xref: /openbmc/linux/arch/s390/kvm/kvm-s390.c (revision 8fdf9062)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * hosting IBM Z kernel virtual machines (s390x)
4  *
5  * Copyright IBM Corp. 2008, 2018
6  *
7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
8  *               Christian Borntraeger <borntraeger@de.ibm.com>
9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
11  *               Jason J. Herne <jjherne@us.ibm.com>
12  */
13 
14 #define KMSG_COMPONENT "kvm-s390"
15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
16 
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <linux/sched/signal.h>
33 #include <linux/string.h>
34 
35 #include <asm/asm-offsets.h>
36 #include <asm/lowcore.h>
37 #include <asm/stp.h>
38 #include <asm/pgtable.h>
39 #include <asm/gmap.h>
40 #include <asm/nmi.h>
41 #include <asm/switch_to.h>
42 #include <asm/isc.h>
43 #include <asm/sclp.h>
44 #include <asm/cpacf.h>
45 #include <asm/timex.h>
46 #include <asm/ap.h>
47 #include "kvm-s390.h"
48 #include "gaccess.h"
49 
50 #define CREATE_TRACE_POINTS
51 #include "trace.h"
52 #include "trace-s390.h"
53 
54 #define MEM_OP_MAX_SIZE 65536	/* Maximum transfer size for KVM_S390_MEM_OP */
55 #define LOCAL_IRQS 32
56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 			   (KVM_MAX_VCPUS + LOCAL_IRQS))
58 
59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
61 
62 struct kvm_stats_debugfs_item debugfs_entries[] = {
63 	{ "userspace_handled", VCPU_STAT(exit_userspace) },
64 	{ "exit_null", VCPU_STAT(exit_null) },
65 	{ "exit_validity", VCPU_STAT(exit_validity) },
66 	{ "exit_stop_request", VCPU_STAT(exit_stop_request) },
67 	{ "exit_external_request", VCPU_STAT(exit_external_request) },
68 	{ "exit_io_request", VCPU_STAT(exit_io_request) },
69 	{ "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
70 	{ "exit_instruction", VCPU_STAT(exit_instruction) },
71 	{ "exit_pei", VCPU_STAT(exit_pei) },
72 	{ "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 	{ "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
74 	{ "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
75 	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
76 	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
77 	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
78 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
79 	{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
80 	{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
81 	{ "instruction_stctl", VCPU_STAT(instruction_stctl) },
82 	{ "instruction_stctg", VCPU_STAT(instruction_stctg) },
83 	{ "deliver_ckc", VCPU_STAT(deliver_ckc) },
84 	{ "deliver_cputm", VCPU_STAT(deliver_cputm) },
85 	{ "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
86 	{ "deliver_external_call", VCPU_STAT(deliver_external_call) },
87 	{ "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
88 	{ "deliver_virtio", VCPU_STAT(deliver_virtio) },
89 	{ "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
90 	{ "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
91 	{ "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
92 	{ "deliver_program", VCPU_STAT(deliver_program) },
93 	{ "deliver_io", VCPU_STAT(deliver_io) },
94 	{ "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
95 	{ "exit_wait_state", VCPU_STAT(exit_wait_state) },
96 	{ "inject_ckc", VCPU_STAT(inject_ckc) },
97 	{ "inject_cputm", VCPU_STAT(inject_cputm) },
98 	{ "inject_external_call", VCPU_STAT(inject_external_call) },
99 	{ "inject_float_mchk", VM_STAT(inject_float_mchk) },
100 	{ "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
101 	{ "inject_io", VM_STAT(inject_io) },
102 	{ "inject_mchk", VCPU_STAT(inject_mchk) },
103 	{ "inject_pfault_done", VM_STAT(inject_pfault_done) },
104 	{ "inject_program", VCPU_STAT(inject_program) },
105 	{ "inject_restart", VCPU_STAT(inject_restart) },
106 	{ "inject_service_signal", VM_STAT(inject_service_signal) },
107 	{ "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
108 	{ "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
109 	{ "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
110 	{ "inject_virtio", VM_STAT(inject_virtio) },
111 	{ "instruction_epsw", VCPU_STAT(instruction_epsw) },
112 	{ "instruction_gs", VCPU_STAT(instruction_gs) },
113 	{ "instruction_io_other", VCPU_STAT(instruction_io_other) },
114 	{ "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
115 	{ "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
116 	{ "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
117 	{ "instruction_ptff", VCPU_STAT(instruction_ptff) },
118 	{ "instruction_stidp", VCPU_STAT(instruction_stidp) },
119 	{ "instruction_sck", VCPU_STAT(instruction_sck) },
120 	{ "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
121 	{ "instruction_spx", VCPU_STAT(instruction_spx) },
122 	{ "instruction_stpx", VCPU_STAT(instruction_stpx) },
123 	{ "instruction_stap", VCPU_STAT(instruction_stap) },
124 	{ "instruction_iske", VCPU_STAT(instruction_iske) },
125 	{ "instruction_ri", VCPU_STAT(instruction_ri) },
126 	{ "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
127 	{ "instruction_sske", VCPU_STAT(instruction_sske) },
128 	{ "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
129 	{ "instruction_essa", VCPU_STAT(instruction_essa) },
130 	{ "instruction_stsi", VCPU_STAT(instruction_stsi) },
131 	{ "instruction_stfl", VCPU_STAT(instruction_stfl) },
132 	{ "instruction_tb", VCPU_STAT(instruction_tb) },
133 	{ "instruction_tpi", VCPU_STAT(instruction_tpi) },
134 	{ "instruction_tprot", VCPU_STAT(instruction_tprot) },
135 	{ "instruction_tsch", VCPU_STAT(instruction_tsch) },
136 	{ "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
137 	{ "instruction_sie", VCPU_STAT(instruction_sie) },
138 	{ "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
139 	{ "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
140 	{ "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
141 	{ "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
142 	{ "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
143 	{ "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
144 	{ "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
145 	{ "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
146 	{ "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
147 	{ "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
148 	{ "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
149 	{ "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
150 	{ "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
151 	{ "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
152 	{ "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
153 	{ "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
154 	{ "instruction_diag_10", VCPU_STAT(diagnose_10) },
155 	{ "instruction_diag_44", VCPU_STAT(diagnose_44) },
156 	{ "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
157 	{ "instruction_diag_258", VCPU_STAT(diagnose_258) },
158 	{ "instruction_diag_308", VCPU_STAT(diagnose_308) },
159 	{ "instruction_diag_500", VCPU_STAT(diagnose_500) },
160 	{ "instruction_diag_other", VCPU_STAT(diagnose_other) },
161 	{ NULL }
162 };
163 
164 struct kvm_s390_tod_clock_ext {
165 	__u8 epoch_idx;
166 	__u64 tod;
167 	__u8 reserved[7];
168 } __packed;
169 
170 /* allow nested virtualization in KVM (if enabled by user space) */
171 static int nested;
172 module_param(nested, int, S_IRUGO);
173 MODULE_PARM_DESC(nested, "Nested virtualization support");
174 
175 /* allow 1m huge page guest backing, if !nested */
176 static int hpage;
177 module_param(hpage, int, 0444);
178 MODULE_PARM_DESC(hpage, "1m huge page backing support");
179 
180 /*
181  * For now we handle at most 16 double words as this is what the s390 base
182  * kernel handles and stores in the prefix page. If we ever need to go beyond
183  * this, this requires changes to code, but the external uapi can stay.
184  */
185 #define SIZE_INTERNAL 16
186 
187 /*
188  * Base feature mask that defines default mask for facilities. Consists of the
189  * defines in FACILITIES_KVM and the non-hypervisor managed bits.
190  */
191 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
192 /*
193  * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
194  * and defines the facilities that can be enabled via a cpu model.
195  */
196 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
197 
198 static unsigned long kvm_s390_fac_size(void)
199 {
200 	BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
201 	BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
202 	BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
203 		sizeof(S390_lowcore.stfle_fac_list));
204 
205 	return SIZE_INTERNAL;
206 }
207 
208 /* available cpu features supported by kvm */
209 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
210 /* available subfunctions indicated via query / "test bit" */
211 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
212 
213 static struct gmap_notifier gmap_notifier;
214 static struct gmap_notifier vsie_gmap_notifier;
215 debug_info_t *kvm_s390_dbf;
216 
217 /* Section: not file related */
218 int kvm_arch_hardware_enable(void)
219 {
220 	/* every s390 is virtualization enabled ;-) */
221 	return 0;
222 }
223 
224 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
225 			      unsigned long end);
226 
227 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
228 {
229 	u8 delta_idx = 0;
230 
231 	/*
232 	 * The TOD jumps by delta, we have to compensate this by adding
233 	 * -delta to the epoch.
234 	 */
235 	delta = -delta;
236 
237 	/* sign-extension - we're adding to signed values below */
238 	if ((s64)delta < 0)
239 		delta_idx = -1;
240 
241 	scb->epoch += delta;
242 	if (scb->ecd & ECD_MEF) {
243 		scb->epdx += delta_idx;
244 		if (scb->epoch < delta)
245 			scb->epdx += 1;
246 	}
247 }
248 
249 /*
250  * This callback is executed during stop_machine(). All CPUs are therefore
251  * temporarily stopped. In order not to change guest behavior, we have to
252  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
253  * so a CPU won't be stopped while calculating with the epoch.
254  */
255 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
256 			  void *v)
257 {
258 	struct kvm *kvm;
259 	struct kvm_vcpu *vcpu;
260 	int i;
261 	unsigned long long *delta = v;
262 
263 	list_for_each_entry(kvm, &vm_list, vm_list) {
264 		kvm_for_each_vcpu(i, vcpu, kvm) {
265 			kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
266 			if (i == 0) {
267 				kvm->arch.epoch = vcpu->arch.sie_block->epoch;
268 				kvm->arch.epdx = vcpu->arch.sie_block->epdx;
269 			}
270 			if (vcpu->arch.cputm_enabled)
271 				vcpu->arch.cputm_start += *delta;
272 			if (vcpu->arch.vsie_block)
273 				kvm_clock_sync_scb(vcpu->arch.vsie_block,
274 						   *delta);
275 		}
276 	}
277 	return NOTIFY_OK;
278 }
279 
280 static struct notifier_block kvm_clock_notifier = {
281 	.notifier_call = kvm_clock_sync,
282 };
283 
284 int kvm_arch_hardware_setup(void)
285 {
286 	gmap_notifier.notifier_call = kvm_gmap_notifier;
287 	gmap_register_pte_notifier(&gmap_notifier);
288 	vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
289 	gmap_register_pte_notifier(&vsie_gmap_notifier);
290 	atomic_notifier_chain_register(&s390_epoch_delta_notifier,
291 				       &kvm_clock_notifier);
292 	return 0;
293 }
294 
295 void kvm_arch_hardware_unsetup(void)
296 {
297 	gmap_unregister_pte_notifier(&gmap_notifier);
298 	gmap_unregister_pte_notifier(&vsie_gmap_notifier);
299 	atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
300 					 &kvm_clock_notifier);
301 }
302 
303 static void allow_cpu_feat(unsigned long nr)
304 {
305 	set_bit_inv(nr, kvm_s390_available_cpu_feat);
306 }
307 
308 static inline int plo_test_bit(unsigned char nr)
309 {
310 	register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
311 	int cc;
312 
313 	asm volatile(
314 		/* Parameter registers are ignored for "test bit" */
315 		"	plo	0,0,0,0(0)\n"
316 		"	ipm	%0\n"
317 		"	srl	%0,28\n"
318 		: "=d" (cc)
319 		: "d" (r0)
320 		: "cc");
321 	return cc == 0;
322 }
323 
324 static void kvm_s390_cpu_feat_init(void)
325 {
326 	int i;
327 
328 	for (i = 0; i < 256; ++i) {
329 		if (plo_test_bit(i))
330 			kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
331 	}
332 
333 	if (test_facility(28)) /* TOD-clock steering */
334 		ptff(kvm_s390_available_subfunc.ptff,
335 		     sizeof(kvm_s390_available_subfunc.ptff),
336 		     PTFF_QAF);
337 
338 	if (test_facility(17)) { /* MSA */
339 		__cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
340 			      kvm_s390_available_subfunc.kmac);
341 		__cpacf_query(CPACF_KMC, (cpacf_mask_t *)
342 			      kvm_s390_available_subfunc.kmc);
343 		__cpacf_query(CPACF_KM, (cpacf_mask_t *)
344 			      kvm_s390_available_subfunc.km);
345 		__cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
346 			      kvm_s390_available_subfunc.kimd);
347 		__cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
348 			      kvm_s390_available_subfunc.klmd);
349 	}
350 	if (test_facility(76)) /* MSA3 */
351 		__cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
352 			      kvm_s390_available_subfunc.pckmo);
353 	if (test_facility(77)) { /* MSA4 */
354 		__cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
355 			      kvm_s390_available_subfunc.kmctr);
356 		__cpacf_query(CPACF_KMF, (cpacf_mask_t *)
357 			      kvm_s390_available_subfunc.kmf);
358 		__cpacf_query(CPACF_KMO, (cpacf_mask_t *)
359 			      kvm_s390_available_subfunc.kmo);
360 		__cpacf_query(CPACF_PCC, (cpacf_mask_t *)
361 			      kvm_s390_available_subfunc.pcc);
362 	}
363 	if (test_facility(57)) /* MSA5 */
364 		__cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
365 			      kvm_s390_available_subfunc.ppno);
366 
367 	if (test_facility(146)) /* MSA8 */
368 		__cpacf_query(CPACF_KMA, (cpacf_mask_t *)
369 			      kvm_s390_available_subfunc.kma);
370 
371 	if (MACHINE_HAS_ESOP)
372 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
373 	/*
374 	 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
375 	 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
376 	 */
377 	if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
378 	    !test_facility(3) || !nested)
379 		return;
380 	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
381 	if (sclp.has_64bscao)
382 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
383 	if (sclp.has_siif)
384 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
385 	if (sclp.has_gpere)
386 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
387 	if (sclp.has_gsls)
388 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
389 	if (sclp.has_ib)
390 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
391 	if (sclp.has_cei)
392 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
393 	if (sclp.has_ibs)
394 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
395 	if (sclp.has_kss)
396 		allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
397 	/*
398 	 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
399 	 * all skey handling functions read/set the skey from the PGSTE
400 	 * instead of the real storage key.
401 	 *
402 	 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
403 	 * pages being detected as preserved although they are resident.
404 	 *
405 	 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
406 	 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
407 	 *
408 	 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
409 	 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
410 	 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
411 	 *
412 	 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
413 	 * cannot easily shadow the SCA because of the ipte lock.
414 	 */
415 }
416 
417 int kvm_arch_init(void *opaque)
418 {
419 	int rc;
420 
421 	kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
422 	if (!kvm_s390_dbf)
423 		return -ENOMEM;
424 
425 	if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
426 		rc = -ENOMEM;
427 		goto out_debug_unreg;
428 	}
429 
430 	kvm_s390_cpu_feat_init();
431 
432 	/* Register floating interrupt controller interface. */
433 	rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
434 	if (rc) {
435 		pr_err("Failed to register FLIC rc=%d\n", rc);
436 		goto out_debug_unreg;
437 	}
438 	return 0;
439 
440 out_debug_unreg:
441 	debug_unregister(kvm_s390_dbf);
442 	return rc;
443 }
444 
445 void kvm_arch_exit(void)
446 {
447 	debug_unregister(kvm_s390_dbf);
448 }
449 
450 /* Section: device related */
451 long kvm_arch_dev_ioctl(struct file *filp,
452 			unsigned int ioctl, unsigned long arg)
453 {
454 	if (ioctl == KVM_S390_ENABLE_SIE)
455 		return s390_enable_sie();
456 	return -EINVAL;
457 }
458 
459 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
460 {
461 	int r;
462 
463 	switch (ext) {
464 	case KVM_CAP_S390_PSW:
465 	case KVM_CAP_S390_GMAP:
466 	case KVM_CAP_SYNC_MMU:
467 #ifdef CONFIG_KVM_S390_UCONTROL
468 	case KVM_CAP_S390_UCONTROL:
469 #endif
470 	case KVM_CAP_ASYNC_PF:
471 	case KVM_CAP_SYNC_REGS:
472 	case KVM_CAP_ONE_REG:
473 	case KVM_CAP_ENABLE_CAP:
474 	case KVM_CAP_S390_CSS_SUPPORT:
475 	case KVM_CAP_IOEVENTFD:
476 	case KVM_CAP_DEVICE_CTRL:
477 	case KVM_CAP_S390_IRQCHIP:
478 	case KVM_CAP_VM_ATTRIBUTES:
479 	case KVM_CAP_MP_STATE:
480 	case KVM_CAP_IMMEDIATE_EXIT:
481 	case KVM_CAP_S390_INJECT_IRQ:
482 	case KVM_CAP_S390_USER_SIGP:
483 	case KVM_CAP_S390_USER_STSI:
484 	case KVM_CAP_S390_SKEYS:
485 	case KVM_CAP_S390_IRQ_STATE:
486 	case KVM_CAP_S390_USER_INSTR0:
487 	case KVM_CAP_S390_CMMA_MIGRATION:
488 	case KVM_CAP_S390_AIS:
489 	case KVM_CAP_S390_AIS_MIGRATION:
490 		r = 1;
491 		break;
492 	case KVM_CAP_S390_HPAGE_1M:
493 		r = 0;
494 		if (hpage && !kvm_is_ucontrol(kvm))
495 			r = 1;
496 		break;
497 	case KVM_CAP_S390_MEM_OP:
498 		r = MEM_OP_MAX_SIZE;
499 		break;
500 	case KVM_CAP_NR_VCPUS:
501 	case KVM_CAP_MAX_VCPUS:
502 		r = KVM_S390_BSCA_CPU_SLOTS;
503 		if (!kvm_s390_use_sca_entries())
504 			r = KVM_MAX_VCPUS;
505 		else if (sclp.has_esca && sclp.has_64bscao)
506 			r = KVM_S390_ESCA_CPU_SLOTS;
507 		break;
508 	case KVM_CAP_NR_MEMSLOTS:
509 		r = KVM_USER_MEM_SLOTS;
510 		break;
511 	case KVM_CAP_S390_COW:
512 		r = MACHINE_HAS_ESOP;
513 		break;
514 	case KVM_CAP_S390_VECTOR_REGISTERS:
515 		r = MACHINE_HAS_VX;
516 		break;
517 	case KVM_CAP_S390_RI:
518 		r = test_facility(64);
519 		break;
520 	case KVM_CAP_S390_GS:
521 		r = test_facility(133);
522 		break;
523 	case KVM_CAP_S390_BPB:
524 		r = test_facility(82);
525 		break;
526 	default:
527 		r = 0;
528 	}
529 	return r;
530 }
531 
532 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
533 				    struct kvm_memory_slot *memslot)
534 {
535 	int i;
536 	gfn_t cur_gfn, last_gfn;
537 	unsigned long gaddr, vmaddr;
538 	struct gmap *gmap = kvm->arch.gmap;
539 	DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
540 
541 	/* Loop over all guest segments */
542 	cur_gfn = memslot->base_gfn;
543 	last_gfn = memslot->base_gfn + memslot->npages;
544 	for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
545 		gaddr = gfn_to_gpa(cur_gfn);
546 		vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
547 		if (kvm_is_error_hva(vmaddr))
548 			continue;
549 
550 		bitmap_zero(bitmap, _PAGE_ENTRIES);
551 		gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
552 		for (i = 0; i < _PAGE_ENTRIES; i++) {
553 			if (test_bit(i, bitmap))
554 				mark_page_dirty(kvm, cur_gfn + i);
555 		}
556 
557 		if (fatal_signal_pending(current))
558 			return;
559 		cond_resched();
560 	}
561 }
562 
563 /* Section: vm related */
564 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
565 
566 /*
567  * Get (and clear) the dirty memory log for a memory slot.
568  */
569 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
570 			       struct kvm_dirty_log *log)
571 {
572 	int r;
573 	unsigned long n;
574 	struct kvm_memslots *slots;
575 	struct kvm_memory_slot *memslot;
576 	int is_dirty = 0;
577 
578 	if (kvm_is_ucontrol(kvm))
579 		return -EINVAL;
580 
581 	mutex_lock(&kvm->slots_lock);
582 
583 	r = -EINVAL;
584 	if (log->slot >= KVM_USER_MEM_SLOTS)
585 		goto out;
586 
587 	slots = kvm_memslots(kvm);
588 	memslot = id_to_memslot(slots, log->slot);
589 	r = -ENOENT;
590 	if (!memslot->dirty_bitmap)
591 		goto out;
592 
593 	kvm_s390_sync_dirty_log(kvm, memslot);
594 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
595 	if (r)
596 		goto out;
597 
598 	/* Clear the dirty log */
599 	if (is_dirty) {
600 		n = kvm_dirty_bitmap_bytes(memslot);
601 		memset(memslot->dirty_bitmap, 0, n);
602 	}
603 	r = 0;
604 out:
605 	mutex_unlock(&kvm->slots_lock);
606 	return r;
607 }
608 
609 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
610 {
611 	unsigned int i;
612 	struct kvm_vcpu *vcpu;
613 
614 	kvm_for_each_vcpu(i, vcpu, kvm) {
615 		kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
616 	}
617 }
618 
619 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
620 {
621 	int r;
622 
623 	if (cap->flags)
624 		return -EINVAL;
625 
626 	switch (cap->cap) {
627 	case KVM_CAP_S390_IRQCHIP:
628 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
629 		kvm->arch.use_irqchip = 1;
630 		r = 0;
631 		break;
632 	case KVM_CAP_S390_USER_SIGP:
633 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
634 		kvm->arch.user_sigp = 1;
635 		r = 0;
636 		break;
637 	case KVM_CAP_S390_VECTOR_REGISTERS:
638 		mutex_lock(&kvm->lock);
639 		if (kvm->created_vcpus) {
640 			r = -EBUSY;
641 		} else if (MACHINE_HAS_VX) {
642 			set_kvm_facility(kvm->arch.model.fac_mask, 129);
643 			set_kvm_facility(kvm->arch.model.fac_list, 129);
644 			if (test_facility(134)) {
645 				set_kvm_facility(kvm->arch.model.fac_mask, 134);
646 				set_kvm_facility(kvm->arch.model.fac_list, 134);
647 			}
648 			if (test_facility(135)) {
649 				set_kvm_facility(kvm->arch.model.fac_mask, 135);
650 				set_kvm_facility(kvm->arch.model.fac_list, 135);
651 			}
652 			r = 0;
653 		} else
654 			r = -EINVAL;
655 		mutex_unlock(&kvm->lock);
656 		VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
657 			 r ? "(not available)" : "(success)");
658 		break;
659 	case KVM_CAP_S390_RI:
660 		r = -EINVAL;
661 		mutex_lock(&kvm->lock);
662 		if (kvm->created_vcpus) {
663 			r = -EBUSY;
664 		} else if (test_facility(64)) {
665 			set_kvm_facility(kvm->arch.model.fac_mask, 64);
666 			set_kvm_facility(kvm->arch.model.fac_list, 64);
667 			r = 0;
668 		}
669 		mutex_unlock(&kvm->lock);
670 		VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
671 			 r ? "(not available)" : "(success)");
672 		break;
673 	case KVM_CAP_S390_AIS:
674 		mutex_lock(&kvm->lock);
675 		if (kvm->created_vcpus) {
676 			r = -EBUSY;
677 		} else {
678 			set_kvm_facility(kvm->arch.model.fac_mask, 72);
679 			set_kvm_facility(kvm->arch.model.fac_list, 72);
680 			r = 0;
681 		}
682 		mutex_unlock(&kvm->lock);
683 		VM_EVENT(kvm, 3, "ENABLE: AIS %s",
684 			 r ? "(not available)" : "(success)");
685 		break;
686 	case KVM_CAP_S390_GS:
687 		r = -EINVAL;
688 		mutex_lock(&kvm->lock);
689 		if (kvm->created_vcpus) {
690 			r = -EBUSY;
691 		} else if (test_facility(133)) {
692 			set_kvm_facility(kvm->arch.model.fac_mask, 133);
693 			set_kvm_facility(kvm->arch.model.fac_list, 133);
694 			r = 0;
695 		}
696 		mutex_unlock(&kvm->lock);
697 		VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
698 			 r ? "(not available)" : "(success)");
699 		break;
700 	case KVM_CAP_S390_HPAGE_1M:
701 		mutex_lock(&kvm->lock);
702 		if (kvm->created_vcpus)
703 			r = -EBUSY;
704 		else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
705 			r = -EINVAL;
706 		else {
707 			r = 0;
708 			down_write(&kvm->mm->mmap_sem);
709 			kvm->mm->context.allow_gmap_hpage_1m = 1;
710 			up_write(&kvm->mm->mmap_sem);
711 			/*
712 			 * We might have to create fake 4k page
713 			 * tables. To avoid that the hardware works on
714 			 * stale PGSTEs, we emulate these instructions.
715 			 */
716 			kvm->arch.use_skf = 0;
717 			kvm->arch.use_pfmfi = 0;
718 		}
719 		mutex_unlock(&kvm->lock);
720 		VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
721 			 r ? "(not available)" : "(success)");
722 		break;
723 	case KVM_CAP_S390_USER_STSI:
724 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
725 		kvm->arch.user_stsi = 1;
726 		r = 0;
727 		break;
728 	case KVM_CAP_S390_USER_INSTR0:
729 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
730 		kvm->arch.user_instr0 = 1;
731 		icpt_operexc_on_all_vcpus(kvm);
732 		r = 0;
733 		break;
734 	default:
735 		r = -EINVAL;
736 		break;
737 	}
738 	return r;
739 }
740 
741 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
742 {
743 	int ret;
744 
745 	switch (attr->attr) {
746 	case KVM_S390_VM_MEM_LIMIT_SIZE:
747 		ret = 0;
748 		VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
749 			 kvm->arch.mem_limit);
750 		if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
751 			ret = -EFAULT;
752 		break;
753 	default:
754 		ret = -ENXIO;
755 		break;
756 	}
757 	return ret;
758 }
759 
760 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
761 {
762 	int ret;
763 	unsigned int idx;
764 	switch (attr->attr) {
765 	case KVM_S390_VM_MEM_ENABLE_CMMA:
766 		ret = -ENXIO;
767 		if (!sclp.has_cmma)
768 			break;
769 
770 		VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
771 		mutex_lock(&kvm->lock);
772 		if (kvm->created_vcpus)
773 			ret = -EBUSY;
774 		else if (kvm->mm->context.allow_gmap_hpage_1m)
775 			ret = -EINVAL;
776 		else {
777 			kvm->arch.use_cmma = 1;
778 			/* Not compatible with cmma. */
779 			kvm->arch.use_pfmfi = 0;
780 			ret = 0;
781 		}
782 		mutex_unlock(&kvm->lock);
783 		break;
784 	case KVM_S390_VM_MEM_CLR_CMMA:
785 		ret = -ENXIO;
786 		if (!sclp.has_cmma)
787 			break;
788 		ret = -EINVAL;
789 		if (!kvm->arch.use_cmma)
790 			break;
791 
792 		VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
793 		mutex_lock(&kvm->lock);
794 		idx = srcu_read_lock(&kvm->srcu);
795 		s390_reset_cmma(kvm->arch.gmap->mm);
796 		srcu_read_unlock(&kvm->srcu, idx);
797 		mutex_unlock(&kvm->lock);
798 		ret = 0;
799 		break;
800 	case KVM_S390_VM_MEM_LIMIT_SIZE: {
801 		unsigned long new_limit;
802 
803 		if (kvm_is_ucontrol(kvm))
804 			return -EINVAL;
805 
806 		if (get_user(new_limit, (u64 __user *)attr->addr))
807 			return -EFAULT;
808 
809 		if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
810 		    new_limit > kvm->arch.mem_limit)
811 			return -E2BIG;
812 
813 		if (!new_limit)
814 			return -EINVAL;
815 
816 		/* gmap_create takes last usable address */
817 		if (new_limit != KVM_S390_NO_MEM_LIMIT)
818 			new_limit -= 1;
819 
820 		ret = -EBUSY;
821 		mutex_lock(&kvm->lock);
822 		if (!kvm->created_vcpus) {
823 			/* gmap_create will round the limit up */
824 			struct gmap *new = gmap_create(current->mm, new_limit);
825 
826 			if (!new) {
827 				ret = -ENOMEM;
828 			} else {
829 				gmap_remove(kvm->arch.gmap);
830 				new->private = kvm;
831 				kvm->arch.gmap = new;
832 				ret = 0;
833 			}
834 		}
835 		mutex_unlock(&kvm->lock);
836 		VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
837 		VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
838 			 (void *) kvm->arch.gmap->asce);
839 		break;
840 	}
841 	default:
842 		ret = -ENXIO;
843 		break;
844 	}
845 	return ret;
846 }
847 
848 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
849 
850 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
851 {
852 	struct kvm_vcpu *vcpu;
853 	int i;
854 
855 	kvm_s390_vcpu_block_all(kvm);
856 
857 	kvm_for_each_vcpu(i, vcpu, kvm) {
858 		kvm_s390_vcpu_crypto_setup(vcpu);
859 		/* recreate the shadow crycb by leaving the VSIE handler */
860 		kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
861 	}
862 
863 	kvm_s390_vcpu_unblock_all(kvm);
864 }
865 
866 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
867 {
868 	mutex_lock(&kvm->lock);
869 	switch (attr->attr) {
870 	case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
871 		if (!test_kvm_facility(kvm, 76)) {
872 			mutex_unlock(&kvm->lock);
873 			return -EINVAL;
874 		}
875 		get_random_bytes(
876 			kvm->arch.crypto.crycb->aes_wrapping_key_mask,
877 			sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
878 		kvm->arch.crypto.aes_kw = 1;
879 		VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
880 		break;
881 	case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
882 		if (!test_kvm_facility(kvm, 76)) {
883 			mutex_unlock(&kvm->lock);
884 			return -EINVAL;
885 		}
886 		get_random_bytes(
887 			kvm->arch.crypto.crycb->dea_wrapping_key_mask,
888 			sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
889 		kvm->arch.crypto.dea_kw = 1;
890 		VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
891 		break;
892 	case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
893 		if (!test_kvm_facility(kvm, 76)) {
894 			mutex_unlock(&kvm->lock);
895 			return -EINVAL;
896 		}
897 		kvm->arch.crypto.aes_kw = 0;
898 		memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
899 			sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
900 		VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
901 		break;
902 	case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
903 		if (!test_kvm_facility(kvm, 76)) {
904 			mutex_unlock(&kvm->lock);
905 			return -EINVAL;
906 		}
907 		kvm->arch.crypto.dea_kw = 0;
908 		memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
909 			sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
910 		VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
911 		break;
912 	case KVM_S390_VM_CRYPTO_ENABLE_APIE:
913 		if (!ap_instructions_available()) {
914 			mutex_unlock(&kvm->lock);
915 			return -EOPNOTSUPP;
916 		}
917 		kvm->arch.crypto.apie = 1;
918 		break;
919 	case KVM_S390_VM_CRYPTO_DISABLE_APIE:
920 		if (!ap_instructions_available()) {
921 			mutex_unlock(&kvm->lock);
922 			return -EOPNOTSUPP;
923 		}
924 		kvm->arch.crypto.apie = 0;
925 		break;
926 	default:
927 		mutex_unlock(&kvm->lock);
928 		return -ENXIO;
929 	}
930 
931 	kvm_s390_vcpu_crypto_reset_all(kvm);
932 	mutex_unlock(&kvm->lock);
933 	return 0;
934 }
935 
936 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
937 {
938 	int cx;
939 	struct kvm_vcpu *vcpu;
940 
941 	kvm_for_each_vcpu(cx, vcpu, kvm)
942 		kvm_s390_sync_request(req, vcpu);
943 }
944 
945 /*
946  * Must be called with kvm->srcu held to avoid races on memslots, and with
947  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
948  */
949 static int kvm_s390_vm_start_migration(struct kvm *kvm)
950 {
951 	struct kvm_memory_slot *ms;
952 	struct kvm_memslots *slots;
953 	unsigned long ram_pages = 0;
954 	int slotnr;
955 
956 	/* migration mode already enabled */
957 	if (kvm->arch.migration_mode)
958 		return 0;
959 	slots = kvm_memslots(kvm);
960 	if (!slots || !slots->used_slots)
961 		return -EINVAL;
962 
963 	if (!kvm->arch.use_cmma) {
964 		kvm->arch.migration_mode = 1;
965 		return 0;
966 	}
967 	/* mark all the pages in active slots as dirty */
968 	for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
969 		ms = slots->memslots + slotnr;
970 		/*
971 		 * The second half of the bitmap is only used on x86,
972 		 * and would be wasted otherwise, so we put it to good
973 		 * use here to keep track of the state of the storage
974 		 * attributes.
975 		 */
976 		memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
977 		ram_pages += ms->npages;
978 	}
979 	atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
980 	kvm->arch.migration_mode = 1;
981 	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
982 	return 0;
983 }
984 
985 /*
986  * Must be called with kvm->slots_lock to avoid races with ourselves and
987  * kvm_s390_vm_start_migration.
988  */
989 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
990 {
991 	/* migration mode already disabled */
992 	if (!kvm->arch.migration_mode)
993 		return 0;
994 	kvm->arch.migration_mode = 0;
995 	if (kvm->arch.use_cmma)
996 		kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
997 	return 0;
998 }
999 
1000 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1001 				     struct kvm_device_attr *attr)
1002 {
1003 	int res = -ENXIO;
1004 
1005 	mutex_lock(&kvm->slots_lock);
1006 	switch (attr->attr) {
1007 	case KVM_S390_VM_MIGRATION_START:
1008 		res = kvm_s390_vm_start_migration(kvm);
1009 		break;
1010 	case KVM_S390_VM_MIGRATION_STOP:
1011 		res = kvm_s390_vm_stop_migration(kvm);
1012 		break;
1013 	default:
1014 		break;
1015 	}
1016 	mutex_unlock(&kvm->slots_lock);
1017 
1018 	return res;
1019 }
1020 
1021 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1022 				     struct kvm_device_attr *attr)
1023 {
1024 	u64 mig = kvm->arch.migration_mode;
1025 
1026 	if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1027 		return -ENXIO;
1028 
1029 	if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1030 		return -EFAULT;
1031 	return 0;
1032 }
1033 
1034 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1035 {
1036 	struct kvm_s390_vm_tod_clock gtod;
1037 
1038 	if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1039 		return -EFAULT;
1040 
1041 	if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1042 		return -EINVAL;
1043 	kvm_s390_set_tod_clock(kvm, &gtod);
1044 
1045 	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1046 		gtod.epoch_idx, gtod.tod);
1047 
1048 	return 0;
1049 }
1050 
1051 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1052 {
1053 	u8 gtod_high;
1054 
1055 	if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1056 					   sizeof(gtod_high)))
1057 		return -EFAULT;
1058 
1059 	if (gtod_high != 0)
1060 		return -EINVAL;
1061 	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1062 
1063 	return 0;
1064 }
1065 
1066 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1067 {
1068 	struct kvm_s390_vm_tod_clock gtod = { 0 };
1069 
1070 	if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1071 			   sizeof(gtod.tod)))
1072 		return -EFAULT;
1073 
1074 	kvm_s390_set_tod_clock(kvm, &gtod);
1075 	VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1076 	return 0;
1077 }
1078 
1079 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1080 {
1081 	int ret;
1082 
1083 	if (attr->flags)
1084 		return -EINVAL;
1085 
1086 	switch (attr->attr) {
1087 	case KVM_S390_VM_TOD_EXT:
1088 		ret = kvm_s390_set_tod_ext(kvm, attr);
1089 		break;
1090 	case KVM_S390_VM_TOD_HIGH:
1091 		ret = kvm_s390_set_tod_high(kvm, attr);
1092 		break;
1093 	case KVM_S390_VM_TOD_LOW:
1094 		ret = kvm_s390_set_tod_low(kvm, attr);
1095 		break;
1096 	default:
1097 		ret = -ENXIO;
1098 		break;
1099 	}
1100 	return ret;
1101 }
1102 
1103 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1104 				   struct kvm_s390_vm_tod_clock *gtod)
1105 {
1106 	struct kvm_s390_tod_clock_ext htod;
1107 
1108 	preempt_disable();
1109 
1110 	get_tod_clock_ext((char *)&htod);
1111 
1112 	gtod->tod = htod.tod + kvm->arch.epoch;
1113 	gtod->epoch_idx = 0;
1114 	if (test_kvm_facility(kvm, 139)) {
1115 		gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1116 		if (gtod->tod < htod.tod)
1117 			gtod->epoch_idx += 1;
1118 	}
1119 
1120 	preempt_enable();
1121 }
1122 
1123 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1124 {
1125 	struct kvm_s390_vm_tod_clock gtod;
1126 
1127 	memset(&gtod, 0, sizeof(gtod));
1128 	kvm_s390_get_tod_clock(kvm, &gtod);
1129 	if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1130 		return -EFAULT;
1131 
1132 	VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1133 		gtod.epoch_idx, gtod.tod);
1134 	return 0;
1135 }
1136 
1137 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1138 {
1139 	u8 gtod_high = 0;
1140 
1141 	if (copy_to_user((void __user *)attr->addr, &gtod_high,
1142 					 sizeof(gtod_high)))
1143 		return -EFAULT;
1144 	VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1145 
1146 	return 0;
1147 }
1148 
1149 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1150 {
1151 	u64 gtod;
1152 
1153 	gtod = kvm_s390_get_tod_clock_fast(kvm);
1154 	if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1155 		return -EFAULT;
1156 	VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1157 
1158 	return 0;
1159 }
1160 
1161 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1162 {
1163 	int ret;
1164 
1165 	if (attr->flags)
1166 		return -EINVAL;
1167 
1168 	switch (attr->attr) {
1169 	case KVM_S390_VM_TOD_EXT:
1170 		ret = kvm_s390_get_tod_ext(kvm, attr);
1171 		break;
1172 	case KVM_S390_VM_TOD_HIGH:
1173 		ret = kvm_s390_get_tod_high(kvm, attr);
1174 		break;
1175 	case KVM_S390_VM_TOD_LOW:
1176 		ret = kvm_s390_get_tod_low(kvm, attr);
1177 		break;
1178 	default:
1179 		ret = -ENXIO;
1180 		break;
1181 	}
1182 	return ret;
1183 }
1184 
1185 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1186 {
1187 	struct kvm_s390_vm_cpu_processor *proc;
1188 	u16 lowest_ibc, unblocked_ibc;
1189 	int ret = 0;
1190 
1191 	mutex_lock(&kvm->lock);
1192 	if (kvm->created_vcpus) {
1193 		ret = -EBUSY;
1194 		goto out;
1195 	}
1196 	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1197 	if (!proc) {
1198 		ret = -ENOMEM;
1199 		goto out;
1200 	}
1201 	if (!copy_from_user(proc, (void __user *)attr->addr,
1202 			    sizeof(*proc))) {
1203 		kvm->arch.model.cpuid = proc->cpuid;
1204 		lowest_ibc = sclp.ibc >> 16 & 0xfff;
1205 		unblocked_ibc = sclp.ibc & 0xfff;
1206 		if (lowest_ibc && proc->ibc) {
1207 			if (proc->ibc > unblocked_ibc)
1208 				kvm->arch.model.ibc = unblocked_ibc;
1209 			else if (proc->ibc < lowest_ibc)
1210 				kvm->arch.model.ibc = lowest_ibc;
1211 			else
1212 				kvm->arch.model.ibc = proc->ibc;
1213 		}
1214 		memcpy(kvm->arch.model.fac_list, proc->fac_list,
1215 		       S390_ARCH_FAC_LIST_SIZE_BYTE);
1216 		VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1217 			 kvm->arch.model.ibc,
1218 			 kvm->arch.model.cpuid);
1219 		VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1220 			 kvm->arch.model.fac_list[0],
1221 			 kvm->arch.model.fac_list[1],
1222 			 kvm->arch.model.fac_list[2]);
1223 	} else
1224 		ret = -EFAULT;
1225 	kfree(proc);
1226 out:
1227 	mutex_unlock(&kvm->lock);
1228 	return ret;
1229 }
1230 
1231 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1232 				       struct kvm_device_attr *attr)
1233 {
1234 	struct kvm_s390_vm_cpu_feat data;
1235 
1236 	if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1237 		return -EFAULT;
1238 	if (!bitmap_subset((unsigned long *) data.feat,
1239 			   kvm_s390_available_cpu_feat,
1240 			   KVM_S390_VM_CPU_FEAT_NR_BITS))
1241 		return -EINVAL;
1242 
1243 	mutex_lock(&kvm->lock);
1244 	if (kvm->created_vcpus) {
1245 		mutex_unlock(&kvm->lock);
1246 		return -EBUSY;
1247 	}
1248 	bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1249 		    KVM_S390_VM_CPU_FEAT_NR_BITS);
1250 	mutex_unlock(&kvm->lock);
1251 	VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1252 			 data.feat[0],
1253 			 data.feat[1],
1254 			 data.feat[2]);
1255 	return 0;
1256 }
1257 
1258 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1259 					  struct kvm_device_attr *attr)
1260 {
1261 	/*
1262 	 * Once supported by kernel + hw, we have to store the subfunctions
1263 	 * in kvm->arch and remember that user space configured them.
1264 	 */
1265 	return -ENXIO;
1266 }
1267 
1268 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1269 {
1270 	int ret = -ENXIO;
1271 
1272 	switch (attr->attr) {
1273 	case KVM_S390_VM_CPU_PROCESSOR:
1274 		ret = kvm_s390_set_processor(kvm, attr);
1275 		break;
1276 	case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1277 		ret = kvm_s390_set_processor_feat(kvm, attr);
1278 		break;
1279 	case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1280 		ret = kvm_s390_set_processor_subfunc(kvm, attr);
1281 		break;
1282 	}
1283 	return ret;
1284 }
1285 
1286 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1287 {
1288 	struct kvm_s390_vm_cpu_processor *proc;
1289 	int ret = 0;
1290 
1291 	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1292 	if (!proc) {
1293 		ret = -ENOMEM;
1294 		goto out;
1295 	}
1296 	proc->cpuid = kvm->arch.model.cpuid;
1297 	proc->ibc = kvm->arch.model.ibc;
1298 	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1299 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
1300 	VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1301 		 kvm->arch.model.ibc,
1302 		 kvm->arch.model.cpuid);
1303 	VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1304 		 kvm->arch.model.fac_list[0],
1305 		 kvm->arch.model.fac_list[1],
1306 		 kvm->arch.model.fac_list[2]);
1307 	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1308 		ret = -EFAULT;
1309 	kfree(proc);
1310 out:
1311 	return ret;
1312 }
1313 
1314 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1315 {
1316 	struct kvm_s390_vm_cpu_machine *mach;
1317 	int ret = 0;
1318 
1319 	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1320 	if (!mach) {
1321 		ret = -ENOMEM;
1322 		goto out;
1323 	}
1324 	get_cpu_id((struct cpuid *) &mach->cpuid);
1325 	mach->ibc = sclp.ibc;
1326 	memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1327 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
1328 	memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1329 	       sizeof(S390_lowcore.stfle_fac_list));
1330 	VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1331 		 kvm->arch.model.ibc,
1332 		 kvm->arch.model.cpuid);
1333 	VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1334 		 mach->fac_mask[0],
1335 		 mach->fac_mask[1],
1336 		 mach->fac_mask[2]);
1337 	VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1338 		 mach->fac_list[0],
1339 		 mach->fac_list[1],
1340 		 mach->fac_list[2]);
1341 	if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1342 		ret = -EFAULT;
1343 	kfree(mach);
1344 out:
1345 	return ret;
1346 }
1347 
1348 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1349 				       struct kvm_device_attr *attr)
1350 {
1351 	struct kvm_s390_vm_cpu_feat data;
1352 
1353 	bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1354 		    KVM_S390_VM_CPU_FEAT_NR_BITS);
1355 	if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1356 		return -EFAULT;
1357 	VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1358 			 data.feat[0],
1359 			 data.feat[1],
1360 			 data.feat[2]);
1361 	return 0;
1362 }
1363 
1364 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1365 				     struct kvm_device_attr *attr)
1366 {
1367 	struct kvm_s390_vm_cpu_feat data;
1368 
1369 	bitmap_copy((unsigned long *) data.feat,
1370 		    kvm_s390_available_cpu_feat,
1371 		    KVM_S390_VM_CPU_FEAT_NR_BITS);
1372 	if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1373 		return -EFAULT;
1374 	VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
1375 			 data.feat[0],
1376 			 data.feat[1],
1377 			 data.feat[2]);
1378 	return 0;
1379 }
1380 
1381 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1382 					  struct kvm_device_attr *attr)
1383 {
1384 	/*
1385 	 * Once we can actually configure subfunctions (kernel + hw support),
1386 	 * we have to check if they were already set by user space, if so copy
1387 	 * them from kvm->arch.
1388 	 */
1389 	return -ENXIO;
1390 }
1391 
1392 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1393 					struct kvm_device_attr *attr)
1394 {
1395 	if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1396 	    sizeof(struct kvm_s390_vm_cpu_subfunc)))
1397 		return -EFAULT;
1398 	return 0;
1399 }
1400 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1401 {
1402 	int ret = -ENXIO;
1403 
1404 	switch (attr->attr) {
1405 	case KVM_S390_VM_CPU_PROCESSOR:
1406 		ret = kvm_s390_get_processor(kvm, attr);
1407 		break;
1408 	case KVM_S390_VM_CPU_MACHINE:
1409 		ret = kvm_s390_get_machine(kvm, attr);
1410 		break;
1411 	case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1412 		ret = kvm_s390_get_processor_feat(kvm, attr);
1413 		break;
1414 	case KVM_S390_VM_CPU_MACHINE_FEAT:
1415 		ret = kvm_s390_get_machine_feat(kvm, attr);
1416 		break;
1417 	case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1418 		ret = kvm_s390_get_processor_subfunc(kvm, attr);
1419 		break;
1420 	case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1421 		ret = kvm_s390_get_machine_subfunc(kvm, attr);
1422 		break;
1423 	}
1424 	return ret;
1425 }
1426 
1427 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1428 {
1429 	int ret;
1430 
1431 	switch (attr->group) {
1432 	case KVM_S390_VM_MEM_CTRL:
1433 		ret = kvm_s390_set_mem_control(kvm, attr);
1434 		break;
1435 	case KVM_S390_VM_TOD:
1436 		ret = kvm_s390_set_tod(kvm, attr);
1437 		break;
1438 	case KVM_S390_VM_CPU_MODEL:
1439 		ret = kvm_s390_set_cpu_model(kvm, attr);
1440 		break;
1441 	case KVM_S390_VM_CRYPTO:
1442 		ret = kvm_s390_vm_set_crypto(kvm, attr);
1443 		break;
1444 	case KVM_S390_VM_MIGRATION:
1445 		ret = kvm_s390_vm_set_migration(kvm, attr);
1446 		break;
1447 	default:
1448 		ret = -ENXIO;
1449 		break;
1450 	}
1451 
1452 	return ret;
1453 }
1454 
1455 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1456 {
1457 	int ret;
1458 
1459 	switch (attr->group) {
1460 	case KVM_S390_VM_MEM_CTRL:
1461 		ret = kvm_s390_get_mem_control(kvm, attr);
1462 		break;
1463 	case KVM_S390_VM_TOD:
1464 		ret = kvm_s390_get_tod(kvm, attr);
1465 		break;
1466 	case KVM_S390_VM_CPU_MODEL:
1467 		ret = kvm_s390_get_cpu_model(kvm, attr);
1468 		break;
1469 	case KVM_S390_VM_MIGRATION:
1470 		ret = kvm_s390_vm_get_migration(kvm, attr);
1471 		break;
1472 	default:
1473 		ret = -ENXIO;
1474 		break;
1475 	}
1476 
1477 	return ret;
1478 }
1479 
1480 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1481 {
1482 	int ret;
1483 
1484 	switch (attr->group) {
1485 	case KVM_S390_VM_MEM_CTRL:
1486 		switch (attr->attr) {
1487 		case KVM_S390_VM_MEM_ENABLE_CMMA:
1488 		case KVM_S390_VM_MEM_CLR_CMMA:
1489 			ret = sclp.has_cmma ? 0 : -ENXIO;
1490 			break;
1491 		case KVM_S390_VM_MEM_LIMIT_SIZE:
1492 			ret = 0;
1493 			break;
1494 		default:
1495 			ret = -ENXIO;
1496 			break;
1497 		}
1498 		break;
1499 	case KVM_S390_VM_TOD:
1500 		switch (attr->attr) {
1501 		case KVM_S390_VM_TOD_LOW:
1502 		case KVM_S390_VM_TOD_HIGH:
1503 			ret = 0;
1504 			break;
1505 		default:
1506 			ret = -ENXIO;
1507 			break;
1508 		}
1509 		break;
1510 	case KVM_S390_VM_CPU_MODEL:
1511 		switch (attr->attr) {
1512 		case KVM_S390_VM_CPU_PROCESSOR:
1513 		case KVM_S390_VM_CPU_MACHINE:
1514 		case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1515 		case KVM_S390_VM_CPU_MACHINE_FEAT:
1516 		case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1517 			ret = 0;
1518 			break;
1519 		/* configuring subfunctions is not supported yet */
1520 		case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1521 		default:
1522 			ret = -ENXIO;
1523 			break;
1524 		}
1525 		break;
1526 	case KVM_S390_VM_CRYPTO:
1527 		switch (attr->attr) {
1528 		case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1529 		case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1530 		case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1531 		case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1532 			ret = 0;
1533 			break;
1534 		case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1535 		case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1536 			ret = ap_instructions_available() ? 0 : -ENXIO;
1537 			break;
1538 		default:
1539 			ret = -ENXIO;
1540 			break;
1541 		}
1542 		break;
1543 	case KVM_S390_VM_MIGRATION:
1544 		ret = 0;
1545 		break;
1546 	default:
1547 		ret = -ENXIO;
1548 		break;
1549 	}
1550 
1551 	return ret;
1552 }
1553 
1554 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1555 {
1556 	uint8_t *keys;
1557 	uint64_t hva;
1558 	int srcu_idx, i, r = 0;
1559 
1560 	if (args->flags != 0)
1561 		return -EINVAL;
1562 
1563 	/* Is this guest using storage keys? */
1564 	if (!mm_uses_skeys(current->mm))
1565 		return KVM_S390_GET_SKEYS_NONE;
1566 
1567 	/* Enforce sane limit on memory allocation */
1568 	if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1569 		return -EINVAL;
1570 
1571 	keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1572 	if (!keys)
1573 		return -ENOMEM;
1574 
1575 	down_read(&current->mm->mmap_sem);
1576 	srcu_idx = srcu_read_lock(&kvm->srcu);
1577 	for (i = 0; i < args->count; i++) {
1578 		hva = gfn_to_hva(kvm, args->start_gfn + i);
1579 		if (kvm_is_error_hva(hva)) {
1580 			r = -EFAULT;
1581 			break;
1582 		}
1583 
1584 		r = get_guest_storage_key(current->mm, hva, &keys[i]);
1585 		if (r)
1586 			break;
1587 	}
1588 	srcu_read_unlock(&kvm->srcu, srcu_idx);
1589 	up_read(&current->mm->mmap_sem);
1590 
1591 	if (!r) {
1592 		r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1593 				 sizeof(uint8_t) * args->count);
1594 		if (r)
1595 			r = -EFAULT;
1596 	}
1597 
1598 	kvfree(keys);
1599 	return r;
1600 }
1601 
1602 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1603 {
1604 	uint8_t *keys;
1605 	uint64_t hva;
1606 	int srcu_idx, i, r = 0;
1607 	bool unlocked;
1608 
1609 	if (args->flags != 0)
1610 		return -EINVAL;
1611 
1612 	/* Enforce sane limit on memory allocation */
1613 	if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1614 		return -EINVAL;
1615 
1616 	keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1617 	if (!keys)
1618 		return -ENOMEM;
1619 
1620 	r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1621 			   sizeof(uint8_t) * args->count);
1622 	if (r) {
1623 		r = -EFAULT;
1624 		goto out;
1625 	}
1626 
1627 	/* Enable storage key handling for the guest */
1628 	r = s390_enable_skey();
1629 	if (r)
1630 		goto out;
1631 
1632 	i = 0;
1633 	down_read(&current->mm->mmap_sem);
1634 	srcu_idx = srcu_read_lock(&kvm->srcu);
1635         while (i < args->count) {
1636 		unlocked = false;
1637 		hva = gfn_to_hva(kvm, args->start_gfn + i);
1638 		if (kvm_is_error_hva(hva)) {
1639 			r = -EFAULT;
1640 			break;
1641 		}
1642 
1643 		/* Lowest order bit is reserved */
1644 		if (keys[i] & 0x01) {
1645 			r = -EINVAL;
1646 			break;
1647 		}
1648 
1649 		r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1650 		if (r) {
1651 			r = fixup_user_fault(current, current->mm, hva,
1652 					     FAULT_FLAG_WRITE, &unlocked);
1653 			if (r)
1654 				break;
1655 		}
1656 		if (!r)
1657 			i++;
1658 	}
1659 	srcu_read_unlock(&kvm->srcu, srcu_idx);
1660 	up_read(&current->mm->mmap_sem);
1661 out:
1662 	kvfree(keys);
1663 	return r;
1664 }
1665 
1666 /*
1667  * Base address and length must be sent at the start of each block, therefore
1668  * it's cheaper to send some clean data, as long as it's less than the size of
1669  * two longs.
1670  */
1671 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1672 /* for consistency */
1673 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1674 
1675 /*
1676  * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1677  * address falls in a hole. In that case the index of one of the memslots
1678  * bordering the hole is returned.
1679  */
1680 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1681 {
1682 	int start = 0, end = slots->used_slots;
1683 	int slot = atomic_read(&slots->lru_slot);
1684 	struct kvm_memory_slot *memslots = slots->memslots;
1685 
1686 	if (gfn >= memslots[slot].base_gfn &&
1687 	    gfn < memslots[slot].base_gfn + memslots[slot].npages)
1688 		return slot;
1689 
1690 	while (start < end) {
1691 		slot = start + (end - start) / 2;
1692 
1693 		if (gfn >= memslots[slot].base_gfn)
1694 			end = slot;
1695 		else
1696 			start = slot + 1;
1697 	}
1698 
1699 	if (gfn >= memslots[start].base_gfn &&
1700 	    gfn < memslots[start].base_gfn + memslots[start].npages) {
1701 		atomic_set(&slots->lru_slot, start);
1702 	}
1703 
1704 	return start;
1705 }
1706 
1707 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1708 			      u8 *res, unsigned long bufsize)
1709 {
1710 	unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1711 
1712 	args->count = 0;
1713 	while (args->count < bufsize) {
1714 		hva = gfn_to_hva(kvm, cur_gfn);
1715 		/*
1716 		 * We return an error if the first value was invalid, but we
1717 		 * return successfully if at least one value was copied.
1718 		 */
1719 		if (kvm_is_error_hva(hva))
1720 			return args->count ? 0 : -EFAULT;
1721 		if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1722 			pgstev = 0;
1723 		res[args->count++] = (pgstev >> 24) & 0x43;
1724 		cur_gfn++;
1725 	}
1726 
1727 	return 0;
1728 }
1729 
1730 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1731 					      unsigned long cur_gfn)
1732 {
1733 	int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1734 	struct kvm_memory_slot *ms = slots->memslots + slotidx;
1735 	unsigned long ofs = cur_gfn - ms->base_gfn;
1736 
1737 	if (ms->base_gfn + ms->npages <= cur_gfn) {
1738 		slotidx--;
1739 		/* If we are above the highest slot, wrap around */
1740 		if (slotidx < 0)
1741 			slotidx = slots->used_slots - 1;
1742 
1743 		ms = slots->memslots + slotidx;
1744 		ofs = 0;
1745 	}
1746 	ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1747 	while ((slotidx > 0) && (ofs >= ms->npages)) {
1748 		slotidx--;
1749 		ms = slots->memslots + slotidx;
1750 		ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1751 	}
1752 	return ms->base_gfn + ofs;
1753 }
1754 
1755 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1756 			     u8 *res, unsigned long bufsize)
1757 {
1758 	unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
1759 	struct kvm_memslots *slots = kvm_memslots(kvm);
1760 	struct kvm_memory_slot *ms;
1761 
1762 	cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1763 	ms = gfn_to_memslot(kvm, cur_gfn);
1764 	args->count = 0;
1765 	args->start_gfn = cur_gfn;
1766 	if (!ms)
1767 		return 0;
1768 	next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
1769 	mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
1770 
1771 	while (args->count < bufsize) {
1772 		hva = gfn_to_hva(kvm, cur_gfn);
1773 		if (kvm_is_error_hva(hva))
1774 			return 0;
1775 		/* Decrement only if we actually flipped the bit to 0 */
1776 		if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
1777 			atomic64_dec(&kvm->arch.cmma_dirty_pages);
1778 		if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1779 			pgstev = 0;
1780 		/* Save the value */
1781 		res[args->count++] = (pgstev >> 24) & 0x43;
1782 		/* If the next bit is too far away, stop. */
1783 		if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
1784 			return 0;
1785 		/* If we reached the previous "next", find the next one */
1786 		if (cur_gfn == next_gfn)
1787 			next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
1788 		/* Reached the end of memory or of the buffer, stop */
1789 		if ((next_gfn >= mem_end) ||
1790 		    (next_gfn - args->start_gfn >= bufsize))
1791 			return 0;
1792 		cur_gfn++;
1793 		/* Reached the end of the current memslot, take the next one. */
1794 		if (cur_gfn - ms->base_gfn >= ms->npages) {
1795 			ms = gfn_to_memslot(kvm, cur_gfn);
1796 			if (!ms)
1797 				return 0;
1798 		}
1799 	}
1800 	return 0;
1801 }
1802 
1803 /*
1804  * This function searches for the next page with dirty CMMA attributes, and
1805  * saves the attributes in the buffer up to either the end of the buffer or
1806  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1807  * no trailing clean bytes are saved.
1808  * In case no dirty bits were found, or if CMMA was not enabled or used, the
1809  * output buffer will indicate 0 as length.
1810  */
1811 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1812 				  struct kvm_s390_cmma_log *args)
1813 {
1814 	unsigned long bufsize;
1815 	int srcu_idx, peek, ret;
1816 	u8 *values;
1817 
1818 	if (!kvm->arch.use_cmma)
1819 		return -ENXIO;
1820 	/* Invalid/unsupported flags were specified */
1821 	if (args->flags & ~KVM_S390_CMMA_PEEK)
1822 		return -EINVAL;
1823 	/* Migration mode query, and we are not doing a migration */
1824 	peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1825 	if (!peek && !kvm->arch.migration_mode)
1826 		return -EINVAL;
1827 	/* CMMA is disabled or was not used, or the buffer has length zero */
1828 	bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1829 	if (!bufsize || !kvm->mm->context.uses_cmm) {
1830 		memset(args, 0, sizeof(*args));
1831 		return 0;
1832 	}
1833 	/* We are not peeking, and there are no dirty pages */
1834 	if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
1835 		memset(args, 0, sizeof(*args));
1836 		return 0;
1837 	}
1838 
1839 	values = vmalloc(bufsize);
1840 	if (!values)
1841 		return -ENOMEM;
1842 
1843 	down_read(&kvm->mm->mmap_sem);
1844 	srcu_idx = srcu_read_lock(&kvm->srcu);
1845 	if (peek)
1846 		ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
1847 	else
1848 		ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
1849 	srcu_read_unlock(&kvm->srcu, srcu_idx);
1850 	up_read(&kvm->mm->mmap_sem);
1851 
1852 	if (kvm->arch.migration_mode)
1853 		args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
1854 	else
1855 		args->remaining = 0;
1856 
1857 	if (copy_to_user((void __user *)args->values, values, args->count))
1858 		ret = -EFAULT;
1859 
1860 	vfree(values);
1861 	return ret;
1862 }
1863 
1864 /*
1865  * This function sets the CMMA attributes for the given pages. If the input
1866  * buffer has zero length, no action is taken, otherwise the attributes are
1867  * set and the mm->context.uses_cmm flag is set.
1868  */
1869 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1870 				  const struct kvm_s390_cmma_log *args)
1871 {
1872 	unsigned long hva, mask, pgstev, i;
1873 	uint8_t *bits;
1874 	int srcu_idx, r = 0;
1875 
1876 	mask = args->mask;
1877 
1878 	if (!kvm->arch.use_cmma)
1879 		return -ENXIO;
1880 	/* invalid/unsupported flags */
1881 	if (args->flags != 0)
1882 		return -EINVAL;
1883 	/* Enforce sane limit on memory allocation */
1884 	if (args->count > KVM_S390_CMMA_SIZE_MAX)
1885 		return -EINVAL;
1886 	/* Nothing to do */
1887 	if (args->count == 0)
1888 		return 0;
1889 
1890 	bits = vmalloc(array_size(sizeof(*bits), args->count));
1891 	if (!bits)
1892 		return -ENOMEM;
1893 
1894 	r = copy_from_user(bits, (void __user *)args->values, args->count);
1895 	if (r) {
1896 		r = -EFAULT;
1897 		goto out;
1898 	}
1899 
1900 	down_read(&kvm->mm->mmap_sem);
1901 	srcu_idx = srcu_read_lock(&kvm->srcu);
1902 	for (i = 0; i < args->count; i++) {
1903 		hva = gfn_to_hva(kvm, args->start_gfn + i);
1904 		if (kvm_is_error_hva(hva)) {
1905 			r = -EFAULT;
1906 			break;
1907 		}
1908 
1909 		pgstev = bits[i];
1910 		pgstev = pgstev << 24;
1911 		mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
1912 		set_pgste_bits(kvm->mm, hva, mask, pgstev);
1913 	}
1914 	srcu_read_unlock(&kvm->srcu, srcu_idx);
1915 	up_read(&kvm->mm->mmap_sem);
1916 
1917 	if (!kvm->mm->context.uses_cmm) {
1918 		down_write(&kvm->mm->mmap_sem);
1919 		kvm->mm->context.uses_cmm = 1;
1920 		up_write(&kvm->mm->mmap_sem);
1921 	}
1922 out:
1923 	vfree(bits);
1924 	return r;
1925 }
1926 
1927 long kvm_arch_vm_ioctl(struct file *filp,
1928 		       unsigned int ioctl, unsigned long arg)
1929 {
1930 	struct kvm *kvm = filp->private_data;
1931 	void __user *argp = (void __user *)arg;
1932 	struct kvm_device_attr attr;
1933 	int r;
1934 
1935 	switch (ioctl) {
1936 	case KVM_S390_INTERRUPT: {
1937 		struct kvm_s390_interrupt s390int;
1938 
1939 		r = -EFAULT;
1940 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
1941 			break;
1942 		r = kvm_s390_inject_vm(kvm, &s390int);
1943 		break;
1944 	}
1945 	case KVM_CREATE_IRQCHIP: {
1946 		struct kvm_irq_routing_entry routing;
1947 
1948 		r = -EINVAL;
1949 		if (kvm->arch.use_irqchip) {
1950 			/* Set up dummy routing. */
1951 			memset(&routing, 0, sizeof(routing));
1952 			r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1953 		}
1954 		break;
1955 	}
1956 	case KVM_SET_DEVICE_ATTR: {
1957 		r = -EFAULT;
1958 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1959 			break;
1960 		r = kvm_s390_vm_set_attr(kvm, &attr);
1961 		break;
1962 	}
1963 	case KVM_GET_DEVICE_ATTR: {
1964 		r = -EFAULT;
1965 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1966 			break;
1967 		r = kvm_s390_vm_get_attr(kvm, &attr);
1968 		break;
1969 	}
1970 	case KVM_HAS_DEVICE_ATTR: {
1971 		r = -EFAULT;
1972 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1973 			break;
1974 		r = kvm_s390_vm_has_attr(kvm, &attr);
1975 		break;
1976 	}
1977 	case KVM_S390_GET_SKEYS: {
1978 		struct kvm_s390_skeys args;
1979 
1980 		r = -EFAULT;
1981 		if (copy_from_user(&args, argp,
1982 				   sizeof(struct kvm_s390_skeys)))
1983 			break;
1984 		r = kvm_s390_get_skeys(kvm, &args);
1985 		break;
1986 	}
1987 	case KVM_S390_SET_SKEYS: {
1988 		struct kvm_s390_skeys args;
1989 
1990 		r = -EFAULT;
1991 		if (copy_from_user(&args, argp,
1992 				   sizeof(struct kvm_s390_skeys)))
1993 			break;
1994 		r = kvm_s390_set_skeys(kvm, &args);
1995 		break;
1996 	}
1997 	case KVM_S390_GET_CMMA_BITS: {
1998 		struct kvm_s390_cmma_log args;
1999 
2000 		r = -EFAULT;
2001 		if (copy_from_user(&args, argp, sizeof(args)))
2002 			break;
2003 		mutex_lock(&kvm->slots_lock);
2004 		r = kvm_s390_get_cmma_bits(kvm, &args);
2005 		mutex_unlock(&kvm->slots_lock);
2006 		if (!r) {
2007 			r = copy_to_user(argp, &args, sizeof(args));
2008 			if (r)
2009 				r = -EFAULT;
2010 		}
2011 		break;
2012 	}
2013 	case KVM_S390_SET_CMMA_BITS: {
2014 		struct kvm_s390_cmma_log args;
2015 
2016 		r = -EFAULT;
2017 		if (copy_from_user(&args, argp, sizeof(args)))
2018 			break;
2019 		mutex_lock(&kvm->slots_lock);
2020 		r = kvm_s390_set_cmma_bits(kvm, &args);
2021 		mutex_unlock(&kvm->slots_lock);
2022 		break;
2023 	}
2024 	default:
2025 		r = -ENOTTY;
2026 	}
2027 
2028 	return r;
2029 }
2030 
2031 static int kvm_s390_apxa_installed(void)
2032 {
2033 	struct ap_config_info info;
2034 
2035 	if (ap_instructions_available()) {
2036 		if (ap_qci(&info) == 0)
2037 			return info.apxa;
2038 	}
2039 
2040 	return 0;
2041 }
2042 
2043 /*
2044  * The format of the crypto control block (CRYCB) is specified in the 3 low
2045  * order bits of the CRYCB designation (CRYCBD) field as follows:
2046  * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2047  *	     AP extended addressing (APXA) facility are installed.
2048  * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2049  * Format 2: Both the APXA and MSAX3 facilities are installed
2050  */
2051 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2052 {
2053 	kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2054 
2055 	/* Clear the CRYCB format bits - i.e., set format 0 by default */
2056 	kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2057 
2058 	/* Check whether MSAX3 is installed */
2059 	if (!test_kvm_facility(kvm, 76))
2060 		return;
2061 
2062 	if (kvm_s390_apxa_installed())
2063 		kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2064 	else
2065 		kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2066 }
2067 
2068 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2069 			       unsigned long *aqm, unsigned long *adm)
2070 {
2071 	struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2072 
2073 	mutex_lock(&kvm->lock);
2074 	kvm_s390_vcpu_block_all(kvm);
2075 
2076 	switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2077 	case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2078 		memcpy(crycb->apcb1.apm, apm, 32);
2079 		VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2080 			 apm[0], apm[1], apm[2], apm[3]);
2081 		memcpy(crycb->apcb1.aqm, aqm, 32);
2082 		VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2083 			 aqm[0], aqm[1], aqm[2], aqm[3]);
2084 		memcpy(crycb->apcb1.adm, adm, 32);
2085 		VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2086 			 adm[0], adm[1], adm[2], adm[3]);
2087 		break;
2088 	case CRYCB_FORMAT1:
2089 	case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2090 		memcpy(crycb->apcb0.apm, apm, 8);
2091 		memcpy(crycb->apcb0.aqm, aqm, 2);
2092 		memcpy(crycb->apcb0.adm, adm, 2);
2093 		VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2094 			 apm[0], *((unsigned short *)aqm),
2095 			 *((unsigned short *)adm));
2096 		break;
2097 	default:	/* Can not happen */
2098 		break;
2099 	}
2100 
2101 	/* recreate the shadow crycb for each vcpu */
2102 	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2103 	kvm_s390_vcpu_unblock_all(kvm);
2104 	mutex_unlock(&kvm->lock);
2105 }
2106 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2107 
2108 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2109 {
2110 	mutex_lock(&kvm->lock);
2111 	kvm_s390_vcpu_block_all(kvm);
2112 
2113 	memset(&kvm->arch.crypto.crycb->apcb0, 0,
2114 	       sizeof(kvm->arch.crypto.crycb->apcb0));
2115 	memset(&kvm->arch.crypto.crycb->apcb1, 0,
2116 	       sizeof(kvm->arch.crypto.crycb->apcb1));
2117 
2118 	VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2119 	/* recreate the shadow crycb for each vcpu */
2120 	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2121 	kvm_s390_vcpu_unblock_all(kvm);
2122 	mutex_unlock(&kvm->lock);
2123 }
2124 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2125 
2126 static u64 kvm_s390_get_initial_cpuid(void)
2127 {
2128 	struct cpuid cpuid;
2129 
2130 	get_cpu_id(&cpuid);
2131 	cpuid.version = 0xff;
2132 	return *((u64 *) &cpuid);
2133 }
2134 
2135 static void kvm_s390_crypto_init(struct kvm *kvm)
2136 {
2137 	kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2138 	kvm_s390_set_crycb_format(kvm);
2139 
2140 	if (!test_kvm_facility(kvm, 76))
2141 		return;
2142 
2143 	/* Enable AES/DEA protected key functions by default */
2144 	kvm->arch.crypto.aes_kw = 1;
2145 	kvm->arch.crypto.dea_kw = 1;
2146 	get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2147 			 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2148 	get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2149 			 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2150 }
2151 
2152 static void sca_dispose(struct kvm *kvm)
2153 {
2154 	if (kvm->arch.use_esca)
2155 		free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2156 	else
2157 		free_page((unsigned long)(kvm->arch.sca));
2158 	kvm->arch.sca = NULL;
2159 }
2160 
2161 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2162 {
2163 	gfp_t alloc_flags = GFP_KERNEL;
2164 	int i, rc;
2165 	char debug_name[16];
2166 	static unsigned long sca_offset;
2167 
2168 	rc = -EINVAL;
2169 #ifdef CONFIG_KVM_S390_UCONTROL
2170 	if (type & ~KVM_VM_S390_UCONTROL)
2171 		goto out_err;
2172 	if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2173 		goto out_err;
2174 #else
2175 	if (type)
2176 		goto out_err;
2177 #endif
2178 
2179 	rc = s390_enable_sie();
2180 	if (rc)
2181 		goto out_err;
2182 
2183 	rc = -ENOMEM;
2184 
2185 	if (!sclp.has_64bscao)
2186 		alloc_flags |= GFP_DMA;
2187 	rwlock_init(&kvm->arch.sca_lock);
2188 	/* start with basic SCA */
2189 	kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2190 	if (!kvm->arch.sca)
2191 		goto out_err;
2192 	spin_lock(&kvm_lock);
2193 	sca_offset += 16;
2194 	if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2195 		sca_offset = 0;
2196 	kvm->arch.sca = (struct bsca_block *)
2197 			((char *) kvm->arch.sca + sca_offset);
2198 	spin_unlock(&kvm_lock);
2199 
2200 	sprintf(debug_name, "kvm-%u", current->pid);
2201 
2202 	kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2203 	if (!kvm->arch.dbf)
2204 		goto out_err;
2205 
2206 	BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2207 	kvm->arch.sie_page2 =
2208 	     (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2209 	if (!kvm->arch.sie_page2)
2210 		goto out_err;
2211 
2212 	kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2213 
2214 	for (i = 0; i < kvm_s390_fac_size(); i++) {
2215 		kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2216 					      (kvm_s390_fac_base[i] |
2217 					       kvm_s390_fac_ext[i]);
2218 		kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2219 					      kvm_s390_fac_base[i];
2220 	}
2221 
2222 	/* we are always in czam mode - even on pre z14 machines */
2223 	set_kvm_facility(kvm->arch.model.fac_mask, 138);
2224 	set_kvm_facility(kvm->arch.model.fac_list, 138);
2225 	/* we emulate STHYI in kvm */
2226 	set_kvm_facility(kvm->arch.model.fac_mask, 74);
2227 	set_kvm_facility(kvm->arch.model.fac_list, 74);
2228 	if (MACHINE_HAS_TLB_GUEST) {
2229 		set_kvm_facility(kvm->arch.model.fac_mask, 147);
2230 		set_kvm_facility(kvm->arch.model.fac_list, 147);
2231 	}
2232 
2233 	kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2234 	kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2235 
2236 	kvm_s390_crypto_init(kvm);
2237 
2238 	mutex_init(&kvm->arch.float_int.ais_lock);
2239 	spin_lock_init(&kvm->arch.float_int.lock);
2240 	for (i = 0; i < FIRQ_LIST_COUNT; i++)
2241 		INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2242 	init_waitqueue_head(&kvm->arch.ipte_wq);
2243 	mutex_init(&kvm->arch.ipte_mutex);
2244 
2245 	debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2246 	VM_EVENT(kvm, 3, "vm created with type %lu", type);
2247 
2248 	if (type & KVM_VM_S390_UCONTROL) {
2249 		kvm->arch.gmap = NULL;
2250 		kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2251 	} else {
2252 		if (sclp.hamax == U64_MAX)
2253 			kvm->arch.mem_limit = TASK_SIZE_MAX;
2254 		else
2255 			kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2256 						    sclp.hamax + 1);
2257 		kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2258 		if (!kvm->arch.gmap)
2259 			goto out_err;
2260 		kvm->arch.gmap->private = kvm;
2261 		kvm->arch.gmap->pfault_enabled = 0;
2262 	}
2263 
2264 	kvm->arch.use_pfmfi = sclp.has_pfmfi;
2265 	kvm->arch.use_skf = sclp.has_skey;
2266 	spin_lock_init(&kvm->arch.start_stop_lock);
2267 	kvm_s390_vsie_init(kvm);
2268 	kvm_s390_gisa_init(kvm);
2269 	KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2270 
2271 	return 0;
2272 out_err:
2273 	free_page((unsigned long)kvm->arch.sie_page2);
2274 	debug_unregister(kvm->arch.dbf);
2275 	sca_dispose(kvm);
2276 	KVM_EVENT(3, "creation of vm failed: %d", rc);
2277 	return rc;
2278 }
2279 
2280 bool kvm_arch_has_vcpu_debugfs(void)
2281 {
2282 	return false;
2283 }
2284 
2285 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2286 {
2287 	return 0;
2288 }
2289 
2290 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2291 {
2292 	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2293 	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2294 	kvm_s390_clear_local_irqs(vcpu);
2295 	kvm_clear_async_pf_completion_queue(vcpu);
2296 	if (!kvm_is_ucontrol(vcpu->kvm))
2297 		sca_del_vcpu(vcpu);
2298 
2299 	if (kvm_is_ucontrol(vcpu->kvm))
2300 		gmap_remove(vcpu->arch.gmap);
2301 
2302 	if (vcpu->kvm->arch.use_cmma)
2303 		kvm_s390_vcpu_unsetup_cmma(vcpu);
2304 	free_page((unsigned long)(vcpu->arch.sie_block));
2305 
2306 	kvm_vcpu_uninit(vcpu);
2307 	kmem_cache_free(kvm_vcpu_cache, vcpu);
2308 }
2309 
2310 static void kvm_free_vcpus(struct kvm *kvm)
2311 {
2312 	unsigned int i;
2313 	struct kvm_vcpu *vcpu;
2314 
2315 	kvm_for_each_vcpu(i, vcpu, kvm)
2316 		kvm_arch_vcpu_destroy(vcpu);
2317 
2318 	mutex_lock(&kvm->lock);
2319 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2320 		kvm->vcpus[i] = NULL;
2321 
2322 	atomic_set(&kvm->online_vcpus, 0);
2323 	mutex_unlock(&kvm->lock);
2324 }
2325 
2326 void kvm_arch_destroy_vm(struct kvm *kvm)
2327 {
2328 	kvm_free_vcpus(kvm);
2329 	sca_dispose(kvm);
2330 	debug_unregister(kvm->arch.dbf);
2331 	kvm_s390_gisa_destroy(kvm);
2332 	free_page((unsigned long)kvm->arch.sie_page2);
2333 	if (!kvm_is_ucontrol(kvm))
2334 		gmap_remove(kvm->arch.gmap);
2335 	kvm_s390_destroy_adapters(kvm);
2336 	kvm_s390_clear_float_irqs(kvm);
2337 	kvm_s390_vsie_destroy(kvm);
2338 	KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2339 }
2340 
2341 /* Section: vcpu related */
2342 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2343 {
2344 	vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2345 	if (!vcpu->arch.gmap)
2346 		return -ENOMEM;
2347 	vcpu->arch.gmap->private = vcpu->kvm;
2348 
2349 	return 0;
2350 }
2351 
2352 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2353 {
2354 	if (!kvm_s390_use_sca_entries())
2355 		return;
2356 	read_lock(&vcpu->kvm->arch.sca_lock);
2357 	if (vcpu->kvm->arch.use_esca) {
2358 		struct esca_block *sca = vcpu->kvm->arch.sca;
2359 
2360 		clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2361 		sca->cpu[vcpu->vcpu_id].sda = 0;
2362 	} else {
2363 		struct bsca_block *sca = vcpu->kvm->arch.sca;
2364 
2365 		clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2366 		sca->cpu[vcpu->vcpu_id].sda = 0;
2367 	}
2368 	read_unlock(&vcpu->kvm->arch.sca_lock);
2369 }
2370 
2371 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2372 {
2373 	if (!kvm_s390_use_sca_entries()) {
2374 		struct bsca_block *sca = vcpu->kvm->arch.sca;
2375 
2376 		/* we still need the basic sca for the ipte control */
2377 		vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2378 		vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2379 		return;
2380 	}
2381 	read_lock(&vcpu->kvm->arch.sca_lock);
2382 	if (vcpu->kvm->arch.use_esca) {
2383 		struct esca_block *sca = vcpu->kvm->arch.sca;
2384 
2385 		sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2386 		vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2387 		vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2388 		vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2389 		set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2390 	} else {
2391 		struct bsca_block *sca = vcpu->kvm->arch.sca;
2392 
2393 		sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2394 		vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2395 		vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2396 		set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2397 	}
2398 	read_unlock(&vcpu->kvm->arch.sca_lock);
2399 }
2400 
2401 /* Basic SCA to Extended SCA data copy routines */
2402 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2403 {
2404 	d->sda = s->sda;
2405 	d->sigp_ctrl.c = s->sigp_ctrl.c;
2406 	d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2407 }
2408 
2409 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2410 {
2411 	int i;
2412 
2413 	d->ipte_control = s->ipte_control;
2414 	d->mcn[0] = s->mcn;
2415 	for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2416 		sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2417 }
2418 
2419 static int sca_switch_to_extended(struct kvm *kvm)
2420 {
2421 	struct bsca_block *old_sca = kvm->arch.sca;
2422 	struct esca_block *new_sca;
2423 	struct kvm_vcpu *vcpu;
2424 	unsigned int vcpu_idx;
2425 	u32 scaol, scaoh;
2426 
2427 	new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2428 	if (!new_sca)
2429 		return -ENOMEM;
2430 
2431 	scaoh = (u32)((u64)(new_sca) >> 32);
2432 	scaol = (u32)(u64)(new_sca) & ~0x3fU;
2433 
2434 	kvm_s390_vcpu_block_all(kvm);
2435 	write_lock(&kvm->arch.sca_lock);
2436 
2437 	sca_copy_b_to_e(new_sca, old_sca);
2438 
2439 	kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2440 		vcpu->arch.sie_block->scaoh = scaoh;
2441 		vcpu->arch.sie_block->scaol = scaol;
2442 		vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2443 	}
2444 	kvm->arch.sca = new_sca;
2445 	kvm->arch.use_esca = 1;
2446 
2447 	write_unlock(&kvm->arch.sca_lock);
2448 	kvm_s390_vcpu_unblock_all(kvm);
2449 
2450 	free_page((unsigned long)old_sca);
2451 
2452 	VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2453 		 old_sca, kvm->arch.sca);
2454 	return 0;
2455 }
2456 
2457 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2458 {
2459 	int rc;
2460 
2461 	if (!kvm_s390_use_sca_entries()) {
2462 		if (id < KVM_MAX_VCPUS)
2463 			return true;
2464 		return false;
2465 	}
2466 	if (id < KVM_S390_BSCA_CPU_SLOTS)
2467 		return true;
2468 	if (!sclp.has_esca || !sclp.has_64bscao)
2469 		return false;
2470 
2471 	mutex_lock(&kvm->lock);
2472 	rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2473 	mutex_unlock(&kvm->lock);
2474 
2475 	return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2476 }
2477 
2478 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2479 {
2480 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2481 	kvm_clear_async_pf_completion_queue(vcpu);
2482 	vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2483 				    KVM_SYNC_GPRS |
2484 				    KVM_SYNC_ACRS |
2485 				    KVM_SYNC_CRS |
2486 				    KVM_SYNC_ARCH0 |
2487 				    KVM_SYNC_PFAULT;
2488 	kvm_s390_set_prefix(vcpu, 0);
2489 	if (test_kvm_facility(vcpu->kvm, 64))
2490 		vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2491 	if (test_kvm_facility(vcpu->kvm, 82))
2492 		vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2493 	if (test_kvm_facility(vcpu->kvm, 133))
2494 		vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2495 	if (test_kvm_facility(vcpu->kvm, 156))
2496 		vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
2497 	/* fprs can be synchronized via vrs, even if the guest has no vx. With
2498 	 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2499 	 */
2500 	if (MACHINE_HAS_VX)
2501 		vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2502 	else
2503 		vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2504 
2505 	if (kvm_is_ucontrol(vcpu->kvm))
2506 		return __kvm_ucontrol_vcpu_init(vcpu);
2507 
2508 	return 0;
2509 }
2510 
2511 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2512 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2513 {
2514 	WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2515 	raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2516 	vcpu->arch.cputm_start = get_tod_clock_fast();
2517 	raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2518 }
2519 
2520 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2521 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2522 {
2523 	WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2524 	raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2525 	vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2526 	vcpu->arch.cputm_start = 0;
2527 	raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2528 }
2529 
2530 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2531 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2532 {
2533 	WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2534 	vcpu->arch.cputm_enabled = true;
2535 	__start_cpu_timer_accounting(vcpu);
2536 }
2537 
2538 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2539 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2540 {
2541 	WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2542 	__stop_cpu_timer_accounting(vcpu);
2543 	vcpu->arch.cputm_enabled = false;
2544 }
2545 
2546 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2547 {
2548 	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2549 	__enable_cpu_timer_accounting(vcpu);
2550 	preempt_enable();
2551 }
2552 
2553 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2554 {
2555 	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2556 	__disable_cpu_timer_accounting(vcpu);
2557 	preempt_enable();
2558 }
2559 
2560 /* set the cpu timer - may only be called from the VCPU thread itself */
2561 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2562 {
2563 	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2564 	raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2565 	if (vcpu->arch.cputm_enabled)
2566 		vcpu->arch.cputm_start = get_tod_clock_fast();
2567 	vcpu->arch.sie_block->cputm = cputm;
2568 	raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2569 	preempt_enable();
2570 }
2571 
2572 /* update and get the cpu timer - can also be called from other VCPU threads */
2573 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2574 {
2575 	unsigned int seq;
2576 	__u64 value;
2577 
2578 	if (unlikely(!vcpu->arch.cputm_enabled))
2579 		return vcpu->arch.sie_block->cputm;
2580 
2581 	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2582 	do {
2583 		seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2584 		/*
2585 		 * If the writer would ever execute a read in the critical
2586 		 * section, e.g. in irq context, we have a deadlock.
2587 		 */
2588 		WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2589 		value = vcpu->arch.sie_block->cputm;
2590 		/* if cputm_start is 0, accounting is being started/stopped */
2591 		if (likely(vcpu->arch.cputm_start))
2592 			value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2593 	} while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2594 	preempt_enable();
2595 	return value;
2596 }
2597 
2598 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2599 {
2600 
2601 	gmap_enable(vcpu->arch.enabled_gmap);
2602 	kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2603 	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2604 		__start_cpu_timer_accounting(vcpu);
2605 	vcpu->cpu = cpu;
2606 }
2607 
2608 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2609 {
2610 	vcpu->cpu = -1;
2611 	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2612 		__stop_cpu_timer_accounting(vcpu);
2613 	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2614 	vcpu->arch.enabled_gmap = gmap_get_enabled();
2615 	gmap_disable(vcpu->arch.enabled_gmap);
2616 
2617 }
2618 
2619 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2620 {
2621 	/* this equals initial cpu reset in pop, but we don't switch to ESA */
2622 	vcpu->arch.sie_block->gpsw.mask = 0UL;
2623 	vcpu->arch.sie_block->gpsw.addr = 0UL;
2624 	kvm_s390_set_prefix(vcpu, 0);
2625 	kvm_s390_set_cpu_timer(vcpu, 0);
2626 	vcpu->arch.sie_block->ckc       = 0UL;
2627 	vcpu->arch.sie_block->todpr     = 0;
2628 	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2629 	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
2630 					CR0_INTERRUPT_KEY_SUBMASK |
2631 					CR0_MEASUREMENT_ALERT_SUBMASK;
2632 	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
2633 					CR14_UNUSED_33 |
2634 					CR14_EXTERNAL_DAMAGE_SUBMASK;
2635 	/* make sure the new fpc will be lazily loaded */
2636 	save_fpu_regs();
2637 	current->thread.fpu.fpc = 0;
2638 	vcpu->arch.sie_block->gbea = 1;
2639 	vcpu->arch.sie_block->pp = 0;
2640 	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2641 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2642 	kvm_clear_async_pf_completion_queue(vcpu);
2643 	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2644 		kvm_s390_vcpu_stop(vcpu);
2645 	kvm_s390_clear_local_irqs(vcpu);
2646 }
2647 
2648 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2649 {
2650 	mutex_lock(&vcpu->kvm->lock);
2651 	preempt_disable();
2652 	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2653 	vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2654 	preempt_enable();
2655 	mutex_unlock(&vcpu->kvm->lock);
2656 	if (!kvm_is_ucontrol(vcpu->kvm)) {
2657 		vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2658 		sca_add_vcpu(vcpu);
2659 	}
2660 	if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2661 		vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2662 	/* make vcpu_load load the right gmap on the first trigger */
2663 	vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2664 }
2665 
2666 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2667 {
2668 	/*
2669 	 * If the AP instructions are not being interpreted and the MSAX3
2670 	 * facility is not configured for the guest, there is nothing to set up.
2671 	 */
2672 	if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2673 		return;
2674 
2675 	vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2676 	vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2677 	vcpu->arch.sie_block->eca &= ~ECA_APIE;
2678 
2679 	if (vcpu->kvm->arch.crypto.apie)
2680 		vcpu->arch.sie_block->eca |= ECA_APIE;
2681 
2682 	/* Set up protected key support */
2683 	if (vcpu->kvm->arch.crypto.aes_kw)
2684 		vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2685 	if (vcpu->kvm->arch.crypto.dea_kw)
2686 		vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2687 }
2688 
2689 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2690 {
2691 	free_page(vcpu->arch.sie_block->cbrlo);
2692 	vcpu->arch.sie_block->cbrlo = 0;
2693 }
2694 
2695 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2696 {
2697 	vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2698 	if (!vcpu->arch.sie_block->cbrlo)
2699 		return -ENOMEM;
2700 	return 0;
2701 }
2702 
2703 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2704 {
2705 	struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2706 
2707 	vcpu->arch.sie_block->ibc = model->ibc;
2708 	if (test_kvm_facility(vcpu->kvm, 7))
2709 		vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2710 }
2711 
2712 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2713 {
2714 	int rc = 0;
2715 
2716 	atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2717 						    CPUSTAT_SM |
2718 						    CPUSTAT_STOPPED);
2719 
2720 	if (test_kvm_facility(vcpu->kvm, 78))
2721 		kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2722 	else if (test_kvm_facility(vcpu->kvm, 8))
2723 		kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2724 
2725 	kvm_s390_vcpu_setup_model(vcpu);
2726 
2727 	/* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2728 	if (MACHINE_HAS_ESOP)
2729 		vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2730 	if (test_kvm_facility(vcpu->kvm, 9))
2731 		vcpu->arch.sie_block->ecb |= ECB_SRSI;
2732 	if (test_kvm_facility(vcpu->kvm, 73))
2733 		vcpu->arch.sie_block->ecb |= ECB_TE;
2734 
2735 	if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2736 		vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2737 	if (test_kvm_facility(vcpu->kvm, 130))
2738 		vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2739 	vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2740 	if (sclp.has_cei)
2741 		vcpu->arch.sie_block->eca |= ECA_CEI;
2742 	if (sclp.has_ib)
2743 		vcpu->arch.sie_block->eca |= ECA_IB;
2744 	if (sclp.has_siif)
2745 		vcpu->arch.sie_block->eca |= ECA_SII;
2746 	if (sclp.has_sigpif)
2747 		vcpu->arch.sie_block->eca |= ECA_SIGPI;
2748 	if (test_kvm_facility(vcpu->kvm, 129)) {
2749 		vcpu->arch.sie_block->eca |= ECA_VX;
2750 		vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2751 	}
2752 	if (test_kvm_facility(vcpu->kvm, 139))
2753 		vcpu->arch.sie_block->ecd |= ECD_MEF;
2754 	if (test_kvm_facility(vcpu->kvm, 156))
2755 		vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
2756 	if (vcpu->arch.sie_block->gd) {
2757 		vcpu->arch.sie_block->eca |= ECA_AIV;
2758 		VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2759 			   vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2760 	}
2761 	vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2762 					| SDNXC;
2763 	vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2764 
2765 	if (sclp.has_kss)
2766 		kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2767 	else
2768 		vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2769 
2770 	if (vcpu->kvm->arch.use_cmma) {
2771 		rc = kvm_s390_vcpu_setup_cmma(vcpu);
2772 		if (rc)
2773 			return rc;
2774 	}
2775 	hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2776 	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2777 
2778 	vcpu->arch.sie_block->hpid = HPID_KVM;
2779 
2780 	kvm_s390_vcpu_crypto_setup(vcpu);
2781 
2782 	return rc;
2783 }
2784 
2785 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2786 				      unsigned int id)
2787 {
2788 	struct kvm_vcpu *vcpu;
2789 	struct sie_page *sie_page;
2790 	int rc = -EINVAL;
2791 
2792 	if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2793 		goto out;
2794 
2795 	rc = -ENOMEM;
2796 
2797 	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2798 	if (!vcpu)
2799 		goto out;
2800 
2801 	BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2802 	sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2803 	if (!sie_page)
2804 		goto out_free_cpu;
2805 
2806 	vcpu->arch.sie_block = &sie_page->sie_block;
2807 	vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2808 
2809 	/* the real guest size will always be smaller than msl */
2810 	vcpu->arch.sie_block->mso = 0;
2811 	vcpu->arch.sie_block->msl = sclp.hamax;
2812 
2813 	vcpu->arch.sie_block->icpua = id;
2814 	spin_lock_init(&vcpu->arch.local_int.lock);
2815 	vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
2816 	if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
2817 		vcpu->arch.sie_block->gd |= GISA_FORMAT1;
2818 	seqcount_init(&vcpu->arch.cputm_seqcount);
2819 
2820 	rc = kvm_vcpu_init(vcpu, kvm, id);
2821 	if (rc)
2822 		goto out_free_sie_block;
2823 	VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2824 		 vcpu->arch.sie_block);
2825 	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2826 
2827 	return vcpu;
2828 out_free_sie_block:
2829 	free_page((unsigned long)(vcpu->arch.sie_block));
2830 out_free_cpu:
2831 	kmem_cache_free(kvm_vcpu_cache, vcpu);
2832 out:
2833 	return ERR_PTR(rc);
2834 }
2835 
2836 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2837 {
2838 	return kvm_s390_vcpu_has_irq(vcpu, 0);
2839 }
2840 
2841 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2842 {
2843 	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
2844 }
2845 
2846 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2847 {
2848 	atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2849 	exit_sie(vcpu);
2850 }
2851 
2852 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2853 {
2854 	atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2855 }
2856 
2857 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2858 {
2859 	atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2860 	exit_sie(vcpu);
2861 }
2862 
2863 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
2864 {
2865 	return atomic_read(&vcpu->arch.sie_block->prog20) &
2866 	       (PROG_BLOCK_SIE | PROG_REQUEST);
2867 }
2868 
2869 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2870 {
2871 	atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2872 }
2873 
2874 /*
2875  * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
2876  * If the CPU is not running (e.g. waiting as idle) the function will
2877  * return immediately. */
2878 void exit_sie(struct kvm_vcpu *vcpu)
2879 {
2880 	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
2881 	kvm_s390_vsie_kick(vcpu);
2882 	while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2883 		cpu_relax();
2884 }
2885 
2886 /* Kick a guest cpu out of SIE to process a request synchronously */
2887 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2888 {
2889 	kvm_make_request(req, vcpu);
2890 	kvm_s390_vcpu_request(vcpu);
2891 }
2892 
2893 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2894 			      unsigned long end)
2895 {
2896 	struct kvm *kvm = gmap->private;
2897 	struct kvm_vcpu *vcpu;
2898 	unsigned long prefix;
2899 	int i;
2900 
2901 	if (gmap_is_shadow(gmap))
2902 		return;
2903 	if (start >= 1UL << 31)
2904 		/* We are only interested in prefix pages */
2905 		return;
2906 	kvm_for_each_vcpu(i, vcpu, kvm) {
2907 		/* match against both prefix pages */
2908 		prefix = kvm_s390_get_prefix(vcpu);
2909 		if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2910 			VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2911 				   start, end);
2912 			kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2913 		}
2914 	}
2915 }
2916 
2917 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2918 {
2919 	/* kvm common code refers to this, but never calls it */
2920 	BUG();
2921 	return 0;
2922 }
2923 
2924 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2925 					   struct kvm_one_reg *reg)
2926 {
2927 	int r = -EINVAL;
2928 
2929 	switch (reg->id) {
2930 	case KVM_REG_S390_TODPR:
2931 		r = put_user(vcpu->arch.sie_block->todpr,
2932 			     (u32 __user *)reg->addr);
2933 		break;
2934 	case KVM_REG_S390_EPOCHDIFF:
2935 		r = put_user(vcpu->arch.sie_block->epoch,
2936 			     (u64 __user *)reg->addr);
2937 		break;
2938 	case KVM_REG_S390_CPU_TIMER:
2939 		r = put_user(kvm_s390_get_cpu_timer(vcpu),
2940 			     (u64 __user *)reg->addr);
2941 		break;
2942 	case KVM_REG_S390_CLOCK_COMP:
2943 		r = put_user(vcpu->arch.sie_block->ckc,
2944 			     (u64 __user *)reg->addr);
2945 		break;
2946 	case KVM_REG_S390_PFTOKEN:
2947 		r = put_user(vcpu->arch.pfault_token,
2948 			     (u64 __user *)reg->addr);
2949 		break;
2950 	case KVM_REG_S390_PFCOMPARE:
2951 		r = put_user(vcpu->arch.pfault_compare,
2952 			     (u64 __user *)reg->addr);
2953 		break;
2954 	case KVM_REG_S390_PFSELECT:
2955 		r = put_user(vcpu->arch.pfault_select,
2956 			     (u64 __user *)reg->addr);
2957 		break;
2958 	case KVM_REG_S390_PP:
2959 		r = put_user(vcpu->arch.sie_block->pp,
2960 			     (u64 __user *)reg->addr);
2961 		break;
2962 	case KVM_REG_S390_GBEA:
2963 		r = put_user(vcpu->arch.sie_block->gbea,
2964 			     (u64 __user *)reg->addr);
2965 		break;
2966 	default:
2967 		break;
2968 	}
2969 
2970 	return r;
2971 }
2972 
2973 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2974 					   struct kvm_one_reg *reg)
2975 {
2976 	int r = -EINVAL;
2977 	__u64 val;
2978 
2979 	switch (reg->id) {
2980 	case KVM_REG_S390_TODPR:
2981 		r = get_user(vcpu->arch.sie_block->todpr,
2982 			     (u32 __user *)reg->addr);
2983 		break;
2984 	case KVM_REG_S390_EPOCHDIFF:
2985 		r = get_user(vcpu->arch.sie_block->epoch,
2986 			     (u64 __user *)reg->addr);
2987 		break;
2988 	case KVM_REG_S390_CPU_TIMER:
2989 		r = get_user(val, (u64 __user *)reg->addr);
2990 		if (!r)
2991 			kvm_s390_set_cpu_timer(vcpu, val);
2992 		break;
2993 	case KVM_REG_S390_CLOCK_COMP:
2994 		r = get_user(vcpu->arch.sie_block->ckc,
2995 			     (u64 __user *)reg->addr);
2996 		break;
2997 	case KVM_REG_S390_PFTOKEN:
2998 		r = get_user(vcpu->arch.pfault_token,
2999 			     (u64 __user *)reg->addr);
3000 		if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3001 			kvm_clear_async_pf_completion_queue(vcpu);
3002 		break;
3003 	case KVM_REG_S390_PFCOMPARE:
3004 		r = get_user(vcpu->arch.pfault_compare,
3005 			     (u64 __user *)reg->addr);
3006 		break;
3007 	case KVM_REG_S390_PFSELECT:
3008 		r = get_user(vcpu->arch.pfault_select,
3009 			     (u64 __user *)reg->addr);
3010 		break;
3011 	case KVM_REG_S390_PP:
3012 		r = get_user(vcpu->arch.sie_block->pp,
3013 			     (u64 __user *)reg->addr);
3014 		break;
3015 	case KVM_REG_S390_GBEA:
3016 		r = get_user(vcpu->arch.sie_block->gbea,
3017 			     (u64 __user *)reg->addr);
3018 		break;
3019 	default:
3020 		break;
3021 	}
3022 
3023 	return r;
3024 }
3025 
3026 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3027 {
3028 	kvm_s390_vcpu_initial_reset(vcpu);
3029 	return 0;
3030 }
3031 
3032 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3033 {
3034 	vcpu_load(vcpu);
3035 	memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
3036 	vcpu_put(vcpu);
3037 	return 0;
3038 }
3039 
3040 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3041 {
3042 	vcpu_load(vcpu);
3043 	memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3044 	vcpu_put(vcpu);
3045 	return 0;
3046 }
3047 
3048 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3049 				  struct kvm_sregs *sregs)
3050 {
3051 	vcpu_load(vcpu);
3052 
3053 	memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3054 	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3055 
3056 	vcpu_put(vcpu);
3057 	return 0;
3058 }
3059 
3060 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3061 				  struct kvm_sregs *sregs)
3062 {
3063 	vcpu_load(vcpu);
3064 
3065 	memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3066 	memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3067 
3068 	vcpu_put(vcpu);
3069 	return 0;
3070 }
3071 
3072 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3073 {
3074 	int ret = 0;
3075 
3076 	vcpu_load(vcpu);
3077 
3078 	if (test_fp_ctl(fpu->fpc)) {
3079 		ret = -EINVAL;
3080 		goto out;
3081 	}
3082 	vcpu->run->s.regs.fpc = fpu->fpc;
3083 	if (MACHINE_HAS_VX)
3084 		convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3085 				 (freg_t *) fpu->fprs);
3086 	else
3087 		memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3088 
3089 out:
3090 	vcpu_put(vcpu);
3091 	return ret;
3092 }
3093 
3094 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3095 {
3096 	vcpu_load(vcpu);
3097 
3098 	/* make sure we have the latest values */
3099 	save_fpu_regs();
3100 	if (MACHINE_HAS_VX)
3101 		convert_vx_to_fp((freg_t *) fpu->fprs,
3102 				 (__vector128 *) vcpu->run->s.regs.vrs);
3103 	else
3104 		memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3105 	fpu->fpc = vcpu->run->s.regs.fpc;
3106 
3107 	vcpu_put(vcpu);
3108 	return 0;
3109 }
3110 
3111 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3112 {
3113 	int rc = 0;
3114 
3115 	if (!is_vcpu_stopped(vcpu))
3116 		rc = -EBUSY;
3117 	else {
3118 		vcpu->run->psw_mask = psw.mask;
3119 		vcpu->run->psw_addr = psw.addr;
3120 	}
3121 	return rc;
3122 }
3123 
3124 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3125 				  struct kvm_translation *tr)
3126 {
3127 	return -EINVAL; /* not implemented yet */
3128 }
3129 
3130 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3131 			      KVM_GUESTDBG_USE_HW_BP | \
3132 			      KVM_GUESTDBG_ENABLE)
3133 
3134 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3135 					struct kvm_guest_debug *dbg)
3136 {
3137 	int rc = 0;
3138 
3139 	vcpu_load(vcpu);
3140 
3141 	vcpu->guest_debug = 0;
3142 	kvm_s390_clear_bp_data(vcpu);
3143 
3144 	if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3145 		rc = -EINVAL;
3146 		goto out;
3147 	}
3148 	if (!sclp.has_gpere) {
3149 		rc = -EINVAL;
3150 		goto out;
3151 	}
3152 
3153 	if (dbg->control & KVM_GUESTDBG_ENABLE) {
3154 		vcpu->guest_debug = dbg->control;
3155 		/* enforce guest PER */
3156 		kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3157 
3158 		if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3159 			rc = kvm_s390_import_bp_data(vcpu, dbg);
3160 	} else {
3161 		kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3162 		vcpu->arch.guestdbg.last_bp = 0;
3163 	}
3164 
3165 	if (rc) {
3166 		vcpu->guest_debug = 0;
3167 		kvm_s390_clear_bp_data(vcpu);
3168 		kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3169 	}
3170 
3171 out:
3172 	vcpu_put(vcpu);
3173 	return rc;
3174 }
3175 
3176 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3177 				    struct kvm_mp_state *mp_state)
3178 {
3179 	int ret;
3180 
3181 	vcpu_load(vcpu);
3182 
3183 	/* CHECK_STOP and LOAD are not supported yet */
3184 	ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3185 				      KVM_MP_STATE_OPERATING;
3186 
3187 	vcpu_put(vcpu);
3188 	return ret;
3189 }
3190 
3191 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3192 				    struct kvm_mp_state *mp_state)
3193 {
3194 	int rc = 0;
3195 
3196 	vcpu_load(vcpu);
3197 
3198 	/* user space knows about this interface - let it control the state */
3199 	vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3200 
3201 	switch (mp_state->mp_state) {
3202 	case KVM_MP_STATE_STOPPED:
3203 		kvm_s390_vcpu_stop(vcpu);
3204 		break;
3205 	case KVM_MP_STATE_OPERATING:
3206 		kvm_s390_vcpu_start(vcpu);
3207 		break;
3208 	case KVM_MP_STATE_LOAD:
3209 	case KVM_MP_STATE_CHECK_STOP:
3210 		/* fall through - CHECK_STOP and LOAD are not supported yet */
3211 	default:
3212 		rc = -ENXIO;
3213 	}
3214 
3215 	vcpu_put(vcpu);
3216 	return rc;
3217 }
3218 
3219 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3220 {
3221 	return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3222 }
3223 
3224 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3225 {
3226 retry:
3227 	kvm_s390_vcpu_request_handled(vcpu);
3228 	if (!kvm_request_pending(vcpu))
3229 		return 0;
3230 	/*
3231 	 * We use MMU_RELOAD just to re-arm the ipte notifier for the
3232 	 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3233 	 * This ensures that the ipte instruction for this request has
3234 	 * already finished. We might race against a second unmapper that
3235 	 * wants to set the blocking bit. Lets just retry the request loop.
3236 	 */
3237 	if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3238 		int rc;
3239 		rc = gmap_mprotect_notify(vcpu->arch.gmap,
3240 					  kvm_s390_get_prefix(vcpu),
3241 					  PAGE_SIZE * 2, PROT_WRITE);
3242 		if (rc) {
3243 			kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3244 			return rc;
3245 		}
3246 		goto retry;
3247 	}
3248 
3249 	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3250 		vcpu->arch.sie_block->ihcpu = 0xffff;
3251 		goto retry;
3252 	}
3253 
3254 	if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3255 		if (!ibs_enabled(vcpu)) {
3256 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3257 			kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3258 		}
3259 		goto retry;
3260 	}
3261 
3262 	if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3263 		if (ibs_enabled(vcpu)) {
3264 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3265 			kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3266 		}
3267 		goto retry;
3268 	}
3269 
3270 	if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3271 		vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3272 		goto retry;
3273 	}
3274 
3275 	if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3276 		/*
3277 		 * Disable CMM virtualization; we will emulate the ESSA
3278 		 * instruction manually, in order to provide additional
3279 		 * functionalities needed for live migration.
3280 		 */
3281 		vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3282 		goto retry;
3283 	}
3284 
3285 	if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3286 		/*
3287 		 * Re-enable CMM virtualization if CMMA is available and
3288 		 * CMM has been used.
3289 		 */
3290 		if ((vcpu->kvm->arch.use_cmma) &&
3291 		    (vcpu->kvm->mm->context.uses_cmm))
3292 			vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3293 		goto retry;
3294 	}
3295 
3296 	/* nothing to do, just clear the request */
3297 	kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3298 	/* we left the vsie handler, nothing to do, just clear the request */
3299 	kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3300 
3301 	return 0;
3302 }
3303 
3304 void kvm_s390_set_tod_clock(struct kvm *kvm,
3305 			    const struct kvm_s390_vm_tod_clock *gtod)
3306 {
3307 	struct kvm_vcpu *vcpu;
3308 	struct kvm_s390_tod_clock_ext htod;
3309 	int i;
3310 
3311 	mutex_lock(&kvm->lock);
3312 	preempt_disable();
3313 
3314 	get_tod_clock_ext((char *)&htod);
3315 
3316 	kvm->arch.epoch = gtod->tod - htod.tod;
3317 	kvm->arch.epdx = 0;
3318 	if (test_kvm_facility(kvm, 139)) {
3319 		kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3320 		if (kvm->arch.epoch > gtod->tod)
3321 			kvm->arch.epdx -= 1;
3322 	}
3323 
3324 	kvm_s390_vcpu_block_all(kvm);
3325 	kvm_for_each_vcpu(i, vcpu, kvm) {
3326 		vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3327 		vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
3328 	}
3329 
3330 	kvm_s390_vcpu_unblock_all(kvm);
3331 	preempt_enable();
3332 	mutex_unlock(&kvm->lock);
3333 }
3334 
3335 /**
3336  * kvm_arch_fault_in_page - fault-in guest page if necessary
3337  * @vcpu: The corresponding virtual cpu
3338  * @gpa: Guest physical address
3339  * @writable: Whether the page should be writable or not
3340  *
3341  * Make sure that a guest page has been faulted-in on the host.
3342  *
3343  * Return: Zero on success, negative error code otherwise.
3344  */
3345 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3346 {
3347 	return gmap_fault(vcpu->arch.gmap, gpa,
3348 			  writable ? FAULT_FLAG_WRITE : 0);
3349 }
3350 
3351 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3352 				      unsigned long token)
3353 {
3354 	struct kvm_s390_interrupt inti;
3355 	struct kvm_s390_irq irq;
3356 
3357 	if (start_token) {
3358 		irq.u.ext.ext_params2 = token;
3359 		irq.type = KVM_S390_INT_PFAULT_INIT;
3360 		WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3361 	} else {
3362 		inti.type = KVM_S390_INT_PFAULT_DONE;
3363 		inti.parm64 = token;
3364 		WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3365 	}
3366 }
3367 
3368 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3369 				     struct kvm_async_pf *work)
3370 {
3371 	trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3372 	__kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3373 }
3374 
3375 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3376 				 struct kvm_async_pf *work)
3377 {
3378 	trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3379 	__kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3380 }
3381 
3382 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3383 			       struct kvm_async_pf *work)
3384 {
3385 	/* s390 will always inject the page directly */
3386 }
3387 
3388 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3389 {
3390 	/*
3391 	 * s390 will always inject the page directly,
3392 	 * but we still want check_async_completion to cleanup
3393 	 */
3394 	return true;
3395 }
3396 
3397 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3398 {
3399 	hva_t hva;
3400 	struct kvm_arch_async_pf arch;
3401 	int rc;
3402 
3403 	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3404 		return 0;
3405 	if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3406 	    vcpu->arch.pfault_compare)
3407 		return 0;
3408 	if (psw_extint_disabled(vcpu))
3409 		return 0;
3410 	if (kvm_s390_vcpu_has_irq(vcpu, 0))
3411 		return 0;
3412 	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3413 		return 0;
3414 	if (!vcpu->arch.gmap->pfault_enabled)
3415 		return 0;
3416 
3417 	hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3418 	hva += current->thread.gmap_addr & ~PAGE_MASK;
3419 	if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3420 		return 0;
3421 
3422 	rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3423 	return rc;
3424 }
3425 
3426 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3427 {
3428 	int rc, cpuflags;
3429 
3430 	/*
3431 	 * On s390 notifications for arriving pages will be delivered directly
3432 	 * to the guest but the house keeping for completed pfaults is
3433 	 * handled outside the worker.
3434 	 */
3435 	kvm_check_async_pf_completion(vcpu);
3436 
3437 	vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3438 	vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3439 
3440 	if (need_resched())
3441 		schedule();
3442 
3443 	if (test_cpu_flag(CIF_MCCK_PENDING))
3444 		s390_handle_mcck();
3445 
3446 	if (!kvm_is_ucontrol(vcpu->kvm)) {
3447 		rc = kvm_s390_deliver_pending_interrupts(vcpu);
3448 		if (rc)
3449 			return rc;
3450 	}
3451 
3452 	rc = kvm_s390_handle_requests(vcpu);
3453 	if (rc)
3454 		return rc;
3455 
3456 	if (guestdbg_enabled(vcpu)) {
3457 		kvm_s390_backup_guest_per_regs(vcpu);
3458 		kvm_s390_patch_guest_per_regs(vcpu);
3459 	}
3460 
3461 	vcpu->arch.sie_block->icptcode = 0;
3462 	cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3463 	VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3464 	trace_kvm_s390_sie_enter(vcpu, cpuflags);
3465 
3466 	return 0;
3467 }
3468 
3469 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3470 {
3471 	struct kvm_s390_pgm_info pgm_info = {
3472 		.code = PGM_ADDRESSING,
3473 	};
3474 	u8 opcode, ilen;
3475 	int rc;
3476 
3477 	VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3478 	trace_kvm_s390_sie_fault(vcpu);
3479 
3480 	/*
3481 	 * We want to inject an addressing exception, which is defined as a
3482 	 * suppressing or terminating exception. However, since we came here
3483 	 * by a DAT access exception, the PSW still points to the faulting
3484 	 * instruction since DAT exceptions are nullifying. So we've got
3485 	 * to look up the current opcode to get the length of the instruction
3486 	 * to be able to forward the PSW.
3487 	 */
3488 	rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3489 	ilen = insn_length(opcode);
3490 	if (rc < 0) {
3491 		return rc;
3492 	} else if (rc) {
3493 		/* Instruction-Fetching Exceptions - we can't detect the ilen.
3494 		 * Forward by arbitrary ilc, injection will take care of
3495 		 * nullification if necessary.
3496 		 */
3497 		pgm_info = vcpu->arch.pgm;
3498 		ilen = 4;
3499 	}
3500 	pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3501 	kvm_s390_forward_psw(vcpu, ilen);
3502 	return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3503 }
3504 
3505 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3506 {
3507 	struct mcck_volatile_info *mcck_info;
3508 	struct sie_page *sie_page;
3509 
3510 	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3511 		   vcpu->arch.sie_block->icptcode);
3512 	trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3513 
3514 	if (guestdbg_enabled(vcpu))
3515 		kvm_s390_restore_guest_per_regs(vcpu);
3516 
3517 	vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3518 	vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3519 
3520 	if (exit_reason == -EINTR) {
3521 		VCPU_EVENT(vcpu, 3, "%s", "machine check");
3522 		sie_page = container_of(vcpu->arch.sie_block,
3523 					struct sie_page, sie_block);
3524 		mcck_info = &sie_page->mcck_info;
3525 		kvm_s390_reinject_machine_check(vcpu, mcck_info);
3526 		return 0;
3527 	}
3528 
3529 	if (vcpu->arch.sie_block->icptcode > 0) {
3530 		int rc = kvm_handle_sie_intercept(vcpu);
3531 
3532 		if (rc != -EOPNOTSUPP)
3533 			return rc;
3534 		vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3535 		vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3536 		vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3537 		vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3538 		return -EREMOTE;
3539 	} else if (exit_reason != -EFAULT) {
3540 		vcpu->stat.exit_null++;
3541 		return 0;
3542 	} else if (kvm_is_ucontrol(vcpu->kvm)) {
3543 		vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3544 		vcpu->run->s390_ucontrol.trans_exc_code =
3545 						current->thread.gmap_addr;
3546 		vcpu->run->s390_ucontrol.pgm_code = 0x10;
3547 		return -EREMOTE;
3548 	} else if (current->thread.gmap_pfault) {
3549 		trace_kvm_s390_major_guest_pfault(vcpu);
3550 		current->thread.gmap_pfault = 0;
3551 		if (kvm_arch_setup_async_pf(vcpu))
3552 			return 0;
3553 		return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3554 	}
3555 	return vcpu_post_run_fault_in_sie(vcpu);
3556 }
3557 
3558 static int __vcpu_run(struct kvm_vcpu *vcpu)
3559 {
3560 	int rc, exit_reason;
3561 
3562 	/*
3563 	 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3564 	 * ning the guest), so that memslots (and other stuff) are protected
3565 	 */
3566 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3567 
3568 	do {
3569 		rc = vcpu_pre_run(vcpu);
3570 		if (rc)
3571 			break;
3572 
3573 		srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3574 		/*
3575 		 * As PF_VCPU will be used in fault handler, between
3576 		 * guest_enter and guest_exit should be no uaccess.
3577 		 */
3578 		local_irq_disable();
3579 		guest_enter_irqoff();
3580 		__disable_cpu_timer_accounting(vcpu);
3581 		local_irq_enable();
3582 		exit_reason = sie64a(vcpu->arch.sie_block,
3583 				     vcpu->run->s.regs.gprs);
3584 		local_irq_disable();
3585 		__enable_cpu_timer_accounting(vcpu);
3586 		guest_exit_irqoff();
3587 		local_irq_enable();
3588 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3589 
3590 		rc = vcpu_post_run(vcpu, exit_reason);
3591 	} while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3592 
3593 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3594 	return rc;
3595 }
3596 
3597 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3598 {
3599 	struct runtime_instr_cb *riccb;
3600 	struct gs_cb *gscb;
3601 
3602 	riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3603 	gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3604 	vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3605 	vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3606 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3607 		kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3608 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3609 		memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3610 		/* some control register changes require a tlb flush */
3611 		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3612 	}
3613 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3614 		kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3615 		vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3616 		vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3617 		vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3618 		vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3619 	}
3620 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3621 		vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3622 		vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3623 		vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3624 		if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3625 			kvm_clear_async_pf_completion_queue(vcpu);
3626 	}
3627 	/*
3628 	 * If userspace sets the riccb (e.g. after migration) to a valid state,
3629 	 * we should enable RI here instead of doing the lazy enablement.
3630 	 */
3631 	if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3632 	    test_kvm_facility(vcpu->kvm, 64) &&
3633 	    riccb->v &&
3634 	    !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3635 		VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3636 		vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3637 	}
3638 	/*
3639 	 * If userspace sets the gscb (e.g. after migration) to non-zero,
3640 	 * we should enable GS here instead of doing the lazy enablement.
3641 	 */
3642 	if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3643 	    test_kvm_facility(vcpu->kvm, 133) &&
3644 	    gscb->gssm &&
3645 	    !vcpu->arch.gs_enabled) {
3646 		VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3647 		vcpu->arch.sie_block->ecb |= ECB_GS;
3648 		vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3649 		vcpu->arch.gs_enabled = 1;
3650 	}
3651 	if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3652 	    test_kvm_facility(vcpu->kvm, 82)) {
3653 		vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3654 		vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3655 	}
3656 	save_access_regs(vcpu->arch.host_acrs);
3657 	restore_access_regs(vcpu->run->s.regs.acrs);
3658 	/* save host (userspace) fprs/vrs */
3659 	save_fpu_regs();
3660 	vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3661 	vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3662 	if (MACHINE_HAS_VX)
3663 		current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3664 	else
3665 		current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3666 	current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3667 	if (test_fp_ctl(current->thread.fpu.fpc))
3668 		/* User space provided an invalid FPC, let's clear it */
3669 		current->thread.fpu.fpc = 0;
3670 	if (MACHINE_HAS_GS) {
3671 		preempt_disable();
3672 		__ctl_set_bit(2, 4);
3673 		if (current->thread.gs_cb) {
3674 			vcpu->arch.host_gscb = current->thread.gs_cb;
3675 			save_gs_cb(vcpu->arch.host_gscb);
3676 		}
3677 		if (vcpu->arch.gs_enabled) {
3678 			current->thread.gs_cb = (struct gs_cb *)
3679 						&vcpu->run->s.regs.gscb;
3680 			restore_gs_cb(current->thread.gs_cb);
3681 		}
3682 		preempt_enable();
3683 	}
3684 	/* SIE will load etoken directly from SDNX and therefore kvm_run */
3685 
3686 	kvm_run->kvm_dirty_regs = 0;
3687 }
3688 
3689 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3690 {
3691 	kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3692 	kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3693 	kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3694 	memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3695 	kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3696 	kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3697 	kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3698 	kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3699 	kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3700 	kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3701 	kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3702 	kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3703 	kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3704 	save_access_regs(vcpu->run->s.regs.acrs);
3705 	restore_access_regs(vcpu->arch.host_acrs);
3706 	/* Save guest register state */
3707 	save_fpu_regs();
3708 	vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3709 	/* Restore will be done lazily at return */
3710 	current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3711 	current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3712 	if (MACHINE_HAS_GS) {
3713 		__ctl_set_bit(2, 4);
3714 		if (vcpu->arch.gs_enabled)
3715 			save_gs_cb(current->thread.gs_cb);
3716 		preempt_disable();
3717 		current->thread.gs_cb = vcpu->arch.host_gscb;
3718 		restore_gs_cb(vcpu->arch.host_gscb);
3719 		preempt_enable();
3720 		if (!vcpu->arch.host_gscb)
3721 			__ctl_clear_bit(2, 4);
3722 		vcpu->arch.host_gscb = NULL;
3723 	}
3724 	/* SIE will save etoken directly into SDNX and therefore kvm_run */
3725 }
3726 
3727 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3728 {
3729 	int rc;
3730 
3731 	if (kvm_run->immediate_exit)
3732 		return -EINTR;
3733 
3734 	vcpu_load(vcpu);
3735 
3736 	if (guestdbg_exit_pending(vcpu)) {
3737 		kvm_s390_prepare_debug_exit(vcpu);
3738 		rc = 0;
3739 		goto out;
3740 	}
3741 
3742 	kvm_sigset_activate(vcpu);
3743 
3744 	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3745 		kvm_s390_vcpu_start(vcpu);
3746 	} else if (is_vcpu_stopped(vcpu)) {
3747 		pr_err_ratelimited("can't run stopped vcpu %d\n",
3748 				   vcpu->vcpu_id);
3749 		rc = -EINVAL;
3750 		goto out;
3751 	}
3752 
3753 	sync_regs(vcpu, kvm_run);
3754 	enable_cpu_timer_accounting(vcpu);
3755 
3756 	might_fault();
3757 	rc = __vcpu_run(vcpu);
3758 
3759 	if (signal_pending(current) && !rc) {
3760 		kvm_run->exit_reason = KVM_EXIT_INTR;
3761 		rc = -EINTR;
3762 	}
3763 
3764 	if (guestdbg_exit_pending(vcpu) && !rc)  {
3765 		kvm_s390_prepare_debug_exit(vcpu);
3766 		rc = 0;
3767 	}
3768 
3769 	if (rc == -EREMOTE) {
3770 		/* userspace support is needed, kvm_run has been prepared */
3771 		rc = 0;
3772 	}
3773 
3774 	disable_cpu_timer_accounting(vcpu);
3775 	store_regs(vcpu, kvm_run);
3776 
3777 	kvm_sigset_deactivate(vcpu);
3778 
3779 	vcpu->stat.exit_userspace++;
3780 out:
3781 	vcpu_put(vcpu);
3782 	return rc;
3783 }
3784 
3785 /*
3786  * store status at address
3787  * we use have two special cases:
3788  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3789  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3790  */
3791 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3792 {
3793 	unsigned char archmode = 1;
3794 	freg_t fprs[NUM_FPRS];
3795 	unsigned int px;
3796 	u64 clkcomp, cputm;
3797 	int rc;
3798 
3799 	px = kvm_s390_get_prefix(vcpu);
3800 	if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3801 		if (write_guest_abs(vcpu, 163, &archmode, 1))
3802 			return -EFAULT;
3803 		gpa = 0;
3804 	} else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3805 		if (write_guest_real(vcpu, 163, &archmode, 1))
3806 			return -EFAULT;
3807 		gpa = px;
3808 	} else
3809 		gpa -= __LC_FPREGS_SAVE_AREA;
3810 
3811 	/* manually convert vector registers if necessary */
3812 	if (MACHINE_HAS_VX) {
3813 		convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3814 		rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3815 				     fprs, 128);
3816 	} else {
3817 		rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3818 				     vcpu->run->s.regs.fprs, 128);
3819 	}
3820 	rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3821 			      vcpu->run->s.regs.gprs, 128);
3822 	rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3823 			      &vcpu->arch.sie_block->gpsw, 16);
3824 	rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3825 			      &px, 4);
3826 	rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3827 			      &vcpu->run->s.regs.fpc, 4);
3828 	rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3829 			      &vcpu->arch.sie_block->todpr, 4);
3830 	cputm = kvm_s390_get_cpu_timer(vcpu);
3831 	rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3832 			      &cputm, 8);
3833 	clkcomp = vcpu->arch.sie_block->ckc >> 8;
3834 	rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3835 			      &clkcomp, 8);
3836 	rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3837 			      &vcpu->run->s.regs.acrs, 64);
3838 	rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3839 			      &vcpu->arch.sie_block->gcr, 128);
3840 	return rc ? -EFAULT : 0;
3841 }
3842 
3843 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3844 {
3845 	/*
3846 	 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3847 	 * switch in the run ioctl. Let's update our copies before we save
3848 	 * it into the save area
3849 	 */
3850 	save_fpu_regs();
3851 	vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3852 	save_access_regs(vcpu->run->s.regs.acrs);
3853 
3854 	return kvm_s390_store_status_unloaded(vcpu, addr);
3855 }
3856 
3857 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3858 {
3859 	kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3860 	kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3861 }
3862 
3863 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3864 {
3865 	unsigned int i;
3866 	struct kvm_vcpu *vcpu;
3867 
3868 	kvm_for_each_vcpu(i, vcpu, kvm) {
3869 		__disable_ibs_on_vcpu(vcpu);
3870 	}
3871 }
3872 
3873 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3874 {
3875 	if (!sclp.has_ibs)
3876 		return;
3877 	kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3878 	kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3879 }
3880 
3881 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3882 {
3883 	int i, online_vcpus, started_vcpus = 0;
3884 
3885 	if (!is_vcpu_stopped(vcpu))
3886 		return;
3887 
3888 	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3889 	/* Only one cpu at a time may enter/leave the STOPPED state. */
3890 	spin_lock(&vcpu->kvm->arch.start_stop_lock);
3891 	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3892 
3893 	for (i = 0; i < online_vcpus; i++) {
3894 		if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3895 			started_vcpus++;
3896 	}
3897 
3898 	if (started_vcpus == 0) {
3899 		/* we're the only active VCPU -> speed it up */
3900 		__enable_ibs_on_vcpu(vcpu);
3901 	} else if (started_vcpus == 1) {
3902 		/*
3903 		 * As we are starting a second VCPU, we have to disable
3904 		 * the IBS facility on all VCPUs to remove potentially
3905 		 * oustanding ENABLE requests.
3906 		 */
3907 		__disable_ibs_on_all_vcpus(vcpu->kvm);
3908 	}
3909 
3910 	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
3911 	/*
3912 	 * Another VCPU might have used IBS while we were offline.
3913 	 * Let's play safe and flush the VCPU at startup.
3914 	 */
3915 	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3916 	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3917 	return;
3918 }
3919 
3920 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3921 {
3922 	int i, online_vcpus, started_vcpus = 0;
3923 	struct kvm_vcpu *started_vcpu = NULL;
3924 
3925 	if (is_vcpu_stopped(vcpu))
3926 		return;
3927 
3928 	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3929 	/* Only one cpu at a time may enter/leave the STOPPED state. */
3930 	spin_lock(&vcpu->kvm->arch.start_stop_lock);
3931 	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3932 
3933 	/* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3934 	kvm_s390_clear_stop_irq(vcpu);
3935 
3936 	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
3937 	__disable_ibs_on_vcpu(vcpu);
3938 
3939 	for (i = 0; i < online_vcpus; i++) {
3940 		if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3941 			started_vcpus++;
3942 			started_vcpu = vcpu->kvm->vcpus[i];
3943 		}
3944 	}
3945 
3946 	if (started_vcpus == 1) {
3947 		/*
3948 		 * As we only have one VCPU left, we want to enable the
3949 		 * IBS facility for that VCPU to speed it up.
3950 		 */
3951 		__enable_ibs_on_vcpu(started_vcpu);
3952 	}
3953 
3954 	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3955 	return;
3956 }
3957 
3958 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3959 				     struct kvm_enable_cap *cap)
3960 {
3961 	int r;
3962 
3963 	if (cap->flags)
3964 		return -EINVAL;
3965 
3966 	switch (cap->cap) {
3967 	case KVM_CAP_S390_CSS_SUPPORT:
3968 		if (!vcpu->kvm->arch.css_support) {
3969 			vcpu->kvm->arch.css_support = 1;
3970 			VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3971 			trace_kvm_s390_enable_css(vcpu->kvm);
3972 		}
3973 		r = 0;
3974 		break;
3975 	default:
3976 		r = -EINVAL;
3977 		break;
3978 	}
3979 	return r;
3980 }
3981 
3982 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3983 				  struct kvm_s390_mem_op *mop)
3984 {
3985 	void __user *uaddr = (void __user *)mop->buf;
3986 	void *tmpbuf = NULL;
3987 	int r, srcu_idx;
3988 	const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3989 				    | KVM_S390_MEMOP_F_CHECK_ONLY;
3990 
3991 	if (mop->flags & ~supported_flags)
3992 		return -EINVAL;
3993 
3994 	if (mop->size > MEM_OP_MAX_SIZE)
3995 		return -E2BIG;
3996 
3997 	if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3998 		tmpbuf = vmalloc(mop->size);
3999 		if (!tmpbuf)
4000 			return -ENOMEM;
4001 	}
4002 
4003 	srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4004 
4005 	switch (mop->op) {
4006 	case KVM_S390_MEMOP_LOGICAL_READ:
4007 		if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4008 			r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4009 					    mop->size, GACC_FETCH);
4010 			break;
4011 		}
4012 		r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4013 		if (r == 0) {
4014 			if (copy_to_user(uaddr, tmpbuf, mop->size))
4015 				r = -EFAULT;
4016 		}
4017 		break;
4018 	case KVM_S390_MEMOP_LOGICAL_WRITE:
4019 		if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4020 			r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4021 					    mop->size, GACC_STORE);
4022 			break;
4023 		}
4024 		if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4025 			r = -EFAULT;
4026 			break;
4027 		}
4028 		r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4029 		break;
4030 	default:
4031 		r = -EINVAL;
4032 	}
4033 
4034 	srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4035 
4036 	if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4037 		kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4038 
4039 	vfree(tmpbuf);
4040 	return r;
4041 }
4042 
4043 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4044 			       unsigned int ioctl, unsigned long arg)
4045 {
4046 	struct kvm_vcpu *vcpu = filp->private_data;
4047 	void __user *argp = (void __user *)arg;
4048 
4049 	switch (ioctl) {
4050 	case KVM_S390_IRQ: {
4051 		struct kvm_s390_irq s390irq;
4052 
4053 		if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4054 			return -EFAULT;
4055 		return kvm_s390_inject_vcpu(vcpu, &s390irq);
4056 	}
4057 	case KVM_S390_INTERRUPT: {
4058 		struct kvm_s390_interrupt s390int;
4059 		struct kvm_s390_irq s390irq;
4060 
4061 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
4062 			return -EFAULT;
4063 		if (s390int_to_s390irq(&s390int, &s390irq))
4064 			return -EINVAL;
4065 		return kvm_s390_inject_vcpu(vcpu, &s390irq);
4066 	}
4067 	}
4068 	return -ENOIOCTLCMD;
4069 }
4070 
4071 long kvm_arch_vcpu_ioctl(struct file *filp,
4072 			 unsigned int ioctl, unsigned long arg)
4073 {
4074 	struct kvm_vcpu *vcpu = filp->private_data;
4075 	void __user *argp = (void __user *)arg;
4076 	int idx;
4077 	long r;
4078 
4079 	vcpu_load(vcpu);
4080 
4081 	switch (ioctl) {
4082 	case KVM_S390_STORE_STATUS:
4083 		idx = srcu_read_lock(&vcpu->kvm->srcu);
4084 		r = kvm_s390_vcpu_store_status(vcpu, arg);
4085 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4086 		break;
4087 	case KVM_S390_SET_INITIAL_PSW: {
4088 		psw_t psw;
4089 
4090 		r = -EFAULT;
4091 		if (copy_from_user(&psw, argp, sizeof(psw)))
4092 			break;
4093 		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4094 		break;
4095 	}
4096 	case KVM_S390_INITIAL_RESET:
4097 		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4098 		break;
4099 	case KVM_SET_ONE_REG:
4100 	case KVM_GET_ONE_REG: {
4101 		struct kvm_one_reg reg;
4102 		r = -EFAULT;
4103 		if (copy_from_user(&reg, argp, sizeof(reg)))
4104 			break;
4105 		if (ioctl == KVM_SET_ONE_REG)
4106 			r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4107 		else
4108 			r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4109 		break;
4110 	}
4111 #ifdef CONFIG_KVM_S390_UCONTROL
4112 	case KVM_S390_UCAS_MAP: {
4113 		struct kvm_s390_ucas_mapping ucasmap;
4114 
4115 		if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4116 			r = -EFAULT;
4117 			break;
4118 		}
4119 
4120 		if (!kvm_is_ucontrol(vcpu->kvm)) {
4121 			r = -EINVAL;
4122 			break;
4123 		}
4124 
4125 		r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4126 				     ucasmap.vcpu_addr, ucasmap.length);
4127 		break;
4128 	}
4129 	case KVM_S390_UCAS_UNMAP: {
4130 		struct kvm_s390_ucas_mapping ucasmap;
4131 
4132 		if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4133 			r = -EFAULT;
4134 			break;
4135 		}
4136 
4137 		if (!kvm_is_ucontrol(vcpu->kvm)) {
4138 			r = -EINVAL;
4139 			break;
4140 		}
4141 
4142 		r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4143 			ucasmap.length);
4144 		break;
4145 	}
4146 #endif
4147 	case KVM_S390_VCPU_FAULT: {
4148 		r = gmap_fault(vcpu->arch.gmap, arg, 0);
4149 		break;
4150 	}
4151 	case KVM_ENABLE_CAP:
4152 	{
4153 		struct kvm_enable_cap cap;
4154 		r = -EFAULT;
4155 		if (copy_from_user(&cap, argp, sizeof(cap)))
4156 			break;
4157 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4158 		break;
4159 	}
4160 	case KVM_S390_MEM_OP: {
4161 		struct kvm_s390_mem_op mem_op;
4162 
4163 		if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4164 			r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4165 		else
4166 			r = -EFAULT;
4167 		break;
4168 	}
4169 	case KVM_S390_SET_IRQ_STATE: {
4170 		struct kvm_s390_irq_state irq_state;
4171 
4172 		r = -EFAULT;
4173 		if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4174 			break;
4175 		if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4176 		    irq_state.len == 0 ||
4177 		    irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4178 			r = -EINVAL;
4179 			break;
4180 		}
4181 		/* do not use irq_state.flags, it will break old QEMUs */
4182 		r = kvm_s390_set_irq_state(vcpu,
4183 					   (void __user *) irq_state.buf,
4184 					   irq_state.len);
4185 		break;
4186 	}
4187 	case KVM_S390_GET_IRQ_STATE: {
4188 		struct kvm_s390_irq_state irq_state;
4189 
4190 		r = -EFAULT;
4191 		if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4192 			break;
4193 		if (irq_state.len == 0) {
4194 			r = -EINVAL;
4195 			break;
4196 		}
4197 		/* do not use irq_state.flags, it will break old QEMUs */
4198 		r = kvm_s390_get_irq_state(vcpu,
4199 					   (__u8 __user *)  irq_state.buf,
4200 					   irq_state.len);
4201 		break;
4202 	}
4203 	default:
4204 		r = -ENOTTY;
4205 	}
4206 
4207 	vcpu_put(vcpu);
4208 	return r;
4209 }
4210 
4211 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4212 {
4213 #ifdef CONFIG_KVM_S390_UCONTROL
4214 	if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4215 		 && (kvm_is_ucontrol(vcpu->kvm))) {
4216 		vmf->page = virt_to_page(vcpu->arch.sie_block);
4217 		get_page(vmf->page);
4218 		return 0;
4219 	}
4220 #endif
4221 	return VM_FAULT_SIGBUS;
4222 }
4223 
4224 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4225 			    unsigned long npages)
4226 {
4227 	return 0;
4228 }
4229 
4230 /* Section: memory related */
4231 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4232 				   struct kvm_memory_slot *memslot,
4233 				   const struct kvm_userspace_memory_region *mem,
4234 				   enum kvm_mr_change change)
4235 {
4236 	/* A few sanity checks. We can have memory slots which have to be
4237 	   located/ended at a segment boundary (1MB). The memory in userland is
4238 	   ok to be fragmented into various different vmas. It is okay to mmap()
4239 	   and munmap() stuff in this slot after doing this call at any time */
4240 
4241 	if (mem->userspace_addr & 0xffffful)
4242 		return -EINVAL;
4243 
4244 	if (mem->memory_size & 0xffffful)
4245 		return -EINVAL;
4246 
4247 	if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4248 		return -EINVAL;
4249 
4250 	return 0;
4251 }
4252 
4253 void kvm_arch_commit_memory_region(struct kvm *kvm,
4254 				const struct kvm_userspace_memory_region *mem,
4255 				const struct kvm_memory_slot *old,
4256 				const struct kvm_memory_slot *new,
4257 				enum kvm_mr_change change)
4258 {
4259 	int rc;
4260 
4261 	/* If the basics of the memslot do not change, we do not want
4262 	 * to update the gmap. Every update causes several unnecessary
4263 	 * segment translation exceptions. This is usually handled just
4264 	 * fine by the normal fault handler + gmap, but it will also
4265 	 * cause faults on the prefix page of running guest CPUs.
4266 	 */
4267 	if (old->userspace_addr == mem->userspace_addr &&
4268 	    old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
4269 	    old->npages * PAGE_SIZE == mem->memory_size)
4270 		return;
4271 
4272 	rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4273 		mem->guest_phys_addr, mem->memory_size);
4274 	if (rc)
4275 		pr_warn("failed to commit memory region\n");
4276 	return;
4277 }
4278 
4279 static inline unsigned long nonhyp_mask(int i)
4280 {
4281 	unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4282 
4283 	return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4284 }
4285 
4286 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4287 {
4288 	vcpu->valid_wakeup = false;
4289 }
4290 
4291 static int __init kvm_s390_init(void)
4292 {
4293 	int i;
4294 
4295 	if (!sclp.has_sief2) {
4296 		pr_info("SIE not available\n");
4297 		return -ENODEV;
4298 	}
4299 
4300 	if (nested && hpage) {
4301 		pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently");
4302 		return -EINVAL;
4303 	}
4304 
4305 	for (i = 0; i < 16; i++)
4306 		kvm_s390_fac_base[i] |=
4307 			S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4308 
4309 	return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4310 }
4311 
4312 static void __exit kvm_s390_exit(void)
4313 {
4314 	kvm_exit();
4315 }
4316 
4317 module_init(kvm_s390_init);
4318 module_exit(kvm_s390_exit);
4319 
4320 /*
4321  * Enable autoloading of the kvm module.
4322  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4323  * since x86 takes a different approach.
4324  */
4325 #include <linux/miscdevice.h>
4326 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4327 MODULE_ALIAS("devname:kvm");
4328