xref: /openbmc/linux/arch/s390/kvm/kvm-s390.c (revision 6774def6)
1 /*
2  * hosting zSeries kernel virtual machines
3  *
4  * Copyright IBM Corp. 2008, 2009
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License (version 2 only)
8  * as published by the Free Software Foundation.
9  *
10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
11  *               Christian Borntraeger <borntraeger@de.ibm.com>
12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
14  *               Jason J. Herne <jjherne@us.ibm.com>
15  */
16 
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/slab.h>
26 #include <linux/timer.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/lowcore.h>
29 #include <asm/pgtable.h>
30 #include <asm/nmi.h>
31 #include <asm/switch_to.h>
32 #include <asm/facility.h>
33 #include <asm/sclp.h>
34 #include "kvm-s390.h"
35 #include "gaccess.h"
36 
37 #define CREATE_TRACE_POINTS
38 #include "trace.h"
39 #include "trace-s390.h"
40 
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42 
43 struct kvm_stats_debugfs_item debugfs_entries[] = {
44 	{ "userspace_handled", VCPU_STAT(exit_userspace) },
45 	{ "exit_null", VCPU_STAT(exit_null) },
46 	{ "exit_validity", VCPU_STAT(exit_validity) },
47 	{ "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 	{ "exit_external_request", VCPU_STAT(exit_external_request) },
49 	{ "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
50 	{ "exit_instruction", VCPU_STAT(exit_instruction) },
51 	{ "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 	{ "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
53 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
54 	{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
55 	{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
56 	{ "instruction_stctl", VCPU_STAT(instruction_stctl) },
57 	{ "instruction_stctg", VCPU_STAT(instruction_stctg) },
58 	{ "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
59 	{ "deliver_external_call", VCPU_STAT(deliver_external_call) },
60 	{ "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
61 	{ "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
62 	{ "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
63 	{ "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
64 	{ "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
65 	{ "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
66 	{ "exit_wait_state", VCPU_STAT(exit_wait_state) },
67 	{ "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
68 	{ "instruction_stidp", VCPU_STAT(instruction_stidp) },
69 	{ "instruction_spx", VCPU_STAT(instruction_spx) },
70 	{ "instruction_stpx", VCPU_STAT(instruction_stpx) },
71 	{ "instruction_stap", VCPU_STAT(instruction_stap) },
72 	{ "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
73 	{ "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
74 	{ "instruction_stsch", VCPU_STAT(instruction_stsch) },
75 	{ "instruction_chsc", VCPU_STAT(instruction_chsc) },
76 	{ "instruction_essa", VCPU_STAT(instruction_essa) },
77 	{ "instruction_stsi", VCPU_STAT(instruction_stsi) },
78 	{ "instruction_stfl", VCPU_STAT(instruction_stfl) },
79 	{ "instruction_tprot", VCPU_STAT(instruction_tprot) },
80 	{ "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
81 	{ "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
82 	{ "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
83 	{ "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
84 	{ "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
85 	{ "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
86 	{ "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
87 	{ "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
88 	{ "diagnose_10", VCPU_STAT(diagnose_10) },
89 	{ "diagnose_44", VCPU_STAT(diagnose_44) },
90 	{ "diagnose_9c", VCPU_STAT(diagnose_9c) },
91 	{ NULL }
92 };
93 
94 unsigned long *vfacilities;
95 static struct gmap_notifier gmap_notifier;
96 
97 /* test availability of vfacility */
98 int test_vfacility(unsigned long nr)
99 {
100 	return __test_facility(nr, (void *) vfacilities);
101 }
102 
103 /* Section: not file related */
104 int kvm_arch_hardware_enable(void)
105 {
106 	/* every s390 is virtualization enabled ;-) */
107 	return 0;
108 }
109 
110 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
111 
112 int kvm_arch_hardware_setup(void)
113 {
114 	gmap_notifier.notifier_call = kvm_gmap_notifier;
115 	gmap_register_ipte_notifier(&gmap_notifier);
116 	return 0;
117 }
118 
119 void kvm_arch_hardware_unsetup(void)
120 {
121 	gmap_unregister_ipte_notifier(&gmap_notifier);
122 }
123 
124 int kvm_arch_init(void *opaque)
125 {
126 	/* Register floating interrupt controller interface. */
127 	return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
128 }
129 
130 /* Section: device related */
131 long kvm_arch_dev_ioctl(struct file *filp,
132 			unsigned int ioctl, unsigned long arg)
133 {
134 	if (ioctl == KVM_S390_ENABLE_SIE)
135 		return s390_enable_sie();
136 	return -EINVAL;
137 }
138 
139 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
140 {
141 	int r;
142 
143 	switch (ext) {
144 	case KVM_CAP_S390_PSW:
145 	case KVM_CAP_S390_GMAP:
146 	case KVM_CAP_SYNC_MMU:
147 #ifdef CONFIG_KVM_S390_UCONTROL
148 	case KVM_CAP_S390_UCONTROL:
149 #endif
150 	case KVM_CAP_ASYNC_PF:
151 	case KVM_CAP_SYNC_REGS:
152 	case KVM_CAP_ONE_REG:
153 	case KVM_CAP_ENABLE_CAP:
154 	case KVM_CAP_S390_CSS_SUPPORT:
155 	case KVM_CAP_IRQFD:
156 	case KVM_CAP_IOEVENTFD:
157 	case KVM_CAP_DEVICE_CTRL:
158 	case KVM_CAP_ENABLE_CAP_VM:
159 	case KVM_CAP_S390_IRQCHIP:
160 	case KVM_CAP_VM_ATTRIBUTES:
161 	case KVM_CAP_MP_STATE:
162 		r = 1;
163 		break;
164 	case KVM_CAP_NR_VCPUS:
165 	case KVM_CAP_MAX_VCPUS:
166 		r = KVM_MAX_VCPUS;
167 		break;
168 	case KVM_CAP_NR_MEMSLOTS:
169 		r = KVM_USER_MEM_SLOTS;
170 		break;
171 	case KVM_CAP_S390_COW:
172 		r = MACHINE_HAS_ESOP;
173 		break;
174 	default:
175 		r = 0;
176 	}
177 	return r;
178 }
179 
180 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
181 					struct kvm_memory_slot *memslot)
182 {
183 	gfn_t cur_gfn, last_gfn;
184 	unsigned long address;
185 	struct gmap *gmap = kvm->arch.gmap;
186 
187 	down_read(&gmap->mm->mmap_sem);
188 	/* Loop over all guest pages */
189 	last_gfn = memslot->base_gfn + memslot->npages;
190 	for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
191 		address = gfn_to_hva_memslot(memslot, cur_gfn);
192 
193 		if (gmap_test_and_clear_dirty(address, gmap))
194 			mark_page_dirty(kvm, cur_gfn);
195 	}
196 	up_read(&gmap->mm->mmap_sem);
197 }
198 
199 /* Section: vm related */
200 /*
201  * Get (and clear) the dirty memory log for a memory slot.
202  */
203 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
204 			       struct kvm_dirty_log *log)
205 {
206 	int r;
207 	unsigned long n;
208 	struct kvm_memory_slot *memslot;
209 	int is_dirty = 0;
210 
211 	mutex_lock(&kvm->slots_lock);
212 
213 	r = -EINVAL;
214 	if (log->slot >= KVM_USER_MEM_SLOTS)
215 		goto out;
216 
217 	memslot = id_to_memslot(kvm->memslots, log->slot);
218 	r = -ENOENT;
219 	if (!memslot->dirty_bitmap)
220 		goto out;
221 
222 	kvm_s390_sync_dirty_log(kvm, memslot);
223 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
224 	if (r)
225 		goto out;
226 
227 	/* Clear the dirty log */
228 	if (is_dirty) {
229 		n = kvm_dirty_bitmap_bytes(memslot);
230 		memset(memslot->dirty_bitmap, 0, n);
231 	}
232 	r = 0;
233 out:
234 	mutex_unlock(&kvm->slots_lock);
235 	return r;
236 }
237 
238 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
239 {
240 	int r;
241 
242 	if (cap->flags)
243 		return -EINVAL;
244 
245 	switch (cap->cap) {
246 	case KVM_CAP_S390_IRQCHIP:
247 		kvm->arch.use_irqchip = 1;
248 		r = 0;
249 		break;
250 	default:
251 		r = -EINVAL;
252 		break;
253 	}
254 	return r;
255 }
256 
257 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
258 {
259 	int ret;
260 	unsigned int idx;
261 	switch (attr->attr) {
262 	case KVM_S390_VM_MEM_ENABLE_CMMA:
263 		ret = -EBUSY;
264 		mutex_lock(&kvm->lock);
265 		if (atomic_read(&kvm->online_vcpus) == 0) {
266 			kvm->arch.use_cmma = 1;
267 			ret = 0;
268 		}
269 		mutex_unlock(&kvm->lock);
270 		break;
271 	case KVM_S390_VM_MEM_CLR_CMMA:
272 		mutex_lock(&kvm->lock);
273 		idx = srcu_read_lock(&kvm->srcu);
274 		page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
275 		srcu_read_unlock(&kvm->srcu, idx);
276 		mutex_unlock(&kvm->lock);
277 		ret = 0;
278 		break;
279 	default:
280 		ret = -ENXIO;
281 		break;
282 	}
283 	return ret;
284 }
285 
286 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
287 {
288 	int ret;
289 
290 	switch (attr->group) {
291 	case KVM_S390_VM_MEM_CTRL:
292 		ret = kvm_s390_mem_control(kvm, attr);
293 		break;
294 	default:
295 		ret = -ENXIO;
296 		break;
297 	}
298 
299 	return ret;
300 }
301 
302 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
303 {
304 	return -ENXIO;
305 }
306 
307 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
308 {
309 	int ret;
310 
311 	switch (attr->group) {
312 	case KVM_S390_VM_MEM_CTRL:
313 		switch (attr->attr) {
314 		case KVM_S390_VM_MEM_ENABLE_CMMA:
315 		case KVM_S390_VM_MEM_CLR_CMMA:
316 			ret = 0;
317 			break;
318 		default:
319 			ret = -ENXIO;
320 			break;
321 		}
322 		break;
323 	default:
324 		ret = -ENXIO;
325 		break;
326 	}
327 
328 	return ret;
329 }
330 
331 long kvm_arch_vm_ioctl(struct file *filp,
332 		       unsigned int ioctl, unsigned long arg)
333 {
334 	struct kvm *kvm = filp->private_data;
335 	void __user *argp = (void __user *)arg;
336 	struct kvm_device_attr attr;
337 	int r;
338 
339 	switch (ioctl) {
340 	case KVM_S390_INTERRUPT: {
341 		struct kvm_s390_interrupt s390int;
342 
343 		r = -EFAULT;
344 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
345 			break;
346 		r = kvm_s390_inject_vm(kvm, &s390int);
347 		break;
348 	}
349 	case KVM_ENABLE_CAP: {
350 		struct kvm_enable_cap cap;
351 		r = -EFAULT;
352 		if (copy_from_user(&cap, argp, sizeof(cap)))
353 			break;
354 		r = kvm_vm_ioctl_enable_cap(kvm, &cap);
355 		break;
356 	}
357 	case KVM_CREATE_IRQCHIP: {
358 		struct kvm_irq_routing_entry routing;
359 
360 		r = -EINVAL;
361 		if (kvm->arch.use_irqchip) {
362 			/* Set up dummy routing. */
363 			memset(&routing, 0, sizeof(routing));
364 			kvm_set_irq_routing(kvm, &routing, 0, 0);
365 			r = 0;
366 		}
367 		break;
368 	}
369 	case KVM_SET_DEVICE_ATTR: {
370 		r = -EFAULT;
371 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
372 			break;
373 		r = kvm_s390_vm_set_attr(kvm, &attr);
374 		break;
375 	}
376 	case KVM_GET_DEVICE_ATTR: {
377 		r = -EFAULT;
378 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
379 			break;
380 		r = kvm_s390_vm_get_attr(kvm, &attr);
381 		break;
382 	}
383 	case KVM_HAS_DEVICE_ATTR: {
384 		r = -EFAULT;
385 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
386 			break;
387 		r = kvm_s390_vm_has_attr(kvm, &attr);
388 		break;
389 	}
390 	default:
391 		r = -ENOTTY;
392 	}
393 
394 	return r;
395 }
396 
397 static int kvm_s390_crypto_init(struct kvm *kvm)
398 {
399 	if (!test_vfacility(76))
400 		return 0;
401 
402 	kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
403 					 GFP_KERNEL | GFP_DMA);
404 	if (!kvm->arch.crypto.crycb)
405 		return -ENOMEM;
406 
407 	kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
408 				  CRYCB_FORMAT1;
409 
410 	return 0;
411 }
412 
413 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
414 {
415 	int rc;
416 	char debug_name[16];
417 	static unsigned long sca_offset;
418 
419 	rc = -EINVAL;
420 #ifdef CONFIG_KVM_S390_UCONTROL
421 	if (type & ~KVM_VM_S390_UCONTROL)
422 		goto out_err;
423 	if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
424 		goto out_err;
425 #else
426 	if (type)
427 		goto out_err;
428 #endif
429 
430 	rc = s390_enable_sie();
431 	if (rc)
432 		goto out_err;
433 
434 	rc = -ENOMEM;
435 
436 	kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
437 	if (!kvm->arch.sca)
438 		goto out_err;
439 	spin_lock(&kvm_lock);
440 	sca_offset = (sca_offset + 16) & 0x7f0;
441 	kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
442 	spin_unlock(&kvm_lock);
443 
444 	sprintf(debug_name, "kvm-%u", current->pid);
445 
446 	kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
447 	if (!kvm->arch.dbf)
448 		goto out_nodbf;
449 
450 	if (kvm_s390_crypto_init(kvm) < 0)
451 		goto out_crypto;
452 
453 	spin_lock_init(&kvm->arch.float_int.lock);
454 	INIT_LIST_HEAD(&kvm->arch.float_int.list);
455 	init_waitqueue_head(&kvm->arch.ipte_wq);
456 
457 	debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
458 	VM_EVENT(kvm, 3, "%s", "vm created");
459 
460 	if (type & KVM_VM_S390_UCONTROL) {
461 		kvm->arch.gmap = NULL;
462 	} else {
463 		kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
464 		if (!kvm->arch.gmap)
465 			goto out_nogmap;
466 		kvm->arch.gmap->private = kvm;
467 		kvm->arch.gmap->pfault_enabled = 0;
468 	}
469 
470 	kvm->arch.css_support = 0;
471 	kvm->arch.use_irqchip = 0;
472 
473 	spin_lock_init(&kvm->arch.start_stop_lock);
474 
475 	return 0;
476 out_nogmap:
477 	kfree(kvm->arch.crypto.crycb);
478 out_crypto:
479 	debug_unregister(kvm->arch.dbf);
480 out_nodbf:
481 	free_page((unsigned long)(kvm->arch.sca));
482 out_err:
483 	return rc;
484 }
485 
486 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
487 {
488 	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
489 	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
490 	kvm_s390_clear_local_irqs(vcpu);
491 	kvm_clear_async_pf_completion_queue(vcpu);
492 	if (!kvm_is_ucontrol(vcpu->kvm)) {
493 		clear_bit(63 - vcpu->vcpu_id,
494 			  (unsigned long *) &vcpu->kvm->arch.sca->mcn);
495 		if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
496 		    (__u64) vcpu->arch.sie_block)
497 			vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
498 	}
499 	smp_mb();
500 
501 	if (kvm_is_ucontrol(vcpu->kvm))
502 		gmap_free(vcpu->arch.gmap);
503 
504 	if (kvm_s390_cmma_enabled(vcpu->kvm))
505 		kvm_s390_vcpu_unsetup_cmma(vcpu);
506 	free_page((unsigned long)(vcpu->arch.sie_block));
507 
508 	kvm_vcpu_uninit(vcpu);
509 	kmem_cache_free(kvm_vcpu_cache, vcpu);
510 }
511 
512 static void kvm_free_vcpus(struct kvm *kvm)
513 {
514 	unsigned int i;
515 	struct kvm_vcpu *vcpu;
516 
517 	kvm_for_each_vcpu(i, vcpu, kvm)
518 		kvm_arch_vcpu_destroy(vcpu);
519 
520 	mutex_lock(&kvm->lock);
521 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
522 		kvm->vcpus[i] = NULL;
523 
524 	atomic_set(&kvm->online_vcpus, 0);
525 	mutex_unlock(&kvm->lock);
526 }
527 
528 void kvm_arch_destroy_vm(struct kvm *kvm)
529 {
530 	kvm_free_vcpus(kvm);
531 	free_page((unsigned long)(kvm->arch.sca));
532 	debug_unregister(kvm->arch.dbf);
533 	kfree(kvm->arch.crypto.crycb);
534 	if (!kvm_is_ucontrol(kvm))
535 		gmap_free(kvm->arch.gmap);
536 	kvm_s390_destroy_adapters(kvm);
537 	kvm_s390_clear_float_irqs(kvm);
538 }
539 
540 /* Section: vcpu related */
541 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
542 {
543 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
544 	kvm_clear_async_pf_completion_queue(vcpu);
545 	if (kvm_is_ucontrol(vcpu->kvm)) {
546 		vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
547 		if (!vcpu->arch.gmap)
548 			return -ENOMEM;
549 		vcpu->arch.gmap->private = vcpu->kvm;
550 		return 0;
551 	}
552 
553 	vcpu->arch.gmap = vcpu->kvm->arch.gmap;
554 	vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
555 				    KVM_SYNC_GPRS |
556 				    KVM_SYNC_ACRS |
557 				    KVM_SYNC_CRS |
558 				    KVM_SYNC_ARCH0 |
559 				    KVM_SYNC_PFAULT;
560 	return 0;
561 }
562 
563 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
564 {
565 	save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
566 	save_fp_regs(vcpu->arch.host_fpregs.fprs);
567 	save_access_regs(vcpu->arch.host_acrs);
568 	restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
569 	restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
570 	restore_access_regs(vcpu->run->s.regs.acrs);
571 	gmap_enable(vcpu->arch.gmap);
572 	atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
573 }
574 
575 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
576 {
577 	atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
578 	gmap_disable(vcpu->arch.gmap);
579 	save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
580 	save_fp_regs(vcpu->arch.guest_fpregs.fprs);
581 	save_access_regs(vcpu->run->s.regs.acrs);
582 	restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
583 	restore_fp_regs(vcpu->arch.host_fpregs.fprs);
584 	restore_access_regs(vcpu->arch.host_acrs);
585 }
586 
587 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
588 {
589 	/* this equals initial cpu reset in pop, but we don't switch to ESA */
590 	vcpu->arch.sie_block->gpsw.mask = 0UL;
591 	vcpu->arch.sie_block->gpsw.addr = 0UL;
592 	kvm_s390_set_prefix(vcpu, 0);
593 	vcpu->arch.sie_block->cputm     = 0UL;
594 	vcpu->arch.sie_block->ckc       = 0UL;
595 	vcpu->arch.sie_block->todpr     = 0;
596 	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
597 	vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
598 	vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
599 	vcpu->arch.guest_fpregs.fpc = 0;
600 	asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
601 	vcpu->arch.sie_block->gbea = 1;
602 	vcpu->arch.sie_block->pp = 0;
603 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
604 	kvm_clear_async_pf_completion_queue(vcpu);
605 	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
606 		kvm_s390_vcpu_stop(vcpu);
607 	kvm_s390_clear_local_irqs(vcpu);
608 }
609 
610 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
611 {
612 	return 0;
613 }
614 
615 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
616 {
617 	if (!test_vfacility(76))
618 		return;
619 
620 	vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
621 }
622 
623 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
624 {
625 	free_page(vcpu->arch.sie_block->cbrlo);
626 	vcpu->arch.sie_block->cbrlo = 0;
627 }
628 
629 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
630 {
631 	vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
632 	if (!vcpu->arch.sie_block->cbrlo)
633 		return -ENOMEM;
634 
635 	vcpu->arch.sie_block->ecb2 |= 0x80;
636 	vcpu->arch.sie_block->ecb2 &= ~0x08;
637 	return 0;
638 }
639 
640 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
641 {
642 	int rc = 0;
643 
644 	atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
645 						    CPUSTAT_SM |
646 						    CPUSTAT_STOPPED |
647 						    CPUSTAT_GED);
648 	vcpu->arch.sie_block->ecb   = 6;
649 	if (test_vfacility(50) && test_vfacility(73))
650 		vcpu->arch.sie_block->ecb |= 0x10;
651 
652 	vcpu->arch.sie_block->ecb2  = 8;
653 	vcpu->arch.sie_block->eca   = 0xD1002000U;
654 	if (sclp_has_siif())
655 		vcpu->arch.sie_block->eca |= 1;
656 	vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
657 	vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
658 				      ICTL_TPROT;
659 
660 	if (kvm_s390_cmma_enabled(vcpu->kvm)) {
661 		rc = kvm_s390_vcpu_setup_cmma(vcpu);
662 		if (rc)
663 			return rc;
664 	}
665 	hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
666 	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
667 	get_cpu_id(&vcpu->arch.cpu_id);
668 	vcpu->arch.cpu_id.version = 0xff;
669 
670 	kvm_s390_vcpu_crypto_setup(vcpu);
671 
672 	return rc;
673 }
674 
675 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
676 				      unsigned int id)
677 {
678 	struct kvm_vcpu *vcpu;
679 	struct sie_page *sie_page;
680 	int rc = -EINVAL;
681 
682 	if (id >= KVM_MAX_VCPUS)
683 		goto out;
684 
685 	rc = -ENOMEM;
686 
687 	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
688 	if (!vcpu)
689 		goto out;
690 
691 	sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
692 	if (!sie_page)
693 		goto out_free_cpu;
694 
695 	vcpu->arch.sie_block = &sie_page->sie_block;
696 	vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
697 
698 	vcpu->arch.sie_block->icpua = id;
699 	if (!kvm_is_ucontrol(kvm)) {
700 		if (!kvm->arch.sca) {
701 			WARN_ON_ONCE(1);
702 			goto out_free_cpu;
703 		}
704 		if (!kvm->arch.sca->cpu[id].sda)
705 			kvm->arch.sca->cpu[id].sda =
706 				(__u64) vcpu->arch.sie_block;
707 		vcpu->arch.sie_block->scaoh =
708 			(__u32)(((__u64)kvm->arch.sca) >> 32);
709 		vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
710 		set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
711 	}
712 
713 	spin_lock_init(&vcpu->arch.local_int.lock);
714 	INIT_LIST_HEAD(&vcpu->arch.local_int.list);
715 	vcpu->arch.local_int.float_int = &kvm->arch.float_int;
716 	vcpu->arch.local_int.wq = &vcpu->wq;
717 	vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
718 
719 	rc = kvm_vcpu_init(vcpu, kvm, id);
720 	if (rc)
721 		goto out_free_sie_block;
722 	VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
723 		 vcpu->arch.sie_block);
724 	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
725 
726 	return vcpu;
727 out_free_sie_block:
728 	free_page((unsigned long)(vcpu->arch.sie_block));
729 out_free_cpu:
730 	kmem_cache_free(kvm_vcpu_cache, vcpu);
731 out:
732 	return ERR_PTR(rc);
733 }
734 
735 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
736 {
737 	return kvm_cpu_has_interrupt(vcpu);
738 }
739 
740 void s390_vcpu_block(struct kvm_vcpu *vcpu)
741 {
742 	atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
743 }
744 
745 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
746 {
747 	atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
748 }
749 
750 /*
751  * Kick a guest cpu out of SIE and wait until SIE is not running.
752  * If the CPU is not running (e.g. waiting as idle) the function will
753  * return immediately. */
754 void exit_sie(struct kvm_vcpu *vcpu)
755 {
756 	atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
757 	while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
758 		cpu_relax();
759 }
760 
761 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
762 void exit_sie_sync(struct kvm_vcpu *vcpu)
763 {
764 	s390_vcpu_block(vcpu);
765 	exit_sie(vcpu);
766 }
767 
768 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
769 {
770 	int i;
771 	struct kvm *kvm = gmap->private;
772 	struct kvm_vcpu *vcpu;
773 
774 	kvm_for_each_vcpu(i, vcpu, kvm) {
775 		/* match against both prefix pages */
776 		if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
777 			VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
778 			kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
779 			exit_sie_sync(vcpu);
780 		}
781 	}
782 }
783 
784 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
785 {
786 	/* kvm common code refers to this, but never calls it */
787 	BUG();
788 	return 0;
789 }
790 
791 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
792 					   struct kvm_one_reg *reg)
793 {
794 	int r = -EINVAL;
795 
796 	switch (reg->id) {
797 	case KVM_REG_S390_TODPR:
798 		r = put_user(vcpu->arch.sie_block->todpr,
799 			     (u32 __user *)reg->addr);
800 		break;
801 	case KVM_REG_S390_EPOCHDIFF:
802 		r = put_user(vcpu->arch.sie_block->epoch,
803 			     (u64 __user *)reg->addr);
804 		break;
805 	case KVM_REG_S390_CPU_TIMER:
806 		r = put_user(vcpu->arch.sie_block->cputm,
807 			     (u64 __user *)reg->addr);
808 		break;
809 	case KVM_REG_S390_CLOCK_COMP:
810 		r = put_user(vcpu->arch.sie_block->ckc,
811 			     (u64 __user *)reg->addr);
812 		break;
813 	case KVM_REG_S390_PFTOKEN:
814 		r = put_user(vcpu->arch.pfault_token,
815 			     (u64 __user *)reg->addr);
816 		break;
817 	case KVM_REG_S390_PFCOMPARE:
818 		r = put_user(vcpu->arch.pfault_compare,
819 			     (u64 __user *)reg->addr);
820 		break;
821 	case KVM_REG_S390_PFSELECT:
822 		r = put_user(vcpu->arch.pfault_select,
823 			     (u64 __user *)reg->addr);
824 		break;
825 	case KVM_REG_S390_PP:
826 		r = put_user(vcpu->arch.sie_block->pp,
827 			     (u64 __user *)reg->addr);
828 		break;
829 	case KVM_REG_S390_GBEA:
830 		r = put_user(vcpu->arch.sie_block->gbea,
831 			     (u64 __user *)reg->addr);
832 		break;
833 	default:
834 		break;
835 	}
836 
837 	return r;
838 }
839 
840 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
841 					   struct kvm_one_reg *reg)
842 {
843 	int r = -EINVAL;
844 
845 	switch (reg->id) {
846 	case KVM_REG_S390_TODPR:
847 		r = get_user(vcpu->arch.sie_block->todpr,
848 			     (u32 __user *)reg->addr);
849 		break;
850 	case KVM_REG_S390_EPOCHDIFF:
851 		r = get_user(vcpu->arch.sie_block->epoch,
852 			     (u64 __user *)reg->addr);
853 		break;
854 	case KVM_REG_S390_CPU_TIMER:
855 		r = get_user(vcpu->arch.sie_block->cputm,
856 			     (u64 __user *)reg->addr);
857 		break;
858 	case KVM_REG_S390_CLOCK_COMP:
859 		r = get_user(vcpu->arch.sie_block->ckc,
860 			     (u64 __user *)reg->addr);
861 		break;
862 	case KVM_REG_S390_PFTOKEN:
863 		r = get_user(vcpu->arch.pfault_token,
864 			     (u64 __user *)reg->addr);
865 		break;
866 	case KVM_REG_S390_PFCOMPARE:
867 		r = get_user(vcpu->arch.pfault_compare,
868 			     (u64 __user *)reg->addr);
869 		break;
870 	case KVM_REG_S390_PFSELECT:
871 		r = get_user(vcpu->arch.pfault_select,
872 			     (u64 __user *)reg->addr);
873 		break;
874 	case KVM_REG_S390_PP:
875 		r = get_user(vcpu->arch.sie_block->pp,
876 			     (u64 __user *)reg->addr);
877 		break;
878 	case KVM_REG_S390_GBEA:
879 		r = get_user(vcpu->arch.sie_block->gbea,
880 			     (u64 __user *)reg->addr);
881 		break;
882 	default:
883 		break;
884 	}
885 
886 	return r;
887 }
888 
889 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
890 {
891 	kvm_s390_vcpu_initial_reset(vcpu);
892 	return 0;
893 }
894 
895 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
896 {
897 	memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
898 	return 0;
899 }
900 
901 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
902 {
903 	memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
904 	return 0;
905 }
906 
907 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
908 				  struct kvm_sregs *sregs)
909 {
910 	memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
911 	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
912 	restore_access_regs(vcpu->run->s.regs.acrs);
913 	return 0;
914 }
915 
916 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
917 				  struct kvm_sregs *sregs)
918 {
919 	memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
920 	memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
921 	return 0;
922 }
923 
924 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
925 {
926 	if (test_fp_ctl(fpu->fpc))
927 		return -EINVAL;
928 	memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
929 	vcpu->arch.guest_fpregs.fpc = fpu->fpc;
930 	restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
931 	restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
932 	return 0;
933 }
934 
935 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
936 {
937 	memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
938 	fpu->fpc = vcpu->arch.guest_fpregs.fpc;
939 	return 0;
940 }
941 
942 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
943 {
944 	int rc = 0;
945 
946 	if (!is_vcpu_stopped(vcpu))
947 		rc = -EBUSY;
948 	else {
949 		vcpu->run->psw_mask = psw.mask;
950 		vcpu->run->psw_addr = psw.addr;
951 	}
952 	return rc;
953 }
954 
955 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
956 				  struct kvm_translation *tr)
957 {
958 	return -EINVAL; /* not implemented yet */
959 }
960 
961 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
962 			      KVM_GUESTDBG_USE_HW_BP | \
963 			      KVM_GUESTDBG_ENABLE)
964 
965 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
966 					struct kvm_guest_debug *dbg)
967 {
968 	int rc = 0;
969 
970 	vcpu->guest_debug = 0;
971 	kvm_s390_clear_bp_data(vcpu);
972 
973 	if (dbg->control & ~VALID_GUESTDBG_FLAGS)
974 		return -EINVAL;
975 
976 	if (dbg->control & KVM_GUESTDBG_ENABLE) {
977 		vcpu->guest_debug = dbg->control;
978 		/* enforce guest PER */
979 		atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
980 
981 		if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
982 			rc = kvm_s390_import_bp_data(vcpu, dbg);
983 	} else {
984 		atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
985 		vcpu->arch.guestdbg.last_bp = 0;
986 	}
987 
988 	if (rc) {
989 		vcpu->guest_debug = 0;
990 		kvm_s390_clear_bp_data(vcpu);
991 		atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
992 	}
993 
994 	return rc;
995 }
996 
997 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
998 				    struct kvm_mp_state *mp_state)
999 {
1000 	/* CHECK_STOP and LOAD are not supported yet */
1001 	return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1002 				       KVM_MP_STATE_OPERATING;
1003 }
1004 
1005 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1006 				    struct kvm_mp_state *mp_state)
1007 {
1008 	int rc = 0;
1009 
1010 	/* user space knows about this interface - let it control the state */
1011 	vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1012 
1013 	switch (mp_state->mp_state) {
1014 	case KVM_MP_STATE_STOPPED:
1015 		kvm_s390_vcpu_stop(vcpu);
1016 		break;
1017 	case KVM_MP_STATE_OPERATING:
1018 		kvm_s390_vcpu_start(vcpu);
1019 		break;
1020 	case KVM_MP_STATE_LOAD:
1021 	case KVM_MP_STATE_CHECK_STOP:
1022 		/* fall through - CHECK_STOP and LOAD are not supported yet */
1023 	default:
1024 		rc = -ENXIO;
1025 	}
1026 
1027 	return rc;
1028 }
1029 
1030 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1031 {
1032 	if (!MACHINE_IS_LPAR)
1033 		return false;
1034 	/* only enable for z10 and later */
1035 	if (!MACHINE_HAS_EDAT1)
1036 		return false;
1037 	if (!kvm->arch.use_cmma)
1038 		return false;
1039 	return true;
1040 }
1041 
1042 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1043 {
1044 	return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1045 }
1046 
1047 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1048 {
1049 retry:
1050 	s390_vcpu_unblock(vcpu);
1051 	/*
1052 	 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1053 	 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1054 	 * This ensures that the ipte instruction for this request has
1055 	 * already finished. We might race against a second unmapper that
1056 	 * wants to set the blocking bit. Lets just retry the request loop.
1057 	 */
1058 	if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1059 		int rc;
1060 		rc = gmap_ipte_notify(vcpu->arch.gmap,
1061 				      kvm_s390_get_prefix(vcpu),
1062 				      PAGE_SIZE * 2);
1063 		if (rc)
1064 			return rc;
1065 		goto retry;
1066 	}
1067 
1068 	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1069 		vcpu->arch.sie_block->ihcpu = 0xffff;
1070 		goto retry;
1071 	}
1072 
1073 	if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1074 		if (!ibs_enabled(vcpu)) {
1075 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1076 			atomic_set_mask(CPUSTAT_IBS,
1077 					&vcpu->arch.sie_block->cpuflags);
1078 		}
1079 		goto retry;
1080 	}
1081 
1082 	if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1083 		if (ibs_enabled(vcpu)) {
1084 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1085 			atomic_clear_mask(CPUSTAT_IBS,
1086 					  &vcpu->arch.sie_block->cpuflags);
1087 		}
1088 		goto retry;
1089 	}
1090 
1091 	/* nothing to do, just clear the request */
1092 	clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1093 
1094 	return 0;
1095 }
1096 
1097 /**
1098  * kvm_arch_fault_in_page - fault-in guest page if necessary
1099  * @vcpu: The corresponding virtual cpu
1100  * @gpa: Guest physical address
1101  * @writable: Whether the page should be writable or not
1102  *
1103  * Make sure that a guest page has been faulted-in on the host.
1104  *
1105  * Return: Zero on success, negative error code otherwise.
1106  */
1107 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1108 {
1109 	return gmap_fault(vcpu->arch.gmap, gpa,
1110 			  writable ? FAULT_FLAG_WRITE : 0);
1111 }
1112 
1113 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1114 				      unsigned long token)
1115 {
1116 	struct kvm_s390_interrupt inti;
1117 	inti.parm64 = token;
1118 
1119 	if (start_token) {
1120 		inti.type = KVM_S390_INT_PFAULT_INIT;
1121 		WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1122 	} else {
1123 		inti.type = KVM_S390_INT_PFAULT_DONE;
1124 		WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1125 	}
1126 }
1127 
1128 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1129 				     struct kvm_async_pf *work)
1130 {
1131 	trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1132 	__kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1133 }
1134 
1135 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1136 				 struct kvm_async_pf *work)
1137 {
1138 	trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1139 	__kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1140 }
1141 
1142 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1143 			       struct kvm_async_pf *work)
1144 {
1145 	/* s390 will always inject the page directly */
1146 }
1147 
1148 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1149 {
1150 	/*
1151 	 * s390 will always inject the page directly,
1152 	 * but we still want check_async_completion to cleanup
1153 	 */
1154 	return true;
1155 }
1156 
1157 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1158 {
1159 	hva_t hva;
1160 	struct kvm_arch_async_pf arch;
1161 	int rc;
1162 
1163 	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1164 		return 0;
1165 	if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1166 	    vcpu->arch.pfault_compare)
1167 		return 0;
1168 	if (psw_extint_disabled(vcpu))
1169 		return 0;
1170 	if (kvm_cpu_has_interrupt(vcpu))
1171 		return 0;
1172 	if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1173 		return 0;
1174 	if (!vcpu->arch.gmap->pfault_enabled)
1175 		return 0;
1176 
1177 	hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1178 	hva += current->thread.gmap_addr & ~PAGE_MASK;
1179 	if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1180 		return 0;
1181 
1182 	rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1183 	return rc;
1184 }
1185 
1186 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1187 {
1188 	int rc, cpuflags;
1189 
1190 	/*
1191 	 * On s390 notifications for arriving pages will be delivered directly
1192 	 * to the guest but the house keeping for completed pfaults is
1193 	 * handled outside the worker.
1194 	 */
1195 	kvm_check_async_pf_completion(vcpu);
1196 
1197 	memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1198 
1199 	if (need_resched())
1200 		schedule();
1201 
1202 	if (test_cpu_flag(CIF_MCCK_PENDING))
1203 		s390_handle_mcck();
1204 
1205 	if (!kvm_is_ucontrol(vcpu->kvm)) {
1206 		rc = kvm_s390_deliver_pending_interrupts(vcpu);
1207 		if (rc)
1208 			return rc;
1209 	}
1210 
1211 	rc = kvm_s390_handle_requests(vcpu);
1212 	if (rc)
1213 		return rc;
1214 
1215 	if (guestdbg_enabled(vcpu)) {
1216 		kvm_s390_backup_guest_per_regs(vcpu);
1217 		kvm_s390_patch_guest_per_regs(vcpu);
1218 	}
1219 
1220 	vcpu->arch.sie_block->icptcode = 0;
1221 	cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1222 	VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1223 	trace_kvm_s390_sie_enter(vcpu, cpuflags);
1224 
1225 	return 0;
1226 }
1227 
1228 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1229 {
1230 	int rc = -1;
1231 
1232 	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1233 		   vcpu->arch.sie_block->icptcode);
1234 	trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1235 
1236 	if (guestdbg_enabled(vcpu))
1237 		kvm_s390_restore_guest_per_regs(vcpu);
1238 
1239 	if (exit_reason >= 0) {
1240 		rc = 0;
1241 	} else if (kvm_is_ucontrol(vcpu->kvm)) {
1242 		vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1243 		vcpu->run->s390_ucontrol.trans_exc_code =
1244 						current->thread.gmap_addr;
1245 		vcpu->run->s390_ucontrol.pgm_code = 0x10;
1246 		rc = -EREMOTE;
1247 
1248 	} else if (current->thread.gmap_pfault) {
1249 		trace_kvm_s390_major_guest_pfault(vcpu);
1250 		current->thread.gmap_pfault = 0;
1251 		if (kvm_arch_setup_async_pf(vcpu)) {
1252 			rc = 0;
1253 		} else {
1254 			gpa_t gpa = current->thread.gmap_addr;
1255 			rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1256 		}
1257 	}
1258 
1259 	if (rc == -1) {
1260 		VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1261 		trace_kvm_s390_sie_fault(vcpu);
1262 		rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1263 	}
1264 
1265 	memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1266 
1267 	if (rc == 0) {
1268 		if (kvm_is_ucontrol(vcpu->kvm))
1269 			/* Don't exit for host interrupts. */
1270 			rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1271 		else
1272 			rc = kvm_handle_sie_intercept(vcpu);
1273 	}
1274 
1275 	return rc;
1276 }
1277 
1278 static int __vcpu_run(struct kvm_vcpu *vcpu)
1279 {
1280 	int rc, exit_reason;
1281 
1282 	/*
1283 	 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1284 	 * ning the guest), so that memslots (and other stuff) are protected
1285 	 */
1286 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1287 
1288 	do {
1289 		rc = vcpu_pre_run(vcpu);
1290 		if (rc)
1291 			break;
1292 
1293 		srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1294 		/*
1295 		 * As PF_VCPU will be used in fault handler, between
1296 		 * guest_enter and guest_exit should be no uaccess.
1297 		 */
1298 		preempt_disable();
1299 		kvm_guest_enter();
1300 		preempt_enable();
1301 		exit_reason = sie64a(vcpu->arch.sie_block,
1302 				     vcpu->run->s.regs.gprs);
1303 		kvm_guest_exit();
1304 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1305 
1306 		rc = vcpu_post_run(vcpu, exit_reason);
1307 	} while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1308 
1309 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1310 	return rc;
1311 }
1312 
1313 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1314 {
1315 	vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1316 	vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1317 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1318 		kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1319 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1320 		memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1321 		/* some control register changes require a tlb flush */
1322 		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1323 	}
1324 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1325 		vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1326 		vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1327 		vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1328 		vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1329 		vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1330 	}
1331 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1332 		vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1333 		vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1334 		vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1335 	}
1336 	kvm_run->kvm_dirty_regs = 0;
1337 }
1338 
1339 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1340 {
1341 	kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1342 	kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1343 	kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1344 	memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1345 	kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1346 	kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1347 	kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1348 	kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1349 	kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1350 	kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1351 	kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1352 	kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1353 }
1354 
1355 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1356 {
1357 	int rc;
1358 	sigset_t sigsaved;
1359 
1360 	if (guestdbg_exit_pending(vcpu)) {
1361 		kvm_s390_prepare_debug_exit(vcpu);
1362 		return 0;
1363 	}
1364 
1365 	if (vcpu->sigset_active)
1366 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1367 
1368 	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1369 		kvm_s390_vcpu_start(vcpu);
1370 	} else if (is_vcpu_stopped(vcpu)) {
1371 		pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1372 				   vcpu->vcpu_id);
1373 		return -EINVAL;
1374 	}
1375 
1376 	sync_regs(vcpu, kvm_run);
1377 
1378 	might_fault();
1379 	rc = __vcpu_run(vcpu);
1380 
1381 	if (signal_pending(current) && !rc) {
1382 		kvm_run->exit_reason = KVM_EXIT_INTR;
1383 		rc = -EINTR;
1384 	}
1385 
1386 	if (guestdbg_exit_pending(vcpu) && !rc)  {
1387 		kvm_s390_prepare_debug_exit(vcpu);
1388 		rc = 0;
1389 	}
1390 
1391 	if (rc == -EOPNOTSUPP) {
1392 		/* intercept cannot be handled in-kernel, prepare kvm-run */
1393 		kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1394 		kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1395 		kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1396 		kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1397 		rc = 0;
1398 	}
1399 
1400 	if (rc == -EREMOTE) {
1401 		/* intercept was handled, but userspace support is needed
1402 		 * kvm_run has been prepared by the handler */
1403 		rc = 0;
1404 	}
1405 
1406 	store_regs(vcpu, kvm_run);
1407 
1408 	if (vcpu->sigset_active)
1409 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1410 
1411 	vcpu->stat.exit_userspace++;
1412 	return rc;
1413 }
1414 
1415 /*
1416  * store status at address
1417  * we use have two special cases:
1418  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1419  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1420  */
1421 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1422 {
1423 	unsigned char archmode = 1;
1424 	unsigned int px;
1425 	u64 clkcomp;
1426 	int rc;
1427 
1428 	if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1429 		if (write_guest_abs(vcpu, 163, &archmode, 1))
1430 			return -EFAULT;
1431 		gpa = SAVE_AREA_BASE;
1432 	} else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1433 		if (write_guest_real(vcpu, 163, &archmode, 1))
1434 			return -EFAULT;
1435 		gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1436 	}
1437 	rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1438 			     vcpu->arch.guest_fpregs.fprs, 128);
1439 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1440 			      vcpu->run->s.regs.gprs, 128);
1441 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1442 			      &vcpu->arch.sie_block->gpsw, 16);
1443 	px = kvm_s390_get_prefix(vcpu);
1444 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1445 			      &px, 4);
1446 	rc |= write_guest_abs(vcpu,
1447 			      gpa + offsetof(struct save_area, fp_ctrl_reg),
1448 			      &vcpu->arch.guest_fpregs.fpc, 4);
1449 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1450 			      &vcpu->arch.sie_block->todpr, 4);
1451 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1452 			      &vcpu->arch.sie_block->cputm, 8);
1453 	clkcomp = vcpu->arch.sie_block->ckc >> 8;
1454 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1455 			      &clkcomp, 8);
1456 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1457 			      &vcpu->run->s.regs.acrs, 64);
1458 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1459 			      &vcpu->arch.sie_block->gcr, 128);
1460 	return rc ? -EFAULT : 0;
1461 }
1462 
1463 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1464 {
1465 	/*
1466 	 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1467 	 * copying in vcpu load/put. Lets update our copies before we save
1468 	 * it into the save area
1469 	 */
1470 	save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1471 	save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1472 	save_access_regs(vcpu->run->s.regs.acrs);
1473 
1474 	return kvm_s390_store_status_unloaded(vcpu, addr);
1475 }
1476 
1477 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1478 {
1479 	kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1480 	kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1481 	exit_sie_sync(vcpu);
1482 }
1483 
1484 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1485 {
1486 	unsigned int i;
1487 	struct kvm_vcpu *vcpu;
1488 
1489 	kvm_for_each_vcpu(i, vcpu, kvm) {
1490 		__disable_ibs_on_vcpu(vcpu);
1491 	}
1492 }
1493 
1494 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1495 {
1496 	kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1497 	kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1498 	exit_sie_sync(vcpu);
1499 }
1500 
1501 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1502 {
1503 	int i, online_vcpus, started_vcpus = 0;
1504 
1505 	if (!is_vcpu_stopped(vcpu))
1506 		return;
1507 
1508 	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1509 	/* Only one cpu at a time may enter/leave the STOPPED state. */
1510 	spin_lock(&vcpu->kvm->arch.start_stop_lock);
1511 	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1512 
1513 	for (i = 0; i < online_vcpus; i++) {
1514 		if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1515 			started_vcpus++;
1516 	}
1517 
1518 	if (started_vcpus == 0) {
1519 		/* we're the only active VCPU -> speed it up */
1520 		__enable_ibs_on_vcpu(vcpu);
1521 	} else if (started_vcpus == 1) {
1522 		/*
1523 		 * As we are starting a second VCPU, we have to disable
1524 		 * the IBS facility on all VCPUs to remove potentially
1525 		 * oustanding ENABLE requests.
1526 		 */
1527 		__disable_ibs_on_all_vcpus(vcpu->kvm);
1528 	}
1529 
1530 	atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1531 	/*
1532 	 * Another VCPU might have used IBS while we were offline.
1533 	 * Let's play safe and flush the VCPU at startup.
1534 	 */
1535 	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1536 	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1537 	return;
1538 }
1539 
1540 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1541 {
1542 	int i, online_vcpus, started_vcpus = 0;
1543 	struct kvm_vcpu *started_vcpu = NULL;
1544 
1545 	if (is_vcpu_stopped(vcpu))
1546 		return;
1547 
1548 	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1549 	/* Only one cpu at a time may enter/leave the STOPPED state. */
1550 	spin_lock(&vcpu->kvm->arch.start_stop_lock);
1551 	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1552 
1553 	/* Need to lock access to action_bits to avoid a SIGP race condition */
1554 	spin_lock(&vcpu->arch.local_int.lock);
1555 	atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1556 
1557 	/* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1558 	vcpu->arch.local_int.action_bits &=
1559 				 ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
1560 	spin_unlock(&vcpu->arch.local_int.lock);
1561 
1562 	__disable_ibs_on_vcpu(vcpu);
1563 
1564 	for (i = 0; i < online_vcpus; i++) {
1565 		if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1566 			started_vcpus++;
1567 			started_vcpu = vcpu->kvm->vcpus[i];
1568 		}
1569 	}
1570 
1571 	if (started_vcpus == 1) {
1572 		/*
1573 		 * As we only have one VCPU left, we want to enable the
1574 		 * IBS facility for that VCPU to speed it up.
1575 		 */
1576 		__enable_ibs_on_vcpu(started_vcpu);
1577 	}
1578 
1579 	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1580 	return;
1581 }
1582 
1583 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1584 				     struct kvm_enable_cap *cap)
1585 {
1586 	int r;
1587 
1588 	if (cap->flags)
1589 		return -EINVAL;
1590 
1591 	switch (cap->cap) {
1592 	case KVM_CAP_S390_CSS_SUPPORT:
1593 		if (!vcpu->kvm->arch.css_support) {
1594 			vcpu->kvm->arch.css_support = 1;
1595 			trace_kvm_s390_enable_css(vcpu->kvm);
1596 		}
1597 		r = 0;
1598 		break;
1599 	default:
1600 		r = -EINVAL;
1601 		break;
1602 	}
1603 	return r;
1604 }
1605 
1606 long kvm_arch_vcpu_ioctl(struct file *filp,
1607 			 unsigned int ioctl, unsigned long arg)
1608 {
1609 	struct kvm_vcpu *vcpu = filp->private_data;
1610 	void __user *argp = (void __user *)arg;
1611 	int idx;
1612 	long r;
1613 
1614 	switch (ioctl) {
1615 	case KVM_S390_INTERRUPT: {
1616 		struct kvm_s390_interrupt s390int;
1617 
1618 		r = -EFAULT;
1619 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
1620 			break;
1621 		r = kvm_s390_inject_vcpu(vcpu, &s390int);
1622 		break;
1623 	}
1624 	case KVM_S390_STORE_STATUS:
1625 		idx = srcu_read_lock(&vcpu->kvm->srcu);
1626 		r = kvm_s390_vcpu_store_status(vcpu, arg);
1627 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
1628 		break;
1629 	case KVM_S390_SET_INITIAL_PSW: {
1630 		psw_t psw;
1631 
1632 		r = -EFAULT;
1633 		if (copy_from_user(&psw, argp, sizeof(psw)))
1634 			break;
1635 		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1636 		break;
1637 	}
1638 	case KVM_S390_INITIAL_RESET:
1639 		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1640 		break;
1641 	case KVM_SET_ONE_REG:
1642 	case KVM_GET_ONE_REG: {
1643 		struct kvm_one_reg reg;
1644 		r = -EFAULT;
1645 		if (copy_from_user(&reg, argp, sizeof(reg)))
1646 			break;
1647 		if (ioctl == KVM_SET_ONE_REG)
1648 			r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1649 		else
1650 			r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1651 		break;
1652 	}
1653 #ifdef CONFIG_KVM_S390_UCONTROL
1654 	case KVM_S390_UCAS_MAP: {
1655 		struct kvm_s390_ucas_mapping ucasmap;
1656 
1657 		if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1658 			r = -EFAULT;
1659 			break;
1660 		}
1661 
1662 		if (!kvm_is_ucontrol(vcpu->kvm)) {
1663 			r = -EINVAL;
1664 			break;
1665 		}
1666 
1667 		r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1668 				     ucasmap.vcpu_addr, ucasmap.length);
1669 		break;
1670 	}
1671 	case KVM_S390_UCAS_UNMAP: {
1672 		struct kvm_s390_ucas_mapping ucasmap;
1673 
1674 		if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1675 			r = -EFAULT;
1676 			break;
1677 		}
1678 
1679 		if (!kvm_is_ucontrol(vcpu->kvm)) {
1680 			r = -EINVAL;
1681 			break;
1682 		}
1683 
1684 		r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1685 			ucasmap.length);
1686 		break;
1687 	}
1688 #endif
1689 	case KVM_S390_VCPU_FAULT: {
1690 		r = gmap_fault(vcpu->arch.gmap, arg, 0);
1691 		break;
1692 	}
1693 	case KVM_ENABLE_CAP:
1694 	{
1695 		struct kvm_enable_cap cap;
1696 		r = -EFAULT;
1697 		if (copy_from_user(&cap, argp, sizeof(cap)))
1698 			break;
1699 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1700 		break;
1701 	}
1702 	default:
1703 		r = -ENOTTY;
1704 	}
1705 	return r;
1706 }
1707 
1708 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1709 {
1710 #ifdef CONFIG_KVM_S390_UCONTROL
1711 	if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1712 		 && (kvm_is_ucontrol(vcpu->kvm))) {
1713 		vmf->page = virt_to_page(vcpu->arch.sie_block);
1714 		get_page(vmf->page);
1715 		return 0;
1716 	}
1717 #endif
1718 	return VM_FAULT_SIGBUS;
1719 }
1720 
1721 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1722 			    unsigned long npages)
1723 {
1724 	return 0;
1725 }
1726 
1727 /* Section: memory related */
1728 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1729 				   struct kvm_memory_slot *memslot,
1730 				   struct kvm_userspace_memory_region *mem,
1731 				   enum kvm_mr_change change)
1732 {
1733 	/* A few sanity checks. We can have memory slots which have to be
1734 	   located/ended at a segment boundary (1MB). The memory in userland is
1735 	   ok to be fragmented into various different vmas. It is okay to mmap()
1736 	   and munmap() stuff in this slot after doing this call at any time */
1737 
1738 	if (mem->userspace_addr & 0xffffful)
1739 		return -EINVAL;
1740 
1741 	if (mem->memory_size & 0xffffful)
1742 		return -EINVAL;
1743 
1744 	return 0;
1745 }
1746 
1747 void kvm_arch_commit_memory_region(struct kvm *kvm,
1748 				struct kvm_userspace_memory_region *mem,
1749 				const struct kvm_memory_slot *old,
1750 				enum kvm_mr_change change)
1751 {
1752 	int rc;
1753 
1754 	/* If the basics of the memslot do not change, we do not want
1755 	 * to update the gmap. Every update causes several unnecessary
1756 	 * segment translation exceptions. This is usually handled just
1757 	 * fine by the normal fault handler + gmap, but it will also
1758 	 * cause faults on the prefix page of running guest CPUs.
1759 	 */
1760 	if (old->userspace_addr == mem->userspace_addr &&
1761 	    old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1762 	    old->npages * PAGE_SIZE == mem->memory_size)
1763 		return;
1764 
1765 	rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1766 		mem->guest_phys_addr, mem->memory_size);
1767 	if (rc)
1768 		printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1769 	return;
1770 }
1771 
1772 static int __init kvm_s390_init(void)
1773 {
1774 	int ret;
1775 	ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1776 	if (ret)
1777 		return ret;
1778 
1779 	/*
1780 	 * guests can ask for up to 255+1 double words, we need a full page
1781 	 * to hold the maximum amount of facilities. On the other hand, we
1782 	 * only set facilities that are known to work in KVM.
1783 	 */
1784 	vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1785 	if (!vfacilities) {
1786 		kvm_exit();
1787 		return -ENOMEM;
1788 	}
1789 	memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1790 	vfacilities[0] &= 0xff82fffbf47c2000UL;
1791 	vfacilities[1] &= 0x005c000000000000UL;
1792 	return 0;
1793 }
1794 
1795 static void __exit kvm_s390_exit(void)
1796 {
1797 	free_page((unsigned long) vfacilities);
1798 	kvm_exit();
1799 }
1800 
1801 module_init(kvm_s390_init);
1802 module_exit(kvm_s390_exit);
1803 
1804 /*
1805  * Enable autoloading of the kvm module.
1806  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1807  * since x86 takes a different approach.
1808  */
1809 #include <linux/miscdevice.h>
1810 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1811 MODULE_ALIAS("devname:kvm");
1812