1 /* 2 * hosting zSeries kernel virtual machines 3 * 4 * Copyright IBM Corp. 2008, 2009 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 * Christian Borntraeger <borntraeger@de.ibm.com> 12 * Heiko Carstens <heiko.carstens@de.ibm.com> 13 * Christian Ehrhardt <ehrhardt@de.ibm.com> 14 * Jason J. Herne <jjherne@us.ibm.com> 15 */ 16 17 #include <linux/compiler.h> 18 #include <linux/err.h> 19 #include <linux/fs.h> 20 #include <linux/hrtimer.h> 21 #include <linux/init.h> 22 #include <linux/kvm.h> 23 #include <linux/kvm_host.h> 24 #include <linux/module.h> 25 #include <linux/slab.h> 26 #include <linux/timer.h> 27 #include <asm/asm-offsets.h> 28 #include <asm/lowcore.h> 29 #include <asm/pgtable.h> 30 #include <asm/nmi.h> 31 #include <asm/switch_to.h> 32 #include <asm/facility.h> 33 #include <asm/sclp.h> 34 #include "kvm-s390.h" 35 #include "gaccess.h" 36 37 #define CREATE_TRACE_POINTS 38 #include "trace.h" 39 #include "trace-s390.h" 40 41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 42 43 struct kvm_stats_debugfs_item debugfs_entries[] = { 44 { "userspace_handled", VCPU_STAT(exit_userspace) }, 45 { "exit_null", VCPU_STAT(exit_null) }, 46 { "exit_validity", VCPU_STAT(exit_validity) }, 47 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 48 { "exit_external_request", VCPU_STAT(exit_external_request) }, 49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 50 { "exit_instruction", VCPU_STAT(exit_instruction) }, 51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 53 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 54 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 55 { "instruction_stctl", VCPU_STAT(instruction_stctl) }, 56 { "instruction_stctg", VCPU_STAT(instruction_stctg) }, 57 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 58 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 59 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 60 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 61 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 62 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 63 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 64 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 65 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 66 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 67 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 68 { "instruction_spx", VCPU_STAT(instruction_spx) }, 69 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 70 { "instruction_stap", VCPU_STAT(instruction_stap) }, 71 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 72 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 73 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 74 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 75 { "instruction_essa", VCPU_STAT(instruction_essa) }, 76 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 77 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 78 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 79 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 80 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 81 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 82 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 83 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 84 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 85 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 86 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 87 { "diagnose_10", VCPU_STAT(diagnose_10) }, 88 { "diagnose_44", VCPU_STAT(diagnose_44) }, 89 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 90 { NULL } 91 }; 92 93 unsigned long *vfacilities; 94 static struct gmap_notifier gmap_notifier; 95 96 /* test availability of vfacility */ 97 int test_vfacility(unsigned long nr) 98 { 99 return __test_facility(nr, (void *) vfacilities); 100 } 101 102 /* Section: not file related */ 103 int kvm_arch_hardware_enable(void *garbage) 104 { 105 /* every s390 is virtualization enabled ;-) */ 106 return 0; 107 } 108 109 void kvm_arch_hardware_disable(void *garbage) 110 { 111 } 112 113 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address); 114 115 int kvm_arch_hardware_setup(void) 116 { 117 gmap_notifier.notifier_call = kvm_gmap_notifier; 118 gmap_register_ipte_notifier(&gmap_notifier); 119 return 0; 120 } 121 122 void kvm_arch_hardware_unsetup(void) 123 { 124 gmap_unregister_ipte_notifier(&gmap_notifier); 125 } 126 127 void kvm_arch_check_processor_compat(void *rtn) 128 { 129 } 130 131 int kvm_arch_init(void *opaque) 132 { 133 return 0; 134 } 135 136 void kvm_arch_exit(void) 137 { 138 } 139 140 /* Section: device related */ 141 long kvm_arch_dev_ioctl(struct file *filp, 142 unsigned int ioctl, unsigned long arg) 143 { 144 if (ioctl == KVM_S390_ENABLE_SIE) 145 return s390_enable_sie(); 146 return -EINVAL; 147 } 148 149 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 150 { 151 int r; 152 153 switch (ext) { 154 case KVM_CAP_S390_PSW: 155 case KVM_CAP_S390_GMAP: 156 case KVM_CAP_SYNC_MMU: 157 #ifdef CONFIG_KVM_S390_UCONTROL 158 case KVM_CAP_S390_UCONTROL: 159 #endif 160 case KVM_CAP_ASYNC_PF: 161 case KVM_CAP_SYNC_REGS: 162 case KVM_CAP_ONE_REG: 163 case KVM_CAP_ENABLE_CAP: 164 case KVM_CAP_S390_CSS_SUPPORT: 165 case KVM_CAP_IRQFD: 166 case KVM_CAP_IOEVENTFD: 167 case KVM_CAP_DEVICE_CTRL: 168 case KVM_CAP_ENABLE_CAP_VM: 169 case KVM_CAP_S390_IRQCHIP: 170 case KVM_CAP_VM_ATTRIBUTES: 171 case KVM_CAP_MP_STATE: 172 r = 1; 173 break; 174 case KVM_CAP_NR_VCPUS: 175 case KVM_CAP_MAX_VCPUS: 176 r = KVM_MAX_VCPUS; 177 break; 178 case KVM_CAP_NR_MEMSLOTS: 179 r = KVM_USER_MEM_SLOTS; 180 break; 181 case KVM_CAP_S390_COW: 182 r = MACHINE_HAS_ESOP; 183 break; 184 default: 185 r = 0; 186 } 187 return r; 188 } 189 190 static void kvm_s390_sync_dirty_log(struct kvm *kvm, 191 struct kvm_memory_slot *memslot) 192 { 193 gfn_t cur_gfn, last_gfn; 194 unsigned long address; 195 struct gmap *gmap = kvm->arch.gmap; 196 197 down_read(&gmap->mm->mmap_sem); 198 /* Loop over all guest pages */ 199 last_gfn = memslot->base_gfn + memslot->npages; 200 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { 201 address = gfn_to_hva_memslot(memslot, cur_gfn); 202 203 if (gmap_test_and_clear_dirty(address, gmap)) 204 mark_page_dirty(kvm, cur_gfn); 205 } 206 up_read(&gmap->mm->mmap_sem); 207 } 208 209 /* Section: vm related */ 210 /* 211 * Get (and clear) the dirty memory log for a memory slot. 212 */ 213 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 214 struct kvm_dirty_log *log) 215 { 216 int r; 217 unsigned long n; 218 struct kvm_memory_slot *memslot; 219 int is_dirty = 0; 220 221 mutex_lock(&kvm->slots_lock); 222 223 r = -EINVAL; 224 if (log->slot >= KVM_USER_MEM_SLOTS) 225 goto out; 226 227 memslot = id_to_memslot(kvm->memslots, log->slot); 228 r = -ENOENT; 229 if (!memslot->dirty_bitmap) 230 goto out; 231 232 kvm_s390_sync_dirty_log(kvm, memslot); 233 r = kvm_get_dirty_log(kvm, log, &is_dirty); 234 if (r) 235 goto out; 236 237 /* Clear the dirty log */ 238 if (is_dirty) { 239 n = kvm_dirty_bitmap_bytes(memslot); 240 memset(memslot->dirty_bitmap, 0, n); 241 } 242 r = 0; 243 out: 244 mutex_unlock(&kvm->slots_lock); 245 return r; 246 } 247 248 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) 249 { 250 int r; 251 252 if (cap->flags) 253 return -EINVAL; 254 255 switch (cap->cap) { 256 case KVM_CAP_S390_IRQCHIP: 257 kvm->arch.use_irqchip = 1; 258 r = 0; 259 break; 260 default: 261 r = -EINVAL; 262 break; 263 } 264 return r; 265 } 266 267 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 268 { 269 int ret; 270 unsigned int idx; 271 switch (attr->attr) { 272 case KVM_S390_VM_MEM_ENABLE_CMMA: 273 ret = -EBUSY; 274 mutex_lock(&kvm->lock); 275 if (atomic_read(&kvm->online_vcpus) == 0) { 276 kvm->arch.use_cmma = 1; 277 ret = 0; 278 } 279 mutex_unlock(&kvm->lock); 280 break; 281 case KVM_S390_VM_MEM_CLR_CMMA: 282 mutex_lock(&kvm->lock); 283 idx = srcu_read_lock(&kvm->srcu); 284 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false); 285 srcu_read_unlock(&kvm->srcu, idx); 286 mutex_unlock(&kvm->lock); 287 ret = 0; 288 break; 289 default: 290 ret = -ENXIO; 291 break; 292 } 293 return ret; 294 } 295 296 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 297 { 298 int ret; 299 300 switch (attr->group) { 301 case KVM_S390_VM_MEM_CTRL: 302 ret = kvm_s390_mem_control(kvm, attr); 303 break; 304 default: 305 ret = -ENXIO; 306 break; 307 } 308 309 return ret; 310 } 311 312 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr) 313 { 314 return -ENXIO; 315 } 316 317 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 318 { 319 int ret; 320 321 switch (attr->group) { 322 case KVM_S390_VM_MEM_CTRL: 323 switch (attr->attr) { 324 case KVM_S390_VM_MEM_ENABLE_CMMA: 325 case KVM_S390_VM_MEM_CLR_CMMA: 326 ret = 0; 327 break; 328 default: 329 ret = -ENXIO; 330 break; 331 } 332 break; 333 default: 334 ret = -ENXIO; 335 break; 336 } 337 338 return ret; 339 } 340 341 long kvm_arch_vm_ioctl(struct file *filp, 342 unsigned int ioctl, unsigned long arg) 343 { 344 struct kvm *kvm = filp->private_data; 345 void __user *argp = (void __user *)arg; 346 struct kvm_device_attr attr; 347 int r; 348 349 switch (ioctl) { 350 case KVM_S390_INTERRUPT: { 351 struct kvm_s390_interrupt s390int; 352 353 r = -EFAULT; 354 if (copy_from_user(&s390int, argp, sizeof(s390int))) 355 break; 356 r = kvm_s390_inject_vm(kvm, &s390int); 357 break; 358 } 359 case KVM_ENABLE_CAP: { 360 struct kvm_enable_cap cap; 361 r = -EFAULT; 362 if (copy_from_user(&cap, argp, sizeof(cap))) 363 break; 364 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 365 break; 366 } 367 case KVM_CREATE_IRQCHIP: { 368 struct kvm_irq_routing_entry routing; 369 370 r = -EINVAL; 371 if (kvm->arch.use_irqchip) { 372 /* Set up dummy routing. */ 373 memset(&routing, 0, sizeof(routing)); 374 kvm_set_irq_routing(kvm, &routing, 0, 0); 375 r = 0; 376 } 377 break; 378 } 379 case KVM_SET_DEVICE_ATTR: { 380 r = -EFAULT; 381 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 382 break; 383 r = kvm_s390_vm_set_attr(kvm, &attr); 384 break; 385 } 386 case KVM_GET_DEVICE_ATTR: { 387 r = -EFAULT; 388 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 389 break; 390 r = kvm_s390_vm_get_attr(kvm, &attr); 391 break; 392 } 393 case KVM_HAS_DEVICE_ATTR: { 394 r = -EFAULT; 395 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 396 break; 397 r = kvm_s390_vm_has_attr(kvm, &attr); 398 break; 399 } 400 default: 401 r = -ENOTTY; 402 } 403 404 return r; 405 } 406 407 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 408 { 409 int rc; 410 char debug_name[16]; 411 static unsigned long sca_offset; 412 413 rc = -EINVAL; 414 #ifdef CONFIG_KVM_S390_UCONTROL 415 if (type & ~KVM_VM_S390_UCONTROL) 416 goto out_err; 417 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 418 goto out_err; 419 #else 420 if (type) 421 goto out_err; 422 #endif 423 424 rc = s390_enable_sie(); 425 if (rc) 426 goto out_err; 427 428 rc = -ENOMEM; 429 430 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); 431 if (!kvm->arch.sca) 432 goto out_err; 433 spin_lock(&kvm_lock); 434 sca_offset = (sca_offset + 16) & 0x7f0; 435 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset); 436 spin_unlock(&kvm_lock); 437 438 sprintf(debug_name, "kvm-%u", current->pid); 439 440 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 441 if (!kvm->arch.dbf) 442 goto out_nodbf; 443 444 spin_lock_init(&kvm->arch.float_int.lock); 445 INIT_LIST_HEAD(&kvm->arch.float_int.list); 446 init_waitqueue_head(&kvm->arch.ipte_wq); 447 448 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 449 VM_EVENT(kvm, 3, "%s", "vm created"); 450 451 if (type & KVM_VM_S390_UCONTROL) { 452 kvm->arch.gmap = NULL; 453 } else { 454 kvm->arch.gmap = gmap_alloc(current->mm); 455 if (!kvm->arch.gmap) 456 goto out_nogmap; 457 kvm->arch.gmap->private = kvm; 458 kvm->arch.gmap->pfault_enabled = 0; 459 } 460 461 kvm->arch.css_support = 0; 462 kvm->arch.use_irqchip = 0; 463 464 spin_lock_init(&kvm->arch.start_stop_lock); 465 466 return 0; 467 out_nogmap: 468 debug_unregister(kvm->arch.dbf); 469 out_nodbf: 470 free_page((unsigned long)(kvm->arch.sca)); 471 out_err: 472 return rc; 473 } 474 475 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 476 { 477 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 478 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 479 kvm_s390_clear_local_irqs(vcpu); 480 kvm_clear_async_pf_completion_queue(vcpu); 481 if (!kvm_is_ucontrol(vcpu->kvm)) { 482 clear_bit(63 - vcpu->vcpu_id, 483 (unsigned long *) &vcpu->kvm->arch.sca->mcn); 484 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda == 485 (__u64) vcpu->arch.sie_block) 486 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0; 487 } 488 smp_mb(); 489 490 if (kvm_is_ucontrol(vcpu->kvm)) 491 gmap_free(vcpu->arch.gmap); 492 493 if (kvm_s390_cmma_enabled(vcpu->kvm)) 494 kvm_s390_vcpu_unsetup_cmma(vcpu); 495 free_page((unsigned long)(vcpu->arch.sie_block)); 496 497 kvm_vcpu_uninit(vcpu); 498 kmem_cache_free(kvm_vcpu_cache, vcpu); 499 } 500 501 static void kvm_free_vcpus(struct kvm *kvm) 502 { 503 unsigned int i; 504 struct kvm_vcpu *vcpu; 505 506 kvm_for_each_vcpu(i, vcpu, kvm) 507 kvm_arch_vcpu_destroy(vcpu); 508 509 mutex_lock(&kvm->lock); 510 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 511 kvm->vcpus[i] = NULL; 512 513 atomic_set(&kvm->online_vcpus, 0); 514 mutex_unlock(&kvm->lock); 515 } 516 517 void kvm_arch_sync_events(struct kvm *kvm) 518 { 519 } 520 521 void kvm_arch_destroy_vm(struct kvm *kvm) 522 { 523 kvm_free_vcpus(kvm); 524 free_page((unsigned long)(kvm->arch.sca)); 525 debug_unregister(kvm->arch.dbf); 526 if (!kvm_is_ucontrol(kvm)) 527 gmap_free(kvm->arch.gmap); 528 kvm_s390_destroy_adapters(kvm); 529 kvm_s390_clear_float_irqs(kvm); 530 } 531 532 /* Section: vcpu related */ 533 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 534 { 535 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 536 kvm_clear_async_pf_completion_queue(vcpu); 537 if (kvm_is_ucontrol(vcpu->kvm)) { 538 vcpu->arch.gmap = gmap_alloc(current->mm); 539 if (!vcpu->arch.gmap) 540 return -ENOMEM; 541 vcpu->arch.gmap->private = vcpu->kvm; 542 return 0; 543 } 544 545 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 546 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 547 KVM_SYNC_GPRS | 548 KVM_SYNC_ACRS | 549 KVM_SYNC_CRS; 550 return 0; 551 } 552 553 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 554 { 555 /* Nothing todo */ 556 } 557 558 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 559 { 560 save_fp_ctl(&vcpu->arch.host_fpregs.fpc); 561 save_fp_regs(vcpu->arch.host_fpregs.fprs); 562 save_access_regs(vcpu->arch.host_acrs); 563 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 564 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 565 restore_access_regs(vcpu->run->s.regs.acrs); 566 gmap_enable(vcpu->arch.gmap); 567 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 568 } 569 570 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 571 { 572 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 573 gmap_disable(vcpu->arch.gmap); 574 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 575 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 576 save_access_regs(vcpu->run->s.regs.acrs); 577 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc); 578 restore_fp_regs(vcpu->arch.host_fpregs.fprs); 579 restore_access_regs(vcpu->arch.host_acrs); 580 } 581 582 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 583 { 584 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 585 vcpu->arch.sie_block->gpsw.mask = 0UL; 586 vcpu->arch.sie_block->gpsw.addr = 0UL; 587 kvm_s390_set_prefix(vcpu, 0); 588 vcpu->arch.sie_block->cputm = 0UL; 589 vcpu->arch.sie_block->ckc = 0UL; 590 vcpu->arch.sie_block->todpr = 0; 591 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 592 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 593 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 594 vcpu->arch.guest_fpregs.fpc = 0; 595 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc)); 596 vcpu->arch.sie_block->gbea = 1; 597 vcpu->arch.sie_block->pp = 0; 598 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 599 kvm_clear_async_pf_completion_queue(vcpu); 600 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 601 kvm_s390_vcpu_stop(vcpu); 602 kvm_s390_clear_local_irqs(vcpu); 603 } 604 605 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 606 { 607 return 0; 608 } 609 610 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) 611 { 612 free_page(vcpu->arch.sie_block->cbrlo); 613 vcpu->arch.sie_block->cbrlo = 0; 614 } 615 616 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) 617 { 618 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL); 619 if (!vcpu->arch.sie_block->cbrlo) 620 return -ENOMEM; 621 622 vcpu->arch.sie_block->ecb2 |= 0x80; 623 vcpu->arch.sie_block->ecb2 &= ~0x08; 624 return 0; 625 } 626 627 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 628 { 629 int rc = 0; 630 631 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 632 CPUSTAT_SM | 633 CPUSTAT_STOPPED | 634 CPUSTAT_GED); 635 vcpu->arch.sie_block->ecb = 6; 636 if (test_vfacility(50) && test_vfacility(73)) 637 vcpu->arch.sie_block->ecb |= 0x10; 638 639 vcpu->arch.sie_block->ecb2 = 8; 640 vcpu->arch.sie_block->eca = 0xD1002000U; 641 if (sclp_has_siif()) 642 vcpu->arch.sie_block->eca |= 1; 643 vcpu->arch.sie_block->fac = (int) (long) vfacilities; 644 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE | 645 ICTL_TPROT; 646 647 if (kvm_s390_cmma_enabled(vcpu->kvm)) { 648 rc = kvm_s390_vcpu_setup_cmma(vcpu); 649 if (rc) 650 return rc; 651 } 652 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 653 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 654 get_cpu_id(&vcpu->arch.cpu_id); 655 vcpu->arch.cpu_id.version = 0xff; 656 return rc; 657 } 658 659 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 660 unsigned int id) 661 { 662 struct kvm_vcpu *vcpu; 663 struct sie_page *sie_page; 664 int rc = -EINVAL; 665 666 if (id >= KVM_MAX_VCPUS) 667 goto out; 668 669 rc = -ENOMEM; 670 671 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 672 if (!vcpu) 673 goto out; 674 675 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL); 676 if (!sie_page) 677 goto out_free_cpu; 678 679 vcpu->arch.sie_block = &sie_page->sie_block; 680 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 681 682 vcpu->arch.sie_block->icpua = id; 683 if (!kvm_is_ucontrol(kvm)) { 684 if (!kvm->arch.sca) { 685 WARN_ON_ONCE(1); 686 goto out_free_cpu; 687 } 688 if (!kvm->arch.sca->cpu[id].sda) 689 kvm->arch.sca->cpu[id].sda = 690 (__u64) vcpu->arch.sie_block; 691 vcpu->arch.sie_block->scaoh = 692 (__u32)(((__u64)kvm->arch.sca) >> 32); 693 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 694 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 695 } 696 697 spin_lock_init(&vcpu->arch.local_int.lock); 698 INIT_LIST_HEAD(&vcpu->arch.local_int.list); 699 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 700 vcpu->arch.local_int.wq = &vcpu->wq; 701 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 702 703 rc = kvm_vcpu_init(vcpu, kvm, id); 704 if (rc) 705 goto out_free_sie_block; 706 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, 707 vcpu->arch.sie_block); 708 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 709 710 return vcpu; 711 out_free_sie_block: 712 free_page((unsigned long)(vcpu->arch.sie_block)); 713 out_free_cpu: 714 kmem_cache_free(kvm_vcpu_cache, vcpu); 715 out: 716 return ERR_PTR(rc); 717 } 718 719 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 720 { 721 return kvm_cpu_has_interrupt(vcpu); 722 } 723 724 void s390_vcpu_block(struct kvm_vcpu *vcpu) 725 { 726 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 727 } 728 729 void s390_vcpu_unblock(struct kvm_vcpu *vcpu) 730 { 731 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 732 } 733 734 /* 735 * Kick a guest cpu out of SIE and wait until SIE is not running. 736 * If the CPU is not running (e.g. waiting as idle) the function will 737 * return immediately. */ 738 void exit_sie(struct kvm_vcpu *vcpu) 739 { 740 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 741 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 742 cpu_relax(); 743 } 744 745 /* Kick a guest cpu out of SIE and prevent SIE-reentry */ 746 void exit_sie_sync(struct kvm_vcpu *vcpu) 747 { 748 s390_vcpu_block(vcpu); 749 exit_sie(vcpu); 750 } 751 752 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address) 753 { 754 int i; 755 struct kvm *kvm = gmap->private; 756 struct kvm_vcpu *vcpu; 757 758 kvm_for_each_vcpu(i, vcpu, kvm) { 759 /* match against both prefix pages */ 760 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) { 761 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address); 762 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 763 exit_sie_sync(vcpu); 764 } 765 } 766 } 767 768 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 769 { 770 /* kvm common code refers to this, but never calls it */ 771 BUG(); 772 return 0; 773 } 774 775 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 776 struct kvm_one_reg *reg) 777 { 778 int r = -EINVAL; 779 780 switch (reg->id) { 781 case KVM_REG_S390_TODPR: 782 r = put_user(vcpu->arch.sie_block->todpr, 783 (u32 __user *)reg->addr); 784 break; 785 case KVM_REG_S390_EPOCHDIFF: 786 r = put_user(vcpu->arch.sie_block->epoch, 787 (u64 __user *)reg->addr); 788 break; 789 case KVM_REG_S390_CPU_TIMER: 790 r = put_user(vcpu->arch.sie_block->cputm, 791 (u64 __user *)reg->addr); 792 break; 793 case KVM_REG_S390_CLOCK_COMP: 794 r = put_user(vcpu->arch.sie_block->ckc, 795 (u64 __user *)reg->addr); 796 break; 797 case KVM_REG_S390_PFTOKEN: 798 r = put_user(vcpu->arch.pfault_token, 799 (u64 __user *)reg->addr); 800 break; 801 case KVM_REG_S390_PFCOMPARE: 802 r = put_user(vcpu->arch.pfault_compare, 803 (u64 __user *)reg->addr); 804 break; 805 case KVM_REG_S390_PFSELECT: 806 r = put_user(vcpu->arch.pfault_select, 807 (u64 __user *)reg->addr); 808 break; 809 case KVM_REG_S390_PP: 810 r = put_user(vcpu->arch.sie_block->pp, 811 (u64 __user *)reg->addr); 812 break; 813 case KVM_REG_S390_GBEA: 814 r = put_user(vcpu->arch.sie_block->gbea, 815 (u64 __user *)reg->addr); 816 break; 817 default: 818 break; 819 } 820 821 return r; 822 } 823 824 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 825 struct kvm_one_reg *reg) 826 { 827 int r = -EINVAL; 828 829 switch (reg->id) { 830 case KVM_REG_S390_TODPR: 831 r = get_user(vcpu->arch.sie_block->todpr, 832 (u32 __user *)reg->addr); 833 break; 834 case KVM_REG_S390_EPOCHDIFF: 835 r = get_user(vcpu->arch.sie_block->epoch, 836 (u64 __user *)reg->addr); 837 break; 838 case KVM_REG_S390_CPU_TIMER: 839 r = get_user(vcpu->arch.sie_block->cputm, 840 (u64 __user *)reg->addr); 841 break; 842 case KVM_REG_S390_CLOCK_COMP: 843 r = get_user(vcpu->arch.sie_block->ckc, 844 (u64 __user *)reg->addr); 845 break; 846 case KVM_REG_S390_PFTOKEN: 847 r = get_user(vcpu->arch.pfault_token, 848 (u64 __user *)reg->addr); 849 break; 850 case KVM_REG_S390_PFCOMPARE: 851 r = get_user(vcpu->arch.pfault_compare, 852 (u64 __user *)reg->addr); 853 break; 854 case KVM_REG_S390_PFSELECT: 855 r = get_user(vcpu->arch.pfault_select, 856 (u64 __user *)reg->addr); 857 break; 858 case KVM_REG_S390_PP: 859 r = get_user(vcpu->arch.sie_block->pp, 860 (u64 __user *)reg->addr); 861 break; 862 case KVM_REG_S390_GBEA: 863 r = get_user(vcpu->arch.sie_block->gbea, 864 (u64 __user *)reg->addr); 865 break; 866 default: 867 break; 868 } 869 870 return r; 871 } 872 873 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 874 { 875 kvm_s390_vcpu_initial_reset(vcpu); 876 return 0; 877 } 878 879 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 880 { 881 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 882 return 0; 883 } 884 885 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 886 { 887 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 888 return 0; 889 } 890 891 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 892 struct kvm_sregs *sregs) 893 { 894 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 895 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 896 restore_access_regs(vcpu->run->s.regs.acrs); 897 return 0; 898 } 899 900 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 901 struct kvm_sregs *sregs) 902 { 903 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 904 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 905 return 0; 906 } 907 908 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 909 { 910 if (test_fp_ctl(fpu->fpc)) 911 return -EINVAL; 912 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 913 vcpu->arch.guest_fpregs.fpc = fpu->fpc; 914 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 915 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 916 return 0; 917 } 918 919 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 920 { 921 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs)); 922 fpu->fpc = vcpu->arch.guest_fpregs.fpc; 923 return 0; 924 } 925 926 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 927 { 928 int rc = 0; 929 930 if (!is_vcpu_stopped(vcpu)) 931 rc = -EBUSY; 932 else { 933 vcpu->run->psw_mask = psw.mask; 934 vcpu->run->psw_addr = psw.addr; 935 } 936 return rc; 937 } 938 939 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 940 struct kvm_translation *tr) 941 { 942 return -EINVAL; /* not implemented yet */ 943 } 944 945 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \ 946 KVM_GUESTDBG_USE_HW_BP | \ 947 KVM_GUESTDBG_ENABLE) 948 949 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 950 struct kvm_guest_debug *dbg) 951 { 952 int rc = 0; 953 954 vcpu->guest_debug = 0; 955 kvm_s390_clear_bp_data(vcpu); 956 957 if (dbg->control & ~VALID_GUESTDBG_FLAGS) 958 return -EINVAL; 959 960 if (dbg->control & KVM_GUESTDBG_ENABLE) { 961 vcpu->guest_debug = dbg->control; 962 /* enforce guest PER */ 963 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 964 965 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 966 rc = kvm_s390_import_bp_data(vcpu, dbg); 967 } else { 968 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 969 vcpu->arch.guestdbg.last_bp = 0; 970 } 971 972 if (rc) { 973 vcpu->guest_debug = 0; 974 kvm_s390_clear_bp_data(vcpu); 975 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 976 } 977 978 return rc; 979 } 980 981 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 982 struct kvm_mp_state *mp_state) 983 { 984 /* CHECK_STOP and LOAD are not supported yet */ 985 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED : 986 KVM_MP_STATE_OPERATING; 987 } 988 989 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 990 struct kvm_mp_state *mp_state) 991 { 992 int rc = 0; 993 994 /* user space knows about this interface - let it control the state */ 995 vcpu->kvm->arch.user_cpu_state_ctrl = 1; 996 997 switch (mp_state->mp_state) { 998 case KVM_MP_STATE_STOPPED: 999 kvm_s390_vcpu_stop(vcpu); 1000 break; 1001 case KVM_MP_STATE_OPERATING: 1002 kvm_s390_vcpu_start(vcpu); 1003 break; 1004 case KVM_MP_STATE_LOAD: 1005 case KVM_MP_STATE_CHECK_STOP: 1006 /* fall through - CHECK_STOP and LOAD are not supported yet */ 1007 default: 1008 rc = -ENXIO; 1009 } 1010 1011 return rc; 1012 } 1013 1014 bool kvm_s390_cmma_enabled(struct kvm *kvm) 1015 { 1016 if (!MACHINE_IS_LPAR) 1017 return false; 1018 /* only enable for z10 and later */ 1019 if (!MACHINE_HAS_EDAT1) 1020 return false; 1021 if (!kvm->arch.use_cmma) 1022 return false; 1023 return true; 1024 } 1025 1026 static bool ibs_enabled(struct kvm_vcpu *vcpu) 1027 { 1028 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS; 1029 } 1030 1031 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 1032 { 1033 retry: 1034 s390_vcpu_unblock(vcpu); 1035 /* 1036 * We use MMU_RELOAD just to re-arm the ipte notifier for the 1037 * guest prefix page. gmap_ipte_notify will wait on the ptl lock. 1038 * This ensures that the ipte instruction for this request has 1039 * already finished. We might race against a second unmapper that 1040 * wants to set the blocking bit. Lets just retry the request loop. 1041 */ 1042 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 1043 int rc; 1044 rc = gmap_ipte_notify(vcpu->arch.gmap, 1045 kvm_s390_get_prefix(vcpu), 1046 PAGE_SIZE * 2); 1047 if (rc) 1048 return rc; 1049 goto retry; 1050 } 1051 1052 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 1053 if (!ibs_enabled(vcpu)) { 1054 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 1055 atomic_set_mask(CPUSTAT_IBS, 1056 &vcpu->arch.sie_block->cpuflags); 1057 } 1058 goto retry; 1059 } 1060 1061 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 1062 if (ibs_enabled(vcpu)) { 1063 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 1064 atomic_clear_mask(CPUSTAT_IBS, 1065 &vcpu->arch.sie_block->cpuflags); 1066 } 1067 goto retry; 1068 } 1069 1070 /* nothing to do, just clear the request */ 1071 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 1072 1073 return 0; 1074 } 1075 1076 /** 1077 * kvm_arch_fault_in_page - fault-in guest page if necessary 1078 * @vcpu: The corresponding virtual cpu 1079 * @gpa: Guest physical address 1080 * @writable: Whether the page should be writable or not 1081 * 1082 * Make sure that a guest page has been faulted-in on the host. 1083 * 1084 * Return: Zero on success, negative error code otherwise. 1085 */ 1086 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable) 1087 { 1088 struct mm_struct *mm = current->mm; 1089 hva_t hva; 1090 long rc; 1091 1092 hva = gmap_fault(gpa, vcpu->arch.gmap); 1093 if (IS_ERR_VALUE(hva)) 1094 return (long)hva; 1095 down_read(&mm->mmap_sem); 1096 rc = get_user_pages(current, mm, hva, 1, writable, 0, NULL, NULL); 1097 up_read(&mm->mmap_sem); 1098 1099 return rc < 0 ? rc : 0; 1100 } 1101 1102 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, 1103 unsigned long token) 1104 { 1105 struct kvm_s390_interrupt inti; 1106 inti.parm64 = token; 1107 1108 if (start_token) { 1109 inti.type = KVM_S390_INT_PFAULT_INIT; 1110 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti)); 1111 } else { 1112 inti.type = KVM_S390_INT_PFAULT_DONE; 1113 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); 1114 } 1115 } 1116 1117 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 1118 struct kvm_async_pf *work) 1119 { 1120 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); 1121 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); 1122 } 1123 1124 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 1125 struct kvm_async_pf *work) 1126 { 1127 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token); 1128 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token); 1129 } 1130 1131 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 1132 struct kvm_async_pf *work) 1133 { 1134 /* s390 will always inject the page directly */ 1135 } 1136 1137 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 1138 { 1139 /* 1140 * s390 will always inject the page directly, 1141 * but we still want check_async_completion to cleanup 1142 */ 1143 return true; 1144 } 1145 1146 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) 1147 { 1148 hva_t hva; 1149 struct kvm_arch_async_pf arch; 1150 int rc; 1151 1152 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 1153 return 0; 1154 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) != 1155 vcpu->arch.pfault_compare) 1156 return 0; 1157 if (psw_extint_disabled(vcpu)) 1158 return 0; 1159 if (kvm_cpu_has_interrupt(vcpu)) 1160 return 0; 1161 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 1162 return 0; 1163 if (!vcpu->arch.gmap->pfault_enabled) 1164 return 0; 1165 1166 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr)); 1167 hva += current->thread.gmap_addr & ~PAGE_MASK; 1168 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8)) 1169 return 0; 1170 1171 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch); 1172 return rc; 1173 } 1174 1175 static int vcpu_pre_run(struct kvm_vcpu *vcpu) 1176 { 1177 int rc, cpuflags; 1178 1179 /* 1180 * On s390 notifications for arriving pages will be delivered directly 1181 * to the guest but the house keeping for completed pfaults is 1182 * handled outside the worker. 1183 */ 1184 kvm_check_async_pf_completion(vcpu); 1185 1186 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16); 1187 1188 if (need_resched()) 1189 schedule(); 1190 1191 if (test_cpu_flag(CIF_MCCK_PENDING)) 1192 s390_handle_mcck(); 1193 1194 if (!kvm_is_ucontrol(vcpu->kvm)) 1195 kvm_s390_deliver_pending_interrupts(vcpu); 1196 1197 rc = kvm_s390_handle_requests(vcpu); 1198 if (rc) 1199 return rc; 1200 1201 if (guestdbg_enabled(vcpu)) { 1202 kvm_s390_backup_guest_per_regs(vcpu); 1203 kvm_s390_patch_guest_per_regs(vcpu); 1204 } 1205 1206 vcpu->arch.sie_block->icptcode = 0; 1207 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); 1208 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); 1209 trace_kvm_s390_sie_enter(vcpu, cpuflags); 1210 1211 return 0; 1212 } 1213 1214 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 1215 { 1216 int rc = -1; 1217 1218 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 1219 vcpu->arch.sie_block->icptcode); 1220 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 1221 1222 if (guestdbg_enabled(vcpu)) 1223 kvm_s390_restore_guest_per_regs(vcpu); 1224 1225 if (exit_reason >= 0) { 1226 rc = 0; 1227 } else if (kvm_is_ucontrol(vcpu->kvm)) { 1228 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; 1229 vcpu->run->s390_ucontrol.trans_exc_code = 1230 current->thread.gmap_addr; 1231 vcpu->run->s390_ucontrol.pgm_code = 0x10; 1232 rc = -EREMOTE; 1233 1234 } else if (current->thread.gmap_pfault) { 1235 trace_kvm_s390_major_guest_pfault(vcpu); 1236 current->thread.gmap_pfault = 0; 1237 if (kvm_arch_setup_async_pf(vcpu)) { 1238 rc = 0; 1239 } else { 1240 gpa_t gpa = current->thread.gmap_addr; 1241 rc = kvm_arch_fault_in_page(vcpu, gpa, 1); 1242 } 1243 } 1244 1245 if (rc == -1) { 1246 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 1247 trace_kvm_s390_sie_fault(vcpu); 1248 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1249 } 1250 1251 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 1252 1253 if (rc == 0) { 1254 if (kvm_is_ucontrol(vcpu->kvm)) 1255 /* Don't exit for host interrupts. */ 1256 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0; 1257 else 1258 rc = kvm_handle_sie_intercept(vcpu); 1259 } 1260 1261 return rc; 1262 } 1263 1264 static int __vcpu_run(struct kvm_vcpu *vcpu) 1265 { 1266 int rc, exit_reason; 1267 1268 /* 1269 * We try to hold kvm->srcu during most of vcpu_run (except when run- 1270 * ning the guest), so that memslots (and other stuff) are protected 1271 */ 1272 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1273 1274 do { 1275 rc = vcpu_pre_run(vcpu); 1276 if (rc) 1277 break; 1278 1279 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1280 /* 1281 * As PF_VCPU will be used in fault handler, between 1282 * guest_enter and guest_exit should be no uaccess. 1283 */ 1284 preempt_disable(); 1285 kvm_guest_enter(); 1286 preempt_enable(); 1287 exit_reason = sie64a(vcpu->arch.sie_block, 1288 vcpu->run->s.regs.gprs); 1289 kvm_guest_exit(); 1290 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1291 1292 rc = vcpu_post_run(vcpu, exit_reason); 1293 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc); 1294 1295 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1296 return rc; 1297 } 1298 1299 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 1300 { 1301 int rc; 1302 sigset_t sigsaved; 1303 1304 if (guestdbg_exit_pending(vcpu)) { 1305 kvm_s390_prepare_debug_exit(vcpu); 1306 return 0; 1307 } 1308 1309 if (vcpu->sigset_active) 1310 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 1311 1312 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { 1313 kvm_s390_vcpu_start(vcpu); 1314 } else if (is_vcpu_stopped(vcpu)) { 1315 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n", 1316 vcpu->vcpu_id); 1317 return -EINVAL; 1318 } 1319 1320 switch (kvm_run->exit_reason) { 1321 case KVM_EXIT_S390_SIEIC: 1322 case KVM_EXIT_UNKNOWN: 1323 case KVM_EXIT_INTR: 1324 case KVM_EXIT_S390_RESET: 1325 case KVM_EXIT_S390_UCONTROL: 1326 case KVM_EXIT_S390_TSCH: 1327 case KVM_EXIT_DEBUG: 1328 break; 1329 default: 1330 BUG(); 1331 } 1332 1333 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 1334 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 1335 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) { 1336 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX; 1337 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 1338 } 1339 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 1340 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS; 1341 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 1342 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 1343 } 1344 1345 might_fault(); 1346 rc = __vcpu_run(vcpu); 1347 1348 if (signal_pending(current) && !rc) { 1349 kvm_run->exit_reason = KVM_EXIT_INTR; 1350 rc = -EINTR; 1351 } 1352 1353 if (guestdbg_exit_pending(vcpu) && !rc) { 1354 kvm_s390_prepare_debug_exit(vcpu); 1355 rc = 0; 1356 } 1357 1358 if (rc == -EOPNOTSUPP) { 1359 /* intercept cannot be handled in-kernel, prepare kvm-run */ 1360 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC; 1361 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 1362 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 1363 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 1364 rc = 0; 1365 } 1366 1367 if (rc == -EREMOTE) { 1368 /* intercept was handled, but userspace support is needed 1369 * kvm_run has been prepared by the handler */ 1370 rc = 0; 1371 } 1372 1373 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 1374 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 1375 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); 1376 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 1377 1378 if (vcpu->sigset_active) 1379 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 1380 1381 vcpu->stat.exit_userspace++; 1382 return rc; 1383 } 1384 1385 /* 1386 * store status at address 1387 * we use have two special cases: 1388 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 1389 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 1390 */ 1391 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa) 1392 { 1393 unsigned char archmode = 1; 1394 unsigned int px; 1395 u64 clkcomp; 1396 int rc; 1397 1398 if (gpa == KVM_S390_STORE_STATUS_NOADDR) { 1399 if (write_guest_abs(vcpu, 163, &archmode, 1)) 1400 return -EFAULT; 1401 gpa = SAVE_AREA_BASE; 1402 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) { 1403 if (write_guest_real(vcpu, 163, &archmode, 1)) 1404 return -EFAULT; 1405 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE); 1406 } 1407 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs), 1408 vcpu->arch.guest_fpregs.fprs, 128); 1409 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs), 1410 vcpu->run->s.regs.gprs, 128); 1411 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw), 1412 &vcpu->arch.sie_block->gpsw, 16); 1413 px = kvm_s390_get_prefix(vcpu); 1414 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg), 1415 &px, 4); 1416 rc |= write_guest_abs(vcpu, 1417 gpa + offsetof(struct save_area, fp_ctrl_reg), 1418 &vcpu->arch.guest_fpregs.fpc, 4); 1419 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg), 1420 &vcpu->arch.sie_block->todpr, 4); 1421 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer), 1422 &vcpu->arch.sie_block->cputm, 8); 1423 clkcomp = vcpu->arch.sie_block->ckc >> 8; 1424 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp), 1425 &clkcomp, 8); 1426 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs), 1427 &vcpu->run->s.regs.acrs, 64); 1428 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs), 1429 &vcpu->arch.sie_block->gcr, 128); 1430 return rc ? -EFAULT : 0; 1431 } 1432 1433 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 1434 { 1435 /* 1436 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 1437 * copying in vcpu load/put. Lets update our copies before we save 1438 * it into the save area 1439 */ 1440 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1441 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 1442 save_access_regs(vcpu->run->s.regs.acrs); 1443 1444 return kvm_s390_store_status_unloaded(vcpu, addr); 1445 } 1446 1447 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 1448 { 1449 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 1450 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu); 1451 exit_sie_sync(vcpu); 1452 } 1453 1454 static void __disable_ibs_on_all_vcpus(struct kvm *kvm) 1455 { 1456 unsigned int i; 1457 struct kvm_vcpu *vcpu; 1458 1459 kvm_for_each_vcpu(i, vcpu, kvm) { 1460 __disable_ibs_on_vcpu(vcpu); 1461 } 1462 } 1463 1464 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 1465 { 1466 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu); 1467 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu); 1468 exit_sie_sync(vcpu); 1469 } 1470 1471 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) 1472 { 1473 int i, online_vcpus, started_vcpus = 0; 1474 1475 if (!is_vcpu_stopped(vcpu)) 1476 return; 1477 1478 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); 1479 /* Only one cpu at a time may enter/leave the STOPPED state. */ 1480 spin_lock(&vcpu->kvm->arch.start_stop_lock); 1481 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 1482 1483 for (i = 0; i < online_vcpus; i++) { 1484 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) 1485 started_vcpus++; 1486 } 1487 1488 if (started_vcpus == 0) { 1489 /* we're the only active VCPU -> speed it up */ 1490 __enable_ibs_on_vcpu(vcpu); 1491 } else if (started_vcpus == 1) { 1492 /* 1493 * As we are starting a second VCPU, we have to disable 1494 * the IBS facility on all VCPUs to remove potentially 1495 * oustanding ENABLE requests. 1496 */ 1497 __disable_ibs_on_all_vcpus(vcpu->kvm); 1498 } 1499 1500 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 1501 /* 1502 * Another VCPU might have used IBS while we were offline. 1503 * Let's play safe and flush the VCPU at startup. 1504 */ 1505 vcpu->arch.sie_block->ihcpu = 0xffff; 1506 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 1507 return; 1508 } 1509 1510 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) 1511 { 1512 int i, online_vcpus, started_vcpus = 0; 1513 struct kvm_vcpu *started_vcpu = NULL; 1514 1515 if (is_vcpu_stopped(vcpu)) 1516 return; 1517 1518 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); 1519 /* Only one cpu at a time may enter/leave the STOPPED state. */ 1520 spin_lock(&vcpu->kvm->arch.start_stop_lock); 1521 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 1522 1523 /* Need to lock access to action_bits to avoid a SIGP race condition */ 1524 spin_lock(&vcpu->arch.local_int.lock); 1525 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 1526 1527 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 1528 vcpu->arch.local_int.action_bits &= 1529 ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP); 1530 spin_unlock(&vcpu->arch.local_int.lock); 1531 1532 __disable_ibs_on_vcpu(vcpu); 1533 1534 for (i = 0; i < online_vcpus; i++) { 1535 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) { 1536 started_vcpus++; 1537 started_vcpu = vcpu->kvm->vcpus[i]; 1538 } 1539 } 1540 1541 if (started_vcpus == 1) { 1542 /* 1543 * As we only have one VCPU left, we want to enable the 1544 * IBS facility for that VCPU to speed it up. 1545 */ 1546 __enable_ibs_on_vcpu(started_vcpu); 1547 } 1548 1549 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 1550 return; 1551 } 1552 1553 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 1554 struct kvm_enable_cap *cap) 1555 { 1556 int r; 1557 1558 if (cap->flags) 1559 return -EINVAL; 1560 1561 switch (cap->cap) { 1562 case KVM_CAP_S390_CSS_SUPPORT: 1563 if (!vcpu->kvm->arch.css_support) { 1564 vcpu->kvm->arch.css_support = 1; 1565 trace_kvm_s390_enable_css(vcpu->kvm); 1566 } 1567 r = 0; 1568 break; 1569 default: 1570 r = -EINVAL; 1571 break; 1572 } 1573 return r; 1574 } 1575 1576 long kvm_arch_vcpu_ioctl(struct file *filp, 1577 unsigned int ioctl, unsigned long arg) 1578 { 1579 struct kvm_vcpu *vcpu = filp->private_data; 1580 void __user *argp = (void __user *)arg; 1581 int idx; 1582 long r; 1583 1584 switch (ioctl) { 1585 case KVM_S390_INTERRUPT: { 1586 struct kvm_s390_interrupt s390int; 1587 1588 r = -EFAULT; 1589 if (copy_from_user(&s390int, argp, sizeof(s390int))) 1590 break; 1591 r = kvm_s390_inject_vcpu(vcpu, &s390int); 1592 break; 1593 } 1594 case KVM_S390_STORE_STATUS: 1595 idx = srcu_read_lock(&vcpu->kvm->srcu); 1596 r = kvm_s390_vcpu_store_status(vcpu, arg); 1597 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1598 break; 1599 case KVM_S390_SET_INITIAL_PSW: { 1600 psw_t psw; 1601 1602 r = -EFAULT; 1603 if (copy_from_user(&psw, argp, sizeof(psw))) 1604 break; 1605 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 1606 break; 1607 } 1608 case KVM_S390_INITIAL_RESET: 1609 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 1610 break; 1611 case KVM_SET_ONE_REG: 1612 case KVM_GET_ONE_REG: { 1613 struct kvm_one_reg reg; 1614 r = -EFAULT; 1615 if (copy_from_user(®, argp, sizeof(reg))) 1616 break; 1617 if (ioctl == KVM_SET_ONE_REG) 1618 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 1619 else 1620 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 1621 break; 1622 } 1623 #ifdef CONFIG_KVM_S390_UCONTROL 1624 case KVM_S390_UCAS_MAP: { 1625 struct kvm_s390_ucas_mapping ucasmap; 1626 1627 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 1628 r = -EFAULT; 1629 break; 1630 } 1631 1632 if (!kvm_is_ucontrol(vcpu->kvm)) { 1633 r = -EINVAL; 1634 break; 1635 } 1636 1637 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 1638 ucasmap.vcpu_addr, ucasmap.length); 1639 break; 1640 } 1641 case KVM_S390_UCAS_UNMAP: { 1642 struct kvm_s390_ucas_mapping ucasmap; 1643 1644 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 1645 r = -EFAULT; 1646 break; 1647 } 1648 1649 if (!kvm_is_ucontrol(vcpu->kvm)) { 1650 r = -EINVAL; 1651 break; 1652 } 1653 1654 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 1655 ucasmap.length); 1656 break; 1657 } 1658 #endif 1659 case KVM_S390_VCPU_FAULT: { 1660 r = gmap_fault(arg, vcpu->arch.gmap); 1661 if (!IS_ERR_VALUE(r)) 1662 r = 0; 1663 break; 1664 } 1665 case KVM_ENABLE_CAP: 1666 { 1667 struct kvm_enable_cap cap; 1668 r = -EFAULT; 1669 if (copy_from_user(&cap, argp, sizeof(cap))) 1670 break; 1671 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1672 break; 1673 } 1674 default: 1675 r = -ENOTTY; 1676 } 1677 return r; 1678 } 1679 1680 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 1681 { 1682 #ifdef CONFIG_KVM_S390_UCONTROL 1683 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 1684 && (kvm_is_ucontrol(vcpu->kvm))) { 1685 vmf->page = virt_to_page(vcpu->arch.sie_block); 1686 get_page(vmf->page); 1687 return 0; 1688 } 1689 #endif 1690 return VM_FAULT_SIGBUS; 1691 } 1692 1693 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 1694 struct kvm_memory_slot *dont) 1695 { 1696 } 1697 1698 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 1699 unsigned long npages) 1700 { 1701 return 0; 1702 } 1703 1704 void kvm_arch_memslots_updated(struct kvm *kvm) 1705 { 1706 } 1707 1708 /* Section: memory related */ 1709 int kvm_arch_prepare_memory_region(struct kvm *kvm, 1710 struct kvm_memory_slot *memslot, 1711 struct kvm_userspace_memory_region *mem, 1712 enum kvm_mr_change change) 1713 { 1714 /* A few sanity checks. We can have memory slots which have to be 1715 located/ended at a segment boundary (1MB). The memory in userland is 1716 ok to be fragmented into various different vmas. It is okay to mmap() 1717 and munmap() stuff in this slot after doing this call at any time */ 1718 1719 if (mem->userspace_addr & 0xffffful) 1720 return -EINVAL; 1721 1722 if (mem->memory_size & 0xffffful) 1723 return -EINVAL; 1724 1725 return 0; 1726 } 1727 1728 void kvm_arch_commit_memory_region(struct kvm *kvm, 1729 struct kvm_userspace_memory_region *mem, 1730 const struct kvm_memory_slot *old, 1731 enum kvm_mr_change change) 1732 { 1733 int rc; 1734 1735 /* If the basics of the memslot do not change, we do not want 1736 * to update the gmap. Every update causes several unnecessary 1737 * segment translation exceptions. This is usually handled just 1738 * fine by the normal fault handler + gmap, but it will also 1739 * cause faults on the prefix page of running guest CPUs. 1740 */ 1741 if (old->userspace_addr == mem->userspace_addr && 1742 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 1743 old->npages * PAGE_SIZE == mem->memory_size) 1744 return; 1745 1746 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 1747 mem->guest_phys_addr, mem->memory_size); 1748 if (rc) 1749 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n"); 1750 return; 1751 } 1752 1753 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1754 { 1755 } 1756 1757 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1758 struct kvm_memory_slot *slot) 1759 { 1760 } 1761 1762 static int __init kvm_s390_init(void) 1763 { 1764 int ret; 1765 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 1766 if (ret) 1767 return ret; 1768 1769 /* 1770 * guests can ask for up to 255+1 double words, we need a full page 1771 * to hold the maximum amount of facilities. On the other hand, we 1772 * only set facilities that are known to work in KVM. 1773 */ 1774 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA); 1775 if (!vfacilities) { 1776 kvm_exit(); 1777 return -ENOMEM; 1778 } 1779 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16); 1780 vfacilities[0] &= 0xff82fff3f4fc2000UL; 1781 vfacilities[1] &= 0x005c000000000000UL; 1782 return 0; 1783 } 1784 1785 static void __exit kvm_s390_exit(void) 1786 { 1787 free_page((unsigned long) vfacilities); 1788 kvm_exit(); 1789 } 1790 1791 module_init(kvm_s390_init); 1792 module_exit(kvm_s390_exit); 1793 1794 /* 1795 * Enable autoloading of the kvm module. 1796 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 1797 * since x86 takes a different approach. 1798 */ 1799 #include <linux/miscdevice.h> 1800 MODULE_ALIAS_MISCDEV(KVM_MINOR); 1801 MODULE_ALIAS("devname:kvm"); 1802