1 /* 2 * hosting zSeries kernel virtual machines 3 * 4 * Copyright IBM Corp. 2008, 2009 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 * Christian Borntraeger <borntraeger@de.ibm.com> 12 * Heiko Carstens <heiko.carstens@de.ibm.com> 13 * Christian Ehrhardt <ehrhardt@de.ibm.com> 14 * Jason J. Herne <jjherne@us.ibm.com> 15 */ 16 17 #include <linux/compiler.h> 18 #include <linux/err.h> 19 #include <linux/fs.h> 20 #include <linux/hrtimer.h> 21 #include <linux/init.h> 22 #include <linux/kvm.h> 23 #include <linux/kvm_host.h> 24 #include <linux/module.h> 25 #include <linux/slab.h> 26 #include <linux/timer.h> 27 #include <asm/asm-offsets.h> 28 #include <asm/lowcore.h> 29 #include <asm/pgtable.h> 30 #include <asm/nmi.h> 31 #include <asm/switch_to.h> 32 #include <asm/facility.h> 33 #include <asm/sclp.h> 34 #include "kvm-s390.h" 35 #include "gaccess.h" 36 37 #define CREATE_TRACE_POINTS 38 #include "trace.h" 39 #include "trace-s390.h" 40 41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 42 43 struct kvm_stats_debugfs_item debugfs_entries[] = { 44 { "userspace_handled", VCPU_STAT(exit_userspace) }, 45 { "exit_null", VCPU_STAT(exit_null) }, 46 { "exit_validity", VCPU_STAT(exit_validity) }, 47 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 48 { "exit_external_request", VCPU_STAT(exit_external_request) }, 49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 50 { "exit_instruction", VCPU_STAT(exit_instruction) }, 51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 53 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 54 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 55 { "instruction_stctl", VCPU_STAT(instruction_stctl) }, 56 { "instruction_stctg", VCPU_STAT(instruction_stctg) }, 57 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 58 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 59 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 60 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 61 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 62 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 63 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 64 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 65 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 66 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 67 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 68 { "instruction_spx", VCPU_STAT(instruction_spx) }, 69 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 70 { "instruction_stap", VCPU_STAT(instruction_stap) }, 71 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 72 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 73 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 74 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 75 { "instruction_essa", VCPU_STAT(instruction_essa) }, 76 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 77 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 78 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 79 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 80 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 81 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 82 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 83 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 84 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 85 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 86 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 87 { "diagnose_10", VCPU_STAT(diagnose_10) }, 88 { "diagnose_44", VCPU_STAT(diagnose_44) }, 89 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 90 { NULL } 91 }; 92 93 unsigned long *vfacilities; 94 static struct gmap_notifier gmap_notifier; 95 96 /* test availability of vfacility */ 97 int test_vfacility(unsigned long nr) 98 { 99 return __test_facility(nr, (void *) vfacilities); 100 } 101 102 /* Section: not file related */ 103 int kvm_arch_hardware_enable(void *garbage) 104 { 105 /* every s390 is virtualization enabled ;-) */ 106 return 0; 107 } 108 109 void kvm_arch_hardware_disable(void *garbage) 110 { 111 } 112 113 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address); 114 115 int kvm_arch_hardware_setup(void) 116 { 117 gmap_notifier.notifier_call = kvm_gmap_notifier; 118 gmap_register_ipte_notifier(&gmap_notifier); 119 return 0; 120 } 121 122 void kvm_arch_hardware_unsetup(void) 123 { 124 gmap_unregister_ipte_notifier(&gmap_notifier); 125 } 126 127 void kvm_arch_check_processor_compat(void *rtn) 128 { 129 } 130 131 int kvm_arch_init(void *opaque) 132 { 133 return 0; 134 } 135 136 void kvm_arch_exit(void) 137 { 138 } 139 140 /* Section: device related */ 141 long kvm_arch_dev_ioctl(struct file *filp, 142 unsigned int ioctl, unsigned long arg) 143 { 144 if (ioctl == KVM_S390_ENABLE_SIE) 145 return s390_enable_sie(); 146 return -EINVAL; 147 } 148 149 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 150 { 151 int r; 152 153 switch (ext) { 154 case KVM_CAP_S390_PSW: 155 case KVM_CAP_S390_GMAP: 156 case KVM_CAP_SYNC_MMU: 157 #ifdef CONFIG_KVM_S390_UCONTROL 158 case KVM_CAP_S390_UCONTROL: 159 #endif 160 case KVM_CAP_ASYNC_PF: 161 case KVM_CAP_SYNC_REGS: 162 case KVM_CAP_ONE_REG: 163 case KVM_CAP_ENABLE_CAP: 164 case KVM_CAP_S390_CSS_SUPPORT: 165 case KVM_CAP_IRQFD: 166 case KVM_CAP_IOEVENTFD: 167 case KVM_CAP_DEVICE_CTRL: 168 case KVM_CAP_ENABLE_CAP_VM: 169 case KVM_CAP_S390_IRQCHIP: 170 case KVM_CAP_VM_ATTRIBUTES: 171 case KVM_CAP_MP_STATE: 172 r = 1; 173 break; 174 case KVM_CAP_NR_VCPUS: 175 case KVM_CAP_MAX_VCPUS: 176 r = KVM_MAX_VCPUS; 177 break; 178 case KVM_CAP_NR_MEMSLOTS: 179 r = KVM_USER_MEM_SLOTS; 180 break; 181 case KVM_CAP_S390_COW: 182 r = MACHINE_HAS_ESOP; 183 break; 184 default: 185 r = 0; 186 } 187 return r; 188 } 189 190 static void kvm_s390_sync_dirty_log(struct kvm *kvm, 191 struct kvm_memory_slot *memslot) 192 { 193 gfn_t cur_gfn, last_gfn; 194 unsigned long address; 195 struct gmap *gmap = kvm->arch.gmap; 196 197 down_read(&gmap->mm->mmap_sem); 198 /* Loop over all guest pages */ 199 last_gfn = memslot->base_gfn + memslot->npages; 200 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { 201 address = gfn_to_hva_memslot(memslot, cur_gfn); 202 203 if (gmap_test_and_clear_dirty(address, gmap)) 204 mark_page_dirty(kvm, cur_gfn); 205 } 206 up_read(&gmap->mm->mmap_sem); 207 } 208 209 /* Section: vm related */ 210 /* 211 * Get (and clear) the dirty memory log for a memory slot. 212 */ 213 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 214 struct kvm_dirty_log *log) 215 { 216 int r; 217 unsigned long n; 218 struct kvm_memory_slot *memslot; 219 int is_dirty = 0; 220 221 mutex_lock(&kvm->slots_lock); 222 223 r = -EINVAL; 224 if (log->slot >= KVM_USER_MEM_SLOTS) 225 goto out; 226 227 memslot = id_to_memslot(kvm->memslots, log->slot); 228 r = -ENOENT; 229 if (!memslot->dirty_bitmap) 230 goto out; 231 232 kvm_s390_sync_dirty_log(kvm, memslot); 233 r = kvm_get_dirty_log(kvm, log, &is_dirty); 234 if (r) 235 goto out; 236 237 /* Clear the dirty log */ 238 if (is_dirty) { 239 n = kvm_dirty_bitmap_bytes(memslot); 240 memset(memslot->dirty_bitmap, 0, n); 241 } 242 r = 0; 243 out: 244 mutex_unlock(&kvm->slots_lock); 245 return r; 246 } 247 248 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) 249 { 250 int r; 251 252 if (cap->flags) 253 return -EINVAL; 254 255 switch (cap->cap) { 256 case KVM_CAP_S390_IRQCHIP: 257 kvm->arch.use_irqchip = 1; 258 r = 0; 259 break; 260 default: 261 r = -EINVAL; 262 break; 263 } 264 return r; 265 } 266 267 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 268 { 269 int ret; 270 unsigned int idx; 271 switch (attr->attr) { 272 case KVM_S390_VM_MEM_ENABLE_CMMA: 273 ret = -EBUSY; 274 mutex_lock(&kvm->lock); 275 if (atomic_read(&kvm->online_vcpus) == 0) { 276 kvm->arch.use_cmma = 1; 277 ret = 0; 278 } 279 mutex_unlock(&kvm->lock); 280 break; 281 case KVM_S390_VM_MEM_CLR_CMMA: 282 mutex_lock(&kvm->lock); 283 idx = srcu_read_lock(&kvm->srcu); 284 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false); 285 srcu_read_unlock(&kvm->srcu, idx); 286 mutex_unlock(&kvm->lock); 287 ret = 0; 288 break; 289 default: 290 ret = -ENXIO; 291 break; 292 } 293 return ret; 294 } 295 296 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 297 { 298 int ret; 299 300 switch (attr->group) { 301 case KVM_S390_VM_MEM_CTRL: 302 ret = kvm_s390_mem_control(kvm, attr); 303 break; 304 default: 305 ret = -ENXIO; 306 break; 307 } 308 309 return ret; 310 } 311 312 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr) 313 { 314 return -ENXIO; 315 } 316 317 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 318 { 319 int ret; 320 321 switch (attr->group) { 322 case KVM_S390_VM_MEM_CTRL: 323 switch (attr->attr) { 324 case KVM_S390_VM_MEM_ENABLE_CMMA: 325 case KVM_S390_VM_MEM_CLR_CMMA: 326 ret = 0; 327 break; 328 default: 329 ret = -ENXIO; 330 break; 331 } 332 break; 333 default: 334 ret = -ENXIO; 335 break; 336 } 337 338 return ret; 339 } 340 341 long kvm_arch_vm_ioctl(struct file *filp, 342 unsigned int ioctl, unsigned long arg) 343 { 344 struct kvm *kvm = filp->private_data; 345 void __user *argp = (void __user *)arg; 346 struct kvm_device_attr attr; 347 int r; 348 349 switch (ioctl) { 350 case KVM_S390_INTERRUPT: { 351 struct kvm_s390_interrupt s390int; 352 353 r = -EFAULT; 354 if (copy_from_user(&s390int, argp, sizeof(s390int))) 355 break; 356 r = kvm_s390_inject_vm(kvm, &s390int); 357 break; 358 } 359 case KVM_ENABLE_CAP: { 360 struct kvm_enable_cap cap; 361 r = -EFAULT; 362 if (copy_from_user(&cap, argp, sizeof(cap))) 363 break; 364 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 365 break; 366 } 367 case KVM_CREATE_IRQCHIP: { 368 struct kvm_irq_routing_entry routing; 369 370 r = -EINVAL; 371 if (kvm->arch.use_irqchip) { 372 /* Set up dummy routing. */ 373 memset(&routing, 0, sizeof(routing)); 374 kvm_set_irq_routing(kvm, &routing, 0, 0); 375 r = 0; 376 } 377 break; 378 } 379 case KVM_SET_DEVICE_ATTR: { 380 r = -EFAULT; 381 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 382 break; 383 r = kvm_s390_vm_set_attr(kvm, &attr); 384 break; 385 } 386 case KVM_GET_DEVICE_ATTR: { 387 r = -EFAULT; 388 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 389 break; 390 r = kvm_s390_vm_get_attr(kvm, &attr); 391 break; 392 } 393 case KVM_HAS_DEVICE_ATTR: { 394 r = -EFAULT; 395 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 396 break; 397 r = kvm_s390_vm_has_attr(kvm, &attr); 398 break; 399 } 400 default: 401 r = -ENOTTY; 402 } 403 404 return r; 405 } 406 407 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 408 { 409 int rc; 410 char debug_name[16]; 411 static unsigned long sca_offset; 412 413 rc = -EINVAL; 414 #ifdef CONFIG_KVM_S390_UCONTROL 415 if (type & ~KVM_VM_S390_UCONTROL) 416 goto out_err; 417 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 418 goto out_err; 419 #else 420 if (type) 421 goto out_err; 422 #endif 423 424 rc = s390_enable_sie(); 425 if (rc) 426 goto out_err; 427 428 rc = -ENOMEM; 429 430 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); 431 if (!kvm->arch.sca) 432 goto out_err; 433 spin_lock(&kvm_lock); 434 sca_offset = (sca_offset + 16) & 0x7f0; 435 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset); 436 spin_unlock(&kvm_lock); 437 438 sprintf(debug_name, "kvm-%u", current->pid); 439 440 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 441 if (!kvm->arch.dbf) 442 goto out_nodbf; 443 444 spin_lock_init(&kvm->arch.float_int.lock); 445 INIT_LIST_HEAD(&kvm->arch.float_int.list); 446 init_waitqueue_head(&kvm->arch.ipte_wq); 447 448 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 449 VM_EVENT(kvm, 3, "%s", "vm created"); 450 451 if (type & KVM_VM_S390_UCONTROL) { 452 kvm->arch.gmap = NULL; 453 } else { 454 kvm->arch.gmap = gmap_alloc(current->mm); 455 if (!kvm->arch.gmap) 456 goto out_nogmap; 457 kvm->arch.gmap->private = kvm; 458 kvm->arch.gmap->pfault_enabled = 0; 459 } 460 461 kvm->arch.css_support = 0; 462 kvm->arch.use_irqchip = 0; 463 464 spin_lock_init(&kvm->arch.start_stop_lock); 465 466 return 0; 467 out_nogmap: 468 debug_unregister(kvm->arch.dbf); 469 out_nodbf: 470 free_page((unsigned long)(kvm->arch.sca)); 471 out_err: 472 return rc; 473 } 474 475 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 476 { 477 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 478 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 479 kvm_s390_clear_local_irqs(vcpu); 480 kvm_clear_async_pf_completion_queue(vcpu); 481 if (!kvm_is_ucontrol(vcpu->kvm)) { 482 clear_bit(63 - vcpu->vcpu_id, 483 (unsigned long *) &vcpu->kvm->arch.sca->mcn); 484 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda == 485 (__u64) vcpu->arch.sie_block) 486 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0; 487 } 488 smp_mb(); 489 490 if (kvm_is_ucontrol(vcpu->kvm)) 491 gmap_free(vcpu->arch.gmap); 492 493 if (kvm_s390_cmma_enabled(vcpu->kvm)) 494 kvm_s390_vcpu_unsetup_cmma(vcpu); 495 free_page((unsigned long)(vcpu->arch.sie_block)); 496 497 kvm_vcpu_uninit(vcpu); 498 kmem_cache_free(kvm_vcpu_cache, vcpu); 499 } 500 501 static void kvm_free_vcpus(struct kvm *kvm) 502 { 503 unsigned int i; 504 struct kvm_vcpu *vcpu; 505 506 kvm_for_each_vcpu(i, vcpu, kvm) 507 kvm_arch_vcpu_destroy(vcpu); 508 509 mutex_lock(&kvm->lock); 510 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 511 kvm->vcpus[i] = NULL; 512 513 atomic_set(&kvm->online_vcpus, 0); 514 mutex_unlock(&kvm->lock); 515 } 516 517 void kvm_arch_sync_events(struct kvm *kvm) 518 { 519 } 520 521 void kvm_arch_destroy_vm(struct kvm *kvm) 522 { 523 kvm_free_vcpus(kvm); 524 free_page((unsigned long)(kvm->arch.sca)); 525 debug_unregister(kvm->arch.dbf); 526 if (!kvm_is_ucontrol(kvm)) 527 gmap_free(kvm->arch.gmap); 528 kvm_s390_destroy_adapters(kvm); 529 kvm_s390_clear_float_irqs(kvm); 530 } 531 532 /* Section: vcpu related */ 533 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 534 { 535 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 536 kvm_clear_async_pf_completion_queue(vcpu); 537 if (kvm_is_ucontrol(vcpu->kvm)) { 538 vcpu->arch.gmap = gmap_alloc(current->mm); 539 if (!vcpu->arch.gmap) 540 return -ENOMEM; 541 vcpu->arch.gmap->private = vcpu->kvm; 542 return 0; 543 } 544 545 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 546 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 547 KVM_SYNC_GPRS | 548 KVM_SYNC_ACRS | 549 KVM_SYNC_CRS; 550 return 0; 551 } 552 553 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 554 { 555 /* Nothing todo */ 556 } 557 558 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 559 { 560 save_fp_ctl(&vcpu->arch.host_fpregs.fpc); 561 save_fp_regs(vcpu->arch.host_fpregs.fprs); 562 save_access_regs(vcpu->arch.host_acrs); 563 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 564 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 565 restore_access_regs(vcpu->run->s.regs.acrs); 566 gmap_enable(vcpu->arch.gmap); 567 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 568 } 569 570 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 571 { 572 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 573 gmap_disable(vcpu->arch.gmap); 574 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 575 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 576 save_access_regs(vcpu->run->s.regs.acrs); 577 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc); 578 restore_fp_regs(vcpu->arch.host_fpregs.fprs); 579 restore_access_regs(vcpu->arch.host_acrs); 580 } 581 582 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 583 { 584 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 585 vcpu->arch.sie_block->gpsw.mask = 0UL; 586 vcpu->arch.sie_block->gpsw.addr = 0UL; 587 kvm_s390_set_prefix(vcpu, 0); 588 vcpu->arch.sie_block->cputm = 0UL; 589 vcpu->arch.sie_block->ckc = 0UL; 590 vcpu->arch.sie_block->todpr = 0; 591 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 592 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 593 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 594 vcpu->arch.guest_fpregs.fpc = 0; 595 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc)); 596 vcpu->arch.sie_block->gbea = 1; 597 vcpu->arch.sie_block->pp = 0; 598 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 599 kvm_clear_async_pf_completion_queue(vcpu); 600 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 601 kvm_s390_vcpu_stop(vcpu); 602 kvm_s390_clear_local_irqs(vcpu); 603 } 604 605 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 606 { 607 return 0; 608 } 609 610 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) 611 { 612 free_page(vcpu->arch.sie_block->cbrlo); 613 vcpu->arch.sie_block->cbrlo = 0; 614 } 615 616 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) 617 { 618 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL); 619 if (!vcpu->arch.sie_block->cbrlo) 620 return -ENOMEM; 621 622 vcpu->arch.sie_block->ecb2 |= 0x80; 623 vcpu->arch.sie_block->ecb2 &= ~0x08; 624 return 0; 625 } 626 627 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 628 { 629 int rc = 0; 630 631 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 632 CPUSTAT_SM | 633 CPUSTAT_STOPPED | 634 CPUSTAT_GED); 635 vcpu->arch.sie_block->ecb = 6; 636 if (test_vfacility(50) && test_vfacility(73)) 637 vcpu->arch.sie_block->ecb |= 0x10; 638 639 vcpu->arch.sie_block->ecb2 = 8; 640 vcpu->arch.sie_block->eca = 0xD1002000U; 641 if (sclp_has_siif()) 642 vcpu->arch.sie_block->eca |= 1; 643 vcpu->arch.sie_block->fac = (int) (long) vfacilities; 644 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE | 645 ICTL_TPROT; 646 647 if (kvm_s390_cmma_enabled(vcpu->kvm)) { 648 rc = kvm_s390_vcpu_setup_cmma(vcpu); 649 if (rc) 650 return rc; 651 } 652 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 653 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 654 get_cpu_id(&vcpu->arch.cpu_id); 655 vcpu->arch.cpu_id.version = 0xff; 656 return rc; 657 } 658 659 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 660 unsigned int id) 661 { 662 struct kvm_vcpu *vcpu; 663 struct sie_page *sie_page; 664 int rc = -EINVAL; 665 666 if (id >= KVM_MAX_VCPUS) 667 goto out; 668 669 rc = -ENOMEM; 670 671 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 672 if (!vcpu) 673 goto out; 674 675 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL); 676 if (!sie_page) 677 goto out_free_cpu; 678 679 vcpu->arch.sie_block = &sie_page->sie_block; 680 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 681 682 vcpu->arch.sie_block->icpua = id; 683 if (!kvm_is_ucontrol(kvm)) { 684 if (!kvm->arch.sca) { 685 WARN_ON_ONCE(1); 686 goto out_free_cpu; 687 } 688 if (!kvm->arch.sca->cpu[id].sda) 689 kvm->arch.sca->cpu[id].sda = 690 (__u64) vcpu->arch.sie_block; 691 vcpu->arch.sie_block->scaoh = 692 (__u32)(((__u64)kvm->arch.sca) >> 32); 693 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 694 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 695 } 696 697 spin_lock_init(&vcpu->arch.local_int.lock); 698 INIT_LIST_HEAD(&vcpu->arch.local_int.list); 699 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 700 vcpu->arch.local_int.wq = &vcpu->wq; 701 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 702 703 rc = kvm_vcpu_init(vcpu, kvm, id); 704 if (rc) 705 goto out_free_sie_block; 706 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, 707 vcpu->arch.sie_block); 708 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 709 710 return vcpu; 711 out_free_sie_block: 712 free_page((unsigned long)(vcpu->arch.sie_block)); 713 out_free_cpu: 714 kmem_cache_free(kvm_vcpu_cache, vcpu); 715 out: 716 return ERR_PTR(rc); 717 } 718 719 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 720 { 721 return kvm_cpu_has_interrupt(vcpu); 722 } 723 724 void s390_vcpu_block(struct kvm_vcpu *vcpu) 725 { 726 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 727 } 728 729 void s390_vcpu_unblock(struct kvm_vcpu *vcpu) 730 { 731 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 732 } 733 734 /* 735 * Kick a guest cpu out of SIE and wait until SIE is not running. 736 * If the CPU is not running (e.g. waiting as idle) the function will 737 * return immediately. */ 738 void exit_sie(struct kvm_vcpu *vcpu) 739 { 740 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 741 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 742 cpu_relax(); 743 } 744 745 /* Kick a guest cpu out of SIE and prevent SIE-reentry */ 746 void exit_sie_sync(struct kvm_vcpu *vcpu) 747 { 748 s390_vcpu_block(vcpu); 749 exit_sie(vcpu); 750 } 751 752 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address) 753 { 754 int i; 755 struct kvm *kvm = gmap->private; 756 struct kvm_vcpu *vcpu; 757 758 kvm_for_each_vcpu(i, vcpu, kvm) { 759 /* match against both prefix pages */ 760 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) { 761 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address); 762 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 763 exit_sie_sync(vcpu); 764 } 765 } 766 } 767 768 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 769 { 770 /* kvm common code refers to this, but never calls it */ 771 BUG(); 772 return 0; 773 } 774 775 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 776 struct kvm_one_reg *reg) 777 { 778 int r = -EINVAL; 779 780 switch (reg->id) { 781 case KVM_REG_S390_TODPR: 782 r = put_user(vcpu->arch.sie_block->todpr, 783 (u32 __user *)reg->addr); 784 break; 785 case KVM_REG_S390_EPOCHDIFF: 786 r = put_user(vcpu->arch.sie_block->epoch, 787 (u64 __user *)reg->addr); 788 break; 789 case KVM_REG_S390_CPU_TIMER: 790 r = put_user(vcpu->arch.sie_block->cputm, 791 (u64 __user *)reg->addr); 792 break; 793 case KVM_REG_S390_CLOCK_COMP: 794 r = put_user(vcpu->arch.sie_block->ckc, 795 (u64 __user *)reg->addr); 796 break; 797 case KVM_REG_S390_PFTOKEN: 798 r = put_user(vcpu->arch.pfault_token, 799 (u64 __user *)reg->addr); 800 break; 801 case KVM_REG_S390_PFCOMPARE: 802 r = put_user(vcpu->arch.pfault_compare, 803 (u64 __user *)reg->addr); 804 break; 805 case KVM_REG_S390_PFSELECT: 806 r = put_user(vcpu->arch.pfault_select, 807 (u64 __user *)reg->addr); 808 break; 809 case KVM_REG_S390_PP: 810 r = put_user(vcpu->arch.sie_block->pp, 811 (u64 __user *)reg->addr); 812 break; 813 case KVM_REG_S390_GBEA: 814 r = put_user(vcpu->arch.sie_block->gbea, 815 (u64 __user *)reg->addr); 816 break; 817 default: 818 break; 819 } 820 821 return r; 822 } 823 824 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 825 struct kvm_one_reg *reg) 826 { 827 int r = -EINVAL; 828 829 switch (reg->id) { 830 case KVM_REG_S390_TODPR: 831 r = get_user(vcpu->arch.sie_block->todpr, 832 (u32 __user *)reg->addr); 833 break; 834 case KVM_REG_S390_EPOCHDIFF: 835 r = get_user(vcpu->arch.sie_block->epoch, 836 (u64 __user *)reg->addr); 837 break; 838 case KVM_REG_S390_CPU_TIMER: 839 r = get_user(vcpu->arch.sie_block->cputm, 840 (u64 __user *)reg->addr); 841 break; 842 case KVM_REG_S390_CLOCK_COMP: 843 r = get_user(vcpu->arch.sie_block->ckc, 844 (u64 __user *)reg->addr); 845 break; 846 case KVM_REG_S390_PFTOKEN: 847 r = get_user(vcpu->arch.pfault_token, 848 (u64 __user *)reg->addr); 849 break; 850 case KVM_REG_S390_PFCOMPARE: 851 r = get_user(vcpu->arch.pfault_compare, 852 (u64 __user *)reg->addr); 853 break; 854 case KVM_REG_S390_PFSELECT: 855 r = get_user(vcpu->arch.pfault_select, 856 (u64 __user *)reg->addr); 857 break; 858 case KVM_REG_S390_PP: 859 r = get_user(vcpu->arch.sie_block->pp, 860 (u64 __user *)reg->addr); 861 break; 862 case KVM_REG_S390_GBEA: 863 r = get_user(vcpu->arch.sie_block->gbea, 864 (u64 __user *)reg->addr); 865 break; 866 default: 867 break; 868 } 869 870 return r; 871 } 872 873 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 874 { 875 kvm_s390_vcpu_initial_reset(vcpu); 876 return 0; 877 } 878 879 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 880 { 881 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 882 return 0; 883 } 884 885 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 886 { 887 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 888 return 0; 889 } 890 891 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 892 struct kvm_sregs *sregs) 893 { 894 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 895 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 896 restore_access_regs(vcpu->run->s.regs.acrs); 897 return 0; 898 } 899 900 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 901 struct kvm_sregs *sregs) 902 { 903 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 904 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 905 return 0; 906 } 907 908 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 909 { 910 if (test_fp_ctl(fpu->fpc)) 911 return -EINVAL; 912 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 913 vcpu->arch.guest_fpregs.fpc = fpu->fpc; 914 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 915 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 916 return 0; 917 } 918 919 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 920 { 921 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs)); 922 fpu->fpc = vcpu->arch.guest_fpregs.fpc; 923 return 0; 924 } 925 926 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 927 { 928 int rc = 0; 929 930 if (!is_vcpu_stopped(vcpu)) 931 rc = -EBUSY; 932 else { 933 vcpu->run->psw_mask = psw.mask; 934 vcpu->run->psw_addr = psw.addr; 935 } 936 return rc; 937 } 938 939 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 940 struct kvm_translation *tr) 941 { 942 return -EINVAL; /* not implemented yet */ 943 } 944 945 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \ 946 KVM_GUESTDBG_USE_HW_BP | \ 947 KVM_GUESTDBG_ENABLE) 948 949 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 950 struct kvm_guest_debug *dbg) 951 { 952 int rc = 0; 953 954 vcpu->guest_debug = 0; 955 kvm_s390_clear_bp_data(vcpu); 956 957 if (dbg->control & ~VALID_GUESTDBG_FLAGS) 958 return -EINVAL; 959 960 if (dbg->control & KVM_GUESTDBG_ENABLE) { 961 vcpu->guest_debug = dbg->control; 962 /* enforce guest PER */ 963 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 964 965 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 966 rc = kvm_s390_import_bp_data(vcpu, dbg); 967 } else { 968 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 969 vcpu->arch.guestdbg.last_bp = 0; 970 } 971 972 if (rc) { 973 vcpu->guest_debug = 0; 974 kvm_s390_clear_bp_data(vcpu); 975 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 976 } 977 978 return rc; 979 } 980 981 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 982 struct kvm_mp_state *mp_state) 983 { 984 /* CHECK_STOP and LOAD are not supported yet */ 985 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED : 986 KVM_MP_STATE_OPERATING; 987 } 988 989 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 990 struct kvm_mp_state *mp_state) 991 { 992 int rc = 0; 993 994 /* user space knows about this interface - let it control the state */ 995 vcpu->kvm->arch.user_cpu_state_ctrl = 1; 996 997 switch (mp_state->mp_state) { 998 case KVM_MP_STATE_STOPPED: 999 kvm_s390_vcpu_stop(vcpu); 1000 break; 1001 case KVM_MP_STATE_OPERATING: 1002 kvm_s390_vcpu_start(vcpu); 1003 break; 1004 case KVM_MP_STATE_LOAD: 1005 case KVM_MP_STATE_CHECK_STOP: 1006 /* fall through - CHECK_STOP and LOAD are not supported yet */ 1007 default: 1008 rc = -ENXIO; 1009 } 1010 1011 return rc; 1012 } 1013 1014 bool kvm_s390_cmma_enabled(struct kvm *kvm) 1015 { 1016 if (!MACHINE_IS_LPAR) 1017 return false; 1018 /* only enable for z10 and later */ 1019 if (!MACHINE_HAS_EDAT1) 1020 return false; 1021 if (!kvm->arch.use_cmma) 1022 return false; 1023 return true; 1024 } 1025 1026 static bool ibs_enabled(struct kvm_vcpu *vcpu) 1027 { 1028 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS; 1029 } 1030 1031 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 1032 { 1033 retry: 1034 s390_vcpu_unblock(vcpu); 1035 /* 1036 * We use MMU_RELOAD just to re-arm the ipte notifier for the 1037 * guest prefix page. gmap_ipte_notify will wait on the ptl lock. 1038 * This ensures that the ipte instruction for this request has 1039 * already finished. We might race against a second unmapper that 1040 * wants to set the blocking bit. Lets just retry the request loop. 1041 */ 1042 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 1043 int rc; 1044 rc = gmap_ipte_notify(vcpu->arch.gmap, 1045 kvm_s390_get_prefix(vcpu), 1046 PAGE_SIZE * 2); 1047 if (rc) 1048 return rc; 1049 goto retry; 1050 } 1051 1052 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 1053 if (!ibs_enabled(vcpu)) { 1054 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 1055 atomic_set_mask(CPUSTAT_IBS, 1056 &vcpu->arch.sie_block->cpuflags); 1057 } 1058 goto retry; 1059 } 1060 1061 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 1062 if (ibs_enabled(vcpu)) { 1063 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 1064 atomic_clear_mask(CPUSTAT_IBS, 1065 &vcpu->arch.sie_block->cpuflags); 1066 } 1067 goto retry; 1068 } 1069 1070 /* nothing to do, just clear the request */ 1071 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 1072 1073 return 0; 1074 } 1075 1076 /** 1077 * kvm_arch_fault_in_page - fault-in guest page if necessary 1078 * @vcpu: The corresponding virtual cpu 1079 * @gpa: Guest physical address 1080 * @writable: Whether the page should be writable or not 1081 * 1082 * Make sure that a guest page has been faulted-in on the host. 1083 * 1084 * Return: Zero on success, negative error code otherwise. 1085 */ 1086 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable) 1087 { 1088 struct mm_struct *mm = current->mm; 1089 hva_t hva; 1090 long rc; 1091 1092 hva = gmap_fault(gpa, vcpu->arch.gmap); 1093 if (IS_ERR_VALUE(hva)) 1094 return (long)hva; 1095 down_read(&mm->mmap_sem); 1096 rc = get_user_pages(current, mm, hva, 1, writable, 0, NULL, NULL); 1097 up_read(&mm->mmap_sem); 1098 1099 return rc < 0 ? rc : 0; 1100 } 1101 1102 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, 1103 unsigned long token) 1104 { 1105 struct kvm_s390_interrupt inti; 1106 inti.parm64 = token; 1107 1108 if (start_token) { 1109 inti.type = KVM_S390_INT_PFAULT_INIT; 1110 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti)); 1111 } else { 1112 inti.type = KVM_S390_INT_PFAULT_DONE; 1113 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); 1114 } 1115 } 1116 1117 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 1118 struct kvm_async_pf *work) 1119 { 1120 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); 1121 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); 1122 } 1123 1124 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 1125 struct kvm_async_pf *work) 1126 { 1127 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token); 1128 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token); 1129 } 1130 1131 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 1132 struct kvm_async_pf *work) 1133 { 1134 /* s390 will always inject the page directly */ 1135 } 1136 1137 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 1138 { 1139 /* 1140 * s390 will always inject the page directly, 1141 * but we still want check_async_completion to cleanup 1142 */ 1143 return true; 1144 } 1145 1146 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) 1147 { 1148 hva_t hva; 1149 struct kvm_arch_async_pf arch; 1150 int rc; 1151 1152 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 1153 return 0; 1154 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) != 1155 vcpu->arch.pfault_compare) 1156 return 0; 1157 if (psw_extint_disabled(vcpu)) 1158 return 0; 1159 if (kvm_cpu_has_interrupt(vcpu)) 1160 return 0; 1161 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 1162 return 0; 1163 if (!vcpu->arch.gmap->pfault_enabled) 1164 return 0; 1165 1166 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr)); 1167 hva += current->thread.gmap_addr & ~PAGE_MASK; 1168 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8)) 1169 return 0; 1170 1171 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch); 1172 return rc; 1173 } 1174 1175 static int vcpu_pre_run(struct kvm_vcpu *vcpu) 1176 { 1177 int rc, cpuflags; 1178 1179 /* 1180 * On s390 notifications for arriving pages will be delivered directly 1181 * to the guest but the house keeping for completed pfaults is 1182 * handled outside the worker. 1183 */ 1184 kvm_check_async_pf_completion(vcpu); 1185 1186 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16); 1187 1188 if (need_resched()) 1189 schedule(); 1190 1191 if (test_cpu_flag(CIF_MCCK_PENDING)) 1192 s390_handle_mcck(); 1193 1194 if (!kvm_is_ucontrol(vcpu->kvm)) 1195 kvm_s390_deliver_pending_interrupts(vcpu); 1196 1197 rc = kvm_s390_handle_requests(vcpu); 1198 if (rc) 1199 return rc; 1200 1201 if (guestdbg_enabled(vcpu)) { 1202 kvm_s390_backup_guest_per_regs(vcpu); 1203 kvm_s390_patch_guest_per_regs(vcpu); 1204 } 1205 1206 vcpu->arch.sie_block->icptcode = 0; 1207 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); 1208 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); 1209 trace_kvm_s390_sie_enter(vcpu, cpuflags); 1210 1211 return 0; 1212 } 1213 1214 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 1215 { 1216 int rc = -1; 1217 1218 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 1219 vcpu->arch.sie_block->icptcode); 1220 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 1221 1222 if (guestdbg_enabled(vcpu)) 1223 kvm_s390_restore_guest_per_regs(vcpu); 1224 1225 if (exit_reason >= 0) { 1226 rc = 0; 1227 } else if (kvm_is_ucontrol(vcpu->kvm)) { 1228 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; 1229 vcpu->run->s390_ucontrol.trans_exc_code = 1230 current->thread.gmap_addr; 1231 vcpu->run->s390_ucontrol.pgm_code = 0x10; 1232 rc = -EREMOTE; 1233 1234 } else if (current->thread.gmap_pfault) { 1235 trace_kvm_s390_major_guest_pfault(vcpu); 1236 current->thread.gmap_pfault = 0; 1237 if (kvm_arch_setup_async_pf(vcpu)) { 1238 rc = 0; 1239 } else { 1240 gpa_t gpa = current->thread.gmap_addr; 1241 rc = kvm_arch_fault_in_page(vcpu, gpa, 1); 1242 } 1243 } 1244 1245 if (rc == -1) { 1246 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 1247 trace_kvm_s390_sie_fault(vcpu); 1248 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1249 } 1250 1251 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 1252 1253 if (rc == 0) { 1254 if (kvm_is_ucontrol(vcpu->kvm)) 1255 /* Don't exit for host interrupts. */ 1256 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0; 1257 else 1258 rc = kvm_handle_sie_intercept(vcpu); 1259 } 1260 1261 return rc; 1262 } 1263 1264 static int __vcpu_run(struct kvm_vcpu *vcpu) 1265 { 1266 int rc, exit_reason; 1267 1268 /* 1269 * We try to hold kvm->srcu during most of vcpu_run (except when run- 1270 * ning the guest), so that memslots (and other stuff) are protected 1271 */ 1272 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1273 1274 do { 1275 rc = vcpu_pre_run(vcpu); 1276 if (rc) 1277 break; 1278 1279 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1280 /* 1281 * As PF_VCPU will be used in fault handler, between 1282 * guest_enter and guest_exit should be no uaccess. 1283 */ 1284 preempt_disable(); 1285 kvm_guest_enter(); 1286 preempt_enable(); 1287 exit_reason = sie64a(vcpu->arch.sie_block, 1288 vcpu->run->s.regs.gprs); 1289 kvm_guest_exit(); 1290 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1291 1292 rc = vcpu_post_run(vcpu, exit_reason); 1293 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc); 1294 1295 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1296 return rc; 1297 } 1298 1299 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 1300 { 1301 int rc; 1302 sigset_t sigsaved; 1303 1304 if (guestdbg_exit_pending(vcpu)) { 1305 kvm_s390_prepare_debug_exit(vcpu); 1306 return 0; 1307 } 1308 1309 if (vcpu->sigset_active) 1310 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 1311 1312 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { 1313 kvm_s390_vcpu_start(vcpu); 1314 } else if (is_vcpu_stopped(vcpu)) { 1315 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n", 1316 vcpu->vcpu_id); 1317 return -EINVAL; 1318 } 1319 1320 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 1321 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 1322 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) { 1323 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX; 1324 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 1325 } 1326 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 1327 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS; 1328 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 1329 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 1330 } 1331 1332 might_fault(); 1333 rc = __vcpu_run(vcpu); 1334 1335 if (signal_pending(current) && !rc) { 1336 kvm_run->exit_reason = KVM_EXIT_INTR; 1337 rc = -EINTR; 1338 } 1339 1340 if (guestdbg_exit_pending(vcpu) && !rc) { 1341 kvm_s390_prepare_debug_exit(vcpu); 1342 rc = 0; 1343 } 1344 1345 if (rc == -EOPNOTSUPP) { 1346 /* intercept cannot be handled in-kernel, prepare kvm-run */ 1347 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC; 1348 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 1349 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 1350 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 1351 rc = 0; 1352 } 1353 1354 if (rc == -EREMOTE) { 1355 /* intercept was handled, but userspace support is needed 1356 * kvm_run has been prepared by the handler */ 1357 rc = 0; 1358 } 1359 1360 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 1361 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 1362 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); 1363 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 1364 1365 if (vcpu->sigset_active) 1366 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 1367 1368 vcpu->stat.exit_userspace++; 1369 return rc; 1370 } 1371 1372 /* 1373 * store status at address 1374 * we use have two special cases: 1375 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 1376 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 1377 */ 1378 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa) 1379 { 1380 unsigned char archmode = 1; 1381 unsigned int px; 1382 u64 clkcomp; 1383 int rc; 1384 1385 if (gpa == KVM_S390_STORE_STATUS_NOADDR) { 1386 if (write_guest_abs(vcpu, 163, &archmode, 1)) 1387 return -EFAULT; 1388 gpa = SAVE_AREA_BASE; 1389 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) { 1390 if (write_guest_real(vcpu, 163, &archmode, 1)) 1391 return -EFAULT; 1392 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE); 1393 } 1394 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs), 1395 vcpu->arch.guest_fpregs.fprs, 128); 1396 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs), 1397 vcpu->run->s.regs.gprs, 128); 1398 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw), 1399 &vcpu->arch.sie_block->gpsw, 16); 1400 px = kvm_s390_get_prefix(vcpu); 1401 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg), 1402 &px, 4); 1403 rc |= write_guest_abs(vcpu, 1404 gpa + offsetof(struct save_area, fp_ctrl_reg), 1405 &vcpu->arch.guest_fpregs.fpc, 4); 1406 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg), 1407 &vcpu->arch.sie_block->todpr, 4); 1408 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer), 1409 &vcpu->arch.sie_block->cputm, 8); 1410 clkcomp = vcpu->arch.sie_block->ckc >> 8; 1411 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp), 1412 &clkcomp, 8); 1413 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs), 1414 &vcpu->run->s.regs.acrs, 64); 1415 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs), 1416 &vcpu->arch.sie_block->gcr, 128); 1417 return rc ? -EFAULT : 0; 1418 } 1419 1420 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 1421 { 1422 /* 1423 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 1424 * copying in vcpu load/put. Lets update our copies before we save 1425 * it into the save area 1426 */ 1427 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1428 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 1429 save_access_regs(vcpu->run->s.regs.acrs); 1430 1431 return kvm_s390_store_status_unloaded(vcpu, addr); 1432 } 1433 1434 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 1435 { 1436 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 1437 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu); 1438 exit_sie_sync(vcpu); 1439 } 1440 1441 static void __disable_ibs_on_all_vcpus(struct kvm *kvm) 1442 { 1443 unsigned int i; 1444 struct kvm_vcpu *vcpu; 1445 1446 kvm_for_each_vcpu(i, vcpu, kvm) { 1447 __disable_ibs_on_vcpu(vcpu); 1448 } 1449 } 1450 1451 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 1452 { 1453 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu); 1454 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu); 1455 exit_sie_sync(vcpu); 1456 } 1457 1458 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) 1459 { 1460 int i, online_vcpus, started_vcpus = 0; 1461 1462 if (!is_vcpu_stopped(vcpu)) 1463 return; 1464 1465 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); 1466 /* Only one cpu at a time may enter/leave the STOPPED state. */ 1467 spin_lock(&vcpu->kvm->arch.start_stop_lock); 1468 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 1469 1470 for (i = 0; i < online_vcpus; i++) { 1471 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) 1472 started_vcpus++; 1473 } 1474 1475 if (started_vcpus == 0) { 1476 /* we're the only active VCPU -> speed it up */ 1477 __enable_ibs_on_vcpu(vcpu); 1478 } else if (started_vcpus == 1) { 1479 /* 1480 * As we are starting a second VCPU, we have to disable 1481 * the IBS facility on all VCPUs to remove potentially 1482 * oustanding ENABLE requests. 1483 */ 1484 __disable_ibs_on_all_vcpus(vcpu->kvm); 1485 } 1486 1487 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 1488 /* 1489 * Another VCPU might have used IBS while we were offline. 1490 * Let's play safe and flush the VCPU at startup. 1491 */ 1492 vcpu->arch.sie_block->ihcpu = 0xffff; 1493 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 1494 return; 1495 } 1496 1497 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) 1498 { 1499 int i, online_vcpus, started_vcpus = 0; 1500 struct kvm_vcpu *started_vcpu = NULL; 1501 1502 if (is_vcpu_stopped(vcpu)) 1503 return; 1504 1505 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); 1506 /* Only one cpu at a time may enter/leave the STOPPED state. */ 1507 spin_lock(&vcpu->kvm->arch.start_stop_lock); 1508 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 1509 1510 /* Need to lock access to action_bits to avoid a SIGP race condition */ 1511 spin_lock(&vcpu->arch.local_int.lock); 1512 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 1513 1514 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 1515 vcpu->arch.local_int.action_bits &= 1516 ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP); 1517 spin_unlock(&vcpu->arch.local_int.lock); 1518 1519 __disable_ibs_on_vcpu(vcpu); 1520 1521 for (i = 0; i < online_vcpus; i++) { 1522 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) { 1523 started_vcpus++; 1524 started_vcpu = vcpu->kvm->vcpus[i]; 1525 } 1526 } 1527 1528 if (started_vcpus == 1) { 1529 /* 1530 * As we only have one VCPU left, we want to enable the 1531 * IBS facility for that VCPU to speed it up. 1532 */ 1533 __enable_ibs_on_vcpu(started_vcpu); 1534 } 1535 1536 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 1537 return; 1538 } 1539 1540 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 1541 struct kvm_enable_cap *cap) 1542 { 1543 int r; 1544 1545 if (cap->flags) 1546 return -EINVAL; 1547 1548 switch (cap->cap) { 1549 case KVM_CAP_S390_CSS_SUPPORT: 1550 if (!vcpu->kvm->arch.css_support) { 1551 vcpu->kvm->arch.css_support = 1; 1552 trace_kvm_s390_enable_css(vcpu->kvm); 1553 } 1554 r = 0; 1555 break; 1556 default: 1557 r = -EINVAL; 1558 break; 1559 } 1560 return r; 1561 } 1562 1563 long kvm_arch_vcpu_ioctl(struct file *filp, 1564 unsigned int ioctl, unsigned long arg) 1565 { 1566 struct kvm_vcpu *vcpu = filp->private_data; 1567 void __user *argp = (void __user *)arg; 1568 int idx; 1569 long r; 1570 1571 switch (ioctl) { 1572 case KVM_S390_INTERRUPT: { 1573 struct kvm_s390_interrupt s390int; 1574 1575 r = -EFAULT; 1576 if (copy_from_user(&s390int, argp, sizeof(s390int))) 1577 break; 1578 r = kvm_s390_inject_vcpu(vcpu, &s390int); 1579 break; 1580 } 1581 case KVM_S390_STORE_STATUS: 1582 idx = srcu_read_lock(&vcpu->kvm->srcu); 1583 r = kvm_s390_vcpu_store_status(vcpu, arg); 1584 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1585 break; 1586 case KVM_S390_SET_INITIAL_PSW: { 1587 psw_t psw; 1588 1589 r = -EFAULT; 1590 if (copy_from_user(&psw, argp, sizeof(psw))) 1591 break; 1592 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 1593 break; 1594 } 1595 case KVM_S390_INITIAL_RESET: 1596 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 1597 break; 1598 case KVM_SET_ONE_REG: 1599 case KVM_GET_ONE_REG: { 1600 struct kvm_one_reg reg; 1601 r = -EFAULT; 1602 if (copy_from_user(®, argp, sizeof(reg))) 1603 break; 1604 if (ioctl == KVM_SET_ONE_REG) 1605 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 1606 else 1607 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 1608 break; 1609 } 1610 #ifdef CONFIG_KVM_S390_UCONTROL 1611 case KVM_S390_UCAS_MAP: { 1612 struct kvm_s390_ucas_mapping ucasmap; 1613 1614 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 1615 r = -EFAULT; 1616 break; 1617 } 1618 1619 if (!kvm_is_ucontrol(vcpu->kvm)) { 1620 r = -EINVAL; 1621 break; 1622 } 1623 1624 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 1625 ucasmap.vcpu_addr, ucasmap.length); 1626 break; 1627 } 1628 case KVM_S390_UCAS_UNMAP: { 1629 struct kvm_s390_ucas_mapping ucasmap; 1630 1631 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 1632 r = -EFAULT; 1633 break; 1634 } 1635 1636 if (!kvm_is_ucontrol(vcpu->kvm)) { 1637 r = -EINVAL; 1638 break; 1639 } 1640 1641 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 1642 ucasmap.length); 1643 break; 1644 } 1645 #endif 1646 case KVM_S390_VCPU_FAULT: { 1647 r = gmap_fault(arg, vcpu->arch.gmap); 1648 if (!IS_ERR_VALUE(r)) 1649 r = 0; 1650 break; 1651 } 1652 case KVM_ENABLE_CAP: 1653 { 1654 struct kvm_enable_cap cap; 1655 r = -EFAULT; 1656 if (copy_from_user(&cap, argp, sizeof(cap))) 1657 break; 1658 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1659 break; 1660 } 1661 default: 1662 r = -ENOTTY; 1663 } 1664 return r; 1665 } 1666 1667 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 1668 { 1669 #ifdef CONFIG_KVM_S390_UCONTROL 1670 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 1671 && (kvm_is_ucontrol(vcpu->kvm))) { 1672 vmf->page = virt_to_page(vcpu->arch.sie_block); 1673 get_page(vmf->page); 1674 return 0; 1675 } 1676 #endif 1677 return VM_FAULT_SIGBUS; 1678 } 1679 1680 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 1681 struct kvm_memory_slot *dont) 1682 { 1683 } 1684 1685 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 1686 unsigned long npages) 1687 { 1688 return 0; 1689 } 1690 1691 void kvm_arch_memslots_updated(struct kvm *kvm) 1692 { 1693 } 1694 1695 /* Section: memory related */ 1696 int kvm_arch_prepare_memory_region(struct kvm *kvm, 1697 struct kvm_memory_slot *memslot, 1698 struct kvm_userspace_memory_region *mem, 1699 enum kvm_mr_change change) 1700 { 1701 /* A few sanity checks. We can have memory slots which have to be 1702 located/ended at a segment boundary (1MB). The memory in userland is 1703 ok to be fragmented into various different vmas. It is okay to mmap() 1704 and munmap() stuff in this slot after doing this call at any time */ 1705 1706 if (mem->userspace_addr & 0xffffful) 1707 return -EINVAL; 1708 1709 if (mem->memory_size & 0xffffful) 1710 return -EINVAL; 1711 1712 return 0; 1713 } 1714 1715 void kvm_arch_commit_memory_region(struct kvm *kvm, 1716 struct kvm_userspace_memory_region *mem, 1717 const struct kvm_memory_slot *old, 1718 enum kvm_mr_change change) 1719 { 1720 int rc; 1721 1722 /* If the basics of the memslot do not change, we do not want 1723 * to update the gmap. Every update causes several unnecessary 1724 * segment translation exceptions. This is usually handled just 1725 * fine by the normal fault handler + gmap, but it will also 1726 * cause faults on the prefix page of running guest CPUs. 1727 */ 1728 if (old->userspace_addr == mem->userspace_addr && 1729 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 1730 old->npages * PAGE_SIZE == mem->memory_size) 1731 return; 1732 1733 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 1734 mem->guest_phys_addr, mem->memory_size); 1735 if (rc) 1736 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n"); 1737 return; 1738 } 1739 1740 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1741 { 1742 } 1743 1744 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1745 struct kvm_memory_slot *slot) 1746 { 1747 } 1748 1749 static int __init kvm_s390_init(void) 1750 { 1751 int ret; 1752 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 1753 if (ret) 1754 return ret; 1755 1756 /* 1757 * guests can ask for up to 255+1 double words, we need a full page 1758 * to hold the maximum amount of facilities. On the other hand, we 1759 * only set facilities that are known to work in KVM. 1760 */ 1761 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA); 1762 if (!vfacilities) { 1763 kvm_exit(); 1764 return -ENOMEM; 1765 } 1766 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16); 1767 vfacilities[0] &= 0xff82fff3f4fc2000UL; 1768 vfacilities[1] &= 0x005c000000000000UL; 1769 return 0; 1770 } 1771 1772 static void __exit kvm_s390_exit(void) 1773 { 1774 free_page((unsigned long) vfacilities); 1775 kvm_exit(); 1776 } 1777 1778 module_init(kvm_s390_init); 1779 module_exit(kvm_s390_exit); 1780 1781 /* 1782 * Enable autoloading of the kvm module. 1783 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 1784 * since x86 takes a different approach. 1785 */ 1786 #include <linux/miscdevice.h> 1787 MODULE_ALIAS_MISCDEV(KVM_MINOR); 1788 MODULE_ALIAS("devname:kvm"); 1789