1 /* 2 * hosting zSeries kernel virtual machines 3 * 4 * Copyright IBM Corp. 2008, 2009 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 * Christian Borntraeger <borntraeger@de.ibm.com> 12 * Heiko Carstens <heiko.carstens@de.ibm.com> 13 * Christian Ehrhardt <ehrhardt@de.ibm.com> 14 * Jason J. Herne <jjherne@us.ibm.com> 15 */ 16 17 #include <linux/compiler.h> 18 #include <linux/err.h> 19 #include <linux/fs.h> 20 #include <linux/hrtimer.h> 21 #include <linux/init.h> 22 #include <linux/kvm.h> 23 #include <linux/kvm_host.h> 24 #include <linux/module.h> 25 #include <linux/random.h> 26 #include <linux/slab.h> 27 #include <linux/timer.h> 28 #include <asm/asm-offsets.h> 29 #include <asm/lowcore.h> 30 #include <asm/pgtable.h> 31 #include <asm/nmi.h> 32 #include <asm/switch_to.h> 33 #include <asm/sclp.h> 34 #include "kvm-s390.h" 35 #include "gaccess.h" 36 37 #define CREATE_TRACE_POINTS 38 #include "trace.h" 39 #include "trace-s390.h" 40 41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 42 43 struct kvm_stats_debugfs_item debugfs_entries[] = { 44 { "userspace_handled", VCPU_STAT(exit_userspace) }, 45 { "exit_null", VCPU_STAT(exit_null) }, 46 { "exit_validity", VCPU_STAT(exit_validity) }, 47 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 48 { "exit_external_request", VCPU_STAT(exit_external_request) }, 49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 50 { "exit_instruction", VCPU_STAT(exit_instruction) }, 51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 53 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, 54 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 55 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 56 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 57 { "instruction_stctl", VCPU_STAT(instruction_stctl) }, 58 { "instruction_stctg", VCPU_STAT(instruction_stctg) }, 59 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 60 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 61 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 62 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 63 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 64 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 65 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 66 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 67 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 68 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 69 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 70 { "instruction_spx", VCPU_STAT(instruction_spx) }, 71 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 72 { "instruction_stap", VCPU_STAT(instruction_stap) }, 73 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 74 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 75 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 76 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 77 { "instruction_essa", VCPU_STAT(instruction_essa) }, 78 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 79 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 80 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 81 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 82 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 83 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 84 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 85 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) }, 86 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) }, 87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, 89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, 90 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 91 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 92 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 93 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) }, 94 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) }, 95 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) }, 96 { "diagnose_10", VCPU_STAT(diagnose_10) }, 97 { "diagnose_44", VCPU_STAT(diagnose_44) }, 98 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 99 { NULL } 100 }; 101 102 /* upper facilities limit for kvm */ 103 unsigned long kvm_s390_fac_list_mask[] = { 104 0xff82fffbf4fc2000UL, 105 0x005c000000000000UL, 106 }; 107 108 unsigned long kvm_s390_fac_list_mask_size(void) 109 { 110 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64); 111 return ARRAY_SIZE(kvm_s390_fac_list_mask); 112 } 113 114 static struct gmap_notifier gmap_notifier; 115 116 /* Section: not file related */ 117 int kvm_arch_hardware_enable(void) 118 { 119 /* every s390 is virtualization enabled ;-) */ 120 return 0; 121 } 122 123 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address); 124 125 int kvm_arch_hardware_setup(void) 126 { 127 gmap_notifier.notifier_call = kvm_gmap_notifier; 128 gmap_register_ipte_notifier(&gmap_notifier); 129 return 0; 130 } 131 132 void kvm_arch_hardware_unsetup(void) 133 { 134 gmap_unregister_ipte_notifier(&gmap_notifier); 135 } 136 137 int kvm_arch_init(void *opaque) 138 { 139 /* Register floating interrupt controller interface. */ 140 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC); 141 } 142 143 /* Section: device related */ 144 long kvm_arch_dev_ioctl(struct file *filp, 145 unsigned int ioctl, unsigned long arg) 146 { 147 if (ioctl == KVM_S390_ENABLE_SIE) 148 return s390_enable_sie(); 149 return -EINVAL; 150 } 151 152 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 153 { 154 int r; 155 156 switch (ext) { 157 case KVM_CAP_S390_PSW: 158 case KVM_CAP_S390_GMAP: 159 case KVM_CAP_SYNC_MMU: 160 #ifdef CONFIG_KVM_S390_UCONTROL 161 case KVM_CAP_S390_UCONTROL: 162 #endif 163 case KVM_CAP_ASYNC_PF: 164 case KVM_CAP_SYNC_REGS: 165 case KVM_CAP_ONE_REG: 166 case KVM_CAP_ENABLE_CAP: 167 case KVM_CAP_S390_CSS_SUPPORT: 168 case KVM_CAP_IOEVENTFD: 169 case KVM_CAP_DEVICE_CTRL: 170 case KVM_CAP_ENABLE_CAP_VM: 171 case KVM_CAP_S390_IRQCHIP: 172 case KVM_CAP_VM_ATTRIBUTES: 173 case KVM_CAP_MP_STATE: 174 case KVM_CAP_S390_USER_SIGP: 175 r = 1; 176 break; 177 case KVM_CAP_NR_VCPUS: 178 case KVM_CAP_MAX_VCPUS: 179 r = KVM_MAX_VCPUS; 180 break; 181 case KVM_CAP_NR_MEMSLOTS: 182 r = KVM_USER_MEM_SLOTS; 183 break; 184 case KVM_CAP_S390_COW: 185 r = MACHINE_HAS_ESOP; 186 break; 187 default: 188 r = 0; 189 } 190 return r; 191 } 192 193 static void kvm_s390_sync_dirty_log(struct kvm *kvm, 194 struct kvm_memory_slot *memslot) 195 { 196 gfn_t cur_gfn, last_gfn; 197 unsigned long address; 198 struct gmap *gmap = kvm->arch.gmap; 199 200 down_read(&gmap->mm->mmap_sem); 201 /* Loop over all guest pages */ 202 last_gfn = memslot->base_gfn + memslot->npages; 203 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { 204 address = gfn_to_hva_memslot(memslot, cur_gfn); 205 206 if (gmap_test_and_clear_dirty(address, gmap)) 207 mark_page_dirty(kvm, cur_gfn); 208 } 209 up_read(&gmap->mm->mmap_sem); 210 } 211 212 /* Section: vm related */ 213 /* 214 * Get (and clear) the dirty memory log for a memory slot. 215 */ 216 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 217 struct kvm_dirty_log *log) 218 { 219 int r; 220 unsigned long n; 221 struct kvm_memory_slot *memslot; 222 int is_dirty = 0; 223 224 mutex_lock(&kvm->slots_lock); 225 226 r = -EINVAL; 227 if (log->slot >= KVM_USER_MEM_SLOTS) 228 goto out; 229 230 memslot = id_to_memslot(kvm->memslots, log->slot); 231 r = -ENOENT; 232 if (!memslot->dirty_bitmap) 233 goto out; 234 235 kvm_s390_sync_dirty_log(kvm, memslot); 236 r = kvm_get_dirty_log(kvm, log, &is_dirty); 237 if (r) 238 goto out; 239 240 /* Clear the dirty log */ 241 if (is_dirty) { 242 n = kvm_dirty_bitmap_bytes(memslot); 243 memset(memslot->dirty_bitmap, 0, n); 244 } 245 r = 0; 246 out: 247 mutex_unlock(&kvm->slots_lock); 248 return r; 249 } 250 251 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) 252 { 253 int r; 254 255 if (cap->flags) 256 return -EINVAL; 257 258 switch (cap->cap) { 259 case KVM_CAP_S390_IRQCHIP: 260 kvm->arch.use_irqchip = 1; 261 r = 0; 262 break; 263 case KVM_CAP_S390_USER_SIGP: 264 kvm->arch.user_sigp = 1; 265 r = 0; 266 break; 267 default: 268 r = -EINVAL; 269 break; 270 } 271 return r; 272 } 273 274 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 275 { 276 int ret; 277 278 switch (attr->attr) { 279 case KVM_S390_VM_MEM_LIMIT_SIZE: 280 ret = 0; 281 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr)) 282 ret = -EFAULT; 283 break; 284 default: 285 ret = -ENXIO; 286 break; 287 } 288 return ret; 289 } 290 291 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 292 { 293 int ret; 294 unsigned int idx; 295 switch (attr->attr) { 296 case KVM_S390_VM_MEM_ENABLE_CMMA: 297 ret = -EBUSY; 298 mutex_lock(&kvm->lock); 299 if (atomic_read(&kvm->online_vcpus) == 0) { 300 kvm->arch.use_cmma = 1; 301 ret = 0; 302 } 303 mutex_unlock(&kvm->lock); 304 break; 305 case KVM_S390_VM_MEM_CLR_CMMA: 306 mutex_lock(&kvm->lock); 307 idx = srcu_read_lock(&kvm->srcu); 308 s390_reset_cmma(kvm->arch.gmap->mm); 309 srcu_read_unlock(&kvm->srcu, idx); 310 mutex_unlock(&kvm->lock); 311 ret = 0; 312 break; 313 case KVM_S390_VM_MEM_LIMIT_SIZE: { 314 unsigned long new_limit; 315 316 if (kvm_is_ucontrol(kvm)) 317 return -EINVAL; 318 319 if (get_user(new_limit, (u64 __user *)attr->addr)) 320 return -EFAULT; 321 322 if (new_limit > kvm->arch.gmap->asce_end) 323 return -E2BIG; 324 325 ret = -EBUSY; 326 mutex_lock(&kvm->lock); 327 if (atomic_read(&kvm->online_vcpus) == 0) { 328 /* gmap_alloc will round the limit up */ 329 struct gmap *new = gmap_alloc(current->mm, new_limit); 330 331 if (!new) { 332 ret = -ENOMEM; 333 } else { 334 gmap_free(kvm->arch.gmap); 335 new->private = kvm; 336 kvm->arch.gmap = new; 337 ret = 0; 338 } 339 } 340 mutex_unlock(&kvm->lock); 341 break; 342 } 343 default: 344 ret = -ENXIO; 345 break; 346 } 347 return ret; 348 } 349 350 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu); 351 352 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr) 353 { 354 struct kvm_vcpu *vcpu; 355 int i; 356 357 if (!test_kvm_facility(kvm, 76)) 358 return -EINVAL; 359 360 mutex_lock(&kvm->lock); 361 switch (attr->attr) { 362 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 363 get_random_bytes( 364 kvm->arch.crypto.crycb->aes_wrapping_key_mask, 365 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 366 kvm->arch.crypto.aes_kw = 1; 367 break; 368 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 369 get_random_bytes( 370 kvm->arch.crypto.crycb->dea_wrapping_key_mask, 371 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 372 kvm->arch.crypto.dea_kw = 1; 373 break; 374 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 375 kvm->arch.crypto.aes_kw = 0; 376 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0, 377 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 378 break; 379 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 380 kvm->arch.crypto.dea_kw = 0; 381 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0, 382 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 383 break; 384 default: 385 mutex_unlock(&kvm->lock); 386 return -ENXIO; 387 } 388 389 kvm_for_each_vcpu(i, vcpu, kvm) { 390 kvm_s390_vcpu_crypto_setup(vcpu); 391 exit_sie(vcpu); 392 } 393 mutex_unlock(&kvm->lock); 394 return 0; 395 } 396 397 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 398 { 399 u8 gtod_high; 400 401 if (copy_from_user(>od_high, (void __user *)attr->addr, 402 sizeof(gtod_high))) 403 return -EFAULT; 404 405 if (gtod_high != 0) 406 return -EINVAL; 407 408 return 0; 409 } 410 411 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 412 { 413 struct kvm_vcpu *cur_vcpu; 414 unsigned int vcpu_idx; 415 u64 host_tod, gtod; 416 int r; 417 418 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod))) 419 return -EFAULT; 420 421 r = store_tod_clock(&host_tod); 422 if (r) 423 return r; 424 425 mutex_lock(&kvm->lock); 426 kvm->arch.epoch = gtod - host_tod; 427 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) { 428 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch; 429 exit_sie(cur_vcpu); 430 } 431 mutex_unlock(&kvm->lock); 432 return 0; 433 } 434 435 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr) 436 { 437 int ret; 438 439 if (attr->flags) 440 return -EINVAL; 441 442 switch (attr->attr) { 443 case KVM_S390_VM_TOD_HIGH: 444 ret = kvm_s390_set_tod_high(kvm, attr); 445 break; 446 case KVM_S390_VM_TOD_LOW: 447 ret = kvm_s390_set_tod_low(kvm, attr); 448 break; 449 default: 450 ret = -ENXIO; 451 break; 452 } 453 return ret; 454 } 455 456 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 457 { 458 u8 gtod_high = 0; 459 460 if (copy_to_user((void __user *)attr->addr, >od_high, 461 sizeof(gtod_high))) 462 return -EFAULT; 463 464 return 0; 465 } 466 467 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 468 { 469 u64 host_tod, gtod; 470 int r; 471 472 r = store_tod_clock(&host_tod); 473 if (r) 474 return r; 475 476 gtod = host_tod + kvm->arch.epoch; 477 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod))) 478 return -EFAULT; 479 480 return 0; 481 } 482 483 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr) 484 { 485 int ret; 486 487 if (attr->flags) 488 return -EINVAL; 489 490 switch (attr->attr) { 491 case KVM_S390_VM_TOD_HIGH: 492 ret = kvm_s390_get_tod_high(kvm, attr); 493 break; 494 case KVM_S390_VM_TOD_LOW: 495 ret = kvm_s390_get_tod_low(kvm, attr); 496 break; 497 default: 498 ret = -ENXIO; 499 break; 500 } 501 return ret; 502 } 503 504 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) 505 { 506 struct kvm_s390_vm_cpu_processor *proc; 507 int ret = 0; 508 509 mutex_lock(&kvm->lock); 510 if (atomic_read(&kvm->online_vcpus)) { 511 ret = -EBUSY; 512 goto out; 513 } 514 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 515 if (!proc) { 516 ret = -ENOMEM; 517 goto out; 518 } 519 if (!copy_from_user(proc, (void __user *)attr->addr, 520 sizeof(*proc))) { 521 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid, 522 sizeof(struct cpuid)); 523 kvm->arch.model.ibc = proc->ibc; 524 memcpy(kvm->arch.model.fac->list, proc->fac_list, 525 S390_ARCH_FAC_LIST_SIZE_BYTE); 526 } else 527 ret = -EFAULT; 528 kfree(proc); 529 out: 530 mutex_unlock(&kvm->lock); 531 return ret; 532 } 533 534 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 535 { 536 int ret = -ENXIO; 537 538 switch (attr->attr) { 539 case KVM_S390_VM_CPU_PROCESSOR: 540 ret = kvm_s390_set_processor(kvm, attr); 541 break; 542 } 543 return ret; 544 } 545 546 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) 547 { 548 struct kvm_s390_vm_cpu_processor *proc; 549 int ret = 0; 550 551 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 552 if (!proc) { 553 ret = -ENOMEM; 554 goto out; 555 } 556 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid)); 557 proc->ibc = kvm->arch.model.ibc; 558 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE); 559 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc))) 560 ret = -EFAULT; 561 kfree(proc); 562 out: 563 return ret; 564 } 565 566 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) 567 { 568 struct kvm_s390_vm_cpu_machine *mach; 569 int ret = 0; 570 571 mach = kzalloc(sizeof(*mach), GFP_KERNEL); 572 if (!mach) { 573 ret = -ENOMEM; 574 goto out; 575 } 576 get_cpu_id((struct cpuid *) &mach->cpuid); 577 mach->ibc = sclp_get_ibc(); 578 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask, 579 S390_ARCH_FAC_LIST_SIZE_BYTE); 580 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list, 581 S390_ARCH_FAC_LIST_SIZE_BYTE); 582 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach))) 583 ret = -EFAULT; 584 kfree(mach); 585 out: 586 return ret; 587 } 588 589 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 590 { 591 int ret = -ENXIO; 592 593 switch (attr->attr) { 594 case KVM_S390_VM_CPU_PROCESSOR: 595 ret = kvm_s390_get_processor(kvm, attr); 596 break; 597 case KVM_S390_VM_CPU_MACHINE: 598 ret = kvm_s390_get_machine(kvm, attr); 599 break; 600 } 601 return ret; 602 } 603 604 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 605 { 606 int ret; 607 608 switch (attr->group) { 609 case KVM_S390_VM_MEM_CTRL: 610 ret = kvm_s390_set_mem_control(kvm, attr); 611 break; 612 case KVM_S390_VM_TOD: 613 ret = kvm_s390_set_tod(kvm, attr); 614 break; 615 case KVM_S390_VM_CPU_MODEL: 616 ret = kvm_s390_set_cpu_model(kvm, attr); 617 break; 618 case KVM_S390_VM_CRYPTO: 619 ret = kvm_s390_vm_set_crypto(kvm, attr); 620 break; 621 default: 622 ret = -ENXIO; 623 break; 624 } 625 626 return ret; 627 } 628 629 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr) 630 { 631 int ret; 632 633 switch (attr->group) { 634 case KVM_S390_VM_MEM_CTRL: 635 ret = kvm_s390_get_mem_control(kvm, attr); 636 break; 637 case KVM_S390_VM_TOD: 638 ret = kvm_s390_get_tod(kvm, attr); 639 break; 640 case KVM_S390_VM_CPU_MODEL: 641 ret = kvm_s390_get_cpu_model(kvm, attr); 642 break; 643 default: 644 ret = -ENXIO; 645 break; 646 } 647 648 return ret; 649 } 650 651 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 652 { 653 int ret; 654 655 switch (attr->group) { 656 case KVM_S390_VM_MEM_CTRL: 657 switch (attr->attr) { 658 case KVM_S390_VM_MEM_ENABLE_CMMA: 659 case KVM_S390_VM_MEM_CLR_CMMA: 660 case KVM_S390_VM_MEM_LIMIT_SIZE: 661 ret = 0; 662 break; 663 default: 664 ret = -ENXIO; 665 break; 666 } 667 break; 668 case KVM_S390_VM_TOD: 669 switch (attr->attr) { 670 case KVM_S390_VM_TOD_LOW: 671 case KVM_S390_VM_TOD_HIGH: 672 ret = 0; 673 break; 674 default: 675 ret = -ENXIO; 676 break; 677 } 678 break; 679 case KVM_S390_VM_CPU_MODEL: 680 switch (attr->attr) { 681 case KVM_S390_VM_CPU_PROCESSOR: 682 case KVM_S390_VM_CPU_MACHINE: 683 ret = 0; 684 break; 685 default: 686 ret = -ENXIO; 687 break; 688 } 689 break; 690 case KVM_S390_VM_CRYPTO: 691 switch (attr->attr) { 692 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 693 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 694 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 695 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 696 ret = 0; 697 break; 698 default: 699 ret = -ENXIO; 700 break; 701 } 702 break; 703 default: 704 ret = -ENXIO; 705 break; 706 } 707 708 return ret; 709 } 710 711 long kvm_arch_vm_ioctl(struct file *filp, 712 unsigned int ioctl, unsigned long arg) 713 { 714 struct kvm *kvm = filp->private_data; 715 void __user *argp = (void __user *)arg; 716 struct kvm_device_attr attr; 717 int r; 718 719 switch (ioctl) { 720 case KVM_S390_INTERRUPT: { 721 struct kvm_s390_interrupt s390int; 722 723 r = -EFAULT; 724 if (copy_from_user(&s390int, argp, sizeof(s390int))) 725 break; 726 r = kvm_s390_inject_vm(kvm, &s390int); 727 break; 728 } 729 case KVM_ENABLE_CAP: { 730 struct kvm_enable_cap cap; 731 r = -EFAULT; 732 if (copy_from_user(&cap, argp, sizeof(cap))) 733 break; 734 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 735 break; 736 } 737 case KVM_CREATE_IRQCHIP: { 738 struct kvm_irq_routing_entry routing; 739 740 r = -EINVAL; 741 if (kvm->arch.use_irqchip) { 742 /* Set up dummy routing. */ 743 memset(&routing, 0, sizeof(routing)); 744 kvm_set_irq_routing(kvm, &routing, 0, 0); 745 r = 0; 746 } 747 break; 748 } 749 case KVM_SET_DEVICE_ATTR: { 750 r = -EFAULT; 751 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 752 break; 753 r = kvm_s390_vm_set_attr(kvm, &attr); 754 break; 755 } 756 case KVM_GET_DEVICE_ATTR: { 757 r = -EFAULT; 758 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 759 break; 760 r = kvm_s390_vm_get_attr(kvm, &attr); 761 break; 762 } 763 case KVM_HAS_DEVICE_ATTR: { 764 r = -EFAULT; 765 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 766 break; 767 r = kvm_s390_vm_has_attr(kvm, &attr); 768 break; 769 } 770 default: 771 r = -ENOTTY; 772 } 773 774 return r; 775 } 776 777 static int kvm_s390_query_ap_config(u8 *config) 778 { 779 u32 fcn_code = 0x04000000UL; 780 u32 cc = 0; 781 782 memset(config, 0, 128); 783 asm volatile( 784 "lgr 0,%1\n" 785 "lgr 2,%2\n" 786 ".long 0xb2af0000\n" /* PQAP(QCI) */ 787 "0: ipm %0\n" 788 "srl %0,28\n" 789 "1:\n" 790 EX_TABLE(0b, 1b) 791 : "+r" (cc) 792 : "r" (fcn_code), "r" (config) 793 : "cc", "0", "2", "memory" 794 ); 795 796 return cc; 797 } 798 799 static int kvm_s390_apxa_installed(void) 800 { 801 u8 config[128]; 802 int cc; 803 804 if (test_facility(2) && test_facility(12)) { 805 cc = kvm_s390_query_ap_config(config); 806 807 if (cc) 808 pr_err("PQAP(QCI) failed with cc=%d", cc); 809 else 810 return config[0] & 0x40; 811 } 812 813 return 0; 814 } 815 816 static void kvm_s390_set_crycb_format(struct kvm *kvm) 817 { 818 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb; 819 820 if (kvm_s390_apxa_installed()) 821 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2; 822 else 823 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1; 824 } 825 826 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id) 827 { 828 get_cpu_id(cpu_id); 829 cpu_id->version = 0xff; 830 } 831 832 static int kvm_s390_crypto_init(struct kvm *kvm) 833 { 834 if (!test_kvm_facility(kvm, 76)) 835 return 0; 836 837 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb), 838 GFP_KERNEL | GFP_DMA); 839 if (!kvm->arch.crypto.crycb) 840 return -ENOMEM; 841 842 kvm_s390_set_crycb_format(kvm); 843 844 /* Enable AES/DEA protected key functions by default */ 845 kvm->arch.crypto.aes_kw = 1; 846 kvm->arch.crypto.dea_kw = 1; 847 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 848 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 849 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 850 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 851 852 return 0; 853 } 854 855 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 856 { 857 int i, rc; 858 char debug_name[16]; 859 static unsigned long sca_offset; 860 861 rc = -EINVAL; 862 #ifdef CONFIG_KVM_S390_UCONTROL 863 if (type & ~KVM_VM_S390_UCONTROL) 864 goto out_err; 865 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 866 goto out_err; 867 #else 868 if (type) 869 goto out_err; 870 #endif 871 872 rc = s390_enable_sie(); 873 if (rc) 874 goto out_err; 875 876 rc = -ENOMEM; 877 878 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); 879 if (!kvm->arch.sca) 880 goto out_err; 881 spin_lock(&kvm_lock); 882 sca_offset = (sca_offset + 16) & 0x7f0; 883 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset); 884 spin_unlock(&kvm_lock); 885 886 sprintf(debug_name, "kvm-%u", current->pid); 887 888 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 889 if (!kvm->arch.dbf) 890 goto out_nodbf; 891 892 /* 893 * The architectural maximum amount of facilities is 16 kbit. To store 894 * this amount, 2 kbyte of memory is required. Thus we need a full 895 * page to hold the guest facility list (arch.model.fac->list) and the 896 * facility mask (arch.model.fac->mask). Its address size has to be 897 * 31 bits and word aligned. 898 */ 899 kvm->arch.model.fac = 900 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 901 if (!kvm->arch.model.fac) 902 goto out_nofac; 903 904 /* Populate the facility mask initially. */ 905 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list, 906 S390_ARCH_FAC_LIST_SIZE_BYTE); 907 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) { 908 if (i < kvm_s390_fac_list_mask_size()) 909 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i]; 910 else 911 kvm->arch.model.fac->mask[i] = 0UL; 912 } 913 914 /* Populate the facility list initially. */ 915 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask, 916 S390_ARCH_FAC_LIST_SIZE_BYTE); 917 918 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id); 919 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff; 920 921 if (kvm_s390_crypto_init(kvm) < 0) 922 goto out_crypto; 923 924 spin_lock_init(&kvm->arch.float_int.lock); 925 INIT_LIST_HEAD(&kvm->arch.float_int.list); 926 init_waitqueue_head(&kvm->arch.ipte_wq); 927 mutex_init(&kvm->arch.ipte_mutex); 928 929 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 930 VM_EVENT(kvm, 3, "%s", "vm created"); 931 932 if (type & KVM_VM_S390_UCONTROL) { 933 kvm->arch.gmap = NULL; 934 } else { 935 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1); 936 if (!kvm->arch.gmap) 937 goto out_nogmap; 938 kvm->arch.gmap->private = kvm; 939 kvm->arch.gmap->pfault_enabled = 0; 940 } 941 942 kvm->arch.css_support = 0; 943 kvm->arch.use_irqchip = 0; 944 kvm->arch.epoch = 0; 945 946 spin_lock_init(&kvm->arch.start_stop_lock); 947 948 return 0; 949 out_nogmap: 950 kfree(kvm->arch.crypto.crycb); 951 out_crypto: 952 free_page((unsigned long)kvm->arch.model.fac); 953 out_nofac: 954 debug_unregister(kvm->arch.dbf); 955 out_nodbf: 956 free_page((unsigned long)(kvm->arch.sca)); 957 out_err: 958 return rc; 959 } 960 961 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 962 { 963 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 964 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 965 kvm_s390_clear_local_irqs(vcpu); 966 kvm_clear_async_pf_completion_queue(vcpu); 967 if (!kvm_is_ucontrol(vcpu->kvm)) { 968 clear_bit(63 - vcpu->vcpu_id, 969 (unsigned long *) &vcpu->kvm->arch.sca->mcn); 970 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda == 971 (__u64) vcpu->arch.sie_block) 972 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0; 973 } 974 smp_mb(); 975 976 if (kvm_is_ucontrol(vcpu->kvm)) 977 gmap_free(vcpu->arch.gmap); 978 979 if (kvm_s390_cmma_enabled(vcpu->kvm)) 980 kvm_s390_vcpu_unsetup_cmma(vcpu); 981 free_page((unsigned long)(vcpu->arch.sie_block)); 982 983 kvm_vcpu_uninit(vcpu); 984 kmem_cache_free(kvm_vcpu_cache, vcpu); 985 } 986 987 static void kvm_free_vcpus(struct kvm *kvm) 988 { 989 unsigned int i; 990 struct kvm_vcpu *vcpu; 991 992 kvm_for_each_vcpu(i, vcpu, kvm) 993 kvm_arch_vcpu_destroy(vcpu); 994 995 mutex_lock(&kvm->lock); 996 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 997 kvm->vcpus[i] = NULL; 998 999 atomic_set(&kvm->online_vcpus, 0); 1000 mutex_unlock(&kvm->lock); 1001 } 1002 1003 void kvm_arch_destroy_vm(struct kvm *kvm) 1004 { 1005 kvm_free_vcpus(kvm); 1006 free_page((unsigned long)kvm->arch.model.fac); 1007 free_page((unsigned long)(kvm->arch.sca)); 1008 debug_unregister(kvm->arch.dbf); 1009 kfree(kvm->arch.crypto.crycb); 1010 if (!kvm_is_ucontrol(kvm)) 1011 gmap_free(kvm->arch.gmap); 1012 kvm_s390_destroy_adapters(kvm); 1013 kvm_s390_clear_float_irqs(kvm); 1014 } 1015 1016 /* Section: vcpu related */ 1017 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) 1018 { 1019 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL); 1020 if (!vcpu->arch.gmap) 1021 return -ENOMEM; 1022 vcpu->arch.gmap->private = vcpu->kvm; 1023 1024 return 0; 1025 } 1026 1027 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 1028 { 1029 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 1030 kvm_clear_async_pf_completion_queue(vcpu); 1031 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 1032 KVM_SYNC_GPRS | 1033 KVM_SYNC_ACRS | 1034 KVM_SYNC_CRS | 1035 KVM_SYNC_ARCH0 | 1036 KVM_SYNC_PFAULT; 1037 1038 if (kvm_is_ucontrol(vcpu->kvm)) 1039 return __kvm_ucontrol_vcpu_init(vcpu); 1040 1041 return 0; 1042 } 1043 1044 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1045 { 1046 save_fp_ctl(&vcpu->arch.host_fpregs.fpc); 1047 save_fp_regs(vcpu->arch.host_fpregs.fprs); 1048 save_access_regs(vcpu->arch.host_acrs); 1049 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1050 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 1051 restore_access_regs(vcpu->run->s.regs.acrs); 1052 gmap_enable(vcpu->arch.gmap); 1053 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1054 } 1055 1056 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1057 { 1058 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1059 gmap_disable(vcpu->arch.gmap); 1060 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1061 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 1062 save_access_regs(vcpu->run->s.regs.acrs); 1063 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc); 1064 restore_fp_regs(vcpu->arch.host_fpregs.fprs); 1065 restore_access_regs(vcpu->arch.host_acrs); 1066 } 1067 1068 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 1069 { 1070 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 1071 vcpu->arch.sie_block->gpsw.mask = 0UL; 1072 vcpu->arch.sie_block->gpsw.addr = 0UL; 1073 kvm_s390_set_prefix(vcpu, 0); 1074 vcpu->arch.sie_block->cputm = 0UL; 1075 vcpu->arch.sie_block->ckc = 0UL; 1076 vcpu->arch.sie_block->todpr = 0; 1077 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 1078 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 1079 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 1080 vcpu->arch.guest_fpregs.fpc = 0; 1081 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc)); 1082 vcpu->arch.sie_block->gbea = 1; 1083 vcpu->arch.sie_block->pp = 0; 1084 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 1085 kvm_clear_async_pf_completion_queue(vcpu); 1086 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 1087 kvm_s390_vcpu_stop(vcpu); 1088 kvm_s390_clear_local_irqs(vcpu); 1089 } 1090 1091 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 1092 { 1093 mutex_lock(&vcpu->kvm->lock); 1094 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch; 1095 mutex_unlock(&vcpu->kvm->lock); 1096 if (!kvm_is_ucontrol(vcpu->kvm)) 1097 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 1098 } 1099 1100 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) 1101 { 1102 if (!test_kvm_facility(vcpu->kvm, 76)) 1103 return; 1104 1105 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA); 1106 1107 if (vcpu->kvm->arch.crypto.aes_kw) 1108 vcpu->arch.sie_block->ecb3 |= ECB3_AES; 1109 if (vcpu->kvm->arch.crypto.dea_kw) 1110 vcpu->arch.sie_block->ecb3 |= ECB3_DEA; 1111 1112 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd; 1113 } 1114 1115 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) 1116 { 1117 free_page(vcpu->arch.sie_block->cbrlo); 1118 vcpu->arch.sie_block->cbrlo = 0; 1119 } 1120 1121 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) 1122 { 1123 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL); 1124 if (!vcpu->arch.sie_block->cbrlo) 1125 return -ENOMEM; 1126 1127 vcpu->arch.sie_block->ecb2 |= 0x80; 1128 vcpu->arch.sie_block->ecb2 &= ~0x08; 1129 return 0; 1130 } 1131 1132 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 1133 { 1134 int rc = 0; 1135 1136 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 1137 CPUSTAT_SM | 1138 CPUSTAT_STOPPED | 1139 CPUSTAT_GED); 1140 vcpu->arch.sie_block->ecb = 6; 1141 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73)) 1142 vcpu->arch.sie_block->ecb |= 0x10; 1143 1144 vcpu->arch.sie_block->ecb2 = 8; 1145 vcpu->arch.sie_block->eca = 0xC1002000U; 1146 if (sclp_has_siif()) 1147 vcpu->arch.sie_block->eca |= 1; 1148 if (sclp_has_sigpif()) 1149 vcpu->arch.sie_block->eca |= 0x10000000U; 1150 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE | 1151 ICTL_TPROT; 1152 1153 if (kvm_s390_cmma_enabled(vcpu->kvm)) { 1154 rc = kvm_s390_vcpu_setup_cmma(vcpu); 1155 if (rc) 1156 return rc; 1157 } 1158 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1159 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 1160 1161 mutex_lock(&vcpu->kvm->lock); 1162 vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id; 1163 vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc; 1164 mutex_unlock(&vcpu->kvm->lock); 1165 1166 kvm_s390_vcpu_crypto_setup(vcpu); 1167 1168 return rc; 1169 } 1170 1171 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 1172 unsigned int id) 1173 { 1174 struct kvm_vcpu *vcpu; 1175 struct sie_page *sie_page; 1176 int rc = -EINVAL; 1177 1178 if (id >= KVM_MAX_VCPUS) 1179 goto out; 1180 1181 rc = -ENOMEM; 1182 1183 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1184 if (!vcpu) 1185 goto out; 1186 1187 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL); 1188 if (!sie_page) 1189 goto out_free_cpu; 1190 1191 vcpu->arch.sie_block = &sie_page->sie_block; 1192 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 1193 1194 vcpu->arch.sie_block->icpua = id; 1195 if (!kvm_is_ucontrol(kvm)) { 1196 if (!kvm->arch.sca) { 1197 WARN_ON_ONCE(1); 1198 goto out_free_cpu; 1199 } 1200 if (!kvm->arch.sca->cpu[id].sda) 1201 kvm->arch.sca->cpu[id].sda = 1202 (__u64) vcpu->arch.sie_block; 1203 vcpu->arch.sie_block->scaoh = 1204 (__u32)(((__u64)kvm->arch.sca) >> 32); 1205 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 1206 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 1207 } 1208 vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.fac->list; 1209 1210 spin_lock_init(&vcpu->arch.local_int.lock); 1211 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 1212 vcpu->arch.local_int.wq = &vcpu->wq; 1213 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 1214 1215 rc = kvm_vcpu_init(vcpu, kvm, id); 1216 if (rc) 1217 goto out_free_sie_block; 1218 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, 1219 vcpu->arch.sie_block); 1220 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 1221 1222 return vcpu; 1223 out_free_sie_block: 1224 free_page((unsigned long)(vcpu->arch.sie_block)); 1225 out_free_cpu: 1226 kmem_cache_free(kvm_vcpu_cache, vcpu); 1227 out: 1228 return ERR_PTR(rc); 1229 } 1230 1231 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 1232 { 1233 return kvm_s390_vcpu_has_irq(vcpu, 0); 1234 } 1235 1236 void s390_vcpu_block(struct kvm_vcpu *vcpu) 1237 { 1238 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 1239 } 1240 1241 void s390_vcpu_unblock(struct kvm_vcpu *vcpu) 1242 { 1243 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 1244 } 1245 1246 /* 1247 * Kick a guest cpu out of SIE and wait until SIE is not running. 1248 * If the CPU is not running (e.g. waiting as idle) the function will 1249 * return immediately. */ 1250 void exit_sie(struct kvm_vcpu *vcpu) 1251 { 1252 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 1253 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 1254 cpu_relax(); 1255 } 1256 1257 /* Kick a guest cpu out of SIE and prevent SIE-reentry */ 1258 void exit_sie_sync(struct kvm_vcpu *vcpu) 1259 { 1260 s390_vcpu_block(vcpu); 1261 exit_sie(vcpu); 1262 } 1263 1264 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address) 1265 { 1266 int i; 1267 struct kvm *kvm = gmap->private; 1268 struct kvm_vcpu *vcpu; 1269 1270 kvm_for_each_vcpu(i, vcpu, kvm) { 1271 /* match against both prefix pages */ 1272 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) { 1273 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address); 1274 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 1275 exit_sie_sync(vcpu); 1276 } 1277 } 1278 } 1279 1280 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 1281 { 1282 /* kvm common code refers to this, but never calls it */ 1283 BUG(); 1284 return 0; 1285 } 1286 1287 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 1288 struct kvm_one_reg *reg) 1289 { 1290 int r = -EINVAL; 1291 1292 switch (reg->id) { 1293 case KVM_REG_S390_TODPR: 1294 r = put_user(vcpu->arch.sie_block->todpr, 1295 (u32 __user *)reg->addr); 1296 break; 1297 case KVM_REG_S390_EPOCHDIFF: 1298 r = put_user(vcpu->arch.sie_block->epoch, 1299 (u64 __user *)reg->addr); 1300 break; 1301 case KVM_REG_S390_CPU_TIMER: 1302 r = put_user(vcpu->arch.sie_block->cputm, 1303 (u64 __user *)reg->addr); 1304 break; 1305 case KVM_REG_S390_CLOCK_COMP: 1306 r = put_user(vcpu->arch.sie_block->ckc, 1307 (u64 __user *)reg->addr); 1308 break; 1309 case KVM_REG_S390_PFTOKEN: 1310 r = put_user(vcpu->arch.pfault_token, 1311 (u64 __user *)reg->addr); 1312 break; 1313 case KVM_REG_S390_PFCOMPARE: 1314 r = put_user(vcpu->arch.pfault_compare, 1315 (u64 __user *)reg->addr); 1316 break; 1317 case KVM_REG_S390_PFSELECT: 1318 r = put_user(vcpu->arch.pfault_select, 1319 (u64 __user *)reg->addr); 1320 break; 1321 case KVM_REG_S390_PP: 1322 r = put_user(vcpu->arch.sie_block->pp, 1323 (u64 __user *)reg->addr); 1324 break; 1325 case KVM_REG_S390_GBEA: 1326 r = put_user(vcpu->arch.sie_block->gbea, 1327 (u64 __user *)reg->addr); 1328 break; 1329 default: 1330 break; 1331 } 1332 1333 return r; 1334 } 1335 1336 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 1337 struct kvm_one_reg *reg) 1338 { 1339 int r = -EINVAL; 1340 1341 switch (reg->id) { 1342 case KVM_REG_S390_TODPR: 1343 r = get_user(vcpu->arch.sie_block->todpr, 1344 (u32 __user *)reg->addr); 1345 break; 1346 case KVM_REG_S390_EPOCHDIFF: 1347 r = get_user(vcpu->arch.sie_block->epoch, 1348 (u64 __user *)reg->addr); 1349 break; 1350 case KVM_REG_S390_CPU_TIMER: 1351 r = get_user(vcpu->arch.sie_block->cputm, 1352 (u64 __user *)reg->addr); 1353 break; 1354 case KVM_REG_S390_CLOCK_COMP: 1355 r = get_user(vcpu->arch.sie_block->ckc, 1356 (u64 __user *)reg->addr); 1357 break; 1358 case KVM_REG_S390_PFTOKEN: 1359 r = get_user(vcpu->arch.pfault_token, 1360 (u64 __user *)reg->addr); 1361 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 1362 kvm_clear_async_pf_completion_queue(vcpu); 1363 break; 1364 case KVM_REG_S390_PFCOMPARE: 1365 r = get_user(vcpu->arch.pfault_compare, 1366 (u64 __user *)reg->addr); 1367 break; 1368 case KVM_REG_S390_PFSELECT: 1369 r = get_user(vcpu->arch.pfault_select, 1370 (u64 __user *)reg->addr); 1371 break; 1372 case KVM_REG_S390_PP: 1373 r = get_user(vcpu->arch.sie_block->pp, 1374 (u64 __user *)reg->addr); 1375 break; 1376 case KVM_REG_S390_GBEA: 1377 r = get_user(vcpu->arch.sie_block->gbea, 1378 (u64 __user *)reg->addr); 1379 break; 1380 default: 1381 break; 1382 } 1383 1384 return r; 1385 } 1386 1387 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 1388 { 1389 kvm_s390_vcpu_initial_reset(vcpu); 1390 return 0; 1391 } 1392 1393 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 1394 { 1395 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 1396 return 0; 1397 } 1398 1399 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 1400 { 1401 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 1402 return 0; 1403 } 1404 1405 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 1406 struct kvm_sregs *sregs) 1407 { 1408 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 1409 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 1410 restore_access_regs(vcpu->run->s.regs.acrs); 1411 return 0; 1412 } 1413 1414 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 1415 struct kvm_sregs *sregs) 1416 { 1417 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 1418 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 1419 return 0; 1420 } 1421 1422 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1423 { 1424 if (test_fp_ctl(fpu->fpc)) 1425 return -EINVAL; 1426 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 1427 vcpu->arch.guest_fpregs.fpc = fpu->fpc; 1428 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1429 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 1430 return 0; 1431 } 1432 1433 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1434 { 1435 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs)); 1436 fpu->fpc = vcpu->arch.guest_fpregs.fpc; 1437 return 0; 1438 } 1439 1440 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 1441 { 1442 int rc = 0; 1443 1444 if (!is_vcpu_stopped(vcpu)) 1445 rc = -EBUSY; 1446 else { 1447 vcpu->run->psw_mask = psw.mask; 1448 vcpu->run->psw_addr = psw.addr; 1449 } 1450 return rc; 1451 } 1452 1453 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 1454 struct kvm_translation *tr) 1455 { 1456 return -EINVAL; /* not implemented yet */ 1457 } 1458 1459 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \ 1460 KVM_GUESTDBG_USE_HW_BP | \ 1461 KVM_GUESTDBG_ENABLE) 1462 1463 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 1464 struct kvm_guest_debug *dbg) 1465 { 1466 int rc = 0; 1467 1468 vcpu->guest_debug = 0; 1469 kvm_s390_clear_bp_data(vcpu); 1470 1471 if (dbg->control & ~VALID_GUESTDBG_FLAGS) 1472 return -EINVAL; 1473 1474 if (dbg->control & KVM_GUESTDBG_ENABLE) { 1475 vcpu->guest_debug = dbg->control; 1476 /* enforce guest PER */ 1477 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 1478 1479 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 1480 rc = kvm_s390_import_bp_data(vcpu, dbg); 1481 } else { 1482 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 1483 vcpu->arch.guestdbg.last_bp = 0; 1484 } 1485 1486 if (rc) { 1487 vcpu->guest_debug = 0; 1488 kvm_s390_clear_bp_data(vcpu); 1489 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 1490 } 1491 1492 return rc; 1493 } 1494 1495 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 1496 struct kvm_mp_state *mp_state) 1497 { 1498 /* CHECK_STOP and LOAD are not supported yet */ 1499 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED : 1500 KVM_MP_STATE_OPERATING; 1501 } 1502 1503 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 1504 struct kvm_mp_state *mp_state) 1505 { 1506 int rc = 0; 1507 1508 /* user space knows about this interface - let it control the state */ 1509 vcpu->kvm->arch.user_cpu_state_ctrl = 1; 1510 1511 switch (mp_state->mp_state) { 1512 case KVM_MP_STATE_STOPPED: 1513 kvm_s390_vcpu_stop(vcpu); 1514 break; 1515 case KVM_MP_STATE_OPERATING: 1516 kvm_s390_vcpu_start(vcpu); 1517 break; 1518 case KVM_MP_STATE_LOAD: 1519 case KVM_MP_STATE_CHECK_STOP: 1520 /* fall through - CHECK_STOP and LOAD are not supported yet */ 1521 default: 1522 rc = -ENXIO; 1523 } 1524 1525 return rc; 1526 } 1527 1528 bool kvm_s390_cmma_enabled(struct kvm *kvm) 1529 { 1530 if (!MACHINE_IS_LPAR) 1531 return false; 1532 /* only enable for z10 and later */ 1533 if (!MACHINE_HAS_EDAT1) 1534 return false; 1535 if (!kvm->arch.use_cmma) 1536 return false; 1537 return true; 1538 } 1539 1540 static bool ibs_enabled(struct kvm_vcpu *vcpu) 1541 { 1542 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS; 1543 } 1544 1545 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 1546 { 1547 retry: 1548 s390_vcpu_unblock(vcpu); 1549 /* 1550 * We use MMU_RELOAD just to re-arm the ipte notifier for the 1551 * guest prefix page. gmap_ipte_notify will wait on the ptl lock. 1552 * This ensures that the ipte instruction for this request has 1553 * already finished. We might race against a second unmapper that 1554 * wants to set the blocking bit. Lets just retry the request loop. 1555 */ 1556 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 1557 int rc; 1558 rc = gmap_ipte_notify(vcpu->arch.gmap, 1559 kvm_s390_get_prefix(vcpu), 1560 PAGE_SIZE * 2); 1561 if (rc) 1562 return rc; 1563 goto retry; 1564 } 1565 1566 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 1567 vcpu->arch.sie_block->ihcpu = 0xffff; 1568 goto retry; 1569 } 1570 1571 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 1572 if (!ibs_enabled(vcpu)) { 1573 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 1574 atomic_set_mask(CPUSTAT_IBS, 1575 &vcpu->arch.sie_block->cpuflags); 1576 } 1577 goto retry; 1578 } 1579 1580 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 1581 if (ibs_enabled(vcpu)) { 1582 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 1583 atomic_clear_mask(CPUSTAT_IBS, 1584 &vcpu->arch.sie_block->cpuflags); 1585 } 1586 goto retry; 1587 } 1588 1589 /* nothing to do, just clear the request */ 1590 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 1591 1592 return 0; 1593 } 1594 1595 /** 1596 * kvm_arch_fault_in_page - fault-in guest page if necessary 1597 * @vcpu: The corresponding virtual cpu 1598 * @gpa: Guest physical address 1599 * @writable: Whether the page should be writable or not 1600 * 1601 * Make sure that a guest page has been faulted-in on the host. 1602 * 1603 * Return: Zero on success, negative error code otherwise. 1604 */ 1605 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable) 1606 { 1607 return gmap_fault(vcpu->arch.gmap, gpa, 1608 writable ? FAULT_FLAG_WRITE : 0); 1609 } 1610 1611 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, 1612 unsigned long token) 1613 { 1614 struct kvm_s390_interrupt inti; 1615 struct kvm_s390_irq irq; 1616 1617 if (start_token) { 1618 irq.u.ext.ext_params2 = token; 1619 irq.type = KVM_S390_INT_PFAULT_INIT; 1620 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); 1621 } else { 1622 inti.type = KVM_S390_INT_PFAULT_DONE; 1623 inti.parm64 = token; 1624 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); 1625 } 1626 } 1627 1628 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 1629 struct kvm_async_pf *work) 1630 { 1631 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); 1632 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); 1633 } 1634 1635 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 1636 struct kvm_async_pf *work) 1637 { 1638 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token); 1639 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token); 1640 } 1641 1642 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 1643 struct kvm_async_pf *work) 1644 { 1645 /* s390 will always inject the page directly */ 1646 } 1647 1648 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 1649 { 1650 /* 1651 * s390 will always inject the page directly, 1652 * but we still want check_async_completion to cleanup 1653 */ 1654 return true; 1655 } 1656 1657 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) 1658 { 1659 hva_t hva; 1660 struct kvm_arch_async_pf arch; 1661 int rc; 1662 1663 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 1664 return 0; 1665 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) != 1666 vcpu->arch.pfault_compare) 1667 return 0; 1668 if (psw_extint_disabled(vcpu)) 1669 return 0; 1670 if (kvm_s390_vcpu_has_irq(vcpu, 0)) 1671 return 0; 1672 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 1673 return 0; 1674 if (!vcpu->arch.gmap->pfault_enabled) 1675 return 0; 1676 1677 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr)); 1678 hva += current->thread.gmap_addr & ~PAGE_MASK; 1679 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8)) 1680 return 0; 1681 1682 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch); 1683 return rc; 1684 } 1685 1686 static int vcpu_pre_run(struct kvm_vcpu *vcpu) 1687 { 1688 int rc, cpuflags; 1689 1690 /* 1691 * On s390 notifications for arriving pages will be delivered directly 1692 * to the guest but the house keeping for completed pfaults is 1693 * handled outside the worker. 1694 */ 1695 kvm_check_async_pf_completion(vcpu); 1696 1697 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16); 1698 1699 if (need_resched()) 1700 schedule(); 1701 1702 if (test_cpu_flag(CIF_MCCK_PENDING)) 1703 s390_handle_mcck(); 1704 1705 if (!kvm_is_ucontrol(vcpu->kvm)) { 1706 rc = kvm_s390_deliver_pending_interrupts(vcpu); 1707 if (rc) 1708 return rc; 1709 } 1710 1711 rc = kvm_s390_handle_requests(vcpu); 1712 if (rc) 1713 return rc; 1714 1715 if (guestdbg_enabled(vcpu)) { 1716 kvm_s390_backup_guest_per_regs(vcpu); 1717 kvm_s390_patch_guest_per_regs(vcpu); 1718 } 1719 1720 vcpu->arch.sie_block->icptcode = 0; 1721 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); 1722 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); 1723 trace_kvm_s390_sie_enter(vcpu, cpuflags); 1724 1725 return 0; 1726 } 1727 1728 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 1729 { 1730 int rc = -1; 1731 1732 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 1733 vcpu->arch.sie_block->icptcode); 1734 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 1735 1736 if (guestdbg_enabled(vcpu)) 1737 kvm_s390_restore_guest_per_regs(vcpu); 1738 1739 if (exit_reason >= 0) { 1740 rc = 0; 1741 } else if (kvm_is_ucontrol(vcpu->kvm)) { 1742 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; 1743 vcpu->run->s390_ucontrol.trans_exc_code = 1744 current->thread.gmap_addr; 1745 vcpu->run->s390_ucontrol.pgm_code = 0x10; 1746 rc = -EREMOTE; 1747 1748 } else if (current->thread.gmap_pfault) { 1749 trace_kvm_s390_major_guest_pfault(vcpu); 1750 current->thread.gmap_pfault = 0; 1751 if (kvm_arch_setup_async_pf(vcpu)) { 1752 rc = 0; 1753 } else { 1754 gpa_t gpa = current->thread.gmap_addr; 1755 rc = kvm_arch_fault_in_page(vcpu, gpa, 1); 1756 } 1757 } 1758 1759 if (rc == -1) { 1760 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 1761 trace_kvm_s390_sie_fault(vcpu); 1762 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1763 } 1764 1765 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 1766 1767 if (rc == 0) { 1768 if (kvm_is_ucontrol(vcpu->kvm)) 1769 /* Don't exit for host interrupts. */ 1770 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0; 1771 else 1772 rc = kvm_handle_sie_intercept(vcpu); 1773 } 1774 1775 return rc; 1776 } 1777 1778 static int __vcpu_run(struct kvm_vcpu *vcpu) 1779 { 1780 int rc, exit_reason; 1781 1782 /* 1783 * We try to hold kvm->srcu during most of vcpu_run (except when run- 1784 * ning the guest), so that memslots (and other stuff) are protected 1785 */ 1786 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1787 1788 do { 1789 rc = vcpu_pre_run(vcpu); 1790 if (rc) 1791 break; 1792 1793 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1794 /* 1795 * As PF_VCPU will be used in fault handler, between 1796 * guest_enter and guest_exit should be no uaccess. 1797 */ 1798 preempt_disable(); 1799 kvm_guest_enter(); 1800 preempt_enable(); 1801 exit_reason = sie64a(vcpu->arch.sie_block, 1802 vcpu->run->s.regs.gprs); 1803 kvm_guest_exit(); 1804 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1805 1806 rc = vcpu_post_run(vcpu, exit_reason); 1807 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc); 1808 1809 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1810 return rc; 1811 } 1812 1813 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 1814 { 1815 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 1816 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 1817 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) 1818 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 1819 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 1820 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 1821 /* some control register changes require a tlb flush */ 1822 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1823 } 1824 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { 1825 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm; 1826 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc; 1827 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr; 1828 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp; 1829 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea; 1830 } 1831 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) { 1832 vcpu->arch.pfault_token = kvm_run->s.regs.pft; 1833 vcpu->arch.pfault_select = kvm_run->s.regs.pfs; 1834 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc; 1835 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 1836 kvm_clear_async_pf_completion_queue(vcpu); 1837 } 1838 kvm_run->kvm_dirty_regs = 0; 1839 } 1840 1841 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 1842 { 1843 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 1844 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 1845 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); 1846 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 1847 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm; 1848 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc; 1849 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr; 1850 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp; 1851 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea; 1852 kvm_run->s.regs.pft = vcpu->arch.pfault_token; 1853 kvm_run->s.regs.pfs = vcpu->arch.pfault_select; 1854 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; 1855 } 1856 1857 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 1858 { 1859 int rc; 1860 sigset_t sigsaved; 1861 1862 if (guestdbg_exit_pending(vcpu)) { 1863 kvm_s390_prepare_debug_exit(vcpu); 1864 return 0; 1865 } 1866 1867 if (vcpu->sigset_active) 1868 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 1869 1870 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { 1871 kvm_s390_vcpu_start(vcpu); 1872 } else if (is_vcpu_stopped(vcpu)) { 1873 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n", 1874 vcpu->vcpu_id); 1875 return -EINVAL; 1876 } 1877 1878 sync_regs(vcpu, kvm_run); 1879 1880 might_fault(); 1881 rc = __vcpu_run(vcpu); 1882 1883 if (signal_pending(current) && !rc) { 1884 kvm_run->exit_reason = KVM_EXIT_INTR; 1885 rc = -EINTR; 1886 } 1887 1888 if (guestdbg_exit_pending(vcpu) && !rc) { 1889 kvm_s390_prepare_debug_exit(vcpu); 1890 rc = 0; 1891 } 1892 1893 if (rc == -EOPNOTSUPP) { 1894 /* intercept cannot be handled in-kernel, prepare kvm-run */ 1895 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC; 1896 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 1897 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 1898 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 1899 rc = 0; 1900 } 1901 1902 if (rc == -EREMOTE) { 1903 /* intercept was handled, but userspace support is needed 1904 * kvm_run has been prepared by the handler */ 1905 rc = 0; 1906 } 1907 1908 store_regs(vcpu, kvm_run); 1909 1910 if (vcpu->sigset_active) 1911 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 1912 1913 vcpu->stat.exit_userspace++; 1914 return rc; 1915 } 1916 1917 /* 1918 * store status at address 1919 * we use have two special cases: 1920 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 1921 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 1922 */ 1923 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa) 1924 { 1925 unsigned char archmode = 1; 1926 unsigned int px; 1927 u64 clkcomp; 1928 int rc; 1929 1930 if (gpa == KVM_S390_STORE_STATUS_NOADDR) { 1931 if (write_guest_abs(vcpu, 163, &archmode, 1)) 1932 return -EFAULT; 1933 gpa = SAVE_AREA_BASE; 1934 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) { 1935 if (write_guest_real(vcpu, 163, &archmode, 1)) 1936 return -EFAULT; 1937 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE); 1938 } 1939 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs), 1940 vcpu->arch.guest_fpregs.fprs, 128); 1941 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs), 1942 vcpu->run->s.regs.gprs, 128); 1943 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw), 1944 &vcpu->arch.sie_block->gpsw, 16); 1945 px = kvm_s390_get_prefix(vcpu); 1946 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg), 1947 &px, 4); 1948 rc |= write_guest_abs(vcpu, 1949 gpa + offsetof(struct save_area, fp_ctrl_reg), 1950 &vcpu->arch.guest_fpregs.fpc, 4); 1951 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg), 1952 &vcpu->arch.sie_block->todpr, 4); 1953 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer), 1954 &vcpu->arch.sie_block->cputm, 8); 1955 clkcomp = vcpu->arch.sie_block->ckc >> 8; 1956 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp), 1957 &clkcomp, 8); 1958 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs), 1959 &vcpu->run->s.regs.acrs, 64); 1960 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs), 1961 &vcpu->arch.sie_block->gcr, 128); 1962 return rc ? -EFAULT : 0; 1963 } 1964 1965 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 1966 { 1967 /* 1968 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 1969 * copying in vcpu load/put. Lets update our copies before we save 1970 * it into the save area 1971 */ 1972 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1973 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 1974 save_access_regs(vcpu->run->s.regs.acrs); 1975 1976 return kvm_s390_store_status_unloaded(vcpu, addr); 1977 } 1978 1979 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 1980 { 1981 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 1982 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu); 1983 exit_sie_sync(vcpu); 1984 } 1985 1986 static void __disable_ibs_on_all_vcpus(struct kvm *kvm) 1987 { 1988 unsigned int i; 1989 struct kvm_vcpu *vcpu; 1990 1991 kvm_for_each_vcpu(i, vcpu, kvm) { 1992 __disable_ibs_on_vcpu(vcpu); 1993 } 1994 } 1995 1996 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 1997 { 1998 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu); 1999 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu); 2000 exit_sie_sync(vcpu); 2001 } 2002 2003 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) 2004 { 2005 int i, online_vcpus, started_vcpus = 0; 2006 2007 if (!is_vcpu_stopped(vcpu)) 2008 return; 2009 2010 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); 2011 /* Only one cpu at a time may enter/leave the STOPPED state. */ 2012 spin_lock(&vcpu->kvm->arch.start_stop_lock); 2013 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 2014 2015 for (i = 0; i < online_vcpus; i++) { 2016 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) 2017 started_vcpus++; 2018 } 2019 2020 if (started_vcpus == 0) { 2021 /* we're the only active VCPU -> speed it up */ 2022 __enable_ibs_on_vcpu(vcpu); 2023 } else if (started_vcpus == 1) { 2024 /* 2025 * As we are starting a second VCPU, we have to disable 2026 * the IBS facility on all VCPUs to remove potentially 2027 * oustanding ENABLE requests. 2028 */ 2029 __disable_ibs_on_all_vcpus(vcpu->kvm); 2030 } 2031 2032 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 2033 /* 2034 * Another VCPU might have used IBS while we were offline. 2035 * Let's play safe and flush the VCPU at startup. 2036 */ 2037 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2038 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 2039 return; 2040 } 2041 2042 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) 2043 { 2044 int i, online_vcpus, started_vcpus = 0; 2045 struct kvm_vcpu *started_vcpu = NULL; 2046 2047 if (is_vcpu_stopped(vcpu)) 2048 return; 2049 2050 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); 2051 /* Only one cpu at a time may enter/leave the STOPPED state. */ 2052 spin_lock(&vcpu->kvm->arch.start_stop_lock); 2053 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 2054 2055 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 2056 kvm_s390_clear_stop_irq(vcpu); 2057 2058 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 2059 __disable_ibs_on_vcpu(vcpu); 2060 2061 for (i = 0; i < online_vcpus; i++) { 2062 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) { 2063 started_vcpus++; 2064 started_vcpu = vcpu->kvm->vcpus[i]; 2065 } 2066 } 2067 2068 if (started_vcpus == 1) { 2069 /* 2070 * As we only have one VCPU left, we want to enable the 2071 * IBS facility for that VCPU to speed it up. 2072 */ 2073 __enable_ibs_on_vcpu(started_vcpu); 2074 } 2075 2076 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 2077 return; 2078 } 2079 2080 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 2081 struct kvm_enable_cap *cap) 2082 { 2083 int r; 2084 2085 if (cap->flags) 2086 return -EINVAL; 2087 2088 switch (cap->cap) { 2089 case KVM_CAP_S390_CSS_SUPPORT: 2090 if (!vcpu->kvm->arch.css_support) { 2091 vcpu->kvm->arch.css_support = 1; 2092 trace_kvm_s390_enable_css(vcpu->kvm); 2093 } 2094 r = 0; 2095 break; 2096 default: 2097 r = -EINVAL; 2098 break; 2099 } 2100 return r; 2101 } 2102 2103 long kvm_arch_vcpu_ioctl(struct file *filp, 2104 unsigned int ioctl, unsigned long arg) 2105 { 2106 struct kvm_vcpu *vcpu = filp->private_data; 2107 void __user *argp = (void __user *)arg; 2108 int idx; 2109 long r; 2110 2111 switch (ioctl) { 2112 case KVM_S390_INTERRUPT: { 2113 struct kvm_s390_interrupt s390int; 2114 struct kvm_s390_irq s390irq; 2115 2116 r = -EFAULT; 2117 if (copy_from_user(&s390int, argp, sizeof(s390int))) 2118 break; 2119 if (s390int_to_s390irq(&s390int, &s390irq)) 2120 return -EINVAL; 2121 r = kvm_s390_inject_vcpu(vcpu, &s390irq); 2122 break; 2123 } 2124 case KVM_S390_STORE_STATUS: 2125 idx = srcu_read_lock(&vcpu->kvm->srcu); 2126 r = kvm_s390_vcpu_store_status(vcpu, arg); 2127 srcu_read_unlock(&vcpu->kvm->srcu, idx); 2128 break; 2129 case KVM_S390_SET_INITIAL_PSW: { 2130 psw_t psw; 2131 2132 r = -EFAULT; 2133 if (copy_from_user(&psw, argp, sizeof(psw))) 2134 break; 2135 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 2136 break; 2137 } 2138 case KVM_S390_INITIAL_RESET: 2139 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 2140 break; 2141 case KVM_SET_ONE_REG: 2142 case KVM_GET_ONE_REG: { 2143 struct kvm_one_reg reg; 2144 r = -EFAULT; 2145 if (copy_from_user(®, argp, sizeof(reg))) 2146 break; 2147 if (ioctl == KVM_SET_ONE_REG) 2148 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 2149 else 2150 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 2151 break; 2152 } 2153 #ifdef CONFIG_KVM_S390_UCONTROL 2154 case KVM_S390_UCAS_MAP: { 2155 struct kvm_s390_ucas_mapping ucasmap; 2156 2157 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 2158 r = -EFAULT; 2159 break; 2160 } 2161 2162 if (!kvm_is_ucontrol(vcpu->kvm)) { 2163 r = -EINVAL; 2164 break; 2165 } 2166 2167 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 2168 ucasmap.vcpu_addr, ucasmap.length); 2169 break; 2170 } 2171 case KVM_S390_UCAS_UNMAP: { 2172 struct kvm_s390_ucas_mapping ucasmap; 2173 2174 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 2175 r = -EFAULT; 2176 break; 2177 } 2178 2179 if (!kvm_is_ucontrol(vcpu->kvm)) { 2180 r = -EINVAL; 2181 break; 2182 } 2183 2184 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 2185 ucasmap.length); 2186 break; 2187 } 2188 #endif 2189 case KVM_S390_VCPU_FAULT: { 2190 r = gmap_fault(vcpu->arch.gmap, arg, 0); 2191 break; 2192 } 2193 case KVM_ENABLE_CAP: 2194 { 2195 struct kvm_enable_cap cap; 2196 r = -EFAULT; 2197 if (copy_from_user(&cap, argp, sizeof(cap))) 2198 break; 2199 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 2200 break; 2201 } 2202 default: 2203 r = -ENOTTY; 2204 } 2205 return r; 2206 } 2207 2208 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 2209 { 2210 #ifdef CONFIG_KVM_S390_UCONTROL 2211 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 2212 && (kvm_is_ucontrol(vcpu->kvm))) { 2213 vmf->page = virt_to_page(vcpu->arch.sie_block); 2214 get_page(vmf->page); 2215 return 0; 2216 } 2217 #endif 2218 return VM_FAULT_SIGBUS; 2219 } 2220 2221 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 2222 unsigned long npages) 2223 { 2224 return 0; 2225 } 2226 2227 /* Section: memory related */ 2228 int kvm_arch_prepare_memory_region(struct kvm *kvm, 2229 struct kvm_memory_slot *memslot, 2230 struct kvm_userspace_memory_region *mem, 2231 enum kvm_mr_change change) 2232 { 2233 /* A few sanity checks. We can have memory slots which have to be 2234 located/ended at a segment boundary (1MB). The memory in userland is 2235 ok to be fragmented into various different vmas. It is okay to mmap() 2236 and munmap() stuff in this slot after doing this call at any time */ 2237 2238 if (mem->userspace_addr & 0xffffful) 2239 return -EINVAL; 2240 2241 if (mem->memory_size & 0xffffful) 2242 return -EINVAL; 2243 2244 return 0; 2245 } 2246 2247 void kvm_arch_commit_memory_region(struct kvm *kvm, 2248 struct kvm_userspace_memory_region *mem, 2249 const struct kvm_memory_slot *old, 2250 enum kvm_mr_change change) 2251 { 2252 int rc; 2253 2254 /* If the basics of the memslot do not change, we do not want 2255 * to update the gmap. Every update causes several unnecessary 2256 * segment translation exceptions. This is usually handled just 2257 * fine by the normal fault handler + gmap, but it will also 2258 * cause faults on the prefix page of running guest CPUs. 2259 */ 2260 if (old->userspace_addr == mem->userspace_addr && 2261 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 2262 old->npages * PAGE_SIZE == mem->memory_size) 2263 return; 2264 2265 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 2266 mem->guest_phys_addr, mem->memory_size); 2267 if (rc) 2268 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n"); 2269 return; 2270 } 2271 2272 static int __init kvm_s390_init(void) 2273 { 2274 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 2275 } 2276 2277 static void __exit kvm_s390_exit(void) 2278 { 2279 kvm_exit(); 2280 } 2281 2282 module_init(kvm_s390_init); 2283 module_exit(kvm_s390_exit); 2284 2285 /* 2286 * Enable autoloading of the kvm module. 2287 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 2288 * since x86 takes a different approach. 2289 */ 2290 #include <linux/miscdevice.h> 2291 MODULE_ALIAS_MISCDEV(KVM_MINOR); 2292 MODULE_ALIAS("devname:kvm"); 2293