1 /* 2 * hosting zSeries kernel virtual machines 3 * 4 * Copyright IBM Corp. 2008, 2009 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 * Christian Borntraeger <borntraeger@de.ibm.com> 12 * Heiko Carstens <heiko.carstens@de.ibm.com> 13 * Christian Ehrhardt <ehrhardt@de.ibm.com> 14 * Jason J. Herne <jjherne@us.ibm.com> 15 */ 16 17 #include <linux/compiler.h> 18 #include <linux/err.h> 19 #include <linux/fs.h> 20 #include <linux/hrtimer.h> 21 #include <linux/init.h> 22 #include <linux/kvm.h> 23 #include <linux/kvm_host.h> 24 #include <linux/module.h> 25 #include <linux/random.h> 26 #include <linux/slab.h> 27 #include <linux/timer.h> 28 #include <linux/vmalloc.h> 29 #include <linux/bitmap.h> 30 #include <asm/asm-offsets.h> 31 #include <asm/lowcore.h> 32 #include <asm/etr.h> 33 #include <asm/pgtable.h> 34 #include <asm/gmap.h> 35 #include <asm/nmi.h> 36 #include <asm/switch_to.h> 37 #include <asm/isc.h> 38 #include <asm/sclp.h> 39 #include <asm/cpacf.h> 40 #include <asm/etr.h> 41 #include "kvm-s390.h" 42 #include "gaccess.h" 43 44 #define KMSG_COMPONENT "kvm-s390" 45 #undef pr_fmt 46 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 47 48 #define CREATE_TRACE_POINTS 49 #include "trace.h" 50 #include "trace-s390.h" 51 52 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */ 53 #define LOCAL_IRQS 32 54 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ 55 (KVM_MAX_VCPUS + LOCAL_IRQS)) 56 57 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 58 59 struct kvm_stats_debugfs_item debugfs_entries[] = { 60 { "userspace_handled", VCPU_STAT(exit_userspace) }, 61 { "exit_null", VCPU_STAT(exit_null) }, 62 { "exit_validity", VCPU_STAT(exit_validity) }, 63 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 64 { "exit_external_request", VCPU_STAT(exit_external_request) }, 65 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 66 { "exit_instruction", VCPU_STAT(exit_instruction) }, 67 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 68 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 69 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) }, 70 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, 71 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, 72 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, 73 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 74 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 75 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 76 { "instruction_stctl", VCPU_STAT(instruction_stctl) }, 77 { "instruction_stctg", VCPU_STAT(instruction_stctg) }, 78 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 79 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 80 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 81 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 82 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 83 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 84 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 85 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 86 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 87 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 88 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 89 { "instruction_spx", VCPU_STAT(instruction_spx) }, 90 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 91 { "instruction_stap", VCPU_STAT(instruction_stap) }, 92 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 93 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 94 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 95 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 96 { "instruction_essa", VCPU_STAT(instruction_essa) }, 97 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 98 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 99 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 100 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) }, 101 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 102 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 103 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 104 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 105 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) }, 106 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) }, 107 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 108 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, 109 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, 110 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) }, 111 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 112 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 113 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 114 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) }, 115 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) }, 116 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) }, 117 { "diagnose_10", VCPU_STAT(diagnose_10) }, 118 { "diagnose_44", VCPU_STAT(diagnose_44) }, 119 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 120 { "diagnose_258", VCPU_STAT(diagnose_258) }, 121 { "diagnose_308", VCPU_STAT(diagnose_308) }, 122 { "diagnose_500", VCPU_STAT(diagnose_500) }, 123 { NULL } 124 }; 125 126 /* upper facilities limit for kvm */ 127 unsigned long kvm_s390_fac_list_mask[16] = { 128 0xffe6000000000000UL, 129 0x005e000000000000UL, 130 }; 131 132 unsigned long kvm_s390_fac_list_mask_size(void) 133 { 134 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64); 135 return ARRAY_SIZE(kvm_s390_fac_list_mask); 136 } 137 138 /* available cpu features supported by kvm */ 139 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); 140 /* available subfunctions indicated via query / "test bit" */ 141 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc; 142 143 static struct gmap_notifier gmap_notifier; 144 debug_info_t *kvm_s390_dbf; 145 146 /* Section: not file related */ 147 int kvm_arch_hardware_enable(void) 148 { 149 /* every s390 is virtualization enabled ;-) */ 150 return 0; 151 } 152 153 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address); 154 155 /* 156 * This callback is executed during stop_machine(). All CPUs are therefore 157 * temporarily stopped. In order not to change guest behavior, we have to 158 * disable preemption whenever we touch the epoch of kvm and the VCPUs, 159 * so a CPU won't be stopped while calculating with the epoch. 160 */ 161 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val, 162 void *v) 163 { 164 struct kvm *kvm; 165 struct kvm_vcpu *vcpu; 166 int i; 167 unsigned long long *delta = v; 168 169 list_for_each_entry(kvm, &vm_list, vm_list) { 170 kvm->arch.epoch -= *delta; 171 kvm_for_each_vcpu(i, vcpu, kvm) { 172 vcpu->arch.sie_block->epoch -= *delta; 173 if (vcpu->arch.cputm_enabled) 174 vcpu->arch.cputm_start += *delta; 175 } 176 } 177 return NOTIFY_OK; 178 } 179 180 static struct notifier_block kvm_clock_notifier = { 181 .notifier_call = kvm_clock_sync, 182 }; 183 184 int kvm_arch_hardware_setup(void) 185 { 186 gmap_notifier.notifier_call = kvm_gmap_notifier; 187 gmap_register_ipte_notifier(&gmap_notifier); 188 atomic_notifier_chain_register(&s390_epoch_delta_notifier, 189 &kvm_clock_notifier); 190 return 0; 191 } 192 193 void kvm_arch_hardware_unsetup(void) 194 { 195 gmap_unregister_ipte_notifier(&gmap_notifier); 196 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier, 197 &kvm_clock_notifier); 198 } 199 200 static void allow_cpu_feat(unsigned long nr) 201 { 202 set_bit_inv(nr, kvm_s390_available_cpu_feat); 203 } 204 205 static inline int plo_test_bit(unsigned char nr) 206 { 207 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100; 208 int cc = 3; /* subfunction not available */ 209 210 asm volatile( 211 /* Parameter registers are ignored for "test bit" */ 212 " plo 0,0,0,0(0)\n" 213 " ipm %0\n" 214 " srl %0,28\n" 215 : "=d" (cc) 216 : "d" (r0) 217 : "cc"); 218 return cc == 0; 219 } 220 221 static void kvm_s390_cpu_feat_init(void) 222 { 223 int i; 224 225 for (i = 0; i < 256; ++i) { 226 if (plo_test_bit(i)) 227 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7); 228 } 229 230 if (test_facility(28)) /* TOD-clock steering */ 231 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF); 232 233 if (test_facility(17)) { /* MSA */ 234 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac); 235 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc); 236 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km); 237 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd); 238 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd); 239 } 240 if (test_facility(76)) /* MSA3 */ 241 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo); 242 if (test_facility(77)) { /* MSA4 */ 243 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr); 244 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf); 245 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo); 246 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc); 247 } 248 if (test_facility(57)) /* MSA5 */ 249 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno); 250 251 if (MACHINE_HAS_ESOP) 252 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP); 253 } 254 255 int kvm_arch_init(void *opaque) 256 { 257 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long)); 258 if (!kvm_s390_dbf) 259 return -ENOMEM; 260 261 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) { 262 debug_unregister(kvm_s390_dbf); 263 return -ENOMEM; 264 } 265 266 kvm_s390_cpu_feat_init(); 267 268 /* Register floating interrupt controller interface. */ 269 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC); 270 } 271 272 void kvm_arch_exit(void) 273 { 274 debug_unregister(kvm_s390_dbf); 275 } 276 277 /* Section: device related */ 278 long kvm_arch_dev_ioctl(struct file *filp, 279 unsigned int ioctl, unsigned long arg) 280 { 281 if (ioctl == KVM_S390_ENABLE_SIE) 282 return s390_enable_sie(); 283 return -EINVAL; 284 } 285 286 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 287 { 288 int r; 289 290 switch (ext) { 291 case KVM_CAP_S390_PSW: 292 case KVM_CAP_S390_GMAP: 293 case KVM_CAP_SYNC_MMU: 294 #ifdef CONFIG_KVM_S390_UCONTROL 295 case KVM_CAP_S390_UCONTROL: 296 #endif 297 case KVM_CAP_ASYNC_PF: 298 case KVM_CAP_SYNC_REGS: 299 case KVM_CAP_ONE_REG: 300 case KVM_CAP_ENABLE_CAP: 301 case KVM_CAP_S390_CSS_SUPPORT: 302 case KVM_CAP_IOEVENTFD: 303 case KVM_CAP_DEVICE_CTRL: 304 case KVM_CAP_ENABLE_CAP_VM: 305 case KVM_CAP_S390_IRQCHIP: 306 case KVM_CAP_VM_ATTRIBUTES: 307 case KVM_CAP_MP_STATE: 308 case KVM_CAP_S390_INJECT_IRQ: 309 case KVM_CAP_S390_USER_SIGP: 310 case KVM_CAP_S390_USER_STSI: 311 case KVM_CAP_S390_SKEYS: 312 case KVM_CAP_S390_IRQ_STATE: 313 r = 1; 314 break; 315 case KVM_CAP_S390_MEM_OP: 316 r = MEM_OP_MAX_SIZE; 317 break; 318 case KVM_CAP_NR_VCPUS: 319 case KVM_CAP_MAX_VCPUS: 320 r = KVM_S390_BSCA_CPU_SLOTS; 321 if (sclp.has_esca && sclp.has_64bscao) 322 r = KVM_S390_ESCA_CPU_SLOTS; 323 break; 324 case KVM_CAP_NR_MEMSLOTS: 325 r = KVM_USER_MEM_SLOTS; 326 break; 327 case KVM_CAP_S390_COW: 328 r = MACHINE_HAS_ESOP; 329 break; 330 case KVM_CAP_S390_VECTOR_REGISTERS: 331 r = MACHINE_HAS_VX; 332 break; 333 case KVM_CAP_S390_RI: 334 r = test_facility(64); 335 break; 336 default: 337 r = 0; 338 } 339 return r; 340 } 341 342 static void kvm_s390_sync_dirty_log(struct kvm *kvm, 343 struct kvm_memory_slot *memslot) 344 { 345 gfn_t cur_gfn, last_gfn; 346 unsigned long address; 347 struct gmap *gmap = kvm->arch.gmap; 348 349 /* Loop over all guest pages */ 350 last_gfn = memslot->base_gfn + memslot->npages; 351 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { 352 address = gfn_to_hva_memslot(memslot, cur_gfn); 353 354 if (test_and_clear_guest_dirty(gmap->mm, address)) 355 mark_page_dirty(kvm, cur_gfn); 356 if (fatal_signal_pending(current)) 357 return; 358 cond_resched(); 359 } 360 } 361 362 /* Section: vm related */ 363 static void sca_del_vcpu(struct kvm_vcpu *vcpu); 364 365 /* 366 * Get (and clear) the dirty memory log for a memory slot. 367 */ 368 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 369 struct kvm_dirty_log *log) 370 { 371 int r; 372 unsigned long n; 373 struct kvm_memslots *slots; 374 struct kvm_memory_slot *memslot; 375 int is_dirty = 0; 376 377 mutex_lock(&kvm->slots_lock); 378 379 r = -EINVAL; 380 if (log->slot >= KVM_USER_MEM_SLOTS) 381 goto out; 382 383 slots = kvm_memslots(kvm); 384 memslot = id_to_memslot(slots, log->slot); 385 r = -ENOENT; 386 if (!memslot->dirty_bitmap) 387 goto out; 388 389 kvm_s390_sync_dirty_log(kvm, memslot); 390 r = kvm_get_dirty_log(kvm, log, &is_dirty); 391 if (r) 392 goto out; 393 394 /* Clear the dirty log */ 395 if (is_dirty) { 396 n = kvm_dirty_bitmap_bytes(memslot); 397 memset(memslot->dirty_bitmap, 0, n); 398 } 399 r = 0; 400 out: 401 mutex_unlock(&kvm->slots_lock); 402 return r; 403 } 404 405 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) 406 { 407 int r; 408 409 if (cap->flags) 410 return -EINVAL; 411 412 switch (cap->cap) { 413 case KVM_CAP_S390_IRQCHIP: 414 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP"); 415 kvm->arch.use_irqchip = 1; 416 r = 0; 417 break; 418 case KVM_CAP_S390_USER_SIGP: 419 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP"); 420 kvm->arch.user_sigp = 1; 421 r = 0; 422 break; 423 case KVM_CAP_S390_VECTOR_REGISTERS: 424 mutex_lock(&kvm->lock); 425 if (atomic_read(&kvm->online_vcpus)) { 426 r = -EBUSY; 427 } else if (MACHINE_HAS_VX) { 428 set_kvm_facility(kvm->arch.model.fac_mask, 129); 429 set_kvm_facility(kvm->arch.model.fac_list, 129); 430 r = 0; 431 } else 432 r = -EINVAL; 433 mutex_unlock(&kvm->lock); 434 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s", 435 r ? "(not available)" : "(success)"); 436 break; 437 case KVM_CAP_S390_RI: 438 r = -EINVAL; 439 mutex_lock(&kvm->lock); 440 if (atomic_read(&kvm->online_vcpus)) { 441 r = -EBUSY; 442 } else if (test_facility(64)) { 443 set_kvm_facility(kvm->arch.model.fac_mask, 64); 444 set_kvm_facility(kvm->arch.model.fac_list, 64); 445 r = 0; 446 } 447 mutex_unlock(&kvm->lock); 448 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s", 449 r ? "(not available)" : "(success)"); 450 break; 451 case KVM_CAP_S390_USER_STSI: 452 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); 453 kvm->arch.user_stsi = 1; 454 r = 0; 455 break; 456 default: 457 r = -EINVAL; 458 break; 459 } 460 return r; 461 } 462 463 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 464 { 465 int ret; 466 467 switch (attr->attr) { 468 case KVM_S390_VM_MEM_LIMIT_SIZE: 469 ret = 0; 470 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes", 471 kvm->arch.mem_limit); 472 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr)) 473 ret = -EFAULT; 474 break; 475 default: 476 ret = -ENXIO; 477 break; 478 } 479 return ret; 480 } 481 482 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 483 { 484 int ret; 485 unsigned int idx; 486 switch (attr->attr) { 487 case KVM_S390_VM_MEM_ENABLE_CMMA: 488 ret = -ENXIO; 489 if (!sclp.has_cmma) 490 break; 491 492 ret = -EBUSY; 493 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support"); 494 mutex_lock(&kvm->lock); 495 if (atomic_read(&kvm->online_vcpus) == 0) { 496 kvm->arch.use_cmma = 1; 497 ret = 0; 498 } 499 mutex_unlock(&kvm->lock); 500 break; 501 case KVM_S390_VM_MEM_CLR_CMMA: 502 ret = -ENXIO; 503 if (!sclp.has_cmma) 504 break; 505 ret = -EINVAL; 506 if (!kvm->arch.use_cmma) 507 break; 508 509 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states"); 510 mutex_lock(&kvm->lock); 511 idx = srcu_read_lock(&kvm->srcu); 512 s390_reset_cmma(kvm->arch.gmap->mm); 513 srcu_read_unlock(&kvm->srcu, idx); 514 mutex_unlock(&kvm->lock); 515 ret = 0; 516 break; 517 case KVM_S390_VM_MEM_LIMIT_SIZE: { 518 unsigned long new_limit; 519 520 if (kvm_is_ucontrol(kvm)) 521 return -EINVAL; 522 523 if (get_user(new_limit, (u64 __user *)attr->addr)) 524 return -EFAULT; 525 526 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT && 527 new_limit > kvm->arch.mem_limit) 528 return -E2BIG; 529 530 if (!new_limit) 531 return -EINVAL; 532 533 /* gmap_alloc takes last usable address */ 534 if (new_limit != KVM_S390_NO_MEM_LIMIT) 535 new_limit -= 1; 536 537 ret = -EBUSY; 538 mutex_lock(&kvm->lock); 539 if (atomic_read(&kvm->online_vcpus) == 0) { 540 /* gmap_alloc will round the limit up */ 541 struct gmap *new = gmap_alloc(current->mm, new_limit); 542 543 if (!new) { 544 ret = -ENOMEM; 545 } else { 546 gmap_free(kvm->arch.gmap); 547 new->private = kvm; 548 kvm->arch.gmap = new; 549 ret = 0; 550 } 551 } 552 mutex_unlock(&kvm->lock); 553 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit); 554 VM_EVENT(kvm, 3, "New guest asce: 0x%pK", 555 (void *) kvm->arch.gmap->asce); 556 break; 557 } 558 default: 559 ret = -ENXIO; 560 break; 561 } 562 return ret; 563 } 564 565 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu); 566 567 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr) 568 { 569 struct kvm_vcpu *vcpu; 570 int i; 571 572 if (!test_kvm_facility(kvm, 76)) 573 return -EINVAL; 574 575 mutex_lock(&kvm->lock); 576 switch (attr->attr) { 577 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 578 get_random_bytes( 579 kvm->arch.crypto.crycb->aes_wrapping_key_mask, 580 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 581 kvm->arch.crypto.aes_kw = 1; 582 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support"); 583 break; 584 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 585 get_random_bytes( 586 kvm->arch.crypto.crycb->dea_wrapping_key_mask, 587 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 588 kvm->arch.crypto.dea_kw = 1; 589 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support"); 590 break; 591 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 592 kvm->arch.crypto.aes_kw = 0; 593 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0, 594 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 595 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support"); 596 break; 597 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 598 kvm->arch.crypto.dea_kw = 0; 599 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0, 600 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 601 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support"); 602 break; 603 default: 604 mutex_unlock(&kvm->lock); 605 return -ENXIO; 606 } 607 608 kvm_for_each_vcpu(i, vcpu, kvm) { 609 kvm_s390_vcpu_crypto_setup(vcpu); 610 exit_sie(vcpu); 611 } 612 mutex_unlock(&kvm->lock); 613 return 0; 614 } 615 616 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 617 { 618 u8 gtod_high; 619 620 if (copy_from_user(>od_high, (void __user *)attr->addr, 621 sizeof(gtod_high))) 622 return -EFAULT; 623 624 if (gtod_high != 0) 625 return -EINVAL; 626 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high); 627 628 return 0; 629 } 630 631 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 632 { 633 u64 gtod; 634 635 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod))) 636 return -EFAULT; 637 638 kvm_s390_set_tod_clock(kvm, gtod); 639 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod); 640 return 0; 641 } 642 643 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr) 644 { 645 int ret; 646 647 if (attr->flags) 648 return -EINVAL; 649 650 switch (attr->attr) { 651 case KVM_S390_VM_TOD_HIGH: 652 ret = kvm_s390_set_tod_high(kvm, attr); 653 break; 654 case KVM_S390_VM_TOD_LOW: 655 ret = kvm_s390_set_tod_low(kvm, attr); 656 break; 657 default: 658 ret = -ENXIO; 659 break; 660 } 661 return ret; 662 } 663 664 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 665 { 666 u8 gtod_high = 0; 667 668 if (copy_to_user((void __user *)attr->addr, >od_high, 669 sizeof(gtod_high))) 670 return -EFAULT; 671 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high); 672 673 return 0; 674 } 675 676 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 677 { 678 u64 gtod; 679 680 gtod = kvm_s390_get_tod_clock_fast(kvm); 681 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod))) 682 return -EFAULT; 683 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod); 684 685 return 0; 686 } 687 688 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr) 689 { 690 int ret; 691 692 if (attr->flags) 693 return -EINVAL; 694 695 switch (attr->attr) { 696 case KVM_S390_VM_TOD_HIGH: 697 ret = kvm_s390_get_tod_high(kvm, attr); 698 break; 699 case KVM_S390_VM_TOD_LOW: 700 ret = kvm_s390_get_tod_low(kvm, attr); 701 break; 702 default: 703 ret = -ENXIO; 704 break; 705 } 706 return ret; 707 } 708 709 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) 710 { 711 struct kvm_s390_vm_cpu_processor *proc; 712 u16 lowest_ibc, unblocked_ibc; 713 int ret = 0; 714 715 mutex_lock(&kvm->lock); 716 if (atomic_read(&kvm->online_vcpus)) { 717 ret = -EBUSY; 718 goto out; 719 } 720 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 721 if (!proc) { 722 ret = -ENOMEM; 723 goto out; 724 } 725 if (!copy_from_user(proc, (void __user *)attr->addr, 726 sizeof(*proc))) { 727 kvm->arch.model.cpuid = proc->cpuid; 728 lowest_ibc = sclp.ibc >> 16 & 0xfff; 729 unblocked_ibc = sclp.ibc & 0xfff; 730 if (lowest_ibc) { 731 if (proc->ibc > unblocked_ibc) 732 kvm->arch.model.ibc = unblocked_ibc; 733 else if (proc->ibc < lowest_ibc) 734 kvm->arch.model.ibc = lowest_ibc; 735 else 736 kvm->arch.model.ibc = proc->ibc; 737 } 738 memcpy(kvm->arch.model.fac_list, proc->fac_list, 739 S390_ARCH_FAC_LIST_SIZE_BYTE); 740 } else 741 ret = -EFAULT; 742 kfree(proc); 743 out: 744 mutex_unlock(&kvm->lock); 745 return ret; 746 } 747 748 static int kvm_s390_set_processor_feat(struct kvm *kvm, 749 struct kvm_device_attr *attr) 750 { 751 struct kvm_s390_vm_cpu_feat data; 752 int ret = -EBUSY; 753 754 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data))) 755 return -EFAULT; 756 if (!bitmap_subset((unsigned long *) data.feat, 757 kvm_s390_available_cpu_feat, 758 KVM_S390_VM_CPU_FEAT_NR_BITS)) 759 return -EINVAL; 760 761 mutex_lock(&kvm->lock); 762 if (!atomic_read(&kvm->online_vcpus)) { 763 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat, 764 KVM_S390_VM_CPU_FEAT_NR_BITS); 765 ret = 0; 766 } 767 mutex_unlock(&kvm->lock); 768 return ret; 769 } 770 771 static int kvm_s390_set_processor_subfunc(struct kvm *kvm, 772 struct kvm_device_attr *attr) 773 { 774 /* 775 * Once supported by kernel + hw, we have to store the subfunctions 776 * in kvm->arch and remember that user space configured them. 777 */ 778 return -ENXIO; 779 } 780 781 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 782 { 783 int ret = -ENXIO; 784 785 switch (attr->attr) { 786 case KVM_S390_VM_CPU_PROCESSOR: 787 ret = kvm_s390_set_processor(kvm, attr); 788 break; 789 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 790 ret = kvm_s390_set_processor_feat(kvm, attr); 791 break; 792 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 793 ret = kvm_s390_set_processor_subfunc(kvm, attr); 794 break; 795 } 796 return ret; 797 } 798 799 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) 800 { 801 struct kvm_s390_vm_cpu_processor *proc; 802 int ret = 0; 803 804 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 805 if (!proc) { 806 ret = -ENOMEM; 807 goto out; 808 } 809 proc->cpuid = kvm->arch.model.cpuid; 810 proc->ibc = kvm->arch.model.ibc; 811 memcpy(&proc->fac_list, kvm->arch.model.fac_list, 812 S390_ARCH_FAC_LIST_SIZE_BYTE); 813 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc))) 814 ret = -EFAULT; 815 kfree(proc); 816 out: 817 return ret; 818 } 819 820 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) 821 { 822 struct kvm_s390_vm_cpu_machine *mach; 823 int ret = 0; 824 825 mach = kzalloc(sizeof(*mach), GFP_KERNEL); 826 if (!mach) { 827 ret = -ENOMEM; 828 goto out; 829 } 830 get_cpu_id((struct cpuid *) &mach->cpuid); 831 mach->ibc = sclp.ibc; 832 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask, 833 S390_ARCH_FAC_LIST_SIZE_BYTE); 834 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list, 835 S390_ARCH_FAC_LIST_SIZE_BYTE); 836 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach))) 837 ret = -EFAULT; 838 kfree(mach); 839 out: 840 return ret; 841 } 842 843 static int kvm_s390_get_processor_feat(struct kvm *kvm, 844 struct kvm_device_attr *attr) 845 { 846 struct kvm_s390_vm_cpu_feat data; 847 848 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat, 849 KVM_S390_VM_CPU_FEAT_NR_BITS); 850 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data))) 851 return -EFAULT; 852 return 0; 853 } 854 855 static int kvm_s390_get_machine_feat(struct kvm *kvm, 856 struct kvm_device_attr *attr) 857 { 858 struct kvm_s390_vm_cpu_feat data; 859 860 bitmap_copy((unsigned long *) data.feat, 861 kvm_s390_available_cpu_feat, 862 KVM_S390_VM_CPU_FEAT_NR_BITS); 863 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data))) 864 return -EFAULT; 865 return 0; 866 } 867 868 static int kvm_s390_get_processor_subfunc(struct kvm *kvm, 869 struct kvm_device_attr *attr) 870 { 871 /* 872 * Once we can actually configure subfunctions (kernel + hw support), 873 * we have to check if they were already set by user space, if so copy 874 * them from kvm->arch. 875 */ 876 return -ENXIO; 877 } 878 879 static int kvm_s390_get_machine_subfunc(struct kvm *kvm, 880 struct kvm_device_attr *attr) 881 { 882 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc, 883 sizeof(struct kvm_s390_vm_cpu_subfunc))) 884 return -EFAULT; 885 return 0; 886 } 887 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 888 { 889 int ret = -ENXIO; 890 891 switch (attr->attr) { 892 case KVM_S390_VM_CPU_PROCESSOR: 893 ret = kvm_s390_get_processor(kvm, attr); 894 break; 895 case KVM_S390_VM_CPU_MACHINE: 896 ret = kvm_s390_get_machine(kvm, attr); 897 break; 898 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 899 ret = kvm_s390_get_processor_feat(kvm, attr); 900 break; 901 case KVM_S390_VM_CPU_MACHINE_FEAT: 902 ret = kvm_s390_get_machine_feat(kvm, attr); 903 break; 904 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 905 ret = kvm_s390_get_processor_subfunc(kvm, attr); 906 break; 907 case KVM_S390_VM_CPU_MACHINE_SUBFUNC: 908 ret = kvm_s390_get_machine_subfunc(kvm, attr); 909 break; 910 } 911 return ret; 912 } 913 914 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 915 { 916 int ret; 917 918 switch (attr->group) { 919 case KVM_S390_VM_MEM_CTRL: 920 ret = kvm_s390_set_mem_control(kvm, attr); 921 break; 922 case KVM_S390_VM_TOD: 923 ret = kvm_s390_set_tod(kvm, attr); 924 break; 925 case KVM_S390_VM_CPU_MODEL: 926 ret = kvm_s390_set_cpu_model(kvm, attr); 927 break; 928 case KVM_S390_VM_CRYPTO: 929 ret = kvm_s390_vm_set_crypto(kvm, attr); 930 break; 931 default: 932 ret = -ENXIO; 933 break; 934 } 935 936 return ret; 937 } 938 939 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr) 940 { 941 int ret; 942 943 switch (attr->group) { 944 case KVM_S390_VM_MEM_CTRL: 945 ret = kvm_s390_get_mem_control(kvm, attr); 946 break; 947 case KVM_S390_VM_TOD: 948 ret = kvm_s390_get_tod(kvm, attr); 949 break; 950 case KVM_S390_VM_CPU_MODEL: 951 ret = kvm_s390_get_cpu_model(kvm, attr); 952 break; 953 default: 954 ret = -ENXIO; 955 break; 956 } 957 958 return ret; 959 } 960 961 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 962 { 963 int ret; 964 965 switch (attr->group) { 966 case KVM_S390_VM_MEM_CTRL: 967 switch (attr->attr) { 968 case KVM_S390_VM_MEM_ENABLE_CMMA: 969 case KVM_S390_VM_MEM_CLR_CMMA: 970 ret = sclp.has_cmma ? 0 : -ENXIO; 971 break; 972 case KVM_S390_VM_MEM_LIMIT_SIZE: 973 ret = 0; 974 break; 975 default: 976 ret = -ENXIO; 977 break; 978 } 979 break; 980 case KVM_S390_VM_TOD: 981 switch (attr->attr) { 982 case KVM_S390_VM_TOD_LOW: 983 case KVM_S390_VM_TOD_HIGH: 984 ret = 0; 985 break; 986 default: 987 ret = -ENXIO; 988 break; 989 } 990 break; 991 case KVM_S390_VM_CPU_MODEL: 992 switch (attr->attr) { 993 case KVM_S390_VM_CPU_PROCESSOR: 994 case KVM_S390_VM_CPU_MACHINE: 995 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 996 case KVM_S390_VM_CPU_MACHINE_FEAT: 997 case KVM_S390_VM_CPU_MACHINE_SUBFUNC: 998 ret = 0; 999 break; 1000 /* configuring subfunctions is not supported yet */ 1001 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1002 default: 1003 ret = -ENXIO; 1004 break; 1005 } 1006 break; 1007 case KVM_S390_VM_CRYPTO: 1008 switch (attr->attr) { 1009 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 1010 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 1011 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 1012 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 1013 ret = 0; 1014 break; 1015 default: 1016 ret = -ENXIO; 1017 break; 1018 } 1019 break; 1020 default: 1021 ret = -ENXIO; 1022 break; 1023 } 1024 1025 return ret; 1026 } 1027 1028 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) 1029 { 1030 uint8_t *keys; 1031 uint64_t hva; 1032 unsigned long curkey; 1033 int i, r = 0; 1034 1035 if (args->flags != 0) 1036 return -EINVAL; 1037 1038 /* Is this guest using storage keys? */ 1039 if (!mm_use_skey(current->mm)) 1040 return KVM_S390_GET_SKEYS_NONE; 1041 1042 /* Enforce sane limit on memory allocation */ 1043 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) 1044 return -EINVAL; 1045 1046 keys = kmalloc_array(args->count, sizeof(uint8_t), 1047 GFP_KERNEL | __GFP_NOWARN); 1048 if (!keys) 1049 keys = vmalloc(sizeof(uint8_t) * args->count); 1050 if (!keys) 1051 return -ENOMEM; 1052 1053 for (i = 0; i < args->count; i++) { 1054 hva = gfn_to_hva(kvm, args->start_gfn + i); 1055 if (kvm_is_error_hva(hva)) { 1056 r = -EFAULT; 1057 goto out; 1058 } 1059 1060 curkey = get_guest_storage_key(current->mm, hva); 1061 if (IS_ERR_VALUE(curkey)) { 1062 r = curkey; 1063 goto out; 1064 } 1065 keys[i] = curkey; 1066 } 1067 1068 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys, 1069 sizeof(uint8_t) * args->count); 1070 if (r) 1071 r = -EFAULT; 1072 out: 1073 kvfree(keys); 1074 return r; 1075 } 1076 1077 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) 1078 { 1079 uint8_t *keys; 1080 uint64_t hva; 1081 int i, r = 0; 1082 1083 if (args->flags != 0) 1084 return -EINVAL; 1085 1086 /* Enforce sane limit on memory allocation */ 1087 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) 1088 return -EINVAL; 1089 1090 keys = kmalloc_array(args->count, sizeof(uint8_t), 1091 GFP_KERNEL | __GFP_NOWARN); 1092 if (!keys) 1093 keys = vmalloc(sizeof(uint8_t) * args->count); 1094 if (!keys) 1095 return -ENOMEM; 1096 1097 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr, 1098 sizeof(uint8_t) * args->count); 1099 if (r) { 1100 r = -EFAULT; 1101 goto out; 1102 } 1103 1104 /* Enable storage key handling for the guest */ 1105 r = s390_enable_skey(); 1106 if (r) 1107 goto out; 1108 1109 for (i = 0; i < args->count; i++) { 1110 hva = gfn_to_hva(kvm, args->start_gfn + i); 1111 if (kvm_is_error_hva(hva)) { 1112 r = -EFAULT; 1113 goto out; 1114 } 1115 1116 /* Lowest order bit is reserved */ 1117 if (keys[i] & 0x01) { 1118 r = -EINVAL; 1119 goto out; 1120 } 1121 1122 r = set_guest_storage_key(current->mm, hva, 1123 (unsigned long)keys[i], 0); 1124 if (r) 1125 goto out; 1126 } 1127 out: 1128 kvfree(keys); 1129 return r; 1130 } 1131 1132 long kvm_arch_vm_ioctl(struct file *filp, 1133 unsigned int ioctl, unsigned long arg) 1134 { 1135 struct kvm *kvm = filp->private_data; 1136 void __user *argp = (void __user *)arg; 1137 struct kvm_device_attr attr; 1138 int r; 1139 1140 switch (ioctl) { 1141 case KVM_S390_INTERRUPT: { 1142 struct kvm_s390_interrupt s390int; 1143 1144 r = -EFAULT; 1145 if (copy_from_user(&s390int, argp, sizeof(s390int))) 1146 break; 1147 r = kvm_s390_inject_vm(kvm, &s390int); 1148 break; 1149 } 1150 case KVM_ENABLE_CAP: { 1151 struct kvm_enable_cap cap; 1152 r = -EFAULT; 1153 if (copy_from_user(&cap, argp, sizeof(cap))) 1154 break; 1155 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 1156 break; 1157 } 1158 case KVM_CREATE_IRQCHIP: { 1159 struct kvm_irq_routing_entry routing; 1160 1161 r = -EINVAL; 1162 if (kvm->arch.use_irqchip) { 1163 /* Set up dummy routing. */ 1164 memset(&routing, 0, sizeof(routing)); 1165 r = kvm_set_irq_routing(kvm, &routing, 0, 0); 1166 } 1167 break; 1168 } 1169 case KVM_SET_DEVICE_ATTR: { 1170 r = -EFAULT; 1171 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 1172 break; 1173 r = kvm_s390_vm_set_attr(kvm, &attr); 1174 break; 1175 } 1176 case KVM_GET_DEVICE_ATTR: { 1177 r = -EFAULT; 1178 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 1179 break; 1180 r = kvm_s390_vm_get_attr(kvm, &attr); 1181 break; 1182 } 1183 case KVM_HAS_DEVICE_ATTR: { 1184 r = -EFAULT; 1185 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 1186 break; 1187 r = kvm_s390_vm_has_attr(kvm, &attr); 1188 break; 1189 } 1190 case KVM_S390_GET_SKEYS: { 1191 struct kvm_s390_skeys args; 1192 1193 r = -EFAULT; 1194 if (copy_from_user(&args, argp, 1195 sizeof(struct kvm_s390_skeys))) 1196 break; 1197 r = kvm_s390_get_skeys(kvm, &args); 1198 break; 1199 } 1200 case KVM_S390_SET_SKEYS: { 1201 struct kvm_s390_skeys args; 1202 1203 r = -EFAULT; 1204 if (copy_from_user(&args, argp, 1205 sizeof(struct kvm_s390_skeys))) 1206 break; 1207 r = kvm_s390_set_skeys(kvm, &args); 1208 break; 1209 } 1210 default: 1211 r = -ENOTTY; 1212 } 1213 1214 return r; 1215 } 1216 1217 static int kvm_s390_query_ap_config(u8 *config) 1218 { 1219 u32 fcn_code = 0x04000000UL; 1220 u32 cc = 0; 1221 1222 memset(config, 0, 128); 1223 asm volatile( 1224 "lgr 0,%1\n" 1225 "lgr 2,%2\n" 1226 ".long 0xb2af0000\n" /* PQAP(QCI) */ 1227 "0: ipm %0\n" 1228 "srl %0,28\n" 1229 "1:\n" 1230 EX_TABLE(0b, 1b) 1231 : "+r" (cc) 1232 : "r" (fcn_code), "r" (config) 1233 : "cc", "0", "2", "memory" 1234 ); 1235 1236 return cc; 1237 } 1238 1239 static int kvm_s390_apxa_installed(void) 1240 { 1241 u8 config[128]; 1242 int cc; 1243 1244 if (test_facility(12)) { 1245 cc = kvm_s390_query_ap_config(config); 1246 1247 if (cc) 1248 pr_err("PQAP(QCI) failed with cc=%d", cc); 1249 else 1250 return config[0] & 0x40; 1251 } 1252 1253 return 0; 1254 } 1255 1256 static void kvm_s390_set_crycb_format(struct kvm *kvm) 1257 { 1258 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb; 1259 1260 if (kvm_s390_apxa_installed()) 1261 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2; 1262 else 1263 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1; 1264 } 1265 1266 static u64 kvm_s390_get_initial_cpuid(void) 1267 { 1268 struct cpuid cpuid; 1269 1270 get_cpu_id(&cpuid); 1271 cpuid.version = 0xff; 1272 return *((u64 *) &cpuid); 1273 } 1274 1275 static void kvm_s390_crypto_init(struct kvm *kvm) 1276 { 1277 if (!test_kvm_facility(kvm, 76)) 1278 return; 1279 1280 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb; 1281 kvm_s390_set_crycb_format(kvm); 1282 1283 /* Enable AES/DEA protected key functions by default */ 1284 kvm->arch.crypto.aes_kw = 1; 1285 kvm->arch.crypto.dea_kw = 1; 1286 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 1287 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 1288 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 1289 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 1290 } 1291 1292 static void sca_dispose(struct kvm *kvm) 1293 { 1294 if (kvm->arch.use_esca) 1295 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block)); 1296 else 1297 free_page((unsigned long)(kvm->arch.sca)); 1298 kvm->arch.sca = NULL; 1299 } 1300 1301 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 1302 { 1303 gfp_t alloc_flags = GFP_KERNEL; 1304 int i, rc; 1305 char debug_name[16]; 1306 static unsigned long sca_offset; 1307 1308 rc = -EINVAL; 1309 #ifdef CONFIG_KVM_S390_UCONTROL 1310 if (type & ~KVM_VM_S390_UCONTROL) 1311 goto out_err; 1312 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 1313 goto out_err; 1314 #else 1315 if (type) 1316 goto out_err; 1317 #endif 1318 1319 rc = s390_enable_sie(); 1320 if (rc) 1321 goto out_err; 1322 1323 rc = -ENOMEM; 1324 1325 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500); 1326 1327 kvm->arch.use_esca = 0; /* start with basic SCA */ 1328 if (!sclp.has_64bscao) 1329 alloc_flags |= GFP_DMA; 1330 rwlock_init(&kvm->arch.sca_lock); 1331 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags); 1332 if (!kvm->arch.sca) 1333 goto out_err; 1334 spin_lock(&kvm_lock); 1335 sca_offset += 16; 1336 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE) 1337 sca_offset = 0; 1338 kvm->arch.sca = (struct bsca_block *) 1339 ((char *) kvm->arch.sca + sca_offset); 1340 spin_unlock(&kvm_lock); 1341 1342 sprintf(debug_name, "kvm-%u", current->pid); 1343 1344 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long)); 1345 if (!kvm->arch.dbf) 1346 goto out_err; 1347 1348 kvm->arch.sie_page2 = 1349 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1350 if (!kvm->arch.sie_page2) 1351 goto out_err; 1352 1353 /* Populate the facility mask initially. */ 1354 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list, 1355 S390_ARCH_FAC_LIST_SIZE_BYTE); 1356 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) { 1357 if (i < kvm_s390_fac_list_mask_size()) 1358 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i]; 1359 else 1360 kvm->arch.model.fac_mask[i] = 0UL; 1361 } 1362 1363 /* Populate the facility list initially. */ 1364 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; 1365 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask, 1366 S390_ARCH_FAC_LIST_SIZE_BYTE); 1367 1368 set_kvm_facility(kvm->arch.model.fac_mask, 74); 1369 set_kvm_facility(kvm->arch.model.fac_list, 74); 1370 1371 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid(); 1372 kvm->arch.model.ibc = sclp.ibc & 0x0fff; 1373 1374 kvm_s390_crypto_init(kvm); 1375 1376 spin_lock_init(&kvm->arch.float_int.lock); 1377 for (i = 0; i < FIRQ_LIST_COUNT; i++) 1378 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); 1379 init_waitqueue_head(&kvm->arch.ipte_wq); 1380 mutex_init(&kvm->arch.ipte_mutex); 1381 1382 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 1383 VM_EVENT(kvm, 3, "vm created with type %lu", type); 1384 1385 if (type & KVM_VM_S390_UCONTROL) { 1386 kvm->arch.gmap = NULL; 1387 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT; 1388 } else { 1389 if (sclp.hamax == U64_MAX) 1390 kvm->arch.mem_limit = TASK_MAX_SIZE; 1391 else 1392 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE, 1393 sclp.hamax + 1); 1394 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1); 1395 if (!kvm->arch.gmap) 1396 goto out_err; 1397 kvm->arch.gmap->private = kvm; 1398 kvm->arch.gmap->pfault_enabled = 0; 1399 } 1400 1401 kvm->arch.css_support = 0; 1402 kvm->arch.use_irqchip = 0; 1403 kvm->arch.epoch = 0; 1404 1405 spin_lock_init(&kvm->arch.start_stop_lock); 1406 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid); 1407 1408 return 0; 1409 out_err: 1410 free_page((unsigned long)kvm->arch.sie_page2); 1411 debug_unregister(kvm->arch.dbf); 1412 sca_dispose(kvm); 1413 KVM_EVENT(3, "creation of vm failed: %d", rc); 1414 return rc; 1415 } 1416 1417 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 1418 { 1419 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 1420 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 1421 kvm_s390_clear_local_irqs(vcpu); 1422 kvm_clear_async_pf_completion_queue(vcpu); 1423 if (!kvm_is_ucontrol(vcpu->kvm)) 1424 sca_del_vcpu(vcpu); 1425 1426 if (kvm_is_ucontrol(vcpu->kvm)) 1427 gmap_free(vcpu->arch.gmap); 1428 1429 if (vcpu->kvm->arch.use_cmma) 1430 kvm_s390_vcpu_unsetup_cmma(vcpu); 1431 free_page((unsigned long)(vcpu->arch.sie_block)); 1432 1433 kvm_vcpu_uninit(vcpu); 1434 kmem_cache_free(kvm_vcpu_cache, vcpu); 1435 } 1436 1437 static void kvm_free_vcpus(struct kvm *kvm) 1438 { 1439 unsigned int i; 1440 struct kvm_vcpu *vcpu; 1441 1442 kvm_for_each_vcpu(i, vcpu, kvm) 1443 kvm_arch_vcpu_destroy(vcpu); 1444 1445 mutex_lock(&kvm->lock); 1446 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 1447 kvm->vcpus[i] = NULL; 1448 1449 atomic_set(&kvm->online_vcpus, 0); 1450 mutex_unlock(&kvm->lock); 1451 } 1452 1453 void kvm_arch_destroy_vm(struct kvm *kvm) 1454 { 1455 kvm_free_vcpus(kvm); 1456 sca_dispose(kvm); 1457 debug_unregister(kvm->arch.dbf); 1458 free_page((unsigned long)kvm->arch.sie_page2); 1459 if (!kvm_is_ucontrol(kvm)) 1460 gmap_free(kvm->arch.gmap); 1461 kvm_s390_destroy_adapters(kvm); 1462 kvm_s390_clear_float_irqs(kvm); 1463 KVM_EVENT(3, "vm 0x%pK destroyed", kvm); 1464 } 1465 1466 /* Section: vcpu related */ 1467 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) 1468 { 1469 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL); 1470 if (!vcpu->arch.gmap) 1471 return -ENOMEM; 1472 vcpu->arch.gmap->private = vcpu->kvm; 1473 1474 return 0; 1475 } 1476 1477 static void sca_del_vcpu(struct kvm_vcpu *vcpu) 1478 { 1479 read_lock(&vcpu->kvm->arch.sca_lock); 1480 if (vcpu->kvm->arch.use_esca) { 1481 struct esca_block *sca = vcpu->kvm->arch.sca; 1482 1483 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn); 1484 sca->cpu[vcpu->vcpu_id].sda = 0; 1485 } else { 1486 struct bsca_block *sca = vcpu->kvm->arch.sca; 1487 1488 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn); 1489 sca->cpu[vcpu->vcpu_id].sda = 0; 1490 } 1491 read_unlock(&vcpu->kvm->arch.sca_lock); 1492 } 1493 1494 static void sca_add_vcpu(struct kvm_vcpu *vcpu) 1495 { 1496 read_lock(&vcpu->kvm->arch.sca_lock); 1497 if (vcpu->kvm->arch.use_esca) { 1498 struct esca_block *sca = vcpu->kvm->arch.sca; 1499 1500 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block; 1501 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 1502 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU; 1503 vcpu->arch.sie_block->ecb2 |= 0x04U; 1504 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn); 1505 } else { 1506 struct bsca_block *sca = vcpu->kvm->arch.sca; 1507 1508 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block; 1509 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 1510 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca; 1511 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn); 1512 } 1513 read_unlock(&vcpu->kvm->arch.sca_lock); 1514 } 1515 1516 /* Basic SCA to Extended SCA data copy routines */ 1517 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s) 1518 { 1519 d->sda = s->sda; 1520 d->sigp_ctrl.c = s->sigp_ctrl.c; 1521 d->sigp_ctrl.scn = s->sigp_ctrl.scn; 1522 } 1523 1524 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s) 1525 { 1526 int i; 1527 1528 d->ipte_control = s->ipte_control; 1529 d->mcn[0] = s->mcn; 1530 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++) 1531 sca_copy_entry(&d->cpu[i], &s->cpu[i]); 1532 } 1533 1534 static int sca_switch_to_extended(struct kvm *kvm) 1535 { 1536 struct bsca_block *old_sca = kvm->arch.sca; 1537 struct esca_block *new_sca; 1538 struct kvm_vcpu *vcpu; 1539 unsigned int vcpu_idx; 1540 u32 scaol, scaoh; 1541 1542 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO); 1543 if (!new_sca) 1544 return -ENOMEM; 1545 1546 scaoh = (u32)((u64)(new_sca) >> 32); 1547 scaol = (u32)(u64)(new_sca) & ~0x3fU; 1548 1549 kvm_s390_vcpu_block_all(kvm); 1550 write_lock(&kvm->arch.sca_lock); 1551 1552 sca_copy_b_to_e(new_sca, old_sca); 1553 1554 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) { 1555 vcpu->arch.sie_block->scaoh = scaoh; 1556 vcpu->arch.sie_block->scaol = scaol; 1557 vcpu->arch.sie_block->ecb2 |= 0x04U; 1558 } 1559 kvm->arch.sca = new_sca; 1560 kvm->arch.use_esca = 1; 1561 1562 write_unlock(&kvm->arch.sca_lock); 1563 kvm_s390_vcpu_unblock_all(kvm); 1564 1565 free_page((unsigned long)old_sca); 1566 1567 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)", 1568 old_sca, kvm->arch.sca); 1569 return 0; 1570 } 1571 1572 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id) 1573 { 1574 int rc; 1575 1576 if (id < KVM_S390_BSCA_CPU_SLOTS) 1577 return true; 1578 if (!sclp.has_esca || !sclp.has_64bscao) 1579 return false; 1580 1581 mutex_lock(&kvm->lock); 1582 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm); 1583 mutex_unlock(&kvm->lock); 1584 1585 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS; 1586 } 1587 1588 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 1589 { 1590 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 1591 kvm_clear_async_pf_completion_queue(vcpu); 1592 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 1593 KVM_SYNC_GPRS | 1594 KVM_SYNC_ACRS | 1595 KVM_SYNC_CRS | 1596 KVM_SYNC_ARCH0 | 1597 KVM_SYNC_PFAULT; 1598 if (test_kvm_facility(vcpu->kvm, 64)) 1599 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB; 1600 /* fprs can be synchronized via vrs, even if the guest has no vx. With 1601 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format. 1602 */ 1603 if (MACHINE_HAS_VX) 1604 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS; 1605 else 1606 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS; 1607 1608 if (kvm_is_ucontrol(vcpu->kvm)) 1609 return __kvm_ucontrol_vcpu_init(vcpu); 1610 1611 return 0; 1612 } 1613 1614 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 1615 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu) 1616 { 1617 WARN_ON_ONCE(vcpu->arch.cputm_start != 0); 1618 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 1619 vcpu->arch.cputm_start = get_tod_clock_fast(); 1620 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 1621 } 1622 1623 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 1624 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu) 1625 { 1626 WARN_ON_ONCE(vcpu->arch.cputm_start == 0); 1627 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 1628 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start; 1629 vcpu->arch.cputm_start = 0; 1630 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 1631 } 1632 1633 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 1634 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 1635 { 1636 WARN_ON_ONCE(vcpu->arch.cputm_enabled); 1637 vcpu->arch.cputm_enabled = true; 1638 __start_cpu_timer_accounting(vcpu); 1639 } 1640 1641 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 1642 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 1643 { 1644 WARN_ON_ONCE(!vcpu->arch.cputm_enabled); 1645 __stop_cpu_timer_accounting(vcpu); 1646 vcpu->arch.cputm_enabled = false; 1647 } 1648 1649 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 1650 { 1651 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 1652 __enable_cpu_timer_accounting(vcpu); 1653 preempt_enable(); 1654 } 1655 1656 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 1657 { 1658 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 1659 __disable_cpu_timer_accounting(vcpu); 1660 preempt_enable(); 1661 } 1662 1663 /* set the cpu timer - may only be called from the VCPU thread itself */ 1664 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm) 1665 { 1666 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 1667 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 1668 if (vcpu->arch.cputm_enabled) 1669 vcpu->arch.cputm_start = get_tod_clock_fast(); 1670 vcpu->arch.sie_block->cputm = cputm; 1671 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 1672 preempt_enable(); 1673 } 1674 1675 /* update and get the cpu timer - can also be called from other VCPU threads */ 1676 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu) 1677 { 1678 unsigned int seq; 1679 __u64 value; 1680 1681 if (unlikely(!vcpu->arch.cputm_enabled)) 1682 return vcpu->arch.sie_block->cputm; 1683 1684 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 1685 do { 1686 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount); 1687 /* 1688 * If the writer would ever execute a read in the critical 1689 * section, e.g. in irq context, we have a deadlock. 1690 */ 1691 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu); 1692 value = vcpu->arch.sie_block->cputm; 1693 /* if cputm_start is 0, accounting is being started/stopped */ 1694 if (likely(vcpu->arch.cputm_start)) 1695 value -= get_tod_clock_fast() - vcpu->arch.cputm_start; 1696 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1)); 1697 preempt_enable(); 1698 return value; 1699 } 1700 1701 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1702 { 1703 /* Save host register state */ 1704 save_fpu_regs(); 1705 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; 1706 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; 1707 1708 if (MACHINE_HAS_VX) 1709 current->thread.fpu.regs = vcpu->run->s.regs.vrs; 1710 else 1711 current->thread.fpu.regs = vcpu->run->s.regs.fprs; 1712 current->thread.fpu.fpc = vcpu->run->s.regs.fpc; 1713 if (test_fp_ctl(current->thread.fpu.fpc)) 1714 /* User space provided an invalid FPC, let's clear it */ 1715 current->thread.fpu.fpc = 0; 1716 1717 save_access_regs(vcpu->arch.host_acrs); 1718 restore_access_regs(vcpu->run->s.regs.acrs); 1719 gmap_enable(vcpu->arch.gmap); 1720 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1721 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 1722 __start_cpu_timer_accounting(vcpu); 1723 vcpu->cpu = cpu; 1724 } 1725 1726 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1727 { 1728 vcpu->cpu = -1; 1729 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 1730 __stop_cpu_timer_accounting(vcpu); 1731 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1732 gmap_disable(vcpu->arch.gmap); 1733 1734 /* Save guest register state */ 1735 save_fpu_regs(); 1736 vcpu->run->s.regs.fpc = current->thread.fpu.fpc; 1737 1738 /* Restore host register state */ 1739 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; 1740 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; 1741 1742 save_access_regs(vcpu->run->s.regs.acrs); 1743 restore_access_regs(vcpu->arch.host_acrs); 1744 } 1745 1746 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 1747 { 1748 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 1749 vcpu->arch.sie_block->gpsw.mask = 0UL; 1750 vcpu->arch.sie_block->gpsw.addr = 0UL; 1751 kvm_s390_set_prefix(vcpu, 0); 1752 kvm_s390_set_cpu_timer(vcpu, 0); 1753 vcpu->arch.sie_block->ckc = 0UL; 1754 vcpu->arch.sie_block->todpr = 0; 1755 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 1756 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 1757 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 1758 /* make sure the new fpc will be lazily loaded */ 1759 save_fpu_regs(); 1760 current->thread.fpu.fpc = 0; 1761 vcpu->arch.sie_block->gbea = 1; 1762 vcpu->arch.sie_block->pp = 0; 1763 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 1764 kvm_clear_async_pf_completion_queue(vcpu); 1765 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 1766 kvm_s390_vcpu_stop(vcpu); 1767 kvm_s390_clear_local_irqs(vcpu); 1768 } 1769 1770 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 1771 { 1772 mutex_lock(&vcpu->kvm->lock); 1773 preempt_disable(); 1774 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch; 1775 preempt_enable(); 1776 mutex_unlock(&vcpu->kvm->lock); 1777 if (!kvm_is_ucontrol(vcpu->kvm)) { 1778 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 1779 sca_add_vcpu(vcpu); 1780 } 1781 1782 } 1783 1784 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) 1785 { 1786 if (!test_kvm_facility(vcpu->kvm, 76)) 1787 return; 1788 1789 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA); 1790 1791 if (vcpu->kvm->arch.crypto.aes_kw) 1792 vcpu->arch.sie_block->ecb3 |= ECB3_AES; 1793 if (vcpu->kvm->arch.crypto.dea_kw) 1794 vcpu->arch.sie_block->ecb3 |= ECB3_DEA; 1795 1796 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd; 1797 } 1798 1799 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) 1800 { 1801 free_page(vcpu->arch.sie_block->cbrlo); 1802 vcpu->arch.sie_block->cbrlo = 0; 1803 } 1804 1805 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) 1806 { 1807 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL); 1808 if (!vcpu->arch.sie_block->cbrlo) 1809 return -ENOMEM; 1810 1811 vcpu->arch.sie_block->ecb2 |= 0x80; 1812 vcpu->arch.sie_block->ecb2 &= ~0x08; 1813 return 0; 1814 } 1815 1816 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu) 1817 { 1818 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model; 1819 1820 vcpu->arch.sie_block->ibc = model->ibc; 1821 if (test_kvm_facility(vcpu->kvm, 7)) 1822 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list; 1823 } 1824 1825 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 1826 { 1827 int rc = 0; 1828 1829 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 1830 CPUSTAT_SM | 1831 CPUSTAT_STOPPED); 1832 1833 if (test_kvm_facility(vcpu->kvm, 78)) 1834 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); 1835 else if (test_kvm_facility(vcpu->kvm, 8)) 1836 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); 1837 1838 kvm_s390_vcpu_setup_model(vcpu); 1839 1840 vcpu->arch.sie_block->ecb = 0x02; 1841 if (test_kvm_facility(vcpu->kvm, 9)) 1842 vcpu->arch.sie_block->ecb |= 0x04; 1843 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73)) 1844 vcpu->arch.sie_block->ecb |= 0x10; 1845 1846 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi) 1847 vcpu->arch.sie_block->ecb2 |= 0x08; 1848 vcpu->arch.sie_block->eca = 0x1002000U; 1849 if (sclp.has_cei) 1850 vcpu->arch.sie_block->eca |= 0x80000000U; 1851 if (sclp.has_ib) 1852 vcpu->arch.sie_block->eca |= 0x40000000U; 1853 if (sclp.has_siif) 1854 vcpu->arch.sie_block->eca |= 1; 1855 if (sclp.has_sigpif) 1856 vcpu->arch.sie_block->eca |= 0x10000000U; 1857 if (test_kvm_facility(vcpu->kvm, 64)) 1858 vcpu->arch.sie_block->ecb3 |= 0x01; 1859 if (test_kvm_facility(vcpu->kvm, 129)) { 1860 vcpu->arch.sie_block->eca |= 0x00020000; 1861 vcpu->arch.sie_block->ecd |= 0x20000000; 1862 } 1863 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb; 1864 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 1865 if (test_kvm_facility(vcpu->kvm, 74)) 1866 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; 1867 1868 if (vcpu->kvm->arch.use_cmma) { 1869 rc = kvm_s390_vcpu_setup_cmma(vcpu); 1870 if (rc) 1871 return rc; 1872 } 1873 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1874 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 1875 1876 kvm_s390_vcpu_crypto_setup(vcpu); 1877 1878 return rc; 1879 } 1880 1881 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 1882 unsigned int id) 1883 { 1884 struct kvm_vcpu *vcpu; 1885 struct sie_page *sie_page; 1886 int rc = -EINVAL; 1887 1888 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id)) 1889 goto out; 1890 1891 rc = -ENOMEM; 1892 1893 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1894 if (!vcpu) 1895 goto out; 1896 1897 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL); 1898 if (!sie_page) 1899 goto out_free_cpu; 1900 1901 vcpu->arch.sie_block = &sie_page->sie_block; 1902 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 1903 1904 /* the real guest size will always be smaller than msl */ 1905 vcpu->arch.sie_block->mso = 0; 1906 vcpu->arch.sie_block->msl = sclp.hamax; 1907 1908 vcpu->arch.sie_block->icpua = id; 1909 spin_lock_init(&vcpu->arch.local_int.lock); 1910 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 1911 vcpu->arch.local_int.wq = &vcpu->wq; 1912 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 1913 seqcount_init(&vcpu->arch.cputm_seqcount); 1914 1915 rc = kvm_vcpu_init(vcpu, kvm, id); 1916 if (rc) 1917 goto out_free_sie_block; 1918 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu, 1919 vcpu->arch.sie_block); 1920 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 1921 1922 return vcpu; 1923 out_free_sie_block: 1924 free_page((unsigned long)(vcpu->arch.sie_block)); 1925 out_free_cpu: 1926 kmem_cache_free(kvm_vcpu_cache, vcpu); 1927 out: 1928 return ERR_PTR(rc); 1929 } 1930 1931 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 1932 { 1933 return kvm_s390_vcpu_has_irq(vcpu, 0); 1934 } 1935 1936 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu) 1937 { 1938 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 1939 exit_sie(vcpu); 1940 } 1941 1942 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu) 1943 { 1944 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 1945 } 1946 1947 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu) 1948 { 1949 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); 1950 exit_sie(vcpu); 1951 } 1952 1953 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) 1954 { 1955 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20); 1956 } 1957 1958 /* 1959 * Kick a guest cpu out of SIE and wait until SIE is not running. 1960 * If the CPU is not running (e.g. waiting as idle) the function will 1961 * return immediately. */ 1962 void exit_sie(struct kvm_vcpu *vcpu) 1963 { 1964 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 1965 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 1966 cpu_relax(); 1967 } 1968 1969 /* Kick a guest cpu out of SIE to process a request synchronously */ 1970 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu) 1971 { 1972 kvm_make_request(req, vcpu); 1973 kvm_s390_vcpu_request(vcpu); 1974 } 1975 1976 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address) 1977 { 1978 int i; 1979 struct kvm *kvm = gmap->private; 1980 struct kvm_vcpu *vcpu; 1981 1982 kvm_for_each_vcpu(i, vcpu, kvm) { 1983 /* match against both prefix pages */ 1984 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) { 1985 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address); 1986 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu); 1987 } 1988 } 1989 } 1990 1991 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 1992 { 1993 /* kvm common code refers to this, but never calls it */ 1994 BUG(); 1995 return 0; 1996 } 1997 1998 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 1999 struct kvm_one_reg *reg) 2000 { 2001 int r = -EINVAL; 2002 2003 switch (reg->id) { 2004 case KVM_REG_S390_TODPR: 2005 r = put_user(vcpu->arch.sie_block->todpr, 2006 (u32 __user *)reg->addr); 2007 break; 2008 case KVM_REG_S390_EPOCHDIFF: 2009 r = put_user(vcpu->arch.sie_block->epoch, 2010 (u64 __user *)reg->addr); 2011 break; 2012 case KVM_REG_S390_CPU_TIMER: 2013 r = put_user(kvm_s390_get_cpu_timer(vcpu), 2014 (u64 __user *)reg->addr); 2015 break; 2016 case KVM_REG_S390_CLOCK_COMP: 2017 r = put_user(vcpu->arch.sie_block->ckc, 2018 (u64 __user *)reg->addr); 2019 break; 2020 case KVM_REG_S390_PFTOKEN: 2021 r = put_user(vcpu->arch.pfault_token, 2022 (u64 __user *)reg->addr); 2023 break; 2024 case KVM_REG_S390_PFCOMPARE: 2025 r = put_user(vcpu->arch.pfault_compare, 2026 (u64 __user *)reg->addr); 2027 break; 2028 case KVM_REG_S390_PFSELECT: 2029 r = put_user(vcpu->arch.pfault_select, 2030 (u64 __user *)reg->addr); 2031 break; 2032 case KVM_REG_S390_PP: 2033 r = put_user(vcpu->arch.sie_block->pp, 2034 (u64 __user *)reg->addr); 2035 break; 2036 case KVM_REG_S390_GBEA: 2037 r = put_user(vcpu->arch.sie_block->gbea, 2038 (u64 __user *)reg->addr); 2039 break; 2040 default: 2041 break; 2042 } 2043 2044 return r; 2045 } 2046 2047 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 2048 struct kvm_one_reg *reg) 2049 { 2050 int r = -EINVAL; 2051 __u64 val; 2052 2053 switch (reg->id) { 2054 case KVM_REG_S390_TODPR: 2055 r = get_user(vcpu->arch.sie_block->todpr, 2056 (u32 __user *)reg->addr); 2057 break; 2058 case KVM_REG_S390_EPOCHDIFF: 2059 r = get_user(vcpu->arch.sie_block->epoch, 2060 (u64 __user *)reg->addr); 2061 break; 2062 case KVM_REG_S390_CPU_TIMER: 2063 r = get_user(val, (u64 __user *)reg->addr); 2064 if (!r) 2065 kvm_s390_set_cpu_timer(vcpu, val); 2066 break; 2067 case KVM_REG_S390_CLOCK_COMP: 2068 r = get_user(vcpu->arch.sie_block->ckc, 2069 (u64 __user *)reg->addr); 2070 break; 2071 case KVM_REG_S390_PFTOKEN: 2072 r = get_user(vcpu->arch.pfault_token, 2073 (u64 __user *)reg->addr); 2074 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 2075 kvm_clear_async_pf_completion_queue(vcpu); 2076 break; 2077 case KVM_REG_S390_PFCOMPARE: 2078 r = get_user(vcpu->arch.pfault_compare, 2079 (u64 __user *)reg->addr); 2080 break; 2081 case KVM_REG_S390_PFSELECT: 2082 r = get_user(vcpu->arch.pfault_select, 2083 (u64 __user *)reg->addr); 2084 break; 2085 case KVM_REG_S390_PP: 2086 r = get_user(vcpu->arch.sie_block->pp, 2087 (u64 __user *)reg->addr); 2088 break; 2089 case KVM_REG_S390_GBEA: 2090 r = get_user(vcpu->arch.sie_block->gbea, 2091 (u64 __user *)reg->addr); 2092 break; 2093 default: 2094 break; 2095 } 2096 2097 return r; 2098 } 2099 2100 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 2101 { 2102 kvm_s390_vcpu_initial_reset(vcpu); 2103 return 0; 2104 } 2105 2106 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 2107 { 2108 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 2109 return 0; 2110 } 2111 2112 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 2113 { 2114 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 2115 return 0; 2116 } 2117 2118 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 2119 struct kvm_sregs *sregs) 2120 { 2121 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 2122 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 2123 restore_access_regs(vcpu->run->s.regs.acrs); 2124 return 0; 2125 } 2126 2127 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 2128 struct kvm_sregs *sregs) 2129 { 2130 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 2131 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 2132 return 0; 2133 } 2134 2135 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 2136 { 2137 /* make sure the new values will be lazily loaded */ 2138 save_fpu_regs(); 2139 if (test_fp_ctl(fpu->fpc)) 2140 return -EINVAL; 2141 current->thread.fpu.fpc = fpu->fpc; 2142 if (MACHINE_HAS_VX) 2143 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs); 2144 else 2145 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs)); 2146 return 0; 2147 } 2148 2149 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 2150 { 2151 /* make sure we have the latest values */ 2152 save_fpu_regs(); 2153 if (MACHINE_HAS_VX) 2154 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs); 2155 else 2156 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs)); 2157 fpu->fpc = current->thread.fpu.fpc; 2158 return 0; 2159 } 2160 2161 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 2162 { 2163 int rc = 0; 2164 2165 if (!is_vcpu_stopped(vcpu)) 2166 rc = -EBUSY; 2167 else { 2168 vcpu->run->psw_mask = psw.mask; 2169 vcpu->run->psw_addr = psw.addr; 2170 } 2171 return rc; 2172 } 2173 2174 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 2175 struct kvm_translation *tr) 2176 { 2177 return -EINVAL; /* not implemented yet */ 2178 } 2179 2180 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \ 2181 KVM_GUESTDBG_USE_HW_BP | \ 2182 KVM_GUESTDBG_ENABLE) 2183 2184 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 2185 struct kvm_guest_debug *dbg) 2186 { 2187 int rc = 0; 2188 2189 vcpu->guest_debug = 0; 2190 kvm_s390_clear_bp_data(vcpu); 2191 2192 if (dbg->control & ~VALID_GUESTDBG_FLAGS) 2193 return -EINVAL; 2194 if (!sclp.has_gpere) 2195 return -EINVAL; 2196 2197 if (dbg->control & KVM_GUESTDBG_ENABLE) { 2198 vcpu->guest_debug = dbg->control; 2199 /* enforce guest PER */ 2200 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2201 2202 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 2203 rc = kvm_s390_import_bp_data(vcpu, dbg); 2204 } else { 2205 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2206 vcpu->arch.guestdbg.last_bp = 0; 2207 } 2208 2209 if (rc) { 2210 vcpu->guest_debug = 0; 2211 kvm_s390_clear_bp_data(vcpu); 2212 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2213 } 2214 2215 return rc; 2216 } 2217 2218 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 2219 struct kvm_mp_state *mp_state) 2220 { 2221 /* CHECK_STOP and LOAD are not supported yet */ 2222 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED : 2223 KVM_MP_STATE_OPERATING; 2224 } 2225 2226 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 2227 struct kvm_mp_state *mp_state) 2228 { 2229 int rc = 0; 2230 2231 /* user space knows about this interface - let it control the state */ 2232 vcpu->kvm->arch.user_cpu_state_ctrl = 1; 2233 2234 switch (mp_state->mp_state) { 2235 case KVM_MP_STATE_STOPPED: 2236 kvm_s390_vcpu_stop(vcpu); 2237 break; 2238 case KVM_MP_STATE_OPERATING: 2239 kvm_s390_vcpu_start(vcpu); 2240 break; 2241 case KVM_MP_STATE_LOAD: 2242 case KVM_MP_STATE_CHECK_STOP: 2243 /* fall through - CHECK_STOP and LOAD are not supported yet */ 2244 default: 2245 rc = -ENXIO; 2246 } 2247 2248 return rc; 2249 } 2250 2251 static bool ibs_enabled(struct kvm_vcpu *vcpu) 2252 { 2253 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS; 2254 } 2255 2256 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 2257 { 2258 retry: 2259 kvm_s390_vcpu_request_handled(vcpu); 2260 if (!vcpu->requests) 2261 return 0; 2262 /* 2263 * We use MMU_RELOAD just to re-arm the ipte notifier for the 2264 * guest prefix page. gmap_ipte_notify will wait on the ptl lock. 2265 * This ensures that the ipte instruction for this request has 2266 * already finished. We might race against a second unmapper that 2267 * wants to set the blocking bit. Lets just retry the request loop. 2268 */ 2269 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 2270 int rc; 2271 rc = gmap_ipte_notify(vcpu->arch.gmap, 2272 kvm_s390_get_prefix(vcpu), 2273 PAGE_SIZE * 2); 2274 if (rc) 2275 return rc; 2276 goto retry; 2277 } 2278 2279 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 2280 vcpu->arch.sie_block->ihcpu = 0xffff; 2281 goto retry; 2282 } 2283 2284 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 2285 if (!ibs_enabled(vcpu)) { 2286 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 2287 atomic_or(CPUSTAT_IBS, 2288 &vcpu->arch.sie_block->cpuflags); 2289 } 2290 goto retry; 2291 } 2292 2293 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 2294 if (ibs_enabled(vcpu)) { 2295 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 2296 atomic_andnot(CPUSTAT_IBS, 2297 &vcpu->arch.sie_block->cpuflags); 2298 } 2299 goto retry; 2300 } 2301 2302 /* nothing to do, just clear the request */ 2303 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 2304 2305 return 0; 2306 } 2307 2308 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod) 2309 { 2310 struct kvm_vcpu *vcpu; 2311 int i; 2312 2313 mutex_lock(&kvm->lock); 2314 preempt_disable(); 2315 kvm->arch.epoch = tod - get_tod_clock(); 2316 kvm_s390_vcpu_block_all(kvm); 2317 kvm_for_each_vcpu(i, vcpu, kvm) 2318 vcpu->arch.sie_block->epoch = kvm->arch.epoch; 2319 kvm_s390_vcpu_unblock_all(kvm); 2320 preempt_enable(); 2321 mutex_unlock(&kvm->lock); 2322 } 2323 2324 /** 2325 * kvm_arch_fault_in_page - fault-in guest page if necessary 2326 * @vcpu: The corresponding virtual cpu 2327 * @gpa: Guest physical address 2328 * @writable: Whether the page should be writable or not 2329 * 2330 * Make sure that a guest page has been faulted-in on the host. 2331 * 2332 * Return: Zero on success, negative error code otherwise. 2333 */ 2334 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable) 2335 { 2336 return gmap_fault(vcpu->arch.gmap, gpa, 2337 writable ? FAULT_FLAG_WRITE : 0); 2338 } 2339 2340 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, 2341 unsigned long token) 2342 { 2343 struct kvm_s390_interrupt inti; 2344 struct kvm_s390_irq irq; 2345 2346 if (start_token) { 2347 irq.u.ext.ext_params2 = token; 2348 irq.type = KVM_S390_INT_PFAULT_INIT; 2349 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); 2350 } else { 2351 inti.type = KVM_S390_INT_PFAULT_DONE; 2352 inti.parm64 = token; 2353 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); 2354 } 2355 } 2356 2357 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 2358 struct kvm_async_pf *work) 2359 { 2360 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); 2361 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); 2362 } 2363 2364 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 2365 struct kvm_async_pf *work) 2366 { 2367 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token); 2368 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token); 2369 } 2370 2371 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 2372 struct kvm_async_pf *work) 2373 { 2374 /* s390 will always inject the page directly */ 2375 } 2376 2377 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 2378 { 2379 /* 2380 * s390 will always inject the page directly, 2381 * but we still want check_async_completion to cleanup 2382 */ 2383 return true; 2384 } 2385 2386 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) 2387 { 2388 hva_t hva; 2389 struct kvm_arch_async_pf arch; 2390 int rc; 2391 2392 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 2393 return 0; 2394 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) != 2395 vcpu->arch.pfault_compare) 2396 return 0; 2397 if (psw_extint_disabled(vcpu)) 2398 return 0; 2399 if (kvm_s390_vcpu_has_irq(vcpu, 0)) 2400 return 0; 2401 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 2402 return 0; 2403 if (!vcpu->arch.gmap->pfault_enabled) 2404 return 0; 2405 2406 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr)); 2407 hva += current->thread.gmap_addr & ~PAGE_MASK; 2408 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8)) 2409 return 0; 2410 2411 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch); 2412 return rc; 2413 } 2414 2415 static int vcpu_pre_run(struct kvm_vcpu *vcpu) 2416 { 2417 int rc, cpuflags; 2418 2419 /* 2420 * On s390 notifications for arriving pages will be delivered directly 2421 * to the guest but the house keeping for completed pfaults is 2422 * handled outside the worker. 2423 */ 2424 kvm_check_async_pf_completion(vcpu); 2425 2426 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14]; 2427 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15]; 2428 2429 if (need_resched()) 2430 schedule(); 2431 2432 if (test_cpu_flag(CIF_MCCK_PENDING)) 2433 s390_handle_mcck(); 2434 2435 if (!kvm_is_ucontrol(vcpu->kvm)) { 2436 rc = kvm_s390_deliver_pending_interrupts(vcpu); 2437 if (rc) 2438 return rc; 2439 } 2440 2441 rc = kvm_s390_handle_requests(vcpu); 2442 if (rc) 2443 return rc; 2444 2445 if (guestdbg_enabled(vcpu)) { 2446 kvm_s390_backup_guest_per_regs(vcpu); 2447 kvm_s390_patch_guest_per_regs(vcpu); 2448 } 2449 2450 vcpu->arch.sie_block->icptcode = 0; 2451 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); 2452 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); 2453 trace_kvm_s390_sie_enter(vcpu, cpuflags); 2454 2455 return 0; 2456 } 2457 2458 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu) 2459 { 2460 struct kvm_s390_pgm_info pgm_info = { 2461 .code = PGM_ADDRESSING, 2462 }; 2463 u8 opcode, ilen; 2464 int rc; 2465 2466 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 2467 trace_kvm_s390_sie_fault(vcpu); 2468 2469 /* 2470 * We want to inject an addressing exception, which is defined as a 2471 * suppressing or terminating exception. However, since we came here 2472 * by a DAT access exception, the PSW still points to the faulting 2473 * instruction since DAT exceptions are nullifying. So we've got 2474 * to look up the current opcode to get the length of the instruction 2475 * to be able to forward the PSW. 2476 */ 2477 rc = read_guest_instr(vcpu, &opcode, 1); 2478 ilen = insn_length(opcode); 2479 if (rc < 0) { 2480 return rc; 2481 } else if (rc) { 2482 /* Instruction-Fetching Exceptions - we can't detect the ilen. 2483 * Forward by arbitrary ilc, injection will take care of 2484 * nullification if necessary. 2485 */ 2486 pgm_info = vcpu->arch.pgm; 2487 ilen = 4; 2488 } 2489 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID; 2490 kvm_s390_forward_psw(vcpu, ilen); 2491 return kvm_s390_inject_prog_irq(vcpu, &pgm_info); 2492 } 2493 2494 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 2495 { 2496 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 2497 vcpu->arch.sie_block->icptcode); 2498 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 2499 2500 if (guestdbg_enabled(vcpu)) 2501 kvm_s390_restore_guest_per_regs(vcpu); 2502 2503 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14; 2504 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; 2505 2506 if (vcpu->arch.sie_block->icptcode > 0) { 2507 int rc = kvm_handle_sie_intercept(vcpu); 2508 2509 if (rc != -EOPNOTSUPP) 2510 return rc; 2511 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC; 2512 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 2513 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 2514 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 2515 return -EREMOTE; 2516 } else if (exit_reason != -EFAULT) { 2517 vcpu->stat.exit_null++; 2518 return 0; 2519 } else if (kvm_is_ucontrol(vcpu->kvm)) { 2520 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; 2521 vcpu->run->s390_ucontrol.trans_exc_code = 2522 current->thread.gmap_addr; 2523 vcpu->run->s390_ucontrol.pgm_code = 0x10; 2524 return -EREMOTE; 2525 } else if (current->thread.gmap_pfault) { 2526 trace_kvm_s390_major_guest_pfault(vcpu); 2527 current->thread.gmap_pfault = 0; 2528 if (kvm_arch_setup_async_pf(vcpu)) 2529 return 0; 2530 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1); 2531 } 2532 return vcpu_post_run_fault_in_sie(vcpu); 2533 } 2534 2535 static int __vcpu_run(struct kvm_vcpu *vcpu) 2536 { 2537 int rc, exit_reason; 2538 2539 /* 2540 * We try to hold kvm->srcu during most of vcpu_run (except when run- 2541 * ning the guest), so that memslots (and other stuff) are protected 2542 */ 2543 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 2544 2545 do { 2546 rc = vcpu_pre_run(vcpu); 2547 if (rc) 2548 break; 2549 2550 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 2551 /* 2552 * As PF_VCPU will be used in fault handler, between 2553 * guest_enter and guest_exit should be no uaccess. 2554 */ 2555 local_irq_disable(); 2556 __kvm_guest_enter(); 2557 __disable_cpu_timer_accounting(vcpu); 2558 local_irq_enable(); 2559 exit_reason = sie64a(vcpu->arch.sie_block, 2560 vcpu->run->s.regs.gprs); 2561 local_irq_disable(); 2562 __enable_cpu_timer_accounting(vcpu); 2563 __kvm_guest_exit(); 2564 local_irq_enable(); 2565 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 2566 2567 rc = vcpu_post_run(vcpu, exit_reason); 2568 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc); 2569 2570 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 2571 return rc; 2572 } 2573 2574 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 2575 { 2576 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 2577 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 2578 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) 2579 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 2580 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 2581 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 2582 /* some control register changes require a tlb flush */ 2583 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2584 } 2585 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { 2586 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm); 2587 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc; 2588 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr; 2589 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp; 2590 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea; 2591 } 2592 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) { 2593 vcpu->arch.pfault_token = kvm_run->s.regs.pft; 2594 vcpu->arch.pfault_select = kvm_run->s.regs.pfs; 2595 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc; 2596 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 2597 kvm_clear_async_pf_completion_queue(vcpu); 2598 } 2599 kvm_run->kvm_dirty_regs = 0; 2600 } 2601 2602 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 2603 { 2604 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 2605 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 2606 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); 2607 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 2608 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu); 2609 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc; 2610 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr; 2611 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp; 2612 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea; 2613 kvm_run->s.regs.pft = vcpu->arch.pfault_token; 2614 kvm_run->s.regs.pfs = vcpu->arch.pfault_select; 2615 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; 2616 } 2617 2618 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 2619 { 2620 int rc; 2621 sigset_t sigsaved; 2622 2623 if (guestdbg_exit_pending(vcpu)) { 2624 kvm_s390_prepare_debug_exit(vcpu); 2625 return 0; 2626 } 2627 2628 if (vcpu->sigset_active) 2629 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 2630 2631 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { 2632 kvm_s390_vcpu_start(vcpu); 2633 } else if (is_vcpu_stopped(vcpu)) { 2634 pr_err_ratelimited("can't run stopped vcpu %d\n", 2635 vcpu->vcpu_id); 2636 return -EINVAL; 2637 } 2638 2639 sync_regs(vcpu, kvm_run); 2640 enable_cpu_timer_accounting(vcpu); 2641 2642 might_fault(); 2643 rc = __vcpu_run(vcpu); 2644 2645 if (signal_pending(current) && !rc) { 2646 kvm_run->exit_reason = KVM_EXIT_INTR; 2647 rc = -EINTR; 2648 } 2649 2650 if (guestdbg_exit_pending(vcpu) && !rc) { 2651 kvm_s390_prepare_debug_exit(vcpu); 2652 rc = 0; 2653 } 2654 2655 if (rc == -EREMOTE) { 2656 /* userspace support is needed, kvm_run has been prepared */ 2657 rc = 0; 2658 } 2659 2660 disable_cpu_timer_accounting(vcpu); 2661 store_regs(vcpu, kvm_run); 2662 2663 if (vcpu->sigset_active) 2664 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 2665 2666 vcpu->stat.exit_userspace++; 2667 return rc; 2668 } 2669 2670 /* 2671 * store status at address 2672 * we use have two special cases: 2673 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 2674 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 2675 */ 2676 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa) 2677 { 2678 unsigned char archmode = 1; 2679 freg_t fprs[NUM_FPRS]; 2680 unsigned int px; 2681 u64 clkcomp, cputm; 2682 int rc; 2683 2684 px = kvm_s390_get_prefix(vcpu); 2685 if (gpa == KVM_S390_STORE_STATUS_NOADDR) { 2686 if (write_guest_abs(vcpu, 163, &archmode, 1)) 2687 return -EFAULT; 2688 gpa = 0; 2689 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) { 2690 if (write_guest_real(vcpu, 163, &archmode, 1)) 2691 return -EFAULT; 2692 gpa = px; 2693 } else 2694 gpa -= __LC_FPREGS_SAVE_AREA; 2695 2696 /* manually convert vector registers if necessary */ 2697 if (MACHINE_HAS_VX) { 2698 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 2699 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA, 2700 fprs, 128); 2701 } else { 2702 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA, 2703 vcpu->run->s.regs.fprs, 128); 2704 } 2705 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA, 2706 vcpu->run->s.regs.gprs, 128); 2707 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA, 2708 &vcpu->arch.sie_block->gpsw, 16); 2709 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA, 2710 &px, 4); 2711 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA, 2712 &vcpu->run->s.regs.fpc, 4); 2713 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA, 2714 &vcpu->arch.sie_block->todpr, 4); 2715 cputm = kvm_s390_get_cpu_timer(vcpu); 2716 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA, 2717 &cputm, 8); 2718 clkcomp = vcpu->arch.sie_block->ckc >> 8; 2719 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA, 2720 &clkcomp, 8); 2721 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA, 2722 &vcpu->run->s.regs.acrs, 64); 2723 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA, 2724 &vcpu->arch.sie_block->gcr, 128); 2725 return rc ? -EFAULT : 0; 2726 } 2727 2728 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 2729 { 2730 /* 2731 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 2732 * copying in vcpu load/put. Lets update our copies before we save 2733 * it into the save area 2734 */ 2735 save_fpu_regs(); 2736 vcpu->run->s.regs.fpc = current->thread.fpu.fpc; 2737 save_access_regs(vcpu->run->s.regs.acrs); 2738 2739 return kvm_s390_store_status_unloaded(vcpu, addr); 2740 } 2741 2742 /* 2743 * store additional status at address 2744 */ 2745 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu, 2746 unsigned long gpa) 2747 { 2748 /* Only bits 0-53 are used for address formation */ 2749 if (!(gpa & ~0x3ff)) 2750 return 0; 2751 2752 return write_guest_abs(vcpu, gpa & ~0x3ff, 2753 (void *)&vcpu->run->s.regs.vrs, 512); 2754 } 2755 2756 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr) 2757 { 2758 if (!test_kvm_facility(vcpu->kvm, 129)) 2759 return 0; 2760 2761 /* 2762 * The guest VXRS are in the host VXRs due to the lazy 2763 * copying in vcpu load/put. We can simply call save_fpu_regs() 2764 * to save the current register state because we are in the 2765 * middle of a load/put cycle. 2766 * 2767 * Let's update our copies before we save it into the save area. 2768 */ 2769 save_fpu_regs(); 2770 2771 return kvm_s390_store_adtl_status_unloaded(vcpu, addr); 2772 } 2773 2774 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 2775 { 2776 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 2777 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu); 2778 } 2779 2780 static void __disable_ibs_on_all_vcpus(struct kvm *kvm) 2781 { 2782 unsigned int i; 2783 struct kvm_vcpu *vcpu; 2784 2785 kvm_for_each_vcpu(i, vcpu, kvm) { 2786 __disable_ibs_on_vcpu(vcpu); 2787 } 2788 } 2789 2790 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 2791 { 2792 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu); 2793 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu); 2794 } 2795 2796 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) 2797 { 2798 int i, online_vcpus, started_vcpus = 0; 2799 2800 if (!is_vcpu_stopped(vcpu)) 2801 return; 2802 2803 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); 2804 /* Only one cpu at a time may enter/leave the STOPPED state. */ 2805 spin_lock(&vcpu->kvm->arch.start_stop_lock); 2806 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 2807 2808 for (i = 0; i < online_vcpus; i++) { 2809 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) 2810 started_vcpus++; 2811 } 2812 2813 if (started_vcpus == 0) { 2814 /* we're the only active VCPU -> speed it up */ 2815 __enable_ibs_on_vcpu(vcpu); 2816 } else if (started_vcpus == 1) { 2817 /* 2818 * As we are starting a second VCPU, we have to disable 2819 * the IBS facility on all VCPUs to remove potentially 2820 * oustanding ENABLE requests. 2821 */ 2822 __disable_ibs_on_all_vcpus(vcpu->kvm); 2823 } 2824 2825 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 2826 /* 2827 * Another VCPU might have used IBS while we were offline. 2828 * Let's play safe and flush the VCPU at startup. 2829 */ 2830 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2831 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 2832 return; 2833 } 2834 2835 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) 2836 { 2837 int i, online_vcpus, started_vcpus = 0; 2838 struct kvm_vcpu *started_vcpu = NULL; 2839 2840 if (is_vcpu_stopped(vcpu)) 2841 return; 2842 2843 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); 2844 /* Only one cpu at a time may enter/leave the STOPPED state. */ 2845 spin_lock(&vcpu->kvm->arch.start_stop_lock); 2846 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 2847 2848 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 2849 kvm_s390_clear_stop_irq(vcpu); 2850 2851 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 2852 __disable_ibs_on_vcpu(vcpu); 2853 2854 for (i = 0; i < online_vcpus; i++) { 2855 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) { 2856 started_vcpus++; 2857 started_vcpu = vcpu->kvm->vcpus[i]; 2858 } 2859 } 2860 2861 if (started_vcpus == 1) { 2862 /* 2863 * As we only have one VCPU left, we want to enable the 2864 * IBS facility for that VCPU to speed it up. 2865 */ 2866 __enable_ibs_on_vcpu(started_vcpu); 2867 } 2868 2869 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 2870 return; 2871 } 2872 2873 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 2874 struct kvm_enable_cap *cap) 2875 { 2876 int r; 2877 2878 if (cap->flags) 2879 return -EINVAL; 2880 2881 switch (cap->cap) { 2882 case KVM_CAP_S390_CSS_SUPPORT: 2883 if (!vcpu->kvm->arch.css_support) { 2884 vcpu->kvm->arch.css_support = 1; 2885 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support"); 2886 trace_kvm_s390_enable_css(vcpu->kvm); 2887 } 2888 r = 0; 2889 break; 2890 default: 2891 r = -EINVAL; 2892 break; 2893 } 2894 return r; 2895 } 2896 2897 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, 2898 struct kvm_s390_mem_op *mop) 2899 { 2900 void __user *uaddr = (void __user *)mop->buf; 2901 void *tmpbuf = NULL; 2902 int r, srcu_idx; 2903 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION 2904 | KVM_S390_MEMOP_F_CHECK_ONLY; 2905 2906 if (mop->flags & ~supported_flags) 2907 return -EINVAL; 2908 2909 if (mop->size > MEM_OP_MAX_SIZE) 2910 return -E2BIG; 2911 2912 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) { 2913 tmpbuf = vmalloc(mop->size); 2914 if (!tmpbuf) 2915 return -ENOMEM; 2916 } 2917 2918 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 2919 2920 switch (mop->op) { 2921 case KVM_S390_MEMOP_LOGICAL_READ: 2922 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { 2923 r = check_gva_range(vcpu, mop->gaddr, mop->ar, 2924 mop->size, GACC_FETCH); 2925 break; 2926 } 2927 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); 2928 if (r == 0) { 2929 if (copy_to_user(uaddr, tmpbuf, mop->size)) 2930 r = -EFAULT; 2931 } 2932 break; 2933 case KVM_S390_MEMOP_LOGICAL_WRITE: 2934 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { 2935 r = check_gva_range(vcpu, mop->gaddr, mop->ar, 2936 mop->size, GACC_STORE); 2937 break; 2938 } 2939 if (copy_from_user(tmpbuf, uaddr, mop->size)) { 2940 r = -EFAULT; 2941 break; 2942 } 2943 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); 2944 break; 2945 default: 2946 r = -EINVAL; 2947 } 2948 2949 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 2950 2951 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0) 2952 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 2953 2954 vfree(tmpbuf); 2955 return r; 2956 } 2957 2958 long kvm_arch_vcpu_ioctl(struct file *filp, 2959 unsigned int ioctl, unsigned long arg) 2960 { 2961 struct kvm_vcpu *vcpu = filp->private_data; 2962 void __user *argp = (void __user *)arg; 2963 int idx; 2964 long r; 2965 2966 switch (ioctl) { 2967 case KVM_S390_IRQ: { 2968 struct kvm_s390_irq s390irq; 2969 2970 r = -EFAULT; 2971 if (copy_from_user(&s390irq, argp, sizeof(s390irq))) 2972 break; 2973 r = kvm_s390_inject_vcpu(vcpu, &s390irq); 2974 break; 2975 } 2976 case KVM_S390_INTERRUPT: { 2977 struct kvm_s390_interrupt s390int; 2978 struct kvm_s390_irq s390irq; 2979 2980 r = -EFAULT; 2981 if (copy_from_user(&s390int, argp, sizeof(s390int))) 2982 break; 2983 if (s390int_to_s390irq(&s390int, &s390irq)) 2984 return -EINVAL; 2985 r = kvm_s390_inject_vcpu(vcpu, &s390irq); 2986 break; 2987 } 2988 case KVM_S390_STORE_STATUS: 2989 idx = srcu_read_lock(&vcpu->kvm->srcu); 2990 r = kvm_s390_vcpu_store_status(vcpu, arg); 2991 srcu_read_unlock(&vcpu->kvm->srcu, idx); 2992 break; 2993 case KVM_S390_SET_INITIAL_PSW: { 2994 psw_t psw; 2995 2996 r = -EFAULT; 2997 if (copy_from_user(&psw, argp, sizeof(psw))) 2998 break; 2999 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 3000 break; 3001 } 3002 case KVM_S390_INITIAL_RESET: 3003 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 3004 break; 3005 case KVM_SET_ONE_REG: 3006 case KVM_GET_ONE_REG: { 3007 struct kvm_one_reg reg; 3008 r = -EFAULT; 3009 if (copy_from_user(®, argp, sizeof(reg))) 3010 break; 3011 if (ioctl == KVM_SET_ONE_REG) 3012 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 3013 else 3014 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 3015 break; 3016 } 3017 #ifdef CONFIG_KVM_S390_UCONTROL 3018 case KVM_S390_UCAS_MAP: { 3019 struct kvm_s390_ucas_mapping ucasmap; 3020 3021 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 3022 r = -EFAULT; 3023 break; 3024 } 3025 3026 if (!kvm_is_ucontrol(vcpu->kvm)) { 3027 r = -EINVAL; 3028 break; 3029 } 3030 3031 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 3032 ucasmap.vcpu_addr, ucasmap.length); 3033 break; 3034 } 3035 case KVM_S390_UCAS_UNMAP: { 3036 struct kvm_s390_ucas_mapping ucasmap; 3037 3038 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 3039 r = -EFAULT; 3040 break; 3041 } 3042 3043 if (!kvm_is_ucontrol(vcpu->kvm)) { 3044 r = -EINVAL; 3045 break; 3046 } 3047 3048 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 3049 ucasmap.length); 3050 break; 3051 } 3052 #endif 3053 case KVM_S390_VCPU_FAULT: { 3054 r = gmap_fault(vcpu->arch.gmap, arg, 0); 3055 break; 3056 } 3057 case KVM_ENABLE_CAP: 3058 { 3059 struct kvm_enable_cap cap; 3060 r = -EFAULT; 3061 if (copy_from_user(&cap, argp, sizeof(cap))) 3062 break; 3063 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 3064 break; 3065 } 3066 case KVM_S390_MEM_OP: { 3067 struct kvm_s390_mem_op mem_op; 3068 3069 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0) 3070 r = kvm_s390_guest_mem_op(vcpu, &mem_op); 3071 else 3072 r = -EFAULT; 3073 break; 3074 } 3075 case KVM_S390_SET_IRQ_STATE: { 3076 struct kvm_s390_irq_state irq_state; 3077 3078 r = -EFAULT; 3079 if (copy_from_user(&irq_state, argp, sizeof(irq_state))) 3080 break; 3081 if (irq_state.len > VCPU_IRQS_MAX_BUF || 3082 irq_state.len == 0 || 3083 irq_state.len % sizeof(struct kvm_s390_irq) > 0) { 3084 r = -EINVAL; 3085 break; 3086 } 3087 r = kvm_s390_set_irq_state(vcpu, 3088 (void __user *) irq_state.buf, 3089 irq_state.len); 3090 break; 3091 } 3092 case KVM_S390_GET_IRQ_STATE: { 3093 struct kvm_s390_irq_state irq_state; 3094 3095 r = -EFAULT; 3096 if (copy_from_user(&irq_state, argp, sizeof(irq_state))) 3097 break; 3098 if (irq_state.len == 0) { 3099 r = -EINVAL; 3100 break; 3101 } 3102 r = kvm_s390_get_irq_state(vcpu, 3103 (__u8 __user *) irq_state.buf, 3104 irq_state.len); 3105 break; 3106 } 3107 default: 3108 r = -ENOTTY; 3109 } 3110 return r; 3111 } 3112 3113 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 3114 { 3115 #ifdef CONFIG_KVM_S390_UCONTROL 3116 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 3117 && (kvm_is_ucontrol(vcpu->kvm))) { 3118 vmf->page = virt_to_page(vcpu->arch.sie_block); 3119 get_page(vmf->page); 3120 return 0; 3121 } 3122 #endif 3123 return VM_FAULT_SIGBUS; 3124 } 3125 3126 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 3127 unsigned long npages) 3128 { 3129 return 0; 3130 } 3131 3132 /* Section: memory related */ 3133 int kvm_arch_prepare_memory_region(struct kvm *kvm, 3134 struct kvm_memory_slot *memslot, 3135 const struct kvm_userspace_memory_region *mem, 3136 enum kvm_mr_change change) 3137 { 3138 /* A few sanity checks. We can have memory slots which have to be 3139 located/ended at a segment boundary (1MB). The memory in userland is 3140 ok to be fragmented into various different vmas. It is okay to mmap() 3141 and munmap() stuff in this slot after doing this call at any time */ 3142 3143 if (mem->userspace_addr & 0xffffful) 3144 return -EINVAL; 3145 3146 if (mem->memory_size & 0xffffful) 3147 return -EINVAL; 3148 3149 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit) 3150 return -EINVAL; 3151 3152 return 0; 3153 } 3154 3155 void kvm_arch_commit_memory_region(struct kvm *kvm, 3156 const struct kvm_userspace_memory_region *mem, 3157 const struct kvm_memory_slot *old, 3158 const struct kvm_memory_slot *new, 3159 enum kvm_mr_change change) 3160 { 3161 int rc; 3162 3163 /* If the basics of the memslot do not change, we do not want 3164 * to update the gmap. Every update causes several unnecessary 3165 * segment translation exceptions. This is usually handled just 3166 * fine by the normal fault handler + gmap, but it will also 3167 * cause faults on the prefix page of running guest CPUs. 3168 */ 3169 if (old->userspace_addr == mem->userspace_addr && 3170 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 3171 old->npages * PAGE_SIZE == mem->memory_size) 3172 return; 3173 3174 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 3175 mem->guest_phys_addr, mem->memory_size); 3176 if (rc) 3177 pr_warn("failed to commit memory region\n"); 3178 return; 3179 } 3180 3181 static inline unsigned long nonhyp_mask(int i) 3182 { 3183 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30; 3184 3185 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4); 3186 } 3187 3188 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) 3189 { 3190 vcpu->valid_wakeup = false; 3191 } 3192 3193 static int __init kvm_s390_init(void) 3194 { 3195 int i; 3196 3197 if (!sclp.has_sief2) { 3198 pr_info("SIE not available\n"); 3199 return -ENODEV; 3200 } 3201 3202 for (i = 0; i < 16; i++) 3203 kvm_s390_fac_list_mask[i] |= 3204 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i); 3205 3206 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 3207 } 3208 3209 static void __exit kvm_s390_exit(void) 3210 { 3211 kvm_exit(); 3212 } 3213 3214 module_init(kvm_s390_init); 3215 module_exit(kvm_s390_exit); 3216 3217 /* 3218 * Enable autoloading of the kvm module. 3219 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 3220 * since x86 takes a different approach. 3221 */ 3222 #include <linux/miscdevice.h> 3223 MODULE_ALIAS_MISCDEV(KVM_MINOR); 3224 MODULE_ALIAS("devname:kvm"); 3225