1 /* 2 * hosting zSeries kernel virtual machines 3 * 4 * Copyright IBM Corp. 2008, 2009 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 * Christian Borntraeger <borntraeger@de.ibm.com> 12 * Heiko Carstens <heiko.carstens@de.ibm.com> 13 * Christian Ehrhardt <ehrhardt@de.ibm.com> 14 */ 15 16 #include <linux/compiler.h> 17 #include <linux/err.h> 18 #include <linux/fs.h> 19 #include <linux/hrtimer.h> 20 #include <linux/init.h> 21 #include <linux/kvm.h> 22 #include <linux/kvm_host.h> 23 #include <linux/module.h> 24 #include <linux/slab.h> 25 #include <linux/timer.h> 26 #include <asm/asm-offsets.h> 27 #include <asm/lowcore.h> 28 #include <asm/pgtable.h> 29 #include <asm/nmi.h> 30 #include <asm/switch_to.h> 31 #include <asm/sclp.h> 32 #include "kvm-s390.h" 33 #include "gaccess.h" 34 35 #define CREATE_TRACE_POINTS 36 #include "trace.h" 37 #include "trace-s390.h" 38 39 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 40 41 struct kvm_stats_debugfs_item debugfs_entries[] = { 42 { "userspace_handled", VCPU_STAT(exit_userspace) }, 43 { "exit_null", VCPU_STAT(exit_null) }, 44 { "exit_validity", VCPU_STAT(exit_validity) }, 45 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 46 { "exit_external_request", VCPU_STAT(exit_external_request) }, 47 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 48 { "exit_instruction", VCPU_STAT(exit_instruction) }, 49 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 50 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 51 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 52 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 53 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 54 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 55 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 56 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 57 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 58 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 59 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 60 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 61 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 62 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 63 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 64 { "instruction_spx", VCPU_STAT(instruction_spx) }, 65 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 66 { "instruction_stap", VCPU_STAT(instruction_stap) }, 67 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 68 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 69 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 70 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 71 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 72 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 73 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 74 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 75 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 76 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 77 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 78 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 79 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 80 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 81 { "diagnose_10", VCPU_STAT(diagnose_10) }, 82 { "diagnose_44", VCPU_STAT(diagnose_44) }, 83 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 84 { NULL } 85 }; 86 87 static unsigned long long *facilities; 88 static struct gmap_notifier gmap_notifier; 89 90 /* Section: not file related */ 91 int kvm_arch_hardware_enable(void *garbage) 92 { 93 /* every s390 is virtualization enabled ;-) */ 94 return 0; 95 } 96 97 void kvm_arch_hardware_disable(void *garbage) 98 { 99 } 100 101 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address); 102 103 int kvm_arch_hardware_setup(void) 104 { 105 gmap_notifier.notifier_call = kvm_gmap_notifier; 106 gmap_register_ipte_notifier(&gmap_notifier); 107 return 0; 108 } 109 110 void kvm_arch_hardware_unsetup(void) 111 { 112 gmap_unregister_ipte_notifier(&gmap_notifier); 113 } 114 115 void kvm_arch_check_processor_compat(void *rtn) 116 { 117 } 118 119 int kvm_arch_init(void *opaque) 120 { 121 return 0; 122 } 123 124 void kvm_arch_exit(void) 125 { 126 } 127 128 /* Section: device related */ 129 long kvm_arch_dev_ioctl(struct file *filp, 130 unsigned int ioctl, unsigned long arg) 131 { 132 if (ioctl == KVM_S390_ENABLE_SIE) 133 return s390_enable_sie(); 134 return -EINVAL; 135 } 136 137 int kvm_dev_ioctl_check_extension(long ext) 138 { 139 int r; 140 141 switch (ext) { 142 case KVM_CAP_S390_PSW: 143 case KVM_CAP_S390_GMAP: 144 case KVM_CAP_SYNC_MMU: 145 #ifdef CONFIG_KVM_S390_UCONTROL 146 case KVM_CAP_S390_UCONTROL: 147 #endif 148 case KVM_CAP_SYNC_REGS: 149 case KVM_CAP_ONE_REG: 150 case KVM_CAP_ENABLE_CAP: 151 case KVM_CAP_S390_CSS_SUPPORT: 152 case KVM_CAP_IOEVENTFD: 153 r = 1; 154 break; 155 case KVM_CAP_NR_VCPUS: 156 case KVM_CAP_MAX_VCPUS: 157 r = KVM_MAX_VCPUS; 158 break; 159 case KVM_CAP_NR_MEMSLOTS: 160 r = KVM_USER_MEM_SLOTS; 161 break; 162 case KVM_CAP_S390_COW: 163 r = MACHINE_HAS_ESOP; 164 break; 165 default: 166 r = 0; 167 } 168 return r; 169 } 170 171 /* Section: vm related */ 172 /* 173 * Get (and clear) the dirty memory log for a memory slot. 174 */ 175 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 176 struct kvm_dirty_log *log) 177 { 178 return 0; 179 } 180 181 long kvm_arch_vm_ioctl(struct file *filp, 182 unsigned int ioctl, unsigned long arg) 183 { 184 struct kvm *kvm = filp->private_data; 185 void __user *argp = (void __user *)arg; 186 int r; 187 188 switch (ioctl) { 189 case KVM_S390_INTERRUPT: { 190 struct kvm_s390_interrupt s390int; 191 192 r = -EFAULT; 193 if (copy_from_user(&s390int, argp, sizeof(s390int))) 194 break; 195 r = kvm_s390_inject_vm(kvm, &s390int); 196 break; 197 } 198 default: 199 r = -ENOTTY; 200 } 201 202 return r; 203 } 204 205 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 206 { 207 int rc; 208 char debug_name[16]; 209 210 rc = -EINVAL; 211 #ifdef CONFIG_KVM_S390_UCONTROL 212 if (type & ~KVM_VM_S390_UCONTROL) 213 goto out_err; 214 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 215 goto out_err; 216 #else 217 if (type) 218 goto out_err; 219 #endif 220 221 rc = s390_enable_sie(); 222 if (rc) 223 goto out_err; 224 225 rc = -ENOMEM; 226 227 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); 228 if (!kvm->arch.sca) 229 goto out_err; 230 231 sprintf(debug_name, "kvm-%u", current->pid); 232 233 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 234 if (!kvm->arch.dbf) 235 goto out_nodbf; 236 237 spin_lock_init(&kvm->arch.float_int.lock); 238 INIT_LIST_HEAD(&kvm->arch.float_int.list); 239 240 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 241 VM_EVENT(kvm, 3, "%s", "vm created"); 242 243 if (type & KVM_VM_S390_UCONTROL) { 244 kvm->arch.gmap = NULL; 245 } else { 246 kvm->arch.gmap = gmap_alloc(current->mm); 247 if (!kvm->arch.gmap) 248 goto out_nogmap; 249 kvm->arch.gmap->private = kvm; 250 } 251 252 kvm->arch.css_support = 0; 253 254 return 0; 255 out_nogmap: 256 debug_unregister(kvm->arch.dbf); 257 out_nodbf: 258 free_page((unsigned long)(kvm->arch.sca)); 259 out_err: 260 return rc; 261 } 262 263 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 264 { 265 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 266 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 267 if (!kvm_is_ucontrol(vcpu->kvm)) { 268 clear_bit(63 - vcpu->vcpu_id, 269 (unsigned long *) &vcpu->kvm->arch.sca->mcn); 270 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda == 271 (__u64) vcpu->arch.sie_block) 272 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0; 273 } 274 smp_mb(); 275 276 if (kvm_is_ucontrol(vcpu->kvm)) 277 gmap_free(vcpu->arch.gmap); 278 279 free_page((unsigned long)(vcpu->arch.sie_block)); 280 kvm_vcpu_uninit(vcpu); 281 kmem_cache_free(kvm_vcpu_cache, vcpu); 282 } 283 284 static void kvm_free_vcpus(struct kvm *kvm) 285 { 286 unsigned int i; 287 struct kvm_vcpu *vcpu; 288 289 kvm_for_each_vcpu(i, vcpu, kvm) 290 kvm_arch_vcpu_destroy(vcpu); 291 292 mutex_lock(&kvm->lock); 293 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 294 kvm->vcpus[i] = NULL; 295 296 atomic_set(&kvm->online_vcpus, 0); 297 mutex_unlock(&kvm->lock); 298 } 299 300 void kvm_arch_sync_events(struct kvm *kvm) 301 { 302 } 303 304 void kvm_arch_destroy_vm(struct kvm *kvm) 305 { 306 kvm_free_vcpus(kvm); 307 free_page((unsigned long)(kvm->arch.sca)); 308 debug_unregister(kvm->arch.dbf); 309 if (!kvm_is_ucontrol(kvm)) 310 gmap_free(kvm->arch.gmap); 311 } 312 313 /* Section: vcpu related */ 314 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 315 { 316 if (kvm_is_ucontrol(vcpu->kvm)) { 317 vcpu->arch.gmap = gmap_alloc(current->mm); 318 if (!vcpu->arch.gmap) 319 return -ENOMEM; 320 vcpu->arch.gmap->private = vcpu->kvm; 321 return 0; 322 } 323 324 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 325 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 326 KVM_SYNC_GPRS | 327 KVM_SYNC_ACRS | 328 KVM_SYNC_CRS; 329 return 0; 330 } 331 332 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 333 { 334 /* Nothing todo */ 335 } 336 337 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 338 { 339 save_fp_regs(&vcpu->arch.host_fpregs); 340 save_access_regs(vcpu->arch.host_acrs); 341 vcpu->arch.guest_fpregs.fpc &= FPC_VALID_MASK; 342 restore_fp_regs(&vcpu->arch.guest_fpregs); 343 restore_access_regs(vcpu->run->s.regs.acrs); 344 gmap_enable(vcpu->arch.gmap); 345 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 346 } 347 348 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 349 { 350 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 351 gmap_disable(vcpu->arch.gmap); 352 save_fp_regs(&vcpu->arch.guest_fpregs); 353 save_access_regs(vcpu->run->s.regs.acrs); 354 restore_fp_regs(&vcpu->arch.host_fpregs); 355 restore_access_regs(vcpu->arch.host_acrs); 356 } 357 358 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 359 { 360 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 361 vcpu->arch.sie_block->gpsw.mask = 0UL; 362 vcpu->arch.sie_block->gpsw.addr = 0UL; 363 kvm_s390_set_prefix(vcpu, 0); 364 vcpu->arch.sie_block->cputm = 0UL; 365 vcpu->arch.sie_block->ckc = 0UL; 366 vcpu->arch.sie_block->todpr = 0; 367 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 368 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 369 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 370 vcpu->arch.guest_fpregs.fpc = 0; 371 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc)); 372 vcpu->arch.sie_block->gbea = 1; 373 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 374 } 375 376 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 377 { 378 return 0; 379 } 380 381 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 382 { 383 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 384 CPUSTAT_SM | 385 CPUSTAT_STOPPED | 386 CPUSTAT_GED); 387 vcpu->arch.sie_block->ecb = 6; 388 vcpu->arch.sie_block->ecb2 = 8; 389 vcpu->arch.sie_block->eca = 0xC1002001U; 390 vcpu->arch.sie_block->fac = (int) (long) facilities; 391 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 392 tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet, 393 (unsigned long) vcpu); 394 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 395 get_cpu_id(&vcpu->arch.cpu_id); 396 vcpu->arch.cpu_id.version = 0xff; 397 return 0; 398 } 399 400 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 401 unsigned int id) 402 { 403 struct kvm_vcpu *vcpu; 404 int rc = -EINVAL; 405 406 if (id >= KVM_MAX_VCPUS) 407 goto out; 408 409 rc = -ENOMEM; 410 411 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 412 if (!vcpu) 413 goto out; 414 415 vcpu->arch.sie_block = (struct kvm_s390_sie_block *) 416 get_zeroed_page(GFP_KERNEL); 417 418 if (!vcpu->arch.sie_block) 419 goto out_free_cpu; 420 421 vcpu->arch.sie_block->icpua = id; 422 if (!kvm_is_ucontrol(kvm)) { 423 if (!kvm->arch.sca) { 424 WARN_ON_ONCE(1); 425 goto out_free_cpu; 426 } 427 if (!kvm->arch.sca->cpu[id].sda) 428 kvm->arch.sca->cpu[id].sda = 429 (__u64) vcpu->arch.sie_block; 430 vcpu->arch.sie_block->scaoh = 431 (__u32)(((__u64)kvm->arch.sca) >> 32); 432 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 433 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 434 } 435 436 spin_lock_init(&vcpu->arch.local_int.lock); 437 INIT_LIST_HEAD(&vcpu->arch.local_int.list); 438 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 439 spin_lock(&kvm->arch.float_int.lock); 440 kvm->arch.float_int.local_int[id] = &vcpu->arch.local_int; 441 vcpu->arch.local_int.wq = &vcpu->wq; 442 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 443 spin_unlock(&kvm->arch.float_int.lock); 444 445 rc = kvm_vcpu_init(vcpu, kvm, id); 446 if (rc) 447 goto out_free_sie_block; 448 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, 449 vcpu->arch.sie_block); 450 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 451 452 return vcpu; 453 out_free_sie_block: 454 free_page((unsigned long)(vcpu->arch.sie_block)); 455 out_free_cpu: 456 kmem_cache_free(kvm_vcpu_cache, vcpu); 457 out: 458 return ERR_PTR(rc); 459 } 460 461 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 462 { 463 /* kvm common code refers to this, but never calls it */ 464 BUG(); 465 return 0; 466 } 467 468 void s390_vcpu_block(struct kvm_vcpu *vcpu) 469 { 470 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 471 } 472 473 void s390_vcpu_unblock(struct kvm_vcpu *vcpu) 474 { 475 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 476 } 477 478 /* 479 * Kick a guest cpu out of SIE and wait until SIE is not running. 480 * If the CPU is not running (e.g. waiting as idle) the function will 481 * return immediately. */ 482 void exit_sie(struct kvm_vcpu *vcpu) 483 { 484 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 485 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 486 cpu_relax(); 487 } 488 489 /* Kick a guest cpu out of SIE and prevent SIE-reentry */ 490 void exit_sie_sync(struct kvm_vcpu *vcpu) 491 { 492 s390_vcpu_block(vcpu); 493 exit_sie(vcpu); 494 } 495 496 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address) 497 { 498 int i; 499 struct kvm *kvm = gmap->private; 500 struct kvm_vcpu *vcpu; 501 502 kvm_for_each_vcpu(i, vcpu, kvm) { 503 /* match against both prefix pages */ 504 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) { 505 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address); 506 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 507 exit_sie_sync(vcpu); 508 } 509 } 510 } 511 512 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 513 { 514 /* kvm common code refers to this, but never calls it */ 515 BUG(); 516 return 0; 517 } 518 519 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 520 struct kvm_one_reg *reg) 521 { 522 int r = -EINVAL; 523 524 switch (reg->id) { 525 case KVM_REG_S390_TODPR: 526 r = put_user(vcpu->arch.sie_block->todpr, 527 (u32 __user *)reg->addr); 528 break; 529 case KVM_REG_S390_EPOCHDIFF: 530 r = put_user(vcpu->arch.sie_block->epoch, 531 (u64 __user *)reg->addr); 532 break; 533 case KVM_REG_S390_CPU_TIMER: 534 r = put_user(vcpu->arch.sie_block->cputm, 535 (u64 __user *)reg->addr); 536 break; 537 case KVM_REG_S390_CLOCK_COMP: 538 r = put_user(vcpu->arch.sie_block->ckc, 539 (u64 __user *)reg->addr); 540 break; 541 default: 542 break; 543 } 544 545 return r; 546 } 547 548 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 549 struct kvm_one_reg *reg) 550 { 551 int r = -EINVAL; 552 553 switch (reg->id) { 554 case KVM_REG_S390_TODPR: 555 r = get_user(vcpu->arch.sie_block->todpr, 556 (u32 __user *)reg->addr); 557 break; 558 case KVM_REG_S390_EPOCHDIFF: 559 r = get_user(vcpu->arch.sie_block->epoch, 560 (u64 __user *)reg->addr); 561 break; 562 case KVM_REG_S390_CPU_TIMER: 563 r = get_user(vcpu->arch.sie_block->cputm, 564 (u64 __user *)reg->addr); 565 break; 566 case KVM_REG_S390_CLOCK_COMP: 567 r = get_user(vcpu->arch.sie_block->ckc, 568 (u64 __user *)reg->addr); 569 break; 570 default: 571 break; 572 } 573 574 return r; 575 } 576 577 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 578 { 579 kvm_s390_vcpu_initial_reset(vcpu); 580 return 0; 581 } 582 583 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 584 { 585 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 586 return 0; 587 } 588 589 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 590 { 591 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 592 return 0; 593 } 594 595 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 596 struct kvm_sregs *sregs) 597 { 598 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 599 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 600 restore_access_regs(vcpu->run->s.regs.acrs); 601 return 0; 602 } 603 604 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 605 struct kvm_sregs *sregs) 606 { 607 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 608 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 609 return 0; 610 } 611 612 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 613 { 614 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 615 vcpu->arch.guest_fpregs.fpc = fpu->fpc & FPC_VALID_MASK; 616 restore_fp_regs(&vcpu->arch.guest_fpregs); 617 return 0; 618 } 619 620 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 621 { 622 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs)); 623 fpu->fpc = vcpu->arch.guest_fpregs.fpc; 624 return 0; 625 } 626 627 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 628 { 629 int rc = 0; 630 631 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED)) 632 rc = -EBUSY; 633 else { 634 vcpu->run->psw_mask = psw.mask; 635 vcpu->run->psw_addr = psw.addr; 636 } 637 return rc; 638 } 639 640 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 641 struct kvm_translation *tr) 642 { 643 return -EINVAL; /* not implemented yet */ 644 } 645 646 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 647 struct kvm_guest_debug *dbg) 648 { 649 return -EINVAL; /* not implemented yet */ 650 } 651 652 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 653 struct kvm_mp_state *mp_state) 654 { 655 return -EINVAL; /* not implemented yet */ 656 } 657 658 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 659 struct kvm_mp_state *mp_state) 660 { 661 return -EINVAL; /* not implemented yet */ 662 } 663 664 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 665 { 666 /* 667 * We use MMU_RELOAD just to re-arm the ipte notifier for the 668 * guest prefix page. gmap_ipte_notify will wait on the ptl lock. 669 * This ensures that the ipte instruction for this request has 670 * already finished. We might race against a second unmapper that 671 * wants to set the blocking bit. Lets just retry the request loop. 672 */ 673 while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 674 int rc; 675 rc = gmap_ipte_notify(vcpu->arch.gmap, 676 vcpu->arch.sie_block->prefix, 677 PAGE_SIZE * 2); 678 if (rc) 679 return rc; 680 s390_vcpu_unblock(vcpu); 681 } 682 return 0; 683 } 684 685 static int __vcpu_run(struct kvm_vcpu *vcpu) 686 { 687 int rc; 688 689 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16); 690 691 if (need_resched()) 692 schedule(); 693 694 if (test_thread_flag(TIF_MCCK_PENDING)) 695 s390_handle_mcck(); 696 697 if (!kvm_is_ucontrol(vcpu->kvm)) 698 kvm_s390_deliver_pending_interrupts(vcpu); 699 700 rc = kvm_s390_handle_requests(vcpu); 701 if (rc) 702 return rc; 703 704 vcpu->arch.sie_block->icptcode = 0; 705 preempt_disable(); 706 kvm_guest_enter(); 707 preempt_enable(); 708 VCPU_EVENT(vcpu, 6, "entering sie flags %x", 709 atomic_read(&vcpu->arch.sie_block->cpuflags)); 710 trace_kvm_s390_sie_enter(vcpu, 711 atomic_read(&vcpu->arch.sie_block->cpuflags)); 712 rc = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); 713 if (rc > 0) 714 rc = 0; 715 if (rc < 0) { 716 if (kvm_is_ucontrol(vcpu->kvm)) { 717 rc = SIE_INTERCEPT_UCONTROL; 718 } else { 719 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 720 trace_kvm_s390_sie_fault(vcpu); 721 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 722 } 723 } 724 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 725 vcpu->arch.sie_block->icptcode); 726 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 727 kvm_guest_exit(); 728 729 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 730 return rc; 731 } 732 733 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 734 { 735 int rc; 736 sigset_t sigsaved; 737 738 rerun_vcpu: 739 if (vcpu->sigset_active) 740 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 741 742 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 743 744 BUG_ON(vcpu->kvm->arch.float_int.local_int[vcpu->vcpu_id] == NULL); 745 746 switch (kvm_run->exit_reason) { 747 case KVM_EXIT_S390_SIEIC: 748 case KVM_EXIT_UNKNOWN: 749 case KVM_EXIT_INTR: 750 case KVM_EXIT_S390_RESET: 751 case KVM_EXIT_S390_UCONTROL: 752 case KVM_EXIT_S390_TSCH: 753 break; 754 default: 755 BUG(); 756 } 757 758 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 759 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 760 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) { 761 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX; 762 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 763 } 764 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 765 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS; 766 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 767 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 768 } 769 770 might_fault(); 771 772 do { 773 rc = __vcpu_run(vcpu); 774 if (rc) 775 break; 776 if (kvm_is_ucontrol(vcpu->kvm)) 777 rc = -EOPNOTSUPP; 778 else 779 rc = kvm_handle_sie_intercept(vcpu); 780 } while (!signal_pending(current) && !rc); 781 782 if (rc == SIE_INTERCEPT_RERUNVCPU) 783 goto rerun_vcpu; 784 785 if (signal_pending(current) && !rc) { 786 kvm_run->exit_reason = KVM_EXIT_INTR; 787 rc = -EINTR; 788 } 789 790 #ifdef CONFIG_KVM_S390_UCONTROL 791 if (rc == SIE_INTERCEPT_UCONTROL) { 792 kvm_run->exit_reason = KVM_EXIT_S390_UCONTROL; 793 kvm_run->s390_ucontrol.trans_exc_code = 794 current->thread.gmap_addr; 795 kvm_run->s390_ucontrol.pgm_code = 0x10; 796 rc = 0; 797 } 798 #endif 799 800 if (rc == -EOPNOTSUPP) { 801 /* intercept cannot be handled in-kernel, prepare kvm-run */ 802 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC; 803 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 804 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 805 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 806 rc = 0; 807 } 808 809 if (rc == -EREMOTE) { 810 /* intercept was handled, but userspace support is needed 811 * kvm_run has been prepared by the handler */ 812 rc = 0; 813 } 814 815 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 816 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 817 kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix; 818 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 819 820 if (vcpu->sigset_active) 821 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 822 823 vcpu->stat.exit_userspace++; 824 return rc; 825 } 826 827 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from, 828 unsigned long n, int prefix) 829 { 830 if (prefix) 831 return copy_to_guest(vcpu, guestdest, from, n); 832 else 833 return copy_to_guest_absolute(vcpu, guestdest, from, n); 834 } 835 836 /* 837 * store status at address 838 * we use have two special cases: 839 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 840 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 841 */ 842 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 843 { 844 unsigned char archmode = 1; 845 int prefix; 846 847 if (addr == KVM_S390_STORE_STATUS_NOADDR) { 848 if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1)) 849 return -EFAULT; 850 addr = SAVE_AREA_BASE; 851 prefix = 0; 852 } else if (addr == KVM_S390_STORE_STATUS_PREFIXED) { 853 if (copy_to_guest(vcpu, 163ul, &archmode, 1)) 854 return -EFAULT; 855 addr = SAVE_AREA_BASE; 856 prefix = 1; 857 } else 858 prefix = 0; 859 860 /* 861 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 862 * copying in vcpu load/put. Lets update our copies before we save 863 * it into the save area 864 */ 865 save_fp_regs(&vcpu->arch.guest_fpregs); 866 save_access_regs(vcpu->run->s.regs.acrs); 867 868 if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs), 869 vcpu->arch.guest_fpregs.fprs, 128, prefix)) 870 return -EFAULT; 871 872 if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs), 873 vcpu->run->s.regs.gprs, 128, prefix)) 874 return -EFAULT; 875 876 if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw), 877 &vcpu->arch.sie_block->gpsw, 16, prefix)) 878 return -EFAULT; 879 880 if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg), 881 &vcpu->arch.sie_block->prefix, 4, prefix)) 882 return -EFAULT; 883 884 if (__guestcopy(vcpu, 885 addr + offsetof(struct save_area, fp_ctrl_reg), 886 &vcpu->arch.guest_fpregs.fpc, 4, prefix)) 887 return -EFAULT; 888 889 if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg), 890 &vcpu->arch.sie_block->todpr, 4, prefix)) 891 return -EFAULT; 892 893 if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer), 894 &vcpu->arch.sie_block->cputm, 8, prefix)) 895 return -EFAULT; 896 897 if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp), 898 &vcpu->arch.sie_block->ckc, 8, prefix)) 899 return -EFAULT; 900 901 if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs), 902 &vcpu->run->s.regs.acrs, 64, prefix)) 903 return -EFAULT; 904 905 if (__guestcopy(vcpu, 906 addr + offsetof(struct save_area, ctrl_regs), 907 &vcpu->arch.sie_block->gcr, 128, prefix)) 908 return -EFAULT; 909 return 0; 910 } 911 912 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 913 struct kvm_enable_cap *cap) 914 { 915 int r; 916 917 if (cap->flags) 918 return -EINVAL; 919 920 switch (cap->cap) { 921 case KVM_CAP_S390_CSS_SUPPORT: 922 if (!vcpu->kvm->arch.css_support) { 923 vcpu->kvm->arch.css_support = 1; 924 trace_kvm_s390_enable_css(vcpu->kvm); 925 } 926 r = 0; 927 break; 928 default: 929 r = -EINVAL; 930 break; 931 } 932 return r; 933 } 934 935 long kvm_arch_vcpu_ioctl(struct file *filp, 936 unsigned int ioctl, unsigned long arg) 937 { 938 struct kvm_vcpu *vcpu = filp->private_data; 939 void __user *argp = (void __user *)arg; 940 long r; 941 942 switch (ioctl) { 943 case KVM_S390_INTERRUPT: { 944 struct kvm_s390_interrupt s390int; 945 946 r = -EFAULT; 947 if (copy_from_user(&s390int, argp, sizeof(s390int))) 948 break; 949 r = kvm_s390_inject_vcpu(vcpu, &s390int); 950 break; 951 } 952 case KVM_S390_STORE_STATUS: 953 r = kvm_s390_vcpu_store_status(vcpu, arg); 954 break; 955 case KVM_S390_SET_INITIAL_PSW: { 956 psw_t psw; 957 958 r = -EFAULT; 959 if (copy_from_user(&psw, argp, sizeof(psw))) 960 break; 961 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 962 break; 963 } 964 case KVM_S390_INITIAL_RESET: 965 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 966 break; 967 case KVM_SET_ONE_REG: 968 case KVM_GET_ONE_REG: { 969 struct kvm_one_reg reg; 970 r = -EFAULT; 971 if (copy_from_user(®, argp, sizeof(reg))) 972 break; 973 if (ioctl == KVM_SET_ONE_REG) 974 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 975 else 976 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 977 break; 978 } 979 #ifdef CONFIG_KVM_S390_UCONTROL 980 case KVM_S390_UCAS_MAP: { 981 struct kvm_s390_ucas_mapping ucasmap; 982 983 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 984 r = -EFAULT; 985 break; 986 } 987 988 if (!kvm_is_ucontrol(vcpu->kvm)) { 989 r = -EINVAL; 990 break; 991 } 992 993 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 994 ucasmap.vcpu_addr, ucasmap.length); 995 break; 996 } 997 case KVM_S390_UCAS_UNMAP: { 998 struct kvm_s390_ucas_mapping ucasmap; 999 1000 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 1001 r = -EFAULT; 1002 break; 1003 } 1004 1005 if (!kvm_is_ucontrol(vcpu->kvm)) { 1006 r = -EINVAL; 1007 break; 1008 } 1009 1010 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 1011 ucasmap.length); 1012 break; 1013 } 1014 #endif 1015 case KVM_S390_VCPU_FAULT: { 1016 r = gmap_fault(arg, vcpu->arch.gmap); 1017 if (!IS_ERR_VALUE(r)) 1018 r = 0; 1019 break; 1020 } 1021 case KVM_ENABLE_CAP: 1022 { 1023 struct kvm_enable_cap cap; 1024 r = -EFAULT; 1025 if (copy_from_user(&cap, argp, sizeof(cap))) 1026 break; 1027 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1028 break; 1029 } 1030 default: 1031 r = -ENOTTY; 1032 } 1033 return r; 1034 } 1035 1036 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 1037 { 1038 #ifdef CONFIG_KVM_S390_UCONTROL 1039 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 1040 && (kvm_is_ucontrol(vcpu->kvm))) { 1041 vmf->page = virt_to_page(vcpu->arch.sie_block); 1042 get_page(vmf->page); 1043 return 0; 1044 } 1045 #endif 1046 return VM_FAULT_SIGBUS; 1047 } 1048 1049 void kvm_arch_free_memslot(struct kvm_memory_slot *free, 1050 struct kvm_memory_slot *dont) 1051 { 1052 } 1053 1054 int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages) 1055 { 1056 return 0; 1057 } 1058 1059 /* Section: memory related */ 1060 int kvm_arch_prepare_memory_region(struct kvm *kvm, 1061 struct kvm_memory_slot *memslot, 1062 struct kvm_userspace_memory_region *mem, 1063 enum kvm_mr_change change) 1064 { 1065 /* A few sanity checks. We can have memory slots which have to be 1066 located/ended at a segment boundary (1MB). The memory in userland is 1067 ok to be fragmented into various different vmas. It is okay to mmap() 1068 and munmap() stuff in this slot after doing this call at any time */ 1069 1070 if (mem->userspace_addr & 0xffffful) 1071 return -EINVAL; 1072 1073 if (mem->memory_size & 0xffffful) 1074 return -EINVAL; 1075 1076 return 0; 1077 } 1078 1079 void kvm_arch_commit_memory_region(struct kvm *kvm, 1080 struct kvm_userspace_memory_region *mem, 1081 const struct kvm_memory_slot *old, 1082 enum kvm_mr_change change) 1083 { 1084 int rc; 1085 1086 /* If the basics of the memslot do not change, we do not want 1087 * to update the gmap. Every update causes several unnecessary 1088 * segment translation exceptions. This is usually handled just 1089 * fine by the normal fault handler + gmap, but it will also 1090 * cause faults on the prefix page of running guest CPUs. 1091 */ 1092 if (old->userspace_addr == mem->userspace_addr && 1093 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 1094 old->npages * PAGE_SIZE == mem->memory_size) 1095 return; 1096 1097 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 1098 mem->guest_phys_addr, mem->memory_size); 1099 if (rc) 1100 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n"); 1101 return; 1102 } 1103 1104 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1105 { 1106 } 1107 1108 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1109 struct kvm_memory_slot *slot) 1110 { 1111 } 1112 1113 static int __init kvm_s390_init(void) 1114 { 1115 int ret; 1116 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 1117 if (ret) 1118 return ret; 1119 1120 /* 1121 * guests can ask for up to 255+1 double words, we need a full page 1122 * to hold the maximum amount of facilities. On the other hand, we 1123 * only set facilities that are known to work in KVM. 1124 */ 1125 facilities = (unsigned long long *) get_zeroed_page(GFP_KERNEL|GFP_DMA); 1126 if (!facilities) { 1127 kvm_exit(); 1128 return -ENOMEM; 1129 } 1130 memcpy(facilities, S390_lowcore.stfle_fac_list, 16); 1131 facilities[0] &= 0xff82fff3f47c0000ULL; 1132 facilities[1] &= 0x001c000000000000ULL; 1133 return 0; 1134 } 1135 1136 static void __exit kvm_s390_exit(void) 1137 { 1138 free_page((unsigned long) facilities); 1139 kvm_exit(); 1140 } 1141 1142 module_init(kvm_s390_init); 1143 module_exit(kvm_s390_exit); 1144 1145 /* 1146 * Enable autoloading of the kvm module. 1147 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 1148 * since x86 takes a different approach. 1149 */ 1150 #include <linux/miscdevice.h> 1151 MODULE_ALIAS_MISCDEV(KVM_MINOR); 1152 MODULE_ALIAS("devname:kvm"); 1153