1 /* 2 * hosting zSeries kernel virtual machines 3 * 4 * Copyright IBM Corp. 2008, 2009 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 * Christian Borntraeger <borntraeger@de.ibm.com> 12 * Heiko Carstens <heiko.carstens@de.ibm.com> 13 * Christian Ehrhardt <ehrhardt@de.ibm.com> 14 */ 15 16 #include <linux/compiler.h> 17 #include <linux/err.h> 18 #include <linux/fs.h> 19 #include <linux/hrtimer.h> 20 #include <linux/init.h> 21 #include <linux/kvm.h> 22 #include <linux/kvm_host.h> 23 #include <linux/module.h> 24 #include <linux/slab.h> 25 #include <linux/timer.h> 26 #include <asm/asm-offsets.h> 27 #include <asm/lowcore.h> 28 #include <asm/pgtable.h> 29 #include <asm/nmi.h> 30 #include <asm/switch_to.h> 31 #include <asm/sclp.h> 32 #include "kvm-s390.h" 33 #include "gaccess.h" 34 35 #define CREATE_TRACE_POINTS 36 #include "trace.h" 37 #include "trace-s390.h" 38 39 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 40 41 struct kvm_stats_debugfs_item debugfs_entries[] = { 42 { "userspace_handled", VCPU_STAT(exit_userspace) }, 43 { "exit_null", VCPU_STAT(exit_null) }, 44 { "exit_validity", VCPU_STAT(exit_validity) }, 45 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 46 { "exit_external_request", VCPU_STAT(exit_external_request) }, 47 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 48 { "exit_instruction", VCPU_STAT(exit_instruction) }, 49 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 50 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 51 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 52 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 53 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 54 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 55 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 56 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 57 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 58 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 59 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 60 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 61 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 62 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 63 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 64 { "instruction_spx", VCPU_STAT(instruction_spx) }, 65 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 66 { "instruction_stap", VCPU_STAT(instruction_stap) }, 67 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 68 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 69 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 70 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 71 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 72 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 73 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 74 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 75 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 76 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 77 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 78 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 79 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 80 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 81 { "diagnose_10", VCPU_STAT(diagnose_10) }, 82 { "diagnose_44", VCPU_STAT(diagnose_44) }, 83 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 84 { NULL } 85 }; 86 87 static unsigned long long *facilities; 88 static struct gmap_notifier gmap_notifier; 89 90 /* Section: not file related */ 91 int kvm_arch_hardware_enable(void *garbage) 92 { 93 /* every s390 is virtualization enabled ;-) */ 94 return 0; 95 } 96 97 void kvm_arch_hardware_disable(void *garbage) 98 { 99 } 100 101 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address); 102 103 int kvm_arch_hardware_setup(void) 104 { 105 gmap_notifier.notifier_call = kvm_gmap_notifier; 106 gmap_register_ipte_notifier(&gmap_notifier); 107 return 0; 108 } 109 110 void kvm_arch_hardware_unsetup(void) 111 { 112 gmap_unregister_ipte_notifier(&gmap_notifier); 113 } 114 115 void kvm_arch_check_processor_compat(void *rtn) 116 { 117 } 118 119 int kvm_arch_init(void *opaque) 120 { 121 return 0; 122 } 123 124 void kvm_arch_exit(void) 125 { 126 } 127 128 /* Section: device related */ 129 long kvm_arch_dev_ioctl(struct file *filp, 130 unsigned int ioctl, unsigned long arg) 131 { 132 if (ioctl == KVM_S390_ENABLE_SIE) 133 return s390_enable_sie(); 134 return -EINVAL; 135 } 136 137 int kvm_dev_ioctl_check_extension(long ext) 138 { 139 int r; 140 141 switch (ext) { 142 case KVM_CAP_S390_PSW: 143 case KVM_CAP_S390_GMAP: 144 case KVM_CAP_SYNC_MMU: 145 #ifdef CONFIG_KVM_S390_UCONTROL 146 case KVM_CAP_S390_UCONTROL: 147 #endif 148 case KVM_CAP_SYNC_REGS: 149 case KVM_CAP_ONE_REG: 150 case KVM_CAP_ENABLE_CAP: 151 case KVM_CAP_S390_CSS_SUPPORT: 152 case KVM_CAP_IOEVENTFD: 153 r = 1; 154 break; 155 case KVM_CAP_NR_VCPUS: 156 case KVM_CAP_MAX_VCPUS: 157 r = KVM_MAX_VCPUS; 158 break; 159 case KVM_CAP_NR_MEMSLOTS: 160 r = KVM_USER_MEM_SLOTS; 161 break; 162 case KVM_CAP_S390_COW: 163 r = MACHINE_HAS_ESOP; 164 break; 165 default: 166 r = 0; 167 } 168 return r; 169 } 170 171 /* Section: vm related */ 172 /* 173 * Get (and clear) the dirty memory log for a memory slot. 174 */ 175 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 176 struct kvm_dirty_log *log) 177 { 178 return 0; 179 } 180 181 long kvm_arch_vm_ioctl(struct file *filp, 182 unsigned int ioctl, unsigned long arg) 183 { 184 struct kvm *kvm = filp->private_data; 185 void __user *argp = (void __user *)arg; 186 int r; 187 188 switch (ioctl) { 189 case KVM_S390_INTERRUPT: { 190 struct kvm_s390_interrupt s390int; 191 192 r = -EFAULT; 193 if (copy_from_user(&s390int, argp, sizeof(s390int))) 194 break; 195 r = kvm_s390_inject_vm(kvm, &s390int); 196 break; 197 } 198 default: 199 r = -ENOTTY; 200 } 201 202 return r; 203 } 204 205 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 206 { 207 int rc; 208 char debug_name[16]; 209 210 rc = -EINVAL; 211 #ifdef CONFIG_KVM_S390_UCONTROL 212 if (type & ~KVM_VM_S390_UCONTROL) 213 goto out_err; 214 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 215 goto out_err; 216 #else 217 if (type) 218 goto out_err; 219 #endif 220 221 rc = s390_enable_sie(); 222 if (rc) 223 goto out_err; 224 225 rc = -ENOMEM; 226 227 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); 228 if (!kvm->arch.sca) 229 goto out_err; 230 231 sprintf(debug_name, "kvm-%u", current->pid); 232 233 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 234 if (!kvm->arch.dbf) 235 goto out_nodbf; 236 237 spin_lock_init(&kvm->arch.float_int.lock); 238 INIT_LIST_HEAD(&kvm->arch.float_int.list); 239 240 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 241 VM_EVENT(kvm, 3, "%s", "vm created"); 242 243 if (type & KVM_VM_S390_UCONTROL) { 244 kvm->arch.gmap = NULL; 245 } else { 246 kvm->arch.gmap = gmap_alloc(current->mm); 247 if (!kvm->arch.gmap) 248 goto out_nogmap; 249 kvm->arch.gmap->private = kvm; 250 } 251 252 kvm->arch.css_support = 0; 253 254 return 0; 255 out_nogmap: 256 debug_unregister(kvm->arch.dbf); 257 out_nodbf: 258 free_page((unsigned long)(kvm->arch.sca)); 259 out_err: 260 return rc; 261 } 262 263 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 264 { 265 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 266 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 267 if (!kvm_is_ucontrol(vcpu->kvm)) { 268 clear_bit(63 - vcpu->vcpu_id, 269 (unsigned long *) &vcpu->kvm->arch.sca->mcn); 270 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda == 271 (__u64) vcpu->arch.sie_block) 272 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0; 273 } 274 smp_mb(); 275 276 if (kvm_is_ucontrol(vcpu->kvm)) 277 gmap_free(vcpu->arch.gmap); 278 279 free_page((unsigned long)(vcpu->arch.sie_block)); 280 kvm_vcpu_uninit(vcpu); 281 kmem_cache_free(kvm_vcpu_cache, vcpu); 282 } 283 284 static void kvm_free_vcpus(struct kvm *kvm) 285 { 286 unsigned int i; 287 struct kvm_vcpu *vcpu; 288 289 kvm_for_each_vcpu(i, vcpu, kvm) 290 kvm_arch_vcpu_destroy(vcpu); 291 292 mutex_lock(&kvm->lock); 293 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 294 kvm->vcpus[i] = NULL; 295 296 atomic_set(&kvm->online_vcpus, 0); 297 mutex_unlock(&kvm->lock); 298 } 299 300 void kvm_arch_sync_events(struct kvm *kvm) 301 { 302 } 303 304 void kvm_arch_destroy_vm(struct kvm *kvm) 305 { 306 kvm_free_vcpus(kvm); 307 free_page((unsigned long)(kvm->arch.sca)); 308 debug_unregister(kvm->arch.dbf); 309 if (!kvm_is_ucontrol(kvm)) 310 gmap_free(kvm->arch.gmap); 311 } 312 313 /* Section: vcpu related */ 314 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 315 { 316 if (kvm_is_ucontrol(vcpu->kvm)) { 317 vcpu->arch.gmap = gmap_alloc(current->mm); 318 if (!vcpu->arch.gmap) 319 return -ENOMEM; 320 vcpu->arch.gmap->private = vcpu->kvm; 321 return 0; 322 } 323 324 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 325 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 326 KVM_SYNC_GPRS | 327 KVM_SYNC_ACRS | 328 KVM_SYNC_CRS; 329 return 0; 330 } 331 332 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 333 { 334 /* Nothing todo */ 335 } 336 337 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 338 { 339 save_fp_regs(&vcpu->arch.host_fpregs); 340 save_access_regs(vcpu->arch.host_acrs); 341 vcpu->arch.guest_fpregs.fpc &= FPC_VALID_MASK; 342 restore_fp_regs(&vcpu->arch.guest_fpregs); 343 restore_access_regs(vcpu->run->s.regs.acrs); 344 gmap_enable(vcpu->arch.gmap); 345 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 346 } 347 348 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 349 { 350 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 351 gmap_disable(vcpu->arch.gmap); 352 save_fp_regs(&vcpu->arch.guest_fpregs); 353 save_access_regs(vcpu->run->s.regs.acrs); 354 restore_fp_regs(&vcpu->arch.host_fpregs); 355 restore_access_regs(vcpu->arch.host_acrs); 356 } 357 358 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 359 { 360 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 361 vcpu->arch.sie_block->gpsw.mask = 0UL; 362 vcpu->arch.sie_block->gpsw.addr = 0UL; 363 kvm_s390_set_prefix(vcpu, 0); 364 vcpu->arch.sie_block->cputm = 0UL; 365 vcpu->arch.sie_block->ckc = 0UL; 366 vcpu->arch.sie_block->todpr = 0; 367 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 368 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 369 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 370 vcpu->arch.guest_fpregs.fpc = 0; 371 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc)); 372 vcpu->arch.sie_block->gbea = 1; 373 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 374 } 375 376 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 377 { 378 return 0; 379 } 380 381 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 382 { 383 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 384 CPUSTAT_SM | 385 CPUSTAT_STOPPED | 386 CPUSTAT_GED); 387 vcpu->arch.sie_block->ecb = 6; 388 vcpu->arch.sie_block->ecb2 = 8; 389 vcpu->arch.sie_block->eca = 0xC1002001U; 390 vcpu->arch.sie_block->fac = (int) (long) facilities; 391 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 392 tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet, 393 (unsigned long) vcpu); 394 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 395 get_cpu_id(&vcpu->arch.cpu_id); 396 vcpu->arch.cpu_id.version = 0xff; 397 return 0; 398 } 399 400 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 401 unsigned int id) 402 { 403 struct kvm_vcpu *vcpu; 404 int rc = -EINVAL; 405 406 if (id >= KVM_MAX_VCPUS) 407 goto out; 408 409 rc = -ENOMEM; 410 411 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 412 if (!vcpu) 413 goto out; 414 415 vcpu->arch.sie_block = (struct kvm_s390_sie_block *) 416 get_zeroed_page(GFP_KERNEL); 417 418 if (!vcpu->arch.sie_block) 419 goto out_free_cpu; 420 421 vcpu->arch.sie_block->icpua = id; 422 if (!kvm_is_ucontrol(kvm)) { 423 if (!kvm->arch.sca) { 424 WARN_ON_ONCE(1); 425 goto out_free_cpu; 426 } 427 if (!kvm->arch.sca->cpu[id].sda) 428 kvm->arch.sca->cpu[id].sda = 429 (__u64) vcpu->arch.sie_block; 430 vcpu->arch.sie_block->scaoh = 431 (__u32)(((__u64)kvm->arch.sca) >> 32); 432 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 433 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 434 } 435 436 spin_lock_init(&vcpu->arch.local_int.lock); 437 INIT_LIST_HEAD(&vcpu->arch.local_int.list); 438 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 439 spin_lock(&kvm->arch.float_int.lock); 440 kvm->arch.float_int.local_int[id] = &vcpu->arch.local_int; 441 vcpu->arch.local_int.wq = &vcpu->wq; 442 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 443 spin_unlock(&kvm->arch.float_int.lock); 444 445 rc = kvm_vcpu_init(vcpu, kvm, id); 446 if (rc) 447 goto out_free_sie_block; 448 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, 449 vcpu->arch.sie_block); 450 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 451 452 return vcpu; 453 out_free_sie_block: 454 free_page((unsigned long)(vcpu->arch.sie_block)); 455 out_free_cpu: 456 kmem_cache_free(kvm_vcpu_cache, vcpu); 457 out: 458 return ERR_PTR(rc); 459 } 460 461 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 462 { 463 /* kvm common code refers to this, but never calls it */ 464 BUG(); 465 return 0; 466 } 467 468 void s390_vcpu_block(struct kvm_vcpu *vcpu) 469 { 470 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 471 } 472 473 void s390_vcpu_unblock(struct kvm_vcpu *vcpu) 474 { 475 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 476 } 477 478 /* 479 * Kick a guest cpu out of SIE and wait until SIE is not running. 480 * If the CPU is not running (e.g. waiting as idle) the function will 481 * return immediately. */ 482 void exit_sie(struct kvm_vcpu *vcpu) 483 { 484 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 485 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 486 cpu_relax(); 487 } 488 489 /* Kick a guest cpu out of SIE and prevent SIE-reentry */ 490 void exit_sie_sync(struct kvm_vcpu *vcpu) 491 { 492 s390_vcpu_block(vcpu); 493 exit_sie(vcpu); 494 } 495 496 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address) 497 { 498 int i; 499 struct kvm *kvm = gmap->private; 500 struct kvm_vcpu *vcpu; 501 502 kvm_for_each_vcpu(i, vcpu, kvm) { 503 /* match against both prefix pages */ 504 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) { 505 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address); 506 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 507 exit_sie_sync(vcpu); 508 } 509 } 510 } 511 512 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 513 { 514 /* kvm common code refers to this, but never calls it */ 515 BUG(); 516 return 0; 517 } 518 519 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 520 struct kvm_one_reg *reg) 521 { 522 int r = -EINVAL; 523 524 switch (reg->id) { 525 case KVM_REG_S390_TODPR: 526 r = put_user(vcpu->arch.sie_block->todpr, 527 (u32 __user *)reg->addr); 528 break; 529 case KVM_REG_S390_EPOCHDIFF: 530 r = put_user(vcpu->arch.sie_block->epoch, 531 (u64 __user *)reg->addr); 532 break; 533 case KVM_REG_S390_CPU_TIMER: 534 r = put_user(vcpu->arch.sie_block->cputm, 535 (u64 __user *)reg->addr); 536 break; 537 case KVM_REG_S390_CLOCK_COMP: 538 r = put_user(vcpu->arch.sie_block->ckc, 539 (u64 __user *)reg->addr); 540 break; 541 default: 542 break; 543 } 544 545 return r; 546 } 547 548 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 549 struct kvm_one_reg *reg) 550 { 551 int r = -EINVAL; 552 553 switch (reg->id) { 554 case KVM_REG_S390_TODPR: 555 r = get_user(vcpu->arch.sie_block->todpr, 556 (u32 __user *)reg->addr); 557 break; 558 case KVM_REG_S390_EPOCHDIFF: 559 r = get_user(vcpu->arch.sie_block->epoch, 560 (u64 __user *)reg->addr); 561 break; 562 case KVM_REG_S390_CPU_TIMER: 563 r = get_user(vcpu->arch.sie_block->cputm, 564 (u64 __user *)reg->addr); 565 break; 566 case KVM_REG_S390_CLOCK_COMP: 567 r = get_user(vcpu->arch.sie_block->ckc, 568 (u64 __user *)reg->addr); 569 break; 570 default: 571 break; 572 } 573 574 return r; 575 } 576 577 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 578 { 579 kvm_s390_vcpu_initial_reset(vcpu); 580 return 0; 581 } 582 583 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 584 { 585 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 586 return 0; 587 } 588 589 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 590 { 591 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 592 return 0; 593 } 594 595 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 596 struct kvm_sregs *sregs) 597 { 598 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 599 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 600 restore_access_regs(vcpu->run->s.regs.acrs); 601 return 0; 602 } 603 604 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 605 struct kvm_sregs *sregs) 606 { 607 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 608 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 609 return 0; 610 } 611 612 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 613 { 614 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 615 vcpu->arch.guest_fpregs.fpc = fpu->fpc & FPC_VALID_MASK; 616 restore_fp_regs(&vcpu->arch.guest_fpregs); 617 return 0; 618 } 619 620 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 621 { 622 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs)); 623 fpu->fpc = vcpu->arch.guest_fpregs.fpc; 624 return 0; 625 } 626 627 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 628 { 629 int rc = 0; 630 631 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED)) 632 rc = -EBUSY; 633 else { 634 vcpu->run->psw_mask = psw.mask; 635 vcpu->run->psw_addr = psw.addr; 636 } 637 return rc; 638 } 639 640 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 641 struct kvm_translation *tr) 642 { 643 return -EINVAL; /* not implemented yet */ 644 } 645 646 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 647 struct kvm_guest_debug *dbg) 648 { 649 return -EINVAL; /* not implemented yet */ 650 } 651 652 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 653 struct kvm_mp_state *mp_state) 654 { 655 return -EINVAL; /* not implemented yet */ 656 } 657 658 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 659 struct kvm_mp_state *mp_state) 660 { 661 return -EINVAL; /* not implemented yet */ 662 } 663 664 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 665 { 666 /* 667 * We use MMU_RELOAD just to re-arm the ipte notifier for the 668 * guest prefix page. gmap_ipte_notify will wait on the ptl lock. 669 * This ensures that the ipte instruction for this request has 670 * already finished. We might race against a second unmapper that 671 * wants to set the blocking bit. Lets just retry the request loop. 672 */ 673 while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 674 int rc; 675 rc = gmap_ipte_notify(vcpu->arch.gmap, 676 vcpu->arch.sie_block->prefix, 677 PAGE_SIZE * 2); 678 if (rc) 679 return rc; 680 s390_vcpu_unblock(vcpu); 681 } 682 return 0; 683 } 684 685 static int __vcpu_run(struct kvm_vcpu *vcpu) 686 { 687 int rc; 688 689 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16); 690 691 if (need_resched()) 692 schedule(); 693 694 if (test_thread_flag(TIF_MCCK_PENDING)) 695 s390_handle_mcck(); 696 697 if (!kvm_is_ucontrol(vcpu->kvm)) 698 kvm_s390_deliver_pending_interrupts(vcpu); 699 700 rc = kvm_s390_handle_requests(vcpu); 701 if (rc) 702 return rc; 703 704 vcpu->arch.sie_block->icptcode = 0; 705 VCPU_EVENT(vcpu, 6, "entering sie flags %x", 706 atomic_read(&vcpu->arch.sie_block->cpuflags)); 707 trace_kvm_s390_sie_enter(vcpu, 708 atomic_read(&vcpu->arch.sie_block->cpuflags)); 709 710 /* 711 * As PF_VCPU will be used in fault handler, between guest_enter 712 * and guest_exit should be no uaccess. 713 */ 714 preempt_disable(); 715 kvm_guest_enter(); 716 preempt_enable(); 717 rc = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); 718 kvm_guest_exit(); 719 720 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 721 vcpu->arch.sie_block->icptcode); 722 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 723 724 if (rc > 0) 725 rc = 0; 726 if (rc < 0) { 727 if (kvm_is_ucontrol(vcpu->kvm)) { 728 rc = SIE_INTERCEPT_UCONTROL; 729 } else { 730 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 731 trace_kvm_s390_sie_fault(vcpu); 732 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 733 } 734 } 735 736 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 737 return rc; 738 } 739 740 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 741 { 742 int rc; 743 sigset_t sigsaved; 744 745 rerun_vcpu: 746 if (vcpu->sigset_active) 747 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 748 749 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 750 751 BUG_ON(vcpu->kvm->arch.float_int.local_int[vcpu->vcpu_id] == NULL); 752 753 switch (kvm_run->exit_reason) { 754 case KVM_EXIT_S390_SIEIC: 755 case KVM_EXIT_UNKNOWN: 756 case KVM_EXIT_INTR: 757 case KVM_EXIT_S390_RESET: 758 case KVM_EXIT_S390_UCONTROL: 759 case KVM_EXIT_S390_TSCH: 760 break; 761 default: 762 BUG(); 763 } 764 765 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 766 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 767 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) { 768 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX; 769 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 770 } 771 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 772 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS; 773 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 774 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 775 } 776 777 might_fault(); 778 779 do { 780 rc = __vcpu_run(vcpu); 781 if (rc) 782 break; 783 if (kvm_is_ucontrol(vcpu->kvm)) 784 rc = -EOPNOTSUPP; 785 else 786 rc = kvm_handle_sie_intercept(vcpu); 787 } while (!signal_pending(current) && !rc); 788 789 if (rc == SIE_INTERCEPT_RERUNVCPU) 790 goto rerun_vcpu; 791 792 if (signal_pending(current) && !rc) { 793 kvm_run->exit_reason = KVM_EXIT_INTR; 794 rc = -EINTR; 795 } 796 797 #ifdef CONFIG_KVM_S390_UCONTROL 798 if (rc == SIE_INTERCEPT_UCONTROL) { 799 kvm_run->exit_reason = KVM_EXIT_S390_UCONTROL; 800 kvm_run->s390_ucontrol.trans_exc_code = 801 current->thread.gmap_addr; 802 kvm_run->s390_ucontrol.pgm_code = 0x10; 803 rc = 0; 804 } 805 #endif 806 807 if (rc == -EOPNOTSUPP) { 808 /* intercept cannot be handled in-kernel, prepare kvm-run */ 809 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC; 810 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 811 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 812 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 813 rc = 0; 814 } 815 816 if (rc == -EREMOTE) { 817 /* intercept was handled, but userspace support is needed 818 * kvm_run has been prepared by the handler */ 819 rc = 0; 820 } 821 822 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 823 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 824 kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix; 825 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 826 827 if (vcpu->sigset_active) 828 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 829 830 vcpu->stat.exit_userspace++; 831 return rc; 832 } 833 834 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from, 835 unsigned long n, int prefix) 836 { 837 if (prefix) 838 return copy_to_guest(vcpu, guestdest, from, n); 839 else 840 return copy_to_guest_absolute(vcpu, guestdest, from, n); 841 } 842 843 /* 844 * store status at address 845 * we use have two special cases: 846 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 847 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 848 */ 849 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 850 { 851 unsigned char archmode = 1; 852 int prefix; 853 854 if (addr == KVM_S390_STORE_STATUS_NOADDR) { 855 if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1)) 856 return -EFAULT; 857 addr = SAVE_AREA_BASE; 858 prefix = 0; 859 } else if (addr == KVM_S390_STORE_STATUS_PREFIXED) { 860 if (copy_to_guest(vcpu, 163ul, &archmode, 1)) 861 return -EFAULT; 862 addr = SAVE_AREA_BASE; 863 prefix = 1; 864 } else 865 prefix = 0; 866 867 /* 868 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 869 * copying in vcpu load/put. Lets update our copies before we save 870 * it into the save area 871 */ 872 save_fp_regs(&vcpu->arch.guest_fpregs); 873 save_access_regs(vcpu->run->s.regs.acrs); 874 875 if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs), 876 vcpu->arch.guest_fpregs.fprs, 128, prefix)) 877 return -EFAULT; 878 879 if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs), 880 vcpu->run->s.regs.gprs, 128, prefix)) 881 return -EFAULT; 882 883 if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw), 884 &vcpu->arch.sie_block->gpsw, 16, prefix)) 885 return -EFAULT; 886 887 if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg), 888 &vcpu->arch.sie_block->prefix, 4, prefix)) 889 return -EFAULT; 890 891 if (__guestcopy(vcpu, 892 addr + offsetof(struct save_area, fp_ctrl_reg), 893 &vcpu->arch.guest_fpregs.fpc, 4, prefix)) 894 return -EFAULT; 895 896 if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg), 897 &vcpu->arch.sie_block->todpr, 4, prefix)) 898 return -EFAULT; 899 900 if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer), 901 &vcpu->arch.sie_block->cputm, 8, prefix)) 902 return -EFAULT; 903 904 if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp), 905 &vcpu->arch.sie_block->ckc, 8, prefix)) 906 return -EFAULT; 907 908 if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs), 909 &vcpu->run->s.regs.acrs, 64, prefix)) 910 return -EFAULT; 911 912 if (__guestcopy(vcpu, 913 addr + offsetof(struct save_area, ctrl_regs), 914 &vcpu->arch.sie_block->gcr, 128, prefix)) 915 return -EFAULT; 916 return 0; 917 } 918 919 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 920 struct kvm_enable_cap *cap) 921 { 922 int r; 923 924 if (cap->flags) 925 return -EINVAL; 926 927 switch (cap->cap) { 928 case KVM_CAP_S390_CSS_SUPPORT: 929 if (!vcpu->kvm->arch.css_support) { 930 vcpu->kvm->arch.css_support = 1; 931 trace_kvm_s390_enable_css(vcpu->kvm); 932 } 933 r = 0; 934 break; 935 default: 936 r = -EINVAL; 937 break; 938 } 939 return r; 940 } 941 942 long kvm_arch_vcpu_ioctl(struct file *filp, 943 unsigned int ioctl, unsigned long arg) 944 { 945 struct kvm_vcpu *vcpu = filp->private_data; 946 void __user *argp = (void __user *)arg; 947 long r; 948 949 switch (ioctl) { 950 case KVM_S390_INTERRUPT: { 951 struct kvm_s390_interrupt s390int; 952 953 r = -EFAULT; 954 if (copy_from_user(&s390int, argp, sizeof(s390int))) 955 break; 956 r = kvm_s390_inject_vcpu(vcpu, &s390int); 957 break; 958 } 959 case KVM_S390_STORE_STATUS: 960 r = kvm_s390_vcpu_store_status(vcpu, arg); 961 break; 962 case KVM_S390_SET_INITIAL_PSW: { 963 psw_t psw; 964 965 r = -EFAULT; 966 if (copy_from_user(&psw, argp, sizeof(psw))) 967 break; 968 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 969 break; 970 } 971 case KVM_S390_INITIAL_RESET: 972 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 973 break; 974 case KVM_SET_ONE_REG: 975 case KVM_GET_ONE_REG: { 976 struct kvm_one_reg reg; 977 r = -EFAULT; 978 if (copy_from_user(®, argp, sizeof(reg))) 979 break; 980 if (ioctl == KVM_SET_ONE_REG) 981 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 982 else 983 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 984 break; 985 } 986 #ifdef CONFIG_KVM_S390_UCONTROL 987 case KVM_S390_UCAS_MAP: { 988 struct kvm_s390_ucas_mapping ucasmap; 989 990 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 991 r = -EFAULT; 992 break; 993 } 994 995 if (!kvm_is_ucontrol(vcpu->kvm)) { 996 r = -EINVAL; 997 break; 998 } 999 1000 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 1001 ucasmap.vcpu_addr, ucasmap.length); 1002 break; 1003 } 1004 case KVM_S390_UCAS_UNMAP: { 1005 struct kvm_s390_ucas_mapping ucasmap; 1006 1007 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 1008 r = -EFAULT; 1009 break; 1010 } 1011 1012 if (!kvm_is_ucontrol(vcpu->kvm)) { 1013 r = -EINVAL; 1014 break; 1015 } 1016 1017 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 1018 ucasmap.length); 1019 break; 1020 } 1021 #endif 1022 case KVM_S390_VCPU_FAULT: { 1023 r = gmap_fault(arg, vcpu->arch.gmap); 1024 if (!IS_ERR_VALUE(r)) 1025 r = 0; 1026 break; 1027 } 1028 case KVM_ENABLE_CAP: 1029 { 1030 struct kvm_enable_cap cap; 1031 r = -EFAULT; 1032 if (copy_from_user(&cap, argp, sizeof(cap))) 1033 break; 1034 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1035 break; 1036 } 1037 default: 1038 r = -ENOTTY; 1039 } 1040 return r; 1041 } 1042 1043 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 1044 { 1045 #ifdef CONFIG_KVM_S390_UCONTROL 1046 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 1047 && (kvm_is_ucontrol(vcpu->kvm))) { 1048 vmf->page = virt_to_page(vcpu->arch.sie_block); 1049 get_page(vmf->page); 1050 return 0; 1051 } 1052 #endif 1053 return VM_FAULT_SIGBUS; 1054 } 1055 1056 void kvm_arch_free_memslot(struct kvm_memory_slot *free, 1057 struct kvm_memory_slot *dont) 1058 { 1059 } 1060 1061 int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages) 1062 { 1063 return 0; 1064 } 1065 1066 /* Section: memory related */ 1067 int kvm_arch_prepare_memory_region(struct kvm *kvm, 1068 struct kvm_memory_slot *memslot, 1069 struct kvm_userspace_memory_region *mem, 1070 enum kvm_mr_change change) 1071 { 1072 /* A few sanity checks. We can have memory slots which have to be 1073 located/ended at a segment boundary (1MB). The memory in userland is 1074 ok to be fragmented into various different vmas. It is okay to mmap() 1075 and munmap() stuff in this slot after doing this call at any time */ 1076 1077 if (mem->userspace_addr & 0xffffful) 1078 return -EINVAL; 1079 1080 if (mem->memory_size & 0xffffful) 1081 return -EINVAL; 1082 1083 return 0; 1084 } 1085 1086 void kvm_arch_commit_memory_region(struct kvm *kvm, 1087 struct kvm_userspace_memory_region *mem, 1088 const struct kvm_memory_slot *old, 1089 enum kvm_mr_change change) 1090 { 1091 int rc; 1092 1093 /* If the basics of the memslot do not change, we do not want 1094 * to update the gmap. Every update causes several unnecessary 1095 * segment translation exceptions. This is usually handled just 1096 * fine by the normal fault handler + gmap, but it will also 1097 * cause faults on the prefix page of running guest CPUs. 1098 */ 1099 if (old->userspace_addr == mem->userspace_addr && 1100 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 1101 old->npages * PAGE_SIZE == mem->memory_size) 1102 return; 1103 1104 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 1105 mem->guest_phys_addr, mem->memory_size); 1106 if (rc) 1107 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n"); 1108 return; 1109 } 1110 1111 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1112 { 1113 } 1114 1115 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1116 struct kvm_memory_slot *slot) 1117 { 1118 } 1119 1120 static int __init kvm_s390_init(void) 1121 { 1122 int ret; 1123 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 1124 if (ret) 1125 return ret; 1126 1127 /* 1128 * guests can ask for up to 255+1 double words, we need a full page 1129 * to hold the maximum amount of facilities. On the other hand, we 1130 * only set facilities that are known to work in KVM. 1131 */ 1132 facilities = (unsigned long long *) get_zeroed_page(GFP_KERNEL|GFP_DMA); 1133 if (!facilities) { 1134 kvm_exit(); 1135 return -ENOMEM; 1136 } 1137 memcpy(facilities, S390_lowcore.stfle_fac_list, 16); 1138 facilities[0] &= 0xff82fff3f47c0000ULL; 1139 facilities[1] &= 0x001c000000000000ULL; 1140 return 0; 1141 } 1142 1143 static void __exit kvm_s390_exit(void) 1144 { 1145 free_page((unsigned long) facilities); 1146 kvm_exit(); 1147 } 1148 1149 module_init(kvm_s390_init); 1150 module_exit(kvm_s390_exit); 1151 1152 /* 1153 * Enable autoloading of the kvm module. 1154 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 1155 * since x86 takes a different approach. 1156 */ 1157 #include <linux/miscdevice.h> 1158 MODULE_ALIAS_MISCDEV(KVM_MINOR); 1159 MODULE_ALIAS("devname:kvm"); 1160