1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14 * 15 * Copyright IBM Corp. 2007 16 * 17 * Authors: Hollis Blanchard <hollisb@us.ibm.com> 18 * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com> 19 */ 20 21 #include <linux/errno.h> 22 #include <linux/err.h> 23 #include <linux/kvm_host.h> 24 #include <linux/vmalloc.h> 25 #include <linux/hrtimer.h> 26 #include <linux/sched/signal.h> 27 #include <linux/fs.h> 28 #include <linux/slab.h> 29 #include <linux/file.h> 30 #include <linux/module.h> 31 #include <linux/irqbypass.h> 32 #include <linux/kvm_irqfd.h> 33 #include <asm/cputable.h> 34 #include <linux/uaccess.h> 35 #include <asm/kvm_ppc.h> 36 #include <asm/tlbflush.h> 37 #include <asm/cputhreads.h> 38 #include <asm/irqflags.h> 39 #include <asm/iommu.h> 40 #include <asm/switch_to.h> 41 #include <asm/xive.h> 42 43 #include "timing.h" 44 #include "irq.h" 45 #include "../mm/mmu_decl.h" 46 47 #define CREATE_TRACE_POINTS 48 #include "trace.h" 49 50 struct kvmppc_ops *kvmppc_hv_ops; 51 EXPORT_SYMBOL_GPL(kvmppc_hv_ops); 52 struct kvmppc_ops *kvmppc_pr_ops; 53 EXPORT_SYMBOL_GPL(kvmppc_pr_ops); 54 55 56 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 57 { 58 return !!(v->arch.pending_exceptions) || 59 v->requests; 60 } 61 62 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 63 { 64 return 1; 65 } 66 67 /* 68 * Common checks before entering the guest world. Call with interrupts 69 * disabled. 70 * 71 * returns: 72 * 73 * == 1 if we're ready to go into guest state 74 * <= 0 if we need to go back to the host with return value 75 */ 76 int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu) 77 { 78 int r; 79 80 WARN_ON(irqs_disabled()); 81 hard_irq_disable(); 82 83 while (true) { 84 if (need_resched()) { 85 local_irq_enable(); 86 cond_resched(); 87 hard_irq_disable(); 88 continue; 89 } 90 91 if (signal_pending(current)) { 92 kvmppc_account_exit(vcpu, SIGNAL_EXITS); 93 vcpu->run->exit_reason = KVM_EXIT_INTR; 94 r = -EINTR; 95 break; 96 } 97 98 vcpu->mode = IN_GUEST_MODE; 99 100 /* 101 * Reading vcpu->requests must happen after setting vcpu->mode, 102 * so we don't miss a request because the requester sees 103 * OUTSIDE_GUEST_MODE and assumes we'll be checking requests 104 * before next entering the guest (and thus doesn't IPI). 105 * This also orders the write to mode from any reads 106 * to the page tables done while the VCPU is running. 107 * Please see the comment in kvm_flush_remote_tlbs. 108 */ 109 smp_mb(); 110 111 if (vcpu->requests) { 112 /* Make sure we process requests preemptable */ 113 local_irq_enable(); 114 trace_kvm_check_requests(vcpu); 115 r = kvmppc_core_check_requests(vcpu); 116 hard_irq_disable(); 117 if (r > 0) 118 continue; 119 break; 120 } 121 122 if (kvmppc_core_prepare_to_enter(vcpu)) { 123 /* interrupts got enabled in between, so we 124 are back at square 1 */ 125 continue; 126 } 127 128 guest_enter_irqoff(); 129 return 1; 130 } 131 132 /* return to host */ 133 local_irq_enable(); 134 return r; 135 } 136 EXPORT_SYMBOL_GPL(kvmppc_prepare_to_enter); 137 138 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) 139 static void kvmppc_swab_shared(struct kvm_vcpu *vcpu) 140 { 141 struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared; 142 int i; 143 144 shared->sprg0 = swab64(shared->sprg0); 145 shared->sprg1 = swab64(shared->sprg1); 146 shared->sprg2 = swab64(shared->sprg2); 147 shared->sprg3 = swab64(shared->sprg3); 148 shared->srr0 = swab64(shared->srr0); 149 shared->srr1 = swab64(shared->srr1); 150 shared->dar = swab64(shared->dar); 151 shared->msr = swab64(shared->msr); 152 shared->dsisr = swab32(shared->dsisr); 153 shared->int_pending = swab32(shared->int_pending); 154 for (i = 0; i < ARRAY_SIZE(shared->sr); i++) 155 shared->sr[i] = swab32(shared->sr[i]); 156 } 157 #endif 158 159 int kvmppc_kvm_pv(struct kvm_vcpu *vcpu) 160 { 161 int nr = kvmppc_get_gpr(vcpu, 11); 162 int r; 163 unsigned long __maybe_unused param1 = kvmppc_get_gpr(vcpu, 3); 164 unsigned long __maybe_unused param2 = kvmppc_get_gpr(vcpu, 4); 165 unsigned long __maybe_unused param3 = kvmppc_get_gpr(vcpu, 5); 166 unsigned long __maybe_unused param4 = kvmppc_get_gpr(vcpu, 6); 167 unsigned long r2 = 0; 168 169 if (!(kvmppc_get_msr(vcpu) & MSR_SF)) { 170 /* 32 bit mode */ 171 param1 &= 0xffffffff; 172 param2 &= 0xffffffff; 173 param3 &= 0xffffffff; 174 param4 &= 0xffffffff; 175 } 176 177 switch (nr) { 178 case KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE): 179 { 180 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) 181 /* Book3S can be little endian, find it out here */ 182 int shared_big_endian = true; 183 if (vcpu->arch.intr_msr & MSR_LE) 184 shared_big_endian = false; 185 if (shared_big_endian != vcpu->arch.shared_big_endian) 186 kvmppc_swab_shared(vcpu); 187 vcpu->arch.shared_big_endian = shared_big_endian; 188 #endif 189 190 if (!(param2 & MAGIC_PAGE_FLAG_NOT_MAPPED_NX)) { 191 /* 192 * Older versions of the Linux magic page code had 193 * a bug where they would map their trampoline code 194 * NX. If that's the case, remove !PR NX capability. 195 */ 196 vcpu->arch.disable_kernel_nx = true; 197 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 198 } 199 200 vcpu->arch.magic_page_pa = param1 & ~0xfffULL; 201 vcpu->arch.magic_page_ea = param2 & ~0xfffULL; 202 203 #ifdef CONFIG_PPC_64K_PAGES 204 /* 205 * Make sure our 4k magic page is in the same window of a 64k 206 * page within the guest and within the host's page. 207 */ 208 if ((vcpu->arch.magic_page_pa & 0xf000) != 209 ((ulong)vcpu->arch.shared & 0xf000)) { 210 void *old_shared = vcpu->arch.shared; 211 ulong shared = (ulong)vcpu->arch.shared; 212 void *new_shared; 213 214 shared &= PAGE_MASK; 215 shared |= vcpu->arch.magic_page_pa & 0xf000; 216 new_shared = (void*)shared; 217 memcpy(new_shared, old_shared, 0x1000); 218 vcpu->arch.shared = new_shared; 219 } 220 #endif 221 222 r2 = KVM_MAGIC_FEAT_SR | KVM_MAGIC_FEAT_MAS0_TO_SPRG7; 223 224 r = EV_SUCCESS; 225 break; 226 } 227 case KVM_HCALL_TOKEN(KVM_HC_FEATURES): 228 r = EV_SUCCESS; 229 #if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500V2) 230 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE); 231 #endif 232 233 /* Second return value is in r4 */ 234 break; 235 case EV_HCALL_TOKEN(EV_IDLE): 236 r = EV_SUCCESS; 237 kvm_vcpu_block(vcpu); 238 kvm_clear_request(KVM_REQ_UNHALT, vcpu); 239 break; 240 default: 241 r = EV_UNIMPLEMENTED; 242 break; 243 } 244 245 kvmppc_set_gpr(vcpu, 4, r2); 246 247 return r; 248 } 249 EXPORT_SYMBOL_GPL(kvmppc_kvm_pv); 250 251 int kvmppc_sanity_check(struct kvm_vcpu *vcpu) 252 { 253 int r = false; 254 255 /* We have to know what CPU to virtualize */ 256 if (!vcpu->arch.pvr) 257 goto out; 258 259 /* PAPR only works with book3s_64 */ 260 if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled) 261 goto out; 262 263 /* HV KVM can only do PAPR mode for now */ 264 if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm)) 265 goto out; 266 267 #ifdef CONFIG_KVM_BOOKE_HV 268 if (!cpu_has_feature(CPU_FTR_EMB_HV)) 269 goto out; 270 #endif 271 272 r = true; 273 274 out: 275 vcpu->arch.sane = r; 276 return r ? 0 : -EINVAL; 277 } 278 EXPORT_SYMBOL_GPL(kvmppc_sanity_check); 279 280 int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu) 281 { 282 enum emulation_result er; 283 int r; 284 285 er = kvmppc_emulate_loadstore(vcpu); 286 switch (er) { 287 case EMULATE_DONE: 288 /* Future optimization: only reload non-volatiles if they were 289 * actually modified. */ 290 r = RESUME_GUEST_NV; 291 break; 292 case EMULATE_AGAIN: 293 r = RESUME_GUEST; 294 break; 295 case EMULATE_DO_MMIO: 296 run->exit_reason = KVM_EXIT_MMIO; 297 /* We must reload nonvolatiles because "update" load/store 298 * instructions modify register state. */ 299 /* Future optimization: only reload non-volatiles if they were 300 * actually modified. */ 301 r = RESUME_HOST_NV; 302 break; 303 case EMULATE_FAIL: 304 { 305 u32 last_inst; 306 307 kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst); 308 /* XXX Deliver Program interrupt to guest. */ 309 pr_emerg("%s: emulation failed (%08x)\n", __func__, last_inst); 310 r = RESUME_HOST; 311 break; 312 } 313 default: 314 WARN_ON(1); 315 r = RESUME_GUEST; 316 } 317 318 return r; 319 } 320 EXPORT_SYMBOL_GPL(kvmppc_emulate_mmio); 321 322 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, 323 bool data) 324 { 325 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK; 326 struct kvmppc_pte pte; 327 int r; 328 329 vcpu->stat.st++; 330 331 r = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST, 332 XLATE_WRITE, &pte); 333 if (r < 0) 334 return r; 335 336 *eaddr = pte.raddr; 337 338 if (!pte.may_write) 339 return -EPERM; 340 341 /* Magic page override */ 342 if (kvmppc_supports_magic_page(vcpu) && mp_pa && 343 ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) && 344 !(kvmppc_get_msr(vcpu) & MSR_PR)) { 345 void *magic = vcpu->arch.shared; 346 magic += pte.eaddr & 0xfff; 347 memcpy(magic, ptr, size); 348 return EMULATE_DONE; 349 } 350 351 if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size)) 352 return EMULATE_DO_MMIO; 353 354 return EMULATE_DONE; 355 } 356 EXPORT_SYMBOL_GPL(kvmppc_st); 357 358 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, 359 bool data) 360 { 361 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK; 362 struct kvmppc_pte pte; 363 int rc; 364 365 vcpu->stat.ld++; 366 367 rc = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST, 368 XLATE_READ, &pte); 369 if (rc) 370 return rc; 371 372 *eaddr = pte.raddr; 373 374 if (!pte.may_read) 375 return -EPERM; 376 377 if (!data && !pte.may_execute) 378 return -ENOEXEC; 379 380 /* Magic page override */ 381 if (kvmppc_supports_magic_page(vcpu) && mp_pa && 382 ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) && 383 !(kvmppc_get_msr(vcpu) & MSR_PR)) { 384 void *magic = vcpu->arch.shared; 385 magic += pte.eaddr & 0xfff; 386 memcpy(ptr, magic, size); 387 return EMULATE_DONE; 388 } 389 390 if (kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size)) 391 return EMULATE_DO_MMIO; 392 393 return EMULATE_DONE; 394 } 395 EXPORT_SYMBOL_GPL(kvmppc_ld); 396 397 int kvm_arch_hardware_enable(void) 398 { 399 return 0; 400 } 401 402 int kvm_arch_hardware_setup(void) 403 { 404 return 0; 405 } 406 407 void kvm_arch_check_processor_compat(void *rtn) 408 { 409 *(int *)rtn = kvmppc_core_check_processor_compat(); 410 } 411 412 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 413 { 414 struct kvmppc_ops *kvm_ops = NULL; 415 /* 416 * if we have both HV and PR enabled, default is HV 417 */ 418 if (type == 0) { 419 if (kvmppc_hv_ops) 420 kvm_ops = kvmppc_hv_ops; 421 else 422 kvm_ops = kvmppc_pr_ops; 423 if (!kvm_ops) 424 goto err_out; 425 } else if (type == KVM_VM_PPC_HV) { 426 if (!kvmppc_hv_ops) 427 goto err_out; 428 kvm_ops = kvmppc_hv_ops; 429 } else if (type == KVM_VM_PPC_PR) { 430 if (!kvmppc_pr_ops) 431 goto err_out; 432 kvm_ops = kvmppc_pr_ops; 433 } else 434 goto err_out; 435 436 if (kvm_ops->owner && !try_module_get(kvm_ops->owner)) 437 return -ENOENT; 438 439 kvm->arch.kvm_ops = kvm_ops; 440 return kvmppc_core_init_vm(kvm); 441 err_out: 442 return -EINVAL; 443 } 444 445 bool kvm_arch_has_vcpu_debugfs(void) 446 { 447 return false; 448 } 449 450 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 451 { 452 return 0; 453 } 454 455 void kvm_arch_destroy_vm(struct kvm *kvm) 456 { 457 unsigned int i; 458 struct kvm_vcpu *vcpu; 459 460 #ifdef CONFIG_KVM_XICS 461 /* 462 * We call kick_all_cpus_sync() to ensure that all 463 * CPUs have executed any pending IPIs before we 464 * continue and free VCPUs structures below. 465 */ 466 if (is_kvmppc_hv_enabled(kvm)) 467 kick_all_cpus_sync(); 468 #endif 469 470 kvm_for_each_vcpu(i, vcpu, kvm) 471 kvm_arch_vcpu_free(vcpu); 472 473 mutex_lock(&kvm->lock); 474 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 475 kvm->vcpus[i] = NULL; 476 477 atomic_set(&kvm->online_vcpus, 0); 478 479 kvmppc_core_destroy_vm(kvm); 480 481 mutex_unlock(&kvm->lock); 482 483 /* drop the module reference */ 484 module_put(kvm->arch.kvm_ops->owner); 485 } 486 487 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 488 { 489 int r; 490 /* Assume we're using HV mode when the HV module is loaded */ 491 int hv_enabled = kvmppc_hv_ops ? 1 : 0; 492 493 if (kvm) { 494 /* 495 * Hooray - we know which VM type we're running on. Depend on 496 * that rather than the guess above. 497 */ 498 hv_enabled = is_kvmppc_hv_enabled(kvm); 499 } 500 501 switch (ext) { 502 #ifdef CONFIG_BOOKE 503 case KVM_CAP_PPC_BOOKE_SREGS: 504 case KVM_CAP_PPC_BOOKE_WATCHDOG: 505 case KVM_CAP_PPC_EPR: 506 #else 507 case KVM_CAP_PPC_SEGSTATE: 508 case KVM_CAP_PPC_HIOR: 509 case KVM_CAP_PPC_PAPR: 510 #endif 511 case KVM_CAP_PPC_UNSET_IRQ: 512 case KVM_CAP_PPC_IRQ_LEVEL: 513 case KVM_CAP_ENABLE_CAP: 514 case KVM_CAP_ENABLE_CAP_VM: 515 case KVM_CAP_ONE_REG: 516 case KVM_CAP_IOEVENTFD: 517 case KVM_CAP_DEVICE_CTRL: 518 case KVM_CAP_IMMEDIATE_EXIT: 519 r = 1; 520 break; 521 case KVM_CAP_PPC_PAIRED_SINGLES: 522 case KVM_CAP_PPC_OSI: 523 case KVM_CAP_PPC_GET_PVINFO: 524 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC) 525 case KVM_CAP_SW_TLB: 526 #endif 527 /* We support this only for PR */ 528 r = !hv_enabled; 529 break; 530 #ifdef CONFIG_KVM_MPIC 531 case KVM_CAP_IRQ_MPIC: 532 r = 1; 533 break; 534 #endif 535 536 #ifdef CONFIG_PPC_BOOK3S_64 537 case KVM_CAP_SPAPR_TCE: 538 case KVM_CAP_SPAPR_TCE_64: 539 /* fallthrough */ 540 case KVM_CAP_SPAPR_TCE_VFIO: 541 case KVM_CAP_PPC_RTAS: 542 case KVM_CAP_PPC_FIXUP_HCALL: 543 case KVM_CAP_PPC_ENABLE_HCALL: 544 #ifdef CONFIG_KVM_XICS 545 case KVM_CAP_IRQ_XICS: 546 #endif 547 r = 1; 548 break; 549 550 case KVM_CAP_PPC_ALLOC_HTAB: 551 r = hv_enabled; 552 break; 553 #endif /* CONFIG_PPC_BOOK3S_64 */ 554 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 555 case KVM_CAP_PPC_SMT: 556 r = 0; 557 if (hv_enabled) { 558 if (cpu_has_feature(CPU_FTR_ARCH_300)) 559 r = 1; 560 else 561 r = threads_per_subcore; 562 } 563 break; 564 case KVM_CAP_PPC_RMA: 565 r = 0; 566 break; 567 case KVM_CAP_PPC_HWRNG: 568 r = kvmppc_hwrng_present(); 569 break; 570 case KVM_CAP_PPC_MMU_RADIX: 571 r = !!(hv_enabled && radix_enabled()); 572 break; 573 case KVM_CAP_PPC_MMU_HASH_V3: 574 r = !!(hv_enabled && !radix_enabled() && 575 cpu_has_feature(CPU_FTR_ARCH_300)); 576 break; 577 #endif 578 case KVM_CAP_SYNC_MMU: 579 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 580 r = hv_enabled; 581 #elif defined(KVM_ARCH_WANT_MMU_NOTIFIER) 582 r = 1; 583 #else 584 r = 0; 585 #endif 586 break; 587 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 588 case KVM_CAP_PPC_HTAB_FD: 589 r = hv_enabled; 590 break; 591 #endif 592 case KVM_CAP_NR_VCPUS: 593 /* 594 * Recommending a number of CPUs is somewhat arbitrary; we 595 * return the number of present CPUs for -HV (since a host 596 * will have secondary threads "offline"), and for other KVM 597 * implementations just count online CPUs. 598 */ 599 if (hv_enabled) 600 r = num_present_cpus(); 601 else 602 r = num_online_cpus(); 603 break; 604 case KVM_CAP_NR_MEMSLOTS: 605 r = KVM_USER_MEM_SLOTS; 606 break; 607 case KVM_CAP_MAX_VCPUS: 608 r = KVM_MAX_VCPUS; 609 break; 610 #ifdef CONFIG_PPC_BOOK3S_64 611 case KVM_CAP_PPC_GET_SMMU_INFO: 612 r = 1; 613 break; 614 case KVM_CAP_SPAPR_MULTITCE: 615 r = 1; 616 break; 617 case KVM_CAP_SPAPR_RESIZE_HPT: 618 /* Disable this on POWER9 until code handles new HPTE format */ 619 r = !!hv_enabled && !cpu_has_feature(CPU_FTR_ARCH_300); 620 break; 621 #endif 622 case KVM_CAP_PPC_HTM: 623 r = cpu_has_feature(CPU_FTR_TM_COMP) && 624 is_kvmppc_hv_enabled(kvm); 625 break; 626 default: 627 r = 0; 628 break; 629 } 630 return r; 631 632 } 633 634 long kvm_arch_dev_ioctl(struct file *filp, 635 unsigned int ioctl, unsigned long arg) 636 { 637 return -EINVAL; 638 } 639 640 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 641 struct kvm_memory_slot *dont) 642 { 643 kvmppc_core_free_memslot(kvm, free, dont); 644 } 645 646 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 647 unsigned long npages) 648 { 649 return kvmppc_core_create_memslot(kvm, slot, npages); 650 } 651 652 int kvm_arch_prepare_memory_region(struct kvm *kvm, 653 struct kvm_memory_slot *memslot, 654 const struct kvm_userspace_memory_region *mem, 655 enum kvm_mr_change change) 656 { 657 return kvmppc_core_prepare_memory_region(kvm, memslot, mem); 658 } 659 660 void kvm_arch_commit_memory_region(struct kvm *kvm, 661 const struct kvm_userspace_memory_region *mem, 662 const struct kvm_memory_slot *old, 663 const struct kvm_memory_slot *new, 664 enum kvm_mr_change change) 665 { 666 kvmppc_core_commit_memory_region(kvm, mem, old, new); 667 } 668 669 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 670 struct kvm_memory_slot *slot) 671 { 672 kvmppc_core_flush_memslot(kvm, slot); 673 } 674 675 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) 676 { 677 struct kvm_vcpu *vcpu; 678 vcpu = kvmppc_core_vcpu_create(kvm, id); 679 if (!IS_ERR(vcpu)) { 680 vcpu->arch.wqp = &vcpu->wq; 681 kvmppc_create_vcpu_debugfs(vcpu, id); 682 } 683 return vcpu; 684 } 685 686 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 687 { 688 } 689 690 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) 691 { 692 /* Make sure we're not using the vcpu anymore */ 693 hrtimer_cancel(&vcpu->arch.dec_timer); 694 695 kvmppc_remove_vcpu_debugfs(vcpu); 696 697 switch (vcpu->arch.irq_type) { 698 case KVMPPC_IRQ_MPIC: 699 kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu); 700 break; 701 case KVMPPC_IRQ_XICS: 702 if (xive_enabled()) 703 kvmppc_xive_cleanup_vcpu(vcpu); 704 else 705 kvmppc_xics_free_icp(vcpu); 706 break; 707 } 708 709 kvmppc_core_vcpu_free(vcpu); 710 } 711 712 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 713 { 714 kvm_arch_vcpu_free(vcpu); 715 } 716 717 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 718 { 719 return kvmppc_core_pending_dec(vcpu); 720 } 721 722 static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer) 723 { 724 struct kvm_vcpu *vcpu; 725 726 vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer); 727 kvmppc_decrementer_func(vcpu); 728 729 return HRTIMER_NORESTART; 730 } 731 732 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 733 { 734 int ret; 735 736 hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 737 vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup; 738 vcpu->arch.dec_expires = ~(u64)0; 739 740 #ifdef CONFIG_KVM_EXIT_TIMING 741 mutex_init(&vcpu->arch.exit_timing_lock); 742 #endif 743 ret = kvmppc_subarch_vcpu_init(vcpu); 744 return ret; 745 } 746 747 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 748 { 749 kvmppc_mmu_destroy(vcpu); 750 kvmppc_subarch_vcpu_uninit(vcpu); 751 } 752 753 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 754 { 755 #ifdef CONFIG_BOOKE 756 /* 757 * vrsave (formerly usprg0) isn't used by Linux, but may 758 * be used by the guest. 759 * 760 * On non-booke this is associated with Altivec and 761 * is handled by code in book3s.c. 762 */ 763 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); 764 #endif 765 kvmppc_core_vcpu_load(vcpu, cpu); 766 } 767 768 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 769 { 770 kvmppc_core_vcpu_put(vcpu); 771 #ifdef CONFIG_BOOKE 772 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); 773 #endif 774 } 775 776 /* 777 * irq_bypass_add_producer and irq_bypass_del_producer are only 778 * useful if the architecture supports PCI passthrough. 779 * irq_bypass_stop and irq_bypass_start are not needed and so 780 * kvm_ops are not defined for them. 781 */ 782 bool kvm_arch_has_irq_bypass(void) 783 { 784 return ((kvmppc_hv_ops && kvmppc_hv_ops->irq_bypass_add_producer) || 785 (kvmppc_pr_ops && kvmppc_pr_ops->irq_bypass_add_producer)); 786 } 787 788 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 789 struct irq_bypass_producer *prod) 790 { 791 struct kvm_kernel_irqfd *irqfd = 792 container_of(cons, struct kvm_kernel_irqfd, consumer); 793 struct kvm *kvm = irqfd->kvm; 794 795 if (kvm->arch.kvm_ops->irq_bypass_add_producer) 796 return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod); 797 798 return 0; 799 } 800 801 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 802 struct irq_bypass_producer *prod) 803 { 804 struct kvm_kernel_irqfd *irqfd = 805 container_of(cons, struct kvm_kernel_irqfd, consumer); 806 struct kvm *kvm = irqfd->kvm; 807 808 if (kvm->arch.kvm_ops->irq_bypass_del_producer) 809 kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod); 810 } 811 812 #ifdef CONFIG_VSX 813 static inline int kvmppc_get_vsr_dword_offset(int index) 814 { 815 int offset; 816 817 if ((index != 0) && (index != 1)) 818 return -1; 819 820 #ifdef __BIG_ENDIAN 821 offset = index; 822 #else 823 offset = 1 - index; 824 #endif 825 826 return offset; 827 } 828 829 static inline int kvmppc_get_vsr_word_offset(int index) 830 { 831 int offset; 832 833 if ((index > 3) || (index < 0)) 834 return -1; 835 836 #ifdef __BIG_ENDIAN 837 offset = index; 838 #else 839 offset = 3 - index; 840 #endif 841 return offset; 842 } 843 844 static inline void kvmppc_set_vsr_dword(struct kvm_vcpu *vcpu, 845 u64 gpr) 846 { 847 union kvmppc_one_reg val; 848 int offset = kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset); 849 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 850 851 if (offset == -1) 852 return; 853 854 if (vcpu->arch.mmio_vsx_tx_sx_enabled) { 855 val.vval = VCPU_VSX_VR(vcpu, index); 856 val.vsxval[offset] = gpr; 857 VCPU_VSX_VR(vcpu, index) = val.vval; 858 } else { 859 VCPU_VSX_FPR(vcpu, index, offset) = gpr; 860 } 861 } 862 863 static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu, 864 u64 gpr) 865 { 866 union kvmppc_one_reg val; 867 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 868 869 if (vcpu->arch.mmio_vsx_tx_sx_enabled) { 870 val.vval = VCPU_VSX_VR(vcpu, index); 871 val.vsxval[0] = gpr; 872 val.vsxval[1] = gpr; 873 VCPU_VSX_VR(vcpu, index) = val.vval; 874 } else { 875 VCPU_VSX_FPR(vcpu, index, 0) = gpr; 876 VCPU_VSX_FPR(vcpu, index, 1) = gpr; 877 } 878 } 879 880 static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu, 881 u32 gpr32) 882 { 883 union kvmppc_one_reg val; 884 int offset = kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset); 885 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 886 int dword_offset, word_offset; 887 888 if (offset == -1) 889 return; 890 891 if (vcpu->arch.mmio_vsx_tx_sx_enabled) { 892 val.vval = VCPU_VSX_VR(vcpu, index); 893 val.vsx32val[offset] = gpr32; 894 VCPU_VSX_VR(vcpu, index) = val.vval; 895 } else { 896 dword_offset = offset / 2; 897 word_offset = offset % 2; 898 val.vsxval[0] = VCPU_VSX_FPR(vcpu, index, dword_offset); 899 val.vsx32val[word_offset] = gpr32; 900 VCPU_VSX_FPR(vcpu, index, dword_offset) = val.vsxval[0]; 901 } 902 } 903 #endif /* CONFIG_VSX */ 904 905 #ifdef CONFIG_PPC_FPU 906 static inline u64 sp_to_dp(u32 fprs) 907 { 908 u64 fprd; 909 910 preempt_disable(); 911 enable_kernel_fp(); 912 asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m" (fprd) : "m" (fprs) 913 : "fr0"); 914 preempt_enable(); 915 return fprd; 916 } 917 918 static inline u32 dp_to_sp(u64 fprd) 919 { 920 u32 fprs; 921 922 preempt_disable(); 923 enable_kernel_fp(); 924 asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m" (fprs) : "m" (fprd) 925 : "fr0"); 926 preempt_enable(); 927 return fprs; 928 } 929 930 #else 931 #define sp_to_dp(x) (x) 932 #define dp_to_sp(x) (x) 933 #endif /* CONFIG_PPC_FPU */ 934 935 static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu, 936 struct kvm_run *run) 937 { 938 u64 uninitialized_var(gpr); 939 940 if (run->mmio.len > sizeof(gpr)) { 941 printk(KERN_ERR "bad MMIO length: %d\n", run->mmio.len); 942 return; 943 } 944 945 if (!vcpu->arch.mmio_host_swabbed) { 946 switch (run->mmio.len) { 947 case 8: gpr = *(u64 *)run->mmio.data; break; 948 case 4: gpr = *(u32 *)run->mmio.data; break; 949 case 2: gpr = *(u16 *)run->mmio.data; break; 950 case 1: gpr = *(u8 *)run->mmio.data; break; 951 } 952 } else { 953 switch (run->mmio.len) { 954 case 8: gpr = swab64(*(u64 *)run->mmio.data); break; 955 case 4: gpr = swab32(*(u32 *)run->mmio.data); break; 956 case 2: gpr = swab16(*(u16 *)run->mmio.data); break; 957 case 1: gpr = *(u8 *)run->mmio.data; break; 958 } 959 } 960 961 /* conversion between single and double precision */ 962 if ((vcpu->arch.mmio_sp64_extend) && (run->mmio.len == 4)) 963 gpr = sp_to_dp(gpr); 964 965 if (vcpu->arch.mmio_sign_extend) { 966 switch (run->mmio.len) { 967 #ifdef CONFIG_PPC64 968 case 4: 969 gpr = (s64)(s32)gpr; 970 break; 971 #endif 972 case 2: 973 gpr = (s64)(s16)gpr; 974 break; 975 case 1: 976 gpr = (s64)(s8)gpr; 977 break; 978 } 979 } 980 981 switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) { 982 case KVM_MMIO_REG_GPR: 983 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr); 984 break; 985 case KVM_MMIO_REG_FPR: 986 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr; 987 break; 988 #ifdef CONFIG_PPC_BOOK3S 989 case KVM_MMIO_REG_QPR: 990 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; 991 break; 992 case KVM_MMIO_REG_FQPR: 993 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr; 994 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; 995 break; 996 #endif 997 #ifdef CONFIG_VSX 998 case KVM_MMIO_REG_VSX: 999 if (vcpu->arch.mmio_vsx_copy_type == KVMPPC_VSX_COPY_DWORD) 1000 kvmppc_set_vsr_dword(vcpu, gpr); 1001 else if (vcpu->arch.mmio_vsx_copy_type == KVMPPC_VSX_COPY_WORD) 1002 kvmppc_set_vsr_word(vcpu, gpr); 1003 else if (vcpu->arch.mmio_vsx_copy_type == 1004 KVMPPC_VSX_COPY_DWORD_LOAD_DUMP) 1005 kvmppc_set_vsr_dword_dump(vcpu, gpr); 1006 break; 1007 #endif 1008 default: 1009 BUG(); 1010 } 1011 } 1012 1013 static int __kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, 1014 unsigned int rt, unsigned int bytes, 1015 int is_default_endian, int sign_extend) 1016 { 1017 int idx, ret; 1018 bool host_swabbed; 1019 1020 /* Pity C doesn't have a logical XOR operator */ 1021 if (kvmppc_need_byteswap(vcpu)) { 1022 host_swabbed = is_default_endian; 1023 } else { 1024 host_swabbed = !is_default_endian; 1025 } 1026 1027 if (bytes > sizeof(run->mmio.data)) { 1028 printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__, 1029 run->mmio.len); 1030 } 1031 1032 run->mmio.phys_addr = vcpu->arch.paddr_accessed; 1033 run->mmio.len = bytes; 1034 run->mmio.is_write = 0; 1035 1036 vcpu->arch.io_gpr = rt; 1037 vcpu->arch.mmio_host_swabbed = host_swabbed; 1038 vcpu->mmio_needed = 1; 1039 vcpu->mmio_is_write = 0; 1040 vcpu->arch.mmio_sign_extend = sign_extend; 1041 1042 idx = srcu_read_lock(&vcpu->kvm->srcu); 1043 1044 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, 1045 bytes, &run->mmio.data); 1046 1047 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1048 1049 if (!ret) { 1050 kvmppc_complete_mmio_load(vcpu, run); 1051 vcpu->mmio_needed = 0; 1052 return EMULATE_DONE; 1053 } 1054 1055 return EMULATE_DO_MMIO; 1056 } 1057 1058 int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, 1059 unsigned int rt, unsigned int bytes, 1060 int is_default_endian) 1061 { 1062 return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 0); 1063 } 1064 EXPORT_SYMBOL_GPL(kvmppc_handle_load); 1065 1066 /* Same as above, but sign extends */ 1067 int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu, 1068 unsigned int rt, unsigned int bytes, 1069 int is_default_endian) 1070 { 1071 return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 1); 1072 } 1073 1074 #ifdef CONFIG_VSX 1075 int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, 1076 unsigned int rt, unsigned int bytes, 1077 int is_default_endian, int mmio_sign_extend) 1078 { 1079 enum emulation_result emulated = EMULATE_DONE; 1080 1081 /* Currently, mmio_vsx_copy_nums only allowed to be less than 4 */ 1082 if ( (vcpu->arch.mmio_vsx_copy_nums > 4) || 1083 (vcpu->arch.mmio_vsx_copy_nums < 0) ) { 1084 return EMULATE_FAIL; 1085 } 1086 1087 while (vcpu->arch.mmio_vsx_copy_nums) { 1088 emulated = __kvmppc_handle_load(run, vcpu, rt, bytes, 1089 is_default_endian, mmio_sign_extend); 1090 1091 if (emulated != EMULATE_DONE) 1092 break; 1093 1094 vcpu->arch.paddr_accessed += run->mmio.len; 1095 1096 vcpu->arch.mmio_vsx_copy_nums--; 1097 vcpu->arch.mmio_vsx_offset++; 1098 } 1099 return emulated; 1100 } 1101 #endif /* CONFIG_VSX */ 1102 1103 int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, 1104 u64 val, unsigned int bytes, int is_default_endian) 1105 { 1106 void *data = run->mmio.data; 1107 int idx, ret; 1108 bool host_swabbed; 1109 1110 /* Pity C doesn't have a logical XOR operator */ 1111 if (kvmppc_need_byteswap(vcpu)) { 1112 host_swabbed = is_default_endian; 1113 } else { 1114 host_swabbed = !is_default_endian; 1115 } 1116 1117 if (bytes > sizeof(run->mmio.data)) { 1118 printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__, 1119 run->mmio.len); 1120 } 1121 1122 run->mmio.phys_addr = vcpu->arch.paddr_accessed; 1123 run->mmio.len = bytes; 1124 run->mmio.is_write = 1; 1125 vcpu->mmio_needed = 1; 1126 vcpu->mmio_is_write = 1; 1127 1128 if ((vcpu->arch.mmio_sp64_extend) && (bytes == 4)) 1129 val = dp_to_sp(val); 1130 1131 /* Store the value at the lowest bytes in 'data'. */ 1132 if (!host_swabbed) { 1133 switch (bytes) { 1134 case 8: *(u64 *)data = val; break; 1135 case 4: *(u32 *)data = val; break; 1136 case 2: *(u16 *)data = val; break; 1137 case 1: *(u8 *)data = val; break; 1138 } 1139 } else { 1140 switch (bytes) { 1141 case 8: *(u64 *)data = swab64(val); break; 1142 case 4: *(u32 *)data = swab32(val); break; 1143 case 2: *(u16 *)data = swab16(val); break; 1144 case 1: *(u8 *)data = val; break; 1145 } 1146 } 1147 1148 idx = srcu_read_lock(&vcpu->kvm->srcu); 1149 1150 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, 1151 bytes, &run->mmio.data); 1152 1153 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1154 1155 if (!ret) { 1156 vcpu->mmio_needed = 0; 1157 return EMULATE_DONE; 1158 } 1159 1160 return EMULATE_DO_MMIO; 1161 } 1162 EXPORT_SYMBOL_GPL(kvmppc_handle_store); 1163 1164 #ifdef CONFIG_VSX 1165 static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val) 1166 { 1167 u32 dword_offset, word_offset; 1168 union kvmppc_one_reg reg; 1169 int vsx_offset = 0; 1170 int copy_type = vcpu->arch.mmio_vsx_copy_type; 1171 int result = 0; 1172 1173 switch (copy_type) { 1174 case KVMPPC_VSX_COPY_DWORD: 1175 vsx_offset = 1176 kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset); 1177 1178 if (vsx_offset == -1) { 1179 result = -1; 1180 break; 1181 } 1182 1183 if (!vcpu->arch.mmio_vsx_tx_sx_enabled) { 1184 *val = VCPU_VSX_FPR(vcpu, rs, vsx_offset); 1185 } else { 1186 reg.vval = VCPU_VSX_VR(vcpu, rs); 1187 *val = reg.vsxval[vsx_offset]; 1188 } 1189 break; 1190 1191 case KVMPPC_VSX_COPY_WORD: 1192 vsx_offset = 1193 kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset); 1194 1195 if (vsx_offset == -1) { 1196 result = -1; 1197 break; 1198 } 1199 1200 if (!vcpu->arch.mmio_vsx_tx_sx_enabled) { 1201 dword_offset = vsx_offset / 2; 1202 word_offset = vsx_offset % 2; 1203 reg.vsxval[0] = VCPU_VSX_FPR(vcpu, rs, dword_offset); 1204 *val = reg.vsx32val[word_offset]; 1205 } else { 1206 reg.vval = VCPU_VSX_VR(vcpu, rs); 1207 *val = reg.vsx32val[vsx_offset]; 1208 } 1209 break; 1210 1211 default: 1212 result = -1; 1213 break; 1214 } 1215 1216 return result; 1217 } 1218 1219 int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, 1220 int rs, unsigned int bytes, int is_default_endian) 1221 { 1222 u64 val; 1223 enum emulation_result emulated = EMULATE_DONE; 1224 1225 vcpu->arch.io_gpr = rs; 1226 1227 /* Currently, mmio_vsx_copy_nums only allowed to be less than 4 */ 1228 if ( (vcpu->arch.mmio_vsx_copy_nums > 4) || 1229 (vcpu->arch.mmio_vsx_copy_nums < 0) ) { 1230 return EMULATE_FAIL; 1231 } 1232 1233 while (vcpu->arch.mmio_vsx_copy_nums) { 1234 if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1) 1235 return EMULATE_FAIL; 1236 1237 emulated = kvmppc_handle_store(run, vcpu, 1238 val, bytes, is_default_endian); 1239 1240 if (emulated != EMULATE_DONE) 1241 break; 1242 1243 vcpu->arch.paddr_accessed += run->mmio.len; 1244 1245 vcpu->arch.mmio_vsx_copy_nums--; 1246 vcpu->arch.mmio_vsx_offset++; 1247 } 1248 1249 return emulated; 1250 } 1251 1252 static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu, 1253 struct kvm_run *run) 1254 { 1255 enum emulation_result emulated = EMULATE_FAIL; 1256 int r; 1257 1258 vcpu->arch.paddr_accessed += run->mmio.len; 1259 1260 if (!vcpu->mmio_is_write) { 1261 emulated = kvmppc_handle_vsx_load(run, vcpu, vcpu->arch.io_gpr, 1262 run->mmio.len, 1, vcpu->arch.mmio_sign_extend); 1263 } else { 1264 emulated = kvmppc_handle_vsx_store(run, vcpu, 1265 vcpu->arch.io_gpr, run->mmio.len, 1); 1266 } 1267 1268 switch (emulated) { 1269 case EMULATE_DO_MMIO: 1270 run->exit_reason = KVM_EXIT_MMIO; 1271 r = RESUME_HOST; 1272 break; 1273 case EMULATE_FAIL: 1274 pr_info("KVM: MMIO emulation failed (VSX repeat)\n"); 1275 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1276 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 1277 r = RESUME_HOST; 1278 break; 1279 default: 1280 r = RESUME_GUEST; 1281 break; 1282 } 1283 return r; 1284 } 1285 #endif /* CONFIG_VSX */ 1286 1287 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) 1288 { 1289 int r = 0; 1290 union kvmppc_one_reg val; 1291 int size; 1292 1293 size = one_reg_size(reg->id); 1294 if (size > sizeof(val)) 1295 return -EINVAL; 1296 1297 r = kvmppc_get_one_reg(vcpu, reg->id, &val); 1298 if (r == -EINVAL) { 1299 r = 0; 1300 switch (reg->id) { 1301 #ifdef CONFIG_ALTIVEC 1302 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31: 1303 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1304 r = -ENXIO; 1305 break; 1306 } 1307 val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0]; 1308 break; 1309 case KVM_REG_PPC_VSCR: 1310 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1311 r = -ENXIO; 1312 break; 1313 } 1314 val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]); 1315 break; 1316 case KVM_REG_PPC_VRSAVE: 1317 val = get_reg_val(reg->id, vcpu->arch.vrsave); 1318 break; 1319 #endif /* CONFIG_ALTIVEC */ 1320 default: 1321 r = -EINVAL; 1322 break; 1323 } 1324 } 1325 1326 if (r) 1327 return r; 1328 1329 if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size)) 1330 r = -EFAULT; 1331 1332 return r; 1333 } 1334 1335 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) 1336 { 1337 int r; 1338 union kvmppc_one_reg val; 1339 int size; 1340 1341 size = one_reg_size(reg->id); 1342 if (size > sizeof(val)) 1343 return -EINVAL; 1344 1345 if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size)) 1346 return -EFAULT; 1347 1348 r = kvmppc_set_one_reg(vcpu, reg->id, &val); 1349 if (r == -EINVAL) { 1350 r = 0; 1351 switch (reg->id) { 1352 #ifdef CONFIG_ALTIVEC 1353 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31: 1354 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1355 r = -ENXIO; 1356 break; 1357 } 1358 vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval; 1359 break; 1360 case KVM_REG_PPC_VSCR: 1361 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1362 r = -ENXIO; 1363 break; 1364 } 1365 vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val); 1366 break; 1367 case KVM_REG_PPC_VRSAVE: 1368 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1369 r = -ENXIO; 1370 break; 1371 } 1372 vcpu->arch.vrsave = set_reg_val(reg->id, val); 1373 break; 1374 #endif /* CONFIG_ALTIVEC */ 1375 default: 1376 r = -EINVAL; 1377 break; 1378 } 1379 } 1380 1381 return r; 1382 } 1383 1384 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) 1385 { 1386 int r; 1387 sigset_t sigsaved; 1388 1389 if (vcpu->mmio_needed) { 1390 vcpu->mmio_needed = 0; 1391 if (!vcpu->mmio_is_write) 1392 kvmppc_complete_mmio_load(vcpu, run); 1393 #ifdef CONFIG_VSX 1394 if (vcpu->arch.mmio_vsx_copy_nums > 0) { 1395 vcpu->arch.mmio_vsx_copy_nums--; 1396 vcpu->arch.mmio_vsx_offset++; 1397 } 1398 1399 if (vcpu->arch.mmio_vsx_copy_nums > 0) { 1400 r = kvmppc_emulate_mmio_vsx_loadstore(vcpu, run); 1401 if (r == RESUME_HOST) { 1402 vcpu->mmio_needed = 1; 1403 return r; 1404 } 1405 } 1406 #endif 1407 } else if (vcpu->arch.osi_needed) { 1408 u64 *gprs = run->osi.gprs; 1409 int i; 1410 1411 for (i = 0; i < 32; i++) 1412 kvmppc_set_gpr(vcpu, i, gprs[i]); 1413 vcpu->arch.osi_needed = 0; 1414 } else if (vcpu->arch.hcall_needed) { 1415 int i; 1416 1417 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret); 1418 for (i = 0; i < 9; ++i) 1419 kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]); 1420 vcpu->arch.hcall_needed = 0; 1421 #ifdef CONFIG_BOOKE 1422 } else if (vcpu->arch.epr_needed) { 1423 kvmppc_set_epr(vcpu, run->epr.epr); 1424 vcpu->arch.epr_needed = 0; 1425 #endif 1426 } 1427 1428 if (vcpu->sigset_active) 1429 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 1430 1431 if (run->immediate_exit) 1432 r = -EINTR; 1433 else 1434 r = kvmppc_vcpu_run(run, vcpu); 1435 1436 if (vcpu->sigset_active) 1437 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 1438 1439 return r; 1440 } 1441 1442 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) 1443 { 1444 if (irq->irq == KVM_INTERRUPT_UNSET) { 1445 kvmppc_core_dequeue_external(vcpu); 1446 return 0; 1447 } 1448 1449 kvmppc_core_queue_external(vcpu, irq); 1450 1451 kvm_vcpu_kick(vcpu); 1452 1453 return 0; 1454 } 1455 1456 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 1457 struct kvm_enable_cap *cap) 1458 { 1459 int r; 1460 1461 if (cap->flags) 1462 return -EINVAL; 1463 1464 switch (cap->cap) { 1465 case KVM_CAP_PPC_OSI: 1466 r = 0; 1467 vcpu->arch.osi_enabled = true; 1468 break; 1469 case KVM_CAP_PPC_PAPR: 1470 r = 0; 1471 vcpu->arch.papr_enabled = true; 1472 break; 1473 case KVM_CAP_PPC_EPR: 1474 r = 0; 1475 if (cap->args[0]) 1476 vcpu->arch.epr_flags |= KVMPPC_EPR_USER; 1477 else 1478 vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER; 1479 break; 1480 #ifdef CONFIG_BOOKE 1481 case KVM_CAP_PPC_BOOKE_WATCHDOG: 1482 r = 0; 1483 vcpu->arch.watchdog_enabled = true; 1484 break; 1485 #endif 1486 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC) 1487 case KVM_CAP_SW_TLB: { 1488 struct kvm_config_tlb cfg; 1489 void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0]; 1490 1491 r = -EFAULT; 1492 if (copy_from_user(&cfg, user_ptr, sizeof(cfg))) 1493 break; 1494 1495 r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg); 1496 break; 1497 } 1498 #endif 1499 #ifdef CONFIG_KVM_MPIC 1500 case KVM_CAP_IRQ_MPIC: { 1501 struct fd f; 1502 struct kvm_device *dev; 1503 1504 r = -EBADF; 1505 f = fdget(cap->args[0]); 1506 if (!f.file) 1507 break; 1508 1509 r = -EPERM; 1510 dev = kvm_device_from_filp(f.file); 1511 if (dev) 1512 r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]); 1513 1514 fdput(f); 1515 break; 1516 } 1517 #endif 1518 #ifdef CONFIG_KVM_XICS 1519 case KVM_CAP_IRQ_XICS: { 1520 struct fd f; 1521 struct kvm_device *dev; 1522 1523 r = -EBADF; 1524 f = fdget(cap->args[0]); 1525 if (!f.file) 1526 break; 1527 1528 r = -EPERM; 1529 dev = kvm_device_from_filp(f.file); 1530 if (dev) { 1531 if (xive_enabled()) 1532 r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]); 1533 else 1534 r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]); 1535 } 1536 1537 fdput(f); 1538 break; 1539 } 1540 #endif /* CONFIG_KVM_XICS */ 1541 default: 1542 r = -EINVAL; 1543 break; 1544 } 1545 1546 if (!r) 1547 r = kvmppc_sanity_check(vcpu); 1548 1549 return r; 1550 } 1551 1552 bool kvm_arch_intc_initialized(struct kvm *kvm) 1553 { 1554 #ifdef CONFIG_KVM_MPIC 1555 if (kvm->arch.mpic) 1556 return true; 1557 #endif 1558 #ifdef CONFIG_KVM_XICS 1559 if (kvm->arch.xics || kvm->arch.xive) 1560 return true; 1561 #endif 1562 return false; 1563 } 1564 1565 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 1566 struct kvm_mp_state *mp_state) 1567 { 1568 return -EINVAL; 1569 } 1570 1571 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 1572 struct kvm_mp_state *mp_state) 1573 { 1574 return -EINVAL; 1575 } 1576 1577 long kvm_arch_vcpu_ioctl(struct file *filp, 1578 unsigned int ioctl, unsigned long arg) 1579 { 1580 struct kvm_vcpu *vcpu = filp->private_data; 1581 void __user *argp = (void __user *)arg; 1582 long r; 1583 1584 switch (ioctl) { 1585 case KVM_INTERRUPT: { 1586 struct kvm_interrupt irq; 1587 r = -EFAULT; 1588 if (copy_from_user(&irq, argp, sizeof(irq))) 1589 goto out; 1590 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 1591 goto out; 1592 } 1593 1594 case KVM_ENABLE_CAP: 1595 { 1596 struct kvm_enable_cap cap; 1597 r = -EFAULT; 1598 if (copy_from_user(&cap, argp, sizeof(cap))) 1599 goto out; 1600 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1601 break; 1602 } 1603 1604 case KVM_SET_ONE_REG: 1605 case KVM_GET_ONE_REG: 1606 { 1607 struct kvm_one_reg reg; 1608 r = -EFAULT; 1609 if (copy_from_user(®, argp, sizeof(reg))) 1610 goto out; 1611 if (ioctl == KVM_SET_ONE_REG) 1612 r = kvm_vcpu_ioctl_set_one_reg(vcpu, ®); 1613 else 1614 r = kvm_vcpu_ioctl_get_one_reg(vcpu, ®); 1615 break; 1616 } 1617 1618 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC) 1619 case KVM_DIRTY_TLB: { 1620 struct kvm_dirty_tlb dirty; 1621 r = -EFAULT; 1622 if (copy_from_user(&dirty, argp, sizeof(dirty))) 1623 goto out; 1624 r = kvm_vcpu_ioctl_dirty_tlb(vcpu, &dirty); 1625 break; 1626 } 1627 #endif 1628 default: 1629 r = -EINVAL; 1630 } 1631 1632 out: 1633 return r; 1634 } 1635 1636 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 1637 { 1638 return VM_FAULT_SIGBUS; 1639 } 1640 1641 static int kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo *pvinfo) 1642 { 1643 u32 inst_nop = 0x60000000; 1644 #ifdef CONFIG_KVM_BOOKE_HV 1645 u32 inst_sc1 = 0x44000022; 1646 pvinfo->hcall[0] = cpu_to_be32(inst_sc1); 1647 pvinfo->hcall[1] = cpu_to_be32(inst_nop); 1648 pvinfo->hcall[2] = cpu_to_be32(inst_nop); 1649 pvinfo->hcall[3] = cpu_to_be32(inst_nop); 1650 #else 1651 u32 inst_lis = 0x3c000000; 1652 u32 inst_ori = 0x60000000; 1653 u32 inst_sc = 0x44000002; 1654 u32 inst_imm_mask = 0xffff; 1655 1656 /* 1657 * The hypercall to get into KVM from within guest context is as 1658 * follows: 1659 * 1660 * lis r0, r0, KVM_SC_MAGIC_R0@h 1661 * ori r0, KVM_SC_MAGIC_R0@l 1662 * sc 1663 * nop 1664 */ 1665 pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask)); 1666 pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask)); 1667 pvinfo->hcall[2] = cpu_to_be32(inst_sc); 1668 pvinfo->hcall[3] = cpu_to_be32(inst_nop); 1669 #endif 1670 1671 pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE; 1672 1673 return 0; 1674 } 1675 1676 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, 1677 bool line_status) 1678 { 1679 if (!irqchip_in_kernel(kvm)) 1680 return -ENXIO; 1681 1682 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1683 irq_event->irq, irq_event->level, 1684 line_status); 1685 return 0; 1686 } 1687 1688 1689 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 1690 struct kvm_enable_cap *cap) 1691 { 1692 int r; 1693 1694 if (cap->flags) 1695 return -EINVAL; 1696 1697 switch (cap->cap) { 1698 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1699 case KVM_CAP_PPC_ENABLE_HCALL: { 1700 unsigned long hcall = cap->args[0]; 1701 1702 r = -EINVAL; 1703 if (hcall > MAX_HCALL_OPCODE || (hcall & 3) || 1704 cap->args[1] > 1) 1705 break; 1706 if (!kvmppc_book3s_hcall_implemented(kvm, hcall)) 1707 break; 1708 if (cap->args[1]) 1709 set_bit(hcall / 4, kvm->arch.enabled_hcalls); 1710 else 1711 clear_bit(hcall / 4, kvm->arch.enabled_hcalls); 1712 r = 0; 1713 break; 1714 } 1715 #endif 1716 default: 1717 r = -EINVAL; 1718 break; 1719 } 1720 1721 return r; 1722 } 1723 1724 long kvm_arch_vm_ioctl(struct file *filp, 1725 unsigned int ioctl, unsigned long arg) 1726 { 1727 struct kvm *kvm __maybe_unused = filp->private_data; 1728 void __user *argp = (void __user *)arg; 1729 long r; 1730 1731 switch (ioctl) { 1732 case KVM_PPC_GET_PVINFO: { 1733 struct kvm_ppc_pvinfo pvinfo; 1734 memset(&pvinfo, 0, sizeof(pvinfo)); 1735 r = kvm_vm_ioctl_get_pvinfo(&pvinfo); 1736 if (copy_to_user(argp, &pvinfo, sizeof(pvinfo))) { 1737 r = -EFAULT; 1738 goto out; 1739 } 1740 1741 break; 1742 } 1743 case KVM_ENABLE_CAP: 1744 { 1745 struct kvm_enable_cap cap; 1746 r = -EFAULT; 1747 if (copy_from_user(&cap, argp, sizeof(cap))) 1748 goto out; 1749 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 1750 break; 1751 } 1752 #ifdef CONFIG_SPAPR_TCE_IOMMU 1753 case KVM_CREATE_SPAPR_TCE_64: { 1754 struct kvm_create_spapr_tce_64 create_tce_64; 1755 1756 r = -EFAULT; 1757 if (copy_from_user(&create_tce_64, argp, sizeof(create_tce_64))) 1758 goto out; 1759 if (create_tce_64.flags) { 1760 r = -EINVAL; 1761 goto out; 1762 } 1763 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64); 1764 goto out; 1765 } 1766 case KVM_CREATE_SPAPR_TCE: { 1767 struct kvm_create_spapr_tce create_tce; 1768 struct kvm_create_spapr_tce_64 create_tce_64; 1769 1770 r = -EFAULT; 1771 if (copy_from_user(&create_tce, argp, sizeof(create_tce))) 1772 goto out; 1773 1774 create_tce_64.liobn = create_tce.liobn; 1775 create_tce_64.page_shift = IOMMU_PAGE_SHIFT_4K; 1776 create_tce_64.offset = 0; 1777 create_tce_64.size = create_tce.window_size >> 1778 IOMMU_PAGE_SHIFT_4K; 1779 create_tce_64.flags = 0; 1780 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64); 1781 goto out; 1782 } 1783 #endif 1784 #ifdef CONFIG_PPC_BOOK3S_64 1785 case KVM_PPC_GET_SMMU_INFO: { 1786 struct kvm_ppc_smmu_info info; 1787 struct kvm *kvm = filp->private_data; 1788 1789 memset(&info, 0, sizeof(info)); 1790 r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info); 1791 if (r >= 0 && copy_to_user(argp, &info, sizeof(info))) 1792 r = -EFAULT; 1793 break; 1794 } 1795 case KVM_PPC_RTAS_DEFINE_TOKEN: { 1796 struct kvm *kvm = filp->private_data; 1797 1798 r = kvm_vm_ioctl_rtas_define_token(kvm, argp); 1799 break; 1800 } 1801 case KVM_PPC_CONFIGURE_V3_MMU: { 1802 struct kvm *kvm = filp->private_data; 1803 struct kvm_ppc_mmuv3_cfg cfg; 1804 1805 r = -EINVAL; 1806 if (!kvm->arch.kvm_ops->configure_mmu) 1807 goto out; 1808 r = -EFAULT; 1809 if (copy_from_user(&cfg, argp, sizeof(cfg))) 1810 goto out; 1811 r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg); 1812 break; 1813 } 1814 case KVM_PPC_GET_RMMU_INFO: { 1815 struct kvm *kvm = filp->private_data; 1816 struct kvm_ppc_rmmu_info info; 1817 1818 r = -EINVAL; 1819 if (!kvm->arch.kvm_ops->get_rmmu_info) 1820 goto out; 1821 r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info); 1822 if (r >= 0 && copy_to_user(argp, &info, sizeof(info))) 1823 r = -EFAULT; 1824 break; 1825 } 1826 default: { 1827 struct kvm *kvm = filp->private_data; 1828 r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg); 1829 } 1830 #else /* CONFIG_PPC_BOOK3S_64 */ 1831 default: 1832 r = -ENOTTY; 1833 #endif 1834 } 1835 out: 1836 return r; 1837 } 1838 1839 static unsigned long lpid_inuse[BITS_TO_LONGS(KVMPPC_NR_LPIDS)]; 1840 static unsigned long nr_lpids; 1841 1842 long kvmppc_alloc_lpid(void) 1843 { 1844 long lpid; 1845 1846 do { 1847 lpid = find_first_zero_bit(lpid_inuse, KVMPPC_NR_LPIDS); 1848 if (lpid >= nr_lpids) { 1849 pr_err("%s: No LPIDs free\n", __func__); 1850 return -ENOMEM; 1851 } 1852 } while (test_and_set_bit(lpid, lpid_inuse)); 1853 1854 return lpid; 1855 } 1856 EXPORT_SYMBOL_GPL(kvmppc_alloc_lpid); 1857 1858 void kvmppc_claim_lpid(long lpid) 1859 { 1860 set_bit(lpid, lpid_inuse); 1861 } 1862 EXPORT_SYMBOL_GPL(kvmppc_claim_lpid); 1863 1864 void kvmppc_free_lpid(long lpid) 1865 { 1866 clear_bit(lpid, lpid_inuse); 1867 } 1868 EXPORT_SYMBOL_GPL(kvmppc_free_lpid); 1869 1870 void kvmppc_init_lpid(unsigned long nr_lpids_param) 1871 { 1872 nr_lpids = min_t(unsigned long, KVMPPC_NR_LPIDS, nr_lpids_param); 1873 memset(lpid_inuse, 0, sizeof(lpid_inuse)); 1874 } 1875 EXPORT_SYMBOL_GPL(kvmppc_init_lpid); 1876 1877 int kvm_arch_init(void *opaque) 1878 { 1879 return 0; 1880 } 1881 1882 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ppc_instr); 1883