1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright IBM Corp. 2007 5 * 6 * Authors: Hollis Blanchard <hollisb@us.ibm.com> 7 * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com> 8 */ 9 10 #include <linux/errno.h> 11 #include <linux/err.h> 12 #include <linux/kvm_host.h> 13 #include <linux/vmalloc.h> 14 #include <linux/hrtimer.h> 15 #include <linux/sched/signal.h> 16 #include <linux/fs.h> 17 #include <linux/slab.h> 18 #include <linux/file.h> 19 #include <linux/module.h> 20 #include <linux/irqbypass.h> 21 #include <linux/kvm_irqfd.h> 22 #include <linux/of.h> 23 #include <asm/cputable.h> 24 #include <linux/uaccess.h> 25 #include <asm/kvm_ppc.h> 26 #include <asm/cputhreads.h> 27 #include <asm/irqflags.h> 28 #include <asm/iommu.h> 29 #include <asm/switch_to.h> 30 #include <asm/xive.h> 31 #ifdef CONFIG_PPC_PSERIES 32 #include <asm/hvcall.h> 33 #include <asm/plpar_wrappers.h> 34 #endif 35 #include <asm/ultravisor.h> 36 #include <asm/setup.h> 37 38 #include "timing.h" 39 #include "../mm/mmu_decl.h" 40 41 #define CREATE_TRACE_POINTS 42 #include "trace.h" 43 44 struct kvmppc_ops *kvmppc_hv_ops; 45 EXPORT_SYMBOL_GPL(kvmppc_hv_ops); 46 struct kvmppc_ops *kvmppc_pr_ops; 47 EXPORT_SYMBOL_GPL(kvmppc_pr_ops); 48 49 50 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 51 { 52 return !!(v->arch.pending_exceptions) || kvm_request_pending(v); 53 } 54 55 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) 56 { 57 return kvm_arch_vcpu_runnable(vcpu); 58 } 59 60 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 61 { 62 return false; 63 } 64 65 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 66 { 67 return 1; 68 } 69 70 /* 71 * Common checks before entering the guest world. Call with interrupts 72 * disabled. 73 * 74 * returns: 75 * 76 * == 1 if we're ready to go into guest state 77 * <= 0 if we need to go back to the host with return value 78 */ 79 int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu) 80 { 81 int r; 82 83 WARN_ON(irqs_disabled()); 84 hard_irq_disable(); 85 86 while (true) { 87 if (need_resched()) { 88 local_irq_enable(); 89 cond_resched(); 90 hard_irq_disable(); 91 continue; 92 } 93 94 if (signal_pending(current)) { 95 kvmppc_account_exit(vcpu, SIGNAL_EXITS); 96 vcpu->run->exit_reason = KVM_EXIT_INTR; 97 r = -EINTR; 98 break; 99 } 100 101 vcpu->mode = IN_GUEST_MODE; 102 103 /* 104 * Reading vcpu->requests must happen after setting vcpu->mode, 105 * so we don't miss a request because the requester sees 106 * OUTSIDE_GUEST_MODE and assumes we'll be checking requests 107 * before next entering the guest (and thus doesn't IPI). 108 * This also orders the write to mode from any reads 109 * to the page tables done while the VCPU is running. 110 * Please see the comment in kvm_flush_remote_tlbs. 111 */ 112 smp_mb(); 113 114 if (kvm_request_pending(vcpu)) { 115 /* Make sure we process requests preemptable */ 116 local_irq_enable(); 117 trace_kvm_check_requests(vcpu); 118 r = kvmppc_core_check_requests(vcpu); 119 hard_irq_disable(); 120 if (r > 0) 121 continue; 122 break; 123 } 124 125 if (kvmppc_core_prepare_to_enter(vcpu)) { 126 /* interrupts got enabled in between, so we 127 are back at square 1 */ 128 continue; 129 } 130 131 guest_enter_irqoff(); 132 return 1; 133 } 134 135 /* return to host */ 136 local_irq_enable(); 137 return r; 138 } 139 EXPORT_SYMBOL_GPL(kvmppc_prepare_to_enter); 140 141 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) 142 static void kvmppc_swab_shared(struct kvm_vcpu *vcpu) 143 { 144 struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared; 145 int i; 146 147 shared->sprg0 = swab64(shared->sprg0); 148 shared->sprg1 = swab64(shared->sprg1); 149 shared->sprg2 = swab64(shared->sprg2); 150 shared->sprg3 = swab64(shared->sprg3); 151 shared->srr0 = swab64(shared->srr0); 152 shared->srr1 = swab64(shared->srr1); 153 shared->dar = swab64(shared->dar); 154 shared->msr = swab64(shared->msr); 155 shared->dsisr = swab32(shared->dsisr); 156 shared->int_pending = swab32(shared->int_pending); 157 for (i = 0; i < ARRAY_SIZE(shared->sr); i++) 158 shared->sr[i] = swab32(shared->sr[i]); 159 } 160 #endif 161 162 int kvmppc_kvm_pv(struct kvm_vcpu *vcpu) 163 { 164 int nr = kvmppc_get_gpr(vcpu, 11); 165 int r; 166 unsigned long __maybe_unused param1 = kvmppc_get_gpr(vcpu, 3); 167 unsigned long __maybe_unused param2 = kvmppc_get_gpr(vcpu, 4); 168 unsigned long __maybe_unused param3 = kvmppc_get_gpr(vcpu, 5); 169 unsigned long __maybe_unused param4 = kvmppc_get_gpr(vcpu, 6); 170 unsigned long r2 = 0; 171 172 if (!(kvmppc_get_msr(vcpu) & MSR_SF)) { 173 /* 32 bit mode */ 174 param1 &= 0xffffffff; 175 param2 &= 0xffffffff; 176 param3 &= 0xffffffff; 177 param4 &= 0xffffffff; 178 } 179 180 switch (nr) { 181 case KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE): 182 { 183 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) 184 /* Book3S can be little endian, find it out here */ 185 int shared_big_endian = true; 186 if (vcpu->arch.intr_msr & MSR_LE) 187 shared_big_endian = false; 188 if (shared_big_endian != vcpu->arch.shared_big_endian) 189 kvmppc_swab_shared(vcpu); 190 vcpu->arch.shared_big_endian = shared_big_endian; 191 #endif 192 193 if (!(param2 & MAGIC_PAGE_FLAG_NOT_MAPPED_NX)) { 194 /* 195 * Older versions of the Linux magic page code had 196 * a bug where they would map their trampoline code 197 * NX. If that's the case, remove !PR NX capability. 198 */ 199 vcpu->arch.disable_kernel_nx = true; 200 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 201 } 202 203 vcpu->arch.magic_page_pa = param1 & ~0xfffULL; 204 vcpu->arch.magic_page_ea = param2 & ~0xfffULL; 205 206 #ifdef CONFIG_PPC_64K_PAGES 207 /* 208 * Make sure our 4k magic page is in the same window of a 64k 209 * page within the guest and within the host's page. 210 */ 211 if ((vcpu->arch.magic_page_pa & 0xf000) != 212 ((ulong)vcpu->arch.shared & 0xf000)) { 213 void *old_shared = vcpu->arch.shared; 214 ulong shared = (ulong)vcpu->arch.shared; 215 void *new_shared; 216 217 shared &= PAGE_MASK; 218 shared |= vcpu->arch.magic_page_pa & 0xf000; 219 new_shared = (void*)shared; 220 memcpy(new_shared, old_shared, 0x1000); 221 vcpu->arch.shared = new_shared; 222 } 223 #endif 224 225 r2 = KVM_MAGIC_FEAT_SR | KVM_MAGIC_FEAT_MAS0_TO_SPRG7; 226 227 r = EV_SUCCESS; 228 break; 229 } 230 case KVM_HCALL_TOKEN(KVM_HC_FEATURES): 231 r = EV_SUCCESS; 232 #if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500V2) 233 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE); 234 #endif 235 236 /* Second return value is in r4 */ 237 break; 238 case EV_HCALL_TOKEN(EV_IDLE): 239 r = EV_SUCCESS; 240 kvm_vcpu_halt(vcpu); 241 break; 242 default: 243 r = EV_UNIMPLEMENTED; 244 break; 245 } 246 247 kvmppc_set_gpr(vcpu, 4, r2); 248 249 return r; 250 } 251 EXPORT_SYMBOL_GPL(kvmppc_kvm_pv); 252 253 int kvmppc_sanity_check(struct kvm_vcpu *vcpu) 254 { 255 int r = false; 256 257 /* We have to know what CPU to virtualize */ 258 if (!vcpu->arch.pvr) 259 goto out; 260 261 /* PAPR only works with book3s_64 */ 262 if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled) 263 goto out; 264 265 /* HV KVM can only do PAPR mode for now */ 266 if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm)) 267 goto out; 268 269 #ifdef CONFIG_KVM_BOOKE_HV 270 if (!cpu_has_feature(CPU_FTR_EMB_HV)) 271 goto out; 272 #endif 273 274 r = true; 275 276 out: 277 vcpu->arch.sane = r; 278 return r ? 0 : -EINVAL; 279 } 280 EXPORT_SYMBOL_GPL(kvmppc_sanity_check); 281 282 int kvmppc_emulate_mmio(struct kvm_vcpu *vcpu) 283 { 284 enum emulation_result er; 285 int r; 286 287 er = kvmppc_emulate_loadstore(vcpu); 288 switch (er) { 289 case EMULATE_DONE: 290 /* Future optimization: only reload non-volatiles if they were 291 * actually modified. */ 292 r = RESUME_GUEST_NV; 293 break; 294 case EMULATE_AGAIN: 295 r = RESUME_GUEST; 296 break; 297 case EMULATE_DO_MMIO: 298 vcpu->run->exit_reason = KVM_EXIT_MMIO; 299 /* We must reload nonvolatiles because "update" load/store 300 * instructions modify register state. */ 301 /* Future optimization: only reload non-volatiles if they were 302 * actually modified. */ 303 r = RESUME_HOST_NV; 304 break; 305 case EMULATE_FAIL: 306 { 307 ppc_inst_t last_inst; 308 309 kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst); 310 kvm_debug_ratelimited("Guest access to device memory using unsupported instruction (opcode: %#08x)\n", 311 ppc_inst_val(last_inst)); 312 313 /* 314 * Injecting a Data Storage here is a bit more 315 * accurate since the instruction that caused the 316 * access could still be a valid one. 317 */ 318 if (!IS_ENABLED(CONFIG_BOOKE)) { 319 ulong dsisr = DSISR_BADACCESS; 320 321 if (vcpu->mmio_is_write) 322 dsisr |= DSISR_ISSTORE; 323 324 kvmppc_core_queue_data_storage(vcpu, 325 kvmppc_get_msr(vcpu) & SRR1_PREFIXED, 326 vcpu->arch.vaddr_accessed, dsisr); 327 } else { 328 /* 329 * BookE does not send a SIGBUS on a bad 330 * fault, so use a Program interrupt instead 331 * to avoid a fault loop. 332 */ 333 kvmppc_core_queue_program(vcpu, 0); 334 } 335 336 r = RESUME_GUEST; 337 break; 338 } 339 default: 340 WARN_ON(1); 341 r = RESUME_GUEST; 342 } 343 344 return r; 345 } 346 EXPORT_SYMBOL_GPL(kvmppc_emulate_mmio); 347 348 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, 349 bool data) 350 { 351 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK; 352 struct kvmppc_pte pte; 353 int r = -EINVAL; 354 355 vcpu->stat.st++; 356 357 if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->store_to_eaddr) 358 r = vcpu->kvm->arch.kvm_ops->store_to_eaddr(vcpu, eaddr, ptr, 359 size); 360 361 if ((!r) || (r == -EAGAIN)) 362 return r; 363 364 r = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST, 365 XLATE_WRITE, &pte); 366 if (r < 0) 367 return r; 368 369 *eaddr = pte.raddr; 370 371 if (!pte.may_write) 372 return -EPERM; 373 374 /* Magic page override */ 375 if (kvmppc_supports_magic_page(vcpu) && mp_pa && 376 ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) && 377 !(kvmppc_get_msr(vcpu) & MSR_PR)) { 378 void *magic = vcpu->arch.shared; 379 magic += pte.eaddr & 0xfff; 380 memcpy(magic, ptr, size); 381 return EMULATE_DONE; 382 } 383 384 if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size)) 385 return EMULATE_DO_MMIO; 386 387 return EMULATE_DONE; 388 } 389 EXPORT_SYMBOL_GPL(kvmppc_st); 390 391 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, 392 bool data) 393 { 394 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK; 395 struct kvmppc_pte pte; 396 int rc = -EINVAL; 397 398 vcpu->stat.ld++; 399 400 if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->load_from_eaddr) 401 rc = vcpu->kvm->arch.kvm_ops->load_from_eaddr(vcpu, eaddr, ptr, 402 size); 403 404 if ((!rc) || (rc == -EAGAIN)) 405 return rc; 406 407 rc = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST, 408 XLATE_READ, &pte); 409 if (rc) 410 return rc; 411 412 *eaddr = pte.raddr; 413 414 if (!pte.may_read) 415 return -EPERM; 416 417 if (!data && !pte.may_execute) 418 return -ENOEXEC; 419 420 /* Magic page override */ 421 if (kvmppc_supports_magic_page(vcpu) && mp_pa && 422 ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) && 423 !(kvmppc_get_msr(vcpu) & MSR_PR)) { 424 void *magic = vcpu->arch.shared; 425 magic += pte.eaddr & 0xfff; 426 memcpy(ptr, magic, size); 427 return EMULATE_DONE; 428 } 429 430 kvm_vcpu_srcu_read_lock(vcpu); 431 rc = kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size); 432 kvm_vcpu_srcu_read_unlock(vcpu); 433 if (rc) 434 return EMULATE_DO_MMIO; 435 436 return EMULATE_DONE; 437 } 438 EXPORT_SYMBOL_GPL(kvmppc_ld); 439 440 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 441 { 442 struct kvmppc_ops *kvm_ops = NULL; 443 int r; 444 445 /* 446 * if we have both HV and PR enabled, default is HV 447 */ 448 if (type == 0) { 449 if (kvmppc_hv_ops) 450 kvm_ops = kvmppc_hv_ops; 451 else 452 kvm_ops = kvmppc_pr_ops; 453 if (!kvm_ops) 454 goto err_out; 455 } else if (type == KVM_VM_PPC_HV) { 456 if (!kvmppc_hv_ops) 457 goto err_out; 458 kvm_ops = kvmppc_hv_ops; 459 } else if (type == KVM_VM_PPC_PR) { 460 if (!kvmppc_pr_ops) 461 goto err_out; 462 kvm_ops = kvmppc_pr_ops; 463 } else 464 goto err_out; 465 466 if (!try_module_get(kvm_ops->owner)) 467 return -ENOENT; 468 469 kvm->arch.kvm_ops = kvm_ops; 470 r = kvmppc_core_init_vm(kvm); 471 if (r) 472 module_put(kvm_ops->owner); 473 return r; 474 err_out: 475 return -EINVAL; 476 } 477 478 void kvm_arch_destroy_vm(struct kvm *kvm) 479 { 480 #ifdef CONFIG_KVM_XICS 481 /* 482 * We call kick_all_cpus_sync() to ensure that all 483 * CPUs have executed any pending IPIs before we 484 * continue and free VCPUs structures below. 485 */ 486 if (is_kvmppc_hv_enabled(kvm)) 487 kick_all_cpus_sync(); 488 #endif 489 490 kvm_destroy_vcpus(kvm); 491 492 mutex_lock(&kvm->lock); 493 494 kvmppc_core_destroy_vm(kvm); 495 496 mutex_unlock(&kvm->lock); 497 498 /* drop the module reference */ 499 module_put(kvm->arch.kvm_ops->owner); 500 } 501 502 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 503 { 504 int r; 505 /* Assume we're using HV mode when the HV module is loaded */ 506 int hv_enabled = kvmppc_hv_ops ? 1 : 0; 507 508 if (kvm) { 509 /* 510 * Hooray - we know which VM type we're running on. Depend on 511 * that rather than the guess above. 512 */ 513 hv_enabled = is_kvmppc_hv_enabled(kvm); 514 } 515 516 switch (ext) { 517 #ifdef CONFIG_BOOKE 518 case KVM_CAP_PPC_BOOKE_SREGS: 519 case KVM_CAP_PPC_BOOKE_WATCHDOG: 520 case KVM_CAP_PPC_EPR: 521 #else 522 case KVM_CAP_PPC_SEGSTATE: 523 case KVM_CAP_PPC_HIOR: 524 case KVM_CAP_PPC_PAPR: 525 #endif 526 case KVM_CAP_PPC_UNSET_IRQ: 527 case KVM_CAP_PPC_IRQ_LEVEL: 528 case KVM_CAP_ENABLE_CAP: 529 case KVM_CAP_ONE_REG: 530 case KVM_CAP_IOEVENTFD: 531 case KVM_CAP_DEVICE_CTRL: 532 case KVM_CAP_IMMEDIATE_EXIT: 533 case KVM_CAP_SET_GUEST_DEBUG: 534 r = 1; 535 break; 536 case KVM_CAP_PPC_GUEST_DEBUG_SSTEP: 537 case KVM_CAP_PPC_PAIRED_SINGLES: 538 case KVM_CAP_PPC_OSI: 539 case KVM_CAP_PPC_GET_PVINFO: 540 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC) 541 case KVM_CAP_SW_TLB: 542 #endif 543 /* We support this only for PR */ 544 r = !hv_enabled; 545 break; 546 #ifdef CONFIG_KVM_MPIC 547 case KVM_CAP_IRQ_MPIC: 548 r = 1; 549 break; 550 #endif 551 552 #ifdef CONFIG_PPC_BOOK3S_64 553 case KVM_CAP_SPAPR_TCE: 554 case KVM_CAP_SPAPR_TCE_64: 555 r = 1; 556 break; 557 case KVM_CAP_SPAPR_TCE_VFIO: 558 r = !!cpu_has_feature(CPU_FTR_HVMODE); 559 break; 560 case KVM_CAP_PPC_RTAS: 561 case KVM_CAP_PPC_FIXUP_HCALL: 562 case KVM_CAP_PPC_ENABLE_HCALL: 563 #ifdef CONFIG_KVM_XICS 564 case KVM_CAP_IRQ_XICS: 565 #endif 566 case KVM_CAP_PPC_GET_CPU_CHAR: 567 r = 1; 568 break; 569 #ifdef CONFIG_KVM_XIVE 570 case KVM_CAP_PPC_IRQ_XIVE: 571 /* 572 * We need XIVE to be enabled on the platform (implies 573 * a POWER9 processor) and the PowerNV platform, as 574 * nested is not yet supported. 575 */ 576 r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE) && 577 kvmppc_xive_native_supported(); 578 break; 579 #endif 580 581 case KVM_CAP_PPC_ALLOC_HTAB: 582 r = hv_enabled; 583 break; 584 #endif /* CONFIG_PPC_BOOK3S_64 */ 585 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 586 case KVM_CAP_PPC_SMT: 587 r = 0; 588 if (kvm) { 589 if (kvm->arch.emul_smt_mode > 1) 590 r = kvm->arch.emul_smt_mode; 591 else 592 r = kvm->arch.smt_mode; 593 } else if (hv_enabled) { 594 if (cpu_has_feature(CPU_FTR_ARCH_300)) 595 r = 1; 596 else 597 r = threads_per_subcore; 598 } 599 break; 600 case KVM_CAP_PPC_SMT_POSSIBLE: 601 r = 1; 602 if (hv_enabled) { 603 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 604 r = ((threads_per_subcore << 1) - 1); 605 else 606 /* P9 can emulate dbells, so allow any mode */ 607 r = 8 | 4 | 2 | 1; 608 } 609 break; 610 case KVM_CAP_PPC_RMA: 611 r = 0; 612 break; 613 case KVM_CAP_PPC_HWRNG: 614 r = kvmppc_hwrng_present(); 615 break; 616 case KVM_CAP_PPC_MMU_RADIX: 617 r = !!(hv_enabled && radix_enabled()); 618 break; 619 case KVM_CAP_PPC_MMU_HASH_V3: 620 r = !!(hv_enabled && kvmppc_hv_ops->hash_v3_possible && 621 kvmppc_hv_ops->hash_v3_possible()); 622 break; 623 case KVM_CAP_PPC_NESTED_HV: 624 r = !!(hv_enabled && kvmppc_hv_ops->enable_nested && 625 !kvmppc_hv_ops->enable_nested(NULL)); 626 break; 627 #endif 628 case KVM_CAP_SYNC_MMU: 629 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 630 r = hv_enabled; 631 #elif defined(KVM_ARCH_WANT_MMU_NOTIFIER) 632 r = 1; 633 #else 634 r = 0; 635 #endif 636 break; 637 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 638 case KVM_CAP_PPC_HTAB_FD: 639 r = hv_enabled; 640 break; 641 #endif 642 case KVM_CAP_NR_VCPUS: 643 /* 644 * Recommending a number of CPUs is somewhat arbitrary; we 645 * return the number of present CPUs for -HV (since a host 646 * will have secondary threads "offline"), and for other KVM 647 * implementations just count online CPUs. 648 */ 649 if (hv_enabled) 650 r = min_t(unsigned int, num_present_cpus(), KVM_MAX_VCPUS); 651 else 652 r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); 653 break; 654 case KVM_CAP_MAX_VCPUS: 655 r = KVM_MAX_VCPUS; 656 break; 657 case KVM_CAP_MAX_VCPU_ID: 658 r = KVM_MAX_VCPU_IDS; 659 break; 660 #ifdef CONFIG_PPC_BOOK3S_64 661 case KVM_CAP_PPC_GET_SMMU_INFO: 662 r = 1; 663 break; 664 case KVM_CAP_SPAPR_MULTITCE: 665 r = 1; 666 break; 667 case KVM_CAP_SPAPR_RESIZE_HPT: 668 r = !!hv_enabled; 669 break; 670 #endif 671 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 672 case KVM_CAP_PPC_FWNMI: 673 r = hv_enabled; 674 break; 675 #endif 676 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 677 case KVM_CAP_PPC_HTM: 678 r = !!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_HTM) || 679 (hv_enabled && cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)); 680 break; 681 #endif 682 #if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) 683 case KVM_CAP_PPC_SECURE_GUEST: 684 r = hv_enabled && kvmppc_hv_ops->enable_svm && 685 !kvmppc_hv_ops->enable_svm(NULL); 686 break; 687 case KVM_CAP_PPC_DAWR1: 688 r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 && 689 !kvmppc_hv_ops->enable_dawr1(NULL)); 690 break; 691 case KVM_CAP_PPC_RPT_INVALIDATE: 692 r = 1; 693 break; 694 #endif 695 case KVM_CAP_PPC_AIL_MODE_3: 696 r = 0; 697 /* 698 * KVM PR, POWER7, and some POWER9s don't support AIL=3 mode. 699 * The POWER9s can support it if the guest runs in hash mode, 700 * but QEMU doesn't necessarily query the capability in time. 701 */ 702 if (hv_enabled) { 703 if (kvmhv_on_pseries()) { 704 if (pseries_reloc_on_exception()) 705 r = 1; 706 } else if (cpu_has_feature(CPU_FTR_ARCH_207S) && 707 !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) { 708 r = 1; 709 } 710 } 711 break; 712 default: 713 r = 0; 714 break; 715 } 716 return r; 717 718 } 719 720 long kvm_arch_dev_ioctl(struct file *filp, 721 unsigned int ioctl, unsigned long arg) 722 { 723 return -EINVAL; 724 } 725 726 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 727 { 728 kvmppc_core_free_memslot(kvm, slot); 729 } 730 731 int kvm_arch_prepare_memory_region(struct kvm *kvm, 732 const struct kvm_memory_slot *old, 733 struct kvm_memory_slot *new, 734 enum kvm_mr_change change) 735 { 736 return kvmppc_core_prepare_memory_region(kvm, old, new, change); 737 } 738 739 void kvm_arch_commit_memory_region(struct kvm *kvm, 740 struct kvm_memory_slot *old, 741 const struct kvm_memory_slot *new, 742 enum kvm_mr_change change) 743 { 744 kvmppc_core_commit_memory_region(kvm, old, new, change); 745 } 746 747 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 748 struct kvm_memory_slot *slot) 749 { 750 kvmppc_core_flush_memslot(kvm, slot); 751 } 752 753 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 754 { 755 return 0; 756 } 757 758 static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer) 759 { 760 struct kvm_vcpu *vcpu; 761 762 vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer); 763 kvmppc_decrementer_func(vcpu); 764 765 return HRTIMER_NORESTART; 766 } 767 768 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 769 { 770 int err; 771 772 hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 773 vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup; 774 775 #ifdef CONFIG_KVM_EXIT_TIMING 776 mutex_init(&vcpu->arch.exit_timing_lock); 777 #endif 778 err = kvmppc_subarch_vcpu_init(vcpu); 779 if (err) 780 return err; 781 782 err = kvmppc_core_vcpu_create(vcpu); 783 if (err) 784 goto out_vcpu_uninit; 785 786 rcuwait_init(&vcpu->arch.wait); 787 vcpu->arch.waitp = &vcpu->arch.wait; 788 return 0; 789 790 out_vcpu_uninit: 791 kvmppc_subarch_vcpu_uninit(vcpu); 792 return err; 793 } 794 795 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 796 { 797 } 798 799 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 800 { 801 /* Make sure we're not using the vcpu anymore */ 802 hrtimer_cancel(&vcpu->arch.dec_timer); 803 804 switch (vcpu->arch.irq_type) { 805 case KVMPPC_IRQ_MPIC: 806 kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu); 807 break; 808 case KVMPPC_IRQ_XICS: 809 if (xics_on_xive()) 810 kvmppc_xive_cleanup_vcpu(vcpu); 811 else 812 kvmppc_xics_free_icp(vcpu); 813 break; 814 case KVMPPC_IRQ_XIVE: 815 kvmppc_xive_native_cleanup_vcpu(vcpu); 816 break; 817 } 818 819 kvmppc_core_vcpu_free(vcpu); 820 821 kvmppc_subarch_vcpu_uninit(vcpu); 822 } 823 824 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 825 { 826 return kvmppc_core_pending_dec(vcpu); 827 } 828 829 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 830 { 831 #ifdef CONFIG_BOOKE 832 /* 833 * vrsave (formerly usprg0) isn't used by Linux, but may 834 * be used by the guest. 835 * 836 * On non-booke this is associated with Altivec and 837 * is handled by code in book3s.c. 838 */ 839 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); 840 #endif 841 kvmppc_core_vcpu_load(vcpu, cpu); 842 } 843 844 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 845 { 846 kvmppc_core_vcpu_put(vcpu); 847 #ifdef CONFIG_BOOKE 848 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); 849 #endif 850 } 851 852 /* 853 * irq_bypass_add_producer and irq_bypass_del_producer are only 854 * useful if the architecture supports PCI passthrough. 855 * irq_bypass_stop and irq_bypass_start are not needed and so 856 * kvm_ops are not defined for them. 857 */ 858 bool kvm_arch_has_irq_bypass(void) 859 { 860 return ((kvmppc_hv_ops && kvmppc_hv_ops->irq_bypass_add_producer) || 861 (kvmppc_pr_ops && kvmppc_pr_ops->irq_bypass_add_producer)); 862 } 863 864 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 865 struct irq_bypass_producer *prod) 866 { 867 struct kvm_kernel_irqfd *irqfd = 868 container_of(cons, struct kvm_kernel_irqfd, consumer); 869 struct kvm *kvm = irqfd->kvm; 870 871 if (kvm->arch.kvm_ops->irq_bypass_add_producer) 872 return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod); 873 874 return 0; 875 } 876 877 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 878 struct irq_bypass_producer *prod) 879 { 880 struct kvm_kernel_irqfd *irqfd = 881 container_of(cons, struct kvm_kernel_irqfd, consumer); 882 struct kvm *kvm = irqfd->kvm; 883 884 if (kvm->arch.kvm_ops->irq_bypass_del_producer) 885 kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod); 886 } 887 888 #ifdef CONFIG_VSX 889 static inline int kvmppc_get_vsr_dword_offset(int index) 890 { 891 int offset; 892 893 if ((index != 0) && (index != 1)) 894 return -1; 895 896 #ifdef __BIG_ENDIAN 897 offset = index; 898 #else 899 offset = 1 - index; 900 #endif 901 902 return offset; 903 } 904 905 static inline int kvmppc_get_vsr_word_offset(int index) 906 { 907 int offset; 908 909 if ((index > 3) || (index < 0)) 910 return -1; 911 912 #ifdef __BIG_ENDIAN 913 offset = index; 914 #else 915 offset = 3 - index; 916 #endif 917 return offset; 918 } 919 920 static inline void kvmppc_set_vsr_dword(struct kvm_vcpu *vcpu, 921 u64 gpr) 922 { 923 union kvmppc_one_reg val; 924 int offset = kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset); 925 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 926 927 if (offset == -1) 928 return; 929 930 if (index >= 32) { 931 val.vval = VCPU_VSX_VR(vcpu, index - 32); 932 val.vsxval[offset] = gpr; 933 VCPU_VSX_VR(vcpu, index - 32) = val.vval; 934 } else { 935 VCPU_VSX_FPR(vcpu, index, offset) = gpr; 936 } 937 } 938 939 static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu, 940 u64 gpr) 941 { 942 union kvmppc_one_reg val; 943 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 944 945 if (index >= 32) { 946 val.vval = VCPU_VSX_VR(vcpu, index - 32); 947 val.vsxval[0] = gpr; 948 val.vsxval[1] = gpr; 949 VCPU_VSX_VR(vcpu, index - 32) = val.vval; 950 } else { 951 VCPU_VSX_FPR(vcpu, index, 0) = gpr; 952 VCPU_VSX_FPR(vcpu, index, 1) = gpr; 953 } 954 } 955 956 static inline void kvmppc_set_vsr_word_dump(struct kvm_vcpu *vcpu, 957 u32 gpr) 958 { 959 union kvmppc_one_reg val; 960 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 961 962 if (index >= 32) { 963 val.vsx32val[0] = gpr; 964 val.vsx32val[1] = gpr; 965 val.vsx32val[2] = gpr; 966 val.vsx32val[3] = gpr; 967 VCPU_VSX_VR(vcpu, index - 32) = val.vval; 968 } else { 969 val.vsx32val[0] = gpr; 970 val.vsx32val[1] = gpr; 971 VCPU_VSX_FPR(vcpu, index, 0) = val.vsxval[0]; 972 VCPU_VSX_FPR(vcpu, index, 1) = val.vsxval[0]; 973 } 974 } 975 976 static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu, 977 u32 gpr32) 978 { 979 union kvmppc_one_reg val; 980 int offset = kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset); 981 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 982 int dword_offset, word_offset; 983 984 if (offset == -1) 985 return; 986 987 if (index >= 32) { 988 val.vval = VCPU_VSX_VR(vcpu, index - 32); 989 val.vsx32val[offset] = gpr32; 990 VCPU_VSX_VR(vcpu, index - 32) = val.vval; 991 } else { 992 dword_offset = offset / 2; 993 word_offset = offset % 2; 994 val.vsxval[0] = VCPU_VSX_FPR(vcpu, index, dword_offset); 995 val.vsx32val[word_offset] = gpr32; 996 VCPU_VSX_FPR(vcpu, index, dword_offset) = val.vsxval[0]; 997 } 998 } 999 #endif /* CONFIG_VSX */ 1000 1001 #ifdef CONFIG_ALTIVEC 1002 static inline int kvmppc_get_vmx_offset_generic(struct kvm_vcpu *vcpu, 1003 int index, int element_size) 1004 { 1005 int offset; 1006 int elts = sizeof(vector128)/element_size; 1007 1008 if ((index < 0) || (index >= elts)) 1009 return -1; 1010 1011 if (kvmppc_need_byteswap(vcpu)) 1012 offset = elts - index - 1; 1013 else 1014 offset = index; 1015 1016 return offset; 1017 } 1018 1019 static inline int kvmppc_get_vmx_dword_offset(struct kvm_vcpu *vcpu, 1020 int index) 1021 { 1022 return kvmppc_get_vmx_offset_generic(vcpu, index, 8); 1023 } 1024 1025 static inline int kvmppc_get_vmx_word_offset(struct kvm_vcpu *vcpu, 1026 int index) 1027 { 1028 return kvmppc_get_vmx_offset_generic(vcpu, index, 4); 1029 } 1030 1031 static inline int kvmppc_get_vmx_hword_offset(struct kvm_vcpu *vcpu, 1032 int index) 1033 { 1034 return kvmppc_get_vmx_offset_generic(vcpu, index, 2); 1035 } 1036 1037 static inline int kvmppc_get_vmx_byte_offset(struct kvm_vcpu *vcpu, 1038 int index) 1039 { 1040 return kvmppc_get_vmx_offset_generic(vcpu, index, 1); 1041 } 1042 1043 1044 static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu, 1045 u64 gpr) 1046 { 1047 union kvmppc_one_reg val; 1048 int offset = kvmppc_get_vmx_dword_offset(vcpu, 1049 vcpu->arch.mmio_vmx_offset); 1050 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 1051 1052 if (offset == -1) 1053 return; 1054 1055 val.vval = VCPU_VSX_VR(vcpu, index); 1056 val.vsxval[offset] = gpr; 1057 VCPU_VSX_VR(vcpu, index) = val.vval; 1058 } 1059 1060 static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu, 1061 u32 gpr32) 1062 { 1063 union kvmppc_one_reg val; 1064 int offset = kvmppc_get_vmx_word_offset(vcpu, 1065 vcpu->arch.mmio_vmx_offset); 1066 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 1067 1068 if (offset == -1) 1069 return; 1070 1071 val.vval = VCPU_VSX_VR(vcpu, index); 1072 val.vsx32val[offset] = gpr32; 1073 VCPU_VSX_VR(vcpu, index) = val.vval; 1074 } 1075 1076 static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu, 1077 u16 gpr16) 1078 { 1079 union kvmppc_one_reg val; 1080 int offset = kvmppc_get_vmx_hword_offset(vcpu, 1081 vcpu->arch.mmio_vmx_offset); 1082 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 1083 1084 if (offset == -1) 1085 return; 1086 1087 val.vval = VCPU_VSX_VR(vcpu, index); 1088 val.vsx16val[offset] = gpr16; 1089 VCPU_VSX_VR(vcpu, index) = val.vval; 1090 } 1091 1092 static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu, 1093 u8 gpr8) 1094 { 1095 union kvmppc_one_reg val; 1096 int offset = kvmppc_get_vmx_byte_offset(vcpu, 1097 vcpu->arch.mmio_vmx_offset); 1098 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; 1099 1100 if (offset == -1) 1101 return; 1102 1103 val.vval = VCPU_VSX_VR(vcpu, index); 1104 val.vsx8val[offset] = gpr8; 1105 VCPU_VSX_VR(vcpu, index) = val.vval; 1106 } 1107 #endif /* CONFIG_ALTIVEC */ 1108 1109 #ifdef CONFIG_PPC_FPU 1110 static inline u64 sp_to_dp(u32 fprs) 1111 { 1112 u64 fprd; 1113 1114 preempt_disable(); 1115 enable_kernel_fp(); 1116 asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m<>" (fprd) : "m<>" (fprs) 1117 : "fr0"); 1118 preempt_enable(); 1119 return fprd; 1120 } 1121 1122 static inline u32 dp_to_sp(u64 fprd) 1123 { 1124 u32 fprs; 1125 1126 preempt_disable(); 1127 enable_kernel_fp(); 1128 asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m<>" (fprs) : "m<>" (fprd) 1129 : "fr0"); 1130 preempt_enable(); 1131 return fprs; 1132 } 1133 1134 #else 1135 #define sp_to_dp(x) (x) 1136 #define dp_to_sp(x) (x) 1137 #endif /* CONFIG_PPC_FPU */ 1138 1139 static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu) 1140 { 1141 struct kvm_run *run = vcpu->run; 1142 u64 gpr; 1143 1144 if (run->mmio.len > sizeof(gpr)) 1145 return; 1146 1147 if (!vcpu->arch.mmio_host_swabbed) { 1148 switch (run->mmio.len) { 1149 case 8: gpr = *(u64 *)run->mmio.data; break; 1150 case 4: gpr = *(u32 *)run->mmio.data; break; 1151 case 2: gpr = *(u16 *)run->mmio.data; break; 1152 case 1: gpr = *(u8 *)run->mmio.data; break; 1153 } 1154 } else { 1155 switch (run->mmio.len) { 1156 case 8: gpr = swab64(*(u64 *)run->mmio.data); break; 1157 case 4: gpr = swab32(*(u32 *)run->mmio.data); break; 1158 case 2: gpr = swab16(*(u16 *)run->mmio.data); break; 1159 case 1: gpr = *(u8 *)run->mmio.data; break; 1160 } 1161 } 1162 1163 /* conversion between single and double precision */ 1164 if ((vcpu->arch.mmio_sp64_extend) && (run->mmio.len == 4)) 1165 gpr = sp_to_dp(gpr); 1166 1167 if (vcpu->arch.mmio_sign_extend) { 1168 switch (run->mmio.len) { 1169 #ifdef CONFIG_PPC64 1170 case 4: 1171 gpr = (s64)(s32)gpr; 1172 break; 1173 #endif 1174 case 2: 1175 gpr = (s64)(s16)gpr; 1176 break; 1177 case 1: 1178 gpr = (s64)(s8)gpr; 1179 break; 1180 } 1181 } 1182 1183 switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) { 1184 case KVM_MMIO_REG_GPR: 1185 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr); 1186 break; 1187 case KVM_MMIO_REG_FPR: 1188 if (vcpu->kvm->arch.kvm_ops->giveup_ext) 1189 vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP); 1190 1191 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr; 1192 break; 1193 #ifdef CONFIG_PPC_BOOK3S 1194 case KVM_MMIO_REG_QPR: 1195 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; 1196 break; 1197 case KVM_MMIO_REG_FQPR: 1198 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr; 1199 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; 1200 break; 1201 #endif 1202 #ifdef CONFIG_VSX 1203 case KVM_MMIO_REG_VSX: 1204 if (vcpu->kvm->arch.kvm_ops->giveup_ext) 1205 vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX); 1206 1207 if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_DWORD) 1208 kvmppc_set_vsr_dword(vcpu, gpr); 1209 else if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_WORD) 1210 kvmppc_set_vsr_word(vcpu, gpr); 1211 else if (vcpu->arch.mmio_copy_type == 1212 KVMPPC_VSX_COPY_DWORD_LOAD_DUMP) 1213 kvmppc_set_vsr_dword_dump(vcpu, gpr); 1214 else if (vcpu->arch.mmio_copy_type == 1215 KVMPPC_VSX_COPY_WORD_LOAD_DUMP) 1216 kvmppc_set_vsr_word_dump(vcpu, gpr); 1217 break; 1218 #endif 1219 #ifdef CONFIG_ALTIVEC 1220 case KVM_MMIO_REG_VMX: 1221 if (vcpu->kvm->arch.kvm_ops->giveup_ext) 1222 vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC); 1223 1224 if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_DWORD) 1225 kvmppc_set_vmx_dword(vcpu, gpr); 1226 else if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_WORD) 1227 kvmppc_set_vmx_word(vcpu, gpr); 1228 else if (vcpu->arch.mmio_copy_type == 1229 KVMPPC_VMX_COPY_HWORD) 1230 kvmppc_set_vmx_hword(vcpu, gpr); 1231 else if (vcpu->arch.mmio_copy_type == 1232 KVMPPC_VMX_COPY_BYTE) 1233 kvmppc_set_vmx_byte(vcpu, gpr); 1234 break; 1235 #endif 1236 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1237 case KVM_MMIO_REG_NESTED_GPR: 1238 if (kvmppc_need_byteswap(vcpu)) 1239 gpr = swab64(gpr); 1240 kvm_vcpu_write_guest(vcpu, vcpu->arch.nested_io_gpr, &gpr, 1241 sizeof(gpr)); 1242 break; 1243 #endif 1244 default: 1245 BUG(); 1246 } 1247 } 1248 1249 static int __kvmppc_handle_load(struct kvm_vcpu *vcpu, 1250 unsigned int rt, unsigned int bytes, 1251 int is_default_endian, int sign_extend) 1252 { 1253 struct kvm_run *run = vcpu->run; 1254 int idx, ret; 1255 bool host_swabbed; 1256 1257 /* Pity C doesn't have a logical XOR operator */ 1258 if (kvmppc_need_byteswap(vcpu)) { 1259 host_swabbed = is_default_endian; 1260 } else { 1261 host_swabbed = !is_default_endian; 1262 } 1263 1264 if (bytes > sizeof(run->mmio.data)) 1265 return EMULATE_FAIL; 1266 1267 run->mmio.phys_addr = vcpu->arch.paddr_accessed; 1268 run->mmio.len = bytes; 1269 run->mmio.is_write = 0; 1270 1271 vcpu->arch.io_gpr = rt; 1272 vcpu->arch.mmio_host_swabbed = host_swabbed; 1273 vcpu->mmio_needed = 1; 1274 vcpu->mmio_is_write = 0; 1275 vcpu->arch.mmio_sign_extend = sign_extend; 1276 1277 idx = srcu_read_lock(&vcpu->kvm->srcu); 1278 1279 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, 1280 bytes, &run->mmio.data); 1281 1282 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1283 1284 if (!ret) { 1285 kvmppc_complete_mmio_load(vcpu); 1286 vcpu->mmio_needed = 0; 1287 return EMULATE_DONE; 1288 } 1289 1290 return EMULATE_DO_MMIO; 1291 } 1292 1293 int kvmppc_handle_load(struct kvm_vcpu *vcpu, 1294 unsigned int rt, unsigned int bytes, 1295 int is_default_endian) 1296 { 1297 return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 0); 1298 } 1299 EXPORT_SYMBOL_GPL(kvmppc_handle_load); 1300 1301 /* Same as above, but sign extends */ 1302 int kvmppc_handle_loads(struct kvm_vcpu *vcpu, 1303 unsigned int rt, unsigned int bytes, 1304 int is_default_endian) 1305 { 1306 return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 1); 1307 } 1308 1309 #ifdef CONFIG_VSX 1310 int kvmppc_handle_vsx_load(struct kvm_vcpu *vcpu, 1311 unsigned int rt, unsigned int bytes, 1312 int is_default_endian, int mmio_sign_extend) 1313 { 1314 enum emulation_result emulated = EMULATE_DONE; 1315 1316 /* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */ 1317 if (vcpu->arch.mmio_vsx_copy_nums > 4) 1318 return EMULATE_FAIL; 1319 1320 while (vcpu->arch.mmio_vsx_copy_nums) { 1321 emulated = __kvmppc_handle_load(vcpu, rt, bytes, 1322 is_default_endian, mmio_sign_extend); 1323 1324 if (emulated != EMULATE_DONE) 1325 break; 1326 1327 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; 1328 1329 vcpu->arch.mmio_vsx_copy_nums--; 1330 vcpu->arch.mmio_vsx_offset++; 1331 } 1332 return emulated; 1333 } 1334 #endif /* CONFIG_VSX */ 1335 1336 int kvmppc_handle_store(struct kvm_vcpu *vcpu, 1337 u64 val, unsigned int bytes, int is_default_endian) 1338 { 1339 struct kvm_run *run = vcpu->run; 1340 void *data = run->mmio.data; 1341 int idx, ret; 1342 bool host_swabbed; 1343 1344 /* Pity C doesn't have a logical XOR operator */ 1345 if (kvmppc_need_byteswap(vcpu)) { 1346 host_swabbed = is_default_endian; 1347 } else { 1348 host_swabbed = !is_default_endian; 1349 } 1350 1351 if (bytes > sizeof(run->mmio.data)) 1352 return EMULATE_FAIL; 1353 1354 run->mmio.phys_addr = vcpu->arch.paddr_accessed; 1355 run->mmio.len = bytes; 1356 run->mmio.is_write = 1; 1357 vcpu->mmio_needed = 1; 1358 vcpu->mmio_is_write = 1; 1359 1360 if ((vcpu->arch.mmio_sp64_extend) && (bytes == 4)) 1361 val = dp_to_sp(val); 1362 1363 /* Store the value at the lowest bytes in 'data'. */ 1364 if (!host_swabbed) { 1365 switch (bytes) { 1366 case 8: *(u64 *)data = val; break; 1367 case 4: *(u32 *)data = val; break; 1368 case 2: *(u16 *)data = val; break; 1369 case 1: *(u8 *)data = val; break; 1370 } 1371 } else { 1372 switch (bytes) { 1373 case 8: *(u64 *)data = swab64(val); break; 1374 case 4: *(u32 *)data = swab32(val); break; 1375 case 2: *(u16 *)data = swab16(val); break; 1376 case 1: *(u8 *)data = val; break; 1377 } 1378 } 1379 1380 idx = srcu_read_lock(&vcpu->kvm->srcu); 1381 1382 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, 1383 bytes, &run->mmio.data); 1384 1385 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1386 1387 if (!ret) { 1388 vcpu->mmio_needed = 0; 1389 return EMULATE_DONE; 1390 } 1391 1392 return EMULATE_DO_MMIO; 1393 } 1394 EXPORT_SYMBOL_GPL(kvmppc_handle_store); 1395 1396 #ifdef CONFIG_VSX 1397 static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val) 1398 { 1399 u32 dword_offset, word_offset; 1400 union kvmppc_one_reg reg; 1401 int vsx_offset = 0; 1402 int copy_type = vcpu->arch.mmio_copy_type; 1403 int result = 0; 1404 1405 switch (copy_type) { 1406 case KVMPPC_VSX_COPY_DWORD: 1407 vsx_offset = 1408 kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset); 1409 1410 if (vsx_offset == -1) { 1411 result = -1; 1412 break; 1413 } 1414 1415 if (rs < 32) { 1416 *val = VCPU_VSX_FPR(vcpu, rs, vsx_offset); 1417 } else { 1418 reg.vval = VCPU_VSX_VR(vcpu, rs - 32); 1419 *val = reg.vsxval[vsx_offset]; 1420 } 1421 break; 1422 1423 case KVMPPC_VSX_COPY_WORD: 1424 vsx_offset = 1425 kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset); 1426 1427 if (vsx_offset == -1) { 1428 result = -1; 1429 break; 1430 } 1431 1432 if (rs < 32) { 1433 dword_offset = vsx_offset / 2; 1434 word_offset = vsx_offset % 2; 1435 reg.vsxval[0] = VCPU_VSX_FPR(vcpu, rs, dword_offset); 1436 *val = reg.vsx32val[word_offset]; 1437 } else { 1438 reg.vval = VCPU_VSX_VR(vcpu, rs - 32); 1439 *val = reg.vsx32val[vsx_offset]; 1440 } 1441 break; 1442 1443 default: 1444 result = -1; 1445 break; 1446 } 1447 1448 return result; 1449 } 1450 1451 int kvmppc_handle_vsx_store(struct kvm_vcpu *vcpu, 1452 int rs, unsigned int bytes, int is_default_endian) 1453 { 1454 u64 val; 1455 enum emulation_result emulated = EMULATE_DONE; 1456 1457 vcpu->arch.io_gpr = rs; 1458 1459 /* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */ 1460 if (vcpu->arch.mmio_vsx_copy_nums > 4) 1461 return EMULATE_FAIL; 1462 1463 while (vcpu->arch.mmio_vsx_copy_nums) { 1464 if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1) 1465 return EMULATE_FAIL; 1466 1467 emulated = kvmppc_handle_store(vcpu, 1468 val, bytes, is_default_endian); 1469 1470 if (emulated != EMULATE_DONE) 1471 break; 1472 1473 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; 1474 1475 vcpu->arch.mmio_vsx_copy_nums--; 1476 vcpu->arch.mmio_vsx_offset++; 1477 } 1478 1479 return emulated; 1480 } 1481 1482 static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu) 1483 { 1484 struct kvm_run *run = vcpu->run; 1485 enum emulation_result emulated = EMULATE_FAIL; 1486 int r; 1487 1488 vcpu->arch.paddr_accessed += run->mmio.len; 1489 1490 if (!vcpu->mmio_is_write) { 1491 emulated = kvmppc_handle_vsx_load(vcpu, vcpu->arch.io_gpr, 1492 run->mmio.len, 1, vcpu->arch.mmio_sign_extend); 1493 } else { 1494 emulated = kvmppc_handle_vsx_store(vcpu, 1495 vcpu->arch.io_gpr, run->mmio.len, 1); 1496 } 1497 1498 switch (emulated) { 1499 case EMULATE_DO_MMIO: 1500 run->exit_reason = KVM_EXIT_MMIO; 1501 r = RESUME_HOST; 1502 break; 1503 case EMULATE_FAIL: 1504 pr_info("KVM: MMIO emulation failed (VSX repeat)\n"); 1505 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1506 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 1507 r = RESUME_HOST; 1508 break; 1509 default: 1510 r = RESUME_GUEST; 1511 break; 1512 } 1513 return r; 1514 } 1515 #endif /* CONFIG_VSX */ 1516 1517 #ifdef CONFIG_ALTIVEC 1518 int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu, 1519 unsigned int rt, unsigned int bytes, int is_default_endian) 1520 { 1521 enum emulation_result emulated = EMULATE_DONE; 1522 1523 if (vcpu->arch.mmio_vmx_copy_nums > 2) 1524 return EMULATE_FAIL; 1525 1526 while (vcpu->arch.mmio_vmx_copy_nums) { 1527 emulated = __kvmppc_handle_load(vcpu, rt, bytes, 1528 is_default_endian, 0); 1529 1530 if (emulated != EMULATE_DONE) 1531 break; 1532 1533 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; 1534 vcpu->arch.mmio_vmx_copy_nums--; 1535 vcpu->arch.mmio_vmx_offset++; 1536 } 1537 1538 return emulated; 1539 } 1540 1541 static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) 1542 { 1543 union kvmppc_one_reg reg; 1544 int vmx_offset = 0; 1545 int result = 0; 1546 1547 vmx_offset = 1548 kvmppc_get_vmx_dword_offset(vcpu, vcpu->arch.mmio_vmx_offset); 1549 1550 if (vmx_offset == -1) 1551 return -1; 1552 1553 reg.vval = VCPU_VSX_VR(vcpu, index); 1554 *val = reg.vsxval[vmx_offset]; 1555 1556 return result; 1557 } 1558 1559 static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) 1560 { 1561 union kvmppc_one_reg reg; 1562 int vmx_offset = 0; 1563 int result = 0; 1564 1565 vmx_offset = 1566 kvmppc_get_vmx_word_offset(vcpu, vcpu->arch.mmio_vmx_offset); 1567 1568 if (vmx_offset == -1) 1569 return -1; 1570 1571 reg.vval = VCPU_VSX_VR(vcpu, index); 1572 *val = reg.vsx32val[vmx_offset]; 1573 1574 return result; 1575 } 1576 1577 static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) 1578 { 1579 union kvmppc_one_reg reg; 1580 int vmx_offset = 0; 1581 int result = 0; 1582 1583 vmx_offset = 1584 kvmppc_get_vmx_hword_offset(vcpu, vcpu->arch.mmio_vmx_offset); 1585 1586 if (vmx_offset == -1) 1587 return -1; 1588 1589 reg.vval = VCPU_VSX_VR(vcpu, index); 1590 *val = reg.vsx16val[vmx_offset]; 1591 1592 return result; 1593 } 1594 1595 static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) 1596 { 1597 union kvmppc_one_reg reg; 1598 int vmx_offset = 0; 1599 int result = 0; 1600 1601 vmx_offset = 1602 kvmppc_get_vmx_byte_offset(vcpu, vcpu->arch.mmio_vmx_offset); 1603 1604 if (vmx_offset == -1) 1605 return -1; 1606 1607 reg.vval = VCPU_VSX_VR(vcpu, index); 1608 *val = reg.vsx8val[vmx_offset]; 1609 1610 return result; 1611 } 1612 1613 int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu, 1614 unsigned int rs, unsigned int bytes, int is_default_endian) 1615 { 1616 u64 val = 0; 1617 unsigned int index = rs & KVM_MMIO_REG_MASK; 1618 enum emulation_result emulated = EMULATE_DONE; 1619 1620 if (vcpu->arch.mmio_vmx_copy_nums > 2) 1621 return EMULATE_FAIL; 1622 1623 vcpu->arch.io_gpr = rs; 1624 1625 while (vcpu->arch.mmio_vmx_copy_nums) { 1626 switch (vcpu->arch.mmio_copy_type) { 1627 case KVMPPC_VMX_COPY_DWORD: 1628 if (kvmppc_get_vmx_dword(vcpu, index, &val) == -1) 1629 return EMULATE_FAIL; 1630 1631 break; 1632 case KVMPPC_VMX_COPY_WORD: 1633 if (kvmppc_get_vmx_word(vcpu, index, &val) == -1) 1634 return EMULATE_FAIL; 1635 break; 1636 case KVMPPC_VMX_COPY_HWORD: 1637 if (kvmppc_get_vmx_hword(vcpu, index, &val) == -1) 1638 return EMULATE_FAIL; 1639 break; 1640 case KVMPPC_VMX_COPY_BYTE: 1641 if (kvmppc_get_vmx_byte(vcpu, index, &val) == -1) 1642 return EMULATE_FAIL; 1643 break; 1644 default: 1645 return EMULATE_FAIL; 1646 } 1647 1648 emulated = kvmppc_handle_store(vcpu, val, bytes, 1649 is_default_endian); 1650 if (emulated != EMULATE_DONE) 1651 break; 1652 1653 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; 1654 vcpu->arch.mmio_vmx_copy_nums--; 1655 vcpu->arch.mmio_vmx_offset++; 1656 } 1657 1658 return emulated; 1659 } 1660 1661 static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu) 1662 { 1663 struct kvm_run *run = vcpu->run; 1664 enum emulation_result emulated = EMULATE_FAIL; 1665 int r; 1666 1667 vcpu->arch.paddr_accessed += run->mmio.len; 1668 1669 if (!vcpu->mmio_is_write) { 1670 emulated = kvmppc_handle_vmx_load(vcpu, 1671 vcpu->arch.io_gpr, run->mmio.len, 1); 1672 } else { 1673 emulated = kvmppc_handle_vmx_store(vcpu, 1674 vcpu->arch.io_gpr, run->mmio.len, 1); 1675 } 1676 1677 switch (emulated) { 1678 case EMULATE_DO_MMIO: 1679 run->exit_reason = KVM_EXIT_MMIO; 1680 r = RESUME_HOST; 1681 break; 1682 case EMULATE_FAIL: 1683 pr_info("KVM: MMIO emulation failed (VMX repeat)\n"); 1684 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1685 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 1686 r = RESUME_HOST; 1687 break; 1688 default: 1689 r = RESUME_GUEST; 1690 break; 1691 } 1692 return r; 1693 } 1694 #endif /* CONFIG_ALTIVEC */ 1695 1696 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) 1697 { 1698 int r = 0; 1699 union kvmppc_one_reg val; 1700 int size; 1701 1702 size = one_reg_size(reg->id); 1703 if (size > sizeof(val)) 1704 return -EINVAL; 1705 1706 r = kvmppc_get_one_reg(vcpu, reg->id, &val); 1707 if (r == -EINVAL) { 1708 r = 0; 1709 switch (reg->id) { 1710 #ifdef CONFIG_ALTIVEC 1711 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31: 1712 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1713 r = -ENXIO; 1714 break; 1715 } 1716 val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0]; 1717 break; 1718 case KVM_REG_PPC_VSCR: 1719 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1720 r = -ENXIO; 1721 break; 1722 } 1723 val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]); 1724 break; 1725 case KVM_REG_PPC_VRSAVE: 1726 val = get_reg_val(reg->id, vcpu->arch.vrsave); 1727 break; 1728 #endif /* CONFIG_ALTIVEC */ 1729 default: 1730 r = -EINVAL; 1731 break; 1732 } 1733 } 1734 1735 if (r) 1736 return r; 1737 1738 if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size)) 1739 r = -EFAULT; 1740 1741 return r; 1742 } 1743 1744 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) 1745 { 1746 int r; 1747 union kvmppc_one_reg val; 1748 int size; 1749 1750 size = one_reg_size(reg->id); 1751 if (size > sizeof(val)) 1752 return -EINVAL; 1753 1754 if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size)) 1755 return -EFAULT; 1756 1757 r = kvmppc_set_one_reg(vcpu, reg->id, &val); 1758 if (r == -EINVAL) { 1759 r = 0; 1760 switch (reg->id) { 1761 #ifdef CONFIG_ALTIVEC 1762 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31: 1763 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1764 r = -ENXIO; 1765 break; 1766 } 1767 vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval; 1768 break; 1769 case KVM_REG_PPC_VSCR: 1770 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1771 r = -ENXIO; 1772 break; 1773 } 1774 vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val); 1775 break; 1776 case KVM_REG_PPC_VRSAVE: 1777 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { 1778 r = -ENXIO; 1779 break; 1780 } 1781 vcpu->arch.vrsave = set_reg_val(reg->id, val); 1782 break; 1783 #endif /* CONFIG_ALTIVEC */ 1784 default: 1785 r = -EINVAL; 1786 break; 1787 } 1788 } 1789 1790 return r; 1791 } 1792 1793 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1794 { 1795 struct kvm_run *run = vcpu->run; 1796 int r; 1797 1798 vcpu_load(vcpu); 1799 1800 if (vcpu->mmio_needed) { 1801 vcpu->mmio_needed = 0; 1802 if (!vcpu->mmio_is_write) 1803 kvmppc_complete_mmio_load(vcpu); 1804 #ifdef CONFIG_VSX 1805 if (vcpu->arch.mmio_vsx_copy_nums > 0) { 1806 vcpu->arch.mmio_vsx_copy_nums--; 1807 vcpu->arch.mmio_vsx_offset++; 1808 } 1809 1810 if (vcpu->arch.mmio_vsx_copy_nums > 0) { 1811 r = kvmppc_emulate_mmio_vsx_loadstore(vcpu); 1812 if (r == RESUME_HOST) { 1813 vcpu->mmio_needed = 1; 1814 goto out; 1815 } 1816 } 1817 #endif 1818 #ifdef CONFIG_ALTIVEC 1819 if (vcpu->arch.mmio_vmx_copy_nums > 0) { 1820 vcpu->arch.mmio_vmx_copy_nums--; 1821 vcpu->arch.mmio_vmx_offset++; 1822 } 1823 1824 if (vcpu->arch.mmio_vmx_copy_nums > 0) { 1825 r = kvmppc_emulate_mmio_vmx_loadstore(vcpu); 1826 if (r == RESUME_HOST) { 1827 vcpu->mmio_needed = 1; 1828 goto out; 1829 } 1830 } 1831 #endif 1832 } else if (vcpu->arch.osi_needed) { 1833 u64 *gprs = run->osi.gprs; 1834 int i; 1835 1836 for (i = 0; i < 32; i++) 1837 kvmppc_set_gpr(vcpu, i, gprs[i]); 1838 vcpu->arch.osi_needed = 0; 1839 } else if (vcpu->arch.hcall_needed) { 1840 int i; 1841 1842 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret); 1843 for (i = 0; i < 9; ++i) 1844 kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]); 1845 vcpu->arch.hcall_needed = 0; 1846 #ifdef CONFIG_BOOKE 1847 } else if (vcpu->arch.epr_needed) { 1848 kvmppc_set_epr(vcpu, run->epr.epr); 1849 vcpu->arch.epr_needed = 0; 1850 #endif 1851 } 1852 1853 kvm_sigset_activate(vcpu); 1854 1855 if (run->immediate_exit) 1856 r = -EINTR; 1857 else 1858 r = kvmppc_vcpu_run(vcpu); 1859 1860 kvm_sigset_deactivate(vcpu); 1861 1862 #ifdef CONFIG_ALTIVEC 1863 out: 1864 #endif 1865 1866 /* 1867 * We're already returning to userspace, don't pass the 1868 * RESUME_HOST flags along. 1869 */ 1870 if (r > 0) 1871 r = 0; 1872 1873 vcpu_put(vcpu); 1874 return r; 1875 } 1876 1877 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) 1878 { 1879 if (irq->irq == KVM_INTERRUPT_UNSET) { 1880 kvmppc_core_dequeue_external(vcpu); 1881 return 0; 1882 } 1883 1884 kvmppc_core_queue_external(vcpu, irq); 1885 1886 kvm_vcpu_kick(vcpu); 1887 1888 return 0; 1889 } 1890 1891 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 1892 struct kvm_enable_cap *cap) 1893 { 1894 int r; 1895 1896 if (cap->flags) 1897 return -EINVAL; 1898 1899 switch (cap->cap) { 1900 case KVM_CAP_PPC_OSI: 1901 r = 0; 1902 vcpu->arch.osi_enabled = true; 1903 break; 1904 case KVM_CAP_PPC_PAPR: 1905 r = 0; 1906 vcpu->arch.papr_enabled = true; 1907 break; 1908 case KVM_CAP_PPC_EPR: 1909 r = 0; 1910 if (cap->args[0]) 1911 vcpu->arch.epr_flags |= KVMPPC_EPR_USER; 1912 else 1913 vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER; 1914 break; 1915 #ifdef CONFIG_BOOKE 1916 case KVM_CAP_PPC_BOOKE_WATCHDOG: 1917 r = 0; 1918 vcpu->arch.watchdog_enabled = true; 1919 break; 1920 #endif 1921 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC) 1922 case KVM_CAP_SW_TLB: { 1923 struct kvm_config_tlb cfg; 1924 void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0]; 1925 1926 r = -EFAULT; 1927 if (copy_from_user(&cfg, user_ptr, sizeof(cfg))) 1928 break; 1929 1930 r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg); 1931 break; 1932 } 1933 #endif 1934 #ifdef CONFIG_KVM_MPIC 1935 case KVM_CAP_IRQ_MPIC: { 1936 struct fd f; 1937 struct kvm_device *dev; 1938 1939 r = -EBADF; 1940 f = fdget(cap->args[0]); 1941 if (!f.file) 1942 break; 1943 1944 r = -EPERM; 1945 dev = kvm_device_from_filp(f.file); 1946 if (dev) 1947 r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]); 1948 1949 fdput(f); 1950 break; 1951 } 1952 #endif 1953 #ifdef CONFIG_KVM_XICS 1954 case KVM_CAP_IRQ_XICS: { 1955 struct fd f; 1956 struct kvm_device *dev; 1957 1958 r = -EBADF; 1959 f = fdget(cap->args[0]); 1960 if (!f.file) 1961 break; 1962 1963 r = -EPERM; 1964 dev = kvm_device_from_filp(f.file); 1965 if (dev) { 1966 if (xics_on_xive()) 1967 r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]); 1968 else 1969 r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]); 1970 } 1971 1972 fdput(f); 1973 break; 1974 } 1975 #endif /* CONFIG_KVM_XICS */ 1976 #ifdef CONFIG_KVM_XIVE 1977 case KVM_CAP_PPC_IRQ_XIVE: { 1978 struct fd f; 1979 struct kvm_device *dev; 1980 1981 r = -EBADF; 1982 f = fdget(cap->args[0]); 1983 if (!f.file) 1984 break; 1985 1986 r = -ENXIO; 1987 if (!xive_enabled()) 1988 break; 1989 1990 r = -EPERM; 1991 dev = kvm_device_from_filp(f.file); 1992 if (dev) 1993 r = kvmppc_xive_native_connect_vcpu(dev, vcpu, 1994 cap->args[1]); 1995 1996 fdput(f); 1997 break; 1998 } 1999 #endif /* CONFIG_KVM_XIVE */ 2000 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2001 case KVM_CAP_PPC_FWNMI: 2002 r = -EINVAL; 2003 if (!is_kvmppc_hv_enabled(vcpu->kvm)) 2004 break; 2005 r = 0; 2006 vcpu->kvm->arch.fwnmi_enabled = true; 2007 break; 2008 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 2009 default: 2010 r = -EINVAL; 2011 break; 2012 } 2013 2014 if (!r) 2015 r = kvmppc_sanity_check(vcpu); 2016 2017 return r; 2018 } 2019 2020 bool kvm_arch_intc_initialized(struct kvm *kvm) 2021 { 2022 #ifdef CONFIG_KVM_MPIC 2023 if (kvm->arch.mpic) 2024 return true; 2025 #endif 2026 #ifdef CONFIG_KVM_XICS 2027 if (kvm->arch.xics || kvm->arch.xive) 2028 return true; 2029 #endif 2030 return false; 2031 } 2032 2033 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 2034 struct kvm_mp_state *mp_state) 2035 { 2036 return -EINVAL; 2037 } 2038 2039 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 2040 struct kvm_mp_state *mp_state) 2041 { 2042 return -EINVAL; 2043 } 2044 2045 long kvm_arch_vcpu_async_ioctl(struct file *filp, 2046 unsigned int ioctl, unsigned long arg) 2047 { 2048 struct kvm_vcpu *vcpu = filp->private_data; 2049 void __user *argp = (void __user *)arg; 2050 2051 if (ioctl == KVM_INTERRUPT) { 2052 struct kvm_interrupt irq; 2053 if (copy_from_user(&irq, argp, sizeof(irq))) 2054 return -EFAULT; 2055 return kvm_vcpu_ioctl_interrupt(vcpu, &irq); 2056 } 2057 return -ENOIOCTLCMD; 2058 } 2059 2060 long kvm_arch_vcpu_ioctl(struct file *filp, 2061 unsigned int ioctl, unsigned long arg) 2062 { 2063 struct kvm_vcpu *vcpu = filp->private_data; 2064 void __user *argp = (void __user *)arg; 2065 long r; 2066 2067 switch (ioctl) { 2068 case KVM_ENABLE_CAP: 2069 { 2070 struct kvm_enable_cap cap; 2071 r = -EFAULT; 2072 if (copy_from_user(&cap, argp, sizeof(cap))) 2073 goto out; 2074 vcpu_load(vcpu); 2075 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 2076 vcpu_put(vcpu); 2077 break; 2078 } 2079 2080 case KVM_SET_ONE_REG: 2081 case KVM_GET_ONE_REG: 2082 { 2083 struct kvm_one_reg reg; 2084 r = -EFAULT; 2085 if (copy_from_user(®, argp, sizeof(reg))) 2086 goto out; 2087 if (ioctl == KVM_SET_ONE_REG) 2088 r = kvm_vcpu_ioctl_set_one_reg(vcpu, ®); 2089 else 2090 r = kvm_vcpu_ioctl_get_one_reg(vcpu, ®); 2091 break; 2092 } 2093 2094 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC) 2095 case KVM_DIRTY_TLB: { 2096 struct kvm_dirty_tlb dirty; 2097 r = -EFAULT; 2098 if (copy_from_user(&dirty, argp, sizeof(dirty))) 2099 goto out; 2100 vcpu_load(vcpu); 2101 r = kvm_vcpu_ioctl_dirty_tlb(vcpu, &dirty); 2102 vcpu_put(vcpu); 2103 break; 2104 } 2105 #endif 2106 default: 2107 r = -EINVAL; 2108 } 2109 2110 out: 2111 return r; 2112 } 2113 2114 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 2115 { 2116 return VM_FAULT_SIGBUS; 2117 } 2118 2119 static int kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo *pvinfo) 2120 { 2121 u32 inst_nop = 0x60000000; 2122 #ifdef CONFIG_KVM_BOOKE_HV 2123 u32 inst_sc1 = 0x44000022; 2124 pvinfo->hcall[0] = cpu_to_be32(inst_sc1); 2125 pvinfo->hcall[1] = cpu_to_be32(inst_nop); 2126 pvinfo->hcall[2] = cpu_to_be32(inst_nop); 2127 pvinfo->hcall[3] = cpu_to_be32(inst_nop); 2128 #else 2129 u32 inst_lis = 0x3c000000; 2130 u32 inst_ori = 0x60000000; 2131 u32 inst_sc = 0x44000002; 2132 u32 inst_imm_mask = 0xffff; 2133 2134 /* 2135 * The hypercall to get into KVM from within guest context is as 2136 * follows: 2137 * 2138 * lis r0, r0, KVM_SC_MAGIC_R0@h 2139 * ori r0, KVM_SC_MAGIC_R0@l 2140 * sc 2141 * nop 2142 */ 2143 pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask)); 2144 pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask)); 2145 pvinfo->hcall[2] = cpu_to_be32(inst_sc); 2146 pvinfo->hcall[3] = cpu_to_be32(inst_nop); 2147 #endif 2148 2149 pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE; 2150 2151 return 0; 2152 } 2153 2154 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) 2155 { 2156 int ret = 0; 2157 2158 #ifdef CONFIG_KVM_MPIC 2159 ret = ret || (kvm->arch.mpic != NULL); 2160 #endif 2161 #ifdef CONFIG_KVM_XICS 2162 ret = ret || (kvm->arch.xics != NULL); 2163 ret = ret || (kvm->arch.xive != NULL); 2164 #endif 2165 smp_rmb(); 2166 return ret; 2167 } 2168 2169 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, 2170 bool line_status) 2171 { 2172 if (!kvm_arch_irqchip_in_kernel(kvm)) 2173 return -ENXIO; 2174 2175 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 2176 irq_event->irq, irq_event->level, 2177 line_status); 2178 return 0; 2179 } 2180 2181 2182 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 2183 struct kvm_enable_cap *cap) 2184 { 2185 int r; 2186 2187 if (cap->flags) 2188 return -EINVAL; 2189 2190 switch (cap->cap) { 2191 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER 2192 case KVM_CAP_PPC_ENABLE_HCALL: { 2193 unsigned long hcall = cap->args[0]; 2194 2195 r = -EINVAL; 2196 if (hcall > MAX_HCALL_OPCODE || (hcall & 3) || 2197 cap->args[1] > 1) 2198 break; 2199 if (!kvmppc_book3s_hcall_implemented(kvm, hcall)) 2200 break; 2201 if (cap->args[1]) 2202 set_bit(hcall / 4, kvm->arch.enabled_hcalls); 2203 else 2204 clear_bit(hcall / 4, kvm->arch.enabled_hcalls); 2205 r = 0; 2206 break; 2207 } 2208 case KVM_CAP_PPC_SMT: { 2209 unsigned long mode = cap->args[0]; 2210 unsigned long flags = cap->args[1]; 2211 2212 r = -EINVAL; 2213 if (kvm->arch.kvm_ops->set_smt_mode) 2214 r = kvm->arch.kvm_ops->set_smt_mode(kvm, mode, flags); 2215 break; 2216 } 2217 2218 case KVM_CAP_PPC_NESTED_HV: 2219 r = -EINVAL; 2220 if (!is_kvmppc_hv_enabled(kvm) || 2221 !kvm->arch.kvm_ops->enable_nested) 2222 break; 2223 r = kvm->arch.kvm_ops->enable_nested(kvm); 2224 break; 2225 #endif 2226 #if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) 2227 case KVM_CAP_PPC_SECURE_GUEST: 2228 r = -EINVAL; 2229 if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_svm) 2230 break; 2231 r = kvm->arch.kvm_ops->enable_svm(kvm); 2232 break; 2233 case KVM_CAP_PPC_DAWR1: 2234 r = -EINVAL; 2235 if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_dawr1) 2236 break; 2237 r = kvm->arch.kvm_ops->enable_dawr1(kvm); 2238 break; 2239 #endif 2240 default: 2241 r = -EINVAL; 2242 break; 2243 } 2244 2245 return r; 2246 } 2247 2248 #ifdef CONFIG_PPC_BOOK3S_64 2249 /* 2250 * These functions check whether the underlying hardware is safe 2251 * against attacks based on observing the effects of speculatively 2252 * executed instructions, and whether it supplies instructions for 2253 * use in workarounds. The information comes from firmware, either 2254 * via the device tree on powernv platforms or from an hcall on 2255 * pseries platforms. 2256 */ 2257 #ifdef CONFIG_PPC_PSERIES 2258 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp) 2259 { 2260 struct h_cpu_char_result c; 2261 unsigned long rc; 2262 2263 if (!machine_is(pseries)) 2264 return -ENOTTY; 2265 2266 rc = plpar_get_cpu_characteristics(&c); 2267 if (rc == H_SUCCESS) { 2268 cp->character = c.character; 2269 cp->behaviour = c.behaviour; 2270 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 | 2271 KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED | 2272 KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 | 2273 KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 | 2274 KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV | 2275 KVM_PPC_CPU_CHAR_BR_HINT_HONOURED | 2276 KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF | 2277 KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS | 2278 KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; 2279 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY | 2280 KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR | 2281 KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR | 2282 KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; 2283 } 2284 return 0; 2285 } 2286 #else 2287 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp) 2288 { 2289 return -ENOTTY; 2290 } 2291 #endif 2292 2293 static inline bool have_fw_feat(struct device_node *fw_features, 2294 const char *state, const char *name) 2295 { 2296 struct device_node *np; 2297 bool r = false; 2298 2299 np = of_get_child_by_name(fw_features, name); 2300 if (np) { 2301 r = of_property_read_bool(np, state); 2302 of_node_put(np); 2303 } 2304 return r; 2305 } 2306 2307 static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp) 2308 { 2309 struct device_node *np, *fw_features; 2310 int r; 2311 2312 memset(cp, 0, sizeof(*cp)); 2313 r = pseries_get_cpu_char(cp); 2314 if (r != -ENOTTY) 2315 return r; 2316 2317 np = of_find_node_by_name(NULL, "ibm,opal"); 2318 if (np) { 2319 fw_features = of_get_child_by_name(np, "fw-features"); 2320 of_node_put(np); 2321 if (!fw_features) 2322 return 0; 2323 if (have_fw_feat(fw_features, "enabled", 2324 "inst-spec-barrier-ori31,31,0")) 2325 cp->character |= KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31; 2326 if (have_fw_feat(fw_features, "enabled", 2327 "fw-bcctrl-serialized")) 2328 cp->character |= KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED; 2329 if (have_fw_feat(fw_features, "enabled", 2330 "inst-l1d-flush-ori30,30,0")) 2331 cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30; 2332 if (have_fw_feat(fw_features, "enabled", 2333 "inst-l1d-flush-trig2")) 2334 cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2; 2335 if (have_fw_feat(fw_features, "enabled", 2336 "fw-l1d-thread-split")) 2337 cp->character |= KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV; 2338 if (have_fw_feat(fw_features, "enabled", 2339 "fw-count-cache-disabled")) 2340 cp->character |= KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS; 2341 if (have_fw_feat(fw_features, "enabled", 2342 "fw-count-cache-flush-bcctr2,0,0")) 2343 cp->character |= KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; 2344 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 | 2345 KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED | 2346 KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 | 2347 KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 | 2348 KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV | 2349 KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS | 2350 KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; 2351 2352 if (have_fw_feat(fw_features, "enabled", 2353 "speculation-policy-favor-security")) 2354 cp->behaviour |= KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY; 2355 if (!have_fw_feat(fw_features, "disabled", 2356 "needs-l1d-flush-msr-pr-0-to-1")) 2357 cp->behaviour |= KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR; 2358 if (!have_fw_feat(fw_features, "disabled", 2359 "needs-spec-barrier-for-bound-checks")) 2360 cp->behaviour |= KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR; 2361 if (have_fw_feat(fw_features, "enabled", 2362 "needs-count-cache-flush-on-context-switch")) 2363 cp->behaviour |= KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; 2364 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY | 2365 KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR | 2366 KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR | 2367 KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; 2368 2369 of_node_put(fw_features); 2370 } 2371 2372 return 0; 2373 } 2374 #endif 2375 2376 long kvm_arch_vm_ioctl(struct file *filp, 2377 unsigned int ioctl, unsigned long arg) 2378 { 2379 struct kvm *kvm __maybe_unused = filp->private_data; 2380 void __user *argp = (void __user *)arg; 2381 long r; 2382 2383 switch (ioctl) { 2384 case KVM_PPC_GET_PVINFO: { 2385 struct kvm_ppc_pvinfo pvinfo; 2386 memset(&pvinfo, 0, sizeof(pvinfo)); 2387 r = kvm_vm_ioctl_get_pvinfo(&pvinfo); 2388 if (copy_to_user(argp, &pvinfo, sizeof(pvinfo))) { 2389 r = -EFAULT; 2390 goto out; 2391 } 2392 2393 break; 2394 } 2395 #ifdef CONFIG_SPAPR_TCE_IOMMU 2396 case KVM_CREATE_SPAPR_TCE_64: { 2397 struct kvm_create_spapr_tce_64 create_tce_64; 2398 2399 r = -EFAULT; 2400 if (copy_from_user(&create_tce_64, argp, sizeof(create_tce_64))) 2401 goto out; 2402 if (create_tce_64.flags) { 2403 r = -EINVAL; 2404 goto out; 2405 } 2406 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64); 2407 goto out; 2408 } 2409 case KVM_CREATE_SPAPR_TCE: { 2410 struct kvm_create_spapr_tce create_tce; 2411 struct kvm_create_spapr_tce_64 create_tce_64; 2412 2413 r = -EFAULT; 2414 if (copy_from_user(&create_tce, argp, sizeof(create_tce))) 2415 goto out; 2416 2417 create_tce_64.liobn = create_tce.liobn; 2418 create_tce_64.page_shift = IOMMU_PAGE_SHIFT_4K; 2419 create_tce_64.offset = 0; 2420 create_tce_64.size = create_tce.window_size >> 2421 IOMMU_PAGE_SHIFT_4K; 2422 create_tce_64.flags = 0; 2423 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64); 2424 goto out; 2425 } 2426 #endif 2427 #ifdef CONFIG_PPC_BOOK3S_64 2428 case KVM_PPC_GET_SMMU_INFO: { 2429 struct kvm_ppc_smmu_info info; 2430 struct kvm *kvm = filp->private_data; 2431 2432 memset(&info, 0, sizeof(info)); 2433 r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info); 2434 if (r >= 0 && copy_to_user(argp, &info, sizeof(info))) 2435 r = -EFAULT; 2436 break; 2437 } 2438 case KVM_PPC_RTAS_DEFINE_TOKEN: { 2439 struct kvm *kvm = filp->private_data; 2440 2441 r = kvm_vm_ioctl_rtas_define_token(kvm, argp); 2442 break; 2443 } 2444 case KVM_PPC_CONFIGURE_V3_MMU: { 2445 struct kvm *kvm = filp->private_data; 2446 struct kvm_ppc_mmuv3_cfg cfg; 2447 2448 r = -EINVAL; 2449 if (!kvm->arch.kvm_ops->configure_mmu) 2450 goto out; 2451 r = -EFAULT; 2452 if (copy_from_user(&cfg, argp, sizeof(cfg))) 2453 goto out; 2454 r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg); 2455 break; 2456 } 2457 case KVM_PPC_GET_RMMU_INFO: { 2458 struct kvm *kvm = filp->private_data; 2459 struct kvm_ppc_rmmu_info info; 2460 2461 r = -EINVAL; 2462 if (!kvm->arch.kvm_ops->get_rmmu_info) 2463 goto out; 2464 r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info); 2465 if (r >= 0 && copy_to_user(argp, &info, sizeof(info))) 2466 r = -EFAULT; 2467 break; 2468 } 2469 case KVM_PPC_GET_CPU_CHAR: { 2470 struct kvm_ppc_cpu_char cpuchar; 2471 2472 r = kvmppc_get_cpu_char(&cpuchar); 2473 if (r >= 0 && copy_to_user(argp, &cpuchar, sizeof(cpuchar))) 2474 r = -EFAULT; 2475 break; 2476 } 2477 case KVM_PPC_SVM_OFF: { 2478 struct kvm *kvm = filp->private_data; 2479 2480 r = 0; 2481 if (!kvm->arch.kvm_ops->svm_off) 2482 goto out; 2483 2484 r = kvm->arch.kvm_ops->svm_off(kvm); 2485 break; 2486 } 2487 default: { 2488 struct kvm *kvm = filp->private_data; 2489 r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg); 2490 } 2491 #else /* CONFIG_PPC_BOOK3S_64 */ 2492 default: 2493 r = -ENOTTY; 2494 #endif 2495 } 2496 out: 2497 return r; 2498 } 2499 2500 static DEFINE_IDA(lpid_inuse); 2501 static unsigned long nr_lpids; 2502 2503 long kvmppc_alloc_lpid(void) 2504 { 2505 int lpid; 2506 2507 /* The host LPID must always be 0 (allocation starts at 1) */ 2508 lpid = ida_alloc_range(&lpid_inuse, 1, nr_lpids - 1, GFP_KERNEL); 2509 if (lpid < 0) { 2510 if (lpid == -ENOMEM) 2511 pr_err("%s: Out of memory\n", __func__); 2512 else 2513 pr_err("%s: No LPIDs free\n", __func__); 2514 return -ENOMEM; 2515 } 2516 2517 return lpid; 2518 } 2519 EXPORT_SYMBOL_GPL(kvmppc_alloc_lpid); 2520 2521 void kvmppc_free_lpid(long lpid) 2522 { 2523 ida_free(&lpid_inuse, lpid); 2524 } 2525 EXPORT_SYMBOL_GPL(kvmppc_free_lpid); 2526 2527 /* nr_lpids_param includes the host LPID */ 2528 void kvmppc_init_lpid(unsigned long nr_lpids_param) 2529 { 2530 nr_lpids = nr_lpids_param; 2531 } 2532 EXPORT_SYMBOL_GPL(kvmppc_init_lpid); 2533 2534 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ppc_instr); 2535 2536 void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry) 2537 { 2538 if (vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs) 2539 vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs(vcpu, debugfs_dentry); 2540 } 2541 2542 int kvm_arch_create_vm_debugfs(struct kvm *kvm) 2543 { 2544 if (kvm->arch.kvm_ops->create_vm_debugfs) 2545 kvm->arch.kvm_ops->create_vm_debugfs(kvm); 2546 return 0; 2547 } 2548