1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include "iodev.h" 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/ioctl.h> 56 #include <asm/uaccess.h> 57 #include <asm/pgtable.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 MODULE_AUTHOR("Qumranet"); 67 MODULE_LICENSE("GPL"); 68 69 /* 70 * Ordering of locks: 71 * 72 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 73 */ 74 75 DEFINE_SPINLOCK(kvm_lock); 76 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 77 LIST_HEAD(vm_list); 78 79 static cpumask_var_t cpus_hardware_enabled; 80 static int kvm_usage_count = 0; 81 static atomic_t hardware_enable_failed; 82 83 struct kmem_cache *kvm_vcpu_cache; 84 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 85 86 static __read_mostly struct preempt_ops kvm_preempt_ops; 87 88 struct dentry *kvm_debugfs_dir; 89 90 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 91 unsigned long arg); 92 #ifdef CONFIG_COMPAT 93 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 94 unsigned long arg); 95 #endif 96 static int hardware_enable_all(void); 97 static void hardware_disable_all(void); 98 99 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 100 101 static void kvm_release_pfn_dirty(pfn_t pfn); 102 static void mark_page_dirty_in_slot(struct kvm *kvm, 103 struct kvm_memory_slot *memslot, gfn_t gfn); 104 105 __visible bool kvm_rebooting; 106 EXPORT_SYMBOL_GPL(kvm_rebooting); 107 108 static bool largepages_enabled = true; 109 110 bool kvm_is_mmio_pfn(pfn_t pfn) 111 { 112 if (pfn_valid(pfn)) 113 return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn)); 114 115 return true; 116 } 117 118 /* 119 * Switches to specified vcpu, until a matching vcpu_put() 120 */ 121 int vcpu_load(struct kvm_vcpu *vcpu) 122 { 123 int cpu; 124 125 if (mutex_lock_killable(&vcpu->mutex)) 126 return -EINTR; 127 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 128 /* The thread running this VCPU changed. */ 129 struct pid *oldpid = vcpu->pid; 130 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 131 rcu_assign_pointer(vcpu->pid, newpid); 132 if (oldpid) 133 synchronize_rcu(); 134 put_pid(oldpid); 135 } 136 cpu = get_cpu(); 137 preempt_notifier_register(&vcpu->preempt_notifier); 138 kvm_arch_vcpu_load(vcpu, cpu); 139 put_cpu(); 140 return 0; 141 } 142 143 void vcpu_put(struct kvm_vcpu *vcpu) 144 { 145 preempt_disable(); 146 kvm_arch_vcpu_put(vcpu); 147 preempt_notifier_unregister(&vcpu->preempt_notifier); 148 preempt_enable(); 149 mutex_unlock(&vcpu->mutex); 150 } 151 152 static void ack_flush(void *_completed) 153 { 154 } 155 156 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 157 { 158 int i, cpu, me; 159 cpumask_var_t cpus; 160 bool called = true; 161 struct kvm_vcpu *vcpu; 162 163 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 164 165 me = get_cpu(); 166 kvm_for_each_vcpu(i, vcpu, kvm) { 167 kvm_make_request(req, vcpu); 168 cpu = vcpu->cpu; 169 170 /* Set ->requests bit before we read ->mode */ 171 smp_mb(); 172 173 if (cpus != NULL && cpu != -1 && cpu != me && 174 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 175 cpumask_set_cpu(cpu, cpus); 176 } 177 if (unlikely(cpus == NULL)) 178 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 179 else if (!cpumask_empty(cpus)) 180 smp_call_function_many(cpus, ack_flush, NULL, 1); 181 else 182 called = false; 183 put_cpu(); 184 free_cpumask_var(cpus); 185 return called; 186 } 187 188 void kvm_flush_remote_tlbs(struct kvm *kvm) 189 { 190 long dirty_count = kvm->tlbs_dirty; 191 192 smp_mb(); 193 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 194 ++kvm->stat.remote_tlb_flush; 195 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 196 } 197 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 198 199 void kvm_reload_remote_mmus(struct kvm *kvm) 200 { 201 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 202 } 203 204 void kvm_make_mclock_inprogress_request(struct kvm *kvm) 205 { 206 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 207 } 208 209 void kvm_make_scan_ioapic_request(struct kvm *kvm) 210 { 211 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 212 } 213 214 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 215 { 216 struct page *page; 217 int r; 218 219 mutex_init(&vcpu->mutex); 220 vcpu->cpu = -1; 221 vcpu->kvm = kvm; 222 vcpu->vcpu_id = id; 223 vcpu->pid = NULL; 224 init_waitqueue_head(&vcpu->wq); 225 kvm_async_pf_vcpu_init(vcpu); 226 227 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 228 if (!page) { 229 r = -ENOMEM; 230 goto fail; 231 } 232 vcpu->run = page_address(page); 233 234 kvm_vcpu_set_in_spin_loop(vcpu, false); 235 kvm_vcpu_set_dy_eligible(vcpu, false); 236 vcpu->preempted = false; 237 238 r = kvm_arch_vcpu_init(vcpu); 239 if (r < 0) 240 goto fail_free_run; 241 return 0; 242 243 fail_free_run: 244 free_page((unsigned long)vcpu->run); 245 fail: 246 return r; 247 } 248 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 249 250 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 251 { 252 put_pid(vcpu->pid); 253 kvm_arch_vcpu_uninit(vcpu); 254 free_page((unsigned long)vcpu->run); 255 } 256 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 257 258 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 259 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 260 { 261 return container_of(mn, struct kvm, mmu_notifier); 262 } 263 264 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 265 struct mm_struct *mm, 266 unsigned long address) 267 { 268 struct kvm *kvm = mmu_notifier_to_kvm(mn); 269 int need_tlb_flush, idx; 270 271 /* 272 * When ->invalidate_page runs, the linux pte has been zapped 273 * already but the page is still allocated until 274 * ->invalidate_page returns. So if we increase the sequence 275 * here the kvm page fault will notice if the spte can't be 276 * established because the page is going to be freed. If 277 * instead the kvm page fault establishes the spte before 278 * ->invalidate_page runs, kvm_unmap_hva will release it 279 * before returning. 280 * 281 * The sequence increase only need to be seen at spin_unlock 282 * time, and not at spin_lock time. 283 * 284 * Increasing the sequence after the spin_unlock would be 285 * unsafe because the kvm page fault could then establish the 286 * pte after kvm_unmap_hva returned, without noticing the page 287 * is going to be freed. 288 */ 289 idx = srcu_read_lock(&kvm->srcu); 290 spin_lock(&kvm->mmu_lock); 291 292 kvm->mmu_notifier_seq++; 293 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 294 /* we've to flush the tlb before the pages can be freed */ 295 if (need_tlb_flush) 296 kvm_flush_remote_tlbs(kvm); 297 298 spin_unlock(&kvm->mmu_lock); 299 300 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 301 302 srcu_read_unlock(&kvm->srcu, idx); 303 } 304 305 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 306 struct mm_struct *mm, 307 unsigned long address, 308 pte_t pte) 309 { 310 struct kvm *kvm = mmu_notifier_to_kvm(mn); 311 int idx; 312 313 idx = srcu_read_lock(&kvm->srcu); 314 spin_lock(&kvm->mmu_lock); 315 kvm->mmu_notifier_seq++; 316 kvm_set_spte_hva(kvm, address, pte); 317 spin_unlock(&kvm->mmu_lock); 318 srcu_read_unlock(&kvm->srcu, idx); 319 } 320 321 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 322 struct mm_struct *mm, 323 unsigned long start, 324 unsigned long end) 325 { 326 struct kvm *kvm = mmu_notifier_to_kvm(mn); 327 int need_tlb_flush = 0, idx; 328 329 idx = srcu_read_lock(&kvm->srcu); 330 spin_lock(&kvm->mmu_lock); 331 /* 332 * The count increase must become visible at unlock time as no 333 * spte can be established without taking the mmu_lock and 334 * count is also read inside the mmu_lock critical section. 335 */ 336 kvm->mmu_notifier_count++; 337 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 338 need_tlb_flush |= kvm->tlbs_dirty; 339 /* we've to flush the tlb before the pages can be freed */ 340 if (need_tlb_flush) 341 kvm_flush_remote_tlbs(kvm); 342 343 spin_unlock(&kvm->mmu_lock); 344 srcu_read_unlock(&kvm->srcu, idx); 345 } 346 347 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 348 struct mm_struct *mm, 349 unsigned long start, 350 unsigned long end) 351 { 352 struct kvm *kvm = mmu_notifier_to_kvm(mn); 353 354 spin_lock(&kvm->mmu_lock); 355 /* 356 * This sequence increase will notify the kvm page fault that 357 * the page that is going to be mapped in the spte could have 358 * been freed. 359 */ 360 kvm->mmu_notifier_seq++; 361 smp_wmb(); 362 /* 363 * The above sequence increase must be visible before the 364 * below count decrease, which is ensured by the smp_wmb above 365 * in conjunction with the smp_rmb in mmu_notifier_retry(). 366 */ 367 kvm->mmu_notifier_count--; 368 spin_unlock(&kvm->mmu_lock); 369 370 BUG_ON(kvm->mmu_notifier_count < 0); 371 } 372 373 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 374 struct mm_struct *mm, 375 unsigned long start, 376 unsigned long end) 377 { 378 struct kvm *kvm = mmu_notifier_to_kvm(mn); 379 int young, idx; 380 381 idx = srcu_read_lock(&kvm->srcu); 382 spin_lock(&kvm->mmu_lock); 383 384 young = kvm_age_hva(kvm, start, end); 385 if (young) 386 kvm_flush_remote_tlbs(kvm); 387 388 spin_unlock(&kvm->mmu_lock); 389 srcu_read_unlock(&kvm->srcu, idx); 390 391 return young; 392 } 393 394 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 395 struct mm_struct *mm, 396 unsigned long address) 397 { 398 struct kvm *kvm = mmu_notifier_to_kvm(mn); 399 int young, idx; 400 401 idx = srcu_read_lock(&kvm->srcu); 402 spin_lock(&kvm->mmu_lock); 403 young = kvm_test_age_hva(kvm, address); 404 spin_unlock(&kvm->mmu_lock); 405 srcu_read_unlock(&kvm->srcu, idx); 406 407 return young; 408 } 409 410 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 411 struct mm_struct *mm) 412 { 413 struct kvm *kvm = mmu_notifier_to_kvm(mn); 414 int idx; 415 416 idx = srcu_read_lock(&kvm->srcu); 417 kvm_arch_flush_shadow_all(kvm); 418 srcu_read_unlock(&kvm->srcu, idx); 419 } 420 421 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 422 .invalidate_page = kvm_mmu_notifier_invalidate_page, 423 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 424 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 425 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 426 .test_young = kvm_mmu_notifier_test_young, 427 .change_pte = kvm_mmu_notifier_change_pte, 428 .release = kvm_mmu_notifier_release, 429 }; 430 431 static int kvm_init_mmu_notifier(struct kvm *kvm) 432 { 433 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 434 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 435 } 436 437 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 438 439 static int kvm_init_mmu_notifier(struct kvm *kvm) 440 { 441 return 0; 442 } 443 444 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 445 446 static void kvm_init_memslots_id(struct kvm *kvm) 447 { 448 int i; 449 struct kvm_memslots *slots = kvm->memslots; 450 451 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 452 slots->id_to_index[i] = slots->memslots[i].id = i; 453 } 454 455 static struct kvm *kvm_create_vm(unsigned long type) 456 { 457 int r, i; 458 struct kvm *kvm = kvm_arch_alloc_vm(); 459 460 if (!kvm) 461 return ERR_PTR(-ENOMEM); 462 463 r = kvm_arch_init_vm(kvm, type); 464 if (r) 465 goto out_err_no_disable; 466 467 r = hardware_enable_all(); 468 if (r) 469 goto out_err_no_disable; 470 471 #ifdef CONFIG_HAVE_KVM_IRQCHIP 472 INIT_HLIST_HEAD(&kvm->mask_notifier_list); 473 #endif 474 #ifdef CONFIG_HAVE_KVM_IRQFD 475 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 476 #endif 477 478 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 479 480 r = -ENOMEM; 481 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 482 if (!kvm->memslots) 483 goto out_err_no_srcu; 484 485 /* 486 * Init kvm generation close to the maximum to easily test the 487 * code of handling generation number wrap-around. 488 */ 489 kvm->memslots->generation = -150; 490 491 kvm_init_memslots_id(kvm); 492 if (init_srcu_struct(&kvm->srcu)) 493 goto out_err_no_srcu; 494 if (init_srcu_struct(&kvm->irq_srcu)) 495 goto out_err_no_irq_srcu; 496 for (i = 0; i < KVM_NR_BUSES; i++) { 497 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 498 GFP_KERNEL); 499 if (!kvm->buses[i]) 500 goto out_err; 501 } 502 503 spin_lock_init(&kvm->mmu_lock); 504 kvm->mm = current->mm; 505 atomic_inc(&kvm->mm->mm_count); 506 kvm_eventfd_init(kvm); 507 mutex_init(&kvm->lock); 508 mutex_init(&kvm->irq_lock); 509 mutex_init(&kvm->slots_lock); 510 atomic_set(&kvm->users_count, 1); 511 INIT_LIST_HEAD(&kvm->devices); 512 513 r = kvm_init_mmu_notifier(kvm); 514 if (r) 515 goto out_err; 516 517 spin_lock(&kvm_lock); 518 list_add(&kvm->vm_list, &vm_list); 519 spin_unlock(&kvm_lock); 520 521 return kvm; 522 523 out_err: 524 cleanup_srcu_struct(&kvm->irq_srcu); 525 out_err_no_irq_srcu: 526 cleanup_srcu_struct(&kvm->srcu); 527 out_err_no_srcu: 528 hardware_disable_all(); 529 out_err_no_disable: 530 for (i = 0; i < KVM_NR_BUSES; i++) 531 kfree(kvm->buses[i]); 532 kfree(kvm->memslots); 533 kvm_arch_free_vm(kvm); 534 return ERR_PTR(r); 535 } 536 537 /* 538 * Avoid using vmalloc for a small buffer. 539 * Should not be used when the size is statically known. 540 */ 541 void *kvm_kvzalloc(unsigned long size) 542 { 543 if (size > PAGE_SIZE) 544 return vzalloc(size); 545 else 546 return kzalloc(size, GFP_KERNEL); 547 } 548 549 void kvm_kvfree(const void *addr) 550 { 551 if (is_vmalloc_addr(addr)) 552 vfree(addr); 553 else 554 kfree(addr); 555 } 556 557 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 558 { 559 if (!memslot->dirty_bitmap) 560 return; 561 562 kvm_kvfree(memslot->dirty_bitmap); 563 memslot->dirty_bitmap = NULL; 564 } 565 566 /* 567 * Free any memory in @free but not in @dont. 568 */ 569 static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free, 570 struct kvm_memory_slot *dont) 571 { 572 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 573 kvm_destroy_dirty_bitmap(free); 574 575 kvm_arch_free_memslot(kvm, free, dont); 576 577 free->npages = 0; 578 } 579 580 static void kvm_free_physmem(struct kvm *kvm) 581 { 582 struct kvm_memslots *slots = kvm->memslots; 583 struct kvm_memory_slot *memslot; 584 585 kvm_for_each_memslot(memslot, slots) 586 kvm_free_physmem_slot(kvm, memslot, NULL); 587 588 kfree(kvm->memslots); 589 } 590 591 static void kvm_destroy_devices(struct kvm *kvm) 592 { 593 struct list_head *node, *tmp; 594 595 list_for_each_safe(node, tmp, &kvm->devices) { 596 struct kvm_device *dev = 597 list_entry(node, struct kvm_device, vm_node); 598 599 list_del(node); 600 dev->ops->destroy(dev); 601 } 602 } 603 604 static void kvm_destroy_vm(struct kvm *kvm) 605 { 606 int i; 607 struct mm_struct *mm = kvm->mm; 608 609 kvm_arch_sync_events(kvm); 610 spin_lock(&kvm_lock); 611 list_del(&kvm->vm_list); 612 spin_unlock(&kvm_lock); 613 kvm_free_irq_routing(kvm); 614 for (i = 0; i < KVM_NR_BUSES; i++) 615 kvm_io_bus_destroy(kvm->buses[i]); 616 kvm_coalesced_mmio_free(kvm); 617 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 618 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 619 #else 620 kvm_arch_flush_shadow_all(kvm); 621 #endif 622 kvm_arch_destroy_vm(kvm); 623 kvm_destroy_devices(kvm); 624 kvm_free_physmem(kvm); 625 cleanup_srcu_struct(&kvm->irq_srcu); 626 cleanup_srcu_struct(&kvm->srcu); 627 kvm_arch_free_vm(kvm); 628 hardware_disable_all(); 629 mmdrop(mm); 630 } 631 632 void kvm_get_kvm(struct kvm *kvm) 633 { 634 atomic_inc(&kvm->users_count); 635 } 636 EXPORT_SYMBOL_GPL(kvm_get_kvm); 637 638 void kvm_put_kvm(struct kvm *kvm) 639 { 640 if (atomic_dec_and_test(&kvm->users_count)) 641 kvm_destroy_vm(kvm); 642 } 643 EXPORT_SYMBOL_GPL(kvm_put_kvm); 644 645 646 static int kvm_vm_release(struct inode *inode, struct file *filp) 647 { 648 struct kvm *kvm = filp->private_data; 649 650 kvm_irqfd_release(kvm); 651 652 kvm_put_kvm(kvm); 653 return 0; 654 } 655 656 /* 657 * Allocation size is twice as large as the actual dirty bitmap size. 658 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 659 */ 660 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 661 { 662 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 663 664 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 665 if (!memslot->dirty_bitmap) 666 return -ENOMEM; 667 668 return 0; 669 } 670 671 static int cmp_memslot(const void *slot1, const void *slot2) 672 { 673 struct kvm_memory_slot *s1, *s2; 674 675 s1 = (struct kvm_memory_slot *)slot1; 676 s2 = (struct kvm_memory_slot *)slot2; 677 678 if (s1->npages < s2->npages) 679 return 1; 680 if (s1->npages > s2->npages) 681 return -1; 682 683 return 0; 684 } 685 686 /* 687 * Sort the memslots base on its size, so the larger slots 688 * will get better fit. 689 */ 690 static void sort_memslots(struct kvm_memslots *slots) 691 { 692 int i; 693 694 sort(slots->memslots, KVM_MEM_SLOTS_NUM, 695 sizeof(struct kvm_memory_slot), cmp_memslot, NULL); 696 697 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 698 slots->id_to_index[slots->memslots[i].id] = i; 699 } 700 701 static void update_memslots(struct kvm_memslots *slots, 702 struct kvm_memory_slot *new) 703 { 704 if (new) { 705 int id = new->id; 706 struct kvm_memory_slot *old = id_to_memslot(slots, id); 707 unsigned long npages = old->npages; 708 709 *old = *new; 710 if (new->npages != npages) 711 sort_memslots(slots); 712 } 713 } 714 715 static int check_memory_region_flags(struct kvm_userspace_memory_region *mem) 716 { 717 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 718 719 #ifdef __KVM_HAVE_READONLY_MEM 720 valid_flags |= KVM_MEM_READONLY; 721 #endif 722 723 if (mem->flags & ~valid_flags) 724 return -EINVAL; 725 726 return 0; 727 } 728 729 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 730 struct kvm_memslots *slots, struct kvm_memory_slot *new) 731 { 732 struct kvm_memslots *old_memslots = kvm->memslots; 733 734 /* 735 * Set the low bit in the generation, which disables SPTE caching 736 * until the end of synchronize_srcu_expedited. 737 */ 738 WARN_ON(old_memslots->generation & 1); 739 slots->generation = old_memslots->generation + 1; 740 741 update_memslots(slots, new); 742 rcu_assign_pointer(kvm->memslots, slots); 743 synchronize_srcu_expedited(&kvm->srcu); 744 745 /* 746 * Increment the new memslot generation a second time. This prevents 747 * vm exits that race with memslot updates from caching a memslot 748 * generation that will (potentially) be valid forever. 749 */ 750 slots->generation++; 751 752 kvm_arch_memslots_updated(kvm); 753 754 return old_memslots; 755 } 756 757 /* 758 * Allocate some memory and give it an address in the guest physical address 759 * space. 760 * 761 * Discontiguous memory is allowed, mostly for framebuffers. 762 * 763 * Must be called holding mmap_sem for write. 764 */ 765 int __kvm_set_memory_region(struct kvm *kvm, 766 struct kvm_userspace_memory_region *mem) 767 { 768 int r; 769 gfn_t base_gfn; 770 unsigned long npages; 771 struct kvm_memory_slot *slot; 772 struct kvm_memory_slot old, new; 773 struct kvm_memslots *slots = NULL, *old_memslots; 774 enum kvm_mr_change change; 775 776 r = check_memory_region_flags(mem); 777 if (r) 778 goto out; 779 780 r = -EINVAL; 781 /* General sanity checks */ 782 if (mem->memory_size & (PAGE_SIZE - 1)) 783 goto out; 784 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 785 goto out; 786 /* We can read the guest memory with __xxx_user() later on. */ 787 if ((mem->slot < KVM_USER_MEM_SLOTS) && 788 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 789 !access_ok(VERIFY_WRITE, 790 (void __user *)(unsigned long)mem->userspace_addr, 791 mem->memory_size))) 792 goto out; 793 if (mem->slot >= KVM_MEM_SLOTS_NUM) 794 goto out; 795 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 796 goto out; 797 798 slot = id_to_memslot(kvm->memslots, mem->slot); 799 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 800 npages = mem->memory_size >> PAGE_SHIFT; 801 802 if (npages > KVM_MEM_MAX_NR_PAGES) 803 goto out; 804 805 if (!npages) 806 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; 807 808 new = old = *slot; 809 810 new.id = mem->slot; 811 new.base_gfn = base_gfn; 812 new.npages = npages; 813 new.flags = mem->flags; 814 815 if (npages) { 816 if (!old.npages) 817 change = KVM_MR_CREATE; 818 else { /* Modify an existing slot. */ 819 if ((mem->userspace_addr != old.userspace_addr) || 820 (npages != old.npages) || 821 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 822 goto out; 823 824 if (base_gfn != old.base_gfn) 825 change = KVM_MR_MOVE; 826 else if (new.flags != old.flags) 827 change = KVM_MR_FLAGS_ONLY; 828 else { /* Nothing to change. */ 829 r = 0; 830 goto out; 831 } 832 } 833 } else if (old.npages) { 834 change = KVM_MR_DELETE; 835 } else /* Modify a non-existent slot: disallowed. */ 836 goto out; 837 838 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 839 /* Check for overlaps */ 840 r = -EEXIST; 841 kvm_for_each_memslot(slot, kvm->memslots) { 842 if ((slot->id >= KVM_USER_MEM_SLOTS) || 843 (slot->id == mem->slot)) 844 continue; 845 if (!((base_gfn + npages <= slot->base_gfn) || 846 (base_gfn >= slot->base_gfn + slot->npages))) 847 goto out; 848 } 849 } 850 851 /* Free page dirty bitmap if unneeded */ 852 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 853 new.dirty_bitmap = NULL; 854 855 r = -ENOMEM; 856 if (change == KVM_MR_CREATE) { 857 new.userspace_addr = mem->userspace_addr; 858 859 if (kvm_arch_create_memslot(kvm, &new, npages)) 860 goto out_free; 861 } 862 863 /* Allocate page dirty bitmap if needed */ 864 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 865 if (kvm_create_dirty_bitmap(&new) < 0) 866 goto out_free; 867 } 868 869 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 870 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 871 GFP_KERNEL); 872 if (!slots) 873 goto out_free; 874 slot = id_to_memslot(slots, mem->slot); 875 slot->flags |= KVM_MEMSLOT_INVALID; 876 877 old_memslots = install_new_memslots(kvm, slots, NULL); 878 879 /* slot was deleted or moved, clear iommu mapping */ 880 kvm_iommu_unmap_pages(kvm, &old); 881 /* From this point no new shadow pages pointing to a deleted, 882 * or moved, memslot will be created. 883 * 884 * validation of sp->gfn happens in: 885 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 886 * - kvm_is_visible_gfn (mmu_check_roots) 887 */ 888 kvm_arch_flush_shadow_memslot(kvm, slot); 889 slots = old_memslots; 890 } 891 892 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 893 if (r) 894 goto out_slots; 895 896 r = -ENOMEM; 897 /* 898 * We can re-use the old_memslots from above, the only difference 899 * from the currently installed memslots is the invalid flag. This 900 * will get overwritten by update_memslots anyway. 901 */ 902 if (!slots) { 903 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 904 GFP_KERNEL); 905 if (!slots) 906 goto out_free; 907 } 908 909 /* actual memory is freed via old in kvm_free_physmem_slot below */ 910 if (change == KVM_MR_DELETE) { 911 new.dirty_bitmap = NULL; 912 memset(&new.arch, 0, sizeof(new.arch)); 913 } 914 915 old_memslots = install_new_memslots(kvm, slots, &new); 916 917 kvm_arch_commit_memory_region(kvm, mem, &old, change); 918 919 kvm_free_physmem_slot(kvm, &old, &new); 920 kfree(old_memslots); 921 922 /* 923 * IOMMU mapping: New slots need to be mapped. Old slots need to be 924 * un-mapped and re-mapped if their base changes. Since base change 925 * unmapping is handled above with slot deletion, mapping alone is 926 * needed here. Anything else the iommu might care about for existing 927 * slots (size changes, userspace addr changes and read-only flag 928 * changes) is disallowed above, so any other attribute changes getting 929 * here can be skipped. 930 */ 931 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 932 r = kvm_iommu_map_pages(kvm, &new); 933 return r; 934 } 935 936 return 0; 937 938 out_slots: 939 kfree(slots); 940 out_free: 941 kvm_free_physmem_slot(kvm, &new, &old); 942 out: 943 return r; 944 } 945 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 946 947 int kvm_set_memory_region(struct kvm *kvm, 948 struct kvm_userspace_memory_region *mem) 949 { 950 int r; 951 952 mutex_lock(&kvm->slots_lock); 953 r = __kvm_set_memory_region(kvm, mem); 954 mutex_unlock(&kvm->slots_lock); 955 return r; 956 } 957 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 958 959 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 960 struct kvm_userspace_memory_region *mem) 961 { 962 if (mem->slot >= KVM_USER_MEM_SLOTS) 963 return -EINVAL; 964 return kvm_set_memory_region(kvm, mem); 965 } 966 967 int kvm_get_dirty_log(struct kvm *kvm, 968 struct kvm_dirty_log *log, int *is_dirty) 969 { 970 struct kvm_memory_slot *memslot; 971 int r, i; 972 unsigned long n; 973 unsigned long any = 0; 974 975 r = -EINVAL; 976 if (log->slot >= KVM_USER_MEM_SLOTS) 977 goto out; 978 979 memslot = id_to_memslot(kvm->memslots, log->slot); 980 r = -ENOENT; 981 if (!memslot->dirty_bitmap) 982 goto out; 983 984 n = kvm_dirty_bitmap_bytes(memslot); 985 986 for (i = 0; !any && i < n/sizeof(long); ++i) 987 any = memslot->dirty_bitmap[i]; 988 989 r = -EFAULT; 990 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 991 goto out; 992 993 if (any) 994 *is_dirty = 1; 995 996 r = 0; 997 out: 998 return r; 999 } 1000 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1001 1002 bool kvm_largepages_enabled(void) 1003 { 1004 return largepages_enabled; 1005 } 1006 1007 void kvm_disable_largepages(void) 1008 { 1009 largepages_enabled = false; 1010 } 1011 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1012 1013 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1014 { 1015 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1016 } 1017 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1018 1019 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1020 { 1021 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1022 1023 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1024 memslot->flags & KVM_MEMSLOT_INVALID) 1025 return 0; 1026 1027 return 1; 1028 } 1029 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1030 1031 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1032 { 1033 struct vm_area_struct *vma; 1034 unsigned long addr, size; 1035 1036 size = PAGE_SIZE; 1037 1038 addr = gfn_to_hva(kvm, gfn); 1039 if (kvm_is_error_hva(addr)) 1040 return PAGE_SIZE; 1041 1042 down_read(¤t->mm->mmap_sem); 1043 vma = find_vma(current->mm, addr); 1044 if (!vma) 1045 goto out; 1046 1047 size = vma_kernel_pagesize(vma); 1048 1049 out: 1050 up_read(¤t->mm->mmap_sem); 1051 1052 return size; 1053 } 1054 1055 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1056 { 1057 return slot->flags & KVM_MEM_READONLY; 1058 } 1059 1060 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1061 gfn_t *nr_pages, bool write) 1062 { 1063 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1064 return KVM_HVA_ERR_BAD; 1065 1066 if (memslot_is_readonly(slot) && write) 1067 return KVM_HVA_ERR_RO_BAD; 1068 1069 if (nr_pages) 1070 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1071 1072 return __gfn_to_hva_memslot(slot, gfn); 1073 } 1074 1075 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1076 gfn_t *nr_pages) 1077 { 1078 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1079 } 1080 1081 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1082 gfn_t gfn) 1083 { 1084 return gfn_to_hva_many(slot, gfn, NULL); 1085 } 1086 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1087 1088 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1089 { 1090 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1091 } 1092 EXPORT_SYMBOL_GPL(gfn_to_hva); 1093 1094 /* 1095 * If writable is set to false, the hva returned by this function is only 1096 * allowed to be read. 1097 */ 1098 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1099 gfn_t gfn, bool *writable) 1100 { 1101 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1102 1103 if (!kvm_is_error_hva(hva) && writable) 1104 *writable = !memslot_is_readonly(slot); 1105 1106 return hva; 1107 } 1108 1109 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1110 { 1111 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1112 1113 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1114 } 1115 1116 static int kvm_read_hva(void *data, void __user *hva, int len) 1117 { 1118 return __copy_from_user(data, hva, len); 1119 } 1120 1121 static int kvm_read_hva_atomic(void *data, void __user *hva, int len) 1122 { 1123 return __copy_from_user_inatomic(data, hva, len); 1124 } 1125 1126 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1127 unsigned long start, int write, struct page **page) 1128 { 1129 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1130 1131 if (write) 1132 flags |= FOLL_WRITE; 1133 1134 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1135 } 1136 1137 int kvm_get_user_page_io(struct task_struct *tsk, struct mm_struct *mm, 1138 unsigned long addr, bool write_fault, 1139 struct page **pagep) 1140 { 1141 int npages; 1142 int locked = 1; 1143 int flags = FOLL_TOUCH | FOLL_HWPOISON | 1144 (pagep ? FOLL_GET : 0) | 1145 (write_fault ? FOLL_WRITE : 0); 1146 1147 /* 1148 * If retrying the fault, we get here *not* having allowed the filemap 1149 * to wait on the page lock. We should now allow waiting on the IO with 1150 * the mmap semaphore released. 1151 */ 1152 down_read(&mm->mmap_sem); 1153 npages = __get_user_pages(tsk, mm, addr, 1, flags, pagep, NULL, 1154 &locked); 1155 if (!locked) { 1156 VM_BUG_ON(npages); 1157 1158 if (!pagep) 1159 return 0; 1160 1161 /* 1162 * The previous call has now waited on the IO. Now we can 1163 * retry and complete. Pass TRIED to ensure we do not re 1164 * schedule async IO (see e.g. filemap_fault). 1165 */ 1166 down_read(&mm->mmap_sem); 1167 npages = __get_user_pages(tsk, mm, addr, 1, flags | FOLL_TRIED, 1168 pagep, NULL, NULL); 1169 } 1170 up_read(&mm->mmap_sem); 1171 return npages; 1172 } 1173 1174 static inline int check_user_page_hwpoison(unsigned long addr) 1175 { 1176 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1177 1178 rc = __get_user_pages(current, current->mm, addr, 1, 1179 flags, NULL, NULL, NULL); 1180 return rc == -EHWPOISON; 1181 } 1182 1183 /* 1184 * The atomic path to get the writable pfn which will be stored in @pfn, 1185 * true indicates success, otherwise false is returned. 1186 */ 1187 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1188 bool write_fault, bool *writable, pfn_t *pfn) 1189 { 1190 struct page *page[1]; 1191 int npages; 1192 1193 if (!(async || atomic)) 1194 return false; 1195 1196 /* 1197 * Fast pin a writable pfn only if it is a write fault request 1198 * or the caller allows to map a writable pfn for a read fault 1199 * request. 1200 */ 1201 if (!(write_fault || writable)) 1202 return false; 1203 1204 npages = __get_user_pages_fast(addr, 1, 1, page); 1205 if (npages == 1) { 1206 *pfn = page_to_pfn(page[0]); 1207 1208 if (writable) 1209 *writable = true; 1210 return true; 1211 } 1212 1213 return false; 1214 } 1215 1216 /* 1217 * The slow path to get the pfn of the specified host virtual address, 1218 * 1 indicates success, -errno is returned if error is detected. 1219 */ 1220 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1221 bool *writable, pfn_t *pfn) 1222 { 1223 struct page *page[1]; 1224 int npages = 0; 1225 1226 might_sleep(); 1227 1228 if (writable) 1229 *writable = write_fault; 1230 1231 if (async) { 1232 down_read(¤t->mm->mmap_sem); 1233 npages = get_user_page_nowait(current, current->mm, 1234 addr, write_fault, page); 1235 up_read(¤t->mm->mmap_sem); 1236 } else { 1237 /* 1238 * By now we have tried gup_fast, and possibly async_pf, and we 1239 * are certainly not atomic. Time to retry the gup, allowing 1240 * mmap semaphore to be relinquished in the case of IO. 1241 */ 1242 npages = kvm_get_user_page_io(current, current->mm, addr, 1243 write_fault, page); 1244 } 1245 if (npages != 1) 1246 return npages; 1247 1248 /* map read fault as writable if possible */ 1249 if (unlikely(!write_fault) && writable) { 1250 struct page *wpage[1]; 1251 1252 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1253 if (npages == 1) { 1254 *writable = true; 1255 put_page(page[0]); 1256 page[0] = wpage[0]; 1257 } 1258 1259 npages = 1; 1260 } 1261 *pfn = page_to_pfn(page[0]); 1262 return npages; 1263 } 1264 1265 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1266 { 1267 if (unlikely(!(vma->vm_flags & VM_READ))) 1268 return false; 1269 1270 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1271 return false; 1272 1273 return true; 1274 } 1275 1276 /* 1277 * Pin guest page in memory and return its pfn. 1278 * @addr: host virtual address which maps memory to the guest 1279 * @atomic: whether this function can sleep 1280 * @async: whether this function need to wait IO complete if the 1281 * host page is not in the memory 1282 * @write_fault: whether we should get a writable host page 1283 * @writable: whether it allows to map a writable host page for !@write_fault 1284 * 1285 * The function will map a writable host page for these two cases: 1286 * 1): @write_fault = true 1287 * 2): @write_fault = false && @writable, @writable will tell the caller 1288 * whether the mapping is writable. 1289 */ 1290 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1291 bool write_fault, bool *writable) 1292 { 1293 struct vm_area_struct *vma; 1294 pfn_t pfn = 0; 1295 int npages; 1296 1297 /* we can do it either atomically or asynchronously, not both */ 1298 BUG_ON(atomic && async); 1299 1300 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1301 return pfn; 1302 1303 if (atomic) 1304 return KVM_PFN_ERR_FAULT; 1305 1306 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1307 if (npages == 1) 1308 return pfn; 1309 1310 down_read(¤t->mm->mmap_sem); 1311 if (npages == -EHWPOISON || 1312 (!async && check_user_page_hwpoison(addr))) { 1313 pfn = KVM_PFN_ERR_HWPOISON; 1314 goto exit; 1315 } 1316 1317 vma = find_vma_intersection(current->mm, addr, addr + 1); 1318 1319 if (vma == NULL) 1320 pfn = KVM_PFN_ERR_FAULT; 1321 else if ((vma->vm_flags & VM_PFNMAP)) { 1322 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1323 vma->vm_pgoff; 1324 BUG_ON(!kvm_is_mmio_pfn(pfn)); 1325 } else { 1326 if (async && vma_is_valid(vma, write_fault)) 1327 *async = true; 1328 pfn = KVM_PFN_ERR_FAULT; 1329 } 1330 exit: 1331 up_read(¤t->mm->mmap_sem); 1332 return pfn; 1333 } 1334 1335 static pfn_t 1336 __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic, 1337 bool *async, bool write_fault, bool *writable) 1338 { 1339 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1340 1341 if (addr == KVM_HVA_ERR_RO_BAD) 1342 return KVM_PFN_ERR_RO_FAULT; 1343 1344 if (kvm_is_error_hva(addr)) 1345 return KVM_PFN_NOSLOT; 1346 1347 /* Do not map writable pfn in the readonly memslot. */ 1348 if (writable && memslot_is_readonly(slot)) { 1349 *writable = false; 1350 writable = NULL; 1351 } 1352 1353 return hva_to_pfn(addr, atomic, async, write_fault, 1354 writable); 1355 } 1356 1357 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async, 1358 bool write_fault, bool *writable) 1359 { 1360 struct kvm_memory_slot *slot; 1361 1362 if (async) 1363 *async = false; 1364 1365 slot = gfn_to_memslot(kvm, gfn); 1366 1367 return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault, 1368 writable); 1369 } 1370 1371 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1372 { 1373 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL); 1374 } 1375 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1376 1377 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async, 1378 bool write_fault, bool *writable) 1379 { 1380 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable); 1381 } 1382 EXPORT_SYMBOL_GPL(gfn_to_pfn_async); 1383 1384 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1385 { 1386 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL); 1387 } 1388 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1389 1390 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1391 bool *writable) 1392 { 1393 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable); 1394 } 1395 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1396 1397 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1398 { 1399 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1400 } 1401 1402 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1403 { 1404 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1405 } 1406 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1407 1408 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages, 1409 int nr_pages) 1410 { 1411 unsigned long addr; 1412 gfn_t entry; 1413 1414 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry); 1415 if (kvm_is_error_hva(addr)) 1416 return -1; 1417 1418 if (entry < nr_pages) 1419 return 0; 1420 1421 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1422 } 1423 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1424 1425 static struct page *kvm_pfn_to_page(pfn_t pfn) 1426 { 1427 if (is_error_noslot_pfn(pfn)) 1428 return KVM_ERR_PTR_BAD_PAGE; 1429 1430 if (kvm_is_mmio_pfn(pfn)) { 1431 WARN_ON(1); 1432 return KVM_ERR_PTR_BAD_PAGE; 1433 } 1434 1435 return pfn_to_page(pfn); 1436 } 1437 1438 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1439 { 1440 pfn_t pfn; 1441 1442 pfn = gfn_to_pfn(kvm, gfn); 1443 1444 return kvm_pfn_to_page(pfn); 1445 } 1446 1447 EXPORT_SYMBOL_GPL(gfn_to_page); 1448 1449 void kvm_release_page_clean(struct page *page) 1450 { 1451 WARN_ON(is_error_page(page)); 1452 1453 kvm_release_pfn_clean(page_to_pfn(page)); 1454 } 1455 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1456 1457 void kvm_release_pfn_clean(pfn_t pfn) 1458 { 1459 if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn)) 1460 put_page(pfn_to_page(pfn)); 1461 } 1462 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1463 1464 void kvm_release_page_dirty(struct page *page) 1465 { 1466 WARN_ON(is_error_page(page)); 1467 1468 kvm_release_pfn_dirty(page_to_pfn(page)); 1469 } 1470 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1471 1472 static void kvm_release_pfn_dirty(pfn_t pfn) 1473 { 1474 kvm_set_pfn_dirty(pfn); 1475 kvm_release_pfn_clean(pfn); 1476 } 1477 1478 void kvm_set_pfn_dirty(pfn_t pfn) 1479 { 1480 if (!kvm_is_mmio_pfn(pfn)) { 1481 struct page *page = pfn_to_page(pfn); 1482 if (!PageReserved(page)) 1483 SetPageDirty(page); 1484 } 1485 } 1486 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1487 1488 void kvm_set_pfn_accessed(pfn_t pfn) 1489 { 1490 if (!kvm_is_mmio_pfn(pfn)) 1491 mark_page_accessed(pfn_to_page(pfn)); 1492 } 1493 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1494 1495 void kvm_get_pfn(pfn_t pfn) 1496 { 1497 if (!kvm_is_mmio_pfn(pfn)) 1498 get_page(pfn_to_page(pfn)); 1499 } 1500 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1501 1502 static int next_segment(unsigned long len, int offset) 1503 { 1504 if (len > PAGE_SIZE - offset) 1505 return PAGE_SIZE - offset; 1506 else 1507 return len; 1508 } 1509 1510 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1511 int len) 1512 { 1513 int r; 1514 unsigned long addr; 1515 1516 addr = gfn_to_hva_prot(kvm, gfn, NULL); 1517 if (kvm_is_error_hva(addr)) 1518 return -EFAULT; 1519 r = kvm_read_hva(data, (void __user *)addr + offset, len); 1520 if (r) 1521 return -EFAULT; 1522 return 0; 1523 } 1524 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1525 1526 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1527 { 1528 gfn_t gfn = gpa >> PAGE_SHIFT; 1529 int seg; 1530 int offset = offset_in_page(gpa); 1531 int ret; 1532 1533 while ((seg = next_segment(len, offset)) != 0) { 1534 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1535 if (ret < 0) 1536 return ret; 1537 offset = 0; 1538 len -= seg; 1539 data += seg; 1540 ++gfn; 1541 } 1542 return 0; 1543 } 1544 EXPORT_SYMBOL_GPL(kvm_read_guest); 1545 1546 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1547 unsigned long len) 1548 { 1549 int r; 1550 unsigned long addr; 1551 gfn_t gfn = gpa >> PAGE_SHIFT; 1552 int offset = offset_in_page(gpa); 1553 1554 addr = gfn_to_hva_prot(kvm, gfn, NULL); 1555 if (kvm_is_error_hva(addr)) 1556 return -EFAULT; 1557 pagefault_disable(); 1558 r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len); 1559 pagefault_enable(); 1560 if (r) 1561 return -EFAULT; 1562 return 0; 1563 } 1564 EXPORT_SYMBOL(kvm_read_guest_atomic); 1565 1566 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1567 int offset, int len) 1568 { 1569 int r; 1570 unsigned long addr; 1571 1572 addr = gfn_to_hva(kvm, gfn); 1573 if (kvm_is_error_hva(addr)) 1574 return -EFAULT; 1575 r = __copy_to_user((void __user *)addr + offset, data, len); 1576 if (r) 1577 return -EFAULT; 1578 mark_page_dirty(kvm, gfn); 1579 return 0; 1580 } 1581 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1582 1583 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1584 unsigned long len) 1585 { 1586 gfn_t gfn = gpa >> PAGE_SHIFT; 1587 int seg; 1588 int offset = offset_in_page(gpa); 1589 int ret; 1590 1591 while ((seg = next_segment(len, offset)) != 0) { 1592 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1593 if (ret < 0) 1594 return ret; 1595 offset = 0; 1596 len -= seg; 1597 data += seg; 1598 ++gfn; 1599 } 1600 return 0; 1601 } 1602 1603 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1604 gpa_t gpa, unsigned long len) 1605 { 1606 struct kvm_memslots *slots = kvm_memslots(kvm); 1607 int offset = offset_in_page(gpa); 1608 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1609 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1610 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1611 gfn_t nr_pages_avail; 1612 1613 ghc->gpa = gpa; 1614 ghc->generation = slots->generation; 1615 ghc->len = len; 1616 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1617 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail); 1618 if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) { 1619 ghc->hva += offset; 1620 } else { 1621 /* 1622 * If the requested region crosses two memslots, we still 1623 * verify that the entire region is valid here. 1624 */ 1625 while (start_gfn <= end_gfn) { 1626 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1627 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1628 &nr_pages_avail); 1629 if (kvm_is_error_hva(ghc->hva)) 1630 return -EFAULT; 1631 start_gfn += nr_pages_avail; 1632 } 1633 /* Use the slow path for cross page reads and writes. */ 1634 ghc->memslot = NULL; 1635 } 1636 return 0; 1637 } 1638 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1639 1640 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1641 void *data, unsigned long len) 1642 { 1643 struct kvm_memslots *slots = kvm_memslots(kvm); 1644 int r; 1645 1646 BUG_ON(len > ghc->len); 1647 1648 if (slots->generation != ghc->generation) 1649 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1650 1651 if (unlikely(!ghc->memslot)) 1652 return kvm_write_guest(kvm, ghc->gpa, data, len); 1653 1654 if (kvm_is_error_hva(ghc->hva)) 1655 return -EFAULT; 1656 1657 r = __copy_to_user((void __user *)ghc->hva, data, len); 1658 if (r) 1659 return -EFAULT; 1660 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1661 1662 return 0; 1663 } 1664 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1665 1666 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1667 void *data, unsigned long len) 1668 { 1669 struct kvm_memslots *slots = kvm_memslots(kvm); 1670 int r; 1671 1672 BUG_ON(len > ghc->len); 1673 1674 if (slots->generation != ghc->generation) 1675 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1676 1677 if (unlikely(!ghc->memslot)) 1678 return kvm_read_guest(kvm, ghc->gpa, data, len); 1679 1680 if (kvm_is_error_hva(ghc->hva)) 1681 return -EFAULT; 1682 1683 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1684 if (r) 1685 return -EFAULT; 1686 1687 return 0; 1688 } 1689 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1690 1691 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1692 { 1693 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1694 1695 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1696 } 1697 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1698 1699 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1700 { 1701 gfn_t gfn = gpa >> PAGE_SHIFT; 1702 int seg; 1703 int offset = offset_in_page(gpa); 1704 int ret; 1705 1706 while ((seg = next_segment(len, offset)) != 0) { 1707 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1708 if (ret < 0) 1709 return ret; 1710 offset = 0; 1711 len -= seg; 1712 ++gfn; 1713 } 1714 return 0; 1715 } 1716 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1717 1718 static void mark_page_dirty_in_slot(struct kvm *kvm, 1719 struct kvm_memory_slot *memslot, 1720 gfn_t gfn) 1721 { 1722 if (memslot && memslot->dirty_bitmap) { 1723 unsigned long rel_gfn = gfn - memslot->base_gfn; 1724 1725 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1726 } 1727 } 1728 1729 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1730 { 1731 struct kvm_memory_slot *memslot; 1732 1733 memslot = gfn_to_memslot(kvm, gfn); 1734 mark_page_dirty_in_slot(kvm, memslot, gfn); 1735 } 1736 EXPORT_SYMBOL_GPL(mark_page_dirty); 1737 1738 /* 1739 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1740 */ 1741 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1742 { 1743 DEFINE_WAIT(wait); 1744 1745 for (;;) { 1746 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1747 1748 if (kvm_arch_vcpu_runnable(vcpu)) { 1749 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1750 break; 1751 } 1752 if (kvm_cpu_has_pending_timer(vcpu)) 1753 break; 1754 if (signal_pending(current)) 1755 break; 1756 1757 schedule(); 1758 } 1759 1760 finish_wait(&vcpu->wq, &wait); 1761 } 1762 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 1763 1764 #ifndef CONFIG_S390 1765 /* 1766 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 1767 */ 1768 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 1769 { 1770 int me; 1771 int cpu = vcpu->cpu; 1772 wait_queue_head_t *wqp; 1773 1774 wqp = kvm_arch_vcpu_wq(vcpu); 1775 if (waitqueue_active(wqp)) { 1776 wake_up_interruptible(wqp); 1777 ++vcpu->stat.halt_wakeup; 1778 } 1779 1780 me = get_cpu(); 1781 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 1782 if (kvm_arch_vcpu_should_kick(vcpu)) 1783 smp_send_reschedule(cpu); 1784 put_cpu(); 1785 } 1786 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 1787 #endif /* !CONFIG_S390 */ 1788 1789 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 1790 { 1791 struct pid *pid; 1792 struct task_struct *task = NULL; 1793 int ret = 0; 1794 1795 rcu_read_lock(); 1796 pid = rcu_dereference(target->pid); 1797 if (pid) 1798 task = get_pid_task(pid, PIDTYPE_PID); 1799 rcu_read_unlock(); 1800 if (!task) 1801 return ret; 1802 if (task->flags & PF_VCPU) { 1803 put_task_struct(task); 1804 return ret; 1805 } 1806 ret = yield_to(task, 1); 1807 put_task_struct(task); 1808 1809 return ret; 1810 } 1811 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 1812 1813 /* 1814 * Helper that checks whether a VCPU is eligible for directed yield. 1815 * Most eligible candidate to yield is decided by following heuristics: 1816 * 1817 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 1818 * (preempted lock holder), indicated by @in_spin_loop. 1819 * Set at the beiginning and cleared at the end of interception/PLE handler. 1820 * 1821 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 1822 * chance last time (mostly it has become eligible now since we have probably 1823 * yielded to lockholder in last iteration. This is done by toggling 1824 * @dy_eligible each time a VCPU checked for eligibility.) 1825 * 1826 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 1827 * to preempted lock-holder could result in wrong VCPU selection and CPU 1828 * burning. Giving priority for a potential lock-holder increases lock 1829 * progress. 1830 * 1831 * Since algorithm is based on heuristics, accessing another VCPU data without 1832 * locking does not harm. It may result in trying to yield to same VCPU, fail 1833 * and continue with next VCPU and so on. 1834 */ 1835 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 1836 { 1837 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 1838 bool eligible; 1839 1840 eligible = !vcpu->spin_loop.in_spin_loop || 1841 vcpu->spin_loop.dy_eligible; 1842 1843 if (vcpu->spin_loop.in_spin_loop) 1844 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 1845 1846 return eligible; 1847 #else 1848 return true; 1849 #endif 1850 } 1851 1852 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 1853 { 1854 struct kvm *kvm = me->kvm; 1855 struct kvm_vcpu *vcpu; 1856 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 1857 int yielded = 0; 1858 int try = 3; 1859 int pass; 1860 int i; 1861 1862 kvm_vcpu_set_in_spin_loop(me, true); 1863 /* 1864 * We boost the priority of a VCPU that is runnable but not 1865 * currently running, because it got preempted by something 1866 * else and called schedule in __vcpu_run. Hopefully that 1867 * VCPU is holding the lock that we need and will release it. 1868 * We approximate round-robin by starting at the last boosted VCPU. 1869 */ 1870 for (pass = 0; pass < 2 && !yielded && try; pass++) { 1871 kvm_for_each_vcpu(i, vcpu, kvm) { 1872 if (!pass && i <= last_boosted_vcpu) { 1873 i = last_boosted_vcpu; 1874 continue; 1875 } else if (pass && i > last_boosted_vcpu) 1876 break; 1877 if (!ACCESS_ONCE(vcpu->preempted)) 1878 continue; 1879 if (vcpu == me) 1880 continue; 1881 if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 1882 continue; 1883 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 1884 continue; 1885 1886 yielded = kvm_vcpu_yield_to(vcpu); 1887 if (yielded > 0) { 1888 kvm->last_boosted_vcpu = i; 1889 break; 1890 } else if (yielded < 0) { 1891 try--; 1892 if (!try) 1893 break; 1894 } 1895 } 1896 } 1897 kvm_vcpu_set_in_spin_loop(me, false); 1898 1899 /* Ensure vcpu is not eligible during next spinloop */ 1900 kvm_vcpu_set_dy_eligible(me, false); 1901 } 1902 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 1903 1904 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1905 { 1906 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 1907 struct page *page; 1908 1909 if (vmf->pgoff == 0) 1910 page = virt_to_page(vcpu->run); 1911 #ifdef CONFIG_X86 1912 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 1913 page = virt_to_page(vcpu->arch.pio_data); 1914 #endif 1915 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1916 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 1917 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 1918 #endif 1919 else 1920 return kvm_arch_vcpu_fault(vcpu, vmf); 1921 get_page(page); 1922 vmf->page = page; 1923 return 0; 1924 } 1925 1926 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 1927 .fault = kvm_vcpu_fault, 1928 }; 1929 1930 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 1931 { 1932 vma->vm_ops = &kvm_vcpu_vm_ops; 1933 return 0; 1934 } 1935 1936 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 1937 { 1938 struct kvm_vcpu *vcpu = filp->private_data; 1939 1940 kvm_put_kvm(vcpu->kvm); 1941 return 0; 1942 } 1943 1944 static struct file_operations kvm_vcpu_fops = { 1945 .release = kvm_vcpu_release, 1946 .unlocked_ioctl = kvm_vcpu_ioctl, 1947 #ifdef CONFIG_COMPAT 1948 .compat_ioctl = kvm_vcpu_compat_ioctl, 1949 #endif 1950 .mmap = kvm_vcpu_mmap, 1951 .llseek = noop_llseek, 1952 }; 1953 1954 /* 1955 * Allocates an inode for the vcpu. 1956 */ 1957 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 1958 { 1959 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 1960 } 1961 1962 /* 1963 * Creates some virtual cpus. Good luck creating more than one. 1964 */ 1965 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 1966 { 1967 int r; 1968 struct kvm_vcpu *vcpu, *v; 1969 1970 if (id >= KVM_MAX_VCPUS) 1971 return -EINVAL; 1972 1973 vcpu = kvm_arch_vcpu_create(kvm, id); 1974 if (IS_ERR(vcpu)) 1975 return PTR_ERR(vcpu); 1976 1977 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 1978 1979 r = kvm_arch_vcpu_setup(vcpu); 1980 if (r) 1981 goto vcpu_destroy; 1982 1983 mutex_lock(&kvm->lock); 1984 if (!kvm_vcpu_compatible(vcpu)) { 1985 r = -EINVAL; 1986 goto unlock_vcpu_destroy; 1987 } 1988 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 1989 r = -EINVAL; 1990 goto unlock_vcpu_destroy; 1991 } 1992 1993 kvm_for_each_vcpu(r, v, kvm) 1994 if (v->vcpu_id == id) { 1995 r = -EEXIST; 1996 goto unlock_vcpu_destroy; 1997 } 1998 1999 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2000 2001 /* Now it's all set up, let userspace reach it */ 2002 kvm_get_kvm(kvm); 2003 r = create_vcpu_fd(vcpu); 2004 if (r < 0) { 2005 kvm_put_kvm(kvm); 2006 goto unlock_vcpu_destroy; 2007 } 2008 2009 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2010 smp_wmb(); 2011 atomic_inc(&kvm->online_vcpus); 2012 2013 mutex_unlock(&kvm->lock); 2014 kvm_arch_vcpu_postcreate(vcpu); 2015 return r; 2016 2017 unlock_vcpu_destroy: 2018 mutex_unlock(&kvm->lock); 2019 vcpu_destroy: 2020 kvm_arch_vcpu_destroy(vcpu); 2021 return r; 2022 } 2023 2024 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2025 { 2026 if (sigset) { 2027 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2028 vcpu->sigset_active = 1; 2029 vcpu->sigset = *sigset; 2030 } else 2031 vcpu->sigset_active = 0; 2032 return 0; 2033 } 2034 2035 static long kvm_vcpu_ioctl(struct file *filp, 2036 unsigned int ioctl, unsigned long arg) 2037 { 2038 struct kvm_vcpu *vcpu = filp->private_data; 2039 void __user *argp = (void __user *)arg; 2040 int r; 2041 struct kvm_fpu *fpu = NULL; 2042 struct kvm_sregs *kvm_sregs = NULL; 2043 2044 if (vcpu->kvm->mm != current->mm) 2045 return -EIO; 2046 2047 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2048 return -EINVAL; 2049 2050 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2051 /* 2052 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2053 * so vcpu_load() would break it. 2054 */ 2055 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT) 2056 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2057 #endif 2058 2059 2060 r = vcpu_load(vcpu); 2061 if (r) 2062 return r; 2063 switch (ioctl) { 2064 case KVM_RUN: 2065 r = -EINVAL; 2066 if (arg) 2067 goto out; 2068 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2069 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2070 break; 2071 case KVM_GET_REGS: { 2072 struct kvm_regs *kvm_regs; 2073 2074 r = -ENOMEM; 2075 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2076 if (!kvm_regs) 2077 goto out; 2078 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2079 if (r) 2080 goto out_free1; 2081 r = -EFAULT; 2082 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2083 goto out_free1; 2084 r = 0; 2085 out_free1: 2086 kfree(kvm_regs); 2087 break; 2088 } 2089 case KVM_SET_REGS: { 2090 struct kvm_regs *kvm_regs; 2091 2092 r = -ENOMEM; 2093 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2094 if (IS_ERR(kvm_regs)) { 2095 r = PTR_ERR(kvm_regs); 2096 goto out; 2097 } 2098 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2099 kfree(kvm_regs); 2100 break; 2101 } 2102 case KVM_GET_SREGS: { 2103 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2104 r = -ENOMEM; 2105 if (!kvm_sregs) 2106 goto out; 2107 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2108 if (r) 2109 goto out; 2110 r = -EFAULT; 2111 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2112 goto out; 2113 r = 0; 2114 break; 2115 } 2116 case KVM_SET_SREGS: { 2117 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2118 if (IS_ERR(kvm_sregs)) { 2119 r = PTR_ERR(kvm_sregs); 2120 kvm_sregs = NULL; 2121 goto out; 2122 } 2123 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2124 break; 2125 } 2126 case KVM_GET_MP_STATE: { 2127 struct kvm_mp_state mp_state; 2128 2129 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2130 if (r) 2131 goto out; 2132 r = -EFAULT; 2133 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 2134 goto out; 2135 r = 0; 2136 break; 2137 } 2138 case KVM_SET_MP_STATE: { 2139 struct kvm_mp_state mp_state; 2140 2141 r = -EFAULT; 2142 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 2143 goto out; 2144 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2145 break; 2146 } 2147 case KVM_TRANSLATE: { 2148 struct kvm_translation tr; 2149 2150 r = -EFAULT; 2151 if (copy_from_user(&tr, argp, sizeof tr)) 2152 goto out; 2153 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2154 if (r) 2155 goto out; 2156 r = -EFAULT; 2157 if (copy_to_user(argp, &tr, sizeof tr)) 2158 goto out; 2159 r = 0; 2160 break; 2161 } 2162 case KVM_SET_GUEST_DEBUG: { 2163 struct kvm_guest_debug dbg; 2164 2165 r = -EFAULT; 2166 if (copy_from_user(&dbg, argp, sizeof dbg)) 2167 goto out; 2168 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2169 break; 2170 } 2171 case KVM_SET_SIGNAL_MASK: { 2172 struct kvm_signal_mask __user *sigmask_arg = argp; 2173 struct kvm_signal_mask kvm_sigmask; 2174 sigset_t sigset, *p; 2175 2176 p = NULL; 2177 if (argp) { 2178 r = -EFAULT; 2179 if (copy_from_user(&kvm_sigmask, argp, 2180 sizeof kvm_sigmask)) 2181 goto out; 2182 r = -EINVAL; 2183 if (kvm_sigmask.len != sizeof sigset) 2184 goto out; 2185 r = -EFAULT; 2186 if (copy_from_user(&sigset, sigmask_arg->sigset, 2187 sizeof sigset)) 2188 goto out; 2189 p = &sigset; 2190 } 2191 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2192 break; 2193 } 2194 case KVM_GET_FPU: { 2195 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2196 r = -ENOMEM; 2197 if (!fpu) 2198 goto out; 2199 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2200 if (r) 2201 goto out; 2202 r = -EFAULT; 2203 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2204 goto out; 2205 r = 0; 2206 break; 2207 } 2208 case KVM_SET_FPU: { 2209 fpu = memdup_user(argp, sizeof(*fpu)); 2210 if (IS_ERR(fpu)) { 2211 r = PTR_ERR(fpu); 2212 fpu = NULL; 2213 goto out; 2214 } 2215 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2216 break; 2217 } 2218 default: 2219 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2220 } 2221 out: 2222 vcpu_put(vcpu); 2223 kfree(fpu); 2224 kfree(kvm_sregs); 2225 return r; 2226 } 2227 2228 #ifdef CONFIG_COMPAT 2229 static long kvm_vcpu_compat_ioctl(struct file *filp, 2230 unsigned int ioctl, unsigned long arg) 2231 { 2232 struct kvm_vcpu *vcpu = filp->private_data; 2233 void __user *argp = compat_ptr(arg); 2234 int r; 2235 2236 if (vcpu->kvm->mm != current->mm) 2237 return -EIO; 2238 2239 switch (ioctl) { 2240 case KVM_SET_SIGNAL_MASK: { 2241 struct kvm_signal_mask __user *sigmask_arg = argp; 2242 struct kvm_signal_mask kvm_sigmask; 2243 compat_sigset_t csigset; 2244 sigset_t sigset; 2245 2246 if (argp) { 2247 r = -EFAULT; 2248 if (copy_from_user(&kvm_sigmask, argp, 2249 sizeof kvm_sigmask)) 2250 goto out; 2251 r = -EINVAL; 2252 if (kvm_sigmask.len != sizeof csigset) 2253 goto out; 2254 r = -EFAULT; 2255 if (copy_from_user(&csigset, sigmask_arg->sigset, 2256 sizeof csigset)) 2257 goto out; 2258 sigset_from_compat(&sigset, &csigset); 2259 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2260 } else 2261 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2262 break; 2263 } 2264 default: 2265 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2266 } 2267 2268 out: 2269 return r; 2270 } 2271 #endif 2272 2273 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2274 int (*accessor)(struct kvm_device *dev, 2275 struct kvm_device_attr *attr), 2276 unsigned long arg) 2277 { 2278 struct kvm_device_attr attr; 2279 2280 if (!accessor) 2281 return -EPERM; 2282 2283 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2284 return -EFAULT; 2285 2286 return accessor(dev, &attr); 2287 } 2288 2289 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2290 unsigned long arg) 2291 { 2292 struct kvm_device *dev = filp->private_data; 2293 2294 switch (ioctl) { 2295 case KVM_SET_DEVICE_ATTR: 2296 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2297 case KVM_GET_DEVICE_ATTR: 2298 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2299 case KVM_HAS_DEVICE_ATTR: 2300 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2301 default: 2302 if (dev->ops->ioctl) 2303 return dev->ops->ioctl(dev, ioctl, arg); 2304 2305 return -ENOTTY; 2306 } 2307 } 2308 2309 static int kvm_device_release(struct inode *inode, struct file *filp) 2310 { 2311 struct kvm_device *dev = filp->private_data; 2312 struct kvm *kvm = dev->kvm; 2313 2314 kvm_put_kvm(kvm); 2315 return 0; 2316 } 2317 2318 static const struct file_operations kvm_device_fops = { 2319 .unlocked_ioctl = kvm_device_ioctl, 2320 #ifdef CONFIG_COMPAT 2321 .compat_ioctl = kvm_device_ioctl, 2322 #endif 2323 .release = kvm_device_release, 2324 }; 2325 2326 struct kvm_device *kvm_device_from_filp(struct file *filp) 2327 { 2328 if (filp->f_op != &kvm_device_fops) 2329 return NULL; 2330 2331 return filp->private_data; 2332 } 2333 2334 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2335 #ifdef CONFIG_KVM_MPIC 2336 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2337 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2338 #endif 2339 2340 #ifdef CONFIG_KVM_XICS 2341 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2342 #endif 2343 }; 2344 2345 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2346 { 2347 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2348 return -ENOSPC; 2349 2350 if (kvm_device_ops_table[type] != NULL) 2351 return -EEXIST; 2352 2353 kvm_device_ops_table[type] = ops; 2354 return 0; 2355 } 2356 2357 void kvm_unregister_device_ops(u32 type) 2358 { 2359 if (kvm_device_ops_table[type] != NULL) 2360 kvm_device_ops_table[type] = NULL; 2361 } 2362 2363 static int kvm_ioctl_create_device(struct kvm *kvm, 2364 struct kvm_create_device *cd) 2365 { 2366 struct kvm_device_ops *ops = NULL; 2367 struct kvm_device *dev; 2368 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2369 int ret; 2370 2371 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2372 return -ENODEV; 2373 2374 ops = kvm_device_ops_table[cd->type]; 2375 if (ops == NULL) 2376 return -ENODEV; 2377 2378 if (test) 2379 return 0; 2380 2381 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2382 if (!dev) 2383 return -ENOMEM; 2384 2385 dev->ops = ops; 2386 dev->kvm = kvm; 2387 2388 ret = ops->create(dev, cd->type); 2389 if (ret < 0) { 2390 kfree(dev); 2391 return ret; 2392 } 2393 2394 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2395 if (ret < 0) { 2396 ops->destroy(dev); 2397 return ret; 2398 } 2399 2400 list_add(&dev->vm_node, &kvm->devices); 2401 kvm_get_kvm(kvm); 2402 cd->fd = ret; 2403 return 0; 2404 } 2405 2406 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2407 { 2408 switch (arg) { 2409 case KVM_CAP_USER_MEMORY: 2410 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2411 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2412 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2413 case KVM_CAP_SET_BOOT_CPU_ID: 2414 #endif 2415 case KVM_CAP_INTERNAL_ERROR_DATA: 2416 #ifdef CONFIG_HAVE_KVM_MSI 2417 case KVM_CAP_SIGNAL_MSI: 2418 #endif 2419 #ifdef CONFIG_HAVE_KVM_IRQFD 2420 case KVM_CAP_IRQFD_RESAMPLE: 2421 #endif 2422 case KVM_CAP_CHECK_EXTENSION_VM: 2423 return 1; 2424 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2425 case KVM_CAP_IRQ_ROUTING: 2426 return KVM_MAX_IRQ_ROUTES; 2427 #endif 2428 default: 2429 break; 2430 } 2431 return kvm_vm_ioctl_check_extension(kvm, arg); 2432 } 2433 2434 static long kvm_vm_ioctl(struct file *filp, 2435 unsigned int ioctl, unsigned long arg) 2436 { 2437 struct kvm *kvm = filp->private_data; 2438 void __user *argp = (void __user *)arg; 2439 int r; 2440 2441 if (kvm->mm != current->mm) 2442 return -EIO; 2443 switch (ioctl) { 2444 case KVM_CREATE_VCPU: 2445 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2446 break; 2447 case KVM_SET_USER_MEMORY_REGION: { 2448 struct kvm_userspace_memory_region kvm_userspace_mem; 2449 2450 r = -EFAULT; 2451 if (copy_from_user(&kvm_userspace_mem, argp, 2452 sizeof kvm_userspace_mem)) 2453 goto out; 2454 2455 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2456 break; 2457 } 2458 case KVM_GET_DIRTY_LOG: { 2459 struct kvm_dirty_log log; 2460 2461 r = -EFAULT; 2462 if (copy_from_user(&log, argp, sizeof log)) 2463 goto out; 2464 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2465 break; 2466 } 2467 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2468 case KVM_REGISTER_COALESCED_MMIO: { 2469 struct kvm_coalesced_mmio_zone zone; 2470 r = -EFAULT; 2471 if (copy_from_user(&zone, argp, sizeof zone)) 2472 goto out; 2473 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2474 break; 2475 } 2476 case KVM_UNREGISTER_COALESCED_MMIO: { 2477 struct kvm_coalesced_mmio_zone zone; 2478 r = -EFAULT; 2479 if (copy_from_user(&zone, argp, sizeof zone)) 2480 goto out; 2481 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2482 break; 2483 } 2484 #endif 2485 case KVM_IRQFD: { 2486 struct kvm_irqfd data; 2487 2488 r = -EFAULT; 2489 if (copy_from_user(&data, argp, sizeof data)) 2490 goto out; 2491 r = kvm_irqfd(kvm, &data); 2492 break; 2493 } 2494 case KVM_IOEVENTFD: { 2495 struct kvm_ioeventfd data; 2496 2497 r = -EFAULT; 2498 if (copy_from_user(&data, argp, sizeof data)) 2499 goto out; 2500 r = kvm_ioeventfd(kvm, &data); 2501 break; 2502 } 2503 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2504 case KVM_SET_BOOT_CPU_ID: 2505 r = 0; 2506 mutex_lock(&kvm->lock); 2507 if (atomic_read(&kvm->online_vcpus) != 0) 2508 r = -EBUSY; 2509 else 2510 kvm->bsp_vcpu_id = arg; 2511 mutex_unlock(&kvm->lock); 2512 break; 2513 #endif 2514 #ifdef CONFIG_HAVE_KVM_MSI 2515 case KVM_SIGNAL_MSI: { 2516 struct kvm_msi msi; 2517 2518 r = -EFAULT; 2519 if (copy_from_user(&msi, argp, sizeof msi)) 2520 goto out; 2521 r = kvm_send_userspace_msi(kvm, &msi); 2522 break; 2523 } 2524 #endif 2525 #ifdef __KVM_HAVE_IRQ_LINE 2526 case KVM_IRQ_LINE_STATUS: 2527 case KVM_IRQ_LINE: { 2528 struct kvm_irq_level irq_event; 2529 2530 r = -EFAULT; 2531 if (copy_from_user(&irq_event, argp, sizeof irq_event)) 2532 goto out; 2533 2534 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2535 ioctl == KVM_IRQ_LINE_STATUS); 2536 if (r) 2537 goto out; 2538 2539 r = -EFAULT; 2540 if (ioctl == KVM_IRQ_LINE_STATUS) { 2541 if (copy_to_user(argp, &irq_event, sizeof irq_event)) 2542 goto out; 2543 } 2544 2545 r = 0; 2546 break; 2547 } 2548 #endif 2549 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2550 case KVM_SET_GSI_ROUTING: { 2551 struct kvm_irq_routing routing; 2552 struct kvm_irq_routing __user *urouting; 2553 struct kvm_irq_routing_entry *entries; 2554 2555 r = -EFAULT; 2556 if (copy_from_user(&routing, argp, sizeof(routing))) 2557 goto out; 2558 r = -EINVAL; 2559 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2560 goto out; 2561 if (routing.flags) 2562 goto out; 2563 r = -ENOMEM; 2564 entries = vmalloc(routing.nr * sizeof(*entries)); 2565 if (!entries) 2566 goto out; 2567 r = -EFAULT; 2568 urouting = argp; 2569 if (copy_from_user(entries, urouting->entries, 2570 routing.nr * sizeof(*entries))) 2571 goto out_free_irq_routing; 2572 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2573 routing.flags); 2574 out_free_irq_routing: 2575 vfree(entries); 2576 break; 2577 } 2578 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2579 case KVM_CREATE_DEVICE: { 2580 struct kvm_create_device cd; 2581 2582 r = -EFAULT; 2583 if (copy_from_user(&cd, argp, sizeof(cd))) 2584 goto out; 2585 2586 r = kvm_ioctl_create_device(kvm, &cd); 2587 if (r) 2588 goto out; 2589 2590 r = -EFAULT; 2591 if (copy_to_user(argp, &cd, sizeof(cd))) 2592 goto out; 2593 2594 r = 0; 2595 break; 2596 } 2597 case KVM_CHECK_EXTENSION: 2598 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 2599 break; 2600 default: 2601 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2602 if (r == -ENOTTY) 2603 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg); 2604 } 2605 out: 2606 return r; 2607 } 2608 2609 #ifdef CONFIG_COMPAT 2610 struct compat_kvm_dirty_log { 2611 __u32 slot; 2612 __u32 padding1; 2613 union { 2614 compat_uptr_t dirty_bitmap; /* one bit per page */ 2615 __u64 padding2; 2616 }; 2617 }; 2618 2619 static long kvm_vm_compat_ioctl(struct file *filp, 2620 unsigned int ioctl, unsigned long arg) 2621 { 2622 struct kvm *kvm = filp->private_data; 2623 int r; 2624 2625 if (kvm->mm != current->mm) 2626 return -EIO; 2627 switch (ioctl) { 2628 case KVM_GET_DIRTY_LOG: { 2629 struct compat_kvm_dirty_log compat_log; 2630 struct kvm_dirty_log log; 2631 2632 r = -EFAULT; 2633 if (copy_from_user(&compat_log, (void __user *)arg, 2634 sizeof(compat_log))) 2635 goto out; 2636 log.slot = compat_log.slot; 2637 log.padding1 = compat_log.padding1; 2638 log.padding2 = compat_log.padding2; 2639 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2640 2641 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2642 break; 2643 } 2644 default: 2645 r = kvm_vm_ioctl(filp, ioctl, arg); 2646 } 2647 2648 out: 2649 return r; 2650 } 2651 #endif 2652 2653 static struct file_operations kvm_vm_fops = { 2654 .release = kvm_vm_release, 2655 .unlocked_ioctl = kvm_vm_ioctl, 2656 #ifdef CONFIG_COMPAT 2657 .compat_ioctl = kvm_vm_compat_ioctl, 2658 #endif 2659 .llseek = noop_llseek, 2660 }; 2661 2662 static int kvm_dev_ioctl_create_vm(unsigned long type) 2663 { 2664 int r; 2665 struct kvm *kvm; 2666 2667 kvm = kvm_create_vm(type); 2668 if (IS_ERR(kvm)) 2669 return PTR_ERR(kvm); 2670 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2671 r = kvm_coalesced_mmio_init(kvm); 2672 if (r < 0) { 2673 kvm_put_kvm(kvm); 2674 return r; 2675 } 2676 #endif 2677 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 2678 if (r < 0) 2679 kvm_put_kvm(kvm); 2680 2681 return r; 2682 } 2683 2684 static long kvm_dev_ioctl(struct file *filp, 2685 unsigned int ioctl, unsigned long arg) 2686 { 2687 long r = -EINVAL; 2688 2689 switch (ioctl) { 2690 case KVM_GET_API_VERSION: 2691 if (arg) 2692 goto out; 2693 r = KVM_API_VERSION; 2694 break; 2695 case KVM_CREATE_VM: 2696 r = kvm_dev_ioctl_create_vm(arg); 2697 break; 2698 case KVM_CHECK_EXTENSION: 2699 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 2700 break; 2701 case KVM_GET_VCPU_MMAP_SIZE: 2702 if (arg) 2703 goto out; 2704 r = PAGE_SIZE; /* struct kvm_run */ 2705 #ifdef CONFIG_X86 2706 r += PAGE_SIZE; /* pio data page */ 2707 #endif 2708 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2709 r += PAGE_SIZE; /* coalesced mmio ring page */ 2710 #endif 2711 break; 2712 case KVM_TRACE_ENABLE: 2713 case KVM_TRACE_PAUSE: 2714 case KVM_TRACE_DISABLE: 2715 r = -EOPNOTSUPP; 2716 break; 2717 default: 2718 return kvm_arch_dev_ioctl(filp, ioctl, arg); 2719 } 2720 out: 2721 return r; 2722 } 2723 2724 static struct file_operations kvm_chardev_ops = { 2725 .unlocked_ioctl = kvm_dev_ioctl, 2726 .compat_ioctl = kvm_dev_ioctl, 2727 .llseek = noop_llseek, 2728 }; 2729 2730 static struct miscdevice kvm_dev = { 2731 KVM_MINOR, 2732 "kvm", 2733 &kvm_chardev_ops, 2734 }; 2735 2736 static void hardware_enable_nolock(void *junk) 2737 { 2738 int cpu = raw_smp_processor_id(); 2739 int r; 2740 2741 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2742 return; 2743 2744 cpumask_set_cpu(cpu, cpus_hardware_enabled); 2745 2746 r = kvm_arch_hardware_enable(); 2747 2748 if (r) { 2749 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2750 atomic_inc(&hardware_enable_failed); 2751 printk(KERN_INFO "kvm: enabling virtualization on " 2752 "CPU%d failed\n", cpu); 2753 } 2754 } 2755 2756 static void hardware_enable(void) 2757 { 2758 raw_spin_lock(&kvm_count_lock); 2759 if (kvm_usage_count) 2760 hardware_enable_nolock(NULL); 2761 raw_spin_unlock(&kvm_count_lock); 2762 } 2763 2764 static void hardware_disable_nolock(void *junk) 2765 { 2766 int cpu = raw_smp_processor_id(); 2767 2768 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2769 return; 2770 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2771 kvm_arch_hardware_disable(); 2772 } 2773 2774 static void hardware_disable(void) 2775 { 2776 raw_spin_lock(&kvm_count_lock); 2777 if (kvm_usage_count) 2778 hardware_disable_nolock(NULL); 2779 raw_spin_unlock(&kvm_count_lock); 2780 } 2781 2782 static void hardware_disable_all_nolock(void) 2783 { 2784 BUG_ON(!kvm_usage_count); 2785 2786 kvm_usage_count--; 2787 if (!kvm_usage_count) 2788 on_each_cpu(hardware_disable_nolock, NULL, 1); 2789 } 2790 2791 static void hardware_disable_all(void) 2792 { 2793 raw_spin_lock(&kvm_count_lock); 2794 hardware_disable_all_nolock(); 2795 raw_spin_unlock(&kvm_count_lock); 2796 } 2797 2798 static int hardware_enable_all(void) 2799 { 2800 int r = 0; 2801 2802 raw_spin_lock(&kvm_count_lock); 2803 2804 kvm_usage_count++; 2805 if (kvm_usage_count == 1) { 2806 atomic_set(&hardware_enable_failed, 0); 2807 on_each_cpu(hardware_enable_nolock, NULL, 1); 2808 2809 if (atomic_read(&hardware_enable_failed)) { 2810 hardware_disable_all_nolock(); 2811 r = -EBUSY; 2812 } 2813 } 2814 2815 raw_spin_unlock(&kvm_count_lock); 2816 2817 return r; 2818 } 2819 2820 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 2821 void *v) 2822 { 2823 int cpu = (long)v; 2824 2825 val &= ~CPU_TASKS_FROZEN; 2826 switch (val) { 2827 case CPU_DYING: 2828 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 2829 cpu); 2830 hardware_disable(); 2831 break; 2832 case CPU_STARTING: 2833 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 2834 cpu); 2835 hardware_enable(); 2836 break; 2837 } 2838 return NOTIFY_OK; 2839 } 2840 2841 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 2842 void *v) 2843 { 2844 /* 2845 * Some (well, at least mine) BIOSes hang on reboot if 2846 * in vmx root mode. 2847 * 2848 * And Intel TXT required VMX off for all cpu when system shutdown. 2849 */ 2850 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 2851 kvm_rebooting = true; 2852 on_each_cpu(hardware_disable_nolock, NULL, 1); 2853 return NOTIFY_OK; 2854 } 2855 2856 static struct notifier_block kvm_reboot_notifier = { 2857 .notifier_call = kvm_reboot, 2858 .priority = 0, 2859 }; 2860 2861 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 2862 { 2863 int i; 2864 2865 for (i = 0; i < bus->dev_count; i++) { 2866 struct kvm_io_device *pos = bus->range[i].dev; 2867 2868 kvm_iodevice_destructor(pos); 2869 } 2870 kfree(bus); 2871 } 2872 2873 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 2874 const struct kvm_io_range *r2) 2875 { 2876 if (r1->addr < r2->addr) 2877 return -1; 2878 if (r1->addr + r1->len > r2->addr + r2->len) 2879 return 1; 2880 return 0; 2881 } 2882 2883 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 2884 { 2885 return kvm_io_bus_cmp(p1, p2); 2886 } 2887 2888 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 2889 gpa_t addr, int len) 2890 { 2891 bus->range[bus->dev_count++] = (struct kvm_io_range) { 2892 .addr = addr, 2893 .len = len, 2894 .dev = dev, 2895 }; 2896 2897 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 2898 kvm_io_bus_sort_cmp, NULL); 2899 2900 return 0; 2901 } 2902 2903 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 2904 gpa_t addr, int len) 2905 { 2906 struct kvm_io_range *range, key; 2907 int off; 2908 2909 key = (struct kvm_io_range) { 2910 .addr = addr, 2911 .len = len, 2912 }; 2913 2914 range = bsearch(&key, bus->range, bus->dev_count, 2915 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 2916 if (range == NULL) 2917 return -ENOENT; 2918 2919 off = range - bus->range; 2920 2921 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 2922 off--; 2923 2924 return off; 2925 } 2926 2927 static int __kvm_io_bus_write(struct kvm_io_bus *bus, 2928 struct kvm_io_range *range, const void *val) 2929 { 2930 int idx; 2931 2932 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 2933 if (idx < 0) 2934 return -EOPNOTSUPP; 2935 2936 while (idx < bus->dev_count && 2937 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 2938 if (!kvm_iodevice_write(bus->range[idx].dev, range->addr, 2939 range->len, val)) 2940 return idx; 2941 idx++; 2942 } 2943 2944 return -EOPNOTSUPP; 2945 } 2946 2947 /* kvm_io_bus_write - called under kvm->slots_lock */ 2948 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2949 int len, const void *val) 2950 { 2951 struct kvm_io_bus *bus; 2952 struct kvm_io_range range; 2953 int r; 2954 2955 range = (struct kvm_io_range) { 2956 .addr = addr, 2957 .len = len, 2958 }; 2959 2960 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2961 r = __kvm_io_bus_write(bus, &range, val); 2962 return r < 0 ? r : 0; 2963 } 2964 2965 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 2966 int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2967 int len, const void *val, long cookie) 2968 { 2969 struct kvm_io_bus *bus; 2970 struct kvm_io_range range; 2971 2972 range = (struct kvm_io_range) { 2973 .addr = addr, 2974 .len = len, 2975 }; 2976 2977 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2978 2979 /* First try the device referenced by cookie. */ 2980 if ((cookie >= 0) && (cookie < bus->dev_count) && 2981 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 2982 if (!kvm_iodevice_write(bus->range[cookie].dev, addr, len, 2983 val)) 2984 return cookie; 2985 2986 /* 2987 * cookie contained garbage; fall back to search and return the 2988 * correct cookie value. 2989 */ 2990 return __kvm_io_bus_write(bus, &range, val); 2991 } 2992 2993 static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range, 2994 void *val) 2995 { 2996 int idx; 2997 2998 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 2999 if (idx < 0) 3000 return -EOPNOTSUPP; 3001 3002 while (idx < bus->dev_count && 3003 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3004 if (!kvm_iodevice_read(bus->range[idx].dev, range->addr, 3005 range->len, val)) 3006 return idx; 3007 idx++; 3008 } 3009 3010 return -EOPNOTSUPP; 3011 } 3012 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3013 3014 /* kvm_io_bus_read - called under kvm->slots_lock */ 3015 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3016 int len, void *val) 3017 { 3018 struct kvm_io_bus *bus; 3019 struct kvm_io_range range; 3020 int r; 3021 3022 range = (struct kvm_io_range) { 3023 .addr = addr, 3024 .len = len, 3025 }; 3026 3027 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3028 r = __kvm_io_bus_read(bus, &range, val); 3029 return r < 0 ? r : 0; 3030 } 3031 3032 3033 /* Caller must hold slots_lock. */ 3034 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3035 int len, struct kvm_io_device *dev) 3036 { 3037 struct kvm_io_bus *new_bus, *bus; 3038 3039 bus = kvm->buses[bus_idx]; 3040 /* exclude ioeventfd which is limited by maximum fd */ 3041 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3042 return -ENOSPC; 3043 3044 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3045 sizeof(struct kvm_io_range)), GFP_KERNEL); 3046 if (!new_bus) 3047 return -ENOMEM; 3048 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3049 sizeof(struct kvm_io_range))); 3050 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3051 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3052 synchronize_srcu_expedited(&kvm->srcu); 3053 kfree(bus); 3054 3055 return 0; 3056 } 3057 3058 /* Caller must hold slots_lock. */ 3059 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3060 struct kvm_io_device *dev) 3061 { 3062 int i, r; 3063 struct kvm_io_bus *new_bus, *bus; 3064 3065 bus = kvm->buses[bus_idx]; 3066 r = -ENOENT; 3067 for (i = 0; i < bus->dev_count; i++) 3068 if (bus->range[i].dev == dev) { 3069 r = 0; 3070 break; 3071 } 3072 3073 if (r) 3074 return r; 3075 3076 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3077 sizeof(struct kvm_io_range)), GFP_KERNEL); 3078 if (!new_bus) 3079 return -ENOMEM; 3080 3081 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3082 new_bus->dev_count--; 3083 memcpy(new_bus->range + i, bus->range + i + 1, 3084 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3085 3086 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3087 synchronize_srcu_expedited(&kvm->srcu); 3088 kfree(bus); 3089 return r; 3090 } 3091 3092 static struct notifier_block kvm_cpu_notifier = { 3093 .notifier_call = kvm_cpu_hotplug, 3094 }; 3095 3096 static int vm_stat_get(void *_offset, u64 *val) 3097 { 3098 unsigned offset = (long)_offset; 3099 struct kvm *kvm; 3100 3101 *val = 0; 3102 spin_lock(&kvm_lock); 3103 list_for_each_entry(kvm, &vm_list, vm_list) 3104 *val += *(u32 *)((void *)kvm + offset); 3105 spin_unlock(&kvm_lock); 3106 return 0; 3107 } 3108 3109 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3110 3111 static int vcpu_stat_get(void *_offset, u64 *val) 3112 { 3113 unsigned offset = (long)_offset; 3114 struct kvm *kvm; 3115 struct kvm_vcpu *vcpu; 3116 int i; 3117 3118 *val = 0; 3119 spin_lock(&kvm_lock); 3120 list_for_each_entry(kvm, &vm_list, vm_list) 3121 kvm_for_each_vcpu(i, vcpu, kvm) 3122 *val += *(u32 *)((void *)vcpu + offset); 3123 3124 spin_unlock(&kvm_lock); 3125 return 0; 3126 } 3127 3128 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3129 3130 static const struct file_operations *stat_fops[] = { 3131 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3132 [KVM_STAT_VM] = &vm_stat_fops, 3133 }; 3134 3135 static int kvm_init_debug(void) 3136 { 3137 int r = -EEXIST; 3138 struct kvm_stats_debugfs_item *p; 3139 3140 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3141 if (kvm_debugfs_dir == NULL) 3142 goto out; 3143 3144 for (p = debugfs_entries; p->name; ++p) { 3145 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3146 (void *)(long)p->offset, 3147 stat_fops[p->kind]); 3148 if (p->dentry == NULL) 3149 goto out_dir; 3150 } 3151 3152 return 0; 3153 3154 out_dir: 3155 debugfs_remove_recursive(kvm_debugfs_dir); 3156 out: 3157 return r; 3158 } 3159 3160 static void kvm_exit_debug(void) 3161 { 3162 struct kvm_stats_debugfs_item *p; 3163 3164 for (p = debugfs_entries; p->name; ++p) 3165 debugfs_remove(p->dentry); 3166 debugfs_remove(kvm_debugfs_dir); 3167 } 3168 3169 static int kvm_suspend(void) 3170 { 3171 if (kvm_usage_count) 3172 hardware_disable_nolock(NULL); 3173 return 0; 3174 } 3175 3176 static void kvm_resume(void) 3177 { 3178 if (kvm_usage_count) { 3179 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3180 hardware_enable_nolock(NULL); 3181 } 3182 } 3183 3184 static struct syscore_ops kvm_syscore_ops = { 3185 .suspend = kvm_suspend, 3186 .resume = kvm_resume, 3187 }; 3188 3189 static inline 3190 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3191 { 3192 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3193 } 3194 3195 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3196 { 3197 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3198 if (vcpu->preempted) 3199 vcpu->preempted = false; 3200 3201 kvm_arch_sched_in(vcpu, cpu); 3202 3203 kvm_arch_vcpu_load(vcpu, cpu); 3204 } 3205 3206 static void kvm_sched_out(struct preempt_notifier *pn, 3207 struct task_struct *next) 3208 { 3209 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3210 3211 if (current->state == TASK_RUNNING) 3212 vcpu->preempted = true; 3213 kvm_arch_vcpu_put(vcpu); 3214 } 3215 3216 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3217 struct module *module) 3218 { 3219 int r; 3220 int cpu; 3221 3222 r = kvm_arch_init(opaque); 3223 if (r) 3224 goto out_fail; 3225 3226 /* 3227 * kvm_arch_init makes sure there's at most one caller 3228 * for architectures that support multiple implementations, 3229 * like intel and amd on x86. 3230 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3231 * conflicts in case kvm is already setup for another implementation. 3232 */ 3233 r = kvm_irqfd_init(); 3234 if (r) 3235 goto out_irqfd; 3236 3237 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3238 r = -ENOMEM; 3239 goto out_free_0; 3240 } 3241 3242 r = kvm_arch_hardware_setup(); 3243 if (r < 0) 3244 goto out_free_0a; 3245 3246 for_each_online_cpu(cpu) { 3247 smp_call_function_single(cpu, 3248 kvm_arch_check_processor_compat, 3249 &r, 1); 3250 if (r < 0) 3251 goto out_free_1; 3252 } 3253 3254 r = register_cpu_notifier(&kvm_cpu_notifier); 3255 if (r) 3256 goto out_free_2; 3257 register_reboot_notifier(&kvm_reboot_notifier); 3258 3259 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3260 if (!vcpu_align) 3261 vcpu_align = __alignof__(struct kvm_vcpu); 3262 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3263 0, NULL); 3264 if (!kvm_vcpu_cache) { 3265 r = -ENOMEM; 3266 goto out_free_3; 3267 } 3268 3269 r = kvm_async_pf_init(); 3270 if (r) 3271 goto out_free; 3272 3273 kvm_chardev_ops.owner = module; 3274 kvm_vm_fops.owner = module; 3275 kvm_vcpu_fops.owner = module; 3276 3277 r = misc_register(&kvm_dev); 3278 if (r) { 3279 printk(KERN_ERR "kvm: misc device register failed\n"); 3280 goto out_unreg; 3281 } 3282 3283 register_syscore_ops(&kvm_syscore_ops); 3284 3285 kvm_preempt_ops.sched_in = kvm_sched_in; 3286 kvm_preempt_ops.sched_out = kvm_sched_out; 3287 3288 r = kvm_init_debug(); 3289 if (r) { 3290 printk(KERN_ERR "kvm: create debugfs files failed\n"); 3291 goto out_undebugfs; 3292 } 3293 3294 r = kvm_vfio_ops_init(); 3295 WARN_ON(r); 3296 3297 return 0; 3298 3299 out_undebugfs: 3300 unregister_syscore_ops(&kvm_syscore_ops); 3301 misc_deregister(&kvm_dev); 3302 out_unreg: 3303 kvm_async_pf_deinit(); 3304 out_free: 3305 kmem_cache_destroy(kvm_vcpu_cache); 3306 out_free_3: 3307 unregister_reboot_notifier(&kvm_reboot_notifier); 3308 unregister_cpu_notifier(&kvm_cpu_notifier); 3309 out_free_2: 3310 out_free_1: 3311 kvm_arch_hardware_unsetup(); 3312 out_free_0a: 3313 free_cpumask_var(cpus_hardware_enabled); 3314 out_free_0: 3315 kvm_irqfd_exit(); 3316 out_irqfd: 3317 kvm_arch_exit(); 3318 out_fail: 3319 return r; 3320 } 3321 EXPORT_SYMBOL_GPL(kvm_init); 3322 3323 void kvm_exit(void) 3324 { 3325 kvm_exit_debug(); 3326 misc_deregister(&kvm_dev); 3327 kmem_cache_destroy(kvm_vcpu_cache); 3328 kvm_async_pf_deinit(); 3329 unregister_syscore_ops(&kvm_syscore_ops); 3330 unregister_reboot_notifier(&kvm_reboot_notifier); 3331 unregister_cpu_notifier(&kvm_cpu_notifier); 3332 on_each_cpu(hardware_disable_nolock, NULL, 1); 3333 kvm_arch_hardware_unsetup(); 3334 kvm_arch_exit(); 3335 kvm_irqfd_exit(); 3336 free_cpumask_var(cpus_hardware_enabled); 3337 kvm_vfio_ops_exit(); 3338 } 3339 EXPORT_SYMBOL_GPL(kvm_exit); 3340