1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include "iodev.h" 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/uaccess.h> 56 #include <asm/pgtable.h> 57 58 #include "coalesced_mmio.h" 59 #include "async_pf.h" 60 61 #define CREATE_TRACE_POINTS 62 #include <trace/events/kvm.h> 63 64 MODULE_AUTHOR("Qumranet"); 65 MODULE_LICENSE("GPL"); 66 67 /* 68 * Ordering of locks: 69 * 70 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 71 */ 72 73 DEFINE_SPINLOCK(kvm_lock); 74 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 75 LIST_HEAD(vm_list); 76 77 static cpumask_var_t cpus_hardware_enabled; 78 static int kvm_usage_count = 0; 79 static atomic_t hardware_enable_failed; 80 81 struct kmem_cache *kvm_vcpu_cache; 82 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 83 84 static __read_mostly struct preempt_ops kvm_preempt_ops; 85 86 struct dentry *kvm_debugfs_dir; 87 88 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 89 unsigned long arg); 90 #ifdef CONFIG_COMPAT 91 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 92 unsigned long arg); 93 #endif 94 static int hardware_enable_all(void); 95 static void hardware_disable_all(void); 96 97 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 98 static void update_memslots(struct kvm_memslots *slots, 99 struct kvm_memory_slot *new, u64 last_generation); 100 101 static void kvm_release_pfn_dirty(pfn_t pfn); 102 static void mark_page_dirty_in_slot(struct kvm *kvm, 103 struct kvm_memory_slot *memslot, gfn_t gfn); 104 105 __visible bool kvm_rebooting; 106 EXPORT_SYMBOL_GPL(kvm_rebooting); 107 108 static bool largepages_enabled = true; 109 110 bool kvm_is_mmio_pfn(pfn_t pfn) 111 { 112 if (pfn_valid(pfn)) 113 return PageReserved(pfn_to_page(pfn)); 114 115 return true; 116 } 117 118 /* 119 * Switches to specified vcpu, until a matching vcpu_put() 120 */ 121 int vcpu_load(struct kvm_vcpu *vcpu) 122 { 123 int cpu; 124 125 if (mutex_lock_killable(&vcpu->mutex)) 126 return -EINTR; 127 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 128 /* The thread running this VCPU changed. */ 129 struct pid *oldpid = vcpu->pid; 130 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 131 rcu_assign_pointer(vcpu->pid, newpid); 132 synchronize_rcu(); 133 put_pid(oldpid); 134 } 135 cpu = get_cpu(); 136 preempt_notifier_register(&vcpu->preempt_notifier); 137 kvm_arch_vcpu_load(vcpu, cpu); 138 put_cpu(); 139 return 0; 140 } 141 142 void vcpu_put(struct kvm_vcpu *vcpu) 143 { 144 preempt_disable(); 145 kvm_arch_vcpu_put(vcpu); 146 preempt_notifier_unregister(&vcpu->preempt_notifier); 147 preempt_enable(); 148 mutex_unlock(&vcpu->mutex); 149 } 150 151 static void ack_flush(void *_completed) 152 { 153 } 154 155 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req) 156 { 157 int i, cpu, me; 158 cpumask_var_t cpus; 159 bool called = true; 160 struct kvm_vcpu *vcpu; 161 162 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 163 164 me = get_cpu(); 165 kvm_for_each_vcpu(i, vcpu, kvm) { 166 kvm_make_request(req, vcpu); 167 cpu = vcpu->cpu; 168 169 /* Set ->requests bit before we read ->mode */ 170 smp_mb(); 171 172 if (cpus != NULL && cpu != -1 && cpu != me && 173 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 174 cpumask_set_cpu(cpu, cpus); 175 } 176 if (unlikely(cpus == NULL)) 177 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 178 else if (!cpumask_empty(cpus)) 179 smp_call_function_many(cpus, ack_flush, NULL, 1); 180 else 181 called = false; 182 put_cpu(); 183 free_cpumask_var(cpus); 184 return called; 185 } 186 187 void kvm_flush_remote_tlbs(struct kvm *kvm) 188 { 189 long dirty_count = kvm->tlbs_dirty; 190 191 smp_mb(); 192 if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 193 ++kvm->stat.remote_tlb_flush; 194 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 195 } 196 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 197 198 void kvm_reload_remote_mmus(struct kvm *kvm) 199 { 200 make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 201 } 202 203 void kvm_make_mclock_inprogress_request(struct kvm *kvm) 204 { 205 make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 206 } 207 208 void kvm_make_scan_ioapic_request(struct kvm *kvm) 209 { 210 make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 211 } 212 213 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 214 { 215 struct page *page; 216 int r; 217 218 mutex_init(&vcpu->mutex); 219 vcpu->cpu = -1; 220 vcpu->kvm = kvm; 221 vcpu->vcpu_id = id; 222 vcpu->pid = NULL; 223 init_waitqueue_head(&vcpu->wq); 224 kvm_async_pf_vcpu_init(vcpu); 225 226 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 227 if (!page) { 228 r = -ENOMEM; 229 goto fail; 230 } 231 vcpu->run = page_address(page); 232 233 kvm_vcpu_set_in_spin_loop(vcpu, false); 234 kvm_vcpu_set_dy_eligible(vcpu, false); 235 vcpu->preempted = false; 236 237 r = kvm_arch_vcpu_init(vcpu); 238 if (r < 0) 239 goto fail_free_run; 240 return 0; 241 242 fail_free_run: 243 free_page((unsigned long)vcpu->run); 244 fail: 245 return r; 246 } 247 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 248 249 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 250 { 251 put_pid(vcpu->pid); 252 kvm_arch_vcpu_uninit(vcpu); 253 free_page((unsigned long)vcpu->run); 254 } 255 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 256 257 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 258 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 259 { 260 return container_of(mn, struct kvm, mmu_notifier); 261 } 262 263 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 264 struct mm_struct *mm, 265 unsigned long address) 266 { 267 struct kvm *kvm = mmu_notifier_to_kvm(mn); 268 int need_tlb_flush, idx; 269 270 /* 271 * When ->invalidate_page runs, the linux pte has been zapped 272 * already but the page is still allocated until 273 * ->invalidate_page returns. So if we increase the sequence 274 * here the kvm page fault will notice if the spte can't be 275 * established because the page is going to be freed. If 276 * instead the kvm page fault establishes the spte before 277 * ->invalidate_page runs, kvm_unmap_hva will release it 278 * before returning. 279 * 280 * The sequence increase only need to be seen at spin_unlock 281 * time, and not at spin_lock time. 282 * 283 * Increasing the sequence after the spin_unlock would be 284 * unsafe because the kvm page fault could then establish the 285 * pte after kvm_unmap_hva returned, without noticing the page 286 * is going to be freed. 287 */ 288 idx = srcu_read_lock(&kvm->srcu); 289 spin_lock(&kvm->mmu_lock); 290 291 kvm->mmu_notifier_seq++; 292 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 293 /* we've to flush the tlb before the pages can be freed */ 294 if (need_tlb_flush) 295 kvm_flush_remote_tlbs(kvm); 296 297 spin_unlock(&kvm->mmu_lock); 298 srcu_read_unlock(&kvm->srcu, idx); 299 } 300 301 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 302 struct mm_struct *mm, 303 unsigned long address, 304 pte_t pte) 305 { 306 struct kvm *kvm = mmu_notifier_to_kvm(mn); 307 int idx; 308 309 idx = srcu_read_lock(&kvm->srcu); 310 spin_lock(&kvm->mmu_lock); 311 kvm->mmu_notifier_seq++; 312 kvm_set_spte_hva(kvm, address, pte); 313 spin_unlock(&kvm->mmu_lock); 314 srcu_read_unlock(&kvm->srcu, idx); 315 } 316 317 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 318 struct mm_struct *mm, 319 unsigned long start, 320 unsigned long end) 321 { 322 struct kvm *kvm = mmu_notifier_to_kvm(mn); 323 int need_tlb_flush = 0, idx; 324 325 idx = srcu_read_lock(&kvm->srcu); 326 spin_lock(&kvm->mmu_lock); 327 /* 328 * The count increase must become visible at unlock time as no 329 * spte can be established without taking the mmu_lock and 330 * count is also read inside the mmu_lock critical section. 331 */ 332 kvm->mmu_notifier_count++; 333 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 334 need_tlb_flush |= kvm->tlbs_dirty; 335 /* we've to flush the tlb before the pages can be freed */ 336 if (need_tlb_flush) 337 kvm_flush_remote_tlbs(kvm); 338 339 spin_unlock(&kvm->mmu_lock); 340 srcu_read_unlock(&kvm->srcu, idx); 341 } 342 343 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 344 struct mm_struct *mm, 345 unsigned long start, 346 unsigned long end) 347 { 348 struct kvm *kvm = mmu_notifier_to_kvm(mn); 349 350 spin_lock(&kvm->mmu_lock); 351 /* 352 * This sequence increase will notify the kvm page fault that 353 * the page that is going to be mapped in the spte could have 354 * been freed. 355 */ 356 kvm->mmu_notifier_seq++; 357 smp_wmb(); 358 /* 359 * The above sequence increase must be visible before the 360 * below count decrease, which is ensured by the smp_wmb above 361 * in conjunction with the smp_rmb in mmu_notifier_retry(). 362 */ 363 kvm->mmu_notifier_count--; 364 spin_unlock(&kvm->mmu_lock); 365 366 BUG_ON(kvm->mmu_notifier_count < 0); 367 } 368 369 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 370 struct mm_struct *mm, 371 unsigned long address) 372 { 373 struct kvm *kvm = mmu_notifier_to_kvm(mn); 374 int young, idx; 375 376 idx = srcu_read_lock(&kvm->srcu); 377 spin_lock(&kvm->mmu_lock); 378 379 young = kvm_age_hva(kvm, address); 380 if (young) 381 kvm_flush_remote_tlbs(kvm); 382 383 spin_unlock(&kvm->mmu_lock); 384 srcu_read_unlock(&kvm->srcu, idx); 385 386 return young; 387 } 388 389 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 390 struct mm_struct *mm, 391 unsigned long address) 392 { 393 struct kvm *kvm = mmu_notifier_to_kvm(mn); 394 int young, idx; 395 396 idx = srcu_read_lock(&kvm->srcu); 397 spin_lock(&kvm->mmu_lock); 398 young = kvm_test_age_hva(kvm, address); 399 spin_unlock(&kvm->mmu_lock); 400 srcu_read_unlock(&kvm->srcu, idx); 401 402 return young; 403 } 404 405 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 406 struct mm_struct *mm) 407 { 408 struct kvm *kvm = mmu_notifier_to_kvm(mn); 409 int idx; 410 411 idx = srcu_read_lock(&kvm->srcu); 412 kvm_arch_flush_shadow_all(kvm); 413 srcu_read_unlock(&kvm->srcu, idx); 414 } 415 416 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 417 .invalidate_page = kvm_mmu_notifier_invalidate_page, 418 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 419 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 420 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 421 .test_young = kvm_mmu_notifier_test_young, 422 .change_pte = kvm_mmu_notifier_change_pte, 423 .release = kvm_mmu_notifier_release, 424 }; 425 426 static int kvm_init_mmu_notifier(struct kvm *kvm) 427 { 428 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 429 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 430 } 431 432 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 433 434 static int kvm_init_mmu_notifier(struct kvm *kvm) 435 { 436 return 0; 437 } 438 439 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 440 441 static void kvm_init_memslots_id(struct kvm *kvm) 442 { 443 int i; 444 struct kvm_memslots *slots = kvm->memslots; 445 446 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 447 slots->id_to_index[i] = slots->memslots[i].id = i; 448 } 449 450 static struct kvm *kvm_create_vm(unsigned long type) 451 { 452 int r, i; 453 struct kvm *kvm = kvm_arch_alloc_vm(); 454 455 if (!kvm) 456 return ERR_PTR(-ENOMEM); 457 458 r = kvm_arch_init_vm(kvm, type); 459 if (r) 460 goto out_err_no_disable; 461 462 r = hardware_enable_all(); 463 if (r) 464 goto out_err_no_disable; 465 466 #ifdef CONFIG_HAVE_KVM_IRQCHIP 467 INIT_HLIST_HEAD(&kvm->mask_notifier_list); 468 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 469 #endif 470 471 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 472 473 r = -ENOMEM; 474 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 475 if (!kvm->memslots) 476 goto out_err_no_srcu; 477 kvm_init_memslots_id(kvm); 478 if (init_srcu_struct(&kvm->srcu)) 479 goto out_err_no_srcu; 480 if (init_srcu_struct(&kvm->irq_srcu)) 481 goto out_err_no_irq_srcu; 482 for (i = 0; i < KVM_NR_BUSES; i++) { 483 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 484 GFP_KERNEL); 485 if (!kvm->buses[i]) 486 goto out_err; 487 } 488 489 spin_lock_init(&kvm->mmu_lock); 490 kvm->mm = current->mm; 491 atomic_inc(&kvm->mm->mm_count); 492 kvm_eventfd_init(kvm); 493 mutex_init(&kvm->lock); 494 mutex_init(&kvm->irq_lock); 495 mutex_init(&kvm->slots_lock); 496 atomic_set(&kvm->users_count, 1); 497 INIT_LIST_HEAD(&kvm->devices); 498 499 r = kvm_init_mmu_notifier(kvm); 500 if (r) 501 goto out_err; 502 503 spin_lock(&kvm_lock); 504 list_add(&kvm->vm_list, &vm_list); 505 spin_unlock(&kvm_lock); 506 507 return kvm; 508 509 out_err: 510 cleanup_srcu_struct(&kvm->irq_srcu); 511 out_err_no_irq_srcu: 512 cleanup_srcu_struct(&kvm->srcu); 513 out_err_no_srcu: 514 hardware_disable_all(); 515 out_err_no_disable: 516 for (i = 0; i < KVM_NR_BUSES; i++) 517 kfree(kvm->buses[i]); 518 kfree(kvm->memslots); 519 kvm_arch_free_vm(kvm); 520 return ERR_PTR(r); 521 } 522 523 /* 524 * Avoid using vmalloc for a small buffer. 525 * Should not be used when the size is statically known. 526 */ 527 void *kvm_kvzalloc(unsigned long size) 528 { 529 if (size > PAGE_SIZE) 530 return vzalloc(size); 531 else 532 return kzalloc(size, GFP_KERNEL); 533 } 534 535 void kvm_kvfree(const void *addr) 536 { 537 if (is_vmalloc_addr(addr)) 538 vfree(addr); 539 else 540 kfree(addr); 541 } 542 543 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 544 { 545 if (!memslot->dirty_bitmap) 546 return; 547 548 kvm_kvfree(memslot->dirty_bitmap); 549 memslot->dirty_bitmap = NULL; 550 } 551 552 /* 553 * Free any memory in @free but not in @dont. 554 */ 555 static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free, 556 struct kvm_memory_slot *dont) 557 { 558 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 559 kvm_destroy_dirty_bitmap(free); 560 561 kvm_arch_free_memslot(kvm, free, dont); 562 563 free->npages = 0; 564 } 565 566 static void kvm_free_physmem(struct kvm *kvm) 567 { 568 struct kvm_memslots *slots = kvm->memslots; 569 struct kvm_memory_slot *memslot; 570 571 kvm_for_each_memslot(memslot, slots) 572 kvm_free_physmem_slot(kvm, memslot, NULL); 573 574 kfree(kvm->memslots); 575 } 576 577 static void kvm_destroy_devices(struct kvm *kvm) 578 { 579 struct list_head *node, *tmp; 580 581 list_for_each_safe(node, tmp, &kvm->devices) { 582 struct kvm_device *dev = 583 list_entry(node, struct kvm_device, vm_node); 584 585 list_del(node); 586 dev->ops->destroy(dev); 587 } 588 } 589 590 static void kvm_destroy_vm(struct kvm *kvm) 591 { 592 int i; 593 struct mm_struct *mm = kvm->mm; 594 595 kvm_arch_sync_events(kvm); 596 spin_lock(&kvm_lock); 597 list_del(&kvm->vm_list); 598 spin_unlock(&kvm_lock); 599 kvm_free_irq_routing(kvm); 600 for (i = 0; i < KVM_NR_BUSES; i++) 601 kvm_io_bus_destroy(kvm->buses[i]); 602 kvm_coalesced_mmio_free(kvm); 603 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 604 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 605 #else 606 kvm_arch_flush_shadow_all(kvm); 607 #endif 608 kvm_arch_destroy_vm(kvm); 609 kvm_destroy_devices(kvm); 610 kvm_free_physmem(kvm); 611 cleanup_srcu_struct(&kvm->irq_srcu); 612 cleanup_srcu_struct(&kvm->srcu); 613 kvm_arch_free_vm(kvm); 614 hardware_disable_all(); 615 mmdrop(mm); 616 } 617 618 void kvm_get_kvm(struct kvm *kvm) 619 { 620 atomic_inc(&kvm->users_count); 621 } 622 EXPORT_SYMBOL_GPL(kvm_get_kvm); 623 624 void kvm_put_kvm(struct kvm *kvm) 625 { 626 if (atomic_dec_and_test(&kvm->users_count)) 627 kvm_destroy_vm(kvm); 628 } 629 EXPORT_SYMBOL_GPL(kvm_put_kvm); 630 631 632 static int kvm_vm_release(struct inode *inode, struct file *filp) 633 { 634 struct kvm *kvm = filp->private_data; 635 636 kvm_irqfd_release(kvm); 637 638 kvm_put_kvm(kvm); 639 return 0; 640 } 641 642 /* 643 * Allocation size is twice as large as the actual dirty bitmap size. 644 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 645 */ 646 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 647 { 648 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 649 650 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 651 if (!memslot->dirty_bitmap) 652 return -ENOMEM; 653 654 return 0; 655 } 656 657 static int cmp_memslot(const void *slot1, const void *slot2) 658 { 659 struct kvm_memory_slot *s1, *s2; 660 661 s1 = (struct kvm_memory_slot *)slot1; 662 s2 = (struct kvm_memory_slot *)slot2; 663 664 if (s1->npages < s2->npages) 665 return 1; 666 if (s1->npages > s2->npages) 667 return -1; 668 669 return 0; 670 } 671 672 /* 673 * Sort the memslots base on its size, so the larger slots 674 * will get better fit. 675 */ 676 static void sort_memslots(struct kvm_memslots *slots) 677 { 678 int i; 679 680 sort(slots->memslots, KVM_MEM_SLOTS_NUM, 681 sizeof(struct kvm_memory_slot), cmp_memslot, NULL); 682 683 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 684 slots->id_to_index[slots->memslots[i].id] = i; 685 } 686 687 static void update_memslots(struct kvm_memslots *slots, 688 struct kvm_memory_slot *new, 689 u64 last_generation) 690 { 691 if (new) { 692 int id = new->id; 693 struct kvm_memory_slot *old = id_to_memslot(slots, id); 694 unsigned long npages = old->npages; 695 696 *old = *new; 697 if (new->npages != npages) 698 sort_memslots(slots); 699 } 700 701 slots->generation = last_generation + 1; 702 } 703 704 static int check_memory_region_flags(struct kvm_userspace_memory_region *mem) 705 { 706 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 707 708 #ifdef KVM_CAP_READONLY_MEM 709 valid_flags |= KVM_MEM_READONLY; 710 #endif 711 712 if (mem->flags & ~valid_flags) 713 return -EINVAL; 714 715 return 0; 716 } 717 718 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 719 struct kvm_memslots *slots, struct kvm_memory_slot *new) 720 { 721 struct kvm_memslots *old_memslots = kvm->memslots; 722 723 update_memslots(slots, new, kvm->memslots->generation); 724 rcu_assign_pointer(kvm->memslots, slots); 725 synchronize_srcu_expedited(&kvm->srcu); 726 727 kvm_arch_memslots_updated(kvm); 728 729 return old_memslots; 730 } 731 732 /* 733 * Allocate some memory and give it an address in the guest physical address 734 * space. 735 * 736 * Discontiguous memory is allowed, mostly for framebuffers. 737 * 738 * Must be called holding mmap_sem for write. 739 */ 740 int __kvm_set_memory_region(struct kvm *kvm, 741 struct kvm_userspace_memory_region *mem) 742 { 743 int r; 744 gfn_t base_gfn; 745 unsigned long npages; 746 struct kvm_memory_slot *slot; 747 struct kvm_memory_slot old, new; 748 struct kvm_memslots *slots = NULL, *old_memslots; 749 enum kvm_mr_change change; 750 751 r = check_memory_region_flags(mem); 752 if (r) 753 goto out; 754 755 r = -EINVAL; 756 /* General sanity checks */ 757 if (mem->memory_size & (PAGE_SIZE - 1)) 758 goto out; 759 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 760 goto out; 761 /* We can read the guest memory with __xxx_user() later on. */ 762 if ((mem->slot < KVM_USER_MEM_SLOTS) && 763 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 764 !access_ok(VERIFY_WRITE, 765 (void __user *)(unsigned long)mem->userspace_addr, 766 mem->memory_size))) 767 goto out; 768 if (mem->slot >= KVM_MEM_SLOTS_NUM) 769 goto out; 770 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 771 goto out; 772 773 slot = id_to_memslot(kvm->memslots, mem->slot); 774 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 775 npages = mem->memory_size >> PAGE_SHIFT; 776 777 r = -EINVAL; 778 if (npages > KVM_MEM_MAX_NR_PAGES) 779 goto out; 780 781 if (!npages) 782 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; 783 784 new = old = *slot; 785 786 new.id = mem->slot; 787 new.base_gfn = base_gfn; 788 new.npages = npages; 789 new.flags = mem->flags; 790 791 r = -EINVAL; 792 if (npages) { 793 if (!old.npages) 794 change = KVM_MR_CREATE; 795 else { /* Modify an existing slot. */ 796 if ((mem->userspace_addr != old.userspace_addr) || 797 (npages != old.npages) || 798 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 799 goto out; 800 801 if (base_gfn != old.base_gfn) 802 change = KVM_MR_MOVE; 803 else if (new.flags != old.flags) 804 change = KVM_MR_FLAGS_ONLY; 805 else { /* Nothing to change. */ 806 r = 0; 807 goto out; 808 } 809 } 810 } else if (old.npages) { 811 change = KVM_MR_DELETE; 812 } else /* Modify a non-existent slot: disallowed. */ 813 goto out; 814 815 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 816 /* Check for overlaps */ 817 r = -EEXIST; 818 kvm_for_each_memslot(slot, kvm->memslots) { 819 if ((slot->id >= KVM_USER_MEM_SLOTS) || 820 (slot->id == mem->slot)) 821 continue; 822 if (!((base_gfn + npages <= slot->base_gfn) || 823 (base_gfn >= slot->base_gfn + slot->npages))) 824 goto out; 825 } 826 } 827 828 /* Free page dirty bitmap if unneeded */ 829 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 830 new.dirty_bitmap = NULL; 831 832 r = -ENOMEM; 833 if (change == KVM_MR_CREATE) { 834 new.userspace_addr = mem->userspace_addr; 835 836 if (kvm_arch_create_memslot(kvm, &new, npages)) 837 goto out_free; 838 } 839 840 /* Allocate page dirty bitmap if needed */ 841 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 842 if (kvm_create_dirty_bitmap(&new) < 0) 843 goto out_free; 844 } 845 846 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 847 r = -ENOMEM; 848 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 849 GFP_KERNEL); 850 if (!slots) 851 goto out_free; 852 slot = id_to_memslot(slots, mem->slot); 853 slot->flags |= KVM_MEMSLOT_INVALID; 854 855 old_memslots = install_new_memslots(kvm, slots, NULL); 856 857 /* slot was deleted or moved, clear iommu mapping */ 858 kvm_iommu_unmap_pages(kvm, &old); 859 /* From this point no new shadow pages pointing to a deleted, 860 * or moved, memslot will be created. 861 * 862 * validation of sp->gfn happens in: 863 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 864 * - kvm_is_visible_gfn (mmu_check_roots) 865 */ 866 kvm_arch_flush_shadow_memslot(kvm, slot); 867 slots = old_memslots; 868 } 869 870 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 871 if (r) 872 goto out_slots; 873 874 r = -ENOMEM; 875 /* 876 * We can re-use the old_memslots from above, the only difference 877 * from the currently installed memslots is the invalid flag. This 878 * will get overwritten by update_memslots anyway. 879 */ 880 if (!slots) { 881 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 882 GFP_KERNEL); 883 if (!slots) 884 goto out_free; 885 } 886 887 /* actual memory is freed via old in kvm_free_physmem_slot below */ 888 if (change == KVM_MR_DELETE) { 889 new.dirty_bitmap = NULL; 890 memset(&new.arch, 0, sizeof(new.arch)); 891 } 892 893 old_memslots = install_new_memslots(kvm, slots, &new); 894 895 kvm_arch_commit_memory_region(kvm, mem, &old, change); 896 897 kvm_free_physmem_slot(kvm, &old, &new); 898 kfree(old_memslots); 899 900 /* 901 * IOMMU mapping: New slots need to be mapped. Old slots need to be 902 * un-mapped and re-mapped if their base changes. Since base change 903 * unmapping is handled above with slot deletion, mapping alone is 904 * needed here. Anything else the iommu might care about for existing 905 * slots (size changes, userspace addr changes and read-only flag 906 * changes) is disallowed above, so any other attribute changes getting 907 * here can be skipped. 908 */ 909 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 910 r = kvm_iommu_map_pages(kvm, &new); 911 return r; 912 } 913 914 return 0; 915 916 out_slots: 917 kfree(slots); 918 out_free: 919 kvm_free_physmem_slot(kvm, &new, &old); 920 out: 921 return r; 922 } 923 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 924 925 int kvm_set_memory_region(struct kvm *kvm, 926 struct kvm_userspace_memory_region *mem) 927 { 928 int r; 929 930 mutex_lock(&kvm->slots_lock); 931 r = __kvm_set_memory_region(kvm, mem); 932 mutex_unlock(&kvm->slots_lock); 933 return r; 934 } 935 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 936 937 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 938 struct kvm_userspace_memory_region *mem) 939 { 940 if (mem->slot >= KVM_USER_MEM_SLOTS) 941 return -EINVAL; 942 return kvm_set_memory_region(kvm, mem); 943 } 944 945 int kvm_get_dirty_log(struct kvm *kvm, 946 struct kvm_dirty_log *log, int *is_dirty) 947 { 948 struct kvm_memory_slot *memslot; 949 int r, i; 950 unsigned long n; 951 unsigned long any = 0; 952 953 r = -EINVAL; 954 if (log->slot >= KVM_USER_MEM_SLOTS) 955 goto out; 956 957 memslot = id_to_memslot(kvm->memslots, log->slot); 958 r = -ENOENT; 959 if (!memslot->dirty_bitmap) 960 goto out; 961 962 n = kvm_dirty_bitmap_bytes(memslot); 963 964 for (i = 0; !any && i < n/sizeof(long); ++i) 965 any = memslot->dirty_bitmap[i]; 966 967 r = -EFAULT; 968 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 969 goto out; 970 971 if (any) 972 *is_dirty = 1; 973 974 r = 0; 975 out: 976 return r; 977 } 978 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 979 980 bool kvm_largepages_enabled(void) 981 { 982 return largepages_enabled; 983 } 984 985 void kvm_disable_largepages(void) 986 { 987 largepages_enabled = false; 988 } 989 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 990 991 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 992 { 993 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 994 } 995 EXPORT_SYMBOL_GPL(gfn_to_memslot); 996 997 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 998 { 999 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1000 1001 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1002 memslot->flags & KVM_MEMSLOT_INVALID) 1003 return 0; 1004 1005 return 1; 1006 } 1007 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1008 1009 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1010 { 1011 struct vm_area_struct *vma; 1012 unsigned long addr, size; 1013 1014 size = PAGE_SIZE; 1015 1016 addr = gfn_to_hva(kvm, gfn); 1017 if (kvm_is_error_hva(addr)) 1018 return PAGE_SIZE; 1019 1020 down_read(¤t->mm->mmap_sem); 1021 vma = find_vma(current->mm, addr); 1022 if (!vma) 1023 goto out; 1024 1025 size = vma_kernel_pagesize(vma); 1026 1027 out: 1028 up_read(¤t->mm->mmap_sem); 1029 1030 return size; 1031 } 1032 1033 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1034 { 1035 return slot->flags & KVM_MEM_READONLY; 1036 } 1037 1038 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1039 gfn_t *nr_pages, bool write) 1040 { 1041 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1042 return KVM_HVA_ERR_BAD; 1043 1044 if (memslot_is_readonly(slot) && write) 1045 return KVM_HVA_ERR_RO_BAD; 1046 1047 if (nr_pages) 1048 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1049 1050 return __gfn_to_hva_memslot(slot, gfn); 1051 } 1052 1053 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1054 gfn_t *nr_pages) 1055 { 1056 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1057 } 1058 1059 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1060 gfn_t gfn) 1061 { 1062 return gfn_to_hva_many(slot, gfn, NULL); 1063 } 1064 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1065 1066 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1067 { 1068 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1069 } 1070 EXPORT_SYMBOL_GPL(gfn_to_hva); 1071 1072 /* 1073 * If writable is set to false, the hva returned by this function is only 1074 * allowed to be read. 1075 */ 1076 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1077 { 1078 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1079 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1080 1081 if (!kvm_is_error_hva(hva) && writable) 1082 *writable = !memslot_is_readonly(slot); 1083 1084 return hva; 1085 } 1086 1087 static int kvm_read_hva(void *data, void __user *hva, int len) 1088 { 1089 return __copy_from_user(data, hva, len); 1090 } 1091 1092 static int kvm_read_hva_atomic(void *data, void __user *hva, int len) 1093 { 1094 return __copy_from_user_inatomic(data, hva, len); 1095 } 1096 1097 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1098 unsigned long start, int write, struct page **page) 1099 { 1100 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1101 1102 if (write) 1103 flags |= FOLL_WRITE; 1104 1105 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1106 } 1107 1108 static inline int check_user_page_hwpoison(unsigned long addr) 1109 { 1110 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1111 1112 rc = __get_user_pages(current, current->mm, addr, 1, 1113 flags, NULL, NULL, NULL); 1114 return rc == -EHWPOISON; 1115 } 1116 1117 /* 1118 * The atomic path to get the writable pfn which will be stored in @pfn, 1119 * true indicates success, otherwise false is returned. 1120 */ 1121 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1122 bool write_fault, bool *writable, pfn_t *pfn) 1123 { 1124 struct page *page[1]; 1125 int npages; 1126 1127 if (!(async || atomic)) 1128 return false; 1129 1130 /* 1131 * Fast pin a writable pfn only if it is a write fault request 1132 * or the caller allows to map a writable pfn for a read fault 1133 * request. 1134 */ 1135 if (!(write_fault || writable)) 1136 return false; 1137 1138 npages = __get_user_pages_fast(addr, 1, 1, page); 1139 if (npages == 1) { 1140 *pfn = page_to_pfn(page[0]); 1141 1142 if (writable) 1143 *writable = true; 1144 return true; 1145 } 1146 1147 return false; 1148 } 1149 1150 /* 1151 * The slow path to get the pfn of the specified host virtual address, 1152 * 1 indicates success, -errno is returned if error is detected. 1153 */ 1154 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1155 bool *writable, pfn_t *pfn) 1156 { 1157 struct page *page[1]; 1158 int npages = 0; 1159 1160 might_sleep(); 1161 1162 if (writable) 1163 *writable = write_fault; 1164 1165 if (async) { 1166 down_read(¤t->mm->mmap_sem); 1167 npages = get_user_page_nowait(current, current->mm, 1168 addr, write_fault, page); 1169 up_read(¤t->mm->mmap_sem); 1170 } else 1171 npages = get_user_pages_fast(addr, 1, write_fault, 1172 page); 1173 if (npages != 1) 1174 return npages; 1175 1176 /* map read fault as writable if possible */ 1177 if (unlikely(!write_fault) && writable) { 1178 struct page *wpage[1]; 1179 1180 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1181 if (npages == 1) { 1182 *writable = true; 1183 put_page(page[0]); 1184 page[0] = wpage[0]; 1185 } 1186 1187 npages = 1; 1188 } 1189 *pfn = page_to_pfn(page[0]); 1190 return npages; 1191 } 1192 1193 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1194 { 1195 if (unlikely(!(vma->vm_flags & VM_READ))) 1196 return false; 1197 1198 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1199 return false; 1200 1201 return true; 1202 } 1203 1204 /* 1205 * Pin guest page in memory and return its pfn. 1206 * @addr: host virtual address which maps memory to the guest 1207 * @atomic: whether this function can sleep 1208 * @async: whether this function need to wait IO complete if the 1209 * host page is not in the memory 1210 * @write_fault: whether we should get a writable host page 1211 * @writable: whether it allows to map a writable host page for !@write_fault 1212 * 1213 * The function will map a writable host page for these two cases: 1214 * 1): @write_fault = true 1215 * 2): @write_fault = false && @writable, @writable will tell the caller 1216 * whether the mapping is writable. 1217 */ 1218 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1219 bool write_fault, bool *writable) 1220 { 1221 struct vm_area_struct *vma; 1222 pfn_t pfn = 0; 1223 int npages; 1224 1225 /* we can do it either atomically or asynchronously, not both */ 1226 BUG_ON(atomic && async); 1227 1228 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1229 return pfn; 1230 1231 if (atomic) 1232 return KVM_PFN_ERR_FAULT; 1233 1234 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1235 if (npages == 1) 1236 return pfn; 1237 1238 down_read(¤t->mm->mmap_sem); 1239 if (npages == -EHWPOISON || 1240 (!async && check_user_page_hwpoison(addr))) { 1241 pfn = KVM_PFN_ERR_HWPOISON; 1242 goto exit; 1243 } 1244 1245 vma = find_vma_intersection(current->mm, addr, addr + 1); 1246 1247 if (vma == NULL) 1248 pfn = KVM_PFN_ERR_FAULT; 1249 else if ((vma->vm_flags & VM_PFNMAP)) { 1250 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1251 vma->vm_pgoff; 1252 BUG_ON(!kvm_is_mmio_pfn(pfn)); 1253 } else { 1254 if (async && vma_is_valid(vma, write_fault)) 1255 *async = true; 1256 pfn = KVM_PFN_ERR_FAULT; 1257 } 1258 exit: 1259 up_read(¤t->mm->mmap_sem); 1260 return pfn; 1261 } 1262 1263 static pfn_t 1264 __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic, 1265 bool *async, bool write_fault, bool *writable) 1266 { 1267 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1268 1269 if (addr == KVM_HVA_ERR_RO_BAD) 1270 return KVM_PFN_ERR_RO_FAULT; 1271 1272 if (kvm_is_error_hva(addr)) 1273 return KVM_PFN_NOSLOT; 1274 1275 /* Do not map writable pfn in the readonly memslot. */ 1276 if (writable && memslot_is_readonly(slot)) { 1277 *writable = false; 1278 writable = NULL; 1279 } 1280 1281 return hva_to_pfn(addr, atomic, async, write_fault, 1282 writable); 1283 } 1284 1285 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async, 1286 bool write_fault, bool *writable) 1287 { 1288 struct kvm_memory_slot *slot; 1289 1290 if (async) 1291 *async = false; 1292 1293 slot = gfn_to_memslot(kvm, gfn); 1294 1295 return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault, 1296 writable); 1297 } 1298 1299 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1300 { 1301 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL); 1302 } 1303 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1304 1305 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async, 1306 bool write_fault, bool *writable) 1307 { 1308 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable); 1309 } 1310 EXPORT_SYMBOL_GPL(gfn_to_pfn_async); 1311 1312 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1313 { 1314 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL); 1315 } 1316 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1317 1318 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1319 bool *writable) 1320 { 1321 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable); 1322 } 1323 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1324 1325 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1326 { 1327 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1328 } 1329 1330 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1331 { 1332 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1333 } 1334 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1335 1336 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages, 1337 int nr_pages) 1338 { 1339 unsigned long addr; 1340 gfn_t entry; 1341 1342 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry); 1343 if (kvm_is_error_hva(addr)) 1344 return -1; 1345 1346 if (entry < nr_pages) 1347 return 0; 1348 1349 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1350 } 1351 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1352 1353 static struct page *kvm_pfn_to_page(pfn_t pfn) 1354 { 1355 if (is_error_noslot_pfn(pfn)) 1356 return KVM_ERR_PTR_BAD_PAGE; 1357 1358 if (kvm_is_mmio_pfn(pfn)) { 1359 WARN_ON(1); 1360 return KVM_ERR_PTR_BAD_PAGE; 1361 } 1362 1363 return pfn_to_page(pfn); 1364 } 1365 1366 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1367 { 1368 pfn_t pfn; 1369 1370 pfn = gfn_to_pfn(kvm, gfn); 1371 1372 return kvm_pfn_to_page(pfn); 1373 } 1374 1375 EXPORT_SYMBOL_GPL(gfn_to_page); 1376 1377 void kvm_release_page_clean(struct page *page) 1378 { 1379 WARN_ON(is_error_page(page)); 1380 1381 kvm_release_pfn_clean(page_to_pfn(page)); 1382 } 1383 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1384 1385 void kvm_release_pfn_clean(pfn_t pfn) 1386 { 1387 if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn)) 1388 put_page(pfn_to_page(pfn)); 1389 } 1390 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1391 1392 void kvm_release_page_dirty(struct page *page) 1393 { 1394 WARN_ON(is_error_page(page)); 1395 1396 kvm_release_pfn_dirty(page_to_pfn(page)); 1397 } 1398 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1399 1400 static void kvm_release_pfn_dirty(pfn_t pfn) 1401 { 1402 kvm_set_pfn_dirty(pfn); 1403 kvm_release_pfn_clean(pfn); 1404 } 1405 1406 void kvm_set_pfn_dirty(pfn_t pfn) 1407 { 1408 if (!kvm_is_mmio_pfn(pfn)) { 1409 struct page *page = pfn_to_page(pfn); 1410 if (!PageReserved(page)) 1411 SetPageDirty(page); 1412 } 1413 } 1414 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1415 1416 void kvm_set_pfn_accessed(pfn_t pfn) 1417 { 1418 if (!kvm_is_mmio_pfn(pfn)) 1419 mark_page_accessed(pfn_to_page(pfn)); 1420 } 1421 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1422 1423 void kvm_get_pfn(pfn_t pfn) 1424 { 1425 if (!kvm_is_mmio_pfn(pfn)) 1426 get_page(pfn_to_page(pfn)); 1427 } 1428 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1429 1430 static int next_segment(unsigned long len, int offset) 1431 { 1432 if (len > PAGE_SIZE - offset) 1433 return PAGE_SIZE - offset; 1434 else 1435 return len; 1436 } 1437 1438 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1439 int len) 1440 { 1441 int r; 1442 unsigned long addr; 1443 1444 addr = gfn_to_hva_prot(kvm, gfn, NULL); 1445 if (kvm_is_error_hva(addr)) 1446 return -EFAULT; 1447 r = kvm_read_hva(data, (void __user *)addr + offset, len); 1448 if (r) 1449 return -EFAULT; 1450 return 0; 1451 } 1452 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1453 1454 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1455 { 1456 gfn_t gfn = gpa >> PAGE_SHIFT; 1457 int seg; 1458 int offset = offset_in_page(gpa); 1459 int ret; 1460 1461 while ((seg = next_segment(len, offset)) != 0) { 1462 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1463 if (ret < 0) 1464 return ret; 1465 offset = 0; 1466 len -= seg; 1467 data += seg; 1468 ++gfn; 1469 } 1470 return 0; 1471 } 1472 EXPORT_SYMBOL_GPL(kvm_read_guest); 1473 1474 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1475 unsigned long len) 1476 { 1477 int r; 1478 unsigned long addr; 1479 gfn_t gfn = gpa >> PAGE_SHIFT; 1480 int offset = offset_in_page(gpa); 1481 1482 addr = gfn_to_hva_prot(kvm, gfn, NULL); 1483 if (kvm_is_error_hva(addr)) 1484 return -EFAULT; 1485 pagefault_disable(); 1486 r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len); 1487 pagefault_enable(); 1488 if (r) 1489 return -EFAULT; 1490 return 0; 1491 } 1492 EXPORT_SYMBOL(kvm_read_guest_atomic); 1493 1494 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1495 int offset, int len) 1496 { 1497 int r; 1498 unsigned long addr; 1499 1500 addr = gfn_to_hva(kvm, gfn); 1501 if (kvm_is_error_hva(addr)) 1502 return -EFAULT; 1503 r = __copy_to_user((void __user *)addr + offset, data, len); 1504 if (r) 1505 return -EFAULT; 1506 mark_page_dirty(kvm, gfn); 1507 return 0; 1508 } 1509 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1510 1511 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1512 unsigned long len) 1513 { 1514 gfn_t gfn = gpa >> PAGE_SHIFT; 1515 int seg; 1516 int offset = offset_in_page(gpa); 1517 int ret; 1518 1519 while ((seg = next_segment(len, offset)) != 0) { 1520 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1521 if (ret < 0) 1522 return ret; 1523 offset = 0; 1524 len -= seg; 1525 data += seg; 1526 ++gfn; 1527 } 1528 return 0; 1529 } 1530 1531 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1532 gpa_t gpa, unsigned long len) 1533 { 1534 struct kvm_memslots *slots = kvm_memslots(kvm); 1535 int offset = offset_in_page(gpa); 1536 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1537 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1538 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1539 gfn_t nr_pages_avail; 1540 1541 ghc->gpa = gpa; 1542 ghc->generation = slots->generation; 1543 ghc->len = len; 1544 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1545 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail); 1546 if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) { 1547 ghc->hva += offset; 1548 } else { 1549 /* 1550 * If the requested region crosses two memslots, we still 1551 * verify that the entire region is valid here. 1552 */ 1553 while (start_gfn <= end_gfn) { 1554 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1555 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1556 &nr_pages_avail); 1557 if (kvm_is_error_hva(ghc->hva)) 1558 return -EFAULT; 1559 start_gfn += nr_pages_avail; 1560 } 1561 /* Use the slow path for cross page reads and writes. */ 1562 ghc->memslot = NULL; 1563 } 1564 return 0; 1565 } 1566 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1567 1568 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1569 void *data, unsigned long len) 1570 { 1571 struct kvm_memslots *slots = kvm_memslots(kvm); 1572 int r; 1573 1574 BUG_ON(len > ghc->len); 1575 1576 if (slots->generation != ghc->generation) 1577 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1578 1579 if (unlikely(!ghc->memslot)) 1580 return kvm_write_guest(kvm, ghc->gpa, data, len); 1581 1582 if (kvm_is_error_hva(ghc->hva)) 1583 return -EFAULT; 1584 1585 r = __copy_to_user((void __user *)ghc->hva, data, len); 1586 if (r) 1587 return -EFAULT; 1588 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1589 1590 return 0; 1591 } 1592 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1593 1594 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1595 void *data, unsigned long len) 1596 { 1597 struct kvm_memslots *slots = kvm_memslots(kvm); 1598 int r; 1599 1600 BUG_ON(len > ghc->len); 1601 1602 if (slots->generation != ghc->generation) 1603 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1604 1605 if (unlikely(!ghc->memslot)) 1606 return kvm_read_guest(kvm, ghc->gpa, data, len); 1607 1608 if (kvm_is_error_hva(ghc->hva)) 1609 return -EFAULT; 1610 1611 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1612 if (r) 1613 return -EFAULT; 1614 1615 return 0; 1616 } 1617 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1618 1619 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1620 { 1621 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1622 1623 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1624 } 1625 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1626 1627 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1628 { 1629 gfn_t gfn = gpa >> PAGE_SHIFT; 1630 int seg; 1631 int offset = offset_in_page(gpa); 1632 int ret; 1633 1634 while ((seg = next_segment(len, offset)) != 0) { 1635 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1636 if (ret < 0) 1637 return ret; 1638 offset = 0; 1639 len -= seg; 1640 ++gfn; 1641 } 1642 return 0; 1643 } 1644 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1645 1646 static void mark_page_dirty_in_slot(struct kvm *kvm, 1647 struct kvm_memory_slot *memslot, 1648 gfn_t gfn) 1649 { 1650 if (memslot && memslot->dirty_bitmap) { 1651 unsigned long rel_gfn = gfn - memslot->base_gfn; 1652 1653 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1654 } 1655 } 1656 1657 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1658 { 1659 struct kvm_memory_slot *memslot; 1660 1661 memslot = gfn_to_memslot(kvm, gfn); 1662 mark_page_dirty_in_slot(kvm, memslot, gfn); 1663 } 1664 EXPORT_SYMBOL_GPL(mark_page_dirty); 1665 1666 /* 1667 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1668 */ 1669 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1670 { 1671 DEFINE_WAIT(wait); 1672 1673 for (;;) { 1674 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1675 1676 if (kvm_arch_vcpu_runnable(vcpu)) { 1677 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1678 break; 1679 } 1680 if (kvm_cpu_has_pending_timer(vcpu)) 1681 break; 1682 if (signal_pending(current)) 1683 break; 1684 1685 schedule(); 1686 } 1687 1688 finish_wait(&vcpu->wq, &wait); 1689 } 1690 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 1691 1692 #ifndef CONFIG_S390 1693 /* 1694 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 1695 */ 1696 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 1697 { 1698 int me; 1699 int cpu = vcpu->cpu; 1700 wait_queue_head_t *wqp; 1701 1702 wqp = kvm_arch_vcpu_wq(vcpu); 1703 if (waitqueue_active(wqp)) { 1704 wake_up_interruptible(wqp); 1705 ++vcpu->stat.halt_wakeup; 1706 } 1707 1708 me = get_cpu(); 1709 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 1710 if (kvm_arch_vcpu_should_kick(vcpu)) 1711 smp_send_reschedule(cpu); 1712 put_cpu(); 1713 } 1714 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 1715 #endif /* !CONFIG_S390 */ 1716 1717 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 1718 { 1719 struct pid *pid; 1720 struct task_struct *task = NULL; 1721 int ret = 0; 1722 1723 rcu_read_lock(); 1724 pid = rcu_dereference(target->pid); 1725 if (pid) 1726 task = get_pid_task(target->pid, PIDTYPE_PID); 1727 rcu_read_unlock(); 1728 if (!task) 1729 return ret; 1730 if (task->flags & PF_VCPU) { 1731 put_task_struct(task); 1732 return ret; 1733 } 1734 ret = yield_to(task, 1); 1735 put_task_struct(task); 1736 1737 return ret; 1738 } 1739 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 1740 1741 /* 1742 * Helper that checks whether a VCPU is eligible for directed yield. 1743 * Most eligible candidate to yield is decided by following heuristics: 1744 * 1745 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 1746 * (preempted lock holder), indicated by @in_spin_loop. 1747 * Set at the beiginning and cleared at the end of interception/PLE handler. 1748 * 1749 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 1750 * chance last time (mostly it has become eligible now since we have probably 1751 * yielded to lockholder in last iteration. This is done by toggling 1752 * @dy_eligible each time a VCPU checked for eligibility.) 1753 * 1754 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 1755 * to preempted lock-holder could result in wrong VCPU selection and CPU 1756 * burning. Giving priority for a potential lock-holder increases lock 1757 * progress. 1758 * 1759 * Since algorithm is based on heuristics, accessing another VCPU data without 1760 * locking does not harm. It may result in trying to yield to same VCPU, fail 1761 * and continue with next VCPU and so on. 1762 */ 1763 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 1764 { 1765 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 1766 bool eligible; 1767 1768 eligible = !vcpu->spin_loop.in_spin_loop || 1769 (vcpu->spin_loop.in_spin_loop && 1770 vcpu->spin_loop.dy_eligible); 1771 1772 if (vcpu->spin_loop.in_spin_loop) 1773 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 1774 1775 return eligible; 1776 #else 1777 return true; 1778 #endif 1779 } 1780 1781 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 1782 { 1783 struct kvm *kvm = me->kvm; 1784 struct kvm_vcpu *vcpu; 1785 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 1786 int yielded = 0; 1787 int try = 3; 1788 int pass; 1789 int i; 1790 1791 kvm_vcpu_set_in_spin_loop(me, true); 1792 /* 1793 * We boost the priority of a VCPU that is runnable but not 1794 * currently running, because it got preempted by something 1795 * else and called schedule in __vcpu_run. Hopefully that 1796 * VCPU is holding the lock that we need and will release it. 1797 * We approximate round-robin by starting at the last boosted VCPU. 1798 */ 1799 for (pass = 0; pass < 2 && !yielded && try; pass++) { 1800 kvm_for_each_vcpu(i, vcpu, kvm) { 1801 if (!pass && i <= last_boosted_vcpu) { 1802 i = last_boosted_vcpu; 1803 continue; 1804 } else if (pass && i > last_boosted_vcpu) 1805 break; 1806 if (!ACCESS_ONCE(vcpu->preempted)) 1807 continue; 1808 if (vcpu == me) 1809 continue; 1810 if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 1811 continue; 1812 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 1813 continue; 1814 1815 yielded = kvm_vcpu_yield_to(vcpu); 1816 if (yielded > 0) { 1817 kvm->last_boosted_vcpu = i; 1818 break; 1819 } else if (yielded < 0) { 1820 try--; 1821 if (!try) 1822 break; 1823 } 1824 } 1825 } 1826 kvm_vcpu_set_in_spin_loop(me, false); 1827 1828 /* Ensure vcpu is not eligible during next spinloop */ 1829 kvm_vcpu_set_dy_eligible(me, false); 1830 } 1831 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 1832 1833 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1834 { 1835 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 1836 struct page *page; 1837 1838 if (vmf->pgoff == 0) 1839 page = virt_to_page(vcpu->run); 1840 #ifdef CONFIG_X86 1841 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 1842 page = virt_to_page(vcpu->arch.pio_data); 1843 #endif 1844 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1845 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 1846 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 1847 #endif 1848 else 1849 return kvm_arch_vcpu_fault(vcpu, vmf); 1850 get_page(page); 1851 vmf->page = page; 1852 return 0; 1853 } 1854 1855 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 1856 .fault = kvm_vcpu_fault, 1857 }; 1858 1859 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 1860 { 1861 vma->vm_ops = &kvm_vcpu_vm_ops; 1862 return 0; 1863 } 1864 1865 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 1866 { 1867 struct kvm_vcpu *vcpu = filp->private_data; 1868 1869 kvm_put_kvm(vcpu->kvm); 1870 return 0; 1871 } 1872 1873 static struct file_operations kvm_vcpu_fops = { 1874 .release = kvm_vcpu_release, 1875 .unlocked_ioctl = kvm_vcpu_ioctl, 1876 #ifdef CONFIG_COMPAT 1877 .compat_ioctl = kvm_vcpu_compat_ioctl, 1878 #endif 1879 .mmap = kvm_vcpu_mmap, 1880 .llseek = noop_llseek, 1881 }; 1882 1883 /* 1884 * Allocates an inode for the vcpu. 1885 */ 1886 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 1887 { 1888 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 1889 } 1890 1891 /* 1892 * Creates some virtual cpus. Good luck creating more than one. 1893 */ 1894 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 1895 { 1896 int r; 1897 struct kvm_vcpu *vcpu, *v; 1898 1899 if (id >= KVM_MAX_VCPUS) 1900 return -EINVAL; 1901 1902 vcpu = kvm_arch_vcpu_create(kvm, id); 1903 if (IS_ERR(vcpu)) 1904 return PTR_ERR(vcpu); 1905 1906 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 1907 1908 r = kvm_arch_vcpu_setup(vcpu); 1909 if (r) 1910 goto vcpu_destroy; 1911 1912 mutex_lock(&kvm->lock); 1913 if (!kvm_vcpu_compatible(vcpu)) { 1914 r = -EINVAL; 1915 goto unlock_vcpu_destroy; 1916 } 1917 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 1918 r = -EINVAL; 1919 goto unlock_vcpu_destroy; 1920 } 1921 1922 kvm_for_each_vcpu(r, v, kvm) 1923 if (v->vcpu_id == id) { 1924 r = -EEXIST; 1925 goto unlock_vcpu_destroy; 1926 } 1927 1928 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 1929 1930 /* Now it's all set up, let userspace reach it */ 1931 kvm_get_kvm(kvm); 1932 r = create_vcpu_fd(vcpu); 1933 if (r < 0) { 1934 kvm_put_kvm(kvm); 1935 goto unlock_vcpu_destroy; 1936 } 1937 1938 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 1939 smp_wmb(); 1940 atomic_inc(&kvm->online_vcpus); 1941 1942 mutex_unlock(&kvm->lock); 1943 kvm_arch_vcpu_postcreate(vcpu); 1944 return r; 1945 1946 unlock_vcpu_destroy: 1947 mutex_unlock(&kvm->lock); 1948 vcpu_destroy: 1949 kvm_arch_vcpu_destroy(vcpu); 1950 return r; 1951 } 1952 1953 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 1954 { 1955 if (sigset) { 1956 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 1957 vcpu->sigset_active = 1; 1958 vcpu->sigset = *sigset; 1959 } else 1960 vcpu->sigset_active = 0; 1961 return 0; 1962 } 1963 1964 static long kvm_vcpu_ioctl(struct file *filp, 1965 unsigned int ioctl, unsigned long arg) 1966 { 1967 struct kvm_vcpu *vcpu = filp->private_data; 1968 void __user *argp = (void __user *)arg; 1969 int r; 1970 struct kvm_fpu *fpu = NULL; 1971 struct kvm_sregs *kvm_sregs = NULL; 1972 1973 if (vcpu->kvm->mm != current->mm) 1974 return -EIO; 1975 1976 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 1977 /* 1978 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 1979 * so vcpu_load() would break it. 1980 */ 1981 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT) 1982 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1983 #endif 1984 1985 1986 r = vcpu_load(vcpu); 1987 if (r) 1988 return r; 1989 switch (ioctl) { 1990 case KVM_RUN: 1991 r = -EINVAL; 1992 if (arg) 1993 goto out; 1994 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 1995 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 1996 break; 1997 case KVM_GET_REGS: { 1998 struct kvm_regs *kvm_regs; 1999 2000 r = -ENOMEM; 2001 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2002 if (!kvm_regs) 2003 goto out; 2004 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2005 if (r) 2006 goto out_free1; 2007 r = -EFAULT; 2008 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2009 goto out_free1; 2010 r = 0; 2011 out_free1: 2012 kfree(kvm_regs); 2013 break; 2014 } 2015 case KVM_SET_REGS: { 2016 struct kvm_regs *kvm_regs; 2017 2018 r = -ENOMEM; 2019 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2020 if (IS_ERR(kvm_regs)) { 2021 r = PTR_ERR(kvm_regs); 2022 goto out; 2023 } 2024 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2025 kfree(kvm_regs); 2026 break; 2027 } 2028 case KVM_GET_SREGS: { 2029 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2030 r = -ENOMEM; 2031 if (!kvm_sregs) 2032 goto out; 2033 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2034 if (r) 2035 goto out; 2036 r = -EFAULT; 2037 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2038 goto out; 2039 r = 0; 2040 break; 2041 } 2042 case KVM_SET_SREGS: { 2043 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2044 if (IS_ERR(kvm_sregs)) { 2045 r = PTR_ERR(kvm_sregs); 2046 kvm_sregs = NULL; 2047 goto out; 2048 } 2049 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2050 break; 2051 } 2052 case KVM_GET_MP_STATE: { 2053 struct kvm_mp_state mp_state; 2054 2055 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2056 if (r) 2057 goto out; 2058 r = -EFAULT; 2059 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 2060 goto out; 2061 r = 0; 2062 break; 2063 } 2064 case KVM_SET_MP_STATE: { 2065 struct kvm_mp_state mp_state; 2066 2067 r = -EFAULT; 2068 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 2069 goto out; 2070 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2071 break; 2072 } 2073 case KVM_TRANSLATE: { 2074 struct kvm_translation tr; 2075 2076 r = -EFAULT; 2077 if (copy_from_user(&tr, argp, sizeof tr)) 2078 goto out; 2079 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2080 if (r) 2081 goto out; 2082 r = -EFAULT; 2083 if (copy_to_user(argp, &tr, sizeof tr)) 2084 goto out; 2085 r = 0; 2086 break; 2087 } 2088 case KVM_SET_GUEST_DEBUG: { 2089 struct kvm_guest_debug dbg; 2090 2091 r = -EFAULT; 2092 if (copy_from_user(&dbg, argp, sizeof dbg)) 2093 goto out; 2094 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2095 break; 2096 } 2097 case KVM_SET_SIGNAL_MASK: { 2098 struct kvm_signal_mask __user *sigmask_arg = argp; 2099 struct kvm_signal_mask kvm_sigmask; 2100 sigset_t sigset, *p; 2101 2102 p = NULL; 2103 if (argp) { 2104 r = -EFAULT; 2105 if (copy_from_user(&kvm_sigmask, argp, 2106 sizeof kvm_sigmask)) 2107 goto out; 2108 r = -EINVAL; 2109 if (kvm_sigmask.len != sizeof sigset) 2110 goto out; 2111 r = -EFAULT; 2112 if (copy_from_user(&sigset, sigmask_arg->sigset, 2113 sizeof sigset)) 2114 goto out; 2115 p = &sigset; 2116 } 2117 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2118 break; 2119 } 2120 case KVM_GET_FPU: { 2121 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2122 r = -ENOMEM; 2123 if (!fpu) 2124 goto out; 2125 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2126 if (r) 2127 goto out; 2128 r = -EFAULT; 2129 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2130 goto out; 2131 r = 0; 2132 break; 2133 } 2134 case KVM_SET_FPU: { 2135 fpu = memdup_user(argp, sizeof(*fpu)); 2136 if (IS_ERR(fpu)) { 2137 r = PTR_ERR(fpu); 2138 fpu = NULL; 2139 goto out; 2140 } 2141 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2142 break; 2143 } 2144 default: 2145 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2146 } 2147 out: 2148 vcpu_put(vcpu); 2149 kfree(fpu); 2150 kfree(kvm_sregs); 2151 return r; 2152 } 2153 2154 #ifdef CONFIG_COMPAT 2155 static long kvm_vcpu_compat_ioctl(struct file *filp, 2156 unsigned int ioctl, unsigned long arg) 2157 { 2158 struct kvm_vcpu *vcpu = filp->private_data; 2159 void __user *argp = compat_ptr(arg); 2160 int r; 2161 2162 if (vcpu->kvm->mm != current->mm) 2163 return -EIO; 2164 2165 switch (ioctl) { 2166 case KVM_SET_SIGNAL_MASK: { 2167 struct kvm_signal_mask __user *sigmask_arg = argp; 2168 struct kvm_signal_mask kvm_sigmask; 2169 compat_sigset_t csigset; 2170 sigset_t sigset; 2171 2172 if (argp) { 2173 r = -EFAULT; 2174 if (copy_from_user(&kvm_sigmask, argp, 2175 sizeof kvm_sigmask)) 2176 goto out; 2177 r = -EINVAL; 2178 if (kvm_sigmask.len != sizeof csigset) 2179 goto out; 2180 r = -EFAULT; 2181 if (copy_from_user(&csigset, sigmask_arg->sigset, 2182 sizeof csigset)) 2183 goto out; 2184 sigset_from_compat(&sigset, &csigset); 2185 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2186 } else 2187 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2188 break; 2189 } 2190 default: 2191 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2192 } 2193 2194 out: 2195 return r; 2196 } 2197 #endif 2198 2199 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2200 int (*accessor)(struct kvm_device *dev, 2201 struct kvm_device_attr *attr), 2202 unsigned long arg) 2203 { 2204 struct kvm_device_attr attr; 2205 2206 if (!accessor) 2207 return -EPERM; 2208 2209 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2210 return -EFAULT; 2211 2212 return accessor(dev, &attr); 2213 } 2214 2215 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2216 unsigned long arg) 2217 { 2218 struct kvm_device *dev = filp->private_data; 2219 2220 switch (ioctl) { 2221 case KVM_SET_DEVICE_ATTR: 2222 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2223 case KVM_GET_DEVICE_ATTR: 2224 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2225 case KVM_HAS_DEVICE_ATTR: 2226 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2227 default: 2228 if (dev->ops->ioctl) 2229 return dev->ops->ioctl(dev, ioctl, arg); 2230 2231 return -ENOTTY; 2232 } 2233 } 2234 2235 static int kvm_device_release(struct inode *inode, struct file *filp) 2236 { 2237 struct kvm_device *dev = filp->private_data; 2238 struct kvm *kvm = dev->kvm; 2239 2240 kvm_put_kvm(kvm); 2241 return 0; 2242 } 2243 2244 static const struct file_operations kvm_device_fops = { 2245 .unlocked_ioctl = kvm_device_ioctl, 2246 #ifdef CONFIG_COMPAT 2247 .compat_ioctl = kvm_device_ioctl, 2248 #endif 2249 .release = kvm_device_release, 2250 }; 2251 2252 struct kvm_device *kvm_device_from_filp(struct file *filp) 2253 { 2254 if (filp->f_op != &kvm_device_fops) 2255 return NULL; 2256 2257 return filp->private_data; 2258 } 2259 2260 static int kvm_ioctl_create_device(struct kvm *kvm, 2261 struct kvm_create_device *cd) 2262 { 2263 struct kvm_device_ops *ops = NULL; 2264 struct kvm_device *dev; 2265 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2266 int ret; 2267 2268 switch (cd->type) { 2269 #ifdef CONFIG_KVM_MPIC 2270 case KVM_DEV_TYPE_FSL_MPIC_20: 2271 case KVM_DEV_TYPE_FSL_MPIC_42: 2272 ops = &kvm_mpic_ops; 2273 break; 2274 #endif 2275 #ifdef CONFIG_KVM_XICS 2276 case KVM_DEV_TYPE_XICS: 2277 ops = &kvm_xics_ops; 2278 break; 2279 #endif 2280 #ifdef CONFIG_KVM_VFIO 2281 case KVM_DEV_TYPE_VFIO: 2282 ops = &kvm_vfio_ops; 2283 break; 2284 #endif 2285 #ifdef CONFIG_KVM_ARM_VGIC 2286 case KVM_DEV_TYPE_ARM_VGIC_V2: 2287 ops = &kvm_arm_vgic_v2_ops; 2288 break; 2289 #endif 2290 #ifdef CONFIG_S390 2291 case KVM_DEV_TYPE_FLIC: 2292 ops = &kvm_flic_ops; 2293 break; 2294 #endif 2295 default: 2296 return -ENODEV; 2297 } 2298 2299 if (test) 2300 return 0; 2301 2302 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2303 if (!dev) 2304 return -ENOMEM; 2305 2306 dev->ops = ops; 2307 dev->kvm = kvm; 2308 2309 ret = ops->create(dev, cd->type); 2310 if (ret < 0) { 2311 kfree(dev); 2312 return ret; 2313 } 2314 2315 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2316 if (ret < 0) { 2317 ops->destroy(dev); 2318 return ret; 2319 } 2320 2321 list_add(&dev->vm_node, &kvm->devices); 2322 kvm_get_kvm(kvm); 2323 cd->fd = ret; 2324 return 0; 2325 } 2326 2327 static long kvm_vm_ioctl(struct file *filp, 2328 unsigned int ioctl, unsigned long arg) 2329 { 2330 struct kvm *kvm = filp->private_data; 2331 void __user *argp = (void __user *)arg; 2332 int r; 2333 2334 if (kvm->mm != current->mm) 2335 return -EIO; 2336 switch (ioctl) { 2337 case KVM_CREATE_VCPU: 2338 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2339 break; 2340 case KVM_SET_USER_MEMORY_REGION: { 2341 struct kvm_userspace_memory_region kvm_userspace_mem; 2342 2343 r = -EFAULT; 2344 if (copy_from_user(&kvm_userspace_mem, argp, 2345 sizeof kvm_userspace_mem)) 2346 goto out; 2347 2348 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2349 break; 2350 } 2351 case KVM_GET_DIRTY_LOG: { 2352 struct kvm_dirty_log log; 2353 2354 r = -EFAULT; 2355 if (copy_from_user(&log, argp, sizeof log)) 2356 goto out; 2357 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2358 break; 2359 } 2360 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2361 case KVM_REGISTER_COALESCED_MMIO: { 2362 struct kvm_coalesced_mmio_zone zone; 2363 r = -EFAULT; 2364 if (copy_from_user(&zone, argp, sizeof zone)) 2365 goto out; 2366 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2367 break; 2368 } 2369 case KVM_UNREGISTER_COALESCED_MMIO: { 2370 struct kvm_coalesced_mmio_zone zone; 2371 r = -EFAULT; 2372 if (copy_from_user(&zone, argp, sizeof zone)) 2373 goto out; 2374 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2375 break; 2376 } 2377 #endif 2378 case KVM_IRQFD: { 2379 struct kvm_irqfd data; 2380 2381 r = -EFAULT; 2382 if (copy_from_user(&data, argp, sizeof data)) 2383 goto out; 2384 r = kvm_irqfd(kvm, &data); 2385 break; 2386 } 2387 case KVM_IOEVENTFD: { 2388 struct kvm_ioeventfd data; 2389 2390 r = -EFAULT; 2391 if (copy_from_user(&data, argp, sizeof data)) 2392 goto out; 2393 r = kvm_ioeventfd(kvm, &data); 2394 break; 2395 } 2396 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2397 case KVM_SET_BOOT_CPU_ID: 2398 r = 0; 2399 mutex_lock(&kvm->lock); 2400 if (atomic_read(&kvm->online_vcpus) != 0) 2401 r = -EBUSY; 2402 else 2403 kvm->bsp_vcpu_id = arg; 2404 mutex_unlock(&kvm->lock); 2405 break; 2406 #endif 2407 #ifdef CONFIG_HAVE_KVM_MSI 2408 case KVM_SIGNAL_MSI: { 2409 struct kvm_msi msi; 2410 2411 r = -EFAULT; 2412 if (copy_from_user(&msi, argp, sizeof msi)) 2413 goto out; 2414 r = kvm_send_userspace_msi(kvm, &msi); 2415 break; 2416 } 2417 #endif 2418 #ifdef __KVM_HAVE_IRQ_LINE 2419 case KVM_IRQ_LINE_STATUS: 2420 case KVM_IRQ_LINE: { 2421 struct kvm_irq_level irq_event; 2422 2423 r = -EFAULT; 2424 if (copy_from_user(&irq_event, argp, sizeof irq_event)) 2425 goto out; 2426 2427 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2428 ioctl == KVM_IRQ_LINE_STATUS); 2429 if (r) 2430 goto out; 2431 2432 r = -EFAULT; 2433 if (ioctl == KVM_IRQ_LINE_STATUS) { 2434 if (copy_to_user(argp, &irq_event, sizeof irq_event)) 2435 goto out; 2436 } 2437 2438 r = 0; 2439 break; 2440 } 2441 #endif 2442 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2443 case KVM_SET_GSI_ROUTING: { 2444 struct kvm_irq_routing routing; 2445 struct kvm_irq_routing __user *urouting; 2446 struct kvm_irq_routing_entry *entries; 2447 2448 r = -EFAULT; 2449 if (copy_from_user(&routing, argp, sizeof(routing))) 2450 goto out; 2451 r = -EINVAL; 2452 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2453 goto out; 2454 if (routing.flags) 2455 goto out; 2456 r = -ENOMEM; 2457 entries = vmalloc(routing.nr * sizeof(*entries)); 2458 if (!entries) 2459 goto out; 2460 r = -EFAULT; 2461 urouting = argp; 2462 if (copy_from_user(entries, urouting->entries, 2463 routing.nr * sizeof(*entries))) 2464 goto out_free_irq_routing; 2465 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2466 routing.flags); 2467 out_free_irq_routing: 2468 vfree(entries); 2469 break; 2470 } 2471 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2472 case KVM_CREATE_DEVICE: { 2473 struct kvm_create_device cd; 2474 2475 r = -EFAULT; 2476 if (copy_from_user(&cd, argp, sizeof(cd))) 2477 goto out; 2478 2479 r = kvm_ioctl_create_device(kvm, &cd); 2480 if (r) 2481 goto out; 2482 2483 r = -EFAULT; 2484 if (copy_to_user(argp, &cd, sizeof(cd))) 2485 goto out; 2486 2487 r = 0; 2488 break; 2489 } 2490 default: 2491 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2492 if (r == -ENOTTY) 2493 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg); 2494 } 2495 out: 2496 return r; 2497 } 2498 2499 #ifdef CONFIG_COMPAT 2500 struct compat_kvm_dirty_log { 2501 __u32 slot; 2502 __u32 padding1; 2503 union { 2504 compat_uptr_t dirty_bitmap; /* one bit per page */ 2505 __u64 padding2; 2506 }; 2507 }; 2508 2509 static long kvm_vm_compat_ioctl(struct file *filp, 2510 unsigned int ioctl, unsigned long arg) 2511 { 2512 struct kvm *kvm = filp->private_data; 2513 int r; 2514 2515 if (kvm->mm != current->mm) 2516 return -EIO; 2517 switch (ioctl) { 2518 case KVM_GET_DIRTY_LOG: { 2519 struct compat_kvm_dirty_log compat_log; 2520 struct kvm_dirty_log log; 2521 2522 r = -EFAULT; 2523 if (copy_from_user(&compat_log, (void __user *)arg, 2524 sizeof(compat_log))) 2525 goto out; 2526 log.slot = compat_log.slot; 2527 log.padding1 = compat_log.padding1; 2528 log.padding2 = compat_log.padding2; 2529 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2530 2531 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2532 break; 2533 } 2534 default: 2535 r = kvm_vm_ioctl(filp, ioctl, arg); 2536 } 2537 2538 out: 2539 return r; 2540 } 2541 #endif 2542 2543 static struct file_operations kvm_vm_fops = { 2544 .release = kvm_vm_release, 2545 .unlocked_ioctl = kvm_vm_ioctl, 2546 #ifdef CONFIG_COMPAT 2547 .compat_ioctl = kvm_vm_compat_ioctl, 2548 #endif 2549 .llseek = noop_llseek, 2550 }; 2551 2552 static int kvm_dev_ioctl_create_vm(unsigned long type) 2553 { 2554 int r; 2555 struct kvm *kvm; 2556 2557 kvm = kvm_create_vm(type); 2558 if (IS_ERR(kvm)) 2559 return PTR_ERR(kvm); 2560 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2561 r = kvm_coalesced_mmio_init(kvm); 2562 if (r < 0) { 2563 kvm_put_kvm(kvm); 2564 return r; 2565 } 2566 #endif 2567 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 2568 if (r < 0) 2569 kvm_put_kvm(kvm); 2570 2571 return r; 2572 } 2573 2574 static long kvm_dev_ioctl_check_extension_generic(long arg) 2575 { 2576 switch (arg) { 2577 case KVM_CAP_USER_MEMORY: 2578 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2579 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2580 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2581 case KVM_CAP_SET_BOOT_CPU_ID: 2582 #endif 2583 case KVM_CAP_INTERNAL_ERROR_DATA: 2584 #ifdef CONFIG_HAVE_KVM_MSI 2585 case KVM_CAP_SIGNAL_MSI: 2586 #endif 2587 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2588 case KVM_CAP_IRQFD_RESAMPLE: 2589 #endif 2590 return 1; 2591 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2592 case KVM_CAP_IRQ_ROUTING: 2593 return KVM_MAX_IRQ_ROUTES; 2594 #endif 2595 default: 2596 break; 2597 } 2598 return kvm_dev_ioctl_check_extension(arg); 2599 } 2600 2601 static long kvm_dev_ioctl(struct file *filp, 2602 unsigned int ioctl, unsigned long arg) 2603 { 2604 long r = -EINVAL; 2605 2606 switch (ioctl) { 2607 case KVM_GET_API_VERSION: 2608 r = -EINVAL; 2609 if (arg) 2610 goto out; 2611 r = KVM_API_VERSION; 2612 break; 2613 case KVM_CREATE_VM: 2614 r = kvm_dev_ioctl_create_vm(arg); 2615 break; 2616 case KVM_CHECK_EXTENSION: 2617 r = kvm_dev_ioctl_check_extension_generic(arg); 2618 break; 2619 case KVM_GET_VCPU_MMAP_SIZE: 2620 r = -EINVAL; 2621 if (arg) 2622 goto out; 2623 r = PAGE_SIZE; /* struct kvm_run */ 2624 #ifdef CONFIG_X86 2625 r += PAGE_SIZE; /* pio data page */ 2626 #endif 2627 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2628 r += PAGE_SIZE; /* coalesced mmio ring page */ 2629 #endif 2630 break; 2631 case KVM_TRACE_ENABLE: 2632 case KVM_TRACE_PAUSE: 2633 case KVM_TRACE_DISABLE: 2634 r = -EOPNOTSUPP; 2635 break; 2636 default: 2637 return kvm_arch_dev_ioctl(filp, ioctl, arg); 2638 } 2639 out: 2640 return r; 2641 } 2642 2643 static struct file_operations kvm_chardev_ops = { 2644 .unlocked_ioctl = kvm_dev_ioctl, 2645 .compat_ioctl = kvm_dev_ioctl, 2646 .llseek = noop_llseek, 2647 }; 2648 2649 static struct miscdevice kvm_dev = { 2650 KVM_MINOR, 2651 "kvm", 2652 &kvm_chardev_ops, 2653 }; 2654 2655 static void hardware_enable_nolock(void *junk) 2656 { 2657 int cpu = raw_smp_processor_id(); 2658 int r; 2659 2660 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2661 return; 2662 2663 cpumask_set_cpu(cpu, cpus_hardware_enabled); 2664 2665 r = kvm_arch_hardware_enable(NULL); 2666 2667 if (r) { 2668 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2669 atomic_inc(&hardware_enable_failed); 2670 printk(KERN_INFO "kvm: enabling virtualization on " 2671 "CPU%d failed\n", cpu); 2672 } 2673 } 2674 2675 static void hardware_enable(void) 2676 { 2677 raw_spin_lock(&kvm_count_lock); 2678 if (kvm_usage_count) 2679 hardware_enable_nolock(NULL); 2680 raw_spin_unlock(&kvm_count_lock); 2681 } 2682 2683 static void hardware_disable_nolock(void *junk) 2684 { 2685 int cpu = raw_smp_processor_id(); 2686 2687 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2688 return; 2689 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2690 kvm_arch_hardware_disable(NULL); 2691 } 2692 2693 static void hardware_disable(void) 2694 { 2695 raw_spin_lock(&kvm_count_lock); 2696 if (kvm_usage_count) 2697 hardware_disable_nolock(NULL); 2698 raw_spin_unlock(&kvm_count_lock); 2699 } 2700 2701 static void hardware_disable_all_nolock(void) 2702 { 2703 BUG_ON(!kvm_usage_count); 2704 2705 kvm_usage_count--; 2706 if (!kvm_usage_count) 2707 on_each_cpu(hardware_disable_nolock, NULL, 1); 2708 } 2709 2710 static void hardware_disable_all(void) 2711 { 2712 raw_spin_lock(&kvm_count_lock); 2713 hardware_disable_all_nolock(); 2714 raw_spin_unlock(&kvm_count_lock); 2715 } 2716 2717 static int hardware_enable_all(void) 2718 { 2719 int r = 0; 2720 2721 raw_spin_lock(&kvm_count_lock); 2722 2723 kvm_usage_count++; 2724 if (kvm_usage_count == 1) { 2725 atomic_set(&hardware_enable_failed, 0); 2726 on_each_cpu(hardware_enable_nolock, NULL, 1); 2727 2728 if (atomic_read(&hardware_enable_failed)) { 2729 hardware_disable_all_nolock(); 2730 r = -EBUSY; 2731 } 2732 } 2733 2734 raw_spin_unlock(&kvm_count_lock); 2735 2736 return r; 2737 } 2738 2739 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 2740 void *v) 2741 { 2742 int cpu = (long)v; 2743 2744 val &= ~CPU_TASKS_FROZEN; 2745 switch (val) { 2746 case CPU_DYING: 2747 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 2748 cpu); 2749 hardware_disable(); 2750 break; 2751 case CPU_STARTING: 2752 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 2753 cpu); 2754 hardware_enable(); 2755 break; 2756 } 2757 return NOTIFY_OK; 2758 } 2759 2760 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 2761 void *v) 2762 { 2763 /* 2764 * Some (well, at least mine) BIOSes hang on reboot if 2765 * in vmx root mode. 2766 * 2767 * And Intel TXT required VMX off for all cpu when system shutdown. 2768 */ 2769 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 2770 kvm_rebooting = true; 2771 on_each_cpu(hardware_disable_nolock, NULL, 1); 2772 return NOTIFY_OK; 2773 } 2774 2775 static struct notifier_block kvm_reboot_notifier = { 2776 .notifier_call = kvm_reboot, 2777 .priority = 0, 2778 }; 2779 2780 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 2781 { 2782 int i; 2783 2784 for (i = 0; i < bus->dev_count; i++) { 2785 struct kvm_io_device *pos = bus->range[i].dev; 2786 2787 kvm_iodevice_destructor(pos); 2788 } 2789 kfree(bus); 2790 } 2791 2792 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 2793 const struct kvm_io_range *r2) 2794 { 2795 if (r1->addr < r2->addr) 2796 return -1; 2797 if (r1->addr + r1->len > r2->addr + r2->len) 2798 return 1; 2799 return 0; 2800 } 2801 2802 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 2803 { 2804 return kvm_io_bus_cmp(p1, p2); 2805 } 2806 2807 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 2808 gpa_t addr, int len) 2809 { 2810 bus->range[bus->dev_count++] = (struct kvm_io_range) { 2811 .addr = addr, 2812 .len = len, 2813 .dev = dev, 2814 }; 2815 2816 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 2817 kvm_io_bus_sort_cmp, NULL); 2818 2819 return 0; 2820 } 2821 2822 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 2823 gpa_t addr, int len) 2824 { 2825 struct kvm_io_range *range, key; 2826 int off; 2827 2828 key = (struct kvm_io_range) { 2829 .addr = addr, 2830 .len = len, 2831 }; 2832 2833 range = bsearch(&key, bus->range, bus->dev_count, 2834 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 2835 if (range == NULL) 2836 return -ENOENT; 2837 2838 off = range - bus->range; 2839 2840 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 2841 off--; 2842 2843 return off; 2844 } 2845 2846 static int __kvm_io_bus_write(struct kvm_io_bus *bus, 2847 struct kvm_io_range *range, const void *val) 2848 { 2849 int idx; 2850 2851 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 2852 if (idx < 0) 2853 return -EOPNOTSUPP; 2854 2855 while (idx < bus->dev_count && 2856 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 2857 if (!kvm_iodevice_write(bus->range[idx].dev, range->addr, 2858 range->len, val)) 2859 return idx; 2860 idx++; 2861 } 2862 2863 return -EOPNOTSUPP; 2864 } 2865 2866 /* kvm_io_bus_write - called under kvm->slots_lock */ 2867 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2868 int len, const void *val) 2869 { 2870 struct kvm_io_bus *bus; 2871 struct kvm_io_range range; 2872 int r; 2873 2874 range = (struct kvm_io_range) { 2875 .addr = addr, 2876 .len = len, 2877 }; 2878 2879 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2880 r = __kvm_io_bus_write(bus, &range, val); 2881 return r < 0 ? r : 0; 2882 } 2883 2884 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 2885 int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2886 int len, const void *val, long cookie) 2887 { 2888 struct kvm_io_bus *bus; 2889 struct kvm_io_range range; 2890 2891 range = (struct kvm_io_range) { 2892 .addr = addr, 2893 .len = len, 2894 }; 2895 2896 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2897 2898 /* First try the device referenced by cookie. */ 2899 if ((cookie >= 0) && (cookie < bus->dev_count) && 2900 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 2901 if (!kvm_iodevice_write(bus->range[cookie].dev, addr, len, 2902 val)) 2903 return cookie; 2904 2905 /* 2906 * cookie contained garbage; fall back to search and return the 2907 * correct cookie value. 2908 */ 2909 return __kvm_io_bus_write(bus, &range, val); 2910 } 2911 2912 static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range, 2913 void *val) 2914 { 2915 int idx; 2916 2917 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 2918 if (idx < 0) 2919 return -EOPNOTSUPP; 2920 2921 while (idx < bus->dev_count && 2922 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 2923 if (!kvm_iodevice_read(bus->range[idx].dev, range->addr, 2924 range->len, val)) 2925 return idx; 2926 idx++; 2927 } 2928 2929 return -EOPNOTSUPP; 2930 } 2931 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 2932 2933 /* kvm_io_bus_read - called under kvm->slots_lock */ 2934 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2935 int len, void *val) 2936 { 2937 struct kvm_io_bus *bus; 2938 struct kvm_io_range range; 2939 int r; 2940 2941 range = (struct kvm_io_range) { 2942 .addr = addr, 2943 .len = len, 2944 }; 2945 2946 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2947 r = __kvm_io_bus_read(bus, &range, val); 2948 return r < 0 ? r : 0; 2949 } 2950 2951 2952 /* Caller must hold slots_lock. */ 2953 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2954 int len, struct kvm_io_device *dev) 2955 { 2956 struct kvm_io_bus *new_bus, *bus; 2957 2958 bus = kvm->buses[bus_idx]; 2959 /* exclude ioeventfd which is limited by maximum fd */ 2960 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 2961 return -ENOSPC; 2962 2963 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) * 2964 sizeof(struct kvm_io_range)), GFP_KERNEL); 2965 if (!new_bus) 2966 return -ENOMEM; 2967 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 2968 sizeof(struct kvm_io_range))); 2969 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 2970 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2971 synchronize_srcu_expedited(&kvm->srcu); 2972 kfree(bus); 2973 2974 return 0; 2975 } 2976 2977 /* Caller must hold slots_lock. */ 2978 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 2979 struct kvm_io_device *dev) 2980 { 2981 int i, r; 2982 struct kvm_io_bus *new_bus, *bus; 2983 2984 bus = kvm->buses[bus_idx]; 2985 r = -ENOENT; 2986 for (i = 0; i < bus->dev_count; i++) 2987 if (bus->range[i].dev == dev) { 2988 r = 0; 2989 break; 2990 } 2991 2992 if (r) 2993 return r; 2994 2995 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) * 2996 sizeof(struct kvm_io_range)), GFP_KERNEL); 2997 if (!new_bus) 2998 return -ENOMEM; 2999 3000 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3001 new_bus->dev_count--; 3002 memcpy(new_bus->range + i, bus->range + i + 1, 3003 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3004 3005 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3006 synchronize_srcu_expedited(&kvm->srcu); 3007 kfree(bus); 3008 return r; 3009 } 3010 3011 static struct notifier_block kvm_cpu_notifier = { 3012 .notifier_call = kvm_cpu_hotplug, 3013 }; 3014 3015 static int vm_stat_get(void *_offset, u64 *val) 3016 { 3017 unsigned offset = (long)_offset; 3018 struct kvm *kvm; 3019 3020 *val = 0; 3021 spin_lock(&kvm_lock); 3022 list_for_each_entry(kvm, &vm_list, vm_list) 3023 *val += *(u32 *)((void *)kvm + offset); 3024 spin_unlock(&kvm_lock); 3025 return 0; 3026 } 3027 3028 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3029 3030 static int vcpu_stat_get(void *_offset, u64 *val) 3031 { 3032 unsigned offset = (long)_offset; 3033 struct kvm *kvm; 3034 struct kvm_vcpu *vcpu; 3035 int i; 3036 3037 *val = 0; 3038 spin_lock(&kvm_lock); 3039 list_for_each_entry(kvm, &vm_list, vm_list) 3040 kvm_for_each_vcpu(i, vcpu, kvm) 3041 *val += *(u32 *)((void *)vcpu + offset); 3042 3043 spin_unlock(&kvm_lock); 3044 return 0; 3045 } 3046 3047 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3048 3049 static const struct file_operations *stat_fops[] = { 3050 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3051 [KVM_STAT_VM] = &vm_stat_fops, 3052 }; 3053 3054 static int kvm_init_debug(void) 3055 { 3056 int r = -EEXIST; 3057 struct kvm_stats_debugfs_item *p; 3058 3059 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3060 if (kvm_debugfs_dir == NULL) 3061 goto out; 3062 3063 for (p = debugfs_entries; p->name; ++p) { 3064 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3065 (void *)(long)p->offset, 3066 stat_fops[p->kind]); 3067 if (p->dentry == NULL) 3068 goto out_dir; 3069 } 3070 3071 return 0; 3072 3073 out_dir: 3074 debugfs_remove_recursive(kvm_debugfs_dir); 3075 out: 3076 return r; 3077 } 3078 3079 static void kvm_exit_debug(void) 3080 { 3081 struct kvm_stats_debugfs_item *p; 3082 3083 for (p = debugfs_entries; p->name; ++p) 3084 debugfs_remove(p->dentry); 3085 debugfs_remove(kvm_debugfs_dir); 3086 } 3087 3088 static int kvm_suspend(void) 3089 { 3090 if (kvm_usage_count) 3091 hardware_disable_nolock(NULL); 3092 return 0; 3093 } 3094 3095 static void kvm_resume(void) 3096 { 3097 if (kvm_usage_count) { 3098 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3099 hardware_enable_nolock(NULL); 3100 } 3101 } 3102 3103 static struct syscore_ops kvm_syscore_ops = { 3104 .suspend = kvm_suspend, 3105 .resume = kvm_resume, 3106 }; 3107 3108 static inline 3109 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3110 { 3111 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3112 } 3113 3114 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3115 { 3116 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3117 if (vcpu->preempted) 3118 vcpu->preempted = false; 3119 3120 kvm_arch_vcpu_load(vcpu, cpu); 3121 } 3122 3123 static void kvm_sched_out(struct preempt_notifier *pn, 3124 struct task_struct *next) 3125 { 3126 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3127 3128 if (current->state == TASK_RUNNING) 3129 vcpu->preempted = true; 3130 kvm_arch_vcpu_put(vcpu); 3131 } 3132 3133 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3134 struct module *module) 3135 { 3136 int r; 3137 int cpu; 3138 3139 r = kvm_arch_init(opaque); 3140 if (r) 3141 goto out_fail; 3142 3143 /* 3144 * kvm_arch_init makes sure there's at most one caller 3145 * for architectures that support multiple implementations, 3146 * like intel and amd on x86. 3147 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3148 * conflicts in case kvm is already setup for another implementation. 3149 */ 3150 r = kvm_irqfd_init(); 3151 if (r) 3152 goto out_irqfd; 3153 3154 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3155 r = -ENOMEM; 3156 goto out_free_0; 3157 } 3158 3159 r = kvm_arch_hardware_setup(); 3160 if (r < 0) 3161 goto out_free_0a; 3162 3163 for_each_online_cpu(cpu) { 3164 smp_call_function_single(cpu, 3165 kvm_arch_check_processor_compat, 3166 &r, 1); 3167 if (r < 0) 3168 goto out_free_1; 3169 } 3170 3171 r = register_cpu_notifier(&kvm_cpu_notifier); 3172 if (r) 3173 goto out_free_2; 3174 register_reboot_notifier(&kvm_reboot_notifier); 3175 3176 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3177 if (!vcpu_align) 3178 vcpu_align = __alignof__(struct kvm_vcpu); 3179 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3180 0, NULL); 3181 if (!kvm_vcpu_cache) { 3182 r = -ENOMEM; 3183 goto out_free_3; 3184 } 3185 3186 r = kvm_async_pf_init(); 3187 if (r) 3188 goto out_free; 3189 3190 kvm_chardev_ops.owner = module; 3191 kvm_vm_fops.owner = module; 3192 kvm_vcpu_fops.owner = module; 3193 3194 r = misc_register(&kvm_dev); 3195 if (r) { 3196 printk(KERN_ERR "kvm: misc device register failed\n"); 3197 goto out_unreg; 3198 } 3199 3200 register_syscore_ops(&kvm_syscore_ops); 3201 3202 kvm_preempt_ops.sched_in = kvm_sched_in; 3203 kvm_preempt_ops.sched_out = kvm_sched_out; 3204 3205 r = kvm_init_debug(); 3206 if (r) { 3207 printk(KERN_ERR "kvm: create debugfs files failed\n"); 3208 goto out_undebugfs; 3209 } 3210 3211 return 0; 3212 3213 out_undebugfs: 3214 unregister_syscore_ops(&kvm_syscore_ops); 3215 misc_deregister(&kvm_dev); 3216 out_unreg: 3217 kvm_async_pf_deinit(); 3218 out_free: 3219 kmem_cache_destroy(kvm_vcpu_cache); 3220 out_free_3: 3221 unregister_reboot_notifier(&kvm_reboot_notifier); 3222 unregister_cpu_notifier(&kvm_cpu_notifier); 3223 out_free_2: 3224 out_free_1: 3225 kvm_arch_hardware_unsetup(); 3226 out_free_0a: 3227 free_cpumask_var(cpus_hardware_enabled); 3228 out_free_0: 3229 kvm_irqfd_exit(); 3230 out_irqfd: 3231 kvm_arch_exit(); 3232 out_fail: 3233 return r; 3234 } 3235 EXPORT_SYMBOL_GPL(kvm_init); 3236 3237 void kvm_exit(void) 3238 { 3239 kvm_exit_debug(); 3240 misc_deregister(&kvm_dev); 3241 kmem_cache_destroy(kvm_vcpu_cache); 3242 kvm_async_pf_deinit(); 3243 unregister_syscore_ops(&kvm_syscore_ops); 3244 unregister_reboot_notifier(&kvm_reboot_notifier); 3245 unregister_cpu_notifier(&kvm_cpu_notifier); 3246 on_each_cpu(hardware_disable_nolock, NULL, 1); 3247 kvm_arch_hardware_unsetup(); 3248 kvm_arch_exit(); 3249 kvm_irqfd_exit(); 3250 free_cpumask_var(cpus_hardware_enabled); 3251 } 3252 EXPORT_SYMBOL_GPL(kvm_exit); 3253