1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include "iodev.h" 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/uaccess.h> 56 #include <asm/pgtable.h> 57 58 #include "coalesced_mmio.h" 59 #include "async_pf.h" 60 61 #define CREATE_TRACE_POINTS 62 #include <trace/events/kvm.h> 63 64 MODULE_AUTHOR("Qumranet"); 65 MODULE_LICENSE("GPL"); 66 67 /* 68 * Ordering of locks: 69 * 70 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 71 */ 72 73 DEFINE_RAW_SPINLOCK(kvm_lock); 74 LIST_HEAD(vm_list); 75 76 static cpumask_var_t cpus_hardware_enabled; 77 static int kvm_usage_count = 0; 78 static atomic_t hardware_enable_failed; 79 80 struct kmem_cache *kvm_vcpu_cache; 81 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 82 83 static __read_mostly struct preempt_ops kvm_preempt_ops; 84 85 struct dentry *kvm_debugfs_dir; 86 87 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 88 unsigned long arg); 89 #ifdef CONFIG_COMPAT 90 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 91 unsigned long arg); 92 #endif 93 static int hardware_enable_all(void); 94 static void hardware_disable_all(void); 95 96 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 97 98 bool kvm_rebooting; 99 EXPORT_SYMBOL_GPL(kvm_rebooting); 100 101 static bool largepages_enabled = true; 102 103 bool kvm_is_mmio_pfn(pfn_t pfn) 104 { 105 if (pfn_valid(pfn)) { 106 int reserved; 107 struct page *tail = pfn_to_page(pfn); 108 struct page *head = compound_trans_head(tail); 109 reserved = PageReserved(head); 110 if (head != tail) { 111 /* 112 * "head" is not a dangling pointer 113 * (compound_trans_head takes care of that) 114 * but the hugepage may have been splitted 115 * from under us (and we may not hold a 116 * reference count on the head page so it can 117 * be reused before we run PageReferenced), so 118 * we've to check PageTail before returning 119 * what we just read. 120 */ 121 smp_rmb(); 122 if (PageTail(tail)) 123 return reserved; 124 } 125 return PageReserved(tail); 126 } 127 128 return true; 129 } 130 131 /* 132 * Switches to specified vcpu, until a matching vcpu_put() 133 */ 134 int vcpu_load(struct kvm_vcpu *vcpu) 135 { 136 int cpu; 137 138 if (mutex_lock_killable(&vcpu->mutex)) 139 return -EINTR; 140 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 141 /* The thread running this VCPU changed. */ 142 struct pid *oldpid = vcpu->pid; 143 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 144 rcu_assign_pointer(vcpu->pid, newpid); 145 synchronize_rcu(); 146 put_pid(oldpid); 147 } 148 cpu = get_cpu(); 149 preempt_notifier_register(&vcpu->preempt_notifier); 150 kvm_arch_vcpu_load(vcpu, cpu); 151 put_cpu(); 152 return 0; 153 } 154 155 void vcpu_put(struct kvm_vcpu *vcpu) 156 { 157 preempt_disable(); 158 kvm_arch_vcpu_put(vcpu); 159 preempt_notifier_unregister(&vcpu->preempt_notifier); 160 preempt_enable(); 161 mutex_unlock(&vcpu->mutex); 162 } 163 164 static void ack_flush(void *_completed) 165 { 166 } 167 168 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req) 169 { 170 int i, cpu, me; 171 cpumask_var_t cpus; 172 bool called = true; 173 struct kvm_vcpu *vcpu; 174 175 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 176 177 me = get_cpu(); 178 kvm_for_each_vcpu(i, vcpu, kvm) { 179 kvm_make_request(req, vcpu); 180 cpu = vcpu->cpu; 181 182 /* Set ->requests bit before we read ->mode */ 183 smp_mb(); 184 185 if (cpus != NULL && cpu != -1 && cpu != me && 186 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 187 cpumask_set_cpu(cpu, cpus); 188 } 189 if (unlikely(cpus == NULL)) 190 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 191 else if (!cpumask_empty(cpus)) 192 smp_call_function_many(cpus, ack_flush, NULL, 1); 193 else 194 called = false; 195 put_cpu(); 196 free_cpumask_var(cpus); 197 return called; 198 } 199 200 void kvm_flush_remote_tlbs(struct kvm *kvm) 201 { 202 long dirty_count = kvm->tlbs_dirty; 203 204 smp_mb(); 205 if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 206 ++kvm->stat.remote_tlb_flush; 207 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 208 } 209 210 void kvm_reload_remote_mmus(struct kvm *kvm) 211 { 212 make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 213 } 214 215 void kvm_make_mclock_inprogress_request(struct kvm *kvm) 216 { 217 make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 218 } 219 220 void kvm_make_update_eoibitmap_request(struct kvm *kvm) 221 { 222 make_all_cpus_request(kvm, KVM_REQ_EOIBITMAP); 223 } 224 225 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 226 { 227 struct page *page; 228 int r; 229 230 mutex_init(&vcpu->mutex); 231 vcpu->cpu = -1; 232 vcpu->kvm = kvm; 233 vcpu->vcpu_id = id; 234 vcpu->pid = NULL; 235 init_waitqueue_head(&vcpu->wq); 236 kvm_async_pf_vcpu_init(vcpu); 237 238 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 239 if (!page) { 240 r = -ENOMEM; 241 goto fail; 242 } 243 vcpu->run = page_address(page); 244 245 kvm_vcpu_set_in_spin_loop(vcpu, false); 246 kvm_vcpu_set_dy_eligible(vcpu, false); 247 248 r = kvm_arch_vcpu_init(vcpu); 249 if (r < 0) 250 goto fail_free_run; 251 return 0; 252 253 fail_free_run: 254 free_page((unsigned long)vcpu->run); 255 fail: 256 return r; 257 } 258 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 259 260 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 261 { 262 put_pid(vcpu->pid); 263 kvm_arch_vcpu_uninit(vcpu); 264 free_page((unsigned long)vcpu->run); 265 } 266 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 267 268 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 269 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 270 { 271 return container_of(mn, struct kvm, mmu_notifier); 272 } 273 274 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 275 struct mm_struct *mm, 276 unsigned long address) 277 { 278 struct kvm *kvm = mmu_notifier_to_kvm(mn); 279 int need_tlb_flush, idx; 280 281 /* 282 * When ->invalidate_page runs, the linux pte has been zapped 283 * already but the page is still allocated until 284 * ->invalidate_page returns. So if we increase the sequence 285 * here the kvm page fault will notice if the spte can't be 286 * established because the page is going to be freed. If 287 * instead the kvm page fault establishes the spte before 288 * ->invalidate_page runs, kvm_unmap_hva will release it 289 * before returning. 290 * 291 * The sequence increase only need to be seen at spin_unlock 292 * time, and not at spin_lock time. 293 * 294 * Increasing the sequence after the spin_unlock would be 295 * unsafe because the kvm page fault could then establish the 296 * pte after kvm_unmap_hva returned, without noticing the page 297 * is going to be freed. 298 */ 299 idx = srcu_read_lock(&kvm->srcu); 300 spin_lock(&kvm->mmu_lock); 301 302 kvm->mmu_notifier_seq++; 303 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 304 /* we've to flush the tlb before the pages can be freed */ 305 if (need_tlb_flush) 306 kvm_flush_remote_tlbs(kvm); 307 308 spin_unlock(&kvm->mmu_lock); 309 srcu_read_unlock(&kvm->srcu, idx); 310 } 311 312 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 313 struct mm_struct *mm, 314 unsigned long address, 315 pte_t pte) 316 { 317 struct kvm *kvm = mmu_notifier_to_kvm(mn); 318 int idx; 319 320 idx = srcu_read_lock(&kvm->srcu); 321 spin_lock(&kvm->mmu_lock); 322 kvm->mmu_notifier_seq++; 323 kvm_set_spte_hva(kvm, address, pte); 324 spin_unlock(&kvm->mmu_lock); 325 srcu_read_unlock(&kvm->srcu, idx); 326 } 327 328 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 329 struct mm_struct *mm, 330 unsigned long start, 331 unsigned long end) 332 { 333 struct kvm *kvm = mmu_notifier_to_kvm(mn); 334 int need_tlb_flush = 0, idx; 335 336 idx = srcu_read_lock(&kvm->srcu); 337 spin_lock(&kvm->mmu_lock); 338 /* 339 * The count increase must become visible at unlock time as no 340 * spte can be established without taking the mmu_lock and 341 * count is also read inside the mmu_lock critical section. 342 */ 343 kvm->mmu_notifier_count++; 344 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 345 need_tlb_flush |= kvm->tlbs_dirty; 346 /* we've to flush the tlb before the pages can be freed */ 347 if (need_tlb_flush) 348 kvm_flush_remote_tlbs(kvm); 349 350 spin_unlock(&kvm->mmu_lock); 351 srcu_read_unlock(&kvm->srcu, idx); 352 } 353 354 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 355 struct mm_struct *mm, 356 unsigned long start, 357 unsigned long end) 358 { 359 struct kvm *kvm = mmu_notifier_to_kvm(mn); 360 361 spin_lock(&kvm->mmu_lock); 362 /* 363 * This sequence increase will notify the kvm page fault that 364 * the page that is going to be mapped in the spte could have 365 * been freed. 366 */ 367 kvm->mmu_notifier_seq++; 368 smp_wmb(); 369 /* 370 * The above sequence increase must be visible before the 371 * below count decrease, which is ensured by the smp_wmb above 372 * in conjunction with the smp_rmb in mmu_notifier_retry(). 373 */ 374 kvm->mmu_notifier_count--; 375 spin_unlock(&kvm->mmu_lock); 376 377 BUG_ON(kvm->mmu_notifier_count < 0); 378 } 379 380 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 381 struct mm_struct *mm, 382 unsigned long address) 383 { 384 struct kvm *kvm = mmu_notifier_to_kvm(mn); 385 int young, idx; 386 387 idx = srcu_read_lock(&kvm->srcu); 388 spin_lock(&kvm->mmu_lock); 389 390 young = kvm_age_hva(kvm, address); 391 if (young) 392 kvm_flush_remote_tlbs(kvm); 393 394 spin_unlock(&kvm->mmu_lock); 395 srcu_read_unlock(&kvm->srcu, idx); 396 397 return young; 398 } 399 400 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 401 struct mm_struct *mm, 402 unsigned long address) 403 { 404 struct kvm *kvm = mmu_notifier_to_kvm(mn); 405 int young, idx; 406 407 idx = srcu_read_lock(&kvm->srcu); 408 spin_lock(&kvm->mmu_lock); 409 young = kvm_test_age_hva(kvm, address); 410 spin_unlock(&kvm->mmu_lock); 411 srcu_read_unlock(&kvm->srcu, idx); 412 413 return young; 414 } 415 416 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 417 struct mm_struct *mm) 418 { 419 struct kvm *kvm = mmu_notifier_to_kvm(mn); 420 int idx; 421 422 idx = srcu_read_lock(&kvm->srcu); 423 kvm_arch_flush_shadow_all(kvm); 424 srcu_read_unlock(&kvm->srcu, idx); 425 } 426 427 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 428 .invalidate_page = kvm_mmu_notifier_invalidate_page, 429 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 430 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 431 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 432 .test_young = kvm_mmu_notifier_test_young, 433 .change_pte = kvm_mmu_notifier_change_pte, 434 .release = kvm_mmu_notifier_release, 435 }; 436 437 static int kvm_init_mmu_notifier(struct kvm *kvm) 438 { 439 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 440 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 441 } 442 443 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 444 445 static int kvm_init_mmu_notifier(struct kvm *kvm) 446 { 447 return 0; 448 } 449 450 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 451 452 static void kvm_init_memslots_id(struct kvm *kvm) 453 { 454 int i; 455 struct kvm_memslots *slots = kvm->memslots; 456 457 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 458 slots->id_to_index[i] = slots->memslots[i].id = i; 459 } 460 461 static struct kvm *kvm_create_vm(unsigned long type) 462 { 463 int r, i; 464 struct kvm *kvm = kvm_arch_alloc_vm(); 465 466 if (!kvm) 467 return ERR_PTR(-ENOMEM); 468 469 r = kvm_arch_init_vm(kvm, type); 470 if (r) 471 goto out_err_nodisable; 472 473 r = hardware_enable_all(); 474 if (r) 475 goto out_err_nodisable; 476 477 #ifdef CONFIG_HAVE_KVM_IRQCHIP 478 INIT_HLIST_HEAD(&kvm->mask_notifier_list); 479 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 480 #endif 481 482 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 483 484 r = -ENOMEM; 485 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 486 if (!kvm->memslots) 487 goto out_err_nosrcu; 488 kvm_init_memslots_id(kvm); 489 if (init_srcu_struct(&kvm->srcu)) 490 goto out_err_nosrcu; 491 for (i = 0; i < KVM_NR_BUSES; i++) { 492 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 493 GFP_KERNEL); 494 if (!kvm->buses[i]) 495 goto out_err; 496 } 497 498 spin_lock_init(&kvm->mmu_lock); 499 kvm->mm = current->mm; 500 atomic_inc(&kvm->mm->mm_count); 501 kvm_eventfd_init(kvm); 502 mutex_init(&kvm->lock); 503 mutex_init(&kvm->irq_lock); 504 mutex_init(&kvm->slots_lock); 505 atomic_set(&kvm->users_count, 1); 506 507 r = kvm_init_mmu_notifier(kvm); 508 if (r) 509 goto out_err; 510 511 raw_spin_lock(&kvm_lock); 512 list_add(&kvm->vm_list, &vm_list); 513 raw_spin_unlock(&kvm_lock); 514 515 return kvm; 516 517 out_err: 518 cleanup_srcu_struct(&kvm->srcu); 519 out_err_nosrcu: 520 hardware_disable_all(); 521 out_err_nodisable: 522 for (i = 0; i < KVM_NR_BUSES; i++) 523 kfree(kvm->buses[i]); 524 kfree(kvm->memslots); 525 kvm_arch_free_vm(kvm); 526 return ERR_PTR(r); 527 } 528 529 /* 530 * Avoid using vmalloc for a small buffer. 531 * Should not be used when the size is statically known. 532 */ 533 void *kvm_kvzalloc(unsigned long size) 534 { 535 if (size > PAGE_SIZE) 536 return vzalloc(size); 537 else 538 return kzalloc(size, GFP_KERNEL); 539 } 540 541 void kvm_kvfree(const void *addr) 542 { 543 if (is_vmalloc_addr(addr)) 544 vfree(addr); 545 else 546 kfree(addr); 547 } 548 549 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 550 { 551 if (!memslot->dirty_bitmap) 552 return; 553 554 kvm_kvfree(memslot->dirty_bitmap); 555 memslot->dirty_bitmap = NULL; 556 } 557 558 /* 559 * Free any memory in @free but not in @dont. 560 */ 561 static void kvm_free_physmem_slot(struct kvm_memory_slot *free, 562 struct kvm_memory_slot *dont) 563 { 564 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 565 kvm_destroy_dirty_bitmap(free); 566 567 kvm_arch_free_memslot(free, dont); 568 569 free->npages = 0; 570 } 571 572 void kvm_free_physmem(struct kvm *kvm) 573 { 574 struct kvm_memslots *slots = kvm->memslots; 575 struct kvm_memory_slot *memslot; 576 577 kvm_for_each_memslot(memslot, slots) 578 kvm_free_physmem_slot(memslot, NULL); 579 580 kfree(kvm->memslots); 581 } 582 583 static void kvm_destroy_vm(struct kvm *kvm) 584 { 585 int i; 586 struct mm_struct *mm = kvm->mm; 587 588 kvm_arch_sync_events(kvm); 589 raw_spin_lock(&kvm_lock); 590 list_del(&kvm->vm_list); 591 raw_spin_unlock(&kvm_lock); 592 kvm_free_irq_routing(kvm); 593 for (i = 0; i < KVM_NR_BUSES; i++) 594 kvm_io_bus_destroy(kvm->buses[i]); 595 kvm_coalesced_mmio_free(kvm); 596 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 597 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 598 #else 599 kvm_arch_flush_shadow_all(kvm); 600 #endif 601 kvm_arch_destroy_vm(kvm); 602 kvm_free_physmem(kvm); 603 cleanup_srcu_struct(&kvm->srcu); 604 kvm_arch_free_vm(kvm); 605 hardware_disable_all(); 606 mmdrop(mm); 607 } 608 609 void kvm_get_kvm(struct kvm *kvm) 610 { 611 atomic_inc(&kvm->users_count); 612 } 613 EXPORT_SYMBOL_GPL(kvm_get_kvm); 614 615 void kvm_put_kvm(struct kvm *kvm) 616 { 617 if (atomic_dec_and_test(&kvm->users_count)) 618 kvm_destroy_vm(kvm); 619 } 620 EXPORT_SYMBOL_GPL(kvm_put_kvm); 621 622 623 static int kvm_vm_release(struct inode *inode, struct file *filp) 624 { 625 struct kvm *kvm = filp->private_data; 626 627 kvm_irqfd_release(kvm); 628 629 kvm_put_kvm(kvm); 630 return 0; 631 } 632 633 /* 634 * Allocation size is twice as large as the actual dirty bitmap size. 635 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 636 */ 637 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 638 { 639 #ifndef CONFIG_S390 640 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 641 642 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 643 if (!memslot->dirty_bitmap) 644 return -ENOMEM; 645 646 #endif /* !CONFIG_S390 */ 647 return 0; 648 } 649 650 static int cmp_memslot(const void *slot1, const void *slot2) 651 { 652 struct kvm_memory_slot *s1, *s2; 653 654 s1 = (struct kvm_memory_slot *)slot1; 655 s2 = (struct kvm_memory_slot *)slot2; 656 657 if (s1->npages < s2->npages) 658 return 1; 659 if (s1->npages > s2->npages) 660 return -1; 661 662 return 0; 663 } 664 665 /* 666 * Sort the memslots base on its size, so the larger slots 667 * will get better fit. 668 */ 669 static void sort_memslots(struct kvm_memslots *slots) 670 { 671 int i; 672 673 sort(slots->memslots, KVM_MEM_SLOTS_NUM, 674 sizeof(struct kvm_memory_slot), cmp_memslot, NULL); 675 676 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 677 slots->id_to_index[slots->memslots[i].id] = i; 678 } 679 680 void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new, 681 u64 last_generation) 682 { 683 if (new) { 684 int id = new->id; 685 struct kvm_memory_slot *old = id_to_memslot(slots, id); 686 unsigned long npages = old->npages; 687 688 *old = *new; 689 if (new->npages != npages) 690 sort_memslots(slots); 691 } 692 693 slots->generation = last_generation + 1; 694 } 695 696 static int check_memory_region_flags(struct kvm_userspace_memory_region *mem) 697 { 698 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 699 700 #ifdef KVM_CAP_READONLY_MEM 701 valid_flags |= KVM_MEM_READONLY; 702 #endif 703 704 if (mem->flags & ~valid_flags) 705 return -EINVAL; 706 707 return 0; 708 } 709 710 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 711 struct kvm_memslots *slots, struct kvm_memory_slot *new) 712 { 713 struct kvm_memslots *old_memslots = kvm->memslots; 714 715 update_memslots(slots, new, kvm->memslots->generation); 716 rcu_assign_pointer(kvm->memslots, slots); 717 synchronize_srcu_expedited(&kvm->srcu); 718 return old_memslots; 719 } 720 721 /* 722 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations: 723 * - create a new memory slot 724 * - delete an existing memory slot 725 * - modify an existing memory slot 726 * -- move it in the guest physical memory space 727 * -- just change its flags 728 * 729 * Since flags can be changed by some of these operations, the following 730 * differentiation is the best we can do for __kvm_set_memory_region(): 731 */ 732 enum kvm_mr_change { 733 KVM_MR_CREATE, 734 KVM_MR_DELETE, 735 KVM_MR_MOVE, 736 KVM_MR_FLAGS_ONLY, 737 }; 738 739 /* 740 * Allocate some memory and give it an address in the guest physical address 741 * space. 742 * 743 * Discontiguous memory is allowed, mostly for framebuffers. 744 * 745 * Must be called holding mmap_sem for write. 746 */ 747 int __kvm_set_memory_region(struct kvm *kvm, 748 struct kvm_userspace_memory_region *mem, 749 bool user_alloc) 750 { 751 int r; 752 gfn_t base_gfn; 753 unsigned long npages; 754 struct kvm_memory_slot *slot; 755 struct kvm_memory_slot old, new; 756 struct kvm_memslots *slots = NULL, *old_memslots; 757 enum kvm_mr_change change; 758 759 r = check_memory_region_flags(mem); 760 if (r) 761 goto out; 762 763 r = -EINVAL; 764 /* General sanity checks */ 765 if (mem->memory_size & (PAGE_SIZE - 1)) 766 goto out; 767 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 768 goto out; 769 /* We can read the guest memory with __xxx_user() later on. */ 770 if (user_alloc && 771 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 772 !access_ok(VERIFY_WRITE, 773 (void __user *)(unsigned long)mem->userspace_addr, 774 mem->memory_size))) 775 goto out; 776 if (mem->slot >= KVM_MEM_SLOTS_NUM) 777 goto out; 778 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 779 goto out; 780 781 slot = id_to_memslot(kvm->memslots, mem->slot); 782 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 783 npages = mem->memory_size >> PAGE_SHIFT; 784 785 r = -EINVAL; 786 if (npages > KVM_MEM_MAX_NR_PAGES) 787 goto out; 788 789 if (!npages) 790 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; 791 792 new = old = *slot; 793 794 new.id = mem->slot; 795 new.base_gfn = base_gfn; 796 new.npages = npages; 797 new.flags = mem->flags; 798 799 r = -EINVAL; 800 if (npages) { 801 if (!old.npages) 802 change = KVM_MR_CREATE; 803 else { /* Modify an existing slot. */ 804 if ((mem->userspace_addr != old.userspace_addr) || 805 (npages != old.npages) || 806 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 807 goto out; 808 809 if (base_gfn != old.base_gfn) 810 change = KVM_MR_MOVE; 811 else if (new.flags != old.flags) 812 change = KVM_MR_FLAGS_ONLY; 813 else { /* Nothing to change. */ 814 r = 0; 815 goto out; 816 } 817 } 818 } else if (old.npages) { 819 change = KVM_MR_DELETE; 820 } else /* Modify a non-existent slot: disallowed. */ 821 goto out; 822 823 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 824 /* Check for overlaps */ 825 r = -EEXIST; 826 kvm_for_each_memslot(slot, kvm->memslots) { 827 if ((slot->id >= KVM_USER_MEM_SLOTS) || 828 (slot->id == mem->slot)) 829 continue; 830 if (!((base_gfn + npages <= slot->base_gfn) || 831 (base_gfn >= slot->base_gfn + slot->npages))) 832 goto out; 833 } 834 } 835 836 /* Free page dirty bitmap if unneeded */ 837 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 838 new.dirty_bitmap = NULL; 839 840 r = -ENOMEM; 841 if (change == KVM_MR_CREATE) { 842 new.userspace_addr = mem->userspace_addr; 843 844 if (kvm_arch_create_memslot(&new, npages)) 845 goto out_free; 846 } 847 848 /* Allocate page dirty bitmap if needed */ 849 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 850 if (kvm_create_dirty_bitmap(&new) < 0) 851 goto out_free; 852 } 853 854 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 855 r = -ENOMEM; 856 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 857 GFP_KERNEL); 858 if (!slots) 859 goto out_free; 860 slot = id_to_memslot(slots, mem->slot); 861 slot->flags |= KVM_MEMSLOT_INVALID; 862 863 old_memslots = install_new_memslots(kvm, slots, NULL); 864 865 /* slot was deleted or moved, clear iommu mapping */ 866 kvm_iommu_unmap_pages(kvm, &old); 867 /* From this point no new shadow pages pointing to a deleted, 868 * or moved, memslot will be created. 869 * 870 * validation of sp->gfn happens in: 871 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 872 * - kvm_is_visible_gfn (mmu_check_roots) 873 */ 874 kvm_arch_flush_shadow_memslot(kvm, slot); 875 slots = old_memslots; 876 } 877 878 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc); 879 if (r) 880 goto out_slots; 881 882 r = -ENOMEM; 883 /* 884 * We can re-use the old_memslots from above, the only difference 885 * from the currently installed memslots is the invalid flag. This 886 * will get overwritten by update_memslots anyway. 887 */ 888 if (!slots) { 889 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 890 GFP_KERNEL); 891 if (!slots) 892 goto out_free; 893 } 894 895 /* 896 * IOMMU mapping: New slots need to be mapped. Old slots need to be 897 * un-mapped and re-mapped if their base changes. Since base change 898 * unmapping is handled above with slot deletion, mapping alone is 899 * needed here. Anything else the iommu might care about for existing 900 * slots (size changes, userspace addr changes and read-only flag 901 * changes) is disallowed above, so any other attribute changes getting 902 * here can be skipped. 903 */ 904 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 905 r = kvm_iommu_map_pages(kvm, &new); 906 if (r) 907 goto out_slots; 908 } 909 910 /* actual memory is freed via old in kvm_free_physmem_slot below */ 911 if (change == KVM_MR_DELETE) { 912 new.dirty_bitmap = NULL; 913 memset(&new.arch, 0, sizeof(new.arch)); 914 } 915 916 old_memslots = install_new_memslots(kvm, slots, &new); 917 918 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc); 919 920 kvm_free_physmem_slot(&old, &new); 921 kfree(old_memslots); 922 923 return 0; 924 925 out_slots: 926 kfree(slots); 927 out_free: 928 kvm_free_physmem_slot(&new, &old); 929 out: 930 return r; 931 } 932 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 933 934 int kvm_set_memory_region(struct kvm *kvm, 935 struct kvm_userspace_memory_region *mem, 936 bool user_alloc) 937 { 938 int r; 939 940 mutex_lock(&kvm->slots_lock); 941 r = __kvm_set_memory_region(kvm, mem, user_alloc); 942 mutex_unlock(&kvm->slots_lock); 943 return r; 944 } 945 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 946 947 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 948 struct 949 kvm_userspace_memory_region *mem, 950 bool user_alloc) 951 { 952 if (mem->slot >= KVM_USER_MEM_SLOTS) 953 return -EINVAL; 954 return kvm_set_memory_region(kvm, mem, user_alloc); 955 } 956 957 int kvm_get_dirty_log(struct kvm *kvm, 958 struct kvm_dirty_log *log, int *is_dirty) 959 { 960 struct kvm_memory_slot *memslot; 961 int r, i; 962 unsigned long n; 963 unsigned long any = 0; 964 965 r = -EINVAL; 966 if (log->slot >= KVM_USER_MEM_SLOTS) 967 goto out; 968 969 memslot = id_to_memslot(kvm->memslots, log->slot); 970 r = -ENOENT; 971 if (!memslot->dirty_bitmap) 972 goto out; 973 974 n = kvm_dirty_bitmap_bytes(memslot); 975 976 for (i = 0; !any && i < n/sizeof(long); ++i) 977 any = memslot->dirty_bitmap[i]; 978 979 r = -EFAULT; 980 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 981 goto out; 982 983 if (any) 984 *is_dirty = 1; 985 986 r = 0; 987 out: 988 return r; 989 } 990 991 bool kvm_largepages_enabled(void) 992 { 993 return largepages_enabled; 994 } 995 996 void kvm_disable_largepages(void) 997 { 998 largepages_enabled = false; 999 } 1000 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1001 1002 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1003 { 1004 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1005 } 1006 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1007 1008 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1009 { 1010 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1011 1012 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1013 memslot->flags & KVM_MEMSLOT_INVALID) 1014 return 0; 1015 1016 return 1; 1017 } 1018 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1019 1020 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1021 { 1022 struct vm_area_struct *vma; 1023 unsigned long addr, size; 1024 1025 size = PAGE_SIZE; 1026 1027 addr = gfn_to_hva(kvm, gfn); 1028 if (kvm_is_error_hva(addr)) 1029 return PAGE_SIZE; 1030 1031 down_read(¤t->mm->mmap_sem); 1032 vma = find_vma(current->mm, addr); 1033 if (!vma) 1034 goto out; 1035 1036 size = vma_kernel_pagesize(vma); 1037 1038 out: 1039 up_read(¤t->mm->mmap_sem); 1040 1041 return size; 1042 } 1043 1044 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1045 { 1046 return slot->flags & KVM_MEM_READONLY; 1047 } 1048 1049 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1050 gfn_t *nr_pages, bool write) 1051 { 1052 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1053 return KVM_HVA_ERR_BAD; 1054 1055 if (memslot_is_readonly(slot) && write) 1056 return KVM_HVA_ERR_RO_BAD; 1057 1058 if (nr_pages) 1059 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1060 1061 return __gfn_to_hva_memslot(slot, gfn); 1062 } 1063 1064 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1065 gfn_t *nr_pages) 1066 { 1067 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1068 } 1069 1070 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1071 gfn_t gfn) 1072 { 1073 return gfn_to_hva_many(slot, gfn, NULL); 1074 } 1075 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1076 1077 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1078 { 1079 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1080 } 1081 EXPORT_SYMBOL_GPL(gfn_to_hva); 1082 1083 /* 1084 * The hva returned by this function is only allowed to be read. 1085 * It should pair with kvm_read_hva() or kvm_read_hva_atomic(). 1086 */ 1087 static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn) 1088 { 1089 return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false); 1090 } 1091 1092 static int kvm_read_hva(void *data, void __user *hva, int len) 1093 { 1094 return __copy_from_user(data, hva, len); 1095 } 1096 1097 static int kvm_read_hva_atomic(void *data, void __user *hva, int len) 1098 { 1099 return __copy_from_user_inatomic(data, hva, len); 1100 } 1101 1102 int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1103 unsigned long start, int write, struct page **page) 1104 { 1105 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1106 1107 if (write) 1108 flags |= FOLL_WRITE; 1109 1110 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1111 } 1112 1113 static inline int check_user_page_hwpoison(unsigned long addr) 1114 { 1115 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1116 1117 rc = __get_user_pages(current, current->mm, addr, 1, 1118 flags, NULL, NULL, NULL); 1119 return rc == -EHWPOISON; 1120 } 1121 1122 /* 1123 * The atomic path to get the writable pfn which will be stored in @pfn, 1124 * true indicates success, otherwise false is returned. 1125 */ 1126 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1127 bool write_fault, bool *writable, pfn_t *pfn) 1128 { 1129 struct page *page[1]; 1130 int npages; 1131 1132 if (!(async || atomic)) 1133 return false; 1134 1135 /* 1136 * Fast pin a writable pfn only if it is a write fault request 1137 * or the caller allows to map a writable pfn for a read fault 1138 * request. 1139 */ 1140 if (!(write_fault || writable)) 1141 return false; 1142 1143 npages = __get_user_pages_fast(addr, 1, 1, page); 1144 if (npages == 1) { 1145 *pfn = page_to_pfn(page[0]); 1146 1147 if (writable) 1148 *writable = true; 1149 return true; 1150 } 1151 1152 return false; 1153 } 1154 1155 /* 1156 * The slow path to get the pfn of the specified host virtual address, 1157 * 1 indicates success, -errno is returned if error is detected. 1158 */ 1159 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1160 bool *writable, pfn_t *pfn) 1161 { 1162 struct page *page[1]; 1163 int npages = 0; 1164 1165 might_sleep(); 1166 1167 if (writable) 1168 *writable = write_fault; 1169 1170 if (async) { 1171 down_read(¤t->mm->mmap_sem); 1172 npages = get_user_page_nowait(current, current->mm, 1173 addr, write_fault, page); 1174 up_read(¤t->mm->mmap_sem); 1175 } else 1176 npages = get_user_pages_fast(addr, 1, write_fault, 1177 page); 1178 if (npages != 1) 1179 return npages; 1180 1181 /* map read fault as writable if possible */ 1182 if (unlikely(!write_fault) && writable) { 1183 struct page *wpage[1]; 1184 1185 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1186 if (npages == 1) { 1187 *writable = true; 1188 put_page(page[0]); 1189 page[0] = wpage[0]; 1190 } 1191 1192 npages = 1; 1193 } 1194 *pfn = page_to_pfn(page[0]); 1195 return npages; 1196 } 1197 1198 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1199 { 1200 if (unlikely(!(vma->vm_flags & VM_READ))) 1201 return false; 1202 1203 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1204 return false; 1205 1206 return true; 1207 } 1208 1209 /* 1210 * Pin guest page in memory and return its pfn. 1211 * @addr: host virtual address which maps memory to the guest 1212 * @atomic: whether this function can sleep 1213 * @async: whether this function need to wait IO complete if the 1214 * host page is not in the memory 1215 * @write_fault: whether we should get a writable host page 1216 * @writable: whether it allows to map a writable host page for !@write_fault 1217 * 1218 * The function will map a writable host page for these two cases: 1219 * 1): @write_fault = true 1220 * 2): @write_fault = false && @writable, @writable will tell the caller 1221 * whether the mapping is writable. 1222 */ 1223 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1224 bool write_fault, bool *writable) 1225 { 1226 struct vm_area_struct *vma; 1227 pfn_t pfn = 0; 1228 int npages; 1229 1230 /* we can do it either atomically or asynchronously, not both */ 1231 BUG_ON(atomic && async); 1232 1233 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1234 return pfn; 1235 1236 if (atomic) 1237 return KVM_PFN_ERR_FAULT; 1238 1239 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1240 if (npages == 1) 1241 return pfn; 1242 1243 down_read(¤t->mm->mmap_sem); 1244 if (npages == -EHWPOISON || 1245 (!async && check_user_page_hwpoison(addr))) { 1246 pfn = KVM_PFN_ERR_HWPOISON; 1247 goto exit; 1248 } 1249 1250 vma = find_vma_intersection(current->mm, addr, addr + 1); 1251 1252 if (vma == NULL) 1253 pfn = KVM_PFN_ERR_FAULT; 1254 else if ((vma->vm_flags & VM_PFNMAP)) { 1255 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1256 vma->vm_pgoff; 1257 BUG_ON(!kvm_is_mmio_pfn(pfn)); 1258 } else { 1259 if (async && vma_is_valid(vma, write_fault)) 1260 *async = true; 1261 pfn = KVM_PFN_ERR_FAULT; 1262 } 1263 exit: 1264 up_read(¤t->mm->mmap_sem); 1265 return pfn; 1266 } 1267 1268 static pfn_t 1269 __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic, 1270 bool *async, bool write_fault, bool *writable) 1271 { 1272 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1273 1274 if (addr == KVM_HVA_ERR_RO_BAD) 1275 return KVM_PFN_ERR_RO_FAULT; 1276 1277 if (kvm_is_error_hva(addr)) 1278 return KVM_PFN_NOSLOT; 1279 1280 /* Do not map writable pfn in the readonly memslot. */ 1281 if (writable && memslot_is_readonly(slot)) { 1282 *writable = false; 1283 writable = NULL; 1284 } 1285 1286 return hva_to_pfn(addr, atomic, async, write_fault, 1287 writable); 1288 } 1289 1290 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async, 1291 bool write_fault, bool *writable) 1292 { 1293 struct kvm_memory_slot *slot; 1294 1295 if (async) 1296 *async = false; 1297 1298 slot = gfn_to_memslot(kvm, gfn); 1299 1300 return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault, 1301 writable); 1302 } 1303 1304 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1305 { 1306 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL); 1307 } 1308 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1309 1310 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async, 1311 bool write_fault, bool *writable) 1312 { 1313 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable); 1314 } 1315 EXPORT_SYMBOL_GPL(gfn_to_pfn_async); 1316 1317 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1318 { 1319 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL); 1320 } 1321 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1322 1323 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1324 bool *writable) 1325 { 1326 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable); 1327 } 1328 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1329 1330 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1331 { 1332 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1333 } 1334 1335 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1336 { 1337 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1338 } 1339 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1340 1341 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages, 1342 int nr_pages) 1343 { 1344 unsigned long addr; 1345 gfn_t entry; 1346 1347 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry); 1348 if (kvm_is_error_hva(addr)) 1349 return -1; 1350 1351 if (entry < nr_pages) 1352 return 0; 1353 1354 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1355 } 1356 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1357 1358 static struct page *kvm_pfn_to_page(pfn_t pfn) 1359 { 1360 if (is_error_noslot_pfn(pfn)) 1361 return KVM_ERR_PTR_BAD_PAGE; 1362 1363 if (kvm_is_mmio_pfn(pfn)) { 1364 WARN_ON(1); 1365 return KVM_ERR_PTR_BAD_PAGE; 1366 } 1367 1368 return pfn_to_page(pfn); 1369 } 1370 1371 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1372 { 1373 pfn_t pfn; 1374 1375 pfn = gfn_to_pfn(kvm, gfn); 1376 1377 return kvm_pfn_to_page(pfn); 1378 } 1379 1380 EXPORT_SYMBOL_GPL(gfn_to_page); 1381 1382 void kvm_release_page_clean(struct page *page) 1383 { 1384 WARN_ON(is_error_page(page)); 1385 1386 kvm_release_pfn_clean(page_to_pfn(page)); 1387 } 1388 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1389 1390 void kvm_release_pfn_clean(pfn_t pfn) 1391 { 1392 if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn)) 1393 put_page(pfn_to_page(pfn)); 1394 } 1395 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1396 1397 void kvm_release_page_dirty(struct page *page) 1398 { 1399 WARN_ON(is_error_page(page)); 1400 1401 kvm_release_pfn_dirty(page_to_pfn(page)); 1402 } 1403 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1404 1405 void kvm_release_pfn_dirty(pfn_t pfn) 1406 { 1407 kvm_set_pfn_dirty(pfn); 1408 kvm_release_pfn_clean(pfn); 1409 } 1410 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 1411 1412 void kvm_set_page_dirty(struct page *page) 1413 { 1414 kvm_set_pfn_dirty(page_to_pfn(page)); 1415 } 1416 EXPORT_SYMBOL_GPL(kvm_set_page_dirty); 1417 1418 void kvm_set_pfn_dirty(pfn_t pfn) 1419 { 1420 if (!kvm_is_mmio_pfn(pfn)) { 1421 struct page *page = pfn_to_page(pfn); 1422 if (!PageReserved(page)) 1423 SetPageDirty(page); 1424 } 1425 } 1426 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1427 1428 void kvm_set_pfn_accessed(pfn_t pfn) 1429 { 1430 if (!kvm_is_mmio_pfn(pfn)) 1431 mark_page_accessed(pfn_to_page(pfn)); 1432 } 1433 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1434 1435 void kvm_get_pfn(pfn_t pfn) 1436 { 1437 if (!kvm_is_mmio_pfn(pfn)) 1438 get_page(pfn_to_page(pfn)); 1439 } 1440 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1441 1442 static int next_segment(unsigned long len, int offset) 1443 { 1444 if (len > PAGE_SIZE - offset) 1445 return PAGE_SIZE - offset; 1446 else 1447 return len; 1448 } 1449 1450 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1451 int len) 1452 { 1453 int r; 1454 unsigned long addr; 1455 1456 addr = gfn_to_hva_read(kvm, gfn); 1457 if (kvm_is_error_hva(addr)) 1458 return -EFAULT; 1459 r = kvm_read_hva(data, (void __user *)addr + offset, len); 1460 if (r) 1461 return -EFAULT; 1462 return 0; 1463 } 1464 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1465 1466 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1467 { 1468 gfn_t gfn = gpa >> PAGE_SHIFT; 1469 int seg; 1470 int offset = offset_in_page(gpa); 1471 int ret; 1472 1473 while ((seg = next_segment(len, offset)) != 0) { 1474 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1475 if (ret < 0) 1476 return ret; 1477 offset = 0; 1478 len -= seg; 1479 data += seg; 1480 ++gfn; 1481 } 1482 return 0; 1483 } 1484 EXPORT_SYMBOL_GPL(kvm_read_guest); 1485 1486 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1487 unsigned long len) 1488 { 1489 int r; 1490 unsigned long addr; 1491 gfn_t gfn = gpa >> PAGE_SHIFT; 1492 int offset = offset_in_page(gpa); 1493 1494 addr = gfn_to_hva_read(kvm, gfn); 1495 if (kvm_is_error_hva(addr)) 1496 return -EFAULT; 1497 pagefault_disable(); 1498 r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len); 1499 pagefault_enable(); 1500 if (r) 1501 return -EFAULT; 1502 return 0; 1503 } 1504 EXPORT_SYMBOL(kvm_read_guest_atomic); 1505 1506 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1507 int offset, int len) 1508 { 1509 int r; 1510 unsigned long addr; 1511 1512 addr = gfn_to_hva(kvm, gfn); 1513 if (kvm_is_error_hva(addr)) 1514 return -EFAULT; 1515 r = __copy_to_user((void __user *)addr + offset, data, len); 1516 if (r) 1517 return -EFAULT; 1518 mark_page_dirty(kvm, gfn); 1519 return 0; 1520 } 1521 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1522 1523 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1524 unsigned long len) 1525 { 1526 gfn_t gfn = gpa >> PAGE_SHIFT; 1527 int seg; 1528 int offset = offset_in_page(gpa); 1529 int ret; 1530 1531 while ((seg = next_segment(len, offset)) != 0) { 1532 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1533 if (ret < 0) 1534 return ret; 1535 offset = 0; 1536 len -= seg; 1537 data += seg; 1538 ++gfn; 1539 } 1540 return 0; 1541 } 1542 1543 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1544 gpa_t gpa) 1545 { 1546 struct kvm_memslots *slots = kvm_memslots(kvm); 1547 int offset = offset_in_page(gpa); 1548 gfn_t gfn = gpa >> PAGE_SHIFT; 1549 1550 ghc->gpa = gpa; 1551 ghc->generation = slots->generation; 1552 ghc->memslot = gfn_to_memslot(kvm, gfn); 1553 ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL); 1554 if (!kvm_is_error_hva(ghc->hva)) 1555 ghc->hva += offset; 1556 else 1557 return -EFAULT; 1558 1559 return 0; 1560 } 1561 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1562 1563 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1564 void *data, unsigned long len) 1565 { 1566 struct kvm_memslots *slots = kvm_memslots(kvm); 1567 int r; 1568 1569 if (slots->generation != ghc->generation) 1570 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa); 1571 1572 if (kvm_is_error_hva(ghc->hva)) 1573 return -EFAULT; 1574 1575 r = __copy_to_user((void __user *)ghc->hva, data, len); 1576 if (r) 1577 return -EFAULT; 1578 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1579 1580 return 0; 1581 } 1582 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1583 1584 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1585 void *data, unsigned long len) 1586 { 1587 struct kvm_memslots *slots = kvm_memslots(kvm); 1588 int r; 1589 1590 if (slots->generation != ghc->generation) 1591 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa); 1592 1593 if (kvm_is_error_hva(ghc->hva)) 1594 return -EFAULT; 1595 1596 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1597 if (r) 1598 return -EFAULT; 1599 1600 return 0; 1601 } 1602 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1603 1604 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1605 { 1606 return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page, 1607 offset, len); 1608 } 1609 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1610 1611 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1612 { 1613 gfn_t gfn = gpa >> PAGE_SHIFT; 1614 int seg; 1615 int offset = offset_in_page(gpa); 1616 int ret; 1617 1618 while ((seg = next_segment(len, offset)) != 0) { 1619 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1620 if (ret < 0) 1621 return ret; 1622 offset = 0; 1623 len -= seg; 1624 ++gfn; 1625 } 1626 return 0; 1627 } 1628 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1629 1630 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot, 1631 gfn_t gfn) 1632 { 1633 if (memslot && memslot->dirty_bitmap) { 1634 unsigned long rel_gfn = gfn - memslot->base_gfn; 1635 1636 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1637 } 1638 } 1639 1640 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1641 { 1642 struct kvm_memory_slot *memslot; 1643 1644 memslot = gfn_to_memslot(kvm, gfn); 1645 mark_page_dirty_in_slot(kvm, memslot, gfn); 1646 } 1647 1648 /* 1649 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1650 */ 1651 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1652 { 1653 DEFINE_WAIT(wait); 1654 1655 for (;;) { 1656 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1657 1658 if (kvm_arch_vcpu_runnable(vcpu)) { 1659 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1660 break; 1661 } 1662 if (kvm_cpu_has_pending_timer(vcpu)) 1663 break; 1664 if (signal_pending(current)) 1665 break; 1666 1667 schedule(); 1668 } 1669 1670 finish_wait(&vcpu->wq, &wait); 1671 } 1672 1673 #ifndef CONFIG_S390 1674 /* 1675 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 1676 */ 1677 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 1678 { 1679 int me; 1680 int cpu = vcpu->cpu; 1681 wait_queue_head_t *wqp; 1682 1683 wqp = kvm_arch_vcpu_wq(vcpu); 1684 if (waitqueue_active(wqp)) { 1685 wake_up_interruptible(wqp); 1686 ++vcpu->stat.halt_wakeup; 1687 } 1688 1689 me = get_cpu(); 1690 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 1691 if (kvm_arch_vcpu_should_kick(vcpu)) 1692 smp_send_reschedule(cpu); 1693 put_cpu(); 1694 } 1695 #endif /* !CONFIG_S390 */ 1696 1697 void kvm_resched(struct kvm_vcpu *vcpu) 1698 { 1699 if (!need_resched()) 1700 return; 1701 cond_resched(); 1702 } 1703 EXPORT_SYMBOL_GPL(kvm_resched); 1704 1705 bool kvm_vcpu_yield_to(struct kvm_vcpu *target) 1706 { 1707 struct pid *pid; 1708 struct task_struct *task = NULL; 1709 bool ret = false; 1710 1711 rcu_read_lock(); 1712 pid = rcu_dereference(target->pid); 1713 if (pid) 1714 task = get_pid_task(target->pid, PIDTYPE_PID); 1715 rcu_read_unlock(); 1716 if (!task) 1717 return ret; 1718 if (task->flags & PF_VCPU) { 1719 put_task_struct(task); 1720 return ret; 1721 } 1722 ret = yield_to(task, 1); 1723 put_task_struct(task); 1724 1725 return ret; 1726 } 1727 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 1728 1729 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 1730 /* 1731 * Helper that checks whether a VCPU is eligible for directed yield. 1732 * Most eligible candidate to yield is decided by following heuristics: 1733 * 1734 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 1735 * (preempted lock holder), indicated by @in_spin_loop. 1736 * Set at the beiginning and cleared at the end of interception/PLE handler. 1737 * 1738 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 1739 * chance last time (mostly it has become eligible now since we have probably 1740 * yielded to lockholder in last iteration. This is done by toggling 1741 * @dy_eligible each time a VCPU checked for eligibility.) 1742 * 1743 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 1744 * to preempted lock-holder could result in wrong VCPU selection and CPU 1745 * burning. Giving priority for a potential lock-holder increases lock 1746 * progress. 1747 * 1748 * Since algorithm is based on heuristics, accessing another VCPU data without 1749 * locking does not harm. It may result in trying to yield to same VCPU, fail 1750 * and continue with next VCPU and so on. 1751 */ 1752 bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 1753 { 1754 bool eligible; 1755 1756 eligible = !vcpu->spin_loop.in_spin_loop || 1757 (vcpu->spin_loop.in_spin_loop && 1758 vcpu->spin_loop.dy_eligible); 1759 1760 if (vcpu->spin_loop.in_spin_loop) 1761 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 1762 1763 return eligible; 1764 } 1765 #endif 1766 1767 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 1768 { 1769 struct kvm *kvm = me->kvm; 1770 struct kvm_vcpu *vcpu; 1771 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 1772 int yielded = 0; 1773 int try = 3; 1774 int pass; 1775 int i; 1776 1777 kvm_vcpu_set_in_spin_loop(me, true); 1778 /* 1779 * We boost the priority of a VCPU that is runnable but not 1780 * currently running, because it got preempted by something 1781 * else and called schedule in __vcpu_run. Hopefully that 1782 * VCPU is holding the lock that we need and will release it. 1783 * We approximate round-robin by starting at the last boosted VCPU. 1784 */ 1785 for (pass = 0; pass < 2 && !yielded && try; pass++) { 1786 kvm_for_each_vcpu(i, vcpu, kvm) { 1787 if (!pass && i <= last_boosted_vcpu) { 1788 i = last_boosted_vcpu; 1789 continue; 1790 } else if (pass && i > last_boosted_vcpu) 1791 break; 1792 if (vcpu == me) 1793 continue; 1794 if (waitqueue_active(&vcpu->wq)) 1795 continue; 1796 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 1797 continue; 1798 1799 yielded = kvm_vcpu_yield_to(vcpu); 1800 if (yielded > 0) { 1801 kvm->last_boosted_vcpu = i; 1802 break; 1803 } else if (yielded < 0) { 1804 try--; 1805 if (!try) 1806 break; 1807 } 1808 } 1809 } 1810 kvm_vcpu_set_in_spin_loop(me, false); 1811 1812 /* Ensure vcpu is not eligible during next spinloop */ 1813 kvm_vcpu_set_dy_eligible(me, false); 1814 } 1815 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 1816 1817 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1818 { 1819 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 1820 struct page *page; 1821 1822 if (vmf->pgoff == 0) 1823 page = virt_to_page(vcpu->run); 1824 #ifdef CONFIG_X86 1825 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 1826 page = virt_to_page(vcpu->arch.pio_data); 1827 #endif 1828 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1829 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 1830 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 1831 #endif 1832 else 1833 return kvm_arch_vcpu_fault(vcpu, vmf); 1834 get_page(page); 1835 vmf->page = page; 1836 return 0; 1837 } 1838 1839 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 1840 .fault = kvm_vcpu_fault, 1841 }; 1842 1843 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 1844 { 1845 vma->vm_ops = &kvm_vcpu_vm_ops; 1846 return 0; 1847 } 1848 1849 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 1850 { 1851 struct kvm_vcpu *vcpu = filp->private_data; 1852 1853 kvm_put_kvm(vcpu->kvm); 1854 return 0; 1855 } 1856 1857 static struct file_operations kvm_vcpu_fops = { 1858 .release = kvm_vcpu_release, 1859 .unlocked_ioctl = kvm_vcpu_ioctl, 1860 #ifdef CONFIG_COMPAT 1861 .compat_ioctl = kvm_vcpu_compat_ioctl, 1862 #endif 1863 .mmap = kvm_vcpu_mmap, 1864 .llseek = noop_llseek, 1865 }; 1866 1867 /* 1868 * Allocates an inode for the vcpu. 1869 */ 1870 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 1871 { 1872 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR); 1873 } 1874 1875 /* 1876 * Creates some virtual cpus. Good luck creating more than one. 1877 */ 1878 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 1879 { 1880 int r; 1881 struct kvm_vcpu *vcpu, *v; 1882 1883 vcpu = kvm_arch_vcpu_create(kvm, id); 1884 if (IS_ERR(vcpu)) 1885 return PTR_ERR(vcpu); 1886 1887 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 1888 1889 r = kvm_arch_vcpu_setup(vcpu); 1890 if (r) 1891 goto vcpu_destroy; 1892 1893 mutex_lock(&kvm->lock); 1894 if (!kvm_vcpu_compatible(vcpu)) { 1895 r = -EINVAL; 1896 goto unlock_vcpu_destroy; 1897 } 1898 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 1899 r = -EINVAL; 1900 goto unlock_vcpu_destroy; 1901 } 1902 1903 kvm_for_each_vcpu(r, v, kvm) 1904 if (v->vcpu_id == id) { 1905 r = -EEXIST; 1906 goto unlock_vcpu_destroy; 1907 } 1908 1909 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 1910 1911 /* Now it's all set up, let userspace reach it */ 1912 kvm_get_kvm(kvm); 1913 r = create_vcpu_fd(vcpu); 1914 if (r < 0) { 1915 kvm_put_kvm(kvm); 1916 goto unlock_vcpu_destroy; 1917 } 1918 1919 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 1920 smp_wmb(); 1921 atomic_inc(&kvm->online_vcpus); 1922 1923 mutex_unlock(&kvm->lock); 1924 kvm_arch_vcpu_postcreate(vcpu); 1925 return r; 1926 1927 unlock_vcpu_destroy: 1928 mutex_unlock(&kvm->lock); 1929 vcpu_destroy: 1930 kvm_arch_vcpu_destroy(vcpu); 1931 return r; 1932 } 1933 1934 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 1935 { 1936 if (sigset) { 1937 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 1938 vcpu->sigset_active = 1; 1939 vcpu->sigset = *sigset; 1940 } else 1941 vcpu->sigset_active = 0; 1942 return 0; 1943 } 1944 1945 static long kvm_vcpu_ioctl(struct file *filp, 1946 unsigned int ioctl, unsigned long arg) 1947 { 1948 struct kvm_vcpu *vcpu = filp->private_data; 1949 void __user *argp = (void __user *)arg; 1950 int r; 1951 struct kvm_fpu *fpu = NULL; 1952 struct kvm_sregs *kvm_sregs = NULL; 1953 1954 if (vcpu->kvm->mm != current->mm) 1955 return -EIO; 1956 1957 #if defined(CONFIG_S390) || defined(CONFIG_PPC) 1958 /* 1959 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 1960 * so vcpu_load() would break it. 1961 */ 1962 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT) 1963 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1964 #endif 1965 1966 1967 r = vcpu_load(vcpu); 1968 if (r) 1969 return r; 1970 switch (ioctl) { 1971 case KVM_RUN: 1972 r = -EINVAL; 1973 if (arg) 1974 goto out; 1975 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 1976 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 1977 break; 1978 case KVM_GET_REGS: { 1979 struct kvm_regs *kvm_regs; 1980 1981 r = -ENOMEM; 1982 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 1983 if (!kvm_regs) 1984 goto out; 1985 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 1986 if (r) 1987 goto out_free1; 1988 r = -EFAULT; 1989 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 1990 goto out_free1; 1991 r = 0; 1992 out_free1: 1993 kfree(kvm_regs); 1994 break; 1995 } 1996 case KVM_SET_REGS: { 1997 struct kvm_regs *kvm_regs; 1998 1999 r = -ENOMEM; 2000 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2001 if (IS_ERR(kvm_regs)) { 2002 r = PTR_ERR(kvm_regs); 2003 goto out; 2004 } 2005 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2006 kfree(kvm_regs); 2007 break; 2008 } 2009 case KVM_GET_SREGS: { 2010 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2011 r = -ENOMEM; 2012 if (!kvm_sregs) 2013 goto out; 2014 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2015 if (r) 2016 goto out; 2017 r = -EFAULT; 2018 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2019 goto out; 2020 r = 0; 2021 break; 2022 } 2023 case KVM_SET_SREGS: { 2024 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2025 if (IS_ERR(kvm_sregs)) { 2026 r = PTR_ERR(kvm_sregs); 2027 kvm_sregs = NULL; 2028 goto out; 2029 } 2030 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2031 break; 2032 } 2033 case KVM_GET_MP_STATE: { 2034 struct kvm_mp_state mp_state; 2035 2036 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2037 if (r) 2038 goto out; 2039 r = -EFAULT; 2040 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 2041 goto out; 2042 r = 0; 2043 break; 2044 } 2045 case KVM_SET_MP_STATE: { 2046 struct kvm_mp_state mp_state; 2047 2048 r = -EFAULT; 2049 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 2050 goto out; 2051 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2052 break; 2053 } 2054 case KVM_TRANSLATE: { 2055 struct kvm_translation tr; 2056 2057 r = -EFAULT; 2058 if (copy_from_user(&tr, argp, sizeof tr)) 2059 goto out; 2060 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2061 if (r) 2062 goto out; 2063 r = -EFAULT; 2064 if (copy_to_user(argp, &tr, sizeof tr)) 2065 goto out; 2066 r = 0; 2067 break; 2068 } 2069 case KVM_SET_GUEST_DEBUG: { 2070 struct kvm_guest_debug dbg; 2071 2072 r = -EFAULT; 2073 if (copy_from_user(&dbg, argp, sizeof dbg)) 2074 goto out; 2075 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2076 break; 2077 } 2078 case KVM_SET_SIGNAL_MASK: { 2079 struct kvm_signal_mask __user *sigmask_arg = argp; 2080 struct kvm_signal_mask kvm_sigmask; 2081 sigset_t sigset, *p; 2082 2083 p = NULL; 2084 if (argp) { 2085 r = -EFAULT; 2086 if (copy_from_user(&kvm_sigmask, argp, 2087 sizeof kvm_sigmask)) 2088 goto out; 2089 r = -EINVAL; 2090 if (kvm_sigmask.len != sizeof sigset) 2091 goto out; 2092 r = -EFAULT; 2093 if (copy_from_user(&sigset, sigmask_arg->sigset, 2094 sizeof sigset)) 2095 goto out; 2096 p = &sigset; 2097 } 2098 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2099 break; 2100 } 2101 case KVM_GET_FPU: { 2102 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2103 r = -ENOMEM; 2104 if (!fpu) 2105 goto out; 2106 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2107 if (r) 2108 goto out; 2109 r = -EFAULT; 2110 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2111 goto out; 2112 r = 0; 2113 break; 2114 } 2115 case KVM_SET_FPU: { 2116 fpu = memdup_user(argp, sizeof(*fpu)); 2117 if (IS_ERR(fpu)) { 2118 r = PTR_ERR(fpu); 2119 fpu = NULL; 2120 goto out; 2121 } 2122 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2123 break; 2124 } 2125 default: 2126 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2127 } 2128 out: 2129 vcpu_put(vcpu); 2130 kfree(fpu); 2131 kfree(kvm_sregs); 2132 return r; 2133 } 2134 2135 #ifdef CONFIG_COMPAT 2136 static long kvm_vcpu_compat_ioctl(struct file *filp, 2137 unsigned int ioctl, unsigned long arg) 2138 { 2139 struct kvm_vcpu *vcpu = filp->private_data; 2140 void __user *argp = compat_ptr(arg); 2141 int r; 2142 2143 if (vcpu->kvm->mm != current->mm) 2144 return -EIO; 2145 2146 switch (ioctl) { 2147 case KVM_SET_SIGNAL_MASK: { 2148 struct kvm_signal_mask __user *sigmask_arg = argp; 2149 struct kvm_signal_mask kvm_sigmask; 2150 compat_sigset_t csigset; 2151 sigset_t sigset; 2152 2153 if (argp) { 2154 r = -EFAULT; 2155 if (copy_from_user(&kvm_sigmask, argp, 2156 sizeof kvm_sigmask)) 2157 goto out; 2158 r = -EINVAL; 2159 if (kvm_sigmask.len != sizeof csigset) 2160 goto out; 2161 r = -EFAULT; 2162 if (copy_from_user(&csigset, sigmask_arg->sigset, 2163 sizeof csigset)) 2164 goto out; 2165 sigset_from_compat(&sigset, &csigset); 2166 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2167 } else 2168 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2169 break; 2170 } 2171 default: 2172 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2173 } 2174 2175 out: 2176 return r; 2177 } 2178 #endif 2179 2180 static long kvm_vm_ioctl(struct file *filp, 2181 unsigned int ioctl, unsigned long arg) 2182 { 2183 struct kvm *kvm = filp->private_data; 2184 void __user *argp = (void __user *)arg; 2185 int r; 2186 2187 if (kvm->mm != current->mm) 2188 return -EIO; 2189 switch (ioctl) { 2190 case KVM_CREATE_VCPU: 2191 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2192 break; 2193 case KVM_SET_USER_MEMORY_REGION: { 2194 struct kvm_userspace_memory_region kvm_userspace_mem; 2195 2196 r = -EFAULT; 2197 if (copy_from_user(&kvm_userspace_mem, argp, 2198 sizeof kvm_userspace_mem)) 2199 goto out; 2200 2201 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, true); 2202 break; 2203 } 2204 case KVM_GET_DIRTY_LOG: { 2205 struct kvm_dirty_log log; 2206 2207 r = -EFAULT; 2208 if (copy_from_user(&log, argp, sizeof log)) 2209 goto out; 2210 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2211 break; 2212 } 2213 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2214 case KVM_REGISTER_COALESCED_MMIO: { 2215 struct kvm_coalesced_mmio_zone zone; 2216 r = -EFAULT; 2217 if (copy_from_user(&zone, argp, sizeof zone)) 2218 goto out; 2219 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2220 break; 2221 } 2222 case KVM_UNREGISTER_COALESCED_MMIO: { 2223 struct kvm_coalesced_mmio_zone zone; 2224 r = -EFAULT; 2225 if (copy_from_user(&zone, argp, sizeof zone)) 2226 goto out; 2227 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2228 break; 2229 } 2230 #endif 2231 case KVM_IRQFD: { 2232 struct kvm_irqfd data; 2233 2234 r = -EFAULT; 2235 if (copy_from_user(&data, argp, sizeof data)) 2236 goto out; 2237 r = kvm_irqfd(kvm, &data); 2238 break; 2239 } 2240 case KVM_IOEVENTFD: { 2241 struct kvm_ioeventfd data; 2242 2243 r = -EFAULT; 2244 if (copy_from_user(&data, argp, sizeof data)) 2245 goto out; 2246 r = kvm_ioeventfd(kvm, &data); 2247 break; 2248 } 2249 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2250 case KVM_SET_BOOT_CPU_ID: 2251 r = 0; 2252 mutex_lock(&kvm->lock); 2253 if (atomic_read(&kvm->online_vcpus) != 0) 2254 r = -EBUSY; 2255 else 2256 kvm->bsp_vcpu_id = arg; 2257 mutex_unlock(&kvm->lock); 2258 break; 2259 #endif 2260 #ifdef CONFIG_HAVE_KVM_MSI 2261 case KVM_SIGNAL_MSI: { 2262 struct kvm_msi msi; 2263 2264 r = -EFAULT; 2265 if (copy_from_user(&msi, argp, sizeof msi)) 2266 goto out; 2267 r = kvm_send_userspace_msi(kvm, &msi); 2268 break; 2269 } 2270 #endif 2271 #ifdef __KVM_HAVE_IRQ_LINE 2272 case KVM_IRQ_LINE_STATUS: 2273 case KVM_IRQ_LINE: { 2274 struct kvm_irq_level irq_event; 2275 2276 r = -EFAULT; 2277 if (copy_from_user(&irq_event, argp, sizeof irq_event)) 2278 goto out; 2279 2280 r = kvm_vm_ioctl_irq_line(kvm, &irq_event); 2281 if (r) 2282 goto out; 2283 2284 r = -EFAULT; 2285 if (ioctl == KVM_IRQ_LINE_STATUS) { 2286 if (copy_to_user(argp, &irq_event, sizeof irq_event)) 2287 goto out; 2288 } 2289 2290 r = 0; 2291 break; 2292 } 2293 #endif 2294 default: 2295 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2296 if (r == -ENOTTY) 2297 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg); 2298 } 2299 out: 2300 return r; 2301 } 2302 2303 #ifdef CONFIG_COMPAT 2304 struct compat_kvm_dirty_log { 2305 __u32 slot; 2306 __u32 padding1; 2307 union { 2308 compat_uptr_t dirty_bitmap; /* one bit per page */ 2309 __u64 padding2; 2310 }; 2311 }; 2312 2313 static long kvm_vm_compat_ioctl(struct file *filp, 2314 unsigned int ioctl, unsigned long arg) 2315 { 2316 struct kvm *kvm = filp->private_data; 2317 int r; 2318 2319 if (kvm->mm != current->mm) 2320 return -EIO; 2321 switch (ioctl) { 2322 case KVM_GET_DIRTY_LOG: { 2323 struct compat_kvm_dirty_log compat_log; 2324 struct kvm_dirty_log log; 2325 2326 r = -EFAULT; 2327 if (copy_from_user(&compat_log, (void __user *)arg, 2328 sizeof(compat_log))) 2329 goto out; 2330 log.slot = compat_log.slot; 2331 log.padding1 = compat_log.padding1; 2332 log.padding2 = compat_log.padding2; 2333 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2334 2335 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2336 break; 2337 } 2338 default: 2339 r = kvm_vm_ioctl(filp, ioctl, arg); 2340 } 2341 2342 out: 2343 return r; 2344 } 2345 #endif 2346 2347 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2348 { 2349 struct page *page[1]; 2350 unsigned long addr; 2351 int npages; 2352 gfn_t gfn = vmf->pgoff; 2353 struct kvm *kvm = vma->vm_file->private_data; 2354 2355 addr = gfn_to_hva(kvm, gfn); 2356 if (kvm_is_error_hva(addr)) 2357 return VM_FAULT_SIGBUS; 2358 2359 npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page, 2360 NULL); 2361 if (unlikely(npages != 1)) 2362 return VM_FAULT_SIGBUS; 2363 2364 vmf->page = page[0]; 2365 return 0; 2366 } 2367 2368 static const struct vm_operations_struct kvm_vm_vm_ops = { 2369 .fault = kvm_vm_fault, 2370 }; 2371 2372 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) 2373 { 2374 vma->vm_ops = &kvm_vm_vm_ops; 2375 return 0; 2376 } 2377 2378 static struct file_operations kvm_vm_fops = { 2379 .release = kvm_vm_release, 2380 .unlocked_ioctl = kvm_vm_ioctl, 2381 #ifdef CONFIG_COMPAT 2382 .compat_ioctl = kvm_vm_compat_ioctl, 2383 #endif 2384 .mmap = kvm_vm_mmap, 2385 .llseek = noop_llseek, 2386 }; 2387 2388 static int kvm_dev_ioctl_create_vm(unsigned long type) 2389 { 2390 int r; 2391 struct kvm *kvm; 2392 2393 kvm = kvm_create_vm(type); 2394 if (IS_ERR(kvm)) 2395 return PTR_ERR(kvm); 2396 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2397 r = kvm_coalesced_mmio_init(kvm); 2398 if (r < 0) { 2399 kvm_put_kvm(kvm); 2400 return r; 2401 } 2402 #endif 2403 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 2404 if (r < 0) 2405 kvm_put_kvm(kvm); 2406 2407 return r; 2408 } 2409 2410 static long kvm_dev_ioctl_check_extension_generic(long arg) 2411 { 2412 switch (arg) { 2413 case KVM_CAP_USER_MEMORY: 2414 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2415 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2416 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2417 case KVM_CAP_SET_BOOT_CPU_ID: 2418 #endif 2419 case KVM_CAP_INTERNAL_ERROR_DATA: 2420 #ifdef CONFIG_HAVE_KVM_MSI 2421 case KVM_CAP_SIGNAL_MSI: 2422 #endif 2423 return 1; 2424 #ifdef KVM_CAP_IRQ_ROUTING 2425 case KVM_CAP_IRQ_ROUTING: 2426 return KVM_MAX_IRQ_ROUTES; 2427 #endif 2428 default: 2429 break; 2430 } 2431 return kvm_dev_ioctl_check_extension(arg); 2432 } 2433 2434 static long kvm_dev_ioctl(struct file *filp, 2435 unsigned int ioctl, unsigned long arg) 2436 { 2437 long r = -EINVAL; 2438 2439 switch (ioctl) { 2440 case KVM_GET_API_VERSION: 2441 r = -EINVAL; 2442 if (arg) 2443 goto out; 2444 r = KVM_API_VERSION; 2445 break; 2446 case KVM_CREATE_VM: 2447 r = kvm_dev_ioctl_create_vm(arg); 2448 break; 2449 case KVM_CHECK_EXTENSION: 2450 r = kvm_dev_ioctl_check_extension_generic(arg); 2451 break; 2452 case KVM_GET_VCPU_MMAP_SIZE: 2453 r = -EINVAL; 2454 if (arg) 2455 goto out; 2456 r = PAGE_SIZE; /* struct kvm_run */ 2457 #ifdef CONFIG_X86 2458 r += PAGE_SIZE; /* pio data page */ 2459 #endif 2460 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2461 r += PAGE_SIZE; /* coalesced mmio ring page */ 2462 #endif 2463 break; 2464 case KVM_TRACE_ENABLE: 2465 case KVM_TRACE_PAUSE: 2466 case KVM_TRACE_DISABLE: 2467 r = -EOPNOTSUPP; 2468 break; 2469 default: 2470 return kvm_arch_dev_ioctl(filp, ioctl, arg); 2471 } 2472 out: 2473 return r; 2474 } 2475 2476 static struct file_operations kvm_chardev_ops = { 2477 .unlocked_ioctl = kvm_dev_ioctl, 2478 .compat_ioctl = kvm_dev_ioctl, 2479 .llseek = noop_llseek, 2480 }; 2481 2482 static struct miscdevice kvm_dev = { 2483 KVM_MINOR, 2484 "kvm", 2485 &kvm_chardev_ops, 2486 }; 2487 2488 static void hardware_enable_nolock(void *junk) 2489 { 2490 int cpu = raw_smp_processor_id(); 2491 int r; 2492 2493 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2494 return; 2495 2496 cpumask_set_cpu(cpu, cpus_hardware_enabled); 2497 2498 r = kvm_arch_hardware_enable(NULL); 2499 2500 if (r) { 2501 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2502 atomic_inc(&hardware_enable_failed); 2503 printk(KERN_INFO "kvm: enabling virtualization on " 2504 "CPU%d failed\n", cpu); 2505 } 2506 } 2507 2508 static void hardware_enable(void *junk) 2509 { 2510 raw_spin_lock(&kvm_lock); 2511 hardware_enable_nolock(junk); 2512 raw_spin_unlock(&kvm_lock); 2513 } 2514 2515 static void hardware_disable_nolock(void *junk) 2516 { 2517 int cpu = raw_smp_processor_id(); 2518 2519 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2520 return; 2521 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2522 kvm_arch_hardware_disable(NULL); 2523 } 2524 2525 static void hardware_disable(void *junk) 2526 { 2527 raw_spin_lock(&kvm_lock); 2528 hardware_disable_nolock(junk); 2529 raw_spin_unlock(&kvm_lock); 2530 } 2531 2532 static void hardware_disable_all_nolock(void) 2533 { 2534 BUG_ON(!kvm_usage_count); 2535 2536 kvm_usage_count--; 2537 if (!kvm_usage_count) 2538 on_each_cpu(hardware_disable_nolock, NULL, 1); 2539 } 2540 2541 static void hardware_disable_all(void) 2542 { 2543 raw_spin_lock(&kvm_lock); 2544 hardware_disable_all_nolock(); 2545 raw_spin_unlock(&kvm_lock); 2546 } 2547 2548 static int hardware_enable_all(void) 2549 { 2550 int r = 0; 2551 2552 raw_spin_lock(&kvm_lock); 2553 2554 kvm_usage_count++; 2555 if (kvm_usage_count == 1) { 2556 atomic_set(&hardware_enable_failed, 0); 2557 on_each_cpu(hardware_enable_nolock, NULL, 1); 2558 2559 if (atomic_read(&hardware_enable_failed)) { 2560 hardware_disable_all_nolock(); 2561 r = -EBUSY; 2562 } 2563 } 2564 2565 raw_spin_unlock(&kvm_lock); 2566 2567 return r; 2568 } 2569 2570 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 2571 void *v) 2572 { 2573 int cpu = (long)v; 2574 2575 if (!kvm_usage_count) 2576 return NOTIFY_OK; 2577 2578 val &= ~CPU_TASKS_FROZEN; 2579 switch (val) { 2580 case CPU_DYING: 2581 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 2582 cpu); 2583 hardware_disable(NULL); 2584 break; 2585 case CPU_STARTING: 2586 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 2587 cpu); 2588 hardware_enable(NULL); 2589 break; 2590 } 2591 return NOTIFY_OK; 2592 } 2593 2594 2595 asmlinkage void kvm_spurious_fault(void) 2596 { 2597 /* Fault while not rebooting. We want the trace. */ 2598 BUG(); 2599 } 2600 EXPORT_SYMBOL_GPL(kvm_spurious_fault); 2601 2602 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 2603 void *v) 2604 { 2605 /* 2606 * Some (well, at least mine) BIOSes hang on reboot if 2607 * in vmx root mode. 2608 * 2609 * And Intel TXT required VMX off for all cpu when system shutdown. 2610 */ 2611 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 2612 kvm_rebooting = true; 2613 on_each_cpu(hardware_disable_nolock, NULL, 1); 2614 return NOTIFY_OK; 2615 } 2616 2617 static struct notifier_block kvm_reboot_notifier = { 2618 .notifier_call = kvm_reboot, 2619 .priority = 0, 2620 }; 2621 2622 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 2623 { 2624 int i; 2625 2626 for (i = 0; i < bus->dev_count; i++) { 2627 struct kvm_io_device *pos = bus->range[i].dev; 2628 2629 kvm_iodevice_destructor(pos); 2630 } 2631 kfree(bus); 2632 } 2633 2634 int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 2635 { 2636 const struct kvm_io_range *r1 = p1; 2637 const struct kvm_io_range *r2 = p2; 2638 2639 if (r1->addr < r2->addr) 2640 return -1; 2641 if (r1->addr + r1->len > r2->addr + r2->len) 2642 return 1; 2643 return 0; 2644 } 2645 2646 int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 2647 gpa_t addr, int len) 2648 { 2649 bus->range[bus->dev_count++] = (struct kvm_io_range) { 2650 .addr = addr, 2651 .len = len, 2652 .dev = dev, 2653 }; 2654 2655 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 2656 kvm_io_bus_sort_cmp, NULL); 2657 2658 return 0; 2659 } 2660 2661 int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 2662 gpa_t addr, int len) 2663 { 2664 struct kvm_io_range *range, key; 2665 int off; 2666 2667 key = (struct kvm_io_range) { 2668 .addr = addr, 2669 .len = len, 2670 }; 2671 2672 range = bsearch(&key, bus->range, bus->dev_count, 2673 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 2674 if (range == NULL) 2675 return -ENOENT; 2676 2677 off = range - bus->range; 2678 2679 while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0) 2680 off--; 2681 2682 return off; 2683 } 2684 2685 /* kvm_io_bus_write - called under kvm->slots_lock */ 2686 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2687 int len, const void *val) 2688 { 2689 int idx; 2690 struct kvm_io_bus *bus; 2691 struct kvm_io_range range; 2692 2693 range = (struct kvm_io_range) { 2694 .addr = addr, 2695 .len = len, 2696 }; 2697 2698 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2699 idx = kvm_io_bus_get_first_dev(bus, addr, len); 2700 if (idx < 0) 2701 return -EOPNOTSUPP; 2702 2703 while (idx < bus->dev_count && 2704 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) { 2705 if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val)) 2706 return 0; 2707 idx++; 2708 } 2709 2710 return -EOPNOTSUPP; 2711 } 2712 2713 /* kvm_io_bus_read - called under kvm->slots_lock */ 2714 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2715 int len, void *val) 2716 { 2717 int idx; 2718 struct kvm_io_bus *bus; 2719 struct kvm_io_range range; 2720 2721 range = (struct kvm_io_range) { 2722 .addr = addr, 2723 .len = len, 2724 }; 2725 2726 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2727 idx = kvm_io_bus_get_first_dev(bus, addr, len); 2728 if (idx < 0) 2729 return -EOPNOTSUPP; 2730 2731 while (idx < bus->dev_count && 2732 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) { 2733 if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val)) 2734 return 0; 2735 idx++; 2736 } 2737 2738 return -EOPNOTSUPP; 2739 } 2740 2741 /* Caller must hold slots_lock. */ 2742 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2743 int len, struct kvm_io_device *dev) 2744 { 2745 struct kvm_io_bus *new_bus, *bus; 2746 2747 bus = kvm->buses[bus_idx]; 2748 if (bus->dev_count > NR_IOBUS_DEVS - 1) 2749 return -ENOSPC; 2750 2751 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) * 2752 sizeof(struct kvm_io_range)), GFP_KERNEL); 2753 if (!new_bus) 2754 return -ENOMEM; 2755 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 2756 sizeof(struct kvm_io_range))); 2757 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 2758 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2759 synchronize_srcu_expedited(&kvm->srcu); 2760 kfree(bus); 2761 2762 return 0; 2763 } 2764 2765 /* Caller must hold slots_lock. */ 2766 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 2767 struct kvm_io_device *dev) 2768 { 2769 int i, r; 2770 struct kvm_io_bus *new_bus, *bus; 2771 2772 bus = kvm->buses[bus_idx]; 2773 r = -ENOENT; 2774 for (i = 0; i < bus->dev_count; i++) 2775 if (bus->range[i].dev == dev) { 2776 r = 0; 2777 break; 2778 } 2779 2780 if (r) 2781 return r; 2782 2783 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) * 2784 sizeof(struct kvm_io_range)), GFP_KERNEL); 2785 if (!new_bus) 2786 return -ENOMEM; 2787 2788 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 2789 new_bus->dev_count--; 2790 memcpy(new_bus->range + i, bus->range + i + 1, 2791 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 2792 2793 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2794 synchronize_srcu_expedited(&kvm->srcu); 2795 kfree(bus); 2796 return r; 2797 } 2798 2799 static struct notifier_block kvm_cpu_notifier = { 2800 .notifier_call = kvm_cpu_hotplug, 2801 }; 2802 2803 static int vm_stat_get(void *_offset, u64 *val) 2804 { 2805 unsigned offset = (long)_offset; 2806 struct kvm *kvm; 2807 2808 *val = 0; 2809 raw_spin_lock(&kvm_lock); 2810 list_for_each_entry(kvm, &vm_list, vm_list) 2811 *val += *(u32 *)((void *)kvm + offset); 2812 raw_spin_unlock(&kvm_lock); 2813 return 0; 2814 } 2815 2816 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 2817 2818 static int vcpu_stat_get(void *_offset, u64 *val) 2819 { 2820 unsigned offset = (long)_offset; 2821 struct kvm *kvm; 2822 struct kvm_vcpu *vcpu; 2823 int i; 2824 2825 *val = 0; 2826 raw_spin_lock(&kvm_lock); 2827 list_for_each_entry(kvm, &vm_list, vm_list) 2828 kvm_for_each_vcpu(i, vcpu, kvm) 2829 *val += *(u32 *)((void *)vcpu + offset); 2830 2831 raw_spin_unlock(&kvm_lock); 2832 return 0; 2833 } 2834 2835 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 2836 2837 static const struct file_operations *stat_fops[] = { 2838 [KVM_STAT_VCPU] = &vcpu_stat_fops, 2839 [KVM_STAT_VM] = &vm_stat_fops, 2840 }; 2841 2842 static int kvm_init_debug(void) 2843 { 2844 int r = -EFAULT; 2845 struct kvm_stats_debugfs_item *p; 2846 2847 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 2848 if (kvm_debugfs_dir == NULL) 2849 goto out; 2850 2851 for (p = debugfs_entries; p->name; ++p) { 2852 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 2853 (void *)(long)p->offset, 2854 stat_fops[p->kind]); 2855 if (p->dentry == NULL) 2856 goto out_dir; 2857 } 2858 2859 return 0; 2860 2861 out_dir: 2862 debugfs_remove_recursive(kvm_debugfs_dir); 2863 out: 2864 return r; 2865 } 2866 2867 static void kvm_exit_debug(void) 2868 { 2869 struct kvm_stats_debugfs_item *p; 2870 2871 for (p = debugfs_entries; p->name; ++p) 2872 debugfs_remove(p->dentry); 2873 debugfs_remove(kvm_debugfs_dir); 2874 } 2875 2876 static int kvm_suspend(void) 2877 { 2878 if (kvm_usage_count) 2879 hardware_disable_nolock(NULL); 2880 return 0; 2881 } 2882 2883 static void kvm_resume(void) 2884 { 2885 if (kvm_usage_count) { 2886 WARN_ON(raw_spin_is_locked(&kvm_lock)); 2887 hardware_enable_nolock(NULL); 2888 } 2889 } 2890 2891 static struct syscore_ops kvm_syscore_ops = { 2892 .suspend = kvm_suspend, 2893 .resume = kvm_resume, 2894 }; 2895 2896 static inline 2897 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 2898 { 2899 return container_of(pn, struct kvm_vcpu, preempt_notifier); 2900 } 2901 2902 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 2903 { 2904 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 2905 2906 kvm_arch_vcpu_load(vcpu, cpu); 2907 } 2908 2909 static void kvm_sched_out(struct preempt_notifier *pn, 2910 struct task_struct *next) 2911 { 2912 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 2913 2914 kvm_arch_vcpu_put(vcpu); 2915 } 2916 2917 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 2918 struct module *module) 2919 { 2920 int r; 2921 int cpu; 2922 2923 r = kvm_arch_init(opaque); 2924 if (r) 2925 goto out_fail; 2926 2927 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 2928 r = -ENOMEM; 2929 goto out_free_0; 2930 } 2931 2932 r = kvm_arch_hardware_setup(); 2933 if (r < 0) 2934 goto out_free_0a; 2935 2936 for_each_online_cpu(cpu) { 2937 smp_call_function_single(cpu, 2938 kvm_arch_check_processor_compat, 2939 &r, 1); 2940 if (r < 0) 2941 goto out_free_1; 2942 } 2943 2944 r = register_cpu_notifier(&kvm_cpu_notifier); 2945 if (r) 2946 goto out_free_2; 2947 register_reboot_notifier(&kvm_reboot_notifier); 2948 2949 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 2950 if (!vcpu_align) 2951 vcpu_align = __alignof__(struct kvm_vcpu); 2952 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 2953 0, NULL); 2954 if (!kvm_vcpu_cache) { 2955 r = -ENOMEM; 2956 goto out_free_3; 2957 } 2958 2959 r = kvm_async_pf_init(); 2960 if (r) 2961 goto out_free; 2962 2963 kvm_chardev_ops.owner = module; 2964 kvm_vm_fops.owner = module; 2965 kvm_vcpu_fops.owner = module; 2966 2967 r = misc_register(&kvm_dev); 2968 if (r) { 2969 printk(KERN_ERR "kvm: misc device register failed\n"); 2970 goto out_unreg; 2971 } 2972 2973 register_syscore_ops(&kvm_syscore_ops); 2974 2975 kvm_preempt_ops.sched_in = kvm_sched_in; 2976 kvm_preempt_ops.sched_out = kvm_sched_out; 2977 2978 r = kvm_init_debug(); 2979 if (r) { 2980 printk(KERN_ERR "kvm: create debugfs files failed\n"); 2981 goto out_undebugfs; 2982 } 2983 2984 return 0; 2985 2986 out_undebugfs: 2987 unregister_syscore_ops(&kvm_syscore_ops); 2988 out_unreg: 2989 kvm_async_pf_deinit(); 2990 out_free: 2991 kmem_cache_destroy(kvm_vcpu_cache); 2992 out_free_3: 2993 unregister_reboot_notifier(&kvm_reboot_notifier); 2994 unregister_cpu_notifier(&kvm_cpu_notifier); 2995 out_free_2: 2996 out_free_1: 2997 kvm_arch_hardware_unsetup(); 2998 out_free_0a: 2999 free_cpumask_var(cpus_hardware_enabled); 3000 out_free_0: 3001 kvm_arch_exit(); 3002 out_fail: 3003 return r; 3004 } 3005 EXPORT_SYMBOL_GPL(kvm_init); 3006 3007 void kvm_exit(void) 3008 { 3009 kvm_exit_debug(); 3010 misc_deregister(&kvm_dev); 3011 kmem_cache_destroy(kvm_vcpu_cache); 3012 kvm_async_pf_deinit(); 3013 unregister_syscore_ops(&kvm_syscore_ops); 3014 unregister_reboot_notifier(&kvm_reboot_notifier); 3015 unregister_cpu_notifier(&kvm_cpu_notifier); 3016 on_each_cpu(hardware_disable_nolock, NULL, 1); 3017 kvm_arch_hardware_unsetup(); 3018 kvm_arch_exit(); 3019 free_cpumask_var(cpus_hardware_enabled); 3020 } 3021 EXPORT_SYMBOL_GPL(kvm_exit); 3022