1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include "iodev.h" 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 51 #include <asm/processor.h> 52 #include <asm/io.h> 53 #include <asm/uaccess.h> 54 #include <asm/pgtable.h> 55 56 #include "coalesced_mmio.h" 57 #include "async_pf.h" 58 59 #define CREATE_TRACE_POINTS 60 #include <trace/events/kvm.h> 61 62 MODULE_AUTHOR("Qumranet"); 63 MODULE_LICENSE("GPL"); 64 65 /* 66 * Ordering of locks: 67 * 68 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 69 */ 70 71 DEFINE_RAW_SPINLOCK(kvm_lock); 72 LIST_HEAD(vm_list); 73 74 static cpumask_var_t cpus_hardware_enabled; 75 static int kvm_usage_count = 0; 76 static atomic_t hardware_enable_failed; 77 78 struct kmem_cache *kvm_vcpu_cache; 79 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 80 81 static __read_mostly struct preempt_ops kvm_preempt_ops; 82 83 struct dentry *kvm_debugfs_dir; 84 85 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 86 unsigned long arg); 87 #ifdef CONFIG_COMPAT 88 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 89 unsigned long arg); 90 #endif 91 static int hardware_enable_all(void); 92 static void hardware_disable_all(void); 93 94 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 95 96 bool kvm_rebooting; 97 EXPORT_SYMBOL_GPL(kvm_rebooting); 98 99 static bool largepages_enabled = true; 100 101 static struct page *hwpoison_page; 102 static pfn_t hwpoison_pfn; 103 104 struct page *fault_page; 105 pfn_t fault_pfn; 106 107 inline int kvm_is_mmio_pfn(pfn_t pfn) 108 { 109 if (pfn_valid(pfn)) { 110 int reserved; 111 struct page *tail = pfn_to_page(pfn); 112 struct page *head = compound_trans_head(tail); 113 reserved = PageReserved(head); 114 if (head != tail) { 115 /* 116 * "head" is not a dangling pointer 117 * (compound_trans_head takes care of that) 118 * but the hugepage may have been splitted 119 * from under us (and we may not hold a 120 * reference count on the head page so it can 121 * be reused before we run PageReferenced), so 122 * we've to check PageTail before returning 123 * what we just read. 124 */ 125 smp_rmb(); 126 if (PageTail(tail)) 127 return reserved; 128 } 129 return PageReserved(tail); 130 } 131 132 return true; 133 } 134 135 /* 136 * Switches to specified vcpu, until a matching vcpu_put() 137 */ 138 void vcpu_load(struct kvm_vcpu *vcpu) 139 { 140 int cpu; 141 142 mutex_lock(&vcpu->mutex); 143 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 144 /* The thread running this VCPU changed. */ 145 struct pid *oldpid = vcpu->pid; 146 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 147 rcu_assign_pointer(vcpu->pid, newpid); 148 synchronize_rcu(); 149 put_pid(oldpid); 150 } 151 cpu = get_cpu(); 152 preempt_notifier_register(&vcpu->preempt_notifier); 153 kvm_arch_vcpu_load(vcpu, cpu); 154 put_cpu(); 155 } 156 157 void vcpu_put(struct kvm_vcpu *vcpu) 158 { 159 preempt_disable(); 160 kvm_arch_vcpu_put(vcpu); 161 preempt_notifier_unregister(&vcpu->preempt_notifier); 162 preempt_enable(); 163 mutex_unlock(&vcpu->mutex); 164 } 165 166 static void ack_flush(void *_completed) 167 { 168 } 169 170 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req) 171 { 172 int i, cpu, me; 173 cpumask_var_t cpus; 174 bool called = true; 175 struct kvm_vcpu *vcpu; 176 177 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 178 179 me = get_cpu(); 180 kvm_for_each_vcpu(i, vcpu, kvm) { 181 kvm_make_request(req, vcpu); 182 cpu = vcpu->cpu; 183 184 /* Set ->requests bit before we read ->mode */ 185 smp_mb(); 186 187 if (cpus != NULL && cpu != -1 && cpu != me && 188 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 189 cpumask_set_cpu(cpu, cpus); 190 } 191 if (unlikely(cpus == NULL)) 192 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 193 else if (!cpumask_empty(cpus)) 194 smp_call_function_many(cpus, ack_flush, NULL, 1); 195 else 196 called = false; 197 put_cpu(); 198 free_cpumask_var(cpus); 199 return called; 200 } 201 202 void kvm_flush_remote_tlbs(struct kvm *kvm) 203 { 204 int dirty_count = kvm->tlbs_dirty; 205 206 smp_mb(); 207 if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 208 ++kvm->stat.remote_tlb_flush; 209 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 210 } 211 212 void kvm_reload_remote_mmus(struct kvm *kvm) 213 { 214 make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 215 } 216 217 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 218 { 219 struct page *page; 220 int r; 221 222 mutex_init(&vcpu->mutex); 223 vcpu->cpu = -1; 224 vcpu->kvm = kvm; 225 vcpu->vcpu_id = id; 226 vcpu->pid = NULL; 227 init_waitqueue_head(&vcpu->wq); 228 kvm_async_pf_vcpu_init(vcpu); 229 230 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 231 if (!page) { 232 r = -ENOMEM; 233 goto fail; 234 } 235 vcpu->run = page_address(page); 236 237 r = kvm_arch_vcpu_init(vcpu); 238 if (r < 0) 239 goto fail_free_run; 240 return 0; 241 242 fail_free_run: 243 free_page((unsigned long)vcpu->run); 244 fail: 245 return r; 246 } 247 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 248 249 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 250 { 251 put_pid(vcpu->pid); 252 kvm_arch_vcpu_uninit(vcpu); 253 free_page((unsigned long)vcpu->run); 254 } 255 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 256 257 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 258 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 259 { 260 return container_of(mn, struct kvm, mmu_notifier); 261 } 262 263 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 264 struct mm_struct *mm, 265 unsigned long address) 266 { 267 struct kvm *kvm = mmu_notifier_to_kvm(mn); 268 int need_tlb_flush, idx; 269 270 /* 271 * When ->invalidate_page runs, the linux pte has been zapped 272 * already but the page is still allocated until 273 * ->invalidate_page returns. So if we increase the sequence 274 * here the kvm page fault will notice if the spte can't be 275 * established because the page is going to be freed. If 276 * instead the kvm page fault establishes the spte before 277 * ->invalidate_page runs, kvm_unmap_hva will release it 278 * before returning. 279 * 280 * The sequence increase only need to be seen at spin_unlock 281 * time, and not at spin_lock time. 282 * 283 * Increasing the sequence after the spin_unlock would be 284 * unsafe because the kvm page fault could then establish the 285 * pte after kvm_unmap_hva returned, without noticing the page 286 * is going to be freed. 287 */ 288 idx = srcu_read_lock(&kvm->srcu); 289 spin_lock(&kvm->mmu_lock); 290 kvm->mmu_notifier_seq++; 291 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 292 spin_unlock(&kvm->mmu_lock); 293 srcu_read_unlock(&kvm->srcu, idx); 294 295 /* we've to flush the tlb before the pages can be freed */ 296 if (need_tlb_flush) 297 kvm_flush_remote_tlbs(kvm); 298 299 } 300 301 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 302 struct mm_struct *mm, 303 unsigned long address, 304 pte_t pte) 305 { 306 struct kvm *kvm = mmu_notifier_to_kvm(mn); 307 int idx; 308 309 idx = srcu_read_lock(&kvm->srcu); 310 spin_lock(&kvm->mmu_lock); 311 kvm->mmu_notifier_seq++; 312 kvm_set_spte_hva(kvm, address, pte); 313 spin_unlock(&kvm->mmu_lock); 314 srcu_read_unlock(&kvm->srcu, idx); 315 } 316 317 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 318 struct mm_struct *mm, 319 unsigned long start, 320 unsigned long end) 321 { 322 struct kvm *kvm = mmu_notifier_to_kvm(mn); 323 int need_tlb_flush = 0, idx; 324 325 idx = srcu_read_lock(&kvm->srcu); 326 spin_lock(&kvm->mmu_lock); 327 /* 328 * The count increase must become visible at unlock time as no 329 * spte can be established without taking the mmu_lock and 330 * count is also read inside the mmu_lock critical section. 331 */ 332 kvm->mmu_notifier_count++; 333 for (; start < end; start += PAGE_SIZE) 334 need_tlb_flush |= kvm_unmap_hva(kvm, start); 335 need_tlb_flush |= kvm->tlbs_dirty; 336 spin_unlock(&kvm->mmu_lock); 337 srcu_read_unlock(&kvm->srcu, idx); 338 339 /* we've to flush the tlb before the pages can be freed */ 340 if (need_tlb_flush) 341 kvm_flush_remote_tlbs(kvm); 342 } 343 344 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 345 struct mm_struct *mm, 346 unsigned long start, 347 unsigned long end) 348 { 349 struct kvm *kvm = mmu_notifier_to_kvm(mn); 350 351 spin_lock(&kvm->mmu_lock); 352 /* 353 * This sequence increase will notify the kvm page fault that 354 * the page that is going to be mapped in the spte could have 355 * been freed. 356 */ 357 kvm->mmu_notifier_seq++; 358 /* 359 * The above sequence increase must be visible before the 360 * below count decrease but both values are read by the kvm 361 * page fault under mmu_lock spinlock so we don't need to add 362 * a smb_wmb() here in between the two. 363 */ 364 kvm->mmu_notifier_count--; 365 spin_unlock(&kvm->mmu_lock); 366 367 BUG_ON(kvm->mmu_notifier_count < 0); 368 } 369 370 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 371 struct mm_struct *mm, 372 unsigned long address) 373 { 374 struct kvm *kvm = mmu_notifier_to_kvm(mn); 375 int young, idx; 376 377 idx = srcu_read_lock(&kvm->srcu); 378 spin_lock(&kvm->mmu_lock); 379 young = kvm_age_hva(kvm, address); 380 spin_unlock(&kvm->mmu_lock); 381 srcu_read_unlock(&kvm->srcu, idx); 382 383 if (young) 384 kvm_flush_remote_tlbs(kvm); 385 386 return young; 387 } 388 389 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 390 struct mm_struct *mm, 391 unsigned long address) 392 { 393 struct kvm *kvm = mmu_notifier_to_kvm(mn); 394 int young, idx; 395 396 idx = srcu_read_lock(&kvm->srcu); 397 spin_lock(&kvm->mmu_lock); 398 young = kvm_test_age_hva(kvm, address); 399 spin_unlock(&kvm->mmu_lock); 400 srcu_read_unlock(&kvm->srcu, idx); 401 402 return young; 403 } 404 405 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 406 struct mm_struct *mm) 407 { 408 struct kvm *kvm = mmu_notifier_to_kvm(mn); 409 int idx; 410 411 idx = srcu_read_lock(&kvm->srcu); 412 kvm_arch_flush_shadow(kvm); 413 srcu_read_unlock(&kvm->srcu, idx); 414 } 415 416 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 417 .invalidate_page = kvm_mmu_notifier_invalidate_page, 418 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 419 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 420 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 421 .test_young = kvm_mmu_notifier_test_young, 422 .change_pte = kvm_mmu_notifier_change_pte, 423 .release = kvm_mmu_notifier_release, 424 }; 425 426 static int kvm_init_mmu_notifier(struct kvm *kvm) 427 { 428 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 429 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 430 } 431 432 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 433 434 static int kvm_init_mmu_notifier(struct kvm *kvm) 435 { 436 return 0; 437 } 438 439 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 440 441 static struct kvm *kvm_create_vm(void) 442 { 443 int r, i; 444 struct kvm *kvm = kvm_arch_alloc_vm(); 445 446 if (!kvm) 447 return ERR_PTR(-ENOMEM); 448 449 r = kvm_arch_init_vm(kvm); 450 if (r) 451 goto out_err_nodisable; 452 453 r = hardware_enable_all(); 454 if (r) 455 goto out_err_nodisable; 456 457 #ifdef CONFIG_HAVE_KVM_IRQCHIP 458 INIT_HLIST_HEAD(&kvm->mask_notifier_list); 459 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 460 #endif 461 462 r = -ENOMEM; 463 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 464 if (!kvm->memslots) 465 goto out_err_nosrcu; 466 if (init_srcu_struct(&kvm->srcu)) 467 goto out_err_nosrcu; 468 for (i = 0; i < KVM_NR_BUSES; i++) { 469 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 470 GFP_KERNEL); 471 if (!kvm->buses[i]) 472 goto out_err; 473 } 474 475 spin_lock_init(&kvm->mmu_lock); 476 kvm->mm = current->mm; 477 atomic_inc(&kvm->mm->mm_count); 478 kvm_eventfd_init(kvm); 479 mutex_init(&kvm->lock); 480 mutex_init(&kvm->irq_lock); 481 mutex_init(&kvm->slots_lock); 482 atomic_set(&kvm->users_count, 1); 483 484 r = kvm_init_mmu_notifier(kvm); 485 if (r) 486 goto out_err; 487 488 raw_spin_lock(&kvm_lock); 489 list_add(&kvm->vm_list, &vm_list); 490 raw_spin_unlock(&kvm_lock); 491 492 return kvm; 493 494 out_err: 495 cleanup_srcu_struct(&kvm->srcu); 496 out_err_nosrcu: 497 hardware_disable_all(); 498 out_err_nodisable: 499 for (i = 0; i < KVM_NR_BUSES; i++) 500 kfree(kvm->buses[i]); 501 kfree(kvm->memslots); 502 kvm_arch_free_vm(kvm); 503 return ERR_PTR(r); 504 } 505 506 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 507 { 508 if (!memslot->dirty_bitmap) 509 return; 510 511 if (2 * kvm_dirty_bitmap_bytes(memslot) > PAGE_SIZE) 512 vfree(memslot->dirty_bitmap_head); 513 else 514 kfree(memslot->dirty_bitmap_head); 515 516 memslot->dirty_bitmap = NULL; 517 memslot->dirty_bitmap_head = NULL; 518 } 519 520 /* 521 * Free any memory in @free but not in @dont. 522 */ 523 static void kvm_free_physmem_slot(struct kvm_memory_slot *free, 524 struct kvm_memory_slot *dont) 525 { 526 int i; 527 528 if (!dont || free->rmap != dont->rmap) 529 vfree(free->rmap); 530 531 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 532 kvm_destroy_dirty_bitmap(free); 533 534 535 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { 536 if (!dont || free->lpage_info[i] != dont->lpage_info[i]) { 537 vfree(free->lpage_info[i]); 538 free->lpage_info[i] = NULL; 539 } 540 } 541 542 free->npages = 0; 543 free->rmap = NULL; 544 } 545 546 void kvm_free_physmem(struct kvm *kvm) 547 { 548 int i; 549 struct kvm_memslots *slots = kvm->memslots; 550 551 for (i = 0; i < slots->nmemslots; ++i) 552 kvm_free_physmem_slot(&slots->memslots[i], NULL); 553 554 kfree(kvm->memslots); 555 } 556 557 static void kvm_destroy_vm(struct kvm *kvm) 558 { 559 int i; 560 struct mm_struct *mm = kvm->mm; 561 562 kvm_arch_sync_events(kvm); 563 raw_spin_lock(&kvm_lock); 564 list_del(&kvm->vm_list); 565 raw_spin_unlock(&kvm_lock); 566 kvm_free_irq_routing(kvm); 567 for (i = 0; i < KVM_NR_BUSES; i++) 568 kvm_io_bus_destroy(kvm->buses[i]); 569 kvm_coalesced_mmio_free(kvm); 570 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 571 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 572 #else 573 kvm_arch_flush_shadow(kvm); 574 #endif 575 kvm_arch_destroy_vm(kvm); 576 kvm_free_physmem(kvm); 577 cleanup_srcu_struct(&kvm->srcu); 578 kvm_arch_free_vm(kvm); 579 hardware_disable_all(); 580 mmdrop(mm); 581 } 582 583 void kvm_get_kvm(struct kvm *kvm) 584 { 585 atomic_inc(&kvm->users_count); 586 } 587 EXPORT_SYMBOL_GPL(kvm_get_kvm); 588 589 void kvm_put_kvm(struct kvm *kvm) 590 { 591 if (atomic_dec_and_test(&kvm->users_count)) 592 kvm_destroy_vm(kvm); 593 } 594 EXPORT_SYMBOL_GPL(kvm_put_kvm); 595 596 597 static int kvm_vm_release(struct inode *inode, struct file *filp) 598 { 599 struct kvm *kvm = filp->private_data; 600 601 kvm_irqfd_release(kvm); 602 603 kvm_put_kvm(kvm); 604 return 0; 605 } 606 607 #ifndef CONFIG_S390 608 /* 609 * Allocation size is twice as large as the actual dirty bitmap size. 610 * This makes it possible to do double buffering: see x86's 611 * kvm_vm_ioctl_get_dirty_log(). 612 */ 613 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 614 { 615 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 616 617 if (dirty_bytes > PAGE_SIZE) 618 memslot->dirty_bitmap = vzalloc(dirty_bytes); 619 else 620 memslot->dirty_bitmap = kzalloc(dirty_bytes, GFP_KERNEL); 621 622 if (!memslot->dirty_bitmap) 623 return -ENOMEM; 624 625 memslot->dirty_bitmap_head = memslot->dirty_bitmap; 626 return 0; 627 } 628 #endif /* !CONFIG_S390 */ 629 630 /* 631 * Allocate some memory and give it an address in the guest physical address 632 * space. 633 * 634 * Discontiguous memory is allowed, mostly for framebuffers. 635 * 636 * Must be called holding mmap_sem for write. 637 */ 638 int __kvm_set_memory_region(struct kvm *kvm, 639 struct kvm_userspace_memory_region *mem, 640 int user_alloc) 641 { 642 int r; 643 gfn_t base_gfn; 644 unsigned long npages; 645 unsigned long i; 646 struct kvm_memory_slot *memslot; 647 struct kvm_memory_slot old, new; 648 struct kvm_memslots *slots, *old_memslots; 649 650 r = -EINVAL; 651 /* General sanity checks */ 652 if (mem->memory_size & (PAGE_SIZE - 1)) 653 goto out; 654 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 655 goto out; 656 /* We can read the guest memory with __xxx_user() later on. */ 657 if (user_alloc && 658 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 659 !access_ok(VERIFY_WRITE, 660 (void __user *)(unsigned long)mem->userspace_addr, 661 mem->memory_size))) 662 goto out; 663 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) 664 goto out; 665 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 666 goto out; 667 668 memslot = &kvm->memslots->memslots[mem->slot]; 669 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 670 npages = mem->memory_size >> PAGE_SHIFT; 671 672 r = -EINVAL; 673 if (npages > KVM_MEM_MAX_NR_PAGES) 674 goto out; 675 676 if (!npages) 677 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; 678 679 new = old = *memslot; 680 681 new.id = mem->slot; 682 new.base_gfn = base_gfn; 683 new.npages = npages; 684 new.flags = mem->flags; 685 686 /* Disallow changing a memory slot's size. */ 687 r = -EINVAL; 688 if (npages && old.npages && npages != old.npages) 689 goto out_free; 690 691 /* Check for overlaps */ 692 r = -EEXIST; 693 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { 694 struct kvm_memory_slot *s = &kvm->memslots->memslots[i]; 695 696 if (s == memslot || !s->npages) 697 continue; 698 if (!((base_gfn + npages <= s->base_gfn) || 699 (base_gfn >= s->base_gfn + s->npages))) 700 goto out_free; 701 } 702 703 /* Free page dirty bitmap if unneeded */ 704 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 705 new.dirty_bitmap = NULL; 706 707 r = -ENOMEM; 708 709 /* Allocate if a slot is being created */ 710 #ifndef CONFIG_S390 711 if (npages && !new.rmap) { 712 new.rmap = vzalloc(npages * sizeof(*new.rmap)); 713 714 if (!new.rmap) 715 goto out_free; 716 717 new.user_alloc = user_alloc; 718 new.userspace_addr = mem->userspace_addr; 719 } 720 if (!npages) 721 goto skip_lpage; 722 723 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { 724 unsigned long ugfn; 725 unsigned long j; 726 int lpages; 727 int level = i + 2; 728 729 /* Avoid unused variable warning if no large pages */ 730 (void)level; 731 732 if (new.lpage_info[i]) 733 continue; 734 735 lpages = 1 + ((base_gfn + npages - 1) 736 >> KVM_HPAGE_GFN_SHIFT(level)); 737 lpages -= base_gfn >> KVM_HPAGE_GFN_SHIFT(level); 738 739 new.lpage_info[i] = vzalloc(lpages * sizeof(*new.lpage_info[i])); 740 741 if (!new.lpage_info[i]) 742 goto out_free; 743 744 if (base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) 745 new.lpage_info[i][0].write_count = 1; 746 if ((base_gfn+npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) 747 new.lpage_info[i][lpages - 1].write_count = 1; 748 ugfn = new.userspace_addr >> PAGE_SHIFT; 749 /* 750 * If the gfn and userspace address are not aligned wrt each 751 * other, or if explicitly asked to, disable large page 752 * support for this slot 753 */ 754 if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) || 755 !largepages_enabled) 756 for (j = 0; j < lpages; ++j) 757 new.lpage_info[i][j].write_count = 1; 758 } 759 760 skip_lpage: 761 762 /* Allocate page dirty bitmap if needed */ 763 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 764 if (kvm_create_dirty_bitmap(&new) < 0) 765 goto out_free; 766 /* destroy any largepage mappings for dirty tracking */ 767 } 768 #else /* not defined CONFIG_S390 */ 769 new.user_alloc = user_alloc; 770 if (user_alloc) 771 new.userspace_addr = mem->userspace_addr; 772 #endif /* not defined CONFIG_S390 */ 773 774 if (!npages) { 775 r = -ENOMEM; 776 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 777 if (!slots) 778 goto out_free; 779 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); 780 if (mem->slot >= slots->nmemslots) 781 slots->nmemslots = mem->slot + 1; 782 slots->generation++; 783 slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID; 784 785 old_memslots = kvm->memslots; 786 rcu_assign_pointer(kvm->memslots, slots); 787 synchronize_srcu_expedited(&kvm->srcu); 788 /* From this point no new shadow pages pointing to a deleted 789 * memslot will be created. 790 * 791 * validation of sp->gfn happens in: 792 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 793 * - kvm_is_visible_gfn (mmu_check_roots) 794 */ 795 kvm_arch_flush_shadow(kvm); 796 kfree(old_memslots); 797 } 798 799 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc); 800 if (r) 801 goto out_free; 802 803 /* map the pages in iommu page table */ 804 if (npages) { 805 r = kvm_iommu_map_pages(kvm, &new); 806 if (r) 807 goto out_free; 808 } 809 810 r = -ENOMEM; 811 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 812 if (!slots) 813 goto out_free; 814 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); 815 if (mem->slot >= slots->nmemslots) 816 slots->nmemslots = mem->slot + 1; 817 slots->generation++; 818 819 /* actual memory is freed via old in kvm_free_physmem_slot below */ 820 if (!npages) { 821 new.rmap = NULL; 822 new.dirty_bitmap = NULL; 823 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) 824 new.lpage_info[i] = NULL; 825 } 826 827 slots->memslots[mem->slot] = new; 828 old_memslots = kvm->memslots; 829 rcu_assign_pointer(kvm->memslots, slots); 830 synchronize_srcu_expedited(&kvm->srcu); 831 832 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc); 833 834 /* 835 * If the new memory slot is created, we need to clear all 836 * mmio sptes. 837 */ 838 if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT) 839 kvm_arch_flush_shadow(kvm); 840 841 kvm_free_physmem_slot(&old, &new); 842 kfree(old_memslots); 843 844 return 0; 845 846 out_free: 847 kvm_free_physmem_slot(&new, &old); 848 out: 849 return r; 850 851 } 852 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 853 854 int kvm_set_memory_region(struct kvm *kvm, 855 struct kvm_userspace_memory_region *mem, 856 int user_alloc) 857 { 858 int r; 859 860 mutex_lock(&kvm->slots_lock); 861 r = __kvm_set_memory_region(kvm, mem, user_alloc); 862 mutex_unlock(&kvm->slots_lock); 863 return r; 864 } 865 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 866 867 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 868 struct 869 kvm_userspace_memory_region *mem, 870 int user_alloc) 871 { 872 if (mem->slot >= KVM_MEMORY_SLOTS) 873 return -EINVAL; 874 return kvm_set_memory_region(kvm, mem, user_alloc); 875 } 876 877 int kvm_get_dirty_log(struct kvm *kvm, 878 struct kvm_dirty_log *log, int *is_dirty) 879 { 880 struct kvm_memory_slot *memslot; 881 int r, i; 882 unsigned long n; 883 unsigned long any = 0; 884 885 r = -EINVAL; 886 if (log->slot >= KVM_MEMORY_SLOTS) 887 goto out; 888 889 memslot = &kvm->memslots->memslots[log->slot]; 890 r = -ENOENT; 891 if (!memslot->dirty_bitmap) 892 goto out; 893 894 n = kvm_dirty_bitmap_bytes(memslot); 895 896 for (i = 0; !any && i < n/sizeof(long); ++i) 897 any = memslot->dirty_bitmap[i]; 898 899 r = -EFAULT; 900 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 901 goto out; 902 903 if (any) 904 *is_dirty = 1; 905 906 r = 0; 907 out: 908 return r; 909 } 910 911 void kvm_disable_largepages(void) 912 { 913 largepages_enabled = false; 914 } 915 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 916 917 int is_error_page(struct page *page) 918 { 919 return page == bad_page || page == hwpoison_page || page == fault_page; 920 } 921 EXPORT_SYMBOL_GPL(is_error_page); 922 923 int is_error_pfn(pfn_t pfn) 924 { 925 return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn; 926 } 927 EXPORT_SYMBOL_GPL(is_error_pfn); 928 929 int is_hwpoison_pfn(pfn_t pfn) 930 { 931 return pfn == hwpoison_pfn; 932 } 933 EXPORT_SYMBOL_GPL(is_hwpoison_pfn); 934 935 int is_fault_pfn(pfn_t pfn) 936 { 937 return pfn == fault_pfn; 938 } 939 EXPORT_SYMBOL_GPL(is_fault_pfn); 940 941 int is_noslot_pfn(pfn_t pfn) 942 { 943 return pfn == bad_pfn; 944 } 945 EXPORT_SYMBOL_GPL(is_noslot_pfn); 946 947 int is_invalid_pfn(pfn_t pfn) 948 { 949 return pfn == hwpoison_pfn || pfn == fault_pfn; 950 } 951 EXPORT_SYMBOL_GPL(is_invalid_pfn); 952 953 static inline unsigned long bad_hva(void) 954 { 955 return PAGE_OFFSET; 956 } 957 958 int kvm_is_error_hva(unsigned long addr) 959 { 960 return addr == bad_hva(); 961 } 962 EXPORT_SYMBOL_GPL(kvm_is_error_hva); 963 964 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm_memslots *slots, 965 gfn_t gfn) 966 { 967 int i; 968 969 for (i = 0; i < slots->nmemslots; ++i) { 970 struct kvm_memory_slot *memslot = &slots->memslots[i]; 971 972 if (gfn >= memslot->base_gfn 973 && gfn < memslot->base_gfn + memslot->npages) 974 return memslot; 975 } 976 return NULL; 977 } 978 979 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 980 { 981 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 982 } 983 EXPORT_SYMBOL_GPL(gfn_to_memslot); 984 985 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 986 { 987 int i; 988 struct kvm_memslots *slots = kvm_memslots(kvm); 989 990 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { 991 struct kvm_memory_slot *memslot = &slots->memslots[i]; 992 993 if (memslot->flags & KVM_MEMSLOT_INVALID) 994 continue; 995 996 if (gfn >= memslot->base_gfn 997 && gfn < memslot->base_gfn + memslot->npages) 998 return 1; 999 } 1000 return 0; 1001 } 1002 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1003 1004 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1005 { 1006 struct vm_area_struct *vma; 1007 unsigned long addr, size; 1008 1009 size = PAGE_SIZE; 1010 1011 addr = gfn_to_hva(kvm, gfn); 1012 if (kvm_is_error_hva(addr)) 1013 return PAGE_SIZE; 1014 1015 down_read(¤t->mm->mmap_sem); 1016 vma = find_vma(current->mm, addr); 1017 if (!vma) 1018 goto out; 1019 1020 size = vma_kernel_pagesize(vma); 1021 1022 out: 1023 up_read(¤t->mm->mmap_sem); 1024 1025 return size; 1026 } 1027 1028 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1029 gfn_t *nr_pages) 1030 { 1031 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1032 return bad_hva(); 1033 1034 if (nr_pages) 1035 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1036 1037 return gfn_to_hva_memslot(slot, gfn); 1038 } 1039 1040 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1041 { 1042 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1043 } 1044 EXPORT_SYMBOL_GPL(gfn_to_hva); 1045 1046 static pfn_t get_fault_pfn(void) 1047 { 1048 get_page(fault_page); 1049 return fault_pfn; 1050 } 1051 1052 int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1053 unsigned long start, int write, struct page **page) 1054 { 1055 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1056 1057 if (write) 1058 flags |= FOLL_WRITE; 1059 1060 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1061 } 1062 1063 static inline int check_user_page_hwpoison(unsigned long addr) 1064 { 1065 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1066 1067 rc = __get_user_pages(current, current->mm, addr, 1, 1068 flags, NULL, NULL, NULL); 1069 return rc == -EHWPOISON; 1070 } 1071 1072 static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic, 1073 bool *async, bool write_fault, bool *writable) 1074 { 1075 struct page *page[1]; 1076 int npages = 0; 1077 pfn_t pfn; 1078 1079 /* we can do it either atomically or asynchronously, not both */ 1080 BUG_ON(atomic && async); 1081 1082 BUG_ON(!write_fault && !writable); 1083 1084 if (writable) 1085 *writable = true; 1086 1087 if (atomic || async) 1088 npages = __get_user_pages_fast(addr, 1, 1, page); 1089 1090 if (unlikely(npages != 1) && !atomic) { 1091 might_sleep(); 1092 1093 if (writable) 1094 *writable = write_fault; 1095 1096 if (async) { 1097 down_read(¤t->mm->mmap_sem); 1098 npages = get_user_page_nowait(current, current->mm, 1099 addr, write_fault, page); 1100 up_read(¤t->mm->mmap_sem); 1101 } else 1102 npages = get_user_pages_fast(addr, 1, write_fault, 1103 page); 1104 1105 /* map read fault as writable if possible */ 1106 if (unlikely(!write_fault) && npages == 1) { 1107 struct page *wpage[1]; 1108 1109 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1110 if (npages == 1) { 1111 *writable = true; 1112 put_page(page[0]); 1113 page[0] = wpage[0]; 1114 } 1115 npages = 1; 1116 } 1117 } 1118 1119 if (unlikely(npages != 1)) { 1120 struct vm_area_struct *vma; 1121 1122 if (atomic) 1123 return get_fault_pfn(); 1124 1125 down_read(¤t->mm->mmap_sem); 1126 if (npages == -EHWPOISON || 1127 (!async && check_user_page_hwpoison(addr))) { 1128 up_read(¤t->mm->mmap_sem); 1129 get_page(hwpoison_page); 1130 return page_to_pfn(hwpoison_page); 1131 } 1132 1133 vma = find_vma_intersection(current->mm, addr, addr+1); 1134 1135 if (vma == NULL) 1136 pfn = get_fault_pfn(); 1137 else if ((vma->vm_flags & VM_PFNMAP)) { 1138 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1139 vma->vm_pgoff; 1140 BUG_ON(!kvm_is_mmio_pfn(pfn)); 1141 } else { 1142 if (async && (vma->vm_flags & VM_WRITE)) 1143 *async = true; 1144 pfn = get_fault_pfn(); 1145 } 1146 up_read(¤t->mm->mmap_sem); 1147 } else 1148 pfn = page_to_pfn(page[0]); 1149 1150 return pfn; 1151 } 1152 1153 pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr) 1154 { 1155 return hva_to_pfn(kvm, addr, true, NULL, true, NULL); 1156 } 1157 EXPORT_SYMBOL_GPL(hva_to_pfn_atomic); 1158 1159 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async, 1160 bool write_fault, bool *writable) 1161 { 1162 unsigned long addr; 1163 1164 if (async) 1165 *async = false; 1166 1167 addr = gfn_to_hva(kvm, gfn); 1168 if (kvm_is_error_hva(addr)) { 1169 get_page(bad_page); 1170 return page_to_pfn(bad_page); 1171 } 1172 1173 return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable); 1174 } 1175 1176 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1177 { 1178 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL); 1179 } 1180 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1181 1182 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async, 1183 bool write_fault, bool *writable) 1184 { 1185 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable); 1186 } 1187 EXPORT_SYMBOL_GPL(gfn_to_pfn_async); 1188 1189 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1190 { 1191 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL); 1192 } 1193 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1194 1195 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1196 bool *writable) 1197 { 1198 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable); 1199 } 1200 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1201 1202 pfn_t gfn_to_pfn_memslot(struct kvm *kvm, 1203 struct kvm_memory_slot *slot, gfn_t gfn) 1204 { 1205 unsigned long addr = gfn_to_hva_memslot(slot, gfn); 1206 return hva_to_pfn(kvm, addr, false, NULL, true, NULL); 1207 } 1208 1209 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages, 1210 int nr_pages) 1211 { 1212 unsigned long addr; 1213 gfn_t entry; 1214 1215 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry); 1216 if (kvm_is_error_hva(addr)) 1217 return -1; 1218 1219 if (entry < nr_pages) 1220 return 0; 1221 1222 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1223 } 1224 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1225 1226 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1227 { 1228 pfn_t pfn; 1229 1230 pfn = gfn_to_pfn(kvm, gfn); 1231 if (!kvm_is_mmio_pfn(pfn)) 1232 return pfn_to_page(pfn); 1233 1234 WARN_ON(kvm_is_mmio_pfn(pfn)); 1235 1236 get_page(bad_page); 1237 return bad_page; 1238 } 1239 1240 EXPORT_SYMBOL_GPL(gfn_to_page); 1241 1242 void kvm_release_page_clean(struct page *page) 1243 { 1244 kvm_release_pfn_clean(page_to_pfn(page)); 1245 } 1246 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1247 1248 void kvm_release_pfn_clean(pfn_t pfn) 1249 { 1250 if (!kvm_is_mmio_pfn(pfn)) 1251 put_page(pfn_to_page(pfn)); 1252 } 1253 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1254 1255 void kvm_release_page_dirty(struct page *page) 1256 { 1257 kvm_release_pfn_dirty(page_to_pfn(page)); 1258 } 1259 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1260 1261 void kvm_release_pfn_dirty(pfn_t pfn) 1262 { 1263 kvm_set_pfn_dirty(pfn); 1264 kvm_release_pfn_clean(pfn); 1265 } 1266 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 1267 1268 void kvm_set_page_dirty(struct page *page) 1269 { 1270 kvm_set_pfn_dirty(page_to_pfn(page)); 1271 } 1272 EXPORT_SYMBOL_GPL(kvm_set_page_dirty); 1273 1274 void kvm_set_pfn_dirty(pfn_t pfn) 1275 { 1276 if (!kvm_is_mmio_pfn(pfn)) { 1277 struct page *page = pfn_to_page(pfn); 1278 if (!PageReserved(page)) 1279 SetPageDirty(page); 1280 } 1281 } 1282 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1283 1284 void kvm_set_pfn_accessed(pfn_t pfn) 1285 { 1286 if (!kvm_is_mmio_pfn(pfn)) 1287 mark_page_accessed(pfn_to_page(pfn)); 1288 } 1289 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1290 1291 void kvm_get_pfn(pfn_t pfn) 1292 { 1293 if (!kvm_is_mmio_pfn(pfn)) 1294 get_page(pfn_to_page(pfn)); 1295 } 1296 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1297 1298 static int next_segment(unsigned long len, int offset) 1299 { 1300 if (len > PAGE_SIZE - offset) 1301 return PAGE_SIZE - offset; 1302 else 1303 return len; 1304 } 1305 1306 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1307 int len) 1308 { 1309 int r; 1310 unsigned long addr; 1311 1312 addr = gfn_to_hva(kvm, gfn); 1313 if (kvm_is_error_hva(addr)) 1314 return -EFAULT; 1315 r = __copy_from_user(data, (void __user *)addr + offset, len); 1316 if (r) 1317 return -EFAULT; 1318 return 0; 1319 } 1320 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1321 1322 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1323 { 1324 gfn_t gfn = gpa >> PAGE_SHIFT; 1325 int seg; 1326 int offset = offset_in_page(gpa); 1327 int ret; 1328 1329 while ((seg = next_segment(len, offset)) != 0) { 1330 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1331 if (ret < 0) 1332 return ret; 1333 offset = 0; 1334 len -= seg; 1335 data += seg; 1336 ++gfn; 1337 } 1338 return 0; 1339 } 1340 EXPORT_SYMBOL_GPL(kvm_read_guest); 1341 1342 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1343 unsigned long len) 1344 { 1345 int r; 1346 unsigned long addr; 1347 gfn_t gfn = gpa >> PAGE_SHIFT; 1348 int offset = offset_in_page(gpa); 1349 1350 addr = gfn_to_hva(kvm, gfn); 1351 if (kvm_is_error_hva(addr)) 1352 return -EFAULT; 1353 pagefault_disable(); 1354 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1355 pagefault_enable(); 1356 if (r) 1357 return -EFAULT; 1358 return 0; 1359 } 1360 EXPORT_SYMBOL(kvm_read_guest_atomic); 1361 1362 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1363 int offset, int len) 1364 { 1365 int r; 1366 unsigned long addr; 1367 1368 addr = gfn_to_hva(kvm, gfn); 1369 if (kvm_is_error_hva(addr)) 1370 return -EFAULT; 1371 r = __copy_to_user((void __user *)addr + offset, data, len); 1372 if (r) 1373 return -EFAULT; 1374 mark_page_dirty(kvm, gfn); 1375 return 0; 1376 } 1377 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1378 1379 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1380 unsigned long len) 1381 { 1382 gfn_t gfn = gpa >> PAGE_SHIFT; 1383 int seg; 1384 int offset = offset_in_page(gpa); 1385 int ret; 1386 1387 while ((seg = next_segment(len, offset)) != 0) { 1388 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1389 if (ret < 0) 1390 return ret; 1391 offset = 0; 1392 len -= seg; 1393 data += seg; 1394 ++gfn; 1395 } 1396 return 0; 1397 } 1398 1399 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1400 gpa_t gpa) 1401 { 1402 struct kvm_memslots *slots = kvm_memslots(kvm); 1403 int offset = offset_in_page(gpa); 1404 gfn_t gfn = gpa >> PAGE_SHIFT; 1405 1406 ghc->gpa = gpa; 1407 ghc->generation = slots->generation; 1408 ghc->memslot = __gfn_to_memslot(slots, gfn); 1409 ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL); 1410 if (!kvm_is_error_hva(ghc->hva)) 1411 ghc->hva += offset; 1412 else 1413 return -EFAULT; 1414 1415 return 0; 1416 } 1417 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1418 1419 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1420 void *data, unsigned long len) 1421 { 1422 struct kvm_memslots *slots = kvm_memslots(kvm); 1423 int r; 1424 1425 if (slots->generation != ghc->generation) 1426 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa); 1427 1428 if (kvm_is_error_hva(ghc->hva)) 1429 return -EFAULT; 1430 1431 r = __copy_to_user((void __user *)ghc->hva, data, len); 1432 if (r) 1433 return -EFAULT; 1434 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1435 1436 return 0; 1437 } 1438 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1439 1440 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1441 void *data, unsigned long len) 1442 { 1443 struct kvm_memslots *slots = kvm_memslots(kvm); 1444 int r; 1445 1446 if (slots->generation != ghc->generation) 1447 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa); 1448 1449 if (kvm_is_error_hva(ghc->hva)) 1450 return -EFAULT; 1451 1452 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1453 if (r) 1454 return -EFAULT; 1455 1456 return 0; 1457 } 1458 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1459 1460 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1461 { 1462 return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page, 1463 offset, len); 1464 } 1465 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1466 1467 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1468 { 1469 gfn_t gfn = gpa >> PAGE_SHIFT; 1470 int seg; 1471 int offset = offset_in_page(gpa); 1472 int ret; 1473 1474 while ((seg = next_segment(len, offset)) != 0) { 1475 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1476 if (ret < 0) 1477 return ret; 1478 offset = 0; 1479 len -= seg; 1480 ++gfn; 1481 } 1482 return 0; 1483 } 1484 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1485 1486 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot, 1487 gfn_t gfn) 1488 { 1489 if (memslot && memslot->dirty_bitmap) { 1490 unsigned long rel_gfn = gfn - memslot->base_gfn; 1491 1492 __set_bit_le(rel_gfn, memslot->dirty_bitmap); 1493 } 1494 } 1495 1496 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1497 { 1498 struct kvm_memory_slot *memslot; 1499 1500 memslot = gfn_to_memslot(kvm, gfn); 1501 mark_page_dirty_in_slot(kvm, memslot, gfn); 1502 } 1503 1504 /* 1505 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1506 */ 1507 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1508 { 1509 DEFINE_WAIT(wait); 1510 1511 for (;;) { 1512 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1513 1514 if (kvm_arch_vcpu_runnable(vcpu)) { 1515 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1516 break; 1517 } 1518 if (kvm_cpu_has_pending_timer(vcpu)) 1519 break; 1520 if (signal_pending(current)) 1521 break; 1522 1523 schedule(); 1524 } 1525 1526 finish_wait(&vcpu->wq, &wait); 1527 } 1528 1529 void kvm_resched(struct kvm_vcpu *vcpu) 1530 { 1531 if (!need_resched()) 1532 return; 1533 cond_resched(); 1534 } 1535 EXPORT_SYMBOL_GPL(kvm_resched); 1536 1537 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 1538 { 1539 struct kvm *kvm = me->kvm; 1540 struct kvm_vcpu *vcpu; 1541 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 1542 int yielded = 0; 1543 int pass; 1544 int i; 1545 1546 /* 1547 * We boost the priority of a VCPU that is runnable but not 1548 * currently running, because it got preempted by something 1549 * else and called schedule in __vcpu_run. Hopefully that 1550 * VCPU is holding the lock that we need and will release it. 1551 * We approximate round-robin by starting at the last boosted VCPU. 1552 */ 1553 for (pass = 0; pass < 2 && !yielded; pass++) { 1554 kvm_for_each_vcpu(i, vcpu, kvm) { 1555 struct task_struct *task = NULL; 1556 struct pid *pid; 1557 if (!pass && i < last_boosted_vcpu) { 1558 i = last_boosted_vcpu; 1559 continue; 1560 } else if (pass && i > last_boosted_vcpu) 1561 break; 1562 if (vcpu == me) 1563 continue; 1564 if (waitqueue_active(&vcpu->wq)) 1565 continue; 1566 rcu_read_lock(); 1567 pid = rcu_dereference(vcpu->pid); 1568 if (pid) 1569 task = get_pid_task(vcpu->pid, PIDTYPE_PID); 1570 rcu_read_unlock(); 1571 if (!task) 1572 continue; 1573 if (task->flags & PF_VCPU) { 1574 put_task_struct(task); 1575 continue; 1576 } 1577 if (yield_to(task, 1)) { 1578 put_task_struct(task); 1579 kvm->last_boosted_vcpu = i; 1580 yielded = 1; 1581 break; 1582 } 1583 put_task_struct(task); 1584 } 1585 } 1586 } 1587 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 1588 1589 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1590 { 1591 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 1592 struct page *page; 1593 1594 if (vmf->pgoff == 0) 1595 page = virt_to_page(vcpu->run); 1596 #ifdef CONFIG_X86 1597 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 1598 page = virt_to_page(vcpu->arch.pio_data); 1599 #endif 1600 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1601 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 1602 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 1603 #endif 1604 else 1605 return VM_FAULT_SIGBUS; 1606 get_page(page); 1607 vmf->page = page; 1608 return 0; 1609 } 1610 1611 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 1612 .fault = kvm_vcpu_fault, 1613 }; 1614 1615 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 1616 { 1617 vma->vm_ops = &kvm_vcpu_vm_ops; 1618 return 0; 1619 } 1620 1621 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 1622 { 1623 struct kvm_vcpu *vcpu = filp->private_data; 1624 1625 kvm_put_kvm(vcpu->kvm); 1626 return 0; 1627 } 1628 1629 static struct file_operations kvm_vcpu_fops = { 1630 .release = kvm_vcpu_release, 1631 .unlocked_ioctl = kvm_vcpu_ioctl, 1632 #ifdef CONFIG_COMPAT 1633 .compat_ioctl = kvm_vcpu_compat_ioctl, 1634 #endif 1635 .mmap = kvm_vcpu_mmap, 1636 .llseek = noop_llseek, 1637 }; 1638 1639 /* 1640 * Allocates an inode for the vcpu. 1641 */ 1642 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 1643 { 1644 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR); 1645 } 1646 1647 /* 1648 * Creates some virtual cpus. Good luck creating more than one. 1649 */ 1650 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 1651 { 1652 int r; 1653 struct kvm_vcpu *vcpu, *v; 1654 1655 vcpu = kvm_arch_vcpu_create(kvm, id); 1656 if (IS_ERR(vcpu)) 1657 return PTR_ERR(vcpu); 1658 1659 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 1660 1661 r = kvm_arch_vcpu_setup(vcpu); 1662 if (r) 1663 goto vcpu_destroy; 1664 1665 mutex_lock(&kvm->lock); 1666 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 1667 r = -EINVAL; 1668 goto unlock_vcpu_destroy; 1669 } 1670 1671 kvm_for_each_vcpu(r, v, kvm) 1672 if (v->vcpu_id == id) { 1673 r = -EEXIST; 1674 goto unlock_vcpu_destroy; 1675 } 1676 1677 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 1678 1679 /* Now it's all set up, let userspace reach it */ 1680 kvm_get_kvm(kvm); 1681 r = create_vcpu_fd(vcpu); 1682 if (r < 0) { 1683 kvm_put_kvm(kvm); 1684 goto unlock_vcpu_destroy; 1685 } 1686 1687 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 1688 smp_wmb(); 1689 atomic_inc(&kvm->online_vcpus); 1690 1691 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 1692 if (kvm->bsp_vcpu_id == id) 1693 kvm->bsp_vcpu = vcpu; 1694 #endif 1695 mutex_unlock(&kvm->lock); 1696 return r; 1697 1698 unlock_vcpu_destroy: 1699 mutex_unlock(&kvm->lock); 1700 vcpu_destroy: 1701 kvm_arch_vcpu_destroy(vcpu); 1702 return r; 1703 } 1704 1705 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 1706 { 1707 if (sigset) { 1708 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 1709 vcpu->sigset_active = 1; 1710 vcpu->sigset = *sigset; 1711 } else 1712 vcpu->sigset_active = 0; 1713 return 0; 1714 } 1715 1716 static long kvm_vcpu_ioctl(struct file *filp, 1717 unsigned int ioctl, unsigned long arg) 1718 { 1719 struct kvm_vcpu *vcpu = filp->private_data; 1720 void __user *argp = (void __user *)arg; 1721 int r; 1722 struct kvm_fpu *fpu = NULL; 1723 struct kvm_sregs *kvm_sregs = NULL; 1724 1725 if (vcpu->kvm->mm != current->mm) 1726 return -EIO; 1727 1728 #if defined(CONFIG_S390) || defined(CONFIG_PPC) 1729 /* 1730 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 1731 * so vcpu_load() would break it. 1732 */ 1733 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT) 1734 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1735 #endif 1736 1737 1738 vcpu_load(vcpu); 1739 switch (ioctl) { 1740 case KVM_RUN: 1741 r = -EINVAL; 1742 if (arg) 1743 goto out; 1744 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 1745 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 1746 break; 1747 case KVM_GET_REGS: { 1748 struct kvm_regs *kvm_regs; 1749 1750 r = -ENOMEM; 1751 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 1752 if (!kvm_regs) 1753 goto out; 1754 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 1755 if (r) 1756 goto out_free1; 1757 r = -EFAULT; 1758 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 1759 goto out_free1; 1760 r = 0; 1761 out_free1: 1762 kfree(kvm_regs); 1763 break; 1764 } 1765 case KVM_SET_REGS: { 1766 struct kvm_regs *kvm_regs; 1767 1768 r = -ENOMEM; 1769 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 1770 if (!kvm_regs) 1771 goto out; 1772 r = -EFAULT; 1773 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs))) 1774 goto out_free2; 1775 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 1776 if (r) 1777 goto out_free2; 1778 r = 0; 1779 out_free2: 1780 kfree(kvm_regs); 1781 break; 1782 } 1783 case KVM_GET_SREGS: { 1784 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 1785 r = -ENOMEM; 1786 if (!kvm_sregs) 1787 goto out; 1788 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 1789 if (r) 1790 goto out; 1791 r = -EFAULT; 1792 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 1793 goto out; 1794 r = 0; 1795 break; 1796 } 1797 case KVM_SET_SREGS: { 1798 kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 1799 r = -ENOMEM; 1800 if (!kvm_sregs) 1801 goto out; 1802 r = -EFAULT; 1803 if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs))) 1804 goto out; 1805 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 1806 if (r) 1807 goto out; 1808 r = 0; 1809 break; 1810 } 1811 case KVM_GET_MP_STATE: { 1812 struct kvm_mp_state mp_state; 1813 1814 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 1815 if (r) 1816 goto out; 1817 r = -EFAULT; 1818 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 1819 goto out; 1820 r = 0; 1821 break; 1822 } 1823 case KVM_SET_MP_STATE: { 1824 struct kvm_mp_state mp_state; 1825 1826 r = -EFAULT; 1827 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 1828 goto out; 1829 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 1830 if (r) 1831 goto out; 1832 r = 0; 1833 break; 1834 } 1835 case KVM_TRANSLATE: { 1836 struct kvm_translation tr; 1837 1838 r = -EFAULT; 1839 if (copy_from_user(&tr, argp, sizeof tr)) 1840 goto out; 1841 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 1842 if (r) 1843 goto out; 1844 r = -EFAULT; 1845 if (copy_to_user(argp, &tr, sizeof tr)) 1846 goto out; 1847 r = 0; 1848 break; 1849 } 1850 case KVM_SET_GUEST_DEBUG: { 1851 struct kvm_guest_debug dbg; 1852 1853 r = -EFAULT; 1854 if (copy_from_user(&dbg, argp, sizeof dbg)) 1855 goto out; 1856 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 1857 if (r) 1858 goto out; 1859 r = 0; 1860 break; 1861 } 1862 case KVM_SET_SIGNAL_MASK: { 1863 struct kvm_signal_mask __user *sigmask_arg = argp; 1864 struct kvm_signal_mask kvm_sigmask; 1865 sigset_t sigset, *p; 1866 1867 p = NULL; 1868 if (argp) { 1869 r = -EFAULT; 1870 if (copy_from_user(&kvm_sigmask, argp, 1871 sizeof kvm_sigmask)) 1872 goto out; 1873 r = -EINVAL; 1874 if (kvm_sigmask.len != sizeof sigset) 1875 goto out; 1876 r = -EFAULT; 1877 if (copy_from_user(&sigset, sigmask_arg->sigset, 1878 sizeof sigset)) 1879 goto out; 1880 p = &sigset; 1881 } 1882 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 1883 break; 1884 } 1885 case KVM_GET_FPU: { 1886 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 1887 r = -ENOMEM; 1888 if (!fpu) 1889 goto out; 1890 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 1891 if (r) 1892 goto out; 1893 r = -EFAULT; 1894 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 1895 goto out; 1896 r = 0; 1897 break; 1898 } 1899 case KVM_SET_FPU: { 1900 fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 1901 r = -ENOMEM; 1902 if (!fpu) 1903 goto out; 1904 r = -EFAULT; 1905 if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu))) 1906 goto out; 1907 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 1908 if (r) 1909 goto out; 1910 r = 0; 1911 break; 1912 } 1913 default: 1914 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1915 } 1916 out: 1917 vcpu_put(vcpu); 1918 kfree(fpu); 1919 kfree(kvm_sregs); 1920 return r; 1921 } 1922 1923 #ifdef CONFIG_COMPAT 1924 static long kvm_vcpu_compat_ioctl(struct file *filp, 1925 unsigned int ioctl, unsigned long arg) 1926 { 1927 struct kvm_vcpu *vcpu = filp->private_data; 1928 void __user *argp = compat_ptr(arg); 1929 int r; 1930 1931 if (vcpu->kvm->mm != current->mm) 1932 return -EIO; 1933 1934 switch (ioctl) { 1935 case KVM_SET_SIGNAL_MASK: { 1936 struct kvm_signal_mask __user *sigmask_arg = argp; 1937 struct kvm_signal_mask kvm_sigmask; 1938 compat_sigset_t csigset; 1939 sigset_t sigset; 1940 1941 if (argp) { 1942 r = -EFAULT; 1943 if (copy_from_user(&kvm_sigmask, argp, 1944 sizeof kvm_sigmask)) 1945 goto out; 1946 r = -EINVAL; 1947 if (kvm_sigmask.len != sizeof csigset) 1948 goto out; 1949 r = -EFAULT; 1950 if (copy_from_user(&csigset, sigmask_arg->sigset, 1951 sizeof csigset)) 1952 goto out; 1953 } 1954 sigset_from_compat(&sigset, &csigset); 1955 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 1956 break; 1957 } 1958 default: 1959 r = kvm_vcpu_ioctl(filp, ioctl, arg); 1960 } 1961 1962 out: 1963 return r; 1964 } 1965 #endif 1966 1967 static long kvm_vm_ioctl(struct file *filp, 1968 unsigned int ioctl, unsigned long arg) 1969 { 1970 struct kvm *kvm = filp->private_data; 1971 void __user *argp = (void __user *)arg; 1972 int r; 1973 1974 if (kvm->mm != current->mm) 1975 return -EIO; 1976 switch (ioctl) { 1977 case KVM_CREATE_VCPU: 1978 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 1979 if (r < 0) 1980 goto out; 1981 break; 1982 case KVM_SET_USER_MEMORY_REGION: { 1983 struct kvm_userspace_memory_region kvm_userspace_mem; 1984 1985 r = -EFAULT; 1986 if (copy_from_user(&kvm_userspace_mem, argp, 1987 sizeof kvm_userspace_mem)) 1988 goto out; 1989 1990 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); 1991 if (r) 1992 goto out; 1993 break; 1994 } 1995 case KVM_GET_DIRTY_LOG: { 1996 struct kvm_dirty_log log; 1997 1998 r = -EFAULT; 1999 if (copy_from_user(&log, argp, sizeof log)) 2000 goto out; 2001 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2002 if (r) 2003 goto out; 2004 break; 2005 } 2006 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2007 case KVM_REGISTER_COALESCED_MMIO: { 2008 struct kvm_coalesced_mmio_zone zone; 2009 r = -EFAULT; 2010 if (copy_from_user(&zone, argp, sizeof zone)) 2011 goto out; 2012 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2013 if (r) 2014 goto out; 2015 r = 0; 2016 break; 2017 } 2018 case KVM_UNREGISTER_COALESCED_MMIO: { 2019 struct kvm_coalesced_mmio_zone zone; 2020 r = -EFAULT; 2021 if (copy_from_user(&zone, argp, sizeof zone)) 2022 goto out; 2023 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2024 if (r) 2025 goto out; 2026 r = 0; 2027 break; 2028 } 2029 #endif 2030 case KVM_IRQFD: { 2031 struct kvm_irqfd data; 2032 2033 r = -EFAULT; 2034 if (copy_from_user(&data, argp, sizeof data)) 2035 goto out; 2036 r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags); 2037 break; 2038 } 2039 case KVM_IOEVENTFD: { 2040 struct kvm_ioeventfd data; 2041 2042 r = -EFAULT; 2043 if (copy_from_user(&data, argp, sizeof data)) 2044 goto out; 2045 r = kvm_ioeventfd(kvm, &data); 2046 break; 2047 } 2048 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2049 case KVM_SET_BOOT_CPU_ID: 2050 r = 0; 2051 mutex_lock(&kvm->lock); 2052 if (atomic_read(&kvm->online_vcpus) != 0) 2053 r = -EBUSY; 2054 else 2055 kvm->bsp_vcpu_id = arg; 2056 mutex_unlock(&kvm->lock); 2057 break; 2058 #endif 2059 default: 2060 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2061 if (r == -ENOTTY) 2062 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg); 2063 } 2064 out: 2065 return r; 2066 } 2067 2068 #ifdef CONFIG_COMPAT 2069 struct compat_kvm_dirty_log { 2070 __u32 slot; 2071 __u32 padding1; 2072 union { 2073 compat_uptr_t dirty_bitmap; /* one bit per page */ 2074 __u64 padding2; 2075 }; 2076 }; 2077 2078 static long kvm_vm_compat_ioctl(struct file *filp, 2079 unsigned int ioctl, unsigned long arg) 2080 { 2081 struct kvm *kvm = filp->private_data; 2082 int r; 2083 2084 if (kvm->mm != current->mm) 2085 return -EIO; 2086 switch (ioctl) { 2087 case KVM_GET_DIRTY_LOG: { 2088 struct compat_kvm_dirty_log compat_log; 2089 struct kvm_dirty_log log; 2090 2091 r = -EFAULT; 2092 if (copy_from_user(&compat_log, (void __user *)arg, 2093 sizeof(compat_log))) 2094 goto out; 2095 log.slot = compat_log.slot; 2096 log.padding1 = compat_log.padding1; 2097 log.padding2 = compat_log.padding2; 2098 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2099 2100 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2101 if (r) 2102 goto out; 2103 break; 2104 } 2105 default: 2106 r = kvm_vm_ioctl(filp, ioctl, arg); 2107 } 2108 2109 out: 2110 return r; 2111 } 2112 #endif 2113 2114 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2115 { 2116 struct page *page[1]; 2117 unsigned long addr; 2118 int npages; 2119 gfn_t gfn = vmf->pgoff; 2120 struct kvm *kvm = vma->vm_file->private_data; 2121 2122 addr = gfn_to_hva(kvm, gfn); 2123 if (kvm_is_error_hva(addr)) 2124 return VM_FAULT_SIGBUS; 2125 2126 npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page, 2127 NULL); 2128 if (unlikely(npages != 1)) 2129 return VM_FAULT_SIGBUS; 2130 2131 vmf->page = page[0]; 2132 return 0; 2133 } 2134 2135 static const struct vm_operations_struct kvm_vm_vm_ops = { 2136 .fault = kvm_vm_fault, 2137 }; 2138 2139 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) 2140 { 2141 vma->vm_ops = &kvm_vm_vm_ops; 2142 return 0; 2143 } 2144 2145 static struct file_operations kvm_vm_fops = { 2146 .release = kvm_vm_release, 2147 .unlocked_ioctl = kvm_vm_ioctl, 2148 #ifdef CONFIG_COMPAT 2149 .compat_ioctl = kvm_vm_compat_ioctl, 2150 #endif 2151 .mmap = kvm_vm_mmap, 2152 .llseek = noop_llseek, 2153 }; 2154 2155 static int kvm_dev_ioctl_create_vm(void) 2156 { 2157 int r; 2158 struct kvm *kvm; 2159 2160 kvm = kvm_create_vm(); 2161 if (IS_ERR(kvm)) 2162 return PTR_ERR(kvm); 2163 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2164 r = kvm_coalesced_mmio_init(kvm); 2165 if (r < 0) { 2166 kvm_put_kvm(kvm); 2167 return r; 2168 } 2169 #endif 2170 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 2171 if (r < 0) 2172 kvm_put_kvm(kvm); 2173 2174 return r; 2175 } 2176 2177 static long kvm_dev_ioctl_check_extension_generic(long arg) 2178 { 2179 switch (arg) { 2180 case KVM_CAP_USER_MEMORY: 2181 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2182 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2183 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2184 case KVM_CAP_SET_BOOT_CPU_ID: 2185 #endif 2186 case KVM_CAP_INTERNAL_ERROR_DATA: 2187 return 1; 2188 #ifdef CONFIG_HAVE_KVM_IRQCHIP 2189 case KVM_CAP_IRQ_ROUTING: 2190 return KVM_MAX_IRQ_ROUTES; 2191 #endif 2192 default: 2193 break; 2194 } 2195 return kvm_dev_ioctl_check_extension(arg); 2196 } 2197 2198 static long kvm_dev_ioctl(struct file *filp, 2199 unsigned int ioctl, unsigned long arg) 2200 { 2201 long r = -EINVAL; 2202 2203 switch (ioctl) { 2204 case KVM_GET_API_VERSION: 2205 r = -EINVAL; 2206 if (arg) 2207 goto out; 2208 r = KVM_API_VERSION; 2209 break; 2210 case KVM_CREATE_VM: 2211 r = -EINVAL; 2212 if (arg) 2213 goto out; 2214 r = kvm_dev_ioctl_create_vm(); 2215 break; 2216 case KVM_CHECK_EXTENSION: 2217 r = kvm_dev_ioctl_check_extension_generic(arg); 2218 break; 2219 case KVM_GET_VCPU_MMAP_SIZE: 2220 r = -EINVAL; 2221 if (arg) 2222 goto out; 2223 r = PAGE_SIZE; /* struct kvm_run */ 2224 #ifdef CONFIG_X86 2225 r += PAGE_SIZE; /* pio data page */ 2226 #endif 2227 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2228 r += PAGE_SIZE; /* coalesced mmio ring page */ 2229 #endif 2230 break; 2231 case KVM_TRACE_ENABLE: 2232 case KVM_TRACE_PAUSE: 2233 case KVM_TRACE_DISABLE: 2234 r = -EOPNOTSUPP; 2235 break; 2236 default: 2237 return kvm_arch_dev_ioctl(filp, ioctl, arg); 2238 } 2239 out: 2240 return r; 2241 } 2242 2243 static struct file_operations kvm_chardev_ops = { 2244 .unlocked_ioctl = kvm_dev_ioctl, 2245 .compat_ioctl = kvm_dev_ioctl, 2246 .llseek = noop_llseek, 2247 }; 2248 2249 static struct miscdevice kvm_dev = { 2250 KVM_MINOR, 2251 "kvm", 2252 &kvm_chardev_ops, 2253 }; 2254 2255 static void hardware_enable_nolock(void *junk) 2256 { 2257 int cpu = raw_smp_processor_id(); 2258 int r; 2259 2260 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2261 return; 2262 2263 cpumask_set_cpu(cpu, cpus_hardware_enabled); 2264 2265 r = kvm_arch_hardware_enable(NULL); 2266 2267 if (r) { 2268 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2269 atomic_inc(&hardware_enable_failed); 2270 printk(KERN_INFO "kvm: enabling virtualization on " 2271 "CPU%d failed\n", cpu); 2272 } 2273 } 2274 2275 static void hardware_enable(void *junk) 2276 { 2277 raw_spin_lock(&kvm_lock); 2278 hardware_enable_nolock(junk); 2279 raw_spin_unlock(&kvm_lock); 2280 } 2281 2282 static void hardware_disable_nolock(void *junk) 2283 { 2284 int cpu = raw_smp_processor_id(); 2285 2286 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2287 return; 2288 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2289 kvm_arch_hardware_disable(NULL); 2290 } 2291 2292 static void hardware_disable(void *junk) 2293 { 2294 raw_spin_lock(&kvm_lock); 2295 hardware_disable_nolock(junk); 2296 raw_spin_unlock(&kvm_lock); 2297 } 2298 2299 static void hardware_disable_all_nolock(void) 2300 { 2301 BUG_ON(!kvm_usage_count); 2302 2303 kvm_usage_count--; 2304 if (!kvm_usage_count) 2305 on_each_cpu(hardware_disable_nolock, NULL, 1); 2306 } 2307 2308 static void hardware_disable_all(void) 2309 { 2310 raw_spin_lock(&kvm_lock); 2311 hardware_disable_all_nolock(); 2312 raw_spin_unlock(&kvm_lock); 2313 } 2314 2315 static int hardware_enable_all(void) 2316 { 2317 int r = 0; 2318 2319 raw_spin_lock(&kvm_lock); 2320 2321 kvm_usage_count++; 2322 if (kvm_usage_count == 1) { 2323 atomic_set(&hardware_enable_failed, 0); 2324 on_each_cpu(hardware_enable_nolock, NULL, 1); 2325 2326 if (atomic_read(&hardware_enable_failed)) { 2327 hardware_disable_all_nolock(); 2328 r = -EBUSY; 2329 } 2330 } 2331 2332 raw_spin_unlock(&kvm_lock); 2333 2334 return r; 2335 } 2336 2337 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 2338 void *v) 2339 { 2340 int cpu = (long)v; 2341 2342 if (!kvm_usage_count) 2343 return NOTIFY_OK; 2344 2345 val &= ~CPU_TASKS_FROZEN; 2346 switch (val) { 2347 case CPU_DYING: 2348 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 2349 cpu); 2350 hardware_disable(NULL); 2351 break; 2352 case CPU_STARTING: 2353 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 2354 cpu); 2355 hardware_enable(NULL); 2356 break; 2357 } 2358 return NOTIFY_OK; 2359 } 2360 2361 2362 asmlinkage void kvm_spurious_fault(void) 2363 { 2364 /* Fault while not rebooting. We want the trace. */ 2365 BUG(); 2366 } 2367 EXPORT_SYMBOL_GPL(kvm_spurious_fault); 2368 2369 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 2370 void *v) 2371 { 2372 /* 2373 * Some (well, at least mine) BIOSes hang on reboot if 2374 * in vmx root mode. 2375 * 2376 * And Intel TXT required VMX off for all cpu when system shutdown. 2377 */ 2378 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 2379 kvm_rebooting = true; 2380 on_each_cpu(hardware_disable_nolock, NULL, 1); 2381 return NOTIFY_OK; 2382 } 2383 2384 static struct notifier_block kvm_reboot_notifier = { 2385 .notifier_call = kvm_reboot, 2386 .priority = 0, 2387 }; 2388 2389 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 2390 { 2391 int i; 2392 2393 for (i = 0; i < bus->dev_count; i++) { 2394 struct kvm_io_device *pos = bus->devs[i]; 2395 2396 kvm_iodevice_destructor(pos); 2397 } 2398 kfree(bus); 2399 } 2400 2401 /* kvm_io_bus_write - called under kvm->slots_lock */ 2402 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2403 int len, const void *val) 2404 { 2405 int i; 2406 struct kvm_io_bus *bus; 2407 2408 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2409 for (i = 0; i < bus->dev_count; i++) 2410 if (!kvm_iodevice_write(bus->devs[i], addr, len, val)) 2411 return 0; 2412 return -EOPNOTSUPP; 2413 } 2414 2415 /* kvm_io_bus_read - called under kvm->slots_lock */ 2416 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2417 int len, void *val) 2418 { 2419 int i; 2420 struct kvm_io_bus *bus; 2421 2422 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2423 for (i = 0; i < bus->dev_count; i++) 2424 if (!kvm_iodevice_read(bus->devs[i], addr, len, val)) 2425 return 0; 2426 return -EOPNOTSUPP; 2427 } 2428 2429 /* Caller must hold slots_lock. */ 2430 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, 2431 struct kvm_io_device *dev) 2432 { 2433 struct kvm_io_bus *new_bus, *bus; 2434 2435 bus = kvm->buses[bus_idx]; 2436 if (bus->dev_count > NR_IOBUS_DEVS-1) 2437 return -ENOSPC; 2438 2439 new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL); 2440 if (!new_bus) 2441 return -ENOMEM; 2442 memcpy(new_bus, bus, sizeof(struct kvm_io_bus)); 2443 new_bus->devs[new_bus->dev_count++] = dev; 2444 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2445 synchronize_srcu_expedited(&kvm->srcu); 2446 kfree(bus); 2447 2448 return 0; 2449 } 2450 2451 /* Caller must hold slots_lock. */ 2452 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 2453 struct kvm_io_device *dev) 2454 { 2455 int i, r; 2456 struct kvm_io_bus *new_bus, *bus; 2457 2458 new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL); 2459 if (!new_bus) 2460 return -ENOMEM; 2461 2462 bus = kvm->buses[bus_idx]; 2463 memcpy(new_bus, bus, sizeof(struct kvm_io_bus)); 2464 2465 r = -ENOENT; 2466 for (i = 0; i < new_bus->dev_count; i++) 2467 if (new_bus->devs[i] == dev) { 2468 r = 0; 2469 new_bus->devs[i] = new_bus->devs[--new_bus->dev_count]; 2470 break; 2471 } 2472 2473 if (r) { 2474 kfree(new_bus); 2475 return r; 2476 } 2477 2478 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2479 synchronize_srcu_expedited(&kvm->srcu); 2480 kfree(bus); 2481 return r; 2482 } 2483 2484 static struct notifier_block kvm_cpu_notifier = { 2485 .notifier_call = kvm_cpu_hotplug, 2486 }; 2487 2488 static int vm_stat_get(void *_offset, u64 *val) 2489 { 2490 unsigned offset = (long)_offset; 2491 struct kvm *kvm; 2492 2493 *val = 0; 2494 raw_spin_lock(&kvm_lock); 2495 list_for_each_entry(kvm, &vm_list, vm_list) 2496 *val += *(u32 *)((void *)kvm + offset); 2497 raw_spin_unlock(&kvm_lock); 2498 return 0; 2499 } 2500 2501 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 2502 2503 static int vcpu_stat_get(void *_offset, u64 *val) 2504 { 2505 unsigned offset = (long)_offset; 2506 struct kvm *kvm; 2507 struct kvm_vcpu *vcpu; 2508 int i; 2509 2510 *val = 0; 2511 raw_spin_lock(&kvm_lock); 2512 list_for_each_entry(kvm, &vm_list, vm_list) 2513 kvm_for_each_vcpu(i, vcpu, kvm) 2514 *val += *(u32 *)((void *)vcpu + offset); 2515 2516 raw_spin_unlock(&kvm_lock); 2517 return 0; 2518 } 2519 2520 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 2521 2522 static const struct file_operations *stat_fops[] = { 2523 [KVM_STAT_VCPU] = &vcpu_stat_fops, 2524 [KVM_STAT_VM] = &vm_stat_fops, 2525 }; 2526 2527 static void kvm_init_debug(void) 2528 { 2529 struct kvm_stats_debugfs_item *p; 2530 2531 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 2532 for (p = debugfs_entries; p->name; ++p) 2533 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 2534 (void *)(long)p->offset, 2535 stat_fops[p->kind]); 2536 } 2537 2538 static void kvm_exit_debug(void) 2539 { 2540 struct kvm_stats_debugfs_item *p; 2541 2542 for (p = debugfs_entries; p->name; ++p) 2543 debugfs_remove(p->dentry); 2544 debugfs_remove(kvm_debugfs_dir); 2545 } 2546 2547 static int kvm_suspend(void) 2548 { 2549 if (kvm_usage_count) 2550 hardware_disable_nolock(NULL); 2551 return 0; 2552 } 2553 2554 static void kvm_resume(void) 2555 { 2556 if (kvm_usage_count) { 2557 WARN_ON(raw_spin_is_locked(&kvm_lock)); 2558 hardware_enable_nolock(NULL); 2559 } 2560 } 2561 2562 static struct syscore_ops kvm_syscore_ops = { 2563 .suspend = kvm_suspend, 2564 .resume = kvm_resume, 2565 }; 2566 2567 struct page *bad_page; 2568 pfn_t bad_pfn; 2569 2570 static inline 2571 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 2572 { 2573 return container_of(pn, struct kvm_vcpu, preempt_notifier); 2574 } 2575 2576 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 2577 { 2578 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 2579 2580 kvm_arch_vcpu_load(vcpu, cpu); 2581 } 2582 2583 static void kvm_sched_out(struct preempt_notifier *pn, 2584 struct task_struct *next) 2585 { 2586 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 2587 2588 kvm_arch_vcpu_put(vcpu); 2589 } 2590 2591 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 2592 struct module *module) 2593 { 2594 int r; 2595 int cpu; 2596 2597 r = kvm_arch_init(opaque); 2598 if (r) 2599 goto out_fail; 2600 2601 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 2602 2603 if (bad_page == NULL) { 2604 r = -ENOMEM; 2605 goto out; 2606 } 2607 2608 bad_pfn = page_to_pfn(bad_page); 2609 2610 hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 2611 2612 if (hwpoison_page == NULL) { 2613 r = -ENOMEM; 2614 goto out_free_0; 2615 } 2616 2617 hwpoison_pfn = page_to_pfn(hwpoison_page); 2618 2619 fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 2620 2621 if (fault_page == NULL) { 2622 r = -ENOMEM; 2623 goto out_free_0; 2624 } 2625 2626 fault_pfn = page_to_pfn(fault_page); 2627 2628 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 2629 r = -ENOMEM; 2630 goto out_free_0; 2631 } 2632 2633 r = kvm_arch_hardware_setup(); 2634 if (r < 0) 2635 goto out_free_0a; 2636 2637 for_each_online_cpu(cpu) { 2638 smp_call_function_single(cpu, 2639 kvm_arch_check_processor_compat, 2640 &r, 1); 2641 if (r < 0) 2642 goto out_free_1; 2643 } 2644 2645 r = register_cpu_notifier(&kvm_cpu_notifier); 2646 if (r) 2647 goto out_free_2; 2648 register_reboot_notifier(&kvm_reboot_notifier); 2649 2650 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 2651 if (!vcpu_align) 2652 vcpu_align = __alignof__(struct kvm_vcpu); 2653 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 2654 0, NULL); 2655 if (!kvm_vcpu_cache) { 2656 r = -ENOMEM; 2657 goto out_free_3; 2658 } 2659 2660 r = kvm_async_pf_init(); 2661 if (r) 2662 goto out_free; 2663 2664 kvm_chardev_ops.owner = module; 2665 kvm_vm_fops.owner = module; 2666 kvm_vcpu_fops.owner = module; 2667 2668 r = misc_register(&kvm_dev); 2669 if (r) { 2670 printk(KERN_ERR "kvm: misc device register failed\n"); 2671 goto out_unreg; 2672 } 2673 2674 register_syscore_ops(&kvm_syscore_ops); 2675 2676 kvm_preempt_ops.sched_in = kvm_sched_in; 2677 kvm_preempt_ops.sched_out = kvm_sched_out; 2678 2679 kvm_init_debug(); 2680 2681 return 0; 2682 2683 out_unreg: 2684 kvm_async_pf_deinit(); 2685 out_free: 2686 kmem_cache_destroy(kvm_vcpu_cache); 2687 out_free_3: 2688 unregister_reboot_notifier(&kvm_reboot_notifier); 2689 unregister_cpu_notifier(&kvm_cpu_notifier); 2690 out_free_2: 2691 out_free_1: 2692 kvm_arch_hardware_unsetup(); 2693 out_free_0a: 2694 free_cpumask_var(cpus_hardware_enabled); 2695 out_free_0: 2696 if (fault_page) 2697 __free_page(fault_page); 2698 if (hwpoison_page) 2699 __free_page(hwpoison_page); 2700 __free_page(bad_page); 2701 out: 2702 kvm_arch_exit(); 2703 out_fail: 2704 return r; 2705 } 2706 EXPORT_SYMBOL_GPL(kvm_init); 2707 2708 void kvm_exit(void) 2709 { 2710 kvm_exit_debug(); 2711 misc_deregister(&kvm_dev); 2712 kmem_cache_destroy(kvm_vcpu_cache); 2713 kvm_async_pf_deinit(); 2714 unregister_syscore_ops(&kvm_syscore_ops); 2715 unregister_reboot_notifier(&kvm_reboot_notifier); 2716 unregister_cpu_notifier(&kvm_cpu_notifier); 2717 on_each_cpu(hardware_disable_nolock, NULL, 1); 2718 kvm_arch_hardware_unsetup(); 2719 kvm_arch_exit(); 2720 free_cpumask_var(cpus_hardware_enabled); 2721 __free_page(hwpoison_page); 2722 __free_page(bad_page); 2723 } 2724 EXPORT_SYMBOL_GPL(kvm_exit); 2725