kvm_main.c (402efdb48d6a6aef9e91e3112a94b679d787fef7) kvm_main.c (2df72e9bc4c505d8357012f2924589f3d16f9d44)
1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.

--- 86 unchanged lines hidden (view full) ---

95
96static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
97
98bool kvm_rebooting;
99EXPORT_SYMBOL_GPL(kvm_rebooting);
100
101static bool largepages_enabled = true;
102
1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.

--- 86 unchanged lines hidden (view full) ---

95
96static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
97
98bool kvm_rebooting;
99EXPORT_SYMBOL_GPL(kvm_rebooting);
100
101static bool largepages_enabled = true;
102
103static struct page *hwpoison_page;
104static pfn_t hwpoison_pfn;
105
106struct page *fault_page;
107pfn_t fault_pfn;
108
109inline int kvm_is_mmio_pfn(pfn_t pfn)
103bool kvm_is_mmio_pfn(pfn_t pfn)
110{
111 if (pfn_valid(pfn)) {
112 int reserved;
113 struct page *tail = pfn_to_page(pfn);
114 struct page *head = compound_trans_head(tail);
115 reserved = PageReserved(head);
116 if (head != tail) {
117 /*

--- 113 unchanged lines hidden (view full) ---

231
232 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
233 if (!page) {
234 r = -ENOMEM;
235 goto fail;
236 }
237 vcpu->run = page_address(page);
238
104{
105 if (pfn_valid(pfn)) {
106 int reserved;
107 struct page *tail = pfn_to_page(pfn);
108 struct page *head = compound_trans_head(tail);
109 reserved = PageReserved(head);
110 if (head != tail) {
111 /*

--- 113 unchanged lines hidden (view full) ---

225
226 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
227 if (!page) {
228 r = -ENOMEM;
229 goto fail;
230 }
231 vcpu->run = page_address(page);
232
233 kvm_vcpu_set_in_spin_loop(vcpu, false);
234 kvm_vcpu_set_dy_eligible(vcpu, false);
235
239 r = kvm_arch_vcpu_init(vcpu);
240 if (r < 0)
241 goto fail_free_run;
242 return 0;
243
244fail_free_run:
245 free_page((unsigned long)vcpu->run);
246fail:

--- 80 unchanged lines hidden (view full) ---

327 idx = srcu_read_lock(&kvm->srcu);
328 spin_lock(&kvm->mmu_lock);
329 /*
330 * The count increase must become visible at unlock time as no
331 * spte can be established without taking the mmu_lock and
332 * count is also read inside the mmu_lock critical section.
333 */
334 kvm->mmu_notifier_count++;
236 r = kvm_arch_vcpu_init(vcpu);
237 if (r < 0)
238 goto fail_free_run;
239 return 0;
240
241fail_free_run:
242 free_page((unsigned long)vcpu->run);
243fail:

--- 80 unchanged lines hidden (view full) ---

324 idx = srcu_read_lock(&kvm->srcu);
325 spin_lock(&kvm->mmu_lock);
326 /*
327 * The count increase must become visible at unlock time as no
328 * spte can be established without taking the mmu_lock and
329 * count is also read inside the mmu_lock critical section.
330 */
331 kvm->mmu_notifier_count++;
335 for (; start < end; start += PAGE_SIZE)
336 need_tlb_flush |= kvm_unmap_hva(kvm, start);
332 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
337 need_tlb_flush |= kvm->tlbs_dirty;
338 /* we've to flush the tlb before the pages can be freed */
339 if (need_tlb_flush)
340 kvm_flush_remote_tlbs(kvm);
341
342 spin_unlock(&kvm->mmu_lock);
343 srcu_read_unlock(&kvm->srcu, idx);
344}

--- 62 unchanged lines hidden (view full) ---

407
408static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
409 struct mm_struct *mm)
410{
411 struct kvm *kvm = mmu_notifier_to_kvm(mn);
412 int idx;
413
414 idx = srcu_read_lock(&kvm->srcu);
333 need_tlb_flush |= kvm->tlbs_dirty;
334 /* we've to flush the tlb before the pages can be freed */
335 if (need_tlb_flush)
336 kvm_flush_remote_tlbs(kvm);
337
338 spin_unlock(&kvm->mmu_lock);
339 srcu_read_unlock(&kvm->srcu, idx);
340}

--- 62 unchanged lines hidden (view full) ---

403
404static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
405 struct mm_struct *mm)
406{
407 struct kvm *kvm = mmu_notifier_to_kvm(mn);
408 int idx;
409
410 idx = srcu_read_lock(&kvm->srcu);
415 kvm_arch_flush_shadow(kvm);
411 kvm_arch_flush_shadow_all(kvm);
416 srcu_read_unlock(&kvm->srcu, idx);
417}
418
419static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
420 .invalidate_page = kvm_mmu_notifier_invalidate_page,
421 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
422 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
423 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,

--- 122 unchanged lines hidden (view full) ---

546}
547
548/*
549 * Free any memory in @free but not in @dont.
550 */
551static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
552 struct kvm_memory_slot *dont)
553{
412 srcu_read_unlock(&kvm->srcu, idx);
413}
414
415static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
416 .invalidate_page = kvm_mmu_notifier_invalidate_page,
417 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
418 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
419 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,

--- 122 unchanged lines hidden (view full) ---

542}
543
544/*
545 * Free any memory in @free but not in @dont.
546 */
547static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
548 struct kvm_memory_slot *dont)
549{
554 if (!dont || free->rmap != dont->rmap)
555 vfree(free->rmap);
556
557 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
558 kvm_destroy_dirty_bitmap(free);
559
560 kvm_arch_free_memslot(free, dont);
561
562 free->npages = 0;
550 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
551 kvm_destroy_dirty_bitmap(free);
552
553 kvm_arch_free_memslot(free, dont);
554
555 free->npages = 0;
563 free->rmap = NULL;
564}
565
566void kvm_free_physmem(struct kvm *kvm)
567{
568 struct kvm_memslots *slots = kvm->memslots;
569 struct kvm_memory_slot *memslot;
570
571 kvm_for_each_memslot(memslot, slots)

--- 13 unchanged lines hidden (view full) ---

585 raw_spin_unlock(&kvm_lock);
586 kvm_free_irq_routing(kvm);
587 for (i = 0; i < KVM_NR_BUSES; i++)
588 kvm_io_bus_destroy(kvm->buses[i]);
589 kvm_coalesced_mmio_free(kvm);
590#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
591 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
592#else
556}
557
558void kvm_free_physmem(struct kvm *kvm)
559{
560 struct kvm_memslots *slots = kvm->memslots;
561 struct kvm_memory_slot *memslot;
562
563 kvm_for_each_memslot(memslot, slots)

--- 13 unchanged lines hidden (view full) ---

577 raw_spin_unlock(&kvm_lock);
578 kvm_free_irq_routing(kvm);
579 for (i = 0; i < KVM_NR_BUSES; i++)
580 kvm_io_bus_destroy(kvm->buses[i]);
581 kvm_coalesced_mmio_free(kvm);
582#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
583 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
584#else
593 kvm_arch_flush_shadow(kvm);
585 kvm_arch_flush_shadow_all(kvm);
594#endif
595 kvm_arch_destroy_vm(kvm);
596 kvm_free_physmem(kvm);
597 cleanup_srcu_struct(&kvm->srcu);
598 kvm_arch_free_vm(kvm);
599 hardware_disable_all();
600 mmdrop(mm);
601}

--- 79 unchanged lines hidden (view full) ---

681 *old = *new;
682 if (new->npages != npages)
683 sort_memslots(slots);
684 }
685
686 slots->generation++;
687}
688
586#endif
587 kvm_arch_destroy_vm(kvm);
588 kvm_free_physmem(kvm);
589 cleanup_srcu_struct(&kvm->srcu);
590 kvm_arch_free_vm(kvm);
591 hardware_disable_all();
592 mmdrop(mm);
593}

--- 79 unchanged lines hidden (view full) ---

673 *old = *new;
674 if (new->npages != npages)
675 sort_memslots(slots);
676 }
677
678 slots->generation++;
679}
680
681static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
682{
683 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
684
685#ifdef KVM_CAP_READONLY_MEM
686 valid_flags |= KVM_MEM_READONLY;
687#endif
688
689 if (mem->flags & ~valid_flags)
690 return -EINVAL;
691
692 return 0;
693}
694
689/*
690 * Allocate some memory and give it an address in the guest physical address
691 * space.
692 *
693 * Discontiguous memory is allowed, mostly for framebuffers.
694 *
695 * Must be called holding mmap_sem for write.
696 */

--- 4 unchanged lines hidden (view full) ---

701 int r;
702 gfn_t base_gfn;
703 unsigned long npages;
704 unsigned long i;
705 struct kvm_memory_slot *memslot;
706 struct kvm_memory_slot old, new;
707 struct kvm_memslots *slots, *old_memslots;
708
695/*
696 * Allocate some memory and give it an address in the guest physical address
697 * space.
698 *
699 * Discontiguous memory is allowed, mostly for framebuffers.
700 *
701 * Must be called holding mmap_sem for write.
702 */

--- 4 unchanged lines hidden (view full) ---

707 int r;
708 gfn_t base_gfn;
709 unsigned long npages;
710 unsigned long i;
711 struct kvm_memory_slot *memslot;
712 struct kvm_memory_slot old, new;
713 struct kvm_memslots *slots, *old_memslots;
714
715 r = check_memory_region_flags(mem);
716 if (r)
717 goto out;
718
709 r = -EINVAL;
710 /* General sanity checks */
711 if (mem->memory_size & (PAGE_SIZE - 1))
712 goto out;
713 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
714 goto out;
715 /* We can read the guest memory with __xxx_user() later on. */
716 if (user_alloc &&

--- 47 unchanged lines hidden (view full) ---

764 new.dirty_bitmap = NULL;
765
766 r = -ENOMEM;
767
768 /* Allocate if a slot is being created */
769 if (npages && !old.npages) {
770 new.user_alloc = user_alloc;
771 new.userspace_addr = mem->userspace_addr;
719 r = -EINVAL;
720 /* General sanity checks */
721 if (mem->memory_size & (PAGE_SIZE - 1))
722 goto out;
723 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
724 goto out;
725 /* We can read the guest memory with __xxx_user() later on. */
726 if (user_alloc &&

--- 47 unchanged lines hidden (view full) ---

774 new.dirty_bitmap = NULL;
775
776 r = -ENOMEM;
777
778 /* Allocate if a slot is being created */
779 if (npages && !old.npages) {
780 new.user_alloc = user_alloc;
781 new.userspace_addr = mem->userspace_addr;
772#ifndef CONFIG_S390
773 new.rmap = vzalloc(npages * sizeof(*new.rmap));
774 if (!new.rmap)
775 goto out_free;
776#endif /* not defined CONFIG_S390 */
782
777 if (kvm_arch_create_memslot(&new, npages))
778 goto out_free;
779 }
780
781 /* Allocate page dirty bitmap if needed */
782 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
783 if (kvm_create_dirty_bitmap(&new) < 0)
784 goto out_free;

--- 18 unchanged lines hidden (view full) ---

803 synchronize_srcu_expedited(&kvm->srcu);
804 /* From this point no new shadow pages pointing to a deleted
805 * memslot will be created.
806 *
807 * validation of sp->gfn happens in:
808 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
809 * - kvm_is_visible_gfn (mmu_check_roots)
810 */
783 if (kvm_arch_create_memslot(&new, npages))
784 goto out_free;
785 }
786
787 /* Allocate page dirty bitmap if needed */
788 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
789 if (kvm_create_dirty_bitmap(&new) < 0)
790 goto out_free;

--- 18 unchanged lines hidden (view full) ---

809 synchronize_srcu_expedited(&kvm->srcu);
810 /* From this point no new shadow pages pointing to a deleted
811 * memslot will be created.
812 *
813 * validation of sp->gfn happens in:
814 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
815 * - kvm_is_visible_gfn (mmu_check_roots)
816 */
811 kvm_arch_flush_shadow(kvm);
817 kvm_arch_flush_shadow_memslot(kvm, slot);
812 kfree(old_memslots);
813 }
814
815 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
816 if (r)
817 goto out_free;
818
819 /* map/unmap the pages in iommu page table */

--- 7 unchanged lines hidden (view full) ---

827 r = -ENOMEM;
828 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
829 GFP_KERNEL);
830 if (!slots)
831 goto out_free;
832
833 /* actual memory is freed via old in kvm_free_physmem_slot below */
834 if (!npages) {
818 kfree(old_memslots);
819 }
820
821 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
822 if (r)
823 goto out_free;
824
825 /* map/unmap the pages in iommu page table */

--- 7 unchanged lines hidden (view full) ---

833 r = -ENOMEM;
834 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
835 GFP_KERNEL);
836 if (!slots)
837 goto out_free;
838
839 /* actual memory is freed via old in kvm_free_physmem_slot below */
840 if (!npages) {
835 new.rmap = NULL;
836 new.dirty_bitmap = NULL;
837 memset(&new.arch, 0, sizeof(new.arch));
838 }
839
840 update_memslots(slots, &new);
841 old_memslots = kvm->memslots;
842 rcu_assign_pointer(kvm->memslots, slots);
843 synchronize_srcu_expedited(&kvm->srcu);
844
845 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
846
847 /*
848 * If the new memory slot is created, we need to clear all
849 * mmio sptes.
850 */
851 if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT)
841 new.dirty_bitmap = NULL;
842 memset(&new.arch, 0, sizeof(new.arch));
843 }
844
845 update_memslots(slots, &new);
846 old_memslots = kvm->memslots;
847 rcu_assign_pointer(kvm->memslots, slots);
848 synchronize_srcu_expedited(&kvm->srcu);
849
850 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
851
852 /*
853 * If the new memory slot is created, we need to clear all
854 * mmio sptes.
855 */
856 if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT)
852 kvm_arch_flush_shadow(kvm);
857 kvm_arch_flush_shadow_all(kvm);
853
854 kvm_free_physmem_slot(&old, &new);
855 kfree(old_memslots);
856
857 return 0;
858
859out_free:
860 kvm_free_physmem_slot(&new, &old);

--- 66 unchanged lines hidden (view full) ---

927}
928
929void kvm_disable_largepages(void)
930{
931 largepages_enabled = false;
932}
933EXPORT_SYMBOL_GPL(kvm_disable_largepages);
934
858
859 kvm_free_physmem_slot(&old, &new);
860 kfree(old_memslots);
861
862 return 0;
863
864out_free:
865 kvm_free_physmem_slot(&new, &old);

--- 66 unchanged lines hidden (view full) ---

932}
933
934void kvm_disable_largepages(void)
935{
936 largepages_enabled = false;
937}
938EXPORT_SYMBOL_GPL(kvm_disable_largepages);
939
935int is_error_page(struct page *page)
936{
937 return page == bad_page || page == hwpoison_page || page == fault_page;
938}
939EXPORT_SYMBOL_GPL(is_error_page);
940
941int is_error_pfn(pfn_t pfn)
942{
943 return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn;
944}
945EXPORT_SYMBOL_GPL(is_error_pfn);
946
947int is_hwpoison_pfn(pfn_t pfn)
948{
949 return pfn == hwpoison_pfn;
950}
951EXPORT_SYMBOL_GPL(is_hwpoison_pfn);
952
953int is_fault_pfn(pfn_t pfn)
954{
955 return pfn == fault_pfn;
956}
957EXPORT_SYMBOL_GPL(is_fault_pfn);
958
959int is_noslot_pfn(pfn_t pfn)
960{
961 return pfn == bad_pfn;
962}
963EXPORT_SYMBOL_GPL(is_noslot_pfn);
964
965int is_invalid_pfn(pfn_t pfn)
966{
967 return pfn == hwpoison_pfn || pfn == fault_pfn;
968}
969EXPORT_SYMBOL_GPL(is_invalid_pfn);
970
971static inline unsigned long bad_hva(void)
972{
973 return PAGE_OFFSET;
974}
975
976int kvm_is_error_hva(unsigned long addr)
977{
978 return addr == bad_hva();
979}
980EXPORT_SYMBOL_GPL(kvm_is_error_hva);
981
982struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
983{
984 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
985}
986EXPORT_SYMBOL_GPL(gfn_to_memslot);
987
988int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
989{

--- 26 unchanged lines hidden (view full) ---

1016 size = vma_kernel_pagesize(vma);
1017
1018out:
1019 up_read(&current->mm->mmap_sem);
1020
1021 return size;
1022}
1023
940struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
941{
942 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
943}
944EXPORT_SYMBOL_GPL(gfn_to_memslot);
945
946int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
947{

--- 26 unchanged lines hidden (view full) ---

974 size = vma_kernel_pagesize(vma);
975
976out:
977 up_read(&current->mm->mmap_sem);
978
979 return size;
980}
981
1024static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1025 gfn_t *nr_pages)
982static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1026{
983{
984 return slot->flags & KVM_MEM_READONLY;
985}
986
987static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
988 gfn_t *nr_pages, bool write)
989{
1027 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
990 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1028 return bad_hva();
991 return KVM_HVA_ERR_BAD;
1029
992
993 if (memslot_is_readonly(slot) && write)
994 return KVM_HVA_ERR_RO_BAD;
995
1030 if (nr_pages)
1031 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1032
996 if (nr_pages)
997 *nr_pages = slot->npages - (gfn - slot->base_gfn);
998
1033 return gfn_to_hva_memslot(slot, gfn);
999 return __gfn_to_hva_memslot(slot, gfn);
1034}
1035
1000}
1001
1002static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1003 gfn_t *nr_pages)
1004{
1005 return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1006}
1007
1008unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1009 gfn_t gfn)
1010{
1011 return gfn_to_hva_many(slot, gfn, NULL);
1012}
1013EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1014
1036unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1037{
1038 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1039}
1040EXPORT_SYMBOL_GPL(gfn_to_hva);
1041
1015unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1016{
1017 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1018}
1019EXPORT_SYMBOL_GPL(gfn_to_hva);
1020
1042static pfn_t get_fault_pfn(void)
1021/*
1022 * The hva returned by this function is only allowed to be read.
1023 * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
1024 */
1025static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
1043{
1026{
1044 get_page(fault_page);
1045 return fault_pfn;
1027 return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
1046}
1047
1028}
1029
1030static int kvm_read_hva(void *data, void __user *hva, int len)
1031{
1032 return __copy_from_user(data, hva, len);
1033}
1034
1035static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
1036{
1037 return __copy_from_user_inatomic(data, hva, len);
1038}
1039
1048int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1049 unsigned long start, int write, struct page **page)
1050{
1051 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1052
1053 if (write)
1054 flags |= FOLL_WRITE;
1055

--- 4 unchanged lines hidden (view full) ---

1060{
1061 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1062
1063 rc = __get_user_pages(current, current->mm, addr, 1,
1064 flags, NULL, NULL, NULL);
1065 return rc == -EHWPOISON;
1066}
1067
1040int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1041 unsigned long start, int write, struct page **page)
1042{
1043 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1044
1045 if (write)
1046 flags |= FOLL_WRITE;
1047

--- 4 unchanged lines hidden (view full) ---

1052{
1053 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1054
1055 rc = __get_user_pages(current, current->mm, addr, 1,
1056 flags, NULL, NULL, NULL);
1057 return rc == -EHWPOISON;
1058}
1059
1068static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic,
1069 bool *async, bool write_fault, bool *writable)
1060/*
1061 * The atomic path to get the writable pfn which will be stored in @pfn,
1062 * true indicates success, otherwise false is returned.
1063 */
1064static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1065 bool write_fault, bool *writable, pfn_t *pfn)
1070{
1071 struct page *page[1];
1066{
1067 struct page *page[1];
1072 int npages = 0;
1073 pfn_t pfn;
1068 int npages;
1074
1069
1075 /* we can do it either atomically or asynchronously, not both */
1076 BUG_ON(atomic && async);
1070 if (!(async || atomic))
1071 return false;
1077
1072
1078 BUG_ON(!write_fault && !writable);
1073 /*
1074 * Fast pin a writable pfn only if it is a write fault request
1075 * or the caller allows to map a writable pfn for a read fault
1076 * request.
1077 */
1078 if (!(write_fault || writable))
1079 return false;
1079
1080
1080 if (writable)
1081 *writable = true;
1081 npages = __get_user_pages_fast(addr, 1, 1, page);
1082 if (npages == 1) {
1083 *pfn = page_to_pfn(page[0]);
1082
1084
1083 if (atomic || async)
1084 npages = __get_user_pages_fast(addr, 1, 1, page);
1085 if (writable)
1086 *writable = true;
1087 return true;
1088 }
1085
1089
1086 if (unlikely(npages != 1) && !atomic) {
1087 might_sleep();
1090 return false;
1091}
1088
1092
1089 if (writable)
1090 *writable = write_fault;
1093/*
1094 * The slow path to get the pfn of the specified host virtual address,
1095 * 1 indicates success, -errno is returned if error is detected.
1096 */
1097static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1098 bool *writable, pfn_t *pfn)
1099{
1100 struct page *page[1];
1101 int npages = 0;
1091
1102
1092 if (async) {
1093 down_read(&current->mm->mmap_sem);
1094 npages = get_user_page_nowait(current, current->mm,
1095 addr, write_fault, page);
1096 up_read(&current->mm->mmap_sem);
1097 } else
1098 npages = get_user_pages_fast(addr, 1, write_fault,
1099 page);
1103 might_sleep();
1100
1104
1101 /* map read fault as writable if possible */
1102 if (unlikely(!write_fault) && npages == 1) {
1103 struct page *wpage[1];
1105 if (writable)
1106 *writable = write_fault;
1104
1107
1105 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1106 if (npages == 1) {
1107 *writable = true;
1108 put_page(page[0]);
1109 page[0] = wpage[0];
1110 }
1111 npages = 1;
1108 if (async) {
1109 down_read(&current->mm->mmap_sem);
1110 npages = get_user_page_nowait(current, current->mm,
1111 addr, write_fault, page);
1112 up_read(&current->mm->mmap_sem);
1113 } else
1114 npages = get_user_pages_fast(addr, 1, write_fault,
1115 page);
1116 if (npages != 1)
1117 return npages;
1118
1119 /* map read fault as writable if possible */
1120 if (unlikely(!write_fault) && writable) {
1121 struct page *wpage[1];
1122
1123 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1124 if (npages == 1) {
1125 *writable = true;
1126 put_page(page[0]);
1127 page[0] = wpage[0];
1112 }
1128 }
1129
1130 npages = 1;
1113 }
1131 }
1132 *pfn = page_to_pfn(page[0]);
1133 return npages;
1134}
1114
1135
1115 if (unlikely(npages != 1)) {
1116 struct vm_area_struct *vma;
1136static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1137{
1138 if (unlikely(!(vma->vm_flags & VM_READ)))
1139 return false;
1117
1140
1118 if (atomic)
1119 return get_fault_pfn();
1141 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1142 return false;
1120
1143
1121 down_read(&current->mm->mmap_sem);
1122 if (npages == -EHWPOISON ||
1123 (!async && check_user_page_hwpoison(addr))) {
1124 up_read(&current->mm->mmap_sem);
1125 get_page(hwpoison_page);
1126 return page_to_pfn(hwpoison_page);
1127 }
1144 return true;
1145}
1128
1146
1129 vma = find_vma_intersection(current->mm, addr, addr+1);
1147/*
1148 * Pin guest page in memory and return its pfn.
1149 * @addr: host virtual address which maps memory to the guest
1150 * @atomic: whether this function can sleep
1151 * @async: whether this function need to wait IO complete if the
1152 * host page is not in the memory
1153 * @write_fault: whether we should get a writable host page
1154 * @writable: whether it allows to map a writable host page for !@write_fault
1155 *
1156 * The function will map a writable host page for these two cases:
1157 * 1): @write_fault = true
1158 * 2): @write_fault = false && @writable, @writable will tell the caller
1159 * whether the mapping is writable.
1160 */
1161static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1162 bool write_fault, bool *writable)
1163{
1164 struct vm_area_struct *vma;
1165 pfn_t pfn = 0;
1166 int npages;
1130
1167
1131 if (vma == NULL)
1132 pfn = get_fault_pfn();
1133 else if ((vma->vm_flags & VM_PFNMAP)) {
1134 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1135 vma->vm_pgoff;
1136 BUG_ON(!kvm_is_mmio_pfn(pfn));
1137 } else {
1138 if (async && (vma->vm_flags & VM_WRITE))
1139 *async = true;
1140 pfn = get_fault_pfn();
1141 }
1142 up_read(&current->mm->mmap_sem);
1143 } else
1144 pfn = page_to_pfn(page[0]);
1168 /* we can do it either atomically or asynchronously, not both */
1169 BUG_ON(atomic && async);
1145
1170
1171 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1172 return pfn;
1173
1174 if (atomic)
1175 return KVM_PFN_ERR_FAULT;
1176
1177 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1178 if (npages == 1)
1179 return pfn;
1180
1181 down_read(&current->mm->mmap_sem);
1182 if (npages == -EHWPOISON ||
1183 (!async && check_user_page_hwpoison(addr))) {
1184 pfn = KVM_PFN_ERR_HWPOISON;
1185 goto exit;
1186 }
1187
1188 vma = find_vma_intersection(current->mm, addr, addr + 1);
1189
1190 if (vma == NULL)
1191 pfn = KVM_PFN_ERR_FAULT;
1192 else if ((vma->vm_flags & VM_PFNMAP)) {
1193 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1194 vma->vm_pgoff;
1195 BUG_ON(!kvm_is_mmio_pfn(pfn));
1196 } else {
1197 if (async && vma_is_valid(vma, write_fault))
1198 *async = true;
1199 pfn = KVM_PFN_ERR_FAULT;
1200 }
1201exit:
1202 up_read(&current->mm->mmap_sem);
1146 return pfn;
1147}
1148
1203 return pfn;
1204}
1205
1149pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr)
1206static pfn_t
1207__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
1208 bool *async, bool write_fault, bool *writable)
1150{
1209{
1151 return hva_to_pfn(kvm, addr, true, NULL, true, NULL);
1210 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1211
1212 if (addr == KVM_HVA_ERR_RO_BAD)
1213 return KVM_PFN_ERR_RO_FAULT;
1214
1215 if (kvm_is_error_hva(addr))
1216 return KVM_PFN_ERR_BAD;
1217
1218 /* Do not map writable pfn in the readonly memslot. */
1219 if (writable && memslot_is_readonly(slot)) {
1220 *writable = false;
1221 writable = NULL;
1222 }
1223
1224 return hva_to_pfn(addr, atomic, async, write_fault,
1225 writable);
1152}
1226}
1153EXPORT_SYMBOL_GPL(hva_to_pfn_atomic);
1154
1155static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1156 bool write_fault, bool *writable)
1157{
1227
1228static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1229 bool write_fault, bool *writable)
1230{
1158 unsigned long addr;
1231 struct kvm_memory_slot *slot;
1159
1160 if (async)
1161 *async = false;
1162
1232
1233 if (async)
1234 *async = false;
1235
1163 addr = gfn_to_hva(kvm, gfn);
1164 if (kvm_is_error_hva(addr)) {
1165 get_page(bad_page);
1166 return page_to_pfn(bad_page);
1167 }
1236 slot = gfn_to_memslot(kvm, gfn);
1168
1237
1169 return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable);
1238 return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
1239 writable);
1170}
1171
1172pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1173{
1174 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1175}
1176EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1177

--- 12 unchanged lines hidden (view full) ---

1190
1191pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1192 bool *writable)
1193{
1194 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1195}
1196EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1197
1240}
1241
1242pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1243{
1244 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1245}
1246EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1247

--- 12 unchanged lines hidden (view full) ---

1260
1261pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1262 bool *writable)
1263{
1264 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1265}
1266EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1267
1198pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
1199 struct kvm_memory_slot *slot, gfn_t gfn)
1268pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1200{
1269{
1201 unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1202 return hva_to_pfn(kvm, addr, false, NULL, true, NULL);
1270 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1203}
1204
1271}
1272
1273pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1274{
1275 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1276}
1277EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1278
1205int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1206 int nr_pages)
1207{
1208 unsigned long addr;
1209 gfn_t entry;
1210
1211 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1212 if (kvm_is_error_hva(addr))
1213 return -1;
1214
1215 if (entry < nr_pages)
1216 return 0;
1217
1218 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1219}
1220EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1221
1279int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1280 int nr_pages)
1281{
1282 unsigned long addr;
1283 gfn_t entry;
1284
1285 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1286 if (kvm_is_error_hva(addr))
1287 return -1;
1288
1289 if (entry < nr_pages)
1290 return 0;
1291
1292 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1293}
1294EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1295
1296static struct page *kvm_pfn_to_page(pfn_t pfn)
1297{
1298 if (is_error_pfn(pfn))
1299 return KVM_ERR_PTR_BAD_PAGE;
1300
1301 if (kvm_is_mmio_pfn(pfn)) {
1302 WARN_ON(1);
1303 return KVM_ERR_PTR_BAD_PAGE;
1304 }
1305
1306 return pfn_to_page(pfn);
1307}
1308
1222struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1223{
1224 pfn_t pfn;
1225
1226 pfn = gfn_to_pfn(kvm, gfn);
1309struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1310{
1311 pfn_t pfn;
1312
1313 pfn = gfn_to_pfn(kvm, gfn);
1227 if (!kvm_is_mmio_pfn(pfn))
1228 return pfn_to_page(pfn);
1229
1314
1230 WARN_ON(kvm_is_mmio_pfn(pfn));
1231
1232 get_page(bad_page);
1233 return bad_page;
1315 return kvm_pfn_to_page(pfn);
1234}
1235
1236EXPORT_SYMBOL_GPL(gfn_to_page);
1237
1238void kvm_release_page_clean(struct page *page)
1239{
1316}
1317
1318EXPORT_SYMBOL_GPL(gfn_to_page);
1319
1320void kvm_release_page_clean(struct page *page)
1321{
1322 WARN_ON(is_error_page(page));
1323
1240 kvm_release_pfn_clean(page_to_pfn(page));
1241}
1242EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1243
1244void kvm_release_pfn_clean(pfn_t pfn)
1245{
1324 kvm_release_pfn_clean(page_to_pfn(page));
1325}
1326EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1327
1328void kvm_release_pfn_clean(pfn_t pfn)
1329{
1330 WARN_ON(is_error_pfn(pfn));
1331
1246 if (!kvm_is_mmio_pfn(pfn))
1247 put_page(pfn_to_page(pfn));
1248}
1249EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1250
1251void kvm_release_page_dirty(struct page *page)
1252{
1332 if (!kvm_is_mmio_pfn(pfn))
1333 put_page(pfn_to_page(pfn));
1334}
1335EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1336
1337void kvm_release_page_dirty(struct page *page)
1338{
1339 WARN_ON(is_error_page(page));
1340
1253 kvm_release_pfn_dirty(page_to_pfn(page));
1254}
1255EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1256
1257void kvm_release_pfn_dirty(pfn_t pfn)
1258{
1259 kvm_set_pfn_dirty(pfn);
1260 kvm_release_pfn_clean(pfn);

--- 39 unchanged lines hidden (view full) ---

1300}
1301
1302int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1303 int len)
1304{
1305 int r;
1306 unsigned long addr;
1307
1341 kvm_release_pfn_dirty(page_to_pfn(page));
1342}
1343EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1344
1345void kvm_release_pfn_dirty(pfn_t pfn)
1346{
1347 kvm_set_pfn_dirty(pfn);
1348 kvm_release_pfn_clean(pfn);

--- 39 unchanged lines hidden (view full) ---

1388}
1389
1390int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1391 int len)
1392{
1393 int r;
1394 unsigned long addr;
1395
1308 addr = gfn_to_hva(kvm, gfn);
1396 addr = gfn_to_hva_read(kvm, gfn);
1309 if (kvm_is_error_hva(addr))
1310 return -EFAULT;
1397 if (kvm_is_error_hva(addr))
1398 return -EFAULT;
1311 r = __copy_from_user(data, (void __user *)addr + offset, len);
1399 r = kvm_read_hva(data, (void __user *)addr + offset, len);
1312 if (r)
1313 return -EFAULT;
1314 return 0;
1315}
1316EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1317
1318int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1319{

--- 18 unchanged lines hidden (view full) ---

1338int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1339 unsigned long len)
1340{
1341 int r;
1342 unsigned long addr;
1343 gfn_t gfn = gpa >> PAGE_SHIFT;
1344 int offset = offset_in_page(gpa);
1345
1400 if (r)
1401 return -EFAULT;
1402 return 0;
1403}
1404EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1405
1406int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1407{

--- 18 unchanged lines hidden (view full) ---

1426int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1427 unsigned long len)
1428{
1429 int r;
1430 unsigned long addr;
1431 gfn_t gfn = gpa >> PAGE_SHIFT;
1432 int offset = offset_in_page(gpa);
1433
1346 addr = gfn_to_hva(kvm, gfn);
1434 addr = gfn_to_hva_read(kvm, gfn);
1347 if (kvm_is_error_hva(addr))
1348 return -EFAULT;
1349 pagefault_disable();
1435 if (kvm_is_error_hva(addr))
1436 return -EFAULT;
1437 pagefault_disable();
1350 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1438 r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1351 pagefault_enable();
1352 if (r)
1353 return -EFAULT;
1354 return 0;
1355}
1356EXPORT_SYMBOL(kvm_read_guest_atomic);
1357
1358int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,

--- 216 unchanged lines hidden (view full) ---

1575 put_task_struct(task);
1576 return true;
1577 }
1578 put_task_struct(task);
1579 return false;
1580}
1581EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
1582
1439 pagefault_enable();
1440 if (r)
1441 return -EFAULT;
1442 return 0;
1443}
1444EXPORT_SYMBOL(kvm_read_guest_atomic);
1445
1446int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,

--- 216 unchanged lines hidden (view full) ---

1663 put_task_struct(task);
1664 return true;
1665 }
1666 put_task_struct(task);
1667 return false;
1668}
1669EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
1670
1671#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1672/*
1673 * Helper that checks whether a VCPU is eligible for directed yield.
1674 * Most eligible candidate to yield is decided by following heuristics:
1675 *
1676 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
1677 * (preempted lock holder), indicated by @in_spin_loop.
1678 * Set at the beiginning and cleared at the end of interception/PLE handler.
1679 *
1680 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
1681 * chance last time (mostly it has become eligible now since we have probably
1682 * yielded to lockholder in last iteration. This is done by toggling
1683 * @dy_eligible each time a VCPU checked for eligibility.)
1684 *
1685 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
1686 * to preempted lock-holder could result in wrong VCPU selection and CPU
1687 * burning. Giving priority for a potential lock-holder increases lock
1688 * progress.
1689 *
1690 * Since algorithm is based on heuristics, accessing another VCPU data without
1691 * locking does not harm. It may result in trying to yield to same VCPU, fail
1692 * and continue with next VCPU and so on.
1693 */
1694bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
1695{
1696 bool eligible;
1697
1698 eligible = !vcpu->spin_loop.in_spin_loop ||
1699 (vcpu->spin_loop.in_spin_loop &&
1700 vcpu->spin_loop.dy_eligible);
1701
1702 if (vcpu->spin_loop.in_spin_loop)
1703 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
1704
1705 return eligible;
1706}
1707#endif
1583void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1584{
1585 struct kvm *kvm = me->kvm;
1586 struct kvm_vcpu *vcpu;
1587 int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1588 int yielded = 0;
1589 int pass;
1590 int i;
1591
1708void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1709{
1710 struct kvm *kvm = me->kvm;
1711 struct kvm_vcpu *vcpu;
1712 int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1713 int yielded = 0;
1714 int pass;
1715 int i;
1716
1717 kvm_vcpu_set_in_spin_loop(me, true);
1592 /*
1593 * We boost the priority of a VCPU that is runnable but not
1594 * currently running, because it got preempted by something
1595 * else and called schedule in __vcpu_run. Hopefully that
1596 * VCPU is holding the lock that we need and will release it.
1597 * We approximate round-robin by starting at the last boosted VCPU.
1598 */
1599 for (pass = 0; pass < 2 && !yielded; pass++) {
1600 kvm_for_each_vcpu(i, vcpu, kvm) {
1601 if (!pass && i <= last_boosted_vcpu) {
1602 i = last_boosted_vcpu;
1603 continue;
1604 } else if (pass && i > last_boosted_vcpu)
1605 break;
1606 if (vcpu == me)
1607 continue;
1608 if (waitqueue_active(&vcpu->wq))
1609 continue;
1718 /*
1719 * We boost the priority of a VCPU that is runnable but not
1720 * currently running, because it got preempted by something
1721 * else and called schedule in __vcpu_run. Hopefully that
1722 * VCPU is holding the lock that we need and will release it.
1723 * We approximate round-robin by starting at the last boosted VCPU.
1724 */
1725 for (pass = 0; pass < 2 && !yielded; pass++) {
1726 kvm_for_each_vcpu(i, vcpu, kvm) {
1727 if (!pass && i <= last_boosted_vcpu) {
1728 i = last_boosted_vcpu;
1729 continue;
1730 } else if (pass && i > last_boosted_vcpu)
1731 break;
1732 if (vcpu == me)
1733 continue;
1734 if (waitqueue_active(&vcpu->wq))
1735 continue;
1736 if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
1737 continue;
1610 if (kvm_vcpu_yield_to(vcpu)) {
1611 kvm->last_boosted_vcpu = i;
1612 yielded = 1;
1613 break;
1614 }
1615 }
1616 }
1738 if (kvm_vcpu_yield_to(vcpu)) {
1739 kvm->last_boosted_vcpu = i;
1740 yielded = 1;
1741 break;
1742 }
1743 }
1744 }
1745 kvm_vcpu_set_in_spin_loop(me, false);
1746
1747 /* Ensure vcpu is not eligible during next spinloop */
1748 kvm_vcpu_set_dy_eligible(me, false);
1617}
1618EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1619
1620static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1621{
1622 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1623 struct page *page;
1624

--- 346 unchanged lines hidden (view full) ---

1971 goto out;
1972 r = -EINVAL;
1973 if (kvm_sigmask.len != sizeof csigset)
1974 goto out;
1975 r = -EFAULT;
1976 if (copy_from_user(&csigset, sigmask_arg->sigset,
1977 sizeof csigset))
1978 goto out;
1749}
1750EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1751
1752static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1753{
1754 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1755 struct page *page;
1756

--- 346 unchanged lines hidden (view full) ---

2103 goto out;
2104 r = -EINVAL;
2105 if (kvm_sigmask.len != sizeof csigset)
2106 goto out;
2107 r = -EFAULT;
2108 if (copy_from_user(&csigset, sigmask_arg->sigset,
2109 sizeof csigset))
2110 goto out;
1979 sigset_from_compat(&sigset, &csigset);
1980 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1981 } else
1982 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2111 }
2112 sigset_from_compat(&sigset, &csigset);
2113 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1983 break;
1984 }
1985 default:
1986 r = kvm_vcpu_ioctl(filp, ioctl, arg);
1987 }
1988
1989out:
1990 return r;

--- 98 unchanged lines hidden (view full) ---

2089
2090 r = -EFAULT;
2091 if (copy_from_user(&msi, argp, sizeof msi))
2092 goto out;
2093 r = kvm_send_userspace_msi(kvm, &msi);
2094 break;
2095 }
2096#endif
2114 break;
2115 }
2116 default:
2117 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2118 }
2119
2120out:
2121 return r;

--- 98 unchanged lines hidden (view full) ---

2220
2221 r = -EFAULT;
2222 if (copy_from_user(&msi, argp, sizeof msi))
2223 goto out;
2224 r = kvm_send_userspace_msi(kvm, &msi);
2225 break;
2226 }
2227#endif
2228#ifdef __KVM_HAVE_IRQ_LINE
2229 case KVM_IRQ_LINE_STATUS:
2230 case KVM_IRQ_LINE: {
2231 struct kvm_irq_level irq_event;
2232
2233 r = -EFAULT;
2234 if (copy_from_user(&irq_event, argp, sizeof irq_event))
2235 goto out;
2236
2237 r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
2238 if (r)
2239 goto out;
2240
2241 r = -EFAULT;
2242 if (ioctl == KVM_IRQ_LINE_STATUS) {
2243 if (copy_to_user(argp, &irq_event, sizeof irq_event))
2244 goto out;
2245 }
2246
2247 r = 0;
2248 break;
2249 }
2250#endif
2097 default:
2098 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2099 if (r == -ENOTTY)
2100 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2101 }
2102out:
2103 return r;
2104}

--- 588 unchanged lines hidden (view full) ---

2693 }
2694}
2695
2696static struct syscore_ops kvm_syscore_ops = {
2697 .suspend = kvm_suspend,
2698 .resume = kvm_resume,
2699};
2700
2251 default:
2252 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2253 if (r == -ENOTTY)
2254 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2255 }
2256out:
2257 return r;
2258}

--- 588 unchanged lines hidden (view full) ---

2847 }
2848}
2849
2850static struct syscore_ops kvm_syscore_ops = {
2851 .suspend = kvm_suspend,
2852 .resume = kvm_resume,
2853};
2854
2701struct page *bad_page;
2702pfn_t bad_pfn;
2703
2704static inline
2705struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2706{
2707 return container_of(pn, struct kvm_vcpu, preempt_notifier);
2708}
2709
2710static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2711{

--- 15 unchanged lines hidden (view full) ---

2727{
2728 int r;
2729 int cpu;
2730
2731 r = kvm_arch_init(opaque);
2732 if (r)
2733 goto out_fail;
2734
2855static inline
2856struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2857{
2858 return container_of(pn, struct kvm_vcpu, preempt_notifier);
2859}
2860
2861static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2862{

--- 15 unchanged lines hidden (view full) ---

2878{
2879 int r;
2880 int cpu;
2881
2882 r = kvm_arch_init(opaque);
2883 if (r)
2884 goto out_fail;
2885
2735 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2736
2737 if (bad_page == NULL) {
2738 r = -ENOMEM;
2739 goto out;
2740 }
2741
2742 bad_pfn = page_to_pfn(bad_page);
2743
2744 hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2745
2746 if (hwpoison_page == NULL) {
2747 r = -ENOMEM;
2748 goto out_free_0;
2749 }
2750
2751 hwpoison_pfn = page_to_pfn(hwpoison_page);
2752
2753 fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2754
2755 if (fault_page == NULL) {
2756 r = -ENOMEM;
2757 goto out_free_0;
2758 }
2759
2760 fault_pfn = page_to_pfn(fault_page);
2761
2762 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2763 r = -ENOMEM;
2764 goto out_free_0;
2765 }
2766
2767 r = kvm_arch_hardware_setup();
2768 if (r < 0)
2769 goto out_free_0a;

--- 58 unchanged lines hidden (view full) ---

2828 unregister_reboot_notifier(&kvm_reboot_notifier);
2829 unregister_cpu_notifier(&kvm_cpu_notifier);
2830out_free_2:
2831out_free_1:
2832 kvm_arch_hardware_unsetup();
2833out_free_0a:
2834 free_cpumask_var(cpus_hardware_enabled);
2835out_free_0:
2886 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2887 r = -ENOMEM;
2888 goto out_free_0;
2889 }
2890
2891 r = kvm_arch_hardware_setup();
2892 if (r < 0)
2893 goto out_free_0a;

--- 58 unchanged lines hidden (view full) ---

2952 unregister_reboot_notifier(&kvm_reboot_notifier);
2953 unregister_cpu_notifier(&kvm_cpu_notifier);
2954out_free_2:
2955out_free_1:
2956 kvm_arch_hardware_unsetup();
2957out_free_0a:
2958 free_cpumask_var(cpus_hardware_enabled);
2959out_free_0:
2836 if (fault_page)
2837 __free_page(fault_page);
2838 if (hwpoison_page)
2839 __free_page(hwpoison_page);
2840 __free_page(bad_page);
2841out:
2842 kvm_arch_exit();
2843out_fail:
2844 return r;
2845}
2846EXPORT_SYMBOL_GPL(kvm_init);
2847
2848void kvm_exit(void)
2849{
2850 kvm_exit_debug();
2851 misc_deregister(&kvm_dev);
2852 kmem_cache_destroy(kvm_vcpu_cache);
2853 kvm_async_pf_deinit();
2854 unregister_syscore_ops(&kvm_syscore_ops);
2855 unregister_reboot_notifier(&kvm_reboot_notifier);
2856 unregister_cpu_notifier(&kvm_cpu_notifier);
2857 on_each_cpu(hardware_disable_nolock, NULL, 1);
2858 kvm_arch_hardware_unsetup();
2859 kvm_arch_exit();
2860 free_cpumask_var(cpus_hardware_enabled);
2960 kvm_arch_exit();
2961out_fail:
2962 return r;
2963}
2964EXPORT_SYMBOL_GPL(kvm_init);
2965
2966void kvm_exit(void)
2967{
2968 kvm_exit_debug();
2969 misc_deregister(&kvm_dev);
2970 kmem_cache_destroy(kvm_vcpu_cache);
2971 kvm_async_pf_deinit();
2972 unregister_syscore_ops(&kvm_syscore_ops);
2973 unregister_reboot_notifier(&kvm_reboot_notifier);
2974 unregister_cpu_notifier(&kvm_cpu_notifier);
2975 on_each_cpu(hardware_disable_nolock, NULL, 1);
2976 kvm_arch_hardware_unsetup();
2977 kvm_arch_exit();
2978 free_cpumask_var(cpus_hardware_enabled);
2861 __free_page(fault_page);
2862 __free_page(hwpoison_page);
2863 __free_page(bad_page);
2864}
2865EXPORT_SYMBOL_GPL(kvm_exit);
2979}
2980EXPORT_SYMBOL_GPL(kvm_exit);