1775c8a3dSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2af585b92SGleb Natapov /* 3af585b92SGleb Natapov * kvm asynchronous fault support 4af585b92SGleb Natapov * 5af585b92SGleb Natapov * Copyright 2010 Red Hat, Inc. 6af585b92SGleb Natapov * 7af585b92SGleb Natapov * Author: 8af585b92SGleb Natapov * Gleb Natapov <gleb@redhat.com> 9af585b92SGleb Natapov */ 10af585b92SGleb Natapov 11af585b92SGleb Natapov #include <linux/kvm_host.h> 12af585b92SGleb Natapov #include <linux/slab.h> 13af585b92SGleb Natapov #include <linux/module.h> 14af585b92SGleb Natapov #include <linux/mmu_context.h> 156e84f315SIngo Molnar #include <linux/sched/mm.h> 16af585b92SGleb Natapov 17af585b92SGleb Natapov #include "async_pf.h" 18af585b92SGleb Natapov #include <trace/events/kvm.h> 19af585b92SGleb Natapov 20af585b92SGleb Natapov static struct kmem_cache *async_pf_cache; 21af585b92SGleb Natapov 22af585b92SGleb Natapov int kvm_async_pf_init(void) 23af585b92SGleb Natapov { 24af585b92SGleb Natapov async_pf_cache = KMEM_CACHE(kvm_async_pf, 0); 25af585b92SGleb Natapov 26af585b92SGleb Natapov if (!async_pf_cache) 27af585b92SGleb Natapov return -ENOMEM; 28af585b92SGleb Natapov 29af585b92SGleb Natapov return 0; 30af585b92SGleb Natapov } 31af585b92SGleb Natapov 32af585b92SGleb Natapov void kvm_async_pf_deinit(void) 33af585b92SGleb Natapov { 34af585b92SGleb Natapov kmem_cache_destroy(async_pf_cache); 35af585b92SGleb Natapov async_pf_cache = NULL; 36af585b92SGleb Natapov } 37af585b92SGleb Natapov 38af585b92SGleb Natapov void kvm_async_pf_vcpu_init(struct kvm_vcpu *vcpu) 39af585b92SGleb Natapov { 40af585b92SGleb Natapov INIT_LIST_HEAD(&vcpu->async_pf.done); 41af585b92SGleb Natapov INIT_LIST_HEAD(&vcpu->async_pf.queue); 42af585b92SGleb Natapov spin_lock_init(&vcpu->async_pf.lock); 43af585b92SGleb Natapov } 44af585b92SGleb Natapov 45af585b92SGleb Natapov static void async_pf_execute(struct work_struct *work) 46af585b92SGleb Natapov { 47af585b92SGleb Natapov struct kvm_async_pf *apf = 48af585b92SGleb Natapov container_of(work, struct kvm_async_pf, work); 49af585b92SGleb Natapov struct mm_struct *mm = apf->mm; 50af585b92SGleb Natapov struct kvm_vcpu *vcpu = apf->vcpu; 51af585b92SGleb Natapov unsigned long addr = apf->addr; 52736c291cSSean Christopherson gpa_t cr2_or_gpa = apf->cr2_or_gpa; 538b7457efSLorenzo Stoakes int locked = 1; 54557a961aSVitaly Kuznetsov bool first; 55af585b92SGleb Natapov 56af585b92SGleb Natapov might_sleep(); 57af585b92SGleb Natapov 581e987790SDave Hansen /* 59bdd303cbSWei Yang * This work is run asynchronously to the task which owns 601e987790SDave Hansen * mm and might be done in another context, so we must 618b7457efSLorenzo Stoakes * access remotely. 621e987790SDave Hansen */ 638b7457efSLorenzo Stoakes down_read(&mm->mmap_sem); 648b7457efSLorenzo Stoakes get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL, 658b7457efSLorenzo Stoakes &locked); 668b7457efSLorenzo Stoakes if (locked) 678b7457efSLorenzo Stoakes up_read(&mm->mmap_sem); 681e987790SDave Hansen 694425f567SPaolo Bonzini if (IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC)) 704425f567SPaolo Bonzini kvm_arch_async_page_present(vcpu, apf); 71af585b92SGleb Natapov 72af585b92SGleb Natapov spin_lock(&vcpu->async_pf.lock); 73557a961aSVitaly Kuznetsov first = list_empty(&vcpu->async_pf.done); 74af585b92SGleb Natapov list_add_tail(&apf->link, &vcpu->async_pf.done); 7522583f0dSPaolo Bonzini apf->vcpu = NULL; 76af585b92SGleb Natapov spin_unlock(&vcpu->async_pf.lock); 77af585b92SGleb Natapov 78557a961aSVitaly Kuznetsov if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC) && first) 79557a961aSVitaly Kuznetsov kvm_arch_async_page_present_queued(vcpu); 80557a961aSVitaly Kuznetsov 81af585b92SGleb Natapov /* 82af585b92SGleb Natapov * apf may be freed by kvm_check_async_pf_completion() after 83af585b92SGleb Natapov * this point 84af585b92SGleb Natapov */ 85af585b92SGleb Natapov 86736c291cSSean Christopherson trace_kvm_async_pf_completed(addr, cr2_or_gpa); 87af585b92SGleb Natapov 88da4ad88cSDavidlohr Bueso rcuwait_wake_up(&vcpu->wait); 89af585b92SGleb Natapov 9041c22f62SOleg Nesterov mmput(mm); 91af585b92SGleb Natapov kvm_put_kvm(vcpu->kvm); 92af585b92SGleb Natapov } 93af585b92SGleb Natapov 94af585b92SGleb Natapov void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu) 95af585b92SGleb Natapov { 9622583f0dSPaolo Bonzini spin_lock(&vcpu->async_pf.lock); 9722583f0dSPaolo Bonzini 98af585b92SGleb Natapov /* cancel outstanding work queue item */ 99af585b92SGleb Natapov while (!list_empty(&vcpu->async_pf.queue)) { 100af585b92SGleb Natapov struct kvm_async_pf *work = 101433da860SGeliang Tang list_first_entry(&vcpu->async_pf.queue, 102af585b92SGleb Natapov typeof(*work), queue); 103af585b92SGleb Natapov list_del(&work->queue); 1049f2ceda4SDominik Dingel 10522583f0dSPaolo Bonzini /* 10622583f0dSPaolo Bonzini * We know it's present in vcpu->async_pf.done, do 10722583f0dSPaolo Bonzini * nothing here. 10822583f0dSPaolo Bonzini */ 10922583f0dSPaolo Bonzini if (!work->vcpu) 11022583f0dSPaolo Bonzini continue; 11122583f0dSPaolo Bonzini 11222583f0dSPaolo Bonzini spin_unlock(&vcpu->async_pf.lock); 1139f2ceda4SDominik Dingel #ifdef CONFIG_KVM_ASYNC_PF_SYNC 1149f2ceda4SDominik Dingel flush_work(&work->work); 1159f2ceda4SDominik Dingel #else 11698fda169SRadim Krčmář if (cancel_work_sync(&work->work)) { 11741c22f62SOleg Nesterov mmput(work->mm); 11828b441e2SRadim Krčmář kvm_put_kvm(vcpu->kvm); /* == work->vcpu->kvm */ 119af585b92SGleb Natapov kmem_cache_free(async_pf_cache, work); 120af585b92SGleb Natapov } 1219f2ceda4SDominik Dingel #endif 12222583f0dSPaolo Bonzini spin_lock(&vcpu->async_pf.lock); 12328b441e2SRadim Krčmář } 124af585b92SGleb Natapov 125af585b92SGleb Natapov while (!list_empty(&vcpu->async_pf.done)) { 126af585b92SGleb Natapov struct kvm_async_pf *work = 127433da860SGeliang Tang list_first_entry(&vcpu->async_pf.done, 128af585b92SGleb Natapov typeof(*work), link); 129af585b92SGleb Natapov list_del(&work->link); 130af585b92SGleb Natapov kmem_cache_free(async_pf_cache, work); 131af585b92SGleb Natapov } 132af585b92SGleb Natapov spin_unlock(&vcpu->async_pf.lock); 133af585b92SGleb Natapov 134af585b92SGleb Natapov vcpu->async_pf.queued = 0; 135af585b92SGleb Natapov } 136af585b92SGleb Natapov 137af585b92SGleb Natapov void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu) 138af585b92SGleb Natapov { 139af585b92SGleb Natapov struct kvm_async_pf *work; 140af585b92SGleb Natapov 14115096ffcSXiao Guangrong while (!list_empty_careful(&vcpu->async_pf.done) && 1427c0ade6cSVitaly Kuznetsov kvm_arch_can_dequeue_async_page_present(vcpu)) { 143af585b92SGleb Natapov spin_lock(&vcpu->async_pf.lock); 14415096ffcSXiao Guangrong work = list_first_entry(&vcpu->async_pf.done, typeof(*work), 14515096ffcSXiao Guangrong link); 146af585b92SGleb Natapov list_del(&work->link); 147af585b92SGleb Natapov spin_unlock(&vcpu->async_pf.lock); 148af585b92SGleb Natapov 14956028d08SGleb Natapov kvm_arch_async_page_ready(vcpu, work); 1504425f567SPaolo Bonzini if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC)) 1514425f567SPaolo Bonzini kvm_arch_async_page_present(vcpu, work); 152af585b92SGleb Natapov 153af585b92SGleb Natapov list_del(&work->queue); 154af585b92SGleb Natapov vcpu->async_pf.queued--; 155af585b92SGleb Natapov kmem_cache_free(async_pf_cache, work); 156af585b92SGleb Natapov } 15715096ffcSXiao Guangrong } 158af585b92SGleb Natapov 159736c291cSSean Christopherson int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 160736c291cSSean Christopherson unsigned long hva, struct kvm_arch_async_pf *arch) 161af585b92SGleb Natapov { 162af585b92SGleb Natapov struct kvm_async_pf *work; 163af585b92SGleb Natapov 164af585b92SGleb Natapov if (vcpu->async_pf.queued >= ASYNC_PF_PER_VCPU) 165af585b92SGleb Natapov return 0; 166af585b92SGleb Natapov 1677863e346SVitaly Kuznetsov /* Arch specific code should not do async PF in this case */ 1687863e346SVitaly Kuznetsov if (unlikely(kvm_is_error_hva(hva))) 1697863e346SVitaly Kuznetsov return 0; 170af585b92SGleb Natapov 171af585b92SGleb Natapov /* 172af585b92SGleb Natapov * do alloc nowait since if we are going to sleep anyway we 173af585b92SGleb Natapov * may as well sleep faulting in page 174af585b92SGleb Natapov */ 175d7444794SChristian Borntraeger work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT | __GFP_NOWARN); 176af585b92SGleb Natapov if (!work) 177af585b92SGleb Natapov return 0; 178af585b92SGleb Natapov 179f2e10669Schai wen work->wakeup_all = false; 180af585b92SGleb Natapov work->vcpu = vcpu; 181736c291cSSean Christopherson work->cr2_or_gpa = cr2_or_gpa; 182e0ead41aSDominik Dingel work->addr = hva; 183af585b92SGleb Natapov work->arch = *arch; 184af585b92SGleb Natapov work->mm = current->mm; 1853fce371bSVegard Nossum mmget(work->mm); 186af585b92SGleb Natapov kvm_get_kvm(work->vcpu->kvm); 187af585b92SGleb Natapov 188af585b92SGleb Natapov INIT_WORK(&work->work, async_pf_execute); 189af585b92SGleb Natapov 190af585b92SGleb Natapov list_add_tail(&work->queue, &vcpu->async_pf.queue); 191af585b92SGleb Natapov vcpu->async_pf.queued++; 192af585b92SGleb Natapov kvm_arch_async_page_not_present(vcpu, work); 1937863e346SVitaly Kuznetsov 1947863e346SVitaly Kuznetsov schedule_work(&work->work); 1957863e346SVitaly Kuznetsov 196af585b92SGleb Natapov return 1; 197af585b92SGleb Natapov } 198344d9588SGleb Natapov 199344d9588SGleb Natapov int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu) 200344d9588SGleb Natapov { 201344d9588SGleb Natapov struct kvm_async_pf *work; 202557a961aSVitaly Kuznetsov bool first; 203344d9588SGleb Natapov 20464f638c7SXiao Guangrong if (!list_empty_careful(&vcpu->async_pf.done)) 205344d9588SGleb Natapov return 0; 206344d9588SGleb Natapov 207344d9588SGleb Natapov work = kmem_cache_zalloc(async_pf_cache, GFP_ATOMIC); 208344d9588SGleb Natapov if (!work) 209344d9588SGleb Natapov return -ENOMEM; 210344d9588SGleb Natapov 211f2e10669Schai wen work->wakeup_all = true; 212344d9588SGleb Natapov INIT_LIST_HEAD(&work->queue); /* for list_del to work */ 213344d9588SGleb Natapov 21464f638c7SXiao Guangrong spin_lock(&vcpu->async_pf.lock); 215557a961aSVitaly Kuznetsov first = list_empty(&vcpu->async_pf.done); 216344d9588SGleb Natapov list_add_tail(&work->link, &vcpu->async_pf.done); 21764f638c7SXiao Guangrong spin_unlock(&vcpu->async_pf.lock); 21864f638c7SXiao Guangrong 219557a961aSVitaly Kuznetsov if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC) && first) 220557a961aSVitaly Kuznetsov kvm_arch_async_page_present_queued(vcpu); 221557a961aSVitaly Kuznetsov 222344d9588SGleb Natapov vcpu->async_pf.queued++; 223344d9588SGleb Natapov return 0; 224344d9588SGleb Natapov } 225