1fe5db27dSBen Gardon // SPDX-License-Identifier: GPL-2.0 2fe5db27dSBen Gardon 302c00b3aSBen Gardon #include "mmu.h" 402c00b3aSBen Gardon #include "mmu_internal.h" 5fe5db27dSBen Gardon #include "tdp_mmu.h" 602c00b3aSBen Gardon #include "spte.h" 7fe5db27dSBen Gardon 8fe5db27dSBen Gardon static bool __read_mostly tdp_mmu_enabled = false; 9fe5db27dSBen Gardon 10fe5db27dSBen Gardon static bool is_tdp_mmu_enabled(void) 11fe5db27dSBen Gardon { 12fe5db27dSBen Gardon #ifdef CONFIG_X86_64 13fe5db27dSBen Gardon return tdp_enabled && READ_ONCE(tdp_mmu_enabled); 14fe5db27dSBen Gardon #else 15fe5db27dSBen Gardon return false; 16fe5db27dSBen Gardon #endif /* CONFIG_X86_64 */ 17fe5db27dSBen Gardon } 18fe5db27dSBen Gardon 19fe5db27dSBen Gardon /* Initializes the TDP MMU for the VM, if enabled. */ 20fe5db27dSBen Gardon void kvm_mmu_init_tdp_mmu(struct kvm *kvm) 21fe5db27dSBen Gardon { 22fe5db27dSBen Gardon if (!is_tdp_mmu_enabled()) 23fe5db27dSBen Gardon return; 24fe5db27dSBen Gardon 25fe5db27dSBen Gardon /* This should not be changed for the lifetime of the VM. */ 26fe5db27dSBen Gardon kvm->arch.tdp_mmu_enabled = true; 2702c00b3aSBen Gardon 2802c00b3aSBen Gardon INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); 29fe5db27dSBen Gardon } 30fe5db27dSBen Gardon 31fe5db27dSBen Gardon void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) 32fe5db27dSBen Gardon { 33fe5db27dSBen Gardon if (!kvm->arch.tdp_mmu_enabled) 34fe5db27dSBen Gardon return; 3502c00b3aSBen Gardon 3602c00b3aSBen Gardon WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots)); 3702c00b3aSBen Gardon } 3802c00b3aSBen Gardon 3902c00b3aSBen Gardon #define for_each_tdp_mmu_root(_kvm, _root) \ 4002c00b3aSBen Gardon list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link) 4102c00b3aSBen Gardon 4202c00b3aSBen Gardon bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa) 4302c00b3aSBen Gardon { 4402c00b3aSBen Gardon struct kvm_mmu_page *sp; 4502c00b3aSBen Gardon 4602c00b3aSBen Gardon sp = to_shadow_page(hpa); 4702c00b3aSBen Gardon 4802c00b3aSBen Gardon return sp->tdp_mmu_page && sp->root_count; 4902c00b3aSBen Gardon } 5002c00b3aSBen Gardon 5102c00b3aSBen Gardon void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root) 5202c00b3aSBen Gardon { 5302c00b3aSBen Gardon lockdep_assert_held(&kvm->mmu_lock); 5402c00b3aSBen Gardon 5502c00b3aSBen Gardon WARN_ON(root->root_count); 5602c00b3aSBen Gardon WARN_ON(!root->tdp_mmu_page); 5702c00b3aSBen Gardon 5802c00b3aSBen Gardon list_del(&root->link); 5902c00b3aSBen Gardon 6002c00b3aSBen Gardon free_page((unsigned long)root->spt); 6102c00b3aSBen Gardon kmem_cache_free(mmu_page_header_cache, root); 6202c00b3aSBen Gardon } 6302c00b3aSBen Gardon 6402c00b3aSBen Gardon static union kvm_mmu_page_role page_role_for_level(struct kvm_vcpu *vcpu, 6502c00b3aSBen Gardon int level) 6602c00b3aSBen Gardon { 6702c00b3aSBen Gardon union kvm_mmu_page_role role; 6802c00b3aSBen Gardon 6902c00b3aSBen Gardon role = vcpu->arch.mmu->mmu_role.base; 7002c00b3aSBen Gardon role.level = level; 7102c00b3aSBen Gardon role.direct = true; 7202c00b3aSBen Gardon role.gpte_is_8_bytes = true; 7302c00b3aSBen Gardon role.access = ACC_ALL; 7402c00b3aSBen Gardon 7502c00b3aSBen Gardon return role; 7602c00b3aSBen Gardon } 7702c00b3aSBen Gardon 7802c00b3aSBen Gardon static struct kvm_mmu_page *alloc_tdp_mmu_page(struct kvm_vcpu *vcpu, gfn_t gfn, 7902c00b3aSBen Gardon int level) 8002c00b3aSBen Gardon { 8102c00b3aSBen Gardon struct kvm_mmu_page *sp; 8202c00b3aSBen Gardon 8302c00b3aSBen Gardon sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 8402c00b3aSBen Gardon sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 8502c00b3aSBen Gardon set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 8602c00b3aSBen Gardon 8702c00b3aSBen Gardon sp->role.word = page_role_for_level(vcpu, level).word; 8802c00b3aSBen Gardon sp->gfn = gfn; 8902c00b3aSBen Gardon sp->tdp_mmu_page = true; 9002c00b3aSBen Gardon 9102c00b3aSBen Gardon return sp; 9202c00b3aSBen Gardon } 9302c00b3aSBen Gardon 9402c00b3aSBen Gardon static struct kvm_mmu_page *get_tdp_mmu_vcpu_root(struct kvm_vcpu *vcpu) 9502c00b3aSBen Gardon { 9602c00b3aSBen Gardon union kvm_mmu_page_role role; 9702c00b3aSBen Gardon struct kvm *kvm = vcpu->kvm; 9802c00b3aSBen Gardon struct kvm_mmu_page *root; 9902c00b3aSBen Gardon 10002c00b3aSBen Gardon role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level); 10102c00b3aSBen Gardon 10202c00b3aSBen Gardon spin_lock(&kvm->mmu_lock); 10302c00b3aSBen Gardon 10402c00b3aSBen Gardon /* Check for an existing root before allocating a new one. */ 10502c00b3aSBen Gardon for_each_tdp_mmu_root(kvm, root) { 10602c00b3aSBen Gardon if (root->role.word == role.word) { 10702c00b3aSBen Gardon kvm_mmu_get_root(kvm, root); 10802c00b3aSBen Gardon spin_unlock(&kvm->mmu_lock); 10902c00b3aSBen Gardon return root; 11002c00b3aSBen Gardon } 11102c00b3aSBen Gardon } 11202c00b3aSBen Gardon 11302c00b3aSBen Gardon root = alloc_tdp_mmu_page(vcpu, 0, vcpu->arch.mmu->shadow_root_level); 11402c00b3aSBen Gardon root->root_count = 1; 11502c00b3aSBen Gardon 11602c00b3aSBen Gardon list_add(&root->link, &kvm->arch.tdp_mmu_roots); 11702c00b3aSBen Gardon 11802c00b3aSBen Gardon spin_unlock(&kvm->mmu_lock); 11902c00b3aSBen Gardon 12002c00b3aSBen Gardon return root; 12102c00b3aSBen Gardon } 12202c00b3aSBen Gardon 12302c00b3aSBen Gardon hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) 12402c00b3aSBen Gardon { 12502c00b3aSBen Gardon struct kvm_mmu_page *root; 12602c00b3aSBen Gardon 12702c00b3aSBen Gardon root = get_tdp_mmu_vcpu_root(vcpu); 12802c00b3aSBen Gardon if (!root) 12902c00b3aSBen Gardon return INVALID_PAGE; 13002c00b3aSBen Gardon 13102c00b3aSBen Gardon return __pa(root->spt); 132fe5db27dSBen Gardon } 133