1fe5db27dSBen Gardon // SPDX-License-Identifier: GPL-2.0 2fe5db27dSBen Gardon 302c00b3aSBen Gardon #include "mmu.h" 402c00b3aSBen Gardon #include "mmu_internal.h" 5bb18842eSBen Gardon #include "mmutrace.h" 62f2fad08SBen Gardon #include "tdp_iter.h" 7fe5db27dSBen Gardon #include "tdp_mmu.h" 802c00b3aSBen Gardon #include "spte.h" 9fe5db27dSBen Gardon 109a77daacSBen Gardon #include <asm/cmpxchg.h> 1133dd3574SBen Gardon #include <trace/events/kvm.h> 1233dd3574SBen Gardon 13fe5db27dSBen Gardon static bool __read_mostly tdp_mmu_enabled = false; 1495fb5b02SBen Gardon module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644); 15fe5db27dSBen Gardon 16fe5db27dSBen Gardon /* Initializes the TDP MMU for the VM, if enabled. */ 17fe5db27dSBen Gardon void kvm_mmu_init_tdp_mmu(struct kvm *kvm) 18fe5db27dSBen Gardon { 19897218ffSPaolo Bonzini if (!tdp_enabled || !READ_ONCE(tdp_mmu_enabled)) 20fe5db27dSBen Gardon return; 21fe5db27dSBen Gardon 22fe5db27dSBen Gardon /* This should not be changed for the lifetime of the VM. */ 23fe5db27dSBen Gardon kvm->arch.tdp_mmu_enabled = true; 2402c00b3aSBen Gardon 2502c00b3aSBen Gardon INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); 269a77daacSBen Gardon spin_lock_init(&kvm->arch.tdp_mmu_pages_lock); 2789c0fd49SBen Gardon INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages); 28fe5db27dSBen Gardon } 29fe5db27dSBen Gardon 30fe5db27dSBen Gardon void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) 31fe5db27dSBen Gardon { 32fe5db27dSBen Gardon if (!kvm->arch.tdp_mmu_enabled) 33fe5db27dSBen Gardon return; 3402c00b3aSBen Gardon 3502c00b3aSBen Gardon WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots)); 367cca2d0bSBen Gardon 377cca2d0bSBen Gardon /* 387cca2d0bSBen Gardon * Ensure that all the outstanding RCU callbacks to free shadow pages 397cca2d0bSBen Gardon * can run before the VM is torn down. 407cca2d0bSBen Gardon */ 417cca2d0bSBen Gardon rcu_barrier(); 4202c00b3aSBen Gardon } 4302c00b3aSBen Gardon 44a889ea54SBen Gardon static void tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) 45a889ea54SBen Gardon { 46a889ea54SBen Gardon if (kvm_mmu_put_root(kvm, root)) 47a889ea54SBen Gardon kvm_tdp_mmu_free_root(kvm, root); 48a889ea54SBen Gardon } 49a889ea54SBen Gardon 50a889ea54SBen Gardon static inline bool tdp_mmu_next_root_valid(struct kvm *kvm, 51a889ea54SBen Gardon struct kvm_mmu_page *root) 52a889ea54SBen Gardon { 53531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 54a889ea54SBen Gardon 55a889ea54SBen Gardon if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link)) 56a889ea54SBen Gardon return false; 57a889ea54SBen Gardon 58a889ea54SBen Gardon kvm_mmu_get_root(kvm, root); 59a889ea54SBen Gardon return true; 60a889ea54SBen Gardon 61a889ea54SBen Gardon } 62a889ea54SBen Gardon 63a889ea54SBen Gardon static inline struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, 64a889ea54SBen Gardon struct kvm_mmu_page *root) 65a889ea54SBen Gardon { 66a889ea54SBen Gardon struct kvm_mmu_page *next_root; 67a889ea54SBen Gardon 68a889ea54SBen Gardon next_root = list_next_entry(root, link); 69a889ea54SBen Gardon tdp_mmu_put_root(kvm, root); 70a889ea54SBen Gardon return next_root; 71a889ea54SBen Gardon } 72a889ea54SBen Gardon 73a889ea54SBen Gardon /* 74a889ea54SBen Gardon * Note: this iterator gets and puts references to the roots it iterates over. 75a889ea54SBen Gardon * This makes it safe to release the MMU lock and yield within the loop, but 76a889ea54SBen Gardon * if exiting the loop early, the caller must drop the reference to the most 77a889ea54SBen Gardon * recent root. (Unless keeping a live reference is desirable.) 78a889ea54SBen Gardon */ 79a889ea54SBen Gardon #define for_each_tdp_mmu_root_yield_safe(_kvm, _root) \ 80a889ea54SBen Gardon for (_root = list_first_entry(&_kvm->arch.tdp_mmu_roots, \ 81a889ea54SBen Gardon typeof(*_root), link); \ 82a889ea54SBen Gardon tdp_mmu_next_root_valid(_kvm, _root); \ 83a889ea54SBen Gardon _root = tdp_mmu_next_root(_kvm, _root)) 84a889ea54SBen Gardon 8502c00b3aSBen Gardon #define for_each_tdp_mmu_root(_kvm, _root) \ 8602c00b3aSBen Gardon list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link) 8702c00b3aSBen Gardon 88faaf05b0SBen Gardon static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 89063afacdSBen Gardon gfn_t start, gfn_t end, bool can_yield); 90faaf05b0SBen Gardon 9102c00b3aSBen Gardon void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root) 9202c00b3aSBen Gardon { 93339f5a7fSRick Edgecombe gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT); 94faaf05b0SBen Gardon 95531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 9602c00b3aSBen Gardon 9702c00b3aSBen Gardon WARN_ON(root->root_count); 9802c00b3aSBen Gardon WARN_ON(!root->tdp_mmu_page); 9902c00b3aSBen Gardon 10002c00b3aSBen Gardon list_del(&root->link); 10102c00b3aSBen Gardon 102063afacdSBen Gardon zap_gfn_range(kvm, root, 0, max_gfn, false); 103faaf05b0SBen Gardon 10402c00b3aSBen Gardon free_page((unsigned long)root->spt); 10502c00b3aSBen Gardon kmem_cache_free(mmu_page_header_cache, root); 10602c00b3aSBen Gardon } 10702c00b3aSBen Gardon 10802c00b3aSBen Gardon static union kvm_mmu_page_role page_role_for_level(struct kvm_vcpu *vcpu, 10902c00b3aSBen Gardon int level) 11002c00b3aSBen Gardon { 11102c00b3aSBen Gardon union kvm_mmu_page_role role; 11202c00b3aSBen Gardon 11302c00b3aSBen Gardon role = vcpu->arch.mmu->mmu_role.base; 11402c00b3aSBen Gardon role.level = level; 11502c00b3aSBen Gardon role.direct = true; 11602c00b3aSBen Gardon role.gpte_is_8_bytes = true; 11702c00b3aSBen Gardon role.access = ACC_ALL; 11802c00b3aSBen Gardon 11902c00b3aSBen Gardon return role; 12002c00b3aSBen Gardon } 12102c00b3aSBen Gardon 12202c00b3aSBen Gardon static struct kvm_mmu_page *alloc_tdp_mmu_page(struct kvm_vcpu *vcpu, gfn_t gfn, 12302c00b3aSBen Gardon int level) 12402c00b3aSBen Gardon { 12502c00b3aSBen Gardon struct kvm_mmu_page *sp; 12602c00b3aSBen Gardon 12702c00b3aSBen Gardon sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 12802c00b3aSBen Gardon sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 12902c00b3aSBen Gardon set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 13002c00b3aSBen Gardon 13102c00b3aSBen Gardon sp->role.word = page_role_for_level(vcpu, level).word; 13202c00b3aSBen Gardon sp->gfn = gfn; 13302c00b3aSBen Gardon sp->tdp_mmu_page = true; 13402c00b3aSBen Gardon 13533dd3574SBen Gardon trace_kvm_mmu_get_page(sp, true); 13633dd3574SBen Gardon 13702c00b3aSBen Gardon return sp; 13802c00b3aSBen Gardon } 13902c00b3aSBen Gardon 14002c00b3aSBen Gardon static struct kvm_mmu_page *get_tdp_mmu_vcpu_root(struct kvm_vcpu *vcpu) 14102c00b3aSBen Gardon { 14202c00b3aSBen Gardon union kvm_mmu_page_role role; 14302c00b3aSBen Gardon struct kvm *kvm = vcpu->kvm; 14402c00b3aSBen Gardon struct kvm_mmu_page *root; 14502c00b3aSBen Gardon 14602c00b3aSBen Gardon role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level); 14702c00b3aSBen Gardon 148531810caSBen Gardon write_lock(&kvm->mmu_lock); 14902c00b3aSBen Gardon 15002c00b3aSBen Gardon /* Check for an existing root before allocating a new one. */ 15102c00b3aSBen Gardon for_each_tdp_mmu_root(kvm, root) { 15202c00b3aSBen Gardon if (root->role.word == role.word) { 15302c00b3aSBen Gardon kvm_mmu_get_root(kvm, root); 154531810caSBen Gardon write_unlock(&kvm->mmu_lock); 15502c00b3aSBen Gardon return root; 15602c00b3aSBen Gardon } 15702c00b3aSBen Gardon } 15802c00b3aSBen Gardon 15902c00b3aSBen Gardon root = alloc_tdp_mmu_page(vcpu, 0, vcpu->arch.mmu->shadow_root_level); 16002c00b3aSBen Gardon root->root_count = 1; 16102c00b3aSBen Gardon 16202c00b3aSBen Gardon list_add(&root->link, &kvm->arch.tdp_mmu_roots); 16302c00b3aSBen Gardon 164531810caSBen Gardon write_unlock(&kvm->mmu_lock); 16502c00b3aSBen Gardon 16602c00b3aSBen Gardon return root; 16702c00b3aSBen Gardon } 16802c00b3aSBen Gardon 16902c00b3aSBen Gardon hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) 17002c00b3aSBen Gardon { 17102c00b3aSBen Gardon struct kvm_mmu_page *root; 17202c00b3aSBen Gardon 17302c00b3aSBen Gardon root = get_tdp_mmu_vcpu_root(vcpu); 17402c00b3aSBen Gardon if (!root) 17502c00b3aSBen Gardon return INVALID_PAGE; 17602c00b3aSBen Gardon 17702c00b3aSBen Gardon return __pa(root->spt); 178fe5db27dSBen Gardon } 1792f2fad08SBen Gardon 1807cca2d0bSBen Gardon static void tdp_mmu_free_sp(struct kvm_mmu_page *sp) 1817cca2d0bSBen Gardon { 1827cca2d0bSBen Gardon free_page((unsigned long)sp->spt); 1837cca2d0bSBen Gardon kmem_cache_free(mmu_page_header_cache, sp); 1847cca2d0bSBen Gardon } 1857cca2d0bSBen Gardon 1867cca2d0bSBen Gardon /* 1877cca2d0bSBen Gardon * This is called through call_rcu in order to free TDP page table memory 1887cca2d0bSBen Gardon * safely with respect to other kernel threads that may be operating on 1897cca2d0bSBen Gardon * the memory. 1907cca2d0bSBen Gardon * By only accessing TDP MMU page table memory in an RCU read critical 1917cca2d0bSBen Gardon * section, and freeing it after a grace period, lockless access to that 1927cca2d0bSBen Gardon * memory won't use it after it is freed. 1937cca2d0bSBen Gardon */ 1947cca2d0bSBen Gardon static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) 1957cca2d0bSBen Gardon { 1967cca2d0bSBen Gardon struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page, 1977cca2d0bSBen Gardon rcu_head); 1987cca2d0bSBen Gardon 1997cca2d0bSBen Gardon tdp_mmu_free_sp(sp); 2007cca2d0bSBen Gardon } 2017cca2d0bSBen Gardon 2022f2fad08SBen Gardon static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 2039a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 2049a77daacSBen Gardon bool shared); 2052f2fad08SBen Gardon 206faaf05b0SBen Gardon static int kvm_mmu_page_as_id(struct kvm_mmu_page *sp) 207faaf05b0SBen Gardon { 208faaf05b0SBen Gardon return sp->role.smm ? 1 : 0; 209faaf05b0SBen Gardon } 210faaf05b0SBen Gardon 211f8e14497SBen Gardon static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level) 212f8e14497SBen Gardon { 213f8e14497SBen Gardon bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 214f8e14497SBen Gardon 215f8e14497SBen Gardon if (!is_shadow_present_pte(old_spte) || !is_last_spte(old_spte, level)) 216f8e14497SBen Gardon return; 217f8e14497SBen Gardon 218f8e14497SBen Gardon if (is_accessed_spte(old_spte) && 219f8e14497SBen Gardon (!is_accessed_spte(new_spte) || pfn_changed)) 220f8e14497SBen Gardon kvm_set_pfn_accessed(spte_to_pfn(old_spte)); 221f8e14497SBen Gardon } 222f8e14497SBen Gardon 223a6a0b05dSBen Gardon static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, 224a6a0b05dSBen Gardon u64 old_spte, u64 new_spte, int level) 225a6a0b05dSBen Gardon { 226a6a0b05dSBen Gardon bool pfn_changed; 227a6a0b05dSBen Gardon struct kvm_memory_slot *slot; 228a6a0b05dSBen Gardon 229a6a0b05dSBen Gardon if (level > PG_LEVEL_4K) 230a6a0b05dSBen Gardon return; 231a6a0b05dSBen Gardon 232a6a0b05dSBen Gardon pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 233a6a0b05dSBen Gardon 234a6a0b05dSBen Gardon if ((!is_writable_pte(old_spte) || pfn_changed) && 235a6a0b05dSBen Gardon is_writable_pte(new_spte)) { 236a6a0b05dSBen Gardon slot = __gfn_to_memslot(__kvm_memslots(kvm, as_id), gfn); 237fb04a1edSPeter Xu mark_page_dirty_in_slot(kvm, slot, gfn); 238a6a0b05dSBen Gardon } 239a6a0b05dSBen Gardon } 240a6a0b05dSBen Gardon 2412f2fad08SBen Gardon /** 242a9442f59SBen Gardon * tdp_mmu_link_page - Add a new page to the list of pages used by the TDP MMU 243a9442f59SBen Gardon * 244a9442f59SBen Gardon * @kvm: kvm instance 245a9442f59SBen Gardon * @sp: the new page 2469a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 2479a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 2489a77daacSBen Gardon * threads that might be adding or removing pages. 249a9442f59SBen Gardon * @account_nx: This page replaces a NX large page and should be marked for 250a9442f59SBen Gardon * eventual reclaim. 251a9442f59SBen Gardon */ 252a9442f59SBen Gardon static void tdp_mmu_link_page(struct kvm *kvm, struct kvm_mmu_page *sp, 2539a77daacSBen Gardon bool shared, bool account_nx) 254a9442f59SBen Gardon { 2559a77daacSBen Gardon if (shared) 2569a77daacSBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 2579a77daacSBen Gardon else 258a9442f59SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 259a9442f59SBen Gardon 260a9442f59SBen Gardon list_add(&sp->link, &kvm->arch.tdp_mmu_pages); 261a9442f59SBen Gardon if (account_nx) 262a9442f59SBen Gardon account_huge_nx_page(kvm, sp); 2639a77daacSBen Gardon 2649a77daacSBen Gardon if (shared) 2659a77daacSBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 266a9442f59SBen Gardon } 267a9442f59SBen Gardon 268a9442f59SBen Gardon /** 269a9442f59SBen Gardon * tdp_mmu_unlink_page - Remove page from the list of pages used by the TDP MMU 270a9442f59SBen Gardon * 271a9442f59SBen Gardon * @kvm: kvm instance 272a9442f59SBen Gardon * @sp: the page to be removed 2739a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 2749a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 2759a77daacSBen Gardon * threads that might be adding or removing pages. 276a9442f59SBen Gardon */ 2779a77daacSBen Gardon static void tdp_mmu_unlink_page(struct kvm *kvm, struct kvm_mmu_page *sp, 2789a77daacSBen Gardon bool shared) 279a9442f59SBen Gardon { 2809a77daacSBen Gardon if (shared) 2819a77daacSBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 2829a77daacSBen Gardon else 283a9442f59SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 284a9442f59SBen Gardon 285a9442f59SBen Gardon list_del(&sp->link); 286a9442f59SBen Gardon if (sp->lpage_disallowed) 287a9442f59SBen Gardon unaccount_huge_nx_page(kvm, sp); 2889a77daacSBen Gardon 2899a77daacSBen Gardon if (shared) 2909a77daacSBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 291a9442f59SBen Gardon } 292a9442f59SBen Gardon 293a9442f59SBen Gardon /** 294a066e61fSBen Gardon * handle_removed_tdp_mmu_page - handle a pt removed from the TDP structure 295a066e61fSBen Gardon * 296a066e61fSBen Gardon * @kvm: kvm instance 297a066e61fSBen Gardon * @pt: the page removed from the paging structure 2989a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use 2999a77daacSBen Gardon * of the MMU lock and the operation must synchronize with other 3009a77daacSBen Gardon * threads that might be modifying SPTEs. 301a066e61fSBen Gardon * 302a066e61fSBen Gardon * Given a page table that has been removed from the TDP paging structure, 303a066e61fSBen Gardon * iterates through the page table to clear SPTEs and free child page tables. 304a066e61fSBen Gardon */ 3059a77daacSBen Gardon static void handle_removed_tdp_mmu_page(struct kvm *kvm, u64 *pt, 3069a77daacSBen Gardon bool shared) 307a066e61fSBen Gardon { 308a066e61fSBen Gardon struct kvm_mmu_page *sp = sptep_to_sp(pt); 309a066e61fSBen Gardon int level = sp->role.level; 310e25f0e0cSBen Gardon gfn_t base_gfn = sp->gfn; 311a066e61fSBen Gardon u64 old_child_spte; 3129a77daacSBen Gardon u64 *sptep; 313e25f0e0cSBen Gardon gfn_t gfn; 314a066e61fSBen Gardon int i; 315a066e61fSBen Gardon 316a066e61fSBen Gardon trace_kvm_mmu_prepare_zap_page(sp); 317a066e61fSBen Gardon 3189a77daacSBen Gardon tdp_mmu_unlink_page(kvm, sp, shared); 319a066e61fSBen Gardon 320a066e61fSBen Gardon for (i = 0; i < PT64_ENT_PER_PAGE; i++) { 3219a77daacSBen Gardon sptep = pt + i; 322e25f0e0cSBen Gardon gfn = base_gfn + (i * KVM_PAGES_PER_HPAGE(level - 1)); 3239a77daacSBen Gardon 3249a77daacSBen Gardon if (shared) { 325e25f0e0cSBen Gardon /* 326e25f0e0cSBen Gardon * Set the SPTE to a nonpresent value that other 327e25f0e0cSBen Gardon * threads will not overwrite. If the SPTE was 328e25f0e0cSBen Gardon * already marked as removed then another thread 329e25f0e0cSBen Gardon * handling a page fault could overwrite it, so 330e25f0e0cSBen Gardon * set the SPTE until it is set from some other 331e25f0e0cSBen Gardon * value to the removed SPTE value. 332e25f0e0cSBen Gardon */ 333e25f0e0cSBen Gardon for (;;) { 334e25f0e0cSBen Gardon old_child_spte = xchg(sptep, REMOVED_SPTE); 335e25f0e0cSBen Gardon if (!is_removed_spte(old_child_spte)) 336e25f0e0cSBen Gardon break; 337e25f0e0cSBen Gardon cpu_relax(); 338e25f0e0cSBen Gardon } 3399a77daacSBen Gardon } else { 340*8df9f1afSSean Christopherson /* 341*8df9f1afSSean Christopherson * If the SPTE is not MMU-present, there is no backing 342*8df9f1afSSean Christopherson * page associated with the SPTE and so no side effects 343*8df9f1afSSean Christopherson * that need to be recorded, and exclusive ownership of 344*8df9f1afSSean Christopherson * mmu_lock ensures the SPTE can't be made present. 345*8df9f1afSSean Christopherson * Note, zapping MMIO SPTEs is also unnecessary as they 346*8df9f1afSSean Christopherson * are guarded by the memslots generation, not by being 347*8df9f1afSSean Christopherson * unreachable. 348*8df9f1afSSean Christopherson */ 3499a77daacSBen Gardon old_child_spte = READ_ONCE(*sptep); 350*8df9f1afSSean Christopherson if (!is_shadow_present_pte(old_child_spte)) 351*8df9f1afSSean Christopherson continue; 352e25f0e0cSBen Gardon 353e25f0e0cSBen Gardon /* 354e25f0e0cSBen Gardon * Marking the SPTE as a removed SPTE is not 355e25f0e0cSBen Gardon * strictly necessary here as the MMU lock will 356e25f0e0cSBen Gardon * stop other threads from concurrently modifying 357e25f0e0cSBen Gardon * this SPTE. Using the removed SPTE value keeps 358e25f0e0cSBen Gardon * the two branches consistent and simplifies 359e25f0e0cSBen Gardon * the function. 360e25f0e0cSBen Gardon */ 361e25f0e0cSBen Gardon WRITE_ONCE(*sptep, REMOVED_SPTE); 3629a77daacSBen Gardon } 363e25f0e0cSBen Gardon handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn, 364e25f0e0cSBen Gardon old_child_spte, REMOVED_SPTE, level - 1, 365e25f0e0cSBen Gardon shared); 366a066e61fSBen Gardon } 367a066e61fSBen Gardon 368a066e61fSBen Gardon kvm_flush_remote_tlbs_with_address(kvm, gfn, 369a066e61fSBen Gardon KVM_PAGES_PER_HPAGE(level)); 370a066e61fSBen Gardon 3717cca2d0bSBen Gardon call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); 372a066e61fSBen Gardon } 373a066e61fSBen Gardon 374a066e61fSBen Gardon /** 3752f2fad08SBen Gardon * handle_changed_spte - handle bookkeeping associated with an SPTE change 3762f2fad08SBen Gardon * @kvm: kvm instance 3772f2fad08SBen Gardon * @as_id: the address space of the paging structure the SPTE was a part of 3782f2fad08SBen Gardon * @gfn: the base GFN that was mapped by the SPTE 3792f2fad08SBen Gardon * @old_spte: The value of the SPTE before the change 3802f2fad08SBen Gardon * @new_spte: The value of the SPTE after the change 3812f2fad08SBen Gardon * @level: the level of the PT the SPTE is part of in the paging structure 3829a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 3839a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 3849a77daacSBen Gardon * threads that might be modifying SPTEs. 3852f2fad08SBen Gardon * 3862f2fad08SBen Gardon * Handle bookkeeping that might result from the modification of a SPTE. 3872f2fad08SBen Gardon * This function must be called for all TDP SPTE modifications. 3882f2fad08SBen Gardon */ 3892f2fad08SBen Gardon static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 3909a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 3919a77daacSBen Gardon bool shared) 3922f2fad08SBen Gardon { 3932f2fad08SBen Gardon bool was_present = is_shadow_present_pte(old_spte); 3942f2fad08SBen Gardon bool is_present = is_shadow_present_pte(new_spte); 3952f2fad08SBen Gardon bool was_leaf = was_present && is_last_spte(old_spte, level); 3962f2fad08SBen Gardon bool is_leaf = is_present && is_last_spte(new_spte, level); 3972f2fad08SBen Gardon bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 3982f2fad08SBen Gardon 3992f2fad08SBen Gardon WARN_ON(level > PT64_ROOT_MAX_LEVEL); 4002f2fad08SBen Gardon WARN_ON(level < PG_LEVEL_4K); 401764388ceSSean Christopherson WARN_ON(gfn & (KVM_PAGES_PER_HPAGE(level) - 1)); 4022f2fad08SBen Gardon 4032f2fad08SBen Gardon /* 4042f2fad08SBen Gardon * If this warning were to trigger it would indicate that there was a 4052f2fad08SBen Gardon * missing MMU notifier or a race with some notifier handler. 4062f2fad08SBen Gardon * A present, leaf SPTE should never be directly replaced with another 4072f2fad08SBen Gardon * present leaf SPTE pointing to a differnt PFN. A notifier handler 4082f2fad08SBen Gardon * should be zapping the SPTE before the main MM's page table is 4092f2fad08SBen Gardon * changed, or the SPTE should be zeroed, and the TLBs flushed by the 4102f2fad08SBen Gardon * thread before replacement. 4112f2fad08SBen Gardon */ 4122f2fad08SBen Gardon if (was_leaf && is_leaf && pfn_changed) { 4132f2fad08SBen Gardon pr_err("Invalid SPTE change: cannot replace a present leaf\n" 4142f2fad08SBen Gardon "SPTE with another present leaf SPTE mapping a\n" 4152f2fad08SBen Gardon "different PFN!\n" 4162f2fad08SBen Gardon "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 4172f2fad08SBen Gardon as_id, gfn, old_spte, new_spte, level); 4182f2fad08SBen Gardon 4192f2fad08SBen Gardon /* 4202f2fad08SBen Gardon * Crash the host to prevent error propagation and guest data 4212f2fad08SBen Gardon * courruption. 4222f2fad08SBen Gardon */ 4232f2fad08SBen Gardon BUG(); 4242f2fad08SBen Gardon } 4252f2fad08SBen Gardon 4262f2fad08SBen Gardon if (old_spte == new_spte) 4272f2fad08SBen Gardon return; 4282f2fad08SBen Gardon 429b9a98c34SBen Gardon trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); 430b9a98c34SBen Gardon 4312f2fad08SBen Gardon /* 4322f2fad08SBen Gardon * The only times a SPTE should be changed from a non-present to 4332f2fad08SBen Gardon * non-present state is when an MMIO entry is installed/modified/ 4342f2fad08SBen Gardon * removed. In that case, there is nothing to do here. 4352f2fad08SBen Gardon */ 4362f2fad08SBen Gardon if (!was_present && !is_present) { 4372f2fad08SBen Gardon /* 43808f07c80SBen Gardon * If this change does not involve a MMIO SPTE or removed SPTE, 43908f07c80SBen Gardon * it is unexpected. Log the change, though it should not 44008f07c80SBen Gardon * impact the guest since both the former and current SPTEs 44108f07c80SBen Gardon * are nonpresent. 4422f2fad08SBen Gardon */ 44308f07c80SBen Gardon if (WARN_ON(!is_mmio_spte(old_spte) && 44408f07c80SBen Gardon !is_mmio_spte(new_spte) && 44508f07c80SBen Gardon !is_removed_spte(new_spte))) 4462f2fad08SBen Gardon pr_err("Unexpected SPTE change! Nonpresent SPTEs\n" 4472f2fad08SBen Gardon "should not be replaced with another,\n" 4482f2fad08SBen Gardon "different nonpresent SPTE, unless one or both\n" 44908f07c80SBen Gardon "are MMIO SPTEs, or the new SPTE is\n" 45008f07c80SBen Gardon "a temporary removed SPTE.\n" 4512f2fad08SBen Gardon "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 4522f2fad08SBen Gardon as_id, gfn, old_spte, new_spte, level); 4532f2fad08SBen Gardon return; 4542f2fad08SBen Gardon } 4552f2fad08SBen Gardon 4562f2fad08SBen Gardon 4572f2fad08SBen Gardon if (was_leaf && is_dirty_spte(old_spte) && 4582f2fad08SBen Gardon (!is_dirty_spte(new_spte) || pfn_changed)) 4592f2fad08SBen Gardon kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 4602f2fad08SBen Gardon 4612f2fad08SBen Gardon /* 4622f2fad08SBen Gardon * Recursively handle child PTs if the change removed a subtree from 4632f2fad08SBen Gardon * the paging structure. 4642f2fad08SBen Gardon */ 465a066e61fSBen Gardon if (was_present && !was_leaf && (pfn_changed || !is_present)) 466a066e61fSBen Gardon handle_removed_tdp_mmu_page(kvm, 4679a77daacSBen Gardon spte_to_child_pt(old_spte, level), shared); 4682f2fad08SBen Gardon } 4692f2fad08SBen Gardon 4702f2fad08SBen Gardon static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 4719a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 4729a77daacSBen Gardon bool shared) 4732f2fad08SBen Gardon { 4749a77daacSBen Gardon __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, 4759a77daacSBen Gardon shared); 476f8e14497SBen Gardon handle_changed_spte_acc_track(old_spte, new_spte, level); 477a6a0b05dSBen Gardon handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte, 478a6a0b05dSBen Gardon new_spte, level); 4792f2fad08SBen Gardon } 480faaf05b0SBen Gardon 481fe43fa2fSBen Gardon /* 4829a77daacSBen Gardon * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically and handle the 4839a77daacSBen Gardon * associated bookkeeping 4849a77daacSBen Gardon * 4859a77daacSBen Gardon * @kvm: kvm instance 4869a77daacSBen Gardon * @iter: a tdp_iter instance currently on the SPTE that should be set 4879a77daacSBen Gardon * @new_spte: The value the SPTE should be set to 4889a77daacSBen Gardon * Returns: true if the SPTE was set, false if it was not. If false is returned, 4899a77daacSBen Gardon * this function will have no side-effects. 4909a77daacSBen Gardon */ 4919a77daacSBen Gardon static inline bool tdp_mmu_set_spte_atomic(struct kvm *kvm, 4929a77daacSBen Gardon struct tdp_iter *iter, 4939a77daacSBen Gardon u64 new_spte) 4949a77daacSBen Gardon { 4959a77daacSBen Gardon u64 *root_pt = tdp_iter_root_pt(iter); 4969a77daacSBen Gardon struct kvm_mmu_page *root = sptep_to_sp(root_pt); 4979a77daacSBen Gardon int as_id = kvm_mmu_page_as_id(root); 4989a77daacSBen Gardon 4999a77daacSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 5009a77daacSBen Gardon 50108f07c80SBen Gardon /* 50208f07c80SBen Gardon * Do not change removed SPTEs. Only the thread that froze the SPTE 50308f07c80SBen Gardon * may modify it. 50408f07c80SBen Gardon */ 50508f07c80SBen Gardon if (iter->old_spte == REMOVED_SPTE) 50608f07c80SBen Gardon return false; 50708f07c80SBen Gardon 5089a77daacSBen Gardon if (cmpxchg64(rcu_dereference(iter->sptep), iter->old_spte, 5099a77daacSBen Gardon new_spte) != iter->old_spte) 5109a77daacSBen Gardon return false; 5119a77daacSBen Gardon 5129a77daacSBen Gardon handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte, 5139a77daacSBen Gardon iter->level, true); 5149a77daacSBen Gardon 5159a77daacSBen Gardon return true; 5169a77daacSBen Gardon } 5179a77daacSBen Gardon 51808f07c80SBen Gardon static inline bool tdp_mmu_zap_spte_atomic(struct kvm *kvm, 51908f07c80SBen Gardon struct tdp_iter *iter) 52008f07c80SBen Gardon { 52108f07c80SBen Gardon /* 52208f07c80SBen Gardon * Freeze the SPTE by setting it to a special, 52308f07c80SBen Gardon * non-present value. This will stop other threads from 52408f07c80SBen Gardon * immediately installing a present entry in its place 52508f07c80SBen Gardon * before the TLBs are flushed. 52608f07c80SBen Gardon */ 52708f07c80SBen Gardon if (!tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE)) 52808f07c80SBen Gardon return false; 52908f07c80SBen Gardon 53008f07c80SBen Gardon kvm_flush_remote_tlbs_with_address(kvm, iter->gfn, 53108f07c80SBen Gardon KVM_PAGES_PER_HPAGE(iter->level)); 53208f07c80SBen Gardon 53308f07c80SBen Gardon /* 53408f07c80SBen Gardon * No other thread can overwrite the removed SPTE as they 53508f07c80SBen Gardon * must either wait on the MMU lock or use 53608f07c80SBen Gardon * tdp_mmu_set_spte_atomic which will not overrite the 53708f07c80SBen Gardon * special removed SPTE value. No bookkeeping is needed 53808f07c80SBen Gardon * here since the SPTE is going from non-present 53908f07c80SBen Gardon * to non-present. 54008f07c80SBen Gardon */ 54108f07c80SBen Gardon WRITE_ONCE(*iter->sptep, 0); 54208f07c80SBen Gardon 54308f07c80SBen Gardon return true; 54408f07c80SBen Gardon } 54508f07c80SBen Gardon 5469a77daacSBen Gardon 5479a77daacSBen Gardon /* 548fe43fa2fSBen Gardon * __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping 549fe43fa2fSBen Gardon * @kvm: kvm instance 550fe43fa2fSBen Gardon * @iter: a tdp_iter instance currently on the SPTE that should be set 551fe43fa2fSBen Gardon * @new_spte: The value the SPTE should be set to 552fe43fa2fSBen Gardon * @record_acc_track: Notify the MM subsystem of changes to the accessed state 553fe43fa2fSBen Gardon * of the page. Should be set unless handling an MMU 554fe43fa2fSBen Gardon * notifier for access tracking. Leaving record_acc_track 555fe43fa2fSBen Gardon * unset in that case prevents page accesses from being 556fe43fa2fSBen Gardon * double counted. 557fe43fa2fSBen Gardon * @record_dirty_log: Record the page as dirty in the dirty bitmap if 558fe43fa2fSBen Gardon * appropriate for the change being made. Should be set 559fe43fa2fSBen Gardon * unless performing certain dirty logging operations. 560fe43fa2fSBen Gardon * Leaving record_dirty_log unset in that case prevents page 561fe43fa2fSBen Gardon * writes from being double counted. 562fe43fa2fSBen Gardon */ 563f8e14497SBen Gardon static inline void __tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, 564a6a0b05dSBen Gardon u64 new_spte, bool record_acc_track, 565a6a0b05dSBen Gardon bool record_dirty_log) 566faaf05b0SBen Gardon { 5677cca2d0bSBen Gardon tdp_ptep_t root_pt = tdp_iter_root_pt(iter); 568faaf05b0SBen Gardon struct kvm_mmu_page *root = sptep_to_sp(root_pt); 569faaf05b0SBen Gardon int as_id = kvm_mmu_page_as_id(root); 570faaf05b0SBen Gardon 571531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 5723a9a4aa5SBen Gardon 57308f07c80SBen Gardon /* 57408f07c80SBen Gardon * No thread should be using this function to set SPTEs to the 57508f07c80SBen Gardon * temporary removed SPTE value. 57608f07c80SBen Gardon * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic 57708f07c80SBen Gardon * should be used. If operating under the MMU lock in write mode, the 57808f07c80SBen Gardon * use of the removed SPTE should not be necessary. 57908f07c80SBen Gardon */ 58008f07c80SBen Gardon WARN_ON(iter->old_spte == REMOVED_SPTE); 58108f07c80SBen Gardon 5827cca2d0bSBen Gardon WRITE_ONCE(*rcu_dereference(iter->sptep), new_spte); 583faaf05b0SBen Gardon 584f8e14497SBen Gardon __handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte, 5859a77daacSBen Gardon iter->level, false); 586f8e14497SBen Gardon if (record_acc_track) 587f8e14497SBen Gardon handle_changed_spte_acc_track(iter->old_spte, new_spte, 588f8e14497SBen Gardon iter->level); 589a6a0b05dSBen Gardon if (record_dirty_log) 590a6a0b05dSBen Gardon handle_changed_spte_dirty_log(kvm, as_id, iter->gfn, 591a6a0b05dSBen Gardon iter->old_spte, new_spte, 592a6a0b05dSBen Gardon iter->level); 593f8e14497SBen Gardon } 594f8e14497SBen Gardon 595f8e14497SBen Gardon static inline void tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, 596f8e14497SBen Gardon u64 new_spte) 597f8e14497SBen Gardon { 598a6a0b05dSBen Gardon __tdp_mmu_set_spte(kvm, iter, new_spte, true, true); 599f8e14497SBen Gardon } 600f8e14497SBen Gardon 601f8e14497SBen Gardon static inline void tdp_mmu_set_spte_no_acc_track(struct kvm *kvm, 602f8e14497SBen Gardon struct tdp_iter *iter, 603f8e14497SBen Gardon u64 new_spte) 604f8e14497SBen Gardon { 605a6a0b05dSBen Gardon __tdp_mmu_set_spte(kvm, iter, new_spte, false, true); 606a6a0b05dSBen Gardon } 607a6a0b05dSBen Gardon 608a6a0b05dSBen Gardon static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm, 609a6a0b05dSBen Gardon struct tdp_iter *iter, 610a6a0b05dSBen Gardon u64 new_spte) 611a6a0b05dSBen Gardon { 612a6a0b05dSBen Gardon __tdp_mmu_set_spte(kvm, iter, new_spte, true, false); 613faaf05b0SBen Gardon } 614faaf05b0SBen Gardon 615faaf05b0SBen Gardon #define tdp_root_for_each_pte(_iter, _root, _start, _end) \ 616faaf05b0SBen Gardon for_each_tdp_pte(_iter, _root->spt, _root->role.level, _start, _end) 617faaf05b0SBen Gardon 618f8e14497SBen Gardon #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \ 619f8e14497SBen Gardon tdp_root_for_each_pte(_iter, _root, _start, _end) \ 620f8e14497SBen Gardon if (!is_shadow_present_pte(_iter.old_spte) || \ 621f8e14497SBen Gardon !is_last_spte(_iter.old_spte, _iter.level)) \ 622f8e14497SBen Gardon continue; \ 623f8e14497SBen Gardon else 624f8e14497SBen Gardon 625bb18842eSBen Gardon #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \ 626bb18842eSBen Gardon for_each_tdp_pte(_iter, __va(_mmu->root_hpa), \ 627bb18842eSBen Gardon _mmu->shadow_root_level, _start, _end) 628bb18842eSBen Gardon 629faaf05b0SBen Gardon /* 630e28a436cSBen Gardon * Yield if the MMU lock is contended or this thread needs to return control 631e28a436cSBen Gardon * to the scheduler. 632e28a436cSBen Gardon * 633e139a34eSBen Gardon * If this function should yield and flush is set, it will perform a remote 634e139a34eSBen Gardon * TLB flush before yielding. 635e139a34eSBen Gardon * 636e28a436cSBen Gardon * If this function yields, it will also reset the tdp_iter's walk over the 637ed5e484bSBen Gardon * paging structure and the calling function should skip to the next 638ed5e484bSBen Gardon * iteration to allow the iterator to continue its traversal from the 639ed5e484bSBen Gardon * paging structure root. 640e28a436cSBen Gardon * 641e28a436cSBen Gardon * Return true if this function yielded and the iterator's traversal was reset. 642e28a436cSBen Gardon * Return false if a yield was not needed. 643e28a436cSBen Gardon */ 644e139a34eSBen Gardon static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm, 645e139a34eSBen Gardon struct tdp_iter *iter, bool flush) 646a6a0b05dSBen Gardon { 647ed5e484bSBen Gardon /* Ensure forward progress has been made before yielding. */ 648ed5e484bSBen Gardon if (iter->next_last_level_gfn == iter->yielded_gfn) 649ed5e484bSBen Gardon return false; 650ed5e484bSBen Gardon 651531810caSBen Gardon if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { 6527cca2d0bSBen Gardon rcu_read_unlock(); 6537cca2d0bSBen Gardon 654e139a34eSBen Gardon if (flush) 655e139a34eSBen Gardon kvm_flush_remote_tlbs(kvm); 656e139a34eSBen Gardon 657531810caSBen Gardon cond_resched_rwlock_write(&kvm->mmu_lock); 6587cca2d0bSBen Gardon rcu_read_lock(); 659ed5e484bSBen Gardon 660ed5e484bSBen Gardon WARN_ON(iter->gfn > iter->next_last_level_gfn); 661ed5e484bSBen Gardon 662ed5e484bSBen Gardon tdp_iter_start(iter, iter->pt_path[iter->root_level - 1], 663ed5e484bSBen Gardon iter->root_level, iter->min_level, 664ed5e484bSBen Gardon iter->next_last_level_gfn); 665ed5e484bSBen Gardon 666e28a436cSBen Gardon return true; 667a6a0b05dSBen Gardon } 668e28a436cSBen Gardon 669e28a436cSBen Gardon return false; 670a6a0b05dSBen Gardon } 671a6a0b05dSBen Gardon 672faaf05b0SBen Gardon /* 673faaf05b0SBen Gardon * Tears down the mappings for the range of gfns, [start, end), and frees the 674faaf05b0SBen Gardon * non-root pages mapping GFNs strictly within that range. Returns true if 675faaf05b0SBen Gardon * SPTEs have been cleared and a TLB flush is needed before releasing the 676faaf05b0SBen Gardon * MMU lock. 677063afacdSBen Gardon * If can_yield is true, will release the MMU lock and reschedule if the 678063afacdSBen Gardon * scheduler needs the CPU or there is contention on the MMU lock. If this 679063afacdSBen Gardon * function cannot yield, it will not release the MMU lock or reschedule and 680063afacdSBen Gardon * the caller must ensure it does not supply too large a GFN range, or the 681063afacdSBen Gardon * operation can cause a soft lockup. 682faaf05b0SBen Gardon */ 683faaf05b0SBen Gardon static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 684063afacdSBen Gardon gfn_t start, gfn_t end, bool can_yield) 685faaf05b0SBen Gardon { 686faaf05b0SBen Gardon struct tdp_iter iter; 687faaf05b0SBen Gardon bool flush_needed = false; 688faaf05b0SBen Gardon 6897cca2d0bSBen Gardon rcu_read_lock(); 6907cca2d0bSBen Gardon 691faaf05b0SBen Gardon tdp_root_for_each_pte(iter, root, start, end) { 6921af4a960SBen Gardon if (can_yield && 6931af4a960SBen Gardon tdp_mmu_iter_cond_resched(kvm, &iter, flush_needed)) { 6941af4a960SBen Gardon flush_needed = false; 6951af4a960SBen Gardon continue; 6961af4a960SBen Gardon } 6971af4a960SBen Gardon 698faaf05b0SBen Gardon if (!is_shadow_present_pte(iter.old_spte)) 699faaf05b0SBen Gardon continue; 700faaf05b0SBen Gardon 701faaf05b0SBen Gardon /* 702faaf05b0SBen Gardon * If this is a non-last-level SPTE that covers a larger range 703faaf05b0SBen Gardon * than should be zapped, continue, and zap the mappings at a 704faaf05b0SBen Gardon * lower level. 705faaf05b0SBen Gardon */ 706faaf05b0SBen Gardon if ((iter.gfn < start || 707faaf05b0SBen Gardon iter.gfn + KVM_PAGES_PER_HPAGE(iter.level) > end) && 708faaf05b0SBen Gardon !is_last_spte(iter.old_spte, iter.level)) 709faaf05b0SBen Gardon continue; 710faaf05b0SBen Gardon 711faaf05b0SBen Gardon tdp_mmu_set_spte(kvm, &iter, 0); 7121af4a960SBen Gardon flush_needed = true; 713faaf05b0SBen Gardon } 7147cca2d0bSBen Gardon 7157cca2d0bSBen Gardon rcu_read_unlock(); 716faaf05b0SBen Gardon return flush_needed; 717faaf05b0SBen Gardon } 718faaf05b0SBen Gardon 719faaf05b0SBen Gardon /* 720faaf05b0SBen Gardon * Tears down the mappings for the range of gfns, [start, end), and frees the 721faaf05b0SBen Gardon * non-root pages mapping GFNs strictly within that range. Returns true if 722faaf05b0SBen Gardon * SPTEs have been cleared and a TLB flush is needed before releasing the 723faaf05b0SBen Gardon * MMU lock. 724faaf05b0SBen Gardon */ 725faaf05b0SBen Gardon bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end) 726faaf05b0SBen Gardon { 727faaf05b0SBen Gardon struct kvm_mmu_page *root; 728faaf05b0SBen Gardon bool flush = false; 729faaf05b0SBen Gardon 730a889ea54SBen Gardon for_each_tdp_mmu_root_yield_safe(kvm, root) 731063afacdSBen Gardon flush |= zap_gfn_range(kvm, root, start, end, true); 732faaf05b0SBen Gardon 733faaf05b0SBen Gardon return flush; 734faaf05b0SBen Gardon } 735faaf05b0SBen Gardon 736faaf05b0SBen Gardon void kvm_tdp_mmu_zap_all(struct kvm *kvm) 737faaf05b0SBen Gardon { 738339f5a7fSRick Edgecombe gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT); 739faaf05b0SBen Gardon bool flush; 740faaf05b0SBen Gardon 741faaf05b0SBen Gardon flush = kvm_tdp_mmu_zap_gfn_range(kvm, 0, max_gfn); 742faaf05b0SBen Gardon if (flush) 743faaf05b0SBen Gardon kvm_flush_remote_tlbs(kvm); 744faaf05b0SBen Gardon } 745bb18842eSBen Gardon 746bb18842eSBen Gardon /* 747bb18842eSBen Gardon * Installs a last-level SPTE to handle a TDP page fault. 748bb18842eSBen Gardon * (NPT/EPT violation/misconfiguration) 749bb18842eSBen Gardon */ 750bb18842eSBen Gardon static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, 751bb18842eSBen Gardon int map_writable, 752bb18842eSBen Gardon struct tdp_iter *iter, 753bb18842eSBen Gardon kvm_pfn_t pfn, bool prefault) 754bb18842eSBen Gardon { 755bb18842eSBen Gardon u64 new_spte; 756bb18842eSBen Gardon int ret = 0; 757bb18842eSBen Gardon int make_spte_ret = 0; 758bb18842eSBen Gardon 7599a77daacSBen Gardon if (unlikely(is_noslot_pfn(pfn))) 760bb18842eSBen Gardon new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL); 7619a77daacSBen Gardon else 762bb18842eSBen Gardon make_spte_ret = make_spte(vcpu, ACC_ALL, iter->level, iter->gfn, 763bb18842eSBen Gardon pfn, iter->old_spte, prefault, true, 764bb18842eSBen Gardon map_writable, !shadow_accessed_mask, 765bb18842eSBen Gardon &new_spte); 766bb18842eSBen Gardon 767bb18842eSBen Gardon if (new_spte == iter->old_spte) 768bb18842eSBen Gardon ret = RET_PF_SPURIOUS; 7699a77daacSBen Gardon else if (!tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte)) 7709a77daacSBen Gardon return RET_PF_RETRY; 771bb18842eSBen Gardon 772bb18842eSBen Gardon /* 773bb18842eSBen Gardon * If the page fault was caused by a write but the page is write 774bb18842eSBen Gardon * protected, emulation is needed. If the emulation was skipped, 775bb18842eSBen Gardon * the vCPU would have the same fault again. 776bb18842eSBen Gardon */ 777bb18842eSBen Gardon if (make_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { 778bb18842eSBen Gardon if (write) 779bb18842eSBen Gardon ret = RET_PF_EMULATE; 780bb18842eSBen Gardon kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 781bb18842eSBen Gardon } 782bb18842eSBen Gardon 783bb18842eSBen Gardon /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */ 7849a77daacSBen Gardon if (unlikely(is_mmio_spte(new_spte))) { 7859a77daacSBen Gardon trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn, 7869a77daacSBen Gardon new_spte); 787bb18842eSBen Gardon ret = RET_PF_EMULATE; 7889a77daacSBen Gardon } else 7899a77daacSBen Gardon trace_kvm_mmu_set_spte(iter->level, iter->gfn, 7909a77daacSBen Gardon rcu_dereference(iter->sptep)); 791bb18842eSBen Gardon 7927cca2d0bSBen Gardon trace_kvm_mmu_set_spte(iter->level, iter->gfn, 7937cca2d0bSBen Gardon rcu_dereference(iter->sptep)); 794bb18842eSBen Gardon if (!prefault) 795bb18842eSBen Gardon vcpu->stat.pf_fixed++; 796bb18842eSBen Gardon 797bb18842eSBen Gardon return ret; 798bb18842eSBen Gardon } 799bb18842eSBen Gardon 800bb18842eSBen Gardon /* 801bb18842eSBen Gardon * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing 802bb18842eSBen Gardon * page tables and SPTEs to translate the faulting guest physical address. 803bb18842eSBen Gardon */ 804bb18842eSBen Gardon int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, 805bb18842eSBen Gardon int map_writable, int max_level, kvm_pfn_t pfn, 806bb18842eSBen Gardon bool prefault) 807bb18842eSBen Gardon { 808bb18842eSBen Gardon bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(); 809bb18842eSBen Gardon bool write = error_code & PFERR_WRITE_MASK; 810bb18842eSBen Gardon bool exec = error_code & PFERR_FETCH_MASK; 811bb18842eSBen Gardon bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled; 812bb18842eSBen Gardon struct kvm_mmu *mmu = vcpu->arch.mmu; 813bb18842eSBen Gardon struct tdp_iter iter; 81489c0fd49SBen Gardon struct kvm_mmu_page *sp; 815bb18842eSBen Gardon u64 *child_pt; 816bb18842eSBen Gardon u64 new_spte; 817bb18842eSBen Gardon int ret; 818bb18842eSBen Gardon gfn_t gfn = gpa >> PAGE_SHIFT; 819bb18842eSBen Gardon int level; 820bb18842eSBen Gardon int req_level; 821bb18842eSBen Gardon 822bb18842eSBen Gardon if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) 823bb18842eSBen Gardon return RET_PF_RETRY; 824bb18842eSBen Gardon if (WARN_ON(!is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa))) 825bb18842eSBen Gardon return RET_PF_RETRY; 826bb18842eSBen Gardon 827bb18842eSBen Gardon level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, 828bb18842eSBen Gardon huge_page_disallowed, &req_level); 829bb18842eSBen Gardon 830bb18842eSBen Gardon trace_kvm_mmu_spte_requested(gpa, level, pfn); 8317cca2d0bSBen Gardon 8327cca2d0bSBen Gardon rcu_read_lock(); 8337cca2d0bSBen Gardon 834bb18842eSBen Gardon tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 835bb18842eSBen Gardon if (nx_huge_page_workaround_enabled) 836bb18842eSBen Gardon disallowed_hugepage_adjust(iter.old_spte, gfn, 837bb18842eSBen Gardon iter.level, &pfn, &level); 838bb18842eSBen Gardon 839bb18842eSBen Gardon if (iter.level == level) 840bb18842eSBen Gardon break; 841bb18842eSBen Gardon 842bb18842eSBen Gardon /* 843bb18842eSBen Gardon * If there is an SPTE mapping a large page at a higher level 844bb18842eSBen Gardon * than the target, that SPTE must be cleared and replaced 845bb18842eSBen Gardon * with a non-leaf SPTE. 846bb18842eSBen Gardon */ 847bb18842eSBen Gardon if (is_shadow_present_pte(iter.old_spte) && 848bb18842eSBen Gardon is_large_pte(iter.old_spte)) { 84908f07c80SBen Gardon if (!tdp_mmu_zap_spte_atomic(vcpu->kvm, &iter)) 8509a77daacSBen Gardon break; 851bb18842eSBen Gardon 852bb18842eSBen Gardon /* 853bb18842eSBen Gardon * The iter must explicitly re-read the spte here 854bb18842eSBen Gardon * because the new value informs the !present 855bb18842eSBen Gardon * path below. 856bb18842eSBen Gardon */ 8577cca2d0bSBen Gardon iter.old_spte = READ_ONCE(*rcu_dereference(iter.sptep)); 858bb18842eSBen Gardon } 859bb18842eSBen Gardon 860bb18842eSBen Gardon if (!is_shadow_present_pte(iter.old_spte)) { 86189c0fd49SBen Gardon sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level); 86289c0fd49SBen Gardon child_pt = sp->spt; 863a9442f59SBen Gardon 864bb18842eSBen Gardon new_spte = make_nonleaf_spte(child_pt, 865bb18842eSBen Gardon !shadow_accessed_mask); 866bb18842eSBen Gardon 8679a77daacSBen Gardon if (tdp_mmu_set_spte_atomic(vcpu->kvm, &iter, 8689a77daacSBen Gardon new_spte)) { 8699a77daacSBen Gardon tdp_mmu_link_page(vcpu->kvm, sp, true, 8709a77daacSBen Gardon huge_page_disallowed && 8719a77daacSBen Gardon req_level >= iter.level); 8729a77daacSBen Gardon 873bb18842eSBen Gardon trace_kvm_mmu_get_page(sp, true); 8749a77daacSBen Gardon } else { 8759a77daacSBen Gardon tdp_mmu_free_sp(sp); 8769a77daacSBen Gardon break; 8779a77daacSBen Gardon } 878bb18842eSBen Gardon } 879bb18842eSBen Gardon } 880bb18842eSBen Gardon 8819a77daacSBen Gardon if (iter.level != level) { 8827cca2d0bSBen Gardon rcu_read_unlock(); 883bb18842eSBen Gardon return RET_PF_RETRY; 8847cca2d0bSBen Gardon } 885bb18842eSBen Gardon 886bb18842eSBen Gardon ret = tdp_mmu_map_handle_target_level(vcpu, write, map_writable, &iter, 887bb18842eSBen Gardon pfn, prefault); 8887cca2d0bSBen Gardon rcu_read_unlock(); 889bb18842eSBen Gardon 890bb18842eSBen Gardon return ret; 891bb18842eSBen Gardon } 892063afacdSBen Gardon 8938f5c44f9SMaciej S. Szmigiero static __always_inline int 8948f5c44f9SMaciej S. Szmigiero kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, 8958f5c44f9SMaciej S. Szmigiero unsigned long start, 8968f5c44f9SMaciej S. Szmigiero unsigned long end, 8978f5c44f9SMaciej S. Szmigiero unsigned long data, 8988f5c44f9SMaciej S. Szmigiero int (*handler)(struct kvm *kvm, 8998f5c44f9SMaciej S. Szmigiero struct kvm_memory_slot *slot, 9008f5c44f9SMaciej S. Szmigiero struct kvm_mmu_page *root, 9018f5c44f9SMaciej S. Szmigiero gfn_t start, 9028f5c44f9SMaciej S. Szmigiero gfn_t end, 9038f5c44f9SMaciej S. Szmigiero unsigned long data)) 904063afacdSBen Gardon { 905063afacdSBen Gardon struct kvm_memslots *slots; 906063afacdSBen Gardon struct kvm_memory_slot *memslot; 907063afacdSBen Gardon struct kvm_mmu_page *root; 908063afacdSBen Gardon int ret = 0; 909063afacdSBen Gardon int as_id; 910063afacdSBen Gardon 911a889ea54SBen Gardon for_each_tdp_mmu_root_yield_safe(kvm, root) { 912063afacdSBen Gardon as_id = kvm_mmu_page_as_id(root); 913063afacdSBen Gardon slots = __kvm_memslots(kvm, as_id); 914063afacdSBen Gardon kvm_for_each_memslot(memslot, slots) { 915063afacdSBen Gardon unsigned long hva_start, hva_end; 916063afacdSBen Gardon gfn_t gfn_start, gfn_end; 917063afacdSBen Gardon 918063afacdSBen Gardon hva_start = max(start, memslot->userspace_addr); 919063afacdSBen Gardon hva_end = min(end, memslot->userspace_addr + 920063afacdSBen Gardon (memslot->npages << PAGE_SHIFT)); 921063afacdSBen Gardon if (hva_start >= hva_end) 922063afacdSBen Gardon continue; 923063afacdSBen Gardon /* 924063afacdSBen Gardon * {gfn(page) | page intersects with [hva_start, hva_end)} = 925063afacdSBen Gardon * {gfn_start, gfn_start+1, ..., gfn_end-1}. 926063afacdSBen Gardon */ 927063afacdSBen Gardon gfn_start = hva_to_gfn_memslot(hva_start, memslot); 928063afacdSBen Gardon gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); 929063afacdSBen Gardon 930063afacdSBen Gardon ret |= handler(kvm, memslot, root, gfn_start, 931063afacdSBen Gardon gfn_end, data); 932063afacdSBen Gardon } 933063afacdSBen Gardon } 934063afacdSBen Gardon 935063afacdSBen Gardon return ret; 936063afacdSBen Gardon } 937063afacdSBen Gardon 938063afacdSBen Gardon static int zap_gfn_range_hva_wrapper(struct kvm *kvm, 939063afacdSBen Gardon struct kvm_memory_slot *slot, 940063afacdSBen Gardon struct kvm_mmu_page *root, gfn_t start, 941063afacdSBen Gardon gfn_t end, unsigned long unused) 942063afacdSBen Gardon { 943063afacdSBen Gardon return zap_gfn_range(kvm, root, start, end, false); 944063afacdSBen Gardon } 945063afacdSBen Gardon 946063afacdSBen Gardon int kvm_tdp_mmu_zap_hva_range(struct kvm *kvm, unsigned long start, 947063afacdSBen Gardon unsigned long end) 948063afacdSBen Gardon { 949063afacdSBen Gardon return kvm_tdp_mmu_handle_hva_range(kvm, start, end, 0, 950063afacdSBen Gardon zap_gfn_range_hva_wrapper); 951063afacdSBen Gardon } 952f8e14497SBen Gardon 953f8e14497SBen Gardon /* 954f8e14497SBen Gardon * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero 955f8e14497SBen Gardon * if any of the GFNs in the range have been accessed. 956f8e14497SBen Gardon */ 957f8e14497SBen Gardon static int age_gfn_range(struct kvm *kvm, struct kvm_memory_slot *slot, 958f8e14497SBen Gardon struct kvm_mmu_page *root, gfn_t start, gfn_t end, 959f8e14497SBen Gardon unsigned long unused) 960f8e14497SBen Gardon { 961f8e14497SBen Gardon struct tdp_iter iter; 962f8e14497SBen Gardon int young = 0; 963f8e14497SBen Gardon u64 new_spte = 0; 964f8e14497SBen Gardon 9657cca2d0bSBen Gardon rcu_read_lock(); 9667cca2d0bSBen Gardon 967f8e14497SBen Gardon tdp_root_for_each_leaf_pte(iter, root, start, end) { 968f8e14497SBen Gardon /* 969f8e14497SBen Gardon * If we have a non-accessed entry we don't need to change the 970f8e14497SBen Gardon * pte. 971f8e14497SBen Gardon */ 972f8e14497SBen Gardon if (!is_accessed_spte(iter.old_spte)) 973f8e14497SBen Gardon continue; 974f8e14497SBen Gardon 975f8e14497SBen Gardon new_spte = iter.old_spte; 976f8e14497SBen Gardon 977f8e14497SBen Gardon if (spte_ad_enabled(new_spte)) { 978f8e14497SBen Gardon clear_bit((ffs(shadow_accessed_mask) - 1), 979f8e14497SBen Gardon (unsigned long *)&new_spte); 980f8e14497SBen Gardon } else { 981f8e14497SBen Gardon /* 982f8e14497SBen Gardon * Capture the dirty status of the page, so that it doesn't get 983f8e14497SBen Gardon * lost when the SPTE is marked for access tracking. 984f8e14497SBen Gardon */ 985f8e14497SBen Gardon if (is_writable_pte(new_spte)) 986f8e14497SBen Gardon kvm_set_pfn_dirty(spte_to_pfn(new_spte)); 987f8e14497SBen Gardon 988f8e14497SBen Gardon new_spte = mark_spte_for_access_track(new_spte); 989f8e14497SBen Gardon } 990a6a0b05dSBen Gardon new_spte &= ~shadow_dirty_mask; 991f8e14497SBen Gardon 992f8e14497SBen Gardon tdp_mmu_set_spte_no_acc_track(kvm, &iter, new_spte); 993f8e14497SBen Gardon young = 1; 99433dd3574SBen Gardon 99533dd3574SBen Gardon trace_kvm_age_page(iter.gfn, iter.level, slot, young); 996f8e14497SBen Gardon } 997f8e14497SBen Gardon 9987cca2d0bSBen Gardon rcu_read_unlock(); 9997cca2d0bSBen Gardon 1000f8e14497SBen Gardon return young; 1001f8e14497SBen Gardon } 1002f8e14497SBen Gardon 1003f8e14497SBen Gardon int kvm_tdp_mmu_age_hva_range(struct kvm *kvm, unsigned long start, 1004f8e14497SBen Gardon unsigned long end) 1005f8e14497SBen Gardon { 1006f8e14497SBen Gardon return kvm_tdp_mmu_handle_hva_range(kvm, start, end, 0, 1007f8e14497SBen Gardon age_gfn_range); 1008f8e14497SBen Gardon } 1009f8e14497SBen Gardon 1010f8e14497SBen Gardon static int test_age_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, 1011f8e14497SBen Gardon struct kvm_mmu_page *root, gfn_t gfn, gfn_t unused, 1012f8e14497SBen Gardon unsigned long unused2) 1013f8e14497SBen Gardon { 1014f8e14497SBen Gardon struct tdp_iter iter; 1015f8e14497SBen Gardon 1016f8e14497SBen Gardon tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) 1017f8e14497SBen Gardon if (is_accessed_spte(iter.old_spte)) 1018f8e14497SBen Gardon return 1; 1019f8e14497SBen Gardon 1020f8e14497SBen Gardon return 0; 1021f8e14497SBen Gardon } 1022f8e14497SBen Gardon 1023f8e14497SBen Gardon int kvm_tdp_mmu_test_age_hva(struct kvm *kvm, unsigned long hva) 1024f8e14497SBen Gardon { 1025f8e14497SBen Gardon return kvm_tdp_mmu_handle_hva_range(kvm, hva, hva + 1, 0, 1026f8e14497SBen Gardon test_age_gfn); 1027f8e14497SBen Gardon } 10281d8dd6b3SBen Gardon 10291d8dd6b3SBen Gardon /* 10301d8dd6b3SBen Gardon * Handle the changed_pte MMU notifier for the TDP MMU. 10311d8dd6b3SBen Gardon * data is a pointer to the new pte_t mapping the HVA specified by the MMU 10321d8dd6b3SBen Gardon * notifier. 10331d8dd6b3SBen Gardon * Returns non-zero if a flush is needed before releasing the MMU lock. 10341d8dd6b3SBen Gardon */ 10351d8dd6b3SBen Gardon static int set_tdp_spte(struct kvm *kvm, struct kvm_memory_slot *slot, 10361d8dd6b3SBen Gardon struct kvm_mmu_page *root, gfn_t gfn, gfn_t unused, 10371d8dd6b3SBen Gardon unsigned long data) 10381d8dd6b3SBen Gardon { 10391d8dd6b3SBen Gardon struct tdp_iter iter; 10401d8dd6b3SBen Gardon pte_t *ptep = (pte_t *)data; 10411d8dd6b3SBen Gardon kvm_pfn_t new_pfn; 10421d8dd6b3SBen Gardon u64 new_spte; 10431d8dd6b3SBen Gardon int need_flush = 0; 10441d8dd6b3SBen Gardon 10457cca2d0bSBen Gardon rcu_read_lock(); 10467cca2d0bSBen Gardon 10471d8dd6b3SBen Gardon WARN_ON(pte_huge(*ptep)); 10481d8dd6b3SBen Gardon 10491d8dd6b3SBen Gardon new_pfn = pte_pfn(*ptep); 10501d8dd6b3SBen Gardon 10511d8dd6b3SBen Gardon tdp_root_for_each_pte(iter, root, gfn, gfn + 1) { 10521d8dd6b3SBen Gardon if (iter.level != PG_LEVEL_4K) 10531d8dd6b3SBen Gardon continue; 10541d8dd6b3SBen Gardon 10551d8dd6b3SBen Gardon if (!is_shadow_present_pte(iter.old_spte)) 10561d8dd6b3SBen Gardon break; 10571d8dd6b3SBen Gardon 10581d8dd6b3SBen Gardon tdp_mmu_set_spte(kvm, &iter, 0); 10591d8dd6b3SBen Gardon 10601d8dd6b3SBen Gardon kvm_flush_remote_tlbs_with_address(kvm, iter.gfn, 1); 10611d8dd6b3SBen Gardon 10621d8dd6b3SBen Gardon if (!pte_write(*ptep)) { 10631d8dd6b3SBen Gardon new_spte = kvm_mmu_changed_pte_notifier_make_spte( 10641d8dd6b3SBen Gardon iter.old_spte, new_pfn); 10651d8dd6b3SBen Gardon 10661d8dd6b3SBen Gardon tdp_mmu_set_spte(kvm, &iter, new_spte); 10671d8dd6b3SBen Gardon } 10681d8dd6b3SBen Gardon 10691d8dd6b3SBen Gardon need_flush = 1; 10701d8dd6b3SBen Gardon } 10711d8dd6b3SBen Gardon 10721d8dd6b3SBen Gardon if (need_flush) 10731d8dd6b3SBen Gardon kvm_flush_remote_tlbs_with_address(kvm, gfn, 1); 10741d8dd6b3SBen Gardon 10757cca2d0bSBen Gardon rcu_read_unlock(); 10767cca2d0bSBen Gardon 10771d8dd6b3SBen Gardon return 0; 10781d8dd6b3SBen Gardon } 10791d8dd6b3SBen Gardon 10801d8dd6b3SBen Gardon int kvm_tdp_mmu_set_spte_hva(struct kvm *kvm, unsigned long address, 10811d8dd6b3SBen Gardon pte_t *host_ptep) 10821d8dd6b3SBen Gardon { 10831d8dd6b3SBen Gardon return kvm_tdp_mmu_handle_hva_range(kvm, address, address + 1, 10841d8dd6b3SBen Gardon (unsigned long)host_ptep, 10851d8dd6b3SBen Gardon set_tdp_spte); 10861d8dd6b3SBen Gardon } 10871d8dd6b3SBen Gardon 1088a6a0b05dSBen Gardon /* 1089a6a0b05dSBen Gardon * Remove write access from all the SPTEs mapping GFNs [start, end). If 1090a6a0b05dSBen Gardon * skip_4k is set, SPTEs that map 4k pages, will not be write-protected. 1091a6a0b05dSBen Gardon * Returns true if an SPTE has been changed and the TLBs need to be flushed. 1092a6a0b05dSBen Gardon */ 1093a6a0b05dSBen Gardon static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1094a6a0b05dSBen Gardon gfn_t start, gfn_t end, int min_level) 1095a6a0b05dSBen Gardon { 1096a6a0b05dSBen Gardon struct tdp_iter iter; 1097a6a0b05dSBen Gardon u64 new_spte; 1098a6a0b05dSBen Gardon bool spte_set = false; 1099a6a0b05dSBen Gardon 11007cca2d0bSBen Gardon rcu_read_lock(); 11017cca2d0bSBen Gardon 1102a6a0b05dSBen Gardon BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 1103a6a0b05dSBen Gardon 1104a6a0b05dSBen Gardon for_each_tdp_pte_min_level(iter, root->spt, root->role.level, 1105a6a0b05dSBen Gardon min_level, start, end) { 11061af4a960SBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, false)) 11071af4a960SBen Gardon continue; 11081af4a960SBen Gardon 1109a6a0b05dSBen Gardon if (!is_shadow_present_pte(iter.old_spte) || 11100f99ee2cSBen Gardon !is_last_spte(iter.old_spte, iter.level) || 11110f99ee2cSBen Gardon !(iter.old_spte & PT_WRITABLE_MASK)) 1112a6a0b05dSBen Gardon continue; 1113a6a0b05dSBen Gardon 1114a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1115a6a0b05dSBen Gardon 1116a6a0b05dSBen Gardon tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte); 1117a6a0b05dSBen Gardon spte_set = true; 1118a6a0b05dSBen Gardon } 11197cca2d0bSBen Gardon 11207cca2d0bSBen Gardon rcu_read_unlock(); 1121a6a0b05dSBen Gardon return spte_set; 1122a6a0b05dSBen Gardon } 1123a6a0b05dSBen Gardon 1124a6a0b05dSBen Gardon /* 1125a6a0b05dSBen Gardon * Remove write access from all the SPTEs mapping GFNs in the memslot. Will 1126a6a0b05dSBen Gardon * only affect leaf SPTEs down to min_level. 1127a6a0b05dSBen Gardon * Returns true if an SPTE has been changed and the TLBs need to be flushed. 1128a6a0b05dSBen Gardon */ 1129a6a0b05dSBen Gardon bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, struct kvm_memory_slot *slot, 1130a6a0b05dSBen Gardon int min_level) 1131a6a0b05dSBen Gardon { 1132a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1133a6a0b05dSBen Gardon int root_as_id; 1134a6a0b05dSBen Gardon bool spte_set = false; 1135a6a0b05dSBen Gardon 1136a889ea54SBen Gardon for_each_tdp_mmu_root_yield_safe(kvm, root) { 1137a6a0b05dSBen Gardon root_as_id = kvm_mmu_page_as_id(root); 1138a6a0b05dSBen Gardon if (root_as_id != slot->as_id) 1139a6a0b05dSBen Gardon continue; 1140a6a0b05dSBen Gardon 1141a6a0b05dSBen Gardon spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn, 1142a6a0b05dSBen Gardon slot->base_gfn + slot->npages, min_level); 1143a6a0b05dSBen Gardon } 1144a6a0b05dSBen Gardon 1145a6a0b05dSBen Gardon return spte_set; 1146a6a0b05dSBen Gardon } 1147a6a0b05dSBen Gardon 1148a6a0b05dSBen Gardon /* 1149a6a0b05dSBen Gardon * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If 1150a6a0b05dSBen Gardon * AD bits are enabled, this will involve clearing the dirty bit on each SPTE. 1151a6a0b05dSBen Gardon * If AD bits are not enabled, this will require clearing the writable bit on 1152a6a0b05dSBen Gardon * each SPTE. Returns true if an SPTE has been changed and the TLBs need to 1153a6a0b05dSBen Gardon * be flushed. 1154a6a0b05dSBen Gardon */ 1155a6a0b05dSBen Gardon static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1156a6a0b05dSBen Gardon gfn_t start, gfn_t end) 1157a6a0b05dSBen Gardon { 1158a6a0b05dSBen Gardon struct tdp_iter iter; 1159a6a0b05dSBen Gardon u64 new_spte; 1160a6a0b05dSBen Gardon bool spte_set = false; 1161a6a0b05dSBen Gardon 11627cca2d0bSBen Gardon rcu_read_lock(); 11637cca2d0bSBen Gardon 1164a6a0b05dSBen Gardon tdp_root_for_each_leaf_pte(iter, root, start, end) { 11651af4a960SBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, false)) 11661af4a960SBen Gardon continue; 11671af4a960SBen Gardon 1168a6a0b05dSBen Gardon if (spte_ad_need_write_protect(iter.old_spte)) { 1169a6a0b05dSBen Gardon if (is_writable_pte(iter.old_spte)) 1170a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1171a6a0b05dSBen Gardon else 1172a6a0b05dSBen Gardon continue; 1173a6a0b05dSBen Gardon } else { 1174a6a0b05dSBen Gardon if (iter.old_spte & shadow_dirty_mask) 1175a6a0b05dSBen Gardon new_spte = iter.old_spte & ~shadow_dirty_mask; 1176a6a0b05dSBen Gardon else 1177a6a0b05dSBen Gardon continue; 1178a6a0b05dSBen Gardon } 1179a6a0b05dSBen Gardon 1180a6a0b05dSBen Gardon tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte); 1181a6a0b05dSBen Gardon spte_set = true; 1182a6a0b05dSBen Gardon } 11837cca2d0bSBen Gardon 11847cca2d0bSBen Gardon rcu_read_unlock(); 1185a6a0b05dSBen Gardon return spte_set; 1186a6a0b05dSBen Gardon } 1187a6a0b05dSBen Gardon 1188a6a0b05dSBen Gardon /* 1189a6a0b05dSBen Gardon * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If 1190a6a0b05dSBen Gardon * AD bits are enabled, this will involve clearing the dirty bit on each SPTE. 1191a6a0b05dSBen Gardon * If AD bits are not enabled, this will require clearing the writable bit on 1192a6a0b05dSBen Gardon * each SPTE. Returns true if an SPTE has been changed and the TLBs need to 1193a6a0b05dSBen Gardon * be flushed. 1194a6a0b05dSBen Gardon */ 1195a6a0b05dSBen Gardon bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 1196a6a0b05dSBen Gardon { 1197a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1198a6a0b05dSBen Gardon int root_as_id; 1199a6a0b05dSBen Gardon bool spte_set = false; 1200a6a0b05dSBen Gardon 1201a889ea54SBen Gardon for_each_tdp_mmu_root_yield_safe(kvm, root) { 1202a6a0b05dSBen Gardon root_as_id = kvm_mmu_page_as_id(root); 1203a6a0b05dSBen Gardon if (root_as_id != slot->as_id) 1204a6a0b05dSBen Gardon continue; 1205a6a0b05dSBen Gardon 1206a6a0b05dSBen Gardon spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn, 1207a6a0b05dSBen Gardon slot->base_gfn + slot->npages); 1208a6a0b05dSBen Gardon } 1209a6a0b05dSBen Gardon 1210a6a0b05dSBen Gardon return spte_set; 1211a6a0b05dSBen Gardon } 1212a6a0b05dSBen Gardon 1213a6a0b05dSBen Gardon /* 1214a6a0b05dSBen Gardon * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is 1215a6a0b05dSBen Gardon * set in mask, starting at gfn. The given memslot is expected to contain all 1216a6a0b05dSBen Gardon * the GFNs represented by set bits in the mask. If AD bits are enabled, 1217a6a0b05dSBen Gardon * clearing the dirty status will involve clearing the dirty bit on each SPTE 1218a6a0b05dSBen Gardon * or, if AD bits are not enabled, clearing the writable bit on each SPTE. 1219a6a0b05dSBen Gardon */ 1220a6a0b05dSBen Gardon static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root, 1221a6a0b05dSBen Gardon gfn_t gfn, unsigned long mask, bool wrprot) 1222a6a0b05dSBen Gardon { 1223a6a0b05dSBen Gardon struct tdp_iter iter; 1224a6a0b05dSBen Gardon u64 new_spte; 1225a6a0b05dSBen Gardon 12267cca2d0bSBen Gardon rcu_read_lock(); 12277cca2d0bSBen Gardon 1228a6a0b05dSBen Gardon tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask), 1229a6a0b05dSBen Gardon gfn + BITS_PER_LONG) { 1230a6a0b05dSBen Gardon if (!mask) 1231a6a0b05dSBen Gardon break; 1232a6a0b05dSBen Gardon 1233a6a0b05dSBen Gardon if (iter.level > PG_LEVEL_4K || 1234a6a0b05dSBen Gardon !(mask & (1UL << (iter.gfn - gfn)))) 1235a6a0b05dSBen Gardon continue; 1236a6a0b05dSBen Gardon 1237f1b3b06aSBen Gardon mask &= ~(1UL << (iter.gfn - gfn)); 1238f1b3b06aSBen Gardon 1239a6a0b05dSBen Gardon if (wrprot || spte_ad_need_write_protect(iter.old_spte)) { 1240a6a0b05dSBen Gardon if (is_writable_pte(iter.old_spte)) 1241a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1242a6a0b05dSBen Gardon else 1243a6a0b05dSBen Gardon continue; 1244a6a0b05dSBen Gardon } else { 1245a6a0b05dSBen Gardon if (iter.old_spte & shadow_dirty_mask) 1246a6a0b05dSBen Gardon new_spte = iter.old_spte & ~shadow_dirty_mask; 1247a6a0b05dSBen Gardon else 1248a6a0b05dSBen Gardon continue; 1249a6a0b05dSBen Gardon } 1250a6a0b05dSBen Gardon 1251a6a0b05dSBen Gardon tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte); 1252a6a0b05dSBen Gardon } 12537cca2d0bSBen Gardon 12547cca2d0bSBen Gardon rcu_read_unlock(); 1255a6a0b05dSBen Gardon } 1256a6a0b05dSBen Gardon 1257a6a0b05dSBen Gardon /* 1258a6a0b05dSBen Gardon * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is 1259a6a0b05dSBen Gardon * set in mask, starting at gfn. The given memslot is expected to contain all 1260a6a0b05dSBen Gardon * the GFNs represented by set bits in the mask. If AD bits are enabled, 1261a6a0b05dSBen Gardon * clearing the dirty status will involve clearing the dirty bit on each SPTE 1262a6a0b05dSBen Gardon * or, if AD bits are not enabled, clearing the writable bit on each SPTE. 1263a6a0b05dSBen Gardon */ 1264a6a0b05dSBen Gardon void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1265a6a0b05dSBen Gardon struct kvm_memory_slot *slot, 1266a6a0b05dSBen Gardon gfn_t gfn, unsigned long mask, 1267a6a0b05dSBen Gardon bool wrprot) 1268a6a0b05dSBen Gardon { 1269a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1270a6a0b05dSBen Gardon int root_as_id; 1271a6a0b05dSBen Gardon 1272531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 1273a6a0b05dSBen Gardon for_each_tdp_mmu_root(kvm, root) { 1274a6a0b05dSBen Gardon root_as_id = kvm_mmu_page_as_id(root); 1275a6a0b05dSBen Gardon if (root_as_id != slot->as_id) 1276a6a0b05dSBen Gardon continue; 1277a6a0b05dSBen Gardon 1278a6a0b05dSBen Gardon clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot); 1279a6a0b05dSBen Gardon } 1280a6a0b05dSBen Gardon } 1281a6a0b05dSBen Gardon 1282a6a0b05dSBen Gardon /* 128387aa9ec9SBen Gardon * Clear leaf entries which could be replaced by large mappings, for 128487aa9ec9SBen Gardon * GFNs within the slot. 128514881998SBen Gardon */ 128614881998SBen Gardon static void zap_collapsible_spte_range(struct kvm *kvm, 128714881998SBen Gardon struct kvm_mmu_page *root, 12889eba50f8SSean Christopherson struct kvm_memory_slot *slot) 128914881998SBen Gardon { 12909eba50f8SSean Christopherson gfn_t start = slot->base_gfn; 12919eba50f8SSean Christopherson gfn_t end = start + slot->npages; 129214881998SBen Gardon struct tdp_iter iter; 129314881998SBen Gardon kvm_pfn_t pfn; 129414881998SBen Gardon bool spte_set = false; 129514881998SBen Gardon 12967cca2d0bSBen Gardon rcu_read_lock(); 12977cca2d0bSBen Gardon 129814881998SBen Gardon tdp_root_for_each_pte(iter, root, start, end) { 12991af4a960SBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, spte_set)) { 13001af4a960SBen Gardon spte_set = false; 13011af4a960SBen Gardon continue; 13021af4a960SBen Gardon } 13031af4a960SBen Gardon 130414881998SBen Gardon if (!is_shadow_present_pte(iter.old_spte) || 130587aa9ec9SBen Gardon !is_last_spte(iter.old_spte, iter.level)) 130614881998SBen Gardon continue; 130714881998SBen Gardon 130814881998SBen Gardon pfn = spte_to_pfn(iter.old_spte); 130914881998SBen Gardon if (kvm_is_reserved_pfn(pfn) || 13109eba50f8SSean Christopherson iter.level >= kvm_mmu_max_mapping_level(kvm, slot, iter.gfn, 13119eba50f8SSean Christopherson pfn, PG_LEVEL_NUM)) 131214881998SBen Gardon continue; 131314881998SBen Gardon 131414881998SBen Gardon tdp_mmu_set_spte(kvm, &iter, 0); 131514881998SBen Gardon 13161af4a960SBen Gardon spte_set = true; 131714881998SBen Gardon } 131814881998SBen Gardon 13197cca2d0bSBen Gardon rcu_read_unlock(); 132014881998SBen Gardon if (spte_set) 132114881998SBen Gardon kvm_flush_remote_tlbs(kvm); 132214881998SBen Gardon } 132314881998SBen Gardon 132414881998SBen Gardon /* 132514881998SBen Gardon * Clear non-leaf entries (and free associated page tables) which could 132614881998SBen Gardon * be replaced by large mappings, for GFNs within the slot. 132714881998SBen Gardon */ 132814881998SBen Gardon void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, 13299eba50f8SSean Christopherson struct kvm_memory_slot *slot) 133014881998SBen Gardon { 133114881998SBen Gardon struct kvm_mmu_page *root; 133214881998SBen Gardon int root_as_id; 133314881998SBen Gardon 1334a889ea54SBen Gardon for_each_tdp_mmu_root_yield_safe(kvm, root) { 133514881998SBen Gardon root_as_id = kvm_mmu_page_as_id(root); 133614881998SBen Gardon if (root_as_id != slot->as_id) 133714881998SBen Gardon continue; 133814881998SBen Gardon 13399eba50f8SSean Christopherson zap_collapsible_spte_range(kvm, root, slot); 134014881998SBen Gardon } 134114881998SBen Gardon } 134246044f72SBen Gardon 134346044f72SBen Gardon /* 134446044f72SBen Gardon * Removes write access on the last level SPTE mapping this GFN and unsets the 134546044f72SBen Gardon * SPTE_MMU_WRITABLE bit to ensure future writes continue to be intercepted. 134646044f72SBen Gardon * Returns true if an SPTE was set and a TLB flush is needed. 134746044f72SBen Gardon */ 134846044f72SBen Gardon static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, 134946044f72SBen Gardon gfn_t gfn) 135046044f72SBen Gardon { 135146044f72SBen Gardon struct tdp_iter iter; 135246044f72SBen Gardon u64 new_spte; 135346044f72SBen Gardon bool spte_set = false; 135446044f72SBen Gardon 13557cca2d0bSBen Gardon rcu_read_lock(); 13567cca2d0bSBen Gardon 135746044f72SBen Gardon tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) { 135846044f72SBen Gardon if (!is_writable_pte(iter.old_spte)) 135946044f72SBen Gardon break; 136046044f72SBen Gardon 136146044f72SBen Gardon new_spte = iter.old_spte & 136246044f72SBen Gardon ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE); 136346044f72SBen Gardon 136446044f72SBen Gardon tdp_mmu_set_spte(kvm, &iter, new_spte); 136546044f72SBen Gardon spte_set = true; 136646044f72SBen Gardon } 136746044f72SBen Gardon 13687cca2d0bSBen Gardon rcu_read_unlock(); 13697cca2d0bSBen Gardon 137046044f72SBen Gardon return spte_set; 137146044f72SBen Gardon } 137246044f72SBen Gardon 137346044f72SBen Gardon /* 137446044f72SBen Gardon * Removes write access on the last level SPTE mapping this GFN and unsets the 137546044f72SBen Gardon * SPTE_MMU_WRITABLE bit to ensure future writes continue to be intercepted. 137646044f72SBen Gardon * Returns true if an SPTE was set and a TLB flush is needed. 137746044f72SBen Gardon */ 137846044f72SBen Gardon bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, 137946044f72SBen Gardon struct kvm_memory_slot *slot, gfn_t gfn) 138046044f72SBen Gardon { 138146044f72SBen Gardon struct kvm_mmu_page *root; 138246044f72SBen Gardon int root_as_id; 138346044f72SBen Gardon bool spte_set = false; 138446044f72SBen Gardon 1385531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 138646044f72SBen Gardon for_each_tdp_mmu_root(kvm, root) { 138746044f72SBen Gardon root_as_id = kvm_mmu_page_as_id(root); 138846044f72SBen Gardon if (root_as_id != slot->as_id) 138946044f72SBen Gardon continue; 139046044f72SBen Gardon 139146044f72SBen Gardon spte_set |= write_protect_gfn(kvm, root, gfn); 139246044f72SBen Gardon } 139346044f72SBen Gardon return spte_set; 139446044f72SBen Gardon } 139546044f72SBen Gardon 139695fb5b02SBen Gardon /* 139795fb5b02SBen Gardon * Return the level of the lowest level SPTE added to sptes. 139895fb5b02SBen Gardon * That SPTE may be non-present. 139995fb5b02SBen Gardon */ 140039b4d43eSSean Christopherson int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, 140139b4d43eSSean Christopherson int *root_level) 140295fb5b02SBen Gardon { 140395fb5b02SBen Gardon struct tdp_iter iter; 140495fb5b02SBen Gardon struct kvm_mmu *mmu = vcpu->arch.mmu; 140595fb5b02SBen Gardon gfn_t gfn = addr >> PAGE_SHIFT; 14062aa07893SSean Christopherson int leaf = -1; 140795fb5b02SBen Gardon 140839b4d43eSSean Christopherson *root_level = vcpu->arch.mmu->shadow_root_level; 140995fb5b02SBen Gardon 14107cca2d0bSBen Gardon rcu_read_lock(); 14117cca2d0bSBen Gardon 141295fb5b02SBen Gardon tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 141395fb5b02SBen Gardon leaf = iter.level; 1414dde81f94SSean Christopherson sptes[leaf] = iter.old_spte; 141595fb5b02SBen Gardon } 141695fb5b02SBen Gardon 14177cca2d0bSBen Gardon rcu_read_unlock(); 14187cca2d0bSBen Gardon 141995fb5b02SBen Gardon return leaf; 142095fb5b02SBen Gardon } 1421