1fe5db27dSBen Gardon // SPDX-License-Identifier: GPL-2.0 2fe5db27dSBen Gardon 302c00b3aSBen Gardon #include "mmu.h" 402c00b3aSBen Gardon #include "mmu_internal.h" 5bb18842eSBen Gardon #include "mmutrace.h" 62f2fad08SBen Gardon #include "tdp_iter.h" 7fe5db27dSBen Gardon #include "tdp_mmu.h" 802c00b3aSBen Gardon #include "spte.h" 9fe5db27dSBen Gardon 109a77daacSBen Gardon #include <asm/cmpxchg.h> 1133dd3574SBen Gardon #include <trace/events/kvm.h> 1233dd3574SBen Gardon 1371ba3f31SPaolo Bonzini static bool __read_mostly tdp_mmu_enabled = true; 1495fb5b02SBen Gardon module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644); 15fe5db27dSBen Gardon 16fe5db27dSBen Gardon /* Initializes the TDP MMU for the VM, if enabled. */ 17d501f747SBen Gardon bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) 18fe5db27dSBen Gardon { 19897218ffSPaolo Bonzini if (!tdp_enabled || !READ_ONCE(tdp_mmu_enabled)) 20d501f747SBen Gardon return false; 21fe5db27dSBen Gardon 22fe5db27dSBen Gardon /* This should not be changed for the lifetime of the VM. */ 23fe5db27dSBen Gardon kvm->arch.tdp_mmu_enabled = true; 2402c00b3aSBen Gardon 2502c00b3aSBen Gardon INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); 269a77daacSBen Gardon spin_lock_init(&kvm->arch.tdp_mmu_pages_lock); 2789c0fd49SBen Gardon INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages); 28d501f747SBen Gardon 29d501f747SBen Gardon return true; 30fe5db27dSBen Gardon } 31fe5db27dSBen Gardon 32*226b8c8fSSean Christopherson /* Arbitrarily returns true so that this may be used in if statements. */ 33*226b8c8fSSean Christopherson static __always_inline bool kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm, 346103bc07SBen Gardon bool shared) 356103bc07SBen Gardon { 366103bc07SBen Gardon if (shared) 376103bc07SBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 386103bc07SBen Gardon else 396103bc07SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 40*226b8c8fSSean Christopherson 41*226b8c8fSSean Christopherson return true; 426103bc07SBen Gardon } 436103bc07SBen Gardon 44fe5db27dSBen Gardon void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) 45fe5db27dSBen Gardon { 46fe5db27dSBen Gardon if (!kvm->arch.tdp_mmu_enabled) 47fe5db27dSBen Gardon return; 4802c00b3aSBen Gardon 49524a1e4eSSean Christopherson WARN_ON(!list_empty(&kvm->arch.tdp_mmu_pages)); 5002c00b3aSBen Gardon WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots)); 517cca2d0bSBen Gardon 527cca2d0bSBen Gardon /* 537cca2d0bSBen Gardon * Ensure that all the outstanding RCU callbacks to free shadow pages 547cca2d0bSBen Gardon * can run before the VM is torn down. 557cca2d0bSBen Gardon */ 567cca2d0bSBen Gardon rcu_barrier(); 5702c00b3aSBen Gardon } 5802c00b3aSBen Gardon 592bdb3d84SBen Gardon static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 606103bc07SBen Gardon gfn_t start, gfn_t end, bool can_yield, bool flush, 616103bc07SBen Gardon bool shared); 622bdb3d84SBen Gardon 632bdb3d84SBen Gardon static void tdp_mmu_free_sp(struct kvm_mmu_page *sp) 64a889ea54SBen Gardon { 652bdb3d84SBen Gardon free_page((unsigned long)sp->spt); 662bdb3d84SBen Gardon kmem_cache_free(mmu_page_header_cache, sp); 67a889ea54SBen Gardon } 68a889ea54SBen Gardon 69c0e64238SBen Gardon /* 70c0e64238SBen Gardon * This is called through call_rcu in order to free TDP page table memory 71c0e64238SBen Gardon * safely with respect to other kernel threads that may be operating on 72c0e64238SBen Gardon * the memory. 73c0e64238SBen Gardon * By only accessing TDP MMU page table memory in an RCU read critical 74c0e64238SBen Gardon * section, and freeing it after a grace period, lockless access to that 75c0e64238SBen Gardon * memory won't use it after it is freed. 76c0e64238SBen Gardon */ 77c0e64238SBen Gardon static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) 78a889ea54SBen Gardon { 79c0e64238SBen Gardon struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page, 80c0e64238SBen Gardon rcu_head); 81a889ea54SBen Gardon 82c0e64238SBen Gardon tdp_mmu_free_sp(sp); 83a889ea54SBen Gardon } 84a889ea54SBen Gardon 856103bc07SBen Gardon void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root, 866103bc07SBen Gardon bool shared) 872bdb3d84SBen Gardon { 886103bc07SBen Gardon kvm_lockdep_assert_mmu_lock_held(kvm, shared); 892bdb3d84SBen Gardon 9011cccf5cSBen Gardon if (!refcount_dec_and_test(&root->tdp_mmu_root_count)) 912bdb3d84SBen Gardon return; 922bdb3d84SBen Gardon 932bdb3d84SBen Gardon WARN_ON(!root->tdp_mmu_page); 942bdb3d84SBen Gardon 95c0e64238SBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 96c0e64238SBen Gardon list_del_rcu(&root->link); 97c0e64238SBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 982bdb3d84SBen Gardon 99db01416bSSean Christopherson /* 100db01416bSSean Christopherson * A TLB flush is not necessary as KVM performs a local TLB flush when 101db01416bSSean Christopherson * allocating a new root (see kvm_mmu_load()), and when migrating vCPU 102db01416bSSean Christopherson * to a different pCPU. Note, the local TLB flush on reuse also 103db01416bSSean Christopherson * invalidates any paging-structure-cache entries, i.e. TLB entries for 104db01416bSSean Christopherson * intermediate paging structures, that may be zapped, as such entries 105db01416bSSean Christopherson * are associated with the ASID on both VMX and SVM. 106db01416bSSean Christopherson */ 107db01416bSSean Christopherson (void)zap_gfn_range(kvm, root, 0, -1ull, false, false, shared); 1082bdb3d84SBen Gardon 109c0e64238SBen Gardon call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback); 110a889ea54SBen Gardon } 111a889ea54SBen Gardon 112cfc10997SBen Gardon /* 113d62007edSSean Christopherson * Returns the next root after @prev_root (or the first root if @prev_root is 114d62007edSSean Christopherson * NULL). A reference to the returned root is acquired, and the reference to 115d62007edSSean Christopherson * @prev_root is released (the caller obviously must hold a reference to 116d62007edSSean Christopherson * @prev_root if it's non-NULL). 117d62007edSSean Christopherson * 118d62007edSSean Christopherson * If @only_valid is true, invalid roots are skipped. 119d62007edSSean Christopherson * 120d62007edSSean Christopherson * Returns NULL if the end of tdp_mmu_roots was reached. 121cfc10997SBen Gardon */ 122cfc10997SBen Gardon static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, 1236103bc07SBen Gardon struct kvm_mmu_page *prev_root, 124d62007edSSean Christopherson bool shared, bool only_valid) 125a889ea54SBen Gardon { 126a889ea54SBen Gardon struct kvm_mmu_page *next_root; 127a889ea54SBen Gardon 128c0e64238SBen Gardon rcu_read_lock(); 129c0e64238SBen Gardon 130cfc10997SBen Gardon if (prev_root) 131c0e64238SBen Gardon next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 132c0e64238SBen Gardon &prev_root->link, 133c0e64238SBen Gardon typeof(*prev_root), link); 134cfc10997SBen Gardon else 135c0e64238SBen Gardon next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots, 136cfc10997SBen Gardon typeof(*next_root), link); 137cfc10997SBen Gardon 13804dc4e6cSSean Christopherson while (next_root) { 139d62007edSSean Christopherson if ((!only_valid || !next_root->role.invalid) && 140ad6d6b94SJinrong Liang kvm_tdp_mmu_get_root(next_root)) 14104dc4e6cSSean Christopherson break; 14204dc4e6cSSean Christopherson 143c0e64238SBen Gardon next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 144c0e64238SBen Gardon &next_root->link, typeof(*next_root), link); 14504dc4e6cSSean Christopherson } 146fb101293SBen Gardon 147c0e64238SBen Gardon rcu_read_unlock(); 148cfc10997SBen Gardon 149cfc10997SBen Gardon if (prev_root) 1506103bc07SBen Gardon kvm_tdp_mmu_put_root(kvm, prev_root, shared); 151cfc10997SBen Gardon 152a889ea54SBen Gardon return next_root; 153a889ea54SBen Gardon } 154a889ea54SBen Gardon 155a889ea54SBen Gardon /* 156a889ea54SBen Gardon * Note: this iterator gets and puts references to the roots it iterates over. 157a889ea54SBen Gardon * This makes it safe to release the MMU lock and yield within the loop, but 158a889ea54SBen Gardon * if exiting the loop early, the caller must drop the reference to the most 159a889ea54SBen Gardon * recent root. (Unless keeping a live reference is desirable.) 1606103bc07SBen Gardon * 1616103bc07SBen Gardon * If shared is set, this function is operating under the MMU lock in read 1626103bc07SBen Gardon * mode. In the unlikely event that this thread must free a root, the lock 1636103bc07SBen Gardon * will be temporarily dropped and reacquired in write mode. 164a889ea54SBen Gardon */ 165d62007edSSean Christopherson #define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, _only_valid)\ 166d62007edSSean Christopherson for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, _only_valid); \ 167cfc10997SBen Gardon _root; \ 168d62007edSSean Christopherson _root = tdp_mmu_next_root(_kvm, _root, _shared, _only_valid)) \ 169a3f15bdaSSean Christopherson if (kvm_mmu_page_as_id(_root) != _as_id) { \ 170a3f15bdaSSean Christopherson } else 171a889ea54SBen Gardon 172d62007edSSean Christopherson #define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \ 173d62007edSSean Christopherson __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true) 174d62007edSSean Christopherson 175d62007edSSean Christopherson #define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \ 176d62007edSSean Christopherson __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, false) 177d62007edSSean Christopherson 178*226b8c8fSSean Christopherson /* 179*226b8c8fSSean Christopherson * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write, 180*226b8c8fSSean Christopherson * the implication being that any flow that holds mmu_lock for read is 181*226b8c8fSSean Christopherson * inherently yield-friendly and should use the yield-safe variant above. 182*226b8c8fSSean Christopherson * Holding mmu_lock for write obviates the need for RCU protection as the list 183*226b8c8fSSean Christopherson * is guaranteed to be stable. 184*226b8c8fSSean Christopherson */ 185a3f15bdaSSean Christopherson #define for_each_tdp_mmu_root(_kvm, _root, _as_id) \ 186*226b8c8fSSean Christopherson list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link) \ 187*226b8c8fSSean Christopherson if (kvm_lockdep_assert_mmu_lock_held(_kvm, false) && \ 188*226b8c8fSSean Christopherson kvm_mmu_page_as_id(_root) != _as_id) { \ 189a3f15bdaSSean Christopherson } else 19002c00b3aSBen Gardon 191a82070b6SDavid Matlack static struct kvm_mmu_page *tdp_mmu_alloc_sp(struct kvm_vcpu *vcpu) 19202c00b3aSBen Gardon { 19302c00b3aSBen Gardon struct kvm_mmu_page *sp; 19402c00b3aSBen Gardon 19502c00b3aSBen Gardon sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 19602c00b3aSBen Gardon sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 197a82070b6SDavid Matlack 198a82070b6SDavid Matlack return sp; 199a82070b6SDavid Matlack } 200a82070b6SDavid Matlack 201a82070b6SDavid Matlack static void tdp_mmu_init_sp(struct kvm_mmu_page *sp, gfn_t gfn, 202a82070b6SDavid Matlack union kvm_mmu_page_role role) 203a82070b6SDavid Matlack { 20402c00b3aSBen Gardon set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 20502c00b3aSBen Gardon 206a3aca4deSDavid Matlack sp->role = role; 20702c00b3aSBen Gardon sp->gfn = gfn; 20802c00b3aSBen Gardon sp->tdp_mmu_page = true; 20902c00b3aSBen Gardon 21033dd3574SBen Gardon trace_kvm_mmu_get_page(sp, true); 21102c00b3aSBen Gardon } 21202c00b3aSBen Gardon 213a82070b6SDavid Matlack static void tdp_mmu_init_child_sp(struct kvm_mmu_page *child_sp, 214a3aca4deSDavid Matlack struct tdp_iter *iter) 215a3aca4deSDavid Matlack { 216a3aca4deSDavid Matlack struct kvm_mmu_page *parent_sp; 217a3aca4deSDavid Matlack union kvm_mmu_page_role role; 218a3aca4deSDavid Matlack 219a3aca4deSDavid Matlack parent_sp = sptep_to_sp(rcu_dereference(iter->sptep)); 220a3aca4deSDavid Matlack 221a3aca4deSDavid Matlack role = parent_sp->role; 222a3aca4deSDavid Matlack role.level--; 223a3aca4deSDavid Matlack 224a82070b6SDavid Matlack tdp_mmu_init_sp(child_sp, iter->gfn, role); 225a3aca4deSDavid Matlack } 226a3aca4deSDavid Matlack 2276e6ec584SSean Christopherson hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) 22802c00b3aSBen Gardon { 229a3aca4deSDavid Matlack union kvm_mmu_page_role role = vcpu->arch.mmu->mmu_role.base; 23002c00b3aSBen Gardon struct kvm *kvm = vcpu->kvm; 23102c00b3aSBen Gardon struct kvm_mmu_page *root; 23202c00b3aSBen Gardon 2336e6ec584SSean Christopherson lockdep_assert_held_write(&kvm->mmu_lock); 23402c00b3aSBen Gardon 23504dc4e6cSSean Christopherson /* 23604dc4e6cSSean Christopherson * Check for an existing root before allocating a new one. Note, the 23704dc4e6cSSean Christopherson * role check prevents consuming an invalid root. 23804dc4e6cSSean Christopherson */ 239a3f15bdaSSean Christopherson for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) { 240fb101293SBen Gardon if (root->role.word == role.word && 241ad6d6b94SJinrong Liang kvm_tdp_mmu_get_root(root)) 2426e6ec584SSean Christopherson goto out; 24302c00b3aSBen Gardon } 24402c00b3aSBen Gardon 245a82070b6SDavid Matlack root = tdp_mmu_alloc_sp(vcpu); 246a82070b6SDavid Matlack tdp_mmu_init_sp(root, 0, role); 247a82070b6SDavid Matlack 24811cccf5cSBen Gardon refcount_set(&root->tdp_mmu_root_count, 1); 24902c00b3aSBen Gardon 250c0e64238SBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 251c0e64238SBen Gardon list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots); 252c0e64238SBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 25302c00b3aSBen Gardon 2546e6ec584SSean Christopherson out: 25502c00b3aSBen Gardon return __pa(root->spt); 256fe5db27dSBen Gardon } 2572f2fad08SBen Gardon 2582f2fad08SBen Gardon static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 2599a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 2609a77daacSBen Gardon bool shared); 2612f2fad08SBen Gardon 262f8e14497SBen Gardon static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level) 263f8e14497SBen Gardon { 264f8e14497SBen Gardon if (!is_shadow_present_pte(old_spte) || !is_last_spte(old_spte, level)) 265f8e14497SBen Gardon return; 266f8e14497SBen Gardon 267f8e14497SBen Gardon if (is_accessed_spte(old_spte) && 26864bb2769SSean Christopherson (!is_shadow_present_pte(new_spte) || !is_accessed_spte(new_spte) || 26964bb2769SSean Christopherson spte_to_pfn(old_spte) != spte_to_pfn(new_spte))) 270f8e14497SBen Gardon kvm_set_pfn_accessed(spte_to_pfn(old_spte)); 271f8e14497SBen Gardon } 272f8e14497SBen Gardon 273a6a0b05dSBen Gardon static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, 274a6a0b05dSBen Gardon u64 old_spte, u64 new_spte, int level) 275a6a0b05dSBen Gardon { 276a6a0b05dSBen Gardon bool pfn_changed; 277a6a0b05dSBen Gardon struct kvm_memory_slot *slot; 278a6a0b05dSBen Gardon 279a6a0b05dSBen Gardon if (level > PG_LEVEL_4K) 280a6a0b05dSBen Gardon return; 281a6a0b05dSBen Gardon 282a6a0b05dSBen Gardon pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 283a6a0b05dSBen Gardon 284a6a0b05dSBen Gardon if ((!is_writable_pte(old_spte) || pfn_changed) && 285a6a0b05dSBen Gardon is_writable_pte(new_spte)) { 286a6a0b05dSBen Gardon slot = __gfn_to_memslot(__kvm_memslots(kvm, as_id), gfn); 287fb04a1edSPeter Xu mark_page_dirty_in_slot(kvm, slot, gfn); 288a6a0b05dSBen Gardon } 289a6a0b05dSBen Gardon } 290a6a0b05dSBen Gardon 2912f2fad08SBen Gardon /** 292c298a30cSDavid Matlack * tdp_mmu_unlink_sp() - Remove a shadow page from the list of used pages 293a9442f59SBen Gardon * 294a9442f59SBen Gardon * @kvm: kvm instance 295a9442f59SBen Gardon * @sp: the page to be removed 2969a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 2979a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 2989a77daacSBen Gardon * threads that might be adding or removing pages. 299a9442f59SBen Gardon */ 300c298a30cSDavid Matlack static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp, 3019a77daacSBen Gardon bool shared) 302a9442f59SBen Gardon { 3039a77daacSBen Gardon if (shared) 3049a77daacSBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 3059a77daacSBen Gardon else 306a9442f59SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 307a9442f59SBen Gardon 308a9442f59SBen Gardon list_del(&sp->link); 309a9442f59SBen Gardon if (sp->lpage_disallowed) 310a9442f59SBen Gardon unaccount_huge_nx_page(kvm, sp); 3119a77daacSBen Gardon 3129a77daacSBen Gardon if (shared) 3139a77daacSBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 314a9442f59SBen Gardon } 315a9442f59SBen Gardon 316a9442f59SBen Gardon /** 3170f53dfa3SDavid Matlack * handle_removed_pt() - handle a page table removed from the TDP structure 318a066e61fSBen Gardon * 319a066e61fSBen Gardon * @kvm: kvm instance 320a066e61fSBen Gardon * @pt: the page removed from the paging structure 3219a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use 3229a77daacSBen Gardon * of the MMU lock and the operation must synchronize with other 3239a77daacSBen Gardon * threads that might be modifying SPTEs. 324a066e61fSBen Gardon * 325a066e61fSBen Gardon * Given a page table that has been removed from the TDP paging structure, 326a066e61fSBen Gardon * iterates through the page table to clear SPTEs and free child page tables. 32770fb3e41SBen Gardon * 32870fb3e41SBen Gardon * Note that pt is passed in as a tdp_ptep_t, but it does not need RCU 32970fb3e41SBen Gardon * protection. Since this thread removed it from the paging structure, 33070fb3e41SBen Gardon * this thread will be responsible for ensuring the page is freed. Hence the 33170fb3e41SBen Gardon * early rcu_dereferences in the function. 332a066e61fSBen Gardon */ 3330f53dfa3SDavid Matlack static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) 334a066e61fSBen Gardon { 33570fb3e41SBen Gardon struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(pt)); 336a066e61fSBen Gardon int level = sp->role.level; 337e25f0e0cSBen Gardon gfn_t base_gfn = sp->gfn; 338a066e61fSBen Gardon int i; 339a066e61fSBen Gardon 340a066e61fSBen Gardon trace_kvm_mmu_prepare_zap_page(sp); 341a066e61fSBen Gardon 342c298a30cSDavid Matlack tdp_mmu_unlink_sp(kvm, sp, shared); 343a066e61fSBen Gardon 344a066e61fSBen Gardon for (i = 0; i < PT64_ENT_PER_PAGE; i++) { 345574c3c55SBen Gardon u64 *sptep = rcu_dereference(pt) + i; 346574c3c55SBen Gardon gfn_t gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level); 347574c3c55SBen Gardon u64 old_child_spte; 3489a77daacSBen Gardon 3499a77daacSBen Gardon if (shared) { 350e25f0e0cSBen Gardon /* 351e25f0e0cSBen Gardon * Set the SPTE to a nonpresent value that other 352e25f0e0cSBen Gardon * threads will not overwrite. If the SPTE was 353e25f0e0cSBen Gardon * already marked as removed then another thread 354e25f0e0cSBen Gardon * handling a page fault could overwrite it, so 355e25f0e0cSBen Gardon * set the SPTE until it is set from some other 356e25f0e0cSBen Gardon * value to the removed SPTE value. 357e25f0e0cSBen Gardon */ 358e25f0e0cSBen Gardon for (;;) { 359e25f0e0cSBen Gardon old_child_spte = xchg(sptep, REMOVED_SPTE); 360e25f0e0cSBen Gardon if (!is_removed_spte(old_child_spte)) 361e25f0e0cSBen Gardon break; 362e25f0e0cSBen Gardon cpu_relax(); 363e25f0e0cSBen Gardon } 3649a77daacSBen Gardon } else { 3658df9f1afSSean Christopherson /* 3668df9f1afSSean Christopherson * If the SPTE is not MMU-present, there is no backing 3678df9f1afSSean Christopherson * page associated with the SPTE and so no side effects 3688df9f1afSSean Christopherson * that need to be recorded, and exclusive ownership of 3698df9f1afSSean Christopherson * mmu_lock ensures the SPTE can't be made present. 3708df9f1afSSean Christopherson * Note, zapping MMIO SPTEs is also unnecessary as they 3718df9f1afSSean Christopherson * are guarded by the memslots generation, not by being 3728df9f1afSSean Christopherson * unreachable. 3738df9f1afSSean Christopherson */ 3749a77daacSBen Gardon old_child_spte = READ_ONCE(*sptep); 3758df9f1afSSean Christopherson if (!is_shadow_present_pte(old_child_spte)) 3768df9f1afSSean Christopherson continue; 377e25f0e0cSBen Gardon 378e25f0e0cSBen Gardon /* 379e25f0e0cSBen Gardon * Marking the SPTE as a removed SPTE is not 380e25f0e0cSBen Gardon * strictly necessary here as the MMU lock will 381e25f0e0cSBen Gardon * stop other threads from concurrently modifying 382e25f0e0cSBen Gardon * this SPTE. Using the removed SPTE value keeps 383e25f0e0cSBen Gardon * the two branches consistent and simplifies 384e25f0e0cSBen Gardon * the function. 385e25f0e0cSBen Gardon */ 386e25f0e0cSBen Gardon WRITE_ONCE(*sptep, REMOVED_SPTE); 3879a77daacSBen Gardon } 388e25f0e0cSBen Gardon handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn, 389f1b83255SKai Huang old_child_spte, REMOVED_SPTE, level, 390e25f0e0cSBen Gardon shared); 391a066e61fSBen Gardon } 392a066e61fSBen Gardon 393574c3c55SBen Gardon kvm_flush_remote_tlbs_with_address(kvm, base_gfn, 394f1b83255SKai Huang KVM_PAGES_PER_HPAGE(level + 1)); 395a066e61fSBen Gardon 3967cca2d0bSBen Gardon call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); 397a066e61fSBen Gardon } 398a066e61fSBen Gardon 399a066e61fSBen Gardon /** 4007f6231a3SKai Huang * __handle_changed_spte - handle bookkeeping associated with an SPTE change 4012f2fad08SBen Gardon * @kvm: kvm instance 4022f2fad08SBen Gardon * @as_id: the address space of the paging structure the SPTE was a part of 4032f2fad08SBen Gardon * @gfn: the base GFN that was mapped by the SPTE 4042f2fad08SBen Gardon * @old_spte: The value of the SPTE before the change 4052f2fad08SBen Gardon * @new_spte: The value of the SPTE after the change 4062f2fad08SBen Gardon * @level: the level of the PT the SPTE is part of in the paging structure 4079a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 4089a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 4099a77daacSBen Gardon * threads that might be modifying SPTEs. 4102f2fad08SBen Gardon * 4112f2fad08SBen Gardon * Handle bookkeeping that might result from the modification of a SPTE. 4122f2fad08SBen Gardon * This function must be called for all TDP SPTE modifications. 4132f2fad08SBen Gardon */ 4142f2fad08SBen Gardon static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 4159a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 4169a77daacSBen Gardon bool shared) 4172f2fad08SBen Gardon { 4182f2fad08SBen Gardon bool was_present = is_shadow_present_pte(old_spte); 4192f2fad08SBen Gardon bool is_present = is_shadow_present_pte(new_spte); 4202f2fad08SBen Gardon bool was_leaf = was_present && is_last_spte(old_spte, level); 4212f2fad08SBen Gardon bool is_leaf = is_present && is_last_spte(new_spte, level); 4222f2fad08SBen Gardon bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 4232f2fad08SBen Gardon 4242f2fad08SBen Gardon WARN_ON(level > PT64_ROOT_MAX_LEVEL); 4252f2fad08SBen Gardon WARN_ON(level < PG_LEVEL_4K); 426764388ceSSean Christopherson WARN_ON(gfn & (KVM_PAGES_PER_HPAGE(level) - 1)); 4272f2fad08SBen Gardon 4282f2fad08SBen Gardon /* 4292f2fad08SBen Gardon * If this warning were to trigger it would indicate that there was a 4302f2fad08SBen Gardon * missing MMU notifier or a race with some notifier handler. 4312f2fad08SBen Gardon * A present, leaf SPTE should never be directly replaced with another 432d9f6e12fSIngo Molnar * present leaf SPTE pointing to a different PFN. A notifier handler 4332f2fad08SBen Gardon * should be zapping the SPTE before the main MM's page table is 4342f2fad08SBen Gardon * changed, or the SPTE should be zeroed, and the TLBs flushed by the 4352f2fad08SBen Gardon * thread before replacement. 4362f2fad08SBen Gardon */ 4372f2fad08SBen Gardon if (was_leaf && is_leaf && pfn_changed) { 4382f2fad08SBen Gardon pr_err("Invalid SPTE change: cannot replace a present leaf\n" 4392f2fad08SBen Gardon "SPTE with another present leaf SPTE mapping a\n" 4402f2fad08SBen Gardon "different PFN!\n" 4412f2fad08SBen Gardon "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 4422f2fad08SBen Gardon as_id, gfn, old_spte, new_spte, level); 4432f2fad08SBen Gardon 4442f2fad08SBen Gardon /* 4452f2fad08SBen Gardon * Crash the host to prevent error propagation and guest data 446d9f6e12fSIngo Molnar * corruption. 4472f2fad08SBen Gardon */ 4482f2fad08SBen Gardon BUG(); 4492f2fad08SBen Gardon } 4502f2fad08SBen Gardon 4512f2fad08SBen Gardon if (old_spte == new_spte) 4522f2fad08SBen Gardon return; 4532f2fad08SBen Gardon 454b9a98c34SBen Gardon trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); 455b9a98c34SBen Gardon 456115111efSDavid Matlack if (is_leaf) 457115111efSDavid Matlack check_spte_writable_invariants(new_spte); 458115111efSDavid Matlack 4592f2fad08SBen Gardon /* 4602f2fad08SBen Gardon * The only times a SPTE should be changed from a non-present to 4612f2fad08SBen Gardon * non-present state is when an MMIO entry is installed/modified/ 4622f2fad08SBen Gardon * removed. In that case, there is nothing to do here. 4632f2fad08SBen Gardon */ 4642f2fad08SBen Gardon if (!was_present && !is_present) { 4652f2fad08SBen Gardon /* 46608f07c80SBen Gardon * If this change does not involve a MMIO SPTE or removed SPTE, 46708f07c80SBen Gardon * it is unexpected. Log the change, though it should not 46808f07c80SBen Gardon * impact the guest since both the former and current SPTEs 46908f07c80SBen Gardon * are nonpresent. 4702f2fad08SBen Gardon */ 47108f07c80SBen Gardon if (WARN_ON(!is_mmio_spte(old_spte) && 47208f07c80SBen Gardon !is_mmio_spte(new_spte) && 47308f07c80SBen Gardon !is_removed_spte(new_spte))) 4742f2fad08SBen Gardon pr_err("Unexpected SPTE change! Nonpresent SPTEs\n" 4752f2fad08SBen Gardon "should not be replaced with another,\n" 4762f2fad08SBen Gardon "different nonpresent SPTE, unless one or both\n" 47708f07c80SBen Gardon "are MMIO SPTEs, or the new SPTE is\n" 47808f07c80SBen Gardon "a temporary removed SPTE.\n" 4792f2fad08SBen Gardon "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 4802f2fad08SBen Gardon as_id, gfn, old_spte, new_spte, level); 4812f2fad08SBen Gardon return; 4822f2fad08SBen Gardon } 4832f2fad08SBen Gardon 48471f51d2cSMingwei Zhang if (is_leaf != was_leaf) 48571f51d2cSMingwei Zhang kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1); 4862f2fad08SBen Gardon 4872f2fad08SBen Gardon if (was_leaf && is_dirty_spte(old_spte) && 48864bb2769SSean Christopherson (!is_present || !is_dirty_spte(new_spte) || pfn_changed)) 4892f2fad08SBen Gardon kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 4902f2fad08SBen Gardon 4912f2fad08SBen Gardon /* 4922f2fad08SBen Gardon * Recursively handle child PTs if the change removed a subtree from 4932f2fad08SBen Gardon * the paging structure. 4942f2fad08SBen Gardon */ 495a066e61fSBen Gardon if (was_present && !was_leaf && (pfn_changed || !is_present)) 4960f53dfa3SDavid Matlack handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared); 4972f2fad08SBen Gardon } 4982f2fad08SBen Gardon 4992f2fad08SBen Gardon static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 5009a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 5019a77daacSBen Gardon bool shared) 5022f2fad08SBen Gardon { 5039a77daacSBen Gardon __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, 5049a77daacSBen Gardon shared); 505f8e14497SBen Gardon handle_changed_spte_acc_track(old_spte, new_spte, level); 506a6a0b05dSBen Gardon handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte, 507a6a0b05dSBen Gardon new_spte, level); 5082f2fad08SBen Gardon } 509faaf05b0SBen Gardon 510fe43fa2fSBen Gardon /* 5116ccf4438SPaolo Bonzini * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically 5126ccf4438SPaolo Bonzini * and handle the associated bookkeeping. Do not mark the page dirty 51324ae4cfaSBen Gardon * in KVM's dirty bitmaps. 5149a77daacSBen Gardon * 5153255530aSDavid Matlack * If setting the SPTE fails because it has changed, iter->old_spte will be 5163255530aSDavid Matlack * refreshed to the current value of the spte. 5173255530aSDavid Matlack * 5189a77daacSBen Gardon * @kvm: kvm instance 5199a77daacSBen Gardon * @iter: a tdp_iter instance currently on the SPTE that should be set 5209a77daacSBen Gardon * @new_spte: The value the SPTE should be set to 5213e72c791SDavid Matlack * Return: 5223e72c791SDavid Matlack * * 0 - If the SPTE was set. 5233e72c791SDavid Matlack * * -EBUSY - If the SPTE cannot be set. In this case this function will have 5243e72c791SDavid Matlack * no side-effects other than setting iter->old_spte to the last 5253e72c791SDavid Matlack * known value of the spte. 5269a77daacSBen Gardon */ 5273e72c791SDavid Matlack static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm, 5289a77daacSBen Gardon struct tdp_iter *iter, 5299a77daacSBen Gardon u64 new_spte) 5309a77daacSBen Gardon { 5313255530aSDavid Matlack u64 *sptep = rcu_dereference(iter->sptep); 5323255530aSDavid Matlack u64 old_spte; 5333255530aSDavid Matlack 5343a0f64deSSean Christopherson WARN_ON_ONCE(iter->yielded); 5353a0f64deSSean Christopherson 5369a77daacSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 5379a77daacSBen Gardon 53808f07c80SBen Gardon /* 53908f07c80SBen Gardon * Do not change removed SPTEs. Only the thread that froze the SPTE 54008f07c80SBen Gardon * may modify it. 54108f07c80SBen Gardon */ 5427a51393aSSean Christopherson if (is_removed_spte(iter->old_spte)) 5433e72c791SDavid Matlack return -EBUSY; 54408f07c80SBen Gardon 5456e8eb206SDavid Matlack /* 5466e8eb206SDavid Matlack * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and 5476e8eb206SDavid Matlack * does not hold the mmu_lock. 5486e8eb206SDavid Matlack */ 5493255530aSDavid Matlack old_spte = cmpxchg64(sptep, iter->old_spte, new_spte); 5503255530aSDavid Matlack if (old_spte != iter->old_spte) { 5513255530aSDavid Matlack /* 5523255530aSDavid Matlack * The page table entry was modified by a different logical 5533255530aSDavid Matlack * CPU. Refresh iter->old_spte with the current value so the 5543255530aSDavid Matlack * caller operates on fresh data, e.g. if it retries 5553255530aSDavid Matlack * tdp_mmu_set_spte_atomic(). 5563255530aSDavid Matlack */ 5573255530aSDavid Matlack iter->old_spte = old_spte; 5583e72c791SDavid Matlack return -EBUSY; 5593255530aSDavid Matlack } 5609a77daacSBen Gardon 56124ae4cfaSBen Gardon __handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte, 56208889894SSean Christopherson new_spte, iter->level, true); 56324ae4cfaSBen Gardon handle_changed_spte_acc_track(iter->old_spte, new_spte, iter->level); 5649a77daacSBen Gardon 5653e72c791SDavid Matlack return 0; 5669a77daacSBen Gardon } 5679a77daacSBen Gardon 5683e72c791SDavid Matlack static inline int tdp_mmu_zap_spte_atomic(struct kvm *kvm, 56908f07c80SBen Gardon struct tdp_iter *iter) 57008f07c80SBen Gardon { 5713e72c791SDavid Matlack int ret; 5723e72c791SDavid Matlack 57308f07c80SBen Gardon /* 57408f07c80SBen Gardon * Freeze the SPTE by setting it to a special, 57508f07c80SBen Gardon * non-present value. This will stop other threads from 57608f07c80SBen Gardon * immediately installing a present entry in its place 57708f07c80SBen Gardon * before the TLBs are flushed. 57808f07c80SBen Gardon */ 5793e72c791SDavid Matlack ret = tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE); 5803e72c791SDavid Matlack if (ret) 5813e72c791SDavid Matlack return ret; 58208f07c80SBen Gardon 58308f07c80SBen Gardon kvm_flush_remote_tlbs_with_address(kvm, iter->gfn, 58408f07c80SBen Gardon KVM_PAGES_PER_HPAGE(iter->level)); 58508f07c80SBen Gardon 58608f07c80SBen Gardon /* 58708f07c80SBen Gardon * No other thread can overwrite the removed SPTE as they 58808f07c80SBen Gardon * must either wait on the MMU lock or use 589d9f6e12fSIngo Molnar * tdp_mmu_set_spte_atomic which will not overwrite the 59008f07c80SBen Gardon * special removed SPTE value. No bookkeeping is needed 59108f07c80SBen Gardon * here since the SPTE is going from non-present 59208f07c80SBen Gardon * to non-present. 59308f07c80SBen Gardon */ 59414f6fec2SBen Gardon WRITE_ONCE(*rcu_dereference(iter->sptep), 0); 59508f07c80SBen Gardon 5963e72c791SDavid Matlack return 0; 59708f07c80SBen Gardon } 59808f07c80SBen Gardon 5999a77daacSBen Gardon 6009a77daacSBen Gardon /* 601fe43fa2fSBen Gardon * __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping 602fe43fa2fSBen Gardon * @kvm: kvm instance 603fe43fa2fSBen Gardon * @iter: a tdp_iter instance currently on the SPTE that should be set 604fe43fa2fSBen Gardon * @new_spte: The value the SPTE should be set to 605fe43fa2fSBen Gardon * @record_acc_track: Notify the MM subsystem of changes to the accessed state 606fe43fa2fSBen Gardon * of the page. Should be set unless handling an MMU 607fe43fa2fSBen Gardon * notifier for access tracking. Leaving record_acc_track 608fe43fa2fSBen Gardon * unset in that case prevents page accesses from being 609fe43fa2fSBen Gardon * double counted. 610fe43fa2fSBen Gardon * @record_dirty_log: Record the page as dirty in the dirty bitmap if 611fe43fa2fSBen Gardon * appropriate for the change being made. Should be set 612fe43fa2fSBen Gardon * unless performing certain dirty logging operations. 613fe43fa2fSBen Gardon * Leaving record_dirty_log unset in that case prevents page 614fe43fa2fSBen Gardon * writes from being double counted. 615fe43fa2fSBen Gardon */ 616f8e14497SBen Gardon static inline void __tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, 617a6a0b05dSBen Gardon u64 new_spte, bool record_acc_track, 618a6a0b05dSBen Gardon bool record_dirty_log) 619faaf05b0SBen Gardon { 6203a0f64deSSean Christopherson WARN_ON_ONCE(iter->yielded); 6213a0f64deSSean Christopherson 622531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 6233a9a4aa5SBen Gardon 62408f07c80SBen Gardon /* 62508f07c80SBen Gardon * No thread should be using this function to set SPTEs to the 62608f07c80SBen Gardon * temporary removed SPTE value. 62708f07c80SBen Gardon * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic 62808f07c80SBen Gardon * should be used. If operating under the MMU lock in write mode, the 62908f07c80SBen Gardon * use of the removed SPTE should not be necessary. 63008f07c80SBen Gardon */ 6317a51393aSSean Christopherson WARN_ON(is_removed_spte(iter->old_spte)); 63208f07c80SBen Gardon 6337cca2d0bSBen Gardon WRITE_ONCE(*rcu_dereference(iter->sptep), new_spte); 634faaf05b0SBen Gardon 63508889894SSean Christopherson __handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte, 63608889894SSean Christopherson new_spte, iter->level, false); 637f8e14497SBen Gardon if (record_acc_track) 638f8e14497SBen Gardon handle_changed_spte_acc_track(iter->old_spte, new_spte, 639f8e14497SBen Gardon iter->level); 640a6a0b05dSBen Gardon if (record_dirty_log) 64108889894SSean Christopherson handle_changed_spte_dirty_log(kvm, iter->as_id, iter->gfn, 642a6a0b05dSBen Gardon iter->old_spte, new_spte, 643a6a0b05dSBen Gardon iter->level); 644f8e14497SBen Gardon } 645f8e14497SBen Gardon 646f8e14497SBen Gardon static inline void tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, 647f8e14497SBen Gardon u64 new_spte) 648f8e14497SBen Gardon { 649a6a0b05dSBen Gardon __tdp_mmu_set_spte(kvm, iter, new_spte, true, true); 650f8e14497SBen Gardon } 651f8e14497SBen Gardon 652f8e14497SBen Gardon static inline void tdp_mmu_set_spte_no_acc_track(struct kvm *kvm, 653f8e14497SBen Gardon struct tdp_iter *iter, 654f8e14497SBen Gardon u64 new_spte) 655f8e14497SBen Gardon { 656a6a0b05dSBen Gardon __tdp_mmu_set_spte(kvm, iter, new_spte, false, true); 657a6a0b05dSBen Gardon } 658a6a0b05dSBen Gardon 659a6a0b05dSBen Gardon static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm, 660a6a0b05dSBen Gardon struct tdp_iter *iter, 661a6a0b05dSBen Gardon u64 new_spte) 662a6a0b05dSBen Gardon { 663a6a0b05dSBen Gardon __tdp_mmu_set_spte(kvm, iter, new_spte, true, false); 664faaf05b0SBen Gardon } 665faaf05b0SBen Gardon 666faaf05b0SBen Gardon #define tdp_root_for_each_pte(_iter, _root, _start, _end) \ 66777aa6075SDavid Matlack for_each_tdp_pte(_iter, _root, _start, _end) 668faaf05b0SBen Gardon 669f8e14497SBen Gardon #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \ 670f8e14497SBen Gardon tdp_root_for_each_pte(_iter, _root, _start, _end) \ 671f8e14497SBen Gardon if (!is_shadow_present_pte(_iter.old_spte) || \ 672f8e14497SBen Gardon !is_last_spte(_iter.old_spte, _iter.level)) \ 673f8e14497SBen Gardon continue; \ 674f8e14497SBen Gardon else 675f8e14497SBen Gardon 676bb18842eSBen Gardon #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \ 677b9e5603cSPaolo Bonzini for_each_tdp_pte(_iter, to_shadow_page(_mmu->root.hpa), _start, _end) 678bb18842eSBen Gardon 679faaf05b0SBen Gardon /* 680e28a436cSBen Gardon * Yield if the MMU lock is contended or this thread needs to return control 681e28a436cSBen Gardon * to the scheduler. 682e28a436cSBen Gardon * 683e139a34eSBen Gardon * If this function should yield and flush is set, it will perform a remote 684e139a34eSBen Gardon * TLB flush before yielding. 685e139a34eSBen Gardon * 6863a0f64deSSean Christopherson * If this function yields, iter->yielded is set and the caller must skip to 6873a0f64deSSean Christopherson * the next iteration, where tdp_iter_next() will reset the tdp_iter's walk 6883a0f64deSSean Christopherson * over the paging structures to allow the iterator to continue its traversal 6893a0f64deSSean Christopherson * from the paging structure root. 690e28a436cSBen Gardon * 6913a0f64deSSean Christopherson * Returns true if this function yielded. 692e28a436cSBen Gardon */ 6933a0f64deSSean Christopherson static inline bool __must_check tdp_mmu_iter_cond_resched(struct kvm *kvm, 6943a0f64deSSean Christopherson struct tdp_iter *iter, 6953a0f64deSSean Christopherson bool flush, bool shared) 696a6a0b05dSBen Gardon { 6973a0f64deSSean Christopherson WARN_ON(iter->yielded); 6983a0f64deSSean Christopherson 699ed5e484bSBen Gardon /* Ensure forward progress has been made before yielding. */ 700ed5e484bSBen Gardon if (iter->next_last_level_gfn == iter->yielded_gfn) 701ed5e484bSBen Gardon return false; 702ed5e484bSBen Gardon 703531810caSBen Gardon if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { 7047cca2d0bSBen Gardon rcu_read_unlock(); 7057cca2d0bSBen Gardon 706e139a34eSBen Gardon if (flush) 707e139a34eSBen Gardon kvm_flush_remote_tlbs(kvm); 708e139a34eSBen Gardon 7096103bc07SBen Gardon if (shared) 7106103bc07SBen Gardon cond_resched_rwlock_read(&kvm->mmu_lock); 7116103bc07SBen Gardon else 712531810caSBen Gardon cond_resched_rwlock_write(&kvm->mmu_lock); 7136103bc07SBen Gardon 7147cca2d0bSBen Gardon rcu_read_lock(); 715ed5e484bSBen Gardon 716ed5e484bSBen Gardon WARN_ON(iter->gfn > iter->next_last_level_gfn); 717ed5e484bSBen Gardon 7183a0f64deSSean Christopherson iter->yielded = true; 719a6a0b05dSBen Gardon } 720e28a436cSBen Gardon 7213a0f64deSSean Christopherson return iter->yielded; 722a6a0b05dSBen Gardon } 723a6a0b05dSBen Gardon 724faaf05b0SBen Gardon /* 725faaf05b0SBen Gardon * Tears down the mappings for the range of gfns, [start, end), and frees the 726faaf05b0SBen Gardon * non-root pages mapping GFNs strictly within that range. Returns true if 727faaf05b0SBen Gardon * SPTEs have been cleared and a TLB flush is needed before releasing the 728faaf05b0SBen Gardon * MMU lock. 7296103bc07SBen Gardon * 730063afacdSBen Gardon * If can_yield is true, will release the MMU lock and reschedule if the 731063afacdSBen Gardon * scheduler needs the CPU or there is contention on the MMU lock. If this 732063afacdSBen Gardon * function cannot yield, it will not release the MMU lock or reschedule and 733063afacdSBen Gardon * the caller must ensure it does not supply too large a GFN range, or the 7346103bc07SBen Gardon * operation can cause a soft lockup. 7356103bc07SBen Gardon * 7366103bc07SBen Gardon * If shared is true, this thread holds the MMU lock in read mode and must 7376103bc07SBen Gardon * account for the possibility that other threads are modifying the paging 7386103bc07SBen Gardon * structures concurrently. If shared is false, this thread should hold the 7396103bc07SBen Gardon * MMU lock in write mode. 740faaf05b0SBen Gardon */ 741faaf05b0SBen Gardon static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 7426103bc07SBen Gardon gfn_t start, gfn_t end, bool can_yield, bool flush, 7436103bc07SBen Gardon bool shared) 744faaf05b0SBen Gardon { 745524a1e4eSSean Christopherson gfn_t max_gfn_host = 1ULL << (shadow_phys_bits - PAGE_SHIFT); 746524a1e4eSSean Christopherson bool zap_all = (start == 0 && end >= max_gfn_host); 747faaf05b0SBen Gardon struct tdp_iter iter; 748faaf05b0SBen Gardon 749524a1e4eSSean Christopherson /* 7500103098fSSean Christopherson * No need to try to step down in the iterator when zapping all SPTEs, 7510103098fSSean Christopherson * zapping the top-level non-leaf SPTEs will recurse on their children. 7520103098fSSean Christopherson */ 7530103098fSSean Christopherson int min_level = zap_all ? root->role.level : PG_LEVEL_4K; 7540103098fSSean Christopherson 7550103098fSSean Christopherson /* 756524a1e4eSSean Christopherson * Bound the walk at host.MAXPHYADDR, guest accesses beyond that will 757524a1e4eSSean Christopherson * hit a #PF(RSVD) and never get to an EPT Violation/Misconfig / #NPF, 758524a1e4eSSean Christopherson * and so KVM will never install a SPTE for such addresses. 759524a1e4eSSean Christopherson */ 760524a1e4eSSean Christopherson end = min(end, max_gfn_host); 761524a1e4eSSean Christopherson 7626103bc07SBen Gardon kvm_lockdep_assert_mmu_lock_held(kvm, shared); 7636103bc07SBen Gardon 7647cca2d0bSBen Gardon rcu_read_lock(); 7657cca2d0bSBen Gardon 76677aa6075SDavid Matlack for_each_tdp_pte_min_level(iter, root, min_level, start, end) { 7676103bc07SBen Gardon retry: 7681af4a960SBen Gardon if (can_yield && 7696103bc07SBen Gardon tdp_mmu_iter_cond_resched(kvm, &iter, flush, shared)) { 770a835429cSSean Christopherson flush = false; 7711af4a960SBen Gardon continue; 7721af4a960SBen Gardon } 7731af4a960SBen Gardon 774faaf05b0SBen Gardon if (!is_shadow_present_pte(iter.old_spte)) 775faaf05b0SBen Gardon continue; 776faaf05b0SBen Gardon 777faaf05b0SBen Gardon /* 778faaf05b0SBen Gardon * If this is a non-last-level SPTE that covers a larger range 779faaf05b0SBen Gardon * than should be zapped, continue, and zap the mappings at a 780524a1e4eSSean Christopherson * lower level, except when zapping all SPTEs. 781faaf05b0SBen Gardon */ 782524a1e4eSSean Christopherson if (!zap_all && 783524a1e4eSSean Christopherson (iter.gfn < start || 784faaf05b0SBen Gardon iter.gfn + KVM_PAGES_PER_HPAGE(iter.level) > end) && 785faaf05b0SBen Gardon !is_last_spte(iter.old_spte, iter.level)) 786faaf05b0SBen Gardon continue; 787faaf05b0SBen Gardon 7886103bc07SBen Gardon if (!shared) { 789faaf05b0SBen Gardon tdp_mmu_set_spte(kvm, &iter, 0); 790a835429cSSean Christopherson flush = true; 7913e72c791SDavid Matlack } else if (tdp_mmu_zap_spte_atomic(kvm, &iter)) { 7926103bc07SBen Gardon goto retry; 7936103bc07SBen Gardon } 794faaf05b0SBen Gardon } 7957cca2d0bSBen Gardon 7967cca2d0bSBen Gardon rcu_read_unlock(); 797a835429cSSean Christopherson return flush; 798faaf05b0SBen Gardon } 799faaf05b0SBen Gardon 800faaf05b0SBen Gardon /* 801faaf05b0SBen Gardon * Tears down the mappings for the range of gfns, [start, end), and frees the 802faaf05b0SBen Gardon * non-root pages mapping GFNs strictly within that range. Returns true if 803faaf05b0SBen Gardon * SPTEs have been cleared and a TLB flush is needed before releasing the 804faaf05b0SBen Gardon * MMU lock. 805faaf05b0SBen Gardon */ 8062b9663d8SSean Christopherson bool __kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, gfn_t start, 8075a324c24SSean Christopherson gfn_t end, bool can_yield, bool flush) 808faaf05b0SBen Gardon { 809faaf05b0SBen Gardon struct kvm_mmu_page *root; 810faaf05b0SBen Gardon 8115a324c24SSean Christopherson for_each_tdp_mmu_root_yield_safe(kvm, root, as_id, false) 8126103bc07SBen Gardon flush = zap_gfn_range(kvm, root, start, end, can_yield, flush, 8135a324c24SSean Christopherson false); 814faaf05b0SBen Gardon 815faaf05b0SBen Gardon return flush; 816faaf05b0SBen Gardon } 817faaf05b0SBen Gardon 818faaf05b0SBen Gardon void kvm_tdp_mmu_zap_all(struct kvm *kvm) 819faaf05b0SBen Gardon { 8202b9663d8SSean Christopherson bool flush = false; 8212b9663d8SSean Christopherson int i; 822faaf05b0SBen Gardon 8232b9663d8SSean Christopherson for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 8245a324c24SSean Christopherson flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, 0, -1ull, flush); 8252b9663d8SSean Christopherson 826faaf05b0SBen Gardon if (flush) 827faaf05b0SBen Gardon kvm_flush_remote_tlbs(kvm); 828faaf05b0SBen Gardon } 829bb18842eSBen Gardon 8304c6654bdSBen Gardon static struct kvm_mmu_page *next_invalidated_root(struct kvm *kvm, 8314c6654bdSBen Gardon struct kvm_mmu_page *prev_root) 8324c6654bdSBen Gardon { 8334c6654bdSBen Gardon struct kvm_mmu_page *next_root; 8344c6654bdSBen Gardon 8354c6654bdSBen Gardon if (prev_root) 8364c6654bdSBen Gardon next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 8374c6654bdSBen Gardon &prev_root->link, 8384c6654bdSBen Gardon typeof(*prev_root), link); 8394c6654bdSBen Gardon else 8404c6654bdSBen Gardon next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots, 8414c6654bdSBen Gardon typeof(*next_root), link); 8424c6654bdSBen Gardon 8434c6654bdSBen Gardon while (next_root && !(next_root->role.invalid && 8444c6654bdSBen Gardon refcount_read(&next_root->tdp_mmu_root_count))) 8454c6654bdSBen Gardon next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 8464c6654bdSBen Gardon &next_root->link, 8474c6654bdSBen Gardon typeof(*next_root), link); 8484c6654bdSBen Gardon 8494c6654bdSBen Gardon return next_root; 8504c6654bdSBen Gardon } 8514c6654bdSBen Gardon 8524c6654bdSBen Gardon /* 853f28e9c7fSSean Christopherson * Zap all invalidated roots to ensure all SPTEs are dropped before the "fast 854f28e9c7fSSean Christopherson * zap" completes. Since kvm_tdp_mmu_invalidate_all_roots() has acquired a 855f28e9c7fSSean Christopherson * reference to each invalidated root, roots will not be freed until after this 856f28e9c7fSSean Christopherson * function drops the gifted reference, e.g. so that vCPUs don't get stuck with 857f28e9c7fSSean Christopherson * tearing down paging structures. 8584c6654bdSBen Gardon */ 8594c6654bdSBen Gardon void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) 8604c6654bdSBen Gardon { 8614c6654bdSBen Gardon struct kvm_mmu_page *next_root; 8624c6654bdSBen Gardon struct kvm_mmu_page *root; 8634c6654bdSBen Gardon 8644c6654bdSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 8654c6654bdSBen Gardon 8664c6654bdSBen Gardon rcu_read_lock(); 8674c6654bdSBen Gardon 8684c6654bdSBen Gardon root = next_invalidated_root(kvm, NULL); 8694c6654bdSBen Gardon 8704c6654bdSBen Gardon while (root) { 8714c6654bdSBen Gardon next_root = next_invalidated_root(kvm, root); 8724c6654bdSBen Gardon 8734c6654bdSBen Gardon rcu_read_unlock(); 8744c6654bdSBen Gardon 8757ae5840eSSean Christopherson /* 8767ae5840eSSean Christopherson * A TLB flush is unnecessary, invalidated roots are guaranteed 8777ae5840eSSean Christopherson * to be unreachable by the guest (see kvm_tdp_mmu_put_root() 8787ae5840eSSean Christopherson * for more details), and unlike the legacy MMU, no vCPU kick 8797ae5840eSSean Christopherson * is needed to play nice with lockless shadow walks as the TDP 8807ae5840eSSean Christopherson * MMU protects its paging structures via RCU. Note, zapping 8817ae5840eSSean Christopherson * will still flush on yield, but that's a minor performance 8827ae5840eSSean Christopherson * blip and not a functional issue. 8837ae5840eSSean Christopherson */ 8847ae5840eSSean Christopherson (void)zap_gfn_range(kvm, root, 0, -1ull, true, false, true); 8854c6654bdSBen Gardon 8864c6654bdSBen Gardon /* 8874c6654bdSBen Gardon * Put the reference acquired in 8884c6654bdSBen Gardon * kvm_tdp_mmu_invalidate_roots 8894c6654bdSBen Gardon */ 8904c6654bdSBen Gardon kvm_tdp_mmu_put_root(kvm, root, true); 8914c6654bdSBen Gardon 8924c6654bdSBen Gardon root = next_root; 8934c6654bdSBen Gardon 8944c6654bdSBen Gardon rcu_read_lock(); 8954c6654bdSBen Gardon } 8964c6654bdSBen Gardon 8974c6654bdSBen Gardon rcu_read_unlock(); 8984c6654bdSBen Gardon } 8994c6654bdSBen Gardon 900bb18842eSBen Gardon /* 901f28e9c7fSSean Christopherson * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that 902f28e9c7fSSean Christopherson * is about to be zapped, e.g. in response to a memslots update. The caller is 903f28e9c7fSSean Christopherson * responsible for invoking kvm_tdp_mmu_zap_invalidated_roots() to do the actual 904f28e9c7fSSean Christopherson * zapping. 905b7cccd39SBen Gardon * 906f28e9c7fSSean Christopherson * Take a reference on all roots to prevent the root from being freed before it 907f28e9c7fSSean Christopherson * is zapped by this thread. Freeing a root is not a correctness issue, but if 908f28e9c7fSSean Christopherson * a vCPU drops the last reference to a root prior to the root being zapped, it 909f28e9c7fSSean Christopherson * will get stuck with tearing down the entire paging structure. 9104c6654bdSBen Gardon * 911f28e9c7fSSean Christopherson * Get a reference even if the root is already invalid, 912f28e9c7fSSean Christopherson * kvm_tdp_mmu_zap_invalidated_roots() assumes it was gifted a reference to all 913f28e9c7fSSean Christopherson * invalid roots, e.g. there's no epoch to identify roots that were invalidated 914f28e9c7fSSean Christopherson * by a previous call. Roots stay on the list until the last reference is 915f28e9c7fSSean Christopherson * dropped, so even though all invalid roots are zapped, a root may not go away 916f28e9c7fSSean Christopherson * for quite some time, e.g. if a vCPU blocks across multiple memslot updates. 917f28e9c7fSSean Christopherson * 918f28e9c7fSSean Christopherson * Because mmu_lock is held for write, it should be impossible to observe a 919f28e9c7fSSean Christopherson * root with zero refcount, i.e. the list of roots cannot be stale. 9204c6654bdSBen Gardon * 921b7cccd39SBen Gardon * This has essentially the same effect for the TDP MMU 922b7cccd39SBen Gardon * as updating mmu_valid_gen does for the shadow MMU. 923b7cccd39SBen Gardon */ 924b7cccd39SBen Gardon void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm) 925b7cccd39SBen Gardon { 926b7cccd39SBen Gardon struct kvm_mmu_page *root; 927b7cccd39SBen Gardon 928b7cccd39SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 929f28e9c7fSSean Christopherson list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) { 930f28e9c7fSSean Christopherson if (!WARN_ON_ONCE(!kvm_tdp_mmu_get_root(root))) 931b7cccd39SBen Gardon root->role.invalid = true; 932b7cccd39SBen Gardon } 933f28e9c7fSSean Christopherson } 934b7cccd39SBen Gardon 935bb18842eSBen Gardon /* 936bb18842eSBen Gardon * Installs a last-level SPTE to handle a TDP page fault. 937bb18842eSBen Gardon * (NPT/EPT violation/misconfiguration) 938bb18842eSBen Gardon */ 939cdc47767SPaolo Bonzini static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, 940cdc47767SPaolo Bonzini struct kvm_page_fault *fault, 941cdc47767SPaolo Bonzini struct tdp_iter *iter) 942bb18842eSBen Gardon { 943c435d4b7SSean Christopherson struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep)); 944bb18842eSBen Gardon u64 new_spte; 94557a3e96dSKai Huang int ret = RET_PF_FIXED; 946ad67e480SPaolo Bonzini bool wrprot = false; 947bb18842eSBen Gardon 9487158bee4SPaolo Bonzini WARN_ON(sp->role.level != fault->goal_level); 949e710c5f6SDavid Matlack if (unlikely(!fault->slot)) 950bb18842eSBen Gardon new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL); 9519a77daacSBen Gardon else 95253597858SDavid Matlack wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn, 9532839180cSPaolo Bonzini fault->pfn, iter->old_spte, fault->prefetch, true, 9547158bee4SPaolo Bonzini fault->map_writable, &new_spte); 955bb18842eSBen Gardon 956bb18842eSBen Gardon if (new_spte == iter->old_spte) 957bb18842eSBen Gardon ret = RET_PF_SPURIOUS; 9583e72c791SDavid Matlack else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte)) 9599a77daacSBen Gardon return RET_PF_RETRY; 960bb18842eSBen Gardon 961bb18842eSBen Gardon /* 962bb18842eSBen Gardon * If the page fault was caused by a write but the page is write 963bb18842eSBen Gardon * protected, emulation is needed. If the emulation was skipped, 964bb18842eSBen Gardon * the vCPU would have the same fault again. 965bb18842eSBen Gardon */ 966ad67e480SPaolo Bonzini if (wrprot) { 967cdc47767SPaolo Bonzini if (fault->write) 968bb18842eSBen Gardon ret = RET_PF_EMULATE; 969bb18842eSBen Gardon } 970bb18842eSBen Gardon 971bb18842eSBen Gardon /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */ 9729a77daacSBen Gardon if (unlikely(is_mmio_spte(new_spte))) { 9739a77daacSBen Gardon trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn, 9749a77daacSBen Gardon new_spte); 975bb18842eSBen Gardon ret = RET_PF_EMULATE; 9763849e092SSean Christopherson } else { 9779a77daacSBen Gardon trace_kvm_mmu_set_spte(iter->level, iter->gfn, 9789a77daacSBen Gardon rcu_dereference(iter->sptep)); 9793849e092SSean Christopherson } 980bb18842eSBen Gardon 981857f8474SKai Huang /* 982857f8474SKai Huang * Increase pf_fixed in both RET_PF_EMULATE and RET_PF_FIXED to be 983857f8474SKai Huang * consistent with legacy MMU behavior. 984857f8474SKai Huang */ 985857f8474SKai Huang if (ret != RET_PF_SPURIOUS) 986bb18842eSBen Gardon vcpu->stat.pf_fixed++; 987bb18842eSBen Gardon 988bb18842eSBen Gardon return ret; 989bb18842eSBen Gardon } 990bb18842eSBen Gardon 991bb18842eSBen Gardon /* 992cb00a70bSDavid Matlack * tdp_mmu_link_sp - Replace the given spte with an spte pointing to the 993cb00a70bSDavid Matlack * provided page table. 9947b7e1ab6SDavid Matlack * 9957b7e1ab6SDavid Matlack * @kvm: kvm instance 9967b7e1ab6SDavid Matlack * @iter: a tdp_iter instance currently on the SPTE that should be set 9977b7e1ab6SDavid Matlack * @sp: The new TDP page table to install. 9987b7e1ab6SDavid Matlack * @account_nx: True if this page table is being installed to split a 9997b7e1ab6SDavid Matlack * non-executable huge page. 1000cb00a70bSDavid Matlack * @shared: This operation is running under the MMU lock in read mode. 10017b7e1ab6SDavid Matlack * 10027b7e1ab6SDavid Matlack * Returns: 0 if the new page table was installed. Non-0 if the page table 10037b7e1ab6SDavid Matlack * could not be installed (e.g. the atomic compare-exchange failed). 10047b7e1ab6SDavid Matlack */ 1005cb00a70bSDavid Matlack static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter, 1006cb00a70bSDavid Matlack struct kvm_mmu_page *sp, bool account_nx, 1007cb00a70bSDavid Matlack bool shared) 10087b7e1ab6SDavid Matlack { 10097b7e1ab6SDavid Matlack u64 spte = make_nonleaf_spte(sp->spt, !shadow_accessed_mask); 1010cb00a70bSDavid Matlack int ret = 0; 10117b7e1ab6SDavid Matlack 1012cb00a70bSDavid Matlack if (shared) { 10137b7e1ab6SDavid Matlack ret = tdp_mmu_set_spte_atomic(kvm, iter, spte); 10147b7e1ab6SDavid Matlack if (ret) 10157b7e1ab6SDavid Matlack return ret; 1016cb00a70bSDavid Matlack } else { 1017cb00a70bSDavid Matlack tdp_mmu_set_spte(kvm, iter, spte); 1018cb00a70bSDavid Matlack } 10197b7e1ab6SDavid Matlack 10207b7e1ab6SDavid Matlack spin_lock(&kvm->arch.tdp_mmu_pages_lock); 10217b7e1ab6SDavid Matlack list_add(&sp->link, &kvm->arch.tdp_mmu_pages); 10227b7e1ab6SDavid Matlack if (account_nx) 10237b7e1ab6SDavid Matlack account_huge_nx_page(kvm, sp); 10247b7e1ab6SDavid Matlack spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 10257b7e1ab6SDavid Matlack 10267b7e1ab6SDavid Matlack return 0; 10277b7e1ab6SDavid Matlack } 10287b7e1ab6SDavid Matlack 10297b7e1ab6SDavid Matlack /* 1030bb18842eSBen Gardon * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing 1031bb18842eSBen Gardon * page tables and SPTEs to translate the faulting guest physical address. 1032bb18842eSBen Gardon */ 10332f6305ddSPaolo Bonzini int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) 1034bb18842eSBen Gardon { 1035bb18842eSBen Gardon struct kvm_mmu *mmu = vcpu->arch.mmu; 1036bb18842eSBen Gardon struct tdp_iter iter; 103789c0fd49SBen Gardon struct kvm_mmu_page *sp; 1038bb18842eSBen Gardon int ret; 1039bb18842eSBen Gardon 104073a3c659SPaolo Bonzini kvm_mmu_hugepage_adjust(vcpu, fault); 1041bb18842eSBen Gardon 1042f0066d94SPaolo Bonzini trace_kvm_mmu_spte_requested(fault); 10437cca2d0bSBen Gardon 10447cca2d0bSBen Gardon rcu_read_lock(); 10457cca2d0bSBen Gardon 10462f6305ddSPaolo Bonzini tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) { 104773a3c659SPaolo Bonzini if (fault->nx_huge_page_workaround_enabled) 1048536f0e6aSPaolo Bonzini disallowed_hugepage_adjust(fault, iter.old_spte, iter.level); 1049bb18842eSBen Gardon 105073a3c659SPaolo Bonzini if (iter.level == fault->goal_level) 1051bb18842eSBen Gardon break; 1052bb18842eSBen Gardon 1053bb18842eSBen Gardon /* 1054bb18842eSBen Gardon * If there is an SPTE mapping a large page at a higher level 1055bb18842eSBen Gardon * than the target, that SPTE must be cleared and replaced 1056bb18842eSBen Gardon * with a non-leaf SPTE. 1057bb18842eSBen Gardon */ 1058bb18842eSBen Gardon if (is_shadow_present_pte(iter.old_spte) && 1059bb18842eSBen Gardon is_large_pte(iter.old_spte)) { 10603e72c791SDavid Matlack if (tdp_mmu_zap_spte_atomic(vcpu->kvm, &iter)) 10619a77daacSBen Gardon break; 1062bb18842eSBen Gardon 1063bb18842eSBen Gardon /* 1064bb18842eSBen Gardon * The iter must explicitly re-read the spte here 1065bb18842eSBen Gardon * because the new value informs the !present 1066bb18842eSBen Gardon * path below. 1067bb18842eSBen Gardon */ 10687cca2d0bSBen Gardon iter.old_spte = READ_ONCE(*rcu_dereference(iter.sptep)); 1069bb18842eSBen Gardon } 1070bb18842eSBen Gardon 1071bb18842eSBen Gardon if (!is_shadow_present_pte(iter.old_spte)) { 10727b7e1ab6SDavid Matlack bool account_nx = fault->huge_page_disallowed && 10737b7e1ab6SDavid Matlack fault->req_level >= iter.level; 10747b7e1ab6SDavid Matlack 1075ff76d506SKai Huang /* 1076c4342633SIngo Molnar * If SPTE has been frozen by another thread, just 1077ff76d506SKai Huang * give up and retry, avoiding unnecessary page table 1078ff76d506SKai Huang * allocation and free. 1079ff76d506SKai Huang */ 1080ff76d506SKai Huang if (is_removed_spte(iter.old_spte)) 1081ff76d506SKai Huang break; 1082ff76d506SKai Huang 1083a82070b6SDavid Matlack sp = tdp_mmu_alloc_sp(vcpu); 1084a82070b6SDavid Matlack tdp_mmu_init_child_sp(sp, &iter); 1085a82070b6SDavid Matlack 1086cb00a70bSDavid Matlack if (tdp_mmu_link_sp(vcpu->kvm, &iter, sp, account_nx, true)) { 10879a77daacSBen Gardon tdp_mmu_free_sp(sp); 10889a77daacSBen Gardon break; 10899a77daacSBen Gardon } 1090bb18842eSBen Gardon } 1091bb18842eSBen Gardon } 1092bb18842eSBen Gardon 109373a3c659SPaolo Bonzini if (iter.level != fault->goal_level) { 10947cca2d0bSBen Gardon rcu_read_unlock(); 1095bb18842eSBen Gardon return RET_PF_RETRY; 10967cca2d0bSBen Gardon } 1097bb18842eSBen Gardon 1098cdc47767SPaolo Bonzini ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter); 10997cca2d0bSBen Gardon rcu_read_unlock(); 1100bb18842eSBen Gardon 1101bb18842eSBen Gardon return ret; 1102bb18842eSBen Gardon } 1103063afacdSBen Gardon 11043039bcc7SSean Christopherson bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, 11053039bcc7SSean Christopherson bool flush) 1106063afacdSBen Gardon { 110783b83a02SSean Christopherson return __kvm_tdp_mmu_zap_gfn_range(kvm, range->slot->as_id, range->start, 110883b83a02SSean Christopherson range->end, range->may_block, flush); 11093039bcc7SSean Christopherson } 11103039bcc7SSean Christopherson 11113039bcc7SSean Christopherson typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter, 11123039bcc7SSean Christopherson struct kvm_gfn_range *range); 11133039bcc7SSean Christopherson 11143039bcc7SSean Christopherson static __always_inline bool kvm_tdp_mmu_handle_gfn(struct kvm *kvm, 11153039bcc7SSean Christopherson struct kvm_gfn_range *range, 1116c1b91493SSean Christopherson tdp_handler_t handler) 1117063afacdSBen Gardon { 1118063afacdSBen Gardon struct kvm_mmu_page *root; 11193039bcc7SSean Christopherson struct tdp_iter iter; 11203039bcc7SSean Christopherson bool ret = false; 1121063afacdSBen Gardon 11223039bcc7SSean Christopherson rcu_read_lock(); 1123063afacdSBen Gardon 1124063afacdSBen Gardon /* 1125e1eed584SSean Christopherson * Don't support rescheduling, none of the MMU notifiers that funnel 1126e1eed584SSean Christopherson * into this helper allow blocking; it'd be dead, wasteful code. 1127063afacdSBen Gardon */ 11283039bcc7SSean Christopherson for_each_tdp_mmu_root(kvm, root, range->slot->as_id) { 11293039bcc7SSean Christopherson tdp_root_for_each_leaf_pte(iter, root, range->start, range->end) 11303039bcc7SSean Christopherson ret |= handler(kvm, &iter, range); 11313039bcc7SSean Christopherson } 1132063afacdSBen Gardon 11333039bcc7SSean Christopherson rcu_read_unlock(); 1134063afacdSBen Gardon 1135063afacdSBen Gardon return ret; 1136063afacdSBen Gardon } 1137063afacdSBen Gardon 1138f8e14497SBen Gardon /* 1139f8e14497SBen Gardon * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero 1140f8e14497SBen Gardon * if any of the GFNs in the range have been accessed. 1141f8e14497SBen Gardon */ 11423039bcc7SSean Christopherson static bool age_gfn_range(struct kvm *kvm, struct tdp_iter *iter, 11433039bcc7SSean Christopherson struct kvm_gfn_range *range) 1144f8e14497SBen Gardon { 1145f8e14497SBen Gardon u64 new_spte = 0; 1146f8e14497SBen Gardon 11473039bcc7SSean Christopherson /* If we have a non-accessed entry we don't need to change the pte. */ 11483039bcc7SSean Christopherson if (!is_accessed_spte(iter->old_spte)) 11493039bcc7SSean Christopherson return false; 11507cca2d0bSBen Gardon 11513039bcc7SSean Christopherson new_spte = iter->old_spte; 1152f8e14497SBen Gardon 1153f8e14497SBen Gardon if (spte_ad_enabled(new_spte)) { 11548f8f52a4SSean Christopherson new_spte &= ~shadow_accessed_mask; 1155f8e14497SBen Gardon } else { 1156f8e14497SBen Gardon /* 1157f8e14497SBen Gardon * Capture the dirty status of the page, so that it doesn't get 1158f8e14497SBen Gardon * lost when the SPTE is marked for access tracking. 1159f8e14497SBen Gardon */ 1160f8e14497SBen Gardon if (is_writable_pte(new_spte)) 1161f8e14497SBen Gardon kvm_set_pfn_dirty(spte_to_pfn(new_spte)); 1162f8e14497SBen Gardon 1163f8e14497SBen Gardon new_spte = mark_spte_for_access_track(new_spte); 1164f8e14497SBen Gardon } 1165f8e14497SBen Gardon 11663039bcc7SSean Christopherson tdp_mmu_set_spte_no_acc_track(kvm, iter, new_spte); 116733dd3574SBen Gardon 11683039bcc7SSean Christopherson return true; 1169f8e14497SBen Gardon } 1170f8e14497SBen Gardon 11713039bcc7SSean Christopherson bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) 1172f8e14497SBen Gardon { 11733039bcc7SSean Christopherson return kvm_tdp_mmu_handle_gfn(kvm, range, age_gfn_range); 1174f8e14497SBen Gardon } 1175f8e14497SBen Gardon 11763039bcc7SSean Christopherson static bool test_age_gfn(struct kvm *kvm, struct tdp_iter *iter, 11773039bcc7SSean Christopherson struct kvm_gfn_range *range) 1178f8e14497SBen Gardon { 11793039bcc7SSean Christopherson return is_accessed_spte(iter->old_spte); 1180f8e14497SBen Gardon } 1181f8e14497SBen Gardon 11823039bcc7SSean Christopherson bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 1183f8e14497SBen Gardon { 11843039bcc7SSean Christopherson return kvm_tdp_mmu_handle_gfn(kvm, range, test_age_gfn); 11853039bcc7SSean Christopherson } 11863039bcc7SSean Christopherson 11873039bcc7SSean Christopherson static bool set_spte_gfn(struct kvm *kvm, struct tdp_iter *iter, 11883039bcc7SSean Christopherson struct kvm_gfn_range *range) 11893039bcc7SSean Christopherson { 11903039bcc7SSean Christopherson u64 new_spte; 11913039bcc7SSean Christopherson 11923039bcc7SSean Christopherson /* Huge pages aren't expected to be modified without first being zapped. */ 11933039bcc7SSean Christopherson WARN_ON(pte_huge(range->pte) || range->start + 1 != range->end); 11943039bcc7SSean Christopherson 11953039bcc7SSean Christopherson if (iter->level != PG_LEVEL_4K || 11963039bcc7SSean Christopherson !is_shadow_present_pte(iter->old_spte)) 11973039bcc7SSean Christopherson return false; 11983039bcc7SSean Christopherson 11993039bcc7SSean Christopherson /* 12003039bcc7SSean Christopherson * Note, when changing a read-only SPTE, it's not strictly necessary to 12013039bcc7SSean Christopherson * zero the SPTE before setting the new PFN, but doing so preserves the 12023039bcc7SSean Christopherson * invariant that the PFN of a present * leaf SPTE can never change. 12033039bcc7SSean Christopherson * See __handle_changed_spte(). 12043039bcc7SSean Christopherson */ 12053039bcc7SSean Christopherson tdp_mmu_set_spte(kvm, iter, 0); 12063039bcc7SSean Christopherson 12073039bcc7SSean Christopherson if (!pte_write(range->pte)) { 12083039bcc7SSean Christopherson new_spte = kvm_mmu_changed_pte_notifier_make_spte(iter->old_spte, 12093039bcc7SSean Christopherson pte_pfn(range->pte)); 12103039bcc7SSean Christopherson 12113039bcc7SSean Christopherson tdp_mmu_set_spte(kvm, iter, new_spte); 12123039bcc7SSean Christopherson } 12133039bcc7SSean Christopherson 12143039bcc7SSean Christopherson return true; 1215f8e14497SBen Gardon } 12161d8dd6b3SBen Gardon 12171d8dd6b3SBen Gardon /* 12181d8dd6b3SBen Gardon * Handle the changed_pte MMU notifier for the TDP MMU. 12191d8dd6b3SBen Gardon * data is a pointer to the new pte_t mapping the HVA specified by the MMU 12201d8dd6b3SBen Gardon * notifier. 12211d8dd6b3SBen Gardon * Returns non-zero if a flush is needed before releasing the MMU lock. 12221d8dd6b3SBen Gardon */ 12233039bcc7SSean Christopherson bool kvm_tdp_mmu_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 12241d8dd6b3SBen Gardon { 12253039bcc7SSean Christopherson bool flush = kvm_tdp_mmu_handle_gfn(kvm, range, set_spte_gfn); 12261d8dd6b3SBen Gardon 12273039bcc7SSean Christopherson /* FIXME: return 'flush' instead of flushing here. */ 12283039bcc7SSean Christopherson if (flush) 12293039bcc7SSean Christopherson kvm_flush_remote_tlbs_with_address(kvm, range->start, 1); 12307cca2d0bSBen Gardon 12313039bcc7SSean Christopherson return false; 12321d8dd6b3SBen Gardon } 12331d8dd6b3SBen Gardon 1234a6a0b05dSBen Gardon /* 1235bedd9195SDavid Matlack * Remove write access from all SPTEs at or above min_level that map GFNs 1236bedd9195SDavid Matlack * [start, end). Returns true if an SPTE has been changed and the TLBs need to 1237bedd9195SDavid Matlack * be flushed. 1238a6a0b05dSBen Gardon */ 1239a6a0b05dSBen Gardon static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1240a6a0b05dSBen Gardon gfn_t start, gfn_t end, int min_level) 1241a6a0b05dSBen Gardon { 1242a6a0b05dSBen Gardon struct tdp_iter iter; 1243a6a0b05dSBen Gardon u64 new_spte; 1244a6a0b05dSBen Gardon bool spte_set = false; 1245a6a0b05dSBen Gardon 12467cca2d0bSBen Gardon rcu_read_lock(); 12477cca2d0bSBen Gardon 1248a6a0b05dSBen Gardon BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 1249a6a0b05dSBen Gardon 125077aa6075SDavid Matlack for_each_tdp_pte_min_level(iter, root, min_level, start, end) { 125124ae4cfaSBen Gardon retry: 125224ae4cfaSBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 12531af4a960SBen Gardon continue; 12541af4a960SBen Gardon 1255a6a0b05dSBen Gardon if (!is_shadow_present_pte(iter.old_spte) || 12560f99ee2cSBen Gardon !is_last_spte(iter.old_spte, iter.level) || 12570f99ee2cSBen Gardon !(iter.old_spte & PT_WRITABLE_MASK)) 1258a6a0b05dSBen Gardon continue; 1259a6a0b05dSBen Gardon 1260a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1261a6a0b05dSBen Gardon 12623e72c791SDavid Matlack if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) 126324ae4cfaSBen Gardon goto retry; 12643255530aSDavid Matlack 1265a6a0b05dSBen Gardon spte_set = true; 1266a6a0b05dSBen Gardon } 12677cca2d0bSBen Gardon 12687cca2d0bSBen Gardon rcu_read_unlock(); 1269a6a0b05dSBen Gardon return spte_set; 1270a6a0b05dSBen Gardon } 1271a6a0b05dSBen Gardon 1272a6a0b05dSBen Gardon /* 1273a6a0b05dSBen Gardon * Remove write access from all the SPTEs mapping GFNs in the memslot. Will 1274a6a0b05dSBen Gardon * only affect leaf SPTEs down to min_level. 1275a6a0b05dSBen Gardon * Returns true if an SPTE has been changed and the TLBs need to be flushed. 1276a6a0b05dSBen Gardon */ 1277269e9552SHamza Mahfooz bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, 1278269e9552SHamza Mahfooz const struct kvm_memory_slot *slot, int min_level) 1279a6a0b05dSBen Gardon { 1280a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1281a6a0b05dSBen Gardon bool spte_set = false; 1282a6a0b05dSBen Gardon 128324ae4cfaSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 1284a6a0b05dSBen Gardon 1285d62007edSSean Christopherson for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) 1286a6a0b05dSBen Gardon spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn, 1287a6a0b05dSBen Gardon slot->base_gfn + slot->npages, min_level); 1288a6a0b05dSBen Gardon 1289a6a0b05dSBen Gardon return spte_set; 1290a6a0b05dSBen Gardon } 1291a6a0b05dSBen Gardon 1292a3fe5dbdSDavid Matlack static struct kvm_mmu_page *__tdp_mmu_alloc_sp_for_split(gfp_t gfp) 1293a3fe5dbdSDavid Matlack { 1294a3fe5dbdSDavid Matlack struct kvm_mmu_page *sp; 1295a3fe5dbdSDavid Matlack 1296a3fe5dbdSDavid Matlack gfp |= __GFP_ZERO; 1297a3fe5dbdSDavid Matlack 1298a3fe5dbdSDavid Matlack sp = kmem_cache_alloc(mmu_page_header_cache, gfp); 1299a3fe5dbdSDavid Matlack if (!sp) 1300a3fe5dbdSDavid Matlack return NULL; 1301a3fe5dbdSDavid Matlack 1302a3fe5dbdSDavid Matlack sp->spt = (void *)__get_free_page(gfp); 1303a3fe5dbdSDavid Matlack if (!sp->spt) { 1304a3fe5dbdSDavid Matlack kmem_cache_free(mmu_page_header_cache, sp); 1305a3fe5dbdSDavid Matlack return NULL; 1306a3fe5dbdSDavid Matlack } 1307a3fe5dbdSDavid Matlack 1308a3fe5dbdSDavid Matlack return sp; 1309a3fe5dbdSDavid Matlack } 1310a3fe5dbdSDavid Matlack 1311a3fe5dbdSDavid Matlack static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct kvm *kvm, 1312cb00a70bSDavid Matlack struct tdp_iter *iter, 1313cb00a70bSDavid Matlack bool shared) 1314a3fe5dbdSDavid Matlack { 1315a3fe5dbdSDavid Matlack struct kvm_mmu_page *sp; 1316a3fe5dbdSDavid Matlack 1317a3fe5dbdSDavid Matlack /* 1318a3fe5dbdSDavid Matlack * Since we are allocating while under the MMU lock we have to be 1319a3fe5dbdSDavid Matlack * careful about GFP flags. Use GFP_NOWAIT to avoid blocking on direct 1320a3fe5dbdSDavid Matlack * reclaim and to avoid making any filesystem callbacks (which can end 1321a3fe5dbdSDavid Matlack * up invoking KVM MMU notifiers, resulting in a deadlock). 1322a3fe5dbdSDavid Matlack * 1323a3fe5dbdSDavid Matlack * If this allocation fails we drop the lock and retry with reclaim 1324a3fe5dbdSDavid Matlack * allowed. 1325a3fe5dbdSDavid Matlack */ 1326a3fe5dbdSDavid Matlack sp = __tdp_mmu_alloc_sp_for_split(GFP_NOWAIT | __GFP_ACCOUNT); 1327a3fe5dbdSDavid Matlack if (sp) 1328a3fe5dbdSDavid Matlack return sp; 1329a3fe5dbdSDavid Matlack 1330a3fe5dbdSDavid Matlack rcu_read_unlock(); 1331cb00a70bSDavid Matlack 1332cb00a70bSDavid Matlack if (shared) 1333a3fe5dbdSDavid Matlack read_unlock(&kvm->mmu_lock); 1334cb00a70bSDavid Matlack else 1335cb00a70bSDavid Matlack write_unlock(&kvm->mmu_lock); 1336a3fe5dbdSDavid Matlack 1337a3fe5dbdSDavid Matlack iter->yielded = true; 1338a3fe5dbdSDavid Matlack sp = __tdp_mmu_alloc_sp_for_split(GFP_KERNEL_ACCOUNT); 1339a3fe5dbdSDavid Matlack 1340cb00a70bSDavid Matlack if (shared) 1341a3fe5dbdSDavid Matlack read_lock(&kvm->mmu_lock); 1342cb00a70bSDavid Matlack else 1343cb00a70bSDavid Matlack write_lock(&kvm->mmu_lock); 1344cb00a70bSDavid Matlack 1345a3fe5dbdSDavid Matlack rcu_read_lock(); 1346a3fe5dbdSDavid Matlack 1347a3fe5dbdSDavid Matlack return sp; 1348a3fe5dbdSDavid Matlack } 1349a3fe5dbdSDavid Matlack 1350cb00a70bSDavid Matlack static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter, 1351cb00a70bSDavid Matlack struct kvm_mmu_page *sp, bool shared) 1352a3fe5dbdSDavid Matlack { 1353a3fe5dbdSDavid Matlack const u64 huge_spte = iter->old_spte; 1354a3fe5dbdSDavid Matlack const int level = iter->level; 1355a3fe5dbdSDavid Matlack int ret, i; 1356a3fe5dbdSDavid Matlack 1357a3fe5dbdSDavid Matlack tdp_mmu_init_child_sp(sp, iter); 1358a3fe5dbdSDavid Matlack 1359a3fe5dbdSDavid Matlack /* 1360a3fe5dbdSDavid Matlack * No need for atomics when writing to sp->spt since the page table has 1361a3fe5dbdSDavid Matlack * not been linked in yet and thus is not reachable from any other CPU. 1362a3fe5dbdSDavid Matlack */ 1363a3fe5dbdSDavid Matlack for (i = 0; i < PT64_ENT_PER_PAGE; i++) 1364a3fe5dbdSDavid Matlack sp->spt[i] = make_huge_page_split_spte(huge_spte, level, i); 1365a3fe5dbdSDavid Matlack 1366a3fe5dbdSDavid Matlack /* 1367a3fe5dbdSDavid Matlack * Replace the huge spte with a pointer to the populated lower level 1368a3fe5dbdSDavid Matlack * page table. Since we are making this change without a TLB flush vCPUs 1369a3fe5dbdSDavid Matlack * will see a mix of the split mappings and the original huge mapping, 1370a3fe5dbdSDavid Matlack * depending on what's currently in their TLB. This is fine from a 1371a3fe5dbdSDavid Matlack * correctness standpoint since the translation will be the same either 1372a3fe5dbdSDavid Matlack * way. 1373a3fe5dbdSDavid Matlack */ 1374cb00a70bSDavid Matlack ret = tdp_mmu_link_sp(kvm, iter, sp, false, shared); 1375a3fe5dbdSDavid Matlack if (ret) 1376e0b728b1SDavid Matlack goto out; 1377a3fe5dbdSDavid Matlack 1378a3fe5dbdSDavid Matlack /* 1379a3fe5dbdSDavid Matlack * tdp_mmu_link_sp_atomic() will handle subtracting the huge page we 1380a3fe5dbdSDavid Matlack * are overwriting from the page stats. But we have to manually update 1381a3fe5dbdSDavid Matlack * the page stats with the new present child pages. 1382a3fe5dbdSDavid Matlack */ 1383a3fe5dbdSDavid Matlack kvm_update_page_stats(kvm, level - 1, PT64_ENT_PER_PAGE); 1384a3fe5dbdSDavid Matlack 1385e0b728b1SDavid Matlack out: 1386e0b728b1SDavid Matlack trace_kvm_mmu_split_huge_page(iter->gfn, huge_spte, level, ret); 1387e0b728b1SDavid Matlack return ret; 1388a3fe5dbdSDavid Matlack } 1389a3fe5dbdSDavid Matlack 1390a3fe5dbdSDavid Matlack static int tdp_mmu_split_huge_pages_root(struct kvm *kvm, 1391a3fe5dbdSDavid Matlack struct kvm_mmu_page *root, 1392a3fe5dbdSDavid Matlack gfn_t start, gfn_t end, 1393cb00a70bSDavid Matlack int target_level, bool shared) 1394a3fe5dbdSDavid Matlack { 1395a3fe5dbdSDavid Matlack struct kvm_mmu_page *sp = NULL; 1396a3fe5dbdSDavid Matlack struct tdp_iter iter; 1397a3fe5dbdSDavid Matlack int ret = 0; 1398a3fe5dbdSDavid Matlack 1399a3fe5dbdSDavid Matlack rcu_read_lock(); 1400a3fe5dbdSDavid Matlack 1401a3fe5dbdSDavid Matlack /* 1402a3fe5dbdSDavid Matlack * Traverse the page table splitting all huge pages above the target 1403a3fe5dbdSDavid Matlack * level into one lower level. For example, if we encounter a 1GB page 1404a3fe5dbdSDavid Matlack * we split it into 512 2MB pages. 1405a3fe5dbdSDavid Matlack * 1406a3fe5dbdSDavid Matlack * Since the TDP iterator uses a pre-order traversal, we are guaranteed 1407a3fe5dbdSDavid Matlack * to visit an SPTE before ever visiting its children, which means we 1408a3fe5dbdSDavid Matlack * will correctly recursively split huge pages that are more than one 1409a3fe5dbdSDavid Matlack * level above the target level (e.g. splitting a 1GB to 512 2MB pages, 1410a3fe5dbdSDavid Matlack * and then splitting each of those to 512 4KB pages). 1411a3fe5dbdSDavid Matlack */ 1412a3fe5dbdSDavid Matlack for_each_tdp_pte_min_level(iter, root, target_level + 1, start, end) { 1413a3fe5dbdSDavid Matlack retry: 1414cb00a70bSDavid Matlack if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) 1415a3fe5dbdSDavid Matlack continue; 1416a3fe5dbdSDavid Matlack 1417a3fe5dbdSDavid Matlack if (!is_shadow_present_pte(iter.old_spte) || !is_large_pte(iter.old_spte)) 1418a3fe5dbdSDavid Matlack continue; 1419a3fe5dbdSDavid Matlack 1420a3fe5dbdSDavid Matlack if (!sp) { 1421cb00a70bSDavid Matlack sp = tdp_mmu_alloc_sp_for_split(kvm, &iter, shared); 1422a3fe5dbdSDavid Matlack if (!sp) { 1423a3fe5dbdSDavid Matlack ret = -ENOMEM; 1424e0b728b1SDavid Matlack trace_kvm_mmu_split_huge_page(iter.gfn, 1425e0b728b1SDavid Matlack iter.old_spte, 1426e0b728b1SDavid Matlack iter.level, ret); 1427a3fe5dbdSDavid Matlack break; 1428a3fe5dbdSDavid Matlack } 1429a3fe5dbdSDavid Matlack 1430a3fe5dbdSDavid Matlack if (iter.yielded) 1431a3fe5dbdSDavid Matlack continue; 1432a3fe5dbdSDavid Matlack } 1433a3fe5dbdSDavid Matlack 1434cb00a70bSDavid Matlack if (tdp_mmu_split_huge_page(kvm, &iter, sp, shared)) 1435a3fe5dbdSDavid Matlack goto retry; 1436a3fe5dbdSDavid Matlack 1437a3fe5dbdSDavid Matlack sp = NULL; 1438a3fe5dbdSDavid Matlack } 1439a3fe5dbdSDavid Matlack 1440a3fe5dbdSDavid Matlack rcu_read_unlock(); 1441a3fe5dbdSDavid Matlack 1442a3fe5dbdSDavid Matlack /* 1443a3fe5dbdSDavid Matlack * It's possible to exit the loop having never used the last sp if, for 1444a3fe5dbdSDavid Matlack * example, a vCPU doing HugePage NX splitting wins the race and 1445a3fe5dbdSDavid Matlack * installs its own sp in place of the last sp we tried to split. 1446a3fe5dbdSDavid Matlack */ 1447a3fe5dbdSDavid Matlack if (sp) 1448a3fe5dbdSDavid Matlack tdp_mmu_free_sp(sp); 1449a3fe5dbdSDavid Matlack 1450a3fe5dbdSDavid Matlack return ret; 1451a3fe5dbdSDavid Matlack } 1452a3fe5dbdSDavid Matlack 1453cb00a70bSDavid Matlack 1454a3fe5dbdSDavid Matlack /* 1455a3fe5dbdSDavid Matlack * Try to split all huge pages mapped by the TDP MMU down to the target level. 1456a3fe5dbdSDavid Matlack */ 1457a3fe5dbdSDavid Matlack void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, 1458a3fe5dbdSDavid Matlack const struct kvm_memory_slot *slot, 1459a3fe5dbdSDavid Matlack gfn_t start, gfn_t end, 1460cb00a70bSDavid Matlack int target_level, bool shared) 1461a3fe5dbdSDavid Matlack { 1462a3fe5dbdSDavid Matlack struct kvm_mmu_page *root; 1463a3fe5dbdSDavid Matlack int r = 0; 1464a3fe5dbdSDavid Matlack 1465cb00a70bSDavid Matlack kvm_lockdep_assert_mmu_lock_held(kvm, shared); 1466a3fe5dbdSDavid Matlack 1467cb00a70bSDavid Matlack for_each_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) { 1468cb00a70bSDavid Matlack r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared); 1469a3fe5dbdSDavid Matlack if (r) { 1470cb00a70bSDavid Matlack kvm_tdp_mmu_put_root(kvm, root, shared); 1471a3fe5dbdSDavid Matlack break; 1472a3fe5dbdSDavid Matlack } 1473a3fe5dbdSDavid Matlack } 1474a3fe5dbdSDavid Matlack } 1475a3fe5dbdSDavid Matlack 1476a6a0b05dSBen Gardon /* 1477a6a0b05dSBen Gardon * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If 1478a6a0b05dSBen Gardon * AD bits are enabled, this will involve clearing the dirty bit on each SPTE. 1479a6a0b05dSBen Gardon * If AD bits are not enabled, this will require clearing the writable bit on 1480a6a0b05dSBen Gardon * each SPTE. Returns true if an SPTE has been changed and the TLBs need to 1481a6a0b05dSBen Gardon * be flushed. 1482a6a0b05dSBen Gardon */ 1483a6a0b05dSBen Gardon static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1484a6a0b05dSBen Gardon gfn_t start, gfn_t end) 1485a6a0b05dSBen Gardon { 1486a6a0b05dSBen Gardon struct tdp_iter iter; 1487a6a0b05dSBen Gardon u64 new_spte; 1488a6a0b05dSBen Gardon bool spte_set = false; 1489a6a0b05dSBen Gardon 14907cca2d0bSBen Gardon rcu_read_lock(); 14917cca2d0bSBen Gardon 1492a6a0b05dSBen Gardon tdp_root_for_each_leaf_pte(iter, root, start, end) { 149324ae4cfaSBen Gardon retry: 149424ae4cfaSBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 14951af4a960SBen Gardon continue; 14961af4a960SBen Gardon 14973354ef5aSSean Christopherson if (!is_shadow_present_pte(iter.old_spte)) 14983354ef5aSSean Christopherson continue; 14993354ef5aSSean Christopherson 1500a6a0b05dSBen Gardon if (spte_ad_need_write_protect(iter.old_spte)) { 1501a6a0b05dSBen Gardon if (is_writable_pte(iter.old_spte)) 1502a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1503a6a0b05dSBen Gardon else 1504a6a0b05dSBen Gardon continue; 1505a6a0b05dSBen Gardon } else { 1506a6a0b05dSBen Gardon if (iter.old_spte & shadow_dirty_mask) 1507a6a0b05dSBen Gardon new_spte = iter.old_spte & ~shadow_dirty_mask; 1508a6a0b05dSBen Gardon else 1509a6a0b05dSBen Gardon continue; 1510a6a0b05dSBen Gardon } 1511a6a0b05dSBen Gardon 15123e72c791SDavid Matlack if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) 151324ae4cfaSBen Gardon goto retry; 15143255530aSDavid Matlack 1515a6a0b05dSBen Gardon spte_set = true; 1516a6a0b05dSBen Gardon } 15177cca2d0bSBen Gardon 15187cca2d0bSBen Gardon rcu_read_unlock(); 1519a6a0b05dSBen Gardon return spte_set; 1520a6a0b05dSBen Gardon } 1521a6a0b05dSBen Gardon 1522a6a0b05dSBen Gardon /* 1523a6a0b05dSBen Gardon * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If 1524a6a0b05dSBen Gardon * AD bits are enabled, this will involve clearing the dirty bit on each SPTE. 1525a6a0b05dSBen Gardon * If AD bits are not enabled, this will require clearing the writable bit on 1526a6a0b05dSBen Gardon * each SPTE. Returns true if an SPTE has been changed and the TLBs need to 1527a6a0b05dSBen Gardon * be flushed. 1528a6a0b05dSBen Gardon */ 1529269e9552SHamza Mahfooz bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, 1530269e9552SHamza Mahfooz const struct kvm_memory_slot *slot) 1531a6a0b05dSBen Gardon { 1532a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1533a6a0b05dSBen Gardon bool spte_set = false; 1534a6a0b05dSBen Gardon 153524ae4cfaSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 1536a6a0b05dSBen Gardon 1537d62007edSSean Christopherson for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) 1538a6a0b05dSBen Gardon spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn, 1539a6a0b05dSBen Gardon slot->base_gfn + slot->npages); 1540a6a0b05dSBen Gardon 1541a6a0b05dSBen Gardon return spte_set; 1542a6a0b05dSBen Gardon } 1543a6a0b05dSBen Gardon 1544a6a0b05dSBen Gardon /* 1545a6a0b05dSBen Gardon * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is 1546a6a0b05dSBen Gardon * set in mask, starting at gfn. The given memslot is expected to contain all 1547a6a0b05dSBen Gardon * the GFNs represented by set bits in the mask. If AD bits are enabled, 1548a6a0b05dSBen Gardon * clearing the dirty status will involve clearing the dirty bit on each SPTE 1549a6a0b05dSBen Gardon * or, if AD bits are not enabled, clearing the writable bit on each SPTE. 1550a6a0b05dSBen Gardon */ 1551a6a0b05dSBen Gardon static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root, 1552a6a0b05dSBen Gardon gfn_t gfn, unsigned long mask, bool wrprot) 1553a6a0b05dSBen Gardon { 1554a6a0b05dSBen Gardon struct tdp_iter iter; 1555a6a0b05dSBen Gardon u64 new_spte; 1556a6a0b05dSBen Gardon 15577cca2d0bSBen Gardon rcu_read_lock(); 15587cca2d0bSBen Gardon 1559a6a0b05dSBen Gardon tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask), 1560a6a0b05dSBen Gardon gfn + BITS_PER_LONG) { 1561a6a0b05dSBen Gardon if (!mask) 1562a6a0b05dSBen Gardon break; 1563a6a0b05dSBen Gardon 1564a6a0b05dSBen Gardon if (iter.level > PG_LEVEL_4K || 1565a6a0b05dSBen Gardon !(mask & (1UL << (iter.gfn - gfn)))) 1566a6a0b05dSBen Gardon continue; 1567a6a0b05dSBen Gardon 1568f1b3b06aSBen Gardon mask &= ~(1UL << (iter.gfn - gfn)); 1569f1b3b06aSBen Gardon 1570a6a0b05dSBen Gardon if (wrprot || spte_ad_need_write_protect(iter.old_spte)) { 1571a6a0b05dSBen Gardon if (is_writable_pte(iter.old_spte)) 1572a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1573a6a0b05dSBen Gardon else 1574a6a0b05dSBen Gardon continue; 1575a6a0b05dSBen Gardon } else { 1576a6a0b05dSBen Gardon if (iter.old_spte & shadow_dirty_mask) 1577a6a0b05dSBen Gardon new_spte = iter.old_spte & ~shadow_dirty_mask; 1578a6a0b05dSBen Gardon else 1579a6a0b05dSBen Gardon continue; 1580a6a0b05dSBen Gardon } 1581a6a0b05dSBen Gardon 1582a6a0b05dSBen Gardon tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte); 1583a6a0b05dSBen Gardon } 15847cca2d0bSBen Gardon 15857cca2d0bSBen Gardon rcu_read_unlock(); 1586a6a0b05dSBen Gardon } 1587a6a0b05dSBen Gardon 1588a6a0b05dSBen Gardon /* 1589a6a0b05dSBen Gardon * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is 1590a6a0b05dSBen Gardon * set in mask, starting at gfn. The given memslot is expected to contain all 1591a6a0b05dSBen Gardon * the GFNs represented by set bits in the mask. If AD bits are enabled, 1592a6a0b05dSBen Gardon * clearing the dirty status will involve clearing the dirty bit on each SPTE 1593a6a0b05dSBen Gardon * or, if AD bits are not enabled, clearing the writable bit on each SPTE. 1594a6a0b05dSBen Gardon */ 1595a6a0b05dSBen Gardon void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1596a6a0b05dSBen Gardon struct kvm_memory_slot *slot, 1597a6a0b05dSBen Gardon gfn_t gfn, unsigned long mask, 1598a6a0b05dSBen Gardon bool wrprot) 1599a6a0b05dSBen Gardon { 1600a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1601a6a0b05dSBen Gardon 1602531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 1603a3f15bdaSSean Christopherson for_each_tdp_mmu_root(kvm, root, slot->as_id) 1604a6a0b05dSBen Gardon clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot); 1605a6a0b05dSBen Gardon } 1606a6a0b05dSBen Gardon 1607a6a0b05dSBen Gardon /* 160887aa9ec9SBen Gardon * Clear leaf entries which could be replaced by large mappings, for 160987aa9ec9SBen Gardon * GFNs within the slot. 161014881998SBen Gardon */ 16114b85c921SSean Christopherson static void zap_collapsible_spte_range(struct kvm *kvm, 161214881998SBen Gardon struct kvm_mmu_page *root, 16134b85c921SSean Christopherson const struct kvm_memory_slot *slot) 161414881998SBen Gardon { 16159eba50f8SSean Christopherson gfn_t start = slot->base_gfn; 16169eba50f8SSean Christopherson gfn_t end = start + slot->npages; 161714881998SBen Gardon struct tdp_iter iter; 161814881998SBen Gardon kvm_pfn_t pfn; 161914881998SBen Gardon 16207cca2d0bSBen Gardon rcu_read_lock(); 16217cca2d0bSBen Gardon 162214881998SBen Gardon tdp_root_for_each_pte(iter, root, start, end) { 16232db6f772SBen Gardon retry: 16244b85c921SSean Christopherson if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 16251af4a960SBen Gardon continue; 16261af4a960SBen Gardon 162714881998SBen Gardon if (!is_shadow_present_pte(iter.old_spte) || 162887aa9ec9SBen Gardon !is_last_spte(iter.old_spte, iter.level)) 162914881998SBen Gardon continue; 163014881998SBen Gardon 163114881998SBen Gardon pfn = spte_to_pfn(iter.old_spte); 163214881998SBen Gardon if (kvm_is_reserved_pfn(pfn) || 16339eba50f8SSean Christopherson iter.level >= kvm_mmu_max_mapping_level(kvm, slot, iter.gfn, 16349eba50f8SSean Christopherson pfn, PG_LEVEL_NUM)) 163514881998SBen Gardon continue; 163614881998SBen Gardon 16374b85c921SSean Christopherson /* Note, a successful atomic zap also does a remote TLB flush. */ 16383e72c791SDavid Matlack if (tdp_mmu_zap_spte_atomic(kvm, &iter)) 16392db6f772SBen Gardon goto retry; 16402db6f772SBen Gardon } 164114881998SBen Gardon 16427cca2d0bSBen Gardon rcu_read_unlock(); 164314881998SBen Gardon } 164414881998SBen Gardon 164514881998SBen Gardon /* 164614881998SBen Gardon * Clear non-leaf entries (and free associated page tables) which could 164714881998SBen Gardon * be replaced by large mappings, for GFNs within the slot. 164814881998SBen Gardon */ 16494b85c921SSean Christopherson void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, 16504b85c921SSean Christopherson const struct kvm_memory_slot *slot) 165114881998SBen Gardon { 165214881998SBen Gardon struct kvm_mmu_page *root; 165314881998SBen Gardon 16542db6f772SBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 165514881998SBen Gardon 1656d62007edSSean Christopherson for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) 16574b85c921SSean Christopherson zap_collapsible_spte_range(kvm, root, slot); 165814881998SBen Gardon } 165946044f72SBen Gardon 166046044f72SBen Gardon /* 166146044f72SBen Gardon * Removes write access on the last level SPTE mapping this GFN and unsets the 16625fc3424fSSean Christopherson * MMU-writable bit to ensure future writes continue to be intercepted. 166346044f72SBen Gardon * Returns true if an SPTE was set and a TLB flush is needed. 166446044f72SBen Gardon */ 166546044f72SBen Gardon static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, 16663ad93562SKeqian Zhu gfn_t gfn, int min_level) 166746044f72SBen Gardon { 166846044f72SBen Gardon struct tdp_iter iter; 166946044f72SBen Gardon u64 new_spte; 167046044f72SBen Gardon bool spte_set = false; 167146044f72SBen Gardon 16723ad93562SKeqian Zhu BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 16733ad93562SKeqian Zhu 16747cca2d0bSBen Gardon rcu_read_lock(); 16757cca2d0bSBen Gardon 167677aa6075SDavid Matlack for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) { 16773ad93562SKeqian Zhu if (!is_shadow_present_pte(iter.old_spte) || 16783ad93562SKeqian Zhu !is_last_spte(iter.old_spte, iter.level)) 16793ad93562SKeqian Zhu continue; 16803ad93562SKeqian Zhu 168146044f72SBen Gardon new_spte = iter.old_spte & 16825fc3424fSSean Christopherson ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask); 168346044f72SBen Gardon 16847c8a4742SDavid Matlack if (new_spte == iter.old_spte) 16857c8a4742SDavid Matlack break; 16867c8a4742SDavid Matlack 168746044f72SBen Gardon tdp_mmu_set_spte(kvm, &iter, new_spte); 168846044f72SBen Gardon spte_set = true; 168946044f72SBen Gardon } 169046044f72SBen Gardon 16917cca2d0bSBen Gardon rcu_read_unlock(); 16927cca2d0bSBen Gardon 169346044f72SBen Gardon return spte_set; 169446044f72SBen Gardon } 169546044f72SBen Gardon 169646044f72SBen Gardon /* 169746044f72SBen Gardon * Removes write access on the last level SPTE mapping this GFN and unsets the 16985fc3424fSSean Christopherson * MMU-writable bit to ensure future writes continue to be intercepted. 169946044f72SBen Gardon * Returns true if an SPTE was set and a TLB flush is needed. 170046044f72SBen Gardon */ 170146044f72SBen Gardon bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, 17023ad93562SKeqian Zhu struct kvm_memory_slot *slot, gfn_t gfn, 17033ad93562SKeqian Zhu int min_level) 170446044f72SBen Gardon { 170546044f72SBen Gardon struct kvm_mmu_page *root; 170646044f72SBen Gardon bool spte_set = false; 170746044f72SBen Gardon 1708531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 1709a3f15bdaSSean Christopherson for_each_tdp_mmu_root(kvm, root, slot->as_id) 17103ad93562SKeqian Zhu spte_set |= write_protect_gfn(kvm, root, gfn, min_level); 1711a3f15bdaSSean Christopherson 171246044f72SBen Gardon return spte_set; 171346044f72SBen Gardon } 171446044f72SBen Gardon 171595fb5b02SBen Gardon /* 171695fb5b02SBen Gardon * Return the level of the lowest level SPTE added to sptes. 171795fb5b02SBen Gardon * That SPTE may be non-present. 1718c5c8c7c5SDavid Matlack * 1719c5c8c7c5SDavid Matlack * Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}. 172095fb5b02SBen Gardon */ 172139b4d43eSSean Christopherson int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, 172239b4d43eSSean Christopherson int *root_level) 172395fb5b02SBen Gardon { 172495fb5b02SBen Gardon struct tdp_iter iter; 172595fb5b02SBen Gardon struct kvm_mmu *mmu = vcpu->arch.mmu; 172695fb5b02SBen Gardon gfn_t gfn = addr >> PAGE_SHIFT; 17272aa07893SSean Christopherson int leaf = -1; 172895fb5b02SBen Gardon 172939b4d43eSSean Christopherson *root_level = vcpu->arch.mmu->shadow_root_level; 173095fb5b02SBen Gardon 173195fb5b02SBen Gardon tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 173295fb5b02SBen Gardon leaf = iter.level; 1733dde81f94SSean Christopherson sptes[leaf] = iter.old_spte; 173495fb5b02SBen Gardon } 173595fb5b02SBen Gardon 173695fb5b02SBen Gardon return leaf; 173795fb5b02SBen Gardon } 17386e8eb206SDavid Matlack 17396e8eb206SDavid Matlack /* 17406e8eb206SDavid Matlack * Returns the last level spte pointer of the shadow page walk for the given 17416e8eb206SDavid Matlack * gpa, and sets *spte to the spte value. This spte may be non-preset. If no 17426e8eb206SDavid Matlack * walk could be performed, returns NULL and *spte does not contain valid data. 17436e8eb206SDavid Matlack * 17446e8eb206SDavid Matlack * Contract: 17456e8eb206SDavid Matlack * - Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}. 17466e8eb206SDavid Matlack * - The returned sptep must not be used after kvm_tdp_mmu_walk_lockless_end. 17476e8eb206SDavid Matlack * 17486e8eb206SDavid Matlack * WARNING: This function is only intended to be called during fast_page_fault. 17496e8eb206SDavid Matlack */ 17506e8eb206SDavid Matlack u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, u64 addr, 17516e8eb206SDavid Matlack u64 *spte) 17526e8eb206SDavid Matlack { 17536e8eb206SDavid Matlack struct tdp_iter iter; 17546e8eb206SDavid Matlack struct kvm_mmu *mmu = vcpu->arch.mmu; 17556e8eb206SDavid Matlack gfn_t gfn = addr >> PAGE_SHIFT; 17566e8eb206SDavid Matlack tdp_ptep_t sptep = NULL; 17576e8eb206SDavid Matlack 17586e8eb206SDavid Matlack tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 17596e8eb206SDavid Matlack *spte = iter.old_spte; 17606e8eb206SDavid Matlack sptep = iter.sptep; 17616e8eb206SDavid Matlack } 17626e8eb206SDavid Matlack 17636e8eb206SDavid Matlack /* 17646e8eb206SDavid Matlack * Perform the rcu_dereference to get the raw spte pointer value since 17656e8eb206SDavid Matlack * we are passing it up to fast_page_fault, which is shared with the 17666e8eb206SDavid Matlack * legacy MMU and thus does not retain the TDP MMU-specific __rcu 17676e8eb206SDavid Matlack * annotation. 17686e8eb206SDavid Matlack * 17696e8eb206SDavid Matlack * This is safe since fast_page_fault obeys the contracts of this 17706e8eb206SDavid Matlack * function as well as all TDP MMU contracts around modifying SPTEs 17716e8eb206SDavid Matlack * outside of mmu_lock. 17726e8eb206SDavid Matlack */ 17736e8eb206SDavid Matlack return rcu_dereference(sptep); 17746e8eb206SDavid Matlack } 1775