1fe5db27dSBen Gardon // SPDX-License-Identifier: GPL-2.0 28d20bd63SSean Christopherson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3fe5db27dSBen Gardon 402c00b3aSBen Gardon #include "mmu.h" 502c00b3aSBen Gardon #include "mmu_internal.h" 6bb18842eSBen Gardon #include "mmutrace.h" 72f2fad08SBen Gardon #include "tdp_iter.h" 8fe5db27dSBen Gardon #include "tdp_mmu.h" 902c00b3aSBen Gardon #include "spte.h" 10fe5db27dSBen Gardon 119a77daacSBen Gardon #include <asm/cmpxchg.h> 1233dd3574SBen Gardon #include <trace/events/kvm.h> 1333dd3574SBen Gardon 14fe5db27dSBen Gardon /* Initializes the TDP MMU for the VM, if enabled. */ 15a1a39128SPaolo Bonzini int kvm_mmu_init_tdp_mmu(struct kvm *kvm) 16fe5db27dSBen Gardon { 17a1a39128SPaolo Bonzini struct workqueue_struct *wq; 18a1a39128SPaolo Bonzini 19a1a39128SPaolo Bonzini wq = alloc_workqueue("kvm", WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 0); 20a1a39128SPaolo Bonzini if (!wq) 21a1a39128SPaolo Bonzini return -ENOMEM; 22fe5db27dSBen Gardon 2302c00b3aSBen Gardon INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); 249a77daacSBen Gardon spin_lock_init(&kvm->arch.tdp_mmu_pages_lock); 25a1a39128SPaolo Bonzini kvm->arch.tdp_mmu_zap_wq = wq; 26a1a39128SPaolo Bonzini return 1; 27fe5db27dSBen Gardon } 28fe5db27dSBen Gardon 29226b8c8fSSean Christopherson /* Arbitrarily returns true so that this may be used in if statements. */ 30226b8c8fSSean Christopherson static __always_inline bool kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm, 316103bc07SBen Gardon bool shared) 326103bc07SBen Gardon { 336103bc07SBen Gardon if (shared) 346103bc07SBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 356103bc07SBen Gardon else 366103bc07SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 37226b8c8fSSean Christopherson 38226b8c8fSSean Christopherson return true; 396103bc07SBen Gardon } 406103bc07SBen Gardon 41fe5db27dSBen Gardon void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) 42fe5db27dSBen Gardon { 433203a56aSLv Ruyi /* Also waits for any queued work items. */ 4422b94c4bSPaolo Bonzini destroy_workqueue(kvm->arch.tdp_mmu_zap_wq); 4522b94c4bSPaolo Bonzini 46d25ceb92SSean Christopherson WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages)); 4702c00b3aSBen Gardon WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots)); 487cca2d0bSBen Gardon 497cca2d0bSBen Gardon /* 507cca2d0bSBen Gardon * Ensure that all the outstanding RCU callbacks to free shadow pages 5122b94c4bSPaolo Bonzini * can run before the VM is torn down. Work items on tdp_mmu_zap_wq 5222b94c4bSPaolo Bonzini * can call kvm_tdp_mmu_put_root and create new callbacks. 537cca2d0bSBen Gardon */ 547cca2d0bSBen Gardon rcu_barrier(); 5502c00b3aSBen Gardon } 5602c00b3aSBen Gardon 572bdb3d84SBen Gardon static void tdp_mmu_free_sp(struct kvm_mmu_page *sp) 58a889ea54SBen Gardon { 592bdb3d84SBen Gardon free_page((unsigned long)sp->spt); 602bdb3d84SBen Gardon kmem_cache_free(mmu_page_header_cache, sp); 61a889ea54SBen Gardon } 62a889ea54SBen Gardon 63c0e64238SBen Gardon /* 64c0e64238SBen Gardon * This is called through call_rcu in order to free TDP page table memory 65c0e64238SBen Gardon * safely with respect to other kernel threads that may be operating on 66c0e64238SBen Gardon * the memory. 67c0e64238SBen Gardon * By only accessing TDP MMU page table memory in an RCU read critical 68c0e64238SBen Gardon * section, and freeing it after a grace period, lockless access to that 69c0e64238SBen Gardon * memory won't use it after it is freed. 70c0e64238SBen Gardon */ 71c0e64238SBen Gardon static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) 72a889ea54SBen Gardon { 73c0e64238SBen Gardon struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page, 74c0e64238SBen Gardon rcu_head); 75a889ea54SBen Gardon 76c0e64238SBen Gardon tdp_mmu_free_sp(sp); 77a889ea54SBen Gardon } 78a889ea54SBen Gardon 79e2b5b21dSSean Christopherson static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root, 80e2b5b21dSSean Christopherson bool shared); 81e2b5b21dSSean Christopherson 8222b94c4bSPaolo Bonzini static void tdp_mmu_zap_root_work(struct work_struct *work) 8322b94c4bSPaolo Bonzini { 8422b94c4bSPaolo Bonzini struct kvm_mmu_page *root = container_of(work, struct kvm_mmu_page, 8522b94c4bSPaolo Bonzini tdp_mmu_async_work); 8622b94c4bSPaolo Bonzini struct kvm *kvm = root->tdp_mmu_async_data; 8722b94c4bSPaolo Bonzini 8822b94c4bSPaolo Bonzini read_lock(&kvm->mmu_lock); 8922b94c4bSPaolo Bonzini 9022b94c4bSPaolo Bonzini /* 9122b94c4bSPaolo Bonzini * A TLB flush is not necessary as KVM performs a local TLB flush when 9222b94c4bSPaolo Bonzini * allocating a new root (see kvm_mmu_load()), and when migrating vCPU 9322b94c4bSPaolo Bonzini * to a different pCPU. Note, the local TLB flush on reuse also 9422b94c4bSPaolo Bonzini * invalidates any paging-structure-cache entries, i.e. TLB entries for 9522b94c4bSPaolo Bonzini * intermediate paging structures, that may be zapped, as such entries 9622b94c4bSPaolo Bonzini * are associated with the ASID on both VMX and SVM. 9722b94c4bSPaolo Bonzini */ 9822b94c4bSPaolo Bonzini tdp_mmu_zap_root(kvm, root, true); 9922b94c4bSPaolo Bonzini 10022b94c4bSPaolo Bonzini /* 10122b94c4bSPaolo Bonzini * Drop the refcount using kvm_tdp_mmu_put_root() to test its logic for 10222b94c4bSPaolo Bonzini * avoiding an infinite loop. By design, the root is reachable while 10322b94c4bSPaolo Bonzini * it's being asynchronously zapped, thus a different task can put its 10422b94c4bSPaolo Bonzini * last reference, i.e. flowing through kvm_tdp_mmu_put_root() for an 10522b94c4bSPaolo Bonzini * asynchronously zapped root is unavoidable. 10622b94c4bSPaolo Bonzini */ 10722b94c4bSPaolo Bonzini kvm_tdp_mmu_put_root(kvm, root, true); 10822b94c4bSPaolo Bonzini 10922b94c4bSPaolo Bonzini read_unlock(&kvm->mmu_lock); 11022b94c4bSPaolo Bonzini } 11122b94c4bSPaolo Bonzini 11222b94c4bSPaolo Bonzini static void tdp_mmu_schedule_zap_root(struct kvm *kvm, struct kvm_mmu_page *root) 11322b94c4bSPaolo Bonzini { 11422b94c4bSPaolo Bonzini root->tdp_mmu_async_data = kvm; 11522b94c4bSPaolo Bonzini INIT_WORK(&root->tdp_mmu_async_work, tdp_mmu_zap_root_work); 11622b94c4bSPaolo Bonzini queue_work(kvm->arch.tdp_mmu_zap_wq, &root->tdp_mmu_async_work); 11722b94c4bSPaolo Bonzini } 11822b94c4bSPaolo Bonzini 1198351779cSPaolo Bonzini static inline bool kvm_tdp_root_mark_invalid(struct kvm_mmu_page *page) 1208351779cSPaolo Bonzini { 1218351779cSPaolo Bonzini union kvm_mmu_page_role role = page->role; 1228351779cSPaolo Bonzini role.invalid = true; 1238351779cSPaolo Bonzini 1248351779cSPaolo Bonzini /* No need to use cmpxchg, only the invalid bit can change. */ 1258351779cSPaolo Bonzini role.word = xchg(&page->role.word, role.word); 1268351779cSPaolo Bonzini return role.invalid; 1278351779cSPaolo Bonzini } 1288351779cSPaolo Bonzini 1296103bc07SBen Gardon void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root, 1306103bc07SBen Gardon bool shared) 1312bdb3d84SBen Gardon { 1326103bc07SBen Gardon kvm_lockdep_assert_mmu_lock_held(kvm, shared); 1332bdb3d84SBen Gardon 13411cccf5cSBen Gardon if (!refcount_dec_and_test(&root->tdp_mmu_root_count)) 1352bdb3d84SBen Gardon return; 1362bdb3d84SBen Gardon 137de0322f5SSean Christopherson WARN_ON(!is_tdp_mmu_page(root)); 1382bdb3d84SBen Gardon 1398351779cSPaolo Bonzini /* 1408351779cSPaolo Bonzini * The root now has refcount=0. It is valid, but readers already 1418351779cSPaolo Bonzini * cannot acquire a reference to it because kvm_tdp_mmu_get_root() 1428351779cSPaolo Bonzini * rejects it. This remains true for the rest of the execution 1438351779cSPaolo Bonzini * of this function, because readers visit valid roots only 1448351779cSPaolo Bonzini * (except for tdp_mmu_zap_root_work(), which however 1458351779cSPaolo Bonzini * does not acquire any reference itself). 1468351779cSPaolo Bonzini * 1478351779cSPaolo Bonzini * Even though there are flows that need to visit all roots for 1488351779cSPaolo Bonzini * correctness, they all take mmu_lock for write, so they cannot yet 1498351779cSPaolo Bonzini * run concurrently. The same is true after kvm_tdp_root_mark_invalid, 1508351779cSPaolo Bonzini * since the root still has refcount=0. 1518351779cSPaolo Bonzini * 1528351779cSPaolo Bonzini * However, tdp_mmu_zap_root can yield, and writers do not expect to 1538351779cSPaolo Bonzini * see refcount=0 (see for example kvm_tdp_mmu_invalidate_all_roots()). 1548351779cSPaolo Bonzini * So the root temporarily gets an extra reference, going to refcount=1 1558351779cSPaolo Bonzini * while staying invalid. Readers still cannot acquire any reference; 1568351779cSPaolo Bonzini * but writers are now allowed to run if tdp_mmu_zap_root yields and 157efd995daSPaolo Bonzini * they might take an extra reference if they themselves yield. 158efd995daSPaolo Bonzini * Therefore, when the reference is given back by the worker, 1598351779cSPaolo Bonzini * there is no guarantee that the refcount is still 1. If not, whoever 1608351779cSPaolo Bonzini * puts the last reference will free the page, but they will not have to 1618351779cSPaolo Bonzini * zap the root because a root cannot go from invalid to valid. 1628351779cSPaolo Bonzini */ 1638351779cSPaolo Bonzini if (!kvm_tdp_root_mark_invalid(root)) { 1648351779cSPaolo Bonzini refcount_set(&root->tdp_mmu_root_count, 1); 1658351779cSPaolo Bonzini 1668351779cSPaolo Bonzini /* 167efd995daSPaolo Bonzini * Zapping the root in a worker is not just "nice to have"; 168efd995daSPaolo Bonzini * it is required because kvm_tdp_mmu_invalidate_all_roots() 169efd995daSPaolo Bonzini * skips already-invalid roots. If kvm_tdp_mmu_put_root() did 170efd995daSPaolo Bonzini * not add the root to the workqueue, kvm_tdp_mmu_zap_all_fast() 171efd995daSPaolo Bonzini * might return with some roots not zapped yet. 1728351779cSPaolo Bonzini */ 173efd995daSPaolo Bonzini tdp_mmu_schedule_zap_root(kvm, root); 1748351779cSPaolo Bonzini return; 1758351779cSPaolo Bonzini } 1768351779cSPaolo Bonzini 177c0e64238SBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 178c0e64238SBen Gardon list_del_rcu(&root->link); 179c0e64238SBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 180c0e64238SBen Gardon call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback); 181a889ea54SBen Gardon } 182a889ea54SBen Gardon 183cfc10997SBen Gardon /* 184d62007edSSean Christopherson * Returns the next root after @prev_root (or the first root if @prev_root is 185d62007edSSean Christopherson * NULL). A reference to the returned root is acquired, and the reference to 186d62007edSSean Christopherson * @prev_root is released (the caller obviously must hold a reference to 187d62007edSSean Christopherson * @prev_root if it's non-NULL). 188d62007edSSean Christopherson * 189d62007edSSean Christopherson * If @only_valid is true, invalid roots are skipped. 190d62007edSSean Christopherson * 191d62007edSSean Christopherson * Returns NULL if the end of tdp_mmu_roots was reached. 192cfc10997SBen Gardon */ 193cfc10997SBen Gardon static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, 1946103bc07SBen Gardon struct kvm_mmu_page *prev_root, 195d62007edSSean Christopherson bool shared, bool only_valid) 196a889ea54SBen Gardon { 197a889ea54SBen Gardon struct kvm_mmu_page *next_root; 198a889ea54SBen Gardon 199c0e64238SBen Gardon rcu_read_lock(); 200c0e64238SBen Gardon 201cfc10997SBen Gardon if (prev_root) 202c0e64238SBen Gardon next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 203c0e64238SBen Gardon &prev_root->link, 204c0e64238SBen Gardon typeof(*prev_root), link); 205cfc10997SBen Gardon else 206c0e64238SBen Gardon next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots, 207cfc10997SBen Gardon typeof(*next_root), link); 208cfc10997SBen Gardon 20904dc4e6cSSean Christopherson while (next_root) { 210d62007edSSean Christopherson if ((!only_valid || !next_root->role.invalid) && 211ad6d6b94SJinrong Liang kvm_tdp_mmu_get_root(next_root)) 21204dc4e6cSSean Christopherson break; 21304dc4e6cSSean Christopherson 214c0e64238SBen Gardon next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 215c0e64238SBen Gardon &next_root->link, typeof(*next_root), link); 21604dc4e6cSSean Christopherson } 217fb101293SBen Gardon 218c0e64238SBen Gardon rcu_read_unlock(); 219cfc10997SBen Gardon 220cfc10997SBen Gardon if (prev_root) 2216103bc07SBen Gardon kvm_tdp_mmu_put_root(kvm, prev_root, shared); 222cfc10997SBen Gardon 223a889ea54SBen Gardon return next_root; 224a889ea54SBen Gardon } 225a889ea54SBen Gardon 226a889ea54SBen Gardon /* 227a889ea54SBen Gardon * Note: this iterator gets and puts references to the roots it iterates over. 228a889ea54SBen Gardon * This makes it safe to release the MMU lock and yield within the loop, but 229a889ea54SBen Gardon * if exiting the loop early, the caller must drop the reference to the most 230a889ea54SBen Gardon * recent root. (Unless keeping a live reference is desirable.) 2316103bc07SBen Gardon * 2326103bc07SBen Gardon * If shared is set, this function is operating under the MMU lock in read 2336103bc07SBen Gardon * mode. In the unlikely event that this thread must free a root, the lock 2346103bc07SBen Gardon * will be temporarily dropped and reacquired in write mode. 235a889ea54SBen Gardon */ 236d62007edSSean Christopherson #define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, _only_valid)\ 237d62007edSSean Christopherson for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, _only_valid); \ 238cfc10997SBen Gardon _root; \ 239d62007edSSean Christopherson _root = tdp_mmu_next_root(_kvm, _root, _shared, _only_valid)) \ 240614f6970SPaolo Bonzini if (kvm_lockdep_assert_mmu_lock_held(_kvm, _shared) && \ 241614f6970SPaolo Bonzini kvm_mmu_page_as_id(_root) != _as_id) { \ 242a3f15bdaSSean Christopherson } else 243a889ea54SBen Gardon 244d62007edSSean Christopherson #define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \ 245d62007edSSean Christopherson __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true) 246d62007edSSean Christopherson 247614f6970SPaolo Bonzini #define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id) \ 248614f6970SPaolo Bonzini __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, false, false) 249d62007edSSean Christopherson 250226b8c8fSSean Christopherson /* 251226b8c8fSSean Christopherson * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write, 252226b8c8fSSean Christopherson * the implication being that any flow that holds mmu_lock for read is 253226b8c8fSSean Christopherson * inherently yield-friendly and should use the yield-safe variant above. 254226b8c8fSSean Christopherson * Holding mmu_lock for write obviates the need for RCU protection as the list 255226b8c8fSSean Christopherson * is guaranteed to be stable. 256226b8c8fSSean Christopherson */ 257a3f15bdaSSean Christopherson #define for_each_tdp_mmu_root(_kvm, _root, _as_id) \ 258226b8c8fSSean Christopherson list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link) \ 259226b8c8fSSean Christopherson if (kvm_lockdep_assert_mmu_lock_held(_kvm, false) && \ 260226b8c8fSSean Christopherson kvm_mmu_page_as_id(_root) != _as_id) { \ 261a3f15bdaSSean Christopherson } else 26202c00b3aSBen Gardon 263a82070b6SDavid Matlack static struct kvm_mmu_page *tdp_mmu_alloc_sp(struct kvm_vcpu *vcpu) 26402c00b3aSBen Gardon { 26502c00b3aSBen Gardon struct kvm_mmu_page *sp; 26602c00b3aSBen Gardon 26702c00b3aSBen Gardon sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 26802c00b3aSBen Gardon sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 269a82070b6SDavid Matlack 270a82070b6SDavid Matlack return sp; 271a82070b6SDavid Matlack } 272a82070b6SDavid Matlack 273c10743a1SSean Christopherson static void tdp_mmu_init_sp(struct kvm_mmu_page *sp, tdp_ptep_t sptep, 274c10743a1SSean Christopherson gfn_t gfn, union kvm_mmu_page_role role) 275a82070b6SDavid Matlack { 27655c510e2SSean Christopherson INIT_LIST_HEAD(&sp->possible_nx_huge_page_link); 277428e9216SSean Christopherson 27802c00b3aSBen Gardon set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 27902c00b3aSBen Gardon 280a3aca4deSDavid Matlack sp->role = role; 28102c00b3aSBen Gardon sp->gfn = gfn; 282c10743a1SSean Christopherson sp->ptep = sptep; 28302c00b3aSBen Gardon sp->tdp_mmu_page = true; 28402c00b3aSBen Gardon 28533dd3574SBen Gardon trace_kvm_mmu_get_page(sp, true); 28602c00b3aSBen Gardon } 28702c00b3aSBen Gardon 288a82070b6SDavid Matlack static void tdp_mmu_init_child_sp(struct kvm_mmu_page *child_sp, 289a3aca4deSDavid Matlack struct tdp_iter *iter) 290a3aca4deSDavid Matlack { 291a3aca4deSDavid Matlack struct kvm_mmu_page *parent_sp; 292a3aca4deSDavid Matlack union kvm_mmu_page_role role; 293a3aca4deSDavid Matlack 294a3aca4deSDavid Matlack parent_sp = sptep_to_sp(rcu_dereference(iter->sptep)); 295a3aca4deSDavid Matlack 296a3aca4deSDavid Matlack role = parent_sp->role; 297a3aca4deSDavid Matlack role.level--; 298a3aca4deSDavid Matlack 299c10743a1SSean Christopherson tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role); 300a3aca4deSDavid Matlack } 301a3aca4deSDavid Matlack 3026e6ec584SSean Christopherson hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) 30302c00b3aSBen Gardon { 3047a458f0eSPaolo Bonzini union kvm_mmu_page_role role = vcpu->arch.mmu->root_role; 30502c00b3aSBen Gardon struct kvm *kvm = vcpu->kvm; 30602c00b3aSBen Gardon struct kvm_mmu_page *root; 30702c00b3aSBen Gardon 3086e6ec584SSean Christopherson lockdep_assert_held_write(&kvm->mmu_lock); 30902c00b3aSBen Gardon 31004dc4e6cSSean Christopherson /* 31104dc4e6cSSean Christopherson * Check for an existing root before allocating a new one. Note, the 31204dc4e6cSSean Christopherson * role check prevents consuming an invalid root. 31304dc4e6cSSean Christopherson */ 314a3f15bdaSSean Christopherson for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) { 315fb101293SBen Gardon if (root->role.word == role.word && 316ad6d6b94SJinrong Liang kvm_tdp_mmu_get_root(root)) 3176e6ec584SSean Christopherson goto out; 31802c00b3aSBen Gardon } 31902c00b3aSBen Gardon 320a82070b6SDavid Matlack root = tdp_mmu_alloc_sp(vcpu); 321c10743a1SSean Christopherson tdp_mmu_init_sp(root, NULL, 0, role); 322a82070b6SDavid Matlack 32311cccf5cSBen Gardon refcount_set(&root->tdp_mmu_root_count, 1); 32402c00b3aSBen Gardon 325c0e64238SBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 326c0e64238SBen Gardon list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots); 327c0e64238SBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 32802c00b3aSBen Gardon 3296e6ec584SSean Christopherson out: 33002c00b3aSBen Gardon return __pa(root->spt); 331fe5db27dSBen Gardon } 3322f2fad08SBen Gardon 3332f2fad08SBen Gardon static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 3349a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 3359a77daacSBen Gardon bool shared); 3362f2fad08SBen Gardon 337f8e14497SBen Gardon static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level) 338f8e14497SBen Gardon { 339f8e14497SBen Gardon if (!is_shadow_present_pte(old_spte) || !is_last_spte(old_spte, level)) 340f8e14497SBen Gardon return; 341f8e14497SBen Gardon 342f8e14497SBen Gardon if (is_accessed_spte(old_spte) && 34364bb2769SSean Christopherson (!is_shadow_present_pte(new_spte) || !is_accessed_spte(new_spte) || 34464bb2769SSean Christopherson spte_to_pfn(old_spte) != spte_to_pfn(new_spte))) 345f8e14497SBen Gardon kvm_set_pfn_accessed(spte_to_pfn(old_spte)); 346f8e14497SBen Gardon } 347f8e14497SBen Gardon 348a6a0b05dSBen Gardon static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, 349a6a0b05dSBen Gardon u64 old_spte, u64 new_spte, int level) 350a6a0b05dSBen Gardon { 351a6a0b05dSBen Gardon bool pfn_changed; 352a6a0b05dSBen Gardon struct kvm_memory_slot *slot; 353a6a0b05dSBen Gardon 354a6a0b05dSBen Gardon if (level > PG_LEVEL_4K) 355a6a0b05dSBen Gardon return; 356a6a0b05dSBen Gardon 357a6a0b05dSBen Gardon pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 358a6a0b05dSBen Gardon 359a6a0b05dSBen Gardon if ((!is_writable_pte(old_spte) || pfn_changed) && 360a6a0b05dSBen Gardon is_writable_pte(new_spte)) { 361a6a0b05dSBen Gardon slot = __gfn_to_memslot(__kvm_memslots(kvm, as_id), gfn); 362fb04a1edSPeter Xu mark_page_dirty_in_slot(kvm, slot, gfn); 363a6a0b05dSBen Gardon } 364a6a0b05dSBen Gardon } 365a6a0b05dSBen Gardon 36643a063caSYosry Ahmed static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) 36743a063caSYosry Ahmed { 36843a063caSYosry Ahmed kvm_account_pgtable_pages((void *)sp->spt, +1); 369d25ceb92SSean Christopherson atomic64_inc(&kvm->arch.tdp_mmu_pages); 37043a063caSYosry Ahmed } 37143a063caSYosry Ahmed 37243a063caSYosry Ahmed static void tdp_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) 37343a063caSYosry Ahmed { 37443a063caSYosry Ahmed kvm_account_pgtable_pages((void *)sp->spt, -1); 375d25ceb92SSean Christopherson atomic64_dec(&kvm->arch.tdp_mmu_pages); 37643a063caSYosry Ahmed } 37743a063caSYosry Ahmed 3782f2fad08SBen Gardon /** 379c298a30cSDavid Matlack * tdp_mmu_unlink_sp() - Remove a shadow page from the list of used pages 380a9442f59SBen Gardon * 381a9442f59SBen Gardon * @kvm: kvm instance 382a9442f59SBen Gardon * @sp: the page to be removed 3839a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 3849a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 3859a77daacSBen Gardon * threads that might be adding or removing pages. 386a9442f59SBen Gardon */ 387c298a30cSDavid Matlack static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp, 3889a77daacSBen Gardon bool shared) 389a9442f59SBen Gardon { 39043a063caSYosry Ahmed tdp_unaccount_mmu_page(kvm, sp); 391d25ceb92SSean Christopherson 392d25ceb92SSean Christopherson if (!sp->nx_huge_page_disallowed) 393d25ceb92SSean Christopherson return; 394d25ceb92SSean Christopherson 3959a77daacSBen Gardon if (shared) 3969a77daacSBen Gardon spin_lock(&kvm->arch.tdp_mmu_pages_lock); 3979a77daacSBen Gardon else 398a9442f59SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 399a9442f59SBen Gardon 40061f94478SSean Christopherson sp->nx_huge_page_disallowed = false; 40161f94478SSean Christopherson untrack_possible_nx_huge_page(kvm, sp); 4029a77daacSBen Gardon 4039a77daacSBen Gardon if (shared) 4049a77daacSBen Gardon spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 405a9442f59SBen Gardon } 406a9442f59SBen Gardon 407a9442f59SBen Gardon /** 4080f53dfa3SDavid Matlack * handle_removed_pt() - handle a page table removed from the TDP structure 409a066e61fSBen Gardon * 410a066e61fSBen Gardon * @kvm: kvm instance 411a066e61fSBen Gardon * @pt: the page removed from the paging structure 4129a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use 4139a77daacSBen Gardon * of the MMU lock and the operation must synchronize with other 4149a77daacSBen Gardon * threads that might be modifying SPTEs. 415a066e61fSBen Gardon * 416a066e61fSBen Gardon * Given a page table that has been removed from the TDP paging structure, 417a066e61fSBen Gardon * iterates through the page table to clear SPTEs and free child page tables. 41870fb3e41SBen Gardon * 41970fb3e41SBen Gardon * Note that pt is passed in as a tdp_ptep_t, but it does not need RCU 42070fb3e41SBen Gardon * protection. Since this thread removed it from the paging structure, 42170fb3e41SBen Gardon * this thread will be responsible for ensuring the page is freed. Hence the 42270fb3e41SBen Gardon * early rcu_dereferences in the function. 423a066e61fSBen Gardon */ 4240f53dfa3SDavid Matlack static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) 425a066e61fSBen Gardon { 42670fb3e41SBen Gardon struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(pt)); 427a066e61fSBen Gardon int level = sp->role.level; 428e25f0e0cSBen Gardon gfn_t base_gfn = sp->gfn; 429a066e61fSBen Gardon int i; 430a066e61fSBen Gardon 431a066e61fSBen Gardon trace_kvm_mmu_prepare_zap_page(sp); 432a066e61fSBen Gardon 433c298a30cSDavid Matlack tdp_mmu_unlink_sp(kvm, sp, shared); 434a066e61fSBen Gardon 4352ca3129eSSean Christopherson for (i = 0; i < SPTE_ENT_PER_PAGE; i++) { 436ba3a6120SSean Christopherson tdp_ptep_t sptep = pt + i; 437574c3c55SBen Gardon gfn_t gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level); 438ba3a6120SSean Christopherson u64 old_spte; 4399a77daacSBen Gardon 4409a77daacSBen Gardon if (shared) { 441e25f0e0cSBen Gardon /* 442e25f0e0cSBen Gardon * Set the SPTE to a nonpresent value that other 443e25f0e0cSBen Gardon * threads will not overwrite. If the SPTE was 444e25f0e0cSBen Gardon * already marked as removed then another thread 445e25f0e0cSBen Gardon * handling a page fault could overwrite it, so 446e25f0e0cSBen Gardon * set the SPTE until it is set from some other 447e25f0e0cSBen Gardon * value to the removed SPTE value. 448e25f0e0cSBen Gardon */ 449e25f0e0cSBen Gardon for (;;) { 450ba3a6120SSean Christopherson old_spte = kvm_tdp_mmu_write_spte_atomic(sptep, REMOVED_SPTE); 451ba3a6120SSean Christopherson if (!is_removed_spte(old_spte)) 452e25f0e0cSBen Gardon break; 453e25f0e0cSBen Gardon cpu_relax(); 454e25f0e0cSBen Gardon } 4559a77daacSBen Gardon } else { 4568df9f1afSSean Christopherson /* 4578df9f1afSSean Christopherson * If the SPTE is not MMU-present, there is no backing 4588df9f1afSSean Christopherson * page associated with the SPTE and so no side effects 4598df9f1afSSean Christopherson * that need to be recorded, and exclusive ownership of 4608df9f1afSSean Christopherson * mmu_lock ensures the SPTE can't be made present. 4618df9f1afSSean Christopherson * Note, zapping MMIO SPTEs is also unnecessary as they 4628df9f1afSSean Christopherson * are guarded by the memslots generation, not by being 4638df9f1afSSean Christopherson * unreachable. 4648df9f1afSSean Christopherson */ 465ba3a6120SSean Christopherson old_spte = kvm_tdp_mmu_read_spte(sptep); 466ba3a6120SSean Christopherson if (!is_shadow_present_pte(old_spte)) 4678df9f1afSSean Christopherson continue; 468e25f0e0cSBen Gardon 469e25f0e0cSBen Gardon /* 470ba3a6120SSean Christopherson * Use the common helper instead of a raw WRITE_ONCE as 471ba3a6120SSean Christopherson * the SPTE needs to be updated atomically if it can be 472ba3a6120SSean Christopherson * modified by a different vCPU outside of mmu_lock. 473ba3a6120SSean Christopherson * Even though the parent SPTE is !PRESENT, the TLB 474ba3a6120SSean Christopherson * hasn't yet been flushed, and both Intel and AMD 475ba3a6120SSean Christopherson * document that A/D assists can use upper-level PxE 476ba3a6120SSean Christopherson * entries that are cached in the TLB, i.e. the CPU can 477ba3a6120SSean Christopherson * still access the page and mark it dirty. 478ba3a6120SSean Christopherson * 479ba3a6120SSean Christopherson * No retry is needed in the atomic update path as the 480ba3a6120SSean Christopherson * sole concern is dropping a Dirty bit, i.e. no other 481ba3a6120SSean Christopherson * task can zap/remove the SPTE as mmu_lock is held for 482ba3a6120SSean Christopherson * write. Marking the SPTE as a removed SPTE is not 483ba3a6120SSean Christopherson * strictly necessary for the same reason, but using 484ba3a6120SSean Christopherson * the remove SPTE value keeps the shared/exclusive 485ba3a6120SSean Christopherson * paths consistent and allows the handle_changed_spte() 486ba3a6120SSean Christopherson * call below to hardcode the new value to REMOVED_SPTE. 487ba3a6120SSean Christopherson * 488ba3a6120SSean Christopherson * Note, even though dropping a Dirty bit is the only 489ba3a6120SSean Christopherson * scenario where a non-atomic update could result in a 490ba3a6120SSean Christopherson * functional bug, simply checking the Dirty bit isn't 491ba3a6120SSean Christopherson * sufficient as a fast page fault could read the upper 492ba3a6120SSean Christopherson * level SPTE before it is zapped, and then make this 493ba3a6120SSean Christopherson * target SPTE writable, resume the guest, and set the 494ba3a6120SSean Christopherson * Dirty bit between reading the SPTE above and writing 495ba3a6120SSean Christopherson * it here. 496e25f0e0cSBen Gardon */ 497ba3a6120SSean Christopherson old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, 498ba3a6120SSean Christopherson REMOVED_SPTE, level); 4999a77daacSBen Gardon } 500e25f0e0cSBen Gardon handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn, 501ba3a6120SSean Christopherson old_spte, REMOVED_SPTE, level, shared); 502a066e61fSBen Gardon } 503a066e61fSBen Gardon 5047cca2d0bSBen Gardon call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); 505a066e61fSBen Gardon } 506a066e61fSBen Gardon 507a066e61fSBen Gardon /** 5087f6231a3SKai Huang * __handle_changed_spte - handle bookkeeping associated with an SPTE change 5092f2fad08SBen Gardon * @kvm: kvm instance 5102f2fad08SBen Gardon * @as_id: the address space of the paging structure the SPTE was a part of 5112f2fad08SBen Gardon * @gfn: the base GFN that was mapped by the SPTE 5122f2fad08SBen Gardon * @old_spte: The value of the SPTE before the change 5132f2fad08SBen Gardon * @new_spte: The value of the SPTE after the change 5142f2fad08SBen Gardon * @level: the level of the PT the SPTE is part of in the paging structure 5159a77daacSBen Gardon * @shared: This operation may not be running under the exclusive use of 5169a77daacSBen Gardon * the MMU lock and the operation must synchronize with other 5179a77daacSBen Gardon * threads that might be modifying SPTEs. 5182f2fad08SBen Gardon * 5192f2fad08SBen Gardon * Handle bookkeeping that might result from the modification of a SPTE. 5202f2fad08SBen Gardon */ 5212f2fad08SBen Gardon static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 5229a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 5239a77daacSBen Gardon bool shared) 5242f2fad08SBen Gardon { 5252f2fad08SBen Gardon bool was_present = is_shadow_present_pte(old_spte); 5262f2fad08SBen Gardon bool is_present = is_shadow_present_pte(new_spte); 5272f2fad08SBen Gardon bool was_leaf = was_present && is_last_spte(old_spte, level); 5282f2fad08SBen Gardon bool is_leaf = is_present && is_last_spte(new_spte, level); 5292f2fad08SBen Gardon bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 5302f2fad08SBen Gardon 5312f2fad08SBen Gardon WARN_ON(level > PT64_ROOT_MAX_LEVEL); 5322f2fad08SBen Gardon WARN_ON(level < PG_LEVEL_4K); 533764388ceSSean Christopherson WARN_ON(gfn & (KVM_PAGES_PER_HPAGE(level) - 1)); 5342f2fad08SBen Gardon 5352f2fad08SBen Gardon /* 5362f2fad08SBen Gardon * If this warning were to trigger it would indicate that there was a 5372f2fad08SBen Gardon * missing MMU notifier or a race with some notifier handler. 5382f2fad08SBen Gardon * A present, leaf SPTE should never be directly replaced with another 539d9f6e12fSIngo Molnar * present leaf SPTE pointing to a different PFN. A notifier handler 5402f2fad08SBen Gardon * should be zapping the SPTE before the main MM's page table is 5412f2fad08SBen Gardon * changed, or the SPTE should be zeroed, and the TLBs flushed by the 5422f2fad08SBen Gardon * thread before replacement. 5432f2fad08SBen Gardon */ 5442f2fad08SBen Gardon if (was_leaf && is_leaf && pfn_changed) { 5452f2fad08SBen Gardon pr_err("Invalid SPTE change: cannot replace a present leaf\n" 5462f2fad08SBen Gardon "SPTE with another present leaf SPTE mapping a\n" 5472f2fad08SBen Gardon "different PFN!\n" 5482f2fad08SBen Gardon "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 5492f2fad08SBen Gardon as_id, gfn, old_spte, new_spte, level); 5502f2fad08SBen Gardon 5512f2fad08SBen Gardon /* 5522f2fad08SBen Gardon * Crash the host to prevent error propagation and guest data 553d9f6e12fSIngo Molnar * corruption. 5542f2fad08SBen Gardon */ 5552f2fad08SBen Gardon BUG(); 5562f2fad08SBen Gardon } 5572f2fad08SBen Gardon 5582f2fad08SBen Gardon if (old_spte == new_spte) 5592f2fad08SBen Gardon return; 5602f2fad08SBen Gardon 561b9a98c34SBen Gardon trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); 562b9a98c34SBen Gardon 563115111efSDavid Matlack if (is_leaf) 564115111efSDavid Matlack check_spte_writable_invariants(new_spte); 565115111efSDavid Matlack 5662f2fad08SBen Gardon /* 5672f2fad08SBen Gardon * The only times a SPTE should be changed from a non-present to 5682f2fad08SBen Gardon * non-present state is when an MMIO entry is installed/modified/ 5692f2fad08SBen Gardon * removed. In that case, there is nothing to do here. 5702f2fad08SBen Gardon */ 5712f2fad08SBen Gardon if (!was_present && !is_present) { 5722f2fad08SBen Gardon /* 57308f07c80SBen Gardon * If this change does not involve a MMIO SPTE or removed SPTE, 57408f07c80SBen Gardon * it is unexpected. Log the change, though it should not 57508f07c80SBen Gardon * impact the guest since both the former and current SPTEs 57608f07c80SBen Gardon * are nonpresent. 5772f2fad08SBen Gardon */ 57808f07c80SBen Gardon if (WARN_ON(!is_mmio_spte(old_spte) && 57908f07c80SBen Gardon !is_mmio_spte(new_spte) && 58008f07c80SBen Gardon !is_removed_spte(new_spte))) 5812f2fad08SBen Gardon pr_err("Unexpected SPTE change! Nonpresent SPTEs\n" 5822f2fad08SBen Gardon "should not be replaced with another,\n" 5832f2fad08SBen Gardon "different nonpresent SPTE, unless one or both\n" 58408f07c80SBen Gardon "are MMIO SPTEs, or the new SPTE is\n" 58508f07c80SBen Gardon "a temporary removed SPTE.\n" 5862f2fad08SBen Gardon "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 5872f2fad08SBen Gardon as_id, gfn, old_spte, new_spte, level); 5882f2fad08SBen Gardon return; 5892f2fad08SBen Gardon } 5902f2fad08SBen Gardon 59171f51d2cSMingwei Zhang if (is_leaf != was_leaf) 59271f51d2cSMingwei Zhang kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1); 5932f2fad08SBen Gardon 5942f2fad08SBen Gardon if (was_leaf && is_dirty_spte(old_spte) && 59564bb2769SSean Christopherson (!is_present || !is_dirty_spte(new_spte) || pfn_changed)) 5962f2fad08SBen Gardon kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 5972f2fad08SBen Gardon 5982f2fad08SBen Gardon /* 5992f2fad08SBen Gardon * Recursively handle child PTs if the change removed a subtree from 600c8e5a0d0SSean Christopherson * the paging structure. Note the WARN on the PFN changing without the 601c8e5a0d0SSean Christopherson * SPTE being converted to a hugepage (leaf) or being zapped. Shadow 602c8e5a0d0SSean Christopherson * pages are kernel allocations and should never be migrated. 6032f2fad08SBen Gardon */ 604c8e5a0d0SSean Christopherson if (was_present && !was_leaf && 605c8e5a0d0SSean Christopherson (is_leaf || !is_present || WARN_ON_ONCE(pfn_changed))) 6060f53dfa3SDavid Matlack handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared); 6072f2fad08SBen Gardon } 6082f2fad08SBen Gardon 6092f2fad08SBen Gardon static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 6109a77daacSBen Gardon u64 old_spte, u64 new_spte, int level, 6119a77daacSBen Gardon bool shared) 6122f2fad08SBen Gardon { 6139a77daacSBen Gardon __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, 6149a77daacSBen Gardon shared); 615f8e14497SBen Gardon handle_changed_spte_acc_track(old_spte, new_spte, level); 616a6a0b05dSBen Gardon handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte, 617a6a0b05dSBen Gardon new_spte, level); 6182f2fad08SBen Gardon } 619faaf05b0SBen Gardon 620fe43fa2fSBen Gardon /* 6216ccf4438SPaolo Bonzini * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically 6226ccf4438SPaolo Bonzini * and handle the associated bookkeeping. Do not mark the page dirty 62324ae4cfaSBen Gardon * in KVM's dirty bitmaps. 6249a77daacSBen Gardon * 6253255530aSDavid Matlack * If setting the SPTE fails because it has changed, iter->old_spte will be 6263255530aSDavid Matlack * refreshed to the current value of the spte. 6273255530aSDavid Matlack * 6289a77daacSBen Gardon * @kvm: kvm instance 6299a77daacSBen Gardon * @iter: a tdp_iter instance currently on the SPTE that should be set 6309a77daacSBen Gardon * @new_spte: The value the SPTE should be set to 6313e72c791SDavid Matlack * Return: 6323e72c791SDavid Matlack * * 0 - If the SPTE was set. 6333e72c791SDavid Matlack * * -EBUSY - If the SPTE cannot be set. In this case this function will have 6343e72c791SDavid Matlack * no side-effects other than setting iter->old_spte to the last 6353e72c791SDavid Matlack * known value of the spte. 6369a77daacSBen Gardon */ 6373e72c791SDavid Matlack static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm, 6389a77daacSBen Gardon struct tdp_iter *iter, 6399a77daacSBen Gardon u64 new_spte) 6409a77daacSBen Gardon { 6413255530aSDavid Matlack u64 *sptep = rcu_dereference(iter->sptep); 6423255530aSDavid Matlack 643396fd74dSSean Christopherson /* 644396fd74dSSean Christopherson * The caller is responsible for ensuring the old SPTE is not a REMOVED 645396fd74dSSean Christopherson * SPTE. KVM should never attempt to zap or manipulate a REMOVED SPTE, 646396fd74dSSean Christopherson * and pre-checking before inserting a new SPTE is advantageous as it 647396fd74dSSean Christopherson * avoids unnecessary work. 648396fd74dSSean Christopherson */ 649396fd74dSSean Christopherson WARN_ON_ONCE(iter->yielded || is_removed_spte(iter->old_spte)); 6503a0f64deSSean Christopherson 6519a77daacSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 6529a77daacSBen Gardon 65308f07c80SBen Gardon /* 6546e8eb206SDavid Matlack * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and 6556e8eb206SDavid Matlack * does not hold the mmu_lock. 6566e8eb206SDavid Matlack */ 657aee98a68SUros Bizjak if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte)) 6583e72c791SDavid Matlack return -EBUSY; 6599a77daacSBen Gardon 66024ae4cfaSBen Gardon __handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte, 66108889894SSean Christopherson new_spte, iter->level, true); 66224ae4cfaSBen Gardon handle_changed_spte_acc_track(iter->old_spte, new_spte, iter->level); 6639a77daacSBen Gardon 6643e72c791SDavid Matlack return 0; 6659a77daacSBen Gardon } 6669a77daacSBen Gardon 6673e72c791SDavid Matlack static inline int tdp_mmu_zap_spte_atomic(struct kvm *kvm, 66808f07c80SBen Gardon struct tdp_iter *iter) 66908f07c80SBen Gardon { 6703e72c791SDavid Matlack int ret; 6713e72c791SDavid Matlack 67208f07c80SBen Gardon /* 67308f07c80SBen Gardon * Freeze the SPTE by setting it to a special, 67408f07c80SBen Gardon * non-present value. This will stop other threads from 67508f07c80SBen Gardon * immediately installing a present entry in its place 67608f07c80SBen Gardon * before the TLBs are flushed. 67708f07c80SBen Gardon */ 6783e72c791SDavid Matlack ret = tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE); 6793e72c791SDavid Matlack if (ret) 6803e72c791SDavid Matlack return ret; 68108f07c80SBen Gardon 6824ad980aeSHou Wenlong kvm_flush_remote_tlbs_gfn(kvm, iter->gfn, iter->level); 68308f07c80SBen Gardon 68408f07c80SBen Gardon /* 685ba3a6120SSean Christopherson * No other thread can overwrite the removed SPTE as they must either 686ba3a6120SSean Christopherson * wait on the MMU lock or use tdp_mmu_set_spte_atomic() which will not 687ba3a6120SSean Christopherson * overwrite the special removed SPTE value. No bookkeeping is needed 688ba3a6120SSean Christopherson * here since the SPTE is going from non-present to non-present. Use 689ba3a6120SSean Christopherson * the raw write helper to avoid an unnecessary check on volatile bits. 69008f07c80SBen Gardon */ 691ba3a6120SSean Christopherson __kvm_tdp_mmu_write_spte(iter->sptep, 0); 69208f07c80SBen Gardon 6933e72c791SDavid Matlack return 0; 69408f07c80SBen Gardon } 69508f07c80SBen Gardon 6969a77daacSBen Gardon 6979a77daacSBen Gardon /* 698fe43fa2fSBen Gardon * __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping 699626808d1SSean Christopherson * @kvm: KVM instance 700626808d1SSean Christopherson * @as_id: Address space ID, i.e. regular vs. SMM 701626808d1SSean Christopherson * @sptep: Pointer to the SPTE 702626808d1SSean Christopherson * @old_spte: The current value of the SPTE 703626808d1SSean Christopherson * @new_spte: The new value that will be set for the SPTE 704626808d1SSean Christopherson * @gfn: The base GFN that was (or will be) mapped by the SPTE 705626808d1SSean Christopherson * @level: The level _containing_ the SPTE (its parent PT's level) 706fe43fa2fSBen Gardon * @record_acc_track: Notify the MM subsystem of changes to the accessed state 707fe43fa2fSBen Gardon * of the page. Should be set unless handling an MMU 708fe43fa2fSBen Gardon * notifier for access tracking. Leaving record_acc_track 709fe43fa2fSBen Gardon * unset in that case prevents page accesses from being 710fe43fa2fSBen Gardon * double counted. 711ba3a6120SSean Christopherson * 712ba3a6120SSean Christopherson * Returns the old SPTE value, which _may_ be different than @old_spte if the 713ba3a6120SSean Christopherson * SPTE had voldatile bits. 714fe43fa2fSBen Gardon */ 715ba3a6120SSean Christopherson static u64 __tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep, 716626808d1SSean Christopherson u64 old_spte, u64 new_spte, gfn_t gfn, int level, 717e7300870SVipin Sharma bool record_acc_track) 718faaf05b0SBen Gardon { 719531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 7203a9a4aa5SBen Gardon 72108f07c80SBen Gardon /* 722966da62aSSean Christopherson * No thread should be using this function to set SPTEs to or from the 72308f07c80SBen Gardon * temporary removed SPTE value. 72408f07c80SBen Gardon * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic 72508f07c80SBen Gardon * should be used. If operating under the MMU lock in write mode, the 72608f07c80SBen Gardon * use of the removed SPTE should not be necessary. 72708f07c80SBen Gardon */ 728626808d1SSean Christopherson WARN_ON(is_removed_spte(old_spte) || is_removed_spte(new_spte)); 72908f07c80SBen Gardon 730ba3a6120SSean Christopherson old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, new_spte, level); 731faaf05b0SBen Gardon 732626808d1SSean Christopherson __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false); 733626808d1SSean Christopherson 734f8e14497SBen Gardon if (record_acc_track) 735626808d1SSean Christopherson handle_changed_spte_acc_track(old_spte, new_spte, level); 736e7300870SVipin Sharma 737e7300870SVipin Sharma handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte, new_spte, 738e7300870SVipin Sharma level); 739ba3a6120SSean Christopherson return old_spte; 740626808d1SSean Christopherson } 741626808d1SSean Christopherson 742626808d1SSean Christopherson static inline void _tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, 743e7300870SVipin Sharma u64 new_spte, bool record_acc_track) 744626808d1SSean Christopherson { 745626808d1SSean Christopherson WARN_ON_ONCE(iter->yielded); 746626808d1SSean Christopherson 747ba3a6120SSean Christopherson iter->old_spte = __tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep, 748ba3a6120SSean Christopherson iter->old_spte, new_spte, 749ba3a6120SSean Christopherson iter->gfn, iter->level, 750e7300870SVipin Sharma record_acc_track); 751f8e14497SBen Gardon } 752f8e14497SBen Gardon 753f8e14497SBen Gardon static inline void tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, 754f8e14497SBen Gardon u64 new_spte) 755f8e14497SBen Gardon { 756e7300870SVipin Sharma _tdp_mmu_set_spte(kvm, iter, new_spte, true); 757f8e14497SBen Gardon } 758f8e14497SBen Gardon 759faaf05b0SBen Gardon #define tdp_root_for_each_pte(_iter, _root, _start, _end) \ 76077aa6075SDavid Matlack for_each_tdp_pte(_iter, _root, _start, _end) 761faaf05b0SBen Gardon 762f8e14497SBen Gardon #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \ 763f8e14497SBen Gardon tdp_root_for_each_pte(_iter, _root, _start, _end) \ 764f8e14497SBen Gardon if (!is_shadow_present_pte(_iter.old_spte) || \ 765f8e14497SBen Gardon !is_last_spte(_iter.old_spte, _iter.level)) \ 766f8e14497SBen Gardon continue; \ 767f8e14497SBen Gardon else 768f8e14497SBen Gardon 769bb18842eSBen Gardon #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \ 770b9e5603cSPaolo Bonzini for_each_tdp_pte(_iter, to_shadow_page(_mmu->root.hpa), _start, _end) 771bb18842eSBen Gardon 772faaf05b0SBen Gardon /* 773e28a436cSBen Gardon * Yield if the MMU lock is contended or this thread needs to return control 774e28a436cSBen Gardon * to the scheduler. 775e28a436cSBen Gardon * 776e139a34eSBen Gardon * If this function should yield and flush is set, it will perform a remote 777e139a34eSBen Gardon * TLB flush before yielding. 778e139a34eSBen Gardon * 7793a0f64deSSean Christopherson * If this function yields, iter->yielded is set and the caller must skip to 7803a0f64deSSean Christopherson * the next iteration, where tdp_iter_next() will reset the tdp_iter's walk 7813a0f64deSSean Christopherson * over the paging structures to allow the iterator to continue its traversal 7823a0f64deSSean Christopherson * from the paging structure root. 783e28a436cSBen Gardon * 7843a0f64deSSean Christopherson * Returns true if this function yielded. 785e28a436cSBen Gardon */ 7863a0f64deSSean Christopherson static inline bool __must_check tdp_mmu_iter_cond_resched(struct kvm *kvm, 7873a0f64deSSean Christopherson struct tdp_iter *iter, 7883a0f64deSSean Christopherson bool flush, bool shared) 789a6a0b05dSBen Gardon { 7903a0f64deSSean Christopherson WARN_ON(iter->yielded); 7913a0f64deSSean Christopherson 792ed5e484bSBen Gardon /* Ensure forward progress has been made before yielding. */ 793ed5e484bSBen Gardon if (iter->next_last_level_gfn == iter->yielded_gfn) 794ed5e484bSBen Gardon return false; 795ed5e484bSBen Gardon 796531810caSBen Gardon if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { 797e139a34eSBen Gardon if (flush) 798e139a34eSBen Gardon kvm_flush_remote_tlbs(kvm); 799e139a34eSBen Gardon 800bd296779SSean Christopherson rcu_read_unlock(); 801bd296779SSean Christopherson 8026103bc07SBen Gardon if (shared) 8036103bc07SBen Gardon cond_resched_rwlock_read(&kvm->mmu_lock); 8046103bc07SBen Gardon else 805531810caSBen Gardon cond_resched_rwlock_write(&kvm->mmu_lock); 8066103bc07SBen Gardon 8077cca2d0bSBen Gardon rcu_read_lock(); 808ed5e484bSBen Gardon 809ed5e484bSBen Gardon WARN_ON(iter->gfn > iter->next_last_level_gfn); 810ed5e484bSBen Gardon 8113a0f64deSSean Christopherson iter->yielded = true; 812a6a0b05dSBen Gardon } 813e28a436cSBen Gardon 8143a0f64deSSean Christopherson return iter->yielded; 815a6a0b05dSBen Gardon } 816a6a0b05dSBen Gardon 81786931ff7SSean Christopherson static inline gfn_t tdp_mmu_max_gfn_exclusive(void) 818e2b5b21dSSean Christopherson { 819e2b5b21dSSean Christopherson /* 82086931ff7SSean Christopherson * Bound TDP MMU walks at host.MAXPHYADDR. KVM disallows memslots with 82186931ff7SSean Christopherson * a gpa range that would exceed the max gfn, and KVM does not create 82286931ff7SSean Christopherson * MMIO SPTEs for "impossible" gfns, instead sending such accesses down 82386931ff7SSean Christopherson * the slow emulation path every time. 824e2b5b21dSSean Christopherson */ 82586931ff7SSean Christopherson return kvm_mmu_max_gfn() + 1; 826e2b5b21dSSean Christopherson } 827e2b5b21dSSean Christopherson 8281b6043e8SSean Christopherson static void __tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root, 8291b6043e8SSean Christopherson bool shared, int zap_level) 830e2b5b21dSSean Christopherson { 831e2b5b21dSSean Christopherson struct tdp_iter iter; 832e2b5b21dSSean Christopherson 83386931ff7SSean Christopherson gfn_t end = tdp_mmu_max_gfn_exclusive(); 834e2b5b21dSSean Christopherson gfn_t start = 0; 835e2b5b21dSSean Christopherson 8361b6043e8SSean Christopherson for_each_tdp_pte_min_level(iter, root, zap_level, start, end) { 8371b6043e8SSean Christopherson retry: 8381b6043e8SSean Christopherson if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) 8391b6043e8SSean Christopherson continue; 8401b6043e8SSean Christopherson 8411b6043e8SSean Christopherson if (!is_shadow_present_pte(iter.old_spte)) 8421b6043e8SSean Christopherson continue; 8431b6043e8SSean Christopherson 8441b6043e8SSean Christopherson if (iter.level > zap_level) 8451b6043e8SSean Christopherson continue; 8461b6043e8SSean Christopherson 8471b6043e8SSean Christopherson if (!shared) 8481b6043e8SSean Christopherson tdp_mmu_set_spte(kvm, &iter, 0); 8491b6043e8SSean Christopherson else if (tdp_mmu_set_spte_atomic(kvm, &iter, 0)) 8501b6043e8SSean Christopherson goto retry; 8511b6043e8SSean Christopherson } 8521b6043e8SSean Christopherson } 8531b6043e8SSean Christopherson 8541b6043e8SSean Christopherson static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root, 8551b6043e8SSean Christopherson bool shared) 8561b6043e8SSean Christopherson { 8571b6043e8SSean Christopherson 8588351779cSPaolo Bonzini /* 8598351779cSPaolo Bonzini * The root must have an elevated refcount so that it's reachable via 8608351779cSPaolo Bonzini * mmu_notifier callbacks, which allows this path to yield and drop 8618351779cSPaolo Bonzini * mmu_lock. When handling an unmap/release mmu_notifier command, KVM 8628351779cSPaolo Bonzini * must drop all references to relevant pages prior to completing the 8638351779cSPaolo Bonzini * callback. Dropping mmu_lock with an unreachable root would result 8648351779cSPaolo Bonzini * in zapping SPTEs after a relevant mmu_notifier callback completes 8658351779cSPaolo Bonzini * and lead to use-after-free as zapping a SPTE triggers "writeback" of 8668351779cSPaolo Bonzini * dirty accessed bits to the SPTE's associated struct page. 8678351779cSPaolo Bonzini */ 8688351779cSPaolo Bonzini WARN_ON_ONCE(!refcount_read(&root->tdp_mmu_root_count)); 8698351779cSPaolo Bonzini 870e2b5b21dSSean Christopherson kvm_lockdep_assert_mmu_lock_held(kvm, shared); 871e2b5b21dSSean Christopherson 872e2b5b21dSSean Christopherson rcu_read_lock(); 873e2b5b21dSSean Christopherson 874e2b5b21dSSean Christopherson /* 8751b6043e8SSean Christopherson * To avoid RCU stalls due to recursively removing huge swaths of SPs, 8761b6043e8SSean Christopherson * split the zap into two passes. On the first pass, zap at the 1gb 8771b6043e8SSean Christopherson * level, and then zap top-level SPs on the second pass. "1gb" is not 8781b6043e8SSean Christopherson * arbitrary, as KVM must be able to zap a 1gb shadow page without 8791b6043e8SSean Christopherson * inducing a stall to allow in-place replacement with a 1gb hugepage. 8801b6043e8SSean Christopherson * 8811b6043e8SSean Christopherson * Because zapping a SP recurses on its children, stepping down to 8821b6043e8SSean Christopherson * PG_LEVEL_4K in the iterator itself is unnecessary. 883e2b5b21dSSean Christopherson */ 8841b6043e8SSean Christopherson __tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_1G); 8851b6043e8SSean Christopherson __tdp_mmu_zap_root(kvm, root, shared, root->role.level); 886e2b5b21dSSean Christopherson 887e2b5b21dSSean Christopherson rcu_read_unlock(); 888e2b5b21dSSean Christopherson } 889e2b5b21dSSean Christopherson 890c10743a1SSean Christopherson bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 891c10743a1SSean Christopherson { 892c10743a1SSean Christopherson u64 old_spte; 893c10743a1SSean Christopherson 894c10743a1SSean Christopherson /* 895c10743a1SSean Christopherson * This helper intentionally doesn't allow zapping a root shadow page, 896c10743a1SSean Christopherson * which doesn't have a parent page table and thus no associated entry. 897c10743a1SSean Christopherson */ 898c10743a1SSean Christopherson if (WARN_ON_ONCE(!sp->ptep)) 899c10743a1SSean Christopherson return false; 900c10743a1SSean Christopherson 901c10743a1SSean Christopherson old_spte = kvm_tdp_mmu_read_spte(sp->ptep); 902bb95dfb9SSean Christopherson if (WARN_ON_ONCE(!is_shadow_present_pte(old_spte))) 903c10743a1SSean Christopherson return false; 904c10743a1SSean Christopherson 905c10743a1SSean Christopherson __tdp_mmu_set_spte(kvm, kvm_mmu_page_as_id(sp), sp->ptep, old_spte, 0, 906e7300870SVipin Sharma sp->gfn, sp->role.level + 1, true); 907c10743a1SSean Christopherson 908c10743a1SSean Christopherson return true; 909c10743a1SSean Christopherson } 910c10743a1SSean Christopherson 911faaf05b0SBen Gardon /* 912063afacdSBen Gardon * If can_yield is true, will release the MMU lock and reschedule if the 913063afacdSBen Gardon * scheduler needs the CPU or there is contention on the MMU lock. If this 914063afacdSBen Gardon * function cannot yield, it will not release the MMU lock or reschedule and 915063afacdSBen Gardon * the caller must ensure it does not supply too large a GFN range, or the 9166103bc07SBen Gardon * operation can cause a soft lockup. 917faaf05b0SBen Gardon */ 918f47e5bbbSSean Christopherson static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root, 919acbda82aSSean Christopherson gfn_t start, gfn_t end, bool can_yield, bool flush) 920faaf05b0SBen Gardon { 921faaf05b0SBen Gardon struct tdp_iter iter; 922faaf05b0SBen Gardon 92386931ff7SSean Christopherson end = min(end, tdp_mmu_max_gfn_exclusive()); 924524a1e4eSSean Christopherson 925acbda82aSSean Christopherson lockdep_assert_held_write(&kvm->mmu_lock); 9266103bc07SBen Gardon 9277cca2d0bSBen Gardon rcu_read_lock(); 9287cca2d0bSBen Gardon 929f47e5bbbSSean Christopherson for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) { 9301af4a960SBen Gardon if (can_yield && 931acbda82aSSean Christopherson tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) { 932a835429cSSean Christopherson flush = false; 9331af4a960SBen Gardon continue; 9341af4a960SBen Gardon } 9351af4a960SBen Gardon 936f47e5bbbSSean Christopherson if (!is_shadow_present_pte(iter.old_spte) || 937faaf05b0SBen Gardon !is_last_spte(iter.old_spte, iter.level)) 938faaf05b0SBen Gardon continue; 939faaf05b0SBen Gardon 940faaf05b0SBen Gardon tdp_mmu_set_spte(kvm, &iter, 0); 941a835429cSSean Christopherson flush = true; 942faaf05b0SBen Gardon } 9437cca2d0bSBen Gardon 9447cca2d0bSBen Gardon rcu_read_unlock(); 945bb95dfb9SSean Christopherson 946f47e5bbbSSean Christopherson /* 947f47e5bbbSSean Christopherson * Because this flow zaps _only_ leaf SPTEs, the caller doesn't need 948f47e5bbbSSean Christopherson * to provide RCU protection as no 'struct kvm_mmu_page' will be freed. 949f47e5bbbSSean Christopherson */ 950f47e5bbbSSean Christopherson return flush; 951faaf05b0SBen Gardon } 952faaf05b0SBen Gardon 953faaf05b0SBen Gardon /* 9547edc3a68SKai Huang * Zap leaf SPTEs for the range of gfns, [start, end), for all roots. Returns 9557edc3a68SKai Huang * true if a TLB flush is needed before releasing the MMU lock, i.e. if one or 9567edc3a68SKai Huang * more SPTEs were zapped since the MMU lock was last acquired. 957faaf05b0SBen Gardon */ 958f47e5bbbSSean Christopherson bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end, 959f47e5bbbSSean Christopherson bool can_yield, bool flush) 960faaf05b0SBen Gardon { 961faaf05b0SBen Gardon struct kvm_mmu_page *root; 962faaf05b0SBen Gardon 963614f6970SPaolo Bonzini for_each_tdp_mmu_root_yield_safe(kvm, root, as_id) 964f47e5bbbSSean Christopherson flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, flush); 965faaf05b0SBen Gardon 966faaf05b0SBen Gardon return flush; 967faaf05b0SBen Gardon } 968faaf05b0SBen Gardon 969faaf05b0SBen Gardon void kvm_tdp_mmu_zap_all(struct kvm *kvm) 970faaf05b0SBen Gardon { 971e2b5b21dSSean Christopherson struct kvm_mmu_page *root; 9722b9663d8SSean Christopherson int i; 973faaf05b0SBen Gardon 97477c8cd6bSSean Christopherson /* 97522b94c4bSPaolo Bonzini * Zap all roots, including invalid roots, as all SPTEs must be dropped 97622b94c4bSPaolo Bonzini * before returning to the caller. Zap directly even if the root is 97722b94c4bSPaolo Bonzini * also being zapped by a worker. Walking zapped top-level SPTEs isn't 97822b94c4bSPaolo Bonzini * all that expensive and mmu_lock is already held, which means the 97922b94c4bSPaolo Bonzini * worker has yielded, i.e. flushing the work instead of zapping here 98022b94c4bSPaolo Bonzini * isn't guaranteed to be any faster. 98122b94c4bSPaolo Bonzini * 98277c8cd6bSSean Christopherson * A TLB flush is unnecessary, KVM zaps everything if and only the VM 98377c8cd6bSSean Christopherson * is being destroyed or the userspace VMM has exited. In both cases, 98477c8cd6bSSean Christopherson * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request. 98577c8cd6bSSean Christopherson */ 986e2b5b21dSSean Christopherson for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 987e2b5b21dSSean Christopherson for_each_tdp_mmu_root_yield_safe(kvm, root, i) 988e2b5b21dSSean Christopherson tdp_mmu_zap_root(kvm, root, false); 989e2b5b21dSSean Christopherson } 990faaf05b0SBen Gardon } 991bb18842eSBen Gardon 9924c6654bdSBen Gardon /* 993f28e9c7fSSean Christopherson * Zap all invalidated roots to ensure all SPTEs are dropped before the "fast 99422b94c4bSPaolo Bonzini * zap" completes. 9954c6654bdSBen Gardon */ 9964c6654bdSBen Gardon void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) 9974c6654bdSBen Gardon { 99822b94c4bSPaolo Bonzini flush_workqueue(kvm->arch.tdp_mmu_zap_wq); 9994c6654bdSBen Gardon } 10004c6654bdSBen Gardon 1001bb18842eSBen Gardon /* 1002f28e9c7fSSean Christopherson * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that 100322b94c4bSPaolo Bonzini * is about to be zapped, e.g. in response to a memslots update. The actual 100422b94c4bSPaolo Bonzini * zapping is performed asynchronously, so a reference is taken on all roots. 100522b94c4bSPaolo Bonzini * Using a separate workqueue makes it easy to ensure that the destruction is 100622b94c4bSPaolo Bonzini * performed before the "fast zap" completes, without keeping a separate list 100722b94c4bSPaolo Bonzini * of invalidated roots; the list is effectively the list of work items in 100822b94c4bSPaolo Bonzini * the workqueue. 1009b7cccd39SBen Gardon * 101022b94c4bSPaolo Bonzini * Get a reference even if the root is already invalid, the asynchronous worker 101122b94c4bSPaolo Bonzini * assumes it was gifted a reference to the root it processes. Because mmu_lock 101222b94c4bSPaolo Bonzini * is held for write, it should be impossible to observe a root with zero refcount, 101322b94c4bSPaolo Bonzini * i.e. the list of roots cannot be stale. 10144c6654bdSBen Gardon * 1015b7cccd39SBen Gardon * This has essentially the same effect for the TDP MMU 1016b7cccd39SBen Gardon * as updating mmu_valid_gen does for the shadow MMU. 1017b7cccd39SBen Gardon */ 1018b7cccd39SBen Gardon void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm) 1019b7cccd39SBen Gardon { 1020b7cccd39SBen Gardon struct kvm_mmu_page *root; 1021b7cccd39SBen Gardon 1022b7cccd39SBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 1023f28e9c7fSSean Christopherson list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) { 1024efd995daSPaolo Bonzini if (!root->role.invalid && 1025efd995daSPaolo Bonzini !WARN_ON_ONCE(!kvm_tdp_mmu_get_root(root))) { 1026b7cccd39SBen Gardon root->role.invalid = true; 102722b94c4bSPaolo Bonzini tdp_mmu_schedule_zap_root(kvm, root); 102822b94c4bSPaolo Bonzini } 1029b7cccd39SBen Gardon } 1030f28e9c7fSSean Christopherson } 1031b7cccd39SBen Gardon 1032bb18842eSBen Gardon /* 1033bb18842eSBen Gardon * Installs a last-level SPTE to handle a TDP page fault. 1034bb18842eSBen Gardon * (NPT/EPT violation/misconfiguration) 1035bb18842eSBen Gardon */ 1036cdc47767SPaolo Bonzini static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, 1037cdc47767SPaolo Bonzini struct kvm_page_fault *fault, 1038cdc47767SPaolo Bonzini struct tdp_iter *iter) 1039bb18842eSBen Gardon { 1040c435d4b7SSean Christopherson struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep)); 1041bb18842eSBen Gardon u64 new_spte; 104257a3e96dSKai Huang int ret = RET_PF_FIXED; 1043ad67e480SPaolo Bonzini bool wrprot = false; 1044bb18842eSBen Gardon 104550a9ac25SSean Christopherson if (WARN_ON_ONCE(sp->role.level != fault->goal_level)) 104650a9ac25SSean Christopherson return RET_PF_RETRY; 104750a9ac25SSean Christopherson 1048e710c5f6SDavid Matlack if (unlikely(!fault->slot)) 1049bb18842eSBen Gardon new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL); 10509a77daacSBen Gardon else 105153597858SDavid Matlack wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn, 10522839180cSPaolo Bonzini fault->pfn, iter->old_spte, fault->prefetch, true, 10537158bee4SPaolo Bonzini fault->map_writable, &new_spte); 1054bb18842eSBen Gardon 1055bb18842eSBen Gardon if (new_spte == iter->old_spte) 1056bb18842eSBen Gardon ret = RET_PF_SPURIOUS; 10573e72c791SDavid Matlack else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte)) 10589a77daacSBen Gardon return RET_PF_RETRY; 1059bb95dfb9SSean Christopherson else if (is_shadow_present_pte(iter->old_spte) && 1060bb95dfb9SSean Christopherson !is_last_spte(iter->old_spte, iter->level)) 10611e203847SHou Wenlong kvm_flush_remote_tlbs_gfn(vcpu->kvm, iter->gfn, iter->level); 1062bb18842eSBen Gardon 1063bb18842eSBen Gardon /* 1064bb18842eSBen Gardon * If the page fault was caused by a write but the page is write 1065bb18842eSBen Gardon * protected, emulation is needed. If the emulation was skipped, 1066bb18842eSBen Gardon * the vCPU would have the same fault again. 1067bb18842eSBen Gardon */ 1068ad67e480SPaolo Bonzini if (wrprot) { 1069cdc47767SPaolo Bonzini if (fault->write) 1070bb18842eSBen Gardon ret = RET_PF_EMULATE; 1071bb18842eSBen Gardon } 1072bb18842eSBen Gardon 1073bb18842eSBen Gardon /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */ 10749a77daacSBen Gardon if (unlikely(is_mmio_spte(new_spte))) { 10751075d41eSSean Christopherson vcpu->stat.pf_mmio_spte_created++; 10769a77daacSBen Gardon trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn, 10779a77daacSBen Gardon new_spte); 1078bb18842eSBen Gardon ret = RET_PF_EMULATE; 10793849e092SSean Christopherson } else { 10809a77daacSBen Gardon trace_kvm_mmu_set_spte(iter->level, iter->gfn, 10819a77daacSBen Gardon rcu_dereference(iter->sptep)); 10823849e092SSean Christopherson } 1083bb18842eSBen Gardon 1084bb18842eSBen Gardon return ret; 1085bb18842eSBen Gardon } 1086bb18842eSBen Gardon 1087bb18842eSBen Gardon /* 1088cb00a70bSDavid Matlack * tdp_mmu_link_sp - Replace the given spte with an spte pointing to the 1089cb00a70bSDavid Matlack * provided page table. 10907b7e1ab6SDavid Matlack * 10917b7e1ab6SDavid Matlack * @kvm: kvm instance 10927b7e1ab6SDavid Matlack * @iter: a tdp_iter instance currently on the SPTE that should be set 10937b7e1ab6SDavid Matlack * @sp: The new TDP page table to install. 1094cb00a70bSDavid Matlack * @shared: This operation is running under the MMU lock in read mode. 10957b7e1ab6SDavid Matlack * 10967b7e1ab6SDavid Matlack * Returns: 0 if the new page table was installed. Non-0 if the page table 10977b7e1ab6SDavid Matlack * could not be installed (e.g. the atomic compare-exchange failed). 10987b7e1ab6SDavid Matlack */ 1099cb00a70bSDavid Matlack static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter, 110061f94478SSean Christopherson struct kvm_mmu_page *sp, bool shared) 11017b7e1ab6SDavid Matlack { 110254275f74SSean Christopherson u64 spte = make_nonleaf_spte(sp->spt, !kvm_ad_enabled()); 1103cb00a70bSDavid Matlack int ret = 0; 11047b7e1ab6SDavid Matlack 1105cb00a70bSDavid Matlack if (shared) { 11067b7e1ab6SDavid Matlack ret = tdp_mmu_set_spte_atomic(kvm, iter, spte); 11077b7e1ab6SDavid Matlack if (ret) 11087b7e1ab6SDavid Matlack return ret; 1109cb00a70bSDavid Matlack } else { 1110cb00a70bSDavid Matlack tdp_mmu_set_spte(kvm, iter, spte); 1111cb00a70bSDavid Matlack } 11127b7e1ab6SDavid Matlack 111343a063caSYosry Ahmed tdp_account_mmu_page(kvm, sp); 11147b7e1ab6SDavid Matlack 11157b7e1ab6SDavid Matlack return 0; 11167b7e1ab6SDavid Matlack } 11177b7e1ab6SDavid Matlack 1118c4b33d28SDavid Matlack static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter, 1119c4b33d28SDavid Matlack struct kvm_mmu_page *sp, bool shared); 1120c4b33d28SDavid Matlack 11217b7e1ab6SDavid Matlack /* 1122bb18842eSBen Gardon * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing 1123bb18842eSBen Gardon * page tables and SPTEs to translate the faulting guest physical address. 1124bb18842eSBen Gardon */ 11252f6305ddSPaolo Bonzini int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) 1126bb18842eSBen Gardon { 1127bb18842eSBen Gardon struct kvm_mmu *mmu = vcpu->arch.mmu; 112861f94478SSean Christopherson struct kvm *kvm = vcpu->kvm; 1129bb18842eSBen Gardon struct tdp_iter iter; 113089c0fd49SBen Gardon struct kvm_mmu_page *sp; 113163d28a25SPaolo Bonzini int ret = RET_PF_RETRY; 1132bb18842eSBen Gardon 113373a3c659SPaolo Bonzini kvm_mmu_hugepage_adjust(vcpu, fault); 1134bb18842eSBen Gardon 1135f0066d94SPaolo Bonzini trace_kvm_mmu_spte_requested(fault); 11367cca2d0bSBen Gardon 11377cca2d0bSBen Gardon rcu_read_lock(); 11387cca2d0bSBen Gardon 11392f6305ddSPaolo Bonzini tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) { 114063d28a25SPaolo Bonzini int r; 114163d28a25SPaolo Bonzini 114273a3c659SPaolo Bonzini if (fault->nx_huge_page_workaround_enabled) 1143536f0e6aSPaolo Bonzini disallowed_hugepage_adjust(fault, iter.old_spte, iter.level); 1144bb18842eSBen Gardon 1145bb18842eSBen Gardon /* 1146c4b33d28SDavid Matlack * If SPTE has been frozen by another thread, just give up and 1147c4b33d28SDavid Matlack * retry, avoiding unnecessary page table allocation and free. 1148ff76d506SKai Huang */ 1149ff76d506SKai Huang if (is_removed_spte(iter.old_spte)) 115063d28a25SPaolo Bonzini goto retry; 115163d28a25SPaolo Bonzini 1152f5d16bb9SSean Christopherson if (iter.level == fault->goal_level) 115380a3e4aeSSean Christopherson goto map_target_level; 1154f5d16bb9SSean Christopherson 115563d28a25SPaolo Bonzini /* Step down into the lower level page table if it exists. */ 115663d28a25SPaolo Bonzini if (is_shadow_present_pte(iter.old_spte) && 115763d28a25SPaolo Bonzini !is_large_pte(iter.old_spte)) 115863d28a25SPaolo Bonzini continue; 1159ff76d506SKai Huang 1160c4b33d28SDavid Matlack /* 1161c4b33d28SDavid Matlack * The SPTE is either non-present or points to a huge page that 1162c4b33d28SDavid Matlack * needs to be split. 1163c4b33d28SDavid Matlack */ 1164a82070b6SDavid Matlack sp = tdp_mmu_alloc_sp(vcpu); 1165a82070b6SDavid Matlack tdp_mmu_init_child_sp(sp, &iter); 1166a82070b6SDavid Matlack 116761f94478SSean Christopherson sp->nx_huge_page_disallowed = fault->huge_page_disallowed; 116861f94478SSean Christopherson 1169c4b33d28SDavid Matlack if (is_shadow_present_pte(iter.old_spte)) 117063d28a25SPaolo Bonzini r = tdp_mmu_split_huge_page(kvm, &iter, sp, true); 1171c4b33d28SDavid Matlack else 117263d28a25SPaolo Bonzini r = tdp_mmu_link_sp(kvm, &iter, sp, true); 1173c4b33d28SDavid Matlack 117463d28a25SPaolo Bonzini /* 117580a3e4aeSSean Christopherson * Force the guest to retry if installing an upper level SPTE 117680a3e4aeSSean Christopherson * failed, e.g. because a different task modified the SPTE. 117763d28a25SPaolo Bonzini */ 117863d28a25SPaolo Bonzini if (r) { 11799a77daacSBen Gardon tdp_mmu_free_sp(sp); 118063d28a25SPaolo Bonzini goto retry; 11819a77daacSBen Gardon } 118261f94478SSean Christopherson 118361f94478SSean Christopherson if (fault->huge_page_disallowed && 118461f94478SSean Christopherson fault->req_level >= iter.level) { 118561f94478SSean Christopherson spin_lock(&kvm->arch.tdp_mmu_pages_lock); 118621a36ac6SSean Christopherson if (sp->nx_huge_page_disallowed) 118761f94478SSean Christopherson track_possible_nx_huge_page(kvm, sp); 118861f94478SSean Christopherson spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 118961f94478SSean Christopherson } 1190bb18842eSBen Gardon } 1191bb18842eSBen Gardon 119280a3e4aeSSean Christopherson /* 119380a3e4aeSSean Christopherson * The walk aborted before reaching the target level, e.g. because the 119480a3e4aeSSean Christopherson * iterator detected an upper level SPTE was frozen during traversal. 119580a3e4aeSSean Christopherson */ 119680a3e4aeSSean Christopherson WARN_ON_ONCE(iter.level == fault->goal_level); 119780a3e4aeSSean Christopherson goto retry; 119880a3e4aeSSean Christopherson 119980a3e4aeSSean Christopherson map_target_level: 1200cdc47767SPaolo Bonzini ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter); 1201bb18842eSBen Gardon 120263d28a25SPaolo Bonzini retry: 120363d28a25SPaolo Bonzini rcu_read_unlock(); 1204bb18842eSBen Gardon return ret; 1205bb18842eSBen Gardon } 1206063afacdSBen Gardon 12073039bcc7SSean Christopherson bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, 12083039bcc7SSean Christopherson bool flush) 1209063afacdSBen Gardon { 1210f47e5bbbSSean Christopherson return kvm_tdp_mmu_zap_leafs(kvm, range->slot->as_id, range->start, 121183b83a02SSean Christopherson range->end, range->may_block, flush); 12123039bcc7SSean Christopherson } 12133039bcc7SSean Christopherson 12143039bcc7SSean Christopherson typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter, 12153039bcc7SSean Christopherson struct kvm_gfn_range *range); 12163039bcc7SSean Christopherson 12173039bcc7SSean Christopherson static __always_inline bool kvm_tdp_mmu_handle_gfn(struct kvm *kvm, 12183039bcc7SSean Christopherson struct kvm_gfn_range *range, 1219c1b91493SSean Christopherson tdp_handler_t handler) 1220063afacdSBen Gardon { 1221063afacdSBen Gardon struct kvm_mmu_page *root; 12223039bcc7SSean Christopherson struct tdp_iter iter; 12233039bcc7SSean Christopherson bool ret = false; 1224063afacdSBen Gardon 1225063afacdSBen Gardon /* 1226e1eed584SSean Christopherson * Don't support rescheduling, none of the MMU notifiers that funnel 1227e1eed584SSean Christopherson * into this helper allow blocking; it'd be dead, wasteful code. 1228063afacdSBen Gardon */ 12293039bcc7SSean Christopherson for_each_tdp_mmu_root(kvm, root, range->slot->as_id) { 1230a151acecSSean Christopherson rcu_read_lock(); 1231a151acecSSean Christopherson 12323039bcc7SSean Christopherson tdp_root_for_each_leaf_pte(iter, root, range->start, range->end) 12333039bcc7SSean Christopherson ret |= handler(kvm, &iter, range); 1234063afacdSBen Gardon 12353039bcc7SSean Christopherson rcu_read_unlock(); 1236a151acecSSean Christopherson } 1237063afacdSBen Gardon 1238063afacdSBen Gardon return ret; 1239063afacdSBen Gardon } 1240063afacdSBen Gardon 1241f8e14497SBen Gardon /* 1242f8e14497SBen Gardon * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero 1243f8e14497SBen Gardon * if any of the GFNs in the range have been accessed. 12447ee131e3SVipin Sharma * 12457ee131e3SVipin Sharma * No need to mark the corresponding PFN as accessed as this call is coming 12467ee131e3SVipin Sharma * from the clear_young() or clear_flush_young() notifier, which uses the 12477ee131e3SVipin Sharma * return value to determine if the page has been accessed. 1248f8e14497SBen Gardon */ 12493039bcc7SSean Christopherson static bool age_gfn_range(struct kvm *kvm, struct tdp_iter *iter, 12503039bcc7SSean Christopherson struct kvm_gfn_range *range) 1251f8e14497SBen Gardon { 12527ee131e3SVipin Sharma u64 new_spte; 1253f8e14497SBen Gardon 12543039bcc7SSean Christopherson /* If we have a non-accessed entry we don't need to change the pte. */ 12553039bcc7SSean Christopherson if (!is_accessed_spte(iter->old_spte)) 12563039bcc7SSean Christopherson return false; 12577cca2d0bSBen Gardon 12587ee131e3SVipin Sharma if (spte_ad_enabled(iter->old_spte)) { 12597ee131e3SVipin Sharma iter->old_spte = tdp_mmu_clear_spte_bits(iter->sptep, 12607ee131e3SVipin Sharma iter->old_spte, 12617ee131e3SVipin Sharma shadow_accessed_mask, 12627ee131e3SVipin Sharma iter->level); 12637ee131e3SVipin Sharma new_spte = iter->old_spte & ~shadow_accessed_mask; 1264f8e14497SBen Gardon } else { 1265f8e14497SBen Gardon /* 1266f8e14497SBen Gardon * Capture the dirty status of the page, so that it doesn't get 1267f8e14497SBen Gardon * lost when the SPTE is marked for access tracking. 1268f8e14497SBen Gardon */ 12697ee131e3SVipin Sharma if (is_writable_pte(iter->old_spte)) 12707ee131e3SVipin Sharma kvm_set_pfn_dirty(spte_to_pfn(iter->old_spte)); 1271f8e14497SBen Gardon 12727ee131e3SVipin Sharma new_spte = mark_spte_for_access_track(iter->old_spte); 12737ee131e3SVipin Sharma iter->old_spte = kvm_tdp_mmu_write_spte(iter->sptep, 12747ee131e3SVipin Sharma iter->old_spte, new_spte, 12757ee131e3SVipin Sharma iter->level); 1276f8e14497SBen Gardon } 1277f8e14497SBen Gardon 1278*891f1159SVipin Sharma trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, 1279*891f1159SVipin Sharma iter->old_spte, new_spte); 12803039bcc7SSean Christopherson return true; 1281f8e14497SBen Gardon } 1282f8e14497SBen Gardon 12833039bcc7SSean Christopherson bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) 1284f8e14497SBen Gardon { 12853039bcc7SSean Christopherson return kvm_tdp_mmu_handle_gfn(kvm, range, age_gfn_range); 1286f8e14497SBen Gardon } 1287f8e14497SBen Gardon 12883039bcc7SSean Christopherson static bool test_age_gfn(struct kvm *kvm, struct tdp_iter *iter, 12893039bcc7SSean Christopherson struct kvm_gfn_range *range) 1290f8e14497SBen Gardon { 12913039bcc7SSean Christopherson return is_accessed_spte(iter->old_spte); 1292f8e14497SBen Gardon } 1293f8e14497SBen Gardon 12943039bcc7SSean Christopherson bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 1295f8e14497SBen Gardon { 12963039bcc7SSean Christopherson return kvm_tdp_mmu_handle_gfn(kvm, range, test_age_gfn); 12973039bcc7SSean Christopherson } 12983039bcc7SSean Christopherson 12993039bcc7SSean Christopherson static bool set_spte_gfn(struct kvm *kvm, struct tdp_iter *iter, 13003039bcc7SSean Christopherson struct kvm_gfn_range *range) 13013039bcc7SSean Christopherson { 13023039bcc7SSean Christopherson u64 new_spte; 13033039bcc7SSean Christopherson 13043039bcc7SSean Christopherson /* Huge pages aren't expected to be modified without first being zapped. */ 13053039bcc7SSean Christopherson WARN_ON(pte_huge(range->pte) || range->start + 1 != range->end); 13063039bcc7SSean Christopherson 13073039bcc7SSean Christopherson if (iter->level != PG_LEVEL_4K || 13083039bcc7SSean Christopherson !is_shadow_present_pte(iter->old_spte)) 13093039bcc7SSean Christopherson return false; 13103039bcc7SSean Christopherson 13113039bcc7SSean Christopherson /* 13123039bcc7SSean Christopherson * Note, when changing a read-only SPTE, it's not strictly necessary to 13133039bcc7SSean Christopherson * zero the SPTE before setting the new PFN, but doing so preserves the 13143039bcc7SSean Christopherson * invariant that the PFN of a present * leaf SPTE can never change. 13153039bcc7SSean Christopherson * See __handle_changed_spte(). 13163039bcc7SSean Christopherson */ 13173039bcc7SSean Christopherson tdp_mmu_set_spte(kvm, iter, 0); 13183039bcc7SSean Christopherson 13193039bcc7SSean Christopherson if (!pte_write(range->pte)) { 13203039bcc7SSean Christopherson new_spte = kvm_mmu_changed_pte_notifier_make_spte(iter->old_spte, 13213039bcc7SSean Christopherson pte_pfn(range->pte)); 13223039bcc7SSean Christopherson 13233039bcc7SSean Christopherson tdp_mmu_set_spte(kvm, iter, new_spte); 13243039bcc7SSean Christopherson } 13253039bcc7SSean Christopherson 13263039bcc7SSean Christopherson return true; 1327f8e14497SBen Gardon } 13281d8dd6b3SBen Gardon 13291d8dd6b3SBen Gardon /* 13301d8dd6b3SBen Gardon * Handle the changed_pte MMU notifier for the TDP MMU. 13311d8dd6b3SBen Gardon * data is a pointer to the new pte_t mapping the HVA specified by the MMU 13321d8dd6b3SBen Gardon * notifier. 13331d8dd6b3SBen Gardon * Returns non-zero if a flush is needed before releasing the MMU lock. 13341d8dd6b3SBen Gardon */ 13353039bcc7SSean Christopherson bool kvm_tdp_mmu_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 13361d8dd6b3SBen Gardon { 133793fa50f6SSean Christopherson /* 133893fa50f6SSean Christopherson * No need to handle the remote TLB flush under RCU protection, the 133993fa50f6SSean Christopherson * target SPTE _must_ be a leaf SPTE, i.e. cannot result in freeing a 134093fa50f6SSean Christopherson * shadow page. See the WARN on pfn_changed in __handle_changed_spte(). 134193fa50f6SSean Christopherson */ 134293fa50f6SSean Christopherson return kvm_tdp_mmu_handle_gfn(kvm, range, set_spte_gfn); 13431d8dd6b3SBen Gardon } 13441d8dd6b3SBen Gardon 1345a6a0b05dSBen Gardon /* 1346bedd9195SDavid Matlack * Remove write access from all SPTEs at or above min_level that map GFNs 1347bedd9195SDavid Matlack * [start, end). Returns true if an SPTE has been changed and the TLBs need to 1348bedd9195SDavid Matlack * be flushed. 1349a6a0b05dSBen Gardon */ 1350a6a0b05dSBen Gardon static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1351a6a0b05dSBen Gardon gfn_t start, gfn_t end, int min_level) 1352a6a0b05dSBen Gardon { 1353a6a0b05dSBen Gardon struct tdp_iter iter; 1354a6a0b05dSBen Gardon u64 new_spte; 1355a6a0b05dSBen Gardon bool spte_set = false; 1356a6a0b05dSBen Gardon 13577cca2d0bSBen Gardon rcu_read_lock(); 13587cca2d0bSBen Gardon 1359a6a0b05dSBen Gardon BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 1360a6a0b05dSBen Gardon 136177aa6075SDavid Matlack for_each_tdp_pte_min_level(iter, root, min_level, start, end) { 136224ae4cfaSBen Gardon retry: 136324ae4cfaSBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 13641af4a960SBen Gardon continue; 13651af4a960SBen Gardon 1366a6a0b05dSBen Gardon if (!is_shadow_present_pte(iter.old_spte) || 13670f99ee2cSBen Gardon !is_last_spte(iter.old_spte, iter.level) || 13680f99ee2cSBen Gardon !(iter.old_spte & PT_WRITABLE_MASK)) 1369a6a0b05dSBen Gardon continue; 1370a6a0b05dSBen Gardon 1371a6a0b05dSBen Gardon new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1372a6a0b05dSBen Gardon 13733e72c791SDavid Matlack if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) 137424ae4cfaSBen Gardon goto retry; 13753255530aSDavid Matlack 1376a6a0b05dSBen Gardon spte_set = true; 1377a6a0b05dSBen Gardon } 13787cca2d0bSBen Gardon 13797cca2d0bSBen Gardon rcu_read_unlock(); 1380a6a0b05dSBen Gardon return spte_set; 1381a6a0b05dSBen Gardon } 1382a6a0b05dSBen Gardon 1383a6a0b05dSBen Gardon /* 1384a6a0b05dSBen Gardon * Remove write access from all the SPTEs mapping GFNs in the memslot. Will 1385a6a0b05dSBen Gardon * only affect leaf SPTEs down to min_level. 1386a6a0b05dSBen Gardon * Returns true if an SPTE has been changed and the TLBs need to be flushed. 1387a6a0b05dSBen Gardon */ 1388269e9552SHamza Mahfooz bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, 1389269e9552SHamza Mahfooz const struct kvm_memory_slot *slot, int min_level) 1390a6a0b05dSBen Gardon { 1391a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1392a6a0b05dSBen Gardon bool spte_set = false; 1393a6a0b05dSBen Gardon 139424ae4cfaSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 1395a6a0b05dSBen Gardon 1396d62007edSSean Christopherson for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) 1397a6a0b05dSBen Gardon spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn, 1398a6a0b05dSBen Gardon slot->base_gfn + slot->npages, min_level); 1399a6a0b05dSBen Gardon 1400a6a0b05dSBen Gardon return spte_set; 1401a6a0b05dSBen Gardon } 1402a6a0b05dSBen Gardon 1403a3fe5dbdSDavid Matlack static struct kvm_mmu_page *__tdp_mmu_alloc_sp_for_split(gfp_t gfp) 1404a3fe5dbdSDavid Matlack { 1405a3fe5dbdSDavid Matlack struct kvm_mmu_page *sp; 1406a3fe5dbdSDavid Matlack 1407a3fe5dbdSDavid Matlack gfp |= __GFP_ZERO; 1408a3fe5dbdSDavid Matlack 1409a3fe5dbdSDavid Matlack sp = kmem_cache_alloc(mmu_page_header_cache, gfp); 1410a3fe5dbdSDavid Matlack if (!sp) 1411a3fe5dbdSDavid Matlack return NULL; 1412a3fe5dbdSDavid Matlack 1413a3fe5dbdSDavid Matlack sp->spt = (void *)__get_free_page(gfp); 1414a3fe5dbdSDavid Matlack if (!sp->spt) { 1415a3fe5dbdSDavid Matlack kmem_cache_free(mmu_page_header_cache, sp); 1416a3fe5dbdSDavid Matlack return NULL; 1417a3fe5dbdSDavid Matlack } 1418a3fe5dbdSDavid Matlack 1419a3fe5dbdSDavid Matlack return sp; 1420a3fe5dbdSDavid Matlack } 1421a3fe5dbdSDavid Matlack 1422a3fe5dbdSDavid Matlack static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct kvm *kvm, 1423cb00a70bSDavid Matlack struct tdp_iter *iter, 1424cb00a70bSDavid Matlack bool shared) 1425a3fe5dbdSDavid Matlack { 1426a3fe5dbdSDavid Matlack struct kvm_mmu_page *sp; 1427a3fe5dbdSDavid Matlack 1428a3fe5dbdSDavid Matlack /* 1429a3fe5dbdSDavid Matlack * Since we are allocating while under the MMU lock we have to be 1430a3fe5dbdSDavid Matlack * careful about GFP flags. Use GFP_NOWAIT to avoid blocking on direct 1431a3fe5dbdSDavid Matlack * reclaim and to avoid making any filesystem callbacks (which can end 1432a3fe5dbdSDavid Matlack * up invoking KVM MMU notifiers, resulting in a deadlock). 1433a3fe5dbdSDavid Matlack * 1434a3fe5dbdSDavid Matlack * If this allocation fails we drop the lock and retry with reclaim 1435a3fe5dbdSDavid Matlack * allowed. 1436a3fe5dbdSDavid Matlack */ 1437a3fe5dbdSDavid Matlack sp = __tdp_mmu_alloc_sp_for_split(GFP_NOWAIT | __GFP_ACCOUNT); 1438a3fe5dbdSDavid Matlack if (sp) 1439a3fe5dbdSDavid Matlack return sp; 1440a3fe5dbdSDavid Matlack 1441a3fe5dbdSDavid Matlack rcu_read_unlock(); 1442cb00a70bSDavid Matlack 1443cb00a70bSDavid Matlack if (shared) 1444a3fe5dbdSDavid Matlack read_unlock(&kvm->mmu_lock); 1445cb00a70bSDavid Matlack else 1446cb00a70bSDavid Matlack write_unlock(&kvm->mmu_lock); 1447a3fe5dbdSDavid Matlack 1448a3fe5dbdSDavid Matlack iter->yielded = true; 1449a3fe5dbdSDavid Matlack sp = __tdp_mmu_alloc_sp_for_split(GFP_KERNEL_ACCOUNT); 1450a3fe5dbdSDavid Matlack 1451cb00a70bSDavid Matlack if (shared) 1452a3fe5dbdSDavid Matlack read_lock(&kvm->mmu_lock); 1453cb00a70bSDavid Matlack else 1454cb00a70bSDavid Matlack write_lock(&kvm->mmu_lock); 1455cb00a70bSDavid Matlack 1456a3fe5dbdSDavid Matlack rcu_read_lock(); 1457a3fe5dbdSDavid Matlack 1458a3fe5dbdSDavid Matlack return sp; 1459a3fe5dbdSDavid Matlack } 1460a3fe5dbdSDavid Matlack 1461c4b33d28SDavid Matlack /* Note, the caller is responsible for initializing @sp. */ 1462cb00a70bSDavid Matlack static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter, 1463cb00a70bSDavid Matlack struct kvm_mmu_page *sp, bool shared) 1464a3fe5dbdSDavid Matlack { 1465a3fe5dbdSDavid Matlack const u64 huge_spte = iter->old_spte; 1466a3fe5dbdSDavid Matlack const int level = iter->level; 1467a3fe5dbdSDavid Matlack int ret, i; 1468a3fe5dbdSDavid Matlack 1469a3fe5dbdSDavid Matlack /* 1470a3fe5dbdSDavid Matlack * No need for atomics when writing to sp->spt since the page table has 1471a3fe5dbdSDavid Matlack * not been linked in yet and thus is not reachable from any other CPU. 1472a3fe5dbdSDavid Matlack */ 14732ca3129eSSean Christopherson for (i = 0; i < SPTE_ENT_PER_PAGE; i++) 147447855da0SDavid Matlack sp->spt[i] = make_huge_page_split_spte(kvm, huge_spte, sp->role, i); 1475a3fe5dbdSDavid Matlack 1476a3fe5dbdSDavid Matlack /* 1477a3fe5dbdSDavid Matlack * Replace the huge spte with a pointer to the populated lower level 1478a3fe5dbdSDavid Matlack * page table. Since we are making this change without a TLB flush vCPUs 1479a3fe5dbdSDavid Matlack * will see a mix of the split mappings and the original huge mapping, 1480a3fe5dbdSDavid Matlack * depending on what's currently in their TLB. This is fine from a 1481a3fe5dbdSDavid Matlack * correctness standpoint since the translation will be the same either 1482a3fe5dbdSDavid Matlack * way. 1483a3fe5dbdSDavid Matlack */ 148461f94478SSean Christopherson ret = tdp_mmu_link_sp(kvm, iter, sp, shared); 1485a3fe5dbdSDavid Matlack if (ret) 1486e0b728b1SDavid Matlack goto out; 1487a3fe5dbdSDavid Matlack 1488a3fe5dbdSDavid Matlack /* 1489a3fe5dbdSDavid Matlack * tdp_mmu_link_sp_atomic() will handle subtracting the huge page we 1490a3fe5dbdSDavid Matlack * are overwriting from the page stats. But we have to manually update 1491a3fe5dbdSDavid Matlack * the page stats with the new present child pages. 1492a3fe5dbdSDavid Matlack */ 14932ca3129eSSean Christopherson kvm_update_page_stats(kvm, level - 1, SPTE_ENT_PER_PAGE); 1494a3fe5dbdSDavid Matlack 1495e0b728b1SDavid Matlack out: 1496e0b728b1SDavid Matlack trace_kvm_mmu_split_huge_page(iter->gfn, huge_spte, level, ret); 1497e0b728b1SDavid Matlack return ret; 1498a3fe5dbdSDavid Matlack } 1499a3fe5dbdSDavid Matlack 1500a3fe5dbdSDavid Matlack static int tdp_mmu_split_huge_pages_root(struct kvm *kvm, 1501a3fe5dbdSDavid Matlack struct kvm_mmu_page *root, 1502a3fe5dbdSDavid Matlack gfn_t start, gfn_t end, 1503cb00a70bSDavid Matlack int target_level, bool shared) 1504a3fe5dbdSDavid Matlack { 1505a3fe5dbdSDavid Matlack struct kvm_mmu_page *sp = NULL; 1506a3fe5dbdSDavid Matlack struct tdp_iter iter; 1507a3fe5dbdSDavid Matlack int ret = 0; 1508a3fe5dbdSDavid Matlack 1509a3fe5dbdSDavid Matlack rcu_read_lock(); 1510a3fe5dbdSDavid Matlack 1511a3fe5dbdSDavid Matlack /* 1512a3fe5dbdSDavid Matlack * Traverse the page table splitting all huge pages above the target 1513a3fe5dbdSDavid Matlack * level into one lower level. For example, if we encounter a 1GB page 1514a3fe5dbdSDavid Matlack * we split it into 512 2MB pages. 1515a3fe5dbdSDavid Matlack * 1516a3fe5dbdSDavid Matlack * Since the TDP iterator uses a pre-order traversal, we are guaranteed 1517a3fe5dbdSDavid Matlack * to visit an SPTE before ever visiting its children, which means we 1518a3fe5dbdSDavid Matlack * will correctly recursively split huge pages that are more than one 1519a3fe5dbdSDavid Matlack * level above the target level (e.g. splitting a 1GB to 512 2MB pages, 1520a3fe5dbdSDavid Matlack * and then splitting each of those to 512 4KB pages). 1521a3fe5dbdSDavid Matlack */ 1522a3fe5dbdSDavid Matlack for_each_tdp_pte_min_level(iter, root, target_level + 1, start, end) { 1523a3fe5dbdSDavid Matlack retry: 1524cb00a70bSDavid Matlack if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) 1525a3fe5dbdSDavid Matlack continue; 1526a3fe5dbdSDavid Matlack 1527a3fe5dbdSDavid Matlack if (!is_shadow_present_pte(iter.old_spte) || !is_large_pte(iter.old_spte)) 1528a3fe5dbdSDavid Matlack continue; 1529a3fe5dbdSDavid Matlack 1530a3fe5dbdSDavid Matlack if (!sp) { 1531cb00a70bSDavid Matlack sp = tdp_mmu_alloc_sp_for_split(kvm, &iter, shared); 1532a3fe5dbdSDavid Matlack if (!sp) { 1533a3fe5dbdSDavid Matlack ret = -ENOMEM; 1534e0b728b1SDavid Matlack trace_kvm_mmu_split_huge_page(iter.gfn, 1535e0b728b1SDavid Matlack iter.old_spte, 1536e0b728b1SDavid Matlack iter.level, ret); 1537a3fe5dbdSDavid Matlack break; 1538a3fe5dbdSDavid Matlack } 1539a3fe5dbdSDavid Matlack 1540a3fe5dbdSDavid Matlack if (iter.yielded) 1541a3fe5dbdSDavid Matlack continue; 1542a3fe5dbdSDavid Matlack } 1543a3fe5dbdSDavid Matlack 1544c4b33d28SDavid Matlack tdp_mmu_init_child_sp(sp, &iter); 1545c4b33d28SDavid Matlack 1546cb00a70bSDavid Matlack if (tdp_mmu_split_huge_page(kvm, &iter, sp, shared)) 1547a3fe5dbdSDavid Matlack goto retry; 1548a3fe5dbdSDavid Matlack 1549a3fe5dbdSDavid Matlack sp = NULL; 1550a3fe5dbdSDavid Matlack } 1551a3fe5dbdSDavid Matlack 1552a3fe5dbdSDavid Matlack rcu_read_unlock(); 1553a3fe5dbdSDavid Matlack 1554a3fe5dbdSDavid Matlack /* 1555a3fe5dbdSDavid Matlack * It's possible to exit the loop having never used the last sp if, for 1556a3fe5dbdSDavid Matlack * example, a vCPU doing HugePage NX splitting wins the race and 1557a3fe5dbdSDavid Matlack * installs its own sp in place of the last sp we tried to split. 1558a3fe5dbdSDavid Matlack */ 1559a3fe5dbdSDavid Matlack if (sp) 1560a3fe5dbdSDavid Matlack tdp_mmu_free_sp(sp); 1561a3fe5dbdSDavid Matlack 1562a3fe5dbdSDavid Matlack return ret; 1563a3fe5dbdSDavid Matlack } 1564a3fe5dbdSDavid Matlack 1565cb00a70bSDavid Matlack 1566a3fe5dbdSDavid Matlack /* 1567a3fe5dbdSDavid Matlack * Try to split all huge pages mapped by the TDP MMU down to the target level. 1568a3fe5dbdSDavid Matlack */ 1569a3fe5dbdSDavid Matlack void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, 1570a3fe5dbdSDavid Matlack const struct kvm_memory_slot *slot, 1571a3fe5dbdSDavid Matlack gfn_t start, gfn_t end, 1572cb00a70bSDavid Matlack int target_level, bool shared) 1573a3fe5dbdSDavid Matlack { 1574a3fe5dbdSDavid Matlack struct kvm_mmu_page *root; 1575a3fe5dbdSDavid Matlack int r = 0; 1576a3fe5dbdSDavid Matlack 1577cb00a70bSDavid Matlack kvm_lockdep_assert_mmu_lock_held(kvm, shared); 1578a3fe5dbdSDavid Matlack 15797c554d8eSPaolo Bonzini for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) { 1580cb00a70bSDavid Matlack r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared); 1581a3fe5dbdSDavid Matlack if (r) { 1582cb00a70bSDavid Matlack kvm_tdp_mmu_put_root(kvm, root, shared); 1583a3fe5dbdSDavid Matlack break; 1584a3fe5dbdSDavid Matlack } 1585a3fe5dbdSDavid Matlack } 1586a3fe5dbdSDavid Matlack } 1587a3fe5dbdSDavid Matlack 1588a6a0b05dSBen Gardon /* 1589a6a0b05dSBen Gardon * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If 1590a6a0b05dSBen Gardon * AD bits are enabled, this will involve clearing the dirty bit on each SPTE. 1591a6a0b05dSBen Gardon * If AD bits are not enabled, this will require clearing the writable bit on 1592a6a0b05dSBen Gardon * each SPTE. Returns true if an SPTE has been changed and the TLBs need to 1593a6a0b05dSBen Gardon * be flushed. 1594a6a0b05dSBen Gardon */ 1595a6a0b05dSBen Gardon static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1596a6a0b05dSBen Gardon gfn_t start, gfn_t end) 1597a6a0b05dSBen Gardon { 1598697c89beSVipin Sharma u64 dbit = kvm_ad_enabled() ? shadow_dirty_mask : PT_WRITABLE_MASK; 1599a6a0b05dSBen Gardon struct tdp_iter iter; 1600a6a0b05dSBen Gardon bool spte_set = false; 1601a6a0b05dSBen Gardon 16027cca2d0bSBen Gardon rcu_read_lock(); 16037cca2d0bSBen Gardon 1604a6a0b05dSBen Gardon tdp_root_for_each_leaf_pte(iter, root, start, end) { 160524ae4cfaSBen Gardon retry: 160624ae4cfaSBen Gardon if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 16071af4a960SBen Gardon continue; 16081af4a960SBen Gardon 16093354ef5aSSean Christopherson if (!is_shadow_present_pte(iter.old_spte)) 16103354ef5aSSean Christopherson continue; 16113354ef5aSSean Christopherson 16125982a539SVipin Sharma MMU_WARN_ON(kvm_ad_enabled() && 16135982a539SVipin Sharma spte_ad_need_write_protect(iter.old_spte)); 16145982a539SVipin Sharma 1615697c89beSVipin Sharma if (!(iter.old_spte & dbit)) 1616a6a0b05dSBen Gardon continue; 1617a6a0b05dSBen Gardon 1618697c89beSVipin Sharma if (tdp_mmu_set_spte_atomic(kvm, &iter, iter.old_spte & ~dbit)) 161924ae4cfaSBen Gardon goto retry; 16203255530aSDavid Matlack 1621a6a0b05dSBen Gardon spte_set = true; 1622a6a0b05dSBen Gardon } 16237cca2d0bSBen Gardon 16247cca2d0bSBen Gardon rcu_read_unlock(); 1625a6a0b05dSBen Gardon return spte_set; 1626a6a0b05dSBen Gardon } 1627a6a0b05dSBen Gardon 1628a6a0b05dSBen Gardon /* 1629a6a0b05dSBen Gardon * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If 1630a6a0b05dSBen Gardon * AD bits are enabled, this will involve clearing the dirty bit on each SPTE. 1631a6a0b05dSBen Gardon * If AD bits are not enabled, this will require clearing the writable bit on 1632a6a0b05dSBen Gardon * each SPTE. Returns true if an SPTE has been changed and the TLBs need to 1633a6a0b05dSBen Gardon * be flushed. 1634a6a0b05dSBen Gardon */ 1635269e9552SHamza Mahfooz bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, 1636269e9552SHamza Mahfooz const struct kvm_memory_slot *slot) 1637a6a0b05dSBen Gardon { 1638a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1639a6a0b05dSBen Gardon bool spte_set = false; 1640a6a0b05dSBen Gardon 164124ae4cfaSBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 1642a6a0b05dSBen Gardon 1643d62007edSSean Christopherson for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) 1644a6a0b05dSBen Gardon spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn, 1645a6a0b05dSBen Gardon slot->base_gfn + slot->npages); 1646a6a0b05dSBen Gardon 1647a6a0b05dSBen Gardon return spte_set; 1648a6a0b05dSBen Gardon } 1649a6a0b05dSBen Gardon 1650a6a0b05dSBen Gardon /* 1651a6a0b05dSBen Gardon * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is 1652a6a0b05dSBen Gardon * set in mask, starting at gfn. The given memslot is expected to contain all 1653a6a0b05dSBen Gardon * the GFNs represented by set bits in the mask. If AD bits are enabled, 1654a6a0b05dSBen Gardon * clearing the dirty status will involve clearing the dirty bit on each SPTE 1655a6a0b05dSBen Gardon * or, if AD bits are not enabled, clearing the writable bit on each SPTE. 1656a6a0b05dSBen Gardon */ 1657a6a0b05dSBen Gardon static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root, 1658a6a0b05dSBen Gardon gfn_t gfn, unsigned long mask, bool wrprot) 1659a6a0b05dSBen Gardon { 1660697c89beSVipin Sharma u64 dbit = (wrprot || !kvm_ad_enabled()) ? PT_WRITABLE_MASK : 1661697c89beSVipin Sharma shadow_dirty_mask; 1662a6a0b05dSBen Gardon struct tdp_iter iter; 1663a6a0b05dSBen Gardon 16647cca2d0bSBen Gardon rcu_read_lock(); 16657cca2d0bSBen Gardon 1666a6a0b05dSBen Gardon tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask), 1667a6a0b05dSBen Gardon gfn + BITS_PER_LONG) { 1668a6a0b05dSBen Gardon if (!mask) 1669a6a0b05dSBen Gardon break; 1670a6a0b05dSBen Gardon 16715982a539SVipin Sharma MMU_WARN_ON(kvm_ad_enabled() && 16725982a539SVipin Sharma spte_ad_need_write_protect(iter.old_spte)); 16735982a539SVipin Sharma 1674a6a0b05dSBen Gardon if (iter.level > PG_LEVEL_4K || 1675a6a0b05dSBen Gardon !(mask & (1UL << (iter.gfn - gfn)))) 1676a6a0b05dSBen Gardon continue; 1677a6a0b05dSBen Gardon 1678f1b3b06aSBen Gardon mask &= ~(1UL << (iter.gfn - gfn)); 1679f1b3b06aSBen Gardon 1680697c89beSVipin Sharma if (!(iter.old_spte & dbit)) 1681a6a0b05dSBen Gardon continue; 1682a6a0b05dSBen Gardon 168389c313f2SVipin Sharma iter.old_spte = tdp_mmu_clear_spte_bits(iter.sptep, 168489c313f2SVipin Sharma iter.old_spte, dbit, 168589c313f2SVipin Sharma iter.level); 168689c313f2SVipin Sharma 16871e0f4298SVipin Sharma trace_kvm_tdp_mmu_spte_changed(iter.as_id, iter.gfn, iter.level, 16881e0f4298SVipin Sharma iter.old_spte, 16891e0f4298SVipin Sharma iter.old_spte & ~dbit); 16901e0f4298SVipin Sharma kvm_set_pfn_dirty(spte_to_pfn(iter.old_spte)); 1691a6a0b05dSBen Gardon } 16927cca2d0bSBen Gardon 16937cca2d0bSBen Gardon rcu_read_unlock(); 1694a6a0b05dSBen Gardon } 1695a6a0b05dSBen Gardon 1696a6a0b05dSBen Gardon /* 1697a6a0b05dSBen Gardon * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is 1698a6a0b05dSBen Gardon * set in mask, starting at gfn. The given memslot is expected to contain all 1699a6a0b05dSBen Gardon * the GFNs represented by set bits in the mask. If AD bits are enabled, 1700a6a0b05dSBen Gardon * clearing the dirty status will involve clearing the dirty bit on each SPTE 1701a6a0b05dSBen Gardon * or, if AD bits are not enabled, clearing the writable bit on each SPTE. 1702a6a0b05dSBen Gardon */ 1703a6a0b05dSBen Gardon void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1704a6a0b05dSBen Gardon struct kvm_memory_slot *slot, 1705a6a0b05dSBen Gardon gfn_t gfn, unsigned long mask, 1706a6a0b05dSBen Gardon bool wrprot) 1707a6a0b05dSBen Gardon { 1708a6a0b05dSBen Gardon struct kvm_mmu_page *root; 1709a6a0b05dSBen Gardon 1710531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 1711a3f15bdaSSean Christopherson for_each_tdp_mmu_root(kvm, root, slot->as_id) 1712a6a0b05dSBen Gardon clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot); 1713a6a0b05dSBen Gardon } 1714a6a0b05dSBen Gardon 17154b85c921SSean Christopherson static void zap_collapsible_spte_range(struct kvm *kvm, 171614881998SBen Gardon struct kvm_mmu_page *root, 17174b85c921SSean Christopherson const struct kvm_memory_slot *slot) 171814881998SBen Gardon { 17199eba50f8SSean Christopherson gfn_t start = slot->base_gfn; 17209eba50f8SSean Christopherson gfn_t end = start + slot->npages; 172114881998SBen Gardon struct tdp_iter iter; 17225ba7c4c6SBen Gardon int max_mapping_level; 172314881998SBen Gardon 17247cca2d0bSBen Gardon rcu_read_lock(); 17257cca2d0bSBen Gardon 172685f44f8cSSean Christopherson for_each_tdp_pte_min_level(iter, root, PG_LEVEL_2M, start, end) { 172785f44f8cSSean Christopherson retry: 17284b85c921SSean Christopherson if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 17291af4a960SBen Gardon continue; 17301af4a960SBen Gardon 173185f44f8cSSean Christopherson if (iter.level > KVM_MAX_HUGEPAGE_LEVEL || 173285f44f8cSSean Christopherson !is_shadow_present_pte(iter.old_spte)) 173385f44f8cSSean Christopherson continue; 173485f44f8cSSean Christopherson 173585f44f8cSSean Christopherson /* 173685f44f8cSSean Christopherson * Don't zap leaf SPTEs, if a leaf SPTE could be replaced with 173785f44f8cSSean Christopherson * a large page size, then its parent would have been zapped 173885f44f8cSSean Christopherson * instead of stepping down. 173985f44f8cSSean Christopherson */ 174085f44f8cSSean Christopherson if (is_last_spte(iter.old_spte, iter.level)) 174185f44f8cSSean Christopherson continue; 174285f44f8cSSean Christopherson 174385f44f8cSSean Christopherson /* 174485f44f8cSSean Christopherson * If iter.gfn resides outside of the slot, i.e. the page for 174585f44f8cSSean Christopherson * the current level overlaps but is not contained by the slot, 174685f44f8cSSean Christopherson * then the SPTE can't be made huge. More importantly, trying 174785f44f8cSSean Christopherson * to query that info from slot->arch.lpage_info will cause an 174885f44f8cSSean Christopherson * out-of-bounds access. 174985f44f8cSSean Christopherson */ 175085f44f8cSSean Christopherson if (iter.gfn < start || iter.gfn >= end) 175114881998SBen Gardon continue; 175214881998SBen Gardon 17535ba7c4c6SBen Gardon max_mapping_level = kvm_mmu_max_mapping_level(kvm, slot, 1754a8ac499bSSean Christopherson iter.gfn, PG_LEVEL_NUM); 175585f44f8cSSean Christopherson if (max_mapping_level < iter.level) 17565ba7c4c6SBen Gardon continue; 17575ba7c4c6SBen Gardon 17584b85c921SSean Christopherson /* Note, a successful atomic zap also does a remote TLB flush. */ 175985f44f8cSSean Christopherson if (tdp_mmu_zap_spte_atomic(kvm, &iter)) 176085f44f8cSSean Christopherson goto retry; 17612db6f772SBen Gardon } 176214881998SBen Gardon 17637cca2d0bSBen Gardon rcu_read_unlock(); 176414881998SBen Gardon } 176514881998SBen Gardon 176614881998SBen Gardon /* 176785f44f8cSSean Christopherson * Zap non-leaf SPTEs (and free their associated page tables) which could 176885f44f8cSSean Christopherson * be replaced by huge pages, for GFNs within the slot. 176914881998SBen Gardon */ 17704b85c921SSean Christopherson void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, 17714b85c921SSean Christopherson const struct kvm_memory_slot *slot) 177214881998SBen Gardon { 177314881998SBen Gardon struct kvm_mmu_page *root; 177414881998SBen Gardon 17752db6f772SBen Gardon lockdep_assert_held_read(&kvm->mmu_lock); 177614881998SBen Gardon 1777d62007edSSean Christopherson for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) 17784b85c921SSean Christopherson zap_collapsible_spte_range(kvm, root, slot); 177914881998SBen Gardon } 178046044f72SBen Gardon 178146044f72SBen Gardon /* 178246044f72SBen Gardon * Removes write access on the last level SPTE mapping this GFN and unsets the 17835fc3424fSSean Christopherson * MMU-writable bit to ensure future writes continue to be intercepted. 178446044f72SBen Gardon * Returns true if an SPTE was set and a TLB flush is needed. 178546044f72SBen Gardon */ 178646044f72SBen Gardon static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, 17873ad93562SKeqian Zhu gfn_t gfn, int min_level) 178846044f72SBen Gardon { 178946044f72SBen Gardon struct tdp_iter iter; 179046044f72SBen Gardon u64 new_spte; 179146044f72SBen Gardon bool spte_set = false; 179246044f72SBen Gardon 17933ad93562SKeqian Zhu BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 17943ad93562SKeqian Zhu 17957cca2d0bSBen Gardon rcu_read_lock(); 17967cca2d0bSBen Gardon 179777aa6075SDavid Matlack for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) { 17983ad93562SKeqian Zhu if (!is_shadow_present_pte(iter.old_spte) || 17993ad93562SKeqian Zhu !is_last_spte(iter.old_spte, iter.level)) 18003ad93562SKeqian Zhu continue; 18013ad93562SKeqian Zhu 180246044f72SBen Gardon new_spte = iter.old_spte & 18035fc3424fSSean Christopherson ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask); 180446044f72SBen Gardon 18057c8a4742SDavid Matlack if (new_spte == iter.old_spte) 18067c8a4742SDavid Matlack break; 18077c8a4742SDavid Matlack 180846044f72SBen Gardon tdp_mmu_set_spte(kvm, &iter, new_spte); 180946044f72SBen Gardon spte_set = true; 181046044f72SBen Gardon } 181146044f72SBen Gardon 18127cca2d0bSBen Gardon rcu_read_unlock(); 18137cca2d0bSBen Gardon 181446044f72SBen Gardon return spte_set; 181546044f72SBen Gardon } 181646044f72SBen Gardon 181746044f72SBen Gardon /* 181846044f72SBen Gardon * Removes write access on the last level SPTE mapping this GFN and unsets the 18195fc3424fSSean Christopherson * MMU-writable bit to ensure future writes continue to be intercepted. 182046044f72SBen Gardon * Returns true if an SPTE was set and a TLB flush is needed. 182146044f72SBen Gardon */ 182246044f72SBen Gardon bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, 18233ad93562SKeqian Zhu struct kvm_memory_slot *slot, gfn_t gfn, 18243ad93562SKeqian Zhu int min_level) 182546044f72SBen Gardon { 182646044f72SBen Gardon struct kvm_mmu_page *root; 182746044f72SBen Gardon bool spte_set = false; 182846044f72SBen Gardon 1829531810caSBen Gardon lockdep_assert_held_write(&kvm->mmu_lock); 1830a3f15bdaSSean Christopherson for_each_tdp_mmu_root(kvm, root, slot->as_id) 18313ad93562SKeqian Zhu spte_set |= write_protect_gfn(kvm, root, gfn, min_level); 1832a3f15bdaSSean Christopherson 183346044f72SBen Gardon return spte_set; 183446044f72SBen Gardon } 183546044f72SBen Gardon 183695fb5b02SBen Gardon /* 183795fb5b02SBen Gardon * Return the level of the lowest level SPTE added to sptes. 183895fb5b02SBen Gardon * That SPTE may be non-present. 1839c5c8c7c5SDavid Matlack * 1840c5c8c7c5SDavid Matlack * Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}. 184195fb5b02SBen Gardon */ 184239b4d43eSSean Christopherson int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, 184339b4d43eSSean Christopherson int *root_level) 184495fb5b02SBen Gardon { 184595fb5b02SBen Gardon struct tdp_iter iter; 184695fb5b02SBen Gardon struct kvm_mmu *mmu = vcpu->arch.mmu; 184795fb5b02SBen Gardon gfn_t gfn = addr >> PAGE_SHIFT; 18482aa07893SSean Christopherson int leaf = -1; 184995fb5b02SBen Gardon 1850a972e29cSPaolo Bonzini *root_level = vcpu->arch.mmu->root_role.level; 185195fb5b02SBen Gardon 185295fb5b02SBen Gardon tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 185395fb5b02SBen Gardon leaf = iter.level; 1854dde81f94SSean Christopherson sptes[leaf] = iter.old_spte; 185595fb5b02SBen Gardon } 185695fb5b02SBen Gardon 185795fb5b02SBen Gardon return leaf; 185895fb5b02SBen Gardon } 18596e8eb206SDavid Matlack 18606e8eb206SDavid Matlack /* 18616e8eb206SDavid Matlack * Returns the last level spte pointer of the shadow page walk for the given 18626e8eb206SDavid Matlack * gpa, and sets *spte to the spte value. This spte may be non-preset. If no 18636e8eb206SDavid Matlack * walk could be performed, returns NULL and *spte does not contain valid data. 18646e8eb206SDavid Matlack * 18656e8eb206SDavid Matlack * Contract: 18666e8eb206SDavid Matlack * - Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}. 18676e8eb206SDavid Matlack * - The returned sptep must not be used after kvm_tdp_mmu_walk_lockless_end. 18686e8eb206SDavid Matlack * 18696e8eb206SDavid Matlack * WARNING: This function is only intended to be called during fast_page_fault. 18706e8eb206SDavid Matlack */ 18716e8eb206SDavid Matlack u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, u64 addr, 18726e8eb206SDavid Matlack u64 *spte) 18736e8eb206SDavid Matlack { 18746e8eb206SDavid Matlack struct tdp_iter iter; 18756e8eb206SDavid Matlack struct kvm_mmu *mmu = vcpu->arch.mmu; 18766e8eb206SDavid Matlack gfn_t gfn = addr >> PAGE_SHIFT; 18776e8eb206SDavid Matlack tdp_ptep_t sptep = NULL; 18786e8eb206SDavid Matlack 18796e8eb206SDavid Matlack tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 18806e8eb206SDavid Matlack *spte = iter.old_spte; 18816e8eb206SDavid Matlack sptep = iter.sptep; 18826e8eb206SDavid Matlack } 18836e8eb206SDavid Matlack 18846e8eb206SDavid Matlack /* 18856e8eb206SDavid Matlack * Perform the rcu_dereference to get the raw spte pointer value since 18866e8eb206SDavid Matlack * we are passing it up to fast_page_fault, which is shared with the 18876e8eb206SDavid Matlack * legacy MMU and thus does not retain the TDP MMU-specific __rcu 18886e8eb206SDavid Matlack * annotation. 18896e8eb206SDavid Matlack * 18906e8eb206SDavid Matlack * This is safe since fast_page_fault obeys the contracts of this 18916e8eb206SDavid Matlack * function as well as all TDP MMU contracts around modifying SPTEs 18926e8eb206SDavid Matlack * outside of mmu_lock. 18936e8eb206SDavid Matlack */ 18946e8eb206SDavid Matlack return rcu_dereference(sptep); 18956e8eb206SDavid Matlack } 1896