Lines Matching refs:sptep
158 u64 *sptep; member
177 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
184 static void mmu_spte_set(u64 *sptep, u64 spte);
288 static void kvm_flush_remote_tlbs_sptep(struct kvm *kvm, u64 *sptep) in kvm_flush_remote_tlbs_sptep() argument
290 struct kvm_mmu_page *sp = sptep_to_sp(sptep); in kvm_flush_remote_tlbs_sptep()
291 gfn_t gfn = kvm_mmu_page_get_gfn(sp, spte_index(sptep)); in kvm_flush_remote_tlbs_sptep()
296 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, in mark_mmio_spte() argument
301 trace_mark_mmio_spte(sptep, gfn, spte); in mark_mmio_spte()
302 mmu_spte_set(sptep, spte); in mark_mmio_spte()
341 static void __set_spte(u64 *sptep, u64 spte) in __set_spte() argument
343 WRITE_ONCE(*sptep, spte); in __set_spte()
346 static void __update_clear_spte_fast(u64 *sptep, u64 spte) in __update_clear_spte_fast() argument
348 WRITE_ONCE(*sptep, spte); in __update_clear_spte_fast()
351 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) in __update_clear_spte_slow() argument
353 return xchg(sptep, spte); in __update_clear_spte_slow()
356 static u64 __get_spte_lockless(u64 *sptep) in __get_spte_lockless() argument
358 return READ_ONCE(*sptep); in __get_spte_lockless()
369 static void count_spte_clear(u64 *sptep, u64 spte) in count_spte_clear() argument
371 struct kvm_mmu_page *sp = sptep_to_sp(sptep); in count_spte_clear()
381 static void __set_spte(u64 *sptep, u64 spte) in __set_spte() argument
385 ssptep = (union split_spte *)sptep; in __set_spte()
400 static void __update_clear_spte_fast(u64 *sptep, u64 spte) in __update_clear_spte_fast() argument
404 ssptep = (union split_spte *)sptep; in __update_clear_spte_fast()
416 count_spte_clear(sptep, spte); in __update_clear_spte_fast()
419 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte) in __update_clear_spte_slow() argument
423 ssptep = (union split_spte *)sptep; in __update_clear_spte_slow()
430 count_spte_clear(sptep, spte); in __update_clear_spte_slow()
453 static u64 __get_spte_lockless(u64 *sptep) in __get_spte_lockless() argument
455 struct kvm_mmu_page *sp = sptep_to_sp(sptep); in __get_spte_lockless()
456 union split_spte spte, *orig = (union split_spte *)sptep; in __get_spte_lockless()
483 static void mmu_spte_set(u64 *sptep, u64 new_spte) in mmu_spte_set() argument
485 WARN_ON_ONCE(is_shadow_present_pte(*sptep)); in mmu_spte_set()
486 __set_spte(sptep, new_spte); in mmu_spte_set()
493 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte) in mmu_spte_update_no_track() argument
495 u64 old_spte = *sptep; in mmu_spte_update_no_track()
501 mmu_spte_set(sptep, new_spte); in mmu_spte_update_no_track()
506 __update_clear_spte_fast(sptep, new_spte); in mmu_spte_update_no_track()
508 old_spte = __update_clear_spte_slow(sptep, new_spte); in mmu_spte_update_no_track()
524 static bool mmu_spte_update(u64 *sptep, u64 new_spte) in mmu_spte_update() argument
527 u64 old_spte = mmu_spte_update_no_track(sptep, new_spte); in mmu_spte_update()
565 static u64 mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep) in mmu_spte_clear_track_bits() argument
568 u64 old_spte = *sptep; in mmu_spte_clear_track_bits()
569 int level = sptep_to_sp(sptep)->role.level; in mmu_spte_clear_track_bits()
574 __update_clear_spte_fast(sptep, 0ull); in mmu_spte_clear_track_bits()
576 old_spte = __update_clear_spte_slow(sptep, 0ull); in mmu_spte_clear_track_bits()
608 static void mmu_spte_clear_no_track(u64 *sptep) in mmu_spte_clear_no_track() argument
610 __update_clear_spte_fast(sptep, 0ull); in mmu_spte_clear_no_track()
613 static u64 mmu_spte_get_lockless(u64 *sptep) in mmu_spte_get_lockless() argument
615 return __get_spte_lockless(sptep); in mmu_spte_get_lockless()
619 static bool mmu_spte_age(u64 *sptep) in mmu_spte_age() argument
621 u64 spte = mmu_spte_get_lockless(sptep); in mmu_spte_age()
628 (unsigned long *)sptep); in mmu_spte_age()
638 mmu_spte_update_no_track(sptep, spte); in mmu_spte_age()
1032 struct kvm_rmap_head *rmap_head, u64 *sptep) in kvm_zap_one_rmap_spte() argument
1034 mmu_spte_clear_track_bits(kvm, sptep); in kvm_zap_one_rmap_spte()
1035 pte_list_remove(kvm, sptep, rmap_head); in kvm_zap_one_rmap_spte()
1133 u64 *sptep; in rmap_get_first() local
1140 sptep = (u64 *)rmap_head->val; in rmap_get_first()
1146 sptep = iter->desc->sptes[iter->pos]; in rmap_get_first()
1148 BUG_ON(!is_shadow_present_pte(*sptep)); in rmap_get_first()
1149 return sptep; in rmap_get_first()
1159 u64 *sptep; in rmap_get_next() local
1164 sptep = iter->desc->sptes[iter->pos]; in rmap_get_next()
1165 if (sptep) in rmap_get_next()
1174 sptep = iter->desc->sptes[iter->pos]; in rmap_get_next()
1181 BUG_ON(!is_shadow_present_pte(*sptep)); in rmap_get_next()
1182 return sptep; in rmap_get_next()
1189 static void drop_spte(struct kvm *kvm, u64 *sptep) in drop_spte() argument
1191 u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep); in drop_spte()
1194 rmap_remove(kvm, sptep); in drop_spte()
1197 static void drop_large_spte(struct kvm *kvm, u64 *sptep, bool flush) in drop_large_spte() argument
1201 sp = sptep_to_sp(sptep); in drop_large_spte()
1204 drop_spte(kvm, sptep); in drop_large_spte()
1207 kvm_flush_remote_tlbs_sptep(kvm, sptep); in drop_large_spte()
1223 static bool spte_write_protect(u64 *sptep, bool pt_protect) in spte_write_protect() argument
1225 u64 spte = *sptep; in spte_write_protect()
1235 return mmu_spte_update(sptep, spte); in spte_write_protect()
1241 u64 *sptep; in rmap_write_protect() local
1245 for_each_rmap_spte(rmap_head, &iter, sptep) in rmap_write_protect()
1246 flush |= spte_write_protect(sptep, pt_protect); in rmap_write_protect()
1251 static bool spte_clear_dirty(u64 *sptep) in spte_clear_dirty() argument
1253 u64 spte = *sptep; in spte_clear_dirty()
1257 return mmu_spte_update(sptep, spte); in spte_clear_dirty()
1260 static bool spte_wrprot_for_clear_dirty(u64 *sptep) in spte_wrprot_for_clear_dirty() argument
1263 (unsigned long *)sptep); in spte_wrprot_for_clear_dirty()
1264 if (was_writable && !spte_ad_enabled(*sptep)) in spte_wrprot_for_clear_dirty()
1265 kvm_set_pfn_dirty(spte_to_pfn(*sptep)); in spte_wrprot_for_clear_dirty()
1279 u64 *sptep; in __rmap_clear_dirty() local
1283 for_each_rmap_spte(rmap_head, &iter, sptep) in __rmap_clear_dirty()
1284 if (spte_ad_need_write_protect(*sptep)) in __rmap_clear_dirty()
1285 flush |= spte_wrprot_for_clear_dirty(sptep); in __rmap_clear_dirty()
1287 flush |= spte_clear_dirty(sptep); in __rmap_clear_dirty()
1455 u64 *sptep; in kvm_set_pte_rmap() local
1465 for_each_rmap_spte(rmap_head, &iter, sptep) { in kvm_set_pte_rmap()
1469 kvm_zap_one_rmap_spte(kvm, rmap_head, sptep); in kvm_set_pte_rmap()
1473 *sptep, new_pfn); in kvm_set_pte_rmap()
1475 mmu_spte_clear_track_bits(kvm, sptep); in kvm_set_pte_rmap()
1476 mmu_spte_set(sptep, new_spte); in kvm_set_pte_rmap()
1610 u64 *sptep; in kvm_age_rmap() local
1614 for_each_rmap_spte(rmap_head, &iter, sptep) in kvm_age_rmap()
1615 young |= mmu_spte_age(sptep); in kvm_age_rmap()
1624 u64 *sptep; in kvm_test_age_rmap() local
1627 for_each_rmap_spte(rmap_head, &iter, sptep) in kvm_test_age_rmap()
1628 if (is_accessed_spte(*sptep)) in kvm_test_age_rmap()
1773 u64 *sptep; in kvm_mmu_mark_parents_unsync() local
1776 for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) { in kvm_mmu_mark_parents_unsync()
1777 mark_unsync(sptep); in kvm_mmu_mark_parents_unsync()
2301 static union kvm_mmu_page_role kvm_mmu_child_role(u64 *sptep, bool direct, in kvm_mmu_child_role() argument
2304 struct kvm_mmu_page *parent_sp = sptep_to_sp(sptep); in kvm_mmu_child_role()
2341 role.quadrant = spte_index(sptep) & 1; in kvm_mmu_child_role()
2348 u64 *sptep, gfn_t gfn, in kvm_mmu_get_child_sp() argument
2353 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) in kvm_mmu_get_child_sp()
2356 role = kvm_mmu_child_role(sptep, direct, access); in kvm_mmu_get_child_sp()
2402 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index; in shadow_walk_okay()
2420 __shadow_walk_next(iterator, *iterator->sptep); in shadow_walk_next()
2424 struct kvm_mmu_memory_cache *cache, u64 *sptep, in __link_shadow_page() argument
2436 if (is_shadow_present_pte(*sptep)) in __link_shadow_page()
2437 drop_large_spte(kvm, sptep, flush); in __link_shadow_page()
2441 mmu_spte_set(sptep, spte); in __link_shadow_page()
2443 mmu_page_add_parent_pte(cache, sp, sptep); in __link_shadow_page()
2455 mark_unsync(sptep); in __link_shadow_page()
2458 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep, in link_shadow_page() argument
2461 __link_shadow_page(vcpu->kvm, &vcpu->arch.mmu_pte_list_desc_cache, sptep, sp, true); in link_shadow_page()
2464 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, in validate_direct_spte() argument
2467 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) { in validate_direct_spte()
2477 child = spte_to_child_sp(*sptep); in validate_direct_spte()
2481 drop_parent_pte(vcpu->kvm, child, sptep); in validate_direct_spte()
2482 kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep); in validate_direct_spte()
2532 u64 *sptep; in kvm_mmu_unlink_parents() local
2535 while ((sptep = rmap_get_first(&sp->parent_ptes, &iter))) in kvm_mmu_unlink_parents()
2536 drop_parent_pte(kvm, sp, sptep); in kvm_mmu_unlink_parents()
2901 u64 *sptep, unsigned int pte_access, gfn_t gfn, in mmu_set_spte() argument
2904 struct kvm_mmu_page *sp = sptep_to_sp(sptep); in mmu_set_spte()
2919 mark_mmio_spte(vcpu, sptep, gfn, pte_access); in mmu_set_spte()
2923 if (is_shadow_present_pte(*sptep)) { in mmu_set_spte()
2928 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) { in mmu_set_spte()
2930 u64 pte = *sptep; in mmu_set_spte()
2933 drop_parent_pte(vcpu->kvm, child, sptep); in mmu_set_spte()
2935 } else if (pfn != spte_to_pfn(*sptep)) { in mmu_set_spte()
2936 drop_spte(vcpu->kvm, sptep); in mmu_set_spte()
2942 wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch, in mmu_set_spte()
2945 if (*sptep == spte) { in mmu_set_spte()
2948 flush |= mmu_spte_update(sptep, spte); in mmu_set_spte()
2949 trace_kvm_mmu_set_spte(level, gfn, sptep); in mmu_set_spte()
2962 rmap_add(vcpu, slot, sptep, gfn, pte_access); in mmu_set_spte()
2965 kvm_mmu_page_set_access(sp, spte_index(sptep), pte_access); in mmu_set_spte()
3000 struct kvm_mmu_page *sp, u64 *sptep) in __direct_pte_prefetch() argument
3007 i = spte_index(sptep) & ~(PTE_PREFETCH_NUM - 1); in __direct_pte_prefetch()
3011 if (is_shadow_present_pte(*spte) || spte == sptep) { in __direct_pte_prefetch()
3024 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) in direct_pte_prefetch() argument
3028 sp = sptep_to_sp(sptep); in direct_pte_prefetch()
3048 __direct_pte_prefetch(vcpu, sp, sptep); in direct_pte_prefetch()
3233 disallowed_hugepage_adjust(fault, *it.sptep, it.level); in direct_map()
3239 sp = kvm_mmu_get_child_sp(vcpu, it.sptep, base_gfn, true, ACC_ALL); in direct_map()
3243 link_shadow_page(vcpu, it.sptep, sp); in direct_map()
3252 ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL, in direct_map()
3257 direct_pte_prefetch(vcpu, it.sptep); in direct_map()
3363 u64 *sptep, u64 old_spte, u64 new_spte) in fast_pf_fix_direct_spte() argument
3377 if (!try_cmpxchg64(sptep, &old_spte, new_spte)) in fast_pf_fix_direct_spte()
3411 u64 *sptep = NULL; in fast_pf_get_last_sptep() local
3414 sptep = iterator.sptep; in fast_pf_get_last_sptep()
3418 return sptep; in fast_pf_get_last_sptep()
3429 u64 *sptep = NULL; in fast_page_fault() local
3441 sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte); in fast_page_fault()
3443 sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte); in fast_page_fault()
3448 sp = sptep_to_sp(sptep); in fast_page_fault()
3517 if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) { in fast_page_fault()
3529 trace_fast_page_fault(vcpu, fault, sptep, spte, ret); in fast_page_fault()
4088 spte = mmu_spte_get_lockless(iterator.sptep); in get_walk()
4097 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) in get_mmio_spte() argument
4114 *sptep = 0ull; in get_mmio_spte()
4118 *sptep = sptes[leaf]; in get_mmio_spte()
4206 clear_sp_write_flooding_count(iterator.sptep); in shadow_page_table_clear_flood()
4671 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, in sync_mmio_spte() argument
4674 if (unlikely(is_mmio_spte(*sptep))) { in sync_mmio_spte()
4675 if (gfn != get_mmio_spte_gfn(*sptep)) { in sync_mmio_spte()
4676 mmu_spte_clear_no_track(sptep); in sync_mmio_spte()
4680 mark_mmio_spte(vcpu, sptep, gfn, access); in sync_mmio_spte()
5794 struct kvm_mmu_page *sp = sptep_to_sp(iterator.sptep); in __kvm_mmu_invalidate_addr()
5800 mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL); in __kvm_mmu_invalidate_addr()
5802 kvm_flush_remote_tlbs_sptep(vcpu->kvm, iterator.sptep); in __kvm_mmu_invalidate_addr()
6383 u64 *sptep, spte; in shadow_mmu_split_huge_page() local
6390 sptep = &sp->spt[index]; in shadow_mmu_split_huge_page()
6407 if (is_shadow_present_pte(*sptep)) { in shadow_mmu_split_huge_page()
6408 flush |= !is_last_spte(*sptep, sp->role.level); in shadow_mmu_split_huge_page()
6413 mmu_spte_set(sptep, spte); in shadow_mmu_split_huge_page()
6414 __rmap_add(kvm, cache, slot, sptep, gfn, sp->role.access); in shadow_mmu_split_huge_page()
6578 u64 *sptep; in kvm_mmu_zap_collapsible_spte() local
6584 for_each_rmap_spte(rmap_head, &iter, sptep) { in kvm_mmu_zap_collapsible_spte()
6585 sp = sptep_to_sp(sptep); in kvm_mmu_zap_collapsible_spte()
6597 kvm_zap_one_rmap_spte(kvm, rmap_head, sptep); in kvm_mmu_zap_collapsible_spte()
6600 kvm_flush_remote_tlbs_sptep(kvm, sptep); in kvm_mmu_zap_collapsible_spte()