Lines Matching refs:iter
207 struct tdp_iter *iter) in tdp_mmu_init_child_sp() argument
212 parent_sp = sptep_to_sp(rcu_dereference(iter->sptep)); in tdp_mmu_init_child_sp()
217 tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role); in tdp_mmu_init_child_sp()
529 struct tdp_iter *iter, in tdp_mmu_set_spte_atomic() argument
532 u64 *sptep = rcu_dereference(iter->sptep); in tdp_mmu_set_spte_atomic()
540 WARN_ON_ONCE(iter->yielded || is_removed_spte(iter->old_spte)); in tdp_mmu_set_spte_atomic()
551 if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte)) in tdp_mmu_set_spte_atomic()
554 handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte, in tdp_mmu_set_spte_atomic()
555 new_spte, iter->level, true); in tdp_mmu_set_spte_atomic()
561 struct tdp_iter *iter) in tdp_mmu_zap_spte_atomic() argument
571 ret = tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE); in tdp_mmu_zap_spte_atomic()
575 kvm_flush_remote_tlbs_gfn(kvm, iter->gfn, iter->level); in tdp_mmu_zap_spte_atomic()
584 __kvm_tdp_mmu_write_spte(iter->sptep, 0); in tdp_mmu_zap_spte_atomic()
623 static inline void tdp_mmu_iter_set_spte(struct kvm *kvm, struct tdp_iter *iter, in tdp_mmu_iter_set_spte() argument
626 WARN_ON_ONCE(iter->yielded); in tdp_mmu_iter_set_spte()
627 iter->old_spte = tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep, in tdp_mmu_iter_set_spte()
628 iter->old_spte, new_spte, in tdp_mmu_iter_set_spte()
629 iter->gfn, iter->level); in tdp_mmu_iter_set_spte()
660 struct tdp_iter *iter, in tdp_mmu_iter_cond_resched() argument
663 WARN_ON_ONCE(iter->yielded); in tdp_mmu_iter_cond_resched()
666 if (iter->next_last_level_gfn == iter->yielded_gfn) in tdp_mmu_iter_cond_resched()
682 WARN_ON_ONCE(iter->gfn > iter->next_last_level_gfn); in tdp_mmu_iter_cond_resched()
684 iter->yielded = true; in tdp_mmu_iter_cond_resched()
687 return iter->yielded; in tdp_mmu_iter_cond_resched()
704 struct tdp_iter iter; in __tdp_mmu_zap_root() local
709 for_each_tdp_pte_min_level(iter, root, zap_level, start, end) { in __tdp_mmu_zap_root()
711 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) in __tdp_mmu_zap_root()
714 if (!is_shadow_present_pte(iter.old_spte)) in __tdp_mmu_zap_root()
717 if (iter.level > zap_level) in __tdp_mmu_zap_root()
721 tdp_mmu_iter_set_spte(kvm, &iter, 0); in __tdp_mmu_zap_root()
722 else if (tdp_mmu_set_spte_atomic(kvm, &iter, 0)) in __tdp_mmu_zap_root()
794 struct tdp_iter iter; in tdp_mmu_zap_leafs() local
802 for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) { in tdp_mmu_zap_leafs()
804 tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) { in tdp_mmu_zap_leafs()
809 if (!is_shadow_present_pte(iter.old_spte) || in tdp_mmu_zap_leafs()
810 !is_last_spte(iter.old_spte, iter.level)) in tdp_mmu_zap_leafs()
813 tdp_mmu_iter_set_spte(kvm, &iter, 0); in tdp_mmu_zap_leafs()
954 struct tdp_iter *iter) in tdp_mmu_map_handle_target_level() argument
956 struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep)); in tdp_mmu_map_handle_target_level()
965 new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL); in tdp_mmu_map_handle_target_level()
967 wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn, in tdp_mmu_map_handle_target_level()
968 fault->pfn, iter->old_spte, fault->prefetch, true, in tdp_mmu_map_handle_target_level()
971 if (new_spte == iter->old_spte) in tdp_mmu_map_handle_target_level()
973 else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte)) in tdp_mmu_map_handle_target_level()
975 else if (is_shadow_present_pte(iter->old_spte) && in tdp_mmu_map_handle_target_level()
976 !is_last_spte(iter->old_spte, iter->level)) in tdp_mmu_map_handle_target_level()
977 kvm_flush_remote_tlbs_gfn(vcpu->kvm, iter->gfn, iter->level); in tdp_mmu_map_handle_target_level()
992 trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn, in tdp_mmu_map_handle_target_level()
996 trace_kvm_mmu_set_spte(iter->level, iter->gfn, in tdp_mmu_map_handle_target_level()
997 rcu_dereference(iter->sptep)); in tdp_mmu_map_handle_target_level()
1015 static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter, in tdp_mmu_link_sp() argument
1022 ret = tdp_mmu_set_spte_atomic(kvm, iter, spte); in tdp_mmu_link_sp()
1026 tdp_mmu_iter_set_spte(kvm, iter, spte); in tdp_mmu_link_sp()
1034 static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter,
1045 struct tdp_iter iter; in kvm_tdp_mmu_map() local
1055 tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) { in kvm_tdp_mmu_map()
1059 disallowed_hugepage_adjust(fault, iter.old_spte, iter.level); in kvm_tdp_mmu_map()
1065 if (is_removed_spte(iter.old_spte)) in kvm_tdp_mmu_map()
1068 if (iter.level == fault->goal_level) in kvm_tdp_mmu_map()
1072 if (is_shadow_present_pte(iter.old_spte) && in kvm_tdp_mmu_map()
1073 !is_large_pte(iter.old_spte)) in kvm_tdp_mmu_map()
1081 tdp_mmu_init_child_sp(sp, &iter); in kvm_tdp_mmu_map()
1085 if (is_shadow_present_pte(iter.old_spte)) in kvm_tdp_mmu_map()
1086 r = tdp_mmu_split_huge_page(kvm, &iter, sp, true); in kvm_tdp_mmu_map()
1088 r = tdp_mmu_link_sp(kvm, &iter, sp, true); in kvm_tdp_mmu_map()
1100 fault->req_level >= iter.level) { in kvm_tdp_mmu_map()
1112 WARN_ON_ONCE(iter.level == fault->goal_level); in kvm_tdp_mmu_map()
1116 ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter); in kvm_tdp_mmu_map()
1135 typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter,
1143 struct tdp_iter iter; in kvm_tdp_mmu_handle_gfn() local
1153 tdp_root_for_each_leaf_pte(iter, root, range->start, range->end) in kvm_tdp_mmu_handle_gfn()
1154 ret |= handler(kvm, &iter, range); in kvm_tdp_mmu_handle_gfn()
1170 static bool age_gfn_range(struct kvm *kvm, struct tdp_iter *iter, in age_gfn_range() argument
1176 if (!is_accessed_spte(iter->old_spte)) in age_gfn_range()
1179 if (spte_ad_enabled(iter->old_spte)) { in age_gfn_range()
1180 iter->old_spte = tdp_mmu_clear_spte_bits(iter->sptep, in age_gfn_range()
1181 iter->old_spte, in age_gfn_range()
1183 iter->level); in age_gfn_range()
1184 new_spte = iter->old_spte & ~shadow_accessed_mask; in age_gfn_range()
1190 if (is_writable_pte(iter->old_spte)) in age_gfn_range()
1191 kvm_set_pfn_dirty(spte_to_pfn(iter->old_spte)); in age_gfn_range()
1193 new_spte = mark_spte_for_access_track(iter->old_spte); in age_gfn_range()
1194 iter->old_spte = kvm_tdp_mmu_write_spte(iter->sptep, in age_gfn_range()
1195 iter->old_spte, new_spte, in age_gfn_range()
1196 iter->level); in age_gfn_range()
1199 trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, in age_gfn_range()
1200 iter->old_spte, new_spte); in age_gfn_range()
1209 static bool test_age_gfn(struct kvm *kvm, struct tdp_iter *iter, in test_age_gfn() argument
1212 return is_accessed_spte(iter->old_spte); in test_age_gfn()
1220 static bool set_spte_gfn(struct kvm *kvm, struct tdp_iter *iter, in set_spte_gfn() argument
1228 if (iter->level != PG_LEVEL_4K || in set_spte_gfn()
1229 !is_shadow_present_pte(iter->old_spte)) in set_spte_gfn()
1238 tdp_mmu_iter_set_spte(kvm, iter, 0); in set_spte_gfn()
1241 new_spte = kvm_mmu_changed_pte_notifier_make_spte(iter->old_spte, in set_spte_gfn()
1244 tdp_mmu_iter_set_spte(kvm, iter, new_spte); in set_spte_gfn()
1274 struct tdp_iter iter; in wrprot_gfn_range() local
1282 for_each_tdp_pte_min_level(iter, root, min_level, start, end) { in wrprot_gfn_range()
1284 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) in wrprot_gfn_range()
1287 if (!is_shadow_present_pte(iter.old_spte) || in wrprot_gfn_range()
1288 !is_last_spte(iter.old_spte, iter.level) || in wrprot_gfn_range()
1289 !(iter.old_spte & PT_WRITABLE_MASK)) in wrprot_gfn_range()
1292 new_spte = iter.old_spte & ~PT_WRITABLE_MASK; in wrprot_gfn_range()
1294 if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) in wrprot_gfn_range()
1344 struct tdp_iter *iter, in tdp_mmu_alloc_sp_for_split() argument
1369 iter->yielded = true; in tdp_mmu_alloc_sp_for_split()
1383 static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter, in tdp_mmu_split_huge_page() argument
1386 const u64 huge_spte = iter->old_spte; in tdp_mmu_split_huge_page()
1387 const int level = iter->level; in tdp_mmu_split_huge_page()
1405 ret = tdp_mmu_link_sp(kvm, iter, sp, shared); in tdp_mmu_split_huge_page()
1417 trace_kvm_mmu_split_huge_page(iter->gfn, huge_spte, level, ret); in tdp_mmu_split_huge_page()
1427 struct tdp_iter iter; in tdp_mmu_split_huge_pages_root() local
1443 for_each_tdp_pte_min_level(iter, root, target_level + 1, start, end) { in tdp_mmu_split_huge_pages_root()
1445 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) in tdp_mmu_split_huge_pages_root()
1448 if (!is_shadow_present_pte(iter.old_spte) || !is_large_pte(iter.old_spte)) in tdp_mmu_split_huge_pages_root()
1452 sp = tdp_mmu_alloc_sp_for_split(kvm, &iter, shared); in tdp_mmu_split_huge_pages_root()
1455 trace_kvm_mmu_split_huge_page(iter.gfn, in tdp_mmu_split_huge_pages_root()
1456 iter.old_spte, in tdp_mmu_split_huge_pages_root()
1457 iter.level, ret); in tdp_mmu_split_huge_pages_root()
1461 if (iter.yielded) in tdp_mmu_split_huge_pages_root()
1465 tdp_mmu_init_child_sp(sp, &iter); in tdp_mmu_split_huge_pages_root()
1467 if (tdp_mmu_split_huge_page(kvm, &iter, sp, shared)) in tdp_mmu_split_huge_pages_root()
1531 struct tdp_iter iter; in clear_dirty_gfn_range() local
1536 tdp_root_for_each_leaf_pte(iter, root, start, end) { in clear_dirty_gfn_range()
1538 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) in clear_dirty_gfn_range()
1541 if (!is_shadow_present_pte(iter.old_spte)) in clear_dirty_gfn_range()
1545 spte_ad_need_write_protect(iter.old_spte)); in clear_dirty_gfn_range()
1547 if (!(iter.old_spte & dbit)) in clear_dirty_gfn_range()
1550 if (tdp_mmu_set_spte_atomic(kvm, &iter, iter.old_spte & ~dbit)) in clear_dirty_gfn_range()
1594 struct tdp_iter iter; in clear_dirty_pt_masked() local
1600 tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask), in clear_dirty_pt_masked()
1606 spte_ad_need_write_protect(iter.old_spte)); in clear_dirty_pt_masked()
1608 if (iter.level > PG_LEVEL_4K || in clear_dirty_pt_masked()
1609 !(mask & (1UL << (iter.gfn - gfn)))) in clear_dirty_pt_masked()
1612 mask &= ~(1UL << (iter.gfn - gfn)); in clear_dirty_pt_masked()
1614 if (!(iter.old_spte & dbit)) in clear_dirty_pt_masked()
1617 iter.old_spte = tdp_mmu_clear_spte_bits(iter.sptep, in clear_dirty_pt_masked()
1618 iter.old_spte, dbit, in clear_dirty_pt_masked()
1619 iter.level); in clear_dirty_pt_masked()
1621 trace_kvm_tdp_mmu_spte_changed(iter.as_id, iter.gfn, iter.level, in clear_dirty_pt_masked()
1622 iter.old_spte, in clear_dirty_pt_masked()
1623 iter.old_spte & ~dbit); in clear_dirty_pt_masked()
1624 kvm_set_pfn_dirty(spte_to_pfn(iter.old_spte)); in clear_dirty_pt_masked()
1654 struct tdp_iter iter; in zap_collapsible_spte_range() local
1659 for_each_tdp_pte_min_level(iter, root, PG_LEVEL_2M, start, end) { in zap_collapsible_spte_range()
1661 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) in zap_collapsible_spte_range()
1664 if (iter.level > KVM_MAX_HUGEPAGE_LEVEL || in zap_collapsible_spte_range()
1665 !is_shadow_present_pte(iter.old_spte)) in zap_collapsible_spte_range()
1673 if (is_last_spte(iter.old_spte, iter.level)) in zap_collapsible_spte_range()
1683 if (iter.gfn < start || iter.gfn >= end) in zap_collapsible_spte_range()
1687 iter.gfn, PG_LEVEL_NUM); in zap_collapsible_spte_range()
1688 if (max_mapping_level < iter.level) in zap_collapsible_spte_range()
1692 if (tdp_mmu_zap_spte_atomic(kvm, &iter)) in zap_collapsible_spte_range()
1722 struct tdp_iter iter; in write_protect_gfn() local
1730 for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) { in write_protect_gfn()
1731 if (!is_shadow_present_pte(iter.old_spte) || in write_protect_gfn()
1732 !is_last_spte(iter.old_spte, iter.level)) in write_protect_gfn()
1735 new_spte = iter.old_spte & in write_protect_gfn()
1738 if (new_spte == iter.old_spte) in write_protect_gfn()
1741 tdp_mmu_iter_set_spte(kvm, &iter, new_spte); in write_protect_gfn()
1778 struct tdp_iter iter; in kvm_tdp_mmu_get_walk() local
1785 tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { in kvm_tdp_mmu_get_walk()
1786 leaf = iter.level; in kvm_tdp_mmu_get_walk()
1787 sptes[leaf] = iter.old_spte; in kvm_tdp_mmu_get_walk()
1807 struct tdp_iter iter; in kvm_tdp_mmu_fast_pf_get_last_sptep() local
1812 tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { in kvm_tdp_mmu_fast_pf_get_last_sptep()
1813 *spte = iter.old_spte; in kvm_tdp_mmu_fast_pf_get_last_sptep()
1814 sptep = iter.sptep; in kvm_tdp_mmu_fast_pf_get_last_sptep()