109 static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgd_page_idx()  argument
111 	u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */  in kvm_pgd_page_idx()
112 	u64 mask = BIT(pgt->ia_bits) - 1;  in kvm_pgd_page_idx()
119 	struct kvm_pgtable pgt = {  in kvm_pgd_pages()  local
124 	return kvm_pgd_page_idx(&pgt, -1ULL) + 1;  in kvm_pgd_pages()
297 static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)  in _kvm_pgtable_walk()  argument
301 	u64 limit = BIT(pgt->ia_bits);  in _kvm_pgtable_walk()
306 	if (!pgt->pgd)  in _kvm_pgtable_walk()
309 	for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {  in _kvm_pgtable_walk()
310 		kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];  in _kvm_pgtable_walk()
312 		ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);  in _kvm_pgtable_walk()
320 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_walk()  argument
335 	r = _kvm_pgtable_walk(pgt, &walk_data);  in kvm_pgtable_walk()
357 int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_get_leaf()  argument
368 	ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE),  in kvm_pgtable_get_leaf()
484 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,  in kvm_pgtable_hyp_map()  argument
501 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_hyp_map()
547 u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_hyp_unmap()  argument
556 	if (!pgt->mm_ops->page_count)  in kvm_pgtable_hyp_unmap()
559 	kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_hyp_unmap()
563 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,  in kvm_pgtable_hyp_init()  argument
568 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);  in kvm_pgtable_hyp_init()
569 	if (!pgt->pgd)  in kvm_pgtable_hyp_init()
572 	pgt->ia_bits		= va_bits;  in kvm_pgtable_hyp_init()
573 	pgt->start_level	= KVM_PGTABLE_MAX_LEVELS - levels;  in kvm_pgtable_hyp_init()
574 	pgt->mm_ops		= mm_ops;  in kvm_pgtable_hyp_init()
575 	pgt->mmu		= NULL;  in kvm_pgtable_hyp_init()
576 	pgt->force_pte_cb	= NULL;  in kvm_pgtable_hyp_init()
597 void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)  in kvm_pgtable_hyp_destroy()  argument
604 	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));  in kvm_pgtable_hyp_destroy()
605 	pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));  in kvm_pgtable_hyp_destroy()
606 	pgt->pgd = NULL;  in kvm_pgtable_hyp_destroy()
665 static bool stage2_has_fwb(struct kvm_pgtable *pgt)  in stage2_has_fwb()  argument
670 	return !(pgt->flags & KVM_PGTABLE_S2_NOFWB);  in stage2_has_fwb()
693 #define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))  argument
695 static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,  in stage2_set_prot_attr()  argument
699 	kvm_pte_t attr = device ? KVM_S2_MEMATTR(pgt, DEVICE_nGnRE) :  in stage2_set_prot_attr()
700 			    KVM_S2_MEMATTR(pgt, NORMAL);  in stage2_set_prot_attr()
837 static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt)  in stage2_unmap_defer_tlb_flush()  argument
847 	return system_supports_tlb_range() && stage2_has_fwb(pgt);  in stage2_unmap_defer_tlb_flush()
854 	struct kvm_pgtable *pgt = ctx->arg;  in stage2_unmap_put_pte()  local
867 		} else if (!stage2_unmap_defer_tlb_flush(pgt)) {  in stage2_unmap_put_pte()
876 static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)  in stage2_pte_cacheable()  argument
879 	return memattr == KVM_S2_MEMATTR(pgt, NORMAL);  in stage2_pte_cacheable()
924 	struct kvm_pgtable *pgt = data->mmu->pgt;  in stage2_map_walker_try_leaf()  local
949 	    stage2_pte_cacheable(pgt, new))  in stage2_map_walker_try_leaf()
1041 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_map()  argument
1048 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_map()
1050 		.force_pte	= pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),  in kvm_pgtable_stage2_map()
1060 	if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))  in kvm_pgtable_stage2_map()
1063 	ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);  in kvm_pgtable_stage2_map()
1067 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_map()
1072 int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_set_owner()  argument
1078 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_set_owner()
1093 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_set_owner()
1100 	struct kvm_pgtable *pgt = ctx->arg;  in stage2_unmap_walker()  local
1101 	struct kvm_s2_mmu *mmu = pgt->mmu;  in stage2_unmap_walker()
1119 	} else if (stage2_pte_cacheable(pgt, ctx->old)) {  in stage2_unmap_walker()
1120 		need_flush = !stage2_has_fwb(pgt);  in stage2_unmap_walker()
1140 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_unmap()  argument
1145 		.arg	= pgt,  in kvm_pgtable_stage2_unmap()
1149 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_unmap()
1150 	if (stage2_unmap_defer_tlb_flush(pgt))  in kvm_pgtable_stage2_unmap()
1152 		kvm_tlb_flush_vmid_range(pgt->mmu, addr, size);  in kvm_pgtable_stage2_unmap()
1201 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,  in stage2_update_leaf_attrs()  argument
1218 	ret = kvm_pgtable_walk(pgt, addr, size, &walker);  in stage2_update_leaf_attrs()
1230 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_wrprotect()  argument
1232 	return stage2_update_leaf_attrs(pgt, addr, size, 0,  in kvm_pgtable_stage2_wrprotect()
1237 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)  in kvm_pgtable_stage2_mkyoung()  argument
1242 	ret = stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,  in kvm_pgtable_stage2_mkyoung()
1286 bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_stage2_test_clear_young()  argument
1298 	WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));  in kvm_pgtable_stage2_test_clear_young()
1302 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,  in kvm_pgtable_stage2_relax_perms()  argument
1321 	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level,  in kvm_pgtable_stage2_relax_perms()
1325 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level);  in kvm_pgtable_stage2_relax_perms()
1332 	struct kvm_pgtable *pgt = ctx->arg;  in stage2_flush_walker()  local
1333 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_flush_walker()
1335 	if (!kvm_pte_valid(ctx->old) || !stage2_pte_cacheable(pgt, ctx->old))  in stage2_flush_walker()
1344 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)  in kvm_pgtable_stage2_flush()  argument
1349 		.arg	= pgt,  in kvm_pgtable_stage2_flush()
1352 	if (stage2_has_fwb(pgt))  in kvm_pgtable_stage2_flush()
1355 	return kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_flush()
1358 kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,  in kvm_pgtable_stage2_create_unlinked()  argument
1365 		.mmu		= pgt->mmu,  in kvm_pgtable_stage2_create_unlinked()
1386 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in kvm_pgtable_stage2_create_unlinked()
1393 	ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);  in kvm_pgtable_stage2_create_unlinked()
1481 	childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys,  in stage2_split_walker()
1503 int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,  in kvm_pgtable_stage2_split()  argument
1512 	return kvm_pgtable_walk(pgt, addr, size, &walker);  in kvm_pgtable_stage2_split()
1515 int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,  in __kvm_pgtable_stage2_init()  argument
1527 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);  in __kvm_pgtable_stage2_init()
1528 	if (!pgt->pgd)  in __kvm_pgtable_stage2_init()
1531 	pgt->ia_bits		= ia_bits;  in __kvm_pgtable_stage2_init()
1532 	pgt->start_level	= start_level;  in __kvm_pgtable_stage2_init()
1533 	pgt->mm_ops		= mm_ops;  in __kvm_pgtable_stage2_init()
1534 	pgt->mmu		= mmu;  in __kvm_pgtable_stage2_init()
1535 	pgt->flags		= flags;  in __kvm_pgtable_stage2_init()
1536 	pgt->force_pte_cb	= force_pte_cb;  in __kvm_pgtable_stage2_init()
1568 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)  in kvm_pgtable_stage2_destroy()  argument
1577 	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));  in kvm_pgtable_stage2_destroy()
1578 	pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;  in kvm_pgtable_stage2_destroy()
1579 	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);  in kvm_pgtable_stage2_destroy()
1580 	pgt->pgd = NULL;  in kvm_pgtable_stage2_destroy()