138 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)  in kvm_pte_follow()  argument
140 	return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));  in kvm_pte_follow()
148 static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)  in kvm_init_table_pte()  argument
150 	kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));  in kvm_init_table_pte()
206 			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level);
209 				      struct kvm_pgtable_mm_ops *mm_ops,  in __kvm_pgtable_visit()  argument
218 		.mm_ops	= mm_ops,  in __kvm_pgtable_visit()
259 	childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);  in __kvm_pgtable_visit()
260 	ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);  in __kvm_pgtable_visit()
275 			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level)  in __kvm_pgtable_walk()  argument
289 		ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);  in __kvm_pgtable_walk()
312 		ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);  in _kvm_pgtable_walk()
452 		ctx->mm_ops->get_page(ctx->ptep);  in hyp_map_walker_try_leaf()
465 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in hyp_map_walker()  local
473 	childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);  in hyp_map_walker()
477 	new = kvm_init_table_pte(childp, mm_ops);  in hyp_map_walker()
478 	mm_ops->get_page(ctx->ptep);  in hyp_map_walker()
513 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in hyp_unmap_walker()  local
519 		childp = kvm_pte_follow(ctx->old, mm_ops);  in hyp_unmap_walker()
521 		if (mm_ops->page_count(childp) != 1)  in hyp_unmap_walker()
539 	mm_ops->put_page(ctx->ptep);  in hyp_unmap_walker()
542 		mm_ops->put_page(childp);  in hyp_unmap_walker()
556 	if (!pgt->mm_ops->page_count)  in kvm_pgtable_hyp_unmap()
564 			 struct kvm_pgtable_mm_ops *mm_ops)  in kvm_pgtable_hyp_init()  argument
568 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);  in kvm_pgtable_hyp_init()
574 	pgt->mm_ops		= mm_ops;  in kvm_pgtable_hyp_init()
584 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in hyp_free_walker()  local
589 	mm_ops->put_page(ctx->ptep);  in hyp_free_walker()
592 		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));  in hyp_free_walker()
605 	pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));  in kvm_pgtable_hyp_destroy()
789 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_try_break_pte()  local
820 		mm_ops->put_page(ctx->ptep);  in stage2_try_break_pte()
827 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_make_pte()  local
832 		mm_ops->get_page(ctx->ptep);  in stage2_make_pte()
852 				struct kvm_pgtable_mm_ops *mm_ops)  in stage2_unmap_put_pte()  argument
873 	mm_ops->put_page(ctx->ptep);  in stage2_unmap_put_pte()
925 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_map_walker_try_leaf()  local
948 	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc &&  in stage2_map_walker_try_leaf()
950 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops),  in stage2_map_walker_try_leaf()
953 	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou &&  in stage2_map_walker_try_leaf()
955 		mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);  in stage2_map_walker_try_leaf()
965 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_map_walk_table_pre()  local
966 	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);  in stage2_map_walk_table_pre()
976 	mm_ops->free_unlinked_table(childp, ctx->level);  in stage2_map_walk_table_pre()
983 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_map_walk_leaf()  local
997 	childp = mm_ops->zalloc_page(data->memcache);  in stage2_map_walk_leaf()
1002 		mm_ops->put_page(childp);  in stage2_map_walk_leaf()
1011 	new = kvm_init_table_pte(childp, mm_ops);  in stage2_map_walk_leaf()
1102 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_unmap_walker()  local
1109 			mm_ops->put_page(ctx->ptep);  in stage2_unmap_walker()
1115 		childp = kvm_pte_follow(ctx->old, mm_ops);  in stage2_unmap_walker()
1117 		if (mm_ops->page_count(childp) != 1)  in stage2_unmap_walker()
1128 	stage2_unmap_put_pte(ctx, mmu, mm_ops);  in stage2_unmap_walker()
1130 	if (need_flush && mm_ops->dcache_clean_inval_poc)  in stage2_unmap_walker()
1131 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),  in stage2_unmap_walker()
1135 		mm_ops->put_page(childp);  in stage2_unmap_walker()
1169 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_attr_walker()  local
1189 		if (mm_ops->icache_inval_pou &&  in stage2_attr_walker()
1191 			mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),  in stage2_attr_walker()
1333 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_flush_walker()  local
1338 	if (mm_ops->dcache_clean_inval_poc)  in stage2_flush_walker()
1339 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),  in stage2_flush_walker()
1386 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in kvm_pgtable_stage2_create_unlinked()  local
1397 	pgtable = mm_ops->zalloc_page(mc);  in kvm_pgtable_stage2_create_unlinked()
1401 	ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable,  in kvm_pgtable_stage2_create_unlinked()
1404 		kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level);  in kvm_pgtable_stage2_create_unlinked()
1405 		mm_ops->put_page(pgtable);  in kvm_pgtable_stage2_create_unlinked()
1436 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_split_walker()  local
1487 		kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level);  in stage2_split_walker()
1488 		mm_ops->put_page(childp);  in stage2_split_walker()
1497 	new = kvm_init_table_pte(childp, mm_ops);  in stage2_split_walker()
1516 			      struct kvm_pgtable_mm_ops *mm_ops,  in __kvm_pgtable_stage2_init()  argument
1527 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);  in __kvm_pgtable_stage2_init()
1533 	pgt->mm_ops		= mm_ops;  in __kvm_pgtable_stage2_init()
1555 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_free_walker()  local
1560 	mm_ops->put_page(ctx->ptep);  in stage2_free_walker()
1563 		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));  in stage2_free_walker()
1579 	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);  in kvm_pgtable_stage2_destroy()
1583 void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level)  in kvm_pgtable_stage2_free_unlinked()  argument
1603 	WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));  in kvm_pgtable_stage2_free_unlinked()
1605 	WARN_ON(mm_ops->page_count(pgtable) != 1);  in kvm_pgtable_stage2_free_unlinked()
1606 	mm_ops->put_page(pgtable);  in kvm_pgtable_stage2_free_unlinked()