65 	iommu_tce_table_put(stit->tbl);  in kvm_spapr_tce_iommu_table_free()
97 				if (table_group->tables[i] != stit->tbl)  in kvm_spapr_tce_release_iommu_group()
113 	struct iommu_table *tbl = NULL;  in kvm_spapr_tce_attach_iommu_group()  local
156 			tbl = iommu_tce_table_get(tbltmp);  in kvm_spapr_tce_attach_iommu_group()
160 	if (!tbl)  in kvm_spapr_tce_attach_iommu_group()
165 		if (tbl != stit->tbl)  in kvm_spapr_tce_attach_iommu_group()
170 			iommu_tce_table_put(tbl);  in kvm_spapr_tce_attach_iommu_group()
185 		iommu_tce_table_put(tbl);  in kvm_spapr_tce_attach_iommu_group()
189 	stit->tbl = tbl;  in kvm_spapr_tce_attach_iommu_group()
392 		long shift = stit->tbl->it_page_shift;  in kvmppc_tce_validate()
414 	u64 *tbl;  in kvmppc_tce_put()  local
430 	tbl = page_to_virt(page);  in kvmppc_tce_put()
432 	tbl[idx % TCES_PER_PAGE] = tce;  in kvmppc_tce_put()
436 		struct iommu_table *tbl, unsigned long entry)  in kvmppc_clear_tce()  argument
439 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_clear_tce()
440 	unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift);  in kvmppc_clear_tce()
446 		iommu_tce_xchg_no_kill(mm, tbl, io_entry + i, &hpa, &dir);  in kvmppc_clear_tce()
451 		struct iommu_table *tbl, unsigned long entry)  in kvmppc_tce_iommu_mapped_dec()  argument
454 	const unsigned long pgsize = 1ULL << tbl->it_page_shift;  in kvmppc_tce_iommu_mapped_dec()
455 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);  in kvmppc_tce_iommu_mapped_dec()
472 		struct iommu_table *tbl, unsigned long entry)  in kvmppc_tce_iommu_do_unmap()  argument
478 	if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa,  in kvmppc_tce_iommu_do_unmap()
485 	ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);  in kvmppc_tce_iommu_do_unmap()
487 		iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);  in kvmppc_tce_iommu_do_unmap()
493 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_unmap()  argument
497 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_unmap()
501 		ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i);  in kvmppc_tce_iommu_unmap()
506 	iommu_tce_kill(tbl, io_entry, subpages);  in kvmppc_tce_iommu_unmap()
511 static long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,  in kvmppc_tce_iommu_do_map()  argument
517 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);  in kvmppc_tce_iommu_do_map()
524 	mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift);  in kvmppc_tce_iommu_do_map()
529 	if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, tbl->it_page_shift, &hpa)))  in kvmppc_tce_iommu_do_map()
535 	ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);  in kvmppc_tce_iommu_do_map()
542 		kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);  in kvmppc_tce_iommu_do_map()
550 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_map()  argument
555 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_map()
559 			++i, pgoff += IOMMU_PAGE_SIZE(tbl)) {  in kvmppc_tce_iommu_map()
561 		ret = kvmppc_tce_iommu_do_map(kvm, tbl,  in kvmppc_tce_iommu_map()
567 	iommu_tce_kill(tbl, io_entry, subpages);  in kvmppc_tce_iommu_map()
610 					stit->tbl, entry);  in kvmppc_h_put_tce()
612 			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,  in kvmppc_h_put_tce()
617 			kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry);  in kvmppc_h_put_tce()
704 					stit->tbl, entry + i, ua,  in kvmppc_h_put_tce_indirect()
708 				kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl,  in kvmppc_h_put_tce_indirect()
749 					stit->tbl, entry + i);  in kvmppc_h_stuff_tce()
758 			kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i);  in kvmppc_h_stuff_tce()
776 	u64 *tbl;  in kvmppc_h_get_tce()  local
792 	tbl = (u64 *)page_address(page);  in kvmppc_h_get_tce()
794 	vcpu->arch.regs.gpr[4] = tbl[idx % TCES_PER_PAGE];  in kvmppc_h_get_tce()