48 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,  in pnv_pci_setup_iommu_table()  argument
52 	tbl->it_blocksize = 16;  in pnv_pci_setup_iommu_table()
53 	tbl->it_base = (unsigned long)tce_mem;  in pnv_pci_setup_iommu_table()
54 	tbl->it_page_shift = page_shift;  in pnv_pci_setup_iommu_table()
55 	tbl->it_offset = dma_offset >> tbl->it_page_shift;  in pnv_pci_setup_iommu_table()
56 	tbl->it_index = 0;  in pnv_pci_setup_iommu_table()
57 	tbl->it_size = tce_size >> 3;  in pnv_pci_setup_iommu_table()
58 	tbl->it_busno = 0;  in pnv_pci_setup_iommu_table()
59 	tbl->it_type = TCE_PCI;  in pnv_pci_setup_iommu_table()
83 static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc)  in pnv_tce()  argument
85 	__be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base;  in pnv_tce()
86 	int  level = tbl->it_indirect_levels;  in pnv_tce()
87 	const long shift = ilog2(tbl->it_level_size);  in pnv_tce()
88 	unsigned long mask = (tbl->it_level_size - 1) << (level * shift);  in pnv_tce()
100 			tmp2 = pnv_alloc_tce_level(tbl->it_nid,  in pnv_tce()
101 					ilog2(tbl->it_level_size) + 3);  in pnv_tce()
110 					ilog2(tbl->it_level_size) + 3, 1);  in pnv_tce()
124 int pnv_tce_build(struct iommu_table *tbl, long index, long npages,  in pnv_tce_build()  argument
129 	u64 rpn = __pa(uaddr) >> tbl->it_page_shift;  in pnv_tce_build()
137 			((rpn + i) << tbl->it_page_shift);  in pnv_tce_build()
138 		unsigned long idx = index - tbl->it_offset + i;  in pnv_tce_build()
140 		*(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce);  in pnv_tce_build()
147 int pnv_tce_xchg(struct iommu_table *tbl, long index,  in pnv_tce_xchg()  argument
152 	unsigned long idx = index - tbl->it_offset;  in pnv_tce_xchg()
155 	BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));  in pnv_tce_xchg()
158 		ptce = pnv_tce(tbl, false, idx, false);  in pnv_tce_xchg()
166 		ptce = pnv_tce(tbl, false, idx, true);  in pnv_tce_xchg()
181 __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc)  in pnv_tce_useraddrptr()  argument
183 	if (WARN_ON_ONCE(!tbl->it_userspace))  in pnv_tce_useraddrptr()
186 	return pnv_tce(tbl, true, index - tbl->it_offset, alloc);  in pnv_tce_useraddrptr()
190 void pnv_tce_free(struct iommu_table *tbl, long index, long npages)  in pnv_tce_free()  argument
195 		unsigned long idx = index - tbl->it_offset + i;  in pnv_tce_free()
196 		__be64 *ptce = pnv_tce(tbl, false, idx,	false);  in pnv_tce_free()
202 			i |= tbl->it_level_size - 1;  in pnv_tce_free()
206 unsigned long pnv_tce_get(struct iommu_table *tbl, long index)  in pnv_tce_get()  argument
208 	__be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false);  in pnv_tce_get()
240 void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)  in pnv_pci_ioda2_table_free_pages()  argument
242 	const unsigned long size = tbl->it_indirect_levels ?  in pnv_pci_ioda2_table_free_pages()
243 			tbl->it_level_size : tbl->it_size;  in pnv_pci_ioda2_table_free_pages()
245 	if (!tbl->it_size)  in pnv_pci_ioda2_table_free_pages()
248 	pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,  in pnv_pci_ioda2_table_free_pages()
249 			tbl->it_indirect_levels);  in pnv_pci_ioda2_table_free_pages()
250 	if (tbl->it_userspace) {  in pnv_pci_ioda2_table_free_pages()
251 		pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size,  in pnv_pci_ioda2_table_free_pages()
252 				tbl->it_indirect_levels);  in pnv_pci_ioda2_table_free_pages()
292 		bool alloc_userspace_copy, struct iommu_table *tbl)  in pnv_pci_ioda2_table_alloc_pages()  argument
347 	pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,  in pnv_pci_ioda2_table_alloc_pages()
349 	tbl->it_level_size = 1ULL << (level_shift - 3);  in pnv_pci_ioda2_table_alloc_pages()
350 	tbl->it_indirect_levels = levels - 1;  in pnv_pci_ioda2_table_alloc_pages()
351 	tbl->it_userspace = uas;  in pnv_pci_ioda2_table_alloc_pages()
352 	tbl->it_nid = nid;  in pnv_pci_ioda2_table_alloc_pages()
355 			window_size, tce_table_size, bus_offset, tbl->it_base,  in pnv_pci_ioda2_table_alloc_pages()
356 			tbl->it_userspace, 1, levels);  in pnv_pci_ioda2_table_alloc_pages()
370 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,  in pnv_pci_unlink_table_and_group()  argument
377 	if (!tbl || !table_group)  in pnv_pci_unlink_table_and_group()
384 	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {  in pnv_pci_unlink_table_and_group()
400 		if (table_group->tables[i] == tbl) {  in pnv_pci_unlink_table_and_group()
401 			iommu_tce_table_put(tbl);  in pnv_pci_unlink_table_and_group()
411 		struct iommu_table *tbl,  in pnv_pci_link_table_and_group()  argument
416 	if (WARN_ON(!tbl || !table_group))  in pnv_pci_link_table_and_group()
425 	list_add_rcu(&tgl->next, &tbl->it_group_list);  in pnv_pci_link_table_and_group()
427 	table_group->tables[num] = iommu_tce_table_get(tbl);  in pnv_pci_link_table_and_group()