1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * TCE helpers for IODA PCI/PCIe on PowerNV platforms
4  *
5  * Copyright 2018 IBM Corp.
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License
9  * as published by the Free Software Foundation; either version
10  * 2 of the License, or (at your option) any later version.
11  */
12 
13 #include <linux/kernel.h>
14 #include <linux/iommu.h>
15 
16 #include <asm/iommu.h>
17 #include <asm/tce.h>
18 #include "pci.h"
19 
20 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
21 		void *tce_mem, u64 tce_size,
22 		u64 dma_offset, unsigned int page_shift)
23 {
24 	tbl->it_blocksize = 16;
25 	tbl->it_base = (unsigned long)tce_mem;
26 	tbl->it_page_shift = page_shift;
27 	tbl->it_offset = dma_offset >> tbl->it_page_shift;
28 	tbl->it_index = 0;
29 	tbl->it_size = tce_size >> 3;
30 	tbl->it_busno = 0;
31 	tbl->it_type = TCE_PCI;
32 }
33 
34 static __be64 *pnv_tce(struct iommu_table *tbl, long idx)
35 {
36 	__be64 *tmp = ((__be64 *)tbl->it_base);
37 	int  level = tbl->it_indirect_levels;
38 	const long shift = ilog2(tbl->it_level_size);
39 	unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
40 
41 	while (level) {
42 		int n = (idx & mask) >> (level * shift);
43 		unsigned long tce = be64_to_cpu(tmp[n]);
44 
45 		tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
46 		idx &= ~mask;
47 		mask >>= shift;
48 		--level;
49 	}
50 
51 	return tmp + idx;
52 }
53 
54 int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
55 		unsigned long uaddr, enum dma_data_direction direction,
56 		unsigned long attrs)
57 {
58 	u64 proto_tce = iommu_direction_to_tce_perm(direction);
59 	u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
60 	long i;
61 
62 	if (proto_tce & TCE_PCI_WRITE)
63 		proto_tce |= TCE_PCI_READ;
64 
65 	for (i = 0; i < npages; i++) {
66 		unsigned long newtce = proto_tce |
67 			((rpn + i) << tbl->it_page_shift);
68 		unsigned long idx = index - tbl->it_offset + i;
69 
70 		*(pnv_tce(tbl, idx)) = cpu_to_be64(newtce);
71 	}
72 
73 	return 0;
74 }
75 
76 #ifdef CONFIG_IOMMU_API
77 int pnv_tce_xchg(struct iommu_table *tbl, long index,
78 		unsigned long *hpa, enum dma_data_direction *direction)
79 {
80 	u64 proto_tce = iommu_direction_to_tce_perm(*direction);
81 	unsigned long newtce = *hpa | proto_tce, oldtce;
82 	unsigned long idx = index - tbl->it_offset;
83 
84 	BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
85 
86 	if (newtce & TCE_PCI_WRITE)
87 		newtce |= TCE_PCI_READ;
88 
89 	oldtce = be64_to_cpu(xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce)));
90 	*hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
91 	*direction = iommu_tce_direction(oldtce);
92 
93 	return 0;
94 }
95 #endif
96 
97 void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
98 {
99 	long i;
100 
101 	for (i = 0; i < npages; i++) {
102 		unsigned long idx = index - tbl->it_offset + i;
103 
104 		*(pnv_tce(tbl, idx)) = cpu_to_be64(0);
105 	}
106 }
107 
108 unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
109 {
110 	return be64_to_cpu(*(pnv_tce(tbl, index - tbl->it_offset)));
111 }
112 
113 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
114 		unsigned long size, unsigned int levels)
115 {
116 	const unsigned long addr_ul = (unsigned long) addr &
117 			~(TCE_PCI_READ | TCE_PCI_WRITE);
118 
119 	if (levels) {
120 		long i;
121 		u64 *tmp = (u64 *) addr_ul;
122 
123 		for (i = 0; i < size; ++i) {
124 			unsigned long hpa = be64_to_cpu(tmp[i]);
125 
126 			if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
127 				continue;
128 
129 			pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
130 					levels - 1);
131 		}
132 	}
133 
134 	free_pages(addr_ul, get_order(size << 3));
135 }
136 
137 void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
138 {
139 	const unsigned long size = tbl->it_indirect_levels ?
140 			tbl->it_level_size : tbl->it_size;
141 
142 	if (!tbl->it_size)
143 		return;
144 
145 	pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
146 			tbl->it_indirect_levels);
147 }
148 
149 static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift,
150 		unsigned int levels, unsigned long limit,
151 		unsigned long *current_offset, unsigned long *total_allocated)
152 {
153 	struct page *tce_mem = NULL;
154 	__be64 *addr, *tmp;
155 	unsigned int order = max_t(unsigned int, shift, PAGE_SHIFT) -
156 			PAGE_SHIFT;
157 	unsigned long allocated = 1UL << (order + PAGE_SHIFT);
158 	unsigned int entries = 1UL << (shift - 3);
159 	long i;
160 
161 	tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
162 	if (!tce_mem) {
163 		pr_err("Failed to allocate a TCE memory, order=%d\n", order);
164 		return NULL;
165 	}
166 	addr = page_address(tce_mem);
167 	memset(addr, 0, allocated);
168 	*total_allocated += allocated;
169 
170 	--levels;
171 	if (!levels) {
172 		*current_offset += allocated;
173 		return addr;
174 	}
175 
176 	for (i = 0; i < entries; ++i) {
177 		tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
178 				levels, limit, current_offset, total_allocated);
179 		if (!tmp)
180 			break;
181 
182 		addr[i] = cpu_to_be64(__pa(tmp) |
183 				TCE_PCI_READ | TCE_PCI_WRITE);
184 
185 		if (*current_offset >= limit)
186 			break;
187 	}
188 
189 	return addr;
190 }
191 
192 long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
193 		__u32 page_shift, __u64 window_size, __u32 levels,
194 		struct iommu_table *tbl)
195 {
196 	void *addr;
197 	unsigned long offset = 0, level_shift, total_allocated = 0;
198 	const unsigned int window_shift = ilog2(window_size);
199 	unsigned int entries_shift = window_shift - page_shift;
200 	unsigned int table_shift = max_t(unsigned int, entries_shift + 3,
201 			PAGE_SHIFT);
202 	const unsigned long tce_table_size = 1UL << table_shift;
203 
204 	if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
205 		return -EINVAL;
206 
207 	if (!is_power_of_2(window_size))
208 		return -EINVAL;
209 
210 	/* Adjust direct table size from window_size and levels */
211 	entries_shift = (entries_shift + levels - 1) / levels;
212 	level_shift = entries_shift + 3;
213 	level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT);
214 
215 	if ((level_shift - 3) * levels + page_shift >= 60)
216 		return -EINVAL;
217 
218 	/* Allocate TCE table */
219 	addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
220 			levels, tce_table_size, &offset, &total_allocated);
221 
222 	/* addr==NULL means that the first level allocation failed */
223 	if (!addr)
224 		return -ENOMEM;
225 
226 	/*
227 	 * First level was allocated but some lower level failed as
228 	 * we did not allocate as much as we wanted,
229 	 * release partially allocated table.
230 	 */
231 	if (offset < tce_table_size) {
232 		pnv_pci_ioda2_table_do_free_pages(addr,
233 				1ULL << (level_shift - 3), levels - 1);
234 		return -ENOMEM;
235 	}
236 
237 	/* Setup linux iommu table */
238 	pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
239 			page_shift);
240 	tbl->it_level_size = 1ULL << (level_shift - 3);
241 	tbl->it_indirect_levels = levels - 1;
242 	tbl->it_allocated_size = total_allocated;
243 
244 	pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
245 			window_size, tce_table_size, bus_offset);
246 
247 	return 0;
248 }
249 
250 static void pnv_iommu_table_group_link_free(struct rcu_head *head)
251 {
252 	struct iommu_table_group_link *tgl = container_of(head,
253 			struct iommu_table_group_link, rcu);
254 
255 	kfree(tgl);
256 }
257 
258 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
259 		struct iommu_table_group *table_group)
260 {
261 	long i;
262 	bool found;
263 	struct iommu_table_group_link *tgl;
264 
265 	if (!tbl || !table_group)
266 		return;
267 
268 	/* Remove link to a group from table's list of attached groups */
269 	found = false;
270 	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
271 		if (tgl->table_group == table_group) {
272 			list_del_rcu(&tgl->next);
273 			call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free);
274 			found = true;
275 			break;
276 		}
277 	}
278 	if (WARN_ON(!found))
279 		return;
280 
281 	/* Clean a pointer to iommu_table in iommu_table_group::tables[] */
282 	found = false;
283 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
284 		if (table_group->tables[i] == tbl) {
285 			table_group->tables[i] = NULL;
286 			found = true;
287 			break;
288 		}
289 	}
290 	WARN_ON(!found);
291 }
292 
293 long pnv_pci_link_table_and_group(int node, int num,
294 		struct iommu_table *tbl,
295 		struct iommu_table_group *table_group)
296 {
297 	struct iommu_table_group_link *tgl = NULL;
298 
299 	if (WARN_ON(!tbl || !table_group))
300 		return -EINVAL;
301 
302 	tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
303 			node);
304 	if (!tgl)
305 		return -ENOMEM;
306 
307 	tgl->table_group = table_group;
308 	list_add_rcu(&tgl->next, &tbl->it_group_list);
309 
310 	table_group->tables[num] = tbl;
311 
312 	return 0;
313 }
314