1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * TCE helpers for IODA PCI/PCIe on PowerNV platforms
4  *
5  * Copyright 2018 IBM Corp.
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License
9  * as published by the Free Software Foundation; either version
10  * 2 of the License, or (at your option) any later version.
11  */
12 
13 #include <linux/kernel.h>
14 #include <linux/iommu.h>
15 
16 #include <asm/iommu.h>
17 #include <asm/tce.h>
18 #include "pci.h"
19 
20 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
21 		void *tce_mem, u64 tce_size,
22 		u64 dma_offset, unsigned int page_shift)
23 {
24 	tbl->it_blocksize = 16;
25 	tbl->it_base = (unsigned long)tce_mem;
26 	tbl->it_page_shift = page_shift;
27 	tbl->it_offset = dma_offset >> tbl->it_page_shift;
28 	tbl->it_index = 0;
29 	tbl->it_size = tce_size >> 3;
30 	tbl->it_busno = 0;
31 	tbl->it_type = TCE_PCI;
32 }
33 
34 static __be64 *pnv_alloc_tce_level(int nid, unsigned int shift)
35 {
36 	struct page *tce_mem = NULL;
37 	__be64 *addr;
38 
39 	tce_mem = alloc_pages_node(nid, GFP_KERNEL, shift - PAGE_SHIFT);
40 	if (!tce_mem) {
41 		pr_err("Failed to allocate a TCE memory, level shift=%d\n",
42 				shift);
43 		return NULL;
44 	}
45 	addr = page_address(tce_mem);
46 	memset(addr, 0, 1UL << shift);
47 
48 	return addr;
49 }
50 
51 static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc)
52 {
53 	__be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base;
54 	int  level = tbl->it_indirect_levels;
55 	const long shift = ilog2(tbl->it_level_size);
56 	unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
57 
58 	while (level) {
59 		int n = (idx & mask) >> (level * shift);
60 		unsigned long tce;
61 
62 		if (tmp[n] == 0) {
63 			__be64 *tmp2;
64 
65 			if (!alloc)
66 				return NULL;
67 
68 			tmp2 = pnv_alloc_tce_level(tbl->it_nid,
69 					ilog2(tbl->it_level_size) + 3);
70 			if (!tmp2)
71 				return NULL;
72 
73 			tmp[n] = cpu_to_be64(__pa(tmp2) |
74 					TCE_PCI_READ | TCE_PCI_WRITE);
75 		}
76 		tce = be64_to_cpu(tmp[n]);
77 
78 		tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
79 		idx &= ~mask;
80 		mask >>= shift;
81 		--level;
82 	}
83 
84 	return tmp + idx;
85 }
86 
87 int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
88 		unsigned long uaddr, enum dma_data_direction direction,
89 		unsigned long attrs)
90 {
91 	u64 proto_tce = iommu_direction_to_tce_perm(direction);
92 	u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
93 	long i;
94 
95 	if (proto_tce & TCE_PCI_WRITE)
96 		proto_tce |= TCE_PCI_READ;
97 
98 	for (i = 0; i < npages; i++) {
99 		unsigned long newtce = proto_tce |
100 			((rpn + i) << tbl->it_page_shift);
101 		unsigned long idx = index - tbl->it_offset + i;
102 
103 		*(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce);
104 	}
105 
106 	return 0;
107 }
108 
109 #ifdef CONFIG_IOMMU_API
110 int pnv_tce_xchg(struct iommu_table *tbl, long index,
111 		unsigned long *hpa, enum dma_data_direction *direction,
112 		bool alloc)
113 {
114 	u64 proto_tce = iommu_direction_to_tce_perm(*direction);
115 	unsigned long newtce = *hpa | proto_tce, oldtce;
116 	unsigned long idx = index - tbl->it_offset;
117 	__be64 *ptce = NULL;
118 
119 	BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
120 
121 	if (*direction == DMA_NONE) {
122 		ptce = pnv_tce(tbl, false, idx, false);
123 		if (!ptce) {
124 			*hpa = 0;
125 			return 0;
126 		}
127 	}
128 
129 	if (!ptce) {
130 		ptce = pnv_tce(tbl, false, idx, alloc);
131 		if (!ptce)
132 			return alloc ? H_HARDWARE : H_TOO_HARD;
133 	}
134 
135 	if (newtce & TCE_PCI_WRITE)
136 		newtce |= TCE_PCI_READ;
137 
138 	oldtce = be64_to_cpu(xchg(ptce, cpu_to_be64(newtce)));
139 	*hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
140 	*direction = iommu_tce_direction(oldtce);
141 
142 	return 0;
143 }
144 
145 __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc)
146 {
147 	if (WARN_ON_ONCE(!tbl->it_userspace))
148 		return NULL;
149 
150 	return pnv_tce(tbl, true, index - tbl->it_offset, alloc);
151 }
152 #endif
153 
154 void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
155 {
156 	long i;
157 
158 	for (i = 0; i < npages; i++) {
159 		unsigned long idx = index - tbl->it_offset + i;
160 		__be64 *ptce = pnv_tce(tbl, false, idx,	false);
161 
162 		if (ptce)
163 			*ptce = cpu_to_be64(0);
164 	}
165 }
166 
167 unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
168 {
169 	__be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false);
170 
171 	if (!ptce)
172 		return 0;
173 
174 	return be64_to_cpu(*ptce);
175 }
176 
177 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
178 		unsigned long size, unsigned int levels)
179 {
180 	const unsigned long addr_ul = (unsigned long) addr &
181 			~(TCE_PCI_READ | TCE_PCI_WRITE);
182 
183 	if (levels) {
184 		long i;
185 		u64 *tmp = (u64 *) addr_ul;
186 
187 		for (i = 0; i < size; ++i) {
188 			unsigned long hpa = be64_to_cpu(tmp[i]);
189 
190 			if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
191 				continue;
192 
193 			pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
194 					levels - 1);
195 		}
196 	}
197 
198 	free_pages(addr_ul, get_order(size << 3));
199 }
200 
201 void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
202 {
203 	const unsigned long size = tbl->it_indirect_levels ?
204 			tbl->it_level_size : tbl->it_size;
205 
206 	if (!tbl->it_size)
207 		return;
208 
209 	pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
210 			tbl->it_indirect_levels);
211 	if (tbl->it_userspace) {
212 		pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size,
213 				tbl->it_indirect_levels);
214 	}
215 }
216 
217 static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift,
218 		unsigned int levels, unsigned long limit,
219 		unsigned long *current_offset, unsigned long *total_allocated)
220 {
221 	__be64 *addr, *tmp;
222 	unsigned long allocated = 1UL << shift;
223 	unsigned int entries = 1UL << (shift - 3);
224 	long i;
225 
226 	addr = pnv_alloc_tce_level(nid, shift);
227 	*total_allocated += allocated;
228 
229 	--levels;
230 	if (!levels) {
231 		*current_offset += allocated;
232 		return addr;
233 	}
234 
235 	for (i = 0; i < entries; ++i) {
236 		tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
237 				levels, limit, current_offset, total_allocated);
238 		if (!tmp)
239 			break;
240 
241 		addr[i] = cpu_to_be64(__pa(tmp) |
242 				TCE_PCI_READ | TCE_PCI_WRITE);
243 
244 		if (*current_offset >= limit)
245 			break;
246 	}
247 
248 	return addr;
249 }
250 
251 long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
252 		__u32 page_shift, __u64 window_size, __u32 levels,
253 		bool alloc_userspace_copy, struct iommu_table *tbl)
254 {
255 	void *addr, *uas = NULL;
256 	unsigned long offset = 0, level_shift, total_allocated = 0;
257 	unsigned long total_allocated_uas = 0;
258 	const unsigned int window_shift = ilog2(window_size);
259 	unsigned int entries_shift = window_shift - page_shift;
260 	unsigned int table_shift = max_t(unsigned int, entries_shift + 3,
261 			PAGE_SHIFT);
262 	const unsigned long tce_table_size = 1UL << table_shift;
263 	unsigned int tmplevels = levels;
264 
265 	if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
266 		return -EINVAL;
267 
268 	if (!is_power_of_2(window_size))
269 		return -EINVAL;
270 
271 	if (alloc_userspace_copy && (window_size > (1ULL << 32)))
272 		tmplevels = 1;
273 
274 	/* Adjust direct table size from window_size and levels */
275 	entries_shift = (entries_shift + levels - 1) / levels;
276 	level_shift = entries_shift + 3;
277 	level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT);
278 
279 	if ((level_shift - 3) * levels + page_shift >= 55)
280 		return -EINVAL;
281 
282 	/* Allocate TCE table */
283 	addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
284 			tmplevels, tce_table_size, &offset, &total_allocated);
285 
286 	/* addr==NULL means that the first level allocation failed */
287 	if (!addr)
288 		return -ENOMEM;
289 
290 	/*
291 	 * First level was allocated but some lower level failed as
292 	 * we did not allocate as much as we wanted,
293 	 * release partially allocated table.
294 	 */
295 	if (tmplevels == levels && offset < tce_table_size)
296 		goto free_tces_exit;
297 
298 	/* Allocate userspace view of the TCE table */
299 	if (alloc_userspace_copy) {
300 		offset = 0;
301 		uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
302 				tmplevels, tce_table_size, &offset,
303 				&total_allocated_uas);
304 		if (!uas)
305 			goto free_tces_exit;
306 		if (tmplevels == levels && (offset < tce_table_size ||
307 				total_allocated_uas != total_allocated))
308 			goto free_uas_exit;
309 	}
310 
311 	/* Setup linux iommu table */
312 	pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
313 			page_shift);
314 	tbl->it_level_size = 1ULL << (level_shift - 3);
315 	tbl->it_indirect_levels = levels - 1;
316 	tbl->it_allocated_size = total_allocated;
317 	tbl->it_userspace = uas;
318 	tbl->it_nid = nid;
319 
320 	pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d/%d\n",
321 			window_size, tce_table_size, bus_offset, tbl->it_base,
322 			tbl->it_userspace, tmplevels, levels);
323 
324 	return 0;
325 
326 free_uas_exit:
327 	pnv_pci_ioda2_table_do_free_pages(uas,
328 			1ULL << (level_shift - 3), levels - 1);
329 free_tces_exit:
330 	pnv_pci_ioda2_table_do_free_pages(addr,
331 			1ULL << (level_shift - 3), levels - 1);
332 
333 	return -ENOMEM;
334 }
335 
336 static void pnv_iommu_table_group_link_free(struct rcu_head *head)
337 {
338 	struct iommu_table_group_link *tgl = container_of(head,
339 			struct iommu_table_group_link, rcu);
340 
341 	kfree(tgl);
342 }
343 
344 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
345 		struct iommu_table_group *table_group)
346 {
347 	long i;
348 	bool found;
349 	struct iommu_table_group_link *tgl;
350 
351 	if (!tbl || !table_group)
352 		return;
353 
354 	/* Remove link to a group from table's list of attached groups */
355 	found = false;
356 	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
357 		if (tgl->table_group == table_group) {
358 			list_del_rcu(&tgl->next);
359 			call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free);
360 			found = true;
361 			break;
362 		}
363 	}
364 	if (WARN_ON(!found))
365 		return;
366 
367 	/* Clean a pointer to iommu_table in iommu_table_group::tables[] */
368 	found = false;
369 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
370 		if (table_group->tables[i] == tbl) {
371 			iommu_tce_table_put(tbl);
372 			table_group->tables[i] = NULL;
373 			found = true;
374 			break;
375 		}
376 	}
377 	WARN_ON(!found);
378 }
379 
380 long pnv_pci_link_table_and_group(int node, int num,
381 		struct iommu_table *tbl,
382 		struct iommu_table_group *table_group)
383 {
384 	struct iommu_table_group_link *tgl = NULL;
385 
386 	if (WARN_ON(!tbl || !table_group))
387 		return -EINVAL;
388 
389 	tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
390 			node);
391 	if (!tgl)
392 		return -ENOMEM;
393 
394 	tgl->table_group = table_group;
395 	list_add_rcu(&tgl->next, &tbl->it_group_list);
396 
397 	table_group->tables[num] = iommu_tce_table_get(tbl);
398 
399 	return 0;
400 }
401