1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VFIO: IOMMU DMA mapping support for TCE on POWER
4  *
5  * Copyright (C) 2013 IBM Corp.  All rights reserved.
6  *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
7  *
8  * Derived from original vfio_iommu_type1.c:
9  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
10  *     Author: Alex Williamson <alex.williamson@redhat.com>
11  */
12 
13 #include <linux/module.h>
14 #include <linux/pci.h>
15 #include <linux/slab.h>
16 #include <linux/uaccess.h>
17 #include <linux/err.h>
18 #include <linux/vfio.h>
19 #include <linux/vmalloc.h>
20 #include <linux/sched/mm.h>
21 #include <linux/sched/signal.h>
22 #include <linux/mm.h>
23 
24 #include <asm/iommu.h>
25 #include <asm/tce.h>
26 #include <asm/mmu_context.h>
27 
28 #define DRIVER_VERSION  "0.1"
29 #define DRIVER_AUTHOR   "aik@ozlabs.ru"
30 #define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"
31 
32 static void tce_iommu_detach_group(void *iommu_data,
33 		struct iommu_group *iommu_group);
34 
35 /*
36  * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
37  *
38  * This code handles mapping and unmapping of user data buffers
39  * into DMA'ble space using the IOMMU
40  */
41 
42 struct tce_iommu_group {
43 	struct list_head next;
44 	struct iommu_group *grp;
45 };
46 
47 /*
48  * A container needs to remember which preregistered region  it has
49  * referenced to do proper cleanup at the userspace process exit.
50  */
51 struct tce_iommu_prereg {
52 	struct list_head next;
53 	struct mm_iommu_table_group_mem_t *mem;
54 };
55 
56 /*
57  * The container descriptor supports only a single group per container.
58  * Required by the API as the container is not supplied with the IOMMU group
59  * at the moment of initialization.
60  */
61 struct tce_container {
62 	struct mutex lock;
63 	bool enabled;
64 	bool v2;
65 	bool def_window_pending;
66 	unsigned long locked_pages;
67 	struct mm_struct *mm;
68 	struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
69 	struct list_head group_list;
70 	struct list_head prereg_list;
71 };
72 
73 static long tce_iommu_mm_set(struct tce_container *container)
74 {
75 	if (container->mm) {
76 		if (container->mm == current->mm)
77 			return 0;
78 		return -EPERM;
79 	}
80 	BUG_ON(!current->mm);
81 	container->mm = current->mm;
82 	mmgrab(container->mm);
83 
84 	return 0;
85 }
86 
87 static long tce_iommu_prereg_free(struct tce_container *container,
88 		struct tce_iommu_prereg *tcemem)
89 {
90 	long ret;
91 
92 	ret = mm_iommu_put(container->mm, tcemem->mem);
93 	if (ret)
94 		return ret;
95 
96 	list_del(&tcemem->next);
97 	kfree(tcemem);
98 
99 	return 0;
100 }
101 
102 static long tce_iommu_unregister_pages(struct tce_container *container,
103 		__u64 vaddr, __u64 size)
104 {
105 	struct mm_iommu_table_group_mem_t *mem;
106 	struct tce_iommu_prereg *tcemem;
107 	bool found = false;
108 	long ret;
109 
110 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
111 		return -EINVAL;
112 
113 	mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
114 	if (!mem)
115 		return -ENOENT;
116 
117 	list_for_each_entry(tcemem, &container->prereg_list, next) {
118 		if (tcemem->mem == mem) {
119 			found = true;
120 			break;
121 		}
122 	}
123 
124 	if (!found)
125 		ret = -ENOENT;
126 	else
127 		ret = tce_iommu_prereg_free(container, tcemem);
128 
129 	mm_iommu_put(container->mm, mem);
130 
131 	return ret;
132 }
133 
134 static long tce_iommu_register_pages(struct tce_container *container,
135 		__u64 vaddr, __u64 size)
136 {
137 	long ret = 0;
138 	struct mm_iommu_table_group_mem_t *mem = NULL;
139 	struct tce_iommu_prereg *tcemem;
140 	unsigned long entries = size >> PAGE_SHIFT;
141 
142 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
143 			((vaddr + size) < vaddr))
144 		return -EINVAL;
145 
146 	mem = mm_iommu_get(container->mm, vaddr, entries);
147 	if (mem) {
148 		list_for_each_entry(tcemem, &container->prereg_list, next) {
149 			if (tcemem->mem == mem) {
150 				ret = -EBUSY;
151 				goto put_exit;
152 			}
153 		}
154 	} else {
155 		ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
156 		if (ret)
157 			return ret;
158 	}
159 
160 	tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
161 	if (!tcemem) {
162 		ret = -ENOMEM;
163 		goto put_exit;
164 	}
165 
166 	tcemem->mem = mem;
167 	list_add(&tcemem->next, &container->prereg_list);
168 
169 	container->enabled = true;
170 
171 	return 0;
172 
173 put_exit:
174 	mm_iommu_put(container->mm, mem);
175 	return ret;
176 }
177 
178 static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
179 		unsigned int it_page_shift)
180 {
181 	struct page *page;
182 	unsigned long size = 0;
183 
184 	if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
185 		return size == (1UL << it_page_shift);
186 
187 	page = pfn_to_page(hpa >> PAGE_SHIFT);
188 	/*
189 	 * Check that the TCE table granularity is not bigger than the size of
190 	 * a page we just found. Otherwise the hardware can get access to
191 	 * a bigger memory chunk that it should.
192 	 */
193 	return page_shift(compound_head(page)) >= it_page_shift;
194 }
195 
196 static inline bool tce_groups_attached(struct tce_container *container)
197 {
198 	return !list_empty(&container->group_list);
199 }
200 
201 static long tce_iommu_find_table(struct tce_container *container,
202 		phys_addr_t ioba, struct iommu_table **ptbl)
203 {
204 	long i;
205 
206 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
207 		struct iommu_table *tbl = container->tables[i];
208 
209 		if (tbl) {
210 			unsigned long entry = ioba >> tbl->it_page_shift;
211 			unsigned long start = tbl->it_offset;
212 			unsigned long end = start + tbl->it_size;
213 
214 			if ((start <= entry) && (entry < end)) {
215 				*ptbl = tbl;
216 				return i;
217 			}
218 		}
219 	}
220 
221 	return -1;
222 }
223 
224 static int tce_iommu_find_free_table(struct tce_container *container)
225 {
226 	int i;
227 
228 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
229 		if (!container->tables[i])
230 			return i;
231 	}
232 
233 	return -ENOSPC;
234 }
235 
236 static int tce_iommu_enable(struct tce_container *container)
237 {
238 	int ret = 0;
239 	unsigned long locked;
240 	struct iommu_table_group *table_group;
241 	struct tce_iommu_group *tcegrp;
242 
243 	if (container->enabled)
244 		return -EBUSY;
245 
246 	/*
247 	 * When userspace pages are mapped into the IOMMU, they are effectively
248 	 * locked memory, so, theoretically, we need to update the accounting
249 	 * of locked pages on each map and unmap.  For powerpc, the map unmap
250 	 * paths can be very hot, though, and the accounting would kill
251 	 * performance, especially since it would be difficult to impossible
252 	 * to handle the accounting in real mode only.
253 	 *
254 	 * To address that, rather than precisely accounting every page, we
255 	 * instead account for a worst case on locked memory when the iommu is
256 	 * enabled and disabled.  The worst case upper bound on locked memory
257 	 * is the size of the whole iommu window, which is usually relatively
258 	 * small (compared to total memory sizes) on POWER hardware.
259 	 *
260 	 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
261 	 * that would effectively kill the guest at random points, much better
262 	 * enforcing the limit based on the max that the guest can map.
263 	 *
264 	 * Unfortunately at the moment it counts whole tables, no matter how
265 	 * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
266 	 * each with 2GB DMA window, 8GB will be counted here. The reason for
267 	 * this is that we cannot tell here the amount of RAM used by the guest
268 	 * as this information is only available from KVM and VFIO is
269 	 * KVM agnostic.
270 	 *
271 	 * So we do not allow enabling a container without a group attached
272 	 * as there is no way to know how much we should increment
273 	 * the locked_vm counter.
274 	 */
275 	if (!tce_groups_attached(container))
276 		return -ENODEV;
277 
278 	tcegrp = list_first_entry(&container->group_list,
279 			struct tce_iommu_group, next);
280 	table_group = iommu_group_get_iommudata(tcegrp->grp);
281 	if (!table_group)
282 		return -ENODEV;
283 
284 	if (!table_group->tce32_size)
285 		return -EPERM;
286 
287 	ret = tce_iommu_mm_set(container);
288 	if (ret)
289 		return ret;
290 
291 	locked = table_group->tce32_size >> PAGE_SHIFT;
292 	ret = account_locked_vm(container->mm, locked, true);
293 	if (ret)
294 		return ret;
295 
296 	container->locked_pages = locked;
297 
298 	container->enabled = true;
299 
300 	return ret;
301 }
302 
303 static void tce_iommu_disable(struct tce_container *container)
304 {
305 	if (!container->enabled)
306 		return;
307 
308 	container->enabled = false;
309 
310 	BUG_ON(!container->mm);
311 	account_locked_vm(container->mm, container->locked_pages, false);
312 }
313 
314 static void *tce_iommu_open(unsigned long arg)
315 {
316 	struct tce_container *container;
317 
318 	if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
319 		pr_err("tce_vfio: Wrong IOMMU type\n");
320 		return ERR_PTR(-EINVAL);
321 	}
322 
323 	container = kzalloc(sizeof(*container), GFP_KERNEL);
324 	if (!container)
325 		return ERR_PTR(-ENOMEM);
326 
327 	mutex_init(&container->lock);
328 	INIT_LIST_HEAD_RCU(&container->group_list);
329 	INIT_LIST_HEAD_RCU(&container->prereg_list);
330 
331 	container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
332 
333 	return container;
334 }
335 
336 static int tce_iommu_clear(struct tce_container *container,
337 		struct iommu_table *tbl,
338 		unsigned long entry, unsigned long pages);
339 static void tce_iommu_free_table(struct tce_container *container,
340 		struct iommu_table *tbl);
341 
342 static void tce_iommu_release(void *iommu_data)
343 {
344 	struct tce_container *container = iommu_data;
345 	struct tce_iommu_group *tcegrp;
346 	struct tce_iommu_prereg *tcemem, *tmtmp;
347 	long i;
348 
349 	while (tce_groups_attached(container)) {
350 		tcegrp = list_first_entry(&container->group_list,
351 				struct tce_iommu_group, next);
352 		tce_iommu_detach_group(iommu_data, tcegrp->grp);
353 	}
354 
355 	/*
356 	 * If VFIO created a table, it was not disposed
357 	 * by tce_iommu_detach_group() so do it now.
358 	 */
359 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
360 		struct iommu_table *tbl = container->tables[i];
361 
362 		if (!tbl)
363 			continue;
364 
365 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
366 		tce_iommu_free_table(container, tbl);
367 	}
368 
369 	list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
370 		WARN_ON(tce_iommu_prereg_free(container, tcemem));
371 
372 	tce_iommu_disable(container);
373 	if (container->mm)
374 		mmdrop(container->mm);
375 	mutex_destroy(&container->lock);
376 
377 	kfree(container);
378 }
379 
380 static void tce_iommu_unuse_page(struct tce_container *container,
381 		unsigned long hpa)
382 {
383 	struct page *page;
384 
385 	page = pfn_to_page(hpa >> PAGE_SHIFT);
386 	put_page(page);
387 }
388 
389 static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
390 		unsigned long tce, unsigned long shift,
391 		unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
392 {
393 	long ret = 0;
394 	struct mm_iommu_table_group_mem_t *mem;
395 
396 	mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
397 	if (!mem)
398 		return -EINVAL;
399 
400 	ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
401 	if (ret)
402 		return -EINVAL;
403 
404 	*pmem = mem;
405 
406 	return 0;
407 }
408 
409 static void tce_iommu_unuse_page_v2(struct tce_container *container,
410 		struct iommu_table *tbl, unsigned long entry)
411 {
412 	struct mm_iommu_table_group_mem_t *mem = NULL;
413 	int ret;
414 	unsigned long hpa = 0;
415 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
416 
417 	if (!pua)
418 		return;
419 
420 	ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
421 			tbl->it_page_shift, &hpa, &mem);
422 	if (ret)
423 		pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
424 				__func__, be64_to_cpu(*pua), entry, ret);
425 	if (mem)
426 		mm_iommu_mapped_dec(mem);
427 
428 	*pua = cpu_to_be64(0);
429 }
430 
431 static int tce_iommu_clear(struct tce_container *container,
432 		struct iommu_table *tbl,
433 		unsigned long entry, unsigned long pages)
434 {
435 	unsigned long oldhpa;
436 	long ret;
437 	enum dma_data_direction direction;
438 	unsigned long lastentry = entry + pages, firstentry = entry;
439 
440 	for ( ; entry < lastentry; ++entry) {
441 		if (tbl->it_indirect_levels && tbl->it_userspace) {
442 			/*
443 			 * For multilevel tables, we can take a shortcut here
444 			 * and skip some TCEs as we know that the userspace
445 			 * addresses cache is a mirror of the real TCE table
446 			 * and if it is missing some indirect levels, then
447 			 * the hardware table does not have them allocated
448 			 * either and therefore does not require updating.
449 			 */
450 			__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
451 					entry);
452 			if (!pua) {
453 				/* align to level_size which is power of two */
454 				entry |= tbl->it_level_size - 1;
455 				continue;
456 			}
457 		}
458 
459 		cond_resched();
460 
461 		direction = DMA_NONE;
462 		oldhpa = 0;
463 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
464 				&direction);
465 		if (ret)
466 			continue;
467 
468 		if (direction == DMA_NONE)
469 			continue;
470 
471 		if (container->v2) {
472 			tce_iommu_unuse_page_v2(container, tbl, entry);
473 			continue;
474 		}
475 
476 		tce_iommu_unuse_page(container, oldhpa);
477 	}
478 
479 	iommu_tce_kill(tbl, firstentry, pages);
480 
481 	return 0;
482 }
483 
484 static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
485 {
486 	struct page *page = NULL;
487 	enum dma_data_direction direction = iommu_tce_direction(tce);
488 
489 	if (get_user_pages_fast(tce & PAGE_MASK, 1,
490 			direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
491 			&page) != 1)
492 		return -EFAULT;
493 
494 	*hpa = __pa((unsigned long) page_address(page));
495 
496 	return 0;
497 }
498 
499 static long tce_iommu_build(struct tce_container *container,
500 		struct iommu_table *tbl,
501 		unsigned long entry, unsigned long tce, unsigned long pages,
502 		enum dma_data_direction direction)
503 {
504 	long i, ret = 0;
505 	unsigned long hpa;
506 	enum dma_data_direction dirtmp;
507 
508 	for (i = 0; i < pages; ++i) {
509 		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
510 
511 		ret = tce_iommu_use_page(tce, &hpa);
512 		if (ret)
513 			break;
514 
515 		if (!tce_page_is_contained(container->mm, hpa,
516 				tbl->it_page_shift)) {
517 			ret = -EPERM;
518 			break;
519 		}
520 
521 		hpa |= offset;
522 		dirtmp = direction;
523 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
524 				&hpa, &dirtmp);
525 		if (ret) {
526 			tce_iommu_unuse_page(container, hpa);
527 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
528 					__func__, entry << tbl->it_page_shift,
529 					tce, ret);
530 			break;
531 		}
532 
533 		if (dirtmp != DMA_NONE)
534 			tce_iommu_unuse_page(container, hpa);
535 
536 		tce += IOMMU_PAGE_SIZE(tbl);
537 	}
538 
539 	if (ret)
540 		tce_iommu_clear(container, tbl, entry, i);
541 	else
542 		iommu_tce_kill(tbl, entry, pages);
543 
544 	return ret;
545 }
546 
547 static long tce_iommu_build_v2(struct tce_container *container,
548 		struct iommu_table *tbl,
549 		unsigned long entry, unsigned long tce, unsigned long pages,
550 		enum dma_data_direction direction)
551 {
552 	long i, ret = 0;
553 	unsigned long hpa;
554 	enum dma_data_direction dirtmp;
555 
556 	for (i = 0; i < pages; ++i) {
557 		struct mm_iommu_table_group_mem_t *mem = NULL;
558 		__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
559 
560 		ret = tce_iommu_prereg_ua_to_hpa(container,
561 				tce, tbl->it_page_shift, &hpa, &mem);
562 		if (ret)
563 			break;
564 
565 		if (!tce_page_is_contained(container->mm, hpa,
566 				tbl->it_page_shift)) {
567 			ret = -EPERM;
568 			break;
569 		}
570 
571 		/* Preserve offset within IOMMU page */
572 		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
573 		dirtmp = direction;
574 
575 		/* The registered region is being unregistered */
576 		if (mm_iommu_mapped_inc(mem))
577 			break;
578 
579 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
580 				&hpa, &dirtmp);
581 		if (ret) {
582 			/* dirtmp cannot be DMA_NONE here */
583 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
584 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
585 					__func__, entry << tbl->it_page_shift,
586 					tce, ret);
587 			break;
588 		}
589 
590 		if (dirtmp != DMA_NONE)
591 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
592 
593 		*pua = cpu_to_be64(tce);
594 
595 		tce += IOMMU_PAGE_SIZE(tbl);
596 	}
597 
598 	if (ret)
599 		tce_iommu_clear(container, tbl, entry, i);
600 	else
601 		iommu_tce_kill(tbl, entry, pages);
602 
603 	return ret;
604 }
605 
606 static long tce_iommu_create_table(struct tce_container *container,
607 			struct iommu_table_group *table_group,
608 			int num,
609 			__u32 page_shift,
610 			__u64 window_size,
611 			__u32 levels,
612 			struct iommu_table **ptbl)
613 {
614 	long ret, table_size;
615 
616 	table_size = table_group->ops->get_table_size(page_shift, window_size,
617 			levels);
618 	if (!table_size)
619 		return -EINVAL;
620 
621 	ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
622 	if (ret)
623 		return ret;
624 
625 	ret = table_group->ops->create_table(table_group, num,
626 			page_shift, window_size, levels, ptbl);
627 
628 	WARN_ON(!ret && !(*ptbl)->it_ops->free);
629 	WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
630 
631 	return ret;
632 }
633 
634 static void tce_iommu_free_table(struct tce_container *container,
635 		struct iommu_table *tbl)
636 {
637 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
638 
639 	iommu_tce_table_put(tbl);
640 	account_locked_vm(container->mm, pages, false);
641 }
642 
643 static long tce_iommu_create_window(struct tce_container *container,
644 		__u32 page_shift, __u64 window_size, __u32 levels,
645 		__u64 *start_addr)
646 {
647 	struct tce_iommu_group *tcegrp;
648 	struct iommu_table_group *table_group;
649 	struct iommu_table *tbl = NULL;
650 	long ret, num;
651 
652 	num = tce_iommu_find_free_table(container);
653 	if (num < 0)
654 		return num;
655 
656 	/* Get the first group for ops::create_table */
657 	tcegrp = list_first_entry(&container->group_list,
658 			struct tce_iommu_group, next);
659 	table_group = iommu_group_get_iommudata(tcegrp->grp);
660 	if (!table_group)
661 		return -EFAULT;
662 
663 	if (!(table_group->pgsizes & (1ULL << page_shift)))
664 		return -EINVAL;
665 
666 	if (!table_group->ops->set_window || !table_group->ops->unset_window ||
667 			!table_group->ops->get_table_size ||
668 			!table_group->ops->create_table)
669 		return -EPERM;
670 
671 	/* Create TCE table */
672 	ret = tce_iommu_create_table(container, table_group, num,
673 			page_shift, window_size, levels, &tbl);
674 	if (ret)
675 		return ret;
676 
677 	BUG_ON(!tbl->it_ops->free);
678 
679 	/*
680 	 * Program the table to every group.
681 	 * Groups have been tested for compatibility at the attach time.
682 	 */
683 	list_for_each_entry(tcegrp, &container->group_list, next) {
684 		table_group = iommu_group_get_iommudata(tcegrp->grp);
685 
686 		ret = table_group->ops->set_window(table_group, num, tbl);
687 		if (ret)
688 			goto unset_exit;
689 	}
690 
691 	container->tables[num] = tbl;
692 
693 	/* Return start address assigned by platform in create_table() */
694 	*start_addr = tbl->it_offset << tbl->it_page_shift;
695 
696 	return 0;
697 
698 unset_exit:
699 	list_for_each_entry(tcegrp, &container->group_list, next) {
700 		table_group = iommu_group_get_iommudata(tcegrp->grp);
701 		table_group->ops->unset_window(table_group, num);
702 	}
703 	tce_iommu_free_table(container, tbl);
704 
705 	return ret;
706 }
707 
708 static long tce_iommu_remove_window(struct tce_container *container,
709 		__u64 start_addr)
710 {
711 	struct iommu_table_group *table_group = NULL;
712 	struct iommu_table *tbl;
713 	struct tce_iommu_group *tcegrp;
714 	int num;
715 
716 	num = tce_iommu_find_table(container, start_addr, &tbl);
717 	if (num < 0)
718 		return -EINVAL;
719 
720 	BUG_ON(!tbl->it_size);
721 
722 	/* Detach groups from IOMMUs */
723 	list_for_each_entry(tcegrp, &container->group_list, next) {
724 		table_group = iommu_group_get_iommudata(tcegrp->grp);
725 
726 		/*
727 		 * SPAPR TCE IOMMU exposes the default DMA window to
728 		 * the guest via dma32_window_start/size of
729 		 * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
730 		 * the userspace to remove this window, some do not so
731 		 * here we check for the platform capability.
732 		 */
733 		if (!table_group->ops || !table_group->ops->unset_window)
734 			return -EPERM;
735 
736 		table_group->ops->unset_window(table_group, num);
737 	}
738 
739 	/* Free table */
740 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
741 	tce_iommu_free_table(container, tbl);
742 	container->tables[num] = NULL;
743 
744 	return 0;
745 }
746 
747 static long tce_iommu_create_default_window(struct tce_container *container)
748 {
749 	long ret;
750 	__u64 start_addr = 0;
751 	struct tce_iommu_group *tcegrp;
752 	struct iommu_table_group *table_group;
753 
754 	if (!container->def_window_pending)
755 		return 0;
756 
757 	if (!tce_groups_attached(container))
758 		return -ENODEV;
759 
760 	tcegrp = list_first_entry(&container->group_list,
761 			struct tce_iommu_group, next);
762 	table_group = iommu_group_get_iommudata(tcegrp->grp);
763 	if (!table_group)
764 		return -ENODEV;
765 
766 	ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
767 			table_group->tce32_size, 1, &start_addr);
768 	WARN_ON_ONCE(!ret && start_addr);
769 
770 	if (!ret)
771 		container->def_window_pending = false;
772 
773 	return ret;
774 }
775 
776 static long tce_iommu_ioctl(void *iommu_data,
777 				 unsigned int cmd, unsigned long arg)
778 {
779 	struct tce_container *container = iommu_data;
780 	unsigned long minsz, ddwsz;
781 	long ret;
782 
783 	switch (cmd) {
784 	case VFIO_CHECK_EXTENSION:
785 		switch (arg) {
786 		case VFIO_SPAPR_TCE_IOMMU:
787 		case VFIO_SPAPR_TCE_v2_IOMMU:
788 			ret = 1;
789 			break;
790 		default:
791 			ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg);
792 			break;
793 		}
794 
795 		return (ret < 0) ? 0 : ret;
796 	}
797 
798 	/*
799 	 * Sanity check to prevent one userspace from manipulating
800 	 * another userspace mm.
801 	 */
802 	BUG_ON(!container);
803 	if (container->mm && container->mm != current->mm)
804 		return -EPERM;
805 
806 	switch (cmd) {
807 	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
808 		struct vfio_iommu_spapr_tce_info info;
809 		struct tce_iommu_group *tcegrp;
810 		struct iommu_table_group *table_group;
811 
812 		if (!tce_groups_attached(container))
813 			return -ENXIO;
814 
815 		tcegrp = list_first_entry(&container->group_list,
816 				struct tce_iommu_group, next);
817 		table_group = iommu_group_get_iommudata(tcegrp->grp);
818 
819 		if (!table_group)
820 			return -ENXIO;
821 
822 		minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
823 				dma32_window_size);
824 
825 		if (copy_from_user(&info, (void __user *)arg, minsz))
826 			return -EFAULT;
827 
828 		if (info.argsz < minsz)
829 			return -EINVAL;
830 
831 		info.dma32_window_start = table_group->tce32_start;
832 		info.dma32_window_size = table_group->tce32_size;
833 		info.flags = 0;
834 		memset(&info.ddw, 0, sizeof(info.ddw));
835 
836 		if (table_group->max_dynamic_windows_supported &&
837 				container->v2) {
838 			info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
839 			info.ddw.pgsizes = table_group->pgsizes;
840 			info.ddw.max_dynamic_windows_supported =
841 				table_group->max_dynamic_windows_supported;
842 			info.ddw.levels = table_group->max_levels;
843 		}
844 
845 		ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
846 
847 		if (info.argsz >= ddwsz)
848 			minsz = ddwsz;
849 
850 		if (copy_to_user((void __user *)arg, &info, minsz))
851 			return -EFAULT;
852 
853 		return 0;
854 	}
855 	case VFIO_IOMMU_MAP_DMA: {
856 		struct vfio_iommu_type1_dma_map param;
857 		struct iommu_table *tbl = NULL;
858 		long num;
859 		enum dma_data_direction direction;
860 
861 		if (!container->enabled)
862 			return -EPERM;
863 
864 		minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
865 
866 		if (copy_from_user(&param, (void __user *)arg, minsz))
867 			return -EFAULT;
868 
869 		if (param.argsz < minsz)
870 			return -EINVAL;
871 
872 		if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
873 				VFIO_DMA_MAP_FLAG_WRITE))
874 			return -EINVAL;
875 
876 		ret = tce_iommu_create_default_window(container);
877 		if (ret)
878 			return ret;
879 
880 		num = tce_iommu_find_table(container, param.iova, &tbl);
881 		if (num < 0)
882 			return -ENXIO;
883 
884 		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
885 				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
886 			return -EINVAL;
887 
888 		/* iova is checked by the IOMMU API */
889 		if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
890 			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
891 				direction = DMA_BIDIRECTIONAL;
892 			else
893 				direction = DMA_TO_DEVICE;
894 		} else {
895 			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
896 				direction = DMA_FROM_DEVICE;
897 			else
898 				return -EINVAL;
899 		}
900 
901 		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
902 		if (ret)
903 			return ret;
904 
905 		if (container->v2)
906 			ret = tce_iommu_build_v2(container, tbl,
907 					param.iova >> tbl->it_page_shift,
908 					param.vaddr,
909 					param.size >> tbl->it_page_shift,
910 					direction);
911 		else
912 			ret = tce_iommu_build(container, tbl,
913 					param.iova >> tbl->it_page_shift,
914 					param.vaddr,
915 					param.size >> tbl->it_page_shift,
916 					direction);
917 
918 		iommu_flush_tce(tbl);
919 
920 		return ret;
921 	}
922 	case VFIO_IOMMU_UNMAP_DMA: {
923 		struct vfio_iommu_type1_dma_unmap param;
924 		struct iommu_table *tbl = NULL;
925 		long num;
926 
927 		if (!container->enabled)
928 			return -EPERM;
929 
930 		minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
931 				size);
932 
933 		if (copy_from_user(&param, (void __user *)arg, minsz))
934 			return -EFAULT;
935 
936 		if (param.argsz < minsz)
937 			return -EINVAL;
938 
939 		/* No flag is supported now */
940 		if (param.flags)
941 			return -EINVAL;
942 
943 		ret = tce_iommu_create_default_window(container);
944 		if (ret)
945 			return ret;
946 
947 		num = tce_iommu_find_table(container, param.iova, &tbl);
948 		if (num < 0)
949 			return -ENXIO;
950 
951 		if (param.size & ~IOMMU_PAGE_MASK(tbl))
952 			return -EINVAL;
953 
954 		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
955 				param.size >> tbl->it_page_shift);
956 		if (ret)
957 			return ret;
958 
959 		ret = tce_iommu_clear(container, tbl,
960 				param.iova >> tbl->it_page_shift,
961 				param.size >> tbl->it_page_shift);
962 		iommu_flush_tce(tbl);
963 
964 		return ret;
965 	}
966 	case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
967 		struct vfio_iommu_spapr_register_memory param;
968 
969 		if (!container->v2)
970 			break;
971 
972 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
973 				size);
974 
975 		ret = tce_iommu_mm_set(container);
976 		if (ret)
977 			return ret;
978 
979 		if (copy_from_user(&param, (void __user *)arg, minsz))
980 			return -EFAULT;
981 
982 		if (param.argsz < minsz)
983 			return -EINVAL;
984 
985 		/* No flag is supported now */
986 		if (param.flags)
987 			return -EINVAL;
988 
989 		mutex_lock(&container->lock);
990 		ret = tce_iommu_register_pages(container, param.vaddr,
991 				param.size);
992 		mutex_unlock(&container->lock);
993 
994 		return ret;
995 	}
996 	case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
997 		struct vfio_iommu_spapr_register_memory param;
998 
999 		if (!container->v2)
1000 			break;
1001 
1002 		if (!container->mm)
1003 			return -EPERM;
1004 
1005 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1006 				size);
1007 
1008 		if (copy_from_user(&param, (void __user *)arg, minsz))
1009 			return -EFAULT;
1010 
1011 		if (param.argsz < minsz)
1012 			return -EINVAL;
1013 
1014 		/* No flag is supported now */
1015 		if (param.flags)
1016 			return -EINVAL;
1017 
1018 		mutex_lock(&container->lock);
1019 		ret = tce_iommu_unregister_pages(container, param.vaddr,
1020 				param.size);
1021 		mutex_unlock(&container->lock);
1022 
1023 		return ret;
1024 	}
1025 	case VFIO_IOMMU_ENABLE:
1026 		if (container->v2)
1027 			break;
1028 
1029 		mutex_lock(&container->lock);
1030 		ret = tce_iommu_enable(container);
1031 		mutex_unlock(&container->lock);
1032 		return ret;
1033 
1034 
1035 	case VFIO_IOMMU_DISABLE:
1036 		if (container->v2)
1037 			break;
1038 
1039 		mutex_lock(&container->lock);
1040 		tce_iommu_disable(container);
1041 		mutex_unlock(&container->lock);
1042 		return 0;
1043 
1044 	case VFIO_EEH_PE_OP: {
1045 		struct tce_iommu_group *tcegrp;
1046 
1047 		ret = 0;
1048 		list_for_each_entry(tcegrp, &container->group_list, next) {
1049 			ret = vfio_spapr_iommu_eeh_ioctl(tcegrp->grp,
1050 					cmd, arg);
1051 			if (ret)
1052 				return ret;
1053 		}
1054 		return ret;
1055 	}
1056 
1057 	case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1058 		struct vfio_iommu_spapr_tce_create create;
1059 
1060 		if (!container->v2)
1061 			break;
1062 
1063 		ret = tce_iommu_mm_set(container);
1064 		if (ret)
1065 			return ret;
1066 
1067 		if (!tce_groups_attached(container))
1068 			return -ENXIO;
1069 
1070 		minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1071 				start_addr);
1072 
1073 		if (copy_from_user(&create, (void __user *)arg, minsz))
1074 			return -EFAULT;
1075 
1076 		if (create.argsz < minsz)
1077 			return -EINVAL;
1078 
1079 		if (create.flags)
1080 			return -EINVAL;
1081 
1082 		mutex_lock(&container->lock);
1083 
1084 		ret = tce_iommu_create_default_window(container);
1085 		if (!ret)
1086 			ret = tce_iommu_create_window(container,
1087 					create.page_shift,
1088 					create.window_size, create.levels,
1089 					&create.start_addr);
1090 
1091 		mutex_unlock(&container->lock);
1092 
1093 		if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1094 			ret = -EFAULT;
1095 
1096 		return ret;
1097 	}
1098 	case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1099 		struct vfio_iommu_spapr_tce_remove remove;
1100 
1101 		if (!container->v2)
1102 			break;
1103 
1104 		ret = tce_iommu_mm_set(container);
1105 		if (ret)
1106 			return ret;
1107 
1108 		if (!tce_groups_attached(container))
1109 			return -ENXIO;
1110 
1111 		minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1112 				start_addr);
1113 
1114 		if (copy_from_user(&remove, (void __user *)arg, minsz))
1115 			return -EFAULT;
1116 
1117 		if (remove.argsz < minsz)
1118 			return -EINVAL;
1119 
1120 		if (remove.flags)
1121 			return -EINVAL;
1122 
1123 		if (container->def_window_pending && !remove.start_addr) {
1124 			container->def_window_pending = false;
1125 			return 0;
1126 		}
1127 
1128 		mutex_lock(&container->lock);
1129 
1130 		ret = tce_iommu_remove_window(container, remove.start_addr);
1131 
1132 		mutex_unlock(&container->lock);
1133 
1134 		return ret;
1135 	}
1136 	}
1137 
1138 	return -ENOTTY;
1139 }
1140 
1141 static void tce_iommu_release_ownership(struct tce_container *container,
1142 		struct iommu_table_group *table_group)
1143 {
1144 	int i;
1145 
1146 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1147 		struct iommu_table *tbl = container->tables[i];
1148 
1149 		if (!tbl)
1150 			continue;
1151 
1152 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
1153 		if (tbl->it_map)
1154 			iommu_release_ownership(tbl);
1155 
1156 		container->tables[i] = NULL;
1157 	}
1158 }
1159 
1160 static int tce_iommu_take_ownership(struct tce_container *container,
1161 		struct iommu_table_group *table_group)
1162 {
1163 	int i, j, rc = 0;
1164 
1165 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1166 		struct iommu_table *tbl = table_group->tables[i];
1167 
1168 		if (!tbl || !tbl->it_map)
1169 			continue;
1170 
1171 		rc = iommu_take_ownership(tbl);
1172 		if (rc) {
1173 			for (j = 0; j < i; ++j)
1174 				iommu_release_ownership(
1175 						table_group->tables[j]);
1176 
1177 			return rc;
1178 		}
1179 	}
1180 
1181 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1182 		container->tables[i] = table_group->tables[i];
1183 
1184 	return 0;
1185 }
1186 
1187 static void tce_iommu_release_ownership_ddw(struct tce_container *container,
1188 		struct iommu_table_group *table_group)
1189 {
1190 	long i;
1191 
1192 	if (!table_group->ops->unset_window) {
1193 		WARN_ON_ONCE(1);
1194 		return;
1195 	}
1196 
1197 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1198 		if (container->tables[i])
1199 			table_group->ops->unset_window(table_group, i);
1200 
1201 	table_group->ops->release_ownership(table_group);
1202 }
1203 
1204 static long tce_iommu_take_ownership_ddw(struct tce_container *container,
1205 		struct iommu_table_group *table_group)
1206 {
1207 	long i, ret = 0;
1208 
1209 	if (!table_group->ops->create_table || !table_group->ops->set_window ||
1210 			!table_group->ops->release_ownership) {
1211 		WARN_ON_ONCE(1);
1212 		return -EFAULT;
1213 	}
1214 
1215 	table_group->ops->take_ownership(table_group);
1216 
1217 	/* Set all windows to the new group */
1218 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1219 		struct iommu_table *tbl = container->tables[i];
1220 
1221 		if (!tbl)
1222 			continue;
1223 
1224 		ret = table_group->ops->set_window(table_group, i, tbl);
1225 		if (ret)
1226 			goto release_exit;
1227 	}
1228 
1229 	return 0;
1230 
1231 release_exit:
1232 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1233 		table_group->ops->unset_window(table_group, i);
1234 
1235 	table_group->ops->release_ownership(table_group);
1236 
1237 	return ret;
1238 }
1239 
1240 static int tce_iommu_attach_group(void *iommu_data,
1241 		struct iommu_group *iommu_group)
1242 {
1243 	int ret = 0;
1244 	struct tce_container *container = iommu_data;
1245 	struct iommu_table_group *table_group;
1246 	struct tce_iommu_group *tcegrp = NULL;
1247 
1248 	mutex_lock(&container->lock);
1249 
1250 	/* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
1251 			iommu_group_id(iommu_group), iommu_group); */
1252 	table_group = iommu_group_get_iommudata(iommu_group);
1253 	if (!table_group) {
1254 		ret = -ENODEV;
1255 		goto unlock_exit;
1256 	}
1257 
1258 	if (tce_groups_attached(container) && (!table_group->ops ||
1259 			!table_group->ops->take_ownership ||
1260 			!table_group->ops->release_ownership)) {
1261 		ret = -EBUSY;
1262 		goto unlock_exit;
1263 	}
1264 
1265 	/* Check if new group has the same iommu_ops (i.e. compatible) */
1266 	list_for_each_entry(tcegrp, &container->group_list, next) {
1267 		struct iommu_table_group *table_group_tmp;
1268 
1269 		if (tcegrp->grp == iommu_group) {
1270 			pr_warn("tce_vfio: Group %d is already attached\n",
1271 					iommu_group_id(iommu_group));
1272 			ret = -EBUSY;
1273 			goto unlock_exit;
1274 		}
1275 		table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
1276 		if (table_group_tmp->ops->create_table !=
1277 				table_group->ops->create_table) {
1278 			pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
1279 					iommu_group_id(iommu_group),
1280 					iommu_group_id(tcegrp->grp));
1281 			ret = -EPERM;
1282 			goto unlock_exit;
1283 		}
1284 	}
1285 
1286 	tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
1287 	if (!tcegrp) {
1288 		ret = -ENOMEM;
1289 		goto unlock_exit;
1290 	}
1291 
1292 	if (!table_group->ops || !table_group->ops->take_ownership ||
1293 			!table_group->ops->release_ownership) {
1294 		if (container->v2) {
1295 			ret = -EPERM;
1296 			goto free_exit;
1297 		}
1298 		ret = tce_iommu_take_ownership(container, table_group);
1299 	} else {
1300 		if (!container->v2) {
1301 			ret = -EPERM;
1302 			goto free_exit;
1303 		}
1304 		ret = tce_iommu_take_ownership_ddw(container, table_group);
1305 		if (!tce_groups_attached(container) && !container->tables[0])
1306 			container->def_window_pending = true;
1307 	}
1308 
1309 	if (!ret) {
1310 		tcegrp->grp = iommu_group;
1311 		list_add(&tcegrp->next, &container->group_list);
1312 	}
1313 
1314 free_exit:
1315 	if (ret && tcegrp)
1316 		kfree(tcegrp);
1317 
1318 unlock_exit:
1319 	mutex_unlock(&container->lock);
1320 
1321 	return ret;
1322 }
1323 
1324 static void tce_iommu_detach_group(void *iommu_data,
1325 		struct iommu_group *iommu_group)
1326 {
1327 	struct tce_container *container = iommu_data;
1328 	struct iommu_table_group *table_group;
1329 	bool found = false;
1330 	struct tce_iommu_group *tcegrp;
1331 
1332 	mutex_lock(&container->lock);
1333 
1334 	list_for_each_entry(tcegrp, &container->group_list, next) {
1335 		if (tcegrp->grp == iommu_group) {
1336 			found = true;
1337 			break;
1338 		}
1339 	}
1340 
1341 	if (!found) {
1342 		pr_warn("tce_vfio: detaching unattached group #%u\n",
1343 				iommu_group_id(iommu_group));
1344 		goto unlock_exit;
1345 	}
1346 
1347 	list_del(&tcegrp->next);
1348 	kfree(tcegrp);
1349 
1350 	table_group = iommu_group_get_iommudata(iommu_group);
1351 	BUG_ON(!table_group);
1352 
1353 	if (!table_group->ops || !table_group->ops->release_ownership)
1354 		tce_iommu_release_ownership(container, table_group);
1355 	else
1356 		tce_iommu_release_ownership_ddw(container, table_group);
1357 
1358 unlock_exit:
1359 	mutex_unlock(&container->lock);
1360 }
1361 
1362 static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
1363 	.name		= "iommu-vfio-powerpc",
1364 	.owner		= THIS_MODULE,
1365 	.open		= tce_iommu_open,
1366 	.release	= tce_iommu_release,
1367 	.ioctl		= tce_iommu_ioctl,
1368 	.attach_group	= tce_iommu_attach_group,
1369 	.detach_group	= tce_iommu_detach_group,
1370 };
1371 
1372 static int __init tce_iommu_init(void)
1373 {
1374 	return vfio_register_iommu_driver(&tce_iommu_driver_ops);
1375 }
1376 
1377 static void __exit tce_iommu_cleanup(void)
1378 {
1379 	vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
1380 }
1381 
1382 module_init(tce_iommu_init);
1383 module_exit(tce_iommu_cleanup);
1384 
1385 MODULE_VERSION(DRIVER_VERSION);
1386 MODULE_LICENSE("GPL v2");
1387 MODULE_AUTHOR(DRIVER_AUTHOR);
1388 MODULE_DESCRIPTION(DRIVER_DESC);
1389 
1390