1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VFIO: IOMMU DMA mapping support for TCE on POWER
4  *
5  * Copyright (C) 2013 IBM Corp.  All rights reserved.
6  *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
7  * Copyright Gavin Shan, IBM Corporation 2014.
8  *
9  * Derived from original vfio_iommu_type1.c:
10  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
11  *     Author: Alex Williamson <alex.williamson@redhat.com>
12  */
13 
14 #include <linux/module.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/uaccess.h>
18 #include <linux/err.h>
19 #include <linux/vfio.h>
20 #include <linux/vmalloc.h>
21 #include <linux/sched/mm.h>
22 #include <linux/sched/signal.h>
23 #include <linux/mm.h>
24 #include "vfio.h"
25 
26 #include <asm/iommu.h>
27 #include <asm/tce.h>
28 #include <asm/mmu_context.h>
29 
30 #define DRIVER_VERSION  "0.1"
31 #define DRIVER_AUTHOR   "aik@ozlabs.ru"
32 #define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"
33 
34 static void tce_iommu_detach_group(void *iommu_data,
35 		struct iommu_group *iommu_group);
36 
37 /*
38  * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
39  *
40  * This code handles mapping and unmapping of user data buffers
41  * into DMA'ble space using the IOMMU
42  */
43 
44 struct tce_iommu_group {
45 	struct list_head next;
46 	struct iommu_group *grp;
47 };
48 
49 /*
50  * A container needs to remember which preregistered region  it has
51  * referenced to do proper cleanup at the userspace process exit.
52  */
53 struct tce_iommu_prereg {
54 	struct list_head next;
55 	struct mm_iommu_table_group_mem_t *mem;
56 };
57 
58 /*
59  * The container descriptor supports only a single group per container.
60  * Required by the API as the container is not supplied with the IOMMU group
61  * at the moment of initialization.
62  */
63 struct tce_container {
64 	struct mutex lock;
65 	bool enabled;
66 	bool v2;
67 	bool def_window_pending;
68 	unsigned long locked_pages;
69 	struct mm_struct *mm;
70 	struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
71 	struct list_head group_list;
72 	struct list_head prereg_list;
73 };
74 
75 static long tce_iommu_mm_set(struct tce_container *container)
76 {
77 	if (container->mm) {
78 		if (container->mm == current->mm)
79 			return 0;
80 		return -EPERM;
81 	}
82 	BUG_ON(!current->mm);
83 	container->mm = current->mm;
84 	mmgrab(container->mm);
85 
86 	return 0;
87 }
88 
89 static long tce_iommu_prereg_free(struct tce_container *container,
90 		struct tce_iommu_prereg *tcemem)
91 {
92 	long ret;
93 
94 	ret = mm_iommu_put(container->mm, tcemem->mem);
95 	if (ret)
96 		return ret;
97 
98 	list_del(&tcemem->next);
99 	kfree(tcemem);
100 
101 	return 0;
102 }
103 
104 static long tce_iommu_unregister_pages(struct tce_container *container,
105 		__u64 vaddr, __u64 size)
106 {
107 	struct mm_iommu_table_group_mem_t *mem;
108 	struct tce_iommu_prereg *tcemem;
109 	bool found = false;
110 	long ret;
111 
112 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
113 		return -EINVAL;
114 
115 	mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
116 	if (!mem)
117 		return -ENOENT;
118 
119 	list_for_each_entry(tcemem, &container->prereg_list, next) {
120 		if (tcemem->mem == mem) {
121 			found = true;
122 			break;
123 		}
124 	}
125 
126 	if (!found)
127 		ret = -ENOENT;
128 	else
129 		ret = tce_iommu_prereg_free(container, tcemem);
130 
131 	mm_iommu_put(container->mm, mem);
132 
133 	return ret;
134 }
135 
136 static long tce_iommu_register_pages(struct tce_container *container,
137 		__u64 vaddr, __u64 size)
138 {
139 	long ret = 0;
140 	struct mm_iommu_table_group_mem_t *mem = NULL;
141 	struct tce_iommu_prereg *tcemem;
142 	unsigned long entries = size >> PAGE_SHIFT;
143 
144 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
145 			((vaddr + size) < vaddr))
146 		return -EINVAL;
147 
148 	mem = mm_iommu_get(container->mm, vaddr, entries);
149 	if (mem) {
150 		list_for_each_entry(tcemem, &container->prereg_list, next) {
151 			if (tcemem->mem == mem) {
152 				ret = -EBUSY;
153 				goto put_exit;
154 			}
155 		}
156 	} else {
157 		ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
158 		if (ret)
159 			return ret;
160 	}
161 
162 	tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
163 	if (!tcemem) {
164 		ret = -ENOMEM;
165 		goto put_exit;
166 	}
167 
168 	tcemem->mem = mem;
169 	list_add(&tcemem->next, &container->prereg_list);
170 
171 	container->enabled = true;
172 
173 	return 0;
174 
175 put_exit:
176 	mm_iommu_put(container->mm, mem);
177 	return ret;
178 }
179 
180 static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
181 		unsigned int it_page_shift)
182 {
183 	struct page *page;
184 	unsigned long size = 0;
185 
186 	if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
187 		return size == (1UL << it_page_shift);
188 
189 	page = pfn_to_page(hpa >> PAGE_SHIFT);
190 	/*
191 	 * Check that the TCE table granularity is not bigger than the size of
192 	 * a page we just found. Otherwise the hardware can get access to
193 	 * a bigger memory chunk that it should.
194 	 */
195 	return page_shift(compound_head(page)) >= it_page_shift;
196 }
197 
198 static inline bool tce_groups_attached(struct tce_container *container)
199 {
200 	return !list_empty(&container->group_list);
201 }
202 
203 static long tce_iommu_find_table(struct tce_container *container,
204 		phys_addr_t ioba, struct iommu_table **ptbl)
205 {
206 	long i;
207 
208 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
209 		struct iommu_table *tbl = container->tables[i];
210 
211 		if (tbl) {
212 			unsigned long entry = ioba >> tbl->it_page_shift;
213 			unsigned long start = tbl->it_offset;
214 			unsigned long end = start + tbl->it_size;
215 
216 			if ((start <= entry) && (entry < end)) {
217 				*ptbl = tbl;
218 				return i;
219 			}
220 		}
221 	}
222 
223 	return -1;
224 }
225 
226 static int tce_iommu_find_free_table(struct tce_container *container)
227 {
228 	int i;
229 
230 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
231 		if (!container->tables[i])
232 			return i;
233 	}
234 
235 	return -ENOSPC;
236 }
237 
238 static int tce_iommu_enable(struct tce_container *container)
239 {
240 	int ret = 0;
241 	unsigned long locked;
242 	struct iommu_table_group *table_group;
243 	struct tce_iommu_group *tcegrp;
244 
245 	if (container->enabled)
246 		return -EBUSY;
247 
248 	/*
249 	 * When userspace pages are mapped into the IOMMU, they are effectively
250 	 * locked memory, so, theoretically, we need to update the accounting
251 	 * of locked pages on each map and unmap.  For powerpc, the map unmap
252 	 * paths can be very hot, though, and the accounting would kill
253 	 * performance, especially since it would be difficult to impossible
254 	 * to handle the accounting in real mode only.
255 	 *
256 	 * To address that, rather than precisely accounting every page, we
257 	 * instead account for a worst case on locked memory when the iommu is
258 	 * enabled and disabled.  The worst case upper bound on locked memory
259 	 * is the size of the whole iommu window, which is usually relatively
260 	 * small (compared to total memory sizes) on POWER hardware.
261 	 *
262 	 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
263 	 * that would effectively kill the guest at random points, much better
264 	 * enforcing the limit based on the max that the guest can map.
265 	 *
266 	 * Unfortunately at the moment it counts whole tables, no matter how
267 	 * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
268 	 * each with 2GB DMA window, 8GB will be counted here. The reason for
269 	 * this is that we cannot tell here the amount of RAM used by the guest
270 	 * as this information is only available from KVM and VFIO is
271 	 * KVM agnostic.
272 	 *
273 	 * So we do not allow enabling a container without a group attached
274 	 * as there is no way to know how much we should increment
275 	 * the locked_vm counter.
276 	 */
277 	if (!tce_groups_attached(container))
278 		return -ENODEV;
279 
280 	tcegrp = list_first_entry(&container->group_list,
281 			struct tce_iommu_group, next);
282 	table_group = iommu_group_get_iommudata(tcegrp->grp);
283 	if (!table_group)
284 		return -ENODEV;
285 
286 	if (!table_group->tce32_size)
287 		return -EPERM;
288 
289 	ret = tce_iommu_mm_set(container);
290 	if (ret)
291 		return ret;
292 
293 	locked = table_group->tce32_size >> PAGE_SHIFT;
294 	ret = account_locked_vm(container->mm, locked, true);
295 	if (ret)
296 		return ret;
297 
298 	container->locked_pages = locked;
299 
300 	container->enabled = true;
301 
302 	return ret;
303 }
304 
305 static void tce_iommu_disable(struct tce_container *container)
306 {
307 	if (!container->enabled)
308 		return;
309 
310 	container->enabled = false;
311 
312 	BUG_ON(!container->mm);
313 	account_locked_vm(container->mm, container->locked_pages, false);
314 }
315 
316 static void *tce_iommu_open(unsigned long arg)
317 {
318 	struct tce_container *container;
319 
320 	if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
321 		pr_err("tce_vfio: Wrong IOMMU type\n");
322 		return ERR_PTR(-EINVAL);
323 	}
324 
325 	container = kzalloc(sizeof(*container), GFP_KERNEL);
326 	if (!container)
327 		return ERR_PTR(-ENOMEM);
328 
329 	mutex_init(&container->lock);
330 	INIT_LIST_HEAD_RCU(&container->group_list);
331 	INIT_LIST_HEAD_RCU(&container->prereg_list);
332 
333 	container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
334 
335 	return container;
336 }
337 
338 static int tce_iommu_clear(struct tce_container *container,
339 		struct iommu_table *tbl,
340 		unsigned long entry, unsigned long pages);
341 static void tce_iommu_free_table(struct tce_container *container,
342 		struct iommu_table *tbl);
343 
344 static void tce_iommu_release(void *iommu_data)
345 {
346 	struct tce_container *container = iommu_data;
347 	struct tce_iommu_group *tcegrp;
348 	struct tce_iommu_prereg *tcemem, *tmtmp;
349 	long i;
350 
351 	while (tce_groups_attached(container)) {
352 		tcegrp = list_first_entry(&container->group_list,
353 				struct tce_iommu_group, next);
354 		tce_iommu_detach_group(iommu_data, tcegrp->grp);
355 	}
356 
357 	/*
358 	 * If VFIO created a table, it was not disposed
359 	 * by tce_iommu_detach_group() so do it now.
360 	 */
361 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
362 		struct iommu_table *tbl = container->tables[i];
363 
364 		if (!tbl)
365 			continue;
366 
367 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
368 		tce_iommu_free_table(container, tbl);
369 	}
370 
371 	list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
372 		WARN_ON(tce_iommu_prereg_free(container, tcemem));
373 
374 	tce_iommu_disable(container);
375 	if (container->mm)
376 		mmdrop(container->mm);
377 	mutex_destroy(&container->lock);
378 
379 	kfree(container);
380 }
381 
382 static void tce_iommu_unuse_page(unsigned long hpa)
383 {
384 	struct page *page;
385 
386 	page = pfn_to_page(hpa >> PAGE_SHIFT);
387 	unpin_user_page(page);
388 }
389 
390 static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
391 		unsigned long tce, unsigned long shift,
392 		unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
393 {
394 	long ret = 0;
395 	struct mm_iommu_table_group_mem_t *mem;
396 
397 	mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
398 	if (!mem)
399 		return -EINVAL;
400 
401 	ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
402 	if (ret)
403 		return -EINVAL;
404 
405 	*pmem = mem;
406 
407 	return 0;
408 }
409 
410 static void tce_iommu_unuse_page_v2(struct tce_container *container,
411 		struct iommu_table *tbl, unsigned long entry)
412 {
413 	struct mm_iommu_table_group_mem_t *mem = NULL;
414 	int ret;
415 	unsigned long hpa = 0;
416 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
417 
418 	if (!pua)
419 		return;
420 
421 	ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
422 			tbl->it_page_shift, &hpa, &mem);
423 	if (ret)
424 		pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
425 				__func__, be64_to_cpu(*pua), entry, ret);
426 	if (mem)
427 		mm_iommu_mapped_dec(mem);
428 
429 	*pua = cpu_to_be64(0);
430 }
431 
432 static int tce_iommu_clear(struct tce_container *container,
433 		struct iommu_table *tbl,
434 		unsigned long entry, unsigned long pages)
435 {
436 	unsigned long oldhpa;
437 	long ret;
438 	enum dma_data_direction direction;
439 	unsigned long lastentry = entry + pages, firstentry = entry;
440 
441 	for ( ; entry < lastentry; ++entry) {
442 		if (tbl->it_indirect_levels && tbl->it_userspace) {
443 			/*
444 			 * For multilevel tables, we can take a shortcut here
445 			 * and skip some TCEs as we know that the userspace
446 			 * addresses cache is a mirror of the real TCE table
447 			 * and if it is missing some indirect levels, then
448 			 * the hardware table does not have them allocated
449 			 * either and therefore does not require updating.
450 			 */
451 			__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
452 					entry);
453 			if (!pua) {
454 				/* align to level_size which is power of two */
455 				entry |= tbl->it_level_size - 1;
456 				continue;
457 			}
458 		}
459 
460 		cond_resched();
461 
462 		direction = DMA_NONE;
463 		oldhpa = 0;
464 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
465 				&direction);
466 		if (ret)
467 			continue;
468 
469 		if (direction == DMA_NONE)
470 			continue;
471 
472 		if (container->v2) {
473 			tce_iommu_unuse_page_v2(container, tbl, entry);
474 			continue;
475 		}
476 
477 		tce_iommu_unuse_page(oldhpa);
478 	}
479 
480 	iommu_tce_kill(tbl, firstentry, pages);
481 
482 	return 0;
483 }
484 
485 static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
486 {
487 	struct page *page = NULL;
488 	enum dma_data_direction direction = iommu_tce_direction(tce);
489 
490 	if (pin_user_pages_fast(tce & PAGE_MASK, 1,
491 			direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
492 			&page) != 1)
493 		return -EFAULT;
494 
495 	*hpa = __pa((unsigned long) page_address(page));
496 
497 	return 0;
498 }
499 
500 static long tce_iommu_build(struct tce_container *container,
501 		struct iommu_table *tbl,
502 		unsigned long entry, unsigned long tce, unsigned long pages,
503 		enum dma_data_direction direction)
504 {
505 	long i, ret = 0;
506 	unsigned long hpa;
507 	enum dma_data_direction dirtmp;
508 
509 	for (i = 0; i < pages; ++i) {
510 		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
511 
512 		ret = tce_iommu_use_page(tce, &hpa);
513 		if (ret)
514 			break;
515 
516 		if (!tce_page_is_contained(container->mm, hpa,
517 				tbl->it_page_shift)) {
518 			ret = -EPERM;
519 			break;
520 		}
521 
522 		hpa |= offset;
523 		dirtmp = direction;
524 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
525 				&hpa, &dirtmp);
526 		if (ret) {
527 			tce_iommu_unuse_page(hpa);
528 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
529 					__func__, entry << tbl->it_page_shift,
530 					tce, ret);
531 			break;
532 		}
533 
534 		if (dirtmp != DMA_NONE)
535 			tce_iommu_unuse_page(hpa);
536 
537 		tce += IOMMU_PAGE_SIZE(tbl);
538 	}
539 
540 	if (ret)
541 		tce_iommu_clear(container, tbl, entry, i);
542 	else
543 		iommu_tce_kill(tbl, entry, pages);
544 
545 	return ret;
546 }
547 
548 static long tce_iommu_build_v2(struct tce_container *container,
549 		struct iommu_table *tbl,
550 		unsigned long entry, unsigned long tce, unsigned long pages,
551 		enum dma_data_direction direction)
552 {
553 	long i, ret = 0;
554 	unsigned long hpa;
555 	enum dma_data_direction dirtmp;
556 
557 	for (i = 0; i < pages; ++i) {
558 		struct mm_iommu_table_group_mem_t *mem = NULL;
559 		__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
560 
561 		ret = tce_iommu_prereg_ua_to_hpa(container,
562 				tce, tbl->it_page_shift, &hpa, &mem);
563 		if (ret)
564 			break;
565 
566 		if (!tce_page_is_contained(container->mm, hpa,
567 				tbl->it_page_shift)) {
568 			ret = -EPERM;
569 			break;
570 		}
571 
572 		/* Preserve offset within IOMMU page */
573 		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
574 		dirtmp = direction;
575 
576 		/* The registered region is being unregistered */
577 		if (mm_iommu_mapped_inc(mem))
578 			break;
579 
580 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
581 				&hpa, &dirtmp);
582 		if (ret) {
583 			/* dirtmp cannot be DMA_NONE here */
584 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
585 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
586 					__func__, entry << tbl->it_page_shift,
587 					tce, ret);
588 			break;
589 		}
590 
591 		if (dirtmp != DMA_NONE)
592 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
593 
594 		*pua = cpu_to_be64(tce);
595 
596 		tce += IOMMU_PAGE_SIZE(tbl);
597 	}
598 
599 	if (ret)
600 		tce_iommu_clear(container, tbl, entry, i);
601 	else
602 		iommu_tce_kill(tbl, entry, pages);
603 
604 	return ret;
605 }
606 
607 static long tce_iommu_create_table(struct tce_container *container,
608 			struct iommu_table_group *table_group,
609 			int num,
610 			__u32 page_shift,
611 			__u64 window_size,
612 			__u32 levels,
613 			struct iommu_table **ptbl)
614 {
615 	long ret, table_size;
616 
617 	table_size = table_group->ops->get_table_size(page_shift, window_size,
618 			levels);
619 	if (!table_size)
620 		return -EINVAL;
621 
622 	ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
623 	if (ret)
624 		return ret;
625 
626 	ret = table_group->ops->create_table(table_group, num,
627 			page_shift, window_size, levels, ptbl);
628 
629 	WARN_ON(!ret && !(*ptbl)->it_ops->free);
630 	WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
631 
632 	return ret;
633 }
634 
635 static void tce_iommu_free_table(struct tce_container *container,
636 		struct iommu_table *tbl)
637 {
638 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
639 
640 	iommu_tce_table_put(tbl);
641 	account_locked_vm(container->mm, pages, false);
642 }
643 
644 static long tce_iommu_create_window(struct tce_container *container,
645 		__u32 page_shift, __u64 window_size, __u32 levels,
646 		__u64 *start_addr)
647 {
648 	struct tce_iommu_group *tcegrp;
649 	struct iommu_table_group *table_group;
650 	struct iommu_table *tbl = NULL;
651 	long ret, num;
652 
653 	num = tce_iommu_find_free_table(container);
654 	if (num < 0)
655 		return num;
656 
657 	/* Get the first group for ops::create_table */
658 	tcegrp = list_first_entry(&container->group_list,
659 			struct tce_iommu_group, next);
660 	table_group = iommu_group_get_iommudata(tcegrp->grp);
661 	if (!table_group)
662 		return -EFAULT;
663 
664 	if (!(table_group->pgsizes & (1ULL << page_shift)))
665 		return -EINVAL;
666 
667 	if (!table_group->ops->set_window || !table_group->ops->unset_window ||
668 			!table_group->ops->get_table_size ||
669 			!table_group->ops->create_table)
670 		return -EPERM;
671 
672 	/* Create TCE table */
673 	ret = tce_iommu_create_table(container, table_group, num,
674 			page_shift, window_size, levels, &tbl);
675 	if (ret)
676 		return ret;
677 
678 	BUG_ON(!tbl->it_ops->free);
679 
680 	/*
681 	 * Program the table to every group.
682 	 * Groups have been tested for compatibility at the attach time.
683 	 */
684 	list_for_each_entry(tcegrp, &container->group_list, next) {
685 		table_group = iommu_group_get_iommudata(tcegrp->grp);
686 
687 		ret = table_group->ops->set_window(table_group, num, tbl);
688 		if (ret)
689 			goto unset_exit;
690 	}
691 
692 	container->tables[num] = tbl;
693 
694 	/* Return start address assigned by platform in create_table() */
695 	*start_addr = tbl->it_offset << tbl->it_page_shift;
696 
697 	return 0;
698 
699 unset_exit:
700 	list_for_each_entry(tcegrp, &container->group_list, next) {
701 		table_group = iommu_group_get_iommudata(tcegrp->grp);
702 		table_group->ops->unset_window(table_group, num);
703 	}
704 	tce_iommu_free_table(container, tbl);
705 
706 	return ret;
707 }
708 
709 static long tce_iommu_remove_window(struct tce_container *container,
710 		__u64 start_addr)
711 {
712 	struct iommu_table_group *table_group = NULL;
713 	struct iommu_table *tbl;
714 	struct tce_iommu_group *tcegrp;
715 	int num;
716 
717 	num = tce_iommu_find_table(container, start_addr, &tbl);
718 	if (num < 0)
719 		return -EINVAL;
720 
721 	BUG_ON(!tbl->it_size);
722 
723 	/* Detach groups from IOMMUs */
724 	list_for_each_entry(tcegrp, &container->group_list, next) {
725 		table_group = iommu_group_get_iommudata(tcegrp->grp);
726 
727 		/*
728 		 * SPAPR TCE IOMMU exposes the default DMA window to
729 		 * the guest via dma32_window_start/size of
730 		 * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
731 		 * the userspace to remove this window, some do not so
732 		 * here we check for the platform capability.
733 		 */
734 		if (!table_group->ops || !table_group->ops->unset_window)
735 			return -EPERM;
736 
737 		table_group->ops->unset_window(table_group, num);
738 	}
739 
740 	/* Free table */
741 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
742 	tce_iommu_free_table(container, tbl);
743 	container->tables[num] = NULL;
744 
745 	return 0;
746 }
747 
748 static long tce_iommu_create_default_window(struct tce_container *container)
749 {
750 	long ret;
751 	__u64 start_addr = 0;
752 	struct tce_iommu_group *tcegrp;
753 	struct iommu_table_group *table_group;
754 
755 	if (!container->def_window_pending)
756 		return 0;
757 
758 	if (!tce_groups_attached(container))
759 		return -ENODEV;
760 
761 	tcegrp = list_first_entry(&container->group_list,
762 			struct tce_iommu_group, next);
763 	table_group = iommu_group_get_iommudata(tcegrp->grp);
764 	if (!table_group)
765 		return -ENODEV;
766 
767 	ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
768 			table_group->tce32_size, 1, &start_addr);
769 	WARN_ON_ONCE(!ret && start_addr);
770 
771 	if (!ret)
772 		container->def_window_pending = false;
773 
774 	return ret;
775 }
776 
777 static long vfio_spapr_ioctl_eeh_pe_op(struct iommu_group *group,
778 				       unsigned long arg)
779 {
780 	struct eeh_pe *pe;
781 	struct vfio_eeh_pe_op op;
782 	unsigned long minsz;
783 
784 	pe = eeh_iommu_group_to_pe(group);
785 	if (!pe)
786 		return -ENODEV;
787 
788 	minsz = offsetofend(struct vfio_eeh_pe_op, op);
789 	if (copy_from_user(&op, (void __user *)arg, minsz))
790 		return -EFAULT;
791 	if (op.argsz < minsz || op.flags)
792 		return -EINVAL;
793 
794 	switch (op.op) {
795 	case VFIO_EEH_PE_DISABLE:
796 		return eeh_pe_set_option(pe, EEH_OPT_DISABLE);
797 	case VFIO_EEH_PE_ENABLE:
798 		return eeh_pe_set_option(pe, EEH_OPT_ENABLE);
799 	case VFIO_EEH_PE_UNFREEZE_IO:
800 		return eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
801 	case VFIO_EEH_PE_UNFREEZE_DMA:
802 		return eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
803 	case VFIO_EEH_PE_GET_STATE:
804 		return eeh_pe_get_state(pe);
805 		break;
806 	case VFIO_EEH_PE_RESET_DEACTIVATE:
807 		return eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, true);
808 	case VFIO_EEH_PE_RESET_HOT:
809 		return eeh_pe_reset(pe, EEH_RESET_HOT, true);
810 	case VFIO_EEH_PE_RESET_FUNDAMENTAL:
811 		return eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL, true);
812 	case VFIO_EEH_PE_CONFIGURE:
813 		return eeh_pe_configure(pe);
814 	case VFIO_EEH_PE_INJECT_ERR:
815 		minsz = offsetofend(struct vfio_eeh_pe_op, err.mask);
816 		if (op.argsz < minsz)
817 			return -EINVAL;
818 		if (copy_from_user(&op, (void __user *)arg, minsz))
819 			return -EFAULT;
820 
821 		return eeh_pe_inject_err(pe, op.err.type, op.err.func,
822 					 op.err.addr, op.err.mask);
823 	default:
824 		return -EINVAL;
825 	}
826 }
827 
828 static long tce_iommu_ioctl(void *iommu_data,
829 				 unsigned int cmd, unsigned long arg)
830 {
831 	struct tce_container *container = iommu_data;
832 	unsigned long minsz, ddwsz;
833 	long ret;
834 
835 	switch (cmd) {
836 	case VFIO_CHECK_EXTENSION:
837 		switch (arg) {
838 		case VFIO_SPAPR_TCE_IOMMU:
839 		case VFIO_SPAPR_TCE_v2_IOMMU:
840 			return 1;
841 		case VFIO_EEH:
842 			return eeh_enabled();
843 		default:
844 			return 0;
845 		}
846 	}
847 
848 	/*
849 	 * Sanity check to prevent one userspace from manipulating
850 	 * another userspace mm.
851 	 */
852 	BUG_ON(!container);
853 	if (container->mm && container->mm != current->mm)
854 		return -EPERM;
855 
856 	switch (cmd) {
857 	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
858 		struct vfio_iommu_spapr_tce_info info;
859 		struct tce_iommu_group *tcegrp;
860 		struct iommu_table_group *table_group;
861 
862 		if (!tce_groups_attached(container))
863 			return -ENXIO;
864 
865 		tcegrp = list_first_entry(&container->group_list,
866 				struct tce_iommu_group, next);
867 		table_group = iommu_group_get_iommudata(tcegrp->grp);
868 
869 		if (!table_group)
870 			return -ENXIO;
871 
872 		minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
873 				dma32_window_size);
874 
875 		if (copy_from_user(&info, (void __user *)arg, minsz))
876 			return -EFAULT;
877 
878 		if (info.argsz < minsz)
879 			return -EINVAL;
880 
881 		info.dma32_window_start = table_group->tce32_start;
882 		info.dma32_window_size = table_group->tce32_size;
883 		info.flags = 0;
884 		memset(&info.ddw, 0, sizeof(info.ddw));
885 
886 		if (table_group->max_dynamic_windows_supported &&
887 				container->v2) {
888 			info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
889 			info.ddw.pgsizes = table_group->pgsizes;
890 			info.ddw.max_dynamic_windows_supported =
891 				table_group->max_dynamic_windows_supported;
892 			info.ddw.levels = table_group->max_levels;
893 		}
894 
895 		ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
896 
897 		if (info.argsz >= ddwsz)
898 			minsz = ddwsz;
899 
900 		if (copy_to_user((void __user *)arg, &info, minsz))
901 			return -EFAULT;
902 
903 		return 0;
904 	}
905 	case VFIO_IOMMU_MAP_DMA: {
906 		struct vfio_iommu_type1_dma_map param;
907 		struct iommu_table *tbl = NULL;
908 		long num;
909 		enum dma_data_direction direction;
910 
911 		if (!container->enabled)
912 			return -EPERM;
913 
914 		minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
915 
916 		if (copy_from_user(&param, (void __user *)arg, minsz))
917 			return -EFAULT;
918 
919 		if (param.argsz < minsz)
920 			return -EINVAL;
921 
922 		if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
923 				VFIO_DMA_MAP_FLAG_WRITE))
924 			return -EINVAL;
925 
926 		ret = tce_iommu_create_default_window(container);
927 		if (ret)
928 			return ret;
929 
930 		num = tce_iommu_find_table(container, param.iova, &tbl);
931 		if (num < 0)
932 			return -ENXIO;
933 
934 		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
935 				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
936 			return -EINVAL;
937 
938 		/* iova is checked by the IOMMU API */
939 		if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
940 			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
941 				direction = DMA_BIDIRECTIONAL;
942 			else
943 				direction = DMA_TO_DEVICE;
944 		} else {
945 			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
946 				direction = DMA_FROM_DEVICE;
947 			else
948 				return -EINVAL;
949 		}
950 
951 		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
952 		if (ret)
953 			return ret;
954 
955 		if (container->v2)
956 			ret = tce_iommu_build_v2(container, tbl,
957 					param.iova >> tbl->it_page_shift,
958 					param.vaddr,
959 					param.size >> tbl->it_page_shift,
960 					direction);
961 		else
962 			ret = tce_iommu_build(container, tbl,
963 					param.iova >> tbl->it_page_shift,
964 					param.vaddr,
965 					param.size >> tbl->it_page_shift,
966 					direction);
967 
968 		iommu_flush_tce(tbl);
969 
970 		return ret;
971 	}
972 	case VFIO_IOMMU_UNMAP_DMA: {
973 		struct vfio_iommu_type1_dma_unmap param;
974 		struct iommu_table *tbl = NULL;
975 		long num;
976 
977 		if (!container->enabled)
978 			return -EPERM;
979 
980 		minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
981 				size);
982 
983 		if (copy_from_user(&param, (void __user *)arg, minsz))
984 			return -EFAULT;
985 
986 		if (param.argsz < minsz)
987 			return -EINVAL;
988 
989 		/* No flag is supported now */
990 		if (param.flags)
991 			return -EINVAL;
992 
993 		ret = tce_iommu_create_default_window(container);
994 		if (ret)
995 			return ret;
996 
997 		num = tce_iommu_find_table(container, param.iova, &tbl);
998 		if (num < 0)
999 			return -ENXIO;
1000 
1001 		if (param.size & ~IOMMU_PAGE_MASK(tbl))
1002 			return -EINVAL;
1003 
1004 		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
1005 				param.size >> tbl->it_page_shift);
1006 		if (ret)
1007 			return ret;
1008 
1009 		ret = tce_iommu_clear(container, tbl,
1010 				param.iova >> tbl->it_page_shift,
1011 				param.size >> tbl->it_page_shift);
1012 		iommu_flush_tce(tbl);
1013 
1014 		return ret;
1015 	}
1016 	case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
1017 		struct vfio_iommu_spapr_register_memory param;
1018 
1019 		if (!container->v2)
1020 			break;
1021 
1022 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1023 				size);
1024 
1025 		ret = tce_iommu_mm_set(container);
1026 		if (ret)
1027 			return ret;
1028 
1029 		if (copy_from_user(&param, (void __user *)arg, minsz))
1030 			return -EFAULT;
1031 
1032 		if (param.argsz < minsz)
1033 			return -EINVAL;
1034 
1035 		/* No flag is supported now */
1036 		if (param.flags)
1037 			return -EINVAL;
1038 
1039 		mutex_lock(&container->lock);
1040 		ret = tce_iommu_register_pages(container, param.vaddr,
1041 				param.size);
1042 		mutex_unlock(&container->lock);
1043 
1044 		return ret;
1045 	}
1046 	case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
1047 		struct vfio_iommu_spapr_register_memory param;
1048 
1049 		if (!container->v2)
1050 			break;
1051 
1052 		if (!container->mm)
1053 			return -EPERM;
1054 
1055 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1056 				size);
1057 
1058 		if (copy_from_user(&param, (void __user *)arg, minsz))
1059 			return -EFAULT;
1060 
1061 		if (param.argsz < minsz)
1062 			return -EINVAL;
1063 
1064 		/* No flag is supported now */
1065 		if (param.flags)
1066 			return -EINVAL;
1067 
1068 		mutex_lock(&container->lock);
1069 		ret = tce_iommu_unregister_pages(container, param.vaddr,
1070 				param.size);
1071 		mutex_unlock(&container->lock);
1072 
1073 		return ret;
1074 	}
1075 	case VFIO_IOMMU_ENABLE:
1076 		if (container->v2)
1077 			break;
1078 
1079 		mutex_lock(&container->lock);
1080 		ret = tce_iommu_enable(container);
1081 		mutex_unlock(&container->lock);
1082 		return ret;
1083 
1084 
1085 	case VFIO_IOMMU_DISABLE:
1086 		if (container->v2)
1087 			break;
1088 
1089 		mutex_lock(&container->lock);
1090 		tce_iommu_disable(container);
1091 		mutex_unlock(&container->lock);
1092 		return 0;
1093 
1094 	case VFIO_EEH_PE_OP: {
1095 		struct tce_iommu_group *tcegrp;
1096 
1097 		ret = 0;
1098 		list_for_each_entry(tcegrp, &container->group_list, next) {
1099 			ret = vfio_spapr_ioctl_eeh_pe_op(tcegrp->grp, arg);
1100 			if (ret)
1101 				return ret;
1102 		}
1103 		return ret;
1104 	}
1105 
1106 	case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1107 		struct vfio_iommu_spapr_tce_create create;
1108 
1109 		if (!container->v2)
1110 			break;
1111 
1112 		ret = tce_iommu_mm_set(container);
1113 		if (ret)
1114 			return ret;
1115 
1116 		if (!tce_groups_attached(container))
1117 			return -ENXIO;
1118 
1119 		minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1120 				start_addr);
1121 
1122 		if (copy_from_user(&create, (void __user *)arg, minsz))
1123 			return -EFAULT;
1124 
1125 		if (create.argsz < minsz)
1126 			return -EINVAL;
1127 
1128 		if (create.flags)
1129 			return -EINVAL;
1130 
1131 		mutex_lock(&container->lock);
1132 
1133 		ret = tce_iommu_create_default_window(container);
1134 		if (!ret)
1135 			ret = tce_iommu_create_window(container,
1136 					create.page_shift,
1137 					create.window_size, create.levels,
1138 					&create.start_addr);
1139 
1140 		mutex_unlock(&container->lock);
1141 
1142 		if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1143 			ret = -EFAULT;
1144 
1145 		return ret;
1146 	}
1147 	case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1148 		struct vfio_iommu_spapr_tce_remove remove;
1149 
1150 		if (!container->v2)
1151 			break;
1152 
1153 		ret = tce_iommu_mm_set(container);
1154 		if (ret)
1155 			return ret;
1156 
1157 		if (!tce_groups_attached(container))
1158 			return -ENXIO;
1159 
1160 		minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1161 				start_addr);
1162 
1163 		if (copy_from_user(&remove, (void __user *)arg, minsz))
1164 			return -EFAULT;
1165 
1166 		if (remove.argsz < minsz)
1167 			return -EINVAL;
1168 
1169 		if (remove.flags)
1170 			return -EINVAL;
1171 
1172 		if (container->def_window_pending && !remove.start_addr) {
1173 			container->def_window_pending = false;
1174 			return 0;
1175 		}
1176 
1177 		mutex_lock(&container->lock);
1178 
1179 		ret = tce_iommu_remove_window(container, remove.start_addr);
1180 
1181 		mutex_unlock(&container->lock);
1182 
1183 		return ret;
1184 	}
1185 	}
1186 
1187 	return -ENOTTY;
1188 }
1189 
1190 static void tce_iommu_release_ownership(struct tce_container *container,
1191 		struct iommu_table_group *table_group)
1192 {
1193 	long i;
1194 
1195 	if (!table_group->ops->unset_window) {
1196 		WARN_ON_ONCE(1);
1197 		return;
1198 	}
1199 
1200 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1201 		if (container->tables[i])
1202 			table_group->ops->unset_window(table_group, i);
1203 }
1204 
1205 static long tce_iommu_take_ownership(struct tce_container *container,
1206 		struct iommu_table_group *table_group)
1207 {
1208 	long i, ret = 0;
1209 
1210 	/* Set all windows to the new group */
1211 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1212 		struct iommu_table *tbl = container->tables[i];
1213 
1214 		if (!tbl)
1215 			continue;
1216 
1217 		ret = table_group->ops->set_window(table_group, i, tbl);
1218 		if (ret)
1219 			goto release_exit;
1220 	}
1221 
1222 	return 0;
1223 
1224 release_exit:
1225 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1226 		table_group->ops->unset_window(table_group, i);
1227 
1228 	return ret;
1229 }
1230 
1231 static int tce_iommu_attach_group(void *iommu_data,
1232 		struct iommu_group *iommu_group, enum vfio_group_type type)
1233 {
1234 	int ret = 0;
1235 	struct tce_container *container = iommu_data;
1236 	struct iommu_table_group *table_group;
1237 	struct tce_iommu_group *tcegrp = NULL;
1238 
1239 	if (type == VFIO_EMULATED_IOMMU)
1240 		return -EINVAL;
1241 
1242 	mutex_lock(&container->lock);
1243 
1244 	/* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
1245 			iommu_group_id(iommu_group), iommu_group); */
1246 	table_group = iommu_group_get_iommudata(iommu_group);
1247 	if (!table_group) {
1248 		ret = -ENODEV;
1249 		goto unlock_exit;
1250 	}
1251 
1252 	/* v2 requires full support of dynamic DMA windows */
1253 	if (container->v2 && table_group->max_dynamic_windows_supported == 0) {
1254 		ret = -EINVAL;
1255 		goto unlock_exit;
1256 	}
1257 
1258 	/* v1 reuses TCE tables and does not share them among PEs */
1259 	if (!container->v2 && tce_groups_attached(container)) {
1260 		ret = -EBUSY;
1261 		goto unlock_exit;
1262 	}
1263 
1264 	/*
1265 	 * Check if new group has the same iommu_table_group_ops
1266 	 * (i.e. compatible)
1267 	 */
1268 	list_for_each_entry(tcegrp, &container->group_list, next) {
1269 		struct iommu_table_group *table_group_tmp;
1270 
1271 		if (tcegrp->grp == iommu_group) {
1272 			pr_warn("tce_vfio: Group %d is already attached\n",
1273 					iommu_group_id(iommu_group));
1274 			ret = -EBUSY;
1275 			goto unlock_exit;
1276 		}
1277 		table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
1278 		if (table_group_tmp->ops->create_table !=
1279 				table_group->ops->create_table) {
1280 			pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
1281 					iommu_group_id(iommu_group),
1282 					iommu_group_id(tcegrp->grp));
1283 			ret = -EPERM;
1284 			goto unlock_exit;
1285 		}
1286 	}
1287 
1288 	tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
1289 	if (!tcegrp) {
1290 		ret = -ENOMEM;
1291 		goto unlock_exit;
1292 	}
1293 
1294 	ret = tce_iommu_take_ownership(container, table_group);
1295 	if (!tce_groups_attached(container) && !container->tables[0])
1296 		container->def_window_pending = true;
1297 
1298 	if (!ret) {
1299 		tcegrp->grp = iommu_group;
1300 		list_add(&tcegrp->next, &container->group_list);
1301 	}
1302 
1303 	if (ret && tcegrp)
1304 		kfree(tcegrp);
1305 
1306 unlock_exit:
1307 	mutex_unlock(&container->lock);
1308 
1309 	return ret;
1310 }
1311 
1312 static void tce_iommu_detach_group(void *iommu_data,
1313 		struct iommu_group *iommu_group)
1314 {
1315 	struct tce_container *container = iommu_data;
1316 	struct iommu_table_group *table_group;
1317 	bool found = false;
1318 	struct tce_iommu_group *tcegrp;
1319 
1320 	mutex_lock(&container->lock);
1321 
1322 	list_for_each_entry(tcegrp, &container->group_list, next) {
1323 		if (tcegrp->grp == iommu_group) {
1324 			found = true;
1325 			break;
1326 		}
1327 	}
1328 
1329 	if (!found) {
1330 		pr_warn("tce_vfio: detaching unattached group #%u\n",
1331 				iommu_group_id(iommu_group));
1332 		goto unlock_exit;
1333 	}
1334 
1335 	list_del(&tcegrp->next);
1336 	kfree(tcegrp);
1337 
1338 	table_group = iommu_group_get_iommudata(iommu_group);
1339 	BUG_ON(!table_group);
1340 
1341 	tce_iommu_release_ownership(container, table_group);
1342 
1343 unlock_exit:
1344 	mutex_unlock(&container->lock);
1345 }
1346 
1347 static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
1348 	.name		= "iommu-vfio-powerpc",
1349 	.owner		= THIS_MODULE,
1350 	.open		= tce_iommu_open,
1351 	.release	= tce_iommu_release,
1352 	.ioctl		= tce_iommu_ioctl,
1353 	.attach_group	= tce_iommu_attach_group,
1354 	.detach_group	= tce_iommu_detach_group,
1355 };
1356 
1357 static int __init tce_iommu_init(void)
1358 {
1359 	return vfio_register_iommu_driver(&tce_iommu_driver_ops);
1360 }
1361 
1362 static void __exit tce_iommu_cleanup(void)
1363 {
1364 	vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
1365 }
1366 
1367 module_init(tce_iommu_init);
1368 module_exit(tce_iommu_cleanup);
1369 
1370 MODULE_VERSION(DRIVER_VERSION);
1371 MODULE_LICENSE("GPL v2");
1372 MODULE_AUTHOR(DRIVER_AUTHOR);
1373 MODULE_DESCRIPTION(DRIVER_DESC);
1374 
1375