1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
25ffd229cSAlexey Kardashevskiy /*
35ffd229cSAlexey Kardashevskiy * VFIO: IOMMU DMA mapping support for TCE on POWER
45ffd229cSAlexey Kardashevskiy *
55ffd229cSAlexey Kardashevskiy * Copyright (C) 2013 IBM Corp. All rights reserved.
65ffd229cSAlexey Kardashevskiy * Author: Alexey Kardashevskiy <aik@ozlabs.ru>
7e276e258SJason Gunthorpe * Copyright Gavin Shan, IBM Corporation 2014.
85ffd229cSAlexey Kardashevskiy *
95ffd229cSAlexey Kardashevskiy * Derived from original vfio_iommu_type1.c:
105ffd229cSAlexey Kardashevskiy * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
115ffd229cSAlexey Kardashevskiy * Author: Alex Williamson <alex.williamson@redhat.com>
125ffd229cSAlexey Kardashevskiy */
135ffd229cSAlexey Kardashevskiy
145ffd229cSAlexey Kardashevskiy #include <linux/module.h>
155ffd229cSAlexey Kardashevskiy #include <linux/pci.h>
165ffd229cSAlexey Kardashevskiy #include <linux/slab.h>
175ffd229cSAlexey Kardashevskiy #include <linux/uaccess.h>
185ffd229cSAlexey Kardashevskiy #include <linux/err.h>
195ffd229cSAlexey Kardashevskiy #include <linux/vfio.h>
202157e7b8SAlexey Kardashevskiy #include <linux/vmalloc.h>
216e84f315SIngo Molnar #include <linux/sched/mm.h>
223f07c014SIngo Molnar #include <linux/sched/signal.h>
2379eb597cSDaniel Jordan #include <linux/mm.h>
248cc02d22SChristoph Hellwig #include "vfio.h"
256e84f315SIngo Molnar
265ffd229cSAlexey Kardashevskiy #include <asm/iommu.h>
275ffd229cSAlexey Kardashevskiy #include <asm/tce.h>
282157e7b8SAlexey Kardashevskiy #include <asm/mmu_context.h>
295ffd229cSAlexey Kardashevskiy
305ffd229cSAlexey Kardashevskiy #define DRIVER_VERSION "0.1"
315ffd229cSAlexey Kardashevskiy #define DRIVER_AUTHOR "aik@ozlabs.ru"
325ffd229cSAlexey Kardashevskiy #define DRIVER_DESC "VFIO IOMMU SPAPR TCE"
335ffd229cSAlexey Kardashevskiy
345ffd229cSAlexey Kardashevskiy static void tce_iommu_detach_group(void *iommu_data,
355ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group);
365ffd229cSAlexey Kardashevskiy
375ffd229cSAlexey Kardashevskiy /*
385ffd229cSAlexey Kardashevskiy * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
395ffd229cSAlexey Kardashevskiy *
405ffd229cSAlexey Kardashevskiy * This code handles mapping and unmapping of user data buffers
415ffd229cSAlexey Kardashevskiy * into DMA'ble space using the IOMMU
425ffd229cSAlexey Kardashevskiy */
435ffd229cSAlexey Kardashevskiy
442157e7b8SAlexey Kardashevskiy struct tce_iommu_group {
452157e7b8SAlexey Kardashevskiy struct list_head next;
462157e7b8SAlexey Kardashevskiy struct iommu_group *grp;
472157e7b8SAlexey Kardashevskiy };
482157e7b8SAlexey Kardashevskiy
495ffd229cSAlexey Kardashevskiy /*
504b6fad70SAlexey Kardashevskiy * A container needs to remember which preregistered region it has
514b6fad70SAlexey Kardashevskiy * referenced to do proper cleanup at the userspace process exit.
524b6fad70SAlexey Kardashevskiy */
534b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg {
544b6fad70SAlexey Kardashevskiy struct list_head next;
554b6fad70SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem;
564b6fad70SAlexey Kardashevskiy };
574b6fad70SAlexey Kardashevskiy
584b6fad70SAlexey Kardashevskiy /*
595ffd229cSAlexey Kardashevskiy * The container descriptor supports only a single group per container.
605ffd229cSAlexey Kardashevskiy * Required by the API as the container is not supplied with the IOMMU group
615ffd229cSAlexey Kardashevskiy * at the moment of initialization.
625ffd229cSAlexey Kardashevskiy */
635ffd229cSAlexey Kardashevskiy struct tce_container {
645ffd229cSAlexey Kardashevskiy struct mutex lock;
655ffd229cSAlexey Kardashevskiy bool enabled;
662157e7b8SAlexey Kardashevskiy bool v2;
67d9c72894SAlexey Kardashevskiy bool def_window_pending;
682d270df8SAlexey Kardashevskiy unsigned long locked_pages;
69bc82d122SAlexey Kardashevskiy struct mm_struct *mm;
702157e7b8SAlexey Kardashevskiy struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
712157e7b8SAlexey Kardashevskiy struct list_head group_list;
724b6fad70SAlexey Kardashevskiy struct list_head prereg_list;
735ffd229cSAlexey Kardashevskiy };
745ffd229cSAlexey Kardashevskiy
tce_iommu_mm_set(struct tce_container * container)75bc82d122SAlexey Kardashevskiy static long tce_iommu_mm_set(struct tce_container *container)
76bc82d122SAlexey Kardashevskiy {
77bc82d122SAlexey Kardashevskiy if (container->mm) {
78bc82d122SAlexey Kardashevskiy if (container->mm == current->mm)
79bc82d122SAlexey Kardashevskiy return 0;
80bc82d122SAlexey Kardashevskiy return -EPERM;
81bc82d122SAlexey Kardashevskiy }
82bc82d122SAlexey Kardashevskiy BUG_ON(!current->mm);
83bc82d122SAlexey Kardashevskiy container->mm = current->mm;
847a49de99SJulia Lawall mmgrab(container->mm);
85bc82d122SAlexey Kardashevskiy
86bc82d122SAlexey Kardashevskiy return 0;
87bc82d122SAlexey Kardashevskiy }
88bc82d122SAlexey Kardashevskiy
tce_iommu_prereg_free(struct tce_container * container,struct tce_iommu_prereg * tcemem)894b6fad70SAlexey Kardashevskiy static long tce_iommu_prereg_free(struct tce_container *container,
904b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem)
914b6fad70SAlexey Kardashevskiy {
924b6fad70SAlexey Kardashevskiy long ret;
934b6fad70SAlexey Kardashevskiy
944b6fad70SAlexey Kardashevskiy ret = mm_iommu_put(container->mm, tcemem->mem);
954b6fad70SAlexey Kardashevskiy if (ret)
964b6fad70SAlexey Kardashevskiy return ret;
974b6fad70SAlexey Kardashevskiy
984b6fad70SAlexey Kardashevskiy list_del(&tcemem->next);
994b6fad70SAlexey Kardashevskiy kfree(tcemem);
1004b6fad70SAlexey Kardashevskiy
1014b6fad70SAlexey Kardashevskiy return 0;
1024b6fad70SAlexey Kardashevskiy }
1034b6fad70SAlexey Kardashevskiy
tce_iommu_unregister_pages(struct tce_container * container,__u64 vaddr,__u64 size)1042157e7b8SAlexey Kardashevskiy static long tce_iommu_unregister_pages(struct tce_container *container,
1052157e7b8SAlexey Kardashevskiy __u64 vaddr, __u64 size)
1062157e7b8SAlexey Kardashevskiy {
1072157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem;
1084b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem;
1094b6fad70SAlexey Kardashevskiy bool found = false;
110e0bf78b0SAlexey Kardashevskiy long ret;
1112157e7b8SAlexey Kardashevskiy
1122157e7b8SAlexey Kardashevskiy if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
1132157e7b8SAlexey Kardashevskiy return -EINVAL;
1142157e7b8SAlexey Kardashevskiy
115e0bf78b0SAlexey Kardashevskiy mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
1162157e7b8SAlexey Kardashevskiy if (!mem)
1172157e7b8SAlexey Kardashevskiy return -ENOENT;
1182157e7b8SAlexey Kardashevskiy
1194b6fad70SAlexey Kardashevskiy list_for_each_entry(tcemem, &container->prereg_list, next) {
1204b6fad70SAlexey Kardashevskiy if (tcemem->mem == mem) {
1214b6fad70SAlexey Kardashevskiy found = true;
1224b6fad70SAlexey Kardashevskiy break;
1234b6fad70SAlexey Kardashevskiy }
1244b6fad70SAlexey Kardashevskiy }
1254b6fad70SAlexey Kardashevskiy
1264b6fad70SAlexey Kardashevskiy if (!found)
127e0bf78b0SAlexey Kardashevskiy ret = -ENOENT;
128e0bf78b0SAlexey Kardashevskiy else
129e0bf78b0SAlexey Kardashevskiy ret = tce_iommu_prereg_free(container, tcemem);
1304b6fad70SAlexey Kardashevskiy
131e0bf78b0SAlexey Kardashevskiy mm_iommu_put(container->mm, mem);
132e0bf78b0SAlexey Kardashevskiy
133e0bf78b0SAlexey Kardashevskiy return ret;
1342157e7b8SAlexey Kardashevskiy }
1352157e7b8SAlexey Kardashevskiy
tce_iommu_register_pages(struct tce_container * container,__u64 vaddr,__u64 size)1362157e7b8SAlexey Kardashevskiy static long tce_iommu_register_pages(struct tce_container *container,
1372157e7b8SAlexey Kardashevskiy __u64 vaddr, __u64 size)
1382157e7b8SAlexey Kardashevskiy {
1392157e7b8SAlexey Kardashevskiy long ret = 0;
1402157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL;
1414b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem;
1422157e7b8SAlexey Kardashevskiy unsigned long entries = size >> PAGE_SHIFT;
1432157e7b8SAlexey Kardashevskiy
1442157e7b8SAlexey Kardashevskiy if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
1452157e7b8SAlexey Kardashevskiy ((vaddr + size) < vaddr))
1462157e7b8SAlexey Kardashevskiy return -EINVAL;
1472157e7b8SAlexey Kardashevskiy
148e0bf78b0SAlexey Kardashevskiy mem = mm_iommu_get(container->mm, vaddr, entries);
1494b6fad70SAlexey Kardashevskiy if (mem) {
1504b6fad70SAlexey Kardashevskiy list_for_each_entry(tcemem, &container->prereg_list, next) {
151e0bf78b0SAlexey Kardashevskiy if (tcemem->mem == mem) {
152e0bf78b0SAlexey Kardashevskiy ret = -EBUSY;
153e0bf78b0SAlexey Kardashevskiy goto put_exit;
1544b6fad70SAlexey Kardashevskiy }
1554b6fad70SAlexey Kardashevskiy }
156e0bf78b0SAlexey Kardashevskiy } else {
157e0bf78b0SAlexey Kardashevskiy ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
1582157e7b8SAlexey Kardashevskiy if (ret)
1592157e7b8SAlexey Kardashevskiy return ret;
160e0bf78b0SAlexey Kardashevskiy }
1612157e7b8SAlexey Kardashevskiy
1624b6fad70SAlexey Kardashevskiy tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
1633393af24SAlexey Kardashevskiy if (!tcemem) {
164e0bf78b0SAlexey Kardashevskiy ret = -ENOMEM;
165e0bf78b0SAlexey Kardashevskiy goto put_exit;
1663393af24SAlexey Kardashevskiy }
1673393af24SAlexey Kardashevskiy
1684b6fad70SAlexey Kardashevskiy tcemem->mem = mem;
1694b6fad70SAlexey Kardashevskiy list_add(&tcemem->next, &container->prereg_list);
1704b6fad70SAlexey Kardashevskiy
1712157e7b8SAlexey Kardashevskiy container->enabled = true;
1722157e7b8SAlexey Kardashevskiy
1732157e7b8SAlexey Kardashevskiy return 0;
174e0bf78b0SAlexey Kardashevskiy
175e0bf78b0SAlexey Kardashevskiy put_exit:
176e0bf78b0SAlexey Kardashevskiy mm_iommu_put(container->mm, mem);
177e0bf78b0SAlexey Kardashevskiy return ret;
1782157e7b8SAlexey Kardashevskiy }
1792157e7b8SAlexey Kardashevskiy
tce_page_is_contained(struct mm_struct * mm,unsigned long hpa,unsigned int it_page_shift)180c10c21efSAlexey Kardashevskiy static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
18194ad9338SMatthew Wilcox (Oracle) unsigned int it_page_shift)
182e432bc7eSAlexey Kardashevskiy {
183c10c21efSAlexey Kardashevskiy struct page *page;
184c10c21efSAlexey Kardashevskiy unsigned long size = 0;
185c10c21efSAlexey Kardashevskiy
18694ad9338SMatthew Wilcox (Oracle) if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
18794ad9338SMatthew Wilcox (Oracle) return size == (1UL << it_page_shift);
188c10c21efSAlexey Kardashevskiy
189c10c21efSAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT);
190e432bc7eSAlexey Kardashevskiy /*
191e432bc7eSAlexey Kardashevskiy * Check that the TCE table granularity is not bigger than the size of
192e432bc7eSAlexey Kardashevskiy * a page we just found. Otherwise the hardware can get access to
193e432bc7eSAlexey Kardashevskiy * a bigger memory chunk that it should.
194e432bc7eSAlexey Kardashevskiy */
19594ad9338SMatthew Wilcox (Oracle) return page_shift(compound_head(page)) >= it_page_shift;
196e432bc7eSAlexey Kardashevskiy }
197e432bc7eSAlexey Kardashevskiy
tce_groups_attached(struct tce_container * container)1982157e7b8SAlexey Kardashevskiy static inline bool tce_groups_attached(struct tce_container *container)
1992157e7b8SAlexey Kardashevskiy {
2002157e7b8SAlexey Kardashevskiy return !list_empty(&container->group_list);
2012157e7b8SAlexey Kardashevskiy }
2022157e7b8SAlexey Kardashevskiy
tce_iommu_find_table(struct tce_container * container,phys_addr_t ioba,struct iommu_table ** ptbl)2030eaf4defSAlexey Kardashevskiy static long tce_iommu_find_table(struct tce_container *container,
2040eaf4defSAlexey Kardashevskiy phys_addr_t ioba, struct iommu_table **ptbl)
2050eaf4defSAlexey Kardashevskiy {
2060eaf4defSAlexey Kardashevskiy long i;
2070eaf4defSAlexey Kardashevskiy
2080eaf4defSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
2092157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i];
2100eaf4defSAlexey Kardashevskiy
2110eaf4defSAlexey Kardashevskiy if (tbl) {
2120eaf4defSAlexey Kardashevskiy unsigned long entry = ioba >> tbl->it_page_shift;
2130eaf4defSAlexey Kardashevskiy unsigned long start = tbl->it_offset;
2140eaf4defSAlexey Kardashevskiy unsigned long end = start + tbl->it_size;
2150eaf4defSAlexey Kardashevskiy
2160eaf4defSAlexey Kardashevskiy if ((start <= entry) && (entry < end)) {
2170eaf4defSAlexey Kardashevskiy *ptbl = tbl;
2180eaf4defSAlexey Kardashevskiy return i;
2190eaf4defSAlexey Kardashevskiy }
2200eaf4defSAlexey Kardashevskiy }
2210eaf4defSAlexey Kardashevskiy }
2220eaf4defSAlexey Kardashevskiy
2230eaf4defSAlexey Kardashevskiy return -1;
2240eaf4defSAlexey Kardashevskiy }
2250eaf4defSAlexey Kardashevskiy
tce_iommu_find_free_table(struct tce_container * container)226e633bc86SAlexey Kardashevskiy static int tce_iommu_find_free_table(struct tce_container *container)
227e633bc86SAlexey Kardashevskiy {
228e633bc86SAlexey Kardashevskiy int i;
229e633bc86SAlexey Kardashevskiy
230e633bc86SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
231e633bc86SAlexey Kardashevskiy if (!container->tables[i])
232e633bc86SAlexey Kardashevskiy return i;
233e633bc86SAlexey Kardashevskiy }
234e633bc86SAlexey Kardashevskiy
235e633bc86SAlexey Kardashevskiy return -ENOSPC;
236e633bc86SAlexey Kardashevskiy }
237e633bc86SAlexey Kardashevskiy
tce_iommu_enable(struct tce_container * container)2385ffd229cSAlexey Kardashevskiy static int tce_iommu_enable(struct tce_container *container)
2395ffd229cSAlexey Kardashevskiy {
2405ffd229cSAlexey Kardashevskiy int ret = 0;
2412d270df8SAlexey Kardashevskiy unsigned long locked;
2420eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group;
2432157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
2445ffd229cSAlexey Kardashevskiy
2455ffd229cSAlexey Kardashevskiy if (container->enabled)
2465ffd229cSAlexey Kardashevskiy return -EBUSY;
2475ffd229cSAlexey Kardashevskiy
2485ffd229cSAlexey Kardashevskiy /*
2495ffd229cSAlexey Kardashevskiy * When userspace pages are mapped into the IOMMU, they are effectively
2505ffd229cSAlexey Kardashevskiy * locked memory, so, theoretically, we need to update the accounting
2515ffd229cSAlexey Kardashevskiy * of locked pages on each map and unmap. For powerpc, the map unmap
2525ffd229cSAlexey Kardashevskiy * paths can be very hot, though, and the accounting would kill
2535ffd229cSAlexey Kardashevskiy * performance, especially since it would be difficult to impossible
2545ffd229cSAlexey Kardashevskiy * to handle the accounting in real mode only.
2555ffd229cSAlexey Kardashevskiy *
2565ffd229cSAlexey Kardashevskiy * To address that, rather than precisely accounting every page, we
2575ffd229cSAlexey Kardashevskiy * instead account for a worst case on locked memory when the iommu is
2585ffd229cSAlexey Kardashevskiy * enabled and disabled. The worst case upper bound on locked memory
2595ffd229cSAlexey Kardashevskiy * is the size of the whole iommu window, which is usually relatively
2605ffd229cSAlexey Kardashevskiy * small (compared to total memory sizes) on POWER hardware.
2615ffd229cSAlexey Kardashevskiy *
2625ffd229cSAlexey Kardashevskiy * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
2635ffd229cSAlexey Kardashevskiy * that would effectively kill the guest at random points, much better
2645ffd229cSAlexey Kardashevskiy * enforcing the limit based on the max that the guest can map.
2652d270df8SAlexey Kardashevskiy *
2662d270df8SAlexey Kardashevskiy * Unfortunately at the moment it counts whole tables, no matter how
2672d270df8SAlexey Kardashevskiy * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
2682d270df8SAlexey Kardashevskiy * each with 2GB DMA window, 8GB will be counted here. The reason for
2692d270df8SAlexey Kardashevskiy * this is that we cannot tell here the amount of RAM used by the guest
2702d270df8SAlexey Kardashevskiy * as this information is only available from KVM and VFIO is
2712d270df8SAlexey Kardashevskiy * KVM agnostic.
2724793d65dSAlexey Kardashevskiy *
2734793d65dSAlexey Kardashevskiy * So we do not allow enabling a container without a group attached
2744793d65dSAlexey Kardashevskiy * as there is no way to know how much we should increment
2754793d65dSAlexey Kardashevskiy * the locked_vm counter.
2765ffd229cSAlexey Kardashevskiy */
2772157e7b8SAlexey Kardashevskiy if (!tce_groups_attached(container))
2782157e7b8SAlexey Kardashevskiy return -ENODEV;
2792157e7b8SAlexey Kardashevskiy
2802157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list,
2812157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next);
2822157e7b8SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
2830eaf4defSAlexey Kardashevskiy if (!table_group)
2840eaf4defSAlexey Kardashevskiy return -ENODEV;
2850eaf4defSAlexey Kardashevskiy
2864793d65dSAlexey Kardashevskiy if (!table_group->tce32_size)
2874793d65dSAlexey Kardashevskiy return -EPERM;
2884793d65dSAlexey Kardashevskiy
289bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container);
290bc82d122SAlexey Kardashevskiy if (ret)
291bc82d122SAlexey Kardashevskiy return ret;
292bc82d122SAlexey Kardashevskiy
2934793d65dSAlexey Kardashevskiy locked = table_group->tce32_size >> PAGE_SHIFT;
29479eb597cSDaniel Jordan ret = account_locked_vm(container->mm, locked, true);
2952d270df8SAlexey Kardashevskiy if (ret)
2962d270df8SAlexey Kardashevskiy return ret;
2975ffd229cSAlexey Kardashevskiy
2982d270df8SAlexey Kardashevskiy container->locked_pages = locked;
2992d270df8SAlexey Kardashevskiy
3005ffd229cSAlexey Kardashevskiy container->enabled = true;
3015ffd229cSAlexey Kardashevskiy
3025ffd229cSAlexey Kardashevskiy return ret;
3035ffd229cSAlexey Kardashevskiy }
3045ffd229cSAlexey Kardashevskiy
tce_iommu_disable(struct tce_container * container)3055ffd229cSAlexey Kardashevskiy static void tce_iommu_disable(struct tce_container *container)
3065ffd229cSAlexey Kardashevskiy {
3075ffd229cSAlexey Kardashevskiy if (!container->enabled)
3085ffd229cSAlexey Kardashevskiy return;
3095ffd229cSAlexey Kardashevskiy
3105ffd229cSAlexey Kardashevskiy container->enabled = false;
3115ffd229cSAlexey Kardashevskiy
312bc82d122SAlexey Kardashevskiy BUG_ON(!container->mm);
31379eb597cSDaniel Jordan account_locked_vm(container->mm, container->locked_pages, false);
3145ffd229cSAlexey Kardashevskiy }
3155ffd229cSAlexey Kardashevskiy
tce_iommu_open(unsigned long arg)3165ffd229cSAlexey Kardashevskiy static void *tce_iommu_open(unsigned long arg)
3175ffd229cSAlexey Kardashevskiy {
3185ffd229cSAlexey Kardashevskiy struct tce_container *container;
3195ffd229cSAlexey Kardashevskiy
3202157e7b8SAlexey Kardashevskiy if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
3215ffd229cSAlexey Kardashevskiy pr_err("tce_vfio: Wrong IOMMU type\n");
3225ffd229cSAlexey Kardashevskiy return ERR_PTR(-EINVAL);
3235ffd229cSAlexey Kardashevskiy }
3245ffd229cSAlexey Kardashevskiy
3255ffd229cSAlexey Kardashevskiy container = kzalloc(sizeof(*container), GFP_KERNEL);
3265ffd229cSAlexey Kardashevskiy if (!container)
3275ffd229cSAlexey Kardashevskiy return ERR_PTR(-ENOMEM);
3285ffd229cSAlexey Kardashevskiy
3295ffd229cSAlexey Kardashevskiy mutex_init(&container->lock);
3302157e7b8SAlexey Kardashevskiy INIT_LIST_HEAD_RCU(&container->group_list);
3314b6fad70SAlexey Kardashevskiy INIT_LIST_HEAD_RCU(&container->prereg_list);
3322157e7b8SAlexey Kardashevskiy
3332157e7b8SAlexey Kardashevskiy container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
3345ffd229cSAlexey Kardashevskiy
3355ffd229cSAlexey Kardashevskiy return container;
3365ffd229cSAlexey Kardashevskiy }
3375ffd229cSAlexey Kardashevskiy
3382157e7b8SAlexey Kardashevskiy static int tce_iommu_clear(struct tce_container *container,
3392157e7b8SAlexey Kardashevskiy struct iommu_table *tbl,
3402157e7b8SAlexey Kardashevskiy unsigned long entry, unsigned long pages);
341bc82d122SAlexey Kardashevskiy static void tce_iommu_free_table(struct tce_container *container,
342bc82d122SAlexey Kardashevskiy struct iommu_table *tbl);
3432157e7b8SAlexey Kardashevskiy
tce_iommu_release(void * iommu_data)3445ffd229cSAlexey Kardashevskiy static void tce_iommu_release(void *iommu_data)
3455ffd229cSAlexey Kardashevskiy {
3465ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data;
3472157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
348517ad4aeSAlexey Kardashevskiy struct tce_iommu_prereg *tcemem, *tmtmp;
3492157e7b8SAlexey Kardashevskiy long i;
3505ffd229cSAlexey Kardashevskiy
3512157e7b8SAlexey Kardashevskiy while (tce_groups_attached(container)) {
3522157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list,
3532157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next);
3542157e7b8SAlexey Kardashevskiy tce_iommu_detach_group(iommu_data, tcegrp->grp);
3552157e7b8SAlexey Kardashevskiy }
3565ffd229cSAlexey Kardashevskiy
3572157e7b8SAlexey Kardashevskiy /*
3582157e7b8SAlexey Kardashevskiy * If VFIO created a table, it was not disposed
3592157e7b8SAlexey Kardashevskiy * by tce_iommu_detach_group() so do it now.
3602157e7b8SAlexey Kardashevskiy */
3612157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
3622157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i];
3632157e7b8SAlexey Kardashevskiy
3642157e7b8SAlexey Kardashevskiy if (!tbl)
3652157e7b8SAlexey Kardashevskiy continue;
3662157e7b8SAlexey Kardashevskiy
3672157e7b8SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
368bc82d122SAlexey Kardashevskiy tce_iommu_free_table(container, tbl);
3692157e7b8SAlexey Kardashevskiy }
3705ffd229cSAlexey Kardashevskiy
371517ad4aeSAlexey Kardashevskiy list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
372517ad4aeSAlexey Kardashevskiy WARN_ON(tce_iommu_prereg_free(container, tcemem));
3734b6fad70SAlexey Kardashevskiy
374649354b7SAlexey Kardashevskiy tce_iommu_disable(container);
375bc82d122SAlexey Kardashevskiy if (container->mm)
376bc82d122SAlexey Kardashevskiy mmdrop(container->mm);
3775ffd229cSAlexey Kardashevskiy mutex_destroy(&container->lock);
3785ffd229cSAlexey Kardashevskiy
3795ffd229cSAlexey Kardashevskiy kfree(container);
3805ffd229cSAlexey Kardashevskiy }
3815ffd229cSAlexey Kardashevskiy
tce_iommu_unuse_page(unsigned long hpa)382ff4f65e4SDeming Wang static void tce_iommu_unuse_page(unsigned long hpa)
383649354b7SAlexey Kardashevskiy {
384649354b7SAlexey Kardashevskiy struct page *page;
385649354b7SAlexey Kardashevskiy
38605c6cfb9SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT);
3879d532f28SJohn Hubbard unpin_user_page(page);
388649354b7SAlexey Kardashevskiy }
389649354b7SAlexey Kardashevskiy
tce_iommu_prereg_ua_to_hpa(struct tce_container * container,unsigned long tce,unsigned long shift,unsigned long * phpa,struct mm_iommu_table_group_mem_t ** pmem)390bc82d122SAlexey Kardashevskiy static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
3911463edcaSAlexey Kardashevskiy unsigned long tce, unsigned long shift,
3922157e7b8SAlexey Kardashevskiy unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
3932157e7b8SAlexey Kardashevskiy {
3942157e7b8SAlexey Kardashevskiy long ret = 0;
3952157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem;
3962157e7b8SAlexey Kardashevskiy
3971463edcaSAlexey Kardashevskiy mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
3982157e7b8SAlexey Kardashevskiy if (!mem)
3992157e7b8SAlexey Kardashevskiy return -EINVAL;
4002157e7b8SAlexey Kardashevskiy
40176fa4975SAlexey Kardashevskiy ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
4022157e7b8SAlexey Kardashevskiy if (ret)
4032157e7b8SAlexey Kardashevskiy return -EINVAL;
4042157e7b8SAlexey Kardashevskiy
4052157e7b8SAlexey Kardashevskiy *pmem = mem;
4062157e7b8SAlexey Kardashevskiy
4072157e7b8SAlexey Kardashevskiy return 0;
4082157e7b8SAlexey Kardashevskiy }
4092157e7b8SAlexey Kardashevskiy
tce_iommu_unuse_page_v2(struct tce_container * container,struct iommu_table * tbl,unsigned long entry)410bc82d122SAlexey Kardashevskiy static void tce_iommu_unuse_page_v2(struct tce_container *container,
411bc82d122SAlexey Kardashevskiy struct iommu_table *tbl, unsigned long entry)
4122157e7b8SAlexey Kardashevskiy {
4132157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL;
4142157e7b8SAlexey Kardashevskiy int ret;
4152157e7b8SAlexey Kardashevskiy unsigned long hpa = 0;
4166e301a8eSAlexey Kardashevskiy __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
4172157e7b8SAlexey Kardashevskiy
418bc82d122SAlexey Kardashevskiy if (!pua)
4192157e7b8SAlexey Kardashevskiy return;
4202157e7b8SAlexey Kardashevskiy
42100a5c58dSAlexey Kardashevskiy ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
422b3124ec2SMichael Ellerman tbl->it_page_shift, &hpa, &mem);
4232157e7b8SAlexey Kardashevskiy if (ret)
42400a5c58dSAlexey Kardashevskiy pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
42500a5c58dSAlexey Kardashevskiy __func__, be64_to_cpu(*pua), entry, ret);
4262157e7b8SAlexey Kardashevskiy if (mem)
4272157e7b8SAlexey Kardashevskiy mm_iommu_mapped_dec(mem);
4282157e7b8SAlexey Kardashevskiy
42900a5c58dSAlexey Kardashevskiy *pua = cpu_to_be64(0);
4302157e7b8SAlexey Kardashevskiy }
4312157e7b8SAlexey Kardashevskiy
tce_iommu_clear(struct tce_container * container,struct iommu_table * tbl,unsigned long entry,unsigned long pages)4329b14a1ffSAlexey Kardashevskiy static int tce_iommu_clear(struct tce_container *container,
4339b14a1ffSAlexey Kardashevskiy struct iommu_table *tbl,
4349b14a1ffSAlexey Kardashevskiy unsigned long entry, unsigned long pages)
4359b14a1ffSAlexey Kardashevskiy {
43605c6cfb9SAlexey Kardashevskiy unsigned long oldhpa;
43705c6cfb9SAlexey Kardashevskiy long ret;
43805c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction;
439650ab1e3SAlexey Kardashevskiy unsigned long lastentry = entry + pages, firstentry = entry;
4409b14a1ffSAlexey Kardashevskiy
4416e301a8eSAlexey Kardashevskiy for ( ; entry < lastentry; ++entry) {
4426e301a8eSAlexey Kardashevskiy if (tbl->it_indirect_levels && tbl->it_userspace) {
4436e301a8eSAlexey Kardashevskiy /*
4446e301a8eSAlexey Kardashevskiy * For multilevel tables, we can take a shortcut here
4456e301a8eSAlexey Kardashevskiy * and skip some TCEs as we know that the userspace
4466e301a8eSAlexey Kardashevskiy * addresses cache is a mirror of the real TCE table
4476e301a8eSAlexey Kardashevskiy * and if it is missing some indirect levels, then
4486e301a8eSAlexey Kardashevskiy * the hardware table does not have them allocated
4496e301a8eSAlexey Kardashevskiy * either and therefore does not require updating.
4506e301a8eSAlexey Kardashevskiy */
4516e301a8eSAlexey Kardashevskiy __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
4526e301a8eSAlexey Kardashevskiy entry);
4536e301a8eSAlexey Kardashevskiy if (!pua) {
4546e301a8eSAlexey Kardashevskiy /* align to level_size which is power of two */
4556e301a8eSAlexey Kardashevskiy entry |= tbl->it_level_size - 1;
4566e301a8eSAlexey Kardashevskiy continue;
4576e301a8eSAlexey Kardashevskiy }
4586e301a8eSAlexey Kardashevskiy }
4596e301a8eSAlexey Kardashevskiy
4605c2fefd8SAlexey Kardashevskiy cond_resched();
4615c2fefd8SAlexey Kardashevskiy
46205c6cfb9SAlexey Kardashevskiy direction = DMA_NONE;
46305c6cfb9SAlexey Kardashevskiy oldhpa = 0;
464650ab1e3SAlexey Kardashevskiy ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
465c10c21efSAlexey Kardashevskiy &direction);
46605c6cfb9SAlexey Kardashevskiy if (ret)
4679b14a1ffSAlexey Kardashevskiy continue;
4689b14a1ffSAlexey Kardashevskiy
46905c6cfb9SAlexey Kardashevskiy if (direction == DMA_NONE)
47005c6cfb9SAlexey Kardashevskiy continue;
47105c6cfb9SAlexey Kardashevskiy
4722157e7b8SAlexey Kardashevskiy if (container->v2) {
473bc82d122SAlexey Kardashevskiy tce_iommu_unuse_page_v2(container, tbl, entry);
4742157e7b8SAlexey Kardashevskiy continue;
4752157e7b8SAlexey Kardashevskiy }
4762157e7b8SAlexey Kardashevskiy
477ff4f65e4SDeming Wang tce_iommu_unuse_page(oldhpa);
4789b14a1ffSAlexey Kardashevskiy }
479649354b7SAlexey Kardashevskiy
480650ab1e3SAlexey Kardashevskiy iommu_tce_kill(tbl, firstentry, pages);
481650ab1e3SAlexey Kardashevskiy
482649354b7SAlexey Kardashevskiy return 0;
4839b14a1ffSAlexey Kardashevskiy }
4849b14a1ffSAlexey Kardashevskiy
tce_iommu_use_page(unsigned long tce,unsigned long * hpa)485649354b7SAlexey Kardashevskiy static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
486649354b7SAlexey Kardashevskiy {
487649354b7SAlexey Kardashevskiy struct page *page = NULL;
488649354b7SAlexey Kardashevskiy enum dma_data_direction direction = iommu_tce_direction(tce);
489649354b7SAlexey Kardashevskiy
4909d532f28SJohn Hubbard if (pin_user_pages_fast(tce & PAGE_MASK, 1,
49173b0140bSIra Weiny direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
49273b0140bSIra Weiny &page) != 1)
493649354b7SAlexey Kardashevskiy return -EFAULT;
494649354b7SAlexey Kardashevskiy
495649354b7SAlexey Kardashevskiy *hpa = __pa((unsigned long) page_address(page));
496649354b7SAlexey Kardashevskiy
4979b14a1ffSAlexey Kardashevskiy return 0;
4989b14a1ffSAlexey Kardashevskiy }
4999b14a1ffSAlexey Kardashevskiy
tce_iommu_build(struct tce_container * container,struct iommu_table * tbl,unsigned long entry,unsigned long tce,unsigned long pages,enum dma_data_direction direction)5009b14a1ffSAlexey Kardashevskiy static long tce_iommu_build(struct tce_container *container,
5019b14a1ffSAlexey Kardashevskiy struct iommu_table *tbl,
50205c6cfb9SAlexey Kardashevskiy unsigned long entry, unsigned long tce, unsigned long pages,
50305c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction)
5049b14a1ffSAlexey Kardashevskiy {
5059b14a1ffSAlexey Kardashevskiy long i, ret = 0;
506649354b7SAlexey Kardashevskiy unsigned long hpa;
50705c6cfb9SAlexey Kardashevskiy enum dma_data_direction dirtmp;
5089b14a1ffSAlexey Kardashevskiy
5099b14a1ffSAlexey Kardashevskiy for (i = 0; i < pages; ++i) {
5109b14a1ffSAlexey Kardashevskiy unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
5119b14a1ffSAlexey Kardashevskiy
512649354b7SAlexey Kardashevskiy ret = tce_iommu_use_page(tce, &hpa);
513649354b7SAlexey Kardashevskiy if (ret)
5149b14a1ffSAlexey Kardashevskiy break;
515e432bc7eSAlexey Kardashevskiy
516c10c21efSAlexey Kardashevskiy if (!tce_page_is_contained(container->mm, hpa,
517c10c21efSAlexey Kardashevskiy tbl->it_page_shift)) {
518e432bc7eSAlexey Kardashevskiy ret = -EPERM;
519e432bc7eSAlexey Kardashevskiy break;
520e432bc7eSAlexey Kardashevskiy }
521e432bc7eSAlexey Kardashevskiy
522649354b7SAlexey Kardashevskiy hpa |= offset;
52305c6cfb9SAlexey Kardashevskiy dirtmp = direction;
524650ab1e3SAlexey Kardashevskiy ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
525650ab1e3SAlexey Kardashevskiy &hpa, &dirtmp);
5269b14a1ffSAlexey Kardashevskiy if (ret) {
527ff4f65e4SDeming Wang tce_iommu_unuse_page(hpa);
5289b14a1ffSAlexey Kardashevskiy pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
5299b14a1ffSAlexey Kardashevskiy __func__, entry << tbl->it_page_shift,
5309b14a1ffSAlexey Kardashevskiy tce, ret);
5319b14a1ffSAlexey Kardashevskiy break;
5329b14a1ffSAlexey Kardashevskiy }
53305c6cfb9SAlexey Kardashevskiy
53405c6cfb9SAlexey Kardashevskiy if (dirtmp != DMA_NONE)
535ff4f65e4SDeming Wang tce_iommu_unuse_page(hpa);
53605c6cfb9SAlexey Kardashevskiy
53700663d4eSAlexey Kardashevskiy tce += IOMMU_PAGE_SIZE(tbl);
5389b14a1ffSAlexey Kardashevskiy }
5399b14a1ffSAlexey Kardashevskiy
5409b14a1ffSAlexey Kardashevskiy if (ret)
5419b14a1ffSAlexey Kardashevskiy tce_iommu_clear(container, tbl, entry, i);
542650ab1e3SAlexey Kardashevskiy else
543650ab1e3SAlexey Kardashevskiy iommu_tce_kill(tbl, entry, pages);
5449b14a1ffSAlexey Kardashevskiy
5459b14a1ffSAlexey Kardashevskiy return ret;
5469b14a1ffSAlexey Kardashevskiy }
5479b14a1ffSAlexey Kardashevskiy
tce_iommu_build_v2(struct tce_container * container,struct iommu_table * tbl,unsigned long entry,unsigned long tce,unsigned long pages,enum dma_data_direction direction)5482157e7b8SAlexey Kardashevskiy static long tce_iommu_build_v2(struct tce_container *container,
5492157e7b8SAlexey Kardashevskiy struct iommu_table *tbl,
5502157e7b8SAlexey Kardashevskiy unsigned long entry, unsigned long tce, unsigned long pages,
5512157e7b8SAlexey Kardashevskiy enum dma_data_direction direction)
5522157e7b8SAlexey Kardashevskiy {
5532157e7b8SAlexey Kardashevskiy long i, ret = 0;
5542157e7b8SAlexey Kardashevskiy unsigned long hpa;
5552157e7b8SAlexey Kardashevskiy enum dma_data_direction dirtmp;
5562157e7b8SAlexey Kardashevskiy
5572157e7b8SAlexey Kardashevskiy for (i = 0; i < pages; ++i) {
5582157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL;
55900a5c58dSAlexey Kardashevskiy __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
5602157e7b8SAlexey Kardashevskiy
561bc82d122SAlexey Kardashevskiy ret = tce_iommu_prereg_ua_to_hpa(container,
5621463edcaSAlexey Kardashevskiy tce, tbl->it_page_shift, &hpa, &mem);
5632157e7b8SAlexey Kardashevskiy if (ret)
5642157e7b8SAlexey Kardashevskiy break;
5652157e7b8SAlexey Kardashevskiy
566c10c21efSAlexey Kardashevskiy if (!tce_page_is_contained(container->mm, hpa,
567c10c21efSAlexey Kardashevskiy tbl->it_page_shift)) {
5682157e7b8SAlexey Kardashevskiy ret = -EPERM;
5692157e7b8SAlexey Kardashevskiy break;
5702157e7b8SAlexey Kardashevskiy }
5712157e7b8SAlexey Kardashevskiy
5722157e7b8SAlexey Kardashevskiy /* Preserve offset within IOMMU page */
5732157e7b8SAlexey Kardashevskiy hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
5742157e7b8SAlexey Kardashevskiy dirtmp = direction;
5752157e7b8SAlexey Kardashevskiy
5762157e7b8SAlexey Kardashevskiy /* The registered region is being unregistered */
5772157e7b8SAlexey Kardashevskiy if (mm_iommu_mapped_inc(mem))
5782157e7b8SAlexey Kardashevskiy break;
5792157e7b8SAlexey Kardashevskiy
580650ab1e3SAlexey Kardashevskiy ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
581650ab1e3SAlexey Kardashevskiy &hpa, &dirtmp);
5822157e7b8SAlexey Kardashevskiy if (ret) {
5832157e7b8SAlexey Kardashevskiy /* dirtmp cannot be DMA_NONE here */
584bc82d122SAlexey Kardashevskiy tce_iommu_unuse_page_v2(container, tbl, entry + i);
5852157e7b8SAlexey Kardashevskiy pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
5862157e7b8SAlexey Kardashevskiy __func__, entry << tbl->it_page_shift,
5872157e7b8SAlexey Kardashevskiy tce, ret);
5882157e7b8SAlexey Kardashevskiy break;
5892157e7b8SAlexey Kardashevskiy }
5902157e7b8SAlexey Kardashevskiy
5912157e7b8SAlexey Kardashevskiy if (dirtmp != DMA_NONE)
592bc82d122SAlexey Kardashevskiy tce_iommu_unuse_page_v2(container, tbl, entry + i);
5932157e7b8SAlexey Kardashevskiy
59400a5c58dSAlexey Kardashevskiy *pua = cpu_to_be64(tce);
5952157e7b8SAlexey Kardashevskiy
5962157e7b8SAlexey Kardashevskiy tce += IOMMU_PAGE_SIZE(tbl);
5972157e7b8SAlexey Kardashevskiy }
5982157e7b8SAlexey Kardashevskiy
5992157e7b8SAlexey Kardashevskiy if (ret)
6002157e7b8SAlexey Kardashevskiy tce_iommu_clear(container, tbl, entry, i);
601650ab1e3SAlexey Kardashevskiy else
602650ab1e3SAlexey Kardashevskiy iommu_tce_kill(tbl, entry, pages);
6032157e7b8SAlexey Kardashevskiy
6042157e7b8SAlexey Kardashevskiy return ret;
6052157e7b8SAlexey Kardashevskiy }
6062157e7b8SAlexey Kardashevskiy
tce_iommu_create_table(struct tce_container * container,struct iommu_table_group * table_group,int num,__u32 page_shift,__u64 window_size,__u32 levels,struct iommu_table ** ptbl)60746d3e1e1SAlexey Kardashevskiy static long tce_iommu_create_table(struct tce_container *container,
60846d3e1e1SAlexey Kardashevskiy struct iommu_table_group *table_group,
60946d3e1e1SAlexey Kardashevskiy int num,
61046d3e1e1SAlexey Kardashevskiy __u32 page_shift,
61146d3e1e1SAlexey Kardashevskiy __u64 window_size,
61246d3e1e1SAlexey Kardashevskiy __u32 levels,
61346d3e1e1SAlexey Kardashevskiy struct iommu_table **ptbl)
61446d3e1e1SAlexey Kardashevskiy {
61546d3e1e1SAlexey Kardashevskiy long ret, table_size;
61646d3e1e1SAlexey Kardashevskiy
61746d3e1e1SAlexey Kardashevskiy table_size = table_group->ops->get_table_size(page_shift, window_size,
61846d3e1e1SAlexey Kardashevskiy levels);
61946d3e1e1SAlexey Kardashevskiy if (!table_size)
62046d3e1e1SAlexey Kardashevskiy return -EINVAL;
62146d3e1e1SAlexey Kardashevskiy
62279eb597cSDaniel Jordan ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
62346d3e1e1SAlexey Kardashevskiy if (ret)
62446d3e1e1SAlexey Kardashevskiy return ret;
62546d3e1e1SAlexey Kardashevskiy
62646d3e1e1SAlexey Kardashevskiy ret = table_group->ops->create_table(table_group, num,
62746d3e1e1SAlexey Kardashevskiy page_shift, window_size, levels, ptbl);
62846d3e1e1SAlexey Kardashevskiy
62946d3e1e1SAlexey Kardashevskiy WARN_ON(!ret && !(*ptbl)->it_ops->free);
630a68bd126SAlexey Kardashevskiy WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
63146d3e1e1SAlexey Kardashevskiy
63246d3e1e1SAlexey Kardashevskiy return ret;
63346d3e1e1SAlexey Kardashevskiy }
63446d3e1e1SAlexey Kardashevskiy
tce_iommu_free_table(struct tce_container * container,struct iommu_table * tbl)635bc82d122SAlexey Kardashevskiy static void tce_iommu_free_table(struct tce_container *container,
636bc82d122SAlexey Kardashevskiy struct iommu_table *tbl)
63746d3e1e1SAlexey Kardashevskiy {
63846d3e1e1SAlexey Kardashevskiy unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
63946d3e1e1SAlexey Kardashevskiy
640e5afdf9dSAlexey Kardashevskiy iommu_tce_table_put(tbl);
64179eb597cSDaniel Jordan account_locked_vm(container->mm, pages, false);
64246d3e1e1SAlexey Kardashevskiy }
64346d3e1e1SAlexey Kardashevskiy
tce_iommu_create_window(struct tce_container * container,__u32 page_shift,__u64 window_size,__u32 levels,__u64 * start_addr)644e633bc86SAlexey Kardashevskiy static long tce_iommu_create_window(struct tce_container *container,
645e633bc86SAlexey Kardashevskiy __u32 page_shift, __u64 window_size, __u32 levels,
646e633bc86SAlexey Kardashevskiy __u64 *start_addr)
647e633bc86SAlexey Kardashevskiy {
648e633bc86SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
649e633bc86SAlexey Kardashevskiy struct iommu_table_group *table_group;
650e633bc86SAlexey Kardashevskiy struct iommu_table *tbl = NULL;
651e633bc86SAlexey Kardashevskiy long ret, num;
652e633bc86SAlexey Kardashevskiy
653e633bc86SAlexey Kardashevskiy num = tce_iommu_find_free_table(container);
654e633bc86SAlexey Kardashevskiy if (num < 0)
655e633bc86SAlexey Kardashevskiy return num;
656e633bc86SAlexey Kardashevskiy
657e633bc86SAlexey Kardashevskiy /* Get the first group for ops::create_table */
658e633bc86SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list,
659e633bc86SAlexey Kardashevskiy struct tce_iommu_group, next);
660e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
661e633bc86SAlexey Kardashevskiy if (!table_group)
662e633bc86SAlexey Kardashevskiy return -EFAULT;
663e633bc86SAlexey Kardashevskiy
664e633bc86SAlexey Kardashevskiy if (!(table_group->pgsizes & (1ULL << page_shift)))
665e633bc86SAlexey Kardashevskiy return -EINVAL;
666e633bc86SAlexey Kardashevskiy
667e633bc86SAlexey Kardashevskiy if (!table_group->ops->set_window || !table_group->ops->unset_window ||
668e633bc86SAlexey Kardashevskiy !table_group->ops->get_table_size ||
669e633bc86SAlexey Kardashevskiy !table_group->ops->create_table)
670e633bc86SAlexey Kardashevskiy return -EPERM;
671e633bc86SAlexey Kardashevskiy
672e633bc86SAlexey Kardashevskiy /* Create TCE table */
673e633bc86SAlexey Kardashevskiy ret = tce_iommu_create_table(container, table_group, num,
674e633bc86SAlexey Kardashevskiy page_shift, window_size, levels, &tbl);
675e633bc86SAlexey Kardashevskiy if (ret)
676e633bc86SAlexey Kardashevskiy return ret;
677e633bc86SAlexey Kardashevskiy
678e633bc86SAlexey Kardashevskiy BUG_ON(!tbl->it_ops->free);
679e633bc86SAlexey Kardashevskiy
680e633bc86SAlexey Kardashevskiy /*
681e633bc86SAlexey Kardashevskiy * Program the table to every group.
682e633bc86SAlexey Kardashevskiy * Groups have been tested for compatibility at the attach time.
683e633bc86SAlexey Kardashevskiy */
684e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) {
685e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
686e633bc86SAlexey Kardashevskiy
687e633bc86SAlexey Kardashevskiy ret = table_group->ops->set_window(table_group, num, tbl);
688e633bc86SAlexey Kardashevskiy if (ret)
689e633bc86SAlexey Kardashevskiy goto unset_exit;
690e633bc86SAlexey Kardashevskiy }
691e633bc86SAlexey Kardashevskiy
692e633bc86SAlexey Kardashevskiy container->tables[num] = tbl;
693e633bc86SAlexey Kardashevskiy
694e633bc86SAlexey Kardashevskiy /* Return start address assigned by platform in create_table() */
695e633bc86SAlexey Kardashevskiy *start_addr = tbl->it_offset << tbl->it_page_shift;
696e633bc86SAlexey Kardashevskiy
697e633bc86SAlexey Kardashevskiy return 0;
698e633bc86SAlexey Kardashevskiy
699e633bc86SAlexey Kardashevskiy unset_exit:
700e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) {
701e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
702e633bc86SAlexey Kardashevskiy table_group->ops->unset_window(table_group, num);
703e633bc86SAlexey Kardashevskiy }
704bc82d122SAlexey Kardashevskiy tce_iommu_free_table(container, tbl);
705e633bc86SAlexey Kardashevskiy
706e633bc86SAlexey Kardashevskiy return ret;
707e633bc86SAlexey Kardashevskiy }
708e633bc86SAlexey Kardashevskiy
tce_iommu_remove_window(struct tce_container * container,__u64 start_addr)709e633bc86SAlexey Kardashevskiy static long tce_iommu_remove_window(struct tce_container *container,
710e633bc86SAlexey Kardashevskiy __u64 start_addr)
711e633bc86SAlexey Kardashevskiy {
712e633bc86SAlexey Kardashevskiy struct iommu_table_group *table_group = NULL;
713e633bc86SAlexey Kardashevskiy struct iommu_table *tbl;
714e633bc86SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
715e633bc86SAlexey Kardashevskiy int num;
716e633bc86SAlexey Kardashevskiy
717e633bc86SAlexey Kardashevskiy num = tce_iommu_find_table(container, start_addr, &tbl);
718e633bc86SAlexey Kardashevskiy if (num < 0)
719e633bc86SAlexey Kardashevskiy return -EINVAL;
720e633bc86SAlexey Kardashevskiy
721e633bc86SAlexey Kardashevskiy BUG_ON(!tbl->it_size);
722e633bc86SAlexey Kardashevskiy
723e633bc86SAlexey Kardashevskiy /* Detach groups from IOMMUs */
724e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) {
725e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
726e633bc86SAlexey Kardashevskiy
727e633bc86SAlexey Kardashevskiy /*
728e633bc86SAlexey Kardashevskiy * SPAPR TCE IOMMU exposes the default DMA window to
729e633bc86SAlexey Kardashevskiy * the guest via dma32_window_start/size of
730e633bc86SAlexey Kardashevskiy * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
731e633bc86SAlexey Kardashevskiy * the userspace to remove this window, some do not so
732e633bc86SAlexey Kardashevskiy * here we check for the platform capability.
733e633bc86SAlexey Kardashevskiy */
734e633bc86SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->unset_window)
735e633bc86SAlexey Kardashevskiy return -EPERM;
736e633bc86SAlexey Kardashevskiy
737e633bc86SAlexey Kardashevskiy table_group->ops->unset_window(table_group, num);
738e633bc86SAlexey Kardashevskiy }
739e633bc86SAlexey Kardashevskiy
740e633bc86SAlexey Kardashevskiy /* Free table */
741e633bc86SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
742bc82d122SAlexey Kardashevskiy tce_iommu_free_table(container, tbl);
743e633bc86SAlexey Kardashevskiy container->tables[num] = NULL;
744e633bc86SAlexey Kardashevskiy
745e633bc86SAlexey Kardashevskiy return 0;
746e633bc86SAlexey Kardashevskiy }
747e633bc86SAlexey Kardashevskiy
tce_iommu_create_default_window(struct tce_container * container)7486f01cc69SAlexey Kardashevskiy static long tce_iommu_create_default_window(struct tce_container *container)
7496f01cc69SAlexey Kardashevskiy {
7506f01cc69SAlexey Kardashevskiy long ret;
7516f01cc69SAlexey Kardashevskiy __u64 start_addr = 0;
7526f01cc69SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
7536f01cc69SAlexey Kardashevskiy struct iommu_table_group *table_group;
7546f01cc69SAlexey Kardashevskiy
755d9c72894SAlexey Kardashevskiy if (!container->def_window_pending)
756d9c72894SAlexey Kardashevskiy return 0;
757d9c72894SAlexey Kardashevskiy
7586f01cc69SAlexey Kardashevskiy if (!tce_groups_attached(container))
7596f01cc69SAlexey Kardashevskiy return -ENODEV;
7606f01cc69SAlexey Kardashevskiy
7616f01cc69SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list,
7626f01cc69SAlexey Kardashevskiy struct tce_iommu_group, next);
7636f01cc69SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
7646f01cc69SAlexey Kardashevskiy if (!table_group)
7656f01cc69SAlexey Kardashevskiy return -ENODEV;
7666f01cc69SAlexey Kardashevskiy
7676f01cc69SAlexey Kardashevskiy ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
7686f01cc69SAlexey Kardashevskiy table_group->tce32_size, 1, &start_addr);
7696f01cc69SAlexey Kardashevskiy WARN_ON_ONCE(!ret && start_addr);
7706f01cc69SAlexey Kardashevskiy
771d9c72894SAlexey Kardashevskiy if (!ret)
772d9c72894SAlexey Kardashevskiy container->def_window_pending = false;
773d9c72894SAlexey Kardashevskiy
7746f01cc69SAlexey Kardashevskiy return ret;
7756f01cc69SAlexey Kardashevskiy }
7766f01cc69SAlexey Kardashevskiy
vfio_spapr_ioctl_eeh_pe_op(struct iommu_group * group,unsigned long arg)777e276e258SJason Gunthorpe static long vfio_spapr_ioctl_eeh_pe_op(struct iommu_group *group,
778e276e258SJason Gunthorpe unsigned long arg)
779e276e258SJason Gunthorpe {
780e276e258SJason Gunthorpe struct eeh_pe *pe;
781e276e258SJason Gunthorpe struct vfio_eeh_pe_op op;
782e276e258SJason Gunthorpe unsigned long minsz;
783e276e258SJason Gunthorpe
784e276e258SJason Gunthorpe pe = eeh_iommu_group_to_pe(group);
785e276e258SJason Gunthorpe if (!pe)
786e276e258SJason Gunthorpe return -ENODEV;
787e276e258SJason Gunthorpe
788e276e258SJason Gunthorpe minsz = offsetofend(struct vfio_eeh_pe_op, op);
789e276e258SJason Gunthorpe if (copy_from_user(&op, (void __user *)arg, minsz))
790e276e258SJason Gunthorpe return -EFAULT;
791e276e258SJason Gunthorpe if (op.argsz < minsz || op.flags)
792e276e258SJason Gunthorpe return -EINVAL;
793e276e258SJason Gunthorpe
794e276e258SJason Gunthorpe switch (op.op) {
795e276e258SJason Gunthorpe case VFIO_EEH_PE_DISABLE:
796e276e258SJason Gunthorpe return eeh_pe_set_option(pe, EEH_OPT_DISABLE);
797e276e258SJason Gunthorpe case VFIO_EEH_PE_ENABLE:
798e276e258SJason Gunthorpe return eeh_pe_set_option(pe, EEH_OPT_ENABLE);
799e276e258SJason Gunthorpe case VFIO_EEH_PE_UNFREEZE_IO:
800e276e258SJason Gunthorpe return eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
801e276e258SJason Gunthorpe case VFIO_EEH_PE_UNFREEZE_DMA:
802e276e258SJason Gunthorpe return eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
803e276e258SJason Gunthorpe case VFIO_EEH_PE_GET_STATE:
804e276e258SJason Gunthorpe return eeh_pe_get_state(pe);
805e276e258SJason Gunthorpe break;
806e276e258SJason Gunthorpe case VFIO_EEH_PE_RESET_DEACTIVATE:
807e276e258SJason Gunthorpe return eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, true);
808e276e258SJason Gunthorpe case VFIO_EEH_PE_RESET_HOT:
809e276e258SJason Gunthorpe return eeh_pe_reset(pe, EEH_RESET_HOT, true);
810e276e258SJason Gunthorpe case VFIO_EEH_PE_RESET_FUNDAMENTAL:
811e276e258SJason Gunthorpe return eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL, true);
812e276e258SJason Gunthorpe case VFIO_EEH_PE_CONFIGURE:
813e276e258SJason Gunthorpe return eeh_pe_configure(pe);
814e276e258SJason Gunthorpe case VFIO_EEH_PE_INJECT_ERR:
815e276e258SJason Gunthorpe minsz = offsetofend(struct vfio_eeh_pe_op, err.mask);
816e276e258SJason Gunthorpe if (op.argsz < minsz)
817e276e258SJason Gunthorpe return -EINVAL;
818e276e258SJason Gunthorpe if (copy_from_user(&op, (void __user *)arg, minsz))
819e276e258SJason Gunthorpe return -EFAULT;
820e276e258SJason Gunthorpe
821e276e258SJason Gunthorpe return eeh_pe_inject_err(pe, op.err.type, op.err.func,
822e276e258SJason Gunthorpe op.err.addr, op.err.mask);
823e276e258SJason Gunthorpe default:
824e276e258SJason Gunthorpe return -EINVAL;
825e276e258SJason Gunthorpe }
826e276e258SJason Gunthorpe }
827e276e258SJason Gunthorpe
tce_iommu_ioctl(void * iommu_data,unsigned int cmd,unsigned long arg)8285ffd229cSAlexey Kardashevskiy static long tce_iommu_ioctl(void *iommu_data,
8295ffd229cSAlexey Kardashevskiy unsigned int cmd, unsigned long arg)
8305ffd229cSAlexey Kardashevskiy {
8315ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data;
832e633bc86SAlexey Kardashevskiy unsigned long minsz, ddwsz;
8335ffd229cSAlexey Kardashevskiy long ret;
8345ffd229cSAlexey Kardashevskiy
8355ffd229cSAlexey Kardashevskiy switch (cmd) {
8365ffd229cSAlexey Kardashevskiy case VFIO_CHECK_EXTENSION:
8371b69be5eSGavin Shan switch (arg) {
8381b69be5eSGavin Shan case VFIO_SPAPR_TCE_IOMMU:
8392157e7b8SAlexey Kardashevskiy case VFIO_SPAPR_TCE_v2_IOMMU:
840e5c38a20SJason Gunthorpe return 1;
841e5c38a20SJason Gunthorpe case VFIO_EEH:
842e5c38a20SJason Gunthorpe return eeh_enabled();
8431b69be5eSGavin Shan default:
844e5c38a20SJason Gunthorpe return 0;
8451b69be5eSGavin Shan }
846bc82d122SAlexey Kardashevskiy }
8475ffd229cSAlexey Kardashevskiy
848bc82d122SAlexey Kardashevskiy /*
849bc82d122SAlexey Kardashevskiy * Sanity check to prevent one userspace from manipulating
850bc82d122SAlexey Kardashevskiy * another userspace mm.
851bc82d122SAlexey Kardashevskiy */
852bc82d122SAlexey Kardashevskiy BUG_ON(!container);
853bc82d122SAlexey Kardashevskiy if (container->mm && container->mm != current->mm)
854bc82d122SAlexey Kardashevskiy return -EPERM;
855bc82d122SAlexey Kardashevskiy
856bc82d122SAlexey Kardashevskiy switch (cmd) {
8575ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
8585ffd229cSAlexey Kardashevskiy struct vfio_iommu_spapr_tce_info info;
8592157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
8600eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group;
8615ffd229cSAlexey Kardashevskiy
8622157e7b8SAlexey Kardashevskiy if (!tce_groups_attached(container))
8630eaf4defSAlexey Kardashevskiy return -ENXIO;
8640eaf4defSAlexey Kardashevskiy
8652157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list,
8662157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next);
8672157e7b8SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp);
8680eaf4defSAlexey Kardashevskiy
8694793d65dSAlexey Kardashevskiy if (!table_group)
8705ffd229cSAlexey Kardashevskiy return -ENXIO;
8715ffd229cSAlexey Kardashevskiy
8725ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
8735ffd229cSAlexey Kardashevskiy dma32_window_size);
8745ffd229cSAlexey Kardashevskiy
8755ffd229cSAlexey Kardashevskiy if (copy_from_user(&info, (void __user *)arg, minsz))
8765ffd229cSAlexey Kardashevskiy return -EFAULT;
8775ffd229cSAlexey Kardashevskiy
8785ffd229cSAlexey Kardashevskiy if (info.argsz < minsz)
8795ffd229cSAlexey Kardashevskiy return -EINVAL;
8805ffd229cSAlexey Kardashevskiy
8814793d65dSAlexey Kardashevskiy info.dma32_window_start = table_group->tce32_start;
8824793d65dSAlexey Kardashevskiy info.dma32_window_size = table_group->tce32_size;
8835ffd229cSAlexey Kardashevskiy info.flags = 0;
884e633bc86SAlexey Kardashevskiy memset(&info.ddw, 0, sizeof(info.ddw));
885e633bc86SAlexey Kardashevskiy
886e633bc86SAlexey Kardashevskiy if (table_group->max_dynamic_windows_supported &&
887e633bc86SAlexey Kardashevskiy container->v2) {
888e633bc86SAlexey Kardashevskiy info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
889e633bc86SAlexey Kardashevskiy info.ddw.pgsizes = table_group->pgsizes;
890e633bc86SAlexey Kardashevskiy info.ddw.max_dynamic_windows_supported =
891e633bc86SAlexey Kardashevskiy table_group->max_dynamic_windows_supported;
892e633bc86SAlexey Kardashevskiy info.ddw.levels = table_group->max_levels;
893e633bc86SAlexey Kardashevskiy }
894e633bc86SAlexey Kardashevskiy
895e633bc86SAlexey Kardashevskiy ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
896e633bc86SAlexey Kardashevskiy
897e633bc86SAlexey Kardashevskiy if (info.argsz >= ddwsz)
898e633bc86SAlexey Kardashevskiy minsz = ddwsz;
8995ffd229cSAlexey Kardashevskiy
9005ffd229cSAlexey Kardashevskiy if (copy_to_user((void __user *)arg, &info, minsz))
9015ffd229cSAlexey Kardashevskiy return -EFAULT;
9025ffd229cSAlexey Kardashevskiy
9035ffd229cSAlexey Kardashevskiy return 0;
9045ffd229cSAlexey Kardashevskiy }
9055ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_MAP_DMA: {
9065ffd229cSAlexey Kardashevskiy struct vfio_iommu_type1_dma_map param;
9070eaf4defSAlexey Kardashevskiy struct iommu_table *tbl = NULL;
9080eaf4defSAlexey Kardashevskiy long num;
90905c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction;
9105ffd229cSAlexey Kardashevskiy
9113c56e822SAlexey Kardashevskiy if (!container->enabled)
9123c56e822SAlexey Kardashevskiy return -EPERM;
9133c56e822SAlexey Kardashevskiy
9145ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
9155ffd229cSAlexey Kardashevskiy
9165ffd229cSAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz))
9175ffd229cSAlexey Kardashevskiy return -EFAULT;
9185ffd229cSAlexey Kardashevskiy
9195ffd229cSAlexey Kardashevskiy if (param.argsz < minsz)
9205ffd229cSAlexey Kardashevskiy return -EINVAL;
9215ffd229cSAlexey Kardashevskiy
9225ffd229cSAlexey Kardashevskiy if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
9235ffd229cSAlexey Kardashevskiy VFIO_DMA_MAP_FLAG_WRITE))
9245ffd229cSAlexey Kardashevskiy return -EINVAL;
9255ffd229cSAlexey Kardashevskiy
926d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container);
927d9c72894SAlexey Kardashevskiy if (ret)
928d9c72894SAlexey Kardashevskiy return ret;
929d9c72894SAlexey Kardashevskiy
9300eaf4defSAlexey Kardashevskiy num = tce_iommu_find_table(container, param.iova, &tbl);
9310eaf4defSAlexey Kardashevskiy if (num < 0)
9320eaf4defSAlexey Kardashevskiy return -ENXIO;
9330eaf4defSAlexey Kardashevskiy
93400663d4eSAlexey Kardashevskiy if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
93500663d4eSAlexey Kardashevskiy (param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
9365ffd229cSAlexey Kardashevskiy return -EINVAL;
9375ffd229cSAlexey Kardashevskiy
9385ffd229cSAlexey Kardashevskiy /* iova is checked by the IOMMU API */
93905c6cfb9SAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
9405ffd229cSAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
94105c6cfb9SAlexey Kardashevskiy direction = DMA_BIDIRECTIONAL;
94205c6cfb9SAlexey Kardashevskiy else
94305c6cfb9SAlexey Kardashevskiy direction = DMA_TO_DEVICE;
94405c6cfb9SAlexey Kardashevskiy } else {
94505c6cfb9SAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
94605c6cfb9SAlexey Kardashevskiy direction = DMA_FROM_DEVICE;
94705c6cfb9SAlexey Kardashevskiy else
94805c6cfb9SAlexey Kardashevskiy return -EINVAL;
94905c6cfb9SAlexey Kardashevskiy }
9505ffd229cSAlexey Kardashevskiy
95105c6cfb9SAlexey Kardashevskiy ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
9525ffd229cSAlexey Kardashevskiy if (ret)
9535ffd229cSAlexey Kardashevskiy return ret;
9545ffd229cSAlexey Kardashevskiy
9552157e7b8SAlexey Kardashevskiy if (container->v2)
9562157e7b8SAlexey Kardashevskiy ret = tce_iommu_build_v2(container, tbl,
9572157e7b8SAlexey Kardashevskiy param.iova >> tbl->it_page_shift,
9582157e7b8SAlexey Kardashevskiy param.vaddr,
9592157e7b8SAlexey Kardashevskiy param.size >> tbl->it_page_shift,
9602157e7b8SAlexey Kardashevskiy direction);
9612157e7b8SAlexey Kardashevskiy else
9629b14a1ffSAlexey Kardashevskiy ret = tce_iommu_build(container, tbl,
96300663d4eSAlexey Kardashevskiy param.iova >> tbl->it_page_shift,
96405c6cfb9SAlexey Kardashevskiy param.vaddr,
96505c6cfb9SAlexey Kardashevskiy param.size >> tbl->it_page_shift,
96605c6cfb9SAlexey Kardashevskiy direction);
9675ffd229cSAlexey Kardashevskiy
9685ffd229cSAlexey Kardashevskiy iommu_flush_tce(tbl);
9695ffd229cSAlexey Kardashevskiy
9705ffd229cSAlexey Kardashevskiy return ret;
9715ffd229cSAlexey Kardashevskiy }
9725ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_UNMAP_DMA: {
9735ffd229cSAlexey Kardashevskiy struct vfio_iommu_type1_dma_unmap param;
9740eaf4defSAlexey Kardashevskiy struct iommu_table *tbl = NULL;
9750eaf4defSAlexey Kardashevskiy long num;
9765ffd229cSAlexey Kardashevskiy
9773c56e822SAlexey Kardashevskiy if (!container->enabled)
9783c56e822SAlexey Kardashevskiy return -EPERM;
9793c56e822SAlexey Kardashevskiy
9805ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
9815ffd229cSAlexey Kardashevskiy size);
9825ffd229cSAlexey Kardashevskiy
9835ffd229cSAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz))
9845ffd229cSAlexey Kardashevskiy return -EFAULT;
9855ffd229cSAlexey Kardashevskiy
9865ffd229cSAlexey Kardashevskiy if (param.argsz < minsz)
9875ffd229cSAlexey Kardashevskiy return -EINVAL;
9885ffd229cSAlexey Kardashevskiy
9895ffd229cSAlexey Kardashevskiy /* No flag is supported now */
9905ffd229cSAlexey Kardashevskiy if (param.flags)
9915ffd229cSAlexey Kardashevskiy return -EINVAL;
9925ffd229cSAlexey Kardashevskiy
993d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container);
994d9c72894SAlexey Kardashevskiy if (ret)
995d9c72894SAlexey Kardashevskiy return ret;
996d9c72894SAlexey Kardashevskiy
9970eaf4defSAlexey Kardashevskiy num = tce_iommu_find_table(container, param.iova, &tbl);
9980eaf4defSAlexey Kardashevskiy if (num < 0)
9990eaf4defSAlexey Kardashevskiy return -ENXIO;
10000eaf4defSAlexey Kardashevskiy
100100663d4eSAlexey Kardashevskiy if (param.size & ~IOMMU_PAGE_MASK(tbl))
10025ffd229cSAlexey Kardashevskiy return -EINVAL;
10035ffd229cSAlexey Kardashevskiy
10045ffd229cSAlexey Kardashevskiy ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
100500663d4eSAlexey Kardashevskiy param.size >> tbl->it_page_shift);
10065ffd229cSAlexey Kardashevskiy if (ret)
10075ffd229cSAlexey Kardashevskiy return ret;
10085ffd229cSAlexey Kardashevskiy
10099b14a1ffSAlexey Kardashevskiy ret = tce_iommu_clear(container, tbl,
101000663d4eSAlexey Kardashevskiy param.iova >> tbl->it_page_shift,
101100663d4eSAlexey Kardashevskiy param.size >> tbl->it_page_shift);
10125ffd229cSAlexey Kardashevskiy iommu_flush_tce(tbl);
10135ffd229cSAlexey Kardashevskiy
10145ffd229cSAlexey Kardashevskiy return ret;
10155ffd229cSAlexey Kardashevskiy }
10162157e7b8SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
10172157e7b8SAlexey Kardashevskiy struct vfio_iommu_spapr_register_memory param;
10182157e7b8SAlexey Kardashevskiy
10192157e7b8SAlexey Kardashevskiy if (!container->v2)
10202157e7b8SAlexey Kardashevskiy break;
10212157e7b8SAlexey Kardashevskiy
10222157e7b8SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
10232157e7b8SAlexey Kardashevskiy size);
10242157e7b8SAlexey Kardashevskiy
1025bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container);
1026bc82d122SAlexey Kardashevskiy if (ret)
1027bc82d122SAlexey Kardashevskiy return ret;
1028bc82d122SAlexey Kardashevskiy
10292157e7b8SAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz))
10302157e7b8SAlexey Kardashevskiy return -EFAULT;
10312157e7b8SAlexey Kardashevskiy
10322157e7b8SAlexey Kardashevskiy if (param.argsz < minsz)
10332157e7b8SAlexey Kardashevskiy return -EINVAL;
10342157e7b8SAlexey Kardashevskiy
10352157e7b8SAlexey Kardashevskiy /* No flag is supported now */
10362157e7b8SAlexey Kardashevskiy if (param.flags)
10372157e7b8SAlexey Kardashevskiy return -EINVAL;
10382157e7b8SAlexey Kardashevskiy
10392157e7b8SAlexey Kardashevskiy mutex_lock(&container->lock);
10402157e7b8SAlexey Kardashevskiy ret = tce_iommu_register_pages(container, param.vaddr,
10412157e7b8SAlexey Kardashevskiy param.size);
10422157e7b8SAlexey Kardashevskiy mutex_unlock(&container->lock);
10432157e7b8SAlexey Kardashevskiy
10442157e7b8SAlexey Kardashevskiy return ret;
10452157e7b8SAlexey Kardashevskiy }
10462157e7b8SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
10472157e7b8SAlexey Kardashevskiy struct vfio_iommu_spapr_register_memory param;
10482157e7b8SAlexey Kardashevskiy
10492157e7b8SAlexey Kardashevskiy if (!container->v2)
10502157e7b8SAlexey Kardashevskiy break;
10512157e7b8SAlexey Kardashevskiy
1052bc82d122SAlexey Kardashevskiy if (!container->mm)
1053bc82d122SAlexey Kardashevskiy return -EPERM;
1054bc82d122SAlexey Kardashevskiy
10552157e7b8SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
10562157e7b8SAlexey Kardashevskiy size);
10572157e7b8SAlexey Kardashevskiy
10582157e7b8SAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz))
10592157e7b8SAlexey Kardashevskiy return -EFAULT;
10602157e7b8SAlexey Kardashevskiy
10612157e7b8SAlexey Kardashevskiy if (param.argsz < minsz)
10622157e7b8SAlexey Kardashevskiy return -EINVAL;
10632157e7b8SAlexey Kardashevskiy
10642157e7b8SAlexey Kardashevskiy /* No flag is supported now */
10652157e7b8SAlexey Kardashevskiy if (param.flags)
10662157e7b8SAlexey Kardashevskiy return -EINVAL;
10672157e7b8SAlexey Kardashevskiy
10682157e7b8SAlexey Kardashevskiy mutex_lock(&container->lock);
10692157e7b8SAlexey Kardashevskiy ret = tce_iommu_unregister_pages(container, param.vaddr,
10702157e7b8SAlexey Kardashevskiy param.size);
10712157e7b8SAlexey Kardashevskiy mutex_unlock(&container->lock);
10722157e7b8SAlexey Kardashevskiy
10732157e7b8SAlexey Kardashevskiy return ret;
10742157e7b8SAlexey Kardashevskiy }
10755ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_ENABLE:
10762157e7b8SAlexey Kardashevskiy if (container->v2)
10772157e7b8SAlexey Kardashevskiy break;
10782157e7b8SAlexey Kardashevskiy
10795ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock);
10805ffd229cSAlexey Kardashevskiy ret = tce_iommu_enable(container);
10815ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock);
10825ffd229cSAlexey Kardashevskiy return ret;
10835ffd229cSAlexey Kardashevskiy
10845ffd229cSAlexey Kardashevskiy
10855ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_DISABLE:
10862157e7b8SAlexey Kardashevskiy if (container->v2)
10872157e7b8SAlexey Kardashevskiy break;
10882157e7b8SAlexey Kardashevskiy
10895ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock);
10905ffd229cSAlexey Kardashevskiy tce_iommu_disable(container);
10915ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock);
10925ffd229cSAlexey Kardashevskiy return 0;
10931b69be5eSGavin Shan
10942157e7b8SAlexey Kardashevskiy case VFIO_EEH_PE_OP: {
10952157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
10962157e7b8SAlexey Kardashevskiy
10972157e7b8SAlexey Kardashevskiy ret = 0;
10982157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) {
1099e276e258SJason Gunthorpe ret = vfio_spapr_ioctl_eeh_pe_op(tcegrp->grp, arg);
11002157e7b8SAlexey Kardashevskiy if (ret)
11012157e7b8SAlexey Kardashevskiy return ret;
11022157e7b8SAlexey Kardashevskiy }
11032157e7b8SAlexey Kardashevskiy return ret;
11042157e7b8SAlexey Kardashevskiy }
11052157e7b8SAlexey Kardashevskiy
1106e633bc86SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1107e633bc86SAlexey Kardashevskiy struct vfio_iommu_spapr_tce_create create;
1108e633bc86SAlexey Kardashevskiy
1109e633bc86SAlexey Kardashevskiy if (!container->v2)
1110e633bc86SAlexey Kardashevskiy break;
1111e633bc86SAlexey Kardashevskiy
1112bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container);
1113bc82d122SAlexey Kardashevskiy if (ret)
1114bc82d122SAlexey Kardashevskiy return ret;
1115bc82d122SAlexey Kardashevskiy
1116e633bc86SAlexey Kardashevskiy if (!tce_groups_attached(container))
1117e633bc86SAlexey Kardashevskiy return -ENXIO;
1118e633bc86SAlexey Kardashevskiy
1119e633bc86SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1120e633bc86SAlexey Kardashevskiy start_addr);
1121e633bc86SAlexey Kardashevskiy
1122e633bc86SAlexey Kardashevskiy if (copy_from_user(&create, (void __user *)arg, minsz))
1123e633bc86SAlexey Kardashevskiy return -EFAULT;
1124e633bc86SAlexey Kardashevskiy
1125e633bc86SAlexey Kardashevskiy if (create.argsz < minsz)
1126e633bc86SAlexey Kardashevskiy return -EINVAL;
1127e633bc86SAlexey Kardashevskiy
1128e633bc86SAlexey Kardashevskiy if (create.flags)
1129e633bc86SAlexey Kardashevskiy return -EINVAL;
1130e633bc86SAlexey Kardashevskiy
1131e633bc86SAlexey Kardashevskiy mutex_lock(&container->lock);
1132e633bc86SAlexey Kardashevskiy
1133d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container);
11342da64d20SAlexey Kardashevskiy if (!ret)
11352da64d20SAlexey Kardashevskiy ret = tce_iommu_create_window(container,
11362da64d20SAlexey Kardashevskiy create.page_shift,
1137e633bc86SAlexey Kardashevskiy create.window_size, create.levels,
1138e633bc86SAlexey Kardashevskiy &create.start_addr);
1139e633bc86SAlexey Kardashevskiy
1140e633bc86SAlexey Kardashevskiy mutex_unlock(&container->lock);
1141e633bc86SAlexey Kardashevskiy
1142e633bc86SAlexey Kardashevskiy if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1143e633bc86SAlexey Kardashevskiy ret = -EFAULT;
1144e633bc86SAlexey Kardashevskiy
1145e633bc86SAlexey Kardashevskiy return ret;
1146e633bc86SAlexey Kardashevskiy }
1147e633bc86SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1148e633bc86SAlexey Kardashevskiy struct vfio_iommu_spapr_tce_remove remove;
1149e633bc86SAlexey Kardashevskiy
1150e633bc86SAlexey Kardashevskiy if (!container->v2)
1151e633bc86SAlexey Kardashevskiy break;
1152e633bc86SAlexey Kardashevskiy
1153bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container);
1154bc82d122SAlexey Kardashevskiy if (ret)
1155bc82d122SAlexey Kardashevskiy return ret;
1156bc82d122SAlexey Kardashevskiy
1157e633bc86SAlexey Kardashevskiy if (!tce_groups_attached(container))
1158e633bc86SAlexey Kardashevskiy return -ENXIO;
1159e633bc86SAlexey Kardashevskiy
1160e633bc86SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1161e633bc86SAlexey Kardashevskiy start_addr);
1162e633bc86SAlexey Kardashevskiy
1163e633bc86SAlexey Kardashevskiy if (copy_from_user(&remove, (void __user *)arg, minsz))
1164e633bc86SAlexey Kardashevskiy return -EFAULT;
1165e633bc86SAlexey Kardashevskiy
1166e633bc86SAlexey Kardashevskiy if (remove.argsz < minsz)
1167e633bc86SAlexey Kardashevskiy return -EINVAL;
1168e633bc86SAlexey Kardashevskiy
1169e633bc86SAlexey Kardashevskiy if (remove.flags)
1170e633bc86SAlexey Kardashevskiy return -EINVAL;
1171e633bc86SAlexey Kardashevskiy
1172d9c72894SAlexey Kardashevskiy if (container->def_window_pending && !remove.start_addr) {
1173d9c72894SAlexey Kardashevskiy container->def_window_pending = false;
1174d9c72894SAlexey Kardashevskiy return 0;
1175d9c72894SAlexey Kardashevskiy }
1176d9c72894SAlexey Kardashevskiy
1177e633bc86SAlexey Kardashevskiy mutex_lock(&container->lock);
1178e633bc86SAlexey Kardashevskiy
1179e633bc86SAlexey Kardashevskiy ret = tce_iommu_remove_window(container, remove.start_addr);
1180e633bc86SAlexey Kardashevskiy
1181e633bc86SAlexey Kardashevskiy mutex_unlock(&container->lock);
1182e633bc86SAlexey Kardashevskiy
1183e633bc86SAlexey Kardashevskiy return ret;
1184e633bc86SAlexey Kardashevskiy }
11855ffd229cSAlexey Kardashevskiy }
11865ffd229cSAlexey Kardashevskiy
11875ffd229cSAlexey Kardashevskiy return -ENOTTY;
11885ffd229cSAlexey Kardashevskiy }
11895ffd229cSAlexey Kardashevskiy
tce_iommu_release_ownership(struct tce_container * container,struct iommu_table_group * table_group)1190f87a8864SAlexey Kardashevskiy static void tce_iommu_release_ownership(struct tce_container *container,
1191f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group)
1192f87a8864SAlexey Kardashevskiy {
119346d3e1e1SAlexey Kardashevskiy long i;
119446d3e1e1SAlexey Kardashevskiy
119546d3e1e1SAlexey Kardashevskiy if (!table_group->ops->unset_window) {
119646d3e1e1SAlexey Kardashevskiy WARN_ON_ONCE(1);
119746d3e1e1SAlexey Kardashevskiy return;
119846d3e1e1SAlexey Kardashevskiy }
119946d3e1e1SAlexey Kardashevskiy
12002157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1201a3906855SAlexey Kardashevskiy if (container->tables[i])
120246d3e1e1SAlexey Kardashevskiy table_group->ops->unset_window(table_group, i);
1203f87a8864SAlexey Kardashevskiy }
1204f87a8864SAlexey Kardashevskiy
tce_iommu_take_ownership(struct tce_container * container,struct iommu_table_group * table_group)1205*9d67c943SAlexey Kardashevskiy static long tce_iommu_take_ownership(struct tce_container *container,
1206f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group)
1207f87a8864SAlexey Kardashevskiy {
1208930a42deSAlexey Kardashevskiy long i, ret = 0;
1209930a42deSAlexey Kardashevskiy
1210930a42deSAlexey Kardashevskiy /* Set all windows to the new group */
1211930a42deSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1212930a42deSAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i];
1213930a42deSAlexey Kardashevskiy
1214930a42deSAlexey Kardashevskiy if (!tbl)
1215930a42deSAlexey Kardashevskiy continue;
1216930a42deSAlexey Kardashevskiy
1217930a42deSAlexey Kardashevskiy ret = table_group->ops->set_window(table_group, i, tbl);
1218930a42deSAlexey Kardashevskiy if (ret)
1219930a42deSAlexey Kardashevskiy goto release_exit;
1220930a42deSAlexey Kardashevskiy }
1221930a42deSAlexey Kardashevskiy
12222157e7b8SAlexey Kardashevskiy return 0;
1223930a42deSAlexey Kardashevskiy
1224930a42deSAlexey Kardashevskiy release_exit:
1225930a42deSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1226930a42deSAlexey Kardashevskiy table_group->ops->unset_window(table_group, i);
1227930a42deSAlexey Kardashevskiy
1228930a42deSAlexey Kardashevskiy return ret;
1229f87a8864SAlexey Kardashevskiy }
1230f87a8864SAlexey Kardashevskiy
tce_iommu_attach_group(void * iommu_data,struct iommu_group * iommu_group,enum vfio_group_type type)12315ffd229cSAlexey Kardashevskiy static int tce_iommu_attach_group(void *iommu_data,
1232c3c0fa9dSChristoph Hellwig struct iommu_group *iommu_group, enum vfio_group_type type)
12335ffd229cSAlexey Kardashevskiy {
123478becab9SAlexey Kardashevskiy int ret = 0;
12355ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data;
12360eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group;
12372157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp = NULL;
12385ffd229cSAlexey Kardashevskiy
123989863904SChristoph Hellwig if (type == VFIO_EMULATED_IOMMU)
124089863904SChristoph Hellwig return -EINVAL;
124189863904SChristoph Hellwig
12425ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock);
12435ffd229cSAlexey Kardashevskiy
12445ffd229cSAlexey Kardashevskiy /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
12455ffd229cSAlexey Kardashevskiy iommu_group_id(iommu_group), iommu_group); */
12460eaf4defSAlexey Kardashevskiy table_group = iommu_group_get_iommudata(iommu_group);
1247bd00fdf1SGreg Kurz if (!table_group) {
1248bd00fdf1SGreg Kurz ret = -ENODEV;
1249bd00fdf1SGreg Kurz goto unlock_exit;
1250bd00fdf1SGreg Kurz }
12512157e7b8SAlexey Kardashevskiy
1252*9d67c943SAlexey Kardashevskiy /* v2 requires full support of dynamic DMA windows */
1253*9d67c943SAlexey Kardashevskiy if (container->v2 && table_group->max_dynamic_windows_supported == 0) {
1254*9d67c943SAlexey Kardashevskiy ret = -EINVAL;
1255*9d67c943SAlexey Kardashevskiy goto unlock_exit;
1256*9d67c943SAlexey Kardashevskiy }
1257*9d67c943SAlexey Kardashevskiy
1258*9d67c943SAlexey Kardashevskiy /* v1 reuses TCE tables and does not share them among PEs */
1259*9d67c943SAlexey Kardashevskiy if (!container->v2 && tce_groups_attached(container)) {
12602157e7b8SAlexey Kardashevskiy ret = -EBUSY;
12612157e7b8SAlexey Kardashevskiy goto unlock_exit;
12622157e7b8SAlexey Kardashevskiy }
12632157e7b8SAlexey Kardashevskiy
12649cb633acSAlexey Kardashevskiy /*
12659cb633acSAlexey Kardashevskiy * Check if new group has the same iommu_table_group_ops
12669cb633acSAlexey Kardashevskiy * (i.e. compatible)
12679cb633acSAlexey Kardashevskiy */
12682157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) {
12692157e7b8SAlexey Kardashevskiy struct iommu_table_group *table_group_tmp;
12702157e7b8SAlexey Kardashevskiy
12712157e7b8SAlexey Kardashevskiy if (tcegrp->grp == iommu_group) {
12722157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: Group %d is already attached\n",
12732157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group));
12742157e7b8SAlexey Kardashevskiy ret = -EBUSY;
12752157e7b8SAlexey Kardashevskiy goto unlock_exit;
12762157e7b8SAlexey Kardashevskiy }
12772157e7b8SAlexey Kardashevskiy table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
127854de285bSAlexey Kardashevskiy if (table_group_tmp->ops->create_table !=
127954de285bSAlexey Kardashevskiy table_group->ops->create_table) {
12802157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
12812157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group),
12822157e7b8SAlexey Kardashevskiy iommu_group_id(tcegrp->grp));
12832157e7b8SAlexey Kardashevskiy ret = -EPERM;
12842157e7b8SAlexey Kardashevskiy goto unlock_exit;
12852157e7b8SAlexey Kardashevskiy }
12862157e7b8SAlexey Kardashevskiy }
12872157e7b8SAlexey Kardashevskiy
12882157e7b8SAlexey Kardashevskiy tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
12892157e7b8SAlexey Kardashevskiy if (!tcegrp) {
12902157e7b8SAlexey Kardashevskiy ret = -ENOMEM;
12910eaf4defSAlexey Kardashevskiy goto unlock_exit;
12920eaf4defSAlexey Kardashevskiy }
12930eaf4defSAlexey Kardashevskiy
1294f87a8864SAlexey Kardashevskiy ret = tce_iommu_take_ownership(container, table_group);
12956f01cc69SAlexey Kardashevskiy if (!tce_groups_attached(container) && !container->tables[0])
1296d9c72894SAlexey Kardashevskiy container->def_window_pending = true;
1297f87a8864SAlexey Kardashevskiy
12982157e7b8SAlexey Kardashevskiy if (!ret) {
12992157e7b8SAlexey Kardashevskiy tcegrp->grp = iommu_group;
13002157e7b8SAlexey Kardashevskiy list_add(&tcegrp->next, &container->group_list);
13012157e7b8SAlexey Kardashevskiy }
13025ffd229cSAlexey Kardashevskiy
13032157e7b8SAlexey Kardashevskiy if (ret && tcegrp)
13042157e7b8SAlexey Kardashevskiy kfree(tcegrp);
13052157e7b8SAlexey Kardashevskiy
130678becab9SAlexey Kardashevskiy unlock_exit:
13075ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock);
13085ffd229cSAlexey Kardashevskiy
13095ffd229cSAlexey Kardashevskiy return ret;
13105ffd229cSAlexey Kardashevskiy }
13115ffd229cSAlexey Kardashevskiy
tce_iommu_detach_group(void * iommu_data,struct iommu_group * iommu_group)13125ffd229cSAlexey Kardashevskiy static void tce_iommu_detach_group(void *iommu_data,
13135ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group)
13145ffd229cSAlexey Kardashevskiy {
13155ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data;
13160eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group;
13172157e7b8SAlexey Kardashevskiy bool found = false;
13182157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp;
13195ffd229cSAlexey Kardashevskiy
13205ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock);
13212157e7b8SAlexey Kardashevskiy
13222157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) {
13232157e7b8SAlexey Kardashevskiy if (tcegrp->grp == iommu_group) {
13242157e7b8SAlexey Kardashevskiy found = true;
13252157e7b8SAlexey Kardashevskiy break;
13262157e7b8SAlexey Kardashevskiy }
13272157e7b8SAlexey Kardashevskiy }
13282157e7b8SAlexey Kardashevskiy
13292157e7b8SAlexey Kardashevskiy if (!found) {
13302157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: detaching unattached group #%u\n",
13312157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group));
133222af4859SAlexey Kardashevskiy goto unlock_exit;
133322af4859SAlexey Kardashevskiy }
133422af4859SAlexey Kardashevskiy
13352157e7b8SAlexey Kardashevskiy list_del(&tcegrp->next);
13362157e7b8SAlexey Kardashevskiy kfree(tcegrp);
13370eaf4defSAlexey Kardashevskiy
13380eaf4defSAlexey Kardashevskiy table_group = iommu_group_get_iommudata(iommu_group);
13390eaf4defSAlexey Kardashevskiy BUG_ON(!table_group);
13400eaf4defSAlexey Kardashevskiy
1341f87a8864SAlexey Kardashevskiy tce_iommu_release_ownership(container, table_group);
134222af4859SAlexey Kardashevskiy
134322af4859SAlexey Kardashevskiy unlock_exit:
13445ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock);
13455ffd229cSAlexey Kardashevskiy }
13465ffd229cSAlexey Kardashevskiy
1347e39dd513SWang Hai static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
13485ffd229cSAlexey Kardashevskiy .name = "iommu-vfio-powerpc",
13495ffd229cSAlexey Kardashevskiy .owner = THIS_MODULE,
13505ffd229cSAlexey Kardashevskiy .open = tce_iommu_open,
13515ffd229cSAlexey Kardashevskiy .release = tce_iommu_release,
13525ffd229cSAlexey Kardashevskiy .ioctl = tce_iommu_ioctl,
13535ffd229cSAlexey Kardashevskiy .attach_group = tce_iommu_attach_group,
13545ffd229cSAlexey Kardashevskiy .detach_group = tce_iommu_detach_group,
13555ffd229cSAlexey Kardashevskiy };
13565ffd229cSAlexey Kardashevskiy
tce_iommu_init(void)13575ffd229cSAlexey Kardashevskiy static int __init tce_iommu_init(void)
13585ffd229cSAlexey Kardashevskiy {
13595ffd229cSAlexey Kardashevskiy return vfio_register_iommu_driver(&tce_iommu_driver_ops);
13605ffd229cSAlexey Kardashevskiy }
13615ffd229cSAlexey Kardashevskiy
tce_iommu_cleanup(void)13625ffd229cSAlexey Kardashevskiy static void __exit tce_iommu_cleanup(void)
13635ffd229cSAlexey Kardashevskiy {
13645ffd229cSAlexey Kardashevskiy vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
13655ffd229cSAlexey Kardashevskiy }
13665ffd229cSAlexey Kardashevskiy
13675ffd229cSAlexey Kardashevskiy module_init(tce_iommu_init);
13685ffd229cSAlexey Kardashevskiy module_exit(tce_iommu_cleanup);
13695ffd229cSAlexey Kardashevskiy
13705ffd229cSAlexey Kardashevskiy MODULE_VERSION(DRIVER_VERSION);
13715ffd229cSAlexey Kardashevskiy MODULE_LICENSE("GPL v2");
13725ffd229cSAlexey Kardashevskiy MODULE_AUTHOR(DRIVER_AUTHOR);
13735ffd229cSAlexey Kardashevskiy MODULE_DESCRIPTION(DRIVER_DESC);
13745ffd229cSAlexey Kardashevskiy
1375