15ffd229cSAlexey Kardashevskiy /* 25ffd229cSAlexey Kardashevskiy * VFIO: IOMMU DMA mapping support for TCE on POWER 35ffd229cSAlexey Kardashevskiy * 45ffd229cSAlexey Kardashevskiy * Copyright (C) 2013 IBM Corp. All rights reserved. 55ffd229cSAlexey Kardashevskiy * Author: Alexey Kardashevskiy <aik@ozlabs.ru> 65ffd229cSAlexey Kardashevskiy * 75ffd229cSAlexey Kardashevskiy * This program is free software; you can redistribute it and/or modify 85ffd229cSAlexey Kardashevskiy * it under the terms of the GNU General Public License version 2 as 95ffd229cSAlexey Kardashevskiy * published by the Free Software Foundation. 105ffd229cSAlexey Kardashevskiy * 115ffd229cSAlexey Kardashevskiy * Derived from original vfio_iommu_type1.c: 125ffd229cSAlexey Kardashevskiy * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 135ffd229cSAlexey Kardashevskiy * Author: Alex Williamson <alex.williamson@redhat.com> 145ffd229cSAlexey Kardashevskiy */ 155ffd229cSAlexey Kardashevskiy 165ffd229cSAlexey Kardashevskiy #include <linux/module.h> 175ffd229cSAlexey Kardashevskiy #include <linux/pci.h> 185ffd229cSAlexey Kardashevskiy #include <linux/slab.h> 195ffd229cSAlexey Kardashevskiy #include <linux/uaccess.h> 205ffd229cSAlexey Kardashevskiy #include <linux/err.h> 215ffd229cSAlexey Kardashevskiy #include <linux/vfio.h> 222157e7b8SAlexey Kardashevskiy #include <linux/vmalloc.h> 235ffd229cSAlexey Kardashevskiy #include <asm/iommu.h> 245ffd229cSAlexey Kardashevskiy #include <asm/tce.h> 252157e7b8SAlexey Kardashevskiy #include <asm/mmu_context.h> 265ffd229cSAlexey Kardashevskiy 275ffd229cSAlexey Kardashevskiy #define DRIVER_VERSION "0.1" 285ffd229cSAlexey Kardashevskiy #define DRIVER_AUTHOR "aik@ozlabs.ru" 295ffd229cSAlexey Kardashevskiy #define DRIVER_DESC "VFIO IOMMU SPAPR TCE" 305ffd229cSAlexey Kardashevskiy 315ffd229cSAlexey Kardashevskiy static void tce_iommu_detach_group(void *iommu_data, 325ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group); 335ffd229cSAlexey Kardashevskiy 342d270df8SAlexey Kardashevskiy static long try_increment_locked_vm(long npages) 352d270df8SAlexey Kardashevskiy { 362d270df8SAlexey Kardashevskiy long ret = 0, locked, lock_limit; 372d270df8SAlexey Kardashevskiy 382d270df8SAlexey Kardashevskiy if (!current || !current->mm) 392d270df8SAlexey Kardashevskiy return -ESRCH; /* process exited */ 402d270df8SAlexey Kardashevskiy 412d270df8SAlexey Kardashevskiy if (!npages) 422d270df8SAlexey Kardashevskiy return 0; 432d270df8SAlexey Kardashevskiy 442d270df8SAlexey Kardashevskiy down_write(¤t->mm->mmap_sem); 452d270df8SAlexey Kardashevskiy locked = current->mm->locked_vm + npages; 462d270df8SAlexey Kardashevskiy lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 472d270df8SAlexey Kardashevskiy if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 482d270df8SAlexey Kardashevskiy ret = -ENOMEM; 492d270df8SAlexey Kardashevskiy else 502d270df8SAlexey Kardashevskiy current->mm->locked_vm += npages; 512d270df8SAlexey Kardashevskiy 522d270df8SAlexey Kardashevskiy pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid, 532d270df8SAlexey Kardashevskiy npages << PAGE_SHIFT, 542d270df8SAlexey Kardashevskiy current->mm->locked_vm << PAGE_SHIFT, 552d270df8SAlexey Kardashevskiy rlimit(RLIMIT_MEMLOCK), 562d270df8SAlexey Kardashevskiy ret ? " - exceeded" : ""); 572d270df8SAlexey Kardashevskiy 582d270df8SAlexey Kardashevskiy up_write(¤t->mm->mmap_sem); 592d270df8SAlexey Kardashevskiy 602d270df8SAlexey Kardashevskiy return ret; 612d270df8SAlexey Kardashevskiy } 622d270df8SAlexey Kardashevskiy 632d270df8SAlexey Kardashevskiy static void decrement_locked_vm(long npages) 642d270df8SAlexey Kardashevskiy { 652d270df8SAlexey Kardashevskiy if (!current || !current->mm || !npages) 662d270df8SAlexey Kardashevskiy return; /* process exited */ 672d270df8SAlexey Kardashevskiy 682d270df8SAlexey Kardashevskiy down_write(¤t->mm->mmap_sem); 692d270df8SAlexey Kardashevskiy if (WARN_ON_ONCE(npages > current->mm->locked_vm)) 702d270df8SAlexey Kardashevskiy npages = current->mm->locked_vm; 712d270df8SAlexey Kardashevskiy current->mm->locked_vm -= npages; 722d270df8SAlexey Kardashevskiy pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid, 732d270df8SAlexey Kardashevskiy npages << PAGE_SHIFT, 742d270df8SAlexey Kardashevskiy current->mm->locked_vm << PAGE_SHIFT, 752d270df8SAlexey Kardashevskiy rlimit(RLIMIT_MEMLOCK)); 762d270df8SAlexey Kardashevskiy up_write(¤t->mm->mmap_sem); 772d270df8SAlexey Kardashevskiy } 782d270df8SAlexey Kardashevskiy 795ffd229cSAlexey Kardashevskiy /* 805ffd229cSAlexey Kardashevskiy * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation 815ffd229cSAlexey Kardashevskiy * 825ffd229cSAlexey Kardashevskiy * This code handles mapping and unmapping of user data buffers 835ffd229cSAlexey Kardashevskiy * into DMA'ble space using the IOMMU 845ffd229cSAlexey Kardashevskiy */ 855ffd229cSAlexey Kardashevskiy 862157e7b8SAlexey Kardashevskiy struct tce_iommu_group { 872157e7b8SAlexey Kardashevskiy struct list_head next; 882157e7b8SAlexey Kardashevskiy struct iommu_group *grp; 892157e7b8SAlexey Kardashevskiy }; 902157e7b8SAlexey Kardashevskiy 915ffd229cSAlexey Kardashevskiy /* 925ffd229cSAlexey Kardashevskiy * The container descriptor supports only a single group per container. 935ffd229cSAlexey Kardashevskiy * Required by the API as the container is not supplied with the IOMMU group 945ffd229cSAlexey Kardashevskiy * at the moment of initialization. 955ffd229cSAlexey Kardashevskiy */ 965ffd229cSAlexey Kardashevskiy struct tce_container { 975ffd229cSAlexey Kardashevskiy struct mutex lock; 985ffd229cSAlexey Kardashevskiy bool enabled; 992157e7b8SAlexey Kardashevskiy bool v2; 100d9c72894SAlexey Kardashevskiy bool def_window_pending; 1012d270df8SAlexey Kardashevskiy unsigned long locked_pages; 1022157e7b8SAlexey Kardashevskiy struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES]; 1032157e7b8SAlexey Kardashevskiy struct list_head group_list; 1045ffd229cSAlexey Kardashevskiy }; 1055ffd229cSAlexey Kardashevskiy 1062157e7b8SAlexey Kardashevskiy static long tce_iommu_unregister_pages(struct tce_container *container, 1072157e7b8SAlexey Kardashevskiy __u64 vaddr, __u64 size) 1082157e7b8SAlexey Kardashevskiy { 1092157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem; 1102157e7b8SAlexey Kardashevskiy 111d7baee69SAlexey Kardashevskiy if (!current || !current->mm) 112d7baee69SAlexey Kardashevskiy return -ESRCH; /* process exited */ 113d7baee69SAlexey Kardashevskiy 1142157e7b8SAlexey Kardashevskiy if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK)) 1152157e7b8SAlexey Kardashevskiy return -EINVAL; 1162157e7b8SAlexey Kardashevskiy 117d7baee69SAlexey Kardashevskiy mem = mm_iommu_find(current->mm, vaddr, size >> PAGE_SHIFT); 1182157e7b8SAlexey Kardashevskiy if (!mem) 1192157e7b8SAlexey Kardashevskiy return -ENOENT; 1202157e7b8SAlexey Kardashevskiy 121d7baee69SAlexey Kardashevskiy return mm_iommu_put(current->mm, mem); 1222157e7b8SAlexey Kardashevskiy } 1232157e7b8SAlexey Kardashevskiy 1242157e7b8SAlexey Kardashevskiy static long tce_iommu_register_pages(struct tce_container *container, 1252157e7b8SAlexey Kardashevskiy __u64 vaddr, __u64 size) 1262157e7b8SAlexey Kardashevskiy { 1272157e7b8SAlexey Kardashevskiy long ret = 0; 1282157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL; 1292157e7b8SAlexey Kardashevskiy unsigned long entries = size >> PAGE_SHIFT; 1302157e7b8SAlexey Kardashevskiy 131d7baee69SAlexey Kardashevskiy if (!current || !current->mm) 132d7baee69SAlexey Kardashevskiy return -ESRCH; /* process exited */ 133d7baee69SAlexey Kardashevskiy 1342157e7b8SAlexey Kardashevskiy if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) || 1352157e7b8SAlexey Kardashevskiy ((vaddr + size) < vaddr)) 1362157e7b8SAlexey Kardashevskiy return -EINVAL; 1372157e7b8SAlexey Kardashevskiy 138d7baee69SAlexey Kardashevskiy ret = mm_iommu_get(current->mm, vaddr, entries, &mem); 1392157e7b8SAlexey Kardashevskiy if (ret) 1402157e7b8SAlexey Kardashevskiy return ret; 1412157e7b8SAlexey Kardashevskiy 1422157e7b8SAlexey Kardashevskiy container->enabled = true; 1432157e7b8SAlexey Kardashevskiy 1442157e7b8SAlexey Kardashevskiy return 0; 1452157e7b8SAlexey Kardashevskiy } 1462157e7b8SAlexey Kardashevskiy 1472157e7b8SAlexey Kardashevskiy static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl) 1482157e7b8SAlexey Kardashevskiy { 1492157e7b8SAlexey Kardashevskiy unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) * 1502157e7b8SAlexey Kardashevskiy tbl->it_size, PAGE_SIZE); 1512157e7b8SAlexey Kardashevskiy unsigned long *uas; 1522157e7b8SAlexey Kardashevskiy long ret; 1532157e7b8SAlexey Kardashevskiy 1542157e7b8SAlexey Kardashevskiy BUG_ON(tbl->it_userspace); 1552157e7b8SAlexey Kardashevskiy 1562157e7b8SAlexey Kardashevskiy ret = try_increment_locked_vm(cb >> PAGE_SHIFT); 1572157e7b8SAlexey Kardashevskiy if (ret) 1582157e7b8SAlexey Kardashevskiy return ret; 1592157e7b8SAlexey Kardashevskiy 1602157e7b8SAlexey Kardashevskiy uas = vzalloc(cb); 1612157e7b8SAlexey Kardashevskiy if (!uas) { 1622157e7b8SAlexey Kardashevskiy decrement_locked_vm(cb >> PAGE_SHIFT); 1632157e7b8SAlexey Kardashevskiy return -ENOMEM; 1642157e7b8SAlexey Kardashevskiy } 1652157e7b8SAlexey Kardashevskiy tbl->it_userspace = uas; 1662157e7b8SAlexey Kardashevskiy 1672157e7b8SAlexey Kardashevskiy return 0; 1682157e7b8SAlexey Kardashevskiy } 1692157e7b8SAlexey Kardashevskiy 1702157e7b8SAlexey Kardashevskiy static void tce_iommu_userspace_view_free(struct iommu_table *tbl) 1712157e7b8SAlexey Kardashevskiy { 1722157e7b8SAlexey Kardashevskiy unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) * 1732157e7b8SAlexey Kardashevskiy tbl->it_size, PAGE_SIZE); 1742157e7b8SAlexey Kardashevskiy 1752157e7b8SAlexey Kardashevskiy if (!tbl->it_userspace) 1762157e7b8SAlexey Kardashevskiy return; 1772157e7b8SAlexey Kardashevskiy 1782157e7b8SAlexey Kardashevskiy vfree(tbl->it_userspace); 1792157e7b8SAlexey Kardashevskiy tbl->it_userspace = NULL; 1802157e7b8SAlexey Kardashevskiy decrement_locked_vm(cb >> PAGE_SHIFT); 1812157e7b8SAlexey Kardashevskiy } 1822157e7b8SAlexey Kardashevskiy 183e432bc7eSAlexey Kardashevskiy static bool tce_page_is_contained(struct page *page, unsigned page_shift) 184e432bc7eSAlexey Kardashevskiy { 185e432bc7eSAlexey Kardashevskiy /* 186e432bc7eSAlexey Kardashevskiy * Check that the TCE table granularity is not bigger than the size of 187e432bc7eSAlexey Kardashevskiy * a page we just found. Otherwise the hardware can get access to 188e432bc7eSAlexey Kardashevskiy * a bigger memory chunk that it should. 189e432bc7eSAlexey Kardashevskiy */ 190e432bc7eSAlexey Kardashevskiy return (PAGE_SHIFT + compound_order(compound_head(page))) >= page_shift; 191e432bc7eSAlexey Kardashevskiy } 192e432bc7eSAlexey Kardashevskiy 1932157e7b8SAlexey Kardashevskiy static inline bool tce_groups_attached(struct tce_container *container) 1942157e7b8SAlexey Kardashevskiy { 1952157e7b8SAlexey Kardashevskiy return !list_empty(&container->group_list); 1962157e7b8SAlexey Kardashevskiy } 1972157e7b8SAlexey Kardashevskiy 1980eaf4defSAlexey Kardashevskiy static long tce_iommu_find_table(struct tce_container *container, 1990eaf4defSAlexey Kardashevskiy phys_addr_t ioba, struct iommu_table **ptbl) 2000eaf4defSAlexey Kardashevskiy { 2010eaf4defSAlexey Kardashevskiy long i; 2020eaf4defSAlexey Kardashevskiy 2030eaf4defSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 2042157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 2050eaf4defSAlexey Kardashevskiy 2060eaf4defSAlexey Kardashevskiy if (tbl) { 2070eaf4defSAlexey Kardashevskiy unsigned long entry = ioba >> tbl->it_page_shift; 2080eaf4defSAlexey Kardashevskiy unsigned long start = tbl->it_offset; 2090eaf4defSAlexey Kardashevskiy unsigned long end = start + tbl->it_size; 2100eaf4defSAlexey Kardashevskiy 2110eaf4defSAlexey Kardashevskiy if ((start <= entry) && (entry < end)) { 2120eaf4defSAlexey Kardashevskiy *ptbl = tbl; 2130eaf4defSAlexey Kardashevskiy return i; 2140eaf4defSAlexey Kardashevskiy } 2150eaf4defSAlexey Kardashevskiy } 2160eaf4defSAlexey Kardashevskiy } 2170eaf4defSAlexey Kardashevskiy 2180eaf4defSAlexey Kardashevskiy return -1; 2190eaf4defSAlexey Kardashevskiy } 2200eaf4defSAlexey Kardashevskiy 221e633bc86SAlexey Kardashevskiy static int tce_iommu_find_free_table(struct tce_container *container) 222e633bc86SAlexey Kardashevskiy { 223e633bc86SAlexey Kardashevskiy int i; 224e633bc86SAlexey Kardashevskiy 225e633bc86SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 226e633bc86SAlexey Kardashevskiy if (!container->tables[i]) 227e633bc86SAlexey Kardashevskiy return i; 228e633bc86SAlexey Kardashevskiy } 229e633bc86SAlexey Kardashevskiy 230e633bc86SAlexey Kardashevskiy return -ENOSPC; 231e633bc86SAlexey Kardashevskiy } 232e633bc86SAlexey Kardashevskiy 2335ffd229cSAlexey Kardashevskiy static int tce_iommu_enable(struct tce_container *container) 2345ffd229cSAlexey Kardashevskiy { 2355ffd229cSAlexey Kardashevskiy int ret = 0; 2362d270df8SAlexey Kardashevskiy unsigned long locked; 2370eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 2382157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 2395ffd229cSAlexey Kardashevskiy 2405ffd229cSAlexey Kardashevskiy if (!current->mm) 2415ffd229cSAlexey Kardashevskiy return -ESRCH; /* process exited */ 2425ffd229cSAlexey Kardashevskiy 2435ffd229cSAlexey Kardashevskiy if (container->enabled) 2445ffd229cSAlexey Kardashevskiy return -EBUSY; 2455ffd229cSAlexey Kardashevskiy 2465ffd229cSAlexey Kardashevskiy /* 2475ffd229cSAlexey Kardashevskiy * When userspace pages are mapped into the IOMMU, they are effectively 2485ffd229cSAlexey Kardashevskiy * locked memory, so, theoretically, we need to update the accounting 2495ffd229cSAlexey Kardashevskiy * of locked pages on each map and unmap. For powerpc, the map unmap 2505ffd229cSAlexey Kardashevskiy * paths can be very hot, though, and the accounting would kill 2515ffd229cSAlexey Kardashevskiy * performance, especially since it would be difficult to impossible 2525ffd229cSAlexey Kardashevskiy * to handle the accounting in real mode only. 2535ffd229cSAlexey Kardashevskiy * 2545ffd229cSAlexey Kardashevskiy * To address that, rather than precisely accounting every page, we 2555ffd229cSAlexey Kardashevskiy * instead account for a worst case on locked memory when the iommu is 2565ffd229cSAlexey Kardashevskiy * enabled and disabled. The worst case upper bound on locked memory 2575ffd229cSAlexey Kardashevskiy * is the size of the whole iommu window, which is usually relatively 2585ffd229cSAlexey Kardashevskiy * small (compared to total memory sizes) on POWER hardware. 2595ffd229cSAlexey Kardashevskiy * 2605ffd229cSAlexey Kardashevskiy * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits, 2615ffd229cSAlexey Kardashevskiy * that would effectively kill the guest at random points, much better 2625ffd229cSAlexey Kardashevskiy * enforcing the limit based on the max that the guest can map. 2632d270df8SAlexey Kardashevskiy * 2642d270df8SAlexey Kardashevskiy * Unfortunately at the moment it counts whole tables, no matter how 2652d270df8SAlexey Kardashevskiy * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups 2662d270df8SAlexey Kardashevskiy * each with 2GB DMA window, 8GB will be counted here. The reason for 2672d270df8SAlexey Kardashevskiy * this is that we cannot tell here the amount of RAM used by the guest 2682d270df8SAlexey Kardashevskiy * as this information is only available from KVM and VFIO is 2692d270df8SAlexey Kardashevskiy * KVM agnostic. 2704793d65dSAlexey Kardashevskiy * 2714793d65dSAlexey Kardashevskiy * So we do not allow enabling a container without a group attached 2724793d65dSAlexey Kardashevskiy * as there is no way to know how much we should increment 2734793d65dSAlexey Kardashevskiy * the locked_vm counter. 2745ffd229cSAlexey Kardashevskiy */ 2752157e7b8SAlexey Kardashevskiy if (!tce_groups_attached(container)) 2762157e7b8SAlexey Kardashevskiy return -ENODEV; 2772157e7b8SAlexey Kardashevskiy 2782157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 2792157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next); 2802157e7b8SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 2810eaf4defSAlexey Kardashevskiy if (!table_group) 2820eaf4defSAlexey Kardashevskiy return -ENODEV; 2830eaf4defSAlexey Kardashevskiy 2844793d65dSAlexey Kardashevskiy if (!table_group->tce32_size) 2854793d65dSAlexey Kardashevskiy return -EPERM; 2864793d65dSAlexey Kardashevskiy 2874793d65dSAlexey Kardashevskiy locked = table_group->tce32_size >> PAGE_SHIFT; 2882d270df8SAlexey Kardashevskiy ret = try_increment_locked_vm(locked); 2892d270df8SAlexey Kardashevskiy if (ret) 2902d270df8SAlexey Kardashevskiy return ret; 2915ffd229cSAlexey Kardashevskiy 2922d270df8SAlexey Kardashevskiy container->locked_pages = locked; 2932d270df8SAlexey Kardashevskiy 2945ffd229cSAlexey Kardashevskiy container->enabled = true; 2955ffd229cSAlexey Kardashevskiy 2965ffd229cSAlexey Kardashevskiy return ret; 2975ffd229cSAlexey Kardashevskiy } 2985ffd229cSAlexey Kardashevskiy 2995ffd229cSAlexey Kardashevskiy static void tce_iommu_disable(struct tce_container *container) 3005ffd229cSAlexey Kardashevskiy { 3015ffd229cSAlexey Kardashevskiy if (!container->enabled) 3025ffd229cSAlexey Kardashevskiy return; 3035ffd229cSAlexey Kardashevskiy 3045ffd229cSAlexey Kardashevskiy container->enabled = false; 3055ffd229cSAlexey Kardashevskiy 3062d270df8SAlexey Kardashevskiy if (!current->mm) 3075ffd229cSAlexey Kardashevskiy return; 3085ffd229cSAlexey Kardashevskiy 3092d270df8SAlexey Kardashevskiy decrement_locked_vm(container->locked_pages); 3105ffd229cSAlexey Kardashevskiy } 3115ffd229cSAlexey Kardashevskiy 3125ffd229cSAlexey Kardashevskiy static void *tce_iommu_open(unsigned long arg) 3135ffd229cSAlexey Kardashevskiy { 3145ffd229cSAlexey Kardashevskiy struct tce_container *container; 3155ffd229cSAlexey Kardashevskiy 3162157e7b8SAlexey Kardashevskiy if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) { 3175ffd229cSAlexey Kardashevskiy pr_err("tce_vfio: Wrong IOMMU type\n"); 3185ffd229cSAlexey Kardashevskiy return ERR_PTR(-EINVAL); 3195ffd229cSAlexey Kardashevskiy } 3205ffd229cSAlexey Kardashevskiy 3215ffd229cSAlexey Kardashevskiy container = kzalloc(sizeof(*container), GFP_KERNEL); 3225ffd229cSAlexey Kardashevskiy if (!container) 3235ffd229cSAlexey Kardashevskiy return ERR_PTR(-ENOMEM); 3245ffd229cSAlexey Kardashevskiy 3255ffd229cSAlexey Kardashevskiy mutex_init(&container->lock); 3262157e7b8SAlexey Kardashevskiy INIT_LIST_HEAD_RCU(&container->group_list); 3272157e7b8SAlexey Kardashevskiy 3282157e7b8SAlexey Kardashevskiy container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU; 3295ffd229cSAlexey Kardashevskiy 3305ffd229cSAlexey Kardashevskiy return container; 3315ffd229cSAlexey Kardashevskiy } 3325ffd229cSAlexey Kardashevskiy 3332157e7b8SAlexey Kardashevskiy static int tce_iommu_clear(struct tce_container *container, 3342157e7b8SAlexey Kardashevskiy struct iommu_table *tbl, 3352157e7b8SAlexey Kardashevskiy unsigned long entry, unsigned long pages); 3362157e7b8SAlexey Kardashevskiy static void tce_iommu_free_table(struct iommu_table *tbl); 3372157e7b8SAlexey Kardashevskiy 3385ffd229cSAlexey Kardashevskiy static void tce_iommu_release(void *iommu_data) 3395ffd229cSAlexey Kardashevskiy { 3405ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 3412157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 3422157e7b8SAlexey Kardashevskiy long i; 3435ffd229cSAlexey Kardashevskiy 3442157e7b8SAlexey Kardashevskiy while (tce_groups_attached(container)) { 3452157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 3462157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next); 3472157e7b8SAlexey Kardashevskiy tce_iommu_detach_group(iommu_data, tcegrp->grp); 3482157e7b8SAlexey Kardashevskiy } 3495ffd229cSAlexey Kardashevskiy 3502157e7b8SAlexey Kardashevskiy /* 3512157e7b8SAlexey Kardashevskiy * If VFIO created a table, it was not disposed 3522157e7b8SAlexey Kardashevskiy * by tce_iommu_detach_group() so do it now. 3532157e7b8SAlexey Kardashevskiy */ 3542157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 3552157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 3562157e7b8SAlexey Kardashevskiy 3572157e7b8SAlexey Kardashevskiy if (!tbl) 3582157e7b8SAlexey Kardashevskiy continue; 3592157e7b8SAlexey Kardashevskiy 3602157e7b8SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 3612157e7b8SAlexey Kardashevskiy tce_iommu_free_table(tbl); 3622157e7b8SAlexey Kardashevskiy } 3635ffd229cSAlexey Kardashevskiy 364649354b7SAlexey Kardashevskiy tce_iommu_disable(container); 3655ffd229cSAlexey Kardashevskiy mutex_destroy(&container->lock); 3665ffd229cSAlexey Kardashevskiy 3675ffd229cSAlexey Kardashevskiy kfree(container); 3685ffd229cSAlexey Kardashevskiy } 3695ffd229cSAlexey Kardashevskiy 370649354b7SAlexey Kardashevskiy static void tce_iommu_unuse_page(struct tce_container *container, 37105c6cfb9SAlexey Kardashevskiy unsigned long hpa) 372649354b7SAlexey Kardashevskiy { 373649354b7SAlexey Kardashevskiy struct page *page; 374649354b7SAlexey Kardashevskiy 37505c6cfb9SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT); 376649354b7SAlexey Kardashevskiy put_page(page); 377649354b7SAlexey Kardashevskiy } 378649354b7SAlexey Kardashevskiy 3792157e7b8SAlexey Kardashevskiy static int tce_iommu_prereg_ua_to_hpa(unsigned long tce, unsigned long size, 3802157e7b8SAlexey Kardashevskiy unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem) 3812157e7b8SAlexey Kardashevskiy { 3822157e7b8SAlexey Kardashevskiy long ret = 0; 3832157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem; 3842157e7b8SAlexey Kardashevskiy 385d7baee69SAlexey Kardashevskiy mem = mm_iommu_lookup(current->mm, tce, size); 3862157e7b8SAlexey Kardashevskiy if (!mem) 3872157e7b8SAlexey Kardashevskiy return -EINVAL; 3882157e7b8SAlexey Kardashevskiy 3892157e7b8SAlexey Kardashevskiy ret = mm_iommu_ua_to_hpa(mem, tce, phpa); 3902157e7b8SAlexey Kardashevskiy if (ret) 3912157e7b8SAlexey Kardashevskiy return -EINVAL; 3922157e7b8SAlexey Kardashevskiy 3932157e7b8SAlexey Kardashevskiy *pmem = mem; 3942157e7b8SAlexey Kardashevskiy 3952157e7b8SAlexey Kardashevskiy return 0; 3962157e7b8SAlexey Kardashevskiy } 3972157e7b8SAlexey Kardashevskiy 3982157e7b8SAlexey Kardashevskiy static void tce_iommu_unuse_page_v2(struct iommu_table *tbl, 3992157e7b8SAlexey Kardashevskiy unsigned long entry) 4002157e7b8SAlexey Kardashevskiy { 4012157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL; 4022157e7b8SAlexey Kardashevskiy int ret; 4032157e7b8SAlexey Kardashevskiy unsigned long hpa = 0; 4042157e7b8SAlexey Kardashevskiy unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 4052157e7b8SAlexey Kardashevskiy 4062157e7b8SAlexey Kardashevskiy if (!pua || !current || !current->mm) 4072157e7b8SAlexey Kardashevskiy return; 4082157e7b8SAlexey Kardashevskiy 4092157e7b8SAlexey Kardashevskiy ret = tce_iommu_prereg_ua_to_hpa(*pua, IOMMU_PAGE_SIZE(tbl), 4102157e7b8SAlexey Kardashevskiy &hpa, &mem); 4112157e7b8SAlexey Kardashevskiy if (ret) 4122157e7b8SAlexey Kardashevskiy pr_debug("%s: tce %lx at #%lx was not cached, ret=%d\n", 4132157e7b8SAlexey Kardashevskiy __func__, *pua, entry, ret); 4142157e7b8SAlexey Kardashevskiy if (mem) 4152157e7b8SAlexey Kardashevskiy mm_iommu_mapped_dec(mem); 4162157e7b8SAlexey Kardashevskiy 4172157e7b8SAlexey Kardashevskiy *pua = 0; 4182157e7b8SAlexey Kardashevskiy } 4192157e7b8SAlexey Kardashevskiy 4209b14a1ffSAlexey Kardashevskiy static int tce_iommu_clear(struct tce_container *container, 4219b14a1ffSAlexey Kardashevskiy struct iommu_table *tbl, 4229b14a1ffSAlexey Kardashevskiy unsigned long entry, unsigned long pages) 4239b14a1ffSAlexey Kardashevskiy { 42405c6cfb9SAlexey Kardashevskiy unsigned long oldhpa; 42505c6cfb9SAlexey Kardashevskiy long ret; 42605c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction; 4279b14a1ffSAlexey Kardashevskiy 4289b14a1ffSAlexey Kardashevskiy for ( ; pages; --pages, ++entry) { 42905c6cfb9SAlexey Kardashevskiy direction = DMA_NONE; 43005c6cfb9SAlexey Kardashevskiy oldhpa = 0; 43105c6cfb9SAlexey Kardashevskiy ret = iommu_tce_xchg(tbl, entry, &oldhpa, &direction); 43205c6cfb9SAlexey Kardashevskiy if (ret) 4339b14a1ffSAlexey Kardashevskiy continue; 4349b14a1ffSAlexey Kardashevskiy 43505c6cfb9SAlexey Kardashevskiy if (direction == DMA_NONE) 43605c6cfb9SAlexey Kardashevskiy continue; 43705c6cfb9SAlexey Kardashevskiy 4382157e7b8SAlexey Kardashevskiy if (container->v2) { 4392157e7b8SAlexey Kardashevskiy tce_iommu_unuse_page_v2(tbl, entry); 4402157e7b8SAlexey Kardashevskiy continue; 4412157e7b8SAlexey Kardashevskiy } 4422157e7b8SAlexey Kardashevskiy 44305c6cfb9SAlexey Kardashevskiy tce_iommu_unuse_page(container, oldhpa); 4449b14a1ffSAlexey Kardashevskiy } 445649354b7SAlexey Kardashevskiy 446649354b7SAlexey Kardashevskiy return 0; 4479b14a1ffSAlexey Kardashevskiy } 4489b14a1ffSAlexey Kardashevskiy 449649354b7SAlexey Kardashevskiy static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa) 450649354b7SAlexey Kardashevskiy { 451649354b7SAlexey Kardashevskiy struct page *page = NULL; 452649354b7SAlexey Kardashevskiy enum dma_data_direction direction = iommu_tce_direction(tce); 453649354b7SAlexey Kardashevskiy 454649354b7SAlexey Kardashevskiy if (get_user_pages_fast(tce & PAGE_MASK, 1, 455649354b7SAlexey Kardashevskiy direction != DMA_TO_DEVICE, &page) != 1) 456649354b7SAlexey Kardashevskiy return -EFAULT; 457649354b7SAlexey Kardashevskiy 458649354b7SAlexey Kardashevskiy *hpa = __pa((unsigned long) page_address(page)); 459649354b7SAlexey Kardashevskiy 4609b14a1ffSAlexey Kardashevskiy return 0; 4619b14a1ffSAlexey Kardashevskiy } 4629b14a1ffSAlexey Kardashevskiy 4639b14a1ffSAlexey Kardashevskiy static long tce_iommu_build(struct tce_container *container, 4649b14a1ffSAlexey Kardashevskiy struct iommu_table *tbl, 46505c6cfb9SAlexey Kardashevskiy unsigned long entry, unsigned long tce, unsigned long pages, 46605c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction) 4679b14a1ffSAlexey Kardashevskiy { 4689b14a1ffSAlexey Kardashevskiy long i, ret = 0; 469649354b7SAlexey Kardashevskiy struct page *page; 470649354b7SAlexey Kardashevskiy unsigned long hpa; 47105c6cfb9SAlexey Kardashevskiy enum dma_data_direction dirtmp; 4729b14a1ffSAlexey Kardashevskiy 4739b14a1ffSAlexey Kardashevskiy for (i = 0; i < pages; ++i) { 4749b14a1ffSAlexey Kardashevskiy unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK; 4759b14a1ffSAlexey Kardashevskiy 476649354b7SAlexey Kardashevskiy ret = tce_iommu_use_page(tce, &hpa); 477649354b7SAlexey Kardashevskiy if (ret) 4789b14a1ffSAlexey Kardashevskiy break; 479e432bc7eSAlexey Kardashevskiy 480649354b7SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT); 481e432bc7eSAlexey Kardashevskiy if (!tce_page_is_contained(page, tbl->it_page_shift)) { 482e432bc7eSAlexey Kardashevskiy ret = -EPERM; 483e432bc7eSAlexey Kardashevskiy break; 484e432bc7eSAlexey Kardashevskiy } 485e432bc7eSAlexey Kardashevskiy 486649354b7SAlexey Kardashevskiy hpa |= offset; 48705c6cfb9SAlexey Kardashevskiy dirtmp = direction; 48805c6cfb9SAlexey Kardashevskiy ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp); 4899b14a1ffSAlexey Kardashevskiy if (ret) { 490649354b7SAlexey Kardashevskiy tce_iommu_unuse_page(container, hpa); 4919b14a1ffSAlexey Kardashevskiy pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n", 4929b14a1ffSAlexey Kardashevskiy __func__, entry << tbl->it_page_shift, 4939b14a1ffSAlexey Kardashevskiy tce, ret); 4949b14a1ffSAlexey Kardashevskiy break; 4959b14a1ffSAlexey Kardashevskiy } 49605c6cfb9SAlexey Kardashevskiy 49705c6cfb9SAlexey Kardashevskiy if (dirtmp != DMA_NONE) 49805c6cfb9SAlexey Kardashevskiy tce_iommu_unuse_page(container, hpa); 49905c6cfb9SAlexey Kardashevskiy 50000663d4eSAlexey Kardashevskiy tce += IOMMU_PAGE_SIZE(tbl); 5019b14a1ffSAlexey Kardashevskiy } 5029b14a1ffSAlexey Kardashevskiy 5039b14a1ffSAlexey Kardashevskiy if (ret) 5049b14a1ffSAlexey Kardashevskiy tce_iommu_clear(container, tbl, entry, i); 5059b14a1ffSAlexey Kardashevskiy 5069b14a1ffSAlexey Kardashevskiy return ret; 5079b14a1ffSAlexey Kardashevskiy } 5089b14a1ffSAlexey Kardashevskiy 5092157e7b8SAlexey Kardashevskiy static long tce_iommu_build_v2(struct tce_container *container, 5102157e7b8SAlexey Kardashevskiy struct iommu_table *tbl, 5112157e7b8SAlexey Kardashevskiy unsigned long entry, unsigned long tce, unsigned long pages, 5122157e7b8SAlexey Kardashevskiy enum dma_data_direction direction) 5132157e7b8SAlexey Kardashevskiy { 5142157e7b8SAlexey Kardashevskiy long i, ret = 0; 5152157e7b8SAlexey Kardashevskiy struct page *page; 5162157e7b8SAlexey Kardashevskiy unsigned long hpa; 5172157e7b8SAlexey Kardashevskiy enum dma_data_direction dirtmp; 5182157e7b8SAlexey Kardashevskiy 51939701e56SAlexey Kardashevskiy if (!tbl->it_userspace) { 52039701e56SAlexey Kardashevskiy ret = tce_iommu_userspace_view_alloc(tbl); 52139701e56SAlexey Kardashevskiy if (ret) 52239701e56SAlexey Kardashevskiy return ret; 52339701e56SAlexey Kardashevskiy } 52439701e56SAlexey Kardashevskiy 5252157e7b8SAlexey Kardashevskiy for (i = 0; i < pages; ++i) { 5262157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL; 5272157e7b8SAlexey Kardashevskiy unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, 5282157e7b8SAlexey Kardashevskiy entry + i); 5292157e7b8SAlexey Kardashevskiy 5302157e7b8SAlexey Kardashevskiy ret = tce_iommu_prereg_ua_to_hpa(tce, IOMMU_PAGE_SIZE(tbl), 5312157e7b8SAlexey Kardashevskiy &hpa, &mem); 5322157e7b8SAlexey Kardashevskiy if (ret) 5332157e7b8SAlexey Kardashevskiy break; 5342157e7b8SAlexey Kardashevskiy 5352157e7b8SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT); 5362157e7b8SAlexey Kardashevskiy if (!tce_page_is_contained(page, tbl->it_page_shift)) { 5372157e7b8SAlexey Kardashevskiy ret = -EPERM; 5382157e7b8SAlexey Kardashevskiy break; 5392157e7b8SAlexey Kardashevskiy } 5402157e7b8SAlexey Kardashevskiy 5412157e7b8SAlexey Kardashevskiy /* Preserve offset within IOMMU page */ 5422157e7b8SAlexey Kardashevskiy hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK; 5432157e7b8SAlexey Kardashevskiy dirtmp = direction; 5442157e7b8SAlexey Kardashevskiy 5452157e7b8SAlexey Kardashevskiy /* The registered region is being unregistered */ 5462157e7b8SAlexey Kardashevskiy if (mm_iommu_mapped_inc(mem)) 5472157e7b8SAlexey Kardashevskiy break; 5482157e7b8SAlexey Kardashevskiy 5492157e7b8SAlexey Kardashevskiy ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp); 5502157e7b8SAlexey Kardashevskiy if (ret) { 5512157e7b8SAlexey Kardashevskiy /* dirtmp cannot be DMA_NONE here */ 5522157e7b8SAlexey Kardashevskiy tce_iommu_unuse_page_v2(tbl, entry + i); 5532157e7b8SAlexey Kardashevskiy pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n", 5542157e7b8SAlexey Kardashevskiy __func__, entry << tbl->it_page_shift, 5552157e7b8SAlexey Kardashevskiy tce, ret); 5562157e7b8SAlexey Kardashevskiy break; 5572157e7b8SAlexey Kardashevskiy } 5582157e7b8SAlexey Kardashevskiy 5592157e7b8SAlexey Kardashevskiy if (dirtmp != DMA_NONE) 5602157e7b8SAlexey Kardashevskiy tce_iommu_unuse_page_v2(tbl, entry + i); 5612157e7b8SAlexey Kardashevskiy 5622157e7b8SAlexey Kardashevskiy *pua = tce; 5632157e7b8SAlexey Kardashevskiy 5642157e7b8SAlexey Kardashevskiy tce += IOMMU_PAGE_SIZE(tbl); 5652157e7b8SAlexey Kardashevskiy } 5662157e7b8SAlexey Kardashevskiy 5672157e7b8SAlexey Kardashevskiy if (ret) 5682157e7b8SAlexey Kardashevskiy tce_iommu_clear(container, tbl, entry, i); 5692157e7b8SAlexey Kardashevskiy 5702157e7b8SAlexey Kardashevskiy return ret; 5712157e7b8SAlexey Kardashevskiy } 5722157e7b8SAlexey Kardashevskiy 57346d3e1e1SAlexey Kardashevskiy static long tce_iommu_create_table(struct tce_container *container, 57446d3e1e1SAlexey Kardashevskiy struct iommu_table_group *table_group, 57546d3e1e1SAlexey Kardashevskiy int num, 57646d3e1e1SAlexey Kardashevskiy __u32 page_shift, 57746d3e1e1SAlexey Kardashevskiy __u64 window_size, 57846d3e1e1SAlexey Kardashevskiy __u32 levels, 57946d3e1e1SAlexey Kardashevskiy struct iommu_table **ptbl) 58046d3e1e1SAlexey Kardashevskiy { 58146d3e1e1SAlexey Kardashevskiy long ret, table_size; 58246d3e1e1SAlexey Kardashevskiy 58346d3e1e1SAlexey Kardashevskiy table_size = table_group->ops->get_table_size(page_shift, window_size, 58446d3e1e1SAlexey Kardashevskiy levels); 58546d3e1e1SAlexey Kardashevskiy if (!table_size) 58646d3e1e1SAlexey Kardashevskiy return -EINVAL; 58746d3e1e1SAlexey Kardashevskiy 58846d3e1e1SAlexey Kardashevskiy ret = try_increment_locked_vm(table_size >> PAGE_SHIFT); 58946d3e1e1SAlexey Kardashevskiy if (ret) 59046d3e1e1SAlexey Kardashevskiy return ret; 59146d3e1e1SAlexey Kardashevskiy 59246d3e1e1SAlexey Kardashevskiy ret = table_group->ops->create_table(table_group, num, 59346d3e1e1SAlexey Kardashevskiy page_shift, window_size, levels, ptbl); 59446d3e1e1SAlexey Kardashevskiy 59546d3e1e1SAlexey Kardashevskiy WARN_ON(!ret && !(*ptbl)->it_ops->free); 59646d3e1e1SAlexey Kardashevskiy WARN_ON(!ret && ((*ptbl)->it_allocated_size != table_size)); 59746d3e1e1SAlexey Kardashevskiy 59846d3e1e1SAlexey Kardashevskiy return ret; 59946d3e1e1SAlexey Kardashevskiy } 60046d3e1e1SAlexey Kardashevskiy 60146d3e1e1SAlexey Kardashevskiy static void tce_iommu_free_table(struct iommu_table *tbl) 60246d3e1e1SAlexey Kardashevskiy { 60346d3e1e1SAlexey Kardashevskiy unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT; 60446d3e1e1SAlexey Kardashevskiy 6052157e7b8SAlexey Kardashevskiy tce_iommu_userspace_view_free(tbl); 60646d3e1e1SAlexey Kardashevskiy tbl->it_ops->free(tbl); 60746d3e1e1SAlexey Kardashevskiy decrement_locked_vm(pages); 60846d3e1e1SAlexey Kardashevskiy } 60946d3e1e1SAlexey Kardashevskiy 610e633bc86SAlexey Kardashevskiy static long tce_iommu_create_window(struct tce_container *container, 611e633bc86SAlexey Kardashevskiy __u32 page_shift, __u64 window_size, __u32 levels, 612e633bc86SAlexey Kardashevskiy __u64 *start_addr) 613e633bc86SAlexey Kardashevskiy { 614e633bc86SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 615e633bc86SAlexey Kardashevskiy struct iommu_table_group *table_group; 616e633bc86SAlexey Kardashevskiy struct iommu_table *tbl = NULL; 617e633bc86SAlexey Kardashevskiy long ret, num; 618e633bc86SAlexey Kardashevskiy 619e633bc86SAlexey Kardashevskiy num = tce_iommu_find_free_table(container); 620e633bc86SAlexey Kardashevskiy if (num < 0) 621e633bc86SAlexey Kardashevskiy return num; 622e633bc86SAlexey Kardashevskiy 623e633bc86SAlexey Kardashevskiy /* Get the first group for ops::create_table */ 624e633bc86SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 625e633bc86SAlexey Kardashevskiy struct tce_iommu_group, next); 626e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 627e633bc86SAlexey Kardashevskiy if (!table_group) 628e633bc86SAlexey Kardashevskiy return -EFAULT; 629e633bc86SAlexey Kardashevskiy 630e633bc86SAlexey Kardashevskiy if (!(table_group->pgsizes & (1ULL << page_shift))) 631e633bc86SAlexey Kardashevskiy return -EINVAL; 632e633bc86SAlexey Kardashevskiy 633e633bc86SAlexey Kardashevskiy if (!table_group->ops->set_window || !table_group->ops->unset_window || 634e633bc86SAlexey Kardashevskiy !table_group->ops->get_table_size || 635e633bc86SAlexey Kardashevskiy !table_group->ops->create_table) 636e633bc86SAlexey Kardashevskiy return -EPERM; 637e633bc86SAlexey Kardashevskiy 638e633bc86SAlexey Kardashevskiy /* Create TCE table */ 639e633bc86SAlexey Kardashevskiy ret = tce_iommu_create_table(container, table_group, num, 640e633bc86SAlexey Kardashevskiy page_shift, window_size, levels, &tbl); 641e633bc86SAlexey Kardashevskiy if (ret) 642e633bc86SAlexey Kardashevskiy return ret; 643e633bc86SAlexey Kardashevskiy 644e633bc86SAlexey Kardashevskiy BUG_ON(!tbl->it_ops->free); 645e633bc86SAlexey Kardashevskiy 646e633bc86SAlexey Kardashevskiy /* 647e633bc86SAlexey Kardashevskiy * Program the table to every group. 648e633bc86SAlexey Kardashevskiy * Groups have been tested for compatibility at the attach time. 649e633bc86SAlexey Kardashevskiy */ 650e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 651e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 652e633bc86SAlexey Kardashevskiy 653e633bc86SAlexey Kardashevskiy ret = table_group->ops->set_window(table_group, num, tbl); 654e633bc86SAlexey Kardashevskiy if (ret) 655e633bc86SAlexey Kardashevskiy goto unset_exit; 656e633bc86SAlexey Kardashevskiy } 657e633bc86SAlexey Kardashevskiy 658e633bc86SAlexey Kardashevskiy container->tables[num] = tbl; 659e633bc86SAlexey Kardashevskiy 660e633bc86SAlexey Kardashevskiy /* Return start address assigned by platform in create_table() */ 661e633bc86SAlexey Kardashevskiy *start_addr = tbl->it_offset << tbl->it_page_shift; 662e633bc86SAlexey Kardashevskiy 663e633bc86SAlexey Kardashevskiy return 0; 664e633bc86SAlexey Kardashevskiy 665e633bc86SAlexey Kardashevskiy unset_exit: 666e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 667e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 668e633bc86SAlexey Kardashevskiy table_group->ops->unset_window(table_group, num); 669e633bc86SAlexey Kardashevskiy } 670e633bc86SAlexey Kardashevskiy tce_iommu_free_table(tbl); 671e633bc86SAlexey Kardashevskiy 672e633bc86SAlexey Kardashevskiy return ret; 673e633bc86SAlexey Kardashevskiy } 674e633bc86SAlexey Kardashevskiy 675e633bc86SAlexey Kardashevskiy static long tce_iommu_remove_window(struct tce_container *container, 676e633bc86SAlexey Kardashevskiy __u64 start_addr) 677e633bc86SAlexey Kardashevskiy { 678e633bc86SAlexey Kardashevskiy struct iommu_table_group *table_group = NULL; 679e633bc86SAlexey Kardashevskiy struct iommu_table *tbl; 680e633bc86SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 681e633bc86SAlexey Kardashevskiy int num; 682e633bc86SAlexey Kardashevskiy 683e633bc86SAlexey Kardashevskiy num = tce_iommu_find_table(container, start_addr, &tbl); 684e633bc86SAlexey Kardashevskiy if (num < 0) 685e633bc86SAlexey Kardashevskiy return -EINVAL; 686e633bc86SAlexey Kardashevskiy 687e633bc86SAlexey Kardashevskiy BUG_ON(!tbl->it_size); 688e633bc86SAlexey Kardashevskiy 689e633bc86SAlexey Kardashevskiy /* Detach groups from IOMMUs */ 690e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 691e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 692e633bc86SAlexey Kardashevskiy 693e633bc86SAlexey Kardashevskiy /* 694e633bc86SAlexey Kardashevskiy * SPAPR TCE IOMMU exposes the default DMA window to 695e633bc86SAlexey Kardashevskiy * the guest via dma32_window_start/size of 696e633bc86SAlexey Kardashevskiy * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow 697e633bc86SAlexey Kardashevskiy * the userspace to remove this window, some do not so 698e633bc86SAlexey Kardashevskiy * here we check for the platform capability. 699e633bc86SAlexey Kardashevskiy */ 700e633bc86SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->unset_window) 701e633bc86SAlexey Kardashevskiy return -EPERM; 702e633bc86SAlexey Kardashevskiy 703e633bc86SAlexey Kardashevskiy table_group->ops->unset_window(table_group, num); 704e633bc86SAlexey Kardashevskiy } 705e633bc86SAlexey Kardashevskiy 706e633bc86SAlexey Kardashevskiy /* Free table */ 707e633bc86SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 708e633bc86SAlexey Kardashevskiy tce_iommu_free_table(tbl); 709e633bc86SAlexey Kardashevskiy container->tables[num] = NULL; 710e633bc86SAlexey Kardashevskiy 711e633bc86SAlexey Kardashevskiy return 0; 712e633bc86SAlexey Kardashevskiy } 713e633bc86SAlexey Kardashevskiy 7146f01cc69SAlexey Kardashevskiy static long tce_iommu_create_default_window(struct tce_container *container) 7156f01cc69SAlexey Kardashevskiy { 7166f01cc69SAlexey Kardashevskiy long ret; 7176f01cc69SAlexey Kardashevskiy __u64 start_addr = 0; 7186f01cc69SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 7196f01cc69SAlexey Kardashevskiy struct iommu_table_group *table_group; 7206f01cc69SAlexey Kardashevskiy 721d9c72894SAlexey Kardashevskiy if (!container->def_window_pending) 722d9c72894SAlexey Kardashevskiy return 0; 723d9c72894SAlexey Kardashevskiy 7246f01cc69SAlexey Kardashevskiy if (!tce_groups_attached(container)) 7256f01cc69SAlexey Kardashevskiy return -ENODEV; 7266f01cc69SAlexey Kardashevskiy 7276f01cc69SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 7286f01cc69SAlexey Kardashevskiy struct tce_iommu_group, next); 7296f01cc69SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 7306f01cc69SAlexey Kardashevskiy if (!table_group) 7316f01cc69SAlexey Kardashevskiy return -ENODEV; 7326f01cc69SAlexey Kardashevskiy 7336f01cc69SAlexey Kardashevskiy ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K, 7346f01cc69SAlexey Kardashevskiy table_group->tce32_size, 1, &start_addr); 7356f01cc69SAlexey Kardashevskiy WARN_ON_ONCE(!ret && start_addr); 7366f01cc69SAlexey Kardashevskiy 737d9c72894SAlexey Kardashevskiy if (!ret) 738d9c72894SAlexey Kardashevskiy container->def_window_pending = false; 739d9c72894SAlexey Kardashevskiy 7406f01cc69SAlexey Kardashevskiy return ret; 7416f01cc69SAlexey Kardashevskiy } 7426f01cc69SAlexey Kardashevskiy 7435ffd229cSAlexey Kardashevskiy static long tce_iommu_ioctl(void *iommu_data, 7445ffd229cSAlexey Kardashevskiy unsigned int cmd, unsigned long arg) 7455ffd229cSAlexey Kardashevskiy { 7465ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 747e633bc86SAlexey Kardashevskiy unsigned long minsz, ddwsz; 7485ffd229cSAlexey Kardashevskiy long ret; 7495ffd229cSAlexey Kardashevskiy 7505ffd229cSAlexey Kardashevskiy switch (cmd) { 7515ffd229cSAlexey Kardashevskiy case VFIO_CHECK_EXTENSION: 7521b69be5eSGavin Shan switch (arg) { 7531b69be5eSGavin Shan case VFIO_SPAPR_TCE_IOMMU: 7542157e7b8SAlexey Kardashevskiy case VFIO_SPAPR_TCE_v2_IOMMU: 7551b69be5eSGavin Shan ret = 1; 7561b69be5eSGavin Shan break; 7571b69be5eSGavin Shan default: 7581b69be5eSGavin Shan ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg); 7591b69be5eSGavin Shan break; 7601b69be5eSGavin Shan } 7611b69be5eSGavin Shan 7621b69be5eSGavin Shan return (ret < 0) ? 0 : ret; 7635ffd229cSAlexey Kardashevskiy 7645ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_GET_INFO: { 7655ffd229cSAlexey Kardashevskiy struct vfio_iommu_spapr_tce_info info; 7662157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 7670eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 7685ffd229cSAlexey Kardashevskiy 7692157e7b8SAlexey Kardashevskiy if (!tce_groups_attached(container)) 7700eaf4defSAlexey Kardashevskiy return -ENXIO; 7710eaf4defSAlexey Kardashevskiy 7722157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 7732157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next); 7742157e7b8SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 7750eaf4defSAlexey Kardashevskiy 7764793d65dSAlexey Kardashevskiy if (!table_group) 7775ffd229cSAlexey Kardashevskiy return -ENXIO; 7785ffd229cSAlexey Kardashevskiy 7795ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_info, 7805ffd229cSAlexey Kardashevskiy dma32_window_size); 7815ffd229cSAlexey Kardashevskiy 7825ffd229cSAlexey Kardashevskiy if (copy_from_user(&info, (void __user *)arg, minsz)) 7835ffd229cSAlexey Kardashevskiy return -EFAULT; 7845ffd229cSAlexey Kardashevskiy 7855ffd229cSAlexey Kardashevskiy if (info.argsz < minsz) 7865ffd229cSAlexey Kardashevskiy return -EINVAL; 7875ffd229cSAlexey Kardashevskiy 7884793d65dSAlexey Kardashevskiy info.dma32_window_start = table_group->tce32_start; 7894793d65dSAlexey Kardashevskiy info.dma32_window_size = table_group->tce32_size; 7905ffd229cSAlexey Kardashevskiy info.flags = 0; 791e633bc86SAlexey Kardashevskiy memset(&info.ddw, 0, sizeof(info.ddw)); 792e633bc86SAlexey Kardashevskiy 793e633bc86SAlexey Kardashevskiy if (table_group->max_dynamic_windows_supported && 794e633bc86SAlexey Kardashevskiy container->v2) { 795e633bc86SAlexey Kardashevskiy info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW; 796e633bc86SAlexey Kardashevskiy info.ddw.pgsizes = table_group->pgsizes; 797e633bc86SAlexey Kardashevskiy info.ddw.max_dynamic_windows_supported = 798e633bc86SAlexey Kardashevskiy table_group->max_dynamic_windows_supported; 799e633bc86SAlexey Kardashevskiy info.ddw.levels = table_group->max_levels; 800e633bc86SAlexey Kardashevskiy } 801e633bc86SAlexey Kardashevskiy 802e633bc86SAlexey Kardashevskiy ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw); 803e633bc86SAlexey Kardashevskiy 804e633bc86SAlexey Kardashevskiy if (info.argsz >= ddwsz) 805e633bc86SAlexey Kardashevskiy minsz = ddwsz; 8065ffd229cSAlexey Kardashevskiy 8075ffd229cSAlexey Kardashevskiy if (copy_to_user((void __user *)arg, &info, minsz)) 8085ffd229cSAlexey Kardashevskiy return -EFAULT; 8095ffd229cSAlexey Kardashevskiy 8105ffd229cSAlexey Kardashevskiy return 0; 8115ffd229cSAlexey Kardashevskiy } 8125ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_MAP_DMA: { 8135ffd229cSAlexey Kardashevskiy struct vfio_iommu_type1_dma_map param; 8140eaf4defSAlexey Kardashevskiy struct iommu_table *tbl = NULL; 8150eaf4defSAlexey Kardashevskiy long num; 81605c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction; 8175ffd229cSAlexey Kardashevskiy 8183c56e822SAlexey Kardashevskiy if (!container->enabled) 8193c56e822SAlexey Kardashevskiy return -EPERM; 8203c56e822SAlexey Kardashevskiy 8215ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); 8225ffd229cSAlexey Kardashevskiy 8235ffd229cSAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 8245ffd229cSAlexey Kardashevskiy return -EFAULT; 8255ffd229cSAlexey Kardashevskiy 8265ffd229cSAlexey Kardashevskiy if (param.argsz < minsz) 8275ffd229cSAlexey Kardashevskiy return -EINVAL; 8285ffd229cSAlexey Kardashevskiy 8295ffd229cSAlexey Kardashevskiy if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ | 8305ffd229cSAlexey Kardashevskiy VFIO_DMA_MAP_FLAG_WRITE)) 8315ffd229cSAlexey Kardashevskiy return -EINVAL; 8325ffd229cSAlexey Kardashevskiy 833d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container); 834d9c72894SAlexey Kardashevskiy if (ret) 835d9c72894SAlexey Kardashevskiy return ret; 836d9c72894SAlexey Kardashevskiy 8370eaf4defSAlexey Kardashevskiy num = tce_iommu_find_table(container, param.iova, &tbl); 8380eaf4defSAlexey Kardashevskiy if (num < 0) 8390eaf4defSAlexey Kardashevskiy return -ENXIO; 8400eaf4defSAlexey Kardashevskiy 84100663d4eSAlexey Kardashevskiy if ((param.size & ~IOMMU_PAGE_MASK(tbl)) || 84200663d4eSAlexey Kardashevskiy (param.vaddr & ~IOMMU_PAGE_MASK(tbl))) 8435ffd229cSAlexey Kardashevskiy return -EINVAL; 8445ffd229cSAlexey Kardashevskiy 8455ffd229cSAlexey Kardashevskiy /* iova is checked by the IOMMU API */ 84605c6cfb9SAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_READ) { 8475ffd229cSAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE) 84805c6cfb9SAlexey Kardashevskiy direction = DMA_BIDIRECTIONAL; 84905c6cfb9SAlexey Kardashevskiy else 85005c6cfb9SAlexey Kardashevskiy direction = DMA_TO_DEVICE; 85105c6cfb9SAlexey Kardashevskiy } else { 85205c6cfb9SAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE) 85305c6cfb9SAlexey Kardashevskiy direction = DMA_FROM_DEVICE; 85405c6cfb9SAlexey Kardashevskiy else 85505c6cfb9SAlexey Kardashevskiy return -EINVAL; 85605c6cfb9SAlexey Kardashevskiy } 8575ffd229cSAlexey Kardashevskiy 85805c6cfb9SAlexey Kardashevskiy ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr); 8595ffd229cSAlexey Kardashevskiy if (ret) 8605ffd229cSAlexey Kardashevskiy return ret; 8615ffd229cSAlexey Kardashevskiy 8622157e7b8SAlexey Kardashevskiy if (container->v2) 8632157e7b8SAlexey Kardashevskiy ret = tce_iommu_build_v2(container, tbl, 8642157e7b8SAlexey Kardashevskiy param.iova >> tbl->it_page_shift, 8652157e7b8SAlexey Kardashevskiy param.vaddr, 8662157e7b8SAlexey Kardashevskiy param.size >> tbl->it_page_shift, 8672157e7b8SAlexey Kardashevskiy direction); 8682157e7b8SAlexey Kardashevskiy else 8699b14a1ffSAlexey Kardashevskiy ret = tce_iommu_build(container, tbl, 87000663d4eSAlexey Kardashevskiy param.iova >> tbl->it_page_shift, 87105c6cfb9SAlexey Kardashevskiy param.vaddr, 87205c6cfb9SAlexey Kardashevskiy param.size >> tbl->it_page_shift, 87305c6cfb9SAlexey Kardashevskiy direction); 8745ffd229cSAlexey Kardashevskiy 8755ffd229cSAlexey Kardashevskiy iommu_flush_tce(tbl); 8765ffd229cSAlexey Kardashevskiy 8775ffd229cSAlexey Kardashevskiy return ret; 8785ffd229cSAlexey Kardashevskiy } 8795ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_UNMAP_DMA: { 8805ffd229cSAlexey Kardashevskiy struct vfio_iommu_type1_dma_unmap param; 8810eaf4defSAlexey Kardashevskiy struct iommu_table *tbl = NULL; 8820eaf4defSAlexey Kardashevskiy long num; 8835ffd229cSAlexey Kardashevskiy 8843c56e822SAlexey Kardashevskiy if (!container->enabled) 8853c56e822SAlexey Kardashevskiy return -EPERM; 8863c56e822SAlexey Kardashevskiy 8875ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, 8885ffd229cSAlexey Kardashevskiy size); 8895ffd229cSAlexey Kardashevskiy 8905ffd229cSAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 8915ffd229cSAlexey Kardashevskiy return -EFAULT; 8925ffd229cSAlexey Kardashevskiy 8935ffd229cSAlexey Kardashevskiy if (param.argsz < minsz) 8945ffd229cSAlexey Kardashevskiy return -EINVAL; 8955ffd229cSAlexey Kardashevskiy 8965ffd229cSAlexey Kardashevskiy /* No flag is supported now */ 8975ffd229cSAlexey Kardashevskiy if (param.flags) 8985ffd229cSAlexey Kardashevskiy return -EINVAL; 8995ffd229cSAlexey Kardashevskiy 900d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container); 901d9c72894SAlexey Kardashevskiy if (ret) 902d9c72894SAlexey Kardashevskiy return ret; 903d9c72894SAlexey Kardashevskiy 9040eaf4defSAlexey Kardashevskiy num = tce_iommu_find_table(container, param.iova, &tbl); 9050eaf4defSAlexey Kardashevskiy if (num < 0) 9060eaf4defSAlexey Kardashevskiy return -ENXIO; 9070eaf4defSAlexey Kardashevskiy 90800663d4eSAlexey Kardashevskiy if (param.size & ~IOMMU_PAGE_MASK(tbl)) 9095ffd229cSAlexey Kardashevskiy return -EINVAL; 9105ffd229cSAlexey Kardashevskiy 9115ffd229cSAlexey Kardashevskiy ret = iommu_tce_clear_param_check(tbl, param.iova, 0, 91200663d4eSAlexey Kardashevskiy param.size >> tbl->it_page_shift); 9135ffd229cSAlexey Kardashevskiy if (ret) 9145ffd229cSAlexey Kardashevskiy return ret; 9155ffd229cSAlexey Kardashevskiy 9169b14a1ffSAlexey Kardashevskiy ret = tce_iommu_clear(container, tbl, 91700663d4eSAlexey Kardashevskiy param.iova >> tbl->it_page_shift, 91800663d4eSAlexey Kardashevskiy param.size >> tbl->it_page_shift); 9195ffd229cSAlexey Kardashevskiy iommu_flush_tce(tbl); 9205ffd229cSAlexey Kardashevskiy 9215ffd229cSAlexey Kardashevskiy return ret; 9225ffd229cSAlexey Kardashevskiy } 9232157e7b8SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: { 9242157e7b8SAlexey Kardashevskiy struct vfio_iommu_spapr_register_memory param; 9252157e7b8SAlexey Kardashevskiy 9262157e7b8SAlexey Kardashevskiy if (!container->v2) 9272157e7b8SAlexey Kardashevskiy break; 9282157e7b8SAlexey Kardashevskiy 9292157e7b8SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_register_memory, 9302157e7b8SAlexey Kardashevskiy size); 9312157e7b8SAlexey Kardashevskiy 9322157e7b8SAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 9332157e7b8SAlexey Kardashevskiy return -EFAULT; 9342157e7b8SAlexey Kardashevskiy 9352157e7b8SAlexey Kardashevskiy if (param.argsz < minsz) 9362157e7b8SAlexey Kardashevskiy return -EINVAL; 9372157e7b8SAlexey Kardashevskiy 9382157e7b8SAlexey Kardashevskiy /* No flag is supported now */ 9392157e7b8SAlexey Kardashevskiy if (param.flags) 9402157e7b8SAlexey Kardashevskiy return -EINVAL; 9412157e7b8SAlexey Kardashevskiy 9422157e7b8SAlexey Kardashevskiy mutex_lock(&container->lock); 9432157e7b8SAlexey Kardashevskiy ret = tce_iommu_register_pages(container, param.vaddr, 9442157e7b8SAlexey Kardashevskiy param.size); 9452157e7b8SAlexey Kardashevskiy mutex_unlock(&container->lock); 9462157e7b8SAlexey Kardashevskiy 9472157e7b8SAlexey Kardashevskiy return ret; 9482157e7b8SAlexey Kardashevskiy } 9492157e7b8SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: { 9502157e7b8SAlexey Kardashevskiy struct vfio_iommu_spapr_register_memory param; 9512157e7b8SAlexey Kardashevskiy 9522157e7b8SAlexey Kardashevskiy if (!container->v2) 9532157e7b8SAlexey Kardashevskiy break; 9542157e7b8SAlexey Kardashevskiy 9552157e7b8SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_register_memory, 9562157e7b8SAlexey Kardashevskiy size); 9572157e7b8SAlexey Kardashevskiy 9582157e7b8SAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 9592157e7b8SAlexey Kardashevskiy return -EFAULT; 9602157e7b8SAlexey Kardashevskiy 9612157e7b8SAlexey Kardashevskiy if (param.argsz < minsz) 9622157e7b8SAlexey Kardashevskiy return -EINVAL; 9632157e7b8SAlexey Kardashevskiy 9642157e7b8SAlexey Kardashevskiy /* No flag is supported now */ 9652157e7b8SAlexey Kardashevskiy if (param.flags) 9662157e7b8SAlexey Kardashevskiy return -EINVAL; 9672157e7b8SAlexey Kardashevskiy 9682157e7b8SAlexey Kardashevskiy mutex_lock(&container->lock); 9692157e7b8SAlexey Kardashevskiy ret = tce_iommu_unregister_pages(container, param.vaddr, 9702157e7b8SAlexey Kardashevskiy param.size); 9712157e7b8SAlexey Kardashevskiy mutex_unlock(&container->lock); 9722157e7b8SAlexey Kardashevskiy 9732157e7b8SAlexey Kardashevskiy return ret; 9742157e7b8SAlexey Kardashevskiy } 9755ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_ENABLE: 9762157e7b8SAlexey Kardashevskiy if (container->v2) 9772157e7b8SAlexey Kardashevskiy break; 9782157e7b8SAlexey Kardashevskiy 9795ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 9805ffd229cSAlexey Kardashevskiy ret = tce_iommu_enable(container); 9815ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 9825ffd229cSAlexey Kardashevskiy return ret; 9835ffd229cSAlexey Kardashevskiy 9845ffd229cSAlexey Kardashevskiy 9855ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_DISABLE: 9862157e7b8SAlexey Kardashevskiy if (container->v2) 9872157e7b8SAlexey Kardashevskiy break; 9882157e7b8SAlexey Kardashevskiy 9895ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 9905ffd229cSAlexey Kardashevskiy tce_iommu_disable(container); 9915ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 9925ffd229cSAlexey Kardashevskiy return 0; 9931b69be5eSGavin Shan 9942157e7b8SAlexey Kardashevskiy case VFIO_EEH_PE_OP: { 9952157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 9962157e7b8SAlexey Kardashevskiy 9972157e7b8SAlexey Kardashevskiy ret = 0; 9982157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 9992157e7b8SAlexey Kardashevskiy ret = vfio_spapr_iommu_eeh_ioctl(tcegrp->grp, 10001b69be5eSGavin Shan cmd, arg); 10012157e7b8SAlexey Kardashevskiy if (ret) 10022157e7b8SAlexey Kardashevskiy return ret; 10032157e7b8SAlexey Kardashevskiy } 10042157e7b8SAlexey Kardashevskiy return ret; 10052157e7b8SAlexey Kardashevskiy } 10062157e7b8SAlexey Kardashevskiy 1007e633bc86SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_CREATE: { 1008e633bc86SAlexey Kardashevskiy struct vfio_iommu_spapr_tce_create create; 1009e633bc86SAlexey Kardashevskiy 1010e633bc86SAlexey Kardashevskiy if (!container->v2) 1011e633bc86SAlexey Kardashevskiy break; 1012e633bc86SAlexey Kardashevskiy 1013e633bc86SAlexey Kardashevskiy if (!tce_groups_attached(container)) 1014e633bc86SAlexey Kardashevskiy return -ENXIO; 1015e633bc86SAlexey Kardashevskiy 1016e633bc86SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_create, 1017e633bc86SAlexey Kardashevskiy start_addr); 1018e633bc86SAlexey Kardashevskiy 1019e633bc86SAlexey Kardashevskiy if (copy_from_user(&create, (void __user *)arg, minsz)) 1020e633bc86SAlexey Kardashevskiy return -EFAULT; 1021e633bc86SAlexey Kardashevskiy 1022e633bc86SAlexey Kardashevskiy if (create.argsz < minsz) 1023e633bc86SAlexey Kardashevskiy return -EINVAL; 1024e633bc86SAlexey Kardashevskiy 1025e633bc86SAlexey Kardashevskiy if (create.flags) 1026e633bc86SAlexey Kardashevskiy return -EINVAL; 1027e633bc86SAlexey Kardashevskiy 1028e633bc86SAlexey Kardashevskiy mutex_lock(&container->lock); 1029e633bc86SAlexey Kardashevskiy 1030d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container); 1031d9c72894SAlexey Kardashevskiy if (ret) 1032d9c72894SAlexey Kardashevskiy return ret; 1033d9c72894SAlexey Kardashevskiy 1034e633bc86SAlexey Kardashevskiy ret = tce_iommu_create_window(container, create.page_shift, 1035e633bc86SAlexey Kardashevskiy create.window_size, create.levels, 1036e633bc86SAlexey Kardashevskiy &create.start_addr); 1037e633bc86SAlexey Kardashevskiy 1038e633bc86SAlexey Kardashevskiy mutex_unlock(&container->lock); 1039e633bc86SAlexey Kardashevskiy 1040e633bc86SAlexey Kardashevskiy if (!ret && copy_to_user((void __user *)arg, &create, minsz)) 1041e633bc86SAlexey Kardashevskiy ret = -EFAULT; 1042e633bc86SAlexey Kardashevskiy 1043e633bc86SAlexey Kardashevskiy return ret; 1044e633bc86SAlexey Kardashevskiy } 1045e633bc86SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_REMOVE: { 1046e633bc86SAlexey Kardashevskiy struct vfio_iommu_spapr_tce_remove remove; 1047e633bc86SAlexey Kardashevskiy 1048e633bc86SAlexey Kardashevskiy if (!container->v2) 1049e633bc86SAlexey Kardashevskiy break; 1050e633bc86SAlexey Kardashevskiy 1051e633bc86SAlexey Kardashevskiy if (!tce_groups_attached(container)) 1052e633bc86SAlexey Kardashevskiy return -ENXIO; 1053e633bc86SAlexey Kardashevskiy 1054e633bc86SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_remove, 1055e633bc86SAlexey Kardashevskiy start_addr); 1056e633bc86SAlexey Kardashevskiy 1057e633bc86SAlexey Kardashevskiy if (copy_from_user(&remove, (void __user *)arg, minsz)) 1058e633bc86SAlexey Kardashevskiy return -EFAULT; 1059e633bc86SAlexey Kardashevskiy 1060e633bc86SAlexey Kardashevskiy if (remove.argsz < minsz) 1061e633bc86SAlexey Kardashevskiy return -EINVAL; 1062e633bc86SAlexey Kardashevskiy 1063e633bc86SAlexey Kardashevskiy if (remove.flags) 1064e633bc86SAlexey Kardashevskiy return -EINVAL; 1065e633bc86SAlexey Kardashevskiy 1066d9c72894SAlexey Kardashevskiy if (container->def_window_pending && !remove.start_addr) { 1067d9c72894SAlexey Kardashevskiy container->def_window_pending = false; 1068d9c72894SAlexey Kardashevskiy return 0; 1069d9c72894SAlexey Kardashevskiy } 1070d9c72894SAlexey Kardashevskiy 1071e633bc86SAlexey Kardashevskiy mutex_lock(&container->lock); 1072e633bc86SAlexey Kardashevskiy 1073e633bc86SAlexey Kardashevskiy ret = tce_iommu_remove_window(container, remove.start_addr); 1074e633bc86SAlexey Kardashevskiy 1075e633bc86SAlexey Kardashevskiy mutex_unlock(&container->lock); 1076e633bc86SAlexey Kardashevskiy 1077e633bc86SAlexey Kardashevskiy return ret; 1078e633bc86SAlexey Kardashevskiy } 10795ffd229cSAlexey Kardashevskiy } 10805ffd229cSAlexey Kardashevskiy 10815ffd229cSAlexey Kardashevskiy return -ENOTTY; 10825ffd229cSAlexey Kardashevskiy } 10835ffd229cSAlexey Kardashevskiy 1084f87a8864SAlexey Kardashevskiy static void tce_iommu_release_ownership(struct tce_container *container, 1085f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1086f87a8864SAlexey Kardashevskiy { 1087f87a8864SAlexey Kardashevskiy int i; 1088f87a8864SAlexey Kardashevskiy 1089f87a8864SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 10902157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 1091f87a8864SAlexey Kardashevskiy 1092f87a8864SAlexey Kardashevskiy if (!tbl) 1093f87a8864SAlexey Kardashevskiy continue; 1094f87a8864SAlexey Kardashevskiy 1095f87a8864SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 10962157e7b8SAlexey Kardashevskiy tce_iommu_userspace_view_free(tbl); 1097f87a8864SAlexey Kardashevskiy if (tbl->it_map) 1098f87a8864SAlexey Kardashevskiy iommu_release_ownership(tbl); 10992157e7b8SAlexey Kardashevskiy 11002157e7b8SAlexey Kardashevskiy container->tables[i] = NULL; 1101f87a8864SAlexey Kardashevskiy } 1102f87a8864SAlexey Kardashevskiy } 1103f87a8864SAlexey Kardashevskiy 1104f87a8864SAlexey Kardashevskiy static int tce_iommu_take_ownership(struct tce_container *container, 1105f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1106f87a8864SAlexey Kardashevskiy { 1107f87a8864SAlexey Kardashevskiy int i, j, rc = 0; 1108f87a8864SAlexey Kardashevskiy 1109f87a8864SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 1110f87a8864SAlexey Kardashevskiy struct iommu_table *tbl = table_group->tables[i]; 1111f87a8864SAlexey Kardashevskiy 1112f87a8864SAlexey Kardashevskiy if (!tbl || !tbl->it_map) 1113f87a8864SAlexey Kardashevskiy continue; 1114f87a8864SAlexey Kardashevskiy 1115f87a8864SAlexey Kardashevskiy rc = iommu_take_ownership(tbl); 1116f87a8864SAlexey Kardashevskiy if (rc) { 1117f87a8864SAlexey Kardashevskiy for (j = 0; j < i; ++j) 1118f87a8864SAlexey Kardashevskiy iommu_release_ownership( 1119f87a8864SAlexey Kardashevskiy table_group->tables[j]); 1120f87a8864SAlexey Kardashevskiy 1121f87a8864SAlexey Kardashevskiy return rc; 1122f87a8864SAlexey Kardashevskiy } 1123f87a8864SAlexey Kardashevskiy } 1124f87a8864SAlexey Kardashevskiy 11252157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) 11262157e7b8SAlexey Kardashevskiy container->tables[i] = table_group->tables[i]; 11272157e7b8SAlexey Kardashevskiy 1128f87a8864SAlexey Kardashevskiy return 0; 1129f87a8864SAlexey Kardashevskiy } 1130f87a8864SAlexey Kardashevskiy 1131f87a8864SAlexey Kardashevskiy static void tce_iommu_release_ownership_ddw(struct tce_container *container, 1132f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1133f87a8864SAlexey Kardashevskiy { 113446d3e1e1SAlexey Kardashevskiy long i; 113546d3e1e1SAlexey Kardashevskiy 113646d3e1e1SAlexey Kardashevskiy if (!table_group->ops->unset_window) { 113746d3e1e1SAlexey Kardashevskiy WARN_ON_ONCE(1); 113846d3e1e1SAlexey Kardashevskiy return; 113946d3e1e1SAlexey Kardashevskiy } 114046d3e1e1SAlexey Kardashevskiy 11412157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) 114246d3e1e1SAlexey Kardashevskiy table_group->ops->unset_window(table_group, i); 114346d3e1e1SAlexey Kardashevskiy 1144f87a8864SAlexey Kardashevskiy table_group->ops->release_ownership(table_group); 1145f87a8864SAlexey Kardashevskiy } 1146f87a8864SAlexey Kardashevskiy 1147f87a8864SAlexey Kardashevskiy static long tce_iommu_take_ownership_ddw(struct tce_container *container, 1148f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1149f87a8864SAlexey Kardashevskiy { 115046d3e1e1SAlexey Kardashevskiy if (!table_group->ops->create_table || !table_group->ops->set_window || 115146d3e1e1SAlexey Kardashevskiy !table_group->ops->release_ownership) { 115246d3e1e1SAlexey Kardashevskiy WARN_ON_ONCE(1); 115346d3e1e1SAlexey Kardashevskiy return -EFAULT; 115446d3e1e1SAlexey Kardashevskiy } 115546d3e1e1SAlexey Kardashevskiy 1156f87a8864SAlexey Kardashevskiy table_group->ops->take_ownership(table_group); 1157f87a8864SAlexey Kardashevskiy 11582157e7b8SAlexey Kardashevskiy return 0; 1159f87a8864SAlexey Kardashevskiy } 1160f87a8864SAlexey Kardashevskiy 11615ffd229cSAlexey Kardashevskiy static int tce_iommu_attach_group(void *iommu_data, 11625ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group) 11635ffd229cSAlexey Kardashevskiy { 11645ffd229cSAlexey Kardashevskiy int ret; 11655ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 11660eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 11672157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp = NULL; 11685ffd229cSAlexey Kardashevskiy 11695ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 11705ffd229cSAlexey Kardashevskiy 11715ffd229cSAlexey Kardashevskiy /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n", 11725ffd229cSAlexey Kardashevskiy iommu_group_id(iommu_group), iommu_group); */ 11730eaf4defSAlexey Kardashevskiy table_group = iommu_group_get_iommudata(iommu_group); 11742157e7b8SAlexey Kardashevskiy 11752157e7b8SAlexey Kardashevskiy if (tce_groups_attached(container) && (!table_group->ops || 11762157e7b8SAlexey Kardashevskiy !table_group->ops->take_ownership || 11772157e7b8SAlexey Kardashevskiy !table_group->ops->release_ownership)) { 11782157e7b8SAlexey Kardashevskiy ret = -EBUSY; 11792157e7b8SAlexey Kardashevskiy goto unlock_exit; 11802157e7b8SAlexey Kardashevskiy } 11812157e7b8SAlexey Kardashevskiy 11822157e7b8SAlexey Kardashevskiy /* Check if new group has the same iommu_ops (i.e. compatible) */ 11832157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 11842157e7b8SAlexey Kardashevskiy struct iommu_table_group *table_group_tmp; 11852157e7b8SAlexey Kardashevskiy 11862157e7b8SAlexey Kardashevskiy if (tcegrp->grp == iommu_group) { 11872157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: Group %d is already attached\n", 11882157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group)); 11892157e7b8SAlexey Kardashevskiy ret = -EBUSY; 11902157e7b8SAlexey Kardashevskiy goto unlock_exit; 11912157e7b8SAlexey Kardashevskiy } 11922157e7b8SAlexey Kardashevskiy table_group_tmp = iommu_group_get_iommudata(tcegrp->grp); 119354de285bSAlexey Kardashevskiy if (table_group_tmp->ops->create_table != 119454de285bSAlexey Kardashevskiy table_group->ops->create_table) { 11952157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: Group %d is incompatible with group %d\n", 11962157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group), 11972157e7b8SAlexey Kardashevskiy iommu_group_id(tcegrp->grp)); 11982157e7b8SAlexey Kardashevskiy ret = -EPERM; 11992157e7b8SAlexey Kardashevskiy goto unlock_exit; 12002157e7b8SAlexey Kardashevskiy } 12012157e7b8SAlexey Kardashevskiy } 12022157e7b8SAlexey Kardashevskiy 12032157e7b8SAlexey Kardashevskiy tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL); 12042157e7b8SAlexey Kardashevskiy if (!tcegrp) { 12052157e7b8SAlexey Kardashevskiy ret = -ENOMEM; 12060eaf4defSAlexey Kardashevskiy goto unlock_exit; 12070eaf4defSAlexey Kardashevskiy } 12080eaf4defSAlexey Kardashevskiy 1209f87a8864SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->take_ownership || 12106f01cc69SAlexey Kardashevskiy !table_group->ops->release_ownership) { 1211f87a8864SAlexey Kardashevskiy ret = tce_iommu_take_ownership(container, table_group); 12126f01cc69SAlexey Kardashevskiy } else { 1213f87a8864SAlexey Kardashevskiy ret = tce_iommu_take_ownership_ddw(container, table_group); 12146f01cc69SAlexey Kardashevskiy if (!tce_groups_attached(container) && !container->tables[0]) 1215d9c72894SAlexey Kardashevskiy container->def_window_pending = true; 12166f01cc69SAlexey Kardashevskiy } 1217f87a8864SAlexey Kardashevskiy 12182157e7b8SAlexey Kardashevskiy if (!ret) { 12192157e7b8SAlexey Kardashevskiy tcegrp->grp = iommu_group; 12202157e7b8SAlexey Kardashevskiy list_add(&tcegrp->next, &container->group_list); 12212157e7b8SAlexey Kardashevskiy } 12225ffd229cSAlexey Kardashevskiy 122322af4859SAlexey Kardashevskiy unlock_exit: 12242157e7b8SAlexey Kardashevskiy if (ret && tcegrp) 12252157e7b8SAlexey Kardashevskiy kfree(tcegrp); 12262157e7b8SAlexey Kardashevskiy 12275ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 12285ffd229cSAlexey Kardashevskiy 12295ffd229cSAlexey Kardashevskiy return ret; 12305ffd229cSAlexey Kardashevskiy } 12315ffd229cSAlexey Kardashevskiy 12325ffd229cSAlexey Kardashevskiy static void tce_iommu_detach_group(void *iommu_data, 12335ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group) 12345ffd229cSAlexey Kardashevskiy { 12355ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 12360eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 12372157e7b8SAlexey Kardashevskiy bool found = false; 12382157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 12395ffd229cSAlexey Kardashevskiy 12405ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 12412157e7b8SAlexey Kardashevskiy 12422157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 12432157e7b8SAlexey Kardashevskiy if (tcegrp->grp == iommu_group) { 12442157e7b8SAlexey Kardashevskiy found = true; 12452157e7b8SAlexey Kardashevskiy break; 12462157e7b8SAlexey Kardashevskiy } 12472157e7b8SAlexey Kardashevskiy } 12482157e7b8SAlexey Kardashevskiy 12492157e7b8SAlexey Kardashevskiy if (!found) { 12502157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: detaching unattached group #%u\n", 12512157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group)); 125222af4859SAlexey Kardashevskiy goto unlock_exit; 125322af4859SAlexey Kardashevskiy } 125422af4859SAlexey Kardashevskiy 12552157e7b8SAlexey Kardashevskiy list_del(&tcegrp->next); 12562157e7b8SAlexey Kardashevskiy kfree(tcegrp); 12570eaf4defSAlexey Kardashevskiy 12580eaf4defSAlexey Kardashevskiy table_group = iommu_group_get_iommudata(iommu_group); 12590eaf4defSAlexey Kardashevskiy BUG_ON(!table_group); 12600eaf4defSAlexey Kardashevskiy 1261f87a8864SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->release_ownership) 1262f87a8864SAlexey Kardashevskiy tce_iommu_release_ownership(container, table_group); 1263f87a8864SAlexey Kardashevskiy else 1264f87a8864SAlexey Kardashevskiy tce_iommu_release_ownership_ddw(container, table_group); 126522af4859SAlexey Kardashevskiy 126622af4859SAlexey Kardashevskiy unlock_exit: 12675ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 12685ffd229cSAlexey Kardashevskiy } 12695ffd229cSAlexey Kardashevskiy 12705ffd229cSAlexey Kardashevskiy const struct vfio_iommu_driver_ops tce_iommu_driver_ops = { 12715ffd229cSAlexey Kardashevskiy .name = "iommu-vfio-powerpc", 12725ffd229cSAlexey Kardashevskiy .owner = THIS_MODULE, 12735ffd229cSAlexey Kardashevskiy .open = tce_iommu_open, 12745ffd229cSAlexey Kardashevskiy .release = tce_iommu_release, 12755ffd229cSAlexey Kardashevskiy .ioctl = tce_iommu_ioctl, 12765ffd229cSAlexey Kardashevskiy .attach_group = tce_iommu_attach_group, 12775ffd229cSAlexey Kardashevskiy .detach_group = tce_iommu_detach_group, 12785ffd229cSAlexey Kardashevskiy }; 12795ffd229cSAlexey Kardashevskiy 12805ffd229cSAlexey Kardashevskiy static int __init tce_iommu_init(void) 12815ffd229cSAlexey Kardashevskiy { 12825ffd229cSAlexey Kardashevskiy return vfio_register_iommu_driver(&tce_iommu_driver_ops); 12835ffd229cSAlexey Kardashevskiy } 12845ffd229cSAlexey Kardashevskiy 12855ffd229cSAlexey Kardashevskiy static void __exit tce_iommu_cleanup(void) 12865ffd229cSAlexey Kardashevskiy { 12875ffd229cSAlexey Kardashevskiy vfio_unregister_iommu_driver(&tce_iommu_driver_ops); 12885ffd229cSAlexey Kardashevskiy } 12895ffd229cSAlexey Kardashevskiy 12905ffd229cSAlexey Kardashevskiy module_init(tce_iommu_init); 12915ffd229cSAlexey Kardashevskiy module_exit(tce_iommu_cleanup); 12925ffd229cSAlexey Kardashevskiy 12935ffd229cSAlexey Kardashevskiy MODULE_VERSION(DRIVER_VERSION); 12945ffd229cSAlexey Kardashevskiy MODULE_LICENSE("GPL v2"); 12955ffd229cSAlexey Kardashevskiy MODULE_AUTHOR(DRIVER_AUTHOR); 12965ffd229cSAlexey Kardashevskiy MODULE_DESCRIPTION(DRIVER_DESC); 12975ffd229cSAlexey Kardashevskiy 1298