15ffd229cSAlexey Kardashevskiy /* 25ffd229cSAlexey Kardashevskiy * VFIO: IOMMU DMA mapping support for TCE on POWER 35ffd229cSAlexey Kardashevskiy * 45ffd229cSAlexey Kardashevskiy * Copyright (C) 2013 IBM Corp. All rights reserved. 55ffd229cSAlexey Kardashevskiy * Author: Alexey Kardashevskiy <aik@ozlabs.ru> 65ffd229cSAlexey Kardashevskiy * 75ffd229cSAlexey Kardashevskiy * This program is free software; you can redistribute it and/or modify 85ffd229cSAlexey Kardashevskiy * it under the terms of the GNU General Public License version 2 as 95ffd229cSAlexey Kardashevskiy * published by the Free Software Foundation. 105ffd229cSAlexey Kardashevskiy * 115ffd229cSAlexey Kardashevskiy * Derived from original vfio_iommu_type1.c: 125ffd229cSAlexey Kardashevskiy * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 135ffd229cSAlexey Kardashevskiy * Author: Alex Williamson <alex.williamson@redhat.com> 145ffd229cSAlexey Kardashevskiy */ 155ffd229cSAlexey Kardashevskiy 165ffd229cSAlexey Kardashevskiy #include <linux/module.h> 175ffd229cSAlexey Kardashevskiy #include <linux/pci.h> 185ffd229cSAlexey Kardashevskiy #include <linux/slab.h> 195ffd229cSAlexey Kardashevskiy #include <linux/uaccess.h> 205ffd229cSAlexey Kardashevskiy #include <linux/err.h> 215ffd229cSAlexey Kardashevskiy #include <linux/vfio.h> 222157e7b8SAlexey Kardashevskiy #include <linux/vmalloc.h> 236e84f315SIngo Molnar #include <linux/sched/mm.h> 243f07c014SIngo Molnar #include <linux/sched/signal.h> 256e84f315SIngo Molnar 265ffd229cSAlexey Kardashevskiy #include <asm/iommu.h> 275ffd229cSAlexey Kardashevskiy #include <asm/tce.h> 282157e7b8SAlexey Kardashevskiy #include <asm/mmu_context.h> 295ffd229cSAlexey Kardashevskiy 305ffd229cSAlexey Kardashevskiy #define DRIVER_VERSION "0.1" 315ffd229cSAlexey Kardashevskiy #define DRIVER_AUTHOR "aik@ozlabs.ru" 325ffd229cSAlexey Kardashevskiy #define DRIVER_DESC "VFIO IOMMU SPAPR TCE" 335ffd229cSAlexey Kardashevskiy 345ffd229cSAlexey Kardashevskiy static void tce_iommu_detach_group(void *iommu_data, 355ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group); 365ffd229cSAlexey Kardashevskiy 37bc82d122SAlexey Kardashevskiy static long try_increment_locked_vm(struct mm_struct *mm, long npages) 382d270df8SAlexey Kardashevskiy { 392d270df8SAlexey Kardashevskiy long ret = 0, locked, lock_limit; 402d270df8SAlexey Kardashevskiy 41bc82d122SAlexey Kardashevskiy if (WARN_ON_ONCE(!mm)) 42bc82d122SAlexey Kardashevskiy return -EPERM; 432d270df8SAlexey Kardashevskiy 442d270df8SAlexey Kardashevskiy if (!npages) 452d270df8SAlexey Kardashevskiy return 0; 462d270df8SAlexey Kardashevskiy 47bc82d122SAlexey Kardashevskiy down_write(&mm->mmap_sem); 48bc82d122SAlexey Kardashevskiy locked = mm->locked_vm + npages; 492d270df8SAlexey Kardashevskiy lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 502d270df8SAlexey Kardashevskiy if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 512d270df8SAlexey Kardashevskiy ret = -ENOMEM; 522d270df8SAlexey Kardashevskiy else 53bc82d122SAlexey Kardashevskiy mm->locked_vm += npages; 542d270df8SAlexey Kardashevskiy 552d270df8SAlexey Kardashevskiy pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid, 562d270df8SAlexey Kardashevskiy npages << PAGE_SHIFT, 57bc82d122SAlexey Kardashevskiy mm->locked_vm << PAGE_SHIFT, 582d270df8SAlexey Kardashevskiy rlimit(RLIMIT_MEMLOCK), 592d270df8SAlexey Kardashevskiy ret ? " - exceeded" : ""); 602d270df8SAlexey Kardashevskiy 61bc82d122SAlexey Kardashevskiy up_write(&mm->mmap_sem); 622d270df8SAlexey Kardashevskiy 632d270df8SAlexey Kardashevskiy return ret; 642d270df8SAlexey Kardashevskiy } 652d270df8SAlexey Kardashevskiy 66bc82d122SAlexey Kardashevskiy static void decrement_locked_vm(struct mm_struct *mm, long npages) 672d270df8SAlexey Kardashevskiy { 68bc82d122SAlexey Kardashevskiy if (!mm || !npages) 69bc82d122SAlexey Kardashevskiy return; 702d270df8SAlexey Kardashevskiy 71bc82d122SAlexey Kardashevskiy down_write(&mm->mmap_sem); 72bc82d122SAlexey Kardashevskiy if (WARN_ON_ONCE(npages > mm->locked_vm)) 73bc82d122SAlexey Kardashevskiy npages = mm->locked_vm; 74bc82d122SAlexey Kardashevskiy mm->locked_vm -= npages; 752d270df8SAlexey Kardashevskiy pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid, 762d270df8SAlexey Kardashevskiy npages << PAGE_SHIFT, 77bc82d122SAlexey Kardashevskiy mm->locked_vm << PAGE_SHIFT, 782d270df8SAlexey Kardashevskiy rlimit(RLIMIT_MEMLOCK)); 79bc82d122SAlexey Kardashevskiy up_write(&mm->mmap_sem); 802d270df8SAlexey Kardashevskiy } 812d270df8SAlexey Kardashevskiy 825ffd229cSAlexey Kardashevskiy /* 835ffd229cSAlexey Kardashevskiy * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation 845ffd229cSAlexey Kardashevskiy * 855ffd229cSAlexey Kardashevskiy * This code handles mapping and unmapping of user data buffers 865ffd229cSAlexey Kardashevskiy * into DMA'ble space using the IOMMU 875ffd229cSAlexey Kardashevskiy */ 885ffd229cSAlexey Kardashevskiy 892157e7b8SAlexey Kardashevskiy struct tce_iommu_group { 902157e7b8SAlexey Kardashevskiy struct list_head next; 912157e7b8SAlexey Kardashevskiy struct iommu_group *grp; 922157e7b8SAlexey Kardashevskiy }; 932157e7b8SAlexey Kardashevskiy 945ffd229cSAlexey Kardashevskiy /* 954b6fad70SAlexey Kardashevskiy * A container needs to remember which preregistered region it has 964b6fad70SAlexey Kardashevskiy * referenced to do proper cleanup at the userspace process exit. 974b6fad70SAlexey Kardashevskiy */ 984b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg { 994b6fad70SAlexey Kardashevskiy struct list_head next; 1004b6fad70SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem; 1014b6fad70SAlexey Kardashevskiy }; 1024b6fad70SAlexey Kardashevskiy 1034b6fad70SAlexey Kardashevskiy /* 1045ffd229cSAlexey Kardashevskiy * The container descriptor supports only a single group per container. 1055ffd229cSAlexey Kardashevskiy * Required by the API as the container is not supplied with the IOMMU group 1065ffd229cSAlexey Kardashevskiy * at the moment of initialization. 1075ffd229cSAlexey Kardashevskiy */ 1085ffd229cSAlexey Kardashevskiy struct tce_container { 1095ffd229cSAlexey Kardashevskiy struct mutex lock; 1105ffd229cSAlexey Kardashevskiy bool enabled; 1112157e7b8SAlexey Kardashevskiy bool v2; 112d9c72894SAlexey Kardashevskiy bool def_window_pending; 1132d270df8SAlexey Kardashevskiy unsigned long locked_pages; 114bc82d122SAlexey Kardashevskiy struct mm_struct *mm; 1152157e7b8SAlexey Kardashevskiy struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES]; 1162157e7b8SAlexey Kardashevskiy struct list_head group_list; 1174b6fad70SAlexey Kardashevskiy struct list_head prereg_list; 1185ffd229cSAlexey Kardashevskiy }; 1195ffd229cSAlexey Kardashevskiy 120bc82d122SAlexey Kardashevskiy static long tce_iommu_mm_set(struct tce_container *container) 121bc82d122SAlexey Kardashevskiy { 122bc82d122SAlexey Kardashevskiy if (container->mm) { 123bc82d122SAlexey Kardashevskiy if (container->mm == current->mm) 124bc82d122SAlexey Kardashevskiy return 0; 125bc82d122SAlexey Kardashevskiy return -EPERM; 126bc82d122SAlexey Kardashevskiy } 127bc82d122SAlexey Kardashevskiy BUG_ON(!current->mm); 128bc82d122SAlexey Kardashevskiy container->mm = current->mm; 129bc82d122SAlexey Kardashevskiy atomic_inc(&container->mm->mm_count); 130bc82d122SAlexey Kardashevskiy 131bc82d122SAlexey Kardashevskiy return 0; 132bc82d122SAlexey Kardashevskiy } 133bc82d122SAlexey Kardashevskiy 1344b6fad70SAlexey Kardashevskiy static long tce_iommu_prereg_free(struct tce_container *container, 1354b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem) 1364b6fad70SAlexey Kardashevskiy { 1374b6fad70SAlexey Kardashevskiy long ret; 1384b6fad70SAlexey Kardashevskiy 1394b6fad70SAlexey Kardashevskiy ret = mm_iommu_put(container->mm, tcemem->mem); 1404b6fad70SAlexey Kardashevskiy if (ret) 1414b6fad70SAlexey Kardashevskiy return ret; 1424b6fad70SAlexey Kardashevskiy 1434b6fad70SAlexey Kardashevskiy list_del(&tcemem->next); 1444b6fad70SAlexey Kardashevskiy kfree(tcemem); 1454b6fad70SAlexey Kardashevskiy 1464b6fad70SAlexey Kardashevskiy return 0; 1474b6fad70SAlexey Kardashevskiy } 1484b6fad70SAlexey Kardashevskiy 1492157e7b8SAlexey Kardashevskiy static long tce_iommu_unregister_pages(struct tce_container *container, 1502157e7b8SAlexey Kardashevskiy __u64 vaddr, __u64 size) 1512157e7b8SAlexey Kardashevskiy { 1522157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem; 1534b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem; 1544b6fad70SAlexey Kardashevskiy bool found = false; 1552157e7b8SAlexey Kardashevskiy 1562157e7b8SAlexey Kardashevskiy if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK)) 1572157e7b8SAlexey Kardashevskiy return -EINVAL; 1582157e7b8SAlexey Kardashevskiy 159bc82d122SAlexey Kardashevskiy mem = mm_iommu_find(container->mm, vaddr, size >> PAGE_SHIFT); 1602157e7b8SAlexey Kardashevskiy if (!mem) 1612157e7b8SAlexey Kardashevskiy return -ENOENT; 1622157e7b8SAlexey Kardashevskiy 1634b6fad70SAlexey Kardashevskiy list_for_each_entry(tcemem, &container->prereg_list, next) { 1644b6fad70SAlexey Kardashevskiy if (tcemem->mem == mem) { 1654b6fad70SAlexey Kardashevskiy found = true; 1664b6fad70SAlexey Kardashevskiy break; 1674b6fad70SAlexey Kardashevskiy } 1684b6fad70SAlexey Kardashevskiy } 1694b6fad70SAlexey Kardashevskiy 1704b6fad70SAlexey Kardashevskiy if (!found) 1714b6fad70SAlexey Kardashevskiy return -ENOENT; 1724b6fad70SAlexey Kardashevskiy 1734b6fad70SAlexey Kardashevskiy return tce_iommu_prereg_free(container, tcemem); 1742157e7b8SAlexey Kardashevskiy } 1752157e7b8SAlexey Kardashevskiy 1762157e7b8SAlexey Kardashevskiy static long tce_iommu_register_pages(struct tce_container *container, 1772157e7b8SAlexey Kardashevskiy __u64 vaddr, __u64 size) 1782157e7b8SAlexey Kardashevskiy { 1792157e7b8SAlexey Kardashevskiy long ret = 0; 1802157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL; 1814b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem; 1822157e7b8SAlexey Kardashevskiy unsigned long entries = size >> PAGE_SHIFT; 1832157e7b8SAlexey Kardashevskiy 1842157e7b8SAlexey Kardashevskiy if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) || 1852157e7b8SAlexey Kardashevskiy ((vaddr + size) < vaddr)) 1862157e7b8SAlexey Kardashevskiy return -EINVAL; 1872157e7b8SAlexey Kardashevskiy 1884b6fad70SAlexey Kardashevskiy mem = mm_iommu_find(container->mm, vaddr, entries); 1894b6fad70SAlexey Kardashevskiy if (mem) { 1904b6fad70SAlexey Kardashevskiy list_for_each_entry(tcemem, &container->prereg_list, next) { 1914b6fad70SAlexey Kardashevskiy if (tcemem->mem == mem) 1924b6fad70SAlexey Kardashevskiy return -EBUSY; 1934b6fad70SAlexey Kardashevskiy } 1944b6fad70SAlexey Kardashevskiy } 1954b6fad70SAlexey Kardashevskiy 196bc82d122SAlexey Kardashevskiy ret = mm_iommu_get(container->mm, vaddr, entries, &mem); 1972157e7b8SAlexey Kardashevskiy if (ret) 1982157e7b8SAlexey Kardashevskiy return ret; 1992157e7b8SAlexey Kardashevskiy 2004b6fad70SAlexey Kardashevskiy tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL); 2014b6fad70SAlexey Kardashevskiy tcemem->mem = mem; 2024b6fad70SAlexey Kardashevskiy list_add(&tcemem->next, &container->prereg_list); 2034b6fad70SAlexey Kardashevskiy 2042157e7b8SAlexey Kardashevskiy container->enabled = true; 2052157e7b8SAlexey Kardashevskiy 2062157e7b8SAlexey Kardashevskiy return 0; 2072157e7b8SAlexey Kardashevskiy } 2082157e7b8SAlexey Kardashevskiy 209bc82d122SAlexey Kardashevskiy static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl, 210bc82d122SAlexey Kardashevskiy struct mm_struct *mm) 2112157e7b8SAlexey Kardashevskiy { 2122157e7b8SAlexey Kardashevskiy unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) * 2132157e7b8SAlexey Kardashevskiy tbl->it_size, PAGE_SIZE); 2142157e7b8SAlexey Kardashevskiy unsigned long *uas; 2152157e7b8SAlexey Kardashevskiy long ret; 2162157e7b8SAlexey Kardashevskiy 2172157e7b8SAlexey Kardashevskiy BUG_ON(tbl->it_userspace); 2182157e7b8SAlexey Kardashevskiy 219bc82d122SAlexey Kardashevskiy ret = try_increment_locked_vm(mm, cb >> PAGE_SHIFT); 2202157e7b8SAlexey Kardashevskiy if (ret) 2212157e7b8SAlexey Kardashevskiy return ret; 2222157e7b8SAlexey Kardashevskiy 2232157e7b8SAlexey Kardashevskiy uas = vzalloc(cb); 2242157e7b8SAlexey Kardashevskiy if (!uas) { 225bc82d122SAlexey Kardashevskiy decrement_locked_vm(mm, cb >> PAGE_SHIFT); 2262157e7b8SAlexey Kardashevskiy return -ENOMEM; 2272157e7b8SAlexey Kardashevskiy } 2282157e7b8SAlexey Kardashevskiy tbl->it_userspace = uas; 2292157e7b8SAlexey Kardashevskiy 2302157e7b8SAlexey Kardashevskiy return 0; 2312157e7b8SAlexey Kardashevskiy } 2322157e7b8SAlexey Kardashevskiy 233bc82d122SAlexey Kardashevskiy static void tce_iommu_userspace_view_free(struct iommu_table *tbl, 234bc82d122SAlexey Kardashevskiy struct mm_struct *mm) 2352157e7b8SAlexey Kardashevskiy { 2362157e7b8SAlexey Kardashevskiy unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) * 2372157e7b8SAlexey Kardashevskiy tbl->it_size, PAGE_SIZE); 2382157e7b8SAlexey Kardashevskiy 2392157e7b8SAlexey Kardashevskiy if (!tbl->it_userspace) 2402157e7b8SAlexey Kardashevskiy return; 2412157e7b8SAlexey Kardashevskiy 2422157e7b8SAlexey Kardashevskiy vfree(tbl->it_userspace); 2432157e7b8SAlexey Kardashevskiy tbl->it_userspace = NULL; 244bc82d122SAlexey Kardashevskiy decrement_locked_vm(mm, cb >> PAGE_SHIFT); 2452157e7b8SAlexey Kardashevskiy } 2462157e7b8SAlexey Kardashevskiy 247e432bc7eSAlexey Kardashevskiy static bool tce_page_is_contained(struct page *page, unsigned page_shift) 248e432bc7eSAlexey Kardashevskiy { 249e432bc7eSAlexey Kardashevskiy /* 250e432bc7eSAlexey Kardashevskiy * Check that the TCE table granularity is not bigger than the size of 251e432bc7eSAlexey Kardashevskiy * a page we just found. Otherwise the hardware can get access to 252e432bc7eSAlexey Kardashevskiy * a bigger memory chunk that it should. 253e432bc7eSAlexey Kardashevskiy */ 254e432bc7eSAlexey Kardashevskiy return (PAGE_SHIFT + compound_order(compound_head(page))) >= page_shift; 255e432bc7eSAlexey Kardashevskiy } 256e432bc7eSAlexey Kardashevskiy 2572157e7b8SAlexey Kardashevskiy static inline bool tce_groups_attached(struct tce_container *container) 2582157e7b8SAlexey Kardashevskiy { 2592157e7b8SAlexey Kardashevskiy return !list_empty(&container->group_list); 2602157e7b8SAlexey Kardashevskiy } 2612157e7b8SAlexey Kardashevskiy 2620eaf4defSAlexey Kardashevskiy static long tce_iommu_find_table(struct tce_container *container, 2630eaf4defSAlexey Kardashevskiy phys_addr_t ioba, struct iommu_table **ptbl) 2640eaf4defSAlexey Kardashevskiy { 2650eaf4defSAlexey Kardashevskiy long i; 2660eaf4defSAlexey Kardashevskiy 2670eaf4defSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 2682157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 2690eaf4defSAlexey Kardashevskiy 2700eaf4defSAlexey Kardashevskiy if (tbl) { 2710eaf4defSAlexey Kardashevskiy unsigned long entry = ioba >> tbl->it_page_shift; 2720eaf4defSAlexey Kardashevskiy unsigned long start = tbl->it_offset; 2730eaf4defSAlexey Kardashevskiy unsigned long end = start + tbl->it_size; 2740eaf4defSAlexey Kardashevskiy 2750eaf4defSAlexey Kardashevskiy if ((start <= entry) && (entry < end)) { 2760eaf4defSAlexey Kardashevskiy *ptbl = tbl; 2770eaf4defSAlexey Kardashevskiy return i; 2780eaf4defSAlexey Kardashevskiy } 2790eaf4defSAlexey Kardashevskiy } 2800eaf4defSAlexey Kardashevskiy } 2810eaf4defSAlexey Kardashevskiy 2820eaf4defSAlexey Kardashevskiy return -1; 2830eaf4defSAlexey Kardashevskiy } 2840eaf4defSAlexey Kardashevskiy 285e633bc86SAlexey Kardashevskiy static int tce_iommu_find_free_table(struct tce_container *container) 286e633bc86SAlexey Kardashevskiy { 287e633bc86SAlexey Kardashevskiy int i; 288e633bc86SAlexey Kardashevskiy 289e633bc86SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 290e633bc86SAlexey Kardashevskiy if (!container->tables[i]) 291e633bc86SAlexey Kardashevskiy return i; 292e633bc86SAlexey Kardashevskiy } 293e633bc86SAlexey Kardashevskiy 294e633bc86SAlexey Kardashevskiy return -ENOSPC; 295e633bc86SAlexey Kardashevskiy } 296e633bc86SAlexey Kardashevskiy 2975ffd229cSAlexey Kardashevskiy static int tce_iommu_enable(struct tce_container *container) 2985ffd229cSAlexey Kardashevskiy { 2995ffd229cSAlexey Kardashevskiy int ret = 0; 3002d270df8SAlexey Kardashevskiy unsigned long locked; 3010eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 3022157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 3035ffd229cSAlexey Kardashevskiy 3045ffd229cSAlexey Kardashevskiy if (container->enabled) 3055ffd229cSAlexey Kardashevskiy return -EBUSY; 3065ffd229cSAlexey Kardashevskiy 3075ffd229cSAlexey Kardashevskiy /* 3085ffd229cSAlexey Kardashevskiy * When userspace pages are mapped into the IOMMU, they are effectively 3095ffd229cSAlexey Kardashevskiy * locked memory, so, theoretically, we need to update the accounting 3105ffd229cSAlexey Kardashevskiy * of locked pages on each map and unmap. For powerpc, the map unmap 3115ffd229cSAlexey Kardashevskiy * paths can be very hot, though, and the accounting would kill 3125ffd229cSAlexey Kardashevskiy * performance, especially since it would be difficult to impossible 3135ffd229cSAlexey Kardashevskiy * to handle the accounting in real mode only. 3145ffd229cSAlexey Kardashevskiy * 3155ffd229cSAlexey Kardashevskiy * To address that, rather than precisely accounting every page, we 3165ffd229cSAlexey Kardashevskiy * instead account for a worst case on locked memory when the iommu is 3175ffd229cSAlexey Kardashevskiy * enabled and disabled. The worst case upper bound on locked memory 3185ffd229cSAlexey Kardashevskiy * is the size of the whole iommu window, which is usually relatively 3195ffd229cSAlexey Kardashevskiy * small (compared to total memory sizes) on POWER hardware. 3205ffd229cSAlexey Kardashevskiy * 3215ffd229cSAlexey Kardashevskiy * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits, 3225ffd229cSAlexey Kardashevskiy * that would effectively kill the guest at random points, much better 3235ffd229cSAlexey Kardashevskiy * enforcing the limit based on the max that the guest can map. 3242d270df8SAlexey Kardashevskiy * 3252d270df8SAlexey Kardashevskiy * Unfortunately at the moment it counts whole tables, no matter how 3262d270df8SAlexey Kardashevskiy * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups 3272d270df8SAlexey Kardashevskiy * each with 2GB DMA window, 8GB will be counted here. The reason for 3282d270df8SAlexey Kardashevskiy * this is that we cannot tell here the amount of RAM used by the guest 3292d270df8SAlexey Kardashevskiy * as this information is only available from KVM and VFIO is 3302d270df8SAlexey Kardashevskiy * KVM agnostic. 3314793d65dSAlexey Kardashevskiy * 3324793d65dSAlexey Kardashevskiy * So we do not allow enabling a container without a group attached 3334793d65dSAlexey Kardashevskiy * as there is no way to know how much we should increment 3344793d65dSAlexey Kardashevskiy * the locked_vm counter. 3355ffd229cSAlexey Kardashevskiy */ 3362157e7b8SAlexey Kardashevskiy if (!tce_groups_attached(container)) 3372157e7b8SAlexey Kardashevskiy return -ENODEV; 3382157e7b8SAlexey Kardashevskiy 3392157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 3402157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next); 3412157e7b8SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 3420eaf4defSAlexey Kardashevskiy if (!table_group) 3430eaf4defSAlexey Kardashevskiy return -ENODEV; 3440eaf4defSAlexey Kardashevskiy 3454793d65dSAlexey Kardashevskiy if (!table_group->tce32_size) 3464793d65dSAlexey Kardashevskiy return -EPERM; 3474793d65dSAlexey Kardashevskiy 348bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container); 349bc82d122SAlexey Kardashevskiy if (ret) 350bc82d122SAlexey Kardashevskiy return ret; 351bc82d122SAlexey Kardashevskiy 3524793d65dSAlexey Kardashevskiy locked = table_group->tce32_size >> PAGE_SHIFT; 353bc82d122SAlexey Kardashevskiy ret = try_increment_locked_vm(container->mm, locked); 3542d270df8SAlexey Kardashevskiy if (ret) 3552d270df8SAlexey Kardashevskiy return ret; 3565ffd229cSAlexey Kardashevskiy 3572d270df8SAlexey Kardashevskiy container->locked_pages = locked; 3582d270df8SAlexey Kardashevskiy 3595ffd229cSAlexey Kardashevskiy container->enabled = true; 3605ffd229cSAlexey Kardashevskiy 3615ffd229cSAlexey Kardashevskiy return ret; 3625ffd229cSAlexey Kardashevskiy } 3635ffd229cSAlexey Kardashevskiy 3645ffd229cSAlexey Kardashevskiy static void tce_iommu_disable(struct tce_container *container) 3655ffd229cSAlexey Kardashevskiy { 3665ffd229cSAlexey Kardashevskiy if (!container->enabled) 3675ffd229cSAlexey Kardashevskiy return; 3685ffd229cSAlexey Kardashevskiy 3695ffd229cSAlexey Kardashevskiy container->enabled = false; 3705ffd229cSAlexey Kardashevskiy 371bc82d122SAlexey Kardashevskiy BUG_ON(!container->mm); 372bc82d122SAlexey Kardashevskiy decrement_locked_vm(container->mm, container->locked_pages); 3735ffd229cSAlexey Kardashevskiy } 3745ffd229cSAlexey Kardashevskiy 3755ffd229cSAlexey Kardashevskiy static void *tce_iommu_open(unsigned long arg) 3765ffd229cSAlexey Kardashevskiy { 3775ffd229cSAlexey Kardashevskiy struct tce_container *container; 3785ffd229cSAlexey Kardashevskiy 3792157e7b8SAlexey Kardashevskiy if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) { 3805ffd229cSAlexey Kardashevskiy pr_err("tce_vfio: Wrong IOMMU type\n"); 3815ffd229cSAlexey Kardashevskiy return ERR_PTR(-EINVAL); 3825ffd229cSAlexey Kardashevskiy } 3835ffd229cSAlexey Kardashevskiy 3845ffd229cSAlexey Kardashevskiy container = kzalloc(sizeof(*container), GFP_KERNEL); 3855ffd229cSAlexey Kardashevskiy if (!container) 3865ffd229cSAlexey Kardashevskiy return ERR_PTR(-ENOMEM); 3875ffd229cSAlexey Kardashevskiy 3885ffd229cSAlexey Kardashevskiy mutex_init(&container->lock); 3892157e7b8SAlexey Kardashevskiy INIT_LIST_HEAD_RCU(&container->group_list); 3904b6fad70SAlexey Kardashevskiy INIT_LIST_HEAD_RCU(&container->prereg_list); 3912157e7b8SAlexey Kardashevskiy 3922157e7b8SAlexey Kardashevskiy container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU; 3935ffd229cSAlexey Kardashevskiy 3945ffd229cSAlexey Kardashevskiy return container; 3955ffd229cSAlexey Kardashevskiy } 3965ffd229cSAlexey Kardashevskiy 3972157e7b8SAlexey Kardashevskiy static int tce_iommu_clear(struct tce_container *container, 3982157e7b8SAlexey Kardashevskiy struct iommu_table *tbl, 3992157e7b8SAlexey Kardashevskiy unsigned long entry, unsigned long pages); 400bc82d122SAlexey Kardashevskiy static void tce_iommu_free_table(struct tce_container *container, 401bc82d122SAlexey Kardashevskiy struct iommu_table *tbl); 4022157e7b8SAlexey Kardashevskiy 4035ffd229cSAlexey Kardashevskiy static void tce_iommu_release(void *iommu_data) 4045ffd229cSAlexey Kardashevskiy { 4055ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 4062157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 4072157e7b8SAlexey Kardashevskiy long i; 4085ffd229cSAlexey Kardashevskiy 4092157e7b8SAlexey Kardashevskiy while (tce_groups_attached(container)) { 4102157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 4112157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next); 4122157e7b8SAlexey Kardashevskiy tce_iommu_detach_group(iommu_data, tcegrp->grp); 4132157e7b8SAlexey Kardashevskiy } 4145ffd229cSAlexey Kardashevskiy 4152157e7b8SAlexey Kardashevskiy /* 4162157e7b8SAlexey Kardashevskiy * If VFIO created a table, it was not disposed 4172157e7b8SAlexey Kardashevskiy * by tce_iommu_detach_group() so do it now. 4182157e7b8SAlexey Kardashevskiy */ 4192157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 4202157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 4212157e7b8SAlexey Kardashevskiy 4222157e7b8SAlexey Kardashevskiy if (!tbl) 4232157e7b8SAlexey Kardashevskiy continue; 4242157e7b8SAlexey Kardashevskiy 4252157e7b8SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 426bc82d122SAlexey Kardashevskiy tce_iommu_free_table(container, tbl); 4272157e7b8SAlexey Kardashevskiy } 4285ffd229cSAlexey Kardashevskiy 4294b6fad70SAlexey Kardashevskiy while (!list_empty(&container->prereg_list)) { 4304b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg *tcemem; 4314b6fad70SAlexey Kardashevskiy 4324b6fad70SAlexey Kardashevskiy tcemem = list_first_entry(&container->prereg_list, 4334b6fad70SAlexey Kardashevskiy struct tce_iommu_prereg, next); 4344b6fad70SAlexey Kardashevskiy WARN_ON_ONCE(tce_iommu_prereg_free(container, tcemem)); 4354b6fad70SAlexey Kardashevskiy } 4364b6fad70SAlexey Kardashevskiy 437649354b7SAlexey Kardashevskiy tce_iommu_disable(container); 438bc82d122SAlexey Kardashevskiy if (container->mm) 439bc82d122SAlexey Kardashevskiy mmdrop(container->mm); 4405ffd229cSAlexey Kardashevskiy mutex_destroy(&container->lock); 4415ffd229cSAlexey Kardashevskiy 4425ffd229cSAlexey Kardashevskiy kfree(container); 4435ffd229cSAlexey Kardashevskiy } 4445ffd229cSAlexey Kardashevskiy 445649354b7SAlexey Kardashevskiy static void tce_iommu_unuse_page(struct tce_container *container, 44605c6cfb9SAlexey Kardashevskiy unsigned long hpa) 447649354b7SAlexey Kardashevskiy { 448649354b7SAlexey Kardashevskiy struct page *page; 449649354b7SAlexey Kardashevskiy 45005c6cfb9SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT); 451649354b7SAlexey Kardashevskiy put_page(page); 452649354b7SAlexey Kardashevskiy } 453649354b7SAlexey Kardashevskiy 454bc82d122SAlexey Kardashevskiy static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container, 455bc82d122SAlexey Kardashevskiy unsigned long tce, unsigned long size, 4562157e7b8SAlexey Kardashevskiy unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem) 4572157e7b8SAlexey Kardashevskiy { 4582157e7b8SAlexey Kardashevskiy long ret = 0; 4592157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem; 4602157e7b8SAlexey Kardashevskiy 461bc82d122SAlexey Kardashevskiy mem = mm_iommu_lookup(container->mm, tce, size); 4622157e7b8SAlexey Kardashevskiy if (!mem) 4632157e7b8SAlexey Kardashevskiy return -EINVAL; 4642157e7b8SAlexey Kardashevskiy 4652157e7b8SAlexey Kardashevskiy ret = mm_iommu_ua_to_hpa(mem, tce, phpa); 4662157e7b8SAlexey Kardashevskiy if (ret) 4672157e7b8SAlexey Kardashevskiy return -EINVAL; 4682157e7b8SAlexey Kardashevskiy 4692157e7b8SAlexey Kardashevskiy *pmem = mem; 4702157e7b8SAlexey Kardashevskiy 4712157e7b8SAlexey Kardashevskiy return 0; 4722157e7b8SAlexey Kardashevskiy } 4732157e7b8SAlexey Kardashevskiy 474bc82d122SAlexey Kardashevskiy static void tce_iommu_unuse_page_v2(struct tce_container *container, 475bc82d122SAlexey Kardashevskiy struct iommu_table *tbl, unsigned long entry) 4762157e7b8SAlexey Kardashevskiy { 4772157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL; 4782157e7b8SAlexey Kardashevskiy int ret; 4792157e7b8SAlexey Kardashevskiy unsigned long hpa = 0; 4802157e7b8SAlexey Kardashevskiy unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 4812157e7b8SAlexey Kardashevskiy 482bc82d122SAlexey Kardashevskiy if (!pua) 4832157e7b8SAlexey Kardashevskiy return; 4842157e7b8SAlexey Kardashevskiy 485bc82d122SAlexey Kardashevskiy ret = tce_iommu_prereg_ua_to_hpa(container, *pua, IOMMU_PAGE_SIZE(tbl), 4862157e7b8SAlexey Kardashevskiy &hpa, &mem); 4872157e7b8SAlexey Kardashevskiy if (ret) 4882157e7b8SAlexey Kardashevskiy pr_debug("%s: tce %lx at #%lx was not cached, ret=%d\n", 4892157e7b8SAlexey Kardashevskiy __func__, *pua, entry, ret); 4902157e7b8SAlexey Kardashevskiy if (mem) 4912157e7b8SAlexey Kardashevskiy mm_iommu_mapped_dec(mem); 4922157e7b8SAlexey Kardashevskiy 4932157e7b8SAlexey Kardashevskiy *pua = 0; 4942157e7b8SAlexey Kardashevskiy } 4952157e7b8SAlexey Kardashevskiy 4969b14a1ffSAlexey Kardashevskiy static int tce_iommu_clear(struct tce_container *container, 4979b14a1ffSAlexey Kardashevskiy struct iommu_table *tbl, 4989b14a1ffSAlexey Kardashevskiy unsigned long entry, unsigned long pages) 4999b14a1ffSAlexey Kardashevskiy { 50005c6cfb9SAlexey Kardashevskiy unsigned long oldhpa; 50105c6cfb9SAlexey Kardashevskiy long ret; 50205c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction; 5039b14a1ffSAlexey Kardashevskiy 5049b14a1ffSAlexey Kardashevskiy for ( ; pages; --pages, ++entry) { 50505c6cfb9SAlexey Kardashevskiy direction = DMA_NONE; 50605c6cfb9SAlexey Kardashevskiy oldhpa = 0; 50705c6cfb9SAlexey Kardashevskiy ret = iommu_tce_xchg(tbl, entry, &oldhpa, &direction); 50805c6cfb9SAlexey Kardashevskiy if (ret) 5099b14a1ffSAlexey Kardashevskiy continue; 5109b14a1ffSAlexey Kardashevskiy 51105c6cfb9SAlexey Kardashevskiy if (direction == DMA_NONE) 51205c6cfb9SAlexey Kardashevskiy continue; 51305c6cfb9SAlexey Kardashevskiy 5142157e7b8SAlexey Kardashevskiy if (container->v2) { 515bc82d122SAlexey Kardashevskiy tce_iommu_unuse_page_v2(container, tbl, entry); 5162157e7b8SAlexey Kardashevskiy continue; 5172157e7b8SAlexey Kardashevskiy } 5182157e7b8SAlexey Kardashevskiy 51905c6cfb9SAlexey Kardashevskiy tce_iommu_unuse_page(container, oldhpa); 5209b14a1ffSAlexey Kardashevskiy } 521649354b7SAlexey Kardashevskiy 522649354b7SAlexey Kardashevskiy return 0; 5239b14a1ffSAlexey Kardashevskiy } 5249b14a1ffSAlexey Kardashevskiy 525649354b7SAlexey Kardashevskiy static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa) 526649354b7SAlexey Kardashevskiy { 527649354b7SAlexey Kardashevskiy struct page *page = NULL; 528649354b7SAlexey Kardashevskiy enum dma_data_direction direction = iommu_tce_direction(tce); 529649354b7SAlexey Kardashevskiy 530649354b7SAlexey Kardashevskiy if (get_user_pages_fast(tce & PAGE_MASK, 1, 531649354b7SAlexey Kardashevskiy direction != DMA_TO_DEVICE, &page) != 1) 532649354b7SAlexey Kardashevskiy return -EFAULT; 533649354b7SAlexey Kardashevskiy 534649354b7SAlexey Kardashevskiy *hpa = __pa((unsigned long) page_address(page)); 535649354b7SAlexey Kardashevskiy 5369b14a1ffSAlexey Kardashevskiy return 0; 5379b14a1ffSAlexey Kardashevskiy } 5389b14a1ffSAlexey Kardashevskiy 5399b14a1ffSAlexey Kardashevskiy static long tce_iommu_build(struct tce_container *container, 5409b14a1ffSAlexey Kardashevskiy struct iommu_table *tbl, 54105c6cfb9SAlexey Kardashevskiy unsigned long entry, unsigned long tce, unsigned long pages, 54205c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction) 5439b14a1ffSAlexey Kardashevskiy { 5449b14a1ffSAlexey Kardashevskiy long i, ret = 0; 545649354b7SAlexey Kardashevskiy struct page *page; 546649354b7SAlexey Kardashevskiy unsigned long hpa; 54705c6cfb9SAlexey Kardashevskiy enum dma_data_direction dirtmp; 5489b14a1ffSAlexey Kardashevskiy 5499b14a1ffSAlexey Kardashevskiy for (i = 0; i < pages; ++i) { 5509b14a1ffSAlexey Kardashevskiy unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK; 5519b14a1ffSAlexey Kardashevskiy 552649354b7SAlexey Kardashevskiy ret = tce_iommu_use_page(tce, &hpa); 553649354b7SAlexey Kardashevskiy if (ret) 5549b14a1ffSAlexey Kardashevskiy break; 555e432bc7eSAlexey Kardashevskiy 556649354b7SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT); 557e432bc7eSAlexey Kardashevskiy if (!tce_page_is_contained(page, tbl->it_page_shift)) { 558e432bc7eSAlexey Kardashevskiy ret = -EPERM; 559e432bc7eSAlexey Kardashevskiy break; 560e432bc7eSAlexey Kardashevskiy } 561e432bc7eSAlexey Kardashevskiy 562649354b7SAlexey Kardashevskiy hpa |= offset; 56305c6cfb9SAlexey Kardashevskiy dirtmp = direction; 56405c6cfb9SAlexey Kardashevskiy ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp); 5659b14a1ffSAlexey Kardashevskiy if (ret) { 566649354b7SAlexey Kardashevskiy tce_iommu_unuse_page(container, hpa); 5679b14a1ffSAlexey Kardashevskiy pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n", 5689b14a1ffSAlexey Kardashevskiy __func__, entry << tbl->it_page_shift, 5699b14a1ffSAlexey Kardashevskiy tce, ret); 5709b14a1ffSAlexey Kardashevskiy break; 5719b14a1ffSAlexey Kardashevskiy } 57205c6cfb9SAlexey Kardashevskiy 57305c6cfb9SAlexey Kardashevskiy if (dirtmp != DMA_NONE) 57405c6cfb9SAlexey Kardashevskiy tce_iommu_unuse_page(container, hpa); 57505c6cfb9SAlexey Kardashevskiy 57600663d4eSAlexey Kardashevskiy tce += IOMMU_PAGE_SIZE(tbl); 5779b14a1ffSAlexey Kardashevskiy } 5789b14a1ffSAlexey Kardashevskiy 5799b14a1ffSAlexey Kardashevskiy if (ret) 5809b14a1ffSAlexey Kardashevskiy tce_iommu_clear(container, tbl, entry, i); 5819b14a1ffSAlexey Kardashevskiy 5829b14a1ffSAlexey Kardashevskiy return ret; 5839b14a1ffSAlexey Kardashevskiy } 5849b14a1ffSAlexey Kardashevskiy 5852157e7b8SAlexey Kardashevskiy static long tce_iommu_build_v2(struct tce_container *container, 5862157e7b8SAlexey Kardashevskiy struct iommu_table *tbl, 5872157e7b8SAlexey Kardashevskiy unsigned long entry, unsigned long tce, unsigned long pages, 5882157e7b8SAlexey Kardashevskiy enum dma_data_direction direction) 5892157e7b8SAlexey Kardashevskiy { 5902157e7b8SAlexey Kardashevskiy long i, ret = 0; 5912157e7b8SAlexey Kardashevskiy struct page *page; 5922157e7b8SAlexey Kardashevskiy unsigned long hpa; 5932157e7b8SAlexey Kardashevskiy enum dma_data_direction dirtmp; 5942157e7b8SAlexey Kardashevskiy 59539701e56SAlexey Kardashevskiy if (!tbl->it_userspace) { 596bc82d122SAlexey Kardashevskiy ret = tce_iommu_userspace_view_alloc(tbl, container->mm); 59739701e56SAlexey Kardashevskiy if (ret) 59839701e56SAlexey Kardashevskiy return ret; 59939701e56SAlexey Kardashevskiy } 60039701e56SAlexey Kardashevskiy 6012157e7b8SAlexey Kardashevskiy for (i = 0; i < pages; ++i) { 6022157e7b8SAlexey Kardashevskiy struct mm_iommu_table_group_mem_t *mem = NULL; 6032157e7b8SAlexey Kardashevskiy unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, 6042157e7b8SAlexey Kardashevskiy entry + i); 6052157e7b8SAlexey Kardashevskiy 606bc82d122SAlexey Kardashevskiy ret = tce_iommu_prereg_ua_to_hpa(container, 607bc82d122SAlexey Kardashevskiy tce, IOMMU_PAGE_SIZE(tbl), &hpa, &mem); 6082157e7b8SAlexey Kardashevskiy if (ret) 6092157e7b8SAlexey Kardashevskiy break; 6102157e7b8SAlexey Kardashevskiy 6112157e7b8SAlexey Kardashevskiy page = pfn_to_page(hpa >> PAGE_SHIFT); 6122157e7b8SAlexey Kardashevskiy if (!tce_page_is_contained(page, tbl->it_page_shift)) { 6132157e7b8SAlexey Kardashevskiy ret = -EPERM; 6142157e7b8SAlexey Kardashevskiy break; 6152157e7b8SAlexey Kardashevskiy } 6162157e7b8SAlexey Kardashevskiy 6172157e7b8SAlexey Kardashevskiy /* Preserve offset within IOMMU page */ 6182157e7b8SAlexey Kardashevskiy hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK; 6192157e7b8SAlexey Kardashevskiy dirtmp = direction; 6202157e7b8SAlexey Kardashevskiy 6212157e7b8SAlexey Kardashevskiy /* The registered region is being unregistered */ 6222157e7b8SAlexey Kardashevskiy if (mm_iommu_mapped_inc(mem)) 6232157e7b8SAlexey Kardashevskiy break; 6242157e7b8SAlexey Kardashevskiy 6252157e7b8SAlexey Kardashevskiy ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp); 6262157e7b8SAlexey Kardashevskiy if (ret) { 6272157e7b8SAlexey Kardashevskiy /* dirtmp cannot be DMA_NONE here */ 628bc82d122SAlexey Kardashevskiy tce_iommu_unuse_page_v2(container, tbl, entry + i); 6292157e7b8SAlexey Kardashevskiy pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n", 6302157e7b8SAlexey Kardashevskiy __func__, entry << tbl->it_page_shift, 6312157e7b8SAlexey Kardashevskiy tce, ret); 6322157e7b8SAlexey Kardashevskiy break; 6332157e7b8SAlexey Kardashevskiy } 6342157e7b8SAlexey Kardashevskiy 6352157e7b8SAlexey Kardashevskiy if (dirtmp != DMA_NONE) 636bc82d122SAlexey Kardashevskiy tce_iommu_unuse_page_v2(container, tbl, entry + i); 6372157e7b8SAlexey Kardashevskiy 6382157e7b8SAlexey Kardashevskiy *pua = tce; 6392157e7b8SAlexey Kardashevskiy 6402157e7b8SAlexey Kardashevskiy tce += IOMMU_PAGE_SIZE(tbl); 6412157e7b8SAlexey Kardashevskiy } 6422157e7b8SAlexey Kardashevskiy 6432157e7b8SAlexey Kardashevskiy if (ret) 6442157e7b8SAlexey Kardashevskiy tce_iommu_clear(container, tbl, entry, i); 6452157e7b8SAlexey Kardashevskiy 6462157e7b8SAlexey Kardashevskiy return ret; 6472157e7b8SAlexey Kardashevskiy } 6482157e7b8SAlexey Kardashevskiy 64946d3e1e1SAlexey Kardashevskiy static long tce_iommu_create_table(struct tce_container *container, 65046d3e1e1SAlexey Kardashevskiy struct iommu_table_group *table_group, 65146d3e1e1SAlexey Kardashevskiy int num, 65246d3e1e1SAlexey Kardashevskiy __u32 page_shift, 65346d3e1e1SAlexey Kardashevskiy __u64 window_size, 65446d3e1e1SAlexey Kardashevskiy __u32 levels, 65546d3e1e1SAlexey Kardashevskiy struct iommu_table **ptbl) 65646d3e1e1SAlexey Kardashevskiy { 65746d3e1e1SAlexey Kardashevskiy long ret, table_size; 65846d3e1e1SAlexey Kardashevskiy 65946d3e1e1SAlexey Kardashevskiy table_size = table_group->ops->get_table_size(page_shift, window_size, 66046d3e1e1SAlexey Kardashevskiy levels); 66146d3e1e1SAlexey Kardashevskiy if (!table_size) 66246d3e1e1SAlexey Kardashevskiy return -EINVAL; 66346d3e1e1SAlexey Kardashevskiy 664bc82d122SAlexey Kardashevskiy ret = try_increment_locked_vm(container->mm, table_size >> PAGE_SHIFT); 66546d3e1e1SAlexey Kardashevskiy if (ret) 66646d3e1e1SAlexey Kardashevskiy return ret; 66746d3e1e1SAlexey Kardashevskiy 66846d3e1e1SAlexey Kardashevskiy ret = table_group->ops->create_table(table_group, num, 66946d3e1e1SAlexey Kardashevskiy page_shift, window_size, levels, ptbl); 67046d3e1e1SAlexey Kardashevskiy 67146d3e1e1SAlexey Kardashevskiy WARN_ON(!ret && !(*ptbl)->it_ops->free); 67246d3e1e1SAlexey Kardashevskiy WARN_ON(!ret && ((*ptbl)->it_allocated_size != table_size)); 67346d3e1e1SAlexey Kardashevskiy 67446d3e1e1SAlexey Kardashevskiy return ret; 67546d3e1e1SAlexey Kardashevskiy } 67646d3e1e1SAlexey Kardashevskiy 677bc82d122SAlexey Kardashevskiy static void tce_iommu_free_table(struct tce_container *container, 678bc82d122SAlexey Kardashevskiy struct iommu_table *tbl) 67946d3e1e1SAlexey Kardashevskiy { 68046d3e1e1SAlexey Kardashevskiy unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT; 68146d3e1e1SAlexey Kardashevskiy 682bc82d122SAlexey Kardashevskiy tce_iommu_userspace_view_free(tbl, container->mm); 683e5afdf9dSAlexey Kardashevskiy iommu_tce_table_put(tbl); 684bc82d122SAlexey Kardashevskiy decrement_locked_vm(container->mm, pages); 68546d3e1e1SAlexey Kardashevskiy } 68646d3e1e1SAlexey Kardashevskiy 687e633bc86SAlexey Kardashevskiy static long tce_iommu_create_window(struct tce_container *container, 688e633bc86SAlexey Kardashevskiy __u32 page_shift, __u64 window_size, __u32 levels, 689e633bc86SAlexey Kardashevskiy __u64 *start_addr) 690e633bc86SAlexey Kardashevskiy { 691e633bc86SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 692e633bc86SAlexey Kardashevskiy struct iommu_table_group *table_group; 693e633bc86SAlexey Kardashevskiy struct iommu_table *tbl = NULL; 694e633bc86SAlexey Kardashevskiy long ret, num; 695e633bc86SAlexey Kardashevskiy 696e633bc86SAlexey Kardashevskiy num = tce_iommu_find_free_table(container); 697e633bc86SAlexey Kardashevskiy if (num < 0) 698e633bc86SAlexey Kardashevskiy return num; 699e633bc86SAlexey Kardashevskiy 700e633bc86SAlexey Kardashevskiy /* Get the first group for ops::create_table */ 701e633bc86SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 702e633bc86SAlexey Kardashevskiy struct tce_iommu_group, next); 703e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 704e633bc86SAlexey Kardashevskiy if (!table_group) 705e633bc86SAlexey Kardashevskiy return -EFAULT; 706e633bc86SAlexey Kardashevskiy 707e633bc86SAlexey Kardashevskiy if (!(table_group->pgsizes & (1ULL << page_shift))) 708e633bc86SAlexey Kardashevskiy return -EINVAL; 709e633bc86SAlexey Kardashevskiy 710e633bc86SAlexey Kardashevskiy if (!table_group->ops->set_window || !table_group->ops->unset_window || 711e633bc86SAlexey Kardashevskiy !table_group->ops->get_table_size || 712e633bc86SAlexey Kardashevskiy !table_group->ops->create_table) 713e633bc86SAlexey Kardashevskiy return -EPERM; 714e633bc86SAlexey Kardashevskiy 715e633bc86SAlexey Kardashevskiy /* Create TCE table */ 716e633bc86SAlexey Kardashevskiy ret = tce_iommu_create_table(container, table_group, num, 717e633bc86SAlexey Kardashevskiy page_shift, window_size, levels, &tbl); 718e633bc86SAlexey Kardashevskiy if (ret) 719e633bc86SAlexey Kardashevskiy return ret; 720e633bc86SAlexey Kardashevskiy 721e633bc86SAlexey Kardashevskiy BUG_ON(!tbl->it_ops->free); 722e633bc86SAlexey Kardashevskiy 723e633bc86SAlexey Kardashevskiy /* 724e633bc86SAlexey Kardashevskiy * Program the table to every group. 725e633bc86SAlexey Kardashevskiy * Groups have been tested for compatibility at the attach time. 726e633bc86SAlexey Kardashevskiy */ 727e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 728e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 729e633bc86SAlexey Kardashevskiy 730e633bc86SAlexey Kardashevskiy ret = table_group->ops->set_window(table_group, num, tbl); 731e633bc86SAlexey Kardashevskiy if (ret) 732e633bc86SAlexey Kardashevskiy goto unset_exit; 733e633bc86SAlexey Kardashevskiy } 734e633bc86SAlexey Kardashevskiy 735e633bc86SAlexey Kardashevskiy container->tables[num] = tbl; 736e633bc86SAlexey Kardashevskiy 737e633bc86SAlexey Kardashevskiy /* Return start address assigned by platform in create_table() */ 738e633bc86SAlexey Kardashevskiy *start_addr = tbl->it_offset << tbl->it_page_shift; 739e633bc86SAlexey Kardashevskiy 740e633bc86SAlexey Kardashevskiy return 0; 741e633bc86SAlexey Kardashevskiy 742e633bc86SAlexey Kardashevskiy unset_exit: 743e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 744e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 745e633bc86SAlexey Kardashevskiy table_group->ops->unset_window(table_group, num); 746e633bc86SAlexey Kardashevskiy } 747bc82d122SAlexey Kardashevskiy tce_iommu_free_table(container, tbl); 748e633bc86SAlexey Kardashevskiy 749e633bc86SAlexey Kardashevskiy return ret; 750e633bc86SAlexey Kardashevskiy } 751e633bc86SAlexey Kardashevskiy 752e633bc86SAlexey Kardashevskiy static long tce_iommu_remove_window(struct tce_container *container, 753e633bc86SAlexey Kardashevskiy __u64 start_addr) 754e633bc86SAlexey Kardashevskiy { 755e633bc86SAlexey Kardashevskiy struct iommu_table_group *table_group = NULL; 756e633bc86SAlexey Kardashevskiy struct iommu_table *tbl; 757e633bc86SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 758e633bc86SAlexey Kardashevskiy int num; 759e633bc86SAlexey Kardashevskiy 760e633bc86SAlexey Kardashevskiy num = tce_iommu_find_table(container, start_addr, &tbl); 761e633bc86SAlexey Kardashevskiy if (num < 0) 762e633bc86SAlexey Kardashevskiy return -EINVAL; 763e633bc86SAlexey Kardashevskiy 764e633bc86SAlexey Kardashevskiy BUG_ON(!tbl->it_size); 765e633bc86SAlexey Kardashevskiy 766e633bc86SAlexey Kardashevskiy /* Detach groups from IOMMUs */ 767e633bc86SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 768e633bc86SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 769e633bc86SAlexey Kardashevskiy 770e633bc86SAlexey Kardashevskiy /* 771e633bc86SAlexey Kardashevskiy * SPAPR TCE IOMMU exposes the default DMA window to 772e633bc86SAlexey Kardashevskiy * the guest via dma32_window_start/size of 773e633bc86SAlexey Kardashevskiy * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow 774e633bc86SAlexey Kardashevskiy * the userspace to remove this window, some do not so 775e633bc86SAlexey Kardashevskiy * here we check for the platform capability. 776e633bc86SAlexey Kardashevskiy */ 777e633bc86SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->unset_window) 778e633bc86SAlexey Kardashevskiy return -EPERM; 779e633bc86SAlexey Kardashevskiy 780e633bc86SAlexey Kardashevskiy table_group->ops->unset_window(table_group, num); 781e633bc86SAlexey Kardashevskiy } 782e633bc86SAlexey Kardashevskiy 783e633bc86SAlexey Kardashevskiy /* Free table */ 784e633bc86SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 785bc82d122SAlexey Kardashevskiy tce_iommu_free_table(container, tbl); 786e633bc86SAlexey Kardashevskiy container->tables[num] = NULL; 787e633bc86SAlexey Kardashevskiy 788e633bc86SAlexey Kardashevskiy return 0; 789e633bc86SAlexey Kardashevskiy } 790e633bc86SAlexey Kardashevskiy 7916f01cc69SAlexey Kardashevskiy static long tce_iommu_create_default_window(struct tce_container *container) 7926f01cc69SAlexey Kardashevskiy { 7936f01cc69SAlexey Kardashevskiy long ret; 7946f01cc69SAlexey Kardashevskiy __u64 start_addr = 0; 7956f01cc69SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 7966f01cc69SAlexey Kardashevskiy struct iommu_table_group *table_group; 7976f01cc69SAlexey Kardashevskiy 798d9c72894SAlexey Kardashevskiy if (!container->def_window_pending) 799d9c72894SAlexey Kardashevskiy return 0; 800d9c72894SAlexey Kardashevskiy 8016f01cc69SAlexey Kardashevskiy if (!tce_groups_attached(container)) 8026f01cc69SAlexey Kardashevskiy return -ENODEV; 8036f01cc69SAlexey Kardashevskiy 8046f01cc69SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 8056f01cc69SAlexey Kardashevskiy struct tce_iommu_group, next); 8066f01cc69SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 8076f01cc69SAlexey Kardashevskiy if (!table_group) 8086f01cc69SAlexey Kardashevskiy return -ENODEV; 8096f01cc69SAlexey Kardashevskiy 8106f01cc69SAlexey Kardashevskiy ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K, 8116f01cc69SAlexey Kardashevskiy table_group->tce32_size, 1, &start_addr); 8126f01cc69SAlexey Kardashevskiy WARN_ON_ONCE(!ret && start_addr); 8136f01cc69SAlexey Kardashevskiy 814d9c72894SAlexey Kardashevskiy if (!ret) 815d9c72894SAlexey Kardashevskiy container->def_window_pending = false; 816d9c72894SAlexey Kardashevskiy 8176f01cc69SAlexey Kardashevskiy return ret; 8186f01cc69SAlexey Kardashevskiy } 8196f01cc69SAlexey Kardashevskiy 8205ffd229cSAlexey Kardashevskiy static long tce_iommu_ioctl(void *iommu_data, 8215ffd229cSAlexey Kardashevskiy unsigned int cmd, unsigned long arg) 8225ffd229cSAlexey Kardashevskiy { 8235ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 824e633bc86SAlexey Kardashevskiy unsigned long minsz, ddwsz; 8255ffd229cSAlexey Kardashevskiy long ret; 8265ffd229cSAlexey Kardashevskiy 8275ffd229cSAlexey Kardashevskiy switch (cmd) { 8285ffd229cSAlexey Kardashevskiy case VFIO_CHECK_EXTENSION: 8291b69be5eSGavin Shan switch (arg) { 8301b69be5eSGavin Shan case VFIO_SPAPR_TCE_IOMMU: 8312157e7b8SAlexey Kardashevskiy case VFIO_SPAPR_TCE_v2_IOMMU: 8321b69be5eSGavin Shan ret = 1; 8331b69be5eSGavin Shan break; 8341b69be5eSGavin Shan default: 8351b69be5eSGavin Shan ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg); 8361b69be5eSGavin Shan break; 8371b69be5eSGavin Shan } 8381b69be5eSGavin Shan 8391b69be5eSGavin Shan return (ret < 0) ? 0 : ret; 840bc82d122SAlexey Kardashevskiy } 8415ffd229cSAlexey Kardashevskiy 842bc82d122SAlexey Kardashevskiy /* 843bc82d122SAlexey Kardashevskiy * Sanity check to prevent one userspace from manipulating 844bc82d122SAlexey Kardashevskiy * another userspace mm. 845bc82d122SAlexey Kardashevskiy */ 846bc82d122SAlexey Kardashevskiy BUG_ON(!container); 847bc82d122SAlexey Kardashevskiy if (container->mm && container->mm != current->mm) 848bc82d122SAlexey Kardashevskiy return -EPERM; 849bc82d122SAlexey Kardashevskiy 850bc82d122SAlexey Kardashevskiy switch (cmd) { 8515ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_GET_INFO: { 8525ffd229cSAlexey Kardashevskiy struct vfio_iommu_spapr_tce_info info; 8532157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 8540eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 8555ffd229cSAlexey Kardashevskiy 8562157e7b8SAlexey Kardashevskiy if (!tce_groups_attached(container)) 8570eaf4defSAlexey Kardashevskiy return -ENXIO; 8580eaf4defSAlexey Kardashevskiy 8592157e7b8SAlexey Kardashevskiy tcegrp = list_first_entry(&container->group_list, 8602157e7b8SAlexey Kardashevskiy struct tce_iommu_group, next); 8612157e7b8SAlexey Kardashevskiy table_group = iommu_group_get_iommudata(tcegrp->grp); 8620eaf4defSAlexey Kardashevskiy 8634793d65dSAlexey Kardashevskiy if (!table_group) 8645ffd229cSAlexey Kardashevskiy return -ENXIO; 8655ffd229cSAlexey Kardashevskiy 8665ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_info, 8675ffd229cSAlexey Kardashevskiy dma32_window_size); 8685ffd229cSAlexey Kardashevskiy 8695ffd229cSAlexey Kardashevskiy if (copy_from_user(&info, (void __user *)arg, minsz)) 8705ffd229cSAlexey Kardashevskiy return -EFAULT; 8715ffd229cSAlexey Kardashevskiy 8725ffd229cSAlexey Kardashevskiy if (info.argsz < minsz) 8735ffd229cSAlexey Kardashevskiy return -EINVAL; 8745ffd229cSAlexey Kardashevskiy 8754793d65dSAlexey Kardashevskiy info.dma32_window_start = table_group->tce32_start; 8764793d65dSAlexey Kardashevskiy info.dma32_window_size = table_group->tce32_size; 8775ffd229cSAlexey Kardashevskiy info.flags = 0; 878e633bc86SAlexey Kardashevskiy memset(&info.ddw, 0, sizeof(info.ddw)); 879e633bc86SAlexey Kardashevskiy 880e633bc86SAlexey Kardashevskiy if (table_group->max_dynamic_windows_supported && 881e633bc86SAlexey Kardashevskiy container->v2) { 882e633bc86SAlexey Kardashevskiy info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW; 883e633bc86SAlexey Kardashevskiy info.ddw.pgsizes = table_group->pgsizes; 884e633bc86SAlexey Kardashevskiy info.ddw.max_dynamic_windows_supported = 885e633bc86SAlexey Kardashevskiy table_group->max_dynamic_windows_supported; 886e633bc86SAlexey Kardashevskiy info.ddw.levels = table_group->max_levels; 887e633bc86SAlexey Kardashevskiy } 888e633bc86SAlexey Kardashevskiy 889e633bc86SAlexey Kardashevskiy ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw); 890e633bc86SAlexey Kardashevskiy 891e633bc86SAlexey Kardashevskiy if (info.argsz >= ddwsz) 892e633bc86SAlexey Kardashevskiy minsz = ddwsz; 8935ffd229cSAlexey Kardashevskiy 8945ffd229cSAlexey Kardashevskiy if (copy_to_user((void __user *)arg, &info, minsz)) 8955ffd229cSAlexey Kardashevskiy return -EFAULT; 8965ffd229cSAlexey Kardashevskiy 8975ffd229cSAlexey Kardashevskiy return 0; 8985ffd229cSAlexey Kardashevskiy } 8995ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_MAP_DMA: { 9005ffd229cSAlexey Kardashevskiy struct vfio_iommu_type1_dma_map param; 9010eaf4defSAlexey Kardashevskiy struct iommu_table *tbl = NULL; 9020eaf4defSAlexey Kardashevskiy long num; 90305c6cfb9SAlexey Kardashevskiy enum dma_data_direction direction; 9045ffd229cSAlexey Kardashevskiy 9053c56e822SAlexey Kardashevskiy if (!container->enabled) 9063c56e822SAlexey Kardashevskiy return -EPERM; 9073c56e822SAlexey Kardashevskiy 9085ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); 9095ffd229cSAlexey Kardashevskiy 9105ffd229cSAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 9115ffd229cSAlexey Kardashevskiy return -EFAULT; 9125ffd229cSAlexey Kardashevskiy 9135ffd229cSAlexey Kardashevskiy if (param.argsz < minsz) 9145ffd229cSAlexey Kardashevskiy return -EINVAL; 9155ffd229cSAlexey Kardashevskiy 9165ffd229cSAlexey Kardashevskiy if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ | 9175ffd229cSAlexey Kardashevskiy VFIO_DMA_MAP_FLAG_WRITE)) 9185ffd229cSAlexey Kardashevskiy return -EINVAL; 9195ffd229cSAlexey Kardashevskiy 920d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container); 921d9c72894SAlexey Kardashevskiy if (ret) 922d9c72894SAlexey Kardashevskiy return ret; 923d9c72894SAlexey Kardashevskiy 9240eaf4defSAlexey Kardashevskiy num = tce_iommu_find_table(container, param.iova, &tbl); 9250eaf4defSAlexey Kardashevskiy if (num < 0) 9260eaf4defSAlexey Kardashevskiy return -ENXIO; 9270eaf4defSAlexey Kardashevskiy 92800663d4eSAlexey Kardashevskiy if ((param.size & ~IOMMU_PAGE_MASK(tbl)) || 92900663d4eSAlexey Kardashevskiy (param.vaddr & ~IOMMU_PAGE_MASK(tbl))) 9305ffd229cSAlexey Kardashevskiy return -EINVAL; 9315ffd229cSAlexey Kardashevskiy 9325ffd229cSAlexey Kardashevskiy /* iova is checked by the IOMMU API */ 93305c6cfb9SAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_READ) { 9345ffd229cSAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE) 93505c6cfb9SAlexey Kardashevskiy direction = DMA_BIDIRECTIONAL; 93605c6cfb9SAlexey Kardashevskiy else 93705c6cfb9SAlexey Kardashevskiy direction = DMA_TO_DEVICE; 93805c6cfb9SAlexey Kardashevskiy } else { 93905c6cfb9SAlexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE) 94005c6cfb9SAlexey Kardashevskiy direction = DMA_FROM_DEVICE; 94105c6cfb9SAlexey Kardashevskiy else 94205c6cfb9SAlexey Kardashevskiy return -EINVAL; 94305c6cfb9SAlexey Kardashevskiy } 9445ffd229cSAlexey Kardashevskiy 94505c6cfb9SAlexey Kardashevskiy ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr); 9465ffd229cSAlexey Kardashevskiy if (ret) 9475ffd229cSAlexey Kardashevskiy return ret; 9485ffd229cSAlexey Kardashevskiy 9492157e7b8SAlexey Kardashevskiy if (container->v2) 9502157e7b8SAlexey Kardashevskiy ret = tce_iommu_build_v2(container, tbl, 9512157e7b8SAlexey Kardashevskiy param.iova >> tbl->it_page_shift, 9522157e7b8SAlexey Kardashevskiy param.vaddr, 9532157e7b8SAlexey Kardashevskiy param.size >> tbl->it_page_shift, 9542157e7b8SAlexey Kardashevskiy direction); 9552157e7b8SAlexey Kardashevskiy else 9569b14a1ffSAlexey Kardashevskiy ret = tce_iommu_build(container, tbl, 95700663d4eSAlexey Kardashevskiy param.iova >> tbl->it_page_shift, 95805c6cfb9SAlexey Kardashevskiy param.vaddr, 95905c6cfb9SAlexey Kardashevskiy param.size >> tbl->it_page_shift, 96005c6cfb9SAlexey Kardashevskiy direction); 9615ffd229cSAlexey Kardashevskiy 9625ffd229cSAlexey Kardashevskiy iommu_flush_tce(tbl); 9635ffd229cSAlexey Kardashevskiy 9645ffd229cSAlexey Kardashevskiy return ret; 9655ffd229cSAlexey Kardashevskiy } 9665ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_UNMAP_DMA: { 9675ffd229cSAlexey Kardashevskiy struct vfio_iommu_type1_dma_unmap param; 9680eaf4defSAlexey Kardashevskiy struct iommu_table *tbl = NULL; 9690eaf4defSAlexey Kardashevskiy long num; 9705ffd229cSAlexey Kardashevskiy 9713c56e822SAlexey Kardashevskiy if (!container->enabled) 9723c56e822SAlexey Kardashevskiy return -EPERM; 9733c56e822SAlexey Kardashevskiy 9745ffd229cSAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, 9755ffd229cSAlexey Kardashevskiy size); 9765ffd229cSAlexey Kardashevskiy 9775ffd229cSAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 9785ffd229cSAlexey Kardashevskiy return -EFAULT; 9795ffd229cSAlexey Kardashevskiy 9805ffd229cSAlexey Kardashevskiy if (param.argsz < minsz) 9815ffd229cSAlexey Kardashevskiy return -EINVAL; 9825ffd229cSAlexey Kardashevskiy 9835ffd229cSAlexey Kardashevskiy /* No flag is supported now */ 9845ffd229cSAlexey Kardashevskiy if (param.flags) 9855ffd229cSAlexey Kardashevskiy return -EINVAL; 9865ffd229cSAlexey Kardashevskiy 987d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container); 988d9c72894SAlexey Kardashevskiy if (ret) 989d9c72894SAlexey Kardashevskiy return ret; 990d9c72894SAlexey Kardashevskiy 9910eaf4defSAlexey Kardashevskiy num = tce_iommu_find_table(container, param.iova, &tbl); 9920eaf4defSAlexey Kardashevskiy if (num < 0) 9930eaf4defSAlexey Kardashevskiy return -ENXIO; 9940eaf4defSAlexey Kardashevskiy 99500663d4eSAlexey Kardashevskiy if (param.size & ~IOMMU_PAGE_MASK(tbl)) 9965ffd229cSAlexey Kardashevskiy return -EINVAL; 9975ffd229cSAlexey Kardashevskiy 9985ffd229cSAlexey Kardashevskiy ret = iommu_tce_clear_param_check(tbl, param.iova, 0, 99900663d4eSAlexey Kardashevskiy param.size >> tbl->it_page_shift); 10005ffd229cSAlexey Kardashevskiy if (ret) 10015ffd229cSAlexey Kardashevskiy return ret; 10025ffd229cSAlexey Kardashevskiy 10039b14a1ffSAlexey Kardashevskiy ret = tce_iommu_clear(container, tbl, 100400663d4eSAlexey Kardashevskiy param.iova >> tbl->it_page_shift, 100500663d4eSAlexey Kardashevskiy param.size >> tbl->it_page_shift); 10065ffd229cSAlexey Kardashevskiy iommu_flush_tce(tbl); 10075ffd229cSAlexey Kardashevskiy 10085ffd229cSAlexey Kardashevskiy return ret; 10095ffd229cSAlexey Kardashevskiy } 10102157e7b8SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: { 10112157e7b8SAlexey Kardashevskiy struct vfio_iommu_spapr_register_memory param; 10122157e7b8SAlexey Kardashevskiy 10132157e7b8SAlexey Kardashevskiy if (!container->v2) 10142157e7b8SAlexey Kardashevskiy break; 10152157e7b8SAlexey Kardashevskiy 10162157e7b8SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_register_memory, 10172157e7b8SAlexey Kardashevskiy size); 10182157e7b8SAlexey Kardashevskiy 1019bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container); 1020bc82d122SAlexey Kardashevskiy if (ret) 1021bc82d122SAlexey Kardashevskiy return ret; 1022bc82d122SAlexey Kardashevskiy 10232157e7b8SAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 10242157e7b8SAlexey Kardashevskiy return -EFAULT; 10252157e7b8SAlexey Kardashevskiy 10262157e7b8SAlexey Kardashevskiy if (param.argsz < minsz) 10272157e7b8SAlexey Kardashevskiy return -EINVAL; 10282157e7b8SAlexey Kardashevskiy 10292157e7b8SAlexey Kardashevskiy /* No flag is supported now */ 10302157e7b8SAlexey Kardashevskiy if (param.flags) 10312157e7b8SAlexey Kardashevskiy return -EINVAL; 10322157e7b8SAlexey Kardashevskiy 10332157e7b8SAlexey Kardashevskiy mutex_lock(&container->lock); 10342157e7b8SAlexey Kardashevskiy ret = tce_iommu_register_pages(container, param.vaddr, 10352157e7b8SAlexey Kardashevskiy param.size); 10362157e7b8SAlexey Kardashevskiy mutex_unlock(&container->lock); 10372157e7b8SAlexey Kardashevskiy 10382157e7b8SAlexey Kardashevskiy return ret; 10392157e7b8SAlexey Kardashevskiy } 10402157e7b8SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: { 10412157e7b8SAlexey Kardashevskiy struct vfio_iommu_spapr_register_memory param; 10422157e7b8SAlexey Kardashevskiy 10432157e7b8SAlexey Kardashevskiy if (!container->v2) 10442157e7b8SAlexey Kardashevskiy break; 10452157e7b8SAlexey Kardashevskiy 1046bc82d122SAlexey Kardashevskiy if (!container->mm) 1047bc82d122SAlexey Kardashevskiy return -EPERM; 1048bc82d122SAlexey Kardashevskiy 10492157e7b8SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_register_memory, 10502157e7b8SAlexey Kardashevskiy size); 10512157e7b8SAlexey Kardashevskiy 10522157e7b8SAlexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 10532157e7b8SAlexey Kardashevskiy return -EFAULT; 10542157e7b8SAlexey Kardashevskiy 10552157e7b8SAlexey Kardashevskiy if (param.argsz < minsz) 10562157e7b8SAlexey Kardashevskiy return -EINVAL; 10572157e7b8SAlexey Kardashevskiy 10582157e7b8SAlexey Kardashevskiy /* No flag is supported now */ 10592157e7b8SAlexey Kardashevskiy if (param.flags) 10602157e7b8SAlexey Kardashevskiy return -EINVAL; 10612157e7b8SAlexey Kardashevskiy 10622157e7b8SAlexey Kardashevskiy mutex_lock(&container->lock); 10632157e7b8SAlexey Kardashevskiy ret = tce_iommu_unregister_pages(container, param.vaddr, 10642157e7b8SAlexey Kardashevskiy param.size); 10652157e7b8SAlexey Kardashevskiy mutex_unlock(&container->lock); 10662157e7b8SAlexey Kardashevskiy 10672157e7b8SAlexey Kardashevskiy return ret; 10682157e7b8SAlexey Kardashevskiy } 10695ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_ENABLE: 10702157e7b8SAlexey Kardashevskiy if (container->v2) 10712157e7b8SAlexey Kardashevskiy break; 10722157e7b8SAlexey Kardashevskiy 10735ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 10745ffd229cSAlexey Kardashevskiy ret = tce_iommu_enable(container); 10755ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 10765ffd229cSAlexey Kardashevskiy return ret; 10775ffd229cSAlexey Kardashevskiy 10785ffd229cSAlexey Kardashevskiy 10795ffd229cSAlexey Kardashevskiy case VFIO_IOMMU_DISABLE: 10802157e7b8SAlexey Kardashevskiy if (container->v2) 10812157e7b8SAlexey Kardashevskiy break; 10822157e7b8SAlexey Kardashevskiy 10835ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 10845ffd229cSAlexey Kardashevskiy tce_iommu_disable(container); 10855ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 10865ffd229cSAlexey Kardashevskiy return 0; 10871b69be5eSGavin Shan 10882157e7b8SAlexey Kardashevskiy case VFIO_EEH_PE_OP: { 10892157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 10902157e7b8SAlexey Kardashevskiy 10912157e7b8SAlexey Kardashevskiy ret = 0; 10922157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 10932157e7b8SAlexey Kardashevskiy ret = vfio_spapr_iommu_eeh_ioctl(tcegrp->grp, 10941b69be5eSGavin Shan cmd, arg); 10952157e7b8SAlexey Kardashevskiy if (ret) 10962157e7b8SAlexey Kardashevskiy return ret; 10972157e7b8SAlexey Kardashevskiy } 10982157e7b8SAlexey Kardashevskiy return ret; 10992157e7b8SAlexey Kardashevskiy } 11002157e7b8SAlexey Kardashevskiy 1101e633bc86SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_CREATE: { 1102e633bc86SAlexey Kardashevskiy struct vfio_iommu_spapr_tce_create create; 1103e633bc86SAlexey Kardashevskiy 1104e633bc86SAlexey Kardashevskiy if (!container->v2) 1105e633bc86SAlexey Kardashevskiy break; 1106e633bc86SAlexey Kardashevskiy 1107bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container); 1108bc82d122SAlexey Kardashevskiy if (ret) 1109bc82d122SAlexey Kardashevskiy return ret; 1110bc82d122SAlexey Kardashevskiy 1111e633bc86SAlexey Kardashevskiy if (!tce_groups_attached(container)) 1112e633bc86SAlexey Kardashevskiy return -ENXIO; 1113e633bc86SAlexey Kardashevskiy 1114e633bc86SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_create, 1115e633bc86SAlexey Kardashevskiy start_addr); 1116e633bc86SAlexey Kardashevskiy 1117e633bc86SAlexey Kardashevskiy if (copy_from_user(&create, (void __user *)arg, minsz)) 1118e633bc86SAlexey Kardashevskiy return -EFAULT; 1119e633bc86SAlexey Kardashevskiy 1120e633bc86SAlexey Kardashevskiy if (create.argsz < minsz) 1121e633bc86SAlexey Kardashevskiy return -EINVAL; 1122e633bc86SAlexey Kardashevskiy 1123e633bc86SAlexey Kardashevskiy if (create.flags) 1124e633bc86SAlexey Kardashevskiy return -EINVAL; 1125e633bc86SAlexey Kardashevskiy 1126e633bc86SAlexey Kardashevskiy mutex_lock(&container->lock); 1127e633bc86SAlexey Kardashevskiy 1128d9c72894SAlexey Kardashevskiy ret = tce_iommu_create_default_window(container); 11292da64d20SAlexey Kardashevskiy if (!ret) 11302da64d20SAlexey Kardashevskiy ret = tce_iommu_create_window(container, 11312da64d20SAlexey Kardashevskiy create.page_shift, 1132e633bc86SAlexey Kardashevskiy create.window_size, create.levels, 1133e633bc86SAlexey Kardashevskiy &create.start_addr); 1134e633bc86SAlexey Kardashevskiy 1135e633bc86SAlexey Kardashevskiy mutex_unlock(&container->lock); 1136e633bc86SAlexey Kardashevskiy 1137e633bc86SAlexey Kardashevskiy if (!ret && copy_to_user((void __user *)arg, &create, minsz)) 1138e633bc86SAlexey Kardashevskiy ret = -EFAULT; 1139e633bc86SAlexey Kardashevskiy 1140e633bc86SAlexey Kardashevskiy return ret; 1141e633bc86SAlexey Kardashevskiy } 1142e633bc86SAlexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_REMOVE: { 1143e633bc86SAlexey Kardashevskiy struct vfio_iommu_spapr_tce_remove remove; 1144e633bc86SAlexey Kardashevskiy 1145e633bc86SAlexey Kardashevskiy if (!container->v2) 1146e633bc86SAlexey Kardashevskiy break; 1147e633bc86SAlexey Kardashevskiy 1148bc82d122SAlexey Kardashevskiy ret = tce_iommu_mm_set(container); 1149bc82d122SAlexey Kardashevskiy if (ret) 1150bc82d122SAlexey Kardashevskiy return ret; 1151bc82d122SAlexey Kardashevskiy 1152e633bc86SAlexey Kardashevskiy if (!tce_groups_attached(container)) 1153e633bc86SAlexey Kardashevskiy return -ENXIO; 1154e633bc86SAlexey Kardashevskiy 1155e633bc86SAlexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_remove, 1156e633bc86SAlexey Kardashevskiy start_addr); 1157e633bc86SAlexey Kardashevskiy 1158e633bc86SAlexey Kardashevskiy if (copy_from_user(&remove, (void __user *)arg, minsz)) 1159e633bc86SAlexey Kardashevskiy return -EFAULT; 1160e633bc86SAlexey Kardashevskiy 1161e633bc86SAlexey Kardashevskiy if (remove.argsz < minsz) 1162e633bc86SAlexey Kardashevskiy return -EINVAL; 1163e633bc86SAlexey Kardashevskiy 1164e633bc86SAlexey Kardashevskiy if (remove.flags) 1165e633bc86SAlexey Kardashevskiy return -EINVAL; 1166e633bc86SAlexey Kardashevskiy 1167d9c72894SAlexey Kardashevskiy if (container->def_window_pending && !remove.start_addr) { 1168d9c72894SAlexey Kardashevskiy container->def_window_pending = false; 1169d9c72894SAlexey Kardashevskiy return 0; 1170d9c72894SAlexey Kardashevskiy } 1171d9c72894SAlexey Kardashevskiy 1172e633bc86SAlexey Kardashevskiy mutex_lock(&container->lock); 1173e633bc86SAlexey Kardashevskiy 1174e633bc86SAlexey Kardashevskiy ret = tce_iommu_remove_window(container, remove.start_addr); 1175e633bc86SAlexey Kardashevskiy 1176e633bc86SAlexey Kardashevskiy mutex_unlock(&container->lock); 1177e633bc86SAlexey Kardashevskiy 1178e633bc86SAlexey Kardashevskiy return ret; 1179e633bc86SAlexey Kardashevskiy } 11805ffd229cSAlexey Kardashevskiy } 11815ffd229cSAlexey Kardashevskiy 11825ffd229cSAlexey Kardashevskiy return -ENOTTY; 11835ffd229cSAlexey Kardashevskiy } 11845ffd229cSAlexey Kardashevskiy 1185f87a8864SAlexey Kardashevskiy static void tce_iommu_release_ownership(struct tce_container *container, 1186f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1187f87a8864SAlexey Kardashevskiy { 1188f87a8864SAlexey Kardashevskiy int i; 1189f87a8864SAlexey Kardashevskiy 1190f87a8864SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 11912157e7b8SAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 1192f87a8864SAlexey Kardashevskiy 1193f87a8864SAlexey Kardashevskiy if (!tbl) 1194f87a8864SAlexey Kardashevskiy continue; 1195f87a8864SAlexey Kardashevskiy 1196f87a8864SAlexey Kardashevskiy tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 1197bc82d122SAlexey Kardashevskiy tce_iommu_userspace_view_free(tbl, container->mm); 1198f87a8864SAlexey Kardashevskiy if (tbl->it_map) 1199f87a8864SAlexey Kardashevskiy iommu_release_ownership(tbl); 12002157e7b8SAlexey Kardashevskiy 12012157e7b8SAlexey Kardashevskiy container->tables[i] = NULL; 1202f87a8864SAlexey Kardashevskiy } 1203f87a8864SAlexey Kardashevskiy } 1204f87a8864SAlexey Kardashevskiy 1205f87a8864SAlexey Kardashevskiy static int tce_iommu_take_ownership(struct tce_container *container, 1206f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1207f87a8864SAlexey Kardashevskiy { 1208f87a8864SAlexey Kardashevskiy int i, j, rc = 0; 1209f87a8864SAlexey Kardashevskiy 1210f87a8864SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 1211f87a8864SAlexey Kardashevskiy struct iommu_table *tbl = table_group->tables[i]; 1212f87a8864SAlexey Kardashevskiy 1213f87a8864SAlexey Kardashevskiy if (!tbl || !tbl->it_map) 1214f87a8864SAlexey Kardashevskiy continue; 1215f87a8864SAlexey Kardashevskiy 1216f87a8864SAlexey Kardashevskiy rc = iommu_take_ownership(tbl); 1217f87a8864SAlexey Kardashevskiy if (rc) { 1218f87a8864SAlexey Kardashevskiy for (j = 0; j < i; ++j) 1219f87a8864SAlexey Kardashevskiy iommu_release_ownership( 1220f87a8864SAlexey Kardashevskiy table_group->tables[j]); 1221f87a8864SAlexey Kardashevskiy 1222f87a8864SAlexey Kardashevskiy return rc; 1223f87a8864SAlexey Kardashevskiy } 1224f87a8864SAlexey Kardashevskiy } 1225f87a8864SAlexey Kardashevskiy 12262157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) 12272157e7b8SAlexey Kardashevskiy container->tables[i] = table_group->tables[i]; 12282157e7b8SAlexey Kardashevskiy 1229f87a8864SAlexey Kardashevskiy return 0; 1230f87a8864SAlexey Kardashevskiy } 1231f87a8864SAlexey Kardashevskiy 1232f87a8864SAlexey Kardashevskiy static void tce_iommu_release_ownership_ddw(struct tce_container *container, 1233f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1234f87a8864SAlexey Kardashevskiy { 123546d3e1e1SAlexey Kardashevskiy long i; 123646d3e1e1SAlexey Kardashevskiy 123746d3e1e1SAlexey Kardashevskiy if (!table_group->ops->unset_window) { 123846d3e1e1SAlexey Kardashevskiy WARN_ON_ONCE(1); 123946d3e1e1SAlexey Kardashevskiy return; 124046d3e1e1SAlexey Kardashevskiy } 124146d3e1e1SAlexey Kardashevskiy 12422157e7b8SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) 124346d3e1e1SAlexey Kardashevskiy table_group->ops->unset_window(table_group, i); 124446d3e1e1SAlexey Kardashevskiy 1245f87a8864SAlexey Kardashevskiy table_group->ops->release_ownership(table_group); 1246f87a8864SAlexey Kardashevskiy } 1247f87a8864SAlexey Kardashevskiy 1248f87a8864SAlexey Kardashevskiy static long tce_iommu_take_ownership_ddw(struct tce_container *container, 1249f87a8864SAlexey Kardashevskiy struct iommu_table_group *table_group) 1250f87a8864SAlexey Kardashevskiy { 1251930a42deSAlexey Kardashevskiy long i, ret = 0; 1252930a42deSAlexey Kardashevskiy 125346d3e1e1SAlexey Kardashevskiy if (!table_group->ops->create_table || !table_group->ops->set_window || 125446d3e1e1SAlexey Kardashevskiy !table_group->ops->release_ownership) { 125546d3e1e1SAlexey Kardashevskiy WARN_ON_ONCE(1); 125646d3e1e1SAlexey Kardashevskiy return -EFAULT; 125746d3e1e1SAlexey Kardashevskiy } 125846d3e1e1SAlexey Kardashevskiy 1259f87a8864SAlexey Kardashevskiy table_group->ops->take_ownership(table_group); 1260f87a8864SAlexey Kardashevskiy 1261930a42deSAlexey Kardashevskiy /* Set all windows to the new group */ 1262930a42deSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 1263930a42deSAlexey Kardashevskiy struct iommu_table *tbl = container->tables[i]; 1264930a42deSAlexey Kardashevskiy 1265930a42deSAlexey Kardashevskiy if (!tbl) 1266930a42deSAlexey Kardashevskiy continue; 1267930a42deSAlexey Kardashevskiy 1268930a42deSAlexey Kardashevskiy ret = table_group->ops->set_window(table_group, i, tbl); 1269930a42deSAlexey Kardashevskiy if (ret) 1270930a42deSAlexey Kardashevskiy goto release_exit; 1271930a42deSAlexey Kardashevskiy } 1272930a42deSAlexey Kardashevskiy 12732157e7b8SAlexey Kardashevskiy return 0; 1274930a42deSAlexey Kardashevskiy 1275930a42deSAlexey Kardashevskiy release_exit: 1276930a42deSAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) 1277930a42deSAlexey Kardashevskiy table_group->ops->unset_window(table_group, i); 1278930a42deSAlexey Kardashevskiy 1279930a42deSAlexey Kardashevskiy table_group->ops->release_ownership(table_group); 1280930a42deSAlexey Kardashevskiy 1281930a42deSAlexey Kardashevskiy return ret; 1282f87a8864SAlexey Kardashevskiy } 1283f87a8864SAlexey Kardashevskiy 12845ffd229cSAlexey Kardashevskiy static int tce_iommu_attach_group(void *iommu_data, 12855ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group) 12865ffd229cSAlexey Kardashevskiy { 12875ffd229cSAlexey Kardashevskiy int ret; 12885ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 12890eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 12902157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp = NULL; 12915ffd229cSAlexey Kardashevskiy 12925ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 12935ffd229cSAlexey Kardashevskiy 12945ffd229cSAlexey Kardashevskiy /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n", 12955ffd229cSAlexey Kardashevskiy iommu_group_id(iommu_group), iommu_group); */ 12960eaf4defSAlexey Kardashevskiy table_group = iommu_group_get_iommudata(iommu_group); 1297bd00fdf1SGreg Kurz if (!table_group) { 1298bd00fdf1SGreg Kurz ret = -ENODEV; 1299bd00fdf1SGreg Kurz goto unlock_exit; 1300bd00fdf1SGreg Kurz } 13012157e7b8SAlexey Kardashevskiy 13022157e7b8SAlexey Kardashevskiy if (tce_groups_attached(container) && (!table_group->ops || 13032157e7b8SAlexey Kardashevskiy !table_group->ops->take_ownership || 13042157e7b8SAlexey Kardashevskiy !table_group->ops->release_ownership)) { 13052157e7b8SAlexey Kardashevskiy ret = -EBUSY; 13062157e7b8SAlexey Kardashevskiy goto unlock_exit; 13072157e7b8SAlexey Kardashevskiy } 13082157e7b8SAlexey Kardashevskiy 13092157e7b8SAlexey Kardashevskiy /* Check if new group has the same iommu_ops (i.e. compatible) */ 13102157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 13112157e7b8SAlexey Kardashevskiy struct iommu_table_group *table_group_tmp; 13122157e7b8SAlexey Kardashevskiy 13132157e7b8SAlexey Kardashevskiy if (tcegrp->grp == iommu_group) { 13142157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: Group %d is already attached\n", 13152157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group)); 13162157e7b8SAlexey Kardashevskiy ret = -EBUSY; 13172157e7b8SAlexey Kardashevskiy goto unlock_exit; 13182157e7b8SAlexey Kardashevskiy } 13192157e7b8SAlexey Kardashevskiy table_group_tmp = iommu_group_get_iommudata(tcegrp->grp); 132054de285bSAlexey Kardashevskiy if (table_group_tmp->ops->create_table != 132154de285bSAlexey Kardashevskiy table_group->ops->create_table) { 13222157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: Group %d is incompatible with group %d\n", 13232157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group), 13242157e7b8SAlexey Kardashevskiy iommu_group_id(tcegrp->grp)); 13252157e7b8SAlexey Kardashevskiy ret = -EPERM; 13262157e7b8SAlexey Kardashevskiy goto unlock_exit; 13272157e7b8SAlexey Kardashevskiy } 13282157e7b8SAlexey Kardashevskiy } 13292157e7b8SAlexey Kardashevskiy 13302157e7b8SAlexey Kardashevskiy tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL); 13312157e7b8SAlexey Kardashevskiy if (!tcegrp) { 13322157e7b8SAlexey Kardashevskiy ret = -ENOMEM; 13330eaf4defSAlexey Kardashevskiy goto unlock_exit; 13340eaf4defSAlexey Kardashevskiy } 13350eaf4defSAlexey Kardashevskiy 1336f87a8864SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->take_ownership || 13376f01cc69SAlexey Kardashevskiy !table_group->ops->release_ownership) { 1338f87a8864SAlexey Kardashevskiy ret = tce_iommu_take_ownership(container, table_group); 13396f01cc69SAlexey Kardashevskiy } else { 1340f87a8864SAlexey Kardashevskiy ret = tce_iommu_take_ownership_ddw(container, table_group); 13416f01cc69SAlexey Kardashevskiy if (!tce_groups_attached(container) && !container->tables[0]) 1342d9c72894SAlexey Kardashevskiy container->def_window_pending = true; 13436f01cc69SAlexey Kardashevskiy } 1344f87a8864SAlexey Kardashevskiy 13452157e7b8SAlexey Kardashevskiy if (!ret) { 13462157e7b8SAlexey Kardashevskiy tcegrp->grp = iommu_group; 13472157e7b8SAlexey Kardashevskiy list_add(&tcegrp->next, &container->group_list); 13482157e7b8SAlexey Kardashevskiy } 13495ffd229cSAlexey Kardashevskiy 135022af4859SAlexey Kardashevskiy unlock_exit: 13512157e7b8SAlexey Kardashevskiy if (ret && tcegrp) 13522157e7b8SAlexey Kardashevskiy kfree(tcegrp); 13532157e7b8SAlexey Kardashevskiy 13545ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 13555ffd229cSAlexey Kardashevskiy 13565ffd229cSAlexey Kardashevskiy return ret; 13575ffd229cSAlexey Kardashevskiy } 13585ffd229cSAlexey Kardashevskiy 13595ffd229cSAlexey Kardashevskiy static void tce_iommu_detach_group(void *iommu_data, 13605ffd229cSAlexey Kardashevskiy struct iommu_group *iommu_group) 13615ffd229cSAlexey Kardashevskiy { 13625ffd229cSAlexey Kardashevskiy struct tce_container *container = iommu_data; 13630eaf4defSAlexey Kardashevskiy struct iommu_table_group *table_group; 13642157e7b8SAlexey Kardashevskiy bool found = false; 13652157e7b8SAlexey Kardashevskiy struct tce_iommu_group *tcegrp; 13665ffd229cSAlexey Kardashevskiy 13675ffd229cSAlexey Kardashevskiy mutex_lock(&container->lock); 13682157e7b8SAlexey Kardashevskiy 13692157e7b8SAlexey Kardashevskiy list_for_each_entry(tcegrp, &container->group_list, next) { 13702157e7b8SAlexey Kardashevskiy if (tcegrp->grp == iommu_group) { 13712157e7b8SAlexey Kardashevskiy found = true; 13722157e7b8SAlexey Kardashevskiy break; 13732157e7b8SAlexey Kardashevskiy } 13742157e7b8SAlexey Kardashevskiy } 13752157e7b8SAlexey Kardashevskiy 13762157e7b8SAlexey Kardashevskiy if (!found) { 13772157e7b8SAlexey Kardashevskiy pr_warn("tce_vfio: detaching unattached group #%u\n", 13782157e7b8SAlexey Kardashevskiy iommu_group_id(iommu_group)); 137922af4859SAlexey Kardashevskiy goto unlock_exit; 138022af4859SAlexey Kardashevskiy } 138122af4859SAlexey Kardashevskiy 13822157e7b8SAlexey Kardashevskiy list_del(&tcegrp->next); 13832157e7b8SAlexey Kardashevskiy kfree(tcegrp); 13840eaf4defSAlexey Kardashevskiy 13850eaf4defSAlexey Kardashevskiy table_group = iommu_group_get_iommudata(iommu_group); 13860eaf4defSAlexey Kardashevskiy BUG_ON(!table_group); 13870eaf4defSAlexey Kardashevskiy 1388f87a8864SAlexey Kardashevskiy if (!table_group->ops || !table_group->ops->release_ownership) 1389f87a8864SAlexey Kardashevskiy tce_iommu_release_ownership(container, table_group); 1390f87a8864SAlexey Kardashevskiy else 1391f87a8864SAlexey Kardashevskiy tce_iommu_release_ownership_ddw(container, table_group); 139222af4859SAlexey Kardashevskiy 139322af4859SAlexey Kardashevskiy unlock_exit: 13945ffd229cSAlexey Kardashevskiy mutex_unlock(&container->lock); 13955ffd229cSAlexey Kardashevskiy } 13965ffd229cSAlexey Kardashevskiy 13975ffd229cSAlexey Kardashevskiy const struct vfio_iommu_driver_ops tce_iommu_driver_ops = { 13985ffd229cSAlexey Kardashevskiy .name = "iommu-vfio-powerpc", 13995ffd229cSAlexey Kardashevskiy .owner = THIS_MODULE, 14005ffd229cSAlexey Kardashevskiy .open = tce_iommu_open, 14015ffd229cSAlexey Kardashevskiy .release = tce_iommu_release, 14025ffd229cSAlexey Kardashevskiy .ioctl = tce_iommu_ioctl, 14035ffd229cSAlexey Kardashevskiy .attach_group = tce_iommu_attach_group, 14045ffd229cSAlexey Kardashevskiy .detach_group = tce_iommu_detach_group, 14055ffd229cSAlexey Kardashevskiy }; 14065ffd229cSAlexey Kardashevskiy 14075ffd229cSAlexey Kardashevskiy static int __init tce_iommu_init(void) 14085ffd229cSAlexey Kardashevskiy { 14095ffd229cSAlexey Kardashevskiy return vfio_register_iommu_driver(&tce_iommu_driver_ops); 14105ffd229cSAlexey Kardashevskiy } 14115ffd229cSAlexey Kardashevskiy 14125ffd229cSAlexey Kardashevskiy static void __exit tce_iommu_cleanup(void) 14135ffd229cSAlexey Kardashevskiy { 14145ffd229cSAlexey Kardashevskiy vfio_unregister_iommu_driver(&tce_iommu_driver_ops); 14155ffd229cSAlexey Kardashevskiy } 14165ffd229cSAlexey Kardashevskiy 14175ffd229cSAlexey Kardashevskiy module_init(tce_iommu_init); 14185ffd229cSAlexey Kardashevskiy module_exit(tce_iommu_cleanup); 14195ffd229cSAlexey Kardashevskiy 14205ffd229cSAlexey Kardashevskiy MODULE_VERSION(DRIVER_VERSION); 14215ffd229cSAlexey Kardashevskiy MODULE_LICENSE("GPL v2"); 14225ffd229cSAlexey Kardashevskiy MODULE_AUTHOR(DRIVER_AUTHOR); 14235ffd229cSAlexey Kardashevskiy MODULE_DESCRIPTION(DRIVER_DESC); 14245ffd229cSAlexey Kardashevskiy 1425