10db2e5d1SRobin Murphy /* 20db2e5d1SRobin Murphy * A fairly generic DMA-API to IOMMU-API glue layer. 30db2e5d1SRobin Murphy * 40db2e5d1SRobin Murphy * Copyright (C) 2014-2015 ARM Ltd. 50db2e5d1SRobin Murphy * 60db2e5d1SRobin Murphy * based in part on arch/arm/mm/dma-mapping.c: 70db2e5d1SRobin Murphy * Copyright (C) 2000-2004 Russell King 80db2e5d1SRobin Murphy * 90db2e5d1SRobin Murphy * This program is free software; you can redistribute it and/or modify 100db2e5d1SRobin Murphy * it under the terms of the GNU General Public License version 2 as 110db2e5d1SRobin Murphy * published by the Free Software Foundation. 120db2e5d1SRobin Murphy * 130db2e5d1SRobin Murphy * This program is distributed in the hope that it will be useful, 140db2e5d1SRobin Murphy * but WITHOUT ANY WARRANTY; without even the implied warranty of 150db2e5d1SRobin Murphy * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 160db2e5d1SRobin Murphy * GNU General Public License for more details. 170db2e5d1SRobin Murphy * 180db2e5d1SRobin Murphy * You should have received a copy of the GNU General Public License 190db2e5d1SRobin Murphy * along with this program. If not, see <http://www.gnu.org/licenses/>. 200db2e5d1SRobin Murphy */ 210db2e5d1SRobin Murphy 22f51dc892SShameer Kolothum #include <linux/acpi_iort.h> 230db2e5d1SRobin Murphy #include <linux/device.h> 2406d60728SChristoph Hellwig #include <linux/dma-contiguous.h> 250db2e5d1SRobin Murphy #include <linux/dma-iommu.h> 26af751d43SChristoph Hellwig #include <linux/dma-noncoherent.h> 275b11e9cdSRobin Murphy #include <linux/gfp.h> 280db2e5d1SRobin Murphy #include <linux/huge_mm.h> 290db2e5d1SRobin Murphy #include <linux/iommu.h> 300db2e5d1SRobin Murphy #include <linux/iova.h> 3144bb7e24SRobin Murphy #include <linux/irq.h> 320db2e5d1SRobin Murphy #include <linux/mm.h> 33fade1ec0SRobin Murphy #include <linux/pci.h> 345b11e9cdSRobin Murphy #include <linux/scatterlist.h> 355b11e9cdSRobin Murphy #include <linux/vmalloc.h> 360db2e5d1SRobin Murphy 3744bb7e24SRobin Murphy struct iommu_dma_msi_page { 3844bb7e24SRobin Murphy struct list_head list; 3944bb7e24SRobin Murphy dma_addr_t iova; 4044bb7e24SRobin Murphy phys_addr_t phys; 4144bb7e24SRobin Murphy }; 4244bb7e24SRobin Murphy 43fdbe574eSRobin Murphy enum iommu_dma_cookie_type { 44fdbe574eSRobin Murphy IOMMU_DMA_IOVA_COOKIE, 45fdbe574eSRobin Murphy IOMMU_DMA_MSI_COOKIE, 46fdbe574eSRobin Murphy }; 47fdbe574eSRobin Murphy 4844bb7e24SRobin Murphy struct iommu_dma_cookie { 49fdbe574eSRobin Murphy enum iommu_dma_cookie_type type; 50fdbe574eSRobin Murphy union { 51fdbe574eSRobin Murphy /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ 5244bb7e24SRobin Murphy struct iova_domain iovad; 53fdbe574eSRobin Murphy /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ 54fdbe574eSRobin Murphy dma_addr_t msi_iova; 55fdbe574eSRobin Murphy }; 5644bb7e24SRobin Murphy struct list_head msi_page_list; 5744bb7e24SRobin Murphy spinlock_t msi_lock; 582da274cdSZhen Lei 592da274cdSZhen Lei /* Domain for flush queue callback; NULL if flush queue not in use */ 602da274cdSZhen Lei struct iommu_domain *fq_domain; 6144bb7e24SRobin Murphy }; 6244bb7e24SRobin Murphy 63fdbe574eSRobin Murphy static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) 64fdbe574eSRobin Murphy { 65fdbe574eSRobin Murphy if (cookie->type == IOMMU_DMA_IOVA_COOKIE) 66fdbe574eSRobin Murphy return cookie->iovad.granule; 67fdbe574eSRobin Murphy return PAGE_SIZE; 68fdbe574eSRobin Murphy } 69fdbe574eSRobin Murphy 70fdbe574eSRobin Murphy static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) 71fdbe574eSRobin Murphy { 72fdbe574eSRobin Murphy struct iommu_dma_cookie *cookie; 73fdbe574eSRobin Murphy 74fdbe574eSRobin Murphy cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); 75fdbe574eSRobin Murphy if (cookie) { 76fdbe574eSRobin Murphy spin_lock_init(&cookie->msi_lock); 77fdbe574eSRobin Murphy INIT_LIST_HEAD(&cookie->msi_page_list); 78fdbe574eSRobin Murphy cookie->type = type; 79fdbe574eSRobin Murphy } 80fdbe574eSRobin Murphy return cookie; 8144bb7e24SRobin Murphy } 8244bb7e24SRobin Murphy 830db2e5d1SRobin Murphy /** 840db2e5d1SRobin Murphy * iommu_get_dma_cookie - Acquire DMA-API resources for a domain 850db2e5d1SRobin Murphy * @domain: IOMMU domain to prepare for DMA-API usage 860db2e5d1SRobin Murphy * 870db2e5d1SRobin Murphy * IOMMU drivers should normally call this from their domain_alloc 880db2e5d1SRobin Murphy * callback when domain->type == IOMMU_DOMAIN_DMA. 890db2e5d1SRobin Murphy */ 900db2e5d1SRobin Murphy int iommu_get_dma_cookie(struct iommu_domain *domain) 910db2e5d1SRobin Murphy { 920db2e5d1SRobin Murphy if (domain->iova_cookie) 930db2e5d1SRobin Murphy return -EEXIST; 940db2e5d1SRobin Murphy 95fdbe574eSRobin Murphy domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE); 96fdbe574eSRobin Murphy if (!domain->iova_cookie) 9744bb7e24SRobin Murphy return -ENOMEM; 980db2e5d1SRobin Murphy 9944bb7e24SRobin Murphy return 0; 1000db2e5d1SRobin Murphy } 1010db2e5d1SRobin Murphy EXPORT_SYMBOL(iommu_get_dma_cookie); 1020db2e5d1SRobin Murphy 1030db2e5d1SRobin Murphy /** 104fdbe574eSRobin Murphy * iommu_get_msi_cookie - Acquire just MSI remapping resources 105fdbe574eSRobin Murphy * @domain: IOMMU domain to prepare 106fdbe574eSRobin Murphy * @base: Start address of IOVA region for MSI mappings 107fdbe574eSRobin Murphy * 108fdbe574eSRobin Murphy * Users who manage their own IOVA allocation and do not want DMA API support, 109fdbe574eSRobin Murphy * but would still like to take advantage of automatic MSI remapping, can use 110fdbe574eSRobin Murphy * this to initialise their own domain appropriately. Users should reserve a 111fdbe574eSRobin Murphy * contiguous IOVA region, starting at @base, large enough to accommodate the 112fdbe574eSRobin Murphy * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address 113fdbe574eSRobin Murphy * used by the devices attached to @domain. 114fdbe574eSRobin Murphy */ 115fdbe574eSRobin Murphy int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) 116fdbe574eSRobin Murphy { 117fdbe574eSRobin Murphy struct iommu_dma_cookie *cookie; 118fdbe574eSRobin Murphy 119fdbe574eSRobin Murphy if (domain->type != IOMMU_DOMAIN_UNMANAGED) 120fdbe574eSRobin Murphy return -EINVAL; 121fdbe574eSRobin Murphy 122fdbe574eSRobin Murphy if (domain->iova_cookie) 123fdbe574eSRobin Murphy return -EEXIST; 124fdbe574eSRobin Murphy 125fdbe574eSRobin Murphy cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE); 126fdbe574eSRobin Murphy if (!cookie) 127fdbe574eSRobin Murphy return -ENOMEM; 128fdbe574eSRobin Murphy 129fdbe574eSRobin Murphy cookie->msi_iova = base; 130fdbe574eSRobin Murphy domain->iova_cookie = cookie; 131fdbe574eSRobin Murphy return 0; 132fdbe574eSRobin Murphy } 133fdbe574eSRobin Murphy EXPORT_SYMBOL(iommu_get_msi_cookie); 134fdbe574eSRobin Murphy 135fdbe574eSRobin Murphy /** 1360db2e5d1SRobin Murphy * iommu_put_dma_cookie - Release a domain's DMA mapping resources 137fdbe574eSRobin Murphy * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or 138fdbe574eSRobin Murphy * iommu_get_msi_cookie() 1390db2e5d1SRobin Murphy * 1400db2e5d1SRobin Murphy * IOMMU drivers should normally call this from their domain_free callback. 1410db2e5d1SRobin Murphy */ 1420db2e5d1SRobin Murphy void iommu_put_dma_cookie(struct iommu_domain *domain) 1430db2e5d1SRobin Murphy { 14444bb7e24SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 14544bb7e24SRobin Murphy struct iommu_dma_msi_page *msi, *tmp; 1460db2e5d1SRobin Murphy 14744bb7e24SRobin Murphy if (!cookie) 1480db2e5d1SRobin Murphy return; 1490db2e5d1SRobin Murphy 150fdbe574eSRobin Murphy if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) 15144bb7e24SRobin Murphy put_iova_domain(&cookie->iovad); 15244bb7e24SRobin Murphy 15344bb7e24SRobin Murphy list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { 15444bb7e24SRobin Murphy list_del(&msi->list); 15544bb7e24SRobin Murphy kfree(msi); 15644bb7e24SRobin Murphy } 15744bb7e24SRobin Murphy kfree(cookie); 1580db2e5d1SRobin Murphy domain->iova_cookie = NULL; 1590db2e5d1SRobin Murphy } 1600db2e5d1SRobin Murphy EXPORT_SYMBOL(iommu_put_dma_cookie); 1610db2e5d1SRobin Murphy 162273df963SRobin Murphy /** 163273df963SRobin Murphy * iommu_dma_get_resv_regions - Reserved region driver helper 164273df963SRobin Murphy * @dev: Device from iommu_get_resv_regions() 165273df963SRobin Murphy * @list: Reserved region list from iommu_get_resv_regions() 166273df963SRobin Murphy * 167273df963SRobin Murphy * IOMMU drivers can use this to implement their .get_resv_regions callback 168cd2c9fcfSShameer Kolothum * for general non-IOMMU-specific reservations. Currently, this covers GICv3 169cd2c9fcfSShameer Kolothum * ITS region reservation on ACPI based ARM platforms that may require HW MSI 170cd2c9fcfSShameer Kolothum * reservation. 171273df963SRobin Murphy */ 172273df963SRobin Murphy void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) 173fade1ec0SRobin Murphy { 174fade1ec0SRobin Murphy 17598cc4f71SJoerg Roedel if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode)) 176cd2c9fcfSShameer Kolothum iort_iommu_msi_get_resv_regions(dev, list); 177f51dc892SShameer Kolothum 178fade1ec0SRobin Murphy } 179273df963SRobin Murphy EXPORT_SYMBOL(iommu_dma_get_resv_regions); 180fade1ec0SRobin Murphy 1817c1b058cSRobin Murphy static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie, 1827c1b058cSRobin Murphy phys_addr_t start, phys_addr_t end) 1837c1b058cSRobin Murphy { 1847c1b058cSRobin Murphy struct iova_domain *iovad = &cookie->iovad; 1857c1b058cSRobin Murphy struct iommu_dma_msi_page *msi_page; 1867c1b058cSRobin Murphy int i, num_pages; 1877c1b058cSRobin Murphy 1887c1b058cSRobin Murphy start -= iova_offset(iovad, start); 1897c1b058cSRobin Murphy num_pages = iova_align(iovad, end - start) >> iova_shift(iovad); 1907c1b058cSRobin Murphy 1917c1b058cSRobin Murphy msi_page = kcalloc(num_pages, sizeof(*msi_page), GFP_KERNEL); 1927c1b058cSRobin Murphy if (!msi_page) 1937c1b058cSRobin Murphy return -ENOMEM; 1947c1b058cSRobin Murphy 1957c1b058cSRobin Murphy for (i = 0; i < num_pages; i++) { 1967c1b058cSRobin Murphy msi_page[i].phys = start; 1977c1b058cSRobin Murphy msi_page[i].iova = start; 1987c1b058cSRobin Murphy INIT_LIST_HEAD(&msi_page[i].list); 1997c1b058cSRobin Murphy list_add(&msi_page[i].list, &cookie->msi_page_list); 2007c1b058cSRobin Murphy start += iovad->granule; 2017c1b058cSRobin Murphy } 2027c1b058cSRobin Murphy 2037c1b058cSRobin Murphy return 0; 2047c1b058cSRobin Murphy } 2057c1b058cSRobin Murphy 206aadad097SSrinath Mannam static int iova_reserve_pci_windows(struct pci_dev *dev, 207cd2c9fcfSShameer Kolothum struct iova_domain *iovad) 208cd2c9fcfSShameer Kolothum { 209cd2c9fcfSShameer Kolothum struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); 210cd2c9fcfSShameer Kolothum struct resource_entry *window; 211cd2c9fcfSShameer Kolothum unsigned long lo, hi; 212aadad097SSrinath Mannam phys_addr_t start = 0, end; 213cd2c9fcfSShameer Kolothum 214cd2c9fcfSShameer Kolothum resource_list_for_each_entry(window, &bridge->windows) { 215cd2c9fcfSShameer Kolothum if (resource_type(window->res) != IORESOURCE_MEM) 216cd2c9fcfSShameer Kolothum continue; 217cd2c9fcfSShameer Kolothum 218cd2c9fcfSShameer Kolothum lo = iova_pfn(iovad, window->res->start - window->offset); 219cd2c9fcfSShameer Kolothum hi = iova_pfn(iovad, window->res->end - window->offset); 220cd2c9fcfSShameer Kolothum reserve_iova(iovad, lo, hi); 221cd2c9fcfSShameer Kolothum } 222aadad097SSrinath Mannam 223aadad097SSrinath Mannam /* Get reserved DMA windows from host bridge */ 224aadad097SSrinath Mannam resource_list_for_each_entry(window, &bridge->dma_ranges) { 225aadad097SSrinath Mannam end = window->res->start - window->offset; 226aadad097SSrinath Mannam resv_iova: 227aadad097SSrinath Mannam if (end > start) { 228aadad097SSrinath Mannam lo = iova_pfn(iovad, start); 229aadad097SSrinath Mannam hi = iova_pfn(iovad, end); 230aadad097SSrinath Mannam reserve_iova(iovad, lo, hi); 231aadad097SSrinath Mannam } else { 232aadad097SSrinath Mannam /* dma_ranges list should be sorted */ 233aadad097SSrinath Mannam dev_err(&dev->dev, "Failed to reserve IOVA\n"); 234aadad097SSrinath Mannam return -EINVAL; 235aadad097SSrinath Mannam } 236aadad097SSrinath Mannam 237aadad097SSrinath Mannam start = window->res->end - window->offset + 1; 238aadad097SSrinath Mannam /* If window is last entry */ 239aadad097SSrinath Mannam if (window->node.next == &bridge->dma_ranges && 240aadad097SSrinath Mannam end != ~(dma_addr_t)0) { 241aadad097SSrinath Mannam end = ~(dma_addr_t)0; 242aadad097SSrinath Mannam goto resv_iova; 243aadad097SSrinath Mannam } 244aadad097SSrinath Mannam } 245aadad097SSrinath Mannam 246aadad097SSrinath Mannam return 0; 247cd2c9fcfSShameer Kolothum } 248cd2c9fcfSShameer Kolothum 2497c1b058cSRobin Murphy static int iova_reserve_iommu_regions(struct device *dev, 2507c1b058cSRobin Murphy struct iommu_domain *domain) 2517c1b058cSRobin Murphy { 2527c1b058cSRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 2537c1b058cSRobin Murphy struct iova_domain *iovad = &cookie->iovad; 2547c1b058cSRobin Murphy struct iommu_resv_region *region; 2557c1b058cSRobin Murphy LIST_HEAD(resv_regions); 2567c1b058cSRobin Murphy int ret = 0; 2577c1b058cSRobin Murphy 258aadad097SSrinath Mannam if (dev_is_pci(dev)) { 259aadad097SSrinath Mannam ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad); 260aadad097SSrinath Mannam if (ret) 261aadad097SSrinath Mannam return ret; 262aadad097SSrinath Mannam } 263cd2c9fcfSShameer Kolothum 2647c1b058cSRobin Murphy iommu_get_resv_regions(dev, &resv_regions); 2657c1b058cSRobin Murphy list_for_each_entry(region, &resv_regions, list) { 2667c1b058cSRobin Murphy unsigned long lo, hi; 2677c1b058cSRobin Murphy 2687c1b058cSRobin Murphy /* We ARE the software that manages these! */ 2697c1b058cSRobin Murphy if (region->type == IOMMU_RESV_SW_MSI) 2707c1b058cSRobin Murphy continue; 2717c1b058cSRobin Murphy 2727c1b058cSRobin Murphy lo = iova_pfn(iovad, region->start); 2737c1b058cSRobin Murphy hi = iova_pfn(iovad, region->start + region->length - 1); 2747c1b058cSRobin Murphy reserve_iova(iovad, lo, hi); 2757c1b058cSRobin Murphy 2767c1b058cSRobin Murphy if (region->type == IOMMU_RESV_MSI) 2777c1b058cSRobin Murphy ret = cookie_init_hw_msi_region(cookie, region->start, 2787c1b058cSRobin Murphy region->start + region->length); 2797c1b058cSRobin Murphy if (ret) 2807c1b058cSRobin Murphy break; 2817c1b058cSRobin Murphy } 2827c1b058cSRobin Murphy iommu_put_resv_regions(dev, &resv_regions); 2837c1b058cSRobin Murphy 2847c1b058cSRobin Murphy return ret; 2857c1b058cSRobin Murphy } 2867c1b058cSRobin Murphy 2872da274cdSZhen Lei static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) 2882da274cdSZhen Lei { 2892da274cdSZhen Lei struct iommu_dma_cookie *cookie; 2902da274cdSZhen Lei struct iommu_domain *domain; 2912da274cdSZhen Lei 2922da274cdSZhen Lei cookie = container_of(iovad, struct iommu_dma_cookie, iovad); 2932da274cdSZhen Lei domain = cookie->fq_domain; 2942da274cdSZhen Lei /* 2952da274cdSZhen Lei * The IOMMU driver supporting DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE 2962da274cdSZhen Lei * implies that ops->flush_iotlb_all must be non-NULL. 2972da274cdSZhen Lei */ 2982da274cdSZhen Lei domain->ops->flush_iotlb_all(domain); 2992da274cdSZhen Lei } 3002da274cdSZhen Lei 3010db2e5d1SRobin Murphy /** 3020db2e5d1SRobin Murphy * iommu_dma_init_domain - Initialise a DMA mapping domain 3030db2e5d1SRobin Murphy * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() 3040db2e5d1SRobin Murphy * @base: IOVA at which the mappable address space starts 3050db2e5d1SRobin Murphy * @size: Size of IOVA space 306fade1ec0SRobin Murphy * @dev: Device the domain is being initialised for 3070db2e5d1SRobin Murphy * 3080db2e5d1SRobin Murphy * @base and @size should be exact multiples of IOMMU page granularity to 3090db2e5d1SRobin Murphy * avoid rounding surprises. If necessary, we reserve the page at address 0 3100db2e5d1SRobin Murphy * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but 3110db2e5d1SRobin Murphy * any change which could make prior IOVAs invalid will fail. 3120db2e5d1SRobin Murphy */ 31306d60728SChristoph Hellwig static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, 314fade1ec0SRobin Murphy u64 size, struct device *dev) 3150db2e5d1SRobin Murphy { 316fdbe574eSRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 317fdbe574eSRobin Murphy struct iova_domain *iovad = &cookie->iovad; 318c61a4633SShaokun Zhang unsigned long order, base_pfn; 3192da274cdSZhen Lei int attr; 3200db2e5d1SRobin Murphy 321fdbe574eSRobin Murphy if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) 322fdbe574eSRobin Murphy return -EINVAL; 3230db2e5d1SRobin Murphy 3240db2e5d1SRobin Murphy /* Use the smallest supported page size for IOVA granularity */ 325d16e0faaSRobin Murphy order = __ffs(domain->pgsize_bitmap); 3260db2e5d1SRobin Murphy base_pfn = max_t(unsigned long, 1, base >> order); 3270db2e5d1SRobin Murphy 3280db2e5d1SRobin Murphy /* Check the domain allows at least some access to the device... */ 3290db2e5d1SRobin Murphy if (domain->geometry.force_aperture) { 3300db2e5d1SRobin Murphy if (base > domain->geometry.aperture_end || 3310db2e5d1SRobin Murphy base + size <= domain->geometry.aperture_start) { 3320db2e5d1SRobin Murphy pr_warn("specified DMA range outside IOMMU capability\n"); 3330db2e5d1SRobin Murphy return -EFAULT; 3340db2e5d1SRobin Murphy } 3350db2e5d1SRobin Murphy /* ...then finally give it a kicking to make sure it fits */ 3360db2e5d1SRobin Murphy base_pfn = max_t(unsigned long, base_pfn, 3370db2e5d1SRobin Murphy domain->geometry.aperture_start >> order); 3380db2e5d1SRobin Murphy } 3390db2e5d1SRobin Murphy 340f51d7bb7SRobin Murphy /* start_pfn is always nonzero for an already-initialised domain */ 3410db2e5d1SRobin Murphy if (iovad->start_pfn) { 3420db2e5d1SRobin Murphy if (1UL << order != iovad->granule || 343f51d7bb7SRobin Murphy base_pfn != iovad->start_pfn) { 3440db2e5d1SRobin Murphy pr_warn("Incompatible range for DMA domain\n"); 3450db2e5d1SRobin Murphy return -EFAULT; 3460db2e5d1SRobin Murphy } 3477c1b058cSRobin Murphy 3480db2e5d1SRobin Murphy return 0; 3490db2e5d1SRobin Murphy } 3507c1b058cSRobin Murphy 351aa3ac946SZhen Lei init_iova_domain(iovad, 1UL << order, base_pfn); 3522da274cdSZhen Lei 3532da274cdSZhen Lei if (!cookie->fq_domain && !iommu_domain_get_attr(domain, 3542da274cdSZhen Lei DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) { 3552da274cdSZhen Lei cookie->fq_domain = domain; 3562da274cdSZhen Lei init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL); 3572da274cdSZhen Lei } 3582da274cdSZhen Lei 3597c1b058cSRobin Murphy if (!dev) 3607c1b058cSRobin Murphy return 0; 3617c1b058cSRobin Murphy 3627c1b058cSRobin Murphy return iova_reserve_iommu_regions(dev, domain); 3637c1b058cSRobin Murphy } 3640db2e5d1SRobin Murphy 3650db2e5d1SRobin Murphy /** 366737c85caSMitchel Humpherys * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API 367737c85caSMitchel Humpherys * page flags. 3680db2e5d1SRobin Murphy * @dir: Direction of DMA transfer 3690db2e5d1SRobin Murphy * @coherent: Is the DMA master cache-coherent? 370737c85caSMitchel Humpherys * @attrs: DMA attributes for the mapping 3710db2e5d1SRobin Murphy * 3720db2e5d1SRobin Murphy * Return: corresponding IOMMU API page protection flags 3730db2e5d1SRobin Murphy */ 37406d60728SChristoph Hellwig static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, 375737c85caSMitchel Humpherys unsigned long attrs) 3760db2e5d1SRobin Murphy { 3770db2e5d1SRobin Murphy int prot = coherent ? IOMMU_CACHE : 0; 3780db2e5d1SRobin Murphy 379737c85caSMitchel Humpherys if (attrs & DMA_ATTR_PRIVILEGED) 380737c85caSMitchel Humpherys prot |= IOMMU_PRIV; 381737c85caSMitchel Humpherys 3820db2e5d1SRobin Murphy switch (dir) { 3830db2e5d1SRobin Murphy case DMA_BIDIRECTIONAL: 3840db2e5d1SRobin Murphy return prot | IOMMU_READ | IOMMU_WRITE; 3850db2e5d1SRobin Murphy case DMA_TO_DEVICE: 3860db2e5d1SRobin Murphy return prot | IOMMU_READ; 3870db2e5d1SRobin Murphy case DMA_FROM_DEVICE: 3880db2e5d1SRobin Murphy return prot | IOMMU_WRITE; 3890db2e5d1SRobin Murphy default: 3900db2e5d1SRobin Murphy return 0; 3910db2e5d1SRobin Murphy } 3920db2e5d1SRobin Murphy } 3930db2e5d1SRobin Murphy 394842fe519SRobin Murphy static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, 395842fe519SRobin Murphy size_t size, dma_addr_t dma_limit, struct device *dev) 3960db2e5d1SRobin Murphy { 397a44e6657SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 398a44e6657SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 399bb65a64cSRobin Murphy unsigned long shift, iova_len, iova = 0; 4000db2e5d1SRobin Murphy 401a44e6657SRobin Murphy if (cookie->type == IOMMU_DMA_MSI_COOKIE) { 402a44e6657SRobin Murphy cookie->msi_iova += size; 403a44e6657SRobin Murphy return cookie->msi_iova - size; 404a44e6657SRobin Murphy } 405a44e6657SRobin Murphy 406a44e6657SRobin Murphy shift = iova_shift(iovad); 407a44e6657SRobin Murphy iova_len = size >> shift; 408bb65a64cSRobin Murphy /* 409bb65a64cSRobin Murphy * Freeing non-power-of-two-sized allocations back into the IOVA caches 410bb65a64cSRobin Murphy * will come back to bite us badly, so we have to waste a bit of space 411bb65a64cSRobin Murphy * rounding up anything cacheable to make sure that can't happen. The 412bb65a64cSRobin Murphy * order of the unadjusted size will still match upon freeing. 413bb65a64cSRobin Murphy */ 414bb65a64cSRobin Murphy if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) 415bb65a64cSRobin Murphy iova_len = roundup_pow_of_two(iova_len); 416a44e6657SRobin Murphy 41703bfdc31SRobin Murphy if (dev->bus_dma_mask) 41803bfdc31SRobin Murphy dma_limit &= dev->bus_dma_mask; 41903bfdc31SRobin Murphy 420c987ff0dSRobin Murphy if (domain->geometry.force_aperture) 421c987ff0dSRobin Murphy dma_limit = min(dma_limit, domain->geometry.aperture_end); 422122fac03SRobin Murphy 423122fac03SRobin Murphy /* Try to get PCI devices a SAC address */ 424122fac03SRobin Murphy if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) 425538d5b33STomasz Nowicki iova = alloc_iova_fast(iovad, iova_len, 426538d5b33STomasz Nowicki DMA_BIT_MASK(32) >> shift, false); 427122fac03SRobin Murphy 428bb65a64cSRobin Murphy if (!iova) 429538d5b33STomasz Nowicki iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, 430538d5b33STomasz Nowicki true); 431bb65a64cSRobin Murphy 432bb65a64cSRobin Murphy return (dma_addr_t)iova << shift; 4330db2e5d1SRobin Murphy } 4340db2e5d1SRobin Murphy 435842fe519SRobin Murphy static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, 436842fe519SRobin Murphy dma_addr_t iova, size_t size) 4370db2e5d1SRobin Murphy { 438842fe519SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 4390db2e5d1SRobin Murphy 440a44e6657SRobin Murphy /* The MSI case is only ever cleaning up its most recent allocation */ 441bb65a64cSRobin Murphy if (cookie->type == IOMMU_DMA_MSI_COOKIE) 442a44e6657SRobin Murphy cookie->msi_iova -= size; 4432da274cdSZhen Lei else if (cookie->fq_domain) /* non-strict mode */ 4442da274cdSZhen Lei queue_iova(iovad, iova_pfn(iovad, iova), 4452da274cdSZhen Lei size >> iova_shift(iovad), 0); 446bb65a64cSRobin Murphy else 4471cc896edSRobin Murphy free_iova_fast(iovad, iova_pfn(iovad, iova), 4481cc896edSRobin Murphy size >> iova_shift(iovad)); 449842fe519SRobin Murphy } 450842fe519SRobin Murphy 451b61d271eSRobin Murphy static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, 452842fe519SRobin Murphy size_t size) 453842fe519SRobin Murphy { 454b61d271eSRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 455a44e6657SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 456a44e6657SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 457842fe519SRobin Murphy size_t iova_off = iova_offset(iovad, dma_addr); 458842fe519SRobin Murphy 459842fe519SRobin Murphy dma_addr -= iova_off; 460842fe519SRobin Murphy size = iova_align(iovad, size + iova_off); 461842fe519SRobin Murphy 4622da274cdSZhen Lei WARN_ON(iommu_unmap_fast(domain, dma_addr, size) != size); 4632da274cdSZhen Lei if (!cookie->fq_domain) 4642da274cdSZhen Lei iommu_tlb_sync(domain); 465a44e6657SRobin Murphy iommu_dma_free_iova(cookie, dma_addr, size); 4660db2e5d1SRobin Murphy } 4670db2e5d1SRobin Murphy 46892aec09cSChristoph Hellwig static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, 469b61d271eSRobin Murphy size_t size, int prot) 47092aec09cSChristoph Hellwig { 471b61d271eSRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 47292aec09cSChristoph Hellwig struct iommu_dma_cookie *cookie = domain->iova_cookie; 47392aec09cSChristoph Hellwig size_t iova_off = 0; 47492aec09cSChristoph Hellwig dma_addr_t iova; 47592aec09cSChristoph Hellwig 47692aec09cSChristoph Hellwig if (cookie->type == IOMMU_DMA_IOVA_COOKIE) { 47792aec09cSChristoph Hellwig iova_off = iova_offset(&cookie->iovad, phys); 47892aec09cSChristoph Hellwig size = iova_align(&cookie->iovad, size + iova_off); 47992aec09cSChristoph Hellwig } 48092aec09cSChristoph Hellwig 48192aec09cSChristoph Hellwig iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); 48292aec09cSChristoph Hellwig if (!iova) 48392aec09cSChristoph Hellwig return DMA_MAPPING_ERROR; 48492aec09cSChristoph Hellwig 48592aec09cSChristoph Hellwig if (iommu_map(domain, iova, phys - iova_off, size, prot)) { 48692aec09cSChristoph Hellwig iommu_dma_free_iova(cookie, iova, size); 48792aec09cSChristoph Hellwig return DMA_MAPPING_ERROR; 48892aec09cSChristoph Hellwig } 48992aec09cSChristoph Hellwig return iova + iova_off; 49092aec09cSChristoph Hellwig } 49192aec09cSChristoph Hellwig 4920db2e5d1SRobin Murphy static void __iommu_dma_free_pages(struct page **pages, int count) 4930db2e5d1SRobin Murphy { 4940db2e5d1SRobin Murphy while (count--) 4950db2e5d1SRobin Murphy __free_page(pages[count]); 4960db2e5d1SRobin Murphy kvfree(pages); 4970db2e5d1SRobin Murphy } 4980db2e5d1SRobin Murphy 499c4b17afbSGanapatrao Kulkarni static struct page **__iommu_dma_alloc_pages(struct device *dev, 500c4b17afbSGanapatrao Kulkarni unsigned int count, unsigned long order_mask, gfp_t gfp) 5010db2e5d1SRobin Murphy { 5020db2e5d1SRobin Murphy struct page **pages; 503c4b17afbSGanapatrao Kulkarni unsigned int i = 0, nid = dev_to_node(dev); 5043b6b7e19SRobin Murphy 5053b6b7e19SRobin Murphy order_mask &= (2U << MAX_ORDER) - 1; 5063b6b7e19SRobin Murphy if (!order_mask) 5073b6b7e19SRobin Murphy return NULL; 5080db2e5d1SRobin Murphy 509c4b17afbSGanapatrao Kulkarni pages = kvzalloc(count * sizeof(*pages), GFP_KERNEL); 5100db2e5d1SRobin Murphy if (!pages) 5110db2e5d1SRobin Murphy return NULL; 5120db2e5d1SRobin Murphy 5130db2e5d1SRobin Murphy /* IOMMU can map any pages, so himem can also be used here */ 5140db2e5d1SRobin Murphy gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 5150db2e5d1SRobin Murphy 5160db2e5d1SRobin Murphy while (count) { 5170db2e5d1SRobin Murphy struct page *page = NULL; 5183b6b7e19SRobin Murphy unsigned int order_size; 5190db2e5d1SRobin Murphy 5200db2e5d1SRobin Murphy /* 5210db2e5d1SRobin Murphy * Higher-order allocations are a convenience rather 5220db2e5d1SRobin Murphy * than a necessity, hence using __GFP_NORETRY until 5233b6b7e19SRobin Murphy * falling back to minimum-order allocations. 5240db2e5d1SRobin Murphy */ 5253b6b7e19SRobin Murphy for (order_mask &= (2U << __fls(count)) - 1; 5263b6b7e19SRobin Murphy order_mask; order_mask &= ~order_size) { 5273b6b7e19SRobin Murphy unsigned int order = __fls(order_mask); 528c4b17afbSGanapatrao Kulkarni gfp_t alloc_flags = gfp; 5293b6b7e19SRobin Murphy 5303b6b7e19SRobin Murphy order_size = 1U << order; 531c4b17afbSGanapatrao Kulkarni if (order_mask > order_size) 532c4b17afbSGanapatrao Kulkarni alloc_flags |= __GFP_NORETRY; 533c4b17afbSGanapatrao Kulkarni page = alloc_pages_node(nid, alloc_flags, order); 5340db2e5d1SRobin Murphy if (!page) 5350db2e5d1SRobin Murphy continue; 5363b6b7e19SRobin Murphy if (!order) 5370db2e5d1SRobin Murphy break; 5383b6b7e19SRobin Murphy if (!PageCompound(page)) { 5390db2e5d1SRobin Murphy split_page(page, order); 5400db2e5d1SRobin Murphy break; 5413b6b7e19SRobin Murphy } else if (!split_huge_page(page)) { 5423b6b7e19SRobin Murphy break; 5430db2e5d1SRobin Murphy } 5443b6b7e19SRobin Murphy __free_pages(page, order); 5450db2e5d1SRobin Murphy } 5460db2e5d1SRobin Murphy if (!page) { 5470db2e5d1SRobin Murphy __iommu_dma_free_pages(pages, i); 5480db2e5d1SRobin Murphy return NULL; 5490db2e5d1SRobin Murphy } 5503b6b7e19SRobin Murphy count -= order_size; 5513b6b7e19SRobin Murphy while (order_size--) 5520db2e5d1SRobin Murphy pages[i++] = page++; 5530db2e5d1SRobin Murphy } 5540db2e5d1SRobin Murphy return pages; 5550db2e5d1SRobin Murphy } 5560db2e5d1SRobin Murphy 5574c360aceSRobin Murphy static struct page **__iommu_dma_get_pages(void *cpu_addr) 5584c360aceSRobin Murphy { 5594c360aceSRobin Murphy struct vm_struct *area = find_vm_area(cpu_addr); 5604c360aceSRobin Murphy 5614c360aceSRobin Murphy if (!area || !area->pages) 5624c360aceSRobin Murphy return NULL; 5634c360aceSRobin Murphy return area->pages; 5644c360aceSRobin Murphy } 5654c360aceSRobin Murphy 5660db2e5d1SRobin Murphy /** 56721b95aafSChristoph Hellwig * iommu_dma_alloc_remap - Allocate and map a buffer contiguous in IOVA space 5680db2e5d1SRobin Murphy * @dev: Device to allocate memory for. Must be a real device 5690db2e5d1SRobin Murphy * attached to an iommu_dma_domain 5700db2e5d1SRobin Murphy * @size: Size of buffer in bytes 57121b95aafSChristoph Hellwig * @dma_handle: Out argument for allocated DMA handle 5720db2e5d1SRobin Murphy * @gfp: Allocation flags 5733b6b7e19SRobin Murphy * @attrs: DMA attributes for this allocation 5740db2e5d1SRobin Murphy * 5750db2e5d1SRobin Murphy * If @size is less than PAGE_SIZE, then a full CPU page will be allocated, 5760db2e5d1SRobin Murphy * but an IOMMU which supports smaller pages might not map the whole thing. 5770db2e5d1SRobin Murphy * 57821b95aafSChristoph Hellwig * Return: Mapped virtual address, or NULL on failure. 5790db2e5d1SRobin Murphy */ 58021b95aafSChristoph Hellwig static void *iommu_dma_alloc_remap(struct device *dev, size_t size, 58121b95aafSChristoph Hellwig dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 5820db2e5d1SRobin Murphy { 58343c5bf11SRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 584842fe519SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 585842fe519SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 58621b95aafSChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 58721b95aafSChristoph Hellwig int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); 58821b95aafSChristoph Hellwig pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); 58921b95aafSChristoph Hellwig unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; 5900db2e5d1SRobin Murphy struct page **pages; 5910db2e5d1SRobin Murphy struct sg_table sgt; 592842fe519SRobin Murphy dma_addr_t iova; 59321b95aafSChristoph Hellwig void *vaddr; 5940db2e5d1SRobin Murphy 59521b95aafSChristoph Hellwig *dma_handle = DMA_MAPPING_ERROR; 5960db2e5d1SRobin Murphy 5973b6b7e19SRobin Murphy min_size = alloc_sizes & -alloc_sizes; 5983b6b7e19SRobin Murphy if (min_size < PAGE_SIZE) { 5993b6b7e19SRobin Murphy min_size = PAGE_SIZE; 6003b6b7e19SRobin Murphy alloc_sizes |= PAGE_SIZE; 6013b6b7e19SRobin Murphy } else { 6023b6b7e19SRobin Murphy size = ALIGN(size, min_size); 6033b6b7e19SRobin Murphy } 60400085f1eSKrzysztof Kozlowski if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) 6053b6b7e19SRobin Murphy alloc_sizes = min_size; 6063b6b7e19SRobin Murphy 6073b6b7e19SRobin Murphy count = PAGE_ALIGN(size) >> PAGE_SHIFT; 608c4b17afbSGanapatrao Kulkarni pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT, 609c4b17afbSGanapatrao Kulkarni gfp); 6100db2e5d1SRobin Murphy if (!pages) 6110db2e5d1SRobin Murphy return NULL; 6120db2e5d1SRobin Murphy 613842fe519SRobin Murphy size = iova_align(iovad, size); 614842fe519SRobin Murphy iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev); 6150db2e5d1SRobin Murphy if (!iova) 6160db2e5d1SRobin Murphy goto out_free_pages; 6170db2e5d1SRobin Murphy 6180db2e5d1SRobin Murphy if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, GFP_KERNEL)) 6190db2e5d1SRobin Murphy goto out_free_iova; 6200db2e5d1SRobin Murphy 62121b95aafSChristoph Hellwig if (!(ioprot & IOMMU_CACHE)) { 62223f88e0aSChristoph Hellwig struct scatterlist *sg; 62323f88e0aSChristoph Hellwig int i; 62423f88e0aSChristoph Hellwig 62523f88e0aSChristoph Hellwig for_each_sg(sgt.sgl, sg, sgt.orig_nents, i) 62623f88e0aSChristoph Hellwig arch_dma_prep_coherent(sg_page(sg), sg->length); 6270db2e5d1SRobin Murphy } 6280db2e5d1SRobin Murphy 62921b95aafSChristoph Hellwig if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, ioprot) 6300db2e5d1SRobin Murphy < size) 6310db2e5d1SRobin Murphy goto out_free_sg; 6320db2e5d1SRobin Murphy 63321b95aafSChristoph Hellwig vaddr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, 63421b95aafSChristoph Hellwig __builtin_return_address(0)); 63521b95aafSChristoph Hellwig if (!vaddr) 63621b95aafSChristoph Hellwig goto out_unmap; 6370db2e5d1SRobin Murphy 63821b95aafSChristoph Hellwig *dma_handle = iova; 63921b95aafSChristoph Hellwig sg_free_table(&sgt); 64021b95aafSChristoph Hellwig return vaddr; 64121b95aafSChristoph Hellwig 64221b95aafSChristoph Hellwig out_unmap: 64321b95aafSChristoph Hellwig __iommu_dma_unmap(dev, iova, size); 6440db2e5d1SRobin Murphy out_free_sg: 6450db2e5d1SRobin Murphy sg_free_table(&sgt); 6460db2e5d1SRobin Murphy out_free_iova: 647842fe519SRobin Murphy iommu_dma_free_iova(cookie, iova, size); 6480db2e5d1SRobin Murphy out_free_pages: 6490db2e5d1SRobin Murphy __iommu_dma_free_pages(pages, count); 6500db2e5d1SRobin Murphy return NULL; 6510db2e5d1SRobin Murphy } 6520db2e5d1SRobin Murphy 6530db2e5d1SRobin Murphy /** 65406d60728SChristoph Hellwig * __iommu_dma_mmap - Map a buffer into provided user VMA 65506d60728SChristoph Hellwig * @pages: Array representing buffer from __iommu_dma_alloc() 6560db2e5d1SRobin Murphy * @size: Size of buffer in bytes 6570db2e5d1SRobin Murphy * @vma: VMA describing requested userspace mapping 6580db2e5d1SRobin Murphy * 6590db2e5d1SRobin Murphy * Maps the pages of the buffer in @pages into @vma. The caller is responsible 6600db2e5d1SRobin Murphy * for verifying the correct size and protection of @vma beforehand. 6610db2e5d1SRobin Murphy */ 66206d60728SChristoph Hellwig static int __iommu_dma_mmap(struct page **pages, size_t size, 66306d60728SChristoph Hellwig struct vm_area_struct *vma) 6640db2e5d1SRobin Murphy { 665b0d0084fSSouptick Joarder return vm_map_pages(vma, pages, PAGE_ALIGN(size) >> PAGE_SHIFT); 6660db2e5d1SRobin Murphy } 6670db2e5d1SRobin Murphy 66806d60728SChristoph Hellwig static void iommu_dma_sync_single_for_cpu(struct device *dev, 66906d60728SChristoph Hellwig dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) 67006d60728SChristoph Hellwig { 67106d60728SChristoph Hellwig phys_addr_t phys; 67206d60728SChristoph Hellwig 67306d60728SChristoph Hellwig if (dev_is_dma_coherent(dev)) 67406d60728SChristoph Hellwig return; 67506d60728SChristoph Hellwig 67606d60728SChristoph Hellwig phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); 67706d60728SChristoph Hellwig arch_sync_dma_for_cpu(dev, phys, size, dir); 67806d60728SChristoph Hellwig } 67906d60728SChristoph Hellwig 68006d60728SChristoph Hellwig static void iommu_dma_sync_single_for_device(struct device *dev, 68106d60728SChristoph Hellwig dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) 68206d60728SChristoph Hellwig { 68306d60728SChristoph Hellwig phys_addr_t phys; 68406d60728SChristoph Hellwig 68506d60728SChristoph Hellwig if (dev_is_dma_coherent(dev)) 68606d60728SChristoph Hellwig return; 68706d60728SChristoph Hellwig 68806d60728SChristoph Hellwig phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); 68906d60728SChristoph Hellwig arch_sync_dma_for_device(dev, phys, size, dir); 69006d60728SChristoph Hellwig } 69106d60728SChristoph Hellwig 69206d60728SChristoph Hellwig static void iommu_dma_sync_sg_for_cpu(struct device *dev, 69306d60728SChristoph Hellwig struct scatterlist *sgl, int nelems, 69406d60728SChristoph Hellwig enum dma_data_direction dir) 69506d60728SChristoph Hellwig { 69606d60728SChristoph Hellwig struct scatterlist *sg; 69706d60728SChristoph Hellwig int i; 69806d60728SChristoph Hellwig 69906d60728SChristoph Hellwig if (dev_is_dma_coherent(dev)) 70006d60728SChristoph Hellwig return; 70106d60728SChristoph Hellwig 70206d60728SChristoph Hellwig for_each_sg(sgl, sg, nelems, i) 70306d60728SChristoph Hellwig arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); 70406d60728SChristoph Hellwig } 70506d60728SChristoph Hellwig 70606d60728SChristoph Hellwig static void iommu_dma_sync_sg_for_device(struct device *dev, 70706d60728SChristoph Hellwig struct scatterlist *sgl, int nelems, 70806d60728SChristoph Hellwig enum dma_data_direction dir) 70906d60728SChristoph Hellwig { 71006d60728SChristoph Hellwig struct scatterlist *sg; 71106d60728SChristoph Hellwig int i; 71206d60728SChristoph Hellwig 71306d60728SChristoph Hellwig if (dev_is_dma_coherent(dev)) 71406d60728SChristoph Hellwig return; 71506d60728SChristoph Hellwig 71606d60728SChristoph Hellwig for_each_sg(sgl, sg, nelems, i) 71706d60728SChristoph Hellwig arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); 71806d60728SChristoph Hellwig } 71906d60728SChristoph Hellwig 72006d60728SChristoph Hellwig static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 72106d60728SChristoph Hellwig unsigned long offset, size_t size, enum dma_data_direction dir, 72206d60728SChristoph Hellwig unsigned long attrs) 72306d60728SChristoph Hellwig { 72406d60728SChristoph Hellwig phys_addr_t phys = page_to_phys(page) + offset; 72506d60728SChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 726b61d271eSRobin Murphy int prot = dma_info_to_prot(dir, coherent, attrs); 72706d60728SChristoph Hellwig dma_addr_t dma_handle; 72806d60728SChristoph Hellwig 729b61d271eSRobin Murphy dma_handle =__iommu_dma_map(dev, phys, size, prot); 73006d60728SChristoph Hellwig if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && 73106d60728SChristoph Hellwig dma_handle != DMA_MAPPING_ERROR) 73206d60728SChristoph Hellwig arch_sync_dma_for_device(dev, phys, size, dir); 73306d60728SChristoph Hellwig return dma_handle; 73406d60728SChristoph Hellwig } 73506d60728SChristoph Hellwig 73606d60728SChristoph Hellwig static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 73706d60728SChristoph Hellwig size_t size, enum dma_data_direction dir, unsigned long attrs) 73806d60728SChristoph Hellwig { 73906d60728SChristoph Hellwig if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 74006d60728SChristoph Hellwig iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); 741b61d271eSRobin Murphy __iommu_dma_unmap(dev, dma_handle, size); 74206d60728SChristoph Hellwig } 74306d60728SChristoph Hellwig 7440db2e5d1SRobin Murphy /* 7450db2e5d1SRobin Murphy * Prepare a successfully-mapped scatterlist to give back to the caller. 746809eac54SRobin Murphy * 747809eac54SRobin Murphy * At this point the segments are already laid out by iommu_dma_map_sg() to 748809eac54SRobin Murphy * avoid individually crossing any boundaries, so we merely need to check a 749809eac54SRobin Murphy * segment's start address to avoid concatenating across one. 7500db2e5d1SRobin Murphy */ 7510db2e5d1SRobin Murphy static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, 7520db2e5d1SRobin Murphy dma_addr_t dma_addr) 7530db2e5d1SRobin Murphy { 754809eac54SRobin Murphy struct scatterlist *s, *cur = sg; 755809eac54SRobin Murphy unsigned long seg_mask = dma_get_seg_boundary(dev); 756809eac54SRobin Murphy unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); 757809eac54SRobin Murphy int i, count = 0; 7580db2e5d1SRobin Murphy 7590db2e5d1SRobin Murphy for_each_sg(sg, s, nents, i) { 760809eac54SRobin Murphy /* Restore this segment's original unaligned fields first */ 761809eac54SRobin Murphy unsigned int s_iova_off = sg_dma_address(s); 7620db2e5d1SRobin Murphy unsigned int s_length = sg_dma_len(s); 763809eac54SRobin Murphy unsigned int s_iova_len = s->length; 7640db2e5d1SRobin Murphy 765809eac54SRobin Murphy s->offset += s_iova_off; 7660db2e5d1SRobin Murphy s->length = s_length; 767cad34be7SChristoph Hellwig sg_dma_address(s) = DMA_MAPPING_ERROR; 768809eac54SRobin Murphy sg_dma_len(s) = 0; 769809eac54SRobin Murphy 770809eac54SRobin Murphy /* 771809eac54SRobin Murphy * Now fill in the real DMA data. If... 772809eac54SRobin Murphy * - there is a valid output segment to append to 773809eac54SRobin Murphy * - and this segment starts on an IOVA page boundary 774809eac54SRobin Murphy * - but doesn't fall at a segment boundary 775809eac54SRobin Murphy * - and wouldn't make the resulting output segment too long 776809eac54SRobin Murphy */ 777809eac54SRobin Murphy if (cur_len && !s_iova_off && (dma_addr & seg_mask) && 778809eac54SRobin Murphy (cur_len + s_length <= max_len)) { 779809eac54SRobin Murphy /* ...then concatenate it with the previous one */ 780809eac54SRobin Murphy cur_len += s_length; 781809eac54SRobin Murphy } else { 782809eac54SRobin Murphy /* Otherwise start the next output segment */ 783809eac54SRobin Murphy if (i > 0) 784809eac54SRobin Murphy cur = sg_next(cur); 785809eac54SRobin Murphy cur_len = s_length; 786809eac54SRobin Murphy count++; 787809eac54SRobin Murphy 788809eac54SRobin Murphy sg_dma_address(cur) = dma_addr + s_iova_off; 7890db2e5d1SRobin Murphy } 790809eac54SRobin Murphy 791809eac54SRobin Murphy sg_dma_len(cur) = cur_len; 792809eac54SRobin Murphy dma_addr += s_iova_len; 793809eac54SRobin Murphy 794809eac54SRobin Murphy if (s_length + s_iova_off < s_iova_len) 795809eac54SRobin Murphy cur_len = 0; 796809eac54SRobin Murphy } 797809eac54SRobin Murphy return count; 7980db2e5d1SRobin Murphy } 7990db2e5d1SRobin Murphy 8000db2e5d1SRobin Murphy /* 8010db2e5d1SRobin Murphy * If mapping failed, then just restore the original list, 8020db2e5d1SRobin Murphy * but making sure the DMA fields are invalidated. 8030db2e5d1SRobin Murphy */ 8040db2e5d1SRobin Murphy static void __invalidate_sg(struct scatterlist *sg, int nents) 8050db2e5d1SRobin Murphy { 8060db2e5d1SRobin Murphy struct scatterlist *s; 8070db2e5d1SRobin Murphy int i; 8080db2e5d1SRobin Murphy 8090db2e5d1SRobin Murphy for_each_sg(sg, s, nents, i) { 810cad34be7SChristoph Hellwig if (sg_dma_address(s) != DMA_MAPPING_ERROR) 81107b48ac4SRobin Murphy s->offset += sg_dma_address(s); 8120db2e5d1SRobin Murphy if (sg_dma_len(s)) 8130db2e5d1SRobin Murphy s->length = sg_dma_len(s); 814cad34be7SChristoph Hellwig sg_dma_address(s) = DMA_MAPPING_ERROR; 8150db2e5d1SRobin Murphy sg_dma_len(s) = 0; 8160db2e5d1SRobin Murphy } 8170db2e5d1SRobin Murphy } 8180db2e5d1SRobin Murphy 8190db2e5d1SRobin Murphy /* 8200db2e5d1SRobin Murphy * The DMA API client is passing in a scatterlist which could describe 8210db2e5d1SRobin Murphy * any old buffer layout, but the IOMMU API requires everything to be 8220db2e5d1SRobin Murphy * aligned to IOMMU pages. Hence the need for this complicated bit of 8230db2e5d1SRobin Murphy * impedance-matching, to be able to hand off a suitably-aligned list, 8240db2e5d1SRobin Murphy * but still preserve the original offsets and sizes for the caller. 8250db2e5d1SRobin Murphy */ 82606d60728SChristoph Hellwig static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, 82706d60728SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 8280db2e5d1SRobin Murphy { 82943c5bf11SRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 830842fe519SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 831842fe519SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 8320db2e5d1SRobin Murphy struct scatterlist *s, *prev = NULL; 83306d60728SChristoph Hellwig int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); 834842fe519SRobin Murphy dma_addr_t iova; 8350db2e5d1SRobin Murphy size_t iova_len = 0; 836809eac54SRobin Murphy unsigned long mask = dma_get_seg_boundary(dev); 8370db2e5d1SRobin Murphy int i; 8380db2e5d1SRobin Murphy 83906d60728SChristoph Hellwig if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 84006d60728SChristoph Hellwig iommu_dma_sync_sg_for_device(dev, sg, nents, dir); 84106d60728SChristoph Hellwig 8420db2e5d1SRobin Murphy /* 8430db2e5d1SRobin Murphy * Work out how much IOVA space we need, and align the segments to 8440db2e5d1SRobin Murphy * IOVA granules for the IOMMU driver to handle. With some clever 8450db2e5d1SRobin Murphy * trickery we can modify the list in-place, but reversibly, by 846809eac54SRobin Murphy * stashing the unaligned parts in the as-yet-unused DMA fields. 8470db2e5d1SRobin Murphy */ 8480db2e5d1SRobin Murphy for_each_sg(sg, s, nents, i) { 849809eac54SRobin Murphy size_t s_iova_off = iova_offset(iovad, s->offset); 8500db2e5d1SRobin Murphy size_t s_length = s->length; 851809eac54SRobin Murphy size_t pad_len = (mask - iova_len + 1) & mask; 8520db2e5d1SRobin Murphy 853809eac54SRobin Murphy sg_dma_address(s) = s_iova_off; 8540db2e5d1SRobin Murphy sg_dma_len(s) = s_length; 855809eac54SRobin Murphy s->offset -= s_iova_off; 856809eac54SRobin Murphy s_length = iova_align(iovad, s_length + s_iova_off); 8570db2e5d1SRobin Murphy s->length = s_length; 8580db2e5d1SRobin Murphy 8590db2e5d1SRobin Murphy /* 860809eac54SRobin Murphy * Due to the alignment of our single IOVA allocation, we can 861809eac54SRobin Murphy * depend on these assumptions about the segment boundary mask: 862809eac54SRobin Murphy * - If mask size >= IOVA size, then the IOVA range cannot 863809eac54SRobin Murphy * possibly fall across a boundary, so we don't care. 864809eac54SRobin Murphy * - If mask size < IOVA size, then the IOVA range must start 865809eac54SRobin Murphy * exactly on a boundary, therefore we can lay things out 866809eac54SRobin Murphy * based purely on segment lengths without needing to know 867809eac54SRobin Murphy * the actual addresses beforehand. 868809eac54SRobin Murphy * - The mask must be a power of 2, so pad_len == 0 if 869809eac54SRobin Murphy * iova_len == 0, thus we cannot dereference prev the first 870809eac54SRobin Murphy * time through here (i.e. before it has a meaningful value). 8710db2e5d1SRobin Murphy */ 872809eac54SRobin Murphy if (pad_len && pad_len < s_length - 1) { 8730db2e5d1SRobin Murphy prev->length += pad_len; 8740db2e5d1SRobin Murphy iova_len += pad_len; 8750db2e5d1SRobin Murphy } 8760db2e5d1SRobin Murphy 8770db2e5d1SRobin Murphy iova_len += s_length; 8780db2e5d1SRobin Murphy prev = s; 8790db2e5d1SRobin Murphy } 8800db2e5d1SRobin Murphy 881842fe519SRobin Murphy iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev); 8820db2e5d1SRobin Murphy if (!iova) 8830db2e5d1SRobin Murphy goto out_restore_sg; 8840db2e5d1SRobin Murphy 8850db2e5d1SRobin Murphy /* 8860db2e5d1SRobin Murphy * We'll leave any physical concatenation to the IOMMU driver's 8870db2e5d1SRobin Murphy * implementation - it knows better than we do. 8880db2e5d1SRobin Murphy */ 889842fe519SRobin Murphy if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len) 8900db2e5d1SRobin Murphy goto out_free_iova; 8910db2e5d1SRobin Murphy 892842fe519SRobin Murphy return __finalise_sg(dev, sg, nents, iova); 8930db2e5d1SRobin Murphy 8940db2e5d1SRobin Murphy out_free_iova: 895842fe519SRobin Murphy iommu_dma_free_iova(cookie, iova, iova_len); 8960db2e5d1SRobin Murphy out_restore_sg: 8970db2e5d1SRobin Murphy __invalidate_sg(sg, nents); 8980db2e5d1SRobin Murphy return 0; 8990db2e5d1SRobin Murphy } 9000db2e5d1SRobin Murphy 90106d60728SChristoph Hellwig static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, 90206d60728SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 9030db2e5d1SRobin Murphy { 904842fe519SRobin Murphy dma_addr_t start, end; 905842fe519SRobin Murphy struct scatterlist *tmp; 906842fe519SRobin Murphy int i; 90706d60728SChristoph Hellwig 90806d60728SChristoph Hellwig if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 90906d60728SChristoph Hellwig iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); 91006d60728SChristoph Hellwig 9110db2e5d1SRobin Murphy /* 9120db2e5d1SRobin Murphy * The scatterlist segments are mapped into a single 9130db2e5d1SRobin Murphy * contiguous IOVA allocation, so this is incredibly easy. 9140db2e5d1SRobin Murphy */ 915842fe519SRobin Murphy start = sg_dma_address(sg); 916842fe519SRobin Murphy for_each_sg(sg_next(sg), tmp, nents - 1, i) { 917842fe519SRobin Murphy if (sg_dma_len(tmp) == 0) 918842fe519SRobin Murphy break; 919842fe519SRobin Murphy sg = tmp; 920842fe519SRobin Murphy } 921842fe519SRobin Murphy end = sg_dma_address(sg) + sg_dma_len(sg); 922b61d271eSRobin Murphy __iommu_dma_unmap(dev, start, end - start); 9230db2e5d1SRobin Murphy } 9240db2e5d1SRobin Murphy 92506d60728SChristoph Hellwig static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, 92651f8cc9eSRobin Murphy size_t size, enum dma_data_direction dir, unsigned long attrs) 92751f8cc9eSRobin Murphy { 92851f8cc9eSRobin Murphy return __iommu_dma_map(dev, phys, size, 929b61d271eSRobin Murphy dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO); 93051f8cc9eSRobin Murphy } 93151f8cc9eSRobin Murphy 93206d60728SChristoph Hellwig static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, 93351f8cc9eSRobin Murphy size_t size, enum dma_data_direction dir, unsigned long attrs) 93451f8cc9eSRobin Murphy { 935b61d271eSRobin Murphy __iommu_dma_unmap(dev, handle, size); 93651f8cc9eSRobin Murphy } 93751f8cc9eSRobin Murphy 9388553f6e6SRobin Murphy static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) 939bcf4b9c4SRobin Murphy { 940bcf4b9c4SRobin Murphy size_t alloc_size = PAGE_ALIGN(size); 941bcf4b9c4SRobin Murphy int count = alloc_size >> PAGE_SHIFT; 942bcf4b9c4SRobin Murphy struct page *page = NULL, **pages = NULL; 943bcf4b9c4SRobin Murphy 944bcf4b9c4SRobin Murphy /* Non-coherent atomic allocation? Easy */ 945bcf4b9c4SRobin Murphy if (dma_free_from_pool(cpu_addr, alloc_size)) 946bcf4b9c4SRobin Murphy return; 947bcf4b9c4SRobin Murphy 948bcf4b9c4SRobin Murphy if (is_vmalloc_addr(cpu_addr)) { 949bcf4b9c4SRobin Murphy /* 950bcf4b9c4SRobin Murphy * If it the address is remapped, then it's either non-coherent 951bcf4b9c4SRobin Murphy * or highmem CMA, or an iommu_dma_alloc_remap() construction. 952bcf4b9c4SRobin Murphy */ 953bcf4b9c4SRobin Murphy pages = __iommu_dma_get_pages(cpu_addr); 954bcf4b9c4SRobin Murphy if (!pages) 955bcf4b9c4SRobin Murphy page = vmalloc_to_page(cpu_addr); 956bcf4b9c4SRobin Murphy dma_common_free_remap(cpu_addr, alloc_size, VM_USERMAP); 957bcf4b9c4SRobin Murphy } else { 958bcf4b9c4SRobin Murphy /* Lowmem means a coherent atomic or CMA allocation */ 959bcf4b9c4SRobin Murphy page = virt_to_page(cpu_addr); 960bcf4b9c4SRobin Murphy } 961bcf4b9c4SRobin Murphy 962bcf4b9c4SRobin Murphy if (pages) 963bcf4b9c4SRobin Murphy __iommu_dma_free_pages(pages, count); 964bcf4b9c4SRobin Murphy if (page && !dma_release_from_contiguous(dev, page, count)) 965bcf4b9c4SRobin Murphy __free_pages(page, get_order(alloc_size)); 966bcf4b9c4SRobin Murphy } 967bcf4b9c4SRobin Murphy 9688553f6e6SRobin Murphy static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, 9698553f6e6SRobin Murphy dma_addr_t handle, unsigned long attrs) 9708553f6e6SRobin Murphy { 9718553f6e6SRobin Murphy __iommu_dma_unmap(dev, handle, size); 9728553f6e6SRobin Murphy __iommu_dma_free(dev, size, cpu_addr); 9738553f6e6SRobin Murphy } 9748553f6e6SRobin Murphy 975ee1ef05dSChristoph Hellwig static void *iommu_dma_alloc_pages(struct device *dev, size_t size, 976ee1ef05dSChristoph Hellwig struct page **pagep, gfp_t gfp, unsigned long attrs) 97706d60728SChristoph Hellwig { 97806d60728SChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 9799ad5d6edSRobin Murphy size_t alloc_size = PAGE_ALIGN(size); 9809a4ab94aSChristoph Hellwig struct page *page = NULL; 9819ad5d6edSRobin Murphy void *cpu_addr; 98206d60728SChristoph Hellwig 9839a4ab94aSChristoph Hellwig if (gfpflags_allow_blocking(gfp)) 9849ad5d6edSRobin Murphy page = dma_alloc_from_contiguous(dev, alloc_size >> PAGE_SHIFT, 9859ad5d6edSRobin Murphy get_order(alloc_size), 9869a4ab94aSChristoph Hellwig gfp & __GFP_NOWARN); 9879a4ab94aSChristoph Hellwig if (!page) 9889ad5d6edSRobin Murphy page = alloc_pages(gfp, get_order(alloc_size)); 98906d60728SChristoph Hellwig if (!page) 99006d60728SChristoph Hellwig return NULL; 99106d60728SChristoph Hellwig 9928680aa5aSRobin Murphy if (!coherent || PageHighMem(page)) { 9938680aa5aSRobin Murphy pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); 9948680aa5aSRobin Murphy 9959ad5d6edSRobin Murphy cpu_addr = dma_common_contiguous_remap(page, alloc_size, 9969ad5d6edSRobin Murphy VM_USERMAP, prot, __builtin_return_address(0)); 9979ad5d6edSRobin Murphy if (!cpu_addr) 998ee1ef05dSChristoph Hellwig goto out_free_pages; 999072bebc0SRobin Murphy 100006d60728SChristoph Hellwig if (!coherent) 10019ad5d6edSRobin Murphy arch_dma_prep_coherent(page, size); 10028680aa5aSRobin Murphy } else { 10039ad5d6edSRobin Murphy cpu_addr = page_address(page); 10048680aa5aSRobin Murphy } 1005ee1ef05dSChristoph Hellwig 1006ee1ef05dSChristoph Hellwig *pagep = page; 10079ad5d6edSRobin Murphy memset(cpu_addr, 0, alloc_size); 10089ad5d6edSRobin Murphy return cpu_addr; 1009072bebc0SRobin Murphy out_free_pages: 10109ad5d6edSRobin Murphy if (!dma_release_from_contiguous(dev, page, alloc_size >> PAGE_SHIFT)) 10119ad5d6edSRobin Murphy __free_pages(page, get_order(alloc_size)); 1012072bebc0SRobin Murphy return NULL; 101306d60728SChristoph Hellwig } 101406d60728SChristoph Hellwig 1015ee1ef05dSChristoph Hellwig static void *iommu_dma_alloc(struct device *dev, size_t size, 1016ee1ef05dSChristoph Hellwig dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 1017ee1ef05dSChristoph Hellwig { 1018ee1ef05dSChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 1019ee1ef05dSChristoph Hellwig int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); 1020ee1ef05dSChristoph Hellwig struct page *page = NULL; 1021ee1ef05dSChristoph Hellwig void *cpu_addr; 1022ee1ef05dSChristoph Hellwig 1023ee1ef05dSChristoph Hellwig gfp |= __GFP_ZERO; 1024ee1ef05dSChristoph Hellwig 1025ee1ef05dSChristoph Hellwig if (gfpflags_allow_blocking(gfp) && 1026ee1ef05dSChristoph Hellwig !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) 1027ee1ef05dSChristoph Hellwig return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); 1028ee1ef05dSChristoph Hellwig 1029ee1ef05dSChristoph Hellwig if (!gfpflags_allow_blocking(gfp) && !coherent) 1030ee1ef05dSChristoph Hellwig cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp); 1031ee1ef05dSChristoph Hellwig else 1032ee1ef05dSChristoph Hellwig cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); 1033ee1ef05dSChristoph Hellwig if (!cpu_addr) 1034ee1ef05dSChristoph Hellwig return NULL; 1035ee1ef05dSChristoph Hellwig 1036ee1ef05dSChristoph Hellwig *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot); 1037ee1ef05dSChristoph Hellwig if (*handle == DMA_MAPPING_ERROR) { 1038ee1ef05dSChristoph Hellwig __iommu_dma_free(dev, size, cpu_addr); 1039ee1ef05dSChristoph Hellwig return NULL; 1040ee1ef05dSChristoph Hellwig } 1041ee1ef05dSChristoph Hellwig 1042ee1ef05dSChristoph Hellwig return cpu_addr; 1043ee1ef05dSChristoph Hellwig } 1044ee1ef05dSChristoph Hellwig 104506d60728SChristoph Hellwig static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, 104606d60728SChristoph Hellwig unsigned long pfn, size_t size) 104706d60728SChristoph Hellwig { 104806d60728SChristoph Hellwig int ret = -ENXIO; 104906d60728SChristoph Hellwig unsigned long nr_vma_pages = vma_pages(vma); 105006d60728SChristoph Hellwig unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 105106d60728SChristoph Hellwig unsigned long off = vma->vm_pgoff; 105206d60728SChristoph Hellwig 105306d60728SChristoph Hellwig if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 105406d60728SChristoph Hellwig ret = remap_pfn_range(vma, vma->vm_start, 105506d60728SChristoph Hellwig pfn + off, 105606d60728SChristoph Hellwig vma->vm_end - vma->vm_start, 105706d60728SChristoph Hellwig vma->vm_page_prot); 105806d60728SChristoph Hellwig } 105906d60728SChristoph Hellwig 106006d60728SChristoph Hellwig return ret; 106106d60728SChristoph Hellwig } 106206d60728SChristoph Hellwig 106306d60728SChristoph Hellwig static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, 106406d60728SChristoph Hellwig void *cpu_addr, dma_addr_t dma_addr, size_t size, 106506d60728SChristoph Hellwig unsigned long attrs) 106606d60728SChristoph Hellwig { 106706d60728SChristoph Hellwig unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 106806d60728SChristoph Hellwig unsigned long off = vma->vm_pgoff; 10694c360aceSRobin Murphy struct page **pages; 107006d60728SChristoph Hellwig int ret; 107106d60728SChristoph Hellwig 107206d60728SChristoph Hellwig vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); 107306d60728SChristoph Hellwig 107406d60728SChristoph Hellwig if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) 107506d60728SChristoph Hellwig return ret; 107606d60728SChristoph Hellwig 107706d60728SChristoph Hellwig if (off >= nr_pages || vma_pages(vma) > nr_pages - off) 107806d60728SChristoph Hellwig return -ENXIO; 107906d60728SChristoph Hellwig 108006d60728SChristoph Hellwig if (!is_vmalloc_addr(cpu_addr)) { 108106d60728SChristoph Hellwig unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); 108206d60728SChristoph Hellwig return __iommu_dma_mmap_pfn(vma, pfn, size); 108306d60728SChristoph Hellwig } 108406d60728SChristoph Hellwig 108506d60728SChristoph Hellwig if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { 108606d60728SChristoph Hellwig /* 108706d60728SChristoph Hellwig * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, 108806d60728SChristoph Hellwig * hence in the vmalloc space. 108906d60728SChristoph Hellwig */ 109006d60728SChristoph Hellwig unsigned long pfn = vmalloc_to_pfn(cpu_addr); 109106d60728SChristoph Hellwig return __iommu_dma_mmap_pfn(vma, pfn, size); 109206d60728SChristoph Hellwig } 109306d60728SChristoph Hellwig 10944c360aceSRobin Murphy pages = __iommu_dma_get_pages(cpu_addr); 10954c360aceSRobin Murphy if (!pages) 109606d60728SChristoph Hellwig return -ENXIO; 10974c360aceSRobin Murphy return __iommu_dma_mmap(pages, size, vma); 109806d60728SChristoph Hellwig } 109906d60728SChristoph Hellwig 110006d60728SChristoph Hellwig static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 110106d60728SChristoph Hellwig void *cpu_addr, dma_addr_t dma_addr, size_t size, 110206d60728SChristoph Hellwig unsigned long attrs) 110306d60728SChristoph Hellwig { 11043fb3378bSChristoph Hellwig struct page *page; 11053fb3378bSChristoph Hellwig int ret; 110606d60728SChristoph Hellwig 11073fb3378bSChristoph Hellwig if (is_vmalloc_addr(cpu_addr)) { 11083fb3378bSChristoph Hellwig struct page **pages = __iommu_dma_get_pages(cpu_addr); 11093fb3378bSChristoph Hellwig 11103fb3378bSChristoph Hellwig if (pages) { 11113fb3378bSChristoph Hellwig return sg_alloc_table_from_pages(sgt, pages, 11123fb3378bSChristoph Hellwig PAGE_ALIGN(size) >> PAGE_SHIFT, 11133fb3378bSChristoph Hellwig 0, size, GFP_KERNEL); 111406d60728SChristoph Hellwig } 111506d60728SChristoph Hellwig 11163fb3378bSChristoph Hellwig page = vmalloc_to_page(cpu_addr); 11173fb3378bSChristoph Hellwig } else { 11183fb3378bSChristoph Hellwig page = virt_to_page(cpu_addr); 111906d60728SChristoph Hellwig } 112006d60728SChristoph Hellwig 11213fb3378bSChristoph Hellwig ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 11223fb3378bSChristoph Hellwig if (!ret) 11233fb3378bSChristoph Hellwig sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 11243fb3378bSChristoph Hellwig return ret; 112506d60728SChristoph Hellwig } 112606d60728SChristoph Hellwig 112706d60728SChristoph Hellwig static const struct dma_map_ops iommu_dma_ops = { 112806d60728SChristoph Hellwig .alloc = iommu_dma_alloc, 112906d60728SChristoph Hellwig .free = iommu_dma_free, 113006d60728SChristoph Hellwig .mmap = iommu_dma_mmap, 113106d60728SChristoph Hellwig .get_sgtable = iommu_dma_get_sgtable, 113206d60728SChristoph Hellwig .map_page = iommu_dma_map_page, 113306d60728SChristoph Hellwig .unmap_page = iommu_dma_unmap_page, 113406d60728SChristoph Hellwig .map_sg = iommu_dma_map_sg, 113506d60728SChristoph Hellwig .unmap_sg = iommu_dma_unmap_sg, 113606d60728SChristoph Hellwig .sync_single_for_cpu = iommu_dma_sync_single_for_cpu, 113706d60728SChristoph Hellwig .sync_single_for_device = iommu_dma_sync_single_for_device, 113806d60728SChristoph Hellwig .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu, 113906d60728SChristoph Hellwig .sync_sg_for_device = iommu_dma_sync_sg_for_device, 114006d60728SChristoph Hellwig .map_resource = iommu_dma_map_resource, 114106d60728SChristoph Hellwig .unmap_resource = iommu_dma_unmap_resource, 114206d60728SChristoph Hellwig }; 114306d60728SChristoph Hellwig 114406d60728SChristoph Hellwig /* 114506d60728SChristoph Hellwig * The IOMMU core code allocates the default DMA domain, which the underlying 114606d60728SChristoph Hellwig * IOMMU driver needs to support via the dma-iommu layer. 114706d60728SChristoph Hellwig */ 114806d60728SChristoph Hellwig void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) 114906d60728SChristoph Hellwig { 115006d60728SChristoph Hellwig struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 115106d60728SChristoph Hellwig 115206d60728SChristoph Hellwig if (!domain) 115306d60728SChristoph Hellwig goto out_err; 115406d60728SChristoph Hellwig 115506d60728SChristoph Hellwig /* 115606d60728SChristoph Hellwig * The IOMMU core code allocates the default DMA domain, which the 115706d60728SChristoph Hellwig * underlying IOMMU driver needs to support via the dma-iommu layer. 115806d60728SChristoph Hellwig */ 115906d60728SChristoph Hellwig if (domain->type == IOMMU_DOMAIN_DMA) { 116006d60728SChristoph Hellwig if (iommu_dma_init_domain(domain, dma_base, size, dev)) 116106d60728SChristoph Hellwig goto out_err; 116206d60728SChristoph Hellwig dev->dma_ops = &iommu_dma_ops; 116306d60728SChristoph Hellwig } 116406d60728SChristoph Hellwig 116506d60728SChristoph Hellwig return; 116606d60728SChristoph Hellwig out_err: 116706d60728SChristoph Hellwig pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", 116806d60728SChristoph Hellwig dev_name(dev)); 116906d60728SChristoph Hellwig } 117006d60728SChristoph Hellwig 117144bb7e24SRobin Murphy static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, 117244bb7e24SRobin Murphy phys_addr_t msi_addr, struct iommu_domain *domain) 117344bb7e24SRobin Murphy { 117444bb7e24SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 117544bb7e24SRobin Murphy struct iommu_dma_msi_page *msi_page; 1176842fe519SRobin Murphy dma_addr_t iova; 117744bb7e24SRobin Murphy int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1178fdbe574eSRobin Murphy size_t size = cookie_msi_granule(cookie); 117944bb7e24SRobin Murphy 1180fdbe574eSRobin Murphy msi_addr &= ~(phys_addr_t)(size - 1); 118144bb7e24SRobin Murphy list_for_each_entry(msi_page, &cookie->msi_page_list, list) 118244bb7e24SRobin Murphy if (msi_page->phys == msi_addr) 118344bb7e24SRobin Murphy return msi_page; 118444bb7e24SRobin Murphy 118544bb7e24SRobin Murphy msi_page = kzalloc(sizeof(*msi_page), GFP_ATOMIC); 118644bb7e24SRobin Murphy if (!msi_page) 118744bb7e24SRobin Murphy return NULL; 118844bb7e24SRobin Murphy 1189b61d271eSRobin Murphy iova = __iommu_dma_map(dev, msi_addr, size, prot); 1190cad34be7SChristoph Hellwig if (iova == DMA_MAPPING_ERROR) 119144bb7e24SRobin Murphy goto out_free_page; 119244bb7e24SRobin Murphy 119344bb7e24SRobin Murphy INIT_LIST_HEAD(&msi_page->list); 1194a44e6657SRobin Murphy msi_page->phys = msi_addr; 1195a44e6657SRobin Murphy msi_page->iova = iova; 119644bb7e24SRobin Murphy list_add(&msi_page->list, &cookie->msi_page_list); 119744bb7e24SRobin Murphy return msi_page; 119844bb7e24SRobin Murphy 119944bb7e24SRobin Murphy out_free_page: 120044bb7e24SRobin Murphy kfree(msi_page); 120144bb7e24SRobin Murphy return NULL; 120244bb7e24SRobin Murphy } 120344bb7e24SRobin Murphy 1204ece6e6f0SJulien Grall int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) 120544bb7e24SRobin Murphy { 1206ece6e6f0SJulien Grall struct device *dev = msi_desc_to_dev(desc); 120744bb7e24SRobin Murphy struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 120844bb7e24SRobin Murphy struct iommu_dma_cookie *cookie; 120944bb7e24SRobin Murphy struct iommu_dma_msi_page *msi_page; 121044bb7e24SRobin Murphy unsigned long flags; 121144bb7e24SRobin Murphy 1212ece6e6f0SJulien Grall if (!domain || !domain->iova_cookie) { 1213ece6e6f0SJulien Grall desc->iommu_cookie = NULL; 1214ece6e6f0SJulien Grall return 0; 1215ece6e6f0SJulien Grall } 121644bb7e24SRobin Murphy 121744bb7e24SRobin Murphy cookie = domain->iova_cookie; 121844bb7e24SRobin Murphy 121944bb7e24SRobin Murphy /* 122044bb7e24SRobin Murphy * We disable IRQs to rule out a possible inversion against 122144bb7e24SRobin Murphy * irq_desc_lock if, say, someone tries to retarget the affinity 122244bb7e24SRobin Murphy * of an MSI from within an IPI handler. 122344bb7e24SRobin Murphy */ 122444bb7e24SRobin Murphy spin_lock_irqsave(&cookie->msi_lock, flags); 122544bb7e24SRobin Murphy msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); 122644bb7e24SRobin Murphy spin_unlock_irqrestore(&cookie->msi_lock, flags); 122744bb7e24SRobin Murphy 1228ece6e6f0SJulien Grall msi_desc_set_iommu_cookie(desc, msi_page); 1229ece6e6f0SJulien Grall 1230ece6e6f0SJulien Grall if (!msi_page) 1231ece6e6f0SJulien Grall return -ENOMEM; 1232ece6e6f0SJulien Grall return 0; 123344bb7e24SRobin Murphy } 1234ece6e6f0SJulien Grall 1235ece6e6f0SJulien Grall void iommu_dma_compose_msi_msg(struct msi_desc *desc, 1236ece6e6f0SJulien Grall struct msi_msg *msg) 1237ece6e6f0SJulien Grall { 1238ece6e6f0SJulien Grall struct device *dev = msi_desc_to_dev(desc); 1239ece6e6f0SJulien Grall const struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 1240ece6e6f0SJulien Grall const struct iommu_dma_msi_page *msi_page; 1241ece6e6f0SJulien Grall 1242ece6e6f0SJulien Grall msi_page = msi_desc_get_iommu_cookie(desc); 1243ece6e6f0SJulien Grall 1244ece6e6f0SJulien Grall if (!domain || !domain->iova_cookie || WARN_ON(!msi_page)) 1245ece6e6f0SJulien Grall return; 1246ece6e6f0SJulien Grall 1247ece6e6f0SJulien Grall msg->address_hi = upper_32_bits(msi_page->iova); 1248ece6e6f0SJulien Grall msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1; 1249ece6e6f0SJulien Grall msg->address_lo += lower_32_bits(msi_page->iova); 125044bb7e24SRobin Murphy } 125106d60728SChristoph Hellwig 125206d60728SChristoph Hellwig static int iommu_dma_init(void) 125306d60728SChristoph Hellwig { 125406d60728SChristoph Hellwig return iova_cache_get(); 125506d60728SChristoph Hellwig } 125606d60728SChristoph Hellwig arch_initcall(iommu_dma_init); 1257