1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20db2e5d1SRobin Murphy /* 30db2e5d1SRobin Murphy * A fairly generic DMA-API to IOMMU-API glue layer. 40db2e5d1SRobin Murphy * 50db2e5d1SRobin Murphy * Copyright (C) 2014-2015 ARM Ltd. 60db2e5d1SRobin Murphy * 70db2e5d1SRobin Murphy * based in part on arch/arm/mm/dma-mapping.c: 80db2e5d1SRobin Murphy * Copyright (C) 2000-2004 Russell King 90db2e5d1SRobin Murphy */ 100db2e5d1SRobin Murphy 11f51dc892SShameer Kolothum #include <linux/acpi_iort.h> 12a17e3026SRobin Murphy #include <linux/atomic.h> 13a17e3026SRobin Murphy #include <linux/crash_dump.h> 140db2e5d1SRobin Murphy #include <linux/device.h> 15a17e3026SRobin Murphy #include <linux/dma-direct.h> 16a17e3026SRobin Murphy #include <linux/dma-map-ops.h> 175b11e9cdSRobin Murphy #include <linux/gfp.h> 180db2e5d1SRobin Murphy #include <linux/huge_mm.h> 190db2e5d1SRobin Murphy #include <linux/iommu.h> 200db2e5d1SRobin Murphy #include <linux/iova.h> 2144bb7e24SRobin Murphy #include <linux/irq.h> 22b8397a8fSRobin Murphy #include <linux/list_sort.h> 2330280eeeSLogan Gunthorpe #include <linux/memremap.h> 240db2e5d1SRobin Murphy #include <linux/mm.h> 25c1864790SRobin Murphy #include <linux/mutex.h> 265cef282eSThierry Reding #include <linux/of_iommu.h> 27fade1ec0SRobin Murphy #include <linux/pci.h> 285b11e9cdSRobin Murphy #include <linux/scatterlist.h> 29a17e3026SRobin Murphy #include <linux/spinlock.h> 30a17e3026SRobin Murphy #include <linux/swiotlb.h> 315b11e9cdSRobin Murphy #include <linux/vmalloc.h> 320db2e5d1SRobin Murphy 33f2042ed2SRobin Murphy #include "dma-iommu.h" 34f2042ed2SRobin Murphy 3544bb7e24SRobin Murphy struct iommu_dma_msi_page { 3644bb7e24SRobin Murphy struct list_head list; 3744bb7e24SRobin Murphy dma_addr_t iova; 3844bb7e24SRobin Murphy phys_addr_t phys; 3944bb7e24SRobin Murphy }; 4044bb7e24SRobin Murphy 41fdbe574eSRobin Murphy enum iommu_dma_cookie_type { 42fdbe574eSRobin Murphy IOMMU_DMA_IOVA_COOKIE, 43fdbe574eSRobin Murphy IOMMU_DMA_MSI_COOKIE, 44fdbe574eSRobin Murphy }; 45fdbe574eSRobin Murphy 4644bb7e24SRobin Murphy struct iommu_dma_cookie { 47fdbe574eSRobin Murphy enum iommu_dma_cookie_type type; 48fdbe574eSRobin Murphy union { 49fdbe574eSRobin Murphy /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ 50a17e3026SRobin Murphy struct { 5144bb7e24SRobin Murphy struct iova_domain iovad; 52a17e3026SRobin Murphy 53a17e3026SRobin Murphy struct iova_fq __percpu *fq; /* Flush queue */ 54a17e3026SRobin Murphy /* Number of TLB flushes that have been started */ 55a17e3026SRobin Murphy atomic64_t fq_flush_start_cnt; 56a17e3026SRobin Murphy /* Number of TLB flushes that have been finished */ 57a17e3026SRobin Murphy atomic64_t fq_flush_finish_cnt; 58a17e3026SRobin Murphy /* Timer to regularily empty the flush queues */ 59a17e3026SRobin Murphy struct timer_list fq_timer; 60a17e3026SRobin Murphy /* 1 when timer is active, 0 when not */ 61a17e3026SRobin Murphy atomic_t fq_timer_on; 62a17e3026SRobin Murphy }; 63fdbe574eSRobin Murphy /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ 64fdbe574eSRobin Murphy dma_addr_t msi_iova; 65fdbe574eSRobin Murphy }; 6644bb7e24SRobin Murphy struct list_head msi_page_list; 672da274cdSZhen Lei 682da274cdSZhen Lei /* Domain for flush queue callback; NULL if flush queue not in use */ 692da274cdSZhen Lei struct iommu_domain *fq_domain; 70ac9a5d52SYunfei Wang struct mutex mutex; 7144bb7e24SRobin Murphy }; 7244bb7e24SRobin Murphy 73a8e8af35SLianbo Jiang static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled); 74af3e9579SLinus Torvalds bool iommu_dma_forcedac __read_mostly; 753542dcb1SRobin Murphy 763542dcb1SRobin Murphy static int __init iommu_dma_forcedac_setup(char *str) 773542dcb1SRobin Murphy { 783542dcb1SRobin Murphy int ret = kstrtobool(str, &iommu_dma_forcedac); 793542dcb1SRobin Murphy 803542dcb1SRobin Murphy if (!ret && iommu_dma_forcedac) 813542dcb1SRobin Murphy pr_info("Forcing DAC for PCI devices\n"); 823542dcb1SRobin Murphy return ret; 833542dcb1SRobin Murphy } 843542dcb1SRobin Murphy early_param("iommu.forcedac", iommu_dma_forcedac_setup); 85a8e8af35SLianbo Jiang 86a17e3026SRobin Murphy /* Number of entries per flush queue */ 87a17e3026SRobin Murphy #define IOVA_FQ_SIZE 256 88a17e3026SRobin Murphy 89a17e3026SRobin Murphy /* Timeout (in ms) after which entries are flushed from the queue */ 90a17e3026SRobin Murphy #define IOVA_FQ_TIMEOUT 10 91a17e3026SRobin Murphy 92a17e3026SRobin Murphy /* Flush queue entry for deferred flushing */ 93a17e3026SRobin Murphy struct iova_fq_entry { 94a17e3026SRobin Murphy unsigned long iova_pfn; 95a17e3026SRobin Murphy unsigned long pages; 96a17e3026SRobin Murphy struct list_head freelist; 97a17e3026SRobin Murphy u64 counter; /* Flush counter when this entry was added */ 98a17e3026SRobin Murphy }; 99a17e3026SRobin Murphy 100a17e3026SRobin Murphy /* Per-CPU flush queue structure */ 101a17e3026SRobin Murphy struct iova_fq { 102a17e3026SRobin Murphy struct iova_fq_entry entries[IOVA_FQ_SIZE]; 103a17e3026SRobin Murphy unsigned int head, tail; 104a17e3026SRobin Murphy spinlock_t lock; 105a17e3026SRobin Murphy }; 106a17e3026SRobin Murphy 107f7f07484SRobin Murphy #define fq_ring_for_each(i, fq) \ 108f7f07484SRobin Murphy for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) 109f7f07484SRobin Murphy 110f7f07484SRobin Murphy static inline bool fq_full(struct iova_fq *fq) 111f7f07484SRobin Murphy { 112f7f07484SRobin Murphy assert_spin_locked(&fq->lock); 113f7f07484SRobin Murphy return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head); 114f7f07484SRobin Murphy } 115f7f07484SRobin Murphy 116a17e3026SRobin Murphy static inline unsigned int fq_ring_add(struct iova_fq *fq) 117f7f07484SRobin Murphy { 118a17e3026SRobin Murphy unsigned int idx = fq->tail; 119f7f07484SRobin Murphy 120f7f07484SRobin Murphy assert_spin_locked(&fq->lock); 121f7f07484SRobin Murphy 122f7f07484SRobin Murphy fq->tail = (idx + 1) % IOVA_FQ_SIZE; 123f7f07484SRobin Murphy 124f7f07484SRobin Murphy return idx; 125f7f07484SRobin Murphy } 126f7f07484SRobin Murphy 127a17e3026SRobin Murphy static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq) 128f7f07484SRobin Murphy { 129a17e3026SRobin Murphy u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt); 130a17e3026SRobin Murphy unsigned int idx; 131f7f07484SRobin Murphy 132f7f07484SRobin Murphy assert_spin_locked(&fq->lock); 133f7f07484SRobin Murphy 134f7f07484SRobin Murphy fq_ring_for_each(idx, fq) { 135f7f07484SRobin Murphy 136f7f07484SRobin Murphy if (fq->entries[idx].counter >= counter) 137f7f07484SRobin Murphy break; 138f7f07484SRobin Murphy 139f7f07484SRobin Murphy put_pages_list(&fq->entries[idx].freelist); 140a17e3026SRobin Murphy free_iova_fast(&cookie->iovad, 141f7f07484SRobin Murphy fq->entries[idx].iova_pfn, 142f7f07484SRobin Murphy fq->entries[idx].pages); 143f7f07484SRobin Murphy 144f7f07484SRobin Murphy fq->head = (fq->head + 1) % IOVA_FQ_SIZE; 145f7f07484SRobin Murphy } 146f7f07484SRobin Murphy } 147f7f07484SRobin Murphy 148a17e3026SRobin Murphy static void fq_flush_iotlb(struct iommu_dma_cookie *cookie) 149f7f07484SRobin Murphy { 150a17e3026SRobin Murphy atomic64_inc(&cookie->fq_flush_start_cnt); 151a17e3026SRobin Murphy cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain); 152a17e3026SRobin Murphy atomic64_inc(&cookie->fq_flush_finish_cnt); 153f7f07484SRobin Murphy } 154f7f07484SRobin Murphy 155f7f07484SRobin Murphy static void fq_flush_timeout(struct timer_list *t) 156f7f07484SRobin Murphy { 157a17e3026SRobin Murphy struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer); 158f7f07484SRobin Murphy int cpu; 159f7f07484SRobin Murphy 160a17e3026SRobin Murphy atomic_set(&cookie->fq_timer_on, 0); 161a17e3026SRobin Murphy fq_flush_iotlb(cookie); 162f7f07484SRobin Murphy 163f7f07484SRobin Murphy for_each_possible_cpu(cpu) { 164f7f07484SRobin Murphy unsigned long flags; 165f7f07484SRobin Murphy struct iova_fq *fq; 166f7f07484SRobin Murphy 167a17e3026SRobin Murphy fq = per_cpu_ptr(cookie->fq, cpu); 168f7f07484SRobin Murphy spin_lock_irqsave(&fq->lock, flags); 169a17e3026SRobin Murphy fq_ring_free(cookie, fq); 170f7f07484SRobin Murphy spin_unlock_irqrestore(&fq->lock, flags); 171f7f07484SRobin Murphy } 172f7f07484SRobin Murphy } 173f7f07484SRobin Murphy 174a17e3026SRobin Murphy static void queue_iova(struct iommu_dma_cookie *cookie, 175f7f07484SRobin Murphy unsigned long pfn, unsigned long pages, 176f7f07484SRobin Murphy struct list_head *freelist) 177f7f07484SRobin Murphy { 178f7f07484SRobin Murphy struct iova_fq *fq; 179f7f07484SRobin Murphy unsigned long flags; 180a17e3026SRobin Murphy unsigned int idx; 181f7f07484SRobin Murphy 182f7f07484SRobin Murphy /* 183f7f07484SRobin Murphy * Order against the IOMMU driver's pagetable update from unmapping 184a17e3026SRobin Murphy * @pte, to guarantee that fq_flush_iotlb() observes that if called 185f7f07484SRobin Murphy * from a different CPU before we release the lock below. Full barrier 186f7f07484SRobin Murphy * so it also pairs with iommu_dma_init_fq() to avoid seeing partially 187f7f07484SRobin Murphy * written fq state here. 188f7f07484SRobin Murphy */ 189f7f07484SRobin Murphy smp_mb(); 190f7f07484SRobin Murphy 191a17e3026SRobin Murphy fq = raw_cpu_ptr(cookie->fq); 192f7f07484SRobin Murphy spin_lock_irqsave(&fq->lock, flags); 193f7f07484SRobin Murphy 194f7f07484SRobin Murphy /* 195f7f07484SRobin Murphy * First remove all entries from the flush queue that have already been 196f7f07484SRobin Murphy * flushed out on another CPU. This makes the fq_full() check below less 197f7f07484SRobin Murphy * likely to be true. 198f7f07484SRobin Murphy */ 199a17e3026SRobin Murphy fq_ring_free(cookie, fq); 200f7f07484SRobin Murphy 201f7f07484SRobin Murphy if (fq_full(fq)) { 202a17e3026SRobin Murphy fq_flush_iotlb(cookie); 203a17e3026SRobin Murphy fq_ring_free(cookie, fq); 204f7f07484SRobin Murphy } 205f7f07484SRobin Murphy 206f7f07484SRobin Murphy idx = fq_ring_add(fq); 207f7f07484SRobin Murphy 208f7f07484SRobin Murphy fq->entries[idx].iova_pfn = pfn; 209f7f07484SRobin Murphy fq->entries[idx].pages = pages; 210a17e3026SRobin Murphy fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt); 211f7f07484SRobin Murphy list_splice(freelist, &fq->entries[idx].freelist); 212f7f07484SRobin Murphy 213f7f07484SRobin Murphy spin_unlock_irqrestore(&fq->lock, flags); 214f7f07484SRobin Murphy 215f7f07484SRobin Murphy /* Avoid false sharing as much as possible. */ 216a17e3026SRobin Murphy if (!atomic_read(&cookie->fq_timer_on) && 217a17e3026SRobin Murphy !atomic_xchg(&cookie->fq_timer_on, 1)) 218a17e3026SRobin Murphy mod_timer(&cookie->fq_timer, 219f7f07484SRobin Murphy jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); 220f7f07484SRobin Murphy } 221f7f07484SRobin Murphy 222a17e3026SRobin Murphy static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie) 223f7f07484SRobin Murphy { 224f7f07484SRobin Murphy int cpu, idx; 225f7f07484SRobin Murphy 226a17e3026SRobin Murphy if (!cookie->fq) 227f7f07484SRobin Murphy return; 228f7f07484SRobin Murphy 229a17e3026SRobin Murphy del_timer_sync(&cookie->fq_timer); 230a17e3026SRobin Murphy /* The IOVAs will be torn down separately, so just free our queued pages */ 231f7f07484SRobin Murphy for_each_possible_cpu(cpu) { 232a17e3026SRobin Murphy struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu); 233f7f07484SRobin Murphy 234f7f07484SRobin Murphy fq_ring_for_each(idx, fq) 235f7f07484SRobin Murphy put_pages_list(&fq->entries[idx].freelist); 236f7f07484SRobin Murphy } 237f7f07484SRobin Murphy 238a17e3026SRobin Murphy free_percpu(cookie->fq); 239f7f07484SRobin Murphy } 240f7f07484SRobin Murphy 241a17e3026SRobin Murphy /* sysfs updates are serialised by the mutex of the group owning @domain */ 242a17e3026SRobin Murphy int iommu_dma_init_fq(struct iommu_domain *domain) 243f7f07484SRobin Murphy { 244a17e3026SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 245f7f07484SRobin Murphy struct iova_fq __percpu *queue; 246f7f07484SRobin Murphy int i, cpu; 247f7f07484SRobin Murphy 248a17e3026SRobin Murphy if (cookie->fq_domain) 249a17e3026SRobin Murphy return 0; 250a17e3026SRobin Murphy 251a17e3026SRobin Murphy atomic64_set(&cookie->fq_flush_start_cnt, 0); 252a17e3026SRobin Murphy atomic64_set(&cookie->fq_flush_finish_cnt, 0); 253f7f07484SRobin Murphy 254f7f07484SRobin Murphy queue = alloc_percpu(struct iova_fq); 255a17e3026SRobin Murphy if (!queue) { 256a17e3026SRobin Murphy pr_warn("iova flush queue initialization failed\n"); 257f7f07484SRobin Murphy return -ENOMEM; 258a17e3026SRobin Murphy } 259f7f07484SRobin Murphy 260f7f07484SRobin Murphy for_each_possible_cpu(cpu) { 261f7f07484SRobin Murphy struct iova_fq *fq = per_cpu_ptr(queue, cpu); 262f7f07484SRobin Murphy 263f7f07484SRobin Murphy fq->head = 0; 264f7f07484SRobin Murphy fq->tail = 0; 265f7f07484SRobin Murphy 266f7f07484SRobin Murphy spin_lock_init(&fq->lock); 267f7f07484SRobin Murphy 268f7f07484SRobin Murphy for (i = 0; i < IOVA_FQ_SIZE; i++) 269f7f07484SRobin Murphy INIT_LIST_HEAD(&fq->entries[i].freelist); 270f7f07484SRobin Murphy } 271f7f07484SRobin Murphy 272a17e3026SRobin Murphy cookie->fq = queue; 273f7f07484SRobin Murphy 274a17e3026SRobin Murphy timer_setup(&cookie->fq_timer, fq_flush_timeout, 0); 275a17e3026SRobin Murphy atomic_set(&cookie->fq_timer_on, 0); 276a17e3026SRobin Murphy /* 277a17e3026SRobin Murphy * Prevent incomplete fq state being observable. Pairs with path from 278a17e3026SRobin Murphy * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() 279a17e3026SRobin Murphy */ 280a17e3026SRobin Murphy smp_wmb(); 281a17e3026SRobin Murphy WRITE_ONCE(cookie->fq_domain, domain); 282f7f07484SRobin Murphy return 0; 283f7f07484SRobin Murphy } 284f7f07484SRobin Murphy 285fdbe574eSRobin Murphy static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) 286fdbe574eSRobin Murphy { 287fdbe574eSRobin Murphy if (cookie->type == IOMMU_DMA_IOVA_COOKIE) 288fdbe574eSRobin Murphy return cookie->iovad.granule; 289fdbe574eSRobin Murphy return PAGE_SIZE; 290fdbe574eSRobin Murphy } 291fdbe574eSRobin Murphy 292fdbe574eSRobin Murphy static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) 293fdbe574eSRobin Murphy { 294fdbe574eSRobin Murphy struct iommu_dma_cookie *cookie; 295fdbe574eSRobin Murphy 296fdbe574eSRobin Murphy cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); 297fdbe574eSRobin Murphy if (cookie) { 298fdbe574eSRobin Murphy INIT_LIST_HEAD(&cookie->msi_page_list); 299fdbe574eSRobin Murphy cookie->type = type; 300fdbe574eSRobin Murphy } 301fdbe574eSRobin Murphy return cookie; 30244bb7e24SRobin Murphy } 30344bb7e24SRobin Murphy 3040db2e5d1SRobin Murphy /** 3050db2e5d1SRobin Murphy * iommu_get_dma_cookie - Acquire DMA-API resources for a domain 3060db2e5d1SRobin Murphy * @domain: IOMMU domain to prepare for DMA-API usage 3070db2e5d1SRobin Murphy */ 3080db2e5d1SRobin Murphy int iommu_get_dma_cookie(struct iommu_domain *domain) 3090db2e5d1SRobin Murphy { 3100db2e5d1SRobin Murphy if (domain->iova_cookie) 3110db2e5d1SRobin Murphy return -EEXIST; 3120db2e5d1SRobin Murphy 313fdbe574eSRobin Murphy domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE); 314fdbe574eSRobin Murphy if (!domain->iova_cookie) 31544bb7e24SRobin Murphy return -ENOMEM; 3160db2e5d1SRobin Murphy 317ac9a5d52SYunfei Wang mutex_init(&domain->iova_cookie->mutex); 31844bb7e24SRobin Murphy return 0; 3190db2e5d1SRobin Murphy } 3200db2e5d1SRobin Murphy 3210db2e5d1SRobin Murphy /** 322fdbe574eSRobin Murphy * iommu_get_msi_cookie - Acquire just MSI remapping resources 323fdbe574eSRobin Murphy * @domain: IOMMU domain to prepare 324fdbe574eSRobin Murphy * @base: Start address of IOVA region for MSI mappings 325fdbe574eSRobin Murphy * 326fdbe574eSRobin Murphy * Users who manage their own IOVA allocation and do not want DMA API support, 327fdbe574eSRobin Murphy * but would still like to take advantage of automatic MSI remapping, can use 328fdbe574eSRobin Murphy * this to initialise their own domain appropriately. Users should reserve a 329fdbe574eSRobin Murphy * contiguous IOVA region, starting at @base, large enough to accommodate the 330fdbe574eSRobin Murphy * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address 331fdbe574eSRobin Murphy * used by the devices attached to @domain. 332fdbe574eSRobin Murphy */ 333fdbe574eSRobin Murphy int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) 334fdbe574eSRobin Murphy { 335fdbe574eSRobin Murphy struct iommu_dma_cookie *cookie; 336fdbe574eSRobin Murphy 337fdbe574eSRobin Murphy if (domain->type != IOMMU_DOMAIN_UNMANAGED) 338fdbe574eSRobin Murphy return -EINVAL; 339fdbe574eSRobin Murphy 340fdbe574eSRobin Murphy if (domain->iova_cookie) 341fdbe574eSRobin Murphy return -EEXIST; 342fdbe574eSRobin Murphy 343fdbe574eSRobin Murphy cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE); 344fdbe574eSRobin Murphy if (!cookie) 345fdbe574eSRobin Murphy return -ENOMEM; 346fdbe574eSRobin Murphy 347fdbe574eSRobin Murphy cookie->msi_iova = base; 348fdbe574eSRobin Murphy domain->iova_cookie = cookie; 349fdbe574eSRobin Murphy return 0; 350fdbe574eSRobin Murphy } 351fdbe574eSRobin Murphy EXPORT_SYMBOL(iommu_get_msi_cookie); 352fdbe574eSRobin Murphy 353fdbe574eSRobin Murphy /** 3540db2e5d1SRobin Murphy * iommu_put_dma_cookie - Release a domain's DMA mapping resources 355fdbe574eSRobin Murphy * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or 356fdbe574eSRobin Murphy * iommu_get_msi_cookie() 3570db2e5d1SRobin Murphy */ 3580db2e5d1SRobin Murphy void iommu_put_dma_cookie(struct iommu_domain *domain) 3590db2e5d1SRobin Murphy { 36044bb7e24SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 36144bb7e24SRobin Murphy struct iommu_dma_msi_page *msi, *tmp; 3620db2e5d1SRobin Murphy 36344bb7e24SRobin Murphy if (!cookie) 3640db2e5d1SRobin Murphy return; 3650db2e5d1SRobin Murphy 366f7f07484SRobin Murphy if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { 367a17e3026SRobin Murphy iommu_dma_free_fq(cookie); 36844bb7e24SRobin Murphy put_iova_domain(&cookie->iovad); 369f7f07484SRobin Murphy } 37044bb7e24SRobin Murphy 37144bb7e24SRobin Murphy list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { 37244bb7e24SRobin Murphy list_del(&msi->list); 37344bb7e24SRobin Murphy kfree(msi); 37444bb7e24SRobin Murphy } 37544bb7e24SRobin Murphy kfree(cookie); 3760db2e5d1SRobin Murphy domain->iova_cookie = NULL; 3770db2e5d1SRobin Murphy } 3780db2e5d1SRobin Murphy 379273df963SRobin Murphy /** 380273df963SRobin Murphy * iommu_dma_get_resv_regions - Reserved region driver helper 381273df963SRobin Murphy * @dev: Device from iommu_get_resv_regions() 382273df963SRobin Murphy * @list: Reserved region list from iommu_get_resv_regions() 383273df963SRobin Murphy * 384273df963SRobin Murphy * IOMMU drivers can use this to implement their .get_resv_regions callback 385cd2c9fcfSShameer Kolothum * for general non-IOMMU-specific reservations. Currently, this covers GICv3 386cd2c9fcfSShameer Kolothum * ITS region reservation on ACPI based ARM platforms that may require HW MSI 387cd2c9fcfSShameer Kolothum * reservation. 388273df963SRobin Murphy */ 389273df963SRobin Murphy void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) 390fade1ec0SRobin Murphy { 391fade1ec0SRobin Murphy 39298cc4f71SJoerg Roedel if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode)) 39355be25b8SShameer Kolothum iort_iommu_get_resv_regions(dev, list); 394f51dc892SShameer Kolothum 3955cef282eSThierry Reding if (dev->of_node) 3965cef282eSThierry Reding of_iommu_get_resv_regions(dev, list); 397fade1ec0SRobin Murphy } 398273df963SRobin Murphy EXPORT_SYMBOL(iommu_dma_get_resv_regions); 399fade1ec0SRobin Murphy 4007c1b058cSRobin Murphy static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie, 4017c1b058cSRobin Murphy phys_addr_t start, phys_addr_t end) 4027c1b058cSRobin Murphy { 4037c1b058cSRobin Murphy struct iova_domain *iovad = &cookie->iovad; 4047c1b058cSRobin Murphy struct iommu_dma_msi_page *msi_page; 4057c1b058cSRobin Murphy int i, num_pages; 4067c1b058cSRobin Murphy 4077c1b058cSRobin Murphy start -= iova_offset(iovad, start); 4087c1b058cSRobin Murphy num_pages = iova_align(iovad, end - start) >> iova_shift(iovad); 4097c1b058cSRobin Murphy 41065ac74f1SMarc Zyngier for (i = 0; i < num_pages; i++) { 41165ac74f1SMarc Zyngier msi_page = kmalloc(sizeof(*msi_page), GFP_KERNEL); 4127c1b058cSRobin Murphy if (!msi_page) 4137c1b058cSRobin Murphy return -ENOMEM; 4147c1b058cSRobin Murphy 41565ac74f1SMarc Zyngier msi_page->phys = start; 41665ac74f1SMarc Zyngier msi_page->iova = start; 41765ac74f1SMarc Zyngier INIT_LIST_HEAD(&msi_page->list); 41865ac74f1SMarc Zyngier list_add(&msi_page->list, &cookie->msi_page_list); 4197c1b058cSRobin Murphy start += iovad->granule; 4207c1b058cSRobin Murphy } 4217c1b058cSRobin Murphy 4227c1b058cSRobin Murphy return 0; 4237c1b058cSRobin Murphy } 4247c1b058cSRobin Murphy 425b8397a8fSRobin Murphy static int iommu_dma_ranges_sort(void *priv, const struct list_head *a, 426b8397a8fSRobin Murphy const struct list_head *b) 427b8397a8fSRobin Murphy { 428b8397a8fSRobin Murphy struct resource_entry *res_a = list_entry(a, typeof(*res_a), node); 429b8397a8fSRobin Murphy struct resource_entry *res_b = list_entry(b, typeof(*res_b), node); 430b8397a8fSRobin Murphy 431b8397a8fSRobin Murphy return res_a->res->start > res_b->res->start; 432b8397a8fSRobin Murphy } 433b8397a8fSRobin Murphy 434aadad097SSrinath Mannam static int iova_reserve_pci_windows(struct pci_dev *dev, 435cd2c9fcfSShameer Kolothum struct iova_domain *iovad) 436cd2c9fcfSShameer Kolothum { 437cd2c9fcfSShameer Kolothum struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); 438cd2c9fcfSShameer Kolothum struct resource_entry *window; 439cd2c9fcfSShameer Kolothum unsigned long lo, hi; 440aadad097SSrinath Mannam phys_addr_t start = 0, end; 441cd2c9fcfSShameer Kolothum 442cd2c9fcfSShameer Kolothum resource_list_for_each_entry(window, &bridge->windows) { 443cd2c9fcfSShameer Kolothum if (resource_type(window->res) != IORESOURCE_MEM) 444cd2c9fcfSShameer Kolothum continue; 445cd2c9fcfSShameer Kolothum 446cd2c9fcfSShameer Kolothum lo = iova_pfn(iovad, window->res->start - window->offset); 447cd2c9fcfSShameer Kolothum hi = iova_pfn(iovad, window->res->end - window->offset); 448cd2c9fcfSShameer Kolothum reserve_iova(iovad, lo, hi); 449cd2c9fcfSShameer Kolothum } 450aadad097SSrinath Mannam 451aadad097SSrinath Mannam /* Get reserved DMA windows from host bridge */ 452b8397a8fSRobin Murphy list_sort(NULL, &bridge->dma_ranges, iommu_dma_ranges_sort); 453aadad097SSrinath Mannam resource_list_for_each_entry(window, &bridge->dma_ranges) { 454aadad097SSrinath Mannam end = window->res->start - window->offset; 455aadad097SSrinath Mannam resv_iova: 456aadad097SSrinath Mannam if (end > start) { 457aadad097SSrinath Mannam lo = iova_pfn(iovad, start); 458aadad097SSrinath Mannam hi = iova_pfn(iovad, end); 459aadad097SSrinath Mannam reserve_iova(iovad, lo, hi); 460571f3160SSrinath Mannam } else if (end < start) { 461b8397a8fSRobin Murphy /* DMA ranges should be non-overlapping */ 462571f3160SSrinath Mannam dev_err(&dev->dev, 4637154cbd3SJoerg Roedel "Failed to reserve IOVA [%pa-%pa]\n", 4647154cbd3SJoerg Roedel &start, &end); 465aadad097SSrinath Mannam return -EINVAL; 466aadad097SSrinath Mannam } 467aadad097SSrinath Mannam 468aadad097SSrinath Mannam start = window->res->end - window->offset + 1; 469aadad097SSrinath Mannam /* If window is last entry */ 470aadad097SSrinath Mannam if (window->node.next == &bridge->dma_ranges && 47129fcea8cSArnd Bergmann end != ~(phys_addr_t)0) { 47229fcea8cSArnd Bergmann end = ~(phys_addr_t)0; 473aadad097SSrinath Mannam goto resv_iova; 474aadad097SSrinath Mannam } 475aadad097SSrinath Mannam } 476aadad097SSrinath Mannam 477aadad097SSrinath Mannam return 0; 478cd2c9fcfSShameer Kolothum } 479cd2c9fcfSShameer Kolothum 4807c1b058cSRobin Murphy static int iova_reserve_iommu_regions(struct device *dev, 4817c1b058cSRobin Murphy struct iommu_domain *domain) 4827c1b058cSRobin Murphy { 4837c1b058cSRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 4847c1b058cSRobin Murphy struct iova_domain *iovad = &cookie->iovad; 4857c1b058cSRobin Murphy struct iommu_resv_region *region; 4867c1b058cSRobin Murphy LIST_HEAD(resv_regions); 4877c1b058cSRobin Murphy int ret = 0; 4887c1b058cSRobin Murphy 489aadad097SSrinath Mannam if (dev_is_pci(dev)) { 490aadad097SSrinath Mannam ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad); 491aadad097SSrinath Mannam if (ret) 492aadad097SSrinath Mannam return ret; 493aadad097SSrinath Mannam } 494cd2c9fcfSShameer Kolothum 4957c1b058cSRobin Murphy iommu_get_resv_regions(dev, &resv_regions); 4967c1b058cSRobin Murphy list_for_each_entry(region, &resv_regions, list) { 4977c1b058cSRobin Murphy unsigned long lo, hi; 4987c1b058cSRobin Murphy 4997c1b058cSRobin Murphy /* We ARE the software that manages these! */ 5007c1b058cSRobin Murphy if (region->type == IOMMU_RESV_SW_MSI) 5017c1b058cSRobin Murphy continue; 5027c1b058cSRobin Murphy 5037c1b058cSRobin Murphy lo = iova_pfn(iovad, region->start); 5047c1b058cSRobin Murphy hi = iova_pfn(iovad, region->start + region->length - 1); 5057c1b058cSRobin Murphy reserve_iova(iovad, lo, hi); 5067c1b058cSRobin Murphy 5077c1b058cSRobin Murphy if (region->type == IOMMU_RESV_MSI) 5087c1b058cSRobin Murphy ret = cookie_init_hw_msi_region(cookie, region->start, 5097c1b058cSRobin Murphy region->start + region->length); 5107c1b058cSRobin Murphy if (ret) 5117c1b058cSRobin Murphy break; 5127c1b058cSRobin Murphy } 5137c1b058cSRobin Murphy iommu_put_resv_regions(dev, &resv_regions); 5147c1b058cSRobin Murphy 5157c1b058cSRobin Murphy return ret; 5167c1b058cSRobin Murphy } 5177c1b058cSRobin Murphy 51882c3cefbSLu Baolu static bool dev_is_untrusted(struct device *dev) 51982c3cefbSLu Baolu { 52082c3cefbSLu Baolu return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; 52182c3cefbSLu Baolu } 52282c3cefbSLu Baolu 5232e727bffSDavid Stevens static bool dev_use_swiotlb(struct device *dev) 5242e727bffSDavid Stevens { 5252e727bffSDavid Stevens return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev); 5262e727bffSDavid Stevens } 5272e727bffSDavid Stevens 5280db2e5d1SRobin Murphy /** 5290db2e5d1SRobin Murphy * iommu_dma_init_domain - Initialise a DMA mapping domain 5300db2e5d1SRobin Murphy * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() 5310db2e5d1SRobin Murphy * @base: IOVA at which the mappable address space starts 532ac6d7046SJean-Philippe Brucker * @limit: Last address of the IOVA space 533fade1ec0SRobin Murphy * @dev: Device the domain is being initialised for 5340db2e5d1SRobin Murphy * 535ac6d7046SJean-Philippe Brucker * @base and @limit + 1 should be exact multiples of IOMMU page granularity to 5360db2e5d1SRobin Murphy * avoid rounding surprises. If necessary, we reserve the page at address 0 5370db2e5d1SRobin Murphy * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but 5380db2e5d1SRobin Murphy * any change which could make prior IOVAs invalid will fail. 5390db2e5d1SRobin Murphy */ 54006d60728SChristoph Hellwig static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, 541ac6d7046SJean-Philippe Brucker dma_addr_t limit, struct device *dev) 5420db2e5d1SRobin Murphy { 543fdbe574eSRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 544c61a4633SShaokun Zhang unsigned long order, base_pfn; 5456b0c54e7SYunsheng Lin struct iova_domain *iovad; 54632e92d9fSJohn Garry int ret; 5470db2e5d1SRobin Murphy 548fdbe574eSRobin Murphy if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) 549fdbe574eSRobin Murphy return -EINVAL; 5500db2e5d1SRobin Murphy 5516b0c54e7SYunsheng Lin iovad = &cookie->iovad; 5526b0c54e7SYunsheng Lin 5530db2e5d1SRobin Murphy /* Use the smallest supported page size for IOVA granularity */ 554d16e0faaSRobin Murphy order = __ffs(domain->pgsize_bitmap); 5550db2e5d1SRobin Murphy base_pfn = max_t(unsigned long, 1, base >> order); 5560db2e5d1SRobin Murphy 5570db2e5d1SRobin Murphy /* Check the domain allows at least some access to the device... */ 5580db2e5d1SRobin Murphy if (domain->geometry.force_aperture) { 5590db2e5d1SRobin Murphy if (base > domain->geometry.aperture_end || 560ac6d7046SJean-Philippe Brucker limit < domain->geometry.aperture_start) { 5610db2e5d1SRobin Murphy pr_warn("specified DMA range outside IOMMU capability\n"); 5620db2e5d1SRobin Murphy return -EFAULT; 5630db2e5d1SRobin Murphy } 5640db2e5d1SRobin Murphy /* ...then finally give it a kicking to make sure it fits */ 5650db2e5d1SRobin Murphy base_pfn = max_t(unsigned long, base_pfn, 5660db2e5d1SRobin Murphy domain->geometry.aperture_start >> order); 5670db2e5d1SRobin Murphy } 5680db2e5d1SRobin Murphy 569f51d7bb7SRobin Murphy /* start_pfn is always nonzero for an already-initialised domain */ 570ac9a5d52SYunfei Wang mutex_lock(&cookie->mutex); 5710db2e5d1SRobin Murphy if (iovad->start_pfn) { 5720db2e5d1SRobin Murphy if (1UL << order != iovad->granule || 573f51d7bb7SRobin Murphy base_pfn != iovad->start_pfn) { 5740db2e5d1SRobin Murphy pr_warn("Incompatible range for DMA domain\n"); 575ac9a5d52SYunfei Wang ret = -EFAULT; 576ac9a5d52SYunfei Wang goto done_unlock; 5770db2e5d1SRobin Murphy } 5787c1b058cSRobin Murphy 579ac9a5d52SYunfei Wang ret = 0; 580ac9a5d52SYunfei Wang goto done_unlock; 5810db2e5d1SRobin Murphy } 5827c1b058cSRobin Murphy 583aa3ac946SZhen Lei init_iova_domain(iovad, 1UL << order, base_pfn); 58432e92d9fSJohn Garry ret = iova_domain_init_rcaches(iovad); 58532e92d9fSJohn Garry if (ret) 586ac9a5d52SYunfei Wang goto done_unlock; 5872da274cdSZhen Lei 588c208916fSRobin Murphy /* If the FQ fails we can simply fall back to strict mode */ 589452e69b5SRobin Murphy if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain)) 590c208916fSRobin Murphy domain->type = IOMMU_DOMAIN_DMA; 5917c1b058cSRobin Murphy 592ac9a5d52SYunfei Wang ret = iova_reserve_iommu_regions(dev, domain); 593ac9a5d52SYunfei Wang 594ac9a5d52SYunfei Wang done_unlock: 595ac9a5d52SYunfei Wang mutex_unlock(&cookie->mutex); 596ac9a5d52SYunfei Wang return ret; 5977c1b058cSRobin Murphy } 5980db2e5d1SRobin Murphy 5990db2e5d1SRobin Murphy /** 600737c85caSMitchel Humpherys * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API 601737c85caSMitchel Humpherys * page flags. 6020db2e5d1SRobin Murphy * @dir: Direction of DMA transfer 6030db2e5d1SRobin Murphy * @coherent: Is the DMA master cache-coherent? 604737c85caSMitchel Humpherys * @attrs: DMA attributes for the mapping 6050db2e5d1SRobin Murphy * 6060db2e5d1SRobin Murphy * Return: corresponding IOMMU API page protection flags 6070db2e5d1SRobin Murphy */ 60806d60728SChristoph Hellwig static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, 609737c85caSMitchel Humpherys unsigned long attrs) 6100db2e5d1SRobin Murphy { 6110db2e5d1SRobin Murphy int prot = coherent ? IOMMU_CACHE : 0; 6120db2e5d1SRobin Murphy 613737c85caSMitchel Humpherys if (attrs & DMA_ATTR_PRIVILEGED) 614737c85caSMitchel Humpherys prot |= IOMMU_PRIV; 615737c85caSMitchel Humpherys 6160db2e5d1SRobin Murphy switch (dir) { 6170db2e5d1SRobin Murphy case DMA_BIDIRECTIONAL: 6180db2e5d1SRobin Murphy return prot | IOMMU_READ | IOMMU_WRITE; 6190db2e5d1SRobin Murphy case DMA_TO_DEVICE: 6200db2e5d1SRobin Murphy return prot | IOMMU_READ; 6210db2e5d1SRobin Murphy case DMA_FROM_DEVICE: 6220db2e5d1SRobin Murphy return prot | IOMMU_WRITE; 6230db2e5d1SRobin Murphy default: 6240db2e5d1SRobin Murphy return 0; 6250db2e5d1SRobin Murphy } 6260db2e5d1SRobin Murphy } 6270db2e5d1SRobin Murphy 628842fe519SRobin Murphy static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, 629bd036d2fSRobin Murphy size_t size, u64 dma_limit, struct device *dev) 6300db2e5d1SRobin Murphy { 631a44e6657SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 632a44e6657SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 633bb65a64cSRobin Murphy unsigned long shift, iova_len, iova = 0; 6340db2e5d1SRobin Murphy 635a44e6657SRobin Murphy if (cookie->type == IOMMU_DMA_MSI_COOKIE) { 636a44e6657SRobin Murphy cookie->msi_iova += size; 637a44e6657SRobin Murphy return cookie->msi_iova - size; 638a44e6657SRobin Murphy } 639a44e6657SRobin Murphy 640a44e6657SRobin Murphy shift = iova_shift(iovad); 641a44e6657SRobin Murphy iova_len = size >> shift; 642a44e6657SRobin Murphy 643a7ba70f1SNicolas Saenz Julienne dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); 64403bfdc31SRobin Murphy 645c987ff0dSRobin Murphy if (domain->geometry.force_aperture) 646bd036d2fSRobin Murphy dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end); 647122fac03SRobin Murphy 648122fac03SRobin Murphy /* Try to get PCI devices a SAC address */ 6493542dcb1SRobin Murphy if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev)) 650538d5b33STomasz Nowicki iova = alloc_iova_fast(iovad, iova_len, 651538d5b33STomasz Nowicki DMA_BIT_MASK(32) >> shift, false); 652122fac03SRobin Murphy 653bb65a64cSRobin Murphy if (!iova) 654538d5b33STomasz Nowicki iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, 655538d5b33STomasz Nowicki true); 656bb65a64cSRobin Murphy 657bb65a64cSRobin Murphy return (dma_addr_t)iova << shift; 6580db2e5d1SRobin Murphy } 6590db2e5d1SRobin Murphy 660842fe519SRobin Murphy static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, 661452e69b5SRobin Murphy dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather) 6620db2e5d1SRobin Murphy { 663842fe519SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 6640db2e5d1SRobin Murphy 665a44e6657SRobin Murphy /* The MSI case is only ever cleaning up its most recent allocation */ 666bb65a64cSRobin Murphy if (cookie->type == IOMMU_DMA_MSI_COOKIE) 667a44e6657SRobin Murphy cookie->msi_iova -= size; 668452e69b5SRobin Murphy else if (gather && gather->queued) 669a17e3026SRobin Murphy queue_iova(cookie, iova_pfn(iovad, iova), 6702a2b8eaaSTom Murphy size >> iova_shift(iovad), 67187f60cc6SMatthew Wilcox (Oracle) &gather->freelist); 672bb65a64cSRobin Murphy else 6731cc896edSRobin Murphy free_iova_fast(iovad, iova_pfn(iovad, iova), 6741cc896edSRobin Murphy size >> iova_shift(iovad)); 675842fe519SRobin Murphy } 676842fe519SRobin Murphy 677b61d271eSRobin Murphy static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, 678842fe519SRobin Murphy size_t size) 679842fe519SRobin Murphy { 680b61d271eSRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 681a44e6657SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 682a44e6657SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 683842fe519SRobin Murphy size_t iova_off = iova_offset(iovad, dma_addr); 684a7d20dc1SWill Deacon struct iommu_iotlb_gather iotlb_gather; 685a7d20dc1SWill Deacon size_t unmapped; 686842fe519SRobin Murphy 687842fe519SRobin Murphy dma_addr -= iova_off; 688842fe519SRobin Murphy size = iova_align(iovad, size + iova_off); 689a7d20dc1SWill Deacon iommu_iotlb_gather_init(&iotlb_gather); 690452e69b5SRobin Murphy iotlb_gather.queued = READ_ONCE(cookie->fq_domain); 691842fe519SRobin Murphy 692a7d20dc1SWill Deacon unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather); 693a7d20dc1SWill Deacon WARN_ON(unmapped != size); 694a7d20dc1SWill Deacon 695452e69b5SRobin Murphy if (!iotlb_gather.queued) 696aae4c8e2STom Murphy iommu_iotlb_sync(domain, &iotlb_gather); 697452e69b5SRobin Murphy iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather); 6980db2e5d1SRobin Murphy } 6990db2e5d1SRobin Murphy 70092aec09cSChristoph Hellwig static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, 701bd036d2fSRobin Murphy size_t size, int prot, u64 dma_mask) 70292aec09cSChristoph Hellwig { 703b61d271eSRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 70492aec09cSChristoph Hellwig struct iommu_dma_cookie *cookie = domain->iova_cookie; 7058af23fadSRobin Murphy struct iova_domain *iovad = &cookie->iovad; 7068af23fadSRobin Murphy size_t iova_off = iova_offset(iovad, phys); 70792aec09cSChristoph Hellwig dma_addr_t iova; 70892aec09cSChristoph Hellwig 709a8e8af35SLianbo Jiang if (static_branch_unlikely(&iommu_deferred_attach_enabled) && 7103ab65729SLianbo Jiang iommu_deferred_attach(dev, domain)) 711795bbbb9STom Murphy return DMA_MAPPING_ERROR; 712795bbbb9STom Murphy 7138af23fadSRobin Murphy size = iova_align(iovad, size + iova_off); 71492aec09cSChristoph Hellwig 7156e235020STom Murphy iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev); 71692aec09cSChristoph Hellwig if (!iova) 71792aec09cSChristoph Hellwig return DMA_MAPPING_ERROR; 71892aec09cSChristoph Hellwig 7194dc6376aSJason Gunthorpe if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) { 7202a2b8eaaSTom Murphy iommu_dma_free_iova(cookie, iova, size, NULL); 72192aec09cSChristoph Hellwig return DMA_MAPPING_ERROR; 72292aec09cSChristoph Hellwig } 72392aec09cSChristoph Hellwig return iova + iova_off; 72492aec09cSChristoph Hellwig } 72592aec09cSChristoph Hellwig 7260db2e5d1SRobin Murphy static void __iommu_dma_free_pages(struct page **pages, int count) 7270db2e5d1SRobin Murphy { 7280db2e5d1SRobin Murphy while (count--) 7290db2e5d1SRobin Murphy __free_page(pages[count]); 7300db2e5d1SRobin Murphy kvfree(pages); 7310db2e5d1SRobin Murphy } 7320db2e5d1SRobin Murphy 733c4b17afbSGanapatrao Kulkarni static struct page **__iommu_dma_alloc_pages(struct device *dev, 734c4b17afbSGanapatrao Kulkarni unsigned int count, unsigned long order_mask, gfp_t gfp) 7350db2e5d1SRobin Murphy { 7360db2e5d1SRobin Murphy struct page **pages; 737c4b17afbSGanapatrao Kulkarni unsigned int i = 0, nid = dev_to_node(dev); 7383b6b7e19SRobin Murphy 73923baf831SKirill A. Shutemov order_mask &= GENMASK(MAX_ORDER, 0); 7403b6b7e19SRobin Murphy if (!order_mask) 7413b6b7e19SRobin Murphy return NULL; 7420db2e5d1SRobin Murphy 743ab6f4b00SGustavo A. R. Silva pages = kvcalloc(count, sizeof(*pages), GFP_KERNEL); 7440db2e5d1SRobin Murphy if (!pages) 7450db2e5d1SRobin Murphy return NULL; 7460db2e5d1SRobin Murphy 7470db2e5d1SRobin Murphy /* IOMMU can map any pages, so himem can also be used here */ 7480db2e5d1SRobin Murphy gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 7490db2e5d1SRobin Murphy 7500db2e5d1SRobin Murphy while (count) { 7510db2e5d1SRobin Murphy struct page *page = NULL; 7523b6b7e19SRobin Murphy unsigned int order_size; 7530db2e5d1SRobin Murphy 7540db2e5d1SRobin Murphy /* 7550db2e5d1SRobin Murphy * Higher-order allocations are a convenience rather 7560db2e5d1SRobin Murphy * than a necessity, hence using __GFP_NORETRY until 7573b6b7e19SRobin Murphy * falling back to minimum-order allocations. 7580db2e5d1SRobin Murphy */ 75961883d3cSKirill A. Shutemov for (order_mask &= GENMASK(__fls(count), 0); 7603b6b7e19SRobin Murphy order_mask; order_mask &= ~order_size) { 7613b6b7e19SRobin Murphy unsigned int order = __fls(order_mask); 762c4b17afbSGanapatrao Kulkarni gfp_t alloc_flags = gfp; 7633b6b7e19SRobin Murphy 7643b6b7e19SRobin Murphy order_size = 1U << order; 765c4b17afbSGanapatrao Kulkarni if (order_mask > order_size) 766c4b17afbSGanapatrao Kulkarni alloc_flags |= __GFP_NORETRY; 767c4b17afbSGanapatrao Kulkarni page = alloc_pages_node(nid, alloc_flags, order); 7680db2e5d1SRobin Murphy if (!page) 7690db2e5d1SRobin Murphy continue; 7704604393cSRobin Murphy if (order) 7710db2e5d1SRobin Murphy split_page(page, order); 7720db2e5d1SRobin Murphy break; 7730db2e5d1SRobin Murphy } 7740db2e5d1SRobin Murphy if (!page) { 7750db2e5d1SRobin Murphy __iommu_dma_free_pages(pages, i); 7760db2e5d1SRobin Murphy return NULL; 7770db2e5d1SRobin Murphy } 7783b6b7e19SRobin Murphy count -= order_size; 7793b6b7e19SRobin Murphy while (order_size--) 7800db2e5d1SRobin Murphy pages[i++] = page++; 7810db2e5d1SRobin Murphy } 7820db2e5d1SRobin Murphy return pages; 7830db2e5d1SRobin Murphy } 7840db2e5d1SRobin Murphy 7858230ce9aSChristoph Hellwig /* 7868230ce9aSChristoph Hellwig * If size is less than PAGE_SIZE, then a full CPU page will be allocated, 7870db2e5d1SRobin Murphy * but an IOMMU which supports smaller pages might not map the whole thing. 7880db2e5d1SRobin Murphy */ 7898230ce9aSChristoph Hellwig static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, 7908230ce9aSChristoph Hellwig size_t size, struct sg_table *sgt, gfp_t gfp, pgprot_t prot, 791e8d39a90SChristoph Hellwig unsigned long attrs) 7920db2e5d1SRobin Murphy { 79343c5bf11SRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 794842fe519SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 795842fe519SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 79621b95aafSChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 79721b95aafSChristoph Hellwig int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); 79821b95aafSChristoph Hellwig unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; 7990db2e5d1SRobin Murphy struct page **pages; 800842fe519SRobin Murphy dma_addr_t iova; 801a3884774SYunfei Wang ssize_t ret; 8020db2e5d1SRobin Murphy 803a8e8af35SLianbo Jiang if (static_branch_unlikely(&iommu_deferred_attach_enabled) && 8043ab65729SLianbo Jiang iommu_deferred_attach(dev, domain)) 805795bbbb9STom Murphy return NULL; 806795bbbb9STom Murphy 8073b6b7e19SRobin Murphy min_size = alloc_sizes & -alloc_sizes; 8083b6b7e19SRobin Murphy if (min_size < PAGE_SIZE) { 8093b6b7e19SRobin Murphy min_size = PAGE_SIZE; 8103b6b7e19SRobin Murphy alloc_sizes |= PAGE_SIZE; 8113b6b7e19SRobin Murphy } else { 8123b6b7e19SRobin Murphy size = ALIGN(size, min_size); 8133b6b7e19SRobin Murphy } 81400085f1eSKrzysztof Kozlowski if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) 8153b6b7e19SRobin Murphy alloc_sizes = min_size; 8163b6b7e19SRobin Murphy 8173b6b7e19SRobin Murphy count = PAGE_ALIGN(size) >> PAGE_SHIFT; 818c4b17afbSGanapatrao Kulkarni pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT, 819c4b17afbSGanapatrao Kulkarni gfp); 8200db2e5d1SRobin Murphy if (!pages) 8210db2e5d1SRobin Murphy return NULL; 8220db2e5d1SRobin Murphy 823842fe519SRobin Murphy size = iova_align(iovad, size); 824842fe519SRobin Murphy iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev); 8250db2e5d1SRobin Murphy if (!iova) 8260db2e5d1SRobin Murphy goto out_free_pages; 8270db2e5d1SRobin Murphy 82896d57808SJason Gunthorpe /* 82996d57808SJason Gunthorpe * Remove the zone/policy flags from the GFP - these are applied to the 83096d57808SJason Gunthorpe * __iommu_dma_alloc_pages() but are not used for the supporting 83196d57808SJason Gunthorpe * internal allocations that follow. 83296d57808SJason Gunthorpe */ 83396d57808SJason Gunthorpe gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); 83496d57808SJason Gunthorpe 83596d57808SJason Gunthorpe if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, gfp)) 8360db2e5d1SRobin Murphy goto out_free_iova; 8370db2e5d1SRobin Murphy 83821b95aafSChristoph Hellwig if (!(ioprot & IOMMU_CACHE)) { 83923f88e0aSChristoph Hellwig struct scatterlist *sg; 84023f88e0aSChristoph Hellwig int i; 84123f88e0aSChristoph Hellwig 8428230ce9aSChristoph Hellwig for_each_sg(sgt->sgl, sg, sgt->orig_nents, i) 84323f88e0aSChristoph Hellwig arch_dma_prep_coherent(sg_page(sg), sg->length); 8440db2e5d1SRobin Murphy } 8450db2e5d1SRobin Murphy 846f2b2c051SJason Gunthorpe ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot, 84796d57808SJason Gunthorpe gfp); 848a3884774SYunfei Wang if (ret < 0 || ret < size) 8490db2e5d1SRobin Murphy goto out_free_sg; 8500db2e5d1SRobin Murphy 8518230ce9aSChristoph Hellwig sgt->sgl->dma_address = iova; 852e817ee5fSChristoph Hellwig sgt->sgl->dma_length = size; 8538230ce9aSChristoph Hellwig return pages; 8540db2e5d1SRobin Murphy 8550db2e5d1SRobin Murphy out_free_sg: 8568230ce9aSChristoph Hellwig sg_free_table(sgt); 8570db2e5d1SRobin Murphy out_free_iova: 8582a2b8eaaSTom Murphy iommu_dma_free_iova(cookie, iova, size, NULL); 8590db2e5d1SRobin Murphy out_free_pages: 8600db2e5d1SRobin Murphy __iommu_dma_free_pages(pages, count); 8610db2e5d1SRobin Murphy return NULL; 8620db2e5d1SRobin Murphy } 8630db2e5d1SRobin Murphy 8648230ce9aSChristoph Hellwig static void *iommu_dma_alloc_remap(struct device *dev, size_t size, 8658230ce9aSChristoph Hellwig dma_addr_t *dma_handle, gfp_t gfp, pgprot_t prot, 8668230ce9aSChristoph Hellwig unsigned long attrs) 8678230ce9aSChristoph Hellwig { 8688230ce9aSChristoph Hellwig struct page **pages; 8698230ce9aSChristoph Hellwig struct sg_table sgt; 8708230ce9aSChristoph Hellwig void *vaddr; 8718230ce9aSChristoph Hellwig 8728230ce9aSChristoph Hellwig pages = __iommu_dma_alloc_noncontiguous(dev, size, &sgt, gfp, prot, 8738230ce9aSChristoph Hellwig attrs); 8748230ce9aSChristoph Hellwig if (!pages) 8758230ce9aSChristoph Hellwig return NULL; 8768230ce9aSChristoph Hellwig *dma_handle = sgt.sgl->dma_address; 8778230ce9aSChristoph Hellwig sg_free_table(&sgt); 8788230ce9aSChristoph Hellwig vaddr = dma_common_pages_remap(pages, size, prot, 8798230ce9aSChristoph Hellwig __builtin_return_address(0)); 8808230ce9aSChristoph Hellwig if (!vaddr) 8818230ce9aSChristoph Hellwig goto out_unmap; 8828230ce9aSChristoph Hellwig return vaddr; 8838230ce9aSChristoph Hellwig 8848230ce9aSChristoph Hellwig out_unmap: 8858230ce9aSChristoph Hellwig __iommu_dma_unmap(dev, *dma_handle, size); 8868230ce9aSChristoph Hellwig __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); 8878230ce9aSChristoph Hellwig return NULL; 8888230ce9aSChristoph Hellwig } 8898230ce9aSChristoph Hellwig 890e817ee5fSChristoph Hellwig static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, 891e817ee5fSChristoph Hellwig size_t size, enum dma_data_direction dir, gfp_t gfp, 892e817ee5fSChristoph Hellwig unsigned long attrs) 893e817ee5fSChristoph Hellwig { 894e817ee5fSChristoph Hellwig struct dma_sgt_handle *sh; 895e817ee5fSChristoph Hellwig 896e817ee5fSChristoph Hellwig sh = kmalloc(sizeof(*sh), gfp); 897e817ee5fSChristoph Hellwig if (!sh) 898e817ee5fSChristoph Hellwig return NULL; 899e817ee5fSChristoph Hellwig 900e817ee5fSChristoph Hellwig sh->pages = __iommu_dma_alloc_noncontiguous(dev, size, &sh->sgt, gfp, 901e817ee5fSChristoph Hellwig PAGE_KERNEL, attrs); 902e817ee5fSChristoph Hellwig if (!sh->pages) { 903e817ee5fSChristoph Hellwig kfree(sh); 904e817ee5fSChristoph Hellwig return NULL; 905e817ee5fSChristoph Hellwig } 906e817ee5fSChristoph Hellwig return &sh->sgt; 907e817ee5fSChristoph Hellwig } 908e817ee5fSChristoph Hellwig 909e817ee5fSChristoph Hellwig static void iommu_dma_free_noncontiguous(struct device *dev, size_t size, 910e817ee5fSChristoph Hellwig struct sg_table *sgt, enum dma_data_direction dir) 911e817ee5fSChristoph Hellwig { 912e817ee5fSChristoph Hellwig struct dma_sgt_handle *sh = sgt_handle(sgt); 913e817ee5fSChristoph Hellwig 914e817ee5fSChristoph Hellwig __iommu_dma_unmap(dev, sgt->sgl->dma_address, size); 915e817ee5fSChristoph Hellwig __iommu_dma_free_pages(sh->pages, PAGE_ALIGN(size) >> PAGE_SHIFT); 916e817ee5fSChristoph Hellwig sg_free_table(&sh->sgt); 9170fbea680SEzequiel Garcia kfree(sh); 918e817ee5fSChristoph Hellwig } 919e817ee5fSChristoph Hellwig 92006d60728SChristoph Hellwig static void iommu_dma_sync_single_for_cpu(struct device *dev, 92106d60728SChristoph Hellwig dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) 9220db2e5d1SRobin Murphy { 92306d60728SChristoph Hellwig phys_addr_t phys; 9240db2e5d1SRobin Murphy 9252e727bffSDavid Stevens if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev)) 92606d60728SChristoph Hellwig return; 92706d60728SChristoph Hellwig 92806d60728SChristoph Hellwig phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); 92982612d66STom Murphy if (!dev_is_dma_coherent(dev)) 93056e35f9cSChristoph Hellwig arch_sync_dma_for_cpu(phys, size, dir); 93182612d66STom Murphy 9327fd856aaSClaire Chang if (is_swiotlb_buffer(dev, phys)) 93380808d27SChristoph Hellwig swiotlb_sync_single_for_cpu(dev, phys, size, dir); 9341cc896edSRobin Murphy } 9351cc896edSRobin Murphy 93606d60728SChristoph Hellwig static void iommu_dma_sync_single_for_device(struct device *dev, 93706d60728SChristoph Hellwig dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) 93851f8cc9eSRobin Murphy { 93906d60728SChristoph Hellwig phys_addr_t phys; 94006d60728SChristoph Hellwig 9412e727bffSDavid Stevens if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev)) 94206d60728SChristoph Hellwig return; 94306d60728SChristoph Hellwig 94406d60728SChristoph Hellwig phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); 9457fd856aaSClaire Chang if (is_swiotlb_buffer(dev, phys)) 94680808d27SChristoph Hellwig swiotlb_sync_single_for_device(dev, phys, size, dir); 94782612d66STom Murphy 94882612d66STom Murphy if (!dev_is_dma_coherent(dev)) 94956e35f9cSChristoph Hellwig arch_sync_dma_for_device(phys, size, dir); 95051f8cc9eSRobin Murphy } 95151f8cc9eSRobin Murphy 95206d60728SChristoph Hellwig static void iommu_dma_sync_sg_for_cpu(struct device *dev, 95306d60728SChristoph Hellwig struct scatterlist *sgl, int nelems, 95406d60728SChristoph Hellwig enum dma_data_direction dir) 9550db2e5d1SRobin Murphy { 95606d60728SChristoph Hellwig struct scatterlist *sg; 95706d60728SChristoph Hellwig int i; 95806d60728SChristoph Hellwig 9592e727bffSDavid Stevens if (dev_use_swiotlb(dev)) 96008ae5d4aSDavid Stevens for_each_sg(sgl, sg, nelems, i) 96108ae5d4aSDavid Stevens iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg), 96280808d27SChristoph Hellwig sg->length, dir); 96308ae5d4aSDavid Stevens else if (!dev_is_dma_coherent(dev)) 96408ae5d4aSDavid Stevens for_each_sg(sgl, sg, nelems, i) 96508ae5d4aSDavid Stevens arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); 96606d60728SChristoph Hellwig } 96706d60728SChristoph Hellwig 96806d60728SChristoph Hellwig static void iommu_dma_sync_sg_for_device(struct device *dev, 96906d60728SChristoph Hellwig struct scatterlist *sgl, int nelems, 97006d60728SChristoph Hellwig enum dma_data_direction dir) 97106d60728SChristoph Hellwig { 97206d60728SChristoph Hellwig struct scatterlist *sg; 97306d60728SChristoph Hellwig int i; 97406d60728SChristoph Hellwig 9752e727bffSDavid Stevens if (dev_use_swiotlb(dev)) 97608ae5d4aSDavid Stevens for_each_sg(sgl, sg, nelems, i) 97708ae5d4aSDavid Stevens iommu_dma_sync_single_for_device(dev, 97808ae5d4aSDavid Stevens sg_dma_address(sg), 97980808d27SChristoph Hellwig sg->length, dir); 98008ae5d4aSDavid Stevens else if (!dev_is_dma_coherent(dev)) 98108ae5d4aSDavid Stevens for_each_sg(sgl, sg, nelems, i) 98256e35f9cSChristoph Hellwig arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); 98306d60728SChristoph Hellwig } 98406d60728SChristoph Hellwig 98506d60728SChristoph Hellwig static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 98606d60728SChristoph Hellwig unsigned long offset, size_t size, enum dma_data_direction dir, 98706d60728SChristoph Hellwig unsigned long attrs) 98806d60728SChristoph Hellwig { 98906d60728SChristoph Hellwig phys_addr_t phys = page_to_phys(page) + offset; 99006d60728SChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 9919b49bbc2SDavid Stevens int prot = dma_info_to_prot(dir, coherent, attrs); 9929b49bbc2SDavid Stevens struct iommu_domain *domain = iommu_get_dma_domain(dev); 9939b49bbc2SDavid Stevens struct iommu_dma_cookie *cookie = domain->iova_cookie; 9949b49bbc2SDavid Stevens struct iova_domain *iovad = &cookie->iovad; 9959b49bbc2SDavid Stevens dma_addr_t iova, dma_mask = dma_get_mask(dev); 99606d60728SChristoph Hellwig 9979b49bbc2SDavid Stevens /* 9989b49bbc2SDavid Stevens * If both the physical buffer start address and size are 9999b49bbc2SDavid Stevens * page aligned, we don't need to use a bounce page. 10009b49bbc2SDavid Stevens */ 10012e727bffSDavid Stevens if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) { 10029b49bbc2SDavid Stevens void *padding_start; 10032cbc61a1SDavid Stevens size_t padding_size, aligned_size; 10049b49bbc2SDavid Stevens 1005f316ba0aSMario Limonciello if (!is_swiotlb_active(dev)) { 1006f316ba0aSMario Limonciello dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n"); 1007f316ba0aSMario Limonciello return DMA_MAPPING_ERROR; 1008f316ba0aSMario Limonciello } 1009f316ba0aSMario Limonciello 10109b49bbc2SDavid Stevens aligned_size = iova_align(iovad, size); 1011e81e99baSDavid Stevens phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size, 1012e81e99baSDavid Stevens iova_mask(iovad), dir, attrs); 10139b49bbc2SDavid Stevens 10149b49bbc2SDavid Stevens if (phys == DMA_MAPPING_ERROR) 10159b49bbc2SDavid Stevens return DMA_MAPPING_ERROR; 10169b49bbc2SDavid Stevens 10179b49bbc2SDavid Stevens /* Cleanup the padding area. */ 10189b49bbc2SDavid Stevens padding_start = phys_to_virt(phys); 10199b49bbc2SDavid Stevens padding_size = aligned_size; 10209b49bbc2SDavid Stevens 10219b49bbc2SDavid Stevens if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && 10229b49bbc2SDavid Stevens (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) { 10239b49bbc2SDavid Stevens padding_start += size; 10249b49bbc2SDavid Stevens padding_size -= size; 10259b49bbc2SDavid Stevens } 10269b49bbc2SDavid Stevens 10279b49bbc2SDavid Stevens memset(padding_start, 0, padding_size); 10289b49bbc2SDavid Stevens } 10299b49bbc2SDavid Stevens 10309b49bbc2SDavid Stevens if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 103156e35f9cSChristoph Hellwig arch_sync_dma_for_device(phys, size, dir); 10329b49bbc2SDavid Stevens 10332cbc61a1SDavid Stevens iova = __iommu_dma_map(dev, phys, size, prot, dma_mask); 10349b49bbc2SDavid Stevens if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys)) 10359b49bbc2SDavid Stevens swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); 10369b49bbc2SDavid Stevens return iova; 103706d60728SChristoph Hellwig } 103806d60728SChristoph Hellwig 103906d60728SChristoph Hellwig static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 104006d60728SChristoph Hellwig size_t size, enum dma_data_direction dir, unsigned long attrs) 104106d60728SChristoph Hellwig { 10429b49bbc2SDavid Stevens struct iommu_domain *domain = iommu_get_dma_domain(dev); 10439b49bbc2SDavid Stevens phys_addr_t phys; 10449b49bbc2SDavid Stevens 10459b49bbc2SDavid Stevens phys = iommu_iova_to_phys(domain, dma_handle); 10469b49bbc2SDavid Stevens if (WARN_ON(!phys)) 10479b49bbc2SDavid Stevens return; 10489b49bbc2SDavid Stevens 10499b49bbc2SDavid Stevens if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev)) 10509b49bbc2SDavid Stevens arch_sync_dma_for_cpu(phys, size, dir); 10519b49bbc2SDavid Stevens 10529b49bbc2SDavid Stevens __iommu_dma_unmap(dev, dma_handle, size); 10539b49bbc2SDavid Stevens 10549b49bbc2SDavid Stevens if (unlikely(is_swiotlb_buffer(dev, phys))) 10559b49bbc2SDavid Stevens swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); 10560db2e5d1SRobin Murphy } 10570db2e5d1SRobin Murphy 10580db2e5d1SRobin Murphy /* 10590db2e5d1SRobin Murphy * Prepare a successfully-mapped scatterlist to give back to the caller. 1060809eac54SRobin Murphy * 1061809eac54SRobin Murphy * At this point the segments are already laid out by iommu_dma_map_sg() to 1062809eac54SRobin Murphy * avoid individually crossing any boundaries, so we merely need to check a 1063809eac54SRobin Murphy * segment's start address to avoid concatenating across one. 10640db2e5d1SRobin Murphy */ 10650db2e5d1SRobin Murphy static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, 10660db2e5d1SRobin Murphy dma_addr_t dma_addr) 10670db2e5d1SRobin Murphy { 1068809eac54SRobin Murphy struct scatterlist *s, *cur = sg; 1069809eac54SRobin Murphy unsigned long seg_mask = dma_get_seg_boundary(dev); 1070809eac54SRobin Murphy unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); 1071809eac54SRobin Murphy int i, count = 0; 10720db2e5d1SRobin Murphy 10730db2e5d1SRobin Murphy for_each_sg(sg, s, nents, i) { 1074809eac54SRobin Murphy /* Restore this segment's original unaligned fields first */ 107530280eeeSLogan Gunthorpe dma_addr_t s_dma_addr = sg_dma_address(s); 1076809eac54SRobin Murphy unsigned int s_iova_off = sg_dma_address(s); 10770db2e5d1SRobin Murphy unsigned int s_length = sg_dma_len(s); 1078809eac54SRobin Murphy unsigned int s_iova_len = s->length; 10790db2e5d1SRobin Murphy 1080cad34be7SChristoph Hellwig sg_dma_address(s) = DMA_MAPPING_ERROR; 1081809eac54SRobin Murphy sg_dma_len(s) = 0; 1082809eac54SRobin Murphy 1083*cb147bbeSRobin Murphy if (sg_dma_is_bus_address(s)) { 108430280eeeSLogan Gunthorpe if (i > 0) 108530280eeeSLogan Gunthorpe cur = sg_next(cur); 108630280eeeSLogan Gunthorpe 108730280eeeSLogan Gunthorpe sg_dma_unmark_bus_address(s); 108830280eeeSLogan Gunthorpe sg_dma_address(cur) = s_dma_addr; 108930280eeeSLogan Gunthorpe sg_dma_len(cur) = s_length; 109030280eeeSLogan Gunthorpe sg_dma_mark_bus_address(cur); 109130280eeeSLogan Gunthorpe count++; 109230280eeeSLogan Gunthorpe cur_len = 0; 109330280eeeSLogan Gunthorpe continue; 109430280eeeSLogan Gunthorpe } 109530280eeeSLogan Gunthorpe 109630280eeeSLogan Gunthorpe s->offset += s_iova_off; 109730280eeeSLogan Gunthorpe s->length = s_length; 109830280eeeSLogan Gunthorpe 1099809eac54SRobin Murphy /* 1100809eac54SRobin Murphy * Now fill in the real DMA data. If... 1101809eac54SRobin Murphy * - there is a valid output segment to append to 1102809eac54SRobin Murphy * - and this segment starts on an IOVA page boundary 1103809eac54SRobin Murphy * - but doesn't fall at a segment boundary 1104809eac54SRobin Murphy * - and wouldn't make the resulting output segment too long 1105809eac54SRobin Murphy */ 1106809eac54SRobin Murphy if (cur_len && !s_iova_off && (dma_addr & seg_mask) && 1107ab2cbeb0SRobin Murphy (max_len - cur_len >= s_length)) { 1108809eac54SRobin Murphy /* ...then concatenate it with the previous one */ 1109809eac54SRobin Murphy cur_len += s_length; 1110809eac54SRobin Murphy } else { 1111809eac54SRobin Murphy /* Otherwise start the next output segment */ 1112809eac54SRobin Murphy if (i > 0) 1113809eac54SRobin Murphy cur = sg_next(cur); 1114809eac54SRobin Murphy cur_len = s_length; 1115809eac54SRobin Murphy count++; 1116809eac54SRobin Murphy 1117809eac54SRobin Murphy sg_dma_address(cur) = dma_addr + s_iova_off; 11180db2e5d1SRobin Murphy } 1119809eac54SRobin Murphy 1120809eac54SRobin Murphy sg_dma_len(cur) = cur_len; 1121809eac54SRobin Murphy dma_addr += s_iova_len; 1122809eac54SRobin Murphy 1123809eac54SRobin Murphy if (s_length + s_iova_off < s_iova_len) 1124809eac54SRobin Murphy cur_len = 0; 1125809eac54SRobin Murphy } 1126809eac54SRobin Murphy return count; 11270db2e5d1SRobin Murphy } 11280db2e5d1SRobin Murphy 11290db2e5d1SRobin Murphy /* 11300db2e5d1SRobin Murphy * If mapping failed, then just restore the original list, 11310db2e5d1SRobin Murphy * but making sure the DMA fields are invalidated. 11320db2e5d1SRobin Murphy */ 11330db2e5d1SRobin Murphy static void __invalidate_sg(struct scatterlist *sg, int nents) 11340db2e5d1SRobin Murphy { 11350db2e5d1SRobin Murphy struct scatterlist *s; 11360db2e5d1SRobin Murphy int i; 11370db2e5d1SRobin Murphy 11380db2e5d1SRobin Murphy for_each_sg(sg, s, nents, i) { 1139*cb147bbeSRobin Murphy if (sg_dma_is_bus_address(s)) { 114030280eeeSLogan Gunthorpe sg_dma_unmark_bus_address(s); 114130280eeeSLogan Gunthorpe } else { 1142cad34be7SChristoph Hellwig if (sg_dma_address(s) != DMA_MAPPING_ERROR) 114307b48ac4SRobin Murphy s->offset += sg_dma_address(s); 11440db2e5d1SRobin Murphy if (sg_dma_len(s)) 11450db2e5d1SRobin Murphy s->length = sg_dma_len(s); 114630280eeeSLogan Gunthorpe } 1147cad34be7SChristoph Hellwig sg_dma_address(s) = DMA_MAPPING_ERROR; 11480db2e5d1SRobin Murphy sg_dma_len(s) = 0; 11490db2e5d1SRobin Murphy } 11500db2e5d1SRobin Murphy } 11510db2e5d1SRobin Murphy 115282612d66STom Murphy static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *sg, 115382612d66STom Murphy int nents, enum dma_data_direction dir, unsigned long attrs) 115482612d66STom Murphy { 115582612d66STom Murphy struct scatterlist *s; 115682612d66STom Murphy int i; 115782612d66STom Murphy 115882612d66STom Murphy for_each_sg(sg, s, nents, i) 11599b49bbc2SDavid Stevens iommu_dma_unmap_page(dev, sg_dma_address(s), 116082612d66STom Murphy sg_dma_len(s), dir, attrs); 116182612d66STom Murphy } 116282612d66STom Murphy 116382612d66STom Murphy static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg, 116482612d66STom Murphy int nents, enum dma_data_direction dir, unsigned long attrs) 116582612d66STom Murphy { 116682612d66STom Murphy struct scatterlist *s; 116782612d66STom Murphy int i; 116882612d66STom Murphy 116982612d66STom Murphy for_each_sg(sg, s, nents, i) { 11709b49bbc2SDavid Stevens sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s), 11719b49bbc2SDavid Stevens s->offset, s->length, dir, attrs); 117282612d66STom Murphy if (sg_dma_address(s) == DMA_MAPPING_ERROR) 117382612d66STom Murphy goto out_unmap; 117482612d66STom Murphy sg_dma_len(s) = s->length; 117582612d66STom Murphy } 117682612d66STom Murphy 117782612d66STom Murphy return nents; 117882612d66STom Murphy 117982612d66STom Murphy out_unmap: 118082612d66STom Murphy iommu_dma_unmap_sg_swiotlb(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); 1181dabb16f6SLogan Gunthorpe return -EIO; 118282612d66STom Murphy } 118382612d66STom Murphy 11840db2e5d1SRobin Murphy /* 11850db2e5d1SRobin Murphy * The DMA API client is passing in a scatterlist which could describe 11860db2e5d1SRobin Murphy * any old buffer layout, but the IOMMU API requires everything to be 11870db2e5d1SRobin Murphy * aligned to IOMMU pages. Hence the need for this complicated bit of 11880db2e5d1SRobin Murphy * impedance-matching, to be able to hand off a suitably-aligned list, 11890db2e5d1SRobin Murphy * but still preserve the original offsets and sizes for the caller. 11900db2e5d1SRobin Murphy */ 119106d60728SChristoph Hellwig static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, 119206d60728SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 11930db2e5d1SRobin Murphy { 119443c5bf11SRobin Murphy struct iommu_domain *domain = iommu_get_dma_domain(dev); 1195842fe519SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 1196842fe519SRobin Murphy struct iova_domain *iovad = &cookie->iovad; 11970db2e5d1SRobin Murphy struct scatterlist *s, *prev = NULL; 119806d60728SChristoph Hellwig int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); 119930280eeeSLogan Gunthorpe struct pci_p2pdma_map_state p2pdma_state = {}; 120030280eeeSLogan Gunthorpe enum pci_p2pdma_map_type map; 1201842fe519SRobin Murphy dma_addr_t iova; 12020db2e5d1SRobin Murphy size_t iova_len = 0; 1203809eac54SRobin Murphy unsigned long mask = dma_get_seg_boundary(dev); 1204dabb16f6SLogan Gunthorpe ssize_t ret; 12050db2e5d1SRobin Murphy int i; 12060db2e5d1SRobin Murphy 1207dabb16f6SLogan Gunthorpe if (static_branch_unlikely(&iommu_deferred_attach_enabled)) { 1208dabb16f6SLogan Gunthorpe ret = iommu_deferred_attach(dev, domain); 1209ac315f96SLogan Gunthorpe if (ret) 1210dabb16f6SLogan Gunthorpe goto out; 1211dabb16f6SLogan Gunthorpe } 1212795bbbb9STom Murphy 12132e727bffSDavid Stevens if (dev_use_swiotlb(dev)) 121482612d66STom Murphy return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs); 121582612d66STom Murphy 12160db2e5d1SRobin Murphy if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 12170db2e5d1SRobin Murphy iommu_dma_sync_sg_for_device(dev, sg, nents, dir); 12180db2e5d1SRobin Murphy 12190db2e5d1SRobin Murphy /* 12200db2e5d1SRobin Murphy * Work out how much IOVA space we need, and align the segments to 12210db2e5d1SRobin Murphy * IOVA granules for the IOMMU driver to handle. With some clever 12220db2e5d1SRobin Murphy * trickery we can modify the list in-place, but reversibly, by 1223809eac54SRobin Murphy * stashing the unaligned parts in the as-yet-unused DMA fields. 12240db2e5d1SRobin Murphy */ 12250db2e5d1SRobin Murphy for_each_sg(sg, s, nents, i) { 1226809eac54SRobin Murphy size_t s_iova_off = iova_offset(iovad, s->offset); 12270db2e5d1SRobin Murphy size_t s_length = s->length; 1228809eac54SRobin Murphy size_t pad_len = (mask - iova_len + 1) & mask; 12290db2e5d1SRobin Murphy 123030280eeeSLogan Gunthorpe if (is_pci_p2pdma_page(sg_page(s))) { 123130280eeeSLogan Gunthorpe map = pci_p2pdma_map_segment(&p2pdma_state, dev, s); 123230280eeeSLogan Gunthorpe switch (map) { 123330280eeeSLogan Gunthorpe case PCI_P2PDMA_MAP_BUS_ADDR: 123430280eeeSLogan Gunthorpe /* 123530280eeeSLogan Gunthorpe * iommu_map_sg() will skip this segment as 123630280eeeSLogan Gunthorpe * it is marked as a bus address, 123730280eeeSLogan Gunthorpe * __finalise_sg() will copy the dma address 123830280eeeSLogan Gunthorpe * into the output segment. 123930280eeeSLogan Gunthorpe */ 124030280eeeSLogan Gunthorpe continue; 124130280eeeSLogan Gunthorpe case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 124230280eeeSLogan Gunthorpe /* 124330280eeeSLogan Gunthorpe * Mapping through host bridge should be 124430280eeeSLogan Gunthorpe * mapped with regular IOVAs, thus we 124530280eeeSLogan Gunthorpe * do nothing here and continue below. 124630280eeeSLogan Gunthorpe */ 124730280eeeSLogan Gunthorpe break; 124830280eeeSLogan Gunthorpe default: 124930280eeeSLogan Gunthorpe ret = -EREMOTEIO; 125030280eeeSLogan Gunthorpe goto out_restore_sg; 125130280eeeSLogan Gunthorpe } 125230280eeeSLogan Gunthorpe } 125330280eeeSLogan Gunthorpe 1254809eac54SRobin Murphy sg_dma_address(s) = s_iova_off; 12550db2e5d1SRobin Murphy sg_dma_len(s) = s_length; 1256809eac54SRobin Murphy s->offset -= s_iova_off; 1257809eac54SRobin Murphy s_length = iova_align(iovad, s_length + s_iova_off); 12580db2e5d1SRobin Murphy s->length = s_length; 12590db2e5d1SRobin Murphy 12600db2e5d1SRobin Murphy /* 1261809eac54SRobin Murphy * Due to the alignment of our single IOVA allocation, we can 1262809eac54SRobin Murphy * depend on these assumptions about the segment boundary mask: 1263809eac54SRobin Murphy * - If mask size >= IOVA size, then the IOVA range cannot 1264809eac54SRobin Murphy * possibly fall across a boundary, so we don't care. 1265809eac54SRobin Murphy * - If mask size < IOVA size, then the IOVA range must start 1266809eac54SRobin Murphy * exactly on a boundary, therefore we can lay things out 1267809eac54SRobin Murphy * based purely on segment lengths without needing to know 1268809eac54SRobin Murphy * the actual addresses beforehand. 1269809eac54SRobin Murphy * - The mask must be a power of 2, so pad_len == 0 if 1270809eac54SRobin Murphy * iova_len == 0, thus we cannot dereference prev the first 1271809eac54SRobin Murphy * time through here (i.e. before it has a meaningful value). 12720db2e5d1SRobin Murphy */ 1273809eac54SRobin Murphy if (pad_len && pad_len < s_length - 1) { 12740db2e5d1SRobin Murphy prev->length += pad_len; 12750db2e5d1SRobin Murphy iova_len += pad_len; 12760db2e5d1SRobin Murphy } 12770db2e5d1SRobin Murphy 12780db2e5d1SRobin Murphy iova_len += s_length; 12790db2e5d1SRobin Murphy prev = s; 12800db2e5d1SRobin Murphy } 12810db2e5d1SRobin Murphy 128230280eeeSLogan Gunthorpe if (!iova_len) 128330280eeeSLogan Gunthorpe return __finalise_sg(dev, sg, nents, 0); 128430280eeeSLogan Gunthorpe 1285842fe519SRobin Murphy iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev); 1286dabb16f6SLogan Gunthorpe if (!iova) { 1287dabb16f6SLogan Gunthorpe ret = -ENOMEM; 12880db2e5d1SRobin Murphy goto out_restore_sg; 1289dabb16f6SLogan Gunthorpe } 12900db2e5d1SRobin Murphy 12910db2e5d1SRobin Murphy /* 12920db2e5d1SRobin Murphy * We'll leave any physical concatenation to the IOMMU driver's 12930db2e5d1SRobin Murphy * implementation - it knows better than we do. 12940db2e5d1SRobin Murphy */ 1295f2b2c051SJason Gunthorpe ret = iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); 1296a3884774SYunfei Wang if (ret < 0 || ret < iova_len) 12970db2e5d1SRobin Murphy goto out_free_iova; 12980db2e5d1SRobin Murphy 1299842fe519SRobin Murphy return __finalise_sg(dev, sg, nents, iova); 13000db2e5d1SRobin Murphy 13010db2e5d1SRobin Murphy out_free_iova: 13022a2b8eaaSTom Murphy iommu_dma_free_iova(cookie, iova, iova_len, NULL); 13030db2e5d1SRobin Murphy out_restore_sg: 13040db2e5d1SRobin Murphy __invalidate_sg(sg, nents); 1305dabb16f6SLogan Gunthorpe out: 130630280eeeSLogan Gunthorpe if (ret != -ENOMEM && ret != -EREMOTEIO) 1307dabb16f6SLogan Gunthorpe return -EINVAL; 1308dabb16f6SLogan Gunthorpe return ret; 13090db2e5d1SRobin Murphy } 13100db2e5d1SRobin Murphy 131106d60728SChristoph Hellwig static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, 131206d60728SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 13130db2e5d1SRobin Murphy { 131430280eeeSLogan Gunthorpe dma_addr_t end = 0, start; 1315842fe519SRobin Murphy struct scatterlist *tmp; 1316842fe519SRobin Murphy int i; 131706d60728SChristoph Hellwig 13182e727bffSDavid Stevens if (dev_use_swiotlb(dev)) { 131982612d66STom Murphy iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs); 132082612d66STom Murphy return; 132182612d66STom Murphy } 132282612d66STom Murphy 1323ee9d4097SDavid Stevens if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1324ee9d4097SDavid Stevens iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); 1325ee9d4097SDavid Stevens 13260db2e5d1SRobin Murphy /* 13270db2e5d1SRobin Murphy * The scatterlist segments are mapped into a single 132830280eeeSLogan Gunthorpe * contiguous IOVA allocation, the start and end points 132930280eeeSLogan Gunthorpe * just have to be determined. 13300db2e5d1SRobin Murphy */ 133130280eeeSLogan Gunthorpe for_each_sg(sg, tmp, nents, i) { 1332*cb147bbeSRobin Murphy if (sg_dma_is_bus_address(tmp)) { 133330280eeeSLogan Gunthorpe sg_dma_unmark_bus_address(tmp); 133430280eeeSLogan Gunthorpe continue; 133530280eeeSLogan Gunthorpe } 133630280eeeSLogan Gunthorpe 1337842fe519SRobin Murphy if (sg_dma_len(tmp) == 0) 1338842fe519SRobin Murphy break; 133930280eeeSLogan Gunthorpe 134030280eeeSLogan Gunthorpe start = sg_dma_address(tmp); 134130280eeeSLogan Gunthorpe break; 1342842fe519SRobin Murphy } 134330280eeeSLogan Gunthorpe 134430280eeeSLogan Gunthorpe nents -= i; 134530280eeeSLogan Gunthorpe for_each_sg(tmp, tmp, nents, i) { 1346*cb147bbeSRobin Murphy if (sg_dma_is_bus_address(tmp)) { 134730280eeeSLogan Gunthorpe sg_dma_unmark_bus_address(tmp); 134830280eeeSLogan Gunthorpe continue; 134930280eeeSLogan Gunthorpe } 135030280eeeSLogan Gunthorpe 135130280eeeSLogan Gunthorpe if (sg_dma_len(tmp) == 0) 135230280eeeSLogan Gunthorpe break; 135330280eeeSLogan Gunthorpe 135430280eeeSLogan Gunthorpe end = sg_dma_address(tmp) + sg_dma_len(tmp); 135530280eeeSLogan Gunthorpe } 135630280eeeSLogan Gunthorpe 135730280eeeSLogan Gunthorpe if (end) 1358b61d271eSRobin Murphy __iommu_dma_unmap(dev, start, end - start); 13590db2e5d1SRobin Murphy } 13600db2e5d1SRobin Murphy 136106d60728SChristoph Hellwig static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, 136251f8cc9eSRobin Murphy size_t size, enum dma_data_direction dir, unsigned long attrs) 136351f8cc9eSRobin Murphy { 136451f8cc9eSRobin Murphy return __iommu_dma_map(dev, phys, size, 13656e235020STom Murphy dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, 13666e235020STom Murphy dma_get_mask(dev)); 136751f8cc9eSRobin Murphy } 136851f8cc9eSRobin Murphy 136906d60728SChristoph Hellwig static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, 137051f8cc9eSRobin Murphy size_t size, enum dma_data_direction dir, unsigned long attrs) 137151f8cc9eSRobin Murphy { 1372b61d271eSRobin Murphy __iommu_dma_unmap(dev, handle, size); 137351f8cc9eSRobin Murphy } 137451f8cc9eSRobin Murphy 13758553f6e6SRobin Murphy static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) 1376bcf4b9c4SRobin Murphy { 1377bcf4b9c4SRobin Murphy size_t alloc_size = PAGE_ALIGN(size); 1378bcf4b9c4SRobin Murphy int count = alloc_size >> PAGE_SHIFT; 1379bcf4b9c4SRobin Murphy struct page *page = NULL, **pages = NULL; 1380bcf4b9c4SRobin Murphy 1381bcf4b9c4SRobin Murphy /* Non-coherent atomic allocation? Easy */ 1382e6475eb0SChristoph Hellwig if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && 1383c84dc6e6SDavid Rientjes dma_free_from_pool(dev, cpu_addr, alloc_size)) 1384bcf4b9c4SRobin Murphy return; 1385bcf4b9c4SRobin Murphy 1386f5ff79fdSChristoph Hellwig if (is_vmalloc_addr(cpu_addr)) { 1387bcf4b9c4SRobin Murphy /* 1388bcf4b9c4SRobin Murphy * If it the address is remapped, then it's either non-coherent 1389bcf4b9c4SRobin Murphy * or highmem CMA, or an iommu_dma_alloc_remap() construction. 1390bcf4b9c4SRobin Murphy */ 13915cf45379SChristoph Hellwig pages = dma_common_find_pages(cpu_addr); 1392bcf4b9c4SRobin Murphy if (!pages) 1393bcf4b9c4SRobin Murphy page = vmalloc_to_page(cpu_addr); 139451231740SChristoph Hellwig dma_common_free_remap(cpu_addr, alloc_size); 1395bcf4b9c4SRobin Murphy } else { 1396bcf4b9c4SRobin Murphy /* Lowmem means a coherent atomic or CMA allocation */ 1397bcf4b9c4SRobin Murphy page = virt_to_page(cpu_addr); 1398bcf4b9c4SRobin Murphy } 1399bcf4b9c4SRobin Murphy 1400bcf4b9c4SRobin Murphy if (pages) 1401bcf4b9c4SRobin Murphy __iommu_dma_free_pages(pages, count); 1402591fcf3bSNicolin Chen if (page) 1403591fcf3bSNicolin Chen dma_free_contiguous(dev, page, alloc_size); 1404bcf4b9c4SRobin Murphy } 1405bcf4b9c4SRobin Murphy 14068553f6e6SRobin Murphy static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, 14078553f6e6SRobin Murphy dma_addr_t handle, unsigned long attrs) 14088553f6e6SRobin Murphy { 14098553f6e6SRobin Murphy __iommu_dma_unmap(dev, handle, size); 14108553f6e6SRobin Murphy __iommu_dma_free(dev, size, cpu_addr); 14118553f6e6SRobin Murphy } 14128553f6e6SRobin Murphy 1413ee1ef05dSChristoph Hellwig static void *iommu_dma_alloc_pages(struct device *dev, size_t size, 1414ee1ef05dSChristoph Hellwig struct page **pagep, gfp_t gfp, unsigned long attrs) 141506d60728SChristoph Hellwig { 141606d60728SChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 14179ad5d6edSRobin Murphy size_t alloc_size = PAGE_ALIGN(size); 141890ae409fSChristoph Hellwig int node = dev_to_node(dev); 14199a4ab94aSChristoph Hellwig struct page *page = NULL; 14209ad5d6edSRobin Murphy void *cpu_addr; 142106d60728SChristoph Hellwig 1422591fcf3bSNicolin Chen page = dma_alloc_contiguous(dev, alloc_size, gfp); 142306d60728SChristoph Hellwig if (!page) 142490ae409fSChristoph Hellwig page = alloc_pages_node(node, gfp, get_order(alloc_size)); 142590ae409fSChristoph Hellwig if (!page) 142606d60728SChristoph Hellwig return NULL; 142706d60728SChristoph Hellwig 1428f5ff79fdSChristoph Hellwig if (!coherent || PageHighMem(page)) { 142933dcb37cSChristoph Hellwig pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); 14308680aa5aSRobin Murphy 14319ad5d6edSRobin Murphy cpu_addr = dma_common_contiguous_remap(page, alloc_size, 143251231740SChristoph Hellwig prot, __builtin_return_address(0)); 14339ad5d6edSRobin Murphy if (!cpu_addr) 1434ee1ef05dSChristoph Hellwig goto out_free_pages; 1435072bebc0SRobin Murphy 143606d60728SChristoph Hellwig if (!coherent) 14379ad5d6edSRobin Murphy arch_dma_prep_coherent(page, size); 14388680aa5aSRobin Murphy } else { 14399ad5d6edSRobin Murphy cpu_addr = page_address(page); 14408680aa5aSRobin Murphy } 1441ee1ef05dSChristoph Hellwig 1442ee1ef05dSChristoph Hellwig *pagep = page; 14439ad5d6edSRobin Murphy memset(cpu_addr, 0, alloc_size); 14449ad5d6edSRobin Murphy return cpu_addr; 1445072bebc0SRobin Murphy out_free_pages: 1446591fcf3bSNicolin Chen dma_free_contiguous(dev, page, alloc_size); 1447072bebc0SRobin Murphy return NULL; 144806d60728SChristoph Hellwig } 144906d60728SChristoph Hellwig 1450ee1ef05dSChristoph Hellwig static void *iommu_dma_alloc(struct device *dev, size_t size, 1451ee1ef05dSChristoph Hellwig dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 1452ee1ef05dSChristoph Hellwig { 1453ee1ef05dSChristoph Hellwig bool coherent = dev_is_dma_coherent(dev); 1454ee1ef05dSChristoph Hellwig int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); 1455ee1ef05dSChristoph Hellwig struct page *page = NULL; 1456ee1ef05dSChristoph Hellwig void *cpu_addr; 1457ee1ef05dSChristoph Hellwig 1458ee1ef05dSChristoph Hellwig gfp |= __GFP_ZERO; 1459ee1ef05dSChristoph Hellwig 1460f5ff79fdSChristoph Hellwig if (gfpflags_allow_blocking(gfp) && 1461e8d39a90SChristoph Hellwig !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) { 1462e8d39a90SChristoph Hellwig return iommu_dma_alloc_remap(dev, size, handle, gfp, 1463e8d39a90SChristoph Hellwig dma_pgprot(dev, PAGE_KERNEL, attrs), attrs); 1464e8d39a90SChristoph Hellwig } 1465ee1ef05dSChristoph Hellwig 1466e6475eb0SChristoph Hellwig if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && 1467e6475eb0SChristoph Hellwig !gfpflags_allow_blocking(gfp) && !coherent) 14689420139fSChristoph Hellwig page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, 14699420139fSChristoph Hellwig gfp, NULL); 1470ee1ef05dSChristoph Hellwig else 1471ee1ef05dSChristoph Hellwig cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); 1472ee1ef05dSChristoph Hellwig if (!cpu_addr) 1473ee1ef05dSChristoph Hellwig return NULL; 1474ee1ef05dSChristoph Hellwig 14756e235020STom Murphy *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot, 14766e235020STom Murphy dev->coherent_dma_mask); 1477ee1ef05dSChristoph Hellwig if (*handle == DMA_MAPPING_ERROR) { 1478ee1ef05dSChristoph Hellwig __iommu_dma_free(dev, size, cpu_addr); 1479ee1ef05dSChristoph Hellwig return NULL; 1480ee1ef05dSChristoph Hellwig } 1481ee1ef05dSChristoph Hellwig 1482ee1ef05dSChristoph Hellwig return cpu_addr; 1483ee1ef05dSChristoph Hellwig } 1484ee1ef05dSChristoph Hellwig 148506d60728SChristoph Hellwig static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, 148606d60728SChristoph Hellwig void *cpu_addr, dma_addr_t dma_addr, size_t size, 148706d60728SChristoph Hellwig unsigned long attrs) 148806d60728SChristoph Hellwig { 148906d60728SChristoph Hellwig unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1490efd9f10bSChristoph Hellwig unsigned long pfn, off = vma->vm_pgoff; 149106d60728SChristoph Hellwig int ret; 149206d60728SChristoph Hellwig 149333dcb37cSChristoph Hellwig vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); 149406d60728SChristoph Hellwig 149506d60728SChristoph Hellwig if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) 149606d60728SChristoph Hellwig return ret; 149706d60728SChristoph Hellwig 149806d60728SChristoph Hellwig if (off >= nr_pages || vma_pages(vma) > nr_pages - off) 149906d60728SChristoph Hellwig return -ENXIO; 150006d60728SChristoph Hellwig 1501f5ff79fdSChristoph Hellwig if (is_vmalloc_addr(cpu_addr)) { 15025cf45379SChristoph Hellwig struct page **pages = dma_common_find_pages(cpu_addr); 150306d60728SChristoph Hellwig 1504efd9f10bSChristoph Hellwig if (pages) 150571fe89ceSChristoph Hellwig return vm_map_pages(vma, pages, nr_pages); 1506efd9f10bSChristoph Hellwig pfn = vmalloc_to_pfn(cpu_addr); 1507efd9f10bSChristoph Hellwig } else { 1508efd9f10bSChristoph Hellwig pfn = page_to_pfn(virt_to_page(cpu_addr)); 1509efd9f10bSChristoph Hellwig } 1510efd9f10bSChristoph Hellwig 1511efd9f10bSChristoph Hellwig return remap_pfn_range(vma, vma->vm_start, pfn + off, 1512efd9f10bSChristoph Hellwig vma->vm_end - vma->vm_start, 1513efd9f10bSChristoph Hellwig vma->vm_page_prot); 151406d60728SChristoph Hellwig } 151506d60728SChristoph Hellwig 151606d60728SChristoph Hellwig static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 151706d60728SChristoph Hellwig void *cpu_addr, dma_addr_t dma_addr, size_t size, 151806d60728SChristoph Hellwig unsigned long attrs) 151906d60728SChristoph Hellwig { 15203fb3378bSChristoph Hellwig struct page *page; 15213fb3378bSChristoph Hellwig int ret; 152206d60728SChristoph Hellwig 1523f5ff79fdSChristoph Hellwig if (is_vmalloc_addr(cpu_addr)) { 15245cf45379SChristoph Hellwig struct page **pages = dma_common_find_pages(cpu_addr); 15253fb3378bSChristoph Hellwig 15263fb3378bSChristoph Hellwig if (pages) { 15273fb3378bSChristoph Hellwig return sg_alloc_table_from_pages(sgt, pages, 15283fb3378bSChristoph Hellwig PAGE_ALIGN(size) >> PAGE_SHIFT, 15293fb3378bSChristoph Hellwig 0, size, GFP_KERNEL); 153006d60728SChristoph Hellwig } 153106d60728SChristoph Hellwig 15323fb3378bSChristoph Hellwig page = vmalloc_to_page(cpu_addr); 15333fb3378bSChristoph Hellwig } else { 15343fb3378bSChristoph Hellwig page = virt_to_page(cpu_addr); 153506d60728SChristoph Hellwig } 153606d60728SChristoph Hellwig 15373fb3378bSChristoph Hellwig ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 15383fb3378bSChristoph Hellwig if (!ret) 15393fb3378bSChristoph Hellwig sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 15403fb3378bSChristoph Hellwig return ret; 154106d60728SChristoph Hellwig } 154206d60728SChristoph Hellwig 1543158a6d3cSYoshihiro Shimoda static unsigned long iommu_dma_get_merge_boundary(struct device *dev) 1544158a6d3cSYoshihiro Shimoda { 1545158a6d3cSYoshihiro Shimoda struct iommu_domain *domain = iommu_get_dma_domain(dev); 1546158a6d3cSYoshihiro Shimoda 1547158a6d3cSYoshihiro Shimoda return (1UL << __ffs(domain->pgsize_bitmap)) - 1; 1548158a6d3cSYoshihiro Shimoda } 1549158a6d3cSYoshihiro Shimoda 15506d9870b7SJohn Garry static size_t iommu_dma_opt_mapping_size(void) 15516d9870b7SJohn Garry { 15526d9870b7SJohn Garry return iova_rcache_range(); 15536d9870b7SJohn Garry } 15546d9870b7SJohn Garry 155506d60728SChristoph Hellwig static const struct dma_map_ops iommu_dma_ops = { 155630280eeeSLogan Gunthorpe .flags = DMA_F_PCI_P2PDMA_SUPPORTED, 155706d60728SChristoph Hellwig .alloc = iommu_dma_alloc, 155806d60728SChristoph Hellwig .free = iommu_dma_free, 1559efa70f2fSChristoph Hellwig .alloc_pages = dma_common_alloc_pages, 1560efa70f2fSChristoph Hellwig .free_pages = dma_common_free_pages, 1561e817ee5fSChristoph Hellwig .alloc_noncontiguous = iommu_dma_alloc_noncontiguous, 1562e817ee5fSChristoph Hellwig .free_noncontiguous = iommu_dma_free_noncontiguous, 156306d60728SChristoph Hellwig .mmap = iommu_dma_mmap, 156406d60728SChristoph Hellwig .get_sgtable = iommu_dma_get_sgtable, 156506d60728SChristoph Hellwig .map_page = iommu_dma_map_page, 156606d60728SChristoph Hellwig .unmap_page = iommu_dma_unmap_page, 156706d60728SChristoph Hellwig .map_sg = iommu_dma_map_sg, 156806d60728SChristoph Hellwig .unmap_sg = iommu_dma_unmap_sg, 156906d60728SChristoph Hellwig .sync_single_for_cpu = iommu_dma_sync_single_for_cpu, 157006d60728SChristoph Hellwig .sync_single_for_device = iommu_dma_sync_single_for_device, 157106d60728SChristoph Hellwig .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu, 157206d60728SChristoph Hellwig .sync_sg_for_device = iommu_dma_sync_sg_for_device, 157306d60728SChristoph Hellwig .map_resource = iommu_dma_map_resource, 157406d60728SChristoph Hellwig .unmap_resource = iommu_dma_unmap_resource, 1575158a6d3cSYoshihiro Shimoda .get_merge_boundary = iommu_dma_get_merge_boundary, 15766d9870b7SJohn Garry .opt_mapping_size = iommu_dma_opt_mapping_size, 157706d60728SChristoph Hellwig }; 157806d60728SChristoph Hellwig 157906d60728SChristoph Hellwig /* 158006d60728SChristoph Hellwig * The IOMMU core code allocates the default DMA domain, which the underlying 158106d60728SChristoph Hellwig * IOMMU driver needs to support via the dma-iommu layer. 158206d60728SChristoph Hellwig */ 1583ac6d7046SJean-Philippe Brucker void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) 158406d60728SChristoph Hellwig { 158506d60728SChristoph Hellwig struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 158606d60728SChristoph Hellwig 158706d60728SChristoph Hellwig if (!domain) 158806d60728SChristoph Hellwig goto out_err; 158906d60728SChristoph Hellwig 159006d60728SChristoph Hellwig /* 159106d60728SChristoph Hellwig * The IOMMU core code allocates the default DMA domain, which the 159206d60728SChristoph Hellwig * underlying IOMMU driver needs to support via the dma-iommu layer. 159306d60728SChristoph Hellwig */ 1594bf3aed46SRobin Murphy if (iommu_is_dma_domain(domain)) { 1595ac6d7046SJean-Philippe Brucker if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev)) 159606d60728SChristoph Hellwig goto out_err; 159706d60728SChristoph Hellwig dev->dma_ops = &iommu_dma_ops; 159806d60728SChristoph Hellwig } 159906d60728SChristoph Hellwig 160006d60728SChristoph Hellwig return; 160106d60728SChristoph Hellwig out_err: 160206d60728SChristoph Hellwig pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", 160306d60728SChristoph Hellwig dev_name(dev)); 160444bb7e24SRobin Murphy } 16058ce4904bSJean-Philippe Brucker EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); 160644bb7e24SRobin Murphy 160744bb7e24SRobin Murphy static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, 160844bb7e24SRobin Murphy phys_addr_t msi_addr, struct iommu_domain *domain) 160944bb7e24SRobin Murphy { 161044bb7e24SRobin Murphy struct iommu_dma_cookie *cookie = domain->iova_cookie; 161144bb7e24SRobin Murphy struct iommu_dma_msi_page *msi_page; 1612842fe519SRobin Murphy dma_addr_t iova; 161344bb7e24SRobin Murphy int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1614fdbe574eSRobin Murphy size_t size = cookie_msi_granule(cookie); 161544bb7e24SRobin Murphy 1616fdbe574eSRobin Murphy msi_addr &= ~(phys_addr_t)(size - 1); 161744bb7e24SRobin Murphy list_for_each_entry(msi_page, &cookie->msi_page_list, list) 161844bb7e24SRobin Murphy if (msi_page->phys == msi_addr) 161944bb7e24SRobin Murphy return msi_page; 162044bb7e24SRobin Murphy 1621c1864790SRobin Murphy msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL); 162244bb7e24SRobin Murphy if (!msi_page) 162344bb7e24SRobin Murphy return NULL; 162444bb7e24SRobin Murphy 16258af23fadSRobin Murphy iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); 16268af23fadSRobin Murphy if (!iova) 162744bb7e24SRobin Murphy goto out_free_page; 162844bb7e24SRobin Murphy 16291369459bSJason Gunthorpe if (iommu_map(domain, iova, msi_addr, size, prot, GFP_KERNEL)) 16308af23fadSRobin Murphy goto out_free_iova; 16318af23fadSRobin Murphy 163244bb7e24SRobin Murphy INIT_LIST_HEAD(&msi_page->list); 1633a44e6657SRobin Murphy msi_page->phys = msi_addr; 1634a44e6657SRobin Murphy msi_page->iova = iova; 163544bb7e24SRobin Murphy list_add(&msi_page->list, &cookie->msi_page_list); 163644bb7e24SRobin Murphy return msi_page; 163744bb7e24SRobin Murphy 16388af23fadSRobin Murphy out_free_iova: 16392a2b8eaaSTom Murphy iommu_dma_free_iova(cookie, iova, size, NULL); 164044bb7e24SRobin Murphy out_free_page: 164144bb7e24SRobin Murphy kfree(msi_page); 164244bb7e24SRobin Murphy return NULL; 164344bb7e24SRobin Murphy } 164444bb7e24SRobin Murphy 1645fa49364cSRobin Murphy /** 1646fa49364cSRobin Murphy * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain 1647fa49364cSRobin Murphy * @desc: MSI descriptor, will store the MSI page 1648fa49364cSRobin Murphy * @msi_addr: MSI target address to be mapped 1649fa49364cSRobin Murphy * 1650fa49364cSRobin Murphy * Return: 0 on success or negative error code if the mapping failed. 1651fa49364cSRobin Murphy */ 1652ece6e6f0SJulien Grall int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) 165344bb7e24SRobin Murphy { 1654ece6e6f0SJulien Grall struct device *dev = msi_desc_to_dev(desc); 165544bb7e24SRobin Murphy struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 165644bb7e24SRobin Murphy struct iommu_dma_msi_page *msi_page; 1657c1864790SRobin Murphy static DEFINE_MUTEX(msi_prepare_lock); /* see below */ 165844bb7e24SRobin Murphy 1659ece6e6f0SJulien Grall if (!domain || !domain->iova_cookie) { 1660ece6e6f0SJulien Grall desc->iommu_cookie = NULL; 1661ece6e6f0SJulien Grall return 0; 1662ece6e6f0SJulien Grall } 166344bb7e24SRobin Murphy 166444bb7e24SRobin Murphy /* 1665c1864790SRobin Murphy * In fact the whole prepare operation should already be serialised by 1666c1864790SRobin Murphy * irq_domain_mutex further up the callchain, but that's pretty subtle 1667c1864790SRobin Murphy * on its own, so consider this locking as failsafe documentation... 166844bb7e24SRobin Murphy */ 1669c1864790SRobin Murphy mutex_lock(&msi_prepare_lock); 167044bb7e24SRobin Murphy msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); 1671c1864790SRobin Murphy mutex_unlock(&msi_prepare_lock); 167244bb7e24SRobin Murphy 1673ece6e6f0SJulien Grall msi_desc_set_iommu_cookie(desc, msi_page); 1674ece6e6f0SJulien Grall 1675ece6e6f0SJulien Grall if (!msi_page) 1676ece6e6f0SJulien Grall return -ENOMEM; 1677ece6e6f0SJulien Grall return 0; 167844bb7e24SRobin Murphy } 1679ece6e6f0SJulien Grall 1680fa49364cSRobin Murphy /** 1681fa49364cSRobin Murphy * iommu_dma_compose_msi_msg() - Apply translation to an MSI message 1682fa49364cSRobin Murphy * @desc: MSI descriptor prepared by iommu_dma_prepare_msi() 1683fa49364cSRobin Murphy * @msg: MSI message containing target physical address 1684fa49364cSRobin Murphy */ 1685fa49364cSRobin Murphy void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 1686ece6e6f0SJulien Grall { 1687ece6e6f0SJulien Grall struct device *dev = msi_desc_to_dev(desc); 1688ece6e6f0SJulien Grall const struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 1689ece6e6f0SJulien Grall const struct iommu_dma_msi_page *msi_page; 1690ece6e6f0SJulien Grall 1691ece6e6f0SJulien Grall msi_page = msi_desc_get_iommu_cookie(desc); 1692ece6e6f0SJulien Grall 1693ece6e6f0SJulien Grall if (!domain || !domain->iova_cookie || WARN_ON(!msi_page)) 1694ece6e6f0SJulien Grall return; 1695ece6e6f0SJulien Grall 1696ece6e6f0SJulien Grall msg->address_hi = upper_32_bits(msi_page->iova); 1697ece6e6f0SJulien Grall msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1; 1698ece6e6f0SJulien Grall msg->address_lo += lower_32_bits(msi_page->iova); 169944bb7e24SRobin Murphy } 170006d60728SChristoph Hellwig 170106d60728SChristoph Hellwig static int iommu_dma_init(void) 170206d60728SChristoph Hellwig { 1703a8e8af35SLianbo Jiang if (is_kdump_kernel()) 1704a8e8af35SLianbo Jiang static_branch_enable(&iommu_deferred_attach_enabled); 1705a8e8af35SLianbo Jiang 170606d60728SChristoph Hellwig return iova_cache_get(); 17070db2e5d1SRobin Murphy } 170806d60728SChristoph Hellwig arch_initcall(iommu_dma_init); 1709