Lines Matching +full:dma +full:- +full:coherent
1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/arch/arm/mm/dma-mapping.c
5 * Copyright (C) 2000-2004 Russell King
7 * DMA uncached mapping support.
17 #include <linux/dma-direct.h>
18 #include <linux/dma-map-ops.h>
33 #include <asm/dma-iommu.h>
36 #include <asm/xen/xen-ops.h>
38 #include "dma.h"
60 #define COHERENT 1 macro
84 if (buf->virt == virt) { in arm_dma_buffer_find()
85 list_del(&buf->list); in arm_dma_buffer_find()
95 * The DMA API is built upon the notion of "buffer ownership". A buffer
97 * by it) or exclusively owned by the DMA device. These helper functions
111 * lurking in the kernel direct-mapped region is invalidated. in __dma_clear_buffer()
119 if (coherent_flag != COHERENT) in __dma_clear_buffer()
123 size -= PAGE_SIZE; in __dma_clear_buffer()
125 if (coherent_flag != COHERENT) in __dma_clear_buffer()
130 if (coherent_flag != COHERENT) { in __dma_clear_buffer()
138 * Allocate a DMA buffer for 'dev' of size 'size' using the
164 * Free a DMA buffer. 'size' must be page aligned.
198 * Initialise the coherent pool for atomic allocations.
207 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); in atomic_pool_init()
211 * The atomic pool is only used for non-coherent allocations in atomic_pool_init()
226 atomic_pool_size, -1); in atomic_pool_init()
233 pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n", in atomic_pool_init()
242 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", in atomic_pool_init()
244 return -ENOMEM; in atomic_pool_init()
286 map.length = end - start; in dma_contiguous_remap()
290 * Clear previous low-memory mapping to ensure that the in dma_contiguous_remap()
336 * non-coherent in __alloc_remap_buffer()
361 WARN(1, "coherent pool not initialised!\n"); in __alloc_from_pool()
450 /* __alloc_simple_buffer is only called when the device is coherent */ in __alloc_simple_buffer()
451 page = __dma_alloc_buffer(dev, size, gfp, COHERENT); in __alloc_simple_buffer()
462 return __alloc_simple_buffer(args->dev, args->size, args->gfp, in simple_allocator_alloc()
468 __dma_free_buffer(args->page, args->size); in simple_allocator_free()
479 return __alloc_from_contiguous(args->dev, args->size, args->prot, in cma_allocator_alloc()
480 ret_page, args->caller, in cma_allocator_alloc()
481 args->want_vaddr, args->coherent_flag, in cma_allocator_alloc()
482 args->gfp); in cma_allocator_alloc()
487 __free_from_contiguous(args->dev, args->page, args->cpu_addr, in cma_allocator_free()
488 args->size, args->want_vaddr); in cma_allocator_free()
499 return __alloc_from_pool(args->size, ret_page); in pool_allocator_alloc()
504 __free_from_pool(args->cpu_addr, args->size); in pool_allocator_free()
515 return __alloc_remap_buffer(args->dev, args->size, args->gfp, in remap_allocator_alloc()
516 args->prot, ret_page, args->caller, in remap_allocator_alloc()
517 args->want_vaddr); in remap_allocator_alloc()
522 if (args->want_vaddr) in remap_allocator_free()
523 dma_common_free_remap(args->cpu_addr, args->size); in remap_allocator_free()
525 __dma_free_buffer(args->page, args->size); in remap_allocator_free()
537 u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); in __dma_alloc()
549 .coherent_flag = is_coherent ? COHERENT : NORMAL, in __dma_alloc()
555 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", in __dma_alloc()
576 buf->allocator = &cma_allocator; in __dma_alloc()
578 buf->allocator = &simple_allocator; in __dma_alloc()
580 buf->allocator = &remap_allocator; in __dma_alloc()
582 buf->allocator = &pool_allocator; in __dma_alloc()
584 addr = buf->allocator->alloc(&args, &page); in __dma_alloc()
590 buf->virt = args.want_vaddr ? addr : page; in __dma_alloc()
593 list_add(&buf->list, &arm_dma_bufs); in __dma_alloc()
623 buf->allocator->free(&args); in __arm_dma_free()
651 len = PAGE_SIZE - offset; in dma_cache_maint_page()
670 left -= len; in dma_cache_maint_page()
677 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
692 /* FIXME: non-speculating: flush on bidirectional mappings? */ in __dma_page_cpu_to_dev()
700 /* FIXME: non-speculating: not required */ in __dma_page_dev_to_cpu()
701 /* in any case, don't bother invalidating if DMA to device */ in __dma_page_dev_to_cpu()
709 * Mark the D-cache clean for these pages to avoid extra flushing. in __dma_page_dev_to_cpu()
716 size_t sz = folio_size(folio) - offset; in __dma_page_dev_to_cpu()
721 set_bit(PG_dcache_clean, &folio->flags); in __dma_page_dev_to_cpu()
723 size -= sz; in __dma_page_dev_to_cpu()
762 size_t mapping_size = mapping->bits << PAGE_SHIFT; in __alloc_iova()
771 align = (1 << order) - 1; in __alloc_iova()
773 spin_lock_irqsave(&mapping->lock, flags); in __alloc_iova()
774 for (i = 0; i < mapping->nr_bitmaps; i++) { in __alloc_iova()
775 start = bitmap_find_next_zero_area(mapping->bitmaps[i], in __alloc_iova()
776 mapping->bits, 0, count, align); in __alloc_iova()
778 if (start > mapping->bits) in __alloc_iova()
781 bitmap_set(mapping->bitmaps[i], start, count); in __alloc_iova()
790 if (i == mapping->nr_bitmaps) { in __alloc_iova()
792 spin_unlock_irqrestore(&mapping->lock, flags); in __alloc_iova()
796 start = bitmap_find_next_zero_area(mapping->bitmaps[i], in __alloc_iova()
797 mapping->bits, 0, count, align); in __alloc_iova()
799 if (start > mapping->bits) { in __alloc_iova()
800 spin_unlock_irqrestore(&mapping->lock, flags); in __alloc_iova()
804 bitmap_set(mapping->bitmaps[i], start, count); in __alloc_iova()
806 spin_unlock_irqrestore(&mapping->lock, flags); in __alloc_iova()
808 iova = mapping->base + (mapping_size * i); in __alloc_iova()
818 size_t mapping_size = mapping->bits << PAGE_SHIFT; in __free_iova()
826 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; in __free_iova()
827 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); in __free_iova()
829 bitmap_base = mapping->base + mapping_size * bitmap_index; in __free_iova()
831 start = (addr - bitmap_base) >> PAGE_SHIFT; in __free_iova()
844 spin_lock_irqsave(&mapping->lock, flags); in __free_iova()
845 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); in __free_iova()
846 spin_unlock_irqrestore(&mapping->lock, flags); in __free_iova()
889 order_idx = ARRAY_SIZE(iommu_order_array) - 1; in __iommu_alloc_buffer()
908 /* See if it's easy to allocate a high-order chunk */ in __iommu_alloc_buffer()
925 while (--j) in __iommu_alloc_buffer()
931 count -= 1 << order; in __iommu_alloc_buffer()
936 while (i--) in __iommu_alloc_buffer()
989 len = (j - i) << PAGE_SHIFT; in __iommu_create_mapping()
990 ret = iommu_map(mapping->domain, iova, phys, len, in __iommu_create_mapping()
1000 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); in __iommu_create_mapping()
1010 * add optional in-page offset from iova to size and align in __iommu_remove_mapping()
1016 iommu_unmap(mapping->domain, iova, size); in __iommu_remove_mapping()
1050 if (coherent_flag == COHERENT) in __iommu_alloc_simple()
1072 if (coherent_flag == COHERENT) in __iommu_free_atomic()
1084 int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; in arm_iommu_alloc_attrs()
1089 if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp)) in arm_iommu_alloc_attrs()
1127 return -ENXIO; in arm_iommu_mmap_attrs()
1129 if (vma->vm_pgoff >= nr_pages) in arm_iommu_mmap_attrs()
1130 return -ENXIO; in arm_iommu_mmap_attrs()
1132 if (!dev->dma_coherent) in arm_iommu_mmap_attrs()
1133 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); in arm_iommu_mmap_attrs()
1149 int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; in arm_iommu_free_attrs()
1153 if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) { in arm_iommu_free_attrs()
1160 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); in arm_iommu_free_attrs()
1179 return -ENXIO; in arm_iommu_get_sgtable()
1186 * Map a part of the scatter-gather list into contiguous io address space
1204 return -ENOMEM; in __map_sg_chunk()
1208 unsigned int len = PAGE_ALIGN(s->offset + s->length); in __map_sg_chunk()
1210 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) in __map_sg_chunk()
1211 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); in __map_sg_chunk()
1215 ret = iommu_map(mapping->domain, iova, phys, len, prot, in __map_sg_chunk()
1226 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); in __map_sg_chunk()
1232 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1236 * @dir: DMA transfer direction
1238 * Map a set of buffers described by scatterlist in streaming mode for DMA.
1240 * tagged with the appropriate dma address and length. They are obtained via
1246 struct scatterlist *s = sg, *dma = sg, *start = sg; in arm_iommu_map_sg() local
1248 unsigned int offset = s->offset; in arm_iommu_map_sg()
1249 unsigned int size = s->offset + s->length; in arm_iommu_map_sg()
1255 s->dma_length = 0; in arm_iommu_map_sg()
1257 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { in arm_iommu_map_sg()
1259 &dma->dma_address, dir, attrs); in arm_iommu_map_sg()
1263 dma->dma_address += offset; in arm_iommu_map_sg()
1264 dma->dma_length = size - offset; in arm_iommu_map_sg()
1266 size = offset = s->offset; in arm_iommu_map_sg()
1268 dma = sg_next(dma); in arm_iommu_map_sg()
1271 size += s->length; in arm_iommu_map_sg()
1273 ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs); in arm_iommu_map_sg()
1277 dma->dma_address += offset; in arm_iommu_map_sg()
1278 dma->dma_length = size - offset; in arm_iommu_map_sg()
1285 if (ret == -ENOMEM) in arm_iommu_map_sg()
1287 return -EINVAL; in arm_iommu_map_sg()
1291 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1295 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1297 * Unmap a set of streaming mode DMA translations. Again, CPU access
1312 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) in arm_iommu_unmap_sg()
1313 __dma_page_dev_to_cpu(sg_page(s), s->offset, in arm_iommu_unmap_sg()
1314 s->length, dir); in arm_iommu_unmap_sg()
1323 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1332 if (dev->dma_coherent) in arm_iommu_sync_sg_for_cpu()
1336 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); in arm_iommu_sync_sg_for_cpu()
1345 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1354 if (dev->dma_coherent) in arm_iommu_sync_sg_for_device()
1358 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); in arm_iommu_sync_sg_for_device()
1367 * @dir: DMA transfer direction
1379 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) in arm_iommu_map_page()
1388 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, in arm_iommu_map_page()
1402 * @handle: DMA address of buffer
1404 * @dir: DMA transfer direction (same as passed to dma_map_page)
1420 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { in arm_iommu_unmap_page()
1421 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); in arm_iommu_unmap_page()
1425 iommu_unmap(mapping->domain, iova, len); in arm_iommu_unmap_page()
1430 * arm_iommu_map_resource - map a device resource for DMA
1434 * @dir: DMA transfer direction
1453 ret = iommu_map(mapping->domain, dma_addr, addr, len, prot, GFP_KERNEL); in arm_iommu_map_resource()
1464 * arm_iommu_unmap_resource - unmap a device DMA resource
1466 * @dma_handle: DMA address to resource
1468 * @dir: DMA transfer direction
1482 iommu_unmap(mapping->domain, iova, len); in arm_iommu_unmap_resource()
1494 if (dev->dma_coherent || !iova) in arm_iommu_sync_single_for_cpu()
1497 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); in arm_iommu_sync_single_for_cpu()
1509 if (dev->dma_coherent || !iova) in arm_iommu_sync_single_for_device()
1512 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); in arm_iommu_sync_single_for_device()
1556 int err = -ENOMEM; in arm_iommu_create_mapping()
1558 /* currently only 32-bit DMA address space is supported */ in arm_iommu_create_mapping()
1560 return ERR_PTR(-ERANGE); in arm_iommu_create_mapping()
1563 return ERR_PTR(-EINVAL); in arm_iommu_create_mapping()
1574 mapping->bitmap_size = bitmap_size; in arm_iommu_create_mapping()
1575 mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *), in arm_iommu_create_mapping()
1577 if (!mapping->bitmaps) in arm_iommu_create_mapping()
1580 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); in arm_iommu_create_mapping()
1581 if (!mapping->bitmaps[0]) in arm_iommu_create_mapping()
1584 mapping->nr_bitmaps = 1; in arm_iommu_create_mapping()
1585 mapping->extensions = extensions; in arm_iommu_create_mapping()
1586 mapping->base = base; in arm_iommu_create_mapping()
1587 mapping->bits = BITS_PER_BYTE * bitmap_size; in arm_iommu_create_mapping()
1589 spin_lock_init(&mapping->lock); in arm_iommu_create_mapping()
1591 mapping->domain = iommu_domain_alloc(bus); in arm_iommu_create_mapping()
1592 if (!mapping->domain) in arm_iommu_create_mapping()
1595 kref_init(&mapping->kref); in arm_iommu_create_mapping()
1598 kfree(mapping->bitmaps[0]); in arm_iommu_create_mapping()
1600 kfree(mapping->bitmaps); in arm_iommu_create_mapping()
1614 iommu_domain_free(mapping->domain); in release_iommu_mapping()
1615 for (i = 0; i < mapping->nr_bitmaps; i++) in release_iommu_mapping()
1616 kfree(mapping->bitmaps[i]); in release_iommu_mapping()
1617 kfree(mapping->bitmaps); in release_iommu_mapping()
1625 if (mapping->nr_bitmaps >= mapping->extensions) in extend_iommu_mapping()
1626 return -EINVAL; in extend_iommu_mapping()
1628 next_bitmap = mapping->nr_bitmaps; in extend_iommu_mapping()
1629 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, in extend_iommu_mapping()
1631 if (!mapping->bitmaps[next_bitmap]) in extend_iommu_mapping()
1632 return -ENOMEM; in extend_iommu_mapping()
1634 mapping->nr_bitmaps++; in extend_iommu_mapping()
1642 kref_put(&mapping->kref, release_iommu_mapping); in arm_iommu_release_mapping()
1651 err = iommu_attach_device(mapping->domain, dev); in __arm_iommu_attach_device()
1655 kref_get(&mapping->kref); in __arm_iommu_attach_device()
1669 * This replaces the dma operations (dma_map_ops pointer) with the
1694 * This overwrites the dma_ops pointer with appropriate non-IOMMU ops.
1706 iommu_detach_device(mapping->domain, dev); in arm_iommu_detach_device()
1707 kref_put(&mapping->kref, release_iommu_mapping); in arm_iommu_detach_device()
1716 const struct iommu_ops *iommu, bool coherent) in arm_setup_iommu_dma_ops() argument
1720 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); in arm_setup_iommu_dma_ops()
1722 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", in arm_setup_iommu_dma_ops()
1751 const struct iommu_ops *iommu, bool coherent) in arm_setup_iommu_dma_ops() argument
1760 const struct iommu_ops *iommu, bool coherent) in arch_setup_dma_ops() argument
1763 * Due to legacy code that sets the ->dma_coherent flag from a bus in arch_setup_dma_ops()
1764 * notifier we can't just assign coherent to the ->dma_coherent flag in arch_setup_dma_ops()
1768 if (coherent) in arch_setup_dma_ops()
1769 dev->dma_coherent = true; in arch_setup_dma_ops()
1776 if (dev->dma_ops) in arch_setup_dma_ops()
1780 arm_setup_iommu_dma_ops(dev, dma_base, size, iommu, coherent); in arch_setup_dma_ops()
1783 dev->archdata.dma_ops_setup = true; in arch_setup_dma_ops()
1788 if (!dev->archdata.dma_ops_setup) in arch_teardown_dma_ops()
1792 /* Let arch_setup_dma_ops() start again from scratch upon re-probe */ in arch_teardown_dma_ops()
1799 __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), in arch_sync_dma_for_device()
1806 __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), in arch_sync_dma_for_cpu()