1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/dma-noncoherent.h> 10 #include <asm/cache.h> 11 #include <asm/cacheflush.h> 12 13 /* 14 * ARCH specific callbacks for generic noncoherent DMA ops (dma/noncoherent.c) 15 * - hardware IOC not available (or "dma-coherent" not set for device in DT) 16 * - But still handle both coherent and non-coherent requests from caller 17 * 18 * For DMA coherent hardware (IOC) generic code suffices 19 */ 20 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 21 gfp_t gfp, unsigned long attrs) 22 { 23 unsigned long order = get_order(size); 24 struct page *page; 25 phys_addr_t paddr; 26 void *kvaddr; 27 bool need_coh = !(attrs & DMA_ATTR_NON_CONSISTENT); 28 29 /* 30 * __GFP_HIGHMEM flag is cleared by upper layer functions 31 * (in include/linux/dma-mapping.h) so we should never get a 32 * __GFP_HIGHMEM here. 33 */ 34 BUG_ON(gfp & __GFP_HIGHMEM); 35 36 page = alloc_pages(gfp, order); 37 if (!page) 38 return NULL; 39 40 /* This is linear addr (0x8000_0000 based) */ 41 paddr = page_to_phys(page); 42 43 *dma_handle = paddr; 44 45 /* 46 * A coherent buffer needs MMU mapping to enforce non-cachability. 47 * kvaddr is kernel Virtual address (0x7000_0000 based). 48 */ 49 if (need_coh) { 50 kvaddr = ioremap_nocache(paddr, size); 51 if (kvaddr == NULL) { 52 __free_pages(page, order); 53 return NULL; 54 } 55 } else { 56 kvaddr = (void *)(u32)paddr; 57 } 58 59 /* 60 * Evict any existing L1 and/or L2 lines for the backing page 61 * in case it was used earlier as a normal "cached" page. 62 * Yeah this bit us - STAR 9000898266 63 * 64 * Although core does call flush_cache_vmap(), it gets kvaddr hence 65 * can't be used to efficiently flush L1 and/or L2 which need paddr 66 * Currently flush_cache_vmap nukes the L1 cache completely which 67 * will be optimized as a separate commit 68 */ 69 if (need_coh) 70 dma_cache_wback_inv(paddr, size); 71 72 return kvaddr; 73 } 74 75 void arch_dma_free(struct device *dev, size_t size, void *vaddr, 76 dma_addr_t dma_handle, unsigned long attrs) 77 { 78 phys_addr_t paddr = dma_handle; 79 struct page *page = virt_to_page(paddr); 80 81 if (!(attrs & DMA_ATTR_NON_CONSISTENT)) 82 iounmap((void __force __iomem *)vaddr); 83 84 __free_pages(page, get_order(size)); 85 } 86 87 long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, 88 dma_addr_t dma_addr) 89 { 90 return __phys_to_pfn(dma_addr); 91 } 92 93 /* 94 * Cache operations depending on function and direction argument, inspired by 95 * https://lkml.org/lkml/2018/5/18/979 96 * "dma_sync_*_for_cpu and direction=TO_DEVICE (was Re: [PATCH 02/20] 97 * dma-mapping: provide a generic dma-noncoherent implementation)" 98 * 99 * | map == for_device | unmap == for_cpu 100 * |---------------------------------------------------------------- 101 * TO_DEV | writeback writeback | none none 102 * FROM_DEV | invalidate invalidate | invalidate* invalidate* 103 * BIDIR | writeback+inv writeback+inv | invalidate invalidate 104 * 105 * [*] needed for CPU speculative prefetches 106 * 107 * NOTE: we don't check the validity of direction argument as it is done in 108 * upper layer functions (in include/linux/dma-mapping.h) 109 */ 110 111 void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, 112 size_t size, enum dma_data_direction dir) 113 { 114 switch (dir) { 115 case DMA_TO_DEVICE: 116 dma_cache_wback(paddr, size); 117 break; 118 119 case DMA_FROM_DEVICE: 120 dma_cache_inv(paddr, size); 121 break; 122 123 case DMA_BIDIRECTIONAL: 124 dma_cache_wback_inv(paddr, size); 125 break; 126 127 default: 128 break; 129 } 130 } 131 132 void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, 133 size_t size, enum dma_data_direction dir) 134 { 135 switch (dir) { 136 case DMA_TO_DEVICE: 137 break; 138 139 /* FROM_DEVICE invalidate needed if speculative CPU prefetch only */ 140 case DMA_FROM_DEVICE: 141 case DMA_BIDIRECTIONAL: 142 dma_cache_inv(paddr, size); 143 break; 144 145 default: 146 break; 147 } 148 } 149 150 /* 151 * Plug in direct dma map ops. 152 */ 153 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 154 const struct iommu_ops *iommu, bool coherent) 155 { 156 /* 157 * IOC hardware snoops all DMA traffic keeping the caches consistent 158 * with memory - eliding need for any explicit cache maintenance of 159 * DMA buffers. 160 */ 161 if (is_isa_arcv2() && ioc_enable && coherent) 162 dev->dma_coherent = true; 163 164 dev_info(dev, "use %sncoherent DMA ops\n", 165 dev->dma_coherent ? "" : "non"); 166 } 167