1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> 4 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> 5 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. 6 */ 7 #include <linux/dma-direct.h> 8 #include <linux/dma-noncoherent.h> 9 #include <linux/dma-contiguous.h> 10 #include <linux/highmem.h> 11 12 #include <asm/cache.h> 13 #include <asm/cpu-type.h> 14 #include <asm/dma-coherence.h> 15 #include <asm/io.h> 16 17 #ifdef CONFIG_DMA_PERDEV_COHERENT 18 static inline int dev_is_coherent(struct device *dev) 19 { 20 return dev->archdata.dma_coherent; 21 } 22 #else 23 static inline int dev_is_coherent(struct device *dev) 24 { 25 switch (coherentio) { 26 default: 27 case IO_COHERENCE_DEFAULT: 28 return hw_coherentio; 29 case IO_COHERENCE_ENABLED: 30 return 1; 31 case IO_COHERENCE_DISABLED: 32 return 0; 33 } 34 } 35 #endif /* CONFIG_DMA_PERDEV_COHERENT */ 36 37 /* 38 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively 39 * fill random cachelines with stale data at any time, requiring an extra 40 * flush post-DMA. 41 * 42 * Warning on the terminology - Linux calls an uncached area coherent; MIPS 43 * terminology calls memory areas with hardware maintained coherency coherent. 44 * 45 * Note that the R14000 and R16000 should also be checked for in this condition. 46 * However this function is only called on non-I/O-coherent systems and only the 47 * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp. 48 * SGI IP32 aka O2. 49 */ 50 static inline bool cpu_needs_post_dma_flush(struct device *dev) 51 { 52 if (dev_is_coherent(dev)) 53 return false; 54 55 switch (boot_cpu_type()) { 56 case CPU_R10000: 57 case CPU_R12000: 58 case CPU_BMIPS5000: 59 return true; 60 default: 61 /* 62 * Presence of MAARs suggests that the CPU supports 63 * speculatively prefetching data, and therefore requires 64 * the post-DMA flush/invalidate. 65 */ 66 return cpu_has_maar; 67 } 68 } 69 70 void *arch_dma_alloc(struct device *dev, size_t size, 71 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 72 { 73 void *ret; 74 75 ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); 76 if (!ret) 77 return NULL; 78 79 if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { 80 dma_cache_wback_inv((unsigned long) ret, size); 81 ret = (void *)UNCAC_ADDR(ret); 82 } 83 84 return ret; 85 } 86 87 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, 88 dma_addr_t dma_addr, unsigned long attrs) 89 { 90 if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev)) 91 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); 92 dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); 93 } 94 95 int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, 96 void *cpu_addr, dma_addr_t dma_addr, size_t size, 97 unsigned long attrs) 98 { 99 unsigned long user_count = vma_pages(vma); 100 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 101 unsigned long addr = (unsigned long)cpu_addr; 102 unsigned long off = vma->vm_pgoff; 103 unsigned long pfn; 104 int ret = -ENXIO; 105 106 if (!dev_is_coherent(dev)) 107 addr = CAC_ADDR(addr); 108 109 pfn = page_to_pfn(virt_to_page((void *)addr)); 110 111 if (attrs & DMA_ATTR_WRITE_COMBINE) 112 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 113 else 114 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 115 116 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) 117 return ret; 118 119 if (off < count && user_count <= (count - off)) { 120 ret = remap_pfn_range(vma, vma->vm_start, 121 pfn + off, 122 user_count << PAGE_SHIFT, 123 vma->vm_page_prot); 124 } 125 126 return ret; 127 } 128 129 static inline void dma_sync_virt(void *addr, size_t size, 130 enum dma_data_direction dir) 131 { 132 switch (dir) { 133 case DMA_TO_DEVICE: 134 dma_cache_wback((unsigned long)addr, size); 135 break; 136 137 case DMA_FROM_DEVICE: 138 dma_cache_inv((unsigned long)addr, size); 139 break; 140 141 case DMA_BIDIRECTIONAL: 142 dma_cache_wback_inv((unsigned long)addr, size); 143 break; 144 145 default: 146 BUG(); 147 } 148 } 149 150 /* 151 * A single sg entry may refer to multiple physically contiguous pages. But 152 * we still need to process highmem pages individually. If highmem is not 153 * configured then the bulk of this loop gets optimized out. 154 */ 155 static inline void dma_sync_phys(phys_addr_t paddr, size_t size, 156 enum dma_data_direction dir) 157 { 158 struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); 159 unsigned long offset = paddr & ~PAGE_MASK; 160 size_t left = size; 161 162 do { 163 size_t len = left; 164 165 if (PageHighMem(page)) { 166 void *addr; 167 168 if (offset + len > PAGE_SIZE) { 169 if (offset >= PAGE_SIZE) { 170 page += offset >> PAGE_SHIFT; 171 offset &= ~PAGE_MASK; 172 } 173 len = PAGE_SIZE - offset; 174 } 175 176 addr = kmap_atomic(page); 177 dma_sync_virt(addr + offset, len, dir); 178 kunmap_atomic(addr); 179 } else 180 dma_sync_virt(page_address(page) + offset, size, dir); 181 offset = 0; 182 page++; 183 left -= len; 184 } while (left); 185 } 186 187 void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, 188 size_t size, enum dma_data_direction dir) 189 { 190 if (!dev_is_coherent(dev)) 191 dma_sync_phys(paddr, size, dir); 192 } 193 194 void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, 195 size_t size, enum dma_data_direction dir) 196 { 197 if (cpu_needs_post_dma_flush(dev)) 198 dma_sync_phys(paddr, size, dir); 199 } 200 201 void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, 202 enum dma_data_direction direction) 203 { 204 BUG_ON(direction == DMA_NONE); 205 206 if (!dev_is_coherent(dev)) 207 dma_sync_virt(vaddr, size, direction); 208 } 209