1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> 4 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> 5 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. 6 */ 7 #include <linux/dma-direct.h> 8 #include <linux/dma-noncoherent.h> 9 #include <linux/dma-contiguous.h> 10 #include <linux/highmem.h> 11 12 #include <asm/cache.h> 13 #include <asm/cpu-type.h> 14 #include <asm/dma-coherence.h> 15 #include <asm/io.h> 16 17 /* 18 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively 19 * fill random cachelines with stale data at any time, requiring an extra 20 * flush post-DMA. 21 * 22 * Warning on the terminology - Linux calls an uncached area coherent; MIPS 23 * terminology calls memory areas with hardware maintained coherency coherent. 24 * 25 * Note that the R14000 and R16000 should also be checked for in this condition. 26 * However this function is only called on non-I/O-coherent systems and only the 27 * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp. 28 * SGI IP32 aka O2. 29 */ 30 static inline bool cpu_needs_post_dma_flush(struct device *dev) 31 { 32 switch (boot_cpu_type()) { 33 case CPU_R10000: 34 case CPU_R12000: 35 case CPU_BMIPS5000: 36 return true; 37 default: 38 /* 39 * Presence of MAARs suggests that the CPU supports 40 * speculatively prefetching data, and therefore requires 41 * the post-DMA flush/invalidate. 42 */ 43 return cpu_has_maar; 44 } 45 } 46 47 void *arch_dma_alloc(struct device *dev, size_t size, 48 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 49 { 50 void *ret; 51 52 ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); 53 if (ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) { 54 dma_cache_wback_inv((unsigned long) ret, size); 55 ret = (void *)UNCAC_ADDR(ret); 56 } 57 58 return ret; 59 } 60 61 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, 62 dma_addr_t dma_addr, unsigned long attrs) 63 { 64 if (!(attrs & DMA_ATTR_NON_CONSISTENT)) 65 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); 66 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); 67 } 68 69 long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, 70 dma_addr_t dma_addr) 71 { 72 unsigned long addr = CAC_ADDR((unsigned long)cpu_addr); 73 return page_to_pfn(virt_to_page((void *)addr)); 74 } 75 76 pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot, 77 unsigned long attrs) 78 { 79 if (attrs & DMA_ATTR_WRITE_COMBINE) 80 return pgprot_writecombine(prot); 81 return pgprot_noncached(prot); 82 } 83 84 static inline void dma_sync_virt(void *addr, size_t size, 85 enum dma_data_direction dir) 86 { 87 switch (dir) { 88 case DMA_TO_DEVICE: 89 dma_cache_wback((unsigned long)addr, size); 90 break; 91 92 case DMA_FROM_DEVICE: 93 dma_cache_inv((unsigned long)addr, size); 94 break; 95 96 case DMA_BIDIRECTIONAL: 97 dma_cache_wback_inv((unsigned long)addr, size); 98 break; 99 100 default: 101 BUG(); 102 } 103 } 104 105 /* 106 * A single sg entry may refer to multiple physically contiguous pages. But 107 * we still need to process highmem pages individually. If highmem is not 108 * configured then the bulk of this loop gets optimized out. 109 */ 110 static inline void dma_sync_phys(phys_addr_t paddr, size_t size, 111 enum dma_data_direction dir) 112 { 113 struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); 114 unsigned long offset = paddr & ~PAGE_MASK; 115 size_t left = size; 116 117 do { 118 size_t len = left; 119 120 if (PageHighMem(page)) { 121 void *addr; 122 123 if (offset + len > PAGE_SIZE) { 124 if (offset >= PAGE_SIZE) { 125 page += offset >> PAGE_SHIFT; 126 offset &= ~PAGE_MASK; 127 } 128 len = PAGE_SIZE - offset; 129 } 130 131 addr = kmap_atomic(page); 132 dma_sync_virt(addr + offset, len, dir); 133 kunmap_atomic(addr); 134 } else 135 dma_sync_virt(page_address(page) + offset, size, dir); 136 offset = 0; 137 page++; 138 left -= len; 139 } while (left); 140 } 141 142 void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, 143 size_t size, enum dma_data_direction dir) 144 { 145 dma_sync_phys(paddr, size, dir); 146 } 147 148 void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, 149 size_t size, enum dma_data_direction dir) 150 { 151 if (cpu_needs_post_dma_flush(dev)) 152 dma_sync_phys(paddr, size, dir); 153 } 154 155 void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, 156 enum dma_data_direction direction) 157 { 158 BUG_ON(direction == DMA_NONE); 159 160 dma_sync_virt(vaddr, size, direction); 161 } 162