1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> 4 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> 5 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. 6 */ 7 #include <linux/dma-direct.h> 8 #include <linux/dma-noncoherent.h> 9 #include <linux/dma-contiguous.h> 10 #include <linux/highmem.h> 11 12 #include <asm/cache.h> 13 #include <asm/cpu-type.h> 14 #include <asm/dma-coherence.h> 15 #include <asm/io.h> 16 17 /* 18 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively 19 * fill random cachelines with stale data at any time, requiring an extra 20 * flush post-DMA. 21 * 22 * Warning on the terminology - Linux calls an uncached area coherent; MIPS 23 * terminology calls memory areas with hardware maintained coherency coherent. 24 * 25 * Note that the R14000 and R16000 should also be checked for in this condition. 26 * However this function is only called on non-I/O-coherent systems and only the 27 * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp. 28 * SGI IP32 aka O2. 29 */ 30 static inline bool cpu_needs_post_dma_flush(void) 31 { 32 switch (boot_cpu_type()) { 33 case CPU_R10000: 34 case CPU_R12000: 35 case CPU_BMIPS5000: 36 case CPU_LOONGSON2EF: 37 return true; 38 default: 39 /* 40 * Presence of MAARs suggests that the CPU supports 41 * speculatively prefetching data, and therefore requires 42 * the post-DMA flush/invalidate. 43 */ 44 return cpu_has_maar; 45 } 46 } 47 48 void arch_dma_prep_coherent(struct page *page, size_t size) 49 { 50 dma_cache_wback_inv((unsigned long)page_address(page), size); 51 } 52 53 void *arch_dma_set_uncached(void *addr, size_t size) 54 { 55 return (void *)(__pa(addr) + UNCAC_BASE); 56 } 57 58 static inline void dma_sync_virt_for_device(void *addr, size_t size, 59 enum dma_data_direction dir) 60 { 61 switch (dir) { 62 case DMA_TO_DEVICE: 63 dma_cache_wback((unsigned long)addr, size); 64 break; 65 case DMA_FROM_DEVICE: 66 dma_cache_inv((unsigned long)addr, size); 67 break; 68 case DMA_BIDIRECTIONAL: 69 dma_cache_wback_inv((unsigned long)addr, size); 70 break; 71 default: 72 BUG(); 73 } 74 } 75 76 static inline void dma_sync_virt_for_cpu(void *addr, size_t size, 77 enum dma_data_direction dir) 78 { 79 switch (dir) { 80 case DMA_TO_DEVICE: 81 break; 82 case DMA_FROM_DEVICE: 83 case DMA_BIDIRECTIONAL: 84 dma_cache_inv((unsigned long)addr, size); 85 break; 86 default: 87 BUG(); 88 } 89 } 90 91 /* 92 * A single sg entry may refer to multiple physically contiguous pages. But 93 * we still need to process highmem pages individually. If highmem is not 94 * configured then the bulk of this loop gets optimized out. 95 */ 96 static inline void dma_sync_phys(phys_addr_t paddr, size_t size, 97 enum dma_data_direction dir, bool for_device) 98 { 99 struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); 100 unsigned long offset = paddr & ~PAGE_MASK; 101 size_t left = size; 102 103 do { 104 size_t len = left; 105 void *addr; 106 107 if (PageHighMem(page)) { 108 if (offset + len > PAGE_SIZE) 109 len = PAGE_SIZE - offset; 110 } 111 112 addr = kmap_atomic(page); 113 if (for_device) 114 dma_sync_virt_for_device(addr + offset, len, dir); 115 else 116 dma_sync_virt_for_cpu(addr + offset, len, dir); 117 kunmap_atomic(addr); 118 119 offset = 0; 120 page++; 121 left -= len; 122 } while (left); 123 } 124 125 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, 126 enum dma_data_direction dir) 127 { 128 dma_sync_phys(paddr, size, dir, true); 129 } 130 131 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU 132 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, 133 enum dma_data_direction dir) 134 { 135 if (cpu_needs_post_dma_flush()) 136 dma_sync_phys(paddr, size, dir, false); 137 } 138 #endif 139 140 void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, 141 enum dma_data_direction direction) 142 { 143 dma_sync_virt_for_device(vaddr, size, direction); 144 } 145 146 #ifdef CONFIG_DMA_PERDEV_COHERENT 147 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 148 const struct iommu_ops *iommu, bool coherent) 149 { 150 dev->dma_coherent = coherent; 151 } 152 #endif 153