1 /* 2 * DMA coherent memory allocation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License as published by the 6 * Free Software Foundation; either version 2 of the License, or (at your 7 * option) any later version. 8 * 9 * Copyright (C) 2002 - 2005 Tensilica Inc. 10 * Copyright (C) 2015 Cadence Design Systems Inc. 11 * 12 * Based on version for i386. 13 * 14 * Chris Zankel <chris@zankel.net> 15 * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 */ 17 18 #include <linux/gfp.h> 19 #include <linux/highmem.h> 20 #include <linux/mm.h> 21 #include <linux/module.h> 22 #include <linux/pci.h> 23 #include <linux/string.h> 24 #include <linux/types.h> 25 #include <asm/cacheflush.h> 26 #include <asm/io.h> 27 28 void dma_cache_sync(struct device *dev, void *vaddr, size_t size, 29 enum dma_data_direction dir) 30 { 31 switch (dir) { 32 case DMA_BIDIRECTIONAL: 33 __flush_invalidate_dcache_range((unsigned long)vaddr, size); 34 break; 35 36 case DMA_FROM_DEVICE: 37 __invalidate_dcache_range((unsigned long)vaddr, size); 38 break; 39 40 case DMA_TO_DEVICE: 41 __flush_dcache_range((unsigned long)vaddr, size); 42 break; 43 44 case DMA_NONE: 45 BUG(); 46 break; 47 } 48 } 49 EXPORT_SYMBOL(dma_cache_sync); 50 51 static void do_cache_op(dma_addr_t dma_handle, size_t size, 52 void (*fn)(unsigned long, unsigned long)) 53 { 54 unsigned long off = dma_handle & (PAGE_SIZE - 1); 55 unsigned long pfn = PFN_DOWN(dma_handle); 56 struct page *page = pfn_to_page(pfn); 57 58 if (!PageHighMem(page)) 59 fn((unsigned long)bus_to_virt(dma_handle), size); 60 else 61 while (size > 0) { 62 size_t sz = min_t(size_t, size, PAGE_SIZE - off); 63 void *vaddr = kmap_atomic(page); 64 65 fn((unsigned long)vaddr + off, sz); 66 kunmap_atomic(vaddr); 67 off = 0; 68 ++page; 69 size -= sz; 70 } 71 } 72 73 static void xtensa_sync_single_for_cpu(struct device *dev, 74 dma_addr_t dma_handle, size_t size, 75 enum dma_data_direction dir) 76 { 77 switch (dir) { 78 case DMA_BIDIRECTIONAL: 79 case DMA_FROM_DEVICE: 80 do_cache_op(dma_handle, size, __invalidate_dcache_range); 81 break; 82 83 case DMA_NONE: 84 BUG(); 85 break; 86 87 default: 88 break; 89 } 90 } 91 92 static void xtensa_sync_single_for_device(struct device *dev, 93 dma_addr_t dma_handle, size_t size, 94 enum dma_data_direction dir) 95 { 96 switch (dir) { 97 case DMA_BIDIRECTIONAL: 98 case DMA_TO_DEVICE: 99 if (XCHAL_DCACHE_IS_WRITEBACK) 100 do_cache_op(dma_handle, size, __flush_dcache_range); 101 break; 102 103 case DMA_NONE: 104 BUG(); 105 break; 106 107 default: 108 break; 109 } 110 } 111 112 static void xtensa_sync_sg_for_cpu(struct device *dev, 113 struct scatterlist *sg, int nents, 114 enum dma_data_direction dir) 115 { 116 struct scatterlist *s; 117 int i; 118 119 for_each_sg(sg, s, nents, i) { 120 xtensa_sync_single_for_cpu(dev, sg_dma_address(s), 121 sg_dma_len(s), dir); 122 } 123 } 124 125 static void xtensa_sync_sg_for_device(struct device *dev, 126 struct scatterlist *sg, int nents, 127 enum dma_data_direction dir) 128 { 129 struct scatterlist *s; 130 int i; 131 132 for_each_sg(sg, s, nents, i) { 133 xtensa_sync_single_for_device(dev, sg_dma_address(s), 134 sg_dma_len(s), dir); 135 } 136 } 137 138 /* 139 * Note: We assume that the full memory space is always mapped to 'kseg' 140 * Otherwise we have to use page attributes (not implemented). 141 */ 142 143 static void *xtensa_dma_alloc(struct device *dev, size_t size, 144 dma_addr_t *handle, gfp_t flag, 145 struct dma_attrs *attrs) 146 { 147 unsigned long ret; 148 unsigned long uncached = 0; 149 150 /* ignore region speicifiers */ 151 152 flag &= ~(__GFP_DMA | __GFP_HIGHMEM); 153 154 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) 155 flag |= GFP_DMA; 156 ret = (unsigned long)__get_free_pages(flag, get_order(size)); 157 158 if (ret == 0) 159 return NULL; 160 161 /* We currently don't support coherent memory outside KSEG */ 162 163 BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR || 164 ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1); 165 166 uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR; 167 *handle = virt_to_bus((void *)ret); 168 __invalidate_dcache_range(ret, size); 169 170 return (void *)uncached; 171 } 172 173 static void xtensa_dma_free(struct device *hwdev, size_t size, void *vaddr, 174 dma_addr_t dma_handle, struct dma_attrs *attrs) 175 { 176 unsigned long addr = (unsigned long)vaddr + 177 XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR; 178 179 BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR || 180 addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1); 181 182 free_pages(addr, get_order(size)); 183 } 184 185 static dma_addr_t xtensa_map_page(struct device *dev, struct page *page, 186 unsigned long offset, size_t size, 187 enum dma_data_direction dir, 188 struct dma_attrs *attrs) 189 { 190 dma_addr_t dma_handle = page_to_phys(page) + offset; 191 192 xtensa_sync_single_for_device(dev, dma_handle, size, dir); 193 return dma_handle; 194 } 195 196 static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle, 197 size_t size, enum dma_data_direction dir, 198 struct dma_attrs *attrs) 199 { 200 xtensa_sync_single_for_cpu(dev, dma_handle, size, dir); 201 } 202 203 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg, 204 int nents, enum dma_data_direction dir, 205 struct dma_attrs *attrs) 206 { 207 struct scatterlist *s; 208 int i; 209 210 for_each_sg(sg, s, nents, i) { 211 s->dma_address = xtensa_map_page(dev, sg_page(s), s->offset, 212 s->length, dir, attrs); 213 } 214 return nents; 215 } 216 217 static void xtensa_unmap_sg(struct device *dev, 218 struct scatterlist *sg, int nents, 219 enum dma_data_direction dir, 220 struct dma_attrs *attrs) 221 { 222 struct scatterlist *s; 223 int i; 224 225 for_each_sg(sg, s, nents, i) { 226 xtensa_unmap_page(dev, sg_dma_address(s), 227 sg_dma_len(s), dir, attrs); 228 } 229 } 230 231 int xtensa_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 232 { 233 return 0; 234 } 235 236 struct dma_map_ops xtensa_dma_map_ops = { 237 .alloc = xtensa_dma_alloc, 238 .free = xtensa_dma_free, 239 .map_page = xtensa_map_page, 240 .unmap_page = xtensa_unmap_page, 241 .map_sg = xtensa_map_sg, 242 .unmap_sg = xtensa_unmap_sg, 243 .sync_single_for_cpu = xtensa_sync_single_for_cpu, 244 .sync_single_for_device = xtensa_sync_single_for_device, 245 .sync_sg_for_cpu = xtensa_sync_sg_for_cpu, 246 .sync_sg_for_device = xtensa_sync_sg_for_device, 247 .mapping_error = xtensa_dma_mapping_error, 248 }; 249 EXPORT_SYMBOL(xtensa_dma_map_ops); 250 251 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) 252 253 static int __init xtensa_dma_init(void) 254 { 255 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 256 return 0; 257 } 258 fs_initcall(xtensa_dma_init); 259