1 /* 2 * DMA coherent memory allocation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License as published by the 6 * Free Software Foundation; either version 2 of the License, or (at your 7 * option) any later version. 8 * 9 * Copyright (C) 2002 - 2005 Tensilica Inc. 10 * Copyright (C) 2015 Cadence Design Systems Inc. 11 * 12 * Based on version for i386. 13 * 14 * Chris Zankel <chris@zankel.net> 15 * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 */ 17 18 #include <linux/dma-contiguous.h> 19 #include <linux/gfp.h> 20 #include <linux/highmem.h> 21 #include <linux/mm.h> 22 #include <linux/module.h> 23 #include <linux/pci.h> 24 #include <linux/string.h> 25 #include <linux/types.h> 26 #include <asm/cacheflush.h> 27 #include <asm/io.h> 28 29 static void do_cache_op(dma_addr_t dma_handle, size_t size, 30 void (*fn)(unsigned long, unsigned long)) 31 { 32 unsigned long off = dma_handle & (PAGE_SIZE - 1); 33 unsigned long pfn = PFN_DOWN(dma_handle); 34 struct page *page = pfn_to_page(pfn); 35 36 if (!PageHighMem(page)) 37 fn((unsigned long)bus_to_virt(dma_handle), size); 38 else 39 while (size > 0) { 40 size_t sz = min_t(size_t, size, PAGE_SIZE - off); 41 void *vaddr = kmap_atomic(page); 42 43 fn((unsigned long)vaddr + off, sz); 44 kunmap_atomic(vaddr); 45 off = 0; 46 ++page; 47 size -= sz; 48 } 49 } 50 51 static void xtensa_sync_single_for_cpu(struct device *dev, 52 dma_addr_t dma_handle, size_t size, 53 enum dma_data_direction dir) 54 { 55 switch (dir) { 56 case DMA_BIDIRECTIONAL: 57 case DMA_FROM_DEVICE: 58 do_cache_op(dma_handle, size, __invalidate_dcache_range); 59 break; 60 61 case DMA_NONE: 62 BUG(); 63 break; 64 65 default: 66 break; 67 } 68 } 69 70 static void xtensa_sync_single_for_device(struct device *dev, 71 dma_addr_t dma_handle, size_t size, 72 enum dma_data_direction dir) 73 { 74 switch (dir) { 75 case DMA_BIDIRECTIONAL: 76 case DMA_TO_DEVICE: 77 if (XCHAL_DCACHE_IS_WRITEBACK) 78 do_cache_op(dma_handle, size, __flush_dcache_range); 79 break; 80 81 case DMA_NONE: 82 BUG(); 83 break; 84 85 default: 86 break; 87 } 88 } 89 90 static void xtensa_sync_sg_for_cpu(struct device *dev, 91 struct scatterlist *sg, int nents, 92 enum dma_data_direction dir) 93 { 94 struct scatterlist *s; 95 int i; 96 97 for_each_sg(sg, s, nents, i) { 98 xtensa_sync_single_for_cpu(dev, sg_dma_address(s), 99 sg_dma_len(s), dir); 100 } 101 } 102 103 static void xtensa_sync_sg_for_device(struct device *dev, 104 struct scatterlist *sg, int nents, 105 enum dma_data_direction dir) 106 { 107 struct scatterlist *s; 108 int i; 109 110 for_each_sg(sg, s, nents, i) { 111 xtensa_sync_single_for_device(dev, sg_dma_address(s), 112 sg_dma_len(s), dir); 113 } 114 } 115 116 /* 117 * Note: We assume that the full memory space is always mapped to 'kseg' 118 * Otherwise we have to use page attributes (not implemented). 119 */ 120 121 static void *xtensa_dma_alloc(struct device *dev, size_t size, 122 dma_addr_t *handle, gfp_t flag, 123 unsigned long attrs) 124 { 125 unsigned long ret; 126 unsigned long uncached = 0; 127 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 128 struct page *page = NULL; 129 130 /* ignore region speicifiers */ 131 132 flag &= ~(__GFP_DMA | __GFP_HIGHMEM); 133 134 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) 135 flag |= GFP_DMA; 136 137 if (gfpflags_allow_blocking(flag)) 138 page = dma_alloc_from_contiguous(dev, count, get_order(size), 139 flag); 140 141 if (!page) 142 page = alloc_pages(flag, get_order(size)); 143 144 if (!page) 145 return NULL; 146 147 ret = (unsigned long)page_address(page); 148 149 /* We currently don't support coherent memory outside KSEG */ 150 151 BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR || 152 ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1); 153 154 uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR; 155 *handle = virt_to_bus((void *)ret); 156 __invalidate_dcache_range(ret, size); 157 158 return (void *)uncached; 159 } 160 161 static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr, 162 dma_addr_t dma_handle, unsigned long attrs) 163 { 164 unsigned long addr = (unsigned long)vaddr + 165 XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR; 166 struct page *page = virt_to_page(addr); 167 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 168 169 BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR || 170 addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1); 171 172 if (!dma_release_from_contiguous(dev, page, count)) 173 __free_pages(page, get_order(size)); 174 } 175 176 static dma_addr_t xtensa_map_page(struct device *dev, struct page *page, 177 unsigned long offset, size_t size, 178 enum dma_data_direction dir, 179 unsigned long attrs) 180 { 181 dma_addr_t dma_handle = page_to_phys(page) + offset; 182 183 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 184 xtensa_sync_single_for_device(dev, dma_handle, size, dir); 185 186 return dma_handle; 187 } 188 189 static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle, 190 size_t size, enum dma_data_direction dir, 191 unsigned long attrs) 192 { 193 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 194 xtensa_sync_single_for_cpu(dev, dma_handle, size, dir); 195 } 196 197 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg, 198 int nents, enum dma_data_direction dir, 199 unsigned long attrs) 200 { 201 struct scatterlist *s; 202 int i; 203 204 for_each_sg(sg, s, nents, i) { 205 s->dma_address = xtensa_map_page(dev, sg_page(s), s->offset, 206 s->length, dir, attrs); 207 } 208 return nents; 209 } 210 211 static void xtensa_unmap_sg(struct device *dev, 212 struct scatterlist *sg, int nents, 213 enum dma_data_direction dir, 214 unsigned long attrs) 215 { 216 struct scatterlist *s; 217 int i; 218 219 for_each_sg(sg, s, nents, i) { 220 xtensa_unmap_page(dev, sg_dma_address(s), 221 sg_dma_len(s), dir, attrs); 222 } 223 } 224 225 int xtensa_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 226 { 227 return 0; 228 } 229 230 const struct dma_map_ops xtensa_dma_map_ops = { 231 .alloc = xtensa_dma_alloc, 232 .free = xtensa_dma_free, 233 .map_page = xtensa_map_page, 234 .unmap_page = xtensa_unmap_page, 235 .map_sg = xtensa_map_sg, 236 .unmap_sg = xtensa_unmap_sg, 237 .sync_single_for_cpu = xtensa_sync_single_for_cpu, 238 .sync_single_for_device = xtensa_sync_single_for_device, 239 .sync_sg_for_cpu = xtensa_sync_sg_for_cpu, 240 .sync_sg_for_device = xtensa_sync_sg_for_device, 241 .mapping_error = xtensa_dma_mapping_error, 242 }; 243 EXPORT_SYMBOL(xtensa_dma_map_ops); 244 245 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) 246 247 static int __init xtensa_dma_init(void) 248 { 249 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 250 return 0; 251 } 252 fs_initcall(xtensa_dma_init); 253