1 /* 2 * linux/arch/arm/mm/dma-mapping.c 3 * 4 * Copyright (C) 2000-2004 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * DMA uncached mapping support. 11 */ 12 #include <linux/module.h> 13 #include <linux/mm.h> 14 #include <linux/gfp.h> 15 #include <linux/errno.h> 16 #include <linux/list.h> 17 #include <linux/init.h> 18 #include <linux/device.h> 19 #include <linux/dma-mapping.h> 20 #include <linux/dma-contiguous.h> 21 #include <linux/highmem.h> 22 #include <linux/memblock.h> 23 #include <linux/slab.h> 24 #include <linux/iommu.h> 25 #include <linux/io.h> 26 #include <linux/vmalloc.h> 27 #include <linux/sizes.h> 28 29 #include <asm/memory.h> 30 #include <asm/highmem.h> 31 #include <asm/cacheflush.h> 32 #include <asm/tlbflush.h> 33 #include <asm/mach/arch.h> 34 #include <asm/dma-iommu.h> 35 #include <asm/mach/map.h> 36 #include <asm/system_info.h> 37 #include <asm/dma-contiguous.h> 38 39 #include "mm.h" 40 41 /* 42 * The DMA API is built upon the notion of "buffer ownership". A buffer 43 * is either exclusively owned by the CPU (and therefore may be accessed 44 * by it) or exclusively owned by the DMA device. These helper functions 45 * represent the transitions between these two ownership states. 46 * 47 * Note, however, that on later ARMs, this notion does not work due to 48 * speculative prefetches. We model our approach on the assumption that 49 * the CPU does do speculative prefetches, which means we clean caches 50 * before transfers and delay cache invalidation until transfer completion. 51 * 52 */ 53 static void __dma_page_cpu_to_dev(struct page *, unsigned long, 54 size_t, enum dma_data_direction); 55 static void __dma_page_dev_to_cpu(struct page *, unsigned long, 56 size_t, enum dma_data_direction); 57 58 /** 59 * arm_dma_map_page - map a portion of a page for streaming DMA 60 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 61 * @page: page that buffer resides in 62 * @offset: offset into page for start of buffer 63 * @size: size of buffer to map 64 * @dir: DMA transfer direction 65 * 66 * Ensure that any data held in the cache is appropriately discarded 67 * or written back. 68 * 69 * The device owns this memory once this call has completed. The CPU 70 * can regain ownership by calling dma_unmap_page(). 71 */ 72 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, 73 unsigned long offset, size_t size, enum dma_data_direction dir, 74 struct dma_attrs *attrs) 75 { 76 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 77 __dma_page_cpu_to_dev(page, offset, size, dir); 78 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 79 } 80 81 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, 82 unsigned long offset, size_t size, enum dma_data_direction dir, 83 struct dma_attrs *attrs) 84 { 85 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 86 } 87 88 /** 89 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() 90 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 91 * @handle: DMA address of buffer 92 * @size: size of buffer (same as passed to dma_map_page) 93 * @dir: DMA transfer direction (same as passed to dma_map_page) 94 * 95 * Unmap a page streaming mode DMA translation. The handle and size 96 * must match what was provided in the previous dma_map_page() call. 97 * All other usages are undefined. 98 * 99 * After this call, reads by the CPU to the buffer are guaranteed to see 100 * whatever the device wrote there. 101 */ 102 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, 103 size_t size, enum dma_data_direction dir, 104 struct dma_attrs *attrs) 105 { 106 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 107 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), 108 handle & ~PAGE_MASK, size, dir); 109 } 110 111 static void arm_dma_sync_single_for_cpu(struct device *dev, 112 dma_addr_t handle, size_t size, enum dma_data_direction dir) 113 { 114 unsigned int offset = handle & (PAGE_SIZE - 1); 115 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 116 __dma_page_dev_to_cpu(page, offset, size, dir); 117 } 118 119 static void arm_dma_sync_single_for_device(struct device *dev, 120 dma_addr_t handle, size_t size, enum dma_data_direction dir) 121 { 122 unsigned int offset = handle & (PAGE_SIZE - 1); 123 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 124 __dma_page_cpu_to_dev(page, offset, size, dir); 125 } 126 127 struct dma_map_ops arm_dma_ops = { 128 .alloc = arm_dma_alloc, 129 .free = arm_dma_free, 130 .mmap = arm_dma_mmap, 131 .get_sgtable = arm_dma_get_sgtable, 132 .map_page = arm_dma_map_page, 133 .unmap_page = arm_dma_unmap_page, 134 .map_sg = arm_dma_map_sg, 135 .unmap_sg = arm_dma_unmap_sg, 136 .sync_single_for_cpu = arm_dma_sync_single_for_cpu, 137 .sync_single_for_device = arm_dma_sync_single_for_device, 138 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, 139 .sync_sg_for_device = arm_dma_sync_sg_for_device, 140 .set_dma_mask = arm_dma_set_mask, 141 }; 142 EXPORT_SYMBOL(arm_dma_ops); 143 144 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 145 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); 146 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 147 dma_addr_t handle, struct dma_attrs *attrs); 148 149 struct dma_map_ops arm_coherent_dma_ops = { 150 .alloc = arm_coherent_dma_alloc, 151 .free = arm_coherent_dma_free, 152 .mmap = arm_dma_mmap, 153 .get_sgtable = arm_dma_get_sgtable, 154 .map_page = arm_coherent_dma_map_page, 155 .map_sg = arm_dma_map_sg, 156 .set_dma_mask = arm_dma_set_mask, 157 }; 158 EXPORT_SYMBOL(arm_coherent_dma_ops); 159 160 static u64 get_coherent_dma_mask(struct device *dev) 161 { 162 u64 mask = (u64)arm_dma_limit; 163 164 if (dev) { 165 mask = dev->coherent_dma_mask; 166 167 /* 168 * Sanity check the DMA mask - it must be non-zero, and 169 * must be able to be satisfied by a DMA allocation. 170 */ 171 if (mask == 0) { 172 dev_warn(dev, "coherent DMA mask is unset\n"); 173 return 0; 174 } 175 176 if ((~mask) & (u64)arm_dma_limit) { 177 dev_warn(dev, "coherent DMA mask %#llx is smaller " 178 "than system GFP_DMA mask %#llx\n", 179 mask, (u64)arm_dma_limit); 180 return 0; 181 } 182 } 183 184 return mask; 185 } 186 187 static void __dma_clear_buffer(struct page *page, size_t size) 188 { 189 /* 190 * Ensure that the allocated pages are zeroed, and that any data 191 * lurking in the kernel direct-mapped region is invalidated. 192 */ 193 if (PageHighMem(page)) { 194 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 195 phys_addr_t end = base + size; 196 while (size > 0) { 197 void *ptr = kmap_atomic(page); 198 memset(ptr, 0, PAGE_SIZE); 199 dmac_flush_range(ptr, ptr + PAGE_SIZE); 200 kunmap_atomic(ptr); 201 page++; 202 size -= PAGE_SIZE; 203 } 204 outer_flush_range(base, end); 205 } else { 206 void *ptr = page_address(page); 207 memset(ptr, 0, size); 208 dmac_flush_range(ptr, ptr + size); 209 outer_flush_range(__pa(ptr), __pa(ptr) + size); 210 } 211 } 212 213 /* 214 * Allocate a DMA buffer for 'dev' of size 'size' using the 215 * specified gfp mask. Note that 'size' must be page aligned. 216 */ 217 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) 218 { 219 unsigned long order = get_order(size); 220 struct page *page, *p, *e; 221 222 page = alloc_pages(gfp, order); 223 if (!page) 224 return NULL; 225 226 /* 227 * Now split the huge page and free the excess pages 228 */ 229 split_page(page, order); 230 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 231 __free_page(p); 232 233 __dma_clear_buffer(page, size); 234 235 return page; 236 } 237 238 /* 239 * Free a DMA buffer. 'size' must be page aligned. 240 */ 241 static void __dma_free_buffer(struct page *page, size_t size) 242 { 243 struct page *e = page + (size >> PAGE_SHIFT); 244 245 while (page < e) { 246 __free_page(page); 247 page++; 248 } 249 } 250 251 #ifdef CONFIG_MMU 252 #ifdef CONFIG_HUGETLB_PAGE 253 #error ARM Coherent DMA allocator does not (yet) support huge TLB 254 #endif 255 256 static void *__alloc_from_contiguous(struct device *dev, size_t size, 257 pgprot_t prot, struct page **ret_page, 258 const void *caller); 259 260 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 261 pgprot_t prot, struct page **ret_page, 262 const void *caller); 263 264 static void * 265 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 266 const void *caller) 267 { 268 struct vm_struct *area; 269 unsigned long addr; 270 271 /* 272 * DMA allocation can be mapped to user space, so lets 273 * set VM_USERMAP flags too. 274 */ 275 area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP, 276 caller); 277 if (!area) 278 return NULL; 279 addr = (unsigned long)area->addr; 280 area->phys_addr = __pfn_to_phys(page_to_pfn(page)); 281 282 if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) { 283 vunmap((void *)addr); 284 return NULL; 285 } 286 return (void *)addr; 287 } 288 289 static void __dma_free_remap(void *cpu_addr, size_t size) 290 { 291 unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP; 292 struct vm_struct *area = find_vm_area(cpu_addr); 293 if (!area || (area->flags & flags) != flags) { 294 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 295 return; 296 } 297 unmap_kernel_range((unsigned long)cpu_addr, size); 298 vunmap(cpu_addr); 299 } 300 301 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 302 303 struct dma_pool { 304 size_t size; 305 spinlock_t lock; 306 unsigned long *bitmap; 307 unsigned long nr_pages; 308 void *vaddr; 309 struct page **pages; 310 }; 311 312 static struct dma_pool atomic_pool = { 313 .size = DEFAULT_DMA_COHERENT_POOL_SIZE, 314 }; 315 316 static int __init early_coherent_pool(char *p) 317 { 318 atomic_pool.size = memparse(p, &p); 319 return 0; 320 } 321 early_param("coherent_pool", early_coherent_pool); 322 323 void __init init_dma_coherent_pool_size(unsigned long size) 324 { 325 /* 326 * Catch any attempt to set the pool size too late. 327 */ 328 BUG_ON(atomic_pool.vaddr); 329 330 /* 331 * Set architecture specific coherent pool size only if 332 * it has not been changed by kernel command line parameter. 333 */ 334 if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE) 335 atomic_pool.size = size; 336 } 337 338 /* 339 * Initialise the coherent pool for atomic allocations. 340 */ 341 static int __init atomic_pool_init(void) 342 { 343 struct dma_pool *pool = &atomic_pool; 344 pgprot_t prot = pgprot_dmacoherent(pgprot_kernel); 345 unsigned long nr_pages = pool->size >> PAGE_SHIFT; 346 unsigned long *bitmap; 347 struct page *page; 348 struct page **pages; 349 void *ptr; 350 int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long); 351 352 bitmap = kzalloc(bitmap_size, GFP_KERNEL); 353 if (!bitmap) 354 goto no_bitmap; 355 356 pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL); 357 if (!pages) 358 goto no_pages; 359 360 if (IS_ENABLED(CONFIG_CMA)) 361 ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page, 362 atomic_pool_init); 363 else 364 ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot, 365 &page, atomic_pool_init); 366 if (ptr) { 367 int i; 368 369 for (i = 0; i < nr_pages; i++) 370 pages[i] = page + i; 371 372 spin_lock_init(&pool->lock); 373 pool->vaddr = ptr; 374 pool->pages = pages; 375 pool->bitmap = bitmap; 376 pool->nr_pages = nr_pages; 377 pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n", 378 (unsigned)pool->size / 1024); 379 return 0; 380 } 381 382 kfree(pages); 383 no_pages: 384 kfree(bitmap); 385 no_bitmap: 386 pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", 387 (unsigned)pool->size / 1024); 388 return -ENOMEM; 389 } 390 /* 391 * CMA is activated by core_initcall, so we must be called after it. 392 */ 393 postcore_initcall(atomic_pool_init); 394 395 struct dma_contig_early_reserve { 396 phys_addr_t base; 397 unsigned long size; 398 }; 399 400 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 401 402 static int dma_mmu_remap_num __initdata; 403 404 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 405 { 406 dma_mmu_remap[dma_mmu_remap_num].base = base; 407 dma_mmu_remap[dma_mmu_remap_num].size = size; 408 dma_mmu_remap_num++; 409 } 410 411 void __init dma_contiguous_remap(void) 412 { 413 int i; 414 for (i = 0; i < dma_mmu_remap_num; i++) { 415 phys_addr_t start = dma_mmu_remap[i].base; 416 phys_addr_t end = start + dma_mmu_remap[i].size; 417 struct map_desc map; 418 unsigned long addr; 419 420 if (end > arm_lowmem_limit) 421 end = arm_lowmem_limit; 422 if (start >= end) 423 continue; 424 425 map.pfn = __phys_to_pfn(start); 426 map.virtual = __phys_to_virt(start); 427 map.length = end - start; 428 map.type = MT_MEMORY_DMA_READY; 429 430 /* 431 * Clear previous low-memory mapping 432 */ 433 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 434 addr += PMD_SIZE) 435 pmd_clear(pmd_off_k(addr)); 436 437 iotable_init(&map, 1); 438 } 439 } 440 441 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, 442 void *data) 443 { 444 struct page *page = virt_to_page(addr); 445 pgprot_t prot = *(pgprot_t *)data; 446 447 set_pte_ext(pte, mk_pte(page, prot), 0); 448 return 0; 449 } 450 451 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 452 { 453 unsigned long start = (unsigned long) page_address(page); 454 unsigned end = start + size; 455 456 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 457 dsb(); 458 flush_tlb_kernel_range(start, end); 459 } 460 461 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 462 pgprot_t prot, struct page **ret_page, 463 const void *caller) 464 { 465 struct page *page; 466 void *ptr; 467 page = __dma_alloc_buffer(dev, size, gfp); 468 if (!page) 469 return NULL; 470 471 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 472 if (!ptr) { 473 __dma_free_buffer(page, size); 474 return NULL; 475 } 476 477 *ret_page = page; 478 return ptr; 479 } 480 481 static void *__alloc_from_pool(size_t size, struct page **ret_page) 482 { 483 struct dma_pool *pool = &atomic_pool; 484 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 485 unsigned int pageno; 486 unsigned long flags; 487 void *ptr = NULL; 488 unsigned long align_mask; 489 490 if (!pool->vaddr) { 491 WARN(1, "coherent pool not initialised!\n"); 492 return NULL; 493 } 494 495 /* 496 * Align the region allocation - allocations from pool are rather 497 * small, so align them to their order in pages, minimum is a page 498 * size. This helps reduce fragmentation of the DMA space. 499 */ 500 align_mask = (1 << get_order(size)) - 1; 501 502 spin_lock_irqsave(&pool->lock, flags); 503 pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages, 504 0, count, align_mask); 505 if (pageno < pool->nr_pages) { 506 bitmap_set(pool->bitmap, pageno, count); 507 ptr = pool->vaddr + PAGE_SIZE * pageno; 508 *ret_page = pool->pages[pageno]; 509 } else { 510 pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n" 511 "Please increase it with coherent_pool= kernel parameter!\n", 512 (unsigned)pool->size / 1024); 513 } 514 spin_unlock_irqrestore(&pool->lock, flags); 515 516 return ptr; 517 } 518 519 static bool __in_atomic_pool(void *start, size_t size) 520 { 521 struct dma_pool *pool = &atomic_pool; 522 void *end = start + size; 523 void *pool_start = pool->vaddr; 524 void *pool_end = pool->vaddr + pool->size; 525 526 if (start < pool_start || start >= pool_end) 527 return false; 528 529 if (end <= pool_end) 530 return true; 531 532 WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n", 533 start, end - 1, pool_start, pool_end - 1); 534 535 return false; 536 } 537 538 static int __free_from_pool(void *start, size_t size) 539 { 540 struct dma_pool *pool = &atomic_pool; 541 unsigned long pageno, count; 542 unsigned long flags; 543 544 if (!__in_atomic_pool(start, size)) 545 return 0; 546 547 pageno = (start - pool->vaddr) >> PAGE_SHIFT; 548 count = size >> PAGE_SHIFT; 549 550 spin_lock_irqsave(&pool->lock, flags); 551 bitmap_clear(pool->bitmap, pageno, count); 552 spin_unlock_irqrestore(&pool->lock, flags); 553 554 return 1; 555 } 556 557 static void *__alloc_from_contiguous(struct device *dev, size_t size, 558 pgprot_t prot, struct page **ret_page, 559 const void *caller) 560 { 561 unsigned long order = get_order(size); 562 size_t count = size >> PAGE_SHIFT; 563 struct page *page; 564 void *ptr; 565 566 page = dma_alloc_from_contiguous(dev, count, order); 567 if (!page) 568 return NULL; 569 570 __dma_clear_buffer(page, size); 571 572 if (PageHighMem(page)) { 573 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); 574 if (!ptr) { 575 dma_release_from_contiguous(dev, page, count); 576 return NULL; 577 } 578 } else { 579 __dma_remap(page, size, prot); 580 ptr = page_address(page); 581 } 582 *ret_page = page; 583 return ptr; 584 } 585 586 static void __free_from_contiguous(struct device *dev, struct page *page, 587 void *cpu_addr, size_t size) 588 { 589 if (PageHighMem(page)) 590 __dma_free_remap(cpu_addr, size); 591 else 592 __dma_remap(page, size, pgprot_kernel); 593 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 594 } 595 596 static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) 597 { 598 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? 599 pgprot_writecombine(prot) : 600 pgprot_dmacoherent(prot); 601 return prot; 602 } 603 604 #define nommu() 0 605 606 #else /* !CONFIG_MMU */ 607 608 #define nommu() 1 609 610 #define __get_dma_pgprot(attrs, prot) __pgprot(0) 611 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL 612 #define __alloc_from_pool(size, ret_page) NULL 613 #define __alloc_from_contiguous(dev, size, prot, ret, c) NULL 614 #define __free_from_pool(cpu_addr, size) 0 615 #define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0) 616 #define __dma_free_remap(cpu_addr, size) do { } while (0) 617 618 #endif /* CONFIG_MMU */ 619 620 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 621 struct page **ret_page) 622 { 623 struct page *page; 624 page = __dma_alloc_buffer(dev, size, gfp); 625 if (!page) 626 return NULL; 627 628 *ret_page = page; 629 return page_address(page); 630 } 631 632 633 634 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 635 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller) 636 { 637 u64 mask = get_coherent_dma_mask(dev); 638 struct page *page = NULL; 639 void *addr; 640 641 #ifdef CONFIG_DMA_API_DEBUG 642 u64 limit = (mask + 1) & ~mask; 643 if (limit && size >= limit) { 644 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 645 size, mask); 646 return NULL; 647 } 648 #endif 649 650 if (!mask) 651 return NULL; 652 653 if (mask < 0xffffffffULL) 654 gfp |= GFP_DMA; 655 656 /* 657 * Following is a work-around (a.k.a. hack) to prevent pages 658 * with __GFP_COMP being passed to split_page() which cannot 659 * handle them. The real problem is that this flag probably 660 * should be 0 on ARM as it is not supported on this 661 * platform; see CONFIG_HUGETLBFS. 662 */ 663 gfp &= ~(__GFP_COMP); 664 665 *handle = DMA_ERROR_CODE; 666 size = PAGE_ALIGN(size); 667 668 if (is_coherent || nommu()) 669 addr = __alloc_simple_buffer(dev, size, gfp, &page); 670 else if (!(gfp & __GFP_WAIT)) 671 addr = __alloc_from_pool(size, &page); 672 else if (!IS_ENABLED(CONFIG_CMA)) 673 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); 674 else 675 addr = __alloc_from_contiguous(dev, size, prot, &page, caller); 676 677 if (addr) 678 *handle = pfn_to_dma(dev, page_to_pfn(page)); 679 680 return addr; 681 } 682 683 /* 684 * Allocate DMA-coherent memory space and return both the kernel remapped 685 * virtual and bus address for that space. 686 */ 687 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 688 gfp_t gfp, struct dma_attrs *attrs) 689 { 690 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); 691 void *memory; 692 693 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 694 return memory; 695 696 return __dma_alloc(dev, size, handle, gfp, prot, false, 697 __builtin_return_address(0)); 698 } 699 700 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 701 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 702 { 703 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); 704 void *memory; 705 706 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 707 return memory; 708 709 return __dma_alloc(dev, size, handle, gfp, prot, true, 710 __builtin_return_address(0)); 711 } 712 713 /* 714 * Create userspace mapping for the DMA-coherent memory. 715 */ 716 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 717 void *cpu_addr, dma_addr_t dma_addr, size_t size, 718 struct dma_attrs *attrs) 719 { 720 int ret = -ENXIO; 721 #ifdef CONFIG_MMU 722 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 723 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 724 unsigned long pfn = dma_to_pfn(dev, dma_addr); 725 unsigned long off = vma->vm_pgoff; 726 727 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 728 729 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 730 return ret; 731 732 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 733 ret = remap_pfn_range(vma, vma->vm_start, 734 pfn + off, 735 vma->vm_end - vma->vm_start, 736 vma->vm_page_prot); 737 } 738 #endif /* CONFIG_MMU */ 739 740 return ret; 741 } 742 743 /* 744 * Free a buffer as defined by the above mapping. 745 */ 746 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 747 dma_addr_t handle, struct dma_attrs *attrs, 748 bool is_coherent) 749 { 750 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 751 752 if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) 753 return; 754 755 size = PAGE_ALIGN(size); 756 757 if (is_coherent || nommu()) { 758 __dma_free_buffer(page, size); 759 } else if (__free_from_pool(cpu_addr, size)) { 760 return; 761 } else if (!IS_ENABLED(CONFIG_CMA)) { 762 __dma_free_remap(cpu_addr, size); 763 __dma_free_buffer(page, size); 764 } else { 765 /* 766 * Non-atomic allocations cannot be freed with IRQs disabled 767 */ 768 WARN_ON(irqs_disabled()); 769 __free_from_contiguous(dev, page, cpu_addr, size); 770 } 771 } 772 773 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 774 dma_addr_t handle, struct dma_attrs *attrs) 775 { 776 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); 777 } 778 779 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 780 dma_addr_t handle, struct dma_attrs *attrs) 781 { 782 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); 783 } 784 785 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 786 void *cpu_addr, dma_addr_t handle, size_t size, 787 struct dma_attrs *attrs) 788 { 789 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 790 int ret; 791 792 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 793 if (unlikely(ret)) 794 return ret; 795 796 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 797 return 0; 798 } 799 800 static void dma_cache_maint_page(struct page *page, unsigned long offset, 801 size_t size, enum dma_data_direction dir, 802 void (*op)(const void *, size_t, int)) 803 { 804 unsigned long pfn; 805 size_t left = size; 806 807 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 808 offset %= PAGE_SIZE; 809 810 /* 811 * A single sg entry may refer to multiple physically contiguous 812 * pages. But we still need to process highmem pages individually. 813 * If highmem is not configured then the bulk of this loop gets 814 * optimized out. 815 */ 816 do { 817 size_t len = left; 818 void *vaddr; 819 820 page = pfn_to_page(pfn); 821 822 if (PageHighMem(page)) { 823 if (len + offset > PAGE_SIZE) 824 len = PAGE_SIZE - offset; 825 vaddr = kmap_high_get(page); 826 if (vaddr) { 827 vaddr += offset; 828 op(vaddr, len, dir); 829 kunmap_high(page); 830 } else if (cache_is_vipt()) { 831 /* unmapped pages might still be cached */ 832 vaddr = kmap_atomic(page); 833 op(vaddr + offset, len, dir); 834 kunmap_atomic(vaddr); 835 } 836 } else { 837 vaddr = page_address(page) + offset; 838 op(vaddr, len, dir); 839 } 840 offset = 0; 841 pfn++; 842 left -= len; 843 } while (left); 844 } 845 846 /* 847 * Make an area consistent for devices. 848 * Note: Drivers should NOT use this function directly, as it will break 849 * platforms with CONFIG_DMABOUNCE. 850 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 851 */ 852 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 853 size_t size, enum dma_data_direction dir) 854 { 855 unsigned long paddr; 856 857 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 858 859 paddr = page_to_phys(page) + off; 860 if (dir == DMA_FROM_DEVICE) { 861 outer_inv_range(paddr, paddr + size); 862 } else { 863 outer_clean_range(paddr, paddr + size); 864 } 865 /* FIXME: non-speculating: flush on bidirectional mappings? */ 866 } 867 868 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 869 size_t size, enum dma_data_direction dir) 870 { 871 unsigned long paddr = page_to_phys(page) + off; 872 873 /* FIXME: non-speculating: not required */ 874 /* don't bother invalidating if DMA to device */ 875 if (dir != DMA_TO_DEVICE) 876 outer_inv_range(paddr, paddr + size); 877 878 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 879 880 /* 881 * Mark the D-cache clean for this page to avoid extra flushing. 882 */ 883 if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) 884 set_bit(PG_dcache_clean, &page->flags); 885 } 886 887 /** 888 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA 889 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 890 * @sg: list of buffers 891 * @nents: number of buffers to map 892 * @dir: DMA transfer direction 893 * 894 * Map a set of buffers described by scatterlist in streaming mode for DMA. 895 * This is the scatter-gather version of the dma_map_single interface. 896 * Here the scatter gather list elements are each tagged with the 897 * appropriate dma address and length. They are obtained via 898 * sg_dma_{address,length}. 899 * 900 * Device ownership issues as mentioned for dma_map_single are the same 901 * here. 902 */ 903 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 904 enum dma_data_direction dir, struct dma_attrs *attrs) 905 { 906 struct dma_map_ops *ops = get_dma_ops(dev); 907 struct scatterlist *s; 908 int i, j; 909 910 for_each_sg(sg, s, nents, i) { 911 #ifdef CONFIG_NEED_SG_DMA_LENGTH 912 s->dma_length = s->length; 913 #endif 914 s->dma_address = ops->map_page(dev, sg_page(s), s->offset, 915 s->length, dir, attrs); 916 if (dma_mapping_error(dev, s->dma_address)) 917 goto bad_mapping; 918 } 919 return nents; 920 921 bad_mapping: 922 for_each_sg(sg, s, i, j) 923 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 924 return 0; 925 } 926 927 /** 928 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 929 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 930 * @sg: list of buffers 931 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 932 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 933 * 934 * Unmap a set of streaming mode DMA translations. Again, CPU access 935 * rules concerning calls here are the same as for dma_unmap_single(). 936 */ 937 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 938 enum dma_data_direction dir, struct dma_attrs *attrs) 939 { 940 struct dma_map_ops *ops = get_dma_ops(dev); 941 struct scatterlist *s; 942 943 int i; 944 945 for_each_sg(sg, s, nents, i) 946 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 947 } 948 949 /** 950 * arm_dma_sync_sg_for_cpu 951 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 952 * @sg: list of buffers 953 * @nents: number of buffers to map (returned from dma_map_sg) 954 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 955 */ 956 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 957 int nents, enum dma_data_direction dir) 958 { 959 struct dma_map_ops *ops = get_dma_ops(dev); 960 struct scatterlist *s; 961 int i; 962 963 for_each_sg(sg, s, nents, i) 964 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, 965 dir); 966 } 967 968 /** 969 * arm_dma_sync_sg_for_device 970 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 971 * @sg: list of buffers 972 * @nents: number of buffers to map (returned from dma_map_sg) 973 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 974 */ 975 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 976 int nents, enum dma_data_direction dir) 977 { 978 struct dma_map_ops *ops = get_dma_ops(dev); 979 struct scatterlist *s; 980 int i; 981 982 for_each_sg(sg, s, nents, i) 983 ops->sync_single_for_device(dev, sg_dma_address(s), s->length, 984 dir); 985 } 986 987 /* 988 * Return whether the given device DMA address mask can be supported 989 * properly. For example, if your device can only drive the low 24-bits 990 * during bus mastering, then you would pass 0x00ffffff as the mask 991 * to this function. 992 */ 993 int dma_supported(struct device *dev, u64 mask) 994 { 995 if (mask < (u64)arm_dma_limit) 996 return 0; 997 return 1; 998 } 999 EXPORT_SYMBOL(dma_supported); 1000 1001 int arm_dma_set_mask(struct device *dev, u64 dma_mask) 1002 { 1003 if (!dev->dma_mask || !dma_supported(dev, dma_mask)) 1004 return -EIO; 1005 1006 *dev->dma_mask = dma_mask; 1007 1008 return 0; 1009 } 1010 1011 #define PREALLOC_DMA_DEBUG_ENTRIES 4096 1012 1013 static int __init dma_debug_do_init(void) 1014 { 1015 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 1016 return 0; 1017 } 1018 fs_initcall(dma_debug_do_init); 1019 1020 #ifdef CONFIG_ARM_DMA_USE_IOMMU 1021 1022 /* IOMMU */ 1023 1024 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 1025 size_t size) 1026 { 1027 unsigned int order = get_order(size); 1028 unsigned int align = 0; 1029 unsigned int count, start; 1030 unsigned long flags; 1031 1032 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 1033 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 1034 1035 count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) + 1036 (1 << mapping->order) - 1) >> mapping->order; 1037 1038 if (order > mapping->order) 1039 align = (1 << (order - mapping->order)) - 1; 1040 1041 spin_lock_irqsave(&mapping->lock, flags); 1042 start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0, 1043 count, align); 1044 if (start > mapping->bits) { 1045 spin_unlock_irqrestore(&mapping->lock, flags); 1046 return DMA_ERROR_CODE; 1047 } 1048 1049 bitmap_set(mapping->bitmap, start, count); 1050 spin_unlock_irqrestore(&mapping->lock, flags); 1051 1052 return mapping->base + (start << (mapping->order + PAGE_SHIFT)); 1053 } 1054 1055 static inline void __free_iova(struct dma_iommu_mapping *mapping, 1056 dma_addr_t addr, size_t size) 1057 { 1058 unsigned int start = (addr - mapping->base) >> 1059 (mapping->order + PAGE_SHIFT); 1060 unsigned int count = ((size >> PAGE_SHIFT) + 1061 (1 << mapping->order) - 1) >> mapping->order; 1062 unsigned long flags; 1063 1064 spin_lock_irqsave(&mapping->lock, flags); 1065 bitmap_clear(mapping->bitmap, start, count); 1066 spin_unlock_irqrestore(&mapping->lock, flags); 1067 } 1068 1069 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 1070 gfp_t gfp, struct dma_attrs *attrs) 1071 { 1072 struct page **pages; 1073 int count = size >> PAGE_SHIFT; 1074 int array_size = count * sizeof(struct page *); 1075 int i = 0; 1076 1077 if (array_size <= PAGE_SIZE) 1078 pages = kzalloc(array_size, gfp); 1079 else 1080 pages = vzalloc(array_size); 1081 if (!pages) 1082 return NULL; 1083 1084 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) 1085 { 1086 unsigned long order = get_order(size); 1087 struct page *page; 1088 1089 page = dma_alloc_from_contiguous(dev, count, order); 1090 if (!page) 1091 goto error; 1092 1093 __dma_clear_buffer(page, size); 1094 1095 for (i = 0; i < count; i++) 1096 pages[i] = page + i; 1097 1098 return pages; 1099 } 1100 1101 /* 1102 * IOMMU can map any pages, so himem can also be used here 1103 */ 1104 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 1105 1106 while (count) { 1107 int j, order = __fls(count); 1108 1109 pages[i] = alloc_pages(gfp, order); 1110 while (!pages[i] && order) 1111 pages[i] = alloc_pages(gfp, --order); 1112 if (!pages[i]) 1113 goto error; 1114 1115 if (order) { 1116 split_page(pages[i], order); 1117 j = 1 << order; 1118 while (--j) 1119 pages[i + j] = pages[i] + j; 1120 } 1121 1122 __dma_clear_buffer(pages[i], PAGE_SIZE << order); 1123 i += 1 << order; 1124 count -= 1 << order; 1125 } 1126 1127 return pages; 1128 error: 1129 while (i--) 1130 if (pages[i]) 1131 __free_pages(pages[i], 0); 1132 if (array_size <= PAGE_SIZE) 1133 kfree(pages); 1134 else 1135 vfree(pages); 1136 return NULL; 1137 } 1138 1139 static int __iommu_free_buffer(struct device *dev, struct page **pages, 1140 size_t size, struct dma_attrs *attrs) 1141 { 1142 int count = size >> PAGE_SHIFT; 1143 int array_size = count * sizeof(struct page *); 1144 int i; 1145 1146 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { 1147 dma_release_from_contiguous(dev, pages[0], count); 1148 } else { 1149 for (i = 0; i < count; i++) 1150 if (pages[i]) 1151 __free_pages(pages[i], 0); 1152 } 1153 1154 if (array_size <= PAGE_SIZE) 1155 kfree(pages); 1156 else 1157 vfree(pages); 1158 return 0; 1159 } 1160 1161 /* 1162 * Create a CPU mapping for a specified pages 1163 */ 1164 static void * 1165 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, 1166 const void *caller) 1167 { 1168 unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1169 struct vm_struct *area; 1170 unsigned long p; 1171 1172 area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP, 1173 caller); 1174 if (!area) 1175 return NULL; 1176 1177 area->pages = pages; 1178 area->nr_pages = nr_pages; 1179 p = (unsigned long)area->addr; 1180 1181 for (i = 0; i < nr_pages; i++) { 1182 phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i])); 1183 if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot)) 1184 goto err; 1185 p += PAGE_SIZE; 1186 } 1187 return area->addr; 1188 err: 1189 unmap_kernel_range((unsigned long)area->addr, size); 1190 vunmap(area->addr); 1191 return NULL; 1192 } 1193 1194 /* 1195 * Create a mapping in device IO address space for specified pages 1196 */ 1197 static dma_addr_t 1198 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1199 { 1200 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1201 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1202 dma_addr_t dma_addr, iova; 1203 int i, ret = DMA_ERROR_CODE; 1204 1205 dma_addr = __alloc_iova(mapping, size); 1206 if (dma_addr == DMA_ERROR_CODE) 1207 return dma_addr; 1208 1209 iova = dma_addr; 1210 for (i = 0; i < count; ) { 1211 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 1212 phys_addr_t phys = page_to_phys(pages[i]); 1213 unsigned int len, j; 1214 1215 for (j = i + 1; j < count; j++, next_pfn++) 1216 if (page_to_pfn(pages[j]) != next_pfn) 1217 break; 1218 1219 len = (j - i) << PAGE_SHIFT; 1220 ret = iommu_map(mapping->domain, iova, phys, len, 0); 1221 if (ret < 0) 1222 goto fail; 1223 iova += len; 1224 i = j; 1225 } 1226 return dma_addr; 1227 fail: 1228 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 1229 __free_iova(mapping, dma_addr, size); 1230 return DMA_ERROR_CODE; 1231 } 1232 1233 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1234 { 1235 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1236 1237 /* 1238 * add optional in-page offset from iova to size and align 1239 * result to page size 1240 */ 1241 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1242 iova &= PAGE_MASK; 1243 1244 iommu_unmap(mapping->domain, iova, size); 1245 __free_iova(mapping, iova, size); 1246 return 0; 1247 } 1248 1249 static struct page **__atomic_get_pages(void *addr) 1250 { 1251 struct dma_pool *pool = &atomic_pool; 1252 struct page **pages = pool->pages; 1253 int offs = (addr - pool->vaddr) >> PAGE_SHIFT; 1254 1255 return pages + offs; 1256 } 1257 1258 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) 1259 { 1260 struct vm_struct *area; 1261 1262 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1263 return __atomic_get_pages(cpu_addr); 1264 1265 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1266 return cpu_addr; 1267 1268 area = find_vm_area(cpu_addr); 1269 if (area && (area->flags & VM_ARM_DMA_CONSISTENT)) 1270 return area->pages; 1271 return NULL; 1272 } 1273 1274 static void *__iommu_alloc_atomic(struct device *dev, size_t size, 1275 dma_addr_t *handle) 1276 { 1277 struct page *page; 1278 void *addr; 1279 1280 addr = __alloc_from_pool(size, &page); 1281 if (!addr) 1282 return NULL; 1283 1284 *handle = __iommu_create_mapping(dev, &page, size); 1285 if (*handle == DMA_ERROR_CODE) 1286 goto err_mapping; 1287 1288 return addr; 1289 1290 err_mapping: 1291 __free_from_pool(addr, size); 1292 return NULL; 1293 } 1294 1295 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1296 dma_addr_t handle, size_t size) 1297 { 1298 __iommu_remove_mapping(dev, handle, size); 1299 __free_from_pool(cpu_addr, size); 1300 } 1301 1302 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1303 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 1304 { 1305 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); 1306 struct page **pages; 1307 void *addr = NULL; 1308 1309 *handle = DMA_ERROR_CODE; 1310 size = PAGE_ALIGN(size); 1311 1312 if (gfp & GFP_ATOMIC) 1313 return __iommu_alloc_atomic(dev, size, handle); 1314 1315 pages = __iommu_alloc_buffer(dev, size, gfp, attrs); 1316 if (!pages) 1317 return NULL; 1318 1319 *handle = __iommu_create_mapping(dev, pages, size); 1320 if (*handle == DMA_ERROR_CODE) 1321 goto err_buffer; 1322 1323 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1324 return pages; 1325 1326 addr = __iommu_alloc_remap(pages, size, gfp, prot, 1327 __builtin_return_address(0)); 1328 if (!addr) 1329 goto err_mapping; 1330 1331 return addr; 1332 1333 err_mapping: 1334 __iommu_remove_mapping(dev, *handle, size); 1335 err_buffer: 1336 __iommu_free_buffer(dev, pages, size, attrs); 1337 return NULL; 1338 } 1339 1340 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1341 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1342 struct dma_attrs *attrs) 1343 { 1344 unsigned long uaddr = vma->vm_start; 1345 unsigned long usize = vma->vm_end - vma->vm_start; 1346 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1347 1348 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1349 1350 if (!pages) 1351 return -ENXIO; 1352 1353 do { 1354 int ret = vm_insert_page(vma, uaddr, *pages++); 1355 if (ret) { 1356 pr_err("Remapping memory failed: %d\n", ret); 1357 return ret; 1358 } 1359 uaddr += PAGE_SIZE; 1360 usize -= PAGE_SIZE; 1361 } while (usize > 0); 1362 1363 return 0; 1364 } 1365 1366 /* 1367 * free a page as defined by the above mapping. 1368 * Must not be called with IRQs disabled. 1369 */ 1370 void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1371 dma_addr_t handle, struct dma_attrs *attrs) 1372 { 1373 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1374 size = PAGE_ALIGN(size); 1375 1376 if (!pages) { 1377 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1378 return; 1379 } 1380 1381 if (__in_atomic_pool(cpu_addr, size)) { 1382 __iommu_free_atomic(dev, cpu_addr, handle, size); 1383 return; 1384 } 1385 1386 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { 1387 unmap_kernel_range((unsigned long)cpu_addr, size); 1388 vunmap(cpu_addr); 1389 } 1390 1391 __iommu_remove_mapping(dev, handle, size); 1392 __iommu_free_buffer(dev, pages, size, attrs); 1393 } 1394 1395 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1396 void *cpu_addr, dma_addr_t dma_addr, 1397 size_t size, struct dma_attrs *attrs) 1398 { 1399 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1400 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1401 1402 if (!pages) 1403 return -ENXIO; 1404 1405 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1406 GFP_KERNEL); 1407 } 1408 1409 /* 1410 * Map a part of the scatter-gather list into contiguous io address space 1411 */ 1412 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1413 size_t size, dma_addr_t *handle, 1414 enum dma_data_direction dir, struct dma_attrs *attrs, 1415 bool is_coherent) 1416 { 1417 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1418 dma_addr_t iova, iova_base; 1419 int ret = 0; 1420 unsigned int count; 1421 struct scatterlist *s; 1422 1423 size = PAGE_ALIGN(size); 1424 *handle = DMA_ERROR_CODE; 1425 1426 iova_base = iova = __alloc_iova(mapping, size); 1427 if (iova == DMA_ERROR_CODE) 1428 return -ENOMEM; 1429 1430 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1431 phys_addr_t phys = page_to_phys(sg_page(s)); 1432 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1433 1434 if (!is_coherent && 1435 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1436 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1437 1438 ret = iommu_map(mapping->domain, iova, phys, len, 0); 1439 if (ret < 0) 1440 goto fail; 1441 count += len >> PAGE_SHIFT; 1442 iova += len; 1443 } 1444 *handle = iova_base; 1445 1446 return 0; 1447 fail: 1448 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1449 __free_iova(mapping, iova_base, size); 1450 return ret; 1451 } 1452 1453 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1454 enum dma_data_direction dir, struct dma_attrs *attrs, 1455 bool is_coherent) 1456 { 1457 struct scatterlist *s = sg, *dma = sg, *start = sg; 1458 int i, count = 0; 1459 unsigned int offset = s->offset; 1460 unsigned int size = s->offset + s->length; 1461 unsigned int max = dma_get_max_seg_size(dev); 1462 1463 for (i = 1; i < nents; i++) { 1464 s = sg_next(s); 1465 1466 s->dma_address = DMA_ERROR_CODE; 1467 s->dma_length = 0; 1468 1469 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1470 if (__map_sg_chunk(dev, start, size, &dma->dma_address, 1471 dir, attrs, is_coherent) < 0) 1472 goto bad_mapping; 1473 1474 dma->dma_address += offset; 1475 dma->dma_length = size - offset; 1476 1477 size = offset = s->offset; 1478 start = s; 1479 dma = sg_next(dma); 1480 count += 1; 1481 } 1482 size += s->length; 1483 } 1484 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, 1485 is_coherent) < 0) 1486 goto bad_mapping; 1487 1488 dma->dma_address += offset; 1489 dma->dma_length = size - offset; 1490 1491 return count+1; 1492 1493 bad_mapping: 1494 for_each_sg(sg, s, count, i) 1495 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1496 return 0; 1497 } 1498 1499 /** 1500 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1501 * @dev: valid struct device pointer 1502 * @sg: list of buffers 1503 * @nents: number of buffers to map 1504 * @dir: DMA transfer direction 1505 * 1506 * Map a set of i/o coherent buffers described by scatterlist in streaming 1507 * mode for DMA. The scatter gather list elements are merged together (if 1508 * possible) and tagged with the appropriate dma address and length. They are 1509 * obtained via sg_dma_{address,length}. 1510 */ 1511 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1512 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1513 { 1514 return __iommu_map_sg(dev, sg, nents, dir, attrs, true); 1515 } 1516 1517 /** 1518 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1519 * @dev: valid struct device pointer 1520 * @sg: list of buffers 1521 * @nents: number of buffers to map 1522 * @dir: DMA transfer direction 1523 * 1524 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1525 * The scatter gather list elements are merged together (if possible) and 1526 * tagged with the appropriate dma address and length. They are obtained via 1527 * sg_dma_{address,length}. 1528 */ 1529 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1530 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1531 { 1532 return __iommu_map_sg(dev, sg, nents, dir, attrs, false); 1533 } 1534 1535 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1536 int nents, enum dma_data_direction dir, struct dma_attrs *attrs, 1537 bool is_coherent) 1538 { 1539 struct scatterlist *s; 1540 int i; 1541 1542 for_each_sg(sg, s, nents, i) { 1543 if (sg_dma_len(s)) 1544 __iommu_remove_mapping(dev, sg_dma_address(s), 1545 sg_dma_len(s)); 1546 if (!is_coherent && 1547 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1548 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1549 s->length, dir); 1550 } 1551 } 1552 1553 /** 1554 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1555 * @dev: valid struct device pointer 1556 * @sg: list of buffers 1557 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1558 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1559 * 1560 * Unmap a set of streaming mode DMA translations. Again, CPU access 1561 * rules concerning calls here are the same as for dma_unmap_single(). 1562 */ 1563 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1564 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1565 { 1566 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); 1567 } 1568 1569 /** 1570 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1571 * @dev: valid struct device pointer 1572 * @sg: list of buffers 1573 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1574 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1575 * 1576 * Unmap a set of streaming mode DMA translations. Again, CPU access 1577 * rules concerning calls here are the same as for dma_unmap_single(). 1578 */ 1579 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1580 enum dma_data_direction dir, struct dma_attrs *attrs) 1581 { 1582 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); 1583 } 1584 1585 /** 1586 * arm_iommu_sync_sg_for_cpu 1587 * @dev: valid struct device pointer 1588 * @sg: list of buffers 1589 * @nents: number of buffers to map (returned from dma_map_sg) 1590 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1591 */ 1592 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1593 int nents, enum dma_data_direction dir) 1594 { 1595 struct scatterlist *s; 1596 int i; 1597 1598 for_each_sg(sg, s, nents, i) 1599 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1600 1601 } 1602 1603 /** 1604 * arm_iommu_sync_sg_for_device 1605 * @dev: valid struct device pointer 1606 * @sg: list of buffers 1607 * @nents: number of buffers to map (returned from dma_map_sg) 1608 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1609 */ 1610 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1611 int nents, enum dma_data_direction dir) 1612 { 1613 struct scatterlist *s; 1614 int i; 1615 1616 for_each_sg(sg, s, nents, i) 1617 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1618 } 1619 1620 1621 /** 1622 * arm_coherent_iommu_map_page 1623 * @dev: valid struct device pointer 1624 * @page: page that buffer resides in 1625 * @offset: offset into page for start of buffer 1626 * @size: size of buffer to map 1627 * @dir: DMA transfer direction 1628 * 1629 * Coherent IOMMU aware version of arm_dma_map_page() 1630 */ 1631 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, 1632 unsigned long offset, size_t size, enum dma_data_direction dir, 1633 struct dma_attrs *attrs) 1634 { 1635 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1636 dma_addr_t dma_addr; 1637 int ret, len = PAGE_ALIGN(size + offset); 1638 1639 dma_addr = __alloc_iova(mapping, len); 1640 if (dma_addr == DMA_ERROR_CODE) 1641 return dma_addr; 1642 1643 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0); 1644 if (ret < 0) 1645 goto fail; 1646 1647 return dma_addr + offset; 1648 fail: 1649 __free_iova(mapping, dma_addr, len); 1650 return DMA_ERROR_CODE; 1651 } 1652 1653 /** 1654 * arm_iommu_map_page 1655 * @dev: valid struct device pointer 1656 * @page: page that buffer resides in 1657 * @offset: offset into page for start of buffer 1658 * @size: size of buffer to map 1659 * @dir: DMA transfer direction 1660 * 1661 * IOMMU aware version of arm_dma_map_page() 1662 */ 1663 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1664 unsigned long offset, size_t size, enum dma_data_direction dir, 1665 struct dma_attrs *attrs) 1666 { 1667 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1668 __dma_page_cpu_to_dev(page, offset, size, dir); 1669 1670 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); 1671 } 1672 1673 /** 1674 * arm_coherent_iommu_unmap_page 1675 * @dev: valid struct device pointer 1676 * @handle: DMA address of buffer 1677 * @size: size of buffer (same as passed to dma_map_page) 1678 * @dir: DMA transfer direction (same as passed to dma_map_page) 1679 * 1680 * Coherent IOMMU aware version of arm_dma_unmap_page() 1681 */ 1682 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1683 size_t size, enum dma_data_direction dir, 1684 struct dma_attrs *attrs) 1685 { 1686 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1687 dma_addr_t iova = handle & PAGE_MASK; 1688 int offset = handle & ~PAGE_MASK; 1689 int len = PAGE_ALIGN(size + offset); 1690 1691 if (!iova) 1692 return; 1693 1694 iommu_unmap(mapping->domain, iova, len); 1695 __free_iova(mapping, iova, len); 1696 } 1697 1698 /** 1699 * arm_iommu_unmap_page 1700 * @dev: valid struct device pointer 1701 * @handle: DMA address of buffer 1702 * @size: size of buffer (same as passed to dma_map_page) 1703 * @dir: DMA transfer direction (same as passed to dma_map_page) 1704 * 1705 * IOMMU aware version of arm_dma_unmap_page() 1706 */ 1707 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1708 size_t size, enum dma_data_direction dir, 1709 struct dma_attrs *attrs) 1710 { 1711 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1712 dma_addr_t iova = handle & PAGE_MASK; 1713 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1714 int offset = handle & ~PAGE_MASK; 1715 int len = PAGE_ALIGN(size + offset); 1716 1717 if (!iova) 1718 return; 1719 1720 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1721 __dma_page_dev_to_cpu(page, offset, size, dir); 1722 1723 iommu_unmap(mapping->domain, iova, len); 1724 __free_iova(mapping, iova, len); 1725 } 1726 1727 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1728 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1729 { 1730 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1731 dma_addr_t iova = handle & PAGE_MASK; 1732 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1733 unsigned int offset = handle & ~PAGE_MASK; 1734 1735 if (!iova) 1736 return; 1737 1738 __dma_page_dev_to_cpu(page, offset, size, dir); 1739 } 1740 1741 static void arm_iommu_sync_single_for_device(struct device *dev, 1742 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1743 { 1744 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1745 dma_addr_t iova = handle & PAGE_MASK; 1746 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1747 unsigned int offset = handle & ~PAGE_MASK; 1748 1749 if (!iova) 1750 return; 1751 1752 __dma_page_cpu_to_dev(page, offset, size, dir); 1753 } 1754 1755 struct dma_map_ops iommu_ops = { 1756 .alloc = arm_iommu_alloc_attrs, 1757 .free = arm_iommu_free_attrs, 1758 .mmap = arm_iommu_mmap_attrs, 1759 .get_sgtable = arm_iommu_get_sgtable, 1760 1761 .map_page = arm_iommu_map_page, 1762 .unmap_page = arm_iommu_unmap_page, 1763 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 1764 .sync_single_for_device = arm_iommu_sync_single_for_device, 1765 1766 .map_sg = arm_iommu_map_sg, 1767 .unmap_sg = arm_iommu_unmap_sg, 1768 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 1769 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 1770 1771 .set_dma_mask = arm_dma_set_mask, 1772 }; 1773 1774 struct dma_map_ops iommu_coherent_ops = { 1775 .alloc = arm_iommu_alloc_attrs, 1776 .free = arm_iommu_free_attrs, 1777 .mmap = arm_iommu_mmap_attrs, 1778 .get_sgtable = arm_iommu_get_sgtable, 1779 1780 .map_page = arm_coherent_iommu_map_page, 1781 .unmap_page = arm_coherent_iommu_unmap_page, 1782 1783 .map_sg = arm_coherent_iommu_map_sg, 1784 .unmap_sg = arm_coherent_iommu_unmap_sg, 1785 1786 .set_dma_mask = arm_dma_set_mask, 1787 }; 1788 1789 /** 1790 * arm_iommu_create_mapping 1791 * @bus: pointer to the bus holding the client device (for IOMMU calls) 1792 * @base: start address of the valid IO address space 1793 * @size: size of the valid IO address space 1794 * @order: accuracy of the IO addresses allocations 1795 * 1796 * Creates a mapping structure which holds information about used/unused 1797 * IO address ranges, which is required to perform memory allocation and 1798 * mapping with IOMMU aware functions. 1799 * 1800 * The client device need to be attached to the mapping with 1801 * arm_iommu_attach_device function. 1802 */ 1803 struct dma_iommu_mapping * 1804 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size, 1805 int order) 1806 { 1807 unsigned int count = size >> (PAGE_SHIFT + order); 1808 unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long); 1809 struct dma_iommu_mapping *mapping; 1810 int err = -ENOMEM; 1811 1812 if (!count) 1813 return ERR_PTR(-EINVAL); 1814 1815 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 1816 if (!mapping) 1817 goto err; 1818 1819 mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL); 1820 if (!mapping->bitmap) 1821 goto err2; 1822 1823 mapping->base = base; 1824 mapping->bits = BITS_PER_BYTE * bitmap_size; 1825 mapping->order = order; 1826 spin_lock_init(&mapping->lock); 1827 1828 mapping->domain = iommu_domain_alloc(bus); 1829 if (!mapping->domain) 1830 goto err3; 1831 1832 kref_init(&mapping->kref); 1833 return mapping; 1834 err3: 1835 kfree(mapping->bitmap); 1836 err2: 1837 kfree(mapping); 1838 err: 1839 return ERR_PTR(err); 1840 } 1841 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 1842 1843 static void release_iommu_mapping(struct kref *kref) 1844 { 1845 struct dma_iommu_mapping *mapping = 1846 container_of(kref, struct dma_iommu_mapping, kref); 1847 1848 iommu_domain_free(mapping->domain); 1849 kfree(mapping->bitmap); 1850 kfree(mapping); 1851 } 1852 1853 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 1854 { 1855 if (mapping) 1856 kref_put(&mapping->kref, release_iommu_mapping); 1857 } 1858 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 1859 1860 /** 1861 * arm_iommu_attach_device 1862 * @dev: valid struct device pointer 1863 * @mapping: io address space mapping structure (returned from 1864 * arm_iommu_create_mapping) 1865 * 1866 * Attaches specified io address space mapping to the provided device, 1867 * this replaces the dma operations (dma_map_ops pointer) with the 1868 * IOMMU aware version. More than one client might be attached to 1869 * the same io address space mapping. 1870 */ 1871 int arm_iommu_attach_device(struct device *dev, 1872 struct dma_iommu_mapping *mapping) 1873 { 1874 int err; 1875 1876 err = iommu_attach_device(mapping->domain, dev); 1877 if (err) 1878 return err; 1879 1880 kref_get(&mapping->kref); 1881 dev->archdata.mapping = mapping; 1882 set_dma_ops(dev, &iommu_ops); 1883 1884 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 1885 return 0; 1886 } 1887 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 1888 1889 /** 1890 * arm_iommu_detach_device 1891 * @dev: valid struct device pointer 1892 * 1893 * Detaches the provided device from a previously attached map. 1894 * This voids the dma operations (dma_map_ops pointer) 1895 */ 1896 void arm_iommu_detach_device(struct device *dev) 1897 { 1898 struct dma_iommu_mapping *mapping; 1899 1900 mapping = to_dma_iommu_mapping(dev); 1901 if (!mapping) { 1902 dev_warn(dev, "Not attached\n"); 1903 return; 1904 } 1905 1906 iommu_detach_device(mapping->domain, dev); 1907 kref_put(&mapping->kref, release_iommu_mapping); 1908 mapping = NULL; 1909 set_dma_ops(dev, NULL); 1910 1911 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 1912 } 1913 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 1914 1915 #endif 1916