1 /* 2 * linux/arch/arm/mm/dma-mapping.c 3 * 4 * Copyright (C) 2000-2004 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * DMA uncached mapping support. 11 */ 12 #include <linux/module.h> 13 #include <linux/mm.h> 14 #include <linux/gfp.h> 15 #include <linux/errno.h> 16 #include <linux/list.h> 17 #include <linux/init.h> 18 #include <linux/device.h> 19 #include <linux/dma-mapping.h> 20 #include <linux/dma-contiguous.h> 21 #include <linux/highmem.h> 22 #include <linux/memblock.h> 23 #include <linux/slab.h> 24 #include <linux/iommu.h> 25 #include <linux/io.h> 26 #include <linux/vmalloc.h> 27 #include <linux/sizes.h> 28 29 #include <asm/memory.h> 30 #include <asm/highmem.h> 31 #include <asm/cacheflush.h> 32 #include <asm/tlbflush.h> 33 #include <asm/mach/arch.h> 34 #include <asm/dma-iommu.h> 35 #include <asm/mach/map.h> 36 #include <asm/system_info.h> 37 #include <asm/dma-contiguous.h> 38 39 #include "mm.h" 40 41 /* 42 * The DMA API is built upon the notion of "buffer ownership". A buffer 43 * is either exclusively owned by the CPU (and therefore may be accessed 44 * by it) or exclusively owned by the DMA device. These helper functions 45 * represent the transitions between these two ownership states. 46 * 47 * Note, however, that on later ARMs, this notion does not work due to 48 * speculative prefetches. We model our approach on the assumption that 49 * the CPU does do speculative prefetches, which means we clean caches 50 * before transfers and delay cache invalidation until transfer completion. 51 * 52 */ 53 static void __dma_page_cpu_to_dev(struct page *, unsigned long, 54 size_t, enum dma_data_direction); 55 static void __dma_page_dev_to_cpu(struct page *, unsigned long, 56 size_t, enum dma_data_direction); 57 58 /** 59 * arm_dma_map_page - map a portion of a page for streaming DMA 60 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 61 * @page: page that buffer resides in 62 * @offset: offset into page for start of buffer 63 * @size: size of buffer to map 64 * @dir: DMA transfer direction 65 * 66 * Ensure that any data held in the cache is appropriately discarded 67 * or written back. 68 * 69 * The device owns this memory once this call has completed. The CPU 70 * can regain ownership by calling dma_unmap_page(). 71 */ 72 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, 73 unsigned long offset, size_t size, enum dma_data_direction dir, 74 struct dma_attrs *attrs) 75 { 76 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 77 __dma_page_cpu_to_dev(page, offset, size, dir); 78 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 79 } 80 81 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, 82 unsigned long offset, size_t size, enum dma_data_direction dir, 83 struct dma_attrs *attrs) 84 { 85 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 86 } 87 88 /** 89 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() 90 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 91 * @handle: DMA address of buffer 92 * @size: size of buffer (same as passed to dma_map_page) 93 * @dir: DMA transfer direction (same as passed to dma_map_page) 94 * 95 * Unmap a page streaming mode DMA translation. The handle and size 96 * must match what was provided in the previous dma_map_page() call. 97 * All other usages are undefined. 98 * 99 * After this call, reads by the CPU to the buffer are guaranteed to see 100 * whatever the device wrote there. 101 */ 102 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, 103 size_t size, enum dma_data_direction dir, 104 struct dma_attrs *attrs) 105 { 106 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 107 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), 108 handle & ~PAGE_MASK, size, dir); 109 } 110 111 static void arm_dma_sync_single_for_cpu(struct device *dev, 112 dma_addr_t handle, size_t size, enum dma_data_direction dir) 113 { 114 unsigned int offset = handle & (PAGE_SIZE - 1); 115 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 116 __dma_page_dev_to_cpu(page, offset, size, dir); 117 } 118 119 static void arm_dma_sync_single_for_device(struct device *dev, 120 dma_addr_t handle, size_t size, enum dma_data_direction dir) 121 { 122 unsigned int offset = handle & (PAGE_SIZE - 1); 123 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 124 __dma_page_cpu_to_dev(page, offset, size, dir); 125 } 126 127 struct dma_map_ops arm_dma_ops = { 128 .alloc = arm_dma_alloc, 129 .free = arm_dma_free, 130 .mmap = arm_dma_mmap, 131 .get_sgtable = arm_dma_get_sgtable, 132 .map_page = arm_dma_map_page, 133 .unmap_page = arm_dma_unmap_page, 134 .map_sg = arm_dma_map_sg, 135 .unmap_sg = arm_dma_unmap_sg, 136 .sync_single_for_cpu = arm_dma_sync_single_for_cpu, 137 .sync_single_for_device = arm_dma_sync_single_for_device, 138 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, 139 .sync_sg_for_device = arm_dma_sync_sg_for_device, 140 .set_dma_mask = arm_dma_set_mask, 141 }; 142 EXPORT_SYMBOL(arm_dma_ops); 143 144 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 145 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); 146 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 147 dma_addr_t handle, struct dma_attrs *attrs); 148 149 struct dma_map_ops arm_coherent_dma_ops = { 150 .alloc = arm_coherent_dma_alloc, 151 .free = arm_coherent_dma_free, 152 .mmap = arm_dma_mmap, 153 .get_sgtable = arm_dma_get_sgtable, 154 .map_page = arm_coherent_dma_map_page, 155 .map_sg = arm_dma_map_sg, 156 .set_dma_mask = arm_dma_set_mask, 157 }; 158 EXPORT_SYMBOL(arm_coherent_dma_ops); 159 160 static u64 get_coherent_dma_mask(struct device *dev) 161 { 162 u64 mask = (u64)arm_dma_limit; 163 164 if (dev) { 165 mask = dev->coherent_dma_mask; 166 167 /* 168 * Sanity check the DMA mask - it must be non-zero, and 169 * must be able to be satisfied by a DMA allocation. 170 */ 171 if (mask == 0) { 172 dev_warn(dev, "coherent DMA mask is unset\n"); 173 return 0; 174 } 175 176 if ((~mask) & (u64)arm_dma_limit) { 177 dev_warn(dev, "coherent DMA mask %#llx is smaller " 178 "than system GFP_DMA mask %#llx\n", 179 mask, (u64)arm_dma_limit); 180 return 0; 181 } 182 } 183 184 return mask; 185 } 186 187 static void __dma_clear_buffer(struct page *page, size_t size) 188 { 189 void *ptr; 190 /* 191 * Ensure that the allocated pages are zeroed, and that any data 192 * lurking in the kernel direct-mapped region is invalidated. 193 */ 194 ptr = page_address(page); 195 if (ptr) { 196 memset(ptr, 0, size); 197 dmac_flush_range(ptr, ptr + size); 198 outer_flush_range(__pa(ptr), __pa(ptr) + size); 199 } 200 } 201 202 /* 203 * Allocate a DMA buffer for 'dev' of size 'size' using the 204 * specified gfp mask. Note that 'size' must be page aligned. 205 */ 206 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) 207 { 208 unsigned long order = get_order(size); 209 struct page *page, *p, *e; 210 211 page = alloc_pages(gfp, order); 212 if (!page) 213 return NULL; 214 215 /* 216 * Now split the huge page and free the excess pages 217 */ 218 split_page(page, order); 219 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 220 __free_page(p); 221 222 __dma_clear_buffer(page, size); 223 224 return page; 225 } 226 227 /* 228 * Free a DMA buffer. 'size' must be page aligned. 229 */ 230 static void __dma_free_buffer(struct page *page, size_t size) 231 { 232 struct page *e = page + (size >> PAGE_SHIFT); 233 234 while (page < e) { 235 __free_page(page); 236 page++; 237 } 238 } 239 240 #ifdef CONFIG_MMU 241 #ifdef CONFIG_HUGETLB_PAGE 242 #error ARM Coherent DMA allocator does not (yet) support huge TLB 243 #endif 244 245 static void *__alloc_from_contiguous(struct device *dev, size_t size, 246 pgprot_t prot, struct page **ret_page); 247 248 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 249 pgprot_t prot, struct page **ret_page, 250 const void *caller); 251 252 static void * 253 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 254 const void *caller) 255 { 256 struct vm_struct *area; 257 unsigned long addr; 258 259 /* 260 * DMA allocation can be mapped to user space, so lets 261 * set VM_USERMAP flags too. 262 */ 263 area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP, 264 caller); 265 if (!area) 266 return NULL; 267 addr = (unsigned long)area->addr; 268 area->phys_addr = __pfn_to_phys(page_to_pfn(page)); 269 270 if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) { 271 vunmap((void *)addr); 272 return NULL; 273 } 274 return (void *)addr; 275 } 276 277 static void __dma_free_remap(void *cpu_addr, size_t size) 278 { 279 unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP; 280 struct vm_struct *area = find_vm_area(cpu_addr); 281 if (!area || (area->flags & flags) != flags) { 282 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 283 return; 284 } 285 unmap_kernel_range((unsigned long)cpu_addr, size); 286 vunmap(cpu_addr); 287 } 288 289 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 290 291 struct dma_pool { 292 size_t size; 293 spinlock_t lock; 294 unsigned long *bitmap; 295 unsigned long nr_pages; 296 void *vaddr; 297 struct page **pages; 298 }; 299 300 static struct dma_pool atomic_pool = { 301 .size = DEFAULT_DMA_COHERENT_POOL_SIZE, 302 }; 303 304 static int __init early_coherent_pool(char *p) 305 { 306 atomic_pool.size = memparse(p, &p); 307 return 0; 308 } 309 early_param("coherent_pool", early_coherent_pool); 310 311 void __init init_dma_coherent_pool_size(unsigned long size) 312 { 313 /* 314 * Catch any attempt to set the pool size too late. 315 */ 316 BUG_ON(atomic_pool.vaddr); 317 318 /* 319 * Set architecture specific coherent pool size only if 320 * it has not been changed by kernel command line parameter. 321 */ 322 if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE) 323 atomic_pool.size = size; 324 } 325 326 /* 327 * Initialise the coherent pool for atomic allocations. 328 */ 329 static int __init atomic_pool_init(void) 330 { 331 struct dma_pool *pool = &atomic_pool; 332 pgprot_t prot = pgprot_dmacoherent(pgprot_kernel); 333 unsigned long nr_pages = pool->size >> PAGE_SHIFT; 334 unsigned long *bitmap; 335 struct page *page; 336 struct page **pages; 337 void *ptr; 338 int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long); 339 340 bitmap = kzalloc(bitmap_size, GFP_KERNEL); 341 if (!bitmap) 342 goto no_bitmap; 343 344 pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL); 345 if (!pages) 346 goto no_pages; 347 348 if (IS_ENABLED(CONFIG_CMA)) 349 ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page); 350 else 351 ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot, 352 &page, NULL); 353 if (ptr) { 354 int i; 355 356 for (i = 0; i < nr_pages; i++) 357 pages[i] = page + i; 358 359 spin_lock_init(&pool->lock); 360 pool->vaddr = ptr; 361 pool->pages = pages; 362 pool->bitmap = bitmap; 363 pool->nr_pages = nr_pages; 364 pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n", 365 (unsigned)pool->size / 1024); 366 return 0; 367 } 368 369 kfree(pages); 370 no_pages: 371 kfree(bitmap); 372 no_bitmap: 373 pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", 374 (unsigned)pool->size / 1024); 375 return -ENOMEM; 376 } 377 /* 378 * CMA is activated by core_initcall, so we must be called after it. 379 */ 380 postcore_initcall(atomic_pool_init); 381 382 struct dma_contig_early_reserve { 383 phys_addr_t base; 384 unsigned long size; 385 }; 386 387 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 388 389 static int dma_mmu_remap_num __initdata; 390 391 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 392 { 393 dma_mmu_remap[dma_mmu_remap_num].base = base; 394 dma_mmu_remap[dma_mmu_remap_num].size = size; 395 dma_mmu_remap_num++; 396 } 397 398 void __init dma_contiguous_remap(void) 399 { 400 int i; 401 for (i = 0; i < dma_mmu_remap_num; i++) { 402 phys_addr_t start = dma_mmu_remap[i].base; 403 phys_addr_t end = start + dma_mmu_remap[i].size; 404 struct map_desc map; 405 unsigned long addr; 406 407 if (end > arm_lowmem_limit) 408 end = arm_lowmem_limit; 409 if (start >= end) 410 continue; 411 412 map.pfn = __phys_to_pfn(start); 413 map.virtual = __phys_to_virt(start); 414 map.length = end - start; 415 map.type = MT_MEMORY_DMA_READY; 416 417 /* 418 * Clear previous low-memory mapping 419 */ 420 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 421 addr += PMD_SIZE) 422 pmd_clear(pmd_off_k(addr)); 423 424 iotable_init(&map, 1); 425 } 426 } 427 428 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, 429 void *data) 430 { 431 struct page *page = virt_to_page(addr); 432 pgprot_t prot = *(pgprot_t *)data; 433 434 set_pte_ext(pte, mk_pte(page, prot), 0); 435 return 0; 436 } 437 438 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 439 { 440 unsigned long start = (unsigned long) page_address(page); 441 unsigned end = start + size; 442 443 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 444 dsb(); 445 flush_tlb_kernel_range(start, end); 446 } 447 448 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 449 pgprot_t prot, struct page **ret_page, 450 const void *caller) 451 { 452 struct page *page; 453 void *ptr; 454 page = __dma_alloc_buffer(dev, size, gfp); 455 if (!page) 456 return NULL; 457 458 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 459 if (!ptr) { 460 __dma_free_buffer(page, size); 461 return NULL; 462 } 463 464 *ret_page = page; 465 return ptr; 466 } 467 468 static void *__alloc_from_pool(size_t size, struct page **ret_page) 469 { 470 struct dma_pool *pool = &atomic_pool; 471 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 472 unsigned int pageno; 473 unsigned long flags; 474 void *ptr = NULL; 475 unsigned long align_mask; 476 477 if (!pool->vaddr) { 478 WARN(1, "coherent pool not initialised!\n"); 479 return NULL; 480 } 481 482 /* 483 * Align the region allocation - allocations from pool are rather 484 * small, so align them to their order in pages, minimum is a page 485 * size. This helps reduce fragmentation of the DMA space. 486 */ 487 align_mask = (1 << get_order(size)) - 1; 488 489 spin_lock_irqsave(&pool->lock, flags); 490 pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages, 491 0, count, align_mask); 492 if (pageno < pool->nr_pages) { 493 bitmap_set(pool->bitmap, pageno, count); 494 ptr = pool->vaddr + PAGE_SIZE * pageno; 495 *ret_page = pool->pages[pageno]; 496 } else { 497 pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n" 498 "Please increase it with coherent_pool= kernel parameter!\n", 499 (unsigned)pool->size / 1024); 500 } 501 spin_unlock_irqrestore(&pool->lock, flags); 502 503 return ptr; 504 } 505 506 static bool __in_atomic_pool(void *start, size_t size) 507 { 508 struct dma_pool *pool = &atomic_pool; 509 void *end = start + size; 510 void *pool_start = pool->vaddr; 511 void *pool_end = pool->vaddr + pool->size; 512 513 if (start < pool_start || start >= pool_end) 514 return false; 515 516 if (end <= pool_end) 517 return true; 518 519 WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n", 520 start, end - 1, pool_start, pool_end - 1); 521 522 return false; 523 } 524 525 static int __free_from_pool(void *start, size_t size) 526 { 527 struct dma_pool *pool = &atomic_pool; 528 unsigned long pageno, count; 529 unsigned long flags; 530 531 if (!__in_atomic_pool(start, size)) 532 return 0; 533 534 pageno = (start - pool->vaddr) >> PAGE_SHIFT; 535 count = size >> PAGE_SHIFT; 536 537 spin_lock_irqsave(&pool->lock, flags); 538 bitmap_clear(pool->bitmap, pageno, count); 539 spin_unlock_irqrestore(&pool->lock, flags); 540 541 return 1; 542 } 543 544 static void *__alloc_from_contiguous(struct device *dev, size_t size, 545 pgprot_t prot, struct page **ret_page) 546 { 547 unsigned long order = get_order(size); 548 size_t count = size >> PAGE_SHIFT; 549 struct page *page; 550 551 page = dma_alloc_from_contiguous(dev, count, order); 552 if (!page) 553 return NULL; 554 555 __dma_clear_buffer(page, size); 556 __dma_remap(page, size, prot); 557 558 *ret_page = page; 559 return page_address(page); 560 } 561 562 static void __free_from_contiguous(struct device *dev, struct page *page, 563 size_t size) 564 { 565 __dma_remap(page, size, pgprot_kernel); 566 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 567 } 568 569 static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) 570 { 571 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? 572 pgprot_writecombine(prot) : 573 pgprot_dmacoherent(prot); 574 return prot; 575 } 576 577 #define nommu() 0 578 579 #else /* !CONFIG_MMU */ 580 581 #define nommu() 1 582 583 #define __get_dma_pgprot(attrs, prot) __pgprot(0) 584 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL 585 #define __alloc_from_pool(size, ret_page) NULL 586 #define __alloc_from_contiguous(dev, size, prot, ret) NULL 587 #define __free_from_pool(cpu_addr, size) 0 588 #define __free_from_contiguous(dev, page, size) do { } while (0) 589 #define __dma_free_remap(cpu_addr, size) do { } while (0) 590 591 #endif /* CONFIG_MMU */ 592 593 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 594 struct page **ret_page) 595 { 596 struct page *page; 597 page = __dma_alloc_buffer(dev, size, gfp); 598 if (!page) 599 return NULL; 600 601 *ret_page = page; 602 return page_address(page); 603 } 604 605 606 607 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 608 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller) 609 { 610 u64 mask = get_coherent_dma_mask(dev); 611 struct page *page = NULL; 612 void *addr; 613 614 #ifdef CONFIG_DMA_API_DEBUG 615 u64 limit = (mask + 1) & ~mask; 616 if (limit && size >= limit) { 617 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 618 size, mask); 619 return NULL; 620 } 621 #endif 622 623 if (!mask) 624 return NULL; 625 626 if (mask < 0xffffffffULL) 627 gfp |= GFP_DMA; 628 629 /* 630 * Following is a work-around (a.k.a. hack) to prevent pages 631 * with __GFP_COMP being passed to split_page() which cannot 632 * handle them. The real problem is that this flag probably 633 * should be 0 on ARM as it is not supported on this 634 * platform; see CONFIG_HUGETLBFS. 635 */ 636 gfp &= ~(__GFP_COMP); 637 638 *handle = DMA_ERROR_CODE; 639 size = PAGE_ALIGN(size); 640 641 if (is_coherent || nommu()) 642 addr = __alloc_simple_buffer(dev, size, gfp, &page); 643 else if (!(gfp & __GFP_WAIT)) 644 addr = __alloc_from_pool(size, &page); 645 else if (!IS_ENABLED(CONFIG_CMA)) 646 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); 647 else 648 addr = __alloc_from_contiguous(dev, size, prot, &page); 649 650 if (addr) 651 *handle = pfn_to_dma(dev, page_to_pfn(page)); 652 653 return addr; 654 } 655 656 /* 657 * Allocate DMA-coherent memory space and return both the kernel remapped 658 * virtual and bus address for that space. 659 */ 660 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 661 gfp_t gfp, struct dma_attrs *attrs) 662 { 663 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); 664 void *memory; 665 666 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 667 return memory; 668 669 return __dma_alloc(dev, size, handle, gfp, prot, false, 670 __builtin_return_address(0)); 671 } 672 673 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 674 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 675 { 676 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); 677 void *memory; 678 679 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 680 return memory; 681 682 return __dma_alloc(dev, size, handle, gfp, prot, true, 683 __builtin_return_address(0)); 684 } 685 686 /* 687 * Create userspace mapping for the DMA-coherent memory. 688 */ 689 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 690 void *cpu_addr, dma_addr_t dma_addr, size_t size, 691 struct dma_attrs *attrs) 692 { 693 int ret = -ENXIO; 694 #ifdef CONFIG_MMU 695 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 696 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 697 unsigned long pfn = dma_to_pfn(dev, dma_addr); 698 unsigned long off = vma->vm_pgoff; 699 700 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 701 702 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 703 return ret; 704 705 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 706 ret = remap_pfn_range(vma, vma->vm_start, 707 pfn + off, 708 vma->vm_end - vma->vm_start, 709 vma->vm_page_prot); 710 } 711 #endif /* CONFIG_MMU */ 712 713 return ret; 714 } 715 716 /* 717 * Free a buffer as defined by the above mapping. 718 */ 719 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 720 dma_addr_t handle, struct dma_attrs *attrs, 721 bool is_coherent) 722 { 723 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 724 725 if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) 726 return; 727 728 size = PAGE_ALIGN(size); 729 730 if (is_coherent || nommu()) { 731 __dma_free_buffer(page, size); 732 } else if (__free_from_pool(cpu_addr, size)) { 733 return; 734 } else if (!IS_ENABLED(CONFIG_CMA)) { 735 __dma_free_remap(cpu_addr, size); 736 __dma_free_buffer(page, size); 737 } else { 738 /* 739 * Non-atomic allocations cannot be freed with IRQs disabled 740 */ 741 WARN_ON(irqs_disabled()); 742 __free_from_contiguous(dev, page, size); 743 } 744 } 745 746 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 747 dma_addr_t handle, struct dma_attrs *attrs) 748 { 749 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); 750 } 751 752 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 753 dma_addr_t handle, struct dma_attrs *attrs) 754 { 755 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); 756 } 757 758 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 759 void *cpu_addr, dma_addr_t handle, size_t size, 760 struct dma_attrs *attrs) 761 { 762 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 763 int ret; 764 765 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 766 if (unlikely(ret)) 767 return ret; 768 769 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 770 return 0; 771 } 772 773 static void dma_cache_maint_page(struct page *page, unsigned long offset, 774 size_t size, enum dma_data_direction dir, 775 void (*op)(const void *, size_t, int)) 776 { 777 unsigned long pfn; 778 size_t left = size; 779 780 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 781 offset %= PAGE_SIZE; 782 783 /* 784 * A single sg entry may refer to multiple physically contiguous 785 * pages. But we still need to process highmem pages individually. 786 * If highmem is not configured then the bulk of this loop gets 787 * optimized out. 788 */ 789 do { 790 size_t len = left; 791 void *vaddr; 792 793 page = pfn_to_page(pfn); 794 795 if (PageHighMem(page)) { 796 if (len + offset > PAGE_SIZE) 797 len = PAGE_SIZE - offset; 798 vaddr = kmap_high_get(page); 799 if (vaddr) { 800 vaddr += offset; 801 op(vaddr, len, dir); 802 kunmap_high(page); 803 } else if (cache_is_vipt()) { 804 /* unmapped pages might still be cached */ 805 vaddr = kmap_atomic(page); 806 op(vaddr + offset, len, dir); 807 kunmap_atomic(vaddr); 808 } 809 } else { 810 vaddr = page_address(page) + offset; 811 op(vaddr, len, dir); 812 } 813 offset = 0; 814 pfn++; 815 left -= len; 816 } while (left); 817 } 818 819 /* 820 * Make an area consistent for devices. 821 * Note: Drivers should NOT use this function directly, as it will break 822 * platforms with CONFIG_DMABOUNCE. 823 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 824 */ 825 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 826 size_t size, enum dma_data_direction dir) 827 { 828 unsigned long paddr; 829 830 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 831 832 paddr = page_to_phys(page) + off; 833 if (dir == DMA_FROM_DEVICE) { 834 outer_inv_range(paddr, paddr + size); 835 } else { 836 outer_clean_range(paddr, paddr + size); 837 } 838 /* FIXME: non-speculating: flush on bidirectional mappings? */ 839 } 840 841 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 842 size_t size, enum dma_data_direction dir) 843 { 844 unsigned long paddr = page_to_phys(page) + off; 845 846 /* FIXME: non-speculating: not required */ 847 /* don't bother invalidating if DMA to device */ 848 if (dir != DMA_TO_DEVICE) 849 outer_inv_range(paddr, paddr + size); 850 851 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 852 853 /* 854 * Mark the D-cache clean for this page to avoid extra flushing. 855 */ 856 if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) 857 set_bit(PG_dcache_clean, &page->flags); 858 } 859 860 /** 861 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA 862 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 863 * @sg: list of buffers 864 * @nents: number of buffers to map 865 * @dir: DMA transfer direction 866 * 867 * Map a set of buffers described by scatterlist in streaming mode for DMA. 868 * This is the scatter-gather version of the dma_map_single interface. 869 * Here the scatter gather list elements are each tagged with the 870 * appropriate dma address and length. They are obtained via 871 * sg_dma_{address,length}. 872 * 873 * Device ownership issues as mentioned for dma_map_single are the same 874 * here. 875 */ 876 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 877 enum dma_data_direction dir, struct dma_attrs *attrs) 878 { 879 struct dma_map_ops *ops = get_dma_ops(dev); 880 struct scatterlist *s; 881 int i, j; 882 883 for_each_sg(sg, s, nents, i) { 884 #ifdef CONFIG_NEED_SG_DMA_LENGTH 885 s->dma_length = s->length; 886 #endif 887 s->dma_address = ops->map_page(dev, sg_page(s), s->offset, 888 s->length, dir, attrs); 889 if (dma_mapping_error(dev, s->dma_address)) 890 goto bad_mapping; 891 } 892 return nents; 893 894 bad_mapping: 895 for_each_sg(sg, s, i, j) 896 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 897 return 0; 898 } 899 900 /** 901 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 902 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 903 * @sg: list of buffers 904 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 905 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 906 * 907 * Unmap a set of streaming mode DMA translations. Again, CPU access 908 * rules concerning calls here are the same as for dma_unmap_single(). 909 */ 910 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 911 enum dma_data_direction dir, struct dma_attrs *attrs) 912 { 913 struct dma_map_ops *ops = get_dma_ops(dev); 914 struct scatterlist *s; 915 916 int i; 917 918 for_each_sg(sg, s, nents, i) 919 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 920 } 921 922 /** 923 * arm_dma_sync_sg_for_cpu 924 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 925 * @sg: list of buffers 926 * @nents: number of buffers to map (returned from dma_map_sg) 927 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 928 */ 929 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 930 int nents, enum dma_data_direction dir) 931 { 932 struct dma_map_ops *ops = get_dma_ops(dev); 933 struct scatterlist *s; 934 int i; 935 936 for_each_sg(sg, s, nents, i) 937 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, 938 dir); 939 } 940 941 /** 942 * arm_dma_sync_sg_for_device 943 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 944 * @sg: list of buffers 945 * @nents: number of buffers to map (returned from dma_map_sg) 946 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 947 */ 948 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 949 int nents, enum dma_data_direction dir) 950 { 951 struct dma_map_ops *ops = get_dma_ops(dev); 952 struct scatterlist *s; 953 int i; 954 955 for_each_sg(sg, s, nents, i) 956 ops->sync_single_for_device(dev, sg_dma_address(s), s->length, 957 dir); 958 } 959 960 /* 961 * Return whether the given device DMA address mask can be supported 962 * properly. For example, if your device can only drive the low 24-bits 963 * during bus mastering, then you would pass 0x00ffffff as the mask 964 * to this function. 965 */ 966 int dma_supported(struct device *dev, u64 mask) 967 { 968 if (mask < (u64)arm_dma_limit) 969 return 0; 970 return 1; 971 } 972 EXPORT_SYMBOL(dma_supported); 973 974 int arm_dma_set_mask(struct device *dev, u64 dma_mask) 975 { 976 if (!dev->dma_mask || !dma_supported(dev, dma_mask)) 977 return -EIO; 978 979 *dev->dma_mask = dma_mask; 980 981 return 0; 982 } 983 984 #define PREALLOC_DMA_DEBUG_ENTRIES 4096 985 986 static int __init dma_debug_do_init(void) 987 { 988 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 989 return 0; 990 } 991 fs_initcall(dma_debug_do_init); 992 993 #ifdef CONFIG_ARM_DMA_USE_IOMMU 994 995 /* IOMMU */ 996 997 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 998 size_t size) 999 { 1000 unsigned int order = get_order(size); 1001 unsigned int align = 0; 1002 unsigned int count, start; 1003 unsigned long flags; 1004 1005 count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) + 1006 (1 << mapping->order) - 1) >> mapping->order; 1007 1008 if (order > mapping->order) 1009 align = (1 << (order - mapping->order)) - 1; 1010 1011 spin_lock_irqsave(&mapping->lock, flags); 1012 start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0, 1013 count, align); 1014 if (start > mapping->bits) { 1015 spin_unlock_irqrestore(&mapping->lock, flags); 1016 return DMA_ERROR_CODE; 1017 } 1018 1019 bitmap_set(mapping->bitmap, start, count); 1020 spin_unlock_irqrestore(&mapping->lock, flags); 1021 1022 return mapping->base + (start << (mapping->order + PAGE_SHIFT)); 1023 } 1024 1025 static inline void __free_iova(struct dma_iommu_mapping *mapping, 1026 dma_addr_t addr, size_t size) 1027 { 1028 unsigned int start = (addr - mapping->base) >> 1029 (mapping->order + PAGE_SHIFT); 1030 unsigned int count = ((size >> PAGE_SHIFT) + 1031 (1 << mapping->order) - 1) >> mapping->order; 1032 unsigned long flags; 1033 1034 spin_lock_irqsave(&mapping->lock, flags); 1035 bitmap_clear(mapping->bitmap, start, count); 1036 spin_unlock_irqrestore(&mapping->lock, flags); 1037 } 1038 1039 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 1040 gfp_t gfp, struct dma_attrs *attrs) 1041 { 1042 struct page **pages; 1043 int count = size >> PAGE_SHIFT; 1044 int array_size = count * sizeof(struct page *); 1045 int i = 0; 1046 1047 if (array_size <= PAGE_SIZE) 1048 pages = kzalloc(array_size, gfp); 1049 else 1050 pages = vzalloc(array_size); 1051 if (!pages) 1052 return NULL; 1053 1054 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) 1055 { 1056 unsigned long order = get_order(size); 1057 struct page *page; 1058 1059 page = dma_alloc_from_contiguous(dev, count, order); 1060 if (!page) 1061 goto error; 1062 1063 __dma_clear_buffer(page, size); 1064 1065 for (i = 0; i < count; i++) 1066 pages[i] = page + i; 1067 1068 return pages; 1069 } 1070 1071 while (count) { 1072 int j, order = __fls(count); 1073 1074 pages[i] = alloc_pages(gfp | __GFP_NOWARN, order); 1075 while (!pages[i] && order) 1076 pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order); 1077 if (!pages[i]) 1078 goto error; 1079 1080 if (order) { 1081 split_page(pages[i], order); 1082 j = 1 << order; 1083 while (--j) 1084 pages[i + j] = pages[i] + j; 1085 } 1086 1087 __dma_clear_buffer(pages[i], PAGE_SIZE << order); 1088 i += 1 << order; 1089 count -= 1 << order; 1090 } 1091 1092 return pages; 1093 error: 1094 while (i--) 1095 if (pages[i]) 1096 __free_pages(pages[i], 0); 1097 if (array_size <= PAGE_SIZE) 1098 kfree(pages); 1099 else 1100 vfree(pages); 1101 return NULL; 1102 } 1103 1104 static int __iommu_free_buffer(struct device *dev, struct page **pages, 1105 size_t size, struct dma_attrs *attrs) 1106 { 1107 int count = size >> PAGE_SHIFT; 1108 int array_size = count * sizeof(struct page *); 1109 int i; 1110 1111 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { 1112 dma_release_from_contiguous(dev, pages[0], count); 1113 } else { 1114 for (i = 0; i < count; i++) 1115 if (pages[i]) 1116 __free_pages(pages[i], 0); 1117 } 1118 1119 if (array_size <= PAGE_SIZE) 1120 kfree(pages); 1121 else 1122 vfree(pages); 1123 return 0; 1124 } 1125 1126 /* 1127 * Create a CPU mapping for a specified pages 1128 */ 1129 static void * 1130 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, 1131 const void *caller) 1132 { 1133 unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1134 struct vm_struct *area; 1135 unsigned long p; 1136 1137 area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP, 1138 caller); 1139 if (!area) 1140 return NULL; 1141 1142 area->pages = pages; 1143 area->nr_pages = nr_pages; 1144 p = (unsigned long)area->addr; 1145 1146 for (i = 0; i < nr_pages; i++) { 1147 phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i])); 1148 if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot)) 1149 goto err; 1150 p += PAGE_SIZE; 1151 } 1152 return area->addr; 1153 err: 1154 unmap_kernel_range((unsigned long)area->addr, size); 1155 vunmap(area->addr); 1156 return NULL; 1157 } 1158 1159 /* 1160 * Create a mapping in device IO address space for specified pages 1161 */ 1162 static dma_addr_t 1163 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1164 { 1165 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1166 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1167 dma_addr_t dma_addr, iova; 1168 int i, ret = DMA_ERROR_CODE; 1169 1170 dma_addr = __alloc_iova(mapping, size); 1171 if (dma_addr == DMA_ERROR_CODE) 1172 return dma_addr; 1173 1174 iova = dma_addr; 1175 for (i = 0; i < count; ) { 1176 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 1177 phys_addr_t phys = page_to_phys(pages[i]); 1178 unsigned int len, j; 1179 1180 for (j = i + 1; j < count; j++, next_pfn++) 1181 if (page_to_pfn(pages[j]) != next_pfn) 1182 break; 1183 1184 len = (j - i) << PAGE_SHIFT; 1185 ret = iommu_map(mapping->domain, iova, phys, len, 0); 1186 if (ret < 0) 1187 goto fail; 1188 iova += len; 1189 i = j; 1190 } 1191 return dma_addr; 1192 fail: 1193 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 1194 __free_iova(mapping, dma_addr, size); 1195 return DMA_ERROR_CODE; 1196 } 1197 1198 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1199 { 1200 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1201 1202 /* 1203 * add optional in-page offset from iova to size and align 1204 * result to page size 1205 */ 1206 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1207 iova &= PAGE_MASK; 1208 1209 iommu_unmap(mapping->domain, iova, size); 1210 __free_iova(mapping, iova, size); 1211 return 0; 1212 } 1213 1214 static struct page **__atomic_get_pages(void *addr) 1215 { 1216 struct dma_pool *pool = &atomic_pool; 1217 struct page **pages = pool->pages; 1218 int offs = (addr - pool->vaddr) >> PAGE_SHIFT; 1219 1220 return pages + offs; 1221 } 1222 1223 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) 1224 { 1225 struct vm_struct *area; 1226 1227 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1228 return __atomic_get_pages(cpu_addr); 1229 1230 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1231 return cpu_addr; 1232 1233 area = find_vm_area(cpu_addr); 1234 if (area && (area->flags & VM_ARM_DMA_CONSISTENT)) 1235 return area->pages; 1236 return NULL; 1237 } 1238 1239 static void *__iommu_alloc_atomic(struct device *dev, size_t size, 1240 dma_addr_t *handle) 1241 { 1242 struct page *page; 1243 void *addr; 1244 1245 addr = __alloc_from_pool(size, &page); 1246 if (!addr) 1247 return NULL; 1248 1249 *handle = __iommu_create_mapping(dev, &page, size); 1250 if (*handle == DMA_ERROR_CODE) 1251 goto err_mapping; 1252 1253 return addr; 1254 1255 err_mapping: 1256 __free_from_pool(addr, size); 1257 return NULL; 1258 } 1259 1260 static void __iommu_free_atomic(struct device *dev, struct page **pages, 1261 dma_addr_t handle, size_t size) 1262 { 1263 __iommu_remove_mapping(dev, handle, size); 1264 __free_from_pool(page_address(pages[0]), size); 1265 } 1266 1267 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1268 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 1269 { 1270 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); 1271 struct page **pages; 1272 void *addr = NULL; 1273 1274 *handle = DMA_ERROR_CODE; 1275 size = PAGE_ALIGN(size); 1276 1277 if (gfp & GFP_ATOMIC) 1278 return __iommu_alloc_atomic(dev, size, handle); 1279 1280 pages = __iommu_alloc_buffer(dev, size, gfp, attrs); 1281 if (!pages) 1282 return NULL; 1283 1284 *handle = __iommu_create_mapping(dev, pages, size); 1285 if (*handle == DMA_ERROR_CODE) 1286 goto err_buffer; 1287 1288 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1289 return pages; 1290 1291 addr = __iommu_alloc_remap(pages, size, gfp, prot, 1292 __builtin_return_address(0)); 1293 if (!addr) 1294 goto err_mapping; 1295 1296 return addr; 1297 1298 err_mapping: 1299 __iommu_remove_mapping(dev, *handle, size); 1300 err_buffer: 1301 __iommu_free_buffer(dev, pages, size, attrs); 1302 return NULL; 1303 } 1304 1305 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1306 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1307 struct dma_attrs *attrs) 1308 { 1309 unsigned long uaddr = vma->vm_start; 1310 unsigned long usize = vma->vm_end - vma->vm_start; 1311 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1312 1313 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1314 1315 if (!pages) 1316 return -ENXIO; 1317 1318 do { 1319 int ret = vm_insert_page(vma, uaddr, *pages++); 1320 if (ret) { 1321 pr_err("Remapping memory failed: %d\n", ret); 1322 return ret; 1323 } 1324 uaddr += PAGE_SIZE; 1325 usize -= PAGE_SIZE; 1326 } while (usize > 0); 1327 1328 return 0; 1329 } 1330 1331 /* 1332 * free a page as defined by the above mapping. 1333 * Must not be called with IRQs disabled. 1334 */ 1335 void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1336 dma_addr_t handle, struct dma_attrs *attrs) 1337 { 1338 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1339 size = PAGE_ALIGN(size); 1340 1341 if (!pages) { 1342 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1343 return; 1344 } 1345 1346 if (__in_atomic_pool(cpu_addr, size)) { 1347 __iommu_free_atomic(dev, pages, handle, size); 1348 return; 1349 } 1350 1351 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { 1352 unmap_kernel_range((unsigned long)cpu_addr, size); 1353 vunmap(cpu_addr); 1354 } 1355 1356 __iommu_remove_mapping(dev, handle, size); 1357 __iommu_free_buffer(dev, pages, size, attrs); 1358 } 1359 1360 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1361 void *cpu_addr, dma_addr_t dma_addr, 1362 size_t size, struct dma_attrs *attrs) 1363 { 1364 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1365 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1366 1367 if (!pages) 1368 return -ENXIO; 1369 1370 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1371 GFP_KERNEL); 1372 } 1373 1374 /* 1375 * Map a part of the scatter-gather list into contiguous io address space 1376 */ 1377 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1378 size_t size, dma_addr_t *handle, 1379 enum dma_data_direction dir, struct dma_attrs *attrs, 1380 bool is_coherent) 1381 { 1382 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1383 dma_addr_t iova, iova_base; 1384 int ret = 0; 1385 unsigned int count; 1386 struct scatterlist *s; 1387 1388 size = PAGE_ALIGN(size); 1389 *handle = DMA_ERROR_CODE; 1390 1391 iova_base = iova = __alloc_iova(mapping, size); 1392 if (iova == DMA_ERROR_CODE) 1393 return -ENOMEM; 1394 1395 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1396 phys_addr_t phys = page_to_phys(sg_page(s)); 1397 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1398 1399 if (!is_coherent && 1400 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1401 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1402 1403 ret = iommu_map(mapping->domain, iova, phys, len, 0); 1404 if (ret < 0) 1405 goto fail; 1406 count += len >> PAGE_SHIFT; 1407 iova += len; 1408 } 1409 *handle = iova_base; 1410 1411 return 0; 1412 fail: 1413 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1414 __free_iova(mapping, iova_base, size); 1415 return ret; 1416 } 1417 1418 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1419 enum dma_data_direction dir, struct dma_attrs *attrs, 1420 bool is_coherent) 1421 { 1422 struct scatterlist *s = sg, *dma = sg, *start = sg; 1423 int i, count = 0; 1424 unsigned int offset = s->offset; 1425 unsigned int size = s->offset + s->length; 1426 unsigned int max = dma_get_max_seg_size(dev); 1427 1428 for (i = 1; i < nents; i++) { 1429 s = sg_next(s); 1430 1431 s->dma_address = DMA_ERROR_CODE; 1432 s->dma_length = 0; 1433 1434 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1435 if (__map_sg_chunk(dev, start, size, &dma->dma_address, 1436 dir, attrs, is_coherent) < 0) 1437 goto bad_mapping; 1438 1439 dma->dma_address += offset; 1440 dma->dma_length = size - offset; 1441 1442 size = offset = s->offset; 1443 start = s; 1444 dma = sg_next(dma); 1445 count += 1; 1446 } 1447 size += s->length; 1448 } 1449 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, 1450 is_coherent) < 0) 1451 goto bad_mapping; 1452 1453 dma->dma_address += offset; 1454 dma->dma_length = size - offset; 1455 1456 return count+1; 1457 1458 bad_mapping: 1459 for_each_sg(sg, s, count, i) 1460 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1461 return 0; 1462 } 1463 1464 /** 1465 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1466 * @dev: valid struct device pointer 1467 * @sg: list of buffers 1468 * @nents: number of buffers to map 1469 * @dir: DMA transfer direction 1470 * 1471 * Map a set of i/o coherent buffers described by scatterlist in streaming 1472 * mode for DMA. The scatter gather list elements are merged together (if 1473 * possible) and tagged with the appropriate dma address and length. They are 1474 * obtained via sg_dma_{address,length}. 1475 */ 1476 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1477 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1478 { 1479 return __iommu_map_sg(dev, sg, nents, dir, attrs, true); 1480 } 1481 1482 /** 1483 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1484 * @dev: valid struct device pointer 1485 * @sg: list of buffers 1486 * @nents: number of buffers to map 1487 * @dir: DMA transfer direction 1488 * 1489 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1490 * The scatter gather list elements are merged together (if possible) and 1491 * tagged with the appropriate dma address and length. They are obtained via 1492 * sg_dma_{address,length}. 1493 */ 1494 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1495 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1496 { 1497 return __iommu_map_sg(dev, sg, nents, dir, attrs, false); 1498 } 1499 1500 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1501 int nents, enum dma_data_direction dir, struct dma_attrs *attrs, 1502 bool is_coherent) 1503 { 1504 struct scatterlist *s; 1505 int i; 1506 1507 for_each_sg(sg, s, nents, i) { 1508 if (sg_dma_len(s)) 1509 __iommu_remove_mapping(dev, sg_dma_address(s), 1510 sg_dma_len(s)); 1511 if (!is_coherent && 1512 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1513 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1514 s->length, dir); 1515 } 1516 } 1517 1518 /** 1519 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1520 * @dev: valid struct device pointer 1521 * @sg: list of buffers 1522 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1523 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1524 * 1525 * Unmap a set of streaming mode DMA translations. Again, CPU access 1526 * rules concerning calls here are the same as for dma_unmap_single(). 1527 */ 1528 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1529 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1530 { 1531 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); 1532 } 1533 1534 /** 1535 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1536 * @dev: valid struct device pointer 1537 * @sg: list of buffers 1538 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1539 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1540 * 1541 * Unmap a set of streaming mode DMA translations. Again, CPU access 1542 * rules concerning calls here are the same as for dma_unmap_single(). 1543 */ 1544 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1545 enum dma_data_direction dir, struct dma_attrs *attrs) 1546 { 1547 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); 1548 } 1549 1550 /** 1551 * arm_iommu_sync_sg_for_cpu 1552 * @dev: valid struct device pointer 1553 * @sg: list of buffers 1554 * @nents: number of buffers to map (returned from dma_map_sg) 1555 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1556 */ 1557 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1558 int nents, enum dma_data_direction dir) 1559 { 1560 struct scatterlist *s; 1561 int i; 1562 1563 for_each_sg(sg, s, nents, i) 1564 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1565 1566 } 1567 1568 /** 1569 * arm_iommu_sync_sg_for_device 1570 * @dev: valid struct device pointer 1571 * @sg: list of buffers 1572 * @nents: number of buffers to map (returned from dma_map_sg) 1573 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1574 */ 1575 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1576 int nents, enum dma_data_direction dir) 1577 { 1578 struct scatterlist *s; 1579 int i; 1580 1581 for_each_sg(sg, s, nents, i) 1582 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1583 } 1584 1585 1586 /** 1587 * arm_coherent_iommu_map_page 1588 * @dev: valid struct device pointer 1589 * @page: page that buffer resides in 1590 * @offset: offset into page for start of buffer 1591 * @size: size of buffer to map 1592 * @dir: DMA transfer direction 1593 * 1594 * Coherent IOMMU aware version of arm_dma_map_page() 1595 */ 1596 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, 1597 unsigned long offset, size_t size, enum dma_data_direction dir, 1598 struct dma_attrs *attrs) 1599 { 1600 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1601 dma_addr_t dma_addr; 1602 int ret, len = PAGE_ALIGN(size + offset); 1603 1604 dma_addr = __alloc_iova(mapping, len); 1605 if (dma_addr == DMA_ERROR_CODE) 1606 return dma_addr; 1607 1608 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0); 1609 if (ret < 0) 1610 goto fail; 1611 1612 return dma_addr + offset; 1613 fail: 1614 __free_iova(mapping, dma_addr, len); 1615 return DMA_ERROR_CODE; 1616 } 1617 1618 /** 1619 * arm_iommu_map_page 1620 * @dev: valid struct device pointer 1621 * @page: page that buffer resides in 1622 * @offset: offset into page for start of buffer 1623 * @size: size of buffer to map 1624 * @dir: DMA transfer direction 1625 * 1626 * IOMMU aware version of arm_dma_map_page() 1627 */ 1628 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1629 unsigned long offset, size_t size, enum dma_data_direction dir, 1630 struct dma_attrs *attrs) 1631 { 1632 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1633 __dma_page_cpu_to_dev(page, offset, size, dir); 1634 1635 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); 1636 } 1637 1638 /** 1639 * arm_coherent_iommu_unmap_page 1640 * @dev: valid struct device pointer 1641 * @handle: DMA address of buffer 1642 * @size: size of buffer (same as passed to dma_map_page) 1643 * @dir: DMA transfer direction (same as passed to dma_map_page) 1644 * 1645 * Coherent IOMMU aware version of arm_dma_unmap_page() 1646 */ 1647 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1648 size_t size, enum dma_data_direction dir, 1649 struct dma_attrs *attrs) 1650 { 1651 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1652 dma_addr_t iova = handle & PAGE_MASK; 1653 int offset = handle & ~PAGE_MASK; 1654 int len = PAGE_ALIGN(size + offset); 1655 1656 if (!iova) 1657 return; 1658 1659 iommu_unmap(mapping->domain, iova, len); 1660 __free_iova(mapping, iova, len); 1661 } 1662 1663 /** 1664 * arm_iommu_unmap_page 1665 * @dev: valid struct device pointer 1666 * @handle: DMA address of buffer 1667 * @size: size of buffer (same as passed to dma_map_page) 1668 * @dir: DMA transfer direction (same as passed to dma_map_page) 1669 * 1670 * IOMMU aware version of arm_dma_unmap_page() 1671 */ 1672 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1673 size_t size, enum dma_data_direction dir, 1674 struct dma_attrs *attrs) 1675 { 1676 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1677 dma_addr_t iova = handle & PAGE_MASK; 1678 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1679 int offset = handle & ~PAGE_MASK; 1680 int len = PAGE_ALIGN(size + offset); 1681 1682 if (!iova) 1683 return; 1684 1685 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1686 __dma_page_dev_to_cpu(page, offset, size, dir); 1687 1688 iommu_unmap(mapping->domain, iova, len); 1689 __free_iova(mapping, iova, len); 1690 } 1691 1692 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1693 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1694 { 1695 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1696 dma_addr_t iova = handle & PAGE_MASK; 1697 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1698 unsigned int offset = handle & ~PAGE_MASK; 1699 1700 if (!iova) 1701 return; 1702 1703 __dma_page_dev_to_cpu(page, offset, size, dir); 1704 } 1705 1706 static void arm_iommu_sync_single_for_device(struct device *dev, 1707 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1708 { 1709 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1710 dma_addr_t iova = handle & PAGE_MASK; 1711 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1712 unsigned int offset = handle & ~PAGE_MASK; 1713 1714 if (!iova) 1715 return; 1716 1717 __dma_page_cpu_to_dev(page, offset, size, dir); 1718 } 1719 1720 struct dma_map_ops iommu_ops = { 1721 .alloc = arm_iommu_alloc_attrs, 1722 .free = arm_iommu_free_attrs, 1723 .mmap = arm_iommu_mmap_attrs, 1724 .get_sgtable = arm_iommu_get_sgtable, 1725 1726 .map_page = arm_iommu_map_page, 1727 .unmap_page = arm_iommu_unmap_page, 1728 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 1729 .sync_single_for_device = arm_iommu_sync_single_for_device, 1730 1731 .map_sg = arm_iommu_map_sg, 1732 .unmap_sg = arm_iommu_unmap_sg, 1733 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 1734 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 1735 }; 1736 1737 struct dma_map_ops iommu_coherent_ops = { 1738 .alloc = arm_iommu_alloc_attrs, 1739 .free = arm_iommu_free_attrs, 1740 .mmap = arm_iommu_mmap_attrs, 1741 .get_sgtable = arm_iommu_get_sgtable, 1742 1743 .map_page = arm_coherent_iommu_map_page, 1744 .unmap_page = arm_coherent_iommu_unmap_page, 1745 1746 .map_sg = arm_coherent_iommu_map_sg, 1747 .unmap_sg = arm_coherent_iommu_unmap_sg, 1748 }; 1749 1750 /** 1751 * arm_iommu_create_mapping 1752 * @bus: pointer to the bus holding the client device (for IOMMU calls) 1753 * @base: start address of the valid IO address space 1754 * @size: size of the valid IO address space 1755 * @order: accuracy of the IO addresses allocations 1756 * 1757 * Creates a mapping structure which holds information about used/unused 1758 * IO address ranges, which is required to perform memory allocation and 1759 * mapping with IOMMU aware functions. 1760 * 1761 * The client device need to be attached to the mapping with 1762 * arm_iommu_attach_device function. 1763 */ 1764 struct dma_iommu_mapping * 1765 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size, 1766 int order) 1767 { 1768 unsigned int count = size >> (PAGE_SHIFT + order); 1769 unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long); 1770 struct dma_iommu_mapping *mapping; 1771 int err = -ENOMEM; 1772 1773 if (!count) 1774 return ERR_PTR(-EINVAL); 1775 1776 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 1777 if (!mapping) 1778 goto err; 1779 1780 mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL); 1781 if (!mapping->bitmap) 1782 goto err2; 1783 1784 mapping->base = base; 1785 mapping->bits = BITS_PER_BYTE * bitmap_size; 1786 mapping->order = order; 1787 spin_lock_init(&mapping->lock); 1788 1789 mapping->domain = iommu_domain_alloc(bus); 1790 if (!mapping->domain) 1791 goto err3; 1792 1793 kref_init(&mapping->kref); 1794 return mapping; 1795 err3: 1796 kfree(mapping->bitmap); 1797 err2: 1798 kfree(mapping); 1799 err: 1800 return ERR_PTR(err); 1801 } 1802 1803 static void release_iommu_mapping(struct kref *kref) 1804 { 1805 struct dma_iommu_mapping *mapping = 1806 container_of(kref, struct dma_iommu_mapping, kref); 1807 1808 iommu_domain_free(mapping->domain); 1809 kfree(mapping->bitmap); 1810 kfree(mapping); 1811 } 1812 1813 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 1814 { 1815 if (mapping) 1816 kref_put(&mapping->kref, release_iommu_mapping); 1817 } 1818 1819 /** 1820 * arm_iommu_attach_device 1821 * @dev: valid struct device pointer 1822 * @mapping: io address space mapping structure (returned from 1823 * arm_iommu_create_mapping) 1824 * 1825 * Attaches specified io address space mapping to the provided device, 1826 * this replaces the dma operations (dma_map_ops pointer) with the 1827 * IOMMU aware version. More than one client might be attached to 1828 * the same io address space mapping. 1829 */ 1830 int arm_iommu_attach_device(struct device *dev, 1831 struct dma_iommu_mapping *mapping) 1832 { 1833 int err; 1834 1835 err = iommu_attach_device(mapping->domain, dev); 1836 if (err) 1837 return err; 1838 1839 kref_get(&mapping->kref); 1840 dev->archdata.mapping = mapping; 1841 set_dma_ops(dev, &iommu_ops); 1842 1843 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 1844 return 0; 1845 } 1846 1847 #endif 1848