1 /* 2 * linux/arch/arm/mm/dma-mapping.c 3 * 4 * Copyright (C) 2000-2004 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * DMA uncached mapping support. 11 */ 12 #include <linux/bootmem.h> 13 #include <linux/module.h> 14 #include <linux/mm.h> 15 #include <linux/genalloc.h> 16 #include <linux/gfp.h> 17 #include <linux/errno.h> 18 #include <linux/list.h> 19 #include <linux/init.h> 20 #include <linux/device.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/dma-contiguous.h> 23 #include <linux/highmem.h> 24 #include <linux/memblock.h> 25 #include <linux/slab.h> 26 #include <linux/iommu.h> 27 #include <linux/io.h> 28 #include <linux/vmalloc.h> 29 #include <linux/sizes.h> 30 #include <linux/cma.h> 31 32 #include <asm/memory.h> 33 #include <asm/highmem.h> 34 #include <asm/cacheflush.h> 35 #include <asm/tlbflush.h> 36 #include <asm/mach/arch.h> 37 #include <asm/dma-iommu.h> 38 #include <asm/mach/map.h> 39 #include <asm/system_info.h> 40 #include <asm/dma-contiguous.h> 41 42 #include "dma.h" 43 #include "mm.h" 44 45 struct arm_dma_alloc_args { 46 struct device *dev; 47 size_t size; 48 gfp_t gfp; 49 pgprot_t prot; 50 const void *caller; 51 bool want_vaddr; 52 }; 53 54 struct arm_dma_free_args { 55 struct device *dev; 56 size_t size; 57 void *cpu_addr; 58 struct page *page; 59 bool want_vaddr; 60 }; 61 62 struct arm_dma_allocator { 63 void *(*alloc)(struct arm_dma_alloc_args *args, 64 struct page **ret_page); 65 void (*free)(struct arm_dma_free_args *args); 66 }; 67 68 struct arm_dma_buffer { 69 struct list_head list; 70 void *virt; 71 struct arm_dma_allocator *allocator; 72 }; 73 74 static LIST_HEAD(arm_dma_bufs); 75 static DEFINE_SPINLOCK(arm_dma_bufs_lock); 76 77 static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) 78 { 79 struct arm_dma_buffer *buf, *found = NULL; 80 unsigned long flags; 81 82 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 83 list_for_each_entry(buf, &arm_dma_bufs, list) { 84 if (buf->virt == virt) { 85 list_del(&buf->list); 86 found = buf; 87 break; 88 } 89 } 90 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 91 return found; 92 } 93 94 /* 95 * The DMA API is built upon the notion of "buffer ownership". A buffer 96 * is either exclusively owned by the CPU (and therefore may be accessed 97 * by it) or exclusively owned by the DMA device. These helper functions 98 * represent the transitions between these two ownership states. 99 * 100 * Note, however, that on later ARMs, this notion does not work due to 101 * speculative prefetches. We model our approach on the assumption that 102 * the CPU does do speculative prefetches, which means we clean caches 103 * before transfers and delay cache invalidation until transfer completion. 104 * 105 */ 106 static void __dma_page_cpu_to_dev(struct page *, unsigned long, 107 size_t, enum dma_data_direction); 108 static void __dma_page_dev_to_cpu(struct page *, unsigned long, 109 size_t, enum dma_data_direction); 110 111 /** 112 * arm_dma_map_page - map a portion of a page for streaming DMA 113 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 114 * @page: page that buffer resides in 115 * @offset: offset into page for start of buffer 116 * @size: size of buffer to map 117 * @dir: DMA transfer direction 118 * 119 * Ensure that any data held in the cache is appropriately discarded 120 * or written back. 121 * 122 * The device owns this memory once this call has completed. The CPU 123 * can regain ownership by calling dma_unmap_page(). 124 */ 125 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, 126 unsigned long offset, size_t size, enum dma_data_direction dir, 127 struct dma_attrs *attrs) 128 { 129 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 130 __dma_page_cpu_to_dev(page, offset, size, dir); 131 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 132 } 133 134 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, 135 unsigned long offset, size_t size, enum dma_data_direction dir, 136 struct dma_attrs *attrs) 137 { 138 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 139 } 140 141 /** 142 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() 143 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 144 * @handle: DMA address of buffer 145 * @size: size of buffer (same as passed to dma_map_page) 146 * @dir: DMA transfer direction (same as passed to dma_map_page) 147 * 148 * Unmap a page streaming mode DMA translation. The handle and size 149 * must match what was provided in the previous dma_map_page() call. 150 * All other usages are undefined. 151 * 152 * After this call, reads by the CPU to the buffer are guaranteed to see 153 * whatever the device wrote there. 154 */ 155 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, 156 size_t size, enum dma_data_direction dir, 157 struct dma_attrs *attrs) 158 { 159 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 160 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), 161 handle & ~PAGE_MASK, size, dir); 162 } 163 164 static void arm_dma_sync_single_for_cpu(struct device *dev, 165 dma_addr_t handle, size_t size, enum dma_data_direction dir) 166 { 167 unsigned int offset = handle & (PAGE_SIZE - 1); 168 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 169 __dma_page_dev_to_cpu(page, offset, size, dir); 170 } 171 172 static void arm_dma_sync_single_for_device(struct device *dev, 173 dma_addr_t handle, size_t size, enum dma_data_direction dir) 174 { 175 unsigned int offset = handle & (PAGE_SIZE - 1); 176 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 177 __dma_page_cpu_to_dev(page, offset, size, dir); 178 } 179 180 struct dma_map_ops arm_dma_ops = { 181 .alloc = arm_dma_alloc, 182 .free = arm_dma_free, 183 .mmap = arm_dma_mmap, 184 .get_sgtable = arm_dma_get_sgtable, 185 .map_page = arm_dma_map_page, 186 .unmap_page = arm_dma_unmap_page, 187 .map_sg = arm_dma_map_sg, 188 .unmap_sg = arm_dma_unmap_sg, 189 .sync_single_for_cpu = arm_dma_sync_single_for_cpu, 190 .sync_single_for_device = arm_dma_sync_single_for_device, 191 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, 192 .sync_sg_for_device = arm_dma_sync_sg_for_device, 193 .set_dma_mask = arm_dma_set_mask, 194 }; 195 EXPORT_SYMBOL(arm_dma_ops); 196 197 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 198 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); 199 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 200 dma_addr_t handle, struct dma_attrs *attrs); 201 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, 202 void *cpu_addr, dma_addr_t dma_addr, size_t size, 203 struct dma_attrs *attrs); 204 205 struct dma_map_ops arm_coherent_dma_ops = { 206 .alloc = arm_coherent_dma_alloc, 207 .free = arm_coherent_dma_free, 208 .mmap = arm_coherent_dma_mmap, 209 .get_sgtable = arm_dma_get_sgtable, 210 .map_page = arm_coherent_dma_map_page, 211 .map_sg = arm_dma_map_sg, 212 .set_dma_mask = arm_dma_set_mask, 213 }; 214 EXPORT_SYMBOL(arm_coherent_dma_ops); 215 216 static int __dma_supported(struct device *dev, u64 mask, bool warn) 217 { 218 unsigned long max_dma_pfn; 219 220 /* 221 * If the mask allows for more memory than we can address, 222 * and we actually have that much memory, then we must 223 * indicate that DMA to this device is not supported. 224 */ 225 if (sizeof(mask) != sizeof(dma_addr_t) && 226 mask > (dma_addr_t)~0 && 227 dma_to_pfn(dev, ~0) < max_pfn - 1) { 228 if (warn) { 229 dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n", 230 mask); 231 dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n"); 232 } 233 return 0; 234 } 235 236 max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); 237 238 /* 239 * Translate the device's DMA mask to a PFN limit. This 240 * PFN number includes the page which we can DMA to. 241 */ 242 if (dma_to_pfn(dev, mask) < max_dma_pfn) { 243 if (warn) 244 dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n", 245 mask, 246 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1, 247 max_dma_pfn + 1); 248 return 0; 249 } 250 251 return 1; 252 } 253 254 static u64 get_coherent_dma_mask(struct device *dev) 255 { 256 u64 mask = (u64)DMA_BIT_MASK(32); 257 258 if (dev) { 259 mask = dev->coherent_dma_mask; 260 261 /* 262 * Sanity check the DMA mask - it must be non-zero, and 263 * must be able to be satisfied by a DMA allocation. 264 */ 265 if (mask == 0) { 266 dev_warn(dev, "coherent DMA mask is unset\n"); 267 return 0; 268 } 269 270 if (!__dma_supported(dev, mask, true)) 271 return 0; 272 } 273 274 return mask; 275 } 276 277 static void __dma_clear_buffer(struct page *page, size_t size) 278 { 279 /* 280 * Ensure that the allocated pages are zeroed, and that any data 281 * lurking in the kernel direct-mapped region is invalidated. 282 */ 283 if (PageHighMem(page)) { 284 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 285 phys_addr_t end = base + size; 286 while (size > 0) { 287 void *ptr = kmap_atomic(page); 288 memset(ptr, 0, PAGE_SIZE); 289 dmac_flush_range(ptr, ptr + PAGE_SIZE); 290 kunmap_atomic(ptr); 291 page++; 292 size -= PAGE_SIZE; 293 } 294 outer_flush_range(base, end); 295 } else { 296 void *ptr = page_address(page); 297 memset(ptr, 0, size); 298 dmac_flush_range(ptr, ptr + size); 299 outer_flush_range(__pa(ptr), __pa(ptr) + size); 300 } 301 } 302 303 /* 304 * Allocate a DMA buffer for 'dev' of size 'size' using the 305 * specified gfp mask. Note that 'size' must be page aligned. 306 */ 307 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) 308 { 309 unsigned long order = get_order(size); 310 struct page *page, *p, *e; 311 312 page = alloc_pages(gfp, order); 313 if (!page) 314 return NULL; 315 316 /* 317 * Now split the huge page and free the excess pages 318 */ 319 split_page(page, order); 320 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 321 __free_page(p); 322 323 __dma_clear_buffer(page, size); 324 325 return page; 326 } 327 328 /* 329 * Free a DMA buffer. 'size' must be page aligned. 330 */ 331 static void __dma_free_buffer(struct page *page, size_t size) 332 { 333 struct page *e = page + (size >> PAGE_SHIFT); 334 335 while (page < e) { 336 __free_page(page); 337 page++; 338 } 339 } 340 341 #ifdef CONFIG_MMU 342 343 static void *__alloc_from_contiguous(struct device *dev, size_t size, 344 pgprot_t prot, struct page **ret_page, 345 const void *caller, bool want_vaddr); 346 347 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 348 pgprot_t prot, struct page **ret_page, 349 const void *caller, bool want_vaddr); 350 351 static void * 352 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 353 const void *caller) 354 { 355 /* 356 * DMA allocation can be mapped to user space, so lets 357 * set VM_USERMAP flags too. 358 */ 359 return dma_common_contiguous_remap(page, size, 360 VM_ARM_DMA_CONSISTENT | VM_USERMAP, 361 prot, caller); 362 } 363 364 static void __dma_free_remap(void *cpu_addr, size_t size) 365 { 366 dma_common_free_remap(cpu_addr, size, 367 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 368 } 369 370 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 371 static struct gen_pool *atomic_pool; 372 373 static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; 374 375 static int __init early_coherent_pool(char *p) 376 { 377 atomic_pool_size = memparse(p, &p); 378 return 0; 379 } 380 early_param("coherent_pool", early_coherent_pool); 381 382 void __init init_dma_coherent_pool_size(unsigned long size) 383 { 384 /* 385 * Catch any attempt to set the pool size too late. 386 */ 387 BUG_ON(atomic_pool); 388 389 /* 390 * Set architecture specific coherent pool size only if 391 * it has not been changed by kernel command line parameter. 392 */ 393 if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE) 394 atomic_pool_size = size; 395 } 396 397 /* 398 * Initialise the coherent pool for atomic allocations. 399 */ 400 static int __init atomic_pool_init(void) 401 { 402 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); 403 gfp_t gfp = GFP_KERNEL | GFP_DMA; 404 struct page *page; 405 void *ptr; 406 407 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 408 if (!atomic_pool) 409 goto out; 410 411 if (dev_get_cma_area(NULL)) 412 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 413 &page, atomic_pool_init, true); 414 else 415 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 416 &page, atomic_pool_init, true); 417 if (ptr) { 418 int ret; 419 420 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr, 421 page_to_phys(page), 422 atomic_pool_size, -1); 423 if (ret) 424 goto destroy_genpool; 425 426 gen_pool_set_algo(atomic_pool, 427 gen_pool_first_fit_order_align, 428 (void *)PAGE_SHIFT); 429 pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n", 430 atomic_pool_size / 1024); 431 return 0; 432 } 433 434 destroy_genpool: 435 gen_pool_destroy(atomic_pool); 436 atomic_pool = NULL; 437 out: 438 pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n", 439 atomic_pool_size / 1024); 440 return -ENOMEM; 441 } 442 /* 443 * CMA is activated by core_initcall, so we must be called after it. 444 */ 445 postcore_initcall(atomic_pool_init); 446 447 struct dma_contig_early_reserve { 448 phys_addr_t base; 449 unsigned long size; 450 }; 451 452 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 453 454 static int dma_mmu_remap_num __initdata; 455 456 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 457 { 458 dma_mmu_remap[dma_mmu_remap_num].base = base; 459 dma_mmu_remap[dma_mmu_remap_num].size = size; 460 dma_mmu_remap_num++; 461 } 462 463 void __init dma_contiguous_remap(void) 464 { 465 int i; 466 for (i = 0; i < dma_mmu_remap_num; i++) { 467 phys_addr_t start = dma_mmu_remap[i].base; 468 phys_addr_t end = start + dma_mmu_remap[i].size; 469 struct map_desc map; 470 unsigned long addr; 471 472 if (end > arm_lowmem_limit) 473 end = arm_lowmem_limit; 474 if (start >= end) 475 continue; 476 477 map.pfn = __phys_to_pfn(start); 478 map.virtual = __phys_to_virt(start); 479 map.length = end - start; 480 map.type = MT_MEMORY_DMA_READY; 481 482 /* 483 * Clear previous low-memory mapping to ensure that the 484 * TLB does not see any conflicting entries, then flush 485 * the TLB of the old entries before creating new mappings. 486 * 487 * This ensures that any speculatively loaded TLB entries 488 * (even though they may be rare) can not cause any problems, 489 * and ensures that this code is architecturally compliant. 490 */ 491 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 492 addr += PMD_SIZE) 493 pmd_clear(pmd_off_k(addr)); 494 495 flush_tlb_kernel_range(__phys_to_virt(start), 496 __phys_to_virt(end)); 497 498 iotable_init(&map, 1); 499 } 500 } 501 502 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, 503 void *data) 504 { 505 struct page *page = virt_to_page(addr); 506 pgprot_t prot = *(pgprot_t *)data; 507 508 set_pte_ext(pte, mk_pte(page, prot), 0); 509 return 0; 510 } 511 512 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 513 { 514 unsigned long start = (unsigned long) page_address(page); 515 unsigned end = start + size; 516 517 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 518 flush_tlb_kernel_range(start, end); 519 } 520 521 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 522 pgprot_t prot, struct page **ret_page, 523 const void *caller, bool want_vaddr) 524 { 525 struct page *page; 526 void *ptr = NULL; 527 page = __dma_alloc_buffer(dev, size, gfp); 528 if (!page) 529 return NULL; 530 if (!want_vaddr) 531 goto out; 532 533 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 534 if (!ptr) { 535 __dma_free_buffer(page, size); 536 return NULL; 537 } 538 539 out: 540 *ret_page = page; 541 return ptr; 542 } 543 544 static void *__alloc_from_pool(size_t size, struct page **ret_page) 545 { 546 unsigned long val; 547 void *ptr = NULL; 548 549 if (!atomic_pool) { 550 WARN(1, "coherent pool not initialised!\n"); 551 return NULL; 552 } 553 554 val = gen_pool_alloc(atomic_pool, size); 555 if (val) { 556 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 557 558 *ret_page = phys_to_page(phys); 559 ptr = (void *)val; 560 } 561 562 return ptr; 563 } 564 565 static bool __in_atomic_pool(void *start, size_t size) 566 { 567 return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); 568 } 569 570 static int __free_from_pool(void *start, size_t size) 571 { 572 if (!__in_atomic_pool(start, size)) 573 return 0; 574 575 gen_pool_free(atomic_pool, (unsigned long)start, size); 576 577 return 1; 578 } 579 580 static void *__alloc_from_contiguous(struct device *dev, size_t size, 581 pgprot_t prot, struct page **ret_page, 582 const void *caller, bool want_vaddr) 583 { 584 unsigned long order = get_order(size); 585 size_t count = size >> PAGE_SHIFT; 586 struct page *page; 587 void *ptr = NULL; 588 589 page = dma_alloc_from_contiguous(dev, count, order); 590 if (!page) 591 return NULL; 592 593 __dma_clear_buffer(page, size); 594 595 if (!want_vaddr) 596 goto out; 597 598 if (PageHighMem(page)) { 599 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); 600 if (!ptr) { 601 dma_release_from_contiguous(dev, page, count); 602 return NULL; 603 } 604 } else { 605 __dma_remap(page, size, prot); 606 ptr = page_address(page); 607 } 608 609 out: 610 *ret_page = page; 611 return ptr; 612 } 613 614 static void __free_from_contiguous(struct device *dev, struct page *page, 615 void *cpu_addr, size_t size, bool want_vaddr) 616 { 617 if (want_vaddr) { 618 if (PageHighMem(page)) 619 __dma_free_remap(cpu_addr, size); 620 else 621 __dma_remap(page, size, PAGE_KERNEL); 622 } 623 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 624 } 625 626 static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) 627 { 628 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? 629 pgprot_writecombine(prot) : 630 pgprot_dmacoherent(prot); 631 return prot; 632 } 633 634 #define nommu() 0 635 636 #else /* !CONFIG_MMU */ 637 638 #define nommu() 1 639 640 #define __get_dma_pgprot(attrs, prot) __pgprot(0) 641 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL 642 #define __alloc_from_pool(size, ret_page) NULL 643 #define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL 644 #define __free_from_pool(cpu_addr, size) do { } while (0) 645 #define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0) 646 #define __dma_free_remap(cpu_addr, size) do { } while (0) 647 648 #endif /* CONFIG_MMU */ 649 650 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 651 struct page **ret_page) 652 { 653 struct page *page; 654 page = __dma_alloc_buffer(dev, size, gfp); 655 if (!page) 656 return NULL; 657 658 *ret_page = page; 659 return page_address(page); 660 } 661 662 static void *simple_allocator_alloc(struct arm_dma_alloc_args *args, 663 struct page **ret_page) 664 { 665 return __alloc_simple_buffer(args->dev, args->size, args->gfp, 666 ret_page); 667 } 668 669 static void simple_allocator_free(struct arm_dma_free_args *args) 670 { 671 __dma_free_buffer(args->page, args->size); 672 } 673 674 static struct arm_dma_allocator simple_allocator = { 675 .alloc = simple_allocator_alloc, 676 .free = simple_allocator_free, 677 }; 678 679 static void *cma_allocator_alloc(struct arm_dma_alloc_args *args, 680 struct page **ret_page) 681 { 682 return __alloc_from_contiguous(args->dev, args->size, args->prot, 683 ret_page, args->caller, 684 args->want_vaddr); 685 } 686 687 static void cma_allocator_free(struct arm_dma_free_args *args) 688 { 689 __free_from_contiguous(args->dev, args->page, args->cpu_addr, 690 args->size, args->want_vaddr); 691 } 692 693 static struct arm_dma_allocator cma_allocator = { 694 .alloc = cma_allocator_alloc, 695 .free = cma_allocator_free, 696 }; 697 698 static void *pool_allocator_alloc(struct arm_dma_alloc_args *args, 699 struct page **ret_page) 700 { 701 return __alloc_from_pool(args->size, ret_page); 702 } 703 704 static void pool_allocator_free(struct arm_dma_free_args *args) 705 { 706 __free_from_pool(args->cpu_addr, args->size); 707 } 708 709 static struct arm_dma_allocator pool_allocator = { 710 .alloc = pool_allocator_alloc, 711 .free = pool_allocator_free, 712 }; 713 714 static void *remap_allocator_alloc(struct arm_dma_alloc_args *args, 715 struct page **ret_page) 716 { 717 return __alloc_remap_buffer(args->dev, args->size, args->gfp, 718 args->prot, ret_page, args->caller, 719 args->want_vaddr); 720 } 721 722 static void remap_allocator_free(struct arm_dma_free_args *args) 723 { 724 if (args->want_vaddr) 725 __dma_free_remap(args->cpu_addr, args->size); 726 727 __dma_free_buffer(args->page, args->size); 728 } 729 730 static struct arm_dma_allocator remap_allocator = { 731 .alloc = remap_allocator_alloc, 732 .free = remap_allocator_free, 733 }; 734 735 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 736 gfp_t gfp, pgprot_t prot, bool is_coherent, 737 struct dma_attrs *attrs, const void *caller) 738 { 739 u64 mask = get_coherent_dma_mask(dev); 740 struct page *page = NULL; 741 void *addr; 742 bool allowblock, cma; 743 struct arm_dma_buffer *buf; 744 struct arm_dma_alloc_args args = { 745 .dev = dev, 746 .size = PAGE_ALIGN(size), 747 .gfp = gfp, 748 .prot = prot, 749 .caller = caller, 750 .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), 751 }; 752 753 #ifdef CONFIG_DMA_API_DEBUG 754 u64 limit = (mask + 1) & ~mask; 755 if (limit && size >= limit) { 756 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 757 size, mask); 758 return NULL; 759 } 760 #endif 761 762 if (!mask) 763 return NULL; 764 765 buf = kzalloc(sizeof(*buf), 766 gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)); 767 if (!buf) 768 return NULL; 769 770 if (mask < 0xffffffffULL) 771 gfp |= GFP_DMA; 772 773 /* 774 * Following is a work-around (a.k.a. hack) to prevent pages 775 * with __GFP_COMP being passed to split_page() which cannot 776 * handle them. The real problem is that this flag probably 777 * should be 0 on ARM as it is not supported on this 778 * platform; see CONFIG_HUGETLBFS. 779 */ 780 gfp &= ~(__GFP_COMP); 781 args.gfp = gfp; 782 783 *handle = DMA_ERROR_CODE; 784 allowblock = gfpflags_allow_blocking(gfp); 785 cma = allowblock ? dev_get_cma_area(dev) : false; 786 787 if (cma) 788 buf->allocator = &cma_allocator; 789 else if (nommu() || is_coherent) 790 buf->allocator = &simple_allocator; 791 else if (allowblock) 792 buf->allocator = &remap_allocator; 793 else 794 buf->allocator = &pool_allocator; 795 796 addr = buf->allocator->alloc(&args, &page); 797 798 if (page) { 799 unsigned long flags; 800 801 *handle = pfn_to_dma(dev, page_to_pfn(page)); 802 buf->virt = args.want_vaddr ? addr : page; 803 804 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 805 list_add(&buf->list, &arm_dma_bufs); 806 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 807 } else { 808 kfree(buf); 809 } 810 811 return args.want_vaddr ? addr : page; 812 } 813 814 /* 815 * Allocate DMA-coherent memory space and return both the kernel remapped 816 * virtual and bus address for that space. 817 */ 818 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 819 gfp_t gfp, struct dma_attrs *attrs) 820 { 821 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 822 823 return __dma_alloc(dev, size, handle, gfp, prot, false, 824 attrs, __builtin_return_address(0)); 825 } 826 827 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 828 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 829 { 830 return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, 831 attrs, __builtin_return_address(0)); 832 } 833 834 static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 835 void *cpu_addr, dma_addr_t dma_addr, size_t size, 836 struct dma_attrs *attrs) 837 { 838 int ret = -ENXIO; 839 #ifdef CONFIG_MMU 840 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 841 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 842 unsigned long pfn = dma_to_pfn(dev, dma_addr); 843 unsigned long off = vma->vm_pgoff; 844 845 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 846 return ret; 847 848 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 849 ret = remap_pfn_range(vma, vma->vm_start, 850 pfn + off, 851 vma->vm_end - vma->vm_start, 852 vma->vm_page_prot); 853 } 854 #endif /* CONFIG_MMU */ 855 856 return ret; 857 } 858 859 /* 860 * Create userspace mapping for the DMA-coherent memory. 861 */ 862 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, 863 void *cpu_addr, dma_addr_t dma_addr, size_t size, 864 struct dma_attrs *attrs) 865 { 866 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 867 } 868 869 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 870 void *cpu_addr, dma_addr_t dma_addr, size_t size, 871 struct dma_attrs *attrs) 872 { 873 #ifdef CONFIG_MMU 874 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 875 #endif /* CONFIG_MMU */ 876 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 877 } 878 879 /* 880 * Free a buffer as defined by the above mapping. 881 */ 882 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 883 dma_addr_t handle, struct dma_attrs *attrs, 884 bool is_coherent) 885 { 886 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 887 struct arm_dma_buffer *buf; 888 struct arm_dma_free_args args = { 889 .dev = dev, 890 .size = PAGE_ALIGN(size), 891 .cpu_addr = cpu_addr, 892 .page = page, 893 .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), 894 }; 895 896 buf = arm_dma_buffer_find(cpu_addr); 897 if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) 898 return; 899 900 buf->allocator->free(&args); 901 kfree(buf); 902 } 903 904 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 905 dma_addr_t handle, struct dma_attrs *attrs) 906 { 907 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); 908 } 909 910 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 911 dma_addr_t handle, struct dma_attrs *attrs) 912 { 913 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); 914 } 915 916 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 917 void *cpu_addr, dma_addr_t handle, size_t size, 918 struct dma_attrs *attrs) 919 { 920 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 921 int ret; 922 923 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 924 if (unlikely(ret)) 925 return ret; 926 927 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 928 return 0; 929 } 930 931 static void dma_cache_maint_page(struct page *page, unsigned long offset, 932 size_t size, enum dma_data_direction dir, 933 void (*op)(const void *, size_t, int)) 934 { 935 unsigned long pfn; 936 size_t left = size; 937 938 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 939 offset %= PAGE_SIZE; 940 941 /* 942 * A single sg entry may refer to multiple physically contiguous 943 * pages. But we still need to process highmem pages individually. 944 * If highmem is not configured then the bulk of this loop gets 945 * optimized out. 946 */ 947 do { 948 size_t len = left; 949 void *vaddr; 950 951 page = pfn_to_page(pfn); 952 953 if (PageHighMem(page)) { 954 if (len + offset > PAGE_SIZE) 955 len = PAGE_SIZE - offset; 956 957 if (cache_is_vipt_nonaliasing()) { 958 vaddr = kmap_atomic(page); 959 op(vaddr + offset, len, dir); 960 kunmap_atomic(vaddr); 961 } else { 962 vaddr = kmap_high_get(page); 963 if (vaddr) { 964 op(vaddr + offset, len, dir); 965 kunmap_high(page); 966 } 967 } 968 } else { 969 vaddr = page_address(page) + offset; 970 op(vaddr, len, dir); 971 } 972 offset = 0; 973 pfn++; 974 left -= len; 975 } while (left); 976 } 977 978 /* 979 * Make an area consistent for devices. 980 * Note: Drivers should NOT use this function directly, as it will break 981 * platforms with CONFIG_DMABOUNCE. 982 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 983 */ 984 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 985 size_t size, enum dma_data_direction dir) 986 { 987 phys_addr_t paddr; 988 989 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 990 991 paddr = page_to_phys(page) + off; 992 if (dir == DMA_FROM_DEVICE) { 993 outer_inv_range(paddr, paddr + size); 994 } else { 995 outer_clean_range(paddr, paddr + size); 996 } 997 /* FIXME: non-speculating: flush on bidirectional mappings? */ 998 } 999 1000 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 1001 size_t size, enum dma_data_direction dir) 1002 { 1003 phys_addr_t paddr = page_to_phys(page) + off; 1004 1005 /* FIXME: non-speculating: not required */ 1006 /* in any case, don't bother invalidating if DMA to device */ 1007 if (dir != DMA_TO_DEVICE) { 1008 outer_inv_range(paddr, paddr + size); 1009 1010 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 1011 } 1012 1013 /* 1014 * Mark the D-cache clean for these pages to avoid extra flushing. 1015 */ 1016 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { 1017 unsigned long pfn; 1018 size_t left = size; 1019 1020 pfn = page_to_pfn(page) + off / PAGE_SIZE; 1021 off %= PAGE_SIZE; 1022 if (off) { 1023 pfn++; 1024 left -= PAGE_SIZE - off; 1025 } 1026 while (left >= PAGE_SIZE) { 1027 page = pfn_to_page(pfn++); 1028 set_bit(PG_dcache_clean, &page->flags); 1029 left -= PAGE_SIZE; 1030 } 1031 } 1032 } 1033 1034 /** 1035 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA 1036 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1037 * @sg: list of buffers 1038 * @nents: number of buffers to map 1039 * @dir: DMA transfer direction 1040 * 1041 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1042 * This is the scatter-gather version of the dma_map_single interface. 1043 * Here the scatter gather list elements are each tagged with the 1044 * appropriate dma address and length. They are obtained via 1045 * sg_dma_{address,length}. 1046 * 1047 * Device ownership issues as mentioned for dma_map_single are the same 1048 * here. 1049 */ 1050 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1051 enum dma_data_direction dir, struct dma_attrs *attrs) 1052 { 1053 struct dma_map_ops *ops = get_dma_ops(dev); 1054 struct scatterlist *s; 1055 int i, j; 1056 1057 for_each_sg(sg, s, nents, i) { 1058 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1059 s->dma_length = s->length; 1060 #endif 1061 s->dma_address = ops->map_page(dev, sg_page(s), s->offset, 1062 s->length, dir, attrs); 1063 if (dma_mapping_error(dev, s->dma_address)) 1064 goto bad_mapping; 1065 } 1066 return nents; 1067 1068 bad_mapping: 1069 for_each_sg(sg, s, i, j) 1070 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 1071 return 0; 1072 } 1073 1074 /** 1075 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1076 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1077 * @sg: list of buffers 1078 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1079 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1080 * 1081 * Unmap a set of streaming mode DMA translations. Again, CPU access 1082 * rules concerning calls here are the same as for dma_unmap_single(). 1083 */ 1084 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1085 enum dma_data_direction dir, struct dma_attrs *attrs) 1086 { 1087 struct dma_map_ops *ops = get_dma_ops(dev); 1088 struct scatterlist *s; 1089 1090 int i; 1091 1092 for_each_sg(sg, s, nents, i) 1093 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 1094 } 1095 1096 /** 1097 * arm_dma_sync_sg_for_cpu 1098 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1099 * @sg: list of buffers 1100 * @nents: number of buffers to map (returned from dma_map_sg) 1101 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1102 */ 1103 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1104 int nents, enum dma_data_direction dir) 1105 { 1106 struct dma_map_ops *ops = get_dma_ops(dev); 1107 struct scatterlist *s; 1108 int i; 1109 1110 for_each_sg(sg, s, nents, i) 1111 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, 1112 dir); 1113 } 1114 1115 /** 1116 * arm_dma_sync_sg_for_device 1117 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1118 * @sg: list of buffers 1119 * @nents: number of buffers to map (returned from dma_map_sg) 1120 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1121 */ 1122 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1123 int nents, enum dma_data_direction dir) 1124 { 1125 struct dma_map_ops *ops = get_dma_ops(dev); 1126 struct scatterlist *s; 1127 int i; 1128 1129 for_each_sg(sg, s, nents, i) 1130 ops->sync_single_for_device(dev, sg_dma_address(s), s->length, 1131 dir); 1132 } 1133 1134 /* 1135 * Return whether the given device DMA address mask can be supported 1136 * properly. For example, if your device can only drive the low 24-bits 1137 * during bus mastering, then you would pass 0x00ffffff as the mask 1138 * to this function. 1139 */ 1140 int dma_supported(struct device *dev, u64 mask) 1141 { 1142 return __dma_supported(dev, mask, false); 1143 } 1144 EXPORT_SYMBOL(dma_supported); 1145 1146 int arm_dma_set_mask(struct device *dev, u64 dma_mask) 1147 { 1148 if (!dev->dma_mask || !dma_supported(dev, dma_mask)) 1149 return -EIO; 1150 1151 *dev->dma_mask = dma_mask; 1152 1153 return 0; 1154 } 1155 1156 #define PREALLOC_DMA_DEBUG_ENTRIES 4096 1157 1158 static int __init dma_debug_do_init(void) 1159 { 1160 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 1161 return 0; 1162 } 1163 fs_initcall(dma_debug_do_init); 1164 1165 #ifdef CONFIG_ARM_DMA_USE_IOMMU 1166 1167 /* IOMMU */ 1168 1169 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); 1170 1171 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 1172 size_t size) 1173 { 1174 unsigned int order = get_order(size); 1175 unsigned int align = 0; 1176 unsigned int count, start; 1177 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1178 unsigned long flags; 1179 dma_addr_t iova; 1180 int i; 1181 1182 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 1183 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 1184 1185 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1186 align = (1 << order) - 1; 1187 1188 spin_lock_irqsave(&mapping->lock, flags); 1189 for (i = 0; i < mapping->nr_bitmaps; i++) { 1190 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1191 mapping->bits, 0, count, align); 1192 1193 if (start > mapping->bits) 1194 continue; 1195 1196 bitmap_set(mapping->bitmaps[i], start, count); 1197 break; 1198 } 1199 1200 /* 1201 * No unused range found. Try to extend the existing mapping 1202 * and perform a second attempt to reserve an IO virtual 1203 * address range of size bytes. 1204 */ 1205 if (i == mapping->nr_bitmaps) { 1206 if (extend_iommu_mapping(mapping)) { 1207 spin_unlock_irqrestore(&mapping->lock, flags); 1208 return DMA_ERROR_CODE; 1209 } 1210 1211 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1212 mapping->bits, 0, count, align); 1213 1214 if (start > mapping->bits) { 1215 spin_unlock_irqrestore(&mapping->lock, flags); 1216 return DMA_ERROR_CODE; 1217 } 1218 1219 bitmap_set(mapping->bitmaps[i], start, count); 1220 } 1221 spin_unlock_irqrestore(&mapping->lock, flags); 1222 1223 iova = mapping->base + (mapping_size * i); 1224 iova += start << PAGE_SHIFT; 1225 1226 return iova; 1227 } 1228 1229 static inline void __free_iova(struct dma_iommu_mapping *mapping, 1230 dma_addr_t addr, size_t size) 1231 { 1232 unsigned int start, count; 1233 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1234 unsigned long flags; 1235 dma_addr_t bitmap_base; 1236 u32 bitmap_index; 1237 1238 if (!size) 1239 return; 1240 1241 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; 1242 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); 1243 1244 bitmap_base = mapping->base + mapping_size * bitmap_index; 1245 1246 start = (addr - bitmap_base) >> PAGE_SHIFT; 1247 1248 if (addr + size > bitmap_base + mapping_size) { 1249 /* 1250 * The address range to be freed reaches into the iova 1251 * range of the next bitmap. This should not happen as 1252 * we don't allow this in __alloc_iova (at the 1253 * moment). 1254 */ 1255 BUG(); 1256 } else 1257 count = size >> PAGE_SHIFT; 1258 1259 spin_lock_irqsave(&mapping->lock, flags); 1260 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); 1261 spin_unlock_irqrestore(&mapping->lock, flags); 1262 } 1263 1264 /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */ 1265 static const int iommu_order_array[] = { 9, 8, 4, 0 }; 1266 1267 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 1268 gfp_t gfp, struct dma_attrs *attrs) 1269 { 1270 struct page **pages; 1271 int count = size >> PAGE_SHIFT; 1272 int array_size = count * sizeof(struct page *); 1273 int i = 0; 1274 int order_idx = 0; 1275 1276 if (array_size <= PAGE_SIZE) 1277 pages = kzalloc(array_size, GFP_KERNEL); 1278 else 1279 pages = vzalloc(array_size); 1280 if (!pages) 1281 return NULL; 1282 1283 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) 1284 { 1285 unsigned long order = get_order(size); 1286 struct page *page; 1287 1288 page = dma_alloc_from_contiguous(dev, count, order); 1289 if (!page) 1290 goto error; 1291 1292 __dma_clear_buffer(page, size); 1293 1294 for (i = 0; i < count; i++) 1295 pages[i] = page + i; 1296 1297 return pages; 1298 } 1299 1300 /* Go straight to 4K chunks if caller says it's OK. */ 1301 if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) 1302 order_idx = ARRAY_SIZE(iommu_order_array) - 1; 1303 1304 /* 1305 * IOMMU can map any pages, so himem can also be used here 1306 */ 1307 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 1308 1309 while (count) { 1310 int j, order; 1311 1312 order = iommu_order_array[order_idx]; 1313 1314 /* Drop down when we get small */ 1315 if (__fls(count) < order) { 1316 order_idx++; 1317 continue; 1318 } 1319 1320 if (order) { 1321 /* See if it's easy to allocate a high-order chunk */ 1322 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); 1323 1324 /* Go down a notch at first sign of pressure */ 1325 if (!pages[i]) { 1326 order_idx++; 1327 continue; 1328 } 1329 } else { 1330 pages[i] = alloc_pages(gfp, 0); 1331 if (!pages[i]) 1332 goto error; 1333 } 1334 1335 if (order) { 1336 split_page(pages[i], order); 1337 j = 1 << order; 1338 while (--j) 1339 pages[i + j] = pages[i] + j; 1340 } 1341 1342 __dma_clear_buffer(pages[i], PAGE_SIZE << order); 1343 i += 1 << order; 1344 count -= 1 << order; 1345 } 1346 1347 return pages; 1348 error: 1349 while (i--) 1350 if (pages[i]) 1351 __free_pages(pages[i], 0); 1352 kvfree(pages); 1353 return NULL; 1354 } 1355 1356 static int __iommu_free_buffer(struct device *dev, struct page **pages, 1357 size_t size, struct dma_attrs *attrs) 1358 { 1359 int count = size >> PAGE_SHIFT; 1360 int i; 1361 1362 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { 1363 dma_release_from_contiguous(dev, pages[0], count); 1364 } else { 1365 for (i = 0; i < count; i++) 1366 if (pages[i]) 1367 __free_pages(pages[i], 0); 1368 } 1369 1370 kvfree(pages); 1371 return 0; 1372 } 1373 1374 /* 1375 * Create a CPU mapping for a specified pages 1376 */ 1377 static void * 1378 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, 1379 const void *caller) 1380 { 1381 return dma_common_pages_remap(pages, size, 1382 VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller); 1383 } 1384 1385 /* 1386 * Create a mapping in device IO address space for specified pages 1387 */ 1388 static dma_addr_t 1389 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1390 { 1391 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1392 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1393 dma_addr_t dma_addr, iova; 1394 int i; 1395 1396 dma_addr = __alloc_iova(mapping, size); 1397 if (dma_addr == DMA_ERROR_CODE) 1398 return dma_addr; 1399 1400 iova = dma_addr; 1401 for (i = 0; i < count; ) { 1402 int ret; 1403 1404 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 1405 phys_addr_t phys = page_to_phys(pages[i]); 1406 unsigned int len, j; 1407 1408 for (j = i + 1; j < count; j++, next_pfn++) 1409 if (page_to_pfn(pages[j]) != next_pfn) 1410 break; 1411 1412 len = (j - i) << PAGE_SHIFT; 1413 ret = iommu_map(mapping->domain, iova, phys, len, 1414 IOMMU_READ|IOMMU_WRITE); 1415 if (ret < 0) 1416 goto fail; 1417 iova += len; 1418 i = j; 1419 } 1420 return dma_addr; 1421 fail: 1422 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 1423 __free_iova(mapping, dma_addr, size); 1424 return DMA_ERROR_CODE; 1425 } 1426 1427 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1428 { 1429 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1430 1431 /* 1432 * add optional in-page offset from iova to size and align 1433 * result to page size 1434 */ 1435 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1436 iova &= PAGE_MASK; 1437 1438 iommu_unmap(mapping->domain, iova, size); 1439 __free_iova(mapping, iova, size); 1440 return 0; 1441 } 1442 1443 static struct page **__atomic_get_pages(void *addr) 1444 { 1445 struct page *page; 1446 phys_addr_t phys; 1447 1448 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); 1449 page = phys_to_page(phys); 1450 1451 return (struct page **)page; 1452 } 1453 1454 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) 1455 { 1456 struct vm_struct *area; 1457 1458 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1459 return __atomic_get_pages(cpu_addr); 1460 1461 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1462 return cpu_addr; 1463 1464 area = find_vm_area(cpu_addr); 1465 if (area && (area->flags & VM_ARM_DMA_CONSISTENT)) 1466 return area->pages; 1467 return NULL; 1468 } 1469 1470 static void *__iommu_alloc_atomic(struct device *dev, size_t size, 1471 dma_addr_t *handle) 1472 { 1473 struct page *page; 1474 void *addr; 1475 1476 addr = __alloc_from_pool(size, &page); 1477 if (!addr) 1478 return NULL; 1479 1480 *handle = __iommu_create_mapping(dev, &page, size); 1481 if (*handle == DMA_ERROR_CODE) 1482 goto err_mapping; 1483 1484 return addr; 1485 1486 err_mapping: 1487 __free_from_pool(addr, size); 1488 return NULL; 1489 } 1490 1491 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1492 dma_addr_t handle, size_t size) 1493 { 1494 __iommu_remove_mapping(dev, handle, size); 1495 __free_from_pool(cpu_addr, size); 1496 } 1497 1498 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1499 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 1500 { 1501 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 1502 struct page **pages; 1503 void *addr = NULL; 1504 1505 *handle = DMA_ERROR_CODE; 1506 size = PAGE_ALIGN(size); 1507 1508 if (!gfpflags_allow_blocking(gfp)) 1509 return __iommu_alloc_atomic(dev, size, handle); 1510 1511 /* 1512 * Following is a work-around (a.k.a. hack) to prevent pages 1513 * with __GFP_COMP being passed to split_page() which cannot 1514 * handle them. The real problem is that this flag probably 1515 * should be 0 on ARM as it is not supported on this 1516 * platform; see CONFIG_HUGETLBFS. 1517 */ 1518 gfp &= ~(__GFP_COMP); 1519 1520 pages = __iommu_alloc_buffer(dev, size, gfp, attrs); 1521 if (!pages) 1522 return NULL; 1523 1524 *handle = __iommu_create_mapping(dev, pages, size); 1525 if (*handle == DMA_ERROR_CODE) 1526 goto err_buffer; 1527 1528 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1529 return pages; 1530 1531 addr = __iommu_alloc_remap(pages, size, gfp, prot, 1532 __builtin_return_address(0)); 1533 if (!addr) 1534 goto err_mapping; 1535 1536 return addr; 1537 1538 err_mapping: 1539 __iommu_remove_mapping(dev, *handle, size); 1540 err_buffer: 1541 __iommu_free_buffer(dev, pages, size, attrs); 1542 return NULL; 1543 } 1544 1545 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1546 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1547 struct dma_attrs *attrs) 1548 { 1549 unsigned long uaddr = vma->vm_start; 1550 unsigned long usize = vma->vm_end - vma->vm_start; 1551 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1552 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1553 unsigned long off = vma->vm_pgoff; 1554 1555 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1556 1557 if (!pages) 1558 return -ENXIO; 1559 1560 if (off >= nr_pages || (usize >> PAGE_SHIFT) > nr_pages - off) 1561 return -ENXIO; 1562 1563 pages += off; 1564 1565 do { 1566 int ret = vm_insert_page(vma, uaddr, *pages++); 1567 if (ret) { 1568 pr_err("Remapping memory failed: %d\n", ret); 1569 return ret; 1570 } 1571 uaddr += PAGE_SIZE; 1572 usize -= PAGE_SIZE; 1573 } while (usize > 0); 1574 1575 return 0; 1576 } 1577 1578 /* 1579 * free a page as defined by the above mapping. 1580 * Must not be called with IRQs disabled. 1581 */ 1582 void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1583 dma_addr_t handle, struct dma_attrs *attrs) 1584 { 1585 struct page **pages; 1586 size = PAGE_ALIGN(size); 1587 1588 if (__in_atomic_pool(cpu_addr, size)) { 1589 __iommu_free_atomic(dev, cpu_addr, handle, size); 1590 return; 1591 } 1592 1593 pages = __iommu_get_pages(cpu_addr, attrs); 1594 if (!pages) { 1595 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1596 return; 1597 } 1598 1599 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { 1600 dma_common_free_remap(cpu_addr, size, 1601 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 1602 } 1603 1604 __iommu_remove_mapping(dev, handle, size); 1605 __iommu_free_buffer(dev, pages, size, attrs); 1606 } 1607 1608 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1609 void *cpu_addr, dma_addr_t dma_addr, 1610 size_t size, struct dma_attrs *attrs) 1611 { 1612 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1613 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1614 1615 if (!pages) 1616 return -ENXIO; 1617 1618 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1619 GFP_KERNEL); 1620 } 1621 1622 static int __dma_direction_to_prot(enum dma_data_direction dir) 1623 { 1624 int prot; 1625 1626 switch (dir) { 1627 case DMA_BIDIRECTIONAL: 1628 prot = IOMMU_READ | IOMMU_WRITE; 1629 break; 1630 case DMA_TO_DEVICE: 1631 prot = IOMMU_READ; 1632 break; 1633 case DMA_FROM_DEVICE: 1634 prot = IOMMU_WRITE; 1635 break; 1636 default: 1637 prot = 0; 1638 } 1639 1640 return prot; 1641 } 1642 1643 /* 1644 * Map a part of the scatter-gather list into contiguous io address space 1645 */ 1646 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1647 size_t size, dma_addr_t *handle, 1648 enum dma_data_direction dir, struct dma_attrs *attrs, 1649 bool is_coherent) 1650 { 1651 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1652 dma_addr_t iova, iova_base; 1653 int ret = 0; 1654 unsigned int count; 1655 struct scatterlist *s; 1656 int prot; 1657 1658 size = PAGE_ALIGN(size); 1659 *handle = DMA_ERROR_CODE; 1660 1661 iova_base = iova = __alloc_iova(mapping, size); 1662 if (iova == DMA_ERROR_CODE) 1663 return -ENOMEM; 1664 1665 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1666 phys_addr_t phys = page_to_phys(sg_page(s)); 1667 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1668 1669 if (!is_coherent && 1670 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1671 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1672 1673 prot = __dma_direction_to_prot(dir); 1674 1675 ret = iommu_map(mapping->domain, iova, phys, len, prot); 1676 if (ret < 0) 1677 goto fail; 1678 count += len >> PAGE_SHIFT; 1679 iova += len; 1680 } 1681 *handle = iova_base; 1682 1683 return 0; 1684 fail: 1685 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1686 __free_iova(mapping, iova_base, size); 1687 return ret; 1688 } 1689 1690 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1691 enum dma_data_direction dir, struct dma_attrs *attrs, 1692 bool is_coherent) 1693 { 1694 struct scatterlist *s = sg, *dma = sg, *start = sg; 1695 int i, count = 0; 1696 unsigned int offset = s->offset; 1697 unsigned int size = s->offset + s->length; 1698 unsigned int max = dma_get_max_seg_size(dev); 1699 1700 for (i = 1; i < nents; i++) { 1701 s = sg_next(s); 1702 1703 s->dma_address = DMA_ERROR_CODE; 1704 s->dma_length = 0; 1705 1706 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1707 if (__map_sg_chunk(dev, start, size, &dma->dma_address, 1708 dir, attrs, is_coherent) < 0) 1709 goto bad_mapping; 1710 1711 dma->dma_address += offset; 1712 dma->dma_length = size - offset; 1713 1714 size = offset = s->offset; 1715 start = s; 1716 dma = sg_next(dma); 1717 count += 1; 1718 } 1719 size += s->length; 1720 } 1721 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, 1722 is_coherent) < 0) 1723 goto bad_mapping; 1724 1725 dma->dma_address += offset; 1726 dma->dma_length = size - offset; 1727 1728 return count+1; 1729 1730 bad_mapping: 1731 for_each_sg(sg, s, count, i) 1732 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1733 return 0; 1734 } 1735 1736 /** 1737 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1738 * @dev: valid struct device pointer 1739 * @sg: list of buffers 1740 * @nents: number of buffers to map 1741 * @dir: DMA transfer direction 1742 * 1743 * Map a set of i/o coherent buffers described by scatterlist in streaming 1744 * mode for DMA. The scatter gather list elements are merged together (if 1745 * possible) and tagged with the appropriate dma address and length. They are 1746 * obtained via sg_dma_{address,length}. 1747 */ 1748 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1749 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1750 { 1751 return __iommu_map_sg(dev, sg, nents, dir, attrs, true); 1752 } 1753 1754 /** 1755 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1756 * @dev: valid struct device pointer 1757 * @sg: list of buffers 1758 * @nents: number of buffers to map 1759 * @dir: DMA transfer direction 1760 * 1761 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1762 * The scatter gather list elements are merged together (if possible) and 1763 * tagged with the appropriate dma address and length. They are obtained via 1764 * sg_dma_{address,length}. 1765 */ 1766 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1767 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1768 { 1769 return __iommu_map_sg(dev, sg, nents, dir, attrs, false); 1770 } 1771 1772 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1773 int nents, enum dma_data_direction dir, struct dma_attrs *attrs, 1774 bool is_coherent) 1775 { 1776 struct scatterlist *s; 1777 int i; 1778 1779 for_each_sg(sg, s, nents, i) { 1780 if (sg_dma_len(s)) 1781 __iommu_remove_mapping(dev, sg_dma_address(s), 1782 sg_dma_len(s)); 1783 if (!is_coherent && 1784 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1785 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1786 s->length, dir); 1787 } 1788 } 1789 1790 /** 1791 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1792 * @dev: valid struct device pointer 1793 * @sg: list of buffers 1794 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1795 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1796 * 1797 * Unmap a set of streaming mode DMA translations. Again, CPU access 1798 * rules concerning calls here are the same as for dma_unmap_single(). 1799 */ 1800 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1801 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1802 { 1803 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); 1804 } 1805 1806 /** 1807 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1808 * @dev: valid struct device pointer 1809 * @sg: list of buffers 1810 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1811 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1812 * 1813 * Unmap a set of streaming mode DMA translations. Again, CPU access 1814 * rules concerning calls here are the same as for dma_unmap_single(). 1815 */ 1816 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1817 enum dma_data_direction dir, struct dma_attrs *attrs) 1818 { 1819 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); 1820 } 1821 1822 /** 1823 * arm_iommu_sync_sg_for_cpu 1824 * @dev: valid struct device pointer 1825 * @sg: list of buffers 1826 * @nents: number of buffers to map (returned from dma_map_sg) 1827 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1828 */ 1829 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1830 int nents, enum dma_data_direction dir) 1831 { 1832 struct scatterlist *s; 1833 int i; 1834 1835 for_each_sg(sg, s, nents, i) 1836 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1837 1838 } 1839 1840 /** 1841 * arm_iommu_sync_sg_for_device 1842 * @dev: valid struct device pointer 1843 * @sg: list of buffers 1844 * @nents: number of buffers to map (returned from dma_map_sg) 1845 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1846 */ 1847 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1848 int nents, enum dma_data_direction dir) 1849 { 1850 struct scatterlist *s; 1851 int i; 1852 1853 for_each_sg(sg, s, nents, i) 1854 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1855 } 1856 1857 1858 /** 1859 * arm_coherent_iommu_map_page 1860 * @dev: valid struct device pointer 1861 * @page: page that buffer resides in 1862 * @offset: offset into page for start of buffer 1863 * @size: size of buffer to map 1864 * @dir: DMA transfer direction 1865 * 1866 * Coherent IOMMU aware version of arm_dma_map_page() 1867 */ 1868 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, 1869 unsigned long offset, size_t size, enum dma_data_direction dir, 1870 struct dma_attrs *attrs) 1871 { 1872 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1873 dma_addr_t dma_addr; 1874 int ret, prot, len = PAGE_ALIGN(size + offset); 1875 1876 dma_addr = __alloc_iova(mapping, len); 1877 if (dma_addr == DMA_ERROR_CODE) 1878 return dma_addr; 1879 1880 prot = __dma_direction_to_prot(dir); 1881 1882 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); 1883 if (ret < 0) 1884 goto fail; 1885 1886 return dma_addr + offset; 1887 fail: 1888 __free_iova(mapping, dma_addr, len); 1889 return DMA_ERROR_CODE; 1890 } 1891 1892 /** 1893 * arm_iommu_map_page 1894 * @dev: valid struct device pointer 1895 * @page: page that buffer resides in 1896 * @offset: offset into page for start of buffer 1897 * @size: size of buffer to map 1898 * @dir: DMA transfer direction 1899 * 1900 * IOMMU aware version of arm_dma_map_page() 1901 */ 1902 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1903 unsigned long offset, size_t size, enum dma_data_direction dir, 1904 struct dma_attrs *attrs) 1905 { 1906 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1907 __dma_page_cpu_to_dev(page, offset, size, dir); 1908 1909 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); 1910 } 1911 1912 /** 1913 * arm_coherent_iommu_unmap_page 1914 * @dev: valid struct device pointer 1915 * @handle: DMA address of buffer 1916 * @size: size of buffer (same as passed to dma_map_page) 1917 * @dir: DMA transfer direction (same as passed to dma_map_page) 1918 * 1919 * Coherent IOMMU aware version of arm_dma_unmap_page() 1920 */ 1921 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1922 size_t size, enum dma_data_direction dir, 1923 struct dma_attrs *attrs) 1924 { 1925 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1926 dma_addr_t iova = handle & PAGE_MASK; 1927 int offset = handle & ~PAGE_MASK; 1928 int len = PAGE_ALIGN(size + offset); 1929 1930 if (!iova) 1931 return; 1932 1933 iommu_unmap(mapping->domain, iova, len); 1934 __free_iova(mapping, iova, len); 1935 } 1936 1937 /** 1938 * arm_iommu_unmap_page 1939 * @dev: valid struct device pointer 1940 * @handle: DMA address of buffer 1941 * @size: size of buffer (same as passed to dma_map_page) 1942 * @dir: DMA transfer direction (same as passed to dma_map_page) 1943 * 1944 * IOMMU aware version of arm_dma_unmap_page() 1945 */ 1946 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1947 size_t size, enum dma_data_direction dir, 1948 struct dma_attrs *attrs) 1949 { 1950 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1951 dma_addr_t iova = handle & PAGE_MASK; 1952 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1953 int offset = handle & ~PAGE_MASK; 1954 int len = PAGE_ALIGN(size + offset); 1955 1956 if (!iova) 1957 return; 1958 1959 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1960 __dma_page_dev_to_cpu(page, offset, size, dir); 1961 1962 iommu_unmap(mapping->domain, iova, len); 1963 __free_iova(mapping, iova, len); 1964 } 1965 1966 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1967 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1968 { 1969 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1970 dma_addr_t iova = handle & PAGE_MASK; 1971 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1972 unsigned int offset = handle & ~PAGE_MASK; 1973 1974 if (!iova) 1975 return; 1976 1977 __dma_page_dev_to_cpu(page, offset, size, dir); 1978 } 1979 1980 static void arm_iommu_sync_single_for_device(struct device *dev, 1981 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1982 { 1983 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1984 dma_addr_t iova = handle & PAGE_MASK; 1985 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1986 unsigned int offset = handle & ~PAGE_MASK; 1987 1988 if (!iova) 1989 return; 1990 1991 __dma_page_cpu_to_dev(page, offset, size, dir); 1992 } 1993 1994 struct dma_map_ops iommu_ops = { 1995 .alloc = arm_iommu_alloc_attrs, 1996 .free = arm_iommu_free_attrs, 1997 .mmap = arm_iommu_mmap_attrs, 1998 .get_sgtable = arm_iommu_get_sgtable, 1999 2000 .map_page = arm_iommu_map_page, 2001 .unmap_page = arm_iommu_unmap_page, 2002 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 2003 .sync_single_for_device = arm_iommu_sync_single_for_device, 2004 2005 .map_sg = arm_iommu_map_sg, 2006 .unmap_sg = arm_iommu_unmap_sg, 2007 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 2008 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 2009 2010 .set_dma_mask = arm_dma_set_mask, 2011 }; 2012 2013 struct dma_map_ops iommu_coherent_ops = { 2014 .alloc = arm_iommu_alloc_attrs, 2015 .free = arm_iommu_free_attrs, 2016 .mmap = arm_iommu_mmap_attrs, 2017 .get_sgtable = arm_iommu_get_sgtable, 2018 2019 .map_page = arm_coherent_iommu_map_page, 2020 .unmap_page = arm_coherent_iommu_unmap_page, 2021 2022 .map_sg = arm_coherent_iommu_map_sg, 2023 .unmap_sg = arm_coherent_iommu_unmap_sg, 2024 2025 .set_dma_mask = arm_dma_set_mask, 2026 }; 2027 2028 /** 2029 * arm_iommu_create_mapping 2030 * @bus: pointer to the bus holding the client device (for IOMMU calls) 2031 * @base: start address of the valid IO address space 2032 * @size: maximum size of the valid IO address space 2033 * 2034 * Creates a mapping structure which holds information about used/unused 2035 * IO address ranges, which is required to perform memory allocation and 2036 * mapping with IOMMU aware functions. 2037 * 2038 * The client device need to be attached to the mapping with 2039 * arm_iommu_attach_device function. 2040 */ 2041 struct dma_iommu_mapping * 2042 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) 2043 { 2044 unsigned int bits = size >> PAGE_SHIFT; 2045 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); 2046 struct dma_iommu_mapping *mapping; 2047 int extensions = 1; 2048 int err = -ENOMEM; 2049 2050 /* currently only 32-bit DMA address space is supported */ 2051 if (size > DMA_BIT_MASK(32) + 1) 2052 return ERR_PTR(-ERANGE); 2053 2054 if (!bitmap_size) 2055 return ERR_PTR(-EINVAL); 2056 2057 if (bitmap_size > PAGE_SIZE) { 2058 extensions = bitmap_size / PAGE_SIZE; 2059 bitmap_size = PAGE_SIZE; 2060 } 2061 2062 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 2063 if (!mapping) 2064 goto err; 2065 2066 mapping->bitmap_size = bitmap_size; 2067 mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *), 2068 GFP_KERNEL); 2069 if (!mapping->bitmaps) 2070 goto err2; 2071 2072 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); 2073 if (!mapping->bitmaps[0]) 2074 goto err3; 2075 2076 mapping->nr_bitmaps = 1; 2077 mapping->extensions = extensions; 2078 mapping->base = base; 2079 mapping->bits = BITS_PER_BYTE * bitmap_size; 2080 2081 spin_lock_init(&mapping->lock); 2082 2083 mapping->domain = iommu_domain_alloc(bus); 2084 if (!mapping->domain) 2085 goto err4; 2086 2087 kref_init(&mapping->kref); 2088 return mapping; 2089 err4: 2090 kfree(mapping->bitmaps[0]); 2091 err3: 2092 kfree(mapping->bitmaps); 2093 err2: 2094 kfree(mapping); 2095 err: 2096 return ERR_PTR(err); 2097 } 2098 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 2099 2100 static void release_iommu_mapping(struct kref *kref) 2101 { 2102 int i; 2103 struct dma_iommu_mapping *mapping = 2104 container_of(kref, struct dma_iommu_mapping, kref); 2105 2106 iommu_domain_free(mapping->domain); 2107 for (i = 0; i < mapping->nr_bitmaps; i++) 2108 kfree(mapping->bitmaps[i]); 2109 kfree(mapping->bitmaps); 2110 kfree(mapping); 2111 } 2112 2113 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping) 2114 { 2115 int next_bitmap; 2116 2117 if (mapping->nr_bitmaps >= mapping->extensions) 2118 return -EINVAL; 2119 2120 next_bitmap = mapping->nr_bitmaps; 2121 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, 2122 GFP_ATOMIC); 2123 if (!mapping->bitmaps[next_bitmap]) 2124 return -ENOMEM; 2125 2126 mapping->nr_bitmaps++; 2127 2128 return 0; 2129 } 2130 2131 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 2132 { 2133 if (mapping) 2134 kref_put(&mapping->kref, release_iommu_mapping); 2135 } 2136 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 2137 2138 static int __arm_iommu_attach_device(struct device *dev, 2139 struct dma_iommu_mapping *mapping) 2140 { 2141 int err; 2142 2143 err = iommu_attach_device(mapping->domain, dev); 2144 if (err) 2145 return err; 2146 2147 kref_get(&mapping->kref); 2148 to_dma_iommu_mapping(dev) = mapping; 2149 2150 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 2151 return 0; 2152 } 2153 2154 /** 2155 * arm_iommu_attach_device 2156 * @dev: valid struct device pointer 2157 * @mapping: io address space mapping structure (returned from 2158 * arm_iommu_create_mapping) 2159 * 2160 * Attaches specified io address space mapping to the provided device. 2161 * This replaces the dma operations (dma_map_ops pointer) with the 2162 * IOMMU aware version. 2163 * 2164 * More than one client might be attached to the same io address space 2165 * mapping. 2166 */ 2167 int arm_iommu_attach_device(struct device *dev, 2168 struct dma_iommu_mapping *mapping) 2169 { 2170 int err; 2171 2172 err = __arm_iommu_attach_device(dev, mapping); 2173 if (err) 2174 return err; 2175 2176 set_dma_ops(dev, &iommu_ops); 2177 return 0; 2178 } 2179 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 2180 2181 static void __arm_iommu_detach_device(struct device *dev) 2182 { 2183 struct dma_iommu_mapping *mapping; 2184 2185 mapping = to_dma_iommu_mapping(dev); 2186 if (!mapping) { 2187 dev_warn(dev, "Not attached\n"); 2188 return; 2189 } 2190 2191 iommu_detach_device(mapping->domain, dev); 2192 kref_put(&mapping->kref, release_iommu_mapping); 2193 to_dma_iommu_mapping(dev) = NULL; 2194 2195 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 2196 } 2197 2198 /** 2199 * arm_iommu_detach_device 2200 * @dev: valid struct device pointer 2201 * 2202 * Detaches the provided device from a previously attached map. 2203 * This voids the dma operations (dma_map_ops pointer) 2204 */ 2205 void arm_iommu_detach_device(struct device *dev) 2206 { 2207 __arm_iommu_detach_device(dev); 2208 set_dma_ops(dev, NULL); 2209 } 2210 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 2211 2212 static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) 2213 { 2214 return coherent ? &iommu_coherent_ops : &iommu_ops; 2215 } 2216 2217 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2218 const struct iommu_ops *iommu) 2219 { 2220 struct dma_iommu_mapping *mapping; 2221 2222 if (!iommu) 2223 return false; 2224 2225 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 2226 if (IS_ERR(mapping)) { 2227 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 2228 size, dev_name(dev)); 2229 return false; 2230 } 2231 2232 if (__arm_iommu_attach_device(dev, mapping)) { 2233 pr_warn("Failed to attached device %s to IOMMU_mapping\n", 2234 dev_name(dev)); 2235 arm_iommu_release_mapping(mapping); 2236 return false; 2237 } 2238 2239 return true; 2240 } 2241 2242 static void arm_teardown_iommu_dma_ops(struct device *dev) 2243 { 2244 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2245 2246 if (!mapping) 2247 return; 2248 2249 __arm_iommu_detach_device(dev); 2250 arm_iommu_release_mapping(mapping); 2251 } 2252 2253 #else 2254 2255 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2256 const struct iommu_ops *iommu) 2257 { 2258 return false; 2259 } 2260 2261 static void arm_teardown_iommu_dma_ops(struct device *dev) { } 2262 2263 #define arm_get_iommu_dma_map_ops arm_get_dma_map_ops 2264 2265 #endif /* CONFIG_ARM_DMA_USE_IOMMU */ 2266 2267 static struct dma_map_ops *arm_get_dma_map_ops(bool coherent) 2268 { 2269 return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; 2270 } 2271 2272 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 2273 const struct iommu_ops *iommu, bool coherent) 2274 { 2275 struct dma_map_ops *dma_ops; 2276 2277 dev->archdata.dma_coherent = coherent; 2278 if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) 2279 dma_ops = arm_get_iommu_dma_map_ops(coherent); 2280 else 2281 dma_ops = arm_get_dma_map_ops(coherent); 2282 2283 set_dma_ops(dev, dma_ops); 2284 } 2285 2286 void arch_teardown_dma_ops(struct device *dev) 2287 { 2288 arm_teardown_iommu_dma_ops(dev); 2289 } 2290