1 /* 2 * linux/arch/arm/mm/dma-mapping.c 3 * 4 * Copyright (C) 2000-2004 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * DMA uncached mapping support. 11 */ 12 #include <linux/bootmem.h> 13 #include <linux/module.h> 14 #include <linux/mm.h> 15 #include <linux/genalloc.h> 16 #include <linux/gfp.h> 17 #include <linux/errno.h> 18 #include <linux/list.h> 19 #include <linux/init.h> 20 #include <linux/device.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/dma-contiguous.h> 23 #include <linux/highmem.h> 24 #include <linux/memblock.h> 25 #include <linux/slab.h> 26 #include <linux/iommu.h> 27 #include <linux/io.h> 28 #include <linux/vmalloc.h> 29 #include <linux/sizes.h> 30 #include <linux/cma.h> 31 32 #include <asm/memory.h> 33 #include <asm/highmem.h> 34 #include <asm/cacheflush.h> 35 #include <asm/tlbflush.h> 36 #include <asm/mach/arch.h> 37 #include <asm/dma-iommu.h> 38 #include <asm/mach/map.h> 39 #include <asm/system_info.h> 40 #include <asm/dma-contiguous.h> 41 42 #include "dma.h" 43 #include "mm.h" 44 45 struct arm_dma_alloc_args { 46 struct device *dev; 47 size_t size; 48 gfp_t gfp; 49 pgprot_t prot; 50 const void *caller; 51 bool want_vaddr; 52 }; 53 54 struct arm_dma_free_args { 55 struct device *dev; 56 size_t size; 57 void *cpu_addr; 58 struct page *page; 59 bool want_vaddr; 60 }; 61 62 struct arm_dma_allocator { 63 void *(*alloc)(struct arm_dma_alloc_args *args, 64 struct page **ret_page); 65 void (*free)(struct arm_dma_free_args *args); 66 }; 67 68 struct arm_dma_buffer { 69 struct list_head list; 70 void *virt; 71 struct arm_dma_allocator *allocator; 72 }; 73 74 static LIST_HEAD(arm_dma_bufs); 75 static DEFINE_SPINLOCK(arm_dma_bufs_lock); 76 77 static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) 78 { 79 struct arm_dma_buffer *buf, *found = NULL; 80 unsigned long flags; 81 82 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 83 list_for_each_entry(buf, &arm_dma_bufs, list) { 84 if (buf->virt == virt) { 85 list_del(&buf->list); 86 found = buf; 87 break; 88 } 89 } 90 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 91 return found; 92 } 93 94 /* 95 * The DMA API is built upon the notion of "buffer ownership". A buffer 96 * is either exclusively owned by the CPU (and therefore may be accessed 97 * by it) or exclusively owned by the DMA device. These helper functions 98 * represent the transitions between these two ownership states. 99 * 100 * Note, however, that on later ARMs, this notion does not work due to 101 * speculative prefetches. We model our approach on the assumption that 102 * the CPU does do speculative prefetches, which means we clean caches 103 * before transfers and delay cache invalidation until transfer completion. 104 * 105 */ 106 static void __dma_page_cpu_to_dev(struct page *, unsigned long, 107 size_t, enum dma_data_direction); 108 static void __dma_page_dev_to_cpu(struct page *, unsigned long, 109 size_t, enum dma_data_direction); 110 111 /** 112 * arm_dma_map_page - map a portion of a page for streaming DMA 113 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 114 * @page: page that buffer resides in 115 * @offset: offset into page for start of buffer 116 * @size: size of buffer to map 117 * @dir: DMA transfer direction 118 * 119 * Ensure that any data held in the cache is appropriately discarded 120 * or written back. 121 * 122 * The device owns this memory once this call has completed. The CPU 123 * can regain ownership by calling dma_unmap_page(). 124 */ 125 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, 126 unsigned long offset, size_t size, enum dma_data_direction dir, 127 struct dma_attrs *attrs) 128 { 129 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 130 __dma_page_cpu_to_dev(page, offset, size, dir); 131 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 132 } 133 134 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, 135 unsigned long offset, size_t size, enum dma_data_direction dir, 136 struct dma_attrs *attrs) 137 { 138 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 139 } 140 141 /** 142 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() 143 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 144 * @handle: DMA address of buffer 145 * @size: size of buffer (same as passed to dma_map_page) 146 * @dir: DMA transfer direction (same as passed to dma_map_page) 147 * 148 * Unmap a page streaming mode DMA translation. The handle and size 149 * must match what was provided in the previous dma_map_page() call. 150 * All other usages are undefined. 151 * 152 * After this call, reads by the CPU to the buffer are guaranteed to see 153 * whatever the device wrote there. 154 */ 155 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, 156 size_t size, enum dma_data_direction dir, 157 struct dma_attrs *attrs) 158 { 159 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 160 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), 161 handle & ~PAGE_MASK, size, dir); 162 } 163 164 static void arm_dma_sync_single_for_cpu(struct device *dev, 165 dma_addr_t handle, size_t size, enum dma_data_direction dir) 166 { 167 unsigned int offset = handle & (PAGE_SIZE - 1); 168 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 169 __dma_page_dev_to_cpu(page, offset, size, dir); 170 } 171 172 static void arm_dma_sync_single_for_device(struct device *dev, 173 dma_addr_t handle, size_t size, enum dma_data_direction dir) 174 { 175 unsigned int offset = handle & (PAGE_SIZE - 1); 176 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 177 __dma_page_cpu_to_dev(page, offset, size, dir); 178 } 179 180 struct dma_map_ops arm_dma_ops = { 181 .alloc = arm_dma_alloc, 182 .free = arm_dma_free, 183 .mmap = arm_dma_mmap, 184 .get_sgtable = arm_dma_get_sgtable, 185 .map_page = arm_dma_map_page, 186 .unmap_page = arm_dma_unmap_page, 187 .map_sg = arm_dma_map_sg, 188 .unmap_sg = arm_dma_unmap_sg, 189 .sync_single_for_cpu = arm_dma_sync_single_for_cpu, 190 .sync_single_for_device = arm_dma_sync_single_for_device, 191 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, 192 .sync_sg_for_device = arm_dma_sync_sg_for_device, 193 .set_dma_mask = arm_dma_set_mask, 194 }; 195 EXPORT_SYMBOL(arm_dma_ops); 196 197 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 198 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); 199 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 200 dma_addr_t handle, struct dma_attrs *attrs); 201 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, 202 void *cpu_addr, dma_addr_t dma_addr, size_t size, 203 struct dma_attrs *attrs); 204 205 struct dma_map_ops arm_coherent_dma_ops = { 206 .alloc = arm_coherent_dma_alloc, 207 .free = arm_coherent_dma_free, 208 .mmap = arm_coherent_dma_mmap, 209 .get_sgtable = arm_dma_get_sgtable, 210 .map_page = arm_coherent_dma_map_page, 211 .map_sg = arm_dma_map_sg, 212 .set_dma_mask = arm_dma_set_mask, 213 }; 214 EXPORT_SYMBOL(arm_coherent_dma_ops); 215 216 static int __dma_supported(struct device *dev, u64 mask, bool warn) 217 { 218 unsigned long max_dma_pfn; 219 220 /* 221 * If the mask allows for more memory than we can address, 222 * and we actually have that much memory, then we must 223 * indicate that DMA to this device is not supported. 224 */ 225 if (sizeof(mask) != sizeof(dma_addr_t) && 226 mask > (dma_addr_t)~0 && 227 dma_to_pfn(dev, ~0) < max_pfn - 1) { 228 if (warn) { 229 dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n", 230 mask); 231 dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n"); 232 } 233 return 0; 234 } 235 236 max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); 237 238 /* 239 * Translate the device's DMA mask to a PFN limit. This 240 * PFN number includes the page which we can DMA to. 241 */ 242 if (dma_to_pfn(dev, mask) < max_dma_pfn) { 243 if (warn) 244 dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n", 245 mask, 246 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1, 247 max_dma_pfn + 1); 248 return 0; 249 } 250 251 return 1; 252 } 253 254 static u64 get_coherent_dma_mask(struct device *dev) 255 { 256 u64 mask = (u64)DMA_BIT_MASK(32); 257 258 if (dev) { 259 mask = dev->coherent_dma_mask; 260 261 /* 262 * Sanity check the DMA mask - it must be non-zero, and 263 * must be able to be satisfied by a DMA allocation. 264 */ 265 if (mask == 0) { 266 dev_warn(dev, "coherent DMA mask is unset\n"); 267 return 0; 268 } 269 270 if (!__dma_supported(dev, mask, true)) 271 return 0; 272 } 273 274 return mask; 275 } 276 277 static void __dma_clear_buffer(struct page *page, size_t size) 278 { 279 /* 280 * Ensure that the allocated pages are zeroed, and that any data 281 * lurking in the kernel direct-mapped region is invalidated. 282 */ 283 if (PageHighMem(page)) { 284 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 285 phys_addr_t end = base + size; 286 while (size > 0) { 287 void *ptr = kmap_atomic(page); 288 memset(ptr, 0, PAGE_SIZE); 289 dmac_flush_range(ptr, ptr + PAGE_SIZE); 290 kunmap_atomic(ptr); 291 page++; 292 size -= PAGE_SIZE; 293 } 294 outer_flush_range(base, end); 295 } else { 296 void *ptr = page_address(page); 297 memset(ptr, 0, size); 298 dmac_flush_range(ptr, ptr + size); 299 outer_flush_range(__pa(ptr), __pa(ptr) + size); 300 } 301 } 302 303 /* 304 * Allocate a DMA buffer for 'dev' of size 'size' using the 305 * specified gfp mask. Note that 'size' must be page aligned. 306 */ 307 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) 308 { 309 unsigned long order = get_order(size); 310 struct page *page, *p, *e; 311 312 page = alloc_pages(gfp, order); 313 if (!page) 314 return NULL; 315 316 /* 317 * Now split the huge page and free the excess pages 318 */ 319 split_page(page, order); 320 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 321 __free_page(p); 322 323 __dma_clear_buffer(page, size); 324 325 return page; 326 } 327 328 /* 329 * Free a DMA buffer. 'size' must be page aligned. 330 */ 331 static void __dma_free_buffer(struct page *page, size_t size) 332 { 333 struct page *e = page + (size >> PAGE_SHIFT); 334 335 while (page < e) { 336 __free_page(page); 337 page++; 338 } 339 } 340 341 #ifdef CONFIG_MMU 342 343 static void *__alloc_from_contiguous(struct device *dev, size_t size, 344 pgprot_t prot, struct page **ret_page, 345 const void *caller, bool want_vaddr); 346 347 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 348 pgprot_t prot, struct page **ret_page, 349 const void *caller, bool want_vaddr); 350 351 static void * 352 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 353 const void *caller) 354 { 355 /* 356 * DMA allocation can be mapped to user space, so lets 357 * set VM_USERMAP flags too. 358 */ 359 return dma_common_contiguous_remap(page, size, 360 VM_ARM_DMA_CONSISTENT | VM_USERMAP, 361 prot, caller); 362 } 363 364 static void __dma_free_remap(void *cpu_addr, size_t size) 365 { 366 dma_common_free_remap(cpu_addr, size, 367 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 368 } 369 370 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 371 static struct gen_pool *atomic_pool; 372 373 static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; 374 375 static int __init early_coherent_pool(char *p) 376 { 377 atomic_pool_size = memparse(p, &p); 378 return 0; 379 } 380 early_param("coherent_pool", early_coherent_pool); 381 382 void __init init_dma_coherent_pool_size(unsigned long size) 383 { 384 /* 385 * Catch any attempt to set the pool size too late. 386 */ 387 BUG_ON(atomic_pool); 388 389 /* 390 * Set architecture specific coherent pool size only if 391 * it has not been changed by kernel command line parameter. 392 */ 393 if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE) 394 atomic_pool_size = size; 395 } 396 397 /* 398 * Initialise the coherent pool for atomic allocations. 399 */ 400 static int __init atomic_pool_init(void) 401 { 402 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); 403 gfp_t gfp = GFP_KERNEL | GFP_DMA; 404 struct page *page; 405 void *ptr; 406 407 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 408 if (!atomic_pool) 409 goto out; 410 411 if (dev_get_cma_area(NULL)) 412 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 413 &page, atomic_pool_init, true); 414 else 415 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 416 &page, atomic_pool_init, true); 417 if (ptr) { 418 int ret; 419 420 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr, 421 page_to_phys(page), 422 atomic_pool_size, -1); 423 if (ret) 424 goto destroy_genpool; 425 426 gen_pool_set_algo(atomic_pool, 427 gen_pool_first_fit_order_align, 428 (void *)PAGE_SHIFT); 429 pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n", 430 atomic_pool_size / 1024); 431 return 0; 432 } 433 434 destroy_genpool: 435 gen_pool_destroy(atomic_pool); 436 atomic_pool = NULL; 437 out: 438 pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n", 439 atomic_pool_size / 1024); 440 return -ENOMEM; 441 } 442 /* 443 * CMA is activated by core_initcall, so we must be called after it. 444 */ 445 postcore_initcall(atomic_pool_init); 446 447 struct dma_contig_early_reserve { 448 phys_addr_t base; 449 unsigned long size; 450 }; 451 452 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 453 454 static int dma_mmu_remap_num __initdata; 455 456 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 457 { 458 dma_mmu_remap[dma_mmu_remap_num].base = base; 459 dma_mmu_remap[dma_mmu_remap_num].size = size; 460 dma_mmu_remap_num++; 461 } 462 463 void __init dma_contiguous_remap(void) 464 { 465 int i; 466 for (i = 0; i < dma_mmu_remap_num; i++) { 467 phys_addr_t start = dma_mmu_remap[i].base; 468 phys_addr_t end = start + dma_mmu_remap[i].size; 469 struct map_desc map; 470 unsigned long addr; 471 472 if (end > arm_lowmem_limit) 473 end = arm_lowmem_limit; 474 if (start >= end) 475 continue; 476 477 map.pfn = __phys_to_pfn(start); 478 map.virtual = __phys_to_virt(start); 479 map.length = end - start; 480 map.type = MT_MEMORY_DMA_READY; 481 482 /* 483 * Clear previous low-memory mapping to ensure that the 484 * TLB does not see any conflicting entries, then flush 485 * the TLB of the old entries before creating new mappings. 486 * 487 * This ensures that any speculatively loaded TLB entries 488 * (even though they may be rare) can not cause any problems, 489 * and ensures that this code is architecturally compliant. 490 */ 491 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 492 addr += PMD_SIZE) 493 pmd_clear(pmd_off_k(addr)); 494 495 flush_tlb_kernel_range(__phys_to_virt(start), 496 __phys_to_virt(end)); 497 498 iotable_init(&map, 1); 499 } 500 } 501 502 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, 503 void *data) 504 { 505 struct page *page = virt_to_page(addr); 506 pgprot_t prot = *(pgprot_t *)data; 507 508 set_pte_ext(pte, mk_pte(page, prot), 0); 509 return 0; 510 } 511 512 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 513 { 514 unsigned long start = (unsigned long) page_address(page); 515 unsigned end = start + size; 516 517 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 518 flush_tlb_kernel_range(start, end); 519 } 520 521 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 522 pgprot_t prot, struct page **ret_page, 523 const void *caller, bool want_vaddr) 524 { 525 struct page *page; 526 void *ptr = NULL; 527 page = __dma_alloc_buffer(dev, size, gfp); 528 if (!page) 529 return NULL; 530 if (!want_vaddr) 531 goto out; 532 533 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 534 if (!ptr) { 535 __dma_free_buffer(page, size); 536 return NULL; 537 } 538 539 out: 540 *ret_page = page; 541 return ptr; 542 } 543 544 static void *__alloc_from_pool(size_t size, struct page **ret_page) 545 { 546 unsigned long val; 547 void *ptr = NULL; 548 549 if (!atomic_pool) { 550 WARN(1, "coherent pool not initialised!\n"); 551 return NULL; 552 } 553 554 val = gen_pool_alloc(atomic_pool, size); 555 if (val) { 556 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 557 558 *ret_page = phys_to_page(phys); 559 ptr = (void *)val; 560 } 561 562 return ptr; 563 } 564 565 static bool __in_atomic_pool(void *start, size_t size) 566 { 567 return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); 568 } 569 570 static int __free_from_pool(void *start, size_t size) 571 { 572 if (!__in_atomic_pool(start, size)) 573 return 0; 574 575 gen_pool_free(atomic_pool, (unsigned long)start, size); 576 577 return 1; 578 } 579 580 static void *__alloc_from_contiguous(struct device *dev, size_t size, 581 pgprot_t prot, struct page **ret_page, 582 const void *caller, bool want_vaddr) 583 { 584 unsigned long order = get_order(size); 585 size_t count = size >> PAGE_SHIFT; 586 struct page *page; 587 void *ptr = NULL; 588 589 page = dma_alloc_from_contiguous(dev, count, order); 590 if (!page) 591 return NULL; 592 593 __dma_clear_buffer(page, size); 594 595 if (!want_vaddr) 596 goto out; 597 598 if (PageHighMem(page)) { 599 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); 600 if (!ptr) { 601 dma_release_from_contiguous(dev, page, count); 602 return NULL; 603 } 604 } else { 605 __dma_remap(page, size, prot); 606 ptr = page_address(page); 607 } 608 609 out: 610 *ret_page = page; 611 return ptr; 612 } 613 614 static void __free_from_contiguous(struct device *dev, struct page *page, 615 void *cpu_addr, size_t size, bool want_vaddr) 616 { 617 if (want_vaddr) { 618 if (PageHighMem(page)) 619 __dma_free_remap(cpu_addr, size); 620 else 621 __dma_remap(page, size, PAGE_KERNEL); 622 } 623 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 624 } 625 626 static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) 627 { 628 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? 629 pgprot_writecombine(prot) : 630 pgprot_dmacoherent(prot); 631 return prot; 632 } 633 634 #define nommu() 0 635 636 #else /* !CONFIG_MMU */ 637 638 #define nommu() 1 639 640 #define __get_dma_pgprot(attrs, prot) __pgprot(0) 641 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL 642 #define __alloc_from_pool(size, ret_page) NULL 643 #define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL 644 #define __free_from_pool(cpu_addr, size) do { } while (0) 645 #define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0) 646 #define __dma_free_remap(cpu_addr, size) do { } while (0) 647 648 #endif /* CONFIG_MMU */ 649 650 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 651 struct page **ret_page) 652 { 653 struct page *page; 654 page = __dma_alloc_buffer(dev, size, gfp); 655 if (!page) 656 return NULL; 657 658 *ret_page = page; 659 return page_address(page); 660 } 661 662 static void *simple_allocator_alloc(struct arm_dma_alloc_args *args, 663 struct page **ret_page) 664 { 665 return __alloc_simple_buffer(args->dev, args->size, args->gfp, 666 ret_page); 667 } 668 669 static void simple_allocator_free(struct arm_dma_free_args *args) 670 { 671 __dma_free_buffer(args->page, args->size); 672 } 673 674 static struct arm_dma_allocator simple_allocator = { 675 .alloc = simple_allocator_alloc, 676 .free = simple_allocator_free, 677 }; 678 679 static void *cma_allocator_alloc(struct arm_dma_alloc_args *args, 680 struct page **ret_page) 681 { 682 return __alloc_from_contiguous(args->dev, args->size, args->prot, 683 ret_page, args->caller, 684 args->want_vaddr); 685 } 686 687 static void cma_allocator_free(struct arm_dma_free_args *args) 688 { 689 __free_from_contiguous(args->dev, args->page, args->cpu_addr, 690 args->size, args->want_vaddr); 691 } 692 693 static struct arm_dma_allocator cma_allocator = { 694 .alloc = cma_allocator_alloc, 695 .free = cma_allocator_free, 696 }; 697 698 static void *pool_allocator_alloc(struct arm_dma_alloc_args *args, 699 struct page **ret_page) 700 { 701 return __alloc_from_pool(args->size, ret_page); 702 } 703 704 static void pool_allocator_free(struct arm_dma_free_args *args) 705 { 706 __free_from_pool(args->cpu_addr, args->size); 707 } 708 709 static struct arm_dma_allocator pool_allocator = { 710 .alloc = pool_allocator_alloc, 711 .free = pool_allocator_free, 712 }; 713 714 static void *remap_allocator_alloc(struct arm_dma_alloc_args *args, 715 struct page **ret_page) 716 { 717 return __alloc_remap_buffer(args->dev, args->size, args->gfp, 718 args->prot, ret_page, args->caller, 719 args->want_vaddr); 720 } 721 722 static void remap_allocator_free(struct arm_dma_free_args *args) 723 { 724 if (args->want_vaddr) 725 __dma_free_remap(args->cpu_addr, args->size); 726 727 __dma_free_buffer(args->page, args->size); 728 } 729 730 static struct arm_dma_allocator remap_allocator = { 731 .alloc = remap_allocator_alloc, 732 .free = remap_allocator_free, 733 }; 734 735 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 736 gfp_t gfp, pgprot_t prot, bool is_coherent, 737 struct dma_attrs *attrs, const void *caller) 738 { 739 u64 mask = get_coherent_dma_mask(dev); 740 struct page *page = NULL; 741 void *addr; 742 bool allowblock, cma; 743 struct arm_dma_buffer *buf; 744 struct arm_dma_alloc_args args = { 745 .dev = dev, 746 .size = PAGE_ALIGN(size), 747 .gfp = gfp, 748 .prot = prot, 749 .caller = caller, 750 .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), 751 }; 752 753 #ifdef CONFIG_DMA_API_DEBUG 754 u64 limit = (mask + 1) & ~mask; 755 if (limit && size >= limit) { 756 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 757 size, mask); 758 return NULL; 759 } 760 #endif 761 762 if (!mask) 763 return NULL; 764 765 buf = kzalloc(sizeof(*buf), gfp); 766 if (!buf) 767 return NULL; 768 769 if (mask < 0xffffffffULL) 770 gfp |= GFP_DMA; 771 772 /* 773 * Following is a work-around (a.k.a. hack) to prevent pages 774 * with __GFP_COMP being passed to split_page() which cannot 775 * handle them. The real problem is that this flag probably 776 * should be 0 on ARM as it is not supported on this 777 * platform; see CONFIG_HUGETLBFS. 778 */ 779 gfp &= ~(__GFP_COMP); 780 args.gfp = gfp; 781 782 *handle = DMA_ERROR_CODE; 783 allowblock = gfpflags_allow_blocking(gfp); 784 cma = allowblock ? dev_get_cma_area(dev) : false; 785 786 if (cma) 787 buf->allocator = &cma_allocator; 788 else if (nommu() || is_coherent) 789 buf->allocator = &simple_allocator; 790 else if (allowblock) 791 buf->allocator = &remap_allocator; 792 else 793 buf->allocator = &pool_allocator; 794 795 addr = buf->allocator->alloc(&args, &page); 796 797 if (page) { 798 unsigned long flags; 799 800 *handle = pfn_to_dma(dev, page_to_pfn(page)); 801 buf->virt = args.want_vaddr ? addr : page; 802 803 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 804 list_add(&buf->list, &arm_dma_bufs); 805 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 806 } else { 807 kfree(buf); 808 } 809 810 return args.want_vaddr ? addr : page; 811 } 812 813 /* 814 * Allocate DMA-coherent memory space and return both the kernel remapped 815 * virtual and bus address for that space. 816 */ 817 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 818 gfp_t gfp, struct dma_attrs *attrs) 819 { 820 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 821 822 return __dma_alloc(dev, size, handle, gfp, prot, false, 823 attrs, __builtin_return_address(0)); 824 } 825 826 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 827 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 828 { 829 return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, 830 attrs, __builtin_return_address(0)); 831 } 832 833 static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 834 void *cpu_addr, dma_addr_t dma_addr, size_t size, 835 struct dma_attrs *attrs) 836 { 837 int ret = -ENXIO; 838 #ifdef CONFIG_MMU 839 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 840 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 841 unsigned long pfn = dma_to_pfn(dev, dma_addr); 842 unsigned long off = vma->vm_pgoff; 843 844 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 845 return ret; 846 847 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 848 ret = remap_pfn_range(vma, vma->vm_start, 849 pfn + off, 850 vma->vm_end - vma->vm_start, 851 vma->vm_page_prot); 852 } 853 #endif /* CONFIG_MMU */ 854 855 return ret; 856 } 857 858 /* 859 * Create userspace mapping for the DMA-coherent memory. 860 */ 861 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, 862 void *cpu_addr, dma_addr_t dma_addr, size_t size, 863 struct dma_attrs *attrs) 864 { 865 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 866 } 867 868 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 869 void *cpu_addr, dma_addr_t dma_addr, size_t size, 870 struct dma_attrs *attrs) 871 { 872 #ifdef CONFIG_MMU 873 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 874 #endif /* CONFIG_MMU */ 875 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 876 } 877 878 /* 879 * Free a buffer as defined by the above mapping. 880 */ 881 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 882 dma_addr_t handle, struct dma_attrs *attrs, 883 bool is_coherent) 884 { 885 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 886 struct arm_dma_buffer *buf; 887 struct arm_dma_free_args args = { 888 .dev = dev, 889 .size = PAGE_ALIGN(size), 890 .cpu_addr = cpu_addr, 891 .page = page, 892 .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), 893 }; 894 895 buf = arm_dma_buffer_find(cpu_addr); 896 if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) 897 return; 898 899 buf->allocator->free(&args); 900 kfree(buf); 901 } 902 903 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 904 dma_addr_t handle, struct dma_attrs *attrs) 905 { 906 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); 907 } 908 909 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 910 dma_addr_t handle, struct dma_attrs *attrs) 911 { 912 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); 913 } 914 915 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 916 void *cpu_addr, dma_addr_t handle, size_t size, 917 struct dma_attrs *attrs) 918 { 919 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 920 int ret; 921 922 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 923 if (unlikely(ret)) 924 return ret; 925 926 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 927 return 0; 928 } 929 930 static void dma_cache_maint_page(struct page *page, unsigned long offset, 931 size_t size, enum dma_data_direction dir, 932 void (*op)(const void *, size_t, int)) 933 { 934 unsigned long pfn; 935 size_t left = size; 936 937 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 938 offset %= PAGE_SIZE; 939 940 /* 941 * A single sg entry may refer to multiple physically contiguous 942 * pages. But we still need to process highmem pages individually. 943 * If highmem is not configured then the bulk of this loop gets 944 * optimized out. 945 */ 946 do { 947 size_t len = left; 948 void *vaddr; 949 950 page = pfn_to_page(pfn); 951 952 if (PageHighMem(page)) { 953 if (len + offset > PAGE_SIZE) 954 len = PAGE_SIZE - offset; 955 956 if (cache_is_vipt_nonaliasing()) { 957 vaddr = kmap_atomic(page); 958 op(vaddr + offset, len, dir); 959 kunmap_atomic(vaddr); 960 } else { 961 vaddr = kmap_high_get(page); 962 if (vaddr) { 963 op(vaddr + offset, len, dir); 964 kunmap_high(page); 965 } 966 } 967 } else { 968 vaddr = page_address(page) + offset; 969 op(vaddr, len, dir); 970 } 971 offset = 0; 972 pfn++; 973 left -= len; 974 } while (left); 975 } 976 977 /* 978 * Make an area consistent for devices. 979 * Note: Drivers should NOT use this function directly, as it will break 980 * platforms with CONFIG_DMABOUNCE. 981 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 982 */ 983 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 984 size_t size, enum dma_data_direction dir) 985 { 986 phys_addr_t paddr; 987 988 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 989 990 paddr = page_to_phys(page) + off; 991 if (dir == DMA_FROM_DEVICE) { 992 outer_inv_range(paddr, paddr + size); 993 } else { 994 outer_clean_range(paddr, paddr + size); 995 } 996 /* FIXME: non-speculating: flush on bidirectional mappings? */ 997 } 998 999 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 1000 size_t size, enum dma_data_direction dir) 1001 { 1002 phys_addr_t paddr = page_to_phys(page) + off; 1003 1004 /* FIXME: non-speculating: not required */ 1005 /* in any case, don't bother invalidating if DMA to device */ 1006 if (dir != DMA_TO_DEVICE) { 1007 outer_inv_range(paddr, paddr + size); 1008 1009 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 1010 } 1011 1012 /* 1013 * Mark the D-cache clean for these pages to avoid extra flushing. 1014 */ 1015 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { 1016 unsigned long pfn; 1017 size_t left = size; 1018 1019 pfn = page_to_pfn(page) + off / PAGE_SIZE; 1020 off %= PAGE_SIZE; 1021 if (off) { 1022 pfn++; 1023 left -= PAGE_SIZE - off; 1024 } 1025 while (left >= PAGE_SIZE) { 1026 page = pfn_to_page(pfn++); 1027 set_bit(PG_dcache_clean, &page->flags); 1028 left -= PAGE_SIZE; 1029 } 1030 } 1031 } 1032 1033 /** 1034 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA 1035 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1036 * @sg: list of buffers 1037 * @nents: number of buffers to map 1038 * @dir: DMA transfer direction 1039 * 1040 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1041 * This is the scatter-gather version of the dma_map_single interface. 1042 * Here the scatter gather list elements are each tagged with the 1043 * appropriate dma address and length. They are obtained via 1044 * sg_dma_{address,length}. 1045 * 1046 * Device ownership issues as mentioned for dma_map_single are the same 1047 * here. 1048 */ 1049 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1050 enum dma_data_direction dir, struct dma_attrs *attrs) 1051 { 1052 struct dma_map_ops *ops = get_dma_ops(dev); 1053 struct scatterlist *s; 1054 int i, j; 1055 1056 for_each_sg(sg, s, nents, i) { 1057 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1058 s->dma_length = s->length; 1059 #endif 1060 s->dma_address = ops->map_page(dev, sg_page(s), s->offset, 1061 s->length, dir, attrs); 1062 if (dma_mapping_error(dev, s->dma_address)) 1063 goto bad_mapping; 1064 } 1065 return nents; 1066 1067 bad_mapping: 1068 for_each_sg(sg, s, i, j) 1069 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 1070 return 0; 1071 } 1072 1073 /** 1074 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1075 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1076 * @sg: list of buffers 1077 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1078 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1079 * 1080 * Unmap a set of streaming mode DMA translations. Again, CPU access 1081 * rules concerning calls here are the same as for dma_unmap_single(). 1082 */ 1083 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1084 enum dma_data_direction dir, struct dma_attrs *attrs) 1085 { 1086 struct dma_map_ops *ops = get_dma_ops(dev); 1087 struct scatterlist *s; 1088 1089 int i; 1090 1091 for_each_sg(sg, s, nents, i) 1092 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 1093 } 1094 1095 /** 1096 * arm_dma_sync_sg_for_cpu 1097 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1098 * @sg: list of buffers 1099 * @nents: number of buffers to map (returned from dma_map_sg) 1100 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1101 */ 1102 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1103 int nents, enum dma_data_direction dir) 1104 { 1105 struct dma_map_ops *ops = get_dma_ops(dev); 1106 struct scatterlist *s; 1107 int i; 1108 1109 for_each_sg(sg, s, nents, i) 1110 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, 1111 dir); 1112 } 1113 1114 /** 1115 * arm_dma_sync_sg_for_device 1116 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1117 * @sg: list of buffers 1118 * @nents: number of buffers to map (returned from dma_map_sg) 1119 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1120 */ 1121 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1122 int nents, enum dma_data_direction dir) 1123 { 1124 struct dma_map_ops *ops = get_dma_ops(dev); 1125 struct scatterlist *s; 1126 int i; 1127 1128 for_each_sg(sg, s, nents, i) 1129 ops->sync_single_for_device(dev, sg_dma_address(s), s->length, 1130 dir); 1131 } 1132 1133 /* 1134 * Return whether the given device DMA address mask can be supported 1135 * properly. For example, if your device can only drive the low 24-bits 1136 * during bus mastering, then you would pass 0x00ffffff as the mask 1137 * to this function. 1138 */ 1139 int dma_supported(struct device *dev, u64 mask) 1140 { 1141 return __dma_supported(dev, mask, false); 1142 } 1143 EXPORT_SYMBOL(dma_supported); 1144 1145 int arm_dma_set_mask(struct device *dev, u64 dma_mask) 1146 { 1147 if (!dev->dma_mask || !dma_supported(dev, dma_mask)) 1148 return -EIO; 1149 1150 *dev->dma_mask = dma_mask; 1151 1152 return 0; 1153 } 1154 1155 #define PREALLOC_DMA_DEBUG_ENTRIES 4096 1156 1157 static int __init dma_debug_do_init(void) 1158 { 1159 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 1160 return 0; 1161 } 1162 fs_initcall(dma_debug_do_init); 1163 1164 #ifdef CONFIG_ARM_DMA_USE_IOMMU 1165 1166 /* IOMMU */ 1167 1168 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); 1169 1170 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 1171 size_t size) 1172 { 1173 unsigned int order = get_order(size); 1174 unsigned int align = 0; 1175 unsigned int count, start; 1176 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1177 unsigned long flags; 1178 dma_addr_t iova; 1179 int i; 1180 1181 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 1182 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 1183 1184 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1185 align = (1 << order) - 1; 1186 1187 spin_lock_irqsave(&mapping->lock, flags); 1188 for (i = 0; i < mapping->nr_bitmaps; i++) { 1189 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1190 mapping->bits, 0, count, align); 1191 1192 if (start > mapping->bits) 1193 continue; 1194 1195 bitmap_set(mapping->bitmaps[i], start, count); 1196 break; 1197 } 1198 1199 /* 1200 * No unused range found. Try to extend the existing mapping 1201 * and perform a second attempt to reserve an IO virtual 1202 * address range of size bytes. 1203 */ 1204 if (i == mapping->nr_bitmaps) { 1205 if (extend_iommu_mapping(mapping)) { 1206 spin_unlock_irqrestore(&mapping->lock, flags); 1207 return DMA_ERROR_CODE; 1208 } 1209 1210 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1211 mapping->bits, 0, count, align); 1212 1213 if (start > mapping->bits) { 1214 spin_unlock_irqrestore(&mapping->lock, flags); 1215 return DMA_ERROR_CODE; 1216 } 1217 1218 bitmap_set(mapping->bitmaps[i], start, count); 1219 } 1220 spin_unlock_irqrestore(&mapping->lock, flags); 1221 1222 iova = mapping->base + (mapping_size * i); 1223 iova += start << PAGE_SHIFT; 1224 1225 return iova; 1226 } 1227 1228 static inline void __free_iova(struct dma_iommu_mapping *mapping, 1229 dma_addr_t addr, size_t size) 1230 { 1231 unsigned int start, count; 1232 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1233 unsigned long flags; 1234 dma_addr_t bitmap_base; 1235 u32 bitmap_index; 1236 1237 if (!size) 1238 return; 1239 1240 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; 1241 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); 1242 1243 bitmap_base = mapping->base + mapping_size * bitmap_index; 1244 1245 start = (addr - bitmap_base) >> PAGE_SHIFT; 1246 1247 if (addr + size > bitmap_base + mapping_size) { 1248 /* 1249 * The address range to be freed reaches into the iova 1250 * range of the next bitmap. This should not happen as 1251 * we don't allow this in __alloc_iova (at the 1252 * moment). 1253 */ 1254 BUG(); 1255 } else 1256 count = size >> PAGE_SHIFT; 1257 1258 spin_lock_irqsave(&mapping->lock, flags); 1259 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); 1260 spin_unlock_irqrestore(&mapping->lock, flags); 1261 } 1262 1263 /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */ 1264 static const int iommu_order_array[] = { 9, 8, 4, 0 }; 1265 1266 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 1267 gfp_t gfp, struct dma_attrs *attrs) 1268 { 1269 struct page **pages; 1270 int count = size >> PAGE_SHIFT; 1271 int array_size = count * sizeof(struct page *); 1272 int i = 0; 1273 int order_idx = 0; 1274 1275 if (array_size <= PAGE_SIZE) 1276 pages = kzalloc(array_size, GFP_KERNEL); 1277 else 1278 pages = vzalloc(array_size); 1279 if (!pages) 1280 return NULL; 1281 1282 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) 1283 { 1284 unsigned long order = get_order(size); 1285 struct page *page; 1286 1287 page = dma_alloc_from_contiguous(dev, count, order); 1288 if (!page) 1289 goto error; 1290 1291 __dma_clear_buffer(page, size); 1292 1293 for (i = 0; i < count; i++) 1294 pages[i] = page + i; 1295 1296 return pages; 1297 } 1298 1299 /* Go straight to 4K chunks if caller says it's OK. */ 1300 if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) 1301 order_idx = ARRAY_SIZE(iommu_order_array) - 1; 1302 1303 /* 1304 * IOMMU can map any pages, so himem can also be used here 1305 */ 1306 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 1307 1308 while (count) { 1309 int j, order; 1310 1311 order = iommu_order_array[order_idx]; 1312 1313 /* Drop down when we get small */ 1314 if (__fls(count) < order) { 1315 order_idx++; 1316 continue; 1317 } 1318 1319 if (order) { 1320 /* See if it's easy to allocate a high-order chunk */ 1321 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); 1322 1323 /* Go down a notch at first sign of pressure */ 1324 if (!pages[i]) { 1325 order_idx++; 1326 continue; 1327 } 1328 } else { 1329 pages[i] = alloc_pages(gfp, 0); 1330 if (!pages[i]) 1331 goto error; 1332 } 1333 1334 if (order) { 1335 split_page(pages[i], order); 1336 j = 1 << order; 1337 while (--j) 1338 pages[i + j] = pages[i] + j; 1339 } 1340 1341 __dma_clear_buffer(pages[i], PAGE_SIZE << order); 1342 i += 1 << order; 1343 count -= 1 << order; 1344 } 1345 1346 return pages; 1347 error: 1348 while (i--) 1349 if (pages[i]) 1350 __free_pages(pages[i], 0); 1351 kvfree(pages); 1352 return NULL; 1353 } 1354 1355 static int __iommu_free_buffer(struct device *dev, struct page **pages, 1356 size_t size, struct dma_attrs *attrs) 1357 { 1358 int count = size >> PAGE_SHIFT; 1359 int i; 1360 1361 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { 1362 dma_release_from_contiguous(dev, pages[0], count); 1363 } else { 1364 for (i = 0; i < count; i++) 1365 if (pages[i]) 1366 __free_pages(pages[i], 0); 1367 } 1368 1369 kvfree(pages); 1370 return 0; 1371 } 1372 1373 /* 1374 * Create a CPU mapping for a specified pages 1375 */ 1376 static void * 1377 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, 1378 const void *caller) 1379 { 1380 return dma_common_pages_remap(pages, size, 1381 VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller); 1382 } 1383 1384 /* 1385 * Create a mapping in device IO address space for specified pages 1386 */ 1387 static dma_addr_t 1388 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1389 { 1390 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1391 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1392 dma_addr_t dma_addr, iova; 1393 int i; 1394 1395 dma_addr = __alloc_iova(mapping, size); 1396 if (dma_addr == DMA_ERROR_CODE) 1397 return dma_addr; 1398 1399 iova = dma_addr; 1400 for (i = 0; i < count; ) { 1401 int ret; 1402 1403 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 1404 phys_addr_t phys = page_to_phys(pages[i]); 1405 unsigned int len, j; 1406 1407 for (j = i + 1; j < count; j++, next_pfn++) 1408 if (page_to_pfn(pages[j]) != next_pfn) 1409 break; 1410 1411 len = (j - i) << PAGE_SHIFT; 1412 ret = iommu_map(mapping->domain, iova, phys, len, 1413 IOMMU_READ|IOMMU_WRITE); 1414 if (ret < 0) 1415 goto fail; 1416 iova += len; 1417 i = j; 1418 } 1419 return dma_addr; 1420 fail: 1421 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 1422 __free_iova(mapping, dma_addr, size); 1423 return DMA_ERROR_CODE; 1424 } 1425 1426 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1427 { 1428 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1429 1430 /* 1431 * add optional in-page offset from iova to size and align 1432 * result to page size 1433 */ 1434 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1435 iova &= PAGE_MASK; 1436 1437 iommu_unmap(mapping->domain, iova, size); 1438 __free_iova(mapping, iova, size); 1439 return 0; 1440 } 1441 1442 static struct page **__atomic_get_pages(void *addr) 1443 { 1444 struct page *page; 1445 phys_addr_t phys; 1446 1447 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); 1448 page = phys_to_page(phys); 1449 1450 return (struct page **)page; 1451 } 1452 1453 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) 1454 { 1455 struct vm_struct *area; 1456 1457 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1458 return __atomic_get_pages(cpu_addr); 1459 1460 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1461 return cpu_addr; 1462 1463 area = find_vm_area(cpu_addr); 1464 if (area && (area->flags & VM_ARM_DMA_CONSISTENT)) 1465 return area->pages; 1466 return NULL; 1467 } 1468 1469 static void *__iommu_alloc_atomic(struct device *dev, size_t size, 1470 dma_addr_t *handle) 1471 { 1472 struct page *page; 1473 void *addr; 1474 1475 addr = __alloc_from_pool(size, &page); 1476 if (!addr) 1477 return NULL; 1478 1479 *handle = __iommu_create_mapping(dev, &page, size); 1480 if (*handle == DMA_ERROR_CODE) 1481 goto err_mapping; 1482 1483 return addr; 1484 1485 err_mapping: 1486 __free_from_pool(addr, size); 1487 return NULL; 1488 } 1489 1490 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1491 dma_addr_t handle, size_t size) 1492 { 1493 __iommu_remove_mapping(dev, handle, size); 1494 __free_from_pool(cpu_addr, size); 1495 } 1496 1497 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1498 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 1499 { 1500 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 1501 struct page **pages; 1502 void *addr = NULL; 1503 1504 *handle = DMA_ERROR_CODE; 1505 size = PAGE_ALIGN(size); 1506 1507 if (!gfpflags_allow_blocking(gfp)) 1508 return __iommu_alloc_atomic(dev, size, handle); 1509 1510 /* 1511 * Following is a work-around (a.k.a. hack) to prevent pages 1512 * with __GFP_COMP being passed to split_page() which cannot 1513 * handle them. The real problem is that this flag probably 1514 * should be 0 on ARM as it is not supported on this 1515 * platform; see CONFIG_HUGETLBFS. 1516 */ 1517 gfp &= ~(__GFP_COMP); 1518 1519 pages = __iommu_alloc_buffer(dev, size, gfp, attrs); 1520 if (!pages) 1521 return NULL; 1522 1523 *handle = __iommu_create_mapping(dev, pages, size); 1524 if (*handle == DMA_ERROR_CODE) 1525 goto err_buffer; 1526 1527 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1528 return pages; 1529 1530 addr = __iommu_alloc_remap(pages, size, gfp, prot, 1531 __builtin_return_address(0)); 1532 if (!addr) 1533 goto err_mapping; 1534 1535 return addr; 1536 1537 err_mapping: 1538 __iommu_remove_mapping(dev, *handle, size); 1539 err_buffer: 1540 __iommu_free_buffer(dev, pages, size, attrs); 1541 return NULL; 1542 } 1543 1544 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1545 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1546 struct dma_attrs *attrs) 1547 { 1548 unsigned long uaddr = vma->vm_start; 1549 unsigned long usize = vma->vm_end - vma->vm_start; 1550 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1551 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1552 unsigned long off = vma->vm_pgoff; 1553 1554 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1555 1556 if (!pages) 1557 return -ENXIO; 1558 1559 if (off >= nr_pages || (usize >> PAGE_SHIFT) > nr_pages - off) 1560 return -ENXIO; 1561 1562 pages += off; 1563 1564 do { 1565 int ret = vm_insert_page(vma, uaddr, *pages++); 1566 if (ret) { 1567 pr_err("Remapping memory failed: %d\n", ret); 1568 return ret; 1569 } 1570 uaddr += PAGE_SIZE; 1571 usize -= PAGE_SIZE; 1572 } while (usize > 0); 1573 1574 return 0; 1575 } 1576 1577 /* 1578 * free a page as defined by the above mapping. 1579 * Must not be called with IRQs disabled. 1580 */ 1581 void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1582 dma_addr_t handle, struct dma_attrs *attrs) 1583 { 1584 struct page **pages; 1585 size = PAGE_ALIGN(size); 1586 1587 if (__in_atomic_pool(cpu_addr, size)) { 1588 __iommu_free_atomic(dev, cpu_addr, handle, size); 1589 return; 1590 } 1591 1592 pages = __iommu_get_pages(cpu_addr, attrs); 1593 if (!pages) { 1594 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1595 return; 1596 } 1597 1598 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { 1599 dma_common_free_remap(cpu_addr, size, 1600 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 1601 } 1602 1603 __iommu_remove_mapping(dev, handle, size); 1604 __iommu_free_buffer(dev, pages, size, attrs); 1605 } 1606 1607 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1608 void *cpu_addr, dma_addr_t dma_addr, 1609 size_t size, struct dma_attrs *attrs) 1610 { 1611 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1612 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1613 1614 if (!pages) 1615 return -ENXIO; 1616 1617 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1618 GFP_KERNEL); 1619 } 1620 1621 static int __dma_direction_to_prot(enum dma_data_direction dir) 1622 { 1623 int prot; 1624 1625 switch (dir) { 1626 case DMA_BIDIRECTIONAL: 1627 prot = IOMMU_READ | IOMMU_WRITE; 1628 break; 1629 case DMA_TO_DEVICE: 1630 prot = IOMMU_READ; 1631 break; 1632 case DMA_FROM_DEVICE: 1633 prot = IOMMU_WRITE; 1634 break; 1635 default: 1636 prot = 0; 1637 } 1638 1639 return prot; 1640 } 1641 1642 /* 1643 * Map a part of the scatter-gather list into contiguous io address space 1644 */ 1645 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1646 size_t size, dma_addr_t *handle, 1647 enum dma_data_direction dir, struct dma_attrs *attrs, 1648 bool is_coherent) 1649 { 1650 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1651 dma_addr_t iova, iova_base; 1652 int ret = 0; 1653 unsigned int count; 1654 struct scatterlist *s; 1655 int prot; 1656 1657 size = PAGE_ALIGN(size); 1658 *handle = DMA_ERROR_CODE; 1659 1660 iova_base = iova = __alloc_iova(mapping, size); 1661 if (iova == DMA_ERROR_CODE) 1662 return -ENOMEM; 1663 1664 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1665 phys_addr_t phys = page_to_phys(sg_page(s)); 1666 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1667 1668 if (!is_coherent && 1669 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1670 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1671 1672 prot = __dma_direction_to_prot(dir); 1673 1674 ret = iommu_map(mapping->domain, iova, phys, len, prot); 1675 if (ret < 0) 1676 goto fail; 1677 count += len >> PAGE_SHIFT; 1678 iova += len; 1679 } 1680 *handle = iova_base; 1681 1682 return 0; 1683 fail: 1684 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1685 __free_iova(mapping, iova_base, size); 1686 return ret; 1687 } 1688 1689 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1690 enum dma_data_direction dir, struct dma_attrs *attrs, 1691 bool is_coherent) 1692 { 1693 struct scatterlist *s = sg, *dma = sg, *start = sg; 1694 int i, count = 0; 1695 unsigned int offset = s->offset; 1696 unsigned int size = s->offset + s->length; 1697 unsigned int max = dma_get_max_seg_size(dev); 1698 1699 for (i = 1; i < nents; i++) { 1700 s = sg_next(s); 1701 1702 s->dma_address = DMA_ERROR_CODE; 1703 s->dma_length = 0; 1704 1705 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1706 if (__map_sg_chunk(dev, start, size, &dma->dma_address, 1707 dir, attrs, is_coherent) < 0) 1708 goto bad_mapping; 1709 1710 dma->dma_address += offset; 1711 dma->dma_length = size - offset; 1712 1713 size = offset = s->offset; 1714 start = s; 1715 dma = sg_next(dma); 1716 count += 1; 1717 } 1718 size += s->length; 1719 } 1720 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, 1721 is_coherent) < 0) 1722 goto bad_mapping; 1723 1724 dma->dma_address += offset; 1725 dma->dma_length = size - offset; 1726 1727 return count+1; 1728 1729 bad_mapping: 1730 for_each_sg(sg, s, count, i) 1731 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1732 return 0; 1733 } 1734 1735 /** 1736 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1737 * @dev: valid struct device pointer 1738 * @sg: list of buffers 1739 * @nents: number of buffers to map 1740 * @dir: DMA transfer direction 1741 * 1742 * Map a set of i/o coherent buffers described by scatterlist in streaming 1743 * mode for DMA. The scatter gather list elements are merged together (if 1744 * possible) and tagged with the appropriate dma address and length. They are 1745 * obtained via sg_dma_{address,length}. 1746 */ 1747 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1748 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1749 { 1750 return __iommu_map_sg(dev, sg, nents, dir, attrs, true); 1751 } 1752 1753 /** 1754 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1755 * @dev: valid struct device pointer 1756 * @sg: list of buffers 1757 * @nents: number of buffers to map 1758 * @dir: DMA transfer direction 1759 * 1760 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1761 * The scatter gather list elements are merged together (if possible) and 1762 * tagged with the appropriate dma address and length. They are obtained via 1763 * sg_dma_{address,length}. 1764 */ 1765 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1766 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1767 { 1768 return __iommu_map_sg(dev, sg, nents, dir, attrs, false); 1769 } 1770 1771 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1772 int nents, enum dma_data_direction dir, struct dma_attrs *attrs, 1773 bool is_coherent) 1774 { 1775 struct scatterlist *s; 1776 int i; 1777 1778 for_each_sg(sg, s, nents, i) { 1779 if (sg_dma_len(s)) 1780 __iommu_remove_mapping(dev, sg_dma_address(s), 1781 sg_dma_len(s)); 1782 if (!is_coherent && 1783 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1784 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1785 s->length, dir); 1786 } 1787 } 1788 1789 /** 1790 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1791 * @dev: valid struct device pointer 1792 * @sg: list of buffers 1793 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1794 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1795 * 1796 * Unmap a set of streaming mode DMA translations. Again, CPU access 1797 * rules concerning calls here are the same as for dma_unmap_single(). 1798 */ 1799 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1800 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1801 { 1802 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); 1803 } 1804 1805 /** 1806 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1807 * @dev: valid struct device pointer 1808 * @sg: list of buffers 1809 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1810 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1811 * 1812 * Unmap a set of streaming mode DMA translations. Again, CPU access 1813 * rules concerning calls here are the same as for dma_unmap_single(). 1814 */ 1815 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1816 enum dma_data_direction dir, struct dma_attrs *attrs) 1817 { 1818 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); 1819 } 1820 1821 /** 1822 * arm_iommu_sync_sg_for_cpu 1823 * @dev: valid struct device pointer 1824 * @sg: list of buffers 1825 * @nents: number of buffers to map (returned from dma_map_sg) 1826 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1827 */ 1828 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1829 int nents, enum dma_data_direction dir) 1830 { 1831 struct scatterlist *s; 1832 int i; 1833 1834 for_each_sg(sg, s, nents, i) 1835 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1836 1837 } 1838 1839 /** 1840 * arm_iommu_sync_sg_for_device 1841 * @dev: valid struct device pointer 1842 * @sg: list of buffers 1843 * @nents: number of buffers to map (returned from dma_map_sg) 1844 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1845 */ 1846 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1847 int nents, enum dma_data_direction dir) 1848 { 1849 struct scatterlist *s; 1850 int i; 1851 1852 for_each_sg(sg, s, nents, i) 1853 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1854 } 1855 1856 1857 /** 1858 * arm_coherent_iommu_map_page 1859 * @dev: valid struct device pointer 1860 * @page: page that buffer resides in 1861 * @offset: offset into page for start of buffer 1862 * @size: size of buffer to map 1863 * @dir: DMA transfer direction 1864 * 1865 * Coherent IOMMU aware version of arm_dma_map_page() 1866 */ 1867 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, 1868 unsigned long offset, size_t size, enum dma_data_direction dir, 1869 struct dma_attrs *attrs) 1870 { 1871 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1872 dma_addr_t dma_addr; 1873 int ret, prot, len = PAGE_ALIGN(size + offset); 1874 1875 dma_addr = __alloc_iova(mapping, len); 1876 if (dma_addr == DMA_ERROR_CODE) 1877 return dma_addr; 1878 1879 prot = __dma_direction_to_prot(dir); 1880 1881 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); 1882 if (ret < 0) 1883 goto fail; 1884 1885 return dma_addr + offset; 1886 fail: 1887 __free_iova(mapping, dma_addr, len); 1888 return DMA_ERROR_CODE; 1889 } 1890 1891 /** 1892 * arm_iommu_map_page 1893 * @dev: valid struct device pointer 1894 * @page: page that buffer resides in 1895 * @offset: offset into page for start of buffer 1896 * @size: size of buffer to map 1897 * @dir: DMA transfer direction 1898 * 1899 * IOMMU aware version of arm_dma_map_page() 1900 */ 1901 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1902 unsigned long offset, size_t size, enum dma_data_direction dir, 1903 struct dma_attrs *attrs) 1904 { 1905 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1906 __dma_page_cpu_to_dev(page, offset, size, dir); 1907 1908 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); 1909 } 1910 1911 /** 1912 * arm_coherent_iommu_unmap_page 1913 * @dev: valid struct device pointer 1914 * @handle: DMA address of buffer 1915 * @size: size of buffer (same as passed to dma_map_page) 1916 * @dir: DMA transfer direction (same as passed to dma_map_page) 1917 * 1918 * Coherent IOMMU aware version of arm_dma_unmap_page() 1919 */ 1920 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1921 size_t size, enum dma_data_direction dir, 1922 struct dma_attrs *attrs) 1923 { 1924 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1925 dma_addr_t iova = handle & PAGE_MASK; 1926 int offset = handle & ~PAGE_MASK; 1927 int len = PAGE_ALIGN(size + offset); 1928 1929 if (!iova) 1930 return; 1931 1932 iommu_unmap(mapping->domain, iova, len); 1933 __free_iova(mapping, iova, len); 1934 } 1935 1936 /** 1937 * arm_iommu_unmap_page 1938 * @dev: valid struct device pointer 1939 * @handle: DMA address of buffer 1940 * @size: size of buffer (same as passed to dma_map_page) 1941 * @dir: DMA transfer direction (same as passed to dma_map_page) 1942 * 1943 * IOMMU aware version of arm_dma_unmap_page() 1944 */ 1945 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1946 size_t size, enum dma_data_direction dir, 1947 struct dma_attrs *attrs) 1948 { 1949 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1950 dma_addr_t iova = handle & PAGE_MASK; 1951 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1952 int offset = handle & ~PAGE_MASK; 1953 int len = PAGE_ALIGN(size + offset); 1954 1955 if (!iova) 1956 return; 1957 1958 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1959 __dma_page_dev_to_cpu(page, offset, size, dir); 1960 1961 iommu_unmap(mapping->domain, iova, len); 1962 __free_iova(mapping, iova, len); 1963 } 1964 1965 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1966 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1967 { 1968 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1969 dma_addr_t iova = handle & PAGE_MASK; 1970 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1971 unsigned int offset = handle & ~PAGE_MASK; 1972 1973 if (!iova) 1974 return; 1975 1976 __dma_page_dev_to_cpu(page, offset, size, dir); 1977 } 1978 1979 static void arm_iommu_sync_single_for_device(struct device *dev, 1980 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1981 { 1982 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1983 dma_addr_t iova = handle & PAGE_MASK; 1984 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1985 unsigned int offset = handle & ~PAGE_MASK; 1986 1987 if (!iova) 1988 return; 1989 1990 __dma_page_cpu_to_dev(page, offset, size, dir); 1991 } 1992 1993 struct dma_map_ops iommu_ops = { 1994 .alloc = arm_iommu_alloc_attrs, 1995 .free = arm_iommu_free_attrs, 1996 .mmap = arm_iommu_mmap_attrs, 1997 .get_sgtable = arm_iommu_get_sgtable, 1998 1999 .map_page = arm_iommu_map_page, 2000 .unmap_page = arm_iommu_unmap_page, 2001 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 2002 .sync_single_for_device = arm_iommu_sync_single_for_device, 2003 2004 .map_sg = arm_iommu_map_sg, 2005 .unmap_sg = arm_iommu_unmap_sg, 2006 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 2007 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 2008 2009 .set_dma_mask = arm_dma_set_mask, 2010 }; 2011 2012 struct dma_map_ops iommu_coherent_ops = { 2013 .alloc = arm_iommu_alloc_attrs, 2014 .free = arm_iommu_free_attrs, 2015 .mmap = arm_iommu_mmap_attrs, 2016 .get_sgtable = arm_iommu_get_sgtable, 2017 2018 .map_page = arm_coherent_iommu_map_page, 2019 .unmap_page = arm_coherent_iommu_unmap_page, 2020 2021 .map_sg = arm_coherent_iommu_map_sg, 2022 .unmap_sg = arm_coherent_iommu_unmap_sg, 2023 2024 .set_dma_mask = arm_dma_set_mask, 2025 }; 2026 2027 /** 2028 * arm_iommu_create_mapping 2029 * @bus: pointer to the bus holding the client device (for IOMMU calls) 2030 * @base: start address of the valid IO address space 2031 * @size: maximum size of the valid IO address space 2032 * 2033 * Creates a mapping structure which holds information about used/unused 2034 * IO address ranges, which is required to perform memory allocation and 2035 * mapping with IOMMU aware functions. 2036 * 2037 * The client device need to be attached to the mapping with 2038 * arm_iommu_attach_device function. 2039 */ 2040 struct dma_iommu_mapping * 2041 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) 2042 { 2043 unsigned int bits = size >> PAGE_SHIFT; 2044 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); 2045 struct dma_iommu_mapping *mapping; 2046 int extensions = 1; 2047 int err = -ENOMEM; 2048 2049 /* currently only 32-bit DMA address space is supported */ 2050 if (size > DMA_BIT_MASK(32) + 1) 2051 return ERR_PTR(-ERANGE); 2052 2053 if (!bitmap_size) 2054 return ERR_PTR(-EINVAL); 2055 2056 if (bitmap_size > PAGE_SIZE) { 2057 extensions = bitmap_size / PAGE_SIZE; 2058 bitmap_size = PAGE_SIZE; 2059 } 2060 2061 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 2062 if (!mapping) 2063 goto err; 2064 2065 mapping->bitmap_size = bitmap_size; 2066 mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *), 2067 GFP_KERNEL); 2068 if (!mapping->bitmaps) 2069 goto err2; 2070 2071 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); 2072 if (!mapping->bitmaps[0]) 2073 goto err3; 2074 2075 mapping->nr_bitmaps = 1; 2076 mapping->extensions = extensions; 2077 mapping->base = base; 2078 mapping->bits = BITS_PER_BYTE * bitmap_size; 2079 2080 spin_lock_init(&mapping->lock); 2081 2082 mapping->domain = iommu_domain_alloc(bus); 2083 if (!mapping->domain) 2084 goto err4; 2085 2086 kref_init(&mapping->kref); 2087 return mapping; 2088 err4: 2089 kfree(mapping->bitmaps[0]); 2090 err3: 2091 kfree(mapping->bitmaps); 2092 err2: 2093 kfree(mapping); 2094 err: 2095 return ERR_PTR(err); 2096 } 2097 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 2098 2099 static void release_iommu_mapping(struct kref *kref) 2100 { 2101 int i; 2102 struct dma_iommu_mapping *mapping = 2103 container_of(kref, struct dma_iommu_mapping, kref); 2104 2105 iommu_domain_free(mapping->domain); 2106 for (i = 0; i < mapping->nr_bitmaps; i++) 2107 kfree(mapping->bitmaps[i]); 2108 kfree(mapping->bitmaps); 2109 kfree(mapping); 2110 } 2111 2112 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping) 2113 { 2114 int next_bitmap; 2115 2116 if (mapping->nr_bitmaps >= mapping->extensions) 2117 return -EINVAL; 2118 2119 next_bitmap = mapping->nr_bitmaps; 2120 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, 2121 GFP_ATOMIC); 2122 if (!mapping->bitmaps[next_bitmap]) 2123 return -ENOMEM; 2124 2125 mapping->nr_bitmaps++; 2126 2127 return 0; 2128 } 2129 2130 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 2131 { 2132 if (mapping) 2133 kref_put(&mapping->kref, release_iommu_mapping); 2134 } 2135 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 2136 2137 static int __arm_iommu_attach_device(struct device *dev, 2138 struct dma_iommu_mapping *mapping) 2139 { 2140 int err; 2141 2142 err = iommu_attach_device(mapping->domain, dev); 2143 if (err) 2144 return err; 2145 2146 kref_get(&mapping->kref); 2147 to_dma_iommu_mapping(dev) = mapping; 2148 2149 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 2150 return 0; 2151 } 2152 2153 /** 2154 * arm_iommu_attach_device 2155 * @dev: valid struct device pointer 2156 * @mapping: io address space mapping structure (returned from 2157 * arm_iommu_create_mapping) 2158 * 2159 * Attaches specified io address space mapping to the provided device. 2160 * This replaces the dma operations (dma_map_ops pointer) with the 2161 * IOMMU aware version. 2162 * 2163 * More than one client might be attached to the same io address space 2164 * mapping. 2165 */ 2166 int arm_iommu_attach_device(struct device *dev, 2167 struct dma_iommu_mapping *mapping) 2168 { 2169 int err; 2170 2171 err = __arm_iommu_attach_device(dev, mapping); 2172 if (err) 2173 return err; 2174 2175 set_dma_ops(dev, &iommu_ops); 2176 return 0; 2177 } 2178 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 2179 2180 static void __arm_iommu_detach_device(struct device *dev) 2181 { 2182 struct dma_iommu_mapping *mapping; 2183 2184 mapping = to_dma_iommu_mapping(dev); 2185 if (!mapping) { 2186 dev_warn(dev, "Not attached\n"); 2187 return; 2188 } 2189 2190 iommu_detach_device(mapping->domain, dev); 2191 kref_put(&mapping->kref, release_iommu_mapping); 2192 to_dma_iommu_mapping(dev) = NULL; 2193 2194 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 2195 } 2196 2197 /** 2198 * arm_iommu_detach_device 2199 * @dev: valid struct device pointer 2200 * 2201 * Detaches the provided device from a previously attached map. 2202 * This voids the dma operations (dma_map_ops pointer) 2203 */ 2204 void arm_iommu_detach_device(struct device *dev) 2205 { 2206 __arm_iommu_detach_device(dev); 2207 set_dma_ops(dev, NULL); 2208 } 2209 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 2210 2211 static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) 2212 { 2213 return coherent ? &iommu_coherent_ops : &iommu_ops; 2214 } 2215 2216 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2217 struct iommu_ops *iommu) 2218 { 2219 struct dma_iommu_mapping *mapping; 2220 2221 if (!iommu) 2222 return false; 2223 2224 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 2225 if (IS_ERR(mapping)) { 2226 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 2227 size, dev_name(dev)); 2228 return false; 2229 } 2230 2231 if (__arm_iommu_attach_device(dev, mapping)) { 2232 pr_warn("Failed to attached device %s to IOMMU_mapping\n", 2233 dev_name(dev)); 2234 arm_iommu_release_mapping(mapping); 2235 return false; 2236 } 2237 2238 return true; 2239 } 2240 2241 static void arm_teardown_iommu_dma_ops(struct device *dev) 2242 { 2243 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2244 2245 if (!mapping) 2246 return; 2247 2248 __arm_iommu_detach_device(dev); 2249 arm_iommu_release_mapping(mapping); 2250 } 2251 2252 #else 2253 2254 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2255 struct iommu_ops *iommu) 2256 { 2257 return false; 2258 } 2259 2260 static void arm_teardown_iommu_dma_ops(struct device *dev) { } 2261 2262 #define arm_get_iommu_dma_map_ops arm_get_dma_map_ops 2263 2264 #endif /* CONFIG_ARM_DMA_USE_IOMMU */ 2265 2266 static struct dma_map_ops *arm_get_dma_map_ops(bool coherent) 2267 { 2268 return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; 2269 } 2270 2271 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 2272 struct iommu_ops *iommu, bool coherent) 2273 { 2274 struct dma_map_ops *dma_ops; 2275 2276 dev->archdata.dma_coherent = coherent; 2277 if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) 2278 dma_ops = arm_get_iommu_dma_map_ops(coherent); 2279 else 2280 dma_ops = arm_get_dma_map_ops(coherent); 2281 2282 set_dma_ops(dev, dma_ops); 2283 } 2284 2285 void arch_teardown_dma_ops(struct device *dev) 2286 { 2287 arm_teardown_iommu_dma_ops(dev); 2288 } 2289