1 /* 2 * linux/arch/arm/mm/dma-mapping.c 3 * 4 * Copyright (C) 2000-2004 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * DMA uncached mapping support. 11 */ 12 #include <linux/bootmem.h> 13 #include <linux/module.h> 14 #include <linux/mm.h> 15 #include <linux/genalloc.h> 16 #include <linux/gfp.h> 17 #include <linux/errno.h> 18 #include <linux/list.h> 19 #include <linux/init.h> 20 #include <linux/device.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/dma-contiguous.h> 23 #include <linux/highmem.h> 24 #include <linux/memblock.h> 25 #include <linux/slab.h> 26 #include <linux/iommu.h> 27 #include <linux/io.h> 28 #include <linux/vmalloc.h> 29 #include <linux/sizes.h> 30 #include <linux/cma.h> 31 32 #include <asm/memory.h> 33 #include <asm/highmem.h> 34 #include <asm/cacheflush.h> 35 #include <asm/tlbflush.h> 36 #include <asm/mach/arch.h> 37 #include <asm/dma-iommu.h> 38 #include <asm/mach/map.h> 39 #include <asm/system_info.h> 40 #include <asm/dma-contiguous.h> 41 42 #include "mm.h" 43 44 /* 45 * The DMA API is built upon the notion of "buffer ownership". A buffer 46 * is either exclusively owned by the CPU (and therefore may be accessed 47 * by it) or exclusively owned by the DMA device. These helper functions 48 * represent the transitions between these two ownership states. 49 * 50 * Note, however, that on later ARMs, this notion does not work due to 51 * speculative prefetches. We model our approach on the assumption that 52 * the CPU does do speculative prefetches, which means we clean caches 53 * before transfers and delay cache invalidation until transfer completion. 54 * 55 */ 56 static void __dma_page_cpu_to_dev(struct page *, unsigned long, 57 size_t, enum dma_data_direction); 58 static void __dma_page_dev_to_cpu(struct page *, unsigned long, 59 size_t, enum dma_data_direction); 60 61 /** 62 * arm_dma_map_page - map a portion of a page for streaming DMA 63 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 64 * @page: page that buffer resides in 65 * @offset: offset into page for start of buffer 66 * @size: size of buffer to map 67 * @dir: DMA transfer direction 68 * 69 * Ensure that any data held in the cache is appropriately discarded 70 * or written back. 71 * 72 * The device owns this memory once this call has completed. The CPU 73 * can regain ownership by calling dma_unmap_page(). 74 */ 75 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, 76 unsigned long offset, size_t size, enum dma_data_direction dir, 77 struct dma_attrs *attrs) 78 { 79 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 80 __dma_page_cpu_to_dev(page, offset, size, dir); 81 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 82 } 83 84 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, 85 unsigned long offset, size_t size, enum dma_data_direction dir, 86 struct dma_attrs *attrs) 87 { 88 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 89 } 90 91 /** 92 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() 93 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 94 * @handle: DMA address of buffer 95 * @size: size of buffer (same as passed to dma_map_page) 96 * @dir: DMA transfer direction (same as passed to dma_map_page) 97 * 98 * Unmap a page streaming mode DMA translation. The handle and size 99 * must match what was provided in the previous dma_map_page() call. 100 * All other usages are undefined. 101 * 102 * After this call, reads by the CPU to the buffer are guaranteed to see 103 * whatever the device wrote there. 104 */ 105 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, 106 size_t size, enum dma_data_direction dir, 107 struct dma_attrs *attrs) 108 { 109 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 110 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), 111 handle & ~PAGE_MASK, size, dir); 112 } 113 114 static void arm_dma_sync_single_for_cpu(struct device *dev, 115 dma_addr_t handle, size_t size, enum dma_data_direction dir) 116 { 117 unsigned int offset = handle & (PAGE_SIZE - 1); 118 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 119 __dma_page_dev_to_cpu(page, offset, size, dir); 120 } 121 122 static void arm_dma_sync_single_for_device(struct device *dev, 123 dma_addr_t handle, size_t size, enum dma_data_direction dir) 124 { 125 unsigned int offset = handle & (PAGE_SIZE - 1); 126 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 127 __dma_page_cpu_to_dev(page, offset, size, dir); 128 } 129 130 struct dma_map_ops arm_dma_ops = { 131 .alloc = arm_dma_alloc, 132 .free = arm_dma_free, 133 .mmap = arm_dma_mmap, 134 .get_sgtable = arm_dma_get_sgtable, 135 .map_page = arm_dma_map_page, 136 .unmap_page = arm_dma_unmap_page, 137 .map_sg = arm_dma_map_sg, 138 .unmap_sg = arm_dma_unmap_sg, 139 .sync_single_for_cpu = arm_dma_sync_single_for_cpu, 140 .sync_single_for_device = arm_dma_sync_single_for_device, 141 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, 142 .sync_sg_for_device = arm_dma_sync_sg_for_device, 143 .set_dma_mask = arm_dma_set_mask, 144 }; 145 EXPORT_SYMBOL(arm_dma_ops); 146 147 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 148 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); 149 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 150 dma_addr_t handle, struct dma_attrs *attrs); 151 152 struct dma_map_ops arm_coherent_dma_ops = { 153 .alloc = arm_coherent_dma_alloc, 154 .free = arm_coherent_dma_free, 155 .mmap = arm_dma_mmap, 156 .get_sgtable = arm_dma_get_sgtable, 157 .map_page = arm_coherent_dma_map_page, 158 .map_sg = arm_dma_map_sg, 159 .set_dma_mask = arm_dma_set_mask, 160 }; 161 EXPORT_SYMBOL(arm_coherent_dma_ops); 162 163 static int __dma_supported(struct device *dev, u64 mask, bool warn) 164 { 165 unsigned long max_dma_pfn; 166 167 /* 168 * If the mask allows for more memory than we can address, 169 * and we actually have that much memory, then we must 170 * indicate that DMA to this device is not supported. 171 */ 172 if (sizeof(mask) != sizeof(dma_addr_t) && 173 mask > (dma_addr_t)~0 && 174 dma_to_pfn(dev, ~0) < max_pfn) { 175 if (warn) { 176 dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n", 177 mask); 178 dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n"); 179 } 180 return 0; 181 } 182 183 max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); 184 185 /* 186 * Translate the device's DMA mask to a PFN limit. This 187 * PFN number includes the page which we can DMA to. 188 */ 189 if (dma_to_pfn(dev, mask) < max_dma_pfn) { 190 if (warn) 191 dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n", 192 mask, 193 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1, 194 max_dma_pfn + 1); 195 return 0; 196 } 197 198 return 1; 199 } 200 201 static u64 get_coherent_dma_mask(struct device *dev) 202 { 203 u64 mask = (u64)DMA_BIT_MASK(32); 204 205 if (dev) { 206 mask = dev->coherent_dma_mask; 207 208 /* 209 * Sanity check the DMA mask - it must be non-zero, and 210 * must be able to be satisfied by a DMA allocation. 211 */ 212 if (mask == 0) { 213 dev_warn(dev, "coherent DMA mask is unset\n"); 214 return 0; 215 } 216 217 if (!__dma_supported(dev, mask, true)) 218 return 0; 219 } 220 221 return mask; 222 } 223 224 static void __dma_clear_buffer(struct page *page, size_t size) 225 { 226 /* 227 * Ensure that the allocated pages are zeroed, and that any data 228 * lurking in the kernel direct-mapped region is invalidated. 229 */ 230 if (PageHighMem(page)) { 231 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 232 phys_addr_t end = base + size; 233 while (size > 0) { 234 void *ptr = kmap_atomic(page); 235 memset(ptr, 0, PAGE_SIZE); 236 dmac_flush_range(ptr, ptr + PAGE_SIZE); 237 kunmap_atomic(ptr); 238 page++; 239 size -= PAGE_SIZE; 240 } 241 outer_flush_range(base, end); 242 } else { 243 void *ptr = page_address(page); 244 memset(ptr, 0, size); 245 dmac_flush_range(ptr, ptr + size); 246 outer_flush_range(__pa(ptr), __pa(ptr) + size); 247 } 248 } 249 250 /* 251 * Allocate a DMA buffer for 'dev' of size 'size' using the 252 * specified gfp mask. Note that 'size' must be page aligned. 253 */ 254 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) 255 { 256 unsigned long order = get_order(size); 257 struct page *page, *p, *e; 258 259 page = alloc_pages(gfp, order); 260 if (!page) 261 return NULL; 262 263 /* 264 * Now split the huge page and free the excess pages 265 */ 266 split_page(page, order); 267 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 268 __free_page(p); 269 270 __dma_clear_buffer(page, size); 271 272 return page; 273 } 274 275 /* 276 * Free a DMA buffer. 'size' must be page aligned. 277 */ 278 static void __dma_free_buffer(struct page *page, size_t size) 279 { 280 struct page *e = page + (size >> PAGE_SHIFT); 281 282 while (page < e) { 283 __free_page(page); 284 page++; 285 } 286 } 287 288 #ifdef CONFIG_MMU 289 290 static void *__alloc_from_contiguous(struct device *dev, size_t size, 291 pgprot_t prot, struct page **ret_page, 292 const void *caller); 293 294 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 295 pgprot_t prot, struct page **ret_page, 296 const void *caller); 297 298 static void * 299 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 300 const void *caller) 301 { 302 /* 303 * DMA allocation can be mapped to user space, so lets 304 * set VM_USERMAP flags too. 305 */ 306 return dma_common_contiguous_remap(page, size, 307 VM_ARM_DMA_CONSISTENT | VM_USERMAP, 308 prot, caller); 309 } 310 311 static void __dma_free_remap(void *cpu_addr, size_t size) 312 { 313 dma_common_free_remap(cpu_addr, size, 314 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 315 } 316 317 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 318 static struct gen_pool *atomic_pool; 319 320 static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; 321 322 static int __init early_coherent_pool(char *p) 323 { 324 atomic_pool_size = memparse(p, &p); 325 return 0; 326 } 327 early_param("coherent_pool", early_coherent_pool); 328 329 void __init init_dma_coherent_pool_size(unsigned long size) 330 { 331 /* 332 * Catch any attempt to set the pool size too late. 333 */ 334 BUG_ON(atomic_pool); 335 336 /* 337 * Set architecture specific coherent pool size only if 338 * it has not been changed by kernel command line parameter. 339 */ 340 if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE) 341 atomic_pool_size = size; 342 } 343 344 /* 345 * Initialise the coherent pool for atomic allocations. 346 */ 347 static int __init atomic_pool_init(void) 348 { 349 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); 350 gfp_t gfp = GFP_KERNEL | GFP_DMA; 351 struct page *page; 352 void *ptr; 353 354 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 355 if (!atomic_pool) 356 goto out; 357 358 if (dev_get_cma_area(NULL)) 359 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 360 &page, atomic_pool_init); 361 else 362 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 363 &page, atomic_pool_init); 364 if (ptr) { 365 int ret; 366 367 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr, 368 page_to_phys(page), 369 atomic_pool_size, -1); 370 if (ret) 371 goto destroy_genpool; 372 373 gen_pool_set_algo(atomic_pool, 374 gen_pool_first_fit_order_align, 375 (void *)PAGE_SHIFT); 376 pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n", 377 atomic_pool_size / 1024); 378 return 0; 379 } 380 381 destroy_genpool: 382 gen_pool_destroy(atomic_pool); 383 atomic_pool = NULL; 384 out: 385 pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n", 386 atomic_pool_size / 1024); 387 return -ENOMEM; 388 } 389 /* 390 * CMA is activated by core_initcall, so we must be called after it. 391 */ 392 postcore_initcall(atomic_pool_init); 393 394 struct dma_contig_early_reserve { 395 phys_addr_t base; 396 unsigned long size; 397 }; 398 399 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 400 401 static int dma_mmu_remap_num __initdata; 402 403 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 404 { 405 dma_mmu_remap[dma_mmu_remap_num].base = base; 406 dma_mmu_remap[dma_mmu_remap_num].size = size; 407 dma_mmu_remap_num++; 408 } 409 410 void __init dma_contiguous_remap(void) 411 { 412 int i; 413 for (i = 0; i < dma_mmu_remap_num; i++) { 414 phys_addr_t start = dma_mmu_remap[i].base; 415 phys_addr_t end = start + dma_mmu_remap[i].size; 416 struct map_desc map; 417 unsigned long addr; 418 419 if (end > arm_lowmem_limit) 420 end = arm_lowmem_limit; 421 if (start >= end) 422 continue; 423 424 map.pfn = __phys_to_pfn(start); 425 map.virtual = __phys_to_virt(start); 426 map.length = end - start; 427 map.type = MT_MEMORY_DMA_READY; 428 429 /* 430 * Clear previous low-memory mapping to ensure that the 431 * TLB does not see any conflicting entries, then flush 432 * the TLB of the old entries before creating new mappings. 433 * 434 * This ensures that any speculatively loaded TLB entries 435 * (even though they may be rare) can not cause any problems, 436 * and ensures that this code is architecturally compliant. 437 */ 438 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 439 addr += PMD_SIZE) 440 pmd_clear(pmd_off_k(addr)); 441 442 flush_tlb_kernel_range(__phys_to_virt(start), 443 __phys_to_virt(end)); 444 445 iotable_init(&map, 1); 446 } 447 } 448 449 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, 450 void *data) 451 { 452 struct page *page = virt_to_page(addr); 453 pgprot_t prot = *(pgprot_t *)data; 454 455 set_pte_ext(pte, mk_pte(page, prot), 0); 456 return 0; 457 } 458 459 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 460 { 461 unsigned long start = (unsigned long) page_address(page); 462 unsigned end = start + size; 463 464 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 465 flush_tlb_kernel_range(start, end); 466 } 467 468 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 469 pgprot_t prot, struct page **ret_page, 470 const void *caller) 471 { 472 struct page *page; 473 void *ptr; 474 page = __dma_alloc_buffer(dev, size, gfp); 475 if (!page) 476 return NULL; 477 478 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 479 if (!ptr) { 480 __dma_free_buffer(page, size); 481 return NULL; 482 } 483 484 *ret_page = page; 485 return ptr; 486 } 487 488 static void *__alloc_from_pool(size_t size, struct page **ret_page) 489 { 490 unsigned long val; 491 void *ptr = NULL; 492 493 if (!atomic_pool) { 494 WARN(1, "coherent pool not initialised!\n"); 495 return NULL; 496 } 497 498 val = gen_pool_alloc(atomic_pool, size); 499 if (val) { 500 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 501 502 *ret_page = phys_to_page(phys); 503 ptr = (void *)val; 504 } 505 506 return ptr; 507 } 508 509 static bool __in_atomic_pool(void *start, size_t size) 510 { 511 return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); 512 } 513 514 static int __free_from_pool(void *start, size_t size) 515 { 516 if (!__in_atomic_pool(start, size)) 517 return 0; 518 519 gen_pool_free(atomic_pool, (unsigned long)start, size); 520 521 return 1; 522 } 523 524 static void *__alloc_from_contiguous(struct device *dev, size_t size, 525 pgprot_t prot, struct page **ret_page, 526 const void *caller) 527 { 528 unsigned long order = get_order(size); 529 size_t count = size >> PAGE_SHIFT; 530 struct page *page; 531 void *ptr; 532 533 page = dma_alloc_from_contiguous(dev, count, order); 534 if (!page) 535 return NULL; 536 537 __dma_clear_buffer(page, size); 538 539 if (PageHighMem(page)) { 540 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); 541 if (!ptr) { 542 dma_release_from_contiguous(dev, page, count); 543 return NULL; 544 } 545 } else { 546 __dma_remap(page, size, prot); 547 ptr = page_address(page); 548 } 549 *ret_page = page; 550 return ptr; 551 } 552 553 static void __free_from_contiguous(struct device *dev, struct page *page, 554 void *cpu_addr, size_t size) 555 { 556 if (PageHighMem(page)) 557 __dma_free_remap(cpu_addr, size); 558 else 559 __dma_remap(page, size, PAGE_KERNEL); 560 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 561 } 562 563 static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) 564 { 565 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? 566 pgprot_writecombine(prot) : 567 pgprot_dmacoherent(prot); 568 return prot; 569 } 570 571 #define nommu() 0 572 573 #else /* !CONFIG_MMU */ 574 575 #define nommu() 1 576 577 #define __get_dma_pgprot(attrs, prot) __pgprot(0) 578 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL 579 #define __alloc_from_pool(size, ret_page) NULL 580 #define __alloc_from_contiguous(dev, size, prot, ret, c) NULL 581 #define __free_from_pool(cpu_addr, size) 0 582 #define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0) 583 #define __dma_free_remap(cpu_addr, size) do { } while (0) 584 585 #endif /* CONFIG_MMU */ 586 587 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 588 struct page **ret_page) 589 { 590 struct page *page; 591 page = __dma_alloc_buffer(dev, size, gfp); 592 if (!page) 593 return NULL; 594 595 *ret_page = page; 596 return page_address(page); 597 } 598 599 600 601 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 602 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller) 603 { 604 u64 mask = get_coherent_dma_mask(dev); 605 struct page *page = NULL; 606 void *addr; 607 608 #ifdef CONFIG_DMA_API_DEBUG 609 u64 limit = (mask + 1) & ~mask; 610 if (limit && size >= limit) { 611 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 612 size, mask); 613 return NULL; 614 } 615 #endif 616 617 if (!mask) 618 return NULL; 619 620 if (mask < 0xffffffffULL) 621 gfp |= GFP_DMA; 622 623 /* 624 * Following is a work-around (a.k.a. hack) to prevent pages 625 * with __GFP_COMP being passed to split_page() which cannot 626 * handle them. The real problem is that this flag probably 627 * should be 0 on ARM as it is not supported on this 628 * platform; see CONFIG_HUGETLBFS. 629 */ 630 gfp &= ~(__GFP_COMP); 631 632 *handle = DMA_ERROR_CODE; 633 size = PAGE_ALIGN(size); 634 635 if (is_coherent || nommu()) 636 addr = __alloc_simple_buffer(dev, size, gfp, &page); 637 else if (!(gfp & __GFP_WAIT)) 638 addr = __alloc_from_pool(size, &page); 639 else if (!dev_get_cma_area(dev)) 640 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); 641 else 642 addr = __alloc_from_contiguous(dev, size, prot, &page, caller); 643 644 if (addr) 645 *handle = pfn_to_dma(dev, page_to_pfn(page)); 646 647 return addr; 648 } 649 650 /* 651 * Allocate DMA-coherent memory space and return both the kernel remapped 652 * virtual and bus address for that space. 653 */ 654 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 655 gfp_t gfp, struct dma_attrs *attrs) 656 { 657 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 658 void *memory; 659 660 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 661 return memory; 662 663 return __dma_alloc(dev, size, handle, gfp, prot, false, 664 __builtin_return_address(0)); 665 } 666 667 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 668 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 669 { 670 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 671 void *memory; 672 673 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 674 return memory; 675 676 return __dma_alloc(dev, size, handle, gfp, prot, true, 677 __builtin_return_address(0)); 678 } 679 680 /* 681 * Create userspace mapping for the DMA-coherent memory. 682 */ 683 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 684 void *cpu_addr, dma_addr_t dma_addr, size_t size, 685 struct dma_attrs *attrs) 686 { 687 int ret = -ENXIO; 688 #ifdef CONFIG_MMU 689 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 690 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 691 unsigned long pfn = dma_to_pfn(dev, dma_addr); 692 unsigned long off = vma->vm_pgoff; 693 694 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 695 696 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 697 return ret; 698 699 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 700 ret = remap_pfn_range(vma, vma->vm_start, 701 pfn + off, 702 vma->vm_end - vma->vm_start, 703 vma->vm_page_prot); 704 } 705 #endif /* CONFIG_MMU */ 706 707 return ret; 708 } 709 710 /* 711 * Free a buffer as defined by the above mapping. 712 */ 713 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 714 dma_addr_t handle, struct dma_attrs *attrs, 715 bool is_coherent) 716 { 717 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 718 719 if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) 720 return; 721 722 size = PAGE_ALIGN(size); 723 724 if (is_coherent || nommu()) { 725 __dma_free_buffer(page, size); 726 } else if (__free_from_pool(cpu_addr, size)) { 727 return; 728 } else if (!dev_get_cma_area(dev)) { 729 __dma_free_remap(cpu_addr, size); 730 __dma_free_buffer(page, size); 731 } else { 732 /* 733 * Non-atomic allocations cannot be freed with IRQs disabled 734 */ 735 WARN_ON(irqs_disabled()); 736 __free_from_contiguous(dev, page, cpu_addr, size); 737 } 738 } 739 740 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 741 dma_addr_t handle, struct dma_attrs *attrs) 742 { 743 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); 744 } 745 746 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 747 dma_addr_t handle, struct dma_attrs *attrs) 748 { 749 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); 750 } 751 752 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 753 void *cpu_addr, dma_addr_t handle, size_t size, 754 struct dma_attrs *attrs) 755 { 756 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 757 int ret; 758 759 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 760 if (unlikely(ret)) 761 return ret; 762 763 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 764 return 0; 765 } 766 767 static void dma_cache_maint_page(struct page *page, unsigned long offset, 768 size_t size, enum dma_data_direction dir, 769 void (*op)(const void *, size_t, int)) 770 { 771 unsigned long pfn; 772 size_t left = size; 773 774 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 775 offset %= PAGE_SIZE; 776 777 /* 778 * A single sg entry may refer to multiple physically contiguous 779 * pages. But we still need to process highmem pages individually. 780 * If highmem is not configured then the bulk of this loop gets 781 * optimized out. 782 */ 783 do { 784 size_t len = left; 785 void *vaddr; 786 787 page = pfn_to_page(pfn); 788 789 if (PageHighMem(page)) { 790 if (len + offset > PAGE_SIZE) 791 len = PAGE_SIZE - offset; 792 793 if (cache_is_vipt_nonaliasing()) { 794 vaddr = kmap_atomic(page); 795 op(vaddr + offset, len, dir); 796 kunmap_atomic(vaddr); 797 } else { 798 vaddr = kmap_high_get(page); 799 if (vaddr) { 800 op(vaddr + offset, len, dir); 801 kunmap_high(page); 802 } 803 } 804 } else { 805 vaddr = page_address(page) + offset; 806 op(vaddr, len, dir); 807 } 808 offset = 0; 809 pfn++; 810 left -= len; 811 } while (left); 812 } 813 814 /* 815 * Make an area consistent for devices. 816 * Note: Drivers should NOT use this function directly, as it will break 817 * platforms with CONFIG_DMABOUNCE. 818 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 819 */ 820 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 821 size_t size, enum dma_data_direction dir) 822 { 823 phys_addr_t paddr; 824 825 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 826 827 paddr = page_to_phys(page) + off; 828 if (dir == DMA_FROM_DEVICE) { 829 outer_inv_range(paddr, paddr + size); 830 } else { 831 outer_clean_range(paddr, paddr + size); 832 } 833 /* FIXME: non-speculating: flush on bidirectional mappings? */ 834 } 835 836 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 837 size_t size, enum dma_data_direction dir) 838 { 839 phys_addr_t paddr = page_to_phys(page) + off; 840 841 /* FIXME: non-speculating: not required */ 842 /* in any case, don't bother invalidating if DMA to device */ 843 if (dir != DMA_TO_DEVICE) { 844 outer_inv_range(paddr, paddr + size); 845 846 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 847 } 848 849 /* 850 * Mark the D-cache clean for these pages to avoid extra flushing. 851 */ 852 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { 853 unsigned long pfn; 854 size_t left = size; 855 856 pfn = page_to_pfn(page) + off / PAGE_SIZE; 857 off %= PAGE_SIZE; 858 if (off) { 859 pfn++; 860 left -= PAGE_SIZE - off; 861 } 862 while (left >= PAGE_SIZE) { 863 page = pfn_to_page(pfn++); 864 set_bit(PG_dcache_clean, &page->flags); 865 left -= PAGE_SIZE; 866 } 867 } 868 } 869 870 /** 871 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA 872 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 873 * @sg: list of buffers 874 * @nents: number of buffers to map 875 * @dir: DMA transfer direction 876 * 877 * Map a set of buffers described by scatterlist in streaming mode for DMA. 878 * This is the scatter-gather version of the dma_map_single interface. 879 * Here the scatter gather list elements are each tagged with the 880 * appropriate dma address and length. They are obtained via 881 * sg_dma_{address,length}. 882 * 883 * Device ownership issues as mentioned for dma_map_single are the same 884 * here. 885 */ 886 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 887 enum dma_data_direction dir, struct dma_attrs *attrs) 888 { 889 struct dma_map_ops *ops = get_dma_ops(dev); 890 struct scatterlist *s; 891 int i, j; 892 893 for_each_sg(sg, s, nents, i) { 894 #ifdef CONFIG_NEED_SG_DMA_LENGTH 895 s->dma_length = s->length; 896 #endif 897 s->dma_address = ops->map_page(dev, sg_page(s), s->offset, 898 s->length, dir, attrs); 899 if (dma_mapping_error(dev, s->dma_address)) 900 goto bad_mapping; 901 } 902 return nents; 903 904 bad_mapping: 905 for_each_sg(sg, s, i, j) 906 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 907 return 0; 908 } 909 910 /** 911 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 912 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 913 * @sg: list of buffers 914 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 915 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 916 * 917 * Unmap a set of streaming mode DMA translations. Again, CPU access 918 * rules concerning calls here are the same as for dma_unmap_single(). 919 */ 920 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 921 enum dma_data_direction dir, struct dma_attrs *attrs) 922 { 923 struct dma_map_ops *ops = get_dma_ops(dev); 924 struct scatterlist *s; 925 926 int i; 927 928 for_each_sg(sg, s, nents, i) 929 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 930 } 931 932 /** 933 * arm_dma_sync_sg_for_cpu 934 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 935 * @sg: list of buffers 936 * @nents: number of buffers to map (returned from dma_map_sg) 937 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 938 */ 939 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 940 int nents, enum dma_data_direction dir) 941 { 942 struct dma_map_ops *ops = get_dma_ops(dev); 943 struct scatterlist *s; 944 int i; 945 946 for_each_sg(sg, s, nents, i) 947 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, 948 dir); 949 } 950 951 /** 952 * arm_dma_sync_sg_for_device 953 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 954 * @sg: list of buffers 955 * @nents: number of buffers to map (returned from dma_map_sg) 956 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 957 */ 958 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 959 int nents, enum dma_data_direction dir) 960 { 961 struct dma_map_ops *ops = get_dma_ops(dev); 962 struct scatterlist *s; 963 int i; 964 965 for_each_sg(sg, s, nents, i) 966 ops->sync_single_for_device(dev, sg_dma_address(s), s->length, 967 dir); 968 } 969 970 /* 971 * Return whether the given device DMA address mask can be supported 972 * properly. For example, if your device can only drive the low 24-bits 973 * during bus mastering, then you would pass 0x00ffffff as the mask 974 * to this function. 975 */ 976 int dma_supported(struct device *dev, u64 mask) 977 { 978 return __dma_supported(dev, mask, false); 979 } 980 EXPORT_SYMBOL(dma_supported); 981 982 int arm_dma_set_mask(struct device *dev, u64 dma_mask) 983 { 984 if (!dev->dma_mask || !dma_supported(dev, dma_mask)) 985 return -EIO; 986 987 *dev->dma_mask = dma_mask; 988 989 return 0; 990 } 991 992 #define PREALLOC_DMA_DEBUG_ENTRIES 4096 993 994 static int __init dma_debug_do_init(void) 995 { 996 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 997 return 0; 998 } 999 fs_initcall(dma_debug_do_init); 1000 1001 #ifdef CONFIG_ARM_DMA_USE_IOMMU 1002 1003 /* IOMMU */ 1004 1005 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); 1006 1007 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 1008 size_t size) 1009 { 1010 unsigned int order = get_order(size); 1011 unsigned int align = 0; 1012 unsigned int count, start; 1013 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1014 unsigned long flags; 1015 dma_addr_t iova; 1016 int i; 1017 1018 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 1019 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 1020 1021 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1022 align = (1 << order) - 1; 1023 1024 spin_lock_irqsave(&mapping->lock, flags); 1025 for (i = 0; i < mapping->nr_bitmaps; i++) { 1026 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1027 mapping->bits, 0, count, align); 1028 1029 if (start > mapping->bits) 1030 continue; 1031 1032 bitmap_set(mapping->bitmaps[i], start, count); 1033 break; 1034 } 1035 1036 /* 1037 * No unused range found. Try to extend the existing mapping 1038 * and perform a second attempt to reserve an IO virtual 1039 * address range of size bytes. 1040 */ 1041 if (i == mapping->nr_bitmaps) { 1042 if (extend_iommu_mapping(mapping)) { 1043 spin_unlock_irqrestore(&mapping->lock, flags); 1044 return DMA_ERROR_CODE; 1045 } 1046 1047 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1048 mapping->bits, 0, count, align); 1049 1050 if (start > mapping->bits) { 1051 spin_unlock_irqrestore(&mapping->lock, flags); 1052 return DMA_ERROR_CODE; 1053 } 1054 1055 bitmap_set(mapping->bitmaps[i], start, count); 1056 } 1057 spin_unlock_irqrestore(&mapping->lock, flags); 1058 1059 iova = mapping->base + (mapping_size * i); 1060 iova += start << PAGE_SHIFT; 1061 1062 return iova; 1063 } 1064 1065 static inline void __free_iova(struct dma_iommu_mapping *mapping, 1066 dma_addr_t addr, size_t size) 1067 { 1068 unsigned int start, count; 1069 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1070 unsigned long flags; 1071 dma_addr_t bitmap_base; 1072 u32 bitmap_index; 1073 1074 if (!size) 1075 return; 1076 1077 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; 1078 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); 1079 1080 bitmap_base = mapping->base + mapping_size * bitmap_index; 1081 1082 start = (addr - bitmap_base) >> PAGE_SHIFT; 1083 1084 if (addr + size > bitmap_base + mapping_size) { 1085 /* 1086 * The address range to be freed reaches into the iova 1087 * range of the next bitmap. This should not happen as 1088 * we don't allow this in __alloc_iova (at the 1089 * moment). 1090 */ 1091 BUG(); 1092 } else 1093 count = size >> PAGE_SHIFT; 1094 1095 spin_lock_irqsave(&mapping->lock, flags); 1096 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); 1097 spin_unlock_irqrestore(&mapping->lock, flags); 1098 } 1099 1100 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 1101 gfp_t gfp, struct dma_attrs *attrs) 1102 { 1103 struct page **pages; 1104 int count = size >> PAGE_SHIFT; 1105 int array_size = count * sizeof(struct page *); 1106 int i = 0; 1107 1108 if (array_size <= PAGE_SIZE) 1109 pages = kzalloc(array_size, gfp); 1110 else 1111 pages = vzalloc(array_size); 1112 if (!pages) 1113 return NULL; 1114 1115 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) 1116 { 1117 unsigned long order = get_order(size); 1118 struct page *page; 1119 1120 page = dma_alloc_from_contiguous(dev, count, order); 1121 if (!page) 1122 goto error; 1123 1124 __dma_clear_buffer(page, size); 1125 1126 for (i = 0; i < count; i++) 1127 pages[i] = page + i; 1128 1129 return pages; 1130 } 1131 1132 /* 1133 * IOMMU can map any pages, so himem can also be used here 1134 */ 1135 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 1136 1137 while (count) { 1138 int j, order = __fls(count); 1139 1140 pages[i] = alloc_pages(gfp, order); 1141 while (!pages[i] && order) 1142 pages[i] = alloc_pages(gfp, --order); 1143 if (!pages[i]) 1144 goto error; 1145 1146 if (order) { 1147 split_page(pages[i], order); 1148 j = 1 << order; 1149 while (--j) 1150 pages[i + j] = pages[i] + j; 1151 } 1152 1153 __dma_clear_buffer(pages[i], PAGE_SIZE << order); 1154 i += 1 << order; 1155 count -= 1 << order; 1156 } 1157 1158 return pages; 1159 error: 1160 while (i--) 1161 if (pages[i]) 1162 __free_pages(pages[i], 0); 1163 if (array_size <= PAGE_SIZE) 1164 kfree(pages); 1165 else 1166 vfree(pages); 1167 return NULL; 1168 } 1169 1170 static int __iommu_free_buffer(struct device *dev, struct page **pages, 1171 size_t size, struct dma_attrs *attrs) 1172 { 1173 int count = size >> PAGE_SHIFT; 1174 int array_size = count * sizeof(struct page *); 1175 int i; 1176 1177 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { 1178 dma_release_from_contiguous(dev, pages[0], count); 1179 } else { 1180 for (i = 0; i < count; i++) 1181 if (pages[i]) 1182 __free_pages(pages[i], 0); 1183 } 1184 1185 if (array_size <= PAGE_SIZE) 1186 kfree(pages); 1187 else 1188 vfree(pages); 1189 return 0; 1190 } 1191 1192 /* 1193 * Create a CPU mapping for a specified pages 1194 */ 1195 static void * 1196 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, 1197 const void *caller) 1198 { 1199 return dma_common_pages_remap(pages, size, 1200 VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller); 1201 } 1202 1203 /* 1204 * Create a mapping in device IO address space for specified pages 1205 */ 1206 static dma_addr_t 1207 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1208 { 1209 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1210 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1211 dma_addr_t dma_addr, iova; 1212 int i, ret = DMA_ERROR_CODE; 1213 1214 dma_addr = __alloc_iova(mapping, size); 1215 if (dma_addr == DMA_ERROR_CODE) 1216 return dma_addr; 1217 1218 iova = dma_addr; 1219 for (i = 0; i < count; ) { 1220 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 1221 phys_addr_t phys = page_to_phys(pages[i]); 1222 unsigned int len, j; 1223 1224 for (j = i + 1; j < count; j++, next_pfn++) 1225 if (page_to_pfn(pages[j]) != next_pfn) 1226 break; 1227 1228 len = (j - i) << PAGE_SHIFT; 1229 ret = iommu_map(mapping->domain, iova, phys, len, 1230 IOMMU_READ|IOMMU_WRITE); 1231 if (ret < 0) 1232 goto fail; 1233 iova += len; 1234 i = j; 1235 } 1236 return dma_addr; 1237 fail: 1238 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 1239 __free_iova(mapping, dma_addr, size); 1240 return DMA_ERROR_CODE; 1241 } 1242 1243 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1244 { 1245 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1246 1247 /* 1248 * add optional in-page offset from iova to size and align 1249 * result to page size 1250 */ 1251 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1252 iova &= PAGE_MASK; 1253 1254 iommu_unmap(mapping->domain, iova, size); 1255 __free_iova(mapping, iova, size); 1256 return 0; 1257 } 1258 1259 static struct page **__atomic_get_pages(void *addr) 1260 { 1261 struct page *page; 1262 phys_addr_t phys; 1263 1264 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); 1265 page = phys_to_page(phys); 1266 1267 return (struct page **)page; 1268 } 1269 1270 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) 1271 { 1272 struct vm_struct *area; 1273 1274 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1275 return __atomic_get_pages(cpu_addr); 1276 1277 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1278 return cpu_addr; 1279 1280 area = find_vm_area(cpu_addr); 1281 if (area && (area->flags & VM_ARM_DMA_CONSISTENT)) 1282 return area->pages; 1283 return NULL; 1284 } 1285 1286 static void *__iommu_alloc_atomic(struct device *dev, size_t size, 1287 dma_addr_t *handle) 1288 { 1289 struct page *page; 1290 void *addr; 1291 1292 addr = __alloc_from_pool(size, &page); 1293 if (!addr) 1294 return NULL; 1295 1296 *handle = __iommu_create_mapping(dev, &page, size); 1297 if (*handle == DMA_ERROR_CODE) 1298 goto err_mapping; 1299 1300 return addr; 1301 1302 err_mapping: 1303 __free_from_pool(addr, size); 1304 return NULL; 1305 } 1306 1307 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1308 dma_addr_t handle, size_t size) 1309 { 1310 __iommu_remove_mapping(dev, handle, size); 1311 __free_from_pool(cpu_addr, size); 1312 } 1313 1314 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1315 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) 1316 { 1317 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 1318 struct page **pages; 1319 void *addr = NULL; 1320 1321 *handle = DMA_ERROR_CODE; 1322 size = PAGE_ALIGN(size); 1323 1324 if (!(gfp & __GFP_WAIT)) 1325 return __iommu_alloc_atomic(dev, size, handle); 1326 1327 /* 1328 * Following is a work-around (a.k.a. hack) to prevent pages 1329 * with __GFP_COMP being passed to split_page() which cannot 1330 * handle them. The real problem is that this flag probably 1331 * should be 0 on ARM as it is not supported on this 1332 * platform; see CONFIG_HUGETLBFS. 1333 */ 1334 gfp &= ~(__GFP_COMP); 1335 1336 pages = __iommu_alloc_buffer(dev, size, gfp, attrs); 1337 if (!pages) 1338 return NULL; 1339 1340 *handle = __iommu_create_mapping(dev, pages, size); 1341 if (*handle == DMA_ERROR_CODE) 1342 goto err_buffer; 1343 1344 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) 1345 return pages; 1346 1347 addr = __iommu_alloc_remap(pages, size, gfp, prot, 1348 __builtin_return_address(0)); 1349 if (!addr) 1350 goto err_mapping; 1351 1352 return addr; 1353 1354 err_mapping: 1355 __iommu_remove_mapping(dev, *handle, size); 1356 err_buffer: 1357 __iommu_free_buffer(dev, pages, size, attrs); 1358 return NULL; 1359 } 1360 1361 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1362 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1363 struct dma_attrs *attrs) 1364 { 1365 unsigned long uaddr = vma->vm_start; 1366 unsigned long usize = vma->vm_end - vma->vm_start; 1367 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1368 1369 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1370 1371 if (!pages) 1372 return -ENXIO; 1373 1374 do { 1375 int ret = vm_insert_page(vma, uaddr, *pages++); 1376 if (ret) { 1377 pr_err("Remapping memory failed: %d\n", ret); 1378 return ret; 1379 } 1380 uaddr += PAGE_SIZE; 1381 usize -= PAGE_SIZE; 1382 } while (usize > 0); 1383 1384 return 0; 1385 } 1386 1387 /* 1388 * free a page as defined by the above mapping. 1389 * Must not be called with IRQs disabled. 1390 */ 1391 void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1392 dma_addr_t handle, struct dma_attrs *attrs) 1393 { 1394 struct page **pages; 1395 size = PAGE_ALIGN(size); 1396 1397 if (__in_atomic_pool(cpu_addr, size)) { 1398 __iommu_free_atomic(dev, cpu_addr, handle, size); 1399 return; 1400 } 1401 1402 pages = __iommu_get_pages(cpu_addr, attrs); 1403 if (!pages) { 1404 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1405 return; 1406 } 1407 1408 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { 1409 dma_common_free_remap(cpu_addr, size, 1410 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 1411 } 1412 1413 __iommu_remove_mapping(dev, handle, size); 1414 __iommu_free_buffer(dev, pages, size, attrs); 1415 } 1416 1417 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1418 void *cpu_addr, dma_addr_t dma_addr, 1419 size_t size, struct dma_attrs *attrs) 1420 { 1421 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1422 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1423 1424 if (!pages) 1425 return -ENXIO; 1426 1427 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1428 GFP_KERNEL); 1429 } 1430 1431 static int __dma_direction_to_prot(enum dma_data_direction dir) 1432 { 1433 int prot; 1434 1435 switch (dir) { 1436 case DMA_BIDIRECTIONAL: 1437 prot = IOMMU_READ | IOMMU_WRITE; 1438 break; 1439 case DMA_TO_DEVICE: 1440 prot = IOMMU_READ; 1441 break; 1442 case DMA_FROM_DEVICE: 1443 prot = IOMMU_WRITE; 1444 break; 1445 default: 1446 prot = 0; 1447 } 1448 1449 return prot; 1450 } 1451 1452 /* 1453 * Map a part of the scatter-gather list into contiguous io address space 1454 */ 1455 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1456 size_t size, dma_addr_t *handle, 1457 enum dma_data_direction dir, struct dma_attrs *attrs, 1458 bool is_coherent) 1459 { 1460 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1461 dma_addr_t iova, iova_base; 1462 int ret = 0; 1463 unsigned int count; 1464 struct scatterlist *s; 1465 int prot; 1466 1467 size = PAGE_ALIGN(size); 1468 *handle = DMA_ERROR_CODE; 1469 1470 iova_base = iova = __alloc_iova(mapping, size); 1471 if (iova == DMA_ERROR_CODE) 1472 return -ENOMEM; 1473 1474 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1475 phys_addr_t phys = page_to_phys(sg_page(s)); 1476 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1477 1478 if (!is_coherent && 1479 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1480 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1481 1482 prot = __dma_direction_to_prot(dir); 1483 1484 ret = iommu_map(mapping->domain, iova, phys, len, prot); 1485 if (ret < 0) 1486 goto fail; 1487 count += len >> PAGE_SHIFT; 1488 iova += len; 1489 } 1490 *handle = iova_base; 1491 1492 return 0; 1493 fail: 1494 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1495 __free_iova(mapping, iova_base, size); 1496 return ret; 1497 } 1498 1499 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1500 enum dma_data_direction dir, struct dma_attrs *attrs, 1501 bool is_coherent) 1502 { 1503 struct scatterlist *s = sg, *dma = sg, *start = sg; 1504 int i, count = 0; 1505 unsigned int offset = s->offset; 1506 unsigned int size = s->offset + s->length; 1507 unsigned int max = dma_get_max_seg_size(dev); 1508 1509 for (i = 1; i < nents; i++) { 1510 s = sg_next(s); 1511 1512 s->dma_address = DMA_ERROR_CODE; 1513 s->dma_length = 0; 1514 1515 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1516 if (__map_sg_chunk(dev, start, size, &dma->dma_address, 1517 dir, attrs, is_coherent) < 0) 1518 goto bad_mapping; 1519 1520 dma->dma_address += offset; 1521 dma->dma_length = size - offset; 1522 1523 size = offset = s->offset; 1524 start = s; 1525 dma = sg_next(dma); 1526 count += 1; 1527 } 1528 size += s->length; 1529 } 1530 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, 1531 is_coherent) < 0) 1532 goto bad_mapping; 1533 1534 dma->dma_address += offset; 1535 dma->dma_length = size - offset; 1536 1537 return count+1; 1538 1539 bad_mapping: 1540 for_each_sg(sg, s, count, i) 1541 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1542 return 0; 1543 } 1544 1545 /** 1546 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1547 * @dev: valid struct device pointer 1548 * @sg: list of buffers 1549 * @nents: number of buffers to map 1550 * @dir: DMA transfer direction 1551 * 1552 * Map a set of i/o coherent buffers described by scatterlist in streaming 1553 * mode for DMA. The scatter gather list elements are merged together (if 1554 * possible) and tagged with the appropriate dma address and length. They are 1555 * obtained via sg_dma_{address,length}. 1556 */ 1557 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1558 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1559 { 1560 return __iommu_map_sg(dev, sg, nents, dir, attrs, true); 1561 } 1562 1563 /** 1564 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1565 * @dev: valid struct device pointer 1566 * @sg: list of buffers 1567 * @nents: number of buffers to map 1568 * @dir: DMA transfer direction 1569 * 1570 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1571 * The scatter gather list elements are merged together (if possible) and 1572 * tagged with the appropriate dma address and length. They are obtained via 1573 * sg_dma_{address,length}. 1574 */ 1575 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1576 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1577 { 1578 return __iommu_map_sg(dev, sg, nents, dir, attrs, false); 1579 } 1580 1581 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1582 int nents, enum dma_data_direction dir, struct dma_attrs *attrs, 1583 bool is_coherent) 1584 { 1585 struct scatterlist *s; 1586 int i; 1587 1588 for_each_sg(sg, s, nents, i) { 1589 if (sg_dma_len(s)) 1590 __iommu_remove_mapping(dev, sg_dma_address(s), 1591 sg_dma_len(s)); 1592 if (!is_coherent && 1593 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1594 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1595 s->length, dir); 1596 } 1597 } 1598 1599 /** 1600 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1601 * @dev: valid struct device pointer 1602 * @sg: list of buffers 1603 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1604 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1605 * 1606 * Unmap a set of streaming mode DMA translations. Again, CPU access 1607 * rules concerning calls here are the same as for dma_unmap_single(). 1608 */ 1609 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1610 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 1611 { 1612 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); 1613 } 1614 1615 /** 1616 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1617 * @dev: valid struct device pointer 1618 * @sg: list of buffers 1619 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1620 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1621 * 1622 * Unmap a set of streaming mode DMA translations. Again, CPU access 1623 * rules concerning calls here are the same as for dma_unmap_single(). 1624 */ 1625 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1626 enum dma_data_direction dir, struct dma_attrs *attrs) 1627 { 1628 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); 1629 } 1630 1631 /** 1632 * arm_iommu_sync_sg_for_cpu 1633 * @dev: valid struct device pointer 1634 * @sg: list of buffers 1635 * @nents: number of buffers to map (returned from dma_map_sg) 1636 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1637 */ 1638 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1639 int nents, enum dma_data_direction dir) 1640 { 1641 struct scatterlist *s; 1642 int i; 1643 1644 for_each_sg(sg, s, nents, i) 1645 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1646 1647 } 1648 1649 /** 1650 * arm_iommu_sync_sg_for_device 1651 * @dev: valid struct device pointer 1652 * @sg: list of buffers 1653 * @nents: number of buffers to map (returned from dma_map_sg) 1654 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1655 */ 1656 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1657 int nents, enum dma_data_direction dir) 1658 { 1659 struct scatterlist *s; 1660 int i; 1661 1662 for_each_sg(sg, s, nents, i) 1663 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1664 } 1665 1666 1667 /** 1668 * arm_coherent_iommu_map_page 1669 * @dev: valid struct device pointer 1670 * @page: page that buffer resides in 1671 * @offset: offset into page for start of buffer 1672 * @size: size of buffer to map 1673 * @dir: DMA transfer direction 1674 * 1675 * Coherent IOMMU aware version of arm_dma_map_page() 1676 */ 1677 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, 1678 unsigned long offset, size_t size, enum dma_data_direction dir, 1679 struct dma_attrs *attrs) 1680 { 1681 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1682 dma_addr_t dma_addr; 1683 int ret, prot, len = PAGE_ALIGN(size + offset); 1684 1685 dma_addr = __alloc_iova(mapping, len); 1686 if (dma_addr == DMA_ERROR_CODE) 1687 return dma_addr; 1688 1689 prot = __dma_direction_to_prot(dir); 1690 1691 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); 1692 if (ret < 0) 1693 goto fail; 1694 1695 return dma_addr + offset; 1696 fail: 1697 __free_iova(mapping, dma_addr, len); 1698 return DMA_ERROR_CODE; 1699 } 1700 1701 /** 1702 * arm_iommu_map_page 1703 * @dev: valid struct device pointer 1704 * @page: page that buffer resides in 1705 * @offset: offset into page for start of buffer 1706 * @size: size of buffer to map 1707 * @dir: DMA transfer direction 1708 * 1709 * IOMMU aware version of arm_dma_map_page() 1710 */ 1711 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1712 unsigned long offset, size_t size, enum dma_data_direction dir, 1713 struct dma_attrs *attrs) 1714 { 1715 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1716 __dma_page_cpu_to_dev(page, offset, size, dir); 1717 1718 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); 1719 } 1720 1721 /** 1722 * arm_coherent_iommu_unmap_page 1723 * @dev: valid struct device pointer 1724 * @handle: DMA address of buffer 1725 * @size: size of buffer (same as passed to dma_map_page) 1726 * @dir: DMA transfer direction (same as passed to dma_map_page) 1727 * 1728 * Coherent IOMMU aware version of arm_dma_unmap_page() 1729 */ 1730 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1731 size_t size, enum dma_data_direction dir, 1732 struct dma_attrs *attrs) 1733 { 1734 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1735 dma_addr_t iova = handle & PAGE_MASK; 1736 int offset = handle & ~PAGE_MASK; 1737 int len = PAGE_ALIGN(size + offset); 1738 1739 if (!iova) 1740 return; 1741 1742 iommu_unmap(mapping->domain, iova, len); 1743 __free_iova(mapping, iova, len); 1744 } 1745 1746 /** 1747 * arm_iommu_unmap_page 1748 * @dev: valid struct device pointer 1749 * @handle: DMA address of buffer 1750 * @size: size of buffer (same as passed to dma_map_page) 1751 * @dir: DMA transfer direction (same as passed to dma_map_page) 1752 * 1753 * IOMMU aware version of arm_dma_unmap_page() 1754 */ 1755 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1756 size_t size, enum dma_data_direction dir, 1757 struct dma_attrs *attrs) 1758 { 1759 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1760 dma_addr_t iova = handle & PAGE_MASK; 1761 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1762 int offset = handle & ~PAGE_MASK; 1763 int len = PAGE_ALIGN(size + offset); 1764 1765 if (!iova) 1766 return; 1767 1768 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) 1769 __dma_page_dev_to_cpu(page, offset, size, dir); 1770 1771 iommu_unmap(mapping->domain, iova, len); 1772 __free_iova(mapping, iova, len); 1773 } 1774 1775 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1776 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1777 { 1778 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1779 dma_addr_t iova = handle & PAGE_MASK; 1780 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1781 unsigned int offset = handle & ~PAGE_MASK; 1782 1783 if (!iova) 1784 return; 1785 1786 __dma_page_dev_to_cpu(page, offset, size, dir); 1787 } 1788 1789 static void arm_iommu_sync_single_for_device(struct device *dev, 1790 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1791 { 1792 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1793 dma_addr_t iova = handle & PAGE_MASK; 1794 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1795 unsigned int offset = handle & ~PAGE_MASK; 1796 1797 if (!iova) 1798 return; 1799 1800 __dma_page_cpu_to_dev(page, offset, size, dir); 1801 } 1802 1803 struct dma_map_ops iommu_ops = { 1804 .alloc = arm_iommu_alloc_attrs, 1805 .free = arm_iommu_free_attrs, 1806 .mmap = arm_iommu_mmap_attrs, 1807 .get_sgtable = arm_iommu_get_sgtable, 1808 1809 .map_page = arm_iommu_map_page, 1810 .unmap_page = arm_iommu_unmap_page, 1811 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 1812 .sync_single_for_device = arm_iommu_sync_single_for_device, 1813 1814 .map_sg = arm_iommu_map_sg, 1815 .unmap_sg = arm_iommu_unmap_sg, 1816 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 1817 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 1818 1819 .set_dma_mask = arm_dma_set_mask, 1820 }; 1821 1822 struct dma_map_ops iommu_coherent_ops = { 1823 .alloc = arm_iommu_alloc_attrs, 1824 .free = arm_iommu_free_attrs, 1825 .mmap = arm_iommu_mmap_attrs, 1826 .get_sgtable = arm_iommu_get_sgtable, 1827 1828 .map_page = arm_coherent_iommu_map_page, 1829 .unmap_page = arm_coherent_iommu_unmap_page, 1830 1831 .map_sg = arm_coherent_iommu_map_sg, 1832 .unmap_sg = arm_coherent_iommu_unmap_sg, 1833 1834 .set_dma_mask = arm_dma_set_mask, 1835 }; 1836 1837 /** 1838 * arm_iommu_create_mapping 1839 * @bus: pointer to the bus holding the client device (for IOMMU calls) 1840 * @base: start address of the valid IO address space 1841 * @size: maximum size of the valid IO address space 1842 * 1843 * Creates a mapping structure which holds information about used/unused 1844 * IO address ranges, which is required to perform memory allocation and 1845 * mapping with IOMMU aware functions. 1846 * 1847 * The client device need to be attached to the mapping with 1848 * arm_iommu_attach_device function. 1849 */ 1850 struct dma_iommu_mapping * 1851 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size) 1852 { 1853 unsigned int bits = size >> PAGE_SHIFT; 1854 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); 1855 struct dma_iommu_mapping *mapping; 1856 int extensions = 1; 1857 int err = -ENOMEM; 1858 1859 if (!bitmap_size) 1860 return ERR_PTR(-EINVAL); 1861 1862 if (bitmap_size > PAGE_SIZE) { 1863 extensions = bitmap_size / PAGE_SIZE; 1864 bitmap_size = PAGE_SIZE; 1865 } 1866 1867 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 1868 if (!mapping) 1869 goto err; 1870 1871 mapping->bitmap_size = bitmap_size; 1872 mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *), 1873 GFP_KERNEL); 1874 if (!mapping->bitmaps) 1875 goto err2; 1876 1877 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); 1878 if (!mapping->bitmaps[0]) 1879 goto err3; 1880 1881 mapping->nr_bitmaps = 1; 1882 mapping->extensions = extensions; 1883 mapping->base = base; 1884 mapping->bits = BITS_PER_BYTE * bitmap_size; 1885 1886 spin_lock_init(&mapping->lock); 1887 1888 mapping->domain = iommu_domain_alloc(bus); 1889 if (!mapping->domain) 1890 goto err4; 1891 1892 kref_init(&mapping->kref); 1893 return mapping; 1894 err4: 1895 kfree(mapping->bitmaps[0]); 1896 err3: 1897 kfree(mapping->bitmaps); 1898 err2: 1899 kfree(mapping); 1900 err: 1901 return ERR_PTR(err); 1902 } 1903 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 1904 1905 static void release_iommu_mapping(struct kref *kref) 1906 { 1907 int i; 1908 struct dma_iommu_mapping *mapping = 1909 container_of(kref, struct dma_iommu_mapping, kref); 1910 1911 iommu_domain_free(mapping->domain); 1912 for (i = 0; i < mapping->nr_bitmaps; i++) 1913 kfree(mapping->bitmaps[i]); 1914 kfree(mapping->bitmaps); 1915 kfree(mapping); 1916 } 1917 1918 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping) 1919 { 1920 int next_bitmap; 1921 1922 if (mapping->nr_bitmaps > mapping->extensions) 1923 return -EINVAL; 1924 1925 next_bitmap = mapping->nr_bitmaps; 1926 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, 1927 GFP_ATOMIC); 1928 if (!mapping->bitmaps[next_bitmap]) 1929 return -ENOMEM; 1930 1931 mapping->nr_bitmaps++; 1932 1933 return 0; 1934 } 1935 1936 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 1937 { 1938 if (mapping) 1939 kref_put(&mapping->kref, release_iommu_mapping); 1940 } 1941 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 1942 1943 /** 1944 * arm_iommu_attach_device 1945 * @dev: valid struct device pointer 1946 * @mapping: io address space mapping structure (returned from 1947 * arm_iommu_create_mapping) 1948 * 1949 * Attaches specified io address space mapping to the provided device, 1950 * More than one client might be attached to the same io address space 1951 * mapping. 1952 */ 1953 int arm_iommu_attach_device(struct device *dev, 1954 struct dma_iommu_mapping *mapping) 1955 { 1956 int err; 1957 1958 err = iommu_attach_device(mapping->domain, dev); 1959 if (err) 1960 return err; 1961 1962 kref_get(&mapping->kref); 1963 dev->archdata.mapping = mapping; 1964 1965 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 1966 return 0; 1967 } 1968 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 1969 1970 /** 1971 * arm_iommu_detach_device 1972 * @dev: valid struct device pointer 1973 * 1974 * Detaches the provided device from a previously attached map. 1975 */ 1976 void arm_iommu_detach_device(struct device *dev) 1977 { 1978 struct dma_iommu_mapping *mapping; 1979 1980 mapping = to_dma_iommu_mapping(dev); 1981 if (!mapping) { 1982 dev_warn(dev, "Not attached\n"); 1983 return; 1984 } 1985 1986 iommu_detach_device(mapping->domain, dev); 1987 kref_put(&mapping->kref, release_iommu_mapping); 1988 dev->archdata.mapping = NULL; 1989 1990 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 1991 } 1992 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 1993 1994 static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) 1995 { 1996 return coherent ? &iommu_coherent_ops : &iommu_ops; 1997 } 1998 1999 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2000 struct iommu_ops *iommu) 2001 { 2002 struct dma_iommu_mapping *mapping; 2003 2004 if (!iommu) 2005 return false; 2006 2007 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 2008 if (IS_ERR(mapping)) { 2009 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 2010 size, dev_name(dev)); 2011 return false; 2012 } 2013 2014 if (arm_iommu_attach_device(dev, mapping)) { 2015 pr_warn("Failed to attached device %s to IOMMU_mapping\n", 2016 dev_name(dev)); 2017 arm_iommu_release_mapping(mapping); 2018 return false; 2019 } 2020 2021 return true; 2022 } 2023 2024 static void arm_teardown_iommu_dma_ops(struct device *dev) 2025 { 2026 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 2027 2028 arm_iommu_detach_device(dev); 2029 arm_iommu_release_mapping(mapping); 2030 } 2031 2032 #else 2033 2034 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2035 struct iommu_ops *iommu) 2036 { 2037 return false; 2038 } 2039 2040 static void arm_teardown_iommu_dma_ops(struct device *dev) { } 2041 2042 #define arm_get_iommu_dma_map_ops arm_get_dma_map_ops 2043 2044 #endif /* CONFIG_ARM_DMA_USE_IOMMU */ 2045 2046 static struct dma_map_ops *arm_get_dma_map_ops(bool coherent) 2047 { 2048 return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; 2049 } 2050 2051 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 2052 struct iommu_ops *iommu, bool coherent) 2053 { 2054 struct dma_map_ops *dma_ops; 2055 2056 dev->archdata.dma_coherent = coherent; 2057 if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) 2058 dma_ops = arm_get_iommu_dma_map_ops(coherent); 2059 else 2060 dma_ops = arm_get_dma_map_ops(coherent); 2061 2062 set_dma_ops(dev, dma_ops); 2063 } 2064 2065 void arch_teardown_dma_ops(struct device *dev) 2066 { 2067 arm_teardown_iommu_dma_ops(dev); 2068 } 2069