10ddbccd1SRussell King /* 20ddbccd1SRussell King * linux/arch/arm/mm/dma-mapping.c 30ddbccd1SRussell King * 40ddbccd1SRussell King * Copyright (C) 2000-2004 Russell King 50ddbccd1SRussell King * 60ddbccd1SRussell King * This program is free software; you can redistribute it and/or modify 70ddbccd1SRussell King * it under the terms of the GNU General Public License version 2 as 80ddbccd1SRussell King * published by the Free Software Foundation. 90ddbccd1SRussell King * 100ddbccd1SRussell King * DMA uncached mapping support. 110ddbccd1SRussell King */ 120ddbccd1SRussell King #include <linux/module.h> 130ddbccd1SRussell King #include <linux/mm.h> 145a0e3ad6STejun Heo #include <linux/gfp.h> 150ddbccd1SRussell King #include <linux/errno.h> 160ddbccd1SRussell King #include <linux/list.h> 170ddbccd1SRussell King #include <linux/init.h> 180ddbccd1SRussell King #include <linux/device.h> 190ddbccd1SRussell King #include <linux/dma-mapping.h> 200ddbccd1SRussell King 210ddbccd1SRussell King #include <asm/memory.h> 2243377453SNicolas Pitre #include <asm/highmem.h> 230ddbccd1SRussell King #include <asm/cacheflush.h> 240ddbccd1SRussell King #include <asm/tlbflush.h> 250ddbccd1SRussell King #include <asm/sizes.h> 260ddbccd1SRussell King 27ab6494f0SCatalin Marinas static u64 get_coherent_dma_mask(struct device *dev) 28ab6494f0SCatalin Marinas { 29ab6494f0SCatalin Marinas u64 mask = ISA_DMA_THRESHOLD; 300ddbccd1SRussell King 31ab6494f0SCatalin Marinas if (dev) { 32ab6494f0SCatalin Marinas mask = dev->coherent_dma_mask; 33ab6494f0SCatalin Marinas 34ab6494f0SCatalin Marinas /* 35ab6494f0SCatalin Marinas * Sanity check the DMA mask - it must be non-zero, and 36ab6494f0SCatalin Marinas * must be able to be satisfied by a DMA allocation. 37ab6494f0SCatalin Marinas */ 38ab6494f0SCatalin Marinas if (mask == 0) { 39ab6494f0SCatalin Marinas dev_warn(dev, "coherent DMA mask is unset\n"); 40ab6494f0SCatalin Marinas return 0; 41ab6494f0SCatalin Marinas } 42ab6494f0SCatalin Marinas 43ab6494f0SCatalin Marinas if ((~mask) & ISA_DMA_THRESHOLD) { 44ab6494f0SCatalin Marinas dev_warn(dev, "coherent DMA mask %#llx is smaller " 45ab6494f0SCatalin Marinas "than system GFP_DMA mask %#llx\n", 46ab6494f0SCatalin Marinas mask, (unsigned long long)ISA_DMA_THRESHOLD); 47ab6494f0SCatalin Marinas return 0; 48ab6494f0SCatalin Marinas } 49ab6494f0SCatalin Marinas } 50ab6494f0SCatalin Marinas 51ab6494f0SCatalin Marinas return mask; 52ab6494f0SCatalin Marinas } 53ab6494f0SCatalin Marinas 547a9a32a9SRussell King /* 557a9a32a9SRussell King * Allocate a DMA buffer for 'dev' of size 'size' using the 567a9a32a9SRussell King * specified gfp mask. Note that 'size' must be page aligned. 577a9a32a9SRussell King */ 587a9a32a9SRussell King static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) 597a9a32a9SRussell King { 607a9a32a9SRussell King unsigned long order = get_order(size); 617a9a32a9SRussell King struct page *page, *p, *e; 627a9a32a9SRussell King void *ptr; 637a9a32a9SRussell King u64 mask = get_coherent_dma_mask(dev); 647a9a32a9SRussell King 657a9a32a9SRussell King #ifdef CONFIG_DMA_API_DEBUG 667a9a32a9SRussell King u64 limit = (mask + 1) & ~mask; 677a9a32a9SRussell King if (limit && size >= limit) { 687a9a32a9SRussell King dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 697a9a32a9SRussell King size, mask); 707a9a32a9SRussell King return NULL; 717a9a32a9SRussell King } 727a9a32a9SRussell King #endif 737a9a32a9SRussell King 747a9a32a9SRussell King if (!mask) 757a9a32a9SRussell King return NULL; 767a9a32a9SRussell King 777a9a32a9SRussell King if (mask < 0xffffffffULL) 787a9a32a9SRussell King gfp |= GFP_DMA; 797a9a32a9SRussell King 807a9a32a9SRussell King page = alloc_pages(gfp, order); 817a9a32a9SRussell King if (!page) 827a9a32a9SRussell King return NULL; 837a9a32a9SRussell King 847a9a32a9SRussell King /* 857a9a32a9SRussell King * Now split the huge page and free the excess pages 867a9a32a9SRussell King */ 877a9a32a9SRussell King split_page(page, order); 887a9a32a9SRussell King for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 897a9a32a9SRussell King __free_page(p); 907a9a32a9SRussell King 917a9a32a9SRussell King /* 927a9a32a9SRussell King * Ensure that the allocated pages are zeroed, and that any data 937a9a32a9SRussell King * lurking in the kernel direct-mapped region is invalidated. 947a9a32a9SRussell King */ 957a9a32a9SRussell King ptr = page_address(page); 967a9a32a9SRussell King memset(ptr, 0, size); 977a9a32a9SRussell King dmac_flush_range(ptr, ptr + size); 987a9a32a9SRussell King outer_flush_range(__pa(ptr), __pa(ptr) + size); 997a9a32a9SRussell King 1007a9a32a9SRussell King return page; 1017a9a32a9SRussell King } 1027a9a32a9SRussell King 1037a9a32a9SRussell King /* 1047a9a32a9SRussell King * Free a DMA buffer. 'size' must be page aligned. 1057a9a32a9SRussell King */ 1067a9a32a9SRussell King static void __dma_free_buffer(struct page *page, size_t size) 1077a9a32a9SRussell King { 1087a9a32a9SRussell King struct page *e = page + (size >> PAGE_SHIFT); 1097a9a32a9SRussell King 1107a9a32a9SRussell King while (page < e) { 1117a9a32a9SRussell King __free_page(page); 1127a9a32a9SRussell King page++; 1137a9a32a9SRussell King } 1147a9a32a9SRussell King } 1157a9a32a9SRussell King 116ab6494f0SCatalin Marinas #ifdef CONFIG_MMU 117a5e9d38bSCatalin Marinas /* Sanity check size */ 118a5e9d38bSCatalin Marinas #if (CONSISTENT_DMA_SIZE % SZ_2M) 119a5e9d38bSCatalin Marinas #error "CONSISTENT_DMA_SIZE must be multiple of 2MiB" 120a5e9d38bSCatalin Marinas #endif 121a5e9d38bSCatalin Marinas 122a5e9d38bSCatalin Marinas #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT) 123a5e9d38bSCatalin Marinas #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT) 124a5e9d38bSCatalin Marinas #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT) 125a5e9d38bSCatalin Marinas 1260ddbccd1SRussell King /* 1270ddbccd1SRussell King * These are the page tables (2MB each) covering uncached, DMA consistent allocations 1280ddbccd1SRussell King */ 1290ddbccd1SRussell King static pte_t *consistent_pte[NUM_CONSISTENT_PTES]; 1300ddbccd1SRussell King 13113ccf3adSRussell King #include "vmregion.h" 1320ddbccd1SRussell King 13313ccf3adSRussell King static struct arm_vmregion_head consistent_head = { 13413ccf3adSRussell King .vm_lock = __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock), 1350ddbccd1SRussell King .vm_list = LIST_HEAD_INIT(consistent_head.vm_list), 1360ddbccd1SRussell King .vm_start = CONSISTENT_BASE, 1370ddbccd1SRussell King .vm_end = CONSISTENT_END, 1380ddbccd1SRussell King }; 1390ddbccd1SRussell King 1400ddbccd1SRussell King #ifdef CONFIG_HUGETLB_PAGE 1410ddbccd1SRussell King #error ARM Coherent DMA allocator does not (yet) support huge TLB 1420ddbccd1SRussell King #endif 1430ddbccd1SRussell King 14488c58f3bSRussell King /* 14588c58f3bSRussell King * Initialise the consistent memory allocation. 14688c58f3bSRussell King */ 14788c58f3bSRussell King static int __init consistent_init(void) 14888c58f3bSRussell King { 14988c58f3bSRussell King int ret = 0; 15088c58f3bSRussell King pgd_t *pgd; 15188c58f3bSRussell King pmd_t *pmd; 15288c58f3bSRussell King pte_t *pte; 15388c58f3bSRussell King int i = 0; 15488c58f3bSRussell King u32 base = CONSISTENT_BASE; 15588c58f3bSRussell King 15688c58f3bSRussell King do { 15788c58f3bSRussell King pgd = pgd_offset(&init_mm, base); 15888c58f3bSRussell King pmd = pmd_alloc(&init_mm, pgd, base); 15988c58f3bSRussell King if (!pmd) { 16088c58f3bSRussell King printk(KERN_ERR "%s: no pmd tables\n", __func__); 16188c58f3bSRussell King ret = -ENOMEM; 16288c58f3bSRussell King break; 16388c58f3bSRussell King } 16488c58f3bSRussell King WARN_ON(!pmd_none(*pmd)); 16588c58f3bSRussell King 16688c58f3bSRussell King pte = pte_alloc_kernel(pmd, base); 16788c58f3bSRussell King if (!pte) { 16888c58f3bSRussell King printk(KERN_ERR "%s: no pte tables\n", __func__); 16988c58f3bSRussell King ret = -ENOMEM; 17088c58f3bSRussell King break; 17188c58f3bSRussell King } 17288c58f3bSRussell King 17388c58f3bSRussell King consistent_pte[i++] = pte; 17488c58f3bSRussell King base += (1 << PGDIR_SHIFT); 17588c58f3bSRussell King } while (base < CONSISTENT_END); 17688c58f3bSRussell King 17788c58f3bSRussell King return ret; 17888c58f3bSRussell King } 17988c58f3bSRussell King 18088c58f3bSRussell King core_initcall(consistent_init); 18188c58f3bSRussell King 1820ddbccd1SRussell King static void * 18331ebf944SRussell King __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot) 1840ddbccd1SRussell King { 18513ccf3adSRussell King struct arm_vmregion *c; 1865bc23d32SRussell King size_t align; 1875bc23d32SRussell King int bit; 1880ddbccd1SRussell King 189ebd7a845SRussell King if (!consistent_pte[0]) { 190ebd7a845SRussell King printk(KERN_ERR "%s: not initialised\n", __func__); 191ebd7a845SRussell King dump_stack(); 192ebd7a845SRussell King return NULL; 193ebd7a845SRussell King } 194ebd7a845SRussell King 1950ddbccd1SRussell King /* 1965bc23d32SRussell King * Align the virtual region allocation - maximum alignment is 1975bc23d32SRussell King * a section size, minimum is a page size. This helps reduce 1985bc23d32SRussell King * fragmentation of the DMA space, and also prevents allocations 1995bc23d32SRussell King * smaller than a section from crossing a section boundary. 2005bc23d32SRussell King */ 2015bc23d32SRussell King bit = fls(size - 1) + 1; 2025bc23d32SRussell King if (bit > SECTION_SHIFT) 2035bc23d32SRussell King bit = SECTION_SHIFT; 2045bc23d32SRussell King align = 1 << bit; 2055bc23d32SRussell King 2065bc23d32SRussell King /* 2070ddbccd1SRussell King * Allocate a virtual address in the consistent mapping region. 2080ddbccd1SRussell King */ 2095bc23d32SRussell King c = arm_vmregion_alloc(&consistent_head, align, size, 2100ddbccd1SRussell King gfp & ~(__GFP_DMA | __GFP_HIGHMEM)); 2110ddbccd1SRussell King if (c) { 2120ddbccd1SRussell King pte_t *pte; 2130ddbccd1SRussell King int idx = CONSISTENT_PTE_INDEX(c->vm_start); 2140ddbccd1SRussell King u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); 2150ddbccd1SRussell King 2160ddbccd1SRussell King pte = consistent_pte[idx] + off; 2170ddbccd1SRussell King c->vm_pages = page; 2180ddbccd1SRussell King 2190ddbccd1SRussell King do { 2200ddbccd1SRussell King BUG_ON(!pte_none(*pte)); 2210ddbccd1SRussell King 2220ddbccd1SRussell King set_pte_ext(pte, mk_pte(page, prot), 0); 2230ddbccd1SRussell King page++; 2240ddbccd1SRussell King pte++; 2250ddbccd1SRussell King off++; 2260ddbccd1SRussell King if (off >= PTRS_PER_PTE) { 2270ddbccd1SRussell King off = 0; 2280ddbccd1SRussell King pte = consistent_pte[++idx]; 2290ddbccd1SRussell King } 2300ddbccd1SRussell King } while (size -= PAGE_SIZE); 2310ddbccd1SRussell King 2322be23c47SRussell King dsb(); 2332be23c47SRussell King 2340ddbccd1SRussell King return (void *)c->vm_start; 2350ddbccd1SRussell King } 2360ddbccd1SRussell King return NULL; 2370ddbccd1SRussell King } 238695ae0afSRussell King 239695ae0afSRussell King static void __dma_free_remap(void *cpu_addr, size_t size) 240695ae0afSRussell King { 241695ae0afSRussell King struct arm_vmregion *c; 242695ae0afSRussell King unsigned long addr; 243695ae0afSRussell King pte_t *ptep; 244695ae0afSRussell King int idx; 245695ae0afSRussell King u32 off; 246695ae0afSRussell King 247695ae0afSRussell King c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); 248695ae0afSRussell King if (!c) { 249695ae0afSRussell King printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n", 250695ae0afSRussell King __func__, cpu_addr); 251695ae0afSRussell King dump_stack(); 252695ae0afSRussell King return; 253695ae0afSRussell King } 254695ae0afSRussell King 255695ae0afSRussell King if ((c->vm_end - c->vm_start) != size) { 256695ae0afSRussell King printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n", 257695ae0afSRussell King __func__, c->vm_end - c->vm_start, size); 258695ae0afSRussell King dump_stack(); 259695ae0afSRussell King size = c->vm_end - c->vm_start; 260695ae0afSRussell King } 261695ae0afSRussell King 262695ae0afSRussell King idx = CONSISTENT_PTE_INDEX(c->vm_start); 263695ae0afSRussell King off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); 264695ae0afSRussell King ptep = consistent_pte[idx] + off; 265695ae0afSRussell King addr = c->vm_start; 266695ae0afSRussell King do { 267695ae0afSRussell King pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); 268695ae0afSRussell King 269695ae0afSRussell King ptep++; 270695ae0afSRussell King addr += PAGE_SIZE; 271695ae0afSRussell King off++; 272695ae0afSRussell King if (off >= PTRS_PER_PTE) { 273695ae0afSRussell King off = 0; 274695ae0afSRussell King ptep = consistent_pte[++idx]; 275695ae0afSRussell King } 276695ae0afSRussell King 277acaac256SRussell King if (pte_none(pte) || !pte_present(pte)) 278695ae0afSRussell King printk(KERN_CRIT "%s: bad page in kernel page table\n", 279695ae0afSRussell King __func__); 280695ae0afSRussell King } while (size -= PAGE_SIZE); 281695ae0afSRussell King 282695ae0afSRussell King flush_tlb_kernel_range(c->vm_start, c->vm_end); 283695ae0afSRussell King 284695ae0afSRussell King arm_vmregion_free(&consistent_head, c); 285695ae0afSRussell King } 286695ae0afSRussell King 287ab6494f0SCatalin Marinas #else /* !CONFIG_MMU */ 288695ae0afSRussell King 28931ebf944SRussell King #define __dma_alloc_remap(page, size, gfp, prot) page_address(page) 29031ebf944SRussell King #define __dma_free_remap(addr, size) do { } while (0) 29131ebf944SRussell King 29231ebf944SRussell King #endif /* CONFIG_MMU */ 29331ebf944SRussell King 294ab6494f0SCatalin Marinas static void * 295ab6494f0SCatalin Marinas __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, 296ab6494f0SCatalin Marinas pgprot_t prot) 297ab6494f0SCatalin Marinas { 29804da5694SRussell King struct page *page; 29931ebf944SRussell King void *addr; 300ab6494f0SCatalin Marinas 301ab6494f0SCatalin Marinas *handle = ~0; 30204da5694SRussell King size = PAGE_ALIGN(size); 30304da5694SRussell King 30404da5694SRussell King page = __dma_alloc_buffer(dev, size, gfp); 30504da5694SRussell King if (!page) 306ab6494f0SCatalin Marinas return NULL; 30704da5694SRussell King 30831ebf944SRussell King if (!arch_is_coherent()) 30931ebf944SRussell King addr = __dma_alloc_remap(page, size, gfp, prot); 31031ebf944SRussell King else 31131ebf944SRussell King addr = page_address(page); 31231ebf944SRussell King 31331ebf944SRussell King if (addr) 31404da5694SRussell King *handle = page_to_dma(dev, page); 31531ebf944SRussell King 31631ebf944SRussell King return addr; 317ab6494f0SCatalin Marinas } 318695ae0afSRussell King 3190ddbccd1SRussell King /* 3200ddbccd1SRussell King * Allocate DMA-coherent memory space and return both the kernel remapped 3210ddbccd1SRussell King * virtual and bus address for that space. 3220ddbccd1SRussell King */ 3230ddbccd1SRussell King void * 3240ddbccd1SRussell King dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp) 3250ddbccd1SRussell King { 3260ddbccd1SRussell King void *memory; 3270ddbccd1SRussell King 3280ddbccd1SRussell King if (dma_alloc_from_coherent(dev, size, handle, &memory)) 3290ddbccd1SRussell King return memory; 3300ddbccd1SRussell King 3310ddbccd1SRussell King return __dma_alloc(dev, size, handle, gfp, 33226a26d32SRussell King pgprot_dmacoherent(pgprot_kernel)); 3330ddbccd1SRussell King } 3340ddbccd1SRussell King EXPORT_SYMBOL(dma_alloc_coherent); 3350ddbccd1SRussell King 3360ddbccd1SRussell King /* 3370ddbccd1SRussell King * Allocate a writecombining region, in much the same way as 3380ddbccd1SRussell King * dma_alloc_coherent above. 3390ddbccd1SRussell King */ 3400ddbccd1SRussell King void * 3410ddbccd1SRussell King dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp) 3420ddbccd1SRussell King { 3430ddbccd1SRussell King return __dma_alloc(dev, size, handle, gfp, 3440ddbccd1SRussell King pgprot_writecombine(pgprot_kernel)); 3450ddbccd1SRussell King } 3460ddbccd1SRussell King EXPORT_SYMBOL(dma_alloc_writecombine); 3470ddbccd1SRussell King 3480ddbccd1SRussell King static int dma_mmap(struct device *dev, struct vm_area_struct *vma, 3490ddbccd1SRussell King void *cpu_addr, dma_addr_t dma_addr, size_t size) 3500ddbccd1SRussell King { 351ab6494f0SCatalin Marinas int ret = -ENXIO; 352ab6494f0SCatalin Marinas #ifdef CONFIG_MMU 35313ccf3adSRussell King unsigned long user_size, kern_size; 35413ccf3adSRussell King struct arm_vmregion *c; 3550ddbccd1SRussell King 3560ddbccd1SRussell King user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 3570ddbccd1SRussell King 35813ccf3adSRussell King c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); 3590ddbccd1SRussell King if (c) { 3600ddbccd1SRussell King unsigned long off = vma->vm_pgoff; 3610ddbccd1SRussell King 3620ddbccd1SRussell King kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT; 3630ddbccd1SRussell King 3640ddbccd1SRussell King if (off < kern_size && 3650ddbccd1SRussell King user_size <= (kern_size - off)) { 3660ddbccd1SRussell King ret = remap_pfn_range(vma, vma->vm_start, 3670ddbccd1SRussell King page_to_pfn(c->vm_pages) + off, 3680ddbccd1SRussell King user_size << PAGE_SHIFT, 3690ddbccd1SRussell King vma->vm_page_prot); 3700ddbccd1SRussell King } 3710ddbccd1SRussell King } 372ab6494f0SCatalin Marinas #endif /* CONFIG_MMU */ 3730ddbccd1SRussell King 3740ddbccd1SRussell King return ret; 3750ddbccd1SRussell King } 3760ddbccd1SRussell King 3770ddbccd1SRussell King int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma, 3780ddbccd1SRussell King void *cpu_addr, dma_addr_t dma_addr, size_t size) 3790ddbccd1SRussell King { 38026a26d32SRussell King vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot); 3810ddbccd1SRussell King return dma_mmap(dev, vma, cpu_addr, dma_addr, size); 3820ddbccd1SRussell King } 3830ddbccd1SRussell King EXPORT_SYMBOL(dma_mmap_coherent); 3840ddbccd1SRussell King 3850ddbccd1SRussell King int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma, 3860ddbccd1SRussell King void *cpu_addr, dma_addr_t dma_addr, size_t size) 3870ddbccd1SRussell King { 3880ddbccd1SRussell King vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 3890ddbccd1SRussell King return dma_mmap(dev, vma, cpu_addr, dma_addr, size); 3900ddbccd1SRussell King } 3910ddbccd1SRussell King EXPORT_SYMBOL(dma_mmap_writecombine); 3920ddbccd1SRussell King 3930ddbccd1SRussell King /* 3940ddbccd1SRussell King * free a page as defined by the above mapping. 3950ddbccd1SRussell King * Must not be called with IRQs disabled. 3960ddbccd1SRussell King */ 3970ddbccd1SRussell King void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) 3980ddbccd1SRussell King { 3990ddbccd1SRussell King WARN_ON(irqs_disabled()); 4000ddbccd1SRussell King 4010ddbccd1SRussell King if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) 4020ddbccd1SRussell King return; 4030ddbccd1SRussell King 4043e82d012SRussell King size = PAGE_ALIGN(size); 4053e82d012SRussell King 406695ae0afSRussell King if (!arch_is_coherent()) 407695ae0afSRussell King __dma_free_remap(cpu_addr, size); 4087a9a32a9SRussell King 4097a9a32a9SRussell King __dma_free_buffer(dma_to_page(dev, handle), size); 4100ddbccd1SRussell King } 4110ddbccd1SRussell King EXPORT_SYMBOL(dma_free_coherent); 4120ddbccd1SRussell King 4130ddbccd1SRussell King /* 4140ddbccd1SRussell King * Make an area consistent for devices. 4150ddbccd1SRussell King * Note: Drivers should NOT use this function directly, as it will break 4160ddbccd1SRussell King * platforms with CONFIG_DMABOUNCE. 4170ddbccd1SRussell King * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 4180ddbccd1SRussell King */ 4194ea0d737SRussell King void ___dma_single_cpu_to_dev(const void *kaddr, size_t size, 4204ea0d737SRussell King enum dma_data_direction dir) 4214ea0d737SRussell King { 4222ffe2da3SRussell King unsigned long paddr; 4232ffe2da3SRussell King 424a9c9147eSRussell King BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1)); 425a9c9147eSRussell King 426a9c9147eSRussell King dmac_map_area(kaddr, size, dir); 4272ffe2da3SRussell King 4282ffe2da3SRussell King paddr = __pa(kaddr); 4292ffe2da3SRussell King if (dir == DMA_FROM_DEVICE) { 4302ffe2da3SRussell King outer_inv_range(paddr, paddr + size); 4312ffe2da3SRussell King } else { 4322ffe2da3SRussell King outer_clean_range(paddr, paddr + size); 4332ffe2da3SRussell King } 4342ffe2da3SRussell King /* FIXME: non-speculating: flush on bidirectional mappings? */ 4354ea0d737SRussell King } 4364ea0d737SRussell King EXPORT_SYMBOL(___dma_single_cpu_to_dev); 4374ea0d737SRussell King 4384ea0d737SRussell King void ___dma_single_dev_to_cpu(const void *kaddr, size_t size, 4394ea0d737SRussell King enum dma_data_direction dir) 4404ea0d737SRussell King { 441a9c9147eSRussell King BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1)); 442a9c9147eSRussell King 4432ffe2da3SRussell King /* FIXME: non-speculating: not required */ 4442ffe2da3SRussell King /* don't bother invalidating if DMA to device */ 4452ffe2da3SRussell King if (dir != DMA_TO_DEVICE) { 4462ffe2da3SRussell King unsigned long paddr = __pa(kaddr); 4472ffe2da3SRussell King outer_inv_range(paddr, paddr + size); 4482ffe2da3SRussell King } 4492ffe2da3SRussell King 450a9c9147eSRussell King dmac_unmap_area(kaddr, size, dir); 4514ea0d737SRussell King } 4524ea0d737SRussell King EXPORT_SYMBOL(___dma_single_dev_to_cpu); 453afd1a321SRussell King 45465af191aSRussell King static void dma_cache_maint_page(struct page *page, unsigned long offset, 455a9c9147eSRussell King size_t size, enum dma_data_direction dir, 456a9c9147eSRussell King void (*op)(const void *, size_t, int)) 45765af191aSRussell King { 45865af191aSRussell King /* 45965af191aSRussell King * A single sg entry may refer to multiple physically contiguous 46065af191aSRussell King * pages. But we still need to process highmem pages individually. 46165af191aSRussell King * If highmem is not configured then the bulk of this loop gets 46265af191aSRussell King * optimized out. 46365af191aSRussell King */ 46465af191aSRussell King size_t left = size; 46565af191aSRussell King do { 46665af191aSRussell King size_t len = left; 46793f1d629SRussell King void *vaddr; 46893f1d629SRussell King 46993f1d629SRussell King if (PageHighMem(page)) { 47093f1d629SRussell King if (len + offset > PAGE_SIZE) { 47165af191aSRussell King if (offset >= PAGE_SIZE) { 47265af191aSRussell King page += offset / PAGE_SIZE; 47365af191aSRussell King offset %= PAGE_SIZE; 47465af191aSRussell King } 47565af191aSRussell King len = PAGE_SIZE - offset; 47665af191aSRussell King } 47793f1d629SRussell King vaddr = kmap_high_get(page); 47893f1d629SRussell King if (vaddr) { 47993f1d629SRussell King vaddr += offset; 480a9c9147eSRussell King op(vaddr, len, dir); 48193f1d629SRussell King kunmap_high(page); 4827e5a69e8SNicolas Pitre } else if (cache_is_vipt()) { 4837e5a69e8SNicolas Pitre pte_t saved_pte; 4847e5a69e8SNicolas Pitre vaddr = kmap_high_l1_vipt(page, &saved_pte); 4857e5a69e8SNicolas Pitre op(vaddr + offset, len, dir); 4867e5a69e8SNicolas Pitre kunmap_high_l1_vipt(page, saved_pte); 48793f1d629SRussell King } 48893f1d629SRussell King } else { 48993f1d629SRussell King vaddr = page_address(page) + offset; 490a9c9147eSRussell King op(vaddr, len, dir); 49193f1d629SRussell King } 49265af191aSRussell King offset = 0; 49365af191aSRussell King page++; 49465af191aSRussell King left -= len; 49565af191aSRussell King } while (left); 49665af191aSRussell King } 49765af191aSRussell King 49865af191aSRussell King void ___dma_page_cpu_to_dev(struct page *page, unsigned long off, 49965af191aSRussell King size_t size, enum dma_data_direction dir) 50065af191aSRussell King { 50143377453SNicolas Pitre unsigned long paddr; 50243377453SNicolas Pitre 503a9c9147eSRussell King dma_cache_maint_page(page, off, size, dir, dmac_map_area); 50443377453SNicolas Pitre 50565af191aSRussell King paddr = page_to_phys(page) + off; 5062ffe2da3SRussell King if (dir == DMA_FROM_DEVICE) { 5072ffe2da3SRussell King outer_inv_range(paddr, paddr + size); 5082ffe2da3SRussell King } else { 5092ffe2da3SRussell King outer_clean_range(paddr, paddr + size); 5102ffe2da3SRussell King } 5112ffe2da3SRussell King /* FIXME: non-speculating: flush on bidirectional mappings? */ 51243377453SNicolas Pitre } 5134ea0d737SRussell King EXPORT_SYMBOL(___dma_page_cpu_to_dev); 5144ea0d737SRussell King 5154ea0d737SRussell King void ___dma_page_dev_to_cpu(struct page *page, unsigned long off, 5164ea0d737SRussell King size_t size, enum dma_data_direction dir) 5174ea0d737SRussell King { 5182ffe2da3SRussell King unsigned long paddr = page_to_phys(page) + off; 5192ffe2da3SRussell King 5202ffe2da3SRussell King /* FIXME: non-speculating: not required */ 5212ffe2da3SRussell King /* don't bother invalidating if DMA to device */ 5222ffe2da3SRussell King if (dir != DMA_TO_DEVICE) 5232ffe2da3SRussell King outer_inv_range(paddr, paddr + size); 5242ffe2da3SRussell King 525a9c9147eSRussell King dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 526c0177800SCatalin Marinas 527c0177800SCatalin Marinas /* 528c0177800SCatalin Marinas * Mark the D-cache clean for this page to avoid extra flushing. 529c0177800SCatalin Marinas */ 530c0177800SCatalin Marinas if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) 531c0177800SCatalin Marinas set_bit(PG_dcache_clean, &page->flags); 5324ea0d737SRussell King } 5334ea0d737SRussell King EXPORT_SYMBOL(___dma_page_dev_to_cpu); 53443377453SNicolas Pitre 535afd1a321SRussell King /** 536afd1a321SRussell King * dma_map_sg - map a set of SG buffers for streaming mode DMA 537afd1a321SRussell King * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 538afd1a321SRussell King * @sg: list of buffers 539afd1a321SRussell King * @nents: number of buffers to map 540afd1a321SRussell King * @dir: DMA transfer direction 541afd1a321SRussell King * 542afd1a321SRussell King * Map a set of buffers described by scatterlist in streaming mode for DMA. 543afd1a321SRussell King * This is the scatter-gather version of the dma_map_single interface. 544afd1a321SRussell King * Here the scatter gather list elements are each tagged with the 545afd1a321SRussell King * appropriate dma address and length. They are obtained via 546afd1a321SRussell King * sg_dma_{address,length}. 547afd1a321SRussell King * 548afd1a321SRussell King * Device ownership issues as mentioned for dma_map_single are the same 549afd1a321SRussell King * here. 550afd1a321SRussell King */ 551afd1a321SRussell King int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 552afd1a321SRussell King enum dma_data_direction dir) 553afd1a321SRussell King { 554afd1a321SRussell King struct scatterlist *s; 55501135d92SRussell King int i, j; 556afd1a321SRussell King 557afd1a321SRussell King for_each_sg(sg, s, nents, i) { 55801135d92SRussell King s->dma_address = dma_map_page(dev, sg_page(s), s->offset, 55901135d92SRussell King s->length, dir); 56001135d92SRussell King if (dma_mapping_error(dev, s->dma_address)) 56101135d92SRussell King goto bad_mapping; 562afd1a321SRussell King } 563afd1a321SRussell King return nents; 56401135d92SRussell King 56501135d92SRussell King bad_mapping: 56601135d92SRussell King for_each_sg(sg, s, i, j) 56701135d92SRussell King dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir); 56801135d92SRussell King return 0; 569afd1a321SRussell King } 570afd1a321SRussell King EXPORT_SYMBOL(dma_map_sg); 571afd1a321SRussell King 572afd1a321SRussell King /** 573afd1a321SRussell King * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 574afd1a321SRussell King * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 575afd1a321SRussell King * @sg: list of buffers 576afd1a321SRussell King * @nents: number of buffers to unmap (returned from dma_map_sg) 577afd1a321SRussell King * @dir: DMA transfer direction (same as was passed to dma_map_sg) 578afd1a321SRussell King * 579afd1a321SRussell King * Unmap a set of streaming mode DMA translations. Again, CPU access 580afd1a321SRussell King * rules concerning calls here are the same as for dma_unmap_single(). 581afd1a321SRussell King */ 582afd1a321SRussell King void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 583afd1a321SRussell King enum dma_data_direction dir) 584afd1a321SRussell King { 58501135d92SRussell King struct scatterlist *s; 58601135d92SRussell King int i; 58701135d92SRussell King 58801135d92SRussell King for_each_sg(sg, s, nents, i) 58901135d92SRussell King dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir); 590afd1a321SRussell King } 591afd1a321SRussell King EXPORT_SYMBOL(dma_unmap_sg); 592afd1a321SRussell King 593afd1a321SRussell King /** 594afd1a321SRussell King * dma_sync_sg_for_cpu 595afd1a321SRussell King * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 596afd1a321SRussell King * @sg: list of buffers 597afd1a321SRussell King * @nents: number of buffers to map (returned from dma_map_sg) 598afd1a321SRussell King * @dir: DMA transfer direction (same as was passed to dma_map_sg) 599afd1a321SRussell King */ 600afd1a321SRussell King void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 601afd1a321SRussell King int nents, enum dma_data_direction dir) 602afd1a321SRussell King { 603afd1a321SRussell King struct scatterlist *s; 604afd1a321SRussell King int i; 605afd1a321SRussell King 606afd1a321SRussell King for_each_sg(sg, s, nents, i) { 60718eabe23SRussell King if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0, 60818eabe23SRussell King sg_dma_len(s), dir)) 60918eabe23SRussell King continue; 61018eabe23SRussell King 61118eabe23SRussell King __dma_page_dev_to_cpu(sg_page(s), s->offset, 61218eabe23SRussell King s->length, dir); 613afd1a321SRussell King } 614afd1a321SRussell King } 615afd1a321SRussell King EXPORT_SYMBOL(dma_sync_sg_for_cpu); 616afd1a321SRussell King 617afd1a321SRussell King /** 618afd1a321SRussell King * dma_sync_sg_for_device 619afd1a321SRussell King * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 620afd1a321SRussell King * @sg: list of buffers 621afd1a321SRussell King * @nents: number of buffers to map (returned from dma_map_sg) 622afd1a321SRussell King * @dir: DMA transfer direction (same as was passed to dma_map_sg) 623afd1a321SRussell King */ 624afd1a321SRussell King void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 625afd1a321SRussell King int nents, enum dma_data_direction dir) 626afd1a321SRussell King { 627afd1a321SRussell King struct scatterlist *s; 628afd1a321SRussell King int i; 629afd1a321SRussell King 630afd1a321SRussell King for_each_sg(sg, s, nents, i) { 6312638b4dbSRussell King if (!dmabounce_sync_for_device(dev, sg_dma_address(s), 0, 6322638b4dbSRussell King sg_dma_len(s), dir)) 6332638b4dbSRussell King continue; 6342638b4dbSRussell King 63518eabe23SRussell King __dma_page_cpu_to_dev(sg_page(s), s->offset, 63643377453SNicolas Pitre s->length, dir); 637afd1a321SRussell King } 638afd1a321SRussell King } 639afd1a321SRussell King EXPORT_SYMBOL(dma_sync_sg_for_device); 640