1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2012 ARM Ltd. 4 * Copyright (c) 2014 The Linux Foundation 5 */ 6 #include <linux/dma-direct.h> 7 #include <linux/dma-noncoherent.h> 8 #include <linux/dma-contiguous.h> 9 #include <linux/init.h> 10 #include <linux/genalloc.h> 11 #include <linux/slab.h> 12 #include <linux/vmalloc.h> 13 14 struct page **dma_common_find_pages(void *cpu_addr) 15 { 16 struct vm_struct *area = find_vm_area(cpu_addr); 17 18 if (!area || area->flags != VM_DMA_COHERENT) 19 return NULL; 20 return area->pages; 21 } 22 23 /* 24 * Remaps an array of PAGE_SIZE pages into another vm_area. 25 * Cannot be used in non-sleeping contexts 26 */ 27 void *dma_common_pages_remap(struct page **pages, size_t size, 28 pgprot_t prot, const void *caller) 29 { 30 void *vaddr; 31 32 vaddr = vmap(pages, size >> PAGE_SHIFT, VM_DMA_COHERENT, prot); 33 if (vaddr) 34 find_vm_area(vaddr)->pages = pages; 35 return vaddr; 36 } 37 38 /* 39 * Remaps an allocated contiguous region into another vm_area. 40 * Cannot be used in non-sleeping contexts 41 */ 42 void *dma_common_contiguous_remap(struct page *page, size_t size, 43 pgprot_t prot, const void *caller) 44 { 45 int count = size >> PAGE_SHIFT; 46 struct page **pages; 47 void *vaddr; 48 int i; 49 50 pages = kmalloc_array(count, sizeof(struct page *), GFP_KERNEL); 51 if (!pages) 52 return NULL; 53 for (i = 0; i < count; i++) 54 pages[i] = nth_page(page, i); 55 vaddr = vmap(pages, count, VM_DMA_COHERENT, prot); 56 kfree(pages); 57 58 return vaddr; 59 } 60 61 /* 62 * Unmaps a range previously mapped by dma_common_*_remap 63 */ 64 void dma_common_free_remap(void *cpu_addr, size_t size) 65 { 66 struct vm_struct *area = find_vm_area(cpu_addr); 67 68 if (!area || area->flags != VM_DMA_COHERENT) { 69 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 70 return; 71 } 72 73 unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size)); 74 vunmap(cpu_addr); 75 } 76 77 #ifdef CONFIG_DMA_DIRECT_REMAP 78 static struct gen_pool *atomic_pool __ro_after_init; 79 80 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 81 static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; 82 83 static int __init early_coherent_pool(char *p) 84 { 85 atomic_pool_size = memparse(p, &p); 86 return 0; 87 } 88 early_param("coherent_pool", early_coherent_pool); 89 90 static gfp_t dma_atomic_pool_gfp(void) 91 { 92 if (IS_ENABLED(CONFIG_ZONE_DMA)) 93 return GFP_DMA; 94 if (IS_ENABLED(CONFIG_ZONE_DMA32)) 95 return GFP_DMA32; 96 return GFP_KERNEL; 97 } 98 99 static int __init dma_atomic_pool_init(void) 100 { 101 unsigned int pool_size_order = get_order(atomic_pool_size); 102 unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT; 103 struct page *page; 104 void *addr; 105 int ret; 106 107 if (dev_get_cma_area(NULL)) 108 page = dma_alloc_from_contiguous(NULL, nr_pages, 109 pool_size_order, false); 110 else 111 page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order); 112 if (!page) 113 goto out; 114 115 arch_dma_prep_coherent(page, atomic_pool_size); 116 117 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 118 if (!atomic_pool) 119 goto free_page; 120 121 addr = dma_common_contiguous_remap(page, atomic_pool_size, 122 pgprot_dmacoherent(PAGE_KERNEL), 123 __builtin_return_address(0)); 124 if (!addr) 125 goto destroy_genpool; 126 127 ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr, 128 page_to_phys(page), atomic_pool_size, -1); 129 if (ret) 130 goto remove_mapping; 131 gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL); 132 133 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", 134 atomic_pool_size / 1024); 135 return 0; 136 137 remove_mapping: 138 dma_common_free_remap(addr, atomic_pool_size); 139 destroy_genpool: 140 gen_pool_destroy(atomic_pool); 141 atomic_pool = NULL; 142 free_page: 143 if (!dma_release_from_contiguous(NULL, page, nr_pages)) 144 __free_pages(page, pool_size_order); 145 out: 146 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", 147 atomic_pool_size / 1024); 148 return -ENOMEM; 149 } 150 postcore_initcall(dma_atomic_pool_init); 151 152 bool dma_in_atomic_pool(void *start, size_t size) 153 { 154 if (unlikely(!atomic_pool)) 155 return false; 156 157 return gen_pool_has_addr(atomic_pool, (unsigned long)start, size); 158 } 159 160 void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags) 161 { 162 unsigned long val; 163 void *ptr = NULL; 164 165 if (!atomic_pool) { 166 WARN(1, "coherent pool not initialised!\n"); 167 return NULL; 168 } 169 170 val = gen_pool_alloc(atomic_pool, size); 171 if (val) { 172 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 173 174 *ret_page = pfn_to_page(__phys_to_pfn(phys)); 175 ptr = (void *)val; 176 memset(ptr, 0, size); 177 } 178 179 return ptr; 180 } 181 182 bool dma_free_from_pool(void *start, size_t size) 183 { 184 if (!dma_in_atomic_pool(start, size)) 185 return false; 186 gen_pool_free(atomic_pool, (unsigned long)start, size); 187 return true; 188 } 189 #endif /* CONFIG_DMA_DIRECT_REMAP */ 190