1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * DMABUF System heap exporter
4  *
5  * Copyright (C) 2011 Google, Inc.
6  * Copyright (C) 2019, 2020 Linaro Ltd.
7  *
8  * Portions based off of Andrew Davis' SRAM heap:
9  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
10  *	Andrew F. Davis <afd@ti.com>
11  */
12 
13 #include <linux/dma-buf.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dma-heap.h>
16 #include <linux/err.h>
17 #include <linux/highmem.h>
18 #include <linux/mm.h>
19 #include <linux/module.h>
20 #include <linux/scatterlist.h>
21 #include <linux/slab.h>
22 #include <linux/vmalloc.h>
23 
24 static struct dma_heap *sys_heap;
25 
26 struct system_heap_buffer {
27 	struct dma_heap *heap;
28 	struct list_head attachments;
29 	struct mutex lock;
30 	unsigned long len;
31 	struct sg_table sg_table;
32 	int vmap_cnt;
33 	void *vaddr;
34 };
35 
36 struct dma_heap_attachment {
37 	struct device *dev;
38 	struct sg_table *table;
39 	struct list_head list;
40 	bool mapped;
41 };
42 
43 #define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO)
44 #define HIGH_ORDER_GFP  (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
45 				| __GFP_NORETRY) & ~__GFP_RECLAIM) \
46 				| __GFP_COMP)
47 static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP};
48 /*
49  * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
50  * to match with the sizes often found in IOMMUs. Using order 4 pages instead
51  * of order 0 pages can significantly improve the performance of many IOMMUs
52  * by reducing TLB pressure and time spent updating page tables.
53  */
54 static const unsigned int orders[] = {8, 4, 0};
55 #define NUM_ORDERS ARRAY_SIZE(orders)
56 
dup_sg_table(struct sg_table * table)57 static struct sg_table *dup_sg_table(struct sg_table *table)
58 {
59 	struct sg_table *new_table;
60 	int ret, i;
61 	struct scatterlist *sg, *new_sg;
62 
63 	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
64 	if (!new_table)
65 		return ERR_PTR(-ENOMEM);
66 
67 	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
68 	if (ret) {
69 		kfree(new_table);
70 		return ERR_PTR(-ENOMEM);
71 	}
72 
73 	new_sg = new_table->sgl;
74 	for_each_sgtable_sg(table, sg, i) {
75 		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
76 		new_sg = sg_next(new_sg);
77 	}
78 
79 	return new_table;
80 }
81 
system_heap_attach(struct dma_buf * dmabuf,struct dma_buf_attachment * attachment)82 static int system_heap_attach(struct dma_buf *dmabuf,
83 			      struct dma_buf_attachment *attachment)
84 {
85 	struct system_heap_buffer *buffer = dmabuf->priv;
86 	struct dma_heap_attachment *a;
87 	struct sg_table *table;
88 
89 	a = kzalloc(sizeof(*a), GFP_KERNEL);
90 	if (!a)
91 		return -ENOMEM;
92 
93 	table = dup_sg_table(&buffer->sg_table);
94 	if (IS_ERR(table)) {
95 		kfree(a);
96 		return -ENOMEM;
97 	}
98 
99 	a->table = table;
100 	a->dev = attachment->dev;
101 	INIT_LIST_HEAD(&a->list);
102 	a->mapped = false;
103 
104 	attachment->priv = a;
105 
106 	mutex_lock(&buffer->lock);
107 	list_add(&a->list, &buffer->attachments);
108 	mutex_unlock(&buffer->lock);
109 
110 	return 0;
111 }
112 
system_heap_detach(struct dma_buf * dmabuf,struct dma_buf_attachment * attachment)113 static void system_heap_detach(struct dma_buf *dmabuf,
114 			       struct dma_buf_attachment *attachment)
115 {
116 	struct system_heap_buffer *buffer = dmabuf->priv;
117 	struct dma_heap_attachment *a = attachment->priv;
118 
119 	mutex_lock(&buffer->lock);
120 	list_del(&a->list);
121 	mutex_unlock(&buffer->lock);
122 
123 	sg_free_table(a->table);
124 	kfree(a->table);
125 	kfree(a);
126 }
127 
system_heap_map_dma_buf(struct dma_buf_attachment * attachment,enum dma_data_direction direction)128 static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
129 						enum dma_data_direction direction)
130 {
131 	struct dma_heap_attachment *a = attachment->priv;
132 	struct sg_table *table = a->table;
133 	int ret;
134 
135 	ret = dma_map_sgtable(attachment->dev, table, direction, 0);
136 	if (ret)
137 		return ERR_PTR(ret);
138 
139 	a->mapped = true;
140 	return table;
141 }
142 
system_heap_unmap_dma_buf(struct dma_buf_attachment * attachment,struct sg_table * table,enum dma_data_direction direction)143 static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
144 				      struct sg_table *table,
145 				      enum dma_data_direction direction)
146 {
147 	struct dma_heap_attachment *a = attachment->priv;
148 
149 	a->mapped = false;
150 	dma_unmap_sgtable(attachment->dev, table, direction, 0);
151 }
152 
system_heap_dma_buf_begin_cpu_access(struct dma_buf * dmabuf,enum dma_data_direction direction)153 static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
154 						enum dma_data_direction direction)
155 {
156 	struct system_heap_buffer *buffer = dmabuf->priv;
157 	struct dma_heap_attachment *a;
158 
159 	mutex_lock(&buffer->lock);
160 
161 	if (buffer->vmap_cnt)
162 		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
163 
164 	list_for_each_entry(a, &buffer->attachments, list) {
165 		if (!a->mapped)
166 			continue;
167 		dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
168 	}
169 	mutex_unlock(&buffer->lock);
170 
171 	return 0;
172 }
173 
system_heap_dma_buf_end_cpu_access(struct dma_buf * dmabuf,enum dma_data_direction direction)174 static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
175 					      enum dma_data_direction direction)
176 {
177 	struct system_heap_buffer *buffer = dmabuf->priv;
178 	struct dma_heap_attachment *a;
179 
180 	mutex_lock(&buffer->lock);
181 
182 	if (buffer->vmap_cnt)
183 		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
184 
185 	list_for_each_entry(a, &buffer->attachments, list) {
186 		if (!a->mapped)
187 			continue;
188 		dma_sync_sgtable_for_device(a->dev, a->table, direction);
189 	}
190 	mutex_unlock(&buffer->lock);
191 
192 	return 0;
193 }
194 
system_heap_mmap(struct dma_buf * dmabuf,struct vm_area_struct * vma)195 static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
196 {
197 	struct system_heap_buffer *buffer = dmabuf->priv;
198 	struct sg_table *table = &buffer->sg_table;
199 	unsigned long addr = vma->vm_start;
200 	struct sg_page_iter piter;
201 	int ret;
202 
203 	for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
204 		struct page *page = sg_page_iter_page(&piter);
205 
206 		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
207 				      vma->vm_page_prot);
208 		if (ret)
209 			return ret;
210 		addr += PAGE_SIZE;
211 		if (addr >= vma->vm_end)
212 			return 0;
213 	}
214 	return 0;
215 }
216 
system_heap_do_vmap(struct system_heap_buffer * buffer)217 static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
218 {
219 	struct sg_table *table = &buffer->sg_table;
220 	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
221 	struct page **pages = vmalloc(sizeof(struct page *) * npages);
222 	struct page **tmp = pages;
223 	struct sg_page_iter piter;
224 	void *vaddr;
225 
226 	if (!pages)
227 		return ERR_PTR(-ENOMEM);
228 
229 	for_each_sgtable_page(table, &piter, 0) {
230 		WARN_ON(tmp - pages >= npages);
231 		*tmp++ = sg_page_iter_page(&piter);
232 	}
233 
234 	vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
235 	vfree(pages);
236 
237 	if (!vaddr)
238 		return ERR_PTR(-ENOMEM);
239 
240 	return vaddr;
241 }
242 
system_heap_vmap(struct dma_buf * dmabuf,struct iosys_map * map)243 static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
244 {
245 	struct system_heap_buffer *buffer = dmabuf->priv;
246 	void *vaddr;
247 	int ret = 0;
248 
249 	mutex_lock(&buffer->lock);
250 	if (buffer->vmap_cnt) {
251 		buffer->vmap_cnt++;
252 		iosys_map_set_vaddr(map, buffer->vaddr);
253 		goto out;
254 	}
255 
256 	vaddr = system_heap_do_vmap(buffer);
257 	if (IS_ERR(vaddr)) {
258 		ret = PTR_ERR(vaddr);
259 		goto out;
260 	}
261 
262 	buffer->vaddr = vaddr;
263 	buffer->vmap_cnt++;
264 	iosys_map_set_vaddr(map, buffer->vaddr);
265 out:
266 	mutex_unlock(&buffer->lock);
267 
268 	return ret;
269 }
270 
system_heap_vunmap(struct dma_buf * dmabuf,struct iosys_map * map)271 static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
272 {
273 	struct system_heap_buffer *buffer = dmabuf->priv;
274 
275 	mutex_lock(&buffer->lock);
276 	if (!--buffer->vmap_cnt) {
277 		vunmap(buffer->vaddr);
278 		buffer->vaddr = NULL;
279 	}
280 	mutex_unlock(&buffer->lock);
281 	iosys_map_clear(map);
282 }
283 
system_heap_dma_buf_release(struct dma_buf * dmabuf)284 static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
285 {
286 	struct system_heap_buffer *buffer = dmabuf->priv;
287 	struct sg_table *table;
288 	struct scatterlist *sg;
289 	int i;
290 
291 	table = &buffer->sg_table;
292 	for_each_sgtable_sg(table, sg, i) {
293 		struct page *page = sg_page(sg);
294 
295 		__free_pages(page, compound_order(page));
296 	}
297 	sg_free_table(table);
298 	kfree(buffer);
299 }
300 
301 static const struct dma_buf_ops system_heap_buf_ops = {
302 	.attach = system_heap_attach,
303 	.detach = system_heap_detach,
304 	.map_dma_buf = system_heap_map_dma_buf,
305 	.unmap_dma_buf = system_heap_unmap_dma_buf,
306 	.begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
307 	.end_cpu_access = system_heap_dma_buf_end_cpu_access,
308 	.mmap = system_heap_mmap,
309 	.vmap = system_heap_vmap,
310 	.vunmap = system_heap_vunmap,
311 	.release = system_heap_dma_buf_release,
312 };
313 
alloc_largest_available(unsigned long size,unsigned int max_order)314 static struct page *alloc_largest_available(unsigned long size,
315 					    unsigned int max_order)
316 {
317 	struct page *page;
318 	int i;
319 
320 	for (i = 0; i < NUM_ORDERS; i++) {
321 		if (size <  (PAGE_SIZE << orders[i]))
322 			continue;
323 		if (max_order < orders[i])
324 			continue;
325 
326 		page = alloc_pages(order_flags[i], orders[i]);
327 		if (!page)
328 			continue;
329 		return page;
330 	}
331 	return NULL;
332 }
333 
system_heap_allocate(struct dma_heap * heap,unsigned long len,unsigned long fd_flags,unsigned long heap_flags)334 static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
335 					    unsigned long len,
336 					    unsigned long fd_flags,
337 					    unsigned long heap_flags)
338 {
339 	struct system_heap_buffer *buffer;
340 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
341 	unsigned long size_remaining = len;
342 	unsigned int max_order = orders[0];
343 	struct dma_buf *dmabuf;
344 	struct sg_table *table;
345 	struct scatterlist *sg;
346 	struct list_head pages;
347 	struct page *page, *tmp_page;
348 	int i, ret = -ENOMEM;
349 
350 	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
351 	if (!buffer)
352 		return ERR_PTR(-ENOMEM);
353 
354 	INIT_LIST_HEAD(&buffer->attachments);
355 	mutex_init(&buffer->lock);
356 	buffer->heap = heap;
357 	buffer->len = len;
358 
359 	INIT_LIST_HEAD(&pages);
360 	i = 0;
361 	while (size_remaining > 0) {
362 		/*
363 		 * Avoid trying to allocate memory if the process
364 		 * has been killed by SIGKILL
365 		 */
366 		if (fatal_signal_pending(current)) {
367 			ret = -EINTR;
368 			goto free_buffer;
369 		}
370 
371 		page = alloc_largest_available(size_remaining, max_order);
372 		if (!page)
373 			goto free_buffer;
374 
375 		list_add_tail(&page->lru, &pages);
376 		size_remaining -= page_size(page);
377 		max_order = compound_order(page);
378 		i++;
379 	}
380 
381 	table = &buffer->sg_table;
382 	if (sg_alloc_table(table, i, GFP_KERNEL))
383 		goto free_buffer;
384 
385 	sg = table->sgl;
386 	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
387 		sg_set_page(sg, page, page_size(page), 0);
388 		sg = sg_next(sg);
389 		list_del(&page->lru);
390 	}
391 
392 	/* create the dmabuf */
393 	exp_info.exp_name = dma_heap_get_name(heap);
394 	exp_info.ops = &system_heap_buf_ops;
395 	exp_info.size = buffer->len;
396 	exp_info.flags = fd_flags;
397 	exp_info.priv = buffer;
398 	dmabuf = dma_buf_export(&exp_info);
399 	if (IS_ERR(dmabuf)) {
400 		ret = PTR_ERR(dmabuf);
401 		goto free_pages;
402 	}
403 	return dmabuf;
404 
405 free_pages:
406 	for_each_sgtable_sg(table, sg, i) {
407 		struct page *p = sg_page(sg);
408 
409 		__free_pages(p, compound_order(p));
410 	}
411 	sg_free_table(table);
412 free_buffer:
413 	list_for_each_entry_safe(page, tmp_page, &pages, lru)
414 		__free_pages(page, compound_order(page));
415 	kfree(buffer);
416 
417 	return ERR_PTR(ret);
418 }
419 
420 static const struct dma_heap_ops system_heap_ops = {
421 	.allocate = system_heap_allocate,
422 };
423 
system_heap_create(void)424 static int system_heap_create(void)
425 {
426 	struct dma_heap_export_info exp_info;
427 
428 	exp_info.name = "system";
429 	exp_info.ops = &system_heap_ops;
430 	exp_info.priv = NULL;
431 
432 	sys_heap = dma_heap_add(&exp_info);
433 	if (IS_ERR(sys_heap))
434 		return PTR_ERR(sys_heap);
435 
436 	return 0;
437 }
438 module_init(system_heap_create);
439