1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * DMABUF System heap exporter
4  *
5  * Copyright (C) 2011 Google, Inc.
6  * Copyright (C) 2019, 2020 Linaro Ltd.
7  *
8  * Portions based off of Andrew Davis' SRAM heap:
9  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
10  *	Andrew F. Davis <afd@ti.com>
11  */
12 
13 #include <linux/dma-buf.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dma-heap.h>
16 #include <linux/dma-resv.h>
17 #include <linux/err.h>
18 #include <linux/highmem.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 #include <linux/scatterlist.h>
22 #include <linux/slab.h>
23 #include <linux/vmalloc.h>
24 
25 static struct dma_heap *sys_heap;
26 
27 struct system_heap_buffer {
28 	struct dma_heap *heap;
29 	struct list_head attachments;
30 	struct mutex lock;
31 	unsigned long len;
32 	struct sg_table sg_table;
33 	int vmap_cnt;
34 	void *vaddr;
35 };
36 
37 struct dma_heap_attachment {
38 	struct device *dev;
39 	struct sg_table *table;
40 	struct list_head list;
41 	bool mapped;
42 };
43 
44 #define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
45 #define MID_ORDER_GFP (LOW_ORDER_GFP | __GFP_NOWARN)
46 #define HIGH_ORDER_GFP  (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
47 				| __GFP_NORETRY) & ~__GFP_RECLAIM) \
48 				| __GFP_COMP)
49 static gfp_t order_flags[] = {HIGH_ORDER_GFP, MID_ORDER_GFP, LOW_ORDER_GFP};
50 /*
51  * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
52  * to match with the sizes often found in IOMMUs. Using order 4 pages instead
53  * of order 0 pages can significantly improve the performance of many IOMMUs
54  * by reducing TLB pressure and time spent updating page tables.
55  */
56 static const unsigned int orders[] = {8, 4, 0};
57 #define NUM_ORDERS ARRAY_SIZE(orders)
58 
59 static struct sg_table *dup_sg_table(struct sg_table *table)
60 {
61 	struct sg_table *new_table;
62 	int ret, i;
63 	struct scatterlist *sg, *new_sg;
64 
65 	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
66 	if (!new_table)
67 		return ERR_PTR(-ENOMEM);
68 
69 	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
70 	if (ret) {
71 		kfree(new_table);
72 		return ERR_PTR(-ENOMEM);
73 	}
74 
75 	new_sg = new_table->sgl;
76 	for_each_sgtable_sg(table, sg, i) {
77 		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
78 		new_sg = sg_next(new_sg);
79 	}
80 
81 	return new_table;
82 }
83 
84 static int system_heap_attach(struct dma_buf *dmabuf,
85 			      struct dma_buf_attachment *attachment)
86 {
87 	struct system_heap_buffer *buffer = dmabuf->priv;
88 	struct dma_heap_attachment *a;
89 	struct sg_table *table;
90 
91 	a = kzalloc(sizeof(*a), GFP_KERNEL);
92 	if (!a)
93 		return -ENOMEM;
94 
95 	table = dup_sg_table(&buffer->sg_table);
96 	if (IS_ERR(table)) {
97 		kfree(a);
98 		return -ENOMEM;
99 	}
100 
101 	a->table = table;
102 	a->dev = attachment->dev;
103 	INIT_LIST_HEAD(&a->list);
104 	a->mapped = false;
105 
106 	attachment->priv = a;
107 
108 	mutex_lock(&buffer->lock);
109 	list_add(&a->list, &buffer->attachments);
110 	mutex_unlock(&buffer->lock);
111 
112 	return 0;
113 }
114 
115 static void system_heap_detach(struct dma_buf *dmabuf,
116 			       struct dma_buf_attachment *attachment)
117 {
118 	struct system_heap_buffer *buffer = dmabuf->priv;
119 	struct dma_heap_attachment *a = attachment->priv;
120 
121 	mutex_lock(&buffer->lock);
122 	list_del(&a->list);
123 	mutex_unlock(&buffer->lock);
124 
125 	sg_free_table(a->table);
126 	kfree(a->table);
127 	kfree(a);
128 }
129 
130 static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
131 						enum dma_data_direction direction)
132 {
133 	struct dma_heap_attachment *a = attachment->priv;
134 	struct sg_table *table = a->table;
135 	int ret;
136 
137 	ret = dma_map_sgtable(attachment->dev, table, direction, 0);
138 	if (ret)
139 		return ERR_PTR(ret);
140 
141 	a->mapped = true;
142 	return table;
143 }
144 
145 static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
146 				      struct sg_table *table,
147 				      enum dma_data_direction direction)
148 {
149 	struct dma_heap_attachment *a = attachment->priv;
150 
151 	a->mapped = false;
152 	dma_unmap_sgtable(attachment->dev, table, direction, 0);
153 }
154 
155 static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
156 						enum dma_data_direction direction)
157 {
158 	struct system_heap_buffer *buffer = dmabuf->priv;
159 	struct dma_heap_attachment *a;
160 
161 	mutex_lock(&buffer->lock);
162 
163 	if (buffer->vmap_cnt)
164 		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
165 
166 	list_for_each_entry(a, &buffer->attachments, list) {
167 		if (!a->mapped)
168 			continue;
169 		dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
170 	}
171 	mutex_unlock(&buffer->lock);
172 
173 	return 0;
174 }
175 
176 static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
177 					      enum dma_data_direction direction)
178 {
179 	struct system_heap_buffer *buffer = dmabuf->priv;
180 	struct dma_heap_attachment *a;
181 
182 	mutex_lock(&buffer->lock);
183 
184 	if (buffer->vmap_cnt)
185 		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
186 
187 	list_for_each_entry(a, &buffer->attachments, list) {
188 		if (!a->mapped)
189 			continue;
190 		dma_sync_sgtable_for_device(a->dev, a->table, direction);
191 	}
192 	mutex_unlock(&buffer->lock);
193 
194 	return 0;
195 }
196 
197 static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
198 {
199 	struct system_heap_buffer *buffer = dmabuf->priv;
200 	struct sg_table *table = &buffer->sg_table;
201 	unsigned long addr = vma->vm_start;
202 	struct sg_page_iter piter;
203 	int ret;
204 
205 	dma_resv_assert_held(dmabuf->resv);
206 
207 	for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
208 		struct page *page = sg_page_iter_page(&piter);
209 
210 		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
211 				      vma->vm_page_prot);
212 		if (ret)
213 			return ret;
214 		addr += PAGE_SIZE;
215 		if (addr >= vma->vm_end)
216 			return 0;
217 	}
218 	return 0;
219 }
220 
221 static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
222 {
223 	struct sg_table *table = &buffer->sg_table;
224 	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
225 	struct page **pages = vmalloc(sizeof(struct page *) * npages);
226 	struct page **tmp = pages;
227 	struct sg_page_iter piter;
228 	void *vaddr;
229 
230 	if (!pages)
231 		return ERR_PTR(-ENOMEM);
232 
233 	for_each_sgtable_page(table, &piter, 0) {
234 		WARN_ON(tmp - pages >= npages);
235 		*tmp++ = sg_page_iter_page(&piter);
236 	}
237 
238 	vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
239 	vfree(pages);
240 
241 	if (!vaddr)
242 		return ERR_PTR(-ENOMEM);
243 
244 	return vaddr;
245 }
246 
247 static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
248 {
249 	struct system_heap_buffer *buffer = dmabuf->priv;
250 	void *vaddr;
251 	int ret = 0;
252 
253 	mutex_lock(&buffer->lock);
254 	if (buffer->vmap_cnt) {
255 		buffer->vmap_cnt++;
256 		iosys_map_set_vaddr(map, buffer->vaddr);
257 		goto out;
258 	}
259 
260 	vaddr = system_heap_do_vmap(buffer);
261 	if (IS_ERR(vaddr)) {
262 		ret = PTR_ERR(vaddr);
263 		goto out;
264 	}
265 
266 	buffer->vaddr = vaddr;
267 	buffer->vmap_cnt++;
268 	iosys_map_set_vaddr(map, buffer->vaddr);
269 out:
270 	mutex_unlock(&buffer->lock);
271 
272 	return ret;
273 }
274 
275 static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
276 {
277 	struct system_heap_buffer *buffer = dmabuf->priv;
278 
279 	mutex_lock(&buffer->lock);
280 	if (!--buffer->vmap_cnt) {
281 		vunmap(buffer->vaddr);
282 		buffer->vaddr = NULL;
283 	}
284 	mutex_unlock(&buffer->lock);
285 	iosys_map_clear(map);
286 }
287 
288 static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
289 {
290 	struct system_heap_buffer *buffer = dmabuf->priv;
291 	struct sg_table *table;
292 	struct scatterlist *sg;
293 	int i;
294 
295 	table = &buffer->sg_table;
296 	for_each_sgtable_sg(table, sg, i) {
297 		struct page *page = sg_page(sg);
298 
299 		__free_pages(page, compound_order(page));
300 	}
301 	sg_free_table(table);
302 	kfree(buffer);
303 }
304 
305 static const struct dma_buf_ops system_heap_buf_ops = {
306 	.attach = system_heap_attach,
307 	.detach = system_heap_detach,
308 	.map_dma_buf = system_heap_map_dma_buf,
309 	.unmap_dma_buf = system_heap_unmap_dma_buf,
310 	.begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
311 	.end_cpu_access = system_heap_dma_buf_end_cpu_access,
312 	.mmap = system_heap_mmap,
313 	.vmap = system_heap_vmap,
314 	.vunmap = system_heap_vunmap,
315 	.release = system_heap_dma_buf_release,
316 };
317 
318 static struct page *alloc_largest_available(unsigned long size,
319 					    unsigned int max_order)
320 {
321 	struct page *page;
322 	int i;
323 
324 	for (i = 0; i < NUM_ORDERS; i++) {
325 		if (size <  (PAGE_SIZE << orders[i]))
326 			continue;
327 		if (max_order < orders[i])
328 			continue;
329 
330 		page = alloc_pages(order_flags[i], orders[i]);
331 		if (!page)
332 			continue;
333 		return page;
334 	}
335 	return NULL;
336 }
337 
338 static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
339 					    unsigned long len,
340 					    unsigned long fd_flags,
341 					    unsigned long heap_flags)
342 {
343 	struct system_heap_buffer *buffer;
344 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
345 	unsigned long size_remaining = len;
346 	unsigned int max_order = orders[0];
347 	struct dma_buf *dmabuf;
348 	struct sg_table *table;
349 	struct scatterlist *sg;
350 	struct list_head pages;
351 	struct page *page, *tmp_page;
352 	int i, ret = -ENOMEM;
353 
354 	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
355 	if (!buffer)
356 		return ERR_PTR(-ENOMEM);
357 
358 	INIT_LIST_HEAD(&buffer->attachments);
359 	mutex_init(&buffer->lock);
360 	buffer->heap = heap;
361 	buffer->len = len;
362 
363 	INIT_LIST_HEAD(&pages);
364 	i = 0;
365 	while (size_remaining > 0) {
366 		/*
367 		 * Avoid trying to allocate memory if the process
368 		 * has been killed by SIGKILL
369 		 */
370 		if (fatal_signal_pending(current)) {
371 			ret = -EINTR;
372 			goto free_buffer;
373 		}
374 
375 		page = alloc_largest_available(size_remaining, max_order);
376 		if (!page)
377 			goto free_buffer;
378 
379 		list_add_tail(&page->lru, &pages);
380 		size_remaining -= page_size(page);
381 		max_order = compound_order(page);
382 		i++;
383 	}
384 
385 	table = &buffer->sg_table;
386 	if (sg_alloc_table(table, i, GFP_KERNEL))
387 		goto free_buffer;
388 
389 	sg = table->sgl;
390 	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
391 		sg_set_page(sg, page, page_size(page), 0);
392 		sg = sg_next(sg);
393 		list_del(&page->lru);
394 	}
395 
396 	/* create the dmabuf */
397 	exp_info.exp_name = dma_heap_get_name(heap);
398 	exp_info.ops = &system_heap_buf_ops;
399 	exp_info.size = buffer->len;
400 	exp_info.flags = fd_flags;
401 	exp_info.priv = buffer;
402 	dmabuf = dma_buf_export(&exp_info);
403 	if (IS_ERR(dmabuf)) {
404 		ret = PTR_ERR(dmabuf);
405 		goto free_pages;
406 	}
407 	return dmabuf;
408 
409 free_pages:
410 	for_each_sgtable_sg(table, sg, i) {
411 		struct page *p = sg_page(sg);
412 
413 		__free_pages(p, compound_order(p));
414 	}
415 	sg_free_table(table);
416 free_buffer:
417 	list_for_each_entry_safe(page, tmp_page, &pages, lru)
418 		__free_pages(page, compound_order(page));
419 	kfree(buffer);
420 
421 	return ERR_PTR(ret);
422 }
423 
424 static const struct dma_heap_ops system_heap_ops = {
425 	.allocate = system_heap_allocate,
426 };
427 
428 static int system_heap_create(void)
429 {
430 	struct dma_heap_export_info exp_info;
431 
432 	exp_info.name = "system";
433 	exp_info.ops = &system_heap_ops;
434 	exp_info.priv = NULL;
435 
436 	sys_heap = dma_heap_add(&exp_info);
437 	if (IS_ERR(sys_heap))
438 		return PTR_ERR(sys_heap);
439 
440 	return 0;
441 }
442 module_init(system_heap_create);
443 MODULE_LICENSE("GPL v2");
444