1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * DMABUF System heap exporter
4  *
5  * Copyright (C) 2011 Google, Inc.
6  * Copyright (C) 2019, 2020 Linaro Ltd.
7  *
8  * Portions based off of Andrew Davis' SRAM heap:
9  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
10  *	Andrew F. Davis <afd@ti.com>
11  */
12 
13 #include <linux/dma-buf.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dma-heap.h>
16 #include <linux/dma-resv.h>
17 #include <linux/err.h>
18 #include <linux/highmem.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 #include <linux/scatterlist.h>
22 #include <linux/slab.h>
23 #include <linux/vmalloc.h>
24 
25 static struct dma_heap *sys_heap;
26 
27 struct system_heap_buffer {
28 	struct dma_heap *heap;
29 	struct list_head attachments;
30 	struct mutex lock;
31 	unsigned long len;
32 	struct sg_table sg_table;
33 	int vmap_cnt;
34 	void *vaddr;
35 };
36 
37 struct dma_heap_attachment {
38 	struct device *dev;
39 	struct sg_table *table;
40 	struct list_head list;
41 	bool mapped;
42 };
43 
44 #define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO)
45 #define HIGH_ORDER_GFP  (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
46 				| __GFP_NORETRY) & ~__GFP_RECLAIM) \
47 				| __GFP_COMP)
48 static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP};
49 /*
50  * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
51  * to match with the sizes often found in IOMMUs. Using order 4 pages instead
52  * of order 0 pages can significantly improve the performance of many IOMMUs
53  * by reducing TLB pressure and time spent updating page tables.
54  */
55 static const unsigned int orders[] = {8, 4, 0};
56 #define NUM_ORDERS ARRAY_SIZE(orders)
57 
58 static struct sg_table *dup_sg_table(struct sg_table *table)
59 {
60 	struct sg_table *new_table;
61 	int ret, i;
62 	struct scatterlist *sg, *new_sg;
63 
64 	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
65 	if (!new_table)
66 		return ERR_PTR(-ENOMEM);
67 
68 	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
69 	if (ret) {
70 		kfree(new_table);
71 		return ERR_PTR(-ENOMEM);
72 	}
73 
74 	new_sg = new_table->sgl;
75 	for_each_sgtable_sg(table, sg, i) {
76 		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
77 		new_sg = sg_next(new_sg);
78 	}
79 
80 	return new_table;
81 }
82 
83 static int system_heap_attach(struct dma_buf *dmabuf,
84 			      struct dma_buf_attachment *attachment)
85 {
86 	struct system_heap_buffer *buffer = dmabuf->priv;
87 	struct dma_heap_attachment *a;
88 	struct sg_table *table;
89 
90 	a = kzalloc(sizeof(*a), GFP_KERNEL);
91 	if (!a)
92 		return -ENOMEM;
93 
94 	table = dup_sg_table(&buffer->sg_table);
95 	if (IS_ERR(table)) {
96 		kfree(a);
97 		return -ENOMEM;
98 	}
99 
100 	a->table = table;
101 	a->dev = attachment->dev;
102 	INIT_LIST_HEAD(&a->list);
103 	a->mapped = false;
104 
105 	attachment->priv = a;
106 
107 	mutex_lock(&buffer->lock);
108 	list_add(&a->list, &buffer->attachments);
109 	mutex_unlock(&buffer->lock);
110 
111 	return 0;
112 }
113 
114 static void system_heap_detach(struct dma_buf *dmabuf,
115 			       struct dma_buf_attachment *attachment)
116 {
117 	struct system_heap_buffer *buffer = dmabuf->priv;
118 	struct dma_heap_attachment *a = attachment->priv;
119 
120 	mutex_lock(&buffer->lock);
121 	list_del(&a->list);
122 	mutex_unlock(&buffer->lock);
123 
124 	sg_free_table(a->table);
125 	kfree(a->table);
126 	kfree(a);
127 }
128 
129 static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
130 						enum dma_data_direction direction)
131 {
132 	struct dma_heap_attachment *a = attachment->priv;
133 	struct sg_table *table = a->table;
134 	int ret;
135 
136 	ret = dma_map_sgtable(attachment->dev, table, direction, 0);
137 	if (ret)
138 		return ERR_PTR(ret);
139 
140 	a->mapped = true;
141 	return table;
142 }
143 
144 static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
145 				      struct sg_table *table,
146 				      enum dma_data_direction direction)
147 {
148 	struct dma_heap_attachment *a = attachment->priv;
149 
150 	a->mapped = false;
151 	dma_unmap_sgtable(attachment->dev, table, direction, 0);
152 }
153 
154 static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
155 						enum dma_data_direction direction)
156 {
157 	struct system_heap_buffer *buffer = dmabuf->priv;
158 	struct dma_heap_attachment *a;
159 
160 	mutex_lock(&buffer->lock);
161 
162 	if (buffer->vmap_cnt)
163 		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
164 
165 	list_for_each_entry(a, &buffer->attachments, list) {
166 		if (!a->mapped)
167 			continue;
168 		dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
169 	}
170 	mutex_unlock(&buffer->lock);
171 
172 	return 0;
173 }
174 
175 static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
176 					      enum dma_data_direction direction)
177 {
178 	struct system_heap_buffer *buffer = dmabuf->priv;
179 	struct dma_heap_attachment *a;
180 
181 	mutex_lock(&buffer->lock);
182 
183 	if (buffer->vmap_cnt)
184 		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
185 
186 	list_for_each_entry(a, &buffer->attachments, list) {
187 		if (!a->mapped)
188 			continue;
189 		dma_sync_sgtable_for_device(a->dev, a->table, direction);
190 	}
191 	mutex_unlock(&buffer->lock);
192 
193 	return 0;
194 }
195 
196 static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
197 {
198 	struct system_heap_buffer *buffer = dmabuf->priv;
199 	struct sg_table *table = &buffer->sg_table;
200 	unsigned long addr = vma->vm_start;
201 	struct sg_page_iter piter;
202 	int ret;
203 
204 	dma_resv_assert_held(dmabuf->resv);
205 
206 	for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
207 		struct page *page = sg_page_iter_page(&piter);
208 
209 		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
210 				      vma->vm_page_prot);
211 		if (ret)
212 			return ret;
213 		addr += PAGE_SIZE;
214 		if (addr >= vma->vm_end)
215 			return 0;
216 	}
217 	return 0;
218 }
219 
220 static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
221 {
222 	struct sg_table *table = &buffer->sg_table;
223 	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
224 	struct page **pages = vmalloc(sizeof(struct page *) * npages);
225 	struct page **tmp = pages;
226 	struct sg_page_iter piter;
227 	void *vaddr;
228 
229 	if (!pages)
230 		return ERR_PTR(-ENOMEM);
231 
232 	for_each_sgtable_page(table, &piter, 0) {
233 		WARN_ON(tmp - pages >= npages);
234 		*tmp++ = sg_page_iter_page(&piter);
235 	}
236 
237 	vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
238 	vfree(pages);
239 
240 	if (!vaddr)
241 		return ERR_PTR(-ENOMEM);
242 
243 	return vaddr;
244 }
245 
246 static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
247 {
248 	struct system_heap_buffer *buffer = dmabuf->priv;
249 	void *vaddr;
250 	int ret = 0;
251 
252 	mutex_lock(&buffer->lock);
253 	if (buffer->vmap_cnt) {
254 		buffer->vmap_cnt++;
255 		iosys_map_set_vaddr(map, buffer->vaddr);
256 		goto out;
257 	}
258 
259 	vaddr = system_heap_do_vmap(buffer);
260 	if (IS_ERR(vaddr)) {
261 		ret = PTR_ERR(vaddr);
262 		goto out;
263 	}
264 
265 	buffer->vaddr = vaddr;
266 	buffer->vmap_cnt++;
267 	iosys_map_set_vaddr(map, buffer->vaddr);
268 out:
269 	mutex_unlock(&buffer->lock);
270 
271 	return ret;
272 }
273 
274 static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
275 {
276 	struct system_heap_buffer *buffer = dmabuf->priv;
277 
278 	mutex_lock(&buffer->lock);
279 	if (!--buffer->vmap_cnt) {
280 		vunmap(buffer->vaddr);
281 		buffer->vaddr = NULL;
282 	}
283 	mutex_unlock(&buffer->lock);
284 	iosys_map_clear(map);
285 }
286 
287 static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
288 {
289 	struct system_heap_buffer *buffer = dmabuf->priv;
290 	struct sg_table *table;
291 	struct scatterlist *sg;
292 	int i;
293 
294 	table = &buffer->sg_table;
295 	for_each_sgtable_sg(table, sg, i) {
296 		struct page *page = sg_page(sg);
297 
298 		__free_pages(page, compound_order(page));
299 	}
300 	sg_free_table(table);
301 	kfree(buffer);
302 }
303 
304 static const struct dma_buf_ops system_heap_buf_ops = {
305 	.attach = system_heap_attach,
306 	.detach = system_heap_detach,
307 	.map_dma_buf = system_heap_map_dma_buf,
308 	.unmap_dma_buf = system_heap_unmap_dma_buf,
309 	.begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
310 	.end_cpu_access = system_heap_dma_buf_end_cpu_access,
311 	.mmap = system_heap_mmap,
312 	.vmap = system_heap_vmap,
313 	.vunmap = system_heap_vunmap,
314 	.release = system_heap_dma_buf_release,
315 };
316 
317 static struct page *alloc_largest_available(unsigned long size,
318 					    unsigned int max_order)
319 {
320 	struct page *page;
321 	int i;
322 
323 	for (i = 0; i < NUM_ORDERS; i++) {
324 		if (size <  (PAGE_SIZE << orders[i]))
325 			continue;
326 		if (max_order < orders[i])
327 			continue;
328 
329 		page = alloc_pages(order_flags[i], orders[i]);
330 		if (!page)
331 			continue;
332 		return page;
333 	}
334 	return NULL;
335 }
336 
337 static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
338 					    unsigned long len,
339 					    unsigned long fd_flags,
340 					    unsigned long heap_flags)
341 {
342 	struct system_heap_buffer *buffer;
343 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
344 	unsigned long size_remaining = len;
345 	unsigned int max_order = orders[0];
346 	struct dma_buf *dmabuf;
347 	struct sg_table *table;
348 	struct scatterlist *sg;
349 	struct list_head pages;
350 	struct page *page, *tmp_page;
351 	int i, ret = -ENOMEM;
352 
353 	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
354 	if (!buffer)
355 		return ERR_PTR(-ENOMEM);
356 
357 	INIT_LIST_HEAD(&buffer->attachments);
358 	mutex_init(&buffer->lock);
359 	buffer->heap = heap;
360 	buffer->len = len;
361 
362 	INIT_LIST_HEAD(&pages);
363 	i = 0;
364 	while (size_remaining > 0) {
365 		/*
366 		 * Avoid trying to allocate memory if the process
367 		 * has been killed by SIGKILL
368 		 */
369 		if (fatal_signal_pending(current)) {
370 			ret = -EINTR;
371 			goto free_buffer;
372 		}
373 
374 		page = alloc_largest_available(size_remaining, max_order);
375 		if (!page)
376 			goto free_buffer;
377 
378 		list_add_tail(&page->lru, &pages);
379 		size_remaining -= page_size(page);
380 		max_order = compound_order(page);
381 		i++;
382 	}
383 
384 	table = &buffer->sg_table;
385 	if (sg_alloc_table(table, i, GFP_KERNEL))
386 		goto free_buffer;
387 
388 	sg = table->sgl;
389 	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
390 		sg_set_page(sg, page, page_size(page), 0);
391 		sg = sg_next(sg);
392 		list_del(&page->lru);
393 	}
394 
395 	/* create the dmabuf */
396 	exp_info.exp_name = dma_heap_get_name(heap);
397 	exp_info.ops = &system_heap_buf_ops;
398 	exp_info.size = buffer->len;
399 	exp_info.flags = fd_flags;
400 	exp_info.priv = buffer;
401 	dmabuf = dma_buf_export(&exp_info);
402 	if (IS_ERR(dmabuf)) {
403 		ret = PTR_ERR(dmabuf);
404 		goto free_pages;
405 	}
406 	return dmabuf;
407 
408 free_pages:
409 	for_each_sgtable_sg(table, sg, i) {
410 		struct page *p = sg_page(sg);
411 
412 		__free_pages(p, compound_order(p));
413 	}
414 	sg_free_table(table);
415 free_buffer:
416 	list_for_each_entry_safe(page, tmp_page, &pages, lru)
417 		__free_pages(page, compound_order(page));
418 	kfree(buffer);
419 
420 	return ERR_PTR(ret);
421 }
422 
423 static const struct dma_heap_ops system_heap_ops = {
424 	.allocate = system_heap_allocate,
425 };
426 
427 static int system_heap_create(void)
428 {
429 	struct dma_heap_export_info exp_info;
430 
431 	exp_info.name = "system";
432 	exp_info.ops = &system_heap_ops;
433 	exp_info.priv = NULL;
434 
435 	sys_heap = dma_heap_add(&exp_info);
436 	if (IS_ERR(sys_heap))
437 		return PTR_ERR(sys_heap);
438 
439 	return 0;
440 }
441 module_init(system_heap_create);
442