1 /*
2  * videobuf2-dma-contig.c - DMA contig memory allocator for videobuf2
3  *
4  * Copyright (C) 2010 Samsung Electronics
5  *
6  * Author: Pawel Osciak <pawel@osciak.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation.
11  */
12 
13 #include <linux/dma-buf.h>
14 #include <linux/module.h>
15 #include <linux/refcount.h>
16 #include <linux/scatterlist.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/highmem.h>
21 
22 #include <media/videobuf2-v4l2.h>
23 #include <media/videobuf2-dma-contig.h>
24 #include <media/videobuf2-memops.h>
25 
26 struct vb2_dc_buf {
27 	struct device			*dev;
28 	void				*vaddr;
29 	unsigned long			size;
30 	void				*cookie;
31 	dma_addr_t			dma_addr;
32 	unsigned long			attrs;
33 	enum dma_data_direction		dma_dir;
34 	struct sg_table			*dma_sgt;
35 	struct frame_vector		*vec;
36 
37 	/* MMAP related */
38 	struct vb2_vmarea_handler	handler;
39 	refcount_t			refcount;
40 	struct sg_table			*sgt_base;
41 
42 	/* DMABUF related */
43 	struct dma_buf_attachment	*db_attach;
44 
45 	struct vb2_buffer		*vb;
46 	bool				non_coherent_mem;
47 };
48 
49 /*********************************************/
50 /*        scatterlist table functions        */
51 /*********************************************/
52 
53 static unsigned long vb2_dc_get_contiguous_size(struct sg_table *sgt)
54 {
55 	struct scatterlist *s;
56 	dma_addr_t expected = sg_dma_address(sgt->sgl);
57 	unsigned int i;
58 	unsigned long size = 0;
59 
60 	for_each_sgtable_dma_sg(sgt, s, i) {
61 		if (sg_dma_address(s) != expected)
62 			break;
63 		expected += sg_dma_len(s);
64 		size += sg_dma_len(s);
65 	}
66 	return size;
67 }
68 
69 /*********************************************/
70 /*         callbacks for all buffers         */
71 /*********************************************/
72 
73 static void *vb2_dc_cookie(struct vb2_buffer *vb, void *buf_priv)
74 {
75 	struct vb2_dc_buf *buf = buf_priv;
76 
77 	return &buf->dma_addr;
78 }
79 
80 /*
81  * This function may fail if:
82  *
83  * - dma_buf_vmap() fails
84  *   E.g. due to lack of virtual mapping address space, or due to
85  *   dmabuf->ops misconfiguration.
86  *
87  * - dma_vmap_noncontiguous() fails
88  *   For instance, when requested buffer size is larger than totalram_pages().
89  *   Relevant for buffers that use non-coherent memory.
90  *
91  * - Queue DMA attrs have DMA_ATTR_NO_KERNEL_MAPPING set
92  *   Relevant for buffers that use coherent memory.
93  */
94 static void *vb2_dc_vaddr(struct vb2_buffer *vb, void *buf_priv)
95 {
96 	struct vb2_dc_buf *buf = buf_priv;
97 
98 	if (buf->vaddr)
99 		return buf->vaddr;
100 
101 	if (buf->db_attach) {
102 		struct iosys_map map;
103 
104 		if (!dma_buf_vmap(buf->db_attach->dmabuf, &map))
105 			buf->vaddr = map.vaddr;
106 
107 		return buf->vaddr;
108 	}
109 
110 	if (buf->non_coherent_mem)
111 		buf->vaddr = dma_vmap_noncontiguous(buf->dev, buf->size,
112 						    buf->dma_sgt);
113 	return buf->vaddr;
114 }
115 
116 static unsigned int vb2_dc_num_users(void *buf_priv)
117 {
118 	struct vb2_dc_buf *buf = buf_priv;
119 
120 	return refcount_read(&buf->refcount);
121 }
122 
123 static void vb2_dc_prepare(void *buf_priv)
124 {
125 	struct vb2_dc_buf *buf = buf_priv;
126 	struct sg_table *sgt = buf->dma_sgt;
127 
128 	/* This takes care of DMABUF and user-enforced cache sync hint */
129 	if (buf->vb->skip_cache_sync_on_prepare)
130 		return;
131 
132 	if (!buf->non_coherent_mem)
133 		return;
134 
135 	/* Non-coherent MMAP only */
136 	if (buf->vaddr)
137 		flush_kernel_vmap_range(buf->vaddr, buf->size);
138 
139 	/* For both USERPTR and non-coherent MMAP */
140 	dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);
141 }
142 
143 static void vb2_dc_finish(void *buf_priv)
144 {
145 	struct vb2_dc_buf *buf = buf_priv;
146 	struct sg_table *sgt = buf->dma_sgt;
147 
148 	/* This takes care of DMABUF and user-enforced cache sync hint */
149 	if (buf->vb->skip_cache_sync_on_finish)
150 		return;
151 
152 	if (!buf->non_coherent_mem)
153 		return;
154 
155 	/* Non-coherent MMAP only */
156 	if (buf->vaddr)
157 		invalidate_kernel_vmap_range(buf->vaddr, buf->size);
158 
159 	/* For both USERPTR and non-coherent MMAP */
160 	dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);
161 }
162 
163 /*********************************************/
164 /*        callbacks for MMAP buffers         */
165 /*********************************************/
166 
167 static void vb2_dc_put(void *buf_priv)
168 {
169 	struct vb2_dc_buf *buf = buf_priv;
170 
171 	if (!refcount_dec_and_test(&buf->refcount))
172 		return;
173 
174 	if (buf->non_coherent_mem) {
175 		if (buf->vaddr)
176 			dma_vunmap_noncontiguous(buf->dev, buf->vaddr);
177 		dma_free_noncontiguous(buf->dev, buf->size,
178 				       buf->dma_sgt, buf->dma_dir);
179 	} else {
180 		if (buf->sgt_base) {
181 			sg_free_table(buf->sgt_base);
182 			kfree(buf->sgt_base);
183 		}
184 		dma_free_attrs(buf->dev, buf->size, buf->cookie,
185 			       buf->dma_addr, buf->attrs);
186 	}
187 	put_device(buf->dev);
188 	kfree(buf);
189 }
190 
191 static int vb2_dc_alloc_coherent(struct vb2_dc_buf *buf)
192 {
193 	struct vb2_queue *q = buf->vb->vb2_queue;
194 
195 	buf->cookie = dma_alloc_attrs(buf->dev,
196 				      buf->size,
197 				      &buf->dma_addr,
198 				      GFP_KERNEL | q->gfp_flags,
199 				      buf->attrs);
200 	if (!buf->cookie)
201 		return -ENOMEM;
202 
203 	if (q->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)
204 		return 0;
205 
206 	buf->vaddr = buf->cookie;
207 	return 0;
208 }
209 
210 static int vb2_dc_alloc_non_coherent(struct vb2_dc_buf *buf)
211 {
212 	struct vb2_queue *q = buf->vb->vb2_queue;
213 
214 	buf->dma_sgt = dma_alloc_noncontiguous(buf->dev,
215 					       buf->size,
216 					       buf->dma_dir,
217 					       GFP_KERNEL | q->gfp_flags,
218 					       buf->attrs);
219 	if (!buf->dma_sgt)
220 		return -ENOMEM;
221 
222 	buf->dma_addr = sg_dma_address(buf->dma_sgt->sgl);
223 
224 	/*
225 	 * For non-coherent buffers the kernel mapping is created on demand
226 	 * in vb2_dc_vaddr().
227 	 */
228 	return 0;
229 }
230 
231 static void *vb2_dc_alloc(struct vb2_buffer *vb,
232 			  struct device *dev,
233 			  unsigned long size)
234 {
235 	struct vb2_dc_buf *buf;
236 	int ret;
237 
238 	if (WARN_ON(!dev))
239 		return ERR_PTR(-EINVAL);
240 
241 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
242 	if (!buf)
243 		return ERR_PTR(-ENOMEM);
244 
245 	buf->attrs = vb->vb2_queue->dma_attrs;
246 	buf->dma_dir = vb->vb2_queue->dma_dir;
247 	buf->vb = vb;
248 	buf->non_coherent_mem = vb->vb2_queue->non_coherent_mem;
249 
250 	buf->size = size;
251 	/* Prevent the device from being released while the buffer is used */
252 	buf->dev = get_device(dev);
253 
254 	if (buf->non_coherent_mem)
255 		ret = vb2_dc_alloc_non_coherent(buf);
256 	else
257 		ret = vb2_dc_alloc_coherent(buf);
258 
259 	if (ret) {
260 		dev_err(dev, "dma alloc of size %lu failed\n", size);
261 		kfree(buf);
262 		return ERR_PTR(-ENOMEM);
263 	}
264 
265 	buf->handler.refcount = &buf->refcount;
266 	buf->handler.put = vb2_dc_put;
267 	buf->handler.arg = buf;
268 
269 	refcount_set(&buf->refcount, 1);
270 
271 	return buf;
272 }
273 
274 static int vb2_dc_mmap(void *buf_priv, struct vm_area_struct *vma)
275 {
276 	struct vb2_dc_buf *buf = buf_priv;
277 	int ret;
278 
279 	if (!buf) {
280 		printk(KERN_ERR "No buffer to map\n");
281 		return -EINVAL;
282 	}
283 
284 	if (buf->non_coherent_mem)
285 		ret = dma_mmap_noncontiguous(buf->dev, vma, buf->size,
286 					     buf->dma_sgt);
287 	else
288 		ret = dma_mmap_attrs(buf->dev, vma, buf->cookie, buf->dma_addr,
289 				     buf->size, buf->attrs);
290 	if (ret) {
291 		pr_err("Remapping memory failed, error: %d\n", ret);
292 		return ret;
293 	}
294 
295 	vma->vm_flags		|= VM_DONTEXPAND | VM_DONTDUMP;
296 	vma->vm_private_data	= &buf->handler;
297 	vma->vm_ops		= &vb2_common_vm_ops;
298 
299 	vma->vm_ops->open(vma);
300 
301 	pr_debug("%s: mapped dma addr 0x%08lx at 0x%08lx, size %lu\n",
302 		 __func__, (unsigned long)buf->dma_addr, vma->vm_start,
303 		 buf->size);
304 
305 	return 0;
306 }
307 
308 /*********************************************/
309 /*         DMABUF ops for exporters          */
310 /*********************************************/
311 
312 struct vb2_dc_attachment {
313 	struct sg_table sgt;
314 	enum dma_data_direction dma_dir;
315 };
316 
317 static int vb2_dc_dmabuf_ops_attach(struct dma_buf *dbuf,
318 	struct dma_buf_attachment *dbuf_attach)
319 {
320 	struct vb2_dc_attachment *attach;
321 	unsigned int i;
322 	struct scatterlist *rd, *wr;
323 	struct sg_table *sgt;
324 	struct vb2_dc_buf *buf = dbuf->priv;
325 	int ret;
326 
327 	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
328 	if (!attach)
329 		return -ENOMEM;
330 
331 	sgt = &attach->sgt;
332 	/* Copy the buf->base_sgt scatter list to the attachment, as we can't
333 	 * map the same scatter list to multiple attachments at the same time.
334 	 */
335 	ret = sg_alloc_table(sgt, buf->sgt_base->orig_nents, GFP_KERNEL);
336 	if (ret) {
337 		kfree(attach);
338 		return -ENOMEM;
339 	}
340 
341 	rd = buf->sgt_base->sgl;
342 	wr = sgt->sgl;
343 	for (i = 0; i < sgt->orig_nents; ++i) {
344 		sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
345 		rd = sg_next(rd);
346 		wr = sg_next(wr);
347 	}
348 
349 	attach->dma_dir = DMA_NONE;
350 	dbuf_attach->priv = attach;
351 
352 	return 0;
353 }
354 
355 static void vb2_dc_dmabuf_ops_detach(struct dma_buf *dbuf,
356 	struct dma_buf_attachment *db_attach)
357 {
358 	struct vb2_dc_attachment *attach = db_attach->priv;
359 	struct sg_table *sgt;
360 
361 	if (!attach)
362 		return;
363 
364 	sgt = &attach->sgt;
365 
366 	/* release the scatterlist cache */
367 	if (attach->dma_dir != DMA_NONE)
368 		/*
369 		 * Cache sync can be skipped here, as the vb2_dc memory is
370 		 * allocated from device coherent memory, which means the
371 		 * memory locations do not require any explicit cache
372 		 * maintenance prior or after being used by the device.
373 		 */
374 		dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir,
375 				  DMA_ATTR_SKIP_CPU_SYNC);
376 	sg_free_table(sgt);
377 	kfree(attach);
378 	db_attach->priv = NULL;
379 }
380 
381 static struct sg_table *vb2_dc_dmabuf_ops_map(
382 	struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
383 {
384 	struct vb2_dc_attachment *attach = db_attach->priv;
385 	/* stealing dmabuf mutex to serialize map/unmap operations */
386 	struct mutex *lock = &db_attach->dmabuf->lock;
387 	struct sg_table *sgt;
388 
389 	mutex_lock(lock);
390 
391 	sgt = &attach->sgt;
392 	/* return previously mapped sg table */
393 	if (attach->dma_dir == dma_dir) {
394 		mutex_unlock(lock);
395 		return sgt;
396 	}
397 
398 	/* release any previous cache */
399 	if (attach->dma_dir != DMA_NONE) {
400 		dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir,
401 				  DMA_ATTR_SKIP_CPU_SYNC);
402 		attach->dma_dir = DMA_NONE;
403 	}
404 
405 	/*
406 	 * mapping to the client with new direction, no cache sync
407 	 * required see comment in vb2_dc_dmabuf_ops_detach()
408 	 */
409 	if (dma_map_sgtable(db_attach->dev, sgt, dma_dir,
410 			    DMA_ATTR_SKIP_CPU_SYNC)) {
411 		pr_err("failed to map scatterlist\n");
412 		mutex_unlock(lock);
413 		return ERR_PTR(-EIO);
414 	}
415 
416 	attach->dma_dir = dma_dir;
417 
418 	mutex_unlock(lock);
419 
420 	return sgt;
421 }
422 
423 static void vb2_dc_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
424 	struct sg_table *sgt, enum dma_data_direction dma_dir)
425 {
426 	/* nothing to be done here */
427 }
428 
429 static void vb2_dc_dmabuf_ops_release(struct dma_buf *dbuf)
430 {
431 	/* drop reference obtained in vb2_dc_get_dmabuf */
432 	vb2_dc_put(dbuf->priv);
433 }
434 
435 static int
436 vb2_dc_dmabuf_ops_begin_cpu_access(struct dma_buf *dbuf,
437 				   enum dma_data_direction direction)
438 {
439 	return 0;
440 }
441 
442 static int
443 vb2_dc_dmabuf_ops_end_cpu_access(struct dma_buf *dbuf,
444 				 enum dma_data_direction direction)
445 {
446 	return 0;
447 }
448 
449 static int vb2_dc_dmabuf_ops_vmap(struct dma_buf *dbuf, struct iosys_map *map)
450 {
451 	struct vb2_dc_buf *buf;
452 	void *vaddr;
453 
454 	buf = dbuf->priv;
455 	vaddr = vb2_dc_vaddr(buf->vb, buf);
456 	if (!vaddr)
457 		return -EINVAL;
458 
459 	iosys_map_set_vaddr(map, vaddr);
460 
461 	return 0;
462 }
463 
464 static int vb2_dc_dmabuf_ops_mmap(struct dma_buf *dbuf,
465 	struct vm_area_struct *vma)
466 {
467 	return vb2_dc_mmap(dbuf->priv, vma);
468 }
469 
470 static const struct dma_buf_ops vb2_dc_dmabuf_ops = {
471 	.attach = vb2_dc_dmabuf_ops_attach,
472 	.detach = vb2_dc_dmabuf_ops_detach,
473 	.map_dma_buf = vb2_dc_dmabuf_ops_map,
474 	.unmap_dma_buf = vb2_dc_dmabuf_ops_unmap,
475 	.begin_cpu_access = vb2_dc_dmabuf_ops_begin_cpu_access,
476 	.end_cpu_access = vb2_dc_dmabuf_ops_end_cpu_access,
477 	.vmap = vb2_dc_dmabuf_ops_vmap,
478 	.mmap = vb2_dc_dmabuf_ops_mmap,
479 	.release = vb2_dc_dmabuf_ops_release,
480 };
481 
482 static struct sg_table *vb2_dc_get_base_sgt(struct vb2_dc_buf *buf)
483 {
484 	int ret;
485 	struct sg_table *sgt;
486 
487 	if (buf->non_coherent_mem)
488 		return buf->dma_sgt;
489 
490 	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
491 	if (!sgt) {
492 		dev_err(buf->dev, "failed to alloc sg table\n");
493 		return NULL;
494 	}
495 
496 	ret = dma_get_sgtable_attrs(buf->dev, sgt, buf->cookie, buf->dma_addr,
497 		buf->size, buf->attrs);
498 	if (ret < 0) {
499 		dev_err(buf->dev, "failed to get scatterlist from DMA API\n");
500 		kfree(sgt);
501 		return NULL;
502 	}
503 
504 	return sgt;
505 }
506 
507 static struct dma_buf *vb2_dc_get_dmabuf(struct vb2_buffer *vb,
508 					 void *buf_priv,
509 					 unsigned long flags)
510 {
511 	struct vb2_dc_buf *buf = buf_priv;
512 	struct dma_buf *dbuf;
513 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
514 
515 	exp_info.ops = &vb2_dc_dmabuf_ops;
516 	exp_info.size = buf->size;
517 	exp_info.flags = flags;
518 	exp_info.priv = buf;
519 
520 	if (!buf->sgt_base)
521 		buf->sgt_base = vb2_dc_get_base_sgt(buf);
522 
523 	if (WARN_ON(!buf->sgt_base))
524 		return NULL;
525 
526 	dbuf = dma_buf_export(&exp_info);
527 	if (IS_ERR(dbuf))
528 		return NULL;
529 
530 	/* dmabuf keeps reference to vb2 buffer */
531 	refcount_inc(&buf->refcount);
532 
533 	return dbuf;
534 }
535 
536 /*********************************************/
537 /*       callbacks for USERPTR buffers       */
538 /*********************************************/
539 
540 static void vb2_dc_put_userptr(void *buf_priv)
541 {
542 	struct vb2_dc_buf *buf = buf_priv;
543 	struct sg_table *sgt = buf->dma_sgt;
544 	int i;
545 	struct page **pages;
546 
547 	if (sgt) {
548 		/*
549 		 * No need to sync to CPU, it's already synced to the CPU
550 		 * since the finish() memop will have been called before this.
551 		 */
552 		dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir,
553 				  DMA_ATTR_SKIP_CPU_SYNC);
554 		pages = frame_vector_pages(buf->vec);
555 		/* sgt should exist only if vector contains pages... */
556 		BUG_ON(IS_ERR(pages));
557 		if (buf->dma_dir == DMA_FROM_DEVICE ||
558 		    buf->dma_dir == DMA_BIDIRECTIONAL)
559 			for (i = 0; i < frame_vector_count(buf->vec); i++)
560 				set_page_dirty_lock(pages[i]);
561 		sg_free_table(sgt);
562 		kfree(sgt);
563 	} else {
564 		dma_unmap_resource(buf->dev, buf->dma_addr, buf->size,
565 				   buf->dma_dir, 0);
566 	}
567 	vb2_destroy_framevec(buf->vec);
568 	kfree(buf);
569 }
570 
571 static void *vb2_dc_get_userptr(struct vb2_buffer *vb, struct device *dev,
572 				unsigned long vaddr, unsigned long size)
573 {
574 	struct vb2_dc_buf *buf;
575 	struct frame_vector *vec;
576 	unsigned int offset;
577 	int n_pages, i;
578 	int ret = 0;
579 	struct sg_table *sgt;
580 	unsigned long contig_size;
581 	unsigned long dma_align = dma_get_cache_alignment();
582 
583 	/* Only cache aligned DMA transfers are reliable */
584 	if (!IS_ALIGNED(vaddr | size, dma_align)) {
585 		pr_debug("user data must be aligned to %lu bytes\n", dma_align);
586 		return ERR_PTR(-EINVAL);
587 	}
588 
589 	if (!size) {
590 		pr_debug("size is zero\n");
591 		return ERR_PTR(-EINVAL);
592 	}
593 
594 	if (WARN_ON(!dev))
595 		return ERR_PTR(-EINVAL);
596 
597 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
598 	if (!buf)
599 		return ERR_PTR(-ENOMEM);
600 
601 	buf->dev = dev;
602 	buf->dma_dir = vb->vb2_queue->dma_dir;
603 	buf->vb = vb;
604 
605 	offset = lower_32_bits(offset_in_page(vaddr));
606 	vec = vb2_create_framevec(vaddr, size);
607 	if (IS_ERR(vec)) {
608 		ret = PTR_ERR(vec);
609 		goto fail_buf;
610 	}
611 	buf->vec = vec;
612 	n_pages = frame_vector_count(vec);
613 	ret = frame_vector_to_pages(vec);
614 	if (ret < 0) {
615 		unsigned long *nums = frame_vector_pfns(vec);
616 
617 		/*
618 		 * Failed to convert to pages... Check the memory is physically
619 		 * contiguous and use direct mapping
620 		 */
621 		for (i = 1; i < n_pages; i++)
622 			if (nums[i-1] + 1 != nums[i])
623 				goto fail_pfnvec;
624 		buf->dma_addr = dma_map_resource(buf->dev,
625 				__pfn_to_phys(nums[0]), size, buf->dma_dir, 0);
626 		if (dma_mapping_error(buf->dev, buf->dma_addr)) {
627 			ret = -ENOMEM;
628 			goto fail_pfnvec;
629 		}
630 		goto out;
631 	}
632 
633 	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
634 	if (!sgt) {
635 		pr_err("failed to allocate sg table\n");
636 		ret = -ENOMEM;
637 		goto fail_pfnvec;
638 	}
639 
640 	ret = sg_alloc_table_from_pages(sgt, frame_vector_pages(vec), n_pages,
641 		offset, size, GFP_KERNEL);
642 	if (ret) {
643 		pr_err("failed to initialize sg table\n");
644 		goto fail_sgt;
645 	}
646 
647 	/*
648 	 * No need to sync to the device, this will happen later when the
649 	 * prepare() memop is called.
650 	 */
651 	if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
652 			    DMA_ATTR_SKIP_CPU_SYNC)) {
653 		pr_err("failed to map scatterlist\n");
654 		ret = -EIO;
655 		goto fail_sgt_init;
656 	}
657 
658 	contig_size = vb2_dc_get_contiguous_size(sgt);
659 	if (contig_size < size) {
660 		pr_err("contiguous mapping is too small %lu/%lu\n",
661 			contig_size, size);
662 		ret = -EFAULT;
663 		goto fail_map_sg;
664 	}
665 
666 	buf->dma_addr = sg_dma_address(sgt->sgl);
667 	buf->dma_sgt = sgt;
668 	buf->non_coherent_mem = 1;
669 
670 out:
671 	buf->size = size;
672 
673 	return buf;
674 
675 fail_map_sg:
676 	dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
677 
678 fail_sgt_init:
679 	sg_free_table(sgt);
680 
681 fail_sgt:
682 	kfree(sgt);
683 
684 fail_pfnvec:
685 	vb2_destroy_framevec(vec);
686 
687 fail_buf:
688 	kfree(buf);
689 
690 	return ERR_PTR(ret);
691 }
692 
693 /*********************************************/
694 /*       callbacks for DMABUF buffers        */
695 /*********************************************/
696 
697 static int vb2_dc_map_dmabuf(void *mem_priv)
698 {
699 	struct vb2_dc_buf *buf = mem_priv;
700 	struct sg_table *sgt;
701 	unsigned long contig_size;
702 
703 	if (WARN_ON(!buf->db_attach)) {
704 		pr_err("trying to pin a non attached buffer\n");
705 		return -EINVAL;
706 	}
707 
708 	if (WARN_ON(buf->dma_sgt)) {
709 		pr_err("dmabuf buffer is already pinned\n");
710 		return 0;
711 	}
712 
713 	/* get the associated scatterlist for this buffer */
714 	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);
715 	if (IS_ERR(sgt)) {
716 		pr_err("Error getting dmabuf scatterlist\n");
717 		return -EINVAL;
718 	}
719 
720 	/* checking if dmabuf is big enough to store contiguous chunk */
721 	contig_size = vb2_dc_get_contiguous_size(sgt);
722 	if (contig_size < buf->size) {
723 		pr_err("contiguous chunk is too small %lu/%lu\n",
724 		       contig_size, buf->size);
725 		dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);
726 		return -EFAULT;
727 	}
728 
729 	buf->dma_addr = sg_dma_address(sgt->sgl);
730 	buf->dma_sgt = sgt;
731 	buf->vaddr = NULL;
732 
733 	return 0;
734 }
735 
736 static void vb2_dc_unmap_dmabuf(void *mem_priv)
737 {
738 	struct vb2_dc_buf *buf = mem_priv;
739 	struct sg_table *sgt = buf->dma_sgt;
740 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(buf->vaddr);
741 
742 	if (WARN_ON(!buf->db_attach)) {
743 		pr_err("trying to unpin a not attached buffer\n");
744 		return;
745 	}
746 
747 	if (WARN_ON(!sgt)) {
748 		pr_err("dmabuf buffer is already unpinned\n");
749 		return;
750 	}
751 
752 	if (buf->vaddr) {
753 		dma_buf_vunmap(buf->db_attach->dmabuf, &map);
754 		buf->vaddr = NULL;
755 	}
756 	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);
757 
758 	buf->dma_addr = 0;
759 	buf->dma_sgt = NULL;
760 }
761 
762 static void vb2_dc_detach_dmabuf(void *mem_priv)
763 {
764 	struct vb2_dc_buf *buf = mem_priv;
765 
766 	/* if vb2 works correctly you should never detach mapped buffer */
767 	if (WARN_ON(buf->dma_addr))
768 		vb2_dc_unmap_dmabuf(buf);
769 
770 	/* detach this attachment */
771 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);
772 	kfree(buf);
773 }
774 
775 static void *vb2_dc_attach_dmabuf(struct vb2_buffer *vb, struct device *dev,
776 				  struct dma_buf *dbuf, unsigned long size)
777 {
778 	struct vb2_dc_buf *buf;
779 	struct dma_buf_attachment *dba;
780 
781 	if (dbuf->size < size)
782 		return ERR_PTR(-EFAULT);
783 
784 	if (WARN_ON(!dev))
785 		return ERR_PTR(-EINVAL);
786 
787 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
788 	if (!buf)
789 		return ERR_PTR(-ENOMEM);
790 
791 	buf->dev = dev;
792 	buf->vb = vb;
793 
794 	/* create attachment for the dmabuf with the user device */
795 	dba = dma_buf_attach(dbuf, buf->dev);
796 	if (IS_ERR(dba)) {
797 		pr_err("failed to attach dmabuf\n");
798 		kfree(buf);
799 		return dba;
800 	}
801 
802 	buf->dma_dir = vb->vb2_queue->dma_dir;
803 	buf->size = size;
804 	buf->db_attach = dba;
805 
806 	return buf;
807 }
808 
809 /*********************************************/
810 /*       DMA CONTIG exported functions       */
811 /*********************************************/
812 
813 const struct vb2_mem_ops vb2_dma_contig_memops = {
814 	.alloc		= vb2_dc_alloc,
815 	.put		= vb2_dc_put,
816 	.get_dmabuf	= vb2_dc_get_dmabuf,
817 	.cookie		= vb2_dc_cookie,
818 	.vaddr		= vb2_dc_vaddr,
819 	.mmap		= vb2_dc_mmap,
820 	.get_userptr	= vb2_dc_get_userptr,
821 	.put_userptr	= vb2_dc_put_userptr,
822 	.prepare	= vb2_dc_prepare,
823 	.finish		= vb2_dc_finish,
824 	.map_dmabuf	= vb2_dc_map_dmabuf,
825 	.unmap_dmabuf	= vb2_dc_unmap_dmabuf,
826 	.attach_dmabuf	= vb2_dc_attach_dmabuf,
827 	.detach_dmabuf	= vb2_dc_detach_dmabuf,
828 	.num_users	= vb2_dc_num_users,
829 };
830 EXPORT_SYMBOL_GPL(vb2_dma_contig_memops);
831 
832 /**
833  * vb2_dma_contig_set_max_seg_size() - configure DMA max segment size
834  * @dev:	device for configuring DMA parameters
835  * @size:	size of DMA max segment size to set
836  *
837  * To allow mapping the scatter-list into a single chunk in the DMA
838  * address space, the device is required to have the DMA max segment
839  * size parameter set to a value larger than the buffer size. Otherwise,
840  * the DMA-mapping subsystem will split the mapping into max segment
841  * size chunks. This function sets the DMA max segment size
842  * parameter to let DMA-mapping map a buffer as a single chunk in DMA
843  * address space.
844  * This code assumes that the DMA-mapping subsystem will merge all
845  * scatterlist segments if this is really possible (for example when
846  * an IOMMU is available and enabled).
847  * Ideally, this parameter should be set by the generic bus code, but it
848  * is left with the default 64KiB value due to historical litmiations in
849  * other subsystems (like limited USB host drivers) and there no good
850  * place to set it to the proper value.
851  * This function should be called from the drivers, which are known to
852  * operate on platforms with IOMMU and provide access to shared buffers
853  * (either USERPTR or DMABUF). This should be done before initializing
854  * videobuf2 queue.
855  */
856 int vb2_dma_contig_set_max_seg_size(struct device *dev, unsigned int size)
857 {
858 	if (!dev->dma_parms) {
859 		dev_err(dev, "Failed to set max_seg_size: dma_parms is NULL\n");
860 		return -ENODEV;
861 	}
862 	if (dma_get_max_seg_size(dev) < size)
863 		return dma_set_max_seg_size(dev, size);
864 
865 	return 0;
866 }
867 EXPORT_SYMBOL_GPL(vb2_dma_contig_set_max_seg_size);
868 
869 MODULE_DESCRIPTION("DMA-contig memory handling routines for videobuf2");
870 MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>");
871 MODULE_LICENSE("GPL");
872 MODULE_IMPORT_NS(DMA_BUF);
873