17952be9bSHans Verkuil /*
27952be9bSHans Verkuil * videobuf2-dma-sg.c - dma scatter/gather memory allocator for videobuf2
37952be9bSHans Verkuil *
47952be9bSHans Verkuil * Copyright (C) 2010 Samsung Electronics
57952be9bSHans Verkuil *
666e988e9SAndrzej Pietrasiewicz * Author: Andrzej Pietrasiewicz <andrzejtp2010@gmail.com>
77952be9bSHans Verkuil *
87952be9bSHans Verkuil * This program is free software; you can redistribute it and/or modify
97952be9bSHans Verkuil * it under the terms of the GNU General Public License as published by
107952be9bSHans Verkuil * the Free Software Foundation.
117952be9bSHans Verkuil */
127952be9bSHans Verkuil
137952be9bSHans Verkuil #include <linux/module.h>
147952be9bSHans Verkuil #include <linux/mm.h>
157952be9bSHans Verkuil #include <linux/refcount.h>
167952be9bSHans Verkuil #include <linux/scatterlist.h>
177952be9bSHans Verkuil #include <linux/sched.h>
187952be9bSHans Verkuil #include <linux/slab.h>
197952be9bSHans Verkuil #include <linux/vmalloc.h>
207952be9bSHans Verkuil
217952be9bSHans Verkuil #include <media/videobuf2-v4l2.h>
227952be9bSHans Verkuil #include <media/videobuf2-memops.h>
237952be9bSHans Verkuil #include <media/videobuf2-dma-sg.h>
247952be9bSHans Verkuil
257952be9bSHans Verkuil static int debug;
267952be9bSHans Verkuil module_param(debug, int, 0644);
277952be9bSHans Verkuil
287952be9bSHans Verkuil #define dprintk(level, fmt, arg...) \
297952be9bSHans Verkuil do { \
307952be9bSHans Verkuil if (debug >= level) \
317952be9bSHans Verkuil printk(KERN_DEBUG "vb2-dma-sg: " fmt, ## arg); \
327952be9bSHans Verkuil } while (0)
337952be9bSHans Verkuil
347952be9bSHans Verkuil struct vb2_dma_sg_buf {
357952be9bSHans Verkuil struct device *dev;
367952be9bSHans Verkuil void *vaddr;
377952be9bSHans Verkuil struct page **pages;
387952be9bSHans Verkuil struct frame_vector *vec;
397952be9bSHans Verkuil int offset;
407952be9bSHans Verkuil enum dma_data_direction dma_dir;
417952be9bSHans Verkuil struct sg_table sg_table;
427952be9bSHans Verkuil /*
437952be9bSHans Verkuil * This will point to sg_table when used with the MMAP or USERPTR
447952be9bSHans Verkuil * memory model, and to the dma_buf sglist when used with the
457952be9bSHans Verkuil * DMABUF memory model.
467952be9bSHans Verkuil */
477952be9bSHans Verkuil struct sg_table *dma_sgt;
487952be9bSHans Verkuil size_t size;
497952be9bSHans Verkuil unsigned int num_pages;
507952be9bSHans Verkuil refcount_t refcount;
517952be9bSHans Verkuil struct vb2_vmarea_handler handler;
527952be9bSHans Verkuil
537952be9bSHans Verkuil struct dma_buf_attachment *db_attach;
54a4b83debSSergey Senozhatsky
55a4b83debSSergey Senozhatsky struct vb2_buffer *vb;
567952be9bSHans Verkuil };
577952be9bSHans Verkuil
587952be9bSHans Verkuil static void vb2_dma_sg_put(void *buf_priv);
597952be9bSHans Verkuil
vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf * buf,gfp_t gfp_flags)607952be9bSHans Verkuil static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,
617952be9bSHans Verkuil gfp_t gfp_flags)
627952be9bSHans Verkuil {
637952be9bSHans Verkuil unsigned int last_page = 0;
6414f28f5cSSakari Ailus unsigned long size = buf->size;
657952be9bSHans Verkuil
667952be9bSHans Verkuil while (size > 0) {
677952be9bSHans Verkuil struct page *pages;
687952be9bSHans Verkuil int order;
697952be9bSHans Verkuil int i;
707952be9bSHans Verkuil
717952be9bSHans Verkuil order = get_order(size);
724b129dc9SMauro Carvalho Chehab /* Don't over allocate*/
737952be9bSHans Verkuil if ((PAGE_SIZE << order) > size)
747952be9bSHans Verkuil order--;
757952be9bSHans Verkuil
767952be9bSHans Verkuil pages = NULL;
777952be9bSHans Verkuil while (!pages) {
787952be9bSHans Verkuil pages = alloc_pages(GFP_KERNEL | __GFP_ZERO |
797952be9bSHans Verkuil __GFP_NOWARN | gfp_flags, order);
807952be9bSHans Verkuil if (pages)
817952be9bSHans Verkuil break;
827952be9bSHans Verkuil
837952be9bSHans Verkuil if (order == 0) {
847952be9bSHans Verkuil while (last_page--)
857952be9bSHans Verkuil __free_page(buf->pages[last_page]);
867952be9bSHans Verkuil return -ENOMEM;
877952be9bSHans Verkuil }
887952be9bSHans Verkuil order--;
897952be9bSHans Verkuil }
907952be9bSHans Verkuil
917952be9bSHans Verkuil split_page(pages, order);
927952be9bSHans Verkuil for (i = 0; i < (1 << order); i++)
937952be9bSHans Verkuil buf->pages[last_page++] = &pages[i];
947952be9bSHans Verkuil
957952be9bSHans Verkuil size -= PAGE_SIZE << order;
967952be9bSHans Verkuil }
977952be9bSHans Verkuil
987952be9bSHans Verkuil return 0;
997952be9bSHans Verkuil }
1007952be9bSHans Verkuil
vb2_dma_sg_alloc(struct vb2_buffer * vb,struct device * dev,unsigned long size)101a4b83debSSergey Senozhatsky static void *vb2_dma_sg_alloc(struct vb2_buffer *vb, struct device *dev,
102a4b83debSSergey Senozhatsky unsigned long size)
1037952be9bSHans Verkuil {
1047952be9bSHans Verkuil struct vb2_dma_sg_buf *buf;
1057952be9bSHans Verkuil struct sg_table *sgt;
1067952be9bSHans Verkuil int ret;
1077952be9bSHans Verkuil int num_pages;
1087952be9bSHans Verkuil
1099cc25c4bSRicardo Ribalda if (WARN_ON(!dev) || WARN_ON(!size))
1107952be9bSHans Verkuil return ERR_PTR(-EINVAL);
1117952be9bSHans Verkuil
1127952be9bSHans Verkuil buf = kzalloc(sizeof *buf, GFP_KERNEL);
1137952be9bSHans Verkuil if (!buf)
1147952be9bSHans Verkuil return ERR_PTR(-ENOMEM);
1157952be9bSHans Verkuil
1167952be9bSHans Verkuil buf->vaddr = NULL;
117a4b83debSSergey Senozhatsky buf->dma_dir = vb->vb2_queue->dma_dir;
1187952be9bSHans Verkuil buf->offset = 0;
1197952be9bSHans Verkuil buf->size = size;
1207952be9bSHans Verkuil /* size is already page aligned */
1217952be9bSHans Verkuil buf->num_pages = size >> PAGE_SHIFT;
1227952be9bSHans Verkuil buf->dma_sgt = &buf->sg_table;
1237952be9bSHans Verkuil
124d4db5eb5SSergey Senozhatsky /*
125d4db5eb5SSergey Senozhatsky * NOTE: dma-sg allocates memory using the page allocator directly, so
126d4db5eb5SSergey Senozhatsky * there is no memory consistency guarantee, hence dma-sg ignores DMA
127129134e5SSergey Senozhatsky * attributes passed from the upper layer.
128d4db5eb5SSergey Senozhatsky */
129d5a8099cSHans Verkuil buf->pages = kvcalloc(buf->num_pages, sizeof(struct page *), GFP_KERNEL);
1307952be9bSHans Verkuil if (!buf->pages)
1317952be9bSHans Verkuil goto fail_pages_array_alloc;
1327952be9bSHans Verkuil
133a4b83debSSergey Senozhatsky ret = vb2_dma_sg_alloc_compacted(buf, vb->vb2_queue->gfp_flags);
1347952be9bSHans Verkuil if (ret)
1357952be9bSHans Verkuil goto fail_pages_alloc;
1367952be9bSHans Verkuil
1377952be9bSHans Verkuil ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
1387952be9bSHans Verkuil buf->num_pages, 0, size, GFP_KERNEL);
1397952be9bSHans Verkuil if (ret)
1407952be9bSHans Verkuil goto fail_table_alloc;
1417952be9bSHans Verkuil
1427952be9bSHans Verkuil /* Prevent the device from being released while the buffer is used */
1437952be9bSHans Verkuil buf->dev = get_device(dev);
1447952be9bSHans Verkuil
1457952be9bSHans Verkuil sgt = &buf->sg_table;
1467952be9bSHans Verkuil /*
1477952be9bSHans Verkuil * No need to sync to the device, this will happen later when the
1487952be9bSHans Verkuil * prepare() memop is called.
1497952be9bSHans Verkuil */
1508b7c0280SMarek Szyprowski if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
1518b7c0280SMarek Szyprowski DMA_ATTR_SKIP_CPU_SYNC))
1527952be9bSHans Verkuil goto fail_map;
1537952be9bSHans Verkuil
1547952be9bSHans Verkuil buf->handler.refcount = &buf->refcount;
1557952be9bSHans Verkuil buf->handler.put = vb2_dma_sg_put;
1567952be9bSHans Verkuil buf->handler.arg = buf;
157a4b83debSSergey Senozhatsky buf->vb = vb;
1587952be9bSHans Verkuil
1597952be9bSHans Verkuil refcount_set(&buf->refcount, 1);
1607952be9bSHans Verkuil
1617952be9bSHans Verkuil dprintk(1, "%s: Allocated buffer of %d pages\n",
1627952be9bSHans Verkuil __func__, buf->num_pages);
1637952be9bSHans Verkuil return buf;
1647952be9bSHans Verkuil
1657952be9bSHans Verkuil fail_map:
1667952be9bSHans Verkuil put_device(buf->dev);
1677952be9bSHans Verkuil sg_free_table(buf->dma_sgt);
1687952be9bSHans Verkuil fail_table_alloc:
1697952be9bSHans Verkuil num_pages = buf->num_pages;
1707952be9bSHans Verkuil while (num_pages--)
1717952be9bSHans Verkuil __free_page(buf->pages[num_pages]);
1727952be9bSHans Verkuil fail_pages_alloc:
1737952be9bSHans Verkuil kvfree(buf->pages);
1747952be9bSHans Verkuil fail_pages_array_alloc:
1757952be9bSHans Verkuil kfree(buf);
1767952be9bSHans Verkuil return ERR_PTR(-ENOMEM);
1777952be9bSHans Verkuil }
1787952be9bSHans Verkuil
vb2_dma_sg_put(void * buf_priv)1797952be9bSHans Verkuil static void vb2_dma_sg_put(void *buf_priv)
1807952be9bSHans Verkuil {
1817952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
1827952be9bSHans Verkuil struct sg_table *sgt = &buf->sg_table;
1837952be9bSHans Verkuil int i = buf->num_pages;
1847952be9bSHans Verkuil
1857952be9bSHans Verkuil if (refcount_dec_and_test(&buf->refcount)) {
1867952be9bSHans Verkuil dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
1877952be9bSHans Verkuil buf->num_pages);
1888b7c0280SMarek Szyprowski dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir,
1898b7c0280SMarek Szyprowski DMA_ATTR_SKIP_CPU_SYNC);
1907952be9bSHans Verkuil if (buf->vaddr)
1917952be9bSHans Verkuil vm_unmap_ram(buf->vaddr, buf->num_pages);
1927952be9bSHans Verkuil sg_free_table(buf->dma_sgt);
1937952be9bSHans Verkuil while (--i >= 0)
1947952be9bSHans Verkuil __free_page(buf->pages[i]);
1957952be9bSHans Verkuil kvfree(buf->pages);
1967952be9bSHans Verkuil put_device(buf->dev);
1977952be9bSHans Verkuil kfree(buf);
1987952be9bSHans Verkuil }
1997952be9bSHans Verkuil }
2007952be9bSHans Verkuil
vb2_dma_sg_prepare(void * buf_priv)2017952be9bSHans Verkuil static void vb2_dma_sg_prepare(void *buf_priv)
2027952be9bSHans Verkuil {
2037952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
2047952be9bSHans Verkuil struct sg_table *sgt = buf->dma_sgt;
2057952be9bSHans Verkuil
206cde513fdSSergey Senozhatsky if (buf->vb->skip_cache_sync_on_prepare)
207cde513fdSSergey Senozhatsky return;
208cde513fdSSergey Senozhatsky
2098b7c0280SMarek Szyprowski dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);
2107952be9bSHans Verkuil }
2117952be9bSHans Verkuil
vb2_dma_sg_finish(void * buf_priv)2127952be9bSHans Verkuil static void vb2_dma_sg_finish(void *buf_priv)
2137952be9bSHans Verkuil {
2147952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
2157952be9bSHans Verkuil struct sg_table *sgt = buf->dma_sgt;
2167952be9bSHans Verkuil
217cde513fdSSergey Senozhatsky if (buf->vb->skip_cache_sync_on_finish)
218cde513fdSSergey Senozhatsky return;
219cde513fdSSergey Senozhatsky
2208b7c0280SMarek Szyprowski dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);
2217952be9bSHans Verkuil }
2227952be9bSHans Verkuil
vb2_dma_sg_get_userptr(struct vb2_buffer * vb,struct device * dev,unsigned long vaddr,unsigned long size)223a4b83debSSergey Senozhatsky static void *vb2_dma_sg_get_userptr(struct vb2_buffer *vb, struct device *dev,
224a4b83debSSergey Senozhatsky unsigned long vaddr, unsigned long size)
2257952be9bSHans Verkuil {
2267952be9bSHans Verkuil struct vb2_dma_sg_buf *buf;
2277952be9bSHans Verkuil struct sg_table *sgt;
2287952be9bSHans Verkuil struct frame_vector *vec;
2297952be9bSHans Verkuil
2307952be9bSHans Verkuil if (WARN_ON(!dev))
2317952be9bSHans Verkuil return ERR_PTR(-EINVAL);
2327952be9bSHans Verkuil
2337952be9bSHans Verkuil buf = kzalloc(sizeof *buf, GFP_KERNEL);
2347952be9bSHans Verkuil if (!buf)
2357952be9bSHans Verkuil return ERR_PTR(-ENOMEM);
2367952be9bSHans Verkuil
2377952be9bSHans Verkuil buf->vaddr = NULL;
2387952be9bSHans Verkuil buf->dev = dev;
239a4b83debSSergey Senozhatsky buf->dma_dir = vb->vb2_queue->dma_dir;
2407952be9bSHans Verkuil buf->offset = vaddr & ~PAGE_MASK;
2417952be9bSHans Verkuil buf->size = size;
2427952be9bSHans Verkuil buf->dma_sgt = &buf->sg_table;
243d55c3ee6SHans de Goede buf->vb = vb;
244e2fc6eddSHans Verkuil vec = vb2_create_framevec(vaddr, size,
245e2fc6eddSHans Verkuil buf->dma_dir == DMA_FROM_DEVICE ||
246e2fc6eddSHans Verkuil buf->dma_dir == DMA_BIDIRECTIONAL);
2477952be9bSHans Verkuil if (IS_ERR(vec))
2487952be9bSHans Verkuil goto userptr_fail_pfnvec;
2497952be9bSHans Verkuil buf->vec = vec;
2507952be9bSHans Verkuil
2517952be9bSHans Verkuil buf->pages = frame_vector_pages(vec);
2527952be9bSHans Verkuil if (IS_ERR(buf->pages))
2537952be9bSHans Verkuil goto userptr_fail_sgtable;
2547952be9bSHans Verkuil buf->num_pages = frame_vector_count(vec);
2557952be9bSHans Verkuil
2567952be9bSHans Verkuil if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
2577952be9bSHans Verkuil buf->num_pages, buf->offset, size, 0))
2587952be9bSHans Verkuil goto userptr_fail_sgtable;
2597952be9bSHans Verkuil
2607952be9bSHans Verkuil sgt = &buf->sg_table;
2617952be9bSHans Verkuil /*
2627952be9bSHans Verkuil * No need to sync to the device, this will happen later when the
2637952be9bSHans Verkuil * prepare() memop is called.
2647952be9bSHans Verkuil */
2658b7c0280SMarek Szyprowski if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
2668b7c0280SMarek Szyprowski DMA_ATTR_SKIP_CPU_SYNC))
2677952be9bSHans Verkuil goto userptr_fail_map;
2687952be9bSHans Verkuil
2697952be9bSHans Verkuil return buf;
2707952be9bSHans Verkuil
2717952be9bSHans Verkuil userptr_fail_map:
2727952be9bSHans Verkuil sg_free_table(&buf->sg_table);
2737952be9bSHans Verkuil userptr_fail_sgtable:
2747952be9bSHans Verkuil vb2_destroy_framevec(vec);
2757952be9bSHans Verkuil userptr_fail_pfnvec:
2767952be9bSHans Verkuil kfree(buf);
2777952be9bSHans Verkuil return ERR_PTR(-ENOMEM);
2787952be9bSHans Verkuil }
2797952be9bSHans Verkuil
2807952be9bSHans Verkuil /*
2817952be9bSHans Verkuil * @put_userptr: inform the allocator that a USERPTR buffer will no longer
2827952be9bSHans Verkuil * be used
2837952be9bSHans Verkuil */
vb2_dma_sg_put_userptr(void * buf_priv)2847952be9bSHans Verkuil static void vb2_dma_sg_put_userptr(void *buf_priv)
2857952be9bSHans Verkuil {
2867952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
2877952be9bSHans Verkuil struct sg_table *sgt = &buf->sg_table;
2887952be9bSHans Verkuil int i = buf->num_pages;
2897952be9bSHans Verkuil
2907952be9bSHans Verkuil dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
2917952be9bSHans Verkuil __func__, buf->num_pages);
2928b7c0280SMarek Szyprowski dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
2937952be9bSHans Verkuil if (buf->vaddr)
2947952be9bSHans Verkuil vm_unmap_ram(buf->vaddr, buf->num_pages);
2957952be9bSHans Verkuil sg_free_table(buf->dma_sgt);
2967952be9bSHans Verkuil if (buf->dma_dir == DMA_FROM_DEVICE ||
2977952be9bSHans Verkuil buf->dma_dir == DMA_BIDIRECTIONAL)
2987952be9bSHans Verkuil while (--i >= 0)
2997952be9bSHans Verkuil set_page_dirty_lock(buf->pages[i]);
3007952be9bSHans Verkuil vb2_destroy_framevec(buf->vec);
3017952be9bSHans Verkuil kfree(buf);
3027952be9bSHans Verkuil }
3037952be9bSHans Verkuil
vb2_dma_sg_vaddr(struct vb2_buffer * vb,void * buf_priv)304a4b83debSSergey Senozhatsky static void *vb2_dma_sg_vaddr(struct vb2_buffer *vb, void *buf_priv)
3057952be9bSHans Verkuil {
3067952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
3077938f421SLucas De Marchi struct iosys_map map;
3086619ccf1SThomas Zimmermann int ret;
3097952be9bSHans Verkuil
3107952be9bSHans Verkuil BUG_ON(!buf);
3117952be9bSHans Verkuil
3127952be9bSHans Verkuil if (!buf->vaddr) {
3136619ccf1SThomas Zimmermann if (buf->db_attach) {
314a26ee3b7SDmitry Osipenko ret = dma_buf_vmap_unlocked(buf->db_attach->dmabuf, &map);
3156619ccf1SThomas Zimmermann buf->vaddr = ret ? NULL : map.vaddr;
3166619ccf1SThomas Zimmermann } else {
317d4efd79aSChristoph Hellwig buf->vaddr = vm_map_ram(buf->pages, buf->num_pages, -1);
3187952be9bSHans Verkuil }
3196619ccf1SThomas Zimmermann }
3207952be9bSHans Verkuil
3217952be9bSHans Verkuil /* add offset in case userptr is not page-aligned */
3227952be9bSHans Verkuil return buf->vaddr ? buf->vaddr + buf->offset : NULL;
3237952be9bSHans Verkuil }
3247952be9bSHans Verkuil
vb2_dma_sg_num_users(void * buf_priv)3257952be9bSHans Verkuil static unsigned int vb2_dma_sg_num_users(void *buf_priv)
3267952be9bSHans Verkuil {
3277952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
3287952be9bSHans Verkuil
3297952be9bSHans Verkuil return refcount_read(&buf->refcount);
3307952be9bSHans Verkuil }
3317952be9bSHans Verkuil
vb2_dma_sg_mmap(void * buf_priv,struct vm_area_struct * vma)3327952be9bSHans Verkuil static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma)
3337952be9bSHans Verkuil {
3347952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
335a17ae147SSouptick Joarder int err;
3367952be9bSHans Verkuil
3377952be9bSHans Verkuil if (!buf) {
3387952be9bSHans Verkuil printk(KERN_ERR "No memory to map\n");
3397952be9bSHans Verkuil return -EINVAL;
3407952be9bSHans Verkuil }
3417952be9bSHans Verkuil
342a17ae147SSouptick Joarder err = vm_map_pages(vma, buf->pages, buf->num_pages);
343a17ae147SSouptick Joarder if (err) {
344a17ae147SSouptick Joarder printk(KERN_ERR "Remapping memory, error: %d\n", err);
345a17ae147SSouptick Joarder return err;
3467952be9bSHans Verkuil }
3477952be9bSHans Verkuil
3487952be9bSHans Verkuil /*
3497952be9bSHans Verkuil * Use common vm_area operations to track buffer refcount.
3507952be9bSHans Verkuil */
3517952be9bSHans Verkuil vma->vm_private_data = &buf->handler;
3527952be9bSHans Verkuil vma->vm_ops = &vb2_common_vm_ops;
3537952be9bSHans Verkuil
3547952be9bSHans Verkuil vma->vm_ops->open(vma);
3557952be9bSHans Verkuil
3567952be9bSHans Verkuil return 0;
3577952be9bSHans Verkuil }
3587952be9bSHans Verkuil
3597952be9bSHans Verkuil /*********************************************/
3607952be9bSHans Verkuil /* DMABUF ops for exporters */
3617952be9bSHans Verkuil /*********************************************/
3627952be9bSHans Verkuil
3637952be9bSHans Verkuil struct vb2_dma_sg_attachment {
3647952be9bSHans Verkuil struct sg_table sgt;
3657952be9bSHans Verkuil enum dma_data_direction dma_dir;
3667952be9bSHans Verkuil };
3677952be9bSHans Verkuil
vb2_dma_sg_dmabuf_ops_attach(struct dma_buf * dbuf,struct dma_buf_attachment * dbuf_attach)368a19741e5SChristian König static int vb2_dma_sg_dmabuf_ops_attach(struct dma_buf *dbuf,
3697952be9bSHans Verkuil struct dma_buf_attachment *dbuf_attach)
3707952be9bSHans Verkuil {
3717952be9bSHans Verkuil struct vb2_dma_sg_attachment *attach;
3727952be9bSHans Verkuil unsigned int i;
3737952be9bSHans Verkuil struct scatterlist *rd, *wr;
3747952be9bSHans Verkuil struct sg_table *sgt;
3757952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = dbuf->priv;
3767952be9bSHans Verkuil int ret;
3777952be9bSHans Verkuil
3787952be9bSHans Verkuil attach = kzalloc(sizeof(*attach), GFP_KERNEL);
3797952be9bSHans Verkuil if (!attach)
3807952be9bSHans Verkuil return -ENOMEM;
3817952be9bSHans Verkuil
3827952be9bSHans Verkuil sgt = &attach->sgt;
3837952be9bSHans Verkuil /* Copy the buf->base_sgt scatter list to the attachment, as we can't
3847952be9bSHans Verkuil * map the same scatter list to multiple attachments at the same time.
3857952be9bSHans Verkuil */
3867952be9bSHans Verkuil ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);
3877952be9bSHans Verkuil if (ret) {
3887952be9bSHans Verkuil kfree(attach);
3897952be9bSHans Verkuil return -ENOMEM;
3907952be9bSHans Verkuil }
3917952be9bSHans Verkuil
3927952be9bSHans Verkuil rd = buf->dma_sgt->sgl;
3937952be9bSHans Verkuil wr = sgt->sgl;
3947952be9bSHans Verkuil for (i = 0; i < sgt->orig_nents; ++i) {
3957952be9bSHans Verkuil sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
3967952be9bSHans Verkuil rd = sg_next(rd);
3977952be9bSHans Verkuil wr = sg_next(wr);
3987952be9bSHans Verkuil }
3997952be9bSHans Verkuil
4007952be9bSHans Verkuil attach->dma_dir = DMA_NONE;
4017952be9bSHans Verkuil dbuf_attach->priv = attach;
4027952be9bSHans Verkuil
4037952be9bSHans Verkuil return 0;
4047952be9bSHans Verkuil }
4057952be9bSHans Verkuil
vb2_dma_sg_dmabuf_ops_detach(struct dma_buf * dbuf,struct dma_buf_attachment * db_attach)4067952be9bSHans Verkuil static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf,
4077952be9bSHans Verkuil struct dma_buf_attachment *db_attach)
4087952be9bSHans Verkuil {
4097952be9bSHans Verkuil struct vb2_dma_sg_attachment *attach = db_attach->priv;
4107952be9bSHans Verkuil struct sg_table *sgt;
4117952be9bSHans Verkuil
4127952be9bSHans Verkuil if (!attach)
4137952be9bSHans Verkuil return;
4147952be9bSHans Verkuil
4157952be9bSHans Verkuil sgt = &attach->sgt;
4167952be9bSHans Verkuil
4177952be9bSHans Verkuil /* release the scatterlist cache */
4187952be9bSHans Verkuil if (attach->dma_dir != DMA_NONE)
4198b7c0280SMarek Szyprowski dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
4207952be9bSHans Verkuil sg_free_table(sgt);
4217952be9bSHans Verkuil kfree(attach);
4227952be9bSHans Verkuil db_attach->priv = NULL;
4237952be9bSHans Verkuil }
4247952be9bSHans Verkuil
vb2_dma_sg_dmabuf_ops_map(struct dma_buf_attachment * db_attach,enum dma_data_direction dma_dir)4257952be9bSHans Verkuil static struct sg_table *vb2_dma_sg_dmabuf_ops_map(
4267952be9bSHans Verkuil struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
4277952be9bSHans Verkuil {
4287952be9bSHans Verkuil struct vb2_dma_sg_attachment *attach = db_attach->priv;
4297952be9bSHans Verkuil struct sg_table *sgt;
4307952be9bSHans Verkuil
4317952be9bSHans Verkuil sgt = &attach->sgt;
4327952be9bSHans Verkuil /* return previously mapped sg table */
43323543b3cSDmitry Osipenko if (attach->dma_dir == dma_dir)
4347952be9bSHans Verkuil return sgt;
4357952be9bSHans Verkuil
4367952be9bSHans Verkuil /* release any previous cache */
4377952be9bSHans Verkuil if (attach->dma_dir != DMA_NONE) {
4388b7c0280SMarek Szyprowski dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
4397952be9bSHans Verkuil attach->dma_dir = DMA_NONE;
4407952be9bSHans Verkuil }
4417952be9bSHans Verkuil
4427952be9bSHans Verkuil /* mapping to the client with new direction */
4438b7c0280SMarek Szyprowski if (dma_map_sgtable(db_attach->dev, sgt, dma_dir, 0)) {
4447952be9bSHans Verkuil pr_err("failed to map scatterlist\n");
4457952be9bSHans Verkuil return ERR_PTR(-EIO);
4467952be9bSHans Verkuil }
4477952be9bSHans Verkuil
4487952be9bSHans Verkuil attach->dma_dir = dma_dir;
4497952be9bSHans Verkuil
4507952be9bSHans Verkuil return sgt;
4517952be9bSHans Verkuil }
4527952be9bSHans Verkuil
vb2_dma_sg_dmabuf_ops_unmap(struct dma_buf_attachment * db_attach,struct sg_table * sgt,enum dma_data_direction dma_dir)4537952be9bSHans Verkuil static void vb2_dma_sg_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
4547952be9bSHans Verkuil struct sg_table *sgt, enum dma_data_direction dma_dir)
4557952be9bSHans Verkuil {
4567952be9bSHans Verkuil /* nothing to be done here */
4577952be9bSHans Verkuil }
4587952be9bSHans Verkuil
vb2_dma_sg_dmabuf_ops_release(struct dma_buf * dbuf)4597952be9bSHans Verkuil static void vb2_dma_sg_dmabuf_ops_release(struct dma_buf *dbuf)
4607952be9bSHans Verkuil {
4617952be9bSHans Verkuil /* drop reference obtained in vb2_dma_sg_get_dmabuf */
4627952be9bSHans Verkuil vb2_dma_sg_put(dbuf->priv);
4637952be9bSHans Verkuil }
4647952be9bSHans Verkuil
465d4db5eb5SSergey Senozhatsky static int
vb2_dma_sg_dmabuf_ops_begin_cpu_access(struct dma_buf * dbuf,enum dma_data_direction direction)466d4db5eb5SSergey Senozhatsky vb2_dma_sg_dmabuf_ops_begin_cpu_access(struct dma_buf *dbuf,
467d4db5eb5SSergey Senozhatsky enum dma_data_direction direction)
468d4db5eb5SSergey Senozhatsky {
469d4db5eb5SSergey Senozhatsky struct vb2_dma_sg_buf *buf = dbuf->priv;
470d4db5eb5SSergey Senozhatsky struct sg_table *sgt = buf->dma_sgt;
471d4db5eb5SSergey Senozhatsky
472d4db5eb5SSergey Senozhatsky dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
473d4db5eb5SSergey Senozhatsky return 0;
474d4db5eb5SSergey Senozhatsky }
475d4db5eb5SSergey Senozhatsky
476d4db5eb5SSergey Senozhatsky static int
vb2_dma_sg_dmabuf_ops_end_cpu_access(struct dma_buf * dbuf,enum dma_data_direction direction)477d4db5eb5SSergey Senozhatsky vb2_dma_sg_dmabuf_ops_end_cpu_access(struct dma_buf *dbuf,
478d4db5eb5SSergey Senozhatsky enum dma_data_direction direction)
479d4db5eb5SSergey Senozhatsky {
480d4db5eb5SSergey Senozhatsky struct vb2_dma_sg_buf *buf = dbuf->priv;
481d4db5eb5SSergey Senozhatsky struct sg_table *sgt = buf->dma_sgt;
482d4db5eb5SSergey Senozhatsky
483d4db5eb5SSergey Senozhatsky dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
484d4db5eb5SSergey Senozhatsky return 0;
485d4db5eb5SSergey Senozhatsky }
486d4db5eb5SSergey Senozhatsky
vb2_dma_sg_dmabuf_ops_vmap(struct dma_buf * dbuf,struct iosys_map * map)4877938f421SLucas De Marchi static int vb2_dma_sg_dmabuf_ops_vmap(struct dma_buf *dbuf,
4887938f421SLucas De Marchi struct iosys_map *map)
4897952be9bSHans Verkuil {
490*72a28c92SMichael Grzeschik struct vb2_dma_sg_buf *buf;
491*72a28c92SMichael Grzeschik void *vaddr;
4927952be9bSHans Verkuil
493*72a28c92SMichael Grzeschik buf = dbuf->priv;
494*72a28c92SMichael Grzeschik vaddr = vb2_dma_sg_vaddr(buf->vb, buf);
495*72a28c92SMichael Grzeschik if (!vaddr)
496*72a28c92SMichael Grzeschik return -EINVAL;
497*72a28c92SMichael Grzeschik
498*72a28c92SMichael Grzeschik iosys_map_set_vaddr(map, vaddr);
4996619ccf1SThomas Zimmermann
5006619ccf1SThomas Zimmermann return 0;
5017952be9bSHans Verkuil }
5027952be9bSHans Verkuil
vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf * dbuf,struct vm_area_struct * vma)5037952be9bSHans Verkuil static int vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf *dbuf,
5047952be9bSHans Verkuil struct vm_area_struct *vma)
5057952be9bSHans Verkuil {
5067952be9bSHans Verkuil return vb2_dma_sg_mmap(dbuf->priv, vma);
5077952be9bSHans Verkuil }
5087952be9bSHans Verkuil
5097952be9bSHans Verkuil static const struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
5107952be9bSHans Verkuil .attach = vb2_dma_sg_dmabuf_ops_attach,
5117952be9bSHans Verkuil .detach = vb2_dma_sg_dmabuf_ops_detach,
5127952be9bSHans Verkuil .map_dma_buf = vb2_dma_sg_dmabuf_ops_map,
5137952be9bSHans Verkuil .unmap_dma_buf = vb2_dma_sg_dmabuf_ops_unmap,
514d4db5eb5SSergey Senozhatsky .begin_cpu_access = vb2_dma_sg_dmabuf_ops_begin_cpu_access,
515d4db5eb5SSergey Senozhatsky .end_cpu_access = vb2_dma_sg_dmabuf_ops_end_cpu_access,
5167952be9bSHans Verkuil .vmap = vb2_dma_sg_dmabuf_ops_vmap,
5177952be9bSHans Verkuil .mmap = vb2_dma_sg_dmabuf_ops_mmap,
5187952be9bSHans Verkuil .release = vb2_dma_sg_dmabuf_ops_release,
5197952be9bSHans Verkuil };
5207952be9bSHans Verkuil
vb2_dma_sg_get_dmabuf(struct vb2_buffer * vb,void * buf_priv,unsigned long flags)521a4b83debSSergey Senozhatsky static struct dma_buf *vb2_dma_sg_get_dmabuf(struct vb2_buffer *vb,
522a4b83debSSergey Senozhatsky void *buf_priv,
523a4b83debSSergey Senozhatsky unsigned long flags)
5247952be9bSHans Verkuil {
5257952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
5267952be9bSHans Verkuil struct dma_buf *dbuf;
5277952be9bSHans Verkuil DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
5287952be9bSHans Verkuil
5297952be9bSHans Verkuil exp_info.ops = &vb2_dma_sg_dmabuf_ops;
5307952be9bSHans Verkuil exp_info.size = buf->size;
5317952be9bSHans Verkuil exp_info.flags = flags;
5327952be9bSHans Verkuil exp_info.priv = buf;
5337952be9bSHans Verkuil
5347952be9bSHans Verkuil if (WARN_ON(!buf->dma_sgt))
5357952be9bSHans Verkuil return NULL;
5367952be9bSHans Verkuil
5377952be9bSHans Verkuil dbuf = dma_buf_export(&exp_info);
5387952be9bSHans Verkuil if (IS_ERR(dbuf))
5397952be9bSHans Verkuil return NULL;
5407952be9bSHans Verkuil
5417952be9bSHans Verkuil /* dmabuf keeps reference to vb2 buffer */
5427952be9bSHans Verkuil refcount_inc(&buf->refcount);
5437952be9bSHans Verkuil
5447952be9bSHans Verkuil return dbuf;
5457952be9bSHans Verkuil }
5467952be9bSHans Verkuil
5477952be9bSHans Verkuil /*********************************************/
5487952be9bSHans Verkuil /* callbacks for DMABUF buffers */
5497952be9bSHans Verkuil /*********************************************/
5507952be9bSHans Verkuil
vb2_dma_sg_map_dmabuf(void * mem_priv)5517952be9bSHans Verkuil static int vb2_dma_sg_map_dmabuf(void *mem_priv)
5527952be9bSHans Verkuil {
5537952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = mem_priv;
5547952be9bSHans Verkuil struct sg_table *sgt;
5557952be9bSHans Verkuil
5567952be9bSHans Verkuil if (WARN_ON(!buf->db_attach)) {
5577952be9bSHans Verkuil pr_err("trying to pin a non attached buffer\n");
5587952be9bSHans Verkuil return -EINVAL;
5597952be9bSHans Verkuil }
5607952be9bSHans Verkuil
5617952be9bSHans Verkuil if (WARN_ON(buf->dma_sgt)) {
5627952be9bSHans Verkuil pr_err("dmabuf buffer is already pinned\n");
5637952be9bSHans Verkuil return 0;
5647952be9bSHans Verkuil }
5657952be9bSHans Verkuil
5667952be9bSHans Verkuil /* get the associated scatterlist for this buffer */
567a26ee3b7SDmitry Osipenko sgt = dma_buf_map_attachment_unlocked(buf->db_attach, buf->dma_dir);
5687952be9bSHans Verkuil if (IS_ERR(sgt)) {
5697952be9bSHans Verkuil pr_err("Error getting dmabuf scatterlist\n");
5707952be9bSHans Verkuil return -EINVAL;
5717952be9bSHans Verkuil }
5727952be9bSHans Verkuil
5737952be9bSHans Verkuil buf->dma_sgt = sgt;
5747952be9bSHans Verkuil buf->vaddr = NULL;
5757952be9bSHans Verkuil
5767952be9bSHans Verkuil return 0;
5777952be9bSHans Verkuil }
5787952be9bSHans Verkuil
vb2_dma_sg_unmap_dmabuf(void * mem_priv)5797952be9bSHans Verkuil static void vb2_dma_sg_unmap_dmabuf(void *mem_priv)
5807952be9bSHans Verkuil {
5817952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = mem_priv;
5827952be9bSHans Verkuil struct sg_table *sgt = buf->dma_sgt;
5837938f421SLucas De Marchi struct iosys_map map = IOSYS_MAP_INIT_VADDR(buf->vaddr);
5847952be9bSHans Verkuil
5857952be9bSHans Verkuil if (WARN_ON(!buf->db_attach)) {
5867952be9bSHans Verkuil pr_err("trying to unpin a not attached buffer\n");
5877952be9bSHans Verkuil return;
5887952be9bSHans Verkuil }
5897952be9bSHans Verkuil
5907952be9bSHans Verkuil if (WARN_ON(!sgt)) {
5917952be9bSHans Verkuil pr_err("dmabuf buffer is already unpinned\n");
5927952be9bSHans Verkuil return;
5937952be9bSHans Verkuil }
5947952be9bSHans Verkuil
5957952be9bSHans Verkuil if (buf->vaddr) {
596a26ee3b7SDmitry Osipenko dma_buf_vunmap_unlocked(buf->db_attach->dmabuf, &map);
5977952be9bSHans Verkuil buf->vaddr = NULL;
5987952be9bSHans Verkuil }
599a26ee3b7SDmitry Osipenko dma_buf_unmap_attachment_unlocked(buf->db_attach, sgt, buf->dma_dir);
6007952be9bSHans Verkuil
6017952be9bSHans Verkuil buf->dma_sgt = NULL;
6027952be9bSHans Verkuil }
6037952be9bSHans Verkuil
vb2_dma_sg_detach_dmabuf(void * mem_priv)6047952be9bSHans Verkuil static void vb2_dma_sg_detach_dmabuf(void *mem_priv)
6057952be9bSHans Verkuil {
6067952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = mem_priv;
6077952be9bSHans Verkuil
6087952be9bSHans Verkuil /* if vb2 works correctly you should never detach mapped buffer */
6097952be9bSHans Verkuil if (WARN_ON(buf->dma_sgt))
6107952be9bSHans Verkuil vb2_dma_sg_unmap_dmabuf(buf);
6117952be9bSHans Verkuil
6127952be9bSHans Verkuil /* detach this attachment */
6137952be9bSHans Verkuil dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);
6147952be9bSHans Verkuil kfree(buf);
6157952be9bSHans Verkuil }
6167952be9bSHans Verkuil
vb2_dma_sg_attach_dmabuf(struct vb2_buffer * vb,struct device * dev,struct dma_buf * dbuf,unsigned long size)617a4b83debSSergey Senozhatsky static void *vb2_dma_sg_attach_dmabuf(struct vb2_buffer *vb, struct device *dev,
618a4b83debSSergey Senozhatsky struct dma_buf *dbuf, unsigned long size)
6197952be9bSHans Verkuil {
6207952be9bSHans Verkuil struct vb2_dma_sg_buf *buf;
6217952be9bSHans Verkuil struct dma_buf_attachment *dba;
6227952be9bSHans Verkuil
6237952be9bSHans Verkuil if (WARN_ON(!dev))
6247952be9bSHans Verkuil return ERR_PTR(-EINVAL);
6257952be9bSHans Verkuil
6267952be9bSHans Verkuil if (dbuf->size < size)
6277952be9bSHans Verkuil return ERR_PTR(-EFAULT);
6287952be9bSHans Verkuil
6297952be9bSHans Verkuil buf = kzalloc(sizeof(*buf), GFP_KERNEL);
6307952be9bSHans Verkuil if (!buf)
6317952be9bSHans Verkuil return ERR_PTR(-ENOMEM);
6327952be9bSHans Verkuil
6337952be9bSHans Verkuil buf->dev = dev;
6347952be9bSHans Verkuil /* create attachment for the dmabuf with the user device */
6357952be9bSHans Verkuil dba = dma_buf_attach(dbuf, buf->dev);
6367952be9bSHans Verkuil if (IS_ERR(dba)) {
6377952be9bSHans Verkuil pr_err("failed to attach dmabuf\n");
6387952be9bSHans Verkuil kfree(buf);
6397952be9bSHans Verkuil return dba;
6407952be9bSHans Verkuil }
6417952be9bSHans Verkuil
642a4b83debSSergey Senozhatsky buf->dma_dir = vb->vb2_queue->dma_dir;
6437952be9bSHans Verkuil buf->size = size;
6447952be9bSHans Verkuil buf->db_attach = dba;
645d55c3ee6SHans de Goede buf->vb = vb;
6467952be9bSHans Verkuil
6477952be9bSHans Verkuil return buf;
6487952be9bSHans Verkuil }
6497952be9bSHans Verkuil
vb2_dma_sg_cookie(struct vb2_buffer * vb,void * buf_priv)650a4b83debSSergey Senozhatsky static void *vb2_dma_sg_cookie(struct vb2_buffer *vb, void *buf_priv)
6517952be9bSHans Verkuil {
6527952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv;
6537952be9bSHans Verkuil
6547952be9bSHans Verkuil return buf->dma_sgt;
6557952be9bSHans Verkuil }
6567952be9bSHans Verkuil
6577952be9bSHans Verkuil const struct vb2_mem_ops vb2_dma_sg_memops = {
6587952be9bSHans Verkuil .alloc = vb2_dma_sg_alloc,
6597952be9bSHans Verkuil .put = vb2_dma_sg_put,
6607952be9bSHans Verkuil .get_userptr = vb2_dma_sg_get_userptr,
6617952be9bSHans Verkuil .put_userptr = vb2_dma_sg_put_userptr,
6627952be9bSHans Verkuil .prepare = vb2_dma_sg_prepare,
6637952be9bSHans Verkuil .finish = vb2_dma_sg_finish,
6647952be9bSHans Verkuil .vaddr = vb2_dma_sg_vaddr,
6657952be9bSHans Verkuil .mmap = vb2_dma_sg_mmap,
6667952be9bSHans Verkuil .num_users = vb2_dma_sg_num_users,
6677952be9bSHans Verkuil .get_dmabuf = vb2_dma_sg_get_dmabuf,
6687952be9bSHans Verkuil .map_dmabuf = vb2_dma_sg_map_dmabuf,
6697952be9bSHans Verkuil .unmap_dmabuf = vb2_dma_sg_unmap_dmabuf,
6707952be9bSHans Verkuil .attach_dmabuf = vb2_dma_sg_attach_dmabuf,
6717952be9bSHans Verkuil .detach_dmabuf = vb2_dma_sg_detach_dmabuf,
6727952be9bSHans Verkuil .cookie = vb2_dma_sg_cookie,
6737952be9bSHans Verkuil };
6747952be9bSHans Verkuil EXPORT_SYMBOL_GPL(vb2_dma_sg_memops);
6757952be9bSHans Verkuil
6767952be9bSHans Verkuil MODULE_DESCRIPTION("dma scatter/gather memory handling routines for videobuf2");
6777952be9bSHans Verkuil MODULE_AUTHOR("Andrzej Pietrasiewicz");
6787952be9bSHans Verkuil MODULE_LICENSE("GPL");
67916b0314aSGreg Kroah-Hartman MODULE_IMPORT_NS(DMA_BUF);
680