17952be9bSHans Verkuil /* 27952be9bSHans Verkuil * videobuf2-dma-sg.c - dma scatter/gather memory allocator for videobuf2 37952be9bSHans Verkuil * 47952be9bSHans Verkuil * Copyright (C) 2010 Samsung Electronics 57952be9bSHans Verkuil * 666e988e9SAndrzej Pietrasiewicz * Author: Andrzej Pietrasiewicz <andrzejtp2010@gmail.com> 77952be9bSHans Verkuil * 87952be9bSHans Verkuil * This program is free software; you can redistribute it and/or modify 97952be9bSHans Verkuil * it under the terms of the GNU General Public License as published by 107952be9bSHans Verkuil * the Free Software Foundation. 117952be9bSHans Verkuil */ 127952be9bSHans Verkuil 137952be9bSHans Verkuil #include <linux/module.h> 147952be9bSHans Verkuil #include <linux/mm.h> 157952be9bSHans Verkuil #include <linux/refcount.h> 167952be9bSHans Verkuil #include <linux/scatterlist.h> 177952be9bSHans Verkuil #include <linux/sched.h> 187952be9bSHans Verkuil #include <linux/slab.h> 197952be9bSHans Verkuil #include <linux/vmalloc.h> 207952be9bSHans Verkuil 217952be9bSHans Verkuil #include <media/videobuf2-v4l2.h> 227952be9bSHans Verkuil #include <media/videobuf2-memops.h> 237952be9bSHans Verkuil #include <media/videobuf2-dma-sg.h> 247952be9bSHans Verkuil 257952be9bSHans Verkuil static int debug; 267952be9bSHans Verkuil module_param(debug, int, 0644); 277952be9bSHans Verkuil 287952be9bSHans Verkuil #define dprintk(level, fmt, arg...) \ 297952be9bSHans Verkuil do { \ 307952be9bSHans Verkuil if (debug >= level) \ 317952be9bSHans Verkuil printk(KERN_DEBUG "vb2-dma-sg: " fmt, ## arg); \ 327952be9bSHans Verkuil } while (0) 337952be9bSHans Verkuil 347952be9bSHans Verkuil struct vb2_dma_sg_buf { 357952be9bSHans Verkuil struct device *dev; 367952be9bSHans Verkuil void *vaddr; 377952be9bSHans Verkuil struct page **pages; 387952be9bSHans Verkuil struct frame_vector *vec; 397952be9bSHans Verkuil int offset; 407952be9bSHans Verkuil enum dma_data_direction dma_dir; 417952be9bSHans Verkuil struct sg_table sg_table; 427952be9bSHans Verkuil /* 437952be9bSHans Verkuil * This will point to sg_table when used with the MMAP or USERPTR 447952be9bSHans Verkuil * memory model, and to the dma_buf sglist when used with the 457952be9bSHans Verkuil * DMABUF memory model. 467952be9bSHans Verkuil */ 477952be9bSHans Verkuil struct sg_table *dma_sgt; 487952be9bSHans Verkuil size_t size; 497952be9bSHans Verkuil unsigned int num_pages; 507952be9bSHans Verkuil refcount_t refcount; 517952be9bSHans Verkuil struct vb2_vmarea_handler handler; 527952be9bSHans Verkuil 537952be9bSHans Verkuil struct dma_buf_attachment *db_attach; 54a4b83debSSergey Senozhatsky 55a4b83debSSergey Senozhatsky struct vb2_buffer *vb; 567952be9bSHans Verkuil }; 577952be9bSHans Verkuil 587952be9bSHans Verkuil static void vb2_dma_sg_put(void *buf_priv); 597952be9bSHans Verkuil 607952be9bSHans Verkuil static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf, 617952be9bSHans Verkuil gfp_t gfp_flags) 627952be9bSHans Verkuil { 637952be9bSHans Verkuil unsigned int last_page = 0; 6414f28f5cSSakari Ailus unsigned long size = buf->size; 657952be9bSHans Verkuil 667952be9bSHans Verkuil while (size > 0) { 677952be9bSHans Verkuil struct page *pages; 687952be9bSHans Verkuil int order; 697952be9bSHans Verkuil int i; 707952be9bSHans Verkuil 717952be9bSHans Verkuil order = get_order(size); 724b129dc9SMauro Carvalho Chehab /* Don't over allocate*/ 737952be9bSHans Verkuil if ((PAGE_SIZE << order) > size) 747952be9bSHans Verkuil order--; 757952be9bSHans Verkuil 767952be9bSHans Verkuil pages = NULL; 777952be9bSHans Verkuil while (!pages) { 787952be9bSHans Verkuil pages = alloc_pages(GFP_KERNEL | __GFP_ZERO | 797952be9bSHans Verkuil __GFP_NOWARN | gfp_flags, order); 807952be9bSHans Verkuil if (pages) 817952be9bSHans Verkuil break; 827952be9bSHans Verkuil 837952be9bSHans Verkuil if (order == 0) { 847952be9bSHans Verkuil while (last_page--) 857952be9bSHans Verkuil __free_page(buf->pages[last_page]); 867952be9bSHans Verkuil return -ENOMEM; 877952be9bSHans Verkuil } 887952be9bSHans Verkuil order--; 897952be9bSHans Verkuil } 907952be9bSHans Verkuil 917952be9bSHans Verkuil split_page(pages, order); 927952be9bSHans Verkuil for (i = 0; i < (1 << order); i++) 937952be9bSHans Verkuil buf->pages[last_page++] = &pages[i]; 947952be9bSHans Verkuil 957952be9bSHans Verkuil size -= PAGE_SIZE << order; 967952be9bSHans Verkuil } 977952be9bSHans Verkuil 987952be9bSHans Verkuil return 0; 997952be9bSHans Verkuil } 1007952be9bSHans Verkuil 101a4b83debSSergey Senozhatsky static void *vb2_dma_sg_alloc(struct vb2_buffer *vb, struct device *dev, 102a4b83debSSergey Senozhatsky unsigned long size) 1037952be9bSHans Verkuil { 1047952be9bSHans Verkuil struct vb2_dma_sg_buf *buf; 1057952be9bSHans Verkuil struct sg_table *sgt; 1067952be9bSHans Verkuil int ret; 1077952be9bSHans Verkuil int num_pages; 1087952be9bSHans Verkuil 1099cc25c4bSRicardo Ribalda if (WARN_ON(!dev) || WARN_ON(!size)) 1107952be9bSHans Verkuil return ERR_PTR(-EINVAL); 1117952be9bSHans Verkuil 1127952be9bSHans Verkuil buf = kzalloc(sizeof *buf, GFP_KERNEL); 1137952be9bSHans Verkuil if (!buf) 1147952be9bSHans Verkuil return ERR_PTR(-ENOMEM); 1157952be9bSHans Verkuil 1167952be9bSHans Verkuil buf->vaddr = NULL; 117a4b83debSSergey Senozhatsky buf->dma_dir = vb->vb2_queue->dma_dir; 1187952be9bSHans Verkuil buf->offset = 0; 1197952be9bSHans Verkuil buf->size = size; 1207952be9bSHans Verkuil /* size is already page aligned */ 1217952be9bSHans Verkuil buf->num_pages = size >> PAGE_SHIFT; 1227952be9bSHans Verkuil buf->dma_sgt = &buf->sg_table; 1237952be9bSHans Verkuil 124d4db5eb5SSergey Senozhatsky /* 125d4db5eb5SSergey Senozhatsky * NOTE: dma-sg allocates memory using the page allocator directly, so 126d4db5eb5SSergey Senozhatsky * there is no memory consistency guarantee, hence dma-sg ignores DMA 127129134e5SSergey Senozhatsky * attributes passed from the upper layer. 128d4db5eb5SSergey Senozhatsky */ 129d5a8099cSHans Verkuil buf->pages = kvcalloc(buf->num_pages, sizeof(struct page *), GFP_KERNEL); 1307952be9bSHans Verkuil if (!buf->pages) 1317952be9bSHans Verkuil goto fail_pages_array_alloc; 1327952be9bSHans Verkuil 133a4b83debSSergey Senozhatsky ret = vb2_dma_sg_alloc_compacted(buf, vb->vb2_queue->gfp_flags); 1347952be9bSHans Verkuil if (ret) 1357952be9bSHans Verkuil goto fail_pages_alloc; 1367952be9bSHans Verkuil 1377952be9bSHans Verkuil ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages, 1387952be9bSHans Verkuil buf->num_pages, 0, size, GFP_KERNEL); 1397952be9bSHans Verkuil if (ret) 1407952be9bSHans Verkuil goto fail_table_alloc; 1417952be9bSHans Verkuil 1427952be9bSHans Verkuil /* Prevent the device from being released while the buffer is used */ 1437952be9bSHans Verkuil buf->dev = get_device(dev); 1447952be9bSHans Verkuil 1457952be9bSHans Verkuil sgt = &buf->sg_table; 1467952be9bSHans Verkuil /* 1477952be9bSHans Verkuil * No need to sync to the device, this will happen later when the 1487952be9bSHans Verkuil * prepare() memop is called. 1497952be9bSHans Verkuil */ 1508b7c0280SMarek Szyprowski if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir, 1518b7c0280SMarek Szyprowski DMA_ATTR_SKIP_CPU_SYNC)) 1527952be9bSHans Verkuil goto fail_map; 1537952be9bSHans Verkuil 1547952be9bSHans Verkuil buf->handler.refcount = &buf->refcount; 1557952be9bSHans Verkuil buf->handler.put = vb2_dma_sg_put; 1567952be9bSHans Verkuil buf->handler.arg = buf; 157a4b83debSSergey Senozhatsky buf->vb = vb; 1587952be9bSHans Verkuil 1597952be9bSHans Verkuil refcount_set(&buf->refcount, 1); 1607952be9bSHans Verkuil 1617952be9bSHans Verkuil dprintk(1, "%s: Allocated buffer of %d pages\n", 1627952be9bSHans Verkuil __func__, buf->num_pages); 1637952be9bSHans Verkuil return buf; 1647952be9bSHans Verkuil 1657952be9bSHans Verkuil fail_map: 1667952be9bSHans Verkuil put_device(buf->dev); 1677952be9bSHans Verkuil sg_free_table(buf->dma_sgt); 1687952be9bSHans Verkuil fail_table_alloc: 1697952be9bSHans Verkuil num_pages = buf->num_pages; 1707952be9bSHans Verkuil while (num_pages--) 1717952be9bSHans Verkuil __free_page(buf->pages[num_pages]); 1727952be9bSHans Verkuil fail_pages_alloc: 1737952be9bSHans Verkuil kvfree(buf->pages); 1747952be9bSHans Verkuil fail_pages_array_alloc: 1757952be9bSHans Verkuil kfree(buf); 1767952be9bSHans Verkuil return ERR_PTR(-ENOMEM); 1777952be9bSHans Verkuil } 1787952be9bSHans Verkuil 1797952be9bSHans Verkuil static void vb2_dma_sg_put(void *buf_priv) 1807952be9bSHans Verkuil { 1817952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 1827952be9bSHans Verkuil struct sg_table *sgt = &buf->sg_table; 1837952be9bSHans Verkuil int i = buf->num_pages; 1847952be9bSHans Verkuil 1857952be9bSHans Verkuil if (refcount_dec_and_test(&buf->refcount)) { 1867952be9bSHans Verkuil dprintk(1, "%s: Freeing buffer of %d pages\n", __func__, 1877952be9bSHans Verkuil buf->num_pages); 1888b7c0280SMarek Szyprowski dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, 1898b7c0280SMarek Szyprowski DMA_ATTR_SKIP_CPU_SYNC); 1907952be9bSHans Verkuil if (buf->vaddr) 1917952be9bSHans Verkuil vm_unmap_ram(buf->vaddr, buf->num_pages); 1927952be9bSHans Verkuil sg_free_table(buf->dma_sgt); 1937952be9bSHans Verkuil while (--i >= 0) 1947952be9bSHans Verkuil __free_page(buf->pages[i]); 1957952be9bSHans Verkuil kvfree(buf->pages); 1967952be9bSHans Verkuil put_device(buf->dev); 1977952be9bSHans Verkuil kfree(buf); 1987952be9bSHans Verkuil } 1997952be9bSHans Verkuil } 2007952be9bSHans Verkuil 2017952be9bSHans Verkuil static void vb2_dma_sg_prepare(void *buf_priv) 2027952be9bSHans Verkuil { 2037952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 2047952be9bSHans Verkuil struct sg_table *sgt = buf->dma_sgt; 2057952be9bSHans Verkuil 206cde513fdSSergey Senozhatsky if (buf->vb->skip_cache_sync_on_prepare) 207cde513fdSSergey Senozhatsky return; 208cde513fdSSergey Senozhatsky 2098b7c0280SMarek Szyprowski dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir); 2107952be9bSHans Verkuil } 2117952be9bSHans Verkuil 2127952be9bSHans Verkuil static void vb2_dma_sg_finish(void *buf_priv) 2137952be9bSHans Verkuil { 2147952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 2157952be9bSHans Verkuil struct sg_table *sgt = buf->dma_sgt; 2167952be9bSHans Verkuil 217cde513fdSSergey Senozhatsky if (buf->vb->skip_cache_sync_on_finish) 218cde513fdSSergey Senozhatsky return; 219cde513fdSSergey Senozhatsky 2208b7c0280SMarek Szyprowski dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir); 2217952be9bSHans Verkuil } 2227952be9bSHans Verkuil 223a4b83debSSergey Senozhatsky static void *vb2_dma_sg_get_userptr(struct vb2_buffer *vb, struct device *dev, 224a4b83debSSergey Senozhatsky unsigned long vaddr, unsigned long size) 2257952be9bSHans Verkuil { 2267952be9bSHans Verkuil struct vb2_dma_sg_buf *buf; 2277952be9bSHans Verkuil struct sg_table *sgt; 2287952be9bSHans Verkuil struct frame_vector *vec; 2297952be9bSHans Verkuil 2307952be9bSHans Verkuil if (WARN_ON(!dev)) 2317952be9bSHans Verkuil return ERR_PTR(-EINVAL); 2327952be9bSHans Verkuil 2337952be9bSHans Verkuil buf = kzalloc(sizeof *buf, GFP_KERNEL); 2347952be9bSHans Verkuil if (!buf) 2357952be9bSHans Verkuil return ERR_PTR(-ENOMEM); 2367952be9bSHans Verkuil 2377952be9bSHans Verkuil buf->vaddr = NULL; 2387952be9bSHans Verkuil buf->dev = dev; 239a4b83debSSergey Senozhatsky buf->dma_dir = vb->vb2_queue->dma_dir; 2407952be9bSHans Verkuil buf->offset = vaddr & ~PAGE_MASK; 2417952be9bSHans Verkuil buf->size = size; 2427952be9bSHans Verkuil buf->dma_sgt = &buf->sg_table; 243d55c3ee6SHans de Goede buf->vb = vb; 24470794724SHans Verkuil vec = vb2_create_framevec(vaddr, size); 2457952be9bSHans Verkuil if (IS_ERR(vec)) 2467952be9bSHans Verkuil goto userptr_fail_pfnvec; 2477952be9bSHans Verkuil buf->vec = vec; 2487952be9bSHans Verkuil 2497952be9bSHans Verkuil buf->pages = frame_vector_pages(vec); 2507952be9bSHans Verkuil if (IS_ERR(buf->pages)) 2517952be9bSHans Verkuil goto userptr_fail_sgtable; 2527952be9bSHans Verkuil buf->num_pages = frame_vector_count(vec); 2537952be9bSHans Verkuil 2547952be9bSHans Verkuil if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages, 2557952be9bSHans Verkuil buf->num_pages, buf->offset, size, 0)) 2567952be9bSHans Verkuil goto userptr_fail_sgtable; 2577952be9bSHans Verkuil 2587952be9bSHans Verkuil sgt = &buf->sg_table; 2597952be9bSHans Verkuil /* 2607952be9bSHans Verkuil * No need to sync to the device, this will happen later when the 2617952be9bSHans Verkuil * prepare() memop is called. 2627952be9bSHans Verkuil */ 2638b7c0280SMarek Szyprowski if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir, 2648b7c0280SMarek Szyprowski DMA_ATTR_SKIP_CPU_SYNC)) 2657952be9bSHans Verkuil goto userptr_fail_map; 2667952be9bSHans Verkuil 2677952be9bSHans Verkuil return buf; 2687952be9bSHans Verkuil 2697952be9bSHans Verkuil userptr_fail_map: 2707952be9bSHans Verkuil sg_free_table(&buf->sg_table); 2717952be9bSHans Verkuil userptr_fail_sgtable: 2727952be9bSHans Verkuil vb2_destroy_framevec(vec); 2737952be9bSHans Verkuil userptr_fail_pfnvec: 2747952be9bSHans Verkuil kfree(buf); 2757952be9bSHans Verkuil return ERR_PTR(-ENOMEM); 2767952be9bSHans Verkuil } 2777952be9bSHans Verkuil 2787952be9bSHans Verkuil /* 2797952be9bSHans Verkuil * @put_userptr: inform the allocator that a USERPTR buffer will no longer 2807952be9bSHans Verkuil * be used 2817952be9bSHans Verkuil */ 2827952be9bSHans Verkuil static void vb2_dma_sg_put_userptr(void *buf_priv) 2837952be9bSHans Verkuil { 2847952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 2857952be9bSHans Verkuil struct sg_table *sgt = &buf->sg_table; 2867952be9bSHans Verkuil int i = buf->num_pages; 2877952be9bSHans Verkuil 2887952be9bSHans Verkuil dprintk(1, "%s: Releasing userspace buffer of %d pages\n", 2897952be9bSHans Verkuil __func__, buf->num_pages); 2908b7c0280SMarek Szyprowski dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); 2917952be9bSHans Verkuil if (buf->vaddr) 2927952be9bSHans Verkuil vm_unmap_ram(buf->vaddr, buf->num_pages); 2937952be9bSHans Verkuil sg_free_table(buf->dma_sgt); 2947952be9bSHans Verkuil if (buf->dma_dir == DMA_FROM_DEVICE || 2957952be9bSHans Verkuil buf->dma_dir == DMA_BIDIRECTIONAL) 2967952be9bSHans Verkuil while (--i >= 0) 2977952be9bSHans Verkuil set_page_dirty_lock(buf->pages[i]); 2987952be9bSHans Verkuil vb2_destroy_framevec(buf->vec); 2997952be9bSHans Verkuil kfree(buf); 3007952be9bSHans Verkuil } 3017952be9bSHans Verkuil 302a4b83debSSergey Senozhatsky static void *vb2_dma_sg_vaddr(struct vb2_buffer *vb, void *buf_priv) 3037952be9bSHans Verkuil { 3047952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 3057938f421SLucas De Marchi struct iosys_map map; 3066619ccf1SThomas Zimmermann int ret; 3077952be9bSHans Verkuil 3087952be9bSHans Verkuil BUG_ON(!buf); 3097952be9bSHans Verkuil 3107952be9bSHans Verkuil if (!buf->vaddr) { 3116619ccf1SThomas Zimmermann if (buf->db_attach) { 312a26ee3b7SDmitry Osipenko ret = dma_buf_vmap_unlocked(buf->db_attach->dmabuf, &map); 3136619ccf1SThomas Zimmermann buf->vaddr = ret ? NULL : map.vaddr; 3146619ccf1SThomas Zimmermann } else { 315d4efd79aSChristoph Hellwig buf->vaddr = vm_map_ram(buf->pages, buf->num_pages, -1); 3167952be9bSHans Verkuil } 3176619ccf1SThomas Zimmermann } 3187952be9bSHans Verkuil 3197952be9bSHans Verkuil /* add offset in case userptr is not page-aligned */ 3207952be9bSHans Verkuil return buf->vaddr ? buf->vaddr + buf->offset : NULL; 3217952be9bSHans Verkuil } 3227952be9bSHans Verkuil 3237952be9bSHans Verkuil static unsigned int vb2_dma_sg_num_users(void *buf_priv) 3247952be9bSHans Verkuil { 3257952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 3267952be9bSHans Verkuil 3277952be9bSHans Verkuil return refcount_read(&buf->refcount); 3287952be9bSHans Verkuil } 3297952be9bSHans Verkuil 3307952be9bSHans Verkuil static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma) 3317952be9bSHans Verkuil { 3327952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 333a17ae147SSouptick Joarder int err; 3347952be9bSHans Verkuil 3357952be9bSHans Verkuil if (!buf) { 3367952be9bSHans Verkuil printk(KERN_ERR "No memory to map\n"); 3377952be9bSHans Verkuil return -EINVAL; 3387952be9bSHans Verkuil } 3397952be9bSHans Verkuil 340a17ae147SSouptick Joarder err = vm_map_pages(vma, buf->pages, buf->num_pages); 341a17ae147SSouptick Joarder if (err) { 342a17ae147SSouptick Joarder printk(KERN_ERR "Remapping memory, error: %d\n", err); 343a17ae147SSouptick Joarder return err; 3447952be9bSHans Verkuil } 3457952be9bSHans Verkuil 3467952be9bSHans Verkuil /* 3477952be9bSHans Verkuil * Use common vm_area operations to track buffer refcount. 3487952be9bSHans Verkuil */ 3497952be9bSHans Verkuil vma->vm_private_data = &buf->handler; 3507952be9bSHans Verkuil vma->vm_ops = &vb2_common_vm_ops; 3517952be9bSHans Verkuil 3527952be9bSHans Verkuil vma->vm_ops->open(vma); 3537952be9bSHans Verkuil 3547952be9bSHans Verkuil return 0; 3557952be9bSHans Verkuil } 3567952be9bSHans Verkuil 3577952be9bSHans Verkuil /*********************************************/ 3587952be9bSHans Verkuil /* DMABUF ops for exporters */ 3597952be9bSHans Verkuil /*********************************************/ 3607952be9bSHans Verkuil 3617952be9bSHans Verkuil struct vb2_dma_sg_attachment { 3627952be9bSHans Verkuil struct sg_table sgt; 3637952be9bSHans Verkuil enum dma_data_direction dma_dir; 3647952be9bSHans Verkuil }; 3657952be9bSHans Verkuil 366a19741e5SChristian König static int vb2_dma_sg_dmabuf_ops_attach(struct dma_buf *dbuf, 3677952be9bSHans Verkuil struct dma_buf_attachment *dbuf_attach) 3687952be9bSHans Verkuil { 3697952be9bSHans Verkuil struct vb2_dma_sg_attachment *attach; 3707952be9bSHans Verkuil unsigned int i; 3717952be9bSHans Verkuil struct scatterlist *rd, *wr; 3727952be9bSHans Verkuil struct sg_table *sgt; 3737952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = dbuf->priv; 3747952be9bSHans Verkuil int ret; 3757952be9bSHans Verkuil 3767952be9bSHans Verkuil attach = kzalloc(sizeof(*attach), GFP_KERNEL); 3777952be9bSHans Verkuil if (!attach) 3787952be9bSHans Verkuil return -ENOMEM; 3797952be9bSHans Verkuil 3807952be9bSHans Verkuil sgt = &attach->sgt; 3817952be9bSHans Verkuil /* Copy the buf->base_sgt scatter list to the attachment, as we can't 3827952be9bSHans Verkuil * map the same scatter list to multiple attachments at the same time. 3837952be9bSHans Verkuil */ 3847952be9bSHans Verkuil ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL); 3857952be9bSHans Verkuil if (ret) { 3867952be9bSHans Verkuil kfree(attach); 3877952be9bSHans Verkuil return -ENOMEM; 3887952be9bSHans Verkuil } 3897952be9bSHans Verkuil 3907952be9bSHans Verkuil rd = buf->dma_sgt->sgl; 3917952be9bSHans Verkuil wr = sgt->sgl; 3927952be9bSHans Verkuil for (i = 0; i < sgt->orig_nents; ++i) { 3937952be9bSHans Verkuil sg_set_page(wr, sg_page(rd), rd->length, rd->offset); 3947952be9bSHans Verkuil rd = sg_next(rd); 3957952be9bSHans Verkuil wr = sg_next(wr); 3967952be9bSHans Verkuil } 3977952be9bSHans Verkuil 3987952be9bSHans Verkuil attach->dma_dir = DMA_NONE; 3997952be9bSHans Verkuil dbuf_attach->priv = attach; 4007952be9bSHans Verkuil 4017952be9bSHans Verkuil return 0; 4027952be9bSHans Verkuil } 4037952be9bSHans Verkuil 4047952be9bSHans Verkuil static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf, 4057952be9bSHans Verkuil struct dma_buf_attachment *db_attach) 4067952be9bSHans Verkuil { 4077952be9bSHans Verkuil struct vb2_dma_sg_attachment *attach = db_attach->priv; 4087952be9bSHans Verkuil struct sg_table *sgt; 4097952be9bSHans Verkuil 4107952be9bSHans Verkuil if (!attach) 4117952be9bSHans Verkuil return; 4127952be9bSHans Verkuil 4137952be9bSHans Verkuil sgt = &attach->sgt; 4147952be9bSHans Verkuil 4157952be9bSHans Verkuil /* release the scatterlist cache */ 4167952be9bSHans Verkuil if (attach->dma_dir != DMA_NONE) 4178b7c0280SMarek Szyprowski dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0); 4187952be9bSHans Verkuil sg_free_table(sgt); 4197952be9bSHans Verkuil kfree(attach); 4207952be9bSHans Verkuil db_attach->priv = NULL; 4217952be9bSHans Verkuil } 4227952be9bSHans Verkuil 4237952be9bSHans Verkuil static struct sg_table *vb2_dma_sg_dmabuf_ops_map( 4247952be9bSHans Verkuil struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir) 4257952be9bSHans Verkuil { 4267952be9bSHans Verkuil struct vb2_dma_sg_attachment *attach = db_attach->priv; 4277952be9bSHans Verkuil struct sg_table *sgt; 4287952be9bSHans Verkuil 4297952be9bSHans Verkuil sgt = &attach->sgt; 4307952be9bSHans Verkuil /* return previously mapped sg table */ 431*23543b3cSDmitry Osipenko if (attach->dma_dir == dma_dir) 4327952be9bSHans Verkuil return sgt; 4337952be9bSHans Verkuil 4347952be9bSHans Verkuil /* release any previous cache */ 4357952be9bSHans Verkuil if (attach->dma_dir != DMA_NONE) { 4368b7c0280SMarek Szyprowski dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0); 4377952be9bSHans Verkuil attach->dma_dir = DMA_NONE; 4387952be9bSHans Verkuil } 4397952be9bSHans Verkuil 4407952be9bSHans Verkuil /* mapping to the client with new direction */ 4418b7c0280SMarek Szyprowski if (dma_map_sgtable(db_attach->dev, sgt, dma_dir, 0)) { 4427952be9bSHans Verkuil pr_err("failed to map scatterlist\n"); 4437952be9bSHans Verkuil return ERR_PTR(-EIO); 4447952be9bSHans Verkuil } 4457952be9bSHans Verkuil 4467952be9bSHans Verkuil attach->dma_dir = dma_dir; 4477952be9bSHans Verkuil 4487952be9bSHans Verkuil return sgt; 4497952be9bSHans Verkuil } 4507952be9bSHans Verkuil 4517952be9bSHans Verkuil static void vb2_dma_sg_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach, 4527952be9bSHans Verkuil struct sg_table *sgt, enum dma_data_direction dma_dir) 4537952be9bSHans Verkuil { 4547952be9bSHans Verkuil /* nothing to be done here */ 4557952be9bSHans Verkuil } 4567952be9bSHans Verkuil 4577952be9bSHans Verkuil static void vb2_dma_sg_dmabuf_ops_release(struct dma_buf *dbuf) 4587952be9bSHans Verkuil { 4597952be9bSHans Verkuil /* drop reference obtained in vb2_dma_sg_get_dmabuf */ 4607952be9bSHans Verkuil vb2_dma_sg_put(dbuf->priv); 4617952be9bSHans Verkuil } 4627952be9bSHans Verkuil 463d4db5eb5SSergey Senozhatsky static int 464d4db5eb5SSergey Senozhatsky vb2_dma_sg_dmabuf_ops_begin_cpu_access(struct dma_buf *dbuf, 465d4db5eb5SSergey Senozhatsky enum dma_data_direction direction) 466d4db5eb5SSergey Senozhatsky { 467d4db5eb5SSergey Senozhatsky struct vb2_dma_sg_buf *buf = dbuf->priv; 468d4db5eb5SSergey Senozhatsky struct sg_table *sgt = buf->dma_sgt; 469d4db5eb5SSergey Senozhatsky 470d4db5eb5SSergey Senozhatsky dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir); 471d4db5eb5SSergey Senozhatsky return 0; 472d4db5eb5SSergey Senozhatsky } 473d4db5eb5SSergey Senozhatsky 474d4db5eb5SSergey Senozhatsky static int 475d4db5eb5SSergey Senozhatsky vb2_dma_sg_dmabuf_ops_end_cpu_access(struct dma_buf *dbuf, 476d4db5eb5SSergey Senozhatsky enum dma_data_direction direction) 477d4db5eb5SSergey Senozhatsky { 478d4db5eb5SSergey Senozhatsky struct vb2_dma_sg_buf *buf = dbuf->priv; 479d4db5eb5SSergey Senozhatsky struct sg_table *sgt = buf->dma_sgt; 480d4db5eb5SSergey Senozhatsky 481d4db5eb5SSergey Senozhatsky dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir); 482d4db5eb5SSergey Senozhatsky return 0; 483d4db5eb5SSergey Senozhatsky } 484d4db5eb5SSergey Senozhatsky 4857938f421SLucas De Marchi static int vb2_dma_sg_dmabuf_ops_vmap(struct dma_buf *dbuf, 4867938f421SLucas De Marchi struct iosys_map *map) 4877952be9bSHans Verkuil { 4887952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = dbuf->priv; 4897952be9bSHans Verkuil 4907938f421SLucas De Marchi iosys_map_set_vaddr(map, buf->vaddr); 4916619ccf1SThomas Zimmermann 4926619ccf1SThomas Zimmermann return 0; 4937952be9bSHans Verkuil } 4947952be9bSHans Verkuil 4957952be9bSHans Verkuil static int vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf *dbuf, 4967952be9bSHans Verkuil struct vm_area_struct *vma) 4977952be9bSHans Verkuil { 4987952be9bSHans Verkuil return vb2_dma_sg_mmap(dbuf->priv, vma); 4997952be9bSHans Verkuil } 5007952be9bSHans Verkuil 5017952be9bSHans Verkuil static const struct dma_buf_ops vb2_dma_sg_dmabuf_ops = { 5027952be9bSHans Verkuil .attach = vb2_dma_sg_dmabuf_ops_attach, 5037952be9bSHans Verkuil .detach = vb2_dma_sg_dmabuf_ops_detach, 5047952be9bSHans Verkuil .map_dma_buf = vb2_dma_sg_dmabuf_ops_map, 5057952be9bSHans Verkuil .unmap_dma_buf = vb2_dma_sg_dmabuf_ops_unmap, 506d4db5eb5SSergey Senozhatsky .begin_cpu_access = vb2_dma_sg_dmabuf_ops_begin_cpu_access, 507d4db5eb5SSergey Senozhatsky .end_cpu_access = vb2_dma_sg_dmabuf_ops_end_cpu_access, 5087952be9bSHans Verkuil .vmap = vb2_dma_sg_dmabuf_ops_vmap, 5097952be9bSHans Verkuil .mmap = vb2_dma_sg_dmabuf_ops_mmap, 5107952be9bSHans Verkuil .release = vb2_dma_sg_dmabuf_ops_release, 5117952be9bSHans Verkuil }; 5127952be9bSHans Verkuil 513a4b83debSSergey Senozhatsky static struct dma_buf *vb2_dma_sg_get_dmabuf(struct vb2_buffer *vb, 514a4b83debSSergey Senozhatsky void *buf_priv, 515a4b83debSSergey Senozhatsky unsigned long flags) 5167952be9bSHans Verkuil { 5177952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 5187952be9bSHans Verkuil struct dma_buf *dbuf; 5197952be9bSHans Verkuil DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 5207952be9bSHans Verkuil 5217952be9bSHans Verkuil exp_info.ops = &vb2_dma_sg_dmabuf_ops; 5227952be9bSHans Verkuil exp_info.size = buf->size; 5237952be9bSHans Verkuil exp_info.flags = flags; 5247952be9bSHans Verkuil exp_info.priv = buf; 5257952be9bSHans Verkuil 5267952be9bSHans Verkuil if (WARN_ON(!buf->dma_sgt)) 5277952be9bSHans Verkuil return NULL; 5287952be9bSHans Verkuil 5297952be9bSHans Verkuil dbuf = dma_buf_export(&exp_info); 5307952be9bSHans Verkuil if (IS_ERR(dbuf)) 5317952be9bSHans Verkuil return NULL; 5327952be9bSHans Verkuil 5337952be9bSHans Verkuil /* dmabuf keeps reference to vb2 buffer */ 5347952be9bSHans Verkuil refcount_inc(&buf->refcount); 5357952be9bSHans Verkuil 5367952be9bSHans Verkuil return dbuf; 5377952be9bSHans Verkuil } 5387952be9bSHans Verkuil 5397952be9bSHans Verkuil /*********************************************/ 5407952be9bSHans Verkuil /* callbacks for DMABUF buffers */ 5417952be9bSHans Verkuil /*********************************************/ 5427952be9bSHans Verkuil 5437952be9bSHans Verkuil static int vb2_dma_sg_map_dmabuf(void *mem_priv) 5447952be9bSHans Verkuil { 5457952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = mem_priv; 5467952be9bSHans Verkuil struct sg_table *sgt; 5477952be9bSHans Verkuil 5487952be9bSHans Verkuil if (WARN_ON(!buf->db_attach)) { 5497952be9bSHans Verkuil pr_err("trying to pin a non attached buffer\n"); 5507952be9bSHans Verkuil return -EINVAL; 5517952be9bSHans Verkuil } 5527952be9bSHans Verkuil 5537952be9bSHans Verkuil if (WARN_ON(buf->dma_sgt)) { 5547952be9bSHans Verkuil pr_err("dmabuf buffer is already pinned\n"); 5557952be9bSHans Verkuil return 0; 5567952be9bSHans Verkuil } 5577952be9bSHans Verkuil 5587952be9bSHans Verkuil /* get the associated scatterlist for this buffer */ 559a26ee3b7SDmitry Osipenko sgt = dma_buf_map_attachment_unlocked(buf->db_attach, buf->dma_dir); 5607952be9bSHans Verkuil if (IS_ERR(sgt)) { 5617952be9bSHans Verkuil pr_err("Error getting dmabuf scatterlist\n"); 5627952be9bSHans Verkuil return -EINVAL; 5637952be9bSHans Verkuil } 5647952be9bSHans Verkuil 5657952be9bSHans Verkuil buf->dma_sgt = sgt; 5667952be9bSHans Verkuil buf->vaddr = NULL; 5677952be9bSHans Verkuil 5687952be9bSHans Verkuil return 0; 5697952be9bSHans Verkuil } 5707952be9bSHans Verkuil 5717952be9bSHans Verkuil static void vb2_dma_sg_unmap_dmabuf(void *mem_priv) 5727952be9bSHans Verkuil { 5737952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = mem_priv; 5747952be9bSHans Verkuil struct sg_table *sgt = buf->dma_sgt; 5757938f421SLucas De Marchi struct iosys_map map = IOSYS_MAP_INIT_VADDR(buf->vaddr); 5767952be9bSHans Verkuil 5777952be9bSHans Verkuil if (WARN_ON(!buf->db_attach)) { 5787952be9bSHans Verkuil pr_err("trying to unpin a not attached buffer\n"); 5797952be9bSHans Verkuil return; 5807952be9bSHans Verkuil } 5817952be9bSHans Verkuil 5827952be9bSHans Verkuil if (WARN_ON(!sgt)) { 5837952be9bSHans Verkuil pr_err("dmabuf buffer is already unpinned\n"); 5847952be9bSHans Verkuil return; 5857952be9bSHans Verkuil } 5867952be9bSHans Verkuil 5877952be9bSHans Verkuil if (buf->vaddr) { 588a26ee3b7SDmitry Osipenko dma_buf_vunmap_unlocked(buf->db_attach->dmabuf, &map); 5897952be9bSHans Verkuil buf->vaddr = NULL; 5907952be9bSHans Verkuil } 591a26ee3b7SDmitry Osipenko dma_buf_unmap_attachment_unlocked(buf->db_attach, sgt, buf->dma_dir); 5927952be9bSHans Verkuil 5937952be9bSHans Verkuil buf->dma_sgt = NULL; 5947952be9bSHans Verkuil } 5957952be9bSHans Verkuil 5967952be9bSHans Verkuil static void vb2_dma_sg_detach_dmabuf(void *mem_priv) 5977952be9bSHans Verkuil { 5987952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = mem_priv; 5997952be9bSHans Verkuil 6007952be9bSHans Verkuil /* if vb2 works correctly you should never detach mapped buffer */ 6017952be9bSHans Verkuil if (WARN_ON(buf->dma_sgt)) 6027952be9bSHans Verkuil vb2_dma_sg_unmap_dmabuf(buf); 6037952be9bSHans Verkuil 6047952be9bSHans Verkuil /* detach this attachment */ 6057952be9bSHans Verkuil dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach); 6067952be9bSHans Verkuil kfree(buf); 6077952be9bSHans Verkuil } 6087952be9bSHans Verkuil 609a4b83debSSergey Senozhatsky static void *vb2_dma_sg_attach_dmabuf(struct vb2_buffer *vb, struct device *dev, 610a4b83debSSergey Senozhatsky struct dma_buf *dbuf, unsigned long size) 6117952be9bSHans Verkuil { 6127952be9bSHans Verkuil struct vb2_dma_sg_buf *buf; 6137952be9bSHans Verkuil struct dma_buf_attachment *dba; 6147952be9bSHans Verkuil 6157952be9bSHans Verkuil if (WARN_ON(!dev)) 6167952be9bSHans Verkuil return ERR_PTR(-EINVAL); 6177952be9bSHans Verkuil 6187952be9bSHans Verkuil if (dbuf->size < size) 6197952be9bSHans Verkuil return ERR_PTR(-EFAULT); 6207952be9bSHans Verkuil 6217952be9bSHans Verkuil buf = kzalloc(sizeof(*buf), GFP_KERNEL); 6227952be9bSHans Verkuil if (!buf) 6237952be9bSHans Verkuil return ERR_PTR(-ENOMEM); 6247952be9bSHans Verkuil 6257952be9bSHans Verkuil buf->dev = dev; 6267952be9bSHans Verkuil /* create attachment for the dmabuf with the user device */ 6277952be9bSHans Verkuil dba = dma_buf_attach(dbuf, buf->dev); 6287952be9bSHans Verkuil if (IS_ERR(dba)) { 6297952be9bSHans Verkuil pr_err("failed to attach dmabuf\n"); 6307952be9bSHans Verkuil kfree(buf); 6317952be9bSHans Verkuil return dba; 6327952be9bSHans Verkuil } 6337952be9bSHans Verkuil 634a4b83debSSergey Senozhatsky buf->dma_dir = vb->vb2_queue->dma_dir; 6357952be9bSHans Verkuil buf->size = size; 6367952be9bSHans Verkuil buf->db_attach = dba; 637d55c3ee6SHans de Goede buf->vb = vb; 6387952be9bSHans Verkuil 6397952be9bSHans Verkuil return buf; 6407952be9bSHans Verkuil } 6417952be9bSHans Verkuil 642a4b83debSSergey Senozhatsky static void *vb2_dma_sg_cookie(struct vb2_buffer *vb, void *buf_priv) 6437952be9bSHans Verkuil { 6447952be9bSHans Verkuil struct vb2_dma_sg_buf *buf = buf_priv; 6457952be9bSHans Verkuil 6467952be9bSHans Verkuil return buf->dma_sgt; 6477952be9bSHans Verkuil } 6487952be9bSHans Verkuil 6497952be9bSHans Verkuil const struct vb2_mem_ops vb2_dma_sg_memops = { 6507952be9bSHans Verkuil .alloc = vb2_dma_sg_alloc, 6517952be9bSHans Verkuil .put = vb2_dma_sg_put, 6527952be9bSHans Verkuil .get_userptr = vb2_dma_sg_get_userptr, 6537952be9bSHans Verkuil .put_userptr = vb2_dma_sg_put_userptr, 6547952be9bSHans Verkuil .prepare = vb2_dma_sg_prepare, 6557952be9bSHans Verkuil .finish = vb2_dma_sg_finish, 6567952be9bSHans Verkuil .vaddr = vb2_dma_sg_vaddr, 6577952be9bSHans Verkuil .mmap = vb2_dma_sg_mmap, 6587952be9bSHans Verkuil .num_users = vb2_dma_sg_num_users, 6597952be9bSHans Verkuil .get_dmabuf = vb2_dma_sg_get_dmabuf, 6607952be9bSHans Verkuil .map_dmabuf = vb2_dma_sg_map_dmabuf, 6617952be9bSHans Verkuil .unmap_dmabuf = vb2_dma_sg_unmap_dmabuf, 6627952be9bSHans Verkuil .attach_dmabuf = vb2_dma_sg_attach_dmabuf, 6637952be9bSHans Verkuil .detach_dmabuf = vb2_dma_sg_detach_dmabuf, 6647952be9bSHans Verkuil .cookie = vb2_dma_sg_cookie, 6657952be9bSHans Verkuil }; 6667952be9bSHans Verkuil EXPORT_SYMBOL_GPL(vb2_dma_sg_memops); 6677952be9bSHans Verkuil 6687952be9bSHans Verkuil MODULE_DESCRIPTION("dma scatter/gather memory handling routines for videobuf2"); 6697952be9bSHans Verkuil MODULE_AUTHOR("Andrzej Pietrasiewicz"); 6707952be9bSHans Verkuil MODULE_LICENSE("GPL"); 67116b0314aSGreg Kroah-Hartman MODULE_IMPORT_NS(DMA_BUF); 672