1c02a81fbSAndrew F. Davis // SPDX-License-Identifier: GPL-2.0
2c02a81fbSAndrew F. Davis /*
3c02a81fbSAndrew F. Davis * Framework for userspace DMA-BUF allocations
4c02a81fbSAndrew F. Davis *
5c02a81fbSAndrew F. Davis * Copyright (C) 2011 Google, Inc.
6c02a81fbSAndrew F. Davis * Copyright (C) 2019 Linaro Ltd.
7c02a81fbSAndrew F. Davis */
8c02a81fbSAndrew F. Davis
9c02a81fbSAndrew F. Davis #include <linux/cdev.h>
10c02a81fbSAndrew F. Davis #include <linux/debugfs.h>
11c02a81fbSAndrew F. Davis #include <linux/device.h>
12c02a81fbSAndrew F. Davis #include <linux/dma-buf.h>
13c02a81fbSAndrew F. Davis #include <linux/err.h>
14c02a81fbSAndrew F. Davis #include <linux/xarray.h>
15c02a81fbSAndrew F. Davis #include <linux/list.h>
16c02a81fbSAndrew F. Davis #include <linux/slab.h>
1792c4cfaeSJordy Zomer #include <linux/nospec.h>
18c02a81fbSAndrew F. Davis #include <linux/uaccess.h>
19c02a81fbSAndrew F. Davis #include <linux/syscalls.h>
20c02a81fbSAndrew F. Davis #include <linux/dma-heap.h>
21c02a81fbSAndrew F. Davis #include <uapi/linux/dma-heap.h>
22c02a81fbSAndrew F. Davis
23c02a81fbSAndrew F. Davis #define DEVNAME "dma_heap"
24c02a81fbSAndrew F. Davis
25c02a81fbSAndrew F. Davis #define NUM_HEAP_MINORS 128
26c02a81fbSAndrew F. Davis
27c02a81fbSAndrew F. Davis /**
28c02a81fbSAndrew F. Davis * struct dma_heap - represents a dmabuf heap in the system
29c02a81fbSAndrew F. Davis * @name: used for debugging/device-node name
30c02a81fbSAndrew F. Davis * @ops: ops struct for this heap
31c02a81fbSAndrew F. Davis * @heap_devt heap device node
32c02a81fbSAndrew F. Davis * @list list head connecting to list of heaps
33c02a81fbSAndrew F. Davis * @heap_cdev heap char device
34c02a81fbSAndrew F. Davis *
35c02a81fbSAndrew F. Davis * Represents a heap of memory from which buffers can be made.
36c02a81fbSAndrew F. Davis */
37c02a81fbSAndrew F. Davis struct dma_heap {
38c02a81fbSAndrew F. Davis const char *name;
39c02a81fbSAndrew F. Davis const struct dma_heap_ops *ops;
40c02a81fbSAndrew F. Davis void *priv;
41c02a81fbSAndrew F. Davis dev_t heap_devt;
42c02a81fbSAndrew F. Davis struct list_head list;
43c02a81fbSAndrew F. Davis struct cdev heap_cdev;
44c02a81fbSAndrew F. Davis };
45c02a81fbSAndrew F. Davis
46c02a81fbSAndrew F. Davis static LIST_HEAD(heap_list);
47c02a81fbSAndrew F. Davis static DEFINE_MUTEX(heap_list_lock);
48c02a81fbSAndrew F. Davis static dev_t dma_heap_devt;
49c02a81fbSAndrew F. Davis static struct class *dma_heap_class;
50c02a81fbSAndrew F. Davis static DEFINE_XARRAY_ALLOC(dma_heap_minors);
51c02a81fbSAndrew F. Davis
dma_heap_buffer_alloc(struct dma_heap * heap,size_t len,unsigned int fd_flags,unsigned int heap_flags)52c02a81fbSAndrew F. Davis static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
53c02a81fbSAndrew F. Davis unsigned int fd_flags,
54c02a81fbSAndrew F. Davis unsigned int heap_flags)
55c02a81fbSAndrew F. Davis {
56c7f59e3dSJohn Stultz struct dma_buf *dmabuf;
57c7f59e3dSJohn Stultz int fd;
58c7f59e3dSJohn Stultz
59c02a81fbSAndrew F. Davis /*
60c02a81fbSAndrew F. Davis * Allocations from all heaps have to begin
61c02a81fbSAndrew F. Davis * and end on page boundaries.
62c02a81fbSAndrew F. Davis */
63c02a81fbSAndrew F. Davis len = PAGE_ALIGN(len);
64c02a81fbSAndrew F. Davis if (!len)
65c02a81fbSAndrew F. Davis return -EINVAL;
66c02a81fbSAndrew F. Davis
67c7f59e3dSJohn Stultz dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags);
68c7f59e3dSJohn Stultz if (IS_ERR(dmabuf))
69c7f59e3dSJohn Stultz return PTR_ERR(dmabuf);
70c7f59e3dSJohn Stultz
71c7f59e3dSJohn Stultz fd = dma_buf_fd(dmabuf, fd_flags);
72c7f59e3dSJohn Stultz if (fd < 0) {
73c7f59e3dSJohn Stultz dma_buf_put(dmabuf);
74c7f59e3dSJohn Stultz /* just return, as put will call release and that will free */
75c7f59e3dSJohn Stultz }
76c7f59e3dSJohn Stultz return fd;
77c02a81fbSAndrew F. Davis }
78c02a81fbSAndrew F. Davis
dma_heap_open(struct inode * inode,struct file * file)79c02a81fbSAndrew F. Davis static int dma_heap_open(struct inode *inode, struct file *file)
80c02a81fbSAndrew F. Davis {
81c02a81fbSAndrew F. Davis struct dma_heap *heap;
82c02a81fbSAndrew F. Davis
83c02a81fbSAndrew F. Davis heap = xa_load(&dma_heap_minors, iminor(inode));
84c02a81fbSAndrew F. Davis if (!heap) {
85c02a81fbSAndrew F. Davis pr_err("dma_heap: minor %d unknown.\n", iminor(inode));
86c02a81fbSAndrew F. Davis return -ENODEV;
87c02a81fbSAndrew F. Davis }
88c02a81fbSAndrew F. Davis
89c02a81fbSAndrew F. Davis /* instance data as context */
90c02a81fbSAndrew F. Davis file->private_data = heap;
91c02a81fbSAndrew F. Davis nonseekable_open(inode, file);
92c02a81fbSAndrew F. Davis
93c02a81fbSAndrew F. Davis return 0;
94c02a81fbSAndrew F. Davis }
95c02a81fbSAndrew F. Davis
dma_heap_ioctl_allocate(struct file * file,void * data)96c02a81fbSAndrew F. Davis static long dma_heap_ioctl_allocate(struct file *file, void *data)
97c02a81fbSAndrew F. Davis {
98c02a81fbSAndrew F. Davis struct dma_heap_allocation_data *heap_allocation = data;
99c02a81fbSAndrew F. Davis struct dma_heap *heap = file->private_data;
100c02a81fbSAndrew F. Davis int fd;
101c02a81fbSAndrew F. Davis
102c02a81fbSAndrew F. Davis if (heap_allocation->fd)
103c02a81fbSAndrew F. Davis return -EINVAL;
104c02a81fbSAndrew F. Davis
105c02a81fbSAndrew F. Davis if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS)
106c02a81fbSAndrew F. Davis return -EINVAL;
107c02a81fbSAndrew F. Davis
108c02a81fbSAndrew F. Davis if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS)
109c02a81fbSAndrew F. Davis return -EINVAL;
110c02a81fbSAndrew F. Davis
111c02a81fbSAndrew F. Davis fd = dma_heap_buffer_alloc(heap, heap_allocation->len,
112c02a81fbSAndrew F. Davis heap_allocation->fd_flags,
113c02a81fbSAndrew F. Davis heap_allocation->heap_flags);
114c02a81fbSAndrew F. Davis if (fd < 0)
115c02a81fbSAndrew F. Davis return fd;
116c02a81fbSAndrew F. Davis
117c02a81fbSAndrew F. Davis heap_allocation->fd = fd;
118c02a81fbSAndrew F. Davis
119c02a81fbSAndrew F. Davis return 0;
120c02a81fbSAndrew F. Davis }
121c02a81fbSAndrew F. Davis
1227d411afeSzhong jiang static unsigned int dma_heap_ioctl_cmds[] = {
123b3b43465SAndrew F. Davis DMA_HEAP_IOCTL_ALLOC,
124c02a81fbSAndrew F. Davis };
125c02a81fbSAndrew F. Davis
dma_heap_ioctl(struct file * file,unsigned int ucmd,unsigned long arg)126c02a81fbSAndrew F. Davis static long dma_heap_ioctl(struct file *file, unsigned int ucmd,
127c02a81fbSAndrew F. Davis unsigned long arg)
128c02a81fbSAndrew F. Davis {
129c02a81fbSAndrew F. Davis char stack_kdata[128];
130c02a81fbSAndrew F. Davis char *kdata = stack_kdata;
131c02a81fbSAndrew F. Davis unsigned int kcmd;
132c02a81fbSAndrew F. Davis unsigned int in_size, out_size, drv_size, ksize;
133c02a81fbSAndrew F. Davis int nr = _IOC_NR(ucmd);
134c02a81fbSAndrew F. Davis int ret = 0;
135c02a81fbSAndrew F. Davis
136c02a81fbSAndrew F. Davis if (nr >= ARRAY_SIZE(dma_heap_ioctl_cmds))
137c02a81fbSAndrew F. Davis return -EINVAL;
138c02a81fbSAndrew F. Davis
13992c4cfaeSJordy Zomer nr = array_index_nospec(nr, ARRAY_SIZE(dma_heap_ioctl_cmds));
140c02a81fbSAndrew F. Davis /* Get the kernel ioctl cmd that matches */
141c02a81fbSAndrew F. Davis kcmd = dma_heap_ioctl_cmds[nr];
142c02a81fbSAndrew F. Davis
143c02a81fbSAndrew F. Davis /* Figure out the delta between user cmd size and kernel cmd size */
144c02a81fbSAndrew F. Davis drv_size = _IOC_SIZE(kcmd);
145c02a81fbSAndrew F. Davis out_size = _IOC_SIZE(ucmd);
146c02a81fbSAndrew F. Davis in_size = out_size;
147c02a81fbSAndrew F. Davis if ((ucmd & kcmd & IOC_IN) == 0)
148c02a81fbSAndrew F. Davis in_size = 0;
149c02a81fbSAndrew F. Davis if ((ucmd & kcmd & IOC_OUT) == 0)
150c02a81fbSAndrew F. Davis out_size = 0;
151c02a81fbSAndrew F. Davis ksize = max(max(in_size, out_size), drv_size);
152c02a81fbSAndrew F. Davis
153c02a81fbSAndrew F. Davis /* If necessary, allocate buffer for ioctl argument */
154c02a81fbSAndrew F. Davis if (ksize > sizeof(stack_kdata)) {
155c02a81fbSAndrew F. Davis kdata = kmalloc(ksize, GFP_KERNEL);
156c02a81fbSAndrew F. Davis if (!kdata)
157c02a81fbSAndrew F. Davis return -ENOMEM;
158c02a81fbSAndrew F. Davis }
159c02a81fbSAndrew F. Davis
160c02a81fbSAndrew F. Davis if (copy_from_user(kdata, (void __user *)arg, in_size) != 0) {
161c02a81fbSAndrew F. Davis ret = -EFAULT;
162c02a81fbSAndrew F. Davis goto err;
163c02a81fbSAndrew F. Davis }
164c02a81fbSAndrew F. Davis
165c02a81fbSAndrew F. Davis /* zero out any difference between the kernel/user structure size */
166c02a81fbSAndrew F. Davis if (ksize > in_size)
167c02a81fbSAndrew F. Davis memset(kdata + in_size, 0, ksize - in_size);
168c02a81fbSAndrew F. Davis
169c02a81fbSAndrew F. Davis switch (kcmd) {
170b3b43465SAndrew F. Davis case DMA_HEAP_IOCTL_ALLOC:
171c02a81fbSAndrew F. Davis ret = dma_heap_ioctl_allocate(file, kdata);
172c02a81fbSAndrew F. Davis break;
173c02a81fbSAndrew F. Davis default:
174f9d3b2c6SColin Ian King ret = -ENOTTY;
175f9d3b2c6SColin Ian King goto err;
176c02a81fbSAndrew F. Davis }
177c02a81fbSAndrew F. Davis
178c02a81fbSAndrew F. Davis if (copy_to_user((void __user *)arg, kdata, out_size) != 0)
179c02a81fbSAndrew F. Davis ret = -EFAULT;
180c02a81fbSAndrew F. Davis err:
181c02a81fbSAndrew F. Davis if (kdata != stack_kdata)
182c02a81fbSAndrew F. Davis kfree(kdata);
183c02a81fbSAndrew F. Davis return ret;
184c02a81fbSAndrew F. Davis }
185c02a81fbSAndrew F. Davis
186c02a81fbSAndrew F. Davis static const struct file_operations dma_heap_fops = {
187c02a81fbSAndrew F. Davis .owner = THIS_MODULE,
188c02a81fbSAndrew F. Davis .open = dma_heap_open,
189c02a81fbSAndrew F. Davis .unlocked_ioctl = dma_heap_ioctl,
190c02a81fbSAndrew F. Davis #ifdef CONFIG_COMPAT
191c02a81fbSAndrew F. Davis .compat_ioctl = dma_heap_ioctl,
192c02a81fbSAndrew F. Davis #endif
193c02a81fbSAndrew F. Davis };
194c02a81fbSAndrew F. Davis
195c02a81fbSAndrew F. Davis /**
196c02a81fbSAndrew F. Davis * dma_heap_get_drvdata() - get per-subdriver data for the heap
197c02a81fbSAndrew F. Davis * @heap: DMA-Heap to retrieve private data for
198c02a81fbSAndrew F. Davis *
199c02a81fbSAndrew F. Davis * Returns:
200c02a81fbSAndrew F. Davis * The per-subdriver data for the heap.
201c02a81fbSAndrew F. Davis */
dma_heap_get_drvdata(struct dma_heap * heap)202c02a81fbSAndrew F. Davis void *dma_heap_get_drvdata(struct dma_heap *heap)
203c02a81fbSAndrew F. Davis {
204c02a81fbSAndrew F. Davis return heap->priv;
205c02a81fbSAndrew F. Davis }
206c02a81fbSAndrew F. Davis
20798cd02daSJohn Stultz /**
20898cd02daSJohn Stultz * dma_heap_get_name() - get heap name
20998cd02daSJohn Stultz * @heap: DMA-Heap to retrieve private data for
21098cd02daSJohn Stultz *
21198cd02daSJohn Stultz * Returns:
21298cd02daSJohn Stultz * The char* for the heap name.
21398cd02daSJohn Stultz */
dma_heap_get_name(struct dma_heap * heap)21498cd02daSJohn Stultz const char *dma_heap_get_name(struct dma_heap *heap)
21598cd02daSJohn Stultz {
21698cd02daSJohn Stultz return heap->name;
21798cd02daSJohn Stultz }
21898cd02daSJohn Stultz
dma_heap_add(const struct dma_heap_export_info * exp_info)219c02a81fbSAndrew F. Davis struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
220c02a81fbSAndrew F. Davis {
221c02a81fbSAndrew F. Davis struct dma_heap *heap, *h, *err_ret;
222c02a81fbSAndrew F. Davis struct device *dev_ret;
223c02a81fbSAndrew F. Davis unsigned int minor;
224c02a81fbSAndrew F. Davis int ret;
225c02a81fbSAndrew F. Davis
226c02a81fbSAndrew F. Davis if (!exp_info->name || !strcmp(exp_info->name, "")) {
227c02a81fbSAndrew F. Davis pr_err("dma_heap: Cannot add heap without a name\n");
228c02a81fbSAndrew F. Davis return ERR_PTR(-EINVAL);
229c02a81fbSAndrew F. Davis }
230c02a81fbSAndrew F. Davis
231c02a81fbSAndrew F. Davis if (!exp_info->ops || !exp_info->ops->allocate) {
232c02a81fbSAndrew F. Davis pr_err("dma_heap: Cannot add heap with invalid ops struct\n");
233c02a81fbSAndrew F. Davis return ERR_PTR(-EINVAL);
234c02a81fbSAndrew F. Davis }
235c02a81fbSAndrew F. Davis
236c02a81fbSAndrew F. Davis heap = kzalloc(sizeof(*heap), GFP_KERNEL);
237c02a81fbSAndrew F. Davis if (!heap)
238c02a81fbSAndrew F. Davis return ERR_PTR(-ENOMEM);
239c02a81fbSAndrew F. Davis
240c02a81fbSAndrew F. Davis heap->name = exp_info->name;
241c02a81fbSAndrew F. Davis heap->ops = exp_info->ops;
242c02a81fbSAndrew F. Davis heap->priv = exp_info->priv;
243c02a81fbSAndrew F. Davis
244c02a81fbSAndrew F. Davis /* Find unused minor number */
245c02a81fbSAndrew F. Davis ret = xa_alloc(&dma_heap_minors, &minor, heap,
246c02a81fbSAndrew F. Davis XA_LIMIT(0, NUM_HEAP_MINORS - 1), GFP_KERNEL);
247c02a81fbSAndrew F. Davis if (ret < 0) {
248c02a81fbSAndrew F. Davis pr_err("dma_heap: Unable to get minor number for heap\n");
249c02a81fbSAndrew F. Davis err_ret = ERR_PTR(ret);
250c02a81fbSAndrew F. Davis goto err0;
251c02a81fbSAndrew F. Davis }
252c02a81fbSAndrew F. Davis
253c02a81fbSAndrew F. Davis /* Create device */
254c02a81fbSAndrew F. Davis heap->heap_devt = MKDEV(MAJOR(dma_heap_devt), minor);
255c02a81fbSAndrew F. Davis
256c02a81fbSAndrew F. Davis cdev_init(&heap->heap_cdev, &dma_heap_fops);
257c02a81fbSAndrew F. Davis ret = cdev_add(&heap->heap_cdev, heap->heap_devt, 1);
258c02a81fbSAndrew F. Davis if (ret < 0) {
259c02a81fbSAndrew F. Davis pr_err("dma_heap: Unable to add char device\n");
260c02a81fbSAndrew F. Davis err_ret = ERR_PTR(ret);
261c02a81fbSAndrew F. Davis goto err1;
262c02a81fbSAndrew F. Davis }
263c02a81fbSAndrew F. Davis
264c02a81fbSAndrew F. Davis dev_ret = device_create(dma_heap_class,
265c02a81fbSAndrew F. Davis NULL,
266c02a81fbSAndrew F. Davis heap->heap_devt,
267c02a81fbSAndrew F. Davis NULL,
268c02a81fbSAndrew F. Davis heap->name);
269c02a81fbSAndrew F. Davis if (IS_ERR(dev_ret)) {
270c02a81fbSAndrew F. Davis pr_err("dma_heap: Unable to create device\n");
271c02a81fbSAndrew F. Davis err_ret = ERR_CAST(dev_ret);
272c02a81fbSAndrew F. Davis goto err2;
273c02a81fbSAndrew F. Davis }
274432e2590SDawei Li
275c02a81fbSAndrew F. Davis mutex_lock(&heap_list_lock);
276432e2590SDawei Li /* check the name is unique */
277432e2590SDawei Li list_for_each_entry(h, &heap_list, list) {
278432e2590SDawei Li if (!strcmp(h->name, exp_info->name)) {
279432e2590SDawei Li mutex_unlock(&heap_list_lock);
280432e2590SDawei Li pr_err("dma_heap: Already registered heap named %s\n",
281432e2590SDawei Li exp_info->name);
282432e2590SDawei Li err_ret = ERR_PTR(-EINVAL);
283432e2590SDawei Li goto err3;
284432e2590SDawei Li }
285432e2590SDawei Li }
286432e2590SDawei Li
287432e2590SDawei Li /* Add heap to the list */
288c02a81fbSAndrew F. Davis list_add(&heap->list, &heap_list);
289c02a81fbSAndrew F. Davis mutex_unlock(&heap_list_lock);
290c02a81fbSAndrew F. Davis
291c02a81fbSAndrew F. Davis return heap;
292c02a81fbSAndrew F. Davis
293432e2590SDawei Li err3:
294432e2590SDawei Li device_destroy(dma_heap_class, heap->heap_devt);
295c02a81fbSAndrew F. Davis err2:
296c02a81fbSAndrew F. Davis cdev_del(&heap->heap_cdev);
297c02a81fbSAndrew F. Davis err1:
298c02a81fbSAndrew F. Davis xa_erase(&dma_heap_minors, minor);
299c02a81fbSAndrew F. Davis err0:
300c02a81fbSAndrew F. Davis kfree(heap);
301c02a81fbSAndrew F. Davis return err_ret;
302c02a81fbSAndrew F. Davis }
303c02a81fbSAndrew F. Davis
dma_heap_devnode(const struct device * dev,umode_t * mode)304ff62b8e6SGreg Kroah-Hartman static char *dma_heap_devnode(const struct device *dev, umode_t *mode)
305c02a81fbSAndrew F. Davis {
306c02a81fbSAndrew F. Davis return kasprintf(GFP_KERNEL, "dma_heap/%s", dev_name(dev));
307c02a81fbSAndrew F. Davis }
308c02a81fbSAndrew F. Davis
dma_heap_init(void)309c02a81fbSAndrew F. Davis static int dma_heap_init(void)
310c02a81fbSAndrew F. Davis {
311c02a81fbSAndrew F. Davis int ret;
312c02a81fbSAndrew F. Davis
313c02a81fbSAndrew F. Davis ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME);
314c02a81fbSAndrew F. Davis if (ret)
315c02a81fbSAndrew F. Davis return ret;
316c02a81fbSAndrew F. Davis
317*1aaba11dSGreg Kroah-Hartman dma_heap_class = class_create(DEVNAME);
318c02a81fbSAndrew F. Davis if (IS_ERR(dma_heap_class)) {
319c02a81fbSAndrew F. Davis unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS);
320c02a81fbSAndrew F. Davis return PTR_ERR(dma_heap_class);
321c02a81fbSAndrew F. Davis }
322c02a81fbSAndrew F. Davis dma_heap_class->devnode = dma_heap_devnode;
323c02a81fbSAndrew F. Davis
324c02a81fbSAndrew F. Davis return 0;
325c02a81fbSAndrew F. Davis }
326c02a81fbSAndrew F. Davis subsys_initcall(dma_heap_init);
327