xref: /openbmc/linux/drivers/misc/uacce/uacce.c (revision 77423a62)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/compat.h>
3 #include <linux/dma-mapping.h>
4 #include <linux/iommu.h>
5 #include <linux/module.h>
6 #include <linux/poll.h>
7 #include <linux/slab.h>
8 #include <linux/uacce.h>
9 
10 static struct class *uacce_class;
11 static dev_t uacce_devt;
12 static DEFINE_XARRAY_ALLOC(uacce_xa);
13 
14 /*
15  * If the parent driver or the device disappears, the queue state is invalid and
16  * ops are not usable anymore.
17  */
18 static bool uacce_queue_is_valid(struct uacce_queue *q)
19 {
20 	return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;
21 }
22 
23 static int uacce_start_queue(struct uacce_queue *q)
24 {
25 	int ret;
26 
27 	if (q->state != UACCE_Q_INIT)
28 		return -EINVAL;
29 
30 	if (q->uacce->ops->start_queue) {
31 		ret = q->uacce->ops->start_queue(q);
32 		if (ret < 0)
33 			return ret;
34 	}
35 
36 	q->state = UACCE_Q_STARTED;
37 	return 0;
38 }
39 
40 static int uacce_put_queue(struct uacce_queue *q)
41 {
42 	struct uacce_device *uacce = q->uacce;
43 
44 	if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue)
45 		uacce->ops->stop_queue(q);
46 
47 	if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) &&
48 	     uacce->ops->put_queue)
49 		uacce->ops->put_queue(q);
50 
51 	q->state = UACCE_Q_ZOMBIE;
52 
53 	return 0;
54 }
55 
56 static long uacce_fops_unl_ioctl(struct file *filep,
57 				 unsigned int cmd, unsigned long arg)
58 {
59 	struct uacce_queue *q = filep->private_data;
60 	struct uacce_device *uacce = q->uacce;
61 	long ret = -ENXIO;
62 
63 	/*
64 	 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from
65 	 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which
66 	 * gets called with mmap_lock held, by taking uacce->mutex instead of
67 	 * q->mutex. Doing this in uacce_fops_mmap() is not possible because
68 	 * uacce_fops_open() calls iommu_sva_bind_device(), which takes
69 	 * mmap_lock, while holding uacce->mutex.
70 	 */
71 	mutex_lock(&uacce->mutex);
72 	if (!uacce_queue_is_valid(q))
73 		goto out_unlock;
74 
75 	switch (cmd) {
76 	case UACCE_CMD_START_Q:
77 		ret = uacce_start_queue(q);
78 		break;
79 	case UACCE_CMD_PUT_Q:
80 		ret = uacce_put_queue(q);
81 		break;
82 	default:
83 		if (uacce->ops->ioctl)
84 			ret = uacce->ops->ioctl(q, cmd, arg);
85 		else
86 			ret = -EINVAL;
87 	}
88 out_unlock:
89 	mutex_unlock(&uacce->mutex);
90 	return ret;
91 }
92 
93 #ifdef CONFIG_COMPAT
94 static long uacce_fops_compat_ioctl(struct file *filep,
95 				   unsigned int cmd, unsigned long arg)
96 {
97 	arg = (unsigned long)compat_ptr(arg);
98 
99 	return uacce_fops_unl_ioctl(filep, cmd, arg);
100 }
101 #endif
102 
103 static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
104 {
105 	u32 pasid;
106 	struct iommu_sva *handle;
107 
108 	if (!(uacce->flags & UACCE_DEV_SVA))
109 		return 0;
110 
111 	handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL);
112 	if (IS_ERR(handle))
113 		return PTR_ERR(handle);
114 
115 	pasid = iommu_sva_get_pasid(handle);
116 	if (pasid == IOMMU_PASID_INVALID) {
117 		iommu_sva_unbind_device(handle);
118 		return -ENODEV;
119 	}
120 
121 	q->handle = handle;
122 	q->pasid = pasid;
123 	return 0;
124 }
125 
126 static void uacce_unbind_queue(struct uacce_queue *q)
127 {
128 	if (!q->handle)
129 		return;
130 	iommu_sva_unbind_device(q->handle);
131 	q->handle = NULL;
132 }
133 
134 static int uacce_fops_open(struct inode *inode, struct file *filep)
135 {
136 	struct uacce_device *uacce;
137 	struct uacce_queue *q;
138 	int ret;
139 
140 	uacce = xa_load(&uacce_xa, iminor(inode));
141 	if (!uacce)
142 		return -ENODEV;
143 
144 	q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL);
145 	if (!q)
146 		return -ENOMEM;
147 
148 	mutex_lock(&uacce->mutex);
149 
150 	if (!uacce->parent) {
151 		ret = -EINVAL;
152 		goto out_with_mem;
153 	}
154 
155 	ret = uacce_bind_queue(uacce, q);
156 	if (ret)
157 		goto out_with_mem;
158 
159 	q->uacce = uacce;
160 
161 	if (uacce->ops->get_queue) {
162 		ret = uacce->ops->get_queue(uacce, q->pasid, q);
163 		if (ret < 0)
164 			goto out_with_bond;
165 	}
166 
167 	init_waitqueue_head(&q->wait);
168 	filep->private_data = q;
169 	uacce->inode = inode;
170 	q->state = UACCE_Q_INIT;
171 	mutex_init(&q->mutex);
172 	list_add(&q->list, &uacce->queues);
173 	mutex_unlock(&uacce->mutex);
174 
175 	return 0;
176 
177 out_with_bond:
178 	uacce_unbind_queue(q);
179 out_with_mem:
180 	kfree(q);
181 	mutex_unlock(&uacce->mutex);
182 	return ret;
183 }
184 
185 static int uacce_fops_release(struct inode *inode, struct file *filep)
186 {
187 	struct uacce_queue *q = filep->private_data;
188 	struct uacce_device *uacce = q->uacce;
189 
190 	mutex_lock(&uacce->mutex);
191 	uacce_put_queue(q);
192 	uacce_unbind_queue(q);
193 	list_del(&q->list);
194 	mutex_unlock(&uacce->mutex);
195 	kfree(q);
196 
197 	return 0;
198 }
199 
200 static void uacce_vma_close(struct vm_area_struct *vma)
201 {
202 	struct uacce_queue *q = vma->vm_private_data;
203 	struct uacce_qfile_region *qfr = NULL;
204 
205 	if (vma->vm_pgoff < UACCE_MAX_REGION)
206 		qfr = q->qfrs[vma->vm_pgoff];
207 
208 	kfree(qfr);
209 }
210 
211 static const struct vm_operations_struct uacce_vm_ops = {
212 	.close = uacce_vma_close,
213 };
214 
215 static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)
216 {
217 	struct uacce_queue *q = filep->private_data;
218 	struct uacce_device *uacce = q->uacce;
219 	struct uacce_qfile_region *qfr;
220 	enum uacce_qfrt type = UACCE_MAX_REGION;
221 	int ret = 0;
222 
223 	if (vma->vm_pgoff < UACCE_MAX_REGION)
224 		type = vma->vm_pgoff;
225 	else
226 		return -EINVAL;
227 
228 	qfr = kzalloc(sizeof(*qfr), GFP_KERNEL);
229 	if (!qfr)
230 		return -ENOMEM;
231 
232 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK;
233 	vma->vm_ops = &uacce_vm_ops;
234 	vma->vm_private_data = q;
235 	qfr->type = type;
236 
237 	mutex_lock(&q->mutex);
238 	if (!uacce_queue_is_valid(q)) {
239 		ret = -ENXIO;
240 		goto out_with_lock;
241 	}
242 
243 	if (q->qfrs[type]) {
244 		ret = -EEXIST;
245 		goto out_with_lock;
246 	}
247 
248 	switch (type) {
249 	case UACCE_QFRT_MMIO:
250 	case UACCE_QFRT_DUS:
251 		if (!uacce->ops->mmap) {
252 			ret = -EINVAL;
253 			goto out_with_lock;
254 		}
255 
256 		ret = uacce->ops->mmap(q, vma, qfr);
257 		if (ret)
258 			goto out_with_lock;
259 		break;
260 
261 	default:
262 		ret = -EINVAL;
263 		goto out_with_lock;
264 	}
265 
266 	q->qfrs[type] = qfr;
267 	mutex_unlock(&q->mutex);
268 
269 	return ret;
270 
271 out_with_lock:
272 	mutex_unlock(&q->mutex);
273 	kfree(qfr);
274 	return ret;
275 }
276 
277 static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)
278 {
279 	struct uacce_queue *q = file->private_data;
280 	struct uacce_device *uacce = q->uacce;
281 	__poll_t ret = 0;
282 
283 	mutex_lock(&q->mutex);
284 	if (!uacce_queue_is_valid(q))
285 		goto out_unlock;
286 
287 	poll_wait(file, &q->wait, wait);
288 
289 	if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))
290 		ret = EPOLLIN | EPOLLRDNORM;
291 
292 out_unlock:
293 	mutex_unlock(&q->mutex);
294 	return ret;
295 }
296 
297 static const struct file_operations uacce_fops = {
298 	.owner		= THIS_MODULE,
299 	.open		= uacce_fops_open,
300 	.release	= uacce_fops_release,
301 	.unlocked_ioctl	= uacce_fops_unl_ioctl,
302 #ifdef CONFIG_COMPAT
303 	.compat_ioctl	= uacce_fops_compat_ioctl,
304 #endif
305 	.mmap		= uacce_fops_mmap,
306 	.poll		= uacce_fops_poll,
307 };
308 
309 #define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
310 
311 static ssize_t api_show(struct device *dev,
312 			struct device_attribute *attr, char *buf)
313 {
314 	struct uacce_device *uacce = to_uacce_device(dev);
315 
316 	return sysfs_emit(buf, "%s\n", uacce->api_ver);
317 }
318 
319 static ssize_t flags_show(struct device *dev,
320 			  struct device_attribute *attr, char *buf)
321 {
322 	struct uacce_device *uacce = to_uacce_device(dev);
323 
324 	return sysfs_emit(buf, "%u\n", uacce->flags);
325 }
326 
327 static ssize_t available_instances_show(struct device *dev,
328 					struct device_attribute *attr,
329 					char *buf)
330 {
331 	struct uacce_device *uacce = to_uacce_device(dev);
332 
333 	if (!uacce->ops->get_available_instances)
334 		return -ENODEV;
335 
336 	return sysfs_emit(buf, "%d\n",
337 		       uacce->ops->get_available_instances(uacce));
338 }
339 
340 static ssize_t algorithms_show(struct device *dev,
341 			       struct device_attribute *attr, char *buf)
342 {
343 	struct uacce_device *uacce = to_uacce_device(dev);
344 
345 	return sysfs_emit(buf, "%s\n", uacce->algs);
346 }
347 
348 static ssize_t region_mmio_size_show(struct device *dev,
349 				     struct device_attribute *attr, char *buf)
350 {
351 	struct uacce_device *uacce = to_uacce_device(dev);
352 
353 	return sysfs_emit(buf, "%lu\n",
354 		       uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);
355 }
356 
357 static ssize_t region_dus_size_show(struct device *dev,
358 				    struct device_attribute *attr, char *buf)
359 {
360 	struct uacce_device *uacce = to_uacce_device(dev);
361 
362 	return sysfs_emit(buf, "%lu\n",
363 		       uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);
364 }
365 
366 static DEVICE_ATTR_RO(api);
367 static DEVICE_ATTR_RO(flags);
368 static DEVICE_ATTR_RO(available_instances);
369 static DEVICE_ATTR_RO(algorithms);
370 static DEVICE_ATTR_RO(region_mmio_size);
371 static DEVICE_ATTR_RO(region_dus_size);
372 
373 static struct attribute *uacce_dev_attrs[] = {
374 	&dev_attr_api.attr,
375 	&dev_attr_flags.attr,
376 	&dev_attr_available_instances.attr,
377 	&dev_attr_algorithms.attr,
378 	&dev_attr_region_mmio_size.attr,
379 	&dev_attr_region_dus_size.attr,
380 	NULL,
381 };
382 
383 static umode_t uacce_dev_is_visible(struct kobject *kobj,
384 				    struct attribute *attr, int n)
385 {
386 	struct device *dev = kobj_to_dev(kobj);
387 	struct uacce_device *uacce = to_uacce_device(dev);
388 
389 	if (((attr == &dev_attr_region_mmio_size.attr) &&
390 	    (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||
391 	    ((attr == &dev_attr_region_dus_size.attr) &&
392 	    (!uacce->qf_pg_num[UACCE_QFRT_DUS])))
393 		return 0;
394 
395 	return attr->mode;
396 }
397 
398 static struct attribute_group uacce_dev_group = {
399 	.is_visible	= uacce_dev_is_visible,
400 	.attrs		= uacce_dev_attrs,
401 };
402 
403 __ATTRIBUTE_GROUPS(uacce_dev);
404 
405 static void uacce_release(struct device *dev)
406 {
407 	struct uacce_device *uacce = to_uacce_device(dev);
408 
409 	kfree(uacce);
410 }
411 
412 static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags)
413 {
414 	int ret;
415 
416 	if (!(flags & UACCE_DEV_SVA))
417 		return flags;
418 
419 	flags &= ~UACCE_DEV_SVA;
420 
421 	ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF);
422 	if (ret) {
423 		dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret));
424 		return flags;
425 	}
426 
427 	ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA);
428 	if (ret) {
429 		dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret));
430 		iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF);
431 		return flags;
432 	}
433 
434 	return flags | UACCE_DEV_SVA;
435 }
436 
437 static void uacce_disable_sva(struct uacce_device *uacce)
438 {
439 	if (!(uacce->flags & UACCE_DEV_SVA))
440 		return;
441 
442 	iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA);
443 	iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF);
444 }
445 
446 /**
447  * uacce_alloc() - alloc an accelerator
448  * @parent: pointer of uacce parent device
449  * @interface: pointer of uacce_interface for register
450  *
451  * Returns uacce pointer if success and ERR_PTR if not
452  * Need check returned negotiated uacce->flags
453  */
454 struct uacce_device *uacce_alloc(struct device *parent,
455 				 struct uacce_interface *interface)
456 {
457 	unsigned int flags = interface->flags;
458 	struct uacce_device *uacce;
459 	int ret;
460 
461 	uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL);
462 	if (!uacce)
463 		return ERR_PTR(-ENOMEM);
464 
465 	flags = uacce_enable_sva(parent, flags);
466 
467 	uacce->parent = parent;
468 	uacce->flags = flags;
469 	uacce->ops = interface->ops;
470 
471 	ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,
472 		       GFP_KERNEL);
473 	if (ret < 0)
474 		goto err_with_uacce;
475 
476 	INIT_LIST_HEAD(&uacce->queues);
477 	mutex_init(&uacce->mutex);
478 	device_initialize(&uacce->dev);
479 	uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
480 	uacce->dev.class = uacce_class;
481 	uacce->dev.groups = uacce_dev_groups;
482 	uacce->dev.parent = uacce->parent;
483 	uacce->dev.release = uacce_release;
484 	dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);
485 
486 	return uacce;
487 
488 err_with_uacce:
489 	uacce_disable_sva(uacce);
490 	kfree(uacce);
491 	return ERR_PTR(ret);
492 }
493 EXPORT_SYMBOL_GPL(uacce_alloc);
494 
495 /**
496  * uacce_register() - add the accelerator to cdev and export to user space
497  * @uacce: The initialized uacce device
498  *
499  * Return 0 if register succeeded, or an error.
500  */
501 int uacce_register(struct uacce_device *uacce)
502 {
503 	if (!uacce)
504 		return -ENODEV;
505 
506 	uacce->cdev = cdev_alloc();
507 	if (!uacce->cdev)
508 		return -ENOMEM;
509 
510 	uacce->cdev->ops = &uacce_fops;
511 	uacce->cdev->owner = THIS_MODULE;
512 
513 	return cdev_device_add(uacce->cdev, &uacce->dev);
514 }
515 EXPORT_SYMBOL_GPL(uacce_register);
516 
517 /**
518  * uacce_remove() - remove the accelerator
519  * @uacce: the accelerator to remove
520  */
521 void uacce_remove(struct uacce_device *uacce)
522 {
523 	struct uacce_queue *q, *next_q;
524 
525 	if (!uacce)
526 		return;
527 	/*
528 	 * unmap remaining mapping from user space, preventing user still
529 	 * access the mmaped area while parent device is already removed
530 	 */
531 	if (uacce->inode)
532 		unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1);
533 
534 	/*
535 	 * uacce_fops_open() may be running concurrently, even after we remove
536 	 * the cdev. Holding uacce->mutex ensures that open() does not obtain a
537 	 * removed uacce device.
538 	 */
539 	mutex_lock(&uacce->mutex);
540 	/* ensure no open queue remains */
541 	list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
542 		/*
543 		 * Taking q->mutex ensures that fops do not use the defunct
544 		 * uacce->ops after the queue is disabled.
545 		 */
546 		mutex_lock(&q->mutex);
547 		uacce_put_queue(q);
548 		mutex_unlock(&q->mutex);
549 		uacce_unbind_queue(q);
550 	}
551 
552 	/* disable sva now since no opened queues */
553 	uacce_disable_sva(uacce);
554 
555 	if (uacce->cdev)
556 		cdev_device_del(uacce->cdev, &uacce->dev);
557 	xa_erase(&uacce_xa, uacce->dev_id);
558 	/*
559 	 * uacce exists as long as there are open fds, but ops will be freed
560 	 * now. Ensure that bugs cause NULL deref rather than use-after-free.
561 	 */
562 	uacce->ops = NULL;
563 	uacce->parent = NULL;
564 	mutex_unlock(&uacce->mutex);
565 	put_device(&uacce->dev);
566 }
567 EXPORT_SYMBOL_GPL(uacce_remove);
568 
569 static int __init uacce_init(void)
570 {
571 	int ret;
572 
573 	uacce_class = class_create(THIS_MODULE, UACCE_NAME);
574 	if (IS_ERR(uacce_class))
575 		return PTR_ERR(uacce_class);
576 
577 	ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);
578 	if (ret)
579 		class_destroy(uacce_class);
580 
581 	return ret;
582 }
583 
584 static __exit void uacce_exit(void)
585 {
586 	unregister_chrdev_region(uacce_devt, MINORMASK);
587 	class_destroy(uacce_class);
588 }
589 
590 subsys_initcall(uacce_init);
591 module_exit(uacce_exit);
592 
593 MODULE_LICENSE("GPL");
594 MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");
595 MODULE_DESCRIPTION("Accelerator interface for Userland applications");
596