1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/compat.h> 3 #include <linux/dma-mapping.h> 4 #include <linux/iommu.h> 5 #include <linux/module.h> 6 #include <linux/poll.h> 7 #include <linux/slab.h> 8 #include <linux/uacce.h> 9 10 static struct class *uacce_class; 11 static dev_t uacce_devt; 12 static DEFINE_XARRAY_ALLOC(uacce_xa); 13 14 /* 15 * If the parent driver or the device disappears, the queue state is invalid and 16 * ops are not usable anymore. 17 */ 18 static bool uacce_queue_is_valid(struct uacce_queue *q) 19 { 20 return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED; 21 } 22 23 static int uacce_start_queue(struct uacce_queue *q) 24 { 25 int ret; 26 27 if (q->state != UACCE_Q_INIT) 28 return -EINVAL; 29 30 if (q->uacce->ops->start_queue) { 31 ret = q->uacce->ops->start_queue(q); 32 if (ret < 0) 33 return ret; 34 } 35 36 q->state = UACCE_Q_STARTED; 37 return 0; 38 } 39 40 static int uacce_put_queue(struct uacce_queue *q) 41 { 42 struct uacce_device *uacce = q->uacce; 43 44 if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) 45 uacce->ops->stop_queue(q); 46 47 if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) && 48 uacce->ops->put_queue) 49 uacce->ops->put_queue(q); 50 51 q->state = UACCE_Q_ZOMBIE; 52 53 return 0; 54 } 55 56 static long uacce_fops_unl_ioctl(struct file *filep, 57 unsigned int cmd, unsigned long arg) 58 { 59 struct uacce_queue *q = filep->private_data; 60 struct uacce_device *uacce = q->uacce; 61 long ret = -ENXIO; 62 63 /* 64 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from 65 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which 66 * gets called with mmap_lock held, by taking uacce->mutex instead of 67 * q->mutex. Doing this in uacce_fops_mmap() is not possible because 68 * uacce_fops_open() calls iommu_sva_bind_device(), which takes 69 * mmap_lock, while holding uacce->mutex. 70 */ 71 mutex_lock(&uacce->mutex); 72 if (!uacce_queue_is_valid(q)) 73 goto out_unlock; 74 75 switch (cmd) { 76 case UACCE_CMD_START_Q: 77 ret = uacce_start_queue(q); 78 break; 79 case UACCE_CMD_PUT_Q: 80 ret = uacce_put_queue(q); 81 break; 82 default: 83 if (uacce->ops->ioctl) 84 ret = uacce->ops->ioctl(q, cmd, arg); 85 else 86 ret = -EINVAL; 87 } 88 out_unlock: 89 mutex_unlock(&uacce->mutex); 90 return ret; 91 } 92 93 #ifdef CONFIG_COMPAT 94 static long uacce_fops_compat_ioctl(struct file *filep, 95 unsigned int cmd, unsigned long arg) 96 { 97 arg = (unsigned long)compat_ptr(arg); 98 99 return uacce_fops_unl_ioctl(filep, cmd, arg); 100 } 101 #endif 102 103 static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q) 104 { 105 u32 pasid; 106 struct iommu_sva *handle; 107 108 if (!(uacce->flags & UACCE_DEV_SVA)) 109 return 0; 110 111 handle = iommu_sva_bind_device(uacce->parent, current->mm); 112 if (IS_ERR(handle)) 113 return PTR_ERR(handle); 114 115 pasid = iommu_sva_get_pasid(handle); 116 if (pasid == IOMMU_PASID_INVALID) { 117 iommu_sva_unbind_device(handle); 118 return -ENODEV; 119 } 120 121 q->handle = handle; 122 q->pasid = pasid; 123 return 0; 124 } 125 126 static void uacce_unbind_queue(struct uacce_queue *q) 127 { 128 if (!q->handle) 129 return; 130 iommu_sva_unbind_device(q->handle); 131 q->handle = NULL; 132 } 133 134 static int uacce_fops_open(struct inode *inode, struct file *filep) 135 { 136 struct uacce_device *uacce; 137 struct uacce_queue *q; 138 int ret; 139 140 uacce = xa_load(&uacce_xa, iminor(inode)); 141 if (!uacce) 142 return -ENODEV; 143 144 q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL); 145 if (!q) 146 return -ENOMEM; 147 148 mutex_lock(&uacce->mutex); 149 150 if (!uacce->parent) { 151 ret = -EINVAL; 152 goto out_with_mem; 153 } 154 155 ret = uacce_bind_queue(uacce, q); 156 if (ret) 157 goto out_with_mem; 158 159 q->uacce = uacce; 160 161 if (uacce->ops->get_queue) { 162 ret = uacce->ops->get_queue(uacce, q->pasid, q); 163 if (ret < 0) 164 goto out_with_bond; 165 } 166 167 init_waitqueue_head(&q->wait); 168 filep->private_data = q; 169 uacce->inode = inode; 170 q->state = UACCE_Q_INIT; 171 mutex_init(&q->mutex); 172 list_add(&q->list, &uacce->queues); 173 mutex_unlock(&uacce->mutex); 174 175 return 0; 176 177 out_with_bond: 178 uacce_unbind_queue(q); 179 out_with_mem: 180 kfree(q); 181 mutex_unlock(&uacce->mutex); 182 return ret; 183 } 184 185 static int uacce_fops_release(struct inode *inode, struct file *filep) 186 { 187 struct uacce_queue *q = filep->private_data; 188 struct uacce_device *uacce = q->uacce; 189 190 mutex_lock(&uacce->mutex); 191 uacce_put_queue(q); 192 uacce_unbind_queue(q); 193 list_del(&q->list); 194 mutex_unlock(&uacce->mutex); 195 kfree(q); 196 197 return 0; 198 } 199 200 static void uacce_vma_close(struct vm_area_struct *vma) 201 { 202 struct uacce_queue *q = vma->vm_private_data; 203 struct uacce_qfile_region *qfr = NULL; 204 205 if (vma->vm_pgoff < UACCE_MAX_REGION) 206 qfr = q->qfrs[vma->vm_pgoff]; 207 208 kfree(qfr); 209 } 210 211 static const struct vm_operations_struct uacce_vm_ops = { 212 .close = uacce_vma_close, 213 }; 214 215 static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) 216 { 217 struct uacce_queue *q = filep->private_data; 218 struct uacce_device *uacce = q->uacce; 219 struct uacce_qfile_region *qfr; 220 enum uacce_qfrt type = UACCE_MAX_REGION; 221 int ret = 0; 222 223 if (vma->vm_pgoff < UACCE_MAX_REGION) 224 type = vma->vm_pgoff; 225 else 226 return -EINVAL; 227 228 qfr = kzalloc(sizeof(*qfr), GFP_KERNEL); 229 if (!qfr) 230 return -ENOMEM; 231 232 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK; 233 vma->vm_ops = &uacce_vm_ops; 234 vma->vm_private_data = q; 235 qfr->type = type; 236 237 mutex_lock(&q->mutex); 238 if (!uacce_queue_is_valid(q)) { 239 ret = -ENXIO; 240 goto out_with_lock; 241 } 242 243 if (q->qfrs[type]) { 244 ret = -EEXIST; 245 goto out_with_lock; 246 } 247 248 switch (type) { 249 case UACCE_QFRT_MMIO: 250 case UACCE_QFRT_DUS: 251 if (!uacce->ops->mmap) { 252 ret = -EINVAL; 253 goto out_with_lock; 254 } 255 256 ret = uacce->ops->mmap(q, vma, qfr); 257 if (ret) 258 goto out_with_lock; 259 break; 260 261 default: 262 ret = -EINVAL; 263 goto out_with_lock; 264 } 265 266 q->qfrs[type] = qfr; 267 mutex_unlock(&q->mutex); 268 269 return ret; 270 271 out_with_lock: 272 mutex_unlock(&q->mutex); 273 kfree(qfr); 274 return ret; 275 } 276 277 static __poll_t uacce_fops_poll(struct file *file, poll_table *wait) 278 { 279 struct uacce_queue *q = file->private_data; 280 struct uacce_device *uacce = q->uacce; 281 __poll_t ret = 0; 282 283 mutex_lock(&q->mutex); 284 if (!uacce_queue_is_valid(q)) 285 goto out_unlock; 286 287 poll_wait(file, &q->wait, wait); 288 289 if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q)) 290 ret = EPOLLIN | EPOLLRDNORM; 291 292 out_unlock: 293 mutex_unlock(&q->mutex); 294 return ret; 295 } 296 297 static const struct file_operations uacce_fops = { 298 .owner = THIS_MODULE, 299 .open = uacce_fops_open, 300 .release = uacce_fops_release, 301 .unlocked_ioctl = uacce_fops_unl_ioctl, 302 #ifdef CONFIG_COMPAT 303 .compat_ioctl = uacce_fops_compat_ioctl, 304 #endif 305 .mmap = uacce_fops_mmap, 306 .poll = uacce_fops_poll, 307 }; 308 309 #define to_uacce_device(dev) container_of(dev, struct uacce_device, dev) 310 311 static ssize_t api_show(struct device *dev, 312 struct device_attribute *attr, char *buf) 313 { 314 struct uacce_device *uacce = to_uacce_device(dev); 315 316 return sysfs_emit(buf, "%s\n", uacce->api_ver); 317 } 318 319 static ssize_t flags_show(struct device *dev, 320 struct device_attribute *attr, char *buf) 321 { 322 struct uacce_device *uacce = to_uacce_device(dev); 323 324 return sysfs_emit(buf, "%u\n", uacce->flags); 325 } 326 327 static ssize_t available_instances_show(struct device *dev, 328 struct device_attribute *attr, 329 char *buf) 330 { 331 struct uacce_device *uacce = to_uacce_device(dev); 332 333 if (!uacce->ops->get_available_instances) 334 return -ENODEV; 335 336 return sysfs_emit(buf, "%d\n", 337 uacce->ops->get_available_instances(uacce)); 338 } 339 340 static ssize_t algorithms_show(struct device *dev, 341 struct device_attribute *attr, char *buf) 342 { 343 struct uacce_device *uacce = to_uacce_device(dev); 344 345 return sysfs_emit(buf, "%s\n", uacce->algs); 346 } 347 348 static ssize_t region_mmio_size_show(struct device *dev, 349 struct device_attribute *attr, char *buf) 350 { 351 struct uacce_device *uacce = to_uacce_device(dev); 352 353 return sysfs_emit(buf, "%lu\n", 354 uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT); 355 } 356 357 static ssize_t region_dus_size_show(struct device *dev, 358 struct device_attribute *attr, char *buf) 359 { 360 struct uacce_device *uacce = to_uacce_device(dev); 361 362 return sysfs_emit(buf, "%lu\n", 363 uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT); 364 } 365 366 static DEVICE_ATTR_RO(api); 367 static DEVICE_ATTR_RO(flags); 368 static DEVICE_ATTR_RO(available_instances); 369 static DEVICE_ATTR_RO(algorithms); 370 static DEVICE_ATTR_RO(region_mmio_size); 371 static DEVICE_ATTR_RO(region_dus_size); 372 373 static struct attribute *uacce_dev_attrs[] = { 374 &dev_attr_api.attr, 375 &dev_attr_flags.attr, 376 &dev_attr_available_instances.attr, 377 &dev_attr_algorithms.attr, 378 &dev_attr_region_mmio_size.attr, 379 &dev_attr_region_dus_size.attr, 380 NULL, 381 }; 382 383 static umode_t uacce_dev_is_visible(struct kobject *kobj, 384 struct attribute *attr, int n) 385 { 386 struct device *dev = kobj_to_dev(kobj); 387 struct uacce_device *uacce = to_uacce_device(dev); 388 389 if (((attr == &dev_attr_region_mmio_size.attr) && 390 (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) || 391 ((attr == &dev_attr_region_dus_size.attr) && 392 (!uacce->qf_pg_num[UACCE_QFRT_DUS]))) 393 return 0; 394 395 return attr->mode; 396 } 397 398 static struct attribute_group uacce_dev_group = { 399 .is_visible = uacce_dev_is_visible, 400 .attrs = uacce_dev_attrs, 401 }; 402 403 __ATTRIBUTE_GROUPS(uacce_dev); 404 405 static void uacce_release(struct device *dev) 406 { 407 struct uacce_device *uacce = to_uacce_device(dev); 408 409 kfree(uacce); 410 } 411 412 static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags) 413 { 414 int ret; 415 416 if (!(flags & UACCE_DEV_SVA)) 417 return flags; 418 419 flags &= ~UACCE_DEV_SVA; 420 421 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF); 422 if (ret) { 423 dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret)); 424 return flags; 425 } 426 427 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA); 428 if (ret) { 429 dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret)); 430 iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF); 431 return flags; 432 } 433 434 return flags | UACCE_DEV_SVA; 435 } 436 437 static void uacce_disable_sva(struct uacce_device *uacce) 438 { 439 if (!(uacce->flags & UACCE_DEV_SVA)) 440 return; 441 442 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA); 443 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF); 444 } 445 446 /** 447 * uacce_alloc() - alloc an accelerator 448 * @parent: pointer of uacce parent device 449 * @interface: pointer of uacce_interface for register 450 * 451 * Returns uacce pointer if success and ERR_PTR if not 452 * Need check returned negotiated uacce->flags 453 */ 454 struct uacce_device *uacce_alloc(struct device *parent, 455 struct uacce_interface *interface) 456 { 457 unsigned int flags = interface->flags; 458 struct uacce_device *uacce; 459 int ret; 460 461 uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL); 462 if (!uacce) 463 return ERR_PTR(-ENOMEM); 464 465 flags = uacce_enable_sva(parent, flags); 466 467 uacce->parent = parent; 468 uacce->flags = flags; 469 uacce->ops = interface->ops; 470 471 ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b, 472 GFP_KERNEL); 473 if (ret < 0) 474 goto err_with_uacce; 475 476 INIT_LIST_HEAD(&uacce->queues); 477 mutex_init(&uacce->mutex); 478 device_initialize(&uacce->dev); 479 uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id); 480 uacce->dev.class = uacce_class; 481 uacce->dev.groups = uacce_dev_groups; 482 uacce->dev.parent = uacce->parent; 483 uacce->dev.release = uacce_release; 484 dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id); 485 486 return uacce; 487 488 err_with_uacce: 489 uacce_disable_sva(uacce); 490 kfree(uacce); 491 return ERR_PTR(ret); 492 } 493 EXPORT_SYMBOL_GPL(uacce_alloc); 494 495 /** 496 * uacce_register() - add the accelerator to cdev and export to user space 497 * @uacce: The initialized uacce device 498 * 499 * Return 0 if register succeeded, or an error. 500 */ 501 int uacce_register(struct uacce_device *uacce) 502 { 503 if (!uacce) 504 return -ENODEV; 505 506 uacce->cdev = cdev_alloc(); 507 if (!uacce->cdev) 508 return -ENOMEM; 509 510 uacce->cdev->ops = &uacce_fops; 511 uacce->cdev->owner = THIS_MODULE; 512 513 return cdev_device_add(uacce->cdev, &uacce->dev); 514 } 515 EXPORT_SYMBOL_GPL(uacce_register); 516 517 /** 518 * uacce_remove() - remove the accelerator 519 * @uacce: the accelerator to remove 520 */ 521 void uacce_remove(struct uacce_device *uacce) 522 { 523 struct uacce_queue *q, *next_q; 524 525 if (!uacce) 526 return; 527 /* 528 * unmap remaining mapping from user space, preventing user still 529 * access the mmaped area while parent device is already removed 530 */ 531 if (uacce->inode) 532 unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1); 533 534 /* 535 * uacce_fops_open() may be running concurrently, even after we remove 536 * the cdev. Holding uacce->mutex ensures that open() does not obtain a 537 * removed uacce device. 538 */ 539 mutex_lock(&uacce->mutex); 540 /* ensure no open queue remains */ 541 list_for_each_entry_safe(q, next_q, &uacce->queues, list) { 542 /* 543 * Taking q->mutex ensures that fops do not use the defunct 544 * uacce->ops after the queue is disabled. 545 */ 546 mutex_lock(&q->mutex); 547 uacce_put_queue(q); 548 mutex_unlock(&q->mutex); 549 uacce_unbind_queue(q); 550 } 551 552 /* disable sva now since no opened queues */ 553 uacce_disable_sva(uacce); 554 555 if (uacce->cdev) 556 cdev_device_del(uacce->cdev, &uacce->dev); 557 xa_erase(&uacce_xa, uacce->dev_id); 558 /* 559 * uacce exists as long as there are open fds, but ops will be freed 560 * now. Ensure that bugs cause NULL deref rather than use-after-free. 561 */ 562 uacce->ops = NULL; 563 uacce->parent = NULL; 564 mutex_unlock(&uacce->mutex); 565 put_device(&uacce->dev); 566 } 567 EXPORT_SYMBOL_GPL(uacce_remove); 568 569 static int __init uacce_init(void) 570 { 571 int ret; 572 573 uacce_class = class_create(THIS_MODULE, UACCE_NAME); 574 if (IS_ERR(uacce_class)) 575 return PTR_ERR(uacce_class); 576 577 ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME); 578 if (ret) 579 class_destroy(uacce_class); 580 581 return ret; 582 } 583 584 static __exit void uacce_exit(void) 585 { 586 unregister_chrdev_region(uacce_devt, MINORMASK); 587 class_destroy(uacce_class); 588 } 589 590 subsys_initcall(uacce_init); 591 module_exit(uacce_exit); 592 593 MODULE_LICENSE("GPL"); 594 MODULE_AUTHOR("HiSilicon Tech. Co., Ltd."); 595 MODULE_DESCRIPTION("Accelerator interface for Userland applications"); 596