1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/compat.h> 3 #include <linux/dma-mapping.h> 4 #include <linux/iommu.h> 5 #include <linux/module.h> 6 #include <linux/poll.h> 7 #include <linux/slab.h> 8 #include <linux/uacce.h> 9 10 static struct class *uacce_class; 11 static dev_t uacce_devt; 12 static DEFINE_XARRAY_ALLOC(uacce_xa); 13 14 /* 15 * If the parent driver or the device disappears, the queue state is invalid and 16 * ops are not usable anymore. 17 */ 18 static bool uacce_queue_is_valid(struct uacce_queue *q) 19 { 20 return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED; 21 } 22 23 static int uacce_start_queue(struct uacce_queue *q) 24 { 25 int ret; 26 27 if (q->state != UACCE_Q_INIT) 28 return -EINVAL; 29 30 if (q->uacce->ops->start_queue) { 31 ret = q->uacce->ops->start_queue(q); 32 if (ret < 0) 33 return ret; 34 } 35 36 q->state = UACCE_Q_STARTED; 37 return 0; 38 } 39 40 static int uacce_put_queue(struct uacce_queue *q) 41 { 42 struct uacce_device *uacce = q->uacce; 43 44 if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) 45 uacce->ops->stop_queue(q); 46 47 if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) && 48 uacce->ops->put_queue) 49 uacce->ops->put_queue(q); 50 51 q->state = UACCE_Q_ZOMBIE; 52 53 return 0; 54 } 55 56 static long uacce_fops_unl_ioctl(struct file *filep, 57 unsigned int cmd, unsigned long arg) 58 { 59 struct uacce_queue *q = filep->private_data; 60 struct uacce_device *uacce = q->uacce; 61 long ret = -ENXIO; 62 63 /* 64 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from 65 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which 66 * gets called with mmap_lock held, by taking uacce->mutex instead of 67 * q->mutex. Doing this in uacce_fops_mmap() is not possible because 68 * uacce_fops_open() calls iommu_sva_bind_device(), which takes 69 * mmap_lock, while holding uacce->mutex. 70 */ 71 mutex_lock(&uacce->mutex); 72 if (!uacce_queue_is_valid(q)) 73 goto out_unlock; 74 75 switch (cmd) { 76 case UACCE_CMD_START_Q: 77 ret = uacce_start_queue(q); 78 break; 79 case UACCE_CMD_PUT_Q: 80 ret = uacce_put_queue(q); 81 break; 82 default: 83 if (uacce->ops->ioctl) 84 ret = uacce->ops->ioctl(q, cmd, arg); 85 else 86 ret = -EINVAL; 87 } 88 out_unlock: 89 mutex_unlock(&uacce->mutex); 90 return ret; 91 } 92 93 #ifdef CONFIG_COMPAT 94 static long uacce_fops_compat_ioctl(struct file *filep, 95 unsigned int cmd, unsigned long arg) 96 { 97 arg = (unsigned long)compat_ptr(arg); 98 99 return uacce_fops_unl_ioctl(filep, cmd, arg); 100 } 101 #endif 102 103 static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q) 104 { 105 u32 pasid; 106 struct iommu_sva *handle; 107 108 if (!(uacce->flags & UACCE_DEV_SVA)) 109 return 0; 110 111 handle = iommu_sva_bind_device(uacce->parent, current->mm); 112 if (IS_ERR(handle)) 113 return PTR_ERR(handle); 114 115 pasid = iommu_sva_get_pasid(handle); 116 if (pasid == IOMMU_PASID_INVALID) { 117 iommu_sva_unbind_device(handle); 118 return -ENODEV; 119 } 120 121 q->handle = handle; 122 q->pasid = pasid; 123 return 0; 124 } 125 126 static void uacce_unbind_queue(struct uacce_queue *q) 127 { 128 if (!q->handle) 129 return; 130 iommu_sva_unbind_device(q->handle); 131 q->handle = NULL; 132 } 133 134 static int uacce_fops_open(struct inode *inode, struct file *filep) 135 { 136 struct uacce_device *uacce; 137 struct uacce_queue *q; 138 int ret; 139 140 uacce = xa_load(&uacce_xa, iminor(inode)); 141 if (!uacce) 142 return -ENODEV; 143 144 q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL); 145 if (!q) 146 return -ENOMEM; 147 148 mutex_lock(&uacce->mutex); 149 150 if (!uacce->parent) { 151 ret = -EINVAL; 152 goto out_with_mem; 153 } 154 155 ret = uacce_bind_queue(uacce, q); 156 if (ret) 157 goto out_with_mem; 158 159 q->uacce = uacce; 160 161 if (uacce->ops->get_queue) { 162 ret = uacce->ops->get_queue(uacce, q->pasid, q); 163 if (ret < 0) 164 goto out_with_bond; 165 } 166 167 init_waitqueue_head(&q->wait); 168 filep->private_data = q; 169 q->state = UACCE_Q_INIT; 170 q->mapping = filep->f_mapping; 171 mutex_init(&q->mutex); 172 list_add(&q->list, &uacce->queues); 173 mutex_unlock(&uacce->mutex); 174 175 return 0; 176 177 out_with_bond: 178 uacce_unbind_queue(q); 179 out_with_mem: 180 kfree(q); 181 mutex_unlock(&uacce->mutex); 182 return ret; 183 } 184 185 static int uacce_fops_release(struct inode *inode, struct file *filep) 186 { 187 struct uacce_queue *q = filep->private_data; 188 struct uacce_device *uacce = q->uacce; 189 190 mutex_lock(&uacce->mutex); 191 uacce_put_queue(q); 192 uacce_unbind_queue(q); 193 list_del(&q->list); 194 mutex_unlock(&uacce->mutex); 195 kfree(q); 196 197 return 0; 198 } 199 200 static void uacce_vma_close(struct vm_area_struct *vma) 201 { 202 struct uacce_queue *q = vma->vm_private_data; 203 204 if (vma->vm_pgoff < UACCE_MAX_REGION) { 205 struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff]; 206 207 mutex_lock(&q->mutex); 208 q->qfrs[vma->vm_pgoff] = NULL; 209 mutex_unlock(&q->mutex); 210 kfree(qfr); 211 } 212 } 213 214 static const struct vm_operations_struct uacce_vm_ops = { 215 .close = uacce_vma_close, 216 }; 217 218 static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) 219 { 220 struct uacce_queue *q = filep->private_data; 221 struct uacce_device *uacce = q->uacce; 222 struct uacce_qfile_region *qfr; 223 enum uacce_qfrt type = UACCE_MAX_REGION; 224 int ret = 0; 225 226 if (vma->vm_pgoff < UACCE_MAX_REGION) 227 type = vma->vm_pgoff; 228 else 229 return -EINVAL; 230 231 qfr = kzalloc(sizeof(*qfr), GFP_KERNEL); 232 if (!qfr) 233 return -ENOMEM; 234 235 vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK); 236 vma->vm_ops = &uacce_vm_ops; 237 vma->vm_private_data = q; 238 qfr->type = type; 239 240 mutex_lock(&q->mutex); 241 if (!uacce_queue_is_valid(q)) { 242 ret = -ENXIO; 243 goto out_with_lock; 244 } 245 246 if (q->qfrs[type]) { 247 ret = -EEXIST; 248 goto out_with_lock; 249 } 250 251 switch (type) { 252 case UACCE_QFRT_MMIO: 253 case UACCE_QFRT_DUS: 254 if (!uacce->ops->mmap) { 255 ret = -EINVAL; 256 goto out_with_lock; 257 } 258 259 ret = uacce->ops->mmap(q, vma, qfr); 260 if (ret) 261 goto out_with_lock; 262 break; 263 264 default: 265 ret = -EINVAL; 266 goto out_with_lock; 267 } 268 269 q->qfrs[type] = qfr; 270 mutex_unlock(&q->mutex); 271 272 return ret; 273 274 out_with_lock: 275 mutex_unlock(&q->mutex); 276 kfree(qfr); 277 return ret; 278 } 279 280 static __poll_t uacce_fops_poll(struct file *file, poll_table *wait) 281 { 282 struct uacce_queue *q = file->private_data; 283 struct uacce_device *uacce = q->uacce; 284 __poll_t ret = 0; 285 286 mutex_lock(&q->mutex); 287 if (!uacce_queue_is_valid(q)) 288 goto out_unlock; 289 290 poll_wait(file, &q->wait, wait); 291 292 if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q)) 293 ret = EPOLLIN | EPOLLRDNORM; 294 295 out_unlock: 296 mutex_unlock(&q->mutex); 297 return ret; 298 } 299 300 static const struct file_operations uacce_fops = { 301 .owner = THIS_MODULE, 302 .open = uacce_fops_open, 303 .release = uacce_fops_release, 304 .unlocked_ioctl = uacce_fops_unl_ioctl, 305 #ifdef CONFIG_COMPAT 306 .compat_ioctl = uacce_fops_compat_ioctl, 307 #endif 308 .mmap = uacce_fops_mmap, 309 .poll = uacce_fops_poll, 310 }; 311 312 #define to_uacce_device(dev) container_of(dev, struct uacce_device, dev) 313 314 static ssize_t api_show(struct device *dev, 315 struct device_attribute *attr, char *buf) 316 { 317 struct uacce_device *uacce = to_uacce_device(dev); 318 319 return sysfs_emit(buf, "%s\n", uacce->api_ver); 320 } 321 322 static ssize_t flags_show(struct device *dev, 323 struct device_attribute *attr, char *buf) 324 { 325 struct uacce_device *uacce = to_uacce_device(dev); 326 327 return sysfs_emit(buf, "%u\n", uacce->flags); 328 } 329 330 static ssize_t available_instances_show(struct device *dev, 331 struct device_attribute *attr, 332 char *buf) 333 { 334 struct uacce_device *uacce = to_uacce_device(dev); 335 336 if (!uacce->ops->get_available_instances) 337 return -ENODEV; 338 339 return sysfs_emit(buf, "%d\n", 340 uacce->ops->get_available_instances(uacce)); 341 } 342 343 static ssize_t algorithms_show(struct device *dev, 344 struct device_attribute *attr, char *buf) 345 { 346 struct uacce_device *uacce = to_uacce_device(dev); 347 348 return sysfs_emit(buf, "%s\n", uacce->algs); 349 } 350 351 static ssize_t region_mmio_size_show(struct device *dev, 352 struct device_attribute *attr, char *buf) 353 { 354 struct uacce_device *uacce = to_uacce_device(dev); 355 356 return sysfs_emit(buf, "%lu\n", 357 uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT); 358 } 359 360 static ssize_t region_dus_size_show(struct device *dev, 361 struct device_attribute *attr, char *buf) 362 { 363 struct uacce_device *uacce = to_uacce_device(dev); 364 365 return sysfs_emit(buf, "%lu\n", 366 uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT); 367 } 368 369 static ssize_t isolate_show(struct device *dev, 370 struct device_attribute *attr, char *buf) 371 { 372 struct uacce_device *uacce = to_uacce_device(dev); 373 374 return sysfs_emit(buf, "%d\n", uacce->ops->get_isolate_state(uacce)); 375 } 376 377 static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute *attr, char *buf) 378 { 379 struct uacce_device *uacce = to_uacce_device(dev); 380 u32 val; 381 382 val = uacce->ops->isolate_err_threshold_read(uacce); 383 384 return sysfs_emit(buf, "%u\n", val); 385 } 386 387 static ssize_t isolate_strategy_store(struct device *dev, struct device_attribute *attr, 388 const char *buf, size_t count) 389 { 390 struct uacce_device *uacce = to_uacce_device(dev); 391 unsigned long val; 392 int ret; 393 394 if (kstrtoul(buf, 0, &val) < 0) 395 return -EINVAL; 396 397 if (val > UACCE_MAX_ERR_THRESHOLD) 398 return -EINVAL; 399 400 ret = uacce->ops->isolate_err_threshold_write(uacce, val); 401 if (ret) 402 return ret; 403 404 return count; 405 } 406 407 static DEVICE_ATTR_RO(api); 408 static DEVICE_ATTR_RO(flags); 409 static DEVICE_ATTR_RO(available_instances); 410 static DEVICE_ATTR_RO(algorithms); 411 static DEVICE_ATTR_RO(region_mmio_size); 412 static DEVICE_ATTR_RO(region_dus_size); 413 static DEVICE_ATTR_RO(isolate); 414 static DEVICE_ATTR_RW(isolate_strategy); 415 416 static struct attribute *uacce_dev_attrs[] = { 417 &dev_attr_api.attr, 418 &dev_attr_flags.attr, 419 &dev_attr_available_instances.attr, 420 &dev_attr_algorithms.attr, 421 &dev_attr_region_mmio_size.attr, 422 &dev_attr_region_dus_size.attr, 423 &dev_attr_isolate.attr, 424 &dev_attr_isolate_strategy.attr, 425 NULL, 426 }; 427 428 static umode_t uacce_dev_is_visible(struct kobject *kobj, 429 struct attribute *attr, int n) 430 { 431 struct device *dev = kobj_to_dev(kobj); 432 struct uacce_device *uacce = to_uacce_device(dev); 433 434 if (((attr == &dev_attr_region_mmio_size.attr) && 435 (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) || 436 ((attr == &dev_attr_region_dus_size.attr) && 437 (!uacce->qf_pg_num[UACCE_QFRT_DUS]))) 438 return 0; 439 440 if (attr == &dev_attr_isolate_strategy.attr && 441 (!uacce->ops->isolate_err_threshold_read && 442 !uacce->ops->isolate_err_threshold_write)) 443 return 0; 444 445 if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state) 446 return 0; 447 448 return attr->mode; 449 } 450 451 static struct attribute_group uacce_dev_group = { 452 .is_visible = uacce_dev_is_visible, 453 .attrs = uacce_dev_attrs, 454 }; 455 456 __ATTRIBUTE_GROUPS(uacce_dev); 457 458 static void uacce_release(struct device *dev) 459 { 460 struct uacce_device *uacce = to_uacce_device(dev); 461 462 kfree(uacce); 463 } 464 465 static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags) 466 { 467 int ret; 468 469 if (!(flags & UACCE_DEV_SVA)) 470 return flags; 471 472 flags &= ~UACCE_DEV_SVA; 473 474 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF); 475 if (ret) { 476 dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret)); 477 return flags; 478 } 479 480 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA); 481 if (ret) { 482 dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret)); 483 iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF); 484 return flags; 485 } 486 487 return flags | UACCE_DEV_SVA; 488 } 489 490 static void uacce_disable_sva(struct uacce_device *uacce) 491 { 492 if (!(uacce->flags & UACCE_DEV_SVA)) 493 return; 494 495 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA); 496 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF); 497 } 498 499 /** 500 * uacce_alloc() - alloc an accelerator 501 * @parent: pointer of uacce parent device 502 * @interface: pointer of uacce_interface for register 503 * 504 * Returns uacce pointer if success and ERR_PTR if not 505 * Need check returned negotiated uacce->flags 506 */ 507 struct uacce_device *uacce_alloc(struct device *parent, 508 struct uacce_interface *interface) 509 { 510 unsigned int flags = interface->flags; 511 struct uacce_device *uacce; 512 int ret; 513 514 uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL); 515 if (!uacce) 516 return ERR_PTR(-ENOMEM); 517 518 flags = uacce_enable_sva(parent, flags); 519 520 uacce->parent = parent; 521 uacce->flags = flags; 522 uacce->ops = interface->ops; 523 524 ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b, 525 GFP_KERNEL); 526 if (ret < 0) 527 goto err_with_uacce; 528 529 INIT_LIST_HEAD(&uacce->queues); 530 mutex_init(&uacce->mutex); 531 device_initialize(&uacce->dev); 532 uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id); 533 uacce->dev.class = uacce_class; 534 uacce->dev.groups = uacce_dev_groups; 535 uacce->dev.parent = uacce->parent; 536 uacce->dev.release = uacce_release; 537 dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id); 538 539 return uacce; 540 541 err_with_uacce: 542 uacce_disable_sva(uacce); 543 kfree(uacce); 544 return ERR_PTR(ret); 545 } 546 EXPORT_SYMBOL_GPL(uacce_alloc); 547 548 /** 549 * uacce_register() - add the accelerator to cdev and export to user space 550 * @uacce: The initialized uacce device 551 * 552 * Return 0 if register succeeded, or an error. 553 */ 554 int uacce_register(struct uacce_device *uacce) 555 { 556 if (!uacce) 557 return -ENODEV; 558 559 uacce->cdev = cdev_alloc(); 560 if (!uacce->cdev) 561 return -ENOMEM; 562 563 uacce->cdev->ops = &uacce_fops; 564 uacce->cdev->owner = THIS_MODULE; 565 566 return cdev_device_add(uacce->cdev, &uacce->dev); 567 } 568 EXPORT_SYMBOL_GPL(uacce_register); 569 570 /** 571 * uacce_remove() - remove the accelerator 572 * @uacce: the accelerator to remove 573 */ 574 void uacce_remove(struct uacce_device *uacce) 575 { 576 struct uacce_queue *q, *next_q; 577 578 if (!uacce) 579 return; 580 581 /* 582 * uacce_fops_open() may be running concurrently, even after we remove 583 * the cdev. Holding uacce->mutex ensures that open() does not obtain a 584 * removed uacce device. 585 */ 586 mutex_lock(&uacce->mutex); 587 /* ensure no open queue remains */ 588 list_for_each_entry_safe(q, next_q, &uacce->queues, list) { 589 /* 590 * Taking q->mutex ensures that fops do not use the defunct 591 * uacce->ops after the queue is disabled. 592 */ 593 mutex_lock(&q->mutex); 594 uacce_put_queue(q); 595 mutex_unlock(&q->mutex); 596 uacce_unbind_queue(q); 597 598 /* 599 * unmap remaining mapping from user space, preventing user still 600 * access the mmaped area while parent device is already removed 601 */ 602 unmap_mapping_range(q->mapping, 0, 0, 1); 603 } 604 605 /* disable sva now since no opened queues */ 606 uacce_disable_sva(uacce); 607 608 if (uacce->cdev) 609 cdev_device_del(uacce->cdev, &uacce->dev); 610 xa_erase(&uacce_xa, uacce->dev_id); 611 /* 612 * uacce exists as long as there are open fds, but ops will be freed 613 * now. Ensure that bugs cause NULL deref rather than use-after-free. 614 */ 615 uacce->ops = NULL; 616 uacce->parent = NULL; 617 mutex_unlock(&uacce->mutex); 618 put_device(&uacce->dev); 619 } 620 EXPORT_SYMBOL_GPL(uacce_remove); 621 622 static int __init uacce_init(void) 623 { 624 int ret; 625 626 uacce_class = class_create(UACCE_NAME); 627 if (IS_ERR(uacce_class)) 628 return PTR_ERR(uacce_class); 629 630 ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME); 631 if (ret) 632 class_destroy(uacce_class); 633 634 return ret; 635 } 636 637 static __exit void uacce_exit(void) 638 { 639 unregister_chrdev_region(uacce_devt, MINORMASK); 640 class_destroy(uacce_class); 641 } 642 643 subsys_initcall(uacce_init); 644 module_exit(uacce_exit); 645 646 MODULE_LICENSE("GPL"); 647 MODULE_AUTHOR("HiSilicon Tech. Co., Ltd."); 648 MODULE_DESCRIPTION("Accelerator interface for Userland applications"); 649