1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/compat.h> 3 #include <linux/dma-mapping.h> 4 #include <linux/iommu.h> 5 #include <linux/module.h> 6 #include <linux/poll.h> 7 #include <linux/slab.h> 8 #include <linux/uacce.h> 9 10 static struct class *uacce_class; 11 static dev_t uacce_devt; 12 static DEFINE_MUTEX(uacce_mutex); 13 static DEFINE_XARRAY_ALLOC(uacce_xa); 14 15 static int uacce_start_queue(struct uacce_queue *q) 16 { 17 int ret = 0; 18 19 mutex_lock(&uacce_mutex); 20 21 if (q->state != UACCE_Q_INIT) { 22 ret = -EINVAL; 23 goto out_with_lock; 24 } 25 26 if (q->uacce->ops->start_queue) { 27 ret = q->uacce->ops->start_queue(q); 28 if (ret < 0) 29 goto out_with_lock; 30 } 31 32 q->state = UACCE_Q_STARTED; 33 34 out_with_lock: 35 mutex_unlock(&uacce_mutex); 36 37 return ret; 38 } 39 40 static int uacce_put_queue(struct uacce_queue *q) 41 { 42 struct uacce_device *uacce = q->uacce; 43 44 mutex_lock(&uacce_mutex); 45 46 if (q->state == UACCE_Q_ZOMBIE) 47 goto out; 48 49 if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) 50 uacce->ops->stop_queue(q); 51 52 if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) && 53 uacce->ops->put_queue) 54 uacce->ops->put_queue(q); 55 56 q->state = UACCE_Q_ZOMBIE; 57 out: 58 mutex_unlock(&uacce_mutex); 59 60 return 0; 61 } 62 63 static long uacce_fops_unl_ioctl(struct file *filep, 64 unsigned int cmd, unsigned long arg) 65 { 66 struct uacce_queue *q = filep->private_data; 67 struct uacce_device *uacce = q->uacce; 68 69 switch (cmd) { 70 case UACCE_CMD_START_Q: 71 return uacce_start_queue(q); 72 73 case UACCE_CMD_PUT_Q: 74 return uacce_put_queue(q); 75 76 default: 77 if (!uacce->ops->ioctl) 78 return -EINVAL; 79 80 return uacce->ops->ioctl(q, cmd, arg); 81 } 82 } 83 84 #ifdef CONFIG_COMPAT 85 static long uacce_fops_compat_ioctl(struct file *filep, 86 unsigned int cmd, unsigned long arg) 87 { 88 arg = (unsigned long)compat_ptr(arg); 89 90 return uacce_fops_unl_ioctl(filep, cmd, arg); 91 } 92 #endif 93 94 static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q) 95 { 96 u32 pasid; 97 struct iommu_sva *handle; 98 99 if (!(uacce->flags & UACCE_DEV_SVA)) 100 return 0; 101 102 handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL); 103 if (IS_ERR(handle)) 104 return PTR_ERR(handle); 105 106 pasid = iommu_sva_get_pasid(handle); 107 if (pasid == IOMMU_PASID_INVALID) { 108 iommu_sva_unbind_device(handle); 109 return -ENODEV; 110 } 111 112 q->handle = handle; 113 q->pasid = pasid; 114 return 0; 115 } 116 117 static void uacce_unbind_queue(struct uacce_queue *q) 118 { 119 if (!q->handle) 120 return; 121 iommu_sva_unbind_device(q->handle); 122 q->handle = NULL; 123 } 124 125 static int uacce_fops_open(struct inode *inode, struct file *filep) 126 { 127 struct uacce_device *uacce; 128 struct uacce_queue *q; 129 int ret; 130 131 uacce = xa_load(&uacce_xa, iminor(inode)); 132 if (!uacce) 133 return -ENODEV; 134 135 q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL); 136 if (!q) 137 return -ENOMEM; 138 139 ret = uacce_bind_queue(uacce, q); 140 if (ret) 141 goto out_with_mem; 142 143 q->uacce = uacce; 144 145 if (uacce->ops->get_queue) { 146 ret = uacce->ops->get_queue(uacce, q->pasid, q); 147 if (ret < 0) 148 goto out_with_bond; 149 } 150 151 init_waitqueue_head(&q->wait); 152 filep->private_data = q; 153 uacce->inode = inode; 154 q->state = UACCE_Q_INIT; 155 156 mutex_lock(&uacce->queues_lock); 157 list_add(&q->list, &uacce->queues); 158 mutex_unlock(&uacce->queues_lock); 159 160 return 0; 161 162 out_with_bond: 163 uacce_unbind_queue(q); 164 out_with_mem: 165 kfree(q); 166 return ret; 167 } 168 169 static int uacce_fops_release(struct inode *inode, struct file *filep) 170 { 171 struct uacce_queue *q = filep->private_data; 172 173 mutex_lock(&q->uacce->queues_lock); 174 list_del(&q->list); 175 mutex_unlock(&q->uacce->queues_lock); 176 uacce_put_queue(q); 177 uacce_unbind_queue(q); 178 kfree(q); 179 180 return 0; 181 } 182 183 static void uacce_vma_close(struct vm_area_struct *vma) 184 { 185 struct uacce_queue *q = vma->vm_private_data; 186 struct uacce_qfile_region *qfr = NULL; 187 188 if (vma->vm_pgoff < UACCE_MAX_REGION) 189 qfr = q->qfrs[vma->vm_pgoff]; 190 191 kfree(qfr); 192 } 193 194 static const struct vm_operations_struct uacce_vm_ops = { 195 .close = uacce_vma_close, 196 }; 197 198 static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) 199 { 200 struct uacce_queue *q = filep->private_data; 201 struct uacce_device *uacce = q->uacce; 202 struct uacce_qfile_region *qfr; 203 enum uacce_qfrt type = UACCE_MAX_REGION; 204 int ret = 0; 205 206 if (vma->vm_pgoff < UACCE_MAX_REGION) 207 type = vma->vm_pgoff; 208 else 209 return -EINVAL; 210 211 qfr = kzalloc(sizeof(*qfr), GFP_KERNEL); 212 if (!qfr) 213 return -ENOMEM; 214 215 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK; 216 vma->vm_ops = &uacce_vm_ops; 217 vma->vm_private_data = q; 218 qfr->type = type; 219 220 mutex_lock(&uacce_mutex); 221 222 if (q->state != UACCE_Q_INIT && q->state != UACCE_Q_STARTED) { 223 ret = -EINVAL; 224 goto out_with_lock; 225 } 226 227 if (q->qfrs[type]) { 228 ret = -EEXIST; 229 goto out_with_lock; 230 } 231 232 switch (type) { 233 case UACCE_QFRT_MMIO: 234 case UACCE_QFRT_DUS: 235 if (!uacce->ops->mmap) { 236 ret = -EINVAL; 237 goto out_with_lock; 238 } 239 240 ret = uacce->ops->mmap(q, vma, qfr); 241 if (ret) 242 goto out_with_lock; 243 break; 244 245 default: 246 ret = -EINVAL; 247 goto out_with_lock; 248 } 249 250 q->qfrs[type] = qfr; 251 mutex_unlock(&uacce_mutex); 252 253 return ret; 254 255 out_with_lock: 256 mutex_unlock(&uacce_mutex); 257 kfree(qfr); 258 return ret; 259 } 260 261 static __poll_t uacce_fops_poll(struct file *file, poll_table *wait) 262 { 263 struct uacce_queue *q = file->private_data; 264 struct uacce_device *uacce = q->uacce; 265 266 poll_wait(file, &q->wait, wait); 267 if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q)) 268 return EPOLLIN | EPOLLRDNORM; 269 270 return 0; 271 } 272 273 static const struct file_operations uacce_fops = { 274 .owner = THIS_MODULE, 275 .open = uacce_fops_open, 276 .release = uacce_fops_release, 277 .unlocked_ioctl = uacce_fops_unl_ioctl, 278 #ifdef CONFIG_COMPAT 279 .compat_ioctl = uacce_fops_compat_ioctl, 280 #endif 281 .mmap = uacce_fops_mmap, 282 .poll = uacce_fops_poll, 283 }; 284 285 #define to_uacce_device(dev) container_of(dev, struct uacce_device, dev) 286 287 static ssize_t api_show(struct device *dev, 288 struct device_attribute *attr, char *buf) 289 { 290 struct uacce_device *uacce = to_uacce_device(dev); 291 292 return sysfs_emit(buf, "%s\n", uacce->api_ver); 293 } 294 295 static ssize_t flags_show(struct device *dev, 296 struct device_attribute *attr, char *buf) 297 { 298 struct uacce_device *uacce = to_uacce_device(dev); 299 300 return sysfs_emit(buf, "%u\n", uacce->flags); 301 } 302 303 static ssize_t available_instances_show(struct device *dev, 304 struct device_attribute *attr, 305 char *buf) 306 { 307 struct uacce_device *uacce = to_uacce_device(dev); 308 309 if (!uacce->ops->get_available_instances) 310 return -ENODEV; 311 312 return sysfs_emit(buf, "%d\n", 313 uacce->ops->get_available_instances(uacce)); 314 } 315 316 static ssize_t algorithms_show(struct device *dev, 317 struct device_attribute *attr, char *buf) 318 { 319 struct uacce_device *uacce = to_uacce_device(dev); 320 321 return sysfs_emit(buf, "%s\n", uacce->algs); 322 } 323 324 static ssize_t region_mmio_size_show(struct device *dev, 325 struct device_attribute *attr, char *buf) 326 { 327 struct uacce_device *uacce = to_uacce_device(dev); 328 329 return sysfs_emit(buf, "%lu\n", 330 uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT); 331 } 332 333 static ssize_t region_dus_size_show(struct device *dev, 334 struct device_attribute *attr, char *buf) 335 { 336 struct uacce_device *uacce = to_uacce_device(dev); 337 338 return sysfs_emit(buf, "%lu\n", 339 uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT); 340 } 341 342 static DEVICE_ATTR_RO(api); 343 static DEVICE_ATTR_RO(flags); 344 static DEVICE_ATTR_RO(available_instances); 345 static DEVICE_ATTR_RO(algorithms); 346 static DEVICE_ATTR_RO(region_mmio_size); 347 static DEVICE_ATTR_RO(region_dus_size); 348 349 static struct attribute *uacce_dev_attrs[] = { 350 &dev_attr_api.attr, 351 &dev_attr_flags.attr, 352 &dev_attr_available_instances.attr, 353 &dev_attr_algorithms.attr, 354 &dev_attr_region_mmio_size.attr, 355 &dev_attr_region_dus_size.attr, 356 NULL, 357 }; 358 359 static umode_t uacce_dev_is_visible(struct kobject *kobj, 360 struct attribute *attr, int n) 361 { 362 struct device *dev = kobj_to_dev(kobj); 363 struct uacce_device *uacce = to_uacce_device(dev); 364 365 if (((attr == &dev_attr_region_mmio_size.attr) && 366 (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) || 367 ((attr == &dev_attr_region_dus_size.attr) && 368 (!uacce->qf_pg_num[UACCE_QFRT_DUS]))) 369 return 0; 370 371 return attr->mode; 372 } 373 374 static struct attribute_group uacce_dev_group = { 375 .is_visible = uacce_dev_is_visible, 376 .attrs = uacce_dev_attrs, 377 }; 378 379 __ATTRIBUTE_GROUPS(uacce_dev); 380 381 static void uacce_release(struct device *dev) 382 { 383 struct uacce_device *uacce = to_uacce_device(dev); 384 385 kfree(uacce); 386 } 387 388 static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags) 389 { 390 int ret; 391 392 if (!(flags & UACCE_DEV_SVA)) 393 return flags; 394 395 flags &= ~UACCE_DEV_SVA; 396 397 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF); 398 if (ret) { 399 dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret)); 400 return flags; 401 } 402 403 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA); 404 if (ret) { 405 dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret)); 406 iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF); 407 return flags; 408 } 409 410 return flags | UACCE_DEV_SVA; 411 } 412 413 static void uacce_disable_sva(struct uacce_device *uacce) 414 { 415 if (!(uacce->flags & UACCE_DEV_SVA)) 416 return; 417 418 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA); 419 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF); 420 } 421 422 /** 423 * uacce_alloc() - alloc an accelerator 424 * @parent: pointer of uacce parent device 425 * @interface: pointer of uacce_interface for register 426 * 427 * Returns uacce pointer if success and ERR_PTR if not 428 * Need check returned negotiated uacce->flags 429 */ 430 struct uacce_device *uacce_alloc(struct device *parent, 431 struct uacce_interface *interface) 432 { 433 unsigned int flags = interface->flags; 434 struct uacce_device *uacce; 435 int ret; 436 437 uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL); 438 if (!uacce) 439 return ERR_PTR(-ENOMEM); 440 441 flags = uacce_enable_sva(parent, flags); 442 443 uacce->parent = parent; 444 uacce->flags = flags; 445 uacce->ops = interface->ops; 446 447 ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b, 448 GFP_KERNEL); 449 if (ret < 0) 450 goto err_with_uacce; 451 452 INIT_LIST_HEAD(&uacce->queues); 453 mutex_init(&uacce->queues_lock); 454 device_initialize(&uacce->dev); 455 uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id); 456 uacce->dev.class = uacce_class; 457 uacce->dev.groups = uacce_dev_groups; 458 uacce->dev.parent = uacce->parent; 459 uacce->dev.release = uacce_release; 460 dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id); 461 462 return uacce; 463 464 err_with_uacce: 465 uacce_disable_sva(uacce); 466 kfree(uacce); 467 return ERR_PTR(ret); 468 } 469 EXPORT_SYMBOL_GPL(uacce_alloc); 470 471 /** 472 * uacce_register() - add the accelerator to cdev and export to user space 473 * @uacce: The initialized uacce device 474 * 475 * Return 0 if register succeeded, or an error. 476 */ 477 int uacce_register(struct uacce_device *uacce) 478 { 479 if (!uacce) 480 return -ENODEV; 481 482 uacce->cdev = cdev_alloc(); 483 if (!uacce->cdev) 484 return -ENOMEM; 485 486 uacce->cdev->ops = &uacce_fops; 487 uacce->cdev->owner = THIS_MODULE; 488 489 return cdev_device_add(uacce->cdev, &uacce->dev); 490 } 491 EXPORT_SYMBOL_GPL(uacce_register); 492 493 /** 494 * uacce_remove() - remove the accelerator 495 * @uacce: the accelerator to remove 496 */ 497 void uacce_remove(struct uacce_device *uacce) 498 { 499 struct uacce_queue *q, *next_q; 500 501 if (!uacce) 502 return; 503 /* 504 * unmap remaining mapping from user space, preventing user still 505 * access the mmaped area while parent device is already removed 506 */ 507 if (uacce->inode) 508 unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1); 509 510 /* ensure no open queue remains */ 511 mutex_lock(&uacce->queues_lock); 512 list_for_each_entry_safe(q, next_q, &uacce->queues, list) { 513 uacce_put_queue(q); 514 uacce_unbind_queue(q); 515 } 516 mutex_unlock(&uacce->queues_lock); 517 518 /* disable sva now since no opened queues */ 519 uacce_disable_sva(uacce); 520 521 if (uacce->cdev) 522 cdev_device_del(uacce->cdev, &uacce->dev); 523 xa_erase(&uacce_xa, uacce->dev_id); 524 put_device(&uacce->dev); 525 } 526 EXPORT_SYMBOL_GPL(uacce_remove); 527 528 static int __init uacce_init(void) 529 { 530 int ret; 531 532 uacce_class = class_create(THIS_MODULE, UACCE_NAME); 533 if (IS_ERR(uacce_class)) 534 return PTR_ERR(uacce_class); 535 536 ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME); 537 if (ret) 538 class_destroy(uacce_class); 539 540 return ret; 541 } 542 543 static __exit void uacce_exit(void) 544 { 545 unregister_chrdev_region(uacce_devt, MINORMASK); 546 class_destroy(uacce_class); 547 } 548 549 subsys_initcall(uacce_init); 550 module_exit(uacce_exit); 551 552 MODULE_LICENSE("GPL"); 553 MODULE_AUTHOR("HiSilicon Tech. Co., Ltd."); 554 MODULE_DESCRIPTION("Accelerator interface for Userland applications"); 555