1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/cdev.h> 11 #include <linux/fs.h> 12 #include <linux/poll.h> 13 #include <linux/iommu.h> 14 #include <linux/highmem.h> 15 #include <uapi/linux/idxd.h> 16 #include <linux/xarray.h> 17 #include "registers.h" 18 #include "idxd.h" 19 20 struct idxd_cdev_context { 21 const char *name; 22 dev_t devt; 23 struct ida minor_ida; 24 }; 25 26 /* 27 * Since user file names are global in DSA devices, define their ida's as 28 * global to avoid conflict file names. 29 */ 30 static DEFINE_IDA(file_ida); 31 static DEFINE_MUTEX(ida_lock); 32 33 /* 34 * ictx is an array based off of accelerator types. enum idxd_type 35 * is used as index 36 */ 37 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 38 { .name = "dsa" }, 39 { .name = "iax" } 40 }; 41 42 struct idxd_user_context { 43 struct idxd_wq *wq; 44 struct task_struct *task; 45 unsigned int pasid; 46 struct mm_struct *mm; 47 unsigned int flags; 48 struct iommu_sva *sva; 49 struct idxd_dev idxd_dev; 50 u64 counters[COUNTER_MAX]; 51 int id; 52 pid_t pid; 53 }; 54 55 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid); 56 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx); 57 58 static inline struct idxd_user_context *dev_to_uctx(struct device *dev) 59 { 60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 61 62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev); 63 } 64 65 static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf) 66 { 67 struct idxd_user_context *ctx = dev_to_uctx(dev); 68 69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]); 70 } 71 static DEVICE_ATTR_RO(cr_faults); 72 73 static ssize_t cr_fault_failures_show(struct device *dev, 74 struct device_attribute *attr, char *buf) 75 { 76 struct idxd_user_context *ctx = dev_to_uctx(dev); 77 78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]); 79 } 80 static DEVICE_ATTR_RO(cr_fault_failures); 81 82 static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) 83 { 84 struct idxd_user_context *ctx = dev_to_uctx(dev); 85 86 return sysfs_emit(buf, "%u\n", ctx->pid); 87 } 88 static DEVICE_ATTR_RO(pid); 89 90 static struct attribute *cdev_file_attributes[] = { 91 &dev_attr_cr_faults.attr, 92 &dev_attr_cr_fault_failures.attr, 93 &dev_attr_pid.attr, 94 NULL 95 }; 96 97 static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n) 98 { 99 struct device *dev = container_of(kobj, typeof(*dev), kobj); 100 struct idxd_user_context *ctx = dev_to_uctx(dev); 101 struct idxd_wq *wq = ctx->wq; 102 103 if (!wq_pasid_enabled(wq)) 104 return 0; 105 106 return a->mode; 107 } 108 109 static const struct attribute_group cdev_file_attribute_group = { 110 .attrs = cdev_file_attributes, 111 .is_visible = cdev_file_attr_visible, 112 }; 113 114 static const struct attribute_group *cdev_file_attribute_groups[] = { 115 &cdev_file_attribute_group, 116 NULL 117 }; 118 119 static void idxd_file_dev_release(struct device *dev) 120 { 121 struct idxd_user_context *ctx = dev_to_uctx(dev); 122 struct idxd_wq *wq = ctx->wq; 123 struct idxd_device *idxd = wq->idxd; 124 int rc; 125 126 mutex_lock(&ida_lock); 127 ida_free(&file_ida, ctx->id); 128 mutex_unlock(&ida_lock); 129 130 /* Wait for in-flight operations to complete. */ 131 if (wq_shared(wq)) { 132 idxd_device_drain_pasid(idxd, ctx->pasid); 133 } else { 134 if (device_user_pasid_enabled(idxd)) { 135 /* The wq disable in the disable pasid function will drain the wq */ 136 rc = idxd_wq_disable_pasid(wq); 137 if (rc < 0) 138 dev_err(dev, "wq disable pasid failed.\n"); 139 } else { 140 idxd_wq_drain(wq); 141 } 142 } 143 144 if (ctx->sva) { 145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid); 146 iommu_sva_unbind_device(ctx->sva); 147 idxd_xa_pasid_remove(ctx); 148 } 149 kfree(ctx); 150 mutex_lock(&wq->wq_lock); 151 idxd_wq_put(wq); 152 mutex_unlock(&wq->wq_lock); 153 } 154 155 static struct device_type idxd_cdev_file_type = { 156 .name = "idxd_file", 157 .release = idxd_file_dev_release, 158 .groups = cdev_file_attribute_groups, 159 }; 160 161 static void idxd_cdev_dev_release(struct device *dev) 162 { 163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); 164 struct idxd_cdev_context *cdev_ctx; 165 struct idxd_wq *wq = idxd_cdev->wq; 166 167 cdev_ctx = &ictx[wq->idxd->data->type]; 168 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 169 kfree(idxd_cdev); 170 } 171 172 static struct device_type idxd_cdev_device_type = { 173 .name = "idxd_cdev", 174 .release = idxd_cdev_dev_release, 175 }; 176 177 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 178 { 179 struct cdev *cdev = inode->i_cdev; 180 181 return container_of(cdev, struct idxd_cdev, cdev); 182 } 183 184 static inline struct idxd_wq *inode_wq(struct inode *inode) 185 { 186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 187 188 return idxd_cdev->wq; 189 } 190 191 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx) 192 { 193 struct idxd_wq *wq = ctx->wq; 194 void *ptr; 195 196 mutex_lock(&wq->uc_lock); 197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); 198 if (ptr != (void *)ctx) 199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", 200 ctx->pasid); 201 mutex_unlock(&wq->uc_lock); 202 } 203 204 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index) 205 { 206 struct idxd_user_context *ctx; 207 208 if (index >= COUNTER_MAX) 209 return; 210 211 mutex_lock(&wq->uc_lock); 212 ctx = xa_load(&wq->upasid_xa, pasid); 213 if (!ctx) { 214 mutex_unlock(&wq->uc_lock); 215 return; 216 } 217 ctx->counters[index]++; 218 mutex_unlock(&wq->uc_lock); 219 } 220 221 static int idxd_cdev_open(struct inode *inode, struct file *filp) 222 { 223 struct idxd_user_context *ctx; 224 struct idxd_device *idxd; 225 struct idxd_wq *wq; 226 struct device *dev, *fdev; 227 int rc = 0; 228 struct iommu_sva *sva; 229 unsigned int pasid; 230 struct idxd_cdev *idxd_cdev; 231 232 wq = inode_wq(inode); 233 idxd = wq->idxd; 234 dev = &idxd->pdev->dev; 235 236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 237 238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 239 if (!ctx) 240 return -ENOMEM; 241 242 mutex_lock(&wq->wq_lock); 243 244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 245 rc = -EBUSY; 246 goto failed; 247 } 248 249 ctx->wq = wq; 250 filp->private_data = ctx; 251 ctx->pid = current->pid; 252 253 if (device_user_pasid_enabled(idxd)) { 254 sva = iommu_sva_bind_device(dev, current->mm); 255 if (IS_ERR(sva)) { 256 rc = PTR_ERR(sva); 257 dev_err(dev, "pasid allocation failed: %d\n", rc); 258 goto failed; 259 } 260 261 pasid = iommu_sva_get_pasid(sva); 262 if (pasid == IOMMU_PASID_INVALID) { 263 rc = -EINVAL; 264 goto failed_get_pasid; 265 } 266 267 ctx->sva = sva; 268 ctx->pasid = pasid; 269 ctx->mm = current->mm; 270 271 mutex_lock(&wq->uc_lock); 272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); 273 mutex_unlock(&wq->uc_lock); 274 if (rc < 0) 275 dev_warn(dev, "PASID entry already exist in xarray.\n"); 276 277 if (wq_dedicated(wq)) { 278 rc = idxd_wq_set_pasid(wq, pasid); 279 if (rc < 0) { 280 iommu_sva_unbind_device(sva); 281 dev_err(dev, "wq set pasid failed: %d\n", rc); 282 goto failed_set_pasid; 283 } 284 } 285 } 286 287 idxd_cdev = wq->idxd_cdev; 288 mutex_lock(&ida_lock); 289 ctx->id = ida_alloc(&file_ida, GFP_KERNEL); 290 mutex_unlock(&ida_lock); 291 if (ctx->id < 0) { 292 dev_warn(dev, "ida alloc failure\n"); 293 goto failed_ida; 294 } 295 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; 296 fdev = user_ctx_dev(ctx); 297 device_initialize(fdev); 298 fdev->parent = cdev_dev(idxd_cdev); 299 fdev->bus = &dsa_bus_type; 300 fdev->type = &idxd_cdev_file_type; 301 302 rc = dev_set_name(fdev, "file%d", ctx->id); 303 if (rc < 0) { 304 dev_warn(dev, "set name failure\n"); 305 goto failed_dev_name; 306 } 307 308 rc = device_add(fdev); 309 if (rc < 0) { 310 dev_warn(dev, "file device add failure\n"); 311 goto failed_dev_add; 312 } 313 314 idxd_wq_get(wq); 315 mutex_unlock(&wq->wq_lock); 316 return 0; 317 318 failed_dev_add: 319 failed_dev_name: 320 put_device(fdev); 321 failed_ida: 322 failed_set_pasid: 323 if (device_user_pasid_enabled(idxd)) 324 idxd_xa_pasid_remove(ctx); 325 failed_get_pasid: 326 if (device_user_pasid_enabled(idxd)) 327 iommu_sva_unbind_device(sva); 328 failed: 329 mutex_unlock(&wq->wq_lock); 330 kfree(ctx); 331 return rc; 332 } 333 334 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) 335 { 336 struct idxd_device *idxd = wq->idxd; 337 struct idxd_evl *evl = idxd->evl; 338 union evl_status_reg status; 339 u16 h, t, size; 340 int ent_size = evl_ent_size(idxd); 341 struct __evl_entry *entry_head; 342 343 if (!evl) 344 return; 345 346 spin_lock(&evl->lock); 347 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); 348 t = status.tail; 349 h = evl->head; 350 size = evl->size; 351 352 while (h != t) { 353 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); 354 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) 355 set_bit(h, evl->bmap); 356 h = (h + 1) % size; 357 } 358 spin_unlock(&evl->lock); 359 360 drain_workqueue(wq->wq); 361 } 362 363 static int idxd_cdev_release(struct inode *node, struct file *filep) 364 { 365 struct idxd_user_context *ctx = filep->private_data; 366 struct idxd_wq *wq = ctx->wq; 367 struct idxd_device *idxd = wq->idxd; 368 struct device *dev = &idxd->pdev->dev; 369 370 dev_dbg(dev, "%s called\n", __func__); 371 filep->private_data = NULL; 372 373 device_unregister(user_ctx_dev(ctx)); 374 375 return 0; 376 } 377 378 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 379 const char *func) 380 { 381 struct device *dev = &wq->idxd->pdev->dev; 382 383 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 384 dev_info_ratelimited(dev, 385 "%s: %s: mapping too large: %lu\n", 386 current->comm, func, 387 vma->vm_end - vma->vm_start); 388 return -EINVAL; 389 } 390 391 return 0; 392 } 393 394 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 395 { 396 struct idxd_user_context *ctx = filp->private_data; 397 struct idxd_wq *wq = ctx->wq; 398 struct idxd_device *idxd = wq->idxd; 399 struct pci_dev *pdev = idxd->pdev; 400 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 401 unsigned long pfn; 402 int rc; 403 404 dev_dbg(&pdev->dev, "%s called\n", __func__); 405 rc = check_vma(wq, vma, __func__); 406 if (rc < 0) 407 return rc; 408 409 vm_flags_set(vma, VM_DONTCOPY); 410 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 411 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 412 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 413 vma->vm_private_data = ctx; 414 415 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 416 vma->vm_page_prot); 417 } 418 419 static __poll_t idxd_cdev_poll(struct file *filp, 420 struct poll_table_struct *wait) 421 { 422 struct idxd_user_context *ctx = filp->private_data; 423 struct idxd_wq *wq = ctx->wq; 424 struct idxd_device *idxd = wq->idxd; 425 __poll_t out = 0; 426 427 poll_wait(filp, &wq->err_queue, wait); 428 spin_lock(&idxd->dev_lock); 429 if (idxd->sw_err.valid) 430 out = EPOLLIN | EPOLLRDNORM; 431 spin_unlock(&idxd->dev_lock); 432 433 return out; 434 } 435 436 static const struct file_operations idxd_cdev_fops = { 437 .owner = THIS_MODULE, 438 .open = idxd_cdev_open, 439 .release = idxd_cdev_release, 440 .mmap = idxd_cdev_mmap, 441 .poll = idxd_cdev_poll, 442 }; 443 444 int idxd_cdev_get_major(struct idxd_device *idxd) 445 { 446 return MAJOR(ictx[idxd->data->type].devt); 447 } 448 449 int idxd_wq_add_cdev(struct idxd_wq *wq) 450 { 451 struct idxd_device *idxd = wq->idxd; 452 struct idxd_cdev *idxd_cdev; 453 struct cdev *cdev; 454 struct device *dev; 455 struct idxd_cdev_context *cdev_ctx; 456 int rc, minor; 457 458 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 459 if (!idxd_cdev) 460 return -ENOMEM; 461 462 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; 463 idxd_cdev->wq = wq; 464 cdev = &idxd_cdev->cdev; 465 dev = cdev_dev(idxd_cdev); 466 cdev_ctx = &ictx[wq->idxd->data->type]; 467 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 468 if (minor < 0) { 469 kfree(idxd_cdev); 470 return minor; 471 } 472 idxd_cdev->minor = minor; 473 474 device_initialize(dev); 475 dev->parent = wq_confdev(wq); 476 dev->bus = &dsa_bus_type; 477 dev->type = &idxd_cdev_device_type; 478 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 479 480 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); 481 if (rc < 0) 482 goto err; 483 484 wq->idxd_cdev = idxd_cdev; 485 cdev_init(cdev, &idxd_cdev_fops); 486 rc = cdev_device_add(cdev, dev); 487 if (rc) { 488 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 489 goto err; 490 } 491 492 return 0; 493 494 err: 495 put_device(dev); 496 wq->idxd_cdev = NULL; 497 return rc; 498 } 499 500 void idxd_wq_del_cdev(struct idxd_wq *wq) 501 { 502 struct idxd_cdev *idxd_cdev; 503 504 idxd_cdev = wq->idxd_cdev; 505 ida_destroy(&file_ida); 506 wq->idxd_cdev = NULL; 507 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); 508 put_device(cdev_dev(idxd_cdev)); 509 } 510 511 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) 512 { 513 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 514 struct idxd_device *idxd = wq->idxd; 515 int rc; 516 517 if (idxd->state != IDXD_DEV_ENABLED) 518 return -ENXIO; 519 520 /* 521 * User type WQ is enabled only when SVA is enabled for two reasons: 522 * - If no IOMMU or IOMMU Passthrough without SVA, userspace 523 * can directly access physical address through the WQ. 524 * - The IDXD cdev driver does not provide any ways to pin 525 * user pages and translate the address from user VA to IOVA or 526 * PA without IOMMU SVA. Therefore the application has no way 527 * to instruct the device to perform DMA function. This makes 528 * the cdev not usable for normal application usage. 529 */ 530 if (!device_user_pasid_enabled(idxd)) { 531 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; 532 dev_dbg(&idxd->pdev->dev, 533 "User type WQ cannot be enabled without SVA.\n"); 534 535 return -EOPNOTSUPP; 536 } 537 538 mutex_lock(&wq->wq_lock); 539 540 wq->wq = create_workqueue(dev_name(wq_confdev(wq))); 541 if (!wq->wq) { 542 rc = -ENOMEM; 543 goto wq_err; 544 } 545 546 wq->type = IDXD_WQT_USER; 547 rc = drv_enable_wq(wq); 548 if (rc < 0) 549 goto err; 550 551 rc = idxd_wq_add_cdev(wq); 552 if (rc < 0) { 553 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; 554 goto err_cdev; 555 } 556 557 idxd->cmd_status = 0; 558 mutex_unlock(&wq->wq_lock); 559 return 0; 560 561 err_cdev: 562 drv_disable_wq(wq); 563 err: 564 destroy_workqueue(wq->wq); 565 wq->type = IDXD_WQT_NONE; 566 wq_err: 567 mutex_unlock(&wq->wq_lock); 568 return rc; 569 } 570 571 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) 572 { 573 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 574 575 mutex_lock(&wq->wq_lock); 576 idxd_wq_del_cdev(wq); 577 drv_disable_wq(wq); 578 wq->type = IDXD_WQT_NONE; 579 destroy_workqueue(wq->wq); 580 wq->wq = NULL; 581 mutex_unlock(&wq->wq_lock); 582 } 583 584 static enum idxd_dev_type dev_types[] = { 585 IDXD_DEV_WQ, 586 IDXD_DEV_NONE, 587 }; 588 589 struct idxd_device_driver idxd_user_drv = { 590 .probe = idxd_user_drv_probe, 591 .remove = idxd_user_drv_remove, 592 .name = "user", 593 .type = dev_types, 594 }; 595 EXPORT_SYMBOL_GPL(idxd_user_drv); 596 597 int idxd_cdev_register(void) 598 { 599 int rc, i; 600 601 for (i = 0; i < IDXD_TYPE_MAX; i++) { 602 ida_init(&ictx[i].minor_ida); 603 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 604 ictx[i].name); 605 if (rc) 606 goto err_free_chrdev_region; 607 } 608 609 return 0; 610 611 err_free_chrdev_region: 612 for (i--; i >= 0; i--) 613 unregister_chrdev_region(ictx[i].devt, MINORMASK); 614 615 return rc; 616 } 617 618 void idxd_cdev_remove(void) 619 { 620 int i; 621 622 for (i = 0; i < IDXD_TYPE_MAX; i++) { 623 unregister_chrdev_region(ictx[i].devt, MINORMASK); 624 ida_destroy(&ictx[i].minor_ida); 625 } 626 } 627 628 /** 629 * idxd_copy_cr - copy completion record to user address space found by wq and 630 * PASID 631 * @wq: work queue 632 * @pasid: PASID 633 * @addr: user fault address to write 634 * @cr: completion record 635 * @len: number of bytes to copy 636 * 637 * This is called by a work that handles completion record fault. 638 * 639 * Return: number of bytes copied. 640 */ 641 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 642 void *cr, int len) 643 { 644 struct device *dev = &wq->idxd->pdev->dev; 645 int left = len, status_size = 1; 646 struct idxd_user_context *ctx; 647 struct mm_struct *mm; 648 649 mutex_lock(&wq->uc_lock); 650 651 ctx = xa_load(&wq->upasid_xa, pasid); 652 if (!ctx) { 653 dev_warn(dev, "No user context\n"); 654 goto out; 655 } 656 657 mm = ctx->mm; 658 /* 659 * The completion record fault handling work is running in kernel 660 * thread context. It temporarily switches to the mm to copy cr 661 * to addr in the mm. 662 */ 663 kthread_use_mm(mm); 664 left = copy_to_user((void __user *)addr + status_size, cr + status_size, 665 len - status_size); 666 /* 667 * Copy status only after the rest of completion record is copied 668 * successfully so that the user gets the complete completion record 669 * when a non-zero status is polled. 670 */ 671 if (!left) { 672 u8 status; 673 674 /* 675 * Ensure that the completion record's status field is written 676 * after the rest of the completion record has been written. 677 * This ensures that the user receives the correct completion 678 * record information once polling for a non-zero status. 679 */ 680 wmb(); 681 status = *(u8 *)cr; 682 if (put_user(status, (u8 __user *)addr)) 683 left += status_size; 684 } else { 685 left += status_size; 686 } 687 kthread_unuse_mm(mm); 688 689 out: 690 mutex_unlock(&wq->uc_lock); 691 692 return len - left; 693 } 694