1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/cdev.h> 11 #include <linux/fs.h> 12 #include <linux/poll.h> 13 #include <linux/iommu.h> 14 #include <linux/highmem.h> 15 #include <uapi/linux/idxd.h> 16 #include <linux/xarray.h> 17 #include "registers.h" 18 #include "idxd.h" 19 20 struct idxd_cdev_context { 21 const char *name; 22 dev_t devt; 23 struct ida minor_ida; 24 }; 25 26 /* 27 * Since user file names are global in DSA devices, define their ida's as 28 * global to avoid conflict file names. 29 */ 30 static DEFINE_IDA(file_ida); 31 static DEFINE_MUTEX(ida_lock); 32 33 /* 34 * ictx is an array based off of accelerator types. enum idxd_type 35 * is used as index 36 */ 37 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 38 { .name = "dsa" }, 39 { .name = "iax" } 40 }; 41 42 struct idxd_user_context { 43 struct idxd_wq *wq; 44 struct task_struct *task; 45 unsigned int pasid; 46 struct mm_struct *mm; 47 unsigned int flags; 48 struct iommu_sva *sva; 49 struct idxd_dev idxd_dev; 50 u64 counters[COUNTER_MAX]; 51 int id; 52 pid_t pid; 53 }; 54 55 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid); 56 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx); 57 58 static inline struct idxd_user_context *dev_to_uctx(struct device *dev) 59 { 60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 61 62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev); 63 } 64 65 static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf) 66 { 67 struct idxd_user_context *ctx = dev_to_uctx(dev); 68 69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]); 70 } 71 static DEVICE_ATTR_RO(cr_faults); 72 73 static ssize_t cr_fault_failures_show(struct device *dev, 74 struct device_attribute *attr, char *buf) 75 { 76 struct idxd_user_context *ctx = dev_to_uctx(dev); 77 78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]); 79 } 80 static DEVICE_ATTR_RO(cr_fault_failures); 81 82 static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) 83 { 84 struct idxd_user_context *ctx = dev_to_uctx(dev); 85 86 return sysfs_emit(buf, "%u\n", ctx->pid); 87 } 88 static DEVICE_ATTR_RO(pid); 89 90 static struct attribute *cdev_file_attributes[] = { 91 &dev_attr_cr_faults.attr, 92 &dev_attr_cr_fault_failures.attr, 93 &dev_attr_pid.attr, 94 NULL 95 }; 96 97 static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n) 98 { 99 struct device *dev = container_of(kobj, typeof(*dev), kobj); 100 struct idxd_user_context *ctx = dev_to_uctx(dev); 101 struct idxd_wq *wq = ctx->wq; 102 103 if (!wq_pasid_enabled(wq)) 104 return 0; 105 106 return a->mode; 107 } 108 109 static const struct attribute_group cdev_file_attribute_group = { 110 .attrs = cdev_file_attributes, 111 .is_visible = cdev_file_attr_visible, 112 }; 113 114 static const struct attribute_group *cdev_file_attribute_groups[] = { 115 &cdev_file_attribute_group, 116 NULL 117 }; 118 119 static void idxd_file_dev_release(struct device *dev) 120 { 121 struct idxd_user_context *ctx = dev_to_uctx(dev); 122 struct idxd_wq *wq = ctx->wq; 123 struct idxd_device *idxd = wq->idxd; 124 int rc; 125 126 mutex_lock(&ida_lock); 127 ida_free(&file_ida, ctx->id); 128 mutex_unlock(&ida_lock); 129 130 /* Wait for in-flight operations to complete. */ 131 if (wq_shared(wq)) { 132 idxd_device_drain_pasid(idxd, ctx->pasid); 133 } else { 134 if (device_user_pasid_enabled(idxd)) { 135 /* The wq disable in the disable pasid function will drain the wq */ 136 rc = idxd_wq_disable_pasid(wq); 137 if (rc < 0) 138 dev_err(dev, "wq disable pasid failed.\n"); 139 } else { 140 idxd_wq_drain(wq); 141 } 142 } 143 144 if (ctx->sva) { 145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid); 146 iommu_sva_unbind_device(ctx->sva); 147 idxd_xa_pasid_remove(ctx); 148 } 149 kfree(ctx); 150 mutex_lock(&wq->wq_lock); 151 idxd_wq_put(wq); 152 mutex_unlock(&wq->wq_lock); 153 } 154 155 static struct device_type idxd_cdev_file_type = { 156 .name = "idxd_file", 157 .release = idxd_file_dev_release, 158 .groups = cdev_file_attribute_groups, 159 }; 160 161 static void idxd_cdev_dev_release(struct device *dev) 162 { 163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); 164 struct idxd_cdev_context *cdev_ctx; 165 struct idxd_wq *wq = idxd_cdev->wq; 166 167 cdev_ctx = &ictx[wq->idxd->data->type]; 168 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 169 kfree(idxd_cdev); 170 } 171 172 static struct device_type idxd_cdev_device_type = { 173 .name = "idxd_cdev", 174 .release = idxd_cdev_dev_release, 175 }; 176 177 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 178 { 179 struct cdev *cdev = inode->i_cdev; 180 181 return container_of(cdev, struct idxd_cdev, cdev); 182 } 183 184 static inline struct idxd_wq *inode_wq(struct inode *inode) 185 { 186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 187 188 return idxd_cdev->wq; 189 } 190 191 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx) 192 { 193 struct idxd_wq *wq = ctx->wq; 194 void *ptr; 195 196 mutex_lock(&wq->uc_lock); 197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); 198 if (ptr != (void *)ctx) 199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", 200 ctx->pasid); 201 mutex_unlock(&wq->uc_lock); 202 } 203 204 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index) 205 { 206 struct idxd_user_context *ctx; 207 208 if (index >= COUNTER_MAX) 209 return; 210 211 mutex_lock(&wq->uc_lock); 212 ctx = xa_load(&wq->upasid_xa, pasid); 213 if (!ctx) { 214 mutex_unlock(&wq->uc_lock); 215 return; 216 } 217 ctx->counters[index]++; 218 mutex_unlock(&wq->uc_lock); 219 } 220 221 static int idxd_cdev_open(struct inode *inode, struct file *filp) 222 { 223 struct idxd_user_context *ctx; 224 struct idxd_device *idxd; 225 struct idxd_wq *wq; 226 struct device *dev, *fdev; 227 int rc = 0; 228 struct iommu_sva *sva; 229 unsigned int pasid; 230 struct idxd_cdev *idxd_cdev; 231 232 wq = inode_wq(inode); 233 idxd = wq->idxd; 234 dev = &idxd->pdev->dev; 235 236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 237 238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 239 if (!ctx) 240 return -ENOMEM; 241 242 mutex_lock(&wq->wq_lock); 243 244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 245 rc = -EBUSY; 246 goto failed; 247 } 248 249 ctx->wq = wq; 250 filp->private_data = ctx; 251 ctx->pid = current->pid; 252 253 if (device_user_pasid_enabled(idxd)) { 254 sva = iommu_sva_bind_device(dev, current->mm); 255 if (IS_ERR(sva)) { 256 rc = PTR_ERR(sva); 257 dev_err(dev, "pasid allocation failed: %d\n", rc); 258 goto failed; 259 } 260 261 pasid = iommu_sva_get_pasid(sva); 262 if (pasid == IOMMU_PASID_INVALID) { 263 rc = -EINVAL; 264 goto failed_get_pasid; 265 } 266 267 ctx->sva = sva; 268 ctx->pasid = pasid; 269 ctx->mm = current->mm; 270 271 mutex_lock(&wq->uc_lock); 272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); 273 mutex_unlock(&wq->uc_lock); 274 if (rc < 0) 275 dev_warn(dev, "PASID entry already exist in xarray.\n"); 276 277 if (wq_dedicated(wq)) { 278 rc = idxd_wq_set_pasid(wq, pasid); 279 if (rc < 0) { 280 dev_err(dev, "wq set pasid failed: %d\n", rc); 281 goto failed_set_pasid; 282 } 283 } 284 } 285 286 idxd_cdev = wq->idxd_cdev; 287 mutex_lock(&ida_lock); 288 ctx->id = ida_alloc(&file_ida, GFP_KERNEL); 289 mutex_unlock(&ida_lock); 290 if (ctx->id < 0) { 291 dev_warn(dev, "ida alloc failure\n"); 292 goto failed_ida; 293 } 294 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; 295 fdev = user_ctx_dev(ctx); 296 device_initialize(fdev); 297 fdev->parent = cdev_dev(idxd_cdev); 298 fdev->bus = &dsa_bus_type; 299 fdev->type = &idxd_cdev_file_type; 300 301 rc = dev_set_name(fdev, "file%d", ctx->id); 302 if (rc < 0) { 303 dev_warn(dev, "set name failure\n"); 304 goto failed_dev_name; 305 } 306 307 rc = device_add(fdev); 308 if (rc < 0) { 309 dev_warn(dev, "file device add failure\n"); 310 goto failed_dev_add; 311 } 312 313 idxd_wq_get(wq); 314 mutex_unlock(&wq->wq_lock); 315 return 0; 316 317 failed_dev_add: 318 failed_dev_name: 319 put_device(fdev); 320 failed_ida: 321 failed_set_pasid: 322 if (device_user_pasid_enabled(idxd)) 323 idxd_xa_pasid_remove(ctx); 324 failed_get_pasid: 325 if (device_user_pasid_enabled(idxd)) 326 iommu_sva_unbind_device(sva); 327 failed: 328 mutex_unlock(&wq->wq_lock); 329 kfree(ctx); 330 return rc; 331 } 332 333 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) 334 { 335 struct idxd_device *idxd = wq->idxd; 336 struct idxd_evl *evl = idxd->evl; 337 union evl_status_reg status; 338 u16 h, t, size; 339 int ent_size = evl_ent_size(idxd); 340 struct __evl_entry *entry_head; 341 342 if (!evl) 343 return; 344 345 mutex_lock(&evl->lock); 346 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); 347 t = status.tail; 348 h = status.head; 349 size = evl->size; 350 351 while (h != t) { 352 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); 353 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) 354 set_bit(h, evl->bmap); 355 h = (h + 1) % size; 356 } 357 drain_workqueue(wq->wq); 358 mutex_unlock(&evl->lock); 359 } 360 361 static int idxd_cdev_release(struct inode *node, struct file *filep) 362 { 363 struct idxd_user_context *ctx = filep->private_data; 364 struct idxd_wq *wq = ctx->wq; 365 struct idxd_device *idxd = wq->idxd; 366 struct device *dev = &idxd->pdev->dev; 367 368 dev_dbg(dev, "%s called\n", __func__); 369 filep->private_data = NULL; 370 371 device_unregister(user_ctx_dev(ctx)); 372 373 return 0; 374 } 375 376 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 377 const char *func) 378 { 379 struct device *dev = &wq->idxd->pdev->dev; 380 381 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 382 dev_info_ratelimited(dev, 383 "%s: %s: mapping too large: %lu\n", 384 current->comm, func, 385 vma->vm_end - vma->vm_start); 386 return -EINVAL; 387 } 388 389 return 0; 390 } 391 392 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 393 { 394 struct idxd_user_context *ctx = filp->private_data; 395 struct idxd_wq *wq = ctx->wq; 396 struct idxd_device *idxd = wq->idxd; 397 struct pci_dev *pdev = idxd->pdev; 398 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 399 unsigned long pfn; 400 int rc; 401 402 dev_dbg(&pdev->dev, "%s called\n", __func__); 403 rc = check_vma(wq, vma, __func__); 404 if (rc < 0) 405 return rc; 406 407 vm_flags_set(vma, VM_DONTCOPY); 408 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 409 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 410 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 411 vma->vm_private_data = ctx; 412 413 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 414 vma->vm_page_prot); 415 } 416 417 static __poll_t idxd_cdev_poll(struct file *filp, 418 struct poll_table_struct *wait) 419 { 420 struct idxd_user_context *ctx = filp->private_data; 421 struct idxd_wq *wq = ctx->wq; 422 struct idxd_device *idxd = wq->idxd; 423 __poll_t out = 0; 424 425 poll_wait(filp, &wq->err_queue, wait); 426 spin_lock(&idxd->dev_lock); 427 if (idxd->sw_err.valid) 428 out = EPOLLIN | EPOLLRDNORM; 429 spin_unlock(&idxd->dev_lock); 430 431 return out; 432 } 433 434 static const struct file_operations idxd_cdev_fops = { 435 .owner = THIS_MODULE, 436 .open = idxd_cdev_open, 437 .release = idxd_cdev_release, 438 .mmap = idxd_cdev_mmap, 439 .poll = idxd_cdev_poll, 440 }; 441 442 int idxd_cdev_get_major(struct idxd_device *idxd) 443 { 444 return MAJOR(ictx[idxd->data->type].devt); 445 } 446 447 int idxd_wq_add_cdev(struct idxd_wq *wq) 448 { 449 struct idxd_device *idxd = wq->idxd; 450 struct idxd_cdev *idxd_cdev; 451 struct cdev *cdev; 452 struct device *dev; 453 struct idxd_cdev_context *cdev_ctx; 454 int rc, minor; 455 456 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 457 if (!idxd_cdev) 458 return -ENOMEM; 459 460 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; 461 idxd_cdev->wq = wq; 462 cdev = &idxd_cdev->cdev; 463 dev = cdev_dev(idxd_cdev); 464 cdev_ctx = &ictx[wq->idxd->data->type]; 465 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 466 if (minor < 0) { 467 kfree(idxd_cdev); 468 return minor; 469 } 470 idxd_cdev->minor = minor; 471 472 device_initialize(dev); 473 dev->parent = wq_confdev(wq); 474 dev->bus = &dsa_bus_type; 475 dev->type = &idxd_cdev_device_type; 476 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 477 478 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); 479 if (rc < 0) 480 goto err; 481 482 wq->idxd_cdev = idxd_cdev; 483 cdev_init(cdev, &idxd_cdev_fops); 484 rc = cdev_device_add(cdev, dev); 485 if (rc) { 486 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 487 goto err; 488 } 489 490 return 0; 491 492 err: 493 put_device(dev); 494 wq->idxd_cdev = NULL; 495 return rc; 496 } 497 498 void idxd_wq_del_cdev(struct idxd_wq *wq) 499 { 500 struct idxd_cdev *idxd_cdev; 501 502 idxd_cdev = wq->idxd_cdev; 503 ida_destroy(&file_ida); 504 wq->idxd_cdev = NULL; 505 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); 506 put_device(cdev_dev(idxd_cdev)); 507 } 508 509 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) 510 { 511 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 512 struct idxd_device *idxd = wq->idxd; 513 int rc; 514 515 if (idxd->state != IDXD_DEV_ENABLED) 516 return -ENXIO; 517 518 /* 519 * User type WQ is enabled only when SVA is enabled for two reasons: 520 * - If no IOMMU or IOMMU Passthrough without SVA, userspace 521 * can directly access physical address through the WQ. 522 * - The IDXD cdev driver does not provide any ways to pin 523 * user pages and translate the address from user VA to IOVA or 524 * PA without IOMMU SVA. Therefore the application has no way 525 * to instruct the device to perform DMA function. This makes 526 * the cdev not usable for normal application usage. 527 */ 528 if (!device_user_pasid_enabled(idxd)) { 529 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; 530 dev_dbg(&idxd->pdev->dev, 531 "User type WQ cannot be enabled without SVA.\n"); 532 533 return -EOPNOTSUPP; 534 } 535 536 mutex_lock(&wq->wq_lock); 537 538 wq->wq = create_workqueue(dev_name(wq_confdev(wq))); 539 if (!wq->wq) { 540 rc = -ENOMEM; 541 goto wq_err; 542 } 543 544 wq->type = IDXD_WQT_USER; 545 rc = drv_enable_wq(wq); 546 if (rc < 0) 547 goto err; 548 549 rc = idxd_wq_add_cdev(wq); 550 if (rc < 0) { 551 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; 552 goto err_cdev; 553 } 554 555 idxd->cmd_status = 0; 556 mutex_unlock(&wq->wq_lock); 557 return 0; 558 559 err_cdev: 560 drv_disable_wq(wq); 561 err: 562 destroy_workqueue(wq->wq); 563 wq->type = IDXD_WQT_NONE; 564 wq_err: 565 mutex_unlock(&wq->wq_lock); 566 return rc; 567 } 568 569 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) 570 { 571 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 572 573 mutex_lock(&wq->wq_lock); 574 idxd_wq_del_cdev(wq); 575 drv_disable_wq(wq); 576 wq->type = IDXD_WQT_NONE; 577 destroy_workqueue(wq->wq); 578 wq->wq = NULL; 579 mutex_unlock(&wq->wq_lock); 580 } 581 582 static enum idxd_dev_type dev_types[] = { 583 IDXD_DEV_WQ, 584 IDXD_DEV_NONE, 585 }; 586 587 struct idxd_device_driver idxd_user_drv = { 588 .probe = idxd_user_drv_probe, 589 .remove = idxd_user_drv_remove, 590 .name = "user", 591 .type = dev_types, 592 }; 593 EXPORT_SYMBOL_GPL(idxd_user_drv); 594 595 int idxd_cdev_register(void) 596 { 597 int rc, i; 598 599 for (i = 0; i < IDXD_TYPE_MAX; i++) { 600 ida_init(&ictx[i].minor_ida); 601 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 602 ictx[i].name); 603 if (rc) 604 goto err_free_chrdev_region; 605 } 606 607 return 0; 608 609 err_free_chrdev_region: 610 for (i--; i >= 0; i--) 611 unregister_chrdev_region(ictx[i].devt, MINORMASK); 612 613 return rc; 614 } 615 616 void idxd_cdev_remove(void) 617 { 618 int i; 619 620 for (i = 0; i < IDXD_TYPE_MAX; i++) { 621 unregister_chrdev_region(ictx[i].devt, MINORMASK); 622 ida_destroy(&ictx[i].minor_ida); 623 } 624 } 625 626 /** 627 * idxd_copy_cr - copy completion record to user address space found by wq and 628 * PASID 629 * @wq: work queue 630 * @pasid: PASID 631 * @addr: user fault address to write 632 * @cr: completion record 633 * @len: number of bytes to copy 634 * 635 * This is called by a work that handles completion record fault. 636 * 637 * Return: number of bytes copied. 638 */ 639 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 640 void *cr, int len) 641 { 642 struct device *dev = &wq->idxd->pdev->dev; 643 int left = len, status_size = 1; 644 struct idxd_user_context *ctx; 645 struct mm_struct *mm; 646 647 mutex_lock(&wq->uc_lock); 648 649 ctx = xa_load(&wq->upasid_xa, pasid); 650 if (!ctx) { 651 dev_warn(dev, "No user context\n"); 652 goto out; 653 } 654 655 mm = ctx->mm; 656 /* 657 * The completion record fault handling work is running in kernel 658 * thread context. It temporarily switches to the mm to copy cr 659 * to addr in the mm. 660 */ 661 kthread_use_mm(mm); 662 left = copy_to_user((void __user *)addr + status_size, cr + status_size, 663 len - status_size); 664 /* 665 * Copy status only after the rest of completion record is copied 666 * successfully so that the user gets the complete completion record 667 * when a non-zero status is polled. 668 */ 669 if (!left) { 670 u8 status; 671 672 /* 673 * Ensure that the completion record's status field is written 674 * after the rest of the completion record has been written. 675 * This ensures that the user receives the correct completion 676 * record information once polling for a non-zero status. 677 */ 678 wmb(); 679 status = *(u8 *)cr; 680 if (put_user(status, (u8 __user *)addr)) 681 left += status_size; 682 } else { 683 left += status_size; 684 } 685 kthread_unuse_mm(mm); 686 687 out: 688 mutex_unlock(&wq->uc_lock); 689 690 return len - left; 691 } 692