1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/cdev.h> 11 #include <linux/fs.h> 12 #include <linux/poll.h> 13 #include <linux/iommu.h> 14 #include <linux/highmem.h> 15 #include <uapi/linux/idxd.h> 16 #include <linux/xarray.h> 17 #include "registers.h" 18 #include "idxd.h" 19 20 struct idxd_cdev_context { 21 const char *name; 22 dev_t devt; 23 struct ida minor_ida; 24 }; 25 26 /* 27 * Since user file names are global in DSA devices, define their ida's as 28 * global to avoid conflict file names. 29 */ 30 static DEFINE_IDA(file_ida); 31 static DEFINE_MUTEX(ida_lock); 32 33 /* 34 * ictx is an array based off of accelerator types. enum idxd_type 35 * is used as index 36 */ 37 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 38 { .name = "dsa" }, 39 { .name = "iax" } 40 }; 41 42 struct idxd_user_context { 43 struct idxd_wq *wq; 44 struct task_struct *task; 45 unsigned int pasid; 46 struct mm_struct *mm; 47 unsigned int flags; 48 struct iommu_sva *sva; 49 struct idxd_dev idxd_dev; 50 u64 counters[COUNTER_MAX]; 51 int id; 52 pid_t pid; 53 }; 54 55 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid); 56 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx); 57 58 static inline struct idxd_user_context *dev_to_uctx(struct device *dev) 59 { 60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 61 62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev); 63 } 64 65 static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf) 66 { 67 struct idxd_user_context *ctx = dev_to_uctx(dev); 68 69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]); 70 } 71 static DEVICE_ATTR_RO(cr_faults); 72 73 static ssize_t cr_fault_failures_show(struct device *dev, 74 struct device_attribute *attr, char *buf) 75 { 76 struct idxd_user_context *ctx = dev_to_uctx(dev); 77 78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]); 79 } 80 static DEVICE_ATTR_RO(cr_fault_failures); 81 82 static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) 83 { 84 struct idxd_user_context *ctx = dev_to_uctx(dev); 85 86 return sysfs_emit(buf, "%u\n", ctx->pid); 87 } 88 static DEVICE_ATTR_RO(pid); 89 90 static struct attribute *cdev_file_attributes[] = { 91 &dev_attr_cr_faults.attr, 92 &dev_attr_cr_fault_failures.attr, 93 &dev_attr_pid.attr, 94 NULL 95 }; 96 97 static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n) 98 { 99 struct device *dev = container_of(kobj, typeof(*dev), kobj); 100 struct idxd_user_context *ctx = dev_to_uctx(dev); 101 struct idxd_wq *wq = ctx->wq; 102 103 if (!wq_pasid_enabled(wq)) 104 return 0; 105 106 return a->mode; 107 } 108 109 static const struct attribute_group cdev_file_attribute_group = { 110 .attrs = cdev_file_attributes, 111 .is_visible = cdev_file_attr_visible, 112 }; 113 114 static const struct attribute_group *cdev_file_attribute_groups[] = { 115 &cdev_file_attribute_group, 116 NULL 117 }; 118 119 static void idxd_file_dev_release(struct device *dev) 120 { 121 struct idxd_user_context *ctx = dev_to_uctx(dev); 122 struct idxd_wq *wq = ctx->wq; 123 struct idxd_device *idxd = wq->idxd; 124 int rc; 125 126 mutex_lock(&ida_lock); 127 ida_free(&file_ida, ctx->id); 128 mutex_unlock(&ida_lock); 129 130 /* Wait for in-flight operations to complete. */ 131 if (wq_shared(wq)) { 132 idxd_device_drain_pasid(idxd, ctx->pasid); 133 } else { 134 if (device_user_pasid_enabled(idxd)) { 135 /* The wq disable in the disable pasid function will drain the wq */ 136 rc = idxd_wq_disable_pasid(wq); 137 if (rc < 0) 138 dev_err(dev, "wq disable pasid failed.\n"); 139 } else { 140 idxd_wq_drain(wq); 141 } 142 } 143 144 if (ctx->sva) { 145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid); 146 iommu_sva_unbind_device(ctx->sva); 147 idxd_xa_pasid_remove(ctx); 148 } 149 kfree(ctx); 150 mutex_lock(&wq->wq_lock); 151 idxd_wq_put(wq); 152 mutex_unlock(&wq->wq_lock); 153 } 154 155 static struct device_type idxd_cdev_file_type = { 156 .name = "idxd_file", 157 .release = idxd_file_dev_release, 158 .groups = cdev_file_attribute_groups, 159 }; 160 161 static void idxd_cdev_dev_release(struct device *dev) 162 { 163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); 164 struct idxd_cdev_context *cdev_ctx; 165 struct idxd_wq *wq = idxd_cdev->wq; 166 167 cdev_ctx = &ictx[wq->idxd->data->type]; 168 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 169 kfree(idxd_cdev); 170 } 171 172 static struct device_type idxd_cdev_device_type = { 173 .name = "idxd_cdev", 174 .release = idxd_cdev_dev_release, 175 }; 176 177 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 178 { 179 struct cdev *cdev = inode->i_cdev; 180 181 return container_of(cdev, struct idxd_cdev, cdev); 182 } 183 184 static inline struct idxd_wq *inode_wq(struct inode *inode) 185 { 186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 187 188 return idxd_cdev->wq; 189 } 190 191 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx) 192 { 193 struct idxd_wq *wq = ctx->wq; 194 void *ptr; 195 196 mutex_lock(&wq->uc_lock); 197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); 198 if (ptr != (void *)ctx) 199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", 200 ctx->pasid); 201 mutex_unlock(&wq->uc_lock); 202 } 203 204 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index) 205 { 206 struct idxd_user_context *ctx; 207 208 if (index >= COUNTER_MAX) 209 return; 210 211 mutex_lock(&wq->uc_lock); 212 ctx = xa_load(&wq->upasid_xa, pasid); 213 if (!ctx) { 214 mutex_unlock(&wq->uc_lock); 215 return; 216 } 217 ctx->counters[index]++; 218 mutex_unlock(&wq->uc_lock); 219 } 220 221 static int idxd_cdev_open(struct inode *inode, struct file *filp) 222 { 223 struct idxd_user_context *ctx; 224 struct idxd_device *idxd; 225 struct idxd_wq *wq; 226 struct device *dev, *fdev; 227 int rc = 0; 228 struct iommu_sva *sva; 229 unsigned int pasid; 230 struct idxd_cdev *idxd_cdev; 231 232 wq = inode_wq(inode); 233 idxd = wq->idxd; 234 dev = &idxd->pdev->dev; 235 236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 237 238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 239 if (!ctx) 240 return -ENOMEM; 241 242 mutex_lock(&wq->wq_lock); 243 244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 245 rc = -EBUSY; 246 goto failed; 247 } 248 249 ctx->wq = wq; 250 filp->private_data = ctx; 251 ctx->pid = current->pid; 252 253 if (device_user_pasid_enabled(idxd)) { 254 sva = iommu_sva_bind_device(dev, current->mm); 255 if (IS_ERR(sva)) { 256 rc = PTR_ERR(sva); 257 dev_err(dev, "pasid allocation failed: %d\n", rc); 258 goto failed; 259 } 260 261 pasid = iommu_sva_get_pasid(sva); 262 if (pasid == IOMMU_PASID_INVALID) { 263 rc = -EINVAL; 264 goto failed_get_pasid; 265 } 266 267 ctx->sva = sva; 268 ctx->pasid = pasid; 269 ctx->mm = current->mm; 270 271 mutex_lock(&wq->uc_lock); 272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); 273 mutex_unlock(&wq->uc_lock); 274 if (rc < 0) 275 dev_warn(dev, "PASID entry already exist in xarray.\n"); 276 277 if (wq_dedicated(wq)) { 278 rc = idxd_wq_set_pasid(wq, pasid); 279 if (rc < 0) { 280 dev_err(dev, "wq set pasid failed: %d\n", rc); 281 goto failed_set_pasid; 282 } 283 } 284 } 285 286 idxd_cdev = wq->idxd_cdev; 287 mutex_lock(&ida_lock); 288 ctx->id = ida_alloc(&file_ida, GFP_KERNEL); 289 mutex_unlock(&ida_lock); 290 if (ctx->id < 0) { 291 dev_warn(dev, "ida alloc failure\n"); 292 goto failed_ida; 293 } 294 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; 295 fdev = user_ctx_dev(ctx); 296 device_initialize(fdev); 297 fdev->parent = cdev_dev(idxd_cdev); 298 fdev->bus = &dsa_bus_type; 299 fdev->type = &idxd_cdev_file_type; 300 301 rc = dev_set_name(fdev, "file%d", ctx->id); 302 if (rc < 0) { 303 dev_warn(dev, "set name failure\n"); 304 goto failed_dev_name; 305 } 306 307 rc = device_add(fdev); 308 if (rc < 0) { 309 dev_warn(dev, "file device add failure\n"); 310 goto failed_dev_add; 311 } 312 313 idxd_wq_get(wq); 314 mutex_unlock(&wq->wq_lock); 315 return 0; 316 317 failed_dev_add: 318 failed_dev_name: 319 put_device(fdev); 320 failed_ida: 321 failed_set_pasid: 322 if (device_user_pasid_enabled(idxd)) 323 idxd_xa_pasid_remove(ctx); 324 failed_get_pasid: 325 if (device_user_pasid_enabled(idxd)) 326 iommu_sva_unbind_device(sva); 327 failed: 328 mutex_unlock(&wq->wq_lock); 329 kfree(ctx); 330 return rc; 331 } 332 333 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) 334 { 335 struct idxd_device *idxd = wq->idxd; 336 struct idxd_evl *evl = idxd->evl; 337 union evl_status_reg status; 338 u16 h, t, size; 339 int ent_size = evl_ent_size(idxd); 340 struct __evl_entry *entry_head; 341 342 if (!evl) 343 return; 344 345 mutex_lock(&evl->lock); 346 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); 347 t = status.tail; 348 h = status.head; 349 size = evl->size; 350 351 while (h != t) { 352 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); 353 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) 354 set_bit(h, evl->bmap); 355 h = (h + 1) % size; 356 } 357 drain_workqueue(wq->wq); 358 mutex_unlock(&evl->lock); 359 } 360 361 static int idxd_cdev_release(struct inode *node, struct file *filep) 362 { 363 struct idxd_user_context *ctx = filep->private_data; 364 struct idxd_wq *wq = ctx->wq; 365 struct idxd_device *idxd = wq->idxd; 366 struct device *dev = &idxd->pdev->dev; 367 368 dev_dbg(dev, "%s called\n", __func__); 369 filep->private_data = NULL; 370 371 device_unregister(user_ctx_dev(ctx)); 372 373 return 0; 374 } 375 376 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 377 const char *func) 378 { 379 struct device *dev = &wq->idxd->pdev->dev; 380 381 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 382 dev_info_ratelimited(dev, 383 "%s: %s: mapping too large: %lu\n", 384 current->comm, func, 385 vma->vm_end - vma->vm_start); 386 return -EINVAL; 387 } 388 389 return 0; 390 } 391 392 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 393 { 394 struct idxd_user_context *ctx = filp->private_data; 395 struct idxd_wq *wq = ctx->wq; 396 struct idxd_device *idxd = wq->idxd; 397 struct pci_dev *pdev = idxd->pdev; 398 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 399 unsigned long pfn; 400 int rc; 401 402 dev_dbg(&pdev->dev, "%s called\n", __func__); 403 404 /* 405 * Due to an erratum in some of the devices supported by the driver, 406 * direct user submission to the device can be unsafe. 407 * (See the INTEL-SA-01084 security advisory) 408 * 409 * For the devices that exhibit this behavior, require that the user 410 * has CAP_SYS_RAWIO capabilities. 411 */ 412 if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) 413 return -EPERM; 414 415 rc = check_vma(wq, vma, __func__); 416 if (rc < 0) 417 return rc; 418 419 vm_flags_set(vma, VM_DONTCOPY); 420 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 421 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 422 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 423 vma->vm_private_data = ctx; 424 425 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 426 vma->vm_page_prot); 427 } 428 429 static int idxd_submit_user_descriptor(struct idxd_user_context *ctx, 430 struct dsa_hw_desc __user *udesc) 431 { 432 struct idxd_wq *wq = ctx->wq; 433 struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev; 434 const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40; 435 void __iomem *portal = idxd_wq_portal_addr(wq); 436 struct dsa_hw_desc descriptor __aligned(64); 437 int rc; 438 439 rc = copy_from_user(&descriptor, udesc, sizeof(descriptor)); 440 if (rc) 441 return -EFAULT; 442 443 /* 444 * DSA devices are capable of indirect ("batch") command submission. 445 * On devices where direct user submissions are not safe, we cannot 446 * allow this since there is no good way for us to verify these 447 * indirect commands. 448 */ 449 if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH && 450 !wq->idxd->user_submission_safe) 451 return -EINVAL; 452 /* 453 * As per the programming specification, the completion address must be 454 * aligned to 32 or 64 bytes. If this is violated the hardware 455 * engine can get very confused (security issue). 456 */ 457 if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align)) 458 return -EINVAL; 459 460 if (wq_dedicated(wq)) 461 iosubmit_cmds512(portal, &descriptor, 1); 462 else { 463 descriptor.priv = 0; 464 descriptor.pasid = ctx->pasid; 465 rc = idxd_enqcmds(wq, portal, &descriptor); 466 if (rc < 0) 467 return rc; 468 } 469 470 return 0; 471 } 472 473 static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len, 474 loff_t *unused) 475 { 476 struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf; 477 struct idxd_user_context *ctx = filp->private_data; 478 ssize_t written = 0; 479 int i; 480 481 for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) { 482 int rc = idxd_submit_user_descriptor(ctx, udesc + i); 483 484 if (rc) 485 return written ? written : rc; 486 487 written += sizeof(struct dsa_hw_desc); 488 } 489 490 return written; 491 } 492 493 static __poll_t idxd_cdev_poll(struct file *filp, 494 struct poll_table_struct *wait) 495 { 496 struct idxd_user_context *ctx = filp->private_data; 497 struct idxd_wq *wq = ctx->wq; 498 struct idxd_device *idxd = wq->idxd; 499 __poll_t out = 0; 500 501 poll_wait(filp, &wq->err_queue, wait); 502 spin_lock(&idxd->dev_lock); 503 if (idxd->sw_err.valid) 504 out = EPOLLIN | EPOLLRDNORM; 505 spin_unlock(&idxd->dev_lock); 506 507 return out; 508 } 509 510 static const struct file_operations idxd_cdev_fops = { 511 .owner = THIS_MODULE, 512 .open = idxd_cdev_open, 513 .release = idxd_cdev_release, 514 .mmap = idxd_cdev_mmap, 515 .write = idxd_cdev_write, 516 .poll = idxd_cdev_poll, 517 }; 518 519 int idxd_cdev_get_major(struct idxd_device *idxd) 520 { 521 return MAJOR(ictx[idxd->data->type].devt); 522 } 523 524 int idxd_wq_add_cdev(struct idxd_wq *wq) 525 { 526 struct idxd_device *idxd = wq->idxd; 527 struct idxd_cdev *idxd_cdev; 528 struct cdev *cdev; 529 struct device *dev; 530 struct idxd_cdev_context *cdev_ctx; 531 int rc, minor; 532 533 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 534 if (!idxd_cdev) 535 return -ENOMEM; 536 537 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; 538 idxd_cdev->wq = wq; 539 cdev = &idxd_cdev->cdev; 540 dev = cdev_dev(idxd_cdev); 541 cdev_ctx = &ictx[wq->idxd->data->type]; 542 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 543 if (minor < 0) { 544 kfree(idxd_cdev); 545 return minor; 546 } 547 idxd_cdev->minor = minor; 548 549 device_initialize(dev); 550 dev->parent = wq_confdev(wq); 551 dev->bus = &dsa_bus_type; 552 dev->type = &idxd_cdev_device_type; 553 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 554 555 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); 556 if (rc < 0) 557 goto err; 558 559 wq->idxd_cdev = idxd_cdev; 560 cdev_init(cdev, &idxd_cdev_fops); 561 rc = cdev_device_add(cdev, dev); 562 if (rc) { 563 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 564 goto err; 565 } 566 567 return 0; 568 569 err: 570 put_device(dev); 571 wq->idxd_cdev = NULL; 572 return rc; 573 } 574 575 void idxd_wq_del_cdev(struct idxd_wq *wq) 576 { 577 struct idxd_cdev *idxd_cdev; 578 579 idxd_cdev = wq->idxd_cdev; 580 wq->idxd_cdev = NULL; 581 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); 582 put_device(cdev_dev(idxd_cdev)); 583 } 584 585 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) 586 { 587 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 588 struct idxd_device *idxd = wq->idxd; 589 int rc; 590 591 if (idxd->state != IDXD_DEV_ENABLED) 592 return -ENXIO; 593 594 /* 595 * User type WQ is enabled only when SVA is enabled for two reasons: 596 * - If no IOMMU or IOMMU Passthrough without SVA, userspace 597 * can directly access physical address through the WQ. 598 * - The IDXD cdev driver does not provide any ways to pin 599 * user pages and translate the address from user VA to IOVA or 600 * PA without IOMMU SVA. Therefore the application has no way 601 * to instruct the device to perform DMA function. This makes 602 * the cdev not usable for normal application usage. 603 */ 604 if (!device_user_pasid_enabled(idxd)) { 605 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; 606 dev_dbg(&idxd->pdev->dev, 607 "User type WQ cannot be enabled without SVA.\n"); 608 609 return -EOPNOTSUPP; 610 } 611 612 mutex_lock(&wq->wq_lock); 613 614 wq->wq = create_workqueue(dev_name(wq_confdev(wq))); 615 if (!wq->wq) { 616 rc = -ENOMEM; 617 goto wq_err; 618 } 619 620 wq->type = IDXD_WQT_USER; 621 rc = drv_enable_wq(wq); 622 if (rc < 0) 623 goto err; 624 625 rc = idxd_wq_add_cdev(wq); 626 if (rc < 0) { 627 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; 628 goto err_cdev; 629 } 630 631 idxd->cmd_status = 0; 632 mutex_unlock(&wq->wq_lock); 633 return 0; 634 635 err_cdev: 636 drv_disable_wq(wq); 637 err: 638 destroy_workqueue(wq->wq); 639 wq->type = IDXD_WQT_NONE; 640 wq_err: 641 mutex_unlock(&wq->wq_lock); 642 return rc; 643 } 644 645 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) 646 { 647 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 648 649 mutex_lock(&wq->wq_lock); 650 idxd_wq_del_cdev(wq); 651 drv_disable_wq(wq); 652 wq->type = IDXD_WQT_NONE; 653 destroy_workqueue(wq->wq); 654 wq->wq = NULL; 655 mutex_unlock(&wq->wq_lock); 656 } 657 658 static enum idxd_dev_type dev_types[] = { 659 IDXD_DEV_WQ, 660 IDXD_DEV_NONE, 661 }; 662 663 struct idxd_device_driver idxd_user_drv = { 664 .probe = idxd_user_drv_probe, 665 .remove = idxd_user_drv_remove, 666 .name = "user", 667 .type = dev_types, 668 }; 669 EXPORT_SYMBOL_GPL(idxd_user_drv); 670 671 int idxd_cdev_register(void) 672 { 673 int rc, i; 674 675 for (i = 0; i < IDXD_TYPE_MAX; i++) { 676 ida_init(&ictx[i].minor_ida); 677 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 678 ictx[i].name); 679 if (rc) 680 goto err_free_chrdev_region; 681 } 682 683 return 0; 684 685 err_free_chrdev_region: 686 for (i--; i >= 0; i--) 687 unregister_chrdev_region(ictx[i].devt, MINORMASK); 688 689 return rc; 690 } 691 692 void idxd_cdev_remove(void) 693 { 694 int i; 695 696 for (i = 0; i < IDXD_TYPE_MAX; i++) { 697 unregister_chrdev_region(ictx[i].devt, MINORMASK); 698 ida_destroy(&ictx[i].minor_ida); 699 } 700 } 701 702 /** 703 * idxd_copy_cr - copy completion record to user address space found by wq and 704 * PASID 705 * @wq: work queue 706 * @pasid: PASID 707 * @addr: user fault address to write 708 * @cr: completion record 709 * @len: number of bytes to copy 710 * 711 * This is called by a work that handles completion record fault. 712 * 713 * Return: number of bytes copied. 714 */ 715 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 716 void *cr, int len) 717 { 718 struct device *dev = &wq->idxd->pdev->dev; 719 int left = len, status_size = 1; 720 struct idxd_user_context *ctx; 721 struct mm_struct *mm; 722 723 mutex_lock(&wq->uc_lock); 724 725 ctx = xa_load(&wq->upasid_xa, pasid); 726 if (!ctx) { 727 dev_warn(dev, "No user context\n"); 728 goto out; 729 } 730 731 mm = ctx->mm; 732 /* 733 * The completion record fault handling work is running in kernel 734 * thread context. It temporarily switches to the mm to copy cr 735 * to addr in the mm. 736 */ 737 kthread_use_mm(mm); 738 left = copy_to_user((void __user *)addr + status_size, cr + status_size, 739 len - status_size); 740 /* 741 * Copy status only after the rest of completion record is copied 742 * successfully so that the user gets the complete completion record 743 * when a non-zero status is polled. 744 */ 745 if (!left) { 746 u8 status; 747 748 /* 749 * Ensure that the completion record's status field is written 750 * after the rest of the completion record has been written. 751 * This ensures that the user receives the correct completion 752 * record information once polling for a non-zero status. 753 */ 754 wmb(); 755 status = *(u8 *)cr; 756 if (put_user(status, (u8 __user *)addr)) 757 left += status_size; 758 } else { 759 left += status_size; 760 } 761 kthread_unuse_mm(mm); 762 763 out: 764 mutex_unlock(&wq->uc_lock); 765 766 return len - left; 767 } 768