1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * VAS user space API for its accelerators (Only NX-GZIP is supported now) 4 * Copyright (C) 2019 Haren Myneni, IBM Corp 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/device.h> 9 #include <linux/cdev.h> 10 #include <linux/fs.h> 11 #include <linux/slab.h> 12 #include <linux/uaccess.h> 13 #include <linux/kthread.h> 14 #include <linux/sched/signal.h> 15 #include <linux/mmu_context.h> 16 #include <linux/io.h> 17 #include <asm/vas.h> 18 #include <uapi/asm/vas-api.h> 19 20 /* 21 * The driver creates the device node that can be used as follows: 22 * For NX-GZIP 23 * 24 * fd = open("/dev/crypto/nx-gzip", O_RDWR); 25 * rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr); 26 * paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL). 27 * vas_copy(&crb, 0, 1); 28 * vas_paste(paste_addr, 0, 1); 29 * close(fd) or exit process to close window. 30 * 31 * where "vas_copy" and "vas_paste" are defined in copy-paste.h. 32 * copy/paste returns to the user space directly. So refer NX hardware 33 * documentation for exact copy/paste usage and completion / error 34 * conditions. 35 */ 36 37 /* 38 * Wrapper object for the nx-gzip device - there is just one instance of 39 * this node for the whole system. 40 */ 41 static struct coproc_dev { 42 struct cdev cdev; 43 struct device *device; 44 char *name; 45 dev_t devt; 46 struct class *class; 47 enum vas_cop_type cop_type; 48 const struct vas_user_win_ops *vops; 49 } coproc_device; 50 51 struct coproc_instance { 52 struct coproc_dev *coproc; 53 struct vas_window *txwin; 54 }; 55 56 static char *coproc_devnode(const struct device *dev, umode_t *mode) 57 { 58 return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev)); 59 } 60 61 /* 62 * Take reference to pid and mm 63 */ 64 int get_vas_user_win_ref(struct vas_user_win_ref *task_ref) 65 { 66 /* 67 * Window opened by a child thread may not be closed when 68 * it exits. So take reference to its pid and release it 69 * when the window is free by parent thread. 70 * Acquire a reference to the task's pid to make sure 71 * pid will not be re-used - needed only for multithread 72 * applications. 73 */ 74 task_ref->pid = get_task_pid(current, PIDTYPE_PID); 75 /* 76 * Acquire a reference to the task's mm. 77 */ 78 task_ref->mm = get_task_mm(current); 79 if (!task_ref->mm) { 80 put_pid(task_ref->pid); 81 pr_err("VAS: pid(%d): mm_struct is not found\n", 82 current->pid); 83 return -EPERM; 84 } 85 86 mmgrab(task_ref->mm); 87 mmput(task_ref->mm); 88 /* 89 * Process closes window during exit. In the case of 90 * multithread application, the child thread can open 91 * window and can exit without closing it. So takes tgid 92 * reference until window closed to make sure tgid is not 93 * reused. 94 */ 95 task_ref->tgid = find_get_pid(task_tgid_vnr(current)); 96 97 return 0; 98 } 99 100 /* 101 * Successful return must release the task reference with 102 * put_task_struct 103 */ 104 static bool ref_get_pid_and_task(struct vas_user_win_ref *task_ref, 105 struct task_struct **tskp, struct pid **pidp) 106 { 107 struct task_struct *tsk; 108 struct pid *pid; 109 110 pid = task_ref->pid; 111 tsk = get_pid_task(pid, PIDTYPE_PID); 112 if (!tsk) { 113 pid = task_ref->tgid; 114 tsk = get_pid_task(pid, PIDTYPE_PID); 115 /* 116 * Parent thread (tgid) will be closing window when it 117 * exits. So should not get here. 118 */ 119 if (WARN_ON_ONCE(!tsk)) 120 return false; 121 } 122 123 /* Return if the task is exiting. */ 124 if (tsk->flags & PF_EXITING) { 125 put_task_struct(tsk); 126 return false; 127 } 128 129 *tskp = tsk; 130 *pidp = pid; 131 132 return true; 133 } 134 135 /* 136 * Update the CSB to indicate a translation error. 137 * 138 * User space will be polling on CSB after the request is issued. 139 * If NX can handle the request without any issues, it updates CSB. 140 * Whereas if NX encounters page fault, the kernel will handle the 141 * fault and update CSB with translation error. 142 * 143 * If we are unable to update the CSB means copy_to_user failed due to 144 * invalid csb_addr, send a signal to the process. 145 */ 146 void vas_update_csb(struct coprocessor_request_block *crb, 147 struct vas_user_win_ref *task_ref) 148 { 149 struct coprocessor_status_block csb; 150 struct kernel_siginfo info; 151 struct task_struct *tsk; 152 void __user *csb_addr; 153 struct pid *pid; 154 int rc; 155 156 /* 157 * NX user space windows can not be opened for task->mm=NULL 158 * and faults will not be generated for kernel requests. 159 */ 160 if (WARN_ON_ONCE(!task_ref->mm)) 161 return; 162 163 csb_addr = (void __user *)be64_to_cpu(crb->csb_addr); 164 165 memset(&csb, 0, sizeof(csb)); 166 csb.cc = CSB_CC_FAULT_ADDRESS; 167 csb.ce = CSB_CE_TERMINATION; 168 csb.cs = 0; 169 csb.count = 0; 170 171 /* 172 * NX operates and returns in BE format as defined CRB struct. 173 * So saves fault_storage_addr in BE as NX pastes in FIFO and 174 * expects user space to convert to CPU format. 175 */ 176 csb.address = crb->stamp.nx.fault_storage_addr; 177 csb.flags = 0; 178 179 /* 180 * Process closes send window after all pending NX requests are 181 * completed. In multi-thread applications, a child thread can 182 * open a window and can exit without closing it. May be some 183 * requests are pending or this window can be used by other 184 * threads later. We should handle faults if NX encounters 185 * pages faults on these requests. Update CSB with translation 186 * error and fault address. If csb_addr passed by user space is 187 * invalid, send SEGV signal to pid saved in window. If the 188 * child thread is not running, send the signal to tgid. 189 * Parent thread (tgid) will close this window upon its exit. 190 * 191 * pid and mm references are taken when window is opened by 192 * process (pid). So tgid is used only when child thread opens 193 * a window and exits without closing it. 194 */ 195 196 if (!ref_get_pid_and_task(task_ref, &tsk, &pid)) 197 return; 198 199 kthread_use_mm(task_ref->mm); 200 rc = copy_to_user(csb_addr, &csb, sizeof(csb)); 201 /* 202 * User space polls on csb.flags (first byte). So add barrier 203 * then copy first byte with csb flags update. 204 */ 205 if (!rc) { 206 csb.flags = CSB_V; 207 /* Make sure update to csb.flags is visible now */ 208 smp_mb(); 209 rc = copy_to_user(csb_addr, &csb, sizeof(u8)); 210 } 211 kthread_unuse_mm(task_ref->mm); 212 put_task_struct(tsk); 213 214 /* Success */ 215 if (!rc) 216 return; 217 218 219 pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n", 220 csb_addr, pid_vnr(pid)); 221 222 clear_siginfo(&info); 223 info.si_signo = SIGSEGV; 224 info.si_errno = EFAULT; 225 info.si_code = SEGV_MAPERR; 226 info.si_addr = csb_addr; 227 /* 228 * process will be polling on csb.flags after request is sent to 229 * NX. So generally CSB update should not fail except when an 230 * application passes invalid csb_addr. So an error message will 231 * be displayed and leave it to user space whether to ignore or 232 * handle this signal. 233 */ 234 rcu_read_lock(); 235 rc = kill_pid_info(SIGSEGV, &info, pid); 236 rcu_read_unlock(); 237 238 pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__, 239 pid_vnr(pid), rc); 240 } 241 242 void vas_dump_crb(struct coprocessor_request_block *crb) 243 { 244 struct data_descriptor_entry *dde; 245 struct nx_fault_stamp *nx; 246 247 dde = &crb->source; 248 pr_devel("SrcDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n", 249 be64_to_cpu(dde->address), be32_to_cpu(dde->length), 250 dde->count, dde->index, dde->flags); 251 252 dde = &crb->target; 253 pr_devel("TgtDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n", 254 be64_to_cpu(dde->address), be32_to_cpu(dde->length), 255 dde->count, dde->index, dde->flags); 256 257 nx = &crb->stamp.nx; 258 pr_devel("NX Stamp: PSWID 0x%x, FSA 0x%llx, flags 0x%x, FS 0x%x\n", 259 be32_to_cpu(nx->pswid), 260 be64_to_cpu(crb->stamp.nx.fault_storage_addr), 261 nx->flags, nx->fault_status); 262 } 263 264 static int coproc_open(struct inode *inode, struct file *fp) 265 { 266 struct coproc_instance *cp_inst; 267 268 cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL); 269 if (!cp_inst) 270 return -ENOMEM; 271 272 cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev, 273 cdev); 274 fp->private_data = cp_inst; 275 276 return 0; 277 } 278 279 static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg) 280 { 281 void __user *uptr = (void __user *)arg; 282 struct vas_tx_win_open_attr uattr; 283 struct coproc_instance *cp_inst; 284 struct vas_window *txwin; 285 int rc; 286 287 cp_inst = fp->private_data; 288 289 /* 290 * One window for file descriptor 291 */ 292 if (cp_inst->txwin) 293 return -EEXIST; 294 295 rc = copy_from_user(&uattr, uptr, sizeof(uattr)); 296 if (rc) { 297 pr_err("%s(): copy_from_user() returns %d\n", __func__, rc); 298 return -EFAULT; 299 } 300 301 if (uattr.version != 1) { 302 pr_err("Invalid window open API version\n"); 303 return -EINVAL; 304 } 305 306 if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->open_win) { 307 pr_err("VAS API is not registered\n"); 308 return -EACCES; 309 } 310 311 txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags, 312 cp_inst->coproc->cop_type); 313 if (IS_ERR(txwin)) { 314 pr_err("%s() VAS window open failed, %ld\n", __func__, 315 PTR_ERR(txwin)); 316 return PTR_ERR(txwin); 317 } 318 319 mutex_init(&txwin->task_ref.mmap_mutex); 320 cp_inst->txwin = txwin; 321 322 return 0; 323 } 324 325 static int coproc_release(struct inode *inode, struct file *fp) 326 { 327 struct coproc_instance *cp_inst = fp->private_data; 328 int rc; 329 330 if (cp_inst->txwin) { 331 if (cp_inst->coproc->vops && 332 cp_inst->coproc->vops->close_win) { 333 rc = cp_inst->coproc->vops->close_win(cp_inst->txwin); 334 if (rc) 335 return rc; 336 } 337 cp_inst->txwin = NULL; 338 } 339 340 kfree(cp_inst); 341 fp->private_data = NULL; 342 343 /* 344 * We don't know here if user has other receive windows 345 * open, so we can't really call clear_thread_tidr(). 346 * So, once the process calls set_thread_tidr(), the 347 * TIDR value sticks around until process exits, resulting 348 * in an extra copy in restore_sprs(). 349 */ 350 351 return 0; 352 } 353 354 /* 355 * If the executed instruction that caused the fault was a paste, then 356 * clear regs CR0[EQ], advance NIP, and return 0. Else return error code. 357 */ 358 static int do_fail_paste(void) 359 { 360 struct pt_regs *regs = current->thread.regs; 361 u32 instword; 362 363 if (WARN_ON_ONCE(!regs)) 364 return -EINVAL; 365 366 if (WARN_ON_ONCE(!user_mode(regs))) 367 return -EINVAL; 368 369 /* 370 * If we couldn't translate the instruction, the driver should 371 * return success without handling the fault, it will be retried 372 * or the instruction fetch will fault. 373 */ 374 if (get_user(instword, (u32 __user *)(regs->nip))) 375 return -EAGAIN; 376 377 /* 378 * Not a paste instruction, driver may fail the fault. 379 */ 380 if ((instword & PPC_INST_PASTE_MASK) != PPC_INST_PASTE) 381 return -ENOENT; 382 383 regs->ccr &= ~0xe0000000; /* Clear CR0[0-2] to fail paste */ 384 regs_add_return_ip(regs, 4); /* Emulate the paste */ 385 386 return 0; 387 } 388 389 /* 390 * This fault handler is invoked when the core generates page fault on 391 * the paste address. Happens if the kernel closes window in hypervisor 392 * (on pseries) due to lost credit or the paste address is not mapped. 393 */ 394 static vm_fault_t vas_mmap_fault(struct vm_fault *vmf) 395 { 396 struct vm_area_struct *vma = vmf->vma; 397 struct file *fp = vma->vm_file; 398 struct coproc_instance *cp_inst = fp->private_data; 399 struct vas_window *txwin; 400 vm_fault_t fault; 401 u64 paste_addr; 402 int ret; 403 404 /* 405 * window is not opened. Shouldn't expect this error. 406 */ 407 if (!cp_inst || !cp_inst->txwin) { 408 pr_err("%s(): Unexpected fault on paste address with TX window closed\n", 409 __func__); 410 return VM_FAULT_SIGBUS; 411 } 412 413 txwin = cp_inst->txwin; 414 /* 415 * When the LPAR lost credits due to core removal or during 416 * migration, invalidate the existing mapping for the current 417 * paste addresses and set windows in-active (zap_vma_pages in 418 * reconfig_close_windows()). 419 * New mapping will be done later after migration or new credits 420 * available. So continue to receive faults if the user space 421 * issue NX request. 422 */ 423 if (txwin->task_ref.vma != vmf->vma) { 424 pr_err("%s(): No previous mapping with paste address\n", 425 __func__); 426 return VM_FAULT_SIGBUS; 427 } 428 429 mutex_lock(&txwin->task_ref.mmap_mutex); 430 /* 431 * The window may be inactive due to lost credit (Ex: core 432 * removal with DLPAR). If the window is active again when 433 * the credit is available, map the new paste address at the 434 * window virtual address. 435 */ 436 if (txwin->status == VAS_WIN_ACTIVE) { 437 paste_addr = cp_inst->coproc->vops->paste_addr(txwin); 438 if (paste_addr) { 439 fault = vmf_insert_pfn(vma, vma->vm_start, 440 (paste_addr >> PAGE_SHIFT)); 441 mutex_unlock(&txwin->task_ref.mmap_mutex); 442 return fault; 443 } 444 } 445 mutex_unlock(&txwin->task_ref.mmap_mutex); 446 447 /* 448 * Received this fault due to closing the actual window. 449 * It can happen during migration or lost credits. 450 * Since no mapping, return the paste instruction failure 451 * to the user space. 452 */ 453 ret = do_fail_paste(); 454 /* 455 * The user space can retry several times until success (needed 456 * for migration) or should fallback to SW compression or 457 * manage with the existing open windows if available. 458 * Looking at sysfs interface, it can determine whether these 459 * failures are coming during migration or core removal: 460 * nr_used_credits > nr_total_credits when lost credits 461 */ 462 if (!ret || (ret == -EAGAIN)) 463 return VM_FAULT_NOPAGE; 464 465 return VM_FAULT_SIGBUS; 466 } 467 468 static const struct vm_operations_struct vas_vm_ops = { 469 .fault = vas_mmap_fault, 470 }; 471 472 static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) 473 { 474 struct coproc_instance *cp_inst = fp->private_data; 475 struct vas_window *txwin; 476 unsigned long pfn; 477 u64 paste_addr; 478 pgprot_t prot; 479 int rc; 480 481 txwin = cp_inst->txwin; 482 483 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 484 pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__, 485 (vma->vm_end - vma->vm_start), PAGE_SIZE); 486 return -EINVAL; 487 } 488 489 /* Ensure instance has an open send window */ 490 if (!txwin) { 491 pr_err("%s(): No send window open?\n", __func__); 492 return -EINVAL; 493 } 494 495 if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) { 496 pr_err("%s(): VAS API is not registered\n", __func__); 497 return -EACCES; 498 } 499 500 /* 501 * The initial mmap is done after the window is opened 502 * with ioctl. But before mmap(), this window can be closed in 503 * the hypervisor due to lost credit (core removal on pseries). 504 * So if the window is not active, return mmap() failure with 505 * -EACCES and expects the user space reissue mmap() when it 506 * is active again or open new window when the credit is available. 507 * mmap_mutex protects the paste address mmap() with DLPAR 508 * close/open event and allows mmap() only when the window is 509 * active. 510 */ 511 mutex_lock(&txwin->task_ref.mmap_mutex); 512 if (txwin->status != VAS_WIN_ACTIVE) { 513 pr_err("%s(): Window is not active\n", __func__); 514 rc = -EACCES; 515 goto out; 516 } 517 518 paste_addr = cp_inst->coproc->vops->paste_addr(txwin); 519 if (!paste_addr) { 520 pr_err("%s(): Window paste address failed\n", __func__); 521 rc = -EINVAL; 522 goto out; 523 } 524 525 pfn = paste_addr >> PAGE_SHIFT; 526 527 /* flags, page_prot from cxl_mmap(), except we want cachable */ 528 vm_flags_set(vma, VM_IO | VM_PFNMAP); 529 vma->vm_page_prot = pgprot_cached(vma->vm_page_prot); 530 531 prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY); 532 533 rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, 534 vma->vm_end - vma->vm_start, prot); 535 536 pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__, 537 paste_addr, vma->vm_start, rc); 538 539 txwin->task_ref.vma = vma; 540 vma->vm_ops = &vas_vm_ops; 541 542 out: 543 mutex_unlock(&txwin->task_ref.mmap_mutex); 544 return rc; 545 } 546 547 static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) 548 { 549 switch (cmd) { 550 case VAS_TX_WIN_OPEN: 551 return coproc_ioc_tx_win_open(fp, arg); 552 default: 553 return -EINVAL; 554 } 555 } 556 557 static struct file_operations coproc_fops = { 558 .open = coproc_open, 559 .release = coproc_release, 560 .mmap = coproc_mmap, 561 .unlocked_ioctl = coproc_ioctl, 562 }; 563 564 /* 565 * Supporting only nx-gzip coprocessor type now, but this API code 566 * extended to other coprocessor types later. 567 */ 568 int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, 569 const char *name, 570 const struct vas_user_win_ops *vops) 571 { 572 int rc = -EINVAL; 573 dev_t devno; 574 575 rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name); 576 if (rc) { 577 pr_err("Unable to allocate coproc major number: %i\n", rc); 578 return rc; 579 } 580 581 pr_devel("%s device allocated, dev [%i,%i]\n", name, 582 MAJOR(coproc_device.devt), MINOR(coproc_device.devt)); 583 584 coproc_device.class = class_create(name); 585 if (IS_ERR(coproc_device.class)) { 586 rc = PTR_ERR(coproc_device.class); 587 pr_err("Unable to create %s class %d\n", name, rc); 588 goto err_class; 589 } 590 coproc_device.class->devnode = coproc_devnode; 591 coproc_device.cop_type = cop_type; 592 coproc_device.vops = vops; 593 594 coproc_fops.owner = mod; 595 cdev_init(&coproc_device.cdev, &coproc_fops); 596 597 devno = MKDEV(MAJOR(coproc_device.devt), 0); 598 rc = cdev_add(&coproc_device.cdev, devno, 1); 599 if (rc) { 600 pr_err("cdev_add() failed %d\n", rc); 601 goto err_cdev; 602 } 603 604 coproc_device.device = device_create(coproc_device.class, NULL, 605 devno, NULL, name, MINOR(devno)); 606 if (IS_ERR(coproc_device.device)) { 607 rc = PTR_ERR(coproc_device.device); 608 pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc); 609 goto err; 610 } 611 612 pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno), 613 MINOR(devno)); 614 615 return 0; 616 617 err: 618 cdev_del(&coproc_device.cdev); 619 err_cdev: 620 class_destroy(coproc_device.class); 621 err_class: 622 unregister_chrdev_region(coproc_device.devt, 1); 623 return rc; 624 } 625 626 void vas_unregister_coproc_api(void) 627 { 628 dev_t devno; 629 630 cdev_del(&coproc_device.cdev); 631 devno = MKDEV(MAJOR(coproc_device.devt), 0); 632 device_destroy(coproc_device.class, devno); 633 634 class_destroy(coproc_device.class); 635 unregister_chrdev_region(coproc_device.devt, 1); 636 } 637