1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * VAS user space API for its accelerators (Only NX-GZIP is supported now)
4  * Copyright (C) 2019 Haren Myneni, IBM Corp
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/device.h>
9 #include <linux/cdev.h>
10 #include <linux/fs.h>
11 #include <linux/slab.h>
12 #include <linux/uaccess.h>
13 #include <linux/kthread.h>
14 #include <linux/sched/signal.h>
15 #include <linux/mmu_context.h>
16 #include <linux/io.h>
17 #include <asm/vas.h>
18 #include <uapi/asm/vas-api.h>
19 
20 /*
21  * The driver creates the device node that can be used as follows:
22  * For NX-GZIP
23  *
24  *	fd = open("/dev/crypto/nx-gzip", O_RDWR);
25  *	rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
26  *	paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL).
27  *	vas_copy(&crb, 0, 1);
28  *	vas_paste(paste_addr, 0, 1);
29  *	close(fd) or exit process to close window.
30  *
31  * where "vas_copy" and "vas_paste" are defined in copy-paste.h.
32  * copy/paste returns to the user space directly. So refer NX hardware
33  * documentation for exact copy/paste usage and completion / error
34  * conditions.
35  */
36 
37 /*
38  * Wrapper object for the nx-gzip device - there is just one instance of
39  * this node for the whole system.
40  */
41 static struct coproc_dev {
42 	struct cdev cdev;
43 	struct device *device;
44 	char *name;
45 	dev_t devt;
46 	struct class *class;
47 	enum vas_cop_type cop_type;
48 	const struct vas_user_win_ops *vops;
49 } coproc_device;
50 
51 struct coproc_instance {
52 	struct coproc_dev *coproc;
53 	struct vas_window *txwin;
54 };
55 
56 static char *coproc_devnode(struct device *dev, umode_t *mode)
57 {
58 	return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev));
59 }
60 
61 /*
62  * Take reference to pid and mm
63  */
64 int get_vas_user_win_ref(struct vas_user_win_ref *task_ref)
65 {
66 	/*
67 	 * Window opened by a child thread may not be closed when
68 	 * it exits. So take reference to its pid and release it
69 	 * when the window is free by parent thread.
70 	 * Acquire a reference to the task's pid to make sure
71 	 * pid will not be re-used - needed only for multithread
72 	 * applications.
73 	 */
74 	task_ref->pid = get_task_pid(current, PIDTYPE_PID);
75 	/*
76 	 * Acquire a reference to the task's mm.
77 	 */
78 	task_ref->mm = get_task_mm(current);
79 	if (!task_ref->mm) {
80 		put_pid(task_ref->pid);
81 		pr_err("VAS: pid(%d): mm_struct is not found\n",
82 				current->pid);
83 		return -EPERM;
84 	}
85 
86 	mmgrab(task_ref->mm);
87 	mmput(task_ref->mm);
88 	/*
89 	 * Process closes window during exit. In the case of
90 	 * multithread application, the child thread can open
91 	 * window and can exit without closing it. So takes tgid
92 	 * reference until window closed to make sure tgid is not
93 	 * reused.
94 	 */
95 	task_ref->tgid = find_get_pid(task_tgid_vnr(current));
96 
97 	return 0;
98 }
99 
100 /*
101  * Successful return must release the task reference with
102  * put_task_struct
103  */
104 static bool ref_get_pid_and_task(struct vas_user_win_ref *task_ref,
105 			  struct task_struct **tskp, struct pid **pidp)
106 {
107 	struct task_struct *tsk;
108 	struct pid *pid;
109 
110 	pid = task_ref->pid;
111 	tsk = get_pid_task(pid, PIDTYPE_PID);
112 	if (!tsk) {
113 		pid = task_ref->tgid;
114 		tsk = get_pid_task(pid, PIDTYPE_PID);
115 		/*
116 		 * Parent thread (tgid) will be closing window when it
117 		 * exits. So should not get here.
118 		 */
119 		if (WARN_ON_ONCE(!tsk))
120 			return false;
121 	}
122 
123 	/* Return if the task is exiting. */
124 	if (tsk->flags & PF_EXITING) {
125 		put_task_struct(tsk);
126 		return false;
127 	}
128 
129 	*tskp = tsk;
130 	*pidp = pid;
131 
132 	return true;
133 }
134 
135 /*
136  * Update the CSB to indicate a translation error.
137  *
138  * User space will be polling on CSB after the request is issued.
139  * If NX can handle the request without any issues, it updates CSB.
140  * Whereas if NX encounters page fault, the kernel will handle the
141  * fault and update CSB with translation error.
142  *
143  * If we are unable to update the CSB means copy_to_user failed due to
144  * invalid csb_addr, send a signal to the process.
145  */
146 void vas_update_csb(struct coprocessor_request_block *crb,
147 		    struct vas_user_win_ref *task_ref)
148 {
149 	struct coprocessor_status_block csb;
150 	struct kernel_siginfo info;
151 	struct task_struct *tsk;
152 	void __user *csb_addr;
153 	struct pid *pid;
154 	int rc;
155 
156 	/*
157 	 * NX user space windows can not be opened for task->mm=NULL
158 	 * and faults will not be generated for kernel requests.
159 	 */
160 	if (WARN_ON_ONCE(!task_ref->mm))
161 		return;
162 
163 	csb_addr = (void __user *)be64_to_cpu(crb->csb_addr);
164 
165 	memset(&csb, 0, sizeof(csb));
166 	csb.cc = CSB_CC_FAULT_ADDRESS;
167 	csb.ce = CSB_CE_TERMINATION;
168 	csb.cs = 0;
169 	csb.count = 0;
170 
171 	/*
172 	 * NX operates and returns in BE format as defined CRB struct.
173 	 * So saves fault_storage_addr in BE as NX pastes in FIFO and
174 	 * expects user space to convert to CPU format.
175 	 */
176 	csb.address = crb->stamp.nx.fault_storage_addr;
177 	csb.flags = 0;
178 
179 	/*
180 	 * Process closes send window after all pending NX requests are
181 	 * completed. In multi-thread applications, a child thread can
182 	 * open a window and can exit without closing it. May be some
183 	 * requests are pending or this window can be used by other
184 	 * threads later. We should handle faults if NX encounters
185 	 * pages faults on these requests. Update CSB with translation
186 	 * error and fault address. If csb_addr passed by user space is
187 	 * invalid, send SEGV signal to pid saved in window. If the
188 	 * child thread is not running, send the signal to tgid.
189 	 * Parent thread (tgid) will close this window upon its exit.
190 	 *
191 	 * pid and mm references are taken when window is opened by
192 	 * process (pid). So tgid is used only when child thread opens
193 	 * a window and exits without closing it.
194 	 */
195 
196 	if (!ref_get_pid_and_task(task_ref, &tsk, &pid))
197 		return;
198 
199 	kthread_use_mm(task_ref->mm);
200 	rc = copy_to_user(csb_addr, &csb, sizeof(csb));
201 	/*
202 	 * User space polls on csb.flags (first byte). So add barrier
203 	 * then copy first byte with csb flags update.
204 	 */
205 	if (!rc) {
206 		csb.flags = CSB_V;
207 		/* Make sure update to csb.flags is visible now */
208 		smp_mb();
209 		rc = copy_to_user(csb_addr, &csb, sizeof(u8));
210 	}
211 	kthread_unuse_mm(task_ref->mm);
212 	put_task_struct(tsk);
213 
214 	/* Success */
215 	if (!rc)
216 		return;
217 
218 
219 	pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n",
220 			csb_addr, pid_vnr(pid));
221 
222 	clear_siginfo(&info);
223 	info.si_signo = SIGSEGV;
224 	info.si_errno = EFAULT;
225 	info.si_code = SEGV_MAPERR;
226 	info.si_addr = csb_addr;
227 	/*
228 	 * process will be polling on csb.flags after request is sent to
229 	 * NX. So generally CSB update should not fail except when an
230 	 * application passes invalid csb_addr. So an error message will
231 	 * be displayed and leave it to user space whether to ignore or
232 	 * handle this signal.
233 	 */
234 	rcu_read_lock();
235 	rc = kill_pid_info(SIGSEGV, &info, pid);
236 	rcu_read_unlock();
237 
238 	pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__,
239 			pid_vnr(pid), rc);
240 }
241 
242 void vas_dump_crb(struct coprocessor_request_block *crb)
243 {
244 	struct data_descriptor_entry *dde;
245 	struct nx_fault_stamp *nx;
246 
247 	dde = &crb->source;
248 	pr_devel("SrcDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n",
249 		be64_to_cpu(dde->address), be32_to_cpu(dde->length),
250 		dde->count, dde->index, dde->flags);
251 
252 	dde = &crb->target;
253 	pr_devel("TgtDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n",
254 		be64_to_cpu(dde->address), be32_to_cpu(dde->length),
255 		dde->count, dde->index, dde->flags);
256 
257 	nx = &crb->stamp.nx;
258 	pr_devel("NX Stamp: PSWID 0x%x, FSA 0x%llx, flags 0x%x, FS 0x%x\n",
259 		be32_to_cpu(nx->pswid),
260 		be64_to_cpu(crb->stamp.nx.fault_storage_addr),
261 		nx->flags, nx->fault_status);
262 }
263 
264 static int coproc_open(struct inode *inode, struct file *fp)
265 {
266 	struct coproc_instance *cp_inst;
267 
268 	cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL);
269 	if (!cp_inst)
270 		return -ENOMEM;
271 
272 	cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev,
273 					cdev);
274 	fp->private_data = cp_inst;
275 
276 	return 0;
277 }
278 
279 static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg)
280 {
281 	void __user *uptr = (void __user *)arg;
282 	struct vas_tx_win_open_attr uattr;
283 	struct coproc_instance *cp_inst;
284 	struct vas_window *txwin;
285 	int rc;
286 
287 	cp_inst = fp->private_data;
288 
289 	/*
290 	 * One window for file descriptor
291 	 */
292 	if (cp_inst->txwin)
293 		return -EEXIST;
294 
295 	rc = copy_from_user(&uattr, uptr, sizeof(uattr));
296 	if (rc) {
297 		pr_err("%s(): copy_from_user() returns %d\n", __func__, rc);
298 		return -EFAULT;
299 	}
300 
301 	if (uattr.version != 1) {
302 		pr_err("Invalid window open API version\n");
303 		return -EINVAL;
304 	}
305 
306 	if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->open_win) {
307 		pr_err("VAS API is not registered\n");
308 		return -EACCES;
309 	}
310 
311 	txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags,
312 						cp_inst->coproc->cop_type);
313 	if (IS_ERR(txwin)) {
314 		pr_err("%s() VAS window open failed, %ld\n", __func__,
315 				PTR_ERR(txwin));
316 		return PTR_ERR(txwin);
317 	}
318 
319 	mutex_init(&txwin->task_ref.mmap_mutex);
320 	cp_inst->txwin = txwin;
321 
322 	return 0;
323 }
324 
325 static int coproc_release(struct inode *inode, struct file *fp)
326 {
327 	struct coproc_instance *cp_inst = fp->private_data;
328 	int rc;
329 
330 	if (cp_inst->txwin) {
331 		if (cp_inst->coproc->vops &&
332 			cp_inst->coproc->vops->close_win) {
333 			rc = cp_inst->coproc->vops->close_win(cp_inst->txwin);
334 			if (rc)
335 				return rc;
336 		}
337 		cp_inst->txwin = NULL;
338 	}
339 
340 	kfree(cp_inst);
341 	fp->private_data = NULL;
342 
343 	/*
344 	 * We don't know here if user has other receive windows
345 	 * open, so we can't really call clear_thread_tidr().
346 	 * So, once the process calls set_thread_tidr(), the
347 	 * TIDR value sticks around until process exits, resulting
348 	 * in an extra copy in restore_sprs().
349 	 */
350 
351 	return 0;
352 }
353 
354 /*
355  * If the executed instruction that caused the fault was a paste, then
356  * clear regs CR0[EQ], advance NIP, and return 0. Else return error code.
357  */
358 static int do_fail_paste(void)
359 {
360 	struct pt_regs *regs = current->thread.regs;
361 	u32 instword;
362 
363 	if (WARN_ON_ONCE(!regs))
364 		return -EINVAL;
365 
366 	if (WARN_ON_ONCE(!user_mode(regs)))
367 		return -EINVAL;
368 
369 	/*
370 	 * If we couldn't translate the instruction, the driver should
371 	 * return success without handling the fault, it will be retried
372 	 * or the instruction fetch will fault.
373 	 */
374 	if (get_user(instword, (u32 __user *)(regs->nip)))
375 		return -EAGAIN;
376 
377 	/*
378 	 * Not a paste instruction, driver may fail the fault.
379 	 */
380 	if ((instword & PPC_INST_PASTE_MASK) != PPC_INST_PASTE)
381 		return -ENOENT;
382 
383 	regs->ccr &= ~0xe0000000;	/* Clear CR0[0-2] to fail paste */
384 	regs_add_return_ip(regs, 4);	/* Emulate the paste */
385 
386 	return 0;
387 }
388 
389 /*
390  * This fault handler is invoked when the core generates page fault on
391  * the paste address. Happens if the kernel closes window in hypervisor
392  * (on pseries) due to lost credit or the paste address is not mapped.
393  */
394 static vm_fault_t vas_mmap_fault(struct vm_fault *vmf)
395 {
396 	struct vm_area_struct *vma = vmf->vma;
397 	struct file *fp = vma->vm_file;
398 	struct coproc_instance *cp_inst = fp->private_data;
399 	struct vas_window *txwin;
400 	vm_fault_t fault;
401 	u64 paste_addr;
402 	int ret;
403 
404 	/*
405 	 * window is not opened. Shouldn't expect this error.
406 	 */
407 	if (!cp_inst || !cp_inst->txwin) {
408 		pr_err("%s(): Unexpected fault on paste address with TX window closed\n",
409 				__func__);
410 		return VM_FAULT_SIGBUS;
411 	}
412 
413 	txwin = cp_inst->txwin;
414 	/*
415 	 * When the LPAR lost credits due to core removal or during
416 	 * migration, invalidate the existing mapping for the current
417 	 * paste addresses and set windows in-active (zap_page_range in
418 	 * reconfig_close_windows()).
419 	 * New mapping will be done later after migration or new credits
420 	 * available. So continue to receive faults if the user space
421 	 * issue NX request.
422 	 */
423 	if (txwin->task_ref.vma != vmf->vma) {
424 		pr_err("%s(): No previous mapping with paste address\n",
425 			__func__);
426 		return VM_FAULT_SIGBUS;
427 	}
428 
429 	mutex_lock(&txwin->task_ref.mmap_mutex);
430 	/*
431 	 * The window may be inactive due to lost credit (Ex: core
432 	 * removal with DLPAR). If the window is active again when
433 	 * the credit is available, map the new paste address at the
434 	 * the window virtual address.
435 	 */
436 	if (txwin->status == VAS_WIN_ACTIVE) {
437 		paste_addr = cp_inst->coproc->vops->paste_addr(txwin);
438 		if (paste_addr) {
439 			fault = vmf_insert_pfn(vma, vma->vm_start,
440 					(paste_addr >> PAGE_SHIFT));
441 			mutex_unlock(&txwin->task_ref.mmap_mutex);
442 			return fault;
443 		}
444 	}
445 	mutex_unlock(&txwin->task_ref.mmap_mutex);
446 
447 	/*
448 	 * Received this fault due to closing the actual window.
449 	 * It can happen during migration or lost credits.
450 	 * Since no mapping, return the paste instruction failure
451 	 * to the user space.
452 	 */
453 	ret = do_fail_paste();
454 	/*
455 	 * The user space can retry several times until success (needed
456 	 * for migration) or should fallback to SW compression or
457 	 * manage with the existing open windows if available.
458 	 * Looking at sysfs interface, it can determine whether these
459 	 * failures are coming during migration or core removal:
460 	 * nr_used_credits > nr_total_credits when lost credits
461 	 */
462 	if (!ret || (ret == -EAGAIN))
463 		return VM_FAULT_NOPAGE;
464 
465 	return VM_FAULT_SIGBUS;
466 }
467 
468 static const struct vm_operations_struct vas_vm_ops = {
469 	.fault = vas_mmap_fault,
470 };
471 
472 static int coproc_mmap(struct file *fp, struct vm_area_struct *vma)
473 {
474 	struct coproc_instance *cp_inst = fp->private_data;
475 	struct vas_window *txwin;
476 	unsigned long pfn;
477 	u64 paste_addr;
478 	pgprot_t prot;
479 	int rc;
480 
481 	txwin = cp_inst->txwin;
482 
483 	if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
484 		pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__,
485 				(vma->vm_end - vma->vm_start), PAGE_SIZE);
486 		return -EINVAL;
487 	}
488 
489 	/* Ensure instance has an open send window */
490 	if (!txwin) {
491 		pr_err("%s(): No send window open?\n", __func__);
492 		return -EINVAL;
493 	}
494 
495 	if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) {
496 		pr_err("%s(): VAS API is not registered\n", __func__);
497 		return -EACCES;
498 	}
499 
500 	/*
501 	 * The initial mmap is done after the window is opened
502 	 * with ioctl. But before mmap(), this window can be closed in
503 	 * the hypervisor due to lost credit (core removal on pseries).
504 	 * So if the window is not active, return mmap() failure with
505 	 * -EACCES and expects the user space reissue mmap() when it
506 	 * is active again or open new window when the credit is available.
507 	 * mmap_mutex protects the paste address mmap() with DLPAR
508 	 * close/open event and allows mmap() only when the window is
509 	 * active.
510 	 */
511 	mutex_lock(&txwin->task_ref.mmap_mutex);
512 	if (txwin->status != VAS_WIN_ACTIVE) {
513 		pr_err("%s(): Window is not active\n", __func__);
514 		rc = -EACCES;
515 		goto out;
516 	}
517 
518 	paste_addr = cp_inst->coproc->vops->paste_addr(txwin);
519 	if (!paste_addr) {
520 		pr_err("%s(): Window paste address failed\n", __func__);
521 		rc = -EINVAL;
522 		goto out;
523 	}
524 
525 	pfn = paste_addr >> PAGE_SHIFT;
526 
527 	/* flags, page_prot from cxl_mmap(), except we want cachable */
528 	vma->vm_flags |= VM_IO | VM_PFNMAP;
529 	vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
530 
531 	prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY);
532 
533 	rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
534 			vma->vm_end - vma->vm_start, prot);
535 
536 	pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
537 			paste_addr, vma->vm_start, rc);
538 
539 	txwin->task_ref.vma = vma;
540 	vma->vm_ops = &vas_vm_ops;
541 
542 out:
543 	mutex_unlock(&txwin->task_ref.mmap_mutex);
544 	return rc;
545 }
546 
547 static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
548 {
549 	switch (cmd) {
550 	case VAS_TX_WIN_OPEN:
551 		return coproc_ioc_tx_win_open(fp, arg);
552 	default:
553 		return -EINVAL;
554 	}
555 }
556 
557 static struct file_operations coproc_fops = {
558 	.open = coproc_open,
559 	.release = coproc_release,
560 	.mmap = coproc_mmap,
561 	.unlocked_ioctl = coproc_ioctl,
562 };
563 
564 /*
565  * Supporting only nx-gzip coprocessor type now, but this API code
566  * extended to other coprocessor types later.
567  */
568 int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type,
569 			    const char *name,
570 			    const struct vas_user_win_ops *vops)
571 {
572 	int rc = -EINVAL;
573 	dev_t devno;
574 
575 	rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name);
576 	if (rc) {
577 		pr_err("Unable to allocate coproc major number: %i\n", rc);
578 		return rc;
579 	}
580 
581 	pr_devel("%s device allocated, dev [%i,%i]\n", name,
582 			MAJOR(coproc_device.devt), MINOR(coproc_device.devt));
583 
584 	coproc_device.class = class_create(mod, name);
585 	if (IS_ERR(coproc_device.class)) {
586 		rc = PTR_ERR(coproc_device.class);
587 		pr_err("Unable to create %s class %d\n", name, rc);
588 		goto err_class;
589 	}
590 	coproc_device.class->devnode = coproc_devnode;
591 	coproc_device.cop_type = cop_type;
592 	coproc_device.vops = vops;
593 
594 	coproc_fops.owner = mod;
595 	cdev_init(&coproc_device.cdev, &coproc_fops);
596 
597 	devno = MKDEV(MAJOR(coproc_device.devt), 0);
598 	rc = cdev_add(&coproc_device.cdev, devno, 1);
599 	if (rc) {
600 		pr_err("cdev_add() failed %d\n", rc);
601 		goto err_cdev;
602 	}
603 
604 	coproc_device.device = device_create(coproc_device.class, NULL,
605 			devno, NULL, name, MINOR(devno));
606 	if (IS_ERR(coproc_device.device)) {
607 		rc = PTR_ERR(coproc_device.device);
608 		pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc);
609 		goto err;
610 	}
611 
612 	pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno),
613 			MINOR(devno));
614 
615 	return 0;
616 
617 err:
618 	cdev_del(&coproc_device.cdev);
619 err_cdev:
620 	class_destroy(coproc_device.class);
621 err_class:
622 	unregister_chrdev_region(coproc_device.devt, 1);
623 	return rc;
624 }
625 
626 void vas_unregister_coproc_api(void)
627 {
628 	dev_t devno;
629 
630 	cdev_del(&coproc_device.cdev);
631 	devno = MKDEV(MAJOR(coproc_device.devt), 0);
632 	device_destroy(coproc_device.class, devno);
633 
634 	class_destroy(coproc_device.class);
635 	unregister_chrdev_region(coproc_device.devt, 1);
636 }
637