1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * VAS user space API for its accelerators (Only NX-GZIP is supported now)
4  * Copyright (C) 2019 Haren Myneni, IBM Corp
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/device.h>
9 #include <linux/cdev.h>
10 #include <linux/fs.h>
11 #include <linux/slab.h>
12 #include <linux/uaccess.h>
13 #include <linux/kthread.h>
14 #include <linux/sched/signal.h>
15 #include <linux/mmu_context.h>
16 #include <linux/io.h>
17 #include <asm/vas.h>
18 #include <uapi/asm/vas-api.h>
19 
20 /*
21  * The driver creates the device node that can be used as follows:
22  * For NX-GZIP
23  *
24  *	fd = open("/dev/crypto/nx-gzip", O_RDWR);
25  *	rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
26  *	paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL).
27  *	vas_copy(&crb, 0, 1);
28  *	vas_paste(paste_addr, 0, 1);
29  *	close(fd) or exit process to close window.
30  *
31  * where "vas_copy" and "vas_paste" are defined in copy-paste.h.
32  * copy/paste returns to the user space directly. So refer NX hardware
33  * documententation for exact copy/paste usage and completion / error
34  * conditions.
35  */
36 
37 /*
38  * Wrapper object for the nx-gzip device - there is just one instance of
39  * this node for the whole system.
40  */
41 static struct coproc_dev {
42 	struct cdev cdev;
43 	struct device *device;
44 	char *name;
45 	dev_t devt;
46 	struct class *class;
47 	enum vas_cop_type cop_type;
48 	const struct vas_user_win_ops *vops;
49 } coproc_device;
50 
51 struct coproc_instance {
52 	struct coproc_dev *coproc;
53 	struct vas_window *txwin;
54 };
55 
56 static char *coproc_devnode(struct device *dev, umode_t *mode)
57 {
58 	return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev));
59 }
60 
61 /*
62  * Take reference to pid and mm
63  */
64 int get_vas_user_win_ref(struct vas_user_win_ref *task_ref)
65 {
66 	/*
67 	 * Window opened by a child thread may not be closed when
68 	 * it exits. So take reference to its pid and release it
69 	 * when the window is free by parent thread.
70 	 * Acquire a reference to the task's pid to make sure
71 	 * pid will not be re-used - needed only for multithread
72 	 * applications.
73 	 */
74 	task_ref->pid = get_task_pid(current, PIDTYPE_PID);
75 	/*
76 	 * Acquire a reference to the task's mm.
77 	 */
78 	task_ref->mm = get_task_mm(current);
79 	if (!task_ref->mm) {
80 		put_pid(task_ref->pid);
81 		pr_err("VAS: pid(%d): mm_struct is not found\n",
82 				current->pid);
83 		return -EPERM;
84 	}
85 
86 	mmgrab(task_ref->mm);
87 	mmput(task_ref->mm);
88 	/*
89 	 * Process closes window during exit. In the case of
90 	 * multithread application, the child thread can open
91 	 * window and can exit without closing it. So takes tgid
92 	 * reference until window closed to make sure tgid is not
93 	 * reused.
94 	 */
95 	task_ref->tgid = find_get_pid(task_tgid_vnr(current));
96 
97 	return 0;
98 }
99 
100 /*
101  * Successful return must release the task reference with
102  * put_task_struct
103  */
104 static bool ref_get_pid_and_task(struct vas_user_win_ref *task_ref,
105 			  struct task_struct **tskp, struct pid **pidp)
106 {
107 	struct task_struct *tsk;
108 	struct pid *pid;
109 
110 	pid = task_ref->pid;
111 	tsk = get_pid_task(pid, PIDTYPE_PID);
112 	if (!tsk) {
113 		pid = task_ref->tgid;
114 		tsk = get_pid_task(pid, PIDTYPE_PID);
115 		/*
116 		 * Parent thread (tgid) will be closing window when it
117 		 * exits. So should not get here.
118 		 */
119 		if (WARN_ON_ONCE(!tsk))
120 			return false;
121 	}
122 
123 	/* Return if the task is exiting. */
124 	if (tsk->flags & PF_EXITING) {
125 		put_task_struct(tsk);
126 		return false;
127 	}
128 
129 	*tskp = tsk;
130 	*pidp = pid;
131 
132 	return true;
133 }
134 
135 /*
136  * Update the CSB to indicate a translation error.
137  *
138  * User space will be polling on CSB after the request is issued.
139  * If NX can handle the request without any issues, it updates CSB.
140  * Whereas if NX encounters page fault, the kernel will handle the
141  * fault and update CSB with translation error.
142  *
143  * If we are unable to update the CSB means copy_to_user failed due to
144  * invalid csb_addr, send a signal to the process.
145  */
146 void vas_update_csb(struct coprocessor_request_block *crb,
147 		    struct vas_user_win_ref *task_ref)
148 {
149 	struct coprocessor_status_block csb;
150 	struct kernel_siginfo info;
151 	struct task_struct *tsk;
152 	void __user *csb_addr;
153 	struct pid *pid;
154 	int rc;
155 
156 	/*
157 	 * NX user space windows can not be opened for task->mm=NULL
158 	 * and faults will not be generated for kernel requests.
159 	 */
160 	if (WARN_ON_ONCE(!task_ref->mm))
161 		return;
162 
163 	csb_addr = (void __user *)be64_to_cpu(crb->csb_addr);
164 
165 	memset(&csb, 0, sizeof(csb));
166 	csb.cc = CSB_CC_FAULT_ADDRESS;
167 	csb.ce = CSB_CE_TERMINATION;
168 	csb.cs = 0;
169 	csb.count = 0;
170 
171 	/*
172 	 * NX operates and returns in BE format as defined CRB struct.
173 	 * So saves fault_storage_addr in BE as NX pastes in FIFO and
174 	 * expects user space to convert to CPU format.
175 	 */
176 	csb.address = crb->stamp.nx.fault_storage_addr;
177 	csb.flags = 0;
178 
179 	/*
180 	 * Process closes send window after all pending NX requests are
181 	 * completed. In multi-thread applications, a child thread can
182 	 * open a window and can exit without closing it. May be some
183 	 * requests are pending or this window can be used by other
184 	 * threads later. We should handle faults if NX encounters
185 	 * pages faults on these requests. Update CSB with translation
186 	 * error and fault address. If csb_addr passed by user space is
187 	 * invalid, send SEGV signal to pid saved in window. If the
188 	 * child thread is not running, send the signal to tgid.
189 	 * Parent thread (tgid) will close this window upon its exit.
190 	 *
191 	 * pid and mm references are taken when window is opened by
192 	 * process (pid). So tgid is used only when child thread opens
193 	 * a window and exits without closing it.
194 	 */
195 
196 	if (!ref_get_pid_and_task(task_ref, &tsk, &pid))
197 		return;
198 
199 	kthread_use_mm(task_ref->mm);
200 	rc = copy_to_user(csb_addr, &csb, sizeof(csb));
201 	/*
202 	 * User space polls on csb.flags (first byte). So add barrier
203 	 * then copy first byte with csb flags update.
204 	 */
205 	if (!rc) {
206 		csb.flags = CSB_V;
207 		/* Make sure update to csb.flags is visible now */
208 		smp_mb();
209 		rc = copy_to_user(csb_addr, &csb, sizeof(u8));
210 	}
211 	kthread_unuse_mm(task_ref->mm);
212 	put_task_struct(tsk);
213 
214 	/* Success */
215 	if (!rc)
216 		return;
217 
218 
219 	pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n",
220 			csb_addr, pid_vnr(pid));
221 
222 	clear_siginfo(&info);
223 	info.si_signo = SIGSEGV;
224 	info.si_errno = EFAULT;
225 	info.si_code = SEGV_MAPERR;
226 	info.si_addr = csb_addr;
227 	/*
228 	 * process will be polling on csb.flags after request is sent to
229 	 * NX. So generally CSB update should not fail except when an
230 	 * application passes invalid csb_addr. So an error message will
231 	 * be displayed and leave it to user space whether to ignore or
232 	 * handle this signal.
233 	 */
234 	rcu_read_lock();
235 	rc = kill_pid_info(SIGSEGV, &info, pid);
236 	rcu_read_unlock();
237 
238 	pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__,
239 			pid_vnr(pid), rc);
240 }
241 
242 void vas_dump_crb(struct coprocessor_request_block *crb)
243 {
244 	struct data_descriptor_entry *dde;
245 	struct nx_fault_stamp *nx;
246 
247 	dde = &crb->source;
248 	pr_devel("SrcDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n",
249 		be64_to_cpu(dde->address), be32_to_cpu(dde->length),
250 		dde->count, dde->index, dde->flags);
251 
252 	dde = &crb->target;
253 	pr_devel("TgtDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n",
254 		be64_to_cpu(dde->address), be32_to_cpu(dde->length),
255 		dde->count, dde->index, dde->flags);
256 
257 	nx = &crb->stamp.nx;
258 	pr_devel("NX Stamp: PSWID 0x%x, FSA 0x%llx, flags 0x%x, FS 0x%x\n",
259 		be32_to_cpu(nx->pswid),
260 		be64_to_cpu(crb->stamp.nx.fault_storage_addr),
261 		nx->flags, nx->fault_status);
262 }
263 
264 static int coproc_open(struct inode *inode, struct file *fp)
265 {
266 	struct coproc_instance *cp_inst;
267 
268 	cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL);
269 	if (!cp_inst)
270 		return -ENOMEM;
271 
272 	cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev,
273 					cdev);
274 	fp->private_data = cp_inst;
275 
276 	return 0;
277 }
278 
279 static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg)
280 {
281 	void __user *uptr = (void __user *)arg;
282 	struct vas_tx_win_open_attr uattr;
283 	struct coproc_instance *cp_inst;
284 	struct vas_window *txwin;
285 	int rc;
286 
287 	cp_inst = fp->private_data;
288 
289 	/*
290 	 * One window for file descriptor
291 	 */
292 	if (cp_inst->txwin)
293 		return -EEXIST;
294 
295 	rc = copy_from_user(&uattr, uptr, sizeof(uattr));
296 	if (rc) {
297 		pr_err("%s(): copy_from_user() returns %d\n", __func__, rc);
298 		return -EFAULT;
299 	}
300 
301 	if (uattr.version != 1) {
302 		pr_err("Invalid window open API version\n");
303 		return -EINVAL;
304 	}
305 
306 	if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->open_win) {
307 		pr_err("VAS API is not registered\n");
308 		return -EACCES;
309 	}
310 
311 	txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags,
312 						cp_inst->coproc->cop_type);
313 	if (IS_ERR(txwin)) {
314 		pr_err("%s() VAS window open failed, %ld\n", __func__,
315 				PTR_ERR(txwin));
316 		return PTR_ERR(txwin);
317 	}
318 
319 	cp_inst->txwin = txwin;
320 
321 	return 0;
322 }
323 
324 static int coproc_release(struct inode *inode, struct file *fp)
325 {
326 	struct coproc_instance *cp_inst = fp->private_data;
327 	int rc;
328 
329 	if (cp_inst->txwin) {
330 		if (cp_inst->coproc->vops &&
331 			cp_inst->coproc->vops->close_win) {
332 			rc = cp_inst->coproc->vops->close_win(cp_inst->txwin);
333 			if (rc)
334 				return rc;
335 		}
336 		cp_inst->txwin = NULL;
337 	}
338 
339 	kfree(cp_inst);
340 	fp->private_data = NULL;
341 
342 	/*
343 	 * We don't know here if user has other receive windows
344 	 * open, so we can't really call clear_thread_tidr().
345 	 * So, once the process calls set_thread_tidr(), the
346 	 * TIDR value sticks around until process exits, resulting
347 	 * in an extra copy in restore_sprs().
348 	 */
349 
350 	return 0;
351 }
352 
353 static int coproc_mmap(struct file *fp, struct vm_area_struct *vma)
354 {
355 	struct coproc_instance *cp_inst = fp->private_data;
356 	struct vas_window *txwin;
357 	unsigned long pfn;
358 	u64 paste_addr;
359 	pgprot_t prot;
360 	int rc;
361 
362 	txwin = cp_inst->txwin;
363 
364 	if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
365 		pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__,
366 				(vma->vm_end - vma->vm_start), PAGE_SIZE);
367 		return -EINVAL;
368 	}
369 
370 	/* Ensure instance has an open send window */
371 	if (!txwin) {
372 		pr_err("%s(): No send window open?\n", __func__);
373 		return -EINVAL;
374 	}
375 
376 	if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) {
377 		pr_err("%s(): VAS API is not registered\n", __func__);
378 		return -EACCES;
379 	}
380 
381 	paste_addr = cp_inst->coproc->vops->paste_addr(txwin);
382 	if (!paste_addr) {
383 		pr_err("%s(): Window paste address failed\n", __func__);
384 		return -EINVAL;
385 	}
386 
387 	pfn = paste_addr >> PAGE_SHIFT;
388 
389 	/* flags, page_prot from cxl_mmap(), except we want cachable */
390 	vma->vm_flags |= VM_IO | VM_PFNMAP;
391 	vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
392 
393 	prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY);
394 
395 	rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
396 			vma->vm_end - vma->vm_start, prot);
397 
398 	pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
399 			paste_addr, vma->vm_start, rc);
400 
401 	return rc;
402 }
403 
404 static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
405 {
406 	switch (cmd) {
407 	case VAS_TX_WIN_OPEN:
408 		return coproc_ioc_tx_win_open(fp, arg);
409 	default:
410 		return -EINVAL;
411 	}
412 }
413 
414 static struct file_operations coproc_fops = {
415 	.open = coproc_open,
416 	.release = coproc_release,
417 	.mmap = coproc_mmap,
418 	.unlocked_ioctl = coproc_ioctl,
419 };
420 
421 /*
422  * Supporting only nx-gzip coprocessor type now, but this API code
423  * extended to other coprocessor types later.
424  */
425 int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type,
426 			    const char *name,
427 			    const struct vas_user_win_ops *vops)
428 {
429 	int rc = -EINVAL;
430 	dev_t devno;
431 
432 	rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name);
433 	if (rc) {
434 		pr_err("Unable to allocate coproc major number: %i\n", rc);
435 		return rc;
436 	}
437 
438 	pr_devel("%s device allocated, dev [%i,%i]\n", name,
439 			MAJOR(coproc_device.devt), MINOR(coproc_device.devt));
440 
441 	coproc_device.class = class_create(mod, name);
442 	if (IS_ERR(coproc_device.class)) {
443 		rc = PTR_ERR(coproc_device.class);
444 		pr_err("Unable to create %s class %d\n", name, rc);
445 		goto err_class;
446 	}
447 	coproc_device.class->devnode = coproc_devnode;
448 	coproc_device.cop_type = cop_type;
449 	coproc_device.vops = vops;
450 
451 	coproc_fops.owner = mod;
452 	cdev_init(&coproc_device.cdev, &coproc_fops);
453 
454 	devno = MKDEV(MAJOR(coproc_device.devt), 0);
455 	rc = cdev_add(&coproc_device.cdev, devno, 1);
456 	if (rc) {
457 		pr_err("cdev_add() failed %d\n", rc);
458 		goto err_cdev;
459 	}
460 
461 	coproc_device.device = device_create(coproc_device.class, NULL,
462 			devno, NULL, name, MINOR(devno));
463 	if (IS_ERR(coproc_device.device)) {
464 		rc = PTR_ERR(coproc_device.device);
465 		pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc);
466 		goto err;
467 	}
468 
469 	pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno),
470 			MINOR(devno));
471 
472 	return 0;
473 
474 err:
475 	cdev_del(&coproc_device.cdev);
476 err_cdev:
477 	class_destroy(coproc_device.class);
478 err_class:
479 	unregister_chrdev_region(coproc_device.devt, 1);
480 	return rc;
481 }
482 
483 void vas_unregister_coproc_api(void)
484 {
485 	dev_t devno;
486 
487 	cdev_del(&coproc_device.cdev);
488 	devno = MKDEV(MAJOR(coproc_device.devt), 0);
489 	device_destroy(coproc_device.class, devno);
490 
491 	class_destroy(coproc_device.class);
492 	unregister_chrdev_region(coproc_device.devt, 1);
493 }
494