xref: /openbmc/linux/drivers/misc/cxl/api.c (revision d8d2af70)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright 2014 IBM Corp.
4  */
5 
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/file.h>
9 #include <misc/cxl.h>
10 #include <linux/module.h>
11 #include <linux/mount.h>
12 #include <linux/pseudo_fs.h>
13 #include <linux/sched/mm.h>
14 #include <linux/mmu_context.h>
15 #include <linux/irqdomain.h>
16 
17 #include "cxl.h"
18 
19 /*
20  * Since we want to track memory mappings to be able to force-unmap
21  * when the AFU is no longer reachable, we need an inode. For devices
22  * opened through the cxl user API, this is not a problem, but a
23  * userland process can also get a cxl fd through the cxl_get_fd()
24  * API, which is used by the cxlflash driver.
25  *
26  * Therefore we implement our own simple pseudo-filesystem and inode
27  * allocator. We don't use the anonymous inode, as we need the
28  * meta-data associated with it (address_space) and it is shared by
29  * other drivers/processes, so it could lead to cxl unmapping VMAs
30  * from random processes.
31  */
32 
33 #define CXL_PSEUDO_FS_MAGIC	0x1697697f
34 
35 static int cxl_fs_cnt;
36 static struct vfsmount *cxl_vfs_mount;
37 
cxl_fs_init_fs_context(struct fs_context * fc)38 static int cxl_fs_init_fs_context(struct fs_context *fc)
39 {
40 	return init_pseudo(fc, CXL_PSEUDO_FS_MAGIC) ? 0 : -ENOMEM;
41 }
42 
43 static struct file_system_type cxl_fs_type = {
44 	.name		= "cxl",
45 	.owner		= THIS_MODULE,
46 	.init_fs_context = cxl_fs_init_fs_context,
47 	.kill_sb	= kill_anon_super,
48 };
49 
50 
cxl_release_mapping(struct cxl_context * ctx)51 void cxl_release_mapping(struct cxl_context *ctx)
52 {
53 	if (ctx->kernelapi && ctx->mapping)
54 		simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
55 }
56 
cxl_getfile(const char * name,const struct file_operations * fops,void * priv,int flags)57 static struct file *cxl_getfile(const char *name,
58 				const struct file_operations *fops,
59 				void *priv, int flags)
60 {
61 	struct file *file;
62 	struct inode *inode;
63 	int rc;
64 
65 	/* strongly inspired by anon_inode_getfile() */
66 
67 	if (fops->owner && !try_module_get(fops->owner))
68 		return ERR_PTR(-ENOENT);
69 
70 	rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);
71 	if (rc < 0) {
72 		pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);
73 		file = ERR_PTR(rc);
74 		goto err_module;
75 	}
76 
77 	inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);
78 	if (IS_ERR(inode)) {
79 		file = ERR_CAST(inode);
80 		goto err_fs;
81 	}
82 
83 	file = alloc_file_pseudo(inode, cxl_vfs_mount, name,
84 				 flags & (O_ACCMODE | O_NONBLOCK), fops);
85 	if (IS_ERR(file))
86 		goto err_inode;
87 
88 	file->private_data = priv;
89 
90 	return file;
91 
92 err_inode:
93 	iput(inode);
94 err_fs:
95 	simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
96 err_module:
97 	module_put(fops->owner);
98 	return file;
99 }
100 
cxl_dev_context_init(struct pci_dev * dev)101 struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
102 {
103 	struct cxl_afu *afu;
104 	struct cxl_context  *ctx;
105 	int rc;
106 
107 	afu = cxl_pci_to_afu(dev);
108 	if (IS_ERR(afu))
109 		return ERR_CAST(afu);
110 
111 	ctx = cxl_context_alloc();
112 	if (!ctx)
113 		return ERR_PTR(-ENOMEM);
114 
115 	ctx->kernelapi = true;
116 
117 	/* Make it a slave context.  We can promote it later? */
118 	rc = cxl_context_init(ctx, afu, false);
119 	if (rc)
120 		goto err_ctx;
121 
122 	return ctx;
123 
124 err_ctx:
125 	kfree(ctx);
126 	return ERR_PTR(rc);
127 }
128 EXPORT_SYMBOL_GPL(cxl_dev_context_init);
129 
cxl_get_context(struct pci_dev * dev)130 struct cxl_context *cxl_get_context(struct pci_dev *dev)
131 {
132 	return dev->dev.archdata.cxl_ctx;
133 }
134 EXPORT_SYMBOL_GPL(cxl_get_context);
135 
cxl_release_context(struct cxl_context * ctx)136 int cxl_release_context(struct cxl_context *ctx)
137 {
138 	if (ctx->status >= STARTED)
139 		return -EBUSY;
140 
141 	cxl_context_free(ctx);
142 
143 	return 0;
144 }
145 EXPORT_SYMBOL_GPL(cxl_release_context);
146 
cxl_find_afu_irq(struct cxl_context * ctx,int num)147 static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
148 {
149 	__u16 range;
150 	int r;
151 
152 	for (r = 0; r < CXL_IRQ_RANGES; r++) {
153 		range = ctx->irqs.range[r];
154 		if (num < range) {
155 			return ctx->irqs.offset[r] + num;
156 		}
157 		num -= range;
158 	}
159 	return 0;
160 }
161 
162 
cxl_set_priv(struct cxl_context * ctx,void * priv)163 int cxl_set_priv(struct cxl_context *ctx, void *priv)
164 {
165 	if (!ctx)
166 		return -EINVAL;
167 
168 	ctx->priv = priv;
169 
170 	return 0;
171 }
172 EXPORT_SYMBOL_GPL(cxl_set_priv);
173 
cxl_get_priv(struct cxl_context * ctx)174 void *cxl_get_priv(struct cxl_context *ctx)
175 {
176 	if (!ctx)
177 		return ERR_PTR(-EINVAL);
178 
179 	return ctx->priv;
180 }
181 EXPORT_SYMBOL_GPL(cxl_get_priv);
182 
cxl_allocate_afu_irqs(struct cxl_context * ctx,int num)183 int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num)
184 {
185 	int res;
186 	irq_hw_number_t hwirq;
187 
188 	if (num == 0)
189 		num = ctx->afu->pp_irqs;
190 	res = afu_allocate_irqs(ctx, num);
191 	if (res)
192 		return res;
193 
194 	if (!cpu_has_feature(CPU_FTR_HVMODE)) {
195 		/* In a guest, the PSL interrupt is not multiplexed. It was
196 		 * allocated above, and we need to set its handler
197 		 */
198 		hwirq = cxl_find_afu_irq(ctx, 0);
199 		if (hwirq)
200 			cxl_map_irq(ctx->afu->adapter, hwirq, cxl_ops->psl_interrupt, ctx, "psl");
201 	}
202 
203 	if (ctx->status == STARTED) {
204 		if (cxl_ops->update_ivtes)
205 			cxl_ops->update_ivtes(ctx);
206 		else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");
207 	}
208 
209 	return res;
210 }
211 EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs);
212 
cxl_free_afu_irqs(struct cxl_context * ctx)213 void cxl_free_afu_irqs(struct cxl_context *ctx)
214 {
215 	irq_hw_number_t hwirq;
216 	unsigned int virq;
217 
218 	if (!cpu_has_feature(CPU_FTR_HVMODE)) {
219 		hwirq = cxl_find_afu_irq(ctx, 0);
220 		if (hwirq) {
221 			virq = irq_find_mapping(NULL, hwirq);
222 			if (virq)
223 				cxl_unmap_irq(virq, ctx);
224 		}
225 	}
226 	afu_irq_name_free(ctx);
227 	cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
228 }
229 EXPORT_SYMBOL_GPL(cxl_free_afu_irqs);
230 
cxl_map_afu_irq(struct cxl_context * ctx,int num,irq_handler_t handler,void * cookie,char * name)231 int cxl_map_afu_irq(struct cxl_context *ctx, int num,
232 		    irq_handler_t handler, void *cookie, char *name)
233 {
234 	irq_hw_number_t hwirq;
235 
236 	/*
237 	 * Find interrupt we are to register.
238 	 */
239 	hwirq = cxl_find_afu_irq(ctx, num);
240 	if (!hwirq)
241 		return -ENOENT;
242 
243 	return cxl_map_irq(ctx->afu->adapter, hwirq, handler, cookie, name);
244 }
245 EXPORT_SYMBOL_GPL(cxl_map_afu_irq);
246 
cxl_unmap_afu_irq(struct cxl_context * ctx,int num,void * cookie)247 void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie)
248 {
249 	irq_hw_number_t hwirq;
250 	unsigned int virq;
251 
252 	hwirq = cxl_find_afu_irq(ctx, num);
253 	if (!hwirq)
254 		return;
255 
256 	virq = irq_find_mapping(NULL, hwirq);
257 	if (virq)
258 		cxl_unmap_irq(virq, cookie);
259 }
260 EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq);
261 
262 /*
263  * Start a context
264  * Code here similar to afu_ioctl_start_work().
265  */
cxl_start_context(struct cxl_context * ctx,u64 wed,struct task_struct * task)266 int cxl_start_context(struct cxl_context *ctx, u64 wed,
267 		      struct task_struct *task)
268 {
269 	int rc = 0;
270 	bool kernel = true;
271 
272 	pr_devel("%s: pe: %i\n", __func__, ctx->pe);
273 
274 	mutex_lock(&ctx->status_mutex);
275 	if (ctx->status == STARTED)
276 		goto out; /* already started */
277 
278 	/*
279 	 * Increment the mapped context count for adapter. This also checks
280 	 * if adapter_context_lock is taken.
281 	 */
282 	rc = cxl_adapter_context_get(ctx->afu->adapter);
283 	if (rc)
284 		goto out;
285 
286 	if (task) {
287 		ctx->pid = get_task_pid(task, PIDTYPE_PID);
288 		kernel = false;
289 
290 		/* acquire a reference to the task's mm */
291 		ctx->mm = get_task_mm(current);
292 
293 		/* ensure this mm_struct can't be freed */
294 		cxl_context_mm_count_get(ctx);
295 
296 		if (ctx->mm) {
297 			/* decrement the use count from above */
298 			mmput(ctx->mm);
299 			/* make TLBIs for this context global */
300 			mm_context_add_copro(ctx->mm);
301 		}
302 	}
303 
304 	/*
305 	 * Increment driver use count. Enables global TLBIs for hash
306 	 * and callbacks to handle the segment table
307 	 */
308 	cxl_ctx_get();
309 
310 	/* See the comment in afu_ioctl_start_work() */
311 	smp_mb();
312 
313 	if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {
314 		put_pid(ctx->pid);
315 		ctx->pid = NULL;
316 		cxl_adapter_context_put(ctx->afu->adapter);
317 		cxl_ctx_put();
318 		if (task) {
319 			cxl_context_mm_count_put(ctx);
320 			if (ctx->mm)
321 				mm_context_remove_copro(ctx->mm);
322 		}
323 		goto out;
324 	}
325 
326 	ctx->status = STARTED;
327 out:
328 	mutex_unlock(&ctx->status_mutex);
329 	return rc;
330 }
331 EXPORT_SYMBOL_GPL(cxl_start_context);
332 
cxl_process_element(struct cxl_context * ctx)333 int cxl_process_element(struct cxl_context *ctx)
334 {
335 	return ctx->external_pe;
336 }
337 EXPORT_SYMBOL_GPL(cxl_process_element);
338 
339 /* Stop a context.  Returns 0 on success, otherwise -Errno */
cxl_stop_context(struct cxl_context * ctx)340 int cxl_stop_context(struct cxl_context *ctx)
341 {
342 	return __detach_context(ctx);
343 }
344 EXPORT_SYMBOL_GPL(cxl_stop_context);
345 
cxl_set_master(struct cxl_context * ctx)346 void cxl_set_master(struct cxl_context *ctx)
347 {
348 	ctx->master = true;
349 }
350 EXPORT_SYMBOL_GPL(cxl_set_master);
351 
352 /* wrappers around afu_* file ops which are EXPORTED */
cxl_fd_open(struct inode * inode,struct file * file)353 int cxl_fd_open(struct inode *inode, struct file *file)
354 {
355 	return afu_open(inode, file);
356 }
357 EXPORT_SYMBOL_GPL(cxl_fd_open);
cxl_fd_release(struct inode * inode,struct file * file)358 int cxl_fd_release(struct inode *inode, struct file *file)
359 {
360 	return afu_release(inode, file);
361 }
362 EXPORT_SYMBOL_GPL(cxl_fd_release);
cxl_fd_ioctl(struct file * file,unsigned int cmd,unsigned long arg)363 long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
364 {
365 	return afu_ioctl(file, cmd, arg);
366 }
367 EXPORT_SYMBOL_GPL(cxl_fd_ioctl);
cxl_fd_mmap(struct file * file,struct vm_area_struct * vm)368 int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)
369 {
370 	return afu_mmap(file, vm);
371 }
372 EXPORT_SYMBOL_GPL(cxl_fd_mmap);
cxl_fd_poll(struct file * file,struct poll_table_struct * poll)373 __poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
374 {
375 	return afu_poll(file, poll);
376 }
377 EXPORT_SYMBOL_GPL(cxl_fd_poll);
cxl_fd_read(struct file * file,char __user * buf,size_t count,loff_t * off)378 ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count,
379 			loff_t *off)
380 {
381 	return afu_read(file, buf, count, off);
382 }
383 EXPORT_SYMBOL_GPL(cxl_fd_read);
384 
385 #define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME
386 
387 /* Get a struct file and fd for a context and attach the ops */
cxl_get_fd(struct cxl_context * ctx,struct file_operations * fops,int * fd)388 struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
389 			int *fd)
390 {
391 	struct file *file;
392 	int rc, flags, fdtmp;
393 	char *name = NULL;
394 
395 	/* only allow one per context */
396 	if (ctx->mapping)
397 		return ERR_PTR(-EEXIST);
398 
399 	flags = O_RDWR | O_CLOEXEC;
400 
401 	/* This code is similar to anon_inode_getfd() */
402 	rc = get_unused_fd_flags(flags);
403 	if (rc < 0)
404 		return ERR_PTR(rc);
405 	fdtmp = rc;
406 
407 	/*
408 	 * Patch the file ops.  Needs to be careful that this is rentrant safe.
409 	 */
410 	if (fops) {
411 		PATCH_FOPS(open);
412 		PATCH_FOPS(poll);
413 		PATCH_FOPS(read);
414 		PATCH_FOPS(release);
415 		PATCH_FOPS(unlocked_ioctl);
416 		PATCH_FOPS(compat_ioctl);
417 		PATCH_FOPS(mmap);
418 	} else /* use default ops */
419 		fops = (struct file_operations *)&afu_fops;
420 
421 	name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);
422 	file = cxl_getfile(name, fops, ctx, flags);
423 	kfree(name);
424 	if (IS_ERR(file))
425 		goto err_fd;
426 
427 	cxl_context_set_mapping(ctx, file->f_mapping);
428 	*fd = fdtmp;
429 	return file;
430 
431 err_fd:
432 	put_unused_fd(fdtmp);
433 	return NULL;
434 }
435 EXPORT_SYMBOL_GPL(cxl_get_fd);
436 
cxl_fops_get_context(struct file * file)437 struct cxl_context *cxl_fops_get_context(struct file *file)
438 {
439 	return file->private_data;
440 }
441 EXPORT_SYMBOL_GPL(cxl_fops_get_context);
442 
cxl_set_driver_ops(struct cxl_context * ctx,struct cxl_afu_driver_ops * ops)443 void cxl_set_driver_ops(struct cxl_context *ctx,
444 			struct cxl_afu_driver_ops *ops)
445 {
446 	WARN_ON(!ops->fetch_event || !ops->event_delivered);
447 	atomic_set(&ctx->afu_driver_events, 0);
448 	ctx->afu_driver_ops = ops;
449 }
450 EXPORT_SYMBOL_GPL(cxl_set_driver_ops);
451 
cxl_context_events_pending(struct cxl_context * ctx,unsigned int new_events)452 void cxl_context_events_pending(struct cxl_context *ctx,
453 				unsigned int new_events)
454 {
455 	atomic_add(new_events, &ctx->afu_driver_events);
456 	wake_up_all(&ctx->wq);
457 }
458 EXPORT_SYMBOL_GPL(cxl_context_events_pending);
459 
cxl_start_work(struct cxl_context * ctx,struct cxl_ioctl_start_work * work)460 int cxl_start_work(struct cxl_context *ctx,
461 		   struct cxl_ioctl_start_work *work)
462 {
463 	int rc;
464 
465 	/* code taken from afu_ioctl_start_work */
466 	if (!(work->flags & CXL_START_WORK_NUM_IRQS))
467 		work->num_interrupts = ctx->afu->pp_irqs;
468 	else if ((work->num_interrupts < ctx->afu->pp_irqs) ||
469 		 (work->num_interrupts > ctx->afu->irqs_max)) {
470 		return -EINVAL;
471 	}
472 
473 	rc = afu_register_irqs(ctx, work->num_interrupts);
474 	if (rc)
475 		return rc;
476 
477 	rc = cxl_start_context(ctx, work->work_element_descriptor, current);
478 	if (rc < 0) {
479 		afu_release_irqs(ctx, ctx);
480 		return rc;
481 	}
482 
483 	return 0;
484 }
485 EXPORT_SYMBOL_GPL(cxl_start_work);
486 
cxl_psa_map(struct cxl_context * ctx)487 void __iomem *cxl_psa_map(struct cxl_context *ctx)
488 {
489 	if (ctx->status != STARTED)
490 		return NULL;
491 
492 	pr_devel("%s: psn_phys%llx size:%llx\n",
493 		__func__, ctx->psn_phys, ctx->psn_size);
494 	return ioremap(ctx->psn_phys, ctx->psn_size);
495 }
496 EXPORT_SYMBOL_GPL(cxl_psa_map);
497 
cxl_psa_unmap(void __iomem * addr)498 void cxl_psa_unmap(void __iomem *addr)
499 {
500 	iounmap(addr);
501 }
502 EXPORT_SYMBOL_GPL(cxl_psa_unmap);
503 
cxl_afu_reset(struct cxl_context * ctx)504 int cxl_afu_reset(struct cxl_context *ctx)
505 {
506 	struct cxl_afu *afu = ctx->afu;
507 	int rc;
508 
509 	rc = cxl_ops->afu_reset(afu);
510 	if (rc)
511 		return rc;
512 
513 	return cxl_ops->afu_check_and_enable(afu);
514 }
515 EXPORT_SYMBOL_GPL(cxl_afu_reset);
516 
cxl_perst_reloads_same_image(struct cxl_afu * afu,bool perst_reloads_same_image)517 void cxl_perst_reloads_same_image(struct cxl_afu *afu,
518 				  bool perst_reloads_same_image)
519 {
520 	afu->adapter->perst_same_image = perst_reloads_same_image;
521 }
522 EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image);
523 
cxl_read_adapter_vpd(struct pci_dev * dev,void * buf,size_t count)524 ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
525 {
526 	struct cxl_afu *afu = cxl_pci_to_afu(dev);
527 	if (IS_ERR(afu))
528 		return -ENODEV;
529 
530 	return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
531 }
532 EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
533