1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/fs.h> 4 #include <linux/poll.h> 5 #include <linux/sched/signal.h> 6 #include <linux/uaccess.h> 7 #include <uapi/misc/ocxl.h> 8 #include "ocxl_internal.h" 9 10 11 #define OCXL_NUM_MINORS 256 /* Total to reserve */ 12 13 static dev_t ocxl_dev; 14 static struct class *ocxl_class; 15 static struct mutex minors_idr_lock; 16 static struct idr minors_idr; 17 18 static struct ocxl_afu *find_and_get_afu(dev_t devno) 19 { 20 struct ocxl_afu *afu; 21 int afu_minor; 22 23 afu_minor = MINOR(devno); 24 /* 25 * We don't declare an RCU critical section here, as our AFU 26 * is protected by a reference counter on the device. By the time the 27 * minor number of a device is removed from the idr, the ref count of 28 * the device is already at 0, so no user API will access that AFU and 29 * this function can't return it. 30 */ 31 afu = idr_find(&minors_idr, afu_minor); 32 if (afu) 33 ocxl_afu_get(afu); 34 return afu; 35 } 36 37 static int allocate_afu_minor(struct ocxl_afu *afu) 38 { 39 int minor; 40 41 mutex_lock(&minors_idr_lock); 42 minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); 43 mutex_unlock(&minors_idr_lock); 44 return minor; 45 } 46 47 static void free_afu_minor(struct ocxl_afu *afu) 48 { 49 mutex_lock(&minors_idr_lock); 50 idr_remove(&minors_idr, MINOR(afu->dev.devt)); 51 mutex_unlock(&minors_idr_lock); 52 } 53 54 static int afu_open(struct inode *inode, struct file *file) 55 { 56 struct ocxl_afu *afu; 57 struct ocxl_context *ctx; 58 int rc; 59 60 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 61 62 afu = find_and_get_afu(inode->i_rdev); 63 if (!afu) 64 return -ENODEV; 65 66 ctx = ocxl_context_alloc(); 67 if (!ctx) { 68 rc = -ENOMEM; 69 goto put_afu; 70 } 71 72 rc = ocxl_context_init(ctx, afu, inode->i_mapping); 73 if (rc) 74 goto put_afu; 75 file->private_data = ctx; 76 ocxl_afu_put(afu); 77 return 0; 78 79 put_afu: 80 ocxl_afu_put(afu); 81 return rc; 82 } 83 84 static long afu_ioctl_attach(struct ocxl_context *ctx, 85 struct ocxl_ioctl_attach __user *uarg) 86 { 87 struct ocxl_ioctl_attach arg; 88 u64 amr = 0; 89 int rc; 90 91 pr_debug("%s for context %d\n", __func__, ctx->pasid); 92 93 if (copy_from_user(&arg, uarg, sizeof(arg))) 94 return -EFAULT; 95 96 /* Make sure reserved fields are not set for forward compatibility */ 97 if (arg.reserved1 || arg.reserved2 || arg.reserved3) 98 return -EINVAL; 99 100 amr = arg.amr & mfspr(SPRN_UAMOR); 101 rc = ocxl_context_attach(ctx, amr); 102 return rc; 103 } 104 105 #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ 106 x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ 107 x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ 108 x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ 109 "UNKNOWN") 110 111 static long afu_ioctl(struct file *file, unsigned int cmd, 112 unsigned long args) 113 { 114 struct ocxl_context *ctx = file->private_data; 115 struct ocxl_ioctl_irq_fd irq_fd; 116 u64 irq_offset; 117 long rc; 118 119 pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, 120 CMD_STR(cmd)); 121 122 if (ctx->status == CLOSED) 123 return -EIO; 124 125 switch (cmd) { 126 case OCXL_IOCTL_ATTACH: 127 rc = afu_ioctl_attach(ctx, 128 (struct ocxl_ioctl_attach __user *) args); 129 break; 130 131 case OCXL_IOCTL_IRQ_ALLOC: 132 rc = ocxl_afu_irq_alloc(ctx, &irq_offset); 133 if (!rc) { 134 rc = copy_to_user((u64 __user *) args, &irq_offset, 135 sizeof(irq_offset)); 136 if (rc) 137 ocxl_afu_irq_free(ctx, irq_offset); 138 } 139 break; 140 141 case OCXL_IOCTL_IRQ_FREE: 142 rc = copy_from_user(&irq_offset, (u64 __user *) args, 143 sizeof(irq_offset)); 144 if (rc) 145 return -EFAULT; 146 rc = ocxl_afu_irq_free(ctx, irq_offset); 147 break; 148 149 case OCXL_IOCTL_IRQ_SET_FD: 150 rc = copy_from_user(&irq_fd, (u64 __user *) args, 151 sizeof(irq_fd)); 152 if (rc) 153 return -EFAULT; 154 if (irq_fd.reserved) 155 return -EINVAL; 156 rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, 157 irq_fd.eventfd); 158 break; 159 160 default: 161 rc = -EINVAL; 162 } 163 return rc; 164 } 165 166 static long afu_compat_ioctl(struct file *file, unsigned int cmd, 167 unsigned long args) 168 { 169 return afu_ioctl(file, cmd, args); 170 } 171 172 static int afu_mmap(struct file *file, struct vm_area_struct *vma) 173 { 174 struct ocxl_context *ctx = file->private_data; 175 176 pr_debug("%s for context %d\n", __func__, ctx->pasid); 177 return ocxl_context_mmap(ctx, vma); 178 } 179 180 static bool has_xsl_error(struct ocxl_context *ctx) 181 { 182 bool ret; 183 184 mutex_lock(&ctx->xsl_error_lock); 185 ret = !!ctx->xsl_error.addr; 186 mutex_unlock(&ctx->xsl_error_lock); 187 188 return ret; 189 } 190 191 /* 192 * Are there any events pending on the AFU 193 * ctx: The AFU context 194 * Returns: true if there are events pending 195 */ 196 static bool afu_events_pending(struct ocxl_context *ctx) 197 { 198 if (has_xsl_error(ctx)) 199 return true; 200 return false; 201 } 202 203 static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait) 204 { 205 struct ocxl_context *ctx = file->private_data; 206 unsigned int mask = 0; 207 bool closed; 208 209 pr_debug("%s for context %d\n", __func__, ctx->pasid); 210 211 poll_wait(file, &ctx->events_wq, wait); 212 213 mutex_lock(&ctx->status_mutex); 214 closed = (ctx->status == CLOSED); 215 mutex_unlock(&ctx->status_mutex); 216 217 if (afu_events_pending(ctx)) 218 mask = POLLIN | POLLRDNORM; 219 else if (closed) 220 mask = POLLERR; 221 222 return mask; 223 } 224 225 /* 226 * Populate the supplied buffer with a single XSL error 227 * ctx: The AFU context to report the error from 228 * header: the event header to populate 229 * buf: The buffer to write the body into (should be at least 230 * AFU_EVENT_BODY_XSL_ERROR_SIZE) 231 * Return: the amount of buffer that was populated 232 */ 233 static ssize_t append_xsl_error(struct ocxl_context *ctx, 234 struct ocxl_kernel_event_header *header, 235 char __user *buf) 236 { 237 struct ocxl_kernel_event_xsl_fault_error body; 238 239 memset(&body, 0, sizeof(body)); 240 241 mutex_lock(&ctx->xsl_error_lock); 242 if (!ctx->xsl_error.addr) { 243 mutex_unlock(&ctx->xsl_error_lock); 244 return 0; 245 } 246 247 body.addr = ctx->xsl_error.addr; 248 body.dsisr = ctx->xsl_error.dsisr; 249 body.count = ctx->xsl_error.count; 250 251 ctx->xsl_error.addr = 0; 252 ctx->xsl_error.dsisr = 0; 253 ctx->xsl_error.count = 0; 254 255 mutex_unlock(&ctx->xsl_error_lock); 256 257 header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR; 258 259 if (copy_to_user(buf, &body, sizeof(body))) 260 return -EFAULT; 261 262 return sizeof(body); 263 } 264 265 #define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error) 266 267 /* 268 * Reports events on the AFU 269 * Format: 270 * Header (struct ocxl_kernel_event_header) 271 * Body (struct ocxl_kernel_event_*) 272 * Header... 273 */ 274 static ssize_t afu_read(struct file *file, char __user *buf, size_t count, 275 loff_t *off) 276 { 277 struct ocxl_context *ctx = file->private_data; 278 struct ocxl_kernel_event_header header; 279 ssize_t rc; 280 size_t used = 0; 281 DEFINE_WAIT(event_wait); 282 283 memset(&header, 0, sizeof(header)); 284 285 /* Require offset to be 0 */ 286 if (*off != 0) 287 return -EINVAL; 288 289 if (count < (sizeof(struct ocxl_kernel_event_header) + 290 AFU_EVENT_BODY_MAX_SIZE)) 291 return -EINVAL; 292 293 for (;;) { 294 prepare_to_wait(&ctx->events_wq, &event_wait, 295 TASK_INTERRUPTIBLE); 296 297 if (afu_events_pending(ctx)) 298 break; 299 300 if (ctx->status == CLOSED) 301 break; 302 303 if (file->f_flags & O_NONBLOCK) { 304 finish_wait(&ctx->events_wq, &event_wait); 305 return -EAGAIN; 306 } 307 308 if (signal_pending(current)) { 309 finish_wait(&ctx->events_wq, &event_wait); 310 return -ERESTARTSYS; 311 } 312 313 schedule(); 314 } 315 316 finish_wait(&ctx->events_wq, &event_wait); 317 318 if (has_xsl_error(ctx)) { 319 used = append_xsl_error(ctx, &header, buf + sizeof(header)); 320 if (used < 0) 321 return used; 322 } 323 324 if (!afu_events_pending(ctx)) 325 header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST; 326 327 if (copy_to_user(buf, &header, sizeof(header))) 328 return -EFAULT; 329 330 used += sizeof(header); 331 332 rc = (ssize_t) used; 333 return rc; 334 } 335 336 static int afu_release(struct inode *inode, struct file *file) 337 { 338 struct ocxl_context *ctx = file->private_data; 339 int rc; 340 341 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 342 rc = ocxl_context_detach(ctx); 343 mutex_lock(&ctx->mapping_lock); 344 ctx->mapping = NULL; 345 mutex_unlock(&ctx->mapping_lock); 346 wake_up_all(&ctx->events_wq); 347 if (rc != -EBUSY) 348 ocxl_context_free(ctx); 349 return 0; 350 } 351 352 static const struct file_operations ocxl_afu_fops = { 353 .owner = THIS_MODULE, 354 .open = afu_open, 355 .unlocked_ioctl = afu_ioctl, 356 .compat_ioctl = afu_compat_ioctl, 357 .mmap = afu_mmap, 358 .poll = afu_poll, 359 .read = afu_read, 360 .release = afu_release, 361 }; 362 363 int ocxl_create_cdev(struct ocxl_afu *afu) 364 { 365 int rc; 366 367 cdev_init(&afu->cdev, &ocxl_afu_fops); 368 rc = cdev_add(&afu->cdev, afu->dev.devt, 1); 369 if (rc) { 370 dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); 371 return rc; 372 } 373 return 0; 374 } 375 376 void ocxl_destroy_cdev(struct ocxl_afu *afu) 377 { 378 cdev_del(&afu->cdev); 379 } 380 381 int ocxl_register_afu(struct ocxl_afu *afu) 382 { 383 int minor; 384 385 minor = allocate_afu_minor(afu); 386 if (minor < 0) 387 return minor; 388 afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); 389 afu->dev.class = ocxl_class; 390 return device_register(&afu->dev); 391 } 392 393 void ocxl_unregister_afu(struct ocxl_afu *afu) 394 { 395 free_afu_minor(afu); 396 } 397 398 static char *ocxl_devnode(struct device *dev, umode_t *mode) 399 { 400 return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev)); 401 } 402 403 int ocxl_file_init(void) 404 { 405 int rc; 406 407 mutex_init(&minors_idr_lock); 408 idr_init(&minors_idr); 409 410 rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl"); 411 if (rc) { 412 pr_err("Unable to allocate ocxl major number: %d\n", rc); 413 return rc; 414 } 415 416 ocxl_class = class_create(THIS_MODULE, "ocxl"); 417 if (IS_ERR(ocxl_class)) { 418 pr_err("Unable to create ocxl class\n"); 419 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 420 return PTR_ERR(ocxl_class); 421 } 422 423 ocxl_class->devnode = ocxl_devnode; 424 return 0; 425 } 426 427 void ocxl_file_exit(void) 428 { 429 class_destroy(ocxl_class); 430 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 431 idr_destroy(&minors_idr); 432 } 433