1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/fs.h> 4 #include <linux/poll.h> 5 #include <linux/sched/signal.h> 6 #include <linux/uaccess.h> 7 #include <uapi/misc/ocxl.h> 8 #include "ocxl_internal.h" 9 10 11 #define OCXL_NUM_MINORS 256 /* Total to reserve */ 12 13 static dev_t ocxl_dev; 14 static struct class *ocxl_class; 15 static struct mutex minors_idr_lock; 16 static struct idr minors_idr; 17 18 static struct ocxl_afu *find_and_get_afu(dev_t devno) 19 { 20 struct ocxl_afu *afu; 21 int afu_minor; 22 23 afu_minor = MINOR(devno); 24 /* 25 * We don't declare an RCU critical section here, as our AFU 26 * is protected by a reference counter on the device. By the time the 27 * minor number of a device is removed from the idr, the ref count of 28 * the device is already at 0, so no user API will access that AFU and 29 * this function can't return it. 30 */ 31 afu = idr_find(&minors_idr, afu_minor); 32 if (afu) 33 ocxl_afu_get(afu); 34 return afu; 35 } 36 37 static int allocate_afu_minor(struct ocxl_afu *afu) 38 { 39 int minor; 40 41 mutex_lock(&minors_idr_lock); 42 minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); 43 mutex_unlock(&minors_idr_lock); 44 return minor; 45 } 46 47 static void free_afu_minor(struct ocxl_afu *afu) 48 { 49 mutex_lock(&minors_idr_lock); 50 idr_remove(&minors_idr, MINOR(afu->dev.devt)); 51 mutex_unlock(&minors_idr_lock); 52 } 53 54 static int afu_open(struct inode *inode, struct file *file) 55 { 56 struct ocxl_afu *afu; 57 struct ocxl_context *ctx; 58 int rc; 59 60 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 61 62 afu = find_and_get_afu(inode->i_rdev); 63 if (!afu) 64 return -ENODEV; 65 66 ctx = ocxl_context_alloc(); 67 if (!ctx) { 68 rc = -ENOMEM; 69 goto put_afu; 70 } 71 72 rc = ocxl_context_init(ctx, afu, inode->i_mapping); 73 if (rc) 74 goto put_afu; 75 file->private_data = ctx; 76 ocxl_afu_put(afu); 77 return 0; 78 79 put_afu: 80 ocxl_afu_put(afu); 81 return rc; 82 } 83 84 static long afu_ioctl_attach(struct ocxl_context *ctx, 85 struct ocxl_ioctl_attach __user *uarg) 86 { 87 struct ocxl_ioctl_attach arg; 88 u64 amr = 0; 89 int rc; 90 91 pr_debug("%s for context %d\n", __func__, ctx->pasid); 92 93 if (copy_from_user(&arg, uarg, sizeof(arg))) 94 return -EFAULT; 95 96 /* Make sure reserved fields are not set for forward compatibility */ 97 if (arg.reserved1 || arg.reserved2 || arg.reserved3) 98 return -EINVAL; 99 100 amr = arg.amr & mfspr(SPRN_UAMOR); 101 rc = ocxl_context_attach(ctx, amr); 102 return rc; 103 } 104 105 static long afu_ioctl_get_metadata(struct ocxl_context *ctx, 106 struct ocxl_ioctl_metadata __user *uarg) 107 { 108 struct ocxl_ioctl_metadata arg; 109 110 memset(&arg, 0, sizeof(arg)); 111 112 arg.version = 0; 113 114 arg.afu_version_major = ctx->afu->config.version_major; 115 arg.afu_version_minor = ctx->afu->config.version_minor; 116 arg.pasid = ctx->pasid; 117 arg.pp_mmio_size = ctx->afu->config.pp_mmio_stride; 118 arg.global_mmio_size = ctx->afu->config.global_mmio_size; 119 120 if (copy_to_user(uarg, &arg, sizeof(arg))) 121 return -EFAULT; 122 123 return 0; 124 } 125 126 #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ 127 x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ 128 x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ 129 x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ 130 x == OCXL_IOCTL_GET_METADATA ? "GET_METADATA" : \ 131 "UNKNOWN") 132 133 static long afu_ioctl(struct file *file, unsigned int cmd, 134 unsigned long args) 135 { 136 struct ocxl_context *ctx = file->private_data; 137 struct ocxl_ioctl_irq_fd irq_fd; 138 u64 irq_offset; 139 long rc; 140 141 pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, 142 CMD_STR(cmd)); 143 144 if (ctx->status == CLOSED) 145 return -EIO; 146 147 switch (cmd) { 148 case OCXL_IOCTL_ATTACH: 149 rc = afu_ioctl_attach(ctx, 150 (struct ocxl_ioctl_attach __user *) args); 151 break; 152 153 case OCXL_IOCTL_IRQ_ALLOC: 154 rc = ocxl_afu_irq_alloc(ctx, &irq_offset); 155 if (!rc) { 156 rc = copy_to_user((u64 __user *) args, &irq_offset, 157 sizeof(irq_offset)); 158 if (rc) { 159 ocxl_afu_irq_free(ctx, irq_offset); 160 return -EFAULT; 161 } 162 } 163 break; 164 165 case OCXL_IOCTL_IRQ_FREE: 166 rc = copy_from_user(&irq_offset, (u64 __user *) args, 167 sizeof(irq_offset)); 168 if (rc) 169 return -EFAULT; 170 rc = ocxl_afu_irq_free(ctx, irq_offset); 171 break; 172 173 case OCXL_IOCTL_IRQ_SET_FD: 174 rc = copy_from_user(&irq_fd, (u64 __user *) args, 175 sizeof(irq_fd)); 176 if (rc) 177 return -EFAULT; 178 if (irq_fd.reserved) 179 return -EINVAL; 180 rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, 181 irq_fd.eventfd); 182 break; 183 184 case OCXL_IOCTL_GET_METADATA: 185 rc = afu_ioctl_get_metadata(ctx, 186 (struct ocxl_ioctl_metadata __user *) args); 187 break; 188 189 default: 190 rc = -EINVAL; 191 } 192 return rc; 193 } 194 195 static long afu_compat_ioctl(struct file *file, unsigned int cmd, 196 unsigned long args) 197 { 198 return afu_ioctl(file, cmd, args); 199 } 200 201 static int afu_mmap(struct file *file, struct vm_area_struct *vma) 202 { 203 struct ocxl_context *ctx = file->private_data; 204 205 pr_debug("%s for context %d\n", __func__, ctx->pasid); 206 return ocxl_context_mmap(ctx, vma); 207 } 208 209 static bool has_xsl_error(struct ocxl_context *ctx) 210 { 211 bool ret; 212 213 mutex_lock(&ctx->xsl_error_lock); 214 ret = !!ctx->xsl_error.addr; 215 mutex_unlock(&ctx->xsl_error_lock); 216 217 return ret; 218 } 219 220 /* 221 * Are there any events pending on the AFU 222 * ctx: The AFU context 223 * Returns: true if there are events pending 224 */ 225 static bool afu_events_pending(struct ocxl_context *ctx) 226 { 227 if (has_xsl_error(ctx)) 228 return true; 229 return false; 230 } 231 232 static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait) 233 { 234 struct ocxl_context *ctx = file->private_data; 235 unsigned int mask = 0; 236 bool closed; 237 238 pr_debug("%s for context %d\n", __func__, ctx->pasid); 239 240 poll_wait(file, &ctx->events_wq, wait); 241 242 mutex_lock(&ctx->status_mutex); 243 closed = (ctx->status == CLOSED); 244 mutex_unlock(&ctx->status_mutex); 245 246 if (afu_events_pending(ctx)) 247 mask = EPOLLIN | EPOLLRDNORM; 248 else if (closed) 249 mask = EPOLLERR; 250 251 return mask; 252 } 253 254 /* 255 * Populate the supplied buffer with a single XSL error 256 * ctx: The AFU context to report the error from 257 * header: the event header to populate 258 * buf: The buffer to write the body into (should be at least 259 * AFU_EVENT_BODY_XSL_ERROR_SIZE) 260 * Return: the amount of buffer that was populated 261 */ 262 static ssize_t append_xsl_error(struct ocxl_context *ctx, 263 struct ocxl_kernel_event_header *header, 264 char __user *buf) 265 { 266 struct ocxl_kernel_event_xsl_fault_error body; 267 268 memset(&body, 0, sizeof(body)); 269 270 mutex_lock(&ctx->xsl_error_lock); 271 if (!ctx->xsl_error.addr) { 272 mutex_unlock(&ctx->xsl_error_lock); 273 return 0; 274 } 275 276 body.addr = ctx->xsl_error.addr; 277 body.dsisr = ctx->xsl_error.dsisr; 278 body.count = ctx->xsl_error.count; 279 280 ctx->xsl_error.addr = 0; 281 ctx->xsl_error.dsisr = 0; 282 ctx->xsl_error.count = 0; 283 284 mutex_unlock(&ctx->xsl_error_lock); 285 286 header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR; 287 288 if (copy_to_user(buf, &body, sizeof(body))) 289 return -EFAULT; 290 291 return sizeof(body); 292 } 293 294 #define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error) 295 296 /* 297 * Reports events on the AFU 298 * Format: 299 * Header (struct ocxl_kernel_event_header) 300 * Body (struct ocxl_kernel_event_*) 301 * Header... 302 */ 303 static ssize_t afu_read(struct file *file, char __user *buf, size_t count, 304 loff_t *off) 305 { 306 struct ocxl_context *ctx = file->private_data; 307 struct ocxl_kernel_event_header header; 308 ssize_t rc; 309 ssize_t used = 0; 310 DEFINE_WAIT(event_wait); 311 312 memset(&header, 0, sizeof(header)); 313 314 /* Require offset to be 0 */ 315 if (*off != 0) 316 return -EINVAL; 317 318 if (count < (sizeof(struct ocxl_kernel_event_header) + 319 AFU_EVENT_BODY_MAX_SIZE)) 320 return -EINVAL; 321 322 for (;;) { 323 prepare_to_wait(&ctx->events_wq, &event_wait, 324 TASK_INTERRUPTIBLE); 325 326 if (afu_events_pending(ctx)) 327 break; 328 329 if (ctx->status == CLOSED) 330 break; 331 332 if (file->f_flags & O_NONBLOCK) { 333 finish_wait(&ctx->events_wq, &event_wait); 334 return -EAGAIN; 335 } 336 337 if (signal_pending(current)) { 338 finish_wait(&ctx->events_wq, &event_wait); 339 return -ERESTARTSYS; 340 } 341 342 schedule(); 343 } 344 345 finish_wait(&ctx->events_wq, &event_wait); 346 347 if (has_xsl_error(ctx)) { 348 used = append_xsl_error(ctx, &header, buf + sizeof(header)); 349 if (used < 0) 350 return used; 351 } 352 353 if (!afu_events_pending(ctx)) 354 header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST; 355 356 if (copy_to_user(buf, &header, sizeof(header))) 357 return -EFAULT; 358 359 used += sizeof(header); 360 361 rc = used; 362 return rc; 363 } 364 365 static int afu_release(struct inode *inode, struct file *file) 366 { 367 struct ocxl_context *ctx = file->private_data; 368 int rc; 369 370 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 371 rc = ocxl_context_detach(ctx); 372 mutex_lock(&ctx->mapping_lock); 373 ctx->mapping = NULL; 374 mutex_unlock(&ctx->mapping_lock); 375 wake_up_all(&ctx->events_wq); 376 if (rc != -EBUSY) 377 ocxl_context_free(ctx); 378 return 0; 379 } 380 381 static const struct file_operations ocxl_afu_fops = { 382 .owner = THIS_MODULE, 383 .open = afu_open, 384 .unlocked_ioctl = afu_ioctl, 385 .compat_ioctl = afu_compat_ioctl, 386 .mmap = afu_mmap, 387 .poll = afu_poll, 388 .read = afu_read, 389 .release = afu_release, 390 }; 391 392 int ocxl_create_cdev(struct ocxl_afu *afu) 393 { 394 int rc; 395 396 cdev_init(&afu->cdev, &ocxl_afu_fops); 397 rc = cdev_add(&afu->cdev, afu->dev.devt, 1); 398 if (rc) { 399 dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); 400 return rc; 401 } 402 return 0; 403 } 404 405 void ocxl_destroy_cdev(struct ocxl_afu *afu) 406 { 407 cdev_del(&afu->cdev); 408 } 409 410 int ocxl_register_afu(struct ocxl_afu *afu) 411 { 412 int minor; 413 414 minor = allocate_afu_minor(afu); 415 if (minor < 0) 416 return minor; 417 afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); 418 afu->dev.class = ocxl_class; 419 return device_register(&afu->dev); 420 } 421 422 void ocxl_unregister_afu(struct ocxl_afu *afu) 423 { 424 free_afu_minor(afu); 425 } 426 427 static char *ocxl_devnode(struct device *dev, umode_t *mode) 428 { 429 return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev)); 430 } 431 432 int ocxl_file_init(void) 433 { 434 int rc; 435 436 mutex_init(&minors_idr_lock); 437 idr_init(&minors_idr); 438 439 rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl"); 440 if (rc) { 441 pr_err("Unable to allocate ocxl major number: %d\n", rc); 442 return rc; 443 } 444 445 ocxl_class = class_create(THIS_MODULE, "ocxl"); 446 if (IS_ERR(ocxl_class)) { 447 pr_err("Unable to create ocxl class\n"); 448 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 449 return PTR_ERR(ocxl_class); 450 } 451 452 ocxl_class->devnode = ocxl_devnode; 453 return 0; 454 } 455 456 void ocxl_file_exit(void) 457 { 458 class_destroy(ocxl_class); 459 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 460 idr_destroy(&minors_idr); 461 } 462