1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/fs.h> 4 #include <linux/poll.h> 5 #include <linux/sched/signal.h> 6 #include <linux/uaccess.h> 7 #include <uapi/misc/ocxl.h> 8 #include <asm/reg.h> 9 #include <asm/switch_to.h> 10 #include "ocxl_internal.h" 11 12 13 #define OCXL_NUM_MINORS 256 /* Total to reserve */ 14 15 static dev_t ocxl_dev; 16 static struct class *ocxl_class; 17 static struct mutex minors_idr_lock; 18 static struct idr minors_idr; 19 20 static struct ocxl_afu *find_and_get_afu(dev_t devno) 21 { 22 struct ocxl_afu *afu; 23 int afu_minor; 24 25 afu_minor = MINOR(devno); 26 /* 27 * We don't declare an RCU critical section here, as our AFU 28 * is protected by a reference counter on the device. By the time the 29 * minor number of a device is removed from the idr, the ref count of 30 * the device is already at 0, so no user API will access that AFU and 31 * this function can't return it. 32 */ 33 afu = idr_find(&minors_idr, afu_minor); 34 if (afu) 35 ocxl_afu_get(afu); 36 return afu; 37 } 38 39 static int allocate_afu_minor(struct ocxl_afu *afu) 40 { 41 int minor; 42 43 mutex_lock(&minors_idr_lock); 44 minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); 45 mutex_unlock(&minors_idr_lock); 46 return minor; 47 } 48 49 static void free_afu_minor(struct ocxl_afu *afu) 50 { 51 mutex_lock(&minors_idr_lock); 52 idr_remove(&minors_idr, MINOR(afu->dev.devt)); 53 mutex_unlock(&minors_idr_lock); 54 } 55 56 static int afu_open(struct inode *inode, struct file *file) 57 { 58 struct ocxl_afu *afu; 59 struct ocxl_context *ctx; 60 int rc; 61 62 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 63 64 afu = find_and_get_afu(inode->i_rdev); 65 if (!afu) 66 return -ENODEV; 67 68 ctx = ocxl_context_alloc(); 69 if (!ctx) { 70 rc = -ENOMEM; 71 goto put_afu; 72 } 73 74 rc = ocxl_context_init(ctx, afu, inode->i_mapping); 75 if (rc) 76 goto put_afu; 77 file->private_data = ctx; 78 ocxl_afu_put(afu); 79 return 0; 80 81 put_afu: 82 ocxl_afu_put(afu); 83 return rc; 84 } 85 86 static long afu_ioctl_attach(struct ocxl_context *ctx, 87 struct ocxl_ioctl_attach __user *uarg) 88 { 89 struct ocxl_ioctl_attach arg; 90 u64 amr = 0; 91 int rc; 92 93 pr_debug("%s for context %d\n", __func__, ctx->pasid); 94 95 if (copy_from_user(&arg, uarg, sizeof(arg))) 96 return -EFAULT; 97 98 /* Make sure reserved fields are not set for forward compatibility */ 99 if (arg.reserved1 || arg.reserved2 || arg.reserved3) 100 return -EINVAL; 101 102 amr = arg.amr & mfspr(SPRN_UAMOR); 103 rc = ocxl_context_attach(ctx, amr); 104 return rc; 105 } 106 107 static long afu_ioctl_get_metadata(struct ocxl_context *ctx, 108 struct ocxl_ioctl_metadata __user *uarg) 109 { 110 struct ocxl_ioctl_metadata arg; 111 112 memset(&arg, 0, sizeof(arg)); 113 114 arg.version = 0; 115 116 arg.afu_version_major = ctx->afu->config.version_major; 117 arg.afu_version_minor = ctx->afu->config.version_minor; 118 arg.pasid = ctx->pasid; 119 arg.pp_mmio_size = ctx->afu->config.pp_mmio_stride; 120 arg.global_mmio_size = ctx->afu->config.global_mmio_size; 121 122 if (copy_to_user(uarg, &arg, sizeof(arg))) 123 return -EFAULT; 124 125 return 0; 126 } 127 128 #ifdef CONFIG_PPC64 129 static long afu_ioctl_enable_p9_wait(struct ocxl_context *ctx, 130 struct ocxl_ioctl_p9_wait __user *uarg) 131 { 132 struct ocxl_ioctl_p9_wait arg; 133 134 memset(&arg, 0, sizeof(arg)); 135 136 if (cpu_has_feature(CPU_FTR_P9_TIDR)) { 137 enum ocxl_context_status status; 138 139 // Locks both status & tidr 140 mutex_lock(&ctx->status_mutex); 141 if (!ctx->tidr) { 142 if (set_thread_tidr(current)) 143 return -ENOENT; 144 145 ctx->tidr = current->thread.tidr; 146 } 147 148 status = ctx->status; 149 mutex_unlock(&ctx->status_mutex); 150 151 if (status == ATTACHED) { 152 int rc; 153 struct link *link = ctx->afu->fn->link; 154 155 rc = ocxl_link_update_pe(link, ctx->pasid, ctx->tidr); 156 if (rc) 157 return rc; 158 } 159 160 arg.thread_id = ctx->tidr; 161 } else 162 return -ENOENT; 163 164 if (copy_to_user(uarg, &arg, sizeof(arg))) 165 return -EFAULT; 166 167 return 0; 168 } 169 #endif 170 171 #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ 172 x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ 173 x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ 174 x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ 175 x == OCXL_IOCTL_GET_METADATA ? "GET_METADATA" : \ 176 x == OCXL_IOCTL_ENABLE_P9_WAIT ? "ENABLE_P9_WAIT" : \ 177 "UNKNOWN") 178 179 static long afu_ioctl(struct file *file, unsigned int cmd, 180 unsigned long args) 181 { 182 struct ocxl_context *ctx = file->private_data; 183 struct ocxl_ioctl_irq_fd irq_fd; 184 u64 irq_offset; 185 long rc; 186 187 pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, 188 CMD_STR(cmd)); 189 190 if (ctx->status == CLOSED) 191 return -EIO; 192 193 switch (cmd) { 194 case OCXL_IOCTL_ATTACH: 195 rc = afu_ioctl_attach(ctx, 196 (struct ocxl_ioctl_attach __user *) args); 197 break; 198 199 case OCXL_IOCTL_IRQ_ALLOC: 200 rc = ocxl_afu_irq_alloc(ctx, &irq_offset); 201 if (!rc) { 202 rc = copy_to_user((u64 __user *) args, &irq_offset, 203 sizeof(irq_offset)); 204 if (rc) { 205 ocxl_afu_irq_free(ctx, irq_offset); 206 return -EFAULT; 207 } 208 } 209 break; 210 211 case OCXL_IOCTL_IRQ_FREE: 212 rc = copy_from_user(&irq_offset, (u64 __user *) args, 213 sizeof(irq_offset)); 214 if (rc) 215 return -EFAULT; 216 rc = ocxl_afu_irq_free(ctx, irq_offset); 217 break; 218 219 case OCXL_IOCTL_IRQ_SET_FD: 220 rc = copy_from_user(&irq_fd, (u64 __user *) args, 221 sizeof(irq_fd)); 222 if (rc) 223 return -EFAULT; 224 if (irq_fd.reserved) 225 return -EINVAL; 226 rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, 227 irq_fd.eventfd); 228 break; 229 230 case OCXL_IOCTL_GET_METADATA: 231 rc = afu_ioctl_get_metadata(ctx, 232 (struct ocxl_ioctl_metadata __user *) args); 233 break; 234 235 #ifdef CONFIG_PPC64 236 case OCXL_IOCTL_ENABLE_P9_WAIT: 237 rc = afu_ioctl_enable_p9_wait(ctx, 238 (struct ocxl_ioctl_p9_wait __user *) args); 239 break; 240 #endif 241 242 default: 243 rc = -EINVAL; 244 } 245 return rc; 246 } 247 248 static long afu_compat_ioctl(struct file *file, unsigned int cmd, 249 unsigned long args) 250 { 251 return afu_ioctl(file, cmd, args); 252 } 253 254 static int afu_mmap(struct file *file, struct vm_area_struct *vma) 255 { 256 struct ocxl_context *ctx = file->private_data; 257 258 pr_debug("%s for context %d\n", __func__, ctx->pasid); 259 return ocxl_context_mmap(ctx, vma); 260 } 261 262 static bool has_xsl_error(struct ocxl_context *ctx) 263 { 264 bool ret; 265 266 mutex_lock(&ctx->xsl_error_lock); 267 ret = !!ctx->xsl_error.addr; 268 mutex_unlock(&ctx->xsl_error_lock); 269 270 return ret; 271 } 272 273 /* 274 * Are there any events pending on the AFU 275 * ctx: The AFU context 276 * Returns: true if there are events pending 277 */ 278 static bool afu_events_pending(struct ocxl_context *ctx) 279 { 280 if (has_xsl_error(ctx)) 281 return true; 282 return false; 283 } 284 285 static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait) 286 { 287 struct ocxl_context *ctx = file->private_data; 288 unsigned int mask = 0; 289 bool closed; 290 291 pr_debug("%s for context %d\n", __func__, ctx->pasid); 292 293 poll_wait(file, &ctx->events_wq, wait); 294 295 mutex_lock(&ctx->status_mutex); 296 closed = (ctx->status == CLOSED); 297 mutex_unlock(&ctx->status_mutex); 298 299 if (afu_events_pending(ctx)) 300 mask = EPOLLIN | EPOLLRDNORM; 301 else if (closed) 302 mask = EPOLLERR; 303 304 return mask; 305 } 306 307 /* 308 * Populate the supplied buffer with a single XSL error 309 * ctx: The AFU context to report the error from 310 * header: the event header to populate 311 * buf: The buffer to write the body into (should be at least 312 * AFU_EVENT_BODY_XSL_ERROR_SIZE) 313 * Return: the amount of buffer that was populated 314 */ 315 static ssize_t append_xsl_error(struct ocxl_context *ctx, 316 struct ocxl_kernel_event_header *header, 317 char __user *buf) 318 { 319 struct ocxl_kernel_event_xsl_fault_error body; 320 321 memset(&body, 0, sizeof(body)); 322 323 mutex_lock(&ctx->xsl_error_lock); 324 if (!ctx->xsl_error.addr) { 325 mutex_unlock(&ctx->xsl_error_lock); 326 return 0; 327 } 328 329 body.addr = ctx->xsl_error.addr; 330 body.dsisr = ctx->xsl_error.dsisr; 331 body.count = ctx->xsl_error.count; 332 333 ctx->xsl_error.addr = 0; 334 ctx->xsl_error.dsisr = 0; 335 ctx->xsl_error.count = 0; 336 337 mutex_unlock(&ctx->xsl_error_lock); 338 339 header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR; 340 341 if (copy_to_user(buf, &body, sizeof(body))) 342 return -EFAULT; 343 344 return sizeof(body); 345 } 346 347 #define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error) 348 349 /* 350 * Reports events on the AFU 351 * Format: 352 * Header (struct ocxl_kernel_event_header) 353 * Body (struct ocxl_kernel_event_*) 354 * Header... 355 */ 356 static ssize_t afu_read(struct file *file, char __user *buf, size_t count, 357 loff_t *off) 358 { 359 struct ocxl_context *ctx = file->private_data; 360 struct ocxl_kernel_event_header header; 361 ssize_t rc; 362 ssize_t used = 0; 363 DEFINE_WAIT(event_wait); 364 365 memset(&header, 0, sizeof(header)); 366 367 /* Require offset to be 0 */ 368 if (*off != 0) 369 return -EINVAL; 370 371 if (count < (sizeof(struct ocxl_kernel_event_header) + 372 AFU_EVENT_BODY_MAX_SIZE)) 373 return -EINVAL; 374 375 for (;;) { 376 prepare_to_wait(&ctx->events_wq, &event_wait, 377 TASK_INTERRUPTIBLE); 378 379 if (afu_events_pending(ctx)) 380 break; 381 382 if (ctx->status == CLOSED) 383 break; 384 385 if (file->f_flags & O_NONBLOCK) { 386 finish_wait(&ctx->events_wq, &event_wait); 387 return -EAGAIN; 388 } 389 390 if (signal_pending(current)) { 391 finish_wait(&ctx->events_wq, &event_wait); 392 return -ERESTARTSYS; 393 } 394 395 schedule(); 396 } 397 398 finish_wait(&ctx->events_wq, &event_wait); 399 400 if (has_xsl_error(ctx)) { 401 used = append_xsl_error(ctx, &header, buf + sizeof(header)); 402 if (used < 0) 403 return used; 404 } 405 406 if (!afu_events_pending(ctx)) 407 header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST; 408 409 if (copy_to_user(buf, &header, sizeof(header))) 410 return -EFAULT; 411 412 used += sizeof(header); 413 414 rc = used; 415 return rc; 416 } 417 418 static int afu_release(struct inode *inode, struct file *file) 419 { 420 struct ocxl_context *ctx = file->private_data; 421 int rc; 422 423 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 424 rc = ocxl_context_detach(ctx); 425 mutex_lock(&ctx->mapping_lock); 426 ctx->mapping = NULL; 427 mutex_unlock(&ctx->mapping_lock); 428 wake_up_all(&ctx->events_wq); 429 if (rc != -EBUSY) 430 ocxl_context_free(ctx); 431 return 0; 432 } 433 434 static const struct file_operations ocxl_afu_fops = { 435 .owner = THIS_MODULE, 436 .open = afu_open, 437 .unlocked_ioctl = afu_ioctl, 438 .compat_ioctl = afu_compat_ioctl, 439 .mmap = afu_mmap, 440 .poll = afu_poll, 441 .read = afu_read, 442 .release = afu_release, 443 }; 444 445 int ocxl_create_cdev(struct ocxl_afu *afu) 446 { 447 int rc; 448 449 cdev_init(&afu->cdev, &ocxl_afu_fops); 450 rc = cdev_add(&afu->cdev, afu->dev.devt, 1); 451 if (rc) { 452 dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); 453 return rc; 454 } 455 return 0; 456 } 457 458 void ocxl_destroy_cdev(struct ocxl_afu *afu) 459 { 460 cdev_del(&afu->cdev); 461 } 462 463 int ocxl_register_afu(struct ocxl_afu *afu) 464 { 465 int minor; 466 467 minor = allocate_afu_minor(afu); 468 if (minor < 0) 469 return minor; 470 afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); 471 afu->dev.class = ocxl_class; 472 return device_register(&afu->dev); 473 } 474 475 void ocxl_unregister_afu(struct ocxl_afu *afu) 476 { 477 free_afu_minor(afu); 478 } 479 480 static char *ocxl_devnode(struct device *dev, umode_t *mode) 481 { 482 return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev)); 483 } 484 485 int ocxl_file_init(void) 486 { 487 int rc; 488 489 mutex_init(&minors_idr_lock); 490 idr_init(&minors_idr); 491 492 rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl"); 493 if (rc) { 494 pr_err("Unable to allocate ocxl major number: %d\n", rc); 495 return rc; 496 } 497 498 ocxl_class = class_create(THIS_MODULE, "ocxl"); 499 if (IS_ERR(ocxl_class)) { 500 pr_err("Unable to create ocxl class\n"); 501 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 502 return PTR_ERR(ocxl_class); 503 } 504 505 ocxl_class->devnode = ocxl_devnode; 506 return 0; 507 } 508 509 void ocxl_file_exit(void) 510 { 511 class_destroy(ocxl_class); 512 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 513 idr_destroy(&minors_idr); 514 } 515