1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/fs.h> 4 #include <linux/poll.h> 5 #include <linux/sched/signal.h> 6 #include <linux/uaccess.h> 7 #include <uapi/misc/ocxl.h> 8 #include <asm/reg.h> 9 #include <asm/switch_to.h> 10 #include "ocxl_internal.h" 11 12 13 #define OCXL_NUM_MINORS 256 /* Total to reserve */ 14 15 static dev_t ocxl_dev; 16 static struct class *ocxl_class; 17 static struct mutex minors_idr_lock; 18 static struct idr minors_idr; 19 20 static struct ocxl_afu *find_and_get_afu(dev_t devno) 21 { 22 struct ocxl_afu *afu; 23 int afu_minor; 24 25 afu_minor = MINOR(devno); 26 /* 27 * We don't declare an RCU critical section here, as our AFU 28 * is protected by a reference counter on the device. By the time the 29 * minor number of a device is removed from the idr, the ref count of 30 * the device is already at 0, so no user API will access that AFU and 31 * this function can't return it. 32 */ 33 afu = idr_find(&minors_idr, afu_minor); 34 if (afu) 35 ocxl_afu_get(afu); 36 return afu; 37 } 38 39 static int allocate_afu_minor(struct ocxl_afu *afu) 40 { 41 int minor; 42 43 mutex_lock(&minors_idr_lock); 44 minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); 45 mutex_unlock(&minors_idr_lock); 46 return minor; 47 } 48 49 static void free_afu_minor(struct ocxl_afu *afu) 50 { 51 mutex_lock(&minors_idr_lock); 52 idr_remove(&minors_idr, MINOR(afu->dev.devt)); 53 mutex_unlock(&minors_idr_lock); 54 } 55 56 static int afu_open(struct inode *inode, struct file *file) 57 { 58 struct ocxl_afu *afu; 59 struct ocxl_context *ctx; 60 int rc; 61 62 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 63 64 afu = find_and_get_afu(inode->i_rdev); 65 if (!afu) 66 return -ENODEV; 67 68 ctx = ocxl_context_alloc(); 69 if (!ctx) { 70 rc = -ENOMEM; 71 goto put_afu; 72 } 73 74 rc = ocxl_context_init(ctx, afu, inode->i_mapping); 75 if (rc) 76 goto put_afu; 77 file->private_data = ctx; 78 ocxl_afu_put(afu); 79 return 0; 80 81 put_afu: 82 ocxl_afu_put(afu); 83 return rc; 84 } 85 86 static long afu_ioctl_attach(struct ocxl_context *ctx, 87 struct ocxl_ioctl_attach __user *uarg) 88 { 89 struct ocxl_ioctl_attach arg; 90 u64 amr = 0; 91 int rc; 92 93 pr_debug("%s for context %d\n", __func__, ctx->pasid); 94 95 if (copy_from_user(&arg, uarg, sizeof(arg))) 96 return -EFAULT; 97 98 /* Make sure reserved fields are not set for forward compatibility */ 99 if (arg.reserved1 || arg.reserved2 || arg.reserved3) 100 return -EINVAL; 101 102 amr = arg.amr & mfspr(SPRN_UAMOR); 103 rc = ocxl_context_attach(ctx, amr); 104 return rc; 105 } 106 107 static long afu_ioctl_get_metadata(struct ocxl_context *ctx, 108 struct ocxl_ioctl_metadata __user *uarg) 109 { 110 struct ocxl_ioctl_metadata arg; 111 112 memset(&arg, 0, sizeof(arg)); 113 114 arg.version = 0; 115 116 arg.afu_version_major = ctx->afu->config.version_major; 117 arg.afu_version_minor = ctx->afu->config.version_minor; 118 arg.pasid = ctx->pasid; 119 arg.pp_mmio_size = ctx->afu->config.pp_mmio_stride; 120 arg.global_mmio_size = ctx->afu->config.global_mmio_size; 121 122 if (copy_to_user(uarg, &arg, sizeof(arg))) 123 return -EFAULT; 124 125 return 0; 126 } 127 128 #ifdef CONFIG_PPC64 129 static long afu_ioctl_enable_p9_wait(struct ocxl_context *ctx, 130 struct ocxl_ioctl_p9_wait __user *uarg) 131 { 132 struct ocxl_ioctl_p9_wait arg; 133 134 memset(&arg, 0, sizeof(arg)); 135 136 if (cpu_has_feature(CPU_FTR_P9_TIDR)) { 137 enum ocxl_context_status status; 138 139 // Locks both status & tidr 140 mutex_lock(&ctx->status_mutex); 141 if (!ctx->tidr) { 142 if (set_thread_tidr(current)) { 143 mutex_unlock(&ctx->status_mutex); 144 return -ENOENT; 145 } 146 147 ctx->tidr = current->thread.tidr; 148 } 149 150 status = ctx->status; 151 mutex_unlock(&ctx->status_mutex); 152 153 if (status == ATTACHED) { 154 int rc; 155 struct link *link = ctx->afu->fn->link; 156 157 rc = ocxl_link_update_pe(link, ctx->pasid, ctx->tidr); 158 if (rc) 159 return rc; 160 } 161 162 arg.thread_id = ctx->tidr; 163 } else 164 return -ENOENT; 165 166 if (copy_to_user(uarg, &arg, sizeof(arg))) 167 return -EFAULT; 168 169 return 0; 170 } 171 #endif 172 173 174 static long afu_ioctl_get_features(struct ocxl_context *ctx, 175 struct ocxl_ioctl_features __user *uarg) 176 { 177 struct ocxl_ioctl_features arg; 178 179 memset(&arg, 0, sizeof(arg)); 180 181 #ifdef CONFIG_PPC64 182 if (cpu_has_feature(CPU_FTR_P9_TIDR)) 183 arg.flags[0] |= OCXL_IOCTL_FEATURES_FLAGS0_P9_WAIT; 184 #endif 185 186 if (copy_to_user(uarg, &arg, sizeof(arg))) 187 return -EFAULT; 188 189 return 0; 190 } 191 192 #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ 193 x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ 194 x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ 195 x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ 196 x == OCXL_IOCTL_GET_METADATA ? "GET_METADATA" : \ 197 x == OCXL_IOCTL_ENABLE_P9_WAIT ? "ENABLE_P9_WAIT" : \ 198 x == OCXL_IOCTL_GET_FEATURES ? "GET_FEATURES" : \ 199 "UNKNOWN") 200 201 static long afu_ioctl(struct file *file, unsigned int cmd, 202 unsigned long args) 203 { 204 struct ocxl_context *ctx = file->private_data; 205 struct ocxl_ioctl_irq_fd irq_fd; 206 u64 irq_offset; 207 long rc; 208 209 pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, 210 CMD_STR(cmd)); 211 212 if (ctx->status == CLOSED) 213 return -EIO; 214 215 switch (cmd) { 216 case OCXL_IOCTL_ATTACH: 217 rc = afu_ioctl_attach(ctx, 218 (struct ocxl_ioctl_attach __user *) args); 219 break; 220 221 case OCXL_IOCTL_IRQ_ALLOC: 222 rc = ocxl_afu_irq_alloc(ctx, &irq_offset); 223 if (!rc) { 224 rc = copy_to_user((u64 __user *) args, &irq_offset, 225 sizeof(irq_offset)); 226 if (rc) { 227 ocxl_afu_irq_free(ctx, irq_offset); 228 return -EFAULT; 229 } 230 } 231 break; 232 233 case OCXL_IOCTL_IRQ_FREE: 234 rc = copy_from_user(&irq_offset, (u64 __user *) args, 235 sizeof(irq_offset)); 236 if (rc) 237 return -EFAULT; 238 rc = ocxl_afu_irq_free(ctx, irq_offset); 239 break; 240 241 case OCXL_IOCTL_IRQ_SET_FD: 242 rc = copy_from_user(&irq_fd, (u64 __user *) args, 243 sizeof(irq_fd)); 244 if (rc) 245 return -EFAULT; 246 if (irq_fd.reserved) 247 return -EINVAL; 248 rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, 249 irq_fd.eventfd); 250 break; 251 252 case OCXL_IOCTL_GET_METADATA: 253 rc = afu_ioctl_get_metadata(ctx, 254 (struct ocxl_ioctl_metadata __user *) args); 255 break; 256 257 #ifdef CONFIG_PPC64 258 case OCXL_IOCTL_ENABLE_P9_WAIT: 259 rc = afu_ioctl_enable_p9_wait(ctx, 260 (struct ocxl_ioctl_p9_wait __user *) args); 261 break; 262 #endif 263 264 case OCXL_IOCTL_GET_FEATURES: 265 rc = afu_ioctl_get_features(ctx, 266 (struct ocxl_ioctl_features __user *) args); 267 break; 268 269 default: 270 rc = -EINVAL; 271 } 272 return rc; 273 } 274 275 static long afu_compat_ioctl(struct file *file, unsigned int cmd, 276 unsigned long args) 277 { 278 return afu_ioctl(file, cmd, args); 279 } 280 281 static int afu_mmap(struct file *file, struct vm_area_struct *vma) 282 { 283 struct ocxl_context *ctx = file->private_data; 284 285 pr_debug("%s for context %d\n", __func__, ctx->pasid); 286 return ocxl_context_mmap(ctx, vma); 287 } 288 289 static bool has_xsl_error(struct ocxl_context *ctx) 290 { 291 bool ret; 292 293 mutex_lock(&ctx->xsl_error_lock); 294 ret = !!ctx->xsl_error.addr; 295 mutex_unlock(&ctx->xsl_error_lock); 296 297 return ret; 298 } 299 300 /* 301 * Are there any events pending on the AFU 302 * ctx: The AFU context 303 * Returns: true if there are events pending 304 */ 305 static bool afu_events_pending(struct ocxl_context *ctx) 306 { 307 if (has_xsl_error(ctx)) 308 return true; 309 return false; 310 } 311 312 static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait) 313 { 314 struct ocxl_context *ctx = file->private_data; 315 unsigned int mask = 0; 316 bool closed; 317 318 pr_debug("%s for context %d\n", __func__, ctx->pasid); 319 320 poll_wait(file, &ctx->events_wq, wait); 321 322 mutex_lock(&ctx->status_mutex); 323 closed = (ctx->status == CLOSED); 324 mutex_unlock(&ctx->status_mutex); 325 326 if (afu_events_pending(ctx)) 327 mask = EPOLLIN | EPOLLRDNORM; 328 else if (closed) 329 mask = EPOLLERR; 330 331 return mask; 332 } 333 334 /* 335 * Populate the supplied buffer with a single XSL error 336 * ctx: The AFU context to report the error from 337 * header: the event header to populate 338 * buf: The buffer to write the body into (should be at least 339 * AFU_EVENT_BODY_XSL_ERROR_SIZE) 340 * Return: the amount of buffer that was populated 341 */ 342 static ssize_t append_xsl_error(struct ocxl_context *ctx, 343 struct ocxl_kernel_event_header *header, 344 char __user *buf) 345 { 346 struct ocxl_kernel_event_xsl_fault_error body; 347 348 memset(&body, 0, sizeof(body)); 349 350 mutex_lock(&ctx->xsl_error_lock); 351 if (!ctx->xsl_error.addr) { 352 mutex_unlock(&ctx->xsl_error_lock); 353 return 0; 354 } 355 356 body.addr = ctx->xsl_error.addr; 357 body.dsisr = ctx->xsl_error.dsisr; 358 body.count = ctx->xsl_error.count; 359 360 ctx->xsl_error.addr = 0; 361 ctx->xsl_error.dsisr = 0; 362 ctx->xsl_error.count = 0; 363 364 mutex_unlock(&ctx->xsl_error_lock); 365 366 header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR; 367 368 if (copy_to_user(buf, &body, sizeof(body))) 369 return -EFAULT; 370 371 return sizeof(body); 372 } 373 374 #define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error) 375 376 /* 377 * Reports events on the AFU 378 * Format: 379 * Header (struct ocxl_kernel_event_header) 380 * Body (struct ocxl_kernel_event_*) 381 * Header... 382 */ 383 static ssize_t afu_read(struct file *file, char __user *buf, size_t count, 384 loff_t *off) 385 { 386 struct ocxl_context *ctx = file->private_data; 387 struct ocxl_kernel_event_header header; 388 ssize_t rc; 389 ssize_t used = 0; 390 DEFINE_WAIT(event_wait); 391 392 memset(&header, 0, sizeof(header)); 393 394 /* Require offset to be 0 */ 395 if (*off != 0) 396 return -EINVAL; 397 398 if (count < (sizeof(struct ocxl_kernel_event_header) + 399 AFU_EVENT_BODY_MAX_SIZE)) 400 return -EINVAL; 401 402 for (;;) { 403 prepare_to_wait(&ctx->events_wq, &event_wait, 404 TASK_INTERRUPTIBLE); 405 406 if (afu_events_pending(ctx)) 407 break; 408 409 if (ctx->status == CLOSED) 410 break; 411 412 if (file->f_flags & O_NONBLOCK) { 413 finish_wait(&ctx->events_wq, &event_wait); 414 return -EAGAIN; 415 } 416 417 if (signal_pending(current)) { 418 finish_wait(&ctx->events_wq, &event_wait); 419 return -ERESTARTSYS; 420 } 421 422 schedule(); 423 } 424 425 finish_wait(&ctx->events_wq, &event_wait); 426 427 if (has_xsl_error(ctx)) { 428 used = append_xsl_error(ctx, &header, buf + sizeof(header)); 429 if (used < 0) 430 return used; 431 } 432 433 if (!afu_events_pending(ctx)) 434 header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST; 435 436 if (copy_to_user(buf, &header, sizeof(header))) 437 return -EFAULT; 438 439 used += sizeof(header); 440 441 rc = used; 442 return rc; 443 } 444 445 static int afu_release(struct inode *inode, struct file *file) 446 { 447 struct ocxl_context *ctx = file->private_data; 448 int rc; 449 450 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 451 rc = ocxl_context_detach(ctx); 452 mutex_lock(&ctx->mapping_lock); 453 ctx->mapping = NULL; 454 mutex_unlock(&ctx->mapping_lock); 455 wake_up_all(&ctx->events_wq); 456 if (rc != -EBUSY) 457 ocxl_context_free(ctx); 458 return 0; 459 } 460 461 static const struct file_operations ocxl_afu_fops = { 462 .owner = THIS_MODULE, 463 .open = afu_open, 464 .unlocked_ioctl = afu_ioctl, 465 .compat_ioctl = afu_compat_ioctl, 466 .mmap = afu_mmap, 467 .poll = afu_poll, 468 .read = afu_read, 469 .release = afu_release, 470 }; 471 472 int ocxl_create_cdev(struct ocxl_afu *afu) 473 { 474 int rc; 475 476 cdev_init(&afu->cdev, &ocxl_afu_fops); 477 rc = cdev_add(&afu->cdev, afu->dev.devt, 1); 478 if (rc) { 479 dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); 480 return rc; 481 } 482 return 0; 483 } 484 485 void ocxl_destroy_cdev(struct ocxl_afu *afu) 486 { 487 cdev_del(&afu->cdev); 488 } 489 490 int ocxl_register_afu(struct ocxl_afu *afu) 491 { 492 int minor; 493 494 minor = allocate_afu_minor(afu); 495 if (minor < 0) 496 return minor; 497 afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); 498 afu->dev.class = ocxl_class; 499 return device_register(&afu->dev); 500 } 501 502 void ocxl_unregister_afu(struct ocxl_afu *afu) 503 { 504 free_afu_minor(afu); 505 } 506 507 static char *ocxl_devnode(struct device *dev, umode_t *mode) 508 { 509 return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev)); 510 } 511 512 int ocxl_file_init(void) 513 { 514 int rc; 515 516 mutex_init(&minors_idr_lock); 517 idr_init(&minors_idr); 518 519 rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl"); 520 if (rc) { 521 pr_err("Unable to allocate ocxl major number: %d\n", rc); 522 return rc; 523 } 524 525 ocxl_class = class_create(THIS_MODULE, "ocxl"); 526 if (IS_ERR(ocxl_class)) { 527 pr_err("Unable to create ocxl class\n"); 528 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 529 return PTR_ERR(ocxl_class); 530 } 531 532 ocxl_class->devnode = ocxl_devnode; 533 return 0; 534 } 535 536 void ocxl_file_exit(void) 537 { 538 class_destroy(ocxl_class); 539 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 540 idr_destroy(&minors_idr); 541 } 542