1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/fs.h> 4 #include <linux/poll.h> 5 #include <linux/sched/signal.h> 6 #include <linux/uaccess.h> 7 #include <uapi/misc/ocxl.h> 8 #include "ocxl_internal.h" 9 10 11 #define OCXL_NUM_MINORS 256 /* Total to reserve */ 12 13 static dev_t ocxl_dev; 14 static struct class *ocxl_class; 15 static struct mutex minors_idr_lock; 16 static struct idr minors_idr; 17 18 static struct ocxl_afu *find_and_get_afu(dev_t devno) 19 { 20 struct ocxl_afu *afu; 21 int afu_minor; 22 23 afu_minor = MINOR(devno); 24 /* 25 * We don't declare an RCU critical section here, as our AFU 26 * is protected by a reference counter on the device. By the time the 27 * minor number of a device is removed from the idr, the ref count of 28 * the device is already at 0, so no user API will access that AFU and 29 * this function can't return it. 30 */ 31 afu = idr_find(&minors_idr, afu_minor); 32 if (afu) 33 ocxl_afu_get(afu); 34 return afu; 35 } 36 37 static int allocate_afu_minor(struct ocxl_afu *afu) 38 { 39 int minor; 40 41 mutex_lock(&minors_idr_lock); 42 minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); 43 mutex_unlock(&minors_idr_lock); 44 return minor; 45 } 46 47 static void free_afu_minor(struct ocxl_afu *afu) 48 { 49 mutex_lock(&minors_idr_lock); 50 idr_remove(&minors_idr, MINOR(afu->dev.devt)); 51 mutex_unlock(&minors_idr_lock); 52 } 53 54 static int afu_open(struct inode *inode, struct file *file) 55 { 56 struct ocxl_afu *afu; 57 struct ocxl_context *ctx; 58 int rc; 59 60 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 61 62 afu = find_and_get_afu(inode->i_rdev); 63 if (!afu) 64 return -ENODEV; 65 66 ctx = ocxl_context_alloc(); 67 if (!ctx) { 68 rc = -ENOMEM; 69 goto put_afu; 70 } 71 72 rc = ocxl_context_init(ctx, afu, inode->i_mapping); 73 if (rc) 74 goto put_afu; 75 file->private_data = ctx; 76 ocxl_afu_put(afu); 77 return 0; 78 79 put_afu: 80 ocxl_afu_put(afu); 81 return rc; 82 } 83 84 static long afu_ioctl_attach(struct ocxl_context *ctx, 85 struct ocxl_ioctl_attach __user *uarg) 86 { 87 struct ocxl_ioctl_attach arg; 88 u64 amr = 0; 89 int rc; 90 91 pr_debug("%s for context %d\n", __func__, ctx->pasid); 92 93 if (copy_from_user(&arg, uarg, sizeof(arg))) 94 return -EFAULT; 95 96 /* Make sure reserved fields are not set for forward compatibility */ 97 if (arg.reserved1 || arg.reserved2 || arg.reserved3) 98 return -EINVAL; 99 100 amr = arg.amr & mfspr(SPRN_UAMOR); 101 rc = ocxl_context_attach(ctx, amr); 102 return rc; 103 } 104 105 #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ 106 x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ 107 x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ 108 x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ 109 "UNKNOWN") 110 111 static long afu_ioctl(struct file *file, unsigned int cmd, 112 unsigned long args) 113 { 114 struct ocxl_context *ctx = file->private_data; 115 struct ocxl_ioctl_irq_fd irq_fd; 116 u64 irq_offset; 117 long rc; 118 119 pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, 120 CMD_STR(cmd)); 121 122 if (ctx->status == CLOSED) 123 return -EIO; 124 125 switch (cmd) { 126 case OCXL_IOCTL_ATTACH: 127 rc = afu_ioctl_attach(ctx, 128 (struct ocxl_ioctl_attach __user *) args); 129 break; 130 131 case OCXL_IOCTL_IRQ_ALLOC: 132 rc = ocxl_afu_irq_alloc(ctx, &irq_offset); 133 if (!rc) { 134 rc = copy_to_user((u64 __user *) args, &irq_offset, 135 sizeof(irq_offset)); 136 if (rc) { 137 ocxl_afu_irq_free(ctx, irq_offset); 138 return -EFAULT; 139 } 140 } 141 break; 142 143 case OCXL_IOCTL_IRQ_FREE: 144 rc = copy_from_user(&irq_offset, (u64 __user *) args, 145 sizeof(irq_offset)); 146 if (rc) 147 return -EFAULT; 148 rc = ocxl_afu_irq_free(ctx, irq_offset); 149 break; 150 151 case OCXL_IOCTL_IRQ_SET_FD: 152 rc = copy_from_user(&irq_fd, (u64 __user *) args, 153 sizeof(irq_fd)); 154 if (rc) 155 return -EFAULT; 156 if (irq_fd.reserved) 157 return -EINVAL; 158 rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, 159 irq_fd.eventfd); 160 break; 161 162 default: 163 rc = -EINVAL; 164 } 165 return rc; 166 } 167 168 static long afu_compat_ioctl(struct file *file, unsigned int cmd, 169 unsigned long args) 170 { 171 return afu_ioctl(file, cmd, args); 172 } 173 174 static int afu_mmap(struct file *file, struct vm_area_struct *vma) 175 { 176 struct ocxl_context *ctx = file->private_data; 177 178 pr_debug("%s for context %d\n", __func__, ctx->pasid); 179 return ocxl_context_mmap(ctx, vma); 180 } 181 182 static bool has_xsl_error(struct ocxl_context *ctx) 183 { 184 bool ret; 185 186 mutex_lock(&ctx->xsl_error_lock); 187 ret = !!ctx->xsl_error.addr; 188 mutex_unlock(&ctx->xsl_error_lock); 189 190 return ret; 191 } 192 193 /* 194 * Are there any events pending on the AFU 195 * ctx: The AFU context 196 * Returns: true if there are events pending 197 */ 198 static bool afu_events_pending(struct ocxl_context *ctx) 199 { 200 if (has_xsl_error(ctx)) 201 return true; 202 return false; 203 } 204 205 static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait) 206 { 207 struct ocxl_context *ctx = file->private_data; 208 unsigned int mask = 0; 209 bool closed; 210 211 pr_debug("%s for context %d\n", __func__, ctx->pasid); 212 213 poll_wait(file, &ctx->events_wq, wait); 214 215 mutex_lock(&ctx->status_mutex); 216 closed = (ctx->status == CLOSED); 217 mutex_unlock(&ctx->status_mutex); 218 219 if (afu_events_pending(ctx)) 220 mask = EPOLLIN | EPOLLRDNORM; 221 else if (closed) 222 mask = EPOLLERR; 223 224 return mask; 225 } 226 227 /* 228 * Populate the supplied buffer with a single XSL error 229 * ctx: The AFU context to report the error from 230 * header: the event header to populate 231 * buf: The buffer to write the body into (should be at least 232 * AFU_EVENT_BODY_XSL_ERROR_SIZE) 233 * Return: the amount of buffer that was populated 234 */ 235 static ssize_t append_xsl_error(struct ocxl_context *ctx, 236 struct ocxl_kernel_event_header *header, 237 char __user *buf) 238 { 239 struct ocxl_kernel_event_xsl_fault_error body; 240 241 memset(&body, 0, sizeof(body)); 242 243 mutex_lock(&ctx->xsl_error_lock); 244 if (!ctx->xsl_error.addr) { 245 mutex_unlock(&ctx->xsl_error_lock); 246 return 0; 247 } 248 249 body.addr = ctx->xsl_error.addr; 250 body.dsisr = ctx->xsl_error.dsisr; 251 body.count = ctx->xsl_error.count; 252 253 ctx->xsl_error.addr = 0; 254 ctx->xsl_error.dsisr = 0; 255 ctx->xsl_error.count = 0; 256 257 mutex_unlock(&ctx->xsl_error_lock); 258 259 header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR; 260 261 if (copy_to_user(buf, &body, sizeof(body))) 262 return -EFAULT; 263 264 return sizeof(body); 265 } 266 267 #define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error) 268 269 /* 270 * Reports events on the AFU 271 * Format: 272 * Header (struct ocxl_kernel_event_header) 273 * Body (struct ocxl_kernel_event_*) 274 * Header... 275 */ 276 static ssize_t afu_read(struct file *file, char __user *buf, size_t count, 277 loff_t *off) 278 { 279 struct ocxl_context *ctx = file->private_data; 280 struct ocxl_kernel_event_header header; 281 ssize_t rc; 282 ssize_t used = 0; 283 DEFINE_WAIT(event_wait); 284 285 memset(&header, 0, sizeof(header)); 286 287 /* Require offset to be 0 */ 288 if (*off != 0) 289 return -EINVAL; 290 291 if (count < (sizeof(struct ocxl_kernel_event_header) + 292 AFU_EVENT_BODY_MAX_SIZE)) 293 return -EINVAL; 294 295 for (;;) { 296 prepare_to_wait(&ctx->events_wq, &event_wait, 297 TASK_INTERRUPTIBLE); 298 299 if (afu_events_pending(ctx)) 300 break; 301 302 if (ctx->status == CLOSED) 303 break; 304 305 if (file->f_flags & O_NONBLOCK) { 306 finish_wait(&ctx->events_wq, &event_wait); 307 return -EAGAIN; 308 } 309 310 if (signal_pending(current)) { 311 finish_wait(&ctx->events_wq, &event_wait); 312 return -ERESTARTSYS; 313 } 314 315 schedule(); 316 } 317 318 finish_wait(&ctx->events_wq, &event_wait); 319 320 if (has_xsl_error(ctx)) { 321 used = append_xsl_error(ctx, &header, buf + sizeof(header)); 322 if (used < 0) 323 return used; 324 } 325 326 if (!afu_events_pending(ctx)) 327 header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST; 328 329 if (copy_to_user(buf, &header, sizeof(header))) 330 return -EFAULT; 331 332 used += sizeof(header); 333 334 rc = used; 335 return rc; 336 } 337 338 static int afu_release(struct inode *inode, struct file *file) 339 { 340 struct ocxl_context *ctx = file->private_data; 341 int rc; 342 343 pr_debug("%s for device %x\n", __func__, inode->i_rdev); 344 rc = ocxl_context_detach(ctx); 345 mutex_lock(&ctx->mapping_lock); 346 ctx->mapping = NULL; 347 mutex_unlock(&ctx->mapping_lock); 348 wake_up_all(&ctx->events_wq); 349 if (rc != -EBUSY) 350 ocxl_context_free(ctx); 351 return 0; 352 } 353 354 static const struct file_operations ocxl_afu_fops = { 355 .owner = THIS_MODULE, 356 .open = afu_open, 357 .unlocked_ioctl = afu_ioctl, 358 .compat_ioctl = afu_compat_ioctl, 359 .mmap = afu_mmap, 360 .poll = afu_poll, 361 .read = afu_read, 362 .release = afu_release, 363 }; 364 365 int ocxl_create_cdev(struct ocxl_afu *afu) 366 { 367 int rc; 368 369 cdev_init(&afu->cdev, &ocxl_afu_fops); 370 rc = cdev_add(&afu->cdev, afu->dev.devt, 1); 371 if (rc) { 372 dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); 373 return rc; 374 } 375 return 0; 376 } 377 378 void ocxl_destroy_cdev(struct ocxl_afu *afu) 379 { 380 cdev_del(&afu->cdev); 381 } 382 383 int ocxl_register_afu(struct ocxl_afu *afu) 384 { 385 int minor; 386 387 minor = allocate_afu_minor(afu); 388 if (minor < 0) 389 return minor; 390 afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); 391 afu->dev.class = ocxl_class; 392 return device_register(&afu->dev); 393 } 394 395 void ocxl_unregister_afu(struct ocxl_afu *afu) 396 { 397 free_afu_minor(afu); 398 } 399 400 static char *ocxl_devnode(struct device *dev, umode_t *mode) 401 { 402 return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev)); 403 } 404 405 int ocxl_file_init(void) 406 { 407 int rc; 408 409 mutex_init(&minors_idr_lock); 410 idr_init(&minors_idr); 411 412 rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl"); 413 if (rc) { 414 pr_err("Unable to allocate ocxl major number: %d\n", rc); 415 return rc; 416 } 417 418 ocxl_class = class_create(THIS_MODULE, "ocxl"); 419 if (IS_ERR(ocxl_class)) { 420 pr_err("Unable to create ocxl class\n"); 421 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 422 return PTR_ERR(ocxl_class); 423 } 424 425 ocxl_class->devnode = ocxl_devnode; 426 return 0; 427 } 428 429 void ocxl_file_exit(void) 430 { 431 class_destroy(ocxl_class); 432 unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); 433 idr_destroy(&minors_idr); 434 } 435