1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/nospec.h> 15 #include <linux/slab.h> 16 17 #include "vfio_ccw_private.h" 18 19 static const struct vfio_device_ops vfio_ccw_dev_ops; 20 21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 22 { 23 /* 24 * If the FSM state is seen as Not Operational after closing 25 * and re-opening the mdev, return an error. 26 */ 27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 29 if (private->state == VFIO_CCW_STATE_NOT_OPER) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 36 { 37 struct vfio_ccw_private *private = 38 container_of(vdev, struct vfio_ccw_private, vdev); 39 40 /* Drivers MUST unpin pages in response to an invalidation. */ 41 if (!cp_iova_pinned(&private->cp, iova, length)) 42 return; 43 44 vfio_ccw_mdev_reset(private); 45 } 46 47 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 48 { 49 struct vfio_ccw_private *private = 50 container_of(vdev, struct vfio_ccw_private, vdev); 51 52 init_completion(&private->release_comp); 53 return 0; 54 } 55 56 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 57 { 58 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 59 int ret; 60 61 if (private->state == VFIO_CCW_STATE_NOT_OPER) 62 return -ENODEV; 63 64 ret = vfio_init_device(&private->vdev, &mdev->dev, &vfio_ccw_dev_ops); 65 if (ret) 66 return ret; 67 68 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 69 private->sch->schid.cssid, 70 private->sch->schid.ssid, 71 private->sch->schid.sch_no); 72 73 ret = vfio_register_emulated_iommu_dev(&private->vdev); 74 if (ret) 75 goto err_put_vdev; 76 dev_set_drvdata(&mdev->dev, private); 77 return 0; 78 79 err_put_vdev: 80 vfio_put_device(&private->vdev); 81 return ret; 82 } 83 84 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 85 { 86 struct vfio_ccw_private *private = 87 container_of(vdev, struct vfio_ccw_private, vdev); 88 89 /* 90 * We cannot free vfio_ccw_private here because it includes 91 * parent info which must be free'ed by css driver. 92 * 93 * Use a workaround by memset'ing the core device part and 94 * then notifying the remove path that all active references 95 * to this device have been released. 96 */ 97 memset(vdev, 0, sizeof(*vdev)); 98 complete(&private->release_comp); 99 } 100 101 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 102 { 103 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 104 105 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 106 private->sch->schid.cssid, 107 private->sch->schid.ssid, 108 private->sch->schid.sch_no); 109 110 vfio_unregister_group_dev(&private->vdev); 111 112 vfio_put_device(&private->vdev); 113 /* 114 * Wait for all active references on mdev are released so it 115 * is safe to defer kfree() to a later point. 116 * 117 * TODO: the clean fix is to split parent/mdev info from ccw 118 * private structure so each can be managed in its own life 119 * cycle. 120 */ 121 wait_for_completion(&private->release_comp); 122 } 123 124 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 125 { 126 struct vfio_ccw_private *private = 127 container_of(vdev, struct vfio_ccw_private, vdev); 128 int ret; 129 130 /* Device cannot simply be opened again from this state */ 131 if (private->state == VFIO_CCW_STATE_NOT_OPER) 132 return -EINVAL; 133 134 ret = vfio_ccw_register_async_dev_regions(private); 135 if (ret) 136 return ret; 137 138 ret = vfio_ccw_register_schib_dev_regions(private); 139 if (ret) 140 goto out_unregister; 141 142 ret = vfio_ccw_register_crw_dev_regions(private); 143 if (ret) 144 goto out_unregister; 145 146 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 147 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 148 ret = -EINVAL; 149 goto out_unregister; 150 } 151 152 return ret; 153 154 out_unregister: 155 vfio_ccw_unregister_dev_regions(private); 156 return ret; 157 } 158 159 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 160 { 161 struct vfio_ccw_private *private = 162 container_of(vdev, struct vfio_ccw_private, vdev); 163 164 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 165 vfio_ccw_unregister_dev_regions(private); 166 } 167 168 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 169 char __user *buf, size_t count, 170 loff_t *ppos) 171 { 172 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 173 struct ccw_io_region *region; 174 int ret; 175 176 if (pos + count > sizeof(*region)) 177 return -EINVAL; 178 179 mutex_lock(&private->io_mutex); 180 region = private->io_region; 181 if (copy_to_user(buf, (void *)region + pos, count)) 182 ret = -EFAULT; 183 else 184 ret = count; 185 mutex_unlock(&private->io_mutex); 186 return ret; 187 } 188 189 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 190 char __user *buf, 191 size_t count, 192 loff_t *ppos) 193 { 194 struct vfio_ccw_private *private = 195 container_of(vdev, struct vfio_ccw_private, vdev); 196 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 197 198 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 199 return -EINVAL; 200 201 switch (index) { 202 case VFIO_CCW_CONFIG_REGION_INDEX: 203 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 204 default: 205 index -= VFIO_CCW_NUM_REGIONS; 206 return private->region[index].ops->read(private, buf, count, 207 ppos); 208 } 209 210 return -EINVAL; 211 } 212 213 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 214 const char __user *buf, 215 size_t count, loff_t *ppos) 216 { 217 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 218 struct ccw_io_region *region; 219 int ret; 220 221 if (pos + count > sizeof(*region)) 222 return -EINVAL; 223 224 if (!mutex_trylock(&private->io_mutex)) 225 return -EAGAIN; 226 227 region = private->io_region; 228 if (copy_from_user((void *)region + pos, buf, count)) { 229 ret = -EFAULT; 230 goto out_unlock; 231 } 232 233 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 234 ret = (region->ret_code != 0) ? region->ret_code : count; 235 236 out_unlock: 237 mutex_unlock(&private->io_mutex); 238 return ret; 239 } 240 241 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 242 const char __user *buf, 243 size_t count, 244 loff_t *ppos) 245 { 246 struct vfio_ccw_private *private = 247 container_of(vdev, struct vfio_ccw_private, vdev); 248 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 249 250 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 251 return -EINVAL; 252 253 switch (index) { 254 case VFIO_CCW_CONFIG_REGION_INDEX: 255 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 256 default: 257 index -= VFIO_CCW_NUM_REGIONS; 258 return private->region[index].ops->write(private, buf, count, 259 ppos); 260 } 261 262 return -EINVAL; 263 } 264 265 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 266 struct vfio_device_info *info) 267 { 268 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 269 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 270 info->num_irqs = VFIO_CCW_NUM_IRQS; 271 272 return 0; 273 } 274 275 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 276 struct vfio_region_info *info, 277 unsigned long arg) 278 { 279 int i; 280 281 switch (info->index) { 282 case VFIO_CCW_CONFIG_REGION_INDEX: 283 info->offset = 0; 284 info->size = sizeof(struct ccw_io_region); 285 info->flags = VFIO_REGION_INFO_FLAG_READ 286 | VFIO_REGION_INFO_FLAG_WRITE; 287 return 0; 288 default: /* all other regions are handled via capability chain */ 289 { 290 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 291 struct vfio_region_info_cap_type cap_type = { 292 .header.id = VFIO_REGION_INFO_CAP_TYPE, 293 .header.version = 1 }; 294 int ret; 295 296 if (info->index >= 297 VFIO_CCW_NUM_REGIONS + private->num_regions) 298 return -EINVAL; 299 300 info->index = array_index_nospec(info->index, 301 VFIO_CCW_NUM_REGIONS + 302 private->num_regions); 303 304 i = info->index - VFIO_CCW_NUM_REGIONS; 305 306 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 307 info->size = private->region[i].size; 308 info->flags = private->region[i].flags; 309 310 cap_type.type = private->region[i].type; 311 cap_type.subtype = private->region[i].subtype; 312 313 ret = vfio_info_add_capability(&caps, &cap_type.header, 314 sizeof(cap_type)); 315 if (ret) 316 return ret; 317 318 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 319 if (info->argsz < sizeof(*info) + caps.size) { 320 info->argsz = sizeof(*info) + caps.size; 321 info->cap_offset = 0; 322 } else { 323 vfio_info_cap_shift(&caps, sizeof(*info)); 324 if (copy_to_user((void __user *)arg + sizeof(*info), 325 caps.buf, caps.size)) { 326 kfree(caps.buf); 327 return -EFAULT; 328 } 329 info->cap_offset = sizeof(*info); 330 } 331 332 kfree(caps.buf); 333 334 } 335 } 336 return 0; 337 } 338 339 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 340 { 341 switch (info->index) { 342 case VFIO_CCW_IO_IRQ_INDEX: 343 case VFIO_CCW_CRW_IRQ_INDEX: 344 case VFIO_CCW_REQ_IRQ_INDEX: 345 info->count = 1; 346 info->flags = VFIO_IRQ_INFO_EVENTFD; 347 break; 348 default: 349 return -EINVAL; 350 } 351 352 return 0; 353 } 354 355 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 356 uint32_t flags, 357 uint32_t index, 358 void __user *data) 359 { 360 struct eventfd_ctx **ctx; 361 362 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 363 return -EINVAL; 364 365 switch (index) { 366 case VFIO_CCW_IO_IRQ_INDEX: 367 ctx = &private->io_trigger; 368 break; 369 case VFIO_CCW_CRW_IRQ_INDEX: 370 ctx = &private->crw_trigger; 371 break; 372 case VFIO_CCW_REQ_IRQ_INDEX: 373 ctx = &private->req_trigger; 374 break; 375 default: 376 return -EINVAL; 377 } 378 379 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 380 case VFIO_IRQ_SET_DATA_NONE: 381 { 382 if (*ctx) 383 eventfd_signal(*ctx, 1); 384 return 0; 385 } 386 case VFIO_IRQ_SET_DATA_BOOL: 387 { 388 uint8_t trigger; 389 390 if (get_user(trigger, (uint8_t __user *)data)) 391 return -EFAULT; 392 393 if (trigger && *ctx) 394 eventfd_signal(*ctx, 1); 395 return 0; 396 } 397 case VFIO_IRQ_SET_DATA_EVENTFD: 398 { 399 int32_t fd; 400 401 if (get_user(fd, (int32_t __user *)data)) 402 return -EFAULT; 403 404 if (fd == -1) { 405 if (*ctx) 406 eventfd_ctx_put(*ctx); 407 *ctx = NULL; 408 } else if (fd >= 0) { 409 struct eventfd_ctx *efdctx; 410 411 efdctx = eventfd_ctx_fdget(fd); 412 if (IS_ERR(efdctx)) 413 return PTR_ERR(efdctx); 414 415 if (*ctx) 416 eventfd_ctx_put(*ctx); 417 418 *ctx = efdctx; 419 } else 420 return -EINVAL; 421 422 return 0; 423 } 424 default: 425 return -EINVAL; 426 } 427 } 428 429 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 430 unsigned int subtype, 431 const struct vfio_ccw_regops *ops, 432 size_t size, u32 flags, void *data) 433 { 434 struct vfio_ccw_region *region; 435 436 region = krealloc(private->region, 437 (private->num_regions + 1) * sizeof(*region), 438 GFP_KERNEL); 439 if (!region) 440 return -ENOMEM; 441 442 private->region = region; 443 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 444 private->region[private->num_regions].subtype = subtype; 445 private->region[private->num_regions].ops = ops; 446 private->region[private->num_regions].size = size; 447 private->region[private->num_regions].flags = flags; 448 private->region[private->num_regions].data = data; 449 450 private->num_regions++; 451 452 return 0; 453 } 454 455 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 456 { 457 int i; 458 459 for (i = 0; i < private->num_regions; i++) 460 private->region[i].ops->release(private, &private->region[i]); 461 private->num_regions = 0; 462 kfree(private->region); 463 private->region = NULL; 464 } 465 466 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 467 unsigned int cmd, 468 unsigned long arg) 469 { 470 struct vfio_ccw_private *private = 471 container_of(vdev, struct vfio_ccw_private, vdev); 472 int ret = 0; 473 unsigned long minsz; 474 475 switch (cmd) { 476 case VFIO_DEVICE_GET_INFO: 477 { 478 struct vfio_device_info info; 479 480 minsz = offsetofend(struct vfio_device_info, num_irqs); 481 482 if (copy_from_user(&info, (void __user *)arg, minsz)) 483 return -EFAULT; 484 485 if (info.argsz < minsz) 486 return -EINVAL; 487 488 ret = vfio_ccw_mdev_get_device_info(private, &info); 489 if (ret) 490 return ret; 491 492 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 493 } 494 case VFIO_DEVICE_GET_REGION_INFO: 495 { 496 struct vfio_region_info info; 497 498 minsz = offsetofend(struct vfio_region_info, offset); 499 500 if (copy_from_user(&info, (void __user *)arg, minsz)) 501 return -EFAULT; 502 503 if (info.argsz < minsz) 504 return -EINVAL; 505 506 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 507 if (ret) 508 return ret; 509 510 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 511 } 512 case VFIO_DEVICE_GET_IRQ_INFO: 513 { 514 struct vfio_irq_info info; 515 516 minsz = offsetofend(struct vfio_irq_info, count); 517 518 if (copy_from_user(&info, (void __user *)arg, minsz)) 519 return -EFAULT; 520 521 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 522 return -EINVAL; 523 524 ret = vfio_ccw_mdev_get_irq_info(&info); 525 if (ret) 526 return ret; 527 528 if (info.count == -1) 529 return -EINVAL; 530 531 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 532 } 533 case VFIO_DEVICE_SET_IRQS: 534 { 535 struct vfio_irq_set hdr; 536 size_t data_size; 537 void __user *data; 538 539 minsz = offsetofend(struct vfio_irq_set, count); 540 541 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 542 return -EFAULT; 543 544 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 545 VFIO_CCW_NUM_IRQS, 546 &data_size); 547 if (ret) 548 return ret; 549 550 data = (void __user *)(arg + minsz); 551 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 552 data); 553 } 554 case VFIO_DEVICE_RESET: 555 return vfio_ccw_mdev_reset(private); 556 default: 557 return -ENOTTY; 558 } 559 } 560 561 /* Request removal of the device*/ 562 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 563 { 564 struct vfio_ccw_private *private = 565 container_of(vdev, struct vfio_ccw_private, vdev); 566 struct device *dev = vdev->dev; 567 568 if (private->req_trigger) { 569 if (!(count % 10)) 570 dev_notice_ratelimited(dev, 571 "Relaying device request to user (#%u)\n", 572 count); 573 574 eventfd_signal(private->req_trigger, 1); 575 } else if (count == 0) { 576 dev_notice(dev, 577 "No device request channel registered, blocked until released by user\n"); 578 } 579 } 580 581 static const struct vfio_device_ops vfio_ccw_dev_ops = { 582 .init = vfio_ccw_mdev_init_dev, 583 .release = vfio_ccw_mdev_release_dev, 584 .open_device = vfio_ccw_mdev_open_device, 585 .close_device = vfio_ccw_mdev_close_device, 586 .read = vfio_ccw_mdev_read, 587 .write = vfio_ccw_mdev_write, 588 .ioctl = vfio_ccw_mdev_ioctl, 589 .request = vfio_ccw_mdev_request, 590 .dma_unmap = vfio_ccw_dma_unmap, 591 }; 592 593 struct mdev_driver vfio_ccw_mdev_driver = { 594 .device_api = VFIO_DEVICE_API_CCW_STRING, 595 .max_instances = 1, 596 .driver = { 597 .name = "vfio_ccw_mdev", 598 .owner = THIS_MODULE, 599 .mod_name = KBUILD_MODNAME, 600 }, 601 .probe = vfio_ccw_mdev_probe, 602 .remove = vfio_ccw_mdev_remove, 603 }; 604