1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/nospec.h> 15 #include <linux/slab.h> 16 17 #include "vfio_ccw_private.h" 18 19 static const struct vfio_device_ops vfio_ccw_dev_ops; 20 21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 22 { 23 /* 24 * If the FSM state is seen as Not Operational after closing 25 * and re-opening the mdev, return an error. 26 */ 27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 29 if (private->state == VFIO_CCW_STATE_NOT_OPER) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 36 { 37 struct vfio_ccw_private *private = 38 container_of(vdev, struct vfio_ccw_private, vdev); 39 40 /* Drivers MUST unpin pages in response to an invalidation. */ 41 if (!cp_iova_pinned(&private->cp, iova, length)) 42 return; 43 44 vfio_ccw_mdev_reset(private); 45 } 46 47 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 48 { 49 struct vfio_ccw_private *private = 50 container_of(vdev, struct vfio_ccw_private, vdev); 51 52 mutex_init(&private->io_mutex); 53 private->state = VFIO_CCW_STATE_STANDBY; 54 INIT_LIST_HEAD(&private->crw); 55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); 56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); 57 58 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), 59 GFP_KERNEL); 60 if (!private->cp.guest_cp) 61 goto out_free_private; 62 63 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, 64 GFP_KERNEL | GFP_DMA); 65 if (!private->io_region) 66 goto out_free_cp; 67 68 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, 69 GFP_KERNEL | GFP_DMA); 70 if (!private->cmd_region) 71 goto out_free_io; 72 73 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region, 74 GFP_KERNEL | GFP_DMA); 75 if (!private->schib_region) 76 goto out_free_cmd; 77 78 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region, 79 GFP_KERNEL | GFP_DMA); 80 if (!private->crw_region) 81 goto out_free_schib; 82 83 return 0; 84 85 out_free_schib: 86 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 87 out_free_cmd: 88 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 89 out_free_io: 90 kmem_cache_free(vfio_ccw_io_region, private->io_region); 91 out_free_cp: 92 kfree(private->cp.guest_cp); 93 out_free_private: 94 mutex_destroy(&private->io_mutex); 95 return -ENOMEM; 96 } 97 98 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 99 { 100 struct subchannel *sch = to_subchannel(mdev->dev.parent); 101 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); 102 struct vfio_ccw_private *private; 103 int ret; 104 105 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev, 106 &vfio_ccw_dev_ops); 107 if (IS_ERR(private)) 108 return PTR_ERR(private); 109 110 dev_set_drvdata(&parent->dev, private); 111 112 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 113 sch->schid.cssid, 114 sch->schid.ssid, 115 sch->schid.sch_no); 116 117 ret = vfio_register_emulated_iommu_dev(&private->vdev); 118 if (ret) 119 goto err_put_vdev; 120 dev_set_drvdata(&mdev->dev, private); 121 return 0; 122 123 err_put_vdev: 124 dev_set_drvdata(&parent->dev, NULL); 125 vfio_put_device(&private->vdev); 126 return ret; 127 } 128 129 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 130 { 131 struct vfio_ccw_private *private = 132 container_of(vdev, struct vfio_ccw_private, vdev); 133 struct vfio_ccw_crw *crw, *temp; 134 135 list_for_each_entry_safe(crw, temp, &private->crw, next) { 136 list_del(&crw->next); 137 kfree(crw); 138 } 139 140 kmem_cache_free(vfio_ccw_crw_region, private->crw_region); 141 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 142 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 143 kmem_cache_free(vfio_ccw_io_region, private->io_region); 144 kfree(private->cp.guest_cp); 145 mutex_destroy(&private->io_mutex); 146 } 147 148 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 149 { 150 struct subchannel *sch = to_subchannel(mdev->dev.parent); 151 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); 152 struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev); 153 154 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 155 sch->schid.cssid, 156 sch->schid.ssid, 157 sch->schid.sch_no); 158 159 vfio_unregister_group_dev(&private->vdev); 160 161 dev_set_drvdata(&parent->dev, NULL); 162 vfio_put_device(&private->vdev); 163 } 164 165 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 166 { 167 struct vfio_ccw_private *private = 168 container_of(vdev, struct vfio_ccw_private, vdev); 169 int ret; 170 171 /* Device cannot simply be opened again from this state */ 172 if (private->state == VFIO_CCW_STATE_NOT_OPER) 173 return -EINVAL; 174 175 ret = vfio_ccw_register_async_dev_regions(private); 176 if (ret) 177 return ret; 178 179 ret = vfio_ccw_register_schib_dev_regions(private); 180 if (ret) 181 goto out_unregister; 182 183 ret = vfio_ccw_register_crw_dev_regions(private); 184 if (ret) 185 goto out_unregister; 186 187 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 188 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 189 ret = -EINVAL; 190 goto out_unregister; 191 } 192 193 return ret; 194 195 out_unregister: 196 vfio_ccw_unregister_dev_regions(private); 197 return ret; 198 } 199 200 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 201 { 202 struct vfio_ccw_private *private = 203 container_of(vdev, struct vfio_ccw_private, vdev); 204 205 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 206 vfio_ccw_unregister_dev_regions(private); 207 } 208 209 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 210 char __user *buf, size_t count, 211 loff_t *ppos) 212 { 213 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 214 struct ccw_io_region *region; 215 int ret; 216 217 if (pos + count > sizeof(*region)) 218 return -EINVAL; 219 220 mutex_lock(&private->io_mutex); 221 region = private->io_region; 222 if (copy_to_user(buf, (void *)region + pos, count)) 223 ret = -EFAULT; 224 else 225 ret = count; 226 mutex_unlock(&private->io_mutex); 227 return ret; 228 } 229 230 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 231 char __user *buf, 232 size_t count, 233 loff_t *ppos) 234 { 235 struct vfio_ccw_private *private = 236 container_of(vdev, struct vfio_ccw_private, vdev); 237 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 238 239 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 240 return -EINVAL; 241 242 switch (index) { 243 case VFIO_CCW_CONFIG_REGION_INDEX: 244 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 245 default: 246 index -= VFIO_CCW_NUM_REGIONS; 247 return private->region[index].ops->read(private, buf, count, 248 ppos); 249 } 250 251 return -EINVAL; 252 } 253 254 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 255 const char __user *buf, 256 size_t count, loff_t *ppos) 257 { 258 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 259 struct ccw_io_region *region; 260 int ret; 261 262 if (pos + count > sizeof(*region)) 263 return -EINVAL; 264 265 if (!mutex_trylock(&private->io_mutex)) 266 return -EAGAIN; 267 268 region = private->io_region; 269 if (copy_from_user((void *)region + pos, buf, count)) { 270 ret = -EFAULT; 271 goto out_unlock; 272 } 273 274 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 275 ret = (region->ret_code != 0) ? region->ret_code : count; 276 277 out_unlock: 278 mutex_unlock(&private->io_mutex); 279 return ret; 280 } 281 282 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 283 const char __user *buf, 284 size_t count, 285 loff_t *ppos) 286 { 287 struct vfio_ccw_private *private = 288 container_of(vdev, struct vfio_ccw_private, vdev); 289 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 290 291 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 292 return -EINVAL; 293 294 switch (index) { 295 case VFIO_CCW_CONFIG_REGION_INDEX: 296 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 297 default: 298 index -= VFIO_CCW_NUM_REGIONS; 299 return private->region[index].ops->write(private, buf, count, 300 ppos); 301 } 302 303 return -EINVAL; 304 } 305 306 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 307 struct vfio_device_info *info) 308 { 309 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 310 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 311 info->num_irqs = VFIO_CCW_NUM_IRQS; 312 313 return 0; 314 } 315 316 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 317 struct vfio_region_info *info, 318 unsigned long arg) 319 { 320 int i; 321 322 switch (info->index) { 323 case VFIO_CCW_CONFIG_REGION_INDEX: 324 info->offset = 0; 325 info->size = sizeof(struct ccw_io_region); 326 info->flags = VFIO_REGION_INFO_FLAG_READ 327 | VFIO_REGION_INFO_FLAG_WRITE; 328 return 0; 329 default: /* all other regions are handled via capability chain */ 330 { 331 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 332 struct vfio_region_info_cap_type cap_type = { 333 .header.id = VFIO_REGION_INFO_CAP_TYPE, 334 .header.version = 1 }; 335 int ret; 336 337 if (info->index >= 338 VFIO_CCW_NUM_REGIONS + private->num_regions) 339 return -EINVAL; 340 341 info->index = array_index_nospec(info->index, 342 VFIO_CCW_NUM_REGIONS + 343 private->num_regions); 344 345 i = info->index - VFIO_CCW_NUM_REGIONS; 346 347 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 348 info->size = private->region[i].size; 349 info->flags = private->region[i].flags; 350 351 cap_type.type = private->region[i].type; 352 cap_type.subtype = private->region[i].subtype; 353 354 ret = vfio_info_add_capability(&caps, &cap_type.header, 355 sizeof(cap_type)); 356 if (ret) 357 return ret; 358 359 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 360 if (info->argsz < sizeof(*info) + caps.size) { 361 info->argsz = sizeof(*info) + caps.size; 362 info->cap_offset = 0; 363 } else { 364 vfio_info_cap_shift(&caps, sizeof(*info)); 365 if (copy_to_user((void __user *)arg + sizeof(*info), 366 caps.buf, caps.size)) { 367 kfree(caps.buf); 368 return -EFAULT; 369 } 370 info->cap_offset = sizeof(*info); 371 } 372 373 kfree(caps.buf); 374 375 } 376 } 377 return 0; 378 } 379 380 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 381 { 382 switch (info->index) { 383 case VFIO_CCW_IO_IRQ_INDEX: 384 case VFIO_CCW_CRW_IRQ_INDEX: 385 case VFIO_CCW_REQ_IRQ_INDEX: 386 info->count = 1; 387 info->flags = VFIO_IRQ_INFO_EVENTFD; 388 break; 389 default: 390 return -EINVAL; 391 } 392 393 return 0; 394 } 395 396 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 397 uint32_t flags, 398 uint32_t index, 399 void __user *data) 400 { 401 struct eventfd_ctx **ctx; 402 403 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 404 return -EINVAL; 405 406 switch (index) { 407 case VFIO_CCW_IO_IRQ_INDEX: 408 ctx = &private->io_trigger; 409 break; 410 case VFIO_CCW_CRW_IRQ_INDEX: 411 ctx = &private->crw_trigger; 412 break; 413 case VFIO_CCW_REQ_IRQ_INDEX: 414 ctx = &private->req_trigger; 415 break; 416 default: 417 return -EINVAL; 418 } 419 420 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 421 case VFIO_IRQ_SET_DATA_NONE: 422 { 423 if (*ctx) 424 eventfd_signal(*ctx, 1); 425 return 0; 426 } 427 case VFIO_IRQ_SET_DATA_BOOL: 428 { 429 uint8_t trigger; 430 431 if (get_user(trigger, (uint8_t __user *)data)) 432 return -EFAULT; 433 434 if (trigger && *ctx) 435 eventfd_signal(*ctx, 1); 436 return 0; 437 } 438 case VFIO_IRQ_SET_DATA_EVENTFD: 439 { 440 int32_t fd; 441 442 if (get_user(fd, (int32_t __user *)data)) 443 return -EFAULT; 444 445 if (fd == -1) { 446 if (*ctx) 447 eventfd_ctx_put(*ctx); 448 *ctx = NULL; 449 } else if (fd >= 0) { 450 struct eventfd_ctx *efdctx; 451 452 efdctx = eventfd_ctx_fdget(fd); 453 if (IS_ERR(efdctx)) 454 return PTR_ERR(efdctx); 455 456 if (*ctx) 457 eventfd_ctx_put(*ctx); 458 459 *ctx = efdctx; 460 } else 461 return -EINVAL; 462 463 return 0; 464 } 465 default: 466 return -EINVAL; 467 } 468 } 469 470 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 471 unsigned int subtype, 472 const struct vfio_ccw_regops *ops, 473 size_t size, u32 flags, void *data) 474 { 475 struct vfio_ccw_region *region; 476 477 region = krealloc(private->region, 478 (private->num_regions + 1) * sizeof(*region), 479 GFP_KERNEL); 480 if (!region) 481 return -ENOMEM; 482 483 private->region = region; 484 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 485 private->region[private->num_regions].subtype = subtype; 486 private->region[private->num_regions].ops = ops; 487 private->region[private->num_regions].size = size; 488 private->region[private->num_regions].flags = flags; 489 private->region[private->num_regions].data = data; 490 491 private->num_regions++; 492 493 return 0; 494 } 495 496 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 497 { 498 int i; 499 500 for (i = 0; i < private->num_regions; i++) 501 private->region[i].ops->release(private, &private->region[i]); 502 private->num_regions = 0; 503 kfree(private->region); 504 private->region = NULL; 505 } 506 507 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 508 unsigned int cmd, 509 unsigned long arg) 510 { 511 struct vfio_ccw_private *private = 512 container_of(vdev, struct vfio_ccw_private, vdev); 513 int ret = 0; 514 unsigned long minsz; 515 516 switch (cmd) { 517 case VFIO_DEVICE_GET_INFO: 518 { 519 struct vfio_device_info info; 520 521 minsz = offsetofend(struct vfio_device_info, num_irqs); 522 523 if (copy_from_user(&info, (void __user *)arg, minsz)) 524 return -EFAULT; 525 526 if (info.argsz < minsz) 527 return -EINVAL; 528 529 ret = vfio_ccw_mdev_get_device_info(private, &info); 530 if (ret) 531 return ret; 532 533 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 534 } 535 case VFIO_DEVICE_GET_REGION_INFO: 536 { 537 struct vfio_region_info info; 538 539 minsz = offsetofend(struct vfio_region_info, offset); 540 541 if (copy_from_user(&info, (void __user *)arg, minsz)) 542 return -EFAULT; 543 544 if (info.argsz < minsz) 545 return -EINVAL; 546 547 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 548 if (ret) 549 return ret; 550 551 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 552 } 553 case VFIO_DEVICE_GET_IRQ_INFO: 554 { 555 struct vfio_irq_info info; 556 557 minsz = offsetofend(struct vfio_irq_info, count); 558 559 if (copy_from_user(&info, (void __user *)arg, minsz)) 560 return -EFAULT; 561 562 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 563 return -EINVAL; 564 565 ret = vfio_ccw_mdev_get_irq_info(&info); 566 if (ret) 567 return ret; 568 569 if (info.count == -1) 570 return -EINVAL; 571 572 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 573 } 574 case VFIO_DEVICE_SET_IRQS: 575 { 576 struct vfio_irq_set hdr; 577 size_t data_size; 578 void __user *data; 579 580 minsz = offsetofend(struct vfio_irq_set, count); 581 582 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 583 return -EFAULT; 584 585 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 586 VFIO_CCW_NUM_IRQS, 587 &data_size); 588 if (ret) 589 return ret; 590 591 data = (void __user *)(arg + minsz); 592 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 593 data); 594 } 595 case VFIO_DEVICE_RESET: 596 return vfio_ccw_mdev_reset(private); 597 default: 598 return -ENOTTY; 599 } 600 } 601 602 /* Request removal of the device*/ 603 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 604 { 605 struct vfio_ccw_private *private = 606 container_of(vdev, struct vfio_ccw_private, vdev); 607 struct device *dev = vdev->dev; 608 609 if (private->req_trigger) { 610 if (!(count % 10)) 611 dev_notice_ratelimited(dev, 612 "Relaying device request to user (#%u)\n", 613 count); 614 615 eventfd_signal(private->req_trigger, 1); 616 } else if (count == 0) { 617 dev_notice(dev, 618 "No device request channel registered, blocked until released by user\n"); 619 } 620 } 621 622 static const struct vfio_device_ops vfio_ccw_dev_ops = { 623 .init = vfio_ccw_mdev_init_dev, 624 .release = vfio_ccw_mdev_release_dev, 625 .open_device = vfio_ccw_mdev_open_device, 626 .close_device = vfio_ccw_mdev_close_device, 627 .read = vfio_ccw_mdev_read, 628 .write = vfio_ccw_mdev_write, 629 .ioctl = vfio_ccw_mdev_ioctl, 630 .request = vfio_ccw_mdev_request, 631 .dma_unmap = vfio_ccw_dma_unmap, 632 .bind_iommufd = vfio_iommufd_emulated_bind, 633 .unbind_iommufd = vfio_iommufd_emulated_unbind, 634 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 635 .detach_ioas = vfio_iommufd_emulated_detach_ioas, 636 }; 637 638 struct mdev_driver vfio_ccw_mdev_driver = { 639 .device_api = VFIO_DEVICE_API_CCW_STRING, 640 .max_instances = 1, 641 .driver = { 642 .name = "vfio_ccw_mdev", 643 .owner = THIS_MODULE, 644 .mod_name = KBUILD_MODNAME, 645 }, 646 .probe = vfio_ccw_mdev_probe, 647 .remove = vfio_ccw_mdev_remove, 648 }; 649