1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/mdev.h> 15 #include <linux/nospec.h> 16 #include <linux/slab.h> 17 18 #include "vfio_ccw_private.h" 19 20 static const struct vfio_device_ops vfio_ccw_dev_ops; 21 22 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 23 { 24 struct subchannel *sch; 25 int ret; 26 27 sch = private->sch; 28 /* 29 * TODO: 30 * In the cureent stage, some things like "no I/O running" and "no 31 * interrupt pending" are clear, but we are not sure what other state 32 * we need to care about. 33 * There are still a lot more instructions need to be handled. We 34 * should come back here later. 35 */ 36 ret = vfio_ccw_sch_quiesce(sch); 37 if (ret) 38 return ret; 39 40 ret = cio_enable_subchannel(sch, (u32)(unsigned long)sch); 41 if (!ret) 42 private->state = VFIO_CCW_STATE_IDLE; 43 44 return ret; 45 } 46 47 static int vfio_ccw_mdev_notifier(struct notifier_block *nb, 48 unsigned long action, 49 void *data) 50 { 51 struct vfio_ccw_private *private = 52 container_of(nb, struct vfio_ccw_private, nb); 53 54 /* 55 * Vendor drivers MUST unpin pages in response to an 56 * invalidation. 57 */ 58 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 59 struct vfio_iommu_type1_dma_unmap *unmap = data; 60 61 if (!cp_iova_pinned(&private->cp, unmap->iova)) 62 return NOTIFY_OK; 63 64 if (vfio_ccw_mdev_reset(private)) 65 return NOTIFY_BAD; 66 67 cp_free(&private->cp); 68 return NOTIFY_OK; 69 } 70 71 return NOTIFY_DONE; 72 } 73 74 static ssize_t name_show(struct mdev_type *mtype, 75 struct mdev_type_attribute *attr, char *buf) 76 { 77 return sprintf(buf, "I/O subchannel (Non-QDIO)\n"); 78 } 79 static MDEV_TYPE_ATTR_RO(name); 80 81 static ssize_t device_api_show(struct mdev_type *mtype, 82 struct mdev_type_attribute *attr, char *buf) 83 { 84 return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING); 85 } 86 static MDEV_TYPE_ATTR_RO(device_api); 87 88 static ssize_t available_instances_show(struct mdev_type *mtype, 89 struct mdev_type_attribute *attr, 90 char *buf) 91 { 92 struct vfio_ccw_private *private = 93 dev_get_drvdata(mtype_get_parent_dev(mtype)); 94 95 return sprintf(buf, "%d\n", atomic_read(&private->avail)); 96 } 97 static MDEV_TYPE_ATTR_RO(available_instances); 98 99 static struct attribute *mdev_types_attrs[] = { 100 &mdev_type_attr_name.attr, 101 &mdev_type_attr_device_api.attr, 102 &mdev_type_attr_available_instances.attr, 103 NULL, 104 }; 105 106 static struct attribute_group mdev_type_group = { 107 .name = "io", 108 .attrs = mdev_types_attrs, 109 }; 110 111 static struct attribute_group *mdev_type_groups[] = { 112 &mdev_type_group, 113 NULL, 114 }; 115 116 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 117 { 118 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 119 int ret; 120 121 if (private->state == VFIO_CCW_STATE_NOT_OPER) 122 return -ENODEV; 123 124 if (atomic_dec_if_positive(&private->avail) < 0) 125 return -EPERM; 126 127 memset(&private->vdev, 0, sizeof(private->vdev)); 128 vfio_init_group_dev(&private->vdev, &mdev->dev, 129 &vfio_ccw_dev_ops); 130 131 private->mdev = mdev; 132 private->state = VFIO_CCW_STATE_IDLE; 133 134 VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: create\n", 135 mdev_uuid(mdev), private->sch->schid.cssid, 136 private->sch->schid.ssid, 137 private->sch->schid.sch_no); 138 139 ret = vfio_register_emulated_iommu_dev(&private->vdev); 140 if (ret) 141 goto err_atomic; 142 dev_set_drvdata(&mdev->dev, private); 143 return 0; 144 145 err_atomic: 146 vfio_uninit_group_dev(&private->vdev); 147 atomic_inc(&private->avail); 148 private->mdev = NULL; 149 private->state = VFIO_CCW_STATE_IDLE; 150 return ret; 151 } 152 153 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 154 { 155 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 156 157 VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: remove\n", 158 mdev_uuid(mdev), private->sch->schid.cssid, 159 private->sch->schid.ssid, 160 private->sch->schid.sch_no); 161 162 vfio_unregister_group_dev(&private->vdev); 163 164 if ((private->state != VFIO_CCW_STATE_NOT_OPER) && 165 (private->state != VFIO_CCW_STATE_STANDBY)) { 166 if (!vfio_ccw_sch_quiesce(private->sch)) 167 private->state = VFIO_CCW_STATE_STANDBY; 168 /* The state will be NOT_OPER on error. */ 169 } 170 171 vfio_uninit_group_dev(&private->vdev); 172 cp_free(&private->cp); 173 private->mdev = NULL; 174 atomic_inc(&private->avail); 175 } 176 177 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 178 { 179 struct vfio_ccw_private *private = 180 container_of(vdev, struct vfio_ccw_private, vdev); 181 unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 182 int ret; 183 184 private->nb.notifier_call = vfio_ccw_mdev_notifier; 185 186 ret = vfio_register_notifier(vdev->dev, VFIO_IOMMU_NOTIFY, 187 &events, &private->nb); 188 if (ret) 189 return ret; 190 191 ret = vfio_ccw_register_async_dev_regions(private); 192 if (ret) 193 goto out_unregister; 194 195 ret = vfio_ccw_register_schib_dev_regions(private); 196 if (ret) 197 goto out_unregister; 198 199 ret = vfio_ccw_register_crw_dev_regions(private); 200 if (ret) 201 goto out_unregister; 202 203 return ret; 204 205 out_unregister: 206 vfio_ccw_unregister_dev_regions(private); 207 vfio_unregister_notifier(vdev->dev, VFIO_IOMMU_NOTIFY, 208 &private->nb); 209 return ret; 210 } 211 212 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 213 { 214 struct vfio_ccw_private *private = 215 container_of(vdev, struct vfio_ccw_private, vdev); 216 217 if ((private->state != VFIO_CCW_STATE_NOT_OPER) && 218 (private->state != VFIO_CCW_STATE_STANDBY)) { 219 if (!vfio_ccw_mdev_reset(private)) 220 private->state = VFIO_CCW_STATE_STANDBY; 221 /* The state will be NOT_OPER on error. */ 222 } 223 224 cp_free(&private->cp); 225 vfio_ccw_unregister_dev_regions(private); 226 vfio_unregister_notifier(vdev->dev, VFIO_IOMMU_NOTIFY, &private->nb); 227 } 228 229 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 230 char __user *buf, size_t count, 231 loff_t *ppos) 232 { 233 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 234 struct ccw_io_region *region; 235 int ret; 236 237 if (pos + count > sizeof(*region)) 238 return -EINVAL; 239 240 mutex_lock(&private->io_mutex); 241 region = private->io_region; 242 if (copy_to_user(buf, (void *)region + pos, count)) 243 ret = -EFAULT; 244 else 245 ret = count; 246 mutex_unlock(&private->io_mutex); 247 return ret; 248 } 249 250 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 251 char __user *buf, 252 size_t count, 253 loff_t *ppos) 254 { 255 struct vfio_ccw_private *private = 256 container_of(vdev, struct vfio_ccw_private, vdev); 257 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 258 259 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 260 return -EINVAL; 261 262 switch (index) { 263 case VFIO_CCW_CONFIG_REGION_INDEX: 264 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 265 default: 266 index -= VFIO_CCW_NUM_REGIONS; 267 return private->region[index].ops->read(private, buf, count, 268 ppos); 269 } 270 271 return -EINVAL; 272 } 273 274 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 275 const char __user *buf, 276 size_t count, loff_t *ppos) 277 { 278 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 279 struct ccw_io_region *region; 280 int ret; 281 282 if (pos + count > sizeof(*region)) 283 return -EINVAL; 284 285 if (!mutex_trylock(&private->io_mutex)) 286 return -EAGAIN; 287 288 region = private->io_region; 289 if (copy_from_user((void *)region + pos, buf, count)) { 290 ret = -EFAULT; 291 goto out_unlock; 292 } 293 294 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 295 ret = (region->ret_code != 0) ? region->ret_code : count; 296 297 out_unlock: 298 mutex_unlock(&private->io_mutex); 299 return ret; 300 } 301 302 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 303 const char __user *buf, 304 size_t count, 305 loff_t *ppos) 306 { 307 struct vfio_ccw_private *private = 308 container_of(vdev, struct vfio_ccw_private, vdev); 309 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 310 311 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 312 return -EINVAL; 313 314 switch (index) { 315 case VFIO_CCW_CONFIG_REGION_INDEX: 316 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 317 default: 318 index -= VFIO_CCW_NUM_REGIONS; 319 return private->region[index].ops->write(private, buf, count, 320 ppos); 321 } 322 323 return -EINVAL; 324 } 325 326 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 327 struct vfio_device_info *info) 328 { 329 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 330 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 331 info->num_irqs = VFIO_CCW_NUM_IRQS; 332 333 return 0; 334 } 335 336 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 337 struct vfio_region_info *info, 338 unsigned long arg) 339 { 340 int i; 341 342 switch (info->index) { 343 case VFIO_CCW_CONFIG_REGION_INDEX: 344 info->offset = 0; 345 info->size = sizeof(struct ccw_io_region); 346 info->flags = VFIO_REGION_INFO_FLAG_READ 347 | VFIO_REGION_INFO_FLAG_WRITE; 348 return 0; 349 default: /* all other regions are handled via capability chain */ 350 { 351 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 352 struct vfio_region_info_cap_type cap_type = { 353 .header.id = VFIO_REGION_INFO_CAP_TYPE, 354 .header.version = 1 }; 355 int ret; 356 357 if (info->index >= 358 VFIO_CCW_NUM_REGIONS + private->num_regions) 359 return -EINVAL; 360 361 info->index = array_index_nospec(info->index, 362 VFIO_CCW_NUM_REGIONS + 363 private->num_regions); 364 365 i = info->index - VFIO_CCW_NUM_REGIONS; 366 367 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 368 info->size = private->region[i].size; 369 info->flags = private->region[i].flags; 370 371 cap_type.type = private->region[i].type; 372 cap_type.subtype = private->region[i].subtype; 373 374 ret = vfio_info_add_capability(&caps, &cap_type.header, 375 sizeof(cap_type)); 376 if (ret) 377 return ret; 378 379 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 380 if (info->argsz < sizeof(*info) + caps.size) { 381 info->argsz = sizeof(*info) + caps.size; 382 info->cap_offset = 0; 383 } else { 384 vfio_info_cap_shift(&caps, sizeof(*info)); 385 if (copy_to_user((void __user *)arg + sizeof(*info), 386 caps.buf, caps.size)) { 387 kfree(caps.buf); 388 return -EFAULT; 389 } 390 info->cap_offset = sizeof(*info); 391 } 392 393 kfree(caps.buf); 394 395 } 396 } 397 return 0; 398 } 399 400 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 401 { 402 switch (info->index) { 403 case VFIO_CCW_IO_IRQ_INDEX: 404 case VFIO_CCW_CRW_IRQ_INDEX: 405 case VFIO_CCW_REQ_IRQ_INDEX: 406 info->count = 1; 407 info->flags = VFIO_IRQ_INFO_EVENTFD; 408 break; 409 default: 410 return -EINVAL; 411 } 412 413 return 0; 414 } 415 416 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 417 uint32_t flags, 418 uint32_t index, 419 void __user *data) 420 { 421 struct eventfd_ctx **ctx; 422 423 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 424 return -EINVAL; 425 426 switch (index) { 427 case VFIO_CCW_IO_IRQ_INDEX: 428 ctx = &private->io_trigger; 429 break; 430 case VFIO_CCW_CRW_IRQ_INDEX: 431 ctx = &private->crw_trigger; 432 break; 433 case VFIO_CCW_REQ_IRQ_INDEX: 434 ctx = &private->req_trigger; 435 break; 436 default: 437 return -EINVAL; 438 } 439 440 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 441 case VFIO_IRQ_SET_DATA_NONE: 442 { 443 if (*ctx) 444 eventfd_signal(*ctx, 1); 445 return 0; 446 } 447 case VFIO_IRQ_SET_DATA_BOOL: 448 { 449 uint8_t trigger; 450 451 if (get_user(trigger, (uint8_t __user *)data)) 452 return -EFAULT; 453 454 if (trigger && *ctx) 455 eventfd_signal(*ctx, 1); 456 return 0; 457 } 458 case VFIO_IRQ_SET_DATA_EVENTFD: 459 { 460 int32_t fd; 461 462 if (get_user(fd, (int32_t __user *)data)) 463 return -EFAULT; 464 465 if (fd == -1) { 466 if (*ctx) 467 eventfd_ctx_put(*ctx); 468 *ctx = NULL; 469 } else if (fd >= 0) { 470 struct eventfd_ctx *efdctx; 471 472 efdctx = eventfd_ctx_fdget(fd); 473 if (IS_ERR(efdctx)) 474 return PTR_ERR(efdctx); 475 476 if (*ctx) 477 eventfd_ctx_put(*ctx); 478 479 *ctx = efdctx; 480 } else 481 return -EINVAL; 482 483 return 0; 484 } 485 default: 486 return -EINVAL; 487 } 488 } 489 490 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 491 unsigned int subtype, 492 const struct vfio_ccw_regops *ops, 493 size_t size, u32 flags, void *data) 494 { 495 struct vfio_ccw_region *region; 496 497 region = krealloc(private->region, 498 (private->num_regions + 1) * sizeof(*region), 499 GFP_KERNEL); 500 if (!region) 501 return -ENOMEM; 502 503 private->region = region; 504 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 505 private->region[private->num_regions].subtype = subtype; 506 private->region[private->num_regions].ops = ops; 507 private->region[private->num_regions].size = size; 508 private->region[private->num_regions].flags = flags; 509 private->region[private->num_regions].data = data; 510 511 private->num_regions++; 512 513 return 0; 514 } 515 516 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 517 { 518 int i; 519 520 for (i = 0; i < private->num_regions; i++) 521 private->region[i].ops->release(private, &private->region[i]); 522 private->num_regions = 0; 523 kfree(private->region); 524 private->region = NULL; 525 } 526 527 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 528 unsigned int cmd, 529 unsigned long arg) 530 { 531 struct vfio_ccw_private *private = 532 container_of(vdev, struct vfio_ccw_private, vdev); 533 int ret = 0; 534 unsigned long minsz; 535 536 switch (cmd) { 537 case VFIO_DEVICE_GET_INFO: 538 { 539 struct vfio_device_info info; 540 541 minsz = offsetofend(struct vfio_device_info, num_irqs); 542 543 if (copy_from_user(&info, (void __user *)arg, minsz)) 544 return -EFAULT; 545 546 if (info.argsz < minsz) 547 return -EINVAL; 548 549 ret = vfio_ccw_mdev_get_device_info(private, &info); 550 if (ret) 551 return ret; 552 553 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 554 } 555 case VFIO_DEVICE_GET_REGION_INFO: 556 { 557 struct vfio_region_info info; 558 559 minsz = offsetofend(struct vfio_region_info, offset); 560 561 if (copy_from_user(&info, (void __user *)arg, minsz)) 562 return -EFAULT; 563 564 if (info.argsz < minsz) 565 return -EINVAL; 566 567 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 568 if (ret) 569 return ret; 570 571 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 572 } 573 case VFIO_DEVICE_GET_IRQ_INFO: 574 { 575 struct vfio_irq_info info; 576 577 minsz = offsetofend(struct vfio_irq_info, count); 578 579 if (copy_from_user(&info, (void __user *)arg, minsz)) 580 return -EFAULT; 581 582 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 583 return -EINVAL; 584 585 ret = vfio_ccw_mdev_get_irq_info(&info); 586 if (ret) 587 return ret; 588 589 if (info.count == -1) 590 return -EINVAL; 591 592 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 593 } 594 case VFIO_DEVICE_SET_IRQS: 595 { 596 struct vfio_irq_set hdr; 597 size_t data_size; 598 void __user *data; 599 600 minsz = offsetofend(struct vfio_irq_set, count); 601 602 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 603 return -EFAULT; 604 605 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 606 VFIO_CCW_NUM_IRQS, 607 &data_size); 608 if (ret) 609 return ret; 610 611 data = (void __user *)(arg + minsz); 612 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 613 data); 614 } 615 case VFIO_DEVICE_RESET: 616 return vfio_ccw_mdev_reset(private); 617 default: 618 return -ENOTTY; 619 } 620 } 621 622 /* Request removal of the device*/ 623 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 624 { 625 struct vfio_ccw_private *private = 626 container_of(vdev, struct vfio_ccw_private, vdev); 627 struct device *dev = vdev->dev; 628 629 if (private->req_trigger) { 630 if (!(count % 10)) 631 dev_notice_ratelimited(dev, 632 "Relaying device request to user (#%u)\n", 633 count); 634 635 eventfd_signal(private->req_trigger, 1); 636 } else if (count == 0) { 637 dev_notice(dev, 638 "No device request channel registered, blocked until released by user\n"); 639 } 640 } 641 642 static const struct vfio_device_ops vfio_ccw_dev_ops = { 643 .open_device = vfio_ccw_mdev_open_device, 644 .close_device = vfio_ccw_mdev_close_device, 645 .read = vfio_ccw_mdev_read, 646 .write = vfio_ccw_mdev_write, 647 .ioctl = vfio_ccw_mdev_ioctl, 648 .request = vfio_ccw_mdev_request, 649 }; 650 651 struct mdev_driver vfio_ccw_mdev_driver = { 652 .driver = { 653 .name = "vfio_ccw_mdev", 654 .owner = THIS_MODULE, 655 .mod_name = KBUILD_MODNAME, 656 }, 657 .probe = vfio_ccw_mdev_probe, 658 .remove = vfio_ccw_mdev_remove, 659 }; 660 661 static const struct mdev_parent_ops vfio_ccw_mdev_ops = { 662 .owner = THIS_MODULE, 663 .device_driver = &vfio_ccw_mdev_driver, 664 .supported_type_groups = mdev_type_groups, 665 }; 666 667 int vfio_ccw_mdev_reg(struct subchannel *sch) 668 { 669 return mdev_register_device(&sch->dev, &vfio_ccw_mdev_ops); 670 } 671 672 void vfio_ccw_mdev_unreg(struct subchannel *sch) 673 { 674 mdev_unregister_device(&sch->dev); 675 } 676