1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/mdev.h> 15 #include <linux/nospec.h> 16 #include <linux/slab.h> 17 18 #include "vfio_ccw_private.h" 19 20 static const struct vfio_device_ops vfio_ccw_dev_ops; 21 22 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 23 { 24 struct subchannel *sch; 25 int ret; 26 27 sch = private->sch; 28 /* 29 * TODO: 30 * In the cureent stage, some things like "no I/O running" and "no 31 * interrupt pending" are clear, but we are not sure what other state 32 * we need to care about. 33 * There are still a lot more instructions need to be handled. We 34 * should come back here later. 35 */ 36 ret = vfio_ccw_sch_quiesce(sch); 37 if (ret) 38 return ret; 39 40 ret = cio_enable_subchannel(sch, (u32)(unsigned long)sch); 41 if (!ret) 42 private->state = VFIO_CCW_STATE_IDLE; 43 44 return ret; 45 } 46 47 static int vfio_ccw_mdev_notifier(struct notifier_block *nb, 48 unsigned long action, 49 void *data) 50 { 51 struct vfio_ccw_private *private = 52 container_of(nb, struct vfio_ccw_private, nb); 53 54 /* 55 * Vendor drivers MUST unpin pages in response to an 56 * invalidation. 57 */ 58 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 59 struct vfio_iommu_type1_dma_unmap *unmap = data; 60 61 if (!cp_iova_pinned(&private->cp, unmap->iova)) 62 return NOTIFY_OK; 63 64 if (vfio_ccw_mdev_reset(private)) 65 return NOTIFY_BAD; 66 67 cp_free(&private->cp); 68 return NOTIFY_OK; 69 } 70 71 return NOTIFY_DONE; 72 } 73 74 static ssize_t name_show(struct mdev_type *mtype, 75 struct mdev_type_attribute *attr, char *buf) 76 { 77 return sprintf(buf, "I/O subchannel (Non-QDIO)\n"); 78 } 79 static MDEV_TYPE_ATTR_RO(name); 80 81 static ssize_t device_api_show(struct mdev_type *mtype, 82 struct mdev_type_attribute *attr, char *buf) 83 { 84 return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING); 85 } 86 static MDEV_TYPE_ATTR_RO(device_api); 87 88 static ssize_t available_instances_show(struct mdev_type *mtype, 89 struct mdev_type_attribute *attr, 90 char *buf) 91 { 92 struct vfio_ccw_private *private = 93 dev_get_drvdata(mtype_get_parent_dev(mtype)); 94 95 return sprintf(buf, "%d\n", atomic_read(&private->avail)); 96 } 97 static MDEV_TYPE_ATTR_RO(available_instances); 98 99 static struct attribute *mdev_types_attrs[] = { 100 &mdev_type_attr_name.attr, 101 &mdev_type_attr_device_api.attr, 102 &mdev_type_attr_available_instances.attr, 103 NULL, 104 }; 105 106 static struct attribute_group mdev_type_group = { 107 .name = "io", 108 .attrs = mdev_types_attrs, 109 }; 110 111 static struct attribute_group *mdev_type_groups[] = { 112 &mdev_type_group, 113 NULL, 114 }; 115 116 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 117 { 118 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 119 int ret; 120 121 if (private->state == VFIO_CCW_STATE_NOT_OPER) 122 return -ENODEV; 123 124 if (atomic_dec_if_positive(&private->avail) < 0) 125 return -EPERM; 126 127 memset(&private->vdev, 0, sizeof(private->vdev)); 128 vfio_init_group_dev(&private->vdev, &mdev->dev, 129 &vfio_ccw_dev_ops); 130 131 private->mdev = mdev; 132 private->state = VFIO_CCW_STATE_IDLE; 133 134 VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: create\n", 135 mdev_uuid(mdev), private->sch->schid.cssid, 136 private->sch->schid.ssid, 137 private->sch->schid.sch_no); 138 139 ret = vfio_register_emulated_iommu_dev(&private->vdev); 140 if (ret) 141 goto err_atomic; 142 dev_set_drvdata(&mdev->dev, private); 143 return 0; 144 145 err_atomic: 146 vfio_uninit_group_dev(&private->vdev); 147 atomic_inc(&private->avail); 148 private->mdev = NULL; 149 private->state = VFIO_CCW_STATE_IDLE; 150 return ret; 151 } 152 153 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 154 { 155 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 156 157 VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: remove\n", 158 mdev_uuid(mdev), private->sch->schid.cssid, 159 private->sch->schid.ssid, 160 private->sch->schid.sch_no); 161 162 vfio_unregister_group_dev(&private->vdev); 163 164 if ((private->state != VFIO_CCW_STATE_NOT_OPER) && 165 (private->state != VFIO_CCW_STATE_STANDBY)) { 166 if (!vfio_ccw_sch_quiesce(private->sch)) 167 private->state = VFIO_CCW_STATE_STANDBY; 168 /* The state will be NOT_OPER on error. */ 169 } 170 171 vfio_uninit_group_dev(&private->vdev); 172 cp_free(&private->cp); 173 private->mdev = NULL; 174 atomic_inc(&private->avail); 175 } 176 177 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 178 { 179 struct vfio_ccw_private *private = 180 container_of(vdev, struct vfio_ccw_private, vdev); 181 unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 182 int ret; 183 184 private->nb.notifier_call = vfio_ccw_mdev_notifier; 185 186 ret = vfio_register_notifier(vdev, VFIO_IOMMU_NOTIFY, 187 &events, &private->nb); 188 if (ret) 189 return ret; 190 191 ret = vfio_ccw_register_async_dev_regions(private); 192 if (ret) 193 goto out_unregister; 194 195 ret = vfio_ccw_register_schib_dev_regions(private); 196 if (ret) 197 goto out_unregister; 198 199 ret = vfio_ccw_register_crw_dev_regions(private); 200 if (ret) 201 goto out_unregister; 202 203 return ret; 204 205 out_unregister: 206 vfio_ccw_unregister_dev_regions(private); 207 vfio_unregister_notifier(vdev, VFIO_IOMMU_NOTIFY, &private->nb); 208 return ret; 209 } 210 211 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 212 { 213 struct vfio_ccw_private *private = 214 container_of(vdev, struct vfio_ccw_private, vdev); 215 216 if ((private->state != VFIO_CCW_STATE_NOT_OPER) && 217 (private->state != VFIO_CCW_STATE_STANDBY)) { 218 if (!vfio_ccw_mdev_reset(private)) 219 private->state = VFIO_CCW_STATE_STANDBY; 220 /* The state will be NOT_OPER on error. */ 221 } 222 223 cp_free(&private->cp); 224 vfio_ccw_unregister_dev_regions(private); 225 vfio_unregister_notifier(vdev, VFIO_IOMMU_NOTIFY, &private->nb); 226 } 227 228 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 229 char __user *buf, size_t count, 230 loff_t *ppos) 231 { 232 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 233 struct ccw_io_region *region; 234 int ret; 235 236 if (pos + count > sizeof(*region)) 237 return -EINVAL; 238 239 mutex_lock(&private->io_mutex); 240 region = private->io_region; 241 if (copy_to_user(buf, (void *)region + pos, count)) 242 ret = -EFAULT; 243 else 244 ret = count; 245 mutex_unlock(&private->io_mutex); 246 return ret; 247 } 248 249 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 250 char __user *buf, 251 size_t count, 252 loff_t *ppos) 253 { 254 struct vfio_ccw_private *private = 255 container_of(vdev, struct vfio_ccw_private, vdev); 256 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 257 258 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 259 return -EINVAL; 260 261 switch (index) { 262 case VFIO_CCW_CONFIG_REGION_INDEX: 263 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 264 default: 265 index -= VFIO_CCW_NUM_REGIONS; 266 return private->region[index].ops->read(private, buf, count, 267 ppos); 268 } 269 270 return -EINVAL; 271 } 272 273 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 274 const char __user *buf, 275 size_t count, loff_t *ppos) 276 { 277 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 278 struct ccw_io_region *region; 279 int ret; 280 281 if (pos + count > sizeof(*region)) 282 return -EINVAL; 283 284 if (!mutex_trylock(&private->io_mutex)) 285 return -EAGAIN; 286 287 region = private->io_region; 288 if (copy_from_user((void *)region + pos, buf, count)) { 289 ret = -EFAULT; 290 goto out_unlock; 291 } 292 293 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 294 ret = (region->ret_code != 0) ? region->ret_code : count; 295 296 out_unlock: 297 mutex_unlock(&private->io_mutex); 298 return ret; 299 } 300 301 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 302 const char __user *buf, 303 size_t count, 304 loff_t *ppos) 305 { 306 struct vfio_ccw_private *private = 307 container_of(vdev, struct vfio_ccw_private, vdev); 308 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 309 310 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 311 return -EINVAL; 312 313 switch (index) { 314 case VFIO_CCW_CONFIG_REGION_INDEX: 315 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 316 default: 317 index -= VFIO_CCW_NUM_REGIONS; 318 return private->region[index].ops->write(private, buf, count, 319 ppos); 320 } 321 322 return -EINVAL; 323 } 324 325 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 326 struct vfio_device_info *info) 327 { 328 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 329 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 330 info->num_irqs = VFIO_CCW_NUM_IRQS; 331 332 return 0; 333 } 334 335 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 336 struct vfio_region_info *info, 337 unsigned long arg) 338 { 339 int i; 340 341 switch (info->index) { 342 case VFIO_CCW_CONFIG_REGION_INDEX: 343 info->offset = 0; 344 info->size = sizeof(struct ccw_io_region); 345 info->flags = VFIO_REGION_INFO_FLAG_READ 346 | VFIO_REGION_INFO_FLAG_WRITE; 347 return 0; 348 default: /* all other regions are handled via capability chain */ 349 { 350 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 351 struct vfio_region_info_cap_type cap_type = { 352 .header.id = VFIO_REGION_INFO_CAP_TYPE, 353 .header.version = 1 }; 354 int ret; 355 356 if (info->index >= 357 VFIO_CCW_NUM_REGIONS + private->num_regions) 358 return -EINVAL; 359 360 info->index = array_index_nospec(info->index, 361 VFIO_CCW_NUM_REGIONS + 362 private->num_regions); 363 364 i = info->index - VFIO_CCW_NUM_REGIONS; 365 366 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 367 info->size = private->region[i].size; 368 info->flags = private->region[i].flags; 369 370 cap_type.type = private->region[i].type; 371 cap_type.subtype = private->region[i].subtype; 372 373 ret = vfio_info_add_capability(&caps, &cap_type.header, 374 sizeof(cap_type)); 375 if (ret) 376 return ret; 377 378 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 379 if (info->argsz < sizeof(*info) + caps.size) { 380 info->argsz = sizeof(*info) + caps.size; 381 info->cap_offset = 0; 382 } else { 383 vfio_info_cap_shift(&caps, sizeof(*info)); 384 if (copy_to_user((void __user *)arg + sizeof(*info), 385 caps.buf, caps.size)) { 386 kfree(caps.buf); 387 return -EFAULT; 388 } 389 info->cap_offset = sizeof(*info); 390 } 391 392 kfree(caps.buf); 393 394 } 395 } 396 return 0; 397 } 398 399 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 400 { 401 switch (info->index) { 402 case VFIO_CCW_IO_IRQ_INDEX: 403 case VFIO_CCW_CRW_IRQ_INDEX: 404 case VFIO_CCW_REQ_IRQ_INDEX: 405 info->count = 1; 406 info->flags = VFIO_IRQ_INFO_EVENTFD; 407 break; 408 default: 409 return -EINVAL; 410 } 411 412 return 0; 413 } 414 415 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 416 uint32_t flags, 417 uint32_t index, 418 void __user *data) 419 { 420 struct eventfd_ctx **ctx; 421 422 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 423 return -EINVAL; 424 425 switch (index) { 426 case VFIO_CCW_IO_IRQ_INDEX: 427 ctx = &private->io_trigger; 428 break; 429 case VFIO_CCW_CRW_IRQ_INDEX: 430 ctx = &private->crw_trigger; 431 break; 432 case VFIO_CCW_REQ_IRQ_INDEX: 433 ctx = &private->req_trigger; 434 break; 435 default: 436 return -EINVAL; 437 } 438 439 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 440 case VFIO_IRQ_SET_DATA_NONE: 441 { 442 if (*ctx) 443 eventfd_signal(*ctx, 1); 444 return 0; 445 } 446 case VFIO_IRQ_SET_DATA_BOOL: 447 { 448 uint8_t trigger; 449 450 if (get_user(trigger, (uint8_t __user *)data)) 451 return -EFAULT; 452 453 if (trigger && *ctx) 454 eventfd_signal(*ctx, 1); 455 return 0; 456 } 457 case VFIO_IRQ_SET_DATA_EVENTFD: 458 { 459 int32_t fd; 460 461 if (get_user(fd, (int32_t __user *)data)) 462 return -EFAULT; 463 464 if (fd == -1) { 465 if (*ctx) 466 eventfd_ctx_put(*ctx); 467 *ctx = NULL; 468 } else if (fd >= 0) { 469 struct eventfd_ctx *efdctx; 470 471 efdctx = eventfd_ctx_fdget(fd); 472 if (IS_ERR(efdctx)) 473 return PTR_ERR(efdctx); 474 475 if (*ctx) 476 eventfd_ctx_put(*ctx); 477 478 *ctx = efdctx; 479 } else 480 return -EINVAL; 481 482 return 0; 483 } 484 default: 485 return -EINVAL; 486 } 487 } 488 489 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 490 unsigned int subtype, 491 const struct vfio_ccw_regops *ops, 492 size_t size, u32 flags, void *data) 493 { 494 struct vfio_ccw_region *region; 495 496 region = krealloc(private->region, 497 (private->num_regions + 1) * sizeof(*region), 498 GFP_KERNEL); 499 if (!region) 500 return -ENOMEM; 501 502 private->region = region; 503 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 504 private->region[private->num_regions].subtype = subtype; 505 private->region[private->num_regions].ops = ops; 506 private->region[private->num_regions].size = size; 507 private->region[private->num_regions].flags = flags; 508 private->region[private->num_regions].data = data; 509 510 private->num_regions++; 511 512 return 0; 513 } 514 515 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 516 { 517 int i; 518 519 for (i = 0; i < private->num_regions; i++) 520 private->region[i].ops->release(private, &private->region[i]); 521 private->num_regions = 0; 522 kfree(private->region); 523 private->region = NULL; 524 } 525 526 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 527 unsigned int cmd, 528 unsigned long arg) 529 { 530 struct vfio_ccw_private *private = 531 container_of(vdev, struct vfio_ccw_private, vdev); 532 int ret = 0; 533 unsigned long minsz; 534 535 switch (cmd) { 536 case VFIO_DEVICE_GET_INFO: 537 { 538 struct vfio_device_info info; 539 540 minsz = offsetofend(struct vfio_device_info, num_irqs); 541 542 if (copy_from_user(&info, (void __user *)arg, minsz)) 543 return -EFAULT; 544 545 if (info.argsz < minsz) 546 return -EINVAL; 547 548 ret = vfio_ccw_mdev_get_device_info(private, &info); 549 if (ret) 550 return ret; 551 552 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 553 } 554 case VFIO_DEVICE_GET_REGION_INFO: 555 { 556 struct vfio_region_info info; 557 558 minsz = offsetofend(struct vfio_region_info, offset); 559 560 if (copy_from_user(&info, (void __user *)arg, minsz)) 561 return -EFAULT; 562 563 if (info.argsz < minsz) 564 return -EINVAL; 565 566 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 567 if (ret) 568 return ret; 569 570 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 571 } 572 case VFIO_DEVICE_GET_IRQ_INFO: 573 { 574 struct vfio_irq_info info; 575 576 minsz = offsetofend(struct vfio_irq_info, count); 577 578 if (copy_from_user(&info, (void __user *)arg, minsz)) 579 return -EFAULT; 580 581 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 582 return -EINVAL; 583 584 ret = vfio_ccw_mdev_get_irq_info(&info); 585 if (ret) 586 return ret; 587 588 if (info.count == -1) 589 return -EINVAL; 590 591 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 592 } 593 case VFIO_DEVICE_SET_IRQS: 594 { 595 struct vfio_irq_set hdr; 596 size_t data_size; 597 void __user *data; 598 599 minsz = offsetofend(struct vfio_irq_set, count); 600 601 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 602 return -EFAULT; 603 604 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 605 VFIO_CCW_NUM_IRQS, 606 &data_size); 607 if (ret) 608 return ret; 609 610 data = (void __user *)(arg + minsz); 611 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 612 data); 613 } 614 case VFIO_DEVICE_RESET: 615 return vfio_ccw_mdev_reset(private); 616 default: 617 return -ENOTTY; 618 } 619 } 620 621 /* Request removal of the device*/ 622 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 623 { 624 struct vfio_ccw_private *private = 625 container_of(vdev, struct vfio_ccw_private, vdev); 626 struct device *dev = vdev->dev; 627 628 if (private->req_trigger) { 629 if (!(count % 10)) 630 dev_notice_ratelimited(dev, 631 "Relaying device request to user (#%u)\n", 632 count); 633 634 eventfd_signal(private->req_trigger, 1); 635 } else if (count == 0) { 636 dev_notice(dev, 637 "No device request channel registered, blocked until released by user\n"); 638 } 639 } 640 641 static const struct vfio_device_ops vfio_ccw_dev_ops = { 642 .open_device = vfio_ccw_mdev_open_device, 643 .close_device = vfio_ccw_mdev_close_device, 644 .read = vfio_ccw_mdev_read, 645 .write = vfio_ccw_mdev_write, 646 .ioctl = vfio_ccw_mdev_ioctl, 647 .request = vfio_ccw_mdev_request, 648 }; 649 650 struct mdev_driver vfio_ccw_mdev_driver = { 651 .driver = { 652 .name = "vfio_ccw_mdev", 653 .owner = THIS_MODULE, 654 .mod_name = KBUILD_MODNAME, 655 }, 656 .probe = vfio_ccw_mdev_probe, 657 .remove = vfio_ccw_mdev_remove, 658 .supported_type_groups = mdev_type_groups, 659 }; 660 661 int vfio_ccw_mdev_reg(struct subchannel *sch) 662 { 663 return mdev_register_device(&sch->dev, &vfio_ccw_mdev_driver); 664 } 665 666 void vfio_ccw_mdev_unreg(struct subchannel *sch) 667 { 668 mdev_unregister_device(&sch->dev); 669 } 670