1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/mdev.h> 15 #include <linux/nospec.h> 16 #include <linux/slab.h> 17 18 #include "vfio_ccw_private.h" 19 20 static const struct vfio_device_ops vfio_ccw_dev_ops; 21 22 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 23 { 24 /* 25 * If the FSM state is seen as Not Operational after closing 26 * and re-opening the mdev, return an error. 27 */ 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 29 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 30 if (private->state == VFIO_CCW_STATE_NOT_OPER) 31 return -EINVAL; 32 33 return 0; 34 } 35 36 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 37 { 38 struct vfio_ccw_private *private = 39 container_of(vdev, struct vfio_ccw_private, vdev); 40 41 /* Drivers MUST unpin pages in response to an invalidation. */ 42 if (!cp_iova_pinned(&private->cp, iova, length)) 43 return; 44 45 vfio_ccw_mdev_reset(private); 46 } 47 48 static ssize_t name_show(struct mdev_type *mtype, 49 struct mdev_type_attribute *attr, char *buf) 50 { 51 return sprintf(buf, "I/O subchannel (Non-QDIO)\n"); 52 } 53 static MDEV_TYPE_ATTR_RO(name); 54 55 static ssize_t device_api_show(struct mdev_type *mtype, 56 struct mdev_type_attribute *attr, char *buf) 57 { 58 return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING); 59 } 60 static MDEV_TYPE_ATTR_RO(device_api); 61 62 static ssize_t available_instances_show(struct mdev_type *mtype, 63 struct mdev_type_attribute *attr, 64 char *buf) 65 { 66 struct vfio_ccw_private *private = 67 dev_get_drvdata(mtype_get_parent_dev(mtype)); 68 69 return sprintf(buf, "%d\n", atomic_read(&private->avail)); 70 } 71 static MDEV_TYPE_ATTR_RO(available_instances); 72 73 static struct attribute *mdev_types_attrs[] = { 74 &mdev_type_attr_name.attr, 75 &mdev_type_attr_device_api.attr, 76 &mdev_type_attr_available_instances.attr, 77 NULL, 78 }; 79 80 static struct attribute_group mdev_type_group = { 81 .name = "io", 82 .attrs = mdev_types_attrs, 83 }; 84 85 static struct attribute_group *mdev_type_groups[] = { 86 &mdev_type_group, 87 NULL, 88 }; 89 90 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 91 { 92 struct vfio_ccw_private *private = 93 container_of(vdev, struct vfio_ccw_private, vdev); 94 95 init_completion(&private->release_comp); 96 return 0; 97 } 98 99 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 100 { 101 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 102 int ret; 103 104 if (private->state == VFIO_CCW_STATE_NOT_OPER) 105 return -ENODEV; 106 107 if (atomic_dec_if_positive(&private->avail) < 0) 108 return -EPERM; 109 110 ret = vfio_init_device(&private->vdev, &mdev->dev, &vfio_ccw_dev_ops); 111 if (ret) 112 return ret; 113 114 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 115 private->sch->schid.cssid, 116 private->sch->schid.ssid, 117 private->sch->schid.sch_no); 118 119 ret = vfio_register_emulated_iommu_dev(&private->vdev); 120 if (ret) 121 goto err_put_vdev; 122 dev_set_drvdata(&mdev->dev, private); 123 return 0; 124 125 err_put_vdev: 126 vfio_put_device(&private->vdev); 127 atomic_inc(&private->avail); 128 return ret; 129 } 130 131 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 132 { 133 struct vfio_ccw_private *private = 134 container_of(vdev, struct vfio_ccw_private, vdev); 135 136 /* 137 * We cannot free vfio_ccw_private here because it includes 138 * parent info which must be free'ed by css driver. 139 * 140 * Use a workaround by memset'ing the core device part and 141 * then notifying the remove path that all active references 142 * to this device have been released. 143 */ 144 memset(vdev, 0, sizeof(*vdev)); 145 complete(&private->release_comp); 146 } 147 148 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 149 { 150 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 151 152 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 153 private->sch->schid.cssid, 154 private->sch->schid.ssid, 155 private->sch->schid.sch_no); 156 157 vfio_unregister_group_dev(&private->vdev); 158 159 vfio_put_device(&private->vdev); 160 /* 161 * Wait for all active references on mdev are released so it 162 * is safe to defer kfree() to a later point. 163 * 164 * TODO: the clean fix is to split parent/mdev info from ccw 165 * private structure so each can be managed in its own life 166 * cycle. 167 */ 168 wait_for_completion(&private->release_comp); 169 170 atomic_inc(&private->avail); 171 } 172 173 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 174 { 175 struct vfio_ccw_private *private = 176 container_of(vdev, struct vfio_ccw_private, vdev); 177 int ret; 178 179 /* Device cannot simply be opened again from this state */ 180 if (private->state == VFIO_CCW_STATE_NOT_OPER) 181 return -EINVAL; 182 183 ret = vfio_ccw_register_async_dev_regions(private); 184 if (ret) 185 return ret; 186 187 ret = vfio_ccw_register_schib_dev_regions(private); 188 if (ret) 189 goto out_unregister; 190 191 ret = vfio_ccw_register_crw_dev_regions(private); 192 if (ret) 193 goto out_unregister; 194 195 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 196 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 197 ret = -EINVAL; 198 goto out_unregister; 199 } 200 201 return ret; 202 203 out_unregister: 204 vfio_ccw_unregister_dev_regions(private); 205 return ret; 206 } 207 208 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 209 { 210 struct vfio_ccw_private *private = 211 container_of(vdev, struct vfio_ccw_private, vdev); 212 213 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 214 vfio_ccw_unregister_dev_regions(private); 215 } 216 217 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 218 char __user *buf, size_t count, 219 loff_t *ppos) 220 { 221 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 222 struct ccw_io_region *region; 223 int ret; 224 225 if (pos + count > sizeof(*region)) 226 return -EINVAL; 227 228 mutex_lock(&private->io_mutex); 229 region = private->io_region; 230 if (copy_to_user(buf, (void *)region + pos, count)) 231 ret = -EFAULT; 232 else 233 ret = count; 234 mutex_unlock(&private->io_mutex); 235 return ret; 236 } 237 238 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 239 char __user *buf, 240 size_t count, 241 loff_t *ppos) 242 { 243 struct vfio_ccw_private *private = 244 container_of(vdev, struct vfio_ccw_private, vdev); 245 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 246 247 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 248 return -EINVAL; 249 250 switch (index) { 251 case VFIO_CCW_CONFIG_REGION_INDEX: 252 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 253 default: 254 index -= VFIO_CCW_NUM_REGIONS; 255 return private->region[index].ops->read(private, buf, count, 256 ppos); 257 } 258 259 return -EINVAL; 260 } 261 262 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 263 const char __user *buf, 264 size_t count, loff_t *ppos) 265 { 266 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 267 struct ccw_io_region *region; 268 int ret; 269 270 if (pos + count > sizeof(*region)) 271 return -EINVAL; 272 273 if (!mutex_trylock(&private->io_mutex)) 274 return -EAGAIN; 275 276 region = private->io_region; 277 if (copy_from_user((void *)region + pos, buf, count)) { 278 ret = -EFAULT; 279 goto out_unlock; 280 } 281 282 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 283 ret = (region->ret_code != 0) ? region->ret_code : count; 284 285 out_unlock: 286 mutex_unlock(&private->io_mutex); 287 return ret; 288 } 289 290 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 291 const char __user *buf, 292 size_t count, 293 loff_t *ppos) 294 { 295 struct vfio_ccw_private *private = 296 container_of(vdev, struct vfio_ccw_private, vdev); 297 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 298 299 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 300 return -EINVAL; 301 302 switch (index) { 303 case VFIO_CCW_CONFIG_REGION_INDEX: 304 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 305 default: 306 index -= VFIO_CCW_NUM_REGIONS; 307 return private->region[index].ops->write(private, buf, count, 308 ppos); 309 } 310 311 return -EINVAL; 312 } 313 314 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 315 struct vfio_device_info *info) 316 { 317 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 318 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 319 info->num_irqs = VFIO_CCW_NUM_IRQS; 320 321 return 0; 322 } 323 324 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 325 struct vfio_region_info *info, 326 unsigned long arg) 327 { 328 int i; 329 330 switch (info->index) { 331 case VFIO_CCW_CONFIG_REGION_INDEX: 332 info->offset = 0; 333 info->size = sizeof(struct ccw_io_region); 334 info->flags = VFIO_REGION_INFO_FLAG_READ 335 | VFIO_REGION_INFO_FLAG_WRITE; 336 return 0; 337 default: /* all other regions are handled via capability chain */ 338 { 339 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 340 struct vfio_region_info_cap_type cap_type = { 341 .header.id = VFIO_REGION_INFO_CAP_TYPE, 342 .header.version = 1 }; 343 int ret; 344 345 if (info->index >= 346 VFIO_CCW_NUM_REGIONS + private->num_regions) 347 return -EINVAL; 348 349 info->index = array_index_nospec(info->index, 350 VFIO_CCW_NUM_REGIONS + 351 private->num_regions); 352 353 i = info->index - VFIO_CCW_NUM_REGIONS; 354 355 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 356 info->size = private->region[i].size; 357 info->flags = private->region[i].flags; 358 359 cap_type.type = private->region[i].type; 360 cap_type.subtype = private->region[i].subtype; 361 362 ret = vfio_info_add_capability(&caps, &cap_type.header, 363 sizeof(cap_type)); 364 if (ret) 365 return ret; 366 367 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 368 if (info->argsz < sizeof(*info) + caps.size) { 369 info->argsz = sizeof(*info) + caps.size; 370 info->cap_offset = 0; 371 } else { 372 vfio_info_cap_shift(&caps, sizeof(*info)); 373 if (copy_to_user((void __user *)arg + sizeof(*info), 374 caps.buf, caps.size)) { 375 kfree(caps.buf); 376 return -EFAULT; 377 } 378 info->cap_offset = sizeof(*info); 379 } 380 381 kfree(caps.buf); 382 383 } 384 } 385 return 0; 386 } 387 388 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 389 { 390 switch (info->index) { 391 case VFIO_CCW_IO_IRQ_INDEX: 392 case VFIO_CCW_CRW_IRQ_INDEX: 393 case VFIO_CCW_REQ_IRQ_INDEX: 394 info->count = 1; 395 info->flags = VFIO_IRQ_INFO_EVENTFD; 396 break; 397 default: 398 return -EINVAL; 399 } 400 401 return 0; 402 } 403 404 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 405 uint32_t flags, 406 uint32_t index, 407 void __user *data) 408 { 409 struct eventfd_ctx **ctx; 410 411 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 412 return -EINVAL; 413 414 switch (index) { 415 case VFIO_CCW_IO_IRQ_INDEX: 416 ctx = &private->io_trigger; 417 break; 418 case VFIO_CCW_CRW_IRQ_INDEX: 419 ctx = &private->crw_trigger; 420 break; 421 case VFIO_CCW_REQ_IRQ_INDEX: 422 ctx = &private->req_trigger; 423 break; 424 default: 425 return -EINVAL; 426 } 427 428 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 429 case VFIO_IRQ_SET_DATA_NONE: 430 { 431 if (*ctx) 432 eventfd_signal(*ctx, 1); 433 return 0; 434 } 435 case VFIO_IRQ_SET_DATA_BOOL: 436 { 437 uint8_t trigger; 438 439 if (get_user(trigger, (uint8_t __user *)data)) 440 return -EFAULT; 441 442 if (trigger && *ctx) 443 eventfd_signal(*ctx, 1); 444 return 0; 445 } 446 case VFIO_IRQ_SET_DATA_EVENTFD: 447 { 448 int32_t fd; 449 450 if (get_user(fd, (int32_t __user *)data)) 451 return -EFAULT; 452 453 if (fd == -1) { 454 if (*ctx) 455 eventfd_ctx_put(*ctx); 456 *ctx = NULL; 457 } else if (fd >= 0) { 458 struct eventfd_ctx *efdctx; 459 460 efdctx = eventfd_ctx_fdget(fd); 461 if (IS_ERR(efdctx)) 462 return PTR_ERR(efdctx); 463 464 if (*ctx) 465 eventfd_ctx_put(*ctx); 466 467 *ctx = efdctx; 468 } else 469 return -EINVAL; 470 471 return 0; 472 } 473 default: 474 return -EINVAL; 475 } 476 } 477 478 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 479 unsigned int subtype, 480 const struct vfio_ccw_regops *ops, 481 size_t size, u32 flags, void *data) 482 { 483 struct vfio_ccw_region *region; 484 485 region = krealloc(private->region, 486 (private->num_regions + 1) * sizeof(*region), 487 GFP_KERNEL); 488 if (!region) 489 return -ENOMEM; 490 491 private->region = region; 492 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 493 private->region[private->num_regions].subtype = subtype; 494 private->region[private->num_regions].ops = ops; 495 private->region[private->num_regions].size = size; 496 private->region[private->num_regions].flags = flags; 497 private->region[private->num_regions].data = data; 498 499 private->num_regions++; 500 501 return 0; 502 } 503 504 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 505 { 506 int i; 507 508 for (i = 0; i < private->num_regions; i++) 509 private->region[i].ops->release(private, &private->region[i]); 510 private->num_regions = 0; 511 kfree(private->region); 512 private->region = NULL; 513 } 514 515 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 516 unsigned int cmd, 517 unsigned long arg) 518 { 519 struct vfio_ccw_private *private = 520 container_of(vdev, struct vfio_ccw_private, vdev); 521 int ret = 0; 522 unsigned long minsz; 523 524 switch (cmd) { 525 case VFIO_DEVICE_GET_INFO: 526 { 527 struct vfio_device_info info; 528 529 minsz = offsetofend(struct vfio_device_info, num_irqs); 530 531 if (copy_from_user(&info, (void __user *)arg, minsz)) 532 return -EFAULT; 533 534 if (info.argsz < minsz) 535 return -EINVAL; 536 537 ret = vfio_ccw_mdev_get_device_info(private, &info); 538 if (ret) 539 return ret; 540 541 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 542 } 543 case VFIO_DEVICE_GET_REGION_INFO: 544 { 545 struct vfio_region_info info; 546 547 minsz = offsetofend(struct vfio_region_info, offset); 548 549 if (copy_from_user(&info, (void __user *)arg, minsz)) 550 return -EFAULT; 551 552 if (info.argsz < minsz) 553 return -EINVAL; 554 555 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 556 if (ret) 557 return ret; 558 559 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 560 } 561 case VFIO_DEVICE_GET_IRQ_INFO: 562 { 563 struct vfio_irq_info info; 564 565 minsz = offsetofend(struct vfio_irq_info, count); 566 567 if (copy_from_user(&info, (void __user *)arg, minsz)) 568 return -EFAULT; 569 570 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 571 return -EINVAL; 572 573 ret = vfio_ccw_mdev_get_irq_info(&info); 574 if (ret) 575 return ret; 576 577 if (info.count == -1) 578 return -EINVAL; 579 580 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 581 } 582 case VFIO_DEVICE_SET_IRQS: 583 { 584 struct vfio_irq_set hdr; 585 size_t data_size; 586 void __user *data; 587 588 minsz = offsetofend(struct vfio_irq_set, count); 589 590 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 591 return -EFAULT; 592 593 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 594 VFIO_CCW_NUM_IRQS, 595 &data_size); 596 if (ret) 597 return ret; 598 599 data = (void __user *)(arg + minsz); 600 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 601 data); 602 } 603 case VFIO_DEVICE_RESET: 604 return vfio_ccw_mdev_reset(private); 605 default: 606 return -ENOTTY; 607 } 608 } 609 610 /* Request removal of the device*/ 611 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 612 { 613 struct vfio_ccw_private *private = 614 container_of(vdev, struct vfio_ccw_private, vdev); 615 struct device *dev = vdev->dev; 616 617 if (private->req_trigger) { 618 if (!(count % 10)) 619 dev_notice_ratelimited(dev, 620 "Relaying device request to user (#%u)\n", 621 count); 622 623 eventfd_signal(private->req_trigger, 1); 624 } else if (count == 0) { 625 dev_notice(dev, 626 "No device request channel registered, blocked until released by user\n"); 627 } 628 } 629 630 static const struct vfio_device_ops vfio_ccw_dev_ops = { 631 .init = vfio_ccw_mdev_init_dev, 632 .release = vfio_ccw_mdev_release_dev, 633 .open_device = vfio_ccw_mdev_open_device, 634 .close_device = vfio_ccw_mdev_close_device, 635 .read = vfio_ccw_mdev_read, 636 .write = vfio_ccw_mdev_write, 637 .ioctl = vfio_ccw_mdev_ioctl, 638 .request = vfio_ccw_mdev_request, 639 .dma_unmap = vfio_ccw_dma_unmap, 640 }; 641 642 struct mdev_driver vfio_ccw_mdev_driver = { 643 .driver = { 644 .name = "vfio_ccw_mdev", 645 .owner = THIS_MODULE, 646 .mod_name = KBUILD_MODNAME, 647 }, 648 .probe = vfio_ccw_mdev_probe, 649 .remove = vfio_ccw_mdev_remove, 650 .supported_type_groups = mdev_type_groups, 651 }; 652