1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/nospec.h> 15 #include <linux/slab.h> 16 17 #include "vfio_ccw_private.h" 18 19 static const struct vfio_device_ops vfio_ccw_dev_ops; 20 21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 22 { 23 /* 24 * If the FSM state is seen as Not Operational after closing 25 * and re-opening the mdev, return an error. 26 */ 27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 29 if (private->state == VFIO_CCW_STATE_NOT_OPER) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 36 { 37 struct vfio_ccw_private *private = 38 container_of(vdev, struct vfio_ccw_private, vdev); 39 40 /* Drivers MUST unpin pages in response to an invalidation. */ 41 if (!cp_iova_pinned(&private->cp, iova, length)) 42 return; 43 44 vfio_ccw_mdev_reset(private); 45 } 46 47 static ssize_t name_show(struct mdev_type *mtype, 48 struct mdev_type_attribute *attr, char *buf) 49 { 50 return sprintf(buf, "I/O subchannel (Non-QDIO)\n"); 51 } 52 static MDEV_TYPE_ATTR_RO(name); 53 54 static ssize_t device_api_show(struct mdev_type *mtype, 55 struct mdev_type_attribute *attr, char *buf) 56 { 57 return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING); 58 } 59 static MDEV_TYPE_ATTR_RO(device_api); 60 61 static ssize_t available_instances_show(struct mdev_type *mtype, 62 struct mdev_type_attribute *attr, 63 char *buf) 64 { 65 struct vfio_ccw_private *private = 66 dev_get_drvdata(mtype_get_parent_dev(mtype)); 67 68 return sprintf(buf, "%d\n", atomic_read(&private->avail)); 69 } 70 static MDEV_TYPE_ATTR_RO(available_instances); 71 72 static struct attribute *mdev_types_attrs[] = { 73 &mdev_type_attr_name.attr, 74 &mdev_type_attr_device_api.attr, 75 &mdev_type_attr_available_instances.attr, 76 NULL, 77 }; 78 79 static struct attribute_group mdev_type_group = { 80 .name = "io", 81 .attrs = mdev_types_attrs, 82 }; 83 84 static struct attribute_group *mdev_type_groups[] = { 85 &mdev_type_group, 86 NULL, 87 }; 88 89 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 90 { 91 struct vfio_ccw_private *private = 92 container_of(vdev, struct vfio_ccw_private, vdev); 93 94 init_completion(&private->release_comp); 95 return 0; 96 } 97 98 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 99 { 100 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 101 int ret; 102 103 if (private->state == VFIO_CCW_STATE_NOT_OPER) 104 return -ENODEV; 105 106 if (atomic_dec_if_positive(&private->avail) < 0) 107 return -EPERM; 108 109 ret = vfio_init_device(&private->vdev, &mdev->dev, &vfio_ccw_dev_ops); 110 if (ret) 111 return ret; 112 113 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 114 private->sch->schid.cssid, 115 private->sch->schid.ssid, 116 private->sch->schid.sch_no); 117 118 ret = vfio_register_emulated_iommu_dev(&private->vdev); 119 if (ret) 120 goto err_put_vdev; 121 dev_set_drvdata(&mdev->dev, private); 122 return 0; 123 124 err_put_vdev: 125 vfio_put_device(&private->vdev); 126 atomic_inc(&private->avail); 127 return ret; 128 } 129 130 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 131 { 132 struct vfio_ccw_private *private = 133 container_of(vdev, struct vfio_ccw_private, vdev); 134 135 /* 136 * We cannot free vfio_ccw_private here because it includes 137 * parent info which must be free'ed by css driver. 138 * 139 * Use a workaround by memset'ing the core device part and 140 * then notifying the remove path that all active references 141 * to this device have been released. 142 */ 143 memset(vdev, 0, sizeof(*vdev)); 144 complete(&private->release_comp); 145 } 146 147 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 148 { 149 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 150 151 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 152 private->sch->schid.cssid, 153 private->sch->schid.ssid, 154 private->sch->schid.sch_no); 155 156 vfio_unregister_group_dev(&private->vdev); 157 158 vfio_put_device(&private->vdev); 159 /* 160 * Wait for all active references on mdev are released so it 161 * is safe to defer kfree() to a later point. 162 * 163 * TODO: the clean fix is to split parent/mdev info from ccw 164 * private structure so each can be managed in its own life 165 * cycle. 166 */ 167 wait_for_completion(&private->release_comp); 168 169 atomic_inc(&private->avail); 170 } 171 172 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 173 { 174 struct vfio_ccw_private *private = 175 container_of(vdev, struct vfio_ccw_private, vdev); 176 int ret; 177 178 /* Device cannot simply be opened again from this state */ 179 if (private->state == VFIO_CCW_STATE_NOT_OPER) 180 return -EINVAL; 181 182 ret = vfio_ccw_register_async_dev_regions(private); 183 if (ret) 184 return ret; 185 186 ret = vfio_ccw_register_schib_dev_regions(private); 187 if (ret) 188 goto out_unregister; 189 190 ret = vfio_ccw_register_crw_dev_regions(private); 191 if (ret) 192 goto out_unregister; 193 194 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 195 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 196 ret = -EINVAL; 197 goto out_unregister; 198 } 199 200 return ret; 201 202 out_unregister: 203 vfio_ccw_unregister_dev_regions(private); 204 return ret; 205 } 206 207 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 208 { 209 struct vfio_ccw_private *private = 210 container_of(vdev, struct vfio_ccw_private, vdev); 211 212 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 213 vfio_ccw_unregister_dev_regions(private); 214 } 215 216 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 217 char __user *buf, size_t count, 218 loff_t *ppos) 219 { 220 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 221 struct ccw_io_region *region; 222 int ret; 223 224 if (pos + count > sizeof(*region)) 225 return -EINVAL; 226 227 mutex_lock(&private->io_mutex); 228 region = private->io_region; 229 if (copy_to_user(buf, (void *)region + pos, count)) 230 ret = -EFAULT; 231 else 232 ret = count; 233 mutex_unlock(&private->io_mutex); 234 return ret; 235 } 236 237 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 238 char __user *buf, 239 size_t count, 240 loff_t *ppos) 241 { 242 struct vfio_ccw_private *private = 243 container_of(vdev, struct vfio_ccw_private, vdev); 244 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 245 246 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 247 return -EINVAL; 248 249 switch (index) { 250 case VFIO_CCW_CONFIG_REGION_INDEX: 251 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 252 default: 253 index -= VFIO_CCW_NUM_REGIONS; 254 return private->region[index].ops->read(private, buf, count, 255 ppos); 256 } 257 258 return -EINVAL; 259 } 260 261 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 262 const char __user *buf, 263 size_t count, loff_t *ppos) 264 { 265 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 266 struct ccw_io_region *region; 267 int ret; 268 269 if (pos + count > sizeof(*region)) 270 return -EINVAL; 271 272 if (!mutex_trylock(&private->io_mutex)) 273 return -EAGAIN; 274 275 region = private->io_region; 276 if (copy_from_user((void *)region + pos, buf, count)) { 277 ret = -EFAULT; 278 goto out_unlock; 279 } 280 281 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 282 ret = (region->ret_code != 0) ? region->ret_code : count; 283 284 out_unlock: 285 mutex_unlock(&private->io_mutex); 286 return ret; 287 } 288 289 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 290 const char __user *buf, 291 size_t count, 292 loff_t *ppos) 293 { 294 struct vfio_ccw_private *private = 295 container_of(vdev, struct vfio_ccw_private, vdev); 296 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 297 298 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 299 return -EINVAL; 300 301 switch (index) { 302 case VFIO_CCW_CONFIG_REGION_INDEX: 303 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 304 default: 305 index -= VFIO_CCW_NUM_REGIONS; 306 return private->region[index].ops->write(private, buf, count, 307 ppos); 308 } 309 310 return -EINVAL; 311 } 312 313 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 314 struct vfio_device_info *info) 315 { 316 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 317 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 318 info->num_irqs = VFIO_CCW_NUM_IRQS; 319 320 return 0; 321 } 322 323 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 324 struct vfio_region_info *info, 325 unsigned long arg) 326 { 327 int i; 328 329 switch (info->index) { 330 case VFIO_CCW_CONFIG_REGION_INDEX: 331 info->offset = 0; 332 info->size = sizeof(struct ccw_io_region); 333 info->flags = VFIO_REGION_INFO_FLAG_READ 334 | VFIO_REGION_INFO_FLAG_WRITE; 335 return 0; 336 default: /* all other regions are handled via capability chain */ 337 { 338 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 339 struct vfio_region_info_cap_type cap_type = { 340 .header.id = VFIO_REGION_INFO_CAP_TYPE, 341 .header.version = 1 }; 342 int ret; 343 344 if (info->index >= 345 VFIO_CCW_NUM_REGIONS + private->num_regions) 346 return -EINVAL; 347 348 info->index = array_index_nospec(info->index, 349 VFIO_CCW_NUM_REGIONS + 350 private->num_regions); 351 352 i = info->index - VFIO_CCW_NUM_REGIONS; 353 354 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 355 info->size = private->region[i].size; 356 info->flags = private->region[i].flags; 357 358 cap_type.type = private->region[i].type; 359 cap_type.subtype = private->region[i].subtype; 360 361 ret = vfio_info_add_capability(&caps, &cap_type.header, 362 sizeof(cap_type)); 363 if (ret) 364 return ret; 365 366 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 367 if (info->argsz < sizeof(*info) + caps.size) { 368 info->argsz = sizeof(*info) + caps.size; 369 info->cap_offset = 0; 370 } else { 371 vfio_info_cap_shift(&caps, sizeof(*info)); 372 if (copy_to_user((void __user *)arg + sizeof(*info), 373 caps.buf, caps.size)) { 374 kfree(caps.buf); 375 return -EFAULT; 376 } 377 info->cap_offset = sizeof(*info); 378 } 379 380 kfree(caps.buf); 381 382 } 383 } 384 return 0; 385 } 386 387 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 388 { 389 switch (info->index) { 390 case VFIO_CCW_IO_IRQ_INDEX: 391 case VFIO_CCW_CRW_IRQ_INDEX: 392 case VFIO_CCW_REQ_IRQ_INDEX: 393 info->count = 1; 394 info->flags = VFIO_IRQ_INFO_EVENTFD; 395 break; 396 default: 397 return -EINVAL; 398 } 399 400 return 0; 401 } 402 403 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 404 uint32_t flags, 405 uint32_t index, 406 void __user *data) 407 { 408 struct eventfd_ctx **ctx; 409 410 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 411 return -EINVAL; 412 413 switch (index) { 414 case VFIO_CCW_IO_IRQ_INDEX: 415 ctx = &private->io_trigger; 416 break; 417 case VFIO_CCW_CRW_IRQ_INDEX: 418 ctx = &private->crw_trigger; 419 break; 420 case VFIO_CCW_REQ_IRQ_INDEX: 421 ctx = &private->req_trigger; 422 break; 423 default: 424 return -EINVAL; 425 } 426 427 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 428 case VFIO_IRQ_SET_DATA_NONE: 429 { 430 if (*ctx) 431 eventfd_signal(*ctx, 1); 432 return 0; 433 } 434 case VFIO_IRQ_SET_DATA_BOOL: 435 { 436 uint8_t trigger; 437 438 if (get_user(trigger, (uint8_t __user *)data)) 439 return -EFAULT; 440 441 if (trigger && *ctx) 442 eventfd_signal(*ctx, 1); 443 return 0; 444 } 445 case VFIO_IRQ_SET_DATA_EVENTFD: 446 { 447 int32_t fd; 448 449 if (get_user(fd, (int32_t __user *)data)) 450 return -EFAULT; 451 452 if (fd == -1) { 453 if (*ctx) 454 eventfd_ctx_put(*ctx); 455 *ctx = NULL; 456 } else if (fd >= 0) { 457 struct eventfd_ctx *efdctx; 458 459 efdctx = eventfd_ctx_fdget(fd); 460 if (IS_ERR(efdctx)) 461 return PTR_ERR(efdctx); 462 463 if (*ctx) 464 eventfd_ctx_put(*ctx); 465 466 *ctx = efdctx; 467 } else 468 return -EINVAL; 469 470 return 0; 471 } 472 default: 473 return -EINVAL; 474 } 475 } 476 477 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 478 unsigned int subtype, 479 const struct vfio_ccw_regops *ops, 480 size_t size, u32 flags, void *data) 481 { 482 struct vfio_ccw_region *region; 483 484 region = krealloc(private->region, 485 (private->num_regions + 1) * sizeof(*region), 486 GFP_KERNEL); 487 if (!region) 488 return -ENOMEM; 489 490 private->region = region; 491 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 492 private->region[private->num_regions].subtype = subtype; 493 private->region[private->num_regions].ops = ops; 494 private->region[private->num_regions].size = size; 495 private->region[private->num_regions].flags = flags; 496 private->region[private->num_regions].data = data; 497 498 private->num_regions++; 499 500 return 0; 501 } 502 503 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 504 { 505 int i; 506 507 for (i = 0; i < private->num_regions; i++) 508 private->region[i].ops->release(private, &private->region[i]); 509 private->num_regions = 0; 510 kfree(private->region); 511 private->region = NULL; 512 } 513 514 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 515 unsigned int cmd, 516 unsigned long arg) 517 { 518 struct vfio_ccw_private *private = 519 container_of(vdev, struct vfio_ccw_private, vdev); 520 int ret = 0; 521 unsigned long minsz; 522 523 switch (cmd) { 524 case VFIO_DEVICE_GET_INFO: 525 { 526 struct vfio_device_info info; 527 528 minsz = offsetofend(struct vfio_device_info, num_irqs); 529 530 if (copy_from_user(&info, (void __user *)arg, minsz)) 531 return -EFAULT; 532 533 if (info.argsz < minsz) 534 return -EINVAL; 535 536 ret = vfio_ccw_mdev_get_device_info(private, &info); 537 if (ret) 538 return ret; 539 540 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 541 } 542 case VFIO_DEVICE_GET_REGION_INFO: 543 { 544 struct vfio_region_info info; 545 546 minsz = offsetofend(struct vfio_region_info, offset); 547 548 if (copy_from_user(&info, (void __user *)arg, minsz)) 549 return -EFAULT; 550 551 if (info.argsz < minsz) 552 return -EINVAL; 553 554 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 555 if (ret) 556 return ret; 557 558 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 559 } 560 case VFIO_DEVICE_GET_IRQ_INFO: 561 { 562 struct vfio_irq_info info; 563 564 minsz = offsetofend(struct vfio_irq_info, count); 565 566 if (copy_from_user(&info, (void __user *)arg, minsz)) 567 return -EFAULT; 568 569 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 570 return -EINVAL; 571 572 ret = vfio_ccw_mdev_get_irq_info(&info); 573 if (ret) 574 return ret; 575 576 if (info.count == -1) 577 return -EINVAL; 578 579 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 580 } 581 case VFIO_DEVICE_SET_IRQS: 582 { 583 struct vfio_irq_set hdr; 584 size_t data_size; 585 void __user *data; 586 587 minsz = offsetofend(struct vfio_irq_set, count); 588 589 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 590 return -EFAULT; 591 592 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 593 VFIO_CCW_NUM_IRQS, 594 &data_size); 595 if (ret) 596 return ret; 597 598 data = (void __user *)(arg + minsz); 599 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 600 data); 601 } 602 case VFIO_DEVICE_RESET: 603 return vfio_ccw_mdev_reset(private); 604 default: 605 return -ENOTTY; 606 } 607 } 608 609 /* Request removal of the device*/ 610 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 611 { 612 struct vfio_ccw_private *private = 613 container_of(vdev, struct vfio_ccw_private, vdev); 614 struct device *dev = vdev->dev; 615 616 if (private->req_trigger) { 617 if (!(count % 10)) 618 dev_notice_ratelimited(dev, 619 "Relaying device request to user (#%u)\n", 620 count); 621 622 eventfd_signal(private->req_trigger, 1); 623 } else if (count == 0) { 624 dev_notice(dev, 625 "No device request channel registered, blocked until released by user\n"); 626 } 627 } 628 629 static const struct vfio_device_ops vfio_ccw_dev_ops = { 630 .init = vfio_ccw_mdev_init_dev, 631 .release = vfio_ccw_mdev_release_dev, 632 .open_device = vfio_ccw_mdev_open_device, 633 .close_device = vfio_ccw_mdev_close_device, 634 .read = vfio_ccw_mdev_read, 635 .write = vfio_ccw_mdev_write, 636 .ioctl = vfio_ccw_mdev_ioctl, 637 .request = vfio_ccw_mdev_request, 638 .dma_unmap = vfio_ccw_dma_unmap, 639 }; 640 641 struct mdev_driver vfio_ccw_mdev_driver = { 642 .driver = { 643 .name = "vfio_ccw_mdev", 644 .owner = THIS_MODULE, 645 .mod_name = KBUILD_MODNAME, 646 }, 647 .probe = vfio_ccw_mdev_probe, 648 .remove = vfio_ccw_mdev_remove, 649 .supported_type_groups = mdev_type_groups, 650 }; 651