1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/nospec.h> 15 #include <linux/slab.h> 16 17 #include "vfio_ccw_private.h" 18 19 static const struct vfio_device_ops vfio_ccw_dev_ops; 20 21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 22 { 23 /* 24 * If the FSM state is seen as Not Operational after closing 25 * and re-opening the mdev, return an error. 26 */ 27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 29 if (private->state == VFIO_CCW_STATE_NOT_OPER) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 36 { 37 struct vfio_ccw_private *private = 38 container_of(vdev, struct vfio_ccw_private, vdev); 39 40 /* Drivers MUST unpin pages in response to an invalidation. */ 41 if (!cp_iova_pinned(&private->cp, iova, length)) 42 return; 43 44 vfio_ccw_mdev_reset(private); 45 } 46 47 static ssize_t name_show(struct mdev_type *mtype, 48 struct mdev_type_attribute *attr, char *buf) 49 { 50 return sprintf(buf, "I/O subchannel (Non-QDIO)\n"); 51 } 52 static MDEV_TYPE_ATTR_RO(name); 53 54 static ssize_t device_api_show(struct mdev_type *mtype, 55 struct mdev_type_attribute *attr, char *buf) 56 { 57 return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING); 58 } 59 static MDEV_TYPE_ATTR_RO(device_api); 60 61 static ssize_t available_instances_show(struct mdev_type *mtype, 62 struct mdev_type_attribute *attr, 63 char *buf) 64 { 65 struct vfio_ccw_private *private = 66 dev_get_drvdata(mtype_get_parent_dev(mtype)); 67 68 return sprintf(buf, "%d\n", atomic_read(&private->avail)); 69 } 70 static MDEV_TYPE_ATTR_RO(available_instances); 71 72 static const struct attribute *mdev_types_attrs[] = { 73 &mdev_type_attr_name.attr, 74 &mdev_type_attr_device_api.attr, 75 &mdev_type_attr_available_instances.attr, 76 NULL, 77 }; 78 79 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 80 { 81 struct vfio_ccw_private *private = 82 container_of(vdev, struct vfio_ccw_private, vdev); 83 84 init_completion(&private->release_comp); 85 return 0; 86 } 87 88 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 89 { 90 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 91 int ret; 92 93 if (private->state == VFIO_CCW_STATE_NOT_OPER) 94 return -ENODEV; 95 96 if (atomic_dec_if_positive(&private->avail) < 0) 97 return -EPERM; 98 99 ret = vfio_init_device(&private->vdev, &mdev->dev, &vfio_ccw_dev_ops); 100 if (ret) 101 return ret; 102 103 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 104 private->sch->schid.cssid, 105 private->sch->schid.ssid, 106 private->sch->schid.sch_no); 107 108 ret = vfio_register_emulated_iommu_dev(&private->vdev); 109 if (ret) 110 goto err_put_vdev; 111 dev_set_drvdata(&mdev->dev, private); 112 return 0; 113 114 err_put_vdev: 115 vfio_put_device(&private->vdev); 116 atomic_inc(&private->avail); 117 return ret; 118 } 119 120 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 121 { 122 struct vfio_ccw_private *private = 123 container_of(vdev, struct vfio_ccw_private, vdev); 124 125 /* 126 * We cannot free vfio_ccw_private here because it includes 127 * parent info which must be free'ed by css driver. 128 * 129 * Use a workaround by memset'ing the core device part and 130 * then notifying the remove path that all active references 131 * to this device have been released. 132 */ 133 memset(vdev, 0, sizeof(*vdev)); 134 complete(&private->release_comp); 135 } 136 137 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 138 { 139 struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); 140 141 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 142 private->sch->schid.cssid, 143 private->sch->schid.ssid, 144 private->sch->schid.sch_no); 145 146 vfio_unregister_group_dev(&private->vdev); 147 148 vfio_put_device(&private->vdev); 149 /* 150 * Wait for all active references on mdev are released so it 151 * is safe to defer kfree() to a later point. 152 * 153 * TODO: the clean fix is to split parent/mdev info from ccw 154 * private structure so each can be managed in its own life 155 * cycle. 156 */ 157 wait_for_completion(&private->release_comp); 158 159 atomic_inc(&private->avail); 160 } 161 162 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 163 { 164 struct vfio_ccw_private *private = 165 container_of(vdev, struct vfio_ccw_private, vdev); 166 int ret; 167 168 /* Device cannot simply be opened again from this state */ 169 if (private->state == VFIO_CCW_STATE_NOT_OPER) 170 return -EINVAL; 171 172 ret = vfio_ccw_register_async_dev_regions(private); 173 if (ret) 174 return ret; 175 176 ret = vfio_ccw_register_schib_dev_regions(private); 177 if (ret) 178 goto out_unregister; 179 180 ret = vfio_ccw_register_crw_dev_regions(private); 181 if (ret) 182 goto out_unregister; 183 184 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 185 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 186 ret = -EINVAL; 187 goto out_unregister; 188 } 189 190 return ret; 191 192 out_unregister: 193 vfio_ccw_unregister_dev_regions(private); 194 return ret; 195 } 196 197 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 198 { 199 struct vfio_ccw_private *private = 200 container_of(vdev, struct vfio_ccw_private, vdev); 201 202 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 203 vfio_ccw_unregister_dev_regions(private); 204 } 205 206 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 207 char __user *buf, size_t count, 208 loff_t *ppos) 209 { 210 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 211 struct ccw_io_region *region; 212 int ret; 213 214 if (pos + count > sizeof(*region)) 215 return -EINVAL; 216 217 mutex_lock(&private->io_mutex); 218 region = private->io_region; 219 if (copy_to_user(buf, (void *)region + pos, count)) 220 ret = -EFAULT; 221 else 222 ret = count; 223 mutex_unlock(&private->io_mutex); 224 return ret; 225 } 226 227 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 228 char __user *buf, 229 size_t count, 230 loff_t *ppos) 231 { 232 struct vfio_ccw_private *private = 233 container_of(vdev, struct vfio_ccw_private, vdev); 234 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 235 236 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 237 return -EINVAL; 238 239 switch (index) { 240 case VFIO_CCW_CONFIG_REGION_INDEX: 241 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 242 default: 243 index -= VFIO_CCW_NUM_REGIONS; 244 return private->region[index].ops->read(private, buf, count, 245 ppos); 246 } 247 248 return -EINVAL; 249 } 250 251 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 252 const char __user *buf, 253 size_t count, loff_t *ppos) 254 { 255 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 256 struct ccw_io_region *region; 257 int ret; 258 259 if (pos + count > sizeof(*region)) 260 return -EINVAL; 261 262 if (!mutex_trylock(&private->io_mutex)) 263 return -EAGAIN; 264 265 region = private->io_region; 266 if (copy_from_user((void *)region + pos, buf, count)) { 267 ret = -EFAULT; 268 goto out_unlock; 269 } 270 271 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 272 ret = (region->ret_code != 0) ? region->ret_code : count; 273 274 out_unlock: 275 mutex_unlock(&private->io_mutex); 276 return ret; 277 } 278 279 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 280 const char __user *buf, 281 size_t count, 282 loff_t *ppos) 283 { 284 struct vfio_ccw_private *private = 285 container_of(vdev, struct vfio_ccw_private, vdev); 286 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 287 288 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 289 return -EINVAL; 290 291 switch (index) { 292 case VFIO_CCW_CONFIG_REGION_INDEX: 293 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 294 default: 295 index -= VFIO_CCW_NUM_REGIONS; 296 return private->region[index].ops->write(private, buf, count, 297 ppos); 298 } 299 300 return -EINVAL; 301 } 302 303 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 304 struct vfio_device_info *info) 305 { 306 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 307 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 308 info->num_irqs = VFIO_CCW_NUM_IRQS; 309 310 return 0; 311 } 312 313 static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, 314 struct vfio_region_info *info, 315 unsigned long arg) 316 { 317 int i; 318 319 switch (info->index) { 320 case VFIO_CCW_CONFIG_REGION_INDEX: 321 info->offset = 0; 322 info->size = sizeof(struct ccw_io_region); 323 info->flags = VFIO_REGION_INFO_FLAG_READ 324 | VFIO_REGION_INFO_FLAG_WRITE; 325 return 0; 326 default: /* all other regions are handled via capability chain */ 327 { 328 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 329 struct vfio_region_info_cap_type cap_type = { 330 .header.id = VFIO_REGION_INFO_CAP_TYPE, 331 .header.version = 1 }; 332 int ret; 333 334 if (info->index >= 335 VFIO_CCW_NUM_REGIONS + private->num_regions) 336 return -EINVAL; 337 338 info->index = array_index_nospec(info->index, 339 VFIO_CCW_NUM_REGIONS + 340 private->num_regions); 341 342 i = info->index - VFIO_CCW_NUM_REGIONS; 343 344 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 345 info->size = private->region[i].size; 346 info->flags = private->region[i].flags; 347 348 cap_type.type = private->region[i].type; 349 cap_type.subtype = private->region[i].subtype; 350 351 ret = vfio_info_add_capability(&caps, &cap_type.header, 352 sizeof(cap_type)); 353 if (ret) 354 return ret; 355 356 info->flags |= VFIO_REGION_INFO_FLAG_CAPS; 357 if (info->argsz < sizeof(*info) + caps.size) { 358 info->argsz = sizeof(*info) + caps.size; 359 info->cap_offset = 0; 360 } else { 361 vfio_info_cap_shift(&caps, sizeof(*info)); 362 if (copy_to_user((void __user *)arg + sizeof(*info), 363 caps.buf, caps.size)) { 364 kfree(caps.buf); 365 return -EFAULT; 366 } 367 info->cap_offset = sizeof(*info); 368 } 369 370 kfree(caps.buf); 371 372 } 373 } 374 return 0; 375 } 376 377 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 378 { 379 switch (info->index) { 380 case VFIO_CCW_IO_IRQ_INDEX: 381 case VFIO_CCW_CRW_IRQ_INDEX: 382 case VFIO_CCW_REQ_IRQ_INDEX: 383 info->count = 1; 384 info->flags = VFIO_IRQ_INFO_EVENTFD; 385 break; 386 default: 387 return -EINVAL; 388 } 389 390 return 0; 391 } 392 393 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 394 uint32_t flags, 395 uint32_t index, 396 void __user *data) 397 { 398 struct eventfd_ctx **ctx; 399 400 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 401 return -EINVAL; 402 403 switch (index) { 404 case VFIO_CCW_IO_IRQ_INDEX: 405 ctx = &private->io_trigger; 406 break; 407 case VFIO_CCW_CRW_IRQ_INDEX: 408 ctx = &private->crw_trigger; 409 break; 410 case VFIO_CCW_REQ_IRQ_INDEX: 411 ctx = &private->req_trigger; 412 break; 413 default: 414 return -EINVAL; 415 } 416 417 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 418 case VFIO_IRQ_SET_DATA_NONE: 419 { 420 if (*ctx) 421 eventfd_signal(*ctx, 1); 422 return 0; 423 } 424 case VFIO_IRQ_SET_DATA_BOOL: 425 { 426 uint8_t trigger; 427 428 if (get_user(trigger, (uint8_t __user *)data)) 429 return -EFAULT; 430 431 if (trigger && *ctx) 432 eventfd_signal(*ctx, 1); 433 return 0; 434 } 435 case VFIO_IRQ_SET_DATA_EVENTFD: 436 { 437 int32_t fd; 438 439 if (get_user(fd, (int32_t __user *)data)) 440 return -EFAULT; 441 442 if (fd == -1) { 443 if (*ctx) 444 eventfd_ctx_put(*ctx); 445 *ctx = NULL; 446 } else if (fd >= 0) { 447 struct eventfd_ctx *efdctx; 448 449 efdctx = eventfd_ctx_fdget(fd); 450 if (IS_ERR(efdctx)) 451 return PTR_ERR(efdctx); 452 453 if (*ctx) 454 eventfd_ctx_put(*ctx); 455 456 *ctx = efdctx; 457 } else 458 return -EINVAL; 459 460 return 0; 461 } 462 default: 463 return -EINVAL; 464 } 465 } 466 467 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 468 unsigned int subtype, 469 const struct vfio_ccw_regops *ops, 470 size_t size, u32 flags, void *data) 471 { 472 struct vfio_ccw_region *region; 473 474 region = krealloc(private->region, 475 (private->num_regions + 1) * sizeof(*region), 476 GFP_KERNEL); 477 if (!region) 478 return -ENOMEM; 479 480 private->region = region; 481 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 482 private->region[private->num_regions].subtype = subtype; 483 private->region[private->num_regions].ops = ops; 484 private->region[private->num_regions].size = size; 485 private->region[private->num_regions].flags = flags; 486 private->region[private->num_regions].data = data; 487 488 private->num_regions++; 489 490 return 0; 491 } 492 493 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 494 { 495 int i; 496 497 for (i = 0; i < private->num_regions; i++) 498 private->region[i].ops->release(private, &private->region[i]); 499 private->num_regions = 0; 500 kfree(private->region); 501 private->region = NULL; 502 } 503 504 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 505 unsigned int cmd, 506 unsigned long arg) 507 { 508 struct vfio_ccw_private *private = 509 container_of(vdev, struct vfio_ccw_private, vdev); 510 int ret = 0; 511 unsigned long minsz; 512 513 switch (cmd) { 514 case VFIO_DEVICE_GET_INFO: 515 { 516 struct vfio_device_info info; 517 518 minsz = offsetofend(struct vfio_device_info, num_irqs); 519 520 if (copy_from_user(&info, (void __user *)arg, minsz)) 521 return -EFAULT; 522 523 if (info.argsz < minsz) 524 return -EINVAL; 525 526 ret = vfio_ccw_mdev_get_device_info(private, &info); 527 if (ret) 528 return ret; 529 530 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 531 } 532 case VFIO_DEVICE_GET_REGION_INFO: 533 { 534 struct vfio_region_info info; 535 536 minsz = offsetofend(struct vfio_region_info, offset); 537 538 if (copy_from_user(&info, (void __user *)arg, minsz)) 539 return -EFAULT; 540 541 if (info.argsz < minsz) 542 return -EINVAL; 543 544 ret = vfio_ccw_mdev_get_region_info(private, &info, arg); 545 if (ret) 546 return ret; 547 548 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 549 } 550 case VFIO_DEVICE_GET_IRQ_INFO: 551 { 552 struct vfio_irq_info info; 553 554 minsz = offsetofend(struct vfio_irq_info, count); 555 556 if (copy_from_user(&info, (void __user *)arg, minsz)) 557 return -EFAULT; 558 559 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 560 return -EINVAL; 561 562 ret = vfio_ccw_mdev_get_irq_info(&info); 563 if (ret) 564 return ret; 565 566 if (info.count == -1) 567 return -EINVAL; 568 569 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 570 } 571 case VFIO_DEVICE_SET_IRQS: 572 { 573 struct vfio_irq_set hdr; 574 size_t data_size; 575 void __user *data; 576 577 minsz = offsetofend(struct vfio_irq_set, count); 578 579 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 580 return -EFAULT; 581 582 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 583 VFIO_CCW_NUM_IRQS, 584 &data_size); 585 if (ret) 586 return ret; 587 588 data = (void __user *)(arg + minsz); 589 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 590 data); 591 } 592 case VFIO_DEVICE_RESET: 593 return vfio_ccw_mdev_reset(private); 594 default: 595 return -ENOTTY; 596 } 597 } 598 599 /* Request removal of the device*/ 600 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 601 { 602 struct vfio_ccw_private *private = 603 container_of(vdev, struct vfio_ccw_private, vdev); 604 struct device *dev = vdev->dev; 605 606 if (private->req_trigger) { 607 if (!(count % 10)) 608 dev_notice_ratelimited(dev, 609 "Relaying device request to user (#%u)\n", 610 count); 611 612 eventfd_signal(private->req_trigger, 1); 613 } else if (count == 0) { 614 dev_notice(dev, 615 "No device request channel registered, blocked until released by user\n"); 616 } 617 } 618 619 static const struct vfio_device_ops vfio_ccw_dev_ops = { 620 .init = vfio_ccw_mdev_init_dev, 621 .release = vfio_ccw_mdev_release_dev, 622 .open_device = vfio_ccw_mdev_open_device, 623 .close_device = vfio_ccw_mdev_close_device, 624 .read = vfio_ccw_mdev_read, 625 .write = vfio_ccw_mdev_write, 626 .ioctl = vfio_ccw_mdev_ioctl, 627 .request = vfio_ccw_mdev_request, 628 .dma_unmap = vfio_ccw_dma_unmap, 629 }; 630 631 struct mdev_driver vfio_ccw_mdev_driver = { 632 .driver = { 633 .name = "vfio_ccw_mdev", 634 .owner = THIS_MODULE, 635 .mod_name = KBUILD_MODNAME, 636 }, 637 .probe = vfio_ccw_mdev_probe, 638 .remove = vfio_ccw_mdev_remove, 639 .types_attrs = mdev_types_attrs, 640 }; 641