1 /* 2 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 3 * Author: Alex Williamson <alex.williamson@redhat.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * Derived from original vfio: 10 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 11 * Author: Tom Lyon, pugs@cisco.com 12 */ 13 14 #include <linux/device.h> 15 #include <linux/eventfd.h> 16 #include <linux/file.h> 17 #include <linux/interrupt.h> 18 #include <linux/iommu.h> 19 #include <linux/module.h> 20 #include <linux/mutex.h> 21 #include <linux/notifier.h> 22 #include <linux/pci.h> 23 #include <linux/pm_runtime.h> 24 #include <linux/slab.h> 25 #include <linux/types.h> 26 #include <linux/uaccess.h> 27 #include <linux/vfio.h> 28 29 #include "vfio_pci_private.h" 30 31 #define DRIVER_VERSION "0.2" 32 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 33 #define DRIVER_DESC "VFIO PCI - User Level meta-driver" 34 35 static bool nointxmask; 36 module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR); 37 MODULE_PARM_DESC(nointxmask, 38 "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag."); 39 40 static int vfio_pci_enable(struct vfio_pci_device *vdev) 41 { 42 struct pci_dev *pdev = vdev->pdev; 43 int ret; 44 u16 cmd; 45 u8 msix_pos; 46 47 ret = pci_enable_device(pdev); 48 if (ret) 49 return ret; 50 51 vdev->reset_works = (pci_reset_function(pdev) == 0); 52 pci_save_state(pdev); 53 vdev->pci_saved_state = pci_store_saved_state(pdev); 54 if (!vdev->pci_saved_state) 55 pr_debug("%s: Couldn't store %s saved state\n", 56 __func__, dev_name(&pdev->dev)); 57 58 ret = vfio_config_init(vdev); 59 if (ret) { 60 pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state); 61 pci_disable_device(pdev); 62 return ret; 63 } 64 65 if (likely(!nointxmask)) 66 vdev->pci_2_3 = pci_intx_mask_supported(pdev); 67 68 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 69 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { 70 cmd &= ~PCI_COMMAND_INTX_DISABLE; 71 pci_write_config_word(pdev, PCI_COMMAND, cmd); 72 } 73 74 msix_pos = pdev->msix_cap; 75 if (msix_pos) { 76 u16 flags; 77 u32 table; 78 79 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); 80 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); 81 82 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; 83 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; 84 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; 85 } else 86 vdev->msix_bar = 0xFF; 87 88 #ifdef CONFIG_VFIO_PCI_VGA 89 if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 90 vdev->has_vga = true; 91 #endif 92 93 return 0; 94 } 95 96 static void vfio_pci_disable(struct vfio_pci_device *vdev) 97 { 98 struct pci_dev *pdev = vdev->pdev; 99 int bar; 100 101 pci_disable_device(pdev); 102 103 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | 104 VFIO_IRQ_SET_ACTION_TRIGGER, 105 vdev->irq_type, 0, 0, NULL); 106 107 vdev->virq_disabled = false; 108 109 vfio_config_free(vdev); 110 111 for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) { 112 if (!vdev->barmap[bar]) 113 continue; 114 pci_iounmap(pdev, vdev->barmap[bar]); 115 pci_release_selected_regions(pdev, 1 << bar); 116 vdev->barmap[bar] = NULL; 117 } 118 119 /* 120 * If we have saved state, restore it. If we can reset the device, 121 * even better. Resetting with current state seems better than 122 * nothing, but saving and restoring current state without reset 123 * is just busy work. 124 */ 125 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { 126 pr_info("%s: Couldn't reload %s saved state\n", 127 __func__, dev_name(&pdev->dev)); 128 129 if (!vdev->reset_works) 130 return; 131 132 pci_save_state(pdev); 133 } 134 135 /* 136 * Disable INTx and MSI, presumably to avoid spurious interrupts 137 * during reset. Stolen from pci_reset_function() 138 */ 139 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 140 141 /* 142 * Careful, device_lock may already be held. This is the case if 143 * a driver unbind is blocked. Try to get the locks ourselves to 144 * prevent a deadlock. 145 */ 146 if (vdev->reset_works) { 147 bool reset_done = false; 148 149 if (pci_cfg_access_trylock(pdev)) { 150 if (device_trylock(&pdev->dev)) { 151 __pci_reset_function_locked(pdev); 152 reset_done = true; 153 device_unlock(&pdev->dev); 154 } 155 pci_cfg_access_unlock(pdev); 156 } 157 158 if (!reset_done) 159 pr_warn("%s: Unable to acquire locks for reset of %s\n", 160 __func__, dev_name(&pdev->dev)); 161 } 162 163 pci_restore_state(pdev); 164 } 165 166 static void vfio_pci_release(void *device_data) 167 { 168 struct vfio_pci_device *vdev = device_data; 169 170 if (atomic_dec_and_test(&vdev->refcnt)) 171 vfio_pci_disable(vdev); 172 173 module_put(THIS_MODULE); 174 } 175 176 static int vfio_pci_open(void *device_data) 177 { 178 struct vfio_pci_device *vdev = device_data; 179 180 if (!try_module_get(THIS_MODULE)) 181 return -ENODEV; 182 183 if (atomic_inc_return(&vdev->refcnt) == 1) { 184 int ret = vfio_pci_enable(vdev); 185 if (ret) { 186 module_put(THIS_MODULE); 187 return ret; 188 } 189 } 190 191 return 0; 192 } 193 194 static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type) 195 { 196 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { 197 u8 pin; 198 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); 199 if (pin) 200 return 1; 201 202 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { 203 u8 pos; 204 u16 flags; 205 206 pos = vdev->pdev->msi_cap; 207 if (pos) { 208 pci_read_config_word(vdev->pdev, 209 pos + PCI_MSI_FLAGS, &flags); 210 211 return 1 << (flags & PCI_MSI_FLAGS_QMASK); 212 } 213 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { 214 u8 pos; 215 u16 flags; 216 217 pos = vdev->pdev->msix_cap; 218 if (pos) { 219 pci_read_config_word(vdev->pdev, 220 pos + PCI_MSIX_FLAGS, &flags); 221 222 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; 223 } 224 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) 225 if (pci_is_pcie(vdev->pdev)) 226 return 1; 227 228 return 0; 229 } 230 231 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) 232 { 233 (*(int *)data)++; 234 return 0; 235 } 236 237 struct vfio_pci_fill_info { 238 int max; 239 int cur; 240 struct vfio_pci_dependent_device *devices; 241 }; 242 243 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) 244 { 245 struct vfio_pci_fill_info *fill = data; 246 struct iommu_group *iommu_group; 247 248 if (fill->cur == fill->max) 249 return -EAGAIN; /* Something changed, try again */ 250 251 iommu_group = iommu_group_get(&pdev->dev); 252 if (!iommu_group) 253 return -EPERM; /* Cannot reset non-isolated devices */ 254 255 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group); 256 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus); 257 fill->devices[fill->cur].bus = pdev->bus->number; 258 fill->devices[fill->cur].devfn = pdev->devfn; 259 fill->cur++; 260 iommu_group_put(iommu_group); 261 return 0; 262 } 263 264 struct vfio_pci_group_entry { 265 struct vfio_group *group; 266 int id; 267 }; 268 269 struct vfio_pci_group_info { 270 int count; 271 struct vfio_pci_group_entry *groups; 272 }; 273 274 static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data) 275 { 276 struct vfio_pci_group_info *info = data; 277 struct iommu_group *group; 278 int id, i; 279 280 group = iommu_group_get(&pdev->dev); 281 if (!group) 282 return -EPERM; 283 284 id = iommu_group_id(group); 285 286 for (i = 0; i < info->count; i++) 287 if (info->groups[i].id == id) 288 break; 289 290 iommu_group_put(group); 291 292 return (i == info->count) ? -EINVAL : 0; 293 } 294 295 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) 296 { 297 for (; pdev; pdev = pdev->bus->self) 298 if (pdev->bus == slot->bus) 299 return (pdev->slot == slot); 300 return false; 301 } 302 303 struct vfio_pci_walk_info { 304 int (*fn)(struct pci_dev *, void *data); 305 void *data; 306 struct pci_dev *pdev; 307 bool slot; 308 int ret; 309 }; 310 311 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) 312 { 313 struct vfio_pci_walk_info *walk = data; 314 315 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) 316 walk->ret = walk->fn(pdev, walk->data); 317 318 return walk->ret; 319 } 320 321 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, 322 int (*fn)(struct pci_dev *, 323 void *data), void *data, 324 bool slot) 325 { 326 struct vfio_pci_walk_info walk = { 327 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, 328 }; 329 330 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); 331 332 return walk.ret; 333 } 334 335 static long vfio_pci_ioctl(void *device_data, 336 unsigned int cmd, unsigned long arg) 337 { 338 struct vfio_pci_device *vdev = device_data; 339 unsigned long minsz; 340 341 if (cmd == VFIO_DEVICE_GET_INFO) { 342 struct vfio_device_info info; 343 344 minsz = offsetofend(struct vfio_device_info, num_irqs); 345 346 if (copy_from_user(&info, (void __user *)arg, minsz)) 347 return -EFAULT; 348 349 if (info.argsz < minsz) 350 return -EINVAL; 351 352 info.flags = VFIO_DEVICE_FLAGS_PCI; 353 354 if (vdev->reset_works) 355 info.flags |= VFIO_DEVICE_FLAGS_RESET; 356 357 info.num_regions = VFIO_PCI_NUM_REGIONS; 358 info.num_irqs = VFIO_PCI_NUM_IRQS; 359 360 return copy_to_user((void __user *)arg, &info, minsz); 361 362 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 363 struct pci_dev *pdev = vdev->pdev; 364 struct vfio_region_info info; 365 366 minsz = offsetofend(struct vfio_region_info, offset); 367 368 if (copy_from_user(&info, (void __user *)arg, minsz)) 369 return -EFAULT; 370 371 if (info.argsz < minsz) 372 return -EINVAL; 373 374 switch (info.index) { 375 case VFIO_PCI_CONFIG_REGION_INDEX: 376 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 377 info.size = pdev->cfg_size; 378 info.flags = VFIO_REGION_INFO_FLAG_READ | 379 VFIO_REGION_INFO_FLAG_WRITE; 380 break; 381 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 382 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 383 info.size = pci_resource_len(pdev, info.index); 384 if (!info.size) { 385 info.flags = 0; 386 break; 387 } 388 389 info.flags = VFIO_REGION_INFO_FLAG_READ | 390 VFIO_REGION_INFO_FLAG_WRITE; 391 if (pci_resource_flags(pdev, info.index) & 392 IORESOURCE_MEM && info.size >= PAGE_SIZE) 393 info.flags |= VFIO_REGION_INFO_FLAG_MMAP; 394 break; 395 case VFIO_PCI_ROM_REGION_INDEX: 396 { 397 void __iomem *io; 398 size_t size; 399 400 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 401 info.flags = 0; 402 403 /* Report the BAR size, not the ROM size */ 404 info.size = pci_resource_len(pdev, info.index); 405 if (!info.size) 406 break; 407 408 /* Is it really there? */ 409 io = pci_map_rom(pdev, &size); 410 if (!io || !size) { 411 info.size = 0; 412 break; 413 } 414 pci_unmap_rom(pdev, io); 415 416 info.flags = VFIO_REGION_INFO_FLAG_READ; 417 break; 418 } 419 case VFIO_PCI_VGA_REGION_INDEX: 420 if (!vdev->has_vga) 421 return -EINVAL; 422 423 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 424 info.size = 0xc0000; 425 info.flags = VFIO_REGION_INFO_FLAG_READ | 426 VFIO_REGION_INFO_FLAG_WRITE; 427 428 break; 429 default: 430 return -EINVAL; 431 } 432 433 return copy_to_user((void __user *)arg, &info, minsz); 434 435 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 436 struct vfio_irq_info info; 437 438 minsz = offsetofend(struct vfio_irq_info, count); 439 440 if (copy_from_user(&info, (void __user *)arg, minsz)) 441 return -EFAULT; 442 443 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 444 return -EINVAL; 445 446 switch (info.index) { 447 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: 448 break; 449 case VFIO_PCI_ERR_IRQ_INDEX: 450 if (pci_is_pcie(vdev->pdev)) 451 break; 452 /* pass thru to return error */ 453 default: 454 return -EINVAL; 455 } 456 457 info.flags = VFIO_IRQ_INFO_EVENTFD; 458 459 info.count = vfio_pci_get_irq_count(vdev, info.index); 460 461 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 462 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 463 VFIO_IRQ_INFO_AUTOMASKED); 464 else 465 info.flags |= VFIO_IRQ_INFO_NORESIZE; 466 467 return copy_to_user((void __user *)arg, &info, minsz); 468 469 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 470 struct vfio_irq_set hdr; 471 u8 *data = NULL; 472 int ret = 0; 473 474 minsz = offsetofend(struct vfio_irq_set, count); 475 476 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 477 return -EFAULT; 478 479 if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS || 480 hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 481 VFIO_IRQ_SET_ACTION_TYPE_MASK)) 482 return -EINVAL; 483 484 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 485 size_t size; 486 int max = vfio_pci_get_irq_count(vdev, hdr.index); 487 488 if (hdr.flags & VFIO_IRQ_SET_DATA_BOOL) 489 size = sizeof(uint8_t); 490 else if (hdr.flags & VFIO_IRQ_SET_DATA_EVENTFD) 491 size = sizeof(int32_t); 492 else 493 return -EINVAL; 494 495 if (hdr.argsz - minsz < hdr.count * size || 496 hdr.start >= max || hdr.start + hdr.count > max) 497 return -EINVAL; 498 499 data = memdup_user((void __user *)(arg + minsz), 500 hdr.count * size); 501 if (IS_ERR(data)) 502 return PTR_ERR(data); 503 } 504 505 mutex_lock(&vdev->igate); 506 507 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, 508 hdr.start, hdr.count, data); 509 510 mutex_unlock(&vdev->igate); 511 kfree(data); 512 513 return ret; 514 515 } else if (cmd == VFIO_DEVICE_RESET) { 516 return vdev->reset_works ? 517 pci_reset_function(vdev->pdev) : -EINVAL; 518 519 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { 520 struct vfio_pci_hot_reset_info hdr; 521 struct vfio_pci_fill_info fill = { 0 }; 522 struct vfio_pci_dependent_device *devices = NULL; 523 bool slot = false; 524 int ret = 0; 525 526 minsz = offsetofend(struct vfio_pci_hot_reset_info, count); 527 528 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 529 return -EFAULT; 530 531 if (hdr.argsz < minsz) 532 return -EINVAL; 533 534 hdr.flags = 0; 535 536 /* Can we do a slot or bus reset or neither? */ 537 if (!pci_probe_reset_slot(vdev->pdev->slot)) 538 slot = true; 539 else if (pci_probe_reset_bus(vdev->pdev->bus)) 540 return -ENODEV; 541 542 /* How many devices are affected? */ 543 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 544 vfio_pci_count_devs, 545 &fill.max, slot); 546 if (ret) 547 return ret; 548 549 WARN_ON(!fill.max); /* Should always be at least one */ 550 551 /* 552 * If there's enough space, fill it now, otherwise return 553 * -ENOSPC and the number of devices affected. 554 */ 555 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { 556 ret = -ENOSPC; 557 hdr.count = fill.max; 558 goto reset_info_exit; 559 } 560 561 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); 562 if (!devices) 563 return -ENOMEM; 564 565 fill.devices = devices; 566 567 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 568 vfio_pci_fill_devs, 569 &fill, slot); 570 571 /* 572 * If a device was removed between counting and filling, 573 * we may come up short of fill.max. If a device was 574 * added, we'll have a return of -EAGAIN above. 575 */ 576 if (!ret) 577 hdr.count = fill.cur; 578 579 reset_info_exit: 580 if (copy_to_user((void __user *)arg, &hdr, minsz)) 581 ret = -EFAULT; 582 583 if (!ret) { 584 if (copy_to_user((void __user *)(arg + minsz), devices, 585 hdr.count * sizeof(*devices))) 586 ret = -EFAULT; 587 } 588 589 kfree(devices); 590 return ret; 591 592 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) { 593 struct vfio_pci_hot_reset hdr; 594 int32_t *group_fds; 595 struct vfio_pci_group_entry *groups; 596 struct vfio_pci_group_info info; 597 bool slot = false; 598 int i, count = 0, ret = 0; 599 600 minsz = offsetofend(struct vfio_pci_hot_reset, count); 601 602 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 603 return -EFAULT; 604 605 if (hdr.argsz < minsz || hdr.flags) 606 return -EINVAL; 607 608 /* Can we do a slot or bus reset or neither? */ 609 if (!pci_probe_reset_slot(vdev->pdev->slot)) 610 slot = true; 611 else if (pci_probe_reset_bus(vdev->pdev->bus)) 612 return -ENODEV; 613 614 /* 615 * We can't let userspace give us an arbitrarily large 616 * buffer to copy, so verify how many we think there 617 * could be. Note groups can have multiple devices so 618 * one group per device is the max. 619 */ 620 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 621 vfio_pci_count_devs, 622 &count, slot); 623 if (ret) 624 return ret; 625 626 /* Somewhere between 1 and count is OK */ 627 if (!hdr.count || hdr.count > count) 628 return -EINVAL; 629 630 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); 631 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL); 632 if (!group_fds || !groups) { 633 kfree(group_fds); 634 kfree(groups); 635 return -ENOMEM; 636 } 637 638 if (copy_from_user(group_fds, (void __user *)(arg + minsz), 639 hdr.count * sizeof(*group_fds))) { 640 kfree(group_fds); 641 kfree(groups); 642 return -EFAULT; 643 } 644 645 /* 646 * For each group_fd, get the group through the vfio external 647 * user interface and store the group and iommu ID. This 648 * ensures the group is held across the reset. 649 */ 650 for (i = 0; i < hdr.count; i++) { 651 struct vfio_group *group; 652 struct fd f = fdget(group_fds[i]); 653 if (!f.file) { 654 ret = -EBADF; 655 break; 656 } 657 658 group = vfio_group_get_external_user(f.file); 659 fdput(f); 660 if (IS_ERR(group)) { 661 ret = PTR_ERR(group); 662 break; 663 } 664 665 groups[i].group = group; 666 groups[i].id = vfio_external_user_iommu_id(group); 667 } 668 669 kfree(group_fds); 670 671 /* release reference to groups on error */ 672 if (ret) 673 goto hot_reset_release; 674 675 info.count = hdr.count; 676 info.groups = groups; 677 678 /* 679 * Test whether all the affected devices are contained 680 * by the set of groups provided by the user. 681 */ 682 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 683 vfio_pci_validate_devs, 684 &info, slot); 685 if (!ret) 686 /* User has access, do the reset */ 687 ret = slot ? pci_reset_slot(vdev->pdev->slot) : 688 pci_reset_bus(vdev->pdev->bus); 689 690 hot_reset_release: 691 for (i--; i >= 0; i--) 692 vfio_group_put_external_user(groups[i].group); 693 694 kfree(groups); 695 return ret; 696 } 697 698 return -ENOTTY; 699 } 700 701 static ssize_t vfio_pci_rw(void *device_data, char __user *buf, 702 size_t count, loff_t *ppos, bool iswrite) 703 { 704 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 705 struct vfio_pci_device *vdev = device_data; 706 707 if (index >= VFIO_PCI_NUM_REGIONS) 708 return -EINVAL; 709 710 switch (index) { 711 case VFIO_PCI_CONFIG_REGION_INDEX: 712 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); 713 714 case VFIO_PCI_ROM_REGION_INDEX: 715 if (iswrite) 716 return -EINVAL; 717 return vfio_pci_bar_rw(vdev, buf, count, ppos, false); 718 719 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 720 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); 721 722 case VFIO_PCI_VGA_REGION_INDEX: 723 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); 724 } 725 726 return -EINVAL; 727 } 728 729 static ssize_t vfio_pci_read(void *device_data, char __user *buf, 730 size_t count, loff_t *ppos) 731 { 732 if (!count) 733 return 0; 734 735 return vfio_pci_rw(device_data, buf, count, ppos, false); 736 } 737 738 static ssize_t vfio_pci_write(void *device_data, const char __user *buf, 739 size_t count, loff_t *ppos) 740 { 741 if (!count) 742 return 0; 743 744 return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); 745 } 746 747 static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) 748 { 749 struct vfio_pci_device *vdev = device_data; 750 struct pci_dev *pdev = vdev->pdev; 751 unsigned int index; 752 u64 phys_len, req_len, pgoff, req_start; 753 int ret; 754 755 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 756 757 if (vma->vm_end < vma->vm_start) 758 return -EINVAL; 759 if ((vma->vm_flags & VM_SHARED) == 0) 760 return -EINVAL; 761 if (index >= VFIO_PCI_ROM_REGION_INDEX) 762 return -EINVAL; 763 if (!(pci_resource_flags(pdev, index) & IORESOURCE_MEM)) 764 return -EINVAL; 765 766 phys_len = pci_resource_len(pdev, index); 767 req_len = vma->vm_end - vma->vm_start; 768 pgoff = vma->vm_pgoff & 769 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 770 req_start = pgoff << PAGE_SHIFT; 771 772 if (phys_len < PAGE_SIZE || req_start + req_len > phys_len) 773 return -EINVAL; 774 775 if (index == vdev->msix_bar) { 776 /* 777 * Disallow mmaps overlapping the MSI-X table; users don't 778 * get to touch this directly. We could find somewhere 779 * else to map the overlap, but page granularity is only 780 * a recommendation, not a requirement, so the user needs 781 * to know which bits are real. Requiring them to mmap 782 * around the table makes that clear. 783 */ 784 785 /* If neither entirely above nor below, then it overlaps */ 786 if (!(req_start >= vdev->msix_offset + vdev->msix_size || 787 req_start + req_len <= vdev->msix_offset)) 788 return -EINVAL; 789 } 790 791 /* 792 * Even though we don't make use of the barmap for the mmap, 793 * we need to request the region and the barmap tracks that. 794 */ 795 if (!vdev->barmap[index]) { 796 ret = pci_request_selected_regions(pdev, 797 1 << index, "vfio-pci"); 798 if (ret) 799 return ret; 800 801 vdev->barmap[index] = pci_iomap(pdev, index, 0); 802 } 803 804 vma->vm_private_data = vdev; 805 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 806 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff; 807 808 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 809 req_len, vma->vm_page_prot); 810 } 811 812 static const struct vfio_device_ops vfio_pci_ops = { 813 .name = "vfio-pci", 814 .open = vfio_pci_open, 815 .release = vfio_pci_release, 816 .ioctl = vfio_pci_ioctl, 817 .read = vfio_pci_read, 818 .write = vfio_pci_write, 819 .mmap = vfio_pci_mmap, 820 }; 821 822 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 823 { 824 u8 type; 825 struct vfio_pci_device *vdev; 826 struct iommu_group *group; 827 int ret; 828 829 pci_read_config_byte(pdev, PCI_HEADER_TYPE, &type); 830 if ((type & PCI_HEADER_TYPE) != PCI_HEADER_TYPE_NORMAL) 831 return -EINVAL; 832 833 group = iommu_group_get(&pdev->dev); 834 if (!group) 835 return -EINVAL; 836 837 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); 838 if (!vdev) { 839 iommu_group_put(group); 840 return -ENOMEM; 841 } 842 843 vdev->pdev = pdev; 844 vdev->irq_type = VFIO_PCI_NUM_IRQS; 845 mutex_init(&vdev->igate); 846 spin_lock_init(&vdev->irqlock); 847 atomic_set(&vdev->refcnt, 0); 848 849 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); 850 if (ret) { 851 iommu_group_put(group); 852 kfree(vdev); 853 } 854 855 return ret; 856 } 857 858 static void vfio_pci_remove(struct pci_dev *pdev) 859 { 860 struct vfio_pci_device *vdev; 861 862 vdev = vfio_del_group_dev(&pdev->dev); 863 if (!vdev) 864 return; 865 866 iommu_group_put(pdev->dev.iommu_group); 867 kfree(vdev); 868 } 869 870 static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, 871 pci_channel_state_t state) 872 { 873 struct vfio_pci_device *vdev; 874 struct vfio_device *device; 875 876 device = vfio_device_get_from_dev(&pdev->dev); 877 if (device == NULL) 878 return PCI_ERS_RESULT_DISCONNECT; 879 880 vdev = vfio_device_data(device); 881 if (vdev == NULL) { 882 vfio_device_put(device); 883 return PCI_ERS_RESULT_DISCONNECT; 884 } 885 886 if (vdev->err_trigger) 887 eventfd_signal(vdev->err_trigger, 1); 888 889 vfio_device_put(device); 890 891 return PCI_ERS_RESULT_CAN_RECOVER; 892 } 893 894 static struct pci_error_handlers vfio_err_handlers = { 895 .error_detected = vfio_pci_aer_err_detected, 896 }; 897 898 static struct pci_driver vfio_pci_driver = { 899 .name = "vfio-pci", 900 .id_table = NULL, /* only dynamic ids */ 901 .probe = vfio_pci_probe, 902 .remove = vfio_pci_remove, 903 .err_handler = &vfio_err_handlers, 904 }; 905 906 static void __exit vfio_pci_cleanup(void) 907 { 908 pci_unregister_driver(&vfio_pci_driver); 909 vfio_pci_virqfd_exit(); 910 vfio_pci_uninit_perm_bits(); 911 } 912 913 static int __init vfio_pci_init(void) 914 { 915 int ret; 916 917 /* Allocate shared config space permision data used by all devices */ 918 ret = vfio_pci_init_perm_bits(); 919 if (ret) 920 return ret; 921 922 /* Start the virqfd cleanup handler */ 923 ret = vfio_pci_virqfd_init(); 924 if (ret) 925 goto out_virqfd; 926 927 /* Register and scan for devices */ 928 ret = pci_register_driver(&vfio_pci_driver); 929 if (ret) 930 goto out_driver; 931 932 return 0; 933 934 out_driver: 935 vfio_pci_virqfd_exit(); 936 out_virqfd: 937 vfio_pci_uninit_perm_bits(); 938 return ret; 939 } 940 941 module_init(vfio_pci_init); 942 module_exit(vfio_pci_cleanup); 943 944 MODULE_VERSION(DRIVER_VERSION); 945 MODULE_LICENSE("GPL v2"); 946 MODULE_AUTHOR(DRIVER_AUTHOR); 947 MODULE_DESCRIPTION(DRIVER_DESC); 948