1 /* 2 * vfio based device assignment support 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <linux/vfio.h> 23 #include <sys/ioctl.h> 24 25 #include "hw/hw.h" 26 #include "hw/pci/msi.h" 27 #include "hw/pci/msix.h" 28 #include "hw/pci/pci_bridge.h" 29 #include "migration/vmstate.h" 30 #include "qemu/error-report.h" 31 #include "qemu/module.h" 32 #include "qemu/option.h" 33 #include "qemu/range.h" 34 #include "qemu/units.h" 35 #include "sysemu/kvm.h" 36 #include "sysemu/sysemu.h" 37 #include "pci.h" 38 #include "trace.h" 39 #include "qapi/error.h" 40 41 #define TYPE_VFIO_PCI "vfio-pci" 42 #define PCI_VFIO(obj) OBJECT_CHECK(VFIOPCIDevice, obj, TYPE_VFIO_PCI) 43 44 #define TYPE_VIFO_PCI_NOHOTPLUG "vfio-pci-nohotplug" 45 46 static void vfio_disable_interrupts(VFIOPCIDevice *vdev); 47 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled); 48 49 /* 50 * Disabling BAR mmaping can be slow, but toggling it around INTx can 51 * also be a huge overhead. We try to get the best of both worlds by 52 * waiting until an interrupt to disable mmaps (subsequent transitions 53 * to the same state are effectively no overhead). If the interrupt has 54 * been serviced and the time gap is long enough, we re-enable mmaps for 55 * performance. This works well for things like graphics cards, which 56 * may not use their interrupt at all and are penalized to an unusable 57 * level by read/write BAR traps. Other devices, like NICs, have more 58 * regular interrupts and see much better latency by staying in non-mmap 59 * mode. We therefore set the default mmap_timeout such that a ping 60 * is just enough to keep the mmap disabled. Users can experiment with 61 * other options with the x-intx-mmap-timeout-ms parameter (a value of 62 * zero disables the timer). 63 */ 64 static void vfio_intx_mmap_enable(void *opaque) 65 { 66 VFIOPCIDevice *vdev = opaque; 67 68 if (vdev->intx.pending) { 69 timer_mod(vdev->intx.mmap_timer, 70 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout); 71 return; 72 } 73 74 vfio_mmap_set_enabled(vdev, true); 75 } 76 77 static void vfio_intx_interrupt(void *opaque) 78 { 79 VFIOPCIDevice *vdev = opaque; 80 81 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) { 82 return; 83 } 84 85 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin); 86 87 vdev->intx.pending = true; 88 pci_irq_assert(&vdev->pdev); 89 vfio_mmap_set_enabled(vdev, false); 90 if (vdev->intx.mmap_timeout) { 91 timer_mod(vdev->intx.mmap_timer, 92 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout); 93 } 94 } 95 96 static void vfio_intx_eoi(VFIODevice *vbasedev) 97 { 98 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 99 100 if (!vdev->intx.pending) { 101 return; 102 } 103 104 trace_vfio_intx_eoi(vbasedev->name); 105 106 vdev->intx.pending = false; 107 pci_irq_deassert(&vdev->pdev); 108 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 109 } 110 111 static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp) 112 { 113 #ifdef CONFIG_KVM 114 struct kvm_irqfd irqfd = { 115 .fd = event_notifier_get_fd(&vdev->intx.interrupt), 116 .gsi = vdev->intx.route.irq, 117 .flags = KVM_IRQFD_FLAG_RESAMPLE, 118 }; 119 Error *err = NULL; 120 121 if (vdev->no_kvm_intx || !kvm_irqfds_enabled() || 122 vdev->intx.route.mode != PCI_INTX_ENABLED || 123 !kvm_resamplefds_enabled()) { 124 return; 125 } 126 127 /* Get to a known interrupt state */ 128 qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev); 129 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 130 vdev->intx.pending = false; 131 pci_irq_deassert(&vdev->pdev); 132 133 /* Get an eventfd for resample/unmask */ 134 if (event_notifier_init(&vdev->intx.unmask, 0)) { 135 error_setg(errp, "event_notifier_init failed eoi"); 136 goto fail; 137 } 138 139 /* KVM triggers it, VFIO listens for it */ 140 irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask); 141 142 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) { 143 error_setg_errno(errp, errno, "failed to setup resample irqfd"); 144 goto fail_irqfd; 145 } 146 147 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0, 148 VFIO_IRQ_SET_ACTION_UNMASK, 149 irqfd.resamplefd, &err)) { 150 error_propagate(errp, err); 151 goto fail_vfio; 152 } 153 154 /* Let'em rip */ 155 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 156 157 vdev->intx.kvm_accel = true; 158 159 trace_vfio_intx_enable_kvm(vdev->vbasedev.name); 160 161 return; 162 163 fail_vfio: 164 irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN; 165 kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd); 166 fail_irqfd: 167 event_notifier_cleanup(&vdev->intx.unmask); 168 fail: 169 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev); 170 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 171 #endif 172 } 173 174 static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev) 175 { 176 #ifdef CONFIG_KVM 177 struct kvm_irqfd irqfd = { 178 .fd = event_notifier_get_fd(&vdev->intx.interrupt), 179 .gsi = vdev->intx.route.irq, 180 .flags = KVM_IRQFD_FLAG_DEASSIGN, 181 }; 182 183 if (!vdev->intx.kvm_accel) { 184 return; 185 } 186 187 /* 188 * Get to a known state, hardware masked, QEMU ready to accept new 189 * interrupts, QEMU IRQ de-asserted. 190 */ 191 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 192 vdev->intx.pending = false; 193 pci_irq_deassert(&vdev->pdev); 194 195 /* Tell KVM to stop listening for an INTx irqfd */ 196 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) { 197 error_report("vfio: Error: Failed to disable INTx irqfd: %m"); 198 } 199 200 /* We only need to close the eventfd for VFIO to cleanup the kernel side */ 201 event_notifier_cleanup(&vdev->intx.unmask); 202 203 /* QEMU starts listening for interrupt events. */ 204 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev); 205 206 vdev->intx.kvm_accel = false; 207 208 /* If we've missed an event, let it re-fire through QEMU */ 209 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 210 211 trace_vfio_intx_disable_kvm(vdev->vbasedev.name); 212 #endif 213 } 214 215 static void vfio_intx_update(PCIDevice *pdev) 216 { 217 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 218 PCIINTxRoute route; 219 Error *err = NULL; 220 221 if (vdev->interrupt != VFIO_INT_INTx) { 222 return; 223 } 224 225 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin); 226 227 if (!pci_intx_route_changed(&vdev->intx.route, &route)) { 228 return; /* Nothing changed */ 229 } 230 231 trace_vfio_intx_update(vdev->vbasedev.name, 232 vdev->intx.route.irq, route.irq); 233 234 vfio_intx_disable_kvm(vdev); 235 236 vdev->intx.route = route; 237 238 if (route.mode != PCI_INTX_ENABLED) { 239 return; 240 } 241 242 vfio_intx_enable_kvm(vdev, &err); 243 if (err) { 244 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 245 } 246 247 /* Re-enable the interrupt in cased we missed an EOI */ 248 vfio_intx_eoi(&vdev->vbasedev); 249 } 250 251 static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp) 252 { 253 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1); 254 Error *err = NULL; 255 int32_t fd; 256 int ret; 257 258 259 if (!pin) { 260 return 0; 261 } 262 263 vfio_disable_interrupts(vdev); 264 265 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */ 266 pci_config_set_interrupt_pin(vdev->pdev.config, pin); 267 268 #ifdef CONFIG_KVM 269 /* 270 * Only conditional to avoid generating error messages on platforms 271 * where we won't actually use the result anyway. 272 */ 273 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) { 274 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev, 275 vdev->intx.pin); 276 } 277 #endif 278 279 ret = event_notifier_init(&vdev->intx.interrupt, 0); 280 if (ret) { 281 error_setg_errno(errp, -ret, "event_notifier_init failed"); 282 return ret; 283 } 284 fd = event_notifier_get_fd(&vdev->intx.interrupt); 285 qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev); 286 287 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0, 288 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { 289 error_propagate(errp, err); 290 qemu_set_fd_handler(fd, NULL, NULL, vdev); 291 event_notifier_cleanup(&vdev->intx.interrupt); 292 return -errno; 293 } 294 295 vfio_intx_enable_kvm(vdev, &err); 296 if (err) { 297 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 298 } 299 300 vdev->interrupt = VFIO_INT_INTx; 301 302 trace_vfio_intx_enable(vdev->vbasedev.name); 303 return 0; 304 } 305 306 static void vfio_intx_disable(VFIOPCIDevice *vdev) 307 { 308 int fd; 309 310 timer_del(vdev->intx.mmap_timer); 311 vfio_intx_disable_kvm(vdev); 312 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 313 vdev->intx.pending = false; 314 pci_irq_deassert(&vdev->pdev); 315 vfio_mmap_set_enabled(vdev, true); 316 317 fd = event_notifier_get_fd(&vdev->intx.interrupt); 318 qemu_set_fd_handler(fd, NULL, NULL, vdev); 319 event_notifier_cleanup(&vdev->intx.interrupt); 320 321 vdev->interrupt = VFIO_INT_NONE; 322 323 trace_vfio_intx_disable(vdev->vbasedev.name); 324 } 325 326 /* 327 * MSI/X 328 */ 329 static void vfio_msi_interrupt(void *opaque) 330 { 331 VFIOMSIVector *vector = opaque; 332 VFIOPCIDevice *vdev = vector->vdev; 333 MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector); 334 void (*notify)(PCIDevice *dev, unsigned vector); 335 MSIMessage msg; 336 int nr = vector - vdev->msi_vectors; 337 338 if (!event_notifier_test_and_clear(&vector->interrupt)) { 339 return; 340 } 341 342 if (vdev->interrupt == VFIO_INT_MSIX) { 343 get_msg = msix_get_message; 344 notify = msix_notify; 345 346 /* A masked vector firing needs to use the PBA, enable it */ 347 if (msix_is_masked(&vdev->pdev, nr)) { 348 set_bit(nr, vdev->msix->pending); 349 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true); 350 trace_vfio_msix_pba_enable(vdev->vbasedev.name); 351 } 352 } else if (vdev->interrupt == VFIO_INT_MSI) { 353 get_msg = msi_get_message; 354 notify = msi_notify; 355 } else { 356 abort(); 357 } 358 359 msg = get_msg(&vdev->pdev, nr); 360 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data); 361 notify(&vdev->pdev, nr); 362 } 363 364 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix) 365 { 366 struct vfio_irq_set *irq_set; 367 int ret = 0, i, argsz; 368 int32_t *fds; 369 370 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds)); 371 372 irq_set = g_malloc0(argsz); 373 irq_set->argsz = argsz; 374 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; 375 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX; 376 irq_set->start = 0; 377 irq_set->count = vdev->nr_vectors; 378 fds = (int32_t *)&irq_set->data; 379 380 for (i = 0; i < vdev->nr_vectors; i++) { 381 int fd = -1; 382 383 /* 384 * MSI vs MSI-X - The guest has direct access to MSI mask and pending 385 * bits, therefore we always use the KVM signaling path when setup. 386 * MSI-X mask and pending bits are emulated, so we want to use the 387 * KVM signaling path only when configured and unmasked. 388 */ 389 if (vdev->msi_vectors[i].use) { 390 if (vdev->msi_vectors[i].virq < 0 || 391 (msix && msix_is_masked(&vdev->pdev, i))) { 392 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt); 393 } else { 394 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt); 395 } 396 } 397 398 fds[i] = fd; 399 } 400 401 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 402 403 g_free(irq_set); 404 405 return ret; 406 } 407 408 static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector, 409 int vector_n, bool msix) 410 { 411 int virq; 412 413 if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) { 414 return; 415 } 416 417 if (event_notifier_init(&vector->kvm_interrupt, 0)) { 418 return; 419 } 420 421 virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev); 422 if (virq < 0) { 423 event_notifier_cleanup(&vector->kvm_interrupt); 424 return; 425 } 426 427 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt, 428 NULL, virq) < 0) { 429 kvm_irqchip_release_virq(kvm_state, virq); 430 event_notifier_cleanup(&vector->kvm_interrupt); 431 return; 432 } 433 434 vector->virq = virq; 435 } 436 437 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector) 438 { 439 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt, 440 vector->virq); 441 kvm_irqchip_release_virq(kvm_state, vector->virq); 442 vector->virq = -1; 443 event_notifier_cleanup(&vector->kvm_interrupt); 444 } 445 446 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg, 447 PCIDevice *pdev) 448 { 449 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev); 450 kvm_irqchip_commit_routes(kvm_state); 451 } 452 453 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, 454 MSIMessage *msg, IOHandler *handler) 455 { 456 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 457 VFIOMSIVector *vector; 458 int ret; 459 460 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr); 461 462 vector = &vdev->msi_vectors[nr]; 463 464 if (!vector->use) { 465 vector->vdev = vdev; 466 vector->virq = -1; 467 if (event_notifier_init(&vector->interrupt, 0)) { 468 error_report("vfio: Error: event_notifier_init failed"); 469 } 470 vector->use = true; 471 msix_vector_use(pdev, nr); 472 } 473 474 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 475 handler, NULL, vector); 476 477 /* 478 * Attempt to enable route through KVM irqchip, 479 * default to userspace handling if unavailable. 480 */ 481 if (vector->virq >= 0) { 482 if (!msg) { 483 vfio_remove_kvm_msi_virq(vector); 484 } else { 485 vfio_update_kvm_msi_virq(vector, *msg, pdev); 486 } 487 } else { 488 if (msg) { 489 vfio_add_kvm_msi_virq(vdev, vector, nr, true); 490 } 491 } 492 493 /* 494 * We don't want to have the host allocate all possible MSI vectors 495 * for a device if they're not in use, so we shutdown and incrementally 496 * increase them as needed. 497 */ 498 if (vdev->nr_vectors < nr + 1) { 499 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); 500 vdev->nr_vectors = nr + 1; 501 ret = vfio_enable_vectors(vdev, true); 502 if (ret) { 503 error_report("vfio: failed to enable vectors, %d", ret); 504 } 505 } else { 506 Error *err = NULL; 507 int32_t fd; 508 509 if (vector->virq >= 0) { 510 fd = event_notifier_get_fd(&vector->kvm_interrupt); 511 } else { 512 fd = event_notifier_get_fd(&vector->interrupt); 513 } 514 515 if (vfio_set_irq_signaling(&vdev->vbasedev, 516 VFIO_PCI_MSIX_IRQ_INDEX, nr, 517 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { 518 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 519 } 520 } 521 522 /* Disable PBA emulation when nothing more is pending. */ 523 clear_bit(nr, vdev->msix->pending); 524 if (find_first_bit(vdev->msix->pending, 525 vdev->nr_vectors) == vdev->nr_vectors) { 526 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false); 527 trace_vfio_msix_pba_disable(vdev->vbasedev.name); 528 } 529 530 return 0; 531 } 532 533 static int vfio_msix_vector_use(PCIDevice *pdev, 534 unsigned int nr, MSIMessage msg) 535 { 536 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt); 537 } 538 539 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) 540 { 541 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 542 VFIOMSIVector *vector = &vdev->msi_vectors[nr]; 543 544 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr); 545 546 /* 547 * There are still old guests that mask and unmask vectors on every 548 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of 549 * the KVM setup in place, simply switch VFIO to use the non-bypass 550 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X 551 * core will mask the interrupt and set pending bits, allowing it to 552 * be re-asserted on unmask. Nothing to do if already using QEMU mode. 553 */ 554 if (vector->virq >= 0) { 555 int32_t fd = event_notifier_get_fd(&vector->interrupt); 556 Error *err = NULL; 557 558 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr, 559 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { 560 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 561 } 562 } 563 } 564 565 static void vfio_msix_enable(VFIOPCIDevice *vdev) 566 { 567 vfio_disable_interrupts(vdev); 568 569 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries); 570 571 vdev->interrupt = VFIO_INT_MSIX; 572 573 /* 574 * Some communication channels between VF & PF or PF & fw rely on the 575 * physical state of the device and expect that enabling MSI-X from the 576 * guest enables the same on the host. When our guest is Linux, the 577 * guest driver call to pci_enable_msix() sets the enabling bit in the 578 * MSI-X capability, but leaves the vector table masked. We therefore 579 * can't rely on a vector_use callback (from request_irq() in the guest) 580 * to switch the physical device into MSI-X mode because that may come a 581 * long time after pci_enable_msix(). This code enables vector 0 with 582 * triggering to userspace, then immediately release the vector, leaving 583 * the physical device with no vectors enabled, but MSI-X enabled, just 584 * like the guest view. 585 */ 586 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); 587 vfio_msix_vector_release(&vdev->pdev, 0); 588 589 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, 590 vfio_msix_vector_release, NULL)) { 591 error_report("vfio: msix_set_vector_notifiers failed"); 592 } 593 594 trace_vfio_msix_enable(vdev->vbasedev.name); 595 } 596 597 static void vfio_msi_enable(VFIOPCIDevice *vdev) 598 { 599 int ret, i; 600 601 vfio_disable_interrupts(vdev); 602 603 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev); 604 retry: 605 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors); 606 607 for (i = 0; i < vdev->nr_vectors; i++) { 608 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 609 610 vector->vdev = vdev; 611 vector->virq = -1; 612 vector->use = true; 613 614 if (event_notifier_init(&vector->interrupt, 0)) { 615 error_report("vfio: Error: event_notifier_init failed"); 616 } 617 618 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 619 vfio_msi_interrupt, NULL, vector); 620 621 /* 622 * Attempt to enable route through KVM irqchip, 623 * default to userspace handling if unavailable. 624 */ 625 vfio_add_kvm_msi_virq(vdev, vector, i, false); 626 } 627 628 /* Set interrupt type prior to possible interrupts */ 629 vdev->interrupt = VFIO_INT_MSI; 630 631 ret = vfio_enable_vectors(vdev, false); 632 if (ret) { 633 if (ret < 0) { 634 error_report("vfio: Error: Failed to setup MSI fds: %m"); 635 } else if (ret != vdev->nr_vectors) { 636 error_report("vfio: Error: Failed to enable %d " 637 "MSI vectors, retry with %d", vdev->nr_vectors, ret); 638 } 639 640 for (i = 0; i < vdev->nr_vectors; i++) { 641 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 642 if (vector->virq >= 0) { 643 vfio_remove_kvm_msi_virq(vector); 644 } 645 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 646 NULL, NULL, NULL); 647 event_notifier_cleanup(&vector->interrupt); 648 } 649 650 g_free(vdev->msi_vectors); 651 652 if (ret > 0 && ret != vdev->nr_vectors) { 653 vdev->nr_vectors = ret; 654 goto retry; 655 } 656 vdev->nr_vectors = 0; 657 658 /* 659 * Failing to setup MSI doesn't really fall within any specification. 660 * Let's try leaving interrupts disabled and hope the guest figures 661 * out to fall back to INTx for this device. 662 */ 663 error_report("vfio: Error: Failed to enable MSI"); 664 vdev->interrupt = VFIO_INT_NONE; 665 666 return; 667 } 668 669 trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors); 670 } 671 672 static void vfio_msi_disable_common(VFIOPCIDevice *vdev) 673 { 674 Error *err = NULL; 675 int i; 676 677 for (i = 0; i < vdev->nr_vectors; i++) { 678 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 679 if (vdev->msi_vectors[i].use) { 680 if (vector->virq >= 0) { 681 vfio_remove_kvm_msi_virq(vector); 682 } 683 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 684 NULL, NULL, NULL); 685 event_notifier_cleanup(&vector->interrupt); 686 } 687 } 688 689 g_free(vdev->msi_vectors); 690 vdev->msi_vectors = NULL; 691 vdev->nr_vectors = 0; 692 vdev->interrupt = VFIO_INT_NONE; 693 694 vfio_intx_enable(vdev, &err); 695 if (err) { 696 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 697 } 698 } 699 700 static void vfio_msix_disable(VFIOPCIDevice *vdev) 701 { 702 int i; 703 704 msix_unset_vector_notifiers(&vdev->pdev); 705 706 /* 707 * MSI-X will only release vectors if MSI-X is still enabled on the 708 * device, check through the rest and release it ourselves if necessary. 709 */ 710 for (i = 0; i < vdev->nr_vectors; i++) { 711 if (vdev->msi_vectors[i].use) { 712 vfio_msix_vector_release(&vdev->pdev, i); 713 msix_vector_unuse(&vdev->pdev, i); 714 } 715 } 716 717 if (vdev->nr_vectors) { 718 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); 719 } 720 721 vfio_msi_disable_common(vdev); 722 723 memset(vdev->msix->pending, 0, 724 BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long)); 725 726 trace_vfio_msix_disable(vdev->vbasedev.name); 727 } 728 729 static void vfio_msi_disable(VFIOPCIDevice *vdev) 730 { 731 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX); 732 vfio_msi_disable_common(vdev); 733 734 trace_vfio_msi_disable(vdev->vbasedev.name); 735 } 736 737 static void vfio_update_msi(VFIOPCIDevice *vdev) 738 { 739 int i; 740 741 for (i = 0; i < vdev->nr_vectors; i++) { 742 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 743 MSIMessage msg; 744 745 if (!vector->use || vector->virq < 0) { 746 continue; 747 } 748 749 msg = msi_get_message(&vdev->pdev, i); 750 vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev); 751 } 752 } 753 754 static void vfio_pci_load_rom(VFIOPCIDevice *vdev) 755 { 756 struct vfio_region_info *reg_info; 757 uint64_t size; 758 off_t off = 0; 759 ssize_t bytes; 760 761 if (vfio_get_region_info(&vdev->vbasedev, 762 VFIO_PCI_ROM_REGION_INDEX, ®_info)) { 763 error_report("vfio: Error getting ROM info: %m"); 764 return; 765 } 766 767 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size, 768 (unsigned long)reg_info->offset, 769 (unsigned long)reg_info->flags); 770 771 vdev->rom_size = size = reg_info->size; 772 vdev->rom_offset = reg_info->offset; 773 774 g_free(reg_info); 775 776 if (!vdev->rom_size) { 777 vdev->rom_read_failed = true; 778 error_report("vfio-pci: Cannot read device rom at " 779 "%s", vdev->vbasedev.name); 780 error_printf("Device option ROM contents are probably invalid " 781 "(check dmesg).\nSkip option ROM probe with rombar=0, " 782 "or load from file with romfile=\n"); 783 return; 784 } 785 786 vdev->rom = g_malloc(size); 787 memset(vdev->rom, 0xff, size); 788 789 while (size) { 790 bytes = pread(vdev->vbasedev.fd, vdev->rom + off, 791 size, vdev->rom_offset + off); 792 if (bytes == 0) { 793 break; 794 } else if (bytes > 0) { 795 off += bytes; 796 size -= bytes; 797 } else { 798 if (errno == EINTR || errno == EAGAIN) { 799 continue; 800 } 801 error_report("vfio: Error reading device ROM: %m"); 802 break; 803 } 804 } 805 806 /* 807 * Test the ROM signature against our device, if the vendor is correct 808 * but the device ID doesn't match, store the correct device ID and 809 * recompute the checksum. Intel IGD devices need this and are known 810 * to have bogus checksums so we can't simply adjust the checksum. 811 */ 812 if (pci_get_word(vdev->rom) == 0xaa55 && 813 pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size && 814 !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) { 815 uint16_t vid, did; 816 817 vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4); 818 did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6); 819 820 if (vid == vdev->vendor_id && did != vdev->device_id) { 821 int i; 822 uint8_t csum, *data = vdev->rom; 823 824 pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6, 825 vdev->device_id); 826 data[6] = 0; 827 828 for (csum = 0, i = 0; i < vdev->rom_size; i++) { 829 csum += data[i]; 830 } 831 832 data[6] = -csum; 833 } 834 } 835 } 836 837 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size) 838 { 839 VFIOPCIDevice *vdev = opaque; 840 union { 841 uint8_t byte; 842 uint16_t word; 843 uint32_t dword; 844 uint64_t qword; 845 } val; 846 uint64_t data = 0; 847 848 /* Load the ROM lazily when the guest tries to read it */ 849 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) { 850 vfio_pci_load_rom(vdev); 851 } 852 853 memcpy(&val, vdev->rom + addr, 854 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0); 855 856 switch (size) { 857 case 1: 858 data = val.byte; 859 break; 860 case 2: 861 data = le16_to_cpu(val.word); 862 break; 863 case 4: 864 data = le32_to_cpu(val.dword); 865 break; 866 default: 867 hw_error("vfio: unsupported read size, %d bytes\n", size); 868 break; 869 } 870 871 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data); 872 873 return data; 874 } 875 876 static void vfio_rom_write(void *opaque, hwaddr addr, 877 uint64_t data, unsigned size) 878 { 879 } 880 881 static const MemoryRegionOps vfio_rom_ops = { 882 .read = vfio_rom_read, 883 .write = vfio_rom_write, 884 .endianness = DEVICE_LITTLE_ENDIAN, 885 }; 886 887 static void vfio_pci_size_rom(VFIOPCIDevice *vdev) 888 { 889 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK); 890 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS; 891 DeviceState *dev = DEVICE(vdev); 892 char *name; 893 int fd = vdev->vbasedev.fd; 894 895 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) { 896 /* Since pci handles romfile, just print a message and return */ 897 if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) { 898 warn_report("Device at %s is known to cause system instability" 899 " issues during option rom execution", 900 vdev->vbasedev.name); 901 error_printf("Proceeding anyway since user specified romfile\n"); 902 } 903 return; 904 } 905 906 /* 907 * Use the same size ROM BAR as the physical device. The contents 908 * will get filled in later when the guest tries to read it. 909 */ 910 if (pread(fd, &orig, 4, offset) != 4 || 911 pwrite(fd, &size, 4, offset) != 4 || 912 pread(fd, &size, 4, offset) != 4 || 913 pwrite(fd, &orig, 4, offset) != 4) { 914 error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name); 915 return; 916 } 917 918 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1; 919 920 if (!size) { 921 return; 922 } 923 924 if (vfio_blacklist_opt_rom(vdev)) { 925 if (dev->opts && qemu_opt_get(dev->opts, "rombar")) { 926 warn_report("Device at %s is known to cause system instability" 927 " issues during option rom execution", 928 vdev->vbasedev.name); 929 error_printf("Proceeding anyway since user specified" 930 " non zero value for rombar\n"); 931 } else { 932 warn_report("Rom loading for device at %s has been disabled" 933 " due to system instability issues", 934 vdev->vbasedev.name); 935 error_printf("Specify rombar=1 or romfile to force\n"); 936 return; 937 } 938 } 939 940 trace_vfio_pci_size_rom(vdev->vbasedev.name, size); 941 942 name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name); 943 944 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev), 945 &vfio_rom_ops, vdev, name, size); 946 g_free(name); 947 948 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT, 949 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom); 950 951 vdev->rom_read_failed = false; 952 } 953 954 void vfio_vga_write(void *opaque, hwaddr addr, 955 uint64_t data, unsigned size) 956 { 957 VFIOVGARegion *region = opaque; 958 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]); 959 union { 960 uint8_t byte; 961 uint16_t word; 962 uint32_t dword; 963 uint64_t qword; 964 } buf; 965 off_t offset = vga->fd_offset + region->offset + addr; 966 967 switch (size) { 968 case 1: 969 buf.byte = data; 970 break; 971 case 2: 972 buf.word = cpu_to_le16(data); 973 break; 974 case 4: 975 buf.dword = cpu_to_le32(data); 976 break; 977 default: 978 hw_error("vfio: unsupported write size, %d bytes", size); 979 break; 980 } 981 982 if (pwrite(vga->fd, &buf, size, offset) != size) { 983 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m", 984 __func__, region->offset + addr, data, size); 985 } 986 987 trace_vfio_vga_write(region->offset + addr, data, size); 988 } 989 990 uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size) 991 { 992 VFIOVGARegion *region = opaque; 993 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]); 994 union { 995 uint8_t byte; 996 uint16_t word; 997 uint32_t dword; 998 uint64_t qword; 999 } buf; 1000 uint64_t data = 0; 1001 off_t offset = vga->fd_offset + region->offset + addr; 1002 1003 if (pread(vga->fd, &buf, size, offset) != size) { 1004 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m", 1005 __func__, region->offset + addr, size); 1006 return (uint64_t)-1; 1007 } 1008 1009 switch (size) { 1010 case 1: 1011 data = buf.byte; 1012 break; 1013 case 2: 1014 data = le16_to_cpu(buf.word); 1015 break; 1016 case 4: 1017 data = le32_to_cpu(buf.dword); 1018 break; 1019 default: 1020 hw_error("vfio: unsupported read size, %d bytes", size); 1021 break; 1022 } 1023 1024 trace_vfio_vga_read(region->offset + addr, size, data); 1025 1026 return data; 1027 } 1028 1029 static const MemoryRegionOps vfio_vga_ops = { 1030 .read = vfio_vga_read, 1031 .write = vfio_vga_write, 1032 .endianness = DEVICE_LITTLE_ENDIAN, 1033 }; 1034 1035 /* 1036 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page 1037 * size if the BAR is in an exclusive page in host so that we could map 1038 * this BAR to guest. But this sub-page BAR may not occupy an exclusive 1039 * page in guest. So we should set the priority of the expanded memory 1040 * region to zero in case of overlap with BARs which share the same page 1041 * with the sub-page BAR in guest. Besides, we should also recover the 1042 * size of this sub-page BAR when its base address is changed in guest 1043 * and not page aligned any more. 1044 */ 1045 static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar) 1046 { 1047 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 1048 VFIORegion *region = &vdev->bars[bar].region; 1049 MemoryRegion *mmap_mr, *region_mr, *base_mr; 1050 PCIIORegion *r; 1051 pcibus_t bar_addr; 1052 uint64_t size = region->size; 1053 1054 /* Make sure that the whole region is allowed to be mmapped */ 1055 if (region->nr_mmaps != 1 || !region->mmaps[0].mmap || 1056 region->mmaps[0].size != region->size) { 1057 return; 1058 } 1059 1060 r = &pdev->io_regions[bar]; 1061 bar_addr = r->addr; 1062 base_mr = vdev->bars[bar].mr; 1063 region_mr = region->mem; 1064 mmap_mr = ®ion->mmaps[0].mem; 1065 1066 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */ 1067 if (bar_addr != PCI_BAR_UNMAPPED && 1068 !(bar_addr & ~qemu_real_host_page_mask)) { 1069 size = qemu_real_host_page_size; 1070 } 1071 1072 memory_region_transaction_begin(); 1073 1074 if (vdev->bars[bar].size < size) { 1075 memory_region_set_size(base_mr, size); 1076 } 1077 memory_region_set_size(region_mr, size); 1078 memory_region_set_size(mmap_mr, size); 1079 if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) { 1080 memory_region_del_subregion(r->address_space, base_mr); 1081 memory_region_add_subregion_overlap(r->address_space, 1082 bar_addr, base_mr, 0); 1083 } 1084 1085 memory_region_transaction_commit(); 1086 } 1087 1088 /* 1089 * PCI config space 1090 */ 1091 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len) 1092 { 1093 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 1094 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val; 1095 1096 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len); 1097 emu_bits = le32_to_cpu(emu_bits); 1098 1099 if (emu_bits) { 1100 emu_val = pci_default_read_config(pdev, addr, len); 1101 } 1102 1103 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) { 1104 ssize_t ret; 1105 1106 ret = pread(vdev->vbasedev.fd, &phys_val, len, 1107 vdev->config_offset + addr); 1108 if (ret != len) { 1109 error_report("%s(%s, 0x%x, 0x%x) failed: %m", 1110 __func__, vdev->vbasedev.name, addr, len); 1111 return -errno; 1112 } 1113 phys_val = le32_to_cpu(phys_val); 1114 } 1115 1116 val = (emu_val & emu_bits) | (phys_val & ~emu_bits); 1117 1118 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val); 1119 1120 return val; 1121 } 1122 1123 void vfio_pci_write_config(PCIDevice *pdev, 1124 uint32_t addr, uint32_t val, int len) 1125 { 1126 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 1127 uint32_t val_le = cpu_to_le32(val); 1128 1129 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len); 1130 1131 /* Write everything to VFIO, let it filter out what we can't write */ 1132 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr) 1133 != len) { 1134 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m", 1135 __func__, vdev->vbasedev.name, addr, val, len); 1136 } 1137 1138 /* MSI/MSI-X Enabling/Disabling */ 1139 if (pdev->cap_present & QEMU_PCI_CAP_MSI && 1140 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) { 1141 int is_enabled, was_enabled = msi_enabled(pdev); 1142 1143 pci_default_write_config(pdev, addr, val, len); 1144 1145 is_enabled = msi_enabled(pdev); 1146 1147 if (!was_enabled) { 1148 if (is_enabled) { 1149 vfio_msi_enable(vdev); 1150 } 1151 } else { 1152 if (!is_enabled) { 1153 vfio_msi_disable(vdev); 1154 } else { 1155 vfio_update_msi(vdev); 1156 } 1157 } 1158 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX && 1159 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) { 1160 int is_enabled, was_enabled = msix_enabled(pdev); 1161 1162 pci_default_write_config(pdev, addr, val, len); 1163 1164 is_enabled = msix_enabled(pdev); 1165 1166 if (!was_enabled && is_enabled) { 1167 vfio_msix_enable(vdev); 1168 } else if (was_enabled && !is_enabled) { 1169 vfio_msix_disable(vdev); 1170 } 1171 } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) || 1172 range_covers_byte(addr, len, PCI_COMMAND)) { 1173 pcibus_t old_addr[PCI_NUM_REGIONS - 1]; 1174 int bar; 1175 1176 for (bar = 0; bar < PCI_ROM_SLOT; bar++) { 1177 old_addr[bar] = pdev->io_regions[bar].addr; 1178 } 1179 1180 pci_default_write_config(pdev, addr, val, len); 1181 1182 for (bar = 0; bar < PCI_ROM_SLOT; bar++) { 1183 if (old_addr[bar] != pdev->io_regions[bar].addr && 1184 vdev->bars[bar].region.size > 0 && 1185 vdev->bars[bar].region.size < qemu_real_host_page_size) { 1186 vfio_sub_page_bar_update_mapping(pdev, bar); 1187 } 1188 } 1189 } else { 1190 /* Write everything to QEMU to keep emulated bits correct */ 1191 pci_default_write_config(pdev, addr, val, len); 1192 } 1193 } 1194 1195 /* 1196 * Interrupt setup 1197 */ 1198 static void vfio_disable_interrupts(VFIOPCIDevice *vdev) 1199 { 1200 /* 1201 * More complicated than it looks. Disabling MSI/X transitions the 1202 * device to INTx mode (if supported). Therefore we need to first 1203 * disable MSI/X and then cleanup by disabling INTx. 1204 */ 1205 if (vdev->interrupt == VFIO_INT_MSIX) { 1206 vfio_msix_disable(vdev); 1207 } else if (vdev->interrupt == VFIO_INT_MSI) { 1208 vfio_msi_disable(vdev); 1209 } 1210 1211 if (vdev->interrupt == VFIO_INT_INTx) { 1212 vfio_intx_disable(vdev); 1213 } 1214 } 1215 1216 static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp) 1217 { 1218 uint16_t ctrl; 1219 bool msi_64bit, msi_maskbit; 1220 int ret, entries; 1221 Error *err = NULL; 1222 1223 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl), 1224 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) { 1225 error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS"); 1226 return -errno; 1227 } 1228 ctrl = le16_to_cpu(ctrl); 1229 1230 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT); 1231 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT); 1232 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1); 1233 1234 trace_vfio_msi_setup(vdev->vbasedev.name, pos); 1235 1236 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err); 1237 if (ret < 0) { 1238 if (ret == -ENOTSUP) { 1239 return 0; 1240 } 1241 error_propagate_prepend(errp, err, "msi_init failed: "); 1242 return ret; 1243 } 1244 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0); 1245 1246 return 0; 1247 } 1248 1249 static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev) 1250 { 1251 off_t start, end; 1252 VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region; 1253 1254 /* 1255 * If the host driver allows mapping of a MSIX data, we are going to 1256 * do map the entire BAR and emulate MSIX table on top of that. 1257 */ 1258 if (vfio_has_region_cap(&vdev->vbasedev, region->nr, 1259 VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) { 1260 return; 1261 } 1262 1263 /* 1264 * We expect to find a single mmap covering the whole BAR, anything else 1265 * means it's either unsupported or already setup. 1266 */ 1267 if (region->nr_mmaps != 1 || region->mmaps[0].offset || 1268 region->size != region->mmaps[0].size) { 1269 return; 1270 } 1271 1272 /* MSI-X table start and end aligned to host page size */ 1273 start = vdev->msix->table_offset & qemu_real_host_page_mask; 1274 end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset + 1275 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE)); 1276 1277 /* 1278 * Does the MSI-X table cover the beginning of the BAR? The whole BAR? 1279 * NB - Host page size is necessarily a power of two and so is the PCI 1280 * BAR (not counting EA yet), therefore if we have host page aligned 1281 * @start and @end, then any remainder of the BAR before or after those 1282 * must be at least host page sized and therefore mmap'able. 1283 */ 1284 if (!start) { 1285 if (end >= region->size) { 1286 region->nr_mmaps = 0; 1287 g_free(region->mmaps); 1288 region->mmaps = NULL; 1289 trace_vfio_msix_fixup(vdev->vbasedev.name, 1290 vdev->msix->table_bar, 0, 0); 1291 } else { 1292 region->mmaps[0].offset = end; 1293 region->mmaps[0].size = region->size - end; 1294 trace_vfio_msix_fixup(vdev->vbasedev.name, 1295 vdev->msix->table_bar, region->mmaps[0].offset, 1296 region->mmaps[0].offset + region->mmaps[0].size); 1297 } 1298 1299 /* Maybe it's aligned at the end of the BAR */ 1300 } else if (end >= region->size) { 1301 region->mmaps[0].size = start; 1302 trace_vfio_msix_fixup(vdev->vbasedev.name, 1303 vdev->msix->table_bar, region->mmaps[0].offset, 1304 region->mmaps[0].offset + region->mmaps[0].size); 1305 1306 /* Otherwise it must split the BAR */ 1307 } else { 1308 region->nr_mmaps = 2; 1309 region->mmaps = g_renew(VFIOMmap, region->mmaps, 2); 1310 1311 memcpy(®ion->mmaps[1], ®ion->mmaps[0], sizeof(VFIOMmap)); 1312 1313 region->mmaps[0].size = start; 1314 trace_vfio_msix_fixup(vdev->vbasedev.name, 1315 vdev->msix->table_bar, region->mmaps[0].offset, 1316 region->mmaps[0].offset + region->mmaps[0].size); 1317 1318 region->mmaps[1].offset = end; 1319 region->mmaps[1].size = region->size - end; 1320 trace_vfio_msix_fixup(vdev->vbasedev.name, 1321 vdev->msix->table_bar, region->mmaps[1].offset, 1322 region->mmaps[1].offset + region->mmaps[1].size); 1323 } 1324 } 1325 1326 static void vfio_pci_relocate_msix(VFIOPCIDevice *vdev, Error **errp) 1327 { 1328 int target_bar = -1; 1329 size_t msix_sz; 1330 1331 if (!vdev->msix || vdev->msix_relo == OFF_AUTOPCIBAR_OFF) { 1332 return; 1333 } 1334 1335 /* The actual minimum size of MSI-X structures */ 1336 msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) + 1337 (QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8); 1338 /* Round up to host pages, we don't want to share a page */ 1339 msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz); 1340 /* PCI BARs must be a power of 2 */ 1341 msix_sz = pow2ceil(msix_sz); 1342 1343 if (vdev->msix_relo == OFF_AUTOPCIBAR_AUTO) { 1344 /* 1345 * TODO: Lookup table for known devices. 1346 * 1347 * Logically we might use an algorithm here to select the BAR adding 1348 * the least additional MMIO space, but we cannot programatically 1349 * predict the driver dependency on BAR ordering or sizing, therefore 1350 * 'auto' becomes a lookup for combinations reported to work. 1351 */ 1352 if (target_bar < 0) { 1353 error_setg(errp, "No automatic MSI-X relocation available for " 1354 "device %04x:%04x", vdev->vendor_id, vdev->device_id); 1355 return; 1356 } 1357 } else { 1358 target_bar = (int)(vdev->msix_relo - OFF_AUTOPCIBAR_BAR0); 1359 } 1360 1361 /* I/O port BARs cannot host MSI-X structures */ 1362 if (vdev->bars[target_bar].ioport) { 1363 error_setg(errp, "Invalid MSI-X relocation BAR %d, " 1364 "I/O port BAR", target_bar); 1365 return; 1366 } 1367 1368 /* Cannot use a BAR in the "shadow" of a 64-bit BAR */ 1369 if (!vdev->bars[target_bar].size && 1370 target_bar > 0 && vdev->bars[target_bar - 1].mem64) { 1371 error_setg(errp, "Invalid MSI-X relocation BAR %d, " 1372 "consumed by 64-bit BAR %d", target_bar, target_bar - 1); 1373 return; 1374 } 1375 1376 /* 2GB max size for 32-bit BARs, cannot double if already > 1G */ 1377 if (vdev->bars[target_bar].size > 1 * GiB && 1378 !vdev->bars[target_bar].mem64) { 1379 error_setg(errp, "Invalid MSI-X relocation BAR %d, " 1380 "no space to extend 32-bit BAR", target_bar); 1381 return; 1382 } 1383 1384 /* 1385 * If adding a new BAR, test if we can make it 64bit. We make it 1386 * prefetchable since QEMU MSI-X emulation has no read side effects 1387 * and doing so makes mapping more flexible. 1388 */ 1389 if (!vdev->bars[target_bar].size) { 1390 if (target_bar < (PCI_ROM_SLOT - 1) && 1391 !vdev->bars[target_bar + 1].size) { 1392 vdev->bars[target_bar].mem64 = true; 1393 vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64; 1394 } 1395 vdev->bars[target_bar].type |= PCI_BASE_ADDRESS_MEM_PREFETCH; 1396 vdev->bars[target_bar].size = msix_sz; 1397 vdev->msix->table_offset = 0; 1398 } else { 1399 vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2, 1400 msix_sz * 2); 1401 /* 1402 * Due to above size calc, MSI-X always starts halfway into the BAR, 1403 * which will always be a separate host page. 1404 */ 1405 vdev->msix->table_offset = vdev->bars[target_bar].size / 2; 1406 } 1407 1408 vdev->msix->table_bar = target_bar; 1409 vdev->msix->pba_bar = target_bar; 1410 /* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */ 1411 vdev->msix->pba_offset = vdev->msix->table_offset + 1412 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE); 1413 1414 trace_vfio_msix_relo(vdev->vbasedev.name, 1415 vdev->msix->table_bar, vdev->msix->table_offset); 1416 } 1417 1418 /* 1419 * We don't have any control over how pci_add_capability() inserts 1420 * capabilities into the chain. In order to setup MSI-X we need a 1421 * MemoryRegion for the BAR. In order to setup the BAR and not 1422 * attempt to mmap the MSI-X table area, which VFIO won't allow, we 1423 * need to first look for where the MSI-X table lives. So we 1424 * unfortunately split MSI-X setup across two functions. 1425 */ 1426 static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp) 1427 { 1428 uint8_t pos; 1429 uint16_t ctrl; 1430 uint32_t table, pba; 1431 int fd = vdev->vbasedev.fd; 1432 VFIOMSIXInfo *msix; 1433 1434 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX); 1435 if (!pos) { 1436 return; 1437 } 1438 1439 if (pread(fd, &ctrl, sizeof(ctrl), 1440 vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) { 1441 error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS"); 1442 return; 1443 } 1444 1445 if (pread(fd, &table, sizeof(table), 1446 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) { 1447 error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE"); 1448 return; 1449 } 1450 1451 if (pread(fd, &pba, sizeof(pba), 1452 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) { 1453 error_setg_errno(errp, errno, "failed to read PCI MSIX PBA"); 1454 return; 1455 } 1456 1457 ctrl = le16_to_cpu(ctrl); 1458 table = le32_to_cpu(table); 1459 pba = le32_to_cpu(pba); 1460 1461 msix = g_malloc0(sizeof(*msix)); 1462 msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK; 1463 msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK; 1464 msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK; 1465 msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK; 1466 msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 1467 1468 /* 1469 * Test the size of the pba_offset variable and catch if it extends outside 1470 * of the specified BAR. If it is the case, we need to apply a hardware 1471 * specific quirk if the device is known or we have a broken configuration. 1472 */ 1473 if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) { 1474 /* 1475 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5 1476 * adapters. The T5 hardware returns an incorrect value of 0x8000 for 1477 * the VF PBA offset while the BAR itself is only 8k. The correct value 1478 * is 0x1000, so we hard code that here. 1479 */ 1480 if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO && 1481 (vdev->device_id & 0xff00) == 0x5800) { 1482 msix->pba_offset = 0x1000; 1483 } else if (vdev->msix_relo == OFF_AUTOPCIBAR_OFF) { 1484 error_setg(errp, "hardware reports invalid configuration, " 1485 "MSIX PBA outside of specified BAR"); 1486 g_free(msix); 1487 return; 1488 } 1489 } 1490 1491 trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar, 1492 msix->table_offset, msix->entries); 1493 vdev->msix = msix; 1494 1495 vfio_pci_fixup_msix_region(vdev); 1496 1497 vfio_pci_relocate_msix(vdev, errp); 1498 } 1499 1500 static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp) 1501 { 1502 int ret; 1503 Error *err = NULL; 1504 1505 vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) * 1506 sizeof(unsigned long)); 1507 ret = msix_init(&vdev->pdev, vdev->msix->entries, 1508 vdev->bars[vdev->msix->table_bar].mr, 1509 vdev->msix->table_bar, vdev->msix->table_offset, 1510 vdev->bars[vdev->msix->pba_bar].mr, 1511 vdev->msix->pba_bar, vdev->msix->pba_offset, pos, 1512 &err); 1513 if (ret < 0) { 1514 if (ret == -ENOTSUP) { 1515 warn_report_err(err); 1516 return 0; 1517 } 1518 1519 error_propagate(errp, err); 1520 return ret; 1521 } 1522 1523 /* 1524 * The PCI spec suggests that devices provide additional alignment for 1525 * MSI-X structures and avoid overlapping non-MSI-X related registers. 1526 * For an assigned device, this hopefully means that emulation of MSI-X 1527 * structures does not affect the performance of the device. If devices 1528 * fail to provide that alignment, a significant performance penalty may 1529 * result, for instance Mellanox MT27500 VFs: 1530 * http://www.spinics.net/lists/kvm/msg125881.html 1531 * 1532 * The PBA is simply not that important for such a serious regression and 1533 * most drivers do not appear to look at it. The solution for this is to 1534 * disable the PBA MemoryRegion unless it's being used. We disable it 1535 * here and only enable it if a masked vector fires through QEMU. As the 1536 * vector-use notifier is called, which occurs on unmask, we test whether 1537 * PBA emulation is needed and again disable if not. 1538 */ 1539 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false); 1540 1541 /* 1542 * The emulated machine may provide a paravirt interface for MSIX setup 1543 * so it is not strictly necessary to emulate MSIX here. This becomes 1544 * helpful when frequently accessed MMIO registers are located in 1545 * subpages adjacent to the MSIX table but the MSIX data containing page 1546 * cannot be mapped because of a host page size bigger than the MSIX table 1547 * alignment. 1548 */ 1549 if (object_property_get_bool(OBJECT(qdev_get_machine()), 1550 "vfio-no-msix-emulation", NULL)) { 1551 memory_region_set_enabled(&vdev->pdev.msix_table_mmio, false); 1552 } 1553 1554 return 0; 1555 } 1556 1557 static void vfio_teardown_msi(VFIOPCIDevice *vdev) 1558 { 1559 msi_uninit(&vdev->pdev); 1560 1561 if (vdev->msix) { 1562 msix_uninit(&vdev->pdev, 1563 vdev->bars[vdev->msix->table_bar].mr, 1564 vdev->bars[vdev->msix->pba_bar].mr); 1565 g_free(vdev->msix->pending); 1566 } 1567 } 1568 1569 /* 1570 * Resource setup 1571 */ 1572 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled) 1573 { 1574 int i; 1575 1576 for (i = 0; i < PCI_ROM_SLOT; i++) { 1577 vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled); 1578 } 1579 } 1580 1581 static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr) 1582 { 1583 VFIOBAR *bar = &vdev->bars[nr]; 1584 1585 uint32_t pci_bar; 1586 int ret; 1587 1588 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */ 1589 if (!bar->region.size) { 1590 return; 1591 } 1592 1593 /* Determine what type of BAR this is for registration */ 1594 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar), 1595 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr)); 1596 if (ret != sizeof(pci_bar)) { 1597 error_report("vfio: Failed to read BAR %d (%m)", nr); 1598 return; 1599 } 1600 1601 pci_bar = le32_to_cpu(pci_bar); 1602 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO); 1603 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64); 1604 bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK : 1605 ~PCI_BASE_ADDRESS_MEM_MASK); 1606 bar->size = bar->region.size; 1607 } 1608 1609 static void vfio_bars_prepare(VFIOPCIDevice *vdev) 1610 { 1611 int i; 1612 1613 for (i = 0; i < PCI_ROM_SLOT; i++) { 1614 vfio_bar_prepare(vdev, i); 1615 } 1616 } 1617 1618 static void vfio_bar_register(VFIOPCIDevice *vdev, int nr) 1619 { 1620 VFIOBAR *bar = &vdev->bars[nr]; 1621 char *name; 1622 1623 if (!bar->size) { 1624 return; 1625 } 1626 1627 bar->mr = g_new0(MemoryRegion, 1); 1628 name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr); 1629 memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size); 1630 g_free(name); 1631 1632 if (bar->region.size) { 1633 memory_region_add_subregion(bar->mr, 0, bar->region.mem); 1634 1635 if (vfio_region_mmap(&bar->region)) { 1636 error_report("Failed to mmap %s BAR %d. Performance may be slow", 1637 vdev->vbasedev.name, nr); 1638 } 1639 } 1640 1641 pci_register_bar(&vdev->pdev, nr, bar->type, bar->mr); 1642 } 1643 1644 static void vfio_bars_register(VFIOPCIDevice *vdev) 1645 { 1646 int i; 1647 1648 for (i = 0; i < PCI_ROM_SLOT; i++) { 1649 vfio_bar_register(vdev, i); 1650 } 1651 } 1652 1653 static void vfio_bars_exit(VFIOPCIDevice *vdev) 1654 { 1655 int i; 1656 1657 for (i = 0; i < PCI_ROM_SLOT; i++) { 1658 VFIOBAR *bar = &vdev->bars[i]; 1659 1660 vfio_bar_quirk_exit(vdev, i); 1661 vfio_region_exit(&bar->region); 1662 if (bar->region.size) { 1663 memory_region_del_subregion(bar->mr, bar->region.mem); 1664 } 1665 } 1666 1667 if (vdev->vga) { 1668 pci_unregister_vga(&vdev->pdev); 1669 vfio_vga_quirk_exit(vdev); 1670 } 1671 } 1672 1673 static void vfio_bars_finalize(VFIOPCIDevice *vdev) 1674 { 1675 int i; 1676 1677 for (i = 0; i < PCI_ROM_SLOT; i++) { 1678 VFIOBAR *bar = &vdev->bars[i]; 1679 1680 vfio_bar_quirk_finalize(vdev, i); 1681 vfio_region_finalize(&bar->region); 1682 if (bar->size) { 1683 object_unparent(OBJECT(bar->mr)); 1684 g_free(bar->mr); 1685 } 1686 } 1687 1688 if (vdev->vga) { 1689 vfio_vga_quirk_finalize(vdev); 1690 for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { 1691 object_unparent(OBJECT(&vdev->vga->region[i].mem)); 1692 } 1693 g_free(vdev->vga); 1694 } 1695 } 1696 1697 /* 1698 * General setup 1699 */ 1700 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos) 1701 { 1702 uint8_t tmp; 1703 uint16_t next = PCI_CONFIG_SPACE_SIZE; 1704 1705 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp; 1706 tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) { 1707 if (tmp > pos && tmp < next) { 1708 next = tmp; 1709 } 1710 } 1711 1712 return next - pos; 1713 } 1714 1715 1716 static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos) 1717 { 1718 uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE; 1719 1720 for (tmp = PCI_CONFIG_SPACE_SIZE; tmp; 1721 tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) { 1722 if (tmp > pos && tmp < next) { 1723 next = tmp; 1724 } 1725 } 1726 1727 return next - pos; 1728 } 1729 1730 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask) 1731 { 1732 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val); 1733 } 1734 1735 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos, 1736 uint16_t val, uint16_t mask) 1737 { 1738 vfio_set_word_bits(vdev->pdev.config + pos, val, mask); 1739 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask); 1740 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask); 1741 } 1742 1743 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask) 1744 { 1745 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val); 1746 } 1747 1748 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos, 1749 uint32_t val, uint32_t mask) 1750 { 1751 vfio_set_long_bits(vdev->pdev.config + pos, val, mask); 1752 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask); 1753 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask); 1754 } 1755 1756 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size, 1757 Error **errp) 1758 { 1759 uint16_t flags; 1760 uint8_t type; 1761 1762 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS); 1763 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4; 1764 1765 if (type != PCI_EXP_TYPE_ENDPOINT && 1766 type != PCI_EXP_TYPE_LEG_END && 1767 type != PCI_EXP_TYPE_RC_END) { 1768 1769 error_setg(errp, "assignment of PCIe type 0x%x " 1770 "devices is not currently supported", type); 1771 return -EINVAL; 1772 } 1773 1774 if (!pci_bus_is_express(pci_get_bus(&vdev->pdev))) { 1775 PCIBus *bus = pci_get_bus(&vdev->pdev); 1776 PCIDevice *bridge; 1777 1778 /* 1779 * Traditionally PCI device assignment exposes the PCIe capability 1780 * as-is on non-express buses. The reason being that some drivers 1781 * simply assume that it's there, for example tg3. However when 1782 * we're running on a native PCIe machine type, like Q35, we need 1783 * to hide the PCIe capability. The reason for this is twofold; 1784 * first Windows guests get a Code 10 error when the PCIe capability 1785 * is exposed in this configuration. Therefore express devices won't 1786 * work at all unless they're attached to express buses in the VM. 1787 * Second, a native PCIe machine introduces the possibility of fine 1788 * granularity IOMMUs supporting both translation and isolation. 1789 * Guest code to discover the IOMMU visibility of a device, such as 1790 * IOMMU grouping code on Linux, is very aware of device types and 1791 * valid transitions between bus types. An express device on a non- 1792 * express bus is not a valid combination on bare metal systems. 1793 * 1794 * Drivers that require a PCIe capability to make the device 1795 * functional are simply going to need to have their devices placed 1796 * on a PCIe bus in the VM. 1797 */ 1798 while (!pci_bus_is_root(bus)) { 1799 bridge = pci_bridge_get_device(bus); 1800 bus = pci_get_bus(bridge); 1801 } 1802 1803 if (pci_bus_is_express(bus)) { 1804 return 0; 1805 } 1806 1807 } else if (pci_bus_is_root(pci_get_bus(&vdev->pdev))) { 1808 /* 1809 * On a Root Complex bus Endpoints become Root Complex Integrated 1810 * Endpoints, which changes the type and clears the LNK & LNK2 fields. 1811 */ 1812 if (type == PCI_EXP_TYPE_ENDPOINT) { 1813 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS, 1814 PCI_EXP_TYPE_RC_END << 4, 1815 PCI_EXP_FLAGS_TYPE); 1816 1817 /* Link Capabilities, Status, and Control goes away */ 1818 if (size > PCI_EXP_LNKCTL) { 1819 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0); 1820 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0); 1821 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0); 1822 1823 #ifndef PCI_EXP_LNKCAP2 1824 #define PCI_EXP_LNKCAP2 44 1825 #endif 1826 #ifndef PCI_EXP_LNKSTA2 1827 #define PCI_EXP_LNKSTA2 50 1828 #endif 1829 /* Link 2 Capabilities, Status, and Control goes away */ 1830 if (size > PCI_EXP_LNKCAP2) { 1831 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0); 1832 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0); 1833 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0); 1834 } 1835 } 1836 1837 } else if (type == PCI_EXP_TYPE_LEG_END) { 1838 /* 1839 * Legacy endpoints don't belong on the root complex. Windows 1840 * seems to be happier with devices if we skip the capability. 1841 */ 1842 return 0; 1843 } 1844 1845 } else { 1846 /* 1847 * Convert Root Complex Integrated Endpoints to regular endpoints. 1848 * These devices don't support LNK/LNK2 capabilities, so make them up. 1849 */ 1850 if (type == PCI_EXP_TYPE_RC_END) { 1851 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS, 1852 PCI_EXP_TYPE_ENDPOINT << 4, 1853 PCI_EXP_FLAGS_TYPE); 1854 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 1855 QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) | 1856 QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0); 1857 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0); 1858 } 1859 } 1860 1861 /* 1862 * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0 1863 * (Niantic errate #35) causing Windows to error with a Code 10 for the 1864 * device on Q35. Fixup any such devices to report version 1. If we 1865 * were to remove the capability entirely the guest would lose extended 1866 * config space. 1867 */ 1868 if ((flags & PCI_EXP_FLAGS_VERS) == 0) { 1869 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS, 1870 1, PCI_EXP_FLAGS_VERS); 1871 } 1872 1873 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size, 1874 errp); 1875 if (pos < 0) { 1876 return pos; 1877 } 1878 1879 vdev->pdev.exp.exp_cap = pos; 1880 1881 return pos; 1882 } 1883 1884 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos) 1885 { 1886 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP); 1887 1888 if (cap & PCI_EXP_DEVCAP_FLR) { 1889 trace_vfio_check_pcie_flr(vdev->vbasedev.name); 1890 vdev->has_flr = true; 1891 } 1892 } 1893 1894 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos) 1895 { 1896 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL); 1897 1898 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) { 1899 trace_vfio_check_pm_reset(vdev->vbasedev.name); 1900 vdev->has_pm_reset = true; 1901 } 1902 } 1903 1904 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos) 1905 { 1906 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP); 1907 1908 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) { 1909 trace_vfio_check_af_flr(vdev->vbasedev.name); 1910 vdev->has_flr = true; 1911 } 1912 } 1913 1914 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp) 1915 { 1916 PCIDevice *pdev = &vdev->pdev; 1917 uint8_t cap_id, next, size; 1918 int ret; 1919 1920 cap_id = pdev->config[pos]; 1921 next = pdev->config[pos + PCI_CAP_LIST_NEXT]; 1922 1923 /* 1924 * If it becomes important to configure capabilities to their actual 1925 * size, use this as the default when it's something we don't recognize. 1926 * Since QEMU doesn't actually handle many of the config accesses, 1927 * exact size doesn't seem worthwhile. 1928 */ 1929 size = vfio_std_cap_max_size(pdev, pos); 1930 1931 /* 1932 * pci_add_capability always inserts the new capability at the head 1933 * of the chain. Therefore to end up with a chain that matches the 1934 * physical device, we insert from the end by making this recursive. 1935 * This is also why we pre-calculate size above as cached config space 1936 * will be changed as we unwind the stack. 1937 */ 1938 if (next) { 1939 ret = vfio_add_std_cap(vdev, next, errp); 1940 if (ret) { 1941 return ret; 1942 } 1943 } else { 1944 /* Begin the rebuild, use QEMU emulated list bits */ 1945 pdev->config[PCI_CAPABILITY_LIST] = 0; 1946 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff; 1947 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST; 1948 1949 ret = vfio_add_virt_caps(vdev, errp); 1950 if (ret) { 1951 return ret; 1952 } 1953 } 1954 1955 /* Scale down size, esp in case virt caps were added above */ 1956 size = MIN(size, vfio_std_cap_max_size(pdev, pos)); 1957 1958 /* Use emulated next pointer to allow dropping caps */ 1959 pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff); 1960 1961 switch (cap_id) { 1962 case PCI_CAP_ID_MSI: 1963 ret = vfio_msi_setup(vdev, pos, errp); 1964 break; 1965 case PCI_CAP_ID_EXP: 1966 vfio_check_pcie_flr(vdev, pos); 1967 ret = vfio_setup_pcie_cap(vdev, pos, size, errp); 1968 break; 1969 case PCI_CAP_ID_MSIX: 1970 ret = vfio_msix_setup(vdev, pos, errp); 1971 break; 1972 case PCI_CAP_ID_PM: 1973 vfio_check_pm_reset(vdev, pos); 1974 vdev->pm_cap = pos; 1975 ret = pci_add_capability(pdev, cap_id, pos, size, errp); 1976 break; 1977 case PCI_CAP_ID_AF: 1978 vfio_check_af_flr(vdev, pos); 1979 ret = pci_add_capability(pdev, cap_id, pos, size, errp); 1980 break; 1981 default: 1982 ret = pci_add_capability(pdev, cap_id, pos, size, errp); 1983 break; 1984 } 1985 1986 if (ret < 0) { 1987 error_prepend(errp, 1988 "failed to add PCI capability 0x%x[0x%x]@0x%x: ", 1989 cap_id, size, pos); 1990 return ret; 1991 } 1992 1993 return 0; 1994 } 1995 1996 static void vfio_add_ext_cap(VFIOPCIDevice *vdev) 1997 { 1998 PCIDevice *pdev = &vdev->pdev; 1999 uint32_t header; 2000 uint16_t cap_id, next, size; 2001 uint8_t cap_ver; 2002 uint8_t *config; 2003 2004 /* Only add extended caps if we have them and the guest can see them */ 2005 if (!pci_is_express(pdev) || !pci_bus_is_express(pci_get_bus(pdev)) || 2006 !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) { 2007 return; 2008 } 2009 2010 /* 2011 * pcie_add_capability always inserts the new capability at the tail 2012 * of the chain. Therefore to end up with a chain that matches the 2013 * physical device, we cache the config space to avoid overwriting 2014 * the original config space when we parse the extended capabilities. 2015 */ 2016 config = g_memdup(pdev->config, vdev->config_size); 2017 2018 /* 2019 * Extended capabilities are chained with each pointing to the next, so we 2020 * can drop anything other than the head of the chain simply by modifying 2021 * the previous next pointer. Seed the head of the chain here such that 2022 * we can simply skip any capabilities we want to drop below, regardless 2023 * of their position in the chain. If this stub capability still exists 2024 * after we add the capabilities we want to expose, update the capability 2025 * ID to zero. Note that we cannot seed with the capability header being 2026 * zero as this conflicts with definition of an absent capability chain 2027 * and prevents capabilities beyond the head of the list from being added. 2028 * By replacing the dummy capability ID with zero after walking the device 2029 * chain, we also transparently mark extended capabilities as absent if 2030 * no capabilities were added. Note that the PCIe spec defines an absence 2031 * of extended capabilities to be determined by a value of zero for the 2032 * capability ID, version, AND next pointer. A non-zero next pointer 2033 * should be sufficient to indicate additional capabilities are present, 2034 * which will occur if we call pcie_add_capability() below. The entire 2035 * first dword is emulated to support this. 2036 * 2037 * NB. The kernel side does similar masking, so be prepared that our 2038 * view of the device may also contain a capability ID zero in the head 2039 * of the chain. Skip it for the same reason that we cannot seed the 2040 * chain with a zero capability. 2041 */ 2042 pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE, 2043 PCI_EXT_CAP(0xFFFF, 0, 0)); 2044 pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0); 2045 pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0); 2046 2047 for (next = PCI_CONFIG_SPACE_SIZE; next; 2048 next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) { 2049 header = pci_get_long(config + next); 2050 cap_id = PCI_EXT_CAP_ID(header); 2051 cap_ver = PCI_EXT_CAP_VER(header); 2052 2053 /* 2054 * If it becomes important to configure extended capabilities to their 2055 * actual size, use this as the default when it's something we don't 2056 * recognize. Since QEMU doesn't actually handle many of the config 2057 * accesses, exact size doesn't seem worthwhile. 2058 */ 2059 size = vfio_ext_cap_max_size(config, next); 2060 2061 /* Use emulated next pointer to allow dropping extended caps */ 2062 pci_long_test_and_set_mask(vdev->emulated_config_bits + next, 2063 PCI_EXT_CAP_NEXT_MASK); 2064 2065 switch (cap_id) { 2066 case 0: /* kernel masked capability */ 2067 case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */ 2068 case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */ 2069 case PCI_EXT_CAP_ID_REBAR: /* Can't expose read-only */ 2070 trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next); 2071 break; 2072 default: 2073 pcie_add_capability(pdev, cap_id, cap_ver, next, size); 2074 } 2075 2076 } 2077 2078 /* Cleanup chain head ID if necessary */ 2079 if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) { 2080 pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0); 2081 } 2082 2083 g_free(config); 2084 return; 2085 } 2086 2087 static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp) 2088 { 2089 PCIDevice *pdev = &vdev->pdev; 2090 int ret; 2091 2092 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) || 2093 !pdev->config[PCI_CAPABILITY_LIST]) { 2094 return 0; /* Nothing to add */ 2095 } 2096 2097 ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp); 2098 if (ret) { 2099 return ret; 2100 } 2101 2102 vfio_add_ext_cap(vdev); 2103 return 0; 2104 } 2105 2106 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev) 2107 { 2108 PCIDevice *pdev = &vdev->pdev; 2109 uint16_t cmd; 2110 2111 vfio_disable_interrupts(vdev); 2112 2113 /* Make sure the device is in D0 */ 2114 if (vdev->pm_cap) { 2115 uint16_t pmcsr; 2116 uint8_t state; 2117 2118 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2); 2119 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 2120 if (state) { 2121 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 2122 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2); 2123 /* vfio handles the necessary delay here */ 2124 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2); 2125 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 2126 if (state) { 2127 error_report("vfio: Unable to power on device, stuck in D%d", 2128 state); 2129 } 2130 } 2131 } 2132 2133 /* 2134 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master. 2135 * Also put INTx Disable in known state. 2136 */ 2137 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2); 2138 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | 2139 PCI_COMMAND_INTX_DISABLE); 2140 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2); 2141 } 2142 2143 static void vfio_pci_post_reset(VFIOPCIDevice *vdev) 2144 { 2145 Error *err = NULL; 2146 int nr; 2147 2148 vfio_intx_enable(vdev, &err); 2149 if (err) { 2150 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 2151 } 2152 2153 for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) { 2154 off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr); 2155 uint32_t val = 0; 2156 uint32_t len = sizeof(val); 2157 2158 if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) { 2159 error_report("%s(%s) reset bar %d failed: %m", __func__, 2160 vdev->vbasedev.name, nr); 2161 } 2162 } 2163 2164 vfio_quirk_reset(vdev); 2165 } 2166 2167 static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name) 2168 { 2169 char tmp[13]; 2170 2171 sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain, 2172 addr->bus, addr->slot, addr->function); 2173 2174 return (strcmp(tmp, name) == 0); 2175 } 2176 2177 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single) 2178 { 2179 VFIOGroup *group; 2180 struct vfio_pci_hot_reset_info *info; 2181 struct vfio_pci_dependent_device *devices; 2182 struct vfio_pci_hot_reset *reset; 2183 int32_t *fds; 2184 int ret, i, count; 2185 bool multi = false; 2186 2187 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi"); 2188 2189 if (!single) { 2190 vfio_pci_pre_reset(vdev); 2191 } 2192 vdev->vbasedev.needs_reset = false; 2193 2194 info = g_malloc0(sizeof(*info)); 2195 info->argsz = sizeof(*info); 2196 2197 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info); 2198 if (ret && errno != ENOSPC) { 2199 ret = -errno; 2200 if (!vdev->has_pm_reset) { 2201 error_report("vfio: Cannot reset device %s, " 2202 "no available reset mechanism.", vdev->vbasedev.name); 2203 } 2204 goto out_single; 2205 } 2206 2207 count = info->count; 2208 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices))); 2209 info->argsz = sizeof(*info) + (count * sizeof(*devices)); 2210 devices = &info->devices[0]; 2211 2212 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info); 2213 if (ret) { 2214 ret = -errno; 2215 error_report("vfio: hot reset info failed: %m"); 2216 goto out_single; 2217 } 2218 2219 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name); 2220 2221 /* Verify that we have all the groups required */ 2222 for (i = 0; i < info->count; i++) { 2223 PCIHostDeviceAddress host; 2224 VFIOPCIDevice *tmp; 2225 VFIODevice *vbasedev_iter; 2226 2227 host.domain = devices[i].segment; 2228 host.bus = devices[i].bus; 2229 host.slot = PCI_SLOT(devices[i].devfn); 2230 host.function = PCI_FUNC(devices[i].devfn); 2231 2232 trace_vfio_pci_hot_reset_dep_devices(host.domain, 2233 host.bus, host.slot, host.function, devices[i].group_id); 2234 2235 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) { 2236 continue; 2237 } 2238 2239 QLIST_FOREACH(group, &vfio_group_list, next) { 2240 if (group->groupid == devices[i].group_id) { 2241 break; 2242 } 2243 } 2244 2245 if (!group) { 2246 if (!vdev->has_pm_reset) { 2247 error_report("vfio: Cannot reset device %s, " 2248 "depends on group %d which is not owned.", 2249 vdev->vbasedev.name, devices[i].group_id); 2250 } 2251 ret = -EPERM; 2252 goto out; 2253 } 2254 2255 /* Prep dependent devices for reset and clear our marker. */ 2256 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) { 2257 if (!vbasedev_iter->dev->realized || 2258 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) { 2259 continue; 2260 } 2261 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev); 2262 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) { 2263 if (single) { 2264 ret = -EINVAL; 2265 goto out_single; 2266 } 2267 vfio_pci_pre_reset(tmp); 2268 tmp->vbasedev.needs_reset = false; 2269 multi = true; 2270 break; 2271 } 2272 } 2273 } 2274 2275 if (!single && !multi) { 2276 ret = -EINVAL; 2277 goto out_single; 2278 } 2279 2280 /* Determine how many group fds need to be passed */ 2281 count = 0; 2282 QLIST_FOREACH(group, &vfio_group_list, next) { 2283 for (i = 0; i < info->count; i++) { 2284 if (group->groupid == devices[i].group_id) { 2285 count++; 2286 break; 2287 } 2288 } 2289 } 2290 2291 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds))); 2292 reset->argsz = sizeof(*reset) + (count * sizeof(*fds)); 2293 fds = &reset->group_fds[0]; 2294 2295 /* Fill in group fds */ 2296 QLIST_FOREACH(group, &vfio_group_list, next) { 2297 for (i = 0; i < info->count; i++) { 2298 if (group->groupid == devices[i].group_id) { 2299 fds[reset->count++] = group->fd; 2300 break; 2301 } 2302 } 2303 } 2304 2305 /* Bus reset! */ 2306 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset); 2307 g_free(reset); 2308 2309 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name, 2310 ret ? "%m" : "Success"); 2311 2312 out: 2313 /* Re-enable INTx on affected devices */ 2314 for (i = 0; i < info->count; i++) { 2315 PCIHostDeviceAddress host; 2316 VFIOPCIDevice *tmp; 2317 VFIODevice *vbasedev_iter; 2318 2319 host.domain = devices[i].segment; 2320 host.bus = devices[i].bus; 2321 host.slot = PCI_SLOT(devices[i].devfn); 2322 host.function = PCI_FUNC(devices[i].devfn); 2323 2324 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) { 2325 continue; 2326 } 2327 2328 QLIST_FOREACH(group, &vfio_group_list, next) { 2329 if (group->groupid == devices[i].group_id) { 2330 break; 2331 } 2332 } 2333 2334 if (!group) { 2335 break; 2336 } 2337 2338 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) { 2339 if (!vbasedev_iter->dev->realized || 2340 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) { 2341 continue; 2342 } 2343 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev); 2344 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) { 2345 vfio_pci_post_reset(tmp); 2346 break; 2347 } 2348 } 2349 } 2350 out_single: 2351 if (!single) { 2352 vfio_pci_post_reset(vdev); 2353 } 2354 g_free(info); 2355 2356 return ret; 2357 } 2358 2359 /* 2360 * We want to differentiate hot reset of mulitple in-use devices vs hot reset 2361 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case 2362 * of doing hot resets when there is only a single device per bus. The in-use 2363 * here refers to how many VFIODevices are affected. A hot reset that affects 2364 * multiple devices, but only a single in-use device, means that we can call 2365 * it from our bus ->reset() callback since the extent is effectively a single 2366 * device. This allows us to make use of it in the hotplug path. When there 2367 * are multiple in-use devices, we can only trigger the hot reset during a 2368 * system reset and thus from our reset handler. We separate _one vs _multi 2369 * here so that we don't overlap and do a double reset on the system reset 2370 * path where both our reset handler and ->reset() callback are used. Calling 2371 * _one() will only do a hot reset for the one in-use devices case, calling 2372 * _multi() will do nothing if a _one() would have been sufficient. 2373 */ 2374 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev) 2375 { 2376 return vfio_pci_hot_reset(vdev, true); 2377 } 2378 2379 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev) 2380 { 2381 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 2382 return vfio_pci_hot_reset(vdev, false); 2383 } 2384 2385 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev) 2386 { 2387 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 2388 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) { 2389 vbasedev->needs_reset = true; 2390 } 2391 } 2392 2393 static VFIODeviceOps vfio_pci_ops = { 2394 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset, 2395 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi, 2396 .vfio_eoi = vfio_intx_eoi, 2397 }; 2398 2399 int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp) 2400 { 2401 VFIODevice *vbasedev = &vdev->vbasedev; 2402 struct vfio_region_info *reg_info; 2403 int ret; 2404 2405 ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, ®_info); 2406 if (ret) { 2407 error_setg_errno(errp, -ret, 2408 "failed getting region info for VGA region index %d", 2409 VFIO_PCI_VGA_REGION_INDEX); 2410 return ret; 2411 } 2412 2413 if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) || 2414 !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) || 2415 reg_info->size < 0xbffff + 1) { 2416 error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx", 2417 (unsigned long)reg_info->flags, 2418 (unsigned long)reg_info->size); 2419 g_free(reg_info); 2420 return -EINVAL; 2421 } 2422 2423 vdev->vga = g_new0(VFIOVGA, 1); 2424 2425 vdev->vga->fd_offset = reg_info->offset; 2426 vdev->vga->fd = vdev->vbasedev.fd; 2427 2428 g_free(reg_info); 2429 2430 vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE; 2431 vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM; 2432 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks); 2433 2434 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem, 2435 OBJECT(vdev), &vfio_vga_ops, 2436 &vdev->vga->region[QEMU_PCI_VGA_MEM], 2437 "vfio-vga-mmio@0xa0000", 2438 QEMU_PCI_VGA_MEM_SIZE); 2439 2440 vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE; 2441 vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO; 2442 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks); 2443 2444 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem, 2445 OBJECT(vdev), &vfio_vga_ops, 2446 &vdev->vga->region[QEMU_PCI_VGA_IO_LO], 2447 "vfio-vga-io@0x3b0", 2448 QEMU_PCI_VGA_IO_LO_SIZE); 2449 2450 vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE; 2451 vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI; 2452 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks); 2453 2454 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, 2455 OBJECT(vdev), &vfio_vga_ops, 2456 &vdev->vga->region[QEMU_PCI_VGA_IO_HI], 2457 "vfio-vga-io@0x3c0", 2458 QEMU_PCI_VGA_IO_HI_SIZE); 2459 2460 pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem, 2461 &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem, 2462 &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem); 2463 2464 return 0; 2465 } 2466 2467 static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp) 2468 { 2469 VFIODevice *vbasedev = &vdev->vbasedev; 2470 struct vfio_region_info *reg_info; 2471 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) }; 2472 int i, ret = -1; 2473 2474 /* Sanity check device */ 2475 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) { 2476 error_setg(errp, "this isn't a PCI device"); 2477 return; 2478 } 2479 2480 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) { 2481 error_setg(errp, "unexpected number of io regions %u", 2482 vbasedev->num_regions); 2483 return; 2484 } 2485 2486 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) { 2487 error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs); 2488 return; 2489 } 2490 2491 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) { 2492 char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i); 2493 2494 ret = vfio_region_setup(OBJECT(vdev), vbasedev, 2495 &vdev->bars[i].region, i, name); 2496 g_free(name); 2497 2498 if (ret) { 2499 error_setg_errno(errp, -ret, "failed to get region %d info", i); 2500 return; 2501 } 2502 2503 QLIST_INIT(&vdev->bars[i].quirks); 2504 } 2505 2506 ret = vfio_get_region_info(vbasedev, 2507 VFIO_PCI_CONFIG_REGION_INDEX, ®_info); 2508 if (ret) { 2509 error_setg_errno(errp, -ret, "failed to get config info"); 2510 return; 2511 } 2512 2513 trace_vfio_populate_device_config(vdev->vbasedev.name, 2514 (unsigned long)reg_info->size, 2515 (unsigned long)reg_info->offset, 2516 (unsigned long)reg_info->flags); 2517 2518 vdev->config_size = reg_info->size; 2519 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) { 2520 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS; 2521 } 2522 vdev->config_offset = reg_info->offset; 2523 2524 g_free(reg_info); 2525 2526 if (vdev->features & VFIO_FEATURE_ENABLE_VGA) { 2527 ret = vfio_populate_vga(vdev, errp); 2528 if (ret) { 2529 error_append_hint(errp, "device does not support " 2530 "requested feature x-vga\n"); 2531 return; 2532 } 2533 } 2534 2535 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX; 2536 2537 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); 2538 if (ret) { 2539 /* This can fail for an old kernel or legacy PCI dev */ 2540 trace_vfio_populate_device_get_irq_info_failure(strerror(errno)); 2541 } else if (irq_info.count == 1) { 2542 vdev->pci_aer = true; 2543 } else { 2544 warn_report(VFIO_MSG_PREFIX 2545 "Could not enable error recovery for the device", 2546 vbasedev->name); 2547 } 2548 } 2549 2550 static void vfio_put_device(VFIOPCIDevice *vdev) 2551 { 2552 g_free(vdev->vbasedev.name); 2553 g_free(vdev->msix); 2554 2555 vfio_put_base_device(&vdev->vbasedev); 2556 } 2557 2558 static void vfio_err_notifier_handler(void *opaque) 2559 { 2560 VFIOPCIDevice *vdev = opaque; 2561 2562 if (!event_notifier_test_and_clear(&vdev->err_notifier)) { 2563 return; 2564 } 2565 2566 /* 2567 * TBD. Retrieve the error details and decide what action 2568 * needs to be taken. One of the actions could be to pass 2569 * the error to the guest and have the guest driver recover 2570 * from the error. This requires that PCIe capabilities be 2571 * exposed to the guest. For now, we just terminate the 2572 * guest to contain the error. 2573 */ 2574 2575 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name); 2576 2577 vm_stop(RUN_STATE_INTERNAL_ERROR); 2578 } 2579 2580 /* 2581 * Registers error notifier for devices supporting error recovery. 2582 * If we encounter a failure in this function, we report an error 2583 * and continue after disabling error recovery support for the 2584 * device. 2585 */ 2586 static void vfio_register_err_notifier(VFIOPCIDevice *vdev) 2587 { 2588 Error *err = NULL; 2589 int32_t fd; 2590 2591 if (!vdev->pci_aer) { 2592 return; 2593 } 2594 2595 if (event_notifier_init(&vdev->err_notifier, 0)) { 2596 error_report("vfio: Unable to init event notifier for error detection"); 2597 vdev->pci_aer = false; 2598 return; 2599 } 2600 2601 fd = event_notifier_get_fd(&vdev->err_notifier); 2602 qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev); 2603 2604 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0, 2605 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { 2606 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 2607 qemu_set_fd_handler(fd, NULL, NULL, vdev); 2608 event_notifier_cleanup(&vdev->err_notifier); 2609 vdev->pci_aer = false; 2610 } 2611 } 2612 2613 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev) 2614 { 2615 Error *err = NULL; 2616 2617 if (!vdev->pci_aer) { 2618 return; 2619 } 2620 2621 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0, 2622 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) { 2623 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 2624 } 2625 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier), 2626 NULL, NULL, vdev); 2627 event_notifier_cleanup(&vdev->err_notifier); 2628 } 2629 2630 static void vfio_req_notifier_handler(void *opaque) 2631 { 2632 VFIOPCIDevice *vdev = opaque; 2633 Error *err = NULL; 2634 2635 if (!event_notifier_test_and_clear(&vdev->req_notifier)) { 2636 return; 2637 } 2638 2639 qdev_unplug(DEVICE(vdev), &err); 2640 if (err) { 2641 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 2642 } 2643 } 2644 2645 static void vfio_register_req_notifier(VFIOPCIDevice *vdev) 2646 { 2647 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info), 2648 .index = VFIO_PCI_REQ_IRQ_INDEX }; 2649 Error *err = NULL; 2650 int32_t fd; 2651 2652 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) { 2653 return; 2654 } 2655 2656 if (ioctl(vdev->vbasedev.fd, 2657 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) { 2658 return; 2659 } 2660 2661 if (event_notifier_init(&vdev->req_notifier, 0)) { 2662 error_report("vfio: Unable to init event notifier for device request"); 2663 return; 2664 } 2665 2666 fd = event_notifier_get_fd(&vdev->req_notifier); 2667 qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev); 2668 2669 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0, 2670 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { 2671 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 2672 qemu_set_fd_handler(fd, NULL, NULL, vdev); 2673 event_notifier_cleanup(&vdev->req_notifier); 2674 } else { 2675 vdev->req_enabled = true; 2676 } 2677 } 2678 2679 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev) 2680 { 2681 Error *err = NULL; 2682 2683 if (!vdev->req_enabled) { 2684 return; 2685 } 2686 2687 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0, 2688 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) { 2689 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); 2690 } 2691 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier), 2692 NULL, NULL, vdev); 2693 event_notifier_cleanup(&vdev->req_notifier); 2694 2695 vdev->req_enabled = false; 2696 } 2697 2698 static void vfio_realize(PCIDevice *pdev, Error **errp) 2699 { 2700 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 2701 VFIODevice *vbasedev_iter; 2702 VFIOGroup *group; 2703 char *tmp, *subsys, group_path[PATH_MAX], *group_name; 2704 Error *err = NULL; 2705 ssize_t len; 2706 struct stat st; 2707 int groupid; 2708 int i, ret; 2709 bool is_mdev; 2710 2711 if (!vdev->vbasedev.sysfsdev) { 2712 if (!(~vdev->host.domain || ~vdev->host.bus || 2713 ~vdev->host.slot || ~vdev->host.function)) { 2714 error_setg(errp, "No provided host device"); 2715 error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F " 2716 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n"); 2717 return; 2718 } 2719 vdev->vbasedev.sysfsdev = 2720 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x", 2721 vdev->host.domain, vdev->host.bus, 2722 vdev->host.slot, vdev->host.function); 2723 } 2724 2725 if (stat(vdev->vbasedev.sysfsdev, &st) < 0) { 2726 error_setg_errno(errp, errno, "no such host device"); 2727 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.sysfsdev); 2728 return; 2729 } 2730 2731 vdev->vbasedev.name = g_path_get_basename(vdev->vbasedev.sysfsdev); 2732 vdev->vbasedev.ops = &vfio_pci_ops; 2733 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI; 2734 vdev->vbasedev.dev = DEVICE(vdev); 2735 2736 tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev); 2737 len = readlink(tmp, group_path, sizeof(group_path)); 2738 g_free(tmp); 2739 2740 if (len <= 0 || len >= sizeof(group_path)) { 2741 error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG, 2742 "no iommu_group found"); 2743 goto error; 2744 } 2745 2746 group_path[len] = 0; 2747 2748 group_name = basename(group_path); 2749 if (sscanf(group_name, "%d", &groupid) != 1) { 2750 error_setg_errno(errp, errno, "failed to read %s", group_path); 2751 goto error; 2752 } 2753 2754 trace_vfio_realize(vdev->vbasedev.name, groupid); 2755 2756 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp); 2757 if (!group) { 2758 goto error; 2759 } 2760 2761 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) { 2762 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) { 2763 error_setg(errp, "device is already attached"); 2764 vfio_put_group(group); 2765 goto error; 2766 } 2767 } 2768 2769 /* 2770 * Mediated devices *might* operate compatibly with memory ballooning, but 2771 * we cannot know for certain, it depends on whether the mdev vendor driver 2772 * stays in sync with the active working set of the guest driver. Prevent 2773 * the x-balloon-allowed option unless this is minimally an mdev device. 2774 */ 2775 tmp = g_strdup_printf("%s/subsystem", vdev->vbasedev.sysfsdev); 2776 subsys = realpath(tmp, NULL); 2777 g_free(tmp); 2778 is_mdev = subsys && (strcmp(subsys, "/sys/bus/mdev") == 0); 2779 free(subsys); 2780 2781 trace_vfio_mdev(vdev->vbasedev.name, is_mdev); 2782 2783 if (vdev->vbasedev.balloon_allowed && !is_mdev) { 2784 error_setg(errp, "x-balloon-allowed only potentially compatible " 2785 "with mdev devices"); 2786 vfio_put_group(group); 2787 goto error; 2788 } 2789 2790 ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp); 2791 if (ret) { 2792 vfio_put_group(group); 2793 goto error; 2794 } 2795 2796 vfio_populate_device(vdev, &err); 2797 if (err) { 2798 error_propagate(errp, err); 2799 goto error; 2800 } 2801 2802 /* Get a copy of config space */ 2803 ret = pread(vdev->vbasedev.fd, vdev->pdev.config, 2804 MIN(pci_config_size(&vdev->pdev), vdev->config_size), 2805 vdev->config_offset); 2806 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) { 2807 ret = ret < 0 ? -errno : -EFAULT; 2808 error_setg_errno(errp, -ret, "failed to read device config space"); 2809 goto error; 2810 } 2811 2812 /* vfio emulates a lot for us, but some bits need extra love */ 2813 vdev->emulated_config_bits = g_malloc0(vdev->config_size); 2814 2815 /* QEMU can choose to expose the ROM or not */ 2816 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4); 2817 /* QEMU can also add or extend BARs */ 2818 memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4); 2819 2820 /* 2821 * The PCI spec reserves vendor ID 0xffff as an invalid value. The 2822 * device ID is managed by the vendor and need only be a 16-bit value. 2823 * Allow any 16-bit value for subsystem so they can be hidden or changed. 2824 */ 2825 if (vdev->vendor_id != PCI_ANY_ID) { 2826 if (vdev->vendor_id >= 0xffff) { 2827 error_setg(errp, "invalid PCI vendor ID provided"); 2828 goto error; 2829 } 2830 vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0); 2831 trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id); 2832 } else { 2833 vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID); 2834 } 2835 2836 if (vdev->device_id != PCI_ANY_ID) { 2837 if (vdev->device_id > 0xffff) { 2838 error_setg(errp, "invalid PCI device ID provided"); 2839 goto error; 2840 } 2841 vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0); 2842 trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id); 2843 } else { 2844 vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID); 2845 } 2846 2847 if (vdev->sub_vendor_id != PCI_ANY_ID) { 2848 if (vdev->sub_vendor_id > 0xffff) { 2849 error_setg(errp, "invalid PCI subsystem vendor ID provided"); 2850 goto error; 2851 } 2852 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID, 2853 vdev->sub_vendor_id, ~0); 2854 trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name, 2855 vdev->sub_vendor_id); 2856 } 2857 2858 if (vdev->sub_device_id != PCI_ANY_ID) { 2859 if (vdev->sub_device_id > 0xffff) { 2860 error_setg(errp, "invalid PCI subsystem device ID provided"); 2861 goto error; 2862 } 2863 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0); 2864 trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name, 2865 vdev->sub_device_id); 2866 } 2867 2868 /* QEMU can change multi-function devices to single function, or reverse */ 2869 vdev->emulated_config_bits[PCI_HEADER_TYPE] = 2870 PCI_HEADER_TYPE_MULTI_FUNCTION; 2871 2872 /* Restore or clear multifunction, this is always controlled by QEMU */ 2873 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) { 2874 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION; 2875 } else { 2876 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION; 2877 } 2878 2879 /* 2880 * Clear host resource mapping info. If we choose not to register a 2881 * BAR, such as might be the case with the option ROM, we can get 2882 * confusing, unwritable, residual addresses from the host here. 2883 */ 2884 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24); 2885 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4); 2886 2887 vfio_pci_size_rom(vdev); 2888 2889 vfio_bars_prepare(vdev); 2890 2891 vfio_msix_early_setup(vdev, &err); 2892 if (err) { 2893 error_propagate(errp, err); 2894 goto error; 2895 } 2896 2897 vfio_bars_register(vdev); 2898 2899 ret = vfio_add_capabilities(vdev, errp); 2900 if (ret) { 2901 goto out_teardown; 2902 } 2903 2904 if (vdev->vga) { 2905 vfio_vga_quirk_setup(vdev); 2906 } 2907 2908 for (i = 0; i < PCI_ROM_SLOT; i++) { 2909 vfio_bar_quirk_setup(vdev, i); 2910 } 2911 2912 if (!vdev->igd_opregion && 2913 vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) { 2914 struct vfio_region_info *opregion; 2915 2916 if (vdev->pdev.qdev.hotplugged) { 2917 error_setg(errp, 2918 "cannot support IGD OpRegion feature on hotplugged " 2919 "device"); 2920 goto out_teardown; 2921 } 2922 2923 ret = vfio_get_dev_region_info(&vdev->vbasedev, 2924 VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, 2925 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion); 2926 if (ret) { 2927 error_setg_errno(errp, -ret, 2928 "does not support requested IGD OpRegion feature"); 2929 goto out_teardown; 2930 } 2931 2932 ret = vfio_pci_igd_opregion_init(vdev, opregion, errp); 2933 g_free(opregion); 2934 if (ret) { 2935 goto out_teardown; 2936 } 2937 } 2938 2939 /* QEMU emulates all of MSI & MSIX */ 2940 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) { 2941 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff, 2942 MSIX_CAP_LENGTH); 2943 } 2944 2945 if (pdev->cap_present & QEMU_PCI_CAP_MSI) { 2946 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff, 2947 vdev->msi_cap_size); 2948 } 2949 2950 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) { 2951 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, 2952 vfio_intx_mmap_enable, vdev); 2953 pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update); 2954 ret = vfio_intx_enable(vdev, errp); 2955 if (ret) { 2956 goto out_teardown; 2957 } 2958 } 2959 2960 if (vdev->display != ON_OFF_AUTO_OFF) { 2961 ret = vfio_display_probe(vdev, errp); 2962 if (ret) { 2963 goto out_teardown; 2964 } 2965 } 2966 if (vdev->enable_ramfb && vdev->dpy == NULL) { 2967 error_setg(errp, "ramfb=on requires display=on"); 2968 goto out_teardown; 2969 } 2970 if (vdev->display_xres || vdev->display_yres) { 2971 if (vdev->dpy == NULL) { 2972 error_setg(errp, "xres and yres properties require display=on"); 2973 goto out_teardown; 2974 } 2975 if (vdev->dpy->edid_regs == NULL) { 2976 error_setg(errp, "xres and yres properties need edid support"); 2977 goto out_teardown; 2978 } 2979 } 2980 2981 if (vdev->vendor_id == PCI_VENDOR_ID_NVIDIA) { 2982 ret = vfio_pci_nvidia_v100_ram_init(vdev, errp); 2983 if (ret && ret != -ENODEV) { 2984 error_report("Failed to setup NVIDIA V100 GPU RAM"); 2985 } 2986 } 2987 2988 if (vdev->vendor_id == PCI_VENDOR_ID_IBM) { 2989 ret = vfio_pci_nvlink2_init(vdev, errp); 2990 if (ret && ret != -ENODEV) { 2991 error_report("Failed to setup NVlink2 bridge"); 2992 } 2993 } 2994 2995 vfio_register_err_notifier(vdev); 2996 vfio_register_req_notifier(vdev); 2997 vfio_setup_resetfn_quirk(vdev); 2998 2999 return; 3000 3001 out_teardown: 3002 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL); 3003 vfio_teardown_msi(vdev); 3004 vfio_bars_exit(vdev); 3005 error: 3006 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.name); 3007 } 3008 3009 static void vfio_instance_finalize(Object *obj) 3010 { 3011 VFIOPCIDevice *vdev = PCI_VFIO(obj); 3012 VFIOGroup *group = vdev->vbasedev.group; 3013 3014 vfio_display_finalize(vdev); 3015 vfio_bars_finalize(vdev); 3016 g_free(vdev->emulated_config_bits); 3017 g_free(vdev->rom); 3018 /* 3019 * XXX Leaking igd_opregion is not an oversight, we can't remove the 3020 * fw_cfg entry therefore leaking this allocation seems like the safest 3021 * option. 3022 * 3023 * g_free(vdev->igd_opregion); 3024 */ 3025 vfio_put_device(vdev); 3026 vfio_put_group(group); 3027 } 3028 3029 static void vfio_exitfn(PCIDevice *pdev) 3030 { 3031 VFIOPCIDevice *vdev = PCI_VFIO(pdev); 3032 3033 vfio_unregister_req_notifier(vdev); 3034 vfio_unregister_err_notifier(vdev); 3035 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL); 3036 vfio_disable_interrupts(vdev); 3037 if (vdev->intx.mmap_timer) { 3038 timer_free(vdev->intx.mmap_timer); 3039 } 3040 vfio_teardown_msi(vdev); 3041 vfio_bars_exit(vdev); 3042 } 3043 3044 static void vfio_pci_reset(DeviceState *dev) 3045 { 3046 VFIOPCIDevice *vdev = PCI_VFIO(dev); 3047 3048 trace_vfio_pci_reset(vdev->vbasedev.name); 3049 3050 vfio_pci_pre_reset(vdev); 3051 3052 if (vdev->display != ON_OFF_AUTO_OFF) { 3053 vfio_display_reset(vdev); 3054 } 3055 3056 if (vdev->resetfn && !vdev->resetfn(vdev)) { 3057 goto post_reset; 3058 } 3059 3060 if (vdev->vbasedev.reset_works && 3061 (vdev->has_flr || !vdev->has_pm_reset) && 3062 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) { 3063 trace_vfio_pci_reset_flr(vdev->vbasedev.name); 3064 goto post_reset; 3065 } 3066 3067 /* See if we can do our own bus reset */ 3068 if (!vfio_pci_hot_reset_one(vdev)) { 3069 goto post_reset; 3070 } 3071 3072 /* If nothing else works and the device supports PM reset, use it */ 3073 if (vdev->vbasedev.reset_works && vdev->has_pm_reset && 3074 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) { 3075 trace_vfio_pci_reset_pm(vdev->vbasedev.name); 3076 goto post_reset; 3077 } 3078 3079 post_reset: 3080 vfio_pci_post_reset(vdev); 3081 } 3082 3083 static void vfio_instance_init(Object *obj) 3084 { 3085 PCIDevice *pci_dev = PCI_DEVICE(obj); 3086 VFIOPCIDevice *vdev = PCI_VFIO(obj); 3087 3088 device_add_bootindex_property(obj, &vdev->bootindex, 3089 "bootindex", NULL, 3090 &pci_dev->qdev, NULL); 3091 vdev->host.domain = ~0U; 3092 vdev->host.bus = ~0U; 3093 vdev->host.slot = ~0U; 3094 vdev->host.function = ~0U; 3095 3096 vdev->nv_gpudirect_clique = 0xFF; 3097 3098 /* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command 3099 * line, therefore, no need to wait to realize like other devices */ 3100 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS; 3101 } 3102 3103 static Property vfio_pci_dev_properties[] = { 3104 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host), 3105 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev), 3106 DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice, 3107 display, ON_OFF_AUTO_OFF), 3108 DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0), 3109 DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0), 3110 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice, 3111 intx.mmap_timeout, 1100), 3112 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features, 3113 VFIO_FEATURE_ENABLE_VGA_BIT, false), 3114 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features, 3115 VFIO_FEATURE_ENABLE_REQ_BIT, true), 3116 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features, 3117 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false), 3118 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false), 3119 DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice, 3120 vbasedev.balloon_allowed, false), 3121 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false), 3122 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false), 3123 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false), 3124 DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice, 3125 no_geforce_quirks, false), 3126 DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd, 3127 false), 3128 DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd, 3129 false), 3130 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID), 3131 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID), 3132 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice, 3133 sub_vendor_id, PCI_ANY_ID), 3134 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice, 3135 sub_device_id, PCI_ANY_ID), 3136 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0), 3137 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice, 3138 nv_gpudirect_clique, 3139 qdev_prop_nv_gpudirect_clique, uint8_t), 3140 DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo, 3141 OFF_AUTOPCIBAR_OFF), 3142 /* 3143 * TODO - support passed fds... is this necessary? 3144 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name), 3145 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name), 3146 */ 3147 DEFINE_PROP_END_OF_LIST(), 3148 }; 3149 3150 static const VMStateDescription vfio_pci_vmstate = { 3151 .name = "vfio-pci", 3152 .unmigratable = 1, 3153 }; 3154 3155 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data) 3156 { 3157 DeviceClass *dc = DEVICE_CLASS(klass); 3158 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass); 3159 3160 dc->reset = vfio_pci_reset; 3161 dc->props = vfio_pci_dev_properties; 3162 dc->vmsd = &vfio_pci_vmstate; 3163 dc->desc = "VFIO-based PCI device assignment"; 3164 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 3165 pdc->realize = vfio_realize; 3166 pdc->exit = vfio_exitfn; 3167 pdc->config_read = vfio_pci_read_config; 3168 pdc->config_write = vfio_pci_write_config; 3169 } 3170 3171 static const TypeInfo vfio_pci_dev_info = { 3172 .name = TYPE_VFIO_PCI, 3173 .parent = TYPE_PCI_DEVICE, 3174 .instance_size = sizeof(VFIOPCIDevice), 3175 .class_init = vfio_pci_dev_class_init, 3176 .instance_init = vfio_instance_init, 3177 .instance_finalize = vfio_instance_finalize, 3178 .interfaces = (InterfaceInfo[]) { 3179 { INTERFACE_PCIE_DEVICE }, 3180 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 3181 { } 3182 }, 3183 }; 3184 3185 static Property vfio_pci_dev_nohotplug_properties[] = { 3186 DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false), 3187 DEFINE_PROP_END_OF_LIST(), 3188 }; 3189 3190 static void vfio_pci_nohotplug_dev_class_init(ObjectClass *klass, void *data) 3191 { 3192 DeviceClass *dc = DEVICE_CLASS(klass); 3193 3194 dc->props = vfio_pci_dev_nohotplug_properties; 3195 dc->hotpluggable = false; 3196 } 3197 3198 static const TypeInfo vfio_pci_nohotplug_dev_info = { 3199 .name = TYPE_VIFO_PCI_NOHOTPLUG, 3200 .parent = TYPE_VFIO_PCI, 3201 .instance_size = sizeof(VFIOPCIDevice), 3202 .class_init = vfio_pci_nohotplug_dev_class_init, 3203 }; 3204 3205 static void register_vfio_pci_dev_type(void) 3206 { 3207 type_register_static(&vfio_pci_dev_info); 3208 type_register_static(&vfio_pci_nohotplug_dev_info); 3209 } 3210 3211 type_init(register_vfio_pci_dev_type) 3212