1 /* 2 * Virtio PCI Bindings 3 * 4 * Copyright IBM, Corp. 2007 5 * Copyright (c) 2009 CodeSourcery 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * Paul Brook <paul@codesourcery.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2. See 12 * the COPYING file in the top-level directory. 13 * 14 * Contributions after 2012-01-13 are licensed under the terms of the 15 * GNU GPL, version 2 or (at your option) any later version. 16 */ 17 18 #include "qemu/osdep.h" 19 20 #include "exec/memop.h" 21 #include "standard-headers/linux/virtio_pci.h" 22 #include "hw/boards.h" 23 #include "hw/virtio/virtio.h" 24 #include "migration/qemu-file-types.h" 25 #include "hw/pci/pci.h" 26 #include "hw/pci/pci_bus.h" 27 #include "hw/qdev-properties.h" 28 #include "qapi/error.h" 29 #include "qemu/error-report.h" 30 #include "qemu/log.h" 31 #include "qemu/module.h" 32 #include "hw/pci/msi.h" 33 #include "hw/pci/msix.h" 34 #include "hw/loader.h" 35 #include "sysemu/kvm.h" 36 #include "virtio-pci.h" 37 #include "qemu/range.h" 38 #include "hw/virtio/virtio-bus.h" 39 #include "qapi/visitor.h" 40 #include "sysemu/replay.h" 41 42 #define VIRTIO_PCI_REGION_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_present(dev)) 43 44 #undef VIRTIO_PCI_CONFIG 45 46 /* The remaining space is defined by each driver as the per-driver 47 * configuration space */ 48 #define VIRTIO_PCI_CONFIG_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev)) 49 50 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size, 51 VirtIOPCIProxy *dev); 52 static void virtio_pci_reset(DeviceState *qdev); 53 54 /* virtio device */ 55 /* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */ 56 static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d) 57 { 58 return container_of(d, VirtIOPCIProxy, pci_dev.qdev); 59 } 60 61 /* DeviceState to VirtIOPCIProxy. Note: used on datapath, 62 * be careful and test performance if you change this. 63 */ 64 static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d) 65 { 66 return container_of(d, VirtIOPCIProxy, pci_dev.qdev); 67 } 68 69 static void virtio_pci_notify(DeviceState *d, uint16_t vector) 70 { 71 VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d); 72 73 if (msix_enabled(&proxy->pci_dev)) 74 msix_notify(&proxy->pci_dev, vector); 75 else { 76 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 77 pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1); 78 } 79 } 80 81 static void virtio_pci_save_config(DeviceState *d, QEMUFile *f) 82 { 83 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 84 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 85 86 pci_device_save(&proxy->pci_dev, f); 87 msix_save(&proxy->pci_dev, f); 88 if (msix_present(&proxy->pci_dev)) 89 qemu_put_be16(f, vdev->config_vector); 90 } 91 92 static const VMStateDescription vmstate_virtio_pci_modern_queue_state = { 93 .name = "virtio_pci/modern_queue_state", 94 .version_id = 1, 95 .minimum_version_id = 1, 96 .fields = (VMStateField[]) { 97 VMSTATE_UINT16(num, VirtIOPCIQueue), 98 VMSTATE_UNUSED(1), /* enabled was stored as be16 */ 99 VMSTATE_BOOL(enabled, VirtIOPCIQueue), 100 VMSTATE_UINT32_ARRAY(desc, VirtIOPCIQueue, 2), 101 VMSTATE_UINT32_ARRAY(avail, VirtIOPCIQueue, 2), 102 VMSTATE_UINT32_ARRAY(used, VirtIOPCIQueue, 2), 103 VMSTATE_END_OF_LIST() 104 } 105 }; 106 107 static bool virtio_pci_modern_state_needed(void *opaque) 108 { 109 VirtIOPCIProxy *proxy = opaque; 110 111 return virtio_pci_modern(proxy); 112 } 113 114 static const VMStateDescription vmstate_virtio_pci_modern_state_sub = { 115 .name = "virtio_pci/modern_state", 116 .version_id = 1, 117 .minimum_version_id = 1, 118 .needed = &virtio_pci_modern_state_needed, 119 .fields = (VMStateField[]) { 120 VMSTATE_UINT32(dfselect, VirtIOPCIProxy), 121 VMSTATE_UINT32(gfselect, VirtIOPCIProxy), 122 VMSTATE_UINT32_ARRAY(guest_features, VirtIOPCIProxy, 2), 123 VMSTATE_STRUCT_ARRAY(vqs, VirtIOPCIProxy, VIRTIO_QUEUE_MAX, 0, 124 vmstate_virtio_pci_modern_queue_state, 125 VirtIOPCIQueue), 126 VMSTATE_END_OF_LIST() 127 } 128 }; 129 130 static const VMStateDescription vmstate_virtio_pci = { 131 .name = "virtio_pci", 132 .version_id = 1, 133 .minimum_version_id = 1, 134 .minimum_version_id_old = 1, 135 .fields = (VMStateField[]) { 136 VMSTATE_END_OF_LIST() 137 }, 138 .subsections = (const VMStateDescription*[]) { 139 &vmstate_virtio_pci_modern_state_sub, 140 NULL 141 } 142 }; 143 144 static bool virtio_pci_has_extra_state(DeviceState *d) 145 { 146 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 147 148 return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA; 149 } 150 151 static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f) 152 { 153 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 154 155 vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL); 156 } 157 158 static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f) 159 { 160 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 161 162 return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1); 163 } 164 165 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f) 166 { 167 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 168 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 169 170 if (msix_present(&proxy->pci_dev)) 171 qemu_put_be16(f, virtio_queue_vector(vdev, n)); 172 } 173 174 static int virtio_pci_load_config(DeviceState *d, QEMUFile *f) 175 { 176 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 177 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 178 179 int ret; 180 ret = pci_device_load(&proxy->pci_dev, f); 181 if (ret) { 182 return ret; 183 } 184 msix_unuse_all_vectors(&proxy->pci_dev); 185 msix_load(&proxy->pci_dev, f); 186 if (msix_present(&proxy->pci_dev)) { 187 qemu_get_be16s(f, &vdev->config_vector); 188 } else { 189 vdev->config_vector = VIRTIO_NO_VECTOR; 190 } 191 if (vdev->config_vector != VIRTIO_NO_VECTOR) { 192 return msix_vector_use(&proxy->pci_dev, vdev->config_vector); 193 } 194 return 0; 195 } 196 197 static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f) 198 { 199 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 200 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 201 202 uint16_t vector; 203 if (msix_present(&proxy->pci_dev)) { 204 qemu_get_be16s(f, &vector); 205 } else { 206 vector = VIRTIO_NO_VECTOR; 207 } 208 virtio_queue_set_vector(vdev, n, vector); 209 if (vector != VIRTIO_NO_VECTOR) { 210 return msix_vector_use(&proxy->pci_dev, vector); 211 } 212 213 return 0; 214 } 215 216 static bool virtio_pci_ioeventfd_enabled(DeviceState *d) 217 { 218 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 219 220 return (proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) != 0; 221 } 222 223 #define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000 224 225 static inline int virtio_pci_queue_mem_mult(struct VirtIOPCIProxy *proxy) 226 { 227 return (proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ) ? 228 QEMU_VIRTIO_PCI_QUEUE_MEM_MULT : 4; 229 } 230 231 static int virtio_pci_ioeventfd_assign(DeviceState *d, EventNotifier *notifier, 232 int n, bool assign) 233 { 234 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 235 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 236 VirtQueue *vq = virtio_get_queue(vdev, n); 237 bool legacy = virtio_pci_legacy(proxy); 238 bool modern = virtio_pci_modern(proxy); 239 bool fast_mmio = kvm_ioeventfd_any_length_enabled(); 240 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 241 MemoryRegion *modern_mr = &proxy->notify.mr; 242 MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr; 243 MemoryRegion *legacy_mr = &proxy->bar; 244 hwaddr modern_addr = virtio_pci_queue_mem_mult(proxy) * 245 virtio_get_queue_index(vq); 246 hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY; 247 248 if (assign) { 249 if (modern) { 250 if (fast_mmio) { 251 memory_region_add_eventfd(modern_mr, modern_addr, 0, 252 false, n, notifier); 253 } else { 254 memory_region_add_eventfd(modern_mr, modern_addr, 2, 255 false, n, notifier); 256 } 257 if (modern_pio) { 258 memory_region_add_eventfd(modern_notify_mr, 0, 2, 259 true, n, notifier); 260 } 261 } 262 if (legacy) { 263 memory_region_add_eventfd(legacy_mr, legacy_addr, 2, 264 true, n, notifier); 265 } 266 } else { 267 if (modern) { 268 if (fast_mmio) { 269 memory_region_del_eventfd(modern_mr, modern_addr, 0, 270 false, n, notifier); 271 } else { 272 memory_region_del_eventfd(modern_mr, modern_addr, 2, 273 false, n, notifier); 274 } 275 if (modern_pio) { 276 memory_region_del_eventfd(modern_notify_mr, 0, 2, 277 true, n, notifier); 278 } 279 } 280 if (legacy) { 281 memory_region_del_eventfd(legacy_mr, legacy_addr, 2, 282 true, n, notifier); 283 } 284 } 285 return 0; 286 } 287 288 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy) 289 { 290 virtio_bus_start_ioeventfd(&proxy->bus); 291 } 292 293 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy) 294 { 295 virtio_bus_stop_ioeventfd(&proxy->bus); 296 } 297 298 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) 299 { 300 VirtIOPCIProxy *proxy = opaque; 301 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 302 hwaddr pa; 303 304 switch (addr) { 305 case VIRTIO_PCI_GUEST_FEATURES: 306 /* Guest does not negotiate properly? We have to assume nothing. */ 307 if (val & (1 << VIRTIO_F_BAD_FEATURE)) { 308 val = virtio_bus_get_vdev_bad_features(&proxy->bus); 309 } 310 virtio_set_features(vdev, val); 311 break; 312 case VIRTIO_PCI_QUEUE_PFN: 313 pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT; 314 if (pa == 0) { 315 virtio_pci_reset(DEVICE(proxy)); 316 } 317 else 318 virtio_queue_set_addr(vdev, vdev->queue_sel, pa); 319 break; 320 case VIRTIO_PCI_QUEUE_SEL: 321 if (val < VIRTIO_QUEUE_MAX) 322 vdev->queue_sel = val; 323 break; 324 case VIRTIO_PCI_QUEUE_NOTIFY: 325 if (val < VIRTIO_QUEUE_MAX) { 326 virtio_queue_notify(vdev, val); 327 } 328 break; 329 case VIRTIO_PCI_STATUS: 330 if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) { 331 virtio_pci_stop_ioeventfd(proxy); 332 } 333 334 virtio_set_status(vdev, val & 0xFF); 335 336 if (val & VIRTIO_CONFIG_S_DRIVER_OK) { 337 virtio_pci_start_ioeventfd(proxy); 338 } 339 340 if (vdev->status == 0) { 341 virtio_pci_reset(DEVICE(proxy)); 342 } 343 344 /* Linux before 2.6.34 drives the device without enabling 345 the PCI device bus master bit. Enable it automatically 346 for the guest. This is a PCI spec violation but so is 347 initiating DMA with bus master bit clear. */ 348 if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER)) { 349 pci_default_write_config(&proxy->pci_dev, PCI_COMMAND, 350 proxy->pci_dev.config[PCI_COMMAND] | 351 PCI_COMMAND_MASTER, 1); 352 } 353 break; 354 case VIRTIO_MSI_CONFIG_VECTOR: 355 msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); 356 /* Make it possible for guest to discover an error took place. */ 357 if (msix_vector_use(&proxy->pci_dev, val) < 0) 358 val = VIRTIO_NO_VECTOR; 359 vdev->config_vector = val; 360 break; 361 case VIRTIO_MSI_QUEUE_VECTOR: 362 msix_vector_unuse(&proxy->pci_dev, 363 virtio_queue_vector(vdev, vdev->queue_sel)); 364 /* Make it possible for guest to discover an error took place. */ 365 if (msix_vector_use(&proxy->pci_dev, val) < 0) 366 val = VIRTIO_NO_VECTOR; 367 virtio_queue_set_vector(vdev, vdev->queue_sel, val); 368 break; 369 default: 370 qemu_log_mask(LOG_GUEST_ERROR, 371 "%s: unexpected address 0x%x value 0x%x\n", 372 __func__, addr, val); 373 break; 374 } 375 } 376 377 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr) 378 { 379 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 380 uint32_t ret = 0xFFFFFFFF; 381 382 switch (addr) { 383 case VIRTIO_PCI_HOST_FEATURES: 384 ret = vdev->host_features; 385 break; 386 case VIRTIO_PCI_GUEST_FEATURES: 387 ret = vdev->guest_features; 388 break; 389 case VIRTIO_PCI_QUEUE_PFN: 390 ret = virtio_queue_get_addr(vdev, vdev->queue_sel) 391 >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; 392 break; 393 case VIRTIO_PCI_QUEUE_NUM: 394 ret = virtio_queue_get_num(vdev, vdev->queue_sel); 395 break; 396 case VIRTIO_PCI_QUEUE_SEL: 397 ret = vdev->queue_sel; 398 break; 399 case VIRTIO_PCI_STATUS: 400 ret = vdev->status; 401 break; 402 case VIRTIO_PCI_ISR: 403 /* reading from the ISR also clears it. */ 404 ret = qatomic_xchg(&vdev->isr, 0); 405 pci_irq_deassert(&proxy->pci_dev); 406 break; 407 case VIRTIO_MSI_CONFIG_VECTOR: 408 ret = vdev->config_vector; 409 break; 410 case VIRTIO_MSI_QUEUE_VECTOR: 411 ret = virtio_queue_vector(vdev, vdev->queue_sel); 412 break; 413 default: 414 break; 415 } 416 417 return ret; 418 } 419 420 static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr, 421 unsigned size) 422 { 423 VirtIOPCIProxy *proxy = opaque; 424 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 425 uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev); 426 uint64_t val = 0; 427 428 if (vdev == NULL) { 429 return UINT64_MAX; 430 } 431 432 if (addr < config) { 433 return virtio_ioport_read(proxy, addr); 434 } 435 addr -= config; 436 437 switch (size) { 438 case 1: 439 val = virtio_config_readb(vdev, addr); 440 break; 441 case 2: 442 val = virtio_config_readw(vdev, addr); 443 if (virtio_is_big_endian(vdev)) { 444 val = bswap16(val); 445 } 446 break; 447 case 4: 448 val = virtio_config_readl(vdev, addr); 449 if (virtio_is_big_endian(vdev)) { 450 val = bswap32(val); 451 } 452 break; 453 } 454 return val; 455 } 456 457 static void virtio_pci_config_write(void *opaque, hwaddr addr, 458 uint64_t val, unsigned size) 459 { 460 VirtIOPCIProxy *proxy = opaque; 461 uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev); 462 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 463 464 if (vdev == NULL) { 465 return; 466 } 467 468 if (addr < config) { 469 virtio_ioport_write(proxy, addr, val); 470 return; 471 } 472 addr -= config; 473 /* 474 * Virtio-PCI is odd. Ioports are LE but config space is target native 475 * endian. 476 */ 477 switch (size) { 478 case 1: 479 virtio_config_writeb(vdev, addr, val); 480 break; 481 case 2: 482 if (virtio_is_big_endian(vdev)) { 483 val = bswap16(val); 484 } 485 virtio_config_writew(vdev, addr, val); 486 break; 487 case 4: 488 if (virtio_is_big_endian(vdev)) { 489 val = bswap32(val); 490 } 491 virtio_config_writel(vdev, addr, val); 492 break; 493 } 494 } 495 496 static const MemoryRegionOps virtio_pci_config_ops = { 497 .read = virtio_pci_config_read, 498 .write = virtio_pci_config_write, 499 .impl = { 500 .min_access_size = 1, 501 .max_access_size = 4, 502 }, 503 .endianness = DEVICE_LITTLE_ENDIAN, 504 }; 505 506 static MemoryRegion *virtio_address_space_lookup(VirtIOPCIProxy *proxy, 507 hwaddr *off, int len) 508 { 509 int i; 510 VirtIOPCIRegion *reg; 511 512 for (i = 0; i < ARRAY_SIZE(proxy->regs); ++i) { 513 reg = &proxy->regs[i]; 514 if (*off >= reg->offset && 515 *off + len <= reg->offset + reg->size) { 516 *off -= reg->offset; 517 return ®->mr; 518 } 519 } 520 521 return NULL; 522 } 523 524 /* Below are generic functions to do memcpy from/to an address space, 525 * without byteswaps, with input validation. 526 * 527 * As regular address_space_* APIs all do some kind of byteswap at least for 528 * some host/target combinations, we are forced to explicitly convert to a 529 * known-endianness integer value. 530 * It doesn't really matter which endian format to go through, so the code 531 * below selects the endian that causes the least amount of work on the given 532 * host. 533 * 534 * Note: host pointer must be aligned. 535 */ 536 static 537 void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr, 538 const uint8_t *buf, int len) 539 { 540 uint64_t val; 541 MemoryRegion *mr; 542 543 /* address_space_* APIs assume an aligned address. 544 * As address is under guest control, handle illegal values. 545 */ 546 addr &= ~(len - 1); 547 548 mr = virtio_address_space_lookup(proxy, &addr, len); 549 if (!mr) { 550 return; 551 } 552 553 /* Make sure caller aligned buf properly */ 554 assert(!(((uintptr_t)buf) & (len - 1))); 555 556 switch (len) { 557 case 1: 558 val = pci_get_byte(buf); 559 break; 560 case 2: 561 val = pci_get_word(buf); 562 break; 563 case 4: 564 val = pci_get_long(buf); 565 break; 566 default: 567 /* As length is under guest control, handle illegal values. */ 568 return; 569 } 570 memory_region_dispatch_write(mr, addr, val, size_memop(len) | MO_LE, 571 MEMTXATTRS_UNSPECIFIED); 572 } 573 574 static void 575 virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr, 576 uint8_t *buf, int len) 577 { 578 uint64_t val; 579 MemoryRegion *mr; 580 581 /* address_space_* APIs assume an aligned address. 582 * As address is under guest control, handle illegal values. 583 */ 584 addr &= ~(len - 1); 585 586 mr = virtio_address_space_lookup(proxy, &addr, len); 587 if (!mr) { 588 return; 589 } 590 591 /* Make sure caller aligned buf properly */ 592 assert(!(((uintptr_t)buf) & (len - 1))); 593 594 memory_region_dispatch_read(mr, addr, &val, size_memop(len) | MO_LE, 595 MEMTXATTRS_UNSPECIFIED); 596 switch (len) { 597 case 1: 598 pci_set_byte(buf, val); 599 break; 600 case 2: 601 pci_set_word(buf, val); 602 break; 603 case 4: 604 pci_set_long(buf, val); 605 break; 606 default: 607 /* As length is under guest control, handle illegal values. */ 608 break; 609 } 610 } 611 612 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address, 613 uint32_t val, int len) 614 { 615 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 616 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 617 struct virtio_pci_cfg_cap *cfg; 618 619 pci_default_write_config(pci_dev, address, val, len); 620 621 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) { 622 pcie_cap_flr_write_config(pci_dev, address, val, len); 623 } 624 625 if (range_covers_byte(address, len, PCI_COMMAND)) { 626 if (!(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { 627 virtio_set_disabled(vdev, true); 628 virtio_pci_stop_ioeventfd(proxy); 629 virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK); 630 } else { 631 virtio_set_disabled(vdev, false); 632 } 633 } 634 635 if (proxy->config_cap && 636 ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap, 637 pci_cfg_data), 638 sizeof cfg->pci_cfg_data)) { 639 uint32_t off; 640 uint32_t len; 641 642 cfg = (void *)(proxy->pci_dev.config + proxy->config_cap); 643 off = le32_to_cpu(cfg->cap.offset); 644 len = le32_to_cpu(cfg->cap.length); 645 646 if (len == 1 || len == 2 || len == 4) { 647 assert(len <= sizeof cfg->pci_cfg_data); 648 virtio_address_space_write(proxy, off, cfg->pci_cfg_data, len); 649 } 650 } 651 } 652 653 static uint32_t virtio_read_config(PCIDevice *pci_dev, 654 uint32_t address, int len) 655 { 656 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 657 struct virtio_pci_cfg_cap *cfg; 658 659 if (proxy->config_cap && 660 ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap, 661 pci_cfg_data), 662 sizeof cfg->pci_cfg_data)) { 663 uint32_t off; 664 uint32_t len; 665 666 cfg = (void *)(proxy->pci_dev.config + proxy->config_cap); 667 off = le32_to_cpu(cfg->cap.offset); 668 len = le32_to_cpu(cfg->cap.length); 669 670 if (len == 1 || len == 2 || len == 4) { 671 assert(len <= sizeof cfg->pci_cfg_data); 672 virtio_address_space_read(proxy, off, cfg->pci_cfg_data, len); 673 } 674 } 675 676 return pci_default_read_config(pci_dev, address, len); 677 } 678 679 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy, 680 unsigned int vector) 681 { 682 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 683 int ret; 684 685 if (irqfd->users == 0) { 686 ret = kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_dev); 687 if (ret < 0) { 688 return ret; 689 } 690 irqfd->virq = ret; 691 } 692 irqfd->users++; 693 return 0; 694 } 695 696 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy, 697 unsigned int vector) 698 { 699 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 700 if (--irqfd->users == 0) { 701 kvm_irqchip_release_virq(kvm_state, irqfd->virq); 702 } 703 } 704 705 static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy, 706 EventNotifier *n, 707 unsigned int vector) 708 { 709 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 710 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq); 711 } 712 713 static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy, 714 EventNotifier *n , 715 unsigned int vector) 716 { 717 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 718 int ret; 719 720 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq); 721 assert(ret == 0); 722 } 723 static int virtio_pci_get_notifier(VirtIOPCIProxy *proxy, int queue_no, 724 EventNotifier **n, unsigned int *vector) 725 { 726 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 727 VirtQueue *vq; 728 729 if (queue_no == VIRTIO_CONFIG_IRQ_IDX) { 730 *n = virtio_config_get_guest_notifier(vdev); 731 *vector = vdev->config_vector; 732 } else { 733 if (!virtio_queue_get_num(vdev, queue_no)) { 734 return -1; 735 } 736 *vector = virtio_queue_vector(vdev, queue_no); 737 vq = virtio_get_queue(vdev, queue_no); 738 *n = virtio_queue_get_guest_notifier(vq); 739 } 740 return 0; 741 } 742 743 static int kvm_virtio_pci_vector_use_one(VirtIOPCIProxy *proxy, int queue_no) 744 { 745 unsigned int vector; 746 int ret; 747 EventNotifier *n; 748 PCIDevice *dev = &proxy->pci_dev; 749 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 750 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 751 752 ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector); 753 if (ret < 0) { 754 return ret; 755 } 756 if (vector >= msix_nr_vectors_allocated(dev)) { 757 return 0; 758 } 759 ret = kvm_virtio_pci_vq_vector_use(proxy, vector); 760 if (ret < 0) { 761 goto undo; 762 } 763 /* 764 * If guest supports masking, set up irqfd now. 765 * Otherwise, delay until unmasked in the frontend. 766 */ 767 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 768 ret = kvm_virtio_pci_irqfd_use(proxy, n, vector); 769 if (ret < 0) { 770 kvm_virtio_pci_vq_vector_release(proxy, vector); 771 goto undo; 772 } 773 } 774 775 return 0; 776 undo: 777 778 vector = virtio_queue_vector(vdev, queue_no); 779 if (vector >= msix_nr_vectors_allocated(dev)) { 780 return ret; 781 } 782 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 783 ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector); 784 if (ret < 0) { 785 return ret; 786 } 787 kvm_virtio_pci_irqfd_release(proxy, n, vector); 788 } 789 return ret; 790 } 791 static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs) 792 { 793 int queue_no; 794 int ret = 0; 795 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 796 797 for (queue_no = 0; queue_no < nvqs; queue_no++) { 798 if (!virtio_queue_get_num(vdev, queue_no)) { 799 return -1; 800 } 801 ret = kvm_virtio_pci_vector_use_one(proxy, queue_no); 802 } 803 return ret; 804 } 805 806 static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy) 807 { 808 return kvm_virtio_pci_vector_use_one(proxy, VIRTIO_CONFIG_IRQ_IDX); 809 } 810 811 static void kvm_virtio_pci_vector_release_one(VirtIOPCIProxy *proxy, 812 int queue_no) 813 { 814 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 815 unsigned int vector; 816 EventNotifier *n; 817 int ret; 818 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 819 PCIDevice *dev = &proxy->pci_dev; 820 821 ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector); 822 if (ret < 0) { 823 return; 824 } 825 if (vector >= msix_nr_vectors_allocated(dev)) { 826 return; 827 } 828 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 829 kvm_virtio_pci_irqfd_release(proxy, n, vector); 830 } 831 kvm_virtio_pci_vq_vector_release(proxy, vector); 832 } 833 834 static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs) 835 { 836 int queue_no; 837 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 838 839 for (queue_no = 0; queue_no < nvqs; queue_no++) { 840 if (!virtio_queue_get_num(vdev, queue_no)) { 841 break; 842 } 843 kvm_virtio_pci_vector_release_one(proxy, queue_no); 844 } 845 } 846 847 static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy) 848 { 849 kvm_virtio_pci_vector_release_one(proxy, VIRTIO_CONFIG_IRQ_IDX); 850 } 851 852 static int virtio_pci_one_vector_unmask(VirtIOPCIProxy *proxy, 853 unsigned int queue_no, 854 unsigned int vector, 855 MSIMessage msg, 856 EventNotifier *n) 857 { 858 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 859 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 860 VirtIOIRQFD *irqfd; 861 int ret = 0; 862 863 if (proxy->vector_irqfd) { 864 irqfd = &proxy->vector_irqfd[vector]; 865 if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) { 866 ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg, 867 &proxy->pci_dev); 868 if (ret < 0) { 869 return ret; 870 } 871 kvm_irqchip_commit_routes(kvm_state); 872 } 873 } 874 875 /* If guest supports masking, irqfd is already setup, unmask it. 876 * Otherwise, set it up now. 877 */ 878 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 879 k->guest_notifier_mask(vdev, queue_no, false); 880 /* Test after unmasking to avoid losing events. */ 881 if (k->guest_notifier_pending && 882 k->guest_notifier_pending(vdev, queue_no)) { 883 event_notifier_set(n); 884 } 885 } else { 886 ret = kvm_virtio_pci_irqfd_use(proxy, n, vector); 887 } 888 return ret; 889 } 890 891 static void virtio_pci_one_vector_mask(VirtIOPCIProxy *proxy, 892 unsigned int queue_no, 893 unsigned int vector, 894 EventNotifier *n) 895 { 896 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 897 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 898 899 /* If guest supports masking, keep irqfd but mask it. 900 * Otherwise, clean it up now. 901 */ 902 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 903 k->guest_notifier_mask(vdev, queue_no, true); 904 } else { 905 kvm_virtio_pci_irqfd_release(proxy, n, vector); 906 } 907 } 908 909 static int virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector, 910 MSIMessage msg) 911 { 912 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); 913 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 914 VirtQueue *vq = virtio_vector_first_queue(vdev, vector); 915 EventNotifier *n; 916 int ret, index, unmasked = 0; 917 918 while (vq) { 919 index = virtio_get_queue_index(vq); 920 if (!virtio_queue_get_num(vdev, index)) { 921 break; 922 } 923 if (index < proxy->nvqs_with_notifiers) { 924 n = virtio_queue_get_guest_notifier(vq); 925 ret = virtio_pci_one_vector_unmask(proxy, index, vector, msg, n); 926 if (ret < 0) { 927 goto undo; 928 } 929 ++unmasked; 930 } 931 vq = virtio_vector_next_queue(vq); 932 } 933 /* unmask config intr */ 934 n = virtio_config_get_guest_notifier(vdev); 935 ret = virtio_pci_one_vector_unmask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, 936 msg, n); 937 if (ret < 0) { 938 goto undo_config; 939 } 940 return 0; 941 undo_config: 942 n = virtio_config_get_guest_notifier(vdev); 943 virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n); 944 undo: 945 vq = virtio_vector_first_queue(vdev, vector); 946 while (vq && unmasked >= 0) { 947 index = virtio_get_queue_index(vq); 948 if (index < proxy->nvqs_with_notifiers) { 949 n = virtio_queue_get_guest_notifier(vq); 950 virtio_pci_one_vector_mask(proxy, index, vector, n); 951 --unmasked; 952 } 953 vq = virtio_vector_next_queue(vq); 954 } 955 return ret; 956 } 957 958 static void virtio_pci_vector_mask(PCIDevice *dev, unsigned vector) 959 { 960 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); 961 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 962 VirtQueue *vq = virtio_vector_first_queue(vdev, vector); 963 EventNotifier *n; 964 int index; 965 966 while (vq) { 967 index = virtio_get_queue_index(vq); 968 n = virtio_queue_get_guest_notifier(vq); 969 if (!virtio_queue_get_num(vdev, index)) { 970 break; 971 } 972 if (index < proxy->nvqs_with_notifiers) { 973 virtio_pci_one_vector_mask(proxy, index, vector, n); 974 } 975 vq = virtio_vector_next_queue(vq); 976 } 977 n = virtio_config_get_guest_notifier(vdev); 978 virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n); 979 } 980 981 static void virtio_pci_vector_poll(PCIDevice *dev, 982 unsigned int vector_start, 983 unsigned int vector_end) 984 { 985 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); 986 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 987 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 988 int queue_no; 989 unsigned int vector; 990 EventNotifier *notifier; 991 int ret; 992 993 for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) { 994 ret = virtio_pci_get_notifier(proxy, queue_no, ¬ifier, &vector); 995 if (ret < 0) { 996 break; 997 } 998 if (vector < vector_start || vector >= vector_end || 999 !msix_is_masked(dev, vector)) { 1000 continue; 1001 } 1002 if (k->guest_notifier_pending) { 1003 if (k->guest_notifier_pending(vdev, queue_no)) { 1004 msix_set_pending(dev, vector); 1005 } 1006 } else if (event_notifier_test_and_clear(notifier)) { 1007 msix_set_pending(dev, vector); 1008 } 1009 } 1010 /* poll the config intr */ 1011 ret = virtio_pci_get_notifier(proxy, VIRTIO_CONFIG_IRQ_IDX, ¬ifier, 1012 &vector); 1013 if (ret < 0) { 1014 return; 1015 } 1016 if (vector < vector_start || vector >= vector_end || 1017 !msix_is_masked(dev, vector)) { 1018 return; 1019 } 1020 if (k->guest_notifier_pending) { 1021 if (k->guest_notifier_pending(vdev, VIRTIO_CONFIG_IRQ_IDX)) { 1022 msix_set_pending(dev, vector); 1023 } 1024 } else if (event_notifier_test_and_clear(notifier)) { 1025 msix_set_pending(dev, vector); 1026 } 1027 } 1028 1029 void virtio_pci_set_guest_notifier_fd_handler(VirtIODevice *vdev, VirtQueue *vq, 1030 int n, bool assign, 1031 bool with_irqfd) 1032 { 1033 if (n == VIRTIO_CONFIG_IRQ_IDX) { 1034 virtio_config_set_guest_notifier_fd_handler(vdev, assign, with_irqfd); 1035 } else { 1036 virtio_queue_set_guest_notifier_fd_handler(vq, assign, with_irqfd); 1037 } 1038 } 1039 1040 static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign, 1041 bool with_irqfd) 1042 { 1043 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1044 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1045 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1046 VirtQueue *vq = NULL; 1047 EventNotifier *notifier = NULL; 1048 1049 if (n == VIRTIO_CONFIG_IRQ_IDX) { 1050 notifier = virtio_config_get_guest_notifier(vdev); 1051 } else { 1052 vq = virtio_get_queue(vdev, n); 1053 notifier = virtio_queue_get_guest_notifier(vq); 1054 } 1055 1056 if (assign) { 1057 int r = event_notifier_init(notifier, 0); 1058 if (r < 0) { 1059 return r; 1060 } 1061 virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, true, with_irqfd); 1062 } else { 1063 virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, false, 1064 with_irqfd); 1065 event_notifier_cleanup(notifier); 1066 } 1067 1068 if (!msix_enabled(&proxy->pci_dev) && 1069 vdev->use_guest_notifier_mask && 1070 vdc->guest_notifier_mask) { 1071 vdc->guest_notifier_mask(vdev, n, !assign); 1072 } 1073 1074 return 0; 1075 } 1076 1077 static bool virtio_pci_query_guest_notifiers(DeviceState *d) 1078 { 1079 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1080 return msix_enabled(&proxy->pci_dev); 1081 } 1082 1083 static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign) 1084 { 1085 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1086 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1087 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1088 int r, n; 1089 bool with_irqfd = msix_enabled(&proxy->pci_dev) && 1090 kvm_msi_via_irqfd_enabled(); 1091 1092 nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX); 1093 1094 /* When deassigning, pass a consistent nvqs value 1095 * to avoid leaking notifiers. 1096 */ 1097 assert(assign || nvqs == proxy->nvqs_with_notifiers); 1098 1099 proxy->nvqs_with_notifiers = nvqs; 1100 1101 /* Must unset vector notifier while guest notifier is still assigned */ 1102 if ((proxy->vector_irqfd || k->guest_notifier_mask) && !assign) { 1103 msix_unset_vector_notifiers(&proxy->pci_dev); 1104 if (proxy->vector_irqfd) { 1105 kvm_virtio_pci_vector_release(proxy, nvqs); 1106 kvm_virtio_pci_vector_config_release(proxy); 1107 g_free(proxy->vector_irqfd); 1108 proxy->vector_irqfd = NULL; 1109 } 1110 } 1111 1112 for (n = 0; n < nvqs; n++) { 1113 if (!virtio_queue_get_num(vdev, n)) { 1114 break; 1115 } 1116 1117 r = virtio_pci_set_guest_notifier(d, n, assign, with_irqfd); 1118 if (r < 0) { 1119 goto assign_error; 1120 } 1121 } 1122 r = virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, assign, 1123 with_irqfd); 1124 if (r < 0) { 1125 goto config_assign_error; 1126 } 1127 /* Must set vector notifier after guest notifier has been assigned */ 1128 if ((with_irqfd || k->guest_notifier_mask) && assign) { 1129 if (with_irqfd) { 1130 proxy->vector_irqfd = 1131 g_malloc0(sizeof(*proxy->vector_irqfd) * 1132 msix_nr_vectors_allocated(&proxy->pci_dev)); 1133 r = kvm_virtio_pci_vector_use(proxy, nvqs); 1134 if (r < 0) { 1135 goto config_assign_error; 1136 } 1137 } 1138 r = kvm_virtio_pci_vector_config_use(proxy); 1139 if (r < 0) { 1140 goto config_error; 1141 } 1142 r = msix_set_vector_notifiers(&proxy->pci_dev, virtio_pci_vector_unmask, 1143 virtio_pci_vector_mask, 1144 virtio_pci_vector_poll); 1145 if (r < 0) { 1146 goto notifiers_error; 1147 } 1148 } 1149 1150 return 0; 1151 1152 notifiers_error: 1153 if (with_irqfd) { 1154 assert(assign); 1155 kvm_virtio_pci_vector_release(proxy, nvqs); 1156 } 1157 config_error: 1158 kvm_virtio_pci_vector_config_release(proxy); 1159 config_assign_error: 1160 virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, !assign, 1161 with_irqfd); 1162 assign_error: 1163 /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */ 1164 assert(assign); 1165 while (--n >= 0) { 1166 virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd); 1167 } 1168 return r; 1169 } 1170 1171 static int virtio_pci_set_host_notifier_mr(DeviceState *d, int n, 1172 MemoryRegion *mr, bool assign) 1173 { 1174 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1175 int offset; 1176 1177 if (n >= VIRTIO_QUEUE_MAX || !virtio_pci_modern(proxy) || 1178 virtio_pci_queue_mem_mult(proxy) != memory_region_size(mr)) { 1179 return -1; 1180 } 1181 1182 if (assign) { 1183 offset = virtio_pci_queue_mem_mult(proxy) * n; 1184 memory_region_add_subregion_overlap(&proxy->notify.mr, offset, mr, 1); 1185 } else { 1186 memory_region_del_subregion(&proxy->notify.mr, mr); 1187 } 1188 1189 return 0; 1190 } 1191 1192 static void virtio_pci_vmstate_change(DeviceState *d, bool running) 1193 { 1194 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1195 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1196 1197 if (running) { 1198 /* Old QEMU versions did not set bus master enable on status write. 1199 * Detect DRIVER set and enable it. 1200 */ 1201 if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) && 1202 (vdev->status & VIRTIO_CONFIG_S_DRIVER) && 1203 !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { 1204 pci_default_write_config(&proxy->pci_dev, PCI_COMMAND, 1205 proxy->pci_dev.config[PCI_COMMAND] | 1206 PCI_COMMAND_MASTER, 1); 1207 } 1208 virtio_pci_start_ioeventfd(proxy); 1209 } else { 1210 virtio_pci_stop_ioeventfd(proxy); 1211 } 1212 } 1213 1214 /* 1215 * virtio-pci: This is the PCIDevice which has a virtio-pci-bus. 1216 */ 1217 1218 static int virtio_pci_query_nvectors(DeviceState *d) 1219 { 1220 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1221 1222 return proxy->nvectors; 1223 } 1224 1225 static AddressSpace *virtio_pci_get_dma_as(DeviceState *d) 1226 { 1227 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1228 PCIDevice *dev = &proxy->pci_dev; 1229 1230 return pci_get_address_space(dev); 1231 } 1232 1233 static bool virtio_pci_iommu_enabled(DeviceState *d) 1234 { 1235 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1236 PCIDevice *dev = &proxy->pci_dev; 1237 AddressSpace *dma_as = pci_device_iommu_address_space(dev); 1238 1239 if (dma_as == &address_space_memory) { 1240 return false; 1241 } 1242 1243 return true; 1244 } 1245 1246 static bool virtio_pci_queue_enabled(DeviceState *d, int n) 1247 { 1248 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1249 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1250 1251 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1252 return proxy->vqs[n].enabled; 1253 } 1254 1255 return virtio_queue_enabled_legacy(vdev, n); 1256 } 1257 1258 static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy, 1259 struct virtio_pci_cap *cap) 1260 { 1261 PCIDevice *dev = &proxy->pci_dev; 1262 int offset; 1263 1264 offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0, 1265 cap->cap_len, &error_abort); 1266 1267 assert(cap->cap_len >= sizeof *cap); 1268 memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len, 1269 cap->cap_len - PCI_CAP_FLAGS); 1270 1271 return offset; 1272 } 1273 1274 static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr, 1275 unsigned size) 1276 { 1277 VirtIOPCIProxy *proxy = opaque; 1278 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1279 uint32_t val = 0; 1280 int i; 1281 1282 if (vdev == NULL) { 1283 return UINT64_MAX; 1284 } 1285 1286 switch (addr) { 1287 case VIRTIO_PCI_COMMON_DFSELECT: 1288 val = proxy->dfselect; 1289 break; 1290 case VIRTIO_PCI_COMMON_DF: 1291 if (proxy->dfselect <= 1) { 1292 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1293 1294 val = (vdev->host_features & ~vdc->legacy_features) >> 1295 (32 * proxy->dfselect); 1296 } 1297 break; 1298 case VIRTIO_PCI_COMMON_GFSELECT: 1299 val = proxy->gfselect; 1300 break; 1301 case VIRTIO_PCI_COMMON_GF: 1302 if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) { 1303 val = proxy->guest_features[proxy->gfselect]; 1304 } 1305 break; 1306 case VIRTIO_PCI_COMMON_MSIX: 1307 val = vdev->config_vector; 1308 break; 1309 case VIRTIO_PCI_COMMON_NUMQ: 1310 for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) { 1311 if (virtio_queue_get_num(vdev, i)) { 1312 val = i + 1; 1313 } 1314 } 1315 break; 1316 case VIRTIO_PCI_COMMON_STATUS: 1317 val = vdev->status; 1318 break; 1319 case VIRTIO_PCI_COMMON_CFGGENERATION: 1320 val = vdev->generation; 1321 break; 1322 case VIRTIO_PCI_COMMON_Q_SELECT: 1323 val = vdev->queue_sel; 1324 break; 1325 case VIRTIO_PCI_COMMON_Q_SIZE: 1326 val = virtio_queue_get_num(vdev, vdev->queue_sel); 1327 break; 1328 case VIRTIO_PCI_COMMON_Q_MSIX: 1329 val = virtio_queue_vector(vdev, vdev->queue_sel); 1330 break; 1331 case VIRTIO_PCI_COMMON_Q_ENABLE: 1332 val = proxy->vqs[vdev->queue_sel].enabled; 1333 break; 1334 case VIRTIO_PCI_COMMON_Q_NOFF: 1335 /* Simply map queues in order */ 1336 val = vdev->queue_sel; 1337 break; 1338 case VIRTIO_PCI_COMMON_Q_DESCLO: 1339 val = proxy->vqs[vdev->queue_sel].desc[0]; 1340 break; 1341 case VIRTIO_PCI_COMMON_Q_DESCHI: 1342 val = proxy->vqs[vdev->queue_sel].desc[1]; 1343 break; 1344 case VIRTIO_PCI_COMMON_Q_AVAILLO: 1345 val = proxy->vqs[vdev->queue_sel].avail[0]; 1346 break; 1347 case VIRTIO_PCI_COMMON_Q_AVAILHI: 1348 val = proxy->vqs[vdev->queue_sel].avail[1]; 1349 break; 1350 case VIRTIO_PCI_COMMON_Q_USEDLO: 1351 val = proxy->vqs[vdev->queue_sel].used[0]; 1352 break; 1353 case VIRTIO_PCI_COMMON_Q_USEDHI: 1354 val = proxy->vqs[vdev->queue_sel].used[1]; 1355 break; 1356 default: 1357 val = 0; 1358 } 1359 1360 return val; 1361 } 1362 1363 static void virtio_pci_common_write(void *opaque, hwaddr addr, 1364 uint64_t val, unsigned size) 1365 { 1366 VirtIOPCIProxy *proxy = opaque; 1367 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1368 1369 if (vdev == NULL) { 1370 return; 1371 } 1372 1373 switch (addr) { 1374 case VIRTIO_PCI_COMMON_DFSELECT: 1375 proxy->dfselect = val; 1376 break; 1377 case VIRTIO_PCI_COMMON_GFSELECT: 1378 proxy->gfselect = val; 1379 break; 1380 case VIRTIO_PCI_COMMON_GF: 1381 if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) { 1382 proxy->guest_features[proxy->gfselect] = val; 1383 virtio_set_features(vdev, 1384 (((uint64_t)proxy->guest_features[1]) << 32) | 1385 proxy->guest_features[0]); 1386 } 1387 break; 1388 case VIRTIO_PCI_COMMON_MSIX: 1389 msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); 1390 /* Make it possible for guest to discover an error took place. */ 1391 if (msix_vector_use(&proxy->pci_dev, val) < 0) { 1392 val = VIRTIO_NO_VECTOR; 1393 } 1394 vdev->config_vector = val; 1395 break; 1396 case VIRTIO_PCI_COMMON_STATUS: 1397 if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) { 1398 virtio_pci_stop_ioeventfd(proxy); 1399 } 1400 1401 virtio_set_status(vdev, val & 0xFF); 1402 1403 if (val & VIRTIO_CONFIG_S_DRIVER_OK) { 1404 virtio_pci_start_ioeventfd(proxy); 1405 } 1406 1407 if (vdev->status == 0) { 1408 virtio_pci_reset(DEVICE(proxy)); 1409 } 1410 1411 break; 1412 case VIRTIO_PCI_COMMON_Q_SELECT: 1413 if (val < VIRTIO_QUEUE_MAX) { 1414 vdev->queue_sel = val; 1415 } 1416 break; 1417 case VIRTIO_PCI_COMMON_Q_SIZE: 1418 proxy->vqs[vdev->queue_sel].num = val; 1419 virtio_queue_set_num(vdev, vdev->queue_sel, 1420 proxy->vqs[vdev->queue_sel].num); 1421 break; 1422 case VIRTIO_PCI_COMMON_Q_MSIX: 1423 msix_vector_unuse(&proxy->pci_dev, 1424 virtio_queue_vector(vdev, vdev->queue_sel)); 1425 /* Make it possible for guest to discover an error took place. */ 1426 if (msix_vector_use(&proxy->pci_dev, val) < 0) { 1427 val = VIRTIO_NO_VECTOR; 1428 } 1429 virtio_queue_set_vector(vdev, vdev->queue_sel, val); 1430 break; 1431 case VIRTIO_PCI_COMMON_Q_ENABLE: 1432 if (val == 1) { 1433 virtio_queue_set_num(vdev, vdev->queue_sel, 1434 proxy->vqs[vdev->queue_sel].num); 1435 virtio_queue_set_rings(vdev, vdev->queue_sel, 1436 ((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 | 1437 proxy->vqs[vdev->queue_sel].desc[0], 1438 ((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 | 1439 proxy->vqs[vdev->queue_sel].avail[0], 1440 ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 | 1441 proxy->vqs[vdev->queue_sel].used[0]); 1442 proxy->vqs[vdev->queue_sel].enabled = 1; 1443 } else { 1444 virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val); 1445 } 1446 break; 1447 case VIRTIO_PCI_COMMON_Q_DESCLO: 1448 proxy->vqs[vdev->queue_sel].desc[0] = val; 1449 break; 1450 case VIRTIO_PCI_COMMON_Q_DESCHI: 1451 proxy->vqs[vdev->queue_sel].desc[1] = val; 1452 break; 1453 case VIRTIO_PCI_COMMON_Q_AVAILLO: 1454 proxy->vqs[vdev->queue_sel].avail[0] = val; 1455 break; 1456 case VIRTIO_PCI_COMMON_Q_AVAILHI: 1457 proxy->vqs[vdev->queue_sel].avail[1] = val; 1458 break; 1459 case VIRTIO_PCI_COMMON_Q_USEDLO: 1460 proxy->vqs[vdev->queue_sel].used[0] = val; 1461 break; 1462 case VIRTIO_PCI_COMMON_Q_USEDHI: 1463 proxy->vqs[vdev->queue_sel].used[1] = val; 1464 break; 1465 default: 1466 break; 1467 } 1468 } 1469 1470 1471 static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr, 1472 unsigned size) 1473 { 1474 VirtIOPCIProxy *proxy = opaque; 1475 if (virtio_bus_get_device(&proxy->bus) == NULL) { 1476 return UINT64_MAX; 1477 } 1478 1479 return 0; 1480 } 1481 1482 static void virtio_pci_notify_write(void *opaque, hwaddr addr, 1483 uint64_t val, unsigned size) 1484 { 1485 VirtIOPCIProxy *proxy = opaque; 1486 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1487 1488 unsigned queue = addr / virtio_pci_queue_mem_mult(proxy); 1489 1490 if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) { 1491 virtio_queue_notify(vdev, queue); 1492 } 1493 } 1494 1495 static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr, 1496 uint64_t val, unsigned size) 1497 { 1498 VirtIOPCIProxy *proxy = opaque; 1499 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1500 1501 unsigned queue = val; 1502 1503 if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) { 1504 virtio_queue_notify(vdev, queue); 1505 } 1506 } 1507 1508 static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr, 1509 unsigned size) 1510 { 1511 VirtIOPCIProxy *proxy = opaque; 1512 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1513 uint64_t val; 1514 1515 if (vdev == NULL) { 1516 return UINT64_MAX; 1517 } 1518 1519 val = qatomic_xchg(&vdev->isr, 0); 1520 pci_irq_deassert(&proxy->pci_dev); 1521 return val; 1522 } 1523 1524 static void virtio_pci_isr_write(void *opaque, hwaddr addr, 1525 uint64_t val, unsigned size) 1526 { 1527 } 1528 1529 static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr, 1530 unsigned size) 1531 { 1532 VirtIOPCIProxy *proxy = opaque; 1533 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1534 uint64_t val; 1535 1536 if (vdev == NULL) { 1537 return UINT64_MAX; 1538 } 1539 1540 switch (size) { 1541 case 1: 1542 val = virtio_config_modern_readb(vdev, addr); 1543 break; 1544 case 2: 1545 val = virtio_config_modern_readw(vdev, addr); 1546 break; 1547 case 4: 1548 val = virtio_config_modern_readl(vdev, addr); 1549 break; 1550 default: 1551 val = 0; 1552 break; 1553 } 1554 return val; 1555 } 1556 1557 static void virtio_pci_device_write(void *opaque, hwaddr addr, 1558 uint64_t val, unsigned size) 1559 { 1560 VirtIOPCIProxy *proxy = opaque; 1561 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1562 1563 if (vdev == NULL) { 1564 return; 1565 } 1566 1567 switch (size) { 1568 case 1: 1569 virtio_config_modern_writeb(vdev, addr, val); 1570 break; 1571 case 2: 1572 virtio_config_modern_writew(vdev, addr, val); 1573 break; 1574 case 4: 1575 virtio_config_modern_writel(vdev, addr, val); 1576 break; 1577 } 1578 } 1579 1580 static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy, 1581 const char *vdev_name) 1582 { 1583 static const MemoryRegionOps common_ops = { 1584 .read = virtio_pci_common_read, 1585 .write = virtio_pci_common_write, 1586 .impl = { 1587 .min_access_size = 1, 1588 .max_access_size = 4, 1589 }, 1590 .endianness = DEVICE_LITTLE_ENDIAN, 1591 }; 1592 static const MemoryRegionOps isr_ops = { 1593 .read = virtio_pci_isr_read, 1594 .write = virtio_pci_isr_write, 1595 .impl = { 1596 .min_access_size = 1, 1597 .max_access_size = 4, 1598 }, 1599 .endianness = DEVICE_LITTLE_ENDIAN, 1600 }; 1601 static const MemoryRegionOps device_ops = { 1602 .read = virtio_pci_device_read, 1603 .write = virtio_pci_device_write, 1604 .impl = { 1605 .min_access_size = 1, 1606 .max_access_size = 4, 1607 }, 1608 .endianness = DEVICE_LITTLE_ENDIAN, 1609 }; 1610 static const MemoryRegionOps notify_ops = { 1611 .read = virtio_pci_notify_read, 1612 .write = virtio_pci_notify_write, 1613 .impl = { 1614 .min_access_size = 1, 1615 .max_access_size = 4, 1616 }, 1617 .endianness = DEVICE_LITTLE_ENDIAN, 1618 }; 1619 static const MemoryRegionOps notify_pio_ops = { 1620 .read = virtio_pci_notify_read, 1621 .write = virtio_pci_notify_write_pio, 1622 .impl = { 1623 .min_access_size = 1, 1624 .max_access_size = 4, 1625 }, 1626 .endianness = DEVICE_LITTLE_ENDIAN, 1627 }; 1628 g_autoptr(GString) name = g_string_new(NULL); 1629 1630 g_string_printf(name, "virtio-pci-common-%s", vdev_name); 1631 memory_region_init_io(&proxy->common.mr, OBJECT(proxy), 1632 &common_ops, 1633 proxy, 1634 name->str, 1635 proxy->common.size); 1636 1637 g_string_printf(name, "virtio-pci-isr-%s", vdev_name); 1638 memory_region_init_io(&proxy->isr.mr, OBJECT(proxy), 1639 &isr_ops, 1640 proxy, 1641 name->str, 1642 proxy->isr.size); 1643 1644 g_string_printf(name, "virtio-pci-device-%s", vdev_name); 1645 memory_region_init_io(&proxy->device.mr, OBJECT(proxy), 1646 &device_ops, 1647 proxy, 1648 name->str, 1649 proxy->device.size); 1650 1651 g_string_printf(name, "virtio-pci-notify-%s", vdev_name); 1652 memory_region_init_io(&proxy->notify.mr, OBJECT(proxy), 1653 ¬ify_ops, 1654 proxy, 1655 name->str, 1656 proxy->notify.size); 1657 1658 g_string_printf(name, "virtio-pci-notify-pio-%s", vdev_name); 1659 memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy), 1660 ¬ify_pio_ops, 1661 proxy, 1662 name->str, 1663 proxy->notify_pio.size); 1664 } 1665 1666 static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy, 1667 VirtIOPCIRegion *region, 1668 struct virtio_pci_cap *cap, 1669 MemoryRegion *mr, 1670 uint8_t bar) 1671 { 1672 memory_region_add_subregion(mr, region->offset, ®ion->mr); 1673 1674 cap->cfg_type = region->type; 1675 cap->bar = bar; 1676 cap->offset = cpu_to_le32(region->offset); 1677 cap->length = cpu_to_le32(region->size); 1678 virtio_pci_add_mem_cap(proxy, cap); 1679 1680 } 1681 1682 static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy, 1683 VirtIOPCIRegion *region, 1684 struct virtio_pci_cap *cap) 1685 { 1686 virtio_pci_modern_region_map(proxy, region, cap, 1687 &proxy->modern_bar, proxy->modern_mem_bar_idx); 1688 } 1689 1690 static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy, 1691 VirtIOPCIRegion *region, 1692 struct virtio_pci_cap *cap) 1693 { 1694 virtio_pci_modern_region_map(proxy, region, cap, 1695 &proxy->io_bar, proxy->modern_io_bar_idx); 1696 } 1697 1698 static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy, 1699 VirtIOPCIRegion *region) 1700 { 1701 memory_region_del_subregion(&proxy->modern_bar, 1702 ®ion->mr); 1703 } 1704 1705 static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy, 1706 VirtIOPCIRegion *region) 1707 { 1708 memory_region_del_subregion(&proxy->io_bar, 1709 ®ion->mr); 1710 } 1711 1712 static void virtio_pci_pre_plugged(DeviceState *d, Error **errp) 1713 { 1714 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1715 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1716 1717 if (virtio_pci_modern(proxy)) { 1718 virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1); 1719 } 1720 1721 virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE); 1722 } 1723 1724 /* This is called by virtio-bus just after the device is plugged. */ 1725 static void virtio_pci_device_plugged(DeviceState *d, Error **errp) 1726 { 1727 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1728 VirtioBusState *bus = &proxy->bus; 1729 bool legacy = virtio_pci_legacy(proxy); 1730 bool modern; 1731 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 1732 uint8_t *config; 1733 uint32_t size; 1734 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1735 1736 /* 1737 * Virtio capabilities present without 1738 * VIRTIO_F_VERSION_1 confuses guests 1739 */ 1740 if (!proxy->ignore_backend_features && 1741 !virtio_has_feature(vdev->host_features, VIRTIO_F_VERSION_1)) { 1742 virtio_pci_disable_modern(proxy); 1743 1744 if (!legacy) { 1745 error_setg(errp, "Device doesn't support modern mode, and legacy" 1746 " mode is disabled"); 1747 error_append_hint(errp, "Set disable-legacy to off\n"); 1748 1749 return; 1750 } 1751 } 1752 1753 modern = virtio_pci_modern(proxy); 1754 1755 config = proxy->pci_dev.config; 1756 if (proxy->class_code) { 1757 pci_config_set_class(config, proxy->class_code); 1758 } 1759 1760 if (legacy) { 1761 if (!virtio_legacy_allowed(vdev)) { 1762 /* 1763 * To avoid migration issues, we allow legacy mode when legacy 1764 * check is disabled in the old machine types (< 5.1). 1765 */ 1766 if (virtio_legacy_check_disabled(vdev)) { 1767 warn_report("device is modern-only, but for backward " 1768 "compatibility legacy is allowed"); 1769 } else { 1770 error_setg(errp, 1771 "device is modern-only, use disable-legacy=on"); 1772 return; 1773 } 1774 } 1775 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 1776 error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by" 1777 " neither legacy nor transitional device"); 1778 return ; 1779 } 1780 /* 1781 * Legacy and transitional devices use specific subsystem IDs. 1782 * Note that the subsystem vendor ID (config + PCI_SUBSYSTEM_VENDOR_ID) 1783 * is set to PCI_SUBVENDOR_ID_REDHAT_QUMRANET by default. 1784 */ 1785 pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus)); 1786 } else { 1787 /* pure virtio-1.0 */ 1788 pci_set_word(config + PCI_VENDOR_ID, 1789 PCI_VENDOR_ID_REDHAT_QUMRANET); 1790 pci_set_word(config + PCI_DEVICE_ID, 1791 0x1040 + virtio_bus_get_vdev_id(bus)); 1792 pci_config_set_revision(config, 1); 1793 } 1794 config[PCI_INTERRUPT_PIN] = 1; 1795 1796 1797 if (modern) { 1798 struct virtio_pci_cap cap = { 1799 .cap_len = sizeof cap, 1800 }; 1801 struct virtio_pci_notify_cap notify = { 1802 .cap.cap_len = sizeof notify, 1803 .notify_off_multiplier = 1804 cpu_to_le32(virtio_pci_queue_mem_mult(proxy)), 1805 }; 1806 struct virtio_pci_cfg_cap cfg = { 1807 .cap.cap_len = sizeof cfg, 1808 .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG, 1809 }; 1810 struct virtio_pci_notify_cap notify_pio = { 1811 .cap.cap_len = sizeof notify, 1812 .notify_off_multiplier = cpu_to_le32(0x0), 1813 }; 1814 1815 struct virtio_pci_cfg_cap *cfg_mask; 1816 1817 virtio_pci_modern_regions_init(proxy, vdev->name); 1818 1819 virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap); 1820 virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap); 1821 virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap); 1822 virtio_pci_modern_mem_region_map(proxy, &proxy->notify, ¬ify.cap); 1823 1824 if (modern_pio) { 1825 memory_region_init(&proxy->io_bar, OBJECT(proxy), 1826 "virtio-pci-io", 0x4); 1827 1828 pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar_idx, 1829 PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar); 1830 1831 virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio, 1832 ¬ify_pio.cap); 1833 } 1834 1835 pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar_idx, 1836 PCI_BASE_ADDRESS_SPACE_MEMORY | 1837 PCI_BASE_ADDRESS_MEM_PREFETCH | 1838 PCI_BASE_ADDRESS_MEM_TYPE_64, 1839 &proxy->modern_bar); 1840 1841 proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap); 1842 cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap); 1843 pci_set_byte(&cfg_mask->cap.bar, ~0x0); 1844 pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0); 1845 pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0); 1846 pci_set_long(cfg_mask->pci_cfg_data, ~0x0); 1847 } 1848 1849 if (proxy->nvectors) { 1850 int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors, 1851 proxy->msix_bar_idx, NULL); 1852 if (err) { 1853 /* Notice when a system that supports MSIx can't initialize it */ 1854 if (err != -ENOTSUP) { 1855 warn_report("unable to init msix vectors to %" PRIu32, 1856 proxy->nvectors); 1857 } 1858 proxy->nvectors = 0; 1859 } 1860 } 1861 1862 proxy->pci_dev.config_write = virtio_write_config; 1863 proxy->pci_dev.config_read = virtio_read_config; 1864 1865 if (legacy) { 1866 size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) 1867 + virtio_bus_get_vdev_config_len(bus); 1868 size = pow2ceil(size); 1869 1870 memory_region_init_io(&proxy->bar, OBJECT(proxy), 1871 &virtio_pci_config_ops, 1872 proxy, "virtio-pci", size); 1873 1874 pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx, 1875 PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar); 1876 } 1877 } 1878 1879 static void virtio_pci_device_unplugged(DeviceState *d) 1880 { 1881 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1882 bool modern = virtio_pci_modern(proxy); 1883 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 1884 1885 virtio_pci_stop_ioeventfd(proxy); 1886 1887 if (modern) { 1888 virtio_pci_modern_mem_region_unmap(proxy, &proxy->common); 1889 virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr); 1890 virtio_pci_modern_mem_region_unmap(proxy, &proxy->device); 1891 virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify); 1892 if (modern_pio) { 1893 virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio); 1894 } 1895 } 1896 } 1897 1898 static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp) 1899 { 1900 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 1901 VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev); 1902 bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) && 1903 !pci_bus_is_root(pci_get_bus(pci_dev)); 1904 1905 if (kvm_enabled() && !kvm_has_many_ioeventfds()) { 1906 proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD; 1907 } 1908 1909 /* fd-based ioevents can't be synchronized in record/replay */ 1910 if (replay_mode != REPLAY_MODE_NONE) { 1911 proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD; 1912 } 1913 1914 /* 1915 * virtio pci bar layout used by default. 1916 * subclasses can re-arrange things if needed. 1917 * 1918 * region 0 -- virtio legacy io bar 1919 * region 1 -- msi-x bar 1920 * region 2 -- virtio modern io bar (off by default) 1921 * region 4+5 -- virtio modern memory (64bit) bar 1922 * 1923 */ 1924 proxy->legacy_io_bar_idx = 0; 1925 proxy->msix_bar_idx = 1; 1926 proxy->modern_io_bar_idx = 2; 1927 proxy->modern_mem_bar_idx = 4; 1928 1929 proxy->common.offset = 0x0; 1930 proxy->common.size = 0x1000; 1931 proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG; 1932 1933 proxy->isr.offset = 0x1000; 1934 proxy->isr.size = 0x1000; 1935 proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG; 1936 1937 proxy->device.offset = 0x2000; 1938 proxy->device.size = 0x1000; 1939 proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG; 1940 1941 proxy->notify.offset = 0x3000; 1942 proxy->notify.size = virtio_pci_queue_mem_mult(proxy) * VIRTIO_QUEUE_MAX; 1943 proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG; 1944 1945 proxy->notify_pio.offset = 0x0; 1946 proxy->notify_pio.size = 0x4; 1947 proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG; 1948 1949 /* subclasses can enforce modern, so do this unconditionally */ 1950 memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci", 1951 /* PCI BAR regions must be powers of 2 */ 1952 pow2ceil(proxy->notify.offset + proxy->notify.size)); 1953 1954 if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) { 1955 proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 1956 } 1957 1958 if (!virtio_pci_modern(proxy) && !virtio_pci_legacy(proxy)) { 1959 error_setg(errp, "device cannot work as neither modern nor legacy mode" 1960 " is enabled"); 1961 error_append_hint(errp, "Set either disable-modern or disable-legacy" 1962 " to off\n"); 1963 return; 1964 } 1965 1966 if (pcie_port && pci_is_express(pci_dev)) { 1967 int pos; 1968 uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE; 1969 1970 pos = pcie_endpoint_cap_init(pci_dev, 0); 1971 assert(pos > 0); 1972 1973 pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0, 1974 PCI_PM_SIZEOF, errp); 1975 if (pos < 0) { 1976 return; 1977 } 1978 1979 pci_dev->exp.pm_cap = pos; 1980 1981 /* 1982 * Indicates that this function complies with revision 1.2 of the 1983 * PCI Power Management Interface Specification. 1984 */ 1985 pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3); 1986 1987 if (proxy->flags & VIRTIO_PCI_FLAG_AER) { 1988 pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset, 1989 PCI_ERR_SIZEOF, NULL); 1990 last_pcie_cap_offset += PCI_ERR_SIZEOF; 1991 } 1992 1993 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) { 1994 /* Init error enabling flags */ 1995 pcie_cap_deverr_init(pci_dev); 1996 } 1997 1998 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_LNKCTL) { 1999 /* Init Link Control Register */ 2000 pcie_cap_lnkctl_init(pci_dev); 2001 } 2002 2003 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_PM) { 2004 /* Init Power Management Control Register */ 2005 pci_set_word(pci_dev->wmask + pos + PCI_PM_CTRL, 2006 PCI_PM_CTRL_STATE_MASK); 2007 } 2008 2009 if (proxy->flags & VIRTIO_PCI_FLAG_ATS) { 2010 pcie_ats_init(pci_dev, last_pcie_cap_offset, 2011 proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED); 2012 last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF; 2013 } 2014 2015 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) { 2016 /* Set Function Level Reset capability bit */ 2017 pcie_cap_flr_init(pci_dev); 2018 } 2019 } else { 2020 /* 2021 * make future invocations of pci_is_express() return false 2022 * and pci_config_size() return PCI_CONFIG_SPACE_SIZE. 2023 */ 2024 pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS; 2025 } 2026 2027 virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy); 2028 if (k->realize) { 2029 k->realize(proxy, errp); 2030 } 2031 } 2032 2033 static void virtio_pci_exit(PCIDevice *pci_dev) 2034 { 2035 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 2036 bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) && 2037 !pci_bus_is_root(pci_get_bus(pci_dev)); 2038 2039 msix_uninit_exclusive_bar(pci_dev); 2040 if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port && 2041 pci_is_express(pci_dev)) { 2042 pcie_aer_exit(pci_dev); 2043 } 2044 } 2045 2046 static void virtio_pci_reset(DeviceState *qdev) 2047 { 2048 VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev); 2049 VirtioBusState *bus = VIRTIO_BUS(&proxy->bus); 2050 PCIDevice *dev = PCI_DEVICE(qdev); 2051 int i; 2052 2053 virtio_pci_stop_ioeventfd(proxy); 2054 virtio_bus_reset(bus); 2055 msix_unuse_all_vectors(&proxy->pci_dev); 2056 2057 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2058 proxy->vqs[i].enabled = 0; 2059 proxy->vqs[i].num = 0; 2060 proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0; 2061 proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0; 2062 proxy->vqs[i].used[0] = proxy->vqs[i].used[1] = 0; 2063 } 2064 2065 if (pci_is_express(dev)) { 2066 pcie_cap_deverr_reset(dev); 2067 pcie_cap_lnkctl_reset(dev); 2068 2069 pci_set_word(dev->config + dev->exp.pm_cap + PCI_PM_CTRL, 0); 2070 } 2071 } 2072 2073 static Property virtio_pci_properties[] = { 2074 DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags, 2075 VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false), 2076 DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags, 2077 VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true), 2078 DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags, 2079 VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false), 2080 DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags, 2081 VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false), 2082 DEFINE_PROP_BIT("page-per-vq", VirtIOPCIProxy, flags, 2083 VIRTIO_PCI_FLAG_PAGE_PER_VQ_BIT, false), 2084 DEFINE_PROP_BOOL("x-ignore-backend-features", VirtIOPCIProxy, 2085 ignore_backend_features, false), 2086 DEFINE_PROP_BIT("ats", VirtIOPCIProxy, flags, 2087 VIRTIO_PCI_FLAG_ATS_BIT, false), 2088 DEFINE_PROP_BIT("x-ats-page-aligned", VirtIOPCIProxy, flags, 2089 VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED_BIT, true), 2090 DEFINE_PROP_BIT("x-pcie-deverr-init", VirtIOPCIProxy, flags, 2091 VIRTIO_PCI_FLAG_INIT_DEVERR_BIT, true), 2092 DEFINE_PROP_BIT("x-pcie-lnkctl-init", VirtIOPCIProxy, flags, 2093 VIRTIO_PCI_FLAG_INIT_LNKCTL_BIT, true), 2094 DEFINE_PROP_BIT("x-pcie-pm-init", VirtIOPCIProxy, flags, 2095 VIRTIO_PCI_FLAG_INIT_PM_BIT, true), 2096 DEFINE_PROP_BIT("x-pcie-flr-init", VirtIOPCIProxy, flags, 2097 VIRTIO_PCI_FLAG_INIT_FLR_BIT, true), 2098 DEFINE_PROP_BIT("aer", VirtIOPCIProxy, flags, 2099 VIRTIO_PCI_FLAG_AER_BIT, false), 2100 DEFINE_PROP_END_OF_LIST(), 2101 }; 2102 2103 static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp) 2104 { 2105 VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev); 2106 VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev); 2107 PCIDevice *pci_dev = &proxy->pci_dev; 2108 2109 if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) && 2110 virtio_pci_modern(proxy)) { 2111 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS; 2112 } 2113 2114 vpciklass->parent_dc_realize(qdev, errp); 2115 } 2116 2117 static void virtio_pci_class_init(ObjectClass *klass, void *data) 2118 { 2119 DeviceClass *dc = DEVICE_CLASS(klass); 2120 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 2121 VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass); 2122 2123 device_class_set_props(dc, virtio_pci_properties); 2124 k->realize = virtio_pci_realize; 2125 k->exit = virtio_pci_exit; 2126 k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET; 2127 k->revision = VIRTIO_PCI_ABI_VERSION; 2128 k->class_id = PCI_CLASS_OTHERS; 2129 device_class_set_parent_realize(dc, virtio_pci_dc_realize, 2130 &vpciklass->parent_dc_realize); 2131 dc->reset = virtio_pci_reset; 2132 } 2133 2134 static const TypeInfo virtio_pci_info = { 2135 .name = TYPE_VIRTIO_PCI, 2136 .parent = TYPE_PCI_DEVICE, 2137 .instance_size = sizeof(VirtIOPCIProxy), 2138 .class_init = virtio_pci_class_init, 2139 .class_size = sizeof(VirtioPCIClass), 2140 .abstract = true, 2141 }; 2142 2143 static Property virtio_pci_generic_properties[] = { 2144 DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy, 2145 ON_OFF_AUTO_AUTO), 2146 DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false), 2147 DEFINE_PROP_END_OF_LIST(), 2148 }; 2149 2150 static void virtio_pci_base_class_init(ObjectClass *klass, void *data) 2151 { 2152 const VirtioPCIDeviceTypeInfo *t = data; 2153 if (t->class_init) { 2154 t->class_init(klass, NULL); 2155 } 2156 } 2157 2158 static void virtio_pci_generic_class_init(ObjectClass *klass, void *data) 2159 { 2160 DeviceClass *dc = DEVICE_CLASS(klass); 2161 2162 device_class_set_props(dc, virtio_pci_generic_properties); 2163 } 2164 2165 static void virtio_pci_transitional_instance_init(Object *obj) 2166 { 2167 VirtIOPCIProxy *proxy = VIRTIO_PCI(obj); 2168 2169 proxy->disable_legacy = ON_OFF_AUTO_OFF; 2170 proxy->disable_modern = false; 2171 } 2172 2173 static void virtio_pci_non_transitional_instance_init(Object *obj) 2174 { 2175 VirtIOPCIProxy *proxy = VIRTIO_PCI(obj); 2176 2177 proxy->disable_legacy = ON_OFF_AUTO_ON; 2178 proxy->disable_modern = false; 2179 } 2180 2181 void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t) 2182 { 2183 char *base_name = NULL; 2184 TypeInfo base_type_info = { 2185 .name = t->base_name, 2186 .parent = t->parent ? t->parent : TYPE_VIRTIO_PCI, 2187 .instance_size = t->instance_size, 2188 .instance_init = t->instance_init, 2189 .class_size = t->class_size, 2190 .abstract = true, 2191 .interfaces = t->interfaces, 2192 }; 2193 TypeInfo generic_type_info = { 2194 .name = t->generic_name, 2195 .parent = base_type_info.name, 2196 .class_init = virtio_pci_generic_class_init, 2197 .interfaces = (InterfaceInfo[]) { 2198 { INTERFACE_PCIE_DEVICE }, 2199 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 2200 { } 2201 }, 2202 }; 2203 2204 if (!base_type_info.name) { 2205 /* No base type -> register a single generic device type */ 2206 /* use intermediate %s-base-type to add generic device props */ 2207 base_name = g_strdup_printf("%s-base-type", t->generic_name); 2208 base_type_info.name = base_name; 2209 base_type_info.class_init = virtio_pci_generic_class_init; 2210 2211 generic_type_info.parent = base_name; 2212 generic_type_info.class_init = virtio_pci_base_class_init; 2213 generic_type_info.class_data = (void *)t; 2214 2215 assert(!t->non_transitional_name); 2216 assert(!t->transitional_name); 2217 } else { 2218 base_type_info.class_init = virtio_pci_base_class_init; 2219 base_type_info.class_data = (void *)t; 2220 } 2221 2222 type_register(&base_type_info); 2223 if (generic_type_info.name) { 2224 type_register(&generic_type_info); 2225 } 2226 2227 if (t->non_transitional_name) { 2228 const TypeInfo non_transitional_type_info = { 2229 .name = t->non_transitional_name, 2230 .parent = base_type_info.name, 2231 .instance_init = virtio_pci_non_transitional_instance_init, 2232 .interfaces = (InterfaceInfo[]) { 2233 { INTERFACE_PCIE_DEVICE }, 2234 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 2235 { } 2236 }, 2237 }; 2238 type_register(&non_transitional_type_info); 2239 } 2240 2241 if (t->transitional_name) { 2242 const TypeInfo transitional_type_info = { 2243 .name = t->transitional_name, 2244 .parent = base_type_info.name, 2245 .instance_init = virtio_pci_transitional_instance_init, 2246 .interfaces = (InterfaceInfo[]) { 2247 /* 2248 * Transitional virtio devices work only as Conventional PCI 2249 * devices because they require PIO ports. 2250 */ 2251 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 2252 { } 2253 }, 2254 }; 2255 type_register(&transitional_type_info); 2256 } 2257 g_free(base_name); 2258 } 2259 2260 unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues) 2261 { 2262 /* 2263 * 1:1 vq to vCPU mapping is ideal because the same vCPU that submitted 2264 * virtqueue buffers can handle their completion. When a different vCPU 2265 * handles completion it may need to IPI the vCPU that submitted the 2266 * request and this adds overhead. 2267 * 2268 * Virtqueues consume guest RAM and MSI-X vectors. This is wasteful in 2269 * guests with very many vCPUs and a device that is only used by a few 2270 * vCPUs. Unfortunately optimizing that case requires manual pinning inside 2271 * the guest, so those users might as well manually set the number of 2272 * queues. There is no upper limit that can be applied automatically and 2273 * doing so arbitrarily would result in a sudden performance drop once the 2274 * threshold number of vCPUs is exceeded. 2275 */ 2276 unsigned num_queues = current_machine->smp.cpus; 2277 2278 /* 2279 * The maximum number of MSI-X vectors is PCI_MSIX_FLAGS_QSIZE + 1, but the 2280 * config change interrupt and the fixed virtqueues must be taken into 2281 * account too. 2282 */ 2283 num_queues = MIN(num_queues, PCI_MSIX_FLAGS_QSIZE - fixed_queues); 2284 2285 /* 2286 * There is a limit to how many virtqueues a device can have. 2287 */ 2288 return MIN(num_queues, VIRTIO_QUEUE_MAX - fixed_queues); 2289 } 2290 2291 /* virtio-pci-bus */ 2292 2293 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size, 2294 VirtIOPCIProxy *dev) 2295 { 2296 DeviceState *qdev = DEVICE(dev); 2297 char virtio_bus_name[] = "virtio-bus"; 2298 2299 qbus_init(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev, virtio_bus_name); 2300 } 2301 2302 static void virtio_pci_bus_class_init(ObjectClass *klass, void *data) 2303 { 2304 BusClass *bus_class = BUS_CLASS(klass); 2305 VirtioBusClass *k = VIRTIO_BUS_CLASS(klass); 2306 bus_class->max_dev = 1; 2307 k->notify = virtio_pci_notify; 2308 k->save_config = virtio_pci_save_config; 2309 k->load_config = virtio_pci_load_config; 2310 k->save_queue = virtio_pci_save_queue; 2311 k->load_queue = virtio_pci_load_queue; 2312 k->save_extra_state = virtio_pci_save_extra_state; 2313 k->load_extra_state = virtio_pci_load_extra_state; 2314 k->has_extra_state = virtio_pci_has_extra_state; 2315 k->query_guest_notifiers = virtio_pci_query_guest_notifiers; 2316 k->set_guest_notifiers = virtio_pci_set_guest_notifiers; 2317 k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr; 2318 k->vmstate_change = virtio_pci_vmstate_change; 2319 k->pre_plugged = virtio_pci_pre_plugged; 2320 k->device_plugged = virtio_pci_device_plugged; 2321 k->device_unplugged = virtio_pci_device_unplugged; 2322 k->query_nvectors = virtio_pci_query_nvectors; 2323 k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled; 2324 k->ioeventfd_assign = virtio_pci_ioeventfd_assign; 2325 k->get_dma_as = virtio_pci_get_dma_as; 2326 k->iommu_enabled = virtio_pci_iommu_enabled; 2327 k->queue_enabled = virtio_pci_queue_enabled; 2328 } 2329 2330 static const TypeInfo virtio_pci_bus_info = { 2331 .name = TYPE_VIRTIO_PCI_BUS, 2332 .parent = TYPE_VIRTIO_BUS, 2333 .instance_size = sizeof(VirtioPCIBusState), 2334 .class_size = sizeof(VirtioPCIBusClass), 2335 .class_init = virtio_pci_bus_class_init, 2336 }; 2337 2338 static void virtio_pci_register_types(void) 2339 { 2340 /* Base types: */ 2341 type_register_static(&virtio_pci_bus_info); 2342 type_register_static(&virtio_pci_info); 2343 } 2344 2345 type_init(virtio_pci_register_types) 2346 2347