1 /* 2 * Virtio PCI Bindings 3 * 4 * Copyright IBM, Corp. 2007 5 * Copyright (c) 2009 CodeSourcery 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * Paul Brook <paul@codesourcery.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2. See 12 * the COPYING file in the top-level directory. 13 * 14 * Contributions after 2012-01-13 are licensed under the terms of the 15 * GNU GPL, version 2 or (at your option) any later version. 16 */ 17 18 #include "qemu/osdep.h" 19 20 #include "exec/memop.h" 21 #include "standard-headers/linux/virtio_pci.h" 22 #include "standard-headers/linux/virtio_ids.h" 23 #include "hw/boards.h" 24 #include "hw/virtio/virtio.h" 25 #include "migration/qemu-file-types.h" 26 #include "hw/pci/pci.h" 27 #include "hw/pci/pci_bus.h" 28 #include "hw/qdev-properties.h" 29 #include "qapi/error.h" 30 #include "qemu/error-report.h" 31 #include "qemu/log.h" 32 #include "qemu/module.h" 33 #include "hw/pci/msi.h" 34 #include "hw/pci/msix.h" 35 #include "hw/loader.h" 36 #include "sysemu/kvm.h" 37 #include "hw/virtio/virtio-pci.h" 38 #include "qemu/range.h" 39 #include "hw/virtio/virtio-bus.h" 40 #include "qapi/visitor.h" 41 #include "sysemu/replay.h" 42 #include "trace.h" 43 44 #define VIRTIO_PCI_REGION_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_present(dev)) 45 46 #undef VIRTIO_PCI_CONFIG 47 48 /* The remaining space is defined by each driver as the per-driver 49 * configuration space */ 50 #define VIRTIO_PCI_CONFIG_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev)) 51 52 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size, 53 VirtIOPCIProxy *dev); 54 static void virtio_pci_reset(DeviceState *qdev); 55 56 /* virtio device */ 57 /* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */ 58 static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d) 59 { 60 return container_of(d, VirtIOPCIProxy, pci_dev.qdev); 61 } 62 63 /* DeviceState to VirtIOPCIProxy. Note: used on datapath, 64 * be careful and test performance if you change this. 65 */ 66 static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d) 67 { 68 return container_of(d, VirtIOPCIProxy, pci_dev.qdev); 69 } 70 71 static void virtio_pci_notify(DeviceState *d, uint16_t vector) 72 { 73 VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d); 74 75 if (msix_enabled(&proxy->pci_dev)) { 76 if (vector != VIRTIO_NO_VECTOR) { 77 msix_notify(&proxy->pci_dev, vector); 78 } 79 } else { 80 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 81 pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1); 82 } 83 } 84 85 static void virtio_pci_save_config(DeviceState *d, QEMUFile *f) 86 { 87 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 88 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 89 90 pci_device_save(&proxy->pci_dev, f); 91 msix_save(&proxy->pci_dev, f); 92 if (msix_present(&proxy->pci_dev)) 93 qemu_put_be16(f, vdev->config_vector); 94 } 95 96 static const VMStateDescription vmstate_virtio_pci_modern_queue_state = { 97 .name = "virtio_pci/modern_queue_state", 98 .version_id = 1, 99 .minimum_version_id = 1, 100 .fields = (const VMStateField[]) { 101 VMSTATE_UINT16(num, VirtIOPCIQueue), 102 VMSTATE_UNUSED(1), /* enabled was stored as be16 */ 103 VMSTATE_BOOL(enabled, VirtIOPCIQueue), 104 VMSTATE_UINT32_ARRAY(desc, VirtIOPCIQueue, 2), 105 VMSTATE_UINT32_ARRAY(avail, VirtIOPCIQueue, 2), 106 VMSTATE_UINT32_ARRAY(used, VirtIOPCIQueue, 2), 107 VMSTATE_END_OF_LIST() 108 } 109 }; 110 111 static bool virtio_pci_modern_state_needed(void *opaque) 112 { 113 VirtIOPCIProxy *proxy = opaque; 114 115 return virtio_pci_modern(proxy); 116 } 117 118 static const VMStateDescription vmstate_virtio_pci_modern_state_sub = { 119 .name = "virtio_pci/modern_state", 120 .version_id = 1, 121 .minimum_version_id = 1, 122 .needed = &virtio_pci_modern_state_needed, 123 .fields = (const VMStateField[]) { 124 VMSTATE_UINT32(dfselect, VirtIOPCIProxy), 125 VMSTATE_UINT32(gfselect, VirtIOPCIProxy), 126 VMSTATE_UINT32_ARRAY(guest_features, VirtIOPCIProxy, 2), 127 VMSTATE_STRUCT_ARRAY(vqs, VirtIOPCIProxy, VIRTIO_QUEUE_MAX, 0, 128 vmstate_virtio_pci_modern_queue_state, 129 VirtIOPCIQueue), 130 VMSTATE_END_OF_LIST() 131 } 132 }; 133 134 static const VMStateDescription vmstate_virtio_pci = { 135 .name = "virtio_pci", 136 .version_id = 1, 137 .minimum_version_id = 1, 138 .fields = (const VMStateField[]) { 139 VMSTATE_END_OF_LIST() 140 }, 141 .subsections = (const VMStateDescription * const []) { 142 &vmstate_virtio_pci_modern_state_sub, 143 NULL 144 } 145 }; 146 147 static bool virtio_pci_has_extra_state(DeviceState *d) 148 { 149 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 150 151 return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA; 152 } 153 154 static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f) 155 { 156 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 157 158 vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL); 159 } 160 161 static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f) 162 { 163 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 164 165 return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1); 166 } 167 168 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f) 169 { 170 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 171 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 172 173 if (msix_present(&proxy->pci_dev)) 174 qemu_put_be16(f, virtio_queue_vector(vdev, n)); 175 } 176 177 static int virtio_pci_load_config(DeviceState *d, QEMUFile *f) 178 { 179 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 180 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 181 uint16_t vector; 182 183 int ret; 184 ret = pci_device_load(&proxy->pci_dev, f); 185 if (ret) { 186 return ret; 187 } 188 msix_unuse_all_vectors(&proxy->pci_dev); 189 msix_load(&proxy->pci_dev, f); 190 if (msix_present(&proxy->pci_dev)) { 191 qemu_get_be16s(f, &vector); 192 193 if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) { 194 return -EINVAL; 195 } 196 } else { 197 vector = VIRTIO_NO_VECTOR; 198 } 199 vdev->config_vector = vector; 200 if (vector != VIRTIO_NO_VECTOR) { 201 msix_vector_use(&proxy->pci_dev, vector); 202 } 203 return 0; 204 } 205 206 static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f) 207 { 208 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 209 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 210 211 uint16_t vector; 212 if (msix_present(&proxy->pci_dev)) { 213 qemu_get_be16s(f, &vector); 214 if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) { 215 return -EINVAL; 216 } 217 } else { 218 vector = VIRTIO_NO_VECTOR; 219 } 220 virtio_queue_set_vector(vdev, n, vector); 221 if (vector != VIRTIO_NO_VECTOR) { 222 msix_vector_use(&proxy->pci_dev, vector); 223 } 224 225 return 0; 226 } 227 228 typedef struct VirtIOPCIIDInfo { 229 /* virtio id */ 230 uint16_t vdev_id; 231 /* pci device id for the transitional device */ 232 uint16_t trans_devid; 233 uint16_t class_id; 234 } VirtIOPCIIDInfo; 235 236 static const VirtIOPCIIDInfo virtio_pci_id_info[] = { 237 { 238 .vdev_id = VIRTIO_ID_CRYPTO, 239 .class_id = PCI_CLASS_OTHERS, 240 }, { 241 .vdev_id = VIRTIO_ID_FS, 242 .class_id = PCI_CLASS_STORAGE_OTHER, 243 }, { 244 .vdev_id = VIRTIO_ID_NET, 245 .trans_devid = PCI_DEVICE_ID_VIRTIO_NET, 246 .class_id = PCI_CLASS_NETWORK_ETHERNET, 247 }, { 248 .vdev_id = VIRTIO_ID_BLOCK, 249 .trans_devid = PCI_DEVICE_ID_VIRTIO_BLOCK, 250 .class_id = PCI_CLASS_STORAGE_SCSI, 251 }, { 252 .vdev_id = VIRTIO_ID_CONSOLE, 253 .trans_devid = PCI_DEVICE_ID_VIRTIO_CONSOLE, 254 .class_id = PCI_CLASS_COMMUNICATION_OTHER, 255 }, { 256 .vdev_id = VIRTIO_ID_SCSI, 257 .trans_devid = PCI_DEVICE_ID_VIRTIO_SCSI, 258 .class_id = PCI_CLASS_STORAGE_SCSI 259 }, { 260 .vdev_id = VIRTIO_ID_9P, 261 .trans_devid = PCI_DEVICE_ID_VIRTIO_9P, 262 .class_id = PCI_BASE_CLASS_NETWORK, 263 }, { 264 .vdev_id = VIRTIO_ID_BALLOON, 265 .trans_devid = PCI_DEVICE_ID_VIRTIO_BALLOON, 266 .class_id = PCI_CLASS_OTHERS, 267 }, { 268 .vdev_id = VIRTIO_ID_RNG, 269 .trans_devid = PCI_DEVICE_ID_VIRTIO_RNG, 270 .class_id = PCI_CLASS_OTHERS, 271 }, 272 }; 273 274 static const VirtIOPCIIDInfo *virtio_pci_get_id_info(uint16_t vdev_id) 275 { 276 const VirtIOPCIIDInfo *info = NULL; 277 int i; 278 279 for (i = 0; i < ARRAY_SIZE(virtio_pci_id_info); i++) { 280 if (virtio_pci_id_info[i].vdev_id == vdev_id) { 281 info = &virtio_pci_id_info[i]; 282 break; 283 } 284 } 285 286 if (!info) { 287 /* The device id is invalid or not added to the id_info yet. */ 288 error_report("Invalid virtio device(id %u)", vdev_id); 289 abort(); 290 } 291 292 return info; 293 } 294 295 /* 296 * Get the Transitional Device ID for the specific device, return 297 * zero if the device is non-transitional. 298 */ 299 uint16_t virtio_pci_get_trans_devid(uint16_t device_id) 300 { 301 return virtio_pci_get_id_info(device_id)->trans_devid; 302 } 303 304 /* 305 * Get the Class ID for the specific device. 306 */ 307 uint16_t virtio_pci_get_class_id(uint16_t device_id) 308 { 309 return virtio_pci_get_id_info(device_id)->class_id; 310 } 311 312 static bool virtio_pci_ioeventfd_enabled(DeviceState *d) 313 { 314 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 315 316 return (proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) != 0; 317 } 318 319 #define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000 320 321 static inline int virtio_pci_queue_mem_mult(struct VirtIOPCIProxy *proxy) 322 { 323 return (proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ) ? 324 QEMU_VIRTIO_PCI_QUEUE_MEM_MULT : 4; 325 } 326 327 static int virtio_pci_ioeventfd_assign(DeviceState *d, EventNotifier *notifier, 328 int n, bool assign) 329 { 330 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 331 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 332 VirtQueue *vq = virtio_get_queue(vdev, n); 333 bool legacy = virtio_pci_legacy(proxy); 334 bool modern = virtio_pci_modern(proxy); 335 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 336 MemoryRegion *modern_mr = &proxy->notify.mr; 337 MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr; 338 MemoryRegion *legacy_mr = &proxy->bar; 339 hwaddr modern_addr = virtio_pci_queue_mem_mult(proxy) * 340 virtio_get_queue_index(vq); 341 hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY; 342 343 if (assign) { 344 if (modern) { 345 memory_region_add_eventfd(modern_mr, modern_addr, 0, 346 false, n, notifier); 347 if (modern_pio) { 348 memory_region_add_eventfd(modern_notify_mr, 0, 2, 349 true, n, notifier); 350 } 351 } 352 if (legacy) { 353 memory_region_add_eventfd(legacy_mr, legacy_addr, 2, 354 true, n, notifier); 355 } 356 } else { 357 if (modern) { 358 memory_region_del_eventfd(modern_mr, modern_addr, 0, 359 false, n, notifier); 360 if (modern_pio) { 361 memory_region_del_eventfd(modern_notify_mr, 0, 2, 362 true, n, notifier); 363 } 364 } 365 if (legacy) { 366 memory_region_del_eventfd(legacy_mr, legacy_addr, 2, 367 true, n, notifier); 368 } 369 } 370 return 0; 371 } 372 373 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy) 374 { 375 virtio_bus_start_ioeventfd(&proxy->bus); 376 } 377 378 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy) 379 { 380 virtio_bus_stop_ioeventfd(&proxy->bus); 381 } 382 383 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) 384 { 385 VirtIOPCIProxy *proxy = opaque; 386 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 387 uint16_t vector, vq_idx; 388 hwaddr pa; 389 390 switch (addr) { 391 case VIRTIO_PCI_GUEST_FEATURES: 392 /* Guest does not negotiate properly? We have to assume nothing. */ 393 if (val & (1 << VIRTIO_F_BAD_FEATURE)) { 394 val = virtio_bus_get_vdev_bad_features(&proxy->bus); 395 } 396 virtio_set_features(vdev, val); 397 break; 398 case VIRTIO_PCI_QUEUE_PFN: 399 pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT; 400 if (pa == 0) { 401 virtio_pci_reset(DEVICE(proxy)); 402 } 403 else 404 virtio_queue_set_addr(vdev, vdev->queue_sel, pa); 405 break; 406 case VIRTIO_PCI_QUEUE_SEL: 407 if (val < VIRTIO_QUEUE_MAX) 408 vdev->queue_sel = val; 409 break; 410 case VIRTIO_PCI_QUEUE_NOTIFY: 411 vq_idx = val; 412 if (vq_idx < VIRTIO_QUEUE_MAX && virtio_queue_get_num(vdev, vq_idx)) { 413 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFICATION_DATA)) { 414 VirtQueue *vq = virtio_get_queue(vdev, vq_idx); 415 416 virtio_queue_set_shadow_avail_idx(vq, val >> 16); 417 } 418 virtio_queue_notify(vdev, vq_idx); 419 } 420 break; 421 case VIRTIO_PCI_STATUS: 422 if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) { 423 virtio_pci_stop_ioeventfd(proxy); 424 } 425 426 virtio_set_status(vdev, val & 0xFF); 427 428 if (val & VIRTIO_CONFIG_S_DRIVER_OK) { 429 virtio_pci_start_ioeventfd(proxy); 430 } 431 432 if (vdev->status == 0) { 433 virtio_pci_reset(DEVICE(proxy)); 434 } 435 436 /* Linux before 2.6.34 drives the device without enabling 437 the PCI device bus master bit. Enable it automatically 438 for the guest. This is a PCI spec violation but so is 439 initiating DMA with bus master bit clear. */ 440 if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER)) { 441 pci_default_write_config(&proxy->pci_dev, PCI_COMMAND, 442 proxy->pci_dev.config[PCI_COMMAND] | 443 PCI_COMMAND_MASTER, 1); 444 } 445 break; 446 case VIRTIO_MSI_CONFIG_VECTOR: 447 if (vdev->config_vector != VIRTIO_NO_VECTOR) { 448 msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); 449 } 450 /* Make it possible for guest to discover an error took place. */ 451 if (val < proxy->nvectors) { 452 msix_vector_use(&proxy->pci_dev, val); 453 } else { 454 val = VIRTIO_NO_VECTOR; 455 } 456 vdev->config_vector = val; 457 break; 458 case VIRTIO_MSI_QUEUE_VECTOR: 459 vector = virtio_queue_vector(vdev, vdev->queue_sel); 460 if (vector != VIRTIO_NO_VECTOR) { 461 msix_vector_unuse(&proxy->pci_dev, vector); 462 } 463 /* Make it possible for guest to discover an error took place. */ 464 if (val < proxy->nvectors) { 465 msix_vector_use(&proxy->pci_dev, val); 466 } else { 467 val = VIRTIO_NO_VECTOR; 468 } 469 virtio_queue_set_vector(vdev, vdev->queue_sel, val); 470 break; 471 default: 472 qemu_log_mask(LOG_GUEST_ERROR, 473 "%s: unexpected address 0x%x value 0x%x\n", 474 __func__, addr, val); 475 break; 476 } 477 } 478 479 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr) 480 { 481 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 482 uint32_t ret = 0xFFFFFFFF; 483 484 switch (addr) { 485 case VIRTIO_PCI_HOST_FEATURES: 486 ret = vdev->host_features; 487 break; 488 case VIRTIO_PCI_GUEST_FEATURES: 489 ret = vdev->guest_features; 490 break; 491 case VIRTIO_PCI_QUEUE_PFN: 492 ret = virtio_queue_get_addr(vdev, vdev->queue_sel) 493 >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; 494 break; 495 case VIRTIO_PCI_QUEUE_NUM: 496 ret = virtio_queue_get_num(vdev, vdev->queue_sel); 497 break; 498 case VIRTIO_PCI_QUEUE_SEL: 499 ret = vdev->queue_sel; 500 break; 501 case VIRTIO_PCI_STATUS: 502 ret = vdev->status; 503 break; 504 case VIRTIO_PCI_ISR: 505 /* reading from the ISR also clears it. */ 506 ret = qatomic_xchg(&vdev->isr, 0); 507 pci_irq_deassert(&proxy->pci_dev); 508 break; 509 case VIRTIO_MSI_CONFIG_VECTOR: 510 ret = vdev->config_vector; 511 break; 512 case VIRTIO_MSI_QUEUE_VECTOR: 513 ret = virtio_queue_vector(vdev, vdev->queue_sel); 514 break; 515 default: 516 break; 517 } 518 519 return ret; 520 } 521 522 static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr, 523 unsigned size) 524 { 525 VirtIOPCIProxy *proxy = opaque; 526 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 527 uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev); 528 uint64_t val = 0; 529 530 if (vdev == NULL) { 531 return UINT64_MAX; 532 } 533 534 if (addr < config) { 535 return virtio_ioport_read(proxy, addr); 536 } 537 addr -= config; 538 539 switch (size) { 540 case 1: 541 val = virtio_config_readb(vdev, addr); 542 break; 543 case 2: 544 val = virtio_config_readw(vdev, addr); 545 if (virtio_is_big_endian(vdev)) { 546 val = bswap16(val); 547 } 548 break; 549 case 4: 550 val = virtio_config_readl(vdev, addr); 551 if (virtio_is_big_endian(vdev)) { 552 val = bswap32(val); 553 } 554 break; 555 } 556 return val; 557 } 558 559 static void virtio_pci_config_write(void *opaque, hwaddr addr, 560 uint64_t val, unsigned size) 561 { 562 VirtIOPCIProxy *proxy = opaque; 563 uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev); 564 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 565 566 if (vdev == NULL) { 567 return; 568 } 569 570 if (addr < config) { 571 virtio_ioport_write(proxy, addr, val); 572 return; 573 } 574 addr -= config; 575 /* 576 * Virtio-PCI is odd. Ioports are LE but config space is target native 577 * endian. 578 */ 579 switch (size) { 580 case 1: 581 virtio_config_writeb(vdev, addr, val); 582 break; 583 case 2: 584 if (virtio_is_big_endian(vdev)) { 585 val = bswap16(val); 586 } 587 virtio_config_writew(vdev, addr, val); 588 break; 589 case 4: 590 if (virtio_is_big_endian(vdev)) { 591 val = bswap32(val); 592 } 593 virtio_config_writel(vdev, addr, val); 594 break; 595 } 596 } 597 598 static const MemoryRegionOps virtio_pci_config_ops = { 599 .read = virtio_pci_config_read, 600 .write = virtio_pci_config_write, 601 .impl = { 602 .min_access_size = 1, 603 .max_access_size = 4, 604 }, 605 .endianness = DEVICE_LITTLE_ENDIAN, 606 }; 607 608 static MemoryRegion *virtio_address_space_lookup(VirtIOPCIProxy *proxy, 609 hwaddr *off, int len) 610 { 611 int i; 612 VirtIOPCIRegion *reg; 613 614 for (i = 0; i < ARRAY_SIZE(proxy->regs); ++i) { 615 reg = &proxy->regs[i]; 616 if (*off >= reg->offset && 617 *off + len <= reg->offset + reg->size) { 618 MemoryRegionSection mrs = memory_region_find(®->mr, 619 *off - reg->offset, len); 620 assert(mrs.mr); 621 *off = mrs.offset_within_region; 622 memory_region_unref(mrs.mr); 623 return mrs.mr; 624 } 625 } 626 627 return NULL; 628 } 629 630 /* Below are generic functions to do memcpy from/to an address space, 631 * without byteswaps, with input validation. 632 * 633 * As regular address_space_* APIs all do some kind of byteswap at least for 634 * some host/target combinations, we are forced to explicitly convert to a 635 * known-endianness integer value. 636 * It doesn't really matter which endian format to go through, so the code 637 * below selects the endian that causes the least amount of work on the given 638 * host. 639 * 640 * Note: host pointer must be aligned. 641 */ 642 static 643 void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr, 644 const uint8_t *buf, int len) 645 { 646 uint64_t val; 647 MemoryRegion *mr; 648 649 /* address_space_* APIs assume an aligned address. 650 * As address is under guest control, handle illegal values. 651 */ 652 addr &= ~(len - 1); 653 654 mr = virtio_address_space_lookup(proxy, &addr, len); 655 if (!mr) { 656 return; 657 } 658 659 /* Make sure caller aligned buf properly */ 660 assert(!(((uintptr_t)buf) & (len - 1))); 661 662 switch (len) { 663 case 1: 664 val = pci_get_byte(buf); 665 break; 666 case 2: 667 val = pci_get_word(buf); 668 break; 669 case 4: 670 val = pci_get_long(buf); 671 break; 672 default: 673 /* As length is under guest control, handle illegal values. */ 674 return; 675 } 676 memory_region_dispatch_write(mr, addr, val, size_memop(len) | MO_LE, 677 MEMTXATTRS_UNSPECIFIED); 678 } 679 680 static void 681 virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr, 682 uint8_t *buf, int len) 683 { 684 uint64_t val; 685 MemoryRegion *mr; 686 687 /* address_space_* APIs assume an aligned address. 688 * As address is under guest control, handle illegal values. 689 */ 690 addr &= ~(len - 1); 691 692 mr = virtio_address_space_lookup(proxy, &addr, len); 693 if (!mr) { 694 return; 695 } 696 697 /* Make sure caller aligned buf properly */ 698 assert(!(((uintptr_t)buf) & (len - 1))); 699 700 memory_region_dispatch_read(mr, addr, &val, size_memop(len) | MO_LE, 701 MEMTXATTRS_UNSPECIFIED); 702 switch (len) { 703 case 1: 704 pci_set_byte(buf, val); 705 break; 706 case 2: 707 pci_set_word(buf, val); 708 break; 709 case 4: 710 pci_set_long(buf, val); 711 break; 712 default: 713 /* As length is under guest control, handle illegal values. */ 714 break; 715 } 716 } 717 718 static void virtio_pci_ats_ctrl_trigger(PCIDevice *pci_dev, bool enable) 719 { 720 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 721 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 722 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 723 724 vdev->device_iotlb_enabled = enable; 725 726 if (k->toggle_device_iotlb) { 727 k->toggle_device_iotlb(vdev); 728 } 729 } 730 731 static void pcie_ats_config_write(PCIDevice *dev, uint32_t address, 732 uint32_t val, int len) 733 { 734 uint32_t off; 735 uint16_t ats_cap = dev->exp.ats_cap; 736 737 if (!ats_cap || address < ats_cap) { 738 return; 739 } 740 off = address - ats_cap; 741 if (off >= PCI_EXT_CAP_ATS_SIZEOF) { 742 return; 743 } 744 745 if (range_covers_byte(off, len, PCI_ATS_CTRL + 1)) { 746 virtio_pci_ats_ctrl_trigger(dev, !!(val & PCI_ATS_CTRL_ENABLE)); 747 } 748 } 749 750 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address, 751 uint32_t val, int len) 752 { 753 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 754 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 755 struct virtio_pci_cfg_cap *cfg; 756 757 pci_default_write_config(pci_dev, address, val, len); 758 759 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) { 760 pcie_cap_flr_write_config(pci_dev, address, val, len); 761 } 762 763 if (proxy->flags & VIRTIO_PCI_FLAG_ATS) { 764 pcie_ats_config_write(pci_dev, address, val, len); 765 } 766 767 if (range_covers_byte(address, len, PCI_COMMAND)) { 768 if (!(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { 769 virtio_set_disabled(vdev, true); 770 virtio_pci_stop_ioeventfd(proxy); 771 virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK); 772 } else { 773 virtio_set_disabled(vdev, false); 774 } 775 } 776 777 if (proxy->config_cap && 778 ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap, 779 pci_cfg_data), 780 sizeof cfg->pci_cfg_data)) { 781 uint32_t off; 782 uint32_t caplen; 783 784 cfg = (void *)(proxy->pci_dev.config + proxy->config_cap); 785 off = le32_to_cpu(cfg->cap.offset); 786 caplen = le32_to_cpu(cfg->cap.length); 787 788 if (caplen == 1 || caplen == 2 || caplen == 4) { 789 assert(caplen <= sizeof cfg->pci_cfg_data); 790 virtio_address_space_write(proxy, off, cfg->pci_cfg_data, caplen); 791 } 792 } 793 } 794 795 static uint32_t virtio_read_config(PCIDevice *pci_dev, 796 uint32_t address, int len) 797 { 798 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 799 struct virtio_pci_cfg_cap *cfg; 800 801 if (proxy->config_cap && 802 ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap, 803 pci_cfg_data), 804 sizeof cfg->pci_cfg_data)) { 805 uint32_t off; 806 uint32_t caplen; 807 808 cfg = (void *)(proxy->pci_dev.config + proxy->config_cap); 809 off = le32_to_cpu(cfg->cap.offset); 810 caplen = le32_to_cpu(cfg->cap.length); 811 812 if (caplen == 1 || caplen == 2 || caplen == 4) { 813 assert(caplen <= sizeof cfg->pci_cfg_data); 814 virtio_address_space_read(proxy, off, cfg->pci_cfg_data, caplen); 815 } 816 } 817 818 return pci_default_read_config(pci_dev, address, len); 819 } 820 821 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy, 822 unsigned int vector) 823 { 824 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 825 int ret; 826 827 if (irqfd->users == 0) { 828 KVMRouteChange c = kvm_irqchip_begin_route_changes(kvm_state); 829 ret = kvm_irqchip_add_msi_route(&c, vector, &proxy->pci_dev); 830 if (ret < 0) { 831 return ret; 832 } 833 kvm_irqchip_commit_route_changes(&c); 834 irqfd->virq = ret; 835 } 836 irqfd->users++; 837 return 0; 838 } 839 840 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy, 841 unsigned int vector) 842 { 843 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 844 if (--irqfd->users == 0) { 845 kvm_irqchip_release_virq(kvm_state, irqfd->virq); 846 } 847 } 848 849 static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy, 850 EventNotifier *n, 851 unsigned int vector) 852 { 853 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 854 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq); 855 } 856 857 static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy, 858 EventNotifier *n , 859 unsigned int vector) 860 { 861 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; 862 int ret; 863 864 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq); 865 assert(ret == 0); 866 } 867 static int virtio_pci_get_notifier(VirtIOPCIProxy *proxy, int queue_no, 868 EventNotifier **n, unsigned int *vector) 869 { 870 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 871 VirtQueue *vq; 872 873 if (!proxy->vector_irqfd && vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) 874 return -1; 875 876 if (queue_no == VIRTIO_CONFIG_IRQ_IDX) { 877 *n = virtio_config_get_guest_notifier(vdev); 878 *vector = vdev->config_vector; 879 } else { 880 if (!virtio_queue_get_num(vdev, queue_no)) { 881 return -1; 882 } 883 *vector = virtio_queue_vector(vdev, queue_no); 884 vq = virtio_get_queue(vdev, queue_no); 885 *n = virtio_queue_get_guest_notifier(vq); 886 } 887 return 0; 888 } 889 890 static int kvm_virtio_pci_vector_use_one(VirtIOPCIProxy *proxy, int queue_no) 891 { 892 unsigned int vector; 893 int ret; 894 EventNotifier *n; 895 PCIDevice *dev = &proxy->pci_dev; 896 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 897 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 898 899 ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector); 900 if (ret < 0) { 901 return ret; 902 } 903 if (vector >= msix_nr_vectors_allocated(dev)) { 904 return 0; 905 } 906 ret = kvm_virtio_pci_vq_vector_use(proxy, vector); 907 if (ret < 0) { 908 return ret; 909 } 910 /* 911 * If guest supports masking, set up irqfd now. 912 * Otherwise, delay until unmasked in the frontend. 913 */ 914 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 915 ret = kvm_virtio_pci_irqfd_use(proxy, n, vector); 916 if (ret < 0) { 917 kvm_virtio_pci_vq_vector_release(proxy, vector); 918 return ret; 919 } 920 } 921 922 return 0; 923 } 924 static int kvm_virtio_pci_vector_vq_use(VirtIOPCIProxy *proxy, int nvqs) 925 { 926 int queue_no; 927 int ret = 0; 928 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 929 930 for (queue_no = 0; queue_no < nvqs; queue_no++) { 931 if (!virtio_queue_get_num(vdev, queue_no)) { 932 return -1; 933 } 934 ret = kvm_virtio_pci_vector_use_one(proxy, queue_no); 935 } 936 return ret; 937 } 938 939 static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy) 940 { 941 return kvm_virtio_pci_vector_use_one(proxy, VIRTIO_CONFIG_IRQ_IDX); 942 } 943 944 static void kvm_virtio_pci_vector_release_one(VirtIOPCIProxy *proxy, 945 int queue_no) 946 { 947 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 948 unsigned int vector; 949 EventNotifier *n; 950 int ret; 951 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 952 PCIDevice *dev = &proxy->pci_dev; 953 954 ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector); 955 if (ret < 0) { 956 return; 957 } 958 if (vector >= msix_nr_vectors_allocated(dev)) { 959 return; 960 } 961 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 962 kvm_virtio_pci_irqfd_release(proxy, n, vector); 963 } 964 kvm_virtio_pci_vq_vector_release(proxy, vector); 965 } 966 967 static void kvm_virtio_pci_vector_vq_release(VirtIOPCIProxy *proxy, int nvqs) 968 { 969 int queue_no; 970 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 971 972 for (queue_no = 0; queue_no < nvqs; queue_no++) { 973 if (!virtio_queue_get_num(vdev, queue_no)) { 974 break; 975 } 976 kvm_virtio_pci_vector_release_one(proxy, queue_no); 977 } 978 } 979 980 static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy) 981 { 982 kvm_virtio_pci_vector_release_one(proxy, VIRTIO_CONFIG_IRQ_IDX); 983 } 984 985 static int virtio_pci_one_vector_unmask(VirtIOPCIProxy *proxy, 986 unsigned int queue_no, 987 unsigned int vector, 988 MSIMessage msg, 989 EventNotifier *n) 990 { 991 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 992 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 993 VirtIOIRQFD *irqfd; 994 int ret = 0; 995 996 if (proxy->vector_irqfd) { 997 irqfd = &proxy->vector_irqfd[vector]; 998 if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) { 999 ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg, 1000 &proxy->pci_dev); 1001 if (ret < 0) { 1002 return ret; 1003 } 1004 kvm_irqchip_commit_routes(kvm_state); 1005 } 1006 } 1007 1008 /* If guest supports masking, irqfd is already setup, unmask it. 1009 * Otherwise, set it up now. 1010 */ 1011 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 1012 k->guest_notifier_mask(vdev, queue_no, false); 1013 /* Test after unmasking to avoid losing events. */ 1014 if (k->guest_notifier_pending && 1015 k->guest_notifier_pending(vdev, queue_no)) { 1016 event_notifier_set(n); 1017 } 1018 } else { 1019 ret = kvm_virtio_pci_irqfd_use(proxy, n, vector); 1020 } 1021 return ret; 1022 } 1023 1024 static void virtio_pci_one_vector_mask(VirtIOPCIProxy *proxy, 1025 unsigned int queue_no, 1026 unsigned int vector, 1027 EventNotifier *n) 1028 { 1029 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1030 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1031 1032 /* If guest supports masking, keep irqfd but mask it. 1033 * Otherwise, clean it up now. 1034 */ 1035 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) { 1036 k->guest_notifier_mask(vdev, queue_no, true); 1037 } else { 1038 kvm_virtio_pci_irqfd_release(proxy, n, vector); 1039 } 1040 } 1041 1042 static int virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector, 1043 MSIMessage msg) 1044 { 1045 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); 1046 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1047 VirtQueue *vq = virtio_vector_first_queue(vdev, vector); 1048 EventNotifier *n; 1049 int ret, index, unmasked = 0; 1050 1051 while (vq) { 1052 index = virtio_get_queue_index(vq); 1053 if (!virtio_queue_get_num(vdev, index)) { 1054 break; 1055 } 1056 if (index < proxy->nvqs_with_notifiers) { 1057 n = virtio_queue_get_guest_notifier(vq); 1058 ret = virtio_pci_one_vector_unmask(proxy, index, vector, msg, n); 1059 if (ret < 0) { 1060 goto undo; 1061 } 1062 ++unmasked; 1063 } 1064 vq = virtio_vector_next_queue(vq); 1065 } 1066 /* unmask config intr */ 1067 if (vector == vdev->config_vector) { 1068 n = virtio_config_get_guest_notifier(vdev); 1069 ret = virtio_pci_one_vector_unmask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, 1070 msg, n); 1071 if (ret < 0) { 1072 goto undo_config; 1073 } 1074 } 1075 return 0; 1076 undo_config: 1077 n = virtio_config_get_guest_notifier(vdev); 1078 virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n); 1079 undo: 1080 vq = virtio_vector_first_queue(vdev, vector); 1081 while (vq && unmasked >= 0) { 1082 index = virtio_get_queue_index(vq); 1083 if (index < proxy->nvqs_with_notifiers) { 1084 n = virtio_queue_get_guest_notifier(vq); 1085 virtio_pci_one_vector_mask(proxy, index, vector, n); 1086 --unmasked; 1087 } 1088 vq = virtio_vector_next_queue(vq); 1089 } 1090 return ret; 1091 } 1092 1093 static void virtio_pci_vector_mask(PCIDevice *dev, unsigned vector) 1094 { 1095 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); 1096 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1097 VirtQueue *vq = virtio_vector_first_queue(vdev, vector); 1098 EventNotifier *n; 1099 int index; 1100 1101 while (vq) { 1102 index = virtio_get_queue_index(vq); 1103 n = virtio_queue_get_guest_notifier(vq); 1104 if (!virtio_queue_get_num(vdev, index)) { 1105 break; 1106 } 1107 if (index < proxy->nvqs_with_notifiers) { 1108 virtio_pci_one_vector_mask(proxy, index, vector, n); 1109 } 1110 vq = virtio_vector_next_queue(vq); 1111 } 1112 1113 if (vector == vdev->config_vector) { 1114 n = virtio_config_get_guest_notifier(vdev); 1115 virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n); 1116 } 1117 } 1118 1119 static void virtio_pci_vector_poll(PCIDevice *dev, 1120 unsigned int vector_start, 1121 unsigned int vector_end) 1122 { 1123 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); 1124 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1125 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1126 int queue_no; 1127 unsigned int vector; 1128 EventNotifier *notifier; 1129 int ret; 1130 1131 for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) { 1132 ret = virtio_pci_get_notifier(proxy, queue_no, ¬ifier, &vector); 1133 if (ret < 0) { 1134 break; 1135 } 1136 if (vector < vector_start || vector >= vector_end || 1137 !msix_is_masked(dev, vector)) { 1138 continue; 1139 } 1140 if (k->guest_notifier_pending) { 1141 if (k->guest_notifier_pending(vdev, queue_no)) { 1142 msix_set_pending(dev, vector); 1143 } 1144 } else if (event_notifier_test_and_clear(notifier)) { 1145 msix_set_pending(dev, vector); 1146 } 1147 } 1148 /* poll the config intr */ 1149 ret = virtio_pci_get_notifier(proxy, VIRTIO_CONFIG_IRQ_IDX, ¬ifier, 1150 &vector); 1151 if (ret < 0) { 1152 return; 1153 } 1154 if (vector < vector_start || vector >= vector_end || 1155 !msix_is_masked(dev, vector)) { 1156 return; 1157 } 1158 if (k->guest_notifier_pending) { 1159 if (k->guest_notifier_pending(vdev, VIRTIO_CONFIG_IRQ_IDX)) { 1160 msix_set_pending(dev, vector); 1161 } 1162 } else if (event_notifier_test_and_clear(notifier)) { 1163 msix_set_pending(dev, vector); 1164 } 1165 } 1166 1167 void virtio_pci_set_guest_notifier_fd_handler(VirtIODevice *vdev, VirtQueue *vq, 1168 int n, bool assign, 1169 bool with_irqfd) 1170 { 1171 if (n == VIRTIO_CONFIG_IRQ_IDX) { 1172 virtio_config_set_guest_notifier_fd_handler(vdev, assign, with_irqfd); 1173 } else { 1174 virtio_queue_set_guest_notifier_fd_handler(vq, assign, with_irqfd); 1175 } 1176 } 1177 1178 static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign, 1179 bool with_irqfd) 1180 { 1181 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1182 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1183 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1184 VirtQueue *vq = NULL; 1185 EventNotifier *notifier = NULL; 1186 1187 if (n == VIRTIO_CONFIG_IRQ_IDX) { 1188 notifier = virtio_config_get_guest_notifier(vdev); 1189 } else { 1190 vq = virtio_get_queue(vdev, n); 1191 notifier = virtio_queue_get_guest_notifier(vq); 1192 } 1193 1194 if (assign) { 1195 int r = event_notifier_init(notifier, 0); 1196 if (r < 0) { 1197 return r; 1198 } 1199 virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, true, with_irqfd); 1200 } else { 1201 virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, false, 1202 with_irqfd); 1203 event_notifier_cleanup(notifier); 1204 } 1205 1206 if (!msix_enabled(&proxy->pci_dev) && 1207 vdev->use_guest_notifier_mask && 1208 vdc->guest_notifier_mask) { 1209 vdc->guest_notifier_mask(vdev, n, !assign); 1210 } 1211 1212 return 0; 1213 } 1214 1215 static bool virtio_pci_query_guest_notifiers(DeviceState *d) 1216 { 1217 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1218 return msix_enabled(&proxy->pci_dev); 1219 } 1220 1221 static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign) 1222 { 1223 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1224 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1225 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1226 int r, n; 1227 bool with_irqfd = msix_enabled(&proxy->pci_dev) && 1228 kvm_msi_via_irqfd_enabled(); 1229 1230 nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX); 1231 1232 /* 1233 * When deassigning, pass a consistent nvqs value to avoid leaking 1234 * notifiers. But first check we've actually been configured, exit 1235 * early if we haven't. 1236 */ 1237 if (!assign && !proxy->nvqs_with_notifiers) { 1238 return 0; 1239 } 1240 assert(assign || nvqs == proxy->nvqs_with_notifiers); 1241 1242 proxy->nvqs_with_notifiers = nvqs; 1243 1244 /* Must unset vector notifier while guest notifier is still assigned */ 1245 if ((proxy->vector_irqfd || 1246 (vdev->use_guest_notifier_mask && k->guest_notifier_mask)) && 1247 !assign) { 1248 msix_unset_vector_notifiers(&proxy->pci_dev); 1249 if (proxy->vector_irqfd) { 1250 kvm_virtio_pci_vector_vq_release(proxy, nvqs); 1251 kvm_virtio_pci_vector_config_release(proxy); 1252 g_free(proxy->vector_irqfd); 1253 proxy->vector_irqfd = NULL; 1254 } 1255 } 1256 1257 for (n = 0; n < nvqs; n++) { 1258 if (!virtio_queue_get_num(vdev, n)) { 1259 break; 1260 } 1261 1262 r = virtio_pci_set_guest_notifier(d, n, assign, with_irqfd); 1263 if (r < 0) { 1264 goto assign_error; 1265 } 1266 } 1267 r = virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, assign, 1268 with_irqfd); 1269 if (r < 0) { 1270 goto config_assign_error; 1271 } 1272 /* Must set vector notifier after guest notifier has been assigned */ 1273 if ((with_irqfd || 1274 (vdev->use_guest_notifier_mask && k->guest_notifier_mask)) && 1275 assign) { 1276 if (with_irqfd) { 1277 proxy->vector_irqfd = 1278 g_malloc0(sizeof(*proxy->vector_irqfd) * 1279 msix_nr_vectors_allocated(&proxy->pci_dev)); 1280 r = kvm_virtio_pci_vector_vq_use(proxy, nvqs); 1281 if (r < 0) { 1282 goto config_assign_error; 1283 } 1284 r = kvm_virtio_pci_vector_config_use(proxy); 1285 if (r < 0) { 1286 goto config_error; 1287 } 1288 } 1289 1290 r = msix_set_vector_notifiers(&proxy->pci_dev, virtio_pci_vector_unmask, 1291 virtio_pci_vector_mask, 1292 virtio_pci_vector_poll); 1293 if (r < 0) { 1294 goto notifiers_error; 1295 } 1296 } 1297 1298 return 0; 1299 1300 notifiers_error: 1301 if (with_irqfd) { 1302 assert(assign); 1303 kvm_virtio_pci_vector_vq_release(proxy, nvqs); 1304 } 1305 config_error: 1306 if (with_irqfd) { 1307 kvm_virtio_pci_vector_config_release(proxy); 1308 } 1309 config_assign_error: 1310 virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, !assign, 1311 with_irqfd); 1312 assign_error: 1313 /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */ 1314 assert(assign); 1315 while (--n >= 0) { 1316 virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd); 1317 } 1318 g_free(proxy->vector_irqfd); 1319 proxy->vector_irqfd = NULL; 1320 return r; 1321 } 1322 1323 static int virtio_pci_set_host_notifier_mr(DeviceState *d, int n, 1324 MemoryRegion *mr, bool assign) 1325 { 1326 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1327 int offset; 1328 1329 if (n >= VIRTIO_QUEUE_MAX || !virtio_pci_modern(proxy) || 1330 virtio_pci_queue_mem_mult(proxy) != memory_region_size(mr)) { 1331 return -1; 1332 } 1333 1334 if (assign) { 1335 offset = virtio_pci_queue_mem_mult(proxy) * n; 1336 memory_region_add_subregion_overlap(&proxy->notify.mr, offset, mr, 1); 1337 } else { 1338 memory_region_del_subregion(&proxy->notify.mr, mr); 1339 } 1340 1341 return 0; 1342 } 1343 1344 static void virtio_pci_vmstate_change(DeviceState *d, bool running) 1345 { 1346 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); 1347 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1348 1349 if (running) { 1350 /* Old QEMU versions did not set bus master enable on status write. 1351 * Detect DRIVER set and enable it. 1352 */ 1353 if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) && 1354 (vdev->status & VIRTIO_CONFIG_S_DRIVER) && 1355 !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { 1356 pci_default_write_config(&proxy->pci_dev, PCI_COMMAND, 1357 proxy->pci_dev.config[PCI_COMMAND] | 1358 PCI_COMMAND_MASTER, 1); 1359 } 1360 virtio_pci_start_ioeventfd(proxy); 1361 } else { 1362 virtio_pci_stop_ioeventfd(proxy); 1363 } 1364 } 1365 1366 /* 1367 * virtio-pci: This is the PCIDevice which has a virtio-pci-bus. 1368 */ 1369 1370 static int virtio_pci_query_nvectors(DeviceState *d) 1371 { 1372 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1373 1374 return proxy->nvectors; 1375 } 1376 1377 static AddressSpace *virtio_pci_get_dma_as(DeviceState *d) 1378 { 1379 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1380 PCIDevice *dev = &proxy->pci_dev; 1381 1382 return pci_get_address_space(dev); 1383 } 1384 1385 static bool virtio_pci_iommu_enabled(DeviceState *d) 1386 { 1387 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1388 PCIDevice *dev = &proxy->pci_dev; 1389 AddressSpace *dma_as = pci_device_iommu_address_space(dev); 1390 1391 if (dma_as == &address_space_memory) { 1392 return false; 1393 } 1394 1395 return true; 1396 } 1397 1398 static bool virtio_pci_queue_enabled(DeviceState *d, int n) 1399 { 1400 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1401 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1402 1403 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1404 return proxy->vqs[n].enabled; 1405 } 1406 1407 return virtio_queue_enabled_legacy(vdev, n); 1408 } 1409 1410 static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy, 1411 struct virtio_pci_cap *cap) 1412 { 1413 PCIDevice *dev = &proxy->pci_dev; 1414 int offset; 1415 1416 offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0, 1417 cap->cap_len, &error_abort); 1418 1419 assert(cap->cap_len >= sizeof *cap); 1420 memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len, 1421 cap->cap_len - PCI_CAP_FLAGS); 1422 1423 return offset; 1424 } 1425 1426 static void virtio_pci_set_vector(VirtIODevice *vdev, 1427 VirtIOPCIProxy *proxy, 1428 int queue_no, uint16_t old_vector, 1429 uint16_t new_vector) 1430 { 1431 bool kvm_irqfd = (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) && 1432 msix_enabled(&proxy->pci_dev) && kvm_msi_via_irqfd_enabled(); 1433 1434 if (new_vector == old_vector) { 1435 return; 1436 } 1437 1438 /* 1439 * If the device uses irqfd and the vector changes after DRIVER_OK is 1440 * set, we need to release the old vector and set up the new one. 1441 * Otherwise just need to set the new vector on the device. 1442 */ 1443 if (kvm_irqfd && old_vector != VIRTIO_NO_VECTOR) { 1444 kvm_virtio_pci_vector_release_one(proxy, queue_no); 1445 } 1446 /* Set the new vector on the device. */ 1447 if (queue_no == VIRTIO_CONFIG_IRQ_IDX) { 1448 vdev->config_vector = new_vector; 1449 } else { 1450 virtio_queue_set_vector(vdev, queue_no, new_vector); 1451 } 1452 /* If the new vector changed need to set it up. */ 1453 if (kvm_irqfd && new_vector != VIRTIO_NO_VECTOR) { 1454 kvm_virtio_pci_vector_use_one(proxy, queue_no); 1455 } 1456 } 1457 1458 int virtio_pci_add_shm_cap(VirtIOPCIProxy *proxy, 1459 uint8_t bar, uint64_t offset, uint64_t length, 1460 uint8_t id) 1461 { 1462 struct virtio_pci_cap64 cap = { 1463 .cap.cap_len = sizeof cap, 1464 .cap.cfg_type = VIRTIO_PCI_CAP_SHARED_MEMORY_CFG, 1465 }; 1466 1467 cap.cap.bar = bar; 1468 cap.cap.length = cpu_to_le32(length); 1469 cap.length_hi = cpu_to_le32(length >> 32); 1470 cap.cap.offset = cpu_to_le32(offset); 1471 cap.offset_hi = cpu_to_le32(offset >> 32); 1472 cap.cap.id = id; 1473 return virtio_pci_add_mem_cap(proxy, &cap.cap); 1474 } 1475 1476 static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr, 1477 unsigned size) 1478 { 1479 VirtIOPCIProxy *proxy = opaque; 1480 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1481 uint32_t val = 0; 1482 int i; 1483 1484 if (vdev == NULL) { 1485 return UINT64_MAX; 1486 } 1487 1488 switch (addr) { 1489 case VIRTIO_PCI_COMMON_DFSELECT: 1490 val = proxy->dfselect; 1491 break; 1492 case VIRTIO_PCI_COMMON_DF: 1493 if (proxy->dfselect <= 1) { 1494 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1495 1496 val = (vdev->host_features & ~vdc->legacy_features) >> 1497 (32 * proxy->dfselect); 1498 } 1499 break; 1500 case VIRTIO_PCI_COMMON_GFSELECT: 1501 val = proxy->gfselect; 1502 break; 1503 case VIRTIO_PCI_COMMON_GF: 1504 if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) { 1505 val = proxy->guest_features[proxy->gfselect]; 1506 } 1507 break; 1508 case VIRTIO_PCI_COMMON_MSIX: 1509 val = vdev->config_vector; 1510 break; 1511 case VIRTIO_PCI_COMMON_NUMQ: 1512 for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) { 1513 if (virtio_queue_get_num(vdev, i)) { 1514 val = i + 1; 1515 } 1516 } 1517 break; 1518 case VIRTIO_PCI_COMMON_STATUS: 1519 val = vdev->status; 1520 break; 1521 case VIRTIO_PCI_COMMON_CFGGENERATION: 1522 val = vdev->generation; 1523 break; 1524 case VIRTIO_PCI_COMMON_Q_SELECT: 1525 val = vdev->queue_sel; 1526 break; 1527 case VIRTIO_PCI_COMMON_Q_SIZE: 1528 val = virtio_queue_get_num(vdev, vdev->queue_sel); 1529 break; 1530 case VIRTIO_PCI_COMMON_Q_MSIX: 1531 val = virtio_queue_vector(vdev, vdev->queue_sel); 1532 break; 1533 case VIRTIO_PCI_COMMON_Q_ENABLE: 1534 val = proxy->vqs[vdev->queue_sel].enabled; 1535 break; 1536 case VIRTIO_PCI_COMMON_Q_NOFF: 1537 /* Simply map queues in order */ 1538 val = vdev->queue_sel; 1539 break; 1540 case VIRTIO_PCI_COMMON_Q_DESCLO: 1541 val = proxy->vqs[vdev->queue_sel].desc[0]; 1542 break; 1543 case VIRTIO_PCI_COMMON_Q_DESCHI: 1544 val = proxy->vqs[vdev->queue_sel].desc[1]; 1545 break; 1546 case VIRTIO_PCI_COMMON_Q_AVAILLO: 1547 val = proxy->vqs[vdev->queue_sel].avail[0]; 1548 break; 1549 case VIRTIO_PCI_COMMON_Q_AVAILHI: 1550 val = proxy->vqs[vdev->queue_sel].avail[1]; 1551 break; 1552 case VIRTIO_PCI_COMMON_Q_USEDLO: 1553 val = proxy->vqs[vdev->queue_sel].used[0]; 1554 break; 1555 case VIRTIO_PCI_COMMON_Q_USEDHI: 1556 val = proxy->vqs[vdev->queue_sel].used[1]; 1557 break; 1558 case VIRTIO_PCI_COMMON_Q_RESET: 1559 val = proxy->vqs[vdev->queue_sel].reset; 1560 break; 1561 default: 1562 val = 0; 1563 } 1564 1565 return val; 1566 } 1567 1568 static void virtio_pci_common_write(void *opaque, hwaddr addr, 1569 uint64_t val, unsigned size) 1570 { 1571 VirtIOPCIProxy *proxy = opaque; 1572 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1573 uint16_t vector; 1574 1575 if (vdev == NULL) { 1576 return; 1577 } 1578 1579 switch (addr) { 1580 case VIRTIO_PCI_COMMON_DFSELECT: 1581 proxy->dfselect = val; 1582 break; 1583 case VIRTIO_PCI_COMMON_GFSELECT: 1584 proxy->gfselect = val; 1585 break; 1586 case VIRTIO_PCI_COMMON_GF: 1587 if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) { 1588 proxy->guest_features[proxy->gfselect] = val; 1589 virtio_set_features(vdev, 1590 (((uint64_t)proxy->guest_features[1]) << 32) | 1591 proxy->guest_features[0]); 1592 } 1593 break; 1594 case VIRTIO_PCI_COMMON_MSIX: 1595 if (vdev->config_vector != VIRTIO_NO_VECTOR) { 1596 msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); 1597 } 1598 /* Make it possible for guest to discover an error took place. */ 1599 if (val < proxy->nvectors) { 1600 msix_vector_use(&proxy->pci_dev, val); 1601 } else { 1602 val = VIRTIO_NO_VECTOR; 1603 } 1604 virtio_pci_set_vector(vdev, proxy, VIRTIO_CONFIG_IRQ_IDX, 1605 vdev->config_vector, val); 1606 break; 1607 case VIRTIO_PCI_COMMON_STATUS: 1608 if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) { 1609 virtio_pci_stop_ioeventfd(proxy); 1610 } 1611 1612 virtio_set_status(vdev, val & 0xFF); 1613 1614 if (val & VIRTIO_CONFIG_S_DRIVER_OK) { 1615 virtio_pci_start_ioeventfd(proxy); 1616 } 1617 1618 if (vdev->status == 0) { 1619 virtio_pci_reset(DEVICE(proxy)); 1620 } 1621 1622 break; 1623 case VIRTIO_PCI_COMMON_Q_SELECT: 1624 if (val < VIRTIO_QUEUE_MAX) { 1625 vdev->queue_sel = val; 1626 } 1627 break; 1628 case VIRTIO_PCI_COMMON_Q_SIZE: 1629 proxy->vqs[vdev->queue_sel].num = val; 1630 virtio_queue_set_num(vdev, vdev->queue_sel, 1631 proxy->vqs[vdev->queue_sel].num); 1632 virtio_init_region_cache(vdev, vdev->queue_sel); 1633 break; 1634 case VIRTIO_PCI_COMMON_Q_MSIX: 1635 vector = virtio_queue_vector(vdev, vdev->queue_sel); 1636 if (vector != VIRTIO_NO_VECTOR) { 1637 msix_vector_unuse(&proxy->pci_dev, vector); 1638 } 1639 /* Make it possible for guest to discover an error took place. */ 1640 if (val < proxy->nvectors) { 1641 msix_vector_use(&proxy->pci_dev, val); 1642 } else { 1643 val = VIRTIO_NO_VECTOR; 1644 } 1645 virtio_pci_set_vector(vdev, proxy, vdev->queue_sel, vector, val); 1646 break; 1647 case VIRTIO_PCI_COMMON_Q_ENABLE: 1648 if (val == 1) { 1649 virtio_queue_set_num(vdev, vdev->queue_sel, 1650 proxy->vqs[vdev->queue_sel].num); 1651 virtio_queue_set_rings(vdev, vdev->queue_sel, 1652 ((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 | 1653 proxy->vqs[vdev->queue_sel].desc[0], 1654 ((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 | 1655 proxy->vqs[vdev->queue_sel].avail[0], 1656 ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 | 1657 proxy->vqs[vdev->queue_sel].used[0]); 1658 proxy->vqs[vdev->queue_sel].enabled = 1; 1659 proxy->vqs[vdev->queue_sel].reset = 0; 1660 virtio_queue_enable(vdev, vdev->queue_sel); 1661 } else { 1662 virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val); 1663 } 1664 break; 1665 case VIRTIO_PCI_COMMON_Q_DESCLO: 1666 proxy->vqs[vdev->queue_sel].desc[0] = val; 1667 break; 1668 case VIRTIO_PCI_COMMON_Q_DESCHI: 1669 proxy->vqs[vdev->queue_sel].desc[1] = val; 1670 break; 1671 case VIRTIO_PCI_COMMON_Q_AVAILLO: 1672 proxy->vqs[vdev->queue_sel].avail[0] = val; 1673 break; 1674 case VIRTIO_PCI_COMMON_Q_AVAILHI: 1675 proxy->vqs[vdev->queue_sel].avail[1] = val; 1676 break; 1677 case VIRTIO_PCI_COMMON_Q_USEDLO: 1678 proxy->vqs[vdev->queue_sel].used[0] = val; 1679 break; 1680 case VIRTIO_PCI_COMMON_Q_USEDHI: 1681 proxy->vqs[vdev->queue_sel].used[1] = val; 1682 break; 1683 case VIRTIO_PCI_COMMON_Q_RESET: 1684 if (val == 1) { 1685 proxy->vqs[vdev->queue_sel].reset = 1; 1686 1687 virtio_queue_reset(vdev, vdev->queue_sel); 1688 1689 proxy->vqs[vdev->queue_sel].reset = 0; 1690 proxy->vqs[vdev->queue_sel].enabled = 0; 1691 } 1692 break; 1693 default: 1694 break; 1695 } 1696 } 1697 1698 1699 static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr, 1700 unsigned size) 1701 { 1702 VirtIOPCIProxy *proxy = opaque; 1703 if (virtio_bus_get_device(&proxy->bus) == NULL) { 1704 return UINT64_MAX; 1705 } 1706 1707 return 0; 1708 } 1709 1710 static void virtio_pci_notify_write(void *opaque, hwaddr addr, 1711 uint64_t val, unsigned size) 1712 { 1713 VirtIOPCIProxy *proxy = opaque; 1714 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1715 1716 unsigned queue = addr / virtio_pci_queue_mem_mult(proxy); 1717 1718 if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) { 1719 trace_virtio_pci_notify_write(addr, val, size); 1720 virtio_queue_notify(vdev, queue); 1721 } 1722 } 1723 1724 static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr, 1725 uint64_t val, unsigned size) 1726 { 1727 VirtIOPCIProxy *proxy = opaque; 1728 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1729 1730 unsigned queue = val; 1731 1732 if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) { 1733 trace_virtio_pci_notify_write_pio(addr, val, size); 1734 virtio_queue_notify(vdev, queue); 1735 } 1736 } 1737 1738 static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr, 1739 unsigned size) 1740 { 1741 VirtIOPCIProxy *proxy = opaque; 1742 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1743 uint64_t val; 1744 1745 if (vdev == NULL) { 1746 return UINT64_MAX; 1747 } 1748 1749 val = qatomic_xchg(&vdev->isr, 0); 1750 pci_irq_deassert(&proxy->pci_dev); 1751 return val; 1752 } 1753 1754 static void virtio_pci_isr_write(void *opaque, hwaddr addr, 1755 uint64_t val, unsigned size) 1756 { 1757 } 1758 1759 static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr, 1760 unsigned size) 1761 { 1762 VirtIOPCIProxy *proxy = opaque; 1763 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1764 uint64_t val; 1765 1766 if (vdev == NULL) { 1767 return UINT64_MAX; 1768 } 1769 1770 switch (size) { 1771 case 1: 1772 val = virtio_config_modern_readb(vdev, addr); 1773 break; 1774 case 2: 1775 val = virtio_config_modern_readw(vdev, addr); 1776 break; 1777 case 4: 1778 val = virtio_config_modern_readl(vdev, addr); 1779 break; 1780 default: 1781 val = 0; 1782 break; 1783 } 1784 return val; 1785 } 1786 1787 static void virtio_pci_device_write(void *opaque, hwaddr addr, 1788 uint64_t val, unsigned size) 1789 { 1790 VirtIOPCIProxy *proxy = opaque; 1791 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1792 1793 if (vdev == NULL) { 1794 return; 1795 } 1796 1797 switch (size) { 1798 case 1: 1799 virtio_config_modern_writeb(vdev, addr, val); 1800 break; 1801 case 2: 1802 virtio_config_modern_writew(vdev, addr, val); 1803 break; 1804 case 4: 1805 virtio_config_modern_writel(vdev, addr, val); 1806 break; 1807 } 1808 } 1809 1810 static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy, 1811 const char *vdev_name) 1812 { 1813 static const MemoryRegionOps common_ops = { 1814 .read = virtio_pci_common_read, 1815 .write = virtio_pci_common_write, 1816 .impl = { 1817 .min_access_size = 1, 1818 .max_access_size = 4, 1819 }, 1820 .endianness = DEVICE_LITTLE_ENDIAN, 1821 }; 1822 static const MemoryRegionOps isr_ops = { 1823 .read = virtio_pci_isr_read, 1824 .write = virtio_pci_isr_write, 1825 .impl = { 1826 .min_access_size = 1, 1827 .max_access_size = 4, 1828 }, 1829 .endianness = DEVICE_LITTLE_ENDIAN, 1830 }; 1831 static const MemoryRegionOps device_ops = { 1832 .read = virtio_pci_device_read, 1833 .write = virtio_pci_device_write, 1834 .impl = { 1835 .min_access_size = 1, 1836 .max_access_size = 4, 1837 }, 1838 .endianness = DEVICE_LITTLE_ENDIAN, 1839 }; 1840 static const MemoryRegionOps notify_ops = { 1841 .read = virtio_pci_notify_read, 1842 .write = virtio_pci_notify_write, 1843 .impl = { 1844 .min_access_size = 1, 1845 .max_access_size = 4, 1846 }, 1847 .endianness = DEVICE_LITTLE_ENDIAN, 1848 }; 1849 static const MemoryRegionOps notify_pio_ops = { 1850 .read = virtio_pci_notify_read, 1851 .write = virtio_pci_notify_write_pio, 1852 .impl = { 1853 .min_access_size = 1, 1854 .max_access_size = 4, 1855 }, 1856 .endianness = DEVICE_LITTLE_ENDIAN, 1857 }; 1858 g_autoptr(GString) name = g_string_new(NULL); 1859 1860 g_string_printf(name, "virtio-pci-common-%s", vdev_name); 1861 memory_region_init_io(&proxy->common.mr, OBJECT(proxy), 1862 &common_ops, 1863 proxy, 1864 name->str, 1865 proxy->common.size); 1866 1867 g_string_printf(name, "virtio-pci-isr-%s", vdev_name); 1868 memory_region_init_io(&proxy->isr.mr, OBJECT(proxy), 1869 &isr_ops, 1870 proxy, 1871 name->str, 1872 proxy->isr.size); 1873 1874 g_string_printf(name, "virtio-pci-device-%s", vdev_name); 1875 memory_region_init_io(&proxy->device.mr, OBJECT(proxy), 1876 &device_ops, 1877 proxy, 1878 name->str, 1879 proxy->device.size); 1880 1881 g_string_printf(name, "virtio-pci-notify-%s", vdev_name); 1882 memory_region_init_io(&proxy->notify.mr, OBJECT(proxy), 1883 ¬ify_ops, 1884 proxy, 1885 name->str, 1886 proxy->notify.size); 1887 1888 g_string_printf(name, "virtio-pci-notify-pio-%s", vdev_name); 1889 memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy), 1890 ¬ify_pio_ops, 1891 proxy, 1892 name->str, 1893 proxy->notify_pio.size); 1894 } 1895 1896 static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy, 1897 VirtIOPCIRegion *region, 1898 struct virtio_pci_cap *cap, 1899 MemoryRegion *mr, 1900 uint8_t bar) 1901 { 1902 memory_region_add_subregion(mr, region->offset, ®ion->mr); 1903 1904 cap->cfg_type = region->type; 1905 cap->bar = bar; 1906 cap->offset = cpu_to_le32(region->offset); 1907 cap->length = cpu_to_le32(region->size); 1908 virtio_pci_add_mem_cap(proxy, cap); 1909 1910 } 1911 1912 static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy, 1913 VirtIOPCIRegion *region, 1914 struct virtio_pci_cap *cap) 1915 { 1916 virtio_pci_modern_region_map(proxy, region, cap, 1917 &proxy->modern_bar, proxy->modern_mem_bar_idx); 1918 } 1919 1920 static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy, 1921 VirtIOPCIRegion *region, 1922 struct virtio_pci_cap *cap) 1923 { 1924 virtio_pci_modern_region_map(proxy, region, cap, 1925 &proxy->io_bar, proxy->modern_io_bar_idx); 1926 } 1927 1928 static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy, 1929 VirtIOPCIRegion *region) 1930 { 1931 memory_region_del_subregion(&proxy->modern_bar, 1932 ®ion->mr); 1933 } 1934 1935 static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy, 1936 VirtIOPCIRegion *region) 1937 { 1938 memory_region_del_subregion(&proxy->io_bar, 1939 ®ion->mr); 1940 } 1941 1942 static void virtio_pci_pre_plugged(DeviceState *d, Error **errp) 1943 { 1944 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1945 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 1946 1947 if (virtio_pci_modern(proxy)) { 1948 virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1); 1949 } 1950 1951 virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE); 1952 } 1953 1954 /* This is called by virtio-bus just after the device is plugged. */ 1955 static void virtio_pci_device_plugged(DeviceState *d, Error **errp) 1956 { 1957 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 1958 VirtioBusState *bus = &proxy->bus; 1959 bool legacy = virtio_pci_legacy(proxy); 1960 bool modern; 1961 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 1962 uint8_t *config; 1963 uint32_t size; 1964 VirtIODevice *vdev = virtio_bus_get_device(bus); 1965 1966 /* 1967 * Virtio capabilities present without 1968 * VIRTIO_F_VERSION_1 confuses guests 1969 */ 1970 if (!proxy->ignore_backend_features && 1971 !virtio_has_feature(vdev->host_features, VIRTIO_F_VERSION_1)) { 1972 virtio_pci_disable_modern(proxy); 1973 1974 if (!legacy) { 1975 error_setg(errp, "Device doesn't support modern mode, and legacy" 1976 " mode is disabled"); 1977 error_append_hint(errp, "Set disable-legacy to off\n"); 1978 1979 return; 1980 } 1981 } 1982 1983 modern = virtio_pci_modern(proxy); 1984 1985 config = proxy->pci_dev.config; 1986 if (proxy->class_code) { 1987 pci_config_set_class(config, proxy->class_code); 1988 } 1989 1990 if (legacy) { 1991 if (!virtio_legacy_allowed(vdev)) { 1992 /* 1993 * To avoid migration issues, we allow legacy mode when legacy 1994 * check is disabled in the old machine types (< 5.1). 1995 */ 1996 if (virtio_legacy_check_disabled(vdev)) { 1997 warn_report("device is modern-only, but for backward " 1998 "compatibility legacy is allowed"); 1999 } else { 2000 error_setg(errp, 2001 "device is modern-only, use disable-legacy=on"); 2002 return; 2003 } 2004 } 2005 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 2006 error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by" 2007 " neither legacy nor transitional device"); 2008 return; 2009 } 2010 /* 2011 * Legacy and transitional devices use specific subsystem IDs. 2012 * Note that the subsystem vendor ID (config + PCI_SUBSYSTEM_VENDOR_ID) 2013 * is set to PCI_SUBVENDOR_ID_REDHAT_QUMRANET by default. 2014 */ 2015 pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus)); 2016 if (proxy->trans_devid) { 2017 pci_config_set_device_id(config, proxy->trans_devid); 2018 } 2019 } else { 2020 /* pure virtio-1.0 */ 2021 pci_set_word(config + PCI_VENDOR_ID, 2022 PCI_VENDOR_ID_REDHAT_QUMRANET); 2023 pci_set_word(config + PCI_DEVICE_ID, 2024 PCI_DEVICE_ID_VIRTIO_10_BASE + virtio_bus_get_vdev_id(bus)); 2025 pci_config_set_revision(config, 1); 2026 } 2027 config[PCI_INTERRUPT_PIN] = 1; 2028 2029 2030 if (modern) { 2031 struct virtio_pci_cap cap = { 2032 .cap_len = sizeof cap, 2033 }; 2034 struct virtio_pci_notify_cap notify = { 2035 .cap.cap_len = sizeof notify, 2036 .notify_off_multiplier = 2037 cpu_to_le32(virtio_pci_queue_mem_mult(proxy)), 2038 }; 2039 struct virtio_pci_cfg_cap cfg = { 2040 .cap.cap_len = sizeof cfg, 2041 .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG, 2042 }; 2043 struct virtio_pci_notify_cap notify_pio = { 2044 .cap.cap_len = sizeof notify, 2045 .notify_off_multiplier = cpu_to_le32(0x0), 2046 }; 2047 2048 struct virtio_pci_cfg_cap *cfg_mask; 2049 2050 virtio_pci_modern_regions_init(proxy, vdev->name); 2051 2052 virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap); 2053 virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap); 2054 virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap); 2055 virtio_pci_modern_mem_region_map(proxy, &proxy->notify, ¬ify.cap); 2056 2057 if (modern_pio) { 2058 memory_region_init(&proxy->io_bar, OBJECT(proxy), 2059 "virtio-pci-io", 0x4); 2060 address_space_init(&proxy->modern_cfg_io_as, &proxy->io_bar, 2061 "virtio-pci-cfg-io-as"); 2062 2063 pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar_idx, 2064 PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar); 2065 2066 virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio, 2067 ¬ify_pio.cap); 2068 } 2069 2070 pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar_idx, 2071 PCI_BASE_ADDRESS_SPACE_MEMORY | 2072 PCI_BASE_ADDRESS_MEM_PREFETCH | 2073 PCI_BASE_ADDRESS_MEM_TYPE_64, 2074 &proxy->modern_bar); 2075 2076 proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap); 2077 cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap); 2078 pci_set_byte(&cfg_mask->cap.bar, ~0x0); 2079 pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0); 2080 pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0); 2081 pci_set_long(cfg_mask->pci_cfg_data, ~0x0); 2082 } 2083 2084 if (proxy->nvectors) { 2085 int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors, 2086 proxy->msix_bar_idx, NULL); 2087 if (err) { 2088 /* Notice when a system that supports MSIx can't initialize it */ 2089 if (err != -ENOTSUP) { 2090 warn_report("unable to init msix vectors to %" PRIu32, 2091 proxy->nvectors); 2092 } 2093 proxy->nvectors = 0; 2094 } 2095 } 2096 2097 proxy->pci_dev.config_write = virtio_write_config; 2098 proxy->pci_dev.config_read = virtio_read_config; 2099 2100 if (legacy) { 2101 size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) 2102 + virtio_bus_get_vdev_config_len(bus); 2103 size = pow2ceil(size); 2104 2105 memory_region_init_io(&proxy->bar, OBJECT(proxy), 2106 &virtio_pci_config_ops, 2107 proxy, "virtio-pci", size); 2108 2109 pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx, 2110 PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar); 2111 } 2112 } 2113 2114 static void virtio_pci_device_unplugged(DeviceState *d) 2115 { 2116 VirtIOPCIProxy *proxy = VIRTIO_PCI(d); 2117 bool modern = virtio_pci_modern(proxy); 2118 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 2119 2120 virtio_pci_stop_ioeventfd(proxy); 2121 2122 if (modern) { 2123 virtio_pci_modern_mem_region_unmap(proxy, &proxy->common); 2124 virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr); 2125 virtio_pci_modern_mem_region_unmap(proxy, &proxy->device); 2126 virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify); 2127 if (modern_pio) { 2128 virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio); 2129 } 2130 } 2131 } 2132 2133 static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp) 2134 { 2135 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 2136 VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev); 2137 bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) && 2138 !pci_bus_is_root(pci_get_bus(pci_dev)); 2139 2140 /* fd-based ioevents can't be synchronized in record/replay */ 2141 if (replay_mode != REPLAY_MODE_NONE) { 2142 proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD; 2143 } 2144 2145 /* 2146 * virtio pci bar layout used by default. 2147 * subclasses can re-arrange things if needed. 2148 * 2149 * region 0 -- virtio legacy io bar 2150 * region 1 -- msi-x bar 2151 * region 2 -- virtio modern io bar (off by default) 2152 * region 4+5 -- virtio modern memory (64bit) bar 2153 * 2154 */ 2155 proxy->legacy_io_bar_idx = 0; 2156 proxy->msix_bar_idx = 1; 2157 proxy->modern_io_bar_idx = 2; 2158 proxy->modern_mem_bar_idx = 4; 2159 2160 proxy->common.offset = 0x0; 2161 proxy->common.size = 0x1000; 2162 proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG; 2163 2164 proxy->isr.offset = 0x1000; 2165 proxy->isr.size = 0x1000; 2166 proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG; 2167 2168 proxy->device.offset = 0x2000; 2169 proxy->device.size = 0x1000; 2170 proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG; 2171 2172 proxy->notify.offset = 0x3000; 2173 proxy->notify.size = virtio_pci_queue_mem_mult(proxy) * VIRTIO_QUEUE_MAX; 2174 proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG; 2175 2176 proxy->notify_pio.offset = 0x0; 2177 proxy->notify_pio.size = 0x4; 2178 proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG; 2179 2180 /* subclasses can enforce modern, so do this unconditionally */ 2181 memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci", 2182 /* PCI BAR regions must be powers of 2 */ 2183 pow2ceil(proxy->notify.offset + proxy->notify.size)); 2184 2185 address_space_init(&proxy->modern_cfg_mem_as, &proxy->modern_bar, 2186 "virtio-pci-cfg-mem-as"); 2187 2188 if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) { 2189 proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 2190 } 2191 2192 if (!virtio_pci_modern(proxy) && !virtio_pci_legacy(proxy)) { 2193 error_setg(errp, "device cannot work as neither modern nor legacy mode" 2194 " is enabled"); 2195 error_append_hint(errp, "Set either disable-modern or disable-legacy" 2196 " to off\n"); 2197 return; 2198 } 2199 2200 if (pcie_port && pci_is_express(pci_dev)) { 2201 int pos; 2202 uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE; 2203 2204 pos = pcie_endpoint_cap_init(pci_dev, 0); 2205 assert(pos > 0); 2206 2207 pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0, 2208 PCI_PM_SIZEOF, errp); 2209 if (pos < 0) { 2210 return; 2211 } 2212 2213 pci_dev->exp.pm_cap = pos; 2214 2215 /* 2216 * Indicates that this function complies with revision 1.2 of the 2217 * PCI Power Management Interface Specification. 2218 */ 2219 pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3); 2220 2221 if (proxy->flags & VIRTIO_PCI_FLAG_AER) { 2222 pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset, 2223 PCI_ERR_SIZEOF, NULL); 2224 last_pcie_cap_offset += PCI_ERR_SIZEOF; 2225 } 2226 2227 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) { 2228 /* Init error enabling flags */ 2229 pcie_cap_deverr_init(pci_dev); 2230 } 2231 2232 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_LNKCTL) { 2233 /* Init Link Control Register */ 2234 pcie_cap_lnkctl_init(pci_dev); 2235 } 2236 2237 if (proxy->flags & VIRTIO_PCI_FLAG_PM_NO_SOFT_RESET) { 2238 pci_set_word(pci_dev->config + pos + PCI_PM_CTRL, 2239 PCI_PM_CTRL_NO_SOFT_RESET); 2240 } 2241 2242 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_PM) { 2243 /* Init Power Management Control Register */ 2244 pci_set_word(pci_dev->wmask + pos + PCI_PM_CTRL, 2245 PCI_PM_CTRL_STATE_MASK); 2246 } 2247 2248 if (proxy->flags & VIRTIO_PCI_FLAG_ATS) { 2249 pcie_ats_init(pci_dev, last_pcie_cap_offset, 2250 proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED); 2251 last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF; 2252 } 2253 2254 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) { 2255 /* Set Function Level Reset capability bit */ 2256 pcie_cap_flr_init(pci_dev); 2257 } 2258 } else { 2259 /* 2260 * make future invocations of pci_is_express() return false 2261 * and pci_config_size() return PCI_CONFIG_SPACE_SIZE. 2262 */ 2263 pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS; 2264 } 2265 2266 virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy); 2267 if (k->realize) { 2268 k->realize(proxy, errp); 2269 } 2270 } 2271 2272 static void virtio_pci_exit(PCIDevice *pci_dev) 2273 { 2274 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev); 2275 bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) && 2276 !pci_bus_is_root(pci_get_bus(pci_dev)); 2277 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY; 2278 2279 msix_uninit_exclusive_bar(pci_dev); 2280 if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port && 2281 pci_is_express(pci_dev)) { 2282 pcie_aer_exit(pci_dev); 2283 } 2284 address_space_destroy(&proxy->modern_cfg_mem_as); 2285 if (modern_pio) { 2286 address_space_destroy(&proxy->modern_cfg_io_as); 2287 } 2288 } 2289 2290 static void virtio_pci_reset(DeviceState *qdev) 2291 { 2292 VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev); 2293 VirtioBusState *bus = VIRTIO_BUS(&proxy->bus); 2294 int i; 2295 2296 virtio_bus_reset(bus); 2297 msix_unuse_all_vectors(&proxy->pci_dev); 2298 2299 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2300 proxy->vqs[i].enabled = 0; 2301 proxy->vqs[i].reset = 0; 2302 proxy->vqs[i].num = 0; 2303 proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0; 2304 proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0; 2305 proxy->vqs[i].used[0] = proxy->vqs[i].used[1] = 0; 2306 } 2307 } 2308 2309 static bool virtio_pci_no_soft_reset(PCIDevice *dev) 2310 { 2311 uint16_t pmcsr; 2312 2313 if (!pci_is_express(dev) || !dev->exp.pm_cap) { 2314 return false; 2315 } 2316 2317 pmcsr = pci_get_word(dev->config + dev->exp.pm_cap + PCI_PM_CTRL); 2318 2319 /* 2320 * When No_Soft_Reset bit is set and the device 2321 * is in D3hot state, don't reset device 2322 */ 2323 return (pmcsr & PCI_PM_CTRL_NO_SOFT_RESET) && 2324 (pmcsr & PCI_PM_CTRL_STATE_MASK) == 3; 2325 } 2326 2327 static void virtio_pci_bus_reset_hold(Object *obj, ResetType type) 2328 { 2329 PCIDevice *dev = PCI_DEVICE(obj); 2330 DeviceState *qdev = DEVICE(obj); 2331 2332 if (virtio_pci_no_soft_reset(dev)) { 2333 return; 2334 } 2335 2336 virtio_pci_reset(qdev); 2337 2338 if (pci_is_express(dev)) { 2339 VirtIOPCIProxy *proxy = VIRTIO_PCI(dev); 2340 2341 pcie_cap_deverr_reset(dev); 2342 pcie_cap_lnkctl_reset(dev); 2343 2344 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_PM) { 2345 pci_word_test_and_clear_mask( 2346 dev->config + dev->exp.pm_cap + PCI_PM_CTRL, 2347 PCI_PM_CTRL_STATE_MASK); 2348 } 2349 } 2350 } 2351 2352 static Property virtio_pci_properties[] = { 2353 DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags, 2354 VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false), 2355 DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags, 2356 VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true), 2357 DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags, 2358 VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false), 2359 DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags, 2360 VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false), 2361 DEFINE_PROP_BIT("page-per-vq", VirtIOPCIProxy, flags, 2362 VIRTIO_PCI_FLAG_PAGE_PER_VQ_BIT, false), 2363 DEFINE_PROP_BOOL("x-ignore-backend-features", VirtIOPCIProxy, 2364 ignore_backend_features, false), 2365 DEFINE_PROP_BIT("ats", VirtIOPCIProxy, flags, 2366 VIRTIO_PCI_FLAG_ATS_BIT, false), 2367 DEFINE_PROP_BIT("x-ats-page-aligned", VirtIOPCIProxy, flags, 2368 VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED_BIT, true), 2369 DEFINE_PROP_BIT("x-pcie-deverr-init", VirtIOPCIProxy, flags, 2370 VIRTIO_PCI_FLAG_INIT_DEVERR_BIT, true), 2371 DEFINE_PROP_BIT("x-pcie-lnkctl-init", VirtIOPCIProxy, flags, 2372 VIRTIO_PCI_FLAG_INIT_LNKCTL_BIT, true), 2373 DEFINE_PROP_BIT("x-pcie-pm-init", VirtIOPCIProxy, flags, 2374 VIRTIO_PCI_FLAG_INIT_PM_BIT, true), 2375 DEFINE_PROP_BIT("x-pcie-pm-no-soft-reset", VirtIOPCIProxy, flags, 2376 VIRTIO_PCI_FLAG_PM_NO_SOFT_RESET_BIT, false), 2377 DEFINE_PROP_BIT("x-pcie-flr-init", VirtIOPCIProxy, flags, 2378 VIRTIO_PCI_FLAG_INIT_FLR_BIT, true), 2379 DEFINE_PROP_BIT("aer", VirtIOPCIProxy, flags, 2380 VIRTIO_PCI_FLAG_AER_BIT, false), 2381 DEFINE_PROP_END_OF_LIST(), 2382 }; 2383 2384 static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp) 2385 { 2386 VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev); 2387 VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev); 2388 PCIDevice *pci_dev = &proxy->pci_dev; 2389 2390 if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) && 2391 virtio_pci_modern(proxy)) { 2392 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS; 2393 } 2394 2395 vpciklass->parent_dc_realize(qdev, errp); 2396 } 2397 2398 static int virtio_pci_sync_config(DeviceState *dev, Error **errp) 2399 { 2400 VirtIOPCIProxy *proxy = VIRTIO_PCI(dev); 2401 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); 2402 2403 return qdev_sync_config(DEVICE(vdev), errp); 2404 } 2405 2406 static void virtio_pci_class_init(ObjectClass *klass, void *data) 2407 { 2408 DeviceClass *dc = DEVICE_CLASS(klass); 2409 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 2410 VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass); 2411 ResettableClass *rc = RESETTABLE_CLASS(klass); 2412 2413 device_class_set_props(dc, virtio_pci_properties); 2414 k->realize = virtio_pci_realize; 2415 k->exit = virtio_pci_exit; 2416 k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET; 2417 k->revision = VIRTIO_PCI_ABI_VERSION; 2418 k->class_id = PCI_CLASS_OTHERS; 2419 device_class_set_parent_realize(dc, virtio_pci_dc_realize, 2420 &vpciklass->parent_dc_realize); 2421 rc->phases.hold = virtio_pci_bus_reset_hold; 2422 dc->sync_config = virtio_pci_sync_config; 2423 } 2424 2425 static const TypeInfo virtio_pci_info = { 2426 .name = TYPE_VIRTIO_PCI, 2427 .parent = TYPE_PCI_DEVICE, 2428 .instance_size = sizeof(VirtIOPCIProxy), 2429 .class_init = virtio_pci_class_init, 2430 .class_size = sizeof(VirtioPCIClass), 2431 .abstract = true, 2432 }; 2433 2434 static Property virtio_pci_generic_properties[] = { 2435 DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy, 2436 ON_OFF_AUTO_AUTO), 2437 DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false), 2438 DEFINE_PROP_END_OF_LIST(), 2439 }; 2440 2441 static void virtio_pci_base_class_init(ObjectClass *klass, void *data) 2442 { 2443 const VirtioPCIDeviceTypeInfo *t = data; 2444 if (t->class_init) { 2445 t->class_init(klass, NULL); 2446 } 2447 } 2448 2449 static void virtio_pci_generic_class_init(ObjectClass *klass, void *data) 2450 { 2451 DeviceClass *dc = DEVICE_CLASS(klass); 2452 2453 device_class_set_props(dc, virtio_pci_generic_properties); 2454 } 2455 2456 static void virtio_pci_transitional_instance_init(Object *obj) 2457 { 2458 VirtIOPCIProxy *proxy = VIRTIO_PCI(obj); 2459 2460 proxy->disable_legacy = ON_OFF_AUTO_OFF; 2461 proxy->disable_modern = false; 2462 } 2463 2464 static void virtio_pci_non_transitional_instance_init(Object *obj) 2465 { 2466 VirtIOPCIProxy *proxy = VIRTIO_PCI(obj); 2467 2468 proxy->disable_legacy = ON_OFF_AUTO_ON; 2469 proxy->disable_modern = false; 2470 } 2471 2472 void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t) 2473 { 2474 char *base_name = NULL; 2475 TypeInfo base_type_info = { 2476 .name = t->base_name, 2477 .parent = t->parent ? t->parent : TYPE_VIRTIO_PCI, 2478 .instance_size = t->instance_size, 2479 .instance_init = t->instance_init, 2480 .instance_finalize = t->instance_finalize, 2481 .class_size = t->class_size, 2482 .abstract = true, 2483 .interfaces = t->interfaces, 2484 }; 2485 TypeInfo generic_type_info = { 2486 .name = t->generic_name, 2487 .parent = base_type_info.name, 2488 .class_init = virtio_pci_generic_class_init, 2489 .interfaces = (InterfaceInfo[]) { 2490 { INTERFACE_PCIE_DEVICE }, 2491 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 2492 { } 2493 }, 2494 }; 2495 2496 if (!base_type_info.name) { 2497 /* No base type -> register a single generic device type */ 2498 /* use intermediate %s-base-type to add generic device props */ 2499 base_name = g_strdup_printf("%s-base-type", t->generic_name); 2500 base_type_info.name = base_name; 2501 base_type_info.class_init = virtio_pci_generic_class_init; 2502 2503 generic_type_info.parent = base_name; 2504 generic_type_info.class_init = virtio_pci_base_class_init; 2505 generic_type_info.class_data = (void *)t; 2506 2507 assert(!t->non_transitional_name); 2508 assert(!t->transitional_name); 2509 } else { 2510 base_type_info.class_init = virtio_pci_base_class_init; 2511 base_type_info.class_data = (void *)t; 2512 } 2513 2514 type_register(&base_type_info); 2515 if (generic_type_info.name) { 2516 type_register(&generic_type_info); 2517 } 2518 2519 if (t->non_transitional_name) { 2520 const TypeInfo non_transitional_type_info = { 2521 .name = t->non_transitional_name, 2522 .parent = base_type_info.name, 2523 .instance_init = virtio_pci_non_transitional_instance_init, 2524 .interfaces = (InterfaceInfo[]) { 2525 { INTERFACE_PCIE_DEVICE }, 2526 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 2527 { } 2528 }, 2529 }; 2530 type_register(&non_transitional_type_info); 2531 } 2532 2533 if (t->transitional_name) { 2534 const TypeInfo transitional_type_info = { 2535 .name = t->transitional_name, 2536 .parent = base_type_info.name, 2537 .instance_init = virtio_pci_transitional_instance_init, 2538 .interfaces = (InterfaceInfo[]) { 2539 /* 2540 * Transitional virtio devices work only as Conventional PCI 2541 * devices because they require PIO ports. 2542 */ 2543 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 2544 { } 2545 }, 2546 }; 2547 type_register(&transitional_type_info); 2548 } 2549 g_free(base_name); 2550 } 2551 2552 unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues) 2553 { 2554 /* 2555 * 1:1 vq to vCPU mapping is ideal because the same vCPU that submitted 2556 * virtqueue buffers can handle their completion. When a different vCPU 2557 * handles completion it may need to IPI the vCPU that submitted the 2558 * request and this adds overhead. 2559 * 2560 * Virtqueues consume guest RAM and MSI-X vectors. This is wasteful in 2561 * guests with very many vCPUs and a device that is only used by a few 2562 * vCPUs. Unfortunately optimizing that case requires manual pinning inside 2563 * the guest, so those users might as well manually set the number of 2564 * queues. There is no upper limit that can be applied automatically and 2565 * doing so arbitrarily would result in a sudden performance drop once the 2566 * threshold number of vCPUs is exceeded. 2567 */ 2568 unsigned num_queues = current_machine->smp.cpus; 2569 2570 /* 2571 * The maximum number of MSI-X vectors is PCI_MSIX_FLAGS_QSIZE + 1, but the 2572 * config change interrupt and the fixed virtqueues must be taken into 2573 * account too. 2574 */ 2575 num_queues = MIN(num_queues, PCI_MSIX_FLAGS_QSIZE - fixed_queues); 2576 2577 /* 2578 * There is a limit to how many virtqueues a device can have. 2579 */ 2580 return MIN(num_queues, VIRTIO_QUEUE_MAX - fixed_queues); 2581 } 2582 2583 /* virtio-pci-bus */ 2584 2585 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size, 2586 VirtIOPCIProxy *dev) 2587 { 2588 DeviceState *qdev = DEVICE(dev); 2589 char virtio_bus_name[] = "virtio-bus"; 2590 2591 qbus_init(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev, virtio_bus_name); 2592 } 2593 2594 static void virtio_pci_bus_class_init(ObjectClass *klass, void *data) 2595 { 2596 BusClass *bus_class = BUS_CLASS(klass); 2597 VirtioBusClass *k = VIRTIO_BUS_CLASS(klass); 2598 bus_class->max_dev = 1; 2599 k->notify = virtio_pci_notify; 2600 k->save_config = virtio_pci_save_config; 2601 k->load_config = virtio_pci_load_config; 2602 k->save_queue = virtio_pci_save_queue; 2603 k->load_queue = virtio_pci_load_queue; 2604 k->save_extra_state = virtio_pci_save_extra_state; 2605 k->load_extra_state = virtio_pci_load_extra_state; 2606 k->has_extra_state = virtio_pci_has_extra_state; 2607 k->query_guest_notifiers = virtio_pci_query_guest_notifiers; 2608 k->set_guest_notifiers = virtio_pci_set_guest_notifiers; 2609 k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr; 2610 k->vmstate_change = virtio_pci_vmstate_change; 2611 k->pre_plugged = virtio_pci_pre_plugged; 2612 k->device_plugged = virtio_pci_device_plugged; 2613 k->device_unplugged = virtio_pci_device_unplugged; 2614 k->query_nvectors = virtio_pci_query_nvectors; 2615 k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled; 2616 k->ioeventfd_assign = virtio_pci_ioeventfd_assign; 2617 k->get_dma_as = virtio_pci_get_dma_as; 2618 k->iommu_enabled = virtio_pci_iommu_enabled; 2619 k->queue_enabled = virtio_pci_queue_enabled; 2620 } 2621 2622 static const TypeInfo virtio_pci_bus_info = { 2623 .name = TYPE_VIRTIO_PCI_BUS, 2624 .parent = TYPE_VIRTIO_BUS, 2625 .instance_size = sizeof(VirtioPCIBusState), 2626 .class_size = sizeof(VirtioPCIBusClass), 2627 .class_init = virtio_pci_bus_class_init, 2628 }; 2629 2630 static void virtio_pci_register_types(void) 2631 { 2632 /* Base types: */ 2633 type_register_static(&virtio_pci_bus_info); 2634 type_register_static(&virtio_pci_info); 2635 } 2636 2637 type_init(virtio_pci_register_types) 2638 2639