1 /* 2 * libqos virtio PCI driver 3 * 4 * Copyright (c) 2014 Marc Marí 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 */ 9 10 #include "qemu/osdep.h" 11 #include "../libqtest.h" 12 #include "virtio.h" 13 #include "virtio-pci.h" 14 #include "pci.h" 15 #include "pci-pc.h" 16 #include "libqos-malloc.h" 17 #include "malloc-pc.h" 18 #include "qgraph.h" 19 #include "standard-headers/linux/virtio_ring.h" 20 #include "standard-headers/linux/virtio_pci.h" 21 22 #include "hw/pci/pci.h" 23 #include "hw/pci/pci_regs.h" 24 25 #include "virtio-pci-modern.h" 26 27 /* virtio-pci is a superclass of all virtio-xxx-pci devices; 28 * the relation between virtio-pci and virtio-xxx-pci is implicit, 29 * and therefore virtio-pci does not produce virtio and is not 30 * reached by any edge, not even as a "contains" edge. 31 * In facts, every device is a QVirtioPCIDevice with 32 * additional fields, since every one has its own 33 * number of queues and various attributes. 34 * Virtio-pci provides default functions to start the 35 * hw and destroy the object, and nodes that want to 36 * override them should always remember to call the 37 * original qvirtio_pci_destructor and qvirtio_pci_start_hw. 38 */ 39 40 #define CONFIG_BASE(dev) (VIRTIO_PCI_CONFIG_OFF((dev)->pdev->msix_enabled)) 41 42 static uint8_t qvirtio_pci_config_readb(QVirtioDevice *d, uint64_t off) 43 { 44 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 45 return qpci_io_readb(dev->pdev, dev->bar, CONFIG_BASE(dev) + off); 46 } 47 48 /* PCI is always read in little-endian order 49 * but virtio ( < 1.0) is in guest order 50 * so with a big-endian guest the order has been reversed, 51 * reverse it again 52 * virtio-1.0 is always little-endian, like PCI 53 */ 54 55 static uint16_t qvirtio_pci_config_readw(QVirtioDevice *d, uint64_t off) 56 { 57 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 58 uint16_t value; 59 60 value = qpci_io_readw(dev->pdev, dev->bar, CONFIG_BASE(dev) + off); 61 if (qvirtio_is_big_endian(d)) { 62 value = bswap16(value); 63 } 64 return value; 65 } 66 67 static uint32_t qvirtio_pci_config_readl(QVirtioDevice *d, uint64_t off) 68 { 69 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 70 uint32_t value; 71 72 value = qpci_io_readl(dev->pdev, dev->bar, CONFIG_BASE(dev) + off); 73 if (qvirtio_is_big_endian(d)) { 74 value = bswap32(value); 75 } 76 return value; 77 } 78 79 static uint64_t qvirtio_pci_config_readq(QVirtioDevice *d, uint64_t off) 80 { 81 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 82 uint64_t val; 83 84 val = qpci_io_readq(dev->pdev, dev->bar, CONFIG_BASE(dev) + off); 85 if (qvirtio_is_big_endian(d)) { 86 val = bswap64(val); 87 } 88 89 return val; 90 } 91 92 static uint64_t qvirtio_pci_get_features(QVirtioDevice *d) 93 { 94 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 95 return qpci_io_readl(dev->pdev, dev->bar, VIRTIO_PCI_HOST_FEATURES); 96 } 97 98 static void qvirtio_pci_set_features(QVirtioDevice *d, uint64_t features) 99 { 100 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 101 qpci_io_writel(dev->pdev, dev->bar, VIRTIO_PCI_GUEST_FEATURES, features); 102 } 103 104 static uint64_t qvirtio_pci_get_guest_features(QVirtioDevice *d) 105 { 106 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 107 return qpci_io_readl(dev->pdev, dev->bar, VIRTIO_PCI_GUEST_FEATURES); 108 } 109 110 static uint8_t qvirtio_pci_get_status(QVirtioDevice *d) 111 { 112 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 113 return qpci_io_readb(dev->pdev, dev->bar, VIRTIO_PCI_STATUS); 114 } 115 116 static void qvirtio_pci_set_status(QVirtioDevice *d, uint8_t status) 117 { 118 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 119 qpci_io_writeb(dev->pdev, dev->bar, VIRTIO_PCI_STATUS, status); 120 } 121 122 static bool qvirtio_pci_get_queue_isr_status(QVirtioDevice *d, QVirtQueue *vq) 123 { 124 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 125 QVirtQueuePCI *vqpci = (QVirtQueuePCI *)vq; 126 uint32_t data; 127 128 if (dev->pdev->msix_enabled) { 129 g_assert_cmpint(vqpci->msix_entry, !=, -1); 130 if (qpci_msix_masked(dev->pdev, vqpci->msix_entry)) { 131 /* No ISR checking should be done if masked, but read anyway */ 132 return qpci_msix_pending(dev->pdev, vqpci->msix_entry); 133 } else { 134 data = qtest_readl(dev->pdev->bus->qts, vqpci->msix_addr); 135 if (data == vqpci->msix_data) { 136 qtest_writel(dev->pdev->bus->qts, vqpci->msix_addr, 0); 137 return true; 138 } else { 139 return false; 140 } 141 } 142 } else { 143 return qpci_io_readb(dev->pdev, dev->bar, VIRTIO_PCI_ISR) & 1; 144 } 145 } 146 147 static bool qvirtio_pci_get_config_isr_status(QVirtioDevice *d) 148 { 149 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 150 uint32_t data; 151 152 if (dev->pdev->msix_enabled) { 153 g_assert_cmpint(dev->config_msix_entry, !=, -1); 154 if (qpci_msix_masked(dev->pdev, dev->config_msix_entry)) { 155 /* No ISR checking should be done if masked, but read anyway */ 156 return qpci_msix_pending(dev->pdev, dev->config_msix_entry); 157 } else { 158 data = qtest_readl(dev->pdev->bus->qts, dev->config_msix_addr); 159 if (data == dev->config_msix_data) { 160 qtest_writel(dev->pdev->bus->qts, dev->config_msix_addr, 0); 161 return true; 162 } else { 163 return false; 164 } 165 } 166 } else { 167 return qpci_io_readb(dev->pdev, dev->bar, VIRTIO_PCI_ISR) & 2; 168 } 169 } 170 171 static void qvirtio_pci_wait_config_isr_status(QVirtioDevice *d, 172 gint64 timeout_us) 173 { 174 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 175 gint64 start_time = g_get_monotonic_time(); 176 177 do { 178 g_assert(g_get_monotonic_time() - start_time <= timeout_us); 179 qtest_clock_step(dev->pdev->bus->qts, 100); 180 } while (!qvirtio_pci_get_config_isr_status(d)); 181 } 182 183 static void qvirtio_pci_queue_select(QVirtioDevice *d, uint16_t index) 184 { 185 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 186 qpci_io_writeb(dev->pdev, dev->bar, VIRTIO_PCI_QUEUE_SEL, index); 187 } 188 189 static uint16_t qvirtio_pci_get_queue_size(QVirtioDevice *d) 190 { 191 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 192 return qpci_io_readw(dev->pdev, dev->bar, VIRTIO_PCI_QUEUE_NUM); 193 } 194 195 static void qvirtio_pci_set_queue_address(QVirtioDevice *d, QVirtQueue *vq) 196 { 197 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 198 uint64_t pfn = vq->desc / VIRTIO_PCI_VRING_ALIGN; 199 200 qpci_io_writel(dev->pdev, dev->bar, VIRTIO_PCI_QUEUE_PFN, pfn); 201 } 202 203 QVirtQueue *qvirtio_pci_virtqueue_setup_common(QVirtioDevice *d, 204 QGuestAllocator *alloc, 205 uint16_t index) 206 { 207 uint64_t feat; 208 uint64_t addr; 209 QVirtQueuePCI *vqpci; 210 QVirtioPCIDevice *qvpcidev = container_of(d, QVirtioPCIDevice, vdev); 211 212 vqpci = g_malloc0(sizeof(*vqpci)); 213 feat = d->bus->get_guest_features(d); 214 215 d->bus->queue_select(d, index); 216 vqpci->vq.vdev = d; 217 vqpci->vq.index = index; 218 vqpci->vq.size = d->bus->get_queue_size(d); 219 vqpci->vq.free_head = 0; 220 vqpci->vq.num_free = vqpci->vq.size; 221 vqpci->vq.align = VIRTIO_PCI_VRING_ALIGN; 222 vqpci->vq.indirect = feat & (1ull << VIRTIO_RING_F_INDIRECT_DESC); 223 vqpci->vq.event = feat & (1ull << VIRTIO_RING_F_EVENT_IDX); 224 225 vqpci->msix_entry = -1; 226 vqpci->msix_addr = 0; 227 vqpci->msix_data = 0x12345678; 228 229 /* Check different than 0 */ 230 g_assert_cmpint(vqpci->vq.size, !=, 0); 231 232 /* Check power of 2 */ 233 g_assert_cmpint(vqpci->vq.size & (vqpci->vq.size - 1), ==, 0); 234 235 addr = guest_alloc(alloc, qvring_size(vqpci->vq.size, 236 VIRTIO_PCI_VRING_ALIGN)); 237 qvring_init(qvpcidev->pdev->bus->qts, alloc, &vqpci->vq, addr); 238 d->bus->set_queue_address(d, &vqpci->vq); 239 240 return &vqpci->vq; 241 } 242 243 void qvirtio_pci_virtqueue_cleanup_common(QVirtQueue *vq, 244 QGuestAllocator *alloc) 245 { 246 QVirtQueuePCI *vqpci = container_of(vq, QVirtQueuePCI, vq); 247 248 guest_free(alloc, vq->desc); 249 g_free(vqpci); 250 } 251 252 static void qvirtio_pci_virtqueue_kick(QVirtioDevice *d, QVirtQueue *vq) 253 { 254 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev); 255 qpci_io_writew(dev->pdev, dev->bar, VIRTIO_PCI_QUEUE_NOTIFY, vq->index); 256 } 257 258 static const QVirtioBus qvirtio_pci_legacy = { 259 .config_readb = qvirtio_pci_config_readb, 260 .config_readw = qvirtio_pci_config_readw, 261 .config_readl = qvirtio_pci_config_readl, 262 .config_readq = qvirtio_pci_config_readq, 263 .get_features = qvirtio_pci_get_features, 264 .set_features = qvirtio_pci_set_features, 265 .get_guest_features = qvirtio_pci_get_guest_features, 266 .get_status = qvirtio_pci_get_status, 267 .set_status = qvirtio_pci_set_status, 268 .get_queue_isr_status = qvirtio_pci_get_queue_isr_status, 269 .wait_config_isr_status = qvirtio_pci_wait_config_isr_status, 270 .queue_select = qvirtio_pci_queue_select, 271 .get_queue_size = qvirtio_pci_get_queue_size, 272 .set_queue_address = qvirtio_pci_set_queue_address, 273 .virtqueue_setup = qvirtio_pci_virtqueue_setup_common, 274 .virtqueue_cleanup = qvirtio_pci_virtqueue_cleanup_common, 275 .virtqueue_kick = qvirtio_pci_virtqueue_kick, 276 }; 277 278 static void qvirtio_pci_set_config_vector(QVirtioPCIDevice *d, uint16_t entry) 279 { 280 uint16_t vector; 281 282 qpci_io_writew(d->pdev, d->bar, VIRTIO_MSI_CONFIG_VECTOR, entry); 283 vector = qpci_io_readw(d->pdev, d->bar, VIRTIO_MSI_CONFIG_VECTOR); 284 g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR); 285 } 286 287 static void qvirtio_pci_set_queue_vector(QVirtioPCIDevice *d, uint16_t vq_idx, 288 uint16_t entry) 289 { 290 uint16_t vector; 291 292 qvirtio_pci_queue_select(&d->vdev, vq_idx); 293 qpci_io_writew(d->pdev, d->bar, VIRTIO_MSI_QUEUE_VECTOR, entry); 294 vector = qpci_io_readw(d->pdev, d->bar, VIRTIO_MSI_QUEUE_VECTOR); 295 g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR); 296 } 297 298 static const QVirtioPCIMSIXOps qvirtio_pci_msix_ops_legacy = { 299 .set_config_vector = qvirtio_pci_set_config_vector, 300 .set_queue_vector = qvirtio_pci_set_queue_vector, 301 }; 302 303 void qvirtio_pci_device_enable(QVirtioPCIDevice *d) 304 { 305 qpci_device_enable(d->pdev); 306 d->bar = qpci_iomap(d->pdev, d->bar_idx, NULL); 307 } 308 309 void qvirtio_pci_device_disable(QVirtioPCIDevice *d) 310 { 311 qpci_iounmap(d->pdev, d->bar); 312 } 313 314 void qvirtqueue_pci_msix_setup(QVirtioPCIDevice *d, QVirtQueuePCI *vqpci, 315 QGuestAllocator *alloc, uint16_t entry) 316 { 317 uint32_t control; 318 uint64_t off; 319 320 g_assert(d->pdev->msix_enabled); 321 off = d->pdev->msix_table_off + (entry * 16); 322 323 g_assert_cmpint(entry, >=, 0); 324 g_assert_cmpint(entry, <, qpci_msix_table_size(d->pdev)); 325 vqpci->msix_entry = entry; 326 327 vqpci->msix_addr = guest_alloc(alloc, 4); 328 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 329 off + PCI_MSIX_ENTRY_LOWER_ADDR, vqpci->msix_addr & ~0UL); 330 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 331 off + PCI_MSIX_ENTRY_UPPER_ADDR, 332 (vqpci->msix_addr >> 32) & ~0UL); 333 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 334 off + PCI_MSIX_ENTRY_DATA, vqpci->msix_data); 335 336 control = qpci_io_readl(d->pdev, d->pdev->msix_table_bar, 337 off + PCI_MSIX_ENTRY_VECTOR_CTRL); 338 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 339 off + PCI_MSIX_ENTRY_VECTOR_CTRL, 340 control & ~PCI_MSIX_ENTRY_CTRL_MASKBIT); 341 342 d->msix_ops->set_queue_vector(d, vqpci->vq.index, entry); 343 } 344 345 void qvirtio_pci_set_msix_configuration_vector(QVirtioPCIDevice *d, 346 QGuestAllocator *alloc, uint16_t entry) 347 { 348 uint32_t control; 349 uint64_t off; 350 351 g_assert(d->pdev->msix_enabled); 352 off = d->pdev->msix_table_off + (entry * 16); 353 354 g_assert_cmpint(entry, >=, 0); 355 g_assert_cmpint(entry, <, qpci_msix_table_size(d->pdev)); 356 d->config_msix_entry = entry; 357 358 d->config_msix_data = 0x12345678; 359 d->config_msix_addr = guest_alloc(alloc, 4); 360 361 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 362 off + PCI_MSIX_ENTRY_LOWER_ADDR, d->config_msix_addr & ~0UL); 363 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 364 off + PCI_MSIX_ENTRY_UPPER_ADDR, 365 (d->config_msix_addr >> 32) & ~0UL); 366 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 367 off + PCI_MSIX_ENTRY_DATA, d->config_msix_data); 368 369 control = qpci_io_readl(d->pdev, d->pdev->msix_table_bar, 370 off + PCI_MSIX_ENTRY_VECTOR_CTRL); 371 qpci_io_writel(d->pdev, d->pdev->msix_table_bar, 372 off + PCI_MSIX_ENTRY_VECTOR_CTRL, 373 control & ~PCI_MSIX_ENTRY_CTRL_MASKBIT); 374 375 d->msix_ops->set_config_vector(d, entry); 376 } 377 378 void qvirtio_pci_destructor(QOSGraphObject *obj) 379 { 380 QVirtioPCIDevice *dev = (QVirtioPCIDevice *)obj; 381 qvirtio_pci_device_disable(dev); 382 g_free(dev->pdev); 383 } 384 385 void qvirtio_pci_start_hw(QOSGraphObject *obj) 386 { 387 QVirtioPCIDevice *dev = (QVirtioPCIDevice *)obj; 388 qvirtio_pci_device_enable(dev); 389 qvirtio_start_device(&dev->vdev); 390 } 391 392 static void qvirtio_pci_init_legacy(QVirtioPCIDevice *dev) 393 { 394 dev->vdev.device_type = qpci_config_readw(dev->pdev, PCI_SUBSYSTEM_ID); 395 dev->bar_idx = 0; 396 dev->vdev.bus = &qvirtio_pci_legacy; 397 dev->msix_ops = &qvirtio_pci_msix_ops_legacy; 398 dev->vdev.big_endian = qtest_big_endian(dev->pdev->bus->qts); 399 } 400 401 static void qvirtio_pci_init_from_pcidev(QVirtioPCIDevice *dev, QPCIDevice *pci_dev) 402 { 403 dev->pdev = pci_dev; 404 dev->config_msix_entry = -1; 405 406 if (!qvirtio_pci_init_virtio_1(dev)) { 407 qvirtio_pci_init_legacy(dev); 408 } 409 410 /* each virtio-xxx-pci device should override at least this function */ 411 dev->obj.get_driver = NULL; 412 dev->obj.start_hw = qvirtio_pci_start_hw; 413 dev->obj.destructor = qvirtio_pci_destructor; 414 } 415 416 void virtio_pci_init(QVirtioPCIDevice *dev, QPCIBus *bus, QPCIAddress * addr) 417 { 418 QPCIDevice *pci_dev = qpci_device_find(bus, addr->devfn); 419 g_assert_nonnull(pci_dev); 420 qvirtio_pci_init_from_pcidev(dev, pci_dev); 421 } 422 423 QVirtioPCIDevice *virtio_pci_new(QPCIBus *bus, QPCIAddress * addr) 424 { 425 QVirtioPCIDevice *dev; 426 QPCIDevice *pci_dev = qpci_device_find(bus, addr->devfn); 427 if (!pci_dev) { 428 return NULL; 429 } 430 431 dev = g_new0(QVirtioPCIDevice, 1); 432 qvirtio_pci_init_from_pcidev(dev, pci_dev); 433 dev->obj.free = g_free; 434 return dev; 435 } 436