1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qapi/error.h" 16 #include "cpu.h" 17 #include "trace.h" 18 #include "exec/address-spaces.h" 19 #include "qemu/error-report.h" 20 #include "qemu/main-loop.h" 21 #include "qemu/module.h" 22 #include "hw/virtio/virtio.h" 23 #include "migration/qemu-file-types.h" 24 #include "qemu/atomic.h" 25 #include "hw/virtio/virtio-bus.h" 26 #include "hw/qdev-properties.h" 27 #include "hw/virtio/virtio-access.h" 28 #include "sysemu/dma.h" 29 30 /* 31 * The alignment to use between consumer and producer parts of vring. 32 * x86 pagesize again. This is the default, used by transports like PCI 33 * which don't provide a means for the guest to tell the host the alignment. 34 */ 35 #define VIRTIO_PCI_VRING_ALIGN 4096 36 37 typedef struct VRingDesc 38 { 39 uint64_t addr; 40 uint32_t len; 41 uint16_t flags; 42 uint16_t next; 43 } VRingDesc; 44 45 typedef struct VRingAvail 46 { 47 uint16_t flags; 48 uint16_t idx; 49 uint16_t ring[0]; 50 } VRingAvail; 51 52 typedef struct VRingUsedElem 53 { 54 uint32_t id; 55 uint32_t len; 56 } VRingUsedElem; 57 58 typedef struct VRingUsed 59 { 60 uint16_t flags; 61 uint16_t idx; 62 VRingUsedElem ring[0]; 63 } VRingUsed; 64 65 typedef struct VRingMemoryRegionCaches { 66 struct rcu_head rcu; 67 MemoryRegionCache desc; 68 MemoryRegionCache avail; 69 MemoryRegionCache used; 70 } VRingMemoryRegionCaches; 71 72 typedef struct VRing 73 { 74 unsigned int num; 75 unsigned int num_default; 76 unsigned int align; 77 hwaddr desc; 78 hwaddr avail; 79 hwaddr used; 80 VRingMemoryRegionCaches *caches; 81 } VRing; 82 83 struct VirtQueue 84 { 85 VRing vring; 86 87 /* Next head to pop */ 88 uint16_t last_avail_idx; 89 90 /* Last avail_idx read from VQ. */ 91 uint16_t shadow_avail_idx; 92 93 uint16_t used_idx; 94 95 /* Last used index value we have signalled on */ 96 uint16_t signalled_used; 97 98 /* Last used index value we have signalled on */ 99 bool signalled_used_valid; 100 101 /* Notification enabled? */ 102 bool notification; 103 104 uint16_t queue_index; 105 106 unsigned int inuse; 107 108 uint16_t vector; 109 VirtIOHandleOutput handle_output; 110 VirtIOHandleAIOOutput handle_aio_output; 111 VirtIODevice *vdev; 112 EventNotifier guest_notifier; 113 EventNotifier host_notifier; 114 QLIST_ENTRY(VirtQueue) node; 115 }; 116 117 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) 118 { 119 if (!caches) { 120 return; 121 } 122 123 address_space_cache_destroy(&caches->desc); 124 address_space_cache_destroy(&caches->avail); 125 address_space_cache_destroy(&caches->used); 126 g_free(caches); 127 } 128 129 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq) 130 { 131 VRingMemoryRegionCaches *caches; 132 133 caches = atomic_read(&vq->vring.caches); 134 atomic_rcu_set(&vq->vring.caches, NULL); 135 if (caches) { 136 call_rcu(caches, virtio_free_region_cache, rcu); 137 } 138 } 139 140 static void virtio_init_region_cache(VirtIODevice *vdev, int n) 141 { 142 VirtQueue *vq = &vdev->vq[n]; 143 VRingMemoryRegionCaches *old = vq->vring.caches; 144 VRingMemoryRegionCaches *new = NULL; 145 hwaddr addr, size; 146 int event_size; 147 int64_t len; 148 149 event_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 150 151 addr = vq->vring.desc; 152 if (!addr) { 153 goto out_no_cache; 154 } 155 new = g_new0(VRingMemoryRegionCaches, 1); 156 size = virtio_queue_get_desc_size(vdev, n); 157 len = address_space_cache_init(&new->desc, vdev->dma_as, 158 addr, size, false); 159 if (len < size) { 160 virtio_error(vdev, "Cannot map desc"); 161 goto err_desc; 162 } 163 164 size = virtio_queue_get_used_size(vdev, n) + event_size; 165 len = address_space_cache_init(&new->used, vdev->dma_as, 166 vq->vring.used, size, true); 167 if (len < size) { 168 virtio_error(vdev, "Cannot map used"); 169 goto err_used; 170 } 171 172 size = virtio_queue_get_avail_size(vdev, n) + event_size; 173 len = address_space_cache_init(&new->avail, vdev->dma_as, 174 vq->vring.avail, size, false); 175 if (len < size) { 176 virtio_error(vdev, "Cannot map avail"); 177 goto err_avail; 178 } 179 180 atomic_rcu_set(&vq->vring.caches, new); 181 if (old) { 182 call_rcu(old, virtio_free_region_cache, rcu); 183 } 184 return; 185 186 err_avail: 187 address_space_cache_destroy(&new->avail); 188 err_used: 189 address_space_cache_destroy(&new->used); 190 err_desc: 191 address_space_cache_destroy(&new->desc); 192 out_no_cache: 193 g_free(new); 194 virtio_virtqueue_reset_region_cache(vq); 195 } 196 197 /* virt queue functions */ 198 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 199 { 200 VRing *vring = &vdev->vq[n].vring; 201 202 if (!vring->num || !vring->desc || !vring->align) { 203 /* not yet setup -> nothing to do */ 204 return; 205 } 206 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 207 vring->used = vring_align(vring->avail + 208 offsetof(VRingAvail, ring[vring->num]), 209 vring->align); 210 virtio_init_region_cache(vdev, n); 211 } 212 213 /* Called within rcu_read_lock(). */ 214 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc, 215 MemoryRegionCache *cache, int i) 216 { 217 address_space_read_cached(cache, i * sizeof(VRingDesc), 218 desc, sizeof(VRingDesc)); 219 virtio_tswap64s(vdev, &desc->addr); 220 virtio_tswap32s(vdev, &desc->len); 221 virtio_tswap16s(vdev, &desc->flags); 222 virtio_tswap16s(vdev, &desc->next); 223 } 224 225 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq) 226 { 227 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 228 assert(caches != NULL); 229 return caches; 230 } 231 /* Called within rcu_read_lock(). */ 232 static inline uint16_t vring_avail_flags(VirtQueue *vq) 233 { 234 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 235 hwaddr pa = offsetof(VRingAvail, flags); 236 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 237 } 238 239 /* Called within rcu_read_lock(). */ 240 static inline uint16_t vring_avail_idx(VirtQueue *vq) 241 { 242 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 243 hwaddr pa = offsetof(VRingAvail, idx); 244 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 245 return vq->shadow_avail_idx; 246 } 247 248 /* Called within rcu_read_lock(). */ 249 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 250 { 251 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 252 hwaddr pa = offsetof(VRingAvail, ring[i]); 253 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 254 } 255 256 /* Called within rcu_read_lock(). */ 257 static inline uint16_t vring_get_used_event(VirtQueue *vq) 258 { 259 return vring_avail_ring(vq, vq->vring.num); 260 } 261 262 /* Called within rcu_read_lock(). */ 263 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, 264 int i) 265 { 266 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 267 hwaddr pa = offsetof(VRingUsed, ring[i]); 268 virtio_tswap32s(vq->vdev, &uelem->id); 269 virtio_tswap32s(vq->vdev, &uelem->len); 270 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); 271 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); 272 } 273 274 /* Called within rcu_read_lock(). */ 275 static uint16_t vring_used_idx(VirtQueue *vq) 276 { 277 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 278 hwaddr pa = offsetof(VRingUsed, idx); 279 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 280 } 281 282 /* Called within rcu_read_lock(). */ 283 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 284 { 285 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 286 hwaddr pa = offsetof(VRingUsed, idx); 287 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 288 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 289 vq->used_idx = val; 290 } 291 292 /* Called within rcu_read_lock(). */ 293 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 294 { 295 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 296 VirtIODevice *vdev = vq->vdev; 297 hwaddr pa = offsetof(VRingUsed, flags); 298 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 299 300 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); 301 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 302 } 303 304 /* Called within rcu_read_lock(). */ 305 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 306 { 307 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq); 308 VirtIODevice *vdev = vq->vdev; 309 hwaddr pa = offsetof(VRingUsed, flags); 310 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 311 312 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); 313 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 314 } 315 316 /* Called within rcu_read_lock(). */ 317 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 318 { 319 VRingMemoryRegionCaches *caches; 320 hwaddr pa; 321 if (!vq->notification) { 322 return; 323 } 324 325 caches = vring_get_region_caches(vq); 326 pa = offsetof(VRingUsed, ring[vq->vring.num]); 327 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 328 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 329 } 330 331 void virtio_queue_set_notification(VirtQueue *vq, int enable) 332 { 333 vq->notification = enable; 334 335 if (!vq->vring.desc) { 336 return; 337 } 338 339 rcu_read_lock(); 340 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 341 vring_set_avail_event(vq, vring_avail_idx(vq)); 342 } else if (enable) { 343 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 344 } else { 345 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 346 } 347 if (enable) { 348 /* Expose avail event/used flags before caller checks the avail idx. */ 349 smp_mb(); 350 } 351 rcu_read_unlock(); 352 } 353 354 int virtio_queue_ready(VirtQueue *vq) 355 { 356 return vq->vring.avail != 0; 357 } 358 359 /* Fetch avail_idx from VQ memory only when we really need to know if 360 * guest has added some buffers. 361 * Called within rcu_read_lock(). */ 362 static int virtio_queue_empty_rcu(VirtQueue *vq) 363 { 364 if (unlikely(vq->vdev->broken)) { 365 return 1; 366 } 367 368 if (unlikely(!vq->vring.avail)) { 369 return 1; 370 } 371 372 if (vq->shadow_avail_idx != vq->last_avail_idx) { 373 return 0; 374 } 375 376 return vring_avail_idx(vq) == vq->last_avail_idx; 377 } 378 379 int virtio_queue_empty(VirtQueue *vq) 380 { 381 bool empty; 382 383 if (unlikely(vq->vdev->broken)) { 384 return 1; 385 } 386 387 if (unlikely(!vq->vring.avail)) { 388 return 1; 389 } 390 391 if (vq->shadow_avail_idx != vq->last_avail_idx) { 392 return 0; 393 } 394 395 rcu_read_lock(); 396 empty = vring_avail_idx(vq) == vq->last_avail_idx; 397 rcu_read_unlock(); 398 return empty; 399 } 400 401 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 402 unsigned int len) 403 { 404 AddressSpace *dma_as = vq->vdev->dma_as; 405 unsigned int offset; 406 int i; 407 408 offset = 0; 409 for (i = 0; i < elem->in_num; i++) { 410 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 411 412 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, 413 elem->in_sg[i].iov_len, 414 DMA_DIRECTION_FROM_DEVICE, size); 415 416 offset += size; 417 } 418 419 for (i = 0; i < elem->out_num; i++) 420 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, 421 elem->out_sg[i].iov_len, 422 DMA_DIRECTION_TO_DEVICE, 423 elem->out_sg[i].iov_len); 424 } 425 426 /* virtqueue_detach_element: 427 * @vq: The #VirtQueue 428 * @elem: The #VirtQueueElement 429 * @len: number of bytes written 430 * 431 * Detach the element from the virtqueue. This function is suitable for device 432 * reset or other situations where a #VirtQueueElement is simply freed and will 433 * not be pushed or discarded. 434 */ 435 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem, 436 unsigned int len) 437 { 438 vq->inuse--; 439 virtqueue_unmap_sg(vq, elem, len); 440 } 441 442 /* virtqueue_unpop: 443 * @vq: The #VirtQueue 444 * @elem: The #VirtQueueElement 445 * @len: number of bytes written 446 * 447 * Pretend the most recent element wasn't popped from the virtqueue. The next 448 * call to virtqueue_pop() will refetch the element. 449 */ 450 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem, 451 unsigned int len) 452 { 453 vq->last_avail_idx--; 454 virtqueue_detach_element(vq, elem, len); 455 } 456 457 /* virtqueue_rewind: 458 * @vq: The #VirtQueue 459 * @num: Number of elements to push back 460 * 461 * Pretend that elements weren't popped from the virtqueue. The next 462 * virtqueue_pop() will refetch the oldest element. 463 * 464 * Use virtqueue_unpop() instead if you have a VirtQueueElement. 465 * 466 * Returns: true on success, false if @num is greater than the number of in use 467 * elements. 468 */ 469 bool virtqueue_rewind(VirtQueue *vq, unsigned int num) 470 { 471 if (num > vq->inuse) { 472 return false; 473 } 474 vq->last_avail_idx -= num; 475 vq->inuse -= num; 476 return true; 477 } 478 479 /* Called within rcu_read_lock(). */ 480 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 481 unsigned int len, unsigned int idx) 482 { 483 VRingUsedElem uelem; 484 485 trace_virtqueue_fill(vq, elem, len, idx); 486 487 virtqueue_unmap_sg(vq, elem, len); 488 489 if (unlikely(vq->vdev->broken)) { 490 return; 491 } 492 493 if (unlikely(!vq->vring.used)) { 494 return; 495 } 496 497 idx = (idx + vq->used_idx) % vq->vring.num; 498 499 uelem.id = elem->index; 500 uelem.len = len; 501 vring_used_write(vq, &uelem, idx); 502 } 503 504 /* Called within rcu_read_lock(). */ 505 void virtqueue_flush(VirtQueue *vq, unsigned int count) 506 { 507 uint16_t old, new; 508 509 if (unlikely(vq->vdev->broken)) { 510 vq->inuse -= count; 511 return; 512 } 513 514 if (unlikely(!vq->vring.used)) { 515 return; 516 } 517 518 /* Make sure buffer is written before we update index. */ 519 smp_wmb(); 520 trace_virtqueue_flush(vq, count); 521 old = vq->used_idx; 522 new = old + count; 523 vring_used_idx_set(vq, new); 524 vq->inuse -= count; 525 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 526 vq->signalled_used_valid = false; 527 } 528 529 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 530 unsigned int len) 531 { 532 rcu_read_lock(); 533 virtqueue_fill(vq, elem, len, 0); 534 virtqueue_flush(vq, 1); 535 rcu_read_unlock(); 536 } 537 538 /* Called within rcu_read_lock(). */ 539 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 540 { 541 uint16_t num_heads = vring_avail_idx(vq) - idx; 542 543 /* Check it isn't doing very strange things with descriptor numbers. */ 544 if (num_heads > vq->vring.num) { 545 virtio_error(vq->vdev, "Guest moved used index from %u to %u", 546 idx, vq->shadow_avail_idx); 547 return -EINVAL; 548 } 549 /* On success, callers read a descriptor at vq->last_avail_idx. 550 * Make sure descriptor read does not bypass avail index read. */ 551 if (num_heads) { 552 smp_rmb(); 553 } 554 555 return num_heads; 556 } 557 558 /* Called within rcu_read_lock(). */ 559 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx, 560 unsigned int *head) 561 { 562 /* Grab the next descriptor number they're advertising, and increment 563 * the index we've seen. */ 564 *head = vring_avail_ring(vq, idx % vq->vring.num); 565 566 /* If their number is silly, that's a fatal mistake. */ 567 if (*head >= vq->vring.num) { 568 virtio_error(vq->vdev, "Guest says index %u is available", *head); 569 return false; 570 } 571 572 return true; 573 } 574 575 enum { 576 VIRTQUEUE_READ_DESC_ERROR = -1, 577 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ 578 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ 579 }; 580 581 static int virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, 582 MemoryRegionCache *desc_cache, unsigned int max, 583 unsigned int *next) 584 { 585 /* If this descriptor says it doesn't chain, we're done. */ 586 if (!(desc->flags & VRING_DESC_F_NEXT)) { 587 return VIRTQUEUE_READ_DESC_DONE; 588 } 589 590 /* Check they're not leading us off end of descriptors. */ 591 *next = desc->next; 592 /* Make sure compiler knows to grab that: we don't want it changing! */ 593 smp_wmb(); 594 595 if (*next >= max) { 596 virtio_error(vdev, "Desc next is %u", *next); 597 return VIRTQUEUE_READ_DESC_ERROR; 598 } 599 600 vring_desc_read(vdev, desc, desc_cache, *next); 601 return VIRTQUEUE_READ_DESC_MORE; 602 } 603 604 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 605 unsigned int *out_bytes, 606 unsigned max_in_bytes, unsigned max_out_bytes) 607 { 608 VirtIODevice *vdev = vq->vdev; 609 unsigned int max, idx; 610 unsigned int total_bufs, in_total, out_total; 611 VRingMemoryRegionCaches *caches; 612 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 613 int64_t len = 0; 614 int rc; 615 616 if (unlikely(!vq->vring.desc)) { 617 if (in_bytes) { 618 *in_bytes = 0; 619 } 620 if (out_bytes) { 621 *out_bytes = 0; 622 } 623 return; 624 } 625 626 rcu_read_lock(); 627 idx = vq->last_avail_idx; 628 total_bufs = in_total = out_total = 0; 629 630 max = vq->vring.num; 631 caches = vring_get_region_caches(vq); 632 if (caches->desc.len < max * sizeof(VRingDesc)) { 633 virtio_error(vdev, "Cannot map descriptor ring"); 634 goto err; 635 } 636 637 while ((rc = virtqueue_num_heads(vq, idx)) > 0) { 638 MemoryRegionCache *desc_cache = &caches->desc; 639 unsigned int num_bufs; 640 VRingDesc desc; 641 unsigned int i; 642 643 num_bufs = total_bufs; 644 645 if (!virtqueue_get_head(vq, idx++, &i)) { 646 goto err; 647 } 648 649 vring_desc_read(vdev, &desc, desc_cache, i); 650 651 if (desc.flags & VRING_DESC_F_INDIRECT) { 652 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 653 virtio_error(vdev, "Invalid size for indirect buffer table"); 654 goto err; 655 } 656 657 /* If we've got too many, that implies a descriptor loop. */ 658 if (num_bufs >= max) { 659 virtio_error(vdev, "Looped descriptor"); 660 goto err; 661 } 662 663 /* loop over the indirect descriptor table */ 664 len = address_space_cache_init(&indirect_desc_cache, 665 vdev->dma_as, 666 desc.addr, desc.len, false); 667 desc_cache = &indirect_desc_cache; 668 if (len < desc.len) { 669 virtio_error(vdev, "Cannot map indirect buffer"); 670 goto err; 671 } 672 673 max = desc.len / sizeof(VRingDesc); 674 num_bufs = i = 0; 675 vring_desc_read(vdev, &desc, desc_cache, i); 676 } 677 678 do { 679 /* If we've got too many, that implies a descriptor loop. */ 680 if (++num_bufs > max) { 681 virtio_error(vdev, "Looped descriptor"); 682 goto err; 683 } 684 685 if (desc.flags & VRING_DESC_F_WRITE) { 686 in_total += desc.len; 687 } else { 688 out_total += desc.len; 689 } 690 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 691 goto done; 692 } 693 694 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i); 695 } while (rc == VIRTQUEUE_READ_DESC_MORE); 696 697 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 698 goto err; 699 } 700 701 if (desc_cache == &indirect_desc_cache) { 702 address_space_cache_destroy(&indirect_desc_cache); 703 total_bufs++; 704 } else { 705 total_bufs = num_bufs; 706 } 707 } 708 709 if (rc < 0) { 710 goto err; 711 } 712 713 done: 714 address_space_cache_destroy(&indirect_desc_cache); 715 if (in_bytes) { 716 *in_bytes = in_total; 717 } 718 if (out_bytes) { 719 *out_bytes = out_total; 720 } 721 rcu_read_unlock(); 722 return; 723 724 err: 725 in_total = out_total = 0; 726 goto done; 727 } 728 729 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 730 unsigned int out_bytes) 731 { 732 unsigned int in_total, out_total; 733 734 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 735 return in_bytes <= in_total && out_bytes <= out_total; 736 } 737 738 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg, 739 hwaddr *addr, struct iovec *iov, 740 unsigned int max_num_sg, bool is_write, 741 hwaddr pa, size_t sz) 742 { 743 bool ok = false; 744 unsigned num_sg = *p_num_sg; 745 assert(num_sg <= max_num_sg); 746 747 if (!sz) { 748 virtio_error(vdev, "virtio: zero sized buffers are not allowed"); 749 goto out; 750 } 751 752 while (sz) { 753 hwaddr len = sz; 754 755 if (num_sg == max_num_sg) { 756 virtio_error(vdev, "virtio: too many write descriptors in " 757 "indirect table"); 758 goto out; 759 } 760 761 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, 762 is_write ? 763 DMA_DIRECTION_FROM_DEVICE : 764 DMA_DIRECTION_TO_DEVICE); 765 if (!iov[num_sg].iov_base) { 766 virtio_error(vdev, "virtio: bogus descriptor or out of resources"); 767 goto out; 768 } 769 770 iov[num_sg].iov_len = len; 771 addr[num_sg] = pa; 772 773 sz -= len; 774 pa += len; 775 num_sg++; 776 } 777 ok = true; 778 779 out: 780 *p_num_sg = num_sg; 781 return ok; 782 } 783 784 /* Only used by error code paths before we have a VirtQueueElement (therefore 785 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to 786 * yet. 787 */ 788 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, 789 struct iovec *iov) 790 { 791 unsigned int i; 792 793 for (i = 0; i < out_num + in_num; i++) { 794 int is_write = i >= out_num; 795 796 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); 797 iov++; 798 } 799 } 800 801 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg, 802 hwaddr *addr, unsigned int num_sg, 803 int is_write) 804 { 805 unsigned int i; 806 hwaddr len; 807 808 for (i = 0; i < num_sg; i++) { 809 len = sg[i].iov_len; 810 sg[i].iov_base = dma_memory_map(vdev->dma_as, 811 addr[i], &len, is_write ? 812 DMA_DIRECTION_FROM_DEVICE : 813 DMA_DIRECTION_TO_DEVICE); 814 if (!sg[i].iov_base) { 815 error_report("virtio: error trying to map MMIO memory"); 816 exit(1); 817 } 818 if (len != sg[i].iov_len) { 819 error_report("virtio: unexpected memory split"); 820 exit(1); 821 } 822 } 823 } 824 825 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem) 826 { 827 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, 1); 828 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, 0); 829 } 830 831 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) 832 { 833 VirtQueueElement *elem; 834 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); 835 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); 836 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); 837 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); 838 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); 839 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); 840 841 assert(sz >= sizeof(VirtQueueElement)); 842 elem = g_malloc(out_sg_end); 843 trace_virtqueue_alloc_element(elem, sz, in_num, out_num); 844 elem->out_num = out_num; 845 elem->in_num = in_num; 846 elem->in_addr = (void *)elem + in_addr_ofs; 847 elem->out_addr = (void *)elem + out_addr_ofs; 848 elem->in_sg = (void *)elem + in_sg_ofs; 849 elem->out_sg = (void *)elem + out_sg_ofs; 850 return elem; 851 } 852 853 void *virtqueue_pop(VirtQueue *vq, size_t sz) 854 { 855 unsigned int i, head, max; 856 VRingMemoryRegionCaches *caches; 857 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 858 MemoryRegionCache *desc_cache; 859 int64_t len; 860 VirtIODevice *vdev = vq->vdev; 861 VirtQueueElement *elem = NULL; 862 unsigned out_num, in_num, elem_entries; 863 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 864 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 865 VRingDesc desc; 866 int rc; 867 868 if (unlikely(vdev->broken)) { 869 return NULL; 870 } 871 rcu_read_lock(); 872 if (virtio_queue_empty_rcu(vq)) { 873 goto done; 874 } 875 /* Needed after virtio_queue_empty(), see comment in 876 * virtqueue_num_heads(). */ 877 smp_rmb(); 878 879 /* When we start there are none of either input nor output. */ 880 out_num = in_num = elem_entries = 0; 881 882 max = vq->vring.num; 883 884 if (vq->inuse >= vq->vring.num) { 885 virtio_error(vdev, "Virtqueue size exceeded"); 886 goto done; 887 } 888 889 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { 890 goto done; 891 } 892 893 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 894 vring_set_avail_event(vq, vq->last_avail_idx); 895 } 896 897 i = head; 898 899 caches = vring_get_region_caches(vq); 900 if (caches->desc.len < max * sizeof(VRingDesc)) { 901 virtio_error(vdev, "Cannot map descriptor ring"); 902 goto done; 903 } 904 905 desc_cache = &caches->desc; 906 vring_desc_read(vdev, &desc, desc_cache, i); 907 if (desc.flags & VRING_DESC_F_INDIRECT) { 908 if (!desc.len || (desc.len % sizeof(VRingDesc))) { 909 virtio_error(vdev, "Invalid size for indirect buffer table"); 910 goto done; 911 } 912 913 /* loop over the indirect descriptor table */ 914 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 915 desc.addr, desc.len, false); 916 desc_cache = &indirect_desc_cache; 917 if (len < desc.len) { 918 virtio_error(vdev, "Cannot map indirect buffer"); 919 goto done; 920 } 921 922 max = desc.len / sizeof(VRingDesc); 923 i = 0; 924 vring_desc_read(vdev, &desc, desc_cache, i); 925 } 926 927 /* Collect all the descriptors */ 928 do { 929 bool map_ok; 930 931 if (desc.flags & VRING_DESC_F_WRITE) { 932 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 933 iov + out_num, 934 VIRTQUEUE_MAX_SIZE - out_num, true, 935 desc.addr, desc.len); 936 } else { 937 if (in_num) { 938 virtio_error(vdev, "Incorrect order for descriptors"); 939 goto err_undo_map; 940 } 941 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 942 VIRTQUEUE_MAX_SIZE, false, 943 desc.addr, desc.len); 944 } 945 if (!map_ok) { 946 goto err_undo_map; 947 } 948 949 /* If we've got too many, that implies a descriptor loop. */ 950 if (++elem_entries > max) { 951 virtio_error(vdev, "Looped descriptor"); 952 goto err_undo_map; 953 } 954 955 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i); 956 } while (rc == VIRTQUEUE_READ_DESC_MORE); 957 958 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 959 goto err_undo_map; 960 } 961 962 /* Now copy what we have collected and mapped */ 963 elem = virtqueue_alloc_element(sz, out_num, in_num); 964 elem->index = head; 965 for (i = 0; i < out_num; i++) { 966 elem->out_addr[i] = addr[i]; 967 elem->out_sg[i] = iov[i]; 968 } 969 for (i = 0; i < in_num; i++) { 970 elem->in_addr[i] = addr[out_num + i]; 971 elem->in_sg[i] = iov[out_num + i]; 972 } 973 974 vq->inuse++; 975 976 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 977 done: 978 address_space_cache_destroy(&indirect_desc_cache); 979 rcu_read_unlock(); 980 981 return elem; 982 983 err_undo_map: 984 virtqueue_undo_map_desc(out_num, in_num, iov); 985 goto done; 986 } 987 988 /* virtqueue_drop_all: 989 * @vq: The #VirtQueue 990 * Drops all queued buffers and indicates them to the guest 991 * as if they are done. Useful when buffers can not be 992 * processed but must be returned to the guest. 993 */ 994 unsigned int virtqueue_drop_all(VirtQueue *vq) 995 { 996 unsigned int dropped = 0; 997 VirtQueueElement elem = {}; 998 VirtIODevice *vdev = vq->vdev; 999 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 1000 1001 if (unlikely(vdev->broken)) { 1002 return 0; 1003 } 1004 1005 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { 1006 /* works similar to virtqueue_pop but does not map buffers 1007 * and does not allocate any memory */ 1008 smp_rmb(); 1009 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { 1010 break; 1011 } 1012 vq->inuse++; 1013 vq->last_avail_idx++; 1014 if (fEventIdx) { 1015 vring_set_avail_event(vq, vq->last_avail_idx); 1016 } 1017 /* immediately push the element, nothing to unmap 1018 * as both in_num and out_num are set to 0 */ 1019 virtqueue_push(vq, &elem, 0); 1020 dropped++; 1021 } 1022 1023 return dropped; 1024 } 1025 1026 /* Reading and writing a structure directly to QEMUFile is *awful*, but 1027 * it is what QEMU has always done by mistake. We can change it sooner 1028 * or later by bumping the version number of the affected vm states. 1029 * In the meanwhile, since the in-memory layout of VirtQueueElement 1030 * has changed, we need to marshal to and from the layout that was 1031 * used before the change. 1032 */ 1033 typedef struct VirtQueueElementOld { 1034 unsigned int index; 1035 unsigned int out_num; 1036 unsigned int in_num; 1037 hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; 1038 hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; 1039 struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; 1040 struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; 1041 } VirtQueueElementOld; 1042 1043 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz) 1044 { 1045 VirtQueueElement *elem; 1046 VirtQueueElementOld data; 1047 int i; 1048 1049 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 1050 1051 /* TODO: teach all callers that this can fail, and return failure instead 1052 * of asserting here. 1053 * This is just one thing (there are probably more) that must be 1054 * fixed before we can allow NDEBUG compilation. 1055 */ 1056 assert(ARRAY_SIZE(data.in_addr) >= data.in_num); 1057 assert(ARRAY_SIZE(data.out_addr) >= data.out_num); 1058 1059 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); 1060 elem->index = data.index; 1061 1062 for (i = 0; i < elem->in_num; i++) { 1063 elem->in_addr[i] = data.in_addr[i]; 1064 } 1065 1066 for (i = 0; i < elem->out_num; i++) { 1067 elem->out_addr[i] = data.out_addr[i]; 1068 } 1069 1070 for (i = 0; i < elem->in_num; i++) { 1071 /* Base is overwritten by virtqueue_map. */ 1072 elem->in_sg[i].iov_base = 0; 1073 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; 1074 } 1075 1076 for (i = 0; i < elem->out_num; i++) { 1077 /* Base is overwritten by virtqueue_map. */ 1078 elem->out_sg[i].iov_base = 0; 1079 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; 1080 } 1081 1082 virtqueue_map(vdev, elem); 1083 return elem; 1084 } 1085 1086 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem) 1087 { 1088 VirtQueueElementOld data; 1089 int i; 1090 1091 memset(&data, 0, sizeof(data)); 1092 data.index = elem->index; 1093 data.in_num = elem->in_num; 1094 data.out_num = elem->out_num; 1095 1096 for (i = 0; i < elem->in_num; i++) { 1097 data.in_addr[i] = elem->in_addr[i]; 1098 } 1099 1100 for (i = 0; i < elem->out_num; i++) { 1101 data.out_addr[i] = elem->out_addr[i]; 1102 } 1103 1104 for (i = 0; i < elem->in_num; i++) { 1105 /* Base is overwritten by virtqueue_map when loading. Do not 1106 * save it, as it would leak the QEMU address space layout. */ 1107 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; 1108 } 1109 1110 for (i = 0; i < elem->out_num; i++) { 1111 /* Do not save iov_base as above. */ 1112 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; 1113 } 1114 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 1115 } 1116 1117 /* virtio device */ 1118 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 1119 { 1120 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1121 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1122 1123 if (unlikely(vdev->broken)) { 1124 return; 1125 } 1126 1127 if (k->notify) { 1128 k->notify(qbus->parent, vector); 1129 } 1130 } 1131 1132 void virtio_update_irq(VirtIODevice *vdev) 1133 { 1134 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 1135 } 1136 1137 static int virtio_validate_features(VirtIODevice *vdev) 1138 { 1139 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1140 1141 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) && 1142 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 1143 return -EFAULT; 1144 } 1145 1146 if (k->validate_features) { 1147 return k->validate_features(vdev); 1148 } else { 1149 return 0; 1150 } 1151 } 1152 1153 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 1154 { 1155 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1156 trace_virtio_set_status(vdev, val); 1157 1158 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1159 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 1160 val & VIRTIO_CONFIG_S_FEATURES_OK) { 1161 int ret = virtio_validate_features(vdev); 1162 1163 if (ret) { 1164 return ret; 1165 } 1166 } 1167 } 1168 1169 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != 1170 (val & VIRTIO_CONFIG_S_DRIVER_OK)) { 1171 virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK); 1172 } 1173 1174 if (k->set_status) { 1175 k->set_status(vdev, val); 1176 } 1177 vdev->status = val; 1178 1179 return 0; 1180 } 1181 1182 static enum virtio_device_endian virtio_default_endian(void) 1183 { 1184 if (target_words_bigendian()) { 1185 return VIRTIO_DEVICE_ENDIAN_BIG; 1186 } else { 1187 return VIRTIO_DEVICE_ENDIAN_LITTLE; 1188 } 1189 } 1190 1191 static enum virtio_device_endian virtio_current_cpu_endian(void) 1192 { 1193 CPUClass *cc = CPU_GET_CLASS(current_cpu); 1194 1195 if (cc->virtio_is_big_endian(current_cpu)) { 1196 return VIRTIO_DEVICE_ENDIAN_BIG; 1197 } else { 1198 return VIRTIO_DEVICE_ENDIAN_LITTLE; 1199 } 1200 } 1201 1202 void virtio_reset(void *opaque) 1203 { 1204 VirtIODevice *vdev = opaque; 1205 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1206 int i; 1207 1208 virtio_set_status(vdev, 0); 1209 if (current_cpu) { 1210 /* Guest initiated reset */ 1211 vdev->device_endian = virtio_current_cpu_endian(); 1212 } else { 1213 /* System reset */ 1214 vdev->device_endian = virtio_default_endian(); 1215 } 1216 1217 if (k->reset) { 1218 k->reset(vdev); 1219 } 1220 1221 vdev->start_on_kick = false; 1222 vdev->started = false; 1223 vdev->broken = false; 1224 vdev->guest_features = 0; 1225 vdev->queue_sel = 0; 1226 vdev->status = 0; 1227 atomic_set(&vdev->isr, 0); 1228 vdev->config_vector = VIRTIO_NO_VECTOR; 1229 virtio_notify_vector(vdev, vdev->config_vector); 1230 1231 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1232 vdev->vq[i].vring.desc = 0; 1233 vdev->vq[i].vring.avail = 0; 1234 vdev->vq[i].vring.used = 0; 1235 vdev->vq[i].last_avail_idx = 0; 1236 vdev->vq[i].shadow_avail_idx = 0; 1237 vdev->vq[i].used_idx = 0; 1238 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 1239 vdev->vq[i].signalled_used = 0; 1240 vdev->vq[i].signalled_used_valid = false; 1241 vdev->vq[i].notification = true; 1242 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 1243 vdev->vq[i].inuse = 0; 1244 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 1245 } 1246 } 1247 1248 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 1249 { 1250 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1251 uint8_t val; 1252 1253 if (addr + sizeof(val) > vdev->config_len) { 1254 return (uint32_t)-1; 1255 } 1256 1257 k->get_config(vdev, vdev->config); 1258 1259 val = ldub_p(vdev->config + addr); 1260 return val; 1261 } 1262 1263 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 1264 { 1265 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1266 uint16_t val; 1267 1268 if (addr + sizeof(val) > vdev->config_len) { 1269 return (uint32_t)-1; 1270 } 1271 1272 k->get_config(vdev, vdev->config); 1273 1274 val = lduw_p(vdev->config + addr); 1275 return val; 1276 } 1277 1278 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 1279 { 1280 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1281 uint32_t val; 1282 1283 if (addr + sizeof(val) > vdev->config_len) { 1284 return (uint32_t)-1; 1285 } 1286 1287 k->get_config(vdev, vdev->config); 1288 1289 val = ldl_p(vdev->config + addr); 1290 return val; 1291 } 1292 1293 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 1294 { 1295 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1296 uint8_t val = data; 1297 1298 if (addr + sizeof(val) > vdev->config_len) { 1299 return; 1300 } 1301 1302 stb_p(vdev->config + addr, val); 1303 1304 if (k->set_config) { 1305 k->set_config(vdev, vdev->config); 1306 } 1307 } 1308 1309 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 1310 { 1311 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1312 uint16_t val = data; 1313 1314 if (addr + sizeof(val) > vdev->config_len) { 1315 return; 1316 } 1317 1318 stw_p(vdev->config + addr, val); 1319 1320 if (k->set_config) { 1321 k->set_config(vdev, vdev->config); 1322 } 1323 } 1324 1325 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 1326 { 1327 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1328 uint32_t val = data; 1329 1330 if (addr + sizeof(val) > vdev->config_len) { 1331 return; 1332 } 1333 1334 stl_p(vdev->config + addr, val); 1335 1336 if (k->set_config) { 1337 k->set_config(vdev, vdev->config); 1338 } 1339 } 1340 1341 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 1342 { 1343 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1344 uint8_t val; 1345 1346 if (addr + sizeof(val) > vdev->config_len) { 1347 return (uint32_t)-1; 1348 } 1349 1350 k->get_config(vdev, vdev->config); 1351 1352 val = ldub_p(vdev->config + addr); 1353 return val; 1354 } 1355 1356 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 1357 { 1358 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1359 uint16_t val; 1360 1361 if (addr + sizeof(val) > vdev->config_len) { 1362 return (uint32_t)-1; 1363 } 1364 1365 k->get_config(vdev, vdev->config); 1366 1367 val = lduw_le_p(vdev->config + addr); 1368 return val; 1369 } 1370 1371 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 1372 { 1373 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1374 uint32_t val; 1375 1376 if (addr + sizeof(val) > vdev->config_len) { 1377 return (uint32_t)-1; 1378 } 1379 1380 k->get_config(vdev, vdev->config); 1381 1382 val = ldl_le_p(vdev->config + addr); 1383 return val; 1384 } 1385 1386 void virtio_config_modern_writeb(VirtIODevice *vdev, 1387 uint32_t addr, uint32_t data) 1388 { 1389 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1390 uint8_t val = data; 1391 1392 if (addr + sizeof(val) > vdev->config_len) { 1393 return; 1394 } 1395 1396 stb_p(vdev->config + addr, val); 1397 1398 if (k->set_config) { 1399 k->set_config(vdev, vdev->config); 1400 } 1401 } 1402 1403 void virtio_config_modern_writew(VirtIODevice *vdev, 1404 uint32_t addr, uint32_t data) 1405 { 1406 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1407 uint16_t val = data; 1408 1409 if (addr + sizeof(val) > vdev->config_len) { 1410 return; 1411 } 1412 1413 stw_le_p(vdev->config + addr, val); 1414 1415 if (k->set_config) { 1416 k->set_config(vdev, vdev->config); 1417 } 1418 } 1419 1420 void virtio_config_modern_writel(VirtIODevice *vdev, 1421 uint32_t addr, uint32_t data) 1422 { 1423 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1424 uint32_t val = data; 1425 1426 if (addr + sizeof(val) > vdev->config_len) { 1427 return; 1428 } 1429 1430 stl_le_p(vdev->config + addr, val); 1431 1432 if (k->set_config) { 1433 k->set_config(vdev, vdev->config); 1434 } 1435 } 1436 1437 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 1438 { 1439 if (!vdev->vq[n].vring.num) { 1440 return; 1441 } 1442 vdev->vq[n].vring.desc = addr; 1443 virtio_queue_update_rings(vdev, n); 1444 } 1445 1446 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 1447 { 1448 return vdev->vq[n].vring.desc; 1449 } 1450 1451 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 1452 hwaddr avail, hwaddr used) 1453 { 1454 if (!vdev->vq[n].vring.num) { 1455 return; 1456 } 1457 vdev->vq[n].vring.desc = desc; 1458 vdev->vq[n].vring.avail = avail; 1459 vdev->vq[n].vring.used = used; 1460 virtio_init_region_cache(vdev, n); 1461 } 1462 1463 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 1464 { 1465 /* Don't allow guest to flip queue between existent and 1466 * nonexistent states, or to set it to an invalid size. 1467 */ 1468 if (!!num != !!vdev->vq[n].vring.num || 1469 num > VIRTQUEUE_MAX_SIZE || 1470 num < 0) { 1471 return; 1472 } 1473 vdev->vq[n].vring.num = num; 1474 } 1475 1476 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 1477 { 1478 return QLIST_FIRST(&vdev->vector_queues[vector]); 1479 } 1480 1481 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 1482 { 1483 return QLIST_NEXT(vq, node); 1484 } 1485 1486 int virtio_queue_get_num(VirtIODevice *vdev, int n) 1487 { 1488 return vdev->vq[n].vring.num; 1489 } 1490 1491 int virtio_queue_get_max_num(VirtIODevice *vdev, int n) 1492 { 1493 return vdev->vq[n].vring.num_default; 1494 } 1495 1496 int virtio_get_num_queues(VirtIODevice *vdev) 1497 { 1498 int i; 1499 1500 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1501 if (!virtio_queue_get_num(vdev, i)) { 1502 break; 1503 } 1504 } 1505 1506 return i; 1507 } 1508 1509 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 1510 { 1511 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1512 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1513 1514 /* virtio-1 compliant devices cannot change the alignment */ 1515 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1516 error_report("tried to modify queue alignment for virtio-1 device"); 1517 return; 1518 } 1519 /* Check that the transport told us it was going to do this 1520 * (so a buggy transport will immediately assert rather than 1521 * silently failing to migrate this state) 1522 */ 1523 assert(k->has_variable_vring_alignment); 1524 1525 if (align) { 1526 vdev->vq[n].vring.align = align; 1527 virtio_queue_update_rings(vdev, n); 1528 } 1529 } 1530 1531 static bool virtio_queue_notify_aio_vq(VirtQueue *vq) 1532 { 1533 bool ret = false; 1534 1535 if (vq->vring.desc && vq->handle_aio_output) { 1536 VirtIODevice *vdev = vq->vdev; 1537 1538 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1539 ret = vq->handle_aio_output(vdev, vq); 1540 1541 if (unlikely(vdev->start_on_kick)) { 1542 virtio_set_started(vdev, true); 1543 } 1544 } 1545 1546 return ret; 1547 } 1548 1549 static void virtio_queue_notify_vq(VirtQueue *vq) 1550 { 1551 if (vq->vring.desc && vq->handle_output) { 1552 VirtIODevice *vdev = vq->vdev; 1553 1554 if (unlikely(vdev->broken)) { 1555 return; 1556 } 1557 1558 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1559 vq->handle_output(vdev, vq); 1560 1561 if (unlikely(vdev->start_on_kick)) { 1562 virtio_set_started(vdev, true); 1563 } 1564 } 1565 } 1566 1567 void virtio_queue_notify(VirtIODevice *vdev, int n) 1568 { 1569 VirtQueue *vq = &vdev->vq[n]; 1570 1571 if (unlikely(!vq->vring.desc || vdev->broken)) { 1572 return; 1573 } 1574 1575 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1576 if (vq->handle_aio_output) { 1577 event_notifier_set(&vq->host_notifier); 1578 } else if (vq->handle_output) { 1579 vq->handle_output(vdev, vq); 1580 1581 if (unlikely(vdev->start_on_kick)) { 1582 virtio_set_started(vdev, true); 1583 } 1584 } 1585 } 1586 1587 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 1588 { 1589 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 1590 VIRTIO_NO_VECTOR; 1591 } 1592 1593 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 1594 { 1595 VirtQueue *vq = &vdev->vq[n]; 1596 1597 if (n < VIRTIO_QUEUE_MAX) { 1598 if (vdev->vector_queues && 1599 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 1600 QLIST_REMOVE(vq, node); 1601 } 1602 vdev->vq[n].vector = vector; 1603 if (vdev->vector_queues && 1604 vector != VIRTIO_NO_VECTOR) { 1605 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 1606 } 1607 } 1608 } 1609 1610 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 1611 VirtIOHandleOutput handle_output) 1612 { 1613 int i; 1614 1615 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1616 if (vdev->vq[i].vring.num == 0) 1617 break; 1618 } 1619 1620 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 1621 abort(); 1622 1623 vdev->vq[i].vring.num = queue_size; 1624 vdev->vq[i].vring.num_default = queue_size; 1625 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 1626 vdev->vq[i].handle_output = handle_output; 1627 vdev->vq[i].handle_aio_output = NULL; 1628 1629 return &vdev->vq[i]; 1630 } 1631 1632 void virtio_del_queue(VirtIODevice *vdev, int n) 1633 { 1634 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 1635 abort(); 1636 } 1637 1638 vdev->vq[n].vring.num = 0; 1639 vdev->vq[n].vring.num_default = 0; 1640 vdev->vq[n].handle_output = NULL; 1641 vdev->vq[n].handle_aio_output = NULL; 1642 } 1643 1644 static void virtio_set_isr(VirtIODevice *vdev, int value) 1645 { 1646 uint8_t old = atomic_read(&vdev->isr); 1647 1648 /* Do not write ISR if it does not change, so that its cacheline remains 1649 * shared in the common case where the guest does not read it. 1650 */ 1651 if ((old & value) != value) { 1652 atomic_or(&vdev->isr, value); 1653 } 1654 } 1655 1656 /* Called within rcu_read_lock(). */ 1657 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) 1658 { 1659 uint16_t old, new; 1660 bool v; 1661 /* We need to expose used array entries before checking used event. */ 1662 smp_mb(); 1663 /* Always notify when queue is empty (when feature acknowledge) */ 1664 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 1665 !vq->inuse && virtio_queue_empty(vq)) { 1666 return true; 1667 } 1668 1669 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1670 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 1671 } 1672 1673 v = vq->signalled_used_valid; 1674 vq->signalled_used_valid = true; 1675 old = vq->signalled_used; 1676 new = vq->signalled_used = vq->used_idx; 1677 return !v || vring_need_event(vring_get_used_event(vq), new, old); 1678 } 1679 1680 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq) 1681 { 1682 bool should_notify; 1683 rcu_read_lock(); 1684 should_notify = virtio_should_notify(vdev, vq); 1685 rcu_read_unlock(); 1686 1687 if (!should_notify) { 1688 return; 1689 } 1690 1691 trace_virtio_notify_irqfd(vdev, vq); 1692 1693 /* 1694 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but 1695 * windows drivers included in virtio-win 1.8.0 (circa 2015) are 1696 * incorrectly polling this bit during crashdump and hibernation 1697 * in MSI mode, causing a hang if this bit is never updated. 1698 * Recent releases of Windows do not really shut down, but rather 1699 * log out and hibernate to make the next startup faster. Hence, 1700 * this manifested as a more serious hang during shutdown with 1701 * 1702 * Next driver release from 2016 fixed this problem, so working around it 1703 * is not a must, but it's easy to do so let's do it here. 1704 * 1705 * Note: it's safe to update ISR from any thread as it was switched 1706 * to an atomic operation. 1707 */ 1708 virtio_set_isr(vq->vdev, 0x1); 1709 event_notifier_set(&vq->guest_notifier); 1710 } 1711 1712 static void virtio_irq(VirtQueue *vq) 1713 { 1714 virtio_set_isr(vq->vdev, 0x1); 1715 virtio_notify_vector(vq->vdev, vq->vector); 1716 } 1717 1718 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 1719 { 1720 bool should_notify; 1721 rcu_read_lock(); 1722 should_notify = virtio_should_notify(vdev, vq); 1723 rcu_read_unlock(); 1724 1725 if (!should_notify) { 1726 return; 1727 } 1728 1729 trace_virtio_notify(vdev, vq); 1730 virtio_irq(vq); 1731 } 1732 1733 void virtio_notify_config(VirtIODevice *vdev) 1734 { 1735 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1736 return; 1737 1738 virtio_set_isr(vdev, 0x3); 1739 vdev->generation++; 1740 virtio_notify_vector(vdev, vdev->config_vector); 1741 } 1742 1743 static bool virtio_device_endian_needed(void *opaque) 1744 { 1745 VirtIODevice *vdev = opaque; 1746 1747 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 1748 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1749 return vdev->device_endian != virtio_default_endian(); 1750 } 1751 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 1752 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 1753 } 1754 1755 static bool virtio_64bit_features_needed(void *opaque) 1756 { 1757 VirtIODevice *vdev = opaque; 1758 1759 return (vdev->host_features >> 32) != 0; 1760 } 1761 1762 static bool virtio_virtqueue_needed(void *opaque) 1763 { 1764 VirtIODevice *vdev = opaque; 1765 1766 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 1767 } 1768 1769 static bool virtio_ringsize_needed(void *opaque) 1770 { 1771 VirtIODevice *vdev = opaque; 1772 int i; 1773 1774 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1775 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 1776 return true; 1777 } 1778 } 1779 return false; 1780 } 1781 1782 static bool virtio_extra_state_needed(void *opaque) 1783 { 1784 VirtIODevice *vdev = opaque; 1785 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1786 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1787 1788 return k->has_extra_state && 1789 k->has_extra_state(qbus->parent); 1790 } 1791 1792 static bool virtio_broken_needed(void *opaque) 1793 { 1794 VirtIODevice *vdev = opaque; 1795 1796 return vdev->broken; 1797 } 1798 1799 static bool virtio_started_needed(void *opaque) 1800 { 1801 VirtIODevice *vdev = opaque; 1802 1803 return vdev->started; 1804 } 1805 1806 static const VMStateDescription vmstate_virtqueue = { 1807 .name = "virtqueue_state", 1808 .version_id = 1, 1809 .minimum_version_id = 1, 1810 .fields = (VMStateField[]) { 1811 VMSTATE_UINT64(vring.avail, struct VirtQueue), 1812 VMSTATE_UINT64(vring.used, struct VirtQueue), 1813 VMSTATE_END_OF_LIST() 1814 } 1815 }; 1816 1817 static const VMStateDescription vmstate_virtio_virtqueues = { 1818 .name = "virtio/virtqueues", 1819 .version_id = 1, 1820 .minimum_version_id = 1, 1821 .needed = &virtio_virtqueue_needed, 1822 .fields = (VMStateField[]) { 1823 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 1824 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), 1825 VMSTATE_END_OF_LIST() 1826 } 1827 }; 1828 1829 static const VMStateDescription vmstate_ringsize = { 1830 .name = "ringsize_state", 1831 .version_id = 1, 1832 .minimum_version_id = 1, 1833 .fields = (VMStateField[]) { 1834 VMSTATE_UINT32(vring.num_default, struct VirtQueue), 1835 VMSTATE_END_OF_LIST() 1836 } 1837 }; 1838 1839 static const VMStateDescription vmstate_virtio_ringsize = { 1840 .name = "virtio/ringsize", 1841 .version_id = 1, 1842 .minimum_version_id = 1, 1843 .needed = &virtio_ringsize_needed, 1844 .fields = (VMStateField[]) { 1845 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 1846 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), 1847 VMSTATE_END_OF_LIST() 1848 } 1849 }; 1850 1851 static int get_extra_state(QEMUFile *f, void *pv, size_t size, 1852 const VMStateField *field) 1853 { 1854 VirtIODevice *vdev = pv; 1855 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1856 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1857 1858 if (!k->load_extra_state) { 1859 return -1; 1860 } else { 1861 return k->load_extra_state(qbus->parent, f); 1862 } 1863 } 1864 1865 static int put_extra_state(QEMUFile *f, void *pv, size_t size, 1866 const VMStateField *field, QJSON *vmdesc) 1867 { 1868 VirtIODevice *vdev = pv; 1869 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1870 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1871 1872 k->save_extra_state(qbus->parent, f); 1873 return 0; 1874 } 1875 1876 static const VMStateInfo vmstate_info_extra_state = { 1877 .name = "virtqueue_extra_state", 1878 .get = get_extra_state, 1879 .put = put_extra_state, 1880 }; 1881 1882 static const VMStateDescription vmstate_virtio_extra_state = { 1883 .name = "virtio/extra_state", 1884 .version_id = 1, 1885 .minimum_version_id = 1, 1886 .needed = &virtio_extra_state_needed, 1887 .fields = (VMStateField[]) { 1888 { 1889 .name = "extra_state", 1890 .version_id = 0, 1891 .field_exists = NULL, 1892 .size = 0, 1893 .info = &vmstate_info_extra_state, 1894 .flags = VMS_SINGLE, 1895 .offset = 0, 1896 }, 1897 VMSTATE_END_OF_LIST() 1898 } 1899 }; 1900 1901 static const VMStateDescription vmstate_virtio_device_endian = { 1902 .name = "virtio/device_endian", 1903 .version_id = 1, 1904 .minimum_version_id = 1, 1905 .needed = &virtio_device_endian_needed, 1906 .fields = (VMStateField[]) { 1907 VMSTATE_UINT8(device_endian, VirtIODevice), 1908 VMSTATE_END_OF_LIST() 1909 } 1910 }; 1911 1912 static const VMStateDescription vmstate_virtio_64bit_features = { 1913 .name = "virtio/64bit_features", 1914 .version_id = 1, 1915 .minimum_version_id = 1, 1916 .needed = &virtio_64bit_features_needed, 1917 .fields = (VMStateField[]) { 1918 VMSTATE_UINT64(guest_features, VirtIODevice), 1919 VMSTATE_END_OF_LIST() 1920 } 1921 }; 1922 1923 static const VMStateDescription vmstate_virtio_broken = { 1924 .name = "virtio/broken", 1925 .version_id = 1, 1926 .minimum_version_id = 1, 1927 .needed = &virtio_broken_needed, 1928 .fields = (VMStateField[]) { 1929 VMSTATE_BOOL(broken, VirtIODevice), 1930 VMSTATE_END_OF_LIST() 1931 } 1932 }; 1933 1934 static const VMStateDescription vmstate_virtio_started = { 1935 .name = "virtio/started", 1936 .version_id = 1, 1937 .minimum_version_id = 1, 1938 .needed = &virtio_started_needed, 1939 .fields = (VMStateField[]) { 1940 VMSTATE_BOOL(started, VirtIODevice), 1941 VMSTATE_END_OF_LIST() 1942 } 1943 }; 1944 1945 static const VMStateDescription vmstate_virtio = { 1946 .name = "virtio", 1947 .version_id = 1, 1948 .minimum_version_id = 1, 1949 .minimum_version_id_old = 1, 1950 .fields = (VMStateField[]) { 1951 VMSTATE_END_OF_LIST() 1952 }, 1953 .subsections = (const VMStateDescription*[]) { 1954 &vmstate_virtio_device_endian, 1955 &vmstate_virtio_64bit_features, 1956 &vmstate_virtio_virtqueues, 1957 &vmstate_virtio_ringsize, 1958 &vmstate_virtio_broken, 1959 &vmstate_virtio_extra_state, 1960 &vmstate_virtio_started, 1961 NULL 1962 } 1963 }; 1964 1965 int virtio_save(VirtIODevice *vdev, QEMUFile *f) 1966 { 1967 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1968 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1969 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1970 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 1971 int i; 1972 1973 if (k->save_config) { 1974 k->save_config(qbus->parent, f); 1975 } 1976 1977 qemu_put_8s(f, &vdev->status); 1978 qemu_put_8s(f, &vdev->isr); 1979 qemu_put_be16s(f, &vdev->queue_sel); 1980 qemu_put_be32s(f, &guest_features_lo); 1981 qemu_put_be32(f, vdev->config_len); 1982 qemu_put_buffer(f, vdev->config, vdev->config_len); 1983 1984 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1985 if (vdev->vq[i].vring.num == 0) 1986 break; 1987 } 1988 1989 qemu_put_be32(f, i); 1990 1991 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1992 if (vdev->vq[i].vring.num == 0) 1993 break; 1994 1995 qemu_put_be32(f, vdev->vq[i].vring.num); 1996 if (k->has_variable_vring_alignment) { 1997 qemu_put_be32(f, vdev->vq[i].vring.align); 1998 } 1999 /* 2000 * Save desc now, the rest of the ring addresses are saved in 2001 * subsections for VIRTIO-1 devices. 2002 */ 2003 qemu_put_be64(f, vdev->vq[i].vring.desc); 2004 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 2005 if (k->save_queue) { 2006 k->save_queue(qbus->parent, i, f); 2007 } 2008 } 2009 2010 if (vdc->save != NULL) { 2011 vdc->save(vdev, f); 2012 } 2013 2014 if (vdc->vmsd) { 2015 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL); 2016 if (ret) { 2017 return ret; 2018 } 2019 } 2020 2021 /* Subsections */ 2022 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 2023 } 2024 2025 /* A wrapper for use as a VMState .put function */ 2026 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, 2027 const VMStateField *field, QJSON *vmdesc) 2028 { 2029 return virtio_save(VIRTIO_DEVICE(opaque), f); 2030 } 2031 2032 /* A wrapper for use as a VMState .get function */ 2033 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size, 2034 const VMStateField *field) 2035 { 2036 VirtIODevice *vdev = VIRTIO_DEVICE(opaque); 2037 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev)); 2038 2039 return virtio_load(vdev, f, dc->vmsd->version_id); 2040 } 2041 2042 const VMStateInfo virtio_vmstate_info = { 2043 .name = "virtio", 2044 .get = virtio_device_get, 2045 .put = virtio_device_put, 2046 }; 2047 2048 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 2049 { 2050 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 2051 bool bad = (val & ~(vdev->host_features)) != 0; 2052 2053 val &= vdev->host_features; 2054 if (k->set_features) { 2055 k->set_features(vdev, val); 2056 } 2057 vdev->guest_features = val; 2058 return bad ? -1 : 0; 2059 } 2060 2061 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 2062 { 2063 int ret; 2064 /* 2065 * The driver must not attempt to set features after feature negotiation 2066 * has finished. 2067 */ 2068 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 2069 return -EINVAL; 2070 } 2071 ret = virtio_set_features_nocheck(vdev, val); 2072 if (!ret) { 2073 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 2074 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */ 2075 int i; 2076 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2077 if (vdev->vq[i].vring.num != 0) { 2078 virtio_init_region_cache(vdev, i); 2079 } 2080 } 2081 } 2082 2083 if (!virtio_device_started(vdev, vdev->status) && 2084 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2085 vdev->start_on_kick = true; 2086 } 2087 } 2088 return ret; 2089 } 2090 2091 size_t virtio_feature_get_config_size(VirtIOFeature *feature_sizes, 2092 uint64_t host_features) 2093 { 2094 size_t config_size = 0; 2095 int i; 2096 2097 for (i = 0; feature_sizes[i].flags != 0; i++) { 2098 if (host_features & feature_sizes[i].flags) { 2099 config_size = MAX(feature_sizes[i].end, config_size); 2100 } 2101 } 2102 2103 return config_size; 2104 } 2105 2106 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 2107 { 2108 int i, ret; 2109 int32_t config_len; 2110 uint32_t num; 2111 uint32_t features; 2112 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2113 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2114 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 2115 2116 /* 2117 * We poison the endianness to ensure it does not get used before 2118 * subsections have been loaded. 2119 */ 2120 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 2121 2122 if (k->load_config) { 2123 ret = k->load_config(qbus->parent, f); 2124 if (ret) 2125 return ret; 2126 } 2127 2128 qemu_get_8s(f, &vdev->status); 2129 qemu_get_8s(f, &vdev->isr); 2130 qemu_get_be16s(f, &vdev->queue_sel); 2131 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 2132 return -1; 2133 } 2134 qemu_get_be32s(f, &features); 2135 2136 /* 2137 * Temporarily set guest_features low bits - needed by 2138 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 2139 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. 2140 * 2141 * Note: devices should always test host features in future - don't create 2142 * new dependencies like this. 2143 */ 2144 vdev->guest_features = features; 2145 2146 config_len = qemu_get_be32(f); 2147 2148 /* 2149 * There are cases where the incoming config can be bigger or smaller 2150 * than what we have; so load what we have space for, and skip 2151 * any excess that's in the stream. 2152 */ 2153 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 2154 2155 while (config_len > vdev->config_len) { 2156 qemu_get_byte(f); 2157 config_len--; 2158 } 2159 2160 num = qemu_get_be32(f); 2161 2162 if (num > VIRTIO_QUEUE_MAX) { 2163 error_report("Invalid number of virtqueues: 0x%x", num); 2164 return -1; 2165 } 2166 2167 for (i = 0; i < num; i++) { 2168 vdev->vq[i].vring.num = qemu_get_be32(f); 2169 if (k->has_variable_vring_alignment) { 2170 vdev->vq[i].vring.align = qemu_get_be32(f); 2171 } 2172 vdev->vq[i].vring.desc = qemu_get_be64(f); 2173 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 2174 vdev->vq[i].signalled_used_valid = false; 2175 vdev->vq[i].notification = true; 2176 2177 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { 2178 error_report("VQ %d address 0x0 " 2179 "inconsistent with Host index 0x%x", 2180 i, vdev->vq[i].last_avail_idx); 2181 return -1; 2182 } 2183 if (k->load_queue) { 2184 ret = k->load_queue(qbus->parent, i, f); 2185 if (ret) 2186 return ret; 2187 } 2188 } 2189 2190 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 2191 2192 if (vdc->load != NULL) { 2193 ret = vdc->load(vdev, f, version_id); 2194 if (ret) { 2195 return ret; 2196 } 2197 } 2198 2199 if (vdc->vmsd) { 2200 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); 2201 if (ret) { 2202 return ret; 2203 } 2204 } 2205 2206 /* Subsections */ 2207 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 2208 if (ret) { 2209 return ret; 2210 } 2211 2212 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 2213 vdev->device_endian = virtio_default_endian(); 2214 } 2215 2216 if (virtio_64bit_features_needed(vdev)) { 2217 /* 2218 * Subsection load filled vdev->guest_features. Run them 2219 * through virtio_set_features to sanity-check them against 2220 * host_features. 2221 */ 2222 uint64_t features64 = vdev->guest_features; 2223 if (virtio_set_features_nocheck(vdev, features64) < 0) { 2224 error_report("Features 0x%" PRIx64 " unsupported. " 2225 "Allowed features: 0x%" PRIx64, 2226 features64, vdev->host_features); 2227 return -1; 2228 } 2229 } else { 2230 if (virtio_set_features_nocheck(vdev, features) < 0) { 2231 error_report("Features 0x%x unsupported. " 2232 "Allowed features: 0x%" PRIx64, 2233 features, vdev->host_features); 2234 return -1; 2235 } 2236 } 2237 2238 if (!virtio_device_started(vdev, vdev->status) && 2239 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2240 vdev->start_on_kick = true; 2241 } 2242 2243 rcu_read_lock(); 2244 for (i = 0; i < num; i++) { 2245 if (vdev->vq[i].vring.desc) { 2246 uint16_t nheads; 2247 2248 /* 2249 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so 2250 * only the region cache needs to be set up. Legacy devices need 2251 * to calculate used and avail ring addresses based on the desc 2252 * address. 2253 */ 2254 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2255 virtio_init_region_cache(vdev, i); 2256 } else { 2257 virtio_queue_update_rings(vdev, i); 2258 } 2259 2260 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 2261 /* Check it isn't doing strange things with descriptor numbers. */ 2262 if (nheads > vdev->vq[i].vring.num) { 2263 error_report("VQ %d size 0x%x Guest index 0x%x " 2264 "inconsistent with Host index 0x%x: delta 0x%x", 2265 i, vdev->vq[i].vring.num, 2266 vring_avail_idx(&vdev->vq[i]), 2267 vdev->vq[i].last_avail_idx, nheads); 2268 return -1; 2269 } 2270 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); 2271 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); 2272 2273 /* 2274 * Some devices migrate VirtQueueElements that have been popped 2275 * from the avail ring but not yet returned to the used ring. 2276 * Since max ring size < UINT16_MAX it's safe to use modulo 2277 * UINT16_MAX + 1 subtraction. 2278 */ 2279 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - 2280 vdev->vq[i].used_idx); 2281 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { 2282 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " 2283 "used_idx 0x%x", 2284 i, vdev->vq[i].vring.num, 2285 vdev->vq[i].last_avail_idx, 2286 vdev->vq[i].used_idx); 2287 return -1; 2288 } 2289 } 2290 } 2291 rcu_read_unlock(); 2292 2293 return 0; 2294 } 2295 2296 void virtio_cleanup(VirtIODevice *vdev) 2297 { 2298 qemu_del_vm_change_state_handler(vdev->vmstate); 2299 } 2300 2301 static void virtio_vmstate_change(void *opaque, int running, RunState state) 2302 { 2303 VirtIODevice *vdev = opaque; 2304 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2305 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2306 bool backend_run = running && virtio_device_started(vdev, vdev->status); 2307 vdev->vm_running = running; 2308 2309 if (backend_run) { 2310 virtio_set_status(vdev, vdev->status); 2311 } 2312 2313 if (k->vmstate_change) { 2314 k->vmstate_change(qbus->parent, backend_run); 2315 } 2316 2317 if (!backend_run) { 2318 virtio_set_status(vdev, vdev->status); 2319 } 2320 } 2321 2322 void virtio_instance_init_common(Object *proxy_obj, void *data, 2323 size_t vdev_size, const char *vdev_name) 2324 { 2325 DeviceState *vdev = data; 2326 2327 object_initialize_child(proxy_obj, "virtio-backend", vdev, vdev_size, 2328 vdev_name, &error_abort, NULL); 2329 qdev_alias_all_properties(vdev, proxy_obj); 2330 } 2331 2332 void virtio_init(VirtIODevice *vdev, const char *name, 2333 uint16_t device_id, size_t config_size) 2334 { 2335 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2336 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2337 int i; 2338 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 2339 2340 if (nvectors) { 2341 vdev->vector_queues = 2342 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 2343 } 2344 2345 vdev->start_on_kick = false; 2346 vdev->started = false; 2347 vdev->device_id = device_id; 2348 vdev->status = 0; 2349 atomic_set(&vdev->isr, 0); 2350 vdev->queue_sel = 0; 2351 vdev->config_vector = VIRTIO_NO_VECTOR; 2352 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX); 2353 vdev->vm_running = runstate_is_running(); 2354 vdev->broken = false; 2355 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2356 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 2357 vdev->vq[i].vdev = vdev; 2358 vdev->vq[i].queue_index = i; 2359 } 2360 2361 vdev->name = name; 2362 vdev->config_len = config_size; 2363 if (vdev->config_len) { 2364 vdev->config = g_malloc0(config_size); 2365 } else { 2366 vdev->config = NULL; 2367 } 2368 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), 2369 virtio_vmstate_change, vdev); 2370 vdev->device_endian = virtio_default_endian(); 2371 vdev->use_guest_notifier_mask = true; 2372 } 2373 2374 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 2375 { 2376 return vdev->vq[n].vring.desc; 2377 } 2378 2379 bool virtio_queue_enabled(VirtIODevice *vdev, int n) 2380 { 2381 return virtio_queue_get_desc_addr(vdev, n) != 0; 2382 } 2383 2384 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 2385 { 2386 return vdev->vq[n].vring.avail; 2387 } 2388 2389 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 2390 { 2391 return vdev->vq[n].vring.used; 2392 } 2393 2394 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 2395 { 2396 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 2397 } 2398 2399 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 2400 { 2401 return offsetof(VRingAvail, ring) + 2402 sizeof(uint16_t) * vdev->vq[n].vring.num; 2403 } 2404 2405 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 2406 { 2407 return offsetof(VRingUsed, ring) + 2408 sizeof(VRingUsedElem) * vdev->vq[n].vring.num; 2409 } 2410 2411 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 2412 { 2413 return vdev->vq[n].last_avail_idx; 2414 } 2415 2416 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx) 2417 { 2418 vdev->vq[n].last_avail_idx = idx; 2419 vdev->vq[n].shadow_avail_idx = idx; 2420 } 2421 2422 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n) 2423 { 2424 rcu_read_lock(); 2425 if (vdev->vq[n].vring.desc) { 2426 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); 2427 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; 2428 } 2429 rcu_read_unlock(); 2430 } 2431 2432 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n) 2433 { 2434 rcu_read_lock(); 2435 if (vdev->vq[n].vring.desc) { 2436 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); 2437 } 2438 rcu_read_unlock(); 2439 } 2440 2441 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 2442 { 2443 vdev->vq[n].signalled_used_valid = false; 2444 } 2445 2446 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 2447 { 2448 return vdev->vq + n; 2449 } 2450 2451 uint16_t virtio_get_queue_index(VirtQueue *vq) 2452 { 2453 return vq->queue_index; 2454 } 2455 2456 static void virtio_queue_guest_notifier_read(EventNotifier *n) 2457 { 2458 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 2459 if (event_notifier_test_and_clear(n)) { 2460 virtio_irq(vq); 2461 } 2462 } 2463 2464 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 2465 bool with_irqfd) 2466 { 2467 if (assign && !with_irqfd) { 2468 event_notifier_set_handler(&vq->guest_notifier, 2469 virtio_queue_guest_notifier_read); 2470 } else { 2471 event_notifier_set_handler(&vq->guest_notifier, NULL); 2472 } 2473 if (!assign) { 2474 /* Test and clear notifier before closing it, 2475 * in case poll callback didn't have time to run. */ 2476 virtio_queue_guest_notifier_read(&vq->guest_notifier); 2477 } 2478 } 2479 2480 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 2481 { 2482 return &vq->guest_notifier; 2483 } 2484 2485 static void virtio_queue_host_notifier_aio_read(EventNotifier *n) 2486 { 2487 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2488 if (event_notifier_test_and_clear(n)) { 2489 virtio_queue_notify_aio_vq(vq); 2490 } 2491 } 2492 2493 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) 2494 { 2495 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2496 2497 virtio_queue_set_notification(vq, 0); 2498 } 2499 2500 static bool virtio_queue_host_notifier_aio_poll(void *opaque) 2501 { 2502 EventNotifier *n = opaque; 2503 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2504 bool progress; 2505 2506 if (!vq->vring.desc || virtio_queue_empty(vq)) { 2507 return false; 2508 } 2509 2510 progress = virtio_queue_notify_aio_vq(vq); 2511 2512 /* In case the handler function re-enabled notifications */ 2513 virtio_queue_set_notification(vq, 0); 2514 return progress; 2515 } 2516 2517 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) 2518 { 2519 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2520 2521 /* Caller polls once more after this to catch requests that race with us */ 2522 virtio_queue_set_notification(vq, 1); 2523 } 2524 2525 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx, 2526 VirtIOHandleAIOOutput handle_output) 2527 { 2528 if (handle_output) { 2529 vq->handle_aio_output = handle_output; 2530 aio_set_event_notifier(ctx, &vq->host_notifier, true, 2531 virtio_queue_host_notifier_aio_read, 2532 virtio_queue_host_notifier_aio_poll); 2533 aio_set_event_notifier_poll(ctx, &vq->host_notifier, 2534 virtio_queue_host_notifier_aio_poll_begin, 2535 virtio_queue_host_notifier_aio_poll_end); 2536 } else { 2537 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL); 2538 /* Test and clear notifier before after disabling event, 2539 * in case poll callback didn't have time to run. */ 2540 virtio_queue_host_notifier_aio_read(&vq->host_notifier); 2541 vq->handle_aio_output = NULL; 2542 } 2543 } 2544 2545 void virtio_queue_host_notifier_read(EventNotifier *n) 2546 { 2547 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2548 if (event_notifier_test_and_clear(n)) { 2549 virtio_queue_notify_vq(vq); 2550 } 2551 } 2552 2553 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 2554 { 2555 return &vq->host_notifier; 2556 } 2557 2558 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, 2559 MemoryRegion *mr, bool assign) 2560 { 2561 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2562 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2563 2564 if (k->set_host_notifier_mr) { 2565 return k->set_host_notifier_mr(qbus->parent, n, mr, assign); 2566 } 2567 2568 return -1; 2569 } 2570 2571 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 2572 { 2573 g_free(vdev->bus_name); 2574 vdev->bus_name = g_strdup(bus_name); 2575 } 2576 2577 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...) 2578 { 2579 va_list ap; 2580 2581 va_start(ap, fmt); 2582 error_vreport(fmt, ap); 2583 va_end(ap); 2584 2585 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2586 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; 2587 virtio_notify_config(vdev); 2588 } 2589 2590 vdev->broken = true; 2591 } 2592 2593 static void virtio_memory_listener_commit(MemoryListener *listener) 2594 { 2595 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener); 2596 int i; 2597 2598 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2599 if (vdev->vq[i].vring.num == 0) { 2600 break; 2601 } 2602 virtio_init_region_cache(vdev, i); 2603 } 2604 } 2605 2606 static void virtio_device_realize(DeviceState *dev, Error **errp) 2607 { 2608 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 2609 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 2610 Error *err = NULL; 2611 2612 /* Devices should either use vmsd or the load/save methods */ 2613 assert(!vdc->vmsd || !vdc->load); 2614 2615 if (vdc->realize != NULL) { 2616 vdc->realize(dev, &err); 2617 if (err != NULL) { 2618 error_propagate(errp, err); 2619 return; 2620 } 2621 } 2622 2623 virtio_bus_device_plugged(vdev, &err); 2624 if (err != NULL) { 2625 error_propagate(errp, err); 2626 vdc->unrealize(dev, NULL); 2627 return; 2628 } 2629 2630 vdev->listener.commit = virtio_memory_listener_commit; 2631 memory_listener_register(&vdev->listener, vdev->dma_as); 2632 } 2633 2634 static void virtio_device_unrealize(DeviceState *dev, Error **errp) 2635 { 2636 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 2637 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 2638 Error *err = NULL; 2639 2640 virtio_bus_device_unplugged(vdev); 2641 2642 if (vdc->unrealize != NULL) { 2643 vdc->unrealize(dev, &err); 2644 if (err != NULL) { 2645 error_propagate(errp, err); 2646 return; 2647 } 2648 } 2649 2650 g_free(vdev->bus_name); 2651 vdev->bus_name = NULL; 2652 } 2653 2654 static void virtio_device_free_virtqueues(VirtIODevice *vdev) 2655 { 2656 int i; 2657 if (!vdev->vq) { 2658 return; 2659 } 2660 2661 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2662 if (vdev->vq[i].vring.num == 0) { 2663 break; 2664 } 2665 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); 2666 } 2667 g_free(vdev->vq); 2668 } 2669 2670 static void virtio_device_instance_finalize(Object *obj) 2671 { 2672 VirtIODevice *vdev = VIRTIO_DEVICE(obj); 2673 2674 memory_listener_unregister(&vdev->listener); 2675 virtio_device_free_virtqueues(vdev); 2676 2677 g_free(vdev->config); 2678 g_free(vdev->vector_queues); 2679 } 2680 2681 static Property virtio_properties[] = { 2682 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 2683 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true), 2684 DEFINE_PROP_END_OF_LIST(), 2685 }; 2686 2687 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev) 2688 { 2689 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 2690 int i, n, r, err; 2691 2692 memory_region_transaction_begin(); 2693 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2694 VirtQueue *vq = &vdev->vq[n]; 2695 if (!virtio_queue_get_num(vdev, n)) { 2696 continue; 2697 } 2698 r = virtio_bus_set_host_notifier(qbus, n, true); 2699 if (r < 0) { 2700 err = r; 2701 goto assign_error; 2702 } 2703 event_notifier_set_handler(&vq->host_notifier, 2704 virtio_queue_host_notifier_read); 2705 } 2706 2707 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2708 /* Kick right away to begin processing requests already in vring */ 2709 VirtQueue *vq = &vdev->vq[n]; 2710 if (!vq->vring.num) { 2711 continue; 2712 } 2713 event_notifier_set(&vq->host_notifier); 2714 } 2715 memory_region_transaction_commit(); 2716 return 0; 2717 2718 assign_error: 2719 i = n; /* save n for a second iteration after transaction is committed. */ 2720 while (--n >= 0) { 2721 VirtQueue *vq = &vdev->vq[n]; 2722 if (!virtio_queue_get_num(vdev, n)) { 2723 continue; 2724 } 2725 2726 event_notifier_set_handler(&vq->host_notifier, NULL); 2727 r = virtio_bus_set_host_notifier(qbus, n, false); 2728 assert(r >= 0); 2729 } 2730 memory_region_transaction_commit(); 2731 2732 while (--i >= 0) { 2733 if (!virtio_queue_get_num(vdev, i)) { 2734 continue; 2735 } 2736 virtio_bus_cleanup_host_notifier(qbus, i); 2737 } 2738 return err; 2739 } 2740 2741 int virtio_device_start_ioeventfd(VirtIODevice *vdev) 2742 { 2743 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2744 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2745 2746 return virtio_bus_start_ioeventfd(vbus); 2747 } 2748 2749 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev) 2750 { 2751 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 2752 int n, r; 2753 2754 memory_region_transaction_begin(); 2755 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2756 VirtQueue *vq = &vdev->vq[n]; 2757 2758 if (!virtio_queue_get_num(vdev, n)) { 2759 continue; 2760 } 2761 event_notifier_set_handler(&vq->host_notifier, NULL); 2762 r = virtio_bus_set_host_notifier(qbus, n, false); 2763 assert(r >= 0); 2764 } 2765 memory_region_transaction_commit(); 2766 2767 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2768 if (!virtio_queue_get_num(vdev, n)) { 2769 continue; 2770 } 2771 virtio_bus_cleanup_host_notifier(qbus, n); 2772 } 2773 } 2774 2775 void virtio_device_stop_ioeventfd(VirtIODevice *vdev) 2776 { 2777 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2778 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2779 2780 virtio_bus_stop_ioeventfd(vbus); 2781 } 2782 2783 int virtio_device_grab_ioeventfd(VirtIODevice *vdev) 2784 { 2785 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2786 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2787 2788 return virtio_bus_grab_ioeventfd(vbus); 2789 } 2790 2791 void virtio_device_release_ioeventfd(VirtIODevice *vdev) 2792 { 2793 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2794 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2795 2796 virtio_bus_release_ioeventfd(vbus); 2797 } 2798 2799 static void virtio_device_class_init(ObjectClass *klass, void *data) 2800 { 2801 /* Set the default value here. */ 2802 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 2803 DeviceClass *dc = DEVICE_CLASS(klass); 2804 2805 dc->realize = virtio_device_realize; 2806 dc->unrealize = virtio_device_unrealize; 2807 dc->bus_type = TYPE_VIRTIO_BUS; 2808 dc->props = virtio_properties; 2809 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; 2810 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; 2811 2812 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; 2813 } 2814 2815 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev) 2816 { 2817 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2818 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2819 2820 return virtio_bus_ioeventfd_enabled(vbus); 2821 } 2822 2823 static const TypeInfo virtio_device_info = { 2824 .name = TYPE_VIRTIO_DEVICE, 2825 .parent = TYPE_DEVICE, 2826 .instance_size = sizeof(VirtIODevice), 2827 .class_init = virtio_device_class_init, 2828 .instance_finalize = virtio_device_instance_finalize, 2829 .abstract = true, 2830 .class_size = sizeof(VirtioDeviceClass), 2831 }; 2832 2833 static void virtio_register_types(void) 2834 { 2835 type_register_static(&virtio_device_info); 2836 } 2837 2838 type_init(virtio_register_types) 2839