1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qapi/error.h" 16 #include "qemu-common.h" 17 #include "cpu.h" 18 #include "trace.h" 19 #include "exec/address-spaces.h" 20 #include "qemu/error-report.h" 21 #include "hw/virtio/virtio.h" 22 #include "qemu/atomic.h" 23 #include "hw/virtio/virtio-bus.h" 24 #include "migration/migration.h" 25 #include "hw/virtio/virtio-access.h" 26 27 /* 28 * The alignment to use between consumer and producer parts of vring. 29 * x86 pagesize again. This is the default, used by transports like PCI 30 * which don't provide a means for the guest to tell the host the alignment. 31 */ 32 #define VIRTIO_PCI_VRING_ALIGN 4096 33 34 typedef struct VRingDesc 35 { 36 uint64_t addr; 37 uint32_t len; 38 uint16_t flags; 39 uint16_t next; 40 } VRingDesc; 41 42 typedef struct VRingAvail 43 { 44 uint16_t flags; 45 uint16_t idx; 46 uint16_t ring[0]; 47 } VRingAvail; 48 49 typedef struct VRingUsedElem 50 { 51 uint32_t id; 52 uint32_t len; 53 } VRingUsedElem; 54 55 typedef struct VRingUsed 56 { 57 uint16_t flags; 58 uint16_t idx; 59 VRingUsedElem ring[0]; 60 } VRingUsed; 61 62 typedef struct VRing 63 { 64 unsigned int num; 65 unsigned int num_default; 66 unsigned int align; 67 hwaddr desc; 68 hwaddr avail; 69 hwaddr used; 70 } VRing; 71 72 struct VirtQueue 73 { 74 VRing vring; 75 76 /* Next head to pop */ 77 uint16_t last_avail_idx; 78 79 /* Last avail_idx read from VQ. */ 80 uint16_t shadow_avail_idx; 81 82 uint16_t used_idx; 83 84 /* Last used index value we have signalled on */ 85 uint16_t signalled_used; 86 87 /* Last used index value we have signalled on */ 88 bool signalled_used_valid; 89 90 /* Notification enabled? */ 91 bool notification; 92 93 uint16_t queue_index; 94 95 int inuse; 96 97 uint16_t vector; 98 VirtIOHandleOutput handle_output; 99 VirtIOHandleOutput handle_aio_output; 100 bool use_aio; 101 VirtIODevice *vdev; 102 EventNotifier guest_notifier; 103 EventNotifier host_notifier; 104 QLIST_ENTRY(VirtQueue) node; 105 }; 106 107 /* virt queue functions */ 108 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 109 { 110 VRing *vring = &vdev->vq[n].vring; 111 112 if (!vring->desc) { 113 /* not yet setup -> nothing to do */ 114 return; 115 } 116 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 117 vring->used = vring_align(vring->avail + 118 offsetof(VRingAvail, ring[vring->num]), 119 vring->align); 120 } 121 122 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc, 123 hwaddr desc_pa, int i) 124 { 125 address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc), 126 MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc)); 127 virtio_tswap64s(vdev, &desc->addr); 128 virtio_tswap32s(vdev, &desc->len); 129 virtio_tswap16s(vdev, &desc->flags); 130 virtio_tswap16s(vdev, &desc->next); 131 } 132 133 static inline uint16_t vring_avail_flags(VirtQueue *vq) 134 { 135 hwaddr pa; 136 pa = vq->vring.avail + offsetof(VRingAvail, flags); 137 return virtio_lduw_phys(vq->vdev, pa); 138 } 139 140 static inline uint16_t vring_avail_idx(VirtQueue *vq) 141 { 142 hwaddr pa; 143 pa = vq->vring.avail + offsetof(VRingAvail, idx); 144 vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa); 145 return vq->shadow_avail_idx; 146 } 147 148 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 149 { 150 hwaddr pa; 151 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]); 152 return virtio_lduw_phys(vq->vdev, pa); 153 } 154 155 static inline uint16_t vring_get_used_event(VirtQueue *vq) 156 { 157 return vring_avail_ring(vq, vq->vring.num); 158 } 159 160 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, 161 int i) 162 { 163 hwaddr pa; 164 virtio_tswap32s(vq->vdev, &uelem->id); 165 virtio_tswap32s(vq->vdev, &uelem->len); 166 pa = vq->vring.used + offsetof(VRingUsed, ring[i]); 167 address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED, 168 (void *)uelem, sizeof(VRingUsedElem)); 169 } 170 171 static uint16_t vring_used_idx(VirtQueue *vq) 172 { 173 hwaddr pa; 174 pa = vq->vring.used + offsetof(VRingUsed, idx); 175 return virtio_lduw_phys(vq->vdev, pa); 176 } 177 178 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 179 { 180 hwaddr pa; 181 pa = vq->vring.used + offsetof(VRingUsed, idx); 182 virtio_stw_phys(vq->vdev, pa, val); 183 vq->used_idx = val; 184 } 185 186 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 187 { 188 VirtIODevice *vdev = vq->vdev; 189 hwaddr pa; 190 pa = vq->vring.used + offsetof(VRingUsed, flags); 191 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask); 192 } 193 194 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 195 { 196 VirtIODevice *vdev = vq->vdev; 197 hwaddr pa; 198 pa = vq->vring.used + offsetof(VRingUsed, flags); 199 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask); 200 } 201 202 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 203 { 204 hwaddr pa; 205 if (!vq->notification) { 206 return; 207 } 208 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]); 209 virtio_stw_phys(vq->vdev, pa, val); 210 } 211 212 void virtio_queue_set_notification(VirtQueue *vq, int enable) 213 { 214 vq->notification = enable; 215 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 216 vring_set_avail_event(vq, vring_avail_idx(vq)); 217 } else if (enable) { 218 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 219 } else { 220 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 221 } 222 if (enable) { 223 /* Expose avail event/used flags before caller checks the avail idx. */ 224 smp_mb(); 225 } 226 } 227 228 int virtio_queue_ready(VirtQueue *vq) 229 { 230 return vq->vring.avail != 0; 231 } 232 233 /* Fetch avail_idx from VQ memory only when we really need to know if 234 * guest has added some buffers. */ 235 int virtio_queue_empty(VirtQueue *vq) 236 { 237 if (vq->shadow_avail_idx != vq->last_avail_idx) { 238 return 0; 239 } 240 241 return vring_avail_idx(vq) == vq->last_avail_idx; 242 } 243 244 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 245 unsigned int len) 246 { 247 unsigned int offset; 248 int i; 249 250 offset = 0; 251 for (i = 0; i < elem->in_num; i++) { 252 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 253 254 cpu_physical_memory_unmap(elem->in_sg[i].iov_base, 255 elem->in_sg[i].iov_len, 256 1, size); 257 258 offset += size; 259 } 260 261 for (i = 0; i < elem->out_num; i++) 262 cpu_physical_memory_unmap(elem->out_sg[i].iov_base, 263 elem->out_sg[i].iov_len, 264 0, elem->out_sg[i].iov_len); 265 } 266 267 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem, 268 unsigned int len) 269 { 270 vq->last_avail_idx--; 271 virtqueue_unmap_sg(vq, elem, len); 272 } 273 274 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 275 unsigned int len, unsigned int idx) 276 { 277 VRingUsedElem uelem; 278 279 trace_virtqueue_fill(vq, elem, len, idx); 280 281 virtqueue_unmap_sg(vq, elem, len); 282 283 idx = (idx + vq->used_idx) % vq->vring.num; 284 285 uelem.id = elem->index; 286 uelem.len = len; 287 vring_used_write(vq, &uelem, idx); 288 } 289 290 void virtqueue_flush(VirtQueue *vq, unsigned int count) 291 { 292 uint16_t old, new; 293 /* Make sure buffer is written before we update index. */ 294 smp_wmb(); 295 trace_virtqueue_flush(vq, count); 296 old = vq->used_idx; 297 new = old + count; 298 vring_used_idx_set(vq, new); 299 vq->inuse -= count; 300 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 301 vq->signalled_used_valid = false; 302 } 303 304 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 305 unsigned int len) 306 { 307 virtqueue_fill(vq, elem, len, 0); 308 virtqueue_flush(vq, 1); 309 } 310 311 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 312 { 313 uint16_t num_heads = vring_avail_idx(vq) - idx; 314 315 /* Check it isn't doing very strange things with descriptor numbers. */ 316 if (num_heads > vq->vring.num) { 317 error_report("Guest moved used index from %u to %u", 318 idx, vq->shadow_avail_idx); 319 exit(1); 320 } 321 /* On success, callers read a descriptor at vq->last_avail_idx. 322 * Make sure descriptor read does not bypass avail index read. */ 323 if (num_heads) { 324 smp_rmb(); 325 } 326 327 return num_heads; 328 } 329 330 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx) 331 { 332 unsigned int head; 333 334 /* Grab the next descriptor number they're advertising, and increment 335 * the index we've seen. */ 336 head = vring_avail_ring(vq, idx % vq->vring.num); 337 338 /* If their number is silly, that's a fatal mistake. */ 339 if (head >= vq->vring.num) { 340 error_report("Guest says index %u is available", head); 341 exit(1); 342 } 343 344 return head; 345 } 346 347 static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, 348 hwaddr desc_pa, unsigned int max) 349 { 350 unsigned int next; 351 352 /* If this descriptor says it doesn't chain, we're done. */ 353 if (!(desc->flags & VRING_DESC_F_NEXT)) { 354 return max; 355 } 356 357 /* Check they're not leading us off end of descriptors. */ 358 next = desc->next; 359 /* Make sure compiler knows to grab that: we don't want it changing! */ 360 smp_wmb(); 361 362 if (next >= max) { 363 error_report("Desc next is %u", next); 364 exit(1); 365 } 366 367 vring_desc_read(vdev, desc, desc_pa, next); 368 return next; 369 } 370 371 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 372 unsigned int *out_bytes, 373 unsigned max_in_bytes, unsigned max_out_bytes) 374 { 375 unsigned int idx; 376 unsigned int total_bufs, in_total, out_total; 377 378 idx = vq->last_avail_idx; 379 380 total_bufs = in_total = out_total = 0; 381 while (virtqueue_num_heads(vq, idx)) { 382 VirtIODevice *vdev = vq->vdev; 383 unsigned int max, num_bufs, indirect = 0; 384 VRingDesc desc; 385 hwaddr desc_pa; 386 int i; 387 388 max = vq->vring.num; 389 num_bufs = total_bufs; 390 i = virtqueue_get_head(vq, idx++); 391 desc_pa = vq->vring.desc; 392 vring_desc_read(vdev, &desc, desc_pa, i); 393 394 if (desc.flags & VRING_DESC_F_INDIRECT) { 395 if (desc.len % sizeof(VRingDesc)) { 396 error_report("Invalid size for indirect buffer table"); 397 exit(1); 398 } 399 400 /* If we've got too many, that implies a descriptor loop. */ 401 if (num_bufs >= max) { 402 error_report("Looped descriptor"); 403 exit(1); 404 } 405 406 /* loop over the indirect descriptor table */ 407 indirect = 1; 408 max = desc.len / sizeof(VRingDesc); 409 desc_pa = desc.addr; 410 num_bufs = i = 0; 411 vring_desc_read(vdev, &desc, desc_pa, i); 412 } 413 414 do { 415 /* If we've got too many, that implies a descriptor loop. */ 416 if (++num_bufs > max) { 417 error_report("Looped descriptor"); 418 exit(1); 419 } 420 421 if (desc.flags & VRING_DESC_F_WRITE) { 422 in_total += desc.len; 423 } else { 424 out_total += desc.len; 425 } 426 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 427 goto done; 428 } 429 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max); 430 431 if (!indirect) 432 total_bufs = num_bufs; 433 else 434 total_bufs++; 435 } 436 done: 437 if (in_bytes) { 438 *in_bytes = in_total; 439 } 440 if (out_bytes) { 441 *out_bytes = out_total; 442 } 443 } 444 445 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 446 unsigned int out_bytes) 447 { 448 unsigned int in_total, out_total; 449 450 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 451 return in_bytes <= in_total && out_bytes <= out_total; 452 } 453 454 static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov, 455 unsigned int max_num_sg, bool is_write, 456 hwaddr pa, size_t sz) 457 { 458 unsigned num_sg = *p_num_sg; 459 assert(num_sg <= max_num_sg); 460 461 if (!sz) { 462 error_report("virtio: zero sized buffers are not allowed"); 463 exit(1); 464 } 465 466 while (sz) { 467 hwaddr len = sz; 468 469 if (num_sg == max_num_sg) { 470 error_report("virtio: too many write descriptors in indirect table"); 471 exit(1); 472 } 473 474 iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write); 475 iov[num_sg].iov_len = len; 476 addr[num_sg] = pa; 477 478 sz -= len; 479 pa += len; 480 num_sg++; 481 } 482 *p_num_sg = num_sg; 483 } 484 485 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr, 486 unsigned int *num_sg, unsigned int max_size, 487 int is_write) 488 { 489 unsigned int i; 490 hwaddr len; 491 492 /* Note: this function MUST validate input, some callers 493 * are passing in num_sg values received over the network. 494 */ 495 /* TODO: teach all callers that this can fail, and return failure instead 496 * of asserting here. 497 * When we do, we might be able to re-enable NDEBUG below. 498 */ 499 #ifdef NDEBUG 500 #error building with NDEBUG is not supported 501 #endif 502 assert(*num_sg <= max_size); 503 504 for (i = 0; i < *num_sg; i++) { 505 len = sg[i].iov_len; 506 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write); 507 if (!sg[i].iov_base) { 508 error_report("virtio: error trying to map MMIO memory"); 509 exit(1); 510 } 511 if (len != sg[i].iov_len) { 512 error_report("virtio: unexpected memory split"); 513 exit(1); 514 } 515 } 516 } 517 518 void virtqueue_map(VirtQueueElement *elem) 519 { 520 virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num, 521 VIRTQUEUE_MAX_SIZE, 1); 522 virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num, 523 VIRTQUEUE_MAX_SIZE, 0); 524 } 525 526 void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) 527 { 528 VirtQueueElement *elem; 529 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); 530 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); 531 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); 532 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); 533 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); 534 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); 535 536 assert(sz >= sizeof(VirtQueueElement)); 537 elem = g_malloc(out_sg_end); 538 elem->out_num = out_num; 539 elem->in_num = in_num; 540 elem->in_addr = (void *)elem + in_addr_ofs; 541 elem->out_addr = (void *)elem + out_addr_ofs; 542 elem->in_sg = (void *)elem + in_sg_ofs; 543 elem->out_sg = (void *)elem + out_sg_ofs; 544 return elem; 545 } 546 547 void *virtqueue_pop(VirtQueue *vq, size_t sz) 548 { 549 unsigned int i, head, max; 550 hwaddr desc_pa = vq->vring.desc; 551 VirtIODevice *vdev = vq->vdev; 552 VirtQueueElement *elem; 553 unsigned out_num, in_num; 554 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 555 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 556 VRingDesc desc; 557 558 if (virtio_queue_empty(vq)) { 559 return NULL; 560 } 561 /* Needed after virtio_queue_empty(), see comment in 562 * virtqueue_num_heads(). */ 563 smp_rmb(); 564 565 /* When we start there are none of either input nor output. */ 566 out_num = in_num = 0; 567 568 max = vq->vring.num; 569 570 if (vq->inuse >= vq->vring.num) { 571 error_report("Virtqueue size exceeded"); 572 exit(1); 573 } 574 575 i = head = virtqueue_get_head(vq, vq->last_avail_idx++); 576 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 577 vring_set_avail_event(vq, vq->last_avail_idx); 578 } 579 580 vring_desc_read(vdev, &desc, desc_pa, i); 581 if (desc.flags & VRING_DESC_F_INDIRECT) { 582 if (desc.len % sizeof(VRingDesc)) { 583 error_report("Invalid size for indirect buffer table"); 584 exit(1); 585 } 586 587 /* loop over the indirect descriptor table */ 588 max = desc.len / sizeof(VRingDesc); 589 desc_pa = desc.addr; 590 i = 0; 591 vring_desc_read(vdev, &desc, desc_pa, i); 592 } 593 594 /* Collect all the descriptors */ 595 do { 596 if (desc.flags & VRING_DESC_F_WRITE) { 597 virtqueue_map_desc(&in_num, addr + out_num, iov + out_num, 598 VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len); 599 } else { 600 if (in_num) { 601 error_report("Incorrect order for descriptors"); 602 exit(1); 603 } 604 virtqueue_map_desc(&out_num, addr, iov, 605 VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len); 606 } 607 608 /* If we've got too many, that implies a descriptor loop. */ 609 if ((in_num + out_num) > max) { 610 error_report("Looped descriptor"); 611 exit(1); 612 } 613 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max); 614 615 /* Now copy what we have collected and mapped */ 616 elem = virtqueue_alloc_element(sz, out_num, in_num); 617 elem->index = head; 618 for (i = 0; i < out_num; i++) { 619 elem->out_addr[i] = addr[i]; 620 elem->out_sg[i] = iov[i]; 621 } 622 for (i = 0; i < in_num; i++) { 623 elem->in_addr[i] = addr[out_num + i]; 624 elem->in_sg[i] = iov[out_num + i]; 625 } 626 627 vq->inuse++; 628 629 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 630 return elem; 631 } 632 633 /* Reading and writing a structure directly to QEMUFile is *awful*, but 634 * it is what QEMU has always done by mistake. We can change it sooner 635 * or later by bumping the version number of the affected vm states. 636 * In the meanwhile, since the in-memory layout of VirtQueueElement 637 * has changed, we need to marshal to and from the layout that was 638 * used before the change. 639 */ 640 typedef struct VirtQueueElementOld { 641 unsigned int index; 642 unsigned int out_num; 643 unsigned int in_num; 644 hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; 645 hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; 646 struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; 647 struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; 648 } VirtQueueElementOld; 649 650 void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz) 651 { 652 VirtQueueElement *elem; 653 VirtQueueElementOld data; 654 int i; 655 656 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 657 658 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); 659 elem->index = data.index; 660 661 for (i = 0; i < elem->in_num; i++) { 662 elem->in_addr[i] = data.in_addr[i]; 663 } 664 665 for (i = 0; i < elem->out_num; i++) { 666 elem->out_addr[i] = data.out_addr[i]; 667 } 668 669 for (i = 0; i < elem->in_num; i++) { 670 /* Base is overwritten by virtqueue_map. */ 671 elem->in_sg[i].iov_base = 0; 672 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; 673 } 674 675 for (i = 0; i < elem->out_num; i++) { 676 /* Base is overwritten by virtqueue_map. */ 677 elem->out_sg[i].iov_base = 0; 678 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; 679 } 680 681 virtqueue_map(elem); 682 return elem; 683 } 684 685 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem) 686 { 687 VirtQueueElementOld data; 688 int i; 689 690 memset(&data, 0, sizeof(data)); 691 data.index = elem->index; 692 data.in_num = elem->in_num; 693 data.out_num = elem->out_num; 694 695 for (i = 0; i < elem->in_num; i++) { 696 data.in_addr[i] = elem->in_addr[i]; 697 } 698 699 for (i = 0; i < elem->out_num; i++) { 700 data.out_addr[i] = elem->out_addr[i]; 701 } 702 703 for (i = 0; i < elem->in_num; i++) { 704 /* Base is overwritten by virtqueue_map when loading. Do not 705 * save it, as it would leak the QEMU address space layout. */ 706 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; 707 } 708 709 for (i = 0; i < elem->out_num; i++) { 710 /* Do not save iov_base as above. */ 711 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; 712 } 713 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 714 } 715 716 /* virtio device */ 717 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 718 { 719 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 720 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 721 722 if (k->notify) { 723 k->notify(qbus->parent, vector); 724 } 725 } 726 727 void virtio_update_irq(VirtIODevice *vdev) 728 { 729 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 730 } 731 732 static int virtio_validate_features(VirtIODevice *vdev) 733 { 734 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 735 736 if (k->validate_features) { 737 return k->validate_features(vdev); 738 } else { 739 return 0; 740 } 741 } 742 743 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 744 { 745 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 746 trace_virtio_set_status(vdev, val); 747 748 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 749 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 750 val & VIRTIO_CONFIG_S_FEATURES_OK) { 751 int ret = virtio_validate_features(vdev); 752 753 if (ret) { 754 return ret; 755 } 756 } 757 } 758 if (k->set_status) { 759 k->set_status(vdev, val); 760 } 761 vdev->status = val; 762 return 0; 763 } 764 765 bool target_words_bigendian(void); 766 static enum virtio_device_endian virtio_default_endian(void) 767 { 768 if (target_words_bigendian()) { 769 return VIRTIO_DEVICE_ENDIAN_BIG; 770 } else { 771 return VIRTIO_DEVICE_ENDIAN_LITTLE; 772 } 773 } 774 775 static enum virtio_device_endian virtio_current_cpu_endian(void) 776 { 777 CPUClass *cc = CPU_GET_CLASS(current_cpu); 778 779 if (cc->virtio_is_big_endian(current_cpu)) { 780 return VIRTIO_DEVICE_ENDIAN_BIG; 781 } else { 782 return VIRTIO_DEVICE_ENDIAN_LITTLE; 783 } 784 } 785 786 void virtio_reset(void *opaque) 787 { 788 VirtIODevice *vdev = opaque; 789 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 790 int i; 791 792 virtio_set_status(vdev, 0); 793 if (current_cpu) { 794 /* Guest initiated reset */ 795 vdev->device_endian = virtio_current_cpu_endian(); 796 } else { 797 /* System reset */ 798 vdev->device_endian = virtio_default_endian(); 799 } 800 801 if (k->reset) { 802 k->reset(vdev); 803 } 804 805 vdev->guest_features = 0; 806 vdev->queue_sel = 0; 807 vdev->status = 0; 808 vdev->isr = 0; 809 vdev->config_vector = VIRTIO_NO_VECTOR; 810 virtio_notify_vector(vdev, vdev->config_vector); 811 812 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 813 vdev->vq[i].vring.desc = 0; 814 vdev->vq[i].vring.avail = 0; 815 vdev->vq[i].vring.used = 0; 816 vdev->vq[i].last_avail_idx = 0; 817 vdev->vq[i].shadow_avail_idx = 0; 818 vdev->vq[i].used_idx = 0; 819 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 820 vdev->vq[i].signalled_used = 0; 821 vdev->vq[i].signalled_used_valid = false; 822 vdev->vq[i].notification = true; 823 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 824 } 825 } 826 827 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 828 { 829 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 830 uint8_t val; 831 832 if (addr + sizeof(val) > vdev->config_len) { 833 return (uint32_t)-1; 834 } 835 836 k->get_config(vdev, vdev->config); 837 838 val = ldub_p(vdev->config + addr); 839 return val; 840 } 841 842 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 843 { 844 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 845 uint16_t val; 846 847 if (addr + sizeof(val) > vdev->config_len) { 848 return (uint32_t)-1; 849 } 850 851 k->get_config(vdev, vdev->config); 852 853 val = lduw_p(vdev->config + addr); 854 return val; 855 } 856 857 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 858 { 859 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 860 uint32_t val; 861 862 if (addr + sizeof(val) > vdev->config_len) { 863 return (uint32_t)-1; 864 } 865 866 k->get_config(vdev, vdev->config); 867 868 val = ldl_p(vdev->config + addr); 869 return val; 870 } 871 872 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 873 { 874 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 875 uint8_t val = data; 876 877 if (addr + sizeof(val) > vdev->config_len) { 878 return; 879 } 880 881 stb_p(vdev->config + addr, val); 882 883 if (k->set_config) { 884 k->set_config(vdev, vdev->config); 885 } 886 } 887 888 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 889 { 890 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 891 uint16_t val = data; 892 893 if (addr + sizeof(val) > vdev->config_len) { 894 return; 895 } 896 897 stw_p(vdev->config + addr, val); 898 899 if (k->set_config) { 900 k->set_config(vdev, vdev->config); 901 } 902 } 903 904 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 905 { 906 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 907 uint32_t val = data; 908 909 if (addr + sizeof(val) > vdev->config_len) { 910 return; 911 } 912 913 stl_p(vdev->config + addr, val); 914 915 if (k->set_config) { 916 k->set_config(vdev, vdev->config); 917 } 918 } 919 920 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 921 { 922 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 923 uint8_t val; 924 925 if (addr + sizeof(val) > vdev->config_len) { 926 return (uint32_t)-1; 927 } 928 929 k->get_config(vdev, vdev->config); 930 931 val = ldub_p(vdev->config + addr); 932 return val; 933 } 934 935 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 936 { 937 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 938 uint16_t val; 939 940 if (addr + sizeof(val) > vdev->config_len) { 941 return (uint32_t)-1; 942 } 943 944 k->get_config(vdev, vdev->config); 945 946 val = lduw_le_p(vdev->config + addr); 947 return val; 948 } 949 950 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 951 { 952 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 953 uint32_t val; 954 955 if (addr + sizeof(val) > vdev->config_len) { 956 return (uint32_t)-1; 957 } 958 959 k->get_config(vdev, vdev->config); 960 961 val = ldl_le_p(vdev->config + addr); 962 return val; 963 } 964 965 void virtio_config_modern_writeb(VirtIODevice *vdev, 966 uint32_t addr, uint32_t data) 967 { 968 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 969 uint8_t val = data; 970 971 if (addr + sizeof(val) > vdev->config_len) { 972 return; 973 } 974 975 stb_p(vdev->config + addr, val); 976 977 if (k->set_config) { 978 k->set_config(vdev, vdev->config); 979 } 980 } 981 982 void virtio_config_modern_writew(VirtIODevice *vdev, 983 uint32_t addr, uint32_t data) 984 { 985 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 986 uint16_t val = data; 987 988 if (addr + sizeof(val) > vdev->config_len) { 989 return; 990 } 991 992 stw_le_p(vdev->config + addr, val); 993 994 if (k->set_config) { 995 k->set_config(vdev, vdev->config); 996 } 997 } 998 999 void virtio_config_modern_writel(VirtIODevice *vdev, 1000 uint32_t addr, uint32_t data) 1001 { 1002 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1003 uint32_t val = data; 1004 1005 if (addr + sizeof(val) > vdev->config_len) { 1006 return; 1007 } 1008 1009 stl_le_p(vdev->config + addr, val); 1010 1011 if (k->set_config) { 1012 k->set_config(vdev, vdev->config); 1013 } 1014 } 1015 1016 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 1017 { 1018 vdev->vq[n].vring.desc = addr; 1019 virtio_queue_update_rings(vdev, n); 1020 } 1021 1022 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 1023 { 1024 return vdev->vq[n].vring.desc; 1025 } 1026 1027 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 1028 hwaddr avail, hwaddr used) 1029 { 1030 vdev->vq[n].vring.desc = desc; 1031 vdev->vq[n].vring.avail = avail; 1032 vdev->vq[n].vring.used = used; 1033 } 1034 1035 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 1036 { 1037 /* Don't allow guest to flip queue between existent and 1038 * nonexistent states, or to set it to an invalid size. 1039 */ 1040 if (!!num != !!vdev->vq[n].vring.num || 1041 num > VIRTQUEUE_MAX_SIZE || 1042 num < 0) { 1043 return; 1044 } 1045 vdev->vq[n].vring.num = num; 1046 } 1047 1048 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 1049 { 1050 return QLIST_FIRST(&vdev->vector_queues[vector]); 1051 } 1052 1053 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 1054 { 1055 return QLIST_NEXT(vq, node); 1056 } 1057 1058 int virtio_queue_get_num(VirtIODevice *vdev, int n) 1059 { 1060 return vdev->vq[n].vring.num; 1061 } 1062 1063 int virtio_get_num_queues(VirtIODevice *vdev) 1064 { 1065 int i; 1066 1067 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1068 if (!virtio_queue_get_num(vdev, i)) { 1069 break; 1070 } 1071 } 1072 1073 return i; 1074 } 1075 1076 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 1077 { 1078 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1079 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1080 1081 /* virtio-1 compliant devices cannot change the alignment */ 1082 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1083 error_report("tried to modify queue alignment for virtio-1 device"); 1084 return; 1085 } 1086 /* Check that the transport told us it was going to do this 1087 * (so a buggy transport will immediately assert rather than 1088 * silently failing to migrate this state) 1089 */ 1090 assert(k->has_variable_vring_alignment); 1091 1092 vdev->vq[n].vring.align = align; 1093 virtio_queue_update_rings(vdev, n); 1094 } 1095 1096 static void virtio_queue_notify_aio_vq(VirtQueue *vq) 1097 { 1098 if (vq->vring.desc && vq->handle_aio_output) { 1099 VirtIODevice *vdev = vq->vdev; 1100 1101 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1102 vq->handle_aio_output(vdev, vq); 1103 } 1104 } 1105 1106 static void virtio_queue_notify_vq(VirtQueue *vq) 1107 { 1108 if (vq->vring.desc && vq->handle_output) { 1109 VirtIODevice *vdev = vq->vdev; 1110 1111 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1112 vq->handle_output(vdev, vq); 1113 } 1114 } 1115 1116 void virtio_queue_notify(VirtIODevice *vdev, int n) 1117 { 1118 virtio_queue_notify_vq(&vdev->vq[n]); 1119 } 1120 1121 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 1122 { 1123 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 1124 VIRTIO_NO_VECTOR; 1125 } 1126 1127 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 1128 { 1129 VirtQueue *vq = &vdev->vq[n]; 1130 1131 if (n < VIRTIO_QUEUE_MAX) { 1132 if (vdev->vector_queues && 1133 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 1134 QLIST_REMOVE(vq, node); 1135 } 1136 vdev->vq[n].vector = vector; 1137 if (vdev->vector_queues && 1138 vector != VIRTIO_NO_VECTOR) { 1139 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 1140 } 1141 } 1142 } 1143 1144 static VirtQueue *virtio_add_queue_internal(VirtIODevice *vdev, int queue_size, 1145 VirtIOHandleOutput handle_output, 1146 bool use_aio) 1147 { 1148 int i; 1149 1150 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1151 if (vdev->vq[i].vring.num == 0) 1152 break; 1153 } 1154 1155 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 1156 abort(); 1157 1158 vdev->vq[i].vring.num = queue_size; 1159 vdev->vq[i].vring.num_default = queue_size; 1160 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 1161 vdev->vq[i].handle_output = handle_output; 1162 vdev->vq[i].handle_aio_output = NULL; 1163 vdev->vq[i].use_aio = use_aio; 1164 1165 return &vdev->vq[i]; 1166 } 1167 1168 /* Add a virt queue and mark AIO. 1169 * An AIO queue will use the AioContext based event interface instead of the 1170 * default IOHandler and EventNotifier interface. 1171 */ 1172 VirtQueue *virtio_add_queue_aio(VirtIODevice *vdev, int queue_size, 1173 VirtIOHandleOutput handle_output) 1174 { 1175 return virtio_add_queue_internal(vdev, queue_size, handle_output, true); 1176 } 1177 1178 /* Add a normal virt queue (on the contrary to the AIO version above. */ 1179 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 1180 VirtIOHandleOutput handle_output) 1181 { 1182 return virtio_add_queue_internal(vdev, queue_size, handle_output, false); 1183 } 1184 1185 void virtio_del_queue(VirtIODevice *vdev, int n) 1186 { 1187 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 1188 abort(); 1189 } 1190 1191 vdev->vq[n].vring.num = 0; 1192 vdev->vq[n].vring.num_default = 0; 1193 } 1194 1195 void virtio_irq(VirtQueue *vq) 1196 { 1197 trace_virtio_irq(vq); 1198 vq->vdev->isr |= 0x01; 1199 virtio_notify_vector(vq->vdev, vq->vector); 1200 } 1201 1202 bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) 1203 { 1204 uint16_t old, new; 1205 bool v; 1206 /* We need to expose used array entries before checking used event. */ 1207 smp_mb(); 1208 /* Always notify when queue is empty (when feature acknowledge) */ 1209 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 1210 !vq->inuse && virtio_queue_empty(vq)) { 1211 return true; 1212 } 1213 1214 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1215 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 1216 } 1217 1218 v = vq->signalled_used_valid; 1219 vq->signalled_used_valid = true; 1220 old = vq->signalled_used; 1221 new = vq->signalled_used = vq->used_idx; 1222 return !v || vring_need_event(vring_get_used_event(vq), new, old); 1223 } 1224 1225 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 1226 { 1227 if (!virtio_should_notify(vdev, vq)) { 1228 return; 1229 } 1230 1231 trace_virtio_notify(vdev, vq); 1232 vdev->isr |= 0x01; 1233 virtio_notify_vector(vdev, vq->vector); 1234 } 1235 1236 void virtio_notify_config(VirtIODevice *vdev) 1237 { 1238 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1239 return; 1240 1241 vdev->isr |= 0x03; 1242 vdev->generation++; 1243 virtio_notify_vector(vdev, vdev->config_vector); 1244 } 1245 1246 static bool virtio_device_endian_needed(void *opaque) 1247 { 1248 VirtIODevice *vdev = opaque; 1249 1250 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 1251 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1252 return vdev->device_endian != virtio_default_endian(); 1253 } 1254 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 1255 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 1256 } 1257 1258 static bool virtio_64bit_features_needed(void *opaque) 1259 { 1260 VirtIODevice *vdev = opaque; 1261 1262 return (vdev->host_features >> 32) != 0; 1263 } 1264 1265 static bool virtio_virtqueue_needed(void *opaque) 1266 { 1267 VirtIODevice *vdev = opaque; 1268 1269 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 1270 } 1271 1272 static bool virtio_ringsize_needed(void *opaque) 1273 { 1274 VirtIODevice *vdev = opaque; 1275 int i; 1276 1277 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1278 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 1279 return true; 1280 } 1281 } 1282 return false; 1283 } 1284 1285 static bool virtio_extra_state_needed(void *opaque) 1286 { 1287 VirtIODevice *vdev = opaque; 1288 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1289 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1290 1291 return k->has_extra_state && 1292 k->has_extra_state(qbus->parent); 1293 } 1294 1295 static const VMStateDescription vmstate_virtqueue = { 1296 .name = "virtqueue_state", 1297 .version_id = 1, 1298 .minimum_version_id = 1, 1299 .fields = (VMStateField[]) { 1300 VMSTATE_UINT64(vring.avail, struct VirtQueue), 1301 VMSTATE_UINT64(vring.used, struct VirtQueue), 1302 VMSTATE_END_OF_LIST() 1303 } 1304 }; 1305 1306 static const VMStateDescription vmstate_virtio_virtqueues = { 1307 .name = "virtio/virtqueues", 1308 .version_id = 1, 1309 .minimum_version_id = 1, 1310 .needed = &virtio_virtqueue_needed, 1311 .fields = (VMStateField[]) { 1312 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 1313 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), 1314 VMSTATE_END_OF_LIST() 1315 } 1316 }; 1317 1318 static const VMStateDescription vmstate_ringsize = { 1319 .name = "ringsize_state", 1320 .version_id = 1, 1321 .minimum_version_id = 1, 1322 .fields = (VMStateField[]) { 1323 VMSTATE_UINT32(vring.num_default, struct VirtQueue), 1324 VMSTATE_END_OF_LIST() 1325 } 1326 }; 1327 1328 static const VMStateDescription vmstate_virtio_ringsize = { 1329 .name = "virtio/ringsize", 1330 .version_id = 1, 1331 .minimum_version_id = 1, 1332 .needed = &virtio_ringsize_needed, 1333 .fields = (VMStateField[]) { 1334 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 1335 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), 1336 VMSTATE_END_OF_LIST() 1337 } 1338 }; 1339 1340 static int get_extra_state(QEMUFile *f, void *pv, size_t size) 1341 { 1342 VirtIODevice *vdev = pv; 1343 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1344 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1345 1346 if (!k->load_extra_state) { 1347 return -1; 1348 } else { 1349 return k->load_extra_state(qbus->parent, f); 1350 } 1351 } 1352 1353 static void put_extra_state(QEMUFile *f, void *pv, size_t size) 1354 { 1355 VirtIODevice *vdev = pv; 1356 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1357 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1358 1359 k->save_extra_state(qbus->parent, f); 1360 } 1361 1362 static const VMStateInfo vmstate_info_extra_state = { 1363 .name = "virtqueue_extra_state", 1364 .get = get_extra_state, 1365 .put = put_extra_state, 1366 }; 1367 1368 static const VMStateDescription vmstate_virtio_extra_state = { 1369 .name = "virtio/extra_state", 1370 .version_id = 1, 1371 .minimum_version_id = 1, 1372 .needed = &virtio_extra_state_needed, 1373 .fields = (VMStateField[]) { 1374 { 1375 .name = "extra_state", 1376 .version_id = 0, 1377 .field_exists = NULL, 1378 .size = 0, 1379 .info = &vmstate_info_extra_state, 1380 .flags = VMS_SINGLE, 1381 .offset = 0, 1382 }, 1383 VMSTATE_END_OF_LIST() 1384 } 1385 }; 1386 1387 static const VMStateDescription vmstate_virtio_device_endian = { 1388 .name = "virtio/device_endian", 1389 .version_id = 1, 1390 .minimum_version_id = 1, 1391 .needed = &virtio_device_endian_needed, 1392 .fields = (VMStateField[]) { 1393 VMSTATE_UINT8(device_endian, VirtIODevice), 1394 VMSTATE_END_OF_LIST() 1395 } 1396 }; 1397 1398 static const VMStateDescription vmstate_virtio_64bit_features = { 1399 .name = "virtio/64bit_features", 1400 .version_id = 1, 1401 .minimum_version_id = 1, 1402 .needed = &virtio_64bit_features_needed, 1403 .fields = (VMStateField[]) { 1404 VMSTATE_UINT64(guest_features, VirtIODevice), 1405 VMSTATE_END_OF_LIST() 1406 } 1407 }; 1408 1409 static const VMStateDescription vmstate_virtio = { 1410 .name = "virtio", 1411 .version_id = 1, 1412 .minimum_version_id = 1, 1413 .minimum_version_id_old = 1, 1414 .fields = (VMStateField[]) { 1415 VMSTATE_END_OF_LIST() 1416 }, 1417 .subsections = (const VMStateDescription*[]) { 1418 &vmstate_virtio_device_endian, 1419 &vmstate_virtio_64bit_features, 1420 &vmstate_virtio_virtqueues, 1421 &vmstate_virtio_ringsize, 1422 &vmstate_virtio_extra_state, 1423 NULL 1424 } 1425 }; 1426 1427 void virtio_save(VirtIODevice *vdev, QEMUFile *f) 1428 { 1429 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1430 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1431 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1432 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 1433 int i; 1434 1435 if (k->save_config) { 1436 k->save_config(qbus->parent, f); 1437 } 1438 1439 qemu_put_8s(f, &vdev->status); 1440 qemu_put_8s(f, &vdev->isr); 1441 qemu_put_be16s(f, &vdev->queue_sel); 1442 qemu_put_be32s(f, &guest_features_lo); 1443 qemu_put_be32(f, vdev->config_len); 1444 qemu_put_buffer(f, vdev->config, vdev->config_len); 1445 1446 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1447 if (vdev->vq[i].vring.num == 0) 1448 break; 1449 } 1450 1451 qemu_put_be32(f, i); 1452 1453 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1454 if (vdev->vq[i].vring.num == 0) 1455 break; 1456 1457 qemu_put_be32(f, vdev->vq[i].vring.num); 1458 if (k->has_variable_vring_alignment) { 1459 qemu_put_be32(f, vdev->vq[i].vring.align); 1460 } 1461 /* XXX virtio-1 devices */ 1462 qemu_put_be64(f, vdev->vq[i].vring.desc); 1463 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 1464 if (k->save_queue) { 1465 k->save_queue(qbus->parent, i, f); 1466 } 1467 } 1468 1469 if (vdc->save != NULL) { 1470 vdc->save(vdev, f); 1471 } 1472 1473 /* Subsections */ 1474 vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 1475 } 1476 1477 /* A wrapper for use as a VMState .put function */ 1478 void virtio_vmstate_save(QEMUFile *f, void *opaque, size_t size) 1479 { 1480 virtio_save(VIRTIO_DEVICE(opaque), f); 1481 } 1482 1483 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 1484 { 1485 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1486 bool bad = (val & ~(vdev->host_features)) != 0; 1487 1488 val &= vdev->host_features; 1489 if (k->set_features) { 1490 k->set_features(vdev, val); 1491 } 1492 vdev->guest_features = val; 1493 return bad ? -1 : 0; 1494 } 1495 1496 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 1497 { 1498 /* 1499 * The driver must not attempt to set features after feature negotiation 1500 * has finished. 1501 */ 1502 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 1503 return -EINVAL; 1504 } 1505 return virtio_set_features_nocheck(vdev, val); 1506 } 1507 1508 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 1509 { 1510 int i, ret; 1511 int32_t config_len; 1512 uint32_t num; 1513 uint32_t features; 1514 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1515 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1516 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1517 1518 /* 1519 * We poison the endianness to ensure it does not get used before 1520 * subsections have been loaded. 1521 */ 1522 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 1523 1524 if (k->load_config) { 1525 ret = k->load_config(qbus->parent, f); 1526 if (ret) 1527 return ret; 1528 } 1529 1530 qemu_get_8s(f, &vdev->status); 1531 qemu_get_8s(f, &vdev->isr); 1532 qemu_get_be16s(f, &vdev->queue_sel); 1533 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 1534 return -1; 1535 } 1536 qemu_get_be32s(f, &features); 1537 1538 /* 1539 * Temporarily set guest_features low bits - needed by 1540 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 1541 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. 1542 * 1543 * Note: devices should always test host features in future - don't create 1544 * new dependencies like this. 1545 */ 1546 vdev->guest_features = features; 1547 1548 config_len = qemu_get_be32(f); 1549 1550 /* 1551 * There are cases where the incoming config can be bigger or smaller 1552 * than what we have; so load what we have space for, and skip 1553 * any excess that's in the stream. 1554 */ 1555 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 1556 1557 while (config_len > vdev->config_len) { 1558 qemu_get_byte(f); 1559 config_len--; 1560 } 1561 1562 num = qemu_get_be32(f); 1563 1564 if (num > VIRTIO_QUEUE_MAX) { 1565 error_report("Invalid number of virtqueues: 0x%x", num); 1566 return -1; 1567 } 1568 1569 for (i = 0; i < num; i++) { 1570 vdev->vq[i].vring.num = qemu_get_be32(f); 1571 if (k->has_variable_vring_alignment) { 1572 vdev->vq[i].vring.align = qemu_get_be32(f); 1573 } 1574 vdev->vq[i].vring.desc = qemu_get_be64(f); 1575 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 1576 vdev->vq[i].signalled_used_valid = false; 1577 vdev->vq[i].notification = true; 1578 1579 if (vdev->vq[i].vring.desc) { 1580 /* XXX virtio-1 devices */ 1581 virtio_queue_update_rings(vdev, i); 1582 } else if (vdev->vq[i].last_avail_idx) { 1583 error_report("VQ %d address 0x0 " 1584 "inconsistent with Host index 0x%x", 1585 i, vdev->vq[i].last_avail_idx); 1586 return -1; 1587 } 1588 if (k->load_queue) { 1589 ret = k->load_queue(qbus->parent, i, f); 1590 if (ret) 1591 return ret; 1592 } 1593 } 1594 1595 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 1596 1597 if (vdc->load != NULL) { 1598 ret = vdc->load(vdev, f, version_id); 1599 if (ret) { 1600 return ret; 1601 } 1602 } 1603 1604 /* Subsections */ 1605 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 1606 if (ret) { 1607 return ret; 1608 } 1609 1610 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 1611 vdev->device_endian = virtio_default_endian(); 1612 } 1613 1614 if (virtio_64bit_features_needed(vdev)) { 1615 /* 1616 * Subsection load filled vdev->guest_features. Run them 1617 * through virtio_set_features to sanity-check them against 1618 * host_features. 1619 */ 1620 uint64_t features64 = vdev->guest_features; 1621 if (virtio_set_features_nocheck(vdev, features64) < 0) { 1622 error_report("Features 0x%" PRIx64 " unsupported. " 1623 "Allowed features: 0x%" PRIx64, 1624 features64, vdev->host_features); 1625 return -1; 1626 } 1627 } else { 1628 if (virtio_set_features_nocheck(vdev, features) < 0) { 1629 error_report("Features 0x%x unsupported. " 1630 "Allowed features: 0x%" PRIx64, 1631 features, vdev->host_features); 1632 return -1; 1633 } 1634 } 1635 1636 for (i = 0; i < num; i++) { 1637 if (vdev->vq[i].vring.desc) { 1638 uint16_t nheads; 1639 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 1640 /* Check it isn't doing strange things with descriptor numbers. */ 1641 if (nheads > vdev->vq[i].vring.num) { 1642 error_report("VQ %d size 0x%x Guest index 0x%x " 1643 "inconsistent with Host index 0x%x: delta 0x%x", 1644 i, vdev->vq[i].vring.num, 1645 vring_avail_idx(&vdev->vq[i]), 1646 vdev->vq[i].last_avail_idx, nheads); 1647 return -1; 1648 } 1649 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); 1650 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); 1651 } 1652 } 1653 1654 return 0; 1655 } 1656 1657 void virtio_cleanup(VirtIODevice *vdev) 1658 { 1659 qemu_del_vm_change_state_handler(vdev->vmstate); 1660 g_free(vdev->config); 1661 g_free(vdev->vq); 1662 g_free(vdev->vector_queues); 1663 } 1664 1665 static void virtio_vmstate_change(void *opaque, int running, RunState state) 1666 { 1667 VirtIODevice *vdev = opaque; 1668 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1669 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1670 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK); 1671 vdev->vm_running = running; 1672 1673 if (backend_run) { 1674 virtio_set_status(vdev, vdev->status); 1675 } 1676 1677 if (k->vmstate_change) { 1678 k->vmstate_change(qbus->parent, backend_run); 1679 } 1680 1681 if (!backend_run) { 1682 virtio_set_status(vdev, vdev->status); 1683 } 1684 } 1685 1686 void virtio_instance_init_common(Object *proxy_obj, void *data, 1687 size_t vdev_size, const char *vdev_name) 1688 { 1689 DeviceState *vdev = data; 1690 1691 object_initialize(vdev, vdev_size, vdev_name); 1692 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL); 1693 object_unref(OBJECT(vdev)); 1694 qdev_alias_all_properties(vdev, proxy_obj); 1695 } 1696 1697 void virtio_init(VirtIODevice *vdev, const char *name, 1698 uint16_t device_id, size_t config_size) 1699 { 1700 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1701 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1702 int i; 1703 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 1704 1705 if (nvectors) { 1706 vdev->vector_queues = 1707 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 1708 } 1709 1710 vdev->device_id = device_id; 1711 vdev->status = 0; 1712 vdev->isr = 0; 1713 vdev->queue_sel = 0; 1714 vdev->config_vector = VIRTIO_NO_VECTOR; 1715 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX); 1716 vdev->vm_running = runstate_is_running(); 1717 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1718 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 1719 vdev->vq[i].vdev = vdev; 1720 vdev->vq[i].queue_index = i; 1721 } 1722 1723 vdev->name = name; 1724 vdev->config_len = config_size; 1725 if (vdev->config_len) { 1726 vdev->config = g_malloc0(config_size); 1727 } else { 1728 vdev->config = NULL; 1729 } 1730 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, 1731 vdev); 1732 vdev->device_endian = virtio_default_endian(); 1733 vdev->use_guest_notifier_mask = true; 1734 } 1735 1736 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 1737 { 1738 return vdev->vq[n].vring.desc; 1739 } 1740 1741 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 1742 { 1743 return vdev->vq[n].vring.avail; 1744 } 1745 1746 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 1747 { 1748 return vdev->vq[n].vring.used; 1749 } 1750 1751 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n) 1752 { 1753 return vdev->vq[n].vring.desc; 1754 } 1755 1756 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 1757 { 1758 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 1759 } 1760 1761 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 1762 { 1763 return offsetof(VRingAvail, ring) + 1764 sizeof(uint16_t) * vdev->vq[n].vring.num; 1765 } 1766 1767 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 1768 { 1769 return offsetof(VRingUsed, ring) + 1770 sizeof(VRingUsedElem) * vdev->vq[n].vring.num; 1771 } 1772 1773 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n) 1774 { 1775 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc + 1776 virtio_queue_get_used_size(vdev, n); 1777 } 1778 1779 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 1780 { 1781 return vdev->vq[n].last_avail_idx; 1782 } 1783 1784 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx) 1785 { 1786 vdev->vq[n].last_avail_idx = idx; 1787 vdev->vq[n].shadow_avail_idx = idx; 1788 } 1789 1790 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 1791 { 1792 vdev->vq[n].signalled_used_valid = false; 1793 } 1794 1795 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 1796 { 1797 return vdev->vq + n; 1798 } 1799 1800 uint16_t virtio_get_queue_index(VirtQueue *vq) 1801 { 1802 return vq->queue_index; 1803 } 1804 1805 static void virtio_queue_guest_notifier_read(EventNotifier *n) 1806 { 1807 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 1808 if (event_notifier_test_and_clear(n)) { 1809 virtio_irq(vq); 1810 } 1811 } 1812 1813 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 1814 bool with_irqfd) 1815 { 1816 if (assign && !with_irqfd) { 1817 event_notifier_set_handler(&vq->guest_notifier, false, 1818 virtio_queue_guest_notifier_read); 1819 } else { 1820 event_notifier_set_handler(&vq->guest_notifier, false, NULL); 1821 } 1822 if (!assign) { 1823 /* Test and clear notifier before closing it, 1824 * in case poll callback didn't have time to run. */ 1825 virtio_queue_guest_notifier_read(&vq->guest_notifier); 1826 } 1827 } 1828 1829 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 1830 { 1831 return &vq->guest_notifier; 1832 } 1833 1834 static void virtio_queue_host_notifier_aio_read(EventNotifier *n) 1835 { 1836 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 1837 if (event_notifier_test_and_clear(n)) { 1838 virtio_queue_notify_aio_vq(vq); 1839 } 1840 } 1841 1842 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx, 1843 VirtIOHandleOutput handle_output) 1844 { 1845 if (handle_output) { 1846 vq->handle_aio_output = handle_output; 1847 aio_set_event_notifier(ctx, &vq->host_notifier, true, 1848 virtio_queue_host_notifier_aio_read); 1849 } else { 1850 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL); 1851 /* Test and clear notifier before after disabling event, 1852 * in case poll callback didn't have time to run. */ 1853 virtio_queue_host_notifier_aio_read(&vq->host_notifier); 1854 vq->handle_aio_output = NULL; 1855 } 1856 } 1857 1858 static void virtio_queue_host_notifier_read(EventNotifier *n) 1859 { 1860 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 1861 if (event_notifier_test_and_clear(n)) { 1862 virtio_queue_notify_vq(vq); 1863 } 1864 } 1865 1866 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign, 1867 bool set_handler) 1868 { 1869 AioContext *ctx = qemu_get_aio_context(); 1870 if (assign && set_handler) { 1871 if (vq->use_aio) { 1872 aio_set_event_notifier(ctx, &vq->host_notifier, true, 1873 virtio_queue_host_notifier_read); 1874 } else { 1875 event_notifier_set_handler(&vq->host_notifier, true, 1876 virtio_queue_host_notifier_read); 1877 } 1878 } else { 1879 if (vq->use_aio) { 1880 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL); 1881 } else { 1882 event_notifier_set_handler(&vq->host_notifier, true, NULL); 1883 } 1884 } 1885 if (!assign) { 1886 /* Test and clear notifier before after disabling event, 1887 * in case poll callback didn't have time to run. */ 1888 virtio_queue_host_notifier_read(&vq->host_notifier); 1889 } 1890 } 1891 1892 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 1893 { 1894 return &vq->host_notifier; 1895 } 1896 1897 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 1898 { 1899 g_free(vdev->bus_name); 1900 vdev->bus_name = g_strdup(bus_name); 1901 } 1902 1903 static void virtio_device_realize(DeviceState *dev, Error **errp) 1904 { 1905 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1906 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 1907 Error *err = NULL; 1908 1909 if (vdc->realize != NULL) { 1910 vdc->realize(dev, &err); 1911 if (err != NULL) { 1912 error_propagate(errp, err); 1913 return; 1914 } 1915 } 1916 1917 virtio_bus_device_plugged(vdev, &err); 1918 if (err != NULL) { 1919 error_propagate(errp, err); 1920 return; 1921 } 1922 } 1923 1924 static void virtio_device_unrealize(DeviceState *dev, Error **errp) 1925 { 1926 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1927 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 1928 Error *err = NULL; 1929 1930 virtio_bus_device_unplugged(vdev); 1931 1932 if (vdc->unrealize != NULL) { 1933 vdc->unrealize(dev, &err); 1934 if (err != NULL) { 1935 error_propagate(errp, err); 1936 return; 1937 } 1938 } 1939 1940 g_free(vdev->bus_name); 1941 vdev->bus_name = NULL; 1942 } 1943 1944 static Property virtio_properties[] = { 1945 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 1946 DEFINE_PROP_END_OF_LIST(), 1947 }; 1948 1949 static void virtio_device_class_init(ObjectClass *klass, void *data) 1950 { 1951 /* Set the default value here. */ 1952 DeviceClass *dc = DEVICE_CLASS(klass); 1953 1954 dc->realize = virtio_device_realize; 1955 dc->unrealize = virtio_device_unrealize; 1956 dc->bus_type = TYPE_VIRTIO_BUS; 1957 dc->props = virtio_properties; 1958 } 1959 1960 static const TypeInfo virtio_device_info = { 1961 .name = TYPE_VIRTIO_DEVICE, 1962 .parent = TYPE_DEVICE, 1963 .instance_size = sizeof(VirtIODevice), 1964 .class_init = virtio_device_class_init, 1965 .abstract = true, 1966 .class_size = sizeof(VirtioDeviceClass), 1967 }; 1968 1969 static void virtio_register_types(void) 1970 { 1971 type_register_static(&virtio_device_info); 1972 } 1973 1974 type_init(virtio_register_types) 1975