1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include <inttypes.h> 15 16 #include "trace.h" 17 #include "exec/address-spaces.h" 18 #include "qemu/error-report.h" 19 #include "hw/virtio/virtio.h" 20 #include "qemu/atomic.h" 21 #include "hw/virtio/virtio-bus.h" 22 #include "migration/migration.h" 23 #include "hw/virtio/virtio-access.h" 24 25 /* 26 * The alignment to use between consumer and producer parts of vring. 27 * x86 pagesize again. This is the default, used by transports like PCI 28 * which don't provide a means for the guest to tell the host the alignment. 29 */ 30 #define VIRTIO_PCI_VRING_ALIGN 4096 31 32 typedef struct VRingDesc 33 { 34 uint64_t addr; 35 uint32_t len; 36 uint16_t flags; 37 uint16_t next; 38 } VRingDesc; 39 40 typedef struct VRingAvail 41 { 42 uint16_t flags; 43 uint16_t idx; 44 uint16_t ring[0]; 45 } VRingAvail; 46 47 typedef struct VRingUsedElem 48 { 49 uint32_t id; 50 uint32_t len; 51 } VRingUsedElem; 52 53 typedef struct VRingUsed 54 { 55 uint16_t flags; 56 uint16_t idx; 57 VRingUsedElem ring[0]; 58 } VRingUsed; 59 60 typedef struct VRing 61 { 62 unsigned int num; 63 unsigned int num_default; 64 unsigned int align; 65 hwaddr desc; 66 hwaddr avail; 67 hwaddr used; 68 } VRing; 69 70 struct VirtQueue 71 { 72 VRing vring; 73 uint16_t last_avail_idx; 74 /* Last used index value we have signalled on */ 75 uint16_t signalled_used; 76 77 /* Last used index value we have signalled on */ 78 bool signalled_used_valid; 79 80 /* Notification enabled? */ 81 bool notification; 82 83 uint16_t queue_index; 84 85 int inuse; 86 87 uint16_t vector; 88 void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq); 89 VirtIODevice *vdev; 90 EventNotifier guest_notifier; 91 EventNotifier host_notifier; 92 QLIST_ENTRY(VirtQueue) node; 93 }; 94 95 /* virt queue functions */ 96 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 97 { 98 VRing *vring = &vdev->vq[n].vring; 99 100 if (!vring->desc) { 101 /* not yet setup -> nothing to do */ 102 return; 103 } 104 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 105 vring->used = vring_align(vring->avail + 106 offsetof(VRingAvail, ring[vring->num]), 107 vring->align); 108 } 109 110 static inline uint64_t vring_desc_addr(VirtIODevice *vdev, hwaddr desc_pa, 111 int i) 112 { 113 hwaddr pa; 114 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr); 115 return virtio_ldq_phys(vdev, pa); 116 } 117 118 static inline uint32_t vring_desc_len(VirtIODevice *vdev, hwaddr desc_pa, int i) 119 { 120 hwaddr pa; 121 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len); 122 return virtio_ldl_phys(vdev, pa); 123 } 124 125 static inline uint16_t vring_desc_flags(VirtIODevice *vdev, hwaddr desc_pa, 126 int i) 127 { 128 hwaddr pa; 129 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags); 130 return virtio_lduw_phys(vdev, pa); 131 } 132 133 static inline uint16_t vring_desc_next(VirtIODevice *vdev, hwaddr desc_pa, 134 int i) 135 { 136 hwaddr pa; 137 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next); 138 return virtio_lduw_phys(vdev, pa); 139 } 140 141 static inline uint16_t vring_avail_flags(VirtQueue *vq) 142 { 143 hwaddr pa; 144 pa = vq->vring.avail + offsetof(VRingAvail, flags); 145 return virtio_lduw_phys(vq->vdev, pa); 146 } 147 148 static inline uint16_t vring_avail_idx(VirtQueue *vq) 149 { 150 hwaddr pa; 151 pa = vq->vring.avail + offsetof(VRingAvail, idx); 152 return virtio_lduw_phys(vq->vdev, pa); 153 } 154 155 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 156 { 157 hwaddr pa; 158 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]); 159 return virtio_lduw_phys(vq->vdev, pa); 160 } 161 162 static inline uint16_t vring_get_used_event(VirtQueue *vq) 163 { 164 return vring_avail_ring(vq, vq->vring.num); 165 } 166 167 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val) 168 { 169 hwaddr pa; 170 pa = vq->vring.used + offsetof(VRingUsed, ring[i].id); 171 virtio_stl_phys(vq->vdev, pa, val); 172 } 173 174 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val) 175 { 176 hwaddr pa; 177 pa = vq->vring.used + offsetof(VRingUsed, ring[i].len); 178 virtio_stl_phys(vq->vdev, pa, val); 179 } 180 181 static uint16_t vring_used_idx(VirtQueue *vq) 182 { 183 hwaddr pa; 184 pa = vq->vring.used + offsetof(VRingUsed, idx); 185 return virtio_lduw_phys(vq->vdev, pa); 186 } 187 188 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 189 { 190 hwaddr pa; 191 pa = vq->vring.used + offsetof(VRingUsed, idx); 192 virtio_stw_phys(vq->vdev, pa, val); 193 } 194 195 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 196 { 197 VirtIODevice *vdev = vq->vdev; 198 hwaddr pa; 199 pa = vq->vring.used + offsetof(VRingUsed, flags); 200 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask); 201 } 202 203 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 204 { 205 VirtIODevice *vdev = vq->vdev; 206 hwaddr pa; 207 pa = vq->vring.used + offsetof(VRingUsed, flags); 208 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask); 209 } 210 211 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 212 { 213 hwaddr pa; 214 if (!vq->notification) { 215 return; 216 } 217 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]); 218 virtio_stw_phys(vq->vdev, pa, val); 219 } 220 221 void virtio_queue_set_notification(VirtQueue *vq, int enable) 222 { 223 vq->notification = enable; 224 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 225 vring_set_avail_event(vq, vring_avail_idx(vq)); 226 } else if (enable) { 227 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 228 } else { 229 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 230 } 231 if (enable) { 232 /* Expose avail event/used flags before caller checks the avail idx. */ 233 smp_mb(); 234 } 235 } 236 237 int virtio_queue_ready(VirtQueue *vq) 238 { 239 return vq->vring.avail != 0; 240 } 241 242 int virtio_queue_empty(VirtQueue *vq) 243 { 244 return vring_avail_idx(vq) == vq->last_avail_idx; 245 } 246 247 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 248 unsigned int len) 249 { 250 unsigned int offset; 251 int i; 252 253 offset = 0; 254 for (i = 0; i < elem->in_num; i++) { 255 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 256 257 cpu_physical_memory_unmap(elem->in_sg[i].iov_base, 258 elem->in_sg[i].iov_len, 259 1, size); 260 261 offset += size; 262 } 263 264 for (i = 0; i < elem->out_num; i++) 265 cpu_physical_memory_unmap(elem->out_sg[i].iov_base, 266 elem->out_sg[i].iov_len, 267 0, elem->out_sg[i].iov_len); 268 } 269 270 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem, 271 unsigned int len) 272 { 273 vq->last_avail_idx--; 274 virtqueue_unmap_sg(vq, elem, len); 275 } 276 277 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 278 unsigned int len, unsigned int idx) 279 { 280 trace_virtqueue_fill(vq, elem, len, idx); 281 282 virtqueue_unmap_sg(vq, elem, len); 283 284 idx = (idx + vring_used_idx(vq)) % vq->vring.num; 285 286 /* Get a pointer to the next entry in the used ring. */ 287 vring_used_ring_id(vq, idx, elem->index); 288 vring_used_ring_len(vq, idx, len); 289 } 290 291 void virtqueue_flush(VirtQueue *vq, unsigned int count) 292 { 293 uint16_t old, new; 294 /* Make sure buffer is written before we update index. */ 295 smp_wmb(); 296 trace_virtqueue_flush(vq, count); 297 old = vring_used_idx(vq); 298 new = old + count; 299 vring_used_idx_set(vq, new); 300 vq->inuse -= count; 301 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 302 vq->signalled_used_valid = false; 303 } 304 305 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 306 unsigned int len) 307 { 308 virtqueue_fill(vq, elem, len, 0); 309 virtqueue_flush(vq, 1); 310 } 311 312 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 313 { 314 uint16_t num_heads = vring_avail_idx(vq) - idx; 315 316 /* Check it isn't doing very strange things with descriptor numbers. */ 317 if (num_heads > vq->vring.num) { 318 error_report("Guest moved used index from %u to %u", 319 idx, vring_avail_idx(vq)); 320 exit(1); 321 } 322 /* On success, callers read a descriptor at vq->last_avail_idx. 323 * Make sure descriptor read does not bypass avail index read. */ 324 if (num_heads) { 325 smp_rmb(); 326 } 327 328 return num_heads; 329 } 330 331 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx) 332 { 333 unsigned int head; 334 335 /* Grab the next descriptor number they're advertising, and increment 336 * the index we've seen. */ 337 head = vring_avail_ring(vq, idx % vq->vring.num); 338 339 /* If their number is silly, that's a fatal mistake. */ 340 if (head >= vq->vring.num) { 341 error_report("Guest says index %u is available", head); 342 exit(1); 343 } 344 345 return head; 346 } 347 348 static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa, 349 unsigned int i, unsigned int max) 350 { 351 unsigned int next; 352 353 /* If this descriptor says it doesn't chain, we're done. */ 354 if (!(vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_NEXT)) { 355 return max; 356 } 357 358 /* Check they're not leading us off end of descriptors. */ 359 next = vring_desc_next(vdev, desc_pa, i); 360 /* Make sure compiler knows to grab that: we don't want it changing! */ 361 smp_wmb(); 362 363 if (next >= max) { 364 error_report("Desc next is %u", next); 365 exit(1); 366 } 367 368 return next; 369 } 370 371 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 372 unsigned int *out_bytes, 373 unsigned max_in_bytes, unsigned max_out_bytes) 374 { 375 unsigned int idx; 376 unsigned int total_bufs, in_total, out_total; 377 378 idx = vq->last_avail_idx; 379 380 total_bufs = in_total = out_total = 0; 381 while (virtqueue_num_heads(vq, idx)) { 382 VirtIODevice *vdev = vq->vdev; 383 unsigned int max, num_bufs, indirect = 0; 384 hwaddr desc_pa; 385 int i; 386 387 max = vq->vring.num; 388 num_bufs = total_bufs; 389 i = virtqueue_get_head(vq, idx++); 390 desc_pa = vq->vring.desc; 391 392 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) { 393 if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) { 394 error_report("Invalid size for indirect buffer table"); 395 exit(1); 396 } 397 398 /* If we've got too many, that implies a descriptor loop. */ 399 if (num_bufs >= max) { 400 error_report("Looped descriptor"); 401 exit(1); 402 } 403 404 /* loop over the indirect descriptor table */ 405 indirect = 1; 406 max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc); 407 desc_pa = vring_desc_addr(vdev, desc_pa, i); 408 num_bufs = i = 0; 409 } 410 411 do { 412 /* If we've got too many, that implies a descriptor loop. */ 413 if (++num_bufs > max) { 414 error_report("Looped descriptor"); 415 exit(1); 416 } 417 418 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) { 419 in_total += vring_desc_len(vdev, desc_pa, i); 420 } else { 421 out_total += vring_desc_len(vdev, desc_pa, i); 422 } 423 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 424 goto done; 425 } 426 } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max); 427 428 if (!indirect) 429 total_bufs = num_bufs; 430 else 431 total_bufs++; 432 } 433 done: 434 if (in_bytes) { 435 *in_bytes = in_total; 436 } 437 if (out_bytes) { 438 *out_bytes = out_total; 439 } 440 } 441 442 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 443 unsigned int out_bytes) 444 { 445 unsigned int in_total, out_total; 446 447 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 448 return in_bytes <= in_total && out_bytes <= out_total; 449 } 450 451 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr, 452 unsigned int *num_sg, unsigned int max_size, 453 int is_write) 454 { 455 unsigned int i; 456 hwaddr len; 457 458 /* Note: this function MUST validate input, some callers 459 * are passing in num_sg values received over the network. 460 */ 461 /* TODO: teach all callers that this can fail, and return failure instead 462 * of asserting here. 463 * When we do, we might be able to re-enable NDEBUG below. 464 */ 465 #ifdef NDEBUG 466 #error building with NDEBUG is not supported 467 #endif 468 assert(*num_sg <= max_size); 469 470 for (i = 0; i < *num_sg; i++) { 471 len = sg[i].iov_len; 472 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write); 473 if (!sg[i].iov_base) { 474 error_report("virtio: error trying to map MMIO memory"); 475 exit(1); 476 } 477 if (len == sg[i].iov_len) { 478 continue; 479 } 480 if (*num_sg >= max_size) { 481 error_report("virtio: memory split makes iovec too large"); 482 exit(1); 483 } 484 memmove(sg + i + 1, sg + i, sizeof(*sg) * (*num_sg - i)); 485 memmove(addr + i + 1, addr + i, sizeof(*addr) * (*num_sg - i)); 486 assert(len < sg[i + 1].iov_len); 487 sg[i].iov_len = len; 488 addr[i + 1] += len; 489 sg[i + 1].iov_len -= len; 490 ++*num_sg; 491 } 492 } 493 494 void virtqueue_map(VirtQueueElement *elem) 495 { 496 virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num, 497 MIN(ARRAY_SIZE(elem->in_sg), ARRAY_SIZE(elem->in_addr)), 498 1); 499 virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num, 500 MIN(ARRAY_SIZE(elem->out_sg), ARRAY_SIZE(elem->out_addr)), 501 0); 502 } 503 504 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem) 505 { 506 unsigned int i, head, max; 507 hwaddr desc_pa = vq->vring.desc; 508 VirtIODevice *vdev = vq->vdev; 509 510 if (!virtqueue_num_heads(vq, vq->last_avail_idx)) 511 return 0; 512 513 /* When we start there are none of either input nor output. */ 514 elem->out_num = elem->in_num = 0; 515 516 max = vq->vring.num; 517 518 i = head = virtqueue_get_head(vq, vq->last_avail_idx++); 519 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 520 vring_set_avail_event(vq, vq->last_avail_idx); 521 } 522 523 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) { 524 if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) { 525 error_report("Invalid size for indirect buffer table"); 526 exit(1); 527 } 528 529 /* loop over the indirect descriptor table */ 530 max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc); 531 desc_pa = vring_desc_addr(vdev, desc_pa, i); 532 i = 0; 533 } 534 535 /* Collect all the descriptors */ 536 do { 537 struct iovec *sg; 538 539 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) { 540 if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) { 541 error_report("Too many write descriptors in indirect table"); 542 exit(1); 543 } 544 elem->in_addr[elem->in_num] = vring_desc_addr(vdev, desc_pa, i); 545 sg = &elem->in_sg[elem->in_num++]; 546 } else { 547 if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) { 548 error_report("Too many read descriptors in indirect table"); 549 exit(1); 550 } 551 elem->out_addr[elem->out_num] = vring_desc_addr(vdev, desc_pa, i); 552 sg = &elem->out_sg[elem->out_num++]; 553 } 554 555 sg->iov_len = vring_desc_len(vdev, desc_pa, i); 556 557 /* If we've got too many, that implies a descriptor loop. */ 558 if ((elem->in_num + elem->out_num) > max) { 559 error_report("Looped descriptor"); 560 exit(1); 561 } 562 } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max); 563 564 /* Now map what we have collected */ 565 virtqueue_map(elem); 566 567 elem->index = head; 568 569 vq->inuse++; 570 571 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 572 return elem->in_num + elem->out_num; 573 } 574 575 /* virtio device */ 576 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 577 { 578 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 579 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 580 581 if (k->notify) { 582 k->notify(qbus->parent, vector); 583 } 584 } 585 586 void virtio_update_irq(VirtIODevice *vdev) 587 { 588 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 589 } 590 591 static int virtio_validate_features(VirtIODevice *vdev) 592 { 593 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 594 595 if (k->validate_features) { 596 return k->validate_features(vdev); 597 } else { 598 return 0; 599 } 600 } 601 602 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 603 { 604 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 605 trace_virtio_set_status(vdev, val); 606 607 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 608 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 609 val & VIRTIO_CONFIG_S_FEATURES_OK) { 610 int ret = virtio_validate_features(vdev); 611 612 if (ret) { 613 return ret; 614 } 615 } 616 } 617 if (k->set_status) { 618 k->set_status(vdev, val); 619 } 620 vdev->status = val; 621 return 0; 622 } 623 624 bool target_words_bigendian(void); 625 static enum virtio_device_endian virtio_default_endian(void) 626 { 627 if (target_words_bigendian()) { 628 return VIRTIO_DEVICE_ENDIAN_BIG; 629 } else { 630 return VIRTIO_DEVICE_ENDIAN_LITTLE; 631 } 632 } 633 634 static enum virtio_device_endian virtio_current_cpu_endian(void) 635 { 636 CPUClass *cc = CPU_GET_CLASS(current_cpu); 637 638 if (cc->virtio_is_big_endian(current_cpu)) { 639 return VIRTIO_DEVICE_ENDIAN_BIG; 640 } else { 641 return VIRTIO_DEVICE_ENDIAN_LITTLE; 642 } 643 } 644 645 void virtio_reset(void *opaque) 646 { 647 VirtIODevice *vdev = opaque; 648 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 649 int i; 650 651 virtio_set_status(vdev, 0); 652 if (current_cpu) { 653 /* Guest initiated reset */ 654 vdev->device_endian = virtio_current_cpu_endian(); 655 } else { 656 /* System reset */ 657 vdev->device_endian = virtio_default_endian(); 658 } 659 660 if (k->reset) { 661 k->reset(vdev); 662 } 663 664 vdev->guest_features = 0; 665 vdev->queue_sel = 0; 666 vdev->status = 0; 667 vdev->isr = 0; 668 vdev->config_vector = VIRTIO_NO_VECTOR; 669 virtio_notify_vector(vdev, vdev->config_vector); 670 671 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 672 vdev->vq[i].vring.desc = 0; 673 vdev->vq[i].vring.avail = 0; 674 vdev->vq[i].vring.used = 0; 675 vdev->vq[i].last_avail_idx = 0; 676 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 677 vdev->vq[i].signalled_used = 0; 678 vdev->vq[i].signalled_used_valid = false; 679 vdev->vq[i].notification = true; 680 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 681 } 682 } 683 684 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 685 { 686 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 687 uint8_t val; 688 689 if (addr + sizeof(val) > vdev->config_len) { 690 return (uint32_t)-1; 691 } 692 693 k->get_config(vdev, vdev->config); 694 695 val = ldub_p(vdev->config + addr); 696 return val; 697 } 698 699 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 700 { 701 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 702 uint16_t val; 703 704 if (addr + sizeof(val) > vdev->config_len) { 705 return (uint32_t)-1; 706 } 707 708 k->get_config(vdev, vdev->config); 709 710 val = lduw_p(vdev->config + addr); 711 return val; 712 } 713 714 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 715 { 716 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 717 uint32_t val; 718 719 if (addr + sizeof(val) > vdev->config_len) { 720 return (uint32_t)-1; 721 } 722 723 k->get_config(vdev, vdev->config); 724 725 val = ldl_p(vdev->config + addr); 726 return val; 727 } 728 729 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 730 { 731 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 732 uint8_t val = data; 733 734 if (addr + sizeof(val) > vdev->config_len) { 735 return; 736 } 737 738 stb_p(vdev->config + addr, val); 739 740 if (k->set_config) { 741 k->set_config(vdev, vdev->config); 742 } 743 } 744 745 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 746 { 747 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 748 uint16_t val = data; 749 750 if (addr + sizeof(val) > vdev->config_len) { 751 return; 752 } 753 754 stw_p(vdev->config + addr, val); 755 756 if (k->set_config) { 757 k->set_config(vdev, vdev->config); 758 } 759 } 760 761 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 762 { 763 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 764 uint32_t val = data; 765 766 if (addr + sizeof(val) > vdev->config_len) { 767 return; 768 } 769 770 stl_p(vdev->config + addr, val); 771 772 if (k->set_config) { 773 k->set_config(vdev, vdev->config); 774 } 775 } 776 777 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 778 { 779 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 780 uint8_t val; 781 782 if (addr + sizeof(val) > vdev->config_len) { 783 return (uint32_t)-1; 784 } 785 786 k->get_config(vdev, vdev->config); 787 788 val = ldub_p(vdev->config + addr); 789 return val; 790 } 791 792 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 793 { 794 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 795 uint16_t val; 796 797 if (addr + sizeof(val) > vdev->config_len) { 798 return (uint32_t)-1; 799 } 800 801 k->get_config(vdev, vdev->config); 802 803 val = lduw_le_p(vdev->config + addr); 804 return val; 805 } 806 807 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 808 { 809 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 810 uint32_t val; 811 812 if (addr + sizeof(val) > vdev->config_len) { 813 return (uint32_t)-1; 814 } 815 816 k->get_config(vdev, vdev->config); 817 818 val = ldl_le_p(vdev->config + addr); 819 return val; 820 } 821 822 void virtio_config_modern_writeb(VirtIODevice *vdev, 823 uint32_t addr, uint32_t data) 824 { 825 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 826 uint8_t val = data; 827 828 if (addr + sizeof(val) > vdev->config_len) { 829 return; 830 } 831 832 stb_p(vdev->config + addr, val); 833 834 if (k->set_config) { 835 k->set_config(vdev, vdev->config); 836 } 837 } 838 839 void virtio_config_modern_writew(VirtIODevice *vdev, 840 uint32_t addr, uint32_t data) 841 { 842 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 843 uint16_t val = data; 844 845 if (addr + sizeof(val) > vdev->config_len) { 846 return; 847 } 848 849 stw_le_p(vdev->config + addr, val); 850 851 if (k->set_config) { 852 k->set_config(vdev, vdev->config); 853 } 854 } 855 856 void virtio_config_modern_writel(VirtIODevice *vdev, 857 uint32_t addr, uint32_t data) 858 { 859 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 860 uint32_t val = data; 861 862 if (addr + sizeof(val) > vdev->config_len) { 863 return; 864 } 865 866 stl_le_p(vdev->config + addr, val); 867 868 if (k->set_config) { 869 k->set_config(vdev, vdev->config); 870 } 871 } 872 873 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 874 { 875 vdev->vq[n].vring.desc = addr; 876 virtio_queue_update_rings(vdev, n); 877 } 878 879 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 880 { 881 return vdev->vq[n].vring.desc; 882 } 883 884 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 885 hwaddr avail, hwaddr used) 886 { 887 vdev->vq[n].vring.desc = desc; 888 vdev->vq[n].vring.avail = avail; 889 vdev->vq[n].vring.used = used; 890 } 891 892 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 893 { 894 /* Don't allow guest to flip queue between existent and 895 * nonexistent states, or to set it to an invalid size. 896 */ 897 if (!!num != !!vdev->vq[n].vring.num || 898 num > VIRTQUEUE_MAX_SIZE || 899 num < 0) { 900 return; 901 } 902 vdev->vq[n].vring.num = num; 903 } 904 905 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 906 { 907 return QLIST_FIRST(&vdev->vector_queues[vector]); 908 } 909 910 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 911 { 912 return QLIST_NEXT(vq, node); 913 } 914 915 int virtio_queue_get_num(VirtIODevice *vdev, int n) 916 { 917 return vdev->vq[n].vring.num; 918 } 919 920 int virtio_get_num_queues(VirtIODevice *vdev) 921 { 922 int i; 923 924 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 925 if (!virtio_queue_get_num(vdev, i)) { 926 break; 927 } 928 } 929 930 return i; 931 } 932 933 int virtio_queue_get_id(VirtQueue *vq) 934 { 935 VirtIODevice *vdev = vq->vdev; 936 assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_QUEUE_MAX]); 937 return vq - &vdev->vq[0]; 938 } 939 940 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 941 { 942 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 943 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 944 945 /* virtio-1 compliant devices cannot change the alignment */ 946 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 947 error_report("tried to modify queue alignment for virtio-1 device"); 948 return; 949 } 950 /* Check that the transport told us it was going to do this 951 * (so a buggy transport will immediately assert rather than 952 * silently failing to migrate this state) 953 */ 954 assert(k->has_variable_vring_alignment); 955 956 vdev->vq[n].vring.align = align; 957 virtio_queue_update_rings(vdev, n); 958 } 959 960 void virtio_queue_notify_vq(VirtQueue *vq) 961 { 962 if (vq->vring.desc && vq->handle_output) { 963 VirtIODevice *vdev = vq->vdev; 964 965 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 966 vq->handle_output(vdev, vq); 967 } 968 } 969 970 void virtio_queue_notify(VirtIODevice *vdev, int n) 971 { 972 virtio_queue_notify_vq(&vdev->vq[n]); 973 } 974 975 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 976 { 977 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 978 VIRTIO_NO_VECTOR; 979 } 980 981 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 982 { 983 VirtQueue *vq = &vdev->vq[n]; 984 985 if (n < VIRTIO_QUEUE_MAX) { 986 if (vdev->vector_queues && 987 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 988 QLIST_REMOVE(vq, node); 989 } 990 vdev->vq[n].vector = vector; 991 if (vdev->vector_queues && 992 vector != VIRTIO_NO_VECTOR) { 993 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 994 } 995 } 996 } 997 998 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 999 void (*handle_output)(VirtIODevice *, VirtQueue *)) 1000 { 1001 int i; 1002 1003 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1004 if (vdev->vq[i].vring.num == 0) 1005 break; 1006 } 1007 1008 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 1009 abort(); 1010 1011 vdev->vq[i].vring.num = queue_size; 1012 vdev->vq[i].vring.num_default = queue_size; 1013 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 1014 vdev->vq[i].handle_output = handle_output; 1015 1016 return &vdev->vq[i]; 1017 } 1018 1019 void virtio_del_queue(VirtIODevice *vdev, int n) 1020 { 1021 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 1022 abort(); 1023 } 1024 1025 vdev->vq[n].vring.num = 0; 1026 vdev->vq[n].vring.num_default = 0; 1027 } 1028 1029 void virtio_irq(VirtQueue *vq) 1030 { 1031 trace_virtio_irq(vq); 1032 vq->vdev->isr |= 0x01; 1033 virtio_notify_vector(vq->vdev, vq->vector); 1034 } 1035 1036 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq) 1037 { 1038 uint16_t old, new; 1039 bool v; 1040 /* We need to expose used array entries before checking used event. */ 1041 smp_mb(); 1042 /* Always notify when queue is empty (when feature acknowledge) */ 1043 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 1044 !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx) { 1045 return true; 1046 } 1047 1048 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1049 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 1050 } 1051 1052 v = vq->signalled_used_valid; 1053 vq->signalled_used_valid = true; 1054 old = vq->signalled_used; 1055 new = vq->signalled_used = vring_used_idx(vq); 1056 return !v || vring_need_event(vring_get_used_event(vq), new, old); 1057 } 1058 1059 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 1060 { 1061 if (!vring_notify(vdev, vq)) { 1062 return; 1063 } 1064 1065 trace_virtio_notify(vdev, vq); 1066 vdev->isr |= 0x01; 1067 virtio_notify_vector(vdev, vq->vector); 1068 } 1069 1070 void virtio_notify_config(VirtIODevice *vdev) 1071 { 1072 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1073 return; 1074 1075 vdev->isr |= 0x03; 1076 vdev->generation++; 1077 virtio_notify_vector(vdev, vdev->config_vector); 1078 } 1079 1080 static bool virtio_device_endian_needed(void *opaque) 1081 { 1082 VirtIODevice *vdev = opaque; 1083 1084 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 1085 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1086 return vdev->device_endian != virtio_default_endian(); 1087 } 1088 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 1089 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 1090 } 1091 1092 static bool virtio_64bit_features_needed(void *opaque) 1093 { 1094 VirtIODevice *vdev = opaque; 1095 1096 return (vdev->host_features >> 32) != 0; 1097 } 1098 1099 static bool virtio_virtqueue_needed(void *opaque) 1100 { 1101 VirtIODevice *vdev = opaque; 1102 1103 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 1104 } 1105 1106 static bool virtio_ringsize_needed(void *opaque) 1107 { 1108 VirtIODevice *vdev = opaque; 1109 int i; 1110 1111 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1112 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 1113 return true; 1114 } 1115 } 1116 return false; 1117 } 1118 1119 static void put_virtqueue_state(QEMUFile *f, void *pv, size_t size) 1120 { 1121 VirtIODevice *vdev = pv; 1122 int i; 1123 1124 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1125 qemu_put_be64(f, vdev->vq[i].vring.avail); 1126 qemu_put_be64(f, vdev->vq[i].vring.used); 1127 } 1128 } 1129 1130 static int get_virtqueue_state(QEMUFile *f, void *pv, size_t size) 1131 { 1132 VirtIODevice *vdev = pv; 1133 int i; 1134 1135 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1136 vdev->vq[i].vring.avail = qemu_get_be64(f); 1137 vdev->vq[i].vring.used = qemu_get_be64(f); 1138 } 1139 return 0; 1140 } 1141 1142 static VMStateInfo vmstate_info_virtqueue = { 1143 .name = "virtqueue_state", 1144 .get = get_virtqueue_state, 1145 .put = put_virtqueue_state, 1146 }; 1147 1148 static const VMStateDescription vmstate_virtio_virtqueues = { 1149 .name = "virtio/virtqueues", 1150 .version_id = 1, 1151 .minimum_version_id = 1, 1152 .needed = &virtio_virtqueue_needed, 1153 .fields = (VMStateField[]) { 1154 { 1155 .name = "virtqueues", 1156 .version_id = 0, 1157 .field_exists = NULL, 1158 .size = 0, 1159 .info = &vmstate_info_virtqueue, 1160 .flags = VMS_SINGLE, 1161 .offset = 0, 1162 }, 1163 VMSTATE_END_OF_LIST() 1164 } 1165 }; 1166 1167 static void put_ringsize_state(QEMUFile *f, void *pv, size_t size) 1168 { 1169 VirtIODevice *vdev = pv; 1170 int i; 1171 1172 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1173 qemu_put_be32(f, vdev->vq[i].vring.num_default); 1174 } 1175 } 1176 1177 static int get_ringsize_state(QEMUFile *f, void *pv, size_t size) 1178 { 1179 VirtIODevice *vdev = pv; 1180 int i; 1181 1182 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1183 vdev->vq[i].vring.num_default = qemu_get_be32(f); 1184 } 1185 return 0; 1186 } 1187 1188 static VMStateInfo vmstate_info_ringsize = { 1189 .name = "ringsize_state", 1190 .get = get_ringsize_state, 1191 .put = put_ringsize_state, 1192 }; 1193 1194 static const VMStateDescription vmstate_virtio_ringsize = { 1195 .name = "virtio/ringsize", 1196 .version_id = 1, 1197 .minimum_version_id = 1, 1198 .needed = &virtio_ringsize_needed, 1199 .fields = (VMStateField[]) { 1200 { 1201 .name = "ringsize", 1202 .version_id = 0, 1203 .field_exists = NULL, 1204 .size = 0, 1205 .info = &vmstate_info_ringsize, 1206 .flags = VMS_SINGLE, 1207 .offset = 0, 1208 }, 1209 VMSTATE_END_OF_LIST() 1210 } 1211 }; 1212 1213 static const VMStateDescription vmstate_virtio_device_endian = { 1214 .name = "virtio/device_endian", 1215 .version_id = 1, 1216 .minimum_version_id = 1, 1217 .needed = &virtio_device_endian_needed, 1218 .fields = (VMStateField[]) { 1219 VMSTATE_UINT8(device_endian, VirtIODevice), 1220 VMSTATE_END_OF_LIST() 1221 } 1222 }; 1223 1224 static const VMStateDescription vmstate_virtio_64bit_features = { 1225 .name = "virtio/64bit_features", 1226 .version_id = 1, 1227 .minimum_version_id = 1, 1228 .needed = &virtio_64bit_features_needed, 1229 .fields = (VMStateField[]) { 1230 VMSTATE_UINT64(guest_features, VirtIODevice), 1231 VMSTATE_END_OF_LIST() 1232 } 1233 }; 1234 1235 static const VMStateDescription vmstate_virtio = { 1236 .name = "virtio", 1237 .version_id = 1, 1238 .minimum_version_id = 1, 1239 .minimum_version_id_old = 1, 1240 .fields = (VMStateField[]) { 1241 VMSTATE_END_OF_LIST() 1242 }, 1243 .subsections = (const VMStateDescription*[]) { 1244 &vmstate_virtio_device_endian, 1245 &vmstate_virtio_64bit_features, 1246 &vmstate_virtio_virtqueues, 1247 &vmstate_virtio_ringsize, 1248 NULL 1249 } 1250 }; 1251 1252 void virtio_save(VirtIODevice *vdev, QEMUFile *f) 1253 { 1254 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1255 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1256 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1257 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 1258 int i; 1259 1260 if (k->save_config) { 1261 k->save_config(qbus->parent, f); 1262 } 1263 1264 qemu_put_8s(f, &vdev->status); 1265 qemu_put_8s(f, &vdev->isr); 1266 qemu_put_be16s(f, &vdev->queue_sel); 1267 qemu_put_be32s(f, &guest_features_lo); 1268 qemu_put_be32(f, vdev->config_len); 1269 qemu_put_buffer(f, vdev->config, vdev->config_len); 1270 1271 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1272 if (vdev->vq[i].vring.num == 0) 1273 break; 1274 } 1275 1276 qemu_put_be32(f, i); 1277 1278 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1279 if (vdev->vq[i].vring.num == 0) 1280 break; 1281 1282 qemu_put_be32(f, vdev->vq[i].vring.num); 1283 if (k->has_variable_vring_alignment) { 1284 qemu_put_be32(f, vdev->vq[i].vring.align); 1285 } 1286 /* XXX virtio-1 devices */ 1287 qemu_put_be64(f, vdev->vq[i].vring.desc); 1288 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 1289 if (k->save_queue) { 1290 k->save_queue(qbus->parent, i, f); 1291 } 1292 } 1293 1294 if (vdc->save != NULL) { 1295 vdc->save(vdev, f); 1296 } 1297 1298 /* Subsections */ 1299 vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 1300 } 1301 1302 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 1303 { 1304 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1305 bool bad = (val & ~(vdev->host_features)) != 0; 1306 1307 val &= vdev->host_features; 1308 if (k->set_features) { 1309 k->set_features(vdev, val); 1310 } 1311 vdev->guest_features = val; 1312 return bad ? -1 : 0; 1313 } 1314 1315 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 1316 { 1317 /* 1318 * The driver must not attempt to set features after feature negotiation 1319 * has finished. 1320 */ 1321 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 1322 return -EINVAL; 1323 } 1324 return virtio_set_features_nocheck(vdev, val); 1325 } 1326 1327 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 1328 { 1329 int i, ret; 1330 int32_t config_len; 1331 uint32_t num; 1332 uint32_t features; 1333 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1334 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1335 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1336 1337 /* 1338 * We poison the endianness to ensure it does not get used before 1339 * subsections have been loaded. 1340 */ 1341 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 1342 1343 if (k->load_config) { 1344 ret = k->load_config(qbus->parent, f); 1345 if (ret) 1346 return ret; 1347 } 1348 1349 qemu_get_8s(f, &vdev->status); 1350 qemu_get_8s(f, &vdev->isr); 1351 qemu_get_be16s(f, &vdev->queue_sel); 1352 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 1353 return -1; 1354 } 1355 qemu_get_be32s(f, &features); 1356 1357 config_len = qemu_get_be32(f); 1358 1359 /* 1360 * There are cases where the incoming config can be bigger or smaller 1361 * than what we have; so load what we have space for, and skip 1362 * any excess that's in the stream. 1363 */ 1364 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 1365 1366 while (config_len > vdev->config_len) { 1367 qemu_get_byte(f); 1368 config_len--; 1369 } 1370 1371 num = qemu_get_be32(f); 1372 1373 if (num > VIRTIO_QUEUE_MAX) { 1374 error_report("Invalid number of PCI queues: 0x%x", num); 1375 return -1; 1376 } 1377 1378 for (i = 0; i < num; i++) { 1379 vdev->vq[i].vring.num = qemu_get_be32(f); 1380 if (k->has_variable_vring_alignment) { 1381 vdev->vq[i].vring.align = qemu_get_be32(f); 1382 } 1383 vdev->vq[i].vring.desc = qemu_get_be64(f); 1384 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 1385 vdev->vq[i].signalled_used_valid = false; 1386 vdev->vq[i].notification = true; 1387 1388 if (vdev->vq[i].vring.desc) { 1389 /* XXX virtio-1 devices */ 1390 virtio_queue_update_rings(vdev, i); 1391 } else if (vdev->vq[i].last_avail_idx) { 1392 error_report("VQ %d address 0x0 " 1393 "inconsistent with Host index 0x%x", 1394 i, vdev->vq[i].last_avail_idx); 1395 return -1; 1396 } 1397 if (k->load_queue) { 1398 ret = k->load_queue(qbus->parent, i, f); 1399 if (ret) 1400 return ret; 1401 } 1402 } 1403 1404 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 1405 1406 if (vdc->load != NULL) { 1407 ret = vdc->load(vdev, f, version_id); 1408 if (ret) { 1409 return ret; 1410 } 1411 } 1412 1413 /* Subsections */ 1414 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 1415 if (ret) { 1416 return ret; 1417 } 1418 1419 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 1420 vdev->device_endian = virtio_default_endian(); 1421 } 1422 1423 if (virtio_64bit_features_needed(vdev)) { 1424 /* 1425 * Subsection load filled vdev->guest_features. Run them 1426 * through virtio_set_features to sanity-check them against 1427 * host_features. 1428 */ 1429 uint64_t features64 = vdev->guest_features; 1430 if (virtio_set_features_nocheck(vdev, features64) < 0) { 1431 error_report("Features 0x%" PRIx64 " unsupported. " 1432 "Allowed features: 0x%" PRIx64, 1433 features64, vdev->host_features); 1434 return -1; 1435 } 1436 } else { 1437 if (virtio_set_features_nocheck(vdev, features) < 0) { 1438 error_report("Features 0x%x unsupported. " 1439 "Allowed features: 0x%" PRIx64, 1440 features, vdev->host_features); 1441 return -1; 1442 } 1443 } 1444 1445 for (i = 0; i < num; i++) { 1446 if (vdev->vq[i].vring.desc) { 1447 uint16_t nheads; 1448 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 1449 /* Check it isn't doing strange things with descriptor numbers. */ 1450 if (nheads > vdev->vq[i].vring.num) { 1451 error_report("VQ %d size 0x%x Guest index 0x%x " 1452 "inconsistent with Host index 0x%x: delta 0x%x", 1453 i, vdev->vq[i].vring.num, 1454 vring_avail_idx(&vdev->vq[i]), 1455 vdev->vq[i].last_avail_idx, nheads); 1456 return -1; 1457 } 1458 } 1459 } 1460 1461 return 0; 1462 } 1463 1464 void virtio_cleanup(VirtIODevice *vdev) 1465 { 1466 qemu_del_vm_change_state_handler(vdev->vmstate); 1467 g_free(vdev->config); 1468 g_free(vdev->vq); 1469 g_free(vdev->vector_queues); 1470 } 1471 1472 static void virtio_vmstate_change(void *opaque, int running, RunState state) 1473 { 1474 VirtIODevice *vdev = opaque; 1475 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1476 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1477 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK); 1478 vdev->vm_running = running; 1479 1480 if (backend_run) { 1481 virtio_set_status(vdev, vdev->status); 1482 } 1483 1484 if (k->vmstate_change) { 1485 k->vmstate_change(qbus->parent, backend_run); 1486 } 1487 1488 if (!backend_run) { 1489 virtio_set_status(vdev, vdev->status); 1490 } 1491 } 1492 1493 void virtio_instance_init_common(Object *proxy_obj, void *data, 1494 size_t vdev_size, const char *vdev_name) 1495 { 1496 DeviceState *vdev = data; 1497 1498 object_initialize(vdev, vdev_size, vdev_name); 1499 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL); 1500 object_unref(OBJECT(vdev)); 1501 qdev_alias_all_properties(vdev, proxy_obj); 1502 } 1503 1504 void virtio_init(VirtIODevice *vdev, const char *name, 1505 uint16_t device_id, size_t config_size) 1506 { 1507 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1508 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1509 int i; 1510 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 1511 1512 if (nvectors) { 1513 vdev->vector_queues = 1514 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 1515 } 1516 1517 vdev->device_id = device_id; 1518 vdev->status = 0; 1519 vdev->isr = 0; 1520 vdev->queue_sel = 0; 1521 vdev->config_vector = VIRTIO_NO_VECTOR; 1522 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX); 1523 vdev->vm_running = runstate_is_running(); 1524 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1525 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 1526 vdev->vq[i].vdev = vdev; 1527 vdev->vq[i].queue_index = i; 1528 } 1529 1530 vdev->name = name; 1531 vdev->config_len = config_size; 1532 if (vdev->config_len) { 1533 vdev->config = g_malloc0(config_size); 1534 } else { 1535 vdev->config = NULL; 1536 } 1537 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, 1538 vdev); 1539 vdev->device_endian = virtio_default_endian(); 1540 } 1541 1542 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 1543 { 1544 return vdev->vq[n].vring.desc; 1545 } 1546 1547 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 1548 { 1549 return vdev->vq[n].vring.avail; 1550 } 1551 1552 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 1553 { 1554 return vdev->vq[n].vring.used; 1555 } 1556 1557 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n) 1558 { 1559 return vdev->vq[n].vring.desc; 1560 } 1561 1562 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 1563 { 1564 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 1565 } 1566 1567 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 1568 { 1569 return offsetof(VRingAvail, ring) + 1570 sizeof(uint16_t) * vdev->vq[n].vring.num; 1571 } 1572 1573 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 1574 { 1575 return offsetof(VRingUsed, ring) + 1576 sizeof(VRingUsedElem) * vdev->vq[n].vring.num; 1577 } 1578 1579 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n) 1580 { 1581 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc + 1582 virtio_queue_get_used_size(vdev, n); 1583 } 1584 1585 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 1586 { 1587 return vdev->vq[n].last_avail_idx; 1588 } 1589 1590 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx) 1591 { 1592 vdev->vq[n].last_avail_idx = idx; 1593 } 1594 1595 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 1596 { 1597 vdev->vq[n].signalled_used_valid = false; 1598 } 1599 1600 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 1601 { 1602 return vdev->vq + n; 1603 } 1604 1605 uint16_t virtio_get_queue_index(VirtQueue *vq) 1606 { 1607 return vq->queue_index; 1608 } 1609 1610 static void virtio_queue_guest_notifier_read(EventNotifier *n) 1611 { 1612 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 1613 if (event_notifier_test_and_clear(n)) { 1614 virtio_irq(vq); 1615 } 1616 } 1617 1618 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 1619 bool with_irqfd) 1620 { 1621 if (assign && !with_irqfd) { 1622 event_notifier_set_handler(&vq->guest_notifier, 1623 virtio_queue_guest_notifier_read); 1624 } else { 1625 event_notifier_set_handler(&vq->guest_notifier, NULL); 1626 } 1627 if (!assign) { 1628 /* Test and clear notifier before closing it, 1629 * in case poll callback didn't have time to run. */ 1630 virtio_queue_guest_notifier_read(&vq->guest_notifier); 1631 } 1632 } 1633 1634 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 1635 { 1636 return &vq->guest_notifier; 1637 } 1638 1639 static void virtio_queue_host_notifier_read(EventNotifier *n) 1640 { 1641 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 1642 if (event_notifier_test_and_clear(n)) { 1643 virtio_queue_notify_vq(vq); 1644 } 1645 } 1646 1647 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign, 1648 bool set_handler) 1649 { 1650 if (assign && set_handler) { 1651 event_notifier_set_handler(&vq->host_notifier, 1652 virtio_queue_host_notifier_read); 1653 } else { 1654 event_notifier_set_handler(&vq->host_notifier, NULL); 1655 } 1656 if (!assign) { 1657 /* Test and clear notifier before after disabling event, 1658 * in case poll callback didn't have time to run. */ 1659 virtio_queue_host_notifier_read(&vq->host_notifier); 1660 } 1661 } 1662 1663 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 1664 { 1665 return &vq->host_notifier; 1666 } 1667 1668 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 1669 { 1670 g_free(vdev->bus_name); 1671 vdev->bus_name = g_strdup(bus_name); 1672 } 1673 1674 static void virtio_device_realize(DeviceState *dev, Error **errp) 1675 { 1676 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1677 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 1678 Error *err = NULL; 1679 1680 if (vdc->realize != NULL) { 1681 vdc->realize(dev, &err); 1682 if (err != NULL) { 1683 error_propagate(errp, err); 1684 return; 1685 } 1686 } 1687 1688 virtio_bus_device_plugged(vdev, &err); 1689 if (err != NULL) { 1690 error_propagate(errp, err); 1691 return; 1692 } 1693 } 1694 1695 static void virtio_device_unrealize(DeviceState *dev, Error **errp) 1696 { 1697 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1698 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 1699 Error *err = NULL; 1700 1701 virtio_bus_device_unplugged(vdev); 1702 1703 if (vdc->unrealize != NULL) { 1704 vdc->unrealize(dev, &err); 1705 if (err != NULL) { 1706 error_propagate(errp, err); 1707 return; 1708 } 1709 } 1710 1711 g_free(vdev->bus_name); 1712 vdev->bus_name = NULL; 1713 } 1714 1715 static Property virtio_properties[] = { 1716 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 1717 DEFINE_PROP_END_OF_LIST(), 1718 }; 1719 1720 static void virtio_device_class_init(ObjectClass *klass, void *data) 1721 { 1722 /* Set the default value here. */ 1723 DeviceClass *dc = DEVICE_CLASS(klass); 1724 1725 dc->realize = virtio_device_realize; 1726 dc->unrealize = virtio_device_unrealize; 1727 dc->bus_type = TYPE_VIRTIO_BUS; 1728 dc->props = virtio_properties; 1729 } 1730 1731 static const TypeInfo virtio_device_info = { 1732 .name = TYPE_VIRTIO_DEVICE, 1733 .parent = TYPE_DEVICE, 1734 .instance_size = sizeof(VirtIODevice), 1735 .class_init = virtio_device_class_init, 1736 .abstract = true, 1737 .class_size = sizeof(VirtioDeviceClass), 1738 }; 1739 1740 static void virtio_register_types(void) 1741 { 1742 type_register_static(&virtio_device_info); 1743 } 1744 1745 type_init(virtio_register_types) 1746