1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include <inttypes.h> 15 16 #include "trace.h" 17 #include "exec/address-spaces.h" 18 #include "qemu/error-report.h" 19 #include "hw/virtio/virtio.h" 20 #include "qemu/atomic.h" 21 #include "hw/virtio/virtio-bus.h" 22 #include "migration/migration.h" 23 #include "hw/virtio/virtio-access.h" 24 25 /* 26 * The alignment to use between consumer and producer parts of vring. 27 * x86 pagesize again. This is the default, used by transports like PCI 28 * which don't provide a means for the guest to tell the host the alignment. 29 */ 30 #define VIRTIO_PCI_VRING_ALIGN 4096 31 32 typedef struct VRingDesc 33 { 34 uint64_t addr; 35 uint32_t len; 36 uint16_t flags; 37 uint16_t next; 38 } VRingDesc; 39 40 typedef struct VRingAvail 41 { 42 uint16_t flags; 43 uint16_t idx; 44 uint16_t ring[0]; 45 } VRingAvail; 46 47 typedef struct VRingUsedElem 48 { 49 uint32_t id; 50 uint32_t len; 51 } VRingUsedElem; 52 53 typedef struct VRingUsed 54 { 55 uint16_t flags; 56 uint16_t idx; 57 VRingUsedElem ring[0]; 58 } VRingUsed; 59 60 typedef struct VRing 61 { 62 unsigned int num; 63 unsigned int num_default; 64 unsigned int align; 65 hwaddr desc; 66 hwaddr avail; 67 hwaddr used; 68 } VRing; 69 70 struct VirtQueue 71 { 72 VRing vring; 73 uint16_t last_avail_idx; 74 /* Last used index value we have signalled on */ 75 uint16_t signalled_used; 76 77 /* Last used index value we have signalled on */ 78 bool signalled_used_valid; 79 80 /* Notification enabled? */ 81 bool notification; 82 83 uint16_t queue_index; 84 85 int inuse; 86 87 uint16_t vector; 88 void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq); 89 VirtIODevice *vdev; 90 EventNotifier guest_notifier; 91 EventNotifier host_notifier; 92 QLIST_ENTRY(VirtQueue) node; 93 }; 94 95 /* virt queue functions */ 96 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 97 { 98 VRing *vring = &vdev->vq[n].vring; 99 100 if (!vring->desc) { 101 /* not yet setup -> nothing to do */ 102 return; 103 } 104 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 105 vring->used = vring_align(vring->avail + 106 offsetof(VRingAvail, ring[vring->num]), 107 vring->align); 108 } 109 110 static inline uint64_t vring_desc_addr(VirtIODevice *vdev, hwaddr desc_pa, 111 int i) 112 { 113 hwaddr pa; 114 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr); 115 return virtio_ldq_phys(vdev, pa); 116 } 117 118 static inline uint32_t vring_desc_len(VirtIODevice *vdev, hwaddr desc_pa, int i) 119 { 120 hwaddr pa; 121 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len); 122 return virtio_ldl_phys(vdev, pa); 123 } 124 125 static inline uint16_t vring_desc_flags(VirtIODevice *vdev, hwaddr desc_pa, 126 int i) 127 { 128 hwaddr pa; 129 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags); 130 return virtio_lduw_phys(vdev, pa); 131 } 132 133 static inline uint16_t vring_desc_next(VirtIODevice *vdev, hwaddr desc_pa, 134 int i) 135 { 136 hwaddr pa; 137 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next); 138 return virtio_lduw_phys(vdev, pa); 139 } 140 141 static inline uint16_t vring_avail_flags(VirtQueue *vq) 142 { 143 hwaddr pa; 144 pa = vq->vring.avail + offsetof(VRingAvail, flags); 145 return virtio_lduw_phys(vq->vdev, pa); 146 } 147 148 static inline uint16_t vring_avail_idx(VirtQueue *vq) 149 { 150 hwaddr pa; 151 pa = vq->vring.avail + offsetof(VRingAvail, idx); 152 return virtio_lduw_phys(vq->vdev, pa); 153 } 154 155 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 156 { 157 hwaddr pa; 158 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]); 159 return virtio_lduw_phys(vq->vdev, pa); 160 } 161 162 static inline uint16_t vring_get_used_event(VirtQueue *vq) 163 { 164 return vring_avail_ring(vq, vq->vring.num); 165 } 166 167 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val) 168 { 169 hwaddr pa; 170 pa = vq->vring.used + offsetof(VRingUsed, ring[i].id); 171 virtio_stl_phys(vq->vdev, pa, val); 172 } 173 174 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val) 175 { 176 hwaddr pa; 177 pa = vq->vring.used + offsetof(VRingUsed, ring[i].len); 178 virtio_stl_phys(vq->vdev, pa, val); 179 } 180 181 static uint16_t vring_used_idx(VirtQueue *vq) 182 { 183 hwaddr pa; 184 pa = vq->vring.used + offsetof(VRingUsed, idx); 185 return virtio_lduw_phys(vq->vdev, pa); 186 } 187 188 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 189 { 190 hwaddr pa; 191 pa = vq->vring.used + offsetof(VRingUsed, idx); 192 virtio_stw_phys(vq->vdev, pa, val); 193 } 194 195 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 196 { 197 VirtIODevice *vdev = vq->vdev; 198 hwaddr pa; 199 pa = vq->vring.used + offsetof(VRingUsed, flags); 200 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask); 201 } 202 203 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 204 { 205 VirtIODevice *vdev = vq->vdev; 206 hwaddr pa; 207 pa = vq->vring.used + offsetof(VRingUsed, flags); 208 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask); 209 } 210 211 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 212 { 213 hwaddr pa; 214 if (!vq->notification) { 215 return; 216 } 217 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]); 218 virtio_stw_phys(vq->vdev, pa, val); 219 } 220 221 void virtio_queue_set_notification(VirtQueue *vq, int enable) 222 { 223 vq->notification = enable; 224 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 225 vring_set_avail_event(vq, vring_avail_idx(vq)); 226 } else if (enable) { 227 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 228 } else { 229 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 230 } 231 if (enable) { 232 /* Expose avail event/used flags before caller checks the avail idx. */ 233 smp_mb(); 234 } 235 } 236 237 int virtio_queue_ready(VirtQueue *vq) 238 { 239 return vq->vring.avail != 0; 240 } 241 242 int virtio_queue_empty(VirtQueue *vq) 243 { 244 return vring_avail_idx(vq) == vq->last_avail_idx; 245 } 246 247 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 248 unsigned int len) 249 { 250 unsigned int offset; 251 int i; 252 253 offset = 0; 254 for (i = 0; i < elem->in_num; i++) { 255 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 256 257 cpu_physical_memory_unmap(elem->in_sg[i].iov_base, 258 elem->in_sg[i].iov_len, 259 1, size); 260 261 offset += size; 262 } 263 264 for (i = 0; i < elem->out_num; i++) 265 cpu_physical_memory_unmap(elem->out_sg[i].iov_base, 266 elem->out_sg[i].iov_len, 267 0, elem->out_sg[i].iov_len); 268 } 269 270 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem, 271 unsigned int len) 272 { 273 vq->last_avail_idx--; 274 virtqueue_unmap_sg(vq, elem, len); 275 } 276 277 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 278 unsigned int len, unsigned int idx) 279 { 280 trace_virtqueue_fill(vq, elem, len, idx); 281 282 virtqueue_unmap_sg(vq, elem, len); 283 284 idx = (idx + vring_used_idx(vq)) % vq->vring.num; 285 286 /* Get a pointer to the next entry in the used ring. */ 287 vring_used_ring_id(vq, idx, elem->index); 288 vring_used_ring_len(vq, idx, len); 289 } 290 291 void virtqueue_flush(VirtQueue *vq, unsigned int count) 292 { 293 uint16_t old, new; 294 /* Make sure buffer is written before we update index. */ 295 smp_wmb(); 296 trace_virtqueue_flush(vq, count); 297 old = vring_used_idx(vq); 298 new = old + count; 299 vring_used_idx_set(vq, new); 300 vq->inuse -= count; 301 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 302 vq->signalled_used_valid = false; 303 } 304 305 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 306 unsigned int len) 307 { 308 virtqueue_fill(vq, elem, len, 0); 309 virtqueue_flush(vq, 1); 310 } 311 312 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 313 { 314 uint16_t num_heads = vring_avail_idx(vq) - idx; 315 316 /* Check it isn't doing very strange things with descriptor numbers. */ 317 if (num_heads > vq->vring.num) { 318 error_report("Guest moved used index from %u to %u", 319 idx, vring_avail_idx(vq)); 320 exit(1); 321 } 322 /* On success, callers read a descriptor at vq->last_avail_idx. 323 * Make sure descriptor read does not bypass avail index read. */ 324 if (num_heads) { 325 smp_rmb(); 326 } 327 328 return num_heads; 329 } 330 331 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx) 332 { 333 unsigned int head; 334 335 /* Grab the next descriptor number they're advertising, and increment 336 * the index we've seen. */ 337 head = vring_avail_ring(vq, idx % vq->vring.num); 338 339 /* If their number is silly, that's a fatal mistake. */ 340 if (head >= vq->vring.num) { 341 error_report("Guest says index %u is available", head); 342 exit(1); 343 } 344 345 return head; 346 } 347 348 static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa, 349 unsigned int i, unsigned int max) 350 { 351 unsigned int next; 352 353 /* If this descriptor says it doesn't chain, we're done. */ 354 if (!(vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_NEXT)) { 355 return max; 356 } 357 358 /* Check they're not leading us off end of descriptors. */ 359 next = vring_desc_next(vdev, desc_pa, i); 360 /* Make sure compiler knows to grab that: we don't want it changing! */ 361 smp_wmb(); 362 363 if (next >= max) { 364 error_report("Desc next is %u", next); 365 exit(1); 366 } 367 368 return next; 369 } 370 371 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 372 unsigned int *out_bytes, 373 unsigned max_in_bytes, unsigned max_out_bytes) 374 { 375 unsigned int idx; 376 unsigned int total_bufs, in_total, out_total; 377 378 idx = vq->last_avail_idx; 379 380 total_bufs = in_total = out_total = 0; 381 while (virtqueue_num_heads(vq, idx)) { 382 VirtIODevice *vdev = vq->vdev; 383 unsigned int max, num_bufs, indirect = 0; 384 hwaddr desc_pa; 385 int i; 386 387 max = vq->vring.num; 388 num_bufs = total_bufs; 389 i = virtqueue_get_head(vq, idx++); 390 desc_pa = vq->vring.desc; 391 392 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) { 393 if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) { 394 error_report("Invalid size for indirect buffer table"); 395 exit(1); 396 } 397 398 /* If we've got too many, that implies a descriptor loop. */ 399 if (num_bufs >= max) { 400 error_report("Looped descriptor"); 401 exit(1); 402 } 403 404 /* loop over the indirect descriptor table */ 405 indirect = 1; 406 max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc); 407 desc_pa = vring_desc_addr(vdev, desc_pa, i); 408 num_bufs = i = 0; 409 } 410 411 do { 412 /* If we've got too many, that implies a descriptor loop. */ 413 if (++num_bufs > max) { 414 error_report("Looped descriptor"); 415 exit(1); 416 } 417 418 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) { 419 in_total += vring_desc_len(vdev, desc_pa, i); 420 } else { 421 out_total += vring_desc_len(vdev, desc_pa, i); 422 } 423 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 424 goto done; 425 } 426 } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max); 427 428 if (!indirect) 429 total_bufs = num_bufs; 430 else 431 total_bufs++; 432 } 433 done: 434 if (in_bytes) { 435 *in_bytes = in_total; 436 } 437 if (out_bytes) { 438 *out_bytes = out_total; 439 } 440 } 441 442 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 443 unsigned int out_bytes) 444 { 445 unsigned int in_total, out_total; 446 447 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 448 return in_bytes <= in_total && out_bytes <= out_total; 449 } 450 451 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr, 452 unsigned int *num_sg, unsigned int max_size, 453 int is_write) 454 { 455 unsigned int i; 456 hwaddr len; 457 458 /* Note: this function MUST validate input, some callers 459 * are passing in num_sg values received over the network. 460 */ 461 /* TODO: teach all callers that this can fail, and return failure instead 462 * of asserting here. 463 * When we do, we might be able to re-enable NDEBUG below. 464 */ 465 #ifdef NDEBUG 466 #error building with NDEBUG is not supported 467 #endif 468 assert(*num_sg <= max_size); 469 470 for (i = 0; i < *num_sg; i++) { 471 len = sg[i].iov_len; 472 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write); 473 if (!sg[i].iov_base) { 474 error_report("virtio: error trying to map MMIO memory"); 475 exit(1); 476 } 477 if (len == sg[i].iov_len) { 478 continue; 479 } 480 if (*num_sg >= max_size) { 481 error_report("virtio: memory split makes iovec too large"); 482 exit(1); 483 } 484 memmove(sg + i + 1, sg + i, sizeof(*sg) * (*num_sg - i)); 485 memmove(addr + i + 1, addr + i, sizeof(*addr) * (*num_sg - i)); 486 assert(len < sg[i + 1].iov_len); 487 sg[i].iov_len = len; 488 addr[i + 1] += len; 489 sg[i + 1].iov_len -= len; 490 ++*num_sg; 491 } 492 } 493 494 void virtqueue_map(VirtQueueElement *elem) 495 { 496 virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num, 497 MIN(ARRAY_SIZE(elem->in_sg), ARRAY_SIZE(elem->in_addr)), 498 1); 499 virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num, 500 MIN(ARRAY_SIZE(elem->out_sg), ARRAY_SIZE(elem->out_addr)), 501 0); 502 } 503 504 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem) 505 { 506 unsigned int i, head, max; 507 hwaddr desc_pa = vq->vring.desc; 508 VirtIODevice *vdev = vq->vdev; 509 510 if (!virtqueue_num_heads(vq, vq->last_avail_idx)) 511 return 0; 512 513 /* When we start there are none of either input nor output. */ 514 elem->out_num = elem->in_num = 0; 515 516 max = vq->vring.num; 517 518 i = head = virtqueue_get_head(vq, vq->last_avail_idx++); 519 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 520 vring_set_avail_event(vq, vq->last_avail_idx); 521 } 522 523 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) { 524 if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) { 525 error_report("Invalid size for indirect buffer table"); 526 exit(1); 527 } 528 529 /* loop over the indirect descriptor table */ 530 max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc); 531 desc_pa = vring_desc_addr(vdev, desc_pa, i); 532 i = 0; 533 } 534 535 /* Collect all the descriptors */ 536 do { 537 struct iovec *sg; 538 539 if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) { 540 if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) { 541 error_report("Too many write descriptors in indirect table"); 542 exit(1); 543 } 544 elem->in_addr[elem->in_num] = vring_desc_addr(vdev, desc_pa, i); 545 sg = &elem->in_sg[elem->in_num++]; 546 } else { 547 if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) { 548 error_report("Too many read descriptors in indirect table"); 549 exit(1); 550 } 551 elem->out_addr[elem->out_num] = vring_desc_addr(vdev, desc_pa, i); 552 sg = &elem->out_sg[elem->out_num++]; 553 } 554 555 sg->iov_len = vring_desc_len(vdev, desc_pa, i); 556 557 /* If we've got too many, that implies a descriptor loop. */ 558 if ((elem->in_num + elem->out_num) > max) { 559 error_report("Looped descriptor"); 560 exit(1); 561 } 562 } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max); 563 564 /* Now map what we have collected */ 565 virtqueue_map(elem); 566 567 elem->index = head; 568 569 vq->inuse++; 570 571 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 572 return elem->in_num + elem->out_num; 573 } 574 575 /* virtio device */ 576 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 577 { 578 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 579 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 580 581 if (k->notify) { 582 k->notify(qbus->parent, vector); 583 } 584 } 585 586 void virtio_update_irq(VirtIODevice *vdev) 587 { 588 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 589 } 590 591 static int virtio_validate_features(VirtIODevice *vdev) 592 { 593 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 594 595 if (k->validate_features) { 596 return k->validate_features(vdev); 597 } else { 598 return 0; 599 } 600 } 601 602 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 603 { 604 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 605 trace_virtio_set_status(vdev, val); 606 607 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 608 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 609 val & VIRTIO_CONFIG_S_FEATURES_OK) { 610 int ret = virtio_validate_features(vdev); 611 612 if (ret) { 613 return ret; 614 } 615 } 616 } 617 if (k->set_status) { 618 k->set_status(vdev, val); 619 } 620 vdev->status = val; 621 return 0; 622 } 623 624 bool target_words_bigendian(void); 625 static enum virtio_device_endian virtio_default_endian(void) 626 { 627 if (target_words_bigendian()) { 628 return VIRTIO_DEVICE_ENDIAN_BIG; 629 } else { 630 return VIRTIO_DEVICE_ENDIAN_LITTLE; 631 } 632 } 633 634 static enum virtio_device_endian virtio_current_cpu_endian(void) 635 { 636 CPUClass *cc = CPU_GET_CLASS(current_cpu); 637 638 if (cc->virtio_is_big_endian(current_cpu)) { 639 return VIRTIO_DEVICE_ENDIAN_BIG; 640 } else { 641 return VIRTIO_DEVICE_ENDIAN_LITTLE; 642 } 643 } 644 645 void virtio_reset(void *opaque) 646 { 647 VirtIODevice *vdev = opaque; 648 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 649 int i; 650 651 virtio_set_status(vdev, 0); 652 if (current_cpu) { 653 /* Guest initiated reset */ 654 vdev->device_endian = virtio_current_cpu_endian(); 655 } else { 656 /* System reset */ 657 vdev->device_endian = virtio_default_endian(); 658 } 659 660 if (k->reset) { 661 k->reset(vdev); 662 } 663 664 vdev->guest_features = 0; 665 vdev->queue_sel = 0; 666 vdev->status = 0; 667 vdev->isr = 0; 668 vdev->config_vector = VIRTIO_NO_VECTOR; 669 virtio_notify_vector(vdev, vdev->config_vector); 670 671 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 672 vdev->vq[i].vring.desc = 0; 673 vdev->vq[i].vring.avail = 0; 674 vdev->vq[i].vring.used = 0; 675 vdev->vq[i].last_avail_idx = 0; 676 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 677 vdev->vq[i].signalled_used = 0; 678 vdev->vq[i].signalled_used_valid = false; 679 vdev->vq[i].notification = true; 680 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 681 } 682 } 683 684 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 685 { 686 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 687 uint8_t val; 688 689 if (addr + sizeof(val) > vdev->config_len) { 690 return (uint32_t)-1; 691 } 692 693 k->get_config(vdev, vdev->config); 694 695 val = ldub_p(vdev->config + addr); 696 return val; 697 } 698 699 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 700 { 701 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 702 uint16_t val; 703 704 if (addr + sizeof(val) > vdev->config_len) { 705 return (uint32_t)-1; 706 } 707 708 k->get_config(vdev, vdev->config); 709 710 val = lduw_p(vdev->config + addr); 711 return val; 712 } 713 714 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 715 { 716 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 717 uint32_t val; 718 719 if (addr + sizeof(val) > vdev->config_len) { 720 return (uint32_t)-1; 721 } 722 723 k->get_config(vdev, vdev->config); 724 725 val = ldl_p(vdev->config + addr); 726 return val; 727 } 728 729 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 730 { 731 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 732 uint8_t val = data; 733 734 if (addr + sizeof(val) > vdev->config_len) { 735 return; 736 } 737 738 stb_p(vdev->config + addr, val); 739 740 if (k->set_config) { 741 k->set_config(vdev, vdev->config); 742 } 743 } 744 745 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 746 { 747 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 748 uint16_t val = data; 749 750 if (addr + sizeof(val) > vdev->config_len) { 751 return; 752 } 753 754 stw_p(vdev->config + addr, val); 755 756 if (k->set_config) { 757 k->set_config(vdev, vdev->config); 758 } 759 } 760 761 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 762 { 763 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 764 uint32_t val = data; 765 766 if (addr + sizeof(val) > vdev->config_len) { 767 return; 768 } 769 770 stl_p(vdev->config + addr, val); 771 772 if (k->set_config) { 773 k->set_config(vdev, vdev->config); 774 } 775 } 776 777 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 778 { 779 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 780 uint8_t val; 781 782 if (addr + sizeof(val) > vdev->config_len) { 783 return (uint32_t)-1; 784 } 785 786 k->get_config(vdev, vdev->config); 787 788 val = ldub_p(vdev->config + addr); 789 return val; 790 } 791 792 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 793 { 794 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 795 uint16_t val; 796 797 if (addr + sizeof(val) > vdev->config_len) { 798 return (uint32_t)-1; 799 } 800 801 k->get_config(vdev, vdev->config); 802 803 val = lduw_le_p(vdev->config + addr); 804 return val; 805 } 806 807 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 808 { 809 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 810 uint32_t val; 811 812 if (addr + sizeof(val) > vdev->config_len) { 813 return (uint32_t)-1; 814 } 815 816 k->get_config(vdev, vdev->config); 817 818 val = ldl_le_p(vdev->config + addr); 819 return val; 820 } 821 822 void virtio_config_modern_writeb(VirtIODevice *vdev, 823 uint32_t addr, uint32_t data) 824 { 825 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 826 uint8_t val = data; 827 828 if (addr + sizeof(val) > vdev->config_len) { 829 return; 830 } 831 832 stb_p(vdev->config + addr, val); 833 834 if (k->set_config) { 835 k->set_config(vdev, vdev->config); 836 } 837 } 838 839 void virtio_config_modern_writew(VirtIODevice *vdev, 840 uint32_t addr, uint32_t data) 841 { 842 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 843 uint16_t val = data; 844 845 if (addr + sizeof(val) > vdev->config_len) { 846 return; 847 } 848 849 stw_le_p(vdev->config + addr, val); 850 851 if (k->set_config) { 852 k->set_config(vdev, vdev->config); 853 } 854 } 855 856 void virtio_config_modern_writel(VirtIODevice *vdev, 857 uint32_t addr, uint32_t data) 858 { 859 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 860 uint32_t val = data; 861 862 if (addr + sizeof(val) > vdev->config_len) { 863 return; 864 } 865 866 stl_le_p(vdev->config + addr, val); 867 868 if (k->set_config) { 869 k->set_config(vdev, vdev->config); 870 } 871 } 872 873 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 874 { 875 vdev->vq[n].vring.desc = addr; 876 virtio_queue_update_rings(vdev, n); 877 } 878 879 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 880 { 881 return vdev->vq[n].vring.desc; 882 } 883 884 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 885 hwaddr avail, hwaddr used) 886 { 887 vdev->vq[n].vring.desc = desc; 888 vdev->vq[n].vring.avail = avail; 889 vdev->vq[n].vring.used = used; 890 } 891 892 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 893 { 894 /* Don't allow guest to flip queue between existent and 895 * nonexistent states, or to set it to an invalid size. 896 */ 897 if (!!num != !!vdev->vq[n].vring.num || 898 num > VIRTQUEUE_MAX_SIZE || 899 num < 0) { 900 return; 901 } 902 vdev->vq[n].vring.num = num; 903 } 904 905 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 906 { 907 return QLIST_FIRST(&vdev->vector_queues[vector]); 908 } 909 910 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 911 { 912 return QLIST_NEXT(vq, node); 913 } 914 915 int virtio_queue_get_num(VirtIODevice *vdev, int n) 916 { 917 return vdev->vq[n].vring.num; 918 } 919 920 int virtio_get_num_queues(VirtIODevice *vdev) 921 { 922 int i; 923 924 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 925 if (!virtio_queue_get_num(vdev, i)) { 926 break; 927 } 928 } 929 930 return i; 931 } 932 933 int virtio_queue_get_id(VirtQueue *vq) 934 { 935 VirtIODevice *vdev = vq->vdev; 936 assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_QUEUE_MAX]); 937 return vq - &vdev->vq[0]; 938 } 939 940 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 941 { 942 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 943 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 944 945 /* virtio-1 compliant devices cannot change the alignment */ 946 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 947 error_report("tried to modify queue alignment for virtio-1 device"); 948 return; 949 } 950 /* Check that the transport told us it was going to do this 951 * (so a buggy transport will immediately assert rather than 952 * silently failing to migrate this state) 953 */ 954 assert(k->has_variable_vring_alignment); 955 956 vdev->vq[n].vring.align = align; 957 virtio_queue_update_rings(vdev, n); 958 } 959 960 void virtio_queue_notify_vq(VirtQueue *vq) 961 { 962 if (vq->vring.desc && vq->handle_output) { 963 VirtIODevice *vdev = vq->vdev; 964 965 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 966 vq->handle_output(vdev, vq); 967 } 968 } 969 970 void virtio_queue_notify(VirtIODevice *vdev, int n) 971 { 972 virtio_queue_notify_vq(&vdev->vq[n]); 973 } 974 975 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 976 { 977 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 978 VIRTIO_NO_VECTOR; 979 } 980 981 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 982 { 983 VirtQueue *vq = &vdev->vq[n]; 984 985 if (n < VIRTIO_QUEUE_MAX) { 986 if (vdev->vector_queues && 987 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 988 QLIST_REMOVE(vq, node); 989 } 990 vdev->vq[n].vector = vector; 991 if (vdev->vector_queues && 992 vector != VIRTIO_NO_VECTOR) { 993 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 994 } 995 } 996 } 997 998 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 999 void (*handle_output)(VirtIODevice *, VirtQueue *)) 1000 { 1001 int i; 1002 1003 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1004 if (vdev->vq[i].vring.num == 0) 1005 break; 1006 } 1007 1008 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 1009 abort(); 1010 1011 vdev->vq[i].vring.num = queue_size; 1012 vdev->vq[i].vring.num_default = queue_size; 1013 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 1014 vdev->vq[i].handle_output = handle_output; 1015 1016 return &vdev->vq[i]; 1017 } 1018 1019 void virtio_del_queue(VirtIODevice *vdev, int n) 1020 { 1021 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 1022 abort(); 1023 } 1024 1025 vdev->vq[n].vring.num = 0; 1026 vdev->vq[n].vring.num_default = 0; 1027 } 1028 1029 void virtio_irq(VirtQueue *vq) 1030 { 1031 trace_virtio_irq(vq); 1032 vq->vdev->isr |= 0x01; 1033 virtio_notify_vector(vq->vdev, vq->vector); 1034 } 1035 1036 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq) 1037 { 1038 uint16_t old, new; 1039 bool v; 1040 /* We need to expose used array entries before checking used event. */ 1041 smp_mb(); 1042 /* Always notify when queue is empty (when feature acknowledge) */ 1043 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 1044 !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx) { 1045 return true; 1046 } 1047 1048 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1049 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 1050 } 1051 1052 v = vq->signalled_used_valid; 1053 vq->signalled_used_valid = true; 1054 old = vq->signalled_used; 1055 new = vq->signalled_used = vring_used_idx(vq); 1056 return !v || vring_need_event(vring_get_used_event(vq), new, old); 1057 } 1058 1059 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 1060 { 1061 if (!vring_notify(vdev, vq)) { 1062 return; 1063 } 1064 1065 trace_virtio_notify(vdev, vq); 1066 vdev->isr |= 0x01; 1067 virtio_notify_vector(vdev, vq->vector); 1068 } 1069 1070 void virtio_notify_config(VirtIODevice *vdev) 1071 { 1072 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1073 return; 1074 1075 vdev->isr |= 0x03; 1076 vdev->generation++; 1077 virtio_notify_vector(vdev, vdev->config_vector); 1078 } 1079 1080 static bool virtio_device_endian_needed(void *opaque) 1081 { 1082 VirtIODevice *vdev = opaque; 1083 1084 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 1085 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1086 return vdev->device_endian != virtio_default_endian(); 1087 } 1088 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 1089 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 1090 } 1091 1092 static bool virtio_64bit_features_needed(void *opaque) 1093 { 1094 VirtIODevice *vdev = opaque; 1095 1096 return (vdev->host_features >> 32) != 0; 1097 } 1098 1099 static bool virtio_virtqueue_needed(void *opaque) 1100 { 1101 VirtIODevice *vdev = opaque; 1102 1103 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 1104 } 1105 1106 static bool virtio_ringsize_needed(void *opaque) 1107 { 1108 VirtIODevice *vdev = opaque; 1109 int i; 1110 1111 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1112 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 1113 return true; 1114 } 1115 } 1116 return false; 1117 } 1118 1119 static bool virtio_extra_state_needed(void *opaque) 1120 { 1121 VirtIODevice *vdev = opaque; 1122 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1123 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1124 1125 return k->has_extra_state && 1126 k->has_extra_state(qbus->parent); 1127 } 1128 1129 static void put_virtqueue_state(QEMUFile *f, void *pv, size_t size) 1130 { 1131 VirtIODevice *vdev = pv; 1132 int i; 1133 1134 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1135 qemu_put_be64(f, vdev->vq[i].vring.avail); 1136 qemu_put_be64(f, vdev->vq[i].vring.used); 1137 } 1138 } 1139 1140 static int get_virtqueue_state(QEMUFile *f, void *pv, size_t size) 1141 { 1142 VirtIODevice *vdev = pv; 1143 int i; 1144 1145 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1146 vdev->vq[i].vring.avail = qemu_get_be64(f); 1147 vdev->vq[i].vring.used = qemu_get_be64(f); 1148 } 1149 return 0; 1150 } 1151 1152 static VMStateInfo vmstate_info_virtqueue = { 1153 .name = "virtqueue_state", 1154 .get = get_virtqueue_state, 1155 .put = put_virtqueue_state, 1156 }; 1157 1158 static const VMStateDescription vmstate_virtio_virtqueues = { 1159 .name = "virtio/virtqueues", 1160 .version_id = 1, 1161 .minimum_version_id = 1, 1162 .needed = &virtio_virtqueue_needed, 1163 .fields = (VMStateField[]) { 1164 { 1165 .name = "virtqueues", 1166 .version_id = 0, 1167 .field_exists = NULL, 1168 .size = 0, 1169 .info = &vmstate_info_virtqueue, 1170 .flags = VMS_SINGLE, 1171 .offset = 0, 1172 }, 1173 VMSTATE_END_OF_LIST() 1174 } 1175 }; 1176 1177 static void put_ringsize_state(QEMUFile *f, void *pv, size_t size) 1178 { 1179 VirtIODevice *vdev = pv; 1180 int i; 1181 1182 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1183 qemu_put_be32(f, vdev->vq[i].vring.num_default); 1184 } 1185 } 1186 1187 static int get_ringsize_state(QEMUFile *f, void *pv, size_t size) 1188 { 1189 VirtIODevice *vdev = pv; 1190 int i; 1191 1192 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1193 vdev->vq[i].vring.num_default = qemu_get_be32(f); 1194 } 1195 return 0; 1196 } 1197 1198 static VMStateInfo vmstate_info_ringsize = { 1199 .name = "ringsize_state", 1200 .get = get_ringsize_state, 1201 .put = put_ringsize_state, 1202 }; 1203 1204 static const VMStateDescription vmstate_virtio_ringsize = { 1205 .name = "virtio/ringsize", 1206 .version_id = 1, 1207 .minimum_version_id = 1, 1208 .needed = &virtio_ringsize_needed, 1209 .fields = (VMStateField[]) { 1210 { 1211 .name = "ringsize", 1212 .version_id = 0, 1213 .field_exists = NULL, 1214 .size = 0, 1215 .info = &vmstate_info_ringsize, 1216 .flags = VMS_SINGLE, 1217 .offset = 0, 1218 }, 1219 VMSTATE_END_OF_LIST() 1220 } 1221 }; 1222 1223 static int get_extra_state(QEMUFile *f, void *pv, size_t size) 1224 { 1225 VirtIODevice *vdev = pv; 1226 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1227 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1228 1229 if (!k->load_extra_state) { 1230 return -1; 1231 } else { 1232 return k->load_extra_state(qbus->parent, f); 1233 } 1234 } 1235 1236 static void put_extra_state(QEMUFile *f, void *pv, size_t size) 1237 { 1238 VirtIODevice *vdev = pv; 1239 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1240 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1241 1242 k->save_extra_state(qbus->parent, f); 1243 } 1244 1245 static const VMStateInfo vmstate_info_extra_state = { 1246 .name = "virtqueue_extra_state", 1247 .get = get_extra_state, 1248 .put = put_extra_state, 1249 }; 1250 1251 static const VMStateDescription vmstate_virtio_extra_state = { 1252 .name = "virtio/extra_state", 1253 .version_id = 1, 1254 .minimum_version_id = 1, 1255 .needed = &virtio_extra_state_needed, 1256 .fields = (VMStateField[]) { 1257 { 1258 .name = "extra_state", 1259 .version_id = 0, 1260 .field_exists = NULL, 1261 .size = 0, 1262 .info = &vmstate_info_extra_state, 1263 .flags = VMS_SINGLE, 1264 .offset = 0, 1265 }, 1266 VMSTATE_END_OF_LIST() 1267 } 1268 }; 1269 1270 static const VMStateDescription vmstate_virtio_device_endian = { 1271 .name = "virtio/device_endian", 1272 .version_id = 1, 1273 .minimum_version_id = 1, 1274 .needed = &virtio_device_endian_needed, 1275 .fields = (VMStateField[]) { 1276 VMSTATE_UINT8(device_endian, VirtIODevice), 1277 VMSTATE_END_OF_LIST() 1278 } 1279 }; 1280 1281 static const VMStateDescription vmstate_virtio_64bit_features = { 1282 .name = "virtio/64bit_features", 1283 .version_id = 1, 1284 .minimum_version_id = 1, 1285 .needed = &virtio_64bit_features_needed, 1286 .fields = (VMStateField[]) { 1287 VMSTATE_UINT64(guest_features, VirtIODevice), 1288 VMSTATE_END_OF_LIST() 1289 } 1290 }; 1291 1292 static const VMStateDescription vmstate_virtio = { 1293 .name = "virtio", 1294 .version_id = 1, 1295 .minimum_version_id = 1, 1296 .minimum_version_id_old = 1, 1297 .fields = (VMStateField[]) { 1298 VMSTATE_END_OF_LIST() 1299 }, 1300 .subsections = (const VMStateDescription*[]) { 1301 &vmstate_virtio_device_endian, 1302 &vmstate_virtio_64bit_features, 1303 &vmstate_virtio_virtqueues, 1304 &vmstate_virtio_ringsize, 1305 &vmstate_virtio_extra_state, 1306 NULL 1307 } 1308 }; 1309 1310 void virtio_save(VirtIODevice *vdev, QEMUFile *f) 1311 { 1312 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1313 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1314 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1315 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 1316 int i; 1317 1318 if (k->save_config) { 1319 k->save_config(qbus->parent, f); 1320 } 1321 1322 qemu_put_8s(f, &vdev->status); 1323 qemu_put_8s(f, &vdev->isr); 1324 qemu_put_be16s(f, &vdev->queue_sel); 1325 qemu_put_be32s(f, &guest_features_lo); 1326 qemu_put_be32(f, vdev->config_len); 1327 qemu_put_buffer(f, vdev->config, vdev->config_len); 1328 1329 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1330 if (vdev->vq[i].vring.num == 0) 1331 break; 1332 } 1333 1334 qemu_put_be32(f, i); 1335 1336 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1337 if (vdev->vq[i].vring.num == 0) 1338 break; 1339 1340 qemu_put_be32(f, vdev->vq[i].vring.num); 1341 if (k->has_variable_vring_alignment) { 1342 qemu_put_be32(f, vdev->vq[i].vring.align); 1343 } 1344 /* XXX virtio-1 devices */ 1345 qemu_put_be64(f, vdev->vq[i].vring.desc); 1346 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 1347 if (k->save_queue) { 1348 k->save_queue(qbus->parent, i, f); 1349 } 1350 } 1351 1352 if (vdc->save != NULL) { 1353 vdc->save(vdev, f); 1354 } 1355 1356 /* Subsections */ 1357 vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 1358 } 1359 1360 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 1361 { 1362 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1363 bool bad = (val & ~(vdev->host_features)) != 0; 1364 1365 val &= vdev->host_features; 1366 if (k->set_features) { 1367 k->set_features(vdev, val); 1368 } 1369 vdev->guest_features = val; 1370 return bad ? -1 : 0; 1371 } 1372 1373 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 1374 { 1375 /* 1376 * The driver must not attempt to set features after feature negotiation 1377 * has finished. 1378 */ 1379 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 1380 return -EINVAL; 1381 } 1382 return virtio_set_features_nocheck(vdev, val); 1383 } 1384 1385 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 1386 { 1387 int i, ret; 1388 int32_t config_len; 1389 uint32_t num; 1390 uint32_t features; 1391 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1392 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1393 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1394 1395 /* 1396 * We poison the endianness to ensure it does not get used before 1397 * subsections have been loaded. 1398 */ 1399 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 1400 1401 if (k->load_config) { 1402 ret = k->load_config(qbus->parent, f); 1403 if (ret) 1404 return ret; 1405 } 1406 1407 qemu_get_8s(f, &vdev->status); 1408 qemu_get_8s(f, &vdev->isr); 1409 qemu_get_be16s(f, &vdev->queue_sel); 1410 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 1411 return -1; 1412 } 1413 qemu_get_be32s(f, &features); 1414 1415 config_len = qemu_get_be32(f); 1416 1417 /* 1418 * There are cases where the incoming config can be bigger or smaller 1419 * than what we have; so load what we have space for, and skip 1420 * any excess that's in the stream. 1421 */ 1422 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 1423 1424 while (config_len > vdev->config_len) { 1425 qemu_get_byte(f); 1426 config_len--; 1427 } 1428 1429 num = qemu_get_be32(f); 1430 1431 if (num > VIRTIO_QUEUE_MAX) { 1432 error_report("Invalid number of PCI queues: 0x%x", num); 1433 return -1; 1434 } 1435 1436 for (i = 0; i < num; i++) { 1437 vdev->vq[i].vring.num = qemu_get_be32(f); 1438 if (k->has_variable_vring_alignment) { 1439 vdev->vq[i].vring.align = qemu_get_be32(f); 1440 } 1441 vdev->vq[i].vring.desc = qemu_get_be64(f); 1442 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 1443 vdev->vq[i].signalled_used_valid = false; 1444 vdev->vq[i].notification = true; 1445 1446 if (vdev->vq[i].vring.desc) { 1447 /* XXX virtio-1 devices */ 1448 virtio_queue_update_rings(vdev, i); 1449 } else if (vdev->vq[i].last_avail_idx) { 1450 error_report("VQ %d address 0x0 " 1451 "inconsistent with Host index 0x%x", 1452 i, vdev->vq[i].last_avail_idx); 1453 return -1; 1454 } 1455 if (k->load_queue) { 1456 ret = k->load_queue(qbus->parent, i, f); 1457 if (ret) 1458 return ret; 1459 } 1460 } 1461 1462 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 1463 1464 if (vdc->load != NULL) { 1465 ret = vdc->load(vdev, f, version_id); 1466 if (ret) { 1467 return ret; 1468 } 1469 } 1470 1471 /* Subsections */ 1472 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 1473 if (ret) { 1474 return ret; 1475 } 1476 1477 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 1478 vdev->device_endian = virtio_default_endian(); 1479 } 1480 1481 if (virtio_64bit_features_needed(vdev)) { 1482 /* 1483 * Subsection load filled vdev->guest_features. Run them 1484 * through virtio_set_features to sanity-check them against 1485 * host_features. 1486 */ 1487 uint64_t features64 = vdev->guest_features; 1488 if (virtio_set_features_nocheck(vdev, features64) < 0) { 1489 error_report("Features 0x%" PRIx64 " unsupported. " 1490 "Allowed features: 0x%" PRIx64, 1491 features64, vdev->host_features); 1492 return -1; 1493 } 1494 } else { 1495 if (virtio_set_features_nocheck(vdev, features) < 0) { 1496 error_report("Features 0x%x unsupported. " 1497 "Allowed features: 0x%" PRIx64, 1498 features, vdev->host_features); 1499 return -1; 1500 } 1501 } 1502 1503 for (i = 0; i < num; i++) { 1504 if (vdev->vq[i].vring.desc) { 1505 uint16_t nheads; 1506 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 1507 /* Check it isn't doing strange things with descriptor numbers. */ 1508 if (nheads > vdev->vq[i].vring.num) { 1509 error_report("VQ %d size 0x%x Guest index 0x%x " 1510 "inconsistent with Host index 0x%x: delta 0x%x", 1511 i, vdev->vq[i].vring.num, 1512 vring_avail_idx(&vdev->vq[i]), 1513 vdev->vq[i].last_avail_idx, nheads); 1514 return -1; 1515 } 1516 } 1517 } 1518 1519 return 0; 1520 } 1521 1522 void virtio_cleanup(VirtIODevice *vdev) 1523 { 1524 qemu_del_vm_change_state_handler(vdev->vmstate); 1525 g_free(vdev->config); 1526 g_free(vdev->vq); 1527 g_free(vdev->vector_queues); 1528 } 1529 1530 static void virtio_vmstate_change(void *opaque, int running, RunState state) 1531 { 1532 VirtIODevice *vdev = opaque; 1533 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1534 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1535 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK); 1536 vdev->vm_running = running; 1537 1538 if (backend_run) { 1539 virtio_set_status(vdev, vdev->status); 1540 } 1541 1542 if (k->vmstate_change) { 1543 k->vmstate_change(qbus->parent, backend_run); 1544 } 1545 1546 if (!backend_run) { 1547 virtio_set_status(vdev, vdev->status); 1548 } 1549 } 1550 1551 void virtio_instance_init_common(Object *proxy_obj, void *data, 1552 size_t vdev_size, const char *vdev_name) 1553 { 1554 DeviceState *vdev = data; 1555 1556 object_initialize(vdev, vdev_size, vdev_name); 1557 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL); 1558 object_unref(OBJECT(vdev)); 1559 qdev_alias_all_properties(vdev, proxy_obj); 1560 } 1561 1562 void virtio_init(VirtIODevice *vdev, const char *name, 1563 uint16_t device_id, size_t config_size) 1564 { 1565 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1566 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1567 int i; 1568 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 1569 1570 if (nvectors) { 1571 vdev->vector_queues = 1572 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 1573 } 1574 1575 vdev->device_id = device_id; 1576 vdev->status = 0; 1577 vdev->isr = 0; 1578 vdev->queue_sel = 0; 1579 vdev->config_vector = VIRTIO_NO_VECTOR; 1580 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX); 1581 vdev->vm_running = runstate_is_running(); 1582 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1583 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 1584 vdev->vq[i].vdev = vdev; 1585 vdev->vq[i].queue_index = i; 1586 } 1587 1588 vdev->name = name; 1589 vdev->config_len = config_size; 1590 if (vdev->config_len) { 1591 vdev->config = g_malloc0(config_size); 1592 } else { 1593 vdev->config = NULL; 1594 } 1595 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, 1596 vdev); 1597 vdev->device_endian = virtio_default_endian(); 1598 } 1599 1600 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 1601 { 1602 return vdev->vq[n].vring.desc; 1603 } 1604 1605 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 1606 { 1607 return vdev->vq[n].vring.avail; 1608 } 1609 1610 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 1611 { 1612 return vdev->vq[n].vring.used; 1613 } 1614 1615 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n) 1616 { 1617 return vdev->vq[n].vring.desc; 1618 } 1619 1620 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 1621 { 1622 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 1623 } 1624 1625 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 1626 { 1627 return offsetof(VRingAvail, ring) + 1628 sizeof(uint16_t) * vdev->vq[n].vring.num; 1629 } 1630 1631 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 1632 { 1633 return offsetof(VRingUsed, ring) + 1634 sizeof(VRingUsedElem) * vdev->vq[n].vring.num; 1635 } 1636 1637 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n) 1638 { 1639 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc + 1640 virtio_queue_get_used_size(vdev, n); 1641 } 1642 1643 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 1644 { 1645 return vdev->vq[n].last_avail_idx; 1646 } 1647 1648 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx) 1649 { 1650 vdev->vq[n].last_avail_idx = idx; 1651 } 1652 1653 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 1654 { 1655 vdev->vq[n].signalled_used_valid = false; 1656 } 1657 1658 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 1659 { 1660 return vdev->vq + n; 1661 } 1662 1663 uint16_t virtio_get_queue_index(VirtQueue *vq) 1664 { 1665 return vq->queue_index; 1666 } 1667 1668 static void virtio_queue_guest_notifier_read(EventNotifier *n) 1669 { 1670 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 1671 if (event_notifier_test_and_clear(n)) { 1672 virtio_irq(vq); 1673 } 1674 } 1675 1676 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 1677 bool with_irqfd) 1678 { 1679 if (assign && !with_irqfd) { 1680 event_notifier_set_handler(&vq->guest_notifier, 1681 virtio_queue_guest_notifier_read); 1682 } else { 1683 event_notifier_set_handler(&vq->guest_notifier, NULL); 1684 } 1685 if (!assign) { 1686 /* Test and clear notifier before closing it, 1687 * in case poll callback didn't have time to run. */ 1688 virtio_queue_guest_notifier_read(&vq->guest_notifier); 1689 } 1690 } 1691 1692 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 1693 { 1694 return &vq->guest_notifier; 1695 } 1696 1697 static void virtio_queue_host_notifier_read(EventNotifier *n) 1698 { 1699 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 1700 if (event_notifier_test_and_clear(n)) { 1701 virtio_queue_notify_vq(vq); 1702 } 1703 } 1704 1705 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign, 1706 bool set_handler) 1707 { 1708 if (assign && set_handler) { 1709 event_notifier_set_handler(&vq->host_notifier, 1710 virtio_queue_host_notifier_read); 1711 } else { 1712 event_notifier_set_handler(&vq->host_notifier, NULL); 1713 } 1714 if (!assign) { 1715 /* Test and clear notifier before after disabling event, 1716 * in case poll callback didn't have time to run. */ 1717 virtio_queue_host_notifier_read(&vq->host_notifier); 1718 } 1719 } 1720 1721 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 1722 { 1723 return &vq->host_notifier; 1724 } 1725 1726 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 1727 { 1728 g_free(vdev->bus_name); 1729 vdev->bus_name = g_strdup(bus_name); 1730 } 1731 1732 static void virtio_device_realize(DeviceState *dev, Error **errp) 1733 { 1734 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1735 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 1736 Error *err = NULL; 1737 1738 if (vdc->realize != NULL) { 1739 vdc->realize(dev, &err); 1740 if (err != NULL) { 1741 error_propagate(errp, err); 1742 return; 1743 } 1744 } 1745 1746 virtio_bus_device_plugged(vdev, &err); 1747 if (err != NULL) { 1748 error_propagate(errp, err); 1749 return; 1750 } 1751 } 1752 1753 static void virtio_device_unrealize(DeviceState *dev, Error **errp) 1754 { 1755 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1756 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 1757 Error *err = NULL; 1758 1759 virtio_bus_device_unplugged(vdev); 1760 1761 if (vdc->unrealize != NULL) { 1762 vdc->unrealize(dev, &err); 1763 if (err != NULL) { 1764 error_propagate(errp, err); 1765 return; 1766 } 1767 } 1768 1769 g_free(vdev->bus_name); 1770 vdev->bus_name = NULL; 1771 } 1772 1773 static Property virtio_properties[] = { 1774 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 1775 DEFINE_PROP_END_OF_LIST(), 1776 }; 1777 1778 static void virtio_device_class_init(ObjectClass *klass, void *data) 1779 { 1780 /* Set the default value here. */ 1781 DeviceClass *dc = DEVICE_CLASS(klass); 1782 1783 dc->realize = virtio_device_realize; 1784 dc->unrealize = virtio_device_unrealize; 1785 dc->bus_type = TYPE_VIRTIO_BUS; 1786 dc->props = virtio_properties; 1787 } 1788 1789 static const TypeInfo virtio_device_info = { 1790 .name = TYPE_VIRTIO_DEVICE, 1791 .parent = TYPE_DEVICE, 1792 .instance_size = sizeof(VirtIODevice), 1793 .class_init = virtio_device_class_init, 1794 .abstract = true, 1795 .class_size = sizeof(VirtioDeviceClass), 1796 }; 1797 1798 static void virtio_register_types(void) 1799 { 1800 type_register_static(&virtio_device_info); 1801 } 1802 1803 type_init(virtio_register_types) 1804