1 /* 2 * Inter-VM Shared Memory PCI device. 3 * 4 * Author: 5 * Cam Macdonell <cam@cs.ualberta.ca> 6 * 7 * Based On: cirrus_vga.c 8 * Copyright (c) 2004 Fabrice Bellard 9 * Copyright (c) 2004 Makoto Suzuki (suzu) 10 * 11 * and rtl8139.c 12 * Copyright (c) 2006 Igor Kovalenko 13 * 14 * This code is licensed under the GNU GPL v2. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 */ 19 #include "qemu/osdep.h" 20 #include "qapi/error.h" 21 #include "qemu/cutils.h" 22 #include "hw/hw.h" 23 #include "hw/pci/pci.h" 24 #include "hw/pci/msi.h" 25 #include "hw/pci/msix.h" 26 #include "sysemu/kvm.h" 27 #include "migration/blocker.h" 28 #include "qemu/error-report.h" 29 #include "qemu/event_notifier.h" 30 #include "qom/object_interfaces.h" 31 #include "chardev/char-fe.h" 32 #include "sysemu/hostmem.h" 33 #include "sysemu/qtest.h" 34 #include "qapi/visitor.h" 35 36 #include "hw/misc/ivshmem.h" 37 38 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET 39 #define PCI_DEVICE_ID_IVSHMEM 0x1110 40 41 #define IVSHMEM_MAX_PEERS UINT16_MAX 42 #define IVSHMEM_IOEVENTFD 0 43 #define IVSHMEM_MSI 1 44 45 #define IVSHMEM_REG_BAR_SIZE 0x100 46 47 #define IVSHMEM_DEBUG 0 48 #define IVSHMEM_DPRINTF(fmt, ...) \ 49 do { \ 50 if (IVSHMEM_DEBUG) { \ 51 printf("IVSHMEM: " fmt, ## __VA_ARGS__); \ 52 } \ 53 } while (0) 54 55 #define TYPE_IVSHMEM_COMMON "ivshmem-common" 56 #define IVSHMEM_COMMON(obj) \ 57 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_COMMON) 58 59 #define TYPE_IVSHMEM_PLAIN "ivshmem-plain" 60 #define IVSHMEM_PLAIN(obj) \ 61 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_PLAIN) 62 63 #define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell" 64 #define IVSHMEM_DOORBELL(obj) \ 65 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_DOORBELL) 66 67 #define TYPE_IVSHMEM "ivshmem" 68 #define IVSHMEM(obj) \ 69 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM) 70 71 typedef struct Peer { 72 int nb_eventfds; 73 EventNotifier *eventfds; 74 } Peer; 75 76 typedef struct MSIVector { 77 PCIDevice *pdev; 78 int virq; 79 } MSIVector; 80 81 typedef struct IVShmemState { 82 /*< private >*/ 83 PCIDevice parent_obj; 84 /*< public >*/ 85 86 uint32_t features; 87 88 /* exactly one of these two may be set */ 89 HostMemoryBackend *hostmem; /* with interrupts */ 90 CharBackend server_chr; /* without interrupts */ 91 92 /* registers */ 93 uint32_t intrmask; 94 uint32_t intrstatus; 95 int vm_id; 96 97 /* BARs */ 98 MemoryRegion ivshmem_mmio; /* BAR 0 (registers) */ 99 MemoryRegion *ivshmem_bar2; /* BAR 2 (shared memory) */ 100 MemoryRegion server_bar2; /* used with server_chr */ 101 102 /* interrupt support */ 103 Peer *peers; 104 int nb_peers; /* space in @peers[] */ 105 uint32_t vectors; 106 MSIVector *msi_vectors; 107 uint64_t msg_buf; /* buffer for receiving server messages */ 108 int msg_buffered_bytes; /* #bytes in @msg_buf */ 109 110 /* migration stuff */ 111 OnOffAuto master; 112 Error *migration_blocker; 113 114 /* legacy cruft */ 115 char *role; 116 char *shmobj; 117 char *sizearg; 118 size_t legacy_size; 119 uint32_t not_legacy_32bit; 120 } IVShmemState; 121 122 /* registers for the Inter-VM shared memory device */ 123 enum ivshmem_registers { 124 INTRMASK = 0, 125 INTRSTATUS = 4, 126 IVPOSITION = 8, 127 DOORBELL = 12, 128 }; 129 130 static inline uint32_t ivshmem_has_feature(IVShmemState *ivs, 131 unsigned int feature) { 132 return (ivs->features & (1 << feature)); 133 } 134 135 static inline bool ivshmem_is_master(IVShmemState *s) 136 { 137 assert(s->master != ON_OFF_AUTO_AUTO); 138 return s->master == ON_OFF_AUTO_ON; 139 } 140 141 static void ivshmem_update_irq(IVShmemState *s) 142 { 143 PCIDevice *d = PCI_DEVICE(s); 144 uint32_t isr = s->intrstatus & s->intrmask; 145 146 /* 147 * Do nothing unless the device actually uses INTx. Here's how 148 * the device variants signal interrupts, what they put in PCI 149 * config space: 150 * Device variant Interrupt Interrupt Pin MSI-X cap. 151 * ivshmem-plain none 0 no 152 * ivshmem-doorbell MSI-X 1 yes(1) 153 * ivshmem,msi=off INTx 1 no 154 * ivshmem,msi=on MSI-X 1(2) yes(1) 155 * (1) if guest enabled MSI-X 156 * (2) the device lies 157 * Leads to the condition for doing nothing: 158 */ 159 if (ivshmem_has_feature(s, IVSHMEM_MSI) 160 || !d->config[PCI_INTERRUPT_PIN]) { 161 return; 162 } 163 164 /* don't print ISR resets */ 165 if (isr) { 166 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n", 167 isr ? 1 : 0, s->intrstatus, s->intrmask); 168 } 169 170 pci_set_irq(d, isr != 0); 171 } 172 173 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val) 174 { 175 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val); 176 177 s->intrmask = val; 178 ivshmem_update_irq(s); 179 } 180 181 static uint32_t ivshmem_IntrMask_read(IVShmemState *s) 182 { 183 uint32_t ret = s->intrmask; 184 185 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret); 186 return ret; 187 } 188 189 static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val) 190 { 191 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val); 192 193 s->intrstatus = val; 194 ivshmem_update_irq(s); 195 } 196 197 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s) 198 { 199 uint32_t ret = s->intrstatus; 200 201 /* reading ISR clears all interrupts */ 202 s->intrstatus = 0; 203 ivshmem_update_irq(s); 204 return ret; 205 } 206 207 static void ivshmem_io_write(void *opaque, hwaddr addr, 208 uint64_t val, unsigned size) 209 { 210 IVShmemState *s = opaque; 211 212 uint16_t dest = val >> 16; 213 uint16_t vector = val & 0xff; 214 215 addr &= 0xfc; 216 217 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx "\n", addr); 218 switch (addr) 219 { 220 case INTRMASK: 221 ivshmem_IntrMask_write(s, val); 222 break; 223 224 case INTRSTATUS: 225 ivshmem_IntrStatus_write(s, val); 226 break; 227 228 case DOORBELL: 229 /* check that dest VM ID is reasonable */ 230 if (dest >= s->nb_peers) { 231 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest); 232 break; 233 } 234 235 /* check doorbell range */ 236 if (vector < s->peers[dest].nb_eventfds) { 237 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector); 238 event_notifier_set(&s->peers[dest].eventfds[vector]); 239 } else { 240 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n", 241 vector, dest); 242 } 243 break; 244 default: 245 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr); 246 } 247 } 248 249 static uint64_t ivshmem_io_read(void *opaque, hwaddr addr, 250 unsigned size) 251 { 252 253 IVShmemState *s = opaque; 254 uint32_t ret; 255 256 switch (addr) 257 { 258 case INTRMASK: 259 ret = ivshmem_IntrMask_read(s); 260 break; 261 262 case INTRSTATUS: 263 ret = ivshmem_IntrStatus_read(s); 264 break; 265 266 case IVPOSITION: 267 ret = s->vm_id; 268 break; 269 270 default: 271 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx "\n", addr); 272 ret = 0; 273 } 274 275 return ret; 276 } 277 278 static const MemoryRegionOps ivshmem_mmio_ops = { 279 .read = ivshmem_io_read, 280 .write = ivshmem_io_write, 281 .endianness = DEVICE_NATIVE_ENDIAN, 282 .impl = { 283 .min_access_size = 4, 284 .max_access_size = 4, 285 }, 286 }; 287 288 static void ivshmem_vector_notify(void *opaque) 289 { 290 MSIVector *entry = opaque; 291 PCIDevice *pdev = entry->pdev; 292 IVShmemState *s = IVSHMEM_COMMON(pdev); 293 int vector = entry - s->msi_vectors; 294 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 295 296 if (!event_notifier_test_and_clear(n)) { 297 return; 298 } 299 300 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector); 301 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 302 if (msix_enabled(pdev)) { 303 msix_notify(pdev, vector); 304 } 305 } else { 306 ivshmem_IntrStatus_write(s, 1); 307 } 308 } 309 310 static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector, 311 MSIMessage msg) 312 { 313 IVShmemState *s = IVSHMEM_COMMON(dev); 314 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 315 MSIVector *v = &s->msi_vectors[vector]; 316 int ret; 317 318 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector); 319 320 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev); 321 if (ret < 0) { 322 return ret; 323 } 324 kvm_irqchip_commit_routes(kvm_state); 325 326 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq); 327 } 328 329 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector) 330 { 331 IVShmemState *s = IVSHMEM_COMMON(dev); 332 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 333 int ret; 334 335 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector); 336 337 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, 338 s->msi_vectors[vector].virq); 339 if (ret != 0) { 340 error_report("remove_irqfd_notifier_gsi failed"); 341 } 342 } 343 344 static void ivshmem_vector_poll(PCIDevice *dev, 345 unsigned int vector_start, 346 unsigned int vector_end) 347 { 348 IVShmemState *s = IVSHMEM_COMMON(dev); 349 unsigned int vector; 350 351 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end); 352 353 vector_end = MIN(vector_end, s->vectors); 354 355 for (vector = vector_start; vector < vector_end; vector++) { 356 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector]; 357 358 if (!msix_is_masked(dev, vector)) { 359 continue; 360 } 361 362 if (event_notifier_test_and_clear(notifier)) { 363 msix_set_pending(dev, vector); 364 } 365 } 366 } 367 368 static void watch_vector_notifier(IVShmemState *s, EventNotifier *n, 369 int vector) 370 { 371 int eventfd = event_notifier_get_fd(n); 372 373 assert(!s->msi_vectors[vector].pdev); 374 s->msi_vectors[vector].pdev = PCI_DEVICE(s); 375 376 qemu_set_fd_handler(eventfd, ivshmem_vector_notify, 377 NULL, &s->msi_vectors[vector]); 378 } 379 380 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i) 381 { 382 memory_region_add_eventfd(&s->ivshmem_mmio, 383 DOORBELL, 384 4, 385 true, 386 (posn << 16) | i, 387 &s->peers[posn].eventfds[i]); 388 } 389 390 static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i) 391 { 392 memory_region_del_eventfd(&s->ivshmem_mmio, 393 DOORBELL, 394 4, 395 true, 396 (posn << 16) | i, 397 &s->peers[posn].eventfds[i]); 398 } 399 400 static void close_peer_eventfds(IVShmemState *s, int posn) 401 { 402 int i, n; 403 404 assert(posn >= 0 && posn < s->nb_peers); 405 n = s->peers[posn].nb_eventfds; 406 407 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 408 memory_region_transaction_begin(); 409 for (i = 0; i < n; i++) { 410 ivshmem_del_eventfd(s, posn, i); 411 } 412 memory_region_transaction_commit(); 413 } 414 415 for (i = 0; i < n; i++) { 416 event_notifier_cleanup(&s->peers[posn].eventfds[i]); 417 } 418 419 g_free(s->peers[posn].eventfds); 420 s->peers[posn].nb_eventfds = 0; 421 } 422 423 static void resize_peers(IVShmemState *s, int nb_peers) 424 { 425 int old_nb_peers = s->nb_peers; 426 int i; 427 428 assert(nb_peers > old_nb_peers); 429 IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers); 430 431 s->peers = g_realloc(s->peers, nb_peers * sizeof(Peer)); 432 s->nb_peers = nb_peers; 433 434 for (i = old_nb_peers; i < nb_peers; i++) { 435 s->peers[i].eventfds = g_new0(EventNotifier, s->vectors); 436 s->peers[i].nb_eventfds = 0; 437 } 438 } 439 440 static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector, 441 Error **errp) 442 { 443 PCIDevice *pdev = PCI_DEVICE(s); 444 int ret; 445 446 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector); 447 assert(!s->msi_vectors[vector].pdev); 448 449 ret = kvm_irqchip_add_msi_route(kvm_state, vector, pdev); 450 if (ret < 0) { 451 error_setg(errp, "kvm_irqchip_add_msi_route failed"); 452 return; 453 } 454 455 s->msi_vectors[vector].virq = ret; 456 s->msi_vectors[vector].pdev = pdev; 457 } 458 459 static void setup_interrupt(IVShmemState *s, int vector, Error **errp) 460 { 461 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 462 bool with_irqfd = kvm_msi_via_irqfd_enabled() && 463 ivshmem_has_feature(s, IVSHMEM_MSI); 464 PCIDevice *pdev = PCI_DEVICE(s); 465 Error *err = NULL; 466 467 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector); 468 469 if (!with_irqfd) { 470 IVSHMEM_DPRINTF("with eventfd\n"); 471 watch_vector_notifier(s, n, vector); 472 } else if (msix_enabled(pdev)) { 473 IVSHMEM_DPRINTF("with irqfd\n"); 474 ivshmem_add_kvm_msi_virq(s, vector, &err); 475 if (err) { 476 error_propagate(errp, err); 477 return; 478 } 479 480 if (!msix_is_masked(pdev, vector)) { 481 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, 482 s->msi_vectors[vector].virq); 483 /* TODO handle error */ 484 } 485 } else { 486 /* it will be delayed until msix is enabled, in write_config */ 487 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n"); 488 } 489 } 490 491 static void process_msg_shmem(IVShmemState *s, int fd, Error **errp) 492 { 493 Error *local_err = NULL; 494 struct stat buf; 495 size_t size; 496 497 if (s->ivshmem_bar2) { 498 error_setg(errp, "server sent unexpected shared memory message"); 499 close(fd); 500 return; 501 } 502 503 if (fstat(fd, &buf) < 0) { 504 error_setg_errno(errp, errno, 505 "can't determine size of shared memory sent by server"); 506 close(fd); 507 return; 508 } 509 510 size = buf.st_size; 511 512 /* Legacy cruft */ 513 if (s->legacy_size != SIZE_MAX) { 514 if (size < s->legacy_size) { 515 error_setg(errp, "server sent only %zd bytes of shared memory", 516 (size_t)buf.st_size); 517 close(fd); 518 return; 519 } 520 size = s->legacy_size; 521 } 522 523 /* mmap the region and map into the BAR2 */ 524 memory_region_init_ram_from_fd(&s->server_bar2, OBJECT(s), 525 "ivshmem.bar2", size, true, fd, &local_err); 526 if (local_err) { 527 error_propagate(errp, local_err); 528 return; 529 } 530 531 s->ivshmem_bar2 = &s->server_bar2; 532 } 533 534 static void process_msg_disconnect(IVShmemState *s, uint16_t posn, 535 Error **errp) 536 { 537 IVSHMEM_DPRINTF("posn %d has gone away\n", posn); 538 if (posn >= s->nb_peers || posn == s->vm_id) { 539 error_setg(errp, "invalid peer %d", posn); 540 return; 541 } 542 close_peer_eventfds(s, posn); 543 } 544 545 static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd, 546 Error **errp) 547 { 548 Peer *peer = &s->peers[posn]; 549 int vector; 550 551 /* 552 * The N-th connect message for this peer comes with the file 553 * descriptor for vector N-1. Count messages to find the vector. 554 */ 555 if (peer->nb_eventfds >= s->vectors) { 556 error_setg(errp, "Too many eventfd received, device has %d vectors", 557 s->vectors); 558 close(fd); 559 return; 560 } 561 vector = peer->nb_eventfds++; 562 563 IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd); 564 event_notifier_init_fd(&peer->eventfds[vector], fd); 565 fcntl_setfl(fd, O_NONBLOCK); /* msix/irqfd poll non block */ 566 567 if (posn == s->vm_id) { 568 setup_interrupt(s, vector, errp); 569 /* TODO do we need to handle the error? */ 570 } 571 572 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 573 ivshmem_add_eventfd(s, posn, vector); 574 } 575 } 576 577 static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp) 578 { 579 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd); 580 581 if (msg < -1 || msg > IVSHMEM_MAX_PEERS) { 582 error_setg(errp, "server sent invalid message %" PRId64, msg); 583 close(fd); 584 return; 585 } 586 587 if (msg == -1) { 588 process_msg_shmem(s, fd, errp); 589 return; 590 } 591 592 if (msg >= s->nb_peers) { 593 resize_peers(s, msg + 1); 594 } 595 596 if (fd >= 0) { 597 process_msg_connect(s, msg, fd, errp); 598 } else { 599 process_msg_disconnect(s, msg, errp); 600 } 601 } 602 603 static int ivshmem_can_receive(void *opaque) 604 { 605 IVShmemState *s = opaque; 606 607 assert(s->msg_buffered_bytes < sizeof(s->msg_buf)); 608 return sizeof(s->msg_buf) - s->msg_buffered_bytes; 609 } 610 611 static void ivshmem_read(void *opaque, const uint8_t *buf, int size) 612 { 613 IVShmemState *s = opaque; 614 Error *err = NULL; 615 int fd; 616 int64_t msg; 617 618 assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf)); 619 memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size); 620 s->msg_buffered_bytes += size; 621 if (s->msg_buffered_bytes < sizeof(s->msg_buf)) { 622 return; 623 } 624 msg = le64_to_cpu(s->msg_buf); 625 s->msg_buffered_bytes = 0; 626 627 fd = qemu_chr_fe_get_msgfd(&s->server_chr); 628 629 process_msg(s, msg, fd, &err); 630 if (err) { 631 error_report_err(err); 632 } 633 } 634 635 static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp) 636 { 637 int64_t msg; 638 int n, ret; 639 640 n = 0; 641 do { 642 ret = qemu_chr_fe_read_all(&s->server_chr, (uint8_t *)&msg + n, 643 sizeof(msg) - n); 644 if (ret < 0) { 645 if (ret == -EINTR) { 646 continue; 647 } 648 error_setg_errno(errp, -ret, "read from server failed"); 649 return INT64_MIN; 650 } 651 n += ret; 652 } while (n < sizeof(msg)); 653 654 *pfd = qemu_chr_fe_get_msgfd(&s->server_chr); 655 return le64_to_cpu(msg); 656 } 657 658 static void ivshmem_recv_setup(IVShmemState *s, Error **errp) 659 { 660 Error *err = NULL; 661 int64_t msg; 662 int fd; 663 664 msg = ivshmem_recv_msg(s, &fd, &err); 665 if (err) { 666 error_propagate(errp, err); 667 return; 668 } 669 if (msg != IVSHMEM_PROTOCOL_VERSION) { 670 error_setg(errp, "server sent version %" PRId64 ", expecting %d", 671 msg, IVSHMEM_PROTOCOL_VERSION); 672 return; 673 } 674 if (fd != -1) { 675 error_setg(errp, "server sent invalid version message"); 676 return; 677 } 678 679 /* 680 * ivshmem-server sends the remaining initial messages in a fixed 681 * order, but the device has always accepted them in any order. 682 * Stay as compatible as practical, just in case people use 683 * servers that behave differently. 684 */ 685 686 /* 687 * ivshmem_device_spec.txt has always required the ID message 688 * right here, and ivshmem-server has always complied. However, 689 * older versions of the device accepted it out of order, but 690 * broke when an interrupt setup message arrived before it. 691 */ 692 msg = ivshmem_recv_msg(s, &fd, &err); 693 if (err) { 694 error_propagate(errp, err); 695 return; 696 } 697 if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) { 698 error_setg(errp, "server sent invalid ID message"); 699 return; 700 } 701 s->vm_id = msg; 702 703 /* 704 * Receive more messages until we got shared memory. 705 */ 706 do { 707 msg = ivshmem_recv_msg(s, &fd, &err); 708 if (err) { 709 error_propagate(errp, err); 710 return; 711 } 712 process_msg(s, msg, fd, &err); 713 if (err) { 714 error_propagate(errp, err); 715 return; 716 } 717 } while (msg != -1); 718 719 /* 720 * This function must either map the shared memory or fail. The 721 * loop above ensures that: it terminates normally only after it 722 * successfully processed the server's shared memory message. 723 * Assert that actually mapped the shared memory: 724 */ 725 assert(s->ivshmem_bar2); 726 } 727 728 /* Select the MSI-X vectors used by device. 729 * ivshmem maps events to vectors statically, so 730 * we just enable all vectors on init and after reset. */ 731 static void ivshmem_msix_vector_use(IVShmemState *s) 732 { 733 PCIDevice *d = PCI_DEVICE(s); 734 int i; 735 736 for (i = 0; i < s->vectors; i++) { 737 msix_vector_use(d, i); 738 } 739 } 740 741 static void ivshmem_reset(DeviceState *d) 742 { 743 IVShmemState *s = IVSHMEM_COMMON(d); 744 745 s->intrstatus = 0; 746 s->intrmask = 0; 747 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 748 ivshmem_msix_vector_use(s); 749 } 750 } 751 752 static int ivshmem_setup_interrupts(IVShmemState *s, Error **errp) 753 { 754 /* allocate QEMU callback data for receiving interrupts */ 755 s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector)); 756 757 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 758 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1, errp)) { 759 return -1; 760 } 761 762 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors); 763 ivshmem_msix_vector_use(s); 764 } 765 766 return 0; 767 } 768 769 static void ivshmem_enable_irqfd(IVShmemState *s) 770 { 771 PCIDevice *pdev = PCI_DEVICE(s); 772 int i; 773 774 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 775 Error *err = NULL; 776 777 ivshmem_add_kvm_msi_virq(s, i, &err); 778 if (err) { 779 error_report_err(err); 780 /* TODO do we need to handle the error? */ 781 } 782 } 783 784 if (msix_set_vector_notifiers(pdev, 785 ivshmem_vector_unmask, 786 ivshmem_vector_mask, 787 ivshmem_vector_poll)) { 788 error_report("ivshmem: msix_set_vector_notifiers failed"); 789 } 790 } 791 792 static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector) 793 { 794 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector); 795 796 if (s->msi_vectors[vector].pdev == NULL) { 797 return; 798 } 799 800 /* it was cleaned when masked in the frontend. */ 801 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq); 802 803 s->msi_vectors[vector].pdev = NULL; 804 } 805 806 static void ivshmem_disable_irqfd(IVShmemState *s) 807 { 808 PCIDevice *pdev = PCI_DEVICE(s); 809 int i; 810 811 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 812 ivshmem_remove_kvm_msi_virq(s, i); 813 } 814 815 msix_unset_vector_notifiers(pdev); 816 } 817 818 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address, 819 uint32_t val, int len) 820 { 821 IVShmemState *s = IVSHMEM_COMMON(pdev); 822 int is_enabled, was_enabled = msix_enabled(pdev); 823 824 pci_default_write_config(pdev, address, val, len); 825 is_enabled = msix_enabled(pdev); 826 827 if (kvm_msi_via_irqfd_enabled()) { 828 if (!was_enabled && is_enabled) { 829 ivshmem_enable_irqfd(s); 830 } else if (was_enabled && !is_enabled) { 831 ivshmem_disable_irqfd(s); 832 } 833 } 834 } 835 836 static void ivshmem_common_realize(PCIDevice *dev, Error **errp) 837 { 838 IVShmemState *s = IVSHMEM_COMMON(dev); 839 Error *err = NULL; 840 uint8_t *pci_conf; 841 uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY | 842 PCI_BASE_ADDRESS_MEM_PREFETCH; 843 Error *local_err = NULL; 844 845 /* IRQFD requires MSI */ 846 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) && 847 !ivshmem_has_feature(s, IVSHMEM_MSI)) { 848 error_setg(errp, "ioeventfd/irqfd requires MSI"); 849 return; 850 } 851 852 pci_conf = dev->config; 853 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY; 854 855 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s, 856 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE); 857 858 /* region for registers*/ 859 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, 860 &s->ivshmem_mmio); 861 862 if (s->not_legacy_32bit) { 863 attr |= PCI_BASE_ADDRESS_MEM_TYPE_64; 864 } 865 866 if (s->hostmem != NULL) { 867 IVSHMEM_DPRINTF("using hostmem\n"); 868 869 s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem, 870 &error_abort); 871 } else { 872 Chardev *chr = qemu_chr_fe_get_driver(&s->server_chr); 873 assert(chr); 874 875 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n", 876 chr->filename); 877 878 /* we allocate enough space for 16 peers and grow as needed */ 879 resize_peers(s, 16); 880 881 /* 882 * Receive setup messages from server synchronously. 883 * Older versions did it asynchronously, but that creates a 884 * number of entertaining race conditions. 885 */ 886 ivshmem_recv_setup(s, &err); 887 if (err) { 888 error_propagate(errp, err); 889 return; 890 } 891 892 if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) { 893 error_setg(errp, 894 "master must connect to the server before any peers"); 895 return; 896 } 897 898 qemu_chr_fe_set_handlers(&s->server_chr, ivshmem_can_receive, 899 ivshmem_read, NULL, NULL, s, NULL, true); 900 901 if (ivshmem_setup_interrupts(s, errp) < 0) { 902 error_prepend(errp, "Failed to initialize interrupts: "); 903 return; 904 } 905 } 906 907 if (s->master == ON_OFF_AUTO_AUTO) { 908 s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 909 } 910 911 if (!ivshmem_is_master(s)) { 912 error_setg(&s->migration_blocker, 913 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'"); 914 migrate_add_blocker(s->migration_blocker, &local_err); 915 if (local_err) { 916 error_propagate(errp, local_err); 917 error_free(s->migration_blocker); 918 return; 919 } 920 } 921 922 vmstate_register_ram(s->ivshmem_bar2, DEVICE(s)); 923 pci_register_bar(PCI_DEVICE(s), 2, attr, s->ivshmem_bar2); 924 } 925 926 static void ivshmem_exit(PCIDevice *dev) 927 { 928 IVShmemState *s = IVSHMEM_COMMON(dev); 929 int i; 930 931 if (s->migration_blocker) { 932 migrate_del_blocker(s->migration_blocker); 933 error_free(s->migration_blocker); 934 } 935 936 if (memory_region_is_mapped(s->ivshmem_bar2)) { 937 if (!s->hostmem) { 938 void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2); 939 int fd; 940 941 if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) { 942 error_report("Failed to munmap shared memory %s", 943 strerror(errno)); 944 } 945 946 fd = memory_region_get_fd(s->ivshmem_bar2); 947 close(fd); 948 } 949 950 vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev)); 951 } 952 953 if (s->peers) { 954 for (i = 0; i < s->nb_peers; i++) { 955 close_peer_eventfds(s, i); 956 } 957 g_free(s->peers); 958 } 959 960 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 961 msix_uninit_exclusive_bar(dev); 962 } 963 964 g_free(s->msi_vectors); 965 } 966 967 static int ivshmem_pre_load(void *opaque) 968 { 969 IVShmemState *s = opaque; 970 971 if (!ivshmem_is_master(s)) { 972 error_report("'peer' devices are not migratable"); 973 return -EINVAL; 974 } 975 976 return 0; 977 } 978 979 static int ivshmem_post_load(void *opaque, int version_id) 980 { 981 IVShmemState *s = opaque; 982 983 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 984 ivshmem_msix_vector_use(s); 985 } 986 return 0; 987 } 988 989 static void ivshmem_common_class_init(ObjectClass *klass, void *data) 990 { 991 DeviceClass *dc = DEVICE_CLASS(klass); 992 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 993 994 k->realize = ivshmem_common_realize; 995 k->exit = ivshmem_exit; 996 k->config_write = ivshmem_write_config; 997 k->vendor_id = PCI_VENDOR_ID_IVSHMEM; 998 k->device_id = PCI_DEVICE_ID_IVSHMEM; 999 k->class_id = PCI_CLASS_MEMORY_RAM; 1000 k->revision = 1; 1001 dc->reset = ivshmem_reset; 1002 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1003 dc->desc = "Inter-VM shared memory"; 1004 } 1005 1006 static const TypeInfo ivshmem_common_info = { 1007 .name = TYPE_IVSHMEM_COMMON, 1008 .parent = TYPE_PCI_DEVICE, 1009 .instance_size = sizeof(IVShmemState), 1010 .abstract = true, 1011 .class_init = ivshmem_common_class_init, 1012 .interfaces = (InterfaceInfo[]) { 1013 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 1014 { }, 1015 }, 1016 }; 1017 1018 static const VMStateDescription ivshmem_plain_vmsd = { 1019 .name = TYPE_IVSHMEM_PLAIN, 1020 .version_id = 0, 1021 .minimum_version_id = 0, 1022 .pre_load = ivshmem_pre_load, 1023 .post_load = ivshmem_post_load, 1024 .fields = (VMStateField[]) { 1025 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1026 VMSTATE_UINT32(intrstatus, IVShmemState), 1027 VMSTATE_UINT32(intrmask, IVShmemState), 1028 VMSTATE_END_OF_LIST() 1029 }, 1030 }; 1031 1032 static Property ivshmem_plain_properties[] = { 1033 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF), 1034 DEFINE_PROP_LINK("memdev", IVShmemState, hostmem, TYPE_MEMORY_BACKEND, 1035 HostMemoryBackend *), 1036 DEFINE_PROP_END_OF_LIST(), 1037 }; 1038 1039 static void ivshmem_plain_init(Object *obj) 1040 { 1041 IVShmemState *s = IVSHMEM_PLAIN(obj); 1042 1043 s->not_legacy_32bit = 1; 1044 } 1045 1046 static void ivshmem_plain_realize(PCIDevice *dev, Error **errp) 1047 { 1048 IVShmemState *s = IVSHMEM_COMMON(dev); 1049 1050 if (!s->hostmem) { 1051 error_setg(errp, "You must specify a 'memdev'"); 1052 return; 1053 } else if (host_memory_backend_is_mapped(s->hostmem)) { 1054 char *path = object_get_canonical_path_component(OBJECT(s->hostmem)); 1055 error_setg(errp, "can't use already busy memdev: %s", path); 1056 g_free(path); 1057 return; 1058 } 1059 1060 ivshmem_common_realize(dev, errp); 1061 host_memory_backend_set_mapped(s->hostmem, true); 1062 } 1063 1064 static void ivshmem_plain_exit(PCIDevice *pci_dev) 1065 { 1066 IVShmemState *s = IVSHMEM_COMMON(pci_dev); 1067 1068 host_memory_backend_set_mapped(s->hostmem, false); 1069 } 1070 1071 static void ivshmem_plain_class_init(ObjectClass *klass, void *data) 1072 { 1073 DeviceClass *dc = DEVICE_CLASS(klass); 1074 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1075 1076 k->realize = ivshmem_plain_realize; 1077 k->exit = ivshmem_plain_exit; 1078 dc->props = ivshmem_plain_properties; 1079 dc->vmsd = &ivshmem_plain_vmsd; 1080 } 1081 1082 static const TypeInfo ivshmem_plain_info = { 1083 .name = TYPE_IVSHMEM_PLAIN, 1084 .parent = TYPE_IVSHMEM_COMMON, 1085 .instance_size = sizeof(IVShmemState), 1086 .instance_init = ivshmem_plain_init, 1087 .class_init = ivshmem_plain_class_init, 1088 }; 1089 1090 static const VMStateDescription ivshmem_doorbell_vmsd = { 1091 .name = TYPE_IVSHMEM_DOORBELL, 1092 .version_id = 0, 1093 .minimum_version_id = 0, 1094 .pre_load = ivshmem_pre_load, 1095 .post_load = ivshmem_post_load, 1096 .fields = (VMStateField[]) { 1097 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1098 VMSTATE_MSIX(parent_obj, IVShmemState), 1099 VMSTATE_UINT32(intrstatus, IVShmemState), 1100 VMSTATE_UINT32(intrmask, IVShmemState), 1101 VMSTATE_END_OF_LIST() 1102 }, 1103 }; 1104 1105 static Property ivshmem_doorbell_properties[] = { 1106 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1107 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1108 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, 1109 true), 1110 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF), 1111 DEFINE_PROP_END_OF_LIST(), 1112 }; 1113 1114 static void ivshmem_doorbell_init(Object *obj) 1115 { 1116 IVShmemState *s = IVSHMEM_DOORBELL(obj); 1117 1118 s->features |= (1 << IVSHMEM_MSI); 1119 s->legacy_size = SIZE_MAX; /* whatever the server sends */ 1120 s->not_legacy_32bit = 1; 1121 } 1122 1123 static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp) 1124 { 1125 IVShmemState *s = IVSHMEM_COMMON(dev); 1126 1127 if (!qemu_chr_fe_backend_connected(&s->server_chr)) { 1128 error_setg(errp, "You must specify a 'chardev'"); 1129 return; 1130 } 1131 1132 ivshmem_common_realize(dev, errp); 1133 } 1134 1135 static void ivshmem_doorbell_class_init(ObjectClass *klass, void *data) 1136 { 1137 DeviceClass *dc = DEVICE_CLASS(klass); 1138 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1139 1140 k->realize = ivshmem_doorbell_realize; 1141 dc->props = ivshmem_doorbell_properties; 1142 dc->vmsd = &ivshmem_doorbell_vmsd; 1143 } 1144 1145 static const TypeInfo ivshmem_doorbell_info = { 1146 .name = TYPE_IVSHMEM_DOORBELL, 1147 .parent = TYPE_IVSHMEM_COMMON, 1148 .instance_size = sizeof(IVShmemState), 1149 .instance_init = ivshmem_doorbell_init, 1150 .class_init = ivshmem_doorbell_class_init, 1151 }; 1152 1153 static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id) 1154 { 1155 IVShmemState *s = opaque; 1156 PCIDevice *pdev = PCI_DEVICE(s); 1157 int ret; 1158 1159 IVSHMEM_DPRINTF("ivshmem_load_old\n"); 1160 1161 if (version_id != 0) { 1162 return -EINVAL; 1163 } 1164 1165 ret = ivshmem_pre_load(s); 1166 if (ret) { 1167 return ret; 1168 } 1169 1170 ret = pci_device_load(pdev, f); 1171 if (ret) { 1172 return ret; 1173 } 1174 1175 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1176 msix_load(pdev, f); 1177 ivshmem_msix_vector_use(s); 1178 } else { 1179 s->intrstatus = qemu_get_be32(f); 1180 s->intrmask = qemu_get_be32(f); 1181 } 1182 1183 return 0; 1184 } 1185 1186 static bool test_msix(void *opaque, int version_id) 1187 { 1188 IVShmemState *s = opaque; 1189 1190 return ivshmem_has_feature(s, IVSHMEM_MSI); 1191 } 1192 1193 static bool test_no_msix(void *opaque, int version_id) 1194 { 1195 return !test_msix(opaque, version_id); 1196 } 1197 1198 static const VMStateDescription ivshmem_vmsd = { 1199 .name = "ivshmem", 1200 .version_id = 1, 1201 .minimum_version_id = 1, 1202 .pre_load = ivshmem_pre_load, 1203 .post_load = ivshmem_post_load, 1204 .fields = (VMStateField[]) { 1205 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1206 1207 VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix), 1208 VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix), 1209 VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix), 1210 1211 VMSTATE_END_OF_LIST() 1212 }, 1213 .load_state_old = ivshmem_load_old, 1214 .minimum_version_id_old = 0 1215 }; 1216 1217 static Property ivshmem_properties[] = { 1218 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1219 DEFINE_PROP_STRING("size", IVShmemState, sizearg), 1220 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1221 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, 1222 false), 1223 DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true), 1224 DEFINE_PROP_STRING("shm", IVShmemState, shmobj), 1225 DEFINE_PROP_STRING("role", IVShmemState, role), 1226 DEFINE_PROP_UINT32("use64", IVShmemState, not_legacy_32bit, 1), 1227 DEFINE_PROP_END_OF_LIST(), 1228 }; 1229 1230 static void desugar_shm(IVShmemState *s) 1231 { 1232 Object *obj; 1233 char *path; 1234 1235 obj = object_new("memory-backend-file"); 1236 path = g_strdup_printf("/dev/shm/%s", s->shmobj); 1237 object_property_set_str(obj, path, "mem-path", &error_abort); 1238 g_free(path); 1239 object_property_set_int(obj, s->legacy_size, "size", &error_abort); 1240 object_property_set_bool(obj, true, "share", &error_abort); 1241 object_property_add_child(OBJECT(s), "internal-shm-backend", obj, 1242 &error_abort); 1243 user_creatable_complete(obj, &error_abort); 1244 s->hostmem = MEMORY_BACKEND(obj); 1245 } 1246 1247 static void ivshmem_realize(PCIDevice *dev, Error **errp) 1248 { 1249 IVShmemState *s = IVSHMEM_COMMON(dev); 1250 1251 if (!qtest_enabled()) { 1252 error_report("ivshmem is deprecated, please use ivshmem-plain" 1253 " or ivshmem-doorbell instead"); 1254 } 1255 1256 if (qemu_chr_fe_backend_connected(&s->server_chr) + !!s->shmobj != 1) { 1257 error_setg(errp, "You must specify either 'shm' or 'chardev'"); 1258 return; 1259 } 1260 1261 if (s->sizearg == NULL) { 1262 s->legacy_size = 4 << 20; /* 4 MB default */ 1263 } else { 1264 int ret; 1265 uint64_t size; 1266 1267 ret = qemu_strtosz_MiB(s->sizearg, NULL, &size); 1268 if (ret < 0 || (size_t)size != size || !is_power_of_2(size)) { 1269 error_setg(errp, "Invalid size %s", s->sizearg); 1270 return; 1271 } 1272 s->legacy_size = size; 1273 } 1274 1275 /* check that role is reasonable */ 1276 if (s->role) { 1277 if (strncmp(s->role, "peer", 5) == 0) { 1278 s->master = ON_OFF_AUTO_OFF; 1279 } else if (strncmp(s->role, "master", 7) == 0) { 1280 s->master = ON_OFF_AUTO_ON; 1281 } else { 1282 error_setg(errp, "'role' must be 'peer' or 'master'"); 1283 return; 1284 } 1285 } else { 1286 s->master = ON_OFF_AUTO_AUTO; 1287 } 1288 1289 if (s->shmobj) { 1290 desugar_shm(s); 1291 } 1292 1293 /* 1294 * Note: we don't use INTx with IVSHMEM_MSI at all, so this is a 1295 * bald-faced lie then. But it's a backwards compatible lie. 1296 */ 1297 pci_config_set_interrupt_pin(dev->config, 1); 1298 1299 ivshmem_common_realize(dev, errp); 1300 } 1301 1302 static void ivshmem_class_init(ObjectClass *klass, void *data) 1303 { 1304 DeviceClass *dc = DEVICE_CLASS(klass); 1305 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1306 1307 k->realize = ivshmem_realize; 1308 k->revision = 0; 1309 dc->desc = "Inter-VM shared memory (legacy)"; 1310 dc->props = ivshmem_properties; 1311 dc->vmsd = &ivshmem_vmsd; 1312 } 1313 1314 static const TypeInfo ivshmem_info = { 1315 .name = TYPE_IVSHMEM, 1316 .parent = TYPE_IVSHMEM_COMMON, 1317 .instance_size = sizeof(IVShmemState), 1318 .class_init = ivshmem_class_init, 1319 }; 1320 1321 static void ivshmem_register_types(void) 1322 { 1323 type_register_static(&ivshmem_common_info); 1324 type_register_static(&ivshmem_plain_info); 1325 type_register_static(&ivshmem_doorbell_info); 1326 type_register_static(&ivshmem_info); 1327 } 1328 1329 type_init(ivshmem_register_types) 1330