1 /* 2 * Inter-VM Shared Memory PCI device. 3 * 4 * Author: 5 * Cam Macdonell <cam@cs.ualberta.ca> 6 * 7 * Based On: cirrus_vga.c 8 * Copyright (c) 2004 Fabrice Bellard 9 * Copyright (c) 2004 Makoto Suzuki (suzu) 10 * 11 * and rtl8139.c 12 * Copyright (c) 2006 Igor Kovalenko 13 * 14 * This code is licensed under the GNU GPL v2. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 */ 19 #include "qemu/osdep.h" 20 #include "qapi/error.h" 21 #include "qemu/cutils.h" 22 #include "hw/hw.h" 23 #include "hw/i386/pc.h" 24 #include "hw/pci/pci.h" 25 #include "hw/pci/msi.h" 26 #include "hw/pci/msix.h" 27 #include "sysemu/kvm.h" 28 #include "migration/blocker.h" 29 #include "qemu/error-report.h" 30 #include "qemu/event_notifier.h" 31 #include "qom/object_interfaces.h" 32 #include "chardev/char-fe.h" 33 #include "sysemu/hostmem.h" 34 #include "sysemu/qtest.h" 35 #include "qapi/visitor.h" 36 37 #include "hw/misc/ivshmem.h" 38 39 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET 40 #define PCI_DEVICE_ID_IVSHMEM 0x1110 41 42 #define IVSHMEM_MAX_PEERS UINT16_MAX 43 #define IVSHMEM_IOEVENTFD 0 44 #define IVSHMEM_MSI 1 45 46 #define IVSHMEM_REG_BAR_SIZE 0x100 47 48 #define IVSHMEM_DEBUG 0 49 #define IVSHMEM_DPRINTF(fmt, ...) \ 50 do { \ 51 if (IVSHMEM_DEBUG) { \ 52 printf("IVSHMEM: " fmt, ## __VA_ARGS__); \ 53 } \ 54 } while (0) 55 56 #define TYPE_IVSHMEM_COMMON "ivshmem-common" 57 #define IVSHMEM_COMMON(obj) \ 58 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_COMMON) 59 60 #define TYPE_IVSHMEM_PLAIN "ivshmem-plain" 61 #define IVSHMEM_PLAIN(obj) \ 62 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_PLAIN) 63 64 #define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell" 65 #define IVSHMEM_DOORBELL(obj) \ 66 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_DOORBELL) 67 68 #define TYPE_IVSHMEM "ivshmem" 69 #define IVSHMEM(obj) \ 70 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM) 71 72 typedef struct Peer { 73 int nb_eventfds; 74 EventNotifier *eventfds; 75 } Peer; 76 77 typedef struct MSIVector { 78 PCIDevice *pdev; 79 int virq; 80 } MSIVector; 81 82 typedef struct IVShmemState { 83 /*< private >*/ 84 PCIDevice parent_obj; 85 /*< public >*/ 86 87 uint32_t features; 88 89 /* exactly one of these two may be set */ 90 HostMemoryBackend *hostmem; /* with interrupts */ 91 CharBackend server_chr; /* without interrupts */ 92 93 /* registers */ 94 uint32_t intrmask; 95 uint32_t intrstatus; 96 int vm_id; 97 98 /* BARs */ 99 MemoryRegion ivshmem_mmio; /* BAR 0 (registers) */ 100 MemoryRegion *ivshmem_bar2; /* BAR 2 (shared memory) */ 101 MemoryRegion server_bar2; /* used with server_chr */ 102 103 /* interrupt support */ 104 Peer *peers; 105 int nb_peers; /* space in @peers[] */ 106 uint32_t vectors; 107 MSIVector *msi_vectors; 108 uint64_t msg_buf; /* buffer for receiving server messages */ 109 int msg_buffered_bytes; /* #bytes in @msg_buf */ 110 111 /* migration stuff */ 112 OnOffAuto master; 113 Error *migration_blocker; 114 115 /* legacy cruft */ 116 char *role; 117 char *shmobj; 118 char *sizearg; 119 size_t legacy_size; 120 uint32_t not_legacy_32bit; 121 } IVShmemState; 122 123 /* registers for the Inter-VM shared memory device */ 124 enum ivshmem_registers { 125 INTRMASK = 0, 126 INTRSTATUS = 4, 127 IVPOSITION = 8, 128 DOORBELL = 12, 129 }; 130 131 static inline uint32_t ivshmem_has_feature(IVShmemState *ivs, 132 unsigned int feature) { 133 return (ivs->features & (1 << feature)); 134 } 135 136 static inline bool ivshmem_is_master(IVShmemState *s) 137 { 138 assert(s->master != ON_OFF_AUTO_AUTO); 139 return s->master == ON_OFF_AUTO_ON; 140 } 141 142 static void ivshmem_update_irq(IVShmemState *s) 143 { 144 PCIDevice *d = PCI_DEVICE(s); 145 uint32_t isr = s->intrstatus & s->intrmask; 146 147 /* 148 * Do nothing unless the device actually uses INTx. Here's how 149 * the device variants signal interrupts, what they put in PCI 150 * config space: 151 * Device variant Interrupt Interrupt Pin MSI-X cap. 152 * ivshmem-plain none 0 no 153 * ivshmem-doorbell MSI-X 1 yes(1) 154 * ivshmem,msi=off INTx 1 no 155 * ivshmem,msi=on MSI-X 1(2) yes(1) 156 * (1) if guest enabled MSI-X 157 * (2) the device lies 158 * Leads to the condition for doing nothing: 159 */ 160 if (ivshmem_has_feature(s, IVSHMEM_MSI) 161 || !d->config[PCI_INTERRUPT_PIN]) { 162 return; 163 } 164 165 /* don't print ISR resets */ 166 if (isr) { 167 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n", 168 isr ? 1 : 0, s->intrstatus, s->intrmask); 169 } 170 171 pci_set_irq(d, isr != 0); 172 } 173 174 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val) 175 { 176 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val); 177 178 s->intrmask = val; 179 ivshmem_update_irq(s); 180 } 181 182 static uint32_t ivshmem_IntrMask_read(IVShmemState *s) 183 { 184 uint32_t ret = s->intrmask; 185 186 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret); 187 return ret; 188 } 189 190 static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val) 191 { 192 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val); 193 194 s->intrstatus = val; 195 ivshmem_update_irq(s); 196 } 197 198 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s) 199 { 200 uint32_t ret = s->intrstatus; 201 202 /* reading ISR clears all interrupts */ 203 s->intrstatus = 0; 204 ivshmem_update_irq(s); 205 return ret; 206 } 207 208 static void ivshmem_io_write(void *opaque, hwaddr addr, 209 uint64_t val, unsigned size) 210 { 211 IVShmemState *s = opaque; 212 213 uint16_t dest = val >> 16; 214 uint16_t vector = val & 0xff; 215 216 addr &= 0xfc; 217 218 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx "\n", addr); 219 switch (addr) 220 { 221 case INTRMASK: 222 ivshmem_IntrMask_write(s, val); 223 break; 224 225 case INTRSTATUS: 226 ivshmem_IntrStatus_write(s, val); 227 break; 228 229 case DOORBELL: 230 /* check that dest VM ID is reasonable */ 231 if (dest >= s->nb_peers) { 232 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest); 233 break; 234 } 235 236 /* check doorbell range */ 237 if (vector < s->peers[dest].nb_eventfds) { 238 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector); 239 event_notifier_set(&s->peers[dest].eventfds[vector]); 240 } else { 241 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n", 242 vector, dest); 243 } 244 break; 245 default: 246 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr); 247 } 248 } 249 250 static uint64_t ivshmem_io_read(void *opaque, hwaddr addr, 251 unsigned size) 252 { 253 254 IVShmemState *s = opaque; 255 uint32_t ret; 256 257 switch (addr) 258 { 259 case INTRMASK: 260 ret = ivshmem_IntrMask_read(s); 261 break; 262 263 case INTRSTATUS: 264 ret = ivshmem_IntrStatus_read(s); 265 break; 266 267 case IVPOSITION: 268 ret = s->vm_id; 269 break; 270 271 default: 272 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx "\n", addr); 273 ret = 0; 274 } 275 276 return ret; 277 } 278 279 static const MemoryRegionOps ivshmem_mmio_ops = { 280 .read = ivshmem_io_read, 281 .write = ivshmem_io_write, 282 .endianness = DEVICE_NATIVE_ENDIAN, 283 .impl = { 284 .min_access_size = 4, 285 .max_access_size = 4, 286 }, 287 }; 288 289 static void ivshmem_vector_notify(void *opaque) 290 { 291 MSIVector *entry = opaque; 292 PCIDevice *pdev = entry->pdev; 293 IVShmemState *s = IVSHMEM_COMMON(pdev); 294 int vector = entry - s->msi_vectors; 295 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 296 297 if (!event_notifier_test_and_clear(n)) { 298 return; 299 } 300 301 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector); 302 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 303 if (msix_enabled(pdev)) { 304 msix_notify(pdev, vector); 305 } 306 } else { 307 ivshmem_IntrStatus_write(s, 1); 308 } 309 } 310 311 static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector, 312 MSIMessage msg) 313 { 314 IVShmemState *s = IVSHMEM_COMMON(dev); 315 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 316 MSIVector *v = &s->msi_vectors[vector]; 317 int ret; 318 319 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector); 320 321 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev); 322 if (ret < 0) { 323 return ret; 324 } 325 kvm_irqchip_commit_routes(kvm_state); 326 327 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq); 328 } 329 330 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector) 331 { 332 IVShmemState *s = IVSHMEM_COMMON(dev); 333 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 334 int ret; 335 336 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector); 337 338 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, 339 s->msi_vectors[vector].virq); 340 if (ret != 0) { 341 error_report("remove_irqfd_notifier_gsi failed"); 342 } 343 } 344 345 static void ivshmem_vector_poll(PCIDevice *dev, 346 unsigned int vector_start, 347 unsigned int vector_end) 348 { 349 IVShmemState *s = IVSHMEM_COMMON(dev); 350 unsigned int vector; 351 352 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end); 353 354 vector_end = MIN(vector_end, s->vectors); 355 356 for (vector = vector_start; vector < vector_end; vector++) { 357 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector]; 358 359 if (!msix_is_masked(dev, vector)) { 360 continue; 361 } 362 363 if (event_notifier_test_and_clear(notifier)) { 364 msix_set_pending(dev, vector); 365 } 366 } 367 } 368 369 static void watch_vector_notifier(IVShmemState *s, EventNotifier *n, 370 int vector) 371 { 372 int eventfd = event_notifier_get_fd(n); 373 374 assert(!s->msi_vectors[vector].pdev); 375 s->msi_vectors[vector].pdev = PCI_DEVICE(s); 376 377 qemu_set_fd_handler(eventfd, ivshmem_vector_notify, 378 NULL, &s->msi_vectors[vector]); 379 } 380 381 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i) 382 { 383 memory_region_add_eventfd(&s->ivshmem_mmio, 384 DOORBELL, 385 4, 386 true, 387 (posn << 16) | i, 388 &s->peers[posn].eventfds[i]); 389 } 390 391 static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i) 392 { 393 memory_region_del_eventfd(&s->ivshmem_mmio, 394 DOORBELL, 395 4, 396 true, 397 (posn << 16) | i, 398 &s->peers[posn].eventfds[i]); 399 } 400 401 static void close_peer_eventfds(IVShmemState *s, int posn) 402 { 403 int i, n; 404 405 assert(posn >= 0 && posn < s->nb_peers); 406 n = s->peers[posn].nb_eventfds; 407 408 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 409 memory_region_transaction_begin(); 410 for (i = 0; i < n; i++) { 411 ivshmem_del_eventfd(s, posn, i); 412 } 413 memory_region_transaction_commit(); 414 } 415 416 for (i = 0; i < n; i++) { 417 event_notifier_cleanup(&s->peers[posn].eventfds[i]); 418 } 419 420 g_free(s->peers[posn].eventfds); 421 s->peers[posn].nb_eventfds = 0; 422 } 423 424 static void resize_peers(IVShmemState *s, int nb_peers) 425 { 426 int old_nb_peers = s->nb_peers; 427 int i; 428 429 assert(nb_peers > old_nb_peers); 430 IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers); 431 432 s->peers = g_realloc(s->peers, nb_peers * sizeof(Peer)); 433 s->nb_peers = nb_peers; 434 435 for (i = old_nb_peers; i < nb_peers; i++) { 436 s->peers[i].eventfds = g_new0(EventNotifier, s->vectors); 437 s->peers[i].nb_eventfds = 0; 438 } 439 } 440 441 static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector, 442 Error **errp) 443 { 444 PCIDevice *pdev = PCI_DEVICE(s); 445 int ret; 446 447 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector); 448 assert(!s->msi_vectors[vector].pdev); 449 450 ret = kvm_irqchip_add_msi_route(kvm_state, vector, pdev); 451 if (ret < 0) { 452 error_setg(errp, "kvm_irqchip_add_msi_route failed"); 453 return; 454 } 455 456 s->msi_vectors[vector].virq = ret; 457 s->msi_vectors[vector].pdev = pdev; 458 } 459 460 static void setup_interrupt(IVShmemState *s, int vector, Error **errp) 461 { 462 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 463 bool with_irqfd = kvm_msi_via_irqfd_enabled() && 464 ivshmem_has_feature(s, IVSHMEM_MSI); 465 PCIDevice *pdev = PCI_DEVICE(s); 466 Error *err = NULL; 467 468 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector); 469 470 if (!with_irqfd) { 471 IVSHMEM_DPRINTF("with eventfd\n"); 472 watch_vector_notifier(s, n, vector); 473 } else if (msix_enabled(pdev)) { 474 IVSHMEM_DPRINTF("with irqfd\n"); 475 ivshmem_add_kvm_msi_virq(s, vector, &err); 476 if (err) { 477 error_propagate(errp, err); 478 return; 479 } 480 481 if (!msix_is_masked(pdev, vector)) { 482 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, 483 s->msi_vectors[vector].virq); 484 /* TODO handle error */ 485 } 486 } else { 487 /* it will be delayed until msix is enabled, in write_config */ 488 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n"); 489 } 490 } 491 492 static void process_msg_shmem(IVShmemState *s, int fd, Error **errp) 493 { 494 Error *local_err = NULL; 495 struct stat buf; 496 size_t size; 497 498 if (s->ivshmem_bar2) { 499 error_setg(errp, "server sent unexpected shared memory message"); 500 close(fd); 501 return; 502 } 503 504 if (fstat(fd, &buf) < 0) { 505 error_setg_errno(errp, errno, 506 "can't determine size of shared memory sent by server"); 507 close(fd); 508 return; 509 } 510 511 size = buf.st_size; 512 513 /* Legacy cruft */ 514 if (s->legacy_size != SIZE_MAX) { 515 if (size < s->legacy_size) { 516 error_setg(errp, "server sent only %zd bytes of shared memory", 517 (size_t)buf.st_size); 518 close(fd); 519 return; 520 } 521 size = s->legacy_size; 522 } 523 524 /* mmap the region and map into the BAR2 */ 525 memory_region_init_ram_from_fd(&s->server_bar2, OBJECT(s), 526 "ivshmem.bar2", size, true, fd, &local_err); 527 if (local_err) { 528 error_propagate(errp, local_err); 529 return; 530 } 531 532 s->ivshmem_bar2 = &s->server_bar2; 533 } 534 535 static void process_msg_disconnect(IVShmemState *s, uint16_t posn, 536 Error **errp) 537 { 538 IVSHMEM_DPRINTF("posn %d has gone away\n", posn); 539 if (posn >= s->nb_peers || posn == s->vm_id) { 540 error_setg(errp, "invalid peer %d", posn); 541 return; 542 } 543 close_peer_eventfds(s, posn); 544 } 545 546 static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd, 547 Error **errp) 548 { 549 Peer *peer = &s->peers[posn]; 550 int vector; 551 552 /* 553 * The N-th connect message for this peer comes with the file 554 * descriptor for vector N-1. Count messages to find the vector. 555 */ 556 if (peer->nb_eventfds >= s->vectors) { 557 error_setg(errp, "Too many eventfd received, device has %d vectors", 558 s->vectors); 559 close(fd); 560 return; 561 } 562 vector = peer->nb_eventfds++; 563 564 IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd); 565 event_notifier_init_fd(&peer->eventfds[vector], fd); 566 fcntl_setfl(fd, O_NONBLOCK); /* msix/irqfd poll non block */ 567 568 if (posn == s->vm_id) { 569 setup_interrupt(s, vector, errp); 570 /* TODO do we need to handle the error? */ 571 } 572 573 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 574 ivshmem_add_eventfd(s, posn, vector); 575 } 576 } 577 578 static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp) 579 { 580 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd); 581 582 if (msg < -1 || msg > IVSHMEM_MAX_PEERS) { 583 error_setg(errp, "server sent invalid message %" PRId64, msg); 584 close(fd); 585 return; 586 } 587 588 if (msg == -1) { 589 process_msg_shmem(s, fd, errp); 590 return; 591 } 592 593 if (msg >= s->nb_peers) { 594 resize_peers(s, msg + 1); 595 } 596 597 if (fd >= 0) { 598 process_msg_connect(s, msg, fd, errp); 599 } else { 600 process_msg_disconnect(s, msg, errp); 601 } 602 } 603 604 static int ivshmem_can_receive(void *opaque) 605 { 606 IVShmemState *s = opaque; 607 608 assert(s->msg_buffered_bytes < sizeof(s->msg_buf)); 609 return sizeof(s->msg_buf) - s->msg_buffered_bytes; 610 } 611 612 static void ivshmem_read(void *opaque, const uint8_t *buf, int size) 613 { 614 IVShmemState *s = opaque; 615 Error *err = NULL; 616 int fd; 617 int64_t msg; 618 619 assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf)); 620 memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size); 621 s->msg_buffered_bytes += size; 622 if (s->msg_buffered_bytes < sizeof(s->msg_buf)) { 623 return; 624 } 625 msg = le64_to_cpu(s->msg_buf); 626 s->msg_buffered_bytes = 0; 627 628 fd = qemu_chr_fe_get_msgfd(&s->server_chr); 629 630 process_msg(s, msg, fd, &err); 631 if (err) { 632 error_report_err(err); 633 } 634 } 635 636 static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp) 637 { 638 int64_t msg; 639 int n, ret; 640 641 n = 0; 642 do { 643 ret = qemu_chr_fe_read_all(&s->server_chr, (uint8_t *)&msg + n, 644 sizeof(msg) - n); 645 if (ret < 0) { 646 if (ret == -EINTR) { 647 continue; 648 } 649 error_setg_errno(errp, -ret, "read from server failed"); 650 return INT64_MIN; 651 } 652 n += ret; 653 } while (n < sizeof(msg)); 654 655 *pfd = qemu_chr_fe_get_msgfd(&s->server_chr); 656 return le64_to_cpu(msg); 657 } 658 659 static void ivshmem_recv_setup(IVShmemState *s, Error **errp) 660 { 661 Error *err = NULL; 662 int64_t msg; 663 int fd; 664 665 msg = ivshmem_recv_msg(s, &fd, &err); 666 if (err) { 667 error_propagate(errp, err); 668 return; 669 } 670 if (msg != IVSHMEM_PROTOCOL_VERSION) { 671 error_setg(errp, "server sent version %" PRId64 ", expecting %d", 672 msg, IVSHMEM_PROTOCOL_VERSION); 673 return; 674 } 675 if (fd != -1) { 676 error_setg(errp, "server sent invalid version message"); 677 return; 678 } 679 680 /* 681 * ivshmem-server sends the remaining initial messages in a fixed 682 * order, but the device has always accepted them in any order. 683 * Stay as compatible as practical, just in case people use 684 * servers that behave differently. 685 */ 686 687 /* 688 * ivshmem_device_spec.txt has always required the ID message 689 * right here, and ivshmem-server has always complied. However, 690 * older versions of the device accepted it out of order, but 691 * broke when an interrupt setup message arrived before it. 692 */ 693 msg = ivshmem_recv_msg(s, &fd, &err); 694 if (err) { 695 error_propagate(errp, err); 696 return; 697 } 698 if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) { 699 error_setg(errp, "server sent invalid ID message"); 700 return; 701 } 702 s->vm_id = msg; 703 704 /* 705 * Receive more messages until we got shared memory. 706 */ 707 do { 708 msg = ivshmem_recv_msg(s, &fd, &err); 709 if (err) { 710 error_propagate(errp, err); 711 return; 712 } 713 process_msg(s, msg, fd, &err); 714 if (err) { 715 error_propagate(errp, err); 716 return; 717 } 718 } while (msg != -1); 719 720 /* 721 * This function must either map the shared memory or fail. The 722 * loop above ensures that: it terminates normally only after it 723 * successfully processed the server's shared memory message. 724 * Assert that actually mapped the shared memory: 725 */ 726 assert(s->ivshmem_bar2); 727 } 728 729 /* Select the MSI-X vectors used by device. 730 * ivshmem maps events to vectors statically, so 731 * we just enable all vectors on init and after reset. */ 732 static void ivshmem_msix_vector_use(IVShmemState *s) 733 { 734 PCIDevice *d = PCI_DEVICE(s); 735 int i; 736 737 for (i = 0; i < s->vectors; i++) { 738 msix_vector_use(d, i); 739 } 740 } 741 742 static void ivshmem_reset(DeviceState *d) 743 { 744 IVShmemState *s = IVSHMEM_COMMON(d); 745 746 s->intrstatus = 0; 747 s->intrmask = 0; 748 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 749 ivshmem_msix_vector_use(s); 750 } 751 } 752 753 static int ivshmem_setup_interrupts(IVShmemState *s, Error **errp) 754 { 755 /* allocate QEMU callback data for receiving interrupts */ 756 s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector)); 757 758 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 759 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1, errp)) { 760 return -1; 761 } 762 763 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors); 764 ivshmem_msix_vector_use(s); 765 } 766 767 return 0; 768 } 769 770 static void ivshmem_enable_irqfd(IVShmemState *s) 771 { 772 PCIDevice *pdev = PCI_DEVICE(s); 773 int i; 774 775 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 776 Error *err = NULL; 777 778 ivshmem_add_kvm_msi_virq(s, i, &err); 779 if (err) { 780 error_report_err(err); 781 /* TODO do we need to handle the error? */ 782 } 783 } 784 785 if (msix_set_vector_notifiers(pdev, 786 ivshmem_vector_unmask, 787 ivshmem_vector_mask, 788 ivshmem_vector_poll)) { 789 error_report("ivshmem: msix_set_vector_notifiers failed"); 790 } 791 } 792 793 static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector) 794 { 795 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector); 796 797 if (s->msi_vectors[vector].pdev == NULL) { 798 return; 799 } 800 801 /* it was cleaned when masked in the frontend. */ 802 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq); 803 804 s->msi_vectors[vector].pdev = NULL; 805 } 806 807 static void ivshmem_disable_irqfd(IVShmemState *s) 808 { 809 PCIDevice *pdev = PCI_DEVICE(s); 810 int i; 811 812 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 813 ivshmem_remove_kvm_msi_virq(s, i); 814 } 815 816 msix_unset_vector_notifiers(pdev); 817 } 818 819 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address, 820 uint32_t val, int len) 821 { 822 IVShmemState *s = IVSHMEM_COMMON(pdev); 823 int is_enabled, was_enabled = msix_enabled(pdev); 824 825 pci_default_write_config(pdev, address, val, len); 826 is_enabled = msix_enabled(pdev); 827 828 if (kvm_msi_via_irqfd_enabled()) { 829 if (!was_enabled && is_enabled) { 830 ivshmem_enable_irqfd(s); 831 } else if (was_enabled && !is_enabled) { 832 ivshmem_disable_irqfd(s); 833 } 834 } 835 } 836 837 static void ivshmem_common_realize(PCIDevice *dev, Error **errp) 838 { 839 IVShmemState *s = IVSHMEM_COMMON(dev); 840 Error *err = NULL; 841 uint8_t *pci_conf; 842 uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY | 843 PCI_BASE_ADDRESS_MEM_PREFETCH; 844 Error *local_err = NULL; 845 846 /* IRQFD requires MSI */ 847 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) && 848 !ivshmem_has_feature(s, IVSHMEM_MSI)) { 849 error_setg(errp, "ioeventfd/irqfd requires MSI"); 850 return; 851 } 852 853 pci_conf = dev->config; 854 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY; 855 856 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s, 857 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE); 858 859 /* region for registers*/ 860 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, 861 &s->ivshmem_mmio); 862 863 if (s->not_legacy_32bit) { 864 attr |= PCI_BASE_ADDRESS_MEM_TYPE_64; 865 } 866 867 if (s->hostmem != NULL) { 868 IVSHMEM_DPRINTF("using hostmem\n"); 869 870 s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem, 871 &error_abort); 872 } else { 873 Chardev *chr = qemu_chr_fe_get_driver(&s->server_chr); 874 assert(chr); 875 876 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n", 877 chr->filename); 878 879 /* we allocate enough space for 16 peers and grow as needed */ 880 resize_peers(s, 16); 881 882 /* 883 * Receive setup messages from server synchronously. 884 * Older versions did it asynchronously, but that creates a 885 * number of entertaining race conditions. 886 */ 887 ivshmem_recv_setup(s, &err); 888 if (err) { 889 error_propagate(errp, err); 890 return; 891 } 892 893 if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) { 894 error_setg(errp, 895 "master must connect to the server before any peers"); 896 return; 897 } 898 899 qemu_chr_fe_set_handlers(&s->server_chr, ivshmem_can_receive, 900 ivshmem_read, NULL, NULL, s, NULL, true); 901 902 if (ivshmem_setup_interrupts(s, errp) < 0) { 903 error_prepend(errp, "Failed to initialize interrupts: "); 904 return; 905 } 906 } 907 908 if (s->master == ON_OFF_AUTO_AUTO) { 909 s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 910 } 911 912 if (!ivshmem_is_master(s)) { 913 error_setg(&s->migration_blocker, 914 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'"); 915 migrate_add_blocker(s->migration_blocker, &local_err); 916 if (local_err) { 917 error_propagate(errp, local_err); 918 error_free(s->migration_blocker); 919 return; 920 } 921 } 922 923 vmstate_register_ram(s->ivshmem_bar2, DEVICE(s)); 924 pci_register_bar(PCI_DEVICE(s), 2, attr, s->ivshmem_bar2); 925 } 926 927 static void ivshmem_exit(PCIDevice *dev) 928 { 929 IVShmemState *s = IVSHMEM_COMMON(dev); 930 int i; 931 932 if (s->migration_blocker) { 933 migrate_del_blocker(s->migration_blocker); 934 error_free(s->migration_blocker); 935 } 936 937 if (memory_region_is_mapped(s->ivshmem_bar2)) { 938 if (!s->hostmem) { 939 void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2); 940 int fd; 941 942 if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) { 943 error_report("Failed to munmap shared memory %s", 944 strerror(errno)); 945 } 946 947 fd = memory_region_get_fd(s->ivshmem_bar2); 948 close(fd); 949 } 950 951 vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev)); 952 } 953 954 if (s->peers) { 955 for (i = 0; i < s->nb_peers; i++) { 956 close_peer_eventfds(s, i); 957 } 958 g_free(s->peers); 959 } 960 961 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 962 msix_uninit_exclusive_bar(dev); 963 } 964 965 g_free(s->msi_vectors); 966 } 967 968 static int ivshmem_pre_load(void *opaque) 969 { 970 IVShmemState *s = opaque; 971 972 if (!ivshmem_is_master(s)) { 973 error_report("'peer' devices are not migratable"); 974 return -EINVAL; 975 } 976 977 return 0; 978 } 979 980 static int ivshmem_post_load(void *opaque, int version_id) 981 { 982 IVShmemState *s = opaque; 983 984 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 985 ivshmem_msix_vector_use(s); 986 } 987 return 0; 988 } 989 990 static void ivshmem_common_class_init(ObjectClass *klass, void *data) 991 { 992 DeviceClass *dc = DEVICE_CLASS(klass); 993 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 994 995 k->realize = ivshmem_common_realize; 996 k->exit = ivshmem_exit; 997 k->config_write = ivshmem_write_config; 998 k->vendor_id = PCI_VENDOR_ID_IVSHMEM; 999 k->device_id = PCI_DEVICE_ID_IVSHMEM; 1000 k->class_id = PCI_CLASS_MEMORY_RAM; 1001 k->revision = 1; 1002 dc->reset = ivshmem_reset; 1003 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1004 dc->desc = "Inter-VM shared memory"; 1005 } 1006 1007 static const TypeInfo ivshmem_common_info = { 1008 .name = TYPE_IVSHMEM_COMMON, 1009 .parent = TYPE_PCI_DEVICE, 1010 .instance_size = sizeof(IVShmemState), 1011 .abstract = true, 1012 .class_init = ivshmem_common_class_init, 1013 .interfaces = (InterfaceInfo[]) { 1014 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 1015 { }, 1016 }, 1017 }; 1018 1019 static const VMStateDescription ivshmem_plain_vmsd = { 1020 .name = TYPE_IVSHMEM_PLAIN, 1021 .version_id = 0, 1022 .minimum_version_id = 0, 1023 .pre_load = ivshmem_pre_load, 1024 .post_load = ivshmem_post_load, 1025 .fields = (VMStateField[]) { 1026 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1027 VMSTATE_UINT32(intrstatus, IVShmemState), 1028 VMSTATE_UINT32(intrmask, IVShmemState), 1029 VMSTATE_END_OF_LIST() 1030 }, 1031 }; 1032 1033 static Property ivshmem_plain_properties[] = { 1034 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF), 1035 DEFINE_PROP_LINK("memdev", IVShmemState, hostmem, TYPE_MEMORY_BACKEND, 1036 HostMemoryBackend *), 1037 DEFINE_PROP_END_OF_LIST(), 1038 }; 1039 1040 static void ivshmem_plain_init(Object *obj) 1041 { 1042 IVShmemState *s = IVSHMEM_PLAIN(obj); 1043 1044 s->not_legacy_32bit = 1; 1045 } 1046 1047 static void ivshmem_plain_realize(PCIDevice *dev, Error **errp) 1048 { 1049 IVShmemState *s = IVSHMEM_COMMON(dev); 1050 1051 if (!s->hostmem) { 1052 error_setg(errp, "You must specify a 'memdev'"); 1053 return; 1054 } else if (host_memory_backend_is_mapped(s->hostmem)) { 1055 char *path = object_get_canonical_path_component(OBJECT(s->hostmem)); 1056 error_setg(errp, "can't use already busy memdev: %s", path); 1057 g_free(path); 1058 return; 1059 } 1060 1061 ivshmem_common_realize(dev, errp); 1062 host_memory_backend_set_mapped(s->hostmem, true); 1063 } 1064 1065 static void ivshmem_plain_exit(PCIDevice *pci_dev) 1066 { 1067 IVShmemState *s = IVSHMEM_COMMON(pci_dev); 1068 1069 host_memory_backend_set_mapped(s->hostmem, false); 1070 } 1071 1072 static void ivshmem_plain_class_init(ObjectClass *klass, void *data) 1073 { 1074 DeviceClass *dc = DEVICE_CLASS(klass); 1075 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1076 1077 k->realize = ivshmem_plain_realize; 1078 k->exit = ivshmem_plain_exit; 1079 dc->props = ivshmem_plain_properties; 1080 dc->vmsd = &ivshmem_plain_vmsd; 1081 } 1082 1083 static const TypeInfo ivshmem_plain_info = { 1084 .name = TYPE_IVSHMEM_PLAIN, 1085 .parent = TYPE_IVSHMEM_COMMON, 1086 .instance_size = sizeof(IVShmemState), 1087 .instance_init = ivshmem_plain_init, 1088 .class_init = ivshmem_plain_class_init, 1089 }; 1090 1091 static const VMStateDescription ivshmem_doorbell_vmsd = { 1092 .name = TYPE_IVSHMEM_DOORBELL, 1093 .version_id = 0, 1094 .minimum_version_id = 0, 1095 .pre_load = ivshmem_pre_load, 1096 .post_load = ivshmem_post_load, 1097 .fields = (VMStateField[]) { 1098 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1099 VMSTATE_MSIX(parent_obj, IVShmemState), 1100 VMSTATE_UINT32(intrstatus, IVShmemState), 1101 VMSTATE_UINT32(intrmask, IVShmemState), 1102 VMSTATE_END_OF_LIST() 1103 }, 1104 }; 1105 1106 static Property ivshmem_doorbell_properties[] = { 1107 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1108 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1109 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, 1110 true), 1111 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF), 1112 DEFINE_PROP_END_OF_LIST(), 1113 }; 1114 1115 static void ivshmem_doorbell_init(Object *obj) 1116 { 1117 IVShmemState *s = IVSHMEM_DOORBELL(obj); 1118 1119 s->features |= (1 << IVSHMEM_MSI); 1120 s->legacy_size = SIZE_MAX; /* whatever the server sends */ 1121 s->not_legacy_32bit = 1; 1122 } 1123 1124 static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp) 1125 { 1126 IVShmemState *s = IVSHMEM_COMMON(dev); 1127 1128 if (!qemu_chr_fe_backend_connected(&s->server_chr)) { 1129 error_setg(errp, "You must specify a 'chardev'"); 1130 return; 1131 } 1132 1133 ivshmem_common_realize(dev, errp); 1134 } 1135 1136 static void ivshmem_doorbell_class_init(ObjectClass *klass, void *data) 1137 { 1138 DeviceClass *dc = DEVICE_CLASS(klass); 1139 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1140 1141 k->realize = ivshmem_doorbell_realize; 1142 dc->props = ivshmem_doorbell_properties; 1143 dc->vmsd = &ivshmem_doorbell_vmsd; 1144 } 1145 1146 static const TypeInfo ivshmem_doorbell_info = { 1147 .name = TYPE_IVSHMEM_DOORBELL, 1148 .parent = TYPE_IVSHMEM_COMMON, 1149 .instance_size = sizeof(IVShmemState), 1150 .instance_init = ivshmem_doorbell_init, 1151 .class_init = ivshmem_doorbell_class_init, 1152 }; 1153 1154 static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id) 1155 { 1156 IVShmemState *s = opaque; 1157 PCIDevice *pdev = PCI_DEVICE(s); 1158 int ret; 1159 1160 IVSHMEM_DPRINTF("ivshmem_load_old\n"); 1161 1162 if (version_id != 0) { 1163 return -EINVAL; 1164 } 1165 1166 ret = ivshmem_pre_load(s); 1167 if (ret) { 1168 return ret; 1169 } 1170 1171 ret = pci_device_load(pdev, f); 1172 if (ret) { 1173 return ret; 1174 } 1175 1176 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1177 msix_load(pdev, f); 1178 ivshmem_msix_vector_use(s); 1179 } else { 1180 s->intrstatus = qemu_get_be32(f); 1181 s->intrmask = qemu_get_be32(f); 1182 } 1183 1184 return 0; 1185 } 1186 1187 static bool test_msix(void *opaque, int version_id) 1188 { 1189 IVShmemState *s = opaque; 1190 1191 return ivshmem_has_feature(s, IVSHMEM_MSI); 1192 } 1193 1194 static bool test_no_msix(void *opaque, int version_id) 1195 { 1196 return !test_msix(opaque, version_id); 1197 } 1198 1199 static const VMStateDescription ivshmem_vmsd = { 1200 .name = "ivshmem", 1201 .version_id = 1, 1202 .minimum_version_id = 1, 1203 .pre_load = ivshmem_pre_load, 1204 .post_load = ivshmem_post_load, 1205 .fields = (VMStateField[]) { 1206 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1207 1208 VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix), 1209 VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix), 1210 VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix), 1211 1212 VMSTATE_END_OF_LIST() 1213 }, 1214 .load_state_old = ivshmem_load_old, 1215 .minimum_version_id_old = 0 1216 }; 1217 1218 static Property ivshmem_properties[] = { 1219 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1220 DEFINE_PROP_STRING("size", IVShmemState, sizearg), 1221 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1222 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, 1223 false), 1224 DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true), 1225 DEFINE_PROP_STRING("shm", IVShmemState, shmobj), 1226 DEFINE_PROP_STRING("role", IVShmemState, role), 1227 DEFINE_PROP_UINT32("use64", IVShmemState, not_legacy_32bit, 1), 1228 DEFINE_PROP_END_OF_LIST(), 1229 }; 1230 1231 static void desugar_shm(IVShmemState *s) 1232 { 1233 Object *obj; 1234 char *path; 1235 1236 obj = object_new("memory-backend-file"); 1237 path = g_strdup_printf("/dev/shm/%s", s->shmobj); 1238 object_property_set_str(obj, path, "mem-path", &error_abort); 1239 g_free(path); 1240 object_property_set_int(obj, s->legacy_size, "size", &error_abort); 1241 object_property_set_bool(obj, true, "share", &error_abort); 1242 object_property_add_child(OBJECT(s), "internal-shm-backend", obj, 1243 &error_abort); 1244 user_creatable_complete(obj, &error_abort); 1245 s->hostmem = MEMORY_BACKEND(obj); 1246 } 1247 1248 static void ivshmem_realize(PCIDevice *dev, Error **errp) 1249 { 1250 IVShmemState *s = IVSHMEM_COMMON(dev); 1251 1252 if (!qtest_enabled()) { 1253 error_report("ivshmem is deprecated, please use ivshmem-plain" 1254 " or ivshmem-doorbell instead"); 1255 } 1256 1257 if (qemu_chr_fe_backend_connected(&s->server_chr) + !!s->shmobj != 1) { 1258 error_setg(errp, "You must specify either 'shm' or 'chardev'"); 1259 return; 1260 } 1261 1262 if (s->sizearg == NULL) { 1263 s->legacy_size = 4 << 20; /* 4 MB default */ 1264 } else { 1265 int ret; 1266 uint64_t size; 1267 1268 ret = qemu_strtosz_MiB(s->sizearg, NULL, &size); 1269 if (ret < 0 || (size_t)size != size || !is_power_of_2(size)) { 1270 error_setg(errp, "Invalid size %s", s->sizearg); 1271 return; 1272 } 1273 s->legacy_size = size; 1274 } 1275 1276 /* check that role is reasonable */ 1277 if (s->role) { 1278 if (strncmp(s->role, "peer", 5) == 0) { 1279 s->master = ON_OFF_AUTO_OFF; 1280 } else if (strncmp(s->role, "master", 7) == 0) { 1281 s->master = ON_OFF_AUTO_ON; 1282 } else { 1283 error_setg(errp, "'role' must be 'peer' or 'master'"); 1284 return; 1285 } 1286 } else { 1287 s->master = ON_OFF_AUTO_AUTO; 1288 } 1289 1290 if (s->shmobj) { 1291 desugar_shm(s); 1292 } 1293 1294 /* 1295 * Note: we don't use INTx with IVSHMEM_MSI at all, so this is a 1296 * bald-faced lie then. But it's a backwards compatible lie. 1297 */ 1298 pci_config_set_interrupt_pin(dev->config, 1); 1299 1300 ivshmem_common_realize(dev, errp); 1301 } 1302 1303 static void ivshmem_class_init(ObjectClass *klass, void *data) 1304 { 1305 DeviceClass *dc = DEVICE_CLASS(klass); 1306 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1307 1308 k->realize = ivshmem_realize; 1309 k->revision = 0; 1310 dc->desc = "Inter-VM shared memory (legacy)"; 1311 dc->props = ivshmem_properties; 1312 dc->vmsd = &ivshmem_vmsd; 1313 } 1314 1315 static const TypeInfo ivshmem_info = { 1316 .name = TYPE_IVSHMEM, 1317 .parent = TYPE_IVSHMEM_COMMON, 1318 .instance_size = sizeof(IVShmemState), 1319 .class_init = ivshmem_class_init, 1320 }; 1321 1322 static void ivshmem_register_types(void) 1323 { 1324 type_register_static(&ivshmem_common_info); 1325 type_register_static(&ivshmem_plain_info); 1326 type_register_static(&ivshmem_doorbell_info); 1327 type_register_static(&ivshmem_info); 1328 } 1329 1330 type_init(ivshmem_register_types) 1331