1 /* 2 * Inter-VM Shared Memory PCI device. 3 * 4 * Author: 5 * Cam Macdonell <cam@cs.ualberta.ca> 6 * 7 * Based On: cirrus_vga.c 8 * Copyright (c) 2004 Fabrice Bellard 9 * Copyright (c) 2004 Makoto Suzuki (suzu) 10 * 11 * and rtl8139.c 12 * Copyright (c) 2006 Igor Kovalenko 13 * 14 * This code is licensed under the GNU GPL v2. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 */ 19 #include "hw/hw.h" 20 #include "hw/i386/pc.h" 21 #include "hw/pci/pci.h" 22 #include "hw/pci/msi.h" 23 #include "hw/pci/msix.h" 24 #include "sysemu/kvm.h" 25 #include "migration/migration.h" 26 #include "qemu/error-report.h" 27 #include "qemu/event_notifier.h" 28 #include "qemu/fifo8.h" 29 #include "sysemu/char.h" 30 #include "sysemu/hostmem.h" 31 #include "qapi/visitor.h" 32 33 #include "hw/misc/ivshmem.h" 34 35 #include <sys/mman.h> 36 #include <sys/types.h> 37 #include <limits.h> 38 39 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET 40 #define PCI_DEVICE_ID_IVSHMEM 0x1110 41 42 #define IVSHMEM_MAX_PEERS G_MAXUINT16 43 #define IVSHMEM_IOEVENTFD 0 44 #define IVSHMEM_MSI 1 45 46 #define IVSHMEM_PEER 0 47 #define IVSHMEM_MASTER 1 48 49 #define IVSHMEM_REG_BAR_SIZE 0x100 50 51 //#define DEBUG_IVSHMEM 52 #ifdef DEBUG_IVSHMEM 53 #define IVSHMEM_DPRINTF(fmt, ...) \ 54 do {printf("IVSHMEM: " fmt, ## __VA_ARGS__); } while (0) 55 #else 56 #define IVSHMEM_DPRINTF(fmt, ...) 57 #endif 58 59 #define TYPE_IVSHMEM "ivshmem" 60 #define IVSHMEM(obj) \ 61 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM) 62 63 typedef struct Peer { 64 int nb_eventfds; 65 EventNotifier *eventfds; 66 } Peer; 67 68 typedef struct MSIVector { 69 PCIDevice *pdev; 70 int virq; 71 } MSIVector; 72 73 typedef struct IVShmemState { 74 /*< private >*/ 75 PCIDevice parent_obj; 76 /*< public >*/ 77 78 HostMemoryBackend *hostmem; 79 uint32_t intrmask; 80 uint32_t intrstatus; 81 82 CharDriverState **eventfd_chr; 83 CharDriverState *server_chr; 84 Fifo8 incoming_fifo; 85 MemoryRegion ivshmem_mmio; 86 87 /* We might need to register the BAR before we actually have the memory. 88 * So prepare a container MemoryRegion for the BAR immediately and 89 * add a subregion when we have the memory. 90 */ 91 MemoryRegion bar; 92 MemoryRegion ivshmem; 93 uint64_t ivshmem_size; /* size of shared memory region */ 94 uint32_t ivshmem_64bit; 95 96 Peer *peers; 97 int nb_peers; /* how many peers we have space for */ 98 99 int vm_id; 100 uint32_t vectors; 101 uint32_t features; 102 MSIVector *msi_vectors; 103 104 Error *migration_blocker; 105 106 char * shmobj; 107 char * sizearg; 108 char * role; 109 int role_val; /* scalar to avoid multiple string comparisons */ 110 } IVShmemState; 111 112 /* registers for the Inter-VM shared memory device */ 113 enum ivshmem_registers { 114 INTRMASK = 0, 115 INTRSTATUS = 4, 116 IVPOSITION = 8, 117 DOORBELL = 12, 118 }; 119 120 static inline uint32_t ivshmem_has_feature(IVShmemState *ivs, 121 unsigned int feature) { 122 return (ivs->features & (1 << feature)); 123 } 124 125 /* accessing registers - based on rtl8139 */ 126 static void ivshmem_update_irq(IVShmemState *s) 127 { 128 PCIDevice *d = PCI_DEVICE(s); 129 int isr; 130 isr = (s->intrstatus & s->intrmask) & 0xffffffff; 131 132 /* don't print ISR resets */ 133 if (isr) { 134 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n", 135 isr ? 1 : 0, s->intrstatus, s->intrmask); 136 } 137 138 pci_set_irq(d, (isr != 0)); 139 } 140 141 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val) 142 { 143 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val); 144 145 s->intrmask = val; 146 147 ivshmem_update_irq(s); 148 } 149 150 static uint32_t ivshmem_IntrMask_read(IVShmemState *s) 151 { 152 uint32_t ret = s->intrmask; 153 154 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret); 155 156 return ret; 157 } 158 159 static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val) 160 { 161 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val); 162 163 s->intrstatus = val; 164 165 ivshmem_update_irq(s); 166 } 167 168 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s) 169 { 170 uint32_t ret = s->intrstatus; 171 172 /* reading ISR clears all interrupts */ 173 s->intrstatus = 0; 174 175 ivshmem_update_irq(s); 176 177 return ret; 178 } 179 180 static void ivshmem_io_write(void *opaque, hwaddr addr, 181 uint64_t val, unsigned size) 182 { 183 IVShmemState *s = opaque; 184 185 uint16_t dest = val >> 16; 186 uint16_t vector = val & 0xff; 187 188 addr &= 0xfc; 189 190 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx "\n", addr); 191 switch (addr) 192 { 193 case INTRMASK: 194 ivshmem_IntrMask_write(s, val); 195 break; 196 197 case INTRSTATUS: 198 ivshmem_IntrStatus_write(s, val); 199 break; 200 201 case DOORBELL: 202 /* check that dest VM ID is reasonable */ 203 if (dest >= s->nb_peers) { 204 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest); 205 break; 206 } 207 208 /* check doorbell range */ 209 if (vector < s->peers[dest].nb_eventfds) { 210 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector); 211 event_notifier_set(&s->peers[dest].eventfds[vector]); 212 } else { 213 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n", 214 vector, dest); 215 } 216 break; 217 default: 218 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr); 219 } 220 } 221 222 static uint64_t ivshmem_io_read(void *opaque, hwaddr addr, 223 unsigned size) 224 { 225 226 IVShmemState *s = opaque; 227 uint32_t ret; 228 229 switch (addr) 230 { 231 case INTRMASK: 232 ret = ivshmem_IntrMask_read(s); 233 break; 234 235 case INTRSTATUS: 236 ret = ivshmem_IntrStatus_read(s); 237 break; 238 239 case IVPOSITION: 240 /* return my VM ID if the memory is mapped */ 241 if (memory_region_is_mapped(&s->ivshmem)) { 242 ret = s->vm_id; 243 } else { 244 ret = -1; 245 } 246 break; 247 248 default: 249 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx "\n", addr); 250 ret = 0; 251 } 252 253 return ret; 254 } 255 256 static const MemoryRegionOps ivshmem_mmio_ops = { 257 .read = ivshmem_io_read, 258 .write = ivshmem_io_write, 259 .endianness = DEVICE_NATIVE_ENDIAN, 260 .impl = { 261 .min_access_size = 4, 262 .max_access_size = 4, 263 }, 264 }; 265 266 static void ivshmem_receive(void *opaque, const uint8_t *buf, int size) 267 { 268 IVShmemState *s = opaque; 269 270 IVSHMEM_DPRINTF("ivshmem_receive 0x%02x size: %d\n", *buf, size); 271 272 ivshmem_IntrStatus_write(s, *buf); 273 } 274 275 static int ivshmem_can_receive(void * opaque) 276 { 277 return sizeof(int64_t); 278 } 279 280 static void ivshmem_event(void *opaque, int event) 281 { 282 IVSHMEM_DPRINTF("ivshmem_event %d\n", event); 283 } 284 285 static void fake_irqfd(void *opaque, const uint8_t *buf, int size) { 286 287 MSIVector *entry = opaque; 288 PCIDevice *pdev = entry->pdev; 289 IVShmemState *s = IVSHMEM(pdev); 290 int vector = entry - s->msi_vectors; 291 292 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector); 293 msix_notify(pdev, vector); 294 } 295 296 static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector, 297 MSIMessage msg) 298 { 299 IVShmemState *s = IVSHMEM(dev); 300 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 301 MSIVector *v = &s->msi_vectors[vector]; 302 int ret; 303 304 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector); 305 306 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev); 307 if (ret < 0) { 308 return ret; 309 } 310 311 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq); 312 } 313 314 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector) 315 { 316 IVShmemState *s = IVSHMEM(dev); 317 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 318 int ret; 319 320 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector); 321 322 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, 323 s->msi_vectors[vector].virq); 324 if (ret != 0) { 325 error_report("remove_irqfd_notifier_gsi failed"); 326 } 327 } 328 329 static void ivshmem_vector_poll(PCIDevice *dev, 330 unsigned int vector_start, 331 unsigned int vector_end) 332 { 333 IVShmemState *s = IVSHMEM(dev); 334 unsigned int vector; 335 336 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end); 337 338 vector_end = MIN(vector_end, s->vectors); 339 340 for (vector = vector_start; vector < vector_end; vector++) { 341 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector]; 342 343 if (!msix_is_masked(dev, vector)) { 344 continue; 345 } 346 347 if (event_notifier_test_and_clear(notifier)) { 348 msix_set_pending(dev, vector); 349 } 350 } 351 } 352 353 static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *n, 354 int vector) 355 { 356 /* create a event character device based on the passed eventfd */ 357 IVShmemState *s = opaque; 358 PCIDevice *pdev = PCI_DEVICE(s); 359 int eventfd = event_notifier_get_fd(n); 360 CharDriverState *chr; 361 362 s->msi_vectors[vector].pdev = pdev; 363 364 chr = qemu_chr_open_eventfd(eventfd); 365 366 if (chr == NULL) { 367 error_report("creating chardriver for eventfd %d failed", eventfd); 368 return NULL; 369 } 370 qemu_chr_fe_claim_no_fail(chr); 371 372 /* if MSI is supported we need multiple interrupts */ 373 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 374 s->msi_vectors[vector].pdev = PCI_DEVICE(s); 375 376 qemu_chr_add_handlers(chr, ivshmem_can_receive, fake_irqfd, 377 ivshmem_event, &s->msi_vectors[vector]); 378 } else { 379 qemu_chr_add_handlers(chr, ivshmem_can_receive, ivshmem_receive, 380 ivshmem_event, s); 381 } 382 383 return chr; 384 385 } 386 387 static int check_shm_size(IVShmemState *s, int fd, Error **errp) 388 { 389 /* check that the guest isn't going to try and map more memory than the 390 * the object has allocated return -1 to indicate error */ 391 392 struct stat buf; 393 394 if (fstat(fd, &buf) < 0) { 395 error_setg(errp, "exiting: fstat on fd %d failed: %s", 396 fd, strerror(errno)); 397 return -1; 398 } 399 400 if (s->ivshmem_size > buf.st_size) { 401 error_setg(errp, "Requested memory size greater" 402 " than shared object size (%" PRIu64 " > %" PRIu64")", 403 s->ivshmem_size, (uint64_t)buf.st_size); 404 return -1; 405 } else { 406 return 0; 407 } 408 } 409 410 /* create the shared memory BAR when we are not using the server, so we can 411 * create the BAR and map the memory immediately */ 412 static int create_shared_memory_BAR(IVShmemState *s, int fd, uint8_t attr, 413 Error **errp) 414 { 415 void * ptr; 416 417 ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 418 if (ptr == MAP_FAILED) { 419 error_setg_errno(errp, errno, "Failed to mmap shared memory"); 420 return -1; 421 } 422 423 memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), "ivshmem.bar2", 424 s->ivshmem_size, ptr); 425 vmstate_register_ram(&s->ivshmem, DEVICE(s)); 426 memory_region_add_subregion(&s->bar, 0, &s->ivshmem); 427 428 /* region for shared memory */ 429 pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar); 430 431 return 0; 432 } 433 434 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i) 435 { 436 memory_region_add_eventfd(&s->ivshmem_mmio, 437 DOORBELL, 438 4, 439 true, 440 (posn << 16) | i, 441 &s->peers[posn].eventfds[i]); 442 } 443 444 static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i) 445 { 446 memory_region_del_eventfd(&s->ivshmem_mmio, 447 DOORBELL, 448 4, 449 true, 450 (posn << 16) | i, 451 &s->peers[posn].eventfds[i]); 452 } 453 454 static void close_peer_eventfds(IVShmemState *s, int posn) 455 { 456 int i, n; 457 458 if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 459 return; 460 } 461 if (posn < 0 || posn >= s->nb_peers) { 462 error_report("invalid peer %d", posn); 463 return; 464 } 465 466 n = s->peers[posn].nb_eventfds; 467 468 memory_region_transaction_begin(); 469 for (i = 0; i < n; i++) { 470 ivshmem_del_eventfd(s, posn, i); 471 } 472 memory_region_transaction_commit(); 473 for (i = 0; i < n; i++) { 474 event_notifier_cleanup(&s->peers[posn].eventfds[i]); 475 } 476 477 g_free(s->peers[posn].eventfds); 478 s->peers[posn].nb_eventfds = 0; 479 } 480 481 /* this function increase the dynamic storage need to store data about other 482 * peers */ 483 static int resize_peers(IVShmemState *s, int new_min_size) 484 { 485 486 int j, old_size; 487 488 /* limit number of max peers */ 489 if (new_min_size <= 0 || new_min_size > IVSHMEM_MAX_PEERS) { 490 return -1; 491 } 492 if (new_min_size <= s->nb_peers) { 493 return 0; 494 } 495 496 old_size = s->nb_peers; 497 s->nb_peers = new_min_size; 498 499 IVSHMEM_DPRINTF("bumping storage to %d peers\n", s->nb_peers); 500 501 s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer)); 502 503 for (j = old_size; j < s->nb_peers; j++) { 504 s->peers[j].eventfds = g_new0(EventNotifier, s->vectors); 505 s->peers[j].nb_eventfds = 0; 506 } 507 508 return 0; 509 } 510 511 static bool fifo_update_and_get(IVShmemState *s, const uint8_t *buf, int size, 512 void *data, size_t len) 513 { 514 const uint8_t *p; 515 uint32_t num; 516 517 assert(len <= sizeof(int64_t)); /* limitation of the fifo */ 518 if (fifo8_is_empty(&s->incoming_fifo) && size == len) { 519 memcpy(data, buf, size); 520 return true; 521 } 522 523 IVSHMEM_DPRINTF("short read of %d bytes\n", size); 524 525 num = MIN(size, sizeof(int64_t) - fifo8_num_used(&s->incoming_fifo)); 526 fifo8_push_all(&s->incoming_fifo, buf, num); 527 528 if (fifo8_num_used(&s->incoming_fifo) < len) { 529 assert(num == 0); 530 return false; 531 } 532 533 size -= num; 534 buf += num; 535 p = fifo8_pop_buf(&s->incoming_fifo, len, &num); 536 assert(num == len); 537 538 memcpy(data, p, len); 539 540 if (size > 0) { 541 fifo8_push_all(&s->incoming_fifo, buf, size); 542 } 543 544 return true; 545 } 546 547 static bool fifo_update_and_get_i64(IVShmemState *s, 548 const uint8_t *buf, int size, int64_t *i64) 549 { 550 if (fifo_update_and_get(s, buf, size, i64, sizeof(*i64))) { 551 *i64 = GINT64_FROM_LE(*i64); 552 return true; 553 } 554 555 return false; 556 } 557 558 static int ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector) 559 { 560 PCIDevice *pdev = PCI_DEVICE(s); 561 MSIMessage msg = msix_get_message(pdev, vector); 562 int ret; 563 564 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector); 565 566 if (s->msi_vectors[vector].pdev != NULL) { 567 return 0; 568 } 569 570 ret = kvm_irqchip_add_msi_route(kvm_state, msg, pdev); 571 if (ret < 0) { 572 error_report("ivshmem: kvm_irqchip_add_msi_route failed"); 573 return -1; 574 } 575 576 s->msi_vectors[vector].virq = ret; 577 s->msi_vectors[vector].pdev = pdev; 578 579 return 0; 580 } 581 582 static void setup_interrupt(IVShmemState *s, int vector) 583 { 584 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 585 bool with_irqfd = kvm_msi_via_irqfd_enabled() && 586 ivshmem_has_feature(s, IVSHMEM_MSI); 587 PCIDevice *pdev = PCI_DEVICE(s); 588 589 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector); 590 591 if (!with_irqfd) { 592 IVSHMEM_DPRINTF("with eventfd"); 593 s->eventfd_chr[vector] = create_eventfd_chr_device(s, n, vector); 594 } else if (msix_enabled(pdev)) { 595 IVSHMEM_DPRINTF("with irqfd"); 596 if (ivshmem_add_kvm_msi_virq(s, vector) < 0) { 597 return; 598 } 599 600 if (!msix_is_masked(pdev, vector)) { 601 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, 602 s->msi_vectors[vector].virq); 603 } 604 } else { 605 /* it will be delayed until msix is enabled, in write_config */ 606 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled"); 607 } 608 } 609 610 static void ivshmem_read(void *opaque, const uint8_t *buf, int size) 611 { 612 IVShmemState *s = opaque; 613 int incoming_fd; 614 int new_eventfd; 615 int64_t incoming_posn; 616 Error *err = NULL; 617 Peer *peer; 618 619 if (!fifo_update_and_get_i64(s, buf, size, &incoming_posn)) { 620 return; 621 } 622 623 if (incoming_posn < -1) { 624 IVSHMEM_DPRINTF("invalid incoming_posn %" PRId64 "\n", incoming_posn); 625 return; 626 } 627 628 /* pick off s->server_chr->msgfd and store it, posn should accompany msg */ 629 incoming_fd = qemu_chr_fe_get_msgfd(s->server_chr); 630 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", 631 incoming_posn, incoming_fd); 632 633 /* make sure we have enough space for this peer */ 634 if (incoming_posn >= s->nb_peers) { 635 if (resize_peers(s, incoming_posn + 1) < 0) { 636 error_report("failed to resize peers array"); 637 if (incoming_fd != -1) { 638 close(incoming_fd); 639 } 640 return; 641 } 642 } 643 644 peer = &s->peers[incoming_posn]; 645 646 if (incoming_fd == -1) { 647 /* if posn is positive and unseen before then this is our posn*/ 648 if (incoming_posn >= 0 && s->vm_id == -1) { 649 /* receive our posn */ 650 s->vm_id = incoming_posn; 651 } else { 652 /* otherwise an fd == -1 means an existing peer has gone away */ 653 IVSHMEM_DPRINTF("posn %" PRId64 " has gone away\n", incoming_posn); 654 close_peer_eventfds(s, incoming_posn); 655 } 656 return; 657 } 658 659 /* if the position is -1, then it's shared memory region fd */ 660 if (incoming_posn == -1) { 661 void * map_ptr; 662 663 if (memory_region_is_mapped(&s->ivshmem)) { 664 error_report("shm already initialized"); 665 close(incoming_fd); 666 return; 667 } 668 669 if (check_shm_size(s, incoming_fd, &err) == -1) { 670 error_report_err(err); 671 close(incoming_fd); 672 return; 673 } 674 675 /* mmap the region and map into the BAR2 */ 676 map_ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, 677 incoming_fd, 0); 678 if (map_ptr == MAP_FAILED) { 679 error_report("Failed to mmap shared memory %s", strerror(errno)); 680 close(incoming_fd); 681 return; 682 } 683 memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), 684 "ivshmem.bar2", s->ivshmem_size, map_ptr); 685 vmstate_register_ram(&s->ivshmem, DEVICE(s)); 686 687 IVSHMEM_DPRINTF("guest h/w addr = %p, size = %" PRIu64 "\n", 688 map_ptr, s->ivshmem_size); 689 690 memory_region_add_subregion(&s->bar, 0, &s->ivshmem); 691 692 close(incoming_fd); 693 return; 694 } 695 696 /* each peer has an associated array of eventfds, and we keep 697 * track of how many eventfds received so far */ 698 /* get a new eventfd: */ 699 if (peer->nb_eventfds >= s->vectors) { 700 error_report("Too many eventfd received, device has %d vectors", 701 s->vectors); 702 close(incoming_fd); 703 return; 704 } 705 706 new_eventfd = peer->nb_eventfds++; 707 708 /* this is an eventfd for a particular peer VM */ 709 IVSHMEM_DPRINTF("eventfds[%" PRId64 "][%d] = %d\n", incoming_posn, 710 new_eventfd, incoming_fd); 711 event_notifier_init_fd(&peer->eventfds[new_eventfd], incoming_fd); 712 fcntl_setfl(incoming_fd, O_NONBLOCK); /* msix/irqfd poll non block */ 713 714 if (incoming_posn == s->vm_id) { 715 setup_interrupt(s, new_eventfd); 716 } 717 718 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 719 ivshmem_add_eventfd(s, incoming_posn, new_eventfd); 720 } 721 } 722 723 static void ivshmem_check_version(void *opaque, const uint8_t * buf, int size) 724 { 725 IVShmemState *s = opaque; 726 int tmp; 727 int64_t version; 728 729 if (!fifo_update_and_get_i64(s, buf, size, &version)) { 730 return; 731 } 732 733 tmp = qemu_chr_fe_get_msgfd(s->server_chr); 734 if (tmp != -1 || version != IVSHMEM_PROTOCOL_VERSION) { 735 fprintf(stderr, "incompatible version, you are connecting to a ivshmem-" 736 "server using a different protocol please check your setup\n"); 737 qemu_chr_delete(s->server_chr); 738 s->server_chr = NULL; 739 return; 740 } 741 742 IVSHMEM_DPRINTF("version check ok, switch to real chardev handler\n"); 743 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read, 744 ivshmem_event, s); 745 } 746 747 /* Select the MSI-X vectors used by device. 748 * ivshmem maps events to vectors statically, so 749 * we just enable all vectors on init and after reset. */ 750 static void ivshmem_use_msix(IVShmemState * s) 751 { 752 PCIDevice *d = PCI_DEVICE(s); 753 int i; 754 755 IVSHMEM_DPRINTF("%s, msix present: %d\n", __func__, msix_present(d)); 756 if (!msix_present(d)) { 757 return; 758 } 759 760 for (i = 0; i < s->vectors; i++) { 761 msix_vector_use(d, i); 762 } 763 } 764 765 static void ivshmem_reset(DeviceState *d) 766 { 767 IVShmemState *s = IVSHMEM(d); 768 769 s->intrstatus = 0; 770 s->intrmask = 0; 771 ivshmem_use_msix(s); 772 } 773 774 static int ivshmem_setup_msi(IVShmemState * s) 775 { 776 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) { 777 return -1; 778 } 779 780 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors); 781 782 /* allocate QEMU char devices for receiving interrupts */ 783 s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector)); 784 785 ivshmem_use_msix(s); 786 return 0; 787 } 788 789 static void ivshmem_enable_irqfd(IVShmemState *s) 790 { 791 PCIDevice *pdev = PCI_DEVICE(s); 792 int i; 793 794 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 795 ivshmem_add_kvm_msi_virq(s, i); 796 } 797 798 if (msix_set_vector_notifiers(pdev, 799 ivshmem_vector_unmask, 800 ivshmem_vector_mask, 801 ivshmem_vector_poll)) { 802 error_report("ivshmem: msix_set_vector_notifiers failed"); 803 } 804 } 805 806 static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector) 807 { 808 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector); 809 810 if (s->msi_vectors[vector].pdev == NULL) { 811 return; 812 } 813 814 /* it was cleaned when masked in the frontend. */ 815 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq); 816 817 s->msi_vectors[vector].pdev = NULL; 818 } 819 820 static void ivshmem_disable_irqfd(IVShmemState *s) 821 { 822 PCIDevice *pdev = PCI_DEVICE(s); 823 int i; 824 825 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 826 ivshmem_remove_kvm_msi_virq(s, i); 827 } 828 829 msix_unset_vector_notifiers(pdev); 830 } 831 832 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address, 833 uint32_t val, int len) 834 { 835 IVShmemState *s = IVSHMEM(pdev); 836 int is_enabled, was_enabled = msix_enabled(pdev); 837 838 pci_default_write_config(pdev, address, val, len); 839 is_enabled = msix_enabled(pdev); 840 841 if (kvm_msi_via_irqfd_enabled() && s->vm_id != -1) { 842 if (!was_enabled && is_enabled) { 843 ivshmem_enable_irqfd(s); 844 } else if (was_enabled && !is_enabled) { 845 ivshmem_disable_irqfd(s); 846 } 847 } 848 } 849 850 static void pci_ivshmem_realize(PCIDevice *dev, Error **errp) 851 { 852 IVShmemState *s = IVSHMEM(dev); 853 uint8_t *pci_conf; 854 uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY | 855 PCI_BASE_ADDRESS_MEM_PREFETCH; 856 857 if (!!s->server_chr + !!s->shmobj + !!s->hostmem != 1) { 858 error_setg(errp, 859 "You must specify either 'shm', 'chardev' or 'x-memdev'"); 860 return; 861 } 862 863 if (s->hostmem) { 864 MemoryRegion *mr; 865 866 if (s->sizearg) { 867 g_warning("size argument ignored with hostmem"); 868 } 869 870 mr = host_memory_backend_get_memory(s->hostmem, errp); 871 s->ivshmem_size = memory_region_size(mr); 872 } else if (s->sizearg == NULL) { 873 s->ivshmem_size = 4 << 20; /* 4 MB default */ 874 } else { 875 char *end; 876 int64_t size = qemu_strtosz(s->sizearg, &end); 877 if (size < 0 || *end != '\0' || !is_power_of_2(size)) { 878 error_setg(errp, "Invalid size %s", s->sizearg); 879 return; 880 } 881 s->ivshmem_size = size; 882 } 883 884 fifo8_create(&s->incoming_fifo, sizeof(int64_t)); 885 886 /* IRQFD requires MSI */ 887 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) && 888 !ivshmem_has_feature(s, IVSHMEM_MSI)) { 889 error_setg(errp, "ioeventfd/irqfd requires MSI"); 890 return; 891 } 892 893 /* check that role is reasonable */ 894 if (s->role) { 895 if (strncmp(s->role, "peer", 5) == 0) { 896 s->role_val = IVSHMEM_PEER; 897 } else if (strncmp(s->role, "master", 7) == 0) { 898 s->role_val = IVSHMEM_MASTER; 899 } else { 900 error_setg(errp, "'role' must be 'peer' or 'master'"); 901 return; 902 } 903 } else { 904 s->role_val = IVSHMEM_MASTER; /* default */ 905 } 906 907 if (s->role_val == IVSHMEM_PEER) { 908 error_setg(&s->migration_blocker, 909 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'"); 910 migrate_add_blocker(s->migration_blocker); 911 } 912 913 pci_conf = dev->config; 914 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY; 915 916 pci_config_set_interrupt_pin(pci_conf, 1); 917 918 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s, 919 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE); 920 921 /* region for registers*/ 922 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, 923 &s->ivshmem_mmio); 924 925 memory_region_init(&s->bar, OBJECT(s), "ivshmem-bar2-container", s->ivshmem_size); 926 if (s->ivshmem_64bit) { 927 attr |= PCI_BASE_ADDRESS_MEM_TYPE_64; 928 } 929 930 if (s->hostmem != NULL) { 931 MemoryRegion *mr; 932 933 IVSHMEM_DPRINTF("using hostmem\n"); 934 935 mr = host_memory_backend_get_memory(MEMORY_BACKEND(s->hostmem), errp); 936 vmstate_register_ram(mr, DEVICE(s)); 937 memory_region_add_subregion(&s->bar, 0, mr); 938 pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar); 939 } else if (s->server_chr != NULL) { 940 /* FIXME do not rely on what chr drivers put into filename */ 941 if (strncmp(s->server_chr->filename, "unix:", 5)) { 942 error_setg(errp, "chardev is not a unix client socket"); 943 return; 944 } 945 946 /* if we get a UNIX socket as the parameter we will talk 947 * to the ivshmem server to receive the memory region */ 948 949 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n", 950 s->server_chr->filename); 951 952 if (ivshmem_has_feature(s, IVSHMEM_MSI) && 953 ivshmem_setup_msi(s)) { 954 error_setg(errp, "msix initialization failed"); 955 return; 956 } 957 958 /* we allocate enough space for 16 peers and grow as needed */ 959 resize_peers(s, 16); 960 s->vm_id = -1; 961 962 pci_register_bar(dev, 2, attr, &s->bar); 963 964 s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *)); 965 966 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, 967 ivshmem_check_version, ivshmem_event, s); 968 } else { 969 /* just map the file immediately, we're not using a server */ 970 int fd; 971 972 IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s->shmobj); 973 974 /* try opening with O_EXCL and if it succeeds zero the memory 975 * by truncating to 0 */ 976 if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR|O_EXCL, 977 S_IRWXU|S_IRWXG|S_IRWXO)) > 0) { 978 /* truncate file to length PCI device's memory */ 979 if (ftruncate(fd, s->ivshmem_size) != 0) { 980 error_report("could not truncate shared file"); 981 } 982 983 } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR, 984 S_IRWXU|S_IRWXG|S_IRWXO)) < 0) { 985 error_setg(errp, "could not open shared file"); 986 return; 987 } 988 989 if (check_shm_size(s, fd, errp) == -1) { 990 return; 991 } 992 993 create_shared_memory_BAR(s, fd, attr, errp); 994 close(fd); 995 } 996 } 997 998 static void pci_ivshmem_exit(PCIDevice *dev) 999 { 1000 IVShmemState *s = IVSHMEM(dev); 1001 int i; 1002 1003 fifo8_destroy(&s->incoming_fifo); 1004 1005 if (s->migration_blocker) { 1006 migrate_del_blocker(s->migration_blocker); 1007 error_free(s->migration_blocker); 1008 } 1009 1010 if (memory_region_is_mapped(&s->ivshmem)) { 1011 if (!s->hostmem) { 1012 void *addr = memory_region_get_ram_ptr(&s->ivshmem); 1013 1014 if (munmap(addr, s->ivshmem_size) == -1) { 1015 error_report("Failed to munmap shared memory %s", 1016 strerror(errno)); 1017 } 1018 } 1019 1020 vmstate_unregister_ram(&s->ivshmem, DEVICE(dev)); 1021 memory_region_del_subregion(&s->bar, &s->ivshmem); 1022 } 1023 1024 if (s->eventfd_chr) { 1025 for (i = 0; i < s->vectors; i++) { 1026 if (s->eventfd_chr[i]) { 1027 qemu_chr_free(s->eventfd_chr[i]); 1028 } 1029 } 1030 g_free(s->eventfd_chr); 1031 } 1032 1033 if (s->peers) { 1034 for (i = 0; i < s->nb_peers; i++) { 1035 close_peer_eventfds(s, i); 1036 } 1037 g_free(s->peers); 1038 } 1039 1040 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1041 msix_uninit_exclusive_bar(dev); 1042 } 1043 1044 g_free(s->msi_vectors); 1045 } 1046 1047 static bool test_msix(void *opaque, int version_id) 1048 { 1049 IVShmemState *s = opaque; 1050 1051 return ivshmem_has_feature(s, IVSHMEM_MSI); 1052 } 1053 1054 static bool test_no_msix(void *opaque, int version_id) 1055 { 1056 return !test_msix(opaque, version_id); 1057 } 1058 1059 static int ivshmem_pre_load(void *opaque) 1060 { 1061 IVShmemState *s = opaque; 1062 1063 if (s->role_val == IVSHMEM_PEER) { 1064 error_report("'peer' devices are not migratable"); 1065 return -EINVAL; 1066 } 1067 1068 return 0; 1069 } 1070 1071 static int ivshmem_post_load(void *opaque, int version_id) 1072 { 1073 IVShmemState *s = opaque; 1074 1075 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1076 ivshmem_use_msix(s); 1077 } 1078 1079 return 0; 1080 } 1081 1082 static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id) 1083 { 1084 IVShmemState *s = opaque; 1085 PCIDevice *pdev = PCI_DEVICE(s); 1086 int ret; 1087 1088 IVSHMEM_DPRINTF("ivshmem_load_old\n"); 1089 1090 if (version_id != 0) { 1091 return -EINVAL; 1092 } 1093 1094 if (s->role_val == IVSHMEM_PEER) { 1095 error_report("'peer' devices are not migratable"); 1096 return -EINVAL; 1097 } 1098 1099 ret = pci_device_load(pdev, f); 1100 if (ret) { 1101 return ret; 1102 } 1103 1104 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1105 msix_load(pdev, f); 1106 ivshmem_use_msix(s); 1107 } else { 1108 s->intrstatus = qemu_get_be32(f); 1109 s->intrmask = qemu_get_be32(f); 1110 } 1111 1112 return 0; 1113 } 1114 1115 static const VMStateDescription ivshmem_vmsd = { 1116 .name = "ivshmem", 1117 .version_id = 1, 1118 .minimum_version_id = 1, 1119 .pre_load = ivshmem_pre_load, 1120 .post_load = ivshmem_post_load, 1121 .fields = (VMStateField[]) { 1122 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1123 1124 VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix), 1125 VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix), 1126 VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix), 1127 1128 VMSTATE_END_OF_LIST() 1129 }, 1130 .load_state_old = ivshmem_load_old, 1131 .minimum_version_id_old = 0 1132 }; 1133 1134 static Property ivshmem_properties[] = { 1135 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1136 DEFINE_PROP_STRING("size", IVShmemState, sizearg), 1137 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1138 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, false), 1139 DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true), 1140 DEFINE_PROP_STRING("shm", IVShmemState, shmobj), 1141 DEFINE_PROP_STRING("role", IVShmemState, role), 1142 DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1), 1143 DEFINE_PROP_END_OF_LIST(), 1144 }; 1145 1146 static void ivshmem_class_init(ObjectClass *klass, void *data) 1147 { 1148 DeviceClass *dc = DEVICE_CLASS(klass); 1149 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1150 1151 k->realize = pci_ivshmem_realize; 1152 k->exit = pci_ivshmem_exit; 1153 k->config_write = ivshmem_write_config; 1154 k->vendor_id = PCI_VENDOR_ID_IVSHMEM; 1155 k->device_id = PCI_DEVICE_ID_IVSHMEM; 1156 k->class_id = PCI_CLASS_MEMORY_RAM; 1157 dc->reset = ivshmem_reset; 1158 dc->props = ivshmem_properties; 1159 dc->vmsd = &ivshmem_vmsd; 1160 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1161 dc->desc = "Inter-VM shared memory"; 1162 } 1163 1164 static void ivshmem_check_memdev_is_busy(Object *obj, const char *name, 1165 Object *val, Error **errp) 1166 { 1167 MemoryRegion *mr; 1168 1169 mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), errp); 1170 if (memory_region_is_mapped(mr)) { 1171 char *path = object_get_canonical_path_component(val); 1172 error_setg(errp, "can't use already busy memdev: %s", path); 1173 g_free(path); 1174 } else { 1175 qdev_prop_allow_set_link_before_realize(obj, name, val, errp); 1176 } 1177 } 1178 1179 static void ivshmem_init(Object *obj) 1180 { 1181 IVShmemState *s = IVSHMEM(obj); 1182 1183 object_property_add_link(obj, "x-memdev", TYPE_MEMORY_BACKEND, 1184 (Object **)&s->hostmem, 1185 ivshmem_check_memdev_is_busy, 1186 OBJ_PROP_LINK_UNREF_ON_RELEASE, 1187 &error_abort); 1188 } 1189 1190 static const TypeInfo ivshmem_info = { 1191 .name = TYPE_IVSHMEM, 1192 .parent = TYPE_PCI_DEVICE, 1193 .instance_size = sizeof(IVShmemState), 1194 .instance_init = ivshmem_init, 1195 .class_init = ivshmem_class_init, 1196 }; 1197 1198 static void ivshmem_register_types(void) 1199 { 1200 type_register_static(&ivshmem_info); 1201 } 1202 1203 type_init(ivshmem_register_types) 1204