1 /* 2 * Inter-VM Shared Memory PCI device. 3 * 4 * Author: 5 * Cam Macdonell <cam@cs.ualberta.ca> 6 * 7 * Based On: cirrus_vga.c 8 * Copyright (c) 2004 Fabrice Bellard 9 * Copyright (c) 2004 Makoto Suzuki (suzu) 10 * 11 * and rtl8139.c 12 * Copyright (c) 2006 Igor Kovalenko 13 * 14 * This code is licensed under the GNU GPL v2. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 */ 19 #include "qemu/osdep.h" 20 #include "hw/hw.h" 21 #include "hw/i386/pc.h" 22 #include "hw/pci/pci.h" 23 #include "hw/pci/msi.h" 24 #include "hw/pci/msix.h" 25 #include "sysemu/kvm.h" 26 #include "migration/migration.h" 27 #include "qemu/error-report.h" 28 #include "qemu/event_notifier.h" 29 #include "qemu/fifo8.h" 30 #include "sysemu/char.h" 31 #include "sysemu/hostmem.h" 32 #include "qapi/visitor.h" 33 #include "exec/ram_addr.h" 34 35 #include "hw/misc/ivshmem.h" 36 37 #include <sys/mman.h> 38 39 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET 40 #define PCI_DEVICE_ID_IVSHMEM 0x1110 41 42 #define IVSHMEM_MAX_PEERS G_MAXUINT16 43 #define IVSHMEM_IOEVENTFD 0 44 #define IVSHMEM_MSI 1 45 46 #define IVSHMEM_PEER 0 47 #define IVSHMEM_MASTER 1 48 49 #define IVSHMEM_REG_BAR_SIZE 0x100 50 51 //#define DEBUG_IVSHMEM 52 #ifdef DEBUG_IVSHMEM 53 #define IVSHMEM_DPRINTF(fmt, ...) \ 54 do {printf("IVSHMEM: " fmt, ## __VA_ARGS__); } while (0) 55 #else 56 #define IVSHMEM_DPRINTF(fmt, ...) 57 #endif 58 59 #define TYPE_IVSHMEM "ivshmem" 60 #define IVSHMEM(obj) \ 61 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM) 62 63 typedef struct Peer { 64 int nb_eventfds; 65 EventNotifier *eventfds; 66 } Peer; 67 68 typedef struct MSIVector { 69 PCIDevice *pdev; 70 int virq; 71 } MSIVector; 72 73 typedef struct IVShmemState { 74 /*< private >*/ 75 PCIDevice parent_obj; 76 /*< public >*/ 77 78 HostMemoryBackend *hostmem; 79 uint32_t intrmask; 80 uint32_t intrstatus; 81 82 CharDriverState **eventfd_chr; 83 CharDriverState *server_chr; 84 Fifo8 incoming_fifo; 85 MemoryRegion ivshmem_mmio; 86 87 /* We might need to register the BAR before we actually have the memory. 88 * So prepare a container MemoryRegion for the BAR immediately and 89 * add a subregion when we have the memory. 90 */ 91 MemoryRegion bar; 92 MemoryRegion ivshmem; 93 uint64_t ivshmem_size; /* size of shared memory region */ 94 uint32_t ivshmem_64bit; 95 96 Peer *peers; 97 int nb_peers; /* how many peers we have space for */ 98 99 int vm_id; 100 uint32_t vectors; 101 uint32_t features; 102 MSIVector *msi_vectors; 103 104 Error *migration_blocker; 105 106 char * shmobj; 107 char * sizearg; 108 char * role; 109 int role_val; /* scalar to avoid multiple string comparisons */ 110 } IVShmemState; 111 112 /* registers for the Inter-VM shared memory device */ 113 enum ivshmem_registers { 114 INTRMASK = 0, 115 INTRSTATUS = 4, 116 IVPOSITION = 8, 117 DOORBELL = 12, 118 }; 119 120 static inline uint32_t ivshmem_has_feature(IVShmemState *ivs, 121 unsigned int feature) { 122 return (ivs->features & (1 << feature)); 123 } 124 125 /* accessing registers - based on rtl8139 */ 126 static void ivshmem_update_irq(IVShmemState *s) 127 { 128 PCIDevice *d = PCI_DEVICE(s); 129 int isr; 130 isr = (s->intrstatus & s->intrmask) & 0xffffffff; 131 132 /* don't print ISR resets */ 133 if (isr) { 134 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n", 135 isr ? 1 : 0, s->intrstatus, s->intrmask); 136 } 137 138 pci_set_irq(d, (isr != 0)); 139 } 140 141 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val) 142 { 143 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val); 144 145 s->intrmask = val; 146 147 ivshmem_update_irq(s); 148 } 149 150 static uint32_t ivshmem_IntrMask_read(IVShmemState *s) 151 { 152 uint32_t ret = s->intrmask; 153 154 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret); 155 156 return ret; 157 } 158 159 static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val) 160 { 161 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val); 162 163 s->intrstatus = val; 164 165 ivshmem_update_irq(s); 166 } 167 168 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s) 169 { 170 uint32_t ret = s->intrstatus; 171 172 /* reading ISR clears all interrupts */ 173 s->intrstatus = 0; 174 175 ivshmem_update_irq(s); 176 177 return ret; 178 } 179 180 static void ivshmem_io_write(void *opaque, hwaddr addr, 181 uint64_t val, unsigned size) 182 { 183 IVShmemState *s = opaque; 184 185 uint16_t dest = val >> 16; 186 uint16_t vector = val & 0xff; 187 188 addr &= 0xfc; 189 190 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx "\n", addr); 191 switch (addr) 192 { 193 case INTRMASK: 194 ivshmem_IntrMask_write(s, val); 195 break; 196 197 case INTRSTATUS: 198 ivshmem_IntrStatus_write(s, val); 199 break; 200 201 case DOORBELL: 202 /* check that dest VM ID is reasonable */ 203 if (dest >= s->nb_peers) { 204 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest); 205 break; 206 } 207 208 /* check doorbell range */ 209 if (vector < s->peers[dest].nb_eventfds) { 210 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector); 211 event_notifier_set(&s->peers[dest].eventfds[vector]); 212 } else { 213 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n", 214 vector, dest); 215 } 216 break; 217 default: 218 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr); 219 } 220 } 221 222 static uint64_t ivshmem_io_read(void *opaque, hwaddr addr, 223 unsigned size) 224 { 225 226 IVShmemState *s = opaque; 227 uint32_t ret; 228 229 switch (addr) 230 { 231 case INTRMASK: 232 ret = ivshmem_IntrMask_read(s); 233 break; 234 235 case INTRSTATUS: 236 ret = ivshmem_IntrStatus_read(s); 237 break; 238 239 case IVPOSITION: 240 /* return my VM ID if the memory is mapped */ 241 if (memory_region_is_mapped(&s->ivshmem)) { 242 ret = s->vm_id; 243 } else { 244 ret = -1; 245 } 246 break; 247 248 default: 249 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx "\n", addr); 250 ret = 0; 251 } 252 253 return ret; 254 } 255 256 static const MemoryRegionOps ivshmem_mmio_ops = { 257 .read = ivshmem_io_read, 258 .write = ivshmem_io_write, 259 .endianness = DEVICE_NATIVE_ENDIAN, 260 .impl = { 261 .min_access_size = 4, 262 .max_access_size = 4, 263 }, 264 }; 265 266 static void ivshmem_receive(void *opaque, const uint8_t *buf, int size) 267 { 268 IVShmemState *s = opaque; 269 270 IVSHMEM_DPRINTF("ivshmem_receive 0x%02x size: %d\n", *buf, size); 271 272 ivshmem_IntrStatus_write(s, *buf); 273 } 274 275 static int ivshmem_can_receive(void * opaque) 276 { 277 return sizeof(int64_t); 278 } 279 280 static void ivshmem_event(void *opaque, int event) 281 { 282 IVSHMEM_DPRINTF("ivshmem_event %d\n", event); 283 } 284 285 static void fake_irqfd(void *opaque, const uint8_t *buf, int size) { 286 287 MSIVector *entry = opaque; 288 PCIDevice *pdev = entry->pdev; 289 IVShmemState *s = IVSHMEM(pdev); 290 int vector = entry - s->msi_vectors; 291 292 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector); 293 msix_notify(pdev, vector); 294 } 295 296 static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector, 297 MSIMessage msg) 298 { 299 IVShmemState *s = IVSHMEM(dev); 300 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 301 MSIVector *v = &s->msi_vectors[vector]; 302 int ret; 303 304 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector); 305 306 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev); 307 if (ret < 0) { 308 return ret; 309 } 310 311 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq); 312 } 313 314 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector) 315 { 316 IVShmemState *s = IVSHMEM(dev); 317 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 318 int ret; 319 320 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector); 321 322 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, 323 s->msi_vectors[vector].virq); 324 if (ret != 0) { 325 error_report("remove_irqfd_notifier_gsi failed"); 326 } 327 } 328 329 static void ivshmem_vector_poll(PCIDevice *dev, 330 unsigned int vector_start, 331 unsigned int vector_end) 332 { 333 IVShmemState *s = IVSHMEM(dev); 334 unsigned int vector; 335 336 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end); 337 338 vector_end = MIN(vector_end, s->vectors); 339 340 for (vector = vector_start; vector < vector_end; vector++) { 341 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector]; 342 343 if (!msix_is_masked(dev, vector)) { 344 continue; 345 } 346 347 if (event_notifier_test_and_clear(notifier)) { 348 msix_set_pending(dev, vector); 349 } 350 } 351 } 352 353 static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *n, 354 int vector) 355 { 356 /* create a event character device based on the passed eventfd */ 357 IVShmemState *s = opaque; 358 PCIDevice *pdev = PCI_DEVICE(s); 359 int eventfd = event_notifier_get_fd(n); 360 CharDriverState *chr; 361 362 s->msi_vectors[vector].pdev = pdev; 363 364 chr = qemu_chr_open_eventfd(eventfd); 365 366 if (chr == NULL) { 367 error_report("creating chardriver for eventfd %d failed", eventfd); 368 return NULL; 369 } 370 qemu_chr_fe_claim_no_fail(chr); 371 372 /* if MSI is supported we need multiple interrupts */ 373 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 374 s->msi_vectors[vector].pdev = PCI_DEVICE(s); 375 376 qemu_chr_add_handlers(chr, ivshmem_can_receive, fake_irqfd, 377 ivshmem_event, &s->msi_vectors[vector]); 378 } else { 379 qemu_chr_add_handlers(chr, ivshmem_can_receive, ivshmem_receive, 380 ivshmem_event, s); 381 } 382 383 return chr; 384 385 } 386 387 static int check_shm_size(IVShmemState *s, int fd, Error **errp) 388 { 389 /* check that the guest isn't going to try and map more memory than the 390 * the object has allocated return -1 to indicate error */ 391 392 struct stat buf; 393 394 if (fstat(fd, &buf) < 0) { 395 error_setg(errp, "exiting: fstat on fd %d failed: %s", 396 fd, strerror(errno)); 397 return -1; 398 } 399 400 if (s->ivshmem_size > buf.st_size) { 401 error_setg(errp, "Requested memory size greater" 402 " than shared object size (%" PRIu64 " > %" PRIu64")", 403 s->ivshmem_size, (uint64_t)buf.st_size); 404 return -1; 405 } else { 406 return 0; 407 } 408 } 409 410 /* create the shared memory BAR when we are not using the server, so we can 411 * create the BAR and map the memory immediately */ 412 static int create_shared_memory_BAR(IVShmemState *s, int fd, uint8_t attr, 413 Error **errp) 414 { 415 void * ptr; 416 417 ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 418 if (ptr == MAP_FAILED) { 419 error_setg_errno(errp, errno, "Failed to mmap shared memory"); 420 return -1; 421 } 422 423 memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), "ivshmem.bar2", 424 s->ivshmem_size, ptr); 425 qemu_set_ram_fd(s->ivshmem.ram_addr, fd); 426 vmstate_register_ram(&s->ivshmem, DEVICE(s)); 427 memory_region_add_subregion(&s->bar, 0, &s->ivshmem); 428 429 /* region for shared memory */ 430 pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar); 431 432 return 0; 433 } 434 435 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i) 436 { 437 memory_region_add_eventfd(&s->ivshmem_mmio, 438 DOORBELL, 439 4, 440 true, 441 (posn << 16) | i, 442 &s->peers[posn].eventfds[i]); 443 } 444 445 static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i) 446 { 447 memory_region_del_eventfd(&s->ivshmem_mmio, 448 DOORBELL, 449 4, 450 true, 451 (posn << 16) | i, 452 &s->peers[posn].eventfds[i]); 453 } 454 455 static void close_peer_eventfds(IVShmemState *s, int posn) 456 { 457 int i, n; 458 459 if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 460 return; 461 } 462 if (posn < 0 || posn >= s->nb_peers) { 463 error_report("invalid peer %d", posn); 464 return; 465 } 466 467 n = s->peers[posn].nb_eventfds; 468 469 memory_region_transaction_begin(); 470 for (i = 0; i < n; i++) { 471 ivshmem_del_eventfd(s, posn, i); 472 } 473 memory_region_transaction_commit(); 474 for (i = 0; i < n; i++) { 475 event_notifier_cleanup(&s->peers[posn].eventfds[i]); 476 } 477 478 g_free(s->peers[posn].eventfds); 479 s->peers[posn].nb_eventfds = 0; 480 } 481 482 /* this function increase the dynamic storage need to store data about other 483 * peers */ 484 static int resize_peers(IVShmemState *s, int new_min_size) 485 { 486 487 int j, old_size; 488 489 /* limit number of max peers */ 490 if (new_min_size <= 0 || new_min_size > IVSHMEM_MAX_PEERS) { 491 return -1; 492 } 493 if (new_min_size <= s->nb_peers) { 494 return 0; 495 } 496 497 old_size = s->nb_peers; 498 s->nb_peers = new_min_size; 499 500 IVSHMEM_DPRINTF("bumping storage to %d peers\n", s->nb_peers); 501 502 s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer)); 503 504 for (j = old_size; j < s->nb_peers; j++) { 505 s->peers[j].eventfds = g_new0(EventNotifier, s->vectors); 506 s->peers[j].nb_eventfds = 0; 507 } 508 509 return 0; 510 } 511 512 static bool fifo_update_and_get(IVShmemState *s, const uint8_t *buf, int size, 513 void *data, size_t len) 514 { 515 const uint8_t *p; 516 uint32_t num; 517 518 assert(len <= sizeof(int64_t)); /* limitation of the fifo */ 519 if (fifo8_is_empty(&s->incoming_fifo) && size == len) { 520 memcpy(data, buf, size); 521 return true; 522 } 523 524 IVSHMEM_DPRINTF("short read of %d bytes\n", size); 525 526 num = MIN(size, sizeof(int64_t) - fifo8_num_used(&s->incoming_fifo)); 527 fifo8_push_all(&s->incoming_fifo, buf, num); 528 529 if (fifo8_num_used(&s->incoming_fifo) < len) { 530 assert(num == 0); 531 return false; 532 } 533 534 size -= num; 535 buf += num; 536 p = fifo8_pop_buf(&s->incoming_fifo, len, &num); 537 assert(num == len); 538 539 memcpy(data, p, len); 540 541 if (size > 0) { 542 fifo8_push_all(&s->incoming_fifo, buf, size); 543 } 544 545 return true; 546 } 547 548 static bool fifo_update_and_get_i64(IVShmemState *s, 549 const uint8_t *buf, int size, int64_t *i64) 550 { 551 if (fifo_update_and_get(s, buf, size, i64, sizeof(*i64))) { 552 *i64 = GINT64_FROM_LE(*i64); 553 return true; 554 } 555 556 return false; 557 } 558 559 static int ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector) 560 { 561 PCIDevice *pdev = PCI_DEVICE(s); 562 MSIMessage msg = msix_get_message(pdev, vector); 563 int ret; 564 565 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector); 566 567 if (s->msi_vectors[vector].pdev != NULL) { 568 return 0; 569 } 570 571 ret = kvm_irqchip_add_msi_route(kvm_state, msg, pdev); 572 if (ret < 0) { 573 error_report("ivshmem: kvm_irqchip_add_msi_route failed"); 574 return -1; 575 } 576 577 s->msi_vectors[vector].virq = ret; 578 s->msi_vectors[vector].pdev = pdev; 579 580 return 0; 581 } 582 583 static void setup_interrupt(IVShmemState *s, int vector) 584 { 585 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 586 bool with_irqfd = kvm_msi_via_irqfd_enabled() && 587 ivshmem_has_feature(s, IVSHMEM_MSI); 588 PCIDevice *pdev = PCI_DEVICE(s); 589 590 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector); 591 592 if (!with_irqfd) { 593 IVSHMEM_DPRINTF("with eventfd"); 594 s->eventfd_chr[vector] = create_eventfd_chr_device(s, n, vector); 595 } else if (msix_enabled(pdev)) { 596 IVSHMEM_DPRINTF("with irqfd"); 597 if (ivshmem_add_kvm_msi_virq(s, vector) < 0) { 598 return; 599 } 600 601 if (!msix_is_masked(pdev, vector)) { 602 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, 603 s->msi_vectors[vector].virq); 604 } 605 } else { 606 /* it will be delayed until msix is enabled, in write_config */ 607 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled"); 608 } 609 } 610 611 static void ivshmem_read(void *opaque, const uint8_t *buf, int size) 612 { 613 IVShmemState *s = opaque; 614 int incoming_fd; 615 int new_eventfd; 616 int64_t incoming_posn; 617 Error *err = NULL; 618 Peer *peer; 619 620 if (!fifo_update_and_get_i64(s, buf, size, &incoming_posn)) { 621 return; 622 } 623 624 if (incoming_posn < -1) { 625 IVSHMEM_DPRINTF("invalid incoming_posn %" PRId64 "\n", incoming_posn); 626 return; 627 } 628 629 /* pick off s->server_chr->msgfd and store it, posn should accompany msg */ 630 incoming_fd = qemu_chr_fe_get_msgfd(s->server_chr); 631 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", 632 incoming_posn, incoming_fd); 633 634 /* make sure we have enough space for this peer */ 635 if (incoming_posn >= s->nb_peers) { 636 if (resize_peers(s, incoming_posn + 1) < 0) { 637 error_report("failed to resize peers array"); 638 if (incoming_fd != -1) { 639 close(incoming_fd); 640 } 641 return; 642 } 643 } 644 645 peer = &s->peers[incoming_posn]; 646 647 if (incoming_fd == -1) { 648 /* if posn is positive and unseen before then this is our posn*/ 649 if (incoming_posn >= 0 && s->vm_id == -1) { 650 /* receive our posn */ 651 s->vm_id = incoming_posn; 652 } else { 653 /* otherwise an fd == -1 means an existing peer has gone away */ 654 IVSHMEM_DPRINTF("posn %" PRId64 " has gone away\n", incoming_posn); 655 close_peer_eventfds(s, incoming_posn); 656 } 657 return; 658 } 659 660 /* if the position is -1, then it's shared memory region fd */ 661 if (incoming_posn == -1) { 662 void * map_ptr; 663 664 if (memory_region_is_mapped(&s->ivshmem)) { 665 error_report("shm already initialized"); 666 close(incoming_fd); 667 return; 668 } 669 670 if (check_shm_size(s, incoming_fd, &err) == -1) { 671 error_report_err(err); 672 close(incoming_fd); 673 return; 674 } 675 676 /* mmap the region and map into the BAR2 */ 677 map_ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, 678 incoming_fd, 0); 679 if (map_ptr == MAP_FAILED) { 680 error_report("Failed to mmap shared memory %s", strerror(errno)); 681 close(incoming_fd); 682 return; 683 } 684 memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), 685 "ivshmem.bar2", s->ivshmem_size, map_ptr); 686 qemu_set_ram_fd(s->ivshmem.ram_addr, incoming_fd); 687 vmstate_register_ram(&s->ivshmem, DEVICE(s)); 688 689 IVSHMEM_DPRINTF("guest h/w addr = %p, size = %" PRIu64 "\n", 690 map_ptr, s->ivshmem_size); 691 692 memory_region_add_subregion(&s->bar, 0, &s->ivshmem); 693 694 return; 695 } 696 697 /* each peer has an associated array of eventfds, and we keep 698 * track of how many eventfds received so far */ 699 /* get a new eventfd: */ 700 if (peer->nb_eventfds >= s->vectors) { 701 error_report("Too many eventfd received, device has %d vectors", 702 s->vectors); 703 close(incoming_fd); 704 return; 705 } 706 707 new_eventfd = peer->nb_eventfds++; 708 709 /* this is an eventfd for a particular peer VM */ 710 IVSHMEM_DPRINTF("eventfds[%" PRId64 "][%d] = %d\n", incoming_posn, 711 new_eventfd, incoming_fd); 712 event_notifier_init_fd(&peer->eventfds[new_eventfd], incoming_fd); 713 fcntl_setfl(incoming_fd, O_NONBLOCK); /* msix/irqfd poll non block */ 714 715 if (incoming_posn == s->vm_id) { 716 setup_interrupt(s, new_eventfd); 717 } 718 719 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 720 ivshmem_add_eventfd(s, incoming_posn, new_eventfd); 721 } 722 } 723 724 static void ivshmem_check_version(void *opaque, const uint8_t * buf, int size) 725 { 726 IVShmemState *s = opaque; 727 int tmp; 728 int64_t version; 729 730 if (!fifo_update_and_get_i64(s, buf, size, &version)) { 731 return; 732 } 733 734 tmp = qemu_chr_fe_get_msgfd(s->server_chr); 735 if (tmp != -1 || version != IVSHMEM_PROTOCOL_VERSION) { 736 fprintf(stderr, "incompatible version, you are connecting to a ivshmem-" 737 "server using a different protocol please check your setup\n"); 738 qemu_chr_delete(s->server_chr); 739 s->server_chr = NULL; 740 return; 741 } 742 743 IVSHMEM_DPRINTF("version check ok, switch to real chardev handler\n"); 744 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read, 745 ivshmem_event, s); 746 } 747 748 /* Select the MSI-X vectors used by device. 749 * ivshmem maps events to vectors statically, so 750 * we just enable all vectors on init and after reset. */ 751 static void ivshmem_use_msix(IVShmemState * s) 752 { 753 PCIDevice *d = PCI_DEVICE(s); 754 int i; 755 756 IVSHMEM_DPRINTF("%s, msix present: %d\n", __func__, msix_present(d)); 757 if (!msix_present(d)) { 758 return; 759 } 760 761 for (i = 0; i < s->vectors; i++) { 762 msix_vector_use(d, i); 763 } 764 } 765 766 static void ivshmem_reset(DeviceState *d) 767 { 768 IVShmemState *s = IVSHMEM(d); 769 770 s->intrstatus = 0; 771 s->intrmask = 0; 772 ivshmem_use_msix(s); 773 } 774 775 static int ivshmem_setup_msi(IVShmemState * s) 776 { 777 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) { 778 return -1; 779 } 780 781 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors); 782 783 /* allocate QEMU char devices for receiving interrupts */ 784 s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector)); 785 786 ivshmem_use_msix(s); 787 return 0; 788 } 789 790 static void ivshmem_enable_irqfd(IVShmemState *s) 791 { 792 PCIDevice *pdev = PCI_DEVICE(s); 793 int i; 794 795 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 796 ivshmem_add_kvm_msi_virq(s, i); 797 } 798 799 if (msix_set_vector_notifiers(pdev, 800 ivshmem_vector_unmask, 801 ivshmem_vector_mask, 802 ivshmem_vector_poll)) { 803 error_report("ivshmem: msix_set_vector_notifiers failed"); 804 } 805 } 806 807 static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector) 808 { 809 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector); 810 811 if (s->msi_vectors[vector].pdev == NULL) { 812 return; 813 } 814 815 /* it was cleaned when masked in the frontend. */ 816 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq); 817 818 s->msi_vectors[vector].pdev = NULL; 819 } 820 821 static void ivshmem_disable_irqfd(IVShmemState *s) 822 { 823 PCIDevice *pdev = PCI_DEVICE(s); 824 int i; 825 826 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 827 ivshmem_remove_kvm_msi_virq(s, i); 828 } 829 830 msix_unset_vector_notifiers(pdev); 831 } 832 833 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address, 834 uint32_t val, int len) 835 { 836 IVShmemState *s = IVSHMEM(pdev); 837 int is_enabled, was_enabled = msix_enabled(pdev); 838 839 pci_default_write_config(pdev, address, val, len); 840 is_enabled = msix_enabled(pdev); 841 842 if (kvm_msi_via_irqfd_enabled() && s->vm_id != -1) { 843 if (!was_enabled && is_enabled) { 844 ivshmem_enable_irqfd(s); 845 } else if (was_enabled && !is_enabled) { 846 ivshmem_disable_irqfd(s); 847 } 848 } 849 } 850 851 static void pci_ivshmem_realize(PCIDevice *dev, Error **errp) 852 { 853 IVShmemState *s = IVSHMEM(dev); 854 uint8_t *pci_conf; 855 uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY | 856 PCI_BASE_ADDRESS_MEM_PREFETCH; 857 858 if (!!s->server_chr + !!s->shmobj + !!s->hostmem != 1) { 859 error_setg(errp, 860 "You must specify either 'shm', 'chardev' or 'x-memdev'"); 861 return; 862 } 863 864 if (s->hostmem) { 865 MemoryRegion *mr; 866 867 if (s->sizearg) { 868 g_warning("size argument ignored with hostmem"); 869 } 870 871 mr = host_memory_backend_get_memory(s->hostmem, errp); 872 s->ivshmem_size = memory_region_size(mr); 873 } else if (s->sizearg == NULL) { 874 s->ivshmem_size = 4 << 20; /* 4 MB default */ 875 } else { 876 char *end; 877 int64_t size = qemu_strtosz(s->sizearg, &end); 878 if (size < 0 || *end != '\0' || !is_power_of_2(size)) { 879 error_setg(errp, "Invalid size %s", s->sizearg); 880 return; 881 } 882 s->ivshmem_size = size; 883 } 884 885 fifo8_create(&s->incoming_fifo, sizeof(int64_t)); 886 887 /* IRQFD requires MSI */ 888 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) && 889 !ivshmem_has_feature(s, IVSHMEM_MSI)) { 890 error_setg(errp, "ioeventfd/irqfd requires MSI"); 891 return; 892 } 893 894 /* check that role is reasonable */ 895 if (s->role) { 896 if (strncmp(s->role, "peer", 5) == 0) { 897 s->role_val = IVSHMEM_PEER; 898 } else if (strncmp(s->role, "master", 7) == 0) { 899 s->role_val = IVSHMEM_MASTER; 900 } else { 901 error_setg(errp, "'role' must be 'peer' or 'master'"); 902 return; 903 } 904 } else { 905 s->role_val = IVSHMEM_MASTER; /* default */ 906 } 907 908 if (s->role_val == IVSHMEM_PEER) { 909 error_setg(&s->migration_blocker, 910 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'"); 911 migrate_add_blocker(s->migration_blocker); 912 } 913 914 pci_conf = dev->config; 915 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY; 916 917 pci_config_set_interrupt_pin(pci_conf, 1); 918 919 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s, 920 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE); 921 922 /* region for registers*/ 923 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, 924 &s->ivshmem_mmio); 925 926 memory_region_init(&s->bar, OBJECT(s), "ivshmem-bar2-container", s->ivshmem_size); 927 if (s->ivshmem_64bit) { 928 attr |= PCI_BASE_ADDRESS_MEM_TYPE_64; 929 } 930 931 if (s->hostmem != NULL) { 932 MemoryRegion *mr; 933 934 IVSHMEM_DPRINTF("using hostmem\n"); 935 936 mr = host_memory_backend_get_memory(MEMORY_BACKEND(s->hostmem), errp); 937 vmstate_register_ram(mr, DEVICE(s)); 938 memory_region_add_subregion(&s->bar, 0, mr); 939 pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar); 940 } else if (s->server_chr != NULL) { 941 /* FIXME do not rely on what chr drivers put into filename */ 942 if (strncmp(s->server_chr->filename, "unix:", 5)) { 943 error_setg(errp, "chardev is not a unix client socket"); 944 return; 945 } 946 947 /* if we get a UNIX socket as the parameter we will talk 948 * to the ivshmem server to receive the memory region */ 949 950 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n", 951 s->server_chr->filename); 952 953 if (ivshmem_has_feature(s, IVSHMEM_MSI) && 954 ivshmem_setup_msi(s)) { 955 error_setg(errp, "msix initialization failed"); 956 return; 957 } 958 959 /* we allocate enough space for 16 peers and grow as needed */ 960 resize_peers(s, 16); 961 s->vm_id = -1; 962 963 pci_register_bar(dev, 2, attr, &s->bar); 964 965 s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *)); 966 967 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, 968 ivshmem_check_version, ivshmem_event, s); 969 } else { 970 /* just map the file immediately, we're not using a server */ 971 int fd; 972 973 IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s->shmobj); 974 975 /* try opening with O_EXCL and if it succeeds zero the memory 976 * by truncating to 0 */ 977 if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR|O_EXCL, 978 S_IRWXU|S_IRWXG|S_IRWXO)) > 0) { 979 /* truncate file to length PCI device's memory */ 980 if (ftruncate(fd, s->ivshmem_size) != 0) { 981 error_report("could not truncate shared file"); 982 } 983 984 } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR, 985 S_IRWXU|S_IRWXG|S_IRWXO)) < 0) { 986 error_setg(errp, "could not open shared file"); 987 return; 988 } 989 990 if (check_shm_size(s, fd, errp) == -1) { 991 return; 992 } 993 994 create_shared_memory_BAR(s, fd, attr, errp); 995 } 996 } 997 998 static void pci_ivshmem_exit(PCIDevice *dev) 999 { 1000 IVShmemState *s = IVSHMEM(dev); 1001 int i; 1002 1003 fifo8_destroy(&s->incoming_fifo); 1004 1005 if (s->migration_blocker) { 1006 migrate_del_blocker(s->migration_blocker); 1007 error_free(s->migration_blocker); 1008 } 1009 1010 if (memory_region_is_mapped(&s->ivshmem)) { 1011 if (!s->hostmem) { 1012 void *addr = memory_region_get_ram_ptr(&s->ivshmem); 1013 int fd; 1014 1015 if (munmap(addr, s->ivshmem_size) == -1) { 1016 error_report("Failed to munmap shared memory %s", 1017 strerror(errno)); 1018 } 1019 1020 if ((fd = qemu_get_ram_fd(s->ivshmem.ram_addr)) != -1) 1021 close(fd); 1022 } 1023 1024 vmstate_unregister_ram(&s->ivshmem, DEVICE(dev)); 1025 memory_region_del_subregion(&s->bar, &s->ivshmem); 1026 } 1027 1028 if (s->eventfd_chr) { 1029 for (i = 0; i < s->vectors; i++) { 1030 if (s->eventfd_chr[i]) { 1031 qemu_chr_free(s->eventfd_chr[i]); 1032 } 1033 } 1034 g_free(s->eventfd_chr); 1035 } 1036 1037 if (s->peers) { 1038 for (i = 0; i < s->nb_peers; i++) { 1039 close_peer_eventfds(s, i); 1040 } 1041 g_free(s->peers); 1042 } 1043 1044 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1045 msix_uninit_exclusive_bar(dev); 1046 } 1047 1048 g_free(s->msi_vectors); 1049 } 1050 1051 static bool test_msix(void *opaque, int version_id) 1052 { 1053 IVShmemState *s = opaque; 1054 1055 return ivshmem_has_feature(s, IVSHMEM_MSI); 1056 } 1057 1058 static bool test_no_msix(void *opaque, int version_id) 1059 { 1060 return !test_msix(opaque, version_id); 1061 } 1062 1063 static int ivshmem_pre_load(void *opaque) 1064 { 1065 IVShmemState *s = opaque; 1066 1067 if (s->role_val == IVSHMEM_PEER) { 1068 error_report("'peer' devices are not migratable"); 1069 return -EINVAL; 1070 } 1071 1072 return 0; 1073 } 1074 1075 static int ivshmem_post_load(void *opaque, int version_id) 1076 { 1077 IVShmemState *s = opaque; 1078 1079 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1080 ivshmem_use_msix(s); 1081 } 1082 1083 return 0; 1084 } 1085 1086 static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id) 1087 { 1088 IVShmemState *s = opaque; 1089 PCIDevice *pdev = PCI_DEVICE(s); 1090 int ret; 1091 1092 IVSHMEM_DPRINTF("ivshmem_load_old\n"); 1093 1094 if (version_id != 0) { 1095 return -EINVAL; 1096 } 1097 1098 if (s->role_val == IVSHMEM_PEER) { 1099 error_report("'peer' devices are not migratable"); 1100 return -EINVAL; 1101 } 1102 1103 ret = pci_device_load(pdev, f); 1104 if (ret) { 1105 return ret; 1106 } 1107 1108 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1109 msix_load(pdev, f); 1110 ivshmem_use_msix(s); 1111 } else { 1112 s->intrstatus = qemu_get_be32(f); 1113 s->intrmask = qemu_get_be32(f); 1114 } 1115 1116 return 0; 1117 } 1118 1119 static const VMStateDescription ivshmem_vmsd = { 1120 .name = "ivshmem", 1121 .version_id = 1, 1122 .minimum_version_id = 1, 1123 .pre_load = ivshmem_pre_load, 1124 .post_load = ivshmem_post_load, 1125 .fields = (VMStateField[]) { 1126 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1127 1128 VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix), 1129 VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix), 1130 VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix), 1131 1132 VMSTATE_END_OF_LIST() 1133 }, 1134 .load_state_old = ivshmem_load_old, 1135 .minimum_version_id_old = 0 1136 }; 1137 1138 static Property ivshmem_properties[] = { 1139 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1140 DEFINE_PROP_STRING("size", IVShmemState, sizearg), 1141 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1142 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, false), 1143 DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true), 1144 DEFINE_PROP_STRING("shm", IVShmemState, shmobj), 1145 DEFINE_PROP_STRING("role", IVShmemState, role), 1146 DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1), 1147 DEFINE_PROP_END_OF_LIST(), 1148 }; 1149 1150 static void ivshmem_class_init(ObjectClass *klass, void *data) 1151 { 1152 DeviceClass *dc = DEVICE_CLASS(klass); 1153 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1154 1155 k->realize = pci_ivshmem_realize; 1156 k->exit = pci_ivshmem_exit; 1157 k->config_write = ivshmem_write_config; 1158 k->vendor_id = PCI_VENDOR_ID_IVSHMEM; 1159 k->device_id = PCI_DEVICE_ID_IVSHMEM; 1160 k->class_id = PCI_CLASS_MEMORY_RAM; 1161 dc->reset = ivshmem_reset; 1162 dc->props = ivshmem_properties; 1163 dc->vmsd = &ivshmem_vmsd; 1164 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1165 dc->desc = "Inter-VM shared memory"; 1166 } 1167 1168 static void ivshmem_check_memdev_is_busy(Object *obj, const char *name, 1169 Object *val, Error **errp) 1170 { 1171 MemoryRegion *mr; 1172 1173 mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), errp); 1174 if (memory_region_is_mapped(mr)) { 1175 char *path = object_get_canonical_path_component(val); 1176 error_setg(errp, "can't use already busy memdev: %s", path); 1177 g_free(path); 1178 } else { 1179 qdev_prop_allow_set_link_before_realize(obj, name, val, errp); 1180 } 1181 } 1182 1183 static void ivshmem_init(Object *obj) 1184 { 1185 IVShmemState *s = IVSHMEM(obj); 1186 1187 object_property_add_link(obj, "x-memdev", TYPE_MEMORY_BACKEND, 1188 (Object **)&s->hostmem, 1189 ivshmem_check_memdev_is_busy, 1190 OBJ_PROP_LINK_UNREF_ON_RELEASE, 1191 &error_abort); 1192 } 1193 1194 static const TypeInfo ivshmem_info = { 1195 .name = TYPE_IVSHMEM, 1196 .parent = TYPE_PCI_DEVICE, 1197 .instance_size = sizeof(IVShmemState), 1198 .instance_init = ivshmem_init, 1199 .class_init = ivshmem_class_init, 1200 }; 1201 1202 static void ivshmem_register_types(void) 1203 { 1204 type_register_static(&ivshmem_info); 1205 } 1206 1207 type_init(ivshmem_register_types) 1208