1 /* 2 * Inter-VM Shared Memory PCI device. 3 * 4 * Author: 5 * Cam Macdonell <cam@cs.ualberta.ca> 6 * 7 * Based On: cirrus_vga.c 8 * Copyright (c) 2004 Fabrice Bellard 9 * Copyright (c) 2004 Makoto Suzuki (suzu) 10 * 11 * and rtl8139.c 12 * Copyright (c) 2006 Igor Kovalenko 13 * 14 * This code is licensed under the GNU GPL v2. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 */ 19 #include "qemu/osdep.h" 20 #include "qapi/error.h" 21 #include "hw/hw.h" 22 #include "hw/i386/pc.h" 23 #include "hw/pci/pci.h" 24 #include "hw/pci/msi.h" 25 #include "hw/pci/msix.h" 26 #include "sysemu/kvm.h" 27 #include "migration/migration.h" 28 #include "qemu/error-report.h" 29 #include "qemu/event_notifier.h" 30 #include "qemu/fifo8.h" 31 #include "sysemu/char.h" 32 #include "sysemu/hostmem.h" 33 #include "qapi/visitor.h" 34 #include "exec/ram_addr.h" 35 36 #include "hw/misc/ivshmem.h" 37 38 #include <sys/mman.h> 39 40 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET 41 #define PCI_DEVICE_ID_IVSHMEM 0x1110 42 43 #define IVSHMEM_MAX_PEERS G_MAXUINT16 44 #define IVSHMEM_IOEVENTFD 0 45 #define IVSHMEM_MSI 1 46 47 #define IVSHMEM_PEER 0 48 #define IVSHMEM_MASTER 1 49 50 #define IVSHMEM_REG_BAR_SIZE 0x100 51 52 //#define DEBUG_IVSHMEM 53 #ifdef DEBUG_IVSHMEM 54 #define IVSHMEM_DPRINTF(fmt, ...) \ 55 do {printf("IVSHMEM: " fmt, ## __VA_ARGS__); } while (0) 56 #else 57 #define IVSHMEM_DPRINTF(fmt, ...) 58 #endif 59 60 #define TYPE_IVSHMEM "ivshmem" 61 #define IVSHMEM(obj) \ 62 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM) 63 64 typedef struct Peer { 65 int nb_eventfds; 66 EventNotifier *eventfds; 67 } Peer; 68 69 typedef struct MSIVector { 70 PCIDevice *pdev; 71 int virq; 72 } MSIVector; 73 74 typedef struct IVShmemState { 75 /*< private >*/ 76 PCIDevice parent_obj; 77 /*< public >*/ 78 79 HostMemoryBackend *hostmem; 80 uint32_t intrmask; 81 uint32_t intrstatus; 82 83 CharDriverState **eventfd_chr; 84 CharDriverState *server_chr; 85 Fifo8 incoming_fifo; 86 MemoryRegion ivshmem_mmio; 87 88 /* We might need to register the BAR before we actually have the memory. 89 * So prepare a container MemoryRegion for the BAR immediately and 90 * add a subregion when we have the memory. 91 */ 92 MemoryRegion bar; 93 MemoryRegion ivshmem; 94 uint64_t ivshmem_size; /* size of shared memory region */ 95 uint32_t ivshmem_64bit; 96 97 Peer *peers; 98 int nb_peers; /* how many peers we have space for */ 99 100 int vm_id; 101 uint32_t vectors; 102 uint32_t features; 103 MSIVector *msi_vectors; 104 105 Error *migration_blocker; 106 107 char * shmobj; 108 char * sizearg; 109 char * role; 110 int role_val; /* scalar to avoid multiple string comparisons */ 111 } IVShmemState; 112 113 /* registers for the Inter-VM shared memory device */ 114 enum ivshmem_registers { 115 INTRMASK = 0, 116 INTRSTATUS = 4, 117 IVPOSITION = 8, 118 DOORBELL = 12, 119 }; 120 121 static inline uint32_t ivshmem_has_feature(IVShmemState *ivs, 122 unsigned int feature) { 123 return (ivs->features & (1 << feature)); 124 } 125 126 /* accessing registers - based on rtl8139 */ 127 static void ivshmem_update_irq(IVShmemState *s) 128 { 129 PCIDevice *d = PCI_DEVICE(s); 130 int isr; 131 isr = (s->intrstatus & s->intrmask) & 0xffffffff; 132 133 /* don't print ISR resets */ 134 if (isr) { 135 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n", 136 isr ? 1 : 0, s->intrstatus, s->intrmask); 137 } 138 139 pci_set_irq(d, (isr != 0)); 140 } 141 142 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val) 143 { 144 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val); 145 146 s->intrmask = val; 147 148 ivshmem_update_irq(s); 149 } 150 151 static uint32_t ivshmem_IntrMask_read(IVShmemState *s) 152 { 153 uint32_t ret = s->intrmask; 154 155 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret); 156 157 return ret; 158 } 159 160 static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val) 161 { 162 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val); 163 164 s->intrstatus = val; 165 166 ivshmem_update_irq(s); 167 } 168 169 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s) 170 { 171 uint32_t ret = s->intrstatus; 172 173 /* reading ISR clears all interrupts */ 174 s->intrstatus = 0; 175 176 ivshmem_update_irq(s); 177 178 return ret; 179 } 180 181 static void ivshmem_io_write(void *opaque, hwaddr addr, 182 uint64_t val, unsigned size) 183 { 184 IVShmemState *s = opaque; 185 186 uint16_t dest = val >> 16; 187 uint16_t vector = val & 0xff; 188 189 addr &= 0xfc; 190 191 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx "\n", addr); 192 switch (addr) 193 { 194 case INTRMASK: 195 ivshmem_IntrMask_write(s, val); 196 break; 197 198 case INTRSTATUS: 199 ivshmem_IntrStatus_write(s, val); 200 break; 201 202 case DOORBELL: 203 /* check that dest VM ID is reasonable */ 204 if (dest >= s->nb_peers) { 205 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest); 206 break; 207 } 208 209 /* check doorbell range */ 210 if (vector < s->peers[dest].nb_eventfds) { 211 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector); 212 event_notifier_set(&s->peers[dest].eventfds[vector]); 213 } else { 214 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n", 215 vector, dest); 216 } 217 break; 218 default: 219 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr); 220 } 221 } 222 223 static uint64_t ivshmem_io_read(void *opaque, hwaddr addr, 224 unsigned size) 225 { 226 227 IVShmemState *s = opaque; 228 uint32_t ret; 229 230 switch (addr) 231 { 232 case INTRMASK: 233 ret = ivshmem_IntrMask_read(s); 234 break; 235 236 case INTRSTATUS: 237 ret = ivshmem_IntrStatus_read(s); 238 break; 239 240 case IVPOSITION: 241 /* return my VM ID if the memory is mapped */ 242 if (memory_region_is_mapped(&s->ivshmem)) { 243 ret = s->vm_id; 244 } else { 245 ret = -1; 246 } 247 break; 248 249 default: 250 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx "\n", addr); 251 ret = 0; 252 } 253 254 return ret; 255 } 256 257 static const MemoryRegionOps ivshmem_mmio_ops = { 258 .read = ivshmem_io_read, 259 .write = ivshmem_io_write, 260 .endianness = DEVICE_NATIVE_ENDIAN, 261 .impl = { 262 .min_access_size = 4, 263 .max_access_size = 4, 264 }, 265 }; 266 267 static int ivshmem_can_receive(void * opaque) 268 { 269 return sizeof(int64_t); 270 } 271 272 static void ivshmem_event(void *opaque, int event) 273 { 274 IVSHMEM_DPRINTF("ivshmem_event %d\n", event); 275 } 276 277 static void ivshmem_vector_notify(void *opaque) 278 { 279 MSIVector *entry = opaque; 280 PCIDevice *pdev = entry->pdev; 281 IVShmemState *s = IVSHMEM(pdev); 282 int vector = entry - s->msi_vectors; 283 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 284 285 if (!event_notifier_test_and_clear(n)) { 286 return; 287 } 288 289 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector); 290 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 291 msix_notify(pdev, vector); 292 } else { 293 ivshmem_IntrStatus_write(s, 1); 294 } 295 } 296 297 static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector, 298 MSIMessage msg) 299 { 300 IVShmemState *s = IVSHMEM(dev); 301 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 302 MSIVector *v = &s->msi_vectors[vector]; 303 int ret; 304 305 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector); 306 307 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev); 308 if (ret < 0) { 309 return ret; 310 } 311 312 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq); 313 } 314 315 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector) 316 { 317 IVShmemState *s = IVSHMEM(dev); 318 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 319 int ret; 320 321 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector); 322 323 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, 324 s->msi_vectors[vector].virq); 325 if (ret != 0) { 326 error_report("remove_irqfd_notifier_gsi failed"); 327 } 328 } 329 330 static void ivshmem_vector_poll(PCIDevice *dev, 331 unsigned int vector_start, 332 unsigned int vector_end) 333 { 334 IVShmemState *s = IVSHMEM(dev); 335 unsigned int vector; 336 337 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end); 338 339 vector_end = MIN(vector_end, s->vectors); 340 341 for (vector = vector_start; vector < vector_end; vector++) { 342 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector]; 343 344 if (!msix_is_masked(dev, vector)) { 345 continue; 346 } 347 348 if (event_notifier_test_and_clear(notifier)) { 349 msix_set_pending(dev, vector); 350 } 351 } 352 } 353 354 static void watch_vector_notifier(IVShmemState *s, EventNotifier *n, 355 int vector) 356 { 357 int eventfd = event_notifier_get_fd(n); 358 359 /* if MSI is supported we need multiple interrupts */ 360 s->msi_vectors[vector].pdev = PCI_DEVICE(s); 361 362 qemu_set_fd_handler(eventfd, ivshmem_vector_notify, 363 NULL, &s->msi_vectors[vector]); 364 } 365 366 static int check_shm_size(IVShmemState *s, int fd, Error **errp) 367 { 368 /* check that the guest isn't going to try and map more memory than the 369 * the object has allocated return -1 to indicate error */ 370 371 struct stat buf; 372 373 if (fstat(fd, &buf) < 0) { 374 error_setg(errp, "exiting: fstat on fd %d failed: %s", 375 fd, strerror(errno)); 376 return -1; 377 } 378 379 if (s->ivshmem_size > buf.st_size) { 380 error_setg(errp, "Requested memory size greater" 381 " than shared object size (%" PRIu64 " > %" PRIu64")", 382 s->ivshmem_size, (uint64_t)buf.st_size); 383 return -1; 384 } else { 385 return 0; 386 } 387 } 388 389 /* create the shared memory BAR when we are not using the server, so we can 390 * create the BAR and map the memory immediately */ 391 static int create_shared_memory_BAR(IVShmemState *s, int fd, uint8_t attr, 392 Error **errp) 393 { 394 void * ptr; 395 396 ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 397 if (ptr == MAP_FAILED) { 398 error_setg_errno(errp, errno, "Failed to mmap shared memory"); 399 return -1; 400 } 401 402 memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), "ivshmem.bar2", 403 s->ivshmem_size, ptr); 404 qemu_set_ram_fd(memory_region_get_ram_addr(&s->ivshmem), fd); 405 vmstate_register_ram(&s->ivshmem, DEVICE(s)); 406 memory_region_add_subregion(&s->bar, 0, &s->ivshmem); 407 408 /* region for shared memory */ 409 pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar); 410 411 return 0; 412 } 413 414 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i) 415 { 416 memory_region_add_eventfd(&s->ivshmem_mmio, 417 DOORBELL, 418 4, 419 true, 420 (posn << 16) | i, 421 &s->peers[posn].eventfds[i]); 422 } 423 424 static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i) 425 { 426 memory_region_del_eventfd(&s->ivshmem_mmio, 427 DOORBELL, 428 4, 429 true, 430 (posn << 16) | i, 431 &s->peers[posn].eventfds[i]); 432 } 433 434 static void close_peer_eventfds(IVShmemState *s, int posn) 435 { 436 int i, n; 437 438 if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 439 return; 440 } 441 if (posn < 0 || posn >= s->nb_peers) { 442 error_report("invalid peer %d", posn); 443 return; 444 } 445 446 n = s->peers[posn].nb_eventfds; 447 448 memory_region_transaction_begin(); 449 for (i = 0; i < n; i++) { 450 ivshmem_del_eventfd(s, posn, i); 451 } 452 memory_region_transaction_commit(); 453 for (i = 0; i < n; i++) { 454 event_notifier_cleanup(&s->peers[posn].eventfds[i]); 455 } 456 457 g_free(s->peers[posn].eventfds); 458 s->peers[posn].nb_eventfds = 0; 459 } 460 461 /* this function increase the dynamic storage need to store data about other 462 * peers */ 463 static int resize_peers(IVShmemState *s, int new_min_size) 464 { 465 466 int j, old_size; 467 468 /* limit number of max peers */ 469 if (new_min_size <= 0 || new_min_size > IVSHMEM_MAX_PEERS) { 470 return -1; 471 } 472 if (new_min_size <= s->nb_peers) { 473 return 0; 474 } 475 476 old_size = s->nb_peers; 477 s->nb_peers = new_min_size; 478 479 IVSHMEM_DPRINTF("bumping storage to %d peers\n", s->nb_peers); 480 481 s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer)); 482 483 for (j = old_size; j < s->nb_peers; j++) { 484 s->peers[j].eventfds = g_new0(EventNotifier, s->vectors); 485 s->peers[j].nb_eventfds = 0; 486 } 487 488 return 0; 489 } 490 491 static bool fifo_update_and_get(IVShmemState *s, const uint8_t *buf, int size, 492 void *data, size_t len) 493 { 494 const uint8_t *p; 495 uint32_t num; 496 497 assert(len <= sizeof(int64_t)); /* limitation of the fifo */ 498 if (fifo8_is_empty(&s->incoming_fifo) && size == len) { 499 memcpy(data, buf, size); 500 return true; 501 } 502 503 IVSHMEM_DPRINTF("short read of %d bytes\n", size); 504 505 num = MIN(size, sizeof(int64_t) - fifo8_num_used(&s->incoming_fifo)); 506 fifo8_push_all(&s->incoming_fifo, buf, num); 507 508 if (fifo8_num_used(&s->incoming_fifo) < len) { 509 assert(num == 0); 510 return false; 511 } 512 513 size -= num; 514 buf += num; 515 p = fifo8_pop_buf(&s->incoming_fifo, len, &num); 516 assert(num == len); 517 518 memcpy(data, p, len); 519 520 if (size > 0) { 521 fifo8_push_all(&s->incoming_fifo, buf, size); 522 } 523 524 return true; 525 } 526 527 static bool fifo_update_and_get_i64(IVShmemState *s, 528 const uint8_t *buf, int size, int64_t *i64) 529 { 530 if (fifo_update_and_get(s, buf, size, i64, sizeof(*i64))) { 531 *i64 = GINT64_FROM_LE(*i64); 532 return true; 533 } 534 535 return false; 536 } 537 538 static int ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector) 539 { 540 PCIDevice *pdev = PCI_DEVICE(s); 541 MSIMessage msg = msix_get_message(pdev, vector); 542 int ret; 543 544 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector); 545 546 if (s->msi_vectors[vector].pdev != NULL) { 547 return 0; 548 } 549 550 ret = kvm_irqchip_add_msi_route(kvm_state, msg, pdev); 551 if (ret < 0) { 552 error_report("ivshmem: kvm_irqchip_add_msi_route failed"); 553 return -1; 554 } 555 556 s->msi_vectors[vector].virq = ret; 557 s->msi_vectors[vector].pdev = pdev; 558 559 return 0; 560 } 561 562 static void setup_interrupt(IVShmemState *s, int vector) 563 { 564 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector]; 565 bool with_irqfd = kvm_msi_via_irqfd_enabled() && 566 ivshmem_has_feature(s, IVSHMEM_MSI); 567 PCIDevice *pdev = PCI_DEVICE(s); 568 569 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector); 570 571 if (!with_irqfd) { 572 IVSHMEM_DPRINTF("with eventfd"); 573 watch_vector_notifier(s, n, vector); 574 } else if (msix_enabled(pdev)) { 575 IVSHMEM_DPRINTF("with irqfd"); 576 if (ivshmem_add_kvm_msi_virq(s, vector) < 0) { 577 return; 578 } 579 580 if (!msix_is_masked(pdev, vector)) { 581 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, 582 s->msi_vectors[vector].virq); 583 } 584 } else { 585 /* it will be delayed until msix is enabled, in write_config */ 586 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled"); 587 } 588 } 589 590 static void ivshmem_read(void *opaque, const uint8_t *buf, int size) 591 { 592 IVShmemState *s = opaque; 593 int incoming_fd; 594 int new_eventfd; 595 int64_t incoming_posn; 596 Error *err = NULL; 597 Peer *peer; 598 599 if (!fifo_update_and_get_i64(s, buf, size, &incoming_posn)) { 600 return; 601 } 602 603 if (incoming_posn < -1) { 604 IVSHMEM_DPRINTF("invalid incoming_posn %" PRId64 "\n", incoming_posn); 605 return; 606 } 607 608 /* pick off s->server_chr->msgfd and store it, posn should accompany msg */ 609 incoming_fd = qemu_chr_fe_get_msgfd(s->server_chr); 610 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", 611 incoming_posn, incoming_fd); 612 613 /* make sure we have enough space for this peer */ 614 if (incoming_posn >= s->nb_peers) { 615 if (resize_peers(s, incoming_posn + 1) < 0) { 616 error_report("failed to resize peers array"); 617 if (incoming_fd != -1) { 618 close(incoming_fd); 619 } 620 return; 621 } 622 } 623 624 peer = &s->peers[incoming_posn]; 625 626 if (incoming_fd == -1) { 627 /* if posn is positive and unseen before then this is our posn*/ 628 if (incoming_posn >= 0 && s->vm_id == -1) { 629 /* receive our posn */ 630 s->vm_id = incoming_posn; 631 } else { 632 /* otherwise an fd == -1 means an existing peer has gone away */ 633 IVSHMEM_DPRINTF("posn %" PRId64 " has gone away\n", incoming_posn); 634 close_peer_eventfds(s, incoming_posn); 635 } 636 return; 637 } 638 639 /* if the position is -1, then it's shared memory region fd */ 640 if (incoming_posn == -1) { 641 void * map_ptr; 642 643 if (memory_region_is_mapped(&s->ivshmem)) { 644 error_report("shm already initialized"); 645 close(incoming_fd); 646 return; 647 } 648 649 if (check_shm_size(s, incoming_fd, &err) == -1) { 650 error_report_err(err); 651 close(incoming_fd); 652 return; 653 } 654 655 /* mmap the region and map into the BAR2 */ 656 map_ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, 657 incoming_fd, 0); 658 if (map_ptr == MAP_FAILED) { 659 error_report("Failed to mmap shared memory %s", strerror(errno)); 660 close(incoming_fd); 661 return; 662 } 663 memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), 664 "ivshmem.bar2", s->ivshmem_size, map_ptr); 665 qemu_set_ram_fd(memory_region_get_ram_addr(&s->ivshmem), 666 incoming_fd); 667 vmstate_register_ram(&s->ivshmem, DEVICE(s)); 668 669 IVSHMEM_DPRINTF("guest h/w addr = %p, size = %" PRIu64 "\n", 670 map_ptr, s->ivshmem_size); 671 672 memory_region_add_subregion(&s->bar, 0, &s->ivshmem); 673 674 return; 675 } 676 677 /* each peer has an associated array of eventfds, and we keep 678 * track of how many eventfds received so far */ 679 /* get a new eventfd: */ 680 if (peer->nb_eventfds >= s->vectors) { 681 error_report("Too many eventfd received, device has %d vectors", 682 s->vectors); 683 close(incoming_fd); 684 return; 685 } 686 687 new_eventfd = peer->nb_eventfds++; 688 689 /* this is an eventfd for a particular peer VM */ 690 IVSHMEM_DPRINTF("eventfds[%" PRId64 "][%d] = %d\n", incoming_posn, 691 new_eventfd, incoming_fd); 692 event_notifier_init_fd(&peer->eventfds[new_eventfd], incoming_fd); 693 fcntl_setfl(incoming_fd, O_NONBLOCK); /* msix/irqfd poll non block */ 694 695 if (incoming_posn == s->vm_id) { 696 setup_interrupt(s, new_eventfd); 697 } 698 699 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) { 700 ivshmem_add_eventfd(s, incoming_posn, new_eventfd); 701 } 702 } 703 704 static void ivshmem_check_version(void *opaque, const uint8_t * buf, int size) 705 { 706 IVShmemState *s = opaque; 707 int tmp; 708 int64_t version; 709 710 if (!fifo_update_and_get_i64(s, buf, size, &version)) { 711 return; 712 } 713 714 tmp = qemu_chr_fe_get_msgfd(s->server_chr); 715 if (tmp != -1 || version != IVSHMEM_PROTOCOL_VERSION) { 716 fprintf(stderr, "incompatible version, you are connecting to a ivshmem-" 717 "server using a different protocol please check your setup\n"); 718 qemu_chr_delete(s->server_chr); 719 s->server_chr = NULL; 720 return; 721 } 722 723 IVSHMEM_DPRINTF("version check ok, switch to real chardev handler\n"); 724 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read, 725 ivshmem_event, s); 726 } 727 728 /* Select the MSI-X vectors used by device. 729 * ivshmem maps events to vectors statically, so 730 * we just enable all vectors on init and after reset. */ 731 static void ivshmem_use_msix(IVShmemState * s) 732 { 733 PCIDevice *d = PCI_DEVICE(s); 734 int i; 735 736 IVSHMEM_DPRINTF("%s, msix present: %d\n", __func__, msix_present(d)); 737 if (!msix_present(d)) { 738 return; 739 } 740 741 for (i = 0; i < s->vectors; i++) { 742 msix_vector_use(d, i); 743 } 744 } 745 746 static void ivshmem_reset(DeviceState *d) 747 { 748 IVShmemState *s = IVSHMEM(d); 749 750 s->intrstatus = 0; 751 s->intrmask = 0; 752 ivshmem_use_msix(s); 753 } 754 755 static int ivshmem_setup_interrupts(IVShmemState *s) 756 { 757 /* allocate QEMU callback data for receiving interrupts */ 758 s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector)); 759 760 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 761 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) { 762 return -1; 763 } 764 765 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors); 766 ivshmem_use_msix(s); 767 } 768 769 return 0; 770 } 771 772 static void ivshmem_enable_irqfd(IVShmemState *s) 773 { 774 PCIDevice *pdev = PCI_DEVICE(s); 775 int i; 776 777 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 778 ivshmem_add_kvm_msi_virq(s, i); 779 } 780 781 if (msix_set_vector_notifiers(pdev, 782 ivshmem_vector_unmask, 783 ivshmem_vector_mask, 784 ivshmem_vector_poll)) { 785 error_report("ivshmem: msix_set_vector_notifiers failed"); 786 } 787 } 788 789 static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector) 790 { 791 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector); 792 793 if (s->msi_vectors[vector].pdev == NULL) { 794 return; 795 } 796 797 /* it was cleaned when masked in the frontend. */ 798 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq); 799 800 s->msi_vectors[vector].pdev = NULL; 801 } 802 803 static void ivshmem_disable_irqfd(IVShmemState *s) 804 { 805 PCIDevice *pdev = PCI_DEVICE(s); 806 int i; 807 808 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) { 809 ivshmem_remove_kvm_msi_virq(s, i); 810 } 811 812 msix_unset_vector_notifiers(pdev); 813 } 814 815 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address, 816 uint32_t val, int len) 817 { 818 IVShmemState *s = IVSHMEM(pdev); 819 int is_enabled, was_enabled = msix_enabled(pdev); 820 821 pci_default_write_config(pdev, address, val, len); 822 is_enabled = msix_enabled(pdev); 823 824 if (kvm_msi_via_irqfd_enabled() && s->vm_id != -1) { 825 if (!was_enabled && is_enabled) { 826 ivshmem_enable_irqfd(s); 827 } else if (was_enabled && !is_enabled) { 828 ivshmem_disable_irqfd(s); 829 } 830 } 831 } 832 833 static void pci_ivshmem_realize(PCIDevice *dev, Error **errp) 834 { 835 IVShmemState *s = IVSHMEM(dev); 836 uint8_t *pci_conf; 837 uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY | 838 PCI_BASE_ADDRESS_MEM_PREFETCH; 839 840 if (!!s->server_chr + !!s->shmobj + !!s->hostmem != 1) { 841 error_setg(errp, 842 "You must specify either 'shm', 'chardev' or 'x-memdev'"); 843 return; 844 } 845 846 if (s->hostmem) { 847 MemoryRegion *mr; 848 849 if (s->sizearg) { 850 g_warning("size argument ignored with hostmem"); 851 } 852 853 mr = host_memory_backend_get_memory(s->hostmem, errp); 854 s->ivshmem_size = memory_region_size(mr); 855 } else if (s->sizearg == NULL) { 856 s->ivshmem_size = 4 << 20; /* 4 MB default */ 857 } else { 858 char *end; 859 int64_t size = qemu_strtosz(s->sizearg, &end); 860 if (size < 0 || *end != '\0' || !is_power_of_2(size)) { 861 error_setg(errp, "Invalid size %s", s->sizearg); 862 return; 863 } 864 s->ivshmem_size = size; 865 } 866 867 fifo8_create(&s->incoming_fifo, sizeof(int64_t)); 868 869 /* IRQFD requires MSI */ 870 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) && 871 !ivshmem_has_feature(s, IVSHMEM_MSI)) { 872 error_setg(errp, "ioeventfd/irqfd requires MSI"); 873 return; 874 } 875 876 /* check that role is reasonable */ 877 if (s->role) { 878 if (strncmp(s->role, "peer", 5) == 0) { 879 s->role_val = IVSHMEM_PEER; 880 } else if (strncmp(s->role, "master", 7) == 0) { 881 s->role_val = IVSHMEM_MASTER; 882 } else { 883 error_setg(errp, "'role' must be 'peer' or 'master'"); 884 return; 885 } 886 } else { 887 s->role_val = IVSHMEM_MASTER; /* default */ 888 } 889 890 if (s->role_val == IVSHMEM_PEER) { 891 error_setg(&s->migration_blocker, 892 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'"); 893 migrate_add_blocker(s->migration_blocker); 894 } 895 896 pci_conf = dev->config; 897 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY; 898 899 pci_config_set_interrupt_pin(pci_conf, 1); 900 901 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s, 902 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE); 903 904 /* region for registers*/ 905 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, 906 &s->ivshmem_mmio); 907 908 memory_region_init(&s->bar, OBJECT(s), "ivshmem-bar2-container", s->ivshmem_size); 909 if (s->ivshmem_64bit) { 910 attr |= PCI_BASE_ADDRESS_MEM_TYPE_64; 911 } 912 913 if (s->hostmem != NULL) { 914 MemoryRegion *mr; 915 916 IVSHMEM_DPRINTF("using hostmem\n"); 917 918 mr = host_memory_backend_get_memory(MEMORY_BACKEND(s->hostmem), errp); 919 vmstate_register_ram(mr, DEVICE(s)); 920 memory_region_add_subregion(&s->bar, 0, mr); 921 pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar); 922 } else if (s->server_chr != NULL) { 923 /* FIXME do not rely on what chr drivers put into filename */ 924 if (strncmp(s->server_chr->filename, "unix:", 5)) { 925 error_setg(errp, "chardev is not a unix client socket"); 926 return; 927 } 928 929 /* if we get a UNIX socket as the parameter we will talk 930 * to the ivshmem server to receive the memory region */ 931 932 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n", 933 s->server_chr->filename); 934 935 if (ivshmem_setup_interrupts(s) < 0) { 936 error_setg(errp, "failed to initialize interrupts"); 937 return; 938 } 939 940 /* we allocate enough space for 16 peers and grow as needed */ 941 resize_peers(s, 16); 942 s->vm_id = -1; 943 944 pci_register_bar(dev, 2, attr, &s->bar); 945 946 s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *)); 947 948 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, 949 ivshmem_check_version, ivshmem_event, s); 950 } else { 951 /* just map the file immediately, we're not using a server */ 952 int fd; 953 954 IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s->shmobj); 955 956 /* try opening with O_EXCL and if it succeeds zero the memory 957 * by truncating to 0 */ 958 if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR|O_EXCL, 959 S_IRWXU|S_IRWXG|S_IRWXO)) > 0) { 960 /* truncate file to length PCI device's memory */ 961 if (ftruncate(fd, s->ivshmem_size) != 0) { 962 error_report("could not truncate shared file"); 963 } 964 965 } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR, 966 S_IRWXU|S_IRWXG|S_IRWXO)) < 0) { 967 error_setg(errp, "could not open shared file"); 968 return; 969 } 970 971 if (check_shm_size(s, fd, errp) == -1) { 972 return; 973 } 974 975 create_shared_memory_BAR(s, fd, attr, errp); 976 } 977 } 978 979 static void pci_ivshmem_exit(PCIDevice *dev) 980 { 981 IVShmemState *s = IVSHMEM(dev); 982 int i; 983 984 fifo8_destroy(&s->incoming_fifo); 985 986 if (s->migration_blocker) { 987 migrate_del_blocker(s->migration_blocker); 988 error_free(s->migration_blocker); 989 } 990 991 if (memory_region_is_mapped(&s->ivshmem)) { 992 if (!s->hostmem) { 993 void *addr = memory_region_get_ram_ptr(&s->ivshmem); 994 int fd; 995 996 if (munmap(addr, s->ivshmem_size) == -1) { 997 error_report("Failed to munmap shared memory %s", 998 strerror(errno)); 999 } 1000 1001 fd = qemu_get_ram_fd(memory_region_get_ram_addr(&s->ivshmem)); 1002 if (fd != -1) { 1003 close(fd); 1004 } 1005 } 1006 1007 vmstate_unregister_ram(&s->ivshmem, DEVICE(dev)); 1008 memory_region_del_subregion(&s->bar, &s->ivshmem); 1009 } 1010 1011 if (s->eventfd_chr) { 1012 for (i = 0; i < s->vectors; i++) { 1013 if (s->eventfd_chr[i]) { 1014 qemu_chr_free(s->eventfd_chr[i]); 1015 } 1016 } 1017 g_free(s->eventfd_chr); 1018 } 1019 1020 if (s->peers) { 1021 for (i = 0; i < s->nb_peers; i++) { 1022 close_peer_eventfds(s, i); 1023 } 1024 g_free(s->peers); 1025 } 1026 1027 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1028 msix_uninit_exclusive_bar(dev); 1029 } 1030 1031 g_free(s->msi_vectors); 1032 } 1033 1034 static bool test_msix(void *opaque, int version_id) 1035 { 1036 IVShmemState *s = opaque; 1037 1038 return ivshmem_has_feature(s, IVSHMEM_MSI); 1039 } 1040 1041 static bool test_no_msix(void *opaque, int version_id) 1042 { 1043 return !test_msix(opaque, version_id); 1044 } 1045 1046 static int ivshmem_pre_load(void *opaque) 1047 { 1048 IVShmemState *s = opaque; 1049 1050 if (s->role_val == IVSHMEM_PEER) { 1051 error_report("'peer' devices are not migratable"); 1052 return -EINVAL; 1053 } 1054 1055 return 0; 1056 } 1057 1058 static int ivshmem_post_load(void *opaque, int version_id) 1059 { 1060 IVShmemState *s = opaque; 1061 1062 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1063 ivshmem_use_msix(s); 1064 } 1065 1066 return 0; 1067 } 1068 1069 static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id) 1070 { 1071 IVShmemState *s = opaque; 1072 PCIDevice *pdev = PCI_DEVICE(s); 1073 int ret; 1074 1075 IVSHMEM_DPRINTF("ivshmem_load_old\n"); 1076 1077 if (version_id != 0) { 1078 return -EINVAL; 1079 } 1080 1081 if (s->role_val == IVSHMEM_PEER) { 1082 error_report("'peer' devices are not migratable"); 1083 return -EINVAL; 1084 } 1085 1086 ret = pci_device_load(pdev, f); 1087 if (ret) { 1088 return ret; 1089 } 1090 1091 if (ivshmem_has_feature(s, IVSHMEM_MSI)) { 1092 msix_load(pdev, f); 1093 ivshmem_use_msix(s); 1094 } else { 1095 s->intrstatus = qemu_get_be32(f); 1096 s->intrmask = qemu_get_be32(f); 1097 } 1098 1099 return 0; 1100 } 1101 1102 static const VMStateDescription ivshmem_vmsd = { 1103 .name = "ivshmem", 1104 .version_id = 1, 1105 .minimum_version_id = 1, 1106 .pre_load = ivshmem_pre_load, 1107 .post_load = ivshmem_post_load, 1108 .fields = (VMStateField[]) { 1109 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState), 1110 1111 VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix), 1112 VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix), 1113 VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix), 1114 1115 VMSTATE_END_OF_LIST() 1116 }, 1117 .load_state_old = ivshmem_load_old, 1118 .minimum_version_id_old = 0 1119 }; 1120 1121 static Property ivshmem_properties[] = { 1122 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr), 1123 DEFINE_PROP_STRING("size", IVShmemState, sizearg), 1124 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1), 1125 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, false), 1126 DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true), 1127 DEFINE_PROP_STRING("shm", IVShmemState, shmobj), 1128 DEFINE_PROP_STRING("role", IVShmemState, role), 1129 DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1), 1130 DEFINE_PROP_END_OF_LIST(), 1131 }; 1132 1133 static void ivshmem_class_init(ObjectClass *klass, void *data) 1134 { 1135 DeviceClass *dc = DEVICE_CLASS(klass); 1136 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1137 1138 k->realize = pci_ivshmem_realize; 1139 k->exit = pci_ivshmem_exit; 1140 k->config_write = ivshmem_write_config; 1141 k->vendor_id = PCI_VENDOR_ID_IVSHMEM; 1142 k->device_id = PCI_DEVICE_ID_IVSHMEM; 1143 k->class_id = PCI_CLASS_MEMORY_RAM; 1144 dc->reset = ivshmem_reset; 1145 dc->props = ivshmem_properties; 1146 dc->vmsd = &ivshmem_vmsd; 1147 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1148 dc->desc = "Inter-VM shared memory"; 1149 } 1150 1151 static void ivshmem_check_memdev_is_busy(Object *obj, const char *name, 1152 Object *val, Error **errp) 1153 { 1154 MemoryRegion *mr; 1155 1156 mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), errp); 1157 if (memory_region_is_mapped(mr)) { 1158 char *path = object_get_canonical_path_component(val); 1159 error_setg(errp, "can't use already busy memdev: %s", path); 1160 g_free(path); 1161 } else { 1162 qdev_prop_allow_set_link_before_realize(obj, name, val, errp); 1163 } 1164 } 1165 1166 static void ivshmem_init(Object *obj) 1167 { 1168 IVShmemState *s = IVSHMEM(obj); 1169 1170 object_property_add_link(obj, "x-memdev", TYPE_MEMORY_BACKEND, 1171 (Object **)&s->hostmem, 1172 ivshmem_check_memdev_is_busy, 1173 OBJ_PROP_LINK_UNREF_ON_RELEASE, 1174 &error_abort); 1175 } 1176 1177 static const TypeInfo ivshmem_info = { 1178 .name = TYPE_IVSHMEM, 1179 .parent = TYPE_PCI_DEVICE, 1180 .instance_size = sizeof(IVShmemState), 1181 .instance_init = ivshmem_init, 1182 .class_init = ivshmem_class_init, 1183 }; 1184 1185 static void ivshmem_register_types(void) 1186 { 1187 type_register_static(&ivshmem_info); 1188 } 1189 1190 type_init(ivshmem_register_types) 1191