1 /* 2 * vhost support 3 * 4 * Copyright Red Hat, Inc. 2010 5 * 6 * Authors: 7 * Michael S. Tsirkin <mst@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu/osdep.h" 17 #include "qapi/error.h" 18 #include "hw/virtio/vhost.h" 19 #include "qemu/atomic.h" 20 #include "qemu/range.h" 21 #include "qemu/error-report.h" 22 #include "qemu/memfd.h" 23 #include "standard-headers/linux/vhost_types.h" 24 #include "exec/address-spaces.h" 25 #include "hw/virtio/virtio-bus.h" 26 #include "hw/virtio/virtio-access.h" 27 #include "migration/blocker.h" 28 #include "migration/qemu-file-types.h" 29 #include "sysemu/dma.h" 30 #include "sysemu/tcg.h" 31 #include "trace.h" 32 33 /* enabled until disconnected backend stabilizes */ 34 #define _VHOST_DEBUG 1 35 36 #ifdef _VHOST_DEBUG 37 #define VHOST_OPS_DEBUG(fmt, ...) \ 38 do { error_report(fmt ": %s (%d)", ## __VA_ARGS__, \ 39 strerror(errno), errno); } while (0) 40 #else 41 #define VHOST_OPS_DEBUG(fmt, ...) \ 42 do { } while (0) 43 #endif 44 45 static struct vhost_log *vhost_log; 46 static struct vhost_log *vhost_log_shm; 47 48 static unsigned int used_memslots; 49 static QLIST_HEAD(, vhost_dev) vhost_devices = 50 QLIST_HEAD_INITIALIZER(vhost_devices); 51 52 bool vhost_has_free_slot(void) 53 { 54 unsigned int slots_limit = ~0U; 55 struct vhost_dev *hdev; 56 57 QLIST_FOREACH(hdev, &vhost_devices, entry) { 58 unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev); 59 slots_limit = MIN(slots_limit, r); 60 } 61 return slots_limit > used_memslots; 62 } 63 64 static void vhost_dev_sync_region(struct vhost_dev *dev, 65 MemoryRegionSection *section, 66 uint64_t mfirst, uint64_t mlast, 67 uint64_t rfirst, uint64_t rlast) 68 { 69 vhost_log_chunk_t *log = dev->log->log; 70 71 uint64_t start = MAX(mfirst, rfirst); 72 uint64_t end = MIN(mlast, rlast); 73 vhost_log_chunk_t *from = log + start / VHOST_LOG_CHUNK; 74 vhost_log_chunk_t *to = log + end / VHOST_LOG_CHUNK + 1; 75 uint64_t addr = QEMU_ALIGN_DOWN(start, VHOST_LOG_CHUNK); 76 77 if (end < start) { 78 return; 79 } 80 assert(end / VHOST_LOG_CHUNK < dev->log_size); 81 assert(start / VHOST_LOG_CHUNK < dev->log_size); 82 83 for (;from < to; ++from) { 84 vhost_log_chunk_t log; 85 /* We first check with non-atomic: much cheaper, 86 * and we expect non-dirty to be the common case. */ 87 if (!*from) { 88 addr += VHOST_LOG_CHUNK; 89 continue; 90 } 91 /* Data must be read atomically. We don't really need barrier semantics 92 * but it's easier to use atomic_* than roll our own. */ 93 log = qatomic_xchg(from, 0); 94 while (log) { 95 int bit = ctzl(log); 96 hwaddr page_addr; 97 hwaddr section_offset; 98 hwaddr mr_offset; 99 page_addr = addr + bit * VHOST_LOG_PAGE; 100 section_offset = page_addr - section->offset_within_address_space; 101 mr_offset = section_offset + section->offset_within_region; 102 memory_region_set_dirty(section->mr, mr_offset, VHOST_LOG_PAGE); 103 log &= ~(0x1ull << bit); 104 } 105 addr += VHOST_LOG_CHUNK; 106 } 107 } 108 109 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev, 110 MemoryRegionSection *section, 111 hwaddr first, 112 hwaddr last) 113 { 114 int i; 115 hwaddr start_addr; 116 hwaddr end_addr; 117 118 if (!dev->log_enabled || !dev->started) { 119 return 0; 120 } 121 start_addr = section->offset_within_address_space; 122 end_addr = range_get_last(start_addr, int128_get64(section->size)); 123 start_addr = MAX(first, start_addr); 124 end_addr = MIN(last, end_addr); 125 126 for (i = 0; i < dev->mem->nregions; ++i) { 127 struct vhost_memory_region *reg = dev->mem->regions + i; 128 vhost_dev_sync_region(dev, section, start_addr, end_addr, 129 reg->guest_phys_addr, 130 range_get_last(reg->guest_phys_addr, 131 reg->memory_size)); 132 } 133 for (i = 0; i < dev->nvqs; ++i) { 134 struct vhost_virtqueue *vq = dev->vqs + i; 135 136 if (!vq->used_phys && !vq->used_size) { 137 continue; 138 } 139 140 vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys, 141 range_get_last(vq->used_phys, vq->used_size)); 142 } 143 return 0; 144 } 145 146 static void vhost_log_sync(MemoryListener *listener, 147 MemoryRegionSection *section) 148 { 149 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 150 memory_listener); 151 vhost_sync_dirty_bitmap(dev, section, 0x0, ~0x0ULL); 152 } 153 154 static void vhost_log_sync_range(struct vhost_dev *dev, 155 hwaddr first, hwaddr last) 156 { 157 int i; 158 /* FIXME: this is N^2 in number of sections */ 159 for (i = 0; i < dev->n_mem_sections; ++i) { 160 MemoryRegionSection *section = &dev->mem_sections[i]; 161 vhost_sync_dirty_bitmap(dev, section, first, last); 162 } 163 } 164 165 static uint64_t vhost_get_log_size(struct vhost_dev *dev) 166 { 167 uint64_t log_size = 0; 168 int i; 169 for (i = 0; i < dev->mem->nregions; ++i) { 170 struct vhost_memory_region *reg = dev->mem->regions + i; 171 uint64_t last = range_get_last(reg->guest_phys_addr, 172 reg->memory_size); 173 log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1); 174 } 175 return log_size; 176 } 177 178 static struct vhost_log *vhost_log_alloc(uint64_t size, bool share) 179 { 180 Error *err = NULL; 181 struct vhost_log *log; 182 uint64_t logsize = size * sizeof(*(log->log)); 183 int fd = -1; 184 185 log = g_new0(struct vhost_log, 1); 186 if (share) { 187 log->log = qemu_memfd_alloc("vhost-log", logsize, 188 F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL, 189 &fd, &err); 190 if (err) { 191 error_report_err(err); 192 g_free(log); 193 return NULL; 194 } 195 memset(log->log, 0, logsize); 196 } else { 197 log->log = g_malloc0(logsize); 198 } 199 200 log->size = size; 201 log->refcnt = 1; 202 log->fd = fd; 203 204 return log; 205 } 206 207 static struct vhost_log *vhost_log_get(uint64_t size, bool share) 208 { 209 struct vhost_log *log = share ? vhost_log_shm : vhost_log; 210 211 if (!log || log->size != size) { 212 log = vhost_log_alloc(size, share); 213 if (share) { 214 vhost_log_shm = log; 215 } else { 216 vhost_log = log; 217 } 218 } else { 219 ++log->refcnt; 220 } 221 222 return log; 223 } 224 225 static void vhost_log_put(struct vhost_dev *dev, bool sync) 226 { 227 struct vhost_log *log = dev->log; 228 229 if (!log) { 230 return; 231 } 232 233 --log->refcnt; 234 if (log->refcnt == 0) { 235 /* Sync only the range covered by the old log */ 236 if (dev->log_size && sync) { 237 vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1); 238 } 239 240 if (vhost_log == log) { 241 g_free(log->log); 242 vhost_log = NULL; 243 } else if (vhost_log_shm == log) { 244 qemu_memfd_free(log->log, log->size * sizeof(*(log->log)), 245 log->fd); 246 vhost_log_shm = NULL; 247 } 248 249 g_free(log); 250 } 251 252 dev->log = NULL; 253 dev->log_size = 0; 254 } 255 256 static bool vhost_dev_log_is_shared(struct vhost_dev *dev) 257 { 258 return dev->vhost_ops->vhost_requires_shm_log && 259 dev->vhost_ops->vhost_requires_shm_log(dev); 260 } 261 262 static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size) 263 { 264 struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev)); 265 uint64_t log_base = (uintptr_t)log->log; 266 int r; 267 268 /* inform backend of log switching, this must be done before 269 releasing the current log, to ensure no logging is lost */ 270 r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log); 271 if (r < 0) { 272 VHOST_OPS_DEBUG("vhost_set_log_base failed"); 273 } 274 275 vhost_log_put(dev, true); 276 dev->log = log; 277 dev->log_size = size; 278 } 279 280 static int vhost_dev_has_iommu(struct vhost_dev *dev) 281 { 282 VirtIODevice *vdev = dev->vdev; 283 284 /* 285 * For vhost, VIRTIO_F_IOMMU_PLATFORM means the backend support 286 * incremental memory mapping API via IOTLB API. For platform that 287 * does not have IOMMU, there's no need to enable this feature 288 * which may cause unnecessary IOTLB miss/update trnasactions. 289 */ 290 return vdev->dma_as != &address_space_memory && 291 virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM); 292 } 293 294 static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr, 295 hwaddr *plen, bool is_write) 296 { 297 if (!vhost_dev_has_iommu(dev)) { 298 return cpu_physical_memory_map(addr, plen, is_write); 299 } else { 300 return (void *)(uintptr_t)addr; 301 } 302 } 303 304 static void vhost_memory_unmap(struct vhost_dev *dev, void *buffer, 305 hwaddr len, int is_write, 306 hwaddr access_len) 307 { 308 if (!vhost_dev_has_iommu(dev)) { 309 cpu_physical_memory_unmap(buffer, len, is_write, access_len); 310 } 311 } 312 313 static int vhost_verify_ring_part_mapping(void *ring_hva, 314 uint64_t ring_gpa, 315 uint64_t ring_size, 316 void *reg_hva, 317 uint64_t reg_gpa, 318 uint64_t reg_size) 319 { 320 uint64_t hva_ring_offset; 321 uint64_t ring_last = range_get_last(ring_gpa, ring_size); 322 uint64_t reg_last = range_get_last(reg_gpa, reg_size); 323 324 if (ring_last < reg_gpa || ring_gpa > reg_last) { 325 return 0; 326 } 327 /* check that whole ring's is mapped */ 328 if (ring_last > reg_last) { 329 return -ENOMEM; 330 } 331 /* check that ring's MemoryRegion wasn't replaced */ 332 hva_ring_offset = ring_gpa - reg_gpa; 333 if (ring_hva != reg_hva + hva_ring_offset) { 334 return -EBUSY; 335 } 336 337 return 0; 338 } 339 340 static int vhost_verify_ring_mappings(struct vhost_dev *dev, 341 void *reg_hva, 342 uint64_t reg_gpa, 343 uint64_t reg_size) 344 { 345 int i, j; 346 int r = 0; 347 const char *part_name[] = { 348 "descriptor table", 349 "available ring", 350 "used ring" 351 }; 352 353 if (vhost_dev_has_iommu(dev)) { 354 return 0; 355 } 356 357 for (i = 0; i < dev->nvqs; ++i) { 358 struct vhost_virtqueue *vq = dev->vqs + i; 359 360 if (vq->desc_phys == 0) { 361 continue; 362 } 363 364 j = 0; 365 r = vhost_verify_ring_part_mapping( 366 vq->desc, vq->desc_phys, vq->desc_size, 367 reg_hva, reg_gpa, reg_size); 368 if (r) { 369 break; 370 } 371 372 j++; 373 r = vhost_verify_ring_part_mapping( 374 vq->avail, vq->avail_phys, vq->avail_size, 375 reg_hva, reg_gpa, reg_size); 376 if (r) { 377 break; 378 } 379 380 j++; 381 r = vhost_verify_ring_part_mapping( 382 vq->used, vq->used_phys, vq->used_size, 383 reg_hva, reg_gpa, reg_size); 384 if (r) { 385 break; 386 } 387 } 388 389 if (r == -ENOMEM) { 390 error_report("Unable to map %s for ring %d", part_name[j], i); 391 } else if (r == -EBUSY) { 392 error_report("%s relocated for ring %d", part_name[j], i); 393 } 394 return r; 395 } 396 397 /* 398 * vhost_section: identify sections needed for vhost access 399 * 400 * We only care about RAM sections here (where virtqueue and guest 401 * internals accessed by virtio might live). If we find one we still 402 * allow the backend to potentially filter it out of our list. 403 */ 404 static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section) 405 { 406 MemoryRegion *mr = section->mr; 407 408 if (memory_region_is_ram(mr) && !memory_region_is_rom(mr)) { 409 uint8_t dirty_mask = memory_region_get_dirty_log_mask(mr); 410 uint8_t handled_dirty; 411 412 /* 413 * Kernel based vhost doesn't handle any block which is doing 414 * dirty-tracking other than migration for which it has 415 * specific logging support. However for TCG the kernel never 416 * gets involved anyway so we can also ignore it's 417 * self-modiying code detection flags. However a vhost-user 418 * client could still confuse a TCG guest if it re-writes 419 * executable memory that has already been translated. 420 */ 421 handled_dirty = (1 << DIRTY_MEMORY_MIGRATION) | 422 (1 << DIRTY_MEMORY_CODE); 423 424 if (dirty_mask & ~handled_dirty) { 425 trace_vhost_reject_section(mr->name, 1); 426 return false; 427 } 428 429 if (dev->vhost_ops->vhost_backend_mem_section_filter && 430 !dev->vhost_ops->vhost_backend_mem_section_filter(dev, section)) { 431 trace_vhost_reject_section(mr->name, 2); 432 return false; 433 } 434 435 trace_vhost_section(mr->name); 436 return true; 437 } else { 438 trace_vhost_reject_section(mr->name, 3); 439 return false; 440 } 441 } 442 443 static void vhost_begin(MemoryListener *listener) 444 { 445 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 446 memory_listener); 447 dev->tmp_sections = NULL; 448 dev->n_tmp_sections = 0; 449 } 450 451 static void vhost_commit(MemoryListener *listener) 452 { 453 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 454 memory_listener); 455 MemoryRegionSection *old_sections; 456 int n_old_sections; 457 uint64_t log_size; 458 size_t regions_size; 459 int r; 460 int i; 461 bool changed = false; 462 463 /* Note we can be called before the device is started, but then 464 * starting the device calls set_mem_table, so we need to have 465 * built the data structures. 466 */ 467 old_sections = dev->mem_sections; 468 n_old_sections = dev->n_mem_sections; 469 dev->mem_sections = dev->tmp_sections; 470 dev->n_mem_sections = dev->n_tmp_sections; 471 472 if (dev->n_mem_sections != n_old_sections) { 473 changed = true; 474 } else { 475 /* Same size, lets check the contents */ 476 for (int i = 0; i < n_old_sections; i++) { 477 if (!MemoryRegionSection_eq(&old_sections[i], 478 &dev->mem_sections[i])) { 479 changed = true; 480 break; 481 } 482 } 483 } 484 485 trace_vhost_commit(dev->started, changed); 486 if (!changed) { 487 goto out; 488 } 489 490 /* Rebuild the regions list from the new sections list */ 491 regions_size = offsetof(struct vhost_memory, regions) + 492 dev->n_mem_sections * sizeof dev->mem->regions[0]; 493 dev->mem = g_realloc(dev->mem, regions_size); 494 dev->mem->nregions = dev->n_mem_sections; 495 used_memslots = dev->mem->nregions; 496 for (i = 0; i < dev->n_mem_sections; i++) { 497 struct vhost_memory_region *cur_vmr = dev->mem->regions + i; 498 struct MemoryRegionSection *mrs = dev->mem_sections + i; 499 500 cur_vmr->guest_phys_addr = mrs->offset_within_address_space; 501 cur_vmr->memory_size = int128_get64(mrs->size); 502 cur_vmr->userspace_addr = 503 (uintptr_t)memory_region_get_ram_ptr(mrs->mr) + 504 mrs->offset_within_region; 505 cur_vmr->flags_padding = 0; 506 } 507 508 if (!dev->started) { 509 goto out; 510 } 511 512 for (i = 0; i < dev->mem->nregions; i++) { 513 if (vhost_verify_ring_mappings(dev, 514 (void *)(uintptr_t)dev->mem->regions[i].userspace_addr, 515 dev->mem->regions[i].guest_phys_addr, 516 dev->mem->regions[i].memory_size)) { 517 error_report("Verify ring failure on region %d", i); 518 abort(); 519 } 520 } 521 522 if (!dev->log_enabled) { 523 r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem); 524 if (r < 0) { 525 VHOST_OPS_DEBUG("vhost_set_mem_table failed"); 526 } 527 goto out; 528 } 529 log_size = vhost_get_log_size(dev); 530 /* We allocate an extra 4K bytes to log, 531 * to reduce the * number of reallocations. */ 532 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log) 533 /* To log more, must increase log size before table update. */ 534 if (dev->log_size < log_size) { 535 vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER); 536 } 537 r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem); 538 if (r < 0) { 539 VHOST_OPS_DEBUG("vhost_set_mem_table failed"); 540 } 541 /* To log less, can only decrease log size after table update. */ 542 if (dev->log_size > log_size + VHOST_LOG_BUFFER) { 543 vhost_dev_log_resize(dev, log_size); 544 } 545 546 out: 547 /* Deref the old list of sections, this must happen _after_ the 548 * vhost_set_mem_table to ensure the client isn't still using the 549 * section we're about to unref. 550 */ 551 while (n_old_sections--) { 552 memory_region_unref(old_sections[n_old_sections].mr); 553 } 554 g_free(old_sections); 555 return; 556 } 557 558 /* Adds the section data to the tmp_section structure. 559 * It relies on the listener calling us in memory address order 560 * and for each region (via the _add and _nop methods) to 561 * join neighbours. 562 */ 563 static void vhost_region_add_section(struct vhost_dev *dev, 564 MemoryRegionSection *section) 565 { 566 bool need_add = true; 567 uint64_t mrs_size = int128_get64(section->size); 568 uint64_t mrs_gpa = section->offset_within_address_space; 569 uintptr_t mrs_host = (uintptr_t)memory_region_get_ram_ptr(section->mr) + 570 section->offset_within_region; 571 RAMBlock *mrs_rb = section->mr->ram_block; 572 573 trace_vhost_region_add_section(section->mr->name, mrs_gpa, mrs_size, 574 mrs_host); 575 576 if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER) { 577 /* Round the section to it's page size */ 578 /* First align the start down to a page boundary */ 579 size_t mrs_page = qemu_ram_pagesize(mrs_rb); 580 uint64_t alignage = mrs_host & (mrs_page - 1); 581 if (alignage) { 582 mrs_host -= alignage; 583 mrs_size += alignage; 584 mrs_gpa -= alignage; 585 } 586 /* Now align the size up to a page boundary */ 587 alignage = mrs_size & (mrs_page - 1); 588 if (alignage) { 589 mrs_size += mrs_page - alignage; 590 } 591 trace_vhost_region_add_section_aligned(section->mr->name, mrs_gpa, 592 mrs_size, mrs_host); 593 } 594 595 if (dev->n_tmp_sections) { 596 /* Since we already have at least one section, lets see if 597 * this extends it; since we're scanning in order, we only 598 * have to look at the last one, and the FlatView that calls 599 * us shouldn't have overlaps. 600 */ 601 MemoryRegionSection *prev_sec = dev->tmp_sections + 602 (dev->n_tmp_sections - 1); 603 uint64_t prev_gpa_start = prev_sec->offset_within_address_space; 604 uint64_t prev_size = int128_get64(prev_sec->size); 605 uint64_t prev_gpa_end = range_get_last(prev_gpa_start, prev_size); 606 uint64_t prev_host_start = 607 (uintptr_t)memory_region_get_ram_ptr(prev_sec->mr) + 608 prev_sec->offset_within_region; 609 uint64_t prev_host_end = range_get_last(prev_host_start, prev_size); 610 611 if (mrs_gpa <= (prev_gpa_end + 1)) { 612 /* OK, looks like overlapping/intersecting - it's possible that 613 * the rounding to page sizes has made them overlap, but they should 614 * match up in the same RAMBlock if they do. 615 */ 616 if (mrs_gpa < prev_gpa_start) { 617 error_report("%s:Section '%s' rounded to %"PRIx64 618 " prior to previous '%s' %"PRIx64, 619 __func__, section->mr->name, mrs_gpa, 620 prev_sec->mr->name, prev_gpa_start); 621 /* A way to cleanly fail here would be better */ 622 return; 623 } 624 /* Offset from the start of the previous GPA to this GPA */ 625 size_t offset = mrs_gpa - prev_gpa_start; 626 627 if (prev_host_start + offset == mrs_host && 628 section->mr == prev_sec->mr && 629 (!dev->vhost_ops->vhost_backend_can_merge || 630 dev->vhost_ops->vhost_backend_can_merge(dev, 631 mrs_host, mrs_size, 632 prev_host_start, prev_size))) { 633 uint64_t max_end = MAX(prev_host_end, mrs_host + mrs_size); 634 need_add = false; 635 prev_sec->offset_within_address_space = 636 MIN(prev_gpa_start, mrs_gpa); 637 prev_sec->offset_within_region = 638 MIN(prev_host_start, mrs_host) - 639 (uintptr_t)memory_region_get_ram_ptr(prev_sec->mr); 640 prev_sec->size = int128_make64(max_end - MIN(prev_host_start, 641 mrs_host)); 642 trace_vhost_region_add_section_merge(section->mr->name, 643 int128_get64(prev_sec->size), 644 prev_sec->offset_within_address_space, 645 prev_sec->offset_within_region); 646 } else { 647 /* adjoining regions are fine, but overlapping ones with 648 * different blocks/offsets shouldn't happen 649 */ 650 if (mrs_gpa != prev_gpa_end + 1) { 651 error_report("%s: Overlapping but not coherent sections " 652 "at %"PRIx64, 653 __func__, mrs_gpa); 654 return; 655 } 656 } 657 } 658 } 659 660 if (need_add) { 661 ++dev->n_tmp_sections; 662 dev->tmp_sections = g_renew(MemoryRegionSection, dev->tmp_sections, 663 dev->n_tmp_sections); 664 dev->tmp_sections[dev->n_tmp_sections - 1] = *section; 665 /* The flatview isn't stable and we don't use it, making it NULL 666 * means we can memcmp the list. 667 */ 668 dev->tmp_sections[dev->n_tmp_sections - 1].fv = NULL; 669 memory_region_ref(section->mr); 670 } 671 } 672 673 /* Used for both add and nop callbacks */ 674 static void vhost_region_addnop(MemoryListener *listener, 675 MemoryRegionSection *section) 676 { 677 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 678 memory_listener); 679 680 if (!vhost_section(dev, section)) { 681 return; 682 } 683 vhost_region_add_section(dev, section); 684 } 685 686 static void vhost_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 687 { 688 struct vhost_iommu *iommu = container_of(n, struct vhost_iommu, n); 689 struct vhost_dev *hdev = iommu->hdev; 690 hwaddr iova = iotlb->iova + iommu->iommu_offset; 691 692 if (vhost_backend_invalidate_device_iotlb(hdev, iova, 693 iotlb->addr_mask + 1)) { 694 error_report("Fail to invalidate device iotlb"); 695 } 696 } 697 698 static void vhost_iommu_region_add(MemoryListener *listener, 699 MemoryRegionSection *section) 700 { 701 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 702 iommu_listener); 703 struct vhost_iommu *iommu; 704 Int128 end; 705 int iommu_idx; 706 IOMMUMemoryRegion *iommu_mr; 707 int ret; 708 709 if (!memory_region_is_iommu(section->mr)) { 710 return; 711 } 712 713 iommu_mr = IOMMU_MEMORY_REGION(section->mr); 714 715 iommu = g_malloc0(sizeof(*iommu)); 716 end = int128_add(int128_make64(section->offset_within_region), 717 section->size); 718 end = int128_sub(end, int128_one()); 719 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 720 MEMTXATTRS_UNSPECIFIED); 721 iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify, 722 IOMMU_NOTIFIER_DEVIOTLB_UNMAP, 723 section->offset_within_region, 724 int128_get64(end), 725 iommu_idx); 726 iommu->mr = section->mr; 727 iommu->iommu_offset = section->offset_within_address_space - 728 section->offset_within_region; 729 iommu->hdev = dev; 730 ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 731 if (ret) { 732 /* 733 * Some vIOMMUs do not support dev-iotlb yet. If so, try to use the 734 * UNMAP legacy message 735 */ 736 iommu->n.notifier_flags = IOMMU_NOTIFIER_UNMAP; 737 memory_region_register_iommu_notifier(section->mr, &iommu->n, 738 &error_fatal); 739 } 740 QLIST_INSERT_HEAD(&dev->iommu_list, iommu, iommu_next); 741 /* TODO: can replay help performance here? */ 742 } 743 744 static void vhost_iommu_region_del(MemoryListener *listener, 745 MemoryRegionSection *section) 746 { 747 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 748 iommu_listener); 749 struct vhost_iommu *iommu; 750 751 if (!memory_region_is_iommu(section->mr)) { 752 return; 753 } 754 755 QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) { 756 if (iommu->mr == section->mr && 757 iommu->n.start == section->offset_within_region) { 758 memory_region_unregister_iommu_notifier(iommu->mr, 759 &iommu->n); 760 QLIST_REMOVE(iommu, iommu_next); 761 g_free(iommu); 762 break; 763 } 764 } 765 } 766 767 static int vhost_virtqueue_set_addr(struct vhost_dev *dev, 768 struct vhost_virtqueue *vq, 769 unsigned idx, bool enable_log) 770 { 771 struct vhost_vring_addr addr; 772 int r; 773 memset(&addr, 0, sizeof(struct vhost_vring_addr)); 774 775 if (dev->vhost_ops->vhost_vq_get_addr) { 776 r = dev->vhost_ops->vhost_vq_get_addr(dev, &addr, vq); 777 if (r < 0) { 778 VHOST_OPS_DEBUG("vhost_vq_get_addr failed"); 779 return -errno; 780 } 781 } else { 782 addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc; 783 addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail; 784 addr.used_user_addr = (uint64_t)(unsigned long)vq->used; 785 } 786 addr.index = idx; 787 addr.log_guest_addr = vq->used_phys; 788 addr.flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0; 789 r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr); 790 if (r < 0) { 791 VHOST_OPS_DEBUG("vhost_set_vring_addr failed"); 792 return -errno; 793 } 794 return 0; 795 } 796 797 static int vhost_dev_set_features(struct vhost_dev *dev, 798 bool enable_log) 799 { 800 uint64_t features = dev->acked_features; 801 int r; 802 if (enable_log) { 803 features |= 0x1ULL << VHOST_F_LOG_ALL; 804 } 805 if (!vhost_dev_has_iommu(dev)) { 806 features &= ~(0x1ULL << VIRTIO_F_IOMMU_PLATFORM); 807 } 808 if (dev->vhost_ops->vhost_force_iommu) { 809 if (dev->vhost_ops->vhost_force_iommu(dev) == true) { 810 features |= 0x1ULL << VIRTIO_F_IOMMU_PLATFORM; 811 } 812 } 813 r = dev->vhost_ops->vhost_set_features(dev, features); 814 if (r < 0) { 815 VHOST_OPS_DEBUG("vhost_set_features failed"); 816 goto out; 817 } 818 if (dev->vhost_ops->vhost_set_backend_cap) { 819 r = dev->vhost_ops->vhost_set_backend_cap(dev); 820 if (r < 0) { 821 VHOST_OPS_DEBUG("vhost_set_backend_cap failed"); 822 goto out; 823 } 824 } 825 826 out: 827 return r < 0 ? -errno : 0; 828 } 829 830 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log) 831 { 832 int r, i, idx; 833 hwaddr addr; 834 835 r = vhost_dev_set_features(dev, enable_log); 836 if (r < 0) { 837 goto err_features; 838 } 839 for (i = 0; i < dev->nvqs; ++i) { 840 idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i); 841 addr = virtio_queue_get_desc_addr(dev->vdev, idx); 842 if (!addr) { 843 /* 844 * The queue might not be ready for start. If this 845 * is the case there is no reason to continue the process. 846 * The similar logic is used by the vhost_virtqueue_start() 847 * routine. 848 */ 849 continue; 850 } 851 r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx, 852 enable_log); 853 if (r < 0) { 854 goto err_vq; 855 } 856 } 857 return 0; 858 err_vq: 859 for (; i >= 0; --i) { 860 idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i); 861 vhost_virtqueue_set_addr(dev, dev->vqs + i, idx, 862 dev->log_enabled); 863 } 864 vhost_dev_set_features(dev, dev->log_enabled); 865 err_features: 866 return r; 867 } 868 869 static int vhost_migration_log(MemoryListener *listener, bool enable) 870 { 871 struct vhost_dev *dev = container_of(listener, struct vhost_dev, 872 memory_listener); 873 int r; 874 if (enable == dev->log_enabled) { 875 return 0; 876 } 877 if (!dev->started) { 878 dev->log_enabled = enable; 879 return 0; 880 } 881 882 r = 0; 883 if (!enable) { 884 r = vhost_dev_set_log(dev, false); 885 if (r < 0) { 886 goto check_dev_state; 887 } 888 vhost_log_put(dev, false); 889 } else { 890 vhost_dev_log_resize(dev, vhost_get_log_size(dev)); 891 r = vhost_dev_set_log(dev, true); 892 if (r < 0) { 893 goto check_dev_state; 894 } 895 } 896 897 check_dev_state: 898 dev->log_enabled = enable; 899 /* 900 * vhost-user-* devices could change their state during log 901 * initialization due to disconnect. So check dev state after 902 * vhost communication. 903 */ 904 if (!dev->started) { 905 /* 906 * Since device is in the stopped state, it is okay for 907 * migration. Return success. 908 */ 909 r = 0; 910 } 911 if (r) { 912 /* An error occurred. */ 913 dev->log_enabled = false; 914 } 915 916 return r; 917 } 918 919 static void vhost_log_global_start(MemoryListener *listener) 920 { 921 int r; 922 923 r = vhost_migration_log(listener, true); 924 if (r < 0) { 925 abort(); 926 } 927 } 928 929 static void vhost_log_global_stop(MemoryListener *listener) 930 { 931 int r; 932 933 r = vhost_migration_log(listener, false); 934 if (r < 0) { 935 abort(); 936 } 937 } 938 939 static void vhost_log_start(MemoryListener *listener, 940 MemoryRegionSection *section, 941 int old, int new) 942 { 943 /* FIXME: implement */ 944 } 945 946 static void vhost_log_stop(MemoryListener *listener, 947 MemoryRegionSection *section, 948 int old, int new) 949 { 950 /* FIXME: implement */ 951 } 952 953 /* The vhost driver natively knows how to handle the vrings of non 954 * cross-endian legacy devices and modern devices. Only legacy devices 955 * exposed to a bi-endian guest may require the vhost driver to use a 956 * specific endianness. 957 */ 958 static inline bool vhost_needs_vring_endian(VirtIODevice *vdev) 959 { 960 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 961 return false; 962 } 963 #ifdef HOST_WORDS_BIGENDIAN 964 return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE; 965 #else 966 return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG; 967 #endif 968 } 969 970 static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev, 971 bool is_big_endian, 972 int vhost_vq_index) 973 { 974 struct vhost_vring_state s = { 975 .index = vhost_vq_index, 976 .num = is_big_endian 977 }; 978 979 if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) { 980 return 0; 981 } 982 983 VHOST_OPS_DEBUG("vhost_set_vring_endian failed"); 984 if (errno == ENOTTY) { 985 error_report("vhost does not support cross-endian"); 986 return -ENOSYS; 987 } 988 989 return -errno; 990 } 991 992 static int vhost_memory_region_lookup(struct vhost_dev *hdev, 993 uint64_t gpa, uint64_t *uaddr, 994 uint64_t *len) 995 { 996 int i; 997 998 for (i = 0; i < hdev->mem->nregions; i++) { 999 struct vhost_memory_region *reg = hdev->mem->regions + i; 1000 1001 if (gpa >= reg->guest_phys_addr && 1002 reg->guest_phys_addr + reg->memory_size > gpa) { 1003 *uaddr = reg->userspace_addr + gpa - reg->guest_phys_addr; 1004 *len = reg->guest_phys_addr + reg->memory_size - gpa; 1005 return 0; 1006 } 1007 } 1008 1009 return -EFAULT; 1010 } 1011 1012 int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write) 1013 { 1014 IOMMUTLBEntry iotlb; 1015 uint64_t uaddr, len; 1016 int ret = -EFAULT; 1017 1018 RCU_READ_LOCK_GUARD(); 1019 1020 trace_vhost_iotlb_miss(dev, 1); 1021 1022 iotlb = address_space_get_iotlb_entry(dev->vdev->dma_as, 1023 iova, write, 1024 MEMTXATTRS_UNSPECIFIED); 1025 if (iotlb.target_as != NULL) { 1026 ret = vhost_memory_region_lookup(dev, iotlb.translated_addr, 1027 &uaddr, &len); 1028 if (ret) { 1029 trace_vhost_iotlb_miss(dev, 3); 1030 error_report("Fail to lookup the translated address " 1031 "%"PRIx64, iotlb.translated_addr); 1032 goto out; 1033 } 1034 1035 len = MIN(iotlb.addr_mask + 1, len); 1036 iova = iova & ~iotlb.addr_mask; 1037 1038 ret = vhost_backend_update_device_iotlb(dev, iova, uaddr, 1039 len, iotlb.perm); 1040 if (ret) { 1041 trace_vhost_iotlb_miss(dev, 4); 1042 error_report("Fail to update device iotlb"); 1043 goto out; 1044 } 1045 } 1046 1047 trace_vhost_iotlb_miss(dev, 2); 1048 1049 out: 1050 return ret; 1051 } 1052 1053 static int vhost_virtqueue_start(struct vhost_dev *dev, 1054 struct VirtIODevice *vdev, 1055 struct vhost_virtqueue *vq, 1056 unsigned idx) 1057 { 1058 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 1059 VirtioBusState *vbus = VIRTIO_BUS(qbus); 1060 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus); 1061 hwaddr s, l, a; 1062 int r; 1063 int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx); 1064 struct vhost_vring_file file = { 1065 .index = vhost_vq_index 1066 }; 1067 struct vhost_vring_state state = { 1068 .index = vhost_vq_index 1069 }; 1070 struct VirtQueue *vvq = virtio_get_queue(vdev, idx); 1071 1072 a = virtio_queue_get_desc_addr(vdev, idx); 1073 if (a == 0) { 1074 /* Queue might not be ready for start */ 1075 return 0; 1076 } 1077 1078 vq->num = state.num = virtio_queue_get_num(vdev, idx); 1079 r = dev->vhost_ops->vhost_set_vring_num(dev, &state); 1080 if (r) { 1081 VHOST_OPS_DEBUG("vhost_set_vring_num failed"); 1082 return -errno; 1083 } 1084 1085 state.num = virtio_queue_get_last_avail_idx(vdev, idx); 1086 r = dev->vhost_ops->vhost_set_vring_base(dev, &state); 1087 if (r) { 1088 VHOST_OPS_DEBUG("vhost_set_vring_base failed"); 1089 return -errno; 1090 } 1091 1092 if (vhost_needs_vring_endian(vdev)) { 1093 r = vhost_virtqueue_set_vring_endian_legacy(dev, 1094 virtio_is_big_endian(vdev), 1095 vhost_vq_index); 1096 if (r) { 1097 return -errno; 1098 } 1099 } 1100 1101 vq->desc_size = s = l = virtio_queue_get_desc_size(vdev, idx); 1102 vq->desc_phys = a; 1103 vq->desc = vhost_memory_map(dev, a, &l, false); 1104 if (!vq->desc || l != s) { 1105 r = -ENOMEM; 1106 goto fail_alloc_desc; 1107 } 1108 vq->avail_size = s = l = virtio_queue_get_avail_size(vdev, idx); 1109 vq->avail_phys = a = virtio_queue_get_avail_addr(vdev, idx); 1110 vq->avail = vhost_memory_map(dev, a, &l, false); 1111 if (!vq->avail || l != s) { 1112 r = -ENOMEM; 1113 goto fail_alloc_avail; 1114 } 1115 vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx); 1116 vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx); 1117 vq->used = vhost_memory_map(dev, a, &l, true); 1118 if (!vq->used || l != s) { 1119 r = -ENOMEM; 1120 goto fail_alloc_used; 1121 } 1122 1123 r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled); 1124 if (r < 0) { 1125 r = -errno; 1126 goto fail_alloc; 1127 } 1128 1129 file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq)); 1130 r = dev->vhost_ops->vhost_set_vring_kick(dev, &file); 1131 if (r) { 1132 VHOST_OPS_DEBUG("vhost_set_vring_kick failed"); 1133 r = -errno; 1134 goto fail_kick; 1135 } 1136 1137 /* Clear and discard previous events if any. */ 1138 event_notifier_test_and_clear(&vq->masked_notifier); 1139 1140 /* Init vring in unmasked state, unless guest_notifier_mask 1141 * will do it later. 1142 */ 1143 if (!vdev->use_guest_notifier_mask) { 1144 /* TODO: check and handle errors. */ 1145 vhost_virtqueue_mask(dev, vdev, idx, false); 1146 } 1147 1148 if (k->query_guest_notifiers && 1149 k->query_guest_notifiers(qbus->parent) && 1150 virtio_queue_vector(vdev, idx) == VIRTIO_NO_VECTOR) { 1151 file.fd = -1; 1152 r = dev->vhost_ops->vhost_set_vring_call(dev, &file); 1153 if (r) { 1154 goto fail_vector; 1155 } 1156 } 1157 1158 return 0; 1159 1160 fail_vector: 1161 fail_kick: 1162 fail_alloc: 1163 vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx), 1164 0, 0); 1165 fail_alloc_used: 1166 vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx), 1167 0, 0); 1168 fail_alloc_avail: 1169 vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx), 1170 0, 0); 1171 fail_alloc_desc: 1172 return r; 1173 } 1174 1175 static void vhost_virtqueue_stop(struct vhost_dev *dev, 1176 struct VirtIODevice *vdev, 1177 struct vhost_virtqueue *vq, 1178 unsigned idx) 1179 { 1180 int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx); 1181 struct vhost_vring_state state = { 1182 .index = vhost_vq_index, 1183 }; 1184 int r; 1185 1186 if (virtio_queue_get_desc_addr(vdev, idx) == 0) { 1187 /* Don't stop the virtqueue which might have not been started */ 1188 return; 1189 } 1190 1191 r = dev->vhost_ops->vhost_get_vring_base(dev, &state); 1192 if (r < 0) { 1193 VHOST_OPS_DEBUG("vhost VQ %u ring restore failed: %d", idx, r); 1194 /* Connection to the backend is broken, so let's sync internal 1195 * last avail idx to the device used idx. 1196 */ 1197 virtio_queue_restore_last_avail_idx(vdev, idx); 1198 } else { 1199 virtio_queue_set_last_avail_idx(vdev, idx, state.num); 1200 } 1201 virtio_queue_invalidate_signalled_used(vdev, idx); 1202 virtio_queue_update_used_idx(vdev, idx); 1203 1204 /* In the cross-endian case, we need to reset the vring endianness to 1205 * native as legacy devices expect so by default. 1206 */ 1207 if (vhost_needs_vring_endian(vdev)) { 1208 vhost_virtqueue_set_vring_endian_legacy(dev, 1209 !virtio_is_big_endian(vdev), 1210 vhost_vq_index); 1211 } 1212 1213 vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx), 1214 1, virtio_queue_get_used_size(vdev, idx)); 1215 vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx), 1216 0, virtio_queue_get_avail_size(vdev, idx)); 1217 vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx), 1218 0, virtio_queue_get_desc_size(vdev, idx)); 1219 } 1220 1221 static void vhost_eventfd_add(MemoryListener *listener, 1222 MemoryRegionSection *section, 1223 bool match_data, uint64_t data, EventNotifier *e) 1224 { 1225 } 1226 1227 static void vhost_eventfd_del(MemoryListener *listener, 1228 MemoryRegionSection *section, 1229 bool match_data, uint64_t data, EventNotifier *e) 1230 { 1231 } 1232 1233 static int vhost_virtqueue_set_busyloop_timeout(struct vhost_dev *dev, 1234 int n, uint32_t timeout) 1235 { 1236 int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n); 1237 struct vhost_vring_state state = { 1238 .index = vhost_vq_index, 1239 .num = timeout, 1240 }; 1241 int r; 1242 1243 if (!dev->vhost_ops->vhost_set_vring_busyloop_timeout) { 1244 return -EINVAL; 1245 } 1246 1247 r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state); 1248 if (r) { 1249 VHOST_OPS_DEBUG("vhost_set_vring_busyloop_timeout failed"); 1250 return r; 1251 } 1252 1253 return 0; 1254 } 1255 1256 static int vhost_virtqueue_init(struct vhost_dev *dev, 1257 struct vhost_virtqueue *vq, int n) 1258 { 1259 int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n); 1260 struct vhost_vring_file file = { 1261 .index = vhost_vq_index, 1262 }; 1263 int r = event_notifier_init(&vq->masked_notifier, 0); 1264 if (r < 0) { 1265 return r; 1266 } 1267 1268 file.fd = event_notifier_get_fd(&vq->masked_notifier); 1269 r = dev->vhost_ops->vhost_set_vring_call(dev, &file); 1270 if (r) { 1271 VHOST_OPS_DEBUG("vhost_set_vring_call failed"); 1272 r = -errno; 1273 goto fail_call; 1274 } 1275 1276 vq->dev = dev; 1277 1278 return 0; 1279 fail_call: 1280 event_notifier_cleanup(&vq->masked_notifier); 1281 return r; 1282 } 1283 1284 static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq) 1285 { 1286 event_notifier_cleanup(&vq->masked_notifier); 1287 } 1288 1289 int vhost_dev_init(struct vhost_dev *hdev, void *opaque, 1290 VhostBackendType backend_type, uint32_t busyloop_timeout) 1291 { 1292 uint64_t features; 1293 int i, r, n_initialized_vqs = 0; 1294 Error *local_err = NULL; 1295 1296 hdev->vdev = NULL; 1297 hdev->migration_blocker = NULL; 1298 1299 r = vhost_set_backend_type(hdev, backend_type); 1300 assert(r >= 0); 1301 1302 r = hdev->vhost_ops->vhost_backend_init(hdev, opaque); 1303 if (r < 0) { 1304 goto fail; 1305 } 1306 1307 r = hdev->vhost_ops->vhost_set_owner(hdev); 1308 if (r < 0) { 1309 VHOST_OPS_DEBUG("vhost_set_owner failed"); 1310 goto fail; 1311 } 1312 1313 r = hdev->vhost_ops->vhost_get_features(hdev, &features); 1314 if (r < 0) { 1315 VHOST_OPS_DEBUG("vhost_get_features failed"); 1316 goto fail; 1317 } 1318 1319 for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) { 1320 r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i); 1321 if (r < 0) { 1322 goto fail; 1323 } 1324 } 1325 1326 if (busyloop_timeout) { 1327 for (i = 0; i < hdev->nvqs; ++i) { 1328 r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 1329 busyloop_timeout); 1330 if (r < 0) { 1331 goto fail_busyloop; 1332 } 1333 } 1334 } 1335 1336 hdev->features = features; 1337 1338 hdev->memory_listener = (MemoryListener) { 1339 .begin = vhost_begin, 1340 .commit = vhost_commit, 1341 .region_add = vhost_region_addnop, 1342 .region_nop = vhost_region_addnop, 1343 .log_start = vhost_log_start, 1344 .log_stop = vhost_log_stop, 1345 .log_sync = vhost_log_sync, 1346 .log_global_start = vhost_log_global_start, 1347 .log_global_stop = vhost_log_global_stop, 1348 .eventfd_add = vhost_eventfd_add, 1349 .eventfd_del = vhost_eventfd_del, 1350 .priority = 10 1351 }; 1352 1353 hdev->iommu_listener = (MemoryListener) { 1354 .region_add = vhost_iommu_region_add, 1355 .region_del = vhost_iommu_region_del, 1356 }; 1357 1358 if (hdev->migration_blocker == NULL) { 1359 if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) { 1360 error_setg(&hdev->migration_blocker, 1361 "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature."); 1362 } else if (vhost_dev_log_is_shared(hdev) && !qemu_memfd_alloc_check()) { 1363 error_setg(&hdev->migration_blocker, 1364 "Migration disabled: failed to allocate shared memory"); 1365 } 1366 } 1367 1368 if (hdev->migration_blocker != NULL) { 1369 r = migrate_add_blocker(hdev->migration_blocker, &local_err); 1370 if (local_err) { 1371 error_report_err(local_err); 1372 error_free(hdev->migration_blocker); 1373 goto fail_busyloop; 1374 } 1375 } 1376 1377 hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions)); 1378 hdev->n_mem_sections = 0; 1379 hdev->mem_sections = NULL; 1380 hdev->log = NULL; 1381 hdev->log_size = 0; 1382 hdev->log_enabled = false; 1383 hdev->started = false; 1384 memory_listener_register(&hdev->memory_listener, &address_space_memory); 1385 QLIST_INSERT_HEAD(&vhost_devices, hdev, entry); 1386 1387 if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) { 1388 error_report("vhost backend memory slots limit is less" 1389 " than current number of present memory slots"); 1390 r = -1; 1391 goto fail_busyloop; 1392 } 1393 1394 return 0; 1395 1396 fail_busyloop: 1397 if (busyloop_timeout) { 1398 while (--i >= 0) { 1399 vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0); 1400 } 1401 } 1402 fail: 1403 hdev->nvqs = n_initialized_vqs; 1404 vhost_dev_cleanup(hdev); 1405 return r; 1406 } 1407 1408 void vhost_dev_cleanup(struct vhost_dev *hdev) 1409 { 1410 int i; 1411 1412 for (i = 0; i < hdev->nvqs; ++i) { 1413 vhost_virtqueue_cleanup(hdev->vqs + i); 1414 } 1415 if (hdev->mem) { 1416 /* those are only safe after successful init */ 1417 memory_listener_unregister(&hdev->memory_listener); 1418 QLIST_REMOVE(hdev, entry); 1419 } 1420 if (hdev->migration_blocker) { 1421 migrate_del_blocker(hdev->migration_blocker); 1422 error_free(hdev->migration_blocker); 1423 } 1424 g_free(hdev->mem); 1425 g_free(hdev->mem_sections); 1426 if (hdev->vhost_ops) { 1427 hdev->vhost_ops->vhost_backend_cleanup(hdev); 1428 } 1429 assert(!hdev->log); 1430 1431 memset(hdev, 0, sizeof(struct vhost_dev)); 1432 } 1433 1434 /* Stop processing guest IO notifications in qemu. 1435 * Start processing them in vhost in kernel. 1436 */ 1437 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev) 1438 { 1439 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 1440 int i, r, e; 1441 1442 /* We will pass the notifiers to the kernel, make sure that QEMU 1443 * doesn't interfere. 1444 */ 1445 r = virtio_device_grab_ioeventfd(vdev); 1446 if (r < 0) { 1447 error_report("binding does not support host notifiers"); 1448 goto fail; 1449 } 1450 1451 for (i = 0; i < hdev->nvqs; ++i) { 1452 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i, 1453 true); 1454 if (r < 0) { 1455 error_report("vhost VQ %d notifier binding failed: %d", i, -r); 1456 goto fail_vq; 1457 } 1458 } 1459 1460 return 0; 1461 fail_vq: 1462 while (--i >= 0) { 1463 e = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i, 1464 false); 1465 if (e < 0) { 1466 error_report("vhost VQ %d notifier cleanup error: %d", i, -r); 1467 } 1468 assert (e >= 0); 1469 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i); 1470 } 1471 virtio_device_release_ioeventfd(vdev); 1472 fail: 1473 return r; 1474 } 1475 1476 /* Stop processing guest IO notifications in vhost. 1477 * Start processing them in qemu. 1478 * This might actually run the qemu handlers right away, 1479 * so virtio in qemu must be completely setup when this is called. 1480 */ 1481 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev) 1482 { 1483 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 1484 int i, r; 1485 1486 for (i = 0; i < hdev->nvqs; ++i) { 1487 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i, 1488 false); 1489 if (r < 0) { 1490 error_report("vhost VQ %d notifier cleanup failed: %d", i, -r); 1491 } 1492 assert (r >= 0); 1493 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i); 1494 } 1495 virtio_device_release_ioeventfd(vdev); 1496 } 1497 1498 /* Test and clear event pending status. 1499 * Should be called after unmask to avoid losing events. 1500 */ 1501 bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n) 1502 { 1503 struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index; 1504 assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs); 1505 return event_notifier_test_and_clear(&vq->masked_notifier); 1506 } 1507 1508 /* Mask/unmask events from this vq. */ 1509 void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n, 1510 bool mask) 1511 { 1512 struct VirtQueue *vvq = virtio_get_queue(vdev, n); 1513 int r, index = n - hdev->vq_index; 1514 struct vhost_vring_file file; 1515 1516 /* should only be called after backend is connected */ 1517 assert(hdev->vhost_ops); 1518 1519 if (mask) { 1520 assert(vdev->use_guest_notifier_mask); 1521 file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier); 1522 } else { 1523 file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq)); 1524 } 1525 1526 file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n); 1527 r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file); 1528 if (r < 0) { 1529 VHOST_OPS_DEBUG("vhost_set_vring_call failed"); 1530 } 1531 } 1532 1533 uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits, 1534 uint64_t features) 1535 { 1536 const int *bit = feature_bits; 1537 while (*bit != VHOST_INVALID_FEATURE_BIT) { 1538 uint64_t bit_mask = (1ULL << *bit); 1539 if (!(hdev->features & bit_mask)) { 1540 features &= ~bit_mask; 1541 } 1542 bit++; 1543 } 1544 return features; 1545 } 1546 1547 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits, 1548 uint64_t features) 1549 { 1550 const int *bit = feature_bits; 1551 while (*bit != VHOST_INVALID_FEATURE_BIT) { 1552 uint64_t bit_mask = (1ULL << *bit); 1553 if (features & bit_mask) { 1554 hdev->acked_features |= bit_mask; 1555 } 1556 bit++; 1557 } 1558 } 1559 1560 int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config, 1561 uint32_t config_len) 1562 { 1563 assert(hdev->vhost_ops); 1564 1565 if (hdev->vhost_ops->vhost_get_config) { 1566 return hdev->vhost_ops->vhost_get_config(hdev, config, config_len); 1567 } 1568 1569 return -1; 1570 } 1571 1572 int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data, 1573 uint32_t offset, uint32_t size, uint32_t flags) 1574 { 1575 assert(hdev->vhost_ops); 1576 1577 if (hdev->vhost_ops->vhost_set_config) { 1578 return hdev->vhost_ops->vhost_set_config(hdev, data, offset, 1579 size, flags); 1580 } 1581 1582 return -1; 1583 } 1584 1585 void vhost_dev_set_config_notifier(struct vhost_dev *hdev, 1586 const VhostDevConfigOps *ops) 1587 { 1588 hdev->config_ops = ops; 1589 } 1590 1591 void vhost_dev_free_inflight(struct vhost_inflight *inflight) 1592 { 1593 if (inflight && inflight->addr) { 1594 qemu_memfd_free(inflight->addr, inflight->size, inflight->fd); 1595 inflight->addr = NULL; 1596 inflight->fd = -1; 1597 } 1598 } 1599 1600 static int vhost_dev_resize_inflight(struct vhost_inflight *inflight, 1601 uint64_t new_size) 1602 { 1603 Error *err = NULL; 1604 int fd = -1; 1605 void *addr = qemu_memfd_alloc("vhost-inflight", new_size, 1606 F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL, 1607 &fd, &err); 1608 1609 if (err) { 1610 error_report_err(err); 1611 return -1; 1612 } 1613 1614 vhost_dev_free_inflight(inflight); 1615 inflight->offset = 0; 1616 inflight->addr = addr; 1617 inflight->fd = fd; 1618 inflight->size = new_size; 1619 1620 return 0; 1621 } 1622 1623 void vhost_dev_save_inflight(struct vhost_inflight *inflight, QEMUFile *f) 1624 { 1625 if (inflight->addr) { 1626 qemu_put_be64(f, inflight->size); 1627 qemu_put_be16(f, inflight->queue_size); 1628 qemu_put_buffer(f, inflight->addr, inflight->size); 1629 } else { 1630 qemu_put_be64(f, 0); 1631 } 1632 } 1633 1634 int vhost_dev_load_inflight(struct vhost_inflight *inflight, QEMUFile *f) 1635 { 1636 uint64_t size; 1637 1638 size = qemu_get_be64(f); 1639 if (!size) { 1640 return 0; 1641 } 1642 1643 if (inflight->size != size) { 1644 if (vhost_dev_resize_inflight(inflight, size)) { 1645 return -1; 1646 } 1647 } 1648 inflight->queue_size = qemu_get_be16(f); 1649 1650 qemu_get_buffer(f, inflight->addr, size); 1651 1652 return 0; 1653 } 1654 1655 int vhost_dev_prepare_inflight(struct vhost_dev *hdev, VirtIODevice *vdev) 1656 { 1657 int r; 1658 1659 if (hdev->vhost_ops->vhost_get_inflight_fd == NULL || 1660 hdev->vhost_ops->vhost_set_inflight_fd == NULL) { 1661 return 0; 1662 } 1663 1664 hdev->vdev = vdev; 1665 1666 r = vhost_dev_set_features(hdev, hdev->log_enabled); 1667 if (r < 0) { 1668 VHOST_OPS_DEBUG("vhost_dev_prepare_inflight failed"); 1669 return r; 1670 } 1671 1672 return 0; 1673 } 1674 1675 int vhost_dev_set_inflight(struct vhost_dev *dev, 1676 struct vhost_inflight *inflight) 1677 { 1678 int r; 1679 1680 if (dev->vhost_ops->vhost_set_inflight_fd && inflight->addr) { 1681 r = dev->vhost_ops->vhost_set_inflight_fd(dev, inflight); 1682 if (r) { 1683 VHOST_OPS_DEBUG("vhost_set_inflight_fd failed"); 1684 return -errno; 1685 } 1686 } 1687 1688 return 0; 1689 } 1690 1691 int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size, 1692 struct vhost_inflight *inflight) 1693 { 1694 int r; 1695 1696 if (dev->vhost_ops->vhost_get_inflight_fd) { 1697 r = dev->vhost_ops->vhost_get_inflight_fd(dev, queue_size, inflight); 1698 if (r) { 1699 VHOST_OPS_DEBUG("vhost_get_inflight_fd failed"); 1700 return -errno; 1701 } 1702 } 1703 1704 return 0; 1705 } 1706 1707 /* Host notifiers must be enabled at this point. */ 1708 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) 1709 { 1710 int i, r; 1711 1712 /* should only be called after backend is connected */ 1713 assert(hdev->vhost_ops); 1714 1715 hdev->started = true; 1716 hdev->vdev = vdev; 1717 1718 r = vhost_dev_set_features(hdev, hdev->log_enabled); 1719 if (r < 0) { 1720 goto fail_features; 1721 } 1722 1723 if (vhost_dev_has_iommu(hdev)) { 1724 memory_listener_register(&hdev->iommu_listener, vdev->dma_as); 1725 } 1726 1727 r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem); 1728 if (r < 0) { 1729 VHOST_OPS_DEBUG("vhost_set_mem_table failed"); 1730 r = -errno; 1731 goto fail_mem; 1732 } 1733 for (i = 0; i < hdev->nvqs; ++i) { 1734 r = vhost_virtqueue_start(hdev, 1735 vdev, 1736 hdev->vqs + i, 1737 hdev->vq_index + i); 1738 if (r < 0) { 1739 goto fail_vq; 1740 } 1741 } 1742 1743 if (hdev->log_enabled) { 1744 uint64_t log_base; 1745 1746 hdev->log_size = vhost_get_log_size(hdev); 1747 hdev->log = vhost_log_get(hdev->log_size, 1748 vhost_dev_log_is_shared(hdev)); 1749 log_base = (uintptr_t)hdev->log->log; 1750 r = hdev->vhost_ops->vhost_set_log_base(hdev, 1751 hdev->log_size ? log_base : 0, 1752 hdev->log); 1753 if (r < 0) { 1754 VHOST_OPS_DEBUG("vhost_set_log_base failed"); 1755 r = -errno; 1756 goto fail_log; 1757 } 1758 } 1759 if (hdev->vhost_ops->vhost_dev_start) { 1760 r = hdev->vhost_ops->vhost_dev_start(hdev, true); 1761 if (r) { 1762 goto fail_log; 1763 } 1764 } 1765 if (vhost_dev_has_iommu(hdev) && 1766 hdev->vhost_ops->vhost_set_iotlb_callback) { 1767 hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true); 1768 1769 /* Update used ring information for IOTLB to work correctly, 1770 * vhost-kernel code requires for this.*/ 1771 for (i = 0; i < hdev->nvqs; ++i) { 1772 struct vhost_virtqueue *vq = hdev->vqs + i; 1773 vhost_device_iotlb_miss(hdev, vq->used_phys, true); 1774 } 1775 } 1776 return 0; 1777 fail_log: 1778 vhost_log_put(hdev, false); 1779 fail_vq: 1780 while (--i >= 0) { 1781 vhost_virtqueue_stop(hdev, 1782 vdev, 1783 hdev->vqs + i, 1784 hdev->vq_index + i); 1785 } 1786 1787 fail_mem: 1788 fail_features: 1789 1790 hdev->started = false; 1791 return r; 1792 } 1793 1794 /* Host notifiers must be enabled at this point. */ 1795 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev) 1796 { 1797 int i; 1798 1799 /* should only be called after backend is connected */ 1800 assert(hdev->vhost_ops); 1801 1802 if (hdev->vhost_ops->vhost_dev_start) { 1803 hdev->vhost_ops->vhost_dev_start(hdev, false); 1804 } 1805 for (i = 0; i < hdev->nvqs; ++i) { 1806 vhost_virtqueue_stop(hdev, 1807 vdev, 1808 hdev->vqs + i, 1809 hdev->vq_index + i); 1810 } 1811 1812 if (vhost_dev_has_iommu(hdev)) { 1813 if (hdev->vhost_ops->vhost_set_iotlb_callback) { 1814 hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false); 1815 } 1816 memory_listener_unregister(&hdev->iommu_listener); 1817 } 1818 vhost_log_put(hdev, true); 1819 hdev->started = false; 1820 hdev->vdev = NULL; 1821 } 1822 1823 int vhost_net_set_backend(struct vhost_dev *hdev, 1824 struct vhost_vring_file *file) 1825 { 1826 if (hdev->vhost_ops->vhost_net_set_backend) { 1827 return hdev->vhost_ops->vhost_net_set_backend(hdev, file); 1828 } 1829 1830 return -1; 1831 } 1832