1 /* 2 * vhost-vdpa 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <linux/vhost.h> 14 #include <linux/vfio.h> 15 #include <sys/eventfd.h> 16 #include <sys/ioctl.h> 17 #include "exec/target_page.h" 18 #include "hw/virtio/vhost.h" 19 #include "hw/virtio/vhost-backend.h" 20 #include "hw/virtio/virtio-net.h" 21 #include "hw/virtio/vhost-shadow-virtqueue.h" 22 #include "hw/virtio/vhost-vdpa.h" 23 #include "exec/address-spaces.h" 24 #include "migration/blocker.h" 25 #include "qemu/cutils.h" 26 #include "qemu/main-loop.h" 27 #include "trace.h" 28 #include "qapi/error.h" 29 30 /* 31 * Return one past the end of the end of section. Be careful with uint64_t 32 * conversions! 33 */ 34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section, 35 int page_mask) 36 { 37 Int128 llend = int128_make64(section->offset_within_address_space); 38 llend = int128_add(llend, section->size); 39 llend = int128_and(llend, int128_exts64(page_mask)); 40 41 return llend; 42 } 43 44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section, 45 uint64_t iova_min, 46 uint64_t iova_max, 47 int page_mask) 48 { 49 Int128 llend; 50 51 if ((!memory_region_is_ram(section->mr) && 52 !memory_region_is_iommu(section->mr)) || 53 memory_region_is_protected(section->mr) || 54 /* vhost-vDPA doesn't allow MMIO to be mapped */ 55 memory_region_is_ram_device(section->mr)) { 56 return true; 57 } 58 59 if (section->offset_within_address_space < iova_min) { 60 error_report("RAM section out of device range (min=0x%" PRIx64 61 ", addr=0x%" HWADDR_PRIx ")", 62 iova_min, section->offset_within_address_space); 63 return true; 64 } 65 /* 66 * While using vIOMMU, sometimes the section will be larger than iova_max, 67 * but the memory that actually maps is smaller, so move the check to 68 * function vhost_vdpa_iommu_map_notify(). That function will use the actual 69 * size that maps to the kernel 70 */ 71 72 if (!memory_region_is_iommu(section->mr)) { 73 llend = vhost_vdpa_section_end(section, page_mask); 74 if (int128_gt(llend, int128_make64(iova_max))) { 75 error_report("RAM section out of device range (max=0x%" PRIx64 76 ", end addr=0x%" PRIx64 ")", 77 iova_max, int128_get64(llend)); 78 return true; 79 } 80 } 81 82 return false; 83 } 84 85 /* 86 * The caller must set asid = 0 if the device does not support asid. 87 * This is not an ABI break since it is set to 0 by the initializer anyway. 88 */ 89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 90 hwaddr size, void *vaddr, bool readonly) 91 { 92 struct vhost_msg_v2 msg = {}; 93 int fd = v->shared->device_fd; 94 int ret = 0; 95 96 msg.type = v->msg_type; 97 msg.asid = asid; 98 msg.iotlb.iova = iova; 99 msg.iotlb.size = size; 100 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 101 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 102 msg.iotlb.type = VHOST_IOTLB_UPDATE; 103 104 trace_vhost_vdpa_dma_map(v->shared, fd, msg.type, msg.asid, msg.iotlb.iova, 105 msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm, 106 msg.iotlb.type); 107 108 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 109 error_report("failed to write, fd=%d, errno=%d (%s)", 110 fd, errno, strerror(errno)); 111 return -EIO ; 112 } 113 114 return ret; 115 } 116 117 /* 118 * The caller must set asid = 0 if the device does not support asid. 119 * This is not an ABI break since it is set to 0 by the initializer anyway. 120 */ 121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 122 hwaddr size) 123 { 124 struct vhost_msg_v2 msg = {}; 125 int fd = v->shared->device_fd; 126 int ret = 0; 127 128 msg.type = v->msg_type; 129 msg.asid = asid; 130 msg.iotlb.iova = iova; 131 msg.iotlb.size = size; 132 msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 133 134 trace_vhost_vdpa_dma_unmap(v->shared, fd, msg.type, msg.asid, 135 msg.iotlb.iova, msg.iotlb.size, msg.iotlb.type); 136 137 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 138 error_report("failed to write, fd=%d, errno=%d (%s)", 139 fd, errno, strerror(errno)); 140 return -EIO ; 141 } 142 143 return ret; 144 } 145 146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v) 147 { 148 int fd = v->shared->device_fd; 149 struct vhost_msg_v2 msg = { 150 .type = v->msg_type, 151 .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, 152 }; 153 154 trace_vhost_vdpa_listener_begin_batch(v->shared, fd, msg.type, 155 msg.iotlb.type); 156 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 157 error_report("failed to write, fd=%d, errno=%d (%s)", 158 fd, errno, strerror(errno)); 159 } 160 } 161 162 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v) 163 { 164 if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && 165 !v->iotlb_batch_begin_sent) { 166 vhost_vdpa_listener_begin_batch(v); 167 } 168 169 v->iotlb_batch_begin_sent = true; 170 } 171 172 static void vhost_vdpa_listener_commit(MemoryListener *listener) 173 { 174 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 175 struct vhost_dev *dev = v->dev; 176 struct vhost_msg_v2 msg = {}; 177 int fd = v->shared->device_fd; 178 179 if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { 180 return; 181 } 182 183 if (!v->iotlb_batch_begin_sent) { 184 return; 185 } 186 187 msg.type = v->msg_type; 188 msg.iotlb.type = VHOST_IOTLB_BATCH_END; 189 190 trace_vhost_vdpa_listener_commit(v->shared, fd, msg.type, msg.iotlb.type); 191 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 192 error_report("failed to write, fd=%d, errno=%d (%s)", 193 fd, errno, strerror(errno)); 194 } 195 196 v->iotlb_batch_begin_sent = false; 197 } 198 199 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 200 { 201 struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n); 202 203 hwaddr iova = iotlb->iova + iommu->iommu_offset; 204 struct vhost_vdpa *v = iommu->dev; 205 void *vaddr; 206 int ret; 207 Int128 llend; 208 209 if (iotlb->target_as != &address_space_memory) { 210 error_report("Wrong target AS \"%s\", only system memory is allowed", 211 iotlb->target_as->name ? iotlb->target_as->name : "none"); 212 return; 213 } 214 RCU_READ_LOCK_GUARD(); 215 /* check if RAM section out of device range */ 216 llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova)); 217 if (int128_gt(llend, int128_make64(v->shared->iova_range.last))) { 218 error_report("RAM section out of device range (max=0x%" PRIx64 219 ", end addr=0x%" PRIx64 ")", 220 v->shared->iova_range.last, int128_get64(llend)); 221 return; 222 } 223 224 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 225 bool read_only; 226 227 if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) { 228 return; 229 } 230 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 231 iotlb->addr_mask + 1, vaddr, read_only); 232 if (ret) { 233 error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", " 234 "0x%" HWADDR_PRIx ", %p) = %d (%m)", 235 v, iova, iotlb->addr_mask + 1, vaddr, ret); 236 } 237 } else { 238 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 239 iotlb->addr_mask + 1); 240 if (ret) { 241 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 242 "0x%" HWADDR_PRIx ") = %d (%m)", 243 v, iova, iotlb->addr_mask + 1, ret); 244 } 245 } 246 } 247 248 static void vhost_vdpa_iommu_region_add(MemoryListener *listener, 249 MemoryRegionSection *section) 250 { 251 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 252 253 struct vdpa_iommu *iommu; 254 Int128 end; 255 int iommu_idx; 256 IOMMUMemoryRegion *iommu_mr; 257 int ret; 258 259 iommu_mr = IOMMU_MEMORY_REGION(section->mr); 260 261 iommu = g_malloc0(sizeof(*iommu)); 262 end = int128_add(int128_make64(section->offset_within_region), 263 section->size); 264 end = int128_sub(end, int128_one()); 265 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 266 MEMTXATTRS_UNSPECIFIED); 267 iommu->iommu_mr = iommu_mr; 268 iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify, 269 IOMMU_NOTIFIER_IOTLB_EVENTS, 270 section->offset_within_region, 271 int128_get64(end), 272 iommu_idx); 273 iommu->iommu_offset = section->offset_within_address_space - 274 section->offset_within_region; 275 iommu->dev = v; 276 277 ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 278 if (ret) { 279 g_free(iommu); 280 return; 281 } 282 283 QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next); 284 memory_region_iommu_replay(iommu->iommu_mr, &iommu->n); 285 286 return; 287 } 288 289 static void vhost_vdpa_iommu_region_del(MemoryListener *listener, 290 MemoryRegionSection *section) 291 { 292 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 293 294 struct vdpa_iommu *iommu; 295 296 QLIST_FOREACH(iommu, &v->iommu_list, iommu_next) 297 { 298 if (MEMORY_REGION(iommu->iommu_mr) == section->mr && 299 iommu->n.start == section->offset_within_region) { 300 memory_region_unregister_iommu_notifier(section->mr, &iommu->n); 301 QLIST_REMOVE(iommu, iommu_next); 302 g_free(iommu); 303 break; 304 } 305 } 306 } 307 308 static void vhost_vdpa_listener_region_add(MemoryListener *listener, 309 MemoryRegionSection *section) 310 { 311 DMAMap mem_region = {}; 312 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 313 hwaddr iova; 314 Int128 llend, llsize; 315 void *vaddr; 316 int ret; 317 int page_size = qemu_target_page_size(); 318 int page_mask = -page_size; 319 320 if (vhost_vdpa_listener_skipped_section(section, 321 v->shared->iova_range.first, 322 v->shared->iova_range.last, 323 page_mask)) { 324 return; 325 } 326 if (memory_region_is_iommu(section->mr)) { 327 vhost_vdpa_iommu_region_add(listener, section); 328 return; 329 } 330 331 if (unlikely((section->offset_within_address_space & ~page_mask) != 332 (section->offset_within_region & ~page_mask))) { 333 trace_vhost_vdpa_listener_region_add_unaligned(v->shared, 334 section->mr->name, 335 section->offset_within_address_space & ~page_mask, 336 section->offset_within_region & ~page_mask); 337 return; 338 } 339 340 iova = ROUND_UP(section->offset_within_address_space, page_size); 341 llend = vhost_vdpa_section_end(section, page_mask); 342 if (int128_ge(int128_make64(iova), llend)) { 343 return; 344 } 345 346 memory_region_ref(section->mr); 347 348 /* Here we assume that memory_region_is_ram(section->mr)==true */ 349 350 vaddr = memory_region_get_ram_ptr(section->mr) + 351 section->offset_within_region + 352 (iova - section->offset_within_address_space); 353 354 trace_vhost_vdpa_listener_region_add(v->shared, iova, int128_get64(llend), 355 vaddr, section->readonly); 356 357 llsize = int128_sub(llend, int128_make64(iova)); 358 if (v->shared->shadow_data) { 359 int r; 360 361 mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr, 362 mem_region.size = int128_get64(llsize) - 1, 363 mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly), 364 365 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &mem_region); 366 if (unlikely(r != IOVA_OK)) { 367 error_report("Can't allocate a mapping (%d)", r); 368 goto fail; 369 } 370 371 iova = mem_region.iova; 372 } 373 374 vhost_vdpa_iotlb_batch_begin_once(v); 375 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 376 int128_get64(llsize), vaddr, section->readonly); 377 if (ret) { 378 error_report("vhost vdpa map fail!"); 379 goto fail_map; 380 } 381 382 return; 383 384 fail_map: 385 if (v->shared->shadow_data) { 386 vhost_iova_tree_remove(v->shared->iova_tree, mem_region); 387 } 388 389 fail: 390 /* 391 * On the initfn path, store the first error in the container so we 392 * can gracefully fail. Runtime, there's not much we can do other 393 * than throw a hardware error. 394 */ 395 error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 396 return; 397 398 } 399 400 static void vhost_vdpa_listener_region_del(MemoryListener *listener, 401 MemoryRegionSection *section) 402 { 403 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 404 hwaddr iova; 405 Int128 llend, llsize; 406 int ret; 407 int page_size = qemu_target_page_size(); 408 int page_mask = -page_size; 409 410 if (vhost_vdpa_listener_skipped_section(section, 411 v->shared->iova_range.first, 412 v->shared->iova_range.last, 413 page_mask)) { 414 return; 415 } 416 if (memory_region_is_iommu(section->mr)) { 417 vhost_vdpa_iommu_region_del(listener, section); 418 } 419 420 if (unlikely((section->offset_within_address_space & ~page_mask) != 421 (section->offset_within_region & ~page_mask))) { 422 trace_vhost_vdpa_listener_region_del_unaligned(v->shared, 423 section->mr->name, 424 section->offset_within_address_space & ~page_mask, 425 section->offset_within_region & ~page_mask); 426 return; 427 } 428 429 iova = ROUND_UP(section->offset_within_address_space, page_size); 430 llend = vhost_vdpa_section_end(section, page_mask); 431 432 trace_vhost_vdpa_listener_region_del(v->shared, iova, 433 int128_get64(int128_sub(llend, int128_one()))); 434 435 if (int128_ge(int128_make64(iova), llend)) { 436 return; 437 } 438 439 llsize = int128_sub(llend, int128_make64(iova)); 440 441 if (v->shared->shadow_data) { 442 const DMAMap *result; 443 const void *vaddr = memory_region_get_ram_ptr(section->mr) + 444 section->offset_within_region + 445 (iova - section->offset_within_address_space); 446 DMAMap mem_region = { 447 .translated_addr = (hwaddr)(uintptr_t)vaddr, 448 .size = int128_get64(llsize) - 1, 449 }; 450 451 result = vhost_iova_tree_find_iova(v->shared->iova_tree, &mem_region); 452 if (!result) { 453 /* The memory listener map wasn't mapped */ 454 return; 455 } 456 iova = result->iova; 457 vhost_iova_tree_remove(v->shared->iova_tree, *result); 458 } 459 vhost_vdpa_iotlb_batch_begin_once(v); 460 /* 461 * The unmap ioctl doesn't accept a full 64-bit. need to check it 462 */ 463 if (int128_eq(llsize, int128_2_64())) { 464 llsize = int128_rshift(llsize, 1); 465 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 466 int128_get64(llsize)); 467 468 if (ret) { 469 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 470 "0x%" HWADDR_PRIx ") = %d (%m)", 471 v, iova, int128_get64(llsize), ret); 472 } 473 iova += int128_get64(llsize); 474 } 475 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 476 int128_get64(llsize)); 477 478 if (ret) { 479 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 480 "0x%" HWADDR_PRIx ") = %d (%m)", 481 v, iova, int128_get64(llsize), ret); 482 } 483 484 memory_region_unref(section->mr); 485 } 486 /* 487 * IOTLB API is used by vhost-vdpa which requires incremental updating 488 * of the mapping. So we can not use generic vhost memory listener which 489 * depends on the addnop(). 490 */ 491 static const MemoryListener vhost_vdpa_memory_listener = { 492 .name = "vhost-vdpa", 493 .commit = vhost_vdpa_listener_commit, 494 .region_add = vhost_vdpa_listener_region_add, 495 .region_del = vhost_vdpa_listener_region_del, 496 }; 497 498 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 499 void *arg) 500 { 501 struct vhost_vdpa *v = dev->opaque; 502 int fd = v->shared->device_fd; 503 int ret; 504 505 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 506 507 ret = ioctl(fd, request, arg); 508 return ret < 0 ? -errno : ret; 509 } 510 511 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 512 { 513 uint8_t s; 514 int ret; 515 516 trace_vhost_vdpa_add_status(dev, status); 517 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 518 if (ret < 0) { 519 return ret; 520 } 521 if ((s & status) == status) { 522 /* Don't set bits already set */ 523 return 0; 524 } 525 526 s |= status; 527 528 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 529 if (ret < 0) { 530 return ret; 531 } 532 533 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 534 if (ret < 0) { 535 return ret; 536 } 537 538 if (!(s & status)) { 539 return -EIO; 540 } 541 542 return 0; 543 } 544 545 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range) 546 { 547 int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range); 548 549 return ret < 0 ? -errno : 0; 550 } 551 552 /* 553 * The use of this function is for requests that only need to be 554 * applied once. Typically such request occurs at the beginning 555 * of operation, and before setting up queues. It should not be 556 * used for request that performs operation until all queues are 557 * set, which would need to check dev->vq_index_end instead. 558 */ 559 static bool vhost_vdpa_first_dev(struct vhost_dev *dev) 560 { 561 struct vhost_vdpa *v = dev->opaque; 562 563 return v->index == 0; 564 } 565 566 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev, 567 uint64_t *features) 568 { 569 int ret; 570 571 ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 572 trace_vhost_vdpa_get_features(dev, *features); 573 return ret; 574 } 575 576 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v) 577 { 578 g_autoptr(GPtrArray) shadow_vqs = NULL; 579 580 shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free); 581 for (unsigned n = 0; n < hdev->nvqs; ++n) { 582 VhostShadowVirtqueue *svq; 583 584 svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque); 585 g_ptr_array_add(shadow_vqs, svq); 586 } 587 588 v->shadow_vqs = g_steal_pointer(&shadow_vqs); 589 } 590 591 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) 592 { 593 struct vhost_vdpa *v = opaque; 594 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 595 trace_vhost_vdpa_init(dev, v->shared, opaque); 596 int ret; 597 598 v->dev = dev; 599 dev->opaque = opaque ; 600 v->listener = vhost_vdpa_memory_listener; 601 v->msg_type = VHOST_IOTLB_MSG_V2; 602 vhost_vdpa_init_svq(dev, v); 603 604 error_propagate(&dev->migration_blocker, v->migration_blocker); 605 if (!vhost_vdpa_first_dev(dev)) { 606 return 0; 607 } 608 609 /* 610 * If dev->shadow_vqs_enabled at initialization that means the device has 611 * been started with x-svq=on, so don't block migration 612 */ 613 if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { 614 /* We don't have dev->features yet */ 615 uint64_t features; 616 ret = vhost_vdpa_get_dev_features(dev, &features); 617 if (unlikely(ret)) { 618 error_setg_errno(errp, -ret, "Could not get device features"); 619 return ret; 620 } 621 vhost_svq_valid_features(features, &dev->migration_blocker); 622 } 623 624 /* 625 * Similar to VFIO, we end up pinning all guest memory and have to 626 * disable discarding of RAM. 627 */ 628 ret = ram_block_discard_disable(true); 629 if (ret) { 630 error_report("Cannot set discarding of RAM broken"); 631 return ret; 632 } 633 634 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 635 VIRTIO_CONFIG_S_DRIVER); 636 637 return 0; 638 } 639 640 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, 641 int queue_index) 642 { 643 size_t page_size = qemu_real_host_page_size(); 644 struct vhost_vdpa *v = dev->opaque; 645 VirtIODevice *vdev = dev->vdev; 646 VhostVDPAHostNotifier *n; 647 648 n = &v->notifier[queue_index]; 649 650 if (n->addr) { 651 virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); 652 object_unparent(OBJECT(&n->mr)); 653 munmap(n->addr, page_size); 654 n->addr = NULL; 655 } 656 } 657 658 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) 659 { 660 size_t page_size = qemu_real_host_page_size(); 661 struct vhost_vdpa *v = dev->opaque; 662 VirtIODevice *vdev = dev->vdev; 663 VhostVDPAHostNotifier *n; 664 int fd = v->shared->device_fd; 665 void *addr; 666 char *name; 667 668 vhost_vdpa_host_notifier_uninit(dev, queue_index); 669 670 n = &v->notifier[queue_index]; 671 672 addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, 673 queue_index * page_size); 674 if (addr == MAP_FAILED) { 675 goto err; 676 } 677 678 name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", 679 v, queue_index); 680 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 681 page_size, addr); 682 g_free(name); 683 684 if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { 685 object_unparent(OBJECT(&n->mr)); 686 munmap(addr, page_size); 687 goto err; 688 } 689 n->addr = addr; 690 691 return 0; 692 693 err: 694 return -1; 695 } 696 697 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) 698 { 699 int i; 700 701 /* 702 * Pack all the changes to the memory regions in a single 703 * transaction to avoid a few updating of the address space 704 * topology. 705 */ 706 memory_region_transaction_begin(); 707 708 for (i = dev->vq_index; i < dev->vq_index + n; i++) { 709 vhost_vdpa_host_notifier_uninit(dev, i); 710 } 711 712 memory_region_transaction_commit(); 713 } 714 715 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) 716 { 717 struct vhost_vdpa *v = dev->opaque; 718 int i; 719 720 if (v->shadow_vqs_enabled) { 721 /* FIXME SVQ is not compatible with host notifiers mr */ 722 return; 723 } 724 725 /* 726 * Pack all the changes to the memory regions in a single 727 * transaction to avoid a few updating of the address space 728 * topology. 729 */ 730 memory_region_transaction_begin(); 731 732 for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { 733 if (vhost_vdpa_host_notifier_init(dev, i)) { 734 vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index); 735 break; 736 } 737 } 738 739 memory_region_transaction_commit(); 740 } 741 742 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev) 743 { 744 struct vhost_vdpa *v = dev->opaque; 745 size_t idx; 746 747 for (idx = 0; idx < v->shadow_vqs->len; ++idx) { 748 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx)); 749 } 750 g_ptr_array_free(v->shadow_vqs, true); 751 } 752 753 static int vhost_vdpa_cleanup(struct vhost_dev *dev) 754 { 755 struct vhost_vdpa *v; 756 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 757 v = dev->opaque; 758 trace_vhost_vdpa_cleanup(dev, v); 759 if (vhost_vdpa_first_dev(dev)) { 760 ram_block_discard_disable(false); 761 } 762 763 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 764 memory_listener_unregister(&v->listener); 765 vhost_vdpa_svq_cleanup(dev); 766 767 dev->opaque = NULL; 768 769 return 0; 770 } 771 772 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 773 { 774 trace_vhost_vdpa_memslots_limit(dev, INT_MAX); 775 return INT_MAX; 776 } 777 778 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 779 struct vhost_memory *mem) 780 { 781 if (!vhost_vdpa_first_dev(dev)) { 782 return 0; 783 } 784 785 trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding); 786 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) && 787 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) { 788 int i; 789 for (i = 0; i < mem->nregions; i++) { 790 trace_vhost_vdpa_dump_regions(dev, i, 791 mem->regions[i].guest_phys_addr, 792 mem->regions[i].memory_size, 793 mem->regions[i].userspace_addr, 794 mem->regions[i].flags_padding); 795 } 796 } 797 if (mem->padding) { 798 return -EINVAL; 799 } 800 801 return 0; 802 } 803 804 static int vhost_vdpa_set_features(struct vhost_dev *dev, 805 uint64_t features) 806 { 807 struct vhost_vdpa *v = dev->opaque; 808 int ret; 809 810 if (!vhost_vdpa_first_dev(dev)) { 811 return 0; 812 } 813 814 if (v->shadow_vqs_enabled) { 815 if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) { 816 /* 817 * QEMU is just trying to enable or disable logging. SVQ handles 818 * this sepparately, so no need to forward this. 819 */ 820 v->acked_features = features; 821 return 0; 822 } 823 824 v->acked_features = features; 825 826 /* We must not ack _F_LOG if SVQ is enabled */ 827 features &= ~BIT_ULL(VHOST_F_LOG_ALL); 828 } 829 830 trace_vhost_vdpa_set_features(dev, features); 831 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 832 if (ret) { 833 return ret; 834 } 835 836 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 837 } 838 839 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) 840 { 841 uint64_t features; 842 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 843 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | 844 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | 845 0x1ULL << VHOST_BACKEND_F_SUSPEND; 846 int r; 847 848 if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { 849 return -EFAULT; 850 } 851 852 features &= f; 853 854 if (vhost_vdpa_first_dev(dev)) { 855 r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); 856 if (r) { 857 return -EFAULT; 858 } 859 } 860 861 dev->backend_cap = features; 862 863 return 0; 864 } 865 866 static int vhost_vdpa_get_device_id(struct vhost_dev *dev, 867 uint32_t *device_id) 868 { 869 int ret; 870 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 871 trace_vhost_vdpa_get_device_id(dev, *device_id); 872 return ret; 873 } 874 875 static int vhost_vdpa_reset_device(struct vhost_dev *dev) 876 { 877 struct vhost_vdpa *v = dev->opaque; 878 int ret; 879 uint8_t status = 0; 880 881 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 882 trace_vhost_vdpa_reset_device(dev); 883 v->suspended = false; 884 return ret; 885 } 886 887 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 888 { 889 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 890 891 trace_vhost_vdpa_get_vq_index(dev, idx, idx); 892 return idx; 893 } 894 895 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx) 896 { 897 struct vhost_dev *dev = v->dev; 898 struct vhost_vring_state state = { 899 .index = idx, 900 .num = 1, 901 }; 902 int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 903 904 trace_vhost_vdpa_set_vring_ready(dev, idx, r); 905 return r; 906 } 907 908 static int vhost_vdpa_set_config_call(struct vhost_dev *dev, 909 int fd) 910 { 911 trace_vhost_vdpa_set_config_call(dev, fd); 912 return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd); 913 } 914 915 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config, 916 uint32_t config_len) 917 { 918 int b, len; 919 char line[QEMU_HEXDUMP_LINE_LEN]; 920 921 for (b = 0; b < config_len; b += 16) { 922 len = config_len - b; 923 qemu_hexdump_line(line, b, config, len, false); 924 trace_vhost_vdpa_dump_config(dev, line); 925 } 926 } 927 928 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 929 uint32_t offset, uint32_t size, 930 uint32_t flags) 931 { 932 struct vhost_vdpa_config *config; 933 int ret; 934 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 935 936 trace_vhost_vdpa_set_config(dev, offset, size, flags); 937 config = g_malloc(size + config_size); 938 config->off = offset; 939 config->len = size; 940 memcpy(config->buf, data, size); 941 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) && 942 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 943 vhost_vdpa_dump_config(dev, data, size); 944 } 945 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 946 g_free(config); 947 return ret; 948 } 949 950 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 951 uint32_t config_len, Error **errp) 952 { 953 struct vhost_vdpa_config *v_config; 954 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 955 int ret; 956 957 trace_vhost_vdpa_get_config(dev, config, config_len); 958 v_config = g_malloc(config_len + config_size); 959 v_config->len = config_len; 960 v_config->off = 0; 961 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 962 memcpy(config, v_config->buf, config_len); 963 g_free(v_config); 964 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) && 965 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 966 vhost_vdpa_dump_config(dev, config, config_len); 967 } 968 return ret; 969 } 970 971 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev, 972 struct vhost_vring_state *ring) 973 { 974 trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num); 975 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 976 } 977 978 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev, 979 struct vhost_vring_file *file) 980 { 981 trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd); 982 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 983 } 984 985 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev, 986 struct vhost_vring_file *file) 987 { 988 trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd); 989 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 990 } 991 992 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev, 993 struct vhost_vring_addr *addr) 994 { 995 trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags, 996 addr->desc_user_addr, addr->used_user_addr, 997 addr->avail_user_addr, 998 addr->log_guest_addr); 999 1000 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 1001 1002 } 1003 1004 /** 1005 * Set the shadow virtqueue descriptors to the device 1006 * 1007 * @dev: The vhost device model 1008 * @svq: The shadow virtqueue 1009 * @idx: The index of the virtqueue in the vhost device 1010 * @errp: Error 1011 * 1012 * Note that this function does not rewind kick file descriptor if cannot set 1013 * call one. 1014 */ 1015 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev, 1016 VhostShadowVirtqueue *svq, unsigned idx, 1017 Error **errp) 1018 { 1019 struct vhost_vring_file file = { 1020 .index = dev->vq_index + idx, 1021 }; 1022 const EventNotifier *event_notifier = &svq->hdev_kick; 1023 int r; 1024 1025 r = event_notifier_init(&svq->hdev_kick, 0); 1026 if (r != 0) { 1027 error_setg_errno(errp, -r, "Couldn't create kick event notifier"); 1028 goto err_init_hdev_kick; 1029 } 1030 1031 r = event_notifier_init(&svq->hdev_call, 0); 1032 if (r != 0) { 1033 error_setg_errno(errp, -r, "Couldn't create call event notifier"); 1034 goto err_init_hdev_call; 1035 } 1036 1037 file.fd = event_notifier_get_fd(event_notifier); 1038 r = vhost_vdpa_set_vring_dev_kick(dev, &file); 1039 if (unlikely(r != 0)) { 1040 error_setg_errno(errp, -r, "Can't set device kick fd"); 1041 goto err_init_set_dev_fd; 1042 } 1043 1044 event_notifier = &svq->hdev_call; 1045 file.fd = event_notifier_get_fd(event_notifier); 1046 r = vhost_vdpa_set_vring_dev_call(dev, &file); 1047 if (unlikely(r != 0)) { 1048 error_setg_errno(errp, -r, "Can't set device call fd"); 1049 goto err_init_set_dev_fd; 1050 } 1051 1052 return 0; 1053 1054 err_init_set_dev_fd: 1055 event_notifier_set_handler(&svq->hdev_call, NULL); 1056 1057 err_init_hdev_call: 1058 event_notifier_cleanup(&svq->hdev_kick); 1059 1060 err_init_hdev_kick: 1061 return r; 1062 } 1063 1064 /** 1065 * Unmap a SVQ area in the device 1066 */ 1067 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr) 1068 { 1069 const DMAMap needle = { 1070 .translated_addr = addr, 1071 }; 1072 const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree, 1073 &needle); 1074 hwaddr size; 1075 int r; 1076 1077 if (unlikely(!result)) { 1078 error_report("Unable to find SVQ address to unmap"); 1079 return; 1080 } 1081 1082 size = ROUND_UP(result->size, qemu_real_host_page_size()); 1083 r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size); 1084 if (unlikely(r < 0)) { 1085 error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r); 1086 return; 1087 } 1088 1089 vhost_iova_tree_remove(v->shared->iova_tree, *result); 1090 } 1091 1092 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev, 1093 const VhostShadowVirtqueue *svq) 1094 { 1095 struct vhost_vdpa *v = dev->opaque; 1096 struct vhost_vring_addr svq_addr; 1097 1098 vhost_svq_get_vring_addr(svq, &svq_addr); 1099 1100 vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr); 1101 1102 vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr); 1103 } 1104 1105 /** 1106 * Map the SVQ area in the device 1107 * 1108 * @v: Vhost-vdpa device 1109 * @needle: The area to search iova 1110 * @errorp: Error pointer 1111 */ 1112 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle, 1113 Error **errp) 1114 { 1115 int r; 1116 1117 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle); 1118 if (unlikely(r != IOVA_OK)) { 1119 error_setg(errp, "Cannot allocate iova (%d)", r); 1120 return false; 1121 } 1122 1123 r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova, 1124 needle->size + 1, 1125 (void *)(uintptr_t)needle->translated_addr, 1126 needle->perm == IOMMU_RO); 1127 if (unlikely(r != 0)) { 1128 error_setg_errno(errp, -r, "Cannot map region to device"); 1129 vhost_iova_tree_remove(v->shared->iova_tree, *needle); 1130 } 1131 1132 return r == 0; 1133 } 1134 1135 /** 1136 * Map the shadow virtqueue rings in the device 1137 * 1138 * @dev: The vhost device 1139 * @svq: The shadow virtqueue 1140 * @addr: Assigned IOVA addresses 1141 * @errp: Error pointer 1142 */ 1143 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev, 1144 const VhostShadowVirtqueue *svq, 1145 struct vhost_vring_addr *addr, 1146 Error **errp) 1147 { 1148 ERRP_GUARD(); 1149 DMAMap device_region, driver_region; 1150 struct vhost_vring_addr svq_addr; 1151 struct vhost_vdpa *v = dev->opaque; 1152 size_t device_size = vhost_svq_device_area_size(svq); 1153 size_t driver_size = vhost_svq_driver_area_size(svq); 1154 size_t avail_offset; 1155 bool ok; 1156 1157 vhost_svq_get_vring_addr(svq, &svq_addr); 1158 1159 driver_region = (DMAMap) { 1160 .translated_addr = svq_addr.desc_user_addr, 1161 .size = driver_size - 1, 1162 .perm = IOMMU_RO, 1163 }; 1164 ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp); 1165 if (unlikely(!ok)) { 1166 error_prepend(errp, "Cannot create vq driver region: "); 1167 return false; 1168 } 1169 addr->desc_user_addr = driver_region.iova; 1170 avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr; 1171 addr->avail_user_addr = driver_region.iova + avail_offset; 1172 1173 device_region = (DMAMap) { 1174 .translated_addr = svq_addr.used_user_addr, 1175 .size = device_size - 1, 1176 .perm = IOMMU_RW, 1177 }; 1178 ok = vhost_vdpa_svq_map_ring(v, &device_region, errp); 1179 if (unlikely(!ok)) { 1180 error_prepend(errp, "Cannot create vq device region: "); 1181 vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr); 1182 } 1183 addr->used_user_addr = device_region.iova; 1184 1185 return ok; 1186 } 1187 1188 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev, 1189 VhostShadowVirtqueue *svq, unsigned idx, 1190 Error **errp) 1191 { 1192 uint16_t vq_index = dev->vq_index + idx; 1193 struct vhost_vring_state s = { 1194 .index = vq_index, 1195 }; 1196 int r; 1197 1198 r = vhost_vdpa_set_dev_vring_base(dev, &s); 1199 if (unlikely(r)) { 1200 error_setg_errno(errp, -r, "Cannot set vring base"); 1201 return false; 1202 } 1203 1204 r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp); 1205 return r == 0; 1206 } 1207 1208 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev) 1209 { 1210 struct vhost_vdpa *v = dev->opaque; 1211 Error *err = NULL; 1212 unsigned i; 1213 1214 if (!v->shadow_vqs_enabled) { 1215 return true; 1216 } 1217 1218 for (i = 0; i < v->shadow_vqs->len; ++i) { 1219 VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i); 1220 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1221 struct vhost_vring_addr addr = { 1222 .index = dev->vq_index + i, 1223 }; 1224 int r; 1225 bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err); 1226 if (unlikely(!ok)) { 1227 goto err; 1228 } 1229 1230 vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree); 1231 ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err); 1232 if (unlikely(!ok)) { 1233 goto err_map; 1234 } 1235 1236 /* Override vring GPA set by vhost subsystem */ 1237 r = vhost_vdpa_set_vring_dev_addr(dev, &addr); 1238 if (unlikely(r != 0)) { 1239 error_setg_errno(&err, -r, "Cannot set device address"); 1240 goto err_set_addr; 1241 } 1242 } 1243 1244 return true; 1245 1246 err_set_addr: 1247 vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i)); 1248 1249 err_map: 1250 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i)); 1251 1252 err: 1253 error_reportf_err(err, "Cannot setup SVQ %u: ", i); 1254 for (unsigned j = 0; j < i; ++j) { 1255 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j); 1256 vhost_vdpa_svq_unmap_rings(dev, svq); 1257 vhost_svq_stop(svq); 1258 } 1259 1260 return false; 1261 } 1262 1263 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) 1264 { 1265 struct vhost_vdpa *v = dev->opaque; 1266 1267 if (!v->shadow_vqs_enabled) { 1268 return; 1269 } 1270 1271 for (unsigned i = 0; i < v->shadow_vqs->len; ++i) { 1272 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1273 1274 vhost_svq_stop(svq); 1275 vhost_vdpa_svq_unmap_rings(dev, svq); 1276 1277 event_notifier_cleanup(&svq->hdev_kick); 1278 event_notifier_cleanup(&svq->hdev_call); 1279 } 1280 } 1281 1282 static void vhost_vdpa_suspend(struct vhost_dev *dev) 1283 { 1284 struct vhost_vdpa *v = dev->opaque; 1285 int r; 1286 1287 if (!vhost_vdpa_first_dev(dev)) { 1288 return; 1289 } 1290 1291 if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { 1292 trace_vhost_vdpa_suspend(dev); 1293 r = ioctl(v->shared->device_fd, VHOST_VDPA_SUSPEND); 1294 if (unlikely(r)) { 1295 error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); 1296 } else { 1297 v->suspended = true; 1298 return; 1299 } 1300 } 1301 1302 vhost_vdpa_reset_device(dev); 1303 } 1304 1305 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 1306 { 1307 struct vhost_vdpa *v = dev->opaque; 1308 bool ok; 1309 trace_vhost_vdpa_dev_start(dev, started); 1310 1311 if (started) { 1312 vhost_vdpa_host_notifiers_init(dev); 1313 ok = vhost_vdpa_svqs_start(dev); 1314 if (unlikely(!ok)) { 1315 return -1; 1316 } 1317 } else { 1318 vhost_vdpa_suspend(dev); 1319 vhost_vdpa_svqs_stop(dev); 1320 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 1321 } 1322 1323 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1324 return 0; 1325 } 1326 1327 if (started) { 1328 if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) { 1329 error_report("SVQ can not work while IOMMU enable, please disable" 1330 "IOMMU and try again"); 1331 return -1; 1332 } 1333 memory_listener_register(&v->listener, dev->vdev->dma_as); 1334 1335 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 1336 } 1337 1338 return 0; 1339 } 1340 1341 static void vhost_vdpa_reset_status(struct vhost_dev *dev) 1342 { 1343 struct vhost_vdpa *v = dev->opaque; 1344 1345 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1346 return; 1347 } 1348 1349 vhost_vdpa_reset_device(dev); 1350 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 1351 VIRTIO_CONFIG_S_DRIVER); 1352 memory_listener_unregister(&v->listener); 1353 } 1354 1355 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 1356 struct vhost_log *log) 1357 { 1358 struct vhost_vdpa *v = dev->opaque; 1359 if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) { 1360 return 0; 1361 } 1362 1363 trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd, 1364 log->log); 1365 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 1366 } 1367 1368 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 1369 struct vhost_vring_addr *addr) 1370 { 1371 struct vhost_vdpa *v = dev->opaque; 1372 1373 if (v->shadow_vqs_enabled) { 1374 /* 1375 * Device vring addr was set at device start. SVQ base is handled by 1376 * VirtQueue code. 1377 */ 1378 return 0; 1379 } 1380 1381 return vhost_vdpa_set_vring_dev_addr(dev, addr); 1382 } 1383 1384 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 1385 struct vhost_vring_state *ring) 1386 { 1387 trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num); 1388 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 1389 } 1390 1391 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 1392 struct vhost_vring_state *ring) 1393 { 1394 struct vhost_vdpa *v = dev->opaque; 1395 1396 if (v->shadow_vqs_enabled) { 1397 /* 1398 * Device vring base was set at device start. SVQ base is handled by 1399 * VirtQueue code. 1400 */ 1401 return 0; 1402 } 1403 1404 return vhost_vdpa_set_dev_vring_base(dev, ring); 1405 } 1406 1407 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 1408 struct vhost_vring_state *ring) 1409 { 1410 struct vhost_vdpa *v = dev->opaque; 1411 int ret; 1412 1413 if (v->shadow_vqs_enabled) { 1414 ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index); 1415 return 0; 1416 } 1417 1418 if (!v->suspended) { 1419 /* 1420 * Cannot trust in value returned by device, let vhost recover used 1421 * idx from guest. 1422 */ 1423 return -1; 1424 } 1425 1426 ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 1427 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); 1428 return ret; 1429 } 1430 1431 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 1432 struct vhost_vring_file *file) 1433 { 1434 struct vhost_vdpa *v = dev->opaque; 1435 int vdpa_idx = file->index - dev->vq_index; 1436 1437 if (v->shadow_vqs_enabled) { 1438 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1439 vhost_svq_set_svq_kick_fd(svq, file->fd); 1440 return 0; 1441 } else { 1442 return vhost_vdpa_set_vring_dev_kick(dev, file); 1443 } 1444 } 1445 1446 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 1447 struct vhost_vring_file *file) 1448 { 1449 struct vhost_vdpa *v = dev->opaque; 1450 int vdpa_idx = file->index - dev->vq_index; 1451 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1452 1453 /* Remember last call fd because we can switch to SVQ anytime. */ 1454 vhost_svq_set_svq_call_fd(svq, file->fd); 1455 if (v->shadow_vqs_enabled) { 1456 return 0; 1457 } 1458 1459 return vhost_vdpa_set_vring_dev_call(dev, file); 1460 } 1461 1462 static int vhost_vdpa_get_features(struct vhost_dev *dev, 1463 uint64_t *features) 1464 { 1465 int ret = vhost_vdpa_get_dev_features(dev, features); 1466 1467 if (ret == 0) { 1468 /* Add SVQ logging capabilities */ 1469 *features |= BIT_ULL(VHOST_F_LOG_ALL); 1470 } 1471 1472 return ret; 1473 } 1474 1475 static int vhost_vdpa_set_owner(struct vhost_dev *dev) 1476 { 1477 if (!vhost_vdpa_first_dev(dev)) { 1478 return 0; 1479 } 1480 1481 trace_vhost_vdpa_set_owner(dev); 1482 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 1483 } 1484 1485 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 1486 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 1487 { 1488 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 1489 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 1490 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 1491 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 1492 trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, 1493 addr->avail_user_addr, addr->used_user_addr); 1494 return 0; 1495 } 1496 1497 static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 1498 { 1499 return true; 1500 } 1501 1502 const VhostOps vdpa_ops = { 1503 .backend_type = VHOST_BACKEND_TYPE_VDPA, 1504 .vhost_backend_init = vhost_vdpa_init, 1505 .vhost_backend_cleanup = vhost_vdpa_cleanup, 1506 .vhost_set_log_base = vhost_vdpa_set_log_base, 1507 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 1508 .vhost_set_vring_num = vhost_vdpa_set_vring_num, 1509 .vhost_set_vring_base = vhost_vdpa_set_vring_base, 1510 .vhost_get_vring_base = vhost_vdpa_get_vring_base, 1511 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 1512 .vhost_set_vring_call = vhost_vdpa_set_vring_call, 1513 .vhost_get_features = vhost_vdpa_get_features, 1514 .vhost_set_backend_cap = vhost_vdpa_set_backend_cap, 1515 .vhost_set_owner = vhost_vdpa_set_owner, 1516 .vhost_set_vring_endian = NULL, 1517 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 1518 .vhost_set_mem_table = vhost_vdpa_set_mem_table, 1519 .vhost_set_features = vhost_vdpa_set_features, 1520 .vhost_reset_device = vhost_vdpa_reset_device, 1521 .vhost_get_vq_index = vhost_vdpa_get_vq_index, 1522 .vhost_get_config = vhost_vdpa_get_config, 1523 .vhost_set_config = vhost_vdpa_set_config, 1524 .vhost_requires_shm_log = NULL, 1525 .vhost_migration_done = NULL, 1526 .vhost_net_set_mtu = NULL, 1527 .vhost_set_iotlb_callback = NULL, 1528 .vhost_send_device_iotlb_msg = NULL, 1529 .vhost_dev_start = vhost_vdpa_dev_start, 1530 .vhost_get_device_id = vhost_vdpa_get_device_id, 1531 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 1532 .vhost_force_iommu = vhost_vdpa_force_iommu, 1533 .vhost_set_config_call = vhost_vdpa_set_config_call, 1534 .vhost_reset_status = vhost_vdpa_reset_status, 1535 }; 1536