1 /* 2 * vhost-vdpa 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <linux/vhost.h> 14 #include <linux/vfio.h> 15 #include <sys/eventfd.h> 16 #include <sys/ioctl.h> 17 #include "exec/target_page.h" 18 #include "hw/virtio/vhost.h" 19 #include "hw/virtio/vhost-backend.h" 20 #include "hw/virtio/virtio-net.h" 21 #include "hw/virtio/vhost-shadow-virtqueue.h" 22 #include "hw/virtio/vhost-vdpa.h" 23 #include "exec/address-spaces.h" 24 #include "migration/blocker.h" 25 #include "qemu/cutils.h" 26 #include "qemu/main-loop.h" 27 #include "trace.h" 28 #include "qapi/error.h" 29 30 /* 31 * Return one past the end of the end of section. Be careful with uint64_t 32 * conversions! 33 */ 34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section, 35 int page_mask) 36 { 37 Int128 llend = int128_make64(section->offset_within_address_space); 38 llend = int128_add(llend, section->size); 39 llend = int128_and(llend, int128_exts64(page_mask)); 40 41 return llend; 42 } 43 44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section, 45 uint64_t iova_min, 46 uint64_t iova_max, 47 int page_mask) 48 { 49 Int128 llend; 50 51 if ((!memory_region_is_ram(section->mr) && 52 !memory_region_is_iommu(section->mr)) || 53 memory_region_is_protected(section->mr) || 54 /* vhost-vDPA doesn't allow MMIO to be mapped */ 55 memory_region_is_ram_device(section->mr)) { 56 return true; 57 } 58 59 if (section->offset_within_address_space < iova_min) { 60 error_report("RAM section out of device range (min=0x%" PRIx64 61 ", addr=0x%" HWADDR_PRIx ")", 62 iova_min, section->offset_within_address_space); 63 return true; 64 } 65 /* 66 * While using vIOMMU, sometimes the section will be larger than iova_max, 67 * but the memory that actually maps is smaller, so move the check to 68 * function vhost_vdpa_iommu_map_notify(). That function will use the actual 69 * size that maps to the kernel 70 */ 71 72 if (!memory_region_is_iommu(section->mr)) { 73 llend = vhost_vdpa_section_end(section, page_mask); 74 if (int128_gt(llend, int128_make64(iova_max))) { 75 error_report("RAM section out of device range (max=0x%" PRIx64 76 ", end addr=0x%" PRIx64 ")", 77 iova_max, int128_get64(llend)); 78 return true; 79 } 80 } 81 82 return false; 83 } 84 85 /* 86 * The caller must set asid = 0 if the device does not support asid. 87 * This is not an ABI break since it is set to 0 by the initializer anyway. 88 */ 89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 90 hwaddr size, void *vaddr, bool readonly) 91 { 92 struct vhost_msg_v2 msg = {}; 93 int fd = v->device_fd; 94 int ret = 0; 95 96 msg.type = v->msg_type; 97 msg.asid = asid; 98 msg.iotlb.iova = iova; 99 msg.iotlb.size = size; 100 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 101 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 102 msg.iotlb.type = VHOST_IOTLB_UPDATE; 103 104 trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.asid, msg.iotlb.iova, 105 msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm, 106 msg.iotlb.type); 107 108 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 109 error_report("failed to write, fd=%d, errno=%d (%s)", 110 fd, errno, strerror(errno)); 111 return -EIO ; 112 } 113 114 return ret; 115 } 116 117 /* 118 * The caller must set asid = 0 if the device does not support asid. 119 * This is not an ABI break since it is set to 0 by the initializer anyway. 120 */ 121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 122 hwaddr size) 123 { 124 struct vhost_msg_v2 msg = {}; 125 int fd = v->device_fd; 126 int ret = 0; 127 128 msg.type = v->msg_type; 129 msg.asid = asid; 130 msg.iotlb.iova = iova; 131 msg.iotlb.size = size; 132 msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 133 134 trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.asid, msg.iotlb.iova, 135 msg.iotlb.size, msg.iotlb.type); 136 137 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 138 error_report("failed to write, fd=%d, errno=%d (%s)", 139 fd, errno, strerror(errno)); 140 return -EIO ; 141 } 142 143 return ret; 144 } 145 146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v) 147 { 148 int fd = v->device_fd; 149 struct vhost_msg_v2 msg = { 150 .type = v->msg_type, 151 .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, 152 }; 153 154 trace_vhost_vdpa_listener_begin_batch(v, fd, msg.type, msg.iotlb.type); 155 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 156 error_report("failed to write, fd=%d, errno=%d (%s)", 157 fd, errno, strerror(errno)); 158 } 159 } 160 161 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v) 162 { 163 if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && 164 !v->iotlb_batch_begin_sent) { 165 vhost_vdpa_listener_begin_batch(v); 166 } 167 168 v->iotlb_batch_begin_sent = true; 169 } 170 171 static void vhost_vdpa_listener_commit(MemoryListener *listener) 172 { 173 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 174 struct vhost_dev *dev = v->dev; 175 struct vhost_msg_v2 msg = {}; 176 int fd = v->device_fd; 177 178 if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { 179 return; 180 } 181 182 if (!v->iotlb_batch_begin_sent) { 183 return; 184 } 185 186 msg.type = v->msg_type; 187 msg.iotlb.type = VHOST_IOTLB_BATCH_END; 188 189 trace_vhost_vdpa_listener_commit(v, fd, msg.type, msg.iotlb.type); 190 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 191 error_report("failed to write, fd=%d, errno=%d (%s)", 192 fd, errno, strerror(errno)); 193 } 194 195 v->iotlb_batch_begin_sent = false; 196 } 197 198 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 199 { 200 struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n); 201 202 hwaddr iova = iotlb->iova + iommu->iommu_offset; 203 struct vhost_vdpa *v = iommu->dev; 204 void *vaddr; 205 int ret; 206 Int128 llend; 207 208 if (iotlb->target_as != &address_space_memory) { 209 error_report("Wrong target AS \"%s\", only system memory is allowed", 210 iotlb->target_as->name ? iotlb->target_as->name : "none"); 211 return; 212 } 213 RCU_READ_LOCK_GUARD(); 214 /* check if RAM section out of device range */ 215 llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova)); 216 if (int128_gt(llend, int128_make64(v->shared->iova_range.last))) { 217 error_report("RAM section out of device range (max=0x%" PRIx64 218 ", end addr=0x%" PRIx64 ")", 219 v->shared->iova_range.last, int128_get64(llend)); 220 return; 221 } 222 223 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 224 bool read_only; 225 226 if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) { 227 return; 228 } 229 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 230 iotlb->addr_mask + 1, vaddr, read_only); 231 if (ret) { 232 error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", " 233 "0x%" HWADDR_PRIx ", %p) = %d (%m)", 234 v, iova, iotlb->addr_mask + 1, vaddr, ret); 235 } 236 } else { 237 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 238 iotlb->addr_mask + 1); 239 if (ret) { 240 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 241 "0x%" HWADDR_PRIx ") = %d (%m)", 242 v, iova, iotlb->addr_mask + 1, ret); 243 } 244 } 245 } 246 247 static void vhost_vdpa_iommu_region_add(MemoryListener *listener, 248 MemoryRegionSection *section) 249 { 250 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 251 252 struct vdpa_iommu *iommu; 253 Int128 end; 254 int iommu_idx; 255 IOMMUMemoryRegion *iommu_mr; 256 int ret; 257 258 iommu_mr = IOMMU_MEMORY_REGION(section->mr); 259 260 iommu = g_malloc0(sizeof(*iommu)); 261 end = int128_add(int128_make64(section->offset_within_region), 262 section->size); 263 end = int128_sub(end, int128_one()); 264 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 265 MEMTXATTRS_UNSPECIFIED); 266 iommu->iommu_mr = iommu_mr; 267 iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify, 268 IOMMU_NOTIFIER_IOTLB_EVENTS, 269 section->offset_within_region, 270 int128_get64(end), 271 iommu_idx); 272 iommu->iommu_offset = section->offset_within_address_space - 273 section->offset_within_region; 274 iommu->dev = v; 275 276 ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 277 if (ret) { 278 g_free(iommu); 279 return; 280 } 281 282 QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next); 283 memory_region_iommu_replay(iommu->iommu_mr, &iommu->n); 284 285 return; 286 } 287 288 static void vhost_vdpa_iommu_region_del(MemoryListener *listener, 289 MemoryRegionSection *section) 290 { 291 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 292 293 struct vdpa_iommu *iommu; 294 295 QLIST_FOREACH(iommu, &v->iommu_list, iommu_next) 296 { 297 if (MEMORY_REGION(iommu->iommu_mr) == section->mr && 298 iommu->n.start == section->offset_within_region) { 299 memory_region_unregister_iommu_notifier(section->mr, &iommu->n); 300 QLIST_REMOVE(iommu, iommu_next); 301 g_free(iommu); 302 break; 303 } 304 } 305 } 306 307 static void vhost_vdpa_listener_region_add(MemoryListener *listener, 308 MemoryRegionSection *section) 309 { 310 DMAMap mem_region = {}; 311 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 312 hwaddr iova; 313 Int128 llend, llsize; 314 void *vaddr; 315 int ret; 316 int page_size = qemu_target_page_size(); 317 int page_mask = -page_size; 318 319 if (vhost_vdpa_listener_skipped_section(section, 320 v->shared->iova_range.first, 321 v->shared->iova_range.last, 322 page_mask)) { 323 return; 324 } 325 if (memory_region_is_iommu(section->mr)) { 326 vhost_vdpa_iommu_region_add(listener, section); 327 return; 328 } 329 330 if (unlikely((section->offset_within_address_space & ~page_mask) != 331 (section->offset_within_region & ~page_mask))) { 332 trace_vhost_vdpa_listener_region_add_unaligned(v, section->mr->name, 333 section->offset_within_address_space & ~page_mask, 334 section->offset_within_region & ~page_mask); 335 return; 336 } 337 338 iova = ROUND_UP(section->offset_within_address_space, page_size); 339 llend = vhost_vdpa_section_end(section, page_mask); 340 if (int128_ge(int128_make64(iova), llend)) { 341 return; 342 } 343 344 memory_region_ref(section->mr); 345 346 /* Here we assume that memory_region_is_ram(section->mr)==true */ 347 348 vaddr = memory_region_get_ram_ptr(section->mr) + 349 section->offset_within_region + 350 (iova - section->offset_within_address_space); 351 352 trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend), 353 vaddr, section->readonly); 354 355 llsize = int128_sub(llend, int128_make64(iova)); 356 if (v->shadow_data) { 357 int r; 358 359 mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr, 360 mem_region.size = int128_get64(llsize) - 1, 361 mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly), 362 363 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &mem_region); 364 if (unlikely(r != IOVA_OK)) { 365 error_report("Can't allocate a mapping (%d)", r); 366 goto fail; 367 } 368 369 iova = mem_region.iova; 370 } 371 372 vhost_vdpa_iotlb_batch_begin_once(v); 373 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 374 int128_get64(llsize), vaddr, section->readonly); 375 if (ret) { 376 error_report("vhost vdpa map fail!"); 377 goto fail_map; 378 } 379 380 return; 381 382 fail_map: 383 if (v->shadow_data) { 384 vhost_iova_tree_remove(v->shared->iova_tree, mem_region); 385 } 386 387 fail: 388 /* 389 * On the initfn path, store the first error in the container so we 390 * can gracefully fail. Runtime, there's not much we can do other 391 * than throw a hardware error. 392 */ 393 error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 394 return; 395 396 } 397 398 static void vhost_vdpa_listener_region_del(MemoryListener *listener, 399 MemoryRegionSection *section) 400 { 401 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 402 hwaddr iova; 403 Int128 llend, llsize; 404 int ret; 405 int page_size = qemu_target_page_size(); 406 int page_mask = -page_size; 407 408 if (vhost_vdpa_listener_skipped_section(section, 409 v->shared->iova_range.first, 410 v->shared->iova_range.last, 411 page_mask)) { 412 return; 413 } 414 if (memory_region_is_iommu(section->mr)) { 415 vhost_vdpa_iommu_region_del(listener, section); 416 } 417 418 if (unlikely((section->offset_within_address_space & ~page_mask) != 419 (section->offset_within_region & ~page_mask))) { 420 trace_vhost_vdpa_listener_region_del_unaligned(v, section->mr->name, 421 section->offset_within_address_space & ~page_mask, 422 section->offset_within_region & ~page_mask); 423 return; 424 } 425 426 iova = ROUND_UP(section->offset_within_address_space, page_size); 427 llend = vhost_vdpa_section_end(section, page_mask); 428 429 trace_vhost_vdpa_listener_region_del(v, iova, 430 int128_get64(int128_sub(llend, int128_one()))); 431 432 if (int128_ge(int128_make64(iova), llend)) { 433 return; 434 } 435 436 llsize = int128_sub(llend, int128_make64(iova)); 437 438 if (v->shadow_data) { 439 const DMAMap *result; 440 const void *vaddr = memory_region_get_ram_ptr(section->mr) + 441 section->offset_within_region + 442 (iova - section->offset_within_address_space); 443 DMAMap mem_region = { 444 .translated_addr = (hwaddr)(uintptr_t)vaddr, 445 .size = int128_get64(llsize) - 1, 446 }; 447 448 result = vhost_iova_tree_find_iova(v->shared->iova_tree, &mem_region); 449 if (!result) { 450 /* The memory listener map wasn't mapped */ 451 return; 452 } 453 iova = result->iova; 454 vhost_iova_tree_remove(v->shared->iova_tree, *result); 455 } 456 vhost_vdpa_iotlb_batch_begin_once(v); 457 /* 458 * The unmap ioctl doesn't accept a full 64-bit. need to check it 459 */ 460 if (int128_eq(llsize, int128_2_64())) { 461 llsize = int128_rshift(llsize, 1); 462 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 463 int128_get64(llsize)); 464 465 if (ret) { 466 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 467 "0x%" HWADDR_PRIx ") = %d (%m)", 468 v, iova, int128_get64(llsize), ret); 469 } 470 iova += int128_get64(llsize); 471 } 472 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 473 int128_get64(llsize)); 474 475 if (ret) { 476 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 477 "0x%" HWADDR_PRIx ") = %d (%m)", 478 v, iova, int128_get64(llsize), ret); 479 } 480 481 memory_region_unref(section->mr); 482 } 483 /* 484 * IOTLB API is used by vhost-vdpa which requires incremental updating 485 * of the mapping. So we can not use generic vhost memory listener which 486 * depends on the addnop(). 487 */ 488 static const MemoryListener vhost_vdpa_memory_listener = { 489 .name = "vhost-vdpa", 490 .commit = vhost_vdpa_listener_commit, 491 .region_add = vhost_vdpa_listener_region_add, 492 .region_del = vhost_vdpa_listener_region_del, 493 }; 494 495 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 496 void *arg) 497 { 498 struct vhost_vdpa *v = dev->opaque; 499 int fd = v->device_fd; 500 int ret; 501 502 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 503 504 ret = ioctl(fd, request, arg); 505 return ret < 0 ? -errno : ret; 506 } 507 508 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 509 { 510 uint8_t s; 511 int ret; 512 513 trace_vhost_vdpa_add_status(dev, status); 514 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 515 if (ret < 0) { 516 return ret; 517 } 518 if ((s & status) == status) { 519 /* Don't set bits already set */ 520 return 0; 521 } 522 523 s |= status; 524 525 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 526 if (ret < 0) { 527 return ret; 528 } 529 530 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 531 if (ret < 0) { 532 return ret; 533 } 534 535 if (!(s & status)) { 536 return -EIO; 537 } 538 539 return 0; 540 } 541 542 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range) 543 { 544 int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range); 545 546 return ret < 0 ? -errno : 0; 547 } 548 549 /* 550 * The use of this function is for requests that only need to be 551 * applied once. Typically such request occurs at the beginning 552 * of operation, and before setting up queues. It should not be 553 * used for request that performs operation until all queues are 554 * set, which would need to check dev->vq_index_end instead. 555 */ 556 static bool vhost_vdpa_first_dev(struct vhost_dev *dev) 557 { 558 struct vhost_vdpa *v = dev->opaque; 559 560 return v->index == 0; 561 } 562 563 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev, 564 uint64_t *features) 565 { 566 int ret; 567 568 ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 569 trace_vhost_vdpa_get_features(dev, *features); 570 return ret; 571 } 572 573 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v) 574 { 575 g_autoptr(GPtrArray) shadow_vqs = NULL; 576 577 shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free); 578 for (unsigned n = 0; n < hdev->nvqs; ++n) { 579 VhostShadowVirtqueue *svq; 580 581 svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque); 582 g_ptr_array_add(shadow_vqs, svq); 583 } 584 585 v->shadow_vqs = g_steal_pointer(&shadow_vqs); 586 } 587 588 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) 589 { 590 struct vhost_vdpa *v; 591 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 592 trace_vhost_vdpa_init(dev, opaque); 593 int ret; 594 595 v = opaque; 596 v->dev = dev; 597 dev->opaque = opaque ; 598 v->listener = vhost_vdpa_memory_listener; 599 v->msg_type = VHOST_IOTLB_MSG_V2; 600 vhost_vdpa_init_svq(dev, v); 601 602 error_propagate(&dev->migration_blocker, v->migration_blocker); 603 if (!vhost_vdpa_first_dev(dev)) { 604 return 0; 605 } 606 607 /* 608 * If dev->shadow_vqs_enabled at initialization that means the device has 609 * been started with x-svq=on, so don't block migration 610 */ 611 if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { 612 /* We don't have dev->features yet */ 613 uint64_t features; 614 ret = vhost_vdpa_get_dev_features(dev, &features); 615 if (unlikely(ret)) { 616 error_setg_errno(errp, -ret, "Could not get device features"); 617 return ret; 618 } 619 vhost_svq_valid_features(features, &dev->migration_blocker); 620 } 621 622 /* 623 * Similar to VFIO, we end up pinning all guest memory and have to 624 * disable discarding of RAM. 625 */ 626 ret = ram_block_discard_disable(true); 627 if (ret) { 628 error_report("Cannot set discarding of RAM broken"); 629 return ret; 630 } 631 632 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 633 VIRTIO_CONFIG_S_DRIVER); 634 635 return 0; 636 } 637 638 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, 639 int queue_index) 640 { 641 size_t page_size = qemu_real_host_page_size(); 642 struct vhost_vdpa *v = dev->opaque; 643 VirtIODevice *vdev = dev->vdev; 644 VhostVDPAHostNotifier *n; 645 646 n = &v->notifier[queue_index]; 647 648 if (n->addr) { 649 virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); 650 object_unparent(OBJECT(&n->mr)); 651 munmap(n->addr, page_size); 652 n->addr = NULL; 653 } 654 } 655 656 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) 657 { 658 size_t page_size = qemu_real_host_page_size(); 659 struct vhost_vdpa *v = dev->opaque; 660 VirtIODevice *vdev = dev->vdev; 661 VhostVDPAHostNotifier *n; 662 int fd = v->device_fd; 663 void *addr; 664 char *name; 665 666 vhost_vdpa_host_notifier_uninit(dev, queue_index); 667 668 n = &v->notifier[queue_index]; 669 670 addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, 671 queue_index * page_size); 672 if (addr == MAP_FAILED) { 673 goto err; 674 } 675 676 name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", 677 v, queue_index); 678 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 679 page_size, addr); 680 g_free(name); 681 682 if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { 683 object_unparent(OBJECT(&n->mr)); 684 munmap(addr, page_size); 685 goto err; 686 } 687 n->addr = addr; 688 689 return 0; 690 691 err: 692 return -1; 693 } 694 695 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) 696 { 697 int i; 698 699 /* 700 * Pack all the changes to the memory regions in a single 701 * transaction to avoid a few updating of the address space 702 * topology. 703 */ 704 memory_region_transaction_begin(); 705 706 for (i = dev->vq_index; i < dev->vq_index + n; i++) { 707 vhost_vdpa_host_notifier_uninit(dev, i); 708 } 709 710 memory_region_transaction_commit(); 711 } 712 713 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) 714 { 715 struct vhost_vdpa *v = dev->opaque; 716 int i; 717 718 if (v->shadow_vqs_enabled) { 719 /* FIXME SVQ is not compatible with host notifiers mr */ 720 return; 721 } 722 723 /* 724 * Pack all the changes to the memory regions in a single 725 * transaction to avoid a few updating of the address space 726 * topology. 727 */ 728 memory_region_transaction_begin(); 729 730 for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { 731 if (vhost_vdpa_host_notifier_init(dev, i)) { 732 vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index); 733 break; 734 } 735 } 736 737 memory_region_transaction_commit(); 738 } 739 740 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev) 741 { 742 struct vhost_vdpa *v = dev->opaque; 743 size_t idx; 744 745 for (idx = 0; idx < v->shadow_vqs->len; ++idx) { 746 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx)); 747 } 748 g_ptr_array_free(v->shadow_vqs, true); 749 } 750 751 static int vhost_vdpa_cleanup(struct vhost_dev *dev) 752 { 753 struct vhost_vdpa *v; 754 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 755 v = dev->opaque; 756 trace_vhost_vdpa_cleanup(dev, v); 757 if (vhost_vdpa_first_dev(dev)) { 758 ram_block_discard_disable(false); 759 } 760 761 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 762 memory_listener_unregister(&v->listener); 763 vhost_vdpa_svq_cleanup(dev); 764 765 dev->opaque = NULL; 766 767 return 0; 768 } 769 770 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 771 { 772 trace_vhost_vdpa_memslots_limit(dev, INT_MAX); 773 return INT_MAX; 774 } 775 776 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 777 struct vhost_memory *mem) 778 { 779 if (!vhost_vdpa_first_dev(dev)) { 780 return 0; 781 } 782 783 trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding); 784 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) && 785 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) { 786 int i; 787 for (i = 0; i < mem->nregions; i++) { 788 trace_vhost_vdpa_dump_regions(dev, i, 789 mem->regions[i].guest_phys_addr, 790 mem->regions[i].memory_size, 791 mem->regions[i].userspace_addr, 792 mem->regions[i].flags_padding); 793 } 794 } 795 if (mem->padding) { 796 return -EINVAL; 797 } 798 799 return 0; 800 } 801 802 static int vhost_vdpa_set_features(struct vhost_dev *dev, 803 uint64_t features) 804 { 805 struct vhost_vdpa *v = dev->opaque; 806 int ret; 807 808 if (!vhost_vdpa_first_dev(dev)) { 809 return 0; 810 } 811 812 if (v->shadow_vqs_enabled) { 813 if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) { 814 /* 815 * QEMU is just trying to enable or disable logging. SVQ handles 816 * this sepparately, so no need to forward this. 817 */ 818 v->acked_features = features; 819 return 0; 820 } 821 822 v->acked_features = features; 823 824 /* We must not ack _F_LOG if SVQ is enabled */ 825 features &= ~BIT_ULL(VHOST_F_LOG_ALL); 826 } 827 828 trace_vhost_vdpa_set_features(dev, features); 829 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 830 if (ret) { 831 return ret; 832 } 833 834 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 835 } 836 837 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) 838 { 839 uint64_t features; 840 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 841 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | 842 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | 843 0x1ULL << VHOST_BACKEND_F_SUSPEND; 844 int r; 845 846 if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { 847 return -EFAULT; 848 } 849 850 features &= f; 851 852 if (vhost_vdpa_first_dev(dev)) { 853 r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); 854 if (r) { 855 return -EFAULT; 856 } 857 } 858 859 dev->backend_cap = features; 860 861 return 0; 862 } 863 864 static int vhost_vdpa_get_device_id(struct vhost_dev *dev, 865 uint32_t *device_id) 866 { 867 int ret; 868 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 869 trace_vhost_vdpa_get_device_id(dev, *device_id); 870 return ret; 871 } 872 873 static int vhost_vdpa_reset_device(struct vhost_dev *dev) 874 { 875 struct vhost_vdpa *v = dev->opaque; 876 int ret; 877 uint8_t status = 0; 878 879 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 880 trace_vhost_vdpa_reset_device(dev); 881 v->suspended = false; 882 return ret; 883 } 884 885 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 886 { 887 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 888 889 trace_vhost_vdpa_get_vq_index(dev, idx, idx); 890 return idx; 891 } 892 893 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx) 894 { 895 struct vhost_dev *dev = v->dev; 896 struct vhost_vring_state state = { 897 .index = idx, 898 .num = 1, 899 }; 900 int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 901 902 trace_vhost_vdpa_set_vring_ready(dev, idx, r); 903 return r; 904 } 905 906 static int vhost_vdpa_set_config_call(struct vhost_dev *dev, 907 int fd) 908 { 909 trace_vhost_vdpa_set_config_call(dev, fd); 910 return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd); 911 } 912 913 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config, 914 uint32_t config_len) 915 { 916 int b, len; 917 char line[QEMU_HEXDUMP_LINE_LEN]; 918 919 for (b = 0; b < config_len; b += 16) { 920 len = config_len - b; 921 qemu_hexdump_line(line, b, config, len, false); 922 trace_vhost_vdpa_dump_config(dev, line); 923 } 924 } 925 926 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 927 uint32_t offset, uint32_t size, 928 uint32_t flags) 929 { 930 struct vhost_vdpa_config *config; 931 int ret; 932 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 933 934 trace_vhost_vdpa_set_config(dev, offset, size, flags); 935 config = g_malloc(size + config_size); 936 config->off = offset; 937 config->len = size; 938 memcpy(config->buf, data, size); 939 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) && 940 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 941 vhost_vdpa_dump_config(dev, data, size); 942 } 943 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 944 g_free(config); 945 return ret; 946 } 947 948 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 949 uint32_t config_len, Error **errp) 950 { 951 struct vhost_vdpa_config *v_config; 952 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 953 int ret; 954 955 trace_vhost_vdpa_get_config(dev, config, config_len); 956 v_config = g_malloc(config_len + config_size); 957 v_config->len = config_len; 958 v_config->off = 0; 959 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 960 memcpy(config, v_config->buf, config_len); 961 g_free(v_config); 962 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) && 963 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 964 vhost_vdpa_dump_config(dev, config, config_len); 965 } 966 return ret; 967 } 968 969 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev, 970 struct vhost_vring_state *ring) 971 { 972 trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num); 973 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 974 } 975 976 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev, 977 struct vhost_vring_file *file) 978 { 979 trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd); 980 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 981 } 982 983 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev, 984 struct vhost_vring_file *file) 985 { 986 trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd); 987 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 988 } 989 990 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev, 991 struct vhost_vring_addr *addr) 992 { 993 trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags, 994 addr->desc_user_addr, addr->used_user_addr, 995 addr->avail_user_addr, 996 addr->log_guest_addr); 997 998 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 999 1000 } 1001 1002 /** 1003 * Set the shadow virtqueue descriptors to the device 1004 * 1005 * @dev: The vhost device model 1006 * @svq: The shadow virtqueue 1007 * @idx: The index of the virtqueue in the vhost device 1008 * @errp: Error 1009 * 1010 * Note that this function does not rewind kick file descriptor if cannot set 1011 * call one. 1012 */ 1013 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev, 1014 VhostShadowVirtqueue *svq, unsigned idx, 1015 Error **errp) 1016 { 1017 struct vhost_vring_file file = { 1018 .index = dev->vq_index + idx, 1019 }; 1020 const EventNotifier *event_notifier = &svq->hdev_kick; 1021 int r; 1022 1023 r = event_notifier_init(&svq->hdev_kick, 0); 1024 if (r != 0) { 1025 error_setg_errno(errp, -r, "Couldn't create kick event notifier"); 1026 goto err_init_hdev_kick; 1027 } 1028 1029 r = event_notifier_init(&svq->hdev_call, 0); 1030 if (r != 0) { 1031 error_setg_errno(errp, -r, "Couldn't create call event notifier"); 1032 goto err_init_hdev_call; 1033 } 1034 1035 file.fd = event_notifier_get_fd(event_notifier); 1036 r = vhost_vdpa_set_vring_dev_kick(dev, &file); 1037 if (unlikely(r != 0)) { 1038 error_setg_errno(errp, -r, "Can't set device kick fd"); 1039 goto err_init_set_dev_fd; 1040 } 1041 1042 event_notifier = &svq->hdev_call; 1043 file.fd = event_notifier_get_fd(event_notifier); 1044 r = vhost_vdpa_set_vring_dev_call(dev, &file); 1045 if (unlikely(r != 0)) { 1046 error_setg_errno(errp, -r, "Can't set device call fd"); 1047 goto err_init_set_dev_fd; 1048 } 1049 1050 return 0; 1051 1052 err_init_set_dev_fd: 1053 event_notifier_set_handler(&svq->hdev_call, NULL); 1054 1055 err_init_hdev_call: 1056 event_notifier_cleanup(&svq->hdev_kick); 1057 1058 err_init_hdev_kick: 1059 return r; 1060 } 1061 1062 /** 1063 * Unmap a SVQ area in the device 1064 */ 1065 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr) 1066 { 1067 const DMAMap needle = { 1068 .translated_addr = addr, 1069 }; 1070 const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree, 1071 &needle); 1072 hwaddr size; 1073 int r; 1074 1075 if (unlikely(!result)) { 1076 error_report("Unable to find SVQ address to unmap"); 1077 return; 1078 } 1079 1080 size = ROUND_UP(result->size, qemu_real_host_page_size()); 1081 r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size); 1082 if (unlikely(r < 0)) { 1083 error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r); 1084 return; 1085 } 1086 1087 vhost_iova_tree_remove(v->shared->iova_tree, *result); 1088 } 1089 1090 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev, 1091 const VhostShadowVirtqueue *svq) 1092 { 1093 struct vhost_vdpa *v = dev->opaque; 1094 struct vhost_vring_addr svq_addr; 1095 1096 vhost_svq_get_vring_addr(svq, &svq_addr); 1097 1098 vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr); 1099 1100 vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr); 1101 } 1102 1103 /** 1104 * Map the SVQ area in the device 1105 * 1106 * @v: Vhost-vdpa device 1107 * @needle: The area to search iova 1108 * @errorp: Error pointer 1109 */ 1110 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle, 1111 Error **errp) 1112 { 1113 int r; 1114 1115 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle); 1116 if (unlikely(r != IOVA_OK)) { 1117 error_setg(errp, "Cannot allocate iova (%d)", r); 1118 return false; 1119 } 1120 1121 r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova, 1122 needle->size + 1, 1123 (void *)(uintptr_t)needle->translated_addr, 1124 needle->perm == IOMMU_RO); 1125 if (unlikely(r != 0)) { 1126 error_setg_errno(errp, -r, "Cannot map region to device"); 1127 vhost_iova_tree_remove(v->shared->iova_tree, *needle); 1128 } 1129 1130 return r == 0; 1131 } 1132 1133 /** 1134 * Map the shadow virtqueue rings in the device 1135 * 1136 * @dev: The vhost device 1137 * @svq: The shadow virtqueue 1138 * @addr: Assigned IOVA addresses 1139 * @errp: Error pointer 1140 */ 1141 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev, 1142 const VhostShadowVirtqueue *svq, 1143 struct vhost_vring_addr *addr, 1144 Error **errp) 1145 { 1146 ERRP_GUARD(); 1147 DMAMap device_region, driver_region; 1148 struct vhost_vring_addr svq_addr; 1149 struct vhost_vdpa *v = dev->opaque; 1150 size_t device_size = vhost_svq_device_area_size(svq); 1151 size_t driver_size = vhost_svq_driver_area_size(svq); 1152 size_t avail_offset; 1153 bool ok; 1154 1155 vhost_svq_get_vring_addr(svq, &svq_addr); 1156 1157 driver_region = (DMAMap) { 1158 .translated_addr = svq_addr.desc_user_addr, 1159 .size = driver_size - 1, 1160 .perm = IOMMU_RO, 1161 }; 1162 ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp); 1163 if (unlikely(!ok)) { 1164 error_prepend(errp, "Cannot create vq driver region: "); 1165 return false; 1166 } 1167 addr->desc_user_addr = driver_region.iova; 1168 avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr; 1169 addr->avail_user_addr = driver_region.iova + avail_offset; 1170 1171 device_region = (DMAMap) { 1172 .translated_addr = svq_addr.used_user_addr, 1173 .size = device_size - 1, 1174 .perm = IOMMU_RW, 1175 }; 1176 ok = vhost_vdpa_svq_map_ring(v, &device_region, errp); 1177 if (unlikely(!ok)) { 1178 error_prepend(errp, "Cannot create vq device region: "); 1179 vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr); 1180 } 1181 addr->used_user_addr = device_region.iova; 1182 1183 return ok; 1184 } 1185 1186 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev, 1187 VhostShadowVirtqueue *svq, unsigned idx, 1188 Error **errp) 1189 { 1190 uint16_t vq_index = dev->vq_index + idx; 1191 struct vhost_vring_state s = { 1192 .index = vq_index, 1193 }; 1194 int r; 1195 1196 r = vhost_vdpa_set_dev_vring_base(dev, &s); 1197 if (unlikely(r)) { 1198 error_setg_errno(errp, -r, "Cannot set vring base"); 1199 return false; 1200 } 1201 1202 r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp); 1203 return r == 0; 1204 } 1205 1206 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev) 1207 { 1208 struct vhost_vdpa *v = dev->opaque; 1209 Error *err = NULL; 1210 unsigned i; 1211 1212 if (!v->shadow_vqs_enabled) { 1213 return true; 1214 } 1215 1216 for (i = 0; i < v->shadow_vqs->len; ++i) { 1217 VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i); 1218 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1219 struct vhost_vring_addr addr = { 1220 .index = dev->vq_index + i, 1221 }; 1222 int r; 1223 bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err); 1224 if (unlikely(!ok)) { 1225 goto err; 1226 } 1227 1228 vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree); 1229 ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err); 1230 if (unlikely(!ok)) { 1231 goto err_map; 1232 } 1233 1234 /* Override vring GPA set by vhost subsystem */ 1235 r = vhost_vdpa_set_vring_dev_addr(dev, &addr); 1236 if (unlikely(r != 0)) { 1237 error_setg_errno(&err, -r, "Cannot set device address"); 1238 goto err_set_addr; 1239 } 1240 } 1241 1242 return true; 1243 1244 err_set_addr: 1245 vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i)); 1246 1247 err_map: 1248 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i)); 1249 1250 err: 1251 error_reportf_err(err, "Cannot setup SVQ %u: ", i); 1252 for (unsigned j = 0; j < i; ++j) { 1253 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j); 1254 vhost_vdpa_svq_unmap_rings(dev, svq); 1255 vhost_svq_stop(svq); 1256 } 1257 1258 return false; 1259 } 1260 1261 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) 1262 { 1263 struct vhost_vdpa *v = dev->opaque; 1264 1265 if (!v->shadow_vqs_enabled) { 1266 return; 1267 } 1268 1269 for (unsigned i = 0; i < v->shadow_vqs->len; ++i) { 1270 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1271 1272 vhost_svq_stop(svq); 1273 vhost_vdpa_svq_unmap_rings(dev, svq); 1274 1275 event_notifier_cleanup(&svq->hdev_kick); 1276 event_notifier_cleanup(&svq->hdev_call); 1277 } 1278 } 1279 1280 static void vhost_vdpa_suspend(struct vhost_dev *dev) 1281 { 1282 struct vhost_vdpa *v = dev->opaque; 1283 int r; 1284 1285 if (!vhost_vdpa_first_dev(dev)) { 1286 return; 1287 } 1288 1289 if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { 1290 trace_vhost_vdpa_suspend(dev); 1291 r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND); 1292 if (unlikely(r)) { 1293 error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); 1294 } else { 1295 v->suspended = true; 1296 return; 1297 } 1298 } 1299 1300 vhost_vdpa_reset_device(dev); 1301 } 1302 1303 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 1304 { 1305 struct vhost_vdpa *v = dev->opaque; 1306 bool ok; 1307 trace_vhost_vdpa_dev_start(dev, started); 1308 1309 if (started) { 1310 vhost_vdpa_host_notifiers_init(dev); 1311 ok = vhost_vdpa_svqs_start(dev); 1312 if (unlikely(!ok)) { 1313 return -1; 1314 } 1315 } else { 1316 vhost_vdpa_suspend(dev); 1317 vhost_vdpa_svqs_stop(dev); 1318 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 1319 } 1320 1321 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1322 return 0; 1323 } 1324 1325 if (started) { 1326 if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) { 1327 error_report("SVQ can not work while IOMMU enable, please disable" 1328 "IOMMU and try again"); 1329 return -1; 1330 } 1331 memory_listener_register(&v->listener, dev->vdev->dma_as); 1332 1333 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 1334 } 1335 1336 return 0; 1337 } 1338 1339 static void vhost_vdpa_reset_status(struct vhost_dev *dev) 1340 { 1341 struct vhost_vdpa *v = dev->opaque; 1342 1343 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1344 return; 1345 } 1346 1347 vhost_vdpa_reset_device(dev); 1348 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 1349 VIRTIO_CONFIG_S_DRIVER); 1350 memory_listener_unregister(&v->listener); 1351 } 1352 1353 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 1354 struct vhost_log *log) 1355 { 1356 struct vhost_vdpa *v = dev->opaque; 1357 if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) { 1358 return 0; 1359 } 1360 1361 trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd, 1362 log->log); 1363 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 1364 } 1365 1366 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 1367 struct vhost_vring_addr *addr) 1368 { 1369 struct vhost_vdpa *v = dev->opaque; 1370 1371 if (v->shadow_vqs_enabled) { 1372 /* 1373 * Device vring addr was set at device start. SVQ base is handled by 1374 * VirtQueue code. 1375 */ 1376 return 0; 1377 } 1378 1379 return vhost_vdpa_set_vring_dev_addr(dev, addr); 1380 } 1381 1382 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 1383 struct vhost_vring_state *ring) 1384 { 1385 trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num); 1386 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 1387 } 1388 1389 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 1390 struct vhost_vring_state *ring) 1391 { 1392 struct vhost_vdpa *v = dev->opaque; 1393 1394 if (v->shadow_vqs_enabled) { 1395 /* 1396 * Device vring base was set at device start. SVQ base is handled by 1397 * VirtQueue code. 1398 */ 1399 return 0; 1400 } 1401 1402 return vhost_vdpa_set_dev_vring_base(dev, ring); 1403 } 1404 1405 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 1406 struct vhost_vring_state *ring) 1407 { 1408 struct vhost_vdpa *v = dev->opaque; 1409 int ret; 1410 1411 if (v->shadow_vqs_enabled) { 1412 ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index); 1413 return 0; 1414 } 1415 1416 if (!v->suspended) { 1417 /* 1418 * Cannot trust in value returned by device, let vhost recover used 1419 * idx from guest. 1420 */ 1421 return -1; 1422 } 1423 1424 ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 1425 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); 1426 return ret; 1427 } 1428 1429 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 1430 struct vhost_vring_file *file) 1431 { 1432 struct vhost_vdpa *v = dev->opaque; 1433 int vdpa_idx = file->index - dev->vq_index; 1434 1435 if (v->shadow_vqs_enabled) { 1436 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1437 vhost_svq_set_svq_kick_fd(svq, file->fd); 1438 return 0; 1439 } else { 1440 return vhost_vdpa_set_vring_dev_kick(dev, file); 1441 } 1442 } 1443 1444 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 1445 struct vhost_vring_file *file) 1446 { 1447 struct vhost_vdpa *v = dev->opaque; 1448 int vdpa_idx = file->index - dev->vq_index; 1449 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1450 1451 /* Remember last call fd because we can switch to SVQ anytime. */ 1452 vhost_svq_set_svq_call_fd(svq, file->fd); 1453 if (v->shadow_vqs_enabled) { 1454 return 0; 1455 } 1456 1457 return vhost_vdpa_set_vring_dev_call(dev, file); 1458 } 1459 1460 static int vhost_vdpa_get_features(struct vhost_dev *dev, 1461 uint64_t *features) 1462 { 1463 int ret = vhost_vdpa_get_dev_features(dev, features); 1464 1465 if (ret == 0) { 1466 /* Add SVQ logging capabilities */ 1467 *features |= BIT_ULL(VHOST_F_LOG_ALL); 1468 } 1469 1470 return ret; 1471 } 1472 1473 static int vhost_vdpa_set_owner(struct vhost_dev *dev) 1474 { 1475 if (!vhost_vdpa_first_dev(dev)) { 1476 return 0; 1477 } 1478 1479 trace_vhost_vdpa_set_owner(dev); 1480 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 1481 } 1482 1483 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 1484 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 1485 { 1486 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 1487 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 1488 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 1489 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 1490 trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, 1491 addr->avail_user_addr, addr->used_user_addr); 1492 return 0; 1493 } 1494 1495 static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 1496 { 1497 return true; 1498 } 1499 1500 const VhostOps vdpa_ops = { 1501 .backend_type = VHOST_BACKEND_TYPE_VDPA, 1502 .vhost_backend_init = vhost_vdpa_init, 1503 .vhost_backend_cleanup = vhost_vdpa_cleanup, 1504 .vhost_set_log_base = vhost_vdpa_set_log_base, 1505 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 1506 .vhost_set_vring_num = vhost_vdpa_set_vring_num, 1507 .vhost_set_vring_base = vhost_vdpa_set_vring_base, 1508 .vhost_get_vring_base = vhost_vdpa_get_vring_base, 1509 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 1510 .vhost_set_vring_call = vhost_vdpa_set_vring_call, 1511 .vhost_get_features = vhost_vdpa_get_features, 1512 .vhost_set_backend_cap = vhost_vdpa_set_backend_cap, 1513 .vhost_set_owner = vhost_vdpa_set_owner, 1514 .vhost_set_vring_endian = NULL, 1515 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 1516 .vhost_set_mem_table = vhost_vdpa_set_mem_table, 1517 .vhost_set_features = vhost_vdpa_set_features, 1518 .vhost_reset_device = vhost_vdpa_reset_device, 1519 .vhost_get_vq_index = vhost_vdpa_get_vq_index, 1520 .vhost_get_config = vhost_vdpa_get_config, 1521 .vhost_set_config = vhost_vdpa_set_config, 1522 .vhost_requires_shm_log = NULL, 1523 .vhost_migration_done = NULL, 1524 .vhost_net_set_mtu = NULL, 1525 .vhost_set_iotlb_callback = NULL, 1526 .vhost_send_device_iotlb_msg = NULL, 1527 .vhost_dev_start = vhost_vdpa_dev_start, 1528 .vhost_get_device_id = vhost_vdpa_get_device_id, 1529 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 1530 .vhost_force_iommu = vhost_vdpa_force_iommu, 1531 .vhost_set_config_call = vhost_vdpa_set_config_call, 1532 .vhost_reset_status = vhost_vdpa_reset_status, 1533 }; 1534