1 /* 2 * vhost-vdpa.c 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include "clients.h" 14 #include "hw/virtio/virtio-net.h" 15 #include "net/vhost_net.h" 16 #include "net/vhost-vdpa.h" 17 #include "hw/virtio/vhost-vdpa.h" 18 #include "qemu/config-file.h" 19 #include "qemu/error-report.h" 20 #include "qemu/log.h" 21 #include "qemu/memalign.h" 22 #include "qemu/option.h" 23 #include "qapi/error.h" 24 #include <linux/vhost.h> 25 #include <sys/ioctl.h> 26 #include <err.h> 27 #include "standard-headers/linux/virtio_net.h" 28 #include "monitor/monitor.h" 29 #include "migration/misc.h" 30 #include "hw/virtio/vhost.h" 31 #include "trace.h" 32 33 /* Todo:need to add the multiqueue support here */ 34 typedef struct VhostVDPAState { 35 NetClientState nc; 36 struct vhost_vdpa vhost_vdpa; 37 NotifierWithReturn migration_state; 38 VHostNetState *vhost_net; 39 40 /* Control commands shadow buffers */ 41 void *cvq_cmd_out_buffer; 42 virtio_net_ctrl_ack *status; 43 44 /* The device always have SVQ enabled */ 45 bool always_svq; 46 47 /* The device can isolate CVQ in its own ASID */ 48 bool cvq_isolated; 49 50 bool started; 51 } VhostVDPAState; 52 53 /* 54 * The array is sorted alphabetically in ascending order, 55 * with the exception of VHOST_INVALID_FEATURE_BIT, 56 * which should always be the last entry. 57 */ 58 const int vdpa_feature_bits[] = { 59 VIRTIO_F_ANY_LAYOUT, 60 VIRTIO_F_IOMMU_PLATFORM, 61 VIRTIO_F_NOTIFY_ON_EMPTY, 62 VIRTIO_F_RING_PACKED, 63 VIRTIO_F_RING_RESET, 64 VIRTIO_F_VERSION_1, 65 VIRTIO_F_IN_ORDER, 66 VIRTIO_F_NOTIFICATION_DATA, 67 VIRTIO_NET_F_CSUM, 68 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 69 VIRTIO_NET_F_CTRL_MAC_ADDR, 70 VIRTIO_NET_F_CTRL_RX, 71 VIRTIO_NET_F_CTRL_RX_EXTRA, 72 VIRTIO_NET_F_CTRL_VLAN, 73 VIRTIO_NET_F_CTRL_VQ, 74 VIRTIO_NET_F_GSO, 75 VIRTIO_NET_F_GUEST_CSUM, 76 VIRTIO_NET_F_GUEST_ECN, 77 VIRTIO_NET_F_GUEST_TSO4, 78 VIRTIO_NET_F_GUEST_TSO6, 79 VIRTIO_NET_F_GUEST_UFO, 80 VIRTIO_NET_F_GUEST_USO4, 81 VIRTIO_NET_F_GUEST_USO6, 82 VIRTIO_NET_F_HASH_REPORT, 83 VIRTIO_NET_F_HOST_ECN, 84 VIRTIO_NET_F_HOST_TSO4, 85 VIRTIO_NET_F_HOST_TSO6, 86 VIRTIO_NET_F_HOST_UFO, 87 VIRTIO_NET_F_HOST_USO, 88 VIRTIO_NET_F_MQ, 89 VIRTIO_NET_F_MRG_RXBUF, 90 VIRTIO_NET_F_MTU, 91 VIRTIO_NET_F_RSC_EXT, 92 VIRTIO_NET_F_RSS, 93 VIRTIO_NET_F_STATUS, 94 VIRTIO_RING_F_EVENT_IDX, 95 VIRTIO_RING_F_INDIRECT_DESC, 96 97 /* VHOST_INVALID_FEATURE_BIT should always be the last entry */ 98 VHOST_INVALID_FEATURE_BIT 99 }; 100 101 /** Supported device specific feature bits with SVQ */ 102 static const uint64_t vdpa_svq_device_features = 103 BIT_ULL(VIRTIO_NET_F_CSUM) | 104 BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | 105 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | 106 BIT_ULL(VIRTIO_NET_F_MTU) | 107 BIT_ULL(VIRTIO_NET_F_MAC) | 108 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | 109 BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | 110 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | 111 BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | 112 BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | 113 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | 114 BIT_ULL(VIRTIO_NET_F_HOST_ECN) | 115 BIT_ULL(VIRTIO_NET_F_HOST_UFO) | 116 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | 117 BIT_ULL(VIRTIO_NET_F_STATUS) | 118 BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | 119 BIT_ULL(VIRTIO_NET_F_CTRL_RX) | 120 BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | 121 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | 122 BIT_ULL(VIRTIO_NET_F_MQ) | 123 BIT_ULL(VIRTIO_F_ANY_LAYOUT) | 124 BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | 125 /* VHOST_F_LOG_ALL is exposed by SVQ */ 126 BIT_ULL(VHOST_F_LOG_ALL) | 127 BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | 128 BIT_ULL(VIRTIO_NET_F_RSS) | 129 BIT_ULL(VIRTIO_NET_F_RSC_EXT) | 130 BIT_ULL(VIRTIO_NET_F_STANDBY) | 131 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX); 132 133 #define VHOST_VDPA_NET_CVQ_ASID 1 134 135 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc) 136 { 137 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 138 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 139 return s->vhost_net; 140 } 141 142 static size_t vhost_vdpa_net_cvq_cmd_len(void) 143 { 144 /* 145 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer. 146 * In buffer is always 1 byte, so it should fit here 147 */ 148 return sizeof(struct virtio_net_ctrl_hdr) + 149 2 * sizeof(struct virtio_net_ctrl_mac) + 150 MAC_TABLE_ENTRIES * ETH_ALEN; 151 } 152 153 static size_t vhost_vdpa_net_cvq_cmd_page_len(void) 154 { 155 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size()); 156 } 157 158 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp) 159 { 160 uint64_t invalid_dev_features = 161 features & ~vdpa_svq_device_features & 162 /* Transport are all accepted at this point */ 163 ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START, 164 VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START); 165 166 if (invalid_dev_features) { 167 error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64, 168 invalid_dev_features); 169 return false; 170 } 171 172 return vhost_svq_valid_features(features, errp); 173 } 174 175 static int vhost_vdpa_net_check_device_id(struct vhost_net *net) 176 { 177 uint32_t device_id; 178 int ret; 179 struct vhost_dev *hdev; 180 181 hdev = (struct vhost_dev *)&net->dev; 182 ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id); 183 if (device_id != VIRTIO_ID_NET) { 184 return -ENOTSUP; 185 } 186 return ret; 187 } 188 189 static int vhost_vdpa_add(NetClientState *ncs, void *be, 190 int queue_pair_index, int nvqs) 191 { 192 VhostNetOptions options; 193 struct vhost_net *net = NULL; 194 VhostVDPAState *s; 195 int ret; 196 197 options.backend_type = VHOST_BACKEND_TYPE_VDPA; 198 assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 199 s = DO_UPCAST(VhostVDPAState, nc, ncs); 200 options.net_backend = ncs; 201 options.opaque = be; 202 options.busyloop_timeout = 0; 203 options.nvqs = nvqs; 204 205 net = vhost_net_init(&options); 206 if (!net) { 207 error_report("failed to init vhost_net for queue"); 208 goto err_init; 209 } 210 s->vhost_net = net; 211 ret = vhost_vdpa_net_check_device_id(net); 212 if (ret) { 213 goto err_check; 214 } 215 return 0; 216 err_check: 217 vhost_net_cleanup(net); 218 g_free(net); 219 err_init: 220 return -1; 221 } 222 223 static void vhost_vdpa_cleanup(NetClientState *nc) 224 { 225 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 226 227 munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len()); 228 munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len()); 229 if (s->vhost_net) { 230 vhost_net_cleanup(s->vhost_net); 231 g_free(s->vhost_net); 232 s->vhost_net = NULL; 233 } 234 if (s->vhost_vdpa.index != 0) { 235 return; 236 } 237 qemu_close(s->vhost_vdpa.shared->device_fd); 238 g_clear_pointer(&s->vhost_vdpa.shared->iova_tree, vhost_iova_tree_delete); 239 g_free(s->vhost_vdpa.shared); 240 } 241 242 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc) 243 { 244 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 245 246 return true; 247 } 248 249 static bool vhost_vdpa_get_vnet_hash_supported_types(NetClientState *nc, 250 uint32_t *types) 251 { 252 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 253 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 254 uint64_t features = s->vhost_vdpa.dev->features; 255 int fd = s->vhost_vdpa.shared->device_fd; 256 struct { 257 struct vhost_vdpa_config hdr; 258 uint32_t supported_hash_types; 259 } config; 260 261 if (!virtio_has_feature(features, VIRTIO_NET_F_HASH_REPORT) && 262 !virtio_has_feature(features, VIRTIO_NET_F_RSS)) { 263 return false; 264 } 265 266 config.hdr.off = offsetof(struct virtio_net_config, supported_hash_types); 267 config.hdr.len = sizeof(config.supported_hash_types); 268 269 assert(!ioctl(fd, VHOST_VDPA_GET_CONFIG, &config)); 270 *types = le32_to_cpu(config.supported_hash_types); 271 272 return true; 273 } 274 275 static bool vhost_vdpa_has_ufo(NetClientState *nc) 276 { 277 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 278 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 279 uint64_t features = 0; 280 features |= (1ULL << VIRTIO_NET_F_HOST_UFO); 281 features = vhost_net_get_features(s->vhost_net, features); 282 return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO)); 283 284 } 285 286 /* 287 * FIXME: vhost_vdpa doesn't have an API to "set h/w endianness". But it's 288 * reasonable to assume that h/w is LE by default, because LE is what 289 * virtio 1.0 and later ask for. So, this function just says "yes, the h/w is 290 * LE". Otherwise, on a BE machine, higher-level code would mistakely think 291 * the h/w is BE and can't support VDPA for a virtio 1.0 client. 292 */ 293 static int vhost_vdpa_set_vnet_le(NetClientState *nc, bool enable) 294 { 295 return 0; 296 } 297 298 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc, 299 Error **errp) 300 { 301 const char *driver = object_class_get_name(oc); 302 303 if (!g_str_has_prefix(driver, "virtio-net-")) { 304 error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*"); 305 return false; 306 } 307 308 return true; 309 } 310 311 /** Dummy receive in case qemu falls back to userland tap networking */ 312 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf, 313 size_t size) 314 { 315 return size; 316 } 317 318 319 /** From any vdpa net client, get the netclient of the i-th queue pair */ 320 static VhostVDPAState *vhost_vdpa_net_get_nc_vdpa(VhostVDPAState *s, int i) 321 { 322 NICState *nic = qemu_get_nic(s->nc.peer); 323 NetClientState *nc_i = qemu_get_peer(nic->ncs, i); 324 325 return DO_UPCAST(VhostVDPAState, nc, nc_i); 326 } 327 328 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s) 329 { 330 return vhost_vdpa_net_get_nc_vdpa(s, 0); 331 } 332 333 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable) 334 { 335 struct vhost_vdpa *v = &s->vhost_vdpa; 336 VirtIONet *n; 337 VirtIODevice *vdev; 338 int data_queue_pairs, cvq, r; 339 340 /* We are only called on the first data vqs and only if x-svq is not set */ 341 if (s->vhost_vdpa.shadow_vqs_enabled == enable) { 342 return; 343 } 344 345 vdev = v->dev->vdev; 346 n = VIRTIO_NET(vdev); 347 if (!n->vhost_started) { 348 return; 349 } 350 351 data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 352 cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ? 353 n->max_ncs - n->max_queue_pairs : 0; 354 v->shared->svq_switching = enable ? 355 SVQ_TSTATE_ENABLING : SVQ_TSTATE_DISABLING; 356 /* 357 * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter 358 * in the future and resume the device if read-only operations between 359 * suspend and reset goes wrong. 360 */ 361 vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq); 362 363 /* Start will check migration setup_or_active to configure or not SVQ */ 364 r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq); 365 if (unlikely(r < 0)) { 366 error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r); 367 } 368 v->shared->svq_switching = SVQ_TSTATE_DONE; 369 } 370 371 static int vdpa_net_migration_state_notifier(NotifierWithReturn *notifier, 372 MigrationEvent *e, Error **errp) 373 { 374 VhostVDPAState *s = container_of(notifier, VhostVDPAState, migration_state); 375 376 if (e->type == MIG_EVENT_PRECOPY_SETUP) { 377 vhost_vdpa_net_log_global_enable(s, true); 378 } else if (e->type == MIG_EVENT_PRECOPY_FAILED) { 379 vhost_vdpa_net_log_global_enable(s, false); 380 } 381 return 0; 382 } 383 384 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s) 385 { 386 migration_add_notifier(&s->migration_state, 387 vdpa_net_migration_state_notifier); 388 } 389 390 static int vhost_vdpa_net_data_start(NetClientState *nc) 391 { 392 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 393 struct vhost_vdpa *v = &s->vhost_vdpa; 394 395 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 396 397 if (s->always_svq || migration_is_running()) { 398 v->shadow_vqs_enabled = true; 399 } else { 400 v->shadow_vqs_enabled = false; 401 } 402 403 if (v->index == 0) { 404 v->shared->shadow_data = v->shadow_vqs_enabled; 405 vhost_vdpa_net_data_start_first(s); 406 return 0; 407 } 408 409 return 0; 410 } 411 412 static int vhost_vdpa_net_data_load(NetClientState *nc) 413 { 414 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 415 struct vhost_vdpa *v = &s->vhost_vdpa; 416 bool has_cvq = v->dev->vq_index_end % 2; 417 418 if (has_cvq) { 419 return 0; 420 } 421 422 for (int i = 0; i < v->dev->nvqs; ++i) { 423 int ret = vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index); 424 if (ret < 0) { 425 return ret; 426 } 427 } 428 return 0; 429 } 430 431 static void vhost_vdpa_net_client_stop(NetClientState *nc) 432 { 433 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 434 435 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 436 437 if (s->vhost_vdpa.index == 0) { 438 migration_remove_notifier(&s->migration_state); 439 } 440 } 441 442 static NetClientInfo net_vhost_vdpa_info = { 443 .type = NET_CLIENT_DRIVER_VHOST_VDPA, 444 .size = sizeof(VhostVDPAState), 445 .receive = vhost_vdpa_receive, 446 .start = vhost_vdpa_net_data_start, 447 .load = vhost_vdpa_net_data_load, 448 .stop = vhost_vdpa_net_client_stop, 449 .cleanup = vhost_vdpa_cleanup, 450 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr, 451 .get_vnet_hash_supported_types = vhost_vdpa_get_vnet_hash_supported_types, 452 .has_ufo = vhost_vdpa_has_ufo, 453 .set_vnet_le = vhost_vdpa_set_vnet_le, 454 .check_peer_type = vhost_vdpa_check_peer_type, 455 }; 456 457 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index, 458 Error **errp) 459 { 460 struct vhost_vring_state state = { 461 .index = vq_index, 462 }; 463 int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state); 464 465 if (unlikely(r < 0)) { 466 r = -errno; 467 error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index); 468 return r; 469 } 470 471 return state.num; 472 } 473 474 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v, 475 unsigned vq_group, 476 unsigned asid_num) 477 { 478 struct vhost_vring_state asid = { 479 .index = vq_group, 480 .num = asid_num, 481 }; 482 int r; 483 484 trace_vhost_vdpa_set_address_space_id(v, vq_group, asid_num); 485 486 r = ioctl(v->shared->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid); 487 if (unlikely(r < 0)) { 488 error_report("Can't set vq group %u asid %u, errno=%d (%s)", 489 asid.index, asid.num, errno, g_strerror(errno)); 490 } 491 return r; 492 } 493 494 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr) 495 { 496 VhostIOVATree *tree = v->shared->iova_tree; 497 DMAMap needle = { 498 /* 499 * No need to specify size or to look for more translations since 500 * this contiguous chunk was allocated by us. 501 */ 502 .translated_addr = (hwaddr)(uintptr_t)addr, 503 }; 504 const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle); 505 int r; 506 507 if (unlikely(!map)) { 508 error_report("Cannot locate expected map"); 509 return; 510 } 511 512 r = vhost_vdpa_dma_unmap(v->shared, v->address_space_id, map->iova, 513 map->size + 1); 514 if (unlikely(r != 0)) { 515 error_report("Device cannot unmap: %s(%d)", g_strerror(r), r); 516 } 517 518 vhost_iova_tree_remove(tree, *map); 519 } 520 521 /** Map CVQ buffer. */ 522 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size, 523 bool write) 524 { 525 DMAMap map = {}; 526 hwaddr taddr = (hwaddr)(uintptr_t)buf; 527 int r; 528 529 map.size = size - 1; 530 map.perm = write ? IOMMU_RW : IOMMU_RO, 531 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &map, taddr); 532 if (unlikely(r != IOVA_OK)) { 533 error_report("Cannot map injected element"); 534 535 if (map.translated_addr == taddr) { 536 error_report("Insertion to IOVA->HVA tree failed"); 537 /* Remove the mapping from the IOVA-only tree */ 538 goto dma_map_err; 539 } 540 return r; 541 } 542 543 r = vhost_vdpa_dma_map(v->shared, v->address_space_id, map.iova, 544 vhost_vdpa_net_cvq_cmd_page_len(), buf, !write); 545 if (unlikely(r < 0)) { 546 goto dma_map_err; 547 } 548 549 return 0; 550 551 dma_map_err: 552 vhost_iova_tree_remove(v->shared->iova_tree, map); 553 return r; 554 } 555 556 static int vhost_vdpa_net_cvq_start(NetClientState *nc) 557 { 558 VhostVDPAState *s, *s0; 559 struct vhost_vdpa *v; 560 int64_t cvq_group; 561 int r; 562 Error *err = NULL; 563 564 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 565 566 s = DO_UPCAST(VhostVDPAState, nc, nc); 567 v = &s->vhost_vdpa; 568 569 s0 = vhost_vdpa_net_first_nc_vdpa(s); 570 v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled; 571 s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID; 572 573 if (v->shared->shadow_data) { 574 /* SVQ is already configured for all virtqueues */ 575 goto out; 576 } 577 578 /* 579 * If we early return in these cases SVQ will not be enabled. The migration 580 * will be blocked as long as vhost-vdpa backends will not offer _F_LOG. 581 */ 582 if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) { 583 return 0; 584 } 585 586 if (!s->cvq_isolated) { 587 return 0; 588 } 589 590 cvq_group = vhost_vdpa_get_vring_group(v->shared->device_fd, 591 v->dev->vq_index_end - 1, 592 &err); 593 if (unlikely(cvq_group < 0)) { 594 error_report_err(err); 595 return cvq_group; 596 } 597 598 r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID); 599 if (unlikely(r < 0)) { 600 return r; 601 } 602 603 v->shadow_vqs_enabled = true; 604 s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID; 605 606 out: 607 if (!s->vhost_vdpa.shadow_vqs_enabled) { 608 return 0; 609 } 610 611 r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer, 612 vhost_vdpa_net_cvq_cmd_page_len(), false); 613 if (unlikely(r < 0)) { 614 return r; 615 } 616 617 r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status, 618 vhost_vdpa_net_cvq_cmd_page_len(), true); 619 if (unlikely(r < 0)) { 620 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer); 621 } 622 623 return r; 624 } 625 626 static void vhost_vdpa_net_cvq_stop(NetClientState *nc) 627 { 628 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 629 630 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 631 632 if (s->vhost_vdpa.shadow_vqs_enabled) { 633 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer); 634 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status); 635 } 636 637 vhost_vdpa_net_client_stop(nc); 638 } 639 640 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s, 641 const struct iovec *out_sg, size_t out_num, 642 const struct iovec *in_sg, size_t in_num) 643 { 644 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0); 645 int r; 646 647 r = vhost_svq_add(svq, out_sg, out_num, NULL, in_sg, in_num, NULL, NULL); 648 if (unlikely(r != 0)) { 649 if (unlikely(r == -ENOSPC)) { 650 qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n", 651 __func__); 652 } 653 } 654 655 return r; 656 } 657 658 /* 659 * Convenience wrapper to poll SVQ for multiple control commands. 660 * 661 * Caller should hold the BQL when invoking this function, and should take 662 * the answer before SVQ pulls by itself when BQL is released. 663 */ 664 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight) 665 { 666 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0); 667 return vhost_svq_poll(svq, cmds_in_flight); 668 } 669 670 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s, 671 struct iovec *out_cursor, 672 struct iovec *in_cursor) 673 { 674 /* reset the cursor of the output buffer for the device */ 675 out_cursor->iov_base = s->cvq_cmd_out_buffer; 676 out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len(); 677 678 /* reset the cursor of the in buffer for the device */ 679 in_cursor->iov_base = s->status; 680 in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len(); 681 } 682 683 /* 684 * Poll SVQ for multiple pending control commands and check the device's ack. 685 * 686 * Caller should hold the BQL when invoking this function. 687 * 688 * @s: The VhostVDPAState 689 * @len: The length of the pending status shadow buffer 690 */ 691 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len) 692 { 693 /* device uses a one-byte length ack for each control command */ 694 ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len); 695 if (unlikely(dev_written != len)) { 696 return -EIO; 697 } 698 699 /* check the device's ack */ 700 for (int i = 0; i < len; ++i) { 701 if (s->status[i] != VIRTIO_NET_OK) { 702 return -EIO; 703 } 704 } 705 return 0; 706 } 707 708 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s, 709 struct iovec *out_cursor, 710 struct iovec *in_cursor, uint8_t class, 711 uint8_t cmd, const struct iovec *data_sg, 712 size_t data_num) 713 { 714 const struct virtio_net_ctrl_hdr ctrl = { 715 .class = class, 716 .cmd = cmd, 717 }; 718 size_t data_size = iov_size(data_sg, data_num), cmd_size; 719 struct iovec out, in; 720 ssize_t r; 721 unsigned dummy_cursor_iov_cnt; 722 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0); 723 724 assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl)); 725 cmd_size = sizeof(ctrl) + data_size; 726 trace_vhost_vdpa_net_load_cmd(s, class, cmd, data_num, data_size); 727 if (vhost_svq_available_slots(svq) < 2 || 728 iov_size(out_cursor, 1) < cmd_size) { 729 /* 730 * It is time to flush all pending control commands if SVQ is full 731 * or control commands shadow buffers are full. 732 * 733 * We can poll here since we've had BQL from the time 734 * we sent the descriptor. 735 */ 736 r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base - 737 (void *)s->status); 738 if (unlikely(r < 0)) { 739 return r; 740 } 741 742 vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor); 743 } 744 745 /* pack the CVQ command header */ 746 iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl)); 747 /* pack the CVQ command command-specific-data */ 748 iov_to_buf(data_sg, data_num, 0, 749 out_cursor->iov_base + sizeof(ctrl), data_size); 750 751 /* extract the required buffer from the cursor for output */ 752 iov_copy(&out, 1, out_cursor, 1, 0, cmd_size); 753 /* extract the required buffer from the cursor for input */ 754 iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status)); 755 756 r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1); 757 if (unlikely(r < 0)) { 758 trace_vhost_vdpa_net_load_cmd_retval(s, class, cmd, r); 759 return r; 760 } 761 762 /* iterate the cursors */ 763 dummy_cursor_iov_cnt = 1; 764 iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size); 765 dummy_cursor_iov_cnt = 1; 766 iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status)); 767 768 return 0; 769 } 770 771 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n, 772 struct iovec *out_cursor, 773 struct iovec *in_cursor) 774 { 775 if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 776 const struct iovec data = { 777 .iov_base = (void *)n->mac, 778 .iov_len = sizeof(n->mac), 779 }; 780 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 781 VIRTIO_NET_CTRL_MAC, 782 VIRTIO_NET_CTRL_MAC_ADDR_SET, 783 &data, 1); 784 if (unlikely(r < 0)) { 785 return r; 786 } 787 } 788 789 /* 790 * According to VirtIO standard, "The device MUST have an 791 * empty MAC filtering table on reset.". 792 * 793 * Therefore, there is no need to send this CVQ command if the 794 * driver also sets an empty MAC filter table, which aligns with 795 * the device's defaults. 796 * 797 * Note that the device's defaults can mismatch the driver's 798 * configuration only at live migration. 799 */ 800 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) || 801 n->mac_table.in_use == 0) { 802 return 0; 803 } 804 805 uint32_t uni_entries = n->mac_table.first_multi, 806 uni_macs_size = uni_entries * ETH_ALEN, 807 mul_entries = n->mac_table.in_use - uni_entries, 808 mul_macs_size = mul_entries * ETH_ALEN; 809 struct virtio_net_ctrl_mac uni = { 810 .entries = cpu_to_le32(uni_entries), 811 }; 812 struct virtio_net_ctrl_mac mul = { 813 .entries = cpu_to_le32(mul_entries), 814 }; 815 const struct iovec data[] = { 816 { 817 .iov_base = &uni, 818 .iov_len = sizeof(uni), 819 }, { 820 .iov_base = n->mac_table.macs, 821 .iov_len = uni_macs_size, 822 }, { 823 .iov_base = &mul, 824 .iov_len = sizeof(mul), 825 }, { 826 .iov_base = &n->mac_table.macs[uni_macs_size], 827 .iov_len = mul_macs_size, 828 }, 829 }; 830 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 831 VIRTIO_NET_CTRL_MAC, 832 VIRTIO_NET_CTRL_MAC_TABLE_SET, 833 data, ARRAY_SIZE(data)); 834 if (unlikely(r < 0)) { 835 return r; 836 } 837 838 return 0; 839 } 840 841 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n, 842 struct iovec *out_cursor, 843 struct iovec *in_cursor, bool do_rss) 844 { 845 struct virtio_net_rss_config cfg = {}; 846 ssize_t r; 847 g_autofree uint16_t *table = NULL; 848 849 /* 850 * According to VirtIO standard, "Initially the device has all hash 851 * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.". 852 * 853 * Therefore, there is no need to send this CVQ command if the 854 * driver disables the all hash types, which aligns with 855 * the device's defaults. 856 * 857 * Note that the device's defaults can mismatch the driver's 858 * configuration only at live migration. 859 */ 860 if (!n->rss_data.enabled || 861 n->rss_data.runtime_hash_types == VIRTIO_NET_HASH_REPORT_NONE) { 862 return 0; 863 } 864 865 table = g_malloc_n(n->rss_data.indirections_len, 866 sizeof(n->rss_data.indirections_table[0])); 867 cfg.hash_types = cpu_to_le32(n->rss_data.runtime_hash_types); 868 869 if (do_rss) { 870 /* 871 * According to VirtIO standard, "Number of entries in indirection_table 872 * is (indirection_table_mask + 1)". 873 */ 874 cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len - 875 1); 876 cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue); 877 for (int i = 0; i < n->rss_data.indirections_len; ++i) { 878 table[i] = cpu_to_le16(n->rss_data.indirections_table[i]); 879 } 880 cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs); 881 } else { 882 /* 883 * According to VirtIO standard, "Field reserved MUST contain zeroes. 884 * It is defined to make the structure to match the layout of 885 * virtio_net_rss_config structure, defined in 5.1.6.5.7.". 886 * 887 * Therefore, we need to zero the fields in 888 * struct virtio_net_rss_config, which corresponds to the 889 * `reserved` field in struct virtio_net_hash_config. 890 * 891 * Note that all other fields are zeroed at their definitions, 892 * except for the `indirection_table` field, where the actual data 893 * is stored in the `table` variable to ensure compatibility 894 * with RSS case. Therefore, we need to zero the `table` variable here. 895 */ 896 table[0] = 0; 897 } 898 899 /* 900 * Considering that virtio_net_handle_rss() currently does not restore 901 * the hash key length parsed from the CVQ command sent from the guest 902 * into n->rss_data and uses the maximum key length in other code, so 903 * we also employ the maximum key length here. 904 */ 905 cfg.hash_key_length = sizeof(n->rss_data.key); 906 907 const struct iovec data[] = { 908 { 909 .iov_base = &cfg, 910 .iov_len = offsetof(struct virtio_net_rss_config, 911 indirection_table), 912 }, { 913 .iov_base = table, 914 .iov_len = n->rss_data.indirections_len * 915 sizeof(n->rss_data.indirections_table[0]), 916 }, { 917 .iov_base = &cfg.max_tx_vq, 918 .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) - 919 offsetof(struct virtio_net_rss_config, max_tx_vq), 920 }, { 921 .iov_base = (void *)n->rss_data.key, 922 .iov_len = sizeof(n->rss_data.key), 923 } 924 }; 925 926 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 927 VIRTIO_NET_CTRL_MQ, 928 do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG : 929 VIRTIO_NET_CTRL_MQ_HASH_CONFIG, 930 data, ARRAY_SIZE(data)); 931 if (unlikely(r < 0)) { 932 return r; 933 } 934 935 return 0; 936 } 937 938 static int vhost_vdpa_net_load_mq(VhostVDPAState *s, 939 const VirtIONet *n, 940 struct iovec *out_cursor, 941 struct iovec *in_cursor) 942 { 943 struct virtio_net_ctrl_mq mq; 944 ssize_t r; 945 946 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) { 947 return 0; 948 } 949 950 trace_vhost_vdpa_net_load_mq(s, n->curr_queue_pairs); 951 952 mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs); 953 const struct iovec data = { 954 .iov_base = &mq, 955 .iov_len = sizeof(mq), 956 }; 957 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 958 VIRTIO_NET_CTRL_MQ, 959 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 960 &data, 1); 961 if (unlikely(r < 0)) { 962 return r; 963 } 964 965 if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) { 966 /* load the receive-side scaling state */ 967 r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true); 968 if (unlikely(r < 0)) { 969 return r; 970 } 971 } else if (virtio_vdev_has_feature(&n->parent_obj, 972 VIRTIO_NET_F_HASH_REPORT)) { 973 /* load the hash calculation state */ 974 r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false); 975 if (unlikely(r < 0)) { 976 return r; 977 } 978 } 979 980 return 0; 981 } 982 983 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s, 984 const VirtIONet *n, 985 struct iovec *out_cursor, 986 struct iovec *in_cursor) 987 { 988 uint64_t offloads; 989 ssize_t r; 990 991 if (!virtio_vdev_has_feature(&n->parent_obj, 992 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 993 return 0; 994 } 995 996 if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) { 997 /* 998 * According to VirtIO standard, "Upon feature negotiation 999 * corresponding offload gets enabled to preserve 1000 * backward compatibility.". 1001 * 1002 * Therefore, there is no need to send this CVQ command if the 1003 * driver also enables all supported offloads, which aligns with 1004 * the device's defaults. 1005 * 1006 * Note that the device's defaults can mismatch the driver's 1007 * configuration only at live migration. 1008 */ 1009 return 0; 1010 } 1011 1012 offloads = cpu_to_le64(n->curr_guest_offloads); 1013 const struct iovec data = { 1014 .iov_base = &offloads, 1015 .iov_len = sizeof(offloads), 1016 }; 1017 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 1018 VIRTIO_NET_CTRL_GUEST_OFFLOADS, 1019 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, 1020 &data, 1); 1021 if (unlikely(r < 0)) { 1022 return r; 1023 } 1024 1025 return 0; 1026 } 1027 1028 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s, 1029 struct iovec *out_cursor, 1030 struct iovec *in_cursor, 1031 uint8_t cmd, 1032 uint8_t on) 1033 { 1034 const struct iovec data = { 1035 .iov_base = &on, 1036 .iov_len = sizeof(on), 1037 }; 1038 ssize_t r; 1039 1040 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 1041 VIRTIO_NET_CTRL_RX, cmd, &data, 1); 1042 if (unlikely(r < 0)) { 1043 return r; 1044 } 1045 1046 return 0; 1047 } 1048 1049 static int vhost_vdpa_net_load_rx(VhostVDPAState *s, 1050 const VirtIONet *n, 1051 struct iovec *out_cursor, 1052 struct iovec *in_cursor) 1053 { 1054 ssize_t r; 1055 1056 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) { 1057 return 0; 1058 } 1059 1060 /* 1061 * According to virtio_net_reset(), device turns promiscuous mode 1062 * on by default. 1063 * 1064 * Additionally, according to VirtIO standard, "Since there are 1065 * no guarantees, it can use a hash filter or silently switch to 1066 * allmulti or promiscuous mode if it is given too many addresses.". 1067 * QEMU marks `n->mac_table.uni_overflow` if guest sets too many 1068 * non-multicast MAC addresses, indicating that promiscuous mode 1069 * should be enabled. 1070 * 1071 * Therefore, QEMU should only send this CVQ command if the 1072 * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off, 1073 * which sets promiscuous mode on, different from the device's defaults. 1074 * 1075 * Note that the device's defaults can mismatch the driver's 1076 * configuration only at live migration. 1077 */ 1078 if (!n->mac_table.uni_overflow && !n->promisc) { 1079 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor, 1080 VIRTIO_NET_CTRL_RX_PROMISC, 0); 1081 if (unlikely(r < 0)) { 1082 return r; 1083 } 1084 } 1085 1086 /* 1087 * According to virtio_net_reset(), device turns all-multicast mode 1088 * off by default. 1089 * 1090 * According to VirtIO standard, "Since there are no guarantees, 1091 * it can use a hash filter or silently switch to allmulti or 1092 * promiscuous mode if it is given too many addresses.". QEMU marks 1093 * `n->mac_table.multi_overflow` if guest sets too many 1094 * non-multicast MAC addresses. 1095 * 1096 * Therefore, QEMU should only send this CVQ command if the 1097 * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on, 1098 * which sets all-multicast mode on, different from the device's defaults. 1099 * 1100 * Note that the device's defaults can mismatch the driver's 1101 * configuration only at live migration. 1102 */ 1103 if (n->mac_table.multi_overflow || n->allmulti) { 1104 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor, 1105 VIRTIO_NET_CTRL_RX_ALLMULTI, 1); 1106 if (unlikely(r < 0)) { 1107 return r; 1108 } 1109 } 1110 1111 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) { 1112 return 0; 1113 } 1114 1115 /* 1116 * According to virtio_net_reset(), device turns all-unicast mode 1117 * off by default. 1118 * 1119 * Therefore, QEMU should only send this CVQ command if the driver 1120 * sets all-unicast mode on, different from the device's defaults. 1121 * 1122 * Note that the device's defaults can mismatch the driver's 1123 * configuration only at live migration. 1124 */ 1125 if (n->alluni) { 1126 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor, 1127 VIRTIO_NET_CTRL_RX_ALLUNI, 1); 1128 if (r < 0) { 1129 return r; 1130 } 1131 } 1132 1133 /* 1134 * According to virtio_net_reset(), device turns non-multicast mode 1135 * off by default. 1136 * 1137 * Therefore, QEMU should only send this CVQ command if the driver 1138 * sets non-multicast mode on, different from the device's defaults. 1139 * 1140 * Note that the device's defaults can mismatch the driver's 1141 * configuration only at live migration. 1142 */ 1143 if (n->nomulti) { 1144 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor, 1145 VIRTIO_NET_CTRL_RX_NOMULTI, 1); 1146 if (r < 0) { 1147 return r; 1148 } 1149 } 1150 1151 /* 1152 * According to virtio_net_reset(), device turns non-unicast mode 1153 * off by default. 1154 * 1155 * Therefore, QEMU should only send this CVQ command if the driver 1156 * sets non-unicast mode on, different from the device's defaults. 1157 * 1158 * Note that the device's defaults can mismatch the driver's 1159 * configuration only at live migration. 1160 */ 1161 if (n->nouni) { 1162 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor, 1163 VIRTIO_NET_CTRL_RX_NOUNI, 1); 1164 if (r < 0) { 1165 return r; 1166 } 1167 } 1168 1169 /* 1170 * According to virtio_net_reset(), device turns non-broadcast mode 1171 * off by default. 1172 * 1173 * Therefore, QEMU should only send this CVQ command if the driver 1174 * sets non-broadcast mode on, different from the device's defaults. 1175 * 1176 * Note that the device's defaults can mismatch the driver's 1177 * configuration only at live migration. 1178 */ 1179 if (n->nobcast) { 1180 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor, 1181 VIRTIO_NET_CTRL_RX_NOBCAST, 1); 1182 if (r < 0) { 1183 return r; 1184 } 1185 } 1186 1187 return 0; 1188 } 1189 1190 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s, 1191 const VirtIONet *n, 1192 struct iovec *out_cursor, 1193 struct iovec *in_cursor, 1194 uint16_t vid) 1195 { 1196 const struct iovec data = { 1197 .iov_base = &vid, 1198 .iov_len = sizeof(vid), 1199 }; 1200 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor, 1201 VIRTIO_NET_CTRL_VLAN, 1202 VIRTIO_NET_CTRL_VLAN_ADD, 1203 &data, 1); 1204 if (unlikely(r < 0)) { 1205 return r; 1206 } 1207 1208 return 0; 1209 } 1210 1211 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s, 1212 const VirtIONet *n, 1213 struct iovec *out_cursor, 1214 struct iovec *in_cursor) 1215 { 1216 int r; 1217 1218 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) { 1219 return 0; 1220 } 1221 1222 for (int i = 0; i < MAX_VLAN >> 5; i++) { 1223 for (int j = 0; n->vlans[i] && j <= 0x1f; j++) { 1224 if (n->vlans[i] & (1U << j)) { 1225 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor, 1226 in_cursor, (i << 5) + j); 1227 if (unlikely(r != 0)) { 1228 return r; 1229 } 1230 } 1231 } 1232 } 1233 1234 return 0; 1235 } 1236 1237 static int vhost_vdpa_net_cvq_load(NetClientState *nc) 1238 { 1239 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); 1240 struct vhost_vdpa *v = &s->vhost_vdpa; 1241 const VirtIONet *n; 1242 int r; 1243 struct iovec out_cursor, in_cursor; 1244 1245 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); 1246 1247 r = vhost_vdpa_set_vring_ready(v, v->dev->vq_index); 1248 if (unlikely(r < 0)) { 1249 return r; 1250 } 1251 1252 if (v->shadow_vqs_enabled) { 1253 n = VIRTIO_NET(v->dev->vdev); 1254 vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor); 1255 r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor); 1256 if (unlikely(r < 0)) { 1257 return r; 1258 } 1259 r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor); 1260 if (unlikely(r)) { 1261 return r; 1262 } 1263 r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor); 1264 if (unlikely(r)) { 1265 return r; 1266 } 1267 r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor); 1268 if (unlikely(r)) { 1269 return r; 1270 } 1271 r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor); 1272 if (unlikely(r)) { 1273 return r; 1274 } 1275 1276 /* 1277 * We need to poll and check all pending device's used buffers. 1278 * 1279 * We can poll here since we've had BQL from the time 1280 * we sent the descriptor. 1281 */ 1282 r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status); 1283 if (unlikely(r)) { 1284 return r; 1285 } 1286 } 1287 1288 for (int i = 0; i < v->dev->vq_index; ++i) { 1289 r = vhost_vdpa_set_vring_ready(v, i); 1290 if (unlikely(r < 0)) { 1291 return r; 1292 } 1293 } 1294 1295 return 0; 1296 } 1297 1298 static NetClientInfo net_vhost_vdpa_cvq_info = { 1299 .type = NET_CLIENT_DRIVER_VHOST_VDPA, 1300 .size = sizeof(VhostVDPAState), 1301 .receive = vhost_vdpa_receive, 1302 .start = vhost_vdpa_net_cvq_start, 1303 .load = vhost_vdpa_net_cvq_load, 1304 .stop = vhost_vdpa_net_cvq_stop, 1305 .cleanup = vhost_vdpa_cleanup, 1306 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr, 1307 .get_vnet_hash_supported_types = vhost_vdpa_get_vnet_hash_supported_types, 1308 .has_ufo = vhost_vdpa_has_ufo, 1309 .check_peer_type = vhost_vdpa_check_peer_type, 1310 }; 1311 1312 /* 1313 * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to 1314 * vdpa device. 1315 * 1316 * Considering that QEMU cannot send the entire filter table to the 1317 * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ 1318 * command to enable promiscuous mode to receive all packets, 1319 * according to VirtIO standard, "Since there are no guarantees, 1320 * it can use a hash filter or silently switch to allmulti or 1321 * promiscuous mode if it is given too many addresses.". 1322 * 1323 * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and 1324 * marks `n->mac_table.x_overflow` accordingly, it should have 1325 * the same effect on the device model to receive 1326 * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses. 1327 * The same applies to multicast MAC addresses. 1328 * 1329 * Therefore, QEMU can provide the device model with a fake 1330 * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1) 1331 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast 1332 * MAC addresses. This ensures that the device model marks 1333 * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`, 1334 * allowing all packets to be received, which aligns with the 1335 * state of the vdpa device. 1336 */ 1337 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s, 1338 VirtQueueElement *elem, 1339 struct iovec *out, 1340 const struct iovec *in) 1341 { 1342 struct virtio_net_ctrl_mac mac_data, *mac_ptr; 1343 struct virtio_net_ctrl_hdr *hdr_ptr; 1344 uint32_t cursor; 1345 ssize_t r; 1346 uint8_t on = 1; 1347 1348 /* parse the non-multicast MAC address entries from CVQ command */ 1349 cursor = sizeof(*hdr_ptr); 1350 r = iov_to_buf(elem->out_sg, elem->out_num, cursor, 1351 &mac_data, sizeof(mac_data)); 1352 if (unlikely(r != sizeof(mac_data))) { 1353 /* 1354 * If the CVQ command is invalid, we should simulate the vdpa device 1355 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command 1356 */ 1357 *s->status = VIRTIO_NET_ERR; 1358 return sizeof(*s->status); 1359 } 1360 cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN; 1361 1362 /* parse the multicast MAC address entries from CVQ command */ 1363 r = iov_to_buf(elem->out_sg, elem->out_num, cursor, 1364 &mac_data, sizeof(mac_data)); 1365 if (r != sizeof(mac_data)) { 1366 /* 1367 * If the CVQ command is invalid, we should simulate the vdpa device 1368 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command 1369 */ 1370 *s->status = VIRTIO_NET_ERR; 1371 return sizeof(*s->status); 1372 } 1373 cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN; 1374 1375 /* validate the CVQ command */ 1376 if (iov_size(elem->out_sg, elem->out_num) != cursor) { 1377 /* 1378 * If the CVQ command is invalid, we should simulate the vdpa device 1379 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command 1380 */ 1381 *s->status = VIRTIO_NET_ERR; 1382 return sizeof(*s->status); 1383 } 1384 1385 /* 1386 * According to VirtIO standard, "Since there are no guarantees, 1387 * it can use a hash filter or silently switch to allmulti or 1388 * promiscuous mode if it is given too many addresses.". 1389 * 1390 * Therefore, considering that QEMU is unable to send the entire 1391 * filter table to the vdpa device, it should send the 1392 * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode 1393 */ 1394 hdr_ptr = out->iov_base; 1395 out->iov_len = sizeof(*hdr_ptr) + sizeof(on); 1396 1397 hdr_ptr->class = VIRTIO_NET_CTRL_RX; 1398 hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC; 1399 iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on)); 1400 r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1); 1401 if (unlikely(r < 0)) { 1402 return r; 1403 } 1404 1405 /* 1406 * We can poll here since we've had BQL from the time 1407 * we sent the descriptor. 1408 */ 1409 r = vhost_vdpa_net_svq_poll(s, 1); 1410 if (unlikely(r < sizeof(*s->status))) { 1411 return r; 1412 } 1413 if (*s->status != VIRTIO_NET_OK) { 1414 return sizeof(*s->status); 1415 } 1416 1417 /* 1418 * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ 1419 * command to the device model, including (`MAC_TABLE_ENTRIES` + 1) 1420 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) 1421 * multicast MAC addresses. 1422 * 1423 * By doing so, the device model can mark `n->mac_table.uni_overflow` 1424 * and `n->mac_table.multi_overflow`, enabling all packets to be 1425 * received, which aligns with the state of the vdpa device. 1426 */ 1427 cursor = 0; 1428 uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1, 1429 fake_mul_entries = MAC_TABLE_ENTRIES + 1, 1430 fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) + 1431 sizeof(mac_data) + fake_uni_entries * ETH_ALEN + 1432 sizeof(mac_data) + fake_mul_entries * ETH_ALEN; 1433 1434 assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len()); 1435 out->iov_len = fake_cvq_size; 1436 1437 /* pack the header for fake CVQ command */ 1438 hdr_ptr = out->iov_base + cursor; 1439 hdr_ptr->class = VIRTIO_NET_CTRL_MAC; 1440 hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1441 cursor += sizeof(*hdr_ptr); 1442 1443 /* 1444 * Pack the non-multicast MAC addresses part for fake CVQ command. 1445 * 1446 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC 1447 * addresses provided in CVQ command. Therefore, only the entries 1448 * field need to be prepared in the CVQ command. 1449 */ 1450 mac_ptr = out->iov_base + cursor; 1451 mac_ptr->entries = cpu_to_le32(fake_uni_entries); 1452 cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN; 1453 1454 /* 1455 * Pack the multicast MAC addresses part for fake CVQ command. 1456 * 1457 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC 1458 * addresses provided in CVQ command. Therefore, only the entries 1459 * field need to be prepared in the CVQ command. 1460 */ 1461 mac_ptr = out->iov_base + cursor; 1462 mac_ptr->entries = cpu_to_le32(fake_mul_entries); 1463 1464 /* 1465 * Simulating QEMU poll a vdpa device used buffer 1466 * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command 1467 */ 1468 return sizeof(*s->status); 1469 } 1470 1471 /** 1472 * Validate and copy control virtqueue commands. 1473 * 1474 * Following QEMU guidelines, we offer a copy of the buffers to the device to 1475 * prevent TOCTOU bugs. 1476 */ 1477 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq, 1478 VirtQueueElement *elem, 1479 void *opaque) 1480 { 1481 VhostVDPAState *s = opaque; 1482 size_t in_len; 1483 const struct virtio_net_ctrl_hdr *ctrl; 1484 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1485 /* Out buffer sent to both the vdpa device and the device model */ 1486 struct iovec out = { 1487 .iov_base = s->cvq_cmd_out_buffer, 1488 }; 1489 /* in buffer used for device model */ 1490 const struct iovec model_in = { 1491 .iov_base = &status, 1492 .iov_len = sizeof(status), 1493 }; 1494 /* in buffer used for vdpa device */ 1495 const struct iovec vdpa_in = { 1496 .iov_base = s->status, 1497 .iov_len = sizeof(*s->status), 1498 }; 1499 ssize_t dev_written = -EINVAL; 1500 1501 out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0, 1502 s->cvq_cmd_out_buffer, 1503 vhost_vdpa_net_cvq_cmd_page_len()); 1504 1505 ctrl = s->cvq_cmd_out_buffer; 1506 if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) { 1507 /* 1508 * Guest announce capability is emulated by qemu, so don't forward to 1509 * the device. 1510 */ 1511 dev_written = sizeof(status); 1512 *s->status = VIRTIO_NET_OK; 1513 } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC && 1514 ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET && 1515 iov_size(elem->out_sg, elem->out_num) > out.iov_len)) { 1516 /* 1517 * Due to the size limitation of the out buffer sent to the vdpa device, 1518 * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive 1519 * MAC addresses set by the driver for the filter table can cause 1520 * truncation of the CVQ command in QEMU. As a result, the vdpa device 1521 * rejects the flawed CVQ command. 1522 * 1523 * Therefore, QEMU must handle this situation instead of sending 1524 * the CVQ command directly. 1525 */ 1526 dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem, 1527 &out, &vdpa_in); 1528 if (unlikely(dev_written < 0)) { 1529 goto out; 1530 } 1531 } else { 1532 ssize_t r; 1533 r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1); 1534 if (unlikely(r < 0)) { 1535 dev_written = r; 1536 goto out; 1537 } 1538 1539 /* 1540 * We can poll here since we've had BQL from the time 1541 * we sent the descriptor. 1542 */ 1543 dev_written = vhost_vdpa_net_svq_poll(s, 1); 1544 } 1545 1546 if (unlikely(dev_written < sizeof(status))) { 1547 error_report("Insufficient written data (%zu)", dev_written); 1548 goto out; 1549 } 1550 1551 if (*s->status != VIRTIO_NET_OK) { 1552 goto out; 1553 } 1554 1555 status = VIRTIO_NET_ERR; 1556 virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1); 1557 if (status != VIRTIO_NET_OK) { 1558 error_report("Bad CVQ processing in model"); 1559 } 1560 1561 out: 1562 in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status, 1563 sizeof(status)); 1564 if (unlikely(in_len < sizeof(status))) { 1565 error_report("Bad device CVQ written length"); 1566 } 1567 vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status))); 1568 /* 1569 * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when 1570 * the function successfully forwards the CVQ command, indicated 1571 * by a non-negative value of `dev_written`. Otherwise, it still 1572 * belongs to SVQ. 1573 * This function should only free the `elem` when it owns. 1574 */ 1575 if (dev_written >= 0) { 1576 g_free(elem); 1577 } 1578 return dev_written < 0 ? dev_written : 0; 1579 } 1580 1581 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = { 1582 .avail_handler = vhost_vdpa_net_handle_ctrl_avail, 1583 }; 1584 1585 /** 1586 * Probe if CVQ is isolated 1587 * 1588 * @device_fd The vdpa device fd 1589 * @features Features offered by the device. 1590 * @cvq_index The control vq pair index 1591 * 1592 * Returns <0 in case of failure, 0 if false and 1 if true. 1593 */ 1594 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features, 1595 int cvq_index, Error **errp) 1596 { 1597 ERRP_GUARD(); 1598 uint64_t backend_features; 1599 int64_t cvq_group; 1600 uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE | 1601 VIRTIO_CONFIG_S_DRIVER; 1602 int r; 1603 1604 r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features); 1605 if (unlikely(r < 0)) { 1606 error_setg_errno(errp, errno, "Cannot get vdpa backend_features"); 1607 return r; 1608 } 1609 1610 if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) { 1611 return 0; 1612 } 1613 1614 r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status); 1615 if (unlikely(r)) { 1616 error_setg_errno(errp, -r, "Cannot set device status"); 1617 goto out; 1618 } 1619 1620 r = ioctl(device_fd, VHOST_SET_FEATURES, &features); 1621 if (unlikely(r)) { 1622 error_setg_errno(errp, -r, "Cannot set features"); 1623 goto out; 1624 } 1625 1626 status |= VIRTIO_CONFIG_S_FEATURES_OK; 1627 r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status); 1628 if (unlikely(r)) { 1629 error_setg_errno(errp, -r, "Cannot set device status"); 1630 goto out; 1631 } 1632 1633 cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp); 1634 if (unlikely(cvq_group < 0)) { 1635 if (cvq_group != -ENOTSUP) { 1636 r = cvq_group; 1637 goto out; 1638 } 1639 1640 /* 1641 * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend 1642 * support ASID even if the parent driver does not. The CVQ cannot be 1643 * isolated in this case. 1644 */ 1645 error_free(*errp); 1646 *errp = NULL; 1647 r = 0; 1648 goto out; 1649 } 1650 1651 for (int i = 0; i < cvq_index; ++i) { 1652 int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp); 1653 if (unlikely(group < 0)) { 1654 r = group; 1655 goto out; 1656 } 1657 1658 if (group == (int64_t)cvq_group) { 1659 r = 0; 1660 goto out; 1661 } 1662 } 1663 1664 r = 1; 1665 1666 out: 1667 status = 0; 1668 ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status); 1669 return r; 1670 } 1671 1672 static NetClientState *net_vhost_vdpa_init(NetClientState *peer, 1673 const char *device, 1674 const char *name, 1675 int vdpa_device_fd, 1676 int queue_pair_index, 1677 int nvqs, 1678 bool is_datapath, 1679 bool svq, 1680 struct vhost_vdpa_iova_range iova_range, 1681 uint64_t features, 1682 VhostVDPAShared *shared, 1683 Error **errp) 1684 { 1685 NetClientState *nc = NULL; 1686 VhostVDPAState *s; 1687 int ret = 0; 1688 assert(name); 1689 int cvq_isolated = 0; 1690 1691 if (is_datapath) { 1692 nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, 1693 name); 1694 } else { 1695 cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features, 1696 queue_pair_index * 2, 1697 errp); 1698 if (unlikely(cvq_isolated < 0)) { 1699 return NULL; 1700 } 1701 1702 nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer, 1703 device, name); 1704 } 1705 qemu_set_info_str(nc, TYPE_VHOST_VDPA); 1706 s = DO_UPCAST(VhostVDPAState, nc, nc); 1707 1708 s->vhost_vdpa.index = queue_pair_index; 1709 s->always_svq = svq; 1710 s->migration_state.notify = NULL; 1711 s->vhost_vdpa.shadow_vqs_enabled = svq; 1712 if (queue_pair_index == 0) { 1713 vhost_vdpa_net_valid_svq_features(features, 1714 &s->vhost_vdpa.migration_blocker); 1715 s->vhost_vdpa.shared = g_new0(VhostVDPAShared, 1); 1716 s->vhost_vdpa.shared->device_fd = vdpa_device_fd; 1717 s->vhost_vdpa.shared->iova_range = iova_range; 1718 s->vhost_vdpa.shared->shadow_data = svq; 1719 s->vhost_vdpa.shared->iova_tree = vhost_iova_tree_new(iova_range.first, 1720 iova_range.last); 1721 } else if (!is_datapath) { 1722 s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(), 1723 PROT_READ | PROT_WRITE, 1724 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1725 s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(), 1726 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, 1727 -1, 0); 1728 1729 s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops; 1730 s->vhost_vdpa.shadow_vq_ops_opaque = s; 1731 s->cvq_isolated = cvq_isolated; 1732 } 1733 if (queue_pair_index != 0) { 1734 s->vhost_vdpa.shared = shared; 1735 } 1736 1737 ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs); 1738 if (ret) { 1739 qemu_del_net_client(nc); 1740 return NULL; 1741 } 1742 1743 return nc; 1744 } 1745 1746 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp) 1747 { 1748 int ret = ioctl(fd, VHOST_GET_FEATURES, features); 1749 if (unlikely(ret < 0)) { 1750 error_setg_errno(errp, errno, 1751 "Fail to query features from vhost-vDPA device"); 1752 } 1753 return ret; 1754 } 1755 1756 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features, 1757 int *has_cvq, Error **errp) 1758 { 1759 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 1760 g_autofree struct vhost_vdpa_config *config = NULL; 1761 __virtio16 *max_queue_pairs; 1762 int ret; 1763 1764 if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) { 1765 *has_cvq = 1; 1766 } else { 1767 *has_cvq = 0; 1768 } 1769 1770 if (features & (1 << VIRTIO_NET_F_MQ)) { 1771 config = g_malloc0(config_size + sizeof(*max_queue_pairs)); 1772 config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs); 1773 config->len = sizeof(*max_queue_pairs); 1774 1775 ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config); 1776 if (ret) { 1777 error_setg(errp, "Fail to get config from vhost-vDPA device"); 1778 return -ret; 1779 } 1780 1781 max_queue_pairs = (__virtio16 *)&config->buf; 1782 1783 return lduw_le_p(max_queue_pairs); 1784 } 1785 1786 return 1; 1787 } 1788 1789 int net_init_vhost_vdpa(const Netdev *netdev, const char *name, 1790 NetClientState *peer, Error **errp) 1791 { 1792 ERRP_GUARD(); 1793 const NetdevVhostVDPAOptions *opts; 1794 uint64_t features; 1795 int vdpa_device_fd; 1796 g_autofree NetClientState **ncs = NULL; 1797 struct vhost_vdpa_iova_range iova_range; 1798 NetClientState *nc; 1799 int queue_pairs, r, i = 0, has_cvq = 0; 1800 1801 assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA); 1802 opts = &netdev->u.vhost_vdpa; 1803 if (!opts->vhostdev && !opts->vhostfd) { 1804 error_setg(errp, 1805 "vhost-vdpa: neither vhostdev= nor vhostfd= was specified"); 1806 return -1; 1807 } 1808 1809 if (opts->vhostdev && opts->vhostfd) { 1810 error_setg(errp, 1811 "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive"); 1812 return -1; 1813 } 1814 1815 if (opts->vhostdev) { 1816 vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp); 1817 if (vdpa_device_fd == -1) { 1818 return -errno; 1819 } 1820 } else { 1821 /* has_vhostfd */ 1822 vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp); 1823 if (vdpa_device_fd == -1) { 1824 error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: "); 1825 return -1; 1826 } 1827 } 1828 1829 r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp); 1830 if (unlikely(r < 0)) { 1831 goto err; 1832 } 1833 1834 queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features, 1835 &has_cvq, errp); 1836 if (queue_pairs < 0) { 1837 qemu_close(vdpa_device_fd); 1838 return queue_pairs; 1839 } 1840 1841 r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range); 1842 if (unlikely(r < 0)) { 1843 error_setg(errp, "vhost-vdpa: get iova range failed: %s", 1844 strerror(-r)); 1845 goto err; 1846 } 1847 1848 if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) { 1849 goto err; 1850 } 1851 1852 ncs = g_malloc0(sizeof(*ncs) * queue_pairs); 1853 1854 for (i = 0; i < queue_pairs; i++) { 1855 VhostVDPAShared *shared = NULL; 1856 1857 if (i) { 1858 shared = DO_UPCAST(VhostVDPAState, nc, ncs[0])->vhost_vdpa.shared; 1859 } 1860 ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name, 1861 vdpa_device_fd, i, 2, true, opts->x_svq, 1862 iova_range, features, shared, errp); 1863 if (!ncs[i]) 1864 goto err; 1865 } 1866 1867 if (has_cvq) { 1868 VhostVDPAState *s0 = DO_UPCAST(VhostVDPAState, nc, ncs[0]); 1869 VhostVDPAShared *shared = s0->vhost_vdpa.shared; 1870 1871 nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name, 1872 vdpa_device_fd, i, 1, false, 1873 opts->x_svq, iova_range, features, shared, 1874 errp); 1875 if (!nc) 1876 goto err; 1877 } 1878 1879 return 0; 1880 1881 err: 1882 if (i) { 1883 for (i--; i >= 0; i--) { 1884 qemu_del_net_client(ncs[i]); 1885 } 1886 } 1887 1888 qemu_close(vdpa_device_fd); 1889 1890 return -1; 1891 } 1892