xref: /openbmc/qemu/net/vhost-vdpa.c (revision 8c5e9809225c387026476f5eefc2f8ae749a72f7)
1 /*
2  * vhost-vdpa.c
3  *
4  * Copyright(c) 2017-2018 Intel Corporation.
5  * Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/log.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
26 #include <err.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/migration.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
32 
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState {
35     NetClientState nc;
36     struct vhost_vdpa vhost_vdpa;
37     Notifier migration_state;
38     VHostNetState *vhost_net;
39 
40     /* Control commands shadow buffers */
41     void *cvq_cmd_out_buffer;
42     virtio_net_ctrl_ack *status;
43 
44     /* The device always have SVQ enabled */
45     bool always_svq;
46 
47     /* The device can isolate CVQ in its own ASID */
48     bool cvq_isolated;
49 
50     bool started;
51 } VhostVDPAState;
52 
53 /*
54  * The array is sorted alphabetically in ascending order,
55  * with the exception of VHOST_INVALID_FEATURE_BIT,
56  * which should always be the last entry.
57  */
58 const int vdpa_feature_bits[] = {
59     VIRTIO_F_ANY_LAYOUT,
60     VIRTIO_F_IOMMU_PLATFORM,
61     VIRTIO_F_NOTIFY_ON_EMPTY,
62     VIRTIO_F_RING_PACKED,
63     VIRTIO_F_RING_RESET,
64     VIRTIO_F_VERSION_1,
65     VIRTIO_NET_F_CSUM,
66     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
67     VIRTIO_NET_F_CTRL_MAC_ADDR,
68     VIRTIO_NET_F_CTRL_RX,
69     VIRTIO_NET_F_CTRL_RX_EXTRA,
70     VIRTIO_NET_F_CTRL_VLAN,
71     VIRTIO_NET_F_CTRL_VQ,
72     VIRTIO_NET_F_GSO,
73     VIRTIO_NET_F_GUEST_CSUM,
74     VIRTIO_NET_F_GUEST_ECN,
75     VIRTIO_NET_F_GUEST_TSO4,
76     VIRTIO_NET_F_GUEST_TSO6,
77     VIRTIO_NET_F_GUEST_UFO,
78     VIRTIO_NET_F_GUEST_USO4,
79     VIRTIO_NET_F_GUEST_USO6,
80     VIRTIO_NET_F_HASH_REPORT,
81     VIRTIO_NET_F_HOST_ECN,
82     VIRTIO_NET_F_HOST_TSO4,
83     VIRTIO_NET_F_HOST_TSO6,
84     VIRTIO_NET_F_HOST_UFO,
85     VIRTIO_NET_F_HOST_USO,
86     VIRTIO_NET_F_MQ,
87     VIRTIO_NET_F_MRG_RXBUF,
88     VIRTIO_NET_F_MTU,
89     VIRTIO_NET_F_RSS,
90     VIRTIO_NET_F_STATUS,
91     VIRTIO_RING_F_EVENT_IDX,
92     VIRTIO_RING_F_INDIRECT_DESC,
93 
94     /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
95     VHOST_INVALID_FEATURE_BIT
96 };
97 
98 /** Supported device specific feature bits with SVQ */
99 static const uint64_t vdpa_svq_device_features =
100     BIT_ULL(VIRTIO_NET_F_CSUM) |
101     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
102     BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
103     BIT_ULL(VIRTIO_NET_F_MTU) |
104     BIT_ULL(VIRTIO_NET_F_MAC) |
105     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
106     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
107     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
108     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
109     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
110     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
111     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
112     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
113     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
114     BIT_ULL(VIRTIO_NET_F_STATUS) |
115     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
116     BIT_ULL(VIRTIO_NET_F_CTRL_RX) |
117     BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) |
118     BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) |
119     BIT_ULL(VIRTIO_NET_F_MQ) |
120     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
121     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
122     /* VHOST_F_LOG_ALL is exposed by SVQ */
123     BIT_ULL(VHOST_F_LOG_ALL) |
124     BIT_ULL(VIRTIO_NET_F_HASH_REPORT) |
125     BIT_ULL(VIRTIO_NET_F_RSS) |
126     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
127     BIT_ULL(VIRTIO_NET_F_STANDBY) |
128     BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
129 
130 #define VHOST_VDPA_NET_CVQ_ASID 1
131 
132 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
133 {
134     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
135     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
136     return s->vhost_net;
137 }
138 
139 static size_t vhost_vdpa_net_cvq_cmd_len(void)
140 {
141     /*
142      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
143      * In buffer is always 1 byte, so it should fit here
144      */
145     return sizeof(struct virtio_net_ctrl_hdr) +
146            2 * sizeof(struct virtio_net_ctrl_mac) +
147            MAC_TABLE_ENTRIES * ETH_ALEN;
148 }
149 
150 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
151 {
152     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
153 }
154 
155 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
156 {
157     uint64_t invalid_dev_features =
158         features & ~vdpa_svq_device_features &
159         /* Transport are all accepted at this point */
160         ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
161                          VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
162 
163     if (invalid_dev_features) {
164         error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
165                    invalid_dev_features);
166         return false;
167     }
168 
169     return vhost_svq_valid_features(features, errp);
170 }
171 
172 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
173 {
174     uint32_t device_id;
175     int ret;
176     struct vhost_dev *hdev;
177 
178     hdev = (struct vhost_dev *)&net->dev;
179     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
180     if (device_id != VIRTIO_ID_NET) {
181         return -ENOTSUP;
182     }
183     return ret;
184 }
185 
186 static int vhost_vdpa_add(NetClientState *ncs, void *be,
187                           int queue_pair_index, int nvqs)
188 {
189     VhostNetOptions options;
190     struct vhost_net *net = NULL;
191     VhostVDPAState *s;
192     int ret;
193 
194     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
195     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
196     s = DO_UPCAST(VhostVDPAState, nc, ncs);
197     options.net_backend = ncs;
198     options.opaque      = be;
199     options.busyloop_timeout = 0;
200     options.nvqs = nvqs;
201 
202     net = vhost_net_init(&options);
203     if (!net) {
204         error_report("failed to init vhost_net for queue");
205         goto err_init;
206     }
207     s->vhost_net = net;
208     ret = vhost_vdpa_net_check_device_id(net);
209     if (ret) {
210         goto err_check;
211     }
212     return 0;
213 err_check:
214     vhost_net_cleanup(net);
215     g_free(net);
216 err_init:
217     return -1;
218 }
219 
220 static void vhost_vdpa_cleanup(NetClientState *nc)
221 {
222     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
223 
224     /*
225      * If a peer NIC is attached, do not cleanup anything.
226      * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
227      * when the guest is shutting down.
228      */
229     if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
230         return;
231     }
232     munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len());
233     munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len());
234     if (s->vhost_net) {
235         vhost_net_cleanup(s->vhost_net);
236         g_free(s->vhost_net);
237         s->vhost_net = NULL;
238     }
239      if (s->vhost_vdpa.device_fd >= 0) {
240         qemu_close(s->vhost_vdpa.device_fd);
241         s->vhost_vdpa.device_fd = -1;
242     }
243     if (s->vhost_vdpa.index != 0) {
244         return;
245     }
246     g_free(s->vhost_vdpa.shared);
247 }
248 
249 /** Dummy SetSteeringEBPF to support RSS for vhost-vdpa backend  */
250 static bool vhost_vdpa_set_steering_ebpf(NetClientState *nc, int prog_fd)
251 {
252     return true;
253 }
254 
255 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
256 {
257     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
258 
259     return true;
260 }
261 
262 static bool vhost_vdpa_has_ufo(NetClientState *nc)
263 {
264     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
265     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
266     uint64_t features = 0;
267     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
268     features = vhost_net_get_features(s->vhost_net, features);
269     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
270 
271 }
272 
273 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
274                                        Error **errp)
275 {
276     const char *driver = object_class_get_name(oc);
277 
278     if (!g_str_has_prefix(driver, "virtio-net-")) {
279         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
280         return false;
281     }
282 
283     return true;
284 }
285 
286 /** Dummy receive in case qemu falls back to userland tap networking */
287 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
288                                   size_t size)
289 {
290     return size;
291 }
292 
293 /** From any vdpa net client, get the netclient of the first queue pair */
294 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
295 {
296     NICState *nic = qemu_get_nic(s->nc.peer);
297     NetClientState *nc0 = qemu_get_peer(nic->ncs, 0);
298 
299     return DO_UPCAST(VhostVDPAState, nc, nc0);
300 }
301 
302 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
303 {
304     struct vhost_vdpa *v = &s->vhost_vdpa;
305     VirtIONet *n;
306     VirtIODevice *vdev;
307     int data_queue_pairs, cvq, r;
308 
309     /* We are only called on the first data vqs and only if x-svq is not set */
310     if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
311         return;
312     }
313 
314     vdev = v->dev->vdev;
315     n = VIRTIO_NET(vdev);
316     if (!n->vhost_started) {
317         return;
318     }
319 
320     data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
321     cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
322                                   n->max_ncs - n->max_queue_pairs : 0;
323     /*
324      * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
325      * in the future and resume the device if read-only operations between
326      * suspend and reset goes wrong.
327      */
328     vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
329 
330     /* Start will check migration setup_or_active to configure or not SVQ */
331     r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
332     if (unlikely(r < 0)) {
333         error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
334     }
335 }
336 
337 static void vdpa_net_migration_state_notifier(Notifier *notifier, void *data)
338 {
339     MigrationState *migration = data;
340     VhostVDPAState *s = container_of(notifier, VhostVDPAState,
341                                      migration_state);
342 
343     if (migration_in_setup(migration)) {
344         vhost_vdpa_net_log_global_enable(s, true);
345     } else if (migration_has_failed(migration)) {
346         vhost_vdpa_net_log_global_enable(s, false);
347     }
348 }
349 
350 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
351 {
352     struct vhost_vdpa *v = &s->vhost_vdpa;
353 
354     migration_add_notifier(&s->migration_state,
355                            vdpa_net_migration_state_notifier);
356     if (v->shadow_vqs_enabled) {
357         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
358                                            v->iova_range.last);
359     }
360 }
361 
362 static int vhost_vdpa_net_data_start(NetClientState *nc)
363 {
364     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
365     struct vhost_vdpa *v = &s->vhost_vdpa;
366 
367     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
368 
369     if (s->always_svq ||
370         migration_is_setup_or_active(migrate_get_current()->state)) {
371         v->shadow_vqs_enabled = true;
372         v->shadow_data = true;
373     } else {
374         v->shadow_vqs_enabled = false;
375         v->shadow_data = false;
376     }
377 
378     if (v->index == 0) {
379         vhost_vdpa_net_data_start_first(s);
380         return 0;
381     }
382 
383     if (v->shadow_vqs_enabled) {
384         VhostVDPAState *s0 = vhost_vdpa_net_first_nc_vdpa(s);
385         v->iova_tree = s0->vhost_vdpa.iova_tree;
386     }
387 
388     return 0;
389 }
390 
391 static int vhost_vdpa_net_data_load(NetClientState *nc)
392 {
393     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
394     struct vhost_vdpa *v = &s->vhost_vdpa;
395     bool has_cvq = v->dev->vq_index_end % 2;
396 
397     if (has_cvq) {
398         return 0;
399     }
400 
401     for (int i = 0; i < v->dev->nvqs; ++i) {
402         vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index);
403     }
404     return 0;
405 }
406 
407 static void vhost_vdpa_net_client_stop(NetClientState *nc)
408 {
409     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
410     struct vhost_dev *dev;
411 
412     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
413 
414     if (s->vhost_vdpa.index == 0) {
415         migration_remove_notifier(&s->migration_state);
416     }
417 
418     dev = s->vhost_vdpa.dev;
419     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
420         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
421     } else {
422         s->vhost_vdpa.iova_tree = NULL;
423     }
424 }
425 
426 static NetClientInfo net_vhost_vdpa_info = {
427         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
428         .size = sizeof(VhostVDPAState),
429         .receive = vhost_vdpa_receive,
430         .start = vhost_vdpa_net_data_start,
431         .load = vhost_vdpa_net_data_load,
432         .stop = vhost_vdpa_net_client_stop,
433         .cleanup = vhost_vdpa_cleanup,
434         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
435         .has_ufo = vhost_vdpa_has_ufo,
436         .check_peer_type = vhost_vdpa_check_peer_type,
437         .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
438 };
439 
440 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
441                                           Error **errp)
442 {
443     struct vhost_vring_state state = {
444         .index = vq_index,
445     };
446     int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
447 
448     if (unlikely(r < 0)) {
449         r = -errno;
450         error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index);
451         return r;
452     }
453 
454     return state.num;
455 }
456 
457 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
458                                            unsigned vq_group,
459                                            unsigned asid_num)
460 {
461     struct vhost_vring_state asid = {
462         .index = vq_group,
463         .num = asid_num,
464     };
465     int r;
466 
467     r = ioctl(v->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
468     if (unlikely(r < 0)) {
469         error_report("Can't set vq group %u asid %u, errno=%d (%s)",
470                      asid.index, asid.num, errno, g_strerror(errno));
471     }
472     return r;
473 }
474 
475 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
476 {
477     VhostIOVATree *tree = v->iova_tree;
478     DMAMap needle = {
479         /*
480          * No need to specify size or to look for more translations since
481          * this contiguous chunk was allocated by us.
482          */
483         .translated_addr = (hwaddr)(uintptr_t)addr,
484     };
485     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
486     int r;
487 
488     if (unlikely(!map)) {
489         error_report("Cannot locate expected map");
490         return;
491     }
492 
493     r = vhost_vdpa_dma_unmap(v, v->address_space_id, map->iova, map->size + 1);
494     if (unlikely(r != 0)) {
495         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
496     }
497 
498     vhost_iova_tree_remove(tree, *map);
499 }
500 
501 /** Map CVQ buffer. */
502 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
503                                   bool write)
504 {
505     DMAMap map = {};
506     int r;
507 
508     map.translated_addr = (hwaddr)(uintptr_t)buf;
509     map.size = size - 1;
510     map.perm = write ? IOMMU_RW : IOMMU_RO,
511     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
512     if (unlikely(r != IOVA_OK)) {
513         error_report("Cannot map injected element");
514         return r;
515     }
516 
517     r = vhost_vdpa_dma_map(v, v->address_space_id, map.iova,
518                            vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
519     if (unlikely(r < 0)) {
520         goto dma_map_err;
521     }
522 
523     return 0;
524 
525 dma_map_err:
526     vhost_iova_tree_remove(v->iova_tree, map);
527     return r;
528 }
529 
530 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
531 {
532     VhostVDPAState *s, *s0;
533     struct vhost_vdpa *v;
534     int64_t cvq_group;
535     int r;
536     Error *err = NULL;
537 
538     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
539 
540     s = DO_UPCAST(VhostVDPAState, nc, nc);
541     v = &s->vhost_vdpa;
542 
543     s0 = vhost_vdpa_net_first_nc_vdpa(s);
544     v->shadow_data = s0->vhost_vdpa.shadow_vqs_enabled;
545     v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled;
546     s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
547 
548     if (s->vhost_vdpa.shadow_data) {
549         /* SVQ is already configured for all virtqueues */
550         goto out;
551     }
552 
553     /*
554      * If we early return in these cases SVQ will not be enabled. The migration
555      * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
556      */
557     if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) {
558         return 0;
559     }
560 
561     if (!s->cvq_isolated) {
562         return 0;
563     }
564 
565     cvq_group = vhost_vdpa_get_vring_group(v->device_fd,
566                                            v->dev->vq_index_end - 1,
567                                            &err);
568     if (unlikely(cvq_group < 0)) {
569         error_report_err(err);
570         return cvq_group;
571     }
572 
573     r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
574     if (unlikely(r < 0)) {
575         return r;
576     }
577 
578     v->shadow_vqs_enabled = true;
579     s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
580 
581 out:
582     if (!s->vhost_vdpa.shadow_vqs_enabled) {
583         return 0;
584     }
585 
586     if (s0->vhost_vdpa.iova_tree) {
587         /*
588          * SVQ is already configured for all virtqueues.  Reuse IOVA tree for
589          * simplicity, whether CVQ shares ASID with guest or not, because:
590          * - Memory listener need access to guest's memory addresses allocated
591          *   in the IOVA tree.
592          * - There should be plenty of IOVA address space for both ASID not to
593          *   worry about collisions between them.  Guest's translations are
594          *   still validated with virtio virtqueue_pop so there is no risk for
595          *   the guest to access memory that it shouldn't.
596          *
597          * To allocate a iova tree per ASID is doable but it complicates the
598          * code and it is not worth it for the moment.
599          */
600         v->iova_tree = s0->vhost_vdpa.iova_tree;
601     } else {
602         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
603                                            v->iova_range.last);
604     }
605 
606     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
607                                vhost_vdpa_net_cvq_cmd_page_len(), false);
608     if (unlikely(r < 0)) {
609         return r;
610     }
611 
612     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
613                                vhost_vdpa_net_cvq_cmd_page_len(), true);
614     if (unlikely(r < 0)) {
615         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
616     }
617 
618     return r;
619 }
620 
621 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
622 {
623     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
624 
625     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
626 
627     if (s->vhost_vdpa.shadow_vqs_enabled) {
628         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
629         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
630     }
631 
632     vhost_vdpa_net_client_stop(nc);
633 }
634 
635 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s,
636                                     const struct iovec *out_sg, size_t out_num,
637                                     const struct iovec *in_sg, size_t in_num)
638 {
639     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
640     int r;
641 
642     r = vhost_svq_add(svq, out_sg, out_num, in_sg, in_num, NULL);
643     if (unlikely(r != 0)) {
644         if (unlikely(r == -ENOSPC)) {
645             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
646                           __func__);
647         }
648     }
649 
650     return r;
651 }
652 
653 /*
654  * Convenience wrapper to poll SVQ for multiple control commands.
655  *
656  * Caller should hold the BQL when invoking this function, and should take
657  * the answer before SVQ pulls by itself when BQL is released.
658  */
659 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight)
660 {
661     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
662     return vhost_svq_poll(svq, cmds_in_flight);
663 }
664 
665 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s,
666                                              struct iovec *out_cursor,
667                                              struct iovec *in_cursor)
668 {
669     /* reset the cursor of the output buffer for the device */
670     out_cursor->iov_base = s->cvq_cmd_out_buffer;
671     out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
672 
673     /* reset the cursor of the in buffer for the device */
674     in_cursor->iov_base = s->status;
675     in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
676 }
677 
678 /*
679  * Poll SVQ for multiple pending control commands and check the device's ack.
680  *
681  * Caller should hold the BQL when invoking this function.
682  *
683  * @s: The VhostVDPAState
684  * @len: The length of the pending status shadow buffer
685  */
686 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len)
687 {
688     /* device uses a one-byte length ack for each control command */
689     ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len);
690     if (unlikely(dev_written != len)) {
691         return -EIO;
692     }
693 
694     /* check the device's ack */
695     for (int i = 0; i < len; ++i) {
696         if (s->status[i] != VIRTIO_NET_OK) {
697             return -EIO;
698         }
699     }
700     return 0;
701 }
702 
703 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s,
704                                        struct iovec *out_cursor,
705                                        struct iovec *in_cursor, uint8_t class,
706                                        uint8_t cmd, const struct iovec *data_sg,
707                                        size_t data_num)
708 {
709     const struct virtio_net_ctrl_hdr ctrl = {
710         .class = class,
711         .cmd = cmd,
712     };
713     size_t data_size = iov_size(data_sg, data_num), cmd_size;
714     struct iovec out, in;
715     ssize_t r;
716     unsigned dummy_cursor_iov_cnt;
717     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
718 
719     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
720     cmd_size = sizeof(ctrl) + data_size;
721     if (vhost_svq_available_slots(svq) < 2 ||
722         iov_size(out_cursor, 1) < cmd_size) {
723         /*
724          * It is time to flush all pending control commands if SVQ is full
725          * or control commands shadow buffers are full.
726          *
727          * We can poll here since we've had BQL from the time
728          * we sent the descriptor.
729          */
730         r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base -
731                                      (void *)s->status);
732         if (unlikely(r < 0)) {
733             return r;
734         }
735 
736         vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor);
737     }
738 
739     /* pack the CVQ command header */
740     iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl));
741     /* pack the CVQ command command-specific-data */
742     iov_to_buf(data_sg, data_num, 0,
743                out_cursor->iov_base + sizeof(ctrl), data_size);
744 
745     /* extract the required buffer from the cursor for output */
746     iov_copy(&out, 1, out_cursor, 1, 0, cmd_size);
747     /* extract the required buffer from the cursor for input */
748     iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status));
749 
750     r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1);
751     if (unlikely(r < 0)) {
752         return r;
753     }
754 
755     /* iterate the cursors */
756     dummy_cursor_iov_cnt = 1;
757     iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size);
758     dummy_cursor_iov_cnt = 1;
759     iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status));
760 
761     return 0;
762 }
763 
764 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n,
765                                    struct iovec *out_cursor,
766                                    struct iovec *in_cursor)
767 {
768     if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
769         const struct iovec data = {
770             .iov_base = (void *)n->mac,
771             .iov_len = sizeof(n->mac),
772         };
773         ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
774                                             VIRTIO_NET_CTRL_MAC,
775                                             VIRTIO_NET_CTRL_MAC_ADDR_SET,
776                                             &data, 1);
777         if (unlikely(r < 0)) {
778             return r;
779         }
780     }
781 
782     /*
783      * According to VirtIO standard, "The device MUST have an
784      * empty MAC filtering table on reset.".
785      *
786      * Therefore, there is no need to send this CVQ command if the
787      * driver also sets an empty MAC filter table, which aligns with
788      * the device's defaults.
789      *
790      * Note that the device's defaults can mismatch the driver's
791      * configuration only at live migration.
792      */
793     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) ||
794         n->mac_table.in_use == 0) {
795         return 0;
796     }
797 
798     uint32_t uni_entries = n->mac_table.first_multi,
799              uni_macs_size = uni_entries * ETH_ALEN,
800              mul_entries = n->mac_table.in_use - uni_entries,
801              mul_macs_size = mul_entries * ETH_ALEN;
802     struct virtio_net_ctrl_mac uni = {
803         .entries = cpu_to_le32(uni_entries),
804     };
805     struct virtio_net_ctrl_mac mul = {
806         .entries = cpu_to_le32(mul_entries),
807     };
808     const struct iovec data[] = {
809         {
810             .iov_base = &uni,
811             .iov_len = sizeof(uni),
812         }, {
813             .iov_base = n->mac_table.macs,
814             .iov_len = uni_macs_size,
815         }, {
816             .iov_base = &mul,
817             .iov_len = sizeof(mul),
818         }, {
819             .iov_base = &n->mac_table.macs[uni_macs_size],
820             .iov_len = mul_macs_size,
821         },
822     };
823     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
824                                         VIRTIO_NET_CTRL_MAC,
825                                         VIRTIO_NET_CTRL_MAC_TABLE_SET,
826                                         data, ARRAY_SIZE(data));
827     if (unlikely(r < 0)) {
828         return r;
829     }
830 
831     return 0;
832 }
833 
834 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
835                                    struct iovec *out_cursor,
836                                    struct iovec *in_cursor, bool do_rss)
837 {
838     struct virtio_net_rss_config cfg = {};
839     ssize_t r;
840     g_autofree uint16_t *table = NULL;
841 
842     /*
843      * According to VirtIO standard, "Initially the device has all hash
844      * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
845      *
846      * Therefore, there is no need to send this CVQ command if the
847      * driver disables the all hash types, which aligns with
848      * the device's defaults.
849      *
850      * Note that the device's defaults can mismatch the driver's
851      * configuration only at live migration.
852      */
853     if (!n->rss_data.enabled ||
854         n->rss_data.hash_types == VIRTIO_NET_HASH_REPORT_NONE) {
855         return 0;
856     }
857 
858     table = g_malloc_n(n->rss_data.indirections_len,
859                        sizeof(n->rss_data.indirections_table[0]));
860     cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
861 
862     if (do_rss) {
863         /*
864          * According to VirtIO standard, "Number of entries in indirection_table
865          * is (indirection_table_mask + 1)".
866          */
867         cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len -
868                                                  1);
869         cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue);
870         for (int i = 0; i < n->rss_data.indirections_len; ++i) {
871             table[i] = cpu_to_le16(n->rss_data.indirections_table[i]);
872         }
873         cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs);
874     } else {
875         /*
876          * According to VirtIO standard, "Field reserved MUST contain zeroes.
877          * It is defined to make the structure to match the layout of
878          * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
879          *
880          * Therefore, we need to zero the fields in
881          * struct virtio_net_rss_config, which corresponds to the
882          * `reserved` field in struct virtio_net_hash_config.
883          *
884          * Note that all other fields are zeroed at their definitions,
885          * except for the `indirection_table` field, where the actual data
886          * is stored in the `table` variable to ensure compatibility
887          * with RSS case. Therefore, we need to zero the `table` variable here.
888          */
889         table[0] = 0;
890     }
891 
892     /*
893      * Considering that virtio_net_handle_rss() currently does not restore
894      * the hash key length parsed from the CVQ command sent from the guest
895      * into n->rss_data and uses the maximum key length in other code, so
896      * we also employ the maximum key length here.
897      */
898     cfg.hash_key_length = sizeof(n->rss_data.key);
899 
900     const struct iovec data[] = {
901         {
902             .iov_base = &cfg,
903             .iov_len = offsetof(struct virtio_net_rss_config,
904                                 indirection_table),
905         }, {
906             .iov_base = table,
907             .iov_len = n->rss_data.indirections_len *
908                        sizeof(n->rss_data.indirections_table[0]),
909         }, {
910             .iov_base = &cfg.max_tx_vq,
911             .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) -
912                        offsetof(struct virtio_net_rss_config, max_tx_vq),
913         }, {
914             .iov_base = (void *)n->rss_data.key,
915             .iov_len = sizeof(n->rss_data.key),
916         }
917     };
918 
919     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
920                                 VIRTIO_NET_CTRL_MQ,
921                                 do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG :
922                                 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
923                                 data, ARRAY_SIZE(data));
924     if (unlikely(r < 0)) {
925         return r;
926     }
927 
928     return 0;
929 }
930 
931 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
932                                   const VirtIONet *n,
933                                   struct iovec *out_cursor,
934                                   struct iovec *in_cursor)
935 {
936     struct virtio_net_ctrl_mq mq;
937     ssize_t r;
938 
939     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) {
940         return 0;
941     }
942 
943     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
944     const struct iovec data = {
945         .iov_base = &mq,
946         .iov_len = sizeof(mq),
947     };
948     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
949                                 VIRTIO_NET_CTRL_MQ,
950                                 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
951                                 &data, 1);
952     if (unlikely(r < 0)) {
953         return r;
954     }
955 
956     if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
957         /* load the receive-side scaling state */
958         r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true);
959         if (unlikely(r < 0)) {
960             return r;
961         }
962     } else if (virtio_vdev_has_feature(&n->parent_obj,
963                                        VIRTIO_NET_F_HASH_REPORT)) {
964         /* load the hash calculation state */
965         r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
966         if (unlikely(r < 0)) {
967             return r;
968         }
969     }
970 
971     return 0;
972 }
973 
974 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s,
975                                         const VirtIONet *n,
976                                         struct iovec *out_cursor,
977                                         struct iovec *in_cursor)
978 {
979     uint64_t offloads;
980     ssize_t r;
981 
982     if (!virtio_vdev_has_feature(&n->parent_obj,
983                                  VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
984         return 0;
985     }
986 
987     if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) {
988         /*
989          * According to VirtIO standard, "Upon feature negotiation
990          * corresponding offload gets enabled to preserve
991          * backward compatibility.".
992          *
993          * Therefore, there is no need to send this CVQ command if the
994          * driver also enables all supported offloads, which aligns with
995          * the device's defaults.
996          *
997          * Note that the device's defaults can mismatch the driver's
998          * configuration only at live migration.
999          */
1000         return 0;
1001     }
1002 
1003     offloads = cpu_to_le64(n->curr_guest_offloads);
1004     const struct iovec data = {
1005         .iov_base = &offloads,
1006         .iov_len = sizeof(offloads),
1007     };
1008     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1009                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS,
1010                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET,
1011                                 &data, 1);
1012     if (unlikely(r < 0)) {
1013         return r;
1014     }
1015 
1016     return 0;
1017 }
1018 
1019 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s,
1020                                        struct iovec *out_cursor,
1021                                        struct iovec *in_cursor,
1022                                        uint8_t cmd,
1023                                        uint8_t on)
1024 {
1025     const struct iovec data = {
1026         .iov_base = &on,
1027         .iov_len = sizeof(on),
1028     };
1029     ssize_t r;
1030 
1031     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1032                                 VIRTIO_NET_CTRL_RX, cmd, &data, 1);
1033     if (unlikely(r < 0)) {
1034         return r;
1035     }
1036 
1037     return 0;
1038 }
1039 
1040 static int vhost_vdpa_net_load_rx(VhostVDPAState *s,
1041                                   const VirtIONet *n,
1042                                   struct iovec *out_cursor,
1043                                   struct iovec *in_cursor)
1044 {
1045     ssize_t r;
1046 
1047     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) {
1048         return 0;
1049     }
1050 
1051     /*
1052      * According to virtio_net_reset(), device turns promiscuous mode
1053      * on by default.
1054      *
1055      * Additionally, according to VirtIO standard, "Since there are
1056      * no guarantees, it can use a hash filter or silently switch to
1057      * allmulti or promiscuous mode if it is given too many addresses.".
1058      * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1059      * non-multicast MAC addresses, indicating that promiscuous mode
1060      * should be enabled.
1061      *
1062      * Therefore, QEMU should only send this CVQ command if the
1063      * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1064      * which sets promiscuous mode on, different from the device's defaults.
1065      *
1066      * Note that the device's defaults can mismatch the driver's
1067      * configuration only at live migration.
1068      */
1069     if (!n->mac_table.uni_overflow && !n->promisc) {
1070         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1071                                         VIRTIO_NET_CTRL_RX_PROMISC, 0);
1072         if (unlikely(r < 0)) {
1073             return r;
1074         }
1075     }
1076 
1077     /*
1078      * According to virtio_net_reset(), device turns all-multicast mode
1079      * off by default.
1080      *
1081      * According to VirtIO standard, "Since there are no guarantees,
1082      * it can use a hash filter or silently switch to allmulti or
1083      * promiscuous mode if it is given too many addresses.". QEMU marks
1084      * `n->mac_table.multi_overflow` if guest sets too many
1085      * non-multicast MAC addresses.
1086      *
1087      * Therefore, QEMU should only send this CVQ command if the
1088      * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1089      * which sets all-multicast mode on, different from the device's defaults.
1090      *
1091      * Note that the device's defaults can mismatch the driver's
1092      * configuration only at live migration.
1093      */
1094     if (n->mac_table.multi_overflow || n->allmulti) {
1095         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1096                                         VIRTIO_NET_CTRL_RX_ALLMULTI, 1);
1097         if (unlikely(r < 0)) {
1098             return r;
1099         }
1100     }
1101 
1102     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) {
1103         return 0;
1104     }
1105 
1106     /*
1107      * According to virtio_net_reset(), device turns all-unicast mode
1108      * off by default.
1109      *
1110      * Therefore, QEMU should only send this CVQ command if the driver
1111      * sets all-unicast mode on, different from the device's defaults.
1112      *
1113      * Note that the device's defaults can mismatch the driver's
1114      * configuration only at live migration.
1115      */
1116     if (n->alluni) {
1117         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1118                                         VIRTIO_NET_CTRL_RX_ALLUNI, 1);
1119         if (r < 0) {
1120             return r;
1121         }
1122     }
1123 
1124     /*
1125      * According to virtio_net_reset(), device turns non-multicast mode
1126      * off by default.
1127      *
1128      * Therefore, QEMU should only send this CVQ command if the driver
1129      * sets non-multicast mode on, different from the device's defaults.
1130      *
1131      * Note that the device's defaults can mismatch the driver's
1132      * configuration only at live migration.
1133      */
1134     if (n->nomulti) {
1135         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1136                                         VIRTIO_NET_CTRL_RX_NOMULTI, 1);
1137         if (r < 0) {
1138             return r;
1139         }
1140     }
1141 
1142     /*
1143      * According to virtio_net_reset(), device turns non-unicast mode
1144      * off by default.
1145      *
1146      * Therefore, QEMU should only send this CVQ command if the driver
1147      * sets non-unicast mode on, different from the device's defaults.
1148      *
1149      * Note that the device's defaults can mismatch the driver's
1150      * configuration only at live migration.
1151      */
1152     if (n->nouni) {
1153         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1154                                         VIRTIO_NET_CTRL_RX_NOUNI, 1);
1155         if (r < 0) {
1156             return r;
1157         }
1158     }
1159 
1160     /*
1161      * According to virtio_net_reset(), device turns non-broadcast mode
1162      * off by default.
1163      *
1164      * Therefore, QEMU should only send this CVQ command if the driver
1165      * sets non-broadcast mode on, different from the device's defaults.
1166      *
1167      * Note that the device's defaults can mismatch the driver's
1168      * configuration only at live migration.
1169      */
1170     if (n->nobcast) {
1171         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1172                                         VIRTIO_NET_CTRL_RX_NOBCAST, 1);
1173         if (r < 0) {
1174             return r;
1175         }
1176     }
1177 
1178     return 0;
1179 }
1180 
1181 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s,
1182                                            const VirtIONet *n,
1183                                            struct iovec *out_cursor,
1184                                            struct iovec *in_cursor,
1185                                            uint16_t vid)
1186 {
1187     const struct iovec data = {
1188         .iov_base = &vid,
1189         .iov_len = sizeof(vid),
1190     };
1191     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1192                                         VIRTIO_NET_CTRL_VLAN,
1193                                         VIRTIO_NET_CTRL_VLAN_ADD,
1194                                         &data, 1);
1195     if (unlikely(r < 0)) {
1196         return r;
1197     }
1198 
1199     return 0;
1200 }
1201 
1202 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s,
1203                                     const VirtIONet *n,
1204                                     struct iovec *out_cursor,
1205                                     struct iovec *in_cursor)
1206 {
1207     int r;
1208 
1209     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) {
1210         return 0;
1211     }
1212 
1213     for (int i = 0; i < MAX_VLAN >> 5; i++) {
1214         for (int j = 0; n->vlans[i] && j <= 0x1f; j++) {
1215             if (n->vlans[i] & (1U << j)) {
1216                 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor,
1217                                                     in_cursor, (i << 5) + j);
1218                 if (unlikely(r != 0)) {
1219                     return r;
1220                 }
1221             }
1222         }
1223     }
1224 
1225     return 0;
1226 }
1227 
1228 static int vhost_vdpa_net_cvq_load(NetClientState *nc)
1229 {
1230     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
1231     struct vhost_vdpa *v = &s->vhost_vdpa;
1232     const VirtIONet *n;
1233     int r;
1234     struct iovec out_cursor, in_cursor;
1235 
1236     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1237 
1238     vhost_vdpa_set_vring_ready(v, v->dev->vq_index);
1239 
1240     if (v->shadow_vqs_enabled) {
1241         n = VIRTIO_NET(v->dev->vdev);
1242         vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor);
1243         r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor);
1244         if (unlikely(r < 0)) {
1245             return r;
1246         }
1247         r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor);
1248         if (unlikely(r)) {
1249             return r;
1250         }
1251         r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor);
1252         if (unlikely(r)) {
1253             return r;
1254         }
1255         r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor);
1256         if (unlikely(r)) {
1257             return r;
1258         }
1259         r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor);
1260         if (unlikely(r)) {
1261             return r;
1262         }
1263 
1264         /*
1265          * We need to poll and check all pending device's used buffers.
1266          *
1267          * We can poll here since we've had BQL from the time
1268          * we sent the descriptor.
1269          */
1270         r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status);
1271         if (unlikely(r)) {
1272             return r;
1273         }
1274     }
1275 
1276     for (int i = 0; i < v->dev->vq_index; ++i) {
1277         vhost_vdpa_set_vring_ready(v, i);
1278     }
1279 
1280     return 0;
1281 }
1282 
1283 static NetClientInfo net_vhost_vdpa_cvq_info = {
1284     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
1285     .size = sizeof(VhostVDPAState),
1286     .receive = vhost_vdpa_receive,
1287     .start = vhost_vdpa_net_cvq_start,
1288     .load = vhost_vdpa_net_cvq_load,
1289     .stop = vhost_vdpa_net_cvq_stop,
1290     .cleanup = vhost_vdpa_cleanup,
1291     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
1292     .has_ufo = vhost_vdpa_has_ufo,
1293     .check_peer_type = vhost_vdpa_check_peer_type,
1294     .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
1295 };
1296 
1297 /*
1298  * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1299  * vdpa device.
1300  *
1301  * Considering that QEMU cannot send the entire filter table to the
1302  * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1303  * command to enable promiscuous mode to receive all packets,
1304  * according to VirtIO standard, "Since there are no guarantees,
1305  * it can use a hash filter or silently switch to allmulti or
1306  * promiscuous mode if it is given too many addresses.".
1307  *
1308  * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1309  * marks `n->mac_table.x_overflow` accordingly, it should have
1310  * the same effect on the device model to receive
1311  * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1312  * The same applies to multicast MAC addresses.
1313  *
1314  * Therefore, QEMU can provide the device model with a fake
1315  * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1316  * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1317  * MAC addresses. This ensures that the device model marks
1318  * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1319  * allowing all packets to be received, which aligns with the
1320  * state of the vdpa device.
1321  */
1322 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s,
1323                                                        VirtQueueElement *elem,
1324                                                        struct iovec *out,
1325                                                        const struct iovec *in)
1326 {
1327     struct virtio_net_ctrl_mac mac_data, *mac_ptr;
1328     struct virtio_net_ctrl_hdr *hdr_ptr;
1329     uint32_t cursor;
1330     ssize_t r;
1331     uint8_t on = 1;
1332 
1333     /* parse the non-multicast MAC address entries from CVQ command */
1334     cursor = sizeof(*hdr_ptr);
1335     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1336                    &mac_data, sizeof(mac_data));
1337     if (unlikely(r != sizeof(mac_data))) {
1338         /*
1339          * If the CVQ command is invalid, we should simulate the vdpa device
1340          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1341          */
1342         *s->status = VIRTIO_NET_ERR;
1343         return sizeof(*s->status);
1344     }
1345     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1346 
1347     /* parse the multicast MAC address entries from CVQ command */
1348     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1349                    &mac_data, sizeof(mac_data));
1350     if (r != sizeof(mac_data)) {
1351         /*
1352          * If the CVQ command is invalid, we should simulate the vdpa device
1353          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1354          */
1355         *s->status = VIRTIO_NET_ERR;
1356         return sizeof(*s->status);
1357     }
1358     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1359 
1360     /* validate the CVQ command */
1361     if (iov_size(elem->out_sg, elem->out_num) != cursor) {
1362         /*
1363          * If the CVQ command is invalid, we should simulate the vdpa device
1364          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1365          */
1366         *s->status = VIRTIO_NET_ERR;
1367         return sizeof(*s->status);
1368     }
1369 
1370     /*
1371      * According to VirtIO standard, "Since there are no guarantees,
1372      * it can use a hash filter or silently switch to allmulti or
1373      * promiscuous mode if it is given too many addresses.".
1374      *
1375      * Therefore, considering that QEMU is unable to send the entire
1376      * filter table to the vdpa device, it should send the
1377      * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1378      */
1379     hdr_ptr = out->iov_base;
1380     out->iov_len = sizeof(*hdr_ptr) + sizeof(on);
1381 
1382     hdr_ptr->class = VIRTIO_NET_CTRL_RX;
1383     hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC;
1384     iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on));
1385     r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1);
1386     if (unlikely(r < 0)) {
1387         return r;
1388     }
1389 
1390     /*
1391      * We can poll here since we've had BQL from the time
1392      * we sent the descriptor.
1393      */
1394     r = vhost_vdpa_net_svq_poll(s, 1);
1395     if (unlikely(r < sizeof(*s->status))) {
1396         return r;
1397     }
1398     if (*s->status != VIRTIO_NET_OK) {
1399         return sizeof(*s->status);
1400     }
1401 
1402     /*
1403      * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1404      * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1405      * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1406      * multicast MAC addresses.
1407      *
1408      * By doing so, the device model can mark `n->mac_table.uni_overflow`
1409      * and `n->mac_table.multi_overflow`, enabling all packets to be
1410      * received, which aligns with the state of the vdpa device.
1411      */
1412     cursor = 0;
1413     uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1,
1414              fake_mul_entries = MAC_TABLE_ENTRIES + 1,
1415              fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) +
1416                              sizeof(mac_data) + fake_uni_entries * ETH_ALEN +
1417                              sizeof(mac_data) + fake_mul_entries * ETH_ALEN;
1418 
1419     assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len());
1420     out->iov_len = fake_cvq_size;
1421 
1422     /* pack the header for fake CVQ command */
1423     hdr_ptr = out->iov_base + cursor;
1424     hdr_ptr->class = VIRTIO_NET_CTRL_MAC;
1425     hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1426     cursor += sizeof(*hdr_ptr);
1427 
1428     /*
1429      * Pack the non-multicast MAC addresses part for fake CVQ command.
1430      *
1431      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1432      * addresses provided in CVQ command. Therefore, only the entries
1433      * field need to be prepared in the CVQ command.
1434      */
1435     mac_ptr = out->iov_base + cursor;
1436     mac_ptr->entries = cpu_to_le32(fake_uni_entries);
1437     cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN;
1438 
1439     /*
1440      * Pack the multicast MAC addresses part for fake CVQ command.
1441      *
1442      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1443      * addresses provided in CVQ command. Therefore, only the entries
1444      * field need to be prepared in the CVQ command.
1445      */
1446     mac_ptr = out->iov_base + cursor;
1447     mac_ptr->entries = cpu_to_le32(fake_mul_entries);
1448 
1449     /*
1450      * Simulating QEMU poll a vdpa device used buffer
1451      * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1452      */
1453     return sizeof(*s->status);
1454 }
1455 
1456 /**
1457  * Validate and copy control virtqueue commands.
1458  *
1459  * Following QEMU guidelines, we offer a copy of the buffers to the device to
1460  * prevent TOCTOU bugs.
1461  */
1462 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
1463                                             VirtQueueElement *elem,
1464                                             void *opaque)
1465 {
1466     VhostVDPAState *s = opaque;
1467     size_t in_len;
1468     const struct virtio_net_ctrl_hdr *ctrl;
1469     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1470     /* Out buffer sent to both the vdpa device and the device model */
1471     struct iovec out = {
1472         .iov_base = s->cvq_cmd_out_buffer,
1473     };
1474     /* in buffer used for device model */
1475     const struct iovec model_in = {
1476         .iov_base = &status,
1477         .iov_len = sizeof(status),
1478     };
1479     /* in buffer used for vdpa device */
1480     const struct iovec vdpa_in = {
1481         .iov_base = s->status,
1482         .iov_len = sizeof(*s->status),
1483     };
1484     ssize_t dev_written = -EINVAL;
1485 
1486     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
1487                              s->cvq_cmd_out_buffer,
1488                              vhost_vdpa_net_cvq_cmd_page_len());
1489 
1490     ctrl = s->cvq_cmd_out_buffer;
1491     if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) {
1492         /*
1493          * Guest announce capability is emulated by qemu, so don't forward to
1494          * the device.
1495          */
1496         dev_written = sizeof(status);
1497         *s->status = VIRTIO_NET_OK;
1498     } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC &&
1499                         ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET &&
1500                         iov_size(elem->out_sg, elem->out_num) > out.iov_len)) {
1501         /*
1502          * Due to the size limitation of the out buffer sent to the vdpa device,
1503          * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1504          * MAC addresses set by the driver for the filter table can cause
1505          * truncation of the CVQ command in QEMU. As a result, the vdpa device
1506          * rejects the flawed CVQ command.
1507          *
1508          * Therefore, QEMU must handle this situation instead of sending
1509          * the CVQ command directly.
1510          */
1511         dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem,
1512                                                             &out, &vdpa_in);
1513         if (unlikely(dev_written < 0)) {
1514             goto out;
1515         }
1516     } else {
1517         ssize_t r;
1518         r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1);
1519         if (unlikely(r < 0)) {
1520             dev_written = r;
1521             goto out;
1522         }
1523 
1524         /*
1525          * We can poll here since we've had BQL from the time
1526          * we sent the descriptor.
1527          */
1528         dev_written = vhost_vdpa_net_svq_poll(s, 1);
1529     }
1530 
1531     if (unlikely(dev_written < sizeof(status))) {
1532         error_report("Insufficient written data (%zu)", dev_written);
1533         goto out;
1534     }
1535 
1536     if (*s->status != VIRTIO_NET_OK) {
1537         goto out;
1538     }
1539 
1540     status = VIRTIO_NET_ERR;
1541     virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1);
1542     if (status != VIRTIO_NET_OK) {
1543         error_report("Bad CVQ processing in model");
1544     }
1545 
1546 out:
1547     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
1548                           sizeof(status));
1549     if (unlikely(in_len < sizeof(status))) {
1550         error_report("Bad device CVQ written length");
1551     }
1552     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
1553     /*
1554      * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1555      * the function successfully forwards the CVQ command, indicated
1556      * by a non-negative value of `dev_written`. Otherwise, it still
1557      * belongs to SVQ.
1558      * This function should only free the `elem` when it owns.
1559      */
1560     if (dev_written >= 0) {
1561         g_free(elem);
1562     }
1563     return dev_written < 0 ? dev_written : 0;
1564 }
1565 
1566 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
1567     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
1568 };
1569 
1570 /**
1571  * Probe if CVQ is isolated
1572  *
1573  * @device_fd         The vdpa device fd
1574  * @features          Features offered by the device.
1575  * @cvq_index         The control vq pair index
1576  *
1577  * Returns <0 in case of failure, 0 if false and 1 if true.
1578  */
1579 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features,
1580                                           int cvq_index, Error **errp)
1581 {
1582     uint64_t backend_features;
1583     int64_t cvq_group;
1584     uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE |
1585                      VIRTIO_CONFIG_S_DRIVER;
1586     int r;
1587 
1588     ERRP_GUARD();
1589 
1590     r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
1591     if (unlikely(r < 0)) {
1592         error_setg_errno(errp, errno, "Cannot get vdpa backend_features");
1593         return r;
1594     }
1595 
1596     if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) {
1597         return 0;
1598     }
1599 
1600     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1601     if (unlikely(r)) {
1602         error_setg_errno(errp, -r, "Cannot set device status");
1603         goto out;
1604     }
1605 
1606     r = ioctl(device_fd, VHOST_SET_FEATURES, &features);
1607     if (unlikely(r)) {
1608         error_setg_errno(errp, -r, "Cannot set features");
1609         goto out;
1610     }
1611 
1612     status |= VIRTIO_CONFIG_S_FEATURES_OK;
1613     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1614     if (unlikely(r)) {
1615         error_setg_errno(errp, -r, "Cannot set device status");
1616         goto out;
1617     }
1618 
1619     cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp);
1620     if (unlikely(cvq_group < 0)) {
1621         if (cvq_group != -ENOTSUP) {
1622             r = cvq_group;
1623             goto out;
1624         }
1625 
1626         /*
1627          * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1628          * support ASID even if the parent driver does not.  The CVQ cannot be
1629          * isolated in this case.
1630          */
1631         error_free(*errp);
1632         *errp = NULL;
1633         r = 0;
1634         goto out;
1635     }
1636 
1637     for (int i = 0; i < cvq_index; ++i) {
1638         int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp);
1639         if (unlikely(group < 0)) {
1640             r = group;
1641             goto out;
1642         }
1643 
1644         if (group == (int64_t)cvq_group) {
1645             r = 0;
1646             goto out;
1647         }
1648     }
1649 
1650     r = 1;
1651 
1652 out:
1653     status = 0;
1654     ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1655     return r;
1656 }
1657 
1658 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
1659                                        const char *device,
1660                                        const char *name,
1661                                        int vdpa_device_fd,
1662                                        int queue_pair_index,
1663                                        int nvqs,
1664                                        bool is_datapath,
1665                                        bool svq,
1666                                        struct vhost_vdpa_iova_range iova_range,
1667                                        uint64_t features,
1668                                        VhostVDPAShared *shared,
1669                                        Error **errp)
1670 {
1671     NetClientState *nc = NULL;
1672     VhostVDPAState *s;
1673     int ret = 0;
1674     assert(name);
1675     int cvq_isolated = 0;
1676 
1677     if (is_datapath) {
1678         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
1679                                  name);
1680     } else {
1681         cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features,
1682                                                       queue_pair_index * 2,
1683                                                       errp);
1684         if (unlikely(cvq_isolated < 0)) {
1685             return NULL;
1686         }
1687 
1688         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
1689                                          device, name);
1690     }
1691     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
1692     s = DO_UPCAST(VhostVDPAState, nc, nc);
1693 
1694     s->vhost_vdpa.device_fd = vdpa_device_fd;
1695     s->vhost_vdpa.index = queue_pair_index;
1696     s->always_svq = svq;
1697     s->migration_state.notify = NULL;
1698     s->vhost_vdpa.shadow_vqs_enabled = svq;
1699     s->vhost_vdpa.iova_range = iova_range;
1700     s->vhost_vdpa.shadow_data = svq;
1701     if (queue_pair_index == 0) {
1702         vhost_vdpa_net_valid_svq_features(features,
1703                                           &s->vhost_vdpa.migration_blocker);
1704         s->vhost_vdpa.shared = g_new0(VhostVDPAShared, 1);
1705     } else if (!is_datapath) {
1706         s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1707                                      PROT_READ | PROT_WRITE,
1708                                      MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1709         s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1710                          PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
1711                          -1, 0);
1712 
1713         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
1714         s->vhost_vdpa.shadow_vq_ops_opaque = s;
1715         s->cvq_isolated = cvq_isolated;
1716     }
1717     if (queue_pair_index != 0) {
1718         s->vhost_vdpa.shared = shared;
1719     }
1720 
1721     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
1722     if (ret) {
1723         qemu_del_net_client(nc);
1724         return NULL;
1725     }
1726 
1727     return nc;
1728 }
1729 
1730 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
1731 {
1732     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
1733     if (unlikely(ret < 0)) {
1734         error_setg_errno(errp, errno,
1735                          "Fail to query features from vhost-vDPA device");
1736     }
1737     return ret;
1738 }
1739 
1740 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
1741                                           int *has_cvq, Error **errp)
1742 {
1743     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
1744     g_autofree struct vhost_vdpa_config *config = NULL;
1745     __virtio16 *max_queue_pairs;
1746     int ret;
1747 
1748     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
1749         *has_cvq = 1;
1750     } else {
1751         *has_cvq = 0;
1752     }
1753 
1754     if (features & (1 << VIRTIO_NET_F_MQ)) {
1755         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
1756         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
1757         config->len = sizeof(*max_queue_pairs);
1758 
1759         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
1760         if (ret) {
1761             error_setg(errp, "Fail to get config from vhost-vDPA device");
1762             return -ret;
1763         }
1764 
1765         max_queue_pairs = (__virtio16 *)&config->buf;
1766 
1767         return lduw_le_p(max_queue_pairs);
1768     }
1769 
1770     return 1;
1771 }
1772 
1773 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
1774                         NetClientState *peer, Error **errp)
1775 {
1776     const NetdevVhostVDPAOptions *opts;
1777     uint64_t features;
1778     int vdpa_device_fd;
1779     g_autofree NetClientState **ncs = NULL;
1780     struct vhost_vdpa_iova_range iova_range;
1781     NetClientState *nc;
1782     int queue_pairs, r, i = 0, has_cvq = 0;
1783 
1784     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1785     opts = &netdev->u.vhost_vdpa;
1786     if (!opts->vhostdev && !opts->vhostfd) {
1787         error_setg(errp,
1788                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1789         return -1;
1790     }
1791 
1792     if (opts->vhostdev && opts->vhostfd) {
1793         error_setg(errp,
1794                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1795         return -1;
1796     }
1797 
1798     if (opts->vhostdev) {
1799         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
1800         if (vdpa_device_fd == -1) {
1801             return -errno;
1802         }
1803     } else {
1804         /* has_vhostfd */
1805         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
1806         if (vdpa_device_fd == -1) {
1807             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
1808             return -1;
1809         }
1810     }
1811 
1812     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
1813     if (unlikely(r < 0)) {
1814         goto err;
1815     }
1816 
1817     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
1818                                                  &has_cvq, errp);
1819     if (queue_pairs < 0) {
1820         qemu_close(vdpa_device_fd);
1821         return queue_pairs;
1822     }
1823 
1824     r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
1825     if (unlikely(r < 0)) {
1826         error_setg(errp, "vhost-vdpa: get iova range failed: %s",
1827                    strerror(-r));
1828         goto err;
1829     }
1830 
1831     if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
1832         goto err;
1833     }
1834 
1835     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
1836 
1837     for (i = 0; i < queue_pairs; i++) {
1838         VhostVDPAShared *shared = NULL;
1839 
1840         if (i) {
1841             shared = DO_UPCAST(VhostVDPAState, nc, ncs[0])->vhost_vdpa.shared;
1842         }
1843         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1844                                      vdpa_device_fd, i, 2, true, opts->x_svq,
1845                                      iova_range, features, shared, errp);
1846         if (!ncs[i])
1847             goto err;
1848     }
1849 
1850     if (has_cvq) {
1851         VhostVDPAState *s0 = DO_UPCAST(VhostVDPAState, nc, ncs[0]);
1852         VhostVDPAShared *shared = s0->vhost_vdpa.shared;
1853 
1854         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1855                                  vdpa_device_fd, i, 1, false,
1856                                  opts->x_svq, iova_range, features, shared,
1857                                  errp);
1858         if (!nc)
1859             goto err;
1860     }
1861 
1862     return 0;
1863 
1864 err:
1865     if (i) {
1866         for (i--; i >= 0; i--) {
1867             qemu_del_net_client(ncs[i]);
1868         }
1869     }
1870 
1871     qemu_close(vdpa_device_fd);
1872 
1873     return -1;
1874 }
1875