xref: /openbmc/qemu/net/vhost-vdpa.c (revision 5edb02e8004c2f1d5026a02cd9378046973f47af)
1 /*
2  * vhost-vdpa.c
3  *
4  * Copyright(c) 2017-2018 Intel Corporation.
5  * Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/log.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
26 #include <err.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/migration.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
32 
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState {
35     NetClientState nc;
36     struct vhost_vdpa vhost_vdpa;
37     Notifier migration_state;
38     VHostNetState *vhost_net;
39 
40     /* Control commands shadow buffers */
41     void *cvq_cmd_out_buffer;
42     virtio_net_ctrl_ack *status;
43 
44     /* The device always have SVQ enabled */
45     bool always_svq;
46 
47     /* The device can isolate CVQ in its own ASID */
48     bool cvq_isolated;
49 
50     bool started;
51 } VhostVDPAState;
52 
53 /*
54  * The array is sorted alphabetically in ascending order,
55  * with the exception of VHOST_INVALID_FEATURE_BIT,
56  * which should always be the last entry.
57  */
58 const int vdpa_feature_bits[] = {
59     VIRTIO_F_ANY_LAYOUT,
60     VIRTIO_F_IOMMU_PLATFORM,
61     VIRTIO_F_NOTIFY_ON_EMPTY,
62     VIRTIO_F_RING_PACKED,
63     VIRTIO_F_RING_RESET,
64     VIRTIO_F_VERSION_1,
65     VIRTIO_NET_F_CSUM,
66     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
67     VIRTIO_NET_F_CTRL_MAC_ADDR,
68     VIRTIO_NET_F_CTRL_RX,
69     VIRTIO_NET_F_CTRL_RX_EXTRA,
70     VIRTIO_NET_F_CTRL_VLAN,
71     VIRTIO_NET_F_CTRL_VQ,
72     VIRTIO_NET_F_GSO,
73     VIRTIO_NET_F_GUEST_CSUM,
74     VIRTIO_NET_F_GUEST_ECN,
75     VIRTIO_NET_F_GUEST_TSO4,
76     VIRTIO_NET_F_GUEST_TSO6,
77     VIRTIO_NET_F_GUEST_UFO,
78     VIRTIO_NET_F_GUEST_USO4,
79     VIRTIO_NET_F_GUEST_USO6,
80     VIRTIO_NET_F_HASH_REPORT,
81     VIRTIO_NET_F_HOST_ECN,
82     VIRTIO_NET_F_HOST_TSO4,
83     VIRTIO_NET_F_HOST_TSO6,
84     VIRTIO_NET_F_HOST_UFO,
85     VIRTIO_NET_F_HOST_USO,
86     VIRTIO_NET_F_MQ,
87     VIRTIO_NET_F_MRG_RXBUF,
88     VIRTIO_NET_F_MTU,
89     VIRTIO_NET_F_RSS,
90     VIRTIO_NET_F_STATUS,
91     VIRTIO_RING_F_EVENT_IDX,
92     VIRTIO_RING_F_INDIRECT_DESC,
93 
94     /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
95     VHOST_INVALID_FEATURE_BIT
96 };
97 
98 /** Supported device specific feature bits with SVQ */
99 static const uint64_t vdpa_svq_device_features =
100     BIT_ULL(VIRTIO_NET_F_CSUM) |
101     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
102     BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
103     BIT_ULL(VIRTIO_NET_F_MTU) |
104     BIT_ULL(VIRTIO_NET_F_MAC) |
105     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
106     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
107     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
108     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
109     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
110     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
111     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
112     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
113     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
114     BIT_ULL(VIRTIO_NET_F_STATUS) |
115     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
116     BIT_ULL(VIRTIO_NET_F_CTRL_RX) |
117     BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) |
118     BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) |
119     BIT_ULL(VIRTIO_NET_F_MQ) |
120     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
121     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
122     /* VHOST_F_LOG_ALL is exposed by SVQ */
123     BIT_ULL(VHOST_F_LOG_ALL) |
124     BIT_ULL(VIRTIO_NET_F_HASH_REPORT) |
125     BIT_ULL(VIRTIO_NET_F_RSS) |
126     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
127     BIT_ULL(VIRTIO_NET_F_STANDBY) |
128     BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
129 
130 #define VHOST_VDPA_NET_CVQ_ASID 1
131 
132 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
133 {
134     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
135     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
136     return s->vhost_net;
137 }
138 
139 static size_t vhost_vdpa_net_cvq_cmd_len(void)
140 {
141     /*
142      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
143      * In buffer is always 1 byte, so it should fit here
144      */
145     return sizeof(struct virtio_net_ctrl_hdr) +
146            2 * sizeof(struct virtio_net_ctrl_mac) +
147            MAC_TABLE_ENTRIES * ETH_ALEN;
148 }
149 
150 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
151 {
152     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
153 }
154 
155 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
156 {
157     uint64_t invalid_dev_features =
158         features & ~vdpa_svq_device_features &
159         /* Transport are all accepted at this point */
160         ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
161                          VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
162 
163     if (invalid_dev_features) {
164         error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
165                    invalid_dev_features);
166         return false;
167     }
168 
169     return vhost_svq_valid_features(features, errp);
170 }
171 
172 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
173 {
174     uint32_t device_id;
175     int ret;
176     struct vhost_dev *hdev;
177 
178     hdev = (struct vhost_dev *)&net->dev;
179     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
180     if (device_id != VIRTIO_ID_NET) {
181         return -ENOTSUP;
182     }
183     return ret;
184 }
185 
186 static int vhost_vdpa_add(NetClientState *ncs, void *be,
187                           int queue_pair_index, int nvqs)
188 {
189     VhostNetOptions options;
190     struct vhost_net *net = NULL;
191     VhostVDPAState *s;
192     int ret;
193 
194     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
195     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
196     s = DO_UPCAST(VhostVDPAState, nc, ncs);
197     options.net_backend = ncs;
198     options.opaque      = be;
199     options.busyloop_timeout = 0;
200     options.nvqs = nvqs;
201 
202     net = vhost_net_init(&options);
203     if (!net) {
204         error_report("failed to init vhost_net for queue");
205         goto err_init;
206     }
207     s->vhost_net = net;
208     ret = vhost_vdpa_net_check_device_id(net);
209     if (ret) {
210         goto err_check;
211     }
212     return 0;
213 err_check:
214     vhost_net_cleanup(net);
215     g_free(net);
216 err_init:
217     return -1;
218 }
219 
220 static void vhost_vdpa_cleanup(NetClientState *nc)
221 {
222     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
223 
224     /*
225      * If a peer NIC is attached, do not cleanup anything.
226      * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
227      * when the guest is shutting down.
228      */
229     if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
230         return;
231     }
232     munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len());
233     munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len());
234     if (s->vhost_net) {
235         vhost_net_cleanup(s->vhost_net);
236         g_free(s->vhost_net);
237         s->vhost_net = NULL;
238     }
239      if (s->vhost_vdpa.device_fd >= 0) {
240         qemu_close(s->vhost_vdpa.device_fd);
241         s->vhost_vdpa.device_fd = -1;
242     }
243     if (s->vhost_vdpa.index != 0) {
244         return;
245     }
246     g_free(s->vhost_vdpa.shared);
247 }
248 
249 /** Dummy SetSteeringEBPF to support RSS for vhost-vdpa backend  */
250 static bool vhost_vdpa_set_steering_ebpf(NetClientState *nc, int prog_fd)
251 {
252     return true;
253 }
254 
255 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
256 {
257     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
258 
259     return true;
260 }
261 
262 static bool vhost_vdpa_has_ufo(NetClientState *nc)
263 {
264     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
265     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
266     uint64_t features = 0;
267     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
268     features = vhost_net_get_features(s->vhost_net, features);
269     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
270 
271 }
272 
273 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
274                                        Error **errp)
275 {
276     const char *driver = object_class_get_name(oc);
277 
278     if (!g_str_has_prefix(driver, "virtio-net-")) {
279         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
280         return false;
281     }
282 
283     return true;
284 }
285 
286 /** Dummy receive in case qemu falls back to userland tap networking */
287 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
288                                   size_t size)
289 {
290     return size;
291 }
292 
293 /** From any vdpa net client, get the netclient of the first queue pair */
294 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
295 {
296     NICState *nic = qemu_get_nic(s->nc.peer);
297     NetClientState *nc0 = qemu_get_peer(nic->ncs, 0);
298 
299     return DO_UPCAST(VhostVDPAState, nc, nc0);
300 }
301 
302 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
303 {
304     struct vhost_vdpa *v = &s->vhost_vdpa;
305     VirtIONet *n;
306     VirtIODevice *vdev;
307     int data_queue_pairs, cvq, r;
308 
309     /* We are only called on the first data vqs and only if x-svq is not set */
310     if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
311         return;
312     }
313 
314     vdev = v->dev->vdev;
315     n = VIRTIO_NET(vdev);
316     if (!n->vhost_started) {
317         return;
318     }
319 
320     data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
321     cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
322                                   n->max_ncs - n->max_queue_pairs : 0;
323     /*
324      * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
325      * in the future and resume the device if read-only operations between
326      * suspend and reset goes wrong.
327      */
328     vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
329 
330     /* Start will check migration setup_or_active to configure or not SVQ */
331     r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
332     if (unlikely(r < 0)) {
333         error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
334     }
335 }
336 
337 static void vdpa_net_migration_state_notifier(Notifier *notifier, void *data)
338 {
339     MigrationState *migration = data;
340     VhostVDPAState *s = container_of(notifier, VhostVDPAState,
341                                      migration_state);
342 
343     if (migration_in_setup(migration)) {
344         vhost_vdpa_net_log_global_enable(s, true);
345     } else if (migration_has_failed(migration)) {
346         vhost_vdpa_net_log_global_enable(s, false);
347     }
348 }
349 
350 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
351 {
352     struct vhost_vdpa *v = &s->vhost_vdpa;
353 
354     migration_add_notifier(&s->migration_state,
355                            vdpa_net_migration_state_notifier);
356     if (v->shadow_vqs_enabled) {
357         v->shared->iova_tree = vhost_iova_tree_new(v->iova_range.first,
358                                                    v->iova_range.last);
359     }
360 }
361 
362 static int vhost_vdpa_net_data_start(NetClientState *nc)
363 {
364     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
365     struct vhost_vdpa *v = &s->vhost_vdpa;
366 
367     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
368 
369     if (s->always_svq ||
370         migration_is_setup_or_active(migrate_get_current()->state)) {
371         v->shadow_vqs_enabled = true;
372         v->shadow_data = true;
373     } else {
374         v->shadow_vqs_enabled = false;
375         v->shadow_data = false;
376     }
377 
378     if (v->index == 0) {
379         vhost_vdpa_net_data_start_first(s);
380         return 0;
381     }
382 
383     return 0;
384 }
385 
386 static int vhost_vdpa_net_data_load(NetClientState *nc)
387 {
388     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
389     struct vhost_vdpa *v = &s->vhost_vdpa;
390     bool has_cvq = v->dev->vq_index_end % 2;
391 
392     if (has_cvq) {
393         return 0;
394     }
395 
396     for (int i = 0; i < v->dev->nvqs; ++i) {
397         vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index);
398     }
399     return 0;
400 }
401 
402 static void vhost_vdpa_net_client_stop(NetClientState *nc)
403 {
404     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
405     struct vhost_dev *dev;
406 
407     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
408 
409     if (s->vhost_vdpa.index == 0) {
410         migration_remove_notifier(&s->migration_state);
411     }
412 
413     dev = s->vhost_vdpa.dev;
414     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
415         g_clear_pointer(&s->vhost_vdpa.shared->iova_tree,
416                         vhost_iova_tree_delete);
417     }
418 }
419 
420 static NetClientInfo net_vhost_vdpa_info = {
421         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
422         .size = sizeof(VhostVDPAState),
423         .receive = vhost_vdpa_receive,
424         .start = vhost_vdpa_net_data_start,
425         .load = vhost_vdpa_net_data_load,
426         .stop = vhost_vdpa_net_client_stop,
427         .cleanup = vhost_vdpa_cleanup,
428         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
429         .has_ufo = vhost_vdpa_has_ufo,
430         .check_peer_type = vhost_vdpa_check_peer_type,
431         .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
432 };
433 
434 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
435                                           Error **errp)
436 {
437     struct vhost_vring_state state = {
438         .index = vq_index,
439     };
440     int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
441 
442     if (unlikely(r < 0)) {
443         r = -errno;
444         error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index);
445         return r;
446     }
447 
448     return state.num;
449 }
450 
451 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
452                                            unsigned vq_group,
453                                            unsigned asid_num)
454 {
455     struct vhost_vring_state asid = {
456         .index = vq_group,
457         .num = asid_num,
458     };
459     int r;
460 
461     r = ioctl(v->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
462     if (unlikely(r < 0)) {
463         error_report("Can't set vq group %u asid %u, errno=%d (%s)",
464                      asid.index, asid.num, errno, g_strerror(errno));
465     }
466     return r;
467 }
468 
469 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
470 {
471     VhostIOVATree *tree = v->shared->iova_tree;
472     DMAMap needle = {
473         /*
474          * No need to specify size or to look for more translations since
475          * this contiguous chunk was allocated by us.
476          */
477         .translated_addr = (hwaddr)(uintptr_t)addr,
478     };
479     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
480     int r;
481 
482     if (unlikely(!map)) {
483         error_report("Cannot locate expected map");
484         return;
485     }
486 
487     r = vhost_vdpa_dma_unmap(v, v->address_space_id, map->iova, map->size + 1);
488     if (unlikely(r != 0)) {
489         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
490     }
491 
492     vhost_iova_tree_remove(tree, *map);
493 }
494 
495 /** Map CVQ buffer. */
496 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
497                                   bool write)
498 {
499     DMAMap map = {};
500     int r;
501 
502     map.translated_addr = (hwaddr)(uintptr_t)buf;
503     map.size = size - 1;
504     map.perm = write ? IOMMU_RW : IOMMU_RO,
505     r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &map);
506     if (unlikely(r != IOVA_OK)) {
507         error_report("Cannot map injected element");
508         return r;
509     }
510 
511     r = vhost_vdpa_dma_map(v, v->address_space_id, map.iova,
512                            vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
513     if (unlikely(r < 0)) {
514         goto dma_map_err;
515     }
516 
517     return 0;
518 
519 dma_map_err:
520     vhost_iova_tree_remove(v->shared->iova_tree, map);
521     return r;
522 }
523 
524 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
525 {
526     VhostVDPAState *s, *s0;
527     struct vhost_vdpa *v;
528     int64_t cvq_group;
529     int r;
530     Error *err = NULL;
531 
532     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
533 
534     s = DO_UPCAST(VhostVDPAState, nc, nc);
535     v = &s->vhost_vdpa;
536 
537     s0 = vhost_vdpa_net_first_nc_vdpa(s);
538     v->shadow_data = s0->vhost_vdpa.shadow_vqs_enabled;
539     v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled;
540     s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
541 
542     if (s->vhost_vdpa.shadow_data) {
543         /* SVQ is already configured for all virtqueues */
544         goto out;
545     }
546 
547     /*
548      * If we early return in these cases SVQ will not be enabled. The migration
549      * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
550      */
551     if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) {
552         return 0;
553     }
554 
555     if (!s->cvq_isolated) {
556         return 0;
557     }
558 
559     cvq_group = vhost_vdpa_get_vring_group(v->device_fd,
560                                            v->dev->vq_index_end - 1,
561                                            &err);
562     if (unlikely(cvq_group < 0)) {
563         error_report_err(err);
564         return cvq_group;
565     }
566 
567     r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
568     if (unlikely(r < 0)) {
569         return r;
570     }
571 
572     v->shadow_vqs_enabled = true;
573     s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
574 
575 out:
576     if (!s->vhost_vdpa.shadow_vqs_enabled) {
577         return 0;
578     }
579 
580     /*
581      * If other vhost_vdpa already have an iova_tree, reuse it for simplicity,
582      * whether CVQ shares ASID with guest or not, because:
583      * - Memory listener need access to guest's memory addresses allocated in
584      *   the IOVA tree.
585      * - There should be plenty of IOVA address space for both ASID not to
586      *   worry about collisions between them.  Guest's translations are still
587      *   validated with virtio virtqueue_pop so there is no risk for the guest
588      *   to access memory that it shouldn't.
589      *
590      * To allocate a iova tree per ASID is doable but it complicates the code
591      * and it is not worth it for the moment.
592      */
593     if (!v->shared->iova_tree) {
594         v->shared->iova_tree = vhost_iova_tree_new(v->iova_range.first,
595                                                    v->iova_range.last);
596     }
597 
598     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
599                                vhost_vdpa_net_cvq_cmd_page_len(), false);
600     if (unlikely(r < 0)) {
601         return r;
602     }
603 
604     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
605                                vhost_vdpa_net_cvq_cmd_page_len(), true);
606     if (unlikely(r < 0)) {
607         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
608     }
609 
610     return r;
611 }
612 
613 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
614 {
615     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
616 
617     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
618 
619     if (s->vhost_vdpa.shadow_vqs_enabled) {
620         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
621         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
622     }
623 
624     vhost_vdpa_net_client_stop(nc);
625 }
626 
627 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s,
628                                     const struct iovec *out_sg, size_t out_num,
629                                     const struct iovec *in_sg, size_t in_num)
630 {
631     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
632     int r;
633 
634     r = vhost_svq_add(svq, out_sg, out_num, in_sg, in_num, NULL);
635     if (unlikely(r != 0)) {
636         if (unlikely(r == -ENOSPC)) {
637             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
638                           __func__);
639         }
640     }
641 
642     return r;
643 }
644 
645 /*
646  * Convenience wrapper to poll SVQ for multiple control commands.
647  *
648  * Caller should hold the BQL when invoking this function, and should take
649  * the answer before SVQ pulls by itself when BQL is released.
650  */
651 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight)
652 {
653     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
654     return vhost_svq_poll(svq, cmds_in_flight);
655 }
656 
657 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s,
658                                              struct iovec *out_cursor,
659                                              struct iovec *in_cursor)
660 {
661     /* reset the cursor of the output buffer for the device */
662     out_cursor->iov_base = s->cvq_cmd_out_buffer;
663     out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
664 
665     /* reset the cursor of the in buffer for the device */
666     in_cursor->iov_base = s->status;
667     in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
668 }
669 
670 /*
671  * Poll SVQ for multiple pending control commands and check the device's ack.
672  *
673  * Caller should hold the BQL when invoking this function.
674  *
675  * @s: The VhostVDPAState
676  * @len: The length of the pending status shadow buffer
677  */
678 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len)
679 {
680     /* device uses a one-byte length ack for each control command */
681     ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len);
682     if (unlikely(dev_written != len)) {
683         return -EIO;
684     }
685 
686     /* check the device's ack */
687     for (int i = 0; i < len; ++i) {
688         if (s->status[i] != VIRTIO_NET_OK) {
689             return -EIO;
690         }
691     }
692     return 0;
693 }
694 
695 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s,
696                                        struct iovec *out_cursor,
697                                        struct iovec *in_cursor, uint8_t class,
698                                        uint8_t cmd, const struct iovec *data_sg,
699                                        size_t data_num)
700 {
701     const struct virtio_net_ctrl_hdr ctrl = {
702         .class = class,
703         .cmd = cmd,
704     };
705     size_t data_size = iov_size(data_sg, data_num), cmd_size;
706     struct iovec out, in;
707     ssize_t r;
708     unsigned dummy_cursor_iov_cnt;
709     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
710 
711     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
712     cmd_size = sizeof(ctrl) + data_size;
713     if (vhost_svq_available_slots(svq) < 2 ||
714         iov_size(out_cursor, 1) < cmd_size) {
715         /*
716          * It is time to flush all pending control commands if SVQ is full
717          * or control commands shadow buffers are full.
718          *
719          * We can poll here since we've had BQL from the time
720          * we sent the descriptor.
721          */
722         r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base -
723                                      (void *)s->status);
724         if (unlikely(r < 0)) {
725             return r;
726         }
727 
728         vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor);
729     }
730 
731     /* pack the CVQ command header */
732     iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl));
733     /* pack the CVQ command command-specific-data */
734     iov_to_buf(data_sg, data_num, 0,
735                out_cursor->iov_base + sizeof(ctrl), data_size);
736 
737     /* extract the required buffer from the cursor for output */
738     iov_copy(&out, 1, out_cursor, 1, 0, cmd_size);
739     /* extract the required buffer from the cursor for input */
740     iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status));
741 
742     r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1);
743     if (unlikely(r < 0)) {
744         return r;
745     }
746 
747     /* iterate the cursors */
748     dummy_cursor_iov_cnt = 1;
749     iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size);
750     dummy_cursor_iov_cnt = 1;
751     iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status));
752 
753     return 0;
754 }
755 
756 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n,
757                                    struct iovec *out_cursor,
758                                    struct iovec *in_cursor)
759 {
760     if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
761         const struct iovec data = {
762             .iov_base = (void *)n->mac,
763             .iov_len = sizeof(n->mac),
764         };
765         ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
766                                             VIRTIO_NET_CTRL_MAC,
767                                             VIRTIO_NET_CTRL_MAC_ADDR_SET,
768                                             &data, 1);
769         if (unlikely(r < 0)) {
770             return r;
771         }
772     }
773 
774     /*
775      * According to VirtIO standard, "The device MUST have an
776      * empty MAC filtering table on reset.".
777      *
778      * Therefore, there is no need to send this CVQ command if the
779      * driver also sets an empty MAC filter table, which aligns with
780      * the device's defaults.
781      *
782      * Note that the device's defaults can mismatch the driver's
783      * configuration only at live migration.
784      */
785     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) ||
786         n->mac_table.in_use == 0) {
787         return 0;
788     }
789 
790     uint32_t uni_entries = n->mac_table.first_multi,
791              uni_macs_size = uni_entries * ETH_ALEN,
792              mul_entries = n->mac_table.in_use - uni_entries,
793              mul_macs_size = mul_entries * ETH_ALEN;
794     struct virtio_net_ctrl_mac uni = {
795         .entries = cpu_to_le32(uni_entries),
796     };
797     struct virtio_net_ctrl_mac mul = {
798         .entries = cpu_to_le32(mul_entries),
799     };
800     const struct iovec data[] = {
801         {
802             .iov_base = &uni,
803             .iov_len = sizeof(uni),
804         }, {
805             .iov_base = n->mac_table.macs,
806             .iov_len = uni_macs_size,
807         }, {
808             .iov_base = &mul,
809             .iov_len = sizeof(mul),
810         }, {
811             .iov_base = &n->mac_table.macs[uni_macs_size],
812             .iov_len = mul_macs_size,
813         },
814     };
815     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
816                                         VIRTIO_NET_CTRL_MAC,
817                                         VIRTIO_NET_CTRL_MAC_TABLE_SET,
818                                         data, ARRAY_SIZE(data));
819     if (unlikely(r < 0)) {
820         return r;
821     }
822 
823     return 0;
824 }
825 
826 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
827                                    struct iovec *out_cursor,
828                                    struct iovec *in_cursor, bool do_rss)
829 {
830     struct virtio_net_rss_config cfg = {};
831     ssize_t r;
832     g_autofree uint16_t *table = NULL;
833 
834     /*
835      * According to VirtIO standard, "Initially the device has all hash
836      * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
837      *
838      * Therefore, there is no need to send this CVQ command if the
839      * driver disables the all hash types, which aligns with
840      * the device's defaults.
841      *
842      * Note that the device's defaults can mismatch the driver's
843      * configuration only at live migration.
844      */
845     if (!n->rss_data.enabled ||
846         n->rss_data.hash_types == VIRTIO_NET_HASH_REPORT_NONE) {
847         return 0;
848     }
849 
850     table = g_malloc_n(n->rss_data.indirections_len,
851                        sizeof(n->rss_data.indirections_table[0]));
852     cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
853 
854     if (do_rss) {
855         /*
856          * According to VirtIO standard, "Number of entries in indirection_table
857          * is (indirection_table_mask + 1)".
858          */
859         cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len -
860                                                  1);
861         cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue);
862         for (int i = 0; i < n->rss_data.indirections_len; ++i) {
863             table[i] = cpu_to_le16(n->rss_data.indirections_table[i]);
864         }
865         cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs);
866     } else {
867         /*
868          * According to VirtIO standard, "Field reserved MUST contain zeroes.
869          * It is defined to make the structure to match the layout of
870          * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
871          *
872          * Therefore, we need to zero the fields in
873          * struct virtio_net_rss_config, which corresponds to the
874          * `reserved` field in struct virtio_net_hash_config.
875          *
876          * Note that all other fields are zeroed at their definitions,
877          * except for the `indirection_table` field, where the actual data
878          * is stored in the `table` variable to ensure compatibility
879          * with RSS case. Therefore, we need to zero the `table` variable here.
880          */
881         table[0] = 0;
882     }
883 
884     /*
885      * Considering that virtio_net_handle_rss() currently does not restore
886      * the hash key length parsed from the CVQ command sent from the guest
887      * into n->rss_data and uses the maximum key length in other code, so
888      * we also employ the maximum key length here.
889      */
890     cfg.hash_key_length = sizeof(n->rss_data.key);
891 
892     const struct iovec data[] = {
893         {
894             .iov_base = &cfg,
895             .iov_len = offsetof(struct virtio_net_rss_config,
896                                 indirection_table),
897         }, {
898             .iov_base = table,
899             .iov_len = n->rss_data.indirections_len *
900                        sizeof(n->rss_data.indirections_table[0]),
901         }, {
902             .iov_base = &cfg.max_tx_vq,
903             .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) -
904                        offsetof(struct virtio_net_rss_config, max_tx_vq),
905         }, {
906             .iov_base = (void *)n->rss_data.key,
907             .iov_len = sizeof(n->rss_data.key),
908         }
909     };
910 
911     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
912                                 VIRTIO_NET_CTRL_MQ,
913                                 do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG :
914                                 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
915                                 data, ARRAY_SIZE(data));
916     if (unlikely(r < 0)) {
917         return r;
918     }
919 
920     return 0;
921 }
922 
923 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
924                                   const VirtIONet *n,
925                                   struct iovec *out_cursor,
926                                   struct iovec *in_cursor)
927 {
928     struct virtio_net_ctrl_mq mq;
929     ssize_t r;
930 
931     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) {
932         return 0;
933     }
934 
935     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
936     const struct iovec data = {
937         .iov_base = &mq,
938         .iov_len = sizeof(mq),
939     };
940     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
941                                 VIRTIO_NET_CTRL_MQ,
942                                 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
943                                 &data, 1);
944     if (unlikely(r < 0)) {
945         return r;
946     }
947 
948     if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
949         /* load the receive-side scaling state */
950         r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true);
951         if (unlikely(r < 0)) {
952             return r;
953         }
954     } else if (virtio_vdev_has_feature(&n->parent_obj,
955                                        VIRTIO_NET_F_HASH_REPORT)) {
956         /* load the hash calculation state */
957         r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
958         if (unlikely(r < 0)) {
959             return r;
960         }
961     }
962 
963     return 0;
964 }
965 
966 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s,
967                                         const VirtIONet *n,
968                                         struct iovec *out_cursor,
969                                         struct iovec *in_cursor)
970 {
971     uint64_t offloads;
972     ssize_t r;
973 
974     if (!virtio_vdev_has_feature(&n->parent_obj,
975                                  VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
976         return 0;
977     }
978 
979     if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) {
980         /*
981          * According to VirtIO standard, "Upon feature negotiation
982          * corresponding offload gets enabled to preserve
983          * backward compatibility.".
984          *
985          * Therefore, there is no need to send this CVQ command if the
986          * driver also enables all supported offloads, which aligns with
987          * the device's defaults.
988          *
989          * Note that the device's defaults can mismatch the driver's
990          * configuration only at live migration.
991          */
992         return 0;
993     }
994 
995     offloads = cpu_to_le64(n->curr_guest_offloads);
996     const struct iovec data = {
997         .iov_base = &offloads,
998         .iov_len = sizeof(offloads),
999     };
1000     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1001                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS,
1002                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET,
1003                                 &data, 1);
1004     if (unlikely(r < 0)) {
1005         return r;
1006     }
1007 
1008     return 0;
1009 }
1010 
1011 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s,
1012                                        struct iovec *out_cursor,
1013                                        struct iovec *in_cursor,
1014                                        uint8_t cmd,
1015                                        uint8_t on)
1016 {
1017     const struct iovec data = {
1018         .iov_base = &on,
1019         .iov_len = sizeof(on),
1020     };
1021     ssize_t r;
1022 
1023     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1024                                 VIRTIO_NET_CTRL_RX, cmd, &data, 1);
1025     if (unlikely(r < 0)) {
1026         return r;
1027     }
1028 
1029     return 0;
1030 }
1031 
1032 static int vhost_vdpa_net_load_rx(VhostVDPAState *s,
1033                                   const VirtIONet *n,
1034                                   struct iovec *out_cursor,
1035                                   struct iovec *in_cursor)
1036 {
1037     ssize_t r;
1038 
1039     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) {
1040         return 0;
1041     }
1042 
1043     /*
1044      * According to virtio_net_reset(), device turns promiscuous mode
1045      * on by default.
1046      *
1047      * Additionally, according to VirtIO standard, "Since there are
1048      * no guarantees, it can use a hash filter or silently switch to
1049      * allmulti or promiscuous mode if it is given too many addresses.".
1050      * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1051      * non-multicast MAC addresses, indicating that promiscuous mode
1052      * should be enabled.
1053      *
1054      * Therefore, QEMU should only send this CVQ command if the
1055      * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1056      * which sets promiscuous mode on, different from the device's defaults.
1057      *
1058      * Note that the device's defaults can mismatch the driver's
1059      * configuration only at live migration.
1060      */
1061     if (!n->mac_table.uni_overflow && !n->promisc) {
1062         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1063                                         VIRTIO_NET_CTRL_RX_PROMISC, 0);
1064         if (unlikely(r < 0)) {
1065             return r;
1066         }
1067     }
1068 
1069     /*
1070      * According to virtio_net_reset(), device turns all-multicast mode
1071      * off by default.
1072      *
1073      * According to VirtIO standard, "Since there are no guarantees,
1074      * it can use a hash filter or silently switch to allmulti or
1075      * promiscuous mode if it is given too many addresses.". QEMU marks
1076      * `n->mac_table.multi_overflow` if guest sets too many
1077      * non-multicast MAC addresses.
1078      *
1079      * Therefore, QEMU should only send this CVQ command if the
1080      * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1081      * which sets all-multicast mode on, different from the device's defaults.
1082      *
1083      * Note that the device's defaults can mismatch the driver's
1084      * configuration only at live migration.
1085      */
1086     if (n->mac_table.multi_overflow || n->allmulti) {
1087         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1088                                         VIRTIO_NET_CTRL_RX_ALLMULTI, 1);
1089         if (unlikely(r < 0)) {
1090             return r;
1091         }
1092     }
1093 
1094     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) {
1095         return 0;
1096     }
1097 
1098     /*
1099      * According to virtio_net_reset(), device turns all-unicast mode
1100      * off by default.
1101      *
1102      * Therefore, QEMU should only send this CVQ command if the driver
1103      * sets all-unicast mode on, different from the device's defaults.
1104      *
1105      * Note that the device's defaults can mismatch the driver's
1106      * configuration only at live migration.
1107      */
1108     if (n->alluni) {
1109         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1110                                         VIRTIO_NET_CTRL_RX_ALLUNI, 1);
1111         if (r < 0) {
1112             return r;
1113         }
1114     }
1115 
1116     /*
1117      * According to virtio_net_reset(), device turns non-multicast mode
1118      * off by default.
1119      *
1120      * Therefore, QEMU should only send this CVQ command if the driver
1121      * sets non-multicast mode on, different from the device's defaults.
1122      *
1123      * Note that the device's defaults can mismatch the driver's
1124      * configuration only at live migration.
1125      */
1126     if (n->nomulti) {
1127         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1128                                         VIRTIO_NET_CTRL_RX_NOMULTI, 1);
1129         if (r < 0) {
1130             return r;
1131         }
1132     }
1133 
1134     /*
1135      * According to virtio_net_reset(), device turns non-unicast mode
1136      * off by default.
1137      *
1138      * Therefore, QEMU should only send this CVQ command if the driver
1139      * sets non-unicast mode on, different from the device's defaults.
1140      *
1141      * Note that the device's defaults can mismatch the driver's
1142      * configuration only at live migration.
1143      */
1144     if (n->nouni) {
1145         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1146                                         VIRTIO_NET_CTRL_RX_NOUNI, 1);
1147         if (r < 0) {
1148             return r;
1149         }
1150     }
1151 
1152     /*
1153      * According to virtio_net_reset(), device turns non-broadcast mode
1154      * off by default.
1155      *
1156      * Therefore, QEMU should only send this CVQ command if the driver
1157      * sets non-broadcast mode on, different from the device's defaults.
1158      *
1159      * Note that the device's defaults can mismatch the driver's
1160      * configuration only at live migration.
1161      */
1162     if (n->nobcast) {
1163         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1164                                         VIRTIO_NET_CTRL_RX_NOBCAST, 1);
1165         if (r < 0) {
1166             return r;
1167         }
1168     }
1169 
1170     return 0;
1171 }
1172 
1173 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s,
1174                                            const VirtIONet *n,
1175                                            struct iovec *out_cursor,
1176                                            struct iovec *in_cursor,
1177                                            uint16_t vid)
1178 {
1179     const struct iovec data = {
1180         .iov_base = &vid,
1181         .iov_len = sizeof(vid),
1182     };
1183     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1184                                         VIRTIO_NET_CTRL_VLAN,
1185                                         VIRTIO_NET_CTRL_VLAN_ADD,
1186                                         &data, 1);
1187     if (unlikely(r < 0)) {
1188         return r;
1189     }
1190 
1191     return 0;
1192 }
1193 
1194 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s,
1195                                     const VirtIONet *n,
1196                                     struct iovec *out_cursor,
1197                                     struct iovec *in_cursor)
1198 {
1199     int r;
1200 
1201     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) {
1202         return 0;
1203     }
1204 
1205     for (int i = 0; i < MAX_VLAN >> 5; i++) {
1206         for (int j = 0; n->vlans[i] && j <= 0x1f; j++) {
1207             if (n->vlans[i] & (1U << j)) {
1208                 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor,
1209                                                     in_cursor, (i << 5) + j);
1210                 if (unlikely(r != 0)) {
1211                     return r;
1212                 }
1213             }
1214         }
1215     }
1216 
1217     return 0;
1218 }
1219 
1220 static int vhost_vdpa_net_cvq_load(NetClientState *nc)
1221 {
1222     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
1223     struct vhost_vdpa *v = &s->vhost_vdpa;
1224     const VirtIONet *n;
1225     int r;
1226     struct iovec out_cursor, in_cursor;
1227 
1228     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1229 
1230     vhost_vdpa_set_vring_ready(v, v->dev->vq_index);
1231 
1232     if (v->shadow_vqs_enabled) {
1233         n = VIRTIO_NET(v->dev->vdev);
1234         vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor);
1235         r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor);
1236         if (unlikely(r < 0)) {
1237             return r;
1238         }
1239         r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor);
1240         if (unlikely(r)) {
1241             return r;
1242         }
1243         r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor);
1244         if (unlikely(r)) {
1245             return r;
1246         }
1247         r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor);
1248         if (unlikely(r)) {
1249             return r;
1250         }
1251         r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor);
1252         if (unlikely(r)) {
1253             return r;
1254         }
1255 
1256         /*
1257          * We need to poll and check all pending device's used buffers.
1258          *
1259          * We can poll here since we've had BQL from the time
1260          * we sent the descriptor.
1261          */
1262         r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status);
1263         if (unlikely(r)) {
1264             return r;
1265         }
1266     }
1267 
1268     for (int i = 0; i < v->dev->vq_index; ++i) {
1269         vhost_vdpa_set_vring_ready(v, i);
1270     }
1271 
1272     return 0;
1273 }
1274 
1275 static NetClientInfo net_vhost_vdpa_cvq_info = {
1276     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
1277     .size = sizeof(VhostVDPAState),
1278     .receive = vhost_vdpa_receive,
1279     .start = vhost_vdpa_net_cvq_start,
1280     .load = vhost_vdpa_net_cvq_load,
1281     .stop = vhost_vdpa_net_cvq_stop,
1282     .cleanup = vhost_vdpa_cleanup,
1283     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
1284     .has_ufo = vhost_vdpa_has_ufo,
1285     .check_peer_type = vhost_vdpa_check_peer_type,
1286     .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
1287 };
1288 
1289 /*
1290  * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1291  * vdpa device.
1292  *
1293  * Considering that QEMU cannot send the entire filter table to the
1294  * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1295  * command to enable promiscuous mode to receive all packets,
1296  * according to VirtIO standard, "Since there are no guarantees,
1297  * it can use a hash filter or silently switch to allmulti or
1298  * promiscuous mode if it is given too many addresses.".
1299  *
1300  * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1301  * marks `n->mac_table.x_overflow` accordingly, it should have
1302  * the same effect on the device model to receive
1303  * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1304  * The same applies to multicast MAC addresses.
1305  *
1306  * Therefore, QEMU can provide the device model with a fake
1307  * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1308  * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1309  * MAC addresses. This ensures that the device model marks
1310  * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1311  * allowing all packets to be received, which aligns with the
1312  * state of the vdpa device.
1313  */
1314 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s,
1315                                                        VirtQueueElement *elem,
1316                                                        struct iovec *out,
1317                                                        const struct iovec *in)
1318 {
1319     struct virtio_net_ctrl_mac mac_data, *mac_ptr;
1320     struct virtio_net_ctrl_hdr *hdr_ptr;
1321     uint32_t cursor;
1322     ssize_t r;
1323     uint8_t on = 1;
1324 
1325     /* parse the non-multicast MAC address entries from CVQ command */
1326     cursor = sizeof(*hdr_ptr);
1327     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1328                    &mac_data, sizeof(mac_data));
1329     if (unlikely(r != sizeof(mac_data))) {
1330         /*
1331          * If the CVQ command is invalid, we should simulate the vdpa device
1332          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1333          */
1334         *s->status = VIRTIO_NET_ERR;
1335         return sizeof(*s->status);
1336     }
1337     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1338 
1339     /* parse the multicast MAC address entries from CVQ command */
1340     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1341                    &mac_data, sizeof(mac_data));
1342     if (r != sizeof(mac_data)) {
1343         /*
1344          * If the CVQ command is invalid, we should simulate the vdpa device
1345          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1346          */
1347         *s->status = VIRTIO_NET_ERR;
1348         return sizeof(*s->status);
1349     }
1350     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1351 
1352     /* validate the CVQ command */
1353     if (iov_size(elem->out_sg, elem->out_num) != cursor) {
1354         /*
1355          * If the CVQ command is invalid, we should simulate the vdpa device
1356          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1357          */
1358         *s->status = VIRTIO_NET_ERR;
1359         return sizeof(*s->status);
1360     }
1361 
1362     /*
1363      * According to VirtIO standard, "Since there are no guarantees,
1364      * it can use a hash filter or silently switch to allmulti or
1365      * promiscuous mode if it is given too many addresses.".
1366      *
1367      * Therefore, considering that QEMU is unable to send the entire
1368      * filter table to the vdpa device, it should send the
1369      * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1370      */
1371     hdr_ptr = out->iov_base;
1372     out->iov_len = sizeof(*hdr_ptr) + sizeof(on);
1373 
1374     hdr_ptr->class = VIRTIO_NET_CTRL_RX;
1375     hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC;
1376     iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on));
1377     r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1);
1378     if (unlikely(r < 0)) {
1379         return r;
1380     }
1381 
1382     /*
1383      * We can poll here since we've had BQL from the time
1384      * we sent the descriptor.
1385      */
1386     r = vhost_vdpa_net_svq_poll(s, 1);
1387     if (unlikely(r < sizeof(*s->status))) {
1388         return r;
1389     }
1390     if (*s->status != VIRTIO_NET_OK) {
1391         return sizeof(*s->status);
1392     }
1393 
1394     /*
1395      * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1396      * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1397      * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1398      * multicast MAC addresses.
1399      *
1400      * By doing so, the device model can mark `n->mac_table.uni_overflow`
1401      * and `n->mac_table.multi_overflow`, enabling all packets to be
1402      * received, which aligns with the state of the vdpa device.
1403      */
1404     cursor = 0;
1405     uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1,
1406              fake_mul_entries = MAC_TABLE_ENTRIES + 1,
1407              fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) +
1408                              sizeof(mac_data) + fake_uni_entries * ETH_ALEN +
1409                              sizeof(mac_data) + fake_mul_entries * ETH_ALEN;
1410 
1411     assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len());
1412     out->iov_len = fake_cvq_size;
1413 
1414     /* pack the header for fake CVQ command */
1415     hdr_ptr = out->iov_base + cursor;
1416     hdr_ptr->class = VIRTIO_NET_CTRL_MAC;
1417     hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1418     cursor += sizeof(*hdr_ptr);
1419 
1420     /*
1421      * Pack the non-multicast MAC addresses part for fake CVQ command.
1422      *
1423      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1424      * addresses provided in CVQ command. Therefore, only the entries
1425      * field need to be prepared in the CVQ command.
1426      */
1427     mac_ptr = out->iov_base + cursor;
1428     mac_ptr->entries = cpu_to_le32(fake_uni_entries);
1429     cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN;
1430 
1431     /*
1432      * Pack the multicast MAC addresses part for fake CVQ command.
1433      *
1434      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1435      * addresses provided in CVQ command. Therefore, only the entries
1436      * field need to be prepared in the CVQ command.
1437      */
1438     mac_ptr = out->iov_base + cursor;
1439     mac_ptr->entries = cpu_to_le32(fake_mul_entries);
1440 
1441     /*
1442      * Simulating QEMU poll a vdpa device used buffer
1443      * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1444      */
1445     return sizeof(*s->status);
1446 }
1447 
1448 /**
1449  * Validate and copy control virtqueue commands.
1450  *
1451  * Following QEMU guidelines, we offer a copy of the buffers to the device to
1452  * prevent TOCTOU bugs.
1453  */
1454 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
1455                                             VirtQueueElement *elem,
1456                                             void *opaque)
1457 {
1458     VhostVDPAState *s = opaque;
1459     size_t in_len;
1460     const struct virtio_net_ctrl_hdr *ctrl;
1461     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1462     /* Out buffer sent to both the vdpa device and the device model */
1463     struct iovec out = {
1464         .iov_base = s->cvq_cmd_out_buffer,
1465     };
1466     /* in buffer used for device model */
1467     const struct iovec model_in = {
1468         .iov_base = &status,
1469         .iov_len = sizeof(status),
1470     };
1471     /* in buffer used for vdpa device */
1472     const struct iovec vdpa_in = {
1473         .iov_base = s->status,
1474         .iov_len = sizeof(*s->status),
1475     };
1476     ssize_t dev_written = -EINVAL;
1477 
1478     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
1479                              s->cvq_cmd_out_buffer,
1480                              vhost_vdpa_net_cvq_cmd_page_len());
1481 
1482     ctrl = s->cvq_cmd_out_buffer;
1483     if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) {
1484         /*
1485          * Guest announce capability is emulated by qemu, so don't forward to
1486          * the device.
1487          */
1488         dev_written = sizeof(status);
1489         *s->status = VIRTIO_NET_OK;
1490     } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC &&
1491                         ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET &&
1492                         iov_size(elem->out_sg, elem->out_num) > out.iov_len)) {
1493         /*
1494          * Due to the size limitation of the out buffer sent to the vdpa device,
1495          * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1496          * MAC addresses set by the driver for the filter table can cause
1497          * truncation of the CVQ command in QEMU. As a result, the vdpa device
1498          * rejects the flawed CVQ command.
1499          *
1500          * Therefore, QEMU must handle this situation instead of sending
1501          * the CVQ command directly.
1502          */
1503         dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem,
1504                                                             &out, &vdpa_in);
1505         if (unlikely(dev_written < 0)) {
1506             goto out;
1507         }
1508     } else {
1509         ssize_t r;
1510         r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1);
1511         if (unlikely(r < 0)) {
1512             dev_written = r;
1513             goto out;
1514         }
1515 
1516         /*
1517          * We can poll here since we've had BQL from the time
1518          * we sent the descriptor.
1519          */
1520         dev_written = vhost_vdpa_net_svq_poll(s, 1);
1521     }
1522 
1523     if (unlikely(dev_written < sizeof(status))) {
1524         error_report("Insufficient written data (%zu)", dev_written);
1525         goto out;
1526     }
1527 
1528     if (*s->status != VIRTIO_NET_OK) {
1529         goto out;
1530     }
1531 
1532     status = VIRTIO_NET_ERR;
1533     virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1);
1534     if (status != VIRTIO_NET_OK) {
1535         error_report("Bad CVQ processing in model");
1536     }
1537 
1538 out:
1539     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
1540                           sizeof(status));
1541     if (unlikely(in_len < sizeof(status))) {
1542         error_report("Bad device CVQ written length");
1543     }
1544     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
1545     /*
1546      * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1547      * the function successfully forwards the CVQ command, indicated
1548      * by a non-negative value of `dev_written`. Otherwise, it still
1549      * belongs to SVQ.
1550      * This function should only free the `elem` when it owns.
1551      */
1552     if (dev_written >= 0) {
1553         g_free(elem);
1554     }
1555     return dev_written < 0 ? dev_written : 0;
1556 }
1557 
1558 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
1559     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
1560 };
1561 
1562 /**
1563  * Probe if CVQ is isolated
1564  *
1565  * @device_fd         The vdpa device fd
1566  * @features          Features offered by the device.
1567  * @cvq_index         The control vq pair index
1568  *
1569  * Returns <0 in case of failure, 0 if false and 1 if true.
1570  */
1571 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features,
1572                                           int cvq_index, Error **errp)
1573 {
1574     uint64_t backend_features;
1575     int64_t cvq_group;
1576     uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE |
1577                      VIRTIO_CONFIG_S_DRIVER;
1578     int r;
1579 
1580     ERRP_GUARD();
1581 
1582     r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
1583     if (unlikely(r < 0)) {
1584         error_setg_errno(errp, errno, "Cannot get vdpa backend_features");
1585         return r;
1586     }
1587 
1588     if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) {
1589         return 0;
1590     }
1591 
1592     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1593     if (unlikely(r)) {
1594         error_setg_errno(errp, -r, "Cannot set device status");
1595         goto out;
1596     }
1597 
1598     r = ioctl(device_fd, VHOST_SET_FEATURES, &features);
1599     if (unlikely(r)) {
1600         error_setg_errno(errp, -r, "Cannot set features");
1601         goto out;
1602     }
1603 
1604     status |= VIRTIO_CONFIG_S_FEATURES_OK;
1605     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1606     if (unlikely(r)) {
1607         error_setg_errno(errp, -r, "Cannot set device status");
1608         goto out;
1609     }
1610 
1611     cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp);
1612     if (unlikely(cvq_group < 0)) {
1613         if (cvq_group != -ENOTSUP) {
1614             r = cvq_group;
1615             goto out;
1616         }
1617 
1618         /*
1619          * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1620          * support ASID even if the parent driver does not.  The CVQ cannot be
1621          * isolated in this case.
1622          */
1623         error_free(*errp);
1624         *errp = NULL;
1625         r = 0;
1626         goto out;
1627     }
1628 
1629     for (int i = 0; i < cvq_index; ++i) {
1630         int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp);
1631         if (unlikely(group < 0)) {
1632             r = group;
1633             goto out;
1634         }
1635 
1636         if (group == (int64_t)cvq_group) {
1637             r = 0;
1638             goto out;
1639         }
1640     }
1641 
1642     r = 1;
1643 
1644 out:
1645     status = 0;
1646     ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1647     return r;
1648 }
1649 
1650 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
1651                                        const char *device,
1652                                        const char *name,
1653                                        int vdpa_device_fd,
1654                                        int queue_pair_index,
1655                                        int nvqs,
1656                                        bool is_datapath,
1657                                        bool svq,
1658                                        struct vhost_vdpa_iova_range iova_range,
1659                                        uint64_t features,
1660                                        VhostVDPAShared *shared,
1661                                        Error **errp)
1662 {
1663     NetClientState *nc = NULL;
1664     VhostVDPAState *s;
1665     int ret = 0;
1666     assert(name);
1667     int cvq_isolated = 0;
1668 
1669     if (is_datapath) {
1670         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
1671                                  name);
1672     } else {
1673         cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features,
1674                                                       queue_pair_index * 2,
1675                                                       errp);
1676         if (unlikely(cvq_isolated < 0)) {
1677             return NULL;
1678         }
1679 
1680         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
1681                                          device, name);
1682     }
1683     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
1684     s = DO_UPCAST(VhostVDPAState, nc, nc);
1685 
1686     s->vhost_vdpa.device_fd = vdpa_device_fd;
1687     s->vhost_vdpa.index = queue_pair_index;
1688     s->always_svq = svq;
1689     s->migration_state.notify = NULL;
1690     s->vhost_vdpa.shadow_vqs_enabled = svq;
1691     s->vhost_vdpa.iova_range = iova_range;
1692     s->vhost_vdpa.shadow_data = svq;
1693     if (queue_pair_index == 0) {
1694         vhost_vdpa_net_valid_svq_features(features,
1695                                           &s->vhost_vdpa.migration_blocker);
1696         s->vhost_vdpa.shared = g_new0(VhostVDPAShared, 1);
1697     } else if (!is_datapath) {
1698         s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1699                                      PROT_READ | PROT_WRITE,
1700                                      MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1701         s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1702                          PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
1703                          -1, 0);
1704 
1705         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
1706         s->vhost_vdpa.shadow_vq_ops_opaque = s;
1707         s->cvq_isolated = cvq_isolated;
1708     }
1709     if (queue_pair_index != 0) {
1710         s->vhost_vdpa.shared = shared;
1711     }
1712 
1713     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
1714     if (ret) {
1715         qemu_del_net_client(nc);
1716         return NULL;
1717     }
1718 
1719     return nc;
1720 }
1721 
1722 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
1723 {
1724     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
1725     if (unlikely(ret < 0)) {
1726         error_setg_errno(errp, errno,
1727                          "Fail to query features from vhost-vDPA device");
1728     }
1729     return ret;
1730 }
1731 
1732 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
1733                                           int *has_cvq, Error **errp)
1734 {
1735     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
1736     g_autofree struct vhost_vdpa_config *config = NULL;
1737     __virtio16 *max_queue_pairs;
1738     int ret;
1739 
1740     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
1741         *has_cvq = 1;
1742     } else {
1743         *has_cvq = 0;
1744     }
1745 
1746     if (features & (1 << VIRTIO_NET_F_MQ)) {
1747         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
1748         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
1749         config->len = sizeof(*max_queue_pairs);
1750 
1751         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
1752         if (ret) {
1753             error_setg(errp, "Fail to get config from vhost-vDPA device");
1754             return -ret;
1755         }
1756 
1757         max_queue_pairs = (__virtio16 *)&config->buf;
1758 
1759         return lduw_le_p(max_queue_pairs);
1760     }
1761 
1762     return 1;
1763 }
1764 
1765 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
1766                         NetClientState *peer, Error **errp)
1767 {
1768     const NetdevVhostVDPAOptions *opts;
1769     uint64_t features;
1770     int vdpa_device_fd;
1771     g_autofree NetClientState **ncs = NULL;
1772     struct vhost_vdpa_iova_range iova_range;
1773     NetClientState *nc;
1774     int queue_pairs, r, i = 0, has_cvq = 0;
1775 
1776     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1777     opts = &netdev->u.vhost_vdpa;
1778     if (!opts->vhostdev && !opts->vhostfd) {
1779         error_setg(errp,
1780                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1781         return -1;
1782     }
1783 
1784     if (opts->vhostdev && opts->vhostfd) {
1785         error_setg(errp,
1786                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1787         return -1;
1788     }
1789 
1790     if (opts->vhostdev) {
1791         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
1792         if (vdpa_device_fd == -1) {
1793             return -errno;
1794         }
1795     } else {
1796         /* has_vhostfd */
1797         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
1798         if (vdpa_device_fd == -1) {
1799             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
1800             return -1;
1801         }
1802     }
1803 
1804     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
1805     if (unlikely(r < 0)) {
1806         goto err;
1807     }
1808 
1809     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
1810                                                  &has_cvq, errp);
1811     if (queue_pairs < 0) {
1812         qemu_close(vdpa_device_fd);
1813         return queue_pairs;
1814     }
1815 
1816     r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
1817     if (unlikely(r < 0)) {
1818         error_setg(errp, "vhost-vdpa: get iova range failed: %s",
1819                    strerror(-r));
1820         goto err;
1821     }
1822 
1823     if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
1824         goto err;
1825     }
1826 
1827     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
1828 
1829     for (i = 0; i < queue_pairs; i++) {
1830         VhostVDPAShared *shared = NULL;
1831 
1832         if (i) {
1833             shared = DO_UPCAST(VhostVDPAState, nc, ncs[0])->vhost_vdpa.shared;
1834         }
1835         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1836                                      vdpa_device_fd, i, 2, true, opts->x_svq,
1837                                      iova_range, features, shared, errp);
1838         if (!ncs[i])
1839             goto err;
1840     }
1841 
1842     if (has_cvq) {
1843         VhostVDPAState *s0 = DO_UPCAST(VhostVDPAState, nc, ncs[0]);
1844         VhostVDPAShared *shared = s0->vhost_vdpa.shared;
1845 
1846         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1847                                  vdpa_device_fd, i, 1, false,
1848                                  opts->x_svq, iova_range, features, shared,
1849                                  errp);
1850         if (!nc)
1851             goto err;
1852     }
1853 
1854     return 0;
1855 
1856 err:
1857     if (i) {
1858         for (i--; i >= 0; i--) {
1859             qemu_del_net_client(ncs[i]);
1860         }
1861     }
1862 
1863     qemu_close(vdpa_device_fd);
1864 
1865     return -1;
1866 }
1867