xref: /openbmc/qemu/net/vhost-vdpa.c (revision 8b98c15f22fca34f23846e70752a70e4a2469317)
1 /*
2  * vhost-vdpa.c
3  *
4  * Copyright(c) 2017-2018 Intel Corporation.
5  * Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/log.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
26 #include <err.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/migration.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
32 
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState {
35     NetClientState nc;
36     struct vhost_vdpa vhost_vdpa;
37     Notifier migration_state;
38     VHostNetState *vhost_net;
39 
40     /* Control commands shadow buffers */
41     void *cvq_cmd_out_buffer;
42     virtio_net_ctrl_ack *status;
43 
44     /* The device always have SVQ enabled */
45     bool always_svq;
46 
47     /* The device can isolate CVQ in its own ASID */
48     bool cvq_isolated;
49 
50     bool started;
51 } VhostVDPAState;
52 
53 /*
54  * The array is sorted alphabetically in ascending order,
55  * with the exception of VHOST_INVALID_FEATURE_BIT,
56  * which should always be the last entry.
57  */
58 const int vdpa_feature_bits[] = {
59     VIRTIO_F_ANY_LAYOUT,
60     VIRTIO_F_IOMMU_PLATFORM,
61     VIRTIO_F_NOTIFY_ON_EMPTY,
62     VIRTIO_F_RING_PACKED,
63     VIRTIO_F_RING_RESET,
64     VIRTIO_F_VERSION_1,
65     VIRTIO_NET_F_CSUM,
66     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
67     VIRTIO_NET_F_CTRL_MAC_ADDR,
68     VIRTIO_NET_F_CTRL_RX,
69     VIRTIO_NET_F_CTRL_RX_EXTRA,
70     VIRTIO_NET_F_CTRL_VLAN,
71     VIRTIO_NET_F_CTRL_VQ,
72     VIRTIO_NET_F_GSO,
73     VIRTIO_NET_F_GUEST_CSUM,
74     VIRTIO_NET_F_GUEST_ECN,
75     VIRTIO_NET_F_GUEST_TSO4,
76     VIRTIO_NET_F_GUEST_TSO6,
77     VIRTIO_NET_F_GUEST_UFO,
78     VIRTIO_NET_F_GUEST_USO4,
79     VIRTIO_NET_F_GUEST_USO6,
80     VIRTIO_NET_F_HASH_REPORT,
81     VIRTIO_NET_F_HOST_ECN,
82     VIRTIO_NET_F_HOST_TSO4,
83     VIRTIO_NET_F_HOST_TSO6,
84     VIRTIO_NET_F_HOST_UFO,
85     VIRTIO_NET_F_HOST_USO,
86     VIRTIO_NET_F_MQ,
87     VIRTIO_NET_F_MRG_RXBUF,
88     VIRTIO_NET_F_MTU,
89     VIRTIO_NET_F_RSS,
90     VIRTIO_NET_F_STATUS,
91     VIRTIO_RING_F_EVENT_IDX,
92     VIRTIO_RING_F_INDIRECT_DESC,
93 
94     /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
95     VHOST_INVALID_FEATURE_BIT
96 };
97 
98 /** Supported device specific feature bits with SVQ */
99 static const uint64_t vdpa_svq_device_features =
100     BIT_ULL(VIRTIO_NET_F_CSUM) |
101     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
102     BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
103     BIT_ULL(VIRTIO_NET_F_MTU) |
104     BIT_ULL(VIRTIO_NET_F_MAC) |
105     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
106     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
107     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
108     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
109     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
110     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
111     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
112     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
113     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
114     BIT_ULL(VIRTIO_NET_F_STATUS) |
115     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
116     BIT_ULL(VIRTIO_NET_F_CTRL_RX) |
117     BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) |
118     BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) |
119     BIT_ULL(VIRTIO_NET_F_MQ) |
120     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
121     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
122     /* VHOST_F_LOG_ALL is exposed by SVQ */
123     BIT_ULL(VHOST_F_LOG_ALL) |
124     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
125     BIT_ULL(VIRTIO_NET_F_STANDBY) |
126     BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
127 
128 #define VHOST_VDPA_NET_CVQ_ASID 1
129 
130 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
131 {
132     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
133     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
134     return s->vhost_net;
135 }
136 
137 static size_t vhost_vdpa_net_cvq_cmd_len(void)
138 {
139     /*
140      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
141      * In buffer is always 1 byte, so it should fit here
142      */
143     return sizeof(struct virtio_net_ctrl_hdr) +
144            2 * sizeof(struct virtio_net_ctrl_mac) +
145            MAC_TABLE_ENTRIES * ETH_ALEN;
146 }
147 
148 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
149 {
150     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
151 }
152 
153 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
154 {
155     uint64_t invalid_dev_features =
156         features & ~vdpa_svq_device_features &
157         /* Transport are all accepted at this point */
158         ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
159                          VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
160 
161     if (invalid_dev_features) {
162         error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
163                    invalid_dev_features);
164         return false;
165     }
166 
167     return vhost_svq_valid_features(features, errp);
168 }
169 
170 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
171 {
172     uint32_t device_id;
173     int ret;
174     struct vhost_dev *hdev;
175 
176     hdev = (struct vhost_dev *)&net->dev;
177     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
178     if (device_id != VIRTIO_ID_NET) {
179         return -ENOTSUP;
180     }
181     return ret;
182 }
183 
184 static int vhost_vdpa_add(NetClientState *ncs, void *be,
185                           int queue_pair_index, int nvqs)
186 {
187     VhostNetOptions options;
188     struct vhost_net *net = NULL;
189     VhostVDPAState *s;
190     int ret;
191 
192     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
193     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
194     s = DO_UPCAST(VhostVDPAState, nc, ncs);
195     options.net_backend = ncs;
196     options.opaque      = be;
197     options.busyloop_timeout = 0;
198     options.nvqs = nvqs;
199 
200     net = vhost_net_init(&options);
201     if (!net) {
202         error_report("failed to init vhost_net for queue");
203         goto err_init;
204     }
205     s->vhost_net = net;
206     ret = vhost_vdpa_net_check_device_id(net);
207     if (ret) {
208         goto err_check;
209     }
210     return 0;
211 err_check:
212     vhost_net_cleanup(net);
213     g_free(net);
214 err_init:
215     return -1;
216 }
217 
218 static void vhost_vdpa_cleanup(NetClientState *nc)
219 {
220     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
221 
222     /*
223      * If a peer NIC is attached, do not cleanup anything.
224      * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
225      * when the guest is shutting down.
226      */
227     if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
228         return;
229     }
230     munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len());
231     munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len());
232     if (s->vhost_net) {
233         vhost_net_cleanup(s->vhost_net);
234         g_free(s->vhost_net);
235         s->vhost_net = NULL;
236     }
237      if (s->vhost_vdpa.device_fd >= 0) {
238         qemu_close(s->vhost_vdpa.device_fd);
239         s->vhost_vdpa.device_fd = -1;
240     }
241 }
242 
243 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
244 {
245     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
246 
247     return true;
248 }
249 
250 static bool vhost_vdpa_has_ufo(NetClientState *nc)
251 {
252     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
253     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
254     uint64_t features = 0;
255     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
256     features = vhost_net_get_features(s->vhost_net, features);
257     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
258 
259 }
260 
261 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
262                                        Error **errp)
263 {
264     const char *driver = object_class_get_name(oc);
265 
266     if (!g_str_has_prefix(driver, "virtio-net-")) {
267         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
268         return false;
269     }
270 
271     return true;
272 }
273 
274 /** Dummy receive in case qemu falls back to userland tap networking */
275 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
276                                   size_t size)
277 {
278     return size;
279 }
280 
281 /** From any vdpa net client, get the netclient of the first queue pair */
282 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
283 {
284     NICState *nic = qemu_get_nic(s->nc.peer);
285     NetClientState *nc0 = qemu_get_peer(nic->ncs, 0);
286 
287     return DO_UPCAST(VhostVDPAState, nc, nc0);
288 }
289 
290 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
291 {
292     struct vhost_vdpa *v = &s->vhost_vdpa;
293     VirtIONet *n;
294     VirtIODevice *vdev;
295     int data_queue_pairs, cvq, r;
296 
297     /* We are only called on the first data vqs and only if x-svq is not set */
298     if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
299         return;
300     }
301 
302     vdev = v->dev->vdev;
303     n = VIRTIO_NET(vdev);
304     if (!n->vhost_started) {
305         return;
306     }
307 
308     data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
309     cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
310                                   n->max_ncs - n->max_queue_pairs : 0;
311     /*
312      * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
313      * in the future and resume the device if read-only operations between
314      * suspend and reset goes wrong.
315      */
316     vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
317 
318     /* Start will check migration setup_or_active to configure or not SVQ */
319     r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
320     if (unlikely(r < 0)) {
321         error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
322     }
323 }
324 
325 static void vdpa_net_migration_state_notifier(Notifier *notifier, void *data)
326 {
327     MigrationState *migration = data;
328     VhostVDPAState *s = container_of(notifier, VhostVDPAState,
329                                      migration_state);
330 
331     if (migration_in_setup(migration)) {
332         vhost_vdpa_net_log_global_enable(s, true);
333     } else if (migration_has_failed(migration)) {
334         vhost_vdpa_net_log_global_enable(s, false);
335     }
336 }
337 
338 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
339 {
340     struct vhost_vdpa *v = &s->vhost_vdpa;
341 
342     migration_add_notifier(&s->migration_state,
343                            vdpa_net_migration_state_notifier);
344     if (v->shadow_vqs_enabled) {
345         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
346                                            v->iova_range.last);
347     }
348 }
349 
350 static int vhost_vdpa_net_data_start(NetClientState *nc)
351 {
352     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
353     struct vhost_vdpa *v = &s->vhost_vdpa;
354 
355     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
356 
357     if (s->always_svq ||
358         migration_is_setup_or_active(migrate_get_current()->state)) {
359         v->shadow_vqs_enabled = true;
360         v->shadow_data = true;
361     } else {
362         v->shadow_vqs_enabled = false;
363         v->shadow_data = false;
364     }
365 
366     if (v->index == 0) {
367         vhost_vdpa_net_data_start_first(s);
368         return 0;
369     }
370 
371     if (v->shadow_vqs_enabled) {
372         VhostVDPAState *s0 = vhost_vdpa_net_first_nc_vdpa(s);
373         v->iova_tree = s0->vhost_vdpa.iova_tree;
374     }
375 
376     return 0;
377 }
378 
379 static int vhost_vdpa_net_data_load(NetClientState *nc)
380 {
381     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
382     struct vhost_vdpa *v = &s->vhost_vdpa;
383     bool has_cvq = v->dev->vq_index_end % 2;
384 
385     if (has_cvq) {
386         return 0;
387     }
388 
389     for (int i = 0; i < v->dev->nvqs; ++i) {
390         vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index);
391     }
392     return 0;
393 }
394 
395 static void vhost_vdpa_net_client_stop(NetClientState *nc)
396 {
397     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
398     struct vhost_dev *dev;
399 
400     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
401 
402     if (s->vhost_vdpa.index == 0) {
403         migration_remove_notifier(&s->migration_state);
404     }
405 
406     dev = s->vhost_vdpa.dev;
407     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
408         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
409     } else {
410         s->vhost_vdpa.iova_tree = NULL;
411     }
412 }
413 
414 static NetClientInfo net_vhost_vdpa_info = {
415         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
416         .size = sizeof(VhostVDPAState),
417         .receive = vhost_vdpa_receive,
418         .start = vhost_vdpa_net_data_start,
419         .load = vhost_vdpa_net_data_load,
420         .stop = vhost_vdpa_net_client_stop,
421         .cleanup = vhost_vdpa_cleanup,
422         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
423         .has_ufo = vhost_vdpa_has_ufo,
424         .check_peer_type = vhost_vdpa_check_peer_type,
425 };
426 
427 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
428                                           Error **errp)
429 {
430     struct vhost_vring_state state = {
431         .index = vq_index,
432     };
433     int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
434 
435     if (unlikely(r < 0)) {
436         r = -errno;
437         error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index);
438         return r;
439     }
440 
441     return state.num;
442 }
443 
444 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
445                                            unsigned vq_group,
446                                            unsigned asid_num)
447 {
448     struct vhost_vring_state asid = {
449         .index = vq_group,
450         .num = asid_num,
451     };
452     int r;
453 
454     r = ioctl(v->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
455     if (unlikely(r < 0)) {
456         error_report("Can't set vq group %u asid %u, errno=%d (%s)",
457                      asid.index, asid.num, errno, g_strerror(errno));
458     }
459     return r;
460 }
461 
462 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
463 {
464     VhostIOVATree *tree = v->iova_tree;
465     DMAMap needle = {
466         /*
467          * No need to specify size or to look for more translations since
468          * this contiguous chunk was allocated by us.
469          */
470         .translated_addr = (hwaddr)(uintptr_t)addr,
471     };
472     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
473     int r;
474 
475     if (unlikely(!map)) {
476         error_report("Cannot locate expected map");
477         return;
478     }
479 
480     r = vhost_vdpa_dma_unmap(v, v->address_space_id, map->iova, map->size + 1);
481     if (unlikely(r != 0)) {
482         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
483     }
484 
485     vhost_iova_tree_remove(tree, *map);
486 }
487 
488 /** Map CVQ buffer. */
489 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
490                                   bool write)
491 {
492     DMAMap map = {};
493     int r;
494 
495     map.translated_addr = (hwaddr)(uintptr_t)buf;
496     map.size = size - 1;
497     map.perm = write ? IOMMU_RW : IOMMU_RO,
498     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
499     if (unlikely(r != IOVA_OK)) {
500         error_report("Cannot map injected element");
501         return r;
502     }
503 
504     r = vhost_vdpa_dma_map(v, v->address_space_id, map.iova,
505                            vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
506     if (unlikely(r < 0)) {
507         goto dma_map_err;
508     }
509 
510     return 0;
511 
512 dma_map_err:
513     vhost_iova_tree_remove(v->iova_tree, map);
514     return r;
515 }
516 
517 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
518 {
519     VhostVDPAState *s, *s0;
520     struct vhost_vdpa *v;
521     int64_t cvq_group;
522     int r;
523     Error *err = NULL;
524 
525     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
526 
527     s = DO_UPCAST(VhostVDPAState, nc, nc);
528     v = &s->vhost_vdpa;
529 
530     s0 = vhost_vdpa_net_first_nc_vdpa(s);
531     v->shadow_data = s0->vhost_vdpa.shadow_vqs_enabled;
532     v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled;
533     s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
534 
535     if (s->vhost_vdpa.shadow_data) {
536         /* SVQ is already configured for all virtqueues */
537         goto out;
538     }
539 
540     /*
541      * If we early return in these cases SVQ will not be enabled. The migration
542      * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
543      */
544     if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) {
545         return 0;
546     }
547 
548     if (!s->cvq_isolated) {
549         return 0;
550     }
551 
552     cvq_group = vhost_vdpa_get_vring_group(v->device_fd,
553                                            v->dev->vq_index_end - 1,
554                                            &err);
555     if (unlikely(cvq_group < 0)) {
556         error_report_err(err);
557         return cvq_group;
558     }
559 
560     r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
561     if (unlikely(r < 0)) {
562         return r;
563     }
564 
565     v->shadow_vqs_enabled = true;
566     s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
567 
568 out:
569     if (!s->vhost_vdpa.shadow_vqs_enabled) {
570         return 0;
571     }
572 
573     if (s0->vhost_vdpa.iova_tree) {
574         /*
575          * SVQ is already configured for all virtqueues.  Reuse IOVA tree for
576          * simplicity, whether CVQ shares ASID with guest or not, because:
577          * - Memory listener need access to guest's memory addresses allocated
578          *   in the IOVA tree.
579          * - There should be plenty of IOVA address space for both ASID not to
580          *   worry about collisions between them.  Guest's translations are
581          *   still validated with virtio virtqueue_pop so there is no risk for
582          *   the guest to access memory that it shouldn't.
583          *
584          * To allocate a iova tree per ASID is doable but it complicates the
585          * code and it is not worth it for the moment.
586          */
587         v->iova_tree = s0->vhost_vdpa.iova_tree;
588     } else {
589         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
590                                            v->iova_range.last);
591     }
592 
593     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
594                                vhost_vdpa_net_cvq_cmd_page_len(), false);
595     if (unlikely(r < 0)) {
596         return r;
597     }
598 
599     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
600                                vhost_vdpa_net_cvq_cmd_page_len(), true);
601     if (unlikely(r < 0)) {
602         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
603     }
604 
605     return r;
606 }
607 
608 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
609 {
610     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
611 
612     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
613 
614     if (s->vhost_vdpa.shadow_vqs_enabled) {
615         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
616         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
617     }
618 
619     vhost_vdpa_net_client_stop(nc);
620 }
621 
622 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s,
623                                     const struct iovec *out_sg, size_t out_num,
624                                     const struct iovec *in_sg, size_t in_num)
625 {
626     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
627     int r;
628 
629     r = vhost_svq_add(svq, out_sg, out_num, in_sg, in_num, NULL);
630     if (unlikely(r != 0)) {
631         if (unlikely(r == -ENOSPC)) {
632             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
633                           __func__);
634         }
635     }
636 
637     return r;
638 }
639 
640 /*
641  * Convenience wrapper to poll SVQ for multiple control commands.
642  *
643  * Caller should hold the BQL when invoking this function, and should take
644  * the answer before SVQ pulls by itself when BQL is released.
645  */
646 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight)
647 {
648     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
649     return vhost_svq_poll(svq, cmds_in_flight);
650 }
651 
652 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s,
653                                              struct iovec *out_cursor,
654                                              struct iovec *in_cursor)
655 {
656     /* reset the cursor of the output buffer for the device */
657     out_cursor->iov_base = s->cvq_cmd_out_buffer;
658     out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
659 
660     /* reset the cursor of the in buffer for the device */
661     in_cursor->iov_base = s->status;
662     in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
663 }
664 
665 /*
666  * Poll SVQ for multiple pending control commands and check the device's ack.
667  *
668  * Caller should hold the BQL when invoking this function.
669  *
670  * @s: The VhostVDPAState
671  * @len: The length of the pending status shadow buffer
672  */
673 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len)
674 {
675     /* device uses a one-byte length ack for each control command */
676     ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len);
677     if (unlikely(dev_written != len)) {
678         return -EIO;
679     }
680 
681     /* check the device's ack */
682     for (int i = 0; i < len; ++i) {
683         if (s->status[i] != VIRTIO_NET_OK) {
684             return -EIO;
685         }
686     }
687     return 0;
688 }
689 
690 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s,
691                                        struct iovec *out_cursor,
692                                        struct iovec *in_cursor, uint8_t class,
693                                        uint8_t cmd, const struct iovec *data_sg,
694                                        size_t data_num)
695 {
696     const struct virtio_net_ctrl_hdr ctrl = {
697         .class = class,
698         .cmd = cmd,
699     };
700     size_t data_size = iov_size(data_sg, data_num), cmd_size;
701     struct iovec out, in;
702     ssize_t r;
703     unsigned dummy_cursor_iov_cnt;
704     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
705 
706     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
707     cmd_size = sizeof(ctrl) + data_size;
708     if (vhost_svq_available_slots(svq) < 2 ||
709         iov_size(out_cursor, 1) < cmd_size) {
710         /*
711          * It is time to flush all pending control commands if SVQ is full
712          * or control commands shadow buffers are full.
713          *
714          * We can poll here since we've had BQL from the time
715          * we sent the descriptor.
716          */
717         r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base -
718                                      (void *)s->status);
719         if (unlikely(r < 0)) {
720             return r;
721         }
722 
723         vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor);
724     }
725 
726     /* pack the CVQ command header */
727     iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl));
728     /* pack the CVQ command command-specific-data */
729     iov_to_buf(data_sg, data_num, 0,
730                out_cursor->iov_base + sizeof(ctrl), data_size);
731 
732     /* extract the required buffer from the cursor for output */
733     iov_copy(&out, 1, out_cursor, 1, 0, cmd_size);
734     /* extract the required buffer from the cursor for input */
735     iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status));
736 
737     r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1);
738     if (unlikely(r < 0)) {
739         return r;
740     }
741 
742     /* iterate the cursors */
743     dummy_cursor_iov_cnt = 1;
744     iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size);
745     dummy_cursor_iov_cnt = 1;
746     iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status));
747 
748     return 0;
749 }
750 
751 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n,
752                                    struct iovec *out_cursor,
753                                    struct iovec *in_cursor)
754 {
755     if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
756         const struct iovec data = {
757             .iov_base = (void *)n->mac,
758             .iov_len = sizeof(n->mac),
759         };
760         ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
761                                             VIRTIO_NET_CTRL_MAC,
762                                             VIRTIO_NET_CTRL_MAC_ADDR_SET,
763                                             &data, 1);
764         if (unlikely(r < 0)) {
765             return r;
766         }
767     }
768 
769     /*
770      * According to VirtIO standard, "The device MUST have an
771      * empty MAC filtering table on reset.".
772      *
773      * Therefore, there is no need to send this CVQ command if the
774      * driver also sets an empty MAC filter table, which aligns with
775      * the device's defaults.
776      *
777      * Note that the device's defaults can mismatch the driver's
778      * configuration only at live migration.
779      */
780     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) ||
781         n->mac_table.in_use == 0) {
782         return 0;
783     }
784 
785     uint32_t uni_entries = n->mac_table.first_multi,
786              uni_macs_size = uni_entries * ETH_ALEN,
787              mul_entries = n->mac_table.in_use - uni_entries,
788              mul_macs_size = mul_entries * ETH_ALEN;
789     struct virtio_net_ctrl_mac uni = {
790         .entries = cpu_to_le32(uni_entries),
791     };
792     struct virtio_net_ctrl_mac mul = {
793         .entries = cpu_to_le32(mul_entries),
794     };
795     const struct iovec data[] = {
796         {
797             .iov_base = &uni,
798             .iov_len = sizeof(uni),
799         }, {
800             .iov_base = n->mac_table.macs,
801             .iov_len = uni_macs_size,
802         }, {
803             .iov_base = &mul,
804             .iov_len = sizeof(mul),
805         }, {
806             .iov_base = &n->mac_table.macs[uni_macs_size],
807             .iov_len = mul_macs_size,
808         },
809     };
810     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
811                                         VIRTIO_NET_CTRL_MAC,
812                                         VIRTIO_NET_CTRL_MAC_TABLE_SET,
813                                         data, ARRAY_SIZE(data));
814     if (unlikely(r < 0)) {
815         return r;
816     }
817 
818     return 0;
819 }
820 
821 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
822                                    struct iovec *out_cursor,
823                                    struct iovec *in_cursor)
824 {
825     struct virtio_net_rss_config cfg = {};
826     ssize_t r;
827     g_autofree uint16_t *table = NULL;
828 
829     /*
830      * According to VirtIO standard, "Initially the device has all hash
831      * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
832      *
833      * Therefore, there is no need to send this CVQ command if the
834      * driver disables the all hash types, which aligns with
835      * the device's defaults.
836      *
837      * Note that the device's defaults can mismatch the driver's
838      * configuration only at live migration.
839      */
840     if (!n->rss_data.enabled ||
841         n->rss_data.hash_types == VIRTIO_NET_HASH_REPORT_NONE) {
842         return 0;
843     }
844 
845     table = g_malloc_n(n->rss_data.indirections_len,
846                        sizeof(n->rss_data.indirections_table[0]));
847     cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
848 
849     /*
850      * According to VirtIO standard, "Field reserved MUST contain zeroes.
851      * It is defined to make the structure to match the layout of
852      * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
853      *
854      * Therefore, we need to zero the fields in
855      * struct virtio_net_rss_config, which corresponds to the
856      * `reserved` field in struct virtio_net_hash_config.
857      *
858      * Note that all other fields are zeroed at their definitions,
859      * except for the `indirection_table` field, where the actual data
860      * is stored in the `table` variable to ensure compatibility
861      * with RSS case. Therefore, we need to zero the `table` variable here.
862      */
863     table[0] = 0;
864 
865     /*
866      * Considering that virtio_net_handle_rss() currently does not restore
867      * the hash key length parsed from the CVQ command sent from the guest
868      * into n->rss_data and uses the maximum key length in other code, so
869      * we also employ the maximum key length here.
870      */
871     cfg.hash_key_length = sizeof(n->rss_data.key);
872 
873     const struct iovec data[] = {
874         {
875             .iov_base = &cfg,
876             .iov_len = offsetof(struct virtio_net_rss_config,
877                                 indirection_table),
878         }, {
879             .iov_base = table,
880             .iov_len = n->rss_data.indirections_len *
881                        sizeof(n->rss_data.indirections_table[0]),
882         }, {
883             .iov_base = &cfg.max_tx_vq,
884             .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) -
885                        offsetof(struct virtio_net_rss_config, max_tx_vq),
886         }, {
887             .iov_base = (void *)n->rss_data.key,
888             .iov_len = sizeof(n->rss_data.key),
889         }
890     };
891 
892     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
893                                 VIRTIO_NET_CTRL_MQ,
894                                 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
895                                 data, ARRAY_SIZE(data));
896     if (unlikely(r < 0)) {
897         return r;
898     }
899 
900     return 0;
901 }
902 
903 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
904                                   const VirtIONet *n,
905                                   struct iovec *out_cursor,
906                                   struct iovec *in_cursor)
907 {
908     struct virtio_net_ctrl_mq mq;
909     ssize_t r;
910 
911     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) {
912         return 0;
913     }
914 
915     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
916     const struct iovec data = {
917         .iov_base = &mq,
918         .iov_len = sizeof(mq),
919     };
920     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
921                                 VIRTIO_NET_CTRL_MQ,
922                                 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
923                                 &data, 1);
924     if (unlikely(r < 0)) {
925         return r;
926     }
927 
928     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_HASH_REPORT)) {
929         return 0;
930     }
931 
932     r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor);
933     if (unlikely(r < 0)) {
934         return r;
935     }
936 
937     return 0;
938 }
939 
940 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s,
941                                         const VirtIONet *n,
942                                         struct iovec *out_cursor,
943                                         struct iovec *in_cursor)
944 {
945     uint64_t offloads;
946     ssize_t r;
947 
948     if (!virtio_vdev_has_feature(&n->parent_obj,
949                                  VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
950         return 0;
951     }
952 
953     if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) {
954         /*
955          * According to VirtIO standard, "Upon feature negotiation
956          * corresponding offload gets enabled to preserve
957          * backward compatibility.".
958          *
959          * Therefore, there is no need to send this CVQ command if the
960          * driver also enables all supported offloads, which aligns with
961          * the device's defaults.
962          *
963          * Note that the device's defaults can mismatch the driver's
964          * configuration only at live migration.
965          */
966         return 0;
967     }
968 
969     offloads = cpu_to_le64(n->curr_guest_offloads);
970     const struct iovec data = {
971         .iov_base = &offloads,
972         .iov_len = sizeof(offloads),
973     };
974     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
975                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS,
976                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET,
977                                 &data, 1);
978     if (unlikely(r < 0)) {
979         return r;
980     }
981 
982     return 0;
983 }
984 
985 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s,
986                                        struct iovec *out_cursor,
987                                        struct iovec *in_cursor,
988                                        uint8_t cmd,
989                                        uint8_t on)
990 {
991     const struct iovec data = {
992         .iov_base = &on,
993         .iov_len = sizeof(on),
994     };
995     ssize_t r;
996 
997     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
998                                 VIRTIO_NET_CTRL_RX, cmd, &data, 1);
999     if (unlikely(r < 0)) {
1000         return r;
1001     }
1002 
1003     return 0;
1004 }
1005 
1006 static int vhost_vdpa_net_load_rx(VhostVDPAState *s,
1007                                   const VirtIONet *n,
1008                                   struct iovec *out_cursor,
1009                                   struct iovec *in_cursor)
1010 {
1011     ssize_t r;
1012 
1013     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) {
1014         return 0;
1015     }
1016 
1017     /*
1018      * According to virtio_net_reset(), device turns promiscuous mode
1019      * on by default.
1020      *
1021      * Additionally, according to VirtIO standard, "Since there are
1022      * no guarantees, it can use a hash filter or silently switch to
1023      * allmulti or promiscuous mode if it is given too many addresses.".
1024      * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1025      * non-multicast MAC addresses, indicating that promiscuous mode
1026      * should be enabled.
1027      *
1028      * Therefore, QEMU should only send this CVQ command if the
1029      * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1030      * which sets promiscuous mode on, different from the device's defaults.
1031      *
1032      * Note that the device's defaults can mismatch the driver's
1033      * configuration only at live migration.
1034      */
1035     if (!n->mac_table.uni_overflow && !n->promisc) {
1036         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1037                                         VIRTIO_NET_CTRL_RX_PROMISC, 0);
1038         if (unlikely(r < 0)) {
1039             return r;
1040         }
1041     }
1042 
1043     /*
1044      * According to virtio_net_reset(), device turns all-multicast mode
1045      * off by default.
1046      *
1047      * According to VirtIO standard, "Since there are no guarantees,
1048      * it can use a hash filter or silently switch to allmulti or
1049      * promiscuous mode if it is given too many addresses.". QEMU marks
1050      * `n->mac_table.multi_overflow` if guest sets too many
1051      * non-multicast MAC addresses.
1052      *
1053      * Therefore, QEMU should only send this CVQ command if the
1054      * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1055      * which sets all-multicast mode on, different from the device's defaults.
1056      *
1057      * Note that the device's defaults can mismatch the driver's
1058      * configuration only at live migration.
1059      */
1060     if (n->mac_table.multi_overflow || n->allmulti) {
1061         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1062                                         VIRTIO_NET_CTRL_RX_ALLMULTI, 1);
1063         if (unlikely(r < 0)) {
1064             return r;
1065         }
1066     }
1067 
1068     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) {
1069         return 0;
1070     }
1071 
1072     /*
1073      * According to virtio_net_reset(), device turns all-unicast mode
1074      * off by default.
1075      *
1076      * Therefore, QEMU should only send this CVQ command if the driver
1077      * sets all-unicast mode on, different from the device's defaults.
1078      *
1079      * Note that the device's defaults can mismatch the driver's
1080      * configuration only at live migration.
1081      */
1082     if (n->alluni) {
1083         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1084                                         VIRTIO_NET_CTRL_RX_ALLUNI, 1);
1085         if (r < 0) {
1086             return r;
1087         }
1088     }
1089 
1090     /*
1091      * According to virtio_net_reset(), device turns non-multicast mode
1092      * off by default.
1093      *
1094      * Therefore, QEMU should only send this CVQ command if the driver
1095      * sets non-multicast mode on, different from the device's defaults.
1096      *
1097      * Note that the device's defaults can mismatch the driver's
1098      * configuration only at live migration.
1099      */
1100     if (n->nomulti) {
1101         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1102                                         VIRTIO_NET_CTRL_RX_NOMULTI, 1);
1103         if (r < 0) {
1104             return r;
1105         }
1106     }
1107 
1108     /*
1109      * According to virtio_net_reset(), device turns non-unicast mode
1110      * off by default.
1111      *
1112      * Therefore, QEMU should only send this CVQ command if the driver
1113      * sets non-unicast mode on, different from the device's defaults.
1114      *
1115      * Note that the device's defaults can mismatch the driver's
1116      * configuration only at live migration.
1117      */
1118     if (n->nouni) {
1119         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1120                                         VIRTIO_NET_CTRL_RX_NOUNI, 1);
1121         if (r < 0) {
1122             return r;
1123         }
1124     }
1125 
1126     /*
1127      * According to virtio_net_reset(), device turns non-broadcast mode
1128      * off by default.
1129      *
1130      * Therefore, QEMU should only send this CVQ command if the driver
1131      * sets non-broadcast mode on, different from the device's defaults.
1132      *
1133      * Note that the device's defaults can mismatch the driver's
1134      * configuration only at live migration.
1135      */
1136     if (n->nobcast) {
1137         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1138                                         VIRTIO_NET_CTRL_RX_NOBCAST, 1);
1139         if (r < 0) {
1140             return r;
1141         }
1142     }
1143 
1144     return 0;
1145 }
1146 
1147 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s,
1148                                            const VirtIONet *n,
1149                                            struct iovec *out_cursor,
1150                                            struct iovec *in_cursor,
1151                                            uint16_t vid)
1152 {
1153     const struct iovec data = {
1154         .iov_base = &vid,
1155         .iov_len = sizeof(vid),
1156     };
1157     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1158                                         VIRTIO_NET_CTRL_VLAN,
1159                                         VIRTIO_NET_CTRL_VLAN_ADD,
1160                                         &data, 1);
1161     if (unlikely(r < 0)) {
1162         return r;
1163     }
1164 
1165     return 0;
1166 }
1167 
1168 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s,
1169                                     const VirtIONet *n,
1170                                     struct iovec *out_cursor,
1171                                     struct iovec *in_cursor)
1172 {
1173     int r;
1174 
1175     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) {
1176         return 0;
1177     }
1178 
1179     for (int i = 0; i < MAX_VLAN >> 5; i++) {
1180         for (int j = 0; n->vlans[i] && j <= 0x1f; j++) {
1181             if (n->vlans[i] & (1U << j)) {
1182                 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor,
1183                                                     in_cursor, (i << 5) + j);
1184                 if (unlikely(r != 0)) {
1185                     return r;
1186                 }
1187             }
1188         }
1189     }
1190 
1191     return 0;
1192 }
1193 
1194 static int vhost_vdpa_net_cvq_load(NetClientState *nc)
1195 {
1196     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
1197     struct vhost_vdpa *v = &s->vhost_vdpa;
1198     const VirtIONet *n;
1199     int r;
1200     struct iovec out_cursor, in_cursor;
1201 
1202     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1203 
1204     vhost_vdpa_set_vring_ready(v, v->dev->vq_index);
1205 
1206     if (v->shadow_vqs_enabled) {
1207         n = VIRTIO_NET(v->dev->vdev);
1208         vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor);
1209         r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor);
1210         if (unlikely(r < 0)) {
1211             return r;
1212         }
1213         r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor);
1214         if (unlikely(r)) {
1215             return r;
1216         }
1217         r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor);
1218         if (unlikely(r)) {
1219             return r;
1220         }
1221         r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor);
1222         if (unlikely(r)) {
1223             return r;
1224         }
1225         r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor);
1226         if (unlikely(r)) {
1227             return r;
1228         }
1229 
1230         /*
1231          * We need to poll and check all pending device's used buffers.
1232          *
1233          * We can poll here since we've had BQL from the time
1234          * we sent the descriptor.
1235          */
1236         r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status);
1237         if (unlikely(r)) {
1238             return r;
1239         }
1240     }
1241 
1242     for (int i = 0; i < v->dev->vq_index; ++i) {
1243         vhost_vdpa_set_vring_ready(v, i);
1244     }
1245 
1246     return 0;
1247 }
1248 
1249 static NetClientInfo net_vhost_vdpa_cvq_info = {
1250     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
1251     .size = sizeof(VhostVDPAState),
1252     .receive = vhost_vdpa_receive,
1253     .start = vhost_vdpa_net_cvq_start,
1254     .load = vhost_vdpa_net_cvq_load,
1255     .stop = vhost_vdpa_net_cvq_stop,
1256     .cleanup = vhost_vdpa_cleanup,
1257     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
1258     .has_ufo = vhost_vdpa_has_ufo,
1259     .check_peer_type = vhost_vdpa_check_peer_type,
1260 };
1261 
1262 /*
1263  * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1264  * vdpa device.
1265  *
1266  * Considering that QEMU cannot send the entire filter table to the
1267  * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1268  * command to enable promiscuous mode to receive all packets,
1269  * according to VirtIO standard, "Since there are no guarantees,
1270  * it can use a hash filter or silently switch to allmulti or
1271  * promiscuous mode if it is given too many addresses.".
1272  *
1273  * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1274  * marks `n->mac_table.x_overflow` accordingly, it should have
1275  * the same effect on the device model to receive
1276  * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1277  * The same applies to multicast MAC addresses.
1278  *
1279  * Therefore, QEMU can provide the device model with a fake
1280  * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1281  * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1282  * MAC addresses. This ensures that the device model marks
1283  * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1284  * allowing all packets to be received, which aligns with the
1285  * state of the vdpa device.
1286  */
1287 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s,
1288                                                        VirtQueueElement *elem,
1289                                                        struct iovec *out,
1290                                                        const struct iovec *in)
1291 {
1292     struct virtio_net_ctrl_mac mac_data, *mac_ptr;
1293     struct virtio_net_ctrl_hdr *hdr_ptr;
1294     uint32_t cursor;
1295     ssize_t r;
1296     uint8_t on = 1;
1297 
1298     /* parse the non-multicast MAC address entries from CVQ command */
1299     cursor = sizeof(*hdr_ptr);
1300     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1301                    &mac_data, sizeof(mac_data));
1302     if (unlikely(r != sizeof(mac_data))) {
1303         /*
1304          * If the CVQ command is invalid, we should simulate the vdpa device
1305          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1306          */
1307         *s->status = VIRTIO_NET_ERR;
1308         return sizeof(*s->status);
1309     }
1310     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1311 
1312     /* parse the multicast MAC address entries from CVQ command */
1313     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1314                    &mac_data, sizeof(mac_data));
1315     if (r != sizeof(mac_data)) {
1316         /*
1317          * If the CVQ command is invalid, we should simulate the vdpa device
1318          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1319          */
1320         *s->status = VIRTIO_NET_ERR;
1321         return sizeof(*s->status);
1322     }
1323     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1324 
1325     /* validate the CVQ command */
1326     if (iov_size(elem->out_sg, elem->out_num) != cursor) {
1327         /*
1328          * If the CVQ command is invalid, we should simulate the vdpa device
1329          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1330          */
1331         *s->status = VIRTIO_NET_ERR;
1332         return sizeof(*s->status);
1333     }
1334 
1335     /*
1336      * According to VirtIO standard, "Since there are no guarantees,
1337      * it can use a hash filter or silently switch to allmulti or
1338      * promiscuous mode if it is given too many addresses.".
1339      *
1340      * Therefore, considering that QEMU is unable to send the entire
1341      * filter table to the vdpa device, it should send the
1342      * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1343      */
1344     hdr_ptr = out->iov_base;
1345     out->iov_len = sizeof(*hdr_ptr) + sizeof(on);
1346 
1347     hdr_ptr->class = VIRTIO_NET_CTRL_RX;
1348     hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC;
1349     iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on));
1350     r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1);
1351     if (unlikely(r < 0)) {
1352         return r;
1353     }
1354 
1355     /*
1356      * We can poll here since we've had BQL from the time
1357      * we sent the descriptor.
1358      */
1359     r = vhost_vdpa_net_svq_poll(s, 1);
1360     if (unlikely(r < sizeof(*s->status))) {
1361         return r;
1362     }
1363     if (*s->status != VIRTIO_NET_OK) {
1364         return sizeof(*s->status);
1365     }
1366 
1367     /*
1368      * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1369      * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1370      * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1371      * multicast MAC addresses.
1372      *
1373      * By doing so, the device model can mark `n->mac_table.uni_overflow`
1374      * and `n->mac_table.multi_overflow`, enabling all packets to be
1375      * received, which aligns with the state of the vdpa device.
1376      */
1377     cursor = 0;
1378     uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1,
1379              fake_mul_entries = MAC_TABLE_ENTRIES + 1,
1380              fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) +
1381                              sizeof(mac_data) + fake_uni_entries * ETH_ALEN +
1382                              sizeof(mac_data) + fake_mul_entries * ETH_ALEN;
1383 
1384     assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len());
1385     out->iov_len = fake_cvq_size;
1386 
1387     /* pack the header for fake CVQ command */
1388     hdr_ptr = out->iov_base + cursor;
1389     hdr_ptr->class = VIRTIO_NET_CTRL_MAC;
1390     hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1391     cursor += sizeof(*hdr_ptr);
1392 
1393     /*
1394      * Pack the non-multicast MAC addresses part for fake CVQ command.
1395      *
1396      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1397      * addresses provided in CVQ command. Therefore, only the entries
1398      * field need to be prepared in the CVQ command.
1399      */
1400     mac_ptr = out->iov_base + cursor;
1401     mac_ptr->entries = cpu_to_le32(fake_uni_entries);
1402     cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN;
1403 
1404     /*
1405      * Pack the multicast MAC addresses part for fake CVQ command.
1406      *
1407      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1408      * addresses provided in CVQ command. Therefore, only the entries
1409      * field need to be prepared in the CVQ command.
1410      */
1411     mac_ptr = out->iov_base + cursor;
1412     mac_ptr->entries = cpu_to_le32(fake_mul_entries);
1413 
1414     /*
1415      * Simulating QEMU poll a vdpa device used buffer
1416      * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1417      */
1418     return sizeof(*s->status);
1419 }
1420 
1421 /**
1422  * Validate and copy control virtqueue commands.
1423  *
1424  * Following QEMU guidelines, we offer a copy of the buffers to the device to
1425  * prevent TOCTOU bugs.
1426  */
1427 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
1428                                             VirtQueueElement *elem,
1429                                             void *opaque)
1430 {
1431     VhostVDPAState *s = opaque;
1432     size_t in_len;
1433     const struct virtio_net_ctrl_hdr *ctrl;
1434     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1435     /* Out buffer sent to both the vdpa device and the device model */
1436     struct iovec out = {
1437         .iov_base = s->cvq_cmd_out_buffer,
1438     };
1439     /* in buffer used for device model */
1440     const struct iovec model_in = {
1441         .iov_base = &status,
1442         .iov_len = sizeof(status),
1443     };
1444     /* in buffer used for vdpa device */
1445     const struct iovec vdpa_in = {
1446         .iov_base = s->status,
1447         .iov_len = sizeof(*s->status),
1448     };
1449     ssize_t dev_written = -EINVAL;
1450 
1451     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
1452                              s->cvq_cmd_out_buffer,
1453                              vhost_vdpa_net_cvq_cmd_page_len());
1454 
1455     ctrl = s->cvq_cmd_out_buffer;
1456     if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) {
1457         /*
1458          * Guest announce capability is emulated by qemu, so don't forward to
1459          * the device.
1460          */
1461         dev_written = sizeof(status);
1462         *s->status = VIRTIO_NET_OK;
1463     } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC &&
1464                         ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET &&
1465                         iov_size(elem->out_sg, elem->out_num) > out.iov_len)) {
1466         /*
1467          * Due to the size limitation of the out buffer sent to the vdpa device,
1468          * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1469          * MAC addresses set by the driver for the filter table can cause
1470          * truncation of the CVQ command in QEMU. As a result, the vdpa device
1471          * rejects the flawed CVQ command.
1472          *
1473          * Therefore, QEMU must handle this situation instead of sending
1474          * the CVQ command directly.
1475          */
1476         dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem,
1477                                                             &out, &vdpa_in);
1478         if (unlikely(dev_written < 0)) {
1479             goto out;
1480         }
1481     } else {
1482         ssize_t r;
1483         r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1);
1484         if (unlikely(r < 0)) {
1485             dev_written = r;
1486             goto out;
1487         }
1488 
1489         /*
1490          * We can poll here since we've had BQL from the time
1491          * we sent the descriptor.
1492          */
1493         dev_written = vhost_vdpa_net_svq_poll(s, 1);
1494     }
1495 
1496     if (unlikely(dev_written < sizeof(status))) {
1497         error_report("Insufficient written data (%zu)", dev_written);
1498         goto out;
1499     }
1500 
1501     if (*s->status != VIRTIO_NET_OK) {
1502         goto out;
1503     }
1504 
1505     status = VIRTIO_NET_ERR;
1506     virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1);
1507     if (status != VIRTIO_NET_OK) {
1508         error_report("Bad CVQ processing in model");
1509     }
1510 
1511 out:
1512     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
1513                           sizeof(status));
1514     if (unlikely(in_len < sizeof(status))) {
1515         error_report("Bad device CVQ written length");
1516     }
1517     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
1518     /*
1519      * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1520      * the function successfully forwards the CVQ command, indicated
1521      * by a non-negative value of `dev_written`. Otherwise, it still
1522      * belongs to SVQ.
1523      * This function should only free the `elem` when it owns.
1524      */
1525     if (dev_written >= 0) {
1526         g_free(elem);
1527     }
1528     return dev_written < 0 ? dev_written : 0;
1529 }
1530 
1531 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
1532     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
1533 };
1534 
1535 /**
1536  * Probe if CVQ is isolated
1537  *
1538  * @device_fd         The vdpa device fd
1539  * @features          Features offered by the device.
1540  * @cvq_index         The control vq pair index
1541  *
1542  * Returns <0 in case of failure, 0 if false and 1 if true.
1543  */
1544 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features,
1545                                           int cvq_index, Error **errp)
1546 {
1547     uint64_t backend_features;
1548     int64_t cvq_group;
1549     uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE |
1550                      VIRTIO_CONFIG_S_DRIVER;
1551     int r;
1552 
1553     ERRP_GUARD();
1554 
1555     r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
1556     if (unlikely(r < 0)) {
1557         error_setg_errno(errp, errno, "Cannot get vdpa backend_features");
1558         return r;
1559     }
1560 
1561     if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) {
1562         return 0;
1563     }
1564 
1565     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1566     if (unlikely(r)) {
1567         error_setg_errno(errp, -r, "Cannot set device status");
1568         goto out;
1569     }
1570 
1571     r = ioctl(device_fd, VHOST_SET_FEATURES, &features);
1572     if (unlikely(r)) {
1573         error_setg_errno(errp, -r, "Cannot set features");
1574         goto out;
1575     }
1576 
1577     status |= VIRTIO_CONFIG_S_FEATURES_OK;
1578     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1579     if (unlikely(r)) {
1580         error_setg_errno(errp, -r, "Cannot set device status");
1581         goto out;
1582     }
1583 
1584     cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp);
1585     if (unlikely(cvq_group < 0)) {
1586         if (cvq_group != -ENOTSUP) {
1587             r = cvq_group;
1588             goto out;
1589         }
1590 
1591         /*
1592          * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1593          * support ASID even if the parent driver does not.  The CVQ cannot be
1594          * isolated in this case.
1595          */
1596         error_free(*errp);
1597         *errp = NULL;
1598         r = 0;
1599         goto out;
1600     }
1601 
1602     for (int i = 0; i < cvq_index; ++i) {
1603         int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp);
1604         if (unlikely(group < 0)) {
1605             r = group;
1606             goto out;
1607         }
1608 
1609         if (group == (int64_t)cvq_group) {
1610             r = 0;
1611             goto out;
1612         }
1613     }
1614 
1615     r = 1;
1616 
1617 out:
1618     status = 0;
1619     ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1620     return r;
1621 }
1622 
1623 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
1624                                        const char *device,
1625                                        const char *name,
1626                                        int vdpa_device_fd,
1627                                        int queue_pair_index,
1628                                        int nvqs,
1629                                        bool is_datapath,
1630                                        bool svq,
1631                                        struct vhost_vdpa_iova_range iova_range,
1632                                        uint64_t features,
1633                                        Error **errp)
1634 {
1635     NetClientState *nc = NULL;
1636     VhostVDPAState *s;
1637     int ret = 0;
1638     assert(name);
1639     int cvq_isolated = 0;
1640 
1641     if (is_datapath) {
1642         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
1643                                  name);
1644     } else {
1645         cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features,
1646                                                       queue_pair_index * 2,
1647                                                       errp);
1648         if (unlikely(cvq_isolated < 0)) {
1649             return NULL;
1650         }
1651 
1652         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
1653                                          device, name);
1654     }
1655     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
1656     s = DO_UPCAST(VhostVDPAState, nc, nc);
1657 
1658     s->vhost_vdpa.device_fd = vdpa_device_fd;
1659     s->vhost_vdpa.index = queue_pair_index;
1660     s->always_svq = svq;
1661     s->migration_state.notify = NULL;
1662     s->vhost_vdpa.shadow_vqs_enabled = svq;
1663     s->vhost_vdpa.iova_range = iova_range;
1664     s->vhost_vdpa.shadow_data = svq;
1665     if (queue_pair_index == 0) {
1666         vhost_vdpa_net_valid_svq_features(features,
1667                                           &s->vhost_vdpa.migration_blocker);
1668     } else if (!is_datapath) {
1669         s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1670                                      PROT_READ | PROT_WRITE,
1671                                      MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1672         s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1673                          PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
1674                          -1, 0);
1675 
1676         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
1677         s->vhost_vdpa.shadow_vq_ops_opaque = s;
1678         s->cvq_isolated = cvq_isolated;
1679     }
1680     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
1681     if (ret) {
1682         qemu_del_net_client(nc);
1683         return NULL;
1684     }
1685     return nc;
1686 }
1687 
1688 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
1689 {
1690     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
1691     if (unlikely(ret < 0)) {
1692         error_setg_errno(errp, errno,
1693                          "Fail to query features from vhost-vDPA device");
1694     }
1695     return ret;
1696 }
1697 
1698 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
1699                                           int *has_cvq, Error **errp)
1700 {
1701     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
1702     g_autofree struct vhost_vdpa_config *config = NULL;
1703     __virtio16 *max_queue_pairs;
1704     int ret;
1705 
1706     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
1707         *has_cvq = 1;
1708     } else {
1709         *has_cvq = 0;
1710     }
1711 
1712     if (features & (1 << VIRTIO_NET_F_MQ)) {
1713         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
1714         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
1715         config->len = sizeof(*max_queue_pairs);
1716 
1717         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
1718         if (ret) {
1719             error_setg(errp, "Fail to get config from vhost-vDPA device");
1720             return -ret;
1721         }
1722 
1723         max_queue_pairs = (__virtio16 *)&config->buf;
1724 
1725         return lduw_le_p(max_queue_pairs);
1726     }
1727 
1728     return 1;
1729 }
1730 
1731 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
1732                         NetClientState *peer, Error **errp)
1733 {
1734     const NetdevVhostVDPAOptions *opts;
1735     uint64_t features;
1736     int vdpa_device_fd;
1737     g_autofree NetClientState **ncs = NULL;
1738     struct vhost_vdpa_iova_range iova_range;
1739     NetClientState *nc;
1740     int queue_pairs, r, i = 0, has_cvq = 0;
1741 
1742     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1743     opts = &netdev->u.vhost_vdpa;
1744     if (!opts->vhostdev && !opts->vhostfd) {
1745         error_setg(errp,
1746                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1747         return -1;
1748     }
1749 
1750     if (opts->vhostdev && opts->vhostfd) {
1751         error_setg(errp,
1752                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1753         return -1;
1754     }
1755 
1756     if (opts->vhostdev) {
1757         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
1758         if (vdpa_device_fd == -1) {
1759             return -errno;
1760         }
1761     } else {
1762         /* has_vhostfd */
1763         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
1764         if (vdpa_device_fd == -1) {
1765             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
1766             return -1;
1767         }
1768     }
1769 
1770     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
1771     if (unlikely(r < 0)) {
1772         goto err;
1773     }
1774 
1775     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
1776                                                  &has_cvq, errp);
1777     if (queue_pairs < 0) {
1778         qemu_close(vdpa_device_fd);
1779         return queue_pairs;
1780     }
1781 
1782     r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
1783     if (unlikely(r < 0)) {
1784         error_setg(errp, "vhost-vdpa: get iova range failed: %s",
1785                    strerror(-r));
1786         goto err;
1787     }
1788 
1789     if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
1790         goto err;
1791     }
1792 
1793     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
1794 
1795     for (i = 0; i < queue_pairs; i++) {
1796         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1797                                      vdpa_device_fd, i, 2, true, opts->x_svq,
1798                                      iova_range, features, errp);
1799         if (!ncs[i])
1800             goto err;
1801     }
1802 
1803     if (has_cvq) {
1804         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1805                                  vdpa_device_fd, i, 1, false,
1806                                  opts->x_svq, iova_range, features, errp);
1807         if (!nc)
1808             goto err;
1809     }
1810 
1811     return 0;
1812 
1813 err:
1814     if (i) {
1815         for (i--; i >= 0; i--) {
1816             qemu_del_net_client(ncs[i]);
1817         }
1818     }
1819 
1820     qemu_close(vdpa_device_fd);
1821 
1822     return -1;
1823 }
1824