xref: /openbmc/qemu/net/vhost-vdpa.c (revision d1fd2d3118a42f36d86efae7ffbdce79f7024584)
1 /*
2  * vhost-vdpa.c
3  *
4  * Copyright(c) 2017-2018 Intel Corporation.
5  * Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/log.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
26 #include <err.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/migration.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
32 
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState {
35     NetClientState nc;
36     struct vhost_vdpa vhost_vdpa;
37     Notifier migration_state;
38     VHostNetState *vhost_net;
39 
40     /* Control commands shadow buffers */
41     void *cvq_cmd_out_buffer;
42     virtio_net_ctrl_ack *status;
43 
44     /* The device always have SVQ enabled */
45     bool always_svq;
46 
47     /* The device can isolate CVQ in its own ASID */
48     bool cvq_isolated;
49 
50     bool started;
51 } VhostVDPAState;
52 
53 /*
54  * The array is sorted alphabetically in ascending order,
55  * with the exception of VHOST_INVALID_FEATURE_BIT,
56  * which should always be the last entry.
57  */
58 const int vdpa_feature_bits[] = {
59     VIRTIO_F_ANY_LAYOUT,
60     VIRTIO_F_IOMMU_PLATFORM,
61     VIRTIO_F_NOTIFY_ON_EMPTY,
62     VIRTIO_F_RING_PACKED,
63     VIRTIO_F_RING_RESET,
64     VIRTIO_F_VERSION_1,
65     VIRTIO_NET_F_CSUM,
66     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
67     VIRTIO_NET_F_CTRL_MAC_ADDR,
68     VIRTIO_NET_F_CTRL_RX,
69     VIRTIO_NET_F_CTRL_RX_EXTRA,
70     VIRTIO_NET_F_CTRL_VLAN,
71     VIRTIO_NET_F_CTRL_VQ,
72     VIRTIO_NET_F_GSO,
73     VIRTIO_NET_F_GUEST_CSUM,
74     VIRTIO_NET_F_GUEST_ECN,
75     VIRTIO_NET_F_GUEST_TSO4,
76     VIRTIO_NET_F_GUEST_TSO6,
77     VIRTIO_NET_F_GUEST_UFO,
78     VIRTIO_NET_F_GUEST_USO4,
79     VIRTIO_NET_F_GUEST_USO6,
80     VIRTIO_NET_F_HASH_REPORT,
81     VIRTIO_NET_F_HOST_ECN,
82     VIRTIO_NET_F_HOST_TSO4,
83     VIRTIO_NET_F_HOST_TSO6,
84     VIRTIO_NET_F_HOST_UFO,
85     VIRTIO_NET_F_HOST_USO,
86     VIRTIO_NET_F_MQ,
87     VIRTIO_NET_F_MRG_RXBUF,
88     VIRTIO_NET_F_MTU,
89     VIRTIO_NET_F_RSS,
90     VIRTIO_NET_F_STATUS,
91     VIRTIO_RING_F_EVENT_IDX,
92     VIRTIO_RING_F_INDIRECT_DESC,
93 
94     /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
95     VHOST_INVALID_FEATURE_BIT
96 };
97 
98 /** Supported device specific feature bits with SVQ */
99 static const uint64_t vdpa_svq_device_features =
100     BIT_ULL(VIRTIO_NET_F_CSUM) |
101     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
102     BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
103     BIT_ULL(VIRTIO_NET_F_MTU) |
104     BIT_ULL(VIRTIO_NET_F_MAC) |
105     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
106     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
107     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
108     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
109     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
110     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
111     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
112     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
113     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
114     BIT_ULL(VIRTIO_NET_F_STATUS) |
115     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
116     BIT_ULL(VIRTIO_NET_F_CTRL_RX) |
117     BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) |
118     BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) |
119     BIT_ULL(VIRTIO_NET_F_MQ) |
120     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
121     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
122     /* VHOST_F_LOG_ALL is exposed by SVQ */
123     BIT_ULL(VHOST_F_LOG_ALL) |
124     BIT_ULL(VIRTIO_NET_F_HASH_REPORT) |
125     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
126     BIT_ULL(VIRTIO_NET_F_STANDBY) |
127     BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
128 
129 #define VHOST_VDPA_NET_CVQ_ASID 1
130 
131 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
132 {
133     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
134     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
135     return s->vhost_net;
136 }
137 
138 static size_t vhost_vdpa_net_cvq_cmd_len(void)
139 {
140     /*
141      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
142      * In buffer is always 1 byte, so it should fit here
143      */
144     return sizeof(struct virtio_net_ctrl_hdr) +
145            2 * sizeof(struct virtio_net_ctrl_mac) +
146            MAC_TABLE_ENTRIES * ETH_ALEN;
147 }
148 
149 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
150 {
151     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
152 }
153 
154 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
155 {
156     uint64_t invalid_dev_features =
157         features & ~vdpa_svq_device_features &
158         /* Transport are all accepted at this point */
159         ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
160                          VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
161 
162     if (invalid_dev_features) {
163         error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
164                    invalid_dev_features);
165         return false;
166     }
167 
168     return vhost_svq_valid_features(features, errp);
169 }
170 
171 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
172 {
173     uint32_t device_id;
174     int ret;
175     struct vhost_dev *hdev;
176 
177     hdev = (struct vhost_dev *)&net->dev;
178     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
179     if (device_id != VIRTIO_ID_NET) {
180         return -ENOTSUP;
181     }
182     return ret;
183 }
184 
185 static int vhost_vdpa_add(NetClientState *ncs, void *be,
186                           int queue_pair_index, int nvqs)
187 {
188     VhostNetOptions options;
189     struct vhost_net *net = NULL;
190     VhostVDPAState *s;
191     int ret;
192 
193     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
194     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
195     s = DO_UPCAST(VhostVDPAState, nc, ncs);
196     options.net_backend = ncs;
197     options.opaque      = be;
198     options.busyloop_timeout = 0;
199     options.nvqs = nvqs;
200 
201     net = vhost_net_init(&options);
202     if (!net) {
203         error_report("failed to init vhost_net for queue");
204         goto err_init;
205     }
206     s->vhost_net = net;
207     ret = vhost_vdpa_net_check_device_id(net);
208     if (ret) {
209         goto err_check;
210     }
211     return 0;
212 err_check:
213     vhost_net_cleanup(net);
214     g_free(net);
215 err_init:
216     return -1;
217 }
218 
219 static void vhost_vdpa_cleanup(NetClientState *nc)
220 {
221     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
222 
223     /*
224      * If a peer NIC is attached, do not cleanup anything.
225      * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
226      * when the guest is shutting down.
227      */
228     if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
229         return;
230     }
231     munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len());
232     munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len());
233     if (s->vhost_net) {
234         vhost_net_cleanup(s->vhost_net);
235         g_free(s->vhost_net);
236         s->vhost_net = NULL;
237     }
238      if (s->vhost_vdpa.device_fd >= 0) {
239         qemu_close(s->vhost_vdpa.device_fd);
240         s->vhost_vdpa.device_fd = -1;
241     }
242 }
243 
244 /** Dummy SetSteeringEBPF to support RSS for vhost-vdpa backend  */
245 static bool vhost_vdpa_set_steering_ebpf(NetClientState *nc, int prog_fd)
246 {
247     return true;
248 }
249 
250 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
251 {
252     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
253 
254     return true;
255 }
256 
257 static bool vhost_vdpa_has_ufo(NetClientState *nc)
258 {
259     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
260     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
261     uint64_t features = 0;
262     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
263     features = vhost_net_get_features(s->vhost_net, features);
264     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
265 
266 }
267 
268 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
269                                        Error **errp)
270 {
271     const char *driver = object_class_get_name(oc);
272 
273     if (!g_str_has_prefix(driver, "virtio-net-")) {
274         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
275         return false;
276     }
277 
278     return true;
279 }
280 
281 /** Dummy receive in case qemu falls back to userland tap networking */
282 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
283                                   size_t size)
284 {
285     return size;
286 }
287 
288 /** From any vdpa net client, get the netclient of the first queue pair */
289 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
290 {
291     NICState *nic = qemu_get_nic(s->nc.peer);
292     NetClientState *nc0 = qemu_get_peer(nic->ncs, 0);
293 
294     return DO_UPCAST(VhostVDPAState, nc, nc0);
295 }
296 
297 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
298 {
299     struct vhost_vdpa *v = &s->vhost_vdpa;
300     VirtIONet *n;
301     VirtIODevice *vdev;
302     int data_queue_pairs, cvq, r;
303 
304     /* We are only called on the first data vqs and only if x-svq is not set */
305     if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
306         return;
307     }
308 
309     vdev = v->dev->vdev;
310     n = VIRTIO_NET(vdev);
311     if (!n->vhost_started) {
312         return;
313     }
314 
315     data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
316     cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
317                                   n->max_ncs - n->max_queue_pairs : 0;
318     /*
319      * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
320      * in the future and resume the device if read-only operations between
321      * suspend and reset goes wrong.
322      */
323     vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
324 
325     /* Start will check migration setup_or_active to configure or not SVQ */
326     r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
327     if (unlikely(r < 0)) {
328         error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
329     }
330 }
331 
332 static void vdpa_net_migration_state_notifier(Notifier *notifier, void *data)
333 {
334     MigrationState *migration = data;
335     VhostVDPAState *s = container_of(notifier, VhostVDPAState,
336                                      migration_state);
337 
338     if (migration_in_setup(migration)) {
339         vhost_vdpa_net_log_global_enable(s, true);
340     } else if (migration_has_failed(migration)) {
341         vhost_vdpa_net_log_global_enable(s, false);
342     }
343 }
344 
345 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
346 {
347     struct vhost_vdpa *v = &s->vhost_vdpa;
348 
349     migration_add_notifier(&s->migration_state,
350                            vdpa_net_migration_state_notifier);
351     if (v->shadow_vqs_enabled) {
352         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
353                                            v->iova_range.last);
354     }
355 }
356 
357 static int vhost_vdpa_net_data_start(NetClientState *nc)
358 {
359     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
360     struct vhost_vdpa *v = &s->vhost_vdpa;
361 
362     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
363 
364     if (s->always_svq ||
365         migration_is_setup_or_active(migrate_get_current()->state)) {
366         v->shadow_vqs_enabled = true;
367         v->shadow_data = true;
368     } else {
369         v->shadow_vqs_enabled = false;
370         v->shadow_data = false;
371     }
372 
373     if (v->index == 0) {
374         vhost_vdpa_net_data_start_first(s);
375         return 0;
376     }
377 
378     if (v->shadow_vqs_enabled) {
379         VhostVDPAState *s0 = vhost_vdpa_net_first_nc_vdpa(s);
380         v->iova_tree = s0->vhost_vdpa.iova_tree;
381     }
382 
383     return 0;
384 }
385 
386 static int vhost_vdpa_net_data_load(NetClientState *nc)
387 {
388     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
389     struct vhost_vdpa *v = &s->vhost_vdpa;
390     bool has_cvq = v->dev->vq_index_end % 2;
391 
392     if (has_cvq) {
393         return 0;
394     }
395 
396     for (int i = 0; i < v->dev->nvqs; ++i) {
397         vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index);
398     }
399     return 0;
400 }
401 
402 static void vhost_vdpa_net_client_stop(NetClientState *nc)
403 {
404     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
405     struct vhost_dev *dev;
406 
407     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
408 
409     if (s->vhost_vdpa.index == 0) {
410         migration_remove_notifier(&s->migration_state);
411     }
412 
413     dev = s->vhost_vdpa.dev;
414     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
415         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
416     } else {
417         s->vhost_vdpa.iova_tree = NULL;
418     }
419 }
420 
421 static NetClientInfo net_vhost_vdpa_info = {
422         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
423         .size = sizeof(VhostVDPAState),
424         .receive = vhost_vdpa_receive,
425         .start = vhost_vdpa_net_data_start,
426         .load = vhost_vdpa_net_data_load,
427         .stop = vhost_vdpa_net_client_stop,
428         .cleanup = vhost_vdpa_cleanup,
429         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
430         .has_ufo = vhost_vdpa_has_ufo,
431         .check_peer_type = vhost_vdpa_check_peer_type,
432         .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
433 };
434 
435 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
436                                           Error **errp)
437 {
438     struct vhost_vring_state state = {
439         .index = vq_index,
440     };
441     int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
442 
443     if (unlikely(r < 0)) {
444         r = -errno;
445         error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index);
446         return r;
447     }
448 
449     return state.num;
450 }
451 
452 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
453                                            unsigned vq_group,
454                                            unsigned asid_num)
455 {
456     struct vhost_vring_state asid = {
457         .index = vq_group,
458         .num = asid_num,
459     };
460     int r;
461 
462     r = ioctl(v->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
463     if (unlikely(r < 0)) {
464         error_report("Can't set vq group %u asid %u, errno=%d (%s)",
465                      asid.index, asid.num, errno, g_strerror(errno));
466     }
467     return r;
468 }
469 
470 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
471 {
472     VhostIOVATree *tree = v->iova_tree;
473     DMAMap needle = {
474         /*
475          * No need to specify size or to look for more translations since
476          * this contiguous chunk was allocated by us.
477          */
478         .translated_addr = (hwaddr)(uintptr_t)addr,
479     };
480     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
481     int r;
482 
483     if (unlikely(!map)) {
484         error_report("Cannot locate expected map");
485         return;
486     }
487 
488     r = vhost_vdpa_dma_unmap(v, v->address_space_id, map->iova, map->size + 1);
489     if (unlikely(r != 0)) {
490         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
491     }
492 
493     vhost_iova_tree_remove(tree, *map);
494 }
495 
496 /** Map CVQ buffer. */
497 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
498                                   bool write)
499 {
500     DMAMap map = {};
501     int r;
502 
503     map.translated_addr = (hwaddr)(uintptr_t)buf;
504     map.size = size - 1;
505     map.perm = write ? IOMMU_RW : IOMMU_RO,
506     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
507     if (unlikely(r != IOVA_OK)) {
508         error_report("Cannot map injected element");
509         return r;
510     }
511 
512     r = vhost_vdpa_dma_map(v, v->address_space_id, map.iova,
513                            vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
514     if (unlikely(r < 0)) {
515         goto dma_map_err;
516     }
517 
518     return 0;
519 
520 dma_map_err:
521     vhost_iova_tree_remove(v->iova_tree, map);
522     return r;
523 }
524 
525 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
526 {
527     VhostVDPAState *s, *s0;
528     struct vhost_vdpa *v;
529     int64_t cvq_group;
530     int r;
531     Error *err = NULL;
532 
533     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
534 
535     s = DO_UPCAST(VhostVDPAState, nc, nc);
536     v = &s->vhost_vdpa;
537 
538     s0 = vhost_vdpa_net_first_nc_vdpa(s);
539     v->shadow_data = s0->vhost_vdpa.shadow_vqs_enabled;
540     v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled;
541     s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
542 
543     if (s->vhost_vdpa.shadow_data) {
544         /* SVQ is already configured for all virtqueues */
545         goto out;
546     }
547 
548     /*
549      * If we early return in these cases SVQ will not be enabled. The migration
550      * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
551      */
552     if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) {
553         return 0;
554     }
555 
556     if (!s->cvq_isolated) {
557         return 0;
558     }
559 
560     cvq_group = vhost_vdpa_get_vring_group(v->device_fd,
561                                            v->dev->vq_index_end - 1,
562                                            &err);
563     if (unlikely(cvq_group < 0)) {
564         error_report_err(err);
565         return cvq_group;
566     }
567 
568     r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
569     if (unlikely(r < 0)) {
570         return r;
571     }
572 
573     v->shadow_vqs_enabled = true;
574     s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
575 
576 out:
577     if (!s->vhost_vdpa.shadow_vqs_enabled) {
578         return 0;
579     }
580 
581     if (s0->vhost_vdpa.iova_tree) {
582         /*
583          * SVQ is already configured for all virtqueues.  Reuse IOVA tree for
584          * simplicity, whether CVQ shares ASID with guest or not, because:
585          * - Memory listener need access to guest's memory addresses allocated
586          *   in the IOVA tree.
587          * - There should be plenty of IOVA address space for both ASID not to
588          *   worry about collisions between them.  Guest's translations are
589          *   still validated with virtio virtqueue_pop so there is no risk for
590          *   the guest to access memory that it shouldn't.
591          *
592          * To allocate a iova tree per ASID is doable but it complicates the
593          * code and it is not worth it for the moment.
594          */
595         v->iova_tree = s0->vhost_vdpa.iova_tree;
596     } else {
597         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
598                                            v->iova_range.last);
599     }
600 
601     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
602                                vhost_vdpa_net_cvq_cmd_page_len(), false);
603     if (unlikely(r < 0)) {
604         return r;
605     }
606 
607     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
608                                vhost_vdpa_net_cvq_cmd_page_len(), true);
609     if (unlikely(r < 0)) {
610         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
611     }
612 
613     return r;
614 }
615 
616 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
617 {
618     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
619 
620     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
621 
622     if (s->vhost_vdpa.shadow_vqs_enabled) {
623         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
624         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
625     }
626 
627     vhost_vdpa_net_client_stop(nc);
628 }
629 
630 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s,
631                                     const struct iovec *out_sg, size_t out_num,
632                                     const struct iovec *in_sg, size_t in_num)
633 {
634     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
635     int r;
636 
637     r = vhost_svq_add(svq, out_sg, out_num, in_sg, in_num, NULL);
638     if (unlikely(r != 0)) {
639         if (unlikely(r == -ENOSPC)) {
640             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
641                           __func__);
642         }
643     }
644 
645     return r;
646 }
647 
648 /*
649  * Convenience wrapper to poll SVQ for multiple control commands.
650  *
651  * Caller should hold the BQL when invoking this function, and should take
652  * the answer before SVQ pulls by itself when BQL is released.
653  */
654 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight)
655 {
656     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
657     return vhost_svq_poll(svq, cmds_in_flight);
658 }
659 
660 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s,
661                                              struct iovec *out_cursor,
662                                              struct iovec *in_cursor)
663 {
664     /* reset the cursor of the output buffer for the device */
665     out_cursor->iov_base = s->cvq_cmd_out_buffer;
666     out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
667 
668     /* reset the cursor of the in buffer for the device */
669     in_cursor->iov_base = s->status;
670     in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
671 }
672 
673 /*
674  * Poll SVQ for multiple pending control commands and check the device's ack.
675  *
676  * Caller should hold the BQL when invoking this function.
677  *
678  * @s: The VhostVDPAState
679  * @len: The length of the pending status shadow buffer
680  */
681 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len)
682 {
683     /* device uses a one-byte length ack for each control command */
684     ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len);
685     if (unlikely(dev_written != len)) {
686         return -EIO;
687     }
688 
689     /* check the device's ack */
690     for (int i = 0; i < len; ++i) {
691         if (s->status[i] != VIRTIO_NET_OK) {
692             return -EIO;
693         }
694     }
695     return 0;
696 }
697 
698 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s,
699                                        struct iovec *out_cursor,
700                                        struct iovec *in_cursor, uint8_t class,
701                                        uint8_t cmd, const struct iovec *data_sg,
702                                        size_t data_num)
703 {
704     const struct virtio_net_ctrl_hdr ctrl = {
705         .class = class,
706         .cmd = cmd,
707     };
708     size_t data_size = iov_size(data_sg, data_num), cmd_size;
709     struct iovec out, in;
710     ssize_t r;
711     unsigned dummy_cursor_iov_cnt;
712     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
713 
714     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
715     cmd_size = sizeof(ctrl) + data_size;
716     if (vhost_svq_available_slots(svq) < 2 ||
717         iov_size(out_cursor, 1) < cmd_size) {
718         /*
719          * It is time to flush all pending control commands if SVQ is full
720          * or control commands shadow buffers are full.
721          *
722          * We can poll here since we've had BQL from the time
723          * we sent the descriptor.
724          */
725         r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base -
726                                      (void *)s->status);
727         if (unlikely(r < 0)) {
728             return r;
729         }
730 
731         vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor);
732     }
733 
734     /* pack the CVQ command header */
735     iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl));
736     /* pack the CVQ command command-specific-data */
737     iov_to_buf(data_sg, data_num, 0,
738                out_cursor->iov_base + sizeof(ctrl), data_size);
739 
740     /* extract the required buffer from the cursor for output */
741     iov_copy(&out, 1, out_cursor, 1, 0, cmd_size);
742     /* extract the required buffer from the cursor for input */
743     iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status));
744 
745     r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1);
746     if (unlikely(r < 0)) {
747         return r;
748     }
749 
750     /* iterate the cursors */
751     dummy_cursor_iov_cnt = 1;
752     iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size);
753     dummy_cursor_iov_cnt = 1;
754     iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status));
755 
756     return 0;
757 }
758 
759 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n,
760                                    struct iovec *out_cursor,
761                                    struct iovec *in_cursor)
762 {
763     if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
764         const struct iovec data = {
765             .iov_base = (void *)n->mac,
766             .iov_len = sizeof(n->mac),
767         };
768         ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
769                                             VIRTIO_NET_CTRL_MAC,
770                                             VIRTIO_NET_CTRL_MAC_ADDR_SET,
771                                             &data, 1);
772         if (unlikely(r < 0)) {
773             return r;
774         }
775     }
776 
777     /*
778      * According to VirtIO standard, "The device MUST have an
779      * empty MAC filtering table on reset.".
780      *
781      * Therefore, there is no need to send this CVQ command if the
782      * driver also sets an empty MAC filter table, which aligns with
783      * the device's defaults.
784      *
785      * Note that the device's defaults can mismatch the driver's
786      * configuration only at live migration.
787      */
788     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) ||
789         n->mac_table.in_use == 0) {
790         return 0;
791     }
792 
793     uint32_t uni_entries = n->mac_table.first_multi,
794              uni_macs_size = uni_entries * ETH_ALEN,
795              mul_entries = n->mac_table.in_use - uni_entries,
796              mul_macs_size = mul_entries * ETH_ALEN;
797     struct virtio_net_ctrl_mac uni = {
798         .entries = cpu_to_le32(uni_entries),
799     };
800     struct virtio_net_ctrl_mac mul = {
801         .entries = cpu_to_le32(mul_entries),
802     };
803     const struct iovec data[] = {
804         {
805             .iov_base = &uni,
806             .iov_len = sizeof(uni),
807         }, {
808             .iov_base = n->mac_table.macs,
809             .iov_len = uni_macs_size,
810         }, {
811             .iov_base = &mul,
812             .iov_len = sizeof(mul),
813         }, {
814             .iov_base = &n->mac_table.macs[uni_macs_size],
815             .iov_len = mul_macs_size,
816         },
817     };
818     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
819                                         VIRTIO_NET_CTRL_MAC,
820                                         VIRTIO_NET_CTRL_MAC_TABLE_SET,
821                                         data, ARRAY_SIZE(data));
822     if (unlikely(r < 0)) {
823         return r;
824     }
825 
826     return 0;
827 }
828 
829 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
830                                    struct iovec *out_cursor,
831                                    struct iovec *in_cursor)
832 {
833     struct virtio_net_rss_config cfg = {};
834     ssize_t r;
835     g_autofree uint16_t *table = NULL;
836 
837     /*
838      * According to VirtIO standard, "Initially the device has all hash
839      * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
840      *
841      * Therefore, there is no need to send this CVQ command if the
842      * driver disables the all hash types, which aligns with
843      * the device's defaults.
844      *
845      * Note that the device's defaults can mismatch the driver's
846      * configuration only at live migration.
847      */
848     if (!n->rss_data.enabled ||
849         n->rss_data.hash_types == VIRTIO_NET_HASH_REPORT_NONE) {
850         return 0;
851     }
852 
853     table = g_malloc_n(n->rss_data.indirections_len,
854                        sizeof(n->rss_data.indirections_table[0]));
855     cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
856 
857     /*
858      * According to VirtIO standard, "Field reserved MUST contain zeroes.
859      * It is defined to make the structure to match the layout of
860      * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
861      *
862      * Therefore, we need to zero the fields in
863      * struct virtio_net_rss_config, which corresponds to the
864      * `reserved` field in struct virtio_net_hash_config.
865      *
866      * Note that all other fields are zeroed at their definitions,
867      * except for the `indirection_table` field, where the actual data
868      * is stored in the `table` variable to ensure compatibility
869      * with RSS case. Therefore, we need to zero the `table` variable here.
870      */
871     table[0] = 0;
872 
873     /*
874      * Considering that virtio_net_handle_rss() currently does not restore
875      * the hash key length parsed from the CVQ command sent from the guest
876      * into n->rss_data and uses the maximum key length in other code, so
877      * we also employ the maximum key length here.
878      */
879     cfg.hash_key_length = sizeof(n->rss_data.key);
880 
881     const struct iovec data[] = {
882         {
883             .iov_base = &cfg,
884             .iov_len = offsetof(struct virtio_net_rss_config,
885                                 indirection_table),
886         }, {
887             .iov_base = table,
888             .iov_len = n->rss_data.indirections_len *
889                        sizeof(n->rss_data.indirections_table[0]),
890         }, {
891             .iov_base = &cfg.max_tx_vq,
892             .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) -
893                        offsetof(struct virtio_net_rss_config, max_tx_vq),
894         }, {
895             .iov_base = (void *)n->rss_data.key,
896             .iov_len = sizeof(n->rss_data.key),
897         }
898     };
899 
900     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
901                                 VIRTIO_NET_CTRL_MQ,
902                                 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
903                                 data, ARRAY_SIZE(data));
904     if (unlikely(r < 0)) {
905         return r;
906     }
907 
908     return 0;
909 }
910 
911 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
912                                   const VirtIONet *n,
913                                   struct iovec *out_cursor,
914                                   struct iovec *in_cursor)
915 {
916     struct virtio_net_ctrl_mq mq;
917     ssize_t r;
918 
919     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) {
920         return 0;
921     }
922 
923     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
924     const struct iovec data = {
925         .iov_base = &mq,
926         .iov_len = sizeof(mq),
927     };
928     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
929                                 VIRTIO_NET_CTRL_MQ,
930                                 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
931                                 &data, 1);
932     if (unlikely(r < 0)) {
933         return r;
934     }
935 
936     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_HASH_REPORT)) {
937         return 0;
938     }
939 
940     r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor);
941     if (unlikely(r < 0)) {
942         return r;
943     }
944 
945     return 0;
946 }
947 
948 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s,
949                                         const VirtIONet *n,
950                                         struct iovec *out_cursor,
951                                         struct iovec *in_cursor)
952 {
953     uint64_t offloads;
954     ssize_t r;
955 
956     if (!virtio_vdev_has_feature(&n->parent_obj,
957                                  VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
958         return 0;
959     }
960 
961     if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) {
962         /*
963          * According to VirtIO standard, "Upon feature negotiation
964          * corresponding offload gets enabled to preserve
965          * backward compatibility.".
966          *
967          * Therefore, there is no need to send this CVQ command if the
968          * driver also enables all supported offloads, which aligns with
969          * the device's defaults.
970          *
971          * Note that the device's defaults can mismatch the driver's
972          * configuration only at live migration.
973          */
974         return 0;
975     }
976 
977     offloads = cpu_to_le64(n->curr_guest_offloads);
978     const struct iovec data = {
979         .iov_base = &offloads,
980         .iov_len = sizeof(offloads),
981     };
982     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
983                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS,
984                                 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET,
985                                 &data, 1);
986     if (unlikely(r < 0)) {
987         return r;
988     }
989 
990     return 0;
991 }
992 
993 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s,
994                                        struct iovec *out_cursor,
995                                        struct iovec *in_cursor,
996                                        uint8_t cmd,
997                                        uint8_t on)
998 {
999     const struct iovec data = {
1000         .iov_base = &on,
1001         .iov_len = sizeof(on),
1002     };
1003     ssize_t r;
1004 
1005     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1006                                 VIRTIO_NET_CTRL_RX, cmd, &data, 1);
1007     if (unlikely(r < 0)) {
1008         return r;
1009     }
1010 
1011     return 0;
1012 }
1013 
1014 static int vhost_vdpa_net_load_rx(VhostVDPAState *s,
1015                                   const VirtIONet *n,
1016                                   struct iovec *out_cursor,
1017                                   struct iovec *in_cursor)
1018 {
1019     ssize_t r;
1020 
1021     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) {
1022         return 0;
1023     }
1024 
1025     /*
1026      * According to virtio_net_reset(), device turns promiscuous mode
1027      * on by default.
1028      *
1029      * Additionally, according to VirtIO standard, "Since there are
1030      * no guarantees, it can use a hash filter or silently switch to
1031      * allmulti or promiscuous mode if it is given too many addresses.".
1032      * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1033      * non-multicast MAC addresses, indicating that promiscuous mode
1034      * should be enabled.
1035      *
1036      * Therefore, QEMU should only send this CVQ command if the
1037      * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1038      * which sets promiscuous mode on, different from the device's defaults.
1039      *
1040      * Note that the device's defaults can mismatch the driver's
1041      * configuration only at live migration.
1042      */
1043     if (!n->mac_table.uni_overflow && !n->promisc) {
1044         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1045                                         VIRTIO_NET_CTRL_RX_PROMISC, 0);
1046         if (unlikely(r < 0)) {
1047             return r;
1048         }
1049     }
1050 
1051     /*
1052      * According to virtio_net_reset(), device turns all-multicast mode
1053      * off by default.
1054      *
1055      * According to VirtIO standard, "Since there are no guarantees,
1056      * it can use a hash filter or silently switch to allmulti or
1057      * promiscuous mode if it is given too many addresses.". QEMU marks
1058      * `n->mac_table.multi_overflow` if guest sets too many
1059      * non-multicast MAC addresses.
1060      *
1061      * Therefore, QEMU should only send this CVQ command if the
1062      * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1063      * which sets all-multicast mode on, different from the device's defaults.
1064      *
1065      * Note that the device's defaults can mismatch the driver's
1066      * configuration only at live migration.
1067      */
1068     if (n->mac_table.multi_overflow || n->allmulti) {
1069         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1070                                         VIRTIO_NET_CTRL_RX_ALLMULTI, 1);
1071         if (unlikely(r < 0)) {
1072             return r;
1073         }
1074     }
1075 
1076     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) {
1077         return 0;
1078     }
1079 
1080     /*
1081      * According to virtio_net_reset(), device turns all-unicast mode
1082      * off by default.
1083      *
1084      * Therefore, QEMU should only send this CVQ command if the driver
1085      * sets all-unicast mode on, different from the device's defaults.
1086      *
1087      * Note that the device's defaults can mismatch the driver's
1088      * configuration only at live migration.
1089      */
1090     if (n->alluni) {
1091         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1092                                         VIRTIO_NET_CTRL_RX_ALLUNI, 1);
1093         if (r < 0) {
1094             return r;
1095         }
1096     }
1097 
1098     /*
1099      * According to virtio_net_reset(), device turns non-multicast mode
1100      * off by default.
1101      *
1102      * Therefore, QEMU should only send this CVQ command if the driver
1103      * sets non-multicast mode on, different from the device's defaults.
1104      *
1105      * Note that the device's defaults can mismatch the driver's
1106      * configuration only at live migration.
1107      */
1108     if (n->nomulti) {
1109         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1110                                         VIRTIO_NET_CTRL_RX_NOMULTI, 1);
1111         if (r < 0) {
1112             return r;
1113         }
1114     }
1115 
1116     /*
1117      * According to virtio_net_reset(), device turns non-unicast mode
1118      * off by default.
1119      *
1120      * Therefore, QEMU should only send this CVQ command if the driver
1121      * sets non-unicast mode on, different from the device's defaults.
1122      *
1123      * Note that the device's defaults can mismatch the driver's
1124      * configuration only at live migration.
1125      */
1126     if (n->nouni) {
1127         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1128                                         VIRTIO_NET_CTRL_RX_NOUNI, 1);
1129         if (r < 0) {
1130             return r;
1131         }
1132     }
1133 
1134     /*
1135      * According to virtio_net_reset(), device turns non-broadcast mode
1136      * off by default.
1137      *
1138      * Therefore, QEMU should only send this CVQ command if the driver
1139      * sets non-broadcast mode on, different from the device's defaults.
1140      *
1141      * Note that the device's defaults can mismatch the driver's
1142      * configuration only at live migration.
1143      */
1144     if (n->nobcast) {
1145         r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1146                                         VIRTIO_NET_CTRL_RX_NOBCAST, 1);
1147         if (r < 0) {
1148             return r;
1149         }
1150     }
1151 
1152     return 0;
1153 }
1154 
1155 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s,
1156                                            const VirtIONet *n,
1157                                            struct iovec *out_cursor,
1158                                            struct iovec *in_cursor,
1159                                            uint16_t vid)
1160 {
1161     const struct iovec data = {
1162         .iov_base = &vid,
1163         .iov_len = sizeof(vid),
1164     };
1165     ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1166                                         VIRTIO_NET_CTRL_VLAN,
1167                                         VIRTIO_NET_CTRL_VLAN_ADD,
1168                                         &data, 1);
1169     if (unlikely(r < 0)) {
1170         return r;
1171     }
1172 
1173     return 0;
1174 }
1175 
1176 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s,
1177                                     const VirtIONet *n,
1178                                     struct iovec *out_cursor,
1179                                     struct iovec *in_cursor)
1180 {
1181     int r;
1182 
1183     if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) {
1184         return 0;
1185     }
1186 
1187     for (int i = 0; i < MAX_VLAN >> 5; i++) {
1188         for (int j = 0; n->vlans[i] && j <= 0x1f; j++) {
1189             if (n->vlans[i] & (1U << j)) {
1190                 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor,
1191                                                     in_cursor, (i << 5) + j);
1192                 if (unlikely(r != 0)) {
1193                     return r;
1194                 }
1195             }
1196         }
1197     }
1198 
1199     return 0;
1200 }
1201 
1202 static int vhost_vdpa_net_cvq_load(NetClientState *nc)
1203 {
1204     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
1205     struct vhost_vdpa *v = &s->vhost_vdpa;
1206     const VirtIONet *n;
1207     int r;
1208     struct iovec out_cursor, in_cursor;
1209 
1210     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1211 
1212     vhost_vdpa_set_vring_ready(v, v->dev->vq_index);
1213 
1214     if (v->shadow_vqs_enabled) {
1215         n = VIRTIO_NET(v->dev->vdev);
1216         vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor);
1217         r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor);
1218         if (unlikely(r < 0)) {
1219             return r;
1220         }
1221         r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor);
1222         if (unlikely(r)) {
1223             return r;
1224         }
1225         r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor);
1226         if (unlikely(r)) {
1227             return r;
1228         }
1229         r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor);
1230         if (unlikely(r)) {
1231             return r;
1232         }
1233         r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor);
1234         if (unlikely(r)) {
1235             return r;
1236         }
1237 
1238         /*
1239          * We need to poll and check all pending device's used buffers.
1240          *
1241          * We can poll here since we've had BQL from the time
1242          * we sent the descriptor.
1243          */
1244         r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status);
1245         if (unlikely(r)) {
1246             return r;
1247         }
1248     }
1249 
1250     for (int i = 0; i < v->dev->vq_index; ++i) {
1251         vhost_vdpa_set_vring_ready(v, i);
1252     }
1253 
1254     return 0;
1255 }
1256 
1257 static NetClientInfo net_vhost_vdpa_cvq_info = {
1258     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
1259     .size = sizeof(VhostVDPAState),
1260     .receive = vhost_vdpa_receive,
1261     .start = vhost_vdpa_net_cvq_start,
1262     .load = vhost_vdpa_net_cvq_load,
1263     .stop = vhost_vdpa_net_cvq_stop,
1264     .cleanup = vhost_vdpa_cleanup,
1265     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
1266     .has_ufo = vhost_vdpa_has_ufo,
1267     .check_peer_type = vhost_vdpa_check_peer_type,
1268     .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
1269 };
1270 
1271 /*
1272  * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1273  * vdpa device.
1274  *
1275  * Considering that QEMU cannot send the entire filter table to the
1276  * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1277  * command to enable promiscuous mode to receive all packets,
1278  * according to VirtIO standard, "Since there are no guarantees,
1279  * it can use a hash filter or silently switch to allmulti or
1280  * promiscuous mode if it is given too many addresses.".
1281  *
1282  * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1283  * marks `n->mac_table.x_overflow` accordingly, it should have
1284  * the same effect on the device model to receive
1285  * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1286  * The same applies to multicast MAC addresses.
1287  *
1288  * Therefore, QEMU can provide the device model with a fake
1289  * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1290  * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1291  * MAC addresses. This ensures that the device model marks
1292  * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1293  * allowing all packets to be received, which aligns with the
1294  * state of the vdpa device.
1295  */
1296 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s,
1297                                                        VirtQueueElement *elem,
1298                                                        struct iovec *out,
1299                                                        const struct iovec *in)
1300 {
1301     struct virtio_net_ctrl_mac mac_data, *mac_ptr;
1302     struct virtio_net_ctrl_hdr *hdr_ptr;
1303     uint32_t cursor;
1304     ssize_t r;
1305     uint8_t on = 1;
1306 
1307     /* parse the non-multicast MAC address entries from CVQ command */
1308     cursor = sizeof(*hdr_ptr);
1309     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1310                    &mac_data, sizeof(mac_data));
1311     if (unlikely(r != sizeof(mac_data))) {
1312         /*
1313          * If the CVQ command is invalid, we should simulate the vdpa device
1314          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1315          */
1316         *s->status = VIRTIO_NET_ERR;
1317         return sizeof(*s->status);
1318     }
1319     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1320 
1321     /* parse the multicast MAC address entries from CVQ command */
1322     r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1323                    &mac_data, sizeof(mac_data));
1324     if (r != sizeof(mac_data)) {
1325         /*
1326          * If the CVQ command is invalid, we should simulate the vdpa device
1327          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1328          */
1329         *s->status = VIRTIO_NET_ERR;
1330         return sizeof(*s->status);
1331     }
1332     cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1333 
1334     /* validate the CVQ command */
1335     if (iov_size(elem->out_sg, elem->out_num) != cursor) {
1336         /*
1337          * If the CVQ command is invalid, we should simulate the vdpa device
1338          * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1339          */
1340         *s->status = VIRTIO_NET_ERR;
1341         return sizeof(*s->status);
1342     }
1343 
1344     /*
1345      * According to VirtIO standard, "Since there are no guarantees,
1346      * it can use a hash filter or silently switch to allmulti or
1347      * promiscuous mode if it is given too many addresses.".
1348      *
1349      * Therefore, considering that QEMU is unable to send the entire
1350      * filter table to the vdpa device, it should send the
1351      * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1352      */
1353     hdr_ptr = out->iov_base;
1354     out->iov_len = sizeof(*hdr_ptr) + sizeof(on);
1355 
1356     hdr_ptr->class = VIRTIO_NET_CTRL_RX;
1357     hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC;
1358     iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on));
1359     r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1);
1360     if (unlikely(r < 0)) {
1361         return r;
1362     }
1363 
1364     /*
1365      * We can poll here since we've had BQL from the time
1366      * we sent the descriptor.
1367      */
1368     r = vhost_vdpa_net_svq_poll(s, 1);
1369     if (unlikely(r < sizeof(*s->status))) {
1370         return r;
1371     }
1372     if (*s->status != VIRTIO_NET_OK) {
1373         return sizeof(*s->status);
1374     }
1375 
1376     /*
1377      * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1378      * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1379      * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1380      * multicast MAC addresses.
1381      *
1382      * By doing so, the device model can mark `n->mac_table.uni_overflow`
1383      * and `n->mac_table.multi_overflow`, enabling all packets to be
1384      * received, which aligns with the state of the vdpa device.
1385      */
1386     cursor = 0;
1387     uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1,
1388              fake_mul_entries = MAC_TABLE_ENTRIES + 1,
1389              fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) +
1390                              sizeof(mac_data) + fake_uni_entries * ETH_ALEN +
1391                              sizeof(mac_data) + fake_mul_entries * ETH_ALEN;
1392 
1393     assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len());
1394     out->iov_len = fake_cvq_size;
1395 
1396     /* pack the header for fake CVQ command */
1397     hdr_ptr = out->iov_base + cursor;
1398     hdr_ptr->class = VIRTIO_NET_CTRL_MAC;
1399     hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1400     cursor += sizeof(*hdr_ptr);
1401 
1402     /*
1403      * Pack the non-multicast MAC addresses part for fake CVQ command.
1404      *
1405      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1406      * addresses provided in CVQ command. Therefore, only the entries
1407      * field need to be prepared in the CVQ command.
1408      */
1409     mac_ptr = out->iov_base + cursor;
1410     mac_ptr->entries = cpu_to_le32(fake_uni_entries);
1411     cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN;
1412 
1413     /*
1414      * Pack the multicast MAC addresses part for fake CVQ command.
1415      *
1416      * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1417      * addresses provided in CVQ command. Therefore, only the entries
1418      * field need to be prepared in the CVQ command.
1419      */
1420     mac_ptr = out->iov_base + cursor;
1421     mac_ptr->entries = cpu_to_le32(fake_mul_entries);
1422 
1423     /*
1424      * Simulating QEMU poll a vdpa device used buffer
1425      * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1426      */
1427     return sizeof(*s->status);
1428 }
1429 
1430 /**
1431  * Validate and copy control virtqueue commands.
1432  *
1433  * Following QEMU guidelines, we offer a copy of the buffers to the device to
1434  * prevent TOCTOU bugs.
1435  */
1436 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
1437                                             VirtQueueElement *elem,
1438                                             void *opaque)
1439 {
1440     VhostVDPAState *s = opaque;
1441     size_t in_len;
1442     const struct virtio_net_ctrl_hdr *ctrl;
1443     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1444     /* Out buffer sent to both the vdpa device and the device model */
1445     struct iovec out = {
1446         .iov_base = s->cvq_cmd_out_buffer,
1447     };
1448     /* in buffer used for device model */
1449     const struct iovec model_in = {
1450         .iov_base = &status,
1451         .iov_len = sizeof(status),
1452     };
1453     /* in buffer used for vdpa device */
1454     const struct iovec vdpa_in = {
1455         .iov_base = s->status,
1456         .iov_len = sizeof(*s->status),
1457     };
1458     ssize_t dev_written = -EINVAL;
1459 
1460     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
1461                              s->cvq_cmd_out_buffer,
1462                              vhost_vdpa_net_cvq_cmd_page_len());
1463 
1464     ctrl = s->cvq_cmd_out_buffer;
1465     if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) {
1466         /*
1467          * Guest announce capability is emulated by qemu, so don't forward to
1468          * the device.
1469          */
1470         dev_written = sizeof(status);
1471         *s->status = VIRTIO_NET_OK;
1472     } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC &&
1473                         ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET &&
1474                         iov_size(elem->out_sg, elem->out_num) > out.iov_len)) {
1475         /*
1476          * Due to the size limitation of the out buffer sent to the vdpa device,
1477          * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1478          * MAC addresses set by the driver for the filter table can cause
1479          * truncation of the CVQ command in QEMU. As a result, the vdpa device
1480          * rejects the flawed CVQ command.
1481          *
1482          * Therefore, QEMU must handle this situation instead of sending
1483          * the CVQ command directly.
1484          */
1485         dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem,
1486                                                             &out, &vdpa_in);
1487         if (unlikely(dev_written < 0)) {
1488             goto out;
1489         }
1490     } else {
1491         ssize_t r;
1492         r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1);
1493         if (unlikely(r < 0)) {
1494             dev_written = r;
1495             goto out;
1496         }
1497 
1498         /*
1499          * We can poll here since we've had BQL from the time
1500          * we sent the descriptor.
1501          */
1502         dev_written = vhost_vdpa_net_svq_poll(s, 1);
1503     }
1504 
1505     if (unlikely(dev_written < sizeof(status))) {
1506         error_report("Insufficient written data (%zu)", dev_written);
1507         goto out;
1508     }
1509 
1510     if (*s->status != VIRTIO_NET_OK) {
1511         goto out;
1512     }
1513 
1514     status = VIRTIO_NET_ERR;
1515     virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1);
1516     if (status != VIRTIO_NET_OK) {
1517         error_report("Bad CVQ processing in model");
1518     }
1519 
1520 out:
1521     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
1522                           sizeof(status));
1523     if (unlikely(in_len < sizeof(status))) {
1524         error_report("Bad device CVQ written length");
1525     }
1526     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
1527     /*
1528      * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1529      * the function successfully forwards the CVQ command, indicated
1530      * by a non-negative value of `dev_written`. Otherwise, it still
1531      * belongs to SVQ.
1532      * This function should only free the `elem` when it owns.
1533      */
1534     if (dev_written >= 0) {
1535         g_free(elem);
1536     }
1537     return dev_written < 0 ? dev_written : 0;
1538 }
1539 
1540 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
1541     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
1542 };
1543 
1544 /**
1545  * Probe if CVQ is isolated
1546  *
1547  * @device_fd         The vdpa device fd
1548  * @features          Features offered by the device.
1549  * @cvq_index         The control vq pair index
1550  *
1551  * Returns <0 in case of failure, 0 if false and 1 if true.
1552  */
1553 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features,
1554                                           int cvq_index, Error **errp)
1555 {
1556     uint64_t backend_features;
1557     int64_t cvq_group;
1558     uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE |
1559                      VIRTIO_CONFIG_S_DRIVER;
1560     int r;
1561 
1562     ERRP_GUARD();
1563 
1564     r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
1565     if (unlikely(r < 0)) {
1566         error_setg_errno(errp, errno, "Cannot get vdpa backend_features");
1567         return r;
1568     }
1569 
1570     if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) {
1571         return 0;
1572     }
1573 
1574     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1575     if (unlikely(r)) {
1576         error_setg_errno(errp, -r, "Cannot set device status");
1577         goto out;
1578     }
1579 
1580     r = ioctl(device_fd, VHOST_SET_FEATURES, &features);
1581     if (unlikely(r)) {
1582         error_setg_errno(errp, -r, "Cannot set features");
1583         goto out;
1584     }
1585 
1586     status |= VIRTIO_CONFIG_S_FEATURES_OK;
1587     r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1588     if (unlikely(r)) {
1589         error_setg_errno(errp, -r, "Cannot set device status");
1590         goto out;
1591     }
1592 
1593     cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp);
1594     if (unlikely(cvq_group < 0)) {
1595         if (cvq_group != -ENOTSUP) {
1596             r = cvq_group;
1597             goto out;
1598         }
1599 
1600         /*
1601          * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1602          * support ASID even if the parent driver does not.  The CVQ cannot be
1603          * isolated in this case.
1604          */
1605         error_free(*errp);
1606         *errp = NULL;
1607         r = 0;
1608         goto out;
1609     }
1610 
1611     for (int i = 0; i < cvq_index; ++i) {
1612         int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp);
1613         if (unlikely(group < 0)) {
1614             r = group;
1615             goto out;
1616         }
1617 
1618         if (group == (int64_t)cvq_group) {
1619             r = 0;
1620             goto out;
1621         }
1622     }
1623 
1624     r = 1;
1625 
1626 out:
1627     status = 0;
1628     ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1629     return r;
1630 }
1631 
1632 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
1633                                        const char *device,
1634                                        const char *name,
1635                                        int vdpa_device_fd,
1636                                        int queue_pair_index,
1637                                        int nvqs,
1638                                        bool is_datapath,
1639                                        bool svq,
1640                                        struct vhost_vdpa_iova_range iova_range,
1641                                        uint64_t features,
1642                                        Error **errp)
1643 {
1644     NetClientState *nc = NULL;
1645     VhostVDPAState *s;
1646     int ret = 0;
1647     assert(name);
1648     int cvq_isolated = 0;
1649 
1650     if (is_datapath) {
1651         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
1652                                  name);
1653     } else {
1654         cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features,
1655                                                       queue_pair_index * 2,
1656                                                       errp);
1657         if (unlikely(cvq_isolated < 0)) {
1658             return NULL;
1659         }
1660 
1661         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
1662                                          device, name);
1663     }
1664     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
1665     s = DO_UPCAST(VhostVDPAState, nc, nc);
1666 
1667     s->vhost_vdpa.device_fd = vdpa_device_fd;
1668     s->vhost_vdpa.index = queue_pair_index;
1669     s->always_svq = svq;
1670     s->migration_state.notify = NULL;
1671     s->vhost_vdpa.shadow_vqs_enabled = svq;
1672     s->vhost_vdpa.iova_range = iova_range;
1673     s->vhost_vdpa.shadow_data = svq;
1674     if (queue_pair_index == 0) {
1675         vhost_vdpa_net_valid_svq_features(features,
1676                                           &s->vhost_vdpa.migration_blocker);
1677     } else if (!is_datapath) {
1678         s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1679                                      PROT_READ | PROT_WRITE,
1680                                      MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1681         s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1682                          PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
1683                          -1, 0);
1684 
1685         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
1686         s->vhost_vdpa.shadow_vq_ops_opaque = s;
1687         s->cvq_isolated = cvq_isolated;
1688     }
1689     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
1690     if (ret) {
1691         qemu_del_net_client(nc);
1692         return NULL;
1693     }
1694     return nc;
1695 }
1696 
1697 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
1698 {
1699     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
1700     if (unlikely(ret < 0)) {
1701         error_setg_errno(errp, errno,
1702                          "Fail to query features from vhost-vDPA device");
1703     }
1704     return ret;
1705 }
1706 
1707 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
1708                                           int *has_cvq, Error **errp)
1709 {
1710     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
1711     g_autofree struct vhost_vdpa_config *config = NULL;
1712     __virtio16 *max_queue_pairs;
1713     int ret;
1714 
1715     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
1716         *has_cvq = 1;
1717     } else {
1718         *has_cvq = 0;
1719     }
1720 
1721     if (features & (1 << VIRTIO_NET_F_MQ)) {
1722         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
1723         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
1724         config->len = sizeof(*max_queue_pairs);
1725 
1726         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
1727         if (ret) {
1728             error_setg(errp, "Fail to get config from vhost-vDPA device");
1729             return -ret;
1730         }
1731 
1732         max_queue_pairs = (__virtio16 *)&config->buf;
1733 
1734         return lduw_le_p(max_queue_pairs);
1735     }
1736 
1737     return 1;
1738 }
1739 
1740 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
1741                         NetClientState *peer, Error **errp)
1742 {
1743     const NetdevVhostVDPAOptions *opts;
1744     uint64_t features;
1745     int vdpa_device_fd;
1746     g_autofree NetClientState **ncs = NULL;
1747     struct vhost_vdpa_iova_range iova_range;
1748     NetClientState *nc;
1749     int queue_pairs, r, i = 0, has_cvq = 0;
1750 
1751     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1752     opts = &netdev->u.vhost_vdpa;
1753     if (!opts->vhostdev && !opts->vhostfd) {
1754         error_setg(errp,
1755                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1756         return -1;
1757     }
1758 
1759     if (opts->vhostdev && opts->vhostfd) {
1760         error_setg(errp,
1761                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1762         return -1;
1763     }
1764 
1765     if (opts->vhostdev) {
1766         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
1767         if (vdpa_device_fd == -1) {
1768             return -errno;
1769         }
1770     } else {
1771         /* has_vhostfd */
1772         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
1773         if (vdpa_device_fd == -1) {
1774             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
1775             return -1;
1776         }
1777     }
1778 
1779     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
1780     if (unlikely(r < 0)) {
1781         goto err;
1782     }
1783 
1784     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
1785                                                  &has_cvq, errp);
1786     if (queue_pairs < 0) {
1787         qemu_close(vdpa_device_fd);
1788         return queue_pairs;
1789     }
1790 
1791     r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
1792     if (unlikely(r < 0)) {
1793         error_setg(errp, "vhost-vdpa: get iova range failed: %s",
1794                    strerror(-r));
1795         goto err;
1796     }
1797 
1798     if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
1799         goto err;
1800     }
1801 
1802     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
1803 
1804     for (i = 0; i < queue_pairs; i++) {
1805         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1806                                      vdpa_device_fd, i, 2, true, opts->x_svq,
1807                                      iova_range, features, errp);
1808         if (!ncs[i])
1809             goto err;
1810     }
1811 
1812     if (has_cvq) {
1813         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1814                                  vdpa_device_fd, i, 1, false,
1815                                  opts->x_svq, iova_range, features, errp);
1816         if (!nc)
1817             goto err;
1818     }
1819 
1820     return 0;
1821 
1822 err:
1823     if (i) {
1824         for (i--; i >= 0; i--) {
1825             qemu_del_net_client(ncs[i]);
1826         }
1827     }
1828 
1829     qemu_close(vdpa_device_fd);
1830 
1831     return -1;
1832 }
1833