1 /*
2 * vhost-vdpa.c
3 *
4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 *
10 */
11
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/log.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
26 #include <err.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/misc.h"
30 #include "hw/virtio/vhost.h"
31 #include "trace.h"
32
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState {
35 NetClientState nc;
36 struct vhost_vdpa vhost_vdpa;
37 NotifierWithReturn migration_state;
38 VHostNetState *vhost_net;
39
40 /* Control commands shadow buffers */
41 void *cvq_cmd_out_buffer;
42 virtio_net_ctrl_ack *status;
43
44 /* The device always have SVQ enabled */
45 bool always_svq;
46
47 /* The device can isolate CVQ in its own ASID */
48 bool cvq_isolated;
49
50 bool started;
51 } VhostVDPAState;
52
53 /*
54 * The array is sorted alphabetically in ascending order,
55 * with the exception of VHOST_INVALID_FEATURE_BIT,
56 * which should always be the last entry.
57 */
58 const int vdpa_feature_bits[] = {
59 VIRTIO_F_ANY_LAYOUT,
60 VIRTIO_F_IOMMU_PLATFORM,
61 VIRTIO_F_NOTIFY_ON_EMPTY,
62 VIRTIO_F_RING_PACKED,
63 VIRTIO_F_RING_RESET,
64 VIRTIO_F_VERSION_1,
65 VIRTIO_F_IN_ORDER,
66 VIRTIO_F_NOTIFICATION_DATA,
67 VIRTIO_NET_F_CSUM,
68 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
69 VIRTIO_NET_F_CTRL_MAC_ADDR,
70 VIRTIO_NET_F_CTRL_RX,
71 VIRTIO_NET_F_CTRL_RX_EXTRA,
72 VIRTIO_NET_F_CTRL_VLAN,
73 VIRTIO_NET_F_CTRL_VQ,
74 VIRTIO_NET_F_GSO,
75 VIRTIO_NET_F_GUEST_CSUM,
76 VIRTIO_NET_F_GUEST_ECN,
77 VIRTIO_NET_F_GUEST_TSO4,
78 VIRTIO_NET_F_GUEST_TSO6,
79 VIRTIO_NET_F_GUEST_UFO,
80 VIRTIO_NET_F_GUEST_USO4,
81 VIRTIO_NET_F_GUEST_USO6,
82 VIRTIO_NET_F_HASH_REPORT,
83 VIRTIO_NET_F_HOST_ECN,
84 VIRTIO_NET_F_HOST_TSO4,
85 VIRTIO_NET_F_HOST_TSO6,
86 VIRTIO_NET_F_HOST_UFO,
87 VIRTIO_NET_F_HOST_USO,
88 VIRTIO_NET_F_MQ,
89 VIRTIO_NET_F_MRG_RXBUF,
90 VIRTIO_NET_F_MTU,
91 VIRTIO_NET_F_RSC_EXT,
92 VIRTIO_NET_F_RSS,
93 VIRTIO_NET_F_STATUS,
94 VIRTIO_RING_F_EVENT_IDX,
95 VIRTIO_RING_F_INDIRECT_DESC,
96
97 /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
98 VHOST_INVALID_FEATURE_BIT
99 };
100
101 /** Supported device specific feature bits with SVQ */
102 static const uint64_t vdpa_svq_device_features =
103 BIT_ULL(VIRTIO_NET_F_CSUM) |
104 BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
105 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
106 BIT_ULL(VIRTIO_NET_F_MTU) |
107 BIT_ULL(VIRTIO_NET_F_MAC) |
108 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
109 BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
110 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
111 BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
112 BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
113 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
114 BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
115 BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
116 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
117 BIT_ULL(VIRTIO_NET_F_STATUS) |
118 BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
119 BIT_ULL(VIRTIO_NET_F_CTRL_RX) |
120 BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) |
121 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) |
122 BIT_ULL(VIRTIO_NET_F_MQ) |
123 BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
124 BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
125 /* VHOST_F_LOG_ALL is exposed by SVQ */
126 BIT_ULL(VHOST_F_LOG_ALL) |
127 BIT_ULL(VIRTIO_NET_F_HASH_REPORT) |
128 BIT_ULL(VIRTIO_NET_F_RSS) |
129 BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
130 BIT_ULL(VIRTIO_NET_F_STANDBY) |
131 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
132
133 #define VHOST_VDPA_NET_CVQ_ASID 1
134
vhost_vdpa_get_vhost_net(NetClientState * nc)135 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
136 {
137 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
138 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
139 return s->vhost_net;
140 }
141
vhost_vdpa_net_cvq_cmd_len(void)142 static size_t vhost_vdpa_net_cvq_cmd_len(void)
143 {
144 /*
145 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
146 * In buffer is always 1 byte, so it should fit here
147 */
148 return sizeof(struct virtio_net_ctrl_hdr) +
149 2 * sizeof(struct virtio_net_ctrl_mac) +
150 MAC_TABLE_ENTRIES * ETH_ALEN;
151 }
152
vhost_vdpa_net_cvq_cmd_page_len(void)153 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
154 {
155 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
156 }
157
vhost_vdpa_net_valid_svq_features(uint64_t features,Error ** errp)158 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
159 {
160 uint64_t invalid_dev_features =
161 features & ~vdpa_svq_device_features &
162 /* Transport are all accepted at this point */
163 ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
164 VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
165
166 if (invalid_dev_features) {
167 error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
168 invalid_dev_features);
169 return false;
170 }
171
172 return vhost_svq_valid_features(features, errp);
173 }
174
vhost_vdpa_net_check_device_id(struct vhost_net * net)175 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
176 {
177 uint32_t device_id;
178 int ret;
179 struct vhost_dev *hdev;
180
181 hdev = (struct vhost_dev *)&net->dev;
182 ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
183 if (device_id != VIRTIO_ID_NET) {
184 return -ENOTSUP;
185 }
186 return ret;
187 }
188
vhost_vdpa_add(NetClientState * ncs,void * be,int queue_pair_index,int nvqs)189 static int vhost_vdpa_add(NetClientState *ncs, void *be,
190 int queue_pair_index, int nvqs)
191 {
192 VhostNetOptions options;
193 struct vhost_net *net = NULL;
194 VhostVDPAState *s;
195 int ret;
196
197 options.backend_type = VHOST_BACKEND_TYPE_VDPA;
198 assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
199 s = DO_UPCAST(VhostVDPAState, nc, ncs);
200 options.net_backend = ncs;
201 options.opaque = be;
202 options.busyloop_timeout = 0;
203 options.nvqs = nvqs;
204
205 net = vhost_net_init(&options);
206 if (!net) {
207 error_report("failed to init vhost_net for queue");
208 goto err_init;
209 }
210 s->vhost_net = net;
211 ret = vhost_vdpa_net_check_device_id(net);
212 if (ret) {
213 goto err_check;
214 }
215 return 0;
216 err_check:
217 vhost_net_cleanup(net);
218 g_free(net);
219 err_init:
220 return -1;
221 }
222
vhost_vdpa_cleanup(NetClientState * nc)223 static void vhost_vdpa_cleanup(NetClientState *nc)
224 {
225 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
226
227 /*
228 * If a peer NIC is attached, do not cleanup anything.
229 * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
230 * when the guest is shutting down.
231 */
232 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
233 return;
234 }
235 munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len());
236 munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len());
237 if (s->vhost_net) {
238 vhost_net_cleanup(s->vhost_net);
239 g_free(s->vhost_net);
240 s->vhost_net = NULL;
241 }
242 if (s->vhost_vdpa.index != 0) {
243 return;
244 }
245 qemu_close(s->vhost_vdpa.shared->device_fd);
246 g_free(s->vhost_vdpa.shared);
247 }
248
249 /** Dummy SetSteeringEBPF to support RSS for vhost-vdpa backend */
vhost_vdpa_set_steering_ebpf(NetClientState * nc,int prog_fd)250 static bool vhost_vdpa_set_steering_ebpf(NetClientState *nc, int prog_fd)
251 {
252 return true;
253 }
254
vhost_vdpa_has_vnet_hdr(NetClientState * nc)255 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
256 {
257 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
258
259 return true;
260 }
261
vhost_vdpa_has_ufo(NetClientState * nc)262 static bool vhost_vdpa_has_ufo(NetClientState *nc)
263 {
264 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
265 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
266 uint64_t features = 0;
267 features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
268 features = vhost_net_get_features(s->vhost_net, features);
269 return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
270
271 }
272
vhost_vdpa_check_peer_type(NetClientState * nc,ObjectClass * oc,Error ** errp)273 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
274 Error **errp)
275 {
276 const char *driver = object_class_get_name(oc);
277
278 if (!g_str_has_prefix(driver, "virtio-net-")) {
279 error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
280 return false;
281 }
282
283 return true;
284 }
285
286 /** Dummy receive in case qemu falls back to userland tap networking */
vhost_vdpa_receive(NetClientState * nc,const uint8_t * buf,size_t size)287 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
288 size_t size)
289 {
290 return size;
291 }
292
293
294 /** From any vdpa net client, get the netclient of the i-th queue pair */
vhost_vdpa_net_get_nc_vdpa(VhostVDPAState * s,int i)295 static VhostVDPAState *vhost_vdpa_net_get_nc_vdpa(VhostVDPAState *s, int i)
296 {
297 NICState *nic = qemu_get_nic(s->nc.peer);
298 NetClientState *nc_i = qemu_get_peer(nic->ncs, i);
299
300 return DO_UPCAST(VhostVDPAState, nc, nc_i);
301 }
302
vhost_vdpa_net_first_nc_vdpa(VhostVDPAState * s)303 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
304 {
305 return vhost_vdpa_net_get_nc_vdpa(s, 0);
306 }
307
vhost_vdpa_net_log_global_enable(VhostVDPAState * s,bool enable)308 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
309 {
310 struct vhost_vdpa *v = &s->vhost_vdpa;
311 VirtIONet *n;
312 VirtIODevice *vdev;
313 int data_queue_pairs, cvq, r;
314
315 /* We are only called on the first data vqs and only if x-svq is not set */
316 if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
317 return;
318 }
319
320 vdev = v->dev->vdev;
321 n = VIRTIO_NET(vdev);
322 if (!n->vhost_started) {
323 return;
324 }
325
326 data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
327 cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
328 n->max_ncs - n->max_queue_pairs : 0;
329 v->shared->svq_switching = enable ?
330 SVQ_TSTATE_ENABLING : SVQ_TSTATE_DISABLING;
331 /*
332 * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
333 * in the future and resume the device if read-only operations between
334 * suspend and reset goes wrong.
335 */
336 vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
337
338 /* Start will check migration setup_or_active to configure or not SVQ */
339 r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
340 if (unlikely(r < 0)) {
341 error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
342 }
343 v->shared->svq_switching = SVQ_TSTATE_DONE;
344 }
345
vdpa_net_migration_state_notifier(NotifierWithReturn * notifier,MigrationEvent * e,Error ** errp)346 static int vdpa_net_migration_state_notifier(NotifierWithReturn *notifier,
347 MigrationEvent *e, Error **errp)
348 {
349 VhostVDPAState *s = container_of(notifier, VhostVDPAState, migration_state);
350
351 if (e->type == MIG_EVENT_PRECOPY_SETUP) {
352 vhost_vdpa_net_log_global_enable(s, true);
353 } else if (e->type == MIG_EVENT_PRECOPY_FAILED) {
354 vhost_vdpa_net_log_global_enable(s, false);
355 }
356 return 0;
357 }
358
vhost_vdpa_net_data_start_first(VhostVDPAState * s)359 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
360 {
361 struct vhost_vdpa *v = &s->vhost_vdpa;
362
363 migration_add_notifier(&s->migration_state,
364 vdpa_net_migration_state_notifier);
365 if (v->shadow_vqs_enabled) {
366 v->shared->iova_tree = vhost_iova_tree_new(v->shared->iova_range.first,
367 v->shared->iova_range.last);
368 }
369 }
370
vhost_vdpa_net_data_start(NetClientState * nc)371 static int vhost_vdpa_net_data_start(NetClientState *nc)
372 {
373 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
374 struct vhost_vdpa *v = &s->vhost_vdpa;
375
376 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
377
378 if (s->always_svq ||
379 migration_is_setup_or_active()) {
380 v->shadow_vqs_enabled = true;
381 } else {
382 v->shadow_vqs_enabled = false;
383 }
384
385 if (v->index == 0) {
386 v->shared->shadow_data = v->shadow_vqs_enabled;
387 vhost_vdpa_net_data_start_first(s);
388 return 0;
389 }
390
391 return 0;
392 }
393
vhost_vdpa_net_data_load(NetClientState * nc)394 static int vhost_vdpa_net_data_load(NetClientState *nc)
395 {
396 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
397 struct vhost_vdpa *v = &s->vhost_vdpa;
398 bool has_cvq = v->dev->vq_index_end % 2;
399
400 if (has_cvq) {
401 return 0;
402 }
403
404 for (int i = 0; i < v->dev->nvqs; ++i) {
405 int ret = vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index);
406 if (ret < 0) {
407 return ret;
408 }
409 }
410 return 0;
411 }
412
vhost_vdpa_net_client_stop(NetClientState * nc)413 static void vhost_vdpa_net_client_stop(NetClientState *nc)
414 {
415 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
416 struct vhost_dev *dev;
417
418 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
419
420 if (s->vhost_vdpa.index == 0) {
421 migration_remove_notifier(&s->migration_state);
422 }
423
424 dev = s->vhost_vdpa.dev;
425 if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
426 g_clear_pointer(&s->vhost_vdpa.shared->iova_tree,
427 vhost_iova_tree_delete);
428 }
429 }
430
431 static NetClientInfo net_vhost_vdpa_info = {
432 .type = NET_CLIENT_DRIVER_VHOST_VDPA,
433 .size = sizeof(VhostVDPAState),
434 .receive = vhost_vdpa_receive,
435 .start = vhost_vdpa_net_data_start,
436 .load = vhost_vdpa_net_data_load,
437 .stop = vhost_vdpa_net_client_stop,
438 .cleanup = vhost_vdpa_cleanup,
439 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
440 .has_ufo = vhost_vdpa_has_ufo,
441 .check_peer_type = vhost_vdpa_check_peer_type,
442 .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
443 };
444
vhost_vdpa_get_vring_group(int device_fd,unsigned vq_index,Error ** errp)445 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
446 Error **errp)
447 {
448 struct vhost_vring_state state = {
449 .index = vq_index,
450 };
451 int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
452
453 if (unlikely(r < 0)) {
454 r = -errno;
455 error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index);
456 return r;
457 }
458
459 return state.num;
460 }
461
vhost_vdpa_set_address_space_id(struct vhost_vdpa * v,unsigned vq_group,unsigned asid_num)462 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
463 unsigned vq_group,
464 unsigned asid_num)
465 {
466 struct vhost_vring_state asid = {
467 .index = vq_group,
468 .num = asid_num,
469 };
470 int r;
471
472 trace_vhost_vdpa_set_address_space_id(v, vq_group, asid_num);
473
474 r = ioctl(v->shared->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
475 if (unlikely(r < 0)) {
476 error_report("Can't set vq group %u asid %u, errno=%d (%s)",
477 asid.index, asid.num, errno, g_strerror(errno));
478 }
479 return r;
480 }
481
vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa * v,void * addr)482 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
483 {
484 VhostIOVATree *tree = v->shared->iova_tree;
485 DMAMap needle = {
486 /*
487 * No need to specify size or to look for more translations since
488 * this contiguous chunk was allocated by us.
489 */
490 .translated_addr = (hwaddr)(uintptr_t)addr,
491 };
492 const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
493 int r;
494
495 if (unlikely(!map)) {
496 error_report("Cannot locate expected map");
497 return;
498 }
499
500 r = vhost_vdpa_dma_unmap(v->shared, v->address_space_id, map->iova,
501 map->size + 1);
502 if (unlikely(r != 0)) {
503 error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
504 }
505
506 vhost_iova_tree_remove(tree, *map);
507 }
508
509 /** Map CVQ buffer. */
vhost_vdpa_cvq_map_buf(struct vhost_vdpa * v,void * buf,size_t size,bool write)510 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
511 bool write)
512 {
513 DMAMap map = {};
514 int r;
515
516 map.translated_addr = (hwaddr)(uintptr_t)buf;
517 map.size = size - 1;
518 map.perm = write ? IOMMU_RW : IOMMU_RO,
519 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &map);
520 if (unlikely(r != IOVA_OK)) {
521 error_report("Cannot map injected element");
522 return r;
523 }
524
525 r = vhost_vdpa_dma_map(v->shared, v->address_space_id, map.iova,
526 vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
527 if (unlikely(r < 0)) {
528 goto dma_map_err;
529 }
530
531 return 0;
532
533 dma_map_err:
534 vhost_iova_tree_remove(v->shared->iova_tree, map);
535 return r;
536 }
537
vhost_vdpa_net_cvq_start(NetClientState * nc)538 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
539 {
540 VhostVDPAState *s, *s0;
541 struct vhost_vdpa *v;
542 int64_t cvq_group;
543 int r;
544 Error *err = NULL;
545
546 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
547
548 s = DO_UPCAST(VhostVDPAState, nc, nc);
549 v = &s->vhost_vdpa;
550
551 s0 = vhost_vdpa_net_first_nc_vdpa(s);
552 v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled;
553 s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
554
555 if (v->shared->shadow_data) {
556 /* SVQ is already configured for all virtqueues */
557 goto out;
558 }
559
560 /*
561 * If we early return in these cases SVQ will not be enabled. The migration
562 * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
563 */
564 if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) {
565 return 0;
566 }
567
568 if (!s->cvq_isolated) {
569 return 0;
570 }
571
572 cvq_group = vhost_vdpa_get_vring_group(v->shared->device_fd,
573 v->dev->vq_index_end - 1,
574 &err);
575 if (unlikely(cvq_group < 0)) {
576 error_report_err(err);
577 return cvq_group;
578 }
579
580 r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
581 if (unlikely(r < 0)) {
582 return r;
583 }
584
585 v->shadow_vqs_enabled = true;
586 s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
587
588 out:
589 if (!s->vhost_vdpa.shadow_vqs_enabled) {
590 return 0;
591 }
592
593 /*
594 * If other vhost_vdpa already have an iova_tree, reuse it for simplicity,
595 * whether CVQ shares ASID with guest or not, because:
596 * - Memory listener need access to guest's memory addresses allocated in
597 * the IOVA tree.
598 * - There should be plenty of IOVA address space for both ASID not to
599 * worry about collisions between them. Guest's translations are still
600 * validated with virtio virtqueue_pop so there is no risk for the guest
601 * to access memory that it shouldn't.
602 *
603 * To allocate a iova tree per ASID is doable but it complicates the code
604 * and it is not worth it for the moment.
605 */
606 if (!v->shared->iova_tree) {
607 v->shared->iova_tree = vhost_iova_tree_new(v->shared->iova_range.first,
608 v->shared->iova_range.last);
609 }
610
611 r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
612 vhost_vdpa_net_cvq_cmd_page_len(), false);
613 if (unlikely(r < 0)) {
614 return r;
615 }
616
617 r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
618 vhost_vdpa_net_cvq_cmd_page_len(), true);
619 if (unlikely(r < 0)) {
620 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
621 }
622
623 return r;
624 }
625
vhost_vdpa_net_cvq_stop(NetClientState * nc)626 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
627 {
628 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
629
630 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
631
632 if (s->vhost_vdpa.shadow_vqs_enabled) {
633 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
634 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
635 }
636
637 vhost_vdpa_net_client_stop(nc);
638 }
639
vhost_vdpa_net_cvq_add(VhostVDPAState * s,const struct iovec * out_sg,size_t out_num,const struct iovec * in_sg,size_t in_num)640 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s,
641 const struct iovec *out_sg, size_t out_num,
642 const struct iovec *in_sg, size_t in_num)
643 {
644 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
645 int r;
646
647 r = vhost_svq_add(svq, out_sg, out_num, in_sg, in_num, NULL);
648 if (unlikely(r != 0)) {
649 if (unlikely(r == -ENOSPC)) {
650 qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
651 __func__);
652 }
653 }
654
655 return r;
656 }
657
658 /*
659 * Convenience wrapper to poll SVQ for multiple control commands.
660 *
661 * Caller should hold the BQL when invoking this function, and should take
662 * the answer before SVQ pulls by itself when BQL is released.
663 */
vhost_vdpa_net_svq_poll(VhostVDPAState * s,size_t cmds_in_flight)664 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight)
665 {
666 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
667 return vhost_svq_poll(svq, cmds_in_flight);
668 }
669
vhost_vdpa_net_load_cursor_reset(VhostVDPAState * s,struct iovec * out_cursor,struct iovec * in_cursor)670 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s,
671 struct iovec *out_cursor,
672 struct iovec *in_cursor)
673 {
674 /* reset the cursor of the output buffer for the device */
675 out_cursor->iov_base = s->cvq_cmd_out_buffer;
676 out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
677
678 /* reset the cursor of the in buffer for the device */
679 in_cursor->iov_base = s->status;
680 in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
681 }
682
683 /*
684 * Poll SVQ for multiple pending control commands and check the device's ack.
685 *
686 * Caller should hold the BQL when invoking this function.
687 *
688 * @s: The VhostVDPAState
689 * @len: The length of the pending status shadow buffer
690 */
vhost_vdpa_net_svq_flush(VhostVDPAState * s,size_t len)691 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len)
692 {
693 /* device uses a one-byte length ack for each control command */
694 ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len);
695 if (unlikely(dev_written != len)) {
696 return -EIO;
697 }
698
699 /* check the device's ack */
700 for (int i = 0; i < len; ++i) {
701 if (s->status[i] != VIRTIO_NET_OK) {
702 return -EIO;
703 }
704 }
705 return 0;
706 }
707
vhost_vdpa_net_load_cmd(VhostVDPAState * s,struct iovec * out_cursor,struct iovec * in_cursor,uint8_t class,uint8_t cmd,const struct iovec * data_sg,size_t data_num)708 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s,
709 struct iovec *out_cursor,
710 struct iovec *in_cursor, uint8_t class,
711 uint8_t cmd, const struct iovec *data_sg,
712 size_t data_num)
713 {
714 const struct virtio_net_ctrl_hdr ctrl = {
715 .class = class,
716 .cmd = cmd,
717 };
718 size_t data_size = iov_size(data_sg, data_num), cmd_size;
719 struct iovec out, in;
720 ssize_t r;
721 unsigned dummy_cursor_iov_cnt;
722 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
723
724 assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
725 cmd_size = sizeof(ctrl) + data_size;
726 trace_vhost_vdpa_net_load_cmd(s, class, cmd, data_num, data_size);
727 if (vhost_svq_available_slots(svq) < 2 ||
728 iov_size(out_cursor, 1) < cmd_size) {
729 /*
730 * It is time to flush all pending control commands if SVQ is full
731 * or control commands shadow buffers are full.
732 *
733 * We can poll here since we've had BQL from the time
734 * we sent the descriptor.
735 */
736 r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base -
737 (void *)s->status);
738 if (unlikely(r < 0)) {
739 return r;
740 }
741
742 vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor);
743 }
744
745 /* pack the CVQ command header */
746 iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl));
747 /* pack the CVQ command command-specific-data */
748 iov_to_buf(data_sg, data_num, 0,
749 out_cursor->iov_base + sizeof(ctrl), data_size);
750
751 /* extract the required buffer from the cursor for output */
752 iov_copy(&out, 1, out_cursor, 1, 0, cmd_size);
753 /* extract the required buffer from the cursor for input */
754 iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status));
755
756 r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1);
757 if (unlikely(r < 0)) {
758 trace_vhost_vdpa_net_load_cmd_retval(s, class, cmd, r);
759 return r;
760 }
761
762 /* iterate the cursors */
763 dummy_cursor_iov_cnt = 1;
764 iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size);
765 dummy_cursor_iov_cnt = 1;
766 iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status));
767
768 return 0;
769 }
770
vhost_vdpa_net_load_mac(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor)771 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n,
772 struct iovec *out_cursor,
773 struct iovec *in_cursor)
774 {
775 if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
776 const struct iovec data = {
777 .iov_base = (void *)n->mac,
778 .iov_len = sizeof(n->mac),
779 };
780 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
781 VIRTIO_NET_CTRL_MAC,
782 VIRTIO_NET_CTRL_MAC_ADDR_SET,
783 &data, 1);
784 if (unlikely(r < 0)) {
785 return r;
786 }
787 }
788
789 /*
790 * According to VirtIO standard, "The device MUST have an
791 * empty MAC filtering table on reset.".
792 *
793 * Therefore, there is no need to send this CVQ command if the
794 * driver also sets an empty MAC filter table, which aligns with
795 * the device's defaults.
796 *
797 * Note that the device's defaults can mismatch the driver's
798 * configuration only at live migration.
799 */
800 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) ||
801 n->mac_table.in_use == 0) {
802 return 0;
803 }
804
805 uint32_t uni_entries = n->mac_table.first_multi,
806 uni_macs_size = uni_entries * ETH_ALEN,
807 mul_entries = n->mac_table.in_use - uni_entries,
808 mul_macs_size = mul_entries * ETH_ALEN;
809 struct virtio_net_ctrl_mac uni = {
810 .entries = cpu_to_le32(uni_entries),
811 };
812 struct virtio_net_ctrl_mac mul = {
813 .entries = cpu_to_le32(mul_entries),
814 };
815 const struct iovec data[] = {
816 {
817 .iov_base = &uni,
818 .iov_len = sizeof(uni),
819 }, {
820 .iov_base = n->mac_table.macs,
821 .iov_len = uni_macs_size,
822 }, {
823 .iov_base = &mul,
824 .iov_len = sizeof(mul),
825 }, {
826 .iov_base = &n->mac_table.macs[uni_macs_size],
827 .iov_len = mul_macs_size,
828 },
829 };
830 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
831 VIRTIO_NET_CTRL_MAC,
832 VIRTIO_NET_CTRL_MAC_TABLE_SET,
833 data, ARRAY_SIZE(data));
834 if (unlikely(r < 0)) {
835 return r;
836 }
837
838 return 0;
839 }
840
vhost_vdpa_net_load_rss(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor,bool do_rss)841 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
842 struct iovec *out_cursor,
843 struct iovec *in_cursor, bool do_rss)
844 {
845 struct virtio_net_rss_config cfg = {};
846 ssize_t r;
847 g_autofree uint16_t *table = NULL;
848
849 /*
850 * According to VirtIO standard, "Initially the device has all hash
851 * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
852 *
853 * Therefore, there is no need to send this CVQ command if the
854 * driver disables the all hash types, which aligns with
855 * the device's defaults.
856 *
857 * Note that the device's defaults can mismatch the driver's
858 * configuration only at live migration.
859 */
860 if (!n->rss_data.enabled ||
861 n->rss_data.hash_types == VIRTIO_NET_HASH_REPORT_NONE) {
862 return 0;
863 }
864
865 table = g_malloc_n(n->rss_data.indirections_len,
866 sizeof(n->rss_data.indirections_table[0]));
867 cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
868
869 if (do_rss) {
870 /*
871 * According to VirtIO standard, "Number of entries in indirection_table
872 * is (indirection_table_mask + 1)".
873 */
874 cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len -
875 1);
876 cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue);
877 for (int i = 0; i < n->rss_data.indirections_len; ++i) {
878 table[i] = cpu_to_le16(n->rss_data.indirections_table[i]);
879 }
880 cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs);
881 } else {
882 /*
883 * According to VirtIO standard, "Field reserved MUST contain zeroes.
884 * It is defined to make the structure to match the layout of
885 * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
886 *
887 * Therefore, we need to zero the fields in
888 * struct virtio_net_rss_config, which corresponds to the
889 * `reserved` field in struct virtio_net_hash_config.
890 *
891 * Note that all other fields are zeroed at their definitions,
892 * except for the `indirection_table` field, where the actual data
893 * is stored in the `table` variable to ensure compatibility
894 * with RSS case. Therefore, we need to zero the `table` variable here.
895 */
896 table[0] = 0;
897 }
898
899 /*
900 * Considering that virtio_net_handle_rss() currently does not restore
901 * the hash key length parsed from the CVQ command sent from the guest
902 * into n->rss_data and uses the maximum key length in other code, so
903 * we also employ the maximum key length here.
904 */
905 cfg.hash_key_length = sizeof(n->rss_data.key);
906
907 const struct iovec data[] = {
908 {
909 .iov_base = &cfg,
910 .iov_len = offsetof(struct virtio_net_rss_config,
911 indirection_table),
912 }, {
913 .iov_base = table,
914 .iov_len = n->rss_data.indirections_len *
915 sizeof(n->rss_data.indirections_table[0]),
916 }, {
917 .iov_base = &cfg.max_tx_vq,
918 .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) -
919 offsetof(struct virtio_net_rss_config, max_tx_vq),
920 }, {
921 .iov_base = (void *)n->rss_data.key,
922 .iov_len = sizeof(n->rss_data.key),
923 }
924 };
925
926 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
927 VIRTIO_NET_CTRL_MQ,
928 do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG :
929 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
930 data, ARRAY_SIZE(data));
931 if (unlikely(r < 0)) {
932 return r;
933 }
934
935 return 0;
936 }
937
vhost_vdpa_net_load_mq(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor)938 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
939 const VirtIONet *n,
940 struct iovec *out_cursor,
941 struct iovec *in_cursor)
942 {
943 struct virtio_net_ctrl_mq mq;
944 ssize_t r;
945
946 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) {
947 return 0;
948 }
949
950 trace_vhost_vdpa_net_load_mq(s, n->curr_queue_pairs);
951
952 mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
953 const struct iovec data = {
954 .iov_base = &mq,
955 .iov_len = sizeof(mq),
956 };
957 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
958 VIRTIO_NET_CTRL_MQ,
959 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
960 &data, 1);
961 if (unlikely(r < 0)) {
962 return r;
963 }
964
965 if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
966 /* load the receive-side scaling state */
967 r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true);
968 if (unlikely(r < 0)) {
969 return r;
970 }
971 } else if (virtio_vdev_has_feature(&n->parent_obj,
972 VIRTIO_NET_F_HASH_REPORT)) {
973 /* load the hash calculation state */
974 r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
975 if (unlikely(r < 0)) {
976 return r;
977 }
978 }
979
980 return 0;
981 }
982
vhost_vdpa_net_load_offloads(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor)983 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s,
984 const VirtIONet *n,
985 struct iovec *out_cursor,
986 struct iovec *in_cursor)
987 {
988 uint64_t offloads;
989 ssize_t r;
990
991 if (!virtio_vdev_has_feature(&n->parent_obj,
992 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
993 return 0;
994 }
995
996 if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) {
997 /*
998 * According to VirtIO standard, "Upon feature negotiation
999 * corresponding offload gets enabled to preserve
1000 * backward compatibility.".
1001 *
1002 * Therefore, there is no need to send this CVQ command if the
1003 * driver also enables all supported offloads, which aligns with
1004 * the device's defaults.
1005 *
1006 * Note that the device's defaults can mismatch the driver's
1007 * configuration only at live migration.
1008 */
1009 return 0;
1010 }
1011
1012 offloads = cpu_to_le64(n->curr_guest_offloads);
1013 const struct iovec data = {
1014 .iov_base = &offloads,
1015 .iov_len = sizeof(offloads),
1016 };
1017 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1018 VIRTIO_NET_CTRL_GUEST_OFFLOADS,
1019 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET,
1020 &data, 1);
1021 if (unlikely(r < 0)) {
1022 return r;
1023 }
1024
1025 return 0;
1026 }
1027
vhost_vdpa_net_load_rx_mode(VhostVDPAState * s,struct iovec * out_cursor,struct iovec * in_cursor,uint8_t cmd,uint8_t on)1028 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s,
1029 struct iovec *out_cursor,
1030 struct iovec *in_cursor,
1031 uint8_t cmd,
1032 uint8_t on)
1033 {
1034 const struct iovec data = {
1035 .iov_base = &on,
1036 .iov_len = sizeof(on),
1037 };
1038 ssize_t r;
1039
1040 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1041 VIRTIO_NET_CTRL_RX, cmd, &data, 1);
1042 if (unlikely(r < 0)) {
1043 return r;
1044 }
1045
1046 return 0;
1047 }
1048
vhost_vdpa_net_load_rx(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor)1049 static int vhost_vdpa_net_load_rx(VhostVDPAState *s,
1050 const VirtIONet *n,
1051 struct iovec *out_cursor,
1052 struct iovec *in_cursor)
1053 {
1054 ssize_t r;
1055
1056 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) {
1057 return 0;
1058 }
1059
1060 /*
1061 * According to virtio_net_reset(), device turns promiscuous mode
1062 * on by default.
1063 *
1064 * Additionally, according to VirtIO standard, "Since there are
1065 * no guarantees, it can use a hash filter or silently switch to
1066 * allmulti or promiscuous mode if it is given too many addresses.".
1067 * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1068 * non-multicast MAC addresses, indicating that promiscuous mode
1069 * should be enabled.
1070 *
1071 * Therefore, QEMU should only send this CVQ command if the
1072 * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1073 * which sets promiscuous mode on, different from the device's defaults.
1074 *
1075 * Note that the device's defaults can mismatch the driver's
1076 * configuration only at live migration.
1077 */
1078 if (!n->mac_table.uni_overflow && !n->promisc) {
1079 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1080 VIRTIO_NET_CTRL_RX_PROMISC, 0);
1081 if (unlikely(r < 0)) {
1082 return r;
1083 }
1084 }
1085
1086 /*
1087 * According to virtio_net_reset(), device turns all-multicast mode
1088 * off by default.
1089 *
1090 * According to VirtIO standard, "Since there are no guarantees,
1091 * it can use a hash filter or silently switch to allmulti or
1092 * promiscuous mode if it is given too many addresses.". QEMU marks
1093 * `n->mac_table.multi_overflow` if guest sets too many
1094 * non-multicast MAC addresses.
1095 *
1096 * Therefore, QEMU should only send this CVQ command if the
1097 * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1098 * which sets all-multicast mode on, different from the device's defaults.
1099 *
1100 * Note that the device's defaults can mismatch the driver's
1101 * configuration only at live migration.
1102 */
1103 if (n->mac_table.multi_overflow || n->allmulti) {
1104 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1105 VIRTIO_NET_CTRL_RX_ALLMULTI, 1);
1106 if (unlikely(r < 0)) {
1107 return r;
1108 }
1109 }
1110
1111 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) {
1112 return 0;
1113 }
1114
1115 /*
1116 * According to virtio_net_reset(), device turns all-unicast mode
1117 * off by default.
1118 *
1119 * Therefore, QEMU should only send this CVQ command if the driver
1120 * sets all-unicast mode on, different from the device's defaults.
1121 *
1122 * Note that the device's defaults can mismatch the driver's
1123 * configuration only at live migration.
1124 */
1125 if (n->alluni) {
1126 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1127 VIRTIO_NET_CTRL_RX_ALLUNI, 1);
1128 if (r < 0) {
1129 return r;
1130 }
1131 }
1132
1133 /*
1134 * According to virtio_net_reset(), device turns non-multicast mode
1135 * off by default.
1136 *
1137 * Therefore, QEMU should only send this CVQ command if the driver
1138 * sets non-multicast mode on, different from the device's defaults.
1139 *
1140 * Note that the device's defaults can mismatch the driver's
1141 * configuration only at live migration.
1142 */
1143 if (n->nomulti) {
1144 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1145 VIRTIO_NET_CTRL_RX_NOMULTI, 1);
1146 if (r < 0) {
1147 return r;
1148 }
1149 }
1150
1151 /*
1152 * According to virtio_net_reset(), device turns non-unicast mode
1153 * off by default.
1154 *
1155 * Therefore, QEMU should only send this CVQ command if the driver
1156 * sets non-unicast mode on, different from the device's defaults.
1157 *
1158 * Note that the device's defaults can mismatch the driver's
1159 * configuration only at live migration.
1160 */
1161 if (n->nouni) {
1162 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1163 VIRTIO_NET_CTRL_RX_NOUNI, 1);
1164 if (r < 0) {
1165 return r;
1166 }
1167 }
1168
1169 /*
1170 * According to virtio_net_reset(), device turns non-broadcast mode
1171 * off by default.
1172 *
1173 * Therefore, QEMU should only send this CVQ command if the driver
1174 * sets non-broadcast mode on, different from the device's defaults.
1175 *
1176 * Note that the device's defaults can mismatch the driver's
1177 * configuration only at live migration.
1178 */
1179 if (n->nobcast) {
1180 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1181 VIRTIO_NET_CTRL_RX_NOBCAST, 1);
1182 if (r < 0) {
1183 return r;
1184 }
1185 }
1186
1187 return 0;
1188 }
1189
vhost_vdpa_net_load_single_vlan(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor,uint16_t vid)1190 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s,
1191 const VirtIONet *n,
1192 struct iovec *out_cursor,
1193 struct iovec *in_cursor,
1194 uint16_t vid)
1195 {
1196 const struct iovec data = {
1197 .iov_base = &vid,
1198 .iov_len = sizeof(vid),
1199 };
1200 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1201 VIRTIO_NET_CTRL_VLAN,
1202 VIRTIO_NET_CTRL_VLAN_ADD,
1203 &data, 1);
1204 if (unlikely(r < 0)) {
1205 return r;
1206 }
1207
1208 return 0;
1209 }
1210
vhost_vdpa_net_load_vlan(VhostVDPAState * s,const VirtIONet * n,struct iovec * out_cursor,struct iovec * in_cursor)1211 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s,
1212 const VirtIONet *n,
1213 struct iovec *out_cursor,
1214 struct iovec *in_cursor)
1215 {
1216 int r;
1217
1218 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) {
1219 return 0;
1220 }
1221
1222 for (int i = 0; i < MAX_VLAN >> 5; i++) {
1223 for (int j = 0; n->vlans[i] && j <= 0x1f; j++) {
1224 if (n->vlans[i] & (1U << j)) {
1225 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor,
1226 in_cursor, (i << 5) + j);
1227 if (unlikely(r != 0)) {
1228 return r;
1229 }
1230 }
1231 }
1232 }
1233
1234 return 0;
1235 }
1236
vhost_vdpa_net_cvq_load(NetClientState * nc)1237 static int vhost_vdpa_net_cvq_load(NetClientState *nc)
1238 {
1239 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
1240 struct vhost_vdpa *v = &s->vhost_vdpa;
1241 const VirtIONet *n;
1242 int r;
1243 struct iovec out_cursor, in_cursor;
1244
1245 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1246
1247 r = vhost_vdpa_set_vring_ready(v, v->dev->vq_index);
1248 if (unlikely(r < 0)) {
1249 return r;
1250 }
1251
1252 if (v->shadow_vqs_enabled) {
1253 n = VIRTIO_NET(v->dev->vdev);
1254 vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor);
1255 r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor);
1256 if (unlikely(r < 0)) {
1257 return r;
1258 }
1259 r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor);
1260 if (unlikely(r)) {
1261 return r;
1262 }
1263 r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor);
1264 if (unlikely(r)) {
1265 return r;
1266 }
1267 r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor);
1268 if (unlikely(r)) {
1269 return r;
1270 }
1271 r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor);
1272 if (unlikely(r)) {
1273 return r;
1274 }
1275
1276 /*
1277 * We need to poll and check all pending device's used buffers.
1278 *
1279 * We can poll here since we've had BQL from the time
1280 * we sent the descriptor.
1281 */
1282 r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status);
1283 if (unlikely(r)) {
1284 return r;
1285 }
1286 }
1287
1288 for (int i = 0; i < v->dev->vq_index; ++i) {
1289 r = vhost_vdpa_set_vring_ready(v, i);
1290 if (unlikely(r < 0)) {
1291 return r;
1292 }
1293 }
1294
1295 return 0;
1296 }
1297
1298 static NetClientInfo net_vhost_vdpa_cvq_info = {
1299 .type = NET_CLIENT_DRIVER_VHOST_VDPA,
1300 .size = sizeof(VhostVDPAState),
1301 .receive = vhost_vdpa_receive,
1302 .start = vhost_vdpa_net_cvq_start,
1303 .load = vhost_vdpa_net_cvq_load,
1304 .stop = vhost_vdpa_net_cvq_stop,
1305 .cleanup = vhost_vdpa_cleanup,
1306 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
1307 .has_ufo = vhost_vdpa_has_ufo,
1308 .check_peer_type = vhost_vdpa_check_peer_type,
1309 .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
1310 };
1311
1312 /*
1313 * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1314 * vdpa device.
1315 *
1316 * Considering that QEMU cannot send the entire filter table to the
1317 * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1318 * command to enable promiscuous mode to receive all packets,
1319 * according to VirtIO standard, "Since there are no guarantees,
1320 * it can use a hash filter or silently switch to allmulti or
1321 * promiscuous mode if it is given too many addresses.".
1322 *
1323 * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1324 * marks `n->mac_table.x_overflow` accordingly, it should have
1325 * the same effect on the device model to receive
1326 * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1327 * The same applies to multicast MAC addresses.
1328 *
1329 * Therefore, QEMU can provide the device model with a fake
1330 * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1331 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1332 * MAC addresses. This ensures that the device model marks
1333 * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1334 * allowing all packets to be received, which aligns with the
1335 * state of the vdpa device.
1336 */
vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState * s,VirtQueueElement * elem,struct iovec * out,const struct iovec * in)1337 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s,
1338 VirtQueueElement *elem,
1339 struct iovec *out,
1340 const struct iovec *in)
1341 {
1342 struct virtio_net_ctrl_mac mac_data, *mac_ptr;
1343 struct virtio_net_ctrl_hdr *hdr_ptr;
1344 uint32_t cursor;
1345 ssize_t r;
1346 uint8_t on = 1;
1347
1348 /* parse the non-multicast MAC address entries from CVQ command */
1349 cursor = sizeof(*hdr_ptr);
1350 r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1351 &mac_data, sizeof(mac_data));
1352 if (unlikely(r != sizeof(mac_data))) {
1353 /*
1354 * If the CVQ command is invalid, we should simulate the vdpa device
1355 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1356 */
1357 *s->status = VIRTIO_NET_ERR;
1358 return sizeof(*s->status);
1359 }
1360 cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1361
1362 /* parse the multicast MAC address entries from CVQ command */
1363 r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1364 &mac_data, sizeof(mac_data));
1365 if (r != sizeof(mac_data)) {
1366 /*
1367 * If the CVQ command is invalid, we should simulate the vdpa device
1368 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1369 */
1370 *s->status = VIRTIO_NET_ERR;
1371 return sizeof(*s->status);
1372 }
1373 cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1374
1375 /* validate the CVQ command */
1376 if (iov_size(elem->out_sg, elem->out_num) != cursor) {
1377 /*
1378 * If the CVQ command is invalid, we should simulate the vdpa device
1379 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1380 */
1381 *s->status = VIRTIO_NET_ERR;
1382 return sizeof(*s->status);
1383 }
1384
1385 /*
1386 * According to VirtIO standard, "Since there are no guarantees,
1387 * it can use a hash filter or silently switch to allmulti or
1388 * promiscuous mode if it is given too many addresses.".
1389 *
1390 * Therefore, considering that QEMU is unable to send the entire
1391 * filter table to the vdpa device, it should send the
1392 * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1393 */
1394 hdr_ptr = out->iov_base;
1395 out->iov_len = sizeof(*hdr_ptr) + sizeof(on);
1396
1397 hdr_ptr->class = VIRTIO_NET_CTRL_RX;
1398 hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC;
1399 iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on));
1400 r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1);
1401 if (unlikely(r < 0)) {
1402 return r;
1403 }
1404
1405 /*
1406 * We can poll here since we've had BQL from the time
1407 * we sent the descriptor.
1408 */
1409 r = vhost_vdpa_net_svq_poll(s, 1);
1410 if (unlikely(r < sizeof(*s->status))) {
1411 return r;
1412 }
1413 if (*s->status != VIRTIO_NET_OK) {
1414 return sizeof(*s->status);
1415 }
1416
1417 /*
1418 * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1419 * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1420 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1421 * multicast MAC addresses.
1422 *
1423 * By doing so, the device model can mark `n->mac_table.uni_overflow`
1424 * and `n->mac_table.multi_overflow`, enabling all packets to be
1425 * received, which aligns with the state of the vdpa device.
1426 */
1427 cursor = 0;
1428 uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1,
1429 fake_mul_entries = MAC_TABLE_ENTRIES + 1,
1430 fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) +
1431 sizeof(mac_data) + fake_uni_entries * ETH_ALEN +
1432 sizeof(mac_data) + fake_mul_entries * ETH_ALEN;
1433
1434 assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len());
1435 out->iov_len = fake_cvq_size;
1436
1437 /* pack the header for fake CVQ command */
1438 hdr_ptr = out->iov_base + cursor;
1439 hdr_ptr->class = VIRTIO_NET_CTRL_MAC;
1440 hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1441 cursor += sizeof(*hdr_ptr);
1442
1443 /*
1444 * Pack the non-multicast MAC addresses part for fake CVQ command.
1445 *
1446 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1447 * addresses provided in CVQ command. Therefore, only the entries
1448 * field need to be prepared in the CVQ command.
1449 */
1450 mac_ptr = out->iov_base + cursor;
1451 mac_ptr->entries = cpu_to_le32(fake_uni_entries);
1452 cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN;
1453
1454 /*
1455 * Pack the multicast MAC addresses part for fake CVQ command.
1456 *
1457 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1458 * addresses provided in CVQ command. Therefore, only the entries
1459 * field need to be prepared in the CVQ command.
1460 */
1461 mac_ptr = out->iov_base + cursor;
1462 mac_ptr->entries = cpu_to_le32(fake_mul_entries);
1463
1464 /*
1465 * Simulating QEMU poll a vdpa device used buffer
1466 * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1467 */
1468 return sizeof(*s->status);
1469 }
1470
1471 /**
1472 * Validate and copy control virtqueue commands.
1473 *
1474 * Following QEMU guidelines, we offer a copy of the buffers to the device to
1475 * prevent TOCTOU bugs.
1476 */
vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue * svq,VirtQueueElement * elem,void * opaque)1477 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
1478 VirtQueueElement *elem,
1479 void *opaque)
1480 {
1481 VhostVDPAState *s = opaque;
1482 size_t in_len;
1483 const struct virtio_net_ctrl_hdr *ctrl;
1484 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1485 /* Out buffer sent to both the vdpa device and the device model */
1486 struct iovec out = {
1487 .iov_base = s->cvq_cmd_out_buffer,
1488 };
1489 /* in buffer used for device model */
1490 const struct iovec model_in = {
1491 .iov_base = &status,
1492 .iov_len = sizeof(status),
1493 };
1494 /* in buffer used for vdpa device */
1495 const struct iovec vdpa_in = {
1496 .iov_base = s->status,
1497 .iov_len = sizeof(*s->status),
1498 };
1499 ssize_t dev_written = -EINVAL;
1500
1501 out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
1502 s->cvq_cmd_out_buffer,
1503 vhost_vdpa_net_cvq_cmd_page_len());
1504
1505 ctrl = s->cvq_cmd_out_buffer;
1506 if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) {
1507 /*
1508 * Guest announce capability is emulated by qemu, so don't forward to
1509 * the device.
1510 */
1511 dev_written = sizeof(status);
1512 *s->status = VIRTIO_NET_OK;
1513 } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC &&
1514 ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET &&
1515 iov_size(elem->out_sg, elem->out_num) > out.iov_len)) {
1516 /*
1517 * Due to the size limitation of the out buffer sent to the vdpa device,
1518 * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1519 * MAC addresses set by the driver for the filter table can cause
1520 * truncation of the CVQ command in QEMU. As a result, the vdpa device
1521 * rejects the flawed CVQ command.
1522 *
1523 * Therefore, QEMU must handle this situation instead of sending
1524 * the CVQ command directly.
1525 */
1526 dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem,
1527 &out, &vdpa_in);
1528 if (unlikely(dev_written < 0)) {
1529 goto out;
1530 }
1531 } else {
1532 ssize_t r;
1533 r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1);
1534 if (unlikely(r < 0)) {
1535 dev_written = r;
1536 goto out;
1537 }
1538
1539 /*
1540 * We can poll here since we've had BQL from the time
1541 * we sent the descriptor.
1542 */
1543 dev_written = vhost_vdpa_net_svq_poll(s, 1);
1544 }
1545
1546 if (unlikely(dev_written < sizeof(status))) {
1547 error_report("Insufficient written data (%zu)", dev_written);
1548 goto out;
1549 }
1550
1551 if (*s->status != VIRTIO_NET_OK) {
1552 goto out;
1553 }
1554
1555 status = VIRTIO_NET_ERR;
1556 virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1);
1557 if (status != VIRTIO_NET_OK) {
1558 error_report("Bad CVQ processing in model");
1559 }
1560
1561 out:
1562 in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
1563 sizeof(status));
1564 if (unlikely(in_len < sizeof(status))) {
1565 error_report("Bad device CVQ written length");
1566 }
1567 vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
1568 /*
1569 * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1570 * the function successfully forwards the CVQ command, indicated
1571 * by a non-negative value of `dev_written`. Otherwise, it still
1572 * belongs to SVQ.
1573 * This function should only free the `elem` when it owns.
1574 */
1575 if (dev_written >= 0) {
1576 g_free(elem);
1577 }
1578 return dev_written < 0 ? dev_written : 0;
1579 }
1580
1581 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
1582 .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
1583 };
1584
1585 /**
1586 * Probe if CVQ is isolated
1587 *
1588 * @device_fd The vdpa device fd
1589 * @features Features offered by the device.
1590 * @cvq_index The control vq pair index
1591 *
1592 * Returns <0 in case of failure, 0 if false and 1 if true.
1593 */
vhost_vdpa_probe_cvq_isolation(int device_fd,uint64_t features,int cvq_index,Error ** errp)1594 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features,
1595 int cvq_index, Error **errp)
1596 {
1597 ERRP_GUARD();
1598 uint64_t backend_features;
1599 int64_t cvq_group;
1600 uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE |
1601 VIRTIO_CONFIG_S_DRIVER;
1602 int r;
1603
1604 r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
1605 if (unlikely(r < 0)) {
1606 error_setg_errno(errp, errno, "Cannot get vdpa backend_features");
1607 return r;
1608 }
1609
1610 if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) {
1611 return 0;
1612 }
1613
1614 r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1615 if (unlikely(r)) {
1616 error_setg_errno(errp, -r, "Cannot set device status");
1617 goto out;
1618 }
1619
1620 r = ioctl(device_fd, VHOST_SET_FEATURES, &features);
1621 if (unlikely(r)) {
1622 error_setg_errno(errp, -r, "Cannot set features");
1623 goto out;
1624 }
1625
1626 status |= VIRTIO_CONFIG_S_FEATURES_OK;
1627 r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1628 if (unlikely(r)) {
1629 error_setg_errno(errp, -r, "Cannot set device status");
1630 goto out;
1631 }
1632
1633 cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp);
1634 if (unlikely(cvq_group < 0)) {
1635 if (cvq_group != -ENOTSUP) {
1636 r = cvq_group;
1637 goto out;
1638 }
1639
1640 /*
1641 * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1642 * support ASID even if the parent driver does not. The CVQ cannot be
1643 * isolated in this case.
1644 */
1645 error_free(*errp);
1646 *errp = NULL;
1647 r = 0;
1648 goto out;
1649 }
1650
1651 for (int i = 0; i < cvq_index; ++i) {
1652 int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp);
1653 if (unlikely(group < 0)) {
1654 r = group;
1655 goto out;
1656 }
1657
1658 if (group == (int64_t)cvq_group) {
1659 r = 0;
1660 goto out;
1661 }
1662 }
1663
1664 r = 1;
1665
1666 out:
1667 status = 0;
1668 ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1669 return r;
1670 }
1671
net_vhost_vdpa_init(NetClientState * peer,const char * device,const char * name,int vdpa_device_fd,int queue_pair_index,int nvqs,bool is_datapath,bool svq,struct vhost_vdpa_iova_range iova_range,uint64_t features,VhostVDPAShared * shared,Error ** errp)1672 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
1673 const char *device,
1674 const char *name,
1675 int vdpa_device_fd,
1676 int queue_pair_index,
1677 int nvqs,
1678 bool is_datapath,
1679 bool svq,
1680 struct vhost_vdpa_iova_range iova_range,
1681 uint64_t features,
1682 VhostVDPAShared *shared,
1683 Error **errp)
1684 {
1685 NetClientState *nc = NULL;
1686 VhostVDPAState *s;
1687 int ret = 0;
1688 assert(name);
1689 int cvq_isolated = 0;
1690
1691 if (is_datapath) {
1692 nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
1693 name);
1694 } else {
1695 cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features,
1696 queue_pair_index * 2,
1697 errp);
1698 if (unlikely(cvq_isolated < 0)) {
1699 return NULL;
1700 }
1701
1702 nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
1703 device, name);
1704 }
1705 qemu_set_info_str(nc, TYPE_VHOST_VDPA);
1706 s = DO_UPCAST(VhostVDPAState, nc, nc);
1707
1708 s->vhost_vdpa.index = queue_pair_index;
1709 s->always_svq = svq;
1710 s->migration_state.notify = NULL;
1711 s->vhost_vdpa.shadow_vqs_enabled = svq;
1712 if (queue_pair_index == 0) {
1713 vhost_vdpa_net_valid_svq_features(features,
1714 &s->vhost_vdpa.migration_blocker);
1715 s->vhost_vdpa.shared = g_new0(VhostVDPAShared, 1);
1716 s->vhost_vdpa.shared->device_fd = vdpa_device_fd;
1717 s->vhost_vdpa.shared->iova_range = iova_range;
1718 s->vhost_vdpa.shared->shadow_data = svq;
1719 } else if (!is_datapath) {
1720 s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1721 PROT_READ | PROT_WRITE,
1722 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1723 s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1724 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
1725 -1, 0);
1726
1727 s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
1728 s->vhost_vdpa.shadow_vq_ops_opaque = s;
1729 s->cvq_isolated = cvq_isolated;
1730 }
1731 if (queue_pair_index != 0) {
1732 s->vhost_vdpa.shared = shared;
1733 }
1734
1735 ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
1736 if (ret) {
1737 qemu_del_net_client(nc);
1738 return NULL;
1739 }
1740
1741 return nc;
1742 }
1743
vhost_vdpa_get_features(int fd,uint64_t * features,Error ** errp)1744 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
1745 {
1746 int ret = ioctl(fd, VHOST_GET_FEATURES, features);
1747 if (unlikely(ret < 0)) {
1748 error_setg_errno(errp, errno,
1749 "Fail to query features from vhost-vDPA device");
1750 }
1751 return ret;
1752 }
1753
vhost_vdpa_get_max_queue_pairs(int fd,uint64_t features,int * has_cvq,Error ** errp)1754 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
1755 int *has_cvq, Error **errp)
1756 {
1757 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
1758 g_autofree struct vhost_vdpa_config *config = NULL;
1759 __virtio16 *max_queue_pairs;
1760 int ret;
1761
1762 if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
1763 *has_cvq = 1;
1764 } else {
1765 *has_cvq = 0;
1766 }
1767
1768 if (features & (1 << VIRTIO_NET_F_MQ)) {
1769 config = g_malloc0(config_size + sizeof(*max_queue_pairs));
1770 config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
1771 config->len = sizeof(*max_queue_pairs);
1772
1773 ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
1774 if (ret) {
1775 error_setg(errp, "Fail to get config from vhost-vDPA device");
1776 return -ret;
1777 }
1778
1779 max_queue_pairs = (__virtio16 *)&config->buf;
1780
1781 return lduw_le_p(max_queue_pairs);
1782 }
1783
1784 return 1;
1785 }
1786
net_init_vhost_vdpa(const Netdev * netdev,const char * name,NetClientState * peer,Error ** errp)1787 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
1788 NetClientState *peer, Error **errp)
1789 {
1790 ERRP_GUARD();
1791 const NetdevVhostVDPAOptions *opts;
1792 uint64_t features;
1793 int vdpa_device_fd;
1794 g_autofree NetClientState **ncs = NULL;
1795 struct vhost_vdpa_iova_range iova_range;
1796 NetClientState *nc;
1797 int queue_pairs, r, i = 0, has_cvq = 0;
1798
1799 assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1800 opts = &netdev->u.vhost_vdpa;
1801 if (!opts->vhostdev && !opts->vhostfd) {
1802 error_setg(errp,
1803 "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1804 return -1;
1805 }
1806
1807 if (opts->vhostdev && opts->vhostfd) {
1808 error_setg(errp,
1809 "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1810 return -1;
1811 }
1812
1813 if (opts->vhostdev) {
1814 vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
1815 if (vdpa_device_fd == -1) {
1816 return -errno;
1817 }
1818 } else {
1819 /* has_vhostfd */
1820 vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
1821 if (vdpa_device_fd == -1) {
1822 error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
1823 return -1;
1824 }
1825 }
1826
1827 r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
1828 if (unlikely(r < 0)) {
1829 goto err;
1830 }
1831
1832 queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
1833 &has_cvq, errp);
1834 if (queue_pairs < 0) {
1835 qemu_close(vdpa_device_fd);
1836 return queue_pairs;
1837 }
1838
1839 r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
1840 if (unlikely(r < 0)) {
1841 error_setg(errp, "vhost-vdpa: get iova range failed: %s",
1842 strerror(-r));
1843 goto err;
1844 }
1845
1846 if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
1847 goto err;
1848 }
1849
1850 ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
1851
1852 for (i = 0; i < queue_pairs; i++) {
1853 VhostVDPAShared *shared = NULL;
1854
1855 if (i) {
1856 shared = DO_UPCAST(VhostVDPAState, nc, ncs[0])->vhost_vdpa.shared;
1857 }
1858 ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1859 vdpa_device_fd, i, 2, true, opts->x_svq,
1860 iova_range, features, shared, errp);
1861 if (!ncs[i])
1862 goto err;
1863 }
1864
1865 if (has_cvq) {
1866 VhostVDPAState *s0 = DO_UPCAST(VhostVDPAState, nc, ncs[0]);
1867 VhostVDPAShared *shared = s0->vhost_vdpa.shared;
1868
1869 nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1870 vdpa_device_fd, i, 1, false,
1871 opts->x_svq, iova_range, features, shared,
1872 errp);
1873 if (!nc)
1874 goto err;
1875 }
1876
1877 return 0;
1878
1879 err:
1880 if (i) {
1881 for (i--; i >= 0; i--) {
1882 qemu_del_net_client(ncs[i]);
1883 }
1884 }
1885
1886 qemu_close(vdpa_device_fd);
1887
1888 return -1;
1889 }
1890