Lines Matching +full:- +full:- +full:disable +full:- +full:vhost +full:- +full:kernel
10 * the COPYING file in the top-level directory.
18 #include "qemu/main-loop.h"
24 #include "qemu/error-report.h"
28 #include "qemu/config-file.h"
30 #include "hw/virtio/virtio-net.h"
33 #include "hw/virtio/virtio-bus.h"
35 #include "qapi/qapi-events-net.h"
36 #include "hw/qdev-properties.h"
37 #include "qapi/qapi-types-migration.h"
38 #include "qapi/qapi-events-migration.h"
39 #include "hw/virtio/virtio-access.h"
41 #include "standard-headers/linux/ethtool.h"
49 #include "hw/virtio/vhost.h"
58 /* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */
68 #define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header))
119 return &n->vqs[nc->queue_index]; in virtio_net_get_subqueue()
129 if (!nc->peer) { in flush_or_purge_queued_packets()
133 qemu_flush_or_purge_queued_packets(nc->peer, true); in flush_or_purge_queued_packets()
134 assert(!virtio_net_get_subqueue(nc)->async_tx.elem); in flush_or_purge_queued_packets()
138 * - we could suppress RX interrupt if we were so inclined.
145 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_get_config()
150 virtio_stw_p(vdev, &netcfg.status, n->status); in virtio_net_get_config()
151 virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs); in virtio_net_get_config()
152 virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu); in virtio_net_get_config()
153 memcpy(netcfg.mac, n->mac, ETH_ALEN); in virtio_net_get_config()
154 virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed); in virtio_net_get_config()
155 netcfg.duplex = n->net_conf.duplex; in virtio_net_get_config()
161 n->rss_data.supported_hash_types); in virtio_net_get_config()
162 memcpy(config, &netcfg, n->config_size); in virtio_net_get_config()
168 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { in virtio_net_get_config()
169 ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg, in virtio_net_get_config()
170 n->config_size); in virtio_net_get_config()
171 if (ret == -1) { in virtio_net_get_config()
176 * Some NIC/kernel combinations present 0 as the mac address. As that in virtio_net_get_config()
179 * correctly elsewhere - just not reported by the device. in virtio_net_get_config()
183 memcpy(netcfg.mac, n->mac, ETH_ALEN); in virtio_net_get_config()
187 n->status & VIRTIO_NET_S_ANNOUNCE); in virtio_net_get_config()
188 memcpy(config, &netcfg, n->config_size); in virtio_net_get_config()
196 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_set_config()
198 memcpy(&netcfg, config, n->config_size); in virtio_net_set_config()
202 memcmp(netcfg.mac, n->mac, ETH_ALEN)) { in virtio_net_set_config()
203 memcpy(n->mac, netcfg.mac, ETH_ALEN); in virtio_net_set_config()
204 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); in virtio_net_set_config()
211 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { in virtio_net_set_config()
212 vhost_net_set_config(get_vhost_net(nc->peer), in virtio_net_set_config()
213 (uint8_t *)&netcfg, 0, n->config_size, in virtio_net_set_config()
222 (n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running; in virtio_net_started()
230 net->status |= VIRTIO_NET_S_ANNOUNCE; in virtio_net_announce_notify()
237 trace_virtio_net_announce_timer(n->announce_timer.round); in virtio_net_announce_timer()
239 n->announce_timer.round--; in virtio_net_announce_timer()
253 if (n->announce_timer.round) { in virtio_net_announce()
266 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_vhost_status()
267 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; in virtio_net_vhost_status()
269 n->max_ncs - n->max_queue_pairs : 0; in virtio_net_vhost_status()
271 if (!get_vhost_net(nc->peer)) { in virtio_net_vhost_status()
275 if ((virtio_net_started(n, status) && !nc->peer->link_down) == in virtio_net_vhost_status()
276 !!n->vhost_started) { in virtio_net_vhost_status()
279 if (!n->vhost_started) { in virtio_net_vhost_status()
282 if (n->needs_vnet_hdr_swap) { in virtio_net_vhost_status()
290 * when vhost is running. in virtio_net_vhost_status()
293 NetClientState *qnc = qemu_get_subqueue(n->nic, i); in virtio_net_vhost_status()
296 qemu_net_queue_purge(qnc->peer->incoming_queue, qnc); in virtio_net_vhost_status()
297 qemu_net_queue_purge(qnc->incoming_queue, qnc->peer); in virtio_net_vhost_status()
300 if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) { in virtio_net_vhost_status()
301 r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu); in virtio_net_vhost_status()
304 n->net_conf.mtu); in virtio_net_vhost_status()
310 n->vhost_started = 1; in virtio_net_vhost_status()
311 r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq); in virtio_net_vhost_status()
313 error_report("unable to start vhost net: %d: " in virtio_net_vhost_status()
314 "falling back on userspace virtio", -r); in virtio_net_vhost_status()
315 n->vhost_started = 0; in virtio_net_vhost_status()
318 vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq); in virtio_net_vhost_status()
319 n->vhost_started = 0; in virtio_net_vhost_status()
342 while (--i >= 0) { in virtio_net_set_vnet_endian()
356 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; in virtio_net_vnet_endian_status()
362 * virtio-net code. in virtio_net_vnet_endian_status()
364 n->needs_vnet_hdr_swap = n->has_vnet_hdr && in virtio_net_vnet_endian_status()
365 virtio_net_set_vnet_endian(vdev, n->nic->ncs, in virtio_net_vnet_endian_status()
367 } else if (virtio_net_started(n, vdev->status)) { in virtio_net_vnet_endian_status()
373 virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false); in virtio_net_vnet_endian_status()
395 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_set_status()
396 NetClientState *ncs = qemu_get_subqueue(n->nic, i); in virtio_net_set_status()
398 q = &n->vqs[i]; in virtio_net_set_status()
400 if ((!n->multiqueue && i != 0) || i >= n->curr_queue_pairs) { in virtio_net_set_status()
406 virtio_net_started(n, queue_status) && !n->vhost_started; in virtio_net_set_status()
412 if (!q->tx_waiting) { in virtio_net_set_status()
417 if (q->tx_timer) { in virtio_net_set_status()
418 timer_mod(q->tx_timer, in virtio_net_set_status()
419 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); in virtio_net_set_status()
421 replay_bh_schedule_event(q->tx_bh); in virtio_net_set_status()
424 if (q->tx_timer) { in virtio_net_set_status()
425 timer_del(q->tx_timer); in virtio_net_set_status()
427 qemu_bh_cancel(q->tx_bh); in virtio_net_set_status()
429 if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 && in virtio_net_set_status()
431 vdev->vm_running) { in virtio_net_set_status()
434 q->tx_waiting = 0; in virtio_net_set_status()
435 virtio_queue_set_notification(q->tx_vq, 1); in virtio_net_set_status()
436 virtio_net_drop_tx_queue_data(vdev, q->tx_vq); in virtio_net_set_status()
447 uint16_t old_status = n->status; in virtio_net_set_link_status()
449 if (nc->link_down) in virtio_net_set_link_status()
450 n->status &= ~VIRTIO_NET_S_LINK_UP; in virtio_net_set_link_status()
452 n->status |= VIRTIO_NET_S_LINK_UP; in virtio_net_set_link_status()
454 if (n->status != old_status) in virtio_net_set_link_status()
457 virtio_net_set_status(vdev, vdev->status); in virtio_net_set_link_status()
464 if (nc->rxfilter_notify_enabled) { in rxfilter_notify()
465 char *path = object_get_canonical_path(OBJECT(n->qdev)); in rxfilter_notify()
466 qapi_event_send_nic_rx_filter_changed(n->netclient_name, path); in rxfilter_notify()
469 /* disable event notification to avoid events flooding */ in rxfilter_notify()
470 nc->rxfilter_notify_enabled = 0; in rxfilter_notify()
481 for (j = 0; n->vlans[i] && j <= 0x1f; j++) { in get_vlan_table()
482 if (n->vlans[i] & (1U << j)) { in get_vlan_table()
500 info->name = g_strdup(nc->name); in virtio_net_query_rxfilter()
501 info->promiscuous = n->promisc; in virtio_net_query_rxfilter()
503 if (n->nouni) { in virtio_net_query_rxfilter()
504 info->unicast = RX_STATE_NONE; in virtio_net_query_rxfilter()
505 } else if (n->alluni) { in virtio_net_query_rxfilter()
506 info->unicast = RX_STATE_ALL; in virtio_net_query_rxfilter()
508 info->unicast = RX_STATE_NORMAL; in virtio_net_query_rxfilter()
511 if (n->nomulti) { in virtio_net_query_rxfilter()
512 info->multicast = RX_STATE_NONE; in virtio_net_query_rxfilter()
513 } else if (n->allmulti) { in virtio_net_query_rxfilter()
514 info->multicast = RX_STATE_ALL; in virtio_net_query_rxfilter()
516 info->multicast = RX_STATE_NORMAL; in virtio_net_query_rxfilter()
519 info->broadcast_allowed = n->nobcast; in virtio_net_query_rxfilter()
520 info->multicast_overflow = n->mac_table.multi_overflow; in virtio_net_query_rxfilter()
521 info->unicast_overflow = n->mac_table.uni_overflow; in virtio_net_query_rxfilter()
523 info->main_mac = qemu_mac_strdup_printf(n->mac); in virtio_net_query_rxfilter()
526 for (i = 0; i < n->mac_table.first_multi; i++) { in virtio_net_query_rxfilter()
528 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); in virtio_net_query_rxfilter()
530 info->unicast_table = str_list; in virtio_net_query_rxfilter()
533 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { in virtio_net_query_rxfilter()
535 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); in virtio_net_query_rxfilter()
537 info->multicast_table = str_list; in virtio_net_query_rxfilter()
538 info->vlan_table = get_vlan_table(n); in virtio_net_query_rxfilter()
541 info->vlan = RX_STATE_ALL; in virtio_net_query_rxfilter()
542 } else if (!info->vlan_table) { in virtio_net_query_rxfilter()
543 info->vlan = RX_STATE_NONE; in virtio_net_query_rxfilter()
545 info->vlan = RX_STATE_NORMAL; in virtio_net_query_rxfilter()
549 nc->rxfilter_notify_enabled = 1; in virtio_net_query_rxfilter()
560 if (queue_index >= n->max_queue_pairs * 2) { in virtio_net_queue_reset()
564 nc = qemu_get_subqueue(n->nic, vq2q(queue_index)); in virtio_net_queue_reset()
566 if (!nc->peer) { in virtio_net_queue_reset()
570 if (get_vhost_net(nc->peer) && in virtio_net_queue_reset()
571 nc->peer->info->type == NET_CLIENT_DRIVER_TAP) { in virtio_net_queue_reset()
585 if (queue_index >= n->max_queue_pairs * 2) { in virtio_net_queue_enable()
589 nc = qemu_get_subqueue(n->nic, vq2q(queue_index)); in virtio_net_queue_enable()
591 if (!nc->peer || !vdev->vhost_started) { in virtio_net_queue_enable()
595 if (get_vhost_net(nc->peer) && in virtio_net_queue_enable()
596 nc->peer->info->type == NET_CLIENT_DRIVER_TAP) { in virtio_net_queue_enable()
599 error_report("unable to restart vhost net virtqueue: %d, " in virtio_net_queue_enable()
607 NetClientState *nc = qemu_get_queue(n->nic); in peer_test_vnet_hdr()
608 if (!nc->peer) { in peer_test_vnet_hdr()
612 n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer); in peer_test_vnet_hdr()
617 return n->has_vnet_hdr; in peer_has_vnet_hdr()
625 n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer); in peer_has_ufo()
627 return n->has_ufo; in peer_has_ufo()
636 return qemu_has_uso(qemu_get_queue(n->nic)->peer); in peer_has_uso()
645 n->mergeable_rx_bufs = mergeable_rx_bufs; in virtio_net_set_mrg_rx_bufs()
648 n->guest_hdr_len = hash_report ? in virtio_net_set_mrg_rx_bufs()
651 n->rss_data.populate_hash = !!hash_report; in virtio_net_set_mrg_rx_bufs()
653 n->guest_hdr_len = n->mergeable_rx_bufs ? in virtio_net_set_mrg_rx_bufs()
656 n->rss_data.populate_hash = false; in virtio_net_set_mrg_rx_bufs()
659 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_set_mrg_rx_bufs()
660 nc = qemu_get_subqueue(n->nic, i); in virtio_net_set_mrg_rx_bufs()
663 qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) { in virtio_net_set_mrg_rx_bufs()
664 qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len); in virtio_net_set_mrg_rx_bufs()
665 n->host_hdr_len = n->guest_hdr_len; in virtio_net_set_mrg_rx_bufs()
672 NetClientState *peer = n->nic_conf.peers.ncs[0]; in virtio_net_max_tx_queue_size()
681 if (!net || !net->max_tx_queue_size) { in virtio_net_max_tx_queue_size()
685 return net->max_tx_queue_size; in virtio_net_max_tx_queue_size()
693 NetClientState *nc = qemu_get_subqueue(n->nic, index); in peer_attach()
696 if (!nc->peer) { in peer_attach()
700 net = get_vhost_net(nc->peer); in peer_attach()
701 if (net && net->is_vhost_user) { in peer_attach()
702 vhost_net_set_vring_enable(nc->peer, 1); in peer_attach()
705 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { in peer_attach()
709 if (n->max_queue_pairs == 1) { in peer_attach()
713 return tap_enable(nc->peer); in peer_attach()
718 NetClientState *nc = qemu_get_subqueue(n->nic, index); in peer_detach()
721 if (!nc->peer) { in peer_detach()
725 net = get_vhost_net(nc->peer); in peer_detach()
726 if (net && net->is_vhost_user) { in peer_detach()
727 vhost_net_set_vring_enable(nc->peer, 0); in peer_detach()
730 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { in peer_detach()
734 return tap_disable(nc->peer); in peer_detach()
742 if (n->nic->peer_deleted) { in virtio_net_set_queue_pairs()
746 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_set_queue_pairs()
747 if (i < n->curr_queue_pairs) { in virtio_net_set_queue_pairs()
763 /* Linux kernel 2.6.25. It understood MAC (as everyone must), in virtio_net_bad_features()
776 qemu_set_offload(qemu_get_queue(n->nic)->peer, in virtio_net_apply_guest_offloads()
777 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)), in virtio_net_apply_guest_offloads()
778 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)), in virtio_net_apply_guest_offloads()
779 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)), in virtio_net_apply_guest_offloads()
780 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)), in virtio_net_apply_guest_offloads()
781 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)), in virtio_net_apply_guest_offloads()
782 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO4)), in virtio_net_apply_guest_offloads()
783 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6))); in virtio_net_apply_guest_offloads()
803 return virtio_net_guest_offloads_by_features(vdev->guest_features); in virtio_net_supported_guest_offloads()
828 if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) { in failover_set_primary()
829 fdev->dev = dev; in failover_set_primary()
837 * Find the primary device for this failover virtio-net
862 if (!n->primary_opts) { in failover_add_primary()
864 error_append_hint(errp, "Virtio-net failover will not work. Make " in failover_add_primary()
866 " failover_pair_id=%s\n", n->netclient_name); in failover_add_primary()
870 dev = qdev_device_add_from_qdict(n->primary_opts, in failover_add_primary()
871 n->primary_opts_from_json, in failover_add_primary()
874 qobject_unref(n->primary_opts); in failover_add_primary()
875 n->primary_opts = NULL; in failover_add_primary()
888 if (n->mtu_bypass_backend && in virtio_net_set_features()
889 !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { in virtio_net_set_features()
905 n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && in virtio_net_set_features()
907 n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && in virtio_net_set_features()
909 n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); in virtio_net_set_features()
911 if (n->has_vnet_hdr) { in virtio_net_set_features()
912 n->curr_guest_offloads = in virtio_net_set_features()
917 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_set_features()
918 NetClientState *nc = qemu_get_subqueue(n->nic, i); in virtio_net_set_features()
920 if (!get_vhost_net(nc->peer)) { in virtio_net_set_features()
923 vhost_net_ack_features(get_vhost_net(nc->peer), features); in virtio_net_set_features()
926 * keep acked_features in NetVhostUserState up-to-date so it in virtio_net_set_features()
929 vhost_net_save_acked_features(nc->peer); in virtio_net_set_features()
932 if (virtio_has_feature(vdev->guest_features ^ features, VIRTIO_NET_F_CTRL_VLAN)) { in virtio_net_set_features()
934 memset(n->vlans, vlan ? 0 : 0xff, MAX_VLAN >> 3); in virtio_net_set_features()
938 qapi_event_send_failover_negotiated(n->netclient_name); in virtio_net_set_features()
939 qatomic_set(&n->failover_primary_hidden, false); in virtio_net_set_features()
956 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_handle_rx_mode()
964 n->promisc = on; in virtio_net_handle_rx_mode()
966 n->allmulti = on; in virtio_net_handle_rx_mode()
968 n->alluni = on; in virtio_net_handle_rx_mode()
970 n->nomulti = on; in virtio_net_handle_rx_mode()
972 n->nouni = on; in virtio_net_handle_rx_mode()
974 n->nobcast = on; in virtio_net_handle_rx_mode()
1005 if (!n->has_vnet_hdr) { in virtio_net_handle_offloads()
1009 n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && in virtio_net_handle_offloads()
1011 n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && in virtio_net_handle_offloads()
1020 n->curr_guest_offloads = offloads; in virtio_net_handle_offloads()
1035 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_handle_mac()
1038 if (iov_size(iov, iov_cnt) != sizeof(n->mac)) { in virtio_net_handle_mac()
1041 s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac)); in virtio_net_handle_mac()
1042 assert(s == sizeof(n->mac)); in virtio_net_handle_mac()
1043 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); in virtio_net_handle_mac()
1099 if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) { in virtio_net_handle_mac()
1110 n->mac_table.in_use = in_use; in virtio_net_handle_mac()
1111 n->mac_table.first_multi = first_multi; in virtio_net_handle_mac()
1112 n->mac_table.uni_overflow = uni_overflow; in virtio_net_handle_mac()
1113 n->mac_table.multi_overflow = multi_overflow; in virtio_net_handle_mac()
1114 memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN); in virtio_net_handle_mac()
1131 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_handle_vlan_table()
1143 n->vlans[vid >> 5] |= (1U << (vid & 0x1f)); in virtio_net_handle_vlan_table()
1145 n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f)); in virtio_net_handle_vlan_table()
1157 trace_virtio_net_handle_announce(n->announce_timer.round); in virtio_net_handle_announce()
1159 n->status & VIRTIO_NET_S_ANNOUNCE) { in virtio_net_handle_announce()
1160 n->status &= ~VIRTIO_NET_S_ANNOUNCE; in virtio_net_handle_announce()
1161 if (n->announce_timer.round) { in virtio_net_handle_announce()
1162 qemu_announce_timer_step(&n->announce_timer); in virtio_net_handle_announce()
1173 if (nc == NULL || nc->info->set_steering_ebpf == NULL) { in virtio_net_attach_ebpf_to_backend()
1178 return nc->info->set_steering_ebpf(nc, prog_fd); in virtio_net_attach_ebpf_to_backend()
1184 config->redirect = data->redirect; in rss_data_to_rss_config()
1185 config->populate_hash = data->populate_hash; in rss_data_to_rss_config()
1186 config->hash_types = data->runtime_hash_types; in rss_data_to_rss_config()
1187 config->indirections_len = data->indirections_len; in rss_data_to_rss_config()
1188 config->default_queue = data->default_queue; in rss_data_to_rss_config()
1195 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { in virtio_net_attach_ebpf_rss()
1199 rss_data_to_rss_config(&n->rss_data, &config); in virtio_net_attach_ebpf_rss()
1201 if (!ebpf_rss_set_all(&n->ebpf_rss, &config, in virtio_net_attach_ebpf_rss()
1202 n->rss_data.indirections_table, n->rss_data.key, in virtio_net_attach_ebpf_rss()
1207 if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { in virtio_net_attach_ebpf_rss()
1216 virtio_net_attach_ebpf_to_backend(n->nic, -1); in virtio_net_detach_ebpf_rss()
1221 if (n->rss_data.peer_hash_available) { in virtio_net_commit_rss_config()
1225 if (n->rss_data.enabled) { in virtio_net_commit_rss_config()
1226 n->rss_data.enabled_software_rss = n->rss_data.populate_hash; in virtio_net_commit_rss_config()
1227 if (n->rss_data.populate_hash) { in virtio_net_commit_rss_config()
1230 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { in virtio_net_commit_rss_config()
1231 warn_report("Can't load eBPF RSS for vhost"); in virtio_net_commit_rss_config()
1233 warn_report("Can't load eBPF RSS - fallback to software RSS"); in virtio_net_commit_rss_config()
1234 n->rss_data.enabled_software_rss = true; in virtio_net_commit_rss_config()
1239 n->rss_data.runtime_hash_types, in virtio_net_commit_rss_config()
1240 n->rss_data.indirections_len, in virtio_net_commit_rss_config()
1241 sizeof(n->rss_data.key)); in virtio_net_commit_rss_config()
1250 if (!n->rss_data.enabled) { in virtio_net_disable_rss()
1254 n->rss_data.enabled = false; in virtio_net_disable_rss()
1260 int fds[EBPF_RSS_MAX_FDS] = { [0 ... EBPF_RSS_MAX_FDS - 1] = -1}; in virtio_net_load_ebpf_fds()
1264 if (n->nr_ebpf_rss_fds != EBPF_RSS_MAX_FDS) { in virtio_net_load_ebpf_fds()
1266 EBPF_RSS_MAX_FDS, n->nr_ebpf_rss_fds); in virtio_net_load_ebpf_fds()
1270 for (i = 0; i < n->nr_ebpf_rss_fds; i++) { in virtio_net_load_ebpf_fds()
1271 fds[i] = monitor_fd_param(monitor_cur(), n->ebpf_rss_fds[i], errp); in virtio_net_load_ebpf_fds()
1278 ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3], errp); in virtio_net_load_ebpf_fds()
1282 for (i = 0; i < n->nr_ebpf_rss_fds && fds[i] != -1; i++) { in virtio_net_load_ebpf_fds()
1292 if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { in virtio_net_load_ebpf()
1296 trace_virtio_net_rss_load(n, n->nr_ebpf_rss_fds, n->ebpf_rss_fds); in virtio_net_load_ebpf()
1305 if (n->ebpf_rss_fds) { in virtio_net_load_ebpf()
1309 ebpf_rss_load(&n->ebpf_rss, &error_warn); in virtio_net_load_ebpf()
1315 virtio_net_attach_ebpf_to_backend(n->nic, -1); in virtio_net_unload_ebpf()
1316 ebpf_rss_unload(&n->ebpf_rss); in virtio_net_unload_ebpf()
1350 n->rss_data.runtime_hash_types = virtio_ldl_p(vdev, &cfg.hash_types); in virtio_net_handle_rss()
1351 n->rss_data.indirections_len = in virtio_net_handle_rss()
1354 n->rss_data.indirections_len = 0; in virtio_net_handle_rss()
1356 if (n->rss_data.indirections_len >= VIRTIO_NET_RSS_MAX_TABLE_LEN) { in virtio_net_handle_rss()
1358 err_value = n->rss_data.indirections_len; in virtio_net_handle_rss()
1361 n->rss_data.indirections_len++; in virtio_net_handle_rss()
1362 if (!is_power_of_2(n->rss_data.indirections_len)) { in virtio_net_handle_rss()
1364 err_value = n->rss_data.indirections_len; in virtio_net_handle_rss()
1367 n->rss_data.default_queue = do_rss ? in virtio_net_handle_rss()
1369 if (n->rss_data.default_queue >= n->max_queue_pairs) { in virtio_net_handle_rss()
1371 err_value = n->rss_data.default_queue; in virtio_net_handle_rss()
1375 size_get = sizeof(uint16_t) * n->rss_data.indirections_len; in virtio_net_handle_rss()
1376 g_free(n->rss_data.indirections_table); in virtio_net_handle_rss()
1377 n->rss_data.indirections_table = g_malloc(size_get); in virtio_net_handle_rss()
1378 if (!n->rss_data.indirections_table) { in virtio_net_handle_rss()
1380 err_value = n->rss_data.indirections_len; in virtio_net_handle_rss()
1384 n->rss_data.indirections_table, size_get); in virtio_net_handle_rss()
1390 for (i = 0; i < n->rss_data.indirections_len; ++i) { in virtio_net_handle_rss()
1391 uint16_t val = n->rss_data.indirections_table[i]; in virtio_net_handle_rss()
1392 n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val); in virtio_net_handle_rss()
1402 queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs; in virtio_net_handle_rss()
1403 if (queue_pairs == 0 || queue_pairs > n->max_queue_pairs) { in virtio_net_handle_rss()
1413 if (!temp.b && n->rss_data.runtime_hash_types) { in virtio_net_handle_rss()
1418 if (!temp.b && !n->rss_data.runtime_hash_types) { in virtio_net_handle_rss()
1424 s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get); in virtio_net_handle_rss()
1430 n->rss_data.enabled = true; in virtio_net_handle_rss()
1444 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_handle_mq()
1471 queue_pairs > n->max_queue_pairs || in virtio_net_handle_mq()
1472 !n->multiqueue) { in virtio_net_handle_mq()
1476 n->curr_queue_pairs = queue_pairs; in virtio_net_handle_mq()
1477 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { in virtio_net_handle_mq()
1486 virtio_net_set_status(vdev, vdev->status); in virtio_net_handle_mq()
1505 virtio_error(vdev, "virtio-net ctrl missing headers"); in virtio_net_handle_ctrl_iov()
1546 written = virtio_net_handle_ctrl_iov(vdev, elem->in_sg, elem->in_num, in virtio_net_handle_ctrl()
1547 elem->out_sg, elem->out_num); in virtio_net_handle_ctrl()
1567 qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index)); in virtio_net_handle_rx()
1576 if (!vdev->vm_running) { in virtio_net_can_receive()
1580 if (nc->queue_index >= n->curr_queue_pairs) { in virtio_net_can_receive()
1584 if (!virtio_queue_ready(q->rx_vq) || in virtio_net_can_receive()
1585 !(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { in virtio_net_can_receive()
1596 VirtIONet *n = q->n; in virtio_net_has_buffers()
1598 while (virtio_queue_empty(q->rx_vq) || n->mergeable_rx_bufs) { in virtio_net_has_buffers()
1599 opaque = virtqueue_get_avail_bytes(q->rx_vq, &in_bytes, NULL, in virtio_net_has_buffers()
1601 /* Buffer is enough, disable notifiaction */ in virtio_net_has_buffers()
1606 if (virtio_queue_enable_notification_and_check(q->rx_vq, opaque)) { in virtio_net_has_buffers()
1614 virtio_queue_set_notification(q->rx_vq, 0); in virtio_net_has_buffers()
1621 virtio_tswap16s(vdev, &hdr->hdr_len); in virtio_net_hdr_swap()
1622 virtio_tswap16s(vdev, &hdr->gso_size); in virtio_net_hdr_swap()
1623 virtio_tswap16s(vdev, &hdr->csum_start); in virtio_net_hdr_swap()
1624 virtio_tswap16s(vdev, &hdr->csum_offset); in virtio_net_hdr_swap()
1627 /* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so
1637 * N.B. if we introduce a zero-copy API, this operation is no longer free so
1638 * we should provide a mechanism to disable it to avoid polluting the host
1647 if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */ in work_around_broken_dhclient()
1653 hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM; in work_around_broken_dhclient()
1660 if (n->has_vnet_hdr) { in receive_header()
1663 work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len, in receive_header()
1664 size - n->host_hdr_len); in receive_header()
1666 if (n->needs_vnet_hdr_swap) { in receive_header()
1686 if (n->promisc) in receive_filter()
1689 ptr += n->host_hdr_len; in receive_filter()
1693 if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f)))) in receive_filter()
1699 return !n->nobcast; in receive_filter()
1700 } else if (n->nomulti) { in receive_filter()
1702 } else if (n->allmulti || n->mac_table.multi_overflow) { in receive_filter()
1706 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { in receive_filter()
1707 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { in receive_filter()
1712 if (n->nouni) { in receive_filter()
1714 } else if (n->alluni || n->mac_table.uni_overflow) { in receive_filter()
1716 } else if (!memcmp(ptr, n->mac, ETH_ALEN)) { in receive_filter()
1720 for (i = 0; i < n->mac_table.first_multi; i++) { in receive_filter()
1721 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { in receive_filter()
1795 unsigned int index = nc->queue_index, new_index = index; in virtio_net_process_rss()
1796 struct NetRxPkt *pkt = n->rx_pkt; in virtio_net_process_rss()
1817 net_rx_pkt_set_protocols(pkt, &iov, 1, n->host_hdr_len); in virtio_net_process_rss()
1820 n->rss_data.runtime_hash_types); in virtio_net_process_rss()
1822 if (n->rss_data.populate_hash) { in virtio_net_process_rss()
1823 hdr->hash_value = VIRTIO_NET_HASH_REPORT_NONE; in virtio_net_process_rss()
1824 hdr->hash_report = 0; in virtio_net_process_rss()
1826 return n->rss_data.redirect ? n->rss_data.default_queue : -1; in virtio_net_process_rss()
1829 hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key); in virtio_net_process_rss()
1831 if (n->rss_data.populate_hash) { in virtio_net_process_rss()
1832 hdr->hash_value = hash; in virtio_net_process_rss()
1833 hdr->hash_report = reports[net_hash_type]; in virtio_net_process_rss()
1836 if (n->rss_data.redirect) { in virtio_net_process_rss()
1837 new_index = hash & (n->rss_data.indirections_len - 1); in virtio_net_process_rss()
1838 new_index = n->rss_data.indirections_table[new_index]; in virtio_net_process_rss()
1841 return (index == new_index) ? -1 : new_index; in virtio_net_process_rss()
1860 if (n->rss_data.enabled && n->rss_data.enabled_software_rss) { in virtio_net_receive_rcu()
1863 nc = qemu_get_subqueue(n->nic, index % n->curr_queue_pairs); in virtio_net_receive_rcu()
1868 return -1; in virtio_net_receive_rcu()
1874 if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) { in virtio_net_receive_rcu()
1891 virtio_error(vdev, "virtio-net unexpected long buffer chain"); in virtio_net_receive_rcu()
1896 elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement)); in virtio_net_receive_rcu()
1899 virtio_error(vdev, "virtio-net unexpected empty queue: " in virtio_net_receive_rcu()
1903 i, n->mergeable_rx_bufs, offset, size, in virtio_net_receive_rcu()
1904 n->guest_hdr_len, n->host_hdr_len, in virtio_net_receive_rcu()
1905 vdev->guest_features); in virtio_net_receive_rcu()
1907 err = -1; in virtio_net_receive_rcu()
1911 if (elem->in_num < 1) { in virtio_net_receive_rcu()
1913 "virtio-net receive queue contains no in buffers"); in virtio_net_receive_rcu()
1914 virtqueue_detach_element(q->rx_vq, elem, 0); in virtio_net_receive_rcu()
1916 err = -1; in virtio_net_receive_rcu()
1920 sg = elem->in_sg; in virtio_net_receive_rcu()
1923 if (n->mergeable_rx_bufs) { in virtio_net_receive_rcu()
1925 sg, elem->in_num, in virtio_net_receive_rcu()
1932 receive_header(n, sg, elem->in_num, buf, size); in virtio_net_receive_rcu()
1933 if (n->rss_data.populate_hash) { in virtio_net_receive_rcu()
1935 iov_from_buf(sg, elem->in_num, offset, in virtio_net_receive_rcu()
1940 offset = n->host_hdr_len; in virtio_net_receive_rcu()
1941 total += n->guest_hdr_len; in virtio_net_receive_rcu()
1942 guest_offset = n->guest_hdr_len; in virtio_net_receive_rcu()
1948 len = iov_from_buf(sg, elem->in_num, guest_offset, in virtio_net_receive_rcu()
1949 buf + offset, size - offset); in virtio_net_receive_rcu()
1955 if (!n->mergeable_rx_bufs && offset < size) { in virtio_net_receive_rcu()
1956 virtqueue_unpop(q->rx_vq, elem, total); in virtio_net_receive_rcu()
1977 virtqueue_fill(q->rx_vq, elems[j], lens[j], j); in virtio_net_receive_rcu()
1981 virtqueue_flush(q->rx_vq, i); in virtio_net_receive_rcu()
1982 virtio_notify(vdev, q->rx_vq); in virtio_net_receive_rcu()
1988 virtqueue_detach_element(q->rx_vq, elems[j], lens[j]); in virtio_net_receive_rcu()
2005 * is a potentially unaligned network-byte-order 16 bit unsigned integer
2006 * pointed to by unit->ip_len.
2010 return lduw_be_p(unit->ip_plen); in read_unit_ip_len()
2015 stw_be_p(unit->ip_plen, l); in write_unit_ip_len()
2025 ip = (struct ip_header *)(buf + chain->n->guest_hdr_len in virtio_net_rsc_extract_unit4()
2027 unit->ip = (void *)ip; in virtio_net_rsc_extract_unit4()
2028 ip_hdrlen = (ip->ip_ver_len & 0xF) << 2; in virtio_net_rsc_extract_unit4()
2029 unit->ip_plen = &ip->ip_len; in virtio_net_rsc_extract_unit4()
2030 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) + ip_hdrlen); in virtio_net_rsc_extract_unit4()
2031 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; in virtio_net_rsc_extract_unit4()
2032 unit->payload = read_unit_ip_len(unit) - ip_hdrlen - unit->tcp_hdrlen; in virtio_net_rsc_extract_unit4()
2041 ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len in virtio_net_rsc_extract_unit6()
2043 unit->ip = ip6; in virtio_net_rsc_extract_unit6()
2044 unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); in virtio_net_rsc_extract_unit6()
2045 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) in virtio_net_rsc_extract_unit6()
2047 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; in virtio_net_rsc_extract_unit6()
2051 unit->payload = read_unit_ip_len(unit) - unit->tcp_hdrlen; in virtio_net_rsc_extract_unit6()
2060 h = (struct virtio_net_hdr_v1 *)seg->buf; in virtio_net_rsc_drain_seg()
2061 h->flags = 0; in virtio_net_rsc_drain_seg()
2062 h->gso_type = VIRTIO_NET_HDR_GSO_NONE; in virtio_net_rsc_drain_seg()
2064 if (seg->is_coalesced) { in virtio_net_rsc_drain_seg()
2065 h->rsc.segments = seg->packets; in virtio_net_rsc_drain_seg()
2066 h->rsc.dup_acks = seg->dup_ack; in virtio_net_rsc_drain_seg()
2067 h->flags = VIRTIO_NET_HDR_F_RSC_INFO; in virtio_net_rsc_drain_seg()
2068 if (chain->proto == ETH_P_IP) { in virtio_net_rsc_drain_seg()
2069 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; in virtio_net_rsc_drain_seg()
2071 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; in virtio_net_rsc_drain_seg()
2075 ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size); in virtio_net_rsc_drain_seg()
2076 QTAILQ_REMOVE(&chain->buffers, seg, next); in virtio_net_rsc_drain_seg()
2077 g_free(seg->buf); in virtio_net_rsc_drain_seg()
2088 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) { in virtio_net_rsc_purge()
2090 chain->stat.purge_failed++; in virtio_net_rsc_purge()
2095 chain->stat.timer++; in virtio_net_rsc_purge()
2096 if (!QTAILQ_EMPTY(&chain->buffers)) { in virtio_net_rsc_purge()
2097 timer_mod(chain->drain_timer, in virtio_net_rsc_purge()
2098 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + chain->n->rsc_timeout); in virtio_net_rsc_purge()
2107 QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) { in virtio_net_rsc_cleanup()
2108 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) { in virtio_net_rsc_cleanup()
2109 QTAILQ_REMOVE(&chain->buffers, seg, next); in virtio_net_rsc_cleanup()
2110 g_free(seg->buf); in virtio_net_rsc_cleanup()
2114 timer_free(chain->drain_timer); in virtio_net_rsc_cleanup()
2115 QTAILQ_REMOVE(&n->rsc_chains, chain, next); in virtio_net_rsc_cleanup()
2127 hdr_len = chain->n->guest_hdr_len; in virtio_net_rsc_cache_buf()
2129 seg->buf = g_malloc(hdr_len + sizeof(struct eth_header) in virtio_net_rsc_cache_buf()
2131 memcpy(seg->buf, buf, size); in virtio_net_rsc_cache_buf()
2132 seg->size = size; in virtio_net_rsc_cache_buf()
2133 seg->packets = 1; in virtio_net_rsc_cache_buf()
2134 seg->dup_ack = 0; in virtio_net_rsc_cache_buf()
2135 seg->is_coalesced = 0; in virtio_net_rsc_cache_buf()
2136 seg->nc = nc; in virtio_net_rsc_cache_buf()
2138 QTAILQ_INSERT_TAIL(&chain->buffers, seg, next); in virtio_net_rsc_cache_buf()
2139 chain->stat.cache++; in virtio_net_rsc_cache_buf()
2141 switch (chain->proto) { in virtio_net_rsc_cache_buf()
2143 virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit); in virtio_net_rsc_cache_buf()
2146 virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit); in virtio_net_rsc_cache_buf()
2162 nack = htonl(n_tcp->th_ack); in virtio_net_rsc_handle_ack()
2163 nwin = htons(n_tcp->th_win); in virtio_net_rsc_handle_ack()
2164 oack = htonl(o_tcp->th_ack); in virtio_net_rsc_handle_ack()
2165 owin = htons(o_tcp->th_win); in virtio_net_rsc_handle_ack()
2167 if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) { in virtio_net_rsc_handle_ack()
2168 chain->stat.ack_out_of_win++; in virtio_net_rsc_handle_ack()
2174 chain->stat.dup_ack++; in virtio_net_rsc_handle_ack()
2178 o_tcp->th_win = n_tcp->th_win; in virtio_net_rsc_handle_ack()
2179 chain->stat.win_update++; in virtio_net_rsc_handle_ack()
2184 chain->stat.pure_ack++; in virtio_net_rsc_handle_ack()
2199 o_unit = &seg->unit; in virtio_net_rsc_coalesce_data()
2201 nseq = htonl(n_unit->tcp->th_seq); in virtio_net_rsc_coalesce_data()
2202 oseq = htonl(o_unit->tcp->th_seq); in virtio_net_rsc_coalesce_data()
2205 if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) { in virtio_net_rsc_coalesce_data()
2206 chain->stat.data_out_of_win++; in virtio_net_rsc_coalesce_data()
2210 data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen; in virtio_net_rsc_coalesce_data()
2212 if ((o_unit->payload == 0) && n_unit->payload) { in virtio_net_rsc_coalesce_data()
2214 chain->stat.data_after_pure_ack++; in virtio_net_rsc_coalesce_data()
2218 n_unit->tcp, o_unit->tcp); in virtio_net_rsc_coalesce_data()
2220 } else if ((nseq - oseq) != o_unit->payload) { in virtio_net_rsc_coalesce_data()
2222 chain->stat.data_out_of_order++; in virtio_net_rsc_coalesce_data()
2226 if ((o_ip_len + n_unit->payload) > chain->max_payload) { in virtio_net_rsc_coalesce_data()
2227 chain->stat.over_size++; in virtio_net_rsc_coalesce_data()
2233 o_unit->payload += n_unit->payload; /* update new data len */ in virtio_net_rsc_coalesce_data()
2236 write_unit_ip_len(o_unit, o_ip_len + n_unit->payload); in virtio_net_rsc_coalesce_data()
2241 o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags; in virtio_net_rsc_coalesce_data()
2243 o_unit->tcp->th_ack = n_unit->tcp->th_ack; in virtio_net_rsc_coalesce_data()
2244 o_unit->tcp->th_win = n_unit->tcp->th_win; in virtio_net_rsc_coalesce_data()
2246 memmove(seg->buf + seg->size, data, n_unit->payload); in virtio_net_rsc_coalesce_data()
2247 seg->size += n_unit->payload; in virtio_net_rsc_coalesce_data()
2248 seg->packets++; in virtio_net_rsc_coalesce_data()
2249 chain->stat.coalesced++; in virtio_net_rsc_coalesce_data()
2261 ip1 = (struct ip_header *)(unit->ip); in virtio_net_rsc_coalesce4()
2262 ip2 = (struct ip_header *)(seg->unit.ip); in virtio_net_rsc_coalesce4()
2263 if ((ip1->ip_src ^ ip2->ip_src) || (ip1->ip_dst ^ ip2->ip_dst) in virtio_net_rsc_coalesce4()
2264 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) in virtio_net_rsc_coalesce4()
2265 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { in virtio_net_rsc_coalesce4()
2266 chain->stat.no_match++; in virtio_net_rsc_coalesce4()
2280 ip1 = (struct ip6_header *)(unit->ip); in virtio_net_rsc_coalesce6()
2281 ip2 = (struct ip6_header *)(seg->unit.ip); in virtio_net_rsc_coalesce6()
2282 if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address)) in virtio_net_rsc_coalesce6()
2283 || memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address)) in virtio_net_rsc_coalesce6()
2284 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) in virtio_net_rsc_coalesce6()
2285 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { in virtio_net_rsc_coalesce6()
2286 chain->stat.no_match++; in virtio_net_rsc_coalesce6()
2301 tcp_flag = htons(tcp->th_offset_flags); in virtio_net_rsc_tcp_ctrl_check()
2305 chain->stat.tcp_syn++; in virtio_net_rsc_tcp_ctrl_check()
2310 chain->stat.tcp_ctrl_drain++; in virtio_net_rsc_tcp_ctrl_check()
2315 chain->stat.tcp_all_opt++; in virtio_net_rsc_tcp_ctrl_check()
2330 if (QTAILQ_EMPTY(&chain->buffers)) { in virtio_net_rsc_do_coalesce()
2331 chain->stat.empty_cache++; in virtio_net_rsc_do_coalesce()
2333 timer_mod(chain->drain_timer, in virtio_net_rsc_do_coalesce()
2334 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + chain->n->rsc_timeout); in virtio_net_rsc_do_coalesce()
2338 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { in virtio_net_rsc_do_coalesce()
2339 if (chain->proto == ETH_P_IP) { in virtio_net_rsc_do_coalesce()
2348 chain->stat.final_failed++; in virtio_net_rsc_do_coalesce()
2358 seg->is_coalesced = 1; in virtio_net_rsc_do_coalesce()
2363 chain->stat.no_match_cache++; in virtio_net_rsc_do_coalesce()
2379 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { in virtio_net_rsc_drain_flow()
2380 ppair2 = *(uint32_t *)(seg->buf + tcp_port); in virtio_net_rsc_drain_flow()
2381 if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size) in virtio_net_rsc_drain_flow()
2386 chain->stat.drain_failed++; in virtio_net_rsc_drain_flow()
2402 if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) { in virtio_net_rsc_sanity_check4()
2403 chain->stat.ip_option++; in virtio_net_rsc_sanity_check4()
2408 if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) { in virtio_net_rsc_sanity_check4()
2409 chain->stat.ip_option++; in virtio_net_rsc_sanity_check4()
2413 if (ip->ip_p != IPPROTO_TCP) { in virtio_net_rsc_sanity_check4()
2414 chain->stat.bypass_not_tcp++; in virtio_net_rsc_sanity_check4()
2419 if (!(htons(ip->ip_off) & IP_DF)) { in virtio_net_rsc_sanity_check4()
2420 chain->stat.ip_frag++; in virtio_net_rsc_sanity_check4()
2425 if (IPTOS_ECN(ip->ip_tos)) { in virtio_net_rsc_sanity_check4()
2426 chain->stat.ip_ecn++; in virtio_net_rsc_sanity_check4()
2430 ip_len = htons(ip->ip_len); in virtio_net_rsc_sanity_check4()
2432 || ip_len > (size - chain->n->guest_hdr_len - in virtio_net_rsc_sanity_check4()
2434 chain->stat.ip_hacked++; in virtio_net_rsc_sanity_check4()
2449 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; in virtio_net_rsc_receive4()
2453 chain->stat.bypass_not_tcp++; in virtio_net_rsc_receive4()
2482 if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4) in virtio_net_rsc_sanity_check6()
2488 if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) { in virtio_net_rsc_sanity_check6()
2489 chain->stat.bypass_not_tcp++; in virtio_net_rsc_sanity_check6()
2493 ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); in virtio_net_rsc_sanity_check6()
2495 ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header) in virtio_net_rsc_sanity_check6()
2496 - sizeof(struct ip6_header))) { in virtio_net_rsc_sanity_check6()
2497 chain->stat.ip_hacked++; in virtio_net_rsc_sanity_check6()
2502 if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) { in virtio_net_rsc_sanity_check6()
2503 chain->stat.ip_ecn++; in virtio_net_rsc_sanity_check6()
2519 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; in virtio_net_rsc_receive6()
2556 QTAILQ_FOREACH(chain, &n->rsc_chains, next) { in virtio_net_rsc_lookup_chain()
2557 if (chain->proto == proto) { in virtio_net_rsc_lookup_chain()
2563 chain->n = n; in virtio_net_rsc_lookup_chain()
2564 chain->proto = proto; in virtio_net_rsc_lookup_chain()
2566 chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD; in virtio_net_rsc_lookup_chain()
2567 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; in virtio_net_rsc_lookup_chain()
2569 chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD; in virtio_net_rsc_lookup_chain()
2570 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; in virtio_net_rsc_lookup_chain()
2572 chain->drain_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, in virtio_net_rsc_lookup_chain()
2574 memset(&chain->stat, 0, sizeof(chain->stat)); in virtio_net_rsc_lookup_chain()
2576 QTAILQ_INIT(&chain->buffers); in virtio_net_rsc_lookup_chain()
2577 QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next); in virtio_net_rsc_lookup_chain()
2592 if (size < (n->host_hdr_len + sizeof(struct eth_header))) { in virtio_net_rsc_receive()
2596 eth = (struct eth_header *)(buf + n->guest_hdr_len); in virtio_net_rsc_receive()
2597 proto = htons(eth->h_proto); in virtio_net_rsc_receive()
2601 chain->stat.received++; in virtio_net_rsc_receive()
2602 if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) { in virtio_net_rsc_receive()
2604 } else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) { in virtio_net_rsc_receive()
2615 if ((n->rsc4_enabled || n->rsc6_enabled)) { in virtio_net_receive()
2631 virtqueue_push(q->tx_vq, q->async_tx.elem, 0); in virtio_net_tx_complete()
2632 virtio_notify(vdev, q->tx_vq); in virtio_net_tx_complete()
2634 g_free(q->async_tx.elem); in virtio_net_tx_complete()
2635 q->async_tx.elem = NULL; in virtio_net_tx_complete()
2637 virtio_queue_set_notification(q->tx_vq, 1); in virtio_net_tx_complete()
2639 if (ret >= n->tx_burst) { in virtio_net_tx_complete()
2643 * remainining part, so re-schedule in virtio_net_tx_complete()
2645 virtio_queue_set_notification(q->tx_vq, 0); in virtio_net_tx_complete()
2646 if (q->tx_bh) { in virtio_net_tx_complete()
2647 replay_bh_schedule_event(q->tx_bh); in virtio_net_tx_complete()
2649 timer_mod(q->tx_timer, in virtio_net_tx_complete()
2650 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); in virtio_net_tx_complete()
2652 q->tx_waiting = 1; in virtio_net_tx_complete()
2659 VirtIONet *n = q->n; in virtio_net_flush_tx()
2663 int queue_index = vq2q(virtio_get_queue_index(q->tx_vq)); in virtio_net_flush_tx()
2664 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { in virtio_net_flush_tx()
2668 if (q->async_tx.elem) { in virtio_net_flush_tx()
2669 virtio_queue_set_notification(q->tx_vq, 0); in virtio_net_flush_tx()
2679 elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement)); in virtio_net_flush_tx()
2684 out_num = elem->out_num; in virtio_net_flush_tx()
2685 out_sg = elem->out_sg; in virtio_net_flush_tx()
2687 virtio_error(vdev, "virtio-net header not in first element"); in virtio_net_flush_tx()
2691 if (n->needs_vnet_hdr_swap) { in virtio_net_flush_tx()
2694 virtio_error(vdev, "virtio-net header incorrect"); in virtio_net_flush_tx()
2700 out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1, out_sg, out_num, in virtio_net_flush_tx()
2701 sizeof(vhdr), -1); in virtio_net_flush_tx()
2713 assert(n->host_hdr_len <= n->guest_hdr_len); in virtio_net_flush_tx()
2714 if (n->host_hdr_len != n->guest_hdr_len) { in virtio_net_flush_tx()
2715 if (iov_size(out_sg, out_num) < n->guest_hdr_len) { in virtio_net_flush_tx()
2716 virtio_error(vdev, "virtio-net header is invalid"); in virtio_net_flush_tx()
2721 0, n->host_hdr_len); in virtio_net_flush_tx()
2722 sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num, in virtio_net_flush_tx()
2724 n->guest_hdr_len, -1); in virtio_net_flush_tx()
2729 virtio_error(vdev, "virtio-net nothing to send"); in virtio_net_flush_tx()
2734 ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index), in virtio_net_flush_tx()
2737 virtio_queue_set_notification(q->tx_vq, 0); in virtio_net_flush_tx()
2738 q->async_tx.elem = elem; in virtio_net_flush_tx()
2739 return -EBUSY; in virtio_net_flush_tx()
2743 virtqueue_push(q->tx_vq, elem, 0); in virtio_net_flush_tx()
2744 virtio_notify(vdev, q->tx_vq); in virtio_net_flush_tx()
2747 if (++num_packets >= n->tx_burst) { in virtio_net_flush_tx()
2754 virtqueue_detach_element(q->tx_vq, elem, 0); in virtio_net_flush_tx()
2756 return -EINVAL; in virtio_net_flush_tx()
2764 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; in virtio_net_handle_tx_timer()
2766 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { in virtio_net_handle_tx_timer()
2772 if (!vdev->vm_running) { in virtio_net_handle_tx_timer()
2773 q->tx_waiting = 1; in virtio_net_handle_tx_timer()
2777 if (q->tx_waiting) { in virtio_net_handle_tx_timer()
2779 timer_del(q->tx_timer); in virtio_net_handle_tx_timer()
2782 /* re-arm timer to flush it (and more) on next tick */ in virtio_net_handle_tx_timer()
2783 timer_mod(q->tx_timer, in virtio_net_handle_tx_timer()
2784 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); in virtio_net_handle_tx_timer()
2785 q->tx_waiting = 1; in virtio_net_handle_tx_timer()
2793 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; in virtio_net_handle_tx_bh()
2795 if (unlikely(n->vhost_started)) { in virtio_net_handle_tx_bh()
2799 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { in virtio_net_handle_tx_bh()
2804 if (unlikely(q->tx_waiting)) { in virtio_net_handle_tx_bh()
2807 q->tx_waiting = 1; in virtio_net_handle_tx_bh()
2809 if (!vdev->vm_running) { in virtio_net_handle_tx_bh()
2813 replay_bh_schedule_event(q->tx_bh); in virtio_net_handle_tx_bh()
2819 VirtIONet *n = q->n; in virtio_net_tx_timer()
2824 if (!vdev->vm_running) { in virtio_net_tx_timer()
2826 assert(q->tx_waiting); in virtio_net_tx_timer()
2830 q->tx_waiting = 0; in virtio_net_tx_timer()
2833 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { in virtio_net_tx_timer()
2838 if (ret == -EBUSY || ret == -EINVAL) { in virtio_net_tx_timer()
2845 if (ret >= n->tx_burst) { in virtio_net_tx_timer()
2846 q->tx_waiting = 1; in virtio_net_tx_timer()
2847 timer_mod(q->tx_timer, in virtio_net_tx_timer()
2848 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); in virtio_net_tx_timer()
2852 * If less than a full burst, re-enable notification and flush in virtio_net_tx_timer()
2856 virtio_queue_set_notification(q->tx_vq, 1); in virtio_net_tx_timer()
2859 virtio_queue_set_notification(q->tx_vq, 0); in virtio_net_tx_timer()
2860 q->tx_waiting = 1; in virtio_net_tx_timer()
2861 timer_mod(q->tx_timer, in virtio_net_tx_timer()
2862 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); in virtio_net_tx_timer()
2869 VirtIONet *n = q->n; in virtio_net_tx_bh()
2874 if (!vdev->vm_running) { in virtio_net_tx_bh()
2876 assert(q->tx_waiting); in virtio_net_tx_bh()
2880 q->tx_waiting = 0; in virtio_net_tx_bh()
2883 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { in virtio_net_tx_bh()
2888 if (ret == -EBUSY || ret == -EINVAL) { in virtio_net_tx_bh()
2889 return; /* Notification re-enable handled by tx_complete or device in virtio_net_tx_bh()
2895 if (ret >= n->tx_burst) { in virtio_net_tx_bh()
2896 replay_bh_schedule_event(q->tx_bh); in virtio_net_tx_bh()
2897 q->tx_waiting = 1; in virtio_net_tx_bh()
2901 /* If less than a full burst, re-enable notification and flush in virtio_net_tx_bh()
2904 virtio_queue_set_notification(q->tx_vq, 1); in virtio_net_tx_bh()
2906 if (ret == -EINVAL) { in virtio_net_tx_bh()
2909 virtio_queue_set_notification(q->tx_vq, 0); in virtio_net_tx_bh()
2910 replay_bh_schedule_event(q->tx_bh); in virtio_net_tx_bh()
2911 q->tx_waiting = 1; in virtio_net_tx_bh()
2919 n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size, in virtio_net_add_queue()
2922 if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) { in virtio_net_add_queue()
2923 n->vqs[index].tx_vq = in virtio_net_add_queue()
2924 virtio_add_queue(vdev, n->net_conf.tx_queue_size, in virtio_net_add_queue()
2926 n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, in virtio_net_add_queue()
2928 &n->vqs[index]); in virtio_net_add_queue()
2930 n->vqs[index].tx_vq = in virtio_net_add_queue()
2931 virtio_add_queue(vdev, n->net_conf.tx_queue_size, in virtio_net_add_queue()
2933 n->vqs[index].tx_bh = qemu_bh_new_guarded(virtio_net_tx_bh, &n->vqs[index], in virtio_net_add_queue()
2934 &DEVICE(vdev)->mem_reentrancy_guard); in virtio_net_add_queue()
2937 n->vqs[index].tx_waiting = 0; in virtio_net_add_queue()
2938 n->vqs[index].n = n; in virtio_net_add_queue()
2944 VirtIONetQueue *q = &n->vqs[index]; in virtio_net_del_queue()
2945 NetClientState *nc = qemu_get_subqueue(n->nic, index); in virtio_net_del_queue()
2950 if (q->tx_timer) { in virtio_net_del_queue()
2951 timer_free(q->tx_timer); in virtio_net_del_queue()
2952 q->tx_timer = NULL; in virtio_net_del_queue()
2954 qemu_bh_delete(q->tx_bh); in virtio_net_del_queue()
2955 q->tx_bh = NULL; in virtio_net_del_queue()
2957 q->tx_waiting = 0; in virtio_net_del_queue()
2979 virtio_del_queue(vdev, old_num_queues - 1); in virtio_net_change_num_queues()
2981 for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) { in virtio_net_change_num_queues()
2986 for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) { in virtio_net_change_num_queues()
2992 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); in virtio_net_change_num_queues()
2997 int max = multiqueue ? n->max_queue_pairs : 1; in virtio_net_set_multiqueue()
2999 n->multiqueue = multiqueue; in virtio_net_set_multiqueue()
3016 NetClientState *nc = qemu_get_queue(n->nic); in virtio_net_get_features()
3017 uint32_t supported_hash_types = n->rss_data.supported_hash_types; in virtio_net_get_features()
3018 uint32_t peer_hash_types = n->rss_data.peer_hash_types; in virtio_net_get_features()
3023 n->rss_data.peer_hash_available && in virtio_net_get_features()
3026 /* Firstly sync all virtio-net possible supported features */ in virtio_net_get_features()
3027 features |= n->host_features; in virtio_net_get_features()
3060 if (!get_vhost_net(nc->peer)) { in virtio_net_get_features()
3074 if (!use_own_hash || !virtio_net_attach_ebpf_to_backend(n->nic, -1)) { in virtio_net_get_features()
3083 features = vhost_net_get_features(get_vhost_net(nc->peer), features); in virtio_net_get_features()
3084 vdev->backend_features = features; in virtio_net_get_features()
3086 if (n->mtu_bypass_backend && in virtio_net_get_features()
3087 (n->host_features & 1ULL << VIRTIO_NET_F_MTU)) { in virtio_net_get_features()
3102 if (!virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_CTRL_VQ)) { in virtio_net_get_features()
3116 virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs, in virtio_net_post_load_device()
3123 if (n->mac_table.in_use > MAC_TABLE_ENTRIES) { in virtio_net_post_load_device()
3124 n->mac_table.in_use = 0; in virtio_net_post_load_device()
3128 n->curr_guest_offloads = virtio_net_supported_guest_offloads(n); in virtio_net_post_load_device()
3137 n->saved_guest_offloads = n->curr_guest_offloads; in virtio_net_post_load_device()
3142 for (i = 0; i < n->mac_table.in_use; i++) { in virtio_net_post_load_device()
3143 if (n->mac_table.macs[i * ETH_ALEN] & 1) { in virtio_net_post_load_device()
3147 n->mac_table.first_multi = i; in virtio_net_post_load_device()
3150 * to link status bit in n->status */ in virtio_net_post_load_device()
3151 link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0; in virtio_net_post_load_device()
3152 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_post_load_device()
3153 qemu_get_subqueue(n->nic, i)->link_down = link_down; in virtio_net_post_load_device()
3158 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), in virtio_net_post_load_device()
3161 if (n->announce_timer.round) { in virtio_net_post_load_device()
3162 timer_mod(n->announce_timer.tm, in virtio_net_post_load_device()
3163 qemu_clock_get_ms(n->announce_timer.type)); in virtio_net_post_load_device()
3165 qemu_announce_timer_del(&n->announce_timer, false); in virtio_net_post_load_device()
3181 n->curr_guest_offloads = n->saved_guest_offloads; in virtio_net_post_load_virtio()
3191 .name = "virtio-net-queue-tx_waiting",
3200 return VIRTIO_NET(opaque)->max_queue_pairs > 1; in max_queue_pairs_gt_1()
3211 return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES; in mac_table_fits()
3240 tmp->vqs_1 = tmp->parent->vqs + 1; in virtio_net_tx_waiting_pre_save()
3241 tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1; in virtio_net_tx_waiting_pre_save()
3242 if (tmp->parent->curr_queue_pairs == 0) { in virtio_net_tx_waiting_pre_save()
3243 tmp->curr_queue_pairs_1 = 0; in virtio_net_tx_waiting_pre_save()
3256 if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) { in virtio_net_tx_waiting_pre_load()
3257 error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x", in virtio_net_tx_waiting_pre_load()
3258 tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs); in virtio_net_tx_waiting_pre_load()
3260 return -EINVAL; in virtio_net_tx_waiting_pre_load()
3267 .name = "virtio-net-tx_waiting",
3286 if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) { in virtio_net_ufo_post_load()
3287 error_report("virtio-net: saved image requires TUN_F_UFO support"); in virtio_net_ufo_post_load()
3288 return -EINVAL; in virtio_net_ufo_post_load()
3298 tmp->has_ufo = tmp->parent->has_ufo; in virtio_net_ufo_pre_save()
3304 .name = "virtio-net-ufo",
3320 if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) { in virtio_net_vnet_post_load()
3321 error_report("virtio-net: saved image requires vnet_hdr=on"); in virtio_net_vnet_post_load()
3322 return -EINVAL; in virtio_net_vnet_post_load()
3332 tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr; in virtio_net_vnet_pre_save()
3338 .name = "virtio-net-vnet",
3352 n->rss_data.supported_hash_types = VIRTIO_NET_RSS_SUPPORTED_HASHES; in virtio_net_rss_post_load()
3360 return VIRTIO_NET(opaque)->rss_data.enabled; in virtio_net_rss_needed()
3364 .name = "virtio-net-device/rss",
3392 if (!n->nic) { in virtio_net_get_vhost()
3396 nc = qemu_get_queue(n->nic); in virtio_net_get_vhost()
3401 net = get_vhost_net(nc->peer); in virtio_net_get_vhost()
3406 return &net->dev; in virtio_net_get_vhost()
3422 "Error getting vhost back-end of %s device %s: ", in vhost_user_net_save_state()
3423 vdev->name, vdev->parent_obj.canonical_path); in vhost_user_net_save_state()
3424 return -1; in vhost_user_net_save_state()
3430 "Error saving back-end state of %s device %s: ", in vhost_user_net_save_state()
3431 vdev->name, vdev->parent_obj.canonical_path); in vhost_user_net_save_state()
3450 "Error getting vhost back-end of %s device %s: ", in vhost_user_net_load_state()
3451 vdev->name, vdev->parent_obj.canonical_path); in vhost_user_net_load_state()
3452 return -1; in vhost_user_net_load_state()
3458 "Error loading back-end state of %s device %s: ", in vhost_user_net_load_state()
3459 vdev->name, vdev->parent_obj.canonical_path); in vhost_user_net_load_state()
3481 .name = "virtio-net-device/backend",
3488 .name = "virtio-net vhost-user backend state",
3498 .name = "virtio-net-device",
3514 * - can happen if source has a larger MAC table.; post-load
3568 assert(n->vhost_started); in virtio_net_guest_notifier_pending()
3569 if (!n->multiqueue && idx == 2) { in virtio_net_guest_notifier_pending()
3579 nc = qemu_get_subqueue(n->nic, n->max_queue_pairs); in virtio_net_guest_notifier_pending()
3581 nc = qemu_get_subqueue(n->nic, vq2q(idx)); in virtio_net_guest_notifier_pending()
3584 * Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1 in virtio_net_guest_notifier_pending()
3590 return vhost_net_config_pending(get_vhost_net(nc->peer)); in virtio_net_guest_notifier_pending()
3592 return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx); in virtio_net_guest_notifier_pending()
3600 assert(n->vhost_started); in virtio_net_guest_notifier_mask()
3601 if (!n->multiqueue && idx == 2) { in virtio_net_guest_notifier_mask()
3611 nc = qemu_get_subqueue(n->nic, n->max_queue_pairs); in virtio_net_guest_notifier_mask()
3613 nc = qemu_get_subqueue(n->nic, vq2q(idx)); in virtio_net_guest_notifier_mask()
3616 *Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1 in virtio_net_guest_notifier_mask()
3622 vhost_net_config_mask(get_vhost_net(nc->peer), vdev, mask); in virtio_net_guest_notifier_mask()
3625 vhost_net_virtqueue_mask(get_vhost_net(nc->peer), vdev, idx, mask); in virtio_net_guest_notifier_mask()
3632 n->config_size = virtio_get_config_size(&cfg_size_params, host_features); in virtio_net_set_config_size()
3643 g_free(n->netclient_name); in virtio_net_set_netclient_name()
3644 g_free(n->netclient_type); in virtio_net_set_netclient_name()
3645 n->netclient_name = g_strdup(name); in virtio_net_set_netclient_name()
3646 n->netclient_type = g_strdup(type); in virtio_net_set_netclient_name()
3658 pci_dev->partially_hotplugged = true; in failover_unplug_primary()
3678 if (!pdev->partially_hotplugged) { in failover_replug_primary()
3681 primary_bus = dev->parent_bus; in failover_replug_primary()
3687 qatomic_set(&n->failover_primary_hidden, false); in failover_replug_primary()
3696 pdev->partially_hotplugged = false; in failover_replug_primary()
3713 should_be_hidden = qatomic_read(&n->failover_primary_hidden); in virtio_net_handle_migration_primary()
3715 if (e->type == MIG_EVENT_PRECOPY_SETUP && !should_be_hidden) { in virtio_net_handle_migration_primary()
3718 qapi_event_send_unplug_primary(dev->id); in virtio_net_handle_migration_primary()
3719 qatomic_set(&n->failover_primary_hidden, true); in virtio_net_handle_migration_primary()
3723 } else if (e->type == MIG_EVENT_PRECOPY_FAILED) { in virtio_net_handle_migration_primary()
3763 if (g_strcmp0(standby_id, n->netclient_name) != 0) { in failover_hide_primary_device()
3769 * Check there is only one primary for a virtio-net device but in failover_hide_primary_device()
3773 if (n->primary_opts) { in failover_hide_primary_device()
3776 old = qdict_get_str(n->primary_opts, "id"); in failover_hide_primary_device()
3780 "'%s': '%s' and '%s'", n->netclient_name, old, new); in failover_hide_primary_device()
3784 n->primary_opts = qdict_clone_shallow(device_opts); in failover_hide_primary_device()
3785 n->primary_opts_from_json = from_json; in failover_hide_primary_device()
3789 return qatomic_read(&n->failover_primary_hidden); in failover_hide_primary_device()
3799 if (n->net_conf.mtu) { in virtio_net_device_realize()
3800 n->host_features |= (1ULL << VIRTIO_NET_F_MTU); in virtio_net_device_realize()
3803 if (n->net_conf.duplex_str) { in virtio_net_device_realize()
3804 if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) { in virtio_net_device_realize()
3805 n->net_conf.duplex = DUPLEX_HALF; in virtio_net_device_realize()
3806 } else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) { in virtio_net_device_realize()
3807 n->net_conf.duplex = DUPLEX_FULL; in virtio_net_device_realize()
3812 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); in virtio_net_device_realize()
3814 n->net_conf.duplex = DUPLEX_UNKNOWN; in virtio_net_device_realize()
3817 if (n->net_conf.speed < SPEED_UNKNOWN) { in virtio_net_device_realize()
3821 if (n->net_conf.speed >= 0) { in virtio_net_device_realize()
3822 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); in virtio_net_device_realize()
3825 if (n->failover) { in virtio_net_device_realize()
3826 n->primary_listener.hide_device = failover_hide_primary_device; in virtio_net_device_realize()
3827 qatomic_set(&n->failover_primary_hidden, true); in virtio_net_device_realize()
3828 device_listener_register(&n->primary_listener); in virtio_net_device_realize()
3829 migration_add_notifier(&n->migration_state, in virtio_net_device_realize()
3831 n->host_features |= (1ULL << VIRTIO_NET_F_STANDBY); in virtio_net_device_realize()
3834 virtio_net_set_config_size(n, n->host_features); in virtio_net_device_realize()
3835 virtio_init(vdev, VIRTIO_ID_NET, n->config_size); in virtio_net_device_realize()
3842 if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE || in virtio_net_device_realize()
3843 n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE || in virtio_net_device_realize()
3844 !is_power_of_2(n->net_conf.rx_queue_size)) { in virtio_net_device_realize()
3847 n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE, in virtio_net_device_realize()
3853 if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE || in virtio_net_device_realize()
3854 n->net_conf.tx_queue_size > virtio_net_max_tx_queue_size(n) || in virtio_net_device_realize()
3855 !is_power_of_2(n->net_conf.tx_queue_size)) { in virtio_net_device_realize()
3858 n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE, in virtio_net_device_realize()
3864 n->max_ncs = MAX(n->nic_conf.peers.queues, 1); in virtio_net_device_realize()
3870 if (n->nic_conf.peers.queues) { in virtio_net_device_realize()
3871 for (i = 0; i < n->max_ncs; i++) { in virtio_net_device_realize()
3872 if (n->nic_conf.peers.ncs[i]->is_datapath) { in virtio_net_device_realize()
3873 ++n->max_queue_pairs; in virtio_net_device_realize()
3877 n->max_queue_pairs = MAX(n->max_queue_pairs, 1); in virtio_net_device_realize()
3879 if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) { in virtio_net_device_realize()
3882 n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2); in virtio_net_device_realize()
3886 n->vqs = g_new0(VirtIONetQueue, n->max_queue_pairs); in virtio_net_device_realize()
3887 n->curr_queue_pairs = 1; in virtio_net_device_realize()
3888 n->tx_timeout = n->net_conf.txtimer; in virtio_net_device_realize()
3890 if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer") in virtio_net_device_realize()
3891 && strcmp(n->net_conf.tx, "bh")) { in virtio_net_device_realize()
3892 warn_report("virtio-net: " in virtio_net_device_realize()
3894 n->net_conf.tx); in virtio_net_device_realize()
3898 n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n), in virtio_net_device_realize()
3899 n->net_conf.tx_queue_size); in virtio_net_device_realize()
3903 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); in virtio_net_device_realize()
3904 qemu_macaddr_default_if_unset(&n->nic_conf.macaddr); in virtio_net_device_realize()
3905 memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac)); in virtio_net_device_realize()
3906 n->status = VIRTIO_NET_S_LINK_UP; in virtio_net_device_realize()
3907 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), in virtio_net_device_realize()
3910 n->announce_timer.round = 0; in virtio_net_device_realize()
3912 if (n->netclient_type) { in virtio_net_device_realize()
3916 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, in virtio_net_device_realize()
3917 n->netclient_type, n->netclient_name, in virtio_net_device_realize()
3918 &dev->mem_reentrancy_guard, n); in virtio_net_device_realize()
3920 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, in virtio_net_device_realize()
3921 object_get_typename(OBJECT(dev)), dev->id, in virtio_net_device_realize()
3922 &dev->mem_reentrancy_guard, n); in virtio_net_device_realize()
3925 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_device_realize()
3926 n->nic->ncs[i].do_not_pad = true; in virtio_net_device_realize()
3931 n->host_hdr_len = sizeof(struct virtio_net_hdr); in virtio_net_device_realize()
3933 n->host_hdr_len = 0; in virtio_net_device_realize()
3936 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a); in virtio_net_device_realize()
3938 n->vqs[0].tx_waiting = 0; in virtio_net_device_realize()
3939 n->tx_burst = n->net_conf.txburst; in virtio_net_device_realize()
3941 n->promisc = 1; /* for compatibility */ in virtio_net_device_realize()
3943 n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); in virtio_net_device_realize()
3945 n->vlans = g_malloc0(MAX_VLAN >> 3); in virtio_net_device_realize()
3946 memset(n->vlans, 0xff, MAX_VLAN >> 3); in virtio_net_device_realize()
3948 nc = qemu_get_queue(n->nic); in virtio_net_device_realize()
3949 nc->rxfilter_notify_enabled = 1; in virtio_net_device_realize()
3951 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { in virtio_net_device_realize()
3953 memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN); in virtio_net_device_realize()
3954 vhost_net_set_config(get_vhost_net(nc->peer), in virtio_net_device_realize()
3957 QTAILQ_INIT(&n->rsc_chains); in virtio_net_device_realize()
3958 n->qdev = dev; in virtio_net_device_realize()
3960 net_rx_pkt_init(&n->rx_pkt); in virtio_net_device_realize()
3962 if (qemu_get_vnet_hash_supported_types(qemu_get_queue(n->nic)->peer, in virtio_net_device_realize()
3963 &n->rss_data.peer_hash_types)) { in virtio_net_device_realize()
3964 n->rss_data.peer_hash_available = true; in virtio_net_device_realize()
3965 n->rss_data.supported_hash_types = in virtio_net_device_realize()
3966 n->rss_data.specified_hash_types.on_bits | in virtio_net_device_realize()
3967 (n->rss_data.specified_hash_types.auto_bits & in virtio_net_device_realize()
3968 n->rss_data.peer_hash_types); in virtio_net_device_realize()
3970 n->rss_data.supported_hash_types = in virtio_net_device_realize()
3971 n->rss_data.specified_hash_types.on_bits | in virtio_net_device_realize()
3972 n->rss_data.specified_hash_types.auto_bits; in virtio_net_device_realize()
3982 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { in virtio_net_device_unrealize()
3986 /* This will stop vhost backend if appropriate. */ in virtio_net_device_unrealize()
3989 g_free(n->netclient_name); in virtio_net_device_unrealize()
3990 n->netclient_name = NULL; in virtio_net_device_unrealize()
3991 g_free(n->netclient_type); in virtio_net_device_unrealize()
3992 n->netclient_type = NULL; in virtio_net_device_unrealize()
3994 g_free(n->mac_table.macs); in virtio_net_device_unrealize()
3995 g_free(n->vlans); in virtio_net_device_unrealize()
3997 if (n->failover) { in virtio_net_device_unrealize()
3998 qobject_unref(n->primary_opts); in virtio_net_device_unrealize()
3999 device_listener_unregister(&n->primary_listener); in virtio_net_device_unrealize()
4000 migration_remove_notifier(&n->migration_state); in virtio_net_device_unrealize()
4002 assert(n->primary_opts == NULL); in virtio_net_device_unrealize()
4005 max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; in virtio_net_device_unrealize()
4011 qemu_announce_timer_del(&n->announce_timer, false); in virtio_net_device_unrealize()
4012 g_free(n->vqs); in virtio_net_device_unrealize()
4013 qemu_del_nic(n->nic); in virtio_net_device_unrealize()
4015 g_free(n->rss_data.indirections_table); in virtio_net_device_unrealize()
4016 net_rx_pkt_uninit(n->rx_pkt); in virtio_net_device_unrealize()
4026 n->promisc = 1; in virtio_net_reset()
4027 n->allmulti = 0; in virtio_net_reset()
4028 n->alluni = 0; in virtio_net_reset()
4029 n->nomulti = 0; in virtio_net_reset()
4030 n->nouni = 0; in virtio_net_reset()
4031 n->nobcast = 0; in virtio_net_reset()
4033 n->curr_queue_pairs = 1; in virtio_net_reset()
4034 timer_del(n->announce_timer.tm); in virtio_net_reset()
4035 n->announce_timer.round = 0; in virtio_net_reset()
4036 n->status &= ~VIRTIO_NET_S_ANNOUNCE; in virtio_net_reset()
4039 n->mac_table.in_use = 0; in virtio_net_reset()
4040 n->mac_table.first_multi = 0; in virtio_net_reset()
4041 n->mac_table.multi_overflow = 0; in virtio_net_reset()
4042 n->mac_table.uni_overflow = 0; in virtio_net_reset()
4043 memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN); in virtio_net_reset()
4044 memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac)); in virtio_net_reset()
4045 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); in virtio_net_reset()
4048 for (i = 0; i < n->max_queue_pairs; i++) { in virtio_net_reset()
4049 flush_or_purge_queued_packets(qemu_get_subqueue(n->nic, i)); in virtio_net_reset()
4063 n->config_size = sizeof(struct virtio_net_config); in virtio_net_instance_init()
4064 device_add_bootindex_property(obj, &n->nic_conf.bootindex, in virtio_net_instance_init()
4065 "bootindex", "/ethernet-phy@0", in virtio_net_instance_init()
4068 ebpf_rss_init(&n->ebpf_rss); in virtio_net_instance_init()
4077 assert(!n->vhost_started); in virtio_net_pre_save()
4093 return primary ? primary->pending_deleted_event : false; in primary_unplug_pending()
4101 return vdc->primary_unplug_pending(dev); in dev_unplug_pending()
4105 .name = "virtio-net",
4161 DEFINE_PROP_ARRAY("ebpf-rss-fds", VirtIONet, nr_ebpf_rss_fds,
4168 DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer,
4170 DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST),
4177 DEFINE_PROP_BOOL("x-mtu-bypass-backend", VirtIONet, mtu_bypass_backend,
4188 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-ipv4", VirtIONet,
4190 VIRTIO_NET_HASH_REPORT_IPv4 - 1,
4192 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-tcp4", VirtIONet,
4194 VIRTIO_NET_HASH_REPORT_TCPv4 - 1,
4196 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-udp4", VirtIONet,
4198 VIRTIO_NET_HASH_REPORT_UDPv4 - 1,
4200 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-ipv6", VirtIONet,
4202 VIRTIO_NET_HASH_REPORT_IPv6 - 1,
4204 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-tcp6", VirtIONet,
4206 VIRTIO_NET_HASH_REPORT_TCPv6 - 1,
4208 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-udp6", VirtIONet,
4210 VIRTIO_NET_HASH_REPORT_UDPv6 - 1,
4212 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-ipv6ex", VirtIONet,
4214 VIRTIO_NET_HASH_REPORT_IPv6_EX - 1,
4216 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-tcp6ex", VirtIONet,
4218 VIRTIO_NET_HASH_REPORT_TCPv6_EX - 1,
4220 DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-udp6ex", VirtIONet,
4222 VIRTIO_NET_HASH_REPORT_UDPv6_EX - 1,
4232 dc->vmsd = &vmstate_virtio_net; in virtio_net_class_init()
4233 set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); in virtio_net_class_init()
4234 vdc->realize = virtio_net_device_realize; in virtio_net_class_init()
4235 vdc->unrealize = virtio_net_device_unrealize; in virtio_net_class_init()
4236 vdc->get_config = virtio_net_get_config; in virtio_net_class_init()
4237 vdc->set_config = virtio_net_set_config; in virtio_net_class_init()
4238 vdc->get_features = virtio_net_get_features; in virtio_net_class_init()
4239 vdc->set_features = virtio_net_set_features; in virtio_net_class_init()
4240 vdc->bad_features = virtio_net_bad_features; in virtio_net_class_init()
4241 vdc->reset = virtio_net_reset; in virtio_net_class_init()
4242 vdc->queue_reset = virtio_net_queue_reset; in virtio_net_class_init()
4243 vdc->queue_enable = virtio_net_queue_enable; in virtio_net_class_init()
4244 vdc->set_status = virtio_net_set_status; in virtio_net_class_init()
4245 vdc->guest_notifier_mask = virtio_net_guest_notifier_mask; in virtio_net_class_init()
4246 vdc->guest_notifier_pending = virtio_net_guest_notifier_pending; in virtio_net_class_init()
4247 vdc->legacy_features |= (0x1 << VIRTIO_NET_F_GSO); in virtio_net_class_init()
4248 vdc->pre_load_queues = virtio_net_pre_load_queues; in virtio_net_class_init()
4249 vdc->post_load = virtio_net_post_load_virtio; in virtio_net_class_init()
4250 vdc->vmsd = &vmstate_virtio_net_device; in virtio_net_class_init()
4251 vdc->primary_unplug_pending = primary_unplug_pending; in virtio_net_class_init()
4252 vdc->get_vhost = virtio_net_get_vhost; in virtio_net_class_init()
4253 vdc->toggle_device_iotlb = vhost_toggle_device_iotlb; in virtio_net_class_init()