1 /* 2 * Virtio Network Device 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/atomic.h" 16 #include "qemu/iov.h" 17 #include "qemu/main-loop.h" 18 #include "qemu/module.h" 19 #include "hw/virtio/virtio.h" 20 #include "net/net.h" 21 #include "net/checksum.h" 22 #include "net/tap.h" 23 #include "qemu/error-report.h" 24 #include "qemu/timer.h" 25 #include "qemu/option.h" 26 #include "qemu/option_int.h" 27 #include "qemu/config-file.h" 28 #include "qapi/qmp/qdict.h" 29 #include "hw/virtio/virtio-net.h" 30 #include "net/vhost_net.h" 31 #include "net/announce.h" 32 #include "hw/virtio/virtio-bus.h" 33 #include "qapi/error.h" 34 #include "qapi/qapi-events-net.h" 35 #include "hw/qdev-properties.h" 36 #include "qapi/qapi-types-migration.h" 37 #include "qapi/qapi-events-migration.h" 38 #include "hw/virtio/virtio-access.h" 39 #include "migration/misc.h" 40 #include "standard-headers/linux/ethtool.h" 41 #include "sysemu/sysemu.h" 42 #include "trace.h" 43 #include "monitor/qdev.h" 44 #include "hw/pci/pci.h" 45 #include "net_rx_pkt.h" 46 #include "hw/virtio/vhost.h" 47 #include "sysemu/qtest.h" 48 49 #define VIRTIO_NET_VM_VERSION 11 50 51 #define MAC_TABLE_ENTRIES 64 52 #define MAX_VLAN (1 << 12) /* Per 802.1Q definition */ 53 54 /* previously fixed value */ 55 #define VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 256 56 #define VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 256 57 58 /* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */ 59 #define VIRTIO_NET_RX_QUEUE_MIN_SIZE VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 60 #define VIRTIO_NET_TX_QUEUE_MIN_SIZE VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 61 62 #define VIRTIO_NET_IP4_ADDR_SIZE 8 /* ipv4 saddr + daddr */ 63 64 #define VIRTIO_NET_TCP_FLAG 0x3F 65 #define VIRTIO_NET_TCP_HDR_LENGTH 0xF000 66 67 /* IPv4 max payload, 16 bits in the header */ 68 #define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header)) 69 #define VIRTIO_NET_MAX_TCP_PAYLOAD 65535 70 71 /* header length value in ip header without option */ 72 #define VIRTIO_NET_IP4_HEADER_LENGTH 5 73 74 #define VIRTIO_NET_IP6_ADDR_SIZE 32 /* ipv6 saddr + daddr */ 75 #define VIRTIO_NET_MAX_IP6_PAYLOAD VIRTIO_NET_MAX_TCP_PAYLOAD 76 77 /* Purge coalesced packets timer interval, This value affects the performance 78 a lot, and should be tuned carefully, '300000'(300us) is the recommended 79 value to pass the WHQL test, '50000' can gain 2x netperf throughput with 80 tso/gso/gro 'off'. */ 81 #define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000 82 83 #define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 | \ 84 VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | \ 85 VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | \ 86 VIRTIO_NET_RSS_HASH_TYPE_IPv6 | \ 87 VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | \ 88 VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | \ 89 VIRTIO_NET_RSS_HASH_TYPE_IP_EX | \ 90 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | \ 91 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) 92 93 static const VirtIOFeature feature_sizes[] = { 94 {.flags = 1ULL << VIRTIO_NET_F_MAC, 95 .end = endof(struct virtio_net_config, mac)}, 96 {.flags = 1ULL << VIRTIO_NET_F_STATUS, 97 .end = endof(struct virtio_net_config, status)}, 98 {.flags = 1ULL << VIRTIO_NET_F_MQ, 99 .end = endof(struct virtio_net_config, max_virtqueue_pairs)}, 100 {.flags = 1ULL << VIRTIO_NET_F_MTU, 101 .end = endof(struct virtio_net_config, mtu)}, 102 {.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX, 103 .end = endof(struct virtio_net_config, duplex)}, 104 {.flags = (1ULL << VIRTIO_NET_F_RSS) | (1ULL << VIRTIO_NET_F_HASH_REPORT), 105 .end = endof(struct virtio_net_config, supported_hash_types)}, 106 {} 107 }; 108 109 static VirtIONetQueue *virtio_net_get_subqueue(NetClientState *nc) 110 { 111 VirtIONet *n = qemu_get_nic_opaque(nc); 112 113 return &n->vqs[nc->queue_index]; 114 } 115 116 static int vq2q(int queue_index) 117 { 118 return queue_index / 2; 119 } 120 121 /* TODO 122 * - we could suppress RX interrupt if we were so inclined. 123 */ 124 125 static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config) 126 { 127 VirtIONet *n = VIRTIO_NET(vdev); 128 struct virtio_net_config netcfg; 129 NetClientState *nc = qemu_get_queue(n->nic); 130 static const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } }; 131 132 int ret = 0; 133 memset(&netcfg, 0 , sizeof(struct virtio_net_config)); 134 virtio_stw_p(vdev, &netcfg.status, n->status); 135 virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs); 136 virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu); 137 memcpy(netcfg.mac, n->mac, ETH_ALEN); 138 virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed); 139 netcfg.duplex = n->net_conf.duplex; 140 netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE; 141 virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length, 142 virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ? 143 VIRTIO_NET_RSS_MAX_TABLE_LEN : 1); 144 virtio_stl_p(vdev, &netcfg.supported_hash_types, 145 VIRTIO_NET_RSS_SUPPORTED_HASHES); 146 memcpy(config, &netcfg, n->config_size); 147 148 /* 149 * Is this VDPA? No peer means not VDPA: there's no way to 150 * disconnect/reconnect a VDPA peer. 151 */ 152 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 153 ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg, 154 n->config_size); 155 if (ret != -1) { 156 /* 157 * Some NIC/kernel combinations present 0 as the mac address. As 158 * that is not a legal address, try to proceed with the 159 * address from the QEMU command line in the hope that the 160 * address has been configured correctly elsewhere - just not 161 * reported by the device. 162 */ 163 if (memcmp(&netcfg.mac, &zero, sizeof(zero)) == 0) { 164 info_report("Zero hardware mac address detected. Ignoring."); 165 memcpy(netcfg.mac, n->mac, ETH_ALEN); 166 } 167 memcpy(config, &netcfg, n->config_size); 168 } 169 } 170 } 171 172 static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config) 173 { 174 VirtIONet *n = VIRTIO_NET(vdev); 175 struct virtio_net_config netcfg = {}; 176 NetClientState *nc = qemu_get_queue(n->nic); 177 178 memcpy(&netcfg, config, n->config_size); 179 180 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) && 181 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) && 182 memcmp(netcfg.mac, n->mac, ETH_ALEN)) { 183 memcpy(n->mac, netcfg.mac, ETH_ALEN); 184 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 185 } 186 187 /* 188 * Is this VDPA? No peer means not VDPA: there's no way to 189 * disconnect/reconnect a VDPA peer. 190 */ 191 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 192 vhost_net_set_config(get_vhost_net(nc->peer), 193 (uint8_t *)&netcfg, 0, n->config_size, 194 VHOST_SET_CONFIG_TYPE_MASTER); 195 } 196 } 197 198 static bool virtio_net_started(VirtIONet *n, uint8_t status) 199 { 200 VirtIODevice *vdev = VIRTIO_DEVICE(n); 201 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 202 (n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running; 203 } 204 205 static void virtio_net_announce_notify(VirtIONet *net) 206 { 207 VirtIODevice *vdev = VIRTIO_DEVICE(net); 208 trace_virtio_net_announce_notify(); 209 210 net->status |= VIRTIO_NET_S_ANNOUNCE; 211 virtio_notify_config(vdev); 212 } 213 214 static void virtio_net_announce_timer(void *opaque) 215 { 216 VirtIONet *n = opaque; 217 trace_virtio_net_announce_timer(n->announce_timer.round); 218 219 n->announce_timer.round--; 220 virtio_net_announce_notify(n); 221 } 222 223 static void virtio_net_announce(NetClientState *nc) 224 { 225 VirtIONet *n = qemu_get_nic_opaque(nc); 226 VirtIODevice *vdev = VIRTIO_DEVICE(n); 227 228 /* 229 * Make sure the virtio migration announcement timer isn't running 230 * If it is, let it trigger announcement so that we do not cause 231 * confusion. 232 */ 233 if (n->announce_timer.round) { 234 return; 235 } 236 237 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 238 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 239 virtio_net_announce_notify(n); 240 } 241 } 242 243 static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) 244 { 245 VirtIODevice *vdev = VIRTIO_DEVICE(n); 246 NetClientState *nc = qemu_get_queue(n->nic); 247 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 248 int cvq = n->max_ncs - n->max_queue_pairs; 249 250 if (!get_vhost_net(nc->peer)) { 251 return; 252 } 253 254 if ((virtio_net_started(n, status) && !nc->peer->link_down) == 255 !!n->vhost_started) { 256 return; 257 } 258 if (!n->vhost_started) { 259 int r, i; 260 261 if (n->needs_vnet_hdr_swap) { 262 error_report("backend does not support %s vnet headers; " 263 "falling back on userspace virtio", 264 virtio_is_big_endian(vdev) ? "BE" : "LE"); 265 return; 266 } 267 268 /* Any packets outstanding? Purge them to avoid touching rings 269 * when vhost is running. 270 */ 271 for (i = 0; i < queue_pairs; i++) { 272 NetClientState *qnc = qemu_get_subqueue(n->nic, i); 273 274 /* Purge both directions: TX and RX. */ 275 qemu_net_queue_purge(qnc->peer->incoming_queue, qnc); 276 qemu_net_queue_purge(qnc->incoming_queue, qnc->peer); 277 } 278 279 if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) { 280 r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu); 281 if (r < 0) { 282 error_report("%uBytes MTU not supported by the backend", 283 n->net_conf.mtu); 284 285 return; 286 } 287 } 288 289 n->vhost_started = 1; 290 r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq); 291 if (r < 0) { 292 error_report("unable to start vhost net: %d: " 293 "falling back on userspace virtio", -r); 294 n->vhost_started = 0; 295 } 296 } else { 297 vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq); 298 n->vhost_started = 0; 299 } 300 } 301 302 static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev, 303 NetClientState *peer, 304 bool enable) 305 { 306 if (virtio_is_big_endian(vdev)) { 307 return qemu_set_vnet_be(peer, enable); 308 } else { 309 return qemu_set_vnet_le(peer, enable); 310 } 311 } 312 313 static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs, 314 int queue_pairs, bool enable) 315 { 316 int i; 317 318 for (i = 0; i < queue_pairs; i++) { 319 if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 && 320 enable) { 321 while (--i >= 0) { 322 virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false); 323 } 324 325 return true; 326 } 327 } 328 329 return false; 330 } 331 332 static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status) 333 { 334 VirtIODevice *vdev = VIRTIO_DEVICE(n); 335 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 336 337 if (virtio_net_started(n, status)) { 338 /* Before using the device, we tell the network backend about the 339 * endianness to use when parsing vnet headers. If the backend 340 * can't do it, we fallback onto fixing the headers in the core 341 * virtio-net code. 342 */ 343 n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs, 344 queue_pairs, true); 345 } else if (virtio_net_started(n, vdev->status)) { 346 /* After using the device, we need to reset the network backend to 347 * the default (guest native endianness), otherwise the guest may 348 * lose network connectivity if it is rebooted into a different 349 * endianness. 350 */ 351 virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false); 352 } 353 } 354 355 static void virtio_net_drop_tx_queue_data(VirtIODevice *vdev, VirtQueue *vq) 356 { 357 unsigned int dropped = virtqueue_drop_all(vq); 358 if (dropped) { 359 virtio_notify(vdev, vq); 360 } 361 } 362 363 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status) 364 { 365 VirtIONet *n = VIRTIO_NET(vdev); 366 VirtIONetQueue *q; 367 int i; 368 uint8_t queue_status; 369 370 virtio_net_vnet_endian_status(n, status); 371 virtio_net_vhost_status(n, status); 372 373 for (i = 0; i < n->max_queue_pairs; i++) { 374 NetClientState *ncs = qemu_get_subqueue(n->nic, i); 375 bool queue_started; 376 q = &n->vqs[i]; 377 378 if ((!n->multiqueue && i != 0) || i >= n->curr_queue_pairs) { 379 queue_status = 0; 380 } else { 381 queue_status = status; 382 } 383 queue_started = 384 virtio_net_started(n, queue_status) && !n->vhost_started; 385 386 if (queue_started) { 387 qemu_flush_queued_packets(ncs); 388 } 389 390 if (!q->tx_waiting) { 391 continue; 392 } 393 394 if (queue_started) { 395 if (q->tx_timer) { 396 timer_mod(q->tx_timer, 397 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 398 } else { 399 qemu_bh_schedule(q->tx_bh); 400 } 401 } else { 402 if (q->tx_timer) { 403 timer_del(q->tx_timer); 404 } else { 405 qemu_bh_cancel(q->tx_bh); 406 } 407 if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 && 408 (queue_status & VIRTIO_CONFIG_S_DRIVER_OK) && 409 vdev->vm_running) { 410 /* if tx is waiting we are likely have some packets in tx queue 411 * and disabled notification */ 412 q->tx_waiting = 0; 413 virtio_queue_set_notification(q->tx_vq, 1); 414 virtio_net_drop_tx_queue_data(vdev, q->tx_vq); 415 } 416 } 417 } 418 } 419 420 static void virtio_net_set_link_status(NetClientState *nc) 421 { 422 VirtIONet *n = qemu_get_nic_opaque(nc); 423 VirtIODevice *vdev = VIRTIO_DEVICE(n); 424 uint16_t old_status = n->status; 425 426 if (nc->link_down) 427 n->status &= ~VIRTIO_NET_S_LINK_UP; 428 else 429 n->status |= VIRTIO_NET_S_LINK_UP; 430 431 if (n->status != old_status) 432 virtio_notify_config(vdev); 433 434 virtio_net_set_status(vdev, vdev->status); 435 } 436 437 static void rxfilter_notify(NetClientState *nc) 438 { 439 VirtIONet *n = qemu_get_nic_opaque(nc); 440 441 if (nc->rxfilter_notify_enabled) { 442 char *path = object_get_canonical_path(OBJECT(n->qdev)); 443 qapi_event_send_nic_rx_filter_changed(!!n->netclient_name, 444 n->netclient_name, path); 445 g_free(path); 446 447 /* disable event notification to avoid events flooding */ 448 nc->rxfilter_notify_enabled = 0; 449 } 450 } 451 452 static intList *get_vlan_table(VirtIONet *n) 453 { 454 intList *list; 455 int i, j; 456 457 list = NULL; 458 for (i = 0; i < MAX_VLAN >> 5; i++) { 459 for (j = 0; n->vlans[i] && j <= 0x1f; j++) { 460 if (n->vlans[i] & (1U << j)) { 461 QAPI_LIST_PREPEND(list, (i << 5) + j); 462 } 463 } 464 } 465 466 return list; 467 } 468 469 static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc) 470 { 471 VirtIONet *n = qemu_get_nic_opaque(nc); 472 VirtIODevice *vdev = VIRTIO_DEVICE(n); 473 RxFilterInfo *info; 474 strList *str_list; 475 int i; 476 477 info = g_malloc0(sizeof(*info)); 478 info->name = g_strdup(nc->name); 479 info->promiscuous = n->promisc; 480 481 if (n->nouni) { 482 info->unicast = RX_STATE_NONE; 483 } else if (n->alluni) { 484 info->unicast = RX_STATE_ALL; 485 } else { 486 info->unicast = RX_STATE_NORMAL; 487 } 488 489 if (n->nomulti) { 490 info->multicast = RX_STATE_NONE; 491 } else if (n->allmulti) { 492 info->multicast = RX_STATE_ALL; 493 } else { 494 info->multicast = RX_STATE_NORMAL; 495 } 496 497 info->broadcast_allowed = n->nobcast; 498 info->multicast_overflow = n->mac_table.multi_overflow; 499 info->unicast_overflow = n->mac_table.uni_overflow; 500 501 info->main_mac = qemu_mac_strdup_printf(n->mac); 502 503 str_list = NULL; 504 for (i = 0; i < n->mac_table.first_multi; i++) { 505 QAPI_LIST_PREPEND(str_list, 506 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 507 } 508 info->unicast_table = str_list; 509 510 str_list = NULL; 511 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 512 QAPI_LIST_PREPEND(str_list, 513 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 514 } 515 info->multicast_table = str_list; 516 info->vlan_table = get_vlan_table(n); 517 518 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) { 519 info->vlan = RX_STATE_ALL; 520 } else if (!info->vlan_table) { 521 info->vlan = RX_STATE_NONE; 522 } else { 523 info->vlan = RX_STATE_NORMAL; 524 } 525 526 /* enable event notification after query */ 527 nc->rxfilter_notify_enabled = 1; 528 529 return info; 530 } 531 532 static void virtio_net_reset(VirtIODevice *vdev) 533 { 534 VirtIONet *n = VIRTIO_NET(vdev); 535 int i; 536 537 /* Reset back to compatibility mode */ 538 n->promisc = 1; 539 n->allmulti = 0; 540 n->alluni = 0; 541 n->nomulti = 0; 542 n->nouni = 0; 543 n->nobcast = 0; 544 /* multiqueue is disabled by default */ 545 n->curr_queue_pairs = 1; 546 timer_del(n->announce_timer.tm); 547 n->announce_timer.round = 0; 548 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 549 550 /* Flush any MAC and VLAN filter table state */ 551 n->mac_table.in_use = 0; 552 n->mac_table.first_multi = 0; 553 n->mac_table.multi_overflow = 0; 554 n->mac_table.uni_overflow = 0; 555 memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN); 556 memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac)); 557 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 558 memset(n->vlans, 0, MAX_VLAN >> 3); 559 560 /* Flush any async TX */ 561 for (i = 0; i < n->max_queue_pairs; i++) { 562 NetClientState *nc = qemu_get_subqueue(n->nic, i); 563 564 if (nc->peer) { 565 qemu_flush_or_purge_queued_packets(nc->peer, true); 566 assert(!virtio_net_get_subqueue(nc)->async_tx.elem); 567 } 568 } 569 } 570 571 static void peer_test_vnet_hdr(VirtIONet *n) 572 { 573 NetClientState *nc = qemu_get_queue(n->nic); 574 if (!nc->peer) { 575 return; 576 } 577 578 n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer); 579 } 580 581 static int peer_has_vnet_hdr(VirtIONet *n) 582 { 583 return n->has_vnet_hdr; 584 } 585 586 static int peer_has_ufo(VirtIONet *n) 587 { 588 if (!peer_has_vnet_hdr(n)) 589 return 0; 590 591 n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer); 592 593 return n->has_ufo; 594 } 595 596 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, 597 int version_1, int hash_report) 598 { 599 int i; 600 NetClientState *nc; 601 602 n->mergeable_rx_bufs = mergeable_rx_bufs; 603 604 if (version_1) { 605 n->guest_hdr_len = hash_report ? 606 sizeof(struct virtio_net_hdr_v1_hash) : 607 sizeof(struct virtio_net_hdr_mrg_rxbuf); 608 n->rss_data.populate_hash = !!hash_report; 609 } else { 610 n->guest_hdr_len = n->mergeable_rx_bufs ? 611 sizeof(struct virtio_net_hdr_mrg_rxbuf) : 612 sizeof(struct virtio_net_hdr); 613 } 614 615 for (i = 0; i < n->max_queue_pairs; i++) { 616 nc = qemu_get_subqueue(n->nic, i); 617 618 if (peer_has_vnet_hdr(n) && 619 qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) { 620 qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len); 621 n->host_hdr_len = n->guest_hdr_len; 622 } 623 } 624 } 625 626 static int virtio_net_max_tx_queue_size(VirtIONet *n) 627 { 628 NetClientState *peer = n->nic_conf.peers.ncs[0]; 629 630 /* 631 * Backends other than vhost-user or vhost-vdpa don't support max queue 632 * size. 633 */ 634 if (!peer) { 635 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 636 } 637 638 switch(peer->info->type) { 639 case NET_CLIENT_DRIVER_VHOST_USER: 640 case NET_CLIENT_DRIVER_VHOST_VDPA: 641 return VIRTQUEUE_MAX_SIZE; 642 default: 643 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 644 }; 645 } 646 647 static int peer_attach(VirtIONet *n, int index) 648 { 649 NetClientState *nc = qemu_get_subqueue(n->nic, index); 650 651 if (!nc->peer) { 652 return 0; 653 } 654 655 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 656 vhost_set_vring_enable(nc->peer, 1); 657 } 658 659 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 660 return 0; 661 } 662 663 if (n->max_queue_pairs == 1) { 664 return 0; 665 } 666 667 return tap_enable(nc->peer); 668 } 669 670 static int peer_detach(VirtIONet *n, int index) 671 { 672 NetClientState *nc = qemu_get_subqueue(n->nic, index); 673 674 if (!nc->peer) { 675 return 0; 676 } 677 678 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 679 vhost_set_vring_enable(nc->peer, 0); 680 } 681 682 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 683 return 0; 684 } 685 686 return tap_disable(nc->peer); 687 } 688 689 static void virtio_net_set_queue_pairs(VirtIONet *n) 690 { 691 int i; 692 int r; 693 694 if (n->nic->peer_deleted) { 695 return; 696 } 697 698 for (i = 0; i < n->max_queue_pairs; i++) { 699 if (i < n->curr_queue_pairs) { 700 r = peer_attach(n, i); 701 assert(!r); 702 } else { 703 r = peer_detach(n, i); 704 assert(!r); 705 } 706 } 707 } 708 709 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue); 710 711 static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, 712 Error **errp) 713 { 714 VirtIONet *n = VIRTIO_NET(vdev); 715 NetClientState *nc = qemu_get_queue(n->nic); 716 717 /* Firstly sync all virtio-net possible supported features */ 718 features |= n->host_features; 719 720 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 721 722 if (!peer_has_vnet_hdr(n)) { 723 virtio_clear_feature(&features, VIRTIO_NET_F_CSUM); 724 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO4); 725 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO6); 726 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_ECN); 727 728 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_CSUM); 729 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4); 730 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6); 731 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN); 732 733 virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); 734 } 735 736 if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) { 737 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_UFO); 738 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO); 739 } 740 741 if (!get_vhost_net(nc->peer)) { 742 return features; 743 } 744 745 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 746 virtio_clear_feature(&features, VIRTIO_NET_F_RSS); 747 } 748 features = vhost_net_get_features(get_vhost_net(nc->peer), features); 749 vdev->backend_features = features; 750 751 if (n->mtu_bypass_backend && 752 (n->host_features & 1ULL << VIRTIO_NET_F_MTU)) { 753 features |= (1ULL << VIRTIO_NET_F_MTU); 754 } 755 756 return features; 757 } 758 759 static uint64_t virtio_net_bad_features(VirtIODevice *vdev) 760 { 761 uint64_t features = 0; 762 763 /* Linux kernel 2.6.25. It understood MAC (as everyone must), 764 * but also these: */ 765 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 766 virtio_add_feature(&features, VIRTIO_NET_F_CSUM); 767 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4); 768 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6); 769 virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN); 770 771 return features; 772 } 773 774 static void virtio_net_apply_guest_offloads(VirtIONet *n) 775 { 776 qemu_set_offload(qemu_get_queue(n->nic)->peer, 777 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)), 778 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)), 779 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)), 780 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)), 781 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO))); 782 } 783 784 static uint64_t virtio_net_guest_offloads_by_features(uint32_t features) 785 { 786 static const uint64_t guest_offloads_mask = 787 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | 788 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 789 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | 790 (1ULL << VIRTIO_NET_F_GUEST_ECN) | 791 (1ULL << VIRTIO_NET_F_GUEST_UFO); 792 793 return guest_offloads_mask & features; 794 } 795 796 static inline uint64_t virtio_net_supported_guest_offloads(VirtIONet *n) 797 { 798 VirtIODevice *vdev = VIRTIO_DEVICE(n); 799 return virtio_net_guest_offloads_by_features(vdev->guest_features); 800 } 801 802 typedef struct { 803 VirtIONet *n; 804 DeviceState *dev; 805 } FailoverDevice; 806 807 /** 808 * Set the failover primary device 809 * 810 * @opaque: FailoverId to setup 811 * @opts: opts for device we are handling 812 * @errp: returns an error if this function fails 813 */ 814 static int failover_set_primary(DeviceState *dev, void *opaque) 815 { 816 FailoverDevice *fdev = opaque; 817 PCIDevice *pci_dev = (PCIDevice *) 818 object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE); 819 820 if (!pci_dev) { 821 return 0; 822 } 823 824 if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) { 825 fdev->dev = dev; 826 return 1; 827 } 828 829 return 0; 830 } 831 832 /** 833 * Find the primary device for this failover virtio-net 834 * 835 * @n: VirtIONet device 836 * @errp: returns an error if this function fails 837 */ 838 static DeviceState *failover_find_primary_device(VirtIONet *n) 839 { 840 FailoverDevice fdev = { 841 .n = n, 842 }; 843 844 qbus_walk_children(sysbus_get_default(), failover_set_primary, NULL, 845 NULL, NULL, &fdev); 846 return fdev.dev; 847 } 848 849 static void failover_add_primary(VirtIONet *n, Error **errp) 850 { 851 Error *err = NULL; 852 DeviceState *dev = failover_find_primary_device(n); 853 854 if (dev) { 855 return; 856 } 857 858 if (!n->primary_opts) { 859 error_setg(errp, "Primary device not found"); 860 error_append_hint(errp, "Virtio-net failover will not work. Make " 861 "sure primary device has parameter" 862 " failover_pair_id=%s\n", n->netclient_name); 863 return; 864 } 865 866 dev = qdev_device_add_from_qdict(n->primary_opts, 867 n->primary_opts_from_json, 868 &err); 869 if (err) { 870 qobject_unref(n->primary_opts); 871 n->primary_opts = NULL; 872 } else { 873 object_unref(OBJECT(dev)); 874 } 875 error_propagate(errp, err); 876 } 877 878 static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) 879 { 880 VirtIONet *n = VIRTIO_NET(vdev); 881 Error *err = NULL; 882 int i; 883 884 if (n->mtu_bypass_backend && 885 !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { 886 features &= ~(1ULL << VIRTIO_NET_F_MTU); 887 } 888 889 virtio_net_set_multiqueue(n, 890 virtio_has_feature(features, VIRTIO_NET_F_RSS) || 891 virtio_has_feature(features, VIRTIO_NET_F_MQ)); 892 893 virtio_net_set_mrg_rx_bufs(n, 894 virtio_has_feature(features, 895 VIRTIO_NET_F_MRG_RXBUF), 896 virtio_has_feature(features, 897 VIRTIO_F_VERSION_1), 898 virtio_has_feature(features, 899 VIRTIO_NET_F_HASH_REPORT)); 900 901 n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 902 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO4); 903 n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 904 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); 905 n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); 906 907 if (n->has_vnet_hdr) { 908 n->curr_guest_offloads = 909 virtio_net_guest_offloads_by_features(features); 910 virtio_net_apply_guest_offloads(n); 911 } 912 913 for (i = 0; i < n->max_queue_pairs; i++) { 914 NetClientState *nc = qemu_get_subqueue(n->nic, i); 915 916 if (!get_vhost_net(nc->peer)) { 917 continue; 918 } 919 vhost_net_ack_features(get_vhost_net(nc->peer), features); 920 } 921 922 if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) { 923 memset(n->vlans, 0, MAX_VLAN >> 3); 924 } else { 925 memset(n->vlans, 0xff, MAX_VLAN >> 3); 926 } 927 928 if (virtio_has_feature(features, VIRTIO_NET_F_STANDBY)) { 929 qapi_event_send_failover_negotiated(n->netclient_name); 930 qatomic_set(&n->failover_primary_hidden, false); 931 failover_add_primary(n, &err); 932 if (err) { 933 if (!qtest_enabled()) { 934 warn_report_err(err); 935 } else { 936 error_free(err); 937 } 938 } 939 } 940 } 941 942 static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd, 943 struct iovec *iov, unsigned int iov_cnt) 944 { 945 uint8_t on; 946 size_t s; 947 NetClientState *nc = qemu_get_queue(n->nic); 948 949 s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on)); 950 if (s != sizeof(on)) { 951 return VIRTIO_NET_ERR; 952 } 953 954 if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) { 955 n->promisc = on; 956 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) { 957 n->allmulti = on; 958 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) { 959 n->alluni = on; 960 } else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) { 961 n->nomulti = on; 962 } else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) { 963 n->nouni = on; 964 } else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) { 965 n->nobcast = on; 966 } else { 967 return VIRTIO_NET_ERR; 968 } 969 970 rxfilter_notify(nc); 971 972 return VIRTIO_NET_OK; 973 } 974 975 static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd, 976 struct iovec *iov, unsigned int iov_cnt) 977 { 978 VirtIODevice *vdev = VIRTIO_DEVICE(n); 979 uint64_t offloads; 980 size_t s; 981 982 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 983 return VIRTIO_NET_ERR; 984 } 985 986 s = iov_to_buf(iov, iov_cnt, 0, &offloads, sizeof(offloads)); 987 if (s != sizeof(offloads)) { 988 return VIRTIO_NET_ERR; 989 } 990 991 if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) { 992 uint64_t supported_offloads; 993 994 offloads = virtio_ldq_p(vdev, &offloads); 995 996 if (!n->has_vnet_hdr) { 997 return VIRTIO_NET_ERR; 998 } 999 1000 n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1001 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO4); 1002 n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1003 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO6); 1004 virtio_clear_feature(&offloads, VIRTIO_NET_F_RSC_EXT); 1005 1006 supported_offloads = virtio_net_supported_guest_offloads(n); 1007 if (offloads & ~supported_offloads) { 1008 return VIRTIO_NET_ERR; 1009 } 1010 1011 n->curr_guest_offloads = offloads; 1012 virtio_net_apply_guest_offloads(n); 1013 1014 return VIRTIO_NET_OK; 1015 } else { 1016 return VIRTIO_NET_ERR; 1017 } 1018 } 1019 1020 static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd, 1021 struct iovec *iov, unsigned int iov_cnt) 1022 { 1023 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1024 struct virtio_net_ctrl_mac mac_data; 1025 size_t s; 1026 NetClientState *nc = qemu_get_queue(n->nic); 1027 1028 if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) { 1029 if (iov_size(iov, iov_cnt) != sizeof(n->mac)) { 1030 return VIRTIO_NET_ERR; 1031 } 1032 s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac)); 1033 assert(s == sizeof(n->mac)); 1034 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 1035 rxfilter_notify(nc); 1036 1037 return VIRTIO_NET_OK; 1038 } 1039 1040 if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) { 1041 return VIRTIO_NET_ERR; 1042 } 1043 1044 int in_use = 0; 1045 int first_multi = 0; 1046 uint8_t uni_overflow = 0; 1047 uint8_t multi_overflow = 0; 1048 uint8_t *macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 1049 1050 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1051 sizeof(mac_data.entries)); 1052 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1053 if (s != sizeof(mac_data.entries)) { 1054 goto error; 1055 } 1056 iov_discard_front(&iov, &iov_cnt, s); 1057 1058 if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) { 1059 goto error; 1060 } 1061 1062 if (mac_data.entries <= MAC_TABLE_ENTRIES) { 1063 s = iov_to_buf(iov, iov_cnt, 0, macs, 1064 mac_data.entries * ETH_ALEN); 1065 if (s != mac_data.entries * ETH_ALEN) { 1066 goto error; 1067 } 1068 in_use += mac_data.entries; 1069 } else { 1070 uni_overflow = 1; 1071 } 1072 1073 iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN); 1074 1075 first_multi = in_use; 1076 1077 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1078 sizeof(mac_data.entries)); 1079 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1080 if (s != sizeof(mac_data.entries)) { 1081 goto error; 1082 } 1083 1084 iov_discard_front(&iov, &iov_cnt, s); 1085 1086 if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) { 1087 goto error; 1088 } 1089 1090 if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) { 1091 s = iov_to_buf(iov, iov_cnt, 0, &macs[in_use * ETH_ALEN], 1092 mac_data.entries * ETH_ALEN); 1093 if (s != mac_data.entries * ETH_ALEN) { 1094 goto error; 1095 } 1096 in_use += mac_data.entries; 1097 } else { 1098 multi_overflow = 1; 1099 } 1100 1101 n->mac_table.in_use = in_use; 1102 n->mac_table.first_multi = first_multi; 1103 n->mac_table.uni_overflow = uni_overflow; 1104 n->mac_table.multi_overflow = multi_overflow; 1105 memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN); 1106 g_free(macs); 1107 rxfilter_notify(nc); 1108 1109 return VIRTIO_NET_OK; 1110 1111 error: 1112 g_free(macs); 1113 return VIRTIO_NET_ERR; 1114 } 1115 1116 static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd, 1117 struct iovec *iov, unsigned int iov_cnt) 1118 { 1119 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1120 uint16_t vid; 1121 size_t s; 1122 NetClientState *nc = qemu_get_queue(n->nic); 1123 1124 s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid)); 1125 vid = virtio_lduw_p(vdev, &vid); 1126 if (s != sizeof(vid)) { 1127 return VIRTIO_NET_ERR; 1128 } 1129 1130 if (vid >= MAX_VLAN) 1131 return VIRTIO_NET_ERR; 1132 1133 if (cmd == VIRTIO_NET_CTRL_VLAN_ADD) 1134 n->vlans[vid >> 5] |= (1U << (vid & 0x1f)); 1135 else if (cmd == VIRTIO_NET_CTRL_VLAN_DEL) 1136 n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f)); 1137 else 1138 return VIRTIO_NET_ERR; 1139 1140 rxfilter_notify(nc); 1141 1142 return VIRTIO_NET_OK; 1143 } 1144 1145 static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, 1146 struct iovec *iov, unsigned int iov_cnt) 1147 { 1148 trace_virtio_net_handle_announce(n->announce_timer.round); 1149 if (cmd == VIRTIO_NET_CTRL_ANNOUNCE_ACK && 1150 n->status & VIRTIO_NET_S_ANNOUNCE) { 1151 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 1152 if (n->announce_timer.round) { 1153 qemu_announce_timer_step(&n->announce_timer); 1154 } 1155 return VIRTIO_NET_OK; 1156 } else { 1157 return VIRTIO_NET_ERR; 1158 } 1159 } 1160 1161 static void virtio_net_detach_epbf_rss(VirtIONet *n); 1162 1163 static void virtio_net_disable_rss(VirtIONet *n) 1164 { 1165 if (n->rss_data.enabled) { 1166 trace_virtio_net_rss_disable(); 1167 } 1168 n->rss_data.enabled = false; 1169 1170 virtio_net_detach_epbf_rss(n); 1171 } 1172 1173 static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd) 1174 { 1175 NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0); 1176 if (nc == NULL || nc->info->set_steering_ebpf == NULL) { 1177 return false; 1178 } 1179 1180 return nc->info->set_steering_ebpf(nc, prog_fd); 1181 } 1182 1183 static void rss_data_to_rss_config(struct VirtioNetRssData *data, 1184 struct EBPFRSSConfig *config) 1185 { 1186 config->redirect = data->redirect; 1187 config->populate_hash = data->populate_hash; 1188 config->hash_types = data->hash_types; 1189 config->indirections_len = data->indirections_len; 1190 config->default_queue = data->default_queue; 1191 } 1192 1193 static bool virtio_net_attach_epbf_rss(VirtIONet *n) 1194 { 1195 struct EBPFRSSConfig config = {}; 1196 1197 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 1198 return false; 1199 } 1200 1201 rss_data_to_rss_config(&n->rss_data, &config); 1202 1203 if (!ebpf_rss_set_all(&n->ebpf_rss, &config, 1204 n->rss_data.indirections_table, n->rss_data.key)) { 1205 return false; 1206 } 1207 1208 if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { 1209 return false; 1210 } 1211 1212 return true; 1213 } 1214 1215 static void virtio_net_detach_epbf_rss(VirtIONet *n) 1216 { 1217 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1218 } 1219 1220 static bool virtio_net_load_ebpf(VirtIONet *n) 1221 { 1222 if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { 1223 /* backend does't support steering ebpf */ 1224 return false; 1225 } 1226 1227 return ebpf_rss_load(&n->ebpf_rss); 1228 } 1229 1230 static void virtio_net_unload_ebpf(VirtIONet *n) 1231 { 1232 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1233 ebpf_rss_unload(&n->ebpf_rss); 1234 } 1235 1236 static uint16_t virtio_net_handle_rss(VirtIONet *n, 1237 struct iovec *iov, 1238 unsigned int iov_cnt, 1239 bool do_rss) 1240 { 1241 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1242 struct virtio_net_rss_config cfg; 1243 size_t s, offset = 0, size_get; 1244 uint16_t queue_pairs, i; 1245 struct { 1246 uint16_t us; 1247 uint8_t b; 1248 } QEMU_PACKED temp; 1249 const char *err_msg = ""; 1250 uint32_t err_value = 0; 1251 1252 if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) { 1253 err_msg = "RSS is not negotiated"; 1254 goto error; 1255 } 1256 if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) { 1257 err_msg = "Hash report is not negotiated"; 1258 goto error; 1259 } 1260 size_get = offsetof(struct virtio_net_rss_config, indirection_table); 1261 s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get); 1262 if (s != size_get) { 1263 err_msg = "Short command buffer"; 1264 err_value = (uint32_t)s; 1265 goto error; 1266 } 1267 n->rss_data.hash_types = virtio_ldl_p(vdev, &cfg.hash_types); 1268 n->rss_data.indirections_len = 1269 virtio_lduw_p(vdev, &cfg.indirection_table_mask); 1270 n->rss_data.indirections_len++; 1271 if (!do_rss) { 1272 n->rss_data.indirections_len = 1; 1273 } 1274 if (!is_power_of_2(n->rss_data.indirections_len)) { 1275 err_msg = "Invalid size of indirection table"; 1276 err_value = n->rss_data.indirections_len; 1277 goto error; 1278 } 1279 if (n->rss_data.indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 1280 err_msg = "Too large indirection table"; 1281 err_value = n->rss_data.indirections_len; 1282 goto error; 1283 } 1284 n->rss_data.default_queue = do_rss ? 1285 virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0; 1286 if (n->rss_data.default_queue >= n->max_queue_pairs) { 1287 err_msg = "Invalid default queue"; 1288 err_value = n->rss_data.default_queue; 1289 goto error; 1290 } 1291 offset += size_get; 1292 size_get = sizeof(uint16_t) * n->rss_data.indirections_len; 1293 g_free(n->rss_data.indirections_table); 1294 n->rss_data.indirections_table = g_malloc(size_get); 1295 if (!n->rss_data.indirections_table) { 1296 err_msg = "Can't allocate indirections table"; 1297 err_value = n->rss_data.indirections_len; 1298 goto error; 1299 } 1300 s = iov_to_buf(iov, iov_cnt, offset, 1301 n->rss_data.indirections_table, size_get); 1302 if (s != size_get) { 1303 err_msg = "Short indirection table buffer"; 1304 err_value = (uint32_t)s; 1305 goto error; 1306 } 1307 for (i = 0; i < n->rss_data.indirections_len; ++i) { 1308 uint16_t val = n->rss_data.indirections_table[i]; 1309 n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val); 1310 } 1311 offset += size_get; 1312 size_get = sizeof(temp); 1313 s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get); 1314 if (s != size_get) { 1315 err_msg = "Can't get queue_pairs"; 1316 err_value = (uint32_t)s; 1317 goto error; 1318 } 1319 queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs; 1320 if (queue_pairs == 0 || queue_pairs > n->max_queue_pairs) { 1321 err_msg = "Invalid number of queue_pairs"; 1322 err_value = queue_pairs; 1323 goto error; 1324 } 1325 if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) { 1326 err_msg = "Invalid key size"; 1327 err_value = temp.b; 1328 goto error; 1329 } 1330 if (!temp.b && n->rss_data.hash_types) { 1331 err_msg = "No key provided"; 1332 err_value = 0; 1333 goto error; 1334 } 1335 if (!temp.b && !n->rss_data.hash_types) { 1336 virtio_net_disable_rss(n); 1337 return queue_pairs; 1338 } 1339 offset += size_get; 1340 size_get = temp.b; 1341 s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get); 1342 if (s != size_get) { 1343 err_msg = "Can get key buffer"; 1344 err_value = (uint32_t)s; 1345 goto error; 1346 } 1347 n->rss_data.enabled = true; 1348 1349 if (!n->rss_data.populate_hash) { 1350 if (!virtio_net_attach_epbf_rss(n)) { 1351 /* EBPF must be loaded for vhost */ 1352 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 1353 warn_report("Can't load eBPF RSS for vhost"); 1354 goto error; 1355 } 1356 /* fallback to software RSS */ 1357 warn_report("Can't load eBPF RSS - fallback to software RSS"); 1358 n->rss_data.enabled_software_rss = true; 1359 } 1360 } else { 1361 /* use software RSS for hash populating */ 1362 /* and detach eBPF if was loaded before */ 1363 virtio_net_detach_epbf_rss(n); 1364 n->rss_data.enabled_software_rss = true; 1365 } 1366 1367 trace_virtio_net_rss_enable(n->rss_data.hash_types, 1368 n->rss_data.indirections_len, 1369 temp.b); 1370 return queue_pairs; 1371 error: 1372 trace_virtio_net_rss_error(err_msg, err_value); 1373 virtio_net_disable_rss(n); 1374 return 0; 1375 } 1376 1377 static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd, 1378 struct iovec *iov, unsigned int iov_cnt) 1379 { 1380 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1381 uint16_t queue_pairs; 1382 1383 virtio_net_disable_rss(n); 1384 if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) { 1385 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, false); 1386 return queue_pairs ? VIRTIO_NET_OK : VIRTIO_NET_ERR; 1387 } 1388 if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 1389 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, true); 1390 } else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 1391 struct virtio_net_ctrl_mq mq; 1392 size_t s; 1393 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) { 1394 return VIRTIO_NET_ERR; 1395 } 1396 s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq)); 1397 if (s != sizeof(mq)) { 1398 return VIRTIO_NET_ERR; 1399 } 1400 queue_pairs = virtio_lduw_p(vdev, &mq.virtqueue_pairs); 1401 1402 } else { 1403 return VIRTIO_NET_ERR; 1404 } 1405 1406 if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1407 queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 1408 queue_pairs > n->max_queue_pairs || 1409 !n->multiqueue) { 1410 return VIRTIO_NET_ERR; 1411 } 1412 1413 n->curr_queue_pairs = queue_pairs; 1414 /* stop the backend before changing the number of queue_pairs to avoid handling a 1415 * disabled queue */ 1416 virtio_net_set_status(vdev, vdev->status); 1417 virtio_net_set_queue_pairs(n); 1418 1419 return VIRTIO_NET_OK; 1420 } 1421 1422 static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 1423 { 1424 VirtIONet *n = VIRTIO_NET(vdev); 1425 struct virtio_net_ctrl_hdr ctrl; 1426 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1427 VirtQueueElement *elem; 1428 size_t s; 1429 struct iovec *iov, *iov2; 1430 unsigned int iov_cnt; 1431 1432 for (;;) { 1433 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 1434 if (!elem) { 1435 break; 1436 } 1437 if (iov_size(elem->in_sg, elem->in_num) < sizeof(status) || 1438 iov_size(elem->out_sg, elem->out_num) < sizeof(ctrl)) { 1439 virtio_error(vdev, "virtio-net ctrl missing headers"); 1440 virtqueue_detach_element(vq, elem, 0); 1441 g_free(elem); 1442 break; 1443 } 1444 1445 iov_cnt = elem->out_num; 1446 iov2 = iov = g_memdup2(elem->out_sg, 1447 sizeof(struct iovec) * elem->out_num); 1448 s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl)); 1449 iov_discard_front(&iov, &iov_cnt, sizeof(ctrl)); 1450 if (s != sizeof(ctrl)) { 1451 status = VIRTIO_NET_ERR; 1452 } else if (ctrl.class == VIRTIO_NET_CTRL_RX) { 1453 status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, iov_cnt); 1454 } else if (ctrl.class == VIRTIO_NET_CTRL_MAC) { 1455 status = virtio_net_handle_mac(n, ctrl.cmd, iov, iov_cnt); 1456 } else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) { 1457 status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, iov_cnt); 1458 } else if (ctrl.class == VIRTIO_NET_CTRL_ANNOUNCE) { 1459 status = virtio_net_handle_announce(n, ctrl.cmd, iov, iov_cnt); 1460 } else if (ctrl.class == VIRTIO_NET_CTRL_MQ) { 1461 status = virtio_net_handle_mq(n, ctrl.cmd, iov, iov_cnt); 1462 } else if (ctrl.class == VIRTIO_NET_CTRL_GUEST_OFFLOADS) { 1463 status = virtio_net_handle_offloads(n, ctrl.cmd, iov, iov_cnt); 1464 } 1465 1466 s = iov_from_buf(elem->in_sg, elem->in_num, 0, &status, sizeof(status)); 1467 assert(s == sizeof(status)); 1468 1469 virtqueue_push(vq, elem, sizeof(status)); 1470 virtio_notify(vdev, vq); 1471 g_free(iov2); 1472 g_free(elem); 1473 } 1474 } 1475 1476 /* RX */ 1477 1478 static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq) 1479 { 1480 VirtIONet *n = VIRTIO_NET(vdev); 1481 int queue_index = vq2q(virtio_get_queue_index(vq)); 1482 1483 qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index)); 1484 } 1485 1486 static bool virtio_net_can_receive(NetClientState *nc) 1487 { 1488 VirtIONet *n = qemu_get_nic_opaque(nc); 1489 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1490 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1491 1492 if (!vdev->vm_running) { 1493 return false; 1494 } 1495 1496 if (nc->queue_index >= n->curr_queue_pairs) { 1497 return false; 1498 } 1499 1500 if (!virtio_queue_ready(q->rx_vq) || 1501 !(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 1502 return false; 1503 } 1504 1505 return true; 1506 } 1507 1508 static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize) 1509 { 1510 VirtIONet *n = q->n; 1511 if (virtio_queue_empty(q->rx_vq) || 1512 (n->mergeable_rx_bufs && 1513 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1514 virtio_queue_set_notification(q->rx_vq, 1); 1515 1516 /* To avoid a race condition where the guest has made some buffers 1517 * available after the above check but before notification was 1518 * enabled, check for available buffers again. 1519 */ 1520 if (virtio_queue_empty(q->rx_vq) || 1521 (n->mergeable_rx_bufs && 1522 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1523 return 0; 1524 } 1525 } 1526 1527 virtio_queue_set_notification(q->rx_vq, 0); 1528 return 1; 1529 } 1530 1531 static void virtio_net_hdr_swap(VirtIODevice *vdev, struct virtio_net_hdr *hdr) 1532 { 1533 virtio_tswap16s(vdev, &hdr->hdr_len); 1534 virtio_tswap16s(vdev, &hdr->gso_size); 1535 virtio_tswap16s(vdev, &hdr->csum_start); 1536 virtio_tswap16s(vdev, &hdr->csum_offset); 1537 } 1538 1539 /* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so 1540 * it never finds out that the packets don't have valid checksums. This 1541 * causes dhclient to get upset. Fedora's carried a patch for ages to 1542 * fix this with Xen but it hasn't appeared in an upstream release of 1543 * dhclient yet. 1544 * 1545 * To avoid breaking existing guests, we catch udp packets and add 1546 * checksums. This is terrible but it's better than hacking the guest 1547 * kernels. 1548 * 1549 * N.B. if we introduce a zero-copy API, this operation is no longer free so 1550 * we should provide a mechanism to disable it to avoid polluting the host 1551 * cache. 1552 */ 1553 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr, 1554 uint8_t *buf, size_t size) 1555 { 1556 if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */ 1557 (size > 27 && size < 1500) && /* normal sized MTU */ 1558 (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */ 1559 (buf[23] == 17) && /* ip.protocol == UDP */ 1560 (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */ 1561 net_checksum_calculate(buf, size, CSUM_UDP); 1562 hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM; 1563 } 1564 } 1565 1566 static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt, 1567 const void *buf, size_t size) 1568 { 1569 if (n->has_vnet_hdr) { 1570 /* FIXME this cast is evil */ 1571 void *wbuf = (void *)buf; 1572 work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len, 1573 size - n->host_hdr_len); 1574 1575 if (n->needs_vnet_hdr_swap) { 1576 virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf); 1577 } 1578 iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr)); 1579 } else { 1580 struct virtio_net_hdr hdr = { 1581 .flags = 0, 1582 .gso_type = VIRTIO_NET_HDR_GSO_NONE 1583 }; 1584 iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr); 1585 } 1586 } 1587 1588 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size) 1589 { 1590 static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 1591 static const uint8_t vlan[] = {0x81, 0x00}; 1592 uint8_t *ptr = (uint8_t *)buf; 1593 int i; 1594 1595 if (n->promisc) 1596 return 1; 1597 1598 ptr += n->host_hdr_len; 1599 1600 if (!memcmp(&ptr[12], vlan, sizeof(vlan))) { 1601 int vid = lduw_be_p(ptr + 14) & 0xfff; 1602 if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f)))) 1603 return 0; 1604 } 1605 1606 if (ptr[0] & 1) { // multicast 1607 if (!memcmp(ptr, bcast, sizeof(bcast))) { 1608 return !n->nobcast; 1609 } else if (n->nomulti) { 1610 return 0; 1611 } else if (n->allmulti || n->mac_table.multi_overflow) { 1612 return 1; 1613 } 1614 1615 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 1616 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1617 return 1; 1618 } 1619 } 1620 } else { // unicast 1621 if (n->nouni) { 1622 return 0; 1623 } else if (n->alluni || n->mac_table.uni_overflow) { 1624 return 1; 1625 } else if (!memcmp(ptr, n->mac, ETH_ALEN)) { 1626 return 1; 1627 } 1628 1629 for (i = 0; i < n->mac_table.first_multi; i++) { 1630 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1631 return 1; 1632 } 1633 } 1634 } 1635 1636 return 0; 1637 } 1638 1639 static uint8_t virtio_net_get_hash_type(bool isip4, 1640 bool isip6, 1641 bool isudp, 1642 bool istcp, 1643 uint32_t types) 1644 { 1645 if (isip4) { 1646 if (istcp && (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4)) { 1647 return NetPktRssIpV4Tcp; 1648 } 1649 if (isudp && (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4)) { 1650 return NetPktRssIpV4Udp; 1651 } 1652 if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 1653 return NetPktRssIpV4; 1654 } 1655 } else if (isip6) { 1656 uint32_t mask = VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 1657 VIRTIO_NET_RSS_HASH_TYPE_TCPv6; 1658 1659 if (istcp && (types & mask)) { 1660 return (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) ? 1661 NetPktRssIpV6TcpEx : NetPktRssIpV6Tcp; 1662 } 1663 mask = VIRTIO_NET_RSS_HASH_TYPE_UDP_EX | VIRTIO_NET_RSS_HASH_TYPE_UDPv6; 1664 if (isudp && (types & mask)) { 1665 return (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) ? 1666 NetPktRssIpV6UdpEx : NetPktRssIpV6Udp; 1667 } 1668 mask = VIRTIO_NET_RSS_HASH_TYPE_IP_EX | VIRTIO_NET_RSS_HASH_TYPE_IPv6; 1669 if (types & mask) { 1670 return (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) ? 1671 NetPktRssIpV6Ex : NetPktRssIpV6; 1672 } 1673 } 1674 return 0xff; 1675 } 1676 1677 static void virtio_set_packet_hash(const uint8_t *buf, uint8_t report, 1678 uint32_t hash) 1679 { 1680 struct virtio_net_hdr_v1_hash *hdr = (void *)buf; 1681 hdr->hash_value = hash; 1682 hdr->hash_report = report; 1683 } 1684 1685 static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, 1686 size_t size) 1687 { 1688 VirtIONet *n = qemu_get_nic_opaque(nc); 1689 unsigned int index = nc->queue_index, new_index = index; 1690 struct NetRxPkt *pkt = n->rx_pkt; 1691 uint8_t net_hash_type; 1692 uint32_t hash; 1693 bool isip4, isip6, isudp, istcp; 1694 static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = { 1695 VIRTIO_NET_HASH_REPORT_IPv4, 1696 VIRTIO_NET_HASH_REPORT_TCPv4, 1697 VIRTIO_NET_HASH_REPORT_TCPv6, 1698 VIRTIO_NET_HASH_REPORT_IPv6, 1699 VIRTIO_NET_HASH_REPORT_IPv6_EX, 1700 VIRTIO_NET_HASH_REPORT_TCPv6_EX, 1701 VIRTIO_NET_HASH_REPORT_UDPv4, 1702 VIRTIO_NET_HASH_REPORT_UDPv6, 1703 VIRTIO_NET_HASH_REPORT_UDPv6_EX 1704 }; 1705 1706 net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len, 1707 size - n->host_hdr_len); 1708 net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); 1709 if (isip4 && (net_rx_pkt_get_ip4_info(pkt)->fragment)) { 1710 istcp = isudp = false; 1711 } 1712 if (isip6 && (net_rx_pkt_get_ip6_info(pkt)->fragment)) { 1713 istcp = isudp = false; 1714 } 1715 net_hash_type = virtio_net_get_hash_type(isip4, isip6, isudp, istcp, 1716 n->rss_data.hash_types); 1717 if (net_hash_type > NetPktRssIpV6UdpEx) { 1718 if (n->rss_data.populate_hash) { 1719 virtio_set_packet_hash(buf, VIRTIO_NET_HASH_REPORT_NONE, 0); 1720 } 1721 return n->rss_data.redirect ? n->rss_data.default_queue : -1; 1722 } 1723 1724 hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key); 1725 1726 if (n->rss_data.populate_hash) { 1727 virtio_set_packet_hash(buf, reports[net_hash_type], hash); 1728 } 1729 1730 if (n->rss_data.redirect) { 1731 new_index = hash & (n->rss_data.indirections_len - 1); 1732 new_index = n->rss_data.indirections_table[new_index]; 1733 } 1734 1735 return (index == new_index) ? -1 : new_index; 1736 } 1737 1738 static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, 1739 size_t size, bool no_rss) 1740 { 1741 VirtIONet *n = qemu_get_nic_opaque(nc); 1742 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1743 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1744 VirtQueueElement *elems[VIRTQUEUE_MAX_SIZE]; 1745 size_t lens[VIRTQUEUE_MAX_SIZE]; 1746 struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE]; 1747 struct virtio_net_hdr_mrg_rxbuf mhdr; 1748 unsigned mhdr_cnt = 0; 1749 size_t offset, i, guest_offset, j; 1750 ssize_t err; 1751 1752 if (!virtio_net_can_receive(nc)) { 1753 return -1; 1754 } 1755 1756 if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) { 1757 int index = virtio_net_process_rss(nc, buf, size); 1758 if (index >= 0) { 1759 NetClientState *nc2 = qemu_get_subqueue(n->nic, index); 1760 return virtio_net_receive_rcu(nc2, buf, size, true); 1761 } 1762 } 1763 1764 /* hdr_len refers to the header we supply to the guest */ 1765 if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) { 1766 return 0; 1767 } 1768 1769 if (!receive_filter(n, buf, size)) 1770 return size; 1771 1772 offset = i = 0; 1773 1774 while (offset < size) { 1775 VirtQueueElement *elem; 1776 int len, total; 1777 const struct iovec *sg; 1778 1779 total = 0; 1780 1781 if (i == VIRTQUEUE_MAX_SIZE) { 1782 virtio_error(vdev, "virtio-net unexpected long buffer chain"); 1783 err = size; 1784 goto err; 1785 } 1786 1787 elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement)); 1788 if (!elem) { 1789 if (i) { 1790 virtio_error(vdev, "virtio-net unexpected empty queue: " 1791 "i %zd mergeable %d offset %zd, size %zd, " 1792 "guest hdr len %zd, host hdr len %zd " 1793 "guest features 0x%" PRIx64, 1794 i, n->mergeable_rx_bufs, offset, size, 1795 n->guest_hdr_len, n->host_hdr_len, 1796 vdev->guest_features); 1797 } 1798 err = -1; 1799 goto err; 1800 } 1801 1802 if (elem->in_num < 1) { 1803 virtio_error(vdev, 1804 "virtio-net receive queue contains no in buffers"); 1805 virtqueue_detach_element(q->rx_vq, elem, 0); 1806 g_free(elem); 1807 err = -1; 1808 goto err; 1809 } 1810 1811 sg = elem->in_sg; 1812 if (i == 0) { 1813 assert(offset == 0); 1814 if (n->mergeable_rx_bufs) { 1815 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg), 1816 sg, elem->in_num, 1817 offsetof(typeof(mhdr), num_buffers), 1818 sizeof(mhdr.num_buffers)); 1819 } 1820 1821 receive_header(n, sg, elem->in_num, buf, size); 1822 if (n->rss_data.populate_hash) { 1823 offset = sizeof(mhdr); 1824 iov_from_buf(sg, elem->in_num, offset, 1825 buf + offset, n->host_hdr_len - sizeof(mhdr)); 1826 } 1827 offset = n->host_hdr_len; 1828 total += n->guest_hdr_len; 1829 guest_offset = n->guest_hdr_len; 1830 } else { 1831 guest_offset = 0; 1832 } 1833 1834 /* copy in packet. ugh */ 1835 len = iov_from_buf(sg, elem->in_num, guest_offset, 1836 buf + offset, size - offset); 1837 total += len; 1838 offset += len; 1839 /* If buffers can't be merged, at this point we 1840 * must have consumed the complete packet. 1841 * Otherwise, drop it. */ 1842 if (!n->mergeable_rx_bufs && offset < size) { 1843 virtqueue_unpop(q->rx_vq, elem, total); 1844 g_free(elem); 1845 err = size; 1846 goto err; 1847 } 1848 1849 elems[i] = elem; 1850 lens[i] = total; 1851 i++; 1852 } 1853 1854 if (mhdr_cnt) { 1855 virtio_stw_p(vdev, &mhdr.num_buffers, i); 1856 iov_from_buf(mhdr_sg, mhdr_cnt, 1857 0, 1858 &mhdr.num_buffers, sizeof mhdr.num_buffers); 1859 } 1860 1861 for (j = 0; j < i; j++) { 1862 /* signal other side */ 1863 virtqueue_fill(q->rx_vq, elems[j], lens[j], j); 1864 g_free(elems[j]); 1865 } 1866 1867 virtqueue_flush(q->rx_vq, i); 1868 virtio_notify(vdev, q->rx_vq); 1869 1870 return size; 1871 1872 err: 1873 for (j = 0; j < i; j++) { 1874 virtqueue_detach_element(q->rx_vq, elems[j], lens[j]); 1875 g_free(elems[j]); 1876 } 1877 1878 return err; 1879 } 1880 1881 static ssize_t virtio_net_do_receive(NetClientState *nc, const uint8_t *buf, 1882 size_t size) 1883 { 1884 RCU_READ_LOCK_GUARD(); 1885 1886 return virtio_net_receive_rcu(nc, buf, size, false); 1887 } 1888 1889 static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain, 1890 const uint8_t *buf, 1891 VirtioNetRscUnit *unit) 1892 { 1893 uint16_t ip_hdrlen; 1894 struct ip_header *ip; 1895 1896 ip = (struct ip_header *)(buf + chain->n->guest_hdr_len 1897 + sizeof(struct eth_header)); 1898 unit->ip = (void *)ip; 1899 ip_hdrlen = (ip->ip_ver_len & 0xF) << 2; 1900 unit->ip_plen = &ip->ip_len; 1901 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) + ip_hdrlen); 1902 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 1903 unit->payload = htons(*unit->ip_plen) - ip_hdrlen - unit->tcp_hdrlen; 1904 } 1905 1906 static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain, 1907 const uint8_t *buf, 1908 VirtioNetRscUnit *unit) 1909 { 1910 struct ip6_header *ip6; 1911 1912 ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len 1913 + sizeof(struct eth_header)); 1914 unit->ip = ip6; 1915 unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 1916 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) 1917 + sizeof(struct ip6_header)); 1918 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 1919 1920 /* There is a difference between payload lenght in ipv4 and v6, 1921 ip header is excluded in ipv6 */ 1922 unit->payload = htons(*unit->ip_plen) - unit->tcp_hdrlen; 1923 } 1924 1925 static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain, 1926 VirtioNetRscSeg *seg) 1927 { 1928 int ret; 1929 struct virtio_net_hdr_v1 *h; 1930 1931 h = (struct virtio_net_hdr_v1 *)seg->buf; 1932 h->flags = 0; 1933 h->gso_type = VIRTIO_NET_HDR_GSO_NONE; 1934 1935 if (seg->is_coalesced) { 1936 h->rsc.segments = seg->packets; 1937 h->rsc.dup_acks = seg->dup_ack; 1938 h->flags = VIRTIO_NET_HDR_F_RSC_INFO; 1939 if (chain->proto == ETH_P_IP) { 1940 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 1941 } else { 1942 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 1943 } 1944 } 1945 1946 ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size); 1947 QTAILQ_REMOVE(&chain->buffers, seg, next); 1948 g_free(seg->buf); 1949 g_free(seg); 1950 1951 return ret; 1952 } 1953 1954 static void virtio_net_rsc_purge(void *opq) 1955 { 1956 VirtioNetRscSeg *seg, *rn; 1957 VirtioNetRscChain *chain = (VirtioNetRscChain *)opq; 1958 1959 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) { 1960 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 1961 chain->stat.purge_failed++; 1962 continue; 1963 } 1964 } 1965 1966 chain->stat.timer++; 1967 if (!QTAILQ_EMPTY(&chain->buffers)) { 1968 timer_mod(chain->drain_timer, 1969 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 1970 } 1971 } 1972 1973 static void virtio_net_rsc_cleanup(VirtIONet *n) 1974 { 1975 VirtioNetRscChain *chain, *rn_chain; 1976 VirtioNetRscSeg *seg, *rn_seg; 1977 1978 QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) { 1979 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) { 1980 QTAILQ_REMOVE(&chain->buffers, seg, next); 1981 g_free(seg->buf); 1982 g_free(seg); 1983 } 1984 1985 timer_free(chain->drain_timer); 1986 QTAILQ_REMOVE(&n->rsc_chains, chain, next); 1987 g_free(chain); 1988 } 1989 } 1990 1991 static void virtio_net_rsc_cache_buf(VirtioNetRscChain *chain, 1992 NetClientState *nc, 1993 const uint8_t *buf, size_t size) 1994 { 1995 uint16_t hdr_len; 1996 VirtioNetRscSeg *seg; 1997 1998 hdr_len = chain->n->guest_hdr_len; 1999 seg = g_new(VirtioNetRscSeg, 1); 2000 seg->buf = g_malloc(hdr_len + sizeof(struct eth_header) 2001 + sizeof(struct ip6_header) + VIRTIO_NET_MAX_TCP_PAYLOAD); 2002 memcpy(seg->buf, buf, size); 2003 seg->size = size; 2004 seg->packets = 1; 2005 seg->dup_ack = 0; 2006 seg->is_coalesced = 0; 2007 seg->nc = nc; 2008 2009 QTAILQ_INSERT_TAIL(&chain->buffers, seg, next); 2010 chain->stat.cache++; 2011 2012 switch (chain->proto) { 2013 case ETH_P_IP: 2014 virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit); 2015 break; 2016 case ETH_P_IPV6: 2017 virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit); 2018 break; 2019 default: 2020 g_assert_not_reached(); 2021 } 2022 } 2023 2024 static int32_t virtio_net_rsc_handle_ack(VirtioNetRscChain *chain, 2025 VirtioNetRscSeg *seg, 2026 const uint8_t *buf, 2027 struct tcp_header *n_tcp, 2028 struct tcp_header *o_tcp) 2029 { 2030 uint32_t nack, oack; 2031 uint16_t nwin, owin; 2032 2033 nack = htonl(n_tcp->th_ack); 2034 nwin = htons(n_tcp->th_win); 2035 oack = htonl(o_tcp->th_ack); 2036 owin = htons(o_tcp->th_win); 2037 2038 if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) { 2039 chain->stat.ack_out_of_win++; 2040 return RSC_FINAL; 2041 } else if (nack == oack) { 2042 /* duplicated ack or window probe */ 2043 if (nwin == owin) { 2044 /* duplicated ack, add dup ack count due to whql test up to 1 */ 2045 chain->stat.dup_ack++; 2046 return RSC_FINAL; 2047 } else { 2048 /* Coalesce window update */ 2049 o_tcp->th_win = n_tcp->th_win; 2050 chain->stat.win_update++; 2051 return RSC_COALESCE; 2052 } 2053 } else { 2054 /* pure ack, go to 'C', finalize*/ 2055 chain->stat.pure_ack++; 2056 return RSC_FINAL; 2057 } 2058 } 2059 2060 static int32_t virtio_net_rsc_coalesce_data(VirtioNetRscChain *chain, 2061 VirtioNetRscSeg *seg, 2062 const uint8_t *buf, 2063 VirtioNetRscUnit *n_unit) 2064 { 2065 void *data; 2066 uint16_t o_ip_len; 2067 uint32_t nseq, oseq; 2068 VirtioNetRscUnit *o_unit; 2069 2070 o_unit = &seg->unit; 2071 o_ip_len = htons(*o_unit->ip_plen); 2072 nseq = htonl(n_unit->tcp->th_seq); 2073 oseq = htonl(o_unit->tcp->th_seq); 2074 2075 /* out of order or retransmitted. */ 2076 if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) { 2077 chain->stat.data_out_of_win++; 2078 return RSC_FINAL; 2079 } 2080 2081 data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen; 2082 if (nseq == oseq) { 2083 if ((o_unit->payload == 0) && n_unit->payload) { 2084 /* From no payload to payload, normal case, not a dup ack or etc */ 2085 chain->stat.data_after_pure_ack++; 2086 goto coalesce; 2087 } else { 2088 return virtio_net_rsc_handle_ack(chain, seg, buf, 2089 n_unit->tcp, o_unit->tcp); 2090 } 2091 } else if ((nseq - oseq) != o_unit->payload) { 2092 /* Not a consistent packet, out of order */ 2093 chain->stat.data_out_of_order++; 2094 return RSC_FINAL; 2095 } else { 2096 coalesce: 2097 if ((o_ip_len + n_unit->payload) > chain->max_payload) { 2098 chain->stat.over_size++; 2099 return RSC_FINAL; 2100 } 2101 2102 /* Here comes the right data, the payload length in v4/v6 is different, 2103 so use the field value to update and record the new data len */ 2104 o_unit->payload += n_unit->payload; /* update new data len */ 2105 2106 /* update field in ip header */ 2107 *o_unit->ip_plen = htons(o_ip_len + n_unit->payload); 2108 2109 /* Bring 'PUSH' big, the whql test guide says 'PUSH' can be coalesced 2110 for windows guest, while this may change the behavior for linux 2111 guest (only if it uses RSC feature). */ 2112 o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags; 2113 2114 o_unit->tcp->th_ack = n_unit->tcp->th_ack; 2115 o_unit->tcp->th_win = n_unit->tcp->th_win; 2116 2117 memmove(seg->buf + seg->size, data, n_unit->payload); 2118 seg->size += n_unit->payload; 2119 seg->packets++; 2120 chain->stat.coalesced++; 2121 return RSC_COALESCE; 2122 } 2123 } 2124 2125 static int32_t virtio_net_rsc_coalesce4(VirtioNetRscChain *chain, 2126 VirtioNetRscSeg *seg, 2127 const uint8_t *buf, size_t size, 2128 VirtioNetRscUnit *unit) 2129 { 2130 struct ip_header *ip1, *ip2; 2131 2132 ip1 = (struct ip_header *)(unit->ip); 2133 ip2 = (struct ip_header *)(seg->unit.ip); 2134 if ((ip1->ip_src ^ ip2->ip_src) || (ip1->ip_dst ^ ip2->ip_dst) 2135 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2136 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2137 chain->stat.no_match++; 2138 return RSC_NO_MATCH; 2139 } 2140 2141 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2142 } 2143 2144 static int32_t virtio_net_rsc_coalesce6(VirtioNetRscChain *chain, 2145 VirtioNetRscSeg *seg, 2146 const uint8_t *buf, size_t size, 2147 VirtioNetRscUnit *unit) 2148 { 2149 struct ip6_header *ip1, *ip2; 2150 2151 ip1 = (struct ip6_header *)(unit->ip); 2152 ip2 = (struct ip6_header *)(seg->unit.ip); 2153 if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address)) 2154 || memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address)) 2155 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2156 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2157 chain->stat.no_match++; 2158 return RSC_NO_MATCH; 2159 } 2160 2161 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2162 } 2163 2164 /* Packets with 'SYN' should bypass, other flag should be sent after drain 2165 * to prevent out of order */ 2166 static int virtio_net_rsc_tcp_ctrl_check(VirtioNetRscChain *chain, 2167 struct tcp_header *tcp) 2168 { 2169 uint16_t tcp_hdr; 2170 uint16_t tcp_flag; 2171 2172 tcp_flag = htons(tcp->th_offset_flags); 2173 tcp_hdr = (tcp_flag & VIRTIO_NET_TCP_HDR_LENGTH) >> 10; 2174 tcp_flag &= VIRTIO_NET_TCP_FLAG; 2175 if (tcp_flag & TH_SYN) { 2176 chain->stat.tcp_syn++; 2177 return RSC_BYPASS; 2178 } 2179 2180 if (tcp_flag & (TH_FIN | TH_URG | TH_RST | TH_ECE | TH_CWR)) { 2181 chain->stat.tcp_ctrl_drain++; 2182 return RSC_FINAL; 2183 } 2184 2185 if (tcp_hdr > sizeof(struct tcp_header)) { 2186 chain->stat.tcp_all_opt++; 2187 return RSC_FINAL; 2188 } 2189 2190 return RSC_CANDIDATE; 2191 } 2192 2193 static size_t virtio_net_rsc_do_coalesce(VirtioNetRscChain *chain, 2194 NetClientState *nc, 2195 const uint8_t *buf, size_t size, 2196 VirtioNetRscUnit *unit) 2197 { 2198 int ret; 2199 VirtioNetRscSeg *seg, *nseg; 2200 2201 if (QTAILQ_EMPTY(&chain->buffers)) { 2202 chain->stat.empty_cache++; 2203 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2204 timer_mod(chain->drain_timer, 2205 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 2206 return size; 2207 } 2208 2209 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2210 if (chain->proto == ETH_P_IP) { 2211 ret = virtio_net_rsc_coalesce4(chain, seg, buf, size, unit); 2212 } else { 2213 ret = virtio_net_rsc_coalesce6(chain, seg, buf, size, unit); 2214 } 2215 2216 if (ret == RSC_FINAL) { 2217 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2218 /* Send failed */ 2219 chain->stat.final_failed++; 2220 return 0; 2221 } 2222 2223 /* Send current packet */ 2224 return virtio_net_do_receive(nc, buf, size); 2225 } else if (ret == RSC_NO_MATCH) { 2226 continue; 2227 } else { 2228 /* Coalesced, mark coalesced flag to tell calc cksum for ipv4 */ 2229 seg->is_coalesced = 1; 2230 return size; 2231 } 2232 } 2233 2234 chain->stat.no_match_cache++; 2235 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2236 return size; 2237 } 2238 2239 /* Drain a connection data, this is to avoid out of order segments */ 2240 static size_t virtio_net_rsc_drain_flow(VirtioNetRscChain *chain, 2241 NetClientState *nc, 2242 const uint8_t *buf, size_t size, 2243 uint16_t ip_start, uint16_t ip_size, 2244 uint16_t tcp_port) 2245 { 2246 VirtioNetRscSeg *seg, *nseg; 2247 uint32_t ppair1, ppair2; 2248 2249 ppair1 = *(uint32_t *)(buf + tcp_port); 2250 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2251 ppair2 = *(uint32_t *)(seg->buf + tcp_port); 2252 if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size) 2253 || (ppair1 != ppair2)) { 2254 continue; 2255 } 2256 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2257 chain->stat.drain_failed++; 2258 } 2259 2260 break; 2261 } 2262 2263 return virtio_net_do_receive(nc, buf, size); 2264 } 2265 2266 static int32_t virtio_net_rsc_sanity_check4(VirtioNetRscChain *chain, 2267 struct ip_header *ip, 2268 const uint8_t *buf, size_t size) 2269 { 2270 uint16_t ip_len; 2271 2272 /* Not an ipv4 packet */ 2273 if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) { 2274 chain->stat.ip_option++; 2275 return RSC_BYPASS; 2276 } 2277 2278 /* Don't handle packets with ip option */ 2279 if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) { 2280 chain->stat.ip_option++; 2281 return RSC_BYPASS; 2282 } 2283 2284 if (ip->ip_p != IPPROTO_TCP) { 2285 chain->stat.bypass_not_tcp++; 2286 return RSC_BYPASS; 2287 } 2288 2289 /* Don't handle packets with ip fragment */ 2290 if (!(htons(ip->ip_off) & IP_DF)) { 2291 chain->stat.ip_frag++; 2292 return RSC_BYPASS; 2293 } 2294 2295 /* Don't handle packets with ecn flag */ 2296 if (IPTOS_ECN(ip->ip_tos)) { 2297 chain->stat.ip_ecn++; 2298 return RSC_BYPASS; 2299 } 2300 2301 ip_len = htons(ip->ip_len); 2302 if (ip_len < (sizeof(struct ip_header) + sizeof(struct tcp_header)) 2303 || ip_len > (size - chain->n->guest_hdr_len - 2304 sizeof(struct eth_header))) { 2305 chain->stat.ip_hacked++; 2306 return RSC_BYPASS; 2307 } 2308 2309 return RSC_CANDIDATE; 2310 } 2311 2312 static size_t virtio_net_rsc_receive4(VirtioNetRscChain *chain, 2313 NetClientState *nc, 2314 const uint8_t *buf, size_t size) 2315 { 2316 int32_t ret; 2317 uint16_t hdr_len; 2318 VirtioNetRscUnit unit; 2319 2320 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2321 2322 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header) 2323 + sizeof(struct tcp_header))) { 2324 chain->stat.bypass_not_tcp++; 2325 return virtio_net_do_receive(nc, buf, size); 2326 } 2327 2328 virtio_net_rsc_extract_unit4(chain, buf, &unit); 2329 if (virtio_net_rsc_sanity_check4(chain, unit.ip, buf, size) 2330 != RSC_CANDIDATE) { 2331 return virtio_net_do_receive(nc, buf, size); 2332 } 2333 2334 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2335 if (ret == RSC_BYPASS) { 2336 return virtio_net_do_receive(nc, buf, size); 2337 } else if (ret == RSC_FINAL) { 2338 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2339 ((hdr_len + sizeof(struct eth_header)) + 12), 2340 VIRTIO_NET_IP4_ADDR_SIZE, 2341 hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header)); 2342 } 2343 2344 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2345 } 2346 2347 static int32_t virtio_net_rsc_sanity_check6(VirtioNetRscChain *chain, 2348 struct ip6_header *ip6, 2349 const uint8_t *buf, size_t size) 2350 { 2351 uint16_t ip_len; 2352 2353 if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4) 2354 != IP_HEADER_VERSION_6) { 2355 return RSC_BYPASS; 2356 } 2357 2358 /* Both option and protocol is checked in this */ 2359 if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) { 2360 chain->stat.bypass_not_tcp++; 2361 return RSC_BYPASS; 2362 } 2363 2364 ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 2365 if (ip_len < sizeof(struct tcp_header) || 2366 ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header) 2367 - sizeof(struct ip6_header))) { 2368 chain->stat.ip_hacked++; 2369 return RSC_BYPASS; 2370 } 2371 2372 /* Don't handle packets with ecn flag */ 2373 if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) { 2374 chain->stat.ip_ecn++; 2375 return RSC_BYPASS; 2376 } 2377 2378 return RSC_CANDIDATE; 2379 } 2380 2381 static size_t virtio_net_rsc_receive6(void *opq, NetClientState *nc, 2382 const uint8_t *buf, size_t size) 2383 { 2384 int32_t ret; 2385 uint16_t hdr_len; 2386 VirtioNetRscChain *chain; 2387 VirtioNetRscUnit unit; 2388 2389 chain = (VirtioNetRscChain *)opq; 2390 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2391 2392 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip6_header) 2393 + sizeof(tcp_header))) { 2394 return virtio_net_do_receive(nc, buf, size); 2395 } 2396 2397 virtio_net_rsc_extract_unit6(chain, buf, &unit); 2398 if (RSC_CANDIDATE != virtio_net_rsc_sanity_check6(chain, 2399 unit.ip, buf, size)) { 2400 return virtio_net_do_receive(nc, buf, size); 2401 } 2402 2403 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2404 if (ret == RSC_BYPASS) { 2405 return virtio_net_do_receive(nc, buf, size); 2406 } else if (ret == RSC_FINAL) { 2407 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2408 ((hdr_len + sizeof(struct eth_header)) + 8), 2409 VIRTIO_NET_IP6_ADDR_SIZE, 2410 hdr_len + sizeof(struct eth_header) 2411 + sizeof(struct ip6_header)); 2412 } 2413 2414 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2415 } 2416 2417 static VirtioNetRscChain *virtio_net_rsc_lookup_chain(VirtIONet *n, 2418 NetClientState *nc, 2419 uint16_t proto) 2420 { 2421 VirtioNetRscChain *chain; 2422 2423 if ((proto != (uint16_t)ETH_P_IP) && (proto != (uint16_t)ETH_P_IPV6)) { 2424 return NULL; 2425 } 2426 2427 QTAILQ_FOREACH(chain, &n->rsc_chains, next) { 2428 if (chain->proto == proto) { 2429 return chain; 2430 } 2431 } 2432 2433 chain = g_malloc(sizeof(*chain)); 2434 chain->n = n; 2435 chain->proto = proto; 2436 if (proto == (uint16_t)ETH_P_IP) { 2437 chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD; 2438 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2439 } else { 2440 chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD; 2441 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 2442 } 2443 chain->drain_timer = timer_new_ns(QEMU_CLOCK_HOST, 2444 virtio_net_rsc_purge, chain); 2445 memset(&chain->stat, 0, sizeof(chain->stat)); 2446 2447 QTAILQ_INIT(&chain->buffers); 2448 QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next); 2449 2450 return chain; 2451 } 2452 2453 static ssize_t virtio_net_rsc_receive(NetClientState *nc, 2454 const uint8_t *buf, 2455 size_t size) 2456 { 2457 uint16_t proto; 2458 VirtioNetRscChain *chain; 2459 struct eth_header *eth; 2460 VirtIONet *n; 2461 2462 n = qemu_get_nic_opaque(nc); 2463 if (size < (n->host_hdr_len + sizeof(struct eth_header))) { 2464 return virtio_net_do_receive(nc, buf, size); 2465 } 2466 2467 eth = (struct eth_header *)(buf + n->guest_hdr_len); 2468 proto = htons(eth->h_proto); 2469 2470 chain = virtio_net_rsc_lookup_chain(n, nc, proto); 2471 if (chain) { 2472 chain->stat.received++; 2473 if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) { 2474 return virtio_net_rsc_receive4(chain, nc, buf, size); 2475 } else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) { 2476 return virtio_net_rsc_receive6(chain, nc, buf, size); 2477 } 2478 } 2479 return virtio_net_do_receive(nc, buf, size); 2480 } 2481 2482 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, 2483 size_t size) 2484 { 2485 VirtIONet *n = qemu_get_nic_opaque(nc); 2486 if ((n->rsc4_enabled || n->rsc6_enabled)) { 2487 return virtio_net_rsc_receive(nc, buf, size); 2488 } else { 2489 return virtio_net_do_receive(nc, buf, size); 2490 } 2491 } 2492 2493 static int32_t virtio_net_flush_tx(VirtIONetQueue *q); 2494 2495 static void virtio_net_tx_complete(NetClientState *nc, ssize_t len) 2496 { 2497 VirtIONet *n = qemu_get_nic_opaque(nc); 2498 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 2499 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2500 2501 virtqueue_push(q->tx_vq, q->async_tx.elem, 0); 2502 virtio_notify(vdev, q->tx_vq); 2503 2504 g_free(q->async_tx.elem); 2505 q->async_tx.elem = NULL; 2506 2507 virtio_queue_set_notification(q->tx_vq, 1); 2508 virtio_net_flush_tx(q); 2509 } 2510 2511 /* TX */ 2512 static int32_t virtio_net_flush_tx(VirtIONetQueue *q) 2513 { 2514 VirtIONet *n = q->n; 2515 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2516 VirtQueueElement *elem; 2517 int32_t num_packets = 0; 2518 int queue_index = vq2q(virtio_get_queue_index(q->tx_vq)); 2519 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2520 return num_packets; 2521 } 2522 2523 if (q->async_tx.elem) { 2524 virtio_queue_set_notification(q->tx_vq, 0); 2525 return num_packets; 2526 } 2527 2528 for (;;) { 2529 ssize_t ret; 2530 unsigned int out_num; 2531 struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg; 2532 struct virtio_net_hdr_mrg_rxbuf mhdr; 2533 2534 elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement)); 2535 if (!elem) { 2536 break; 2537 } 2538 2539 out_num = elem->out_num; 2540 out_sg = elem->out_sg; 2541 if (out_num < 1) { 2542 virtio_error(vdev, "virtio-net header not in first element"); 2543 virtqueue_detach_element(q->tx_vq, elem, 0); 2544 g_free(elem); 2545 return -EINVAL; 2546 } 2547 2548 if (n->has_vnet_hdr) { 2549 if (iov_to_buf(out_sg, out_num, 0, &mhdr, n->guest_hdr_len) < 2550 n->guest_hdr_len) { 2551 virtio_error(vdev, "virtio-net header incorrect"); 2552 virtqueue_detach_element(q->tx_vq, elem, 0); 2553 g_free(elem); 2554 return -EINVAL; 2555 } 2556 if (n->needs_vnet_hdr_swap) { 2557 virtio_net_hdr_swap(vdev, (void *) &mhdr); 2558 sg2[0].iov_base = &mhdr; 2559 sg2[0].iov_len = n->guest_hdr_len; 2560 out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1, 2561 out_sg, out_num, 2562 n->guest_hdr_len, -1); 2563 if (out_num == VIRTQUEUE_MAX_SIZE) { 2564 goto drop; 2565 } 2566 out_num += 1; 2567 out_sg = sg2; 2568 } 2569 } 2570 /* 2571 * If host wants to see the guest header as is, we can 2572 * pass it on unchanged. Otherwise, copy just the parts 2573 * that host is interested in. 2574 */ 2575 assert(n->host_hdr_len <= n->guest_hdr_len); 2576 if (n->host_hdr_len != n->guest_hdr_len) { 2577 unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg), 2578 out_sg, out_num, 2579 0, n->host_hdr_len); 2580 sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num, 2581 out_sg, out_num, 2582 n->guest_hdr_len, -1); 2583 out_num = sg_num; 2584 out_sg = sg; 2585 } 2586 2587 ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index), 2588 out_sg, out_num, virtio_net_tx_complete); 2589 if (ret == 0) { 2590 virtio_queue_set_notification(q->tx_vq, 0); 2591 q->async_tx.elem = elem; 2592 return -EBUSY; 2593 } 2594 2595 drop: 2596 virtqueue_push(q->tx_vq, elem, 0); 2597 virtio_notify(vdev, q->tx_vq); 2598 g_free(elem); 2599 2600 if (++num_packets >= n->tx_burst) { 2601 break; 2602 } 2603 } 2604 return num_packets; 2605 } 2606 2607 static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq) 2608 { 2609 VirtIONet *n = VIRTIO_NET(vdev); 2610 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2611 2612 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2613 virtio_net_drop_tx_queue_data(vdev, vq); 2614 return; 2615 } 2616 2617 /* This happens when device was stopped but VCPU wasn't. */ 2618 if (!vdev->vm_running) { 2619 q->tx_waiting = 1; 2620 return; 2621 } 2622 2623 if (q->tx_waiting) { 2624 virtio_queue_set_notification(vq, 1); 2625 timer_del(q->tx_timer); 2626 q->tx_waiting = 0; 2627 if (virtio_net_flush_tx(q) == -EINVAL) { 2628 return; 2629 } 2630 } else { 2631 timer_mod(q->tx_timer, 2632 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2633 q->tx_waiting = 1; 2634 virtio_queue_set_notification(vq, 0); 2635 } 2636 } 2637 2638 static void virtio_net_handle_tx_bh(VirtIODevice *vdev, VirtQueue *vq) 2639 { 2640 VirtIONet *n = VIRTIO_NET(vdev); 2641 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2642 2643 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2644 virtio_net_drop_tx_queue_data(vdev, vq); 2645 return; 2646 } 2647 2648 if (unlikely(q->tx_waiting)) { 2649 return; 2650 } 2651 q->tx_waiting = 1; 2652 /* This happens when device was stopped but VCPU wasn't. */ 2653 if (!vdev->vm_running) { 2654 return; 2655 } 2656 virtio_queue_set_notification(vq, 0); 2657 qemu_bh_schedule(q->tx_bh); 2658 } 2659 2660 static void virtio_net_tx_timer(void *opaque) 2661 { 2662 VirtIONetQueue *q = opaque; 2663 VirtIONet *n = q->n; 2664 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2665 /* This happens when device was stopped but BH wasn't. */ 2666 if (!vdev->vm_running) { 2667 /* Make sure tx waiting is set, so we'll run when restarted. */ 2668 assert(q->tx_waiting); 2669 return; 2670 } 2671 2672 q->tx_waiting = 0; 2673 2674 /* Just in case the driver is not ready on more */ 2675 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2676 return; 2677 } 2678 2679 virtio_queue_set_notification(q->tx_vq, 1); 2680 virtio_net_flush_tx(q); 2681 } 2682 2683 static void virtio_net_tx_bh(void *opaque) 2684 { 2685 VirtIONetQueue *q = opaque; 2686 VirtIONet *n = q->n; 2687 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2688 int32_t ret; 2689 2690 /* This happens when device was stopped but BH wasn't. */ 2691 if (!vdev->vm_running) { 2692 /* Make sure tx waiting is set, so we'll run when restarted. */ 2693 assert(q->tx_waiting); 2694 return; 2695 } 2696 2697 q->tx_waiting = 0; 2698 2699 /* Just in case the driver is not ready on more */ 2700 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 2701 return; 2702 } 2703 2704 ret = virtio_net_flush_tx(q); 2705 if (ret == -EBUSY || ret == -EINVAL) { 2706 return; /* Notification re-enable handled by tx_complete or device 2707 * broken */ 2708 } 2709 2710 /* If we flush a full burst of packets, assume there are 2711 * more coming and immediately reschedule */ 2712 if (ret >= n->tx_burst) { 2713 qemu_bh_schedule(q->tx_bh); 2714 q->tx_waiting = 1; 2715 return; 2716 } 2717 2718 /* If less than a full burst, re-enable notification and flush 2719 * anything that may have come in while we weren't looking. If 2720 * we find something, assume the guest is still active and reschedule */ 2721 virtio_queue_set_notification(q->tx_vq, 1); 2722 ret = virtio_net_flush_tx(q); 2723 if (ret == -EINVAL) { 2724 return; 2725 } else if (ret > 0) { 2726 virtio_queue_set_notification(q->tx_vq, 0); 2727 qemu_bh_schedule(q->tx_bh); 2728 q->tx_waiting = 1; 2729 } 2730 } 2731 2732 static void virtio_net_add_queue(VirtIONet *n, int index) 2733 { 2734 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2735 2736 n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size, 2737 virtio_net_handle_rx); 2738 2739 if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) { 2740 n->vqs[index].tx_vq = 2741 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2742 virtio_net_handle_tx_timer); 2743 n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, 2744 virtio_net_tx_timer, 2745 &n->vqs[index]); 2746 } else { 2747 n->vqs[index].tx_vq = 2748 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2749 virtio_net_handle_tx_bh); 2750 n->vqs[index].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[index]); 2751 } 2752 2753 n->vqs[index].tx_waiting = 0; 2754 n->vqs[index].n = n; 2755 } 2756 2757 static void virtio_net_del_queue(VirtIONet *n, int index) 2758 { 2759 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2760 VirtIONetQueue *q = &n->vqs[index]; 2761 NetClientState *nc = qemu_get_subqueue(n->nic, index); 2762 2763 qemu_purge_queued_packets(nc); 2764 2765 virtio_del_queue(vdev, index * 2); 2766 if (q->tx_timer) { 2767 timer_free(q->tx_timer); 2768 q->tx_timer = NULL; 2769 } else { 2770 qemu_bh_delete(q->tx_bh); 2771 q->tx_bh = NULL; 2772 } 2773 q->tx_waiting = 0; 2774 virtio_del_queue(vdev, index * 2 + 1); 2775 } 2776 2777 static void virtio_net_change_num_queue_pairs(VirtIONet *n, int new_max_queue_pairs) 2778 { 2779 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2780 int old_num_queues = virtio_get_num_queues(vdev); 2781 int new_num_queues = new_max_queue_pairs * 2 + 1; 2782 int i; 2783 2784 assert(old_num_queues >= 3); 2785 assert(old_num_queues % 2 == 1); 2786 2787 if (old_num_queues == new_num_queues) { 2788 return; 2789 } 2790 2791 /* 2792 * We always need to remove and add ctrl vq if 2793 * old_num_queues != new_num_queues. Remove ctrl_vq first, 2794 * and then we only enter one of the following two loops. 2795 */ 2796 virtio_del_queue(vdev, old_num_queues - 1); 2797 2798 for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) { 2799 /* new_num_queues < old_num_queues */ 2800 virtio_net_del_queue(n, i / 2); 2801 } 2802 2803 for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) { 2804 /* new_num_queues > old_num_queues */ 2805 virtio_net_add_queue(n, i / 2); 2806 } 2807 2808 /* add ctrl_vq last */ 2809 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 2810 } 2811 2812 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue) 2813 { 2814 int max = multiqueue ? n->max_queue_pairs : 1; 2815 2816 n->multiqueue = multiqueue; 2817 virtio_net_change_num_queue_pairs(n, max); 2818 2819 virtio_net_set_queue_pairs(n); 2820 } 2821 2822 static int virtio_net_post_load_device(void *opaque, int version_id) 2823 { 2824 VirtIONet *n = opaque; 2825 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2826 int i, link_down; 2827 2828 trace_virtio_net_post_load_device(); 2829 virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs, 2830 virtio_vdev_has_feature(vdev, 2831 VIRTIO_F_VERSION_1), 2832 virtio_vdev_has_feature(vdev, 2833 VIRTIO_NET_F_HASH_REPORT)); 2834 2835 /* MAC_TABLE_ENTRIES may be different from the saved image */ 2836 if (n->mac_table.in_use > MAC_TABLE_ENTRIES) { 2837 n->mac_table.in_use = 0; 2838 } 2839 2840 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 2841 n->curr_guest_offloads = virtio_net_supported_guest_offloads(n); 2842 } 2843 2844 /* 2845 * curr_guest_offloads will be later overwritten by the 2846 * virtio_set_features_nocheck call done from the virtio_load. 2847 * Here we make sure it is preserved and restored accordingly 2848 * in the virtio_net_post_load_virtio callback. 2849 */ 2850 n->saved_guest_offloads = n->curr_guest_offloads; 2851 2852 virtio_net_set_queue_pairs(n); 2853 2854 /* Find the first multicast entry in the saved MAC filter */ 2855 for (i = 0; i < n->mac_table.in_use; i++) { 2856 if (n->mac_table.macs[i * ETH_ALEN] & 1) { 2857 break; 2858 } 2859 } 2860 n->mac_table.first_multi = i; 2861 2862 /* nc.link_down can't be migrated, so infer link_down according 2863 * to link status bit in n->status */ 2864 link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0; 2865 for (i = 0; i < n->max_queue_pairs; i++) { 2866 qemu_get_subqueue(n->nic, i)->link_down = link_down; 2867 } 2868 2869 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 2870 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 2871 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 2872 QEMU_CLOCK_VIRTUAL, 2873 virtio_net_announce_timer, n); 2874 if (n->announce_timer.round) { 2875 timer_mod(n->announce_timer.tm, 2876 qemu_clock_get_ms(n->announce_timer.type)); 2877 } else { 2878 qemu_announce_timer_del(&n->announce_timer, false); 2879 } 2880 } 2881 2882 if (n->rss_data.enabled) { 2883 n->rss_data.enabled_software_rss = n->rss_data.populate_hash; 2884 if (!n->rss_data.populate_hash) { 2885 if (!virtio_net_attach_epbf_rss(n)) { 2886 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 2887 warn_report("Can't post-load eBPF RSS for vhost"); 2888 } else { 2889 warn_report("Can't post-load eBPF RSS - " 2890 "fallback to software RSS"); 2891 n->rss_data.enabled_software_rss = true; 2892 } 2893 } 2894 } 2895 2896 trace_virtio_net_rss_enable(n->rss_data.hash_types, 2897 n->rss_data.indirections_len, 2898 sizeof(n->rss_data.key)); 2899 } else { 2900 trace_virtio_net_rss_disable(); 2901 } 2902 return 0; 2903 } 2904 2905 static int virtio_net_post_load_virtio(VirtIODevice *vdev) 2906 { 2907 VirtIONet *n = VIRTIO_NET(vdev); 2908 /* 2909 * The actual needed state is now in saved_guest_offloads, 2910 * see virtio_net_post_load_device for detail. 2911 * Restore it back and apply the desired offloads. 2912 */ 2913 n->curr_guest_offloads = n->saved_guest_offloads; 2914 if (peer_has_vnet_hdr(n)) { 2915 virtio_net_apply_guest_offloads(n); 2916 } 2917 2918 return 0; 2919 } 2920 2921 /* tx_waiting field of a VirtIONetQueue */ 2922 static const VMStateDescription vmstate_virtio_net_queue_tx_waiting = { 2923 .name = "virtio-net-queue-tx_waiting", 2924 .fields = (VMStateField[]) { 2925 VMSTATE_UINT32(tx_waiting, VirtIONetQueue), 2926 VMSTATE_END_OF_LIST() 2927 }, 2928 }; 2929 2930 static bool max_queue_pairs_gt_1(void *opaque, int version_id) 2931 { 2932 return VIRTIO_NET(opaque)->max_queue_pairs > 1; 2933 } 2934 2935 static bool has_ctrl_guest_offloads(void *opaque, int version_id) 2936 { 2937 return virtio_vdev_has_feature(VIRTIO_DEVICE(opaque), 2938 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2939 } 2940 2941 static bool mac_table_fits(void *opaque, int version_id) 2942 { 2943 return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES; 2944 } 2945 2946 static bool mac_table_doesnt_fit(void *opaque, int version_id) 2947 { 2948 return !mac_table_fits(opaque, version_id); 2949 } 2950 2951 /* This temporary type is shared by all the WITH_TMP methods 2952 * although only some fields are used by each. 2953 */ 2954 struct VirtIONetMigTmp { 2955 VirtIONet *parent; 2956 VirtIONetQueue *vqs_1; 2957 uint16_t curr_queue_pairs_1; 2958 uint8_t has_ufo; 2959 uint32_t has_vnet_hdr; 2960 }; 2961 2962 /* The 2nd and subsequent tx_waiting flags are loaded later than 2963 * the 1st entry in the queue_pairs and only if there's more than one 2964 * entry. We use the tmp mechanism to calculate a temporary 2965 * pointer and count and also validate the count. 2966 */ 2967 2968 static int virtio_net_tx_waiting_pre_save(void *opaque) 2969 { 2970 struct VirtIONetMigTmp *tmp = opaque; 2971 2972 tmp->vqs_1 = tmp->parent->vqs + 1; 2973 tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1; 2974 if (tmp->parent->curr_queue_pairs == 0) { 2975 tmp->curr_queue_pairs_1 = 0; 2976 } 2977 2978 return 0; 2979 } 2980 2981 static int virtio_net_tx_waiting_pre_load(void *opaque) 2982 { 2983 struct VirtIONetMigTmp *tmp = opaque; 2984 2985 /* Reuse the pointer setup from save */ 2986 virtio_net_tx_waiting_pre_save(opaque); 2987 2988 if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) { 2989 error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x", 2990 tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs); 2991 2992 return -EINVAL; 2993 } 2994 2995 return 0; /* all good */ 2996 } 2997 2998 static const VMStateDescription vmstate_virtio_net_tx_waiting = { 2999 .name = "virtio-net-tx_waiting", 3000 .pre_load = virtio_net_tx_waiting_pre_load, 3001 .pre_save = virtio_net_tx_waiting_pre_save, 3002 .fields = (VMStateField[]) { 3003 VMSTATE_STRUCT_VARRAY_POINTER_UINT16(vqs_1, struct VirtIONetMigTmp, 3004 curr_queue_pairs_1, 3005 vmstate_virtio_net_queue_tx_waiting, 3006 struct VirtIONetQueue), 3007 VMSTATE_END_OF_LIST() 3008 }, 3009 }; 3010 3011 /* the 'has_ufo' flag is just tested; if the incoming stream has the 3012 * flag set we need to check that we have it 3013 */ 3014 static int virtio_net_ufo_post_load(void *opaque, int version_id) 3015 { 3016 struct VirtIONetMigTmp *tmp = opaque; 3017 3018 if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) { 3019 error_report("virtio-net: saved image requires TUN_F_UFO support"); 3020 return -EINVAL; 3021 } 3022 3023 return 0; 3024 } 3025 3026 static int virtio_net_ufo_pre_save(void *opaque) 3027 { 3028 struct VirtIONetMigTmp *tmp = opaque; 3029 3030 tmp->has_ufo = tmp->parent->has_ufo; 3031 3032 return 0; 3033 } 3034 3035 static const VMStateDescription vmstate_virtio_net_has_ufo = { 3036 .name = "virtio-net-ufo", 3037 .post_load = virtio_net_ufo_post_load, 3038 .pre_save = virtio_net_ufo_pre_save, 3039 .fields = (VMStateField[]) { 3040 VMSTATE_UINT8(has_ufo, struct VirtIONetMigTmp), 3041 VMSTATE_END_OF_LIST() 3042 }, 3043 }; 3044 3045 /* the 'has_vnet_hdr' flag is just tested; if the incoming stream has the 3046 * flag set we need to check that we have it 3047 */ 3048 static int virtio_net_vnet_post_load(void *opaque, int version_id) 3049 { 3050 struct VirtIONetMigTmp *tmp = opaque; 3051 3052 if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) { 3053 error_report("virtio-net: saved image requires vnet_hdr=on"); 3054 return -EINVAL; 3055 } 3056 3057 return 0; 3058 } 3059 3060 static int virtio_net_vnet_pre_save(void *opaque) 3061 { 3062 struct VirtIONetMigTmp *tmp = opaque; 3063 3064 tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr; 3065 3066 return 0; 3067 } 3068 3069 static const VMStateDescription vmstate_virtio_net_has_vnet = { 3070 .name = "virtio-net-vnet", 3071 .post_load = virtio_net_vnet_post_load, 3072 .pre_save = virtio_net_vnet_pre_save, 3073 .fields = (VMStateField[]) { 3074 VMSTATE_UINT32(has_vnet_hdr, struct VirtIONetMigTmp), 3075 VMSTATE_END_OF_LIST() 3076 }, 3077 }; 3078 3079 static bool virtio_net_rss_needed(void *opaque) 3080 { 3081 return VIRTIO_NET(opaque)->rss_data.enabled; 3082 } 3083 3084 static const VMStateDescription vmstate_virtio_net_rss = { 3085 .name = "virtio-net-device/rss", 3086 .version_id = 1, 3087 .minimum_version_id = 1, 3088 .needed = virtio_net_rss_needed, 3089 .fields = (VMStateField[]) { 3090 VMSTATE_BOOL(rss_data.enabled, VirtIONet), 3091 VMSTATE_BOOL(rss_data.redirect, VirtIONet), 3092 VMSTATE_BOOL(rss_data.populate_hash, VirtIONet), 3093 VMSTATE_UINT32(rss_data.hash_types, VirtIONet), 3094 VMSTATE_UINT16(rss_data.indirections_len, VirtIONet), 3095 VMSTATE_UINT16(rss_data.default_queue, VirtIONet), 3096 VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet, 3097 VIRTIO_NET_RSS_MAX_KEY_SIZE), 3098 VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet, 3099 rss_data.indirections_len, 0, 3100 vmstate_info_uint16, uint16_t), 3101 VMSTATE_END_OF_LIST() 3102 }, 3103 }; 3104 3105 static const VMStateDescription vmstate_virtio_net_device = { 3106 .name = "virtio-net-device", 3107 .version_id = VIRTIO_NET_VM_VERSION, 3108 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3109 .post_load = virtio_net_post_load_device, 3110 .fields = (VMStateField[]) { 3111 VMSTATE_UINT8_ARRAY(mac, VirtIONet, ETH_ALEN), 3112 VMSTATE_STRUCT_POINTER(vqs, VirtIONet, 3113 vmstate_virtio_net_queue_tx_waiting, 3114 VirtIONetQueue), 3115 VMSTATE_UINT32(mergeable_rx_bufs, VirtIONet), 3116 VMSTATE_UINT16(status, VirtIONet), 3117 VMSTATE_UINT8(promisc, VirtIONet), 3118 VMSTATE_UINT8(allmulti, VirtIONet), 3119 VMSTATE_UINT32(mac_table.in_use, VirtIONet), 3120 3121 /* Guarded pair: If it fits we load it, else we throw it away 3122 * - can happen if source has a larger MAC table.; post-load 3123 * sets flags in this case. 3124 */ 3125 VMSTATE_VBUFFER_MULTIPLY(mac_table.macs, VirtIONet, 3126 0, mac_table_fits, mac_table.in_use, 3127 ETH_ALEN), 3128 VMSTATE_UNUSED_VARRAY_UINT32(VirtIONet, mac_table_doesnt_fit, 0, 3129 mac_table.in_use, ETH_ALEN), 3130 3131 /* Note: This is an array of uint32's that's always been saved as a 3132 * buffer; hold onto your endiannesses; it's actually used as a bitmap 3133 * but based on the uint. 3134 */ 3135 VMSTATE_BUFFER_POINTER_UNSAFE(vlans, VirtIONet, 0, MAX_VLAN >> 3), 3136 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3137 vmstate_virtio_net_has_vnet), 3138 VMSTATE_UINT8(mac_table.multi_overflow, VirtIONet), 3139 VMSTATE_UINT8(mac_table.uni_overflow, VirtIONet), 3140 VMSTATE_UINT8(alluni, VirtIONet), 3141 VMSTATE_UINT8(nomulti, VirtIONet), 3142 VMSTATE_UINT8(nouni, VirtIONet), 3143 VMSTATE_UINT8(nobcast, VirtIONet), 3144 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3145 vmstate_virtio_net_has_ufo), 3146 VMSTATE_SINGLE_TEST(max_queue_pairs, VirtIONet, max_queue_pairs_gt_1, 0, 3147 vmstate_info_uint16_equal, uint16_t), 3148 VMSTATE_UINT16_TEST(curr_queue_pairs, VirtIONet, max_queue_pairs_gt_1), 3149 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3150 vmstate_virtio_net_tx_waiting), 3151 VMSTATE_UINT64_TEST(curr_guest_offloads, VirtIONet, 3152 has_ctrl_guest_offloads), 3153 VMSTATE_END_OF_LIST() 3154 }, 3155 .subsections = (const VMStateDescription * []) { 3156 &vmstate_virtio_net_rss, 3157 NULL 3158 } 3159 }; 3160 3161 static NetClientInfo net_virtio_info = { 3162 .type = NET_CLIENT_DRIVER_NIC, 3163 .size = sizeof(NICState), 3164 .can_receive = virtio_net_can_receive, 3165 .receive = virtio_net_receive, 3166 .link_status_changed = virtio_net_set_link_status, 3167 .query_rx_filter = virtio_net_query_rxfilter, 3168 .announce = virtio_net_announce, 3169 }; 3170 3171 static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx) 3172 { 3173 VirtIONet *n = VIRTIO_NET(vdev); 3174 NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3175 assert(n->vhost_started); 3176 return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx); 3177 } 3178 3179 static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx, 3180 bool mask) 3181 { 3182 VirtIONet *n = VIRTIO_NET(vdev); 3183 NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3184 assert(n->vhost_started); 3185 vhost_net_virtqueue_mask(get_vhost_net(nc->peer), 3186 vdev, idx, mask); 3187 } 3188 3189 static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features) 3190 { 3191 virtio_add_feature(&host_features, VIRTIO_NET_F_MAC); 3192 3193 n->config_size = virtio_feature_get_config_size(feature_sizes, 3194 host_features); 3195 } 3196 3197 void virtio_net_set_netclient_name(VirtIONet *n, const char *name, 3198 const char *type) 3199 { 3200 /* 3201 * The name can be NULL, the netclient name will be type.x. 3202 */ 3203 assert(type != NULL); 3204 3205 g_free(n->netclient_name); 3206 g_free(n->netclient_type); 3207 n->netclient_name = g_strdup(name); 3208 n->netclient_type = g_strdup(type); 3209 } 3210 3211 static bool failover_unplug_primary(VirtIONet *n, DeviceState *dev) 3212 { 3213 HotplugHandler *hotplug_ctrl; 3214 PCIDevice *pci_dev; 3215 Error *err = NULL; 3216 3217 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3218 if (hotplug_ctrl) { 3219 pci_dev = PCI_DEVICE(dev); 3220 pci_dev->partially_hotplugged = true; 3221 hotplug_handler_unplug_request(hotplug_ctrl, dev, &err); 3222 if (err) { 3223 error_report_err(err); 3224 return false; 3225 } 3226 } else { 3227 return false; 3228 } 3229 return true; 3230 } 3231 3232 static bool failover_replug_primary(VirtIONet *n, DeviceState *dev, 3233 Error **errp) 3234 { 3235 Error *err = NULL; 3236 HotplugHandler *hotplug_ctrl; 3237 PCIDevice *pdev = PCI_DEVICE(dev); 3238 BusState *primary_bus; 3239 3240 if (!pdev->partially_hotplugged) { 3241 return true; 3242 } 3243 primary_bus = dev->parent_bus; 3244 if (!primary_bus) { 3245 error_setg(errp, "virtio_net: couldn't find primary bus"); 3246 return false; 3247 } 3248 qdev_set_parent_bus(dev, primary_bus, &error_abort); 3249 qatomic_set(&n->failover_primary_hidden, false); 3250 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3251 if (hotplug_ctrl) { 3252 hotplug_handler_pre_plug(hotplug_ctrl, dev, &err); 3253 if (err) { 3254 goto out; 3255 } 3256 hotplug_handler_plug(hotplug_ctrl, dev, &err); 3257 } 3258 pdev->partially_hotplugged = false; 3259 3260 out: 3261 error_propagate(errp, err); 3262 return !err; 3263 } 3264 3265 static void virtio_net_handle_migration_primary(VirtIONet *n, MigrationState *s) 3266 { 3267 bool should_be_hidden; 3268 Error *err = NULL; 3269 DeviceState *dev = failover_find_primary_device(n); 3270 3271 if (!dev) { 3272 return; 3273 } 3274 3275 should_be_hidden = qatomic_read(&n->failover_primary_hidden); 3276 3277 if (migration_in_setup(s) && !should_be_hidden) { 3278 if (failover_unplug_primary(n, dev)) { 3279 vmstate_unregister(VMSTATE_IF(dev), qdev_get_vmsd(dev), dev); 3280 qapi_event_send_unplug_primary(dev->id); 3281 qatomic_set(&n->failover_primary_hidden, true); 3282 } else { 3283 warn_report("couldn't unplug primary device"); 3284 } 3285 } else if (migration_has_failed(s)) { 3286 /* We already unplugged the device let's plug it back */ 3287 if (!failover_replug_primary(n, dev, &err)) { 3288 if (err) { 3289 error_report_err(err); 3290 } 3291 } 3292 } 3293 } 3294 3295 static void virtio_net_migration_state_notifier(Notifier *notifier, void *data) 3296 { 3297 MigrationState *s = data; 3298 VirtIONet *n = container_of(notifier, VirtIONet, migration_state); 3299 virtio_net_handle_migration_primary(n, s); 3300 } 3301 3302 static bool failover_hide_primary_device(DeviceListener *listener, 3303 const QDict *device_opts, 3304 bool from_json, 3305 Error **errp) 3306 { 3307 VirtIONet *n = container_of(listener, VirtIONet, primary_listener); 3308 const char *standby_id; 3309 3310 if (!device_opts) { 3311 return false; 3312 } 3313 3314 if (!qdict_haskey(device_opts, "failover_pair_id")) { 3315 return false; 3316 } 3317 3318 if (!qdict_haskey(device_opts, "id")) { 3319 error_setg(errp, "Device with failover_pair_id needs to have id"); 3320 return false; 3321 } 3322 3323 standby_id = qdict_get_str(device_opts, "failover_pair_id"); 3324 if (g_strcmp0(standby_id, n->netclient_name) != 0) { 3325 return false; 3326 } 3327 3328 /* 3329 * The hide helper can be called several times for a given device. 3330 * Check there is only one primary for a virtio-net device but 3331 * don't duplicate the qdict several times if it's called for the same 3332 * device. 3333 */ 3334 if (n->primary_opts) { 3335 const char *old, *new; 3336 /* devices with failover_pair_id always have an id */ 3337 old = qdict_get_str(n->primary_opts, "id"); 3338 new = qdict_get_str(device_opts, "id"); 3339 if (strcmp(old, new) != 0) { 3340 error_setg(errp, "Cannot attach more than one primary device to " 3341 "'%s': '%s' and '%s'", n->netclient_name, old, new); 3342 return false; 3343 } 3344 } else { 3345 n->primary_opts = qdict_clone_shallow(device_opts); 3346 n->primary_opts_from_json = from_json; 3347 } 3348 3349 /* failover_primary_hidden is set during feature negotiation */ 3350 return qatomic_read(&n->failover_primary_hidden); 3351 } 3352 3353 static void virtio_net_device_realize(DeviceState *dev, Error **errp) 3354 { 3355 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3356 VirtIONet *n = VIRTIO_NET(dev); 3357 NetClientState *nc; 3358 int i; 3359 3360 if (n->net_conf.mtu) { 3361 n->host_features |= (1ULL << VIRTIO_NET_F_MTU); 3362 } 3363 3364 if (n->net_conf.duplex_str) { 3365 if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) { 3366 n->net_conf.duplex = DUPLEX_HALF; 3367 } else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) { 3368 n->net_conf.duplex = DUPLEX_FULL; 3369 } else { 3370 error_setg(errp, "'duplex' must be 'half' or 'full'"); 3371 return; 3372 } 3373 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3374 } else { 3375 n->net_conf.duplex = DUPLEX_UNKNOWN; 3376 } 3377 3378 if (n->net_conf.speed < SPEED_UNKNOWN) { 3379 error_setg(errp, "'speed' must be between 0 and INT_MAX"); 3380 return; 3381 } 3382 if (n->net_conf.speed >= 0) { 3383 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3384 } 3385 3386 if (n->failover) { 3387 n->primary_listener.hide_device = failover_hide_primary_device; 3388 qatomic_set(&n->failover_primary_hidden, true); 3389 device_listener_register(&n->primary_listener); 3390 n->migration_state.notify = virtio_net_migration_state_notifier; 3391 add_migration_state_change_notifier(&n->migration_state); 3392 n->host_features |= (1ULL << VIRTIO_NET_F_STANDBY); 3393 } 3394 3395 virtio_net_set_config_size(n, n->host_features); 3396 virtio_init(vdev, "virtio-net", VIRTIO_ID_NET, n->config_size); 3397 3398 /* 3399 * We set a lower limit on RX queue size to what it always was. 3400 * Guests that want a smaller ring can always resize it without 3401 * help from us (using virtio 1 and up). 3402 */ 3403 if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE || 3404 n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE || 3405 !is_power_of_2(n->net_conf.rx_queue_size)) { 3406 error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), " 3407 "must be a power of 2 between %d and %d.", 3408 n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE, 3409 VIRTQUEUE_MAX_SIZE); 3410 virtio_cleanup(vdev); 3411 return; 3412 } 3413 3414 if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE || 3415 n->net_conf.tx_queue_size > VIRTQUEUE_MAX_SIZE || 3416 !is_power_of_2(n->net_conf.tx_queue_size)) { 3417 error_setg(errp, "Invalid tx_queue_size (= %" PRIu16 "), " 3418 "must be a power of 2 between %d and %d", 3419 n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE, 3420 VIRTQUEUE_MAX_SIZE); 3421 virtio_cleanup(vdev); 3422 return; 3423 } 3424 3425 n->max_ncs = MAX(n->nic_conf.peers.queues, 1); 3426 3427 /* 3428 * Figure out the datapath queue pairs since the backend could 3429 * provide control queue via peers as well. 3430 */ 3431 if (n->nic_conf.peers.queues) { 3432 for (i = 0; i < n->max_ncs; i++) { 3433 if (n->nic_conf.peers.ncs[i]->is_datapath) { 3434 ++n->max_queue_pairs; 3435 } 3436 } 3437 } 3438 n->max_queue_pairs = MAX(n->max_queue_pairs, 1); 3439 3440 if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) { 3441 error_setg(errp, "Invalid number of queue pairs (= %" PRIu32 "), " 3442 "must be a positive integer less than %d.", 3443 n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2); 3444 virtio_cleanup(vdev); 3445 return; 3446 } 3447 n->vqs = g_new0(VirtIONetQueue, n->max_queue_pairs); 3448 n->curr_queue_pairs = 1; 3449 n->tx_timeout = n->net_conf.txtimer; 3450 3451 if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer") 3452 && strcmp(n->net_conf.tx, "bh")) { 3453 warn_report("virtio-net: " 3454 "Unknown option tx=%s, valid options: \"timer\" \"bh\"", 3455 n->net_conf.tx); 3456 error_printf("Defaulting to \"bh\""); 3457 } 3458 3459 n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n), 3460 n->net_conf.tx_queue_size); 3461 3462 for (i = 0; i < n->max_queue_pairs; i++) { 3463 virtio_net_add_queue(n, i); 3464 } 3465 3466 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 3467 qemu_macaddr_default_if_unset(&n->nic_conf.macaddr); 3468 memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac)); 3469 n->status = VIRTIO_NET_S_LINK_UP; 3470 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 3471 QEMU_CLOCK_VIRTUAL, 3472 virtio_net_announce_timer, n); 3473 n->announce_timer.round = 0; 3474 3475 if (n->netclient_type) { 3476 /* 3477 * Happen when virtio_net_set_netclient_name has been called. 3478 */ 3479 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3480 n->netclient_type, n->netclient_name, n); 3481 } else { 3482 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3483 object_get_typename(OBJECT(dev)), dev->id, n); 3484 } 3485 3486 for (i = 0; i < n->max_queue_pairs; i++) { 3487 n->nic->ncs[i].do_not_pad = true; 3488 } 3489 3490 peer_test_vnet_hdr(n); 3491 if (peer_has_vnet_hdr(n)) { 3492 for (i = 0; i < n->max_queue_pairs; i++) { 3493 qemu_using_vnet_hdr(qemu_get_subqueue(n->nic, i)->peer, true); 3494 } 3495 n->host_hdr_len = sizeof(struct virtio_net_hdr); 3496 } else { 3497 n->host_hdr_len = 0; 3498 } 3499 3500 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a); 3501 3502 n->vqs[0].tx_waiting = 0; 3503 n->tx_burst = n->net_conf.txburst; 3504 virtio_net_set_mrg_rx_bufs(n, 0, 0, 0); 3505 n->promisc = 1; /* for compatibility */ 3506 3507 n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 3508 3509 n->vlans = g_malloc0(MAX_VLAN >> 3); 3510 3511 nc = qemu_get_queue(n->nic); 3512 nc->rxfilter_notify_enabled = 1; 3513 3514 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 3515 struct virtio_net_config netcfg = {}; 3516 memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN); 3517 vhost_net_set_config(get_vhost_net(nc->peer), 3518 (uint8_t *)&netcfg, 0, ETH_ALEN, VHOST_SET_CONFIG_TYPE_MASTER); 3519 } 3520 QTAILQ_INIT(&n->rsc_chains); 3521 n->qdev = dev; 3522 3523 net_rx_pkt_init(&n->rx_pkt, false); 3524 3525 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3526 virtio_net_load_ebpf(n); 3527 } 3528 } 3529 3530 static void virtio_net_device_unrealize(DeviceState *dev) 3531 { 3532 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3533 VirtIONet *n = VIRTIO_NET(dev); 3534 int i, max_queue_pairs; 3535 3536 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3537 virtio_net_unload_ebpf(n); 3538 } 3539 3540 /* This will stop vhost backend if appropriate. */ 3541 virtio_net_set_status(vdev, 0); 3542 3543 g_free(n->netclient_name); 3544 n->netclient_name = NULL; 3545 g_free(n->netclient_type); 3546 n->netclient_type = NULL; 3547 3548 g_free(n->mac_table.macs); 3549 g_free(n->vlans); 3550 3551 if (n->failover) { 3552 qobject_unref(n->primary_opts); 3553 device_listener_unregister(&n->primary_listener); 3554 remove_migration_state_change_notifier(&n->migration_state); 3555 } else { 3556 assert(n->primary_opts == NULL); 3557 } 3558 3559 max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 3560 for (i = 0; i < max_queue_pairs; i++) { 3561 virtio_net_del_queue(n, i); 3562 } 3563 /* delete also control vq */ 3564 virtio_del_queue(vdev, max_queue_pairs * 2); 3565 qemu_announce_timer_del(&n->announce_timer, false); 3566 g_free(n->vqs); 3567 qemu_del_nic(n->nic); 3568 virtio_net_rsc_cleanup(n); 3569 g_free(n->rss_data.indirections_table); 3570 net_rx_pkt_uninit(n->rx_pkt); 3571 virtio_cleanup(vdev); 3572 } 3573 3574 static void virtio_net_instance_init(Object *obj) 3575 { 3576 VirtIONet *n = VIRTIO_NET(obj); 3577 3578 /* 3579 * The default config_size is sizeof(struct virtio_net_config). 3580 * Can be overriden with virtio_net_set_config_size. 3581 */ 3582 n->config_size = sizeof(struct virtio_net_config); 3583 device_add_bootindex_property(obj, &n->nic_conf.bootindex, 3584 "bootindex", "/ethernet-phy@0", 3585 DEVICE(n)); 3586 3587 ebpf_rss_init(&n->ebpf_rss); 3588 } 3589 3590 static int virtio_net_pre_save(void *opaque) 3591 { 3592 VirtIONet *n = opaque; 3593 3594 /* At this point, backend must be stopped, otherwise 3595 * it might keep writing to memory. */ 3596 assert(!n->vhost_started); 3597 3598 return 0; 3599 } 3600 3601 static bool primary_unplug_pending(void *opaque) 3602 { 3603 DeviceState *dev = opaque; 3604 DeviceState *primary; 3605 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3606 VirtIONet *n = VIRTIO_NET(vdev); 3607 3608 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 3609 return false; 3610 } 3611 primary = failover_find_primary_device(n); 3612 return primary ? primary->pending_deleted_event : false; 3613 } 3614 3615 static bool dev_unplug_pending(void *opaque) 3616 { 3617 DeviceState *dev = opaque; 3618 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 3619 3620 return vdc->primary_unplug_pending(dev); 3621 } 3622 3623 static const VMStateDescription vmstate_virtio_net = { 3624 .name = "virtio-net", 3625 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3626 .version_id = VIRTIO_NET_VM_VERSION, 3627 .fields = (VMStateField[]) { 3628 VMSTATE_VIRTIO_DEVICE, 3629 VMSTATE_END_OF_LIST() 3630 }, 3631 .pre_save = virtio_net_pre_save, 3632 .dev_unplug_pending = dev_unplug_pending, 3633 }; 3634 3635 static Property virtio_net_properties[] = { 3636 DEFINE_PROP_BIT64("csum", VirtIONet, host_features, 3637 VIRTIO_NET_F_CSUM, true), 3638 DEFINE_PROP_BIT64("guest_csum", VirtIONet, host_features, 3639 VIRTIO_NET_F_GUEST_CSUM, true), 3640 DEFINE_PROP_BIT64("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true), 3641 DEFINE_PROP_BIT64("guest_tso4", VirtIONet, host_features, 3642 VIRTIO_NET_F_GUEST_TSO4, true), 3643 DEFINE_PROP_BIT64("guest_tso6", VirtIONet, host_features, 3644 VIRTIO_NET_F_GUEST_TSO6, true), 3645 DEFINE_PROP_BIT64("guest_ecn", VirtIONet, host_features, 3646 VIRTIO_NET_F_GUEST_ECN, true), 3647 DEFINE_PROP_BIT64("guest_ufo", VirtIONet, host_features, 3648 VIRTIO_NET_F_GUEST_UFO, true), 3649 DEFINE_PROP_BIT64("guest_announce", VirtIONet, host_features, 3650 VIRTIO_NET_F_GUEST_ANNOUNCE, true), 3651 DEFINE_PROP_BIT64("host_tso4", VirtIONet, host_features, 3652 VIRTIO_NET_F_HOST_TSO4, true), 3653 DEFINE_PROP_BIT64("host_tso6", VirtIONet, host_features, 3654 VIRTIO_NET_F_HOST_TSO6, true), 3655 DEFINE_PROP_BIT64("host_ecn", VirtIONet, host_features, 3656 VIRTIO_NET_F_HOST_ECN, true), 3657 DEFINE_PROP_BIT64("host_ufo", VirtIONet, host_features, 3658 VIRTIO_NET_F_HOST_UFO, true), 3659 DEFINE_PROP_BIT64("mrg_rxbuf", VirtIONet, host_features, 3660 VIRTIO_NET_F_MRG_RXBUF, true), 3661 DEFINE_PROP_BIT64("status", VirtIONet, host_features, 3662 VIRTIO_NET_F_STATUS, true), 3663 DEFINE_PROP_BIT64("ctrl_vq", VirtIONet, host_features, 3664 VIRTIO_NET_F_CTRL_VQ, true), 3665 DEFINE_PROP_BIT64("ctrl_rx", VirtIONet, host_features, 3666 VIRTIO_NET_F_CTRL_RX, true), 3667 DEFINE_PROP_BIT64("ctrl_vlan", VirtIONet, host_features, 3668 VIRTIO_NET_F_CTRL_VLAN, true), 3669 DEFINE_PROP_BIT64("ctrl_rx_extra", VirtIONet, host_features, 3670 VIRTIO_NET_F_CTRL_RX_EXTRA, true), 3671 DEFINE_PROP_BIT64("ctrl_mac_addr", VirtIONet, host_features, 3672 VIRTIO_NET_F_CTRL_MAC_ADDR, true), 3673 DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features, 3674 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true), 3675 DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false), 3676 DEFINE_PROP_BIT64("rss", VirtIONet, host_features, 3677 VIRTIO_NET_F_RSS, false), 3678 DEFINE_PROP_BIT64("hash", VirtIONet, host_features, 3679 VIRTIO_NET_F_HASH_REPORT, false), 3680 DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features, 3681 VIRTIO_NET_F_RSC_EXT, false), 3682 DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout, 3683 VIRTIO_NET_RSC_DEFAULT_INTERVAL), 3684 DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf), 3685 DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer, 3686 TX_TIMER_INTERVAL), 3687 DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST), 3688 DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx), 3689 DEFINE_PROP_UINT16("rx_queue_size", VirtIONet, net_conf.rx_queue_size, 3690 VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE), 3691 DEFINE_PROP_UINT16("tx_queue_size", VirtIONet, net_conf.tx_queue_size, 3692 VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE), 3693 DEFINE_PROP_UINT16("host_mtu", VirtIONet, net_conf.mtu, 0), 3694 DEFINE_PROP_BOOL("x-mtu-bypass-backend", VirtIONet, mtu_bypass_backend, 3695 true), 3696 DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), 3697 DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), 3698 DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), 3699 DEFINE_PROP_END_OF_LIST(), 3700 }; 3701 3702 static void virtio_net_class_init(ObjectClass *klass, void *data) 3703 { 3704 DeviceClass *dc = DEVICE_CLASS(klass); 3705 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 3706 3707 device_class_set_props(dc, virtio_net_properties); 3708 dc->vmsd = &vmstate_virtio_net; 3709 set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); 3710 vdc->realize = virtio_net_device_realize; 3711 vdc->unrealize = virtio_net_device_unrealize; 3712 vdc->get_config = virtio_net_get_config; 3713 vdc->set_config = virtio_net_set_config; 3714 vdc->get_features = virtio_net_get_features; 3715 vdc->set_features = virtio_net_set_features; 3716 vdc->bad_features = virtio_net_bad_features; 3717 vdc->reset = virtio_net_reset; 3718 vdc->set_status = virtio_net_set_status; 3719 vdc->guest_notifier_mask = virtio_net_guest_notifier_mask; 3720 vdc->guest_notifier_pending = virtio_net_guest_notifier_pending; 3721 vdc->legacy_features |= (0x1 << VIRTIO_NET_F_GSO); 3722 vdc->post_load = virtio_net_post_load_virtio; 3723 vdc->vmsd = &vmstate_virtio_net_device; 3724 vdc->primary_unplug_pending = primary_unplug_pending; 3725 } 3726 3727 static const TypeInfo virtio_net_info = { 3728 .name = TYPE_VIRTIO_NET, 3729 .parent = TYPE_VIRTIO_DEVICE, 3730 .instance_size = sizeof(VirtIONet), 3731 .instance_init = virtio_net_instance_init, 3732 .class_init = virtio_net_class_init, 3733 }; 3734 3735 static void virtio_register_types(void) 3736 { 3737 type_register_static(&virtio_net_info); 3738 } 3739 3740 type_init(virtio_register_types) 3741