1 /* 2 * Virtio Network Device 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/atomic.h" 16 #include "qemu/iov.h" 17 #include "qemu/log.h" 18 #include "qemu/main-loop.h" 19 #include "qemu/module.h" 20 #include "hw/virtio/virtio.h" 21 #include "net/net.h" 22 #include "net/checksum.h" 23 #include "net/tap.h" 24 #include "qemu/error-report.h" 25 #include "qemu/timer.h" 26 #include "qemu/option.h" 27 #include "qemu/option_int.h" 28 #include "qemu/config-file.h" 29 #include "qapi/qmp/qdict.h" 30 #include "hw/virtio/virtio-net.h" 31 #include "net/vhost_net.h" 32 #include "net/announce.h" 33 #include "hw/virtio/virtio-bus.h" 34 #include "qapi/error.h" 35 #include "qapi/qapi-events-net.h" 36 #include "hw/qdev-properties.h" 37 #include "qapi/qapi-types-migration.h" 38 #include "qapi/qapi-events-migration.h" 39 #include "hw/virtio/virtio-access.h" 40 #include "migration/misc.h" 41 #include "standard-headers/linux/ethtool.h" 42 #include "sysemu/sysemu.h" 43 #include "trace.h" 44 #include "monitor/qdev.h" 45 #include "hw/pci/pci_device.h" 46 #include "net_rx_pkt.h" 47 #include "hw/virtio/vhost.h" 48 #include "sysemu/qtest.h" 49 50 #define VIRTIO_NET_VM_VERSION 11 51 52 #define MAX_VLAN (1 << 12) /* Per 802.1Q definition */ 53 54 /* previously fixed value */ 55 #define VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 256 56 #define VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 256 57 58 /* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */ 59 #define VIRTIO_NET_RX_QUEUE_MIN_SIZE VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 60 #define VIRTIO_NET_TX_QUEUE_MIN_SIZE VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 61 62 #define VIRTIO_NET_IP4_ADDR_SIZE 8 /* ipv4 saddr + daddr */ 63 64 #define VIRTIO_NET_TCP_FLAG 0x3F 65 #define VIRTIO_NET_TCP_HDR_LENGTH 0xF000 66 67 /* IPv4 max payload, 16 bits in the header */ 68 #define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header)) 69 #define VIRTIO_NET_MAX_TCP_PAYLOAD 65535 70 71 /* header length value in ip header without option */ 72 #define VIRTIO_NET_IP4_HEADER_LENGTH 5 73 74 #define VIRTIO_NET_IP6_ADDR_SIZE 32 /* ipv6 saddr + daddr */ 75 #define VIRTIO_NET_MAX_IP6_PAYLOAD VIRTIO_NET_MAX_TCP_PAYLOAD 76 77 /* Purge coalesced packets timer interval, This value affects the performance 78 a lot, and should be tuned carefully, '300000'(300us) is the recommended 79 value to pass the WHQL test, '50000' can gain 2x netperf throughput with 80 tso/gso/gro 'off'. */ 81 #define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000 82 83 #define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 | \ 84 VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | \ 85 VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | \ 86 VIRTIO_NET_RSS_HASH_TYPE_IPv6 | \ 87 VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | \ 88 VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | \ 89 VIRTIO_NET_RSS_HASH_TYPE_IP_EX | \ 90 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | \ 91 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) 92 93 static const VirtIOFeature feature_sizes[] = { 94 {.flags = 1ULL << VIRTIO_NET_F_MAC, 95 .end = endof(struct virtio_net_config, mac)}, 96 {.flags = 1ULL << VIRTIO_NET_F_STATUS, 97 .end = endof(struct virtio_net_config, status)}, 98 {.flags = 1ULL << VIRTIO_NET_F_MQ, 99 .end = endof(struct virtio_net_config, max_virtqueue_pairs)}, 100 {.flags = 1ULL << VIRTIO_NET_F_MTU, 101 .end = endof(struct virtio_net_config, mtu)}, 102 {.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX, 103 .end = endof(struct virtio_net_config, duplex)}, 104 {.flags = (1ULL << VIRTIO_NET_F_RSS) | (1ULL << VIRTIO_NET_F_HASH_REPORT), 105 .end = endof(struct virtio_net_config, supported_hash_types)}, 106 {} 107 }; 108 109 static const VirtIOConfigSizeParams cfg_size_params = { 110 .min_size = endof(struct virtio_net_config, mac), 111 .max_size = sizeof(struct virtio_net_config), 112 .feature_sizes = feature_sizes 113 }; 114 115 static VirtIONetQueue *virtio_net_get_subqueue(NetClientState *nc) 116 { 117 VirtIONet *n = qemu_get_nic_opaque(nc); 118 119 return &n->vqs[nc->queue_index]; 120 } 121 122 static int vq2q(int queue_index) 123 { 124 return queue_index / 2; 125 } 126 127 static void flush_or_purge_queued_packets(NetClientState *nc) 128 { 129 if (!nc->peer) { 130 return; 131 } 132 133 qemu_flush_or_purge_queued_packets(nc->peer, true); 134 assert(!virtio_net_get_subqueue(nc)->async_tx.elem); 135 } 136 137 /* TODO 138 * - we could suppress RX interrupt if we were so inclined. 139 */ 140 141 static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config) 142 { 143 VirtIONet *n = VIRTIO_NET(vdev); 144 struct virtio_net_config netcfg; 145 NetClientState *nc = qemu_get_queue(n->nic); 146 static const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } }; 147 148 int ret = 0; 149 memset(&netcfg, 0 , sizeof(struct virtio_net_config)); 150 virtio_stw_p(vdev, &netcfg.status, n->status); 151 virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs); 152 virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu); 153 memcpy(netcfg.mac, n->mac, ETH_ALEN); 154 virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed); 155 netcfg.duplex = n->net_conf.duplex; 156 netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE; 157 virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length, 158 virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ? 159 VIRTIO_NET_RSS_MAX_TABLE_LEN : 1); 160 virtio_stl_p(vdev, &netcfg.supported_hash_types, 161 VIRTIO_NET_RSS_SUPPORTED_HASHES); 162 memcpy(config, &netcfg, n->config_size); 163 164 /* 165 * Is this VDPA? No peer means not VDPA: there's no way to 166 * disconnect/reconnect a VDPA peer. 167 */ 168 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 169 ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg, 170 n->config_size); 171 if (ret == -1) { 172 return; 173 } 174 175 /* 176 * Some NIC/kernel combinations present 0 as the mac address. As that 177 * is not a legal address, try to proceed with the address from the 178 * QEMU command line in the hope that the address has been configured 179 * correctly elsewhere - just not reported by the device. 180 */ 181 if (memcmp(&netcfg.mac, &zero, sizeof(zero)) == 0) { 182 info_report("Zero hardware mac address detected. Ignoring."); 183 memcpy(netcfg.mac, n->mac, ETH_ALEN); 184 } 185 186 netcfg.status |= virtio_tswap16(vdev, 187 n->status & VIRTIO_NET_S_ANNOUNCE); 188 memcpy(config, &netcfg, n->config_size); 189 } 190 } 191 192 static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config) 193 { 194 VirtIONet *n = VIRTIO_NET(vdev); 195 struct virtio_net_config netcfg = {}; 196 NetClientState *nc = qemu_get_queue(n->nic); 197 198 memcpy(&netcfg, config, n->config_size); 199 200 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) && 201 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) && 202 memcmp(netcfg.mac, n->mac, ETH_ALEN)) { 203 memcpy(n->mac, netcfg.mac, ETH_ALEN); 204 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 205 } 206 207 /* 208 * Is this VDPA? No peer means not VDPA: there's no way to 209 * disconnect/reconnect a VDPA peer. 210 */ 211 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 212 vhost_net_set_config(get_vhost_net(nc->peer), 213 (uint8_t *)&netcfg, 0, n->config_size, 214 VHOST_SET_CONFIG_TYPE_FRONTEND); 215 } 216 } 217 218 static bool virtio_net_started(VirtIONet *n, uint8_t status) 219 { 220 VirtIODevice *vdev = VIRTIO_DEVICE(n); 221 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 222 (n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running; 223 } 224 225 static void virtio_net_announce_notify(VirtIONet *net) 226 { 227 VirtIODevice *vdev = VIRTIO_DEVICE(net); 228 trace_virtio_net_announce_notify(); 229 230 net->status |= VIRTIO_NET_S_ANNOUNCE; 231 virtio_notify_config(vdev); 232 } 233 234 static void virtio_net_announce_timer(void *opaque) 235 { 236 VirtIONet *n = opaque; 237 trace_virtio_net_announce_timer(n->announce_timer.round); 238 239 n->announce_timer.round--; 240 virtio_net_announce_notify(n); 241 } 242 243 static void virtio_net_announce(NetClientState *nc) 244 { 245 VirtIONet *n = qemu_get_nic_opaque(nc); 246 VirtIODevice *vdev = VIRTIO_DEVICE(n); 247 248 /* 249 * Make sure the virtio migration announcement timer isn't running 250 * If it is, let it trigger announcement so that we do not cause 251 * confusion. 252 */ 253 if (n->announce_timer.round) { 254 return; 255 } 256 257 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 258 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 259 virtio_net_announce_notify(n); 260 } 261 } 262 263 static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) 264 { 265 VirtIODevice *vdev = VIRTIO_DEVICE(n); 266 NetClientState *nc = qemu_get_queue(n->nic); 267 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 268 int cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ? 269 n->max_ncs - n->max_queue_pairs : 0; 270 271 if (!get_vhost_net(nc->peer)) { 272 return; 273 } 274 275 if ((virtio_net_started(n, status) && !nc->peer->link_down) == 276 !!n->vhost_started) { 277 return; 278 } 279 if (!n->vhost_started) { 280 int r, i; 281 282 if (n->needs_vnet_hdr_swap) { 283 error_report("backend does not support %s vnet headers; " 284 "falling back on userspace virtio", 285 virtio_is_big_endian(vdev) ? "BE" : "LE"); 286 return; 287 } 288 289 /* Any packets outstanding? Purge them to avoid touching rings 290 * when vhost is running. 291 */ 292 for (i = 0; i < queue_pairs; i++) { 293 NetClientState *qnc = qemu_get_subqueue(n->nic, i); 294 295 /* Purge both directions: TX and RX. */ 296 qemu_net_queue_purge(qnc->peer->incoming_queue, qnc); 297 qemu_net_queue_purge(qnc->incoming_queue, qnc->peer); 298 } 299 300 if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) { 301 r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu); 302 if (r < 0) { 303 error_report("%uBytes MTU not supported by the backend", 304 n->net_conf.mtu); 305 306 return; 307 } 308 } 309 310 n->vhost_started = 1; 311 r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq); 312 if (r < 0) { 313 error_report("unable to start vhost net: %d: " 314 "falling back on userspace virtio", -r); 315 n->vhost_started = 0; 316 } 317 } else { 318 vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq); 319 n->vhost_started = 0; 320 } 321 } 322 323 static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev, 324 NetClientState *peer, 325 bool enable) 326 { 327 if (virtio_is_big_endian(vdev)) { 328 return qemu_set_vnet_be(peer, enable); 329 } else { 330 return qemu_set_vnet_le(peer, enable); 331 } 332 } 333 334 static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs, 335 int queue_pairs, bool enable) 336 { 337 int i; 338 339 for (i = 0; i < queue_pairs; i++) { 340 if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 && 341 enable) { 342 while (--i >= 0) { 343 virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false); 344 } 345 346 return true; 347 } 348 } 349 350 return false; 351 } 352 353 static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status) 354 { 355 VirtIODevice *vdev = VIRTIO_DEVICE(n); 356 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 357 358 if (virtio_net_started(n, status)) { 359 /* Before using the device, we tell the network backend about the 360 * endianness to use when parsing vnet headers. If the backend 361 * can't do it, we fallback onto fixing the headers in the core 362 * virtio-net code. 363 */ 364 n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs, 365 queue_pairs, true); 366 } else if (virtio_net_started(n, vdev->status)) { 367 /* After using the device, we need to reset the network backend to 368 * the default (guest native endianness), otherwise the guest may 369 * lose network connectivity if it is rebooted into a different 370 * endianness. 371 */ 372 virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false); 373 } 374 } 375 376 static void virtio_net_drop_tx_queue_data(VirtIODevice *vdev, VirtQueue *vq) 377 { 378 unsigned int dropped = virtqueue_drop_all(vq); 379 if (dropped) { 380 virtio_notify(vdev, vq); 381 } 382 } 383 384 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status) 385 { 386 VirtIONet *n = VIRTIO_NET(vdev); 387 VirtIONetQueue *q; 388 int i; 389 uint8_t queue_status; 390 391 virtio_net_vnet_endian_status(n, status); 392 virtio_net_vhost_status(n, status); 393 394 for (i = 0; i < n->max_queue_pairs; i++) { 395 NetClientState *ncs = qemu_get_subqueue(n->nic, i); 396 bool queue_started; 397 q = &n->vqs[i]; 398 399 if ((!n->multiqueue && i != 0) || i >= n->curr_queue_pairs) { 400 queue_status = 0; 401 } else { 402 queue_status = status; 403 } 404 queue_started = 405 virtio_net_started(n, queue_status) && !n->vhost_started; 406 407 if (queue_started) { 408 qemu_flush_queued_packets(ncs); 409 } 410 411 if (!q->tx_waiting) { 412 continue; 413 } 414 415 if (queue_started) { 416 if (q->tx_timer) { 417 timer_mod(q->tx_timer, 418 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 419 } else { 420 qemu_bh_schedule(q->tx_bh); 421 } 422 } else { 423 if (q->tx_timer) { 424 timer_del(q->tx_timer); 425 } else { 426 qemu_bh_cancel(q->tx_bh); 427 } 428 if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 && 429 (queue_status & VIRTIO_CONFIG_S_DRIVER_OK) && 430 vdev->vm_running) { 431 /* if tx is waiting we are likely have some packets in tx queue 432 * and disabled notification */ 433 q->tx_waiting = 0; 434 virtio_queue_set_notification(q->tx_vq, 1); 435 virtio_net_drop_tx_queue_data(vdev, q->tx_vq); 436 } 437 } 438 } 439 } 440 441 static void virtio_net_set_link_status(NetClientState *nc) 442 { 443 VirtIONet *n = qemu_get_nic_opaque(nc); 444 VirtIODevice *vdev = VIRTIO_DEVICE(n); 445 uint16_t old_status = n->status; 446 447 if (nc->link_down) 448 n->status &= ~VIRTIO_NET_S_LINK_UP; 449 else 450 n->status |= VIRTIO_NET_S_LINK_UP; 451 452 if (n->status != old_status) 453 virtio_notify_config(vdev); 454 455 virtio_net_set_status(vdev, vdev->status); 456 } 457 458 static void rxfilter_notify(NetClientState *nc) 459 { 460 VirtIONet *n = qemu_get_nic_opaque(nc); 461 462 if (nc->rxfilter_notify_enabled) { 463 char *path = object_get_canonical_path(OBJECT(n->qdev)); 464 qapi_event_send_nic_rx_filter_changed(n->netclient_name, path); 465 g_free(path); 466 467 /* disable event notification to avoid events flooding */ 468 nc->rxfilter_notify_enabled = 0; 469 } 470 } 471 472 static intList *get_vlan_table(VirtIONet *n) 473 { 474 intList *list; 475 int i, j; 476 477 list = NULL; 478 for (i = 0; i < MAX_VLAN >> 5; i++) { 479 for (j = 0; n->vlans[i] && j <= 0x1f; j++) { 480 if (n->vlans[i] & (1U << j)) { 481 QAPI_LIST_PREPEND(list, (i << 5) + j); 482 } 483 } 484 } 485 486 return list; 487 } 488 489 static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc) 490 { 491 VirtIONet *n = qemu_get_nic_opaque(nc); 492 VirtIODevice *vdev = VIRTIO_DEVICE(n); 493 RxFilterInfo *info; 494 strList *str_list; 495 int i; 496 497 info = g_malloc0(sizeof(*info)); 498 info->name = g_strdup(nc->name); 499 info->promiscuous = n->promisc; 500 501 if (n->nouni) { 502 info->unicast = RX_STATE_NONE; 503 } else if (n->alluni) { 504 info->unicast = RX_STATE_ALL; 505 } else { 506 info->unicast = RX_STATE_NORMAL; 507 } 508 509 if (n->nomulti) { 510 info->multicast = RX_STATE_NONE; 511 } else if (n->allmulti) { 512 info->multicast = RX_STATE_ALL; 513 } else { 514 info->multicast = RX_STATE_NORMAL; 515 } 516 517 info->broadcast_allowed = n->nobcast; 518 info->multicast_overflow = n->mac_table.multi_overflow; 519 info->unicast_overflow = n->mac_table.uni_overflow; 520 521 info->main_mac = qemu_mac_strdup_printf(n->mac); 522 523 str_list = NULL; 524 for (i = 0; i < n->mac_table.first_multi; i++) { 525 QAPI_LIST_PREPEND(str_list, 526 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 527 } 528 info->unicast_table = str_list; 529 530 str_list = NULL; 531 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 532 QAPI_LIST_PREPEND(str_list, 533 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 534 } 535 info->multicast_table = str_list; 536 info->vlan_table = get_vlan_table(n); 537 538 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) { 539 info->vlan = RX_STATE_ALL; 540 } else if (!info->vlan_table) { 541 info->vlan = RX_STATE_NONE; 542 } else { 543 info->vlan = RX_STATE_NORMAL; 544 } 545 546 /* enable event notification after query */ 547 nc->rxfilter_notify_enabled = 1; 548 549 return info; 550 } 551 552 static void virtio_net_queue_reset(VirtIODevice *vdev, uint32_t queue_index) 553 { 554 VirtIONet *n = VIRTIO_NET(vdev); 555 NetClientState *nc; 556 557 /* validate queue_index and skip for cvq */ 558 if (queue_index >= n->max_queue_pairs * 2) { 559 return; 560 } 561 562 nc = qemu_get_subqueue(n->nic, vq2q(queue_index)); 563 564 if (!nc->peer) { 565 return; 566 } 567 568 if (get_vhost_net(nc->peer) && 569 nc->peer->info->type == NET_CLIENT_DRIVER_TAP) { 570 vhost_net_virtqueue_reset(vdev, nc, queue_index); 571 } 572 573 flush_or_purge_queued_packets(nc); 574 } 575 576 static void virtio_net_queue_enable(VirtIODevice *vdev, uint32_t queue_index) 577 { 578 VirtIONet *n = VIRTIO_NET(vdev); 579 NetClientState *nc; 580 int r; 581 582 /* validate queue_index and skip for cvq */ 583 if (queue_index >= n->max_queue_pairs * 2) { 584 return; 585 } 586 587 nc = qemu_get_subqueue(n->nic, vq2q(queue_index)); 588 589 if (!nc->peer || !vdev->vhost_started) { 590 return; 591 } 592 593 if (get_vhost_net(nc->peer) && 594 nc->peer->info->type == NET_CLIENT_DRIVER_TAP) { 595 r = vhost_net_virtqueue_restart(vdev, nc, queue_index); 596 if (r < 0) { 597 error_report("unable to restart vhost net virtqueue: %d, " 598 "when resetting the queue", queue_index); 599 } 600 } 601 } 602 603 static void virtio_net_reset(VirtIODevice *vdev) 604 { 605 VirtIONet *n = VIRTIO_NET(vdev); 606 int i; 607 608 /* Reset back to compatibility mode */ 609 n->promisc = 1; 610 n->allmulti = 0; 611 n->alluni = 0; 612 n->nomulti = 0; 613 n->nouni = 0; 614 n->nobcast = 0; 615 /* multiqueue is disabled by default */ 616 n->curr_queue_pairs = 1; 617 timer_del(n->announce_timer.tm); 618 n->announce_timer.round = 0; 619 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 620 621 /* Flush any MAC and VLAN filter table state */ 622 n->mac_table.in_use = 0; 623 n->mac_table.first_multi = 0; 624 n->mac_table.multi_overflow = 0; 625 n->mac_table.uni_overflow = 0; 626 memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN); 627 memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac)); 628 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 629 memset(n->vlans, 0, MAX_VLAN >> 3); 630 631 /* Flush any async TX */ 632 for (i = 0; i < n->max_queue_pairs; i++) { 633 flush_or_purge_queued_packets(qemu_get_subqueue(n->nic, i)); 634 } 635 } 636 637 static void peer_test_vnet_hdr(VirtIONet *n) 638 { 639 NetClientState *nc = qemu_get_queue(n->nic); 640 if (!nc->peer) { 641 return; 642 } 643 644 n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer); 645 } 646 647 static int peer_has_vnet_hdr(VirtIONet *n) 648 { 649 return n->has_vnet_hdr; 650 } 651 652 static int peer_has_ufo(VirtIONet *n) 653 { 654 if (!peer_has_vnet_hdr(n)) 655 return 0; 656 657 n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer); 658 659 return n->has_ufo; 660 } 661 662 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, 663 int version_1, int hash_report) 664 { 665 int i; 666 NetClientState *nc; 667 668 n->mergeable_rx_bufs = mergeable_rx_bufs; 669 670 if (version_1) { 671 n->guest_hdr_len = hash_report ? 672 sizeof(struct virtio_net_hdr_v1_hash) : 673 sizeof(struct virtio_net_hdr_mrg_rxbuf); 674 n->rss_data.populate_hash = !!hash_report; 675 } else { 676 n->guest_hdr_len = n->mergeable_rx_bufs ? 677 sizeof(struct virtio_net_hdr_mrg_rxbuf) : 678 sizeof(struct virtio_net_hdr); 679 } 680 681 for (i = 0; i < n->max_queue_pairs; i++) { 682 nc = qemu_get_subqueue(n->nic, i); 683 684 if (peer_has_vnet_hdr(n) && 685 qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) { 686 qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len); 687 n->host_hdr_len = n->guest_hdr_len; 688 } 689 } 690 } 691 692 static int virtio_net_max_tx_queue_size(VirtIONet *n) 693 { 694 NetClientState *peer = n->nic_conf.peers.ncs[0]; 695 696 /* 697 * Backends other than vhost-user or vhost-vdpa don't support max queue 698 * size. 699 */ 700 if (!peer) { 701 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 702 } 703 704 switch(peer->info->type) { 705 case NET_CLIENT_DRIVER_VHOST_USER: 706 case NET_CLIENT_DRIVER_VHOST_VDPA: 707 return VIRTQUEUE_MAX_SIZE; 708 default: 709 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 710 }; 711 } 712 713 static int peer_attach(VirtIONet *n, int index) 714 { 715 NetClientState *nc = qemu_get_subqueue(n->nic, index); 716 717 if (!nc->peer) { 718 return 0; 719 } 720 721 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 722 vhost_set_vring_enable(nc->peer, 1); 723 } 724 725 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 726 return 0; 727 } 728 729 if (n->max_queue_pairs == 1) { 730 return 0; 731 } 732 733 return tap_enable(nc->peer); 734 } 735 736 static int peer_detach(VirtIONet *n, int index) 737 { 738 NetClientState *nc = qemu_get_subqueue(n->nic, index); 739 740 if (!nc->peer) { 741 return 0; 742 } 743 744 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 745 vhost_set_vring_enable(nc->peer, 0); 746 } 747 748 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 749 return 0; 750 } 751 752 return tap_disable(nc->peer); 753 } 754 755 static void virtio_net_set_queue_pairs(VirtIONet *n) 756 { 757 int i; 758 int r; 759 760 if (n->nic->peer_deleted) { 761 return; 762 } 763 764 for (i = 0; i < n->max_queue_pairs; i++) { 765 if (i < n->curr_queue_pairs) { 766 r = peer_attach(n, i); 767 assert(!r); 768 } else { 769 r = peer_detach(n, i); 770 assert(!r); 771 } 772 } 773 } 774 775 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue); 776 777 static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, 778 Error **errp) 779 { 780 VirtIONet *n = VIRTIO_NET(vdev); 781 NetClientState *nc = qemu_get_queue(n->nic); 782 783 /* Firstly sync all virtio-net possible supported features */ 784 features |= n->host_features; 785 786 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 787 788 if (!peer_has_vnet_hdr(n)) { 789 virtio_clear_feature(&features, VIRTIO_NET_F_CSUM); 790 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO4); 791 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO6); 792 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_ECN); 793 794 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_CSUM); 795 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4); 796 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6); 797 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN); 798 799 virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); 800 } 801 802 if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) { 803 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_UFO); 804 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO); 805 } 806 807 if (!get_vhost_net(nc->peer)) { 808 return features; 809 } 810 811 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 812 virtio_clear_feature(&features, VIRTIO_NET_F_RSS); 813 } 814 features = vhost_net_get_features(get_vhost_net(nc->peer), features); 815 vdev->backend_features = features; 816 817 if (n->mtu_bypass_backend && 818 (n->host_features & 1ULL << VIRTIO_NET_F_MTU)) { 819 features |= (1ULL << VIRTIO_NET_F_MTU); 820 } 821 822 /* 823 * Since GUEST_ANNOUNCE is emulated the feature bit could be set without 824 * enabled. This happens in the vDPA case. 825 * 826 * Make sure the feature set is not incoherent, as the driver could refuse 827 * to start. 828 * 829 * TODO: QEMU is able to emulate a CVQ just for guest_announce purposes, 830 * helping guest to notify the new location with vDPA devices that does not 831 * support it. 832 */ 833 if (!virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_CTRL_VQ)) { 834 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ANNOUNCE); 835 } 836 837 return features; 838 } 839 840 static uint64_t virtio_net_bad_features(VirtIODevice *vdev) 841 { 842 uint64_t features = 0; 843 844 /* Linux kernel 2.6.25. It understood MAC (as everyone must), 845 * but also these: */ 846 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 847 virtio_add_feature(&features, VIRTIO_NET_F_CSUM); 848 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4); 849 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6); 850 virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN); 851 852 return features; 853 } 854 855 static void virtio_net_apply_guest_offloads(VirtIONet *n) 856 { 857 qemu_set_offload(qemu_get_queue(n->nic)->peer, 858 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)), 859 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)), 860 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)), 861 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)), 862 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO))); 863 } 864 865 static uint64_t virtio_net_guest_offloads_by_features(uint32_t features) 866 { 867 static const uint64_t guest_offloads_mask = 868 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | 869 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 870 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | 871 (1ULL << VIRTIO_NET_F_GUEST_ECN) | 872 (1ULL << VIRTIO_NET_F_GUEST_UFO); 873 874 return guest_offloads_mask & features; 875 } 876 877 uint64_t virtio_net_supported_guest_offloads(const VirtIONet *n) 878 { 879 VirtIODevice *vdev = VIRTIO_DEVICE(n); 880 return virtio_net_guest_offloads_by_features(vdev->guest_features); 881 } 882 883 typedef struct { 884 VirtIONet *n; 885 DeviceState *dev; 886 } FailoverDevice; 887 888 /** 889 * Set the failover primary device 890 * 891 * @opaque: FailoverId to setup 892 * @opts: opts for device we are handling 893 * @errp: returns an error if this function fails 894 */ 895 static int failover_set_primary(DeviceState *dev, void *opaque) 896 { 897 FailoverDevice *fdev = opaque; 898 PCIDevice *pci_dev = (PCIDevice *) 899 object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE); 900 901 if (!pci_dev) { 902 return 0; 903 } 904 905 if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) { 906 fdev->dev = dev; 907 return 1; 908 } 909 910 return 0; 911 } 912 913 /** 914 * Find the primary device for this failover virtio-net 915 * 916 * @n: VirtIONet device 917 * @errp: returns an error if this function fails 918 */ 919 static DeviceState *failover_find_primary_device(VirtIONet *n) 920 { 921 FailoverDevice fdev = { 922 .n = n, 923 }; 924 925 qbus_walk_children(sysbus_get_default(), failover_set_primary, NULL, 926 NULL, NULL, &fdev); 927 return fdev.dev; 928 } 929 930 static void failover_add_primary(VirtIONet *n, Error **errp) 931 { 932 Error *err = NULL; 933 DeviceState *dev = failover_find_primary_device(n); 934 935 if (dev) { 936 return; 937 } 938 939 if (!n->primary_opts) { 940 error_setg(errp, "Primary device not found"); 941 error_append_hint(errp, "Virtio-net failover will not work. Make " 942 "sure primary device has parameter" 943 " failover_pair_id=%s\n", n->netclient_name); 944 return; 945 } 946 947 dev = qdev_device_add_from_qdict(n->primary_opts, 948 n->primary_opts_from_json, 949 &err); 950 if (err) { 951 qobject_unref(n->primary_opts); 952 n->primary_opts = NULL; 953 } else { 954 object_unref(OBJECT(dev)); 955 } 956 error_propagate(errp, err); 957 } 958 959 static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) 960 { 961 VirtIONet *n = VIRTIO_NET(vdev); 962 Error *err = NULL; 963 int i; 964 965 if (n->mtu_bypass_backend && 966 !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { 967 features &= ~(1ULL << VIRTIO_NET_F_MTU); 968 } 969 970 virtio_net_set_multiqueue(n, 971 virtio_has_feature(features, VIRTIO_NET_F_RSS) || 972 virtio_has_feature(features, VIRTIO_NET_F_MQ)); 973 974 virtio_net_set_mrg_rx_bufs(n, 975 virtio_has_feature(features, 976 VIRTIO_NET_F_MRG_RXBUF), 977 virtio_has_feature(features, 978 VIRTIO_F_VERSION_1), 979 virtio_has_feature(features, 980 VIRTIO_NET_F_HASH_REPORT)); 981 982 n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 983 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO4); 984 n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 985 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); 986 n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); 987 988 if (n->has_vnet_hdr) { 989 n->curr_guest_offloads = 990 virtio_net_guest_offloads_by_features(features); 991 virtio_net_apply_guest_offloads(n); 992 } 993 994 for (i = 0; i < n->max_queue_pairs; i++) { 995 NetClientState *nc = qemu_get_subqueue(n->nic, i); 996 997 if (!get_vhost_net(nc->peer)) { 998 continue; 999 } 1000 vhost_net_ack_features(get_vhost_net(nc->peer), features); 1001 1002 /* 1003 * keep acked_features in NetVhostUserState up-to-date so it 1004 * can't miss any features configured by guest virtio driver. 1005 */ 1006 vhost_net_save_acked_features(nc->peer); 1007 } 1008 1009 if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) { 1010 memset(n->vlans, 0, MAX_VLAN >> 3); 1011 } else { 1012 memset(n->vlans, 0xff, MAX_VLAN >> 3); 1013 } 1014 1015 if (virtio_has_feature(features, VIRTIO_NET_F_STANDBY)) { 1016 qapi_event_send_failover_negotiated(n->netclient_name); 1017 qatomic_set(&n->failover_primary_hidden, false); 1018 failover_add_primary(n, &err); 1019 if (err) { 1020 if (!qtest_enabled()) { 1021 warn_report_err(err); 1022 } else { 1023 error_free(err); 1024 } 1025 } 1026 } 1027 } 1028 1029 static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd, 1030 struct iovec *iov, unsigned int iov_cnt) 1031 { 1032 uint8_t on; 1033 size_t s; 1034 NetClientState *nc = qemu_get_queue(n->nic); 1035 1036 s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on)); 1037 if (s != sizeof(on)) { 1038 return VIRTIO_NET_ERR; 1039 } 1040 1041 if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) { 1042 n->promisc = on; 1043 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) { 1044 n->allmulti = on; 1045 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) { 1046 n->alluni = on; 1047 } else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) { 1048 n->nomulti = on; 1049 } else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) { 1050 n->nouni = on; 1051 } else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) { 1052 n->nobcast = on; 1053 } else { 1054 return VIRTIO_NET_ERR; 1055 } 1056 1057 rxfilter_notify(nc); 1058 1059 return VIRTIO_NET_OK; 1060 } 1061 1062 static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd, 1063 struct iovec *iov, unsigned int iov_cnt) 1064 { 1065 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1066 uint64_t offloads; 1067 size_t s; 1068 1069 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 1070 return VIRTIO_NET_ERR; 1071 } 1072 1073 s = iov_to_buf(iov, iov_cnt, 0, &offloads, sizeof(offloads)); 1074 if (s != sizeof(offloads)) { 1075 return VIRTIO_NET_ERR; 1076 } 1077 1078 if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) { 1079 uint64_t supported_offloads; 1080 1081 offloads = virtio_ldq_p(vdev, &offloads); 1082 1083 if (!n->has_vnet_hdr) { 1084 return VIRTIO_NET_ERR; 1085 } 1086 1087 n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1088 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO4); 1089 n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1090 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO6); 1091 virtio_clear_feature(&offloads, VIRTIO_NET_F_RSC_EXT); 1092 1093 supported_offloads = virtio_net_supported_guest_offloads(n); 1094 if (offloads & ~supported_offloads) { 1095 return VIRTIO_NET_ERR; 1096 } 1097 1098 n->curr_guest_offloads = offloads; 1099 virtio_net_apply_guest_offloads(n); 1100 1101 return VIRTIO_NET_OK; 1102 } else { 1103 return VIRTIO_NET_ERR; 1104 } 1105 } 1106 1107 static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd, 1108 struct iovec *iov, unsigned int iov_cnt) 1109 { 1110 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1111 struct virtio_net_ctrl_mac mac_data; 1112 size_t s; 1113 NetClientState *nc = qemu_get_queue(n->nic); 1114 1115 if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) { 1116 if (iov_size(iov, iov_cnt) != sizeof(n->mac)) { 1117 return VIRTIO_NET_ERR; 1118 } 1119 s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac)); 1120 assert(s == sizeof(n->mac)); 1121 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 1122 rxfilter_notify(nc); 1123 1124 return VIRTIO_NET_OK; 1125 } 1126 1127 if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) { 1128 return VIRTIO_NET_ERR; 1129 } 1130 1131 int in_use = 0; 1132 int first_multi = 0; 1133 uint8_t uni_overflow = 0; 1134 uint8_t multi_overflow = 0; 1135 uint8_t *macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 1136 1137 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1138 sizeof(mac_data.entries)); 1139 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1140 if (s != sizeof(mac_data.entries)) { 1141 goto error; 1142 } 1143 iov_discard_front(&iov, &iov_cnt, s); 1144 1145 if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) { 1146 goto error; 1147 } 1148 1149 if (mac_data.entries <= MAC_TABLE_ENTRIES) { 1150 s = iov_to_buf(iov, iov_cnt, 0, macs, 1151 mac_data.entries * ETH_ALEN); 1152 if (s != mac_data.entries * ETH_ALEN) { 1153 goto error; 1154 } 1155 in_use += mac_data.entries; 1156 } else { 1157 uni_overflow = 1; 1158 } 1159 1160 iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN); 1161 1162 first_multi = in_use; 1163 1164 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1165 sizeof(mac_data.entries)); 1166 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1167 if (s != sizeof(mac_data.entries)) { 1168 goto error; 1169 } 1170 1171 iov_discard_front(&iov, &iov_cnt, s); 1172 1173 if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) { 1174 goto error; 1175 } 1176 1177 if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) { 1178 s = iov_to_buf(iov, iov_cnt, 0, &macs[in_use * ETH_ALEN], 1179 mac_data.entries * ETH_ALEN); 1180 if (s != mac_data.entries * ETH_ALEN) { 1181 goto error; 1182 } 1183 in_use += mac_data.entries; 1184 } else { 1185 multi_overflow = 1; 1186 } 1187 1188 n->mac_table.in_use = in_use; 1189 n->mac_table.first_multi = first_multi; 1190 n->mac_table.uni_overflow = uni_overflow; 1191 n->mac_table.multi_overflow = multi_overflow; 1192 memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN); 1193 g_free(macs); 1194 rxfilter_notify(nc); 1195 1196 return VIRTIO_NET_OK; 1197 1198 error: 1199 g_free(macs); 1200 return VIRTIO_NET_ERR; 1201 } 1202 1203 static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd, 1204 struct iovec *iov, unsigned int iov_cnt) 1205 { 1206 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1207 uint16_t vid; 1208 size_t s; 1209 NetClientState *nc = qemu_get_queue(n->nic); 1210 1211 s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid)); 1212 vid = virtio_lduw_p(vdev, &vid); 1213 if (s != sizeof(vid)) { 1214 return VIRTIO_NET_ERR; 1215 } 1216 1217 if (vid >= MAX_VLAN) 1218 return VIRTIO_NET_ERR; 1219 1220 if (cmd == VIRTIO_NET_CTRL_VLAN_ADD) 1221 n->vlans[vid >> 5] |= (1U << (vid & 0x1f)); 1222 else if (cmd == VIRTIO_NET_CTRL_VLAN_DEL) 1223 n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f)); 1224 else 1225 return VIRTIO_NET_ERR; 1226 1227 rxfilter_notify(nc); 1228 1229 return VIRTIO_NET_OK; 1230 } 1231 1232 static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, 1233 struct iovec *iov, unsigned int iov_cnt) 1234 { 1235 trace_virtio_net_handle_announce(n->announce_timer.round); 1236 if (cmd == VIRTIO_NET_CTRL_ANNOUNCE_ACK && 1237 n->status & VIRTIO_NET_S_ANNOUNCE) { 1238 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 1239 if (n->announce_timer.round) { 1240 qemu_announce_timer_step(&n->announce_timer); 1241 } 1242 return VIRTIO_NET_OK; 1243 } else { 1244 return VIRTIO_NET_ERR; 1245 } 1246 } 1247 1248 static void virtio_net_detach_epbf_rss(VirtIONet *n); 1249 1250 static void virtio_net_disable_rss(VirtIONet *n) 1251 { 1252 if (n->rss_data.enabled) { 1253 trace_virtio_net_rss_disable(); 1254 } 1255 n->rss_data.enabled = false; 1256 1257 virtio_net_detach_epbf_rss(n); 1258 } 1259 1260 static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd) 1261 { 1262 NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0); 1263 if (nc == NULL || nc->info->set_steering_ebpf == NULL) { 1264 return false; 1265 } 1266 1267 return nc->info->set_steering_ebpf(nc, prog_fd); 1268 } 1269 1270 static void rss_data_to_rss_config(struct VirtioNetRssData *data, 1271 struct EBPFRSSConfig *config) 1272 { 1273 config->redirect = data->redirect; 1274 config->populate_hash = data->populate_hash; 1275 config->hash_types = data->hash_types; 1276 config->indirections_len = data->indirections_len; 1277 config->default_queue = data->default_queue; 1278 } 1279 1280 static bool virtio_net_attach_epbf_rss(VirtIONet *n) 1281 { 1282 struct EBPFRSSConfig config = {}; 1283 1284 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 1285 return false; 1286 } 1287 1288 rss_data_to_rss_config(&n->rss_data, &config); 1289 1290 if (!ebpf_rss_set_all(&n->ebpf_rss, &config, 1291 n->rss_data.indirections_table, n->rss_data.key)) { 1292 return false; 1293 } 1294 1295 if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { 1296 return false; 1297 } 1298 1299 return true; 1300 } 1301 1302 static void virtio_net_detach_epbf_rss(VirtIONet *n) 1303 { 1304 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1305 } 1306 1307 static bool virtio_net_load_ebpf(VirtIONet *n) 1308 { 1309 if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { 1310 /* backend does't support steering ebpf */ 1311 return false; 1312 } 1313 1314 return ebpf_rss_load(&n->ebpf_rss); 1315 } 1316 1317 static void virtio_net_unload_ebpf(VirtIONet *n) 1318 { 1319 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1320 ebpf_rss_unload(&n->ebpf_rss); 1321 } 1322 1323 static uint16_t virtio_net_handle_rss(VirtIONet *n, 1324 struct iovec *iov, 1325 unsigned int iov_cnt, 1326 bool do_rss) 1327 { 1328 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1329 struct virtio_net_rss_config cfg; 1330 size_t s, offset = 0, size_get; 1331 uint16_t queue_pairs, i; 1332 struct { 1333 uint16_t us; 1334 uint8_t b; 1335 } QEMU_PACKED temp; 1336 const char *err_msg = ""; 1337 uint32_t err_value = 0; 1338 1339 if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) { 1340 err_msg = "RSS is not negotiated"; 1341 goto error; 1342 } 1343 if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) { 1344 err_msg = "Hash report is not negotiated"; 1345 goto error; 1346 } 1347 size_get = offsetof(struct virtio_net_rss_config, indirection_table); 1348 s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get); 1349 if (s != size_get) { 1350 err_msg = "Short command buffer"; 1351 err_value = (uint32_t)s; 1352 goto error; 1353 } 1354 n->rss_data.hash_types = virtio_ldl_p(vdev, &cfg.hash_types); 1355 n->rss_data.indirections_len = 1356 virtio_lduw_p(vdev, &cfg.indirection_table_mask); 1357 n->rss_data.indirections_len++; 1358 if (!do_rss) { 1359 n->rss_data.indirections_len = 1; 1360 } 1361 if (!is_power_of_2(n->rss_data.indirections_len)) { 1362 err_msg = "Invalid size of indirection table"; 1363 err_value = n->rss_data.indirections_len; 1364 goto error; 1365 } 1366 if (n->rss_data.indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 1367 err_msg = "Too large indirection table"; 1368 err_value = n->rss_data.indirections_len; 1369 goto error; 1370 } 1371 n->rss_data.default_queue = do_rss ? 1372 virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0; 1373 if (n->rss_data.default_queue >= n->max_queue_pairs) { 1374 err_msg = "Invalid default queue"; 1375 err_value = n->rss_data.default_queue; 1376 goto error; 1377 } 1378 offset += size_get; 1379 size_get = sizeof(uint16_t) * n->rss_data.indirections_len; 1380 g_free(n->rss_data.indirections_table); 1381 n->rss_data.indirections_table = g_malloc(size_get); 1382 if (!n->rss_data.indirections_table) { 1383 err_msg = "Can't allocate indirections table"; 1384 err_value = n->rss_data.indirections_len; 1385 goto error; 1386 } 1387 s = iov_to_buf(iov, iov_cnt, offset, 1388 n->rss_data.indirections_table, size_get); 1389 if (s != size_get) { 1390 err_msg = "Short indirection table buffer"; 1391 err_value = (uint32_t)s; 1392 goto error; 1393 } 1394 for (i = 0; i < n->rss_data.indirections_len; ++i) { 1395 uint16_t val = n->rss_data.indirections_table[i]; 1396 n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val); 1397 } 1398 offset += size_get; 1399 size_get = sizeof(temp); 1400 s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get); 1401 if (s != size_get) { 1402 err_msg = "Can't get queue_pairs"; 1403 err_value = (uint32_t)s; 1404 goto error; 1405 } 1406 queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs; 1407 if (queue_pairs == 0 || queue_pairs > n->max_queue_pairs) { 1408 err_msg = "Invalid number of queue_pairs"; 1409 err_value = queue_pairs; 1410 goto error; 1411 } 1412 if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) { 1413 err_msg = "Invalid key size"; 1414 err_value = temp.b; 1415 goto error; 1416 } 1417 if (!temp.b && n->rss_data.hash_types) { 1418 err_msg = "No key provided"; 1419 err_value = 0; 1420 goto error; 1421 } 1422 if (!temp.b && !n->rss_data.hash_types) { 1423 virtio_net_disable_rss(n); 1424 return queue_pairs; 1425 } 1426 offset += size_get; 1427 size_get = temp.b; 1428 s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get); 1429 if (s != size_get) { 1430 err_msg = "Can get key buffer"; 1431 err_value = (uint32_t)s; 1432 goto error; 1433 } 1434 n->rss_data.enabled = true; 1435 1436 if (!n->rss_data.populate_hash) { 1437 if (!virtio_net_attach_epbf_rss(n)) { 1438 /* EBPF must be loaded for vhost */ 1439 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 1440 warn_report("Can't load eBPF RSS for vhost"); 1441 goto error; 1442 } 1443 /* fallback to software RSS */ 1444 warn_report("Can't load eBPF RSS - fallback to software RSS"); 1445 n->rss_data.enabled_software_rss = true; 1446 } 1447 } else { 1448 /* use software RSS for hash populating */ 1449 /* and detach eBPF if was loaded before */ 1450 virtio_net_detach_epbf_rss(n); 1451 n->rss_data.enabled_software_rss = true; 1452 } 1453 1454 trace_virtio_net_rss_enable(n->rss_data.hash_types, 1455 n->rss_data.indirections_len, 1456 temp.b); 1457 return queue_pairs; 1458 error: 1459 trace_virtio_net_rss_error(err_msg, err_value); 1460 virtio_net_disable_rss(n); 1461 return 0; 1462 } 1463 1464 static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd, 1465 struct iovec *iov, unsigned int iov_cnt) 1466 { 1467 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1468 uint16_t queue_pairs; 1469 NetClientState *nc = qemu_get_queue(n->nic); 1470 1471 virtio_net_disable_rss(n); 1472 if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) { 1473 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, false); 1474 return queue_pairs ? VIRTIO_NET_OK : VIRTIO_NET_ERR; 1475 } 1476 if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 1477 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, true); 1478 } else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 1479 struct virtio_net_ctrl_mq mq; 1480 size_t s; 1481 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) { 1482 return VIRTIO_NET_ERR; 1483 } 1484 s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq)); 1485 if (s != sizeof(mq)) { 1486 return VIRTIO_NET_ERR; 1487 } 1488 queue_pairs = virtio_lduw_p(vdev, &mq.virtqueue_pairs); 1489 1490 } else { 1491 return VIRTIO_NET_ERR; 1492 } 1493 1494 if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1495 queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 1496 queue_pairs > n->max_queue_pairs || 1497 !n->multiqueue) { 1498 return VIRTIO_NET_ERR; 1499 } 1500 1501 n->curr_queue_pairs = queue_pairs; 1502 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 1503 /* 1504 * Avoid updating the backend for a vdpa device: We're only interested 1505 * in updating the device model queues. 1506 */ 1507 return VIRTIO_NET_OK; 1508 } 1509 /* stop the backend before changing the number of queue_pairs to avoid handling a 1510 * disabled queue */ 1511 virtio_net_set_status(vdev, vdev->status); 1512 virtio_net_set_queue_pairs(n); 1513 1514 return VIRTIO_NET_OK; 1515 } 1516 1517 size_t virtio_net_handle_ctrl_iov(VirtIODevice *vdev, 1518 const struct iovec *in_sg, unsigned in_num, 1519 const struct iovec *out_sg, 1520 unsigned out_num) 1521 { 1522 VirtIONet *n = VIRTIO_NET(vdev); 1523 struct virtio_net_ctrl_hdr ctrl; 1524 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1525 size_t s; 1526 struct iovec *iov, *iov2; 1527 1528 if (iov_size(in_sg, in_num) < sizeof(status) || 1529 iov_size(out_sg, out_num) < sizeof(ctrl)) { 1530 virtio_error(vdev, "virtio-net ctrl missing headers"); 1531 return 0; 1532 } 1533 1534 iov2 = iov = g_memdup2(out_sg, sizeof(struct iovec) * out_num); 1535 s = iov_to_buf(iov, out_num, 0, &ctrl, sizeof(ctrl)); 1536 iov_discard_front(&iov, &out_num, sizeof(ctrl)); 1537 if (s != sizeof(ctrl)) { 1538 status = VIRTIO_NET_ERR; 1539 } else if (ctrl.class == VIRTIO_NET_CTRL_RX) { 1540 status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, out_num); 1541 } else if (ctrl.class == VIRTIO_NET_CTRL_MAC) { 1542 status = virtio_net_handle_mac(n, ctrl.cmd, iov, out_num); 1543 } else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) { 1544 status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, out_num); 1545 } else if (ctrl.class == VIRTIO_NET_CTRL_ANNOUNCE) { 1546 status = virtio_net_handle_announce(n, ctrl.cmd, iov, out_num); 1547 } else if (ctrl.class == VIRTIO_NET_CTRL_MQ) { 1548 status = virtio_net_handle_mq(n, ctrl.cmd, iov, out_num); 1549 } else if (ctrl.class == VIRTIO_NET_CTRL_GUEST_OFFLOADS) { 1550 status = virtio_net_handle_offloads(n, ctrl.cmd, iov, out_num); 1551 } 1552 1553 s = iov_from_buf(in_sg, in_num, 0, &status, sizeof(status)); 1554 assert(s == sizeof(status)); 1555 1556 g_free(iov2); 1557 return sizeof(status); 1558 } 1559 1560 static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 1561 { 1562 VirtQueueElement *elem; 1563 1564 for (;;) { 1565 size_t written; 1566 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 1567 if (!elem) { 1568 break; 1569 } 1570 1571 written = virtio_net_handle_ctrl_iov(vdev, elem->in_sg, elem->in_num, 1572 elem->out_sg, elem->out_num); 1573 if (written > 0) { 1574 virtqueue_push(vq, elem, written); 1575 virtio_notify(vdev, vq); 1576 g_free(elem); 1577 } else { 1578 virtqueue_detach_element(vq, elem, 0); 1579 g_free(elem); 1580 break; 1581 } 1582 } 1583 } 1584 1585 /* RX */ 1586 1587 static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq) 1588 { 1589 VirtIONet *n = VIRTIO_NET(vdev); 1590 int queue_index = vq2q(virtio_get_queue_index(vq)); 1591 1592 qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index)); 1593 } 1594 1595 static bool virtio_net_can_receive(NetClientState *nc) 1596 { 1597 VirtIONet *n = qemu_get_nic_opaque(nc); 1598 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1599 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1600 1601 if (!vdev->vm_running) { 1602 return false; 1603 } 1604 1605 if (nc->queue_index >= n->curr_queue_pairs) { 1606 return false; 1607 } 1608 1609 if (!virtio_queue_ready(q->rx_vq) || 1610 !(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 1611 return false; 1612 } 1613 1614 return true; 1615 } 1616 1617 static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize) 1618 { 1619 VirtIONet *n = q->n; 1620 if (virtio_queue_empty(q->rx_vq) || 1621 (n->mergeable_rx_bufs && 1622 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1623 virtio_queue_set_notification(q->rx_vq, 1); 1624 1625 /* To avoid a race condition where the guest has made some buffers 1626 * available after the above check but before notification was 1627 * enabled, check for available buffers again. 1628 */ 1629 if (virtio_queue_empty(q->rx_vq) || 1630 (n->mergeable_rx_bufs && 1631 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1632 return 0; 1633 } 1634 } 1635 1636 virtio_queue_set_notification(q->rx_vq, 0); 1637 return 1; 1638 } 1639 1640 static void virtio_net_hdr_swap(VirtIODevice *vdev, struct virtio_net_hdr *hdr) 1641 { 1642 virtio_tswap16s(vdev, &hdr->hdr_len); 1643 virtio_tswap16s(vdev, &hdr->gso_size); 1644 virtio_tswap16s(vdev, &hdr->csum_start); 1645 virtio_tswap16s(vdev, &hdr->csum_offset); 1646 } 1647 1648 /* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so 1649 * it never finds out that the packets don't have valid checksums. This 1650 * causes dhclient to get upset. Fedora's carried a patch for ages to 1651 * fix this with Xen but it hasn't appeared in an upstream release of 1652 * dhclient yet. 1653 * 1654 * To avoid breaking existing guests, we catch udp packets and add 1655 * checksums. This is terrible but it's better than hacking the guest 1656 * kernels. 1657 * 1658 * N.B. if we introduce a zero-copy API, this operation is no longer free so 1659 * we should provide a mechanism to disable it to avoid polluting the host 1660 * cache. 1661 */ 1662 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr, 1663 uint8_t *buf, size_t size) 1664 { 1665 if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */ 1666 (size > 27 && size < 1500) && /* normal sized MTU */ 1667 (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */ 1668 (buf[23] == 17) && /* ip.protocol == UDP */ 1669 (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */ 1670 net_checksum_calculate(buf, size, CSUM_UDP); 1671 hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM; 1672 } 1673 } 1674 1675 static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt, 1676 const void *buf, size_t size) 1677 { 1678 if (n->has_vnet_hdr) { 1679 /* FIXME this cast is evil */ 1680 void *wbuf = (void *)buf; 1681 work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len, 1682 size - n->host_hdr_len); 1683 1684 if (n->needs_vnet_hdr_swap) { 1685 virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf); 1686 } 1687 iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr)); 1688 } else { 1689 struct virtio_net_hdr hdr = { 1690 .flags = 0, 1691 .gso_type = VIRTIO_NET_HDR_GSO_NONE 1692 }; 1693 iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr); 1694 } 1695 } 1696 1697 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size) 1698 { 1699 static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 1700 static const uint8_t vlan[] = {0x81, 0x00}; 1701 uint8_t *ptr = (uint8_t *)buf; 1702 int i; 1703 1704 if (n->promisc) 1705 return 1; 1706 1707 ptr += n->host_hdr_len; 1708 1709 if (!memcmp(&ptr[12], vlan, sizeof(vlan))) { 1710 int vid = lduw_be_p(ptr + 14) & 0xfff; 1711 if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f)))) 1712 return 0; 1713 } 1714 1715 if (ptr[0] & 1) { // multicast 1716 if (!memcmp(ptr, bcast, sizeof(bcast))) { 1717 return !n->nobcast; 1718 } else if (n->nomulti) { 1719 return 0; 1720 } else if (n->allmulti || n->mac_table.multi_overflow) { 1721 return 1; 1722 } 1723 1724 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 1725 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1726 return 1; 1727 } 1728 } 1729 } else { // unicast 1730 if (n->nouni) { 1731 return 0; 1732 } else if (n->alluni || n->mac_table.uni_overflow) { 1733 return 1; 1734 } else if (!memcmp(ptr, n->mac, ETH_ALEN)) { 1735 return 1; 1736 } 1737 1738 for (i = 0; i < n->mac_table.first_multi; i++) { 1739 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1740 return 1; 1741 } 1742 } 1743 } 1744 1745 return 0; 1746 } 1747 1748 static uint8_t virtio_net_get_hash_type(bool hasip4, 1749 bool hasip6, 1750 EthL4HdrProto l4hdr_proto, 1751 uint32_t types) 1752 { 1753 if (hasip4) { 1754 switch (l4hdr_proto) { 1755 case ETH_L4_HDR_PROTO_TCP: 1756 if (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { 1757 return NetPktRssIpV4Tcp; 1758 } 1759 break; 1760 1761 case ETH_L4_HDR_PROTO_UDP: 1762 if (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { 1763 return NetPktRssIpV4Udp; 1764 } 1765 break; 1766 1767 default: 1768 break; 1769 } 1770 1771 if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 1772 return NetPktRssIpV4; 1773 } 1774 } else if (hasip6) { 1775 switch (l4hdr_proto) { 1776 case ETH_L4_HDR_PROTO_TCP: 1777 if (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) { 1778 return NetPktRssIpV6TcpEx; 1779 } 1780 if (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { 1781 return NetPktRssIpV6Tcp; 1782 } 1783 break; 1784 1785 case ETH_L4_HDR_PROTO_UDP: 1786 if (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) { 1787 return NetPktRssIpV6UdpEx; 1788 } 1789 if (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { 1790 return NetPktRssIpV6Udp; 1791 } 1792 break; 1793 1794 default: 1795 break; 1796 } 1797 1798 if (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) { 1799 return NetPktRssIpV6Ex; 1800 } 1801 if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 1802 return NetPktRssIpV6; 1803 } 1804 } 1805 return 0xff; 1806 } 1807 1808 static void virtio_set_packet_hash(const uint8_t *buf, uint8_t report, 1809 uint32_t hash) 1810 { 1811 struct virtio_net_hdr_v1_hash *hdr = (void *)buf; 1812 hdr->hash_value = hash; 1813 hdr->hash_report = report; 1814 } 1815 1816 static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, 1817 size_t size) 1818 { 1819 VirtIONet *n = qemu_get_nic_opaque(nc); 1820 unsigned int index = nc->queue_index, new_index = index; 1821 struct NetRxPkt *pkt = n->rx_pkt; 1822 uint8_t net_hash_type; 1823 uint32_t hash; 1824 bool hasip4, hasip6; 1825 EthL4HdrProto l4hdr_proto; 1826 static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = { 1827 VIRTIO_NET_HASH_REPORT_IPv4, 1828 VIRTIO_NET_HASH_REPORT_TCPv4, 1829 VIRTIO_NET_HASH_REPORT_TCPv6, 1830 VIRTIO_NET_HASH_REPORT_IPv6, 1831 VIRTIO_NET_HASH_REPORT_IPv6_EX, 1832 VIRTIO_NET_HASH_REPORT_TCPv6_EX, 1833 VIRTIO_NET_HASH_REPORT_UDPv4, 1834 VIRTIO_NET_HASH_REPORT_UDPv6, 1835 VIRTIO_NET_HASH_REPORT_UDPv6_EX 1836 }; 1837 struct iovec iov = { 1838 .iov_base = (void *)buf, 1839 .iov_len = size 1840 }; 1841 1842 net_rx_pkt_set_protocols(pkt, &iov, 1, n->host_hdr_len); 1843 net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); 1844 net_hash_type = virtio_net_get_hash_type(hasip4, hasip6, l4hdr_proto, 1845 n->rss_data.hash_types); 1846 if (net_hash_type > NetPktRssIpV6UdpEx) { 1847 if (n->rss_data.populate_hash) { 1848 virtio_set_packet_hash(buf, VIRTIO_NET_HASH_REPORT_NONE, 0); 1849 } 1850 return n->rss_data.redirect ? n->rss_data.default_queue : -1; 1851 } 1852 1853 hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key); 1854 1855 if (n->rss_data.populate_hash) { 1856 virtio_set_packet_hash(buf, reports[net_hash_type], hash); 1857 } 1858 1859 if (n->rss_data.redirect) { 1860 new_index = hash & (n->rss_data.indirections_len - 1); 1861 new_index = n->rss_data.indirections_table[new_index]; 1862 } 1863 1864 return (index == new_index) ? -1 : new_index; 1865 } 1866 1867 static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, 1868 size_t size, bool no_rss) 1869 { 1870 VirtIONet *n = qemu_get_nic_opaque(nc); 1871 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1872 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1873 VirtQueueElement *elems[VIRTQUEUE_MAX_SIZE]; 1874 size_t lens[VIRTQUEUE_MAX_SIZE]; 1875 struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE]; 1876 struct virtio_net_hdr_mrg_rxbuf mhdr; 1877 unsigned mhdr_cnt = 0; 1878 size_t offset, i, guest_offset, j; 1879 ssize_t err; 1880 1881 if (!virtio_net_can_receive(nc)) { 1882 return -1; 1883 } 1884 1885 if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) { 1886 int index = virtio_net_process_rss(nc, buf, size); 1887 if (index >= 0) { 1888 NetClientState *nc2 = qemu_get_subqueue(n->nic, index); 1889 return virtio_net_receive_rcu(nc2, buf, size, true); 1890 } 1891 } 1892 1893 /* hdr_len refers to the header we supply to the guest */ 1894 if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) { 1895 return 0; 1896 } 1897 1898 if (!receive_filter(n, buf, size)) 1899 return size; 1900 1901 offset = i = 0; 1902 1903 while (offset < size) { 1904 VirtQueueElement *elem; 1905 int len, total; 1906 const struct iovec *sg; 1907 1908 total = 0; 1909 1910 if (i == VIRTQUEUE_MAX_SIZE) { 1911 virtio_error(vdev, "virtio-net unexpected long buffer chain"); 1912 err = size; 1913 goto err; 1914 } 1915 1916 elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement)); 1917 if (!elem) { 1918 if (i) { 1919 virtio_error(vdev, "virtio-net unexpected empty queue: " 1920 "i %zd mergeable %d offset %zd, size %zd, " 1921 "guest hdr len %zd, host hdr len %zd " 1922 "guest features 0x%" PRIx64, 1923 i, n->mergeable_rx_bufs, offset, size, 1924 n->guest_hdr_len, n->host_hdr_len, 1925 vdev->guest_features); 1926 } 1927 err = -1; 1928 goto err; 1929 } 1930 1931 if (elem->in_num < 1) { 1932 virtio_error(vdev, 1933 "virtio-net receive queue contains no in buffers"); 1934 virtqueue_detach_element(q->rx_vq, elem, 0); 1935 g_free(elem); 1936 err = -1; 1937 goto err; 1938 } 1939 1940 sg = elem->in_sg; 1941 if (i == 0) { 1942 assert(offset == 0); 1943 if (n->mergeable_rx_bufs) { 1944 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg), 1945 sg, elem->in_num, 1946 offsetof(typeof(mhdr), num_buffers), 1947 sizeof(mhdr.num_buffers)); 1948 } 1949 1950 receive_header(n, sg, elem->in_num, buf, size); 1951 if (n->rss_data.populate_hash) { 1952 offset = sizeof(mhdr); 1953 iov_from_buf(sg, elem->in_num, offset, 1954 buf + offset, n->host_hdr_len - sizeof(mhdr)); 1955 } 1956 offset = n->host_hdr_len; 1957 total += n->guest_hdr_len; 1958 guest_offset = n->guest_hdr_len; 1959 } else { 1960 guest_offset = 0; 1961 } 1962 1963 /* copy in packet. ugh */ 1964 len = iov_from_buf(sg, elem->in_num, guest_offset, 1965 buf + offset, size - offset); 1966 total += len; 1967 offset += len; 1968 /* If buffers can't be merged, at this point we 1969 * must have consumed the complete packet. 1970 * Otherwise, drop it. */ 1971 if (!n->mergeable_rx_bufs && offset < size) { 1972 virtqueue_unpop(q->rx_vq, elem, total); 1973 g_free(elem); 1974 err = size; 1975 goto err; 1976 } 1977 1978 elems[i] = elem; 1979 lens[i] = total; 1980 i++; 1981 } 1982 1983 if (mhdr_cnt) { 1984 virtio_stw_p(vdev, &mhdr.num_buffers, i); 1985 iov_from_buf(mhdr_sg, mhdr_cnt, 1986 0, 1987 &mhdr.num_buffers, sizeof mhdr.num_buffers); 1988 } 1989 1990 for (j = 0; j < i; j++) { 1991 /* signal other side */ 1992 virtqueue_fill(q->rx_vq, elems[j], lens[j], j); 1993 g_free(elems[j]); 1994 } 1995 1996 virtqueue_flush(q->rx_vq, i); 1997 virtio_notify(vdev, q->rx_vq); 1998 1999 return size; 2000 2001 err: 2002 for (j = 0; j < i; j++) { 2003 virtqueue_detach_element(q->rx_vq, elems[j], lens[j]); 2004 g_free(elems[j]); 2005 } 2006 2007 return err; 2008 } 2009 2010 static ssize_t virtio_net_do_receive(NetClientState *nc, const uint8_t *buf, 2011 size_t size) 2012 { 2013 RCU_READ_LOCK_GUARD(); 2014 2015 return virtio_net_receive_rcu(nc, buf, size, false); 2016 } 2017 2018 static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain, 2019 const uint8_t *buf, 2020 VirtioNetRscUnit *unit) 2021 { 2022 uint16_t ip_hdrlen; 2023 struct ip_header *ip; 2024 2025 ip = (struct ip_header *)(buf + chain->n->guest_hdr_len 2026 + sizeof(struct eth_header)); 2027 unit->ip = (void *)ip; 2028 ip_hdrlen = (ip->ip_ver_len & 0xF) << 2; 2029 unit->ip_plen = &ip->ip_len; 2030 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) + ip_hdrlen); 2031 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 2032 unit->payload = htons(*unit->ip_plen) - ip_hdrlen - unit->tcp_hdrlen; 2033 } 2034 2035 static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain, 2036 const uint8_t *buf, 2037 VirtioNetRscUnit *unit) 2038 { 2039 struct ip6_header *ip6; 2040 2041 ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len 2042 + sizeof(struct eth_header)); 2043 unit->ip = ip6; 2044 unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 2045 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) 2046 + sizeof(struct ip6_header)); 2047 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 2048 2049 /* There is a difference between payload lenght in ipv4 and v6, 2050 ip header is excluded in ipv6 */ 2051 unit->payload = htons(*unit->ip_plen) - unit->tcp_hdrlen; 2052 } 2053 2054 static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain, 2055 VirtioNetRscSeg *seg) 2056 { 2057 int ret; 2058 struct virtio_net_hdr_v1 *h; 2059 2060 h = (struct virtio_net_hdr_v1 *)seg->buf; 2061 h->flags = 0; 2062 h->gso_type = VIRTIO_NET_HDR_GSO_NONE; 2063 2064 if (seg->is_coalesced) { 2065 h->rsc.segments = seg->packets; 2066 h->rsc.dup_acks = seg->dup_ack; 2067 h->flags = VIRTIO_NET_HDR_F_RSC_INFO; 2068 if (chain->proto == ETH_P_IP) { 2069 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2070 } else { 2071 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 2072 } 2073 } 2074 2075 ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size); 2076 QTAILQ_REMOVE(&chain->buffers, seg, next); 2077 g_free(seg->buf); 2078 g_free(seg); 2079 2080 return ret; 2081 } 2082 2083 static void virtio_net_rsc_purge(void *opq) 2084 { 2085 VirtioNetRscSeg *seg, *rn; 2086 VirtioNetRscChain *chain = (VirtioNetRscChain *)opq; 2087 2088 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) { 2089 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2090 chain->stat.purge_failed++; 2091 continue; 2092 } 2093 } 2094 2095 chain->stat.timer++; 2096 if (!QTAILQ_EMPTY(&chain->buffers)) { 2097 timer_mod(chain->drain_timer, 2098 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 2099 } 2100 } 2101 2102 static void virtio_net_rsc_cleanup(VirtIONet *n) 2103 { 2104 VirtioNetRscChain *chain, *rn_chain; 2105 VirtioNetRscSeg *seg, *rn_seg; 2106 2107 QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) { 2108 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) { 2109 QTAILQ_REMOVE(&chain->buffers, seg, next); 2110 g_free(seg->buf); 2111 g_free(seg); 2112 } 2113 2114 timer_free(chain->drain_timer); 2115 QTAILQ_REMOVE(&n->rsc_chains, chain, next); 2116 g_free(chain); 2117 } 2118 } 2119 2120 static void virtio_net_rsc_cache_buf(VirtioNetRscChain *chain, 2121 NetClientState *nc, 2122 const uint8_t *buf, size_t size) 2123 { 2124 uint16_t hdr_len; 2125 VirtioNetRscSeg *seg; 2126 2127 hdr_len = chain->n->guest_hdr_len; 2128 seg = g_new(VirtioNetRscSeg, 1); 2129 seg->buf = g_malloc(hdr_len + sizeof(struct eth_header) 2130 + sizeof(struct ip6_header) + VIRTIO_NET_MAX_TCP_PAYLOAD); 2131 memcpy(seg->buf, buf, size); 2132 seg->size = size; 2133 seg->packets = 1; 2134 seg->dup_ack = 0; 2135 seg->is_coalesced = 0; 2136 seg->nc = nc; 2137 2138 QTAILQ_INSERT_TAIL(&chain->buffers, seg, next); 2139 chain->stat.cache++; 2140 2141 switch (chain->proto) { 2142 case ETH_P_IP: 2143 virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit); 2144 break; 2145 case ETH_P_IPV6: 2146 virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit); 2147 break; 2148 default: 2149 g_assert_not_reached(); 2150 } 2151 } 2152 2153 static int32_t virtio_net_rsc_handle_ack(VirtioNetRscChain *chain, 2154 VirtioNetRscSeg *seg, 2155 const uint8_t *buf, 2156 struct tcp_header *n_tcp, 2157 struct tcp_header *o_tcp) 2158 { 2159 uint32_t nack, oack; 2160 uint16_t nwin, owin; 2161 2162 nack = htonl(n_tcp->th_ack); 2163 nwin = htons(n_tcp->th_win); 2164 oack = htonl(o_tcp->th_ack); 2165 owin = htons(o_tcp->th_win); 2166 2167 if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) { 2168 chain->stat.ack_out_of_win++; 2169 return RSC_FINAL; 2170 } else if (nack == oack) { 2171 /* duplicated ack or window probe */ 2172 if (nwin == owin) { 2173 /* duplicated ack, add dup ack count due to whql test up to 1 */ 2174 chain->stat.dup_ack++; 2175 return RSC_FINAL; 2176 } else { 2177 /* Coalesce window update */ 2178 o_tcp->th_win = n_tcp->th_win; 2179 chain->stat.win_update++; 2180 return RSC_COALESCE; 2181 } 2182 } else { 2183 /* pure ack, go to 'C', finalize*/ 2184 chain->stat.pure_ack++; 2185 return RSC_FINAL; 2186 } 2187 } 2188 2189 static int32_t virtio_net_rsc_coalesce_data(VirtioNetRscChain *chain, 2190 VirtioNetRscSeg *seg, 2191 const uint8_t *buf, 2192 VirtioNetRscUnit *n_unit) 2193 { 2194 void *data; 2195 uint16_t o_ip_len; 2196 uint32_t nseq, oseq; 2197 VirtioNetRscUnit *o_unit; 2198 2199 o_unit = &seg->unit; 2200 o_ip_len = htons(*o_unit->ip_plen); 2201 nseq = htonl(n_unit->tcp->th_seq); 2202 oseq = htonl(o_unit->tcp->th_seq); 2203 2204 /* out of order or retransmitted. */ 2205 if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) { 2206 chain->stat.data_out_of_win++; 2207 return RSC_FINAL; 2208 } 2209 2210 data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen; 2211 if (nseq == oseq) { 2212 if ((o_unit->payload == 0) && n_unit->payload) { 2213 /* From no payload to payload, normal case, not a dup ack or etc */ 2214 chain->stat.data_after_pure_ack++; 2215 goto coalesce; 2216 } else { 2217 return virtio_net_rsc_handle_ack(chain, seg, buf, 2218 n_unit->tcp, o_unit->tcp); 2219 } 2220 } else if ((nseq - oseq) != o_unit->payload) { 2221 /* Not a consistent packet, out of order */ 2222 chain->stat.data_out_of_order++; 2223 return RSC_FINAL; 2224 } else { 2225 coalesce: 2226 if ((o_ip_len + n_unit->payload) > chain->max_payload) { 2227 chain->stat.over_size++; 2228 return RSC_FINAL; 2229 } 2230 2231 /* Here comes the right data, the payload length in v4/v6 is different, 2232 so use the field value to update and record the new data len */ 2233 o_unit->payload += n_unit->payload; /* update new data len */ 2234 2235 /* update field in ip header */ 2236 *o_unit->ip_plen = htons(o_ip_len + n_unit->payload); 2237 2238 /* Bring 'PUSH' big, the whql test guide says 'PUSH' can be coalesced 2239 for windows guest, while this may change the behavior for linux 2240 guest (only if it uses RSC feature). */ 2241 o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags; 2242 2243 o_unit->tcp->th_ack = n_unit->tcp->th_ack; 2244 o_unit->tcp->th_win = n_unit->tcp->th_win; 2245 2246 memmove(seg->buf + seg->size, data, n_unit->payload); 2247 seg->size += n_unit->payload; 2248 seg->packets++; 2249 chain->stat.coalesced++; 2250 return RSC_COALESCE; 2251 } 2252 } 2253 2254 static int32_t virtio_net_rsc_coalesce4(VirtioNetRscChain *chain, 2255 VirtioNetRscSeg *seg, 2256 const uint8_t *buf, size_t size, 2257 VirtioNetRscUnit *unit) 2258 { 2259 struct ip_header *ip1, *ip2; 2260 2261 ip1 = (struct ip_header *)(unit->ip); 2262 ip2 = (struct ip_header *)(seg->unit.ip); 2263 if ((ip1->ip_src ^ ip2->ip_src) || (ip1->ip_dst ^ ip2->ip_dst) 2264 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2265 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2266 chain->stat.no_match++; 2267 return RSC_NO_MATCH; 2268 } 2269 2270 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2271 } 2272 2273 static int32_t virtio_net_rsc_coalesce6(VirtioNetRscChain *chain, 2274 VirtioNetRscSeg *seg, 2275 const uint8_t *buf, size_t size, 2276 VirtioNetRscUnit *unit) 2277 { 2278 struct ip6_header *ip1, *ip2; 2279 2280 ip1 = (struct ip6_header *)(unit->ip); 2281 ip2 = (struct ip6_header *)(seg->unit.ip); 2282 if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address)) 2283 || memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address)) 2284 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2285 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2286 chain->stat.no_match++; 2287 return RSC_NO_MATCH; 2288 } 2289 2290 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2291 } 2292 2293 /* Packets with 'SYN' should bypass, other flag should be sent after drain 2294 * to prevent out of order */ 2295 static int virtio_net_rsc_tcp_ctrl_check(VirtioNetRscChain *chain, 2296 struct tcp_header *tcp) 2297 { 2298 uint16_t tcp_hdr; 2299 uint16_t tcp_flag; 2300 2301 tcp_flag = htons(tcp->th_offset_flags); 2302 tcp_hdr = (tcp_flag & VIRTIO_NET_TCP_HDR_LENGTH) >> 10; 2303 tcp_flag &= VIRTIO_NET_TCP_FLAG; 2304 if (tcp_flag & TH_SYN) { 2305 chain->stat.tcp_syn++; 2306 return RSC_BYPASS; 2307 } 2308 2309 if (tcp_flag & (TH_FIN | TH_URG | TH_RST | TH_ECE | TH_CWR)) { 2310 chain->stat.tcp_ctrl_drain++; 2311 return RSC_FINAL; 2312 } 2313 2314 if (tcp_hdr > sizeof(struct tcp_header)) { 2315 chain->stat.tcp_all_opt++; 2316 return RSC_FINAL; 2317 } 2318 2319 return RSC_CANDIDATE; 2320 } 2321 2322 static size_t virtio_net_rsc_do_coalesce(VirtioNetRscChain *chain, 2323 NetClientState *nc, 2324 const uint8_t *buf, size_t size, 2325 VirtioNetRscUnit *unit) 2326 { 2327 int ret; 2328 VirtioNetRscSeg *seg, *nseg; 2329 2330 if (QTAILQ_EMPTY(&chain->buffers)) { 2331 chain->stat.empty_cache++; 2332 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2333 timer_mod(chain->drain_timer, 2334 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 2335 return size; 2336 } 2337 2338 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2339 if (chain->proto == ETH_P_IP) { 2340 ret = virtio_net_rsc_coalesce4(chain, seg, buf, size, unit); 2341 } else { 2342 ret = virtio_net_rsc_coalesce6(chain, seg, buf, size, unit); 2343 } 2344 2345 if (ret == RSC_FINAL) { 2346 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2347 /* Send failed */ 2348 chain->stat.final_failed++; 2349 return 0; 2350 } 2351 2352 /* Send current packet */ 2353 return virtio_net_do_receive(nc, buf, size); 2354 } else if (ret == RSC_NO_MATCH) { 2355 continue; 2356 } else { 2357 /* Coalesced, mark coalesced flag to tell calc cksum for ipv4 */ 2358 seg->is_coalesced = 1; 2359 return size; 2360 } 2361 } 2362 2363 chain->stat.no_match_cache++; 2364 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2365 return size; 2366 } 2367 2368 /* Drain a connection data, this is to avoid out of order segments */ 2369 static size_t virtio_net_rsc_drain_flow(VirtioNetRscChain *chain, 2370 NetClientState *nc, 2371 const uint8_t *buf, size_t size, 2372 uint16_t ip_start, uint16_t ip_size, 2373 uint16_t tcp_port) 2374 { 2375 VirtioNetRscSeg *seg, *nseg; 2376 uint32_t ppair1, ppair2; 2377 2378 ppair1 = *(uint32_t *)(buf + tcp_port); 2379 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2380 ppair2 = *(uint32_t *)(seg->buf + tcp_port); 2381 if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size) 2382 || (ppair1 != ppair2)) { 2383 continue; 2384 } 2385 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2386 chain->stat.drain_failed++; 2387 } 2388 2389 break; 2390 } 2391 2392 return virtio_net_do_receive(nc, buf, size); 2393 } 2394 2395 static int32_t virtio_net_rsc_sanity_check4(VirtioNetRscChain *chain, 2396 struct ip_header *ip, 2397 const uint8_t *buf, size_t size) 2398 { 2399 uint16_t ip_len; 2400 2401 /* Not an ipv4 packet */ 2402 if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) { 2403 chain->stat.ip_option++; 2404 return RSC_BYPASS; 2405 } 2406 2407 /* Don't handle packets with ip option */ 2408 if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) { 2409 chain->stat.ip_option++; 2410 return RSC_BYPASS; 2411 } 2412 2413 if (ip->ip_p != IPPROTO_TCP) { 2414 chain->stat.bypass_not_tcp++; 2415 return RSC_BYPASS; 2416 } 2417 2418 /* Don't handle packets with ip fragment */ 2419 if (!(htons(ip->ip_off) & IP_DF)) { 2420 chain->stat.ip_frag++; 2421 return RSC_BYPASS; 2422 } 2423 2424 /* Don't handle packets with ecn flag */ 2425 if (IPTOS_ECN(ip->ip_tos)) { 2426 chain->stat.ip_ecn++; 2427 return RSC_BYPASS; 2428 } 2429 2430 ip_len = htons(ip->ip_len); 2431 if (ip_len < (sizeof(struct ip_header) + sizeof(struct tcp_header)) 2432 || ip_len > (size - chain->n->guest_hdr_len - 2433 sizeof(struct eth_header))) { 2434 chain->stat.ip_hacked++; 2435 return RSC_BYPASS; 2436 } 2437 2438 return RSC_CANDIDATE; 2439 } 2440 2441 static size_t virtio_net_rsc_receive4(VirtioNetRscChain *chain, 2442 NetClientState *nc, 2443 const uint8_t *buf, size_t size) 2444 { 2445 int32_t ret; 2446 uint16_t hdr_len; 2447 VirtioNetRscUnit unit; 2448 2449 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2450 2451 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header) 2452 + sizeof(struct tcp_header))) { 2453 chain->stat.bypass_not_tcp++; 2454 return virtio_net_do_receive(nc, buf, size); 2455 } 2456 2457 virtio_net_rsc_extract_unit4(chain, buf, &unit); 2458 if (virtio_net_rsc_sanity_check4(chain, unit.ip, buf, size) 2459 != RSC_CANDIDATE) { 2460 return virtio_net_do_receive(nc, buf, size); 2461 } 2462 2463 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2464 if (ret == RSC_BYPASS) { 2465 return virtio_net_do_receive(nc, buf, size); 2466 } else if (ret == RSC_FINAL) { 2467 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2468 ((hdr_len + sizeof(struct eth_header)) + 12), 2469 VIRTIO_NET_IP4_ADDR_SIZE, 2470 hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header)); 2471 } 2472 2473 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2474 } 2475 2476 static int32_t virtio_net_rsc_sanity_check6(VirtioNetRscChain *chain, 2477 struct ip6_header *ip6, 2478 const uint8_t *buf, size_t size) 2479 { 2480 uint16_t ip_len; 2481 2482 if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4) 2483 != IP_HEADER_VERSION_6) { 2484 return RSC_BYPASS; 2485 } 2486 2487 /* Both option and protocol is checked in this */ 2488 if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) { 2489 chain->stat.bypass_not_tcp++; 2490 return RSC_BYPASS; 2491 } 2492 2493 ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 2494 if (ip_len < sizeof(struct tcp_header) || 2495 ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header) 2496 - sizeof(struct ip6_header))) { 2497 chain->stat.ip_hacked++; 2498 return RSC_BYPASS; 2499 } 2500 2501 /* Don't handle packets with ecn flag */ 2502 if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) { 2503 chain->stat.ip_ecn++; 2504 return RSC_BYPASS; 2505 } 2506 2507 return RSC_CANDIDATE; 2508 } 2509 2510 static size_t virtio_net_rsc_receive6(void *opq, NetClientState *nc, 2511 const uint8_t *buf, size_t size) 2512 { 2513 int32_t ret; 2514 uint16_t hdr_len; 2515 VirtioNetRscChain *chain; 2516 VirtioNetRscUnit unit; 2517 2518 chain = opq; 2519 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2520 2521 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip6_header) 2522 + sizeof(tcp_header))) { 2523 return virtio_net_do_receive(nc, buf, size); 2524 } 2525 2526 virtio_net_rsc_extract_unit6(chain, buf, &unit); 2527 if (RSC_CANDIDATE != virtio_net_rsc_sanity_check6(chain, 2528 unit.ip, buf, size)) { 2529 return virtio_net_do_receive(nc, buf, size); 2530 } 2531 2532 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2533 if (ret == RSC_BYPASS) { 2534 return virtio_net_do_receive(nc, buf, size); 2535 } else if (ret == RSC_FINAL) { 2536 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2537 ((hdr_len + sizeof(struct eth_header)) + 8), 2538 VIRTIO_NET_IP6_ADDR_SIZE, 2539 hdr_len + sizeof(struct eth_header) 2540 + sizeof(struct ip6_header)); 2541 } 2542 2543 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2544 } 2545 2546 static VirtioNetRscChain *virtio_net_rsc_lookup_chain(VirtIONet *n, 2547 NetClientState *nc, 2548 uint16_t proto) 2549 { 2550 VirtioNetRscChain *chain; 2551 2552 if ((proto != (uint16_t)ETH_P_IP) && (proto != (uint16_t)ETH_P_IPV6)) { 2553 return NULL; 2554 } 2555 2556 QTAILQ_FOREACH(chain, &n->rsc_chains, next) { 2557 if (chain->proto == proto) { 2558 return chain; 2559 } 2560 } 2561 2562 chain = g_malloc(sizeof(*chain)); 2563 chain->n = n; 2564 chain->proto = proto; 2565 if (proto == (uint16_t)ETH_P_IP) { 2566 chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD; 2567 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2568 } else { 2569 chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD; 2570 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 2571 } 2572 chain->drain_timer = timer_new_ns(QEMU_CLOCK_HOST, 2573 virtio_net_rsc_purge, chain); 2574 memset(&chain->stat, 0, sizeof(chain->stat)); 2575 2576 QTAILQ_INIT(&chain->buffers); 2577 QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next); 2578 2579 return chain; 2580 } 2581 2582 static ssize_t virtio_net_rsc_receive(NetClientState *nc, 2583 const uint8_t *buf, 2584 size_t size) 2585 { 2586 uint16_t proto; 2587 VirtioNetRscChain *chain; 2588 struct eth_header *eth; 2589 VirtIONet *n; 2590 2591 n = qemu_get_nic_opaque(nc); 2592 if (size < (n->host_hdr_len + sizeof(struct eth_header))) { 2593 return virtio_net_do_receive(nc, buf, size); 2594 } 2595 2596 eth = (struct eth_header *)(buf + n->guest_hdr_len); 2597 proto = htons(eth->h_proto); 2598 2599 chain = virtio_net_rsc_lookup_chain(n, nc, proto); 2600 if (chain) { 2601 chain->stat.received++; 2602 if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) { 2603 return virtio_net_rsc_receive4(chain, nc, buf, size); 2604 } else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) { 2605 return virtio_net_rsc_receive6(chain, nc, buf, size); 2606 } 2607 } 2608 return virtio_net_do_receive(nc, buf, size); 2609 } 2610 2611 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, 2612 size_t size) 2613 { 2614 VirtIONet *n = qemu_get_nic_opaque(nc); 2615 if ((n->rsc4_enabled || n->rsc6_enabled)) { 2616 return virtio_net_rsc_receive(nc, buf, size); 2617 } else { 2618 return virtio_net_do_receive(nc, buf, size); 2619 } 2620 } 2621 2622 static int32_t virtio_net_flush_tx(VirtIONetQueue *q); 2623 2624 static void virtio_net_tx_complete(NetClientState *nc, ssize_t len) 2625 { 2626 VirtIONet *n = qemu_get_nic_opaque(nc); 2627 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 2628 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2629 int ret; 2630 2631 virtqueue_push(q->tx_vq, q->async_tx.elem, 0); 2632 virtio_notify(vdev, q->tx_vq); 2633 2634 g_free(q->async_tx.elem); 2635 q->async_tx.elem = NULL; 2636 2637 virtio_queue_set_notification(q->tx_vq, 1); 2638 ret = virtio_net_flush_tx(q); 2639 if (ret >= n->tx_burst) { 2640 /* 2641 * the flush has been stopped by tx_burst 2642 * we will not receive notification for the 2643 * remainining part, so re-schedule 2644 */ 2645 virtio_queue_set_notification(q->tx_vq, 0); 2646 if (q->tx_bh) { 2647 qemu_bh_schedule(q->tx_bh); 2648 } else { 2649 timer_mod(q->tx_timer, 2650 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2651 } 2652 q->tx_waiting = 1; 2653 } 2654 } 2655 2656 /* TX */ 2657 static int32_t virtio_net_flush_tx(VirtIONetQueue *q) 2658 { 2659 VirtIONet *n = q->n; 2660 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2661 VirtQueueElement *elem; 2662 int32_t num_packets = 0; 2663 int queue_index = vq2q(virtio_get_queue_index(q->tx_vq)); 2664 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2665 return num_packets; 2666 } 2667 2668 if (q->async_tx.elem) { 2669 virtio_queue_set_notification(q->tx_vq, 0); 2670 return num_packets; 2671 } 2672 2673 for (;;) { 2674 ssize_t ret; 2675 unsigned int out_num; 2676 struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg; 2677 struct virtio_net_hdr_mrg_rxbuf mhdr; 2678 2679 elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement)); 2680 if (!elem) { 2681 break; 2682 } 2683 2684 out_num = elem->out_num; 2685 out_sg = elem->out_sg; 2686 if (out_num < 1) { 2687 virtio_error(vdev, "virtio-net header not in first element"); 2688 virtqueue_detach_element(q->tx_vq, elem, 0); 2689 g_free(elem); 2690 return -EINVAL; 2691 } 2692 2693 if (n->has_vnet_hdr) { 2694 if (iov_to_buf(out_sg, out_num, 0, &mhdr, n->guest_hdr_len) < 2695 n->guest_hdr_len) { 2696 virtio_error(vdev, "virtio-net header incorrect"); 2697 virtqueue_detach_element(q->tx_vq, elem, 0); 2698 g_free(elem); 2699 return -EINVAL; 2700 } 2701 if (n->needs_vnet_hdr_swap) { 2702 virtio_net_hdr_swap(vdev, (void *) &mhdr); 2703 sg2[0].iov_base = &mhdr; 2704 sg2[0].iov_len = n->guest_hdr_len; 2705 out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1, 2706 out_sg, out_num, 2707 n->guest_hdr_len, -1); 2708 if (out_num == VIRTQUEUE_MAX_SIZE) { 2709 goto drop; 2710 } 2711 out_num += 1; 2712 out_sg = sg2; 2713 } 2714 } 2715 /* 2716 * If host wants to see the guest header as is, we can 2717 * pass it on unchanged. Otherwise, copy just the parts 2718 * that host is interested in. 2719 */ 2720 assert(n->host_hdr_len <= n->guest_hdr_len); 2721 if (n->host_hdr_len != n->guest_hdr_len) { 2722 unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg), 2723 out_sg, out_num, 2724 0, n->host_hdr_len); 2725 sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num, 2726 out_sg, out_num, 2727 n->guest_hdr_len, -1); 2728 out_num = sg_num; 2729 out_sg = sg; 2730 } 2731 2732 ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index), 2733 out_sg, out_num, virtio_net_tx_complete); 2734 if (ret == 0) { 2735 virtio_queue_set_notification(q->tx_vq, 0); 2736 q->async_tx.elem = elem; 2737 return -EBUSY; 2738 } 2739 2740 drop: 2741 virtqueue_push(q->tx_vq, elem, 0); 2742 virtio_notify(vdev, q->tx_vq); 2743 g_free(elem); 2744 2745 if (++num_packets >= n->tx_burst) { 2746 break; 2747 } 2748 } 2749 return num_packets; 2750 } 2751 2752 static void virtio_net_tx_timer(void *opaque); 2753 2754 static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq) 2755 { 2756 VirtIONet *n = VIRTIO_NET(vdev); 2757 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2758 2759 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2760 virtio_net_drop_tx_queue_data(vdev, vq); 2761 return; 2762 } 2763 2764 /* This happens when device was stopped but VCPU wasn't. */ 2765 if (!vdev->vm_running) { 2766 q->tx_waiting = 1; 2767 return; 2768 } 2769 2770 if (q->tx_waiting) { 2771 /* We already have queued packets, immediately flush */ 2772 timer_del(q->tx_timer); 2773 virtio_net_tx_timer(q); 2774 } else { 2775 /* re-arm timer to flush it (and more) on next tick */ 2776 timer_mod(q->tx_timer, 2777 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2778 q->tx_waiting = 1; 2779 virtio_queue_set_notification(vq, 0); 2780 } 2781 } 2782 2783 static void virtio_net_handle_tx_bh(VirtIODevice *vdev, VirtQueue *vq) 2784 { 2785 VirtIONet *n = VIRTIO_NET(vdev); 2786 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2787 2788 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2789 virtio_net_drop_tx_queue_data(vdev, vq); 2790 return; 2791 } 2792 2793 if (unlikely(q->tx_waiting)) { 2794 return; 2795 } 2796 q->tx_waiting = 1; 2797 /* This happens when device was stopped but VCPU wasn't. */ 2798 if (!vdev->vm_running) { 2799 return; 2800 } 2801 virtio_queue_set_notification(vq, 0); 2802 qemu_bh_schedule(q->tx_bh); 2803 } 2804 2805 static void virtio_net_tx_timer(void *opaque) 2806 { 2807 VirtIONetQueue *q = opaque; 2808 VirtIONet *n = q->n; 2809 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2810 int ret; 2811 2812 /* This happens when device was stopped but BH wasn't. */ 2813 if (!vdev->vm_running) { 2814 /* Make sure tx waiting is set, so we'll run when restarted. */ 2815 assert(q->tx_waiting); 2816 return; 2817 } 2818 2819 q->tx_waiting = 0; 2820 2821 /* Just in case the driver is not ready on more */ 2822 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2823 return; 2824 } 2825 2826 ret = virtio_net_flush_tx(q); 2827 if (ret == -EBUSY || ret == -EINVAL) { 2828 return; 2829 } 2830 /* 2831 * If we flush a full burst of packets, assume there are 2832 * more coming and immediately rearm 2833 */ 2834 if (ret >= n->tx_burst) { 2835 q->tx_waiting = 1; 2836 timer_mod(q->tx_timer, 2837 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2838 return; 2839 } 2840 /* 2841 * If less than a full burst, re-enable notification and flush 2842 * anything that may have come in while we weren't looking. If 2843 * we find something, assume the guest is still active and rearm 2844 */ 2845 virtio_queue_set_notification(q->tx_vq, 1); 2846 ret = virtio_net_flush_tx(q); 2847 if (ret > 0) { 2848 virtio_queue_set_notification(q->tx_vq, 0); 2849 q->tx_waiting = 1; 2850 timer_mod(q->tx_timer, 2851 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2852 } 2853 } 2854 2855 static void virtio_net_tx_bh(void *opaque) 2856 { 2857 VirtIONetQueue *q = opaque; 2858 VirtIONet *n = q->n; 2859 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2860 int32_t ret; 2861 2862 /* This happens when device was stopped but BH wasn't. */ 2863 if (!vdev->vm_running) { 2864 /* Make sure tx waiting is set, so we'll run when restarted. */ 2865 assert(q->tx_waiting); 2866 return; 2867 } 2868 2869 q->tx_waiting = 0; 2870 2871 /* Just in case the driver is not ready on more */ 2872 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 2873 return; 2874 } 2875 2876 ret = virtio_net_flush_tx(q); 2877 if (ret == -EBUSY || ret == -EINVAL) { 2878 return; /* Notification re-enable handled by tx_complete or device 2879 * broken */ 2880 } 2881 2882 /* If we flush a full burst of packets, assume there are 2883 * more coming and immediately reschedule */ 2884 if (ret >= n->tx_burst) { 2885 qemu_bh_schedule(q->tx_bh); 2886 q->tx_waiting = 1; 2887 return; 2888 } 2889 2890 /* If less than a full burst, re-enable notification and flush 2891 * anything that may have come in while we weren't looking. If 2892 * we find something, assume the guest is still active and reschedule */ 2893 virtio_queue_set_notification(q->tx_vq, 1); 2894 ret = virtio_net_flush_tx(q); 2895 if (ret == -EINVAL) { 2896 return; 2897 } else if (ret > 0) { 2898 virtio_queue_set_notification(q->tx_vq, 0); 2899 qemu_bh_schedule(q->tx_bh); 2900 q->tx_waiting = 1; 2901 } 2902 } 2903 2904 static void virtio_net_add_queue(VirtIONet *n, int index) 2905 { 2906 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2907 2908 n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size, 2909 virtio_net_handle_rx); 2910 2911 if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) { 2912 n->vqs[index].tx_vq = 2913 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2914 virtio_net_handle_tx_timer); 2915 n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, 2916 virtio_net_tx_timer, 2917 &n->vqs[index]); 2918 } else { 2919 n->vqs[index].tx_vq = 2920 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2921 virtio_net_handle_tx_bh); 2922 n->vqs[index].tx_bh = qemu_bh_new_guarded(virtio_net_tx_bh, &n->vqs[index], 2923 &DEVICE(vdev)->mem_reentrancy_guard); 2924 } 2925 2926 n->vqs[index].tx_waiting = 0; 2927 n->vqs[index].n = n; 2928 } 2929 2930 static void virtio_net_del_queue(VirtIONet *n, int index) 2931 { 2932 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2933 VirtIONetQueue *q = &n->vqs[index]; 2934 NetClientState *nc = qemu_get_subqueue(n->nic, index); 2935 2936 qemu_purge_queued_packets(nc); 2937 2938 virtio_del_queue(vdev, index * 2); 2939 if (q->tx_timer) { 2940 timer_free(q->tx_timer); 2941 q->tx_timer = NULL; 2942 } else { 2943 qemu_bh_delete(q->tx_bh); 2944 q->tx_bh = NULL; 2945 } 2946 q->tx_waiting = 0; 2947 virtio_del_queue(vdev, index * 2 + 1); 2948 } 2949 2950 static void virtio_net_change_num_queue_pairs(VirtIONet *n, int new_max_queue_pairs) 2951 { 2952 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2953 int old_num_queues = virtio_get_num_queues(vdev); 2954 int new_num_queues = new_max_queue_pairs * 2 + 1; 2955 int i; 2956 2957 assert(old_num_queues >= 3); 2958 assert(old_num_queues % 2 == 1); 2959 2960 if (old_num_queues == new_num_queues) { 2961 return; 2962 } 2963 2964 /* 2965 * We always need to remove and add ctrl vq if 2966 * old_num_queues != new_num_queues. Remove ctrl_vq first, 2967 * and then we only enter one of the following two loops. 2968 */ 2969 virtio_del_queue(vdev, old_num_queues - 1); 2970 2971 for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) { 2972 /* new_num_queues < old_num_queues */ 2973 virtio_net_del_queue(n, i / 2); 2974 } 2975 2976 for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) { 2977 /* new_num_queues > old_num_queues */ 2978 virtio_net_add_queue(n, i / 2); 2979 } 2980 2981 /* add ctrl_vq last */ 2982 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 2983 } 2984 2985 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue) 2986 { 2987 int max = multiqueue ? n->max_queue_pairs : 1; 2988 2989 n->multiqueue = multiqueue; 2990 virtio_net_change_num_queue_pairs(n, max); 2991 2992 virtio_net_set_queue_pairs(n); 2993 } 2994 2995 static int virtio_net_post_load_device(void *opaque, int version_id) 2996 { 2997 VirtIONet *n = opaque; 2998 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2999 int i, link_down; 3000 3001 trace_virtio_net_post_load_device(); 3002 virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs, 3003 virtio_vdev_has_feature(vdev, 3004 VIRTIO_F_VERSION_1), 3005 virtio_vdev_has_feature(vdev, 3006 VIRTIO_NET_F_HASH_REPORT)); 3007 3008 /* MAC_TABLE_ENTRIES may be different from the saved image */ 3009 if (n->mac_table.in_use > MAC_TABLE_ENTRIES) { 3010 n->mac_table.in_use = 0; 3011 } 3012 3013 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 3014 n->curr_guest_offloads = virtio_net_supported_guest_offloads(n); 3015 } 3016 3017 /* 3018 * curr_guest_offloads will be later overwritten by the 3019 * virtio_set_features_nocheck call done from the virtio_load. 3020 * Here we make sure it is preserved and restored accordingly 3021 * in the virtio_net_post_load_virtio callback. 3022 */ 3023 n->saved_guest_offloads = n->curr_guest_offloads; 3024 3025 virtio_net_set_queue_pairs(n); 3026 3027 /* Find the first multicast entry in the saved MAC filter */ 3028 for (i = 0; i < n->mac_table.in_use; i++) { 3029 if (n->mac_table.macs[i * ETH_ALEN] & 1) { 3030 break; 3031 } 3032 } 3033 n->mac_table.first_multi = i; 3034 3035 /* nc.link_down can't be migrated, so infer link_down according 3036 * to link status bit in n->status */ 3037 link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0; 3038 for (i = 0; i < n->max_queue_pairs; i++) { 3039 qemu_get_subqueue(n->nic, i)->link_down = link_down; 3040 } 3041 3042 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 3043 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 3044 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 3045 QEMU_CLOCK_VIRTUAL, 3046 virtio_net_announce_timer, n); 3047 if (n->announce_timer.round) { 3048 timer_mod(n->announce_timer.tm, 3049 qemu_clock_get_ms(n->announce_timer.type)); 3050 } else { 3051 qemu_announce_timer_del(&n->announce_timer, false); 3052 } 3053 } 3054 3055 if (n->rss_data.enabled) { 3056 n->rss_data.enabled_software_rss = n->rss_data.populate_hash; 3057 if (!n->rss_data.populate_hash) { 3058 if (!virtio_net_attach_epbf_rss(n)) { 3059 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 3060 warn_report("Can't post-load eBPF RSS for vhost"); 3061 } else { 3062 warn_report("Can't post-load eBPF RSS - " 3063 "fallback to software RSS"); 3064 n->rss_data.enabled_software_rss = true; 3065 } 3066 } 3067 } 3068 3069 trace_virtio_net_rss_enable(n->rss_data.hash_types, 3070 n->rss_data.indirections_len, 3071 sizeof(n->rss_data.key)); 3072 } else { 3073 trace_virtio_net_rss_disable(); 3074 } 3075 return 0; 3076 } 3077 3078 static int virtio_net_post_load_virtio(VirtIODevice *vdev) 3079 { 3080 VirtIONet *n = VIRTIO_NET(vdev); 3081 /* 3082 * The actual needed state is now in saved_guest_offloads, 3083 * see virtio_net_post_load_device for detail. 3084 * Restore it back and apply the desired offloads. 3085 */ 3086 n->curr_guest_offloads = n->saved_guest_offloads; 3087 if (peer_has_vnet_hdr(n)) { 3088 virtio_net_apply_guest_offloads(n); 3089 } 3090 3091 return 0; 3092 } 3093 3094 /* tx_waiting field of a VirtIONetQueue */ 3095 static const VMStateDescription vmstate_virtio_net_queue_tx_waiting = { 3096 .name = "virtio-net-queue-tx_waiting", 3097 .fields = (VMStateField[]) { 3098 VMSTATE_UINT32(tx_waiting, VirtIONetQueue), 3099 VMSTATE_END_OF_LIST() 3100 }, 3101 }; 3102 3103 static bool max_queue_pairs_gt_1(void *opaque, int version_id) 3104 { 3105 return VIRTIO_NET(opaque)->max_queue_pairs > 1; 3106 } 3107 3108 static bool has_ctrl_guest_offloads(void *opaque, int version_id) 3109 { 3110 return virtio_vdev_has_feature(VIRTIO_DEVICE(opaque), 3111 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 3112 } 3113 3114 static bool mac_table_fits(void *opaque, int version_id) 3115 { 3116 return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES; 3117 } 3118 3119 static bool mac_table_doesnt_fit(void *opaque, int version_id) 3120 { 3121 return !mac_table_fits(opaque, version_id); 3122 } 3123 3124 /* This temporary type is shared by all the WITH_TMP methods 3125 * although only some fields are used by each. 3126 */ 3127 struct VirtIONetMigTmp { 3128 VirtIONet *parent; 3129 VirtIONetQueue *vqs_1; 3130 uint16_t curr_queue_pairs_1; 3131 uint8_t has_ufo; 3132 uint32_t has_vnet_hdr; 3133 }; 3134 3135 /* The 2nd and subsequent tx_waiting flags are loaded later than 3136 * the 1st entry in the queue_pairs and only if there's more than one 3137 * entry. We use the tmp mechanism to calculate a temporary 3138 * pointer and count and also validate the count. 3139 */ 3140 3141 static int virtio_net_tx_waiting_pre_save(void *opaque) 3142 { 3143 struct VirtIONetMigTmp *tmp = opaque; 3144 3145 tmp->vqs_1 = tmp->parent->vqs + 1; 3146 tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1; 3147 if (tmp->parent->curr_queue_pairs == 0) { 3148 tmp->curr_queue_pairs_1 = 0; 3149 } 3150 3151 return 0; 3152 } 3153 3154 static int virtio_net_tx_waiting_pre_load(void *opaque) 3155 { 3156 struct VirtIONetMigTmp *tmp = opaque; 3157 3158 /* Reuse the pointer setup from save */ 3159 virtio_net_tx_waiting_pre_save(opaque); 3160 3161 if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) { 3162 error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x", 3163 tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs); 3164 3165 return -EINVAL; 3166 } 3167 3168 return 0; /* all good */ 3169 } 3170 3171 static const VMStateDescription vmstate_virtio_net_tx_waiting = { 3172 .name = "virtio-net-tx_waiting", 3173 .pre_load = virtio_net_tx_waiting_pre_load, 3174 .pre_save = virtio_net_tx_waiting_pre_save, 3175 .fields = (VMStateField[]) { 3176 VMSTATE_STRUCT_VARRAY_POINTER_UINT16(vqs_1, struct VirtIONetMigTmp, 3177 curr_queue_pairs_1, 3178 vmstate_virtio_net_queue_tx_waiting, 3179 struct VirtIONetQueue), 3180 VMSTATE_END_OF_LIST() 3181 }, 3182 }; 3183 3184 /* the 'has_ufo' flag is just tested; if the incoming stream has the 3185 * flag set we need to check that we have it 3186 */ 3187 static int virtio_net_ufo_post_load(void *opaque, int version_id) 3188 { 3189 struct VirtIONetMigTmp *tmp = opaque; 3190 3191 if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) { 3192 error_report("virtio-net: saved image requires TUN_F_UFO support"); 3193 return -EINVAL; 3194 } 3195 3196 return 0; 3197 } 3198 3199 static int virtio_net_ufo_pre_save(void *opaque) 3200 { 3201 struct VirtIONetMigTmp *tmp = opaque; 3202 3203 tmp->has_ufo = tmp->parent->has_ufo; 3204 3205 return 0; 3206 } 3207 3208 static const VMStateDescription vmstate_virtio_net_has_ufo = { 3209 .name = "virtio-net-ufo", 3210 .post_load = virtio_net_ufo_post_load, 3211 .pre_save = virtio_net_ufo_pre_save, 3212 .fields = (VMStateField[]) { 3213 VMSTATE_UINT8(has_ufo, struct VirtIONetMigTmp), 3214 VMSTATE_END_OF_LIST() 3215 }, 3216 }; 3217 3218 /* the 'has_vnet_hdr' flag is just tested; if the incoming stream has the 3219 * flag set we need to check that we have it 3220 */ 3221 static int virtio_net_vnet_post_load(void *opaque, int version_id) 3222 { 3223 struct VirtIONetMigTmp *tmp = opaque; 3224 3225 if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) { 3226 error_report("virtio-net: saved image requires vnet_hdr=on"); 3227 return -EINVAL; 3228 } 3229 3230 return 0; 3231 } 3232 3233 static int virtio_net_vnet_pre_save(void *opaque) 3234 { 3235 struct VirtIONetMigTmp *tmp = opaque; 3236 3237 tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr; 3238 3239 return 0; 3240 } 3241 3242 static const VMStateDescription vmstate_virtio_net_has_vnet = { 3243 .name = "virtio-net-vnet", 3244 .post_load = virtio_net_vnet_post_load, 3245 .pre_save = virtio_net_vnet_pre_save, 3246 .fields = (VMStateField[]) { 3247 VMSTATE_UINT32(has_vnet_hdr, struct VirtIONetMigTmp), 3248 VMSTATE_END_OF_LIST() 3249 }, 3250 }; 3251 3252 static bool virtio_net_rss_needed(void *opaque) 3253 { 3254 return VIRTIO_NET(opaque)->rss_data.enabled; 3255 } 3256 3257 static const VMStateDescription vmstate_virtio_net_rss = { 3258 .name = "virtio-net-device/rss", 3259 .version_id = 1, 3260 .minimum_version_id = 1, 3261 .needed = virtio_net_rss_needed, 3262 .fields = (VMStateField[]) { 3263 VMSTATE_BOOL(rss_data.enabled, VirtIONet), 3264 VMSTATE_BOOL(rss_data.redirect, VirtIONet), 3265 VMSTATE_BOOL(rss_data.populate_hash, VirtIONet), 3266 VMSTATE_UINT32(rss_data.hash_types, VirtIONet), 3267 VMSTATE_UINT16(rss_data.indirections_len, VirtIONet), 3268 VMSTATE_UINT16(rss_data.default_queue, VirtIONet), 3269 VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet, 3270 VIRTIO_NET_RSS_MAX_KEY_SIZE), 3271 VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet, 3272 rss_data.indirections_len, 0, 3273 vmstate_info_uint16, uint16_t), 3274 VMSTATE_END_OF_LIST() 3275 }, 3276 }; 3277 3278 static const VMStateDescription vmstate_virtio_net_device = { 3279 .name = "virtio-net-device", 3280 .version_id = VIRTIO_NET_VM_VERSION, 3281 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3282 .post_load = virtio_net_post_load_device, 3283 .fields = (VMStateField[]) { 3284 VMSTATE_UINT8_ARRAY(mac, VirtIONet, ETH_ALEN), 3285 VMSTATE_STRUCT_POINTER(vqs, VirtIONet, 3286 vmstate_virtio_net_queue_tx_waiting, 3287 VirtIONetQueue), 3288 VMSTATE_UINT32(mergeable_rx_bufs, VirtIONet), 3289 VMSTATE_UINT16(status, VirtIONet), 3290 VMSTATE_UINT8(promisc, VirtIONet), 3291 VMSTATE_UINT8(allmulti, VirtIONet), 3292 VMSTATE_UINT32(mac_table.in_use, VirtIONet), 3293 3294 /* Guarded pair: If it fits we load it, else we throw it away 3295 * - can happen if source has a larger MAC table.; post-load 3296 * sets flags in this case. 3297 */ 3298 VMSTATE_VBUFFER_MULTIPLY(mac_table.macs, VirtIONet, 3299 0, mac_table_fits, mac_table.in_use, 3300 ETH_ALEN), 3301 VMSTATE_UNUSED_VARRAY_UINT32(VirtIONet, mac_table_doesnt_fit, 0, 3302 mac_table.in_use, ETH_ALEN), 3303 3304 /* Note: This is an array of uint32's that's always been saved as a 3305 * buffer; hold onto your endiannesses; it's actually used as a bitmap 3306 * but based on the uint. 3307 */ 3308 VMSTATE_BUFFER_POINTER_UNSAFE(vlans, VirtIONet, 0, MAX_VLAN >> 3), 3309 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3310 vmstate_virtio_net_has_vnet), 3311 VMSTATE_UINT8(mac_table.multi_overflow, VirtIONet), 3312 VMSTATE_UINT8(mac_table.uni_overflow, VirtIONet), 3313 VMSTATE_UINT8(alluni, VirtIONet), 3314 VMSTATE_UINT8(nomulti, VirtIONet), 3315 VMSTATE_UINT8(nouni, VirtIONet), 3316 VMSTATE_UINT8(nobcast, VirtIONet), 3317 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3318 vmstate_virtio_net_has_ufo), 3319 VMSTATE_SINGLE_TEST(max_queue_pairs, VirtIONet, max_queue_pairs_gt_1, 0, 3320 vmstate_info_uint16_equal, uint16_t), 3321 VMSTATE_UINT16_TEST(curr_queue_pairs, VirtIONet, max_queue_pairs_gt_1), 3322 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3323 vmstate_virtio_net_tx_waiting), 3324 VMSTATE_UINT64_TEST(curr_guest_offloads, VirtIONet, 3325 has_ctrl_guest_offloads), 3326 VMSTATE_END_OF_LIST() 3327 }, 3328 .subsections = (const VMStateDescription * []) { 3329 &vmstate_virtio_net_rss, 3330 NULL 3331 } 3332 }; 3333 3334 static NetClientInfo net_virtio_info = { 3335 .type = NET_CLIENT_DRIVER_NIC, 3336 .size = sizeof(NICState), 3337 .can_receive = virtio_net_can_receive, 3338 .receive = virtio_net_receive, 3339 .link_status_changed = virtio_net_set_link_status, 3340 .query_rx_filter = virtio_net_query_rxfilter, 3341 .announce = virtio_net_announce, 3342 }; 3343 3344 static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx) 3345 { 3346 VirtIONet *n = VIRTIO_NET(vdev); 3347 NetClientState *nc; 3348 assert(n->vhost_started); 3349 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ) && idx == 2) { 3350 /* Must guard against invalid features and bogus queue index 3351 * from being set by malicious guest, or penetrated through 3352 * buggy migration stream. 3353 */ 3354 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 3355 qemu_log_mask(LOG_GUEST_ERROR, 3356 "%s: bogus vq index ignored\n", __func__); 3357 return false; 3358 } 3359 nc = qemu_get_subqueue(n->nic, n->max_queue_pairs); 3360 } else { 3361 nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3362 } 3363 /* 3364 * Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1 3365 * as the Marco of configure interrupt's IDX, If this driver does not 3366 * support, the function will return false 3367 */ 3368 3369 if (idx == VIRTIO_CONFIG_IRQ_IDX) { 3370 return vhost_net_config_pending(get_vhost_net(nc->peer)); 3371 } 3372 return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx); 3373 } 3374 3375 static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx, 3376 bool mask) 3377 { 3378 VirtIONet *n = VIRTIO_NET(vdev); 3379 NetClientState *nc; 3380 assert(n->vhost_started); 3381 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ) && idx == 2) { 3382 /* Must guard against invalid features and bogus queue index 3383 * from being set by malicious guest, or penetrated through 3384 * buggy migration stream. 3385 */ 3386 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 3387 qemu_log_mask(LOG_GUEST_ERROR, 3388 "%s: bogus vq index ignored\n", __func__); 3389 return; 3390 } 3391 nc = qemu_get_subqueue(n->nic, n->max_queue_pairs); 3392 } else { 3393 nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3394 } 3395 /* 3396 *Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1 3397 * as the Marco of configure interrupt's IDX, If this driver does not 3398 * support, the function will return 3399 */ 3400 3401 if (idx == VIRTIO_CONFIG_IRQ_IDX) { 3402 vhost_net_config_mask(get_vhost_net(nc->peer), vdev, mask); 3403 return; 3404 } 3405 vhost_net_virtqueue_mask(get_vhost_net(nc->peer), vdev, idx, mask); 3406 } 3407 3408 static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features) 3409 { 3410 virtio_add_feature(&host_features, VIRTIO_NET_F_MAC); 3411 3412 n->config_size = virtio_get_config_size(&cfg_size_params, host_features); 3413 } 3414 3415 void virtio_net_set_netclient_name(VirtIONet *n, const char *name, 3416 const char *type) 3417 { 3418 /* 3419 * The name can be NULL, the netclient name will be type.x. 3420 */ 3421 assert(type != NULL); 3422 3423 g_free(n->netclient_name); 3424 g_free(n->netclient_type); 3425 n->netclient_name = g_strdup(name); 3426 n->netclient_type = g_strdup(type); 3427 } 3428 3429 static bool failover_unplug_primary(VirtIONet *n, DeviceState *dev) 3430 { 3431 HotplugHandler *hotplug_ctrl; 3432 PCIDevice *pci_dev; 3433 Error *err = NULL; 3434 3435 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3436 if (hotplug_ctrl) { 3437 pci_dev = PCI_DEVICE(dev); 3438 pci_dev->partially_hotplugged = true; 3439 hotplug_handler_unplug_request(hotplug_ctrl, dev, &err); 3440 if (err) { 3441 error_report_err(err); 3442 return false; 3443 } 3444 } else { 3445 return false; 3446 } 3447 return true; 3448 } 3449 3450 static bool failover_replug_primary(VirtIONet *n, DeviceState *dev, 3451 Error **errp) 3452 { 3453 Error *err = NULL; 3454 HotplugHandler *hotplug_ctrl; 3455 PCIDevice *pdev = PCI_DEVICE(dev); 3456 BusState *primary_bus; 3457 3458 if (!pdev->partially_hotplugged) { 3459 return true; 3460 } 3461 primary_bus = dev->parent_bus; 3462 if (!primary_bus) { 3463 error_setg(errp, "virtio_net: couldn't find primary bus"); 3464 return false; 3465 } 3466 qdev_set_parent_bus(dev, primary_bus, &error_abort); 3467 qatomic_set(&n->failover_primary_hidden, false); 3468 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3469 if (hotplug_ctrl) { 3470 hotplug_handler_pre_plug(hotplug_ctrl, dev, &err); 3471 if (err) { 3472 goto out; 3473 } 3474 hotplug_handler_plug(hotplug_ctrl, dev, &err); 3475 } 3476 pdev->partially_hotplugged = false; 3477 3478 out: 3479 error_propagate(errp, err); 3480 return !err; 3481 } 3482 3483 static void virtio_net_handle_migration_primary(VirtIONet *n, MigrationState *s) 3484 { 3485 bool should_be_hidden; 3486 Error *err = NULL; 3487 DeviceState *dev = failover_find_primary_device(n); 3488 3489 if (!dev) { 3490 return; 3491 } 3492 3493 should_be_hidden = qatomic_read(&n->failover_primary_hidden); 3494 3495 if (migration_in_setup(s) && !should_be_hidden) { 3496 if (failover_unplug_primary(n, dev)) { 3497 vmstate_unregister(VMSTATE_IF(dev), qdev_get_vmsd(dev), dev); 3498 qapi_event_send_unplug_primary(dev->id); 3499 qatomic_set(&n->failover_primary_hidden, true); 3500 } else { 3501 warn_report("couldn't unplug primary device"); 3502 } 3503 } else if (migration_has_failed(s)) { 3504 /* We already unplugged the device let's plug it back */ 3505 if (!failover_replug_primary(n, dev, &err)) { 3506 if (err) { 3507 error_report_err(err); 3508 } 3509 } 3510 } 3511 } 3512 3513 static void virtio_net_migration_state_notifier(Notifier *notifier, void *data) 3514 { 3515 MigrationState *s = data; 3516 VirtIONet *n = container_of(notifier, VirtIONet, migration_state); 3517 virtio_net_handle_migration_primary(n, s); 3518 } 3519 3520 static bool failover_hide_primary_device(DeviceListener *listener, 3521 const QDict *device_opts, 3522 bool from_json, 3523 Error **errp) 3524 { 3525 VirtIONet *n = container_of(listener, VirtIONet, primary_listener); 3526 const char *standby_id; 3527 3528 if (!device_opts) { 3529 return false; 3530 } 3531 3532 if (!qdict_haskey(device_opts, "failover_pair_id")) { 3533 return false; 3534 } 3535 3536 if (!qdict_haskey(device_opts, "id")) { 3537 error_setg(errp, "Device with failover_pair_id needs to have id"); 3538 return false; 3539 } 3540 3541 standby_id = qdict_get_str(device_opts, "failover_pair_id"); 3542 if (g_strcmp0(standby_id, n->netclient_name) != 0) { 3543 return false; 3544 } 3545 3546 /* 3547 * The hide helper can be called several times for a given device. 3548 * Check there is only one primary for a virtio-net device but 3549 * don't duplicate the qdict several times if it's called for the same 3550 * device. 3551 */ 3552 if (n->primary_opts) { 3553 const char *old, *new; 3554 /* devices with failover_pair_id always have an id */ 3555 old = qdict_get_str(n->primary_opts, "id"); 3556 new = qdict_get_str(device_opts, "id"); 3557 if (strcmp(old, new) != 0) { 3558 error_setg(errp, "Cannot attach more than one primary device to " 3559 "'%s': '%s' and '%s'", n->netclient_name, old, new); 3560 return false; 3561 } 3562 } else { 3563 n->primary_opts = qdict_clone_shallow(device_opts); 3564 n->primary_opts_from_json = from_json; 3565 } 3566 3567 /* failover_primary_hidden is set during feature negotiation */ 3568 return qatomic_read(&n->failover_primary_hidden); 3569 } 3570 3571 static void virtio_net_device_realize(DeviceState *dev, Error **errp) 3572 { 3573 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3574 VirtIONet *n = VIRTIO_NET(dev); 3575 NetClientState *nc; 3576 int i; 3577 3578 if (n->net_conf.mtu) { 3579 n->host_features |= (1ULL << VIRTIO_NET_F_MTU); 3580 } 3581 3582 if (n->net_conf.duplex_str) { 3583 if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) { 3584 n->net_conf.duplex = DUPLEX_HALF; 3585 } else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) { 3586 n->net_conf.duplex = DUPLEX_FULL; 3587 } else { 3588 error_setg(errp, "'duplex' must be 'half' or 'full'"); 3589 return; 3590 } 3591 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3592 } else { 3593 n->net_conf.duplex = DUPLEX_UNKNOWN; 3594 } 3595 3596 if (n->net_conf.speed < SPEED_UNKNOWN) { 3597 error_setg(errp, "'speed' must be between 0 and INT_MAX"); 3598 return; 3599 } 3600 if (n->net_conf.speed >= 0) { 3601 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3602 } 3603 3604 if (n->failover) { 3605 n->primary_listener.hide_device = failover_hide_primary_device; 3606 qatomic_set(&n->failover_primary_hidden, true); 3607 device_listener_register(&n->primary_listener); 3608 n->migration_state.notify = virtio_net_migration_state_notifier; 3609 add_migration_state_change_notifier(&n->migration_state); 3610 n->host_features |= (1ULL << VIRTIO_NET_F_STANDBY); 3611 } 3612 3613 virtio_net_set_config_size(n, n->host_features); 3614 virtio_init(vdev, VIRTIO_ID_NET, n->config_size); 3615 3616 /* 3617 * We set a lower limit on RX queue size to what it always was. 3618 * Guests that want a smaller ring can always resize it without 3619 * help from us (using virtio 1 and up). 3620 */ 3621 if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE || 3622 n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE || 3623 !is_power_of_2(n->net_conf.rx_queue_size)) { 3624 error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), " 3625 "must be a power of 2 between %d and %d.", 3626 n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE, 3627 VIRTQUEUE_MAX_SIZE); 3628 virtio_cleanup(vdev); 3629 return; 3630 } 3631 3632 if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE || 3633 n->net_conf.tx_queue_size > virtio_net_max_tx_queue_size(n) || 3634 !is_power_of_2(n->net_conf.tx_queue_size)) { 3635 error_setg(errp, "Invalid tx_queue_size (= %" PRIu16 "), " 3636 "must be a power of 2 between %d and %d", 3637 n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE, 3638 virtio_net_max_tx_queue_size(n)); 3639 virtio_cleanup(vdev); 3640 return; 3641 } 3642 3643 n->max_ncs = MAX(n->nic_conf.peers.queues, 1); 3644 3645 /* 3646 * Figure out the datapath queue pairs since the backend could 3647 * provide control queue via peers as well. 3648 */ 3649 if (n->nic_conf.peers.queues) { 3650 for (i = 0; i < n->max_ncs; i++) { 3651 if (n->nic_conf.peers.ncs[i]->is_datapath) { 3652 ++n->max_queue_pairs; 3653 } 3654 } 3655 } 3656 n->max_queue_pairs = MAX(n->max_queue_pairs, 1); 3657 3658 if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) { 3659 error_setg(errp, "Invalid number of queue pairs (= %" PRIu32 "), " 3660 "must be a positive integer less than %d.", 3661 n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2); 3662 virtio_cleanup(vdev); 3663 return; 3664 } 3665 n->vqs = g_new0(VirtIONetQueue, n->max_queue_pairs); 3666 n->curr_queue_pairs = 1; 3667 n->tx_timeout = n->net_conf.txtimer; 3668 3669 if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer") 3670 && strcmp(n->net_conf.tx, "bh")) { 3671 warn_report("virtio-net: " 3672 "Unknown option tx=%s, valid options: \"timer\" \"bh\"", 3673 n->net_conf.tx); 3674 error_printf("Defaulting to \"bh\""); 3675 } 3676 3677 n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n), 3678 n->net_conf.tx_queue_size); 3679 3680 for (i = 0; i < n->max_queue_pairs; i++) { 3681 virtio_net_add_queue(n, i); 3682 } 3683 3684 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 3685 qemu_macaddr_default_if_unset(&n->nic_conf.macaddr); 3686 memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac)); 3687 n->status = VIRTIO_NET_S_LINK_UP; 3688 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 3689 QEMU_CLOCK_VIRTUAL, 3690 virtio_net_announce_timer, n); 3691 n->announce_timer.round = 0; 3692 3693 if (n->netclient_type) { 3694 /* 3695 * Happen when virtio_net_set_netclient_name has been called. 3696 */ 3697 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3698 n->netclient_type, n->netclient_name, n); 3699 } else { 3700 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3701 object_get_typename(OBJECT(dev)), dev->id, n); 3702 } 3703 3704 for (i = 0; i < n->max_queue_pairs; i++) { 3705 n->nic->ncs[i].do_not_pad = true; 3706 } 3707 3708 peer_test_vnet_hdr(n); 3709 if (peer_has_vnet_hdr(n)) { 3710 for (i = 0; i < n->max_queue_pairs; i++) { 3711 qemu_using_vnet_hdr(qemu_get_subqueue(n->nic, i)->peer, true); 3712 } 3713 n->host_hdr_len = sizeof(struct virtio_net_hdr); 3714 } else { 3715 n->host_hdr_len = 0; 3716 } 3717 3718 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a); 3719 3720 n->vqs[0].tx_waiting = 0; 3721 n->tx_burst = n->net_conf.txburst; 3722 virtio_net_set_mrg_rx_bufs(n, 0, 0, 0); 3723 n->promisc = 1; /* for compatibility */ 3724 3725 n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 3726 3727 n->vlans = g_malloc0(MAX_VLAN >> 3); 3728 3729 nc = qemu_get_queue(n->nic); 3730 nc->rxfilter_notify_enabled = 1; 3731 3732 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 3733 struct virtio_net_config netcfg = {}; 3734 memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN); 3735 vhost_net_set_config(get_vhost_net(nc->peer), 3736 (uint8_t *)&netcfg, 0, ETH_ALEN, VHOST_SET_CONFIG_TYPE_FRONTEND); 3737 } 3738 QTAILQ_INIT(&n->rsc_chains); 3739 n->qdev = dev; 3740 3741 net_rx_pkt_init(&n->rx_pkt); 3742 3743 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3744 virtio_net_load_ebpf(n); 3745 } 3746 } 3747 3748 static void virtio_net_device_unrealize(DeviceState *dev) 3749 { 3750 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3751 VirtIONet *n = VIRTIO_NET(dev); 3752 int i, max_queue_pairs; 3753 3754 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3755 virtio_net_unload_ebpf(n); 3756 } 3757 3758 /* This will stop vhost backend if appropriate. */ 3759 virtio_net_set_status(vdev, 0); 3760 3761 g_free(n->netclient_name); 3762 n->netclient_name = NULL; 3763 g_free(n->netclient_type); 3764 n->netclient_type = NULL; 3765 3766 g_free(n->mac_table.macs); 3767 g_free(n->vlans); 3768 3769 if (n->failover) { 3770 qobject_unref(n->primary_opts); 3771 device_listener_unregister(&n->primary_listener); 3772 remove_migration_state_change_notifier(&n->migration_state); 3773 } else { 3774 assert(n->primary_opts == NULL); 3775 } 3776 3777 max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 3778 for (i = 0; i < max_queue_pairs; i++) { 3779 virtio_net_del_queue(n, i); 3780 } 3781 /* delete also control vq */ 3782 virtio_del_queue(vdev, max_queue_pairs * 2); 3783 qemu_announce_timer_del(&n->announce_timer, false); 3784 g_free(n->vqs); 3785 qemu_del_nic(n->nic); 3786 virtio_net_rsc_cleanup(n); 3787 g_free(n->rss_data.indirections_table); 3788 net_rx_pkt_uninit(n->rx_pkt); 3789 virtio_cleanup(vdev); 3790 } 3791 3792 static void virtio_net_instance_init(Object *obj) 3793 { 3794 VirtIONet *n = VIRTIO_NET(obj); 3795 3796 /* 3797 * The default config_size is sizeof(struct virtio_net_config). 3798 * Can be overriden with virtio_net_set_config_size. 3799 */ 3800 n->config_size = sizeof(struct virtio_net_config); 3801 device_add_bootindex_property(obj, &n->nic_conf.bootindex, 3802 "bootindex", "/ethernet-phy@0", 3803 DEVICE(n)); 3804 3805 ebpf_rss_init(&n->ebpf_rss); 3806 } 3807 3808 static int virtio_net_pre_save(void *opaque) 3809 { 3810 VirtIONet *n = opaque; 3811 3812 /* At this point, backend must be stopped, otherwise 3813 * it might keep writing to memory. */ 3814 assert(!n->vhost_started); 3815 3816 return 0; 3817 } 3818 3819 static bool primary_unplug_pending(void *opaque) 3820 { 3821 DeviceState *dev = opaque; 3822 DeviceState *primary; 3823 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3824 VirtIONet *n = VIRTIO_NET(vdev); 3825 3826 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 3827 return false; 3828 } 3829 primary = failover_find_primary_device(n); 3830 return primary ? primary->pending_deleted_event : false; 3831 } 3832 3833 static bool dev_unplug_pending(void *opaque) 3834 { 3835 DeviceState *dev = opaque; 3836 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 3837 3838 return vdc->primary_unplug_pending(dev); 3839 } 3840 3841 static struct vhost_dev *virtio_net_get_vhost(VirtIODevice *vdev) 3842 { 3843 VirtIONet *n = VIRTIO_NET(vdev); 3844 NetClientState *nc = qemu_get_queue(n->nic); 3845 struct vhost_net *net = get_vhost_net(nc->peer); 3846 return &net->dev; 3847 } 3848 3849 static const VMStateDescription vmstate_virtio_net = { 3850 .name = "virtio-net", 3851 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3852 .version_id = VIRTIO_NET_VM_VERSION, 3853 .fields = (VMStateField[]) { 3854 VMSTATE_VIRTIO_DEVICE, 3855 VMSTATE_END_OF_LIST() 3856 }, 3857 .pre_save = virtio_net_pre_save, 3858 .dev_unplug_pending = dev_unplug_pending, 3859 }; 3860 3861 static Property virtio_net_properties[] = { 3862 DEFINE_PROP_BIT64("csum", VirtIONet, host_features, 3863 VIRTIO_NET_F_CSUM, true), 3864 DEFINE_PROP_BIT64("guest_csum", VirtIONet, host_features, 3865 VIRTIO_NET_F_GUEST_CSUM, true), 3866 DEFINE_PROP_BIT64("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true), 3867 DEFINE_PROP_BIT64("guest_tso4", VirtIONet, host_features, 3868 VIRTIO_NET_F_GUEST_TSO4, true), 3869 DEFINE_PROP_BIT64("guest_tso6", VirtIONet, host_features, 3870 VIRTIO_NET_F_GUEST_TSO6, true), 3871 DEFINE_PROP_BIT64("guest_ecn", VirtIONet, host_features, 3872 VIRTIO_NET_F_GUEST_ECN, true), 3873 DEFINE_PROP_BIT64("guest_ufo", VirtIONet, host_features, 3874 VIRTIO_NET_F_GUEST_UFO, true), 3875 DEFINE_PROP_BIT64("guest_announce", VirtIONet, host_features, 3876 VIRTIO_NET_F_GUEST_ANNOUNCE, true), 3877 DEFINE_PROP_BIT64("host_tso4", VirtIONet, host_features, 3878 VIRTIO_NET_F_HOST_TSO4, true), 3879 DEFINE_PROP_BIT64("host_tso6", VirtIONet, host_features, 3880 VIRTIO_NET_F_HOST_TSO6, true), 3881 DEFINE_PROP_BIT64("host_ecn", VirtIONet, host_features, 3882 VIRTIO_NET_F_HOST_ECN, true), 3883 DEFINE_PROP_BIT64("host_ufo", VirtIONet, host_features, 3884 VIRTIO_NET_F_HOST_UFO, true), 3885 DEFINE_PROP_BIT64("mrg_rxbuf", VirtIONet, host_features, 3886 VIRTIO_NET_F_MRG_RXBUF, true), 3887 DEFINE_PROP_BIT64("status", VirtIONet, host_features, 3888 VIRTIO_NET_F_STATUS, true), 3889 DEFINE_PROP_BIT64("ctrl_vq", VirtIONet, host_features, 3890 VIRTIO_NET_F_CTRL_VQ, true), 3891 DEFINE_PROP_BIT64("ctrl_rx", VirtIONet, host_features, 3892 VIRTIO_NET_F_CTRL_RX, true), 3893 DEFINE_PROP_BIT64("ctrl_vlan", VirtIONet, host_features, 3894 VIRTIO_NET_F_CTRL_VLAN, true), 3895 DEFINE_PROP_BIT64("ctrl_rx_extra", VirtIONet, host_features, 3896 VIRTIO_NET_F_CTRL_RX_EXTRA, true), 3897 DEFINE_PROP_BIT64("ctrl_mac_addr", VirtIONet, host_features, 3898 VIRTIO_NET_F_CTRL_MAC_ADDR, true), 3899 DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features, 3900 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true), 3901 DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false), 3902 DEFINE_PROP_BIT64("rss", VirtIONet, host_features, 3903 VIRTIO_NET_F_RSS, false), 3904 DEFINE_PROP_BIT64("hash", VirtIONet, host_features, 3905 VIRTIO_NET_F_HASH_REPORT, false), 3906 DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features, 3907 VIRTIO_NET_F_RSC_EXT, false), 3908 DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout, 3909 VIRTIO_NET_RSC_DEFAULT_INTERVAL), 3910 DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf), 3911 DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer, 3912 TX_TIMER_INTERVAL), 3913 DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST), 3914 DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx), 3915 DEFINE_PROP_UINT16("rx_queue_size", VirtIONet, net_conf.rx_queue_size, 3916 VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE), 3917 DEFINE_PROP_UINT16("tx_queue_size", VirtIONet, net_conf.tx_queue_size, 3918 VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE), 3919 DEFINE_PROP_UINT16("host_mtu", VirtIONet, net_conf.mtu, 0), 3920 DEFINE_PROP_BOOL("x-mtu-bypass-backend", VirtIONet, mtu_bypass_backend, 3921 true), 3922 DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), 3923 DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), 3924 DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), 3925 DEFINE_PROP_END_OF_LIST(), 3926 }; 3927 3928 static void virtio_net_class_init(ObjectClass *klass, void *data) 3929 { 3930 DeviceClass *dc = DEVICE_CLASS(klass); 3931 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 3932 3933 device_class_set_props(dc, virtio_net_properties); 3934 dc->vmsd = &vmstate_virtio_net; 3935 set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); 3936 vdc->realize = virtio_net_device_realize; 3937 vdc->unrealize = virtio_net_device_unrealize; 3938 vdc->get_config = virtio_net_get_config; 3939 vdc->set_config = virtio_net_set_config; 3940 vdc->get_features = virtio_net_get_features; 3941 vdc->set_features = virtio_net_set_features; 3942 vdc->bad_features = virtio_net_bad_features; 3943 vdc->reset = virtio_net_reset; 3944 vdc->queue_reset = virtio_net_queue_reset; 3945 vdc->queue_enable = virtio_net_queue_enable; 3946 vdc->set_status = virtio_net_set_status; 3947 vdc->guest_notifier_mask = virtio_net_guest_notifier_mask; 3948 vdc->guest_notifier_pending = virtio_net_guest_notifier_pending; 3949 vdc->legacy_features |= (0x1 << VIRTIO_NET_F_GSO); 3950 vdc->post_load = virtio_net_post_load_virtio; 3951 vdc->vmsd = &vmstate_virtio_net_device; 3952 vdc->primary_unplug_pending = primary_unplug_pending; 3953 vdc->get_vhost = virtio_net_get_vhost; 3954 vdc->toggle_device_iotlb = vhost_toggle_device_iotlb; 3955 } 3956 3957 static const TypeInfo virtio_net_info = { 3958 .name = TYPE_VIRTIO_NET, 3959 .parent = TYPE_VIRTIO_DEVICE, 3960 .instance_size = sizeof(VirtIONet), 3961 .instance_init = virtio_net_instance_init, 3962 .class_init = virtio_net_class_init, 3963 }; 3964 3965 static void virtio_register_types(void) 3966 { 3967 type_register_static(&virtio_net_info); 3968 } 3969 3970 type_init(virtio_register_types) 3971