1 /* 2 * Virtio Network Device 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/atomic.h" 16 #include "qemu/iov.h" 17 #include "qemu/log.h" 18 #include "qemu/main-loop.h" 19 #include "qemu/module.h" 20 #include "hw/virtio/virtio.h" 21 #include "net/net.h" 22 #include "net/checksum.h" 23 #include "net/tap.h" 24 #include "qemu/error-report.h" 25 #include "qemu/timer.h" 26 #include "qemu/option.h" 27 #include "qemu/option_int.h" 28 #include "qemu/config-file.h" 29 #include "qapi/qmp/qdict.h" 30 #include "hw/virtio/virtio-net.h" 31 #include "net/vhost_net.h" 32 #include "net/announce.h" 33 #include "hw/virtio/virtio-bus.h" 34 #include "qapi/error.h" 35 #include "qapi/qapi-events-net.h" 36 #include "hw/qdev-properties.h" 37 #include "qapi/qapi-types-migration.h" 38 #include "qapi/qapi-events-migration.h" 39 #include "hw/virtio/virtio-access.h" 40 #include "migration/misc.h" 41 #include "standard-headers/linux/ethtool.h" 42 #include "sysemu/sysemu.h" 43 #include "trace.h" 44 #include "monitor/qdev.h" 45 #include "hw/pci/pci_device.h" 46 #include "net_rx_pkt.h" 47 #include "hw/virtio/vhost.h" 48 #include "sysemu/qtest.h" 49 50 #define VIRTIO_NET_VM_VERSION 11 51 52 #define MAX_VLAN (1 << 12) /* Per 802.1Q definition */ 53 54 /* previously fixed value */ 55 #define VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 256 56 #define VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 256 57 58 /* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */ 59 #define VIRTIO_NET_RX_QUEUE_MIN_SIZE VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 60 #define VIRTIO_NET_TX_QUEUE_MIN_SIZE VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 61 62 #define VIRTIO_NET_IP4_ADDR_SIZE 8 /* ipv4 saddr + daddr */ 63 64 #define VIRTIO_NET_TCP_FLAG 0x3F 65 #define VIRTIO_NET_TCP_HDR_LENGTH 0xF000 66 67 /* IPv4 max payload, 16 bits in the header */ 68 #define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header)) 69 #define VIRTIO_NET_MAX_TCP_PAYLOAD 65535 70 71 /* header length value in ip header without option */ 72 #define VIRTIO_NET_IP4_HEADER_LENGTH 5 73 74 #define VIRTIO_NET_IP6_ADDR_SIZE 32 /* ipv6 saddr + daddr */ 75 #define VIRTIO_NET_MAX_IP6_PAYLOAD VIRTIO_NET_MAX_TCP_PAYLOAD 76 77 /* Purge coalesced packets timer interval, This value affects the performance 78 a lot, and should be tuned carefully, '300000'(300us) is the recommended 79 value to pass the WHQL test, '50000' can gain 2x netperf throughput with 80 tso/gso/gro 'off'. */ 81 #define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000 82 83 #define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 | \ 84 VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | \ 85 VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | \ 86 VIRTIO_NET_RSS_HASH_TYPE_IPv6 | \ 87 VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | \ 88 VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | \ 89 VIRTIO_NET_RSS_HASH_TYPE_IP_EX | \ 90 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | \ 91 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) 92 93 static const VirtIOFeature feature_sizes[] = { 94 {.flags = 1ULL << VIRTIO_NET_F_MAC, 95 .end = endof(struct virtio_net_config, mac)}, 96 {.flags = 1ULL << VIRTIO_NET_F_STATUS, 97 .end = endof(struct virtio_net_config, status)}, 98 {.flags = 1ULL << VIRTIO_NET_F_MQ, 99 .end = endof(struct virtio_net_config, max_virtqueue_pairs)}, 100 {.flags = 1ULL << VIRTIO_NET_F_MTU, 101 .end = endof(struct virtio_net_config, mtu)}, 102 {.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX, 103 .end = endof(struct virtio_net_config, duplex)}, 104 {.flags = (1ULL << VIRTIO_NET_F_RSS) | (1ULL << VIRTIO_NET_F_HASH_REPORT), 105 .end = endof(struct virtio_net_config, supported_hash_types)}, 106 {} 107 }; 108 109 static const VirtIOConfigSizeParams cfg_size_params = { 110 .min_size = endof(struct virtio_net_config, mac), 111 .max_size = sizeof(struct virtio_net_config), 112 .feature_sizes = feature_sizes 113 }; 114 115 static VirtIONetQueue *virtio_net_get_subqueue(NetClientState *nc) 116 { 117 VirtIONet *n = qemu_get_nic_opaque(nc); 118 119 return &n->vqs[nc->queue_index]; 120 } 121 122 static int vq2q(int queue_index) 123 { 124 return queue_index / 2; 125 } 126 127 static void flush_or_purge_queued_packets(NetClientState *nc) 128 { 129 if (!nc->peer) { 130 return; 131 } 132 133 qemu_flush_or_purge_queued_packets(nc->peer, true); 134 assert(!virtio_net_get_subqueue(nc)->async_tx.elem); 135 } 136 137 /* TODO 138 * - we could suppress RX interrupt if we were so inclined. 139 */ 140 141 static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config) 142 { 143 VirtIONet *n = VIRTIO_NET(vdev); 144 struct virtio_net_config netcfg; 145 NetClientState *nc = qemu_get_queue(n->nic); 146 static const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } }; 147 148 int ret = 0; 149 memset(&netcfg, 0 , sizeof(struct virtio_net_config)); 150 virtio_stw_p(vdev, &netcfg.status, n->status); 151 virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs); 152 virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu); 153 memcpy(netcfg.mac, n->mac, ETH_ALEN); 154 virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed); 155 netcfg.duplex = n->net_conf.duplex; 156 netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE; 157 virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length, 158 virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ? 159 VIRTIO_NET_RSS_MAX_TABLE_LEN : 1); 160 virtio_stl_p(vdev, &netcfg.supported_hash_types, 161 VIRTIO_NET_RSS_SUPPORTED_HASHES); 162 memcpy(config, &netcfg, n->config_size); 163 164 /* 165 * Is this VDPA? No peer means not VDPA: there's no way to 166 * disconnect/reconnect a VDPA peer. 167 */ 168 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 169 ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg, 170 n->config_size); 171 if (ret == -1) { 172 return; 173 } 174 175 /* 176 * Some NIC/kernel combinations present 0 as the mac address. As that 177 * is not a legal address, try to proceed with the address from the 178 * QEMU command line in the hope that the address has been configured 179 * correctly elsewhere - just not reported by the device. 180 */ 181 if (memcmp(&netcfg.mac, &zero, sizeof(zero)) == 0) { 182 info_report("Zero hardware mac address detected. Ignoring."); 183 memcpy(netcfg.mac, n->mac, ETH_ALEN); 184 } 185 186 netcfg.status |= virtio_tswap16(vdev, 187 n->status & VIRTIO_NET_S_ANNOUNCE); 188 memcpy(config, &netcfg, n->config_size); 189 } 190 } 191 192 static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config) 193 { 194 VirtIONet *n = VIRTIO_NET(vdev); 195 struct virtio_net_config netcfg = {}; 196 NetClientState *nc = qemu_get_queue(n->nic); 197 198 memcpy(&netcfg, config, n->config_size); 199 200 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) && 201 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) && 202 memcmp(netcfg.mac, n->mac, ETH_ALEN)) { 203 memcpy(n->mac, netcfg.mac, ETH_ALEN); 204 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 205 } 206 207 /* 208 * Is this VDPA? No peer means not VDPA: there's no way to 209 * disconnect/reconnect a VDPA peer. 210 */ 211 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 212 vhost_net_set_config(get_vhost_net(nc->peer), 213 (uint8_t *)&netcfg, 0, n->config_size, 214 VHOST_SET_CONFIG_TYPE_MASTER); 215 } 216 } 217 218 static bool virtio_net_started(VirtIONet *n, uint8_t status) 219 { 220 VirtIODevice *vdev = VIRTIO_DEVICE(n); 221 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 222 (n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running; 223 } 224 225 static void virtio_net_announce_notify(VirtIONet *net) 226 { 227 VirtIODevice *vdev = VIRTIO_DEVICE(net); 228 trace_virtio_net_announce_notify(); 229 230 net->status |= VIRTIO_NET_S_ANNOUNCE; 231 virtio_notify_config(vdev); 232 } 233 234 static void virtio_net_announce_timer(void *opaque) 235 { 236 VirtIONet *n = opaque; 237 trace_virtio_net_announce_timer(n->announce_timer.round); 238 239 n->announce_timer.round--; 240 virtio_net_announce_notify(n); 241 } 242 243 static void virtio_net_announce(NetClientState *nc) 244 { 245 VirtIONet *n = qemu_get_nic_opaque(nc); 246 VirtIODevice *vdev = VIRTIO_DEVICE(n); 247 248 /* 249 * Make sure the virtio migration announcement timer isn't running 250 * If it is, let it trigger announcement so that we do not cause 251 * confusion. 252 */ 253 if (n->announce_timer.round) { 254 return; 255 } 256 257 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 258 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 259 virtio_net_announce_notify(n); 260 } 261 } 262 263 static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) 264 { 265 VirtIODevice *vdev = VIRTIO_DEVICE(n); 266 NetClientState *nc = qemu_get_queue(n->nic); 267 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 268 int cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ? 269 n->max_ncs - n->max_queue_pairs : 0; 270 271 if (!get_vhost_net(nc->peer)) { 272 return; 273 } 274 275 if ((virtio_net_started(n, status) && !nc->peer->link_down) == 276 !!n->vhost_started) { 277 return; 278 } 279 if (!n->vhost_started) { 280 int r, i; 281 282 if (n->needs_vnet_hdr_swap) { 283 error_report("backend does not support %s vnet headers; " 284 "falling back on userspace virtio", 285 virtio_is_big_endian(vdev) ? "BE" : "LE"); 286 return; 287 } 288 289 /* Any packets outstanding? Purge them to avoid touching rings 290 * when vhost is running. 291 */ 292 for (i = 0; i < queue_pairs; i++) { 293 NetClientState *qnc = qemu_get_subqueue(n->nic, i); 294 295 /* Purge both directions: TX and RX. */ 296 qemu_net_queue_purge(qnc->peer->incoming_queue, qnc); 297 qemu_net_queue_purge(qnc->incoming_queue, qnc->peer); 298 } 299 300 if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) { 301 r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu); 302 if (r < 0) { 303 error_report("%uBytes MTU not supported by the backend", 304 n->net_conf.mtu); 305 306 return; 307 } 308 } 309 310 n->vhost_started = 1; 311 r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq); 312 if (r < 0) { 313 error_report("unable to start vhost net: %d: " 314 "falling back on userspace virtio", -r); 315 n->vhost_started = 0; 316 } 317 } else { 318 vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq); 319 n->vhost_started = 0; 320 } 321 } 322 323 static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev, 324 NetClientState *peer, 325 bool enable) 326 { 327 if (virtio_is_big_endian(vdev)) { 328 return qemu_set_vnet_be(peer, enable); 329 } else { 330 return qemu_set_vnet_le(peer, enable); 331 } 332 } 333 334 static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs, 335 int queue_pairs, bool enable) 336 { 337 int i; 338 339 for (i = 0; i < queue_pairs; i++) { 340 if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 && 341 enable) { 342 while (--i >= 0) { 343 virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false); 344 } 345 346 return true; 347 } 348 } 349 350 return false; 351 } 352 353 static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status) 354 { 355 VirtIODevice *vdev = VIRTIO_DEVICE(n); 356 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 357 358 if (virtio_net_started(n, status)) { 359 /* Before using the device, we tell the network backend about the 360 * endianness to use when parsing vnet headers. If the backend 361 * can't do it, we fallback onto fixing the headers in the core 362 * virtio-net code. 363 */ 364 n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs, 365 queue_pairs, true); 366 } else if (virtio_net_started(n, vdev->status)) { 367 /* After using the device, we need to reset the network backend to 368 * the default (guest native endianness), otherwise the guest may 369 * lose network connectivity if it is rebooted into a different 370 * endianness. 371 */ 372 virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false); 373 } 374 } 375 376 static void virtio_net_drop_tx_queue_data(VirtIODevice *vdev, VirtQueue *vq) 377 { 378 unsigned int dropped = virtqueue_drop_all(vq); 379 if (dropped) { 380 virtio_notify(vdev, vq); 381 } 382 } 383 384 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status) 385 { 386 VirtIONet *n = VIRTIO_NET(vdev); 387 VirtIONetQueue *q; 388 int i; 389 uint8_t queue_status; 390 391 virtio_net_vnet_endian_status(n, status); 392 virtio_net_vhost_status(n, status); 393 394 for (i = 0; i < n->max_queue_pairs; i++) { 395 NetClientState *ncs = qemu_get_subqueue(n->nic, i); 396 bool queue_started; 397 q = &n->vqs[i]; 398 399 if ((!n->multiqueue && i != 0) || i >= n->curr_queue_pairs) { 400 queue_status = 0; 401 } else { 402 queue_status = status; 403 } 404 queue_started = 405 virtio_net_started(n, queue_status) && !n->vhost_started; 406 407 if (queue_started) { 408 qemu_flush_queued_packets(ncs); 409 } 410 411 if (!q->tx_waiting) { 412 continue; 413 } 414 415 if (queue_started) { 416 if (q->tx_timer) { 417 timer_mod(q->tx_timer, 418 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 419 } else { 420 qemu_bh_schedule(q->tx_bh); 421 } 422 } else { 423 if (q->tx_timer) { 424 timer_del(q->tx_timer); 425 } else { 426 qemu_bh_cancel(q->tx_bh); 427 } 428 if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 && 429 (queue_status & VIRTIO_CONFIG_S_DRIVER_OK) && 430 vdev->vm_running) { 431 /* if tx is waiting we are likely have some packets in tx queue 432 * and disabled notification */ 433 q->tx_waiting = 0; 434 virtio_queue_set_notification(q->tx_vq, 1); 435 virtio_net_drop_tx_queue_data(vdev, q->tx_vq); 436 } 437 } 438 } 439 } 440 441 static void virtio_net_set_link_status(NetClientState *nc) 442 { 443 VirtIONet *n = qemu_get_nic_opaque(nc); 444 VirtIODevice *vdev = VIRTIO_DEVICE(n); 445 uint16_t old_status = n->status; 446 447 if (nc->link_down) 448 n->status &= ~VIRTIO_NET_S_LINK_UP; 449 else 450 n->status |= VIRTIO_NET_S_LINK_UP; 451 452 if (n->status != old_status) 453 virtio_notify_config(vdev); 454 455 virtio_net_set_status(vdev, vdev->status); 456 } 457 458 static void rxfilter_notify(NetClientState *nc) 459 { 460 VirtIONet *n = qemu_get_nic_opaque(nc); 461 462 if (nc->rxfilter_notify_enabled) { 463 char *path = object_get_canonical_path(OBJECT(n->qdev)); 464 qapi_event_send_nic_rx_filter_changed(n->netclient_name, path); 465 g_free(path); 466 467 /* disable event notification to avoid events flooding */ 468 nc->rxfilter_notify_enabled = 0; 469 } 470 } 471 472 static intList *get_vlan_table(VirtIONet *n) 473 { 474 intList *list; 475 int i, j; 476 477 list = NULL; 478 for (i = 0; i < MAX_VLAN >> 5; i++) { 479 for (j = 0; n->vlans[i] && j <= 0x1f; j++) { 480 if (n->vlans[i] & (1U << j)) { 481 QAPI_LIST_PREPEND(list, (i << 5) + j); 482 } 483 } 484 } 485 486 return list; 487 } 488 489 static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc) 490 { 491 VirtIONet *n = qemu_get_nic_opaque(nc); 492 VirtIODevice *vdev = VIRTIO_DEVICE(n); 493 RxFilterInfo *info; 494 strList *str_list; 495 int i; 496 497 info = g_malloc0(sizeof(*info)); 498 info->name = g_strdup(nc->name); 499 info->promiscuous = n->promisc; 500 501 if (n->nouni) { 502 info->unicast = RX_STATE_NONE; 503 } else if (n->alluni) { 504 info->unicast = RX_STATE_ALL; 505 } else { 506 info->unicast = RX_STATE_NORMAL; 507 } 508 509 if (n->nomulti) { 510 info->multicast = RX_STATE_NONE; 511 } else if (n->allmulti) { 512 info->multicast = RX_STATE_ALL; 513 } else { 514 info->multicast = RX_STATE_NORMAL; 515 } 516 517 info->broadcast_allowed = n->nobcast; 518 info->multicast_overflow = n->mac_table.multi_overflow; 519 info->unicast_overflow = n->mac_table.uni_overflow; 520 521 info->main_mac = qemu_mac_strdup_printf(n->mac); 522 523 str_list = NULL; 524 for (i = 0; i < n->mac_table.first_multi; i++) { 525 QAPI_LIST_PREPEND(str_list, 526 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 527 } 528 info->unicast_table = str_list; 529 530 str_list = NULL; 531 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 532 QAPI_LIST_PREPEND(str_list, 533 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 534 } 535 info->multicast_table = str_list; 536 info->vlan_table = get_vlan_table(n); 537 538 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) { 539 info->vlan = RX_STATE_ALL; 540 } else if (!info->vlan_table) { 541 info->vlan = RX_STATE_NONE; 542 } else { 543 info->vlan = RX_STATE_NORMAL; 544 } 545 546 /* enable event notification after query */ 547 nc->rxfilter_notify_enabled = 1; 548 549 return info; 550 } 551 552 static void virtio_net_queue_reset(VirtIODevice *vdev, uint32_t queue_index) 553 { 554 VirtIONet *n = VIRTIO_NET(vdev); 555 NetClientState *nc; 556 557 /* validate queue_index and skip for cvq */ 558 if (queue_index >= n->max_queue_pairs * 2) { 559 return; 560 } 561 562 nc = qemu_get_subqueue(n->nic, vq2q(queue_index)); 563 564 if (!nc->peer) { 565 return; 566 } 567 568 if (get_vhost_net(nc->peer) && 569 nc->peer->info->type == NET_CLIENT_DRIVER_TAP) { 570 vhost_net_virtqueue_reset(vdev, nc, queue_index); 571 } 572 573 flush_or_purge_queued_packets(nc); 574 } 575 576 static void virtio_net_queue_enable(VirtIODevice *vdev, uint32_t queue_index) 577 { 578 VirtIONet *n = VIRTIO_NET(vdev); 579 NetClientState *nc; 580 int r; 581 582 /* validate queue_index and skip for cvq */ 583 if (queue_index >= n->max_queue_pairs * 2) { 584 return; 585 } 586 587 nc = qemu_get_subqueue(n->nic, vq2q(queue_index)); 588 589 if (!nc->peer || !vdev->vhost_started) { 590 return; 591 } 592 593 if (get_vhost_net(nc->peer) && 594 nc->peer->info->type == NET_CLIENT_DRIVER_TAP) { 595 r = vhost_net_virtqueue_restart(vdev, nc, queue_index); 596 if (r < 0) { 597 error_report("unable to restart vhost net virtqueue: %d, " 598 "when resetting the queue", queue_index); 599 } 600 } 601 } 602 603 static void virtio_net_reset(VirtIODevice *vdev) 604 { 605 VirtIONet *n = VIRTIO_NET(vdev); 606 int i; 607 608 /* Reset back to compatibility mode */ 609 n->promisc = 1; 610 n->allmulti = 0; 611 n->alluni = 0; 612 n->nomulti = 0; 613 n->nouni = 0; 614 n->nobcast = 0; 615 /* multiqueue is disabled by default */ 616 n->curr_queue_pairs = 1; 617 timer_del(n->announce_timer.tm); 618 n->announce_timer.round = 0; 619 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 620 621 /* Flush any MAC and VLAN filter table state */ 622 n->mac_table.in_use = 0; 623 n->mac_table.first_multi = 0; 624 n->mac_table.multi_overflow = 0; 625 n->mac_table.uni_overflow = 0; 626 memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN); 627 memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac)); 628 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 629 memset(n->vlans, 0, MAX_VLAN >> 3); 630 631 /* Flush any async TX */ 632 for (i = 0; i < n->max_queue_pairs; i++) { 633 flush_or_purge_queued_packets(qemu_get_subqueue(n->nic, i)); 634 } 635 } 636 637 static void peer_test_vnet_hdr(VirtIONet *n) 638 { 639 NetClientState *nc = qemu_get_queue(n->nic); 640 if (!nc->peer) { 641 return; 642 } 643 644 n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer); 645 } 646 647 static int peer_has_vnet_hdr(VirtIONet *n) 648 { 649 return n->has_vnet_hdr; 650 } 651 652 static int peer_has_ufo(VirtIONet *n) 653 { 654 if (!peer_has_vnet_hdr(n)) 655 return 0; 656 657 n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer); 658 659 return n->has_ufo; 660 } 661 662 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, 663 int version_1, int hash_report) 664 { 665 int i; 666 NetClientState *nc; 667 668 n->mergeable_rx_bufs = mergeable_rx_bufs; 669 670 if (version_1) { 671 n->guest_hdr_len = hash_report ? 672 sizeof(struct virtio_net_hdr_v1_hash) : 673 sizeof(struct virtio_net_hdr_mrg_rxbuf); 674 n->rss_data.populate_hash = !!hash_report; 675 } else { 676 n->guest_hdr_len = n->mergeable_rx_bufs ? 677 sizeof(struct virtio_net_hdr_mrg_rxbuf) : 678 sizeof(struct virtio_net_hdr); 679 } 680 681 for (i = 0; i < n->max_queue_pairs; i++) { 682 nc = qemu_get_subqueue(n->nic, i); 683 684 if (peer_has_vnet_hdr(n) && 685 qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) { 686 qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len); 687 n->host_hdr_len = n->guest_hdr_len; 688 } 689 } 690 } 691 692 static int virtio_net_max_tx_queue_size(VirtIONet *n) 693 { 694 NetClientState *peer = n->nic_conf.peers.ncs[0]; 695 696 /* 697 * Backends other than vhost-user or vhost-vdpa don't support max queue 698 * size. 699 */ 700 if (!peer) { 701 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 702 } 703 704 switch(peer->info->type) { 705 case NET_CLIENT_DRIVER_VHOST_USER: 706 case NET_CLIENT_DRIVER_VHOST_VDPA: 707 return VIRTQUEUE_MAX_SIZE; 708 default: 709 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 710 }; 711 } 712 713 static int peer_attach(VirtIONet *n, int index) 714 { 715 NetClientState *nc = qemu_get_subqueue(n->nic, index); 716 717 if (!nc->peer) { 718 return 0; 719 } 720 721 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 722 vhost_set_vring_enable(nc->peer, 1); 723 } 724 725 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 726 return 0; 727 } 728 729 if (n->max_queue_pairs == 1) { 730 return 0; 731 } 732 733 return tap_enable(nc->peer); 734 } 735 736 static int peer_detach(VirtIONet *n, int index) 737 { 738 NetClientState *nc = qemu_get_subqueue(n->nic, index); 739 740 if (!nc->peer) { 741 return 0; 742 } 743 744 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 745 vhost_set_vring_enable(nc->peer, 0); 746 } 747 748 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 749 return 0; 750 } 751 752 return tap_disable(nc->peer); 753 } 754 755 static void virtio_net_set_queue_pairs(VirtIONet *n) 756 { 757 int i; 758 int r; 759 760 if (n->nic->peer_deleted) { 761 return; 762 } 763 764 for (i = 0; i < n->max_queue_pairs; i++) { 765 if (i < n->curr_queue_pairs) { 766 r = peer_attach(n, i); 767 assert(!r); 768 } else { 769 r = peer_detach(n, i); 770 assert(!r); 771 } 772 } 773 } 774 775 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue); 776 777 static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, 778 Error **errp) 779 { 780 VirtIONet *n = VIRTIO_NET(vdev); 781 NetClientState *nc = qemu_get_queue(n->nic); 782 783 /* Firstly sync all virtio-net possible supported features */ 784 features |= n->host_features; 785 786 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 787 788 if (!peer_has_vnet_hdr(n)) { 789 virtio_clear_feature(&features, VIRTIO_NET_F_CSUM); 790 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO4); 791 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO6); 792 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_ECN); 793 794 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_CSUM); 795 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4); 796 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6); 797 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN); 798 799 virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); 800 } 801 802 if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) { 803 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_UFO); 804 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO); 805 } 806 807 if (!get_vhost_net(nc->peer)) { 808 virtio_add_feature(&features, VIRTIO_F_RING_RESET); 809 return features; 810 } 811 812 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 813 virtio_clear_feature(&features, VIRTIO_NET_F_RSS); 814 } 815 features = vhost_net_get_features(get_vhost_net(nc->peer), features); 816 vdev->backend_features = features; 817 818 if (n->mtu_bypass_backend && 819 (n->host_features & 1ULL << VIRTIO_NET_F_MTU)) { 820 features |= (1ULL << VIRTIO_NET_F_MTU); 821 } 822 823 /* 824 * Since GUEST_ANNOUNCE is emulated the feature bit could be set without 825 * enabled. This happens in the vDPA case. 826 * 827 * Make sure the feature set is not incoherent, as the driver could refuse 828 * to start. 829 * 830 * TODO: QEMU is able to emulate a CVQ just for guest_announce purposes, 831 * helping guest to notify the new location with vDPA devices that does not 832 * support it. 833 */ 834 if (!virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_CTRL_VQ)) { 835 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ANNOUNCE); 836 } 837 838 return features; 839 } 840 841 static uint64_t virtio_net_bad_features(VirtIODevice *vdev) 842 { 843 uint64_t features = 0; 844 845 /* Linux kernel 2.6.25. It understood MAC (as everyone must), 846 * but also these: */ 847 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 848 virtio_add_feature(&features, VIRTIO_NET_F_CSUM); 849 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4); 850 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6); 851 virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN); 852 853 return features; 854 } 855 856 static void virtio_net_apply_guest_offloads(VirtIONet *n) 857 { 858 qemu_set_offload(qemu_get_queue(n->nic)->peer, 859 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)), 860 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)), 861 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)), 862 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)), 863 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO))); 864 } 865 866 static uint64_t virtio_net_guest_offloads_by_features(uint32_t features) 867 { 868 static const uint64_t guest_offloads_mask = 869 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | 870 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 871 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | 872 (1ULL << VIRTIO_NET_F_GUEST_ECN) | 873 (1ULL << VIRTIO_NET_F_GUEST_UFO); 874 875 return guest_offloads_mask & features; 876 } 877 878 static inline uint64_t virtio_net_supported_guest_offloads(VirtIONet *n) 879 { 880 VirtIODevice *vdev = VIRTIO_DEVICE(n); 881 return virtio_net_guest_offloads_by_features(vdev->guest_features); 882 } 883 884 typedef struct { 885 VirtIONet *n; 886 DeviceState *dev; 887 } FailoverDevice; 888 889 /** 890 * Set the failover primary device 891 * 892 * @opaque: FailoverId to setup 893 * @opts: opts for device we are handling 894 * @errp: returns an error if this function fails 895 */ 896 static int failover_set_primary(DeviceState *dev, void *opaque) 897 { 898 FailoverDevice *fdev = opaque; 899 PCIDevice *pci_dev = (PCIDevice *) 900 object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE); 901 902 if (!pci_dev) { 903 return 0; 904 } 905 906 if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) { 907 fdev->dev = dev; 908 return 1; 909 } 910 911 return 0; 912 } 913 914 /** 915 * Find the primary device for this failover virtio-net 916 * 917 * @n: VirtIONet device 918 * @errp: returns an error if this function fails 919 */ 920 static DeviceState *failover_find_primary_device(VirtIONet *n) 921 { 922 FailoverDevice fdev = { 923 .n = n, 924 }; 925 926 qbus_walk_children(sysbus_get_default(), failover_set_primary, NULL, 927 NULL, NULL, &fdev); 928 return fdev.dev; 929 } 930 931 static void failover_add_primary(VirtIONet *n, Error **errp) 932 { 933 Error *err = NULL; 934 DeviceState *dev = failover_find_primary_device(n); 935 936 if (dev) { 937 return; 938 } 939 940 if (!n->primary_opts) { 941 error_setg(errp, "Primary device not found"); 942 error_append_hint(errp, "Virtio-net failover will not work. Make " 943 "sure primary device has parameter" 944 " failover_pair_id=%s\n", n->netclient_name); 945 return; 946 } 947 948 dev = qdev_device_add_from_qdict(n->primary_opts, 949 n->primary_opts_from_json, 950 &err); 951 if (err) { 952 qobject_unref(n->primary_opts); 953 n->primary_opts = NULL; 954 } else { 955 object_unref(OBJECT(dev)); 956 } 957 error_propagate(errp, err); 958 } 959 960 static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) 961 { 962 VirtIONet *n = VIRTIO_NET(vdev); 963 Error *err = NULL; 964 int i; 965 966 if (n->mtu_bypass_backend && 967 !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { 968 features &= ~(1ULL << VIRTIO_NET_F_MTU); 969 } 970 971 virtio_net_set_multiqueue(n, 972 virtio_has_feature(features, VIRTIO_NET_F_RSS) || 973 virtio_has_feature(features, VIRTIO_NET_F_MQ)); 974 975 virtio_net_set_mrg_rx_bufs(n, 976 virtio_has_feature(features, 977 VIRTIO_NET_F_MRG_RXBUF), 978 virtio_has_feature(features, 979 VIRTIO_F_VERSION_1), 980 virtio_has_feature(features, 981 VIRTIO_NET_F_HASH_REPORT)); 982 983 n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 984 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO4); 985 n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 986 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); 987 n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); 988 989 if (n->has_vnet_hdr) { 990 n->curr_guest_offloads = 991 virtio_net_guest_offloads_by_features(features); 992 virtio_net_apply_guest_offloads(n); 993 } 994 995 for (i = 0; i < n->max_queue_pairs; i++) { 996 NetClientState *nc = qemu_get_subqueue(n->nic, i); 997 998 if (!get_vhost_net(nc->peer)) { 999 continue; 1000 } 1001 vhost_net_ack_features(get_vhost_net(nc->peer), features); 1002 1003 /* 1004 * keep acked_features in NetVhostUserState up-to-date so it 1005 * can't miss any features configured by guest virtio driver. 1006 */ 1007 vhost_net_save_acked_features(nc->peer); 1008 } 1009 1010 if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) { 1011 memset(n->vlans, 0, MAX_VLAN >> 3); 1012 } else { 1013 memset(n->vlans, 0xff, MAX_VLAN >> 3); 1014 } 1015 1016 if (virtio_has_feature(features, VIRTIO_NET_F_STANDBY)) { 1017 qapi_event_send_failover_negotiated(n->netclient_name); 1018 qatomic_set(&n->failover_primary_hidden, false); 1019 failover_add_primary(n, &err); 1020 if (err) { 1021 if (!qtest_enabled()) { 1022 warn_report_err(err); 1023 } else { 1024 error_free(err); 1025 } 1026 } 1027 } 1028 } 1029 1030 static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd, 1031 struct iovec *iov, unsigned int iov_cnt) 1032 { 1033 uint8_t on; 1034 size_t s; 1035 NetClientState *nc = qemu_get_queue(n->nic); 1036 1037 s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on)); 1038 if (s != sizeof(on)) { 1039 return VIRTIO_NET_ERR; 1040 } 1041 1042 if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) { 1043 n->promisc = on; 1044 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) { 1045 n->allmulti = on; 1046 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) { 1047 n->alluni = on; 1048 } else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) { 1049 n->nomulti = on; 1050 } else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) { 1051 n->nouni = on; 1052 } else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) { 1053 n->nobcast = on; 1054 } else { 1055 return VIRTIO_NET_ERR; 1056 } 1057 1058 rxfilter_notify(nc); 1059 1060 return VIRTIO_NET_OK; 1061 } 1062 1063 static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd, 1064 struct iovec *iov, unsigned int iov_cnt) 1065 { 1066 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1067 uint64_t offloads; 1068 size_t s; 1069 1070 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 1071 return VIRTIO_NET_ERR; 1072 } 1073 1074 s = iov_to_buf(iov, iov_cnt, 0, &offloads, sizeof(offloads)); 1075 if (s != sizeof(offloads)) { 1076 return VIRTIO_NET_ERR; 1077 } 1078 1079 if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) { 1080 uint64_t supported_offloads; 1081 1082 offloads = virtio_ldq_p(vdev, &offloads); 1083 1084 if (!n->has_vnet_hdr) { 1085 return VIRTIO_NET_ERR; 1086 } 1087 1088 n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1089 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO4); 1090 n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1091 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO6); 1092 virtio_clear_feature(&offloads, VIRTIO_NET_F_RSC_EXT); 1093 1094 supported_offloads = virtio_net_supported_guest_offloads(n); 1095 if (offloads & ~supported_offloads) { 1096 return VIRTIO_NET_ERR; 1097 } 1098 1099 n->curr_guest_offloads = offloads; 1100 virtio_net_apply_guest_offloads(n); 1101 1102 return VIRTIO_NET_OK; 1103 } else { 1104 return VIRTIO_NET_ERR; 1105 } 1106 } 1107 1108 static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd, 1109 struct iovec *iov, unsigned int iov_cnt) 1110 { 1111 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1112 struct virtio_net_ctrl_mac mac_data; 1113 size_t s; 1114 NetClientState *nc = qemu_get_queue(n->nic); 1115 1116 if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) { 1117 if (iov_size(iov, iov_cnt) != sizeof(n->mac)) { 1118 return VIRTIO_NET_ERR; 1119 } 1120 s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac)); 1121 assert(s == sizeof(n->mac)); 1122 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 1123 rxfilter_notify(nc); 1124 1125 return VIRTIO_NET_OK; 1126 } 1127 1128 if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) { 1129 return VIRTIO_NET_ERR; 1130 } 1131 1132 int in_use = 0; 1133 int first_multi = 0; 1134 uint8_t uni_overflow = 0; 1135 uint8_t multi_overflow = 0; 1136 uint8_t *macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 1137 1138 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1139 sizeof(mac_data.entries)); 1140 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1141 if (s != sizeof(mac_data.entries)) { 1142 goto error; 1143 } 1144 iov_discard_front(&iov, &iov_cnt, s); 1145 1146 if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) { 1147 goto error; 1148 } 1149 1150 if (mac_data.entries <= MAC_TABLE_ENTRIES) { 1151 s = iov_to_buf(iov, iov_cnt, 0, macs, 1152 mac_data.entries * ETH_ALEN); 1153 if (s != mac_data.entries * ETH_ALEN) { 1154 goto error; 1155 } 1156 in_use += mac_data.entries; 1157 } else { 1158 uni_overflow = 1; 1159 } 1160 1161 iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN); 1162 1163 first_multi = in_use; 1164 1165 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1166 sizeof(mac_data.entries)); 1167 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1168 if (s != sizeof(mac_data.entries)) { 1169 goto error; 1170 } 1171 1172 iov_discard_front(&iov, &iov_cnt, s); 1173 1174 if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) { 1175 goto error; 1176 } 1177 1178 if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) { 1179 s = iov_to_buf(iov, iov_cnt, 0, &macs[in_use * ETH_ALEN], 1180 mac_data.entries * ETH_ALEN); 1181 if (s != mac_data.entries * ETH_ALEN) { 1182 goto error; 1183 } 1184 in_use += mac_data.entries; 1185 } else { 1186 multi_overflow = 1; 1187 } 1188 1189 n->mac_table.in_use = in_use; 1190 n->mac_table.first_multi = first_multi; 1191 n->mac_table.uni_overflow = uni_overflow; 1192 n->mac_table.multi_overflow = multi_overflow; 1193 memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN); 1194 g_free(macs); 1195 rxfilter_notify(nc); 1196 1197 return VIRTIO_NET_OK; 1198 1199 error: 1200 g_free(macs); 1201 return VIRTIO_NET_ERR; 1202 } 1203 1204 static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd, 1205 struct iovec *iov, unsigned int iov_cnt) 1206 { 1207 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1208 uint16_t vid; 1209 size_t s; 1210 NetClientState *nc = qemu_get_queue(n->nic); 1211 1212 s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid)); 1213 vid = virtio_lduw_p(vdev, &vid); 1214 if (s != sizeof(vid)) { 1215 return VIRTIO_NET_ERR; 1216 } 1217 1218 if (vid >= MAX_VLAN) 1219 return VIRTIO_NET_ERR; 1220 1221 if (cmd == VIRTIO_NET_CTRL_VLAN_ADD) 1222 n->vlans[vid >> 5] |= (1U << (vid & 0x1f)); 1223 else if (cmd == VIRTIO_NET_CTRL_VLAN_DEL) 1224 n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f)); 1225 else 1226 return VIRTIO_NET_ERR; 1227 1228 rxfilter_notify(nc); 1229 1230 return VIRTIO_NET_OK; 1231 } 1232 1233 static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, 1234 struct iovec *iov, unsigned int iov_cnt) 1235 { 1236 trace_virtio_net_handle_announce(n->announce_timer.round); 1237 if (cmd == VIRTIO_NET_CTRL_ANNOUNCE_ACK && 1238 n->status & VIRTIO_NET_S_ANNOUNCE) { 1239 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 1240 if (n->announce_timer.round) { 1241 qemu_announce_timer_step(&n->announce_timer); 1242 } 1243 return VIRTIO_NET_OK; 1244 } else { 1245 return VIRTIO_NET_ERR; 1246 } 1247 } 1248 1249 static void virtio_net_detach_epbf_rss(VirtIONet *n); 1250 1251 static void virtio_net_disable_rss(VirtIONet *n) 1252 { 1253 if (n->rss_data.enabled) { 1254 trace_virtio_net_rss_disable(); 1255 } 1256 n->rss_data.enabled = false; 1257 1258 virtio_net_detach_epbf_rss(n); 1259 } 1260 1261 static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd) 1262 { 1263 NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0); 1264 if (nc == NULL || nc->info->set_steering_ebpf == NULL) { 1265 return false; 1266 } 1267 1268 return nc->info->set_steering_ebpf(nc, prog_fd); 1269 } 1270 1271 static void rss_data_to_rss_config(struct VirtioNetRssData *data, 1272 struct EBPFRSSConfig *config) 1273 { 1274 config->redirect = data->redirect; 1275 config->populate_hash = data->populate_hash; 1276 config->hash_types = data->hash_types; 1277 config->indirections_len = data->indirections_len; 1278 config->default_queue = data->default_queue; 1279 } 1280 1281 static bool virtio_net_attach_epbf_rss(VirtIONet *n) 1282 { 1283 struct EBPFRSSConfig config = {}; 1284 1285 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 1286 return false; 1287 } 1288 1289 rss_data_to_rss_config(&n->rss_data, &config); 1290 1291 if (!ebpf_rss_set_all(&n->ebpf_rss, &config, 1292 n->rss_data.indirections_table, n->rss_data.key)) { 1293 return false; 1294 } 1295 1296 if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { 1297 return false; 1298 } 1299 1300 return true; 1301 } 1302 1303 static void virtio_net_detach_epbf_rss(VirtIONet *n) 1304 { 1305 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1306 } 1307 1308 static bool virtio_net_load_ebpf(VirtIONet *n) 1309 { 1310 if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { 1311 /* backend does't support steering ebpf */ 1312 return false; 1313 } 1314 1315 return ebpf_rss_load(&n->ebpf_rss); 1316 } 1317 1318 static void virtio_net_unload_ebpf(VirtIONet *n) 1319 { 1320 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1321 ebpf_rss_unload(&n->ebpf_rss); 1322 } 1323 1324 static uint16_t virtio_net_handle_rss(VirtIONet *n, 1325 struct iovec *iov, 1326 unsigned int iov_cnt, 1327 bool do_rss) 1328 { 1329 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1330 struct virtio_net_rss_config cfg; 1331 size_t s, offset = 0, size_get; 1332 uint16_t queue_pairs, i; 1333 struct { 1334 uint16_t us; 1335 uint8_t b; 1336 } QEMU_PACKED temp; 1337 const char *err_msg = ""; 1338 uint32_t err_value = 0; 1339 1340 if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) { 1341 err_msg = "RSS is not negotiated"; 1342 goto error; 1343 } 1344 if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) { 1345 err_msg = "Hash report is not negotiated"; 1346 goto error; 1347 } 1348 size_get = offsetof(struct virtio_net_rss_config, indirection_table); 1349 s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get); 1350 if (s != size_get) { 1351 err_msg = "Short command buffer"; 1352 err_value = (uint32_t)s; 1353 goto error; 1354 } 1355 n->rss_data.hash_types = virtio_ldl_p(vdev, &cfg.hash_types); 1356 n->rss_data.indirections_len = 1357 virtio_lduw_p(vdev, &cfg.indirection_table_mask); 1358 n->rss_data.indirections_len++; 1359 if (!do_rss) { 1360 n->rss_data.indirections_len = 1; 1361 } 1362 if (!is_power_of_2(n->rss_data.indirections_len)) { 1363 err_msg = "Invalid size of indirection table"; 1364 err_value = n->rss_data.indirections_len; 1365 goto error; 1366 } 1367 if (n->rss_data.indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 1368 err_msg = "Too large indirection table"; 1369 err_value = n->rss_data.indirections_len; 1370 goto error; 1371 } 1372 n->rss_data.default_queue = do_rss ? 1373 virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0; 1374 if (n->rss_data.default_queue >= n->max_queue_pairs) { 1375 err_msg = "Invalid default queue"; 1376 err_value = n->rss_data.default_queue; 1377 goto error; 1378 } 1379 offset += size_get; 1380 size_get = sizeof(uint16_t) * n->rss_data.indirections_len; 1381 g_free(n->rss_data.indirections_table); 1382 n->rss_data.indirections_table = g_malloc(size_get); 1383 if (!n->rss_data.indirections_table) { 1384 err_msg = "Can't allocate indirections table"; 1385 err_value = n->rss_data.indirections_len; 1386 goto error; 1387 } 1388 s = iov_to_buf(iov, iov_cnt, offset, 1389 n->rss_data.indirections_table, size_get); 1390 if (s != size_get) { 1391 err_msg = "Short indirection table buffer"; 1392 err_value = (uint32_t)s; 1393 goto error; 1394 } 1395 for (i = 0; i < n->rss_data.indirections_len; ++i) { 1396 uint16_t val = n->rss_data.indirections_table[i]; 1397 n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val); 1398 } 1399 offset += size_get; 1400 size_get = sizeof(temp); 1401 s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get); 1402 if (s != size_get) { 1403 err_msg = "Can't get queue_pairs"; 1404 err_value = (uint32_t)s; 1405 goto error; 1406 } 1407 queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs; 1408 if (queue_pairs == 0 || queue_pairs > n->max_queue_pairs) { 1409 err_msg = "Invalid number of queue_pairs"; 1410 err_value = queue_pairs; 1411 goto error; 1412 } 1413 if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) { 1414 err_msg = "Invalid key size"; 1415 err_value = temp.b; 1416 goto error; 1417 } 1418 if (!temp.b && n->rss_data.hash_types) { 1419 err_msg = "No key provided"; 1420 err_value = 0; 1421 goto error; 1422 } 1423 if (!temp.b && !n->rss_data.hash_types) { 1424 virtio_net_disable_rss(n); 1425 return queue_pairs; 1426 } 1427 offset += size_get; 1428 size_get = temp.b; 1429 s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get); 1430 if (s != size_get) { 1431 err_msg = "Can get key buffer"; 1432 err_value = (uint32_t)s; 1433 goto error; 1434 } 1435 n->rss_data.enabled = true; 1436 1437 if (!n->rss_data.populate_hash) { 1438 if (!virtio_net_attach_epbf_rss(n)) { 1439 /* EBPF must be loaded for vhost */ 1440 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 1441 warn_report("Can't load eBPF RSS for vhost"); 1442 goto error; 1443 } 1444 /* fallback to software RSS */ 1445 warn_report("Can't load eBPF RSS - fallback to software RSS"); 1446 n->rss_data.enabled_software_rss = true; 1447 } 1448 } else { 1449 /* use software RSS for hash populating */ 1450 /* and detach eBPF if was loaded before */ 1451 virtio_net_detach_epbf_rss(n); 1452 n->rss_data.enabled_software_rss = true; 1453 } 1454 1455 trace_virtio_net_rss_enable(n->rss_data.hash_types, 1456 n->rss_data.indirections_len, 1457 temp.b); 1458 return queue_pairs; 1459 error: 1460 trace_virtio_net_rss_error(err_msg, err_value); 1461 virtio_net_disable_rss(n); 1462 return 0; 1463 } 1464 1465 static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd, 1466 struct iovec *iov, unsigned int iov_cnt) 1467 { 1468 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1469 uint16_t queue_pairs; 1470 NetClientState *nc = qemu_get_queue(n->nic); 1471 1472 virtio_net_disable_rss(n); 1473 if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) { 1474 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, false); 1475 return queue_pairs ? VIRTIO_NET_OK : VIRTIO_NET_ERR; 1476 } 1477 if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 1478 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, true); 1479 } else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 1480 struct virtio_net_ctrl_mq mq; 1481 size_t s; 1482 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) { 1483 return VIRTIO_NET_ERR; 1484 } 1485 s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq)); 1486 if (s != sizeof(mq)) { 1487 return VIRTIO_NET_ERR; 1488 } 1489 queue_pairs = virtio_lduw_p(vdev, &mq.virtqueue_pairs); 1490 1491 } else { 1492 return VIRTIO_NET_ERR; 1493 } 1494 1495 if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1496 queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 1497 queue_pairs > n->max_queue_pairs || 1498 !n->multiqueue) { 1499 return VIRTIO_NET_ERR; 1500 } 1501 1502 n->curr_queue_pairs = queue_pairs; 1503 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 1504 /* 1505 * Avoid updating the backend for a vdpa device: We're only interested 1506 * in updating the device model queues. 1507 */ 1508 return VIRTIO_NET_OK; 1509 } 1510 /* stop the backend before changing the number of queue_pairs to avoid handling a 1511 * disabled queue */ 1512 virtio_net_set_status(vdev, vdev->status); 1513 virtio_net_set_queue_pairs(n); 1514 1515 return VIRTIO_NET_OK; 1516 } 1517 1518 size_t virtio_net_handle_ctrl_iov(VirtIODevice *vdev, 1519 const struct iovec *in_sg, unsigned in_num, 1520 const struct iovec *out_sg, 1521 unsigned out_num) 1522 { 1523 VirtIONet *n = VIRTIO_NET(vdev); 1524 struct virtio_net_ctrl_hdr ctrl; 1525 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1526 size_t s; 1527 struct iovec *iov, *iov2; 1528 1529 if (iov_size(in_sg, in_num) < sizeof(status) || 1530 iov_size(out_sg, out_num) < sizeof(ctrl)) { 1531 virtio_error(vdev, "virtio-net ctrl missing headers"); 1532 return 0; 1533 } 1534 1535 iov2 = iov = g_memdup2(out_sg, sizeof(struct iovec) * out_num); 1536 s = iov_to_buf(iov, out_num, 0, &ctrl, sizeof(ctrl)); 1537 iov_discard_front(&iov, &out_num, sizeof(ctrl)); 1538 if (s != sizeof(ctrl)) { 1539 status = VIRTIO_NET_ERR; 1540 } else if (ctrl.class == VIRTIO_NET_CTRL_RX) { 1541 status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, out_num); 1542 } else if (ctrl.class == VIRTIO_NET_CTRL_MAC) { 1543 status = virtio_net_handle_mac(n, ctrl.cmd, iov, out_num); 1544 } else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) { 1545 status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, out_num); 1546 } else if (ctrl.class == VIRTIO_NET_CTRL_ANNOUNCE) { 1547 status = virtio_net_handle_announce(n, ctrl.cmd, iov, out_num); 1548 } else if (ctrl.class == VIRTIO_NET_CTRL_MQ) { 1549 status = virtio_net_handle_mq(n, ctrl.cmd, iov, out_num); 1550 } else if (ctrl.class == VIRTIO_NET_CTRL_GUEST_OFFLOADS) { 1551 status = virtio_net_handle_offloads(n, ctrl.cmd, iov, out_num); 1552 } 1553 1554 s = iov_from_buf(in_sg, in_num, 0, &status, sizeof(status)); 1555 assert(s == sizeof(status)); 1556 1557 g_free(iov2); 1558 return sizeof(status); 1559 } 1560 1561 static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 1562 { 1563 VirtQueueElement *elem; 1564 1565 for (;;) { 1566 size_t written; 1567 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 1568 if (!elem) { 1569 break; 1570 } 1571 1572 written = virtio_net_handle_ctrl_iov(vdev, elem->in_sg, elem->in_num, 1573 elem->out_sg, elem->out_num); 1574 if (written > 0) { 1575 virtqueue_push(vq, elem, written); 1576 virtio_notify(vdev, vq); 1577 g_free(elem); 1578 } else { 1579 virtqueue_detach_element(vq, elem, 0); 1580 g_free(elem); 1581 break; 1582 } 1583 } 1584 } 1585 1586 /* RX */ 1587 1588 static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq) 1589 { 1590 VirtIONet *n = VIRTIO_NET(vdev); 1591 int queue_index = vq2q(virtio_get_queue_index(vq)); 1592 1593 qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index)); 1594 } 1595 1596 static bool virtio_net_can_receive(NetClientState *nc) 1597 { 1598 VirtIONet *n = qemu_get_nic_opaque(nc); 1599 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1600 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1601 1602 if (!vdev->vm_running) { 1603 return false; 1604 } 1605 1606 if (nc->queue_index >= n->curr_queue_pairs) { 1607 return false; 1608 } 1609 1610 if (!virtio_queue_ready(q->rx_vq) || 1611 !(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 1612 return false; 1613 } 1614 1615 return true; 1616 } 1617 1618 static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize) 1619 { 1620 VirtIONet *n = q->n; 1621 if (virtio_queue_empty(q->rx_vq) || 1622 (n->mergeable_rx_bufs && 1623 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1624 virtio_queue_set_notification(q->rx_vq, 1); 1625 1626 /* To avoid a race condition where the guest has made some buffers 1627 * available after the above check but before notification was 1628 * enabled, check for available buffers again. 1629 */ 1630 if (virtio_queue_empty(q->rx_vq) || 1631 (n->mergeable_rx_bufs && 1632 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1633 return 0; 1634 } 1635 } 1636 1637 virtio_queue_set_notification(q->rx_vq, 0); 1638 return 1; 1639 } 1640 1641 static void virtio_net_hdr_swap(VirtIODevice *vdev, struct virtio_net_hdr *hdr) 1642 { 1643 virtio_tswap16s(vdev, &hdr->hdr_len); 1644 virtio_tswap16s(vdev, &hdr->gso_size); 1645 virtio_tswap16s(vdev, &hdr->csum_start); 1646 virtio_tswap16s(vdev, &hdr->csum_offset); 1647 } 1648 1649 /* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so 1650 * it never finds out that the packets don't have valid checksums. This 1651 * causes dhclient to get upset. Fedora's carried a patch for ages to 1652 * fix this with Xen but it hasn't appeared in an upstream release of 1653 * dhclient yet. 1654 * 1655 * To avoid breaking existing guests, we catch udp packets and add 1656 * checksums. This is terrible but it's better than hacking the guest 1657 * kernels. 1658 * 1659 * N.B. if we introduce a zero-copy API, this operation is no longer free so 1660 * we should provide a mechanism to disable it to avoid polluting the host 1661 * cache. 1662 */ 1663 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr, 1664 uint8_t *buf, size_t size) 1665 { 1666 if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */ 1667 (size > 27 && size < 1500) && /* normal sized MTU */ 1668 (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */ 1669 (buf[23] == 17) && /* ip.protocol == UDP */ 1670 (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */ 1671 net_checksum_calculate(buf, size, CSUM_UDP); 1672 hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM; 1673 } 1674 } 1675 1676 static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt, 1677 const void *buf, size_t size) 1678 { 1679 if (n->has_vnet_hdr) { 1680 /* FIXME this cast is evil */ 1681 void *wbuf = (void *)buf; 1682 work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len, 1683 size - n->host_hdr_len); 1684 1685 if (n->needs_vnet_hdr_swap) { 1686 virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf); 1687 } 1688 iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr)); 1689 } else { 1690 struct virtio_net_hdr hdr = { 1691 .flags = 0, 1692 .gso_type = VIRTIO_NET_HDR_GSO_NONE 1693 }; 1694 iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr); 1695 } 1696 } 1697 1698 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size) 1699 { 1700 static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 1701 static const uint8_t vlan[] = {0x81, 0x00}; 1702 uint8_t *ptr = (uint8_t *)buf; 1703 int i; 1704 1705 if (n->promisc) 1706 return 1; 1707 1708 ptr += n->host_hdr_len; 1709 1710 if (!memcmp(&ptr[12], vlan, sizeof(vlan))) { 1711 int vid = lduw_be_p(ptr + 14) & 0xfff; 1712 if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f)))) 1713 return 0; 1714 } 1715 1716 if (ptr[0] & 1) { // multicast 1717 if (!memcmp(ptr, bcast, sizeof(bcast))) { 1718 return !n->nobcast; 1719 } else if (n->nomulti) { 1720 return 0; 1721 } else if (n->allmulti || n->mac_table.multi_overflow) { 1722 return 1; 1723 } 1724 1725 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 1726 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1727 return 1; 1728 } 1729 } 1730 } else { // unicast 1731 if (n->nouni) { 1732 return 0; 1733 } else if (n->alluni || n->mac_table.uni_overflow) { 1734 return 1; 1735 } else if (!memcmp(ptr, n->mac, ETH_ALEN)) { 1736 return 1; 1737 } 1738 1739 for (i = 0; i < n->mac_table.first_multi; i++) { 1740 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1741 return 1; 1742 } 1743 } 1744 } 1745 1746 return 0; 1747 } 1748 1749 static uint8_t virtio_net_get_hash_type(bool hasip4, 1750 bool hasip6, 1751 bool hasudp, 1752 bool hastcp, 1753 uint32_t types) 1754 { 1755 if (hasip4) { 1756 if (hastcp && (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4)) { 1757 return NetPktRssIpV4Tcp; 1758 } 1759 if (hasudp && (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4)) { 1760 return NetPktRssIpV4Udp; 1761 } 1762 if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 1763 return NetPktRssIpV4; 1764 } 1765 } else if (hasip6) { 1766 uint32_t mask = VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 1767 VIRTIO_NET_RSS_HASH_TYPE_TCPv6; 1768 1769 if (hastcp && (types & mask)) { 1770 return (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) ? 1771 NetPktRssIpV6TcpEx : NetPktRssIpV6Tcp; 1772 } 1773 mask = VIRTIO_NET_RSS_HASH_TYPE_UDP_EX | VIRTIO_NET_RSS_HASH_TYPE_UDPv6; 1774 if (hasudp && (types & mask)) { 1775 return (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) ? 1776 NetPktRssIpV6UdpEx : NetPktRssIpV6Udp; 1777 } 1778 mask = VIRTIO_NET_RSS_HASH_TYPE_IP_EX | VIRTIO_NET_RSS_HASH_TYPE_IPv6; 1779 if (types & mask) { 1780 return (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) ? 1781 NetPktRssIpV6Ex : NetPktRssIpV6; 1782 } 1783 } 1784 return 0xff; 1785 } 1786 1787 static void virtio_set_packet_hash(const uint8_t *buf, uint8_t report, 1788 uint32_t hash) 1789 { 1790 struct virtio_net_hdr_v1_hash *hdr = (void *)buf; 1791 hdr->hash_value = hash; 1792 hdr->hash_report = report; 1793 } 1794 1795 static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, 1796 size_t size) 1797 { 1798 VirtIONet *n = qemu_get_nic_opaque(nc); 1799 unsigned int index = nc->queue_index, new_index = index; 1800 struct NetRxPkt *pkt = n->rx_pkt; 1801 uint8_t net_hash_type; 1802 uint32_t hash; 1803 bool hasip4, hasip6, hasudp, hastcp; 1804 static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = { 1805 VIRTIO_NET_HASH_REPORT_IPv4, 1806 VIRTIO_NET_HASH_REPORT_TCPv4, 1807 VIRTIO_NET_HASH_REPORT_TCPv6, 1808 VIRTIO_NET_HASH_REPORT_IPv6, 1809 VIRTIO_NET_HASH_REPORT_IPv6_EX, 1810 VIRTIO_NET_HASH_REPORT_TCPv6_EX, 1811 VIRTIO_NET_HASH_REPORT_UDPv4, 1812 VIRTIO_NET_HASH_REPORT_UDPv6, 1813 VIRTIO_NET_HASH_REPORT_UDPv6_EX 1814 }; 1815 1816 net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len, 1817 size - n->host_hdr_len); 1818 net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &hasudp, &hastcp); 1819 net_hash_type = virtio_net_get_hash_type(hasip4, hasip6, hasudp, hastcp, 1820 n->rss_data.hash_types); 1821 if (net_hash_type > NetPktRssIpV6UdpEx) { 1822 if (n->rss_data.populate_hash) { 1823 virtio_set_packet_hash(buf, VIRTIO_NET_HASH_REPORT_NONE, 0); 1824 } 1825 return n->rss_data.redirect ? n->rss_data.default_queue : -1; 1826 } 1827 1828 hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key); 1829 1830 if (n->rss_data.populate_hash) { 1831 virtio_set_packet_hash(buf, reports[net_hash_type], hash); 1832 } 1833 1834 if (n->rss_data.redirect) { 1835 new_index = hash & (n->rss_data.indirections_len - 1); 1836 new_index = n->rss_data.indirections_table[new_index]; 1837 } 1838 1839 return (index == new_index) ? -1 : new_index; 1840 } 1841 1842 static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, 1843 size_t size, bool no_rss) 1844 { 1845 VirtIONet *n = qemu_get_nic_opaque(nc); 1846 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1847 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1848 VirtQueueElement *elems[VIRTQUEUE_MAX_SIZE]; 1849 size_t lens[VIRTQUEUE_MAX_SIZE]; 1850 struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE]; 1851 struct virtio_net_hdr_mrg_rxbuf mhdr; 1852 unsigned mhdr_cnt = 0; 1853 size_t offset, i, guest_offset, j; 1854 ssize_t err; 1855 1856 if (!virtio_net_can_receive(nc)) { 1857 return -1; 1858 } 1859 1860 if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) { 1861 int index = virtio_net_process_rss(nc, buf, size); 1862 if (index >= 0) { 1863 NetClientState *nc2 = qemu_get_subqueue(n->nic, index); 1864 return virtio_net_receive_rcu(nc2, buf, size, true); 1865 } 1866 } 1867 1868 /* hdr_len refers to the header we supply to the guest */ 1869 if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) { 1870 return 0; 1871 } 1872 1873 if (!receive_filter(n, buf, size)) 1874 return size; 1875 1876 offset = i = 0; 1877 1878 while (offset < size) { 1879 VirtQueueElement *elem; 1880 int len, total; 1881 const struct iovec *sg; 1882 1883 total = 0; 1884 1885 if (i == VIRTQUEUE_MAX_SIZE) { 1886 virtio_error(vdev, "virtio-net unexpected long buffer chain"); 1887 err = size; 1888 goto err; 1889 } 1890 1891 elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement)); 1892 if (!elem) { 1893 if (i) { 1894 virtio_error(vdev, "virtio-net unexpected empty queue: " 1895 "i %zd mergeable %d offset %zd, size %zd, " 1896 "guest hdr len %zd, host hdr len %zd " 1897 "guest features 0x%" PRIx64, 1898 i, n->mergeable_rx_bufs, offset, size, 1899 n->guest_hdr_len, n->host_hdr_len, 1900 vdev->guest_features); 1901 } 1902 err = -1; 1903 goto err; 1904 } 1905 1906 if (elem->in_num < 1) { 1907 virtio_error(vdev, 1908 "virtio-net receive queue contains no in buffers"); 1909 virtqueue_detach_element(q->rx_vq, elem, 0); 1910 g_free(elem); 1911 err = -1; 1912 goto err; 1913 } 1914 1915 sg = elem->in_sg; 1916 if (i == 0) { 1917 assert(offset == 0); 1918 if (n->mergeable_rx_bufs) { 1919 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg), 1920 sg, elem->in_num, 1921 offsetof(typeof(mhdr), num_buffers), 1922 sizeof(mhdr.num_buffers)); 1923 } 1924 1925 receive_header(n, sg, elem->in_num, buf, size); 1926 if (n->rss_data.populate_hash) { 1927 offset = sizeof(mhdr); 1928 iov_from_buf(sg, elem->in_num, offset, 1929 buf + offset, n->host_hdr_len - sizeof(mhdr)); 1930 } 1931 offset = n->host_hdr_len; 1932 total += n->guest_hdr_len; 1933 guest_offset = n->guest_hdr_len; 1934 } else { 1935 guest_offset = 0; 1936 } 1937 1938 /* copy in packet. ugh */ 1939 len = iov_from_buf(sg, elem->in_num, guest_offset, 1940 buf + offset, size - offset); 1941 total += len; 1942 offset += len; 1943 /* If buffers can't be merged, at this point we 1944 * must have consumed the complete packet. 1945 * Otherwise, drop it. */ 1946 if (!n->mergeable_rx_bufs && offset < size) { 1947 virtqueue_unpop(q->rx_vq, elem, total); 1948 g_free(elem); 1949 err = size; 1950 goto err; 1951 } 1952 1953 elems[i] = elem; 1954 lens[i] = total; 1955 i++; 1956 } 1957 1958 if (mhdr_cnt) { 1959 virtio_stw_p(vdev, &mhdr.num_buffers, i); 1960 iov_from_buf(mhdr_sg, mhdr_cnt, 1961 0, 1962 &mhdr.num_buffers, sizeof mhdr.num_buffers); 1963 } 1964 1965 for (j = 0; j < i; j++) { 1966 /* signal other side */ 1967 virtqueue_fill(q->rx_vq, elems[j], lens[j], j); 1968 g_free(elems[j]); 1969 } 1970 1971 virtqueue_flush(q->rx_vq, i); 1972 virtio_notify(vdev, q->rx_vq); 1973 1974 return size; 1975 1976 err: 1977 for (j = 0; j < i; j++) { 1978 virtqueue_detach_element(q->rx_vq, elems[j], lens[j]); 1979 g_free(elems[j]); 1980 } 1981 1982 return err; 1983 } 1984 1985 static ssize_t virtio_net_do_receive(NetClientState *nc, const uint8_t *buf, 1986 size_t size) 1987 { 1988 RCU_READ_LOCK_GUARD(); 1989 1990 return virtio_net_receive_rcu(nc, buf, size, false); 1991 } 1992 1993 static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain, 1994 const uint8_t *buf, 1995 VirtioNetRscUnit *unit) 1996 { 1997 uint16_t ip_hdrlen; 1998 struct ip_header *ip; 1999 2000 ip = (struct ip_header *)(buf + chain->n->guest_hdr_len 2001 + sizeof(struct eth_header)); 2002 unit->ip = (void *)ip; 2003 ip_hdrlen = (ip->ip_ver_len & 0xF) << 2; 2004 unit->ip_plen = &ip->ip_len; 2005 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) + ip_hdrlen); 2006 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 2007 unit->payload = htons(*unit->ip_plen) - ip_hdrlen - unit->tcp_hdrlen; 2008 } 2009 2010 static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain, 2011 const uint8_t *buf, 2012 VirtioNetRscUnit *unit) 2013 { 2014 struct ip6_header *ip6; 2015 2016 ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len 2017 + sizeof(struct eth_header)); 2018 unit->ip = ip6; 2019 unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 2020 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) 2021 + sizeof(struct ip6_header)); 2022 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 2023 2024 /* There is a difference between payload lenght in ipv4 and v6, 2025 ip header is excluded in ipv6 */ 2026 unit->payload = htons(*unit->ip_plen) - unit->tcp_hdrlen; 2027 } 2028 2029 static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain, 2030 VirtioNetRscSeg *seg) 2031 { 2032 int ret; 2033 struct virtio_net_hdr_v1 *h; 2034 2035 h = (struct virtio_net_hdr_v1 *)seg->buf; 2036 h->flags = 0; 2037 h->gso_type = VIRTIO_NET_HDR_GSO_NONE; 2038 2039 if (seg->is_coalesced) { 2040 h->rsc.segments = seg->packets; 2041 h->rsc.dup_acks = seg->dup_ack; 2042 h->flags = VIRTIO_NET_HDR_F_RSC_INFO; 2043 if (chain->proto == ETH_P_IP) { 2044 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2045 } else { 2046 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 2047 } 2048 } 2049 2050 ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size); 2051 QTAILQ_REMOVE(&chain->buffers, seg, next); 2052 g_free(seg->buf); 2053 g_free(seg); 2054 2055 return ret; 2056 } 2057 2058 static void virtio_net_rsc_purge(void *opq) 2059 { 2060 VirtioNetRscSeg *seg, *rn; 2061 VirtioNetRscChain *chain = (VirtioNetRscChain *)opq; 2062 2063 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) { 2064 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2065 chain->stat.purge_failed++; 2066 continue; 2067 } 2068 } 2069 2070 chain->stat.timer++; 2071 if (!QTAILQ_EMPTY(&chain->buffers)) { 2072 timer_mod(chain->drain_timer, 2073 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 2074 } 2075 } 2076 2077 static void virtio_net_rsc_cleanup(VirtIONet *n) 2078 { 2079 VirtioNetRscChain *chain, *rn_chain; 2080 VirtioNetRscSeg *seg, *rn_seg; 2081 2082 QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) { 2083 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) { 2084 QTAILQ_REMOVE(&chain->buffers, seg, next); 2085 g_free(seg->buf); 2086 g_free(seg); 2087 } 2088 2089 timer_free(chain->drain_timer); 2090 QTAILQ_REMOVE(&n->rsc_chains, chain, next); 2091 g_free(chain); 2092 } 2093 } 2094 2095 static void virtio_net_rsc_cache_buf(VirtioNetRscChain *chain, 2096 NetClientState *nc, 2097 const uint8_t *buf, size_t size) 2098 { 2099 uint16_t hdr_len; 2100 VirtioNetRscSeg *seg; 2101 2102 hdr_len = chain->n->guest_hdr_len; 2103 seg = g_new(VirtioNetRscSeg, 1); 2104 seg->buf = g_malloc(hdr_len + sizeof(struct eth_header) 2105 + sizeof(struct ip6_header) + VIRTIO_NET_MAX_TCP_PAYLOAD); 2106 memcpy(seg->buf, buf, size); 2107 seg->size = size; 2108 seg->packets = 1; 2109 seg->dup_ack = 0; 2110 seg->is_coalesced = 0; 2111 seg->nc = nc; 2112 2113 QTAILQ_INSERT_TAIL(&chain->buffers, seg, next); 2114 chain->stat.cache++; 2115 2116 switch (chain->proto) { 2117 case ETH_P_IP: 2118 virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit); 2119 break; 2120 case ETH_P_IPV6: 2121 virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit); 2122 break; 2123 default: 2124 g_assert_not_reached(); 2125 } 2126 } 2127 2128 static int32_t virtio_net_rsc_handle_ack(VirtioNetRscChain *chain, 2129 VirtioNetRscSeg *seg, 2130 const uint8_t *buf, 2131 struct tcp_header *n_tcp, 2132 struct tcp_header *o_tcp) 2133 { 2134 uint32_t nack, oack; 2135 uint16_t nwin, owin; 2136 2137 nack = htonl(n_tcp->th_ack); 2138 nwin = htons(n_tcp->th_win); 2139 oack = htonl(o_tcp->th_ack); 2140 owin = htons(o_tcp->th_win); 2141 2142 if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) { 2143 chain->stat.ack_out_of_win++; 2144 return RSC_FINAL; 2145 } else if (nack == oack) { 2146 /* duplicated ack or window probe */ 2147 if (nwin == owin) { 2148 /* duplicated ack, add dup ack count due to whql test up to 1 */ 2149 chain->stat.dup_ack++; 2150 return RSC_FINAL; 2151 } else { 2152 /* Coalesce window update */ 2153 o_tcp->th_win = n_tcp->th_win; 2154 chain->stat.win_update++; 2155 return RSC_COALESCE; 2156 } 2157 } else { 2158 /* pure ack, go to 'C', finalize*/ 2159 chain->stat.pure_ack++; 2160 return RSC_FINAL; 2161 } 2162 } 2163 2164 static int32_t virtio_net_rsc_coalesce_data(VirtioNetRscChain *chain, 2165 VirtioNetRscSeg *seg, 2166 const uint8_t *buf, 2167 VirtioNetRscUnit *n_unit) 2168 { 2169 void *data; 2170 uint16_t o_ip_len; 2171 uint32_t nseq, oseq; 2172 VirtioNetRscUnit *o_unit; 2173 2174 o_unit = &seg->unit; 2175 o_ip_len = htons(*o_unit->ip_plen); 2176 nseq = htonl(n_unit->tcp->th_seq); 2177 oseq = htonl(o_unit->tcp->th_seq); 2178 2179 /* out of order or retransmitted. */ 2180 if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) { 2181 chain->stat.data_out_of_win++; 2182 return RSC_FINAL; 2183 } 2184 2185 data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen; 2186 if (nseq == oseq) { 2187 if ((o_unit->payload == 0) && n_unit->payload) { 2188 /* From no payload to payload, normal case, not a dup ack or etc */ 2189 chain->stat.data_after_pure_ack++; 2190 goto coalesce; 2191 } else { 2192 return virtio_net_rsc_handle_ack(chain, seg, buf, 2193 n_unit->tcp, o_unit->tcp); 2194 } 2195 } else if ((nseq - oseq) != o_unit->payload) { 2196 /* Not a consistent packet, out of order */ 2197 chain->stat.data_out_of_order++; 2198 return RSC_FINAL; 2199 } else { 2200 coalesce: 2201 if ((o_ip_len + n_unit->payload) > chain->max_payload) { 2202 chain->stat.over_size++; 2203 return RSC_FINAL; 2204 } 2205 2206 /* Here comes the right data, the payload length in v4/v6 is different, 2207 so use the field value to update and record the new data len */ 2208 o_unit->payload += n_unit->payload; /* update new data len */ 2209 2210 /* update field in ip header */ 2211 *o_unit->ip_plen = htons(o_ip_len + n_unit->payload); 2212 2213 /* Bring 'PUSH' big, the whql test guide says 'PUSH' can be coalesced 2214 for windows guest, while this may change the behavior for linux 2215 guest (only if it uses RSC feature). */ 2216 o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags; 2217 2218 o_unit->tcp->th_ack = n_unit->tcp->th_ack; 2219 o_unit->tcp->th_win = n_unit->tcp->th_win; 2220 2221 memmove(seg->buf + seg->size, data, n_unit->payload); 2222 seg->size += n_unit->payload; 2223 seg->packets++; 2224 chain->stat.coalesced++; 2225 return RSC_COALESCE; 2226 } 2227 } 2228 2229 static int32_t virtio_net_rsc_coalesce4(VirtioNetRscChain *chain, 2230 VirtioNetRscSeg *seg, 2231 const uint8_t *buf, size_t size, 2232 VirtioNetRscUnit *unit) 2233 { 2234 struct ip_header *ip1, *ip2; 2235 2236 ip1 = (struct ip_header *)(unit->ip); 2237 ip2 = (struct ip_header *)(seg->unit.ip); 2238 if ((ip1->ip_src ^ ip2->ip_src) || (ip1->ip_dst ^ ip2->ip_dst) 2239 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2240 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2241 chain->stat.no_match++; 2242 return RSC_NO_MATCH; 2243 } 2244 2245 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2246 } 2247 2248 static int32_t virtio_net_rsc_coalesce6(VirtioNetRscChain *chain, 2249 VirtioNetRscSeg *seg, 2250 const uint8_t *buf, size_t size, 2251 VirtioNetRscUnit *unit) 2252 { 2253 struct ip6_header *ip1, *ip2; 2254 2255 ip1 = (struct ip6_header *)(unit->ip); 2256 ip2 = (struct ip6_header *)(seg->unit.ip); 2257 if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address)) 2258 || memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address)) 2259 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2260 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2261 chain->stat.no_match++; 2262 return RSC_NO_MATCH; 2263 } 2264 2265 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2266 } 2267 2268 /* Packets with 'SYN' should bypass, other flag should be sent after drain 2269 * to prevent out of order */ 2270 static int virtio_net_rsc_tcp_ctrl_check(VirtioNetRscChain *chain, 2271 struct tcp_header *tcp) 2272 { 2273 uint16_t tcp_hdr; 2274 uint16_t tcp_flag; 2275 2276 tcp_flag = htons(tcp->th_offset_flags); 2277 tcp_hdr = (tcp_flag & VIRTIO_NET_TCP_HDR_LENGTH) >> 10; 2278 tcp_flag &= VIRTIO_NET_TCP_FLAG; 2279 if (tcp_flag & TH_SYN) { 2280 chain->stat.tcp_syn++; 2281 return RSC_BYPASS; 2282 } 2283 2284 if (tcp_flag & (TH_FIN | TH_URG | TH_RST | TH_ECE | TH_CWR)) { 2285 chain->stat.tcp_ctrl_drain++; 2286 return RSC_FINAL; 2287 } 2288 2289 if (tcp_hdr > sizeof(struct tcp_header)) { 2290 chain->stat.tcp_all_opt++; 2291 return RSC_FINAL; 2292 } 2293 2294 return RSC_CANDIDATE; 2295 } 2296 2297 static size_t virtio_net_rsc_do_coalesce(VirtioNetRscChain *chain, 2298 NetClientState *nc, 2299 const uint8_t *buf, size_t size, 2300 VirtioNetRscUnit *unit) 2301 { 2302 int ret; 2303 VirtioNetRscSeg *seg, *nseg; 2304 2305 if (QTAILQ_EMPTY(&chain->buffers)) { 2306 chain->stat.empty_cache++; 2307 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2308 timer_mod(chain->drain_timer, 2309 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 2310 return size; 2311 } 2312 2313 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2314 if (chain->proto == ETH_P_IP) { 2315 ret = virtio_net_rsc_coalesce4(chain, seg, buf, size, unit); 2316 } else { 2317 ret = virtio_net_rsc_coalesce6(chain, seg, buf, size, unit); 2318 } 2319 2320 if (ret == RSC_FINAL) { 2321 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2322 /* Send failed */ 2323 chain->stat.final_failed++; 2324 return 0; 2325 } 2326 2327 /* Send current packet */ 2328 return virtio_net_do_receive(nc, buf, size); 2329 } else if (ret == RSC_NO_MATCH) { 2330 continue; 2331 } else { 2332 /* Coalesced, mark coalesced flag to tell calc cksum for ipv4 */ 2333 seg->is_coalesced = 1; 2334 return size; 2335 } 2336 } 2337 2338 chain->stat.no_match_cache++; 2339 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2340 return size; 2341 } 2342 2343 /* Drain a connection data, this is to avoid out of order segments */ 2344 static size_t virtio_net_rsc_drain_flow(VirtioNetRscChain *chain, 2345 NetClientState *nc, 2346 const uint8_t *buf, size_t size, 2347 uint16_t ip_start, uint16_t ip_size, 2348 uint16_t tcp_port) 2349 { 2350 VirtioNetRscSeg *seg, *nseg; 2351 uint32_t ppair1, ppair2; 2352 2353 ppair1 = *(uint32_t *)(buf + tcp_port); 2354 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2355 ppair2 = *(uint32_t *)(seg->buf + tcp_port); 2356 if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size) 2357 || (ppair1 != ppair2)) { 2358 continue; 2359 } 2360 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2361 chain->stat.drain_failed++; 2362 } 2363 2364 break; 2365 } 2366 2367 return virtio_net_do_receive(nc, buf, size); 2368 } 2369 2370 static int32_t virtio_net_rsc_sanity_check4(VirtioNetRscChain *chain, 2371 struct ip_header *ip, 2372 const uint8_t *buf, size_t size) 2373 { 2374 uint16_t ip_len; 2375 2376 /* Not an ipv4 packet */ 2377 if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) { 2378 chain->stat.ip_option++; 2379 return RSC_BYPASS; 2380 } 2381 2382 /* Don't handle packets with ip option */ 2383 if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) { 2384 chain->stat.ip_option++; 2385 return RSC_BYPASS; 2386 } 2387 2388 if (ip->ip_p != IPPROTO_TCP) { 2389 chain->stat.bypass_not_tcp++; 2390 return RSC_BYPASS; 2391 } 2392 2393 /* Don't handle packets with ip fragment */ 2394 if (!(htons(ip->ip_off) & IP_DF)) { 2395 chain->stat.ip_frag++; 2396 return RSC_BYPASS; 2397 } 2398 2399 /* Don't handle packets with ecn flag */ 2400 if (IPTOS_ECN(ip->ip_tos)) { 2401 chain->stat.ip_ecn++; 2402 return RSC_BYPASS; 2403 } 2404 2405 ip_len = htons(ip->ip_len); 2406 if (ip_len < (sizeof(struct ip_header) + sizeof(struct tcp_header)) 2407 || ip_len > (size - chain->n->guest_hdr_len - 2408 sizeof(struct eth_header))) { 2409 chain->stat.ip_hacked++; 2410 return RSC_BYPASS; 2411 } 2412 2413 return RSC_CANDIDATE; 2414 } 2415 2416 static size_t virtio_net_rsc_receive4(VirtioNetRscChain *chain, 2417 NetClientState *nc, 2418 const uint8_t *buf, size_t size) 2419 { 2420 int32_t ret; 2421 uint16_t hdr_len; 2422 VirtioNetRscUnit unit; 2423 2424 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2425 2426 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header) 2427 + sizeof(struct tcp_header))) { 2428 chain->stat.bypass_not_tcp++; 2429 return virtio_net_do_receive(nc, buf, size); 2430 } 2431 2432 virtio_net_rsc_extract_unit4(chain, buf, &unit); 2433 if (virtio_net_rsc_sanity_check4(chain, unit.ip, buf, size) 2434 != RSC_CANDIDATE) { 2435 return virtio_net_do_receive(nc, buf, size); 2436 } 2437 2438 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2439 if (ret == RSC_BYPASS) { 2440 return virtio_net_do_receive(nc, buf, size); 2441 } else if (ret == RSC_FINAL) { 2442 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2443 ((hdr_len + sizeof(struct eth_header)) + 12), 2444 VIRTIO_NET_IP4_ADDR_SIZE, 2445 hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header)); 2446 } 2447 2448 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2449 } 2450 2451 static int32_t virtio_net_rsc_sanity_check6(VirtioNetRscChain *chain, 2452 struct ip6_header *ip6, 2453 const uint8_t *buf, size_t size) 2454 { 2455 uint16_t ip_len; 2456 2457 if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4) 2458 != IP_HEADER_VERSION_6) { 2459 return RSC_BYPASS; 2460 } 2461 2462 /* Both option and protocol is checked in this */ 2463 if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) { 2464 chain->stat.bypass_not_tcp++; 2465 return RSC_BYPASS; 2466 } 2467 2468 ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 2469 if (ip_len < sizeof(struct tcp_header) || 2470 ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header) 2471 - sizeof(struct ip6_header))) { 2472 chain->stat.ip_hacked++; 2473 return RSC_BYPASS; 2474 } 2475 2476 /* Don't handle packets with ecn flag */ 2477 if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) { 2478 chain->stat.ip_ecn++; 2479 return RSC_BYPASS; 2480 } 2481 2482 return RSC_CANDIDATE; 2483 } 2484 2485 static size_t virtio_net_rsc_receive6(void *opq, NetClientState *nc, 2486 const uint8_t *buf, size_t size) 2487 { 2488 int32_t ret; 2489 uint16_t hdr_len; 2490 VirtioNetRscChain *chain; 2491 VirtioNetRscUnit unit; 2492 2493 chain = opq; 2494 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2495 2496 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip6_header) 2497 + sizeof(tcp_header))) { 2498 return virtio_net_do_receive(nc, buf, size); 2499 } 2500 2501 virtio_net_rsc_extract_unit6(chain, buf, &unit); 2502 if (RSC_CANDIDATE != virtio_net_rsc_sanity_check6(chain, 2503 unit.ip, buf, size)) { 2504 return virtio_net_do_receive(nc, buf, size); 2505 } 2506 2507 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2508 if (ret == RSC_BYPASS) { 2509 return virtio_net_do_receive(nc, buf, size); 2510 } else if (ret == RSC_FINAL) { 2511 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2512 ((hdr_len + sizeof(struct eth_header)) + 8), 2513 VIRTIO_NET_IP6_ADDR_SIZE, 2514 hdr_len + sizeof(struct eth_header) 2515 + sizeof(struct ip6_header)); 2516 } 2517 2518 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2519 } 2520 2521 static VirtioNetRscChain *virtio_net_rsc_lookup_chain(VirtIONet *n, 2522 NetClientState *nc, 2523 uint16_t proto) 2524 { 2525 VirtioNetRscChain *chain; 2526 2527 if ((proto != (uint16_t)ETH_P_IP) && (proto != (uint16_t)ETH_P_IPV6)) { 2528 return NULL; 2529 } 2530 2531 QTAILQ_FOREACH(chain, &n->rsc_chains, next) { 2532 if (chain->proto == proto) { 2533 return chain; 2534 } 2535 } 2536 2537 chain = g_malloc(sizeof(*chain)); 2538 chain->n = n; 2539 chain->proto = proto; 2540 if (proto == (uint16_t)ETH_P_IP) { 2541 chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD; 2542 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2543 } else { 2544 chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD; 2545 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 2546 } 2547 chain->drain_timer = timer_new_ns(QEMU_CLOCK_HOST, 2548 virtio_net_rsc_purge, chain); 2549 memset(&chain->stat, 0, sizeof(chain->stat)); 2550 2551 QTAILQ_INIT(&chain->buffers); 2552 QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next); 2553 2554 return chain; 2555 } 2556 2557 static ssize_t virtio_net_rsc_receive(NetClientState *nc, 2558 const uint8_t *buf, 2559 size_t size) 2560 { 2561 uint16_t proto; 2562 VirtioNetRscChain *chain; 2563 struct eth_header *eth; 2564 VirtIONet *n; 2565 2566 n = qemu_get_nic_opaque(nc); 2567 if (size < (n->host_hdr_len + sizeof(struct eth_header))) { 2568 return virtio_net_do_receive(nc, buf, size); 2569 } 2570 2571 eth = (struct eth_header *)(buf + n->guest_hdr_len); 2572 proto = htons(eth->h_proto); 2573 2574 chain = virtio_net_rsc_lookup_chain(n, nc, proto); 2575 if (chain) { 2576 chain->stat.received++; 2577 if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) { 2578 return virtio_net_rsc_receive4(chain, nc, buf, size); 2579 } else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) { 2580 return virtio_net_rsc_receive6(chain, nc, buf, size); 2581 } 2582 } 2583 return virtio_net_do_receive(nc, buf, size); 2584 } 2585 2586 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, 2587 size_t size) 2588 { 2589 VirtIONet *n = qemu_get_nic_opaque(nc); 2590 if ((n->rsc4_enabled || n->rsc6_enabled)) { 2591 return virtio_net_rsc_receive(nc, buf, size); 2592 } else { 2593 return virtio_net_do_receive(nc, buf, size); 2594 } 2595 } 2596 2597 static int32_t virtio_net_flush_tx(VirtIONetQueue *q); 2598 2599 static void virtio_net_tx_complete(NetClientState *nc, ssize_t len) 2600 { 2601 VirtIONet *n = qemu_get_nic_opaque(nc); 2602 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 2603 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2604 int ret; 2605 2606 virtqueue_push(q->tx_vq, q->async_tx.elem, 0); 2607 virtio_notify(vdev, q->tx_vq); 2608 2609 g_free(q->async_tx.elem); 2610 q->async_tx.elem = NULL; 2611 2612 virtio_queue_set_notification(q->tx_vq, 1); 2613 ret = virtio_net_flush_tx(q); 2614 if (ret >= n->tx_burst) { 2615 /* 2616 * the flush has been stopped by tx_burst 2617 * we will not receive notification for the 2618 * remainining part, so re-schedule 2619 */ 2620 virtio_queue_set_notification(q->tx_vq, 0); 2621 if (q->tx_bh) { 2622 qemu_bh_schedule(q->tx_bh); 2623 } else { 2624 timer_mod(q->tx_timer, 2625 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2626 } 2627 q->tx_waiting = 1; 2628 } 2629 } 2630 2631 /* TX */ 2632 static int32_t virtio_net_flush_tx(VirtIONetQueue *q) 2633 { 2634 VirtIONet *n = q->n; 2635 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2636 VirtQueueElement *elem; 2637 int32_t num_packets = 0; 2638 int queue_index = vq2q(virtio_get_queue_index(q->tx_vq)); 2639 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2640 return num_packets; 2641 } 2642 2643 if (q->async_tx.elem) { 2644 virtio_queue_set_notification(q->tx_vq, 0); 2645 return num_packets; 2646 } 2647 2648 for (;;) { 2649 ssize_t ret; 2650 unsigned int out_num; 2651 struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg; 2652 struct virtio_net_hdr_mrg_rxbuf mhdr; 2653 2654 elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement)); 2655 if (!elem) { 2656 break; 2657 } 2658 2659 out_num = elem->out_num; 2660 out_sg = elem->out_sg; 2661 if (out_num < 1) { 2662 virtio_error(vdev, "virtio-net header not in first element"); 2663 virtqueue_detach_element(q->tx_vq, elem, 0); 2664 g_free(elem); 2665 return -EINVAL; 2666 } 2667 2668 if (n->has_vnet_hdr) { 2669 if (iov_to_buf(out_sg, out_num, 0, &mhdr, n->guest_hdr_len) < 2670 n->guest_hdr_len) { 2671 virtio_error(vdev, "virtio-net header incorrect"); 2672 virtqueue_detach_element(q->tx_vq, elem, 0); 2673 g_free(elem); 2674 return -EINVAL; 2675 } 2676 if (n->needs_vnet_hdr_swap) { 2677 virtio_net_hdr_swap(vdev, (void *) &mhdr); 2678 sg2[0].iov_base = &mhdr; 2679 sg2[0].iov_len = n->guest_hdr_len; 2680 out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1, 2681 out_sg, out_num, 2682 n->guest_hdr_len, -1); 2683 if (out_num == VIRTQUEUE_MAX_SIZE) { 2684 goto drop; 2685 } 2686 out_num += 1; 2687 out_sg = sg2; 2688 } 2689 } 2690 /* 2691 * If host wants to see the guest header as is, we can 2692 * pass it on unchanged. Otherwise, copy just the parts 2693 * that host is interested in. 2694 */ 2695 assert(n->host_hdr_len <= n->guest_hdr_len); 2696 if (n->host_hdr_len != n->guest_hdr_len) { 2697 unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg), 2698 out_sg, out_num, 2699 0, n->host_hdr_len); 2700 sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num, 2701 out_sg, out_num, 2702 n->guest_hdr_len, -1); 2703 out_num = sg_num; 2704 out_sg = sg; 2705 } 2706 2707 ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index), 2708 out_sg, out_num, virtio_net_tx_complete); 2709 if (ret == 0) { 2710 virtio_queue_set_notification(q->tx_vq, 0); 2711 q->async_tx.elem = elem; 2712 return -EBUSY; 2713 } 2714 2715 drop: 2716 virtqueue_push(q->tx_vq, elem, 0); 2717 virtio_notify(vdev, q->tx_vq); 2718 g_free(elem); 2719 2720 if (++num_packets >= n->tx_burst) { 2721 break; 2722 } 2723 } 2724 return num_packets; 2725 } 2726 2727 static void virtio_net_tx_timer(void *opaque); 2728 2729 static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq) 2730 { 2731 VirtIONet *n = VIRTIO_NET(vdev); 2732 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2733 2734 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2735 virtio_net_drop_tx_queue_data(vdev, vq); 2736 return; 2737 } 2738 2739 /* This happens when device was stopped but VCPU wasn't. */ 2740 if (!vdev->vm_running) { 2741 q->tx_waiting = 1; 2742 return; 2743 } 2744 2745 if (q->tx_waiting) { 2746 /* We already have queued packets, immediately flush */ 2747 timer_del(q->tx_timer); 2748 virtio_net_tx_timer(q); 2749 } else { 2750 /* re-arm timer to flush it (and more) on next tick */ 2751 timer_mod(q->tx_timer, 2752 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2753 q->tx_waiting = 1; 2754 virtio_queue_set_notification(vq, 0); 2755 } 2756 } 2757 2758 static void virtio_net_handle_tx_bh(VirtIODevice *vdev, VirtQueue *vq) 2759 { 2760 VirtIONet *n = VIRTIO_NET(vdev); 2761 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2762 2763 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2764 virtio_net_drop_tx_queue_data(vdev, vq); 2765 return; 2766 } 2767 2768 if (unlikely(q->tx_waiting)) { 2769 return; 2770 } 2771 q->tx_waiting = 1; 2772 /* This happens when device was stopped but VCPU wasn't. */ 2773 if (!vdev->vm_running) { 2774 return; 2775 } 2776 virtio_queue_set_notification(vq, 0); 2777 qemu_bh_schedule(q->tx_bh); 2778 } 2779 2780 static void virtio_net_tx_timer(void *opaque) 2781 { 2782 VirtIONetQueue *q = opaque; 2783 VirtIONet *n = q->n; 2784 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2785 int ret; 2786 2787 /* This happens when device was stopped but BH wasn't. */ 2788 if (!vdev->vm_running) { 2789 /* Make sure tx waiting is set, so we'll run when restarted. */ 2790 assert(q->tx_waiting); 2791 return; 2792 } 2793 2794 q->tx_waiting = 0; 2795 2796 /* Just in case the driver is not ready on more */ 2797 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2798 return; 2799 } 2800 2801 ret = virtio_net_flush_tx(q); 2802 if (ret == -EBUSY || ret == -EINVAL) { 2803 return; 2804 } 2805 /* 2806 * If we flush a full burst of packets, assume there are 2807 * more coming and immediately rearm 2808 */ 2809 if (ret >= n->tx_burst) { 2810 q->tx_waiting = 1; 2811 timer_mod(q->tx_timer, 2812 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2813 return; 2814 } 2815 /* 2816 * If less than a full burst, re-enable notification and flush 2817 * anything that may have come in while we weren't looking. If 2818 * we find something, assume the guest is still active and rearm 2819 */ 2820 virtio_queue_set_notification(q->tx_vq, 1); 2821 ret = virtio_net_flush_tx(q); 2822 if (ret > 0) { 2823 virtio_queue_set_notification(q->tx_vq, 0); 2824 q->tx_waiting = 1; 2825 timer_mod(q->tx_timer, 2826 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2827 } 2828 } 2829 2830 static void virtio_net_tx_bh(void *opaque) 2831 { 2832 VirtIONetQueue *q = opaque; 2833 VirtIONet *n = q->n; 2834 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2835 int32_t ret; 2836 2837 /* This happens when device was stopped but BH wasn't. */ 2838 if (!vdev->vm_running) { 2839 /* Make sure tx waiting is set, so we'll run when restarted. */ 2840 assert(q->tx_waiting); 2841 return; 2842 } 2843 2844 q->tx_waiting = 0; 2845 2846 /* Just in case the driver is not ready on more */ 2847 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 2848 return; 2849 } 2850 2851 ret = virtio_net_flush_tx(q); 2852 if (ret == -EBUSY || ret == -EINVAL) { 2853 return; /* Notification re-enable handled by tx_complete or device 2854 * broken */ 2855 } 2856 2857 /* If we flush a full burst of packets, assume there are 2858 * more coming and immediately reschedule */ 2859 if (ret >= n->tx_burst) { 2860 qemu_bh_schedule(q->tx_bh); 2861 q->tx_waiting = 1; 2862 return; 2863 } 2864 2865 /* If less than a full burst, re-enable notification and flush 2866 * anything that may have come in while we weren't looking. If 2867 * we find something, assume the guest is still active and reschedule */ 2868 virtio_queue_set_notification(q->tx_vq, 1); 2869 ret = virtio_net_flush_tx(q); 2870 if (ret == -EINVAL) { 2871 return; 2872 } else if (ret > 0) { 2873 virtio_queue_set_notification(q->tx_vq, 0); 2874 qemu_bh_schedule(q->tx_bh); 2875 q->tx_waiting = 1; 2876 } 2877 } 2878 2879 static void virtio_net_add_queue(VirtIONet *n, int index) 2880 { 2881 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2882 2883 n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size, 2884 virtio_net_handle_rx); 2885 2886 if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) { 2887 n->vqs[index].tx_vq = 2888 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2889 virtio_net_handle_tx_timer); 2890 n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, 2891 virtio_net_tx_timer, 2892 &n->vqs[index]); 2893 } else { 2894 n->vqs[index].tx_vq = 2895 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2896 virtio_net_handle_tx_bh); 2897 n->vqs[index].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[index]); 2898 } 2899 2900 n->vqs[index].tx_waiting = 0; 2901 n->vqs[index].n = n; 2902 } 2903 2904 static void virtio_net_del_queue(VirtIONet *n, int index) 2905 { 2906 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2907 VirtIONetQueue *q = &n->vqs[index]; 2908 NetClientState *nc = qemu_get_subqueue(n->nic, index); 2909 2910 qemu_purge_queued_packets(nc); 2911 2912 virtio_del_queue(vdev, index * 2); 2913 if (q->tx_timer) { 2914 timer_free(q->tx_timer); 2915 q->tx_timer = NULL; 2916 } else { 2917 qemu_bh_delete(q->tx_bh); 2918 q->tx_bh = NULL; 2919 } 2920 q->tx_waiting = 0; 2921 virtio_del_queue(vdev, index * 2 + 1); 2922 } 2923 2924 static void virtio_net_change_num_queue_pairs(VirtIONet *n, int new_max_queue_pairs) 2925 { 2926 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2927 int old_num_queues = virtio_get_num_queues(vdev); 2928 int new_num_queues = new_max_queue_pairs * 2 + 1; 2929 int i; 2930 2931 assert(old_num_queues >= 3); 2932 assert(old_num_queues % 2 == 1); 2933 2934 if (old_num_queues == new_num_queues) { 2935 return; 2936 } 2937 2938 /* 2939 * We always need to remove and add ctrl vq if 2940 * old_num_queues != new_num_queues. Remove ctrl_vq first, 2941 * and then we only enter one of the following two loops. 2942 */ 2943 virtio_del_queue(vdev, old_num_queues - 1); 2944 2945 for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) { 2946 /* new_num_queues < old_num_queues */ 2947 virtio_net_del_queue(n, i / 2); 2948 } 2949 2950 for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) { 2951 /* new_num_queues > old_num_queues */ 2952 virtio_net_add_queue(n, i / 2); 2953 } 2954 2955 /* add ctrl_vq last */ 2956 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 2957 } 2958 2959 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue) 2960 { 2961 int max = multiqueue ? n->max_queue_pairs : 1; 2962 2963 n->multiqueue = multiqueue; 2964 virtio_net_change_num_queue_pairs(n, max); 2965 2966 virtio_net_set_queue_pairs(n); 2967 } 2968 2969 static int virtio_net_post_load_device(void *opaque, int version_id) 2970 { 2971 VirtIONet *n = opaque; 2972 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2973 int i, link_down; 2974 2975 trace_virtio_net_post_load_device(); 2976 virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs, 2977 virtio_vdev_has_feature(vdev, 2978 VIRTIO_F_VERSION_1), 2979 virtio_vdev_has_feature(vdev, 2980 VIRTIO_NET_F_HASH_REPORT)); 2981 2982 /* MAC_TABLE_ENTRIES may be different from the saved image */ 2983 if (n->mac_table.in_use > MAC_TABLE_ENTRIES) { 2984 n->mac_table.in_use = 0; 2985 } 2986 2987 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 2988 n->curr_guest_offloads = virtio_net_supported_guest_offloads(n); 2989 } 2990 2991 /* 2992 * curr_guest_offloads will be later overwritten by the 2993 * virtio_set_features_nocheck call done from the virtio_load. 2994 * Here we make sure it is preserved and restored accordingly 2995 * in the virtio_net_post_load_virtio callback. 2996 */ 2997 n->saved_guest_offloads = n->curr_guest_offloads; 2998 2999 virtio_net_set_queue_pairs(n); 3000 3001 /* Find the first multicast entry in the saved MAC filter */ 3002 for (i = 0; i < n->mac_table.in_use; i++) { 3003 if (n->mac_table.macs[i * ETH_ALEN] & 1) { 3004 break; 3005 } 3006 } 3007 n->mac_table.first_multi = i; 3008 3009 /* nc.link_down can't be migrated, so infer link_down according 3010 * to link status bit in n->status */ 3011 link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0; 3012 for (i = 0; i < n->max_queue_pairs; i++) { 3013 qemu_get_subqueue(n->nic, i)->link_down = link_down; 3014 } 3015 3016 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 3017 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 3018 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 3019 QEMU_CLOCK_VIRTUAL, 3020 virtio_net_announce_timer, n); 3021 if (n->announce_timer.round) { 3022 timer_mod(n->announce_timer.tm, 3023 qemu_clock_get_ms(n->announce_timer.type)); 3024 } else { 3025 qemu_announce_timer_del(&n->announce_timer, false); 3026 } 3027 } 3028 3029 if (n->rss_data.enabled) { 3030 n->rss_data.enabled_software_rss = n->rss_data.populate_hash; 3031 if (!n->rss_data.populate_hash) { 3032 if (!virtio_net_attach_epbf_rss(n)) { 3033 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 3034 warn_report("Can't post-load eBPF RSS for vhost"); 3035 } else { 3036 warn_report("Can't post-load eBPF RSS - " 3037 "fallback to software RSS"); 3038 n->rss_data.enabled_software_rss = true; 3039 } 3040 } 3041 } 3042 3043 trace_virtio_net_rss_enable(n->rss_data.hash_types, 3044 n->rss_data.indirections_len, 3045 sizeof(n->rss_data.key)); 3046 } else { 3047 trace_virtio_net_rss_disable(); 3048 } 3049 return 0; 3050 } 3051 3052 static int virtio_net_post_load_virtio(VirtIODevice *vdev) 3053 { 3054 VirtIONet *n = VIRTIO_NET(vdev); 3055 /* 3056 * The actual needed state is now in saved_guest_offloads, 3057 * see virtio_net_post_load_device for detail. 3058 * Restore it back and apply the desired offloads. 3059 */ 3060 n->curr_guest_offloads = n->saved_guest_offloads; 3061 if (peer_has_vnet_hdr(n)) { 3062 virtio_net_apply_guest_offloads(n); 3063 } 3064 3065 return 0; 3066 } 3067 3068 /* tx_waiting field of a VirtIONetQueue */ 3069 static const VMStateDescription vmstate_virtio_net_queue_tx_waiting = { 3070 .name = "virtio-net-queue-tx_waiting", 3071 .fields = (VMStateField[]) { 3072 VMSTATE_UINT32(tx_waiting, VirtIONetQueue), 3073 VMSTATE_END_OF_LIST() 3074 }, 3075 }; 3076 3077 static bool max_queue_pairs_gt_1(void *opaque, int version_id) 3078 { 3079 return VIRTIO_NET(opaque)->max_queue_pairs > 1; 3080 } 3081 3082 static bool has_ctrl_guest_offloads(void *opaque, int version_id) 3083 { 3084 return virtio_vdev_has_feature(VIRTIO_DEVICE(opaque), 3085 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 3086 } 3087 3088 static bool mac_table_fits(void *opaque, int version_id) 3089 { 3090 return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES; 3091 } 3092 3093 static bool mac_table_doesnt_fit(void *opaque, int version_id) 3094 { 3095 return !mac_table_fits(opaque, version_id); 3096 } 3097 3098 /* This temporary type is shared by all the WITH_TMP methods 3099 * although only some fields are used by each. 3100 */ 3101 struct VirtIONetMigTmp { 3102 VirtIONet *parent; 3103 VirtIONetQueue *vqs_1; 3104 uint16_t curr_queue_pairs_1; 3105 uint8_t has_ufo; 3106 uint32_t has_vnet_hdr; 3107 }; 3108 3109 /* The 2nd and subsequent tx_waiting flags are loaded later than 3110 * the 1st entry in the queue_pairs and only if there's more than one 3111 * entry. We use the tmp mechanism to calculate a temporary 3112 * pointer and count and also validate the count. 3113 */ 3114 3115 static int virtio_net_tx_waiting_pre_save(void *opaque) 3116 { 3117 struct VirtIONetMigTmp *tmp = opaque; 3118 3119 tmp->vqs_1 = tmp->parent->vqs + 1; 3120 tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1; 3121 if (tmp->parent->curr_queue_pairs == 0) { 3122 tmp->curr_queue_pairs_1 = 0; 3123 } 3124 3125 return 0; 3126 } 3127 3128 static int virtio_net_tx_waiting_pre_load(void *opaque) 3129 { 3130 struct VirtIONetMigTmp *tmp = opaque; 3131 3132 /* Reuse the pointer setup from save */ 3133 virtio_net_tx_waiting_pre_save(opaque); 3134 3135 if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) { 3136 error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x", 3137 tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs); 3138 3139 return -EINVAL; 3140 } 3141 3142 return 0; /* all good */ 3143 } 3144 3145 static const VMStateDescription vmstate_virtio_net_tx_waiting = { 3146 .name = "virtio-net-tx_waiting", 3147 .pre_load = virtio_net_tx_waiting_pre_load, 3148 .pre_save = virtio_net_tx_waiting_pre_save, 3149 .fields = (VMStateField[]) { 3150 VMSTATE_STRUCT_VARRAY_POINTER_UINT16(vqs_1, struct VirtIONetMigTmp, 3151 curr_queue_pairs_1, 3152 vmstate_virtio_net_queue_tx_waiting, 3153 struct VirtIONetQueue), 3154 VMSTATE_END_OF_LIST() 3155 }, 3156 }; 3157 3158 /* the 'has_ufo' flag is just tested; if the incoming stream has the 3159 * flag set we need to check that we have it 3160 */ 3161 static int virtio_net_ufo_post_load(void *opaque, int version_id) 3162 { 3163 struct VirtIONetMigTmp *tmp = opaque; 3164 3165 if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) { 3166 error_report("virtio-net: saved image requires TUN_F_UFO support"); 3167 return -EINVAL; 3168 } 3169 3170 return 0; 3171 } 3172 3173 static int virtio_net_ufo_pre_save(void *opaque) 3174 { 3175 struct VirtIONetMigTmp *tmp = opaque; 3176 3177 tmp->has_ufo = tmp->parent->has_ufo; 3178 3179 return 0; 3180 } 3181 3182 static const VMStateDescription vmstate_virtio_net_has_ufo = { 3183 .name = "virtio-net-ufo", 3184 .post_load = virtio_net_ufo_post_load, 3185 .pre_save = virtio_net_ufo_pre_save, 3186 .fields = (VMStateField[]) { 3187 VMSTATE_UINT8(has_ufo, struct VirtIONetMigTmp), 3188 VMSTATE_END_OF_LIST() 3189 }, 3190 }; 3191 3192 /* the 'has_vnet_hdr' flag is just tested; if the incoming stream has the 3193 * flag set we need to check that we have it 3194 */ 3195 static int virtio_net_vnet_post_load(void *opaque, int version_id) 3196 { 3197 struct VirtIONetMigTmp *tmp = opaque; 3198 3199 if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) { 3200 error_report("virtio-net: saved image requires vnet_hdr=on"); 3201 return -EINVAL; 3202 } 3203 3204 return 0; 3205 } 3206 3207 static int virtio_net_vnet_pre_save(void *opaque) 3208 { 3209 struct VirtIONetMigTmp *tmp = opaque; 3210 3211 tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr; 3212 3213 return 0; 3214 } 3215 3216 static const VMStateDescription vmstate_virtio_net_has_vnet = { 3217 .name = "virtio-net-vnet", 3218 .post_load = virtio_net_vnet_post_load, 3219 .pre_save = virtio_net_vnet_pre_save, 3220 .fields = (VMStateField[]) { 3221 VMSTATE_UINT32(has_vnet_hdr, struct VirtIONetMigTmp), 3222 VMSTATE_END_OF_LIST() 3223 }, 3224 }; 3225 3226 static bool virtio_net_rss_needed(void *opaque) 3227 { 3228 return VIRTIO_NET(opaque)->rss_data.enabled; 3229 } 3230 3231 static const VMStateDescription vmstate_virtio_net_rss = { 3232 .name = "virtio-net-device/rss", 3233 .version_id = 1, 3234 .minimum_version_id = 1, 3235 .needed = virtio_net_rss_needed, 3236 .fields = (VMStateField[]) { 3237 VMSTATE_BOOL(rss_data.enabled, VirtIONet), 3238 VMSTATE_BOOL(rss_data.redirect, VirtIONet), 3239 VMSTATE_BOOL(rss_data.populate_hash, VirtIONet), 3240 VMSTATE_UINT32(rss_data.hash_types, VirtIONet), 3241 VMSTATE_UINT16(rss_data.indirections_len, VirtIONet), 3242 VMSTATE_UINT16(rss_data.default_queue, VirtIONet), 3243 VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet, 3244 VIRTIO_NET_RSS_MAX_KEY_SIZE), 3245 VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet, 3246 rss_data.indirections_len, 0, 3247 vmstate_info_uint16, uint16_t), 3248 VMSTATE_END_OF_LIST() 3249 }, 3250 }; 3251 3252 static const VMStateDescription vmstate_virtio_net_device = { 3253 .name = "virtio-net-device", 3254 .version_id = VIRTIO_NET_VM_VERSION, 3255 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3256 .post_load = virtio_net_post_load_device, 3257 .fields = (VMStateField[]) { 3258 VMSTATE_UINT8_ARRAY(mac, VirtIONet, ETH_ALEN), 3259 VMSTATE_STRUCT_POINTER(vqs, VirtIONet, 3260 vmstate_virtio_net_queue_tx_waiting, 3261 VirtIONetQueue), 3262 VMSTATE_UINT32(mergeable_rx_bufs, VirtIONet), 3263 VMSTATE_UINT16(status, VirtIONet), 3264 VMSTATE_UINT8(promisc, VirtIONet), 3265 VMSTATE_UINT8(allmulti, VirtIONet), 3266 VMSTATE_UINT32(mac_table.in_use, VirtIONet), 3267 3268 /* Guarded pair: If it fits we load it, else we throw it away 3269 * - can happen if source has a larger MAC table.; post-load 3270 * sets flags in this case. 3271 */ 3272 VMSTATE_VBUFFER_MULTIPLY(mac_table.macs, VirtIONet, 3273 0, mac_table_fits, mac_table.in_use, 3274 ETH_ALEN), 3275 VMSTATE_UNUSED_VARRAY_UINT32(VirtIONet, mac_table_doesnt_fit, 0, 3276 mac_table.in_use, ETH_ALEN), 3277 3278 /* Note: This is an array of uint32's that's always been saved as a 3279 * buffer; hold onto your endiannesses; it's actually used as a bitmap 3280 * but based on the uint. 3281 */ 3282 VMSTATE_BUFFER_POINTER_UNSAFE(vlans, VirtIONet, 0, MAX_VLAN >> 3), 3283 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3284 vmstate_virtio_net_has_vnet), 3285 VMSTATE_UINT8(mac_table.multi_overflow, VirtIONet), 3286 VMSTATE_UINT8(mac_table.uni_overflow, VirtIONet), 3287 VMSTATE_UINT8(alluni, VirtIONet), 3288 VMSTATE_UINT8(nomulti, VirtIONet), 3289 VMSTATE_UINT8(nouni, VirtIONet), 3290 VMSTATE_UINT8(nobcast, VirtIONet), 3291 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3292 vmstate_virtio_net_has_ufo), 3293 VMSTATE_SINGLE_TEST(max_queue_pairs, VirtIONet, max_queue_pairs_gt_1, 0, 3294 vmstate_info_uint16_equal, uint16_t), 3295 VMSTATE_UINT16_TEST(curr_queue_pairs, VirtIONet, max_queue_pairs_gt_1), 3296 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3297 vmstate_virtio_net_tx_waiting), 3298 VMSTATE_UINT64_TEST(curr_guest_offloads, VirtIONet, 3299 has_ctrl_guest_offloads), 3300 VMSTATE_END_OF_LIST() 3301 }, 3302 .subsections = (const VMStateDescription * []) { 3303 &vmstate_virtio_net_rss, 3304 NULL 3305 } 3306 }; 3307 3308 static NetClientInfo net_virtio_info = { 3309 .type = NET_CLIENT_DRIVER_NIC, 3310 .size = sizeof(NICState), 3311 .can_receive = virtio_net_can_receive, 3312 .receive = virtio_net_receive, 3313 .link_status_changed = virtio_net_set_link_status, 3314 .query_rx_filter = virtio_net_query_rxfilter, 3315 .announce = virtio_net_announce, 3316 }; 3317 3318 static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx) 3319 { 3320 VirtIONet *n = VIRTIO_NET(vdev); 3321 NetClientState *nc; 3322 assert(n->vhost_started); 3323 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ) && idx == 2) { 3324 /* Must guard against invalid features and bogus queue index 3325 * from being set by malicious guest, or penetrated through 3326 * buggy migration stream. 3327 */ 3328 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 3329 qemu_log_mask(LOG_GUEST_ERROR, 3330 "%s: bogus vq index ignored\n", __func__); 3331 return false; 3332 } 3333 nc = qemu_get_subqueue(n->nic, n->max_queue_pairs); 3334 } else { 3335 nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3336 } 3337 /* 3338 * Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1 3339 * as the Marco of configure interrupt's IDX, If this driver does not 3340 * support, the function will return false 3341 */ 3342 3343 if (idx == VIRTIO_CONFIG_IRQ_IDX) { 3344 return vhost_net_config_pending(get_vhost_net(nc->peer)); 3345 } 3346 return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx); 3347 } 3348 3349 static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx, 3350 bool mask) 3351 { 3352 VirtIONet *n = VIRTIO_NET(vdev); 3353 NetClientState *nc; 3354 assert(n->vhost_started); 3355 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ) && idx == 2) { 3356 /* Must guard against invalid features and bogus queue index 3357 * from being set by malicious guest, or penetrated through 3358 * buggy migration stream. 3359 */ 3360 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 3361 qemu_log_mask(LOG_GUEST_ERROR, 3362 "%s: bogus vq index ignored\n", __func__); 3363 return; 3364 } 3365 nc = qemu_get_subqueue(n->nic, n->max_queue_pairs); 3366 } else { 3367 nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3368 } 3369 /* 3370 *Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1 3371 * as the Marco of configure interrupt's IDX, If this driver does not 3372 * support, the function will return 3373 */ 3374 3375 if (idx == VIRTIO_CONFIG_IRQ_IDX) { 3376 vhost_net_config_mask(get_vhost_net(nc->peer), vdev, mask); 3377 return; 3378 } 3379 vhost_net_virtqueue_mask(get_vhost_net(nc->peer), vdev, idx, mask); 3380 } 3381 3382 static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features) 3383 { 3384 virtio_add_feature(&host_features, VIRTIO_NET_F_MAC); 3385 3386 n->config_size = virtio_get_config_size(&cfg_size_params, host_features); 3387 } 3388 3389 void virtio_net_set_netclient_name(VirtIONet *n, const char *name, 3390 const char *type) 3391 { 3392 /* 3393 * The name can be NULL, the netclient name will be type.x. 3394 */ 3395 assert(type != NULL); 3396 3397 g_free(n->netclient_name); 3398 g_free(n->netclient_type); 3399 n->netclient_name = g_strdup(name); 3400 n->netclient_type = g_strdup(type); 3401 } 3402 3403 static bool failover_unplug_primary(VirtIONet *n, DeviceState *dev) 3404 { 3405 HotplugHandler *hotplug_ctrl; 3406 PCIDevice *pci_dev; 3407 Error *err = NULL; 3408 3409 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3410 if (hotplug_ctrl) { 3411 pci_dev = PCI_DEVICE(dev); 3412 pci_dev->partially_hotplugged = true; 3413 hotplug_handler_unplug_request(hotplug_ctrl, dev, &err); 3414 if (err) { 3415 error_report_err(err); 3416 return false; 3417 } 3418 } else { 3419 return false; 3420 } 3421 return true; 3422 } 3423 3424 static bool failover_replug_primary(VirtIONet *n, DeviceState *dev, 3425 Error **errp) 3426 { 3427 Error *err = NULL; 3428 HotplugHandler *hotplug_ctrl; 3429 PCIDevice *pdev = PCI_DEVICE(dev); 3430 BusState *primary_bus; 3431 3432 if (!pdev->partially_hotplugged) { 3433 return true; 3434 } 3435 primary_bus = dev->parent_bus; 3436 if (!primary_bus) { 3437 error_setg(errp, "virtio_net: couldn't find primary bus"); 3438 return false; 3439 } 3440 qdev_set_parent_bus(dev, primary_bus, &error_abort); 3441 qatomic_set(&n->failover_primary_hidden, false); 3442 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3443 if (hotplug_ctrl) { 3444 hotplug_handler_pre_plug(hotplug_ctrl, dev, &err); 3445 if (err) { 3446 goto out; 3447 } 3448 hotplug_handler_plug(hotplug_ctrl, dev, &err); 3449 } 3450 pdev->partially_hotplugged = false; 3451 3452 out: 3453 error_propagate(errp, err); 3454 return !err; 3455 } 3456 3457 static void virtio_net_handle_migration_primary(VirtIONet *n, MigrationState *s) 3458 { 3459 bool should_be_hidden; 3460 Error *err = NULL; 3461 DeviceState *dev = failover_find_primary_device(n); 3462 3463 if (!dev) { 3464 return; 3465 } 3466 3467 should_be_hidden = qatomic_read(&n->failover_primary_hidden); 3468 3469 if (migration_in_setup(s) && !should_be_hidden) { 3470 if (failover_unplug_primary(n, dev)) { 3471 vmstate_unregister(VMSTATE_IF(dev), qdev_get_vmsd(dev), dev); 3472 qapi_event_send_unplug_primary(dev->id); 3473 qatomic_set(&n->failover_primary_hidden, true); 3474 } else { 3475 warn_report("couldn't unplug primary device"); 3476 } 3477 } else if (migration_has_failed(s)) { 3478 /* We already unplugged the device let's plug it back */ 3479 if (!failover_replug_primary(n, dev, &err)) { 3480 if (err) { 3481 error_report_err(err); 3482 } 3483 } 3484 } 3485 } 3486 3487 static void virtio_net_migration_state_notifier(Notifier *notifier, void *data) 3488 { 3489 MigrationState *s = data; 3490 VirtIONet *n = container_of(notifier, VirtIONet, migration_state); 3491 virtio_net_handle_migration_primary(n, s); 3492 } 3493 3494 static bool failover_hide_primary_device(DeviceListener *listener, 3495 const QDict *device_opts, 3496 bool from_json, 3497 Error **errp) 3498 { 3499 VirtIONet *n = container_of(listener, VirtIONet, primary_listener); 3500 const char *standby_id; 3501 3502 if (!device_opts) { 3503 return false; 3504 } 3505 3506 if (!qdict_haskey(device_opts, "failover_pair_id")) { 3507 return false; 3508 } 3509 3510 if (!qdict_haskey(device_opts, "id")) { 3511 error_setg(errp, "Device with failover_pair_id needs to have id"); 3512 return false; 3513 } 3514 3515 standby_id = qdict_get_str(device_opts, "failover_pair_id"); 3516 if (g_strcmp0(standby_id, n->netclient_name) != 0) { 3517 return false; 3518 } 3519 3520 /* 3521 * The hide helper can be called several times for a given device. 3522 * Check there is only one primary for a virtio-net device but 3523 * don't duplicate the qdict several times if it's called for the same 3524 * device. 3525 */ 3526 if (n->primary_opts) { 3527 const char *old, *new; 3528 /* devices with failover_pair_id always have an id */ 3529 old = qdict_get_str(n->primary_opts, "id"); 3530 new = qdict_get_str(device_opts, "id"); 3531 if (strcmp(old, new) != 0) { 3532 error_setg(errp, "Cannot attach more than one primary device to " 3533 "'%s': '%s' and '%s'", n->netclient_name, old, new); 3534 return false; 3535 } 3536 } else { 3537 n->primary_opts = qdict_clone_shallow(device_opts); 3538 n->primary_opts_from_json = from_json; 3539 } 3540 3541 /* failover_primary_hidden is set during feature negotiation */ 3542 return qatomic_read(&n->failover_primary_hidden); 3543 } 3544 3545 static void virtio_net_device_realize(DeviceState *dev, Error **errp) 3546 { 3547 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3548 VirtIONet *n = VIRTIO_NET(dev); 3549 NetClientState *nc; 3550 int i; 3551 3552 if (n->net_conf.mtu) { 3553 n->host_features |= (1ULL << VIRTIO_NET_F_MTU); 3554 } 3555 3556 if (n->net_conf.duplex_str) { 3557 if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) { 3558 n->net_conf.duplex = DUPLEX_HALF; 3559 } else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) { 3560 n->net_conf.duplex = DUPLEX_FULL; 3561 } else { 3562 error_setg(errp, "'duplex' must be 'half' or 'full'"); 3563 return; 3564 } 3565 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3566 } else { 3567 n->net_conf.duplex = DUPLEX_UNKNOWN; 3568 } 3569 3570 if (n->net_conf.speed < SPEED_UNKNOWN) { 3571 error_setg(errp, "'speed' must be between 0 and INT_MAX"); 3572 return; 3573 } 3574 if (n->net_conf.speed >= 0) { 3575 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3576 } 3577 3578 if (n->failover) { 3579 n->primary_listener.hide_device = failover_hide_primary_device; 3580 qatomic_set(&n->failover_primary_hidden, true); 3581 device_listener_register(&n->primary_listener); 3582 n->migration_state.notify = virtio_net_migration_state_notifier; 3583 add_migration_state_change_notifier(&n->migration_state); 3584 n->host_features |= (1ULL << VIRTIO_NET_F_STANDBY); 3585 } 3586 3587 virtio_net_set_config_size(n, n->host_features); 3588 virtio_init(vdev, VIRTIO_ID_NET, n->config_size); 3589 3590 /* 3591 * We set a lower limit on RX queue size to what it always was. 3592 * Guests that want a smaller ring can always resize it without 3593 * help from us (using virtio 1 and up). 3594 */ 3595 if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE || 3596 n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE || 3597 !is_power_of_2(n->net_conf.rx_queue_size)) { 3598 error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), " 3599 "must be a power of 2 between %d and %d.", 3600 n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE, 3601 VIRTQUEUE_MAX_SIZE); 3602 virtio_cleanup(vdev); 3603 return; 3604 } 3605 3606 if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE || 3607 n->net_conf.tx_queue_size > VIRTQUEUE_MAX_SIZE || 3608 !is_power_of_2(n->net_conf.tx_queue_size)) { 3609 error_setg(errp, "Invalid tx_queue_size (= %" PRIu16 "), " 3610 "must be a power of 2 between %d and %d", 3611 n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE, 3612 VIRTQUEUE_MAX_SIZE); 3613 virtio_cleanup(vdev); 3614 return; 3615 } 3616 3617 n->max_ncs = MAX(n->nic_conf.peers.queues, 1); 3618 3619 /* 3620 * Figure out the datapath queue pairs since the backend could 3621 * provide control queue via peers as well. 3622 */ 3623 if (n->nic_conf.peers.queues) { 3624 for (i = 0; i < n->max_ncs; i++) { 3625 if (n->nic_conf.peers.ncs[i]->is_datapath) { 3626 ++n->max_queue_pairs; 3627 } 3628 } 3629 } 3630 n->max_queue_pairs = MAX(n->max_queue_pairs, 1); 3631 3632 if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) { 3633 error_setg(errp, "Invalid number of queue pairs (= %" PRIu32 "), " 3634 "must be a positive integer less than %d.", 3635 n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2); 3636 virtio_cleanup(vdev); 3637 return; 3638 } 3639 n->vqs = g_new0(VirtIONetQueue, n->max_queue_pairs); 3640 n->curr_queue_pairs = 1; 3641 n->tx_timeout = n->net_conf.txtimer; 3642 3643 if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer") 3644 && strcmp(n->net_conf.tx, "bh")) { 3645 warn_report("virtio-net: " 3646 "Unknown option tx=%s, valid options: \"timer\" \"bh\"", 3647 n->net_conf.tx); 3648 error_printf("Defaulting to \"bh\""); 3649 } 3650 3651 n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n), 3652 n->net_conf.tx_queue_size); 3653 3654 for (i = 0; i < n->max_queue_pairs; i++) { 3655 virtio_net_add_queue(n, i); 3656 } 3657 3658 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 3659 qemu_macaddr_default_if_unset(&n->nic_conf.macaddr); 3660 memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac)); 3661 n->status = VIRTIO_NET_S_LINK_UP; 3662 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 3663 QEMU_CLOCK_VIRTUAL, 3664 virtio_net_announce_timer, n); 3665 n->announce_timer.round = 0; 3666 3667 if (n->netclient_type) { 3668 /* 3669 * Happen when virtio_net_set_netclient_name has been called. 3670 */ 3671 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3672 n->netclient_type, n->netclient_name, n); 3673 } else { 3674 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3675 object_get_typename(OBJECT(dev)), dev->id, n); 3676 } 3677 3678 for (i = 0; i < n->max_queue_pairs; i++) { 3679 n->nic->ncs[i].do_not_pad = true; 3680 } 3681 3682 peer_test_vnet_hdr(n); 3683 if (peer_has_vnet_hdr(n)) { 3684 for (i = 0; i < n->max_queue_pairs; i++) { 3685 qemu_using_vnet_hdr(qemu_get_subqueue(n->nic, i)->peer, true); 3686 } 3687 n->host_hdr_len = sizeof(struct virtio_net_hdr); 3688 } else { 3689 n->host_hdr_len = 0; 3690 } 3691 3692 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a); 3693 3694 n->vqs[0].tx_waiting = 0; 3695 n->tx_burst = n->net_conf.txburst; 3696 virtio_net_set_mrg_rx_bufs(n, 0, 0, 0); 3697 n->promisc = 1; /* for compatibility */ 3698 3699 n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 3700 3701 n->vlans = g_malloc0(MAX_VLAN >> 3); 3702 3703 nc = qemu_get_queue(n->nic); 3704 nc->rxfilter_notify_enabled = 1; 3705 3706 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 3707 struct virtio_net_config netcfg = {}; 3708 memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN); 3709 vhost_net_set_config(get_vhost_net(nc->peer), 3710 (uint8_t *)&netcfg, 0, ETH_ALEN, VHOST_SET_CONFIG_TYPE_MASTER); 3711 } 3712 QTAILQ_INIT(&n->rsc_chains); 3713 n->qdev = dev; 3714 3715 net_rx_pkt_init(&n->rx_pkt); 3716 3717 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3718 virtio_net_load_ebpf(n); 3719 } 3720 } 3721 3722 static void virtio_net_device_unrealize(DeviceState *dev) 3723 { 3724 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3725 VirtIONet *n = VIRTIO_NET(dev); 3726 int i, max_queue_pairs; 3727 3728 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3729 virtio_net_unload_ebpf(n); 3730 } 3731 3732 /* This will stop vhost backend if appropriate. */ 3733 virtio_net_set_status(vdev, 0); 3734 3735 g_free(n->netclient_name); 3736 n->netclient_name = NULL; 3737 g_free(n->netclient_type); 3738 n->netclient_type = NULL; 3739 3740 g_free(n->mac_table.macs); 3741 g_free(n->vlans); 3742 3743 if (n->failover) { 3744 qobject_unref(n->primary_opts); 3745 device_listener_unregister(&n->primary_listener); 3746 remove_migration_state_change_notifier(&n->migration_state); 3747 } else { 3748 assert(n->primary_opts == NULL); 3749 } 3750 3751 max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 3752 for (i = 0; i < max_queue_pairs; i++) { 3753 virtio_net_del_queue(n, i); 3754 } 3755 /* delete also control vq */ 3756 virtio_del_queue(vdev, max_queue_pairs * 2); 3757 qemu_announce_timer_del(&n->announce_timer, false); 3758 g_free(n->vqs); 3759 qemu_del_nic(n->nic); 3760 virtio_net_rsc_cleanup(n); 3761 g_free(n->rss_data.indirections_table); 3762 net_rx_pkt_uninit(n->rx_pkt); 3763 virtio_cleanup(vdev); 3764 } 3765 3766 static void virtio_net_instance_init(Object *obj) 3767 { 3768 VirtIONet *n = VIRTIO_NET(obj); 3769 3770 /* 3771 * The default config_size is sizeof(struct virtio_net_config). 3772 * Can be overriden with virtio_net_set_config_size. 3773 */ 3774 n->config_size = sizeof(struct virtio_net_config); 3775 device_add_bootindex_property(obj, &n->nic_conf.bootindex, 3776 "bootindex", "/ethernet-phy@0", 3777 DEVICE(n)); 3778 3779 ebpf_rss_init(&n->ebpf_rss); 3780 } 3781 3782 static int virtio_net_pre_save(void *opaque) 3783 { 3784 VirtIONet *n = opaque; 3785 3786 /* At this point, backend must be stopped, otherwise 3787 * it might keep writing to memory. */ 3788 assert(!n->vhost_started); 3789 3790 return 0; 3791 } 3792 3793 static bool primary_unplug_pending(void *opaque) 3794 { 3795 DeviceState *dev = opaque; 3796 DeviceState *primary; 3797 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3798 VirtIONet *n = VIRTIO_NET(vdev); 3799 3800 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 3801 return false; 3802 } 3803 primary = failover_find_primary_device(n); 3804 return primary ? primary->pending_deleted_event : false; 3805 } 3806 3807 static bool dev_unplug_pending(void *opaque) 3808 { 3809 DeviceState *dev = opaque; 3810 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 3811 3812 return vdc->primary_unplug_pending(dev); 3813 } 3814 3815 static struct vhost_dev *virtio_net_get_vhost(VirtIODevice *vdev) 3816 { 3817 VirtIONet *n = VIRTIO_NET(vdev); 3818 NetClientState *nc = qemu_get_queue(n->nic); 3819 struct vhost_net *net = get_vhost_net(nc->peer); 3820 return &net->dev; 3821 } 3822 3823 static const VMStateDescription vmstate_virtio_net = { 3824 .name = "virtio-net", 3825 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3826 .version_id = VIRTIO_NET_VM_VERSION, 3827 .fields = (VMStateField[]) { 3828 VMSTATE_VIRTIO_DEVICE, 3829 VMSTATE_END_OF_LIST() 3830 }, 3831 .pre_save = virtio_net_pre_save, 3832 .dev_unplug_pending = dev_unplug_pending, 3833 }; 3834 3835 static Property virtio_net_properties[] = { 3836 DEFINE_PROP_BIT64("csum", VirtIONet, host_features, 3837 VIRTIO_NET_F_CSUM, true), 3838 DEFINE_PROP_BIT64("guest_csum", VirtIONet, host_features, 3839 VIRTIO_NET_F_GUEST_CSUM, true), 3840 DEFINE_PROP_BIT64("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true), 3841 DEFINE_PROP_BIT64("guest_tso4", VirtIONet, host_features, 3842 VIRTIO_NET_F_GUEST_TSO4, true), 3843 DEFINE_PROP_BIT64("guest_tso6", VirtIONet, host_features, 3844 VIRTIO_NET_F_GUEST_TSO6, true), 3845 DEFINE_PROP_BIT64("guest_ecn", VirtIONet, host_features, 3846 VIRTIO_NET_F_GUEST_ECN, true), 3847 DEFINE_PROP_BIT64("guest_ufo", VirtIONet, host_features, 3848 VIRTIO_NET_F_GUEST_UFO, true), 3849 DEFINE_PROP_BIT64("guest_announce", VirtIONet, host_features, 3850 VIRTIO_NET_F_GUEST_ANNOUNCE, true), 3851 DEFINE_PROP_BIT64("host_tso4", VirtIONet, host_features, 3852 VIRTIO_NET_F_HOST_TSO4, true), 3853 DEFINE_PROP_BIT64("host_tso6", VirtIONet, host_features, 3854 VIRTIO_NET_F_HOST_TSO6, true), 3855 DEFINE_PROP_BIT64("host_ecn", VirtIONet, host_features, 3856 VIRTIO_NET_F_HOST_ECN, true), 3857 DEFINE_PROP_BIT64("host_ufo", VirtIONet, host_features, 3858 VIRTIO_NET_F_HOST_UFO, true), 3859 DEFINE_PROP_BIT64("mrg_rxbuf", VirtIONet, host_features, 3860 VIRTIO_NET_F_MRG_RXBUF, true), 3861 DEFINE_PROP_BIT64("status", VirtIONet, host_features, 3862 VIRTIO_NET_F_STATUS, true), 3863 DEFINE_PROP_BIT64("ctrl_vq", VirtIONet, host_features, 3864 VIRTIO_NET_F_CTRL_VQ, true), 3865 DEFINE_PROP_BIT64("ctrl_rx", VirtIONet, host_features, 3866 VIRTIO_NET_F_CTRL_RX, true), 3867 DEFINE_PROP_BIT64("ctrl_vlan", VirtIONet, host_features, 3868 VIRTIO_NET_F_CTRL_VLAN, true), 3869 DEFINE_PROP_BIT64("ctrl_rx_extra", VirtIONet, host_features, 3870 VIRTIO_NET_F_CTRL_RX_EXTRA, true), 3871 DEFINE_PROP_BIT64("ctrl_mac_addr", VirtIONet, host_features, 3872 VIRTIO_NET_F_CTRL_MAC_ADDR, true), 3873 DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features, 3874 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true), 3875 DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false), 3876 DEFINE_PROP_BIT64("rss", VirtIONet, host_features, 3877 VIRTIO_NET_F_RSS, false), 3878 DEFINE_PROP_BIT64("hash", VirtIONet, host_features, 3879 VIRTIO_NET_F_HASH_REPORT, false), 3880 DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features, 3881 VIRTIO_NET_F_RSC_EXT, false), 3882 DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout, 3883 VIRTIO_NET_RSC_DEFAULT_INTERVAL), 3884 DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf), 3885 DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer, 3886 TX_TIMER_INTERVAL), 3887 DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST), 3888 DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx), 3889 DEFINE_PROP_UINT16("rx_queue_size", VirtIONet, net_conf.rx_queue_size, 3890 VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE), 3891 DEFINE_PROP_UINT16("tx_queue_size", VirtIONet, net_conf.tx_queue_size, 3892 VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE), 3893 DEFINE_PROP_UINT16("host_mtu", VirtIONet, net_conf.mtu, 0), 3894 DEFINE_PROP_BOOL("x-mtu-bypass-backend", VirtIONet, mtu_bypass_backend, 3895 true), 3896 DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), 3897 DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), 3898 DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), 3899 DEFINE_PROP_END_OF_LIST(), 3900 }; 3901 3902 static void virtio_net_class_init(ObjectClass *klass, void *data) 3903 { 3904 DeviceClass *dc = DEVICE_CLASS(klass); 3905 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 3906 3907 device_class_set_props(dc, virtio_net_properties); 3908 dc->vmsd = &vmstate_virtio_net; 3909 set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); 3910 vdc->realize = virtio_net_device_realize; 3911 vdc->unrealize = virtio_net_device_unrealize; 3912 vdc->get_config = virtio_net_get_config; 3913 vdc->set_config = virtio_net_set_config; 3914 vdc->get_features = virtio_net_get_features; 3915 vdc->set_features = virtio_net_set_features; 3916 vdc->bad_features = virtio_net_bad_features; 3917 vdc->reset = virtio_net_reset; 3918 vdc->queue_reset = virtio_net_queue_reset; 3919 vdc->queue_enable = virtio_net_queue_enable; 3920 vdc->set_status = virtio_net_set_status; 3921 vdc->guest_notifier_mask = virtio_net_guest_notifier_mask; 3922 vdc->guest_notifier_pending = virtio_net_guest_notifier_pending; 3923 vdc->legacy_features |= (0x1 << VIRTIO_NET_F_GSO); 3924 vdc->post_load = virtio_net_post_load_virtio; 3925 vdc->vmsd = &vmstate_virtio_net_device; 3926 vdc->primary_unplug_pending = primary_unplug_pending; 3927 vdc->get_vhost = virtio_net_get_vhost; 3928 } 3929 3930 static const TypeInfo virtio_net_info = { 3931 .name = TYPE_VIRTIO_NET, 3932 .parent = TYPE_VIRTIO_DEVICE, 3933 .instance_size = sizeof(VirtIONet), 3934 .instance_init = virtio_net_instance_init, 3935 .class_init = virtio_net_class_init, 3936 }; 3937 3938 static void virtio_register_types(void) 3939 { 3940 type_register_static(&virtio_net_info); 3941 } 3942 3943 type_init(virtio_register_types) 3944