1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* A network driver using virtio. 3 * 4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5 */ 6 //#define DEBUG 7 #include <linux/netdevice.h> 8 #include <linux/etherdevice.h> 9 #include <linux/ethtool.h> 10 #include <linux/module.h> 11 #include <linux/virtio.h> 12 #include <linux/virtio_net.h> 13 #include <linux/bpf.h> 14 #include <linux/bpf_trace.h> 15 #include <linux/scatterlist.h> 16 #include <linux/if_vlan.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/average.h> 20 #include <linux/filter.h> 21 #include <linux/kernel.h> 22 #include <net/route.h> 23 #include <net/xdp.h> 24 #include <net/net_failover.h> 25 26 static int napi_weight = NAPI_POLL_WEIGHT; 27 module_param(napi_weight, int, 0444); 28 29 static bool csum = true, gso = true, napi_tx = true; 30 module_param(csum, bool, 0444); 31 module_param(gso, bool, 0444); 32 module_param(napi_tx, bool, 0644); 33 34 /* FIXME: MTU in config. */ 35 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 36 #define GOOD_COPY_LEN 128 37 38 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 39 40 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */ 41 #define VIRTIO_XDP_HEADROOM 256 42 43 /* Separating two types of XDP xmit */ 44 #define VIRTIO_XDP_TX BIT(0) 45 #define VIRTIO_XDP_REDIR BIT(1) 46 47 #define VIRTIO_XDP_FLAG BIT(0) 48 49 /* RX packet size EWMA. The average packet size is used to determine the packet 50 * buffer size when refilling RX rings. As the entire RX ring may be refilled 51 * at once, the weight is chosen so that the EWMA will be insensitive to short- 52 * term, transient changes in packet size. 53 */ 54 DECLARE_EWMA(pkt_len, 0, 64) 55 56 #define VIRTNET_DRIVER_VERSION "1.0.0" 57 58 static const unsigned long guest_offloads[] = { 59 VIRTIO_NET_F_GUEST_TSO4, 60 VIRTIO_NET_F_GUEST_TSO6, 61 VIRTIO_NET_F_GUEST_ECN, 62 VIRTIO_NET_F_GUEST_UFO, 63 VIRTIO_NET_F_GUEST_CSUM, 64 VIRTIO_NET_F_GUEST_USO4, 65 VIRTIO_NET_F_GUEST_USO6 66 }; 67 68 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 69 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 70 (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 71 (1ULL << VIRTIO_NET_F_GUEST_UFO) | \ 72 (1ULL << VIRTIO_NET_F_GUEST_USO4) | \ 73 (1ULL << VIRTIO_NET_F_GUEST_USO6)) 74 75 struct virtnet_stat_desc { 76 char desc[ETH_GSTRING_LEN]; 77 size_t offset; 78 }; 79 80 struct virtnet_sq_stats { 81 struct u64_stats_sync syncp; 82 u64 packets; 83 u64 bytes; 84 u64 xdp_tx; 85 u64 xdp_tx_drops; 86 u64 kicks; 87 u64 tx_timeouts; 88 }; 89 90 struct virtnet_rq_stats { 91 struct u64_stats_sync syncp; 92 u64 packets; 93 u64 bytes; 94 u64 drops; 95 u64 xdp_packets; 96 u64 xdp_tx; 97 u64 xdp_redirects; 98 u64 xdp_drops; 99 u64 kicks; 100 }; 101 102 #define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m) 103 #define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m) 104 105 static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 106 { "packets", VIRTNET_SQ_STAT(packets) }, 107 { "bytes", VIRTNET_SQ_STAT(bytes) }, 108 { "xdp_tx", VIRTNET_SQ_STAT(xdp_tx) }, 109 { "xdp_tx_drops", VIRTNET_SQ_STAT(xdp_tx_drops) }, 110 { "kicks", VIRTNET_SQ_STAT(kicks) }, 111 { "tx_timeouts", VIRTNET_SQ_STAT(tx_timeouts) }, 112 }; 113 114 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 115 { "packets", VIRTNET_RQ_STAT(packets) }, 116 { "bytes", VIRTNET_RQ_STAT(bytes) }, 117 { "drops", VIRTNET_RQ_STAT(drops) }, 118 { "xdp_packets", VIRTNET_RQ_STAT(xdp_packets) }, 119 { "xdp_tx", VIRTNET_RQ_STAT(xdp_tx) }, 120 { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) }, 121 { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) }, 122 { "kicks", VIRTNET_RQ_STAT(kicks) }, 123 }; 124 125 #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc) 126 #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc) 127 128 /* Internal representation of a send virtqueue */ 129 struct send_queue { 130 /* Virtqueue associated with this send _queue */ 131 struct virtqueue *vq; 132 133 /* TX: fragments + linear part + virtio header */ 134 struct scatterlist sg[MAX_SKB_FRAGS + 2]; 135 136 /* Name of the send queue: output.$index */ 137 char name[16]; 138 139 struct virtnet_sq_stats stats; 140 141 struct napi_struct napi; 142 143 /* Record whether sq is in reset state. */ 144 bool reset; 145 }; 146 147 /* Internal representation of a receive virtqueue */ 148 struct receive_queue { 149 /* Virtqueue associated with this receive_queue */ 150 struct virtqueue *vq; 151 152 struct napi_struct napi; 153 154 struct bpf_prog __rcu *xdp_prog; 155 156 struct virtnet_rq_stats stats; 157 158 /* Chain pages by the private ptr. */ 159 struct page *pages; 160 161 /* Average packet length for mergeable receive buffers. */ 162 struct ewma_pkt_len mrg_avg_pkt_len; 163 164 /* Page frag for packet buffer allocation. */ 165 struct page_frag alloc_frag; 166 167 /* RX: fragments + linear part + virtio header */ 168 struct scatterlist sg[MAX_SKB_FRAGS + 2]; 169 170 /* Min single buffer size for mergeable buffers case. */ 171 unsigned int min_buf_len; 172 173 /* Name of this receive queue: input.$index */ 174 char name[16]; 175 176 struct xdp_rxq_info xdp_rxq; 177 }; 178 179 /* This structure can contain rss message with maximum settings for indirection table and keysize 180 * Note, that default structure that describes RSS configuration virtio_net_rss_config 181 * contains same info but can't handle table values. 182 * In any case, structure would be passed to virtio hw through sg_buf split by parts 183 * because table sizes may be differ according to the device configuration. 184 */ 185 #define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 186 #define VIRTIO_NET_RSS_MAX_TABLE_LEN 128 187 struct virtio_net_ctrl_rss { 188 u32 hash_types; 189 u16 indirection_table_mask; 190 u16 unclassified_queue; 191 u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN]; 192 u16 max_tx_vq; 193 u8 hash_key_length; 194 u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE]; 195 }; 196 197 /* Control VQ buffers: protected by the rtnl lock */ 198 struct control_buf { 199 struct virtio_net_ctrl_hdr hdr; 200 virtio_net_ctrl_ack status; 201 struct virtio_net_ctrl_mq mq; 202 u8 promisc; 203 u8 allmulti; 204 __virtio16 vid; 205 __virtio64 offloads; 206 struct virtio_net_ctrl_rss rss; 207 }; 208 209 struct virtnet_info { 210 struct virtio_device *vdev; 211 struct virtqueue *cvq; 212 struct net_device *dev; 213 struct send_queue *sq; 214 struct receive_queue *rq; 215 unsigned int status; 216 217 /* Max # of queue pairs supported by the device */ 218 u16 max_queue_pairs; 219 220 /* # of queue pairs currently used by the driver */ 221 u16 curr_queue_pairs; 222 223 /* # of XDP queue pairs currently used by the driver */ 224 u16 xdp_queue_pairs; 225 226 /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ 227 bool xdp_enabled; 228 229 /* I like... big packets and I cannot lie! */ 230 bool big_packets; 231 232 /* number of sg entries allocated for big packets */ 233 unsigned int big_packets_num_skbfrags; 234 235 /* Host will merge rx buffers for big packets (shake it! shake it!) */ 236 bool mergeable_rx_bufs; 237 238 /* Host supports rss and/or hash report */ 239 bool has_rss; 240 bool has_rss_hash_report; 241 u8 rss_key_size; 242 u16 rss_indir_table_size; 243 u32 rss_hash_types_supported; 244 u32 rss_hash_types_saved; 245 246 /* Has control virtqueue */ 247 bool has_cvq; 248 249 /* Host can handle any s/g split between our header and packet data */ 250 bool any_header_sg; 251 252 /* Packet virtio header size */ 253 u8 hdr_len; 254 255 /* Work struct for delayed refilling if we run low on memory. */ 256 struct delayed_work refill; 257 258 /* Is delayed refill enabled? */ 259 bool refill_enabled; 260 261 /* The lock to synchronize the access to refill_enabled */ 262 spinlock_t refill_lock; 263 264 /* Work struct for config space updates */ 265 struct work_struct config_work; 266 267 /* Does the affinity hint is set for virtqueues? */ 268 bool affinity_hint_set; 269 270 /* CPU hotplug instances for online & dead */ 271 struct hlist_node node; 272 struct hlist_node node_dead; 273 274 struct control_buf *ctrl; 275 276 /* Ethtool settings */ 277 u8 duplex; 278 u32 speed; 279 280 /* Interrupt coalescing settings */ 281 u32 tx_usecs; 282 u32 rx_usecs; 283 u32 tx_max_packets; 284 u32 rx_max_packets; 285 286 unsigned long guest_offloads; 287 unsigned long guest_offloads_capable; 288 289 /* failover when STANDBY feature enabled */ 290 struct failover *failover; 291 }; 292 293 struct padded_vnet_hdr { 294 struct virtio_net_hdr_v1_hash hdr; 295 /* 296 * hdr is in a separate sg buffer, and data sg buffer shares same page 297 * with this header sg. This padding makes next sg 16 byte aligned 298 * after the header. 299 */ 300 char padding[12]; 301 }; 302 303 static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf); 304 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf); 305 306 static bool is_xdp_frame(void *ptr) 307 { 308 return (unsigned long)ptr & VIRTIO_XDP_FLAG; 309 } 310 311 static void *xdp_to_ptr(struct xdp_frame *ptr) 312 { 313 return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG); 314 } 315 316 static struct xdp_frame *ptr_to_xdp(void *ptr) 317 { 318 return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); 319 } 320 321 /* Converting between virtqueue no. and kernel tx/rx queue no. 322 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 323 */ 324 static int vq2txq(struct virtqueue *vq) 325 { 326 return (vq->index - 1) / 2; 327 } 328 329 static int txq2vq(int txq) 330 { 331 return txq * 2 + 1; 332 } 333 334 static int vq2rxq(struct virtqueue *vq) 335 { 336 return vq->index / 2; 337 } 338 339 static int rxq2vq(int rxq) 340 { 341 return rxq * 2; 342 } 343 344 static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb) 345 { 346 return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb; 347 } 348 349 /* 350 * private is used to chain pages for big packets, put the whole 351 * most recent used list in the beginning for reuse 352 */ 353 static void give_pages(struct receive_queue *rq, struct page *page) 354 { 355 struct page *end; 356 357 /* Find end of list, sew whole thing into vi->rq.pages. */ 358 for (end = page; end->private; end = (struct page *)end->private); 359 end->private = (unsigned long)rq->pages; 360 rq->pages = page; 361 } 362 363 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 364 { 365 struct page *p = rq->pages; 366 367 if (p) { 368 rq->pages = (struct page *)p->private; 369 /* clear private here, it is used to chain pages */ 370 p->private = 0; 371 } else 372 p = alloc_page(gfp_mask); 373 return p; 374 } 375 376 static void enable_delayed_refill(struct virtnet_info *vi) 377 { 378 spin_lock_bh(&vi->refill_lock); 379 vi->refill_enabled = true; 380 spin_unlock_bh(&vi->refill_lock); 381 } 382 383 static void disable_delayed_refill(struct virtnet_info *vi) 384 { 385 spin_lock_bh(&vi->refill_lock); 386 vi->refill_enabled = false; 387 spin_unlock_bh(&vi->refill_lock); 388 } 389 390 static void virtqueue_napi_schedule(struct napi_struct *napi, 391 struct virtqueue *vq) 392 { 393 if (napi_schedule_prep(napi)) { 394 virtqueue_disable_cb(vq); 395 __napi_schedule(napi); 396 } 397 } 398 399 static void virtqueue_napi_complete(struct napi_struct *napi, 400 struct virtqueue *vq, int processed) 401 { 402 int opaque; 403 404 opaque = virtqueue_enable_cb_prepare(vq); 405 if (napi_complete_done(napi, processed)) { 406 if (unlikely(virtqueue_poll(vq, opaque))) 407 virtqueue_napi_schedule(napi, vq); 408 } else { 409 virtqueue_disable_cb(vq); 410 } 411 } 412 413 static void skb_xmit_done(struct virtqueue *vq) 414 { 415 struct virtnet_info *vi = vq->vdev->priv; 416 struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi; 417 418 /* Suppress further interrupts. */ 419 virtqueue_disable_cb(vq); 420 421 if (napi->weight) 422 virtqueue_napi_schedule(napi, vq); 423 else 424 /* We were probably waiting for more output buffers. */ 425 netif_wake_subqueue(vi->dev, vq2txq(vq)); 426 } 427 428 #define MRG_CTX_HEADER_SHIFT 22 429 static void *mergeable_len_to_ctx(unsigned int truesize, 430 unsigned int headroom) 431 { 432 return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 433 } 434 435 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 436 { 437 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 438 } 439 440 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 441 { 442 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 443 } 444 445 /* Called from bottom half context */ 446 static struct sk_buff *page_to_skb(struct virtnet_info *vi, 447 struct receive_queue *rq, 448 struct page *page, unsigned int offset, 449 unsigned int len, unsigned int truesize) 450 { 451 struct sk_buff *skb; 452 struct virtio_net_hdr_mrg_rxbuf *hdr; 453 unsigned int copy, hdr_len, hdr_padded_len; 454 struct page *page_to_free = NULL; 455 int tailroom, shinfo_size; 456 char *p, *hdr_p, *buf; 457 458 p = page_address(page) + offset; 459 hdr_p = p; 460 461 hdr_len = vi->hdr_len; 462 if (vi->mergeable_rx_bufs) 463 hdr_padded_len = hdr_len; 464 else 465 hdr_padded_len = sizeof(struct padded_vnet_hdr); 466 467 buf = p; 468 len -= hdr_len; 469 offset += hdr_padded_len; 470 p += hdr_padded_len; 471 tailroom = truesize - hdr_padded_len - len; 472 473 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 474 475 /* copy small packet so we can reuse these pages */ 476 if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) { 477 skb = build_skb(buf, truesize); 478 if (unlikely(!skb)) 479 return NULL; 480 481 skb_reserve(skb, p - buf); 482 skb_put(skb, len); 483 484 page = (struct page *)page->private; 485 if (page) 486 give_pages(rq, page); 487 goto ok; 488 } 489 490 /* copy small packet so we can reuse these pages for small data */ 491 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 492 if (unlikely(!skb)) 493 return NULL; 494 495 /* Copy all frame if it fits skb->head, otherwise 496 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. 497 */ 498 if (len <= skb_tailroom(skb)) 499 copy = len; 500 else 501 copy = ETH_HLEN; 502 skb_put_data(skb, p, copy); 503 504 len -= copy; 505 offset += copy; 506 507 if (vi->mergeable_rx_bufs) { 508 if (len) 509 skb_add_rx_frag(skb, 0, page, offset, len, truesize); 510 else 511 page_to_free = page; 512 goto ok; 513 } 514 515 /* 516 * Verify that we can indeed put this data into a skb. 517 * This is here to handle cases when the device erroneously 518 * tries to receive more than is possible. This is usually 519 * the case of a broken device. 520 */ 521 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { 522 net_dbg_ratelimited("%s: too much data\n", skb->dev->name); 523 dev_kfree_skb(skb); 524 return NULL; 525 } 526 BUG_ON(offset >= PAGE_SIZE); 527 while (len) { 528 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 529 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 530 frag_size, truesize); 531 len -= frag_size; 532 page = (struct page *)page->private; 533 offset = 0; 534 } 535 536 if (page) 537 give_pages(rq, page); 538 539 ok: 540 hdr = skb_vnet_hdr(skb); 541 memcpy(hdr, hdr_p, hdr_len); 542 if (page_to_free) 543 put_page(page_to_free); 544 545 return skb; 546 } 547 548 static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi) 549 { 550 unsigned int len; 551 unsigned int packets = 0; 552 unsigned int bytes = 0; 553 void *ptr; 554 555 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 556 if (likely(!is_xdp_frame(ptr))) { 557 struct sk_buff *skb = ptr; 558 559 pr_debug("Sent skb %p\n", skb); 560 561 bytes += skb->len; 562 napi_consume_skb(skb, in_napi); 563 } else { 564 struct xdp_frame *frame = ptr_to_xdp(ptr); 565 566 bytes += xdp_get_frame_len(frame); 567 xdp_return_frame(frame); 568 } 569 packets++; 570 } 571 572 /* Avoid overhead when no packets have been processed 573 * happens when called speculatively from start_xmit. 574 */ 575 if (!packets) 576 return; 577 578 u64_stats_update_begin(&sq->stats.syncp); 579 sq->stats.bytes += bytes; 580 sq->stats.packets += packets; 581 u64_stats_update_end(&sq->stats.syncp); 582 } 583 584 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 585 { 586 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 587 return false; 588 else if (q < vi->curr_queue_pairs) 589 return true; 590 else 591 return false; 592 } 593 594 static void check_sq_full_and_disable(struct virtnet_info *vi, 595 struct net_device *dev, 596 struct send_queue *sq) 597 { 598 bool use_napi = sq->napi.weight; 599 int qnum; 600 601 qnum = sq - vi->sq; 602 603 /* If running out of space, stop queue to avoid getting packets that we 604 * are then unable to transmit. 605 * An alternative would be to force queuing layer to requeue the skb by 606 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 607 * returned in a normal path of operation: it means that driver is not 608 * maintaining the TX queue stop/start state properly, and causes 609 * the stack to do a non-trivial amount of useless work. 610 * Since most packets only take 1 or 2 ring slots, stopping the queue 611 * early means 16 slots are typically wasted. 612 */ 613 if (sq->vq->num_free < 2+MAX_SKB_FRAGS) { 614 netif_stop_subqueue(dev, qnum); 615 if (use_napi) { 616 if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) 617 virtqueue_napi_schedule(&sq->napi, sq->vq); 618 } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 619 /* More just got used, free them then recheck. */ 620 free_old_xmit_skbs(sq, false); 621 if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { 622 netif_start_subqueue(dev, qnum); 623 virtqueue_disable_cb(sq->vq); 624 } 625 } 626 } 627 } 628 629 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 630 struct send_queue *sq, 631 struct xdp_frame *xdpf) 632 { 633 struct virtio_net_hdr_mrg_rxbuf *hdr; 634 struct skb_shared_info *shinfo; 635 u8 nr_frags = 0; 636 int err, i; 637 638 if (unlikely(xdpf->headroom < vi->hdr_len)) 639 return -EOVERFLOW; 640 641 if (unlikely(xdp_frame_has_frags(xdpf))) { 642 shinfo = xdp_get_shared_info_from_frame(xdpf); 643 nr_frags = shinfo->nr_frags; 644 } 645 646 /* In wrapping function virtnet_xdp_xmit(), we need to free 647 * up the pending old buffers, where we need to calculate the 648 * position of skb_shared_info in xdp_get_frame_len() and 649 * xdp_return_frame(), which will involve to xdpf->data and 650 * xdpf->headroom. Therefore, we need to update the value of 651 * headroom synchronously here. 652 */ 653 xdpf->headroom -= vi->hdr_len; 654 xdpf->data -= vi->hdr_len; 655 /* Zero header and leave csum up to XDP layers */ 656 hdr = xdpf->data; 657 memset(hdr, 0, vi->hdr_len); 658 xdpf->len += vi->hdr_len; 659 660 sg_init_table(sq->sg, nr_frags + 1); 661 sg_set_buf(sq->sg, xdpf->data, xdpf->len); 662 for (i = 0; i < nr_frags; i++) { 663 skb_frag_t *frag = &shinfo->frags[i]; 664 665 sg_set_page(&sq->sg[i + 1], skb_frag_page(frag), 666 skb_frag_size(frag), skb_frag_off(frag)); 667 } 668 669 err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1, 670 xdp_to_ptr(xdpf), GFP_ATOMIC); 671 if (unlikely(err)) 672 return -ENOSPC; /* Caller handle free/refcnt */ 673 674 return 0; 675 } 676 677 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on 678 * the current cpu, so it does not need to be locked. 679 * 680 * Here we use marco instead of inline functions because we have to deal with 681 * three issues at the same time: 1. the choice of sq. 2. judge and execute the 682 * lock/unlock of txq 3. make sparse happy. It is difficult for two inline 683 * functions to perfectly solve these three problems at the same time. 684 */ 685 #define virtnet_xdp_get_sq(vi) ({ \ 686 int cpu = smp_processor_id(); \ 687 struct netdev_queue *txq; \ 688 typeof(vi) v = (vi); \ 689 unsigned int qp; \ 690 \ 691 if (v->curr_queue_pairs > nr_cpu_ids) { \ 692 qp = v->curr_queue_pairs - v->xdp_queue_pairs; \ 693 qp += cpu; \ 694 txq = netdev_get_tx_queue(v->dev, qp); \ 695 __netif_tx_acquire(txq); \ 696 } else { \ 697 qp = cpu % v->curr_queue_pairs; \ 698 txq = netdev_get_tx_queue(v->dev, qp); \ 699 __netif_tx_lock(txq, cpu); \ 700 } \ 701 v->sq + qp; \ 702 }) 703 704 #define virtnet_xdp_put_sq(vi, q) { \ 705 struct netdev_queue *txq; \ 706 typeof(vi) v = (vi); \ 707 \ 708 txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \ 709 if (v->curr_queue_pairs > nr_cpu_ids) \ 710 __netif_tx_release(txq); \ 711 else \ 712 __netif_tx_unlock(txq); \ 713 } 714 715 static int virtnet_xdp_xmit(struct net_device *dev, 716 int n, struct xdp_frame **frames, u32 flags) 717 { 718 struct virtnet_info *vi = netdev_priv(dev); 719 struct receive_queue *rq = vi->rq; 720 struct bpf_prog *xdp_prog; 721 struct send_queue *sq; 722 unsigned int len; 723 int packets = 0; 724 int bytes = 0; 725 int nxmit = 0; 726 int kicks = 0; 727 void *ptr; 728 int ret; 729 int i; 730 731 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 732 * indicate XDP resources have been successfully allocated. 733 */ 734 xdp_prog = rcu_access_pointer(rq->xdp_prog); 735 if (!xdp_prog) 736 return -ENXIO; 737 738 sq = virtnet_xdp_get_sq(vi); 739 740 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 741 ret = -EINVAL; 742 goto out; 743 } 744 745 /* Free up any pending old buffers before queueing new ones. */ 746 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 747 if (likely(is_xdp_frame(ptr))) { 748 struct xdp_frame *frame = ptr_to_xdp(ptr); 749 750 bytes += xdp_get_frame_len(frame); 751 xdp_return_frame(frame); 752 } else { 753 struct sk_buff *skb = ptr; 754 755 bytes += skb->len; 756 napi_consume_skb(skb, false); 757 } 758 packets++; 759 } 760 761 for (i = 0; i < n; i++) { 762 struct xdp_frame *xdpf = frames[i]; 763 764 if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 765 break; 766 nxmit++; 767 } 768 ret = nxmit; 769 770 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) 771 check_sq_full_and_disable(vi, dev, sq); 772 773 if (flags & XDP_XMIT_FLUSH) { 774 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 775 kicks = 1; 776 } 777 out: 778 u64_stats_update_begin(&sq->stats.syncp); 779 sq->stats.bytes += bytes; 780 sq->stats.packets += packets; 781 sq->stats.xdp_tx += n; 782 sq->stats.xdp_tx_drops += n - nxmit; 783 sq->stats.kicks += kicks; 784 u64_stats_update_end(&sq->stats.syncp); 785 786 virtnet_xdp_put_sq(vi, sq); 787 return ret; 788 } 789 790 static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 791 { 792 return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0; 793 } 794 795 /* We copy the packet for XDP in the following cases: 796 * 797 * 1) Packet is scattered across multiple rx buffers. 798 * 2) Headroom space is insufficient. 799 * 800 * This is inefficient but it's a temporary condition that 801 * we hit right after XDP is enabled and until queue is refilled 802 * with large buffers with sufficient headroom - so it should affect 803 * at most queue size packets. 804 * Afterwards, the conditions to enable 805 * XDP should preclude the underlying device from sending packets 806 * across multiple buffers (num_buf > 1), and we make sure buffers 807 * have enough headroom. 808 */ 809 static struct page *xdp_linearize_page(struct receive_queue *rq, 810 int *num_buf, 811 struct page *p, 812 int offset, 813 int page_off, 814 unsigned int *len) 815 { 816 struct page *page = alloc_page(GFP_ATOMIC); 817 818 if (!page) 819 return NULL; 820 821 memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 822 page_off += *len; 823 824 while (--*num_buf) { 825 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 826 unsigned int buflen; 827 void *buf; 828 int off; 829 830 buf = virtqueue_get_buf(rq->vq, &buflen); 831 if (unlikely(!buf)) 832 goto err_buf; 833 834 p = virt_to_head_page(buf); 835 off = buf - page_address(p); 836 837 /* guard against a misconfigured or uncooperative backend that 838 * is sending packet larger than the MTU. 839 */ 840 if ((page_off + buflen + tailroom) > PAGE_SIZE) { 841 put_page(p); 842 goto err_buf; 843 } 844 845 memcpy(page_address(page) + page_off, 846 page_address(p) + off, buflen); 847 page_off += buflen; 848 put_page(p); 849 } 850 851 /* Headroom does not contribute to packet length */ 852 *len = page_off - VIRTIO_XDP_HEADROOM; 853 return page; 854 err_buf: 855 __free_pages(page, 0); 856 return NULL; 857 } 858 859 static struct sk_buff *receive_small(struct net_device *dev, 860 struct virtnet_info *vi, 861 struct receive_queue *rq, 862 void *buf, void *ctx, 863 unsigned int len, 864 unsigned int *xdp_xmit, 865 struct virtnet_rq_stats *stats) 866 { 867 struct sk_buff *skb; 868 struct bpf_prog *xdp_prog; 869 unsigned int xdp_headroom = (unsigned long)ctx; 870 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 871 unsigned int headroom = vi->hdr_len + header_offset; 872 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 873 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 874 struct page *page = virt_to_head_page(buf); 875 unsigned int delta = 0; 876 struct page *xdp_page; 877 int err; 878 unsigned int metasize = 0; 879 880 len -= vi->hdr_len; 881 stats->bytes += len; 882 883 if (unlikely(len > GOOD_PACKET_LEN)) { 884 pr_debug("%s: rx error: len %u exceeds max size %d\n", 885 dev->name, len, GOOD_PACKET_LEN); 886 dev->stats.rx_length_errors++; 887 goto err; 888 } 889 890 if (likely(!vi->xdp_enabled)) { 891 xdp_prog = NULL; 892 goto skip_xdp; 893 } 894 895 rcu_read_lock(); 896 xdp_prog = rcu_dereference(rq->xdp_prog); 897 if (xdp_prog) { 898 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 899 struct xdp_frame *xdpf; 900 struct xdp_buff xdp; 901 void *orig_data; 902 u32 act; 903 904 if (unlikely(hdr->hdr.gso_type)) 905 goto err_xdp; 906 907 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 908 int offset = buf - page_address(page) + header_offset; 909 unsigned int tlen = len + vi->hdr_len; 910 int num_buf = 1; 911 912 xdp_headroom = virtnet_get_headroom(vi); 913 header_offset = VIRTNET_RX_PAD + xdp_headroom; 914 headroom = vi->hdr_len + header_offset; 915 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 916 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 917 xdp_page = xdp_linearize_page(rq, &num_buf, page, 918 offset, header_offset, 919 &tlen); 920 if (!xdp_page) 921 goto err_xdp; 922 923 buf = page_address(xdp_page); 924 put_page(page); 925 page = xdp_page; 926 } 927 928 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq); 929 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, 930 xdp_headroom, len, true); 931 orig_data = xdp.data; 932 act = bpf_prog_run_xdp(xdp_prog, &xdp); 933 stats->xdp_packets++; 934 935 switch (act) { 936 case XDP_PASS: 937 /* Recalculate length in case bpf program changed it */ 938 delta = orig_data - xdp.data; 939 len = xdp.data_end - xdp.data; 940 metasize = xdp.data - xdp.data_meta; 941 break; 942 case XDP_TX: 943 stats->xdp_tx++; 944 xdpf = xdp_convert_buff_to_frame(&xdp); 945 if (unlikely(!xdpf)) 946 goto err_xdp; 947 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 948 if (unlikely(!err)) { 949 xdp_return_frame_rx_napi(xdpf); 950 } else if (unlikely(err < 0)) { 951 trace_xdp_exception(vi->dev, xdp_prog, act); 952 goto err_xdp; 953 } 954 *xdp_xmit |= VIRTIO_XDP_TX; 955 rcu_read_unlock(); 956 goto xdp_xmit; 957 case XDP_REDIRECT: 958 stats->xdp_redirects++; 959 err = xdp_do_redirect(dev, &xdp, xdp_prog); 960 if (err) 961 goto err_xdp; 962 *xdp_xmit |= VIRTIO_XDP_REDIR; 963 rcu_read_unlock(); 964 goto xdp_xmit; 965 default: 966 bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act); 967 fallthrough; 968 case XDP_ABORTED: 969 trace_xdp_exception(vi->dev, xdp_prog, act); 970 goto err_xdp; 971 case XDP_DROP: 972 goto err_xdp; 973 } 974 } 975 rcu_read_unlock(); 976 977 skip_xdp: 978 skb = build_skb(buf, buflen); 979 if (!skb) 980 goto err; 981 skb_reserve(skb, headroom - delta); 982 skb_put(skb, len); 983 if (!xdp_prog) { 984 buf += header_offset; 985 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); 986 } /* keep zeroed vnet hdr since XDP is loaded */ 987 988 if (metasize) 989 skb_metadata_set(skb, metasize); 990 991 return skb; 992 993 err_xdp: 994 rcu_read_unlock(); 995 stats->xdp_drops++; 996 err: 997 stats->drops++; 998 put_page(page); 999 xdp_xmit: 1000 return NULL; 1001 } 1002 1003 static struct sk_buff *receive_big(struct net_device *dev, 1004 struct virtnet_info *vi, 1005 struct receive_queue *rq, 1006 void *buf, 1007 unsigned int len, 1008 struct virtnet_rq_stats *stats) 1009 { 1010 struct page *page = buf; 1011 struct sk_buff *skb = 1012 page_to_skb(vi, rq, page, 0, len, PAGE_SIZE); 1013 1014 stats->bytes += len - vi->hdr_len; 1015 if (unlikely(!skb)) 1016 goto err; 1017 1018 return skb; 1019 1020 err: 1021 stats->drops++; 1022 give_pages(rq, page); 1023 return NULL; 1024 } 1025 1026 /* Why not use xdp_build_skb_from_frame() ? 1027 * XDP core assumes that xdp frags are PAGE_SIZE in length, while in 1028 * virtio-net there are 2 points that do not match its requirements: 1029 * 1. The size of the prefilled buffer is not fixed before xdp is set. 1030 * 2. xdp_build_skb_from_frame() does more checks that we don't need, 1031 * like eth_type_trans() (which virtio-net does in receive_buf()). 1032 */ 1033 static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev, 1034 struct virtnet_info *vi, 1035 struct xdp_buff *xdp, 1036 unsigned int xdp_frags_truesz) 1037 { 1038 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 1039 unsigned int headroom, data_len; 1040 struct sk_buff *skb; 1041 int metasize; 1042 u8 nr_frags; 1043 1044 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { 1045 pr_debug("Error building skb as missing reserved tailroom for xdp"); 1046 return NULL; 1047 } 1048 1049 if (unlikely(xdp_buff_has_frags(xdp))) 1050 nr_frags = sinfo->nr_frags; 1051 1052 skb = build_skb(xdp->data_hard_start, xdp->frame_sz); 1053 if (unlikely(!skb)) 1054 return NULL; 1055 1056 headroom = xdp->data - xdp->data_hard_start; 1057 data_len = xdp->data_end - xdp->data; 1058 skb_reserve(skb, headroom); 1059 __skb_put(skb, data_len); 1060 1061 metasize = xdp->data - xdp->data_meta; 1062 metasize = metasize > 0 ? metasize : 0; 1063 if (metasize) 1064 skb_metadata_set(skb, metasize); 1065 1066 if (unlikely(xdp_buff_has_frags(xdp))) 1067 xdp_update_skb_shared_info(skb, nr_frags, 1068 sinfo->xdp_frags_size, 1069 xdp_frags_truesz, 1070 xdp_buff_is_frag_pfmemalloc(xdp)); 1071 1072 return skb; 1073 } 1074 1075 /* TODO: build xdp in big mode */ 1076 static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 1077 struct virtnet_info *vi, 1078 struct receive_queue *rq, 1079 struct xdp_buff *xdp, 1080 void *buf, 1081 unsigned int len, 1082 unsigned int frame_sz, 1083 int *num_buf, 1084 unsigned int *xdp_frags_truesize, 1085 struct virtnet_rq_stats *stats) 1086 { 1087 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1088 unsigned int headroom, tailroom, room; 1089 unsigned int truesize, cur_frag_size; 1090 struct skb_shared_info *shinfo; 1091 unsigned int xdp_frags_truesz = 0; 1092 struct page *page; 1093 skb_frag_t *frag; 1094 int offset; 1095 void *ctx; 1096 1097 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 1098 xdp_prepare_buff(xdp, buf - VIRTIO_XDP_HEADROOM, 1099 VIRTIO_XDP_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); 1100 1101 if (!*num_buf) 1102 return 0; 1103 1104 if (*num_buf > 1) { 1105 /* If we want to build multi-buffer xdp, we need 1106 * to specify that the flags of xdp_buff have the 1107 * XDP_FLAGS_HAS_FRAG bit. 1108 */ 1109 if (!xdp_buff_has_frags(xdp)) 1110 xdp_buff_set_frags_flag(xdp); 1111 1112 shinfo = xdp_get_shared_info_from_buff(xdp); 1113 shinfo->nr_frags = 0; 1114 shinfo->xdp_frags_size = 0; 1115 } 1116 1117 if (*num_buf > MAX_SKB_FRAGS + 1) 1118 return -EINVAL; 1119 1120 while (--*num_buf > 0) { 1121 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 1122 if (unlikely(!buf)) { 1123 pr_debug("%s: rx error: %d buffers out of %d missing\n", 1124 dev->name, *num_buf, 1125 virtio16_to_cpu(vi->vdev, hdr->num_buffers)); 1126 dev->stats.rx_length_errors++; 1127 return -EINVAL; 1128 } 1129 1130 stats->bytes += len; 1131 page = virt_to_head_page(buf); 1132 offset = buf - page_address(page); 1133 1134 truesize = mergeable_ctx_to_truesize(ctx); 1135 headroom = mergeable_ctx_to_headroom(ctx); 1136 tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1137 room = SKB_DATA_ALIGN(headroom + tailroom); 1138 1139 cur_frag_size = truesize; 1140 xdp_frags_truesz += cur_frag_size; 1141 if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) { 1142 put_page(page); 1143 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1144 dev->name, len, (unsigned long)(truesize - room)); 1145 dev->stats.rx_length_errors++; 1146 return -EINVAL; 1147 } 1148 1149 frag = &shinfo->frags[shinfo->nr_frags++]; 1150 __skb_frag_set_page(frag, page); 1151 skb_frag_off_set(frag, offset); 1152 skb_frag_size_set(frag, len); 1153 if (page_is_pfmemalloc(page)) 1154 xdp_buff_set_frag_pfmemalloc(xdp); 1155 1156 shinfo->xdp_frags_size += len; 1157 } 1158 1159 *xdp_frags_truesize = xdp_frags_truesz; 1160 return 0; 1161 } 1162 1163 static struct sk_buff *receive_mergeable(struct net_device *dev, 1164 struct virtnet_info *vi, 1165 struct receive_queue *rq, 1166 void *buf, 1167 void *ctx, 1168 unsigned int len, 1169 unsigned int *xdp_xmit, 1170 struct virtnet_rq_stats *stats) 1171 { 1172 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1173 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 1174 struct page *page = virt_to_head_page(buf); 1175 int offset = buf - page_address(page); 1176 struct sk_buff *head_skb, *curr_skb; 1177 struct bpf_prog *xdp_prog; 1178 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 1179 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 1180 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1181 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1182 unsigned int frame_sz, xdp_room; 1183 int err; 1184 1185 head_skb = NULL; 1186 stats->bytes += len - vi->hdr_len; 1187 1188 if (unlikely(len > truesize - room)) { 1189 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1190 dev->name, len, (unsigned long)(truesize - room)); 1191 dev->stats.rx_length_errors++; 1192 goto err_skb; 1193 } 1194 1195 if (likely(!vi->xdp_enabled)) { 1196 xdp_prog = NULL; 1197 goto skip_xdp; 1198 } 1199 1200 rcu_read_lock(); 1201 xdp_prog = rcu_dereference(rq->xdp_prog); 1202 if (xdp_prog) { 1203 unsigned int xdp_frags_truesz = 0; 1204 struct skb_shared_info *shinfo; 1205 struct xdp_frame *xdpf; 1206 struct page *xdp_page; 1207 struct xdp_buff xdp; 1208 void *data; 1209 u32 act; 1210 int i; 1211 1212 /* Transient failure which in theory could occur if 1213 * in-flight packets from before XDP was enabled reach 1214 * the receive path after XDP is loaded. 1215 */ 1216 if (unlikely(hdr->hdr.gso_type)) 1217 goto err_xdp; 1218 1219 /* Now XDP core assumes frag size is PAGE_SIZE, but buffers 1220 * with headroom may add hole in truesize, which 1221 * make their length exceed PAGE_SIZE. So we disabled the 1222 * hole mechanism for xdp. See add_recvbuf_mergeable(). 1223 */ 1224 frame_sz = truesize; 1225 1226 /* This happens when headroom is not enough because 1227 * of the buffer was prefilled before XDP is set. 1228 * This should only happen for the first several packets. 1229 * In fact, vq reset can be used here to help us clean up 1230 * the prefilled buffers, but many existing devices do not 1231 * support it, and we don't want to bother users who are 1232 * using xdp normally. 1233 */ 1234 if (!xdp_prog->aux->xdp_has_frags && 1235 (num_buf > 1 || headroom < virtnet_get_headroom(vi))) { 1236 /* linearize data for XDP */ 1237 xdp_page = xdp_linearize_page(rq, &num_buf, 1238 page, offset, 1239 VIRTIO_XDP_HEADROOM, 1240 &len); 1241 frame_sz = PAGE_SIZE; 1242 1243 if (!xdp_page) 1244 goto err_xdp; 1245 offset = VIRTIO_XDP_HEADROOM; 1246 } else if (unlikely(headroom < virtnet_get_headroom(vi))) { 1247 xdp_room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM + 1248 sizeof(struct skb_shared_info)); 1249 if (len + xdp_room > PAGE_SIZE) 1250 goto err_xdp; 1251 1252 xdp_page = alloc_page(GFP_ATOMIC); 1253 if (!xdp_page) 1254 goto err_xdp; 1255 1256 memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM, 1257 page_address(page) + offset, len); 1258 frame_sz = PAGE_SIZE; 1259 offset = VIRTIO_XDP_HEADROOM; 1260 } else { 1261 xdp_page = page; 1262 } 1263 1264 data = page_address(xdp_page) + offset; 1265 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz, 1266 &num_buf, &xdp_frags_truesz, stats); 1267 if (unlikely(err)) 1268 goto err_xdp_frags; 1269 1270 act = bpf_prog_run_xdp(xdp_prog, &xdp); 1271 stats->xdp_packets++; 1272 1273 switch (act) { 1274 case XDP_PASS: 1275 if (unlikely(xdp_page != page)) 1276 put_page(page); 1277 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz); 1278 rcu_read_unlock(); 1279 return head_skb; 1280 case XDP_TX: 1281 stats->xdp_tx++; 1282 xdpf = xdp_convert_buff_to_frame(&xdp); 1283 if (unlikely(!xdpf)) { 1284 netdev_dbg(dev, "convert buff to frame failed for xdp\n"); 1285 goto err_xdp_frags; 1286 } 1287 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 1288 if (unlikely(!err)) { 1289 xdp_return_frame_rx_napi(xdpf); 1290 } else if (unlikely(err < 0)) { 1291 trace_xdp_exception(vi->dev, xdp_prog, act); 1292 goto err_xdp_frags; 1293 } 1294 *xdp_xmit |= VIRTIO_XDP_TX; 1295 if (unlikely(xdp_page != page)) 1296 put_page(page); 1297 rcu_read_unlock(); 1298 goto xdp_xmit; 1299 case XDP_REDIRECT: 1300 stats->xdp_redirects++; 1301 err = xdp_do_redirect(dev, &xdp, xdp_prog); 1302 if (err) 1303 goto err_xdp_frags; 1304 *xdp_xmit |= VIRTIO_XDP_REDIR; 1305 if (unlikely(xdp_page != page)) 1306 put_page(page); 1307 rcu_read_unlock(); 1308 goto xdp_xmit; 1309 default: 1310 bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act); 1311 fallthrough; 1312 case XDP_ABORTED: 1313 trace_xdp_exception(vi->dev, xdp_prog, act); 1314 fallthrough; 1315 case XDP_DROP: 1316 goto err_xdp_frags; 1317 } 1318 err_xdp_frags: 1319 if (unlikely(xdp_page != page)) 1320 __free_pages(xdp_page, 0); 1321 1322 if (xdp_buff_has_frags(&xdp)) { 1323 shinfo = xdp_get_shared_info_from_buff(&xdp); 1324 for (i = 0; i < shinfo->nr_frags; i++) { 1325 xdp_page = skb_frag_page(&shinfo->frags[i]); 1326 put_page(xdp_page); 1327 } 1328 } 1329 1330 goto err_xdp; 1331 } 1332 rcu_read_unlock(); 1333 1334 skip_xdp: 1335 head_skb = page_to_skb(vi, rq, page, offset, len, truesize); 1336 curr_skb = head_skb; 1337 1338 if (unlikely(!curr_skb)) 1339 goto err_skb; 1340 while (--num_buf) { 1341 int num_skb_frags; 1342 1343 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 1344 if (unlikely(!buf)) { 1345 pr_debug("%s: rx error: %d buffers out of %d missing\n", 1346 dev->name, num_buf, 1347 virtio16_to_cpu(vi->vdev, 1348 hdr->num_buffers)); 1349 dev->stats.rx_length_errors++; 1350 goto err_buf; 1351 } 1352 1353 stats->bytes += len; 1354 page = virt_to_head_page(buf); 1355 1356 truesize = mergeable_ctx_to_truesize(ctx); 1357 headroom = mergeable_ctx_to_headroom(ctx); 1358 tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1359 room = SKB_DATA_ALIGN(headroom + tailroom); 1360 if (unlikely(len > truesize - room)) { 1361 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1362 dev->name, len, (unsigned long)(truesize - room)); 1363 dev->stats.rx_length_errors++; 1364 goto err_skb; 1365 } 1366 1367 num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 1368 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 1369 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 1370 1371 if (unlikely(!nskb)) 1372 goto err_skb; 1373 if (curr_skb == head_skb) 1374 skb_shinfo(curr_skb)->frag_list = nskb; 1375 else 1376 curr_skb->next = nskb; 1377 curr_skb = nskb; 1378 head_skb->truesize += nskb->truesize; 1379 num_skb_frags = 0; 1380 } 1381 if (curr_skb != head_skb) { 1382 head_skb->data_len += len; 1383 head_skb->len += len; 1384 head_skb->truesize += truesize; 1385 } 1386 offset = buf - page_address(page); 1387 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 1388 put_page(page); 1389 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 1390 len, truesize); 1391 } else { 1392 skb_add_rx_frag(curr_skb, num_skb_frags, page, 1393 offset, len, truesize); 1394 } 1395 } 1396 1397 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 1398 return head_skb; 1399 1400 err_xdp: 1401 rcu_read_unlock(); 1402 stats->xdp_drops++; 1403 err_skb: 1404 put_page(page); 1405 while (num_buf-- > 1) { 1406 buf = virtqueue_get_buf(rq->vq, &len); 1407 if (unlikely(!buf)) { 1408 pr_debug("%s: rx error: %d buffers missing\n", 1409 dev->name, num_buf); 1410 dev->stats.rx_length_errors++; 1411 break; 1412 } 1413 stats->bytes += len; 1414 page = virt_to_head_page(buf); 1415 put_page(page); 1416 } 1417 err_buf: 1418 stats->drops++; 1419 dev_kfree_skb(head_skb); 1420 xdp_xmit: 1421 return NULL; 1422 } 1423 1424 static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash, 1425 struct sk_buff *skb) 1426 { 1427 enum pkt_hash_types rss_hash_type; 1428 1429 if (!hdr_hash || !skb) 1430 return; 1431 1432 switch (__le16_to_cpu(hdr_hash->hash_report)) { 1433 case VIRTIO_NET_HASH_REPORT_TCPv4: 1434 case VIRTIO_NET_HASH_REPORT_UDPv4: 1435 case VIRTIO_NET_HASH_REPORT_TCPv6: 1436 case VIRTIO_NET_HASH_REPORT_UDPv6: 1437 case VIRTIO_NET_HASH_REPORT_TCPv6_EX: 1438 case VIRTIO_NET_HASH_REPORT_UDPv6_EX: 1439 rss_hash_type = PKT_HASH_TYPE_L4; 1440 break; 1441 case VIRTIO_NET_HASH_REPORT_IPv4: 1442 case VIRTIO_NET_HASH_REPORT_IPv6: 1443 case VIRTIO_NET_HASH_REPORT_IPv6_EX: 1444 rss_hash_type = PKT_HASH_TYPE_L3; 1445 break; 1446 case VIRTIO_NET_HASH_REPORT_NONE: 1447 default: 1448 rss_hash_type = PKT_HASH_TYPE_NONE; 1449 } 1450 skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type); 1451 } 1452 1453 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 1454 void *buf, unsigned int len, void **ctx, 1455 unsigned int *xdp_xmit, 1456 struct virtnet_rq_stats *stats) 1457 { 1458 struct net_device *dev = vi->dev; 1459 struct sk_buff *skb; 1460 struct virtio_net_hdr_mrg_rxbuf *hdr; 1461 1462 if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 1463 pr_debug("%s: short packet %i\n", dev->name, len); 1464 dev->stats.rx_length_errors++; 1465 virtnet_rq_free_unused_buf(rq->vq, buf); 1466 return; 1467 } 1468 1469 if (vi->mergeable_rx_bufs) 1470 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, 1471 stats); 1472 else if (vi->big_packets) 1473 skb = receive_big(dev, vi, rq, buf, len, stats); 1474 else 1475 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); 1476 1477 if (unlikely(!skb)) 1478 return; 1479 1480 hdr = skb_vnet_hdr(skb); 1481 if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report) 1482 virtio_skb_set_hash((const struct virtio_net_hdr_v1_hash *)hdr, skb); 1483 1484 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) 1485 skb->ip_summed = CHECKSUM_UNNECESSARY; 1486 1487 if (virtio_net_hdr_to_skb(skb, &hdr->hdr, 1488 virtio_is_little_endian(vi->vdev))) { 1489 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n", 1490 dev->name, hdr->hdr.gso_type, 1491 hdr->hdr.gso_size); 1492 goto frame_err; 1493 } 1494 1495 skb_record_rx_queue(skb, vq2rxq(rq->vq)); 1496 skb->protocol = eth_type_trans(skb, dev); 1497 pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 1498 ntohs(skb->protocol), skb->len, skb->pkt_type); 1499 1500 napi_gro_receive(&rq->napi, skb); 1501 return; 1502 1503 frame_err: 1504 dev->stats.rx_frame_errors++; 1505 dev_kfree_skb(skb); 1506 } 1507 1508 /* Unlike mergeable buffers, all buffers are allocated to the 1509 * same size, except for the headroom. For this reason we do 1510 * not need to use mergeable_len_to_ctx here - it is enough 1511 * to store the headroom as the context ignoring the truesize. 1512 */ 1513 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 1514 gfp_t gfp) 1515 { 1516 struct page_frag *alloc_frag = &rq->alloc_frag; 1517 char *buf; 1518 unsigned int xdp_headroom = virtnet_get_headroom(vi); 1519 void *ctx = (void *)(unsigned long)xdp_headroom; 1520 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 1521 int err; 1522 1523 len = SKB_DATA_ALIGN(len) + 1524 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1525 if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp))) 1526 return -ENOMEM; 1527 1528 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1529 get_page(alloc_frag->page); 1530 alloc_frag->offset += len; 1531 sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom, 1532 vi->hdr_len + GOOD_PACKET_LEN); 1533 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 1534 if (err < 0) 1535 put_page(virt_to_head_page(buf)); 1536 return err; 1537 } 1538 1539 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 1540 gfp_t gfp) 1541 { 1542 struct page *first, *list = NULL; 1543 char *p; 1544 int i, err, offset; 1545 1546 sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2); 1547 1548 /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */ 1549 for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) { 1550 first = get_a_page(rq, gfp); 1551 if (!first) { 1552 if (list) 1553 give_pages(rq, list); 1554 return -ENOMEM; 1555 } 1556 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 1557 1558 /* chain new page in list head to match sg */ 1559 first->private = (unsigned long)list; 1560 list = first; 1561 } 1562 1563 first = get_a_page(rq, gfp); 1564 if (!first) { 1565 give_pages(rq, list); 1566 return -ENOMEM; 1567 } 1568 p = page_address(first); 1569 1570 /* rq->sg[0], rq->sg[1] share the same page */ 1571 /* a separated rq->sg[0] for header - required in case !any_header_sg */ 1572 sg_set_buf(&rq->sg[0], p, vi->hdr_len); 1573 1574 /* rq->sg[1] for data packet, from offset */ 1575 offset = sizeof(struct padded_vnet_hdr); 1576 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 1577 1578 /* chain first in list head */ 1579 first->private = (unsigned long)list; 1580 err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2, 1581 first, gfp); 1582 if (err < 0) 1583 give_pages(rq, first); 1584 1585 return err; 1586 } 1587 1588 static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 1589 struct ewma_pkt_len *avg_pkt_len, 1590 unsigned int room) 1591 { 1592 struct virtnet_info *vi = rq->vq->vdev->priv; 1593 const size_t hdr_len = vi->hdr_len; 1594 unsigned int len; 1595 1596 if (room) 1597 return PAGE_SIZE - room; 1598 1599 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 1600 rq->min_buf_len, PAGE_SIZE - hdr_len); 1601 1602 return ALIGN(len, L1_CACHE_BYTES); 1603 } 1604 1605 static int add_recvbuf_mergeable(struct virtnet_info *vi, 1606 struct receive_queue *rq, gfp_t gfp) 1607 { 1608 struct page_frag *alloc_frag = &rq->alloc_frag; 1609 unsigned int headroom = virtnet_get_headroom(vi); 1610 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1611 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1612 char *buf; 1613 void *ctx; 1614 int err; 1615 unsigned int len, hole; 1616 1617 /* Extra tailroom is needed to satisfy XDP's assumption. This 1618 * means rx frags coalescing won't work, but consider we've 1619 * disabled GSO for XDP, it won't be a big issue. 1620 */ 1621 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 1622 if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp))) 1623 return -ENOMEM; 1624 1625 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1626 buf += headroom; /* advance address leaving hole at front of pkt */ 1627 get_page(alloc_frag->page); 1628 alloc_frag->offset += len + room; 1629 hole = alloc_frag->size - alloc_frag->offset; 1630 if (hole < len + room) { 1631 /* To avoid internal fragmentation, if there is very likely not 1632 * enough space for another buffer, add the remaining space to 1633 * the current buffer. 1634 * XDP core assumes that frame_size of xdp_buff and the length 1635 * of the frag are PAGE_SIZE, so we disable the hole mechanism. 1636 */ 1637 if (!headroom) 1638 len += hole; 1639 alloc_frag->offset += hole; 1640 } 1641 1642 sg_init_one(rq->sg, buf, len); 1643 ctx = mergeable_len_to_ctx(len + room, headroom); 1644 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 1645 if (err < 0) 1646 put_page(virt_to_head_page(buf)); 1647 1648 return err; 1649 } 1650 1651 /* 1652 * Returns false if we couldn't fill entirely (OOM). 1653 * 1654 * Normally run in the receive path, but can also be run from ndo_open 1655 * before we're receiving packets, or from refill_work which is 1656 * careful to disable receiving (using napi_disable). 1657 */ 1658 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 1659 gfp_t gfp) 1660 { 1661 int err; 1662 bool oom; 1663 1664 do { 1665 if (vi->mergeable_rx_bufs) 1666 err = add_recvbuf_mergeable(vi, rq, gfp); 1667 else if (vi->big_packets) 1668 err = add_recvbuf_big(vi, rq, gfp); 1669 else 1670 err = add_recvbuf_small(vi, rq, gfp); 1671 1672 oom = err == -ENOMEM; 1673 if (err) 1674 break; 1675 } while (rq->vq->num_free); 1676 if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { 1677 unsigned long flags; 1678 1679 flags = u64_stats_update_begin_irqsave(&rq->stats.syncp); 1680 rq->stats.kicks++; 1681 u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); 1682 } 1683 1684 return !oom; 1685 } 1686 1687 static void skb_recv_done(struct virtqueue *rvq) 1688 { 1689 struct virtnet_info *vi = rvq->vdev->priv; 1690 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 1691 1692 virtqueue_napi_schedule(&rq->napi, rvq); 1693 } 1694 1695 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi) 1696 { 1697 napi_enable(napi); 1698 1699 /* If all buffers were filled by other side before we napi_enabled, we 1700 * won't get another interrupt, so process any outstanding packets now. 1701 * Call local_bh_enable after to trigger softIRQ processing. 1702 */ 1703 local_bh_disable(); 1704 virtqueue_napi_schedule(napi, vq); 1705 local_bh_enable(); 1706 } 1707 1708 static void virtnet_napi_tx_enable(struct virtnet_info *vi, 1709 struct virtqueue *vq, 1710 struct napi_struct *napi) 1711 { 1712 if (!napi->weight) 1713 return; 1714 1715 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 1716 * enable the feature if this is likely affine with the transmit path. 1717 */ 1718 if (!vi->affinity_hint_set) { 1719 napi->weight = 0; 1720 return; 1721 } 1722 1723 return virtnet_napi_enable(vq, napi); 1724 } 1725 1726 static void virtnet_napi_tx_disable(struct napi_struct *napi) 1727 { 1728 if (napi->weight) 1729 napi_disable(napi); 1730 } 1731 1732 static void refill_work(struct work_struct *work) 1733 { 1734 struct virtnet_info *vi = 1735 container_of(work, struct virtnet_info, refill.work); 1736 bool still_empty; 1737 int i; 1738 1739 for (i = 0; i < vi->curr_queue_pairs; i++) { 1740 struct receive_queue *rq = &vi->rq[i]; 1741 1742 napi_disable(&rq->napi); 1743 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL); 1744 virtnet_napi_enable(rq->vq, &rq->napi); 1745 1746 /* In theory, this can happen: if we don't get any buffers in 1747 * we will *never* try to fill again. 1748 */ 1749 if (still_empty) 1750 schedule_delayed_work(&vi->refill, HZ/2); 1751 } 1752 } 1753 1754 static int virtnet_receive(struct receive_queue *rq, int budget, 1755 unsigned int *xdp_xmit) 1756 { 1757 struct virtnet_info *vi = rq->vq->vdev->priv; 1758 struct virtnet_rq_stats stats = {}; 1759 unsigned int len; 1760 void *buf; 1761 int i; 1762 1763 if (!vi->big_packets || vi->mergeable_rx_bufs) { 1764 void *ctx; 1765 1766 while (stats.packets < budget && 1767 (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) { 1768 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats); 1769 stats.packets++; 1770 } 1771 } else { 1772 while (stats.packets < budget && 1773 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 1774 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats); 1775 stats.packets++; 1776 } 1777 } 1778 1779 if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 1780 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) { 1781 spin_lock(&vi->refill_lock); 1782 if (vi->refill_enabled) 1783 schedule_delayed_work(&vi->refill, 0); 1784 spin_unlock(&vi->refill_lock); 1785 } 1786 } 1787 1788 u64_stats_update_begin(&rq->stats.syncp); 1789 for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) { 1790 size_t offset = virtnet_rq_stats_desc[i].offset; 1791 u64 *item; 1792 1793 item = (u64 *)((u8 *)&rq->stats + offset); 1794 *item += *(u64 *)((u8 *)&stats + offset); 1795 } 1796 u64_stats_update_end(&rq->stats.syncp); 1797 1798 return stats.packets; 1799 } 1800 1801 static void virtnet_poll_cleantx(struct receive_queue *rq) 1802 { 1803 struct virtnet_info *vi = rq->vq->vdev->priv; 1804 unsigned int index = vq2rxq(rq->vq); 1805 struct send_queue *sq = &vi->sq[index]; 1806 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 1807 1808 if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) 1809 return; 1810 1811 if (__netif_tx_trylock(txq)) { 1812 if (sq->reset) { 1813 __netif_tx_unlock(txq); 1814 return; 1815 } 1816 1817 do { 1818 virtqueue_disable_cb(sq->vq); 1819 free_old_xmit_skbs(sq, true); 1820 } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 1821 1822 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 1823 netif_tx_wake_queue(txq); 1824 1825 __netif_tx_unlock(txq); 1826 } 1827 } 1828 1829 static int virtnet_poll(struct napi_struct *napi, int budget) 1830 { 1831 struct receive_queue *rq = 1832 container_of(napi, struct receive_queue, napi); 1833 struct virtnet_info *vi = rq->vq->vdev->priv; 1834 struct send_queue *sq; 1835 unsigned int received; 1836 unsigned int xdp_xmit = 0; 1837 1838 virtnet_poll_cleantx(rq); 1839 1840 received = virtnet_receive(rq, budget, &xdp_xmit); 1841 1842 if (xdp_xmit & VIRTIO_XDP_REDIR) 1843 xdp_do_flush(); 1844 1845 /* Out of packets? */ 1846 if (received < budget) 1847 virtqueue_napi_complete(napi, rq->vq, received); 1848 1849 if (xdp_xmit & VIRTIO_XDP_TX) { 1850 sq = virtnet_xdp_get_sq(vi); 1851 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 1852 u64_stats_update_begin(&sq->stats.syncp); 1853 sq->stats.kicks++; 1854 u64_stats_update_end(&sq->stats.syncp); 1855 } 1856 virtnet_xdp_put_sq(vi, sq); 1857 } 1858 1859 return received; 1860 } 1861 1862 static int virtnet_open(struct net_device *dev) 1863 { 1864 struct virtnet_info *vi = netdev_priv(dev); 1865 int i, err; 1866 1867 enable_delayed_refill(vi); 1868 1869 for (i = 0; i < vi->max_queue_pairs; i++) { 1870 if (i < vi->curr_queue_pairs) 1871 /* Make sure we have some buffers: if oom use wq. */ 1872 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 1873 schedule_delayed_work(&vi->refill, 0); 1874 1875 err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id); 1876 if (err < 0) 1877 return err; 1878 1879 err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq, 1880 MEM_TYPE_PAGE_SHARED, NULL); 1881 if (err < 0) { 1882 xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 1883 return err; 1884 } 1885 1886 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 1887 virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi); 1888 } 1889 1890 return 0; 1891 } 1892 1893 static int virtnet_poll_tx(struct napi_struct *napi, int budget) 1894 { 1895 struct send_queue *sq = container_of(napi, struct send_queue, napi); 1896 struct virtnet_info *vi = sq->vq->vdev->priv; 1897 unsigned int index = vq2txq(sq->vq); 1898 struct netdev_queue *txq; 1899 int opaque; 1900 bool done; 1901 1902 if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { 1903 /* We don't need to enable cb for XDP */ 1904 napi_complete_done(napi, 0); 1905 return 0; 1906 } 1907 1908 txq = netdev_get_tx_queue(vi->dev, index); 1909 __netif_tx_lock(txq, raw_smp_processor_id()); 1910 virtqueue_disable_cb(sq->vq); 1911 free_old_xmit_skbs(sq, true); 1912 1913 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 1914 netif_tx_wake_queue(txq); 1915 1916 opaque = virtqueue_enable_cb_prepare(sq->vq); 1917 1918 done = napi_complete_done(napi, 0); 1919 1920 if (!done) 1921 virtqueue_disable_cb(sq->vq); 1922 1923 __netif_tx_unlock(txq); 1924 1925 if (done) { 1926 if (unlikely(virtqueue_poll(sq->vq, opaque))) { 1927 if (napi_schedule_prep(napi)) { 1928 __netif_tx_lock(txq, raw_smp_processor_id()); 1929 virtqueue_disable_cb(sq->vq); 1930 __netif_tx_unlock(txq); 1931 __napi_schedule(napi); 1932 } 1933 } 1934 } 1935 1936 return 0; 1937 } 1938 1939 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) 1940 { 1941 struct virtio_net_hdr_mrg_rxbuf *hdr; 1942 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 1943 struct virtnet_info *vi = sq->vq->vdev->priv; 1944 int num_sg; 1945 unsigned hdr_len = vi->hdr_len; 1946 bool can_push; 1947 1948 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 1949 1950 can_push = vi->any_header_sg && 1951 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 1952 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 1953 /* Even if we can, don't push here yet as this would skew 1954 * csum_start offset below. */ 1955 if (can_push) 1956 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len); 1957 else 1958 hdr = skb_vnet_hdr(skb); 1959 1960 if (virtio_net_hdr_from_skb(skb, &hdr->hdr, 1961 virtio_is_little_endian(vi->vdev), false, 1962 0)) 1963 return -EPROTO; 1964 1965 if (vi->mergeable_rx_bufs) 1966 hdr->num_buffers = 0; 1967 1968 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 1969 if (can_push) { 1970 __skb_push(skb, hdr_len); 1971 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 1972 if (unlikely(num_sg < 0)) 1973 return num_sg; 1974 /* Pull header back to avoid skew in tx bytes calculations. */ 1975 __skb_pull(skb, hdr_len); 1976 } else { 1977 sg_set_buf(sq->sg, hdr, hdr_len); 1978 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 1979 if (unlikely(num_sg < 0)) 1980 return num_sg; 1981 num_sg++; 1982 } 1983 return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC); 1984 } 1985 1986 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 1987 { 1988 struct virtnet_info *vi = netdev_priv(dev); 1989 int qnum = skb_get_queue_mapping(skb); 1990 struct send_queue *sq = &vi->sq[qnum]; 1991 int err; 1992 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 1993 bool kick = !netdev_xmit_more(); 1994 bool use_napi = sq->napi.weight; 1995 1996 /* Free up any pending old buffers before queueing new ones. */ 1997 do { 1998 if (use_napi) 1999 virtqueue_disable_cb(sq->vq); 2000 2001 free_old_xmit_skbs(sq, false); 2002 2003 } while (use_napi && kick && 2004 unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 2005 2006 /* timestamp packet in software */ 2007 skb_tx_timestamp(skb); 2008 2009 /* Try to transmit */ 2010 err = xmit_skb(sq, skb); 2011 2012 /* This should not happen! */ 2013 if (unlikely(err)) { 2014 dev->stats.tx_fifo_errors++; 2015 if (net_ratelimit()) 2016 dev_warn(&dev->dev, 2017 "Unexpected TXQ (%d) queue failure: %d\n", 2018 qnum, err); 2019 dev->stats.tx_dropped++; 2020 dev_kfree_skb_any(skb); 2021 return NETDEV_TX_OK; 2022 } 2023 2024 /* Don't wait up for transmitted skbs to be freed. */ 2025 if (!use_napi) { 2026 skb_orphan(skb); 2027 nf_reset_ct(skb); 2028 } 2029 2030 check_sq_full_and_disable(vi, dev, sq); 2031 2032 if (kick || netif_xmit_stopped(txq)) { 2033 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 2034 u64_stats_update_begin(&sq->stats.syncp); 2035 sq->stats.kicks++; 2036 u64_stats_update_end(&sq->stats.syncp); 2037 } 2038 } 2039 2040 return NETDEV_TX_OK; 2041 } 2042 2043 static int virtnet_rx_resize(struct virtnet_info *vi, 2044 struct receive_queue *rq, u32 ring_num) 2045 { 2046 bool running = netif_running(vi->dev); 2047 int err, qindex; 2048 2049 qindex = rq - vi->rq; 2050 2051 if (running) 2052 napi_disable(&rq->napi); 2053 2054 err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf); 2055 if (err) 2056 netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); 2057 2058 if (!try_fill_recv(vi, rq, GFP_KERNEL)) 2059 schedule_delayed_work(&vi->refill, 0); 2060 2061 if (running) 2062 virtnet_napi_enable(rq->vq, &rq->napi); 2063 return err; 2064 } 2065 2066 static int virtnet_tx_resize(struct virtnet_info *vi, 2067 struct send_queue *sq, u32 ring_num) 2068 { 2069 bool running = netif_running(vi->dev); 2070 struct netdev_queue *txq; 2071 int err, qindex; 2072 2073 qindex = sq - vi->sq; 2074 2075 if (running) 2076 virtnet_napi_tx_disable(&sq->napi); 2077 2078 txq = netdev_get_tx_queue(vi->dev, qindex); 2079 2080 /* 1. wait all ximt complete 2081 * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue() 2082 */ 2083 __netif_tx_lock_bh(txq); 2084 2085 /* Prevent rx poll from accessing sq. */ 2086 sq->reset = true; 2087 2088 /* Prevent the upper layer from trying to send packets. */ 2089 netif_stop_subqueue(vi->dev, qindex); 2090 2091 __netif_tx_unlock_bh(txq); 2092 2093 err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf); 2094 if (err) 2095 netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err); 2096 2097 __netif_tx_lock_bh(txq); 2098 sq->reset = false; 2099 netif_tx_wake_queue(txq); 2100 __netif_tx_unlock_bh(txq); 2101 2102 if (running) 2103 virtnet_napi_tx_enable(vi, sq->vq, &sq->napi); 2104 return err; 2105 } 2106 2107 /* 2108 * Send command via the control virtqueue and check status. Commands 2109 * supported by the hypervisor, as indicated by feature bits, should 2110 * never fail unless improperly formatted. 2111 */ 2112 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 2113 struct scatterlist *out) 2114 { 2115 struct scatterlist *sgs[4], hdr, stat; 2116 unsigned out_num = 0, tmp; 2117 int ret; 2118 2119 /* Caller should know better */ 2120 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 2121 2122 vi->ctrl->status = ~0; 2123 vi->ctrl->hdr.class = class; 2124 vi->ctrl->hdr.cmd = cmd; 2125 /* Add header */ 2126 sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 2127 sgs[out_num++] = &hdr; 2128 2129 if (out) 2130 sgs[out_num++] = out; 2131 2132 /* Add return status. */ 2133 sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 2134 sgs[out_num] = &stat; 2135 2136 BUG_ON(out_num + 1 > ARRAY_SIZE(sgs)); 2137 ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC); 2138 if (ret < 0) { 2139 dev_warn(&vi->vdev->dev, 2140 "Failed to add sgs for command vq: %d\n.", ret); 2141 return false; 2142 } 2143 2144 if (unlikely(!virtqueue_kick(vi->cvq))) 2145 return vi->ctrl->status == VIRTIO_NET_OK; 2146 2147 /* Spin for a response, the kick causes an ioport write, trapping 2148 * into the hypervisor, so the request should be handled immediately. 2149 */ 2150 while (!virtqueue_get_buf(vi->cvq, &tmp) && 2151 !virtqueue_is_broken(vi->cvq)) 2152 cpu_relax(); 2153 2154 return vi->ctrl->status == VIRTIO_NET_OK; 2155 } 2156 2157 static int virtnet_set_mac_address(struct net_device *dev, void *p) 2158 { 2159 struct virtnet_info *vi = netdev_priv(dev); 2160 struct virtio_device *vdev = vi->vdev; 2161 int ret; 2162 struct sockaddr *addr; 2163 struct scatterlist sg; 2164 2165 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 2166 return -EOPNOTSUPP; 2167 2168 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 2169 if (!addr) 2170 return -ENOMEM; 2171 2172 ret = eth_prepare_mac_addr_change(dev, addr); 2173 if (ret) 2174 goto out; 2175 2176 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 2177 sg_init_one(&sg, addr->sa_data, dev->addr_len); 2178 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 2179 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 2180 dev_warn(&vdev->dev, 2181 "Failed to set mac address by vq command.\n"); 2182 ret = -EINVAL; 2183 goto out; 2184 } 2185 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 2186 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2187 unsigned int i; 2188 2189 /* Naturally, this has an atomicity problem. */ 2190 for (i = 0; i < dev->addr_len; i++) 2191 virtio_cwrite8(vdev, 2192 offsetof(struct virtio_net_config, mac) + 2193 i, addr->sa_data[i]); 2194 } 2195 2196 eth_commit_mac_addr_change(dev, p); 2197 ret = 0; 2198 2199 out: 2200 kfree(addr); 2201 return ret; 2202 } 2203 2204 static void virtnet_stats(struct net_device *dev, 2205 struct rtnl_link_stats64 *tot) 2206 { 2207 struct virtnet_info *vi = netdev_priv(dev); 2208 unsigned int start; 2209 int i; 2210 2211 for (i = 0; i < vi->max_queue_pairs; i++) { 2212 u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops; 2213 struct receive_queue *rq = &vi->rq[i]; 2214 struct send_queue *sq = &vi->sq[i]; 2215 2216 do { 2217 start = u64_stats_fetch_begin(&sq->stats.syncp); 2218 tpackets = sq->stats.packets; 2219 tbytes = sq->stats.bytes; 2220 terrors = sq->stats.tx_timeouts; 2221 } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 2222 2223 do { 2224 start = u64_stats_fetch_begin(&rq->stats.syncp); 2225 rpackets = rq->stats.packets; 2226 rbytes = rq->stats.bytes; 2227 rdrops = rq->stats.drops; 2228 } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 2229 2230 tot->rx_packets += rpackets; 2231 tot->tx_packets += tpackets; 2232 tot->rx_bytes += rbytes; 2233 tot->tx_bytes += tbytes; 2234 tot->rx_dropped += rdrops; 2235 tot->tx_errors += terrors; 2236 } 2237 2238 tot->tx_dropped = dev->stats.tx_dropped; 2239 tot->tx_fifo_errors = dev->stats.tx_fifo_errors; 2240 tot->rx_length_errors = dev->stats.rx_length_errors; 2241 tot->rx_frame_errors = dev->stats.rx_frame_errors; 2242 } 2243 2244 static void virtnet_ack_link_announce(struct virtnet_info *vi) 2245 { 2246 rtnl_lock(); 2247 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 2248 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 2249 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 2250 rtnl_unlock(); 2251 } 2252 2253 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 2254 { 2255 struct scatterlist sg; 2256 struct net_device *dev = vi->dev; 2257 2258 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 2259 return 0; 2260 2261 vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 2262 sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq)); 2263 2264 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 2265 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 2266 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 2267 queue_pairs); 2268 return -EINVAL; 2269 } else { 2270 vi->curr_queue_pairs = queue_pairs; 2271 /* virtnet_open() will refill when device is going to up. */ 2272 if (dev->flags & IFF_UP) 2273 schedule_delayed_work(&vi->refill, 0); 2274 } 2275 2276 return 0; 2277 } 2278 2279 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 2280 { 2281 int err; 2282 2283 rtnl_lock(); 2284 err = _virtnet_set_queues(vi, queue_pairs); 2285 rtnl_unlock(); 2286 return err; 2287 } 2288 2289 static int virtnet_close(struct net_device *dev) 2290 { 2291 struct virtnet_info *vi = netdev_priv(dev); 2292 int i; 2293 2294 /* Make sure NAPI doesn't schedule refill work */ 2295 disable_delayed_refill(vi); 2296 /* Make sure refill_work doesn't re-enable napi! */ 2297 cancel_delayed_work_sync(&vi->refill); 2298 2299 for (i = 0; i < vi->max_queue_pairs; i++) { 2300 virtnet_napi_tx_disable(&vi->sq[i].napi); 2301 napi_disable(&vi->rq[i].napi); 2302 xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 2303 } 2304 2305 return 0; 2306 } 2307 2308 static void virtnet_set_rx_mode(struct net_device *dev) 2309 { 2310 struct virtnet_info *vi = netdev_priv(dev); 2311 struct scatterlist sg[2]; 2312 struct virtio_net_ctrl_mac *mac_data; 2313 struct netdev_hw_addr *ha; 2314 int uc_count; 2315 int mc_count; 2316 void *buf; 2317 int i; 2318 2319 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 2320 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 2321 return; 2322 2323 vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0); 2324 vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0); 2325 2326 sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc)); 2327 2328 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 2329 VIRTIO_NET_CTRL_RX_PROMISC, sg)) 2330 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 2331 vi->ctrl->promisc ? "en" : "dis"); 2332 2333 sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti)); 2334 2335 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 2336 VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 2337 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 2338 vi->ctrl->allmulti ? "en" : "dis"); 2339 2340 uc_count = netdev_uc_count(dev); 2341 mc_count = netdev_mc_count(dev); 2342 /* MAC filter - use one buffer for both lists */ 2343 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 2344 (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 2345 mac_data = buf; 2346 if (!buf) 2347 return; 2348 2349 sg_init_table(sg, 2); 2350 2351 /* Store the unicast list and count in the front of the buffer */ 2352 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 2353 i = 0; 2354 netdev_for_each_uc_addr(ha, dev) 2355 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2356 2357 sg_set_buf(&sg[0], mac_data, 2358 sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 2359 2360 /* multicast list and count fill the end */ 2361 mac_data = (void *)&mac_data->macs[uc_count][0]; 2362 2363 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 2364 i = 0; 2365 netdev_for_each_mc_addr(ha, dev) 2366 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2367 2368 sg_set_buf(&sg[1], mac_data, 2369 sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 2370 2371 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 2372 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 2373 dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 2374 2375 kfree(buf); 2376 } 2377 2378 static int virtnet_vlan_rx_add_vid(struct net_device *dev, 2379 __be16 proto, u16 vid) 2380 { 2381 struct virtnet_info *vi = netdev_priv(dev); 2382 struct scatterlist sg; 2383 2384 vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 2385 sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 2386 2387 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2388 VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 2389 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 2390 return 0; 2391 } 2392 2393 static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 2394 __be16 proto, u16 vid) 2395 { 2396 struct virtnet_info *vi = netdev_priv(dev); 2397 struct scatterlist sg; 2398 2399 vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 2400 sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 2401 2402 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2403 VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 2404 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 2405 return 0; 2406 } 2407 2408 static void virtnet_clean_affinity(struct virtnet_info *vi) 2409 { 2410 int i; 2411 2412 if (vi->affinity_hint_set) { 2413 for (i = 0; i < vi->max_queue_pairs; i++) { 2414 virtqueue_set_affinity(vi->rq[i].vq, NULL); 2415 virtqueue_set_affinity(vi->sq[i].vq, NULL); 2416 } 2417 2418 vi->affinity_hint_set = false; 2419 } 2420 } 2421 2422 static void virtnet_set_affinity(struct virtnet_info *vi) 2423 { 2424 cpumask_var_t mask; 2425 int stragglers; 2426 int group_size; 2427 int i, j, cpu; 2428 int num_cpu; 2429 int stride; 2430 2431 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 2432 virtnet_clean_affinity(vi); 2433 return; 2434 } 2435 2436 num_cpu = num_online_cpus(); 2437 stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); 2438 stragglers = num_cpu >= vi->curr_queue_pairs ? 2439 num_cpu % vi->curr_queue_pairs : 2440 0; 2441 cpu = cpumask_first(cpu_online_mask); 2442 2443 for (i = 0; i < vi->curr_queue_pairs; i++) { 2444 group_size = stride + (i < stragglers ? 1 : 0); 2445 2446 for (j = 0; j < group_size; j++) { 2447 cpumask_set_cpu(cpu, mask); 2448 cpu = cpumask_next_wrap(cpu, cpu_online_mask, 2449 nr_cpu_ids, false); 2450 } 2451 virtqueue_set_affinity(vi->rq[i].vq, mask); 2452 virtqueue_set_affinity(vi->sq[i].vq, mask); 2453 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); 2454 cpumask_clear(mask); 2455 } 2456 2457 vi->affinity_hint_set = true; 2458 free_cpumask_var(mask); 2459 } 2460 2461 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 2462 { 2463 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 2464 node); 2465 virtnet_set_affinity(vi); 2466 return 0; 2467 } 2468 2469 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 2470 { 2471 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 2472 node_dead); 2473 virtnet_set_affinity(vi); 2474 return 0; 2475 } 2476 2477 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 2478 { 2479 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 2480 node); 2481 2482 virtnet_clean_affinity(vi); 2483 return 0; 2484 } 2485 2486 static enum cpuhp_state virtionet_online; 2487 2488 static int virtnet_cpu_notif_add(struct virtnet_info *vi) 2489 { 2490 int ret; 2491 2492 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 2493 if (ret) 2494 return ret; 2495 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 2496 &vi->node_dead); 2497 if (!ret) 2498 return ret; 2499 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 2500 return ret; 2501 } 2502 2503 static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 2504 { 2505 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 2506 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 2507 &vi->node_dead); 2508 } 2509 2510 static void virtnet_get_ringparam(struct net_device *dev, 2511 struct ethtool_ringparam *ring, 2512 struct kernel_ethtool_ringparam *kernel_ring, 2513 struct netlink_ext_ack *extack) 2514 { 2515 struct virtnet_info *vi = netdev_priv(dev); 2516 2517 ring->rx_max_pending = vi->rq[0].vq->num_max; 2518 ring->tx_max_pending = vi->sq[0].vq->num_max; 2519 ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 2520 ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 2521 } 2522 2523 static int virtnet_set_ringparam(struct net_device *dev, 2524 struct ethtool_ringparam *ring, 2525 struct kernel_ethtool_ringparam *kernel_ring, 2526 struct netlink_ext_ack *extack) 2527 { 2528 struct virtnet_info *vi = netdev_priv(dev); 2529 u32 rx_pending, tx_pending; 2530 struct receive_queue *rq; 2531 struct send_queue *sq; 2532 int i, err; 2533 2534 if (ring->rx_mini_pending || ring->rx_jumbo_pending) 2535 return -EINVAL; 2536 2537 rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 2538 tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 2539 2540 if (ring->rx_pending == rx_pending && 2541 ring->tx_pending == tx_pending) 2542 return 0; 2543 2544 if (ring->rx_pending > vi->rq[0].vq->num_max) 2545 return -EINVAL; 2546 2547 if (ring->tx_pending > vi->sq[0].vq->num_max) 2548 return -EINVAL; 2549 2550 for (i = 0; i < vi->max_queue_pairs; i++) { 2551 rq = vi->rq + i; 2552 sq = vi->sq + i; 2553 2554 if (ring->tx_pending != tx_pending) { 2555 err = virtnet_tx_resize(vi, sq, ring->tx_pending); 2556 if (err) 2557 return err; 2558 } 2559 2560 if (ring->rx_pending != rx_pending) { 2561 err = virtnet_rx_resize(vi, rq, ring->rx_pending); 2562 if (err) 2563 return err; 2564 } 2565 } 2566 2567 return 0; 2568 } 2569 2570 static bool virtnet_commit_rss_command(struct virtnet_info *vi) 2571 { 2572 struct net_device *dev = vi->dev; 2573 struct scatterlist sgs[4]; 2574 unsigned int sg_buf_size; 2575 2576 /* prepare sgs */ 2577 sg_init_table(sgs, 4); 2578 2579 sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table); 2580 sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size); 2581 2582 sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1); 2583 sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size); 2584 2585 sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key) 2586 - offsetof(struct virtio_net_ctrl_rss, max_tx_vq); 2587 sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size); 2588 2589 sg_buf_size = vi->rss_key_size; 2590 sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size); 2591 2592 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 2593 vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG 2594 : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) { 2595 dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n"); 2596 return false; 2597 } 2598 return true; 2599 } 2600 2601 static void virtnet_init_default_rss(struct virtnet_info *vi) 2602 { 2603 u32 indir_val = 0; 2604 int i = 0; 2605 2606 vi->ctrl->rss.hash_types = vi->rss_hash_types_supported; 2607 vi->rss_hash_types_saved = vi->rss_hash_types_supported; 2608 vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size 2609 ? vi->rss_indir_table_size - 1 : 0; 2610 vi->ctrl->rss.unclassified_queue = 0; 2611 2612 for (; i < vi->rss_indir_table_size; ++i) { 2613 indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs); 2614 vi->ctrl->rss.indirection_table[i] = indir_val; 2615 } 2616 2617 vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs; 2618 vi->ctrl->rss.hash_key_length = vi->rss_key_size; 2619 2620 netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size); 2621 } 2622 2623 static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info) 2624 { 2625 info->data = 0; 2626 switch (info->flow_type) { 2627 case TCP_V4_FLOW: 2628 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { 2629 info->data = RXH_IP_SRC | RXH_IP_DST | 2630 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2631 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 2632 info->data = RXH_IP_SRC | RXH_IP_DST; 2633 } 2634 break; 2635 case TCP_V6_FLOW: 2636 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { 2637 info->data = RXH_IP_SRC | RXH_IP_DST | 2638 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2639 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 2640 info->data = RXH_IP_SRC | RXH_IP_DST; 2641 } 2642 break; 2643 case UDP_V4_FLOW: 2644 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { 2645 info->data = RXH_IP_SRC | RXH_IP_DST | 2646 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2647 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 2648 info->data = RXH_IP_SRC | RXH_IP_DST; 2649 } 2650 break; 2651 case UDP_V6_FLOW: 2652 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { 2653 info->data = RXH_IP_SRC | RXH_IP_DST | 2654 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2655 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 2656 info->data = RXH_IP_SRC | RXH_IP_DST; 2657 } 2658 break; 2659 case IPV4_FLOW: 2660 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) 2661 info->data = RXH_IP_SRC | RXH_IP_DST; 2662 2663 break; 2664 case IPV6_FLOW: 2665 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) 2666 info->data = RXH_IP_SRC | RXH_IP_DST; 2667 2668 break; 2669 default: 2670 info->data = 0; 2671 break; 2672 } 2673 } 2674 2675 static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info) 2676 { 2677 u32 new_hashtypes = vi->rss_hash_types_saved; 2678 bool is_disable = info->data & RXH_DISCARD; 2679 bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3); 2680 2681 /* supports only 'sd', 'sdfn' and 'r' */ 2682 if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable)) 2683 return false; 2684 2685 switch (info->flow_type) { 2686 case TCP_V4_FLOW: 2687 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4); 2688 if (!is_disable) 2689 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 2690 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0); 2691 break; 2692 case UDP_V4_FLOW: 2693 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4); 2694 if (!is_disable) 2695 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 2696 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0); 2697 break; 2698 case IPV4_FLOW: 2699 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4; 2700 if (!is_disable) 2701 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4; 2702 break; 2703 case TCP_V6_FLOW: 2704 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6); 2705 if (!is_disable) 2706 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 2707 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0); 2708 break; 2709 case UDP_V6_FLOW: 2710 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6); 2711 if (!is_disable) 2712 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 2713 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0); 2714 break; 2715 case IPV6_FLOW: 2716 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6; 2717 if (!is_disable) 2718 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6; 2719 break; 2720 default: 2721 /* unsupported flow */ 2722 return false; 2723 } 2724 2725 /* if unsupported hashtype was set */ 2726 if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported)) 2727 return false; 2728 2729 if (new_hashtypes != vi->rss_hash_types_saved) { 2730 vi->rss_hash_types_saved = new_hashtypes; 2731 vi->ctrl->rss.hash_types = vi->rss_hash_types_saved; 2732 if (vi->dev->features & NETIF_F_RXHASH) 2733 return virtnet_commit_rss_command(vi); 2734 } 2735 2736 return true; 2737 } 2738 2739 static void virtnet_get_drvinfo(struct net_device *dev, 2740 struct ethtool_drvinfo *info) 2741 { 2742 struct virtnet_info *vi = netdev_priv(dev); 2743 struct virtio_device *vdev = vi->vdev; 2744 2745 strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 2746 strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 2747 strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 2748 2749 } 2750 2751 /* TODO: Eliminate OOO packets during switching */ 2752 static int virtnet_set_channels(struct net_device *dev, 2753 struct ethtool_channels *channels) 2754 { 2755 struct virtnet_info *vi = netdev_priv(dev); 2756 u16 queue_pairs = channels->combined_count; 2757 int err; 2758 2759 /* We don't support separate rx/tx channels. 2760 * We don't allow setting 'other' channels. 2761 */ 2762 if (channels->rx_count || channels->tx_count || channels->other_count) 2763 return -EINVAL; 2764 2765 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 2766 return -EINVAL; 2767 2768 /* For now we don't support modifying channels while XDP is loaded 2769 * also when XDP is loaded all RX queues have XDP programs so we only 2770 * need to check a single RX queue. 2771 */ 2772 if (vi->rq[0].xdp_prog) 2773 return -EINVAL; 2774 2775 cpus_read_lock(); 2776 err = _virtnet_set_queues(vi, queue_pairs); 2777 if (err) { 2778 cpus_read_unlock(); 2779 goto err; 2780 } 2781 virtnet_set_affinity(vi); 2782 cpus_read_unlock(); 2783 2784 netif_set_real_num_tx_queues(dev, queue_pairs); 2785 netif_set_real_num_rx_queues(dev, queue_pairs); 2786 err: 2787 return err; 2788 } 2789 2790 static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 2791 { 2792 struct virtnet_info *vi = netdev_priv(dev); 2793 unsigned int i, j; 2794 u8 *p = data; 2795 2796 switch (stringset) { 2797 case ETH_SS_STATS: 2798 for (i = 0; i < vi->curr_queue_pairs; i++) { 2799 for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) 2800 ethtool_sprintf(&p, "rx_queue_%u_%s", i, 2801 virtnet_rq_stats_desc[j].desc); 2802 } 2803 2804 for (i = 0; i < vi->curr_queue_pairs; i++) { 2805 for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) 2806 ethtool_sprintf(&p, "tx_queue_%u_%s", i, 2807 virtnet_sq_stats_desc[j].desc); 2808 } 2809 break; 2810 } 2811 } 2812 2813 static int virtnet_get_sset_count(struct net_device *dev, int sset) 2814 { 2815 struct virtnet_info *vi = netdev_priv(dev); 2816 2817 switch (sset) { 2818 case ETH_SS_STATS: 2819 return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN + 2820 VIRTNET_SQ_STATS_LEN); 2821 default: 2822 return -EOPNOTSUPP; 2823 } 2824 } 2825 2826 static void virtnet_get_ethtool_stats(struct net_device *dev, 2827 struct ethtool_stats *stats, u64 *data) 2828 { 2829 struct virtnet_info *vi = netdev_priv(dev); 2830 unsigned int idx = 0, start, i, j; 2831 const u8 *stats_base; 2832 size_t offset; 2833 2834 for (i = 0; i < vi->curr_queue_pairs; i++) { 2835 struct receive_queue *rq = &vi->rq[i]; 2836 2837 stats_base = (u8 *)&rq->stats; 2838 do { 2839 start = u64_stats_fetch_begin(&rq->stats.syncp); 2840 for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { 2841 offset = virtnet_rq_stats_desc[j].offset; 2842 data[idx + j] = *(u64 *)(stats_base + offset); 2843 } 2844 } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 2845 idx += VIRTNET_RQ_STATS_LEN; 2846 } 2847 2848 for (i = 0; i < vi->curr_queue_pairs; i++) { 2849 struct send_queue *sq = &vi->sq[i]; 2850 2851 stats_base = (u8 *)&sq->stats; 2852 do { 2853 start = u64_stats_fetch_begin(&sq->stats.syncp); 2854 for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { 2855 offset = virtnet_sq_stats_desc[j].offset; 2856 data[idx + j] = *(u64 *)(stats_base + offset); 2857 } 2858 } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 2859 idx += VIRTNET_SQ_STATS_LEN; 2860 } 2861 } 2862 2863 static void virtnet_get_channels(struct net_device *dev, 2864 struct ethtool_channels *channels) 2865 { 2866 struct virtnet_info *vi = netdev_priv(dev); 2867 2868 channels->combined_count = vi->curr_queue_pairs; 2869 channels->max_combined = vi->max_queue_pairs; 2870 channels->max_other = 0; 2871 channels->rx_count = 0; 2872 channels->tx_count = 0; 2873 channels->other_count = 0; 2874 } 2875 2876 static int virtnet_set_link_ksettings(struct net_device *dev, 2877 const struct ethtool_link_ksettings *cmd) 2878 { 2879 struct virtnet_info *vi = netdev_priv(dev); 2880 2881 return ethtool_virtdev_set_link_ksettings(dev, cmd, 2882 &vi->speed, &vi->duplex); 2883 } 2884 2885 static int virtnet_get_link_ksettings(struct net_device *dev, 2886 struct ethtool_link_ksettings *cmd) 2887 { 2888 struct virtnet_info *vi = netdev_priv(dev); 2889 2890 cmd->base.speed = vi->speed; 2891 cmd->base.duplex = vi->duplex; 2892 cmd->base.port = PORT_OTHER; 2893 2894 return 0; 2895 } 2896 2897 static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, 2898 struct ethtool_coalesce *ec) 2899 { 2900 struct scatterlist sgs_tx, sgs_rx; 2901 struct virtio_net_ctrl_coal_tx coal_tx; 2902 struct virtio_net_ctrl_coal_rx coal_rx; 2903 2904 coal_tx.tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs); 2905 coal_tx.tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames); 2906 sg_init_one(&sgs_tx, &coal_tx, sizeof(coal_tx)); 2907 2908 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 2909 VIRTIO_NET_CTRL_NOTF_COAL_TX_SET, 2910 &sgs_tx)) 2911 return -EINVAL; 2912 2913 /* Save parameters */ 2914 vi->tx_usecs = ec->tx_coalesce_usecs; 2915 vi->tx_max_packets = ec->tx_max_coalesced_frames; 2916 2917 coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); 2918 coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); 2919 sg_init_one(&sgs_rx, &coal_rx, sizeof(coal_rx)); 2920 2921 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 2922 VIRTIO_NET_CTRL_NOTF_COAL_RX_SET, 2923 &sgs_rx)) 2924 return -EINVAL; 2925 2926 /* Save parameters */ 2927 vi->rx_usecs = ec->rx_coalesce_usecs; 2928 vi->rx_max_packets = ec->rx_max_coalesced_frames; 2929 2930 return 0; 2931 } 2932 2933 static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) 2934 { 2935 /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL 2936 * feature is negotiated. 2937 */ 2938 if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs) 2939 return -EOPNOTSUPP; 2940 2941 if (ec->tx_max_coalesced_frames > 1 || 2942 ec->rx_max_coalesced_frames != 1) 2943 return -EINVAL; 2944 2945 return 0; 2946 } 2947 2948 static int virtnet_set_coalesce(struct net_device *dev, 2949 struct ethtool_coalesce *ec, 2950 struct kernel_ethtool_coalesce *kernel_coal, 2951 struct netlink_ext_ack *extack) 2952 { 2953 struct virtnet_info *vi = netdev_priv(dev); 2954 int ret, i, napi_weight; 2955 bool update_napi = false; 2956 2957 /* Can't change NAPI weight if the link is up */ 2958 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 2959 if (napi_weight ^ vi->sq[0].napi.weight) { 2960 if (dev->flags & IFF_UP) 2961 return -EBUSY; 2962 else 2963 update_napi = true; 2964 } 2965 2966 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) 2967 ret = virtnet_send_notf_coal_cmds(vi, ec); 2968 else 2969 ret = virtnet_coal_params_supported(ec); 2970 2971 if (ret) 2972 return ret; 2973 2974 if (update_napi) { 2975 for (i = 0; i < vi->max_queue_pairs; i++) 2976 vi->sq[i].napi.weight = napi_weight; 2977 } 2978 2979 return ret; 2980 } 2981 2982 static int virtnet_get_coalesce(struct net_device *dev, 2983 struct ethtool_coalesce *ec, 2984 struct kernel_ethtool_coalesce *kernel_coal, 2985 struct netlink_ext_ack *extack) 2986 { 2987 struct virtnet_info *vi = netdev_priv(dev); 2988 2989 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 2990 ec->rx_coalesce_usecs = vi->rx_usecs; 2991 ec->tx_coalesce_usecs = vi->tx_usecs; 2992 ec->tx_max_coalesced_frames = vi->tx_max_packets; 2993 ec->rx_max_coalesced_frames = vi->rx_max_packets; 2994 } else { 2995 ec->rx_max_coalesced_frames = 1; 2996 2997 if (vi->sq[0].napi.weight) 2998 ec->tx_max_coalesced_frames = 1; 2999 } 3000 3001 return 0; 3002 } 3003 3004 static void virtnet_init_settings(struct net_device *dev) 3005 { 3006 struct virtnet_info *vi = netdev_priv(dev); 3007 3008 vi->speed = SPEED_UNKNOWN; 3009 vi->duplex = DUPLEX_UNKNOWN; 3010 } 3011 3012 static void virtnet_update_settings(struct virtnet_info *vi) 3013 { 3014 u32 speed; 3015 u8 duplex; 3016 3017 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 3018 return; 3019 3020 virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); 3021 3022 if (ethtool_validate_speed(speed)) 3023 vi->speed = speed; 3024 3025 virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); 3026 3027 if (ethtool_validate_duplex(duplex)) 3028 vi->duplex = duplex; 3029 } 3030 3031 static u32 virtnet_get_rxfh_key_size(struct net_device *dev) 3032 { 3033 return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size; 3034 } 3035 3036 static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) 3037 { 3038 return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; 3039 } 3040 3041 static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) 3042 { 3043 struct virtnet_info *vi = netdev_priv(dev); 3044 int i; 3045 3046 if (indir) { 3047 for (i = 0; i < vi->rss_indir_table_size; ++i) 3048 indir[i] = vi->ctrl->rss.indirection_table[i]; 3049 } 3050 3051 if (key) 3052 memcpy(key, vi->ctrl->rss.key, vi->rss_key_size); 3053 3054 if (hfunc) 3055 *hfunc = ETH_RSS_HASH_TOP; 3056 3057 return 0; 3058 } 3059 3060 static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc) 3061 { 3062 struct virtnet_info *vi = netdev_priv(dev); 3063 int i; 3064 3065 if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) 3066 return -EOPNOTSUPP; 3067 3068 if (indir) { 3069 for (i = 0; i < vi->rss_indir_table_size; ++i) 3070 vi->ctrl->rss.indirection_table[i] = indir[i]; 3071 } 3072 if (key) 3073 memcpy(vi->ctrl->rss.key, key, vi->rss_key_size); 3074 3075 virtnet_commit_rss_command(vi); 3076 3077 return 0; 3078 } 3079 3080 static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs) 3081 { 3082 struct virtnet_info *vi = netdev_priv(dev); 3083 int rc = 0; 3084 3085 switch (info->cmd) { 3086 case ETHTOOL_GRXRINGS: 3087 info->data = vi->curr_queue_pairs; 3088 break; 3089 case ETHTOOL_GRXFH: 3090 virtnet_get_hashflow(vi, info); 3091 break; 3092 default: 3093 rc = -EOPNOTSUPP; 3094 } 3095 3096 return rc; 3097 } 3098 3099 static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) 3100 { 3101 struct virtnet_info *vi = netdev_priv(dev); 3102 int rc = 0; 3103 3104 switch (info->cmd) { 3105 case ETHTOOL_SRXFH: 3106 if (!virtnet_set_hashflow(vi, info)) 3107 rc = -EINVAL; 3108 3109 break; 3110 default: 3111 rc = -EOPNOTSUPP; 3112 } 3113 3114 return rc; 3115 } 3116 3117 static const struct ethtool_ops virtnet_ethtool_ops = { 3118 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | 3119 ETHTOOL_COALESCE_USECS, 3120 .get_drvinfo = virtnet_get_drvinfo, 3121 .get_link = ethtool_op_get_link, 3122 .get_ringparam = virtnet_get_ringparam, 3123 .set_ringparam = virtnet_set_ringparam, 3124 .get_strings = virtnet_get_strings, 3125 .get_sset_count = virtnet_get_sset_count, 3126 .get_ethtool_stats = virtnet_get_ethtool_stats, 3127 .set_channels = virtnet_set_channels, 3128 .get_channels = virtnet_get_channels, 3129 .get_ts_info = ethtool_op_get_ts_info, 3130 .get_link_ksettings = virtnet_get_link_ksettings, 3131 .set_link_ksettings = virtnet_set_link_ksettings, 3132 .set_coalesce = virtnet_set_coalesce, 3133 .get_coalesce = virtnet_get_coalesce, 3134 .get_rxfh_key_size = virtnet_get_rxfh_key_size, 3135 .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, 3136 .get_rxfh = virtnet_get_rxfh, 3137 .set_rxfh = virtnet_set_rxfh, 3138 .get_rxnfc = virtnet_get_rxnfc, 3139 .set_rxnfc = virtnet_set_rxnfc, 3140 }; 3141 3142 static void virtnet_freeze_down(struct virtio_device *vdev) 3143 { 3144 struct virtnet_info *vi = vdev->priv; 3145 3146 /* Make sure no work handler is accessing the device */ 3147 flush_work(&vi->config_work); 3148 3149 netif_tx_lock_bh(vi->dev); 3150 netif_device_detach(vi->dev); 3151 netif_tx_unlock_bh(vi->dev); 3152 if (netif_running(vi->dev)) 3153 virtnet_close(vi->dev); 3154 } 3155 3156 static int init_vqs(struct virtnet_info *vi); 3157 3158 static int virtnet_restore_up(struct virtio_device *vdev) 3159 { 3160 struct virtnet_info *vi = vdev->priv; 3161 int err; 3162 3163 err = init_vqs(vi); 3164 if (err) 3165 return err; 3166 3167 virtio_device_ready(vdev); 3168 3169 enable_delayed_refill(vi); 3170 3171 if (netif_running(vi->dev)) { 3172 err = virtnet_open(vi->dev); 3173 if (err) 3174 return err; 3175 } 3176 3177 netif_tx_lock_bh(vi->dev); 3178 netif_device_attach(vi->dev); 3179 netif_tx_unlock_bh(vi->dev); 3180 return err; 3181 } 3182 3183 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 3184 { 3185 struct scatterlist sg; 3186 vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads); 3187 3188 sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads)); 3189 3190 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 3191 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 3192 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); 3193 return -EINVAL; 3194 } 3195 3196 return 0; 3197 } 3198 3199 static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 3200 { 3201 u64 offloads = 0; 3202 3203 if (!vi->guest_offloads) 3204 return 0; 3205 3206 return virtnet_set_guest_offloads(vi, offloads); 3207 } 3208 3209 static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 3210 { 3211 u64 offloads = vi->guest_offloads; 3212 3213 if (!vi->guest_offloads) 3214 return 0; 3215 3216 return virtnet_set_guest_offloads(vi, offloads); 3217 } 3218 3219 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 3220 struct netlink_ext_ack *extack) 3221 { 3222 unsigned int room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM + 3223 sizeof(struct skb_shared_info)); 3224 unsigned int max_sz = PAGE_SIZE - room - ETH_HLEN; 3225 struct virtnet_info *vi = netdev_priv(dev); 3226 struct bpf_prog *old_prog; 3227 u16 xdp_qp = 0, curr_qp; 3228 int i, err; 3229 3230 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 3231 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 3232 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 3233 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 3234 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 3235 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) || 3236 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) || 3237 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) { 3238 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); 3239 return -EOPNOTSUPP; 3240 } 3241 3242 if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 3243 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 3244 return -EINVAL; 3245 } 3246 3247 if (prog && !prog->aux->xdp_has_frags && dev->mtu > max_sz) { 3248 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP without frags"); 3249 netdev_warn(dev, "single-buffer XDP requires MTU less than %u\n", max_sz); 3250 return -EINVAL; 3251 } 3252 3253 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 3254 if (prog) 3255 xdp_qp = nr_cpu_ids; 3256 3257 /* XDP requires extra queues for XDP_TX */ 3258 if (curr_qp + xdp_qp > vi->max_queue_pairs) { 3259 netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n", 3260 curr_qp + xdp_qp, vi->max_queue_pairs); 3261 xdp_qp = 0; 3262 } 3263 3264 old_prog = rtnl_dereference(vi->rq[0].xdp_prog); 3265 if (!prog && !old_prog) 3266 return 0; 3267 3268 if (prog) 3269 bpf_prog_add(prog, vi->max_queue_pairs - 1); 3270 3271 /* Make sure NAPI is not using any XDP TX queues for RX. */ 3272 if (netif_running(dev)) { 3273 for (i = 0; i < vi->max_queue_pairs; i++) { 3274 napi_disable(&vi->rq[i].napi); 3275 virtnet_napi_tx_disable(&vi->sq[i].napi); 3276 } 3277 } 3278 3279 if (!prog) { 3280 for (i = 0; i < vi->max_queue_pairs; i++) { 3281 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 3282 if (i == 0) 3283 virtnet_restore_guest_offloads(vi); 3284 } 3285 synchronize_net(); 3286 } 3287 3288 err = _virtnet_set_queues(vi, curr_qp + xdp_qp); 3289 if (err) 3290 goto err; 3291 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 3292 vi->xdp_queue_pairs = xdp_qp; 3293 3294 if (prog) { 3295 vi->xdp_enabled = true; 3296 for (i = 0; i < vi->max_queue_pairs; i++) { 3297 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 3298 if (i == 0 && !old_prog) 3299 virtnet_clear_guest_offloads(vi); 3300 } 3301 if (!old_prog) 3302 xdp_features_set_redirect_target(dev, true); 3303 } else { 3304 xdp_features_clear_redirect_target(dev); 3305 vi->xdp_enabled = false; 3306 } 3307 3308 for (i = 0; i < vi->max_queue_pairs; i++) { 3309 if (old_prog) 3310 bpf_prog_put(old_prog); 3311 if (netif_running(dev)) { 3312 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 3313 virtnet_napi_tx_enable(vi, vi->sq[i].vq, 3314 &vi->sq[i].napi); 3315 } 3316 } 3317 3318 return 0; 3319 3320 err: 3321 if (!prog) { 3322 virtnet_clear_guest_offloads(vi); 3323 for (i = 0; i < vi->max_queue_pairs; i++) 3324 rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); 3325 } 3326 3327 if (netif_running(dev)) { 3328 for (i = 0; i < vi->max_queue_pairs; i++) { 3329 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 3330 virtnet_napi_tx_enable(vi, vi->sq[i].vq, 3331 &vi->sq[i].napi); 3332 } 3333 } 3334 if (prog) 3335 bpf_prog_sub(prog, vi->max_queue_pairs - 1); 3336 return err; 3337 } 3338 3339 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 3340 { 3341 switch (xdp->command) { 3342 case XDP_SETUP_PROG: 3343 return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 3344 default: 3345 return -EINVAL; 3346 } 3347 } 3348 3349 static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, 3350 size_t len) 3351 { 3352 struct virtnet_info *vi = netdev_priv(dev); 3353 int ret; 3354 3355 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 3356 return -EOPNOTSUPP; 3357 3358 ret = snprintf(buf, len, "sby"); 3359 if (ret >= len) 3360 return -EOPNOTSUPP; 3361 3362 return 0; 3363 } 3364 3365 static int virtnet_set_features(struct net_device *dev, 3366 netdev_features_t features) 3367 { 3368 struct virtnet_info *vi = netdev_priv(dev); 3369 u64 offloads; 3370 int err; 3371 3372 if ((dev->features ^ features) & NETIF_F_GRO_HW) { 3373 if (vi->xdp_enabled) 3374 return -EBUSY; 3375 3376 if (features & NETIF_F_GRO_HW) 3377 offloads = vi->guest_offloads_capable; 3378 else 3379 offloads = vi->guest_offloads_capable & 3380 ~GUEST_OFFLOAD_GRO_HW_MASK; 3381 3382 err = virtnet_set_guest_offloads(vi, offloads); 3383 if (err) 3384 return err; 3385 vi->guest_offloads = offloads; 3386 } 3387 3388 if ((dev->features ^ features) & NETIF_F_RXHASH) { 3389 if (features & NETIF_F_RXHASH) 3390 vi->ctrl->rss.hash_types = vi->rss_hash_types_saved; 3391 else 3392 vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE; 3393 3394 if (!virtnet_commit_rss_command(vi)) 3395 return -EINVAL; 3396 } 3397 3398 return 0; 3399 } 3400 3401 static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) 3402 { 3403 struct virtnet_info *priv = netdev_priv(dev); 3404 struct send_queue *sq = &priv->sq[txqueue]; 3405 struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); 3406 3407 u64_stats_update_begin(&sq->stats.syncp); 3408 sq->stats.tx_timeouts++; 3409 u64_stats_update_end(&sq->stats.syncp); 3410 3411 netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", 3412 txqueue, sq->name, sq->vq->index, sq->vq->name, 3413 jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); 3414 } 3415 3416 static const struct net_device_ops virtnet_netdev = { 3417 .ndo_open = virtnet_open, 3418 .ndo_stop = virtnet_close, 3419 .ndo_start_xmit = start_xmit, 3420 .ndo_validate_addr = eth_validate_addr, 3421 .ndo_set_mac_address = virtnet_set_mac_address, 3422 .ndo_set_rx_mode = virtnet_set_rx_mode, 3423 .ndo_get_stats64 = virtnet_stats, 3424 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 3425 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 3426 .ndo_bpf = virtnet_xdp, 3427 .ndo_xdp_xmit = virtnet_xdp_xmit, 3428 .ndo_features_check = passthru_features_check, 3429 .ndo_get_phys_port_name = virtnet_get_phys_port_name, 3430 .ndo_set_features = virtnet_set_features, 3431 .ndo_tx_timeout = virtnet_tx_timeout, 3432 }; 3433 3434 static void virtnet_config_changed_work(struct work_struct *work) 3435 { 3436 struct virtnet_info *vi = 3437 container_of(work, struct virtnet_info, config_work); 3438 u16 v; 3439 3440 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 3441 struct virtio_net_config, status, &v) < 0) 3442 return; 3443 3444 if (v & VIRTIO_NET_S_ANNOUNCE) { 3445 netdev_notify_peers(vi->dev); 3446 virtnet_ack_link_announce(vi); 3447 } 3448 3449 /* Ignore unknown (future) status bits */ 3450 v &= VIRTIO_NET_S_LINK_UP; 3451 3452 if (vi->status == v) 3453 return; 3454 3455 vi->status = v; 3456 3457 if (vi->status & VIRTIO_NET_S_LINK_UP) { 3458 virtnet_update_settings(vi); 3459 netif_carrier_on(vi->dev); 3460 netif_tx_wake_all_queues(vi->dev); 3461 } else { 3462 netif_carrier_off(vi->dev); 3463 netif_tx_stop_all_queues(vi->dev); 3464 } 3465 } 3466 3467 static void virtnet_config_changed(struct virtio_device *vdev) 3468 { 3469 struct virtnet_info *vi = vdev->priv; 3470 3471 schedule_work(&vi->config_work); 3472 } 3473 3474 static void virtnet_free_queues(struct virtnet_info *vi) 3475 { 3476 int i; 3477 3478 for (i = 0; i < vi->max_queue_pairs; i++) { 3479 __netif_napi_del(&vi->rq[i].napi); 3480 __netif_napi_del(&vi->sq[i].napi); 3481 } 3482 3483 /* We called __netif_napi_del(), 3484 * we need to respect an RCU grace period before freeing vi->rq 3485 */ 3486 synchronize_net(); 3487 3488 kfree(vi->rq); 3489 kfree(vi->sq); 3490 kfree(vi->ctrl); 3491 } 3492 3493 static void _free_receive_bufs(struct virtnet_info *vi) 3494 { 3495 struct bpf_prog *old_prog; 3496 int i; 3497 3498 for (i = 0; i < vi->max_queue_pairs; i++) { 3499 while (vi->rq[i].pages) 3500 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 3501 3502 old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 3503 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 3504 if (old_prog) 3505 bpf_prog_put(old_prog); 3506 } 3507 } 3508 3509 static void free_receive_bufs(struct virtnet_info *vi) 3510 { 3511 rtnl_lock(); 3512 _free_receive_bufs(vi); 3513 rtnl_unlock(); 3514 } 3515 3516 static void free_receive_page_frags(struct virtnet_info *vi) 3517 { 3518 int i; 3519 for (i = 0; i < vi->max_queue_pairs; i++) 3520 if (vi->rq[i].alloc_frag.page) 3521 put_page(vi->rq[i].alloc_frag.page); 3522 } 3523 3524 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf) 3525 { 3526 if (!is_xdp_frame(buf)) 3527 dev_kfree_skb(buf); 3528 else 3529 xdp_return_frame(ptr_to_xdp(buf)); 3530 } 3531 3532 static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf) 3533 { 3534 struct virtnet_info *vi = vq->vdev->priv; 3535 int i = vq2rxq(vq); 3536 3537 if (vi->mergeable_rx_bufs) 3538 put_page(virt_to_head_page(buf)); 3539 else if (vi->big_packets) 3540 give_pages(&vi->rq[i], buf); 3541 else 3542 put_page(virt_to_head_page(buf)); 3543 } 3544 3545 static void free_unused_bufs(struct virtnet_info *vi) 3546 { 3547 void *buf; 3548 int i; 3549 3550 for (i = 0; i < vi->max_queue_pairs; i++) { 3551 struct virtqueue *vq = vi->sq[i].vq; 3552 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 3553 virtnet_sq_free_unused_buf(vq, buf); 3554 } 3555 3556 for (i = 0; i < vi->max_queue_pairs; i++) { 3557 struct virtqueue *vq = vi->rq[i].vq; 3558 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 3559 virtnet_rq_free_unused_buf(vq, buf); 3560 } 3561 } 3562 3563 static void virtnet_del_vqs(struct virtnet_info *vi) 3564 { 3565 struct virtio_device *vdev = vi->vdev; 3566 3567 virtnet_clean_affinity(vi); 3568 3569 vdev->config->del_vqs(vdev); 3570 3571 virtnet_free_queues(vi); 3572 } 3573 3574 /* How large should a single buffer be so a queue full of these can fit at 3575 * least one full packet? 3576 * Logic below assumes the mergeable buffer header is used. 3577 */ 3578 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 3579 { 3580 const unsigned int hdr_len = vi->hdr_len; 3581 unsigned int rq_size = virtqueue_get_vring_size(vq); 3582 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 3583 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 3584 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 3585 3586 return max(max(min_buf_len, hdr_len) - hdr_len, 3587 (unsigned int)GOOD_PACKET_LEN); 3588 } 3589 3590 static int virtnet_find_vqs(struct virtnet_info *vi) 3591 { 3592 vq_callback_t **callbacks; 3593 struct virtqueue **vqs; 3594 int ret = -ENOMEM; 3595 int i, total_vqs; 3596 const char **names; 3597 bool *ctx; 3598 3599 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 3600 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 3601 * possible control vq. 3602 */ 3603 total_vqs = vi->max_queue_pairs * 2 + 3604 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 3605 3606 /* Allocate space for find_vqs parameters */ 3607 vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); 3608 if (!vqs) 3609 goto err_vq; 3610 callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL); 3611 if (!callbacks) 3612 goto err_callback; 3613 names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL); 3614 if (!names) 3615 goto err_names; 3616 if (!vi->big_packets || vi->mergeable_rx_bufs) { 3617 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); 3618 if (!ctx) 3619 goto err_ctx; 3620 } else { 3621 ctx = NULL; 3622 } 3623 3624 /* Parameters for control virtqueue, if any */ 3625 if (vi->has_cvq) { 3626 callbacks[total_vqs - 1] = NULL; 3627 names[total_vqs - 1] = "control"; 3628 } 3629 3630 /* Allocate/initialize parameters for send/receive virtqueues */ 3631 for (i = 0; i < vi->max_queue_pairs; i++) { 3632 callbacks[rxq2vq(i)] = skb_recv_done; 3633 callbacks[txq2vq(i)] = skb_xmit_done; 3634 sprintf(vi->rq[i].name, "input.%d", i); 3635 sprintf(vi->sq[i].name, "output.%d", i); 3636 names[rxq2vq(i)] = vi->rq[i].name; 3637 names[txq2vq(i)] = vi->sq[i].name; 3638 if (ctx) 3639 ctx[rxq2vq(i)] = true; 3640 } 3641 3642 ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks, 3643 names, ctx, NULL); 3644 if (ret) 3645 goto err_find; 3646 3647 if (vi->has_cvq) { 3648 vi->cvq = vqs[total_vqs - 1]; 3649 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 3650 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 3651 } 3652 3653 for (i = 0; i < vi->max_queue_pairs; i++) { 3654 vi->rq[i].vq = vqs[rxq2vq(i)]; 3655 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 3656 vi->sq[i].vq = vqs[txq2vq(i)]; 3657 } 3658 3659 /* run here: ret == 0. */ 3660 3661 3662 err_find: 3663 kfree(ctx); 3664 err_ctx: 3665 kfree(names); 3666 err_names: 3667 kfree(callbacks); 3668 err_callback: 3669 kfree(vqs); 3670 err_vq: 3671 return ret; 3672 } 3673 3674 static int virtnet_alloc_queues(struct virtnet_info *vi) 3675 { 3676 int i; 3677 3678 if (vi->has_cvq) { 3679 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 3680 if (!vi->ctrl) 3681 goto err_ctrl; 3682 } else { 3683 vi->ctrl = NULL; 3684 } 3685 vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL); 3686 if (!vi->sq) 3687 goto err_sq; 3688 vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL); 3689 if (!vi->rq) 3690 goto err_rq; 3691 3692 INIT_DELAYED_WORK(&vi->refill, refill_work); 3693 for (i = 0; i < vi->max_queue_pairs; i++) { 3694 vi->rq[i].pages = NULL; 3695 netif_napi_add_weight(vi->dev, &vi->rq[i].napi, virtnet_poll, 3696 napi_weight); 3697 netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi, 3698 virtnet_poll_tx, 3699 napi_tx ? napi_weight : 0); 3700 3701 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 3702 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 3703 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 3704 3705 u64_stats_init(&vi->rq[i].stats.syncp); 3706 u64_stats_init(&vi->sq[i].stats.syncp); 3707 } 3708 3709 return 0; 3710 3711 err_rq: 3712 kfree(vi->sq); 3713 err_sq: 3714 kfree(vi->ctrl); 3715 err_ctrl: 3716 return -ENOMEM; 3717 } 3718 3719 static int init_vqs(struct virtnet_info *vi) 3720 { 3721 int ret; 3722 3723 /* Allocate send & receive queues */ 3724 ret = virtnet_alloc_queues(vi); 3725 if (ret) 3726 goto err; 3727 3728 ret = virtnet_find_vqs(vi); 3729 if (ret) 3730 goto err_free; 3731 3732 cpus_read_lock(); 3733 virtnet_set_affinity(vi); 3734 cpus_read_unlock(); 3735 3736 return 0; 3737 3738 err_free: 3739 virtnet_free_queues(vi); 3740 err: 3741 return ret; 3742 } 3743 3744 #ifdef CONFIG_SYSFS 3745 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 3746 char *buf) 3747 { 3748 struct virtnet_info *vi = netdev_priv(queue->dev); 3749 unsigned int queue_index = get_netdev_rx_queue_index(queue); 3750 unsigned int headroom = virtnet_get_headroom(vi); 3751 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 3752 struct ewma_pkt_len *avg; 3753 3754 BUG_ON(queue_index >= vi->max_queue_pairs); 3755 avg = &vi->rq[queue_index].mrg_avg_pkt_len; 3756 return sprintf(buf, "%u\n", 3757 get_mergeable_buf_len(&vi->rq[queue_index], avg, 3758 SKB_DATA_ALIGN(headroom + tailroom))); 3759 } 3760 3761 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 3762 __ATTR_RO(mergeable_rx_buffer_size); 3763 3764 static struct attribute *virtio_net_mrg_rx_attrs[] = { 3765 &mergeable_rx_buffer_size_attribute.attr, 3766 NULL 3767 }; 3768 3769 static const struct attribute_group virtio_net_mrg_rx_group = { 3770 .name = "virtio_net", 3771 .attrs = virtio_net_mrg_rx_attrs 3772 }; 3773 #endif 3774 3775 static bool virtnet_fail_on_feature(struct virtio_device *vdev, 3776 unsigned int fbit, 3777 const char *fname, const char *dname) 3778 { 3779 if (!virtio_has_feature(vdev, fbit)) 3780 return false; 3781 3782 dev_err(&vdev->dev, "device advertises feature %s but not %s", 3783 fname, dname); 3784 3785 return true; 3786 } 3787 3788 #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 3789 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 3790 3791 static bool virtnet_validate_features(struct virtio_device *vdev) 3792 { 3793 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 3794 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 3795 "VIRTIO_NET_F_CTRL_VQ") || 3796 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 3797 "VIRTIO_NET_F_CTRL_VQ") || 3798 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 3799 "VIRTIO_NET_F_CTRL_VQ") || 3800 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 3801 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 3802 "VIRTIO_NET_F_CTRL_VQ") || 3803 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, 3804 "VIRTIO_NET_F_CTRL_VQ") || 3805 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, 3806 "VIRTIO_NET_F_CTRL_VQ") || 3807 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, 3808 "VIRTIO_NET_F_CTRL_VQ"))) { 3809 return false; 3810 } 3811 3812 return true; 3813 } 3814 3815 #define MIN_MTU ETH_MIN_MTU 3816 #define MAX_MTU ETH_MAX_MTU 3817 3818 static int virtnet_validate(struct virtio_device *vdev) 3819 { 3820 if (!vdev->config->get) { 3821 dev_err(&vdev->dev, "%s failure: config access disabled\n", 3822 __func__); 3823 return -EINVAL; 3824 } 3825 3826 if (!virtnet_validate_features(vdev)) 3827 return -EINVAL; 3828 3829 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 3830 int mtu = virtio_cread16(vdev, 3831 offsetof(struct virtio_net_config, 3832 mtu)); 3833 if (mtu < MIN_MTU) 3834 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 3835 } 3836 3837 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY) && 3838 !virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 3839 dev_warn(&vdev->dev, "device advertises feature VIRTIO_NET_F_STANDBY but not VIRTIO_NET_F_MAC, disabling standby"); 3840 __virtio_clear_bit(vdev, VIRTIO_NET_F_STANDBY); 3841 } 3842 3843 return 0; 3844 } 3845 3846 static bool virtnet_check_guest_gso(const struct virtnet_info *vi) 3847 { 3848 return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 3849 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 3850 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 3851 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 3852 (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) && 3853 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6)); 3854 } 3855 3856 static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) 3857 { 3858 bool guest_gso = virtnet_check_guest_gso(vi); 3859 3860 /* If device can receive ANY guest GSO packets, regardless of mtu, 3861 * allocate packets of maximum size, otherwise limit it to only 3862 * mtu size worth only. 3863 */ 3864 if (mtu > ETH_DATA_LEN || guest_gso) { 3865 vi->big_packets = true; 3866 vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE); 3867 } 3868 } 3869 3870 static int virtnet_probe(struct virtio_device *vdev) 3871 { 3872 int i, err = -ENOMEM; 3873 struct net_device *dev; 3874 struct virtnet_info *vi; 3875 u16 max_queue_pairs; 3876 int mtu = 0; 3877 3878 /* Find if host supports multiqueue/rss virtio_net device */ 3879 max_queue_pairs = 1; 3880 if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 3881 max_queue_pairs = 3882 virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); 3883 3884 /* We need at least 2 queue's */ 3885 if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 3886 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 3887 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 3888 max_queue_pairs = 1; 3889 3890 /* Allocate ourselves a network device with room for our info */ 3891 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 3892 if (!dev) 3893 return -ENOMEM; 3894 3895 /* Set up network device as normal. */ 3896 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 3897 IFF_TX_SKB_NO_LINEAR; 3898 dev->netdev_ops = &virtnet_netdev; 3899 dev->features = NETIF_F_HIGHDMA; 3900 3901 dev->ethtool_ops = &virtnet_ethtool_ops; 3902 SET_NETDEV_DEV(dev, &vdev->dev); 3903 3904 /* Do we support "hardware" checksums? */ 3905 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 3906 /* This opens up the world of extra features. */ 3907 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 3908 if (csum) 3909 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 3910 3911 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 3912 dev->hw_features |= NETIF_F_TSO 3913 | NETIF_F_TSO_ECN | NETIF_F_TSO6; 3914 } 3915 /* Individual feature bits: what can host handle? */ 3916 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 3917 dev->hw_features |= NETIF_F_TSO; 3918 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 3919 dev->hw_features |= NETIF_F_TSO6; 3920 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 3921 dev->hw_features |= NETIF_F_TSO_ECN; 3922 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO)) 3923 dev->hw_features |= NETIF_F_GSO_UDP_L4; 3924 3925 dev->features |= NETIF_F_GSO_ROBUST; 3926 3927 if (gso) 3928 dev->features |= dev->hw_features & NETIF_F_ALL_TSO; 3929 /* (!csum && gso) case will be fixed by register_netdev() */ 3930 } 3931 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM)) 3932 dev->features |= NETIF_F_RXCSUM; 3933 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 3934 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) 3935 dev->features |= NETIF_F_GRO_HW; 3936 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) 3937 dev->hw_features |= NETIF_F_GRO_HW; 3938 3939 dev->vlan_features = dev->features; 3940 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; 3941 3942 /* MTU range: 68 - 65535 */ 3943 dev->min_mtu = MIN_MTU; 3944 dev->max_mtu = MAX_MTU; 3945 3946 /* Configuration may specify what MAC to use. Otherwise random. */ 3947 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 3948 u8 addr[ETH_ALEN]; 3949 3950 virtio_cread_bytes(vdev, 3951 offsetof(struct virtio_net_config, mac), 3952 addr, ETH_ALEN); 3953 eth_hw_addr_set(dev, addr); 3954 } else { 3955 eth_hw_addr_random(dev); 3956 dev_info(&vdev->dev, "Assigned random MAC address %pM\n", 3957 dev->dev_addr); 3958 } 3959 3960 /* Set up our device-specific information */ 3961 vi = netdev_priv(dev); 3962 vi->dev = dev; 3963 vi->vdev = vdev; 3964 vdev->priv = vi; 3965 3966 INIT_WORK(&vi->config_work, virtnet_config_changed_work); 3967 spin_lock_init(&vi->refill_lock); 3968 3969 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { 3970 vi->mergeable_rx_bufs = true; 3971 dev->xdp_features |= NETDEV_XDP_ACT_RX_SG; 3972 } 3973 3974 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 3975 vi->rx_usecs = 0; 3976 vi->tx_usecs = 0; 3977 vi->tx_max_packets = 0; 3978 vi->rx_max_packets = 0; 3979 } 3980 3981 if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) 3982 vi->has_rss_hash_report = true; 3983 3984 if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 3985 vi->has_rss = true; 3986 3987 if (vi->has_rss || vi->has_rss_hash_report) { 3988 vi->rss_indir_table_size = 3989 virtio_cread16(vdev, offsetof(struct virtio_net_config, 3990 rss_max_indirection_table_length)); 3991 vi->rss_key_size = 3992 virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); 3993 3994 vi->rss_hash_types_supported = 3995 virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); 3996 vi->rss_hash_types_supported &= 3997 ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX | 3998 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 3999 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX); 4000 4001 dev->hw_features |= NETIF_F_RXHASH; 4002 } 4003 4004 if (vi->has_rss_hash_report) 4005 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash); 4006 else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 4007 virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 4008 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 4009 else 4010 vi->hdr_len = sizeof(struct virtio_net_hdr); 4011 4012 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 4013 virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 4014 vi->any_header_sg = true; 4015 4016 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 4017 vi->has_cvq = true; 4018 4019 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 4020 mtu = virtio_cread16(vdev, 4021 offsetof(struct virtio_net_config, 4022 mtu)); 4023 if (mtu < dev->min_mtu) { 4024 /* Should never trigger: MTU was previously validated 4025 * in virtnet_validate. 4026 */ 4027 dev_err(&vdev->dev, 4028 "device MTU appears to have changed it is now %d < %d", 4029 mtu, dev->min_mtu); 4030 err = -EINVAL; 4031 goto free; 4032 } 4033 4034 dev->mtu = mtu; 4035 dev->max_mtu = mtu; 4036 } 4037 4038 virtnet_set_big_packets(vi, mtu); 4039 4040 if (vi->any_header_sg) 4041 dev->needed_headroom = vi->hdr_len; 4042 4043 /* Enable multiqueue by default */ 4044 if (num_online_cpus() >= max_queue_pairs) 4045 vi->curr_queue_pairs = max_queue_pairs; 4046 else 4047 vi->curr_queue_pairs = num_online_cpus(); 4048 vi->max_queue_pairs = max_queue_pairs; 4049 4050 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 4051 err = init_vqs(vi); 4052 if (err) 4053 goto free; 4054 4055 #ifdef CONFIG_SYSFS 4056 if (vi->mergeable_rx_bufs) 4057 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 4058 #endif 4059 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 4060 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 4061 4062 virtnet_init_settings(dev); 4063 4064 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 4065 vi->failover = net_failover_create(vi->dev); 4066 if (IS_ERR(vi->failover)) { 4067 err = PTR_ERR(vi->failover); 4068 goto free_vqs; 4069 } 4070 } 4071 4072 if (vi->has_rss || vi->has_rss_hash_report) 4073 virtnet_init_default_rss(vi); 4074 4075 /* serialize netdev register + virtio_device_ready() with ndo_open() */ 4076 rtnl_lock(); 4077 4078 err = register_netdevice(dev); 4079 if (err) { 4080 pr_debug("virtio_net: registering device failed\n"); 4081 rtnl_unlock(); 4082 goto free_failover; 4083 } 4084 4085 virtio_device_ready(vdev); 4086 4087 /* a random MAC address has been assigned, notify the device. 4088 * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there 4089 * because many devices work fine without getting MAC explicitly 4090 */ 4091 if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 4092 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 4093 struct scatterlist sg; 4094 4095 sg_init_one(&sg, dev->dev_addr, dev->addr_len); 4096 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 4097 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 4098 pr_debug("virtio_net: setting MAC address failed\n"); 4099 rtnl_unlock(); 4100 err = -EINVAL; 4101 goto free_unregister_netdev; 4102 } 4103 } 4104 4105 rtnl_unlock(); 4106 4107 err = virtnet_cpu_notif_add(vi); 4108 if (err) { 4109 pr_debug("virtio_net: registering cpu notifier failed\n"); 4110 goto free_unregister_netdev; 4111 } 4112 4113 virtnet_set_queues(vi, vi->curr_queue_pairs); 4114 4115 /* Assume link up if device can't report link status, 4116 otherwise get link status from config. */ 4117 netif_carrier_off(dev); 4118 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 4119 schedule_work(&vi->config_work); 4120 } else { 4121 vi->status = VIRTIO_NET_S_LINK_UP; 4122 virtnet_update_settings(vi); 4123 netif_carrier_on(dev); 4124 } 4125 4126 for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) 4127 if (virtio_has_feature(vi->vdev, guest_offloads[i])) 4128 set_bit(guest_offloads[i], &vi->guest_offloads); 4129 vi->guest_offloads_capable = vi->guest_offloads; 4130 4131 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 4132 dev->name, max_queue_pairs); 4133 4134 return 0; 4135 4136 free_unregister_netdev: 4137 unregister_netdev(dev); 4138 free_failover: 4139 net_failover_destroy(vi->failover); 4140 free_vqs: 4141 virtio_reset_device(vdev); 4142 cancel_delayed_work_sync(&vi->refill); 4143 free_receive_page_frags(vi); 4144 virtnet_del_vqs(vi); 4145 free: 4146 free_netdev(dev); 4147 return err; 4148 } 4149 4150 static void remove_vq_common(struct virtnet_info *vi) 4151 { 4152 virtio_reset_device(vi->vdev); 4153 4154 /* Free unused buffers in both send and recv, if any. */ 4155 free_unused_bufs(vi); 4156 4157 free_receive_bufs(vi); 4158 4159 free_receive_page_frags(vi); 4160 4161 virtnet_del_vqs(vi); 4162 } 4163 4164 static void virtnet_remove(struct virtio_device *vdev) 4165 { 4166 struct virtnet_info *vi = vdev->priv; 4167 4168 virtnet_cpu_notif_remove(vi); 4169 4170 /* Make sure no work handler is accessing the device. */ 4171 flush_work(&vi->config_work); 4172 4173 unregister_netdev(vi->dev); 4174 4175 net_failover_destroy(vi->failover); 4176 4177 remove_vq_common(vi); 4178 4179 free_netdev(vi->dev); 4180 } 4181 4182 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 4183 { 4184 struct virtnet_info *vi = vdev->priv; 4185 4186 virtnet_cpu_notif_remove(vi); 4187 virtnet_freeze_down(vdev); 4188 remove_vq_common(vi); 4189 4190 return 0; 4191 } 4192 4193 static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 4194 { 4195 struct virtnet_info *vi = vdev->priv; 4196 int err; 4197 4198 err = virtnet_restore_up(vdev); 4199 if (err) 4200 return err; 4201 virtnet_set_queues(vi, vi->curr_queue_pairs); 4202 4203 err = virtnet_cpu_notif_add(vi); 4204 if (err) { 4205 virtnet_freeze_down(vdev); 4206 remove_vq_common(vi); 4207 return err; 4208 } 4209 4210 return 0; 4211 } 4212 4213 static struct virtio_device_id id_table[] = { 4214 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 4215 { 0 }, 4216 }; 4217 4218 #define VIRTNET_FEATURES \ 4219 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 4220 VIRTIO_NET_F_MAC, \ 4221 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 4222 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 4223 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 4224 VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \ 4225 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 4226 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 4227 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 4228 VIRTIO_NET_F_CTRL_MAC_ADDR, \ 4229 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 4230 VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ 4231 VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL 4232 4233 static unsigned int features[] = { 4234 VIRTNET_FEATURES, 4235 }; 4236 4237 static unsigned int features_legacy[] = { 4238 VIRTNET_FEATURES, 4239 VIRTIO_NET_F_GSO, 4240 VIRTIO_F_ANY_LAYOUT, 4241 }; 4242 4243 static struct virtio_driver virtio_net_driver = { 4244 .feature_table = features, 4245 .feature_table_size = ARRAY_SIZE(features), 4246 .feature_table_legacy = features_legacy, 4247 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 4248 .driver.name = KBUILD_MODNAME, 4249 .driver.owner = THIS_MODULE, 4250 .id_table = id_table, 4251 .validate = virtnet_validate, 4252 .probe = virtnet_probe, 4253 .remove = virtnet_remove, 4254 .config_changed = virtnet_config_changed, 4255 #ifdef CONFIG_PM_SLEEP 4256 .freeze = virtnet_freeze, 4257 .restore = virtnet_restore, 4258 #endif 4259 }; 4260 4261 static __init int virtio_net_driver_init(void) 4262 { 4263 int ret; 4264 4265 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 4266 virtnet_cpu_online, 4267 virtnet_cpu_down_prep); 4268 if (ret < 0) 4269 goto out; 4270 virtionet_online = ret; 4271 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 4272 NULL, virtnet_cpu_dead); 4273 if (ret) 4274 goto err_dead; 4275 ret = register_virtio_driver(&virtio_net_driver); 4276 if (ret) 4277 goto err_virtio; 4278 return 0; 4279 err_virtio: 4280 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 4281 err_dead: 4282 cpuhp_remove_multi_state(virtionet_online); 4283 out: 4284 return ret; 4285 } 4286 module_init(virtio_net_driver_init); 4287 4288 static __exit void virtio_net_driver_exit(void) 4289 { 4290 unregister_virtio_driver(&virtio_net_driver); 4291 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 4292 cpuhp_remove_multi_state(virtionet_online); 4293 } 4294 module_exit(virtio_net_driver_exit); 4295 4296 MODULE_DEVICE_TABLE(virtio, id_table); 4297 MODULE_DESCRIPTION("Virtio network driver"); 4298 MODULE_LICENSE("GPL"); 4299