11ccea77eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 248925e37SRusty Russell /* A network driver using virtio. 3296f96fcSRusty Russell * 4296f96fcSRusty Russell * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5296f96fcSRusty Russell */ 6296f96fcSRusty Russell //#define DEBUG 7296f96fcSRusty Russell #include <linux/netdevice.h> 8296f96fcSRusty Russell #include <linux/etherdevice.h> 9a9ea3fc6SHerbert Xu #include <linux/ethtool.h> 10296f96fcSRusty Russell #include <linux/module.h> 11296f96fcSRusty Russell #include <linux/virtio.h> 12296f96fcSRusty Russell #include <linux/virtio_net.h> 13f600b690SJohn Fastabend #include <linux/bpf.h> 14a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 15296f96fcSRusty Russell #include <linux/scatterlist.h> 16e918085aSAlex Williamson #include <linux/if_vlan.h> 175a0e3ad6STejun Heo #include <linux/slab.h> 188de4b2f3SWanlong Gao #include <linux/cpu.h> 19ab7db917SMichael Dalton #include <linux/average.h> 20186b3c99SJason Wang #include <linux/filter.h> 212ca653d6SCaleb Raitto #include <linux/kernel.h> 22d85b758fSMichael S. Tsirkin #include <net/route.h> 23754b8a21SJesper Dangaard Brouer #include <net/xdp.h> 24ba5e4426SSridhar Samudrala #include <net/net_failover.h> 25296f96fcSRusty Russell 26d34710e3SAmerigo Wang static int napi_weight = NAPI_POLL_WEIGHT; 276c0cd7c0SDor Laor module_param(napi_weight, int, 0444); 286c0cd7c0SDor Laor 2931c03aefSWillem de Bruijn static bool csum = true, gso = true, napi_tx = true; 3034a48579SRusty Russell module_param(csum, bool, 0444); 3134a48579SRusty Russell module_param(gso, bool, 0444); 32b92f1e67SWillem de Bruijn module_param(napi_tx, bool, 0644); 3334a48579SRusty Russell 34296f96fcSRusty Russell /* FIXME: MTU in config. */ 355061de36SMichael Dalton #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 363f2c31d9SMark McLoughlin #define GOOD_COPY_LEN 128 37296f96fcSRusty Russell 38f6b10209SJason Wang #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 39f6b10209SJason Wang 402de2f7f4SJohn Fastabend /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */ 412de2f7f4SJohn Fastabend #define VIRTIO_XDP_HEADROOM 256 422de2f7f4SJohn Fastabend 432471c75eSJesper Dangaard Brouer /* Separating two types of XDP xmit */ 442471c75eSJesper Dangaard Brouer #define VIRTIO_XDP_TX BIT(0) 452471c75eSJesper Dangaard Brouer #define VIRTIO_XDP_REDIR BIT(1) 462471c75eSJesper Dangaard Brouer 475050471dSToshiaki Makita #define VIRTIO_XDP_FLAG BIT(0) 485050471dSToshiaki Makita 495377d758SJohannes Berg /* RX packet size EWMA. The average packet size is used to determine the packet 505377d758SJohannes Berg * buffer size when refilling RX rings. As the entire RX ring may be refilled 515377d758SJohannes Berg * at once, the weight is chosen so that the EWMA will be insensitive to short- 525377d758SJohannes Berg * term, transient changes in packet size. 53ab7db917SMichael Dalton */ 54eb1e011aSJohannes Berg DECLARE_EWMA(pkt_len, 0, 64) 55ab7db917SMichael Dalton 5666846048SRick Jones #define VIRTNET_DRIVER_VERSION "1.0.0" 572a41f71dSAlex Williamson 587acd4329SColin Ian King static const unsigned long guest_offloads[] = { 597acd4329SColin Ian King VIRTIO_NET_F_GUEST_TSO4, 603f93522fSJason Wang VIRTIO_NET_F_GUEST_TSO6, 613f93522fSJason Wang VIRTIO_NET_F_GUEST_ECN, 62e59ff2c4SJason Wang VIRTIO_NET_F_GUEST_UFO, 63e59ff2c4SJason Wang VIRTIO_NET_F_GUEST_CSUM 647acd4329SColin Ian King }; 653f93522fSJason Wang 66dbcf24d1SJason Wang #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 671a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 681a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 691a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_UFO)) 701a03b8a3STonghao Zhang 71d7dfc5cfSToshiaki Makita struct virtnet_stat_desc { 72d7dfc5cfSToshiaki Makita char desc[ETH_GSTRING_LEN]; 73d7dfc5cfSToshiaki Makita size_t offset; 743fa2a1dfSstephen hemminger }; 753fa2a1dfSstephen hemminger 76d7dfc5cfSToshiaki Makita struct virtnet_sq_stats { 77d7dfc5cfSToshiaki Makita struct u64_stats_sync syncp; 78d7dfc5cfSToshiaki Makita u64 packets; 79d7dfc5cfSToshiaki Makita u64 bytes; 805b8f3c8dSToshiaki Makita u64 xdp_tx; 815b8f3c8dSToshiaki Makita u64 xdp_tx_drops; 82461f03dcSToshiaki Makita u64 kicks; 83d7dfc5cfSToshiaki Makita }; 84d7dfc5cfSToshiaki Makita 85d46eeeafSJason Wang struct virtnet_rq_stats { 86d46eeeafSJason Wang struct u64_stats_sync syncp; 87d7dfc5cfSToshiaki Makita u64 packets; 88d7dfc5cfSToshiaki Makita u64 bytes; 892c4a2f7dSToshiaki Makita u64 drops; 905b8f3c8dSToshiaki Makita u64 xdp_packets; 915b8f3c8dSToshiaki Makita u64 xdp_tx; 925b8f3c8dSToshiaki Makita u64 xdp_redirects; 935b8f3c8dSToshiaki Makita u64 xdp_drops; 94461f03dcSToshiaki Makita u64 kicks; 95d7dfc5cfSToshiaki Makita }; 96d7dfc5cfSToshiaki Makita 97d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m) 98d46eeeafSJason Wang #define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m) 99d7dfc5cfSToshiaki Makita 100d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 101d7dfc5cfSToshiaki Makita { "packets", VIRTNET_SQ_STAT(packets) }, 102d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_SQ_STAT(bytes) }, 1035b8f3c8dSToshiaki Makita { "xdp_tx", VIRTNET_SQ_STAT(xdp_tx) }, 1045b8f3c8dSToshiaki Makita { "xdp_tx_drops", VIRTNET_SQ_STAT(xdp_tx_drops) }, 105461f03dcSToshiaki Makita { "kicks", VIRTNET_SQ_STAT(kicks) }, 106d7dfc5cfSToshiaki Makita }; 107d7dfc5cfSToshiaki Makita 108d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 109d7dfc5cfSToshiaki Makita { "packets", VIRTNET_RQ_STAT(packets) }, 110d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_RQ_STAT(bytes) }, 1112c4a2f7dSToshiaki Makita { "drops", VIRTNET_RQ_STAT(drops) }, 1125b8f3c8dSToshiaki Makita { "xdp_packets", VIRTNET_RQ_STAT(xdp_packets) }, 1135b8f3c8dSToshiaki Makita { "xdp_tx", VIRTNET_RQ_STAT(xdp_tx) }, 1145b8f3c8dSToshiaki Makita { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) }, 1155b8f3c8dSToshiaki Makita { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) }, 116461f03dcSToshiaki Makita { "kicks", VIRTNET_RQ_STAT(kicks) }, 117d7dfc5cfSToshiaki Makita }; 118d7dfc5cfSToshiaki Makita 119d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc) 120d7dfc5cfSToshiaki Makita #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc) 121d7dfc5cfSToshiaki Makita 122e9d7417bSJason Wang /* Internal representation of a send virtqueue */ 123e9d7417bSJason Wang struct send_queue { 124e9d7417bSJason Wang /* Virtqueue associated with this send _queue */ 125e9d7417bSJason Wang struct virtqueue *vq; 126e9d7417bSJason Wang 127e9d7417bSJason Wang /* TX: fragments + linear part + virtio header */ 128e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 129986a4f4dSJason Wang 130986a4f4dSJason Wang /* Name of the send queue: output.$index */ 131986a4f4dSJason Wang char name[40]; 132b92f1e67SWillem de Bruijn 133d7dfc5cfSToshiaki Makita struct virtnet_sq_stats stats; 134d7dfc5cfSToshiaki Makita 135b92f1e67SWillem de Bruijn struct napi_struct napi; 136e9d7417bSJason Wang }; 137e9d7417bSJason Wang 138e9d7417bSJason Wang /* Internal representation of a receive virtqueue */ 139e9d7417bSJason Wang struct receive_queue { 140e9d7417bSJason Wang /* Virtqueue associated with this receive_queue */ 141e9d7417bSJason Wang struct virtqueue *vq; 142e9d7417bSJason Wang 143296f96fcSRusty Russell struct napi_struct napi; 144296f96fcSRusty Russell 145f600b690SJohn Fastabend struct bpf_prog __rcu *xdp_prog; 146f600b690SJohn Fastabend 147d7dfc5cfSToshiaki Makita struct virtnet_rq_stats stats; 148d7dfc5cfSToshiaki Makita 149e9d7417bSJason Wang /* Chain pages by the private ptr. */ 150e9d7417bSJason Wang struct page *pages; 151e9d7417bSJason Wang 152ab7db917SMichael Dalton /* Average packet length for mergeable receive buffers. */ 1535377d758SJohannes Berg struct ewma_pkt_len mrg_avg_pkt_len; 154ab7db917SMichael Dalton 155fb51879dSMichael Dalton /* Page frag for packet buffer allocation. */ 156fb51879dSMichael Dalton struct page_frag alloc_frag; 157fb51879dSMichael Dalton 158e9d7417bSJason Wang /* RX: fragments + linear part + virtio header */ 159e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 160986a4f4dSJason Wang 161d85b758fSMichael S. Tsirkin /* Min single buffer size for mergeable buffers case. */ 162d85b758fSMichael S. Tsirkin unsigned int min_buf_len; 163d85b758fSMichael S. Tsirkin 164986a4f4dSJason Wang /* Name of this receive queue: input.$index */ 165986a4f4dSJason Wang char name[40]; 166754b8a21SJesper Dangaard Brouer 167754b8a21SJesper Dangaard Brouer struct xdp_rxq_info xdp_rxq; 168e9d7417bSJason Wang }; 169e9d7417bSJason Wang 17012e57169SMichael S. Tsirkin /* Control VQ buffers: protected by the rtnl lock */ 17112e57169SMichael S. Tsirkin struct control_buf { 17212e57169SMichael S. Tsirkin struct virtio_net_ctrl_hdr hdr; 17312e57169SMichael S. Tsirkin virtio_net_ctrl_ack status; 17412e57169SMichael S. Tsirkin struct virtio_net_ctrl_mq mq; 17512e57169SMichael S. Tsirkin u8 promisc; 17612e57169SMichael S. Tsirkin u8 allmulti; 177d7fad4c8SMichael S. Tsirkin __virtio16 vid; 178f4ee703aSMichael S. Tsirkin __virtio64 offloads; 17912e57169SMichael S. Tsirkin }; 18012e57169SMichael S. Tsirkin 181e9d7417bSJason Wang struct virtnet_info { 182e9d7417bSJason Wang struct virtio_device *vdev; 183e9d7417bSJason Wang struct virtqueue *cvq; 184e9d7417bSJason Wang struct net_device *dev; 185986a4f4dSJason Wang struct send_queue *sq; 186986a4f4dSJason Wang struct receive_queue *rq; 187e9d7417bSJason Wang unsigned int status; 188e9d7417bSJason Wang 189986a4f4dSJason Wang /* Max # of queue pairs supported by the device */ 190986a4f4dSJason Wang u16 max_queue_pairs; 191986a4f4dSJason Wang 192986a4f4dSJason Wang /* # of queue pairs currently used by the driver */ 193986a4f4dSJason Wang u16 curr_queue_pairs; 194986a4f4dSJason Wang 195672aafd5SJohn Fastabend /* # of XDP queue pairs currently used by the driver */ 196672aafd5SJohn Fastabend u16 xdp_queue_pairs; 197672aafd5SJohn Fastabend 19897c2c69eSXuan Zhuo /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ 19997c2c69eSXuan Zhuo bool xdp_enabled; 20097c2c69eSXuan Zhuo 20197402b96SHerbert Xu /* I like... big packets and I cannot lie! */ 20297402b96SHerbert Xu bool big_packets; 20397402b96SHerbert Xu 2043f2c31d9SMark McLoughlin /* Host will merge rx buffers for big packets (shake it! shake it!) */ 2053f2c31d9SMark McLoughlin bool mergeable_rx_bufs; 2063f2c31d9SMark McLoughlin 207986a4f4dSJason Wang /* Has control virtqueue */ 208986a4f4dSJason Wang bool has_cvq; 209986a4f4dSJason Wang 210e7428e95SMichael S. Tsirkin /* Host can handle any s/g split between our header and packet data */ 211e7428e95SMichael S. Tsirkin bool any_header_sg; 212e7428e95SMichael S. Tsirkin 213012873d0SMichael S. Tsirkin /* Packet virtio header size */ 214012873d0SMichael S. Tsirkin u8 hdr_len; 215012873d0SMichael S. Tsirkin 2163161e453SRusty Russell /* Work struct for refilling if we run low on memory. */ 2173161e453SRusty Russell struct delayed_work refill; 2183161e453SRusty Russell 219586d17c5SJason Wang /* Work struct for config space updates */ 220586d17c5SJason Wang struct work_struct config_work; 221586d17c5SJason Wang 222986a4f4dSJason Wang /* Does the affinity hint is set for virtqueues? */ 223986a4f4dSJason Wang bool affinity_hint_set; 22447be2479SWanlong Gao 2258017c279SSebastian Andrzej Siewior /* CPU hotplug instances for online & dead */ 2268017c279SSebastian Andrzej Siewior struct hlist_node node; 2278017c279SSebastian Andrzej Siewior struct hlist_node node_dead; 2282ac46030SMichael S. Tsirkin 22912e57169SMichael S. Tsirkin struct control_buf *ctrl; 23016032be5SNikolay Aleksandrov 23116032be5SNikolay Aleksandrov /* Ethtool settings */ 23216032be5SNikolay Aleksandrov u8 duplex; 23316032be5SNikolay Aleksandrov u32 speed; 2343f93522fSJason Wang 2353f93522fSJason Wang unsigned long guest_offloads; 236a02e8964SWillem de Bruijn unsigned long guest_offloads_capable; 237ba5e4426SSridhar Samudrala 238ba5e4426SSridhar Samudrala /* failover when STANDBY feature enabled */ 239ba5e4426SSridhar Samudrala struct failover *failover; 240296f96fcSRusty Russell }; 241296f96fcSRusty Russell 2429ab86bbcSShirley Ma struct padded_vnet_hdr { 243012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf hdr; 2449ab86bbcSShirley Ma /* 245012873d0SMichael S. Tsirkin * hdr is in a separate sg buffer, and data sg buffer shares same page 246012873d0SMichael S. Tsirkin * with this header sg. This padding makes next sg 16 byte aligned 247012873d0SMichael S. Tsirkin * after the header. 2489ab86bbcSShirley Ma */ 249012873d0SMichael S. Tsirkin char padding[4]; 2509ab86bbcSShirley Ma }; 2519ab86bbcSShirley Ma 2525050471dSToshiaki Makita static bool is_xdp_frame(void *ptr) 2535050471dSToshiaki Makita { 2545050471dSToshiaki Makita return (unsigned long)ptr & VIRTIO_XDP_FLAG; 2555050471dSToshiaki Makita } 2565050471dSToshiaki Makita 2575050471dSToshiaki Makita static void *xdp_to_ptr(struct xdp_frame *ptr) 2585050471dSToshiaki Makita { 2595050471dSToshiaki Makita return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG); 2605050471dSToshiaki Makita } 2615050471dSToshiaki Makita 2625050471dSToshiaki Makita static struct xdp_frame *ptr_to_xdp(void *ptr) 2635050471dSToshiaki Makita { 2645050471dSToshiaki Makita return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); 2655050471dSToshiaki Makita } 2665050471dSToshiaki Makita 267986a4f4dSJason Wang /* Converting between virtqueue no. and kernel tx/rx queue no. 268986a4f4dSJason Wang * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 269986a4f4dSJason Wang */ 270986a4f4dSJason Wang static int vq2txq(struct virtqueue *vq) 271986a4f4dSJason Wang { 2729d0ca6edSRusty Russell return (vq->index - 1) / 2; 273986a4f4dSJason Wang } 274986a4f4dSJason Wang 275986a4f4dSJason Wang static int txq2vq(int txq) 276986a4f4dSJason Wang { 277986a4f4dSJason Wang return txq * 2 + 1; 278986a4f4dSJason Wang } 279986a4f4dSJason Wang 280986a4f4dSJason Wang static int vq2rxq(struct virtqueue *vq) 281986a4f4dSJason Wang { 2829d0ca6edSRusty Russell return vq->index / 2; 283986a4f4dSJason Wang } 284986a4f4dSJason Wang 285986a4f4dSJason Wang static int rxq2vq(int rxq) 286986a4f4dSJason Wang { 287986a4f4dSJason Wang return rxq * 2; 288986a4f4dSJason Wang } 289986a4f4dSJason Wang 290012873d0SMichael S. Tsirkin static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb) 291296f96fcSRusty Russell { 292012873d0SMichael S. Tsirkin return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb; 293296f96fcSRusty Russell } 294296f96fcSRusty Russell 2959ab86bbcSShirley Ma /* 2969ab86bbcSShirley Ma * private is used to chain pages for big packets, put the whole 2979ab86bbcSShirley Ma * most recent used list in the beginning for reuse 2989ab86bbcSShirley Ma */ 299e9d7417bSJason Wang static void give_pages(struct receive_queue *rq, struct page *page) 300fb6813f4SRusty Russell { 3019ab86bbcSShirley Ma struct page *end; 3029ab86bbcSShirley Ma 303e9d7417bSJason Wang /* Find end of list, sew whole thing into vi->rq.pages. */ 3049ab86bbcSShirley Ma for (end = page; end->private; end = (struct page *)end->private); 305e9d7417bSJason Wang end->private = (unsigned long)rq->pages; 306e9d7417bSJason Wang rq->pages = page; 307fb6813f4SRusty Russell } 308fb6813f4SRusty Russell 309e9d7417bSJason Wang static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 310fb6813f4SRusty Russell { 311e9d7417bSJason Wang struct page *p = rq->pages; 312fb6813f4SRusty Russell 3139ab86bbcSShirley Ma if (p) { 314e9d7417bSJason Wang rq->pages = (struct page *)p->private; 3159ab86bbcSShirley Ma /* clear private here, it is used to chain pages */ 3169ab86bbcSShirley Ma p->private = 0; 3179ab86bbcSShirley Ma } else 318fb6813f4SRusty Russell p = alloc_page(gfp_mask); 319fb6813f4SRusty Russell return p; 320fb6813f4SRusty Russell } 321fb6813f4SRusty Russell 322e4e8452aSWillem de Bruijn static void virtqueue_napi_schedule(struct napi_struct *napi, 323e4e8452aSWillem de Bruijn struct virtqueue *vq) 324e4e8452aSWillem de Bruijn { 325e4e8452aSWillem de Bruijn if (napi_schedule_prep(napi)) { 326e4e8452aSWillem de Bruijn virtqueue_disable_cb(vq); 327e4e8452aSWillem de Bruijn __napi_schedule(napi); 328e4e8452aSWillem de Bruijn } 329e4e8452aSWillem de Bruijn } 330e4e8452aSWillem de Bruijn 331e4e8452aSWillem de Bruijn static void virtqueue_napi_complete(struct napi_struct *napi, 332e4e8452aSWillem de Bruijn struct virtqueue *vq, int processed) 333e4e8452aSWillem de Bruijn { 334e4e8452aSWillem de Bruijn int opaque; 335e4e8452aSWillem de Bruijn 336e4e8452aSWillem de Bruijn opaque = virtqueue_enable_cb_prepare(vq); 337fdaa767aSToshiaki Makita if (napi_complete_done(napi, processed)) { 338fdaa767aSToshiaki Makita if (unlikely(virtqueue_poll(vq, opaque))) 339e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 340fdaa767aSToshiaki Makita } else { 341fdaa767aSToshiaki Makita virtqueue_disable_cb(vq); 342fdaa767aSToshiaki Makita } 343e4e8452aSWillem de Bruijn } 344e4e8452aSWillem de Bruijn 345e9d7417bSJason Wang static void skb_xmit_done(struct virtqueue *vq) 346296f96fcSRusty Russell { 347e9d7417bSJason Wang struct virtnet_info *vi = vq->vdev->priv; 348b92f1e67SWillem de Bruijn struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi; 349296f96fcSRusty Russell 3502cb9c6baSRusty Russell /* Suppress further interrupts. */ 351e9d7417bSJason Wang virtqueue_disable_cb(vq); 35211a3a154SRusty Russell 353b92f1e67SWillem de Bruijn if (napi->weight) 354b92f1e67SWillem de Bruijn virtqueue_napi_schedule(napi, vq); 355b92f1e67SWillem de Bruijn else 356363f1514SRusty Russell /* We were probably waiting for more output buffers. */ 357986a4f4dSJason Wang netif_wake_subqueue(vi->dev, vq2txq(vq)); 358296f96fcSRusty Russell } 359296f96fcSRusty Russell 36028b39bc7SJason Wang #define MRG_CTX_HEADER_SHIFT 22 36128b39bc7SJason Wang static void *mergeable_len_to_ctx(unsigned int truesize, 36228b39bc7SJason Wang unsigned int headroom) 36328b39bc7SJason Wang { 36428b39bc7SJason Wang return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 36528b39bc7SJason Wang } 36628b39bc7SJason Wang 36728b39bc7SJason Wang static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 36828b39bc7SJason Wang { 36928b39bc7SJason Wang return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 37028b39bc7SJason Wang } 37128b39bc7SJason Wang 37228b39bc7SJason Wang static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 37328b39bc7SJason Wang { 37428b39bc7SJason Wang return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 37528b39bc7SJason Wang } 37628b39bc7SJason Wang 3773464645aSMike Waychison /* Called from bottom half context */ 378946fa564SMichael S. Tsirkin static struct sk_buff *page_to_skb(struct virtnet_info *vi, 379946fa564SMichael S. Tsirkin struct receive_queue *rq, 3802613af0eSMichael Dalton struct page *page, unsigned int offset, 381436c9453SJason Wang unsigned int len, unsigned int truesize, 382fb32856bSXuan Zhuo bool hdr_valid, unsigned int metasize, 383c32325b8SJakub Kicinski unsigned int headroom) 3849ab86bbcSShirley Ma { 3859ab86bbcSShirley Ma struct sk_buff *skb; 386012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 3872613af0eSMichael Dalton unsigned int copy, hdr_len, hdr_padded_len; 388af39c8f7SEric Dumazet struct page *page_to_free = NULL; 389fb32856bSXuan Zhuo int tailroom, shinfo_size; 390f80bd740SXuan Zhuo char *p, *hdr_p, *buf; 3919ab86bbcSShirley Ma 3922613af0eSMichael Dalton p = page_address(page) + offset; 393fb32856bSXuan Zhuo hdr_p = p; 3949ab86bbcSShirley Ma 395012873d0SMichael S. Tsirkin hdr_len = vi->hdr_len; 396012873d0SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 397a4a76503Sstephen hemminger hdr_padded_len = sizeof(*hdr); 398012873d0SMichael S. Tsirkin else 3992613af0eSMichael Dalton hdr_padded_len = sizeof(struct padded_vnet_hdr); 4003f2c31d9SMark McLoughlin 401c32325b8SJakub Kicinski /* If headroom is not 0, there is an offset between the beginning of the 402fb32856bSXuan Zhuo * data and the allocated space, otherwise the data and the allocated 403fb32856bSXuan Zhuo * space are aligned. 4048fb7da9eSXuan Zhuo * 4058fb7da9eSXuan Zhuo * Buffers with headroom use PAGE_SIZE as alloc size, see 4068fb7da9eSXuan Zhuo * add_recvbuf_mergeable() + get_mergeable_buf_len() 407fb32856bSXuan Zhuo */ 408c32325b8SJakub Kicinski truesize = headroom ? PAGE_SIZE : truesize; 409c32325b8SJakub Kicinski tailroom = truesize - len - headroom; 410c32325b8SJakub Kicinski buf = p - headroom; 4113f2c31d9SMark McLoughlin 4129ab86bbcSShirley Ma len -= hdr_len; 4132613af0eSMichael Dalton offset += hdr_padded_len; 4142613af0eSMichael Dalton p += hdr_padded_len; 4153f2c31d9SMark McLoughlin 416fb32856bSXuan Zhuo shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 417fb32856bSXuan Zhuo 418f80bd740SXuan Zhuo /* copy small packet so we can reuse these pages */ 419f5d7872aSEric Dumazet if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) { 420f80bd740SXuan Zhuo skb = build_skb(buf, truesize); 421fb32856bSXuan Zhuo if (unlikely(!skb)) 422fb32856bSXuan Zhuo return NULL; 423fb32856bSXuan Zhuo 424f80bd740SXuan Zhuo skb_reserve(skb, p - buf); 425fb32856bSXuan Zhuo skb_put(skb, len); 426fb32856bSXuan Zhuo goto ok; 427fb32856bSXuan Zhuo } 428fb32856bSXuan Zhuo 429fb32856bSXuan Zhuo /* copy small packet so we can reuse these pages for small data */ 430fb32856bSXuan Zhuo skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 431fb32856bSXuan Zhuo if (unlikely(!skb)) 432fb32856bSXuan Zhuo return NULL; 433fb32856bSXuan Zhuo 4340f6925b3SEric Dumazet /* Copy all frame if it fits skb->head, otherwise 4350f6925b3SEric Dumazet * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. 4360f6925b3SEric Dumazet */ 4370f6925b3SEric Dumazet if (len <= skb_tailroom(skb)) 4383f2c31d9SMark McLoughlin copy = len; 4390f6925b3SEric Dumazet else 4400f6925b3SEric Dumazet copy = ETH_HLEN + metasize; 44159ae1d12SJohannes Berg skb_put_data(skb, p, copy); 4423f2c31d9SMark McLoughlin 4433f2c31d9SMark McLoughlin len -= copy; 4449ab86bbcSShirley Ma offset += copy; 4453f2c31d9SMark McLoughlin 4462613af0eSMichael Dalton if (vi->mergeable_rx_bufs) { 4472613af0eSMichael Dalton if (len) 4482613af0eSMichael Dalton skb_add_rx_frag(skb, 0, page, offset, len, truesize); 4492613af0eSMichael Dalton else 450af39c8f7SEric Dumazet page_to_free = page; 451fb32856bSXuan Zhuo goto ok; 4522613af0eSMichael Dalton } 4532613af0eSMichael Dalton 454e878d78bSSasha Levin /* 455e878d78bSSasha Levin * Verify that we can indeed put this data into a skb. 456e878d78bSSasha Levin * This is here to handle cases when the device erroneously 457e878d78bSSasha Levin * tries to receive more than is possible. This is usually 458e878d78bSSasha Levin * the case of a broken device. 459e878d78bSSasha Levin */ 460e878d78bSSasha Levin if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { 461be443899SAmerigo Wang net_dbg_ratelimited("%s: too much data\n", skb->dev->name); 462e878d78bSSasha Levin dev_kfree_skb(skb); 463e878d78bSSasha Levin return NULL; 464e878d78bSSasha Levin } 4652613af0eSMichael Dalton BUG_ON(offset >= PAGE_SIZE); 4669ab86bbcSShirley Ma while (len) { 4672613af0eSMichael Dalton unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 4682613af0eSMichael Dalton skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 4692613af0eSMichael Dalton frag_size, truesize); 4702613af0eSMichael Dalton len -= frag_size; 4719ab86bbcSShirley Ma page = (struct page *)page->private; 4729ab86bbcSShirley Ma offset = 0; 4733f2c31d9SMark McLoughlin } 4743f2c31d9SMark McLoughlin 4759ab86bbcSShirley Ma if (page) 476e9d7417bSJason Wang give_pages(rq, page); 4773f2c31d9SMark McLoughlin 478fb32856bSXuan Zhuo ok: 479fb32856bSXuan Zhuo /* hdr_valid means no XDP, so we can copy the vnet header */ 480fb32856bSXuan Zhuo if (hdr_valid) { 481fb32856bSXuan Zhuo hdr = skb_vnet_hdr(skb); 482fb32856bSXuan Zhuo memcpy(hdr, hdr_p, hdr_len); 483fb32856bSXuan Zhuo } 484af39c8f7SEric Dumazet if (page_to_free) 485af39c8f7SEric Dumazet put_page(page_to_free); 486fb32856bSXuan Zhuo 487fb32856bSXuan Zhuo if (metasize) { 488fb32856bSXuan Zhuo __skb_pull(skb, metasize); 489fb32856bSXuan Zhuo skb_metadata_set(skb, metasize); 490fb32856bSXuan Zhuo } 491fb32856bSXuan Zhuo 4929ab86bbcSShirley Ma return skb; 4939ab86bbcSShirley Ma } 4949ab86bbcSShirley Ma 495735fc405SJesper Dangaard Brouer static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 496735fc405SJesper Dangaard Brouer struct send_queue *sq, 49744fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf) 49856434a01SJohn Fastabend { 49956434a01SJohn Fastabend struct virtio_net_hdr_mrg_rxbuf *hdr; 50056434a01SJohn Fastabend int err; 50156434a01SJohn Fastabend 502cac320c8SJesper Dangaard Brouer if (unlikely(xdpf->headroom < vi->hdr_len)) 503cac320c8SJesper Dangaard Brouer return -EOVERFLOW; 504cac320c8SJesper Dangaard Brouer 505cac320c8SJesper Dangaard Brouer /* Make room for virtqueue hdr (also change xdpf->headroom?) */ 506cac320c8SJesper Dangaard Brouer xdpf->data -= vi->hdr_len; 50756434a01SJohn Fastabend /* Zero header and leave csum up to XDP layers */ 508cac320c8SJesper Dangaard Brouer hdr = xdpf->data; 50956434a01SJohn Fastabend memset(hdr, 0, vi->hdr_len); 510cac320c8SJesper Dangaard Brouer xdpf->len += vi->hdr_len; 51156434a01SJohn Fastabend 512cac320c8SJesper Dangaard Brouer sg_init_one(sq->sg, xdpf->data, xdpf->len); 513bb91accfSJason Wang 5145050471dSToshiaki Makita err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf), 5155050471dSToshiaki Makita GFP_ATOMIC); 51611b7d897SJesper Dangaard Brouer if (unlikely(err)) 517cac320c8SJesper Dangaard Brouer return -ENOSPC; /* Caller handle free/refcnt */ 51856434a01SJohn Fastabend 519cac320c8SJesper Dangaard Brouer return 0; 52056434a01SJohn Fastabend } 52156434a01SJohn Fastabend 52297c2c69eSXuan Zhuo /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on 52397c2c69eSXuan Zhuo * the current cpu, so it does not need to be locked. 52497c2c69eSXuan Zhuo * 52597c2c69eSXuan Zhuo * Here we use marco instead of inline functions because we have to deal with 52697c2c69eSXuan Zhuo * three issues at the same time: 1. the choice of sq. 2. judge and execute the 52797c2c69eSXuan Zhuo * lock/unlock of txq 3. make sparse happy. It is difficult for two inline 52897c2c69eSXuan Zhuo * functions to perfectly solve these three problems at the same time. 52997c2c69eSXuan Zhuo */ 53097c2c69eSXuan Zhuo #define virtnet_xdp_get_sq(vi) ({ \ 5313dcc1edcSLi RongQing int cpu = smp_processor_id(); \ 53297c2c69eSXuan Zhuo struct netdev_queue *txq; \ 53397c2c69eSXuan Zhuo typeof(vi) v = (vi); \ 53497c2c69eSXuan Zhuo unsigned int qp; \ 53597c2c69eSXuan Zhuo \ 53697c2c69eSXuan Zhuo if (v->curr_queue_pairs > nr_cpu_ids) { \ 53797c2c69eSXuan Zhuo qp = v->curr_queue_pairs - v->xdp_queue_pairs; \ 5383dcc1edcSLi RongQing qp += cpu; \ 53997c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, qp); \ 54097c2c69eSXuan Zhuo __netif_tx_acquire(txq); \ 54197c2c69eSXuan Zhuo } else { \ 5423dcc1edcSLi RongQing qp = cpu % v->curr_queue_pairs; \ 54397c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, qp); \ 5443dcc1edcSLi RongQing __netif_tx_lock(txq, cpu); \ 54597c2c69eSXuan Zhuo } \ 54697c2c69eSXuan Zhuo v->sq + qp; \ 54797c2c69eSXuan Zhuo }) 5482a43565cSToshiaki Makita 54997c2c69eSXuan Zhuo #define virtnet_xdp_put_sq(vi, q) { \ 55097c2c69eSXuan Zhuo struct netdev_queue *txq; \ 55197c2c69eSXuan Zhuo typeof(vi) v = (vi); \ 55297c2c69eSXuan Zhuo \ 55397c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \ 55497c2c69eSXuan Zhuo if (v->curr_queue_pairs > nr_cpu_ids) \ 55597c2c69eSXuan Zhuo __netif_tx_release(txq); \ 55697c2c69eSXuan Zhuo else \ 55797c2c69eSXuan Zhuo __netif_tx_unlock(txq); \ 5582a43565cSToshiaki Makita } 5592a43565cSToshiaki Makita 560735fc405SJesper Dangaard Brouer static int virtnet_xdp_xmit(struct net_device *dev, 56142b33468SJesper Dangaard Brouer int n, struct xdp_frame **frames, u32 flags) 562186b3c99SJason Wang { 563186b3c99SJason Wang struct virtnet_info *vi = netdev_priv(dev); 5648dcc5b0aSJesper Dangaard Brouer struct receive_queue *rq = vi->rq; 5658dcc5b0aSJesper Dangaard Brouer struct bpf_prog *xdp_prog; 566735fc405SJesper Dangaard Brouer struct send_queue *sq; 567735fc405SJesper Dangaard Brouer unsigned int len; 568546f2897SToshiaki Makita int packets = 0; 569546f2897SToshiaki Makita int bytes = 0; 570fdc13979SLorenzo Bianconi int nxmit = 0; 571461f03dcSToshiaki Makita int kicks = 0; 5725050471dSToshiaki Makita void *ptr; 573fdc13979SLorenzo Bianconi int ret; 574735fc405SJesper Dangaard Brouer int i; 575735fc405SJesper Dangaard Brouer 5768dcc5b0aSJesper Dangaard Brouer /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 5778dcc5b0aSJesper Dangaard Brouer * indicate XDP resources have been successfully allocated. 5788dcc5b0aSJesper Dangaard Brouer */ 5799719c6b9SJohn Fastabend xdp_prog = rcu_access_pointer(rq->xdp_prog); 5801667c08aSToshiaki Makita if (!xdp_prog) 5811667c08aSToshiaki Makita return -ENXIO; 5821667c08aSToshiaki Makita 58397c2c69eSXuan Zhuo sq = virtnet_xdp_get_sq(vi); 5849ab86bbcSShirley Ma 5859ab86bbcSShirley Ma if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 586186b3c99SJason Wang ret = -EINVAL; 587186b3c99SJason Wang goto out; 588186b3c99SJason Wang } 589186b3c99SJason Wang 590735fc405SJesper Dangaard Brouer /* Free up any pending old buffers before queueing new ones. */ 5915050471dSToshiaki Makita while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 592546f2897SToshiaki Makita if (likely(is_xdp_frame(ptr))) { 593546f2897SToshiaki Makita struct xdp_frame *frame = ptr_to_xdp(ptr); 594546f2897SToshiaki Makita 595546f2897SToshiaki Makita bytes += frame->len; 596546f2897SToshiaki Makita xdp_return_frame(frame); 597546f2897SToshiaki Makita } else { 598546f2897SToshiaki Makita struct sk_buff *skb = ptr; 599546f2897SToshiaki Makita 600546f2897SToshiaki Makita bytes += skb->len; 601546f2897SToshiaki Makita napi_consume_skb(skb, false); 602546f2897SToshiaki Makita } 603546f2897SToshiaki Makita packets++; 6045050471dSToshiaki Makita } 605735fc405SJesper Dangaard Brouer 606735fc405SJesper Dangaard Brouer for (i = 0; i < n; i++) { 607735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf = frames[i]; 608735fc405SJesper Dangaard Brouer 609fdc13979SLorenzo Bianconi if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 610fdc13979SLorenzo Bianconi break; 611fdc13979SLorenzo Bianconi nxmit++; 612735fc405SJesper Dangaard Brouer } 613fdc13979SLorenzo Bianconi ret = nxmit; 6145d274cb4SJesper Dangaard Brouer 615461f03dcSToshiaki Makita if (flags & XDP_XMIT_FLUSH) { 616461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 617461f03dcSToshiaki Makita kicks = 1; 618461f03dcSToshiaki Makita } 6195b8f3c8dSToshiaki Makita out: 6205b8f3c8dSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 621546f2897SToshiaki Makita sq->stats.bytes += bytes; 622546f2897SToshiaki Makita sq->stats.packets += packets; 6235b8f3c8dSToshiaki Makita sq->stats.xdp_tx += n; 624fdc13979SLorenzo Bianconi sq->stats.xdp_tx_drops += n - nxmit; 625461f03dcSToshiaki Makita sq->stats.kicks += kicks; 6265b8f3c8dSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 6275d274cb4SJesper Dangaard Brouer 62897c2c69eSXuan Zhuo virtnet_xdp_put_sq(vi, sq); 6295b8f3c8dSToshiaki Makita return ret; 630186b3c99SJason Wang } 631186b3c99SJason Wang 632f6b10209SJason Wang static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 633f6b10209SJason Wang { 63497c2c69eSXuan Zhuo return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0; 635f6b10209SJason Wang } 636f6b10209SJason Wang 6374941d472SJason Wang /* We copy the packet for XDP in the following cases: 6384941d472SJason Wang * 6394941d472SJason Wang * 1) Packet is scattered across multiple rx buffers. 6404941d472SJason Wang * 2) Headroom space is insufficient. 6414941d472SJason Wang * 6424941d472SJason Wang * This is inefficient but it's a temporary condition that 6434941d472SJason Wang * we hit right after XDP is enabled and until queue is refilled 6444941d472SJason Wang * with large buffers with sufficient headroom - so it should affect 6454941d472SJason Wang * at most queue size packets. 6464941d472SJason Wang * Afterwards, the conditions to enable 6474941d472SJason Wang * XDP should preclude the underlying device from sending packets 6484941d472SJason Wang * across multiple buffers (num_buf > 1), and we make sure buffers 6494941d472SJason Wang * have enough headroom. 65072979a6cSJohn Fastabend */ 65172979a6cSJohn Fastabend static struct page *xdp_linearize_page(struct receive_queue *rq, 65256a86f84SJason Wang u16 *num_buf, 65372979a6cSJohn Fastabend struct page *p, 65472979a6cSJohn Fastabend int offset, 6554941d472SJason Wang int page_off, 65672979a6cSJohn Fastabend unsigned int *len) 65772979a6cSJohn Fastabend { 65872979a6cSJohn Fastabend struct page *page = alloc_page(GFP_ATOMIC); 65972979a6cSJohn Fastabend 66072979a6cSJohn Fastabend if (!page) 66172979a6cSJohn Fastabend return NULL; 66272979a6cSJohn Fastabend 66372979a6cSJohn Fastabend memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 66472979a6cSJohn Fastabend page_off += *len; 66572979a6cSJohn Fastabend 66656a86f84SJason Wang while (--*num_buf) { 6673cc81a9aSJason Wang int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 66872979a6cSJohn Fastabend unsigned int buflen; 66972979a6cSJohn Fastabend void *buf; 67072979a6cSJohn Fastabend int off; 67172979a6cSJohn Fastabend 672680557cfSMichael S. Tsirkin buf = virtqueue_get_buf(rq->vq, &buflen); 673680557cfSMichael S. Tsirkin if (unlikely(!buf)) 67472979a6cSJohn Fastabend goto err_buf; 67572979a6cSJohn Fastabend 67672979a6cSJohn Fastabend p = virt_to_head_page(buf); 67772979a6cSJohn Fastabend off = buf - page_address(p); 67872979a6cSJohn Fastabend 67956a86f84SJason Wang /* guard against a misconfigured or uncooperative backend that 68056a86f84SJason Wang * is sending packet larger than the MTU. 68156a86f84SJason Wang */ 6823cc81a9aSJason Wang if ((page_off + buflen + tailroom) > PAGE_SIZE) { 68356a86f84SJason Wang put_page(p); 68456a86f84SJason Wang goto err_buf; 68556a86f84SJason Wang } 68656a86f84SJason Wang 68772979a6cSJohn Fastabend memcpy(page_address(page) + page_off, 68872979a6cSJohn Fastabend page_address(p) + off, buflen); 68972979a6cSJohn Fastabend page_off += buflen; 69056a86f84SJason Wang put_page(p); 69172979a6cSJohn Fastabend } 69272979a6cSJohn Fastabend 6932de2f7f4SJohn Fastabend /* Headroom does not contribute to packet length */ 6942de2f7f4SJohn Fastabend *len = page_off - VIRTIO_XDP_HEADROOM; 69572979a6cSJohn Fastabend return page; 69672979a6cSJohn Fastabend err_buf: 69772979a6cSJohn Fastabend __free_pages(page, 0); 69872979a6cSJohn Fastabend return NULL; 69972979a6cSJohn Fastabend } 70072979a6cSJohn Fastabend 7014941d472SJason Wang static struct sk_buff *receive_small(struct net_device *dev, 7024941d472SJason Wang struct virtnet_info *vi, 7034941d472SJason Wang struct receive_queue *rq, 7044941d472SJason Wang void *buf, void *ctx, 705186b3c99SJason Wang unsigned int len, 7067d9d60fdSToshiaki Makita unsigned int *xdp_xmit, 707d46eeeafSJason Wang struct virtnet_rq_stats *stats) 7084941d472SJason Wang { 7094941d472SJason Wang struct sk_buff *skb; 7104941d472SJason Wang struct bpf_prog *xdp_prog; 7114941d472SJason Wang unsigned int xdp_headroom = (unsigned long)ctx; 7124941d472SJason Wang unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 7134941d472SJason Wang unsigned int headroom = vi->hdr_len + header_offset; 7144941d472SJason Wang unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 7154941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 7164941d472SJason Wang struct page *page = virt_to_head_page(buf); 71711b7d897SJesper Dangaard Brouer unsigned int delta = 0; 7184941d472SJason Wang struct page *xdp_page; 71911b7d897SJesper Dangaard Brouer int err; 720503d539aSYuya Kusakabe unsigned int metasize = 0; 72111b7d897SJesper Dangaard Brouer 7224941d472SJason Wang len -= vi->hdr_len; 723d46eeeafSJason Wang stats->bytes += len; 7244941d472SJason Wang 725ad993a95SXie Yongji if (unlikely(len > GOOD_PACKET_LEN)) { 726ad993a95SXie Yongji pr_debug("%s: rx error: len %u exceeds max size %d\n", 727ad993a95SXie Yongji dev->name, len, GOOD_PACKET_LEN); 728ad993a95SXie Yongji dev->stats.rx_length_errors++; 729ad993a95SXie Yongji goto err_len; 730ad993a95SXie Yongji } 7314941d472SJason Wang rcu_read_lock(); 7324941d472SJason Wang xdp_prog = rcu_dereference(rq->xdp_prog); 7334941d472SJason Wang if (xdp_prog) { 7344941d472SJason Wang struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 73544fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf; 7364941d472SJason Wang struct xdp_buff xdp; 7374941d472SJason Wang void *orig_data; 7384941d472SJason Wang u32 act; 7394941d472SJason Wang 74095dbe9e7SJesper Dangaard Brouer if (unlikely(hdr->hdr.gso_type)) 7414941d472SJason Wang goto err_xdp; 7424941d472SJason Wang 7434941d472SJason Wang if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 7444941d472SJason Wang int offset = buf - page_address(page) + header_offset; 7454941d472SJason Wang unsigned int tlen = len + vi->hdr_len; 7464941d472SJason Wang u16 num_buf = 1; 7474941d472SJason Wang 7484941d472SJason Wang xdp_headroom = virtnet_get_headroom(vi); 7494941d472SJason Wang header_offset = VIRTNET_RX_PAD + xdp_headroom; 7504941d472SJason Wang headroom = vi->hdr_len + header_offset; 7514941d472SJason Wang buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 7524941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 7534941d472SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, page, 7544941d472SJason Wang offset, header_offset, 7554941d472SJason Wang &tlen); 7564941d472SJason Wang if (!xdp_page) 7574941d472SJason Wang goto err_xdp; 7584941d472SJason Wang 7594941d472SJason Wang buf = page_address(xdp_page); 7604941d472SJason Wang put_page(page); 7614941d472SJason Wang page = xdp_page; 7624941d472SJason Wang } 7634941d472SJason Wang 76443b5169dSLorenzo Bianconi xdp_init_buff(&xdp, buflen, &rq->xdp_rxq); 765be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, 766be9df4afSLorenzo Bianconi xdp_headroom, len, true); 7674941d472SJason Wang orig_data = xdp.data; 7684941d472SJason Wang act = bpf_prog_run_xdp(xdp_prog, &xdp); 769d46eeeafSJason Wang stats->xdp_packets++; 7704941d472SJason Wang 7714941d472SJason Wang switch (act) { 7724941d472SJason Wang case XDP_PASS: 7734941d472SJason Wang /* Recalculate length in case bpf program changed it */ 7744941d472SJason Wang delta = orig_data - xdp.data; 7756870de43SNikita V. Shirokov len = xdp.data_end - xdp.data; 776503d539aSYuya Kusakabe metasize = xdp.data - xdp.data_meta; 7774941d472SJason Wang break; 7784941d472SJason Wang case XDP_TX: 779d46eeeafSJason Wang stats->xdp_tx++; 7801b698fa5SLorenzo Bianconi xdpf = xdp_convert_buff_to_frame(&xdp); 78144fa2dbdSJesper Dangaard Brouer if (unlikely(!xdpf)) 78244fa2dbdSJesper Dangaard Brouer goto err_xdp; 783ca9e83b4SJason Wang err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 784fdc13979SLorenzo Bianconi if (unlikely(!err)) { 785fdc13979SLorenzo Bianconi xdp_return_frame_rx_napi(xdpf); 786fdc13979SLorenzo Bianconi } else if (unlikely(err < 0)) { 7874941d472SJason Wang trace_xdp_exception(vi->dev, xdp_prog, act); 78811b7d897SJesper Dangaard Brouer goto err_xdp; 78911b7d897SJesper Dangaard Brouer } 7902471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_TX; 791186b3c99SJason Wang rcu_read_unlock(); 792186b3c99SJason Wang goto xdp_xmit; 793186b3c99SJason Wang case XDP_REDIRECT: 794d46eeeafSJason Wang stats->xdp_redirects++; 795186b3c99SJason Wang err = xdp_do_redirect(dev, &xdp, xdp_prog); 79611b7d897SJesper Dangaard Brouer if (err) 79711b7d897SJesper Dangaard Brouer goto err_xdp; 7982471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_REDIR; 7994941d472SJason Wang rcu_read_unlock(); 8004941d472SJason Wang goto xdp_xmit; 8014941d472SJason Wang default: 8024941d472SJason Wang bpf_warn_invalid_xdp_action(act); 803df561f66SGustavo A. R. Silva fallthrough; 8044941d472SJason Wang case XDP_ABORTED: 8054941d472SJason Wang trace_xdp_exception(vi->dev, xdp_prog, act); 80695efabf0SGustavo A. R. Silva goto err_xdp; 8074941d472SJason Wang case XDP_DROP: 8084941d472SJason Wang goto err_xdp; 8094941d472SJason Wang } 8104941d472SJason Wang } 8114941d472SJason Wang rcu_read_unlock(); 8124941d472SJason Wang 8134941d472SJason Wang skb = build_skb(buf, buflen); 8144941d472SJason Wang if (!skb) { 8154941d472SJason Wang put_page(page); 8164941d472SJason Wang goto err; 8174941d472SJason Wang } 8184941d472SJason Wang skb_reserve(skb, headroom - delta); 8196870de43SNikita V. Shirokov skb_put(skb, len); 820f1d4884dSYuya Kusakabe if (!xdp_prog) { 8214941d472SJason Wang buf += header_offset; 8224941d472SJason Wang memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); 823f1d4884dSYuya Kusakabe } /* keep zeroed vnet hdr since XDP is loaded */ 8244941d472SJason Wang 825503d539aSYuya Kusakabe if (metasize) 826503d539aSYuya Kusakabe skb_metadata_set(skb, metasize); 827503d539aSYuya Kusakabe 8284941d472SJason Wang err: 8294941d472SJason Wang return skb; 8304941d472SJason Wang 8314941d472SJason Wang err_xdp: 8324941d472SJason Wang rcu_read_unlock(); 833d46eeeafSJason Wang stats->xdp_drops++; 834ad993a95SXie Yongji err_len: 835d46eeeafSJason Wang stats->drops++; 8364941d472SJason Wang put_page(page); 8374941d472SJason Wang xdp_xmit: 8384941d472SJason Wang return NULL; 8394941d472SJason Wang } 8404941d472SJason Wang 8414941d472SJason Wang static struct sk_buff *receive_big(struct net_device *dev, 8424941d472SJason Wang struct virtnet_info *vi, 8434941d472SJason Wang struct receive_queue *rq, 8444941d472SJason Wang void *buf, 8457d9d60fdSToshiaki Makita unsigned int len, 846d46eeeafSJason Wang struct virtnet_rq_stats *stats) 8474941d472SJason Wang { 8484941d472SJason Wang struct page *page = buf; 849503d539aSYuya Kusakabe struct sk_buff *skb = 850fb32856bSXuan Zhuo page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0); 8514941d472SJason Wang 852d46eeeafSJason Wang stats->bytes += len - vi->hdr_len; 8534941d472SJason Wang if (unlikely(!skb)) 8544941d472SJason Wang goto err; 8554941d472SJason Wang 8564941d472SJason Wang return skb; 8574941d472SJason Wang 8584941d472SJason Wang err: 859d46eeeafSJason Wang stats->drops++; 8604941d472SJason Wang give_pages(rq, page); 8614941d472SJason Wang return NULL; 8624941d472SJason Wang } 8634941d472SJason Wang 8648fc3b9e9SMichael S. Tsirkin static struct sk_buff *receive_mergeable(struct net_device *dev, 865fdd819b2SMichael S. Tsirkin struct virtnet_info *vi, 8668fc3b9e9SMichael S. Tsirkin struct receive_queue *rq, 867680557cfSMichael S. Tsirkin void *buf, 868680557cfSMichael S. Tsirkin void *ctx, 869186b3c99SJason Wang unsigned int len, 8707d9d60fdSToshiaki Makita unsigned int *xdp_xmit, 871d46eeeafSJason Wang struct virtnet_rq_stats *stats) 8729ab86bbcSShirley Ma { 873012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 874012873d0SMichael S. Tsirkin u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 8758fc3b9e9SMichael S. Tsirkin struct page *page = virt_to_head_page(buf); 8768fc3b9e9SMichael S. Tsirkin int offset = buf - page_address(page); 877f600b690SJohn Fastabend struct sk_buff *head_skb, *curr_skb; 878f600b690SJohn Fastabend struct bpf_prog *xdp_prog; 8799ce6146eSJesper Dangaard Brouer unsigned int truesize = mergeable_ctx_to_truesize(ctx); 8804941d472SJason Wang unsigned int headroom = mergeable_ctx_to_headroom(ctx); 881503d539aSYuya Kusakabe unsigned int metasize = 0; 8829ce6146eSJesper Dangaard Brouer unsigned int frame_sz; 8839ce6146eSJesper Dangaard Brouer int err; 884ab7db917SMichael Dalton 88556434a01SJohn Fastabend head_skb = NULL; 886d46eeeafSJason Wang stats->bytes += len - vi->hdr_len; 88756434a01SJohn Fastabend 888ad993a95SXie Yongji if (unlikely(len > truesize)) { 889ad993a95SXie Yongji pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 890ad993a95SXie Yongji dev->name, len, (unsigned long)ctx); 891ad993a95SXie Yongji dev->stats.rx_length_errors++; 892ad993a95SXie Yongji goto err_skb; 893ad993a95SXie Yongji } 894f600b690SJohn Fastabend rcu_read_lock(); 895f600b690SJohn Fastabend xdp_prog = rcu_dereference(rq->xdp_prog); 896f600b690SJohn Fastabend if (xdp_prog) { 89744fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf; 89872979a6cSJohn Fastabend struct page *xdp_page; 8990354e4d1SJohn Fastabend struct xdp_buff xdp; 9000354e4d1SJohn Fastabend void *data; 901f600b690SJohn Fastabend u32 act; 902f600b690SJohn Fastabend 9033d62b2a0SJason Wang /* Transient failure which in theory could occur if 9043d62b2a0SJason Wang * in-flight packets from before XDP was enabled reach 9053d62b2a0SJason Wang * the receive path after XDP is loaded. 9063d62b2a0SJason Wang */ 9073d62b2a0SJason Wang if (unlikely(hdr->hdr.gso_type)) 9083d62b2a0SJason Wang goto err_xdp; 9093d62b2a0SJason Wang 9109ce6146eSJesper Dangaard Brouer /* Buffers with headroom use PAGE_SIZE as alloc size, 9119ce6146eSJesper Dangaard Brouer * see add_recvbuf_mergeable() + get_mergeable_buf_len() 9129ce6146eSJesper Dangaard Brouer */ 9139ce6146eSJesper Dangaard Brouer frame_sz = headroom ? PAGE_SIZE : truesize; 9149ce6146eSJesper Dangaard Brouer 9153cc81a9aSJason Wang /* This happens when rx buffer size is underestimated 9163cc81a9aSJason Wang * or headroom is not enough because of the buffer 9173cc81a9aSJason Wang * was refilled before XDP is set. This should only 9183cc81a9aSJason Wang * happen for the first several packets, so we don't 9193cc81a9aSJason Wang * care much about its performance. 9203cc81a9aSJason Wang */ 9214941d472SJason Wang if (unlikely(num_buf > 1 || 9224941d472SJason Wang headroom < virtnet_get_headroom(vi))) { 92372979a6cSJohn Fastabend /* linearize data for XDP */ 92456a86f84SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, 9254941d472SJason Wang page, offset, 9264941d472SJason Wang VIRTIO_XDP_HEADROOM, 9274941d472SJason Wang &len); 9289ce6146eSJesper Dangaard Brouer frame_sz = PAGE_SIZE; 9299ce6146eSJesper Dangaard Brouer 93072979a6cSJohn Fastabend if (!xdp_page) 931f600b690SJohn Fastabend goto err_xdp; 9322de2f7f4SJohn Fastabend offset = VIRTIO_XDP_HEADROOM; 93372979a6cSJohn Fastabend } else { 93472979a6cSJohn Fastabend xdp_page = page; 935f600b690SJohn Fastabend } 936f600b690SJohn Fastabend 9372de2f7f4SJohn Fastabend /* Allow consuming headroom but reserve enough space to push 9382de2f7f4SJohn Fastabend * the descriptor on if we get an XDP_TX return code. 9392de2f7f4SJohn Fastabend */ 9400354e4d1SJohn Fastabend data = page_address(xdp_page) + offset; 94143b5169dSLorenzo Bianconi xdp_init_buff(&xdp, frame_sz - vi->hdr_len, &rq->xdp_rxq); 942be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, data - VIRTIO_XDP_HEADROOM + vi->hdr_len, 943be9df4afSLorenzo Bianconi VIRTIO_XDP_HEADROOM, len - vi->hdr_len, true); 944754b8a21SJesper Dangaard Brouer 9450354e4d1SJohn Fastabend act = bpf_prog_run_xdp(xdp_prog, &xdp); 946d46eeeafSJason Wang stats->xdp_packets++; 9470354e4d1SJohn Fastabend 94856434a01SJohn Fastabend switch (act) { 94956434a01SJohn Fastabend case XDP_PASS: 950503d539aSYuya Kusakabe metasize = xdp.data - xdp.data_meta; 9512de2f7f4SJohn Fastabend 952503d539aSYuya Kusakabe /* recalculate offset to account for any header 953503d539aSYuya Kusakabe * adjustments and minus the metasize to copy the 954503d539aSYuya Kusakabe * metadata in page_to_skb(). Note other cases do not 955503d539aSYuya Kusakabe * build an skb and avoid using offset 9566870de43SNikita V. Shirokov */ 957503d539aSYuya Kusakabe offset = xdp.data - page_address(xdp_page) - 958503d539aSYuya Kusakabe vi->hdr_len - metasize; 959503d539aSYuya Kusakabe 960503d539aSYuya Kusakabe /* recalculate len if xdp.data, xdp.data_end or 961503d539aSYuya Kusakabe * xdp.data_meta were adjusted 962503d539aSYuya Kusakabe */ 963503d539aSYuya Kusakabe len = xdp.data_end - xdp.data + vi->hdr_len + metasize; 9641830f893SJason Wang /* We can only create skb based on xdp_page. */ 9651830f893SJason Wang if (unlikely(xdp_page != page)) { 9661830f893SJason Wang rcu_read_unlock(); 9671830f893SJason Wang put_page(page); 968503d539aSYuya Kusakabe head_skb = page_to_skb(vi, rq, xdp_page, offset, 969503d539aSYuya Kusakabe len, PAGE_SIZE, false, 970c32325b8SJakub Kicinski metasize, 971c32325b8SJakub Kicinski VIRTIO_XDP_HEADROOM); 9721830f893SJason Wang return head_skb; 9731830f893SJason Wang } 97456434a01SJohn Fastabend break; 97556434a01SJohn Fastabend case XDP_TX: 976d46eeeafSJason Wang stats->xdp_tx++; 9771b698fa5SLorenzo Bianconi xdpf = xdp_convert_buff_to_frame(&xdp); 97844fa2dbdSJesper Dangaard Brouer if (unlikely(!xdpf)) 97944fa2dbdSJesper Dangaard Brouer goto err_xdp; 980ca9e83b4SJason Wang err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 981fdc13979SLorenzo Bianconi if (unlikely(!err)) { 982fdc13979SLorenzo Bianconi xdp_return_frame_rx_napi(xdpf); 983fdc13979SLorenzo Bianconi } else if (unlikely(err < 0)) { 9840354e4d1SJohn Fastabend trace_xdp_exception(vi->dev, xdp_prog, act); 98511b7d897SJesper Dangaard Brouer if (unlikely(xdp_page != page)) 98611b7d897SJesper Dangaard Brouer put_page(xdp_page); 98711b7d897SJesper Dangaard Brouer goto err_xdp; 98811b7d897SJesper Dangaard Brouer } 9892471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_TX; 99072979a6cSJohn Fastabend if (unlikely(xdp_page != page)) 9915d458a13SJason Wang put_page(page); 99256434a01SJohn Fastabend rcu_read_unlock(); 99356434a01SJohn Fastabend goto xdp_xmit; 9943cc81a9aSJason Wang case XDP_REDIRECT: 995d46eeeafSJason Wang stats->xdp_redirects++; 9963cc81a9aSJason Wang err = xdp_do_redirect(dev, &xdp, xdp_prog); 9973cc81a9aSJason Wang if (err) { 9983cc81a9aSJason Wang if (unlikely(xdp_page != page)) 9993cc81a9aSJason Wang put_page(xdp_page); 10003cc81a9aSJason Wang goto err_xdp; 10013cc81a9aSJason Wang } 10022471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_REDIR; 10033cc81a9aSJason Wang if (unlikely(xdp_page != page)) 10046890418bSJason Wang put_page(page); 10053cc81a9aSJason Wang rcu_read_unlock(); 10063cc81a9aSJason Wang goto xdp_xmit; 100756434a01SJohn Fastabend default: 10080354e4d1SJohn Fastabend bpf_warn_invalid_xdp_action(act); 1009df561f66SGustavo A. R. Silva fallthrough; 10100354e4d1SJohn Fastabend case XDP_ABORTED: 10110354e4d1SJohn Fastabend trace_xdp_exception(vi->dev, xdp_prog, act); 1012df561f66SGustavo A. R. Silva fallthrough; 10130354e4d1SJohn Fastabend case XDP_DROP: 101472979a6cSJohn Fastabend if (unlikely(xdp_page != page)) 101572979a6cSJohn Fastabend __free_pages(xdp_page, 0); 1016f600b690SJohn Fastabend goto err_xdp; 1017f600b690SJohn Fastabend } 101856434a01SJohn Fastabend } 1019f600b690SJohn Fastabend rcu_read_unlock(); 1020f600b690SJohn Fastabend 1021503d539aSYuya Kusakabe head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, 1022c32325b8SJakub Kicinski metasize, headroom); 1023f600b690SJohn Fastabend curr_skb = head_skb; 10249ab86bbcSShirley Ma 10258fc3b9e9SMichael S. Tsirkin if (unlikely(!curr_skb)) 10268fc3b9e9SMichael S. Tsirkin goto err_skb; 10279ab86bbcSShirley Ma while (--num_buf) { 10288fc3b9e9SMichael S. Tsirkin int num_skb_frags; 10298fc3b9e9SMichael S. Tsirkin 1030680557cfSMichael S. Tsirkin buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 103103e9f8a0SYunjian Wang if (unlikely(!buf)) { 10328fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers out of %d missing\n", 1033fdd819b2SMichael S. Tsirkin dev->name, num_buf, 1034012873d0SMichael S. Tsirkin virtio16_to_cpu(vi->vdev, 1035012873d0SMichael S. Tsirkin hdr->num_buffers)); 10368fc3b9e9SMichael S. Tsirkin dev->stats.rx_length_errors++; 10378fc3b9e9SMichael S. Tsirkin goto err_buf; 10383f2c31d9SMark McLoughlin } 10398fc3b9e9SMichael S. Tsirkin 1040d46eeeafSJason Wang stats->bytes += len; 10418fc3b9e9SMichael S. Tsirkin page = virt_to_head_page(buf); 104228b39bc7SJason Wang 104328b39bc7SJason Wang truesize = mergeable_ctx_to_truesize(ctx); 104428b39bc7SJason Wang if (unlikely(len > truesize)) { 104556da5fd0SDan Carpenter pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1046680557cfSMichael S. Tsirkin dev->name, len, (unsigned long)ctx); 1047680557cfSMichael S. Tsirkin dev->stats.rx_length_errors++; 1048680557cfSMichael S. Tsirkin goto err_skb; 1049680557cfSMichael S. Tsirkin } 10508fc3b9e9SMichael S. Tsirkin 10518fc3b9e9SMichael S. Tsirkin num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 10522613af0eSMichael Dalton if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 10532613af0eSMichael Dalton struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 10548fc3b9e9SMichael S. Tsirkin 10558fc3b9e9SMichael S. Tsirkin if (unlikely(!nskb)) 10568fc3b9e9SMichael S. Tsirkin goto err_skb; 10572613af0eSMichael Dalton if (curr_skb == head_skb) 10582613af0eSMichael Dalton skb_shinfo(curr_skb)->frag_list = nskb; 10592613af0eSMichael Dalton else 10602613af0eSMichael Dalton curr_skb->next = nskb; 10612613af0eSMichael Dalton curr_skb = nskb; 10622613af0eSMichael Dalton head_skb->truesize += nskb->truesize; 10632613af0eSMichael Dalton num_skb_frags = 0; 10642613af0eSMichael Dalton } 10652613af0eSMichael Dalton if (curr_skb != head_skb) { 10662613af0eSMichael Dalton head_skb->data_len += len; 10672613af0eSMichael Dalton head_skb->len += len; 1068fb51879dSMichael Dalton head_skb->truesize += truesize; 10692613af0eSMichael Dalton } 10708fc3b9e9SMichael S. Tsirkin offset = buf - page_address(page); 1071ba275241SJason Wang if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 1072ba275241SJason Wang put_page(page); 1073ba275241SJason Wang skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 1074fb51879dSMichael Dalton len, truesize); 1075ba275241SJason Wang } else { 10762613af0eSMichael Dalton skb_add_rx_frag(curr_skb, num_skb_frags, page, 1077fb51879dSMichael Dalton offset, len, truesize); 1078ba275241SJason Wang } 10798fc3b9e9SMichael S. Tsirkin } 10808fc3b9e9SMichael S. Tsirkin 10815377d758SJohannes Berg ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 10828fc3b9e9SMichael S. Tsirkin return head_skb; 10838fc3b9e9SMichael S. Tsirkin 1084f600b690SJohn Fastabend err_xdp: 1085f600b690SJohn Fastabend rcu_read_unlock(); 1086d46eeeafSJason Wang stats->xdp_drops++; 10878fc3b9e9SMichael S. Tsirkin err_skb: 10888fc3b9e9SMichael S. Tsirkin put_page(page); 1089850e088dSJason Wang while (num_buf-- > 1) { 1090680557cfSMichael S. Tsirkin buf = virtqueue_get_buf(rq->vq, &len); 1091680557cfSMichael S. Tsirkin if (unlikely(!buf)) { 10928fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers missing\n", 10938fc3b9e9SMichael S. Tsirkin dev->name, num_buf); 10948fc3b9e9SMichael S. Tsirkin dev->stats.rx_length_errors++; 10958fc3b9e9SMichael S. Tsirkin break; 10968fc3b9e9SMichael S. Tsirkin } 1097d46eeeafSJason Wang stats->bytes += len; 1098680557cfSMichael S. Tsirkin page = virt_to_head_page(buf); 10998fc3b9e9SMichael S. Tsirkin put_page(page); 11003f2c31d9SMark McLoughlin } 11018fc3b9e9SMichael S. Tsirkin err_buf: 1102d46eeeafSJason Wang stats->drops++; 11038fc3b9e9SMichael S. Tsirkin dev_kfree_skb(head_skb); 110456434a01SJohn Fastabend xdp_xmit: 11058fc3b9e9SMichael S. Tsirkin return NULL; 11069ab86bbcSShirley Ma } 11079ab86bbcSShirley Ma 11087d9d60fdSToshiaki Makita static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 11092471c75eSJesper Dangaard Brouer void *buf, unsigned int len, void **ctx, 1110a0929a44SToshiaki Makita unsigned int *xdp_xmit, 1111d46eeeafSJason Wang struct virtnet_rq_stats *stats) 11129ab86bbcSShirley Ma { 1113e9d7417bSJason Wang struct net_device *dev = vi->dev; 11149ab86bbcSShirley Ma struct sk_buff *skb; 1115012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 11169ab86bbcSShirley Ma 1117bcff3162SMichael S. Tsirkin if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 11189ab86bbcSShirley Ma pr_debug("%s: short packet %i\n", dev->name, len); 11199ab86bbcSShirley Ma dev->stats.rx_length_errors++; 1120ab7db917SMichael Dalton if (vi->mergeable_rx_bufs) { 1121680557cfSMichael S. Tsirkin put_page(virt_to_head_page(buf)); 1122ab7db917SMichael Dalton } else if (vi->big_packets) { 112398bfd23cSMichael Dalton give_pages(rq, buf); 1124ab7db917SMichael Dalton } else { 1125f6b10209SJason Wang put_page(virt_to_head_page(buf)); 1126ab7db917SMichael Dalton } 11277d9d60fdSToshiaki Makita return; 11289ab86bbcSShirley Ma } 11299ab86bbcSShirley Ma 1130f121159dSMichael S. Tsirkin if (vi->mergeable_rx_bufs) 11317d9d60fdSToshiaki Makita skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, 1132a0929a44SToshiaki Makita stats); 1133f121159dSMichael S. Tsirkin else if (vi->big_packets) 1134a0929a44SToshiaki Makita skb = receive_big(dev, vi, rq, buf, len, stats); 1135f121159dSMichael S. Tsirkin else 1136a0929a44SToshiaki Makita skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); 1137f121159dSMichael S. Tsirkin 11388fc3b9e9SMichael S. Tsirkin if (unlikely(!skb)) 11397d9d60fdSToshiaki Makita return; 11403f2c31d9SMark McLoughlin 11419ab86bbcSShirley Ma hdr = skb_vnet_hdr(skb); 11423fa2a1dfSstephen hemminger 1143e858fae2SMike Rapoport if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) 114410a8d94aSJason Wang skb->ip_summed = CHECKSUM_UNNECESSARY; 1145296f96fcSRusty Russell 1146e858fae2SMike Rapoport if (virtio_net_hdr_to_skb(skb, &hdr->hdr, 1147e858fae2SMike Rapoport virtio_is_little_endian(vi->vdev))) { 1148e858fae2SMike Rapoport net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n", 1149e858fae2SMike Rapoport dev->name, hdr->hdr.gso_type, 1150fdd819b2SMichael S. Tsirkin hdr->hdr.gso_size); 1151296f96fcSRusty Russell goto frame_err; 1152296f96fcSRusty Russell } 1153296f96fcSRusty Russell 1154133bbb18SWillem de Bruijn skb_record_rx_queue(skb, vq2rxq(rq->vq)); 1155d1dc06dcSMike Rapoport skb->protocol = eth_type_trans(skb, dev); 1156d1dc06dcSMike Rapoport pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 1157d1dc06dcSMike Rapoport ntohs(skb->protocol), skb->len, skb->pkt_type); 1158d1dc06dcSMike Rapoport 11590fbd050aSEric Dumazet napi_gro_receive(&rq->napi, skb); 11607d9d60fdSToshiaki Makita return; 1161296f96fcSRusty Russell 1162296f96fcSRusty Russell frame_err: 1163296f96fcSRusty Russell dev->stats.rx_frame_errors++; 1164296f96fcSRusty Russell dev_kfree_skb(skb); 1165296f96fcSRusty Russell } 1166296f96fcSRusty Russell 1167192f68cfSJason Wang /* Unlike mergeable buffers, all buffers are allocated to the 1168192f68cfSJason Wang * same size, except for the headroom. For this reason we do 1169192f68cfSJason Wang * not need to use mergeable_len_to_ctx here - it is enough 1170192f68cfSJason Wang * to store the headroom as the context ignoring the truesize. 1171192f68cfSJason Wang */ 1172946fa564SMichael S. Tsirkin static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 1173946fa564SMichael S. Tsirkin gfp_t gfp) 1174296f96fcSRusty Russell { 1175f6b10209SJason Wang struct page_frag *alloc_frag = &rq->alloc_frag; 1176f6b10209SJason Wang char *buf; 11772de2f7f4SJohn Fastabend unsigned int xdp_headroom = virtnet_get_headroom(vi); 1178192f68cfSJason Wang void *ctx = (void *)(unsigned long)xdp_headroom; 1179f6b10209SJason Wang int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 11809ab86bbcSShirley Ma int err; 11813f2c31d9SMark McLoughlin 1182f6b10209SJason Wang len = SKB_DATA_ALIGN(len) + 1183f6b10209SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1184f6b10209SJason Wang if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp))) 11859ab86bbcSShirley Ma return -ENOMEM; 1186296f96fcSRusty Russell 1187f6b10209SJason Wang buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1188f6b10209SJason Wang get_page(alloc_frag->page); 1189f6b10209SJason Wang alloc_frag->offset += len; 1190f6b10209SJason Wang sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom, 1191f6b10209SJason Wang vi->hdr_len + GOOD_PACKET_LEN); 1192192f68cfSJason Wang err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 11939ab86bbcSShirley Ma if (err < 0) 1194f6b10209SJason Wang put_page(virt_to_head_page(buf)); 11959ab86bbcSShirley Ma return err; 119697402b96SHerbert Xu } 119797402b96SHerbert Xu 1198012873d0SMichael S. Tsirkin static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 1199012873d0SMichael S. Tsirkin gfp_t gfp) 12009ab86bbcSShirley Ma { 12019ab86bbcSShirley Ma struct page *first, *list = NULL; 12029ab86bbcSShirley Ma char *p; 12039ab86bbcSShirley Ma int i, err, offset; 1204296f96fcSRusty Russell 1205a5835440SRusty Russell sg_init_table(rq->sg, MAX_SKB_FRAGS + 2); 1206a5835440SRusty Russell 1207e9d7417bSJason Wang /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */ 12089ab86bbcSShirley Ma for (i = MAX_SKB_FRAGS + 1; i > 1; --i) { 1209e9d7417bSJason Wang first = get_a_page(rq, gfp); 12109ab86bbcSShirley Ma if (!first) { 12119ab86bbcSShirley Ma if (list) 1212e9d7417bSJason Wang give_pages(rq, list); 12139ab86bbcSShirley Ma return -ENOMEM; 1214296f96fcSRusty Russell } 1215e9d7417bSJason Wang sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 12169ab86bbcSShirley Ma 12179ab86bbcSShirley Ma /* chain new page in list head to match sg */ 12189ab86bbcSShirley Ma first->private = (unsigned long)list; 12199ab86bbcSShirley Ma list = first; 12209ab86bbcSShirley Ma } 12219ab86bbcSShirley Ma 1222e9d7417bSJason Wang first = get_a_page(rq, gfp); 12239ab86bbcSShirley Ma if (!first) { 1224e9d7417bSJason Wang give_pages(rq, list); 12259ab86bbcSShirley Ma return -ENOMEM; 12269ab86bbcSShirley Ma } 12279ab86bbcSShirley Ma p = page_address(first); 12289ab86bbcSShirley Ma 1229e9d7417bSJason Wang /* rq->sg[0], rq->sg[1] share the same page */ 1230012873d0SMichael S. Tsirkin /* a separated rq->sg[0] for header - required in case !any_header_sg */ 1231012873d0SMichael S. Tsirkin sg_set_buf(&rq->sg[0], p, vi->hdr_len); 12329ab86bbcSShirley Ma 1233e9d7417bSJason Wang /* rq->sg[1] for data packet, from offset */ 12349ab86bbcSShirley Ma offset = sizeof(struct padded_vnet_hdr); 1235e9d7417bSJason Wang sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 12369ab86bbcSShirley Ma 12379ab86bbcSShirley Ma /* chain first in list head */ 12389ab86bbcSShirley Ma first->private = (unsigned long)list; 12399dc7b9e4SRusty Russell err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2, 1240aa989f5eSMichael S. Tsirkin first, gfp); 12419ab86bbcSShirley Ma if (err < 0) 1242e9d7417bSJason Wang give_pages(rq, first); 12439ab86bbcSShirley Ma 12449ab86bbcSShirley Ma return err; 12459ab86bbcSShirley Ma } 12469ab86bbcSShirley Ma 1247d85b758fSMichael S. Tsirkin static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 12483cc81a9aSJason Wang struct ewma_pkt_len *avg_pkt_len, 12493cc81a9aSJason Wang unsigned int room) 12509ab86bbcSShirley Ma { 1251ab7db917SMichael Dalton const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1252fbf28d78SMichael Dalton unsigned int len; 1253fbf28d78SMichael Dalton 12543cc81a9aSJason Wang if (room) 12553cc81a9aSJason Wang return PAGE_SIZE - room; 12563cc81a9aSJason Wang 12575377d758SJohannes Berg len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 1258f0c3192cSMichael S. Tsirkin rq->min_buf_len, PAGE_SIZE - hdr_len); 12593cc81a9aSJason Wang 1260e377fcc8SMichael S. Tsirkin return ALIGN(len, L1_CACHE_BYTES); 1261fbf28d78SMichael Dalton } 1262fbf28d78SMichael Dalton 12632de2f7f4SJohn Fastabend static int add_recvbuf_mergeable(struct virtnet_info *vi, 12642de2f7f4SJohn Fastabend struct receive_queue *rq, gfp_t gfp) 1265fbf28d78SMichael Dalton { 1266fb51879dSMichael Dalton struct page_frag *alloc_frag = &rq->alloc_frag; 12672de2f7f4SJohn Fastabend unsigned int headroom = virtnet_get_headroom(vi); 12683cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 12693cc81a9aSJason Wang unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1270fb51879dSMichael Dalton char *buf; 1271680557cfSMichael S. Tsirkin void *ctx; 12729ab86bbcSShirley Ma int err; 1273fb51879dSMichael Dalton unsigned int len, hole; 12749ab86bbcSShirley Ma 12753cc81a9aSJason Wang /* Extra tailroom is needed to satisfy XDP's assumption. This 12763cc81a9aSJason Wang * means rx frags coalescing won't work, but consider we've 12773cc81a9aSJason Wang * disabled GSO for XDP, it won't be a big issue. 12783cc81a9aSJason Wang */ 12793cc81a9aSJason Wang len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 12803cc81a9aSJason Wang if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp))) 12819ab86bbcSShirley Ma return -ENOMEM; 1282ab7db917SMichael Dalton 1283fb51879dSMichael Dalton buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 12842de2f7f4SJohn Fastabend buf += headroom; /* advance address leaving hole at front of pkt */ 1285fb51879dSMichael Dalton get_page(alloc_frag->page); 12863cc81a9aSJason Wang alloc_frag->offset += len + room; 1287fb51879dSMichael Dalton hole = alloc_frag->size - alloc_frag->offset; 12883cc81a9aSJason Wang if (hole < len + room) { 1289ab7db917SMichael Dalton /* To avoid internal fragmentation, if there is very likely not 1290ab7db917SMichael Dalton * enough space for another buffer, add the remaining space to 12911daa8790SMichael S. Tsirkin * the current buffer. 1292ab7db917SMichael Dalton */ 1293fb51879dSMichael Dalton len += hole; 1294fb51879dSMichael Dalton alloc_frag->offset += hole; 1295fb51879dSMichael Dalton } 12969ab86bbcSShirley Ma 1297fb51879dSMichael Dalton sg_init_one(rq->sg, buf, len); 129829fda25aSDavid S. Miller ctx = mergeable_len_to_ctx(len, headroom); 1299680557cfSMichael S. Tsirkin err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 13009ab86bbcSShirley Ma if (err < 0) 13012613af0eSMichael Dalton put_page(virt_to_head_page(buf)); 13029ab86bbcSShirley Ma 13039ab86bbcSShirley Ma return err; 1304296f96fcSRusty Russell } 1305296f96fcSRusty Russell 1306b2baed69SRusty Russell /* 1307b2baed69SRusty Russell * Returns false if we couldn't fill entirely (OOM). 1308b2baed69SRusty Russell * 1309b2baed69SRusty Russell * Normally run in the receive path, but can also be run from ndo_open 1310b2baed69SRusty Russell * before we're receiving packets, or from refill_work which is 1311b2baed69SRusty Russell * careful to disable receiving (using napi_disable). 1312b2baed69SRusty Russell */ 1313946fa564SMichael S. Tsirkin static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 1314946fa564SMichael S. Tsirkin gfp_t gfp) 13153f2c31d9SMark McLoughlin { 13163f2c31d9SMark McLoughlin int err; 13171788f495SMichael S. Tsirkin bool oom; 13183f2c31d9SMark McLoughlin 13190aea51c3SAmit Shah do { 13209ab86bbcSShirley Ma if (vi->mergeable_rx_bufs) 13212de2f7f4SJohn Fastabend err = add_recvbuf_mergeable(vi, rq, gfp); 13229ab86bbcSShirley Ma else if (vi->big_packets) 1323012873d0SMichael S. Tsirkin err = add_recvbuf_big(vi, rq, gfp); 13249ab86bbcSShirley Ma else 1325946fa564SMichael S. Tsirkin err = add_recvbuf_small(vi, rq, gfp); 13263f2c31d9SMark McLoughlin 13271788f495SMichael S. Tsirkin oom = err == -ENOMEM; 13289ed4cb07SRusty Russell if (err) 13293f2c31d9SMark McLoughlin break; 1330b7dfde95SLinus Torvalds } while (rq->vq->num_free); 1331461f03dcSToshiaki Makita if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { 133201c32598SMichael S. Tsirkin unsigned long flags; 133301c32598SMichael S. Tsirkin 133401c32598SMichael S. Tsirkin flags = u64_stats_update_begin_irqsave(&rq->stats.syncp); 1335d46eeeafSJason Wang rq->stats.kicks++; 133601c32598SMichael S. Tsirkin u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); 1337461f03dcSToshiaki Makita } 1338461f03dcSToshiaki Makita 13393161e453SRusty Russell return !oom; 13403f2c31d9SMark McLoughlin } 13413f2c31d9SMark McLoughlin 134218445c4dSRusty Russell static void skb_recv_done(struct virtqueue *rvq) 1343296f96fcSRusty Russell { 1344296f96fcSRusty Russell struct virtnet_info *vi = rvq->vdev->priv; 1345986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 1346e9d7417bSJason Wang 1347e4e8452aSWillem de Bruijn virtqueue_napi_schedule(&rq->napi, rvq); 1348296f96fcSRusty Russell } 1349296f96fcSRusty Russell 1350e4e8452aSWillem de Bruijn static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi) 13513e9d08ecSBruce Rogers { 1352e4e8452aSWillem de Bruijn napi_enable(napi); 13533e9d08ecSBruce Rogers 13543e9d08ecSBruce Rogers /* If all buffers were filled by other side before we napi_enabled, we 1355e4e8452aSWillem de Bruijn * won't get another interrupt, so process any outstanding packets now. 1356e4e8452aSWillem de Bruijn * Call local_bh_enable after to trigger softIRQ processing. 1357e4e8452aSWillem de Bruijn */ 1358ec13ee80SMichael S. Tsirkin local_bh_disable(); 1359e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 1360ec13ee80SMichael S. Tsirkin local_bh_enable(); 13613e9d08ecSBruce Rogers } 13623e9d08ecSBruce Rogers 1363b92f1e67SWillem de Bruijn static void virtnet_napi_tx_enable(struct virtnet_info *vi, 1364b92f1e67SWillem de Bruijn struct virtqueue *vq, 1365b92f1e67SWillem de Bruijn struct napi_struct *napi) 1366b92f1e67SWillem de Bruijn { 1367b92f1e67SWillem de Bruijn if (!napi->weight) 1368b92f1e67SWillem de Bruijn return; 1369b92f1e67SWillem de Bruijn 1370b92f1e67SWillem de Bruijn /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 1371b92f1e67SWillem de Bruijn * enable the feature if this is likely affine with the transmit path. 1372b92f1e67SWillem de Bruijn */ 1373b92f1e67SWillem de Bruijn if (!vi->affinity_hint_set) { 1374b92f1e67SWillem de Bruijn napi->weight = 0; 1375b92f1e67SWillem de Bruijn return; 1376b92f1e67SWillem de Bruijn } 1377b92f1e67SWillem de Bruijn 1378b92f1e67SWillem de Bruijn return virtnet_napi_enable(vq, napi); 1379b92f1e67SWillem de Bruijn } 1380b92f1e67SWillem de Bruijn 138178a57b48SWillem de Bruijn static void virtnet_napi_tx_disable(struct napi_struct *napi) 138278a57b48SWillem de Bruijn { 138378a57b48SWillem de Bruijn if (napi->weight) 138478a57b48SWillem de Bruijn napi_disable(napi); 138578a57b48SWillem de Bruijn } 138678a57b48SWillem de Bruijn 13873161e453SRusty Russell static void refill_work(struct work_struct *work) 13883161e453SRusty Russell { 1389e9d7417bSJason Wang struct virtnet_info *vi = 1390e9d7417bSJason Wang container_of(work, struct virtnet_info, refill.work); 13913161e453SRusty Russell bool still_empty; 1392986a4f4dSJason Wang int i; 13933161e453SRusty Russell 139455257d72SSasha Levin for (i = 0; i < vi->curr_queue_pairs; i++) { 1395986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[i]; 1396986a4f4dSJason Wang 1397986a4f4dSJason Wang napi_disable(&rq->napi); 1398946fa564SMichael S. Tsirkin still_empty = !try_fill_recv(vi, rq, GFP_KERNEL); 1399e4e8452aSWillem de Bruijn virtnet_napi_enable(rq->vq, &rq->napi); 14003161e453SRusty Russell 14013161e453SRusty Russell /* In theory, this can happen: if we don't get any buffers in 1402986a4f4dSJason Wang * we will *never* try to fill again. 1403986a4f4dSJason Wang */ 14043161e453SRusty Russell if (still_empty) 14053b07e9caSTejun Heo schedule_delayed_work(&vi->refill, HZ/2); 14063161e453SRusty Russell } 1407986a4f4dSJason Wang } 14083161e453SRusty Russell 14092471c75eSJesper Dangaard Brouer static int virtnet_receive(struct receive_queue *rq, int budget, 14102471c75eSJesper Dangaard Brouer unsigned int *xdp_xmit) 1411296f96fcSRusty Russell { 1412e9d7417bSJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 1413d46eeeafSJason Wang struct virtnet_rq_stats stats = {}; 1414a0929a44SToshiaki Makita unsigned int len; 14159ab86bbcSShirley Ma void *buf; 1416a0929a44SToshiaki Makita int i; 1417296f96fcSRusty Russell 1418192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 1419680557cfSMichael S. Tsirkin void *ctx; 1420680557cfSMichael S. Tsirkin 1421d46eeeafSJason Wang while (stats.packets < budget && 1422680557cfSMichael S. Tsirkin (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) { 1423a0929a44SToshiaki Makita receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats); 1424d46eeeafSJason Wang stats.packets++; 1425680557cfSMichael S. Tsirkin } 1426680557cfSMichael S. Tsirkin } else { 1427d46eeeafSJason Wang while (stats.packets < budget && 1428e9d7417bSJason Wang (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 1429a0929a44SToshiaki Makita receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats); 1430d46eeeafSJason Wang stats.packets++; 1431296f96fcSRusty Russell } 1432680557cfSMichael S. Tsirkin } 1433296f96fcSRusty Russell 1434718be6baS? jiang if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 1435946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, rq, GFP_ATOMIC)) 14363b07e9caSTejun Heo schedule_delayed_work(&vi->refill, 0); 14373161e453SRusty Russell } 1438296f96fcSRusty Russell 1439d7dfc5cfSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 1440a0929a44SToshiaki Makita for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) { 1441a0929a44SToshiaki Makita size_t offset = virtnet_rq_stats_desc[i].offset; 1442a0929a44SToshiaki Makita u64 *item; 1443a0929a44SToshiaki Makita 1444d46eeeafSJason Wang item = (u64 *)((u8 *)&rq->stats + offset); 1445d46eeeafSJason Wang *item += *(u64 *)((u8 *)&stats + offset); 1446a0929a44SToshiaki Makita } 1447d7dfc5cfSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 144861845d20SJason Wang 1449d46eeeafSJason Wang return stats.packets; 14502ffa7598SJason Wang } 14512ffa7598SJason Wang 1452df133f3fSMichael S. Tsirkin static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi) 1453ea7735d9SWillem de Bruijn { 1454ea7735d9SWillem de Bruijn unsigned int len; 1455ea7735d9SWillem de Bruijn unsigned int packets = 0; 1456ea7735d9SWillem de Bruijn unsigned int bytes = 0; 14575050471dSToshiaki Makita void *ptr; 1458ea7735d9SWillem de Bruijn 14595050471dSToshiaki Makita while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 14605050471dSToshiaki Makita if (likely(!is_xdp_frame(ptr))) { 14615050471dSToshiaki Makita struct sk_buff *skb = ptr; 14625050471dSToshiaki Makita 1463ea7735d9SWillem de Bruijn pr_debug("Sent skb %p\n", skb); 1464ea7735d9SWillem de Bruijn 1465ea7735d9SWillem de Bruijn bytes += skb->len; 1466df133f3fSMichael S. Tsirkin napi_consume_skb(skb, in_napi); 14675050471dSToshiaki Makita } else { 14685050471dSToshiaki Makita struct xdp_frame *frame = ptr_to_xdp(ptr); 14695050471dSToshiaki Makita 14705050471dSToshiaki Makita bytes += frame->len; 14715050471dSToshiaki Makita xdp_return_frame(frame); 14725050471dSToshiaki Makita } 14735050471dSToshiaki Makita packets++; 1474ea7735d9SWillem de Bruijn } 1475ea7735d9SWillem de Bruijn 1476ea7735d9SWillem de Bruijn /* Avoid overhead when no packets have been processed 1477ea7735d9SWillem de Bruijn * happens when called speculatively from start_xmit. 1478ea7735d9SWillem de Bruijn */ 1479ea7735d9SWillem de Bruijn if (!packets) 1480ea7735d9SWillem de Bruijn return; 1481ea7735d9SWillem de Bruijn 1482d7dfc5cfSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1483d7dfc5cfSToshiaki Makita sq->stats.bytes += bytes; 1484d7dfc5cfSToshiaki Makita sq->stats.packets += packets; 1485d7dfc5cfSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1486ea7735d9SWillem de Bruijn } 1487ea7735d9SWillem de Bruijn 1488534da5e8SToshiaki Makita static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 1489534da5e8SToshiaki Makita { 1490534da5e8SToshiaki Makita if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 1491534da5e8SToshiaki Makita return false; 1492534da5e8SToshiaki Makita else if (q < vi->curr_queue_pairs) 1493534da5e8SToshiaki Makita return true; 1494534da5e8SToshiaki Makita else 1495534da5e8SToshiaki Makita return false; 1496534da5e8SToshiaki Makita } 1497534da5e8SToshiaki Makita 14987b0411efSWillem de Bruijn static void virtnet_poll_cleantx(struct receive_queue *rq) 14997b0411efSWillem de Bruijn { 15007b0411efSWillem de Bruijn struct virtnet_info *vi = rq->vq->vdev->priv; 15017b0411efSWillem de Bruijn unsigned int index = vq2rxq(rq->vq); 15027b0411efSWillem de Bruijn struct send_queue *sq = &vi->sq[index]; 15037b0411efSWillem de Bruijn struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 15047b0411efSWillem de Bruijn 1505534da5e8SToshiaki Makita if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) 15067b0411efSWillem de Bruijn return; 15077b0411efSWillem de Bruijn 15087b0411efSWillem de Bruijn if (__netif_tx_trylock(txq)) { 1509a7766ef1SMichael S. Tsirkin do { 1510a7766ef1SMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 1511df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, true); 1512a7766ef1SMichael S. Tsirkin } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 15137b0411efSWillem de Bruijn 15147b0411efSWillem de Bruijn if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 15157b0411efSWillem de Bruijn netif_tx_wake_queue(txq); 151622bc63c5SMichael S. Tsirkin 151722bc63c5SMichael S. Tsirkin __netif_tx_unlock(txq); 151822bc63c5SMichael S. Tsirkin } 15197b0411efSWillem de Bruijn } 15207b0411efSWillem de Bruijn 15212ffa7598SJason Wang static int virtnet_poll(struct napi_struct *napi, int budget) 15222ffa7598SJason Wang { 15232ffa7598SJason Wang struct receive_queue *rq = 15242ffa7598SJason Wang container_of(napi, struct receive_queue, napi); 15259267c430SJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 15269267c430SJason Wang struct send_queue *sq; 15272a43565cSToshiaki Makita unsigned int received; 15282471c75eSJesper Dangaard Brouer unsigned int xdp_xmit = 0; 15292ffa7598SJason Wang 15307b0411efSWillem de Bruijn virtnet_poll_cleantx(rq); 15317b0411efSWillem de Bruijn 1532186b3c99SJason Wang received = virtnet_receive(rq, budget, &xdp_xmit); 15332ffa7598SJason Wang 15348329d98eSRusty Russell /* Out of packets? */ 1535e4e8452aSWillem de Bruijn if (received < budget) 1536e4e8452aSWillem de Bruijn virtqueue_napi_complete(napi, rq->vq, received); 1537296f96fcSRusty Russell 15382471c75eSJesper Dangaard Brouer if (xdp_xmit & VIRTIO_XDP_REDIR) 15391d233886SToke Høiland-Jørgensen xdp_do_flush(); 15402471c75eSJesper Dangaard Brouer 15412471c75eSJesper Dangaard Brouer if (xdp_xmit & VIRTIO_XDP_TX) { 154297c2c69eSXuan Zhuo sq = virtnet_xdp_get_sq(vi); 1543461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 1544461f03dcSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1545461f03dcSToshiaki Makita sq->stats.kicks++; 1546461f03dcSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1547461f03dcSToshiaki Makita } 154897c2c69eSXuan Zhuo virtnet_xdp_put_sq(vi, sq); 15499267c430SJason Wang } 1550186b3c99SJason Wang 1551296f96fcSRusty Russell return received; 1552296f96fcSRusty Russell } 1553296f96fcSRusty Russell 1554986a4f4dSJason Wang static int virtnet_open(struct net_device *dev) 1555986a4f4dSJason Wang { 1556986a4f4dSJason Wang struct virtnet_info *vi = netdev_priv(dev); 1557754b8a21SJesper Dangaard Brouer int i, err; 1558986a4f4dSJason Wang 1559e4166625SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 1560e4166625SJason Wang if (i < vi->curr_queue_pairs) 1561986a4f4dSJason Wang /* Make sure we have some buffers: if oom use wq. */ 1562946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 1563986a4f4dSJason Wang schedule_delayed_work(&vi->refill, 0); 1564754b8a21SJesper Dangaard Brouer 1565b02e5a0eSBjörn Töpel err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id); 1566754b8a21SJesper Dangaard Brouer if (err < 0) 1567754b8a21SJesper Dangaard Brouer return err; 1568754b8a21SJesper Dangaard Brouer 15698d5d8852SJesper Dangaard Brouer err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq, 15708d5d8852SJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED, NULL); 15718d5d8852SJesper Dangaard Brouer if (err < 0) { 15728d5d8852SJesper Dangaard Brouer xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 15738d5d8852SJesper Dangaard Brouer return err; 15748d5d8852SJesper Dangaard Brouer } 15758d5d8852SJesper Dangaard Brouer 1576e4e8452aSWillem de Bruijn virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 1577b92f1e67SWillem de Bruijn virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi); 1578986a4f4dSJason Wang } 1579986a4f4dSJason Wang 1580986a4f4dSJason Wang return 0; 1581986a4f4dSJason Wang } 1582986a4f4dSJason Wang 1583b92f1e67SWillem de Bruijn static int virtnet_poll_tx(struct napi_struct *napi, int budget) 1584b92f1e67SWillem de Bruijn { 1585b92f1e67SWillem de Bruijn struct send_queue *sq = container_of(napi, struct send_queue, napi); 1586b92f1e67SWillem de Bruijn struct virtnet_info *vi = sq->vq->vdev->priv; 1587534da5e8SToshiaki Makita unsigned int index = vq2txq(sq->vq); 1588534da5e8SToshiaki Makita struct netdev_queue *txq; 15895a2f966dSMichael S. Tsirkin int opaque; 15905a2f966dSMichael S. Tsirkin bool done; 1591b92f1e67SWillem de Bruijn 1592534da5e8SToshiaki Makita if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { 1593534da5e8SToshiaki Makita /* We don't need to enable cb for XDP */ 1594534da5e8SToshiaki Makita napi_complete_done(napi, 0); 1595534da5e8SToshiaki Makita return 0; 1596534da5e8SToshiaki Makita } 1597534da5e8SToshiaki Makita 1598534da5e8SToshiaki Makita txq = netdev_get_tx_queue(vi->dev, index); 1599b92f1e67SWillem de Bruijn __netif_tx_lock(txq, raw_smp_processor_id()); 16005a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 1601df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, true); 16025a2f966dSMichael S. Tsirkin 160322bc63c5SMichael S. Tsirkin if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 160422bc63c5SMichael S. Tsirkin netif_tx_wake_queue(txq); 160522bc63c5SMichael S. Tsirkin 16065a2f966dSMichael S. Tsirkin opaque = virtqueue_enable_cb_prepare(sq->vq); 16075a2f966dSMichael S. Tsirkin 16085a2f966dSMichael S. Tsirkin done = napi_complete_done(napi, 0); 16095a2f966dSMichael S. Tsirkin 16105a2f966dSMichael S. Tsirkin if (!done) 16115a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 16125a2f966dSMichael S. Tsirkin 1613b92f1e67SWillem de Bruijn __netif_tx_unlock(txq); 1614b92f1e67SWillem de Bruijn 16155a2f966dSMichael S. Tsirkin if (done) { 16165a2f966dSMichael S. Tsirkin if (unlikely(virtqueue_poll(sq->vq, opaque))) { 16175a2f966dSMichael S. Tsirkin if (napi_schedule_prep(napi)) { 16185a2f966dSMichael S. Tsirkin __netif_tx_lock(txq, raw_smp_processor_id()); 16195a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 16205a2f966dSMichael S. Tsirkin __netif_tx_unlock(txq); 16215a2f966dSMichael S. Tsirkin __napi_schedule(napi); 16225a2f966dSMichael S. Tsirkin } 16235a2f966dSMichael S. Tsirkin } 16245a2f966dSMichael S. Tsirkin } 1625b92f1e67SWillem de Bruijn 1626b92f1e67SWillem de Bruijn return 0; 1627b92f1e67SWillem de Bruijn } 1628b92f1e67SWillem de Bruijn 1629e9d7417bSJason Wang static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) 1630296f96fcSRusty Russell { 1631012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 1632296f96fcSRusty Russell const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 1633e9d7417bSJason Wang struct virtnet_info *vi = sq->vq->vdev->priv; 1634e2fcad58SJason A. Donenfeld int num_sg; 1635012873d0SMichael S. Tsirkin unsigned hdr_len = vi->hdr_len; 1636e7428e95SMichael S. Tsirkin bool can_push; 1637296f96fcSRusty Russell 1638e174961cSJohannes Berg pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 1639e7428e95SMichael S. Tsirkin 1640e7428e95SMichael S. Tsirkin can_push = vi->any_header_sg && 1641e7428e95SMichael S. Tsirkin !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 1642e7428e95SMichael S. Tsirkin !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 1643e7428e95SMichael S. Tsirkin /* Even if we can, don't push here yet as this would skew 1644e7428e95SMichael S. Tsirkin * csum_start offset below. */ 1645e7428e95SMichael S. Tsirkin if (can_push) 1646012873d0SMichael S. Tsirkin hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len); 1647e7428e95SMichael S. Tsirkin else 1648e7428e95SMichael S. Tsirkin hdr = skb_vnet_hdr(skb); 1649296f96fcSRusty Russell 1650e858fae2SMike Rapoport if (virtio_net_hdr_from_skb(skb, &hdr->hdr, 1651fd3a8862SWillem de Bruijn virtio_is_little_endian(vi->vdev), false, 1652fd3a8862SWillem de Bruijn 0)) 165385eb1389SXianting Tian return -EPROTO; 1654296f96fcSRusty Russell 1655e7428e95SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 1656012873d0SMichael S. Tsirkin hdr->num_buffers = 0; 16573f2c31d9SMark McLoughlin 1658547c890cSJason Wang sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 1659e7428e95SMichael S. Tsirkin if (can_push) { 1660e7428e95SMichael S. Tsirkin __skb_push(skb, hdr_len); 1661e7428e95SMichael S. Tsirkin num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 1662e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 1663e2fcad58SJason A. Donenfeld return num_sg; 1664e7428e95SMichael S. Tsirkin /* Pull header back to avoid skew in tx bytes calculations. */ 1665e7428e95SMichael S. Tsirkin __skb_pull(skb, hdr_len); 1666e7428e95SMichael S. Tsirkin } else { 1667e7428e95SMichael S. Tsirkin sg_set_buf(sq->sg, hdr, hdr_len); 1668e2fcad58SJason A. Donenfeld num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 1669e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 1670e2fcad58SJason A. Donenfeld return num_sg; 1671e2fcad58SJason A. Donenfeld num_sg++; 1672e7428e95SMichael S. Tsirkin } 16739dc7b9e4SRusty Russell return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC); 167411a3a154SRusty Russell } 167511a3a154SRusty Russell 1676424efe9cSStephen Hemminger static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 167799ffc696SRusty Russell { 167899ffc696SRusty Russell struct virtnet_info *vi = netdev_priv(dev); 1679986a4f4dSJason Wang int qnum = skb_get_queue_mapping(skb); 1680986a4f4dSJason Wang struct send_queue *sq = &vi->sq[qnum]; 16819ed4cb07SRusty Russell int err; 16824b7fd2e6SMichael S. Tsirkin struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 16836b16f9eeSFlorian Westphal bool kick = !netdev_xmit_more(); 1684b92f1e67SWillem de Bruijn bool use_napi = sq->napi.weight; 16852cb9c6baSRusty Russell 16862cb9c6baSRusty Russell /* Free up any pending old buffers before queueing new ones. */ 1687a7766ef1SMichael S. Tsirkin do { 1688a7766ef1SMichael S. Tsirkin if (use_napi) 1689a7766ef1SMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 1690a7766ef1SMichael S. Tsirkin 1691df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, false); 169299ffc696SRusty Russell 1693a7766ef1SMichael S. Tsirkin } while (use_napi && kick && 1694a7766ef1SMichael S. Tsirkin unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 1695bdb12e0dSWillem de Bruijn 1696074c3582SJacob Keller /* timestamp packet in software */ 1697074c3582SJacob Keller skb_tx_timestamp(skb); 1698074c3582SJacob Keller 169903f191baSMichael S. Tsirkin /* Try to transmit */ 1700b7dfde95SLinus Torvalds err = xmit_skb(sq, skb); 170199ffc696SRusty Russell 17029ed4cb07SRusty Russell /* This should not happen! */ 1703681daee2SJason Wang if (unlikely(err)) { 170458eba97dSRusty Russell dev->stats.tx_fifo_errors++; 17052e57b79cSRick Jones if (net_ratelimit()) 170658eba97dSRusty Russell dev_warn(&dev->dev, 17077934b481SYuval Shaia "Unexpected TXQ (%d) queue failure: %d\n", 17087934b481SYuval Shaia qnum, err); 170958eba97dSRusty Russell dev->stats.tx_dropped++; 171085e94525SEric W. Biederman dev_kfree_skb_any(skb); 171158eba97dSRusty Russell return NETDEV_TX_OK; 1712296f96fcSRusty Russell } 171303f191baSMichael S. Tsirkin 171448925e37SRusty Russell /* Don't wait up for transmitted skbs to be freed. */ 1715b92f1e67SWillem de Bruijn if (!use_napi) { 171648925e37SRusty Russell skb_orphan(skb); 1717895b5c9fSFlorian Westphal nf_reset_ct(skb); 1718b92f1e67SWillem de Bruijn } 171948925e37SRusty Russell 172060302ff6SMichael S. Tsirkin /* If running out of space, stop queue to avoid getting packets that we 172160302ff6SMichael S. Tsirkin * are then unable to transmit. 172260302ff6SMichael S. Tsirkin * An alternative would be to force queuing layer to requeue the skb by 172360302ff6SMichael S. Tsirkin * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 172460302ff6SMichael S. Tsirkin * returned in a normal path of operation: it means that driver is not 172560302ff6SMichael S. Tsirkin * maintaining the TX queue stop/start state properly, and causes 172660302ff6SMichael S. Tsirkin * the stack to do a non-trivial amount of useless work. 172760302ff6SMichael S. Tsirkin * Since most packets only take 1 or 2 ring slots, stopping the queue 172860302ff6SMichael S. Tsirkin * early means 16 slots are typically wasted. 1729d631b94eSstephen hemminger */ 1730b7dfde95SLinus Torvalds if (sq->vq->num_free < 2+MAX_SKB_FRAGS) { 1731986a4f4dSJason Wang netif_stop_subqueue(dev, qnum); 1732b92f1e67SWillem de Bruijn if (!use_napi && 1733b92f1e67SWillem de Bruijn unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 173448925e37SRusty Russell /* More just got used, free them then recheck. */ 1735df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, false); 1736b7dfde95SLinus Torvalds if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { 1737986a4f4dSJason Wang netif_start_subqueue(dev, qnum); 1738e9d7417bSJason Wang virtqueue_disable_cb(sq->vq); 173948925e37SRusty Russell } 174048925e37SRusty Russell } 174148925e37SRusty Russell } 174248925e37SRusty Russell 1743461f03dcSToshiaki Makita if (kick || netif_xmit_stopped(txq)) { 1744461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 1745461f03dcSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1746461f03dcSToshiaki Makita sq->stats.kicks++; 1747461f03dcSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1748461f03dcSToshiaki Makita } 1749461f03dcSToshiaki Makita } 17500b725a2cSDavid S. Miller 17510b725a2cSDavid S. Miller return NETDEV_TX_OK; 1752c223a078SDavid S. Miller } 1753c223a078SDavid S. Miller 175440cbfc37SAmos Kong /* 175540cbfc37SAmos Kong * Send command via the control virtqueue and check status. Commands 175640cbfc37SAmos Kong * supported by the hypervisor, as indicated by feature bits, should 1757788a8b6dSstephen hemminger * never fail unless improperly formatted. 175840cbfc37SAmos Kong */ 175940cbfc37SAmos Kong static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 1760d24bae32Sstephen hemminger struct scatterlist *out) 176140cbfc37SAmos Kong { 1762f7bc9594SRusty Russell struct scatterlist *sgs[4], hdr, stat; 1763d24bae32Sstephen hemminger unsigned out_num = 0, tmp; 1764222722bcSYunjian Wang int ret; 176540cbfc37SAmos Kong 176640cbfc37SAmos Kong /* Caller should know better */ 1767f7bc9594SRusty Russell BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 176840cbfc37SAmos Kong 176912e57169SMichael S. Tsirkin vi->ctrl->status = ~0; 177012e57169SMichael S. Tsirkin vi->ctrl->hdr.class = class; 177112e57169SMichael S. Tsirkin vi->ctrl->hdr.cmd = cmd; 1772f7bc9594SRusty Russell /* Add header */ 177312e57169SMichael S. Tsirkin sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 1774f7bc9594SRusty Russell sgs[out_num++] = &hdr; 177540cbfc37SAmos Kong 1776f7bc9594SRusty Russell if (out) 1777f7bc9594SRusty Russell sgs[out_num++] = out; 177840cbfc37SAmos Kong 1779f7bc9594SRusty Russell /* Add return status. */ 178012e57169SMichael S. Tsirkin sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 1781d24bae32Sstephen hemminger sgs[out_num] = &stat; 178240cbfc37SAmos Kong 1783d24bae32Sstephen hemminger BUG_ON(out_num + 1 > ARRAY_SIZE(sgs)); 1784222722bcSYunjian Wang ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC); 1785222722bcSYunjian Wang if (ret < 0) { 1786222722bcSYunjian Wang dev_warn(&vi->vdev->dev, 1787222722bcSYunjian Wang "Failed to add sgs for command vq: %d\n.", ret); 1788222722bcSYunjian Wang return false; 1789222722bcSYunjian Wang } 179040cbfc37SAmos Kong 179167975901SHeinz Graalfs if (unlikely(!virtqueue_kick(vi->cvq))) 179212e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 179340cbfc37SAmos Kong 179440cbfc37SAmos Kong /* Spin for a response, the kick causes an ioport write, trapping 179540cbfc37SAmos Kong * into the hypervisor, so the request should be handled immediately. 179640cbfc37SAmos Kong */ 1797047b9b94SHeinz Graalfs while (!virtqueue_get_buf(vi->cvq, &tmp) && 1798047b9b94SHeinz Graalfs !virtqueue_is_broken(vi->cvq)) 179940cbfc37SAmos Kong cpu_relax(); 180040cbfc37SAmos Kong 180112e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 180240cbfc37SAmos Kong } 180340cbfc37SAmos Kong 18049c46f6d4SAlex Williamson static int virtnet_set_mac_address(struct net_device *dev, void *p) 18059c46f6d4SAlex Williamson { 18069c46f6d4SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 18079c46f6d4SAlex Williamson struct virtio_device *vdev = vi->vdev; 1808f2f2c8b4SJiri Pirko int ret; 1809e37e2ff3SAndy Lutomirski struct sockaddr *addr; 18107e58d5aeSAmos Kong struct scatterlist sg; 18119c46f6d4SAlex Williamson 1812ba5e4426SSridhar Samudrala if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 1813ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 1814ba5e4426SSridhar Samudrala 1815801822d1SShyam Saini addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 1816e37e2ff3SAndy Lutomirski if (!addr) 1817e37e2ff3SAndy Lutomirski return -ENOMEM; 1818e37e2ff3SAndy Lutomirski 1819e37e2ff3SAndy Lutomirski ret = eth_prepare_mac_addr_change(dev, addr); 1820f2f2c8b4SJiri Pirko if (ret) 1821e37e2ff3SAndy Lutomirski goto out; 18229c46f6d4SAlex Williamson 18237e58d5aeSAmos Kong if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 18247e58d5aeSAmos Kong sg_init_one(&sg, addr->sa_data, dev->addr_len); 18257e58d5aeSAmos Kong if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 1826d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 18277e58d5aeSAmos Kong dev_warn(&vdev->dev, 18287e58d5aeSAmos Kong "Failed to set mac address by vq command.\n"); 1829e37e2ff3SAndy Lutomirski ret = -EINVAL; 1830e37e2ff3SAndy Lutomirski goto out; 18317e58d5aeSAmos Kong } 18327e93a02fSMichael S. Tsirkin } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 18337e93a02fSMichael S. Tsirkin !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1834855e0c52SRusty Russell unsigned int i; 1835855e0c52SRusty Russell 1836855e0c52SRusty Russell /* Naturally, this has an atomicity problem. */ 1837855e0c52SRusty Russell for (i = 0; i < dev->addr_len; i++) 1838855e0c52SRusty Russell virtio_cwrite8(vdev, 1839855e0c52SRusty Russell offsetof(struct virtio_net_config, mac) + 1840855e0c52SRusty Russell i, addr->sa_data[i]); 18417e58d5aeSAmos Kong } 18427e58d5aeSAmos Kong 18437e58d5aeSAmos Kong eth_commit_mac_addr_change(dev, p); 1844e37e2ff3SAndy Lutomirski ret = 0; 18459c46f6d4SAlex Williamson 1846e37e2ff3SAndy Lutomirski out: 1847e37e2ff3SAndy Lutomirski kfree(addr); 1848e37e2ff3SAndy Lutomirski return ret; 18499c46f6d4SAlex Williamson } 18509c46f6d4SAlex Williamson 1851bc1f4470Sstephen hemminger static void virtnet_stats(struct net_device *dev, 18523fa2a1dfSstephen hemminger struct rtnl_link_stats64 *tot) 18533fa2a1dfSstephen hemminger { 18543fa2a1dfSstephen hemminger struct virtnet_info *vi = netdev_priv(dev); 18553fa2a1dfSstephen hemminger unsigned int start; 1856d7dfc5cfSToshiaki Makita int i; 18573fa2a1dfSstephen hemminger 1858d7dfc5cfSToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 18592c4a2f7dSToshiaki Makita u64 tpackets, tbytes, rpackets, rbytes, rdrops; 1860d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 1861d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 18623fa2a1dfSstephen hemminger 18633fa2a1dfSstephen hemminger do { 1864d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&sq->stats.syncp); 1865d7dfc5cfSToshiaki Makita tpackets = sq->stats.packets; 1866d7dfc5cfSToshiaki Makita tbytes = sq->stats.bytes; 1867d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); 186883a27052SEric Dumazet 186983a27052SEric Dumazet do { 1870d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&rq->stats.syncp); 1871d46eeeafSJason Wang rpackets = rq->stats.packets; 1872d46eeeafSJason Wang rbytes = rq->stats.bytes; 1873d46eeeafSJason Wang rdrops = rq->stats.drops; 1874d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); 18753fa2a1dfSstephen hemminger 18763fa2a1dfSstephen hemminger tot->rx_packets += rpackets; 18773fa2a1dfSstephen hemminger tot->tx_packets += tpackets; 18783fa2a1dfSstephen hemminger tot->rx_bytes += rbytes; 18793fa2a1dfSstephen hemminger tot->tx_bytes += tbytes; 18802c4a2f7dSToshiaki Makita tot->rx_dropped += rdrops; 18813fa2a1dfSstephen hemminger } 18823fa2a1dfSstephen hemminger 18833fa2a1dfSstephen hemminger tot->tx_dropped = dev->stats.tx_dropped; 1884021ac8d3SRick Jones tot->tx_fifo_errors = dev->stats.tx_fifo_errors; 18853fa2a1dfSstephen hemminger tot->rx_length_errors = dev->stats.rx_length_errors; 18863fa2a1dfSstephen hemminger tot->rx_frame_errors = dev->stats.rx_frame_errors; 18873fa2a1dfSstephen hemminger } 18883fa2a1dfSstephen hemminger 1889586d17c5SJason Wang static void virtnet_ack_link_announce(struct virtnet_info *vi) 1890586d17c5SJason Wang { 1891586d17c5SJason Wang rtnl_lock(); 1892586d17c5SJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 1893d24bae32Sstephen hemminger VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 1894586d17c5SJason Wang dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 1895586d17c5SJason Wang rtnl_unlock(); 1896586d17c5SJason Wang } 1897586d17c5SJason Wang 189847315329SJohn Fastabend static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 1899986a4f4dSJason Wang { 1900986a4f4dSJason Wang struct scatterlist sg; 1901986a4f4dSJason Wang struct net_device *dev = vi->dev; 1902986a4f4dSJason Wang 1903986a4f4dSJason Wang if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 1904986a4f4dSJason Wang return 0; 1905986a4f4dSJason Wang 190612e57169SMichael S. Tsirkin vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 190712e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq)); 1908986a4f4dSJason Wang 1909986a4f4dSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 1910d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 1911986a4f4dSJason Wang dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 1912986a4f4dSJason Wang queue_pairs); 1913986a4f4dSJason Wang return -EINVAL; 191455257d72SSasha Levin } else { 1915986a4f4dSJason Wang vi->curr_queue_pairs = queue_pairs; 191635ed159bSJason Wang /* virtnet_open() will refill when device is going to up. */ 191735ed159bSJason Wang if (dev->flags & IFF_UP) 19189b9cd802SJason Wang schedule_delayed_work(&vi->refill, 0); 191955257d72SSasha Levin } 1920986a4f4dSJason Wang 1921986a4f4dSJason Wang return 0; 1922986a4f4dSJason Wang } 1923986a4f4dSJason Wang 192447315329SJohn Fastabend static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 192547315329SJohn Fastabend { 192647315329SJohn Fastabend int err; 192747315329SJohn Fastabend 192847315329SJohn Fastabend rtnl_lock(); 192947315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 193047315329SJohn Fastabend rtnl_unlock(); 193147315329SJohn Fastabend return err; 193247315329SJohn Fastabend } 193347315329SJohn Fastabend 1934296f96fcSRusty Russell static int virtnet_close(struct net_device *dev) 1935296f96fcSRusty Russell { 1936296f96fcSRusty Russell struct virtnet_info *vi = netdev_priv(dev); 1937986a4f4dSJason Wang int i; 1938296f96fcSRusty Russell 1939b2baed69SRusty Russell /* Make sure refill_work doesn't re-enable napi! */ 1940b2baed69SRusty Russell cancel_delayed_work_sync(&vi->refill); 1941986a4f4dSJason Wang 1942b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 1943754b8a21SJesper Dangaard Brouer xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 1944986a4f4dSJason Wang napi_disable(&vi->rq[i].napi); 194578a57b48SWillem de Bruijn virtnet_napi_tx_disable(&vi->sq[i].napi); 1946b92f1e67SWillem de Bruijn } 1947296f96fcSRusty Russell 1948296f96fcSRusty Russell return 0; 1949296f96fcSRusty Russell } 1950296f96fcSRusty Russell 19512af7698eSAlex Williamson static void virtnet_set_rx_mode(struct net_device *dev) 19522af7698eSAlex Williamson { 19532af7698eSAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 1954f565a7c2SAlex Williamson struct scatterlist sg[2]; 1955f565a7c2SAlex Williamson struct virtio_net_ctrl_mac *mac_data; 1956ccffad25SJiri Pirko struct netdev_hw_addr *ha; 195732e7bfc4SJiri Pirko int uc_count; 19584cd24eafSJiri Pirko int mc_count; 1959f565a7c2SAlex Williamson void *buf; 1960f565a7c2SAlex Williamson int i; 19612af7698eSAlex Williamson 1962788a8b6dSstephen hemminger /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 19632af7698eSAlex Williamson if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 19642af7698eSAlex Williamson return; 19652af7698eSAlex Williamson 196612e57169SMichael S. Tsirkin vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0); 196712e57169SMichael S. Tsirkin vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0); 19682af7698eSAlex Williamson 196912e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc)); 19702af7698eSAlex Williamson 19712af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 1972d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_PROMISC, sg)) 19732af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 197412e57169SMichael S. Tsirkin vi->ctrl->promisc ? "en" : "dis"); 19752af7698eSAlex Williamson 197612e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti)); 19772af7698eSAlex Williamson 19782af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 1979d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 19802af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 198112e57169SMichael S. Tsirkin vi->ctrl->allmulti ? "en" : "dis"); 1982f565a7c2SAlex Williamson 198332e7bfc4SJiri Pirko uc_count = netdev_uc_count(dev); 19844cd24eafSJiri Pirko mc_count = netdev_mc_count(dev); 1985f565a7c2SAlex Williamson /* MAC filter - use one buffer for both lists */ 19864cd24eafSJiri Pirko buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 1987f565a7c2SAlex Williamson (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 19884cd24eafSJiri Pirko mac_data = buf; 1989e68ed8f0SJoe Perches if (!buf) 1990f565a7c2SAlex Williamson return; 1991f565a7c2SAlex Williamson 199223e258e1SAlex Williamson sg_init_table(sg, 2); 199323e258e1SAlex Williamson 1994f565a7c2SAlex Williamson /* Store the unicast list and count in the front of the buffer */ 1995fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 1996ccffad25SJiri Pirko i = 0; 199732e7bfc4SJiri Pirko netdev_for_each_uc_addr(ha, dev) 1998ccffad25SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 1999f565a7c2SAlex Williamson 2000f565a7c2SAlex Williamson sg_set_buf(&sg[0], mac_data, 200132e7bfc4SJiri Pirko sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 2002f565a7c2SAlex Williamson 2003f565a7c2SAlex Williamson /* multicast list and count fill the end */ 200432e7bfc4SJiri Pirko mac_data = (void *)&mac_data->macs[uc_count][0]; 2005f565a7c2SAlex Williamson 2006fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 2007567ec874SJiri Pirko i = 0; 200822bedad3SJiri Pirko netdev_for_each_mc_addr(ha, dev) 200922bedad3SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2010f565a7c2SAlex Williamson 2011f565a7c2SAlex Williamson sg_set_buf(&sg[1], mac_data, 20124cd24eafSJiri Pirko sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 2013f565a7c2SAlex Williamson 2014f565a7c2SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 2015d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 201699e872aeSThomas Huth dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 2017f565a7c2SAlex Williamson 2018f565a7c2SAlex Williamson kfree(buf); 20192af7698eSAlex Williamson } 20202af7698eSAlex Williamson 202180d5c368SPatrick McHardy static int virtnet_vlan_rx_add_vid(struct net_device *dev, 202280d5c368SPatrick McHardy __be16 proto, u16 vid) 20230bde9569SAlex Williamson { 20240bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 20250bde9569SAlex Williamson struct scatterlist sg; 20260bde9569SAlex Williamson 2027d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 202812e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 20290bde9569SAlex Williamson 20300bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2031d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 20320bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 20338e586137SJiri Pirko return 0; 20340bde9569SAlex Williamson } 20350bde9569SAlex Williamson 203680d5c368SPatrick McHardy static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 203780d5c368SPatrick McHardy __be16 proto, u16 vid) 20380bde9569SAlex Williamson { 20390bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 20400bde9569SAlex Williamson struct scatterlist sg; 20410bde9569SAlex Williamson 2042d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 204312e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 20440bde9569SAlex Williamson 20450bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2046d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 20470bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 20488e586137SJiri Pirko return 0; 20490bde9569SAlex Williamson } 20500bde9569SAlex Williamson 2051310974faSPeter Xu static void virtnet_clean_affinity(struct virtnet_info *vi) 2052986a4f4dSJason Wang { 2053986a4f4dSJason Wang int i; 20548898c21cSWanlong Gao 20558898c21cSWanlong Gao if (vi->affinity_hint_set) { 20568898c21cSWanlong Gao for (i = 0; i < vi->max_queue_pairs; i++) { 205719e226e8SCaleb Raitto virtqueue_set_affinity(vi->rq[i].vq, NULL); 205819e226e8SCaleb Raitto virtqueue_set_affinity(vi->sq[i].vq, NULL); 20598898c21cSWanlong Gao } 20608898c21cSWanlong Gao 20618898c21cSWanlong Gao vi->affinity_hint_set = false; 20628898c21cSWanlong Gao } 20638898c21cSWanlong Gao } 20648898c21cSWanlong Gao 20658898c21cSWanlong Gao static void virtnet_set_affinity(struct virtnet_info *vi) 2066986a4f4dSJason Wang { 20672ca653d6SCaleb Raitto cpumask_var_t mask; 20682ca653d6SCaleb Raitto int stragglers; 20692ca653d6SCaleb Raitto int group_size; 20702ca653d6SCaleb Raitto int i, j, cpu; 20712ca653d6SCaleb Raitto int num_cpu; 20722ca653d6SCaleb Raitto int stride; 2073986a4f4dSJason Wang 20742ca653d6SCaleb Raitto if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 2075310974faSPeter Xu virtnet_clean_affinity(vi); 2076986a4f4dSJason Wang return; 2077986a4f4dSJason Wang } 2078986a4f4dSJason Wang 20792ca653d6SCaleb Raitto num_cpu = num_online_cpus(); 20802ca653d6SCaleb Raitto stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); 20812ca653d6SCaleb Raitto stragglers = num_cpu >= vi->curr_queue_pairs ? 20822ca653d6SCaleb Raitto num_cpu % vi->curr_queue_pairs : 20832ca653d6SCaleb Raitto 0; 20842ca653d6SCaleb Raitto cpu = cpumask_next(-1, cpu_online_mask); 20854d99f660SAndrei Vagin 20862ca653d6SCaleb Raitto for (i = 0; i < vi->curr_queue_pairs; i++) { 20872ca653d6SCaleb Raitto group_size = stride + (i < stragglers ? 1 : 0); 20882ca653d6SCaleb Raitto 20892ca653d6SCaleb Raitto for (j = 0; j < group_size; j++) { 20902ca653d6SCaleb Raitto cpumask_set_cpu(cpu, mask); 20912ca653d6SCaleb Raitto cpu = cpumask_next_wrap(cpu, cpu_online_mask, 20922ca653d6SCaleb Raitto nr_cpu_ids, false); 20932ca653d6SCaleb Raitto } 20942ca653d6SCaleb Raitto virtqueue_set_affinity(vi->rq[i].vq, mask); 20952ca653d6SCaleb Raitto virtqueue_set_affinity(vi->sq[i].vq, mask); 2096044ab86dSAntoine Tenart __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); 20972ca653d6SCaleb Raitto cpumask_clear(mask); 2098986a4f4dSJason Wang } 2099986a4f4dSJason Wang 2100986a4f4dSJason Wang vi->affinity_hint_set = true; 21012ca653d6SCaleb Raitto free_cpumask_var(mask); 210247be2479SWanlong Gao } 2103986a4f4dSJason Wang 21048017c279SSebastian Andrzej Siewior static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 21058de4b2f3SWanlong Gao { 21068017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 21078017c279SSebastian Andrzej Siewior node); 21088de4b2f3SWanlong Gao virtnet_set_affinity(vi); 21098017c279SSebastian Andrzej Siewior return 0; 21108de4b2f3SWanlong Gao } 21113ab098dfSJason Wang 21128017c279SSebastian Andrzej Siewior static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 21138017c279SSebastian Andrzej Siewior { 21148017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 21158017c279SSebastian Andrzej Siewior node_dead); 21168017c279SSebastian Andrzej Siewior virtnet_set_affinity(vi); 21178017c279SSebastian Andrzej Siewior return 0; 21188017c279SSebastian Andrzej Siewior } 21198017c279SSebastian Andrzej Siewior 21208017c279SSebastian Andrzej Siewior static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 21218017c279SSebastian Andrzej Siewior { 21228017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 21238017c279SSebastian Andrzej Siewior node); 21248017c279SSebastian Andrzej Siewior 2125310974faSPeter Xu virtnet_clean_affinity(vi); 21268017c279SSebastian Andrzej Siewior return 0; 21278017c279SSebastian Andrzej Siewior } 21288017c279SSebastian Andrzej Siewior 21298017c279SSebastian Andrzej Siewior static enum cpuhp_state virtionet_online; 21308017c279SSebastian Andrzej Siewior 21318017c279SSebastian Andrzej Siewior static int virtnet_cpu_notif_add(struct virtnet_info *vi) 21328017c279SSebastian Andrzej Siewior { 21338017c279SSebastian Andrzej Siewior int ret; 21348017c279SSebastian Andrzej Siewior 21358017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 21368017c279SSebastian Andrzej Siewior if (ret) 21378017c279SSebastian Andrzej Siewior return ret; 21388017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 21398017c279SSebastian Andrzej Siewior &vi->node_dead); 21408017c279SSebastian Andrzej Siewior if (!ret) 21418017c279SSebastian Andrzej Siewior return ret; 21428017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 21438017c279SSebastian Andrzej Siewior return ret; 21448017c279SSebastian Andrzej Siewior } 21458017c279SSebastian Andrzej Siewior 21468017c279SSebastian Andrzej Siewior static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 21478017c279SSebastian Andrzej Siewior { 21488017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 21498017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 21508017c279SSebastian Andrzej Siewior &vi->node_dead); 2151a9ea3fc6SHerbert Xu } 2152a9ea3fc6SHerbert Xu 21538f9f4668SRick Jones static void virtnet_get_ringparam(struct net_device *dev, 21548f9f4668SRick Jones struct ethtool_ringparam *ring) 21558f9f4668SRick Jones { 21568f9f4668SRick Jones struct virtnet_info *vi = netdev_priv(dev); 21578f9f4668SRick Jones 2158986a4f4dSJason Wang ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq); 2159986a4f4dSJason Wang ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq); 21608f9f4668SRick Jones ring->rx_pending = ring->rx_max_pending; 21618f9f4668SRick Jones ring->tx_pending = ring->tx_max_pending; 21628f9f4668SRick Jones } 21638f9f4668SRick Jones 216466846048SRick Jones 216566846048SRick Jones static void virtnet_get_drvinfo(struct net_device *dev, 216666846048SRick Jones struct ethtool_drvinfo *info) 216766846048SRick Jones { 216866846048SRick Jones struct virtnet_info *vi = netdev_priv(dev); 216966846048SRick Jones struct virtio_device *vdev = vi->vdev; 217066846048SRick Jones 217166846048SRick Jones strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 217266846048SRick Jones strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 217366846048SRick Jones strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 217466846048SRick Jones 217566846048SRick Jones } 217666846048SRick Jones 2177d73bcd2cSJason Wang /* TODO: Eliminate OOO packets during switching */ 2178d73bcd2cSJason Wang static int virtnet_set_channels(struct net_device *dev, 2179d73bcd2cSJason Wang struct ethtool_channels *channels) 2180d73bcd2cSJason Wang { 2181d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 2182d73bcd2cSJason Wang u16 queue_pairs = channels->combined_count; 2183d73bcd2cSJason Wang int err; 2184d73bcd2cSJason Wang 2185d73bcd2cSJason Wang /* We don't support separate rx/tx channels. 2186d73bcd2cSJason Wang * We don't allow setting 'other' channels. 2187d73bcd2cSJason Wang */ 2188d73bcd2cSJason Wang if (channels->rx_count || channels->tx_count || channels->other_count) 2189d73bcd2cSJason Wang return -EINVAL; 2190d73bcd2cSJason Wang 2191c18e9cd6SAmos Kong if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 2192d73bcd2cSJason Wang return -EINVAL; 2193d73bcd2cSJason Wang 2194f600b690SJohn Fastabend /* For now we don't support modifying channels while XDP is loaded 2195f600b690SJohn Fastabend * also when XDP is loaded all RX queues have XDP programs so we only 2196f600b690SJohn Fastabend * need to check a single RX queue. 2197f600b690SJohn Fastabend */ 2198f600b690SJohn Fastabend if (vi->rq[0].xdp_prog) 2199f600b690SJohn Fastabend return -EINVAL; 2200f600b690SJohn Fastabend 2201a0d1d0f4SSebastian Andrzej Siewior cpus_read_lock(); 220247315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 2203de33212fSJeff Dike if (err) { 2204a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 2205de33212fSJeff Dike goto err; 2206d73bcd2cSJason Wang } 2207de33212fSJeff Dike virtnet_set_affinity(vi); 2208a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 2209d73bcd2cSJason Wang 2210de33212fSJeff Dike netif_set_real_num_tx_queues(dev, queue_pairs); 2211de33212fSJeff Dike netif_set_real_num_rx_queues(dev, queue_pairs); 2212de33212fSJeff Dike err: 2213d73bcd2cSJason Wang return err; 2214d73bcd2cSJason Wang } 2215d73bcd2cSJason Wang 2216d7dfc5cfSToshiaki Makita static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 2217d7dfc5cfSToshiaki Makita { 2218d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 2219d7dfc5cfSToshiaki Makita unsigned int i, j; 2220d7a9a01bSAlexander Duyck u8 *p = data; 2221d7dfc5cfSToshiaki Makita 2222d7dfc5cfSToshiaki Makita switch (stringset) { 2223d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 2224d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2225d7a9a01bSAlexander Duyck for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) 2226d7a9a01bSAlexander Duyck ethtool_sprintf(&p, "rx_queue_%u_%s", i, 2227d7a9a01bSAlexander Duyck virtnet_rq_stats_desc[j].desc); 2228d7dfc5cfSToshiaki Makita } 2229d7dfc5cfSToshiaki Makita 2230d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2231d7a9a01bSAlexander Duyck for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) 2232d7a9a01bSAlexander Duyck ethtool_sprintf(&p, "tx_queue_%u_%s", i, 2233d7a9a01bSAlexander Duyck virtnet_sq_stats_desc[j].desc); 2234d7dfc5cfSToshiaki Makita } 2235d7dfc5cfSToshiaki Makita break; 2236d7dfc5cfSToshiaki Makita } 2237d7dfc5cfSToshiaki Makita } 2238d7dfc5cfSToshiaki Makita 2239d7dfc5cfSToshiaki Makita static int virtnet_get_sset_count(struct net_device *dev, int sset) 2240d7dfc5cfSToshiaki Makita { 2241d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 2242d7dfc5cfSToshiaki Makita 2243d7dfc5cfSToshiaki Makita switch (sset) { 2244d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 2245d7dfc5cfSToshiaki Makita return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN + 2246d7dfc5cfSToshiaki Makita VIRTNET_SQ_STATS_LEN); 2247d7dfc5cfSToshiaki Makita default: 2248d7dfc5cfSToshiaki Makita return -EOPNOTSUPP; 2249d7dfc5cfSToshiaki Makita } 2250d7dfc5cfSToshiaki Makita } 2251d7dfc5cfSToshiaki Makita 2252d7dfc5cfSToshiaki Makita static void virtnet_get_ethtool_stats(struct net_device *dev, 2253d7dfc5cfSToshiaki Makita struct ethtool_stats *stats, u64 *data) 2254d7dfc5cfSToshiaki Makita { 2255d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 2256d7dfc5cfSToshiaki Makita unsigned int idx = 0, start, i, j; 2257d7dfc5cfSToshiaki Makita const u8 *stats_base; 2258d7dfc5cfSToshiaki Makita size_t offset; 2259d7dfc5cfSToshiaki Makita 2260d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2261d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 2262d7dfc5cfSToshiaki Makita 2263d46eeeafSJason Wang stats_base = (u8 *)&rq->stats; 2264d7dfc5cfSToshiaki Makita do { 2265d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&rq->stats.syncp); 2266d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { 2267d7dfc5cfSToshiaki Makita offset = virtnet_rq_stats_desc[j].offset; 2268d7dfc5cfSToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 2269d7dfc5cfSToshiaki Makita } 2270d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); 2271d7dfc5cfSToshiaki Makita idx += VIRTNET_RQ_STATS_LEN; 2272d7dfc5cfSToshiaki Makita } 2273d7dfc5cfSToshiaki Makita 2274d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2275d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 2276d7dfc5cfSToshiaki Makita 2277d7dfc5cfSToshiaki Makita stats_base = (u8 *)&sq->stats; 2278d7dfc5cfSToshiaki Makita do { 2279d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&sq->stats.syncp); 2280d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { 2281d7dfc5cfSToshiaki Makita offset = virtnet_sq_stats_desc[j].offset; 2282d7dfc5cfSToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 2283d7dfc5cfSToshiaki Makita } 2284d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); 2285d7dfc5cfSToshiaki Makita idx += VIRTNET_SQ_STATS_LEN; 2286d7dfc5cfSToshiaki Makita } 2287d7dfc5cfSToshiaki Makita } 2288d7dfc5cfSToshiaki Makita 2289d73bcd2cSJason Wang static void virtnet_get_channels(struct net_device *dev, 2290d73bcd2cSJason Wang struct ethtool_channels *channels) 2291d73bcd2cSJason Wang { 2292d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 2293d73bcd2cSJason Wang 2294d73bcd2cSJason Wang channels->combined_count = vi->curr_queue_pairs; 2295d73bcd2cSJason Wang channels->max_combined = vi->max_queue_pairs; 2296d73bcd2cSJason Wang channels->max_other = 0; 2297d73bcd2cSJason Wang channels->rx_count = 0; 2298d73bcd2cSJason Wang channels->tx_count = 0; 2299d73bcd2cSJason Wang channels->other_count = 0; 2300d73bcd2cSJason Wang } 2301d73bcd2cSJason Wang 2302ebb6b4b1SPhilippe Reynes static int virtnet_set_link_ksettings(struct net_device *dev, 2303ebb6b4b1SPhilippe Reynes const struct ethtool_link_ksettings *cmd) 230416032be5SNikolay Aleksandrov { 230516032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 230616032be5SNikolay Aleksandrov 23079aedc6e2SCris Forno return ethtool_virtdev_set_link_ksettings(dev, cmd, 23089aedc6e2SCris Forno &vi->speed, &vi->duplex); 230916032be5SNikolay Aleksandrov } 231016032be5SNikolay Aleksandrov 2311ebb6b4b1SPhilippe Reynes static int virtnet_get_link_ksettings(struct net_device *dev, 2312ebb6b4b1SPhilippe Reynes struct ethtool_link_ksettings *cmd) 231316032be5SNikolay Aleksandrov { 231416032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 231516032be5SNikolay Aleksandrov 2316ebb6b4b1SPhilippe Reynes cmd->base.speed = vi->speed; 2317ebb6b4b1SPhilippe Reynes cmd->base.duplex = vi->duplex; 2318ebb6b4b1SPhilippe Reynes cmd->base.port = PORT_OTHER; 231916032be5SNikolay Aleksandrov 232016032be5SNikolay Aleksandrov return 0; 232116032be5SNikolay Aleksandrov } 232216032be5SNikolay Aleksandrov 23230c465be1SJason Wang static int virtnet_set_coalesce(struct net_device *dev, 2324f3ccfda1SYufeng Mo struct ethtool_coalesce *ec, 2325f3ccfda1SYufeng Mo struct kernel_ethtool_coalesce *kernel_coal, 2326f3ccfda1SYufeng Mo struct netlink_ext_ack *extack) 23270c465be1SJason Wang { 23280c465be1SJason Wang struct virtnet_info *vi = netdev_priv(dev); 23290c465be1SJason Wang int i, napi_weight; 23300c465be1SJason Wang 2331a51e5206SJakub Kicinski if (ec->tx_max_coalesced_frames > 1 || 2332a51e5206SJakub Kicinski ec->rx_max_coalesced_frames != 1) 23330c465be1SJason Wang return -EINVAL; 23340c465be1SJason Wang 23350c465be1SJason Wang napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 23360c465be1SJason Wang if (napi_weight ^ vi->sq[0].napi.weight) { 23370c465be1SJason Wang if (dev->flags & IFF_UP) 23380c465be1SJason Wang return -EBUSY; 23390c465be1SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 23400c465be1SJason Wang vi->sq[i].napi.weight = napi_weight; 23410c465be1SJason Wang } 23420c465be1SJason Wang 23430c465be1SJason Wang return 0; 23440c465be1SJason Wang } 23450c465be1SJason Wang 23460c465be1SJason Wang static int virtnet_get_coalesce(struct net_device *dev, 2347f3ccfda1SYufeng Mo struct ethtool_coalesce *ec, 2348f3ccfda1SYufeng Mo struct kernel_ethtool_coalesce *kernel_coal, 2349f3ccfda1SYufeng Mo struct netlink_ext_ack *extack) 23500c465be1SJason Wang { 23510c465be1SJason Wang struct ethtool_coalesce ec_default = { 23520c465be1SJason Wang .cmd = ETHTOOL_GCOALESCE, 23530c465be1SJason Wang .rx_max_coalesced_frames = 1, 23540c465be1SJason Wang }; 23550c465be1SJason Wang struct virtnet_info *vi = netdev_priv(dev); 23560c465be1SJason Wang 23570c465be1SJason Wang memcpy(ec, &ec_default, sizeof(ec_default)); 23580c465be1SJason Wang 23590c465be1SJason Wang if (vi->sq[0].napi.weight) 23600c465be1SJason Wang ec->tx_max_coalesced_frames = 1; 23610c465be1SJason Wang 23620c465be1SJason Wang return 0; 23630c465be1SJason Wang } 23640c465be1SJason Wang 236516032be5SNikolay Aleksandrov static void virtnet_init_settings(struct net_device *dev) 236616032be5SNikolay Aleksandrov { 236716032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 236816032be5SNikolay Aleksandrov 236916032be5SNikolay Aleksandrov vi->speed = SPEED_UNKNOWN; 237016032be5SNikolay Aleksandrov vi->duplex = DUPLEX_UNKNOWN; 237116032be5SNikolay Aleksandrov } 237216032be5SNikolay Aleksandrov 2373faa9b39fSJason Baron static void virtnet_update_settings(struct virtnet_info *vi) 2374faa9b39fSJason Baron { 2375faa9b39fSJason Baron u32 speed; 2376faa9b39fSJason Baron u8 duplex; 2377faa9b39fSJason Baron 2378faa9b39fSJason Baron if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 2379faa9b39fSJason Baron return; 2380faa9b39fSJason Baron 238164ffa39dSMichael S. Tsirkin virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); 238264ffa39dSMichael S. Tsirkin 2383faa9b39fSJason Baron if (ethtool_validate_speed(speed)) 2384faa9b39fSJason Baron vi->speed = speed; 238564ffa39dSMichael S. Tsirkin 238664ffa39dSMichael S. Tsirkin virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); 238764ffa39dSMichael S. Tsirkin 2388faa9b39fSJason Baron if (ethtool_validate_duplex(duplex)) 2389faa9b39fSJason Baron vi->duplex = duplex; 2390faa9b39fSJason Baron } 2391faa9b39fSJason Baron 23920fc0b732SStephen Hemminger static const struct ethtool_ops virtnet_ethtool_ops = { 2393a51e5206SJakub Kicinski .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES, 239466846048SRick Jones .get_drvinfo = virtnet_get_drvinfo, 23959f4d26d0SMark McLoughlin .get_link = ethtool_op_get_link, 23968f9f4668SRick Jones .get_ringparam = virtnet_get_ringparam, 2397d7dfc5cfSToshiaki Makita .get_strings = virtnet_get_strings, 2398d7dfc5cfSToshiaki Makita .get_sset_count = virtnet_get_sset_count, 2399d7dfc5cfSToshiaki Makita .get_ethtool_stats = virtnet_get_ethtool_stats, 2400d73bcd2cSJason Wang .set_channels = virtnet_set_channels, 2401d73bcd2cSJason Wang .get_channels = virtnet_get_channels, 2402074c3582SJacob Keller .get_ts_info = ethtool_op_get_ts_info, 2403ebb6b4b1SPhilippe Reynes .get_link_ksettings = virtnet_get_link_ksettings, 2404ebb6b4b1SPhilippe Reynes .set_link_ksettings = virtnet_set_link_ksettings, 24050c465be1SJason Wang .set_coalesce = virtnet_set_coalesce, 24060c465be1SJason Wang .get_coalesce = virtnet_get_coalesce, 2407a9ea3fc6SHerbert Xu }; 2408a9ea3fc6SHerbert Xu 24099fe7bfceSJohn Fastabend static void virtnet_freeze_down(struct virtio_device *vdev) 24109fe7bfceSJohn Fastabend { 24119fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 24129fe7bfceSJohn Fastabend int i; 24139fe7bfceSJohn Fastabend 24149fe7bfceSJohn Fastabend /* Make sure no work handler is accessing the device */ 24159fe7bfceSJohn Fastabend flush_work(&vi->config_work); 24169fe7bfceSJohn Fastabend 241705c998b7SAke Koomsin netif_tx_lock_bh(vi->dev); 24189fe7bfceSJohn Fastabend netif_device_detach(vi->dev); 241905c998b7SAke Koomsin netif_tx_unlock_bh(vi->dev); 24209fe7bfceSJohn Fastabend cancel_delayed_work_sync(&vi->refill); 24219fe7bfceSJohn Fastabend 24229fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 2423b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 24249fe7bfceSJohn Fastabend napi_disable(&vi->rq[i].napi); 242578a57b48SWillem de Bruijn virtnet_napi_tx_disable(&vi->sq[i].napi); 2426b92f1e67SWillem de Bruijn } 24279fe7bfceSJohn Fastabend } 24289fe7bfceSJohn Fastabend } 24299fe7bfceSJohn Fastabend 24309fe7bfceSJohn Fastabend static int init_vqs(struct virtnet_info *vi); 24319fe7bfceSJohn Fastabend 24329fe7bfceSJohn Fastabend static int virtnet_restore_up(struct virtio_device *vdev) 24339fe7bfceSJohn Fastabend { 24349fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 24359fe7bfceSJohn Fastabend int err, i; 24369fe7bfceSJohn Fastabend 24379fe7bfceSJohn Fastabend err = init_vqs(vi); 24389fe7bfceSJohn Fastabend if (err) 24399fe7bfceSJohn Fastabend return err; 24409fe7bfceSJohn Fastabend 24419fe7bfceSJohn Fastabend virtio_device_ready(vdev); 24429fe7bfceSJohn Fastabend 24439fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 24449fe7bfceSJohn Fastabend for (i = 0; i < vi->curr_queue_pairs; i++) 24459fe7bfceSJohn Fastabend if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 24469fe7bfceSJohn Fastabend schedule_delayed_work(&vi->refill, 0); 24479fe7bfceSJohn Fastabend 2448b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 2449e4e8452aSWillem de Bruijn virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2450b92f1e67SWillem de Bruijn virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2451b92f1e67SWillem de Bruijn &vi->sq[i].napi); 2452b92f1e67SWillem de Bruijn } 24539fe7bfceSJohn Fastabend } 24549fe7bfceSJohn Fastabend 245505c998b7SAke Koomsin netif_tx_lock_bh(vi->dev); 24569fe7bfceSJohn Fastabend netif_device_attach(vi->dev); 245705c998b7SAke Koomsin netif_tx_unlock_bh(vi->dev); 24589fe7bfceSJohn Fastabend return err; 24599fe7bfceSJohn Fastabend } 24609fe7bfceSJohn Fastabend 24613f93522fSJason Wang static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 24623f93522fSJason Wang { 24633f93522fSJason Wang struct scatterlist sg; 246412e57169SMichael S. Tsirkin vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads); 24653f93522fSJason Wang 246612e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads)); 24673f93522fSJason Wang 24683f93522fSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 24693f93522fSJason Wang VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 24703f93522fSJason Wang dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); 24713f93522fSJason Wang return -EINVAL; 24723f93522fSJason Wang } 24733f93522fSJason Wang 24743f93522fSJason Wang return 0; 24753f93522fSJason Wang } 24763f93522fSJason Wang 24773f93522fSJason Wang static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 24783f93522fSJason Wang { 24793f93522fSJason Wang u64 offloads = 0; 24803f93522fSJason Wang 24813f93522fSJason Wang if (!vi->guest_offloads) 24823f93522fSJason Wang return 0; 24833f93522fSJason Wang 24843f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 24853f93522fSJason Wang } 24863f93522fSJason Wang 24873f93522fSJason Wang static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 24883f93522fSJason Wang { 24893f93522fSJason Wang u64 offloads = vi->guest_offloads; 24903f93522fSJason Wang 24913f93522fSJason Wang if (!vi->guest_offloads) 24923f93522fSJason Wang return 0; 24933f93522fSJason Wang 24943f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 24953f93522fSJason Wang } 24963f93522fSJason Wang 24979861ce03SJakub Kicinski static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 24989861ce03SJakub Kicinski struct netlink_ext_ack *extack) 2499f600b690SJohn Fastabend { 2500f600b690SJohn Fastabend unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr); 2501f600b690SJohn Fastabend struct virtnet_info *vi = netdev_priv(dev); 2502f600b690SJohn Fastabend struct bpf_prog *old_prog; 2503017b29c3SJason Wang u16 xdp_qp = 0, curr_qp; 2504672aafd5SJohn Fastabend int i, err; 2505f600b690SJohn Fastabend 25063f93522fSJason Wang if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 25073f93522fSJason Wang && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 250892502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 250992502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 251018ba58e1SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 251118ba58e1SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) { 2512dbcf24d1SJason Wang NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); 2513f600b690SJohn Fastabend return -EOPNOTSUPP; 2514f600b690SJohn Fastabend } 2515f600b690SJohn Fastabend 2516f600b690SJohn Fastabend if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 25174d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 2518f600b690SJohn Fastabend return -EINVAL; 2519f600b690SJohn Fastabend } 2520f600b690SJohn Fastabend 2521f600b690SJohn Fastabend if (dev->mtu > max_sz) { 25224d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP"); 2523f600b690SJohn Fastabend netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz); 2524f600b690SJohn Fastabend return -EINVAL; 2525f600b690SJohn Fastabend } 2526f600b690SJohn Fastabend 2527672aafd5SJohn Fastabend curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 2528672aafd5SJohn Fastabend if (prog) 2529672aafd5SJohn Fastabend xdp_qp = nr_cpu_ids; 2530672aafd5SJohn Fastabend 2531672aafd5SJohn Fastabend /* XDP requires extra queues for XDP_TX */ 2532672aafd5SJohn Fastabend if (curr_qp + xdp_qp > vi->max_queue_pairs) { 2533*9ce4e3d6SXuan Zhuo netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n", 2534672aafd5SJohn Fastabend curr_qp + xdp_qp, vi->max_queue_pairs); 253597c2c69eSXuan Zhuo xdp_qp = 0; 2536672aafd5SJohn Fastabend } 2537672aafd5SJohn Fastabend 253803aa6d34SToshiaki Makita old_prog = rtnl_dereference(vi->rq[0].xdp_prog); 253903aa6d34SToshiaki Makita if (!prog && !old_prog) 254003aa6d34SToshiaki Makita return 0; 254103aa6d34SToshiaki Makita 254285192dbfSAndrii Nakryiko if (prog) 254385192dbfSAndrii Nakryiko bpf_prog_add(prog, vi->max_queue_pairs - 1); 25442de2f7f4SJohn Fastabend 25454941d472SJason Wang /* Make sure NAPI is not using any XDP TX queues for RX. */ 2546534da5e8SToshiaki Makita if (netif_running(dev)) { 2547534da5e8SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 25484941d472SJason Wang napi_disable(&vi->rq[i].napi); 2549534da5e8SToshiaki Makita virtnet_napi_tx_disable(&vi->sq[i].napi); 2550534da5e8SToshiaki Makita } 2551534da5e8SToshiaki Makita } 25522de2f7f4SJohn Fastabend 255303aa6d34SToshiaki Makita if (!prog) { 255403aa6d34SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 255503aa6d34SToshiaki Makita rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 255603aa6d34SToshiaki Makita if (i == 0) 255703aa6d34SToshiaki Makita virtnet_restore_guest_offloads(vi); 255803aa6d34SToshiaki Makita } 255903aa6d34SToshiaki Makita synchronize_net(); 256003aa6d34SToshiaki Makita } 256103aa6d34SToshiaki Makita 25624941d472SJason Wang err = _virtnet_set_queues(vi, curr_qp + xdp_qp); 25634941d472SJason Wang if (err) 25644941d472SJason Wang goto err; 2565188313c1SToshiaki Makita netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 25664941d472SJason Wang vi->xdp_queue_pairs = xdp_qp; 2567f600b690SJohn Fastabend 256803aa6d34SToshiaki Makita if (prog) { 256997c2c69eSXuan Zhuo vi->xdp_enabled = true; 2570f600b690SJohn Fastabend for (i = 0; i < vi->max_queue_pairs; i++) { 2571f600b690SJohn Fastabend rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 257203aa6d34SToshiaki Makita if (i == 0 && !old_prog) 25733f93522fSJason Wang virtnet_clear_guest_offloads(vi); 25743f93522fSJason Wang } 257597c2c69eSXuan Zhuo } else { 257697c2c69eSXuan Zhuo vi->xdp_enabled = false; 257703aa6d34SToshiaki Makita } 257803aa6d34SToshiaki Makita 257903aa6d34SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 2580f600b690SJohn Fastabend if (old_prog) 2581f600b690SJohn Fastabend bpf_prog_put(old_prog); 2582534da5e8SToshiaki Makita if (netif_running(dev)) { 25834941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2584534da5e8SToshiaki Makita virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2585534da5e8SToshiaki Makita &vi->sq[i].napi); 2586534da5e8SToshiaki Makita } 2587f600b690SJohn Fastabend } 2588f600b690SJohn Fastabend 2589f600b690SJohn Fastabend return 0; 25902de2f7f4SJohn Fastabend 25914941d472SJason Wang err: 259203aa6d34SToshiaki Makita if (!prog) { 259303aa6d34SToshiaki Makita virtnet_clear_guest_offloads(vi); 25944941d472SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 259503aa6d34SToshiaki Makita rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); 259603aa6d34SToshiaki Makita } 259703aa6d34SToshiaki Makita 25988be4d9a4SToshiaki Makita if (netif_running(dev)) { 2599534da5e8SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 26004941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2601534da5e8SToshiaki Makita virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2602534da5e8SToshiaki Makita &vi->sq[i].napi); 2603534da5e8SToshiaki Makita } 26048be4d9a4SToshiaki Makita } 26052de2f7f4SJohn Fastabend if (prog) 26062de2f7f4SJohn Fastabend bpf_prog_sub(prog, vi->max_queue_pairs - 1); 26072de2f7f4SJohn Fastabend return err; 2608f600b690SJohn Fastabend } 2609f600b690SJohn Fastabend 2610f4e63525SJakub Kicinski static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2611f600b690SJohn Fastabend { 2612f600b690SJohn Fastabend switch (xdp->command) { 2613f600b690SJohn Fastabend case XDP_SETUP_PROG: 26149861ce03SJakub Kicinski return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 2615f600b690SJohn Fastabend default: 2616f600b690SJohn Fastabend return -EINVAL; 2617f600b690SJohn Fastabend } 2618f600b690SJohn Fastabend } 2619f600b690SJohn Fastabend 2620ba5e4426SSridhar Samudrala static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, 2621ba5e4426SSridhar Samudrala size_t len) 2622ba5e4426SSridhar Samudrala { 2623ba5e4426SSridhar Samudrala struct virtnet_info *vi = netdev_priv(dev); 2624ba5e4426SSridhar Samudrala int ret; 2625ba5e4426SSridhar Samudrala 2626ba5e4426SSridhar Samudrala if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 2627ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 2628ba5e4426SSridhar Samudrala 2629ba5e4426SSridhar Samudrala ret = snprintf(buf, len, "sby"); 2630ba5e4426SSridhar Samudrala if (ret >= len) 2631ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 2632ba5e4426SSridhar Samudrala 2633ba5e4426SSridhar Samudrala return 0; 2634ba5e4426SSridhar Samudrala } 2635ba5e4426SSridhar Samudrala 2636a02e8964SWillem de Bruijn static int virtnet_set_features(struct net_device *dev, 2637a02e8964SWillem de Bruijn netdev_features_t features) 2638a02e8964SWillem de Bruijn { 2639a02e8964SWillem de Bruijn struct virtnet_info *vi = netdev_priv(dev); 2640cf8691cbSMichael S. Tsirkin u64 offloads; 2641a02e8964SWillem de Bruijn int err; 2642a02e8964SWillem de Bruijn 2643dbcf24d1SJason Wang if ((dev->features ^ features) & NETIF_F_GRO_HW) { 264497c2c69eSXuan Zhuo if (vi->xdp_enabled) 2645a02e8964SWillem de Bruijn return -EBUSY; 2646a02e8964SWillem de Bruijn 2647dbcf24d1SJason Wang if (features & NETIF_F_GRO_HW) 2648cf8691cbSMichael S. Tsirkin offloads = vi->guest_offloads_capable; 2649a02e8964SWillem de Bruijn else 2650cf8691cbSMichael S. Tsirkin offloads = vi->guest_offloads_capable & 2651dbcf24d1SJason Wang ~GUEST_OFFLOAD_GRO_HW_MASK; 2652a02e8964SWillem de Bruijn 2653a02e8964SWillem de Bruijn err = virtnet_set_guest_offloads(vi, offloads); 2654a02e8964SWillem de Bruijn if (err) 2655a02e8964SWillem de Bruijn return err; 26563618ad2aSTonghao Zhang vi->guest_offloads = offloads; 2657cf8691cbSMichael S. Tsirkin } 2658cf8691cbSMichael S. Tsirkin 2659a02e8964SWillem de Bruijn return 0; 2660a02e8964SWillem de Bruijn } 2661a02e8964SWillem de Bruijn 266276288b4eSStephen Hemminger static const struct net_device_ops virtnet_netdev = { 266376288b4eSStephen Hemminger .ndo_open = virtnet_open, 266476288b4eSStephen Hemminger .ndo_stop = virtnet_close, 266576288b4eSStephen Hemminger .ndo_start_xmit = start_xmit, 266676288b4eSStephen Hemminger .ndo_validate_addr = eth_validate_addr, 26679c46f6d4SAlex Williamson .ndo_set_mac_address = virtnet_set_mac_address, 26682af7698eSAlex Williamson .ndo_set_rx_mode = virtnet_set_rx_mode, 26693fa2a1dfSstephen hemminger .ndo_get_stats64 = virtnet_stats, 26701824a989SAlex Williamson .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 26711824a989SAlex Williamson .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 2672f4e63525SJakub Kicinski .ndo_bpf = virtnet_xdp, 2673186b3c99SJason Wang .ndo_xdp_xmit = virtnet_xdp_xmit, 26742836b4f2SVlad Yasevich .ndo_features_check = passthru_features_check, 2675ba5e4426SSridhar Samudrala .ndo_get_phys_port_name = virtnet_get_phys_port_name, 2676a02e8964SWillem de Bruijn .ndo_set_features = virtnet_set_features, 267776288b4eSStephen Hemminger }; 267876288b4eSStephen Hemminger 2679586d17c5SJason Wang static void virtnet_config_changed_work(struct work_struct *work) 26809f4d26d0SMark McLoughlin { 2681586d17c5SJason Wang struct virtnet_info *vi = 2682586d17c5SJason Wang container_of(work, struct virtnet_info, config_work); 26839f4d26d0SMark McLoughlin u16 v; 26849f4d26d0SMark McLoughlin 2685855e0c52SRusty Russell if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 2686855e0c52SRusty Russell struct virtio_net_config, status, &v) < 0) 2687507613bfSMichael S. Tsirkin return; 2688586d17c5SJason Wang 2689586d17c5SJason Wang if (v & VIRTIO_NET_S_ANNOUNCE) { 2690ee89bab1SAmerigo Wang netdev_notify_peers(vi->dev); 2691586d17c5SJason Wang virtnet_ack_link_announce(vi); 2692586d17c5SJason Wang } 26939f4d26d0SMark McLoughlin 26949f4d26d0SMark McLoughlin /* Ignore unknown (future) status bits */ 26959f4d26d0SMark McLoughlin v &= VIRTIO_NET_S_LINK_UP; 26969f4d26d0SMark McLoughlin 26979f4d26d0SMark McLoughlin if (vi->status == v) 2698507613bfSMichael S. Tsirkin return; 26999f4d26d0SMark McLoughlin 27009f4d26d0SMark McLoughlin vi->status = v; 27019f4d26d0SMark McLoughlin 27029f4d26d0SMark McLoughlin if (vi->status & VIRTIO_NET_S_LINK_UP) { 2703faa9b39fSJason Baron virtnet_update_settings(vi); 27049f4d26d0SMark McLoughlin netif_carrier_on(vi->dev); 2705986a4f4dSJason Wang netif_tx_wake_all_queues(vi->dev); 27069f4d26d0SMark McLoughlin } else { 27079f4d26d0SMark McLoughlin netif_carrier_off(vi->dev); 2708986a4f4dSJason Wang netif_tx_stop_all_queues(vi->dev); 27099f4d26d0SMark McLoughlin } 27109f4d26d0SMark McLoughlin } 27119f4d26d0SMark McLoughlin 27129f4d26d0SMark McLoughlin static void virtnet_config_changed(struct virtio_device *vdev) 27139f4d26d0SMark McLoughlin { 27149f4d26d0SMark McLoughlin struct virtnet_info *vi = vdev->priv; 27159f4d26d0SMark McLoughlin 27163b07e9caSTejun Heo schedule_work(&vi->config_work); 27179f4d26d0SMark McLoughlin } 27189f4d26d0SMark McLoughlin 2719986a4f4dSJason Wang static void virtnet_free_queues(struct virtnet_info *vi) 2720986a4f4dSJason Wang { 2721d4fb84eeSAndrey Vagin int i; 2722d4fb84eeSAndrey Vagin 2723ab3971b1SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 27245198d545SJakub Kicinski __netif_napi_del(&vi->rq[i].napi); 27255198d545SJakub Kicinski __netif_napi_del(&vi->sq[i].napi); 2726ab3971b1SJason Wang } 2727d4fb84eeSAndrey Vagin 27285198d545SJakub Kicinski /* We called __netif_napi_del(), 2729963abe5cSEric Dumazet * we need to respect an RCU grace period before freeing vi->rq 2730963abe5cSEric Dumazet */ 2731963abe5cSEric Dumazet synchronize_net(); 2732963abe5cSEric Dumazet 2733986a4f4dSJason Wang kfree(vi->rq); 2734986a4f4dSJason Wang kfree(vi->sq); 273512e57169SMichael S. Tsirkin kfree(vi->ctrl); 2736986a4f4dSJason Wang } 2737986a4f4dSJason Wang 273847315329SJohn Fastabend static void _free_receive_bufs(struct virtnet_info *vi) 2739986a4f4dSJason Wang { 2740f600b690SJohn Fastabend struct bpf_prog *old_prog; 2741986a4f4dSJason Wang int i; 2742986a4f4dSJason Wang 2743986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2744986a4f4dSJason Wang while (vi->rq[i].pages) 2745986a4f4dSJason Wang __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 2746f600b690SJohn Fastabend 2747f600b690SJohn Fastabend old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 2748f600b690SJohn Fastabend RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 2749f600b690SJohn Fastabend if (old_prog) 2750f600b690SJohn Fastabend bpf_prog_put(old_prog); 2751986a4f4dSJason Wang } 275247315329SJohn Fastabend } 275347315329SJohn Fastabend 275447315329SJohn Fastabend static void free_receive_bufs(struct virtnet_info *vi) 275547315329SJohn Fastabend { 275647315329SJohn Fastabend rtnl_lock(); 275747315329SJohn Fastabend _free_receive_bufs(vi); 2758f600b690SJohn Fastabend rtnl_unlock(); 2759986a4f4dSJason Wang } 2760986a4f4dSJason Wang 2761fb51879dSMichael Dalton static void free_receive_page_frags(struct virtnet_info *vi) 2762fb51879dSMichael Dalton { 2763fb51879dSMichael Dalton int i; 2764fb51879dSMichael Dalton for (i = 0; i < vi->max_queue_pairs; i++) 2765fb51879dSMichael Dalton if (vi->rq[i].alloc_frag.page) 2766fb51879dSMichael Dalton put_page(vi->rq[i].alloc_frag.page); 2767fb51879dSMichael Dalton } 2768fb51879dSMichael Dalton 2769986a4f4dSJason Wang static void free_unused_bufs(struct virtnet_info *vi) 2770986a4f4dSJason Wang { 2771986a4f4dSJason Wang void *buf; 2772986a4f4dSJason Wang int i; 2773986a4f4dSJason Wang 2774986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2775986a4f4dSJason Wang struct virtqueue *vq = vi->sq[i].vq; 277656434a01SJohn Fastabend while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { 27775050471dSToshiaki Makita if (!is_xdp_frame(buf)) 2778986a4f4dSJason Wang dev_kfree_skb(buf); 277956434a01SJohn Fastabend else 27805050471dSToshiaki Makita xdp_return_frame(ptr_to_xdp(buf)); 278156434a01SJohn Fastabend } 2782986a4f4dSJason Wang } 2783986a4f4dSJason Wang 2784986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2785986a4f4dSJason Wang struct virtqueue *vq = vi->rq[i].vq; 2786986a4f4dSJason Wang 2787986a4f4dSJason Wang while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { 2788ab7db917SMichael Dalton if (vi->mergeable_rx_bufs) { 2789680557cfSMichael S. Tsirkin put_page(virt_to_head_page(buf)); 2790ab7db917SMichael Dalton } else if (vi->big_packets) { 2791fa9fac17SAndrey Vagin give_pages(&vi->rq[i], buf); 2792ab7db917SMichael Dalton } else { 2793f6b10209SJason Wang put_page(virt_to_head_page(buf)); 2794986a4f4dSJason Wang } 2795986a4f4dSJason Wang } 2796986a4f4dSJason Wang } 2797ab7db917SMichael Dalton } 2798986a4f4dSJason Wang 2799e9d7417bSJason Wang static void virtnet_del_vqs(struct virtnet_info *vi) 2800e9d7417bSJason Wang { 2801e9d7417bSJason Wang struct virtio_device *vdev = vi->vdev; 2802e9d7417bSJason Wang 2803310974faSPeter Xu virtnet_clean_affinity(vi); 2804986a4f4dSJason Wang 2805e9d7417bSJason Wang vdev->config->del_vqs(vdev); 2806986a4f4dSJason Wang 2807986a4f4dSJason Wang virtnet_free_queues(vi); 2808986a4f4dSJason Wang } 2809986a4f4dSJason Wang 2810d85b758fSMichael S. Tsirkin /* How large should a single buffer be so a queue full of these can fit at 2811d85b758fSMichael S. Tsirkin * least one full packet? 2812d85b758fSMichael S. Tsirkin * Logic below assumes the mergeable buffer header is used. 2813d85b758fSMichael S. Tsirkin */ 2814d85b758fSMichael S. Tsirkin static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 2815d85b758fSMichael S. Tsirkin { 2816d85b758fSMichael S. Tsirkin const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 2817d85b758fSMichael S. Tsirkin unsigned int rq_size = virtqueue_get_vring_size(vq); 2818d85b758fSMichael S. Tsirkin unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 2819d85b758fSMichael S. Tsirkin unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 2820d85b758fSMichael S. Tsirkin unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 2821d85b758fSMichael S. Tsirkin 2822f0c3192cSMichael S. Tsirkin return max(max(min_buf_len, hdr_len) - hdr_len, 2823f0c3192cSMichael S. Tsirkin (unsigned int)GOOD_PACKET_LEN); 2824d85b758fSMichael S. Tsirkin } 2825d85b758fSMichael S. Tsirkin 2826986a4f4dSJason Wang static int virtnet_find_vqs(struct virtnet_info *vi) 2827986a4f4dSJason Wang { 2828986a4f4dSJason Wang vq_callback_t **callbacks; 2829986a4f4dSJason Wang struct virtqueue **vqs; 2830986a4f4dSJason Wang int ret = -ENOMEM; 2831986a4f4dSJason Wang int i, total_vqs; 2832986a4f4dSJason Wang const char **names; 2833d45b897bSMichael S. Tsirkin bool *ctx; 2834986a4f4dSJason Wang 2835986a4f4dSJason Wang /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 2836986a4f4dSJason Wang * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 2837986a4f4dSJason Wang * possible control vq. 2838986a4f4dSJason Wang */ 2839986a4f4dSJason Wang total_vqs = vi->max_queue_pairs * 2 + 2840986a4f4dSJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 2841986a4f4dSJason Wang 2842986a4f4dSJason Wang /* Allocate space for find_vqs parameters */ 28436396bb22SKees Cook vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); 2844986a4f4dSJason Wang if (!vqs) 2845986a4f4dSJason Wang goto err_vq; 28466da2ec56SKees Cook callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL); 2847986a4f4dSJason Wang if (!callbacks) 2848986a4f4dSJason Wang goto err_callback; 28496da2ec56SKees Cook names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL); 2850986a4f4dSJason Wang if (!names) 2851986a4f4dSJason Wang goto err_names; 2852192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 28536396bb22SKees Cook ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); 2854d45b897bSMichael S. Tsirkin if (!ctx) 2855d45b897bSMichael S. Tsirkin goto err_ctx; 2856d45b897bSMichael S. Tsirkin } else { 2857d45b897bSMichael S. Tsirkin ctx = NULL; 2858d45b897bSMichael S. Tsirkin } 2859986a4f4dSJason Wang 2860986a4f4dSJason Wang /* Parameters for control virtqueue, if any */ 2861986a4f4dSJason Wang if (vi->has_cvq) { 2862986a4f4dSJason Wang callbacks[total_vqs - 1] = NULL; 2863986a4f4dSJason Wang names[total_vqs - 1] = "control"; 2864986a4f4dSJason Wang } 2865986a4f4dSJason Wang 2866986a4f4dSJason Wang /* Allocate/initialize parameters for send/receive virtqueues */ 2867986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2868986a4f4dSJason Wang callbacks[rxq2vq(i)] = skb_recv_done; 2869986a4f4dSJason Wang callbacks[txq2vq(i)] = skb_xmit_done; 2870986a4f4dSJason Wang sprintf(vi->rq[i].name, "input.%d", i); 2871986a4f4dSJason Wang sprintf(vi->sq[i].name, "output.%d", i); 2872986a4f4dSJason Wang names[rxq2vq(i)] = vi->rq[i].name; 2873986a4f4dSJason Wang names[txq2vq(i)] = vi->sq[i].name; 2874d45b897bSMichael S. Tsirkin if (ctx) 2875d45b897bSMichael S. Tsirkin ctx[rxq2vq(i)] = true; 2876986a4f4dSJason Wang } 2877986a4f4dSJason Wang 2878a2f7dc00SXianting Tian ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks, 2879d45b897bSMichael S. Tsirkin names, ctx, NULL); 2880986a4f4dSJason Wang if (ret) 2881986a4f4dSJason Wang goto err_find; 2882986a4f4dSJason Wang 2883986a4f4dSJason Wang if (vi->has_cvq) { 2884986a4f4dSJason Wang vi->cvq = vqs[total_vqs - 1]; 2885986a4f4dSJason Wang if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 2886f646968fSPatrick McHardy vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 2887986a4f4dSJason Wang } 2888986a4f4dSJason Wang 2889986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2890986a4f4dSJason Wang vi->rq[i].vq = vqs[rxq2vq(i)]; 2891d85b758fSMichael S. Tsirkin vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 2892986a4f4dSJason Wang vi->sq[i].vq = vqs[txq2vq(i)]; 2893986a4f4dSJason Wang } 2894986a4f4dSJason Wang 28952fa3c8a8STonghao Zhang /* run here: ret == 0. */ 2896986a4f4dSJason Wang 2897986a4f4dSJason Wang 2898986a4f4dSJason Wang err_find: 2899d45b897bSMichael S. Tsirkin kfree(ctx); 2900d45b897bSMichael S. Tsirkin err_ctx: 2901986a4f4dSJason Wang kfree(names); 2902986a4f4dSJason Wang err_names: 2903986a4f4dSJason Wang kfree(callbacks); 2904986a4f4dSJason Wang err_callback: 2905986a4f4dSJason Wang kfree(vqs); 2906986a4f4dSJason Wang err_vq: 2907986a4f4dSJason Wang return ret; 2908986a4f4dSJason Wang } 2909986a4f4dSJason Wang 2910986a4f4dSJason Wang static int virtnet_alloc_queues(struct virtnet_info *vi) 2911986a4f4dSJason Wang { 2912986a4f4dSJason Wang int i; 2913986a4f4dSJason Wang 2914122b84a1SMax Gurtovoy if (vi->has_cvq) { 291512e57169SMichael S. Tsirkin vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 291612e57169SMichael S. Tsirkin if (!vi->ctrl) 291712e57169SMichael S. Tsirkin goto err_ctrl; 2918122b84a1SMax Gurtovoy } else { 2919122b84a1SMax Gurtovoy vi->ctrl = NULL; 2920122b84a1SMax Gurtovoy } 29216396bb22SKees Cook vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL); 2922986a4f4dSJason Wang if (!vi->sq) 2923986a4f4dSJason Wang goto err_sq; 29246396bb22SKees Cook vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL); 2925008d4278SAmerigo Wang if (!vi->rq) 2926986a4f4dSJason Wang goto err_rq; 2927986a4f4dSJason Wang 2928986a4f4dSJason Wang INIT_DELAYED_WORK(&vi->refill, refill_work); 2929986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2930986a4f4dSJason Wang vi->rq[i].pages = NULL; 2931986a4f4dSJason Wang netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll, 2932986a4f4dSJason Wang napi_weight); 29331d11e732SWillem de Bruijn netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx, 2934b92f1e67SWillem de Bruijn napi_tx ? napi_weight : 0); 2935986a4f4dSJason Wang 2936986a4f4dSJason Wang sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 29375377d758SJohannes Berg ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 2938986a4f4dSJason Wang sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 2939d7dfc5cfSToshiaki Makita 2940d7dfc5cfSToshiaki Makita u64_stats_init(&vi->rq[i].stats.syncp); 2941d7dfc5cfSToshiaki Makita u64_stats_init(&vi->sq[i].stats.syncp); 2942986a4f4dSJason Wang } 2943986a4f4dSJason Wang 2944986a4f4dSJason Wang return 0; 2945986a4f4dSJason Wang 2946986a4f4dSJason Wang err_rq: 2947986a4f4dSJason Wang kfree(vi->sq); 2948986a4f4dSJason Wang err_sq: 294912e57169SMichael S. Tsirkin kfree(vi->ctrl); 295012e57169SMichael S. Tsirkin err_ctrl: 2951986a4f4dSJason Wang return -ENOMEM; 2952e9d7417bSJason Wang } 2953e9d7417bSJason Wang 29543f9c10b0SAmit Shah static int init_vqs(struct virtnet_info *vi) 29553f9c10b0SAmit Shah { 2956986a4f4dSJason Wang int ret; 29573f9c10b0SAmit Shah 2958986a4f4dSJason Wang /* Allocate send & receive queues */ 2959986a4f4dSJason Wang ret = virtnet_alloc_queues(vi); 2960986a4f4dSJason Wang if (ret) 2961986a4f4dSJason Wang goto err; 29623f9c10b0SAmit Shah 2963986a4f4dSJason Wang ret = virtnet_find_vqs(vi); 2964986a4f4dSJason Wang if (ret) 2965986a4f4dSJason Wang goto err_free; 29663f9c10b0SAmit Shah 2967a0d1d0f4SSebastian Andrzej Siewior cpus_read_lock(); 29688898c21cSWanlong Gao virtnet_set_affinity(vi); 2969a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 297047be2479SWanlong Gao 29713f9c10b0SAmit Shah return 0; 2972986a4f4dSJason Wang 2973986a4f4dSJason Wang err_free: 2974986a4f4dSJason Wang virtnet_free_queues(vi); 2975986a4f4dSJason Wang err: 2976986a4f4dSJason Wang return ret; 29773f9c10b0SAmit Shah } 29783f9c10b0SAmit Shah 2979fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 2980fbf28d78SMichael Dalton static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 2981718ad681Sstephen hemminger char *buf) 2982fbf28d78SMichael Dalton { 2983fbf28d78SMichael Dalton struct virtnet_info *vi = netdev_priv(queue->dev); 2984fbf28d78SMichael Dalton unsigned int queue_index = get_netdev_rx_queue_index(queue); 29853cc81a9aSJason Wang unsigned int headroom = virtnet_get_headroom(vi); 29863cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 29875377d758SJohannes Berg struct ewma_pkt_len *avg; 2988fbf28d78SMichael Dalton 2989fbf28d78SMichael Dalton BUG_ON(queue_index >= vi->max_queue_pairs); 2990fbf28d78SMichael Dalton avg = &vi->rq[queue_index].mrg_avg_pkt_len; 2991d85b758fSMichael S. Tsirkin return sprintf(buf, "%u\n", 29923cc81a9aSJason Wang get_mergeable_buf_len(&vi->rq[queue_index], avg, 29933cc81a9aSJason Wang SKB_DATA_ALIGN(headroom + tailroom))); 2994fbf28d78SMichael Dalton } 2995fbf28d78SMichael Dalton 2996fbf28d78SMichael Dalton static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 2997fbf28d78SMichael Dalton __ATTR_RO(mergeable_rx_buffer_size); 2998fbf28d78SMichael Dalton 2999fbf28d78SMichael Dalton static struct attribute *virtio_net_mrg_rx_attrs[] = { 3000fbf28d78SMichael Dalton &mergeable_rx_buffer_size_attribute.attr, 3001fbf28d78SMichael Dalton NULL 3002fbf28d78SMichael Dalton }; 3003fbf28d78SMichael Dalton 3004fbf28d78SMichael Dalton static const struct attribute_group virtio_net_mrg_rx_group = { 3005fbf28d78SMichael Dalton .name = "virtio_net", 3006fbf28d78SMichael Dalton .attrs = virtio_net_mrg_rx_attrs 3007fbf28d78SMichael Dalton }; 3008fbf28d78SMichael Dalton #endif 3009fbf28d78SMichael Dalton 3010892d6eb1SJason Wang static bool virtnet_fail_on_feature(struct virtio_device *vdev, 3011892d6eb1SJason Wang unsigned int fbit, 3012892d6eb1SJason Wang const char *fname, const char *dname) 3013892d6eb1SJason Wang { 3014892d6eb1SJason Wang if (!virtio_has_feature(vdev, fbit)) 3015892d6eb1SJason Wang return false; 3016892d6eb1SJason Wang 3017892d6eb1SJason Wang dev_err(&vdev->dev, "device advertises feature %s but not %s", 3018892d6eb1SJason Wang fname, dname); 3019892d6eb1SJason Wang 3020892d6eb1SJason Wang return true; 3021892d6eb1SJason Wang } 3022892d6eb1SJason Wang 3023892d6eb1SJason Wang #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 3024892d6eb1SJason Wang virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 3025892d6eb1SJason Wang 3026892d6eb1SJason Wang static bool virtnet_validate_features(struct virtio_device *vdev) 3027892d6eb1SJason Wang { 3028892d6eb1SJason Wang if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 3029892d6eb1SJason Wang (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 3030892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 3031892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 3032892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 3033892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 3034892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 3035892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 3036892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 3037892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ"))) { 3038892d6eb1SJason Wang return false; 3039892d6eb1SJason Wang } 3040892d6eb1SJason Wang 3041892d6eb1SJason Wang return true; 3042892d6eb1SJason Wang } 3043892d6eb1SJason Wang 3044d0c2c997SJarod Wilson #define MIN_MTU ETH_MIN_MTU 3045d0c2c997SJarod Wilson #define MAX_MTU ETH_MAX_MTU 3046d0c2c997SJarod Wilson 3047fe36cbe0SMichael S. Tsirkin static int virtnet_validate(struct virtio_device *vdev) 3048296f96fcSRusty Russell { 30496ba42248SMichael S. Tsirkin if (!vdev->config->get) { 30506ba42248SMichael S. Tsirkin dev_err(&vdev->dev, "%s failure: config access disabled\n", 30516ba42248SMichael S. Tsirkin __func__); 30526ba42248SMichael S. Tsirkin return -EINVAL; 30536ba42248SMichael S. Tsirkin } 30546ba42248SMichael S. Tsirkin 3055892d6eb1SJason Wang if (!virtnet_validate_features(vdev)) 3056892d6eb1SJason Wang return -EINVAL; 3057892d6eb1SJason Wang 3058fe36cbe0SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 3059fe36cbe0SMichael S. Tsirkin int mtu = virtio_cread16(vdev, 3060fe36cbe0SMichael S. Tsirkin offsetof(struct virtio_net_config, 3061fe36cbe0SMichael S. Tsirkin mtu)); 3062fe36cbe0SMichael S. Tsirkin if (mtu < MIN_MTU) 3063fe36cbe0SMichael S. Tsirkin __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 3064fe36cbe0SMichael S. Tsirkin } 3065fe36cbe0SMichael S. Tsirkin 3066fe36cbe0SMichael S. Tsirkin return 0; 3067fe36cbe0SMichael S. Tsirkin } 3068fe36cbe0SMichael S. Tsirkin 3069fe36cbe0SMichael S. Tsirkin static int virtnet_probe(struct virtio_device *vdev) 3070fe36cbe0SMichael S. Tsirkin { 3071d7dfc5cfSToshiaki Makita int i, err = -ENOMEM; 3072fe36cbe0SMichael S. Tsirkin struct net_device *dev; 3073fe36cbe0SMichael S. Tsirkin struct virtnet_info *vi; 3074fe36cbe0SMichael S. Tsirkin u16 max_queue_pairs; 3075fe36cbe0SMichael S. Tsirkin int mtu; 3076fe36cbe0SMichael S. Tsirkin 3077986a4f4dSJason Wang /* Find if host supports multiqueue virtio_net device */ 3078855e0c52SRusty Russell err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ, 3079855e0c52SRusty Russell struct virtio_net_config, 3080855e0c52SRusty Russell max_virtqueue_pairs, &max_queue_pairs); 3081986a4f4dSJason Wang 3082986a4f4dSJason Wang /* We need at least 2 queue's */ 3083986a4f4dSJason Wang if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 3084986a4f4dSJason Wang max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 3085986a4f4dSJason Wang !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 3086986a4f4dSJason Wang max_queue_pairs = 1; 3087296f96fcSRusty Russell 3088296f96fcSRusty Russell /* Allocate ourselves a network device with room for our info */ 3089986a4f4dSJason Wang dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 3090296f96fcSRusty Russell if (!dev) 3091296f96fcSRusty Russell return -ENOMEM; 3092296f96fcSRusty Russell 3093296f96fcSRusty Russell /* Set up network device as normal. */ 3094ab5bd583SXuan Zhuo dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 3095ab5bd583SXuan Zhuo IFF_TX_SKB_NO_LINEAR; 309676288b4eSStephen Hemminger dev->netdev_ops = &virtnet_netdev; 3097296f96fcSRusty Russell dev->features = NETIF_F_HIGHDMA; 30983fa2a1dfSstephen hemminger 30997ad24ea4SWilfried Klaebe dev->ethtool_ops = &virtnet_ethtool_ops; 3100296f96fcSRusty Russell SET_NETDEV_DEV(dev, &vdev->dev); 3101296f96fcSRusty Russell 3102296f96fcSRusty Russell /* Do we support "hardware" checksums? */ 310398e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 3104296f96fcSRusty Russell /* This opens up the world of extra features. */ 310548900cb6SJason Wang dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 310698e778c9SMichał Mirosław if (csum) 310748900cb6SJason Wang dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 310898e778c9SMichał Mirosław 310998e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 3110e078de03SDavid S. Miller dev->hw_features |= NETIF_F_TSO 311134a48579SRusty Russell | NETIF_F_TSO_ECN | NETIF_F_TSO6; 311234a48579SRusty Russell } 31135539ae96SRusty Russell /* Individual feature bits: what can host handle? */ 311498e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 311598e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO; 311698e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 311798e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO6; 311898e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 311998e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO_ECN; 312098e778c9SMichał Mirosław 312141f2f127SJason Wang dev->features |= NETIF_F_GSO_ROBUST; 312241f2f127SJason Wang 312398e778c9SMichał Mirosław if (gso) 3124e078de03SDavid S. Miller dev->features |= dev->hw_features & NETIF_F_ALL_TSO; 312598e778c9SMichał Mirosław /* (!csum && gso) case will be fixed by register_netdev() */ 3126296f96fcSRusty Russell } 31274f49129bSThomas Huth if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM)) 31284f49129bSThomas Huth dev->features |= NETIF_F_RXCSUM; 3129a02e8964SWillem de Bruijn if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 3130a02e8964SWillem de Bruijn virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) 3131dbcf24d1SJason Wang dev->features |= NETIF_F_GRO_HW; 3132cf8691cbSMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) 3133dbcf24d1SJason Wang dev->hw_features |= NETIF_F_GRO_HW; 3134296f96fcSRusty Russell 31354fda8302SJason Wang dev->vlan_features = dev->features; 31364fda8302SJason Wang 3137d0c2c997SJarod Wilson /* MTU range: 68 - 65535 */ 3138d0c2c997SJarod Wilson dev->min_mtu = MIN_MTU; 3139d0c2c997SJarod Wilson dev->max_mtu = MAX_MTU; 3140d0c2c997SJarod Wilson 3141296f96fcSRusty Russell /* Configuration may specify what MAC to use. Otherwise random. */ 3142855e0c52SRusty Russell if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) 3143855e0c52SRusty Russell virtio_cread_bytes(vdev, 3144a586d4f6SRusty Russell offsetof(struct virtio_net_config, mac), 3145855e0c52SRusty Russell dev->dev_addr, dev->addr_len); 3146855e0c52SRusty Russell else 3147f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 3148296f96fcSRusty Russell 3149296f96fcSRusty Russell /* Set up our device-specific information */ 3150296f96fcSRusty Russell vi = netdev_priv(dev); 3151296f96fcSRusty Russell vi->dev = dev; 3152296f96fcSRusty Russell vi->vdev = vdev; 3153d9d5dcc8SChristian Borntraeger vdev->priv = vi; 3154827da44cSJohn Stultz 3155586d17c5SJason Wang INIT_WORK(&vi->config_work, virtnet_config_changed_work); 3156296f96fcSRusty Russell 315797402b96SHerbert Xu /* If we can receive ANY GSO packets, we must allocate large ones. */ 31588e95a202SJoe Perches if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 31598e95a202SJoe Perches virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) || 3160e3e3c423SVlad Yasevich virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) || 3161e3e3c423SVlad Yasevich virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO)) 316297402b96SHerbert Xu vi->big_packets = true; 316397402b96SHerbert Xu 31643f2c31d9SMark McLoughlin if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) 31653f2c31d9SMark McLoughlin vi->mergeable_rx_bufs = true; 31663f2c31d9SMark McLoughlin 3167d04302b3SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 3168d04302b3SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 3169012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 3170012873d0SMichael S. Tsirkin else 3171012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr); 3172012873d0SMichael S. Tsirkin 317375993300SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 317475993300SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 3175e7428e95SMichael S. Tsirkin vi->any_header_sg = true; 3176e7428e95SMichael S. Tsirkin 3177986a4f4dSJason Wang if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 3178986a4f4dSJason Wang vi->has_cvq = true; 3179986a4f4dSJason Wang 318014de9d11SAaron Conole if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 318114de9d11SAaron Conole mtu = virtio_cread16(vdev, 318214de9d11SAaron Conole offsetof(struct virtio_net_config, 318314de9d11SAaron Conole mtu)); 318493a205eeSAaron Conole if (mtu < dev->min_mtu) { 3185fe36cbe0SMichael S. Tsirkin /* Should never trigger: MTU was previously validated 3186fe36cbe0SMichael S. Tsirkin * in virtnet_validate. 3187fe36cbe0SMichael S. Tsirkin */ 31887934b481SYuval Shaia dev_err(&vdev->dev, 31897934b481SYuval Shaia "device MTU appears to have changed it is now %d < %d", 31907934b481SYuval Shaia mtu, dev->min_mtu); 3191411ea23aSDan Carpenter err = -EINVAL; 3192d7dfc5cfSToshiaki Makita goto free; 3193fe36cbe0SMichael S. Tsirkin } 3194fe36cbe0SMichael S. Tsirkin 3195d0c2c997SJarod Wilson dev->mtu = mtu; 319693a205eeSAaron Conole dev->max_mtu = mtu; 31972e123b44SMichael S. Tsirkin 31982e123b44SMichael S. Tsirkin /* TODO: size buffers correctly in this case. */ 31992e123b44SMichael S. Tsirkin if (dev->mtu > ETH_DATA_LEN) 32002e123b44SMichael S. Tsirkin vi->big_packets = true; 320114de9d11SAaron Conole } 320214de9d11SAaron Conole 3203012873d0SMichael S. Tsirkin if (vi->any_header_sg) 3204012873d0SMichael S. Tsirkin dev->needed_headroom = vi->hdr_len; 32056ebbc1a6SZhangjie \(HZ\) 320644900010SJason Wang /* Enable multiqueue by default */ 320744900010SJason Wang if (num_online_cpus() >= max_queue_pairs) 320844900010SJason Wang vi->curr_queue_pairs = max_queue_pairs; 320944900010SJason Wang else 321044900010SJason Wang vi->curr_queue_pairs = num_online_cpus(); 3211986a4f4dSJason Wang vi->max_queue_pairs = max_queue_pairs; 3212986a4f4dSJason Wang 3213986a4f4dSJason Wang /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 32143f9c10b0SAmit Shah err = init_vqs(vi); 3215d2a7dddaSMichael S. Tsirkin if (err) 3216d7dfc5cfSToshiaki Makita goto free; 3217d2a7dddaSMichael S. Tsirkin 3218fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 3219fbf28d78SMichael Dalton if (vi->mergeable_rx_bufs) 3220fbf28d78SMichael Dalton dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 3221fbf28d78SMichael Dalton #endif 32220f13b66bSZhi Yong Wu netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 32230f13b66bSZhi Yong Wu netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 3224986a4f4dSJason Wang 322516032be5SNikolay Aleksandrov virtnet_init_settings(dev); 322616032be5SNikolay Aleksandrov 3227ba5e4426SSridhar Samudrala if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 3228ba5e4426SSridhar Samudrala vi->failover = net_failover_create(vi->dev); 32294b8e6ac4SWei Yongjun if (IS_ERR(vi->failover)) { 32304b8e6ac4SWei Yongjun err = PTR_ERR(vi->failover); 3231ba5e4426SSridhar Samudrala goto free_vqs; 3232ba5e4426SSridhar Samudrala } 32334b8e6ac4SWei Yongjun } 3234ba5e4426SSridhar Samudrala 3235296f96fcSRusty Russell err = register_netdev(dev); 3236296f96fcSRusty Russell if (err) { 3237296f96fcSRusty Russell pr_debug("virtio_net: registering device failed\n"); 3238ba5e4426SSridhar Samudrala goto free_failover; 3239296f96fcSRusty Russell } 3240b3369c1fSRusty Russell 32414baf1e33SMichael S. Tsirkin virtio_device_ready(vdev); 32424baf1e33SMichael S. Tsirkin 32438017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 32448de4b2f3SWanlong Gao if (err) { 32458de4b2f3SWanlong Gao pr_debug("virtio_net: registering cpu notifier failed\n"); 3246f00e35e2Swangyunjian goto free_unregister_netdev; 32478de4b2f3SWanlong Gao } 32488de4b2f3SWanlong Gao 3249a220871bSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 325044900010SJason Wang 3251167c25e4SJason Wang /* Assume link up if device can't report link status, 3252167c25e4SJason Wang otherwise get link status from config. */ 3253167c25e4SJason Wang netif_carrier_off(dev); 3254bda7fab5SJay Vosburgh if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 32553b07e9caSTejun Heo schedule_work(&vi->config_work); 3256167c25e4SJason Wang } else { 3257167c25e4SJason Wang vi->status = VIRTIO_NET_S_LINK_UP; 3258faa9b39fSJason Baron virtnet_update_settings(vi); 32594783256eSPantelis Koukousoulas netif_carrier_on(dev); 3260167c25e4SJason Wang } 32619f4d26d0SMark McLoughlin 32623f93522fSJason Wang for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) 32633f93522fSJason Wang if (virtio_has_feature(vi->vdev, guest_offloads[i])) 32643f93522fSJason Wang set_bit(guest_offloads[i], &vi->guest_offloads); 3265a02e8964SWillem de Bruijn vi->guest_offloads_capable = vi->guest_offloads; 32663f93522fSJason Wang 3267986a4f4dSJason Wang pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 3268986a4f4dSJason Wang dev->name, max_queue_pairs); 3269986a4f4dSJason Wang 3270296f96fcSRusty Russell return 0; 3271296f96fcSRusty Russell 3272f00e35e2Swangyunjian free_unregister_netdev: 327302465555SMichael S. Tsirkin vi->vdev->config->reset(vdev); 327402465555SMichael S. Tsirkin 3275b3369c1fSRusty Russell unregister_netdev(dev); 3276ba5e4426SSridhar Samudrala free_failover: 3277ba5e4426SSridhar Samudrala net_failover_destroy(vi->failover); 3278d2a7dddaSMichael S. Tsirkin free_vqs: 3279986a4f4dSJason Wang cancel_delayed_work_sync(&vi->refill); 3280fb51879dSMichael Dalton free_receive_page_frags(vi); 3281e9d7417bSJason Wang virtnet_del_vqs(vi); 3282296f96fcSRusty Russell free: 3283296f96fcSRusty Russell free_netdev(dev); 3284296f96fcSRusty Russell return err; 3285296f96fcSRusty Russell } 3286296f96fcSRusty Russell 328704486ed0SAmit Shah static void remove_vq_common(struct virtnet_info *vi) 3288296f96fcSRusty Russell { 328904486ed0SAmit Shah vi->vdev->config->reset(vi->vdev); 3290830a8a97SShirley Ma 3291830a8a97SShirley Ma /* Free unused buffers in both send and recv, if any. */ 32929ab86bbcSShirley Ma free_unused_bufs(vi); 3293fb6813f4SRusty Russell 3294986a4f4dSJason Wang free_receive_bufs(vi); 3295d2a7dddaSMichael S. Tsirkin 3296fb51879dSMichael Dalton free_receive_page_frags(vi); 3297fb51879dSMichael Dalton 3298986a4f4dSJason Wang virtnet_del_vqs(vi); 329904486ed0SAmit Shah } 330004486ed0SAmit Shah 33018cc085d6SBill Pemberton static void virtnet_remove(struct virtio_device *vdev) 330204486ed0SAmit Shah { 330304486ed0SAmit Shah struct virtnet_info *vi = vdev->priv; 330404486ed0SAmit Shah 33058017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 33068de4b2f3SWanlong Gao 3307102a2786SMichael S. Tsirkin /* Make sure no work handler is accessing the device. */ 3308102a2786SMichael S. Tsirkin flush_work(&vi->config_work); 3309586d17c5SJason Wang 331004486ed0SAmit Shah unregister_netdev(vi->dev); 331104486ed0SAmit Shah 3312ba5e4426SSridhar Samudrala net_failover_destroy(vi->failover); 3313ba5e4426SSridhar Samudrala 331404486ed0SAmit Shah remove_vq_common(vi); 3315fb6813f4SRusty Russell 331674b2553fSRusty Russell free_netdev(vi->dev); 3317296f96fcSRusty Russell } 3318296f96fcSRusty Russell 331967a75194SArnd Bergmann static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 33200741bcb5SAmit Shah { 33210741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 33220741bcb5SAmit Shah 33238017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 33249fe7bfceSJohn Fastabend virtnet_freeze_down(vdev); 33250741bcb5SAmit Shah remove_vq_common(vi); 33260741bcb5SAmit Shah 33270741bcb5SAmit Shah return 0; 33280741bcb5SAmit Shah } 33290741bcb5SAmit Shah 333067a75194SArnd Bergmann static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 33310741bcb5SAmit Shah { 33320741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 33339fe7bfceSJohn Fastabend int err; 33340741bcb5SAmit Shah 33359fe7bfceSJohn Fastabend err = virtnet_restore_up(vdev); 33360741bcb5SAmit Shah if (err) 33370741bcb5SAmit Shah return err; 3338986a4f4dSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 3339986a4f4dSJason Wang 33408017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 33413f2869caSXie Yongji if (err) { 33423f2869caSXie Yongji virtnet_freeze_down(vdev); 33433f2869caSXie Yongji remove_vq_common(vi); 3344ec9debbdSJason Wang return err; 33453f2869caSXie Yongji } 3346ec9debbdSJason Wang 33470741bcb5SAmit Shah return 0; 33480741bcb5SAmit Shah } 33490741bcb5SAmit Shah 3350296f96fcSRusty Russell static struct virtio_device_id id_table[] = { 3351296f96fcSRusty Russell { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 3352296f96fcSRusty Russell { 0 }, 3353296f96fcSRusty Russell }; 3354296f96fcSRusty Russell 3355f3358507SMichael S. Tsirkin #define VIRTNET_FEATURES \ 3356f3358507SMichael S. Tsirkin VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 3357f3358507SMichael S. Tsirkin VIRTIO_NET_F_MAC, \ 3358f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 3359f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 3360f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 3361f3358507SMichael S. Tsirkin VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 3362f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 3363f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 3364f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_MAC_ADDR, \ 3365faa9b39fSJason Baron VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 33669805069dSSridhar Samudrala VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY 3367f3358507SMichael S. Tsirkin 3368c45a6816SRusty Russell static unsigned int features[] = { 3369f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 3370f3358507SMichael S. Tsirkin }; 3371f3358507SMichael S. Tsirkin 3372f3358507SMichael S. Tsirkin static unsigned int features_legacy[] = { 3373f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 3374f3358507SMichael S. Tsirkin VIRTIO_NET_F_GSO, 3375e7428e95SMichael S. Tsirkin VIRTIO_F_ANY_LAYOUT, 3376c45a6816SRusty Russell }; 3377c45a6816SRusty Russell 337822402529SUwe Kleine-König static struct virtio_driver virtio_net_driver = { 3379c45a6816SRusty Russell .feature_table = features, 3380c45a6816SRusty Russell .feature_table_size = ARRAY_SIZE(features), 3381f3358507SMichael S. Tsirkin .feature_table_legacy = features_legacy, 3382f3358507SMichael S. Tsirkin .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 3383296f96fcSRusty Russell .driver.name = KBUILD_MODNAME, 3384296f96fcSRusty Russell .driver.owner = THIS_MODULE, 3385296f96fcSRusty Russell .id_table = id_table, 3386fe36cbe0SMichael S. Tsirkin .validate = virtnet_validate, 3387296f96fcSRusty Russell .probe = virtnet_probe, 33888cc085d6SBill Pemberton .remove = virtnet_remove, 33899f4d26d0SMark McLoughlin .config_changed = virtnet_config_changed, 339089107000SAaron Lu #ifdef CONFIG_PM_SLEEP 33910741bcb5SAmit Shah .freeze = virtnet_freeze, 33920741bcb5SAmit Shah .restore = virtnet_restore, 33930741bcb5SAmit Shah #endif 3394296f96fcSRusty Russell }; 3395296f96fcSRusty Russell 33968017c279SSebastian Andrzej Siewior static __init int virtio_net_driver_init(void) 33978017c279SSebastian Andrzej Siewior { 33988017c279SSebastian Andrzej Siewior int ret; 33998017c279SSebastian Andrzej Siewior 340073c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 34018017c279SSebastian Andrzej Siewior virtnet_cpu_online, 34028017c279SSebastian Andrzej Siewior virtnet_cpu_down_prep); 34038017c279SSebastian Andrzej Siewior if (ret < 0) 34048017c279SSebastian Andrzej Siewior goto out; 34058017c279SSebastian Andrzej Siewior virtionet_online = ret; 340673c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 34078017c279SSebastian Andrzej Siewior NULL, virtnet_cpu_dead); 34088017c279SSebastian Andrzej Siewior if (ret) 34098017c279SSebastian Andrzej Siewior goto err_dead; 34108017c279SSebastian Andrzej Siewior 34118017c279SSebastian Andrzej Siewior ret = register_virtio_driver(&virtio_net_driver); 34128017c279SSebastian Andrzej Siewior if (ret) 34138017c279SSebastian Andrzej Siewior goto err_virtio; 34148017c279SSebastian Andrzej Siewior return 0; 34158017c279SSebastian Andrzej Siewior err_virtio: 34168017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 34178017c279SSebastian Andrzej Siewior err_dead: 34188017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 34198017c279SSebastian Andrzej Siewior out: 34208017c279SSebastian Andrzej Siewior return ret; 34218017c279SSebastian Andrzej Siewior } 34228017c279SSebastian Andrzej Siewior module_init(virtio_net_driver_init); 34238017c279SSebastian Andrzej Siewior 34248017c279SSebastian Andrzej Siewior static __exit void virtio_net_driver_exit(void) 34258017c279SSebastian Andrzej Siewior { 3426cfa0ebc9SAndrew Jones unregister_virtio_driver(&virtio_net_driver); 34278017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 34288017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 34298017c279SSebastian Andrzej Siewior } 34308017c279SSebastian Andrzej Siewior module_exit(virtio_net_driver_exit); 3431296f96fcSRusty Russell 3432296f96fcSRusty Russell MODULE_DEVICE_TABLE(virtio, id_table); 3433296f96fcSRusty Russell MODULE_DESCRIPTION("Virtio network driver"); 3434296f96fcSRusty Russell MODULE_LICENSE("GPL"); 3435