11ccea77eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 248925e37SRusty Russell /* A network driver using virtio. 3296f96fcSRusty Russell * 4296f96fcSRusty Russell * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5296f96fcSRusty Russell */ 6296f96fcSRusty Russell //#define DEBUG 7296f96fcSRusty Russell #include <linux/netdevice.h> 8296f96fcSRusty Russell #include <linux/etherdevice.h> 9a9ea3fc6SHerbert Xu #include <linux/ethtool.h> 10296f96fcSRusty Russell #include <linux/module.h> 11296f96fcSRusty Russell #include <linux/virtio.h> 12296f96fcSRusty Russell #include <linux/virtio_net.h> 13f600b690SJohn Fastabend #include <linux/bpf.h> 14a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 15296f96fcSRusty Russell #include <linux/scatterlist.h> 16e918085aSAlex Williamson #include <linux/if_vlan.h> 175a0e3ad6STejun Heo #include <linux/slab.h> 188de4b2f3SWanlong Gao #include <linux/cpu.h> 19ab7db917SMichael Dalton #include <linux/average.h> 20186b3c99SJason Wang #include <linux/filter.h> 212ca653d6SCaleb Raitto #include <linux/kernel.h> 22d85b758fSMichael S. Tsirkin #include <net/route.h> 23754b8a21SJesper Dangaard Brouer #include <net/xdp.h> 24ba5e4426SSridhar Samudrala #include <net/net_failover.h> 25296f96fcSRusty Russell 26d34710e3SAmerigo Wang static int napi_weight = NAPI_POLL_WEIGHT; 276c0cd7c0SDor Laor module_param(napi_weight, int, 0444); 286c0cd7c0SDor Laor 2931c03aefSWillem de Bruijn static bool csum = true, gso = true, napi_tx = true; 3034a48579SRusty Russell module_param(csum, bool, 0444); 3134a48579SRusty Russell module_param(gso, bool, 0444); 32b92f1e67SWillem de Bruijn module_param(napi_tx, bool, 0644); 3334a48579SRusty Russell 34296f96fcSRusty Russell /* FIXME: MTU in config. */ 355061de36SMichael Dalton #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 363f2c31d9SMark McLoughlin #define GOOD_COPY_LEN 128 37296f96fcSRusty Russell 38f6b10209SJason Wang #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 39f6b10209SJason Wang 402de2f7f4SJohn Fastabend /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */ 412de2f7f4SJohn Fastabend #define VIRTIO_XDP_HEADROOM 256 422de2f7f4SJohn Fastabend 432471c75eSJesper Dangaard Brouer /* Separating two types of XDP xmit */ 442471c75eSJesper Dangaard Brouer #define VIRTIO_XDP_TX BIT(0) 452471c75eSJesper Dangaard Brouer #define VIRTIO_XDP_REDIR BIT(1) 462471c75eSJesper Dangaard Brouer 475050471dSToshiaki Makita #define VIRTIO_XDP_FLAG BIT(0) 485050471dSToshiaki Makita 495377d758SJohannes Berg /* RX packet size EWMA. The average packet size is used to determine the packet 505377d758SJohannes Berg * buffer size when refilling RX rings. As the entire RX ring may be refilled 515377d758SJohannes Berg * at once, the weight is chosen so that the EWMA will be insensitive to short- 525377d758SJohannes Berg * term, transient changes in packet size. 53ab7db917SMichael Dalton */ 54eb1e011aSJohannes Berg DECLARE_EWMA(pkt_len, 0, 64) 55ab7db917SMichael Dalton 5666846048SRick Jones #define VIRTNET_DRIVER_VERSION "1.0.0" 572a41f71dSAlex Williamson 587acd4329SColin Ian King static const unsigned long guest_offloads[] = { 597acd4329SColin Ian King VIRTIO_NET_F_GUEST_TSO4, 603f93522fSJason Wang VIRTIO_NET_F_GUEST_TSO6, 613f93522fSJason Wang VIRTIO_NET_F_GUEST_ECN, 62e59ff2c4SJason Wang VIRTIO_NET_F_GUEST_UFO, 63e59ff2c4SJason Wang VIRTIO_NET_F_GUEST_CSUM 647acd4329SColin Ian King }; 653f93522fSJason Wang 66dbcf24d1SJason Wang #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 671a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 681a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 691a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_UFO)) 701a03b8a3STonghao Zhang 71d7dfc5cfSToshiaki Makita struct virtnet_stat_desc { 72d7dfc5cfSToshiaki Makita char desc[ETH_GSTRING_LEN]; 73d7dfc5cfSToshiaki Makita size_t offset; 743fa2a1dfSstephen hemminger }; 753fa2a1dfSstephen hemminger 76d7dfc5cfSToshiaki Makita struct virtnet_sq_stats { 77d7dfc5cfSToshiaki Makita struct u64_stats_sync syncp; 78d7dfc5cfSToshiaki Makita u64 packets; 79d7dfc5cfSToshiaki Makita u64 bytes; 805b8f3c8dSToshiaki Makita u64 xdp_tx; 815b8f3c8dSToshiaki Makita u64 xdp_tx_drops; 82461f03dcSToshiaki Makita u64 kicks; 83a520794bSTony Lu u64 tx_timeouts; 84d7dfc5cfSToshiaki Makita }; 85d7dfc5cfSToshiaki Makita 86d46eeeafSJason Wang struct virtnet_rq_stats { 87d46eeeafSJason Wang struct u64_stats_sync syncp; 88d7dfc5cfSToshiaki Makita u64 packets; 89d7dfc5cfSToshiaki Makita u64 bytes; 902c4a2f7dSToshiaki Makita u64 drops; 915b8f3c8dSToshiaki Makita u64 xdp_packets; 925b8f3c8dSToshiaki Makita u64 xdp_tx; 935b8f3c8dSToshiaki Makita u64 xdp_redirects; 945b8f3c8dSToshiaki Makita u64 xdp_drops; 95461f03dcSToshiaki Makita u64 kicks; 96d7dfc5cfSToshiaki Makita }; 97d7dfc5cfSToshiaki Makita 98d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m) 99d46eeeafSJason Wang #define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m) 100d7dfc5cfSToshiaki Makita 101d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 102d7dfc5cfSToshiaki Makita { "packets", VIRTNET_SQ_STAT(packets) }, 103d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_SQ_STAT(bytes) }, 1045b8f3c8dSToshiaki Makita { "xdp_tx", VIRTNET_SQ_STAT(xdp_tx) }, 1055b8f3c8dSToshiaki Makita { "xdp_tx_drops", VIRTNET_SQ_STAT(xdp_tx_drops) }, 106461f03dcSToshiaki Makita { "kicks", VIRTNET_SQ_STAT(kicks) }, 107a520794bSTony Lu { "tx_timeouts", VIRTNET_SQ_STAT(tx_timeouts) }, 108d7dfc5cfSToshiaki Makita }; 109d7dfc5cfSToshiaki Makita 110d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 111d7dfc5cfSToshiaki Makita { "packets", VIRTNET_RQ_STAT(packets) }, 112d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_RQ_STAT(bytes) }, 1132c4a2f7dSToshiaki Makita { "drops", VIRTNET_RQ_STAT(drops) }, 1145b8f3c8dSToshiaki Makita { "xdp_packets", VIRTNET_RQ_STAT(xdp_packets) }, 1155b8f3c8dSToshiaki Makita { "xdp_tx", VIRTNET_RQ_STAT(xdp_tx) }, 1165b8f3c8dSToshiaki Makita { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) }, 1175b8f3c8dSToshiaki Makita { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) }, 118461f03dcSToshiaki Makita { "kicks", VIRTNET_RQ_STAT(kicks) }, 119d7dfc5cfSToshiaki Makita }; 120d7dfc5cfSToshiaki Makita 121d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc) 122d7dfc5cfSToshiaki Makita #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc) 123d7dfc5cfSToshiaki Makita 124e9d7417bSJason Wang /* Internal representation of a send virtqueue */ 125e9d7417bSJason Wang struct send_queue { 126e9d7417bSJason Wang /* Virtqueue associated with this send _queue */ 127e9d7417bSJason Wang struct virtqueue *vq; 128e9d7417bSJason Wang 129e9d7417bSJason Wang /* TX: fragments + linear part + virtio header */ 130e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 131986a4f4dSJason Wang 132986a4f4dSJason Wang /* Name of the send queue: output.$index */ 133986a4f4dSJason Wang char name[40]; 134b92f1e67SWillem de Bruijn 135d7dfc5cfSToshiaki Makita struct virtnet_sq_stats stats; 136d7dfc5cfSToshiaki Makita 137b92f1e67SWillem de Bruijn struct napi_struct napi; 138e9d7417bSJason Wang }; 139e9d7417bSJason Wang 140e9d7417bSJason Wang /* Internal representation of a receive virtqueue */ 141e9d7417bSJason Wang struct receive_queue { 142e9d7417bSJason Wang /* Virtqueue associated with this receive_queue */ 143e9d7417bSJason Wang struct virtqueue *vq; 144e9d7417bSJason Wang 145296f96fcSRusty Russell struct napi_struct napi; 146296f96fcSRusty Russell 147f600b690SJohn Fastabend struct bpf_prog __rcu *xdp_prog; 148f600b690SJohn Fastabend 149d7dfc5cfSToshiaki Makita struct virtnet_rq_stats stats; 150d7dfc5cfSToshiaki Makita 151e9d7417bSJason Wang /* Chain pages by the private ptr. */ 152e9d7417bSJason Wang struct page *pages; 153e9d7417bSJason Wang 154ab7db917SMichael Dalton /* Average packet length for mergeable receive buffers. */ 1555377d758SJohannes Berg struct ewma_pkt_len mrg_avg_pkt_len; 156ab7db917SMichael Dalton 157fb51879dSMichael Dalton /* Page frag for packet buffer allocation. */ 158fb51879dSMichael Dalton struct page_frag alloc_frag; 159fb51879dSMichael Dalton 160e9d7417bSJason Wang /* RX: fragments + linear part + virtio header */ 161e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 162986a4f4dSJason Wang 163d85b758fSMichael S. Tsirkin /* Min single buffer size for mergeable buffers case. */ 164d85b758fSMichael S. Tsirkin unsigned int min_buf_len; 165d85b758fSMichael S. Tsirkin 166986a4f4dSJason Wang /* Name of this receive queue: input.$index */ 167986a4f4dSJason Wang char name[40]; 168754b8a21SJesper Dangaard Brouer 169754b8a21SJesper Dangaard Brouer struct xdp_rxq_info xdp_rxq; 170e9d7417bSJason Wang }; 171e9d7417bSJason Wang 172*c7114b12SAndrew Melnychenko /* This structure can contain rss message with maximum settings for indirection table and keysize 173*c7114b12SAndrew Melnychenko * Note, that default structure that describes RSS configuration virtio_net_rss_config 174*c7114b12SAndrew Melnychenko * contains same info but can't handle table values. 175*c7114b12SAndrew Melnychenko * In any case, structure would be passed to virtio hw through sg_buf split by parts 176*c7114b12SAndrew Melnychenko * because table sizes may be differ according to the device configuration. 177*c7114b12SAndrew Melnychenko */ 178*c7114b12SAndrew Melnychenko #define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 179*c7114b12SAndrew Melnychenko #define VIRTIO_NET_RSS_MAX_TABLE_LEN 128 180*c7114b12SAndrew Melnychenko struct virtio_net_ctrl_rss { 181*c7114b12SAndrew Melnychenko u32 hash_types; 182*c7114b12SAndrew Melnychenko u16 indirection_table_mask; 183*c7114b12SAndrew Melnychenko u16 unclassified_queue; 184*c7114b12SAndrew Melnychenko u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN]; 185*c7114b12SAndrew Melnychenko u16 max_tx_vq; 186*c7114b12SAndrew Melnychenko u8 hash_key_length; 187*c7114b12SAndrew Melnychenko u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE]; 188*c7114b12SAndrew Melnychenko }; 189*c7114b12SAndrew Melnychenko 19012e57169SMichael S. Tsirkin /* Control VQ buffers: protected by the rtnl lock */ 19112e57169SMichael S. Tsirkin struct control_buf { 19212e57169SMichael S. Tsirkin struct virtio_net_ctrl_hdr hdr; 19312e57169SMichael S. Tsirkin virtio_net_ctrl_ack status; 19412e57169SMichael S. Tsirkin struct virtio_net_ctrl_mq mq; 19512e57169SMichael S. Tsirkin u8 promisc; 19612e57169SMichael S. Tsirkin u8 allmulti; 197d7fad4c8SMichael S. Tsirkin __virtio16 vid; 198f4ee703aSMichael S. Tsirkin __virtio64 offloads; 199*c7114b12SAndrew Melnychenko struct virtio_net_ctrl_rss rss; 20012e57169SMichael S. Tsirkin }; 20112e57169SMichael S. Tsirkin 202e9d7417bSJason Wang struct virtnet_info { 203e9d7417bSJason Wang struct virtio_device *vdev; 204e9d7417bSJason Wang struct virtqueue *cvq; 205e9d7417bSJason Wang struct net_device *dev; 206986a4f4dSJason Wang struct send_queue *sq; 207986a4f4dSJason Wang struct receive_queue *rq; 208e9d7417bSJason Wang unsigned int status; 209e9d7417bSJason Wang 210986a4f4dSJason Wang /* Max # of queue pairs supported by the device */ 211986a4f4dSJason Wang u16 max_queue_pairs; 212986a4f4dSJason Wang 213986a4f4dSJason Wang /* # of queue pairs currently used by the driver */ 214986a4f4dSJason Wang u16 curr_queue_pairs; 215986a4f4dSJason Wang 216672aafd5SJohn Fastabend /* # of XDP queue pairs currently used by the driver */ 217672aafd5SJohn Fastabend u16 xdp_queue_pairs; 218672aafd5SJohn Fastabend 21997c2c69eSXuan Zhuo /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ 22097c2c69eSXuan Zhuo bool xdp_enabled; 22197c2c69eSXuan Zhuo 22297402b96SHerbert Xu /* I like... big packets and I cannot lie! */ 22397402b96SHerbert Xu bool big_packets; 22497402b96SHerbert Xu 2253f2c31d9SMark McLoughlin /* Host will merge rx buffers for big packets (shake it! shake it!) */ 2263f2c31d9SMark McLoughlin bool mergeable_rx_bufs; 2273f2c31d9SMark McLoughlin 228*c7114b12SAndrew Melnychenko /* Host supports rss and/or hash report */ 229*c7114b12SAndrew Melnychenko bool has_rss; 230*c7114b12SAndrew Melnychenko u8 rss_key_size; 231*c7114b12SAndrew Melnychenko u16 rss_indir_table_size; 232*c7114b12SAndrew Melnychenko u32 rss_hash_types_supported; 233*c7114b12SAndrew Melnychenko 234986a4f4dSJason Wang /* Has control virtqueue */ 235986a4f4dSJason Wang bool has_cvq; 236986a4f4dSJason Wang 237e7428e95SMichael S. Tsirkin /* Host can handle any s/g split between our header and packet data */ 238e7428e95SMichael S. Tsirkin bool any_header_sg; 239e7428e95SMichael S. Tsirkin 240012873d0SMichael S. Tsirkin /* Packet virtio header size */ 241012873d0SMichael S. Tsirkin u8 hdr_len; 242012873d0SMichael S. Tsirkin 2433161e453SRusty Russell /* Work struct for refilling if we run low on memory. */ 2443161e453SRusty Russell struct delayed_work refill; 2453161e453SRusty Russell 246586d17c5SJason Wang /* Work struct for config space updates */ 247586d17c5SJason Wang struct work_struct config_work; 248586d17c5SJason Wang 249986a4f4dSJason Wang /* Does the affinity hint is set for virtqueues? */ 250986a4f4dSJason Wang bool affinity_hint_set; 25147be2479SWanlong Gao 2528017c279SSebastian Andrzej Siewior /* CPU hotplug instances for online & dead */ 2538017c279SSebastian Andrzej Siewior struct hlist_node node; 2548017c279SSebastian Andrzej Siewior struct hlist_node node_dead; 2552ac46030SMichael S. Tsirkin 25612e57169SMichael S. Tsirkin struct control_buf *ctrl; 25716032be5SNikolay Aleksandrov 25816032be5SNikolay Aleksandrov /* Ethtool settings */ 25916032be5SNikolay Aleksandrov u8 duplex; 26016032be5SNikolay Aleksandrov u32 speed; 2613f93522fSJason Wang 2623f93522fSJason Wang unsigned long guest_offloads; 263a02e8964SWillem de Bruijn unsigned long guest_offloads_capable; 264ba5e4426SSridhar Samudrala 265ba5e4426SSridhar Samudrala /* failover when STANDBY feature enabled */ 266ba5e4426SSridhar Samudrala struct failover *failover; 267296f96fcSRusty Russell }; 268296f96fcSRusty Russell 2699ab86bbcSShirley Ma struct padded_vnet_hdr { 270c1ddc42dSAndrew Melnychenko struct virtio_net_hdr_v1_hash hdr; 2719ab86bbcSShirley Ma /* 272012873d0SMichael S. Tsirkin * hdr is in a separate sg buffer, and data sg buffer shares same page 273012873d0SMichael S. Tsirkin * with this header sg. This padding makes next sg 16 byte aligned 274012873d0SMichael S. Tsirkin * after the header. 2759ab86bbcSShirley Ma */ 276c1ddc42dSAndrew Melnychenko char padding[12]; 2779ab86bbcSShirley Ma }; 2789ab86bbcSShirley Ma 2795050471dSToshiaki Makita static bool is_xdp_frame(void *ptr) 2805050471dSToshiaki Makita { 2815050471dSToshiaki Makita return (unsigned long)ptr & VIRTIO_XDP_FLAG; 2825050471dSToshiaki Makita } 2835050471dSToshiaki Makita 2845050471dSToshiaki Makita static void *xdp_to_ptr(struct xdp_frame *ptr) 2855050471dSToshiaki Makita { 2865050471dSToshiaki Makita return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG); 2875050471dSToshiaki Makita } 2885050471dSToshiaki Makita 2895050471dSToshiaki Makita static struct xdp_frame *ptr_to_xdp(void *ptr) 2905050471dSToshiaki Makita { 2915050471dSToshiaki Makita return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); 2925050471dSToshiaki Makita } 2935050471dSToshiaki Makita 294986a4f4dSJason Wang /* Converting between virtqueue no. and kernel tx/rx queue no. 295986a4f4dSJason Wang * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 296986a4f4dSJason Wang */ 297986a4f4dSJason Wang static int vq2txq(struct virtqueue *vq) 298986a4f4dSJason Wang { 2999d0ca6edSRusty Russell return (vq->index - 1) / 2; 300986a4f4dSJason Wang } 301986a4f4dSJason Wang 302986a4f4dSJason Wang static int txq2vq(int txq) 303986a4f4dSJason Wang { 304986a4f4dSJason Wang return txq * 2 + 1; 305986a4f4dSJason Wang } 306986a4f4dSJason Wang 307986a4f4dSJason Wang static int vq2rxq(struct virtqueue *vq) 308986a4f4dSJason Wang { 3099d0ca6edSRusty Russell return vq->index / 2; 310986a4f4dSJason Wang } 311986a4f4dSJason Wang 312986a4f4dSJason Wang static int rxq2vq(int rxq) 313986a4f4dSJason Wang { 314986a4f4dSJason Wang return rxq * 2; 315986a4f4dSJason Wang } 316986a4f4dSJason Wang 317012873d0SMichael S. Tsirkin static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb) 318296f96fcSRusty Russell { 319012873d0SMichael S. Tsirkin return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb; 320296f96fcSRusty Russell } 321296f96fcSRusty Russell 3229ab86bbcSShirley Ma /* 3239ab86bbcSShirley Ma * private is used to chain pages for big packets, put the whole 3249ab86bbcSShirley Ma * most recent used list in the beginning for reuse 3259ab86bbcSShirley Ma */ 326e9d7417bSJason Wang static void give_pages(struct receive_queue *rq, struct page *page) 327fb6813f4SRusty Russell { 3289ab86bbcSShirley Ma struct page *end; 3299ab86bbcSShirley Ma 330e9d7417bSJason Wang /* Find end of list, sew whole thing into vi->rq.pages. */ 3319ab86bbcSShirley Ma for (end = page; end->private; end = (struct page *)end->private); 332e9d7417bSJason Wang end->private = (unsigned long)rq->pages; 333e9d7417bSJason Wang rq->pages = page; 334fb6813f4SRusty Russell } 335fb6813f4SRusty Russell 336e9d7417bSJason Wang static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 337fb6813f4SRusty Russell { 338e9d7417bSJason Wang struct page *p = rq->pages; 339fb6813f4SRusty Russell 3409ab86bbcSShirley Ma if (p) { 341e9d7417bSJason Wang rq->pages = (struct page *)p->private; 3429ab86bbcSShirley Ma /* clear private here, it is used to chain pages */ 3439ab86bbcSShirley Ma p->private = 0; 3449ab86bbcSShirley Ma } else 345fb6813f4SRusty Russell p = alloc_page(gfp_mask); 346fb6813f4SRusty Russell return p; 347fb6813f4SRusty Russell } 348fb6813f4SRusty Russell 349e4e8452aSWillem de Bruijn static void virtqueue_napi_schedule(struct napi_struct *napi, 350e4e8452aSWillem de Bruijn struct virtqueue *vq) 351e4e8452aSWillem de Bruijn { 352e4e8452aSWillem de Bruijn if (napi_schedule_prep(napi)) { 353e4e8452aSWillem de Bruijn virtqueue_disable_cb(vq); 354e4e8452aSWillem de Bruijn __napi_schedule(napi); 355e4e8452aSWillem de Bruijn } 356e4e8452aSWillem de Bruijn } 357e4e8452aSWillem de Bruijn 358e4e8452aSWillem de Bruijn static void virtqueue_napi_complete(struct napi_struct *napi, 359e4e8452aSWillem de Bruijn struct virtqueue *vq, int processed) 360e4e8452aSWillem de Bruijn { 361e4e8452aSWillem de Bruijn int opaque; 362e4e8452aSWillem de Bruijn 363e4e8452aSWillem de Bruijn opaque = virtqueue_enable_cb_prepare(vq); 364fdaa767aSToshiaki Makita if (napi_complete_done(napi, processed)) { 365fdaa767aSToshiaki Makita if (unlikely(virtqueue_poll(vq, opaque))) 366e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 367fdaa767aSToshiaki Makita } else { 368fdaa767aSToshiaki Makita virtqueue_disable_cb(vq); 369fdaa767aSToshiaki Makita } 370e4e8452aSWillem de Bruijn } 371e4e8452aSWillem de Bruijn 372e9d7417bSJason Wang static void skb_xmit_done(struct virtqueue *vq) 373296f96fcSRusty Russell { 374e9d7417bSJason Wang struct virtnet_info *vi = vq->vdev->priv; 375b92f1e67SWillem de Bruijn struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi; 376296f96fcSRusty Russell 3772cb9c6baSRusty Russell /* Suppress further interrupts. */ 378e9d7417bSJason Wang virtqueue_disable_cb(vq); 37911a3a154SRusty Russell 380b92f1e67SWillem de Bruijn if (napi->weight) 381b92f1e67SWillem de Bruijn virtqueue_napi_schedule(napi, vq); 382b92f1e67SWillem de Bruijn else 383363f1514SRusty Russell /* We were probably waiting for more output buffers. */ 384986a4f4dSJason Wang netif_wake_subqueue(vi->dev, vq2txq(vq)); 385296f96fcSRusty Russell } 386296f96fcSRusty Russell 38728b39bc7SJason Wang #define MRG_CTX_HEADER_SHIFT 22 38828b39bc7SJason Wang static void *mergeable_len_to_ctx(unsigned int truesize, 38928b39bc7SJason Wang unsigned int headroom) 39028b39bc7SJason Wang { 39128b39bc7SJason Wang return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 39228b39bc7SJason Wang } 39328b39bc7SJason Wang 39428b39bc7SJason Wang static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 39528b39bc7SJason Wang { 39628b39bc7SJason Wang return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 39728b39bc7SJason Wang } 39828b39bc7SJason Wang 39928b39bc7SJason Wang static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 40028b39bc7SJason Wang { 40128b39bc7SJason Wang return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 40228b39bc7SJason Wang } 40328b39bc7SJason Wang 4043464645aSMike Waychison /* Called from bottom half context */ 405946fa564SMichael S. Tsirkin static struct sk_buff *page_to_skb(struct virtnet_info *vi, 406946fa564SMichael S. Tsirkin struct receive_queue *rq, 4072613af0eSMichael Dalton struct page *page, unsigned int offset, 408436c9453SJason Wang unsigned int len, unsigned int truesize, 409fb32856bSXuan Zhuo bool hdr_valid, unsigned int metasize, 410c32325b8SJakub Kicinski unsigned int headroom) 4119ab86bbcSShirley Ma { 4129ab86bbcSShirley Ma struct sk_buff *skb; 413012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 4142613af0eSMichael Dalton unsigned int copy, hdr_len, hdr_padded_len; 415af39c8f7SEric Dumazet struct page *page_to_free = NULL; 416fb32856bSXuan Zhuo int tailroom, shinfo_size; 417f80bd740SXuan Zhuo char *p, *hdr_p, *buf; 4189ab86bbcSShirley Ma 4192613af0eSMichael Dalton p = page_address(page) + offset; 420fb32856bSXuan Zhuo hdr_p = p; 4219ab86bbcSShirley Ma 422012873d0SMichael S. Tsirkin hdr_len = vi->hdr_len; 423012873d0SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 424c1ddc42dSAndrew Melnychenko hdr_padded_len = hdr_len; 425012873d0SMichael S. Tsirkin else 4262613af0eSMichael Dalton hdr_padded_len = sizeof(struct padded_vnet_hdr); 4273f2c31d9SMark McLoughlin 428c32325b8SJakub Kicinski /* If headroom is not 0, there is an offset between the beginning of the 429fb32856bSXuan Zhuo * data and the allocated space, otherwise the data and the allocated 430fb32856bSXuan Zhuo * space are aligned. 4318fb7da9eSXuan Zhuo * 4328fb7da9eSXuan Zhuo * Buffers with headroom use PAGE_SIZE as alloc size, see 4338fb7da9eSXuan Zhuo * add_recvbuf_mergeable() + get_mergeable_buf_len() 434fb32856bSXuan Zhuo */ 435c32325b8SJakub Kicinski truesize = headroom ? PAGE_SIZE : truesize; 436fc02e8cbSMichael S. Tsirkin tailroom = truesize - headroom; 437c32325b8SJakub Kicinski buf = p - headroom; 4383f2c31d9SMark McLoughlin 4399ab86bbcSShirley Ma len -= hdr_len; 4402613af0eSMichael Dalton offset += hdr_padded_len; 4412613af0eSMichael Dalton p += hdr_padded_len; 442fc02e8cbSMichael S. Tsirkin tailroom -= hdr_padded_len + len; 4433f2c31d9SMark McLoughlin 444fb32856bSXuan Zhuo shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 445fb32856bSXuan Zhuo 446f80bd740SXuan Zhuo /* copy small packet so we can reuse these pages */ 447f5d7872aSEric Dumazet if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) { 448f80bd740SXuan Zhuo skb = build_skb(buf, truesize); 449fb32856bSXuan Zhuo if (unlikely(!skb)) 450fb32856bSXuan Zhuo return NULL; 451fb32856bSXuan Zhuo 452f80bd740SXuan Zhuo skb_reserve(skb, p - buf); 453fb32856bSXuan Zhuo skb_put(skb, len); 454afd92d82SJason Wang 455afd92d82SJason Wang page = (struct page *)page->private; 456afd92d82SJason Wang if (page) 457afd92d82SJason Wang give_pages(rq, page); 458fb32856bSXuan Zhuo goto ok; 459fb32856bSXuan Zhuo } 460fb32856bSXuan Zhuo 461fb32856bSXuan Zhuo /* copy small packet so we can reuse these pages for small data */ 462fb32856bSXuan Zhuo skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 463fb32856bSXuan Zhuo if (unlikely(!skb)) 464fb32856bSXuan Zhuo return NULL; 465fb32856bSXuan Zhuo 4660f6925b3SEric Dumazet /* Copy all frame if it fits skb->head, otherwise 4670f6925b3SEric Dumazet * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. 4680f6925b3SEric Dumazet */ 4690f6925b3SEric Dumazet if (len <= skb_tailroom(skb)) 4703f2c31d9SMark McLoughlin copy = len; 4710f6925b3SEric Dumazet else 4720f6925b3SEric Dumazet copy = ETH_HLEN + metasize; 47359ae1d12SJohannes Berg skb_put_data(skb, p, copy); 4743f2c31d9SMark McLoughlin 4753f2c31d9SMark McLoughlin len -= copy; 4769ab86bbcSShirley Ma offset += copy; 4773f2c31d9SMark McLoughlin 4782613af0eSMichael Dalton if (vi->mergeable_rx_bufs) { 4792613af0eSMichael Dalton if (len) 4802613af0eSMichael Dalton skb_add_rx_frag(skb, 0, page, offset, len, truesize); 4812613af0eSMichael Dalton else 482af39c8f7SEric Dumazet page_to_free = page; 483fb32856bSXuan Zhuo goto ok; 4842613af0eSMichael Dalton } 4852613af0eSMichael Dalton 486e878d78bSSasha Levin /* 487e878d78bSSasha Levin * Verify that we can indeed put this data into a skb. 488e878d78bSSasha Levin * This is here to handle cases when the device erroneously 489e878d78bSSasha Levin * tries to receive more than is possible. This is usually 490e878d78bSSasha Levin * the case of a broken device. 491e878d78bSSasha Levin */ 492e878d78bSSasha Levin if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { 493be443899SAmerigo Wang net_dbg_ratelimited("%s: too much data\n", skb->dev->name); 494e878d78bSSasha Levin dev_kfree_skb(skb); 495e878d78bSSasha Levin return NULL; 496e878d78bSSasha Levin } 4972613af0eSMichael Dalton BUG_ON(offset >= PAGE_SIZE); 4989ab86bbcSShirley Ma while (len) { 4992613af0eSMichael Dalton unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 5002613af0eSMichael Dalton skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 5012613af0eSMichael Dalton frag_size, truesize); 5022613af0eSMichael Dalton len -= frag_size; 5039ab86bbcSShirley Ma page = (struct page *)page->private; 5049ab86bbcSShirley Ma offset = 0; 5053f2c31d9SMark McLoughlin } 5063f2c31d9SMark McLoughlin 5079ab86bbcSShirley Ma if (page) 508e9d7417bSJason Wang give_pages(rq, page); 5093f2c31d9SMark McLoughlin 510fb32856bSXuan Zhuo ok: 511fb32856bSXuan Zhuo /* hdr_valid means no XDP, so we can copy the vnet header */ 512fb32856bSXuan Zhuo if (hdr_valid) { 513fb32856bSXuan Zhuo hdr = skb_vnet_hdr(skb); 514fb32856bSXuan Zhuo memcpy(hdr, hdr_p, hdr_len); 515fb32856bSXuan Zhuo } 516af39c8f7SEric Dumazet if (page_to_free) 517af39c8f7SEric Dumazet put_page(page_to_free); 518fb32856bSXuan Zhuo 519fb32856bSXuan Zhuo if (metasize) { 520fb32856bSXuan Zhuo __skb_pull(skb, metasize); 521fb32856bSXuan Zhuo skb_metadata_set(skb, metasize); 522fb32856bSXuan Zhuo } 523fb32856bSXuan Zhuo 5249ab86bbcSShirley Ma return skb; 5259ab86bbcSShirley Ma } 5269ab86bbcSShirley Ma 527735fc405SJesper Dangaard Brouer static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 528735fc405SJesper Dangaard Brouer struct send_queue *sq, 52944fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf) 53056434a01SJohn Fastabend { 53156434a01SJohn Fastabend struct virtio_net_hdr_mrg_rxbuf *hdr; 53256434a01SJohn Fastabend int err; 53356434a01SJohn Fastabend 534cac320c8SJesper Dangaard Brouer if (unlikely(xdpf->headroom < vi->hdr_len)) 535cac320c8SJesper Dangaard Brouer return -EOVERFLOW; 536cac320c8SJesper Dangaard Brouer 537cac320c8SJesper Dangaard Brouer /* Make room for virtqueue hdr (also change xdpf->headroom?) */ 538cac320c8SJesper Dangaard Brouer xdpf->data -= vi->hdr_len; 53956434a01SJohn Fastabend /* Zero header and leave csum up to XDP layers */ 540cac320c8SJesper Dangaard Brouer hdr = xdpf->data; 54156434a01SJohn Fastabend memset(hdr, 0, vi->hdr_len); 542cac320c8SJesper Dangaard Brouer xdpf->len += vi->hdr_len; 54356434a01SJohn Fastabend 544cac320c8SJesper Dangaard Brouer sg_init_one(sq->sg, xdpf->data, xdpf->len); 545bb91accfSJason Wang 5465050471dSToshiaki Makita err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf), 5475050471dSToshiaki Makita GFP_ATOMIC); 54811b7d897SJesper Dangaard Brouer if (unlikely(err)) 549cac320c8SJesper Dangaard Brouer return -ENOSPC; /* Caller handle free/refcnt */ 55056434a01SJohn Fastabend 551cac320c8SJesper Dangaard Brouer return 0; 55256434a01SJohn Fastabend } 55356434a01SJohn Fastabend 55497c2c69eSXuan Zhuo /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on 55597c2c69eSXuan Zhuo * the current cpu, so it does not need to be locked. 55697c2c69eSXuan Zhuo * 55797c2c69eSXuan Zhuo * Here we use marco instead of inline functions because we have to deal with 55897c2c69eSXuan Zhuo * three issues at the same time: 1. the choice of sq. 2. judge and execute the 55997c2c69eSXuan Zhuo * lock/unlock of txq 3. make sparse happy. It is difficult for two inline 56097c2c69eSXuan Zhuo * functions to perfectly solve these three problems at the same time. 56197c2c69eSXuan Zhuo */ 56297c2c69eSXuan Zhuo #define virtnet_xdp_get_sq(vi) ({ \ 5633dcc1edcSLi RongQing int cpu = smp_processor_id(); \ 56497c2c69eSXuan Zhuo struct netdev_queue *txq; \ 56597c2c69eSXuan Zhuo typeof(vi) v = (vi); \ 56697c2c69eSXuan Zhuo unsigned int qp; \ 56797c2c69eSXuan Zhuo \ 56897c2c69eSXuan Zhuo if (v->curr_queue_pairs > nr_cpu_ids) { \ 56997c2c69eSXuan Zhuo qp = v->curr_queue_pairs - v->xdp_queue_pairs; \ 5703dcc1edcSLi RongQing qp += cpu; \ 57197c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, qp); \ 57297c2c69eSXuan Zhuo __netif_tx_acquire(txq); \ 57397c2c69eSXuan Zhuo } else { \ 5743dcc1edcSLi RongQing qp = cpu % v->curr_queue_pairs; \ 57597c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, qp); \ 5763dcc1edcSLi RongQing __netif_tx_lock(txq, cpu); \ 57797c2c69eSXuan Zhuo } \ 57897c2c69eSXuan Zhuo v->sq + qp; \ 57997c2c69eSXuan Zhuo }) 5802a43565cSToshiaki Makita 58197c2c69eSXuan Zhuo #define virtnet_xdp_put_sq(vi, q) { \ 58297c2c69eSXuan Zhuo struct netdev_queue *txq; \ 58397c2c69eSXuan Zhuo typeof(vi) v = (vi); \ 58497c2c69eSXuan Zhuo \ 58597c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \ 58697c2c69eSXuan Zhuo if (v->curr_queue_pairs > nr_cpu_ids) \ 58797c2c69eSXuan Zhuo __netif_tx_release(txq); \ 58897c2c69eSXuan Zhuo else \ 58997c2c69eSXuan Zhuo __netif_tx_unlock(txq); \ 5902a43565cSToshiaki Makita } 5912a43565cSToshiaki Makita 592735fc405SJesper Dangaard Brouer static int virtnet_xdp_xmit(struct net_device *dev, 59342b33468SJesper Dangaard Brouer int n, struct xdp_frame **frames, u32 flags) 594186b3c99SJason Wang { 595186b3c99SJason Wang struct virtnet_info *vi = netdev_priv(dev); 5968dcc5b0aSJesper Dangaard Brouer struct receive_queue *rq = vi->rq; 5978dcc5b0aSJesper Dangaard Brouer struct bpf_prog *xdp_prog; 598735fc405SJesper Dangaard Brouer struct send_queue *sq; 599735fc405SJesper Dangaard Brouer unsigned int len; 600546f2897SToshiaki Makita int packets = 0; 601546f2897SToshiaki Makita int bytes = 0; 602fdc13979SLorenzo Bianconi int nxmit = 0; 603461f03dcSToshiaki Makita int kicks = 0; 6045050471dSToshiaki Makita void *ptr; 605fdc13979SLorenzo Bianconi int ret; 606735fc405SJesper Dangaard Brouer int i; 607735fc405SJesper Dangaard Brouer 6088dcc5b0aSJesper Dangaard Brouer /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 6098dcc5b0aSJesper Dangaard Brouer * indicate XDP resources have been successfully allocated. 6108dcc5b0aSJesper Dangaard Brouer */ 6119719c6b9SJohn Fastabend xdp_prog = rcu_access_pointer(rq->xdp_prog); 6121667c08aSToshiaki Makita if (!xdp_prog) 6131667c08aSToshiaki Makita return -ENXIO; 6141667c08aSToshiaki Makita 61597c2c69eSXuan Zhuo sq = virtnet_xdp_get_sq(vi); 6169ab86bbcSShirley Ma 6179ab86bbcSShirley Ma if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 618186b3c99SJason Wang ret = -EINVAL; 619186b3c99SJason Wang goto out; 620186b3c99SJason Wang } 621186b3c99SJason Wang 622735fc405SJesper Dangaard Brouer /* Free up any pending old buffers before queueing new ones. */ 6235050471dSToshiaki Makita while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 624546f2897SToshiaki Makita if (likely(is_xdp_frame(ptr))) { 625546f2897SToshiaki Makita struct xdp_frame *frame = ptr_to_xdp(ptr); 626546f2897SToshiaki Makita 627546f2897SToshiaki Makita bytes += frame->len; 628546f2897SToshiaki Makita xdp_return_frame(frame); 629546f2897SToshiaki Makita } else { 630546f2897SToshiaki Makita struct sk_buff *skb = ptr; 631546f2897SToshiaki Makita 632546f2897SToshiaki Makita bytes += skb->len; 633546f2897SToshiaki Makita napi_consume_skb(skb, false); 634546f2897SToshiaki Makita } 635546f2897SToshiaki Makita packets++; 6365050471dSToshiaki Makita } 637735fc405SJesper Dangaard Brouer 638735fc405SJesper Dangaard Brouer for (i = 0; i < n; i++) { 639735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf = frames[i]; 640735fc405SJesper Dangaard Brouer 641fdc13979SLorenzo Bianconi if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 642fdc13979SLorenzo Bianconi break; 643fdc13979SLorenzo Bianconi nxmit++; 644735fc405SJesper Dangaard Brouer } 645fdc13979SLorenzo Bianconi ret = nxmit; 6465d274cb4SJesper Dangaard Brouer 647461f03dcSToshiaki Makita if (flags & XDP_XMIT_FLUSH) { 648461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 649461f03dcSToshiaki Makita kicks = 1; 650461f03dcSToshiaki Makita } 6515b8f3c8dSToshiaki Makita out: 6525b8f3c8dSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 653546f2897SToshiaki Makita sq->stats.bytes += bytes; 654546f2897SToshiaki Makita sq->stats.packets += packets; 6555b8f3c8dSToshiaki Makita sq->stats.xdp_tx += n; 656fdc13979SLorenzo Bianconi sq->stats.xdp_tx_drops += n - nxmit; 657461f03dcSToshiaki Makita sq->stats.kicks += kicks; 6585b8f3c8dSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 6595d274cb4SJesper Dangaard Brouer 66097c2c69eSXuan Zhuo virtnet_xdp_put_sq(vi, sq); 6615b8f3c8dSToshiaki Makita return ret; 662186b3c99SJason Wang } 663186b3c99SJason Wang 664f6b10209SJason Wang static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 665f6b10209SJason Wang { 66697c2c69eSXuan Zhuo return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0; 667f6b10209SJason Wang } 668f6b10209SJason Wang 6694941d472SJason Wang /* We copy the packet for XDP in the following cases: 6704941d472SJason Wang * 6714941d472SJason Wang * 1) Packet is scattered across multiple rx buffers. 6724941d472SJason Wang * 2) Headroom space is insufficient. 6734941d472SJason Wang * 6744941d472SJason Wang * This is inefficient but it's a temporary condition that 6754941d472SJason Wang * we hit right after XDP is enabled and until queue is refilled 6764941d472SJason Wang * with large buffers with sufficient headroom - so it should affect 6774941d472SJason Wang * at most queue size packets. 6784941d472SJason Wang * Afterwards, the conditions to enable 6794941d472SJason Wang * XDP should preclude the underlying device from sending packets 6804941d472SJason Wang * across multiple buffers (num_buf > 1), and we make sure buffers 6814941d472SJason Wang * have enough headroom. 68272979a6cSJohn Fastabend */ 68372979a6cSJohn Fastabend static struct page *xdp_linearize_page(struct receive_queue *rq, 68456a86f84SJason Wang u16 *num_buf, 68572979a6cSJohn Fastabend struct page *p, 68672979a6cSJohn Fastabend int offset, 6874941d472SJason Wang int page_off, 68872979a6cSJohn Fastabend unsigned int *len) 68972979a6cSJohn Fastabend { 69072979a6cSJohn Fastabend struct page *page = alloc_page(GFP_ATOMIC); 69172979a6cSJohn Fastabend 69272979a6cSJohn Fastabend if (!page) 69372979a6cSJohn Fastabend return NULL; 69472979a6cSJohn Fastabend 69572979a6cSJohn Fastabend memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 69672979a6cSJohn Fastabend page_off += *len; 69772979a6cSJohn Fastabend 69856a86f84SJason Wang while (--*num_buf) { 6993cc81a9aSJason Wang int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 70072979a6cSJohn Fastabend unsigned int buflen; 70172979a6cSJohn Fastabend void *buf; 70272979a6cSJohn Fastabend int off; 70372979a6cSJohn Fastabend 704680557cfSMichael S. Tsirkin buf = virtqueue_get_buf(rq->vq, &buflen); 705680557cfSMichael S. Tsirkin if (unlikely(!buf)) 70672979a6cSJohn Fastabend goto err_buf; 70772979a6cSJohn Fastabend 70872979a6cSJohn Fastabend p = virt_to_head_page(buf); 70972979a6cSJohn Fastabend off = buf - page_address(p); 71072979a6cSJohn Fastabend 71156a86f84SJason Wang /* guard against a misconfigured or uncooperative backend that 71256a86f84SJason Wang * is sending packet larger than the MTU. 71356a86f84SJason Wang */ 7143cc81a9aSJason Wang if ((page_off + buflen + tailroom) > PAGE_SIZE) { 71556a86f84SJason Wang put_page(p); 71656a86f84SJason Wang goto err_buf; 71756a86f84SJason Wang } 71856a86f84SJason Wang 71972979a6cSJohn Fastabend memcpy(page_address(page) + page_off, 72072979a6cSJohn Fastabend page_address(p) + off, buflen); 72172979a6cSJohn Fastabend page_off += buflen; 72256a86f84SJason Wang put_page(p); 72372979a6cSJohn Fastabend } 72472979a6cSJohn Fastabend 7252de2f7f4SJohn Fastabend /* Headroom does not contribute to packet length */ 7262de2f7f4SJohn Fastabend *len = page_off - VIRTIO_XDP_HEADROOM; 72772979a6cSJohn Fastabend return page; 72872979a6cSJohn Fastabend err_buf: 72972979a6cSJohn Fastabend __free_pages(page, 0); 73072979a6cSJohn Fastabend return NULL; 73172979a6cSJohn Fastabend } 73272979a6cSJohn Fastabend 7334941d472SJason Wang static struct sk_buff *receive_small(struct net_device *dev, 7344941d472SJason Wang struct virtnet_info *vi, 7354941d472SJason Wang struct receive_queue *rq, 7364941d472SJason Wang void *buf, void *ctx, 737186b3c99SJason Wang unsigned int len, 7387d9d60fdSToshiaki Makita unsigned int *xdp_xmit, 739d46eeeafSJason Wang struct virtnet_rq_stats *stats) 7404941d472SJason Wang { 7414941d472SJason Wang struct sk_buff *skb; 7424941d472SJason Wang struct bpf_prog *xdp_prog; 7434941d472SJason Wang unsigned int xdp_headroom = (unsigned long)ctx; 7444941d472SJason Wang unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 7454941d472SJason Wang unsigned int headroom = vi->hdr_len + header_offset; 7464941d472SJason Wang unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 7474941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 7484941d472SJason Wang struct page *page = virt_to_head_page(buf); 74911b7d897SJesper Dangaard Brouer unsigned int delta = 0; 7504941d472SJason Wang struct page *xdp_page; 75111b7d897SJesper Dangaard Brouer int err; 752503d539aSYuya Kusakabe unsigned int metasize = 0; 75311b7d897SJesper Dangaard Brouer 7544941d472SJason Wang len -= vi->hdr_len; 755d46eeeafSJason Wang stats->bytes += len; 7564941d472SJason Wang 757ad993a95SXie Yongji if (unlikely(len > GOOD_PACKET_LEN)) { 758ad993a95SXie Yongji pr_debug("%s: rx error: len %u exceeds max size %d\n", 759ad993a95SXie Yongji dev->name, len, GOOD_PACKET_LEN); 760ad993a95SXie Yongji dev->stats.rx_length_errors++; 761053c9e18SWenliang Wang goto err; 762ad993a95SXie Yongji } 7636213f07cSLi RongQing 7646213f07cSLi RongQing if (likely(!vi->xdp_enabled)) { 7656213f07cSLi RongQing xdp_prog = NULL; 7666213f07cSLi RongQing goto skip_xdp; 7676213f07cSLi RongQing } 7686213f07cSLi RongQing 7694941d472SJason Wang rcu_read_lock(); 7704941d472SJason Wang xdp_prog = rcu_dereference(rq->xdp_prog); 7714941d472SJason Wang if (xdp_prog) { 7724941d472SJason Wang struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 77344fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf; 7744941d472SJason Wang struct xdp_buff xdp; 7754941d472SJason Wang void *orig_data; 7764941d472SJason Wang u32 act; 7774941d472SJason Wang 77895dbe9e7SJesper Dangaard Brouer if (unlikely(hdr->hdr.gso_type)) 7794941d472SJason Wang goto err_xdp; 7804941d472SJason Wang 7814941d472SJason Wang if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 7824941d472SJason Wang int offset = buf - page_address(page) + header_offset; 7834941d472SJason Wang unsigned int tlen = len + vi->hdr_len; 7844941d472SJason Wang u16 num_buf = 1; 7854941d472SJason Wang 7864941d472SJason Wang xdp_headroom = virtnet_get_headroom(vi); 7874941d472SJason Wang header_offset = VIRTNET_RX_PAD + xdp_headroom; 7884941d472SJason Wang headroom = vi->hdr_len + header_offset; 7894941d472SJason Wang buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 7904941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 7914941d472SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, page, 7924941d472SJason Wang offset, header_offset, 7934941d472SJason Wang &tlen); 7944941d472SJason Wang if (!xdp_page) 7954941d472SJason Wang goto err_xdp; 7964941d472SJason Wang 7974941d472SJason Wang buf = page_address(xdp_page); 7984941d472SJason Wang put_page(page); 7994941d472SJason Wang page = xdp_page; 8004941d472SJason Wang } 8014941d472SJason Wang 80243b5169dSLorenzo Bianconi xdp_init_buff(&xdp, buflen, &rq->xdp_rxq); 803be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, 804be9df4afSLorenzo Bianconi xdp_headroom, len, true); 8054941d472SJason Wang orig_data = xdp.data; 8064941d472SJason Wang act = bpf_prog_run_xdp(xdp_prog, &xdp); 807d46eeeafSJason Wang stats->xdp_packets++; 8084941d472SJason Wang 8094941d472SJason Wang switch (act) { 8104941d472SJason Wang case XDP_PASS: 8114941d472SJason Wang /* Recalculate length in case bpf program changed it */ 8124941d472SJason Wang delta = orig_data - xdp.data; 8136870de43SNikita V. Shirokov len = xdp.data_end - xdp.data; 814503d539aSYuya Kusakabe metasize = xdp.data - xdp.data_meta; 8154941d472SJason Wang break; 8164941d472SJason Wang case XDP_TX: 817d46eeeafSJason Wang stats->xdp_tx++; 8181b698fa5SLorenzo Bianconi xdpf = xdp_convert_buff_to_frame(&xdp); 81944fa2dbdSJesper Dangaard Brouer if (unlikely(!xdpf)) 82044fa2dbdSJesper Dangaard Brouer goto err_xdp; 821ca9e83b4SJason Wang err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 822fdc13979SLorenzo Bianconi if (unlikely(!err)) { 823fdc13979SLorenzo Bianconi xdp_return_frame_rx_napi(xdpf); 824fdc13979SLorenzo Bianconi } else if (unlikely(err < 0)) { 8254941d472SJason Wang trace_xdp_exception(vi->dev, xdp_prog, act); 82611b7d897SJesper Dangaard Brouer goto err_xdp; 82711b7d897SJesper Dangaard Brouer } 8282471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_TX; 829186b3c99SJason Wang rcu_read_unlock(); 830186b3c99SJason Wang goto xdp_xmit; 831186b3c99SJason Wang case XDP_REDIRECT: 832d46eeeafSJason Wang stats->xdp_redirects++; 833186b3c99SJason Wang err = xdp_do_redirect(dev, &xdp, xdp_prog); 83411b7d897SJesper Dangaard Brouer if (err) 83511b7d897SJesper Dangaard Brouer goto err_xdp; 8362471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_REDIR; 8374941d472SJason Wang rcu_read_unlock(); 8384941d472SJason Wang goto xdp_xmit; 8394941d472SJason Wang default: 840c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act); 841df561f66SGustavo A. R. Silva fallthrough; 8424941d472SJason Wang case XDP_ABORTED: 8434941d472SJason Wang trace_xdp_exception(vi->dev, xdp_prog, act); 84495efabf0SGustavo A. R. Silva goto err_xdp; 8454941d472SJason Wang case XDP_DROP: 8464941d472SJason Wang goto err_xdp; 8474941d472SJason Wang } 8484941d472SJason Wang } 8494941d472SJason Wang rcu_read_unlock(); 8504941d472SJason Wang 8516213f07cSLi RongQing skip_xdp: 8524941d472SJason Wang skb = build_skb(buf, buflen); 853053c9e18SWenliang Wang if (!skb) 8544941d472SJason Wang goto err; 8554941d472SJason Wang skb_reserve(skb, headroom - delta); 8566870de43SNikita V. Shirokov skb_put(skb, len); 857f1d4884dSYuya Kusakabe if (!xdp_prog) { 8584941d472SJason Wang buf += header_offset; 8594941d472SJason Wang memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); 860f1d4884dSYuya Kusakabe } /* keep zeroed vnet hdr since XDP is loaded */ 8614941d472SJason Wang 862503d539aSYuya Kusakabe if (metasize) 863503d539aSYuya Kusakabe skb_metadata_set(skb, metasize); 864503d539aSYuya Kusakabe 8654941d472SJason Wang return skb; 8664941d472SJason Wang 8674941d472SJason Wang err_xdp: 8684941d472SJason Wang rcu_read_unlock(); 869d46eeeafSJason Wang stats->xdp_drops++; 870053c9e18SWenliang Wang err: 871d46eeeafSJason Wang stats->drops++; 8724941d472SJason Wang put_page(page); 8734941d472SJason Wang xdp_xmit: 8744941d472SJason Wang return NULL; 8754941d472SJason Wang } 8764941d472SJason Wang 8774941d472SJason Wang static struct sk_buff *receive_big(struct net_device *dev, 8784941d472SJason Wang struct virtnet_info *vi, 8794941d472SJason Wang struct receive_queue *rq, 8804941d472SJason Wang void *buf, 8817d9d60fdSToshiaki Makita unsigned int len, 882d46eeeafSJason Wang struct virtnet_rq_stats *stats) 8834941d472SJason Wang { 8844941d472SJason Wang struct page *page = buf; 885503d539aSYuya Kusakabe struct sk_buff *skb = 886fb32856bSXuan Zhuo page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0); 8874941d472SJason Wang 888d46eeeafSJason Wang stats->bytes += len - vi->hdr_len; 8894941d472SJason Wang if (unlikely(!skb)) 8904941d472SJason Wang goto err; 8914941d472SJason Wang 8924941d472SJason Wang return skb; 8934941d472SJason Wang 8944941d472SJason Wang err: 895d46eeeafSJason Wang stats->drops++; 8964941d472SJason Wang give_pages(rq, page); 8974941d472SJason Wang return NULL; 8984941d472SJason Wang } 8994941d472SJason Wang 9008fc3b9e9SMichael S. Tsirkin static struct sk_buff *receive_mergeable(struct net_device *dev, 901fdd819b2SMichael S. Tsirkin struct virtnet_info *vi, 9028fc3b9e9SMichael S. Tsirkin struct receive_queue *rq, 903680557cfSMichael S. Tsirkin void *buf, 904680557cfSMichael S. Tsirkin void *ctx, 905186b3c99SJason Wang unsigned int len, 9067d9d60fdSToshiaki Makita unsigned int *xdp_xmit, 907d46eeeafSJason Wang struct virtnet_rq_stats *stats) 9089ab86bbcSShirley Ma { 909012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 910012873d0SMichael S. Tsirkin u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 9118fc3b9e9SMichael S. Tsirkin struct page *page = virt_to_head_page(buf); 9128fc3b9e9SMichael S. Tsirkin int offset = buf - page_address(page); 913f600b690SJohn Fastabend struct sk_buff *head_skb, *curr_skb; 914f600b690SJohn Fastabend struct bpf_prog *xdp_prog; 9159ce6146eSJesper Dangaard Brouer unsigned int truesize = mergeable_ctx_to_truesize(ctx); 9164941d472SJason Wang unsigned int headroom = mergeable_ctx_to_headroom(ctx); 917503d539aSYuya Kusakabe unsigned int metasize = 0; 9189ce6146eSJesper Dangaard Brouer unsigned int frame_sz; 9199ce6146eSJesper Dangaard Brouer int err; 920ab7db917SMichael Dalton 92156434a01SJohn Fastabend head_skb = NULL; 922d46eeeafSJason Wang stats->bytes += len - vi->hdr_len; 92356434a01SJohn Fastabend 924ad993a95SXie Yongji if (unlikely(len > truesize)) { 925ad993a95SXie Yongji pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 926ad993a95SXie Yongji dev->name, len, (unsigned long)ctx); 927ad993a95SXie Yongji dev->stats.rx_length_errors++; 928ad993a95SXie Yongji goto err_skb; 929ad993a95SXie Yongji } 9306213f07cSLi RongQing 9316213f07cSLi RongQing if (likely(!vi->xdp_enabled)) { 9326213f07cSLi RongQing xdp_prog = NULL; 9336213f07cSLi RongQing goto skip_xdp; 9346213f07cSLi RongQing } 9356213f07cSLi RongQing 936f600b690SJohn Fastabend rcu_read_lock(); 937f600b690SJohn Fastabend xdp_prog = rcu_dereference(rq->xdp_prog); 938f600b690SJohn Fastabend if (xdp_prog) { 93944fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf; 94072979a6cSJohn Fastabend struct page *xdp_page; 9410354e4d1SJohn Fastabend struct xdp_buff xdp; 9420354e4d1SJohn Fastabend void *data; 943f600b690SJohn Fastabend u32 act; 944f600b690SJohn Fastabend 9453d62b2a0SJason Wang /* Transient failure which in theory could occur if 9463d62b2a0SJason Wang * in-flight packets from before XDP was enabled reach 9473d62b2a0SJason Wang * the receive path after XDP is loaded. 9483d62b2a0SJason Wang */ 9493d62b2a0SJason Wang if (unlikely(hdr->hdr.gso_type)) 9503d62b2a0SJason Wang goto err_xdp; 9513d62b2a0SJason Wang 9529ce6146eSJesper Dangaard Brouer /* Buffers with headroom use PAGE_SIZE as alloc size, 9539ce6146eSJesper Dangaard Brouer * see add_recvbuf_mergeable() + get_mergeable_buf_len() 9549ce6146eSJesper Dangaard Brouer */ 9559ce6146eSJesper Dangaard Brouer frame_sz = headroom ? PAGE_SIZE : truesize; 9569ce6146eSJesper Dangaard Brouer 9573cc81a9aSJason Wang /* This happens when rx buffer size is underestimated 9583cc81a9aSJason Wang * or headroom is not enough because of the buffer 9593cc81a9aSJason Wang * was refilled before XDP is set. This should only 9603cc81a9aSJason Wang * happen for the first several packets, so we don't 9613cc81a9aSJason Wang * care much about its performance. 9623cc81a9aSJason Wang */ 9634941d472SJason Wang if (unlikely(num_buf > 1 || 9644941d472SJason Wang headroom < virtnet_get_headroom(vi))) { 96572979a6cSJohn Fastabend /* linearize data for XDP */ 96656a86f84SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, 9674941d472SJason Wang page, offset, 9684941d472SJason Wang VIRTIO_XDP_HEADROOM, 9694941d472SJason Wang &len); 9709ce6146eSJesper Dangaard Brouer frame_sz = PAGE_SIZE; 9719ce6146eSJesper Dangaard Brouer 97272979a6cSJohn Fastabend if (!xdp_page) 973f600b690SJohn Fastabend goto err_xdp; 9742de2f7f4SJohn Fastabend offset = VIRTIO_XDP_HEADROOM; 97572979a6cSJohn Fastabend } else { 97672979a6cSJohn Fastabend xdp_page = page; 977f600b690SJohn Fastabend } 978f600b690SJohn Fastabend 9792de2f7f4SJohn Fastabend /* Allow consuming headroom but reserve enough space to push 9802de2f7f4SJohn Fastabend * the descriptor on if we get an XDP_TX return code. 9812de2f7f4SJohn Fastabend */ 9820354e4d1SJohn Fastabend data = page_address(xdp_page) + offset; 98343b5169dSLorenzo Bianconi xdp_init_buff(&xdp, frame_sz - vi->hdr_len, &rq->xdp_rxq); 984be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, data - VIRTIO_XDP_HEADROOM + vi->hdr_len, 985be9df4afSLorenzo Bianconi VIRTIO_XDP_HEADROOM, len - vi->hdr_len, true); 986754b8a21SJesper Dangaard Brouer 9870354e4d1SJohn Fastabend act = bpf_prog_run_xdp(xdp_prog, &xdp); 988d46eeeafSJason Wang stats->xdp_packets++; 9890354e4d1SJohn Fastabend 99056434a01SJohn Fastabend switch (act) { 99156434a01SJohn Fastabend case XDP_PASS: 992503d539aSYuya Kusakabe metasize = xdp.data - xdp.data_meta; 9932de2f7f4SJohn Fastabend 994503d539aSYuya Kusakabe /* recalculate offset to account for any header 995503d539aSYuya Kusakabe * adjustments and minus the metasize to copy the 996503d539aSYuya Kusakabe * metadata in page_to_skb(). Note other cases do not 997503d539aSYuya Kusakabe * build an skb and avoid using offset 9986870de43SNikita V. Shirokov */ 999503d539aSYuya Kusakabe offset = xdp.data - page_address(xdp_page) - 1000503d539aSYuya Kusakabe vi->hdr_len - metasize; 1001503d539aSYuya Kusakabe 1002503d539aSYuya Kusakabe /* recalculate len if xdp.data, xdp.data_end or 1003503d539aSYuya Kusakabe * xdp.data_meta were adjusted 1004503d539aSYuya Kusakabe */ 1005503d539aSYuya Kusakabe len = xdp.data_end - xdp.data + vi->hdr_len + metasize; 10061830f893SJason Wang /* We can only create skb based on xdp_page. */ 10071830f893SJason Wang if (unlikely(xdp_page != page)) { 10081830f893SJason Wang rcu_read_unlock(); 10091830f893SJason Wang put_page(page); 1010503d539aSYuya Kusakabe head_skb = page_to_skb(vi, rq, xdp_page, offset, 1011503d539aSYuya Kusakabe len, PAGE_SIZE, false, 1012c32325b8SJakub Kicinski metasize, 1013c32325b8SJakub Kicinski VIRTIO_XDP_HEADROOM); 10141830f893SJason Wang return head_skb; 10151830f893SJason Wang } 101656434a01SJohn Fastabend break; 101756434a01SJohn Fastabend case XDP_TX: 1018d46eeeafSJason Wang stats->xdp_tx++; 10191b698fa5SLorenzo Bianconi xdpf = xdp_convert_buff_to_frame(&xdp); 102044fa2dbdSJesper Dangaard Brouer if (unlikely(!xdpf)) 102144fa2dbdSJesper Dangaard Brouer goto err_xdp; 1022ca9e83b4SJason Wang err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 1023fdc13979SLorenzo Bianconi if (unlikely(!err)) { 1024fdc13979SLorenzo Bianconi xdp_return_frame_rx_napi(xdpf); 1025fdc13979SLorenzo Bianconi } else if (unlikely(err < 0)) { 10260354e4d1SJohn Fastabend trace_xdp_exception(vi->dev, xdp_prog, act); 102711b7d897SJesper Dangaard Brouer if (unlikely(xdp_page != page)) 102811b7d897SJesper Dangaard Brouer put_page(xdp_page); 102911b7d897SJesper Dangaard Brouer goto err_xdp; 103011b7d897SJesper Dangaard Brouer } 10312471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_TX; 103272979a6cSJohn Fastabend if (unlikely(xdp_page != page)) 10335d458a13SJason Wang put_page(page); 103456434a01SJohn Fastabend rcu_read_unlock(); 103556434a01SJohn Fastabend goto xdp_xmit; 10363cc81a9aSJason Wang case XDP_REDIRECT: 1037d46eeeafSJason Wang stats->xdp_redirects++; 10383cc81a9aSJason Wang err = xdp_do_redirect(dev, &xdp, xdp_prog); 10393cc81a9aSJason Wang if (err) { 10403cc81a9aSJason Wang if (unlikely(xdp_page != page)) 10413cc81a9aSJason Wang put_page(xdp_page); 10423cc81a9aSJason Wang goto err_xdp; 10433cc81a9aSJason Wang } 10442471c75eSJesper Dangaard Brouer *xdp_xmit |= VIRTIO_XDP_REDIR; 10453cc81a9aSJason Wang if (unlikely(xdp_page != page)) 10466890418bSJason Wang put_page(page); 10473cc81a9aSJason Wang rcu_read_unlock(); 10483cc81a9aSJason Wang goto xdp_xmit; 104956434a01SJohn Fastabend default: 1050c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act); 1051df561f66SGustavo A. R. Silva fallthrough; 10520354e4d1SJohn Fastabend case XDP_ABORTED: 10530354e4d1SJohn Fastabend trace_xdp_exception(vi->dev, xdp_prog, act); 1054df561f66SGustavo A. R. Silva fallthrough; 10550354e4d1SJohn Fastabend case XDP_DROP: 105672979a6cSJohn Fastabend if (unlikely(xdp_page != page)) 105772979a6cSJohn Fastabend __free_pages(xdp_page, 0); 1058f600b690SJohn Fastabend goto err_xdp; 1059f600b690SJohn Fastabend } 106056434a01SJohn Fastabend } 1061f600b690SJohn Fastabend rcu_read_unlock(); 1062f600b690SJohn Fastabend 10636213f07cSLi RongQing skip_xdp: 1064503d539aSYuya Kusakabe head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, 1065c32325b8SJakub Kicinski metasize, headroom); 1066f600b690SJohn Fastabend curr_skb = head_skb; 10679ab86bbcSShirley Ma 10688fc3b9e9SMichael S. Tsirkin if (unlikely(!curr_skb)) 10698fc3b9e9SMichael S. Tsirkin goto err_skb; 10709ab86bbcSShirley Ma while (--num_buf) { 10718fc3b9e9SMichael S. Tsirkin int num_skb_frags; 10728fc3b9e9SMichael S. Tsirkin 1073680557cfSMichael S. Tsirkin buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 107403e9f8a0SYunjian Wang if (unlikely(!buf)) { 10758fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers out of %d missing\n", 1076fdd819b2SMichael S. Tsirkin dev->name, num_buf, 1077012873d0SMichael S. Tsirkin virtio16_to_cpu(vi->vdev, 1078012873d0SMichael S. Tsirkin hdr->num_buffers)); 10798fc3b9e9SMichael S. Tsirkin dev->stats.rx_length_errors++; 10808fc3b9e9SMichael S. Tsirkin goto err_buf; 10813f2c31d9SMark McLoughlin } 10828fc3b9e9SMichael S. Tsirkin 1083d46eeeafSJason Wang stats->bytes += len; 10848fc3b9e9SMichael S. Tsirkin page = virt_to_head_page(buf); 108528b39bc7SJason Wang 108628b39bc7SJason Wang truesize = mergeable_ctx_to_truesize(ctx); 108728b39bc7SJason Wang if (unlikely(len > truesize)) { 108856da5fd0SDan Carpenter pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1089680557cfSMichael S. Tsirkin dev->name, len, (unsigned long)ctx); 1090680557cfSMichael S. Tsirkin dev->stats.rx_length_errors++; 1091680557cfSMichael S. Tsirkin goto err_skb; 1092680557cfSMichael S. Tsirkin } 10938fc3b9e9SMichael S. Tsirkin 10948fc3b9e9SMichael S. Tsirkin num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 10952613af0eSMichael Dalton if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 10962613af0eSMichael Dalton struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 10978fc3b9e9SMichael S. Tsirkin 10988fc3b9e9SMichael S. Tsirkin if (unlikely(!nskb)) 10998fc3b9e9SMichael S. Tsirkin goto err_skb; 11002613af0eSMichael Dalton if (curr_skb == head_skb) 11012613af0eSMichael Dalton skb_shinfo(curr_skb)->frag_list = nskb; 11022613af0eSMichael Dalton else 11032613af0eSMichael Dalton curr_skb->next = nskb; 11042613af0eSMichael Dalton curr_skb = nskb; 11052613af0eSMichael Dalton head_skb->truesize += nskb->truesize; 11062613af0eSMichael Dalton num_skb_frags = 0; 11072613af0eSMichael Dalton } 11082613af0eSMichael Dalton if (curr_skb != head_skb) { 11092613af0eSMichael Dalton head_skb->data_len += len; 11102613af0eSMichael Dalton head_skb->len += len; 1111fb51879dSMichael Dalton head_skb->truesize += truesize; 11122613af0eSMichael Dalton } 11138fc3b9e9SMichael S. Tsirkin offset = buf - page_address(page); 1114ba275241SJason Wang if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 1115ba275241SJason Wang put_page(page); 1116ba275241SJason Wang skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 1117fb51879dSMichael Dalton len, truesize); 1118ba275241SJason Wang } else { 11192613af0eSMichael Dalton skb_add_rx_frag(curr_skb, num_skb_frags, page, 1120fb51879dSMichael Dalton offset, len, truesize); 1121ba275241SJason Wang } 11228fc3b9e9SMichael S. Tsirkin } 11238fc3b9e9SMichael S. Tsirkin 11245377d758SJohannes Berg ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 11258fc3b9e9SMichael S. Tsirkin return head_skb; 11268fc3b9e9SMichael S. Tsirkin 1127f600b690SJohn Fastabend err_xdp: 1128f600b690SJohn Fastabend rcu_read_unlock(); 1129d46eeeafSJason Wang stats->xdp_drops++; 11308fc3b9e9SMichael S. Tsirkin err_skb: 11318fc3b9e9SMichael S. Tsirkin put_page(page); 1132850e088dSJason Wang while (num_buf-- > 1) { 1133680557cfSMichael S. Tsirkin buf = virtqueue_get_buf(rq->vq, &len); 1134680557cfSMichael S. Tsirkin if (unlikely(!buf)) { 11358fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers missing\n", 11368fc3b9e9SMichael S. Tsirkin dev->name, num_buf); 11378fc3b9e9SMichael S. Tsirkin dev->stats.rx_length_errors++; 11388fc3b9e9SMichael S. Tsirkin break; 11398fc3b9e9SMichael S. Tsirkin } 1140d46eeeafSJason Wang stats->bytes += len; 1141680557cfSMichael S. Tsirkin page = virt_to_head_page(buf); 11428fc3b9e9SMichael S. Tsirkin put_page(page); 11433f2c31d9SMark McLoughlin } 11448fc3b9e9SMichael S. Tsirkin err_buf: 1145d46eeeafSJason Wang stats->drops++; 11468fc3b9e9SMichael S. Tsirkin dev_kfree_skb(head_skb); 114756434a01SJohn Fastabend xdp_xmit: 11488fc3b9e9SMichael S. Tsirkin return NULL; 11499ab86bbcSShirley Ma } 11509ab86bbcSShirley Ma 11517d9d60fdSToshiaki Makita static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 11522471c75eSJesper Dangaard Brouer void *buf, unsigned int len, void **ctx, 1153a0929a44SToshiaki Makita unsigned int *xdp_xmit, 1154d46eeeafSJason Wang struct virtnet_rq_stats *stats) 11559ab86bbcSShirley Ma { 1156e9d7417bSJason Wang struct net_device *dev = vi->dev; 11579ab86bbcSShirley Ma struct sk_buff *skb; 1158012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 11599ab86bbcSShirley Ma 1160bcff3162SMichael S. Tsirkin if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 11619ab86bbcSShirley Ma pr_debug("%s: short packet %i\n", dev->name, len); 11629ab86bbcSShirley Ma dev->stats.rx_length_errors++; 1163ab7db917SMichael Dalton if (vi->mergeable_rx_bufs) { 1164680557cfSMichael S. Tsirkin put_page(virt_to_head_page(buf)); 1165ab7db917SMichael Dalton } else if (vi->big_packets) { 116698bfd23cSMichael Dalton give_pages(rq, buf); 1167ab7db917SMichael Dalton } else { 1168f6b10209SJason Wang put_page(virt_to_head_page(buf)); 1169ab7db917SMichael Dalton } 11707d9d60fdSToshiaki Makita return; 11719ab86bbcSShirley Ma } 11729ab86bbcSShirley Ma 1173f121159dSMichael S. Tsirkin if (vi->mergeable_rx_bufs) 11747d9d60fdSToshiaki Makita skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, 1175a0929a44SToshiaki Makita stats); 1176f121159dSMichael S. Tsirkin else if (vi->big_packets) 1177a0929a44SToshiaki Makita skb = receive_big(dev, vi, rq, buf, len, stats); 1178f121159dSMichael S. Tsirkin else 1179a0929a44SToshiaki Makita skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); 1180f121159dSMichael S. Tsirkin 11818fc3b9e9SMichael S. Tsirkin if (unlikely(!skb)) 11827d9d60fdSToshiaki Makita return; 11833f2c31d9SMark McLoughlin 11849ab86bbcSShirley Ma hdr = skb_vnet_hdr(skb); 11853fa2a1dfSstephen hemminger 1186e858fae2SMike Rapoport if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) 118710a8d94aSJason Wang skb->ip_summed = CHECKSUM_UNNECESSARY; 1188296f96fcSRusty Russell 1189e858fae2SMike Rapoport if (virtio_net_hdr_to_skb(skb, &hdr->hdr, 1190e858fae2SMike Rapoport virtio_is_little_endian(vi->vdev))) { 1191e858fae2SMike Rapoport net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n", 1192e858fae2SMike Rapoport dev->name, hdr->hdr.gso_type, 1193fdd819b2SMichael S. Tsirkin hdr->hdr.gso_size); 1194296f96fcSRusty Russell goto frame_err; 1195296f96fcSRusty Russell } 1196296f96fcSRusty Russell 1197133bbb18SWillem de Bruijn skb_record_rx_queue(skb, vq2rxq(rq->vq)); 1198d1dc06dcSMike Rapoport skb->protocol = eth_type_trans(skb, dev); 1199d1dc06dcSMike Rapoport pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 1200d1dc06dcSMike Rapoport ntohs(skb->protocol), skb->len, skb->pkt_type); 1201d1dc06dcSMike Rapoport 12020fbd050aSEric Dumazet napi_gro_receive(&rq->napi, skb); 12037d9d60fdSToshiaki Makita return; 1204296f96fcSRusty Russell 1205296f96fcSRusty Russell frame_err: 1206296f96fcSRusty Russell dev->stats.rx_frame_errors++; 1207296f96fcSRusty Russell dev_kfree_skb(skb); 1208296f96fcSRusty Russell } 1209296f96fcSRusty Russell 1210192f68cfSJason Wang /* Unlike mergeable buffers, all buffers are allocated to the 1211192f68cfSJason Wang * same size, except for the headroom. For this reason we do 1212192f68cfSJason Wang * not need to use mergeable_len_to_ctx here - it is enough 1213192f68cfSJason Wang * to store the headroom as the context ignoring the truesize. 1214192f68cfSJason Wang */ 1215946fa564SMichael S. Tsirkin static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 1216946fa564SMichael S. Tsirkin gfp_t gfp) 1217296f96fcSRusty Russell { 1218f6b10209SJason Wang struct page_frag *alloc_frag = &rq->alloc_frag; 1219f6b10209SJason Wang char *buf; 12202de2f7f4SJohn Fastabend unsigned int xdp_headroom = virtnet_get_headroom(vi); 1221192f68cfSJason Wang void *ctx = (void *)(unsigned long)xdp_headroom; 1222f6b10209SJason Wang int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 12239ab86bbcSShirley Ma int err; 12243f2c31d9SMark McLoughlin 1225f6b10209SJason Wang len = SKB_DATA_ALIGN(len) + 1226f6b10209SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1227f6b10209SJason Wang if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp))) 12289ab86bbcSShirley Ma return -ENOMEM; 1229296f96fcSRusty Russell 1230f6b10209SJason Wang buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1231f6b10209SJason Wang get_page(alloc_frag->page); 1232f6b10209SJason Wang alloc_frag->offset += len; 1233f6b10209SJason Wang sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom, 1234f6b10209SJason Wang vi->hdr_len + GOOD_PACKET_LEN); 1235192f68cfSJason Wang err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 12369ab86bbcSShirley Ma if (err < 0) 1237f6b10209SJason Wang put_page(virt_to_head_page(buf)); 12389ab86bbcSShirley Ma return err; 123997402b96SHerbert Xu } 124097402b96SHerbert Xu 1241012873d0SMichael S. Tsirkin static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 1242012873d0SMichael S. Tsirkin gfp_t gfp) 12439ab86bbcSShirley Ma { 12449ab86bbcSShirley Ma struct page *first, *list = NULL; 12459ab86bbcSShirley Ma char *p; 12469ab86bbcSShirley Ma int i, err, offset; 1247296f96fcSRusty Russell 1248a5835440SRusty Russell sg_init_table(rq->sg, MAX_SKB_FRAGS + 2); 1249a5835440SRusty Russell 1250e9d7417bSJason Wang /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */ 12519ab86bbcSShirley Ma for (i = MAX_SKB_FRAGS + 1; i > 1; --i) { 1252e9d7417bSJason Wang first = get_a_page(rq, gfp); 12539ab86bbcSShirley Ma if (!first) { 12549ab86bbcSShirley Ma if (list) 1255e9d7417bSJason Wang give_pages(rq, list); 12569ab86bbcSShirley Ma return -ENOMEM; 1257296f96fcSRusty Russell } 1258e9d7417bSJason Wang sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 12599ab86bbcSShirley Ma 12609ab86bbcSShirley Ma /* chain new page in list head to match sg */ 12619ab86bbcSShirley Ma first->private = (unsigned long)list; 12629ab86bbcSShirley Ma list = first; 12639ab86bbcSShirley Ma } 12649ab86bbcSShirley Ma 1265e9d7417bSJason Wang first = get_a_page(rq, gfp); 12669ab86bbcSShirley Ma if (!first) { 1267e9d7417bSJason Wang give_pages(rq, list); 12689ab86bbcSShirley Ma return -ENOMEM; 12699ab86bbcSShirley Ma } 12709ab86bbcSShirley Ma p = page_address(first); 12719ab86bbcSShirley Ma 1272e9d7417bSJason Wang /* rq->sg[0], rq->sg[1] share the same page */ 1273012873d0SMichael S. Tsirkin /* a separated rq->sg[0] for header - required in case !any_header_sg */ 1274012873d0SMichael S. Tsirkin sg_set_buf(&rq->sg[0], p, vi->hdr_len); 12759ab86bbcSShirley Ma 1276e9d7417bSJason Wang /* rq->sg[1] for data packet, from offset */ 12779ab86bbcSShirley Ma offset = sizeof(struct padded_vnet_hdr); 1278e9d7417bSJason Wang sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 12799ab86bbcSShirley Ma 12809ab86bbcSShirley Ma /* chain first in list head */ 12819ab86bbcSShirley Ma first->private = (unsigned long)list; 12829dc7b9e4SRusty Russell err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2, 1283aa989f5eSMichael S. Tsirkin first, gfp); 12849ab86bbcSShirley Ma if (err < 0) 1285e9d7417bSJason Wang give_pages(rq, first); 12869ab86bbcSShirley Ma 12879ab86bbcSShirley Ma return err; 12889ab86bbcSShirley Ma } 12899ab86bbcSShirley Ma 1290d85b758fSMichael S. Tsirkin static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 12913cc81a9aSJason Wang struct ewma_pkt_len *avg_pkt_len, 12923cc81a9aSJason Wang unsigned int room) 12939ab86bbcSShirley Ma { 1294c1ddc42dSAndrew Melnychenko struct virtnet_info *vi = rq->vq->vdev->priv; 1295c1ddc42dSAndrew Melnychenko const size_t hdr_len = vi->hdr_len; 1296fbf28d78SMichael Dalton unsigned int len; 1297fbf28d78SMichael Dalton 12983cc81a9aSJason Wang if (room) 12993cc81a9aSJason Wang return PAGE_SIZE - room; 13003cc81a9aSJason Wang 13015377d758SJohannes Berg len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 1302f0c3192cSMichael S. Tsirkin rq->min_buf_len, PAGE_SIZE - hdr_len); 13033cc81a9aSJason Wang 1304e377fcc8SMichael S. Tsirkin return ALIGN(len, L1_CACHE_BYTES); 1305fbf28d78SMichael Dalton } 1306fbf28d78SMichael Dalton 13072de2f7f4SJohn Fastabend static int add_recvbuf_mergeable(struct virtnet_info *vi, 13082de2f7f4SJohn Fastabend struct receive_queue *rq, gfp_t gfp) 1309fbf28d78SMichael Dalton { 1310fb51879dSMichael Dalton struct page_frag *alloc_frag = &rq->alloc_frag; 13112de2f7f4SJohn Fastabend unsigned int headroom = virtnet_get_headroom(vi); 13123cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 13133cc81a9aSJason Wang unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1314fb51879dSMichael Dalton char *buf; 1315680557cfSMichael S. Tsirkin void *ctx; 13169ab86bbcSShirley Ma int err; 1317fb51879dSMichael Dalton unsigned int len, hole; 13189ab86bbcSShirley Ma 13193cc81a9aSJason Wang /* Extra tailroom is needed to satisfy XDP's assumption. This 13203cc81a9aSJason Wang * means rx frags coalescing won't work, but consider we've 13213cc81a9aSJason Wang * disabled GSO for XDP, it won't be a big issue. 13223cc81a9aSJason Wang */ 13233cc81a9aSJason Wang len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 13243cc81a9aSJason Wang if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp))) 13259ab86bbcSShirley Ma return -ENOMEM; 1326ab7db917SMichael Dalton 1327fb51879dSMichael Dalton buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 13282de2f7f4SJohn Fastabend buf += headroom; /* advance address leaving hole at front of pkt */ 1329fb51879dSMichael Dalton get_page(alloc_frag->page); 13303cc81a9aSJason Wang alloc_frag->offset += len + room; 1331fb51879dSMichael Dalton hole = alloc_frag->size - alloc_frag->offset; 13323cc81a9aSJason Wang if (hole < len + room) { 1333ab7db917SMichael Dalton /* To avoid internal fragmentation, if there is very likely not 1334ab7db917SMichael Dalton * enough space for another buffer, add the remaining space to 13351daa8790SMichael S. Tsirkin * the current buffer. 1336ab7db917SMichael Dalton */ 1337fb51879dSMichael Dalton len += hole; 1338fb51879dSMichael Dalton alloc_frag->offset += hole; 1339fb51879dSMichael Dalton } 13409ab86bbcSShirley Ma 1341fb51879dSMichael Dalton sg_init_one(rq->sg, buf, len); 134229fda25aSDavid S. Miller ctx = mergeable_len_to_ctx(len, headroom); 1343680557cfSMichael S. Tsirkin err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 13449ab86bbcSShirley Ma if (err < 0) 13452613af0eSMichael Dalton put_page(virt_to_head_page(buf)); 13469ab86bbcSShirley Ma 13479ab86bbcSShirley Ma return err; 1348296f96fcSRusty Russell } 1349296f96fcSRusty Russell 1350b2baed69SRusty Russell /* 1351b2baed69SRusty Russell * Returns false if we couldn't fill entirely (OOM). 1352b2baed69SRusty Russell * 1353b2baed69SRusty Russell * Normally run in the receive path, but can also be run from ndo_open 1354b2baed69SRusty Russell * before we're receiving packets, or from refill_work which is 1355b2baed69SRusty Russell * careful to disable receiving (using napi_disable). 1356b2baed69SRusty Russell */ 1357946fa564SMichael S. Tsirkin static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 1358946fa564SMichael S. Tsirkin gfp_t gfp) 13593f2c31d9SMark McLoughlin { 13603f2c31d9SMark McLoughlin int err; 13611788f495SMichael S. Tsirkin bool oom; 13623f2c31d9SMark McLoughlin 13630aea51c3SAmit Shah do { 13649ab86bbcSShirley Ma if (vi->mergeable_rx_bufs) 13652de2f7f4SJohn Fastabend err = add_recvbuf_mergeable(vi, rq, gfp); 13669ab86bbcSShirley Ma else if (vi->big_packets) 1367012873d0SMichael S. Tsirkin err = add_recvbuf_big(vi, rq, gfp); 13689ab86bbcSShirley Ma else 1369946fa564SMichael S. Tsirkin err = add_recvbuf_small(vi, rq, gfp); 13703f2c31d9SMark McLoughlin 13711788f495SMichael S. Tsirkin oom = err == -ENOMEM; 13729ed4cb07SRusty Russell if (err) 13733f2c31d9SMark McLoughlin break; 1374b7dfde95SLinus Torvalds } while (rq->vq->num_free); 1375461f03dcSToshiaki Makita if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { 137601c32598SMichael S. Tsirkin unsigned long flags; 137701c32598SMichael S. Tsirkin 137801c32598SMichael S. Tsirkin flags = u64_stats_update_begin_irqsave(&rq->stats.syncp); 1379d46eeeafSJason Wang rq->stats.kicks++; 138001c32598SMichael S. Tsirkin u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); 1381461f03dcSToshiaki Makita } 1382461f03dcSToshiaki Makita 13833161e453SRusty Russell return !oom; 13843f2c31d9SMark McLoughlin } 13853f2c31d9SMark McLoughlin 138618445c4dSRusty Russell static void skb_recv_done(struct virtqueue *rvq) 1387296f96fcSRusty Russell { 1388296f96fcSRusty Russell struct virtnet_info *vi = rvq->vdev->priv; 1389986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 1390e9d7417bSJason Wang 1391e4e8452aSWillem de Bruijn virtqueue_napi_schedule(&rq->napi, rvq); 1392296f96fcSRusty Russell } 1393296f96fcSRusty Russell 1394e4e8452aSWillem de Bruijn static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi) 13953e9d08ecSBruce Rogers { 1396e4e8452aSWillem de Bruijn napi_enable(napi); 13973e9d08ecSBruce Rogers 13983e9d08ecSBruce Rogers /* If all buffers were filled by other side before we napi_enabled, we 1399e4e8452aSWillem de Bruijn * won't get another interrupt, so process any outstanding packets now. 1400e4e8452aSWillem de Bruijn * Call local_bh_enable after to trigger softIRQ processing. 1401e4e8452aSWillem de Bruijn */ 1402ec13ee80SMichael S. Tsirkin local_bh_disable(); 1403e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 1404ec13ee80SMichael S. Tsirkin local_bh_enable(); 14053e9d08ecSBruce Rogers } 14063e9d08ecSBruce Rogers 1407b92f1e67SWillem de Bruijn static void virtnet_napi_tx_enable(struct virtnet_info *vi, 1408b92f1e67SWillem de Bruijn struct virtqueue *vq, 1409b92f1e67SWillem de Bruijn struct napi_struct *napi) 1410b92f1e67SWillem de Bruijn { 1411b92f1e67SWillem de Bruijn if (!napi->weight) 1412b92f1e67SWillem de Bruijn return; 1413b92f1e67SWillem de Bruijn 1414b92f1e67SWillem de Bruijn /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 1415b92f1e67SWillem de Bruijn * enable the feature if this is likely affine with the transmit path. 1416b92f1e67SWillem de Bruijn */ 1417b92f1e67SWillem de Bruijn if (!vi->affinity_hint_set) { 1418b92f1e67SWillem de Bruijn napi->weight = 0; 1419b92f1e67SWillem de Bruijn return; 1420b92f1e67SWillem de Bruijn } 1421b92f1e67SWillem de Bruijn 1422b92f1e67SWillem de Bruijn return virtnet_napi_enable(vq, napi); 1423b92f1e67SWillem de Bruijn } 1424b92f1e67SWillem de Bruijn 142578a57b48SWillem de Bruijn static void virtnet_napi_tx_disable(struct napi_struct *napi) 142678a57b48SWillem de Bruijn { 142778a57b48SWillem de Bruijn if (napi->weight) 142878a57b48SWillem de Bruijn napi_disable(napi); 142978a57b48SWillem de Bruijn } 143078a57b48SWillem de Bruijn 14313161e453SRusty Russell static void refill_work(struct work_struct *work) 14323161e453SRusty Russell { 1433e9d7417bSJason Wang struct virtnet_info *vi = 1434e9d7417bSJason Wang container_of(work, struct virtnet_info, refill.work); 14353161e453SRusty Russell bool still_empty; 1436986a4f4dSJason Wang int i; 14373161e453SRusty Russell 143855257d72SSasha Levin for (i = 0; i < vi->curr_queue_pairs; i++) { 1439986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[i]; 1440986a4f4dSJason Wang 1441986a4f4dSJason Wang napi_disable(&rq->napi); 1442946fa564SMichael S. Tsirkin still_empty = !try_fill_recv(vi, rq, GFP_KERNEL); 1443e4e8452aSWillem de Bruijn virtnet_napi_enable(rq->vq, &rq->napi); 14443161e453SRusty Russell 14453161e453SRusty Russell /* In theory, this can happen: if we don't get any buffers in 1446986a4f4dSJason Wang * we will *never* try to fill again. 1447986a4f4dSJason Wang */ 14483161e453SRusty Russell if (still_empty) 14493b07e9caSTejun Heo schedule_delayed_work(&vi->refill, HZ/2); 14503161e453SRusty Russell } 1451986a4f4dSJason Wang } 14523161e453SRusty Russell 14532471c75eSJesper Dangaard Brouer static int virtnet_receive(struct receive_queue *rq, int budget, 14542471c75eSJesper Dangaard Brouer unsigned int *xdp_xmit) 1455296f96fcSRusty Russell { 1456e9d7417bSJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 1457d46eeeafSJason Wang struct virtnet_rq_stats stats = {}; 1458a0929a44SToshiaki Makita unsigned int len; 14599ab86bbcSShirley Ma void *buf; 1460a0929a44SToshiaki Makita int i; 1461296f96fcSRusty Russell 1462192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 1463680557cfSMichael S. Tsirkin void *ctx; 1464680557cfSMichael S. Tsirkin 1465d46eeeafSJason Wang while (stats.packets < budget && 1466680557cfSMichael S. Tsirkin (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) { 1467a0929a44SToshiaki Makita receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats); 1468d46eeeafSJason Wang stats.packets++; 1469680557cfSMichael S. Tsirkin } 1470680557cfSMichael S. Tsirkin } else { 1471d46eeeafSJason Wang while (stats.packets < budget && 1472e9d7417bSJason Wang (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 1473a0929a44SToshiaki Makita receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats); 1474d46eeeafSJason Wang stats.packets++; 1475296f96fcSRusty Russell } 1476680557cfSMichael S. Tsirkin } 1477296f96fcSRusty Russell 1478718be6baS? jiang if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 1479946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, rq, GFP_ATOMIC)) 14803b07e9caSTejun Heo schedule_delayed_work(&vi->refill, 0); 14813161e453SRusty Russell } 1482296f96fcSRusty Russell 1483d7dfc5cfSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 1484a0929a44SToshiaki Makita for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) { 1485a0929a44SToshiaki Makita size_t offset = virtnet_rq_stats_desc[i].offset; 1486a0929a44SToshiaki Makita u64 *item; 1487a0929a44SToshiaki Makita 1488d46eeeafSJason Wang item = (u64 *)((u8 *)&rq->stats + offset); 1489d46eeeafSJason Wang *item += *(u64 *)((u8 *)&stats + offset); 1490a0929a44SToshiaki Makita } 1491d7dfc5cfSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 149261845d20SJason Wang 1493d46eeeafSJason Wang return stats.packets; 14942ffa7598SJason Wang } 14952ffa7598SJason Wang 1496df133f3fSMichael S. Tsirkin static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi) 1497ea7735d9SWillem de Bruijn { 1498ea7735d9SWillem de Bruijn unsigned int len; 1499ea7735d9SWillem de Bruijn unsigned int packets = 0; 1500ea7735d9SWillem de Bruijn unsigned int bytes = 0; 15015050471dSToshiaki Makita void *ptr; 1502ea7735d9SWillem de Bruijn 15035050471dSToshiaki Makita while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 15045050471dSToshiaki Makita if (likely(!is_xdp_frame(ptr))) { 15055050471dSToshiaki Makita struct sk_buff *skb = ptr; 15065050471dSToshiaki Makita 1507ea7735d9SWillem de Bruijn pr_debug("Sent skb %p\n", skb); 1508ea7735d9SWillem de Bruijn 1509ea7735d9SWillem de Bruijn bytes += skb->len; 1510df133f3fSMichael S. Tsirkin napi_consume_skb(skb, in_napi); 15115050471dSToshiaki Makita } else { 15125050471dSToshiaki Makita struct xdp_frame *frame = ptr_to_xdp(ptr); 15135050471dSToshiaki Makita 15145050471dSToshiaki Makita bytes += frame->len; 15155050471dSToshiaki Makita xdp_return_frame(frame); 15165050471dSToshiaki Makita } 15175050471dSToshiaki Makita packets++; 1518ea7735d9SWillem de Bruijn } 1519ea7735d9SWillem de Bruijn 1520ea7735d9SWillem de Bruijn /* Avoid overhead when no packets have been processed 1521ea7735d9SWillem de Bruijn * happens when called speculatively from start_xmit. 1522ea7735d9SWillem de Bruijn */ 1523ea7735d9SWillem de Bruijn if (!packets) 1524ea7735d9SWillem de Bruijn return; 1525ea7735d9SWillem de Bruijn 1526d7dfc5cfSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1527d7dfc5cfSToshiaki Makita sq->stats.bytes += bytes; 1528d7dfc5cfSToshiaki Makita sq->stats.packets += packets; 1529d7dfc5cfSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1530ea7735d9SWillem de Bruijn } 1531ea7735d9SWillem de Bruijn 1532534da5e8SToshiaki Makita static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 1533534da5e8SToshiaki Makita { 1534534da5e8SToshiaki Makita if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 1535534da5e8SToshiaki Makita return false; 1536534da5e8SToshiaki Makita else if (q < vi->curr_queue_pairs) 1537534da5e8SToshiaki Makita return true; 1538534da5e8SToshiaki Makita else 1539534da5e8SToshiaki Makita return false; 1540534da5e8SToshiaki Makita } 1541534da5e8SToshiaki Makita 15427b0411efSWillem de Bruijn static void virtnet_poll_cleantx(struct receive_queue *rq) 15437b0411efSWillem de Bruijn { 15447b0411efSWillem de Bruijn struct virtnet_info *vi = rq->vq->vdev->priv; 15457b0411efSWillem de Bruijn unsigned int index = vq2rxq(rq->vq); 15467b0411efSWillem de Bruijn struct send_queue *sq = &vi->sq[index]; 15477b0411efSWillem de Bruijn struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 15487b0411efSWillem de Bruijn 1549534da5e8SToshiaki Makita if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) 15507b0411efSWillem de Bruijn return; 15517b0411efSWillem de Bruijn 15527b0411efSWillem de Bruijn if (__netif_tx_trylock(txq)) { 1553a7766ef1SMichael S. Tsirkin do { 1554a7766ef1SMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 1555df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, true); 1556a7766ef1SMichael S. Tsirkin } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 15577b0411efSWillem de Bruijn 15587b0411efSWillem de Bruijn if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 15597b0411efSWillem de Bruijn netif_tx_wake_queue(txq); 156022bc63c5SMichael S. Tsirkin 156122bc63c5SMichael S. Tsirkin __netif_tx_unlock(txq); 156222bc63c5SMichael S. Tsirkin } 15637b0411efSWillem de Bruijn } 15647b0411efSWillem de Bruijn 15652ffa7598SJason Wang static int virtnet_poll(struct napi_struct *napi, int budget) 15662ffa7598SJason Wang { 15672ffa7598SJason Wang struct receive_queue *rq = 15682ffa7598SJason Wang container_of(napi, struct receive_queue, napi); 15699267c430SJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 15709267c430SJason Wang struct send_queue *sq; 15712a43565cSToshiaki Makita unsigned int received; 15722471c75eSJesper Dangaard Brouer unsigned int xdp_xmit = 0; 15732ffa7598SJason Wang 15747b0411efSWillem de Bruijn virtnet_poll_cleantx(rq); 15757b0411efSWillem de Bruijn 1576186b3c99SJason Wang received = virtnet_receive(rq, budget, &xdp_xmit); 15772ffa7598SJason Wang 15788329d98eSRusty Russell /* Out of packets? */ 1579e4e8452aSWillem de Bruijn if (received < budget) 1580e4e8452aSWillem de Bruijn virtqueue_napi_complete(napi, rq->vq, received); 1581296f96fcSRusty Russell 15822471c75eSJesper Dangaard Brouer if (xdp_xmit & VIRTIO_XDP_REDIR) 15831d233886SToke Høiland-Jørgensen xdp_do_flush(); 15842471c75eSJesper Dangaard Brouer 15852471c75eSJesper Dangaard Brouer if (xdp_xmit & VIRTIO_XDP_TX) { 158697c2c69eSXuan Zhuo sq = virtnet_xdp_get_sq(vi); 1587461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 1588461f03dcSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1589461f03dcSToshiaki Makita sq->stats.kicks++; 1590461f03dcSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1591461f03dcSToshiaki Makita } 159297c2c69eSXuan Zhuo virtnet_xdp_put_sq(vi, sq); 15939267c430SJason Wang } 1594186b3c99SJason Wang 1595296f96fcSRusty Russell return received; 1596296f96fcSRusty Russell } 1597296f96fcSRusty Russell 1598986a4f4dSJason Wang static int virtnet_open(struct net_device *dev) 1599986a4f4dSJason Wang { 1600986a4f4dSJason Wang struct virtnet_info *vi = netdev_priv(dev); 1601754b8a21SJesper Dangaard Brouer int i, err; 1602986a4f4dSJason Wang 1603e4166625SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 1604e4166625SJason Wang if (i < vi->curr_queue_pairs) 1605986a4f4dSJason Wang /* Make sure we have some buffers: if oom use wq. */ 1606946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 1607986a4f4dSJason Wang schedule_delayed_work(&vi->refill, 0); 1608754b8a21SJesper Dangaard Brouer 1609b02e5a0eSBjörn Töpel err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id); 1610754b8a21SJesper Dangaard Brouer if (err < 0) 1611754b8a21SJesper Dangaard Brouer return err; 1612754b8a21SJesper Dangaard Brouer 16138d5d8852SJesper Dangaard Brouer err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq, 16148d5d8852SJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED, NULL); 16158d5d8852SJesper Dangaard Brouer if (err < 0) { 16168d5d8852SJesper Dangaard Brouer xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 16178d5d8852SJesper Dangaard Brouer return err; 16188d5d8852SJesper Dangaard Brouer } 16198d5d8852SJesper Dangaard Brouer 1620e4e8452aSWillem de Bruijn virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 1621b92f1e67SWillem de Bruijn virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi); 1622986a4f4dSJason Wang } 1623986a4f4dSJason Wang 1624986a4f4dSJason Wang return 0; 1625986a4f4dSJason Wang } 1626986a4f4dSJason Wang 1627b92f1e67SWillem de Bruijn static int virtnet_poll_tx(struct napi_struct *napi, int budget) 1628b92f1e67SWillem de Bruijn { 1629b92f1e67SWillem de Bruijn struct send_queue *sq = container_of(napi, struct send_queue, napi); 1630b92f1e67SWillem de Bruijn struct virtnet_info *vi = sq->vq->vdev->priv; 1631534da5e8SToshiaki Makita unsigned int index = vq2txq(sq->vq); 1632534da5e8SToshiaki Makita struct netdev_queue *txq; 16335a2f966dSMichael S. Tsirkin int opaque; 16345a2f966dSMichael S. Tsirkin bool done; 1635b92f1e67SWillem de Bruijn 1636534da5e8SToshiaki Makita if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { 1637534da5e8SToshiaki Makita /* We don't need to enable cb for XDP */ 1638534da5e8SToshiaki Makita napi_complete_done(napi, 0); 1639534da5e8SToshiaki Makita return 0; 1640534da5e8SToshiaki Makita } 1641534da5e8SToshiaki Makita 1642534da5e8SToshiaki Makita txq = netdev_get_tx_queue(vi->dev, index); 1643b92f1e67SWillem de Bruijn __netif_tx_lock(txq, raw_smp_processor_id()); 16445a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 1645df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, true); 16465a2f966dSMichael S. Tsirkin 164722bc63c5SMichael S. Tsirkin if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 164822bc63c5SMichael S. Tsirkin netif_tx_wake_queue(txq); 164922bc63c5SMichael S. Tsirkin 16505a2f966dSMichael S. Tsirkin opaque = virtqueue_enable_cb_prepare(sq->vq); 16515a2f966dSMichael S. Tsirkin 16525a2f966dSMichael S. Tsirkin done = napi_complete_done(napi, 0); 16535a2f966dSMichael S. Tsirkin 16545a2f966dSMichael S. Tsirkin if (!done) 16555a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 16565a2f966dSMichael S. Tsirkin 1657b92f1e67SWillem de Bruijn __netif_tx_unlock(txq); 1658b92f1e67SWillem de Bruijn 16595a2f966dSMichael S. Tsirkin if (done) { 16605a2f966dSMichael S. Tsirkin if (unlikely(virtqueue_poll(sq->vq, opaque))) { 16615a2f966dSMichael S. Tsirkin if (napi_schedule_prep(napi)) { 16625a2f966dSMichael S. Tsirkin __netif_tx_lock(txq, raw_smp_processor_id()); 16635a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 16645a2f966dSMichael S. Tsirkin __netif_tx_unlock(txq); 16655a2f966dSMichael S. Tsirkin __napi_schedule(napi); 16665a2f966dSMichael S. Tsirkin } 16675a2f966dSMichael S. Tsirkin } 16685a2f966dSMichael S. Tsirkin } 1669b92f1e67SWillem de Bruijn 1670b92f1e67SWillem de Bruijn return 0; 1671b92f1e67SWillem de Bruijn } 1672b92f1e67SWillem de Bruijn 1673e9d7417bSJason Wang static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) 1674296f96fcSRusty Russell { 1675012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 1676296f96fcSRusty Russell const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 1677e9d7417bSJason Wang struct virtnet_info *vi = sq->vq->vdev->priv; 1678e2fcad58SJason A. Donenfeld int num_sg; 1679012873d0SMichael S. Tsirkin unsigned hdr_len = vi->hdr_len; 1680e7428e95SMichael S. Tsirkin bool can_push; 1681296f96fcSRusty Russell 1682e174961cSJohannes Berg pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 1683e7428e95SMichael S. Tsirkin 1684e7428e95SMichael S. Tsirkin can_push = vi->any_header_sg && 1685e7428e95SMichael S. Tsirkin !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 1686e7428e95SMichael S. Tsirkin !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 1687e7428e95SMichael S. Tsirkin /* Even if we can, don't push here yet as this would skew 1688e7428e95SMichael S. Tsirkin * csum_start offset below. */ 1689e7428e95SMichael S. Tsirkin if (can_push) 1690012873d0SMichael S. Tsirkin hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len); 1691e7428e95SMichael S. Tsirkin else 1692e7428e95SMichael S. Tsirkin hdr = skb_vnet_hdr(skb); 1693296f96fcSRusty Russell 1694e858fae2SMike Rapoport if (virtio_net_hdr_from_skb(skb, &hdr->hdr, 1695fd3a8862SWillem de Bruijn virtio_is_little_endian(vi->vdev), false, 1696fd3a8862SWillem de Bruijn 0)) 169785eb1389SXianting Tian return -EPROTO; 1698296f96fcSRusty Russell 1699e7428e95SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 1700012873d0SMichael S. Tsirkin hdr->num_buffers = 0; 17013f2c31d9SMark McLoughlin 1702547c890cSJason Wang sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 1703e7428e95SMichael S. Tsirkin if (can_push) { 1704e7428e95SMichael S. Tsirkin __skb_push(skb, hdr_len); 1705e7428e95SMichael S. Tsirkin num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 1706e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 1707e2fcad58SJason A. Donenfeld return num_sg; 1708e7428e95SMichael S. Tsirkin /* Pull header back to avoid skew in tx bytes calculations. */ 1709e7428e95SMichael S. Tsirkin __skb_pull(skb, hdr_len); 1710e7428e95SMichael S. Tsirkin } else { 1711e7428e95SMichael S. Tsirkin sg_set_buf(sq->sg, hdr, hdr_len); 1712e2fcad58SJason A. Donenfeld num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 1713e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 1714e2fcad58SJason A. Donenfeld return num_sg; 1715e2fcad58SJason A. Donenfeld num_sg++; 1716e7428e95SMichael S. Tsirkin } 17179dc7b9e4SRusty Russell return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC); 171811a3a154SRusty Russell } 171911a3a154SRusty Russell 1720424efe9cSStephen Hemminger static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 172199ffc696SRusty Russell { 172299ffc696SRusty Russell struct virtnet_info *vi = netdev_priv(dev); 1723986a4f4dSJason Wang int qnum = skb_get_queue_mapping(skb); 1724986a4f4dSJason Wang struct send_queue *sq = &vi->sq[qnum]; 17259ed4cb07SRusty Russell int err; 17264b7fd2e6SMichael S. Tsirkin struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 17276b16f9eeSFlorian Westphal bool kick = !netdev_xmit_more(); 1728b92f1e67SWillem de Bruijn bool use_napi = sq->napi.weight; 17292cb9c6baSRusty Russell 17302cb9c6baSRusty Russell /* Free up any pending old buffers before queueing new ones. */ 1731a7766ef1SMichael S. Tsirkin do { 1732a7766ef1SMichael S. Tsirkin if (use_napi) 1733a7766ef1SMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 1734a7766ef1SMichael S. Tsirkin 1735df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, false); 173699ffc696SRusty Russell 1737a7766ef1SMichael S. Tsirkin } while (use_napi && kick && 1738a7766ef1SMichael S. Tsirkin unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 1739bdb12e0dSWillem de Bruijn 1740074c3582SJacob Keller /* timestamp packet in software */ 1741074c3582SJacob Keller skb_tx_timestamp(skb); 1742074c3582SJacob Keller 174303f191baSMichael S. Tsirkin /* Try to transmit */ 1744b7dfde95SLinus Torvalds err = xmit_skb(sq, skb); 174599ffc696SRusty Russell 17469ed4cb07SRusty Russell /* This should not happen! */ 1747681daee2SJason Wang if (unlikely(err)) { 174858eba97dSRusty Russell dev->stats.tx_fifo_errors++; 17492e57b79cSRick Jones if (net_ratelimit()) 175058eba97dSRusty Russell dev_warn(&dev->dev, 17517934b481SYuval Shaia "Unexpected TXQ (%d) queue failure: %d\n", 17527934b481SYuval Shaia qnum, err); 175358eba97dSRusty Russell dev->stats.tx_dropped++; 175485e94525SEric W. Biederman dev_kfree_skb_any(skb); 175558eba97dSRusty Russell return NETDEV_TX_OK; 1756296f96fcSRusty Russell } 175703f191baSMichael S. Tsirkin 175848925e37SRusty Russell /* Don't wait up for transmitted skbs to be freed. */ 1759b92f1e67SWillem de Bruijn if (!use_napi) { 176048925e37SRusty Russell skb_orphan(skb); 1761895b5c9fSFlorian Westphal nf_reset_ct(skb); 1762b92f1e67SWillem de Bruijn } 176348925e37SRusty Russell 176460302ff6SMichael S. Tsirkin /* If running out of space, stop queue to avoid getting packets that we 176560302ff6SMichael S. Tsirkin * are then unable to transmit. 176660302ff6SMichael S. Tsirkin * An alternative would be to force queuing layer to requeue the skb by 176760302ff6SMichael S. Tsirkin * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 176860302ff6SMichael S. Tsirkin * returned in a normal path of operation: it means that driver is not 176960302ff6SMichael S. Tsirkin * maintaining the TX queue stop/start state properly, and causes 177060302ff6SMichael S. Tsirkin * the stack to do a non-trivial amount of useless work. 177160302ff6SMichael S. Tsirkin * Since most packets only take 1 or 2 ring slots, stopping the queue 177260302ff6SMichael S. Tsirkin * early means 16 slots are typically wasted. 1773d631b94eSstephen hemminger */ 1774b7dfde95SLinus Torvalds if (sq->vq->num_free < 2+MAX_SKB_FRAGS) { 1775986a4f4dSJason Wang netif_stop_subqueue(dev, qnum); 1776b92f1e67SWillem de Bruijn if (!use_napi && 1777b92f1e67SWillem de Bruijn unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 177848925e37SRusty Russell /* More just got used, free them then recheck. */ 1779df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, false); 1780b7dfde95SLinus Torvalds if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { 1781986a4f4dSJason Wang netif_start_subqueue(dev, qnum); 1782e9d7417bSJason Wang virtqueue_disable_cb(sq->vq); 178348925e37SRusty Russell } 178448925e37SRusty Russell } 178548925e37SRusty Russell } 178648925e37SRusty Russell 1787461f03dcSToshiaki Makita if (kick || netif_xmit_stopped(txq)) { 1788461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 1789461f03dcSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1790461f03dcSToshiaki Makita sq->stats.kicks++; 1791461f03dcSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1792461f03dcSToshiaki Makita } 1793461f03dcSToshiaki Makita } 17940b725a2cSDavid S. Miller 17950b725a2cSDavid S. Miller return NETDEV_TX_OK; 1796c223a078SDavid S. Miller } 1797c223a078SDavid S. Miller 179840cbfc37SAmos Kong /* 179940cbfc37SAmos Kong * Send command via the control virtqueue and check status. Commands 180040cbfc37SAmos Kong * supported by the hypervisor, as indicated by feature bits, should 1801788a8b6dSstephen hemminger * never fail unless improperly formatted. 180240cbfc37SAmos Kong */ 180340cbfc37SAmos Kong static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 1804d24bae32Sstephen hemminger struct scatterlist *out) 180540cbfc37SAmos Kong { 1806f7bc9594SRusty Russell struct scatterlist *sgs[4], hdr, stat; 1807d24bae32Sstephen hemminger unsigned out_num = 0, tmp; 1808222722bcSYunjian Wang int ret; 180940cbfc37SAmos Kong 181040cbfc37SAmos Kong /* Caller should know better */ 1811f7bc9594SRusty Russell BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 181240cbfc37SAmos Kong 181312e57169SMichael S. Tsirkin vi->ctrl->status = ~0; 181412e57169SMichael S. Tsirkin vi->ctrl->hdr.class = class; 181512e57169SMichael S. Tsirkin vi->ctrl->hdr.cmd = cmd; 1816f7bc9594SRusty Russell /* Add header */ 181712e57169SMichael S. Tsirkin sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 1818f7bc9594SRusty Russell sgs[out_num++] = &hdr; 181940cbfc37SAmos Kong 1820f7bc9594SRusty Russell if (out) 1821f7bc9594SRusty Russell sgs[out_num++] = out; 182240cbfc37SAmos Kong 1823f7bc9594SRusty Russell /* Add return status. */ 182412e57169SMichael S. Tsirkin sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 1825d24bae32Sstephen hemminger sgs[out_num] = &stat; 182640cbfc37SAmos Kong 1827d24bae32Sstephen hemminger BUG_ON(out_num + 1 > ARRAY_SIZE(sgs)); 1828222722bcSYunjian Wang ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC); 1829222722bcSYunjian Wang if (ret < 0) { 1830222722bcSYunjian Wang dev_warn(&vi->vdev->dev, 1831222722bcSYunjian Wang "Failed to add sgs for command vq: %d\n.", ret); 1832222722bcSYunjian Wang return false; 1833222722bcSYunjian Wang } 183440cbfc37SAmos Kong 183567975901SHeinz Graalfs if (unlikely(!virtqueue_kick(vi->cvq))) 183612e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 183740cbfc37SAmos Kong 183840cbfc37SAmos Kong /* Spin for a response, the kick causes an ioport write, trapping 183940cbfc37SAmos Kong * into the hypervisor, so the request should be handled immediately. 184040cbfc37SAmos Kong */ 1841047b9b94SHeinz Graalfs while (!virtqueue_get_buf(vi->cvq, &tmp) && 1842047b9b94SHeinz Graalfs !virtqueue_is_broken(vi->cvq)) 184340cbfc37SAmos Kong cpu_relax(); 184440cbfc37SAmos Kong 184512e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 184640cbfc37SAmos Kong } 184740cbfc37SAmos Kong 18489c46f6d4SAlex Williamson static int virtnet_set_mac_address(struct net_device *dev, void *p) 18499c46f6d4SAlex Williamson { 18509c46f6d4SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 18519c46f6d4SAlex Williamson struct virtio_device *vdev = vi->vdev; 1852f2f2c8b4SJiri Pirko int ret; 1853e37e2ff3SAndy Lutomirski struct sockaddr *addr; 18547e58d5aeSAmos Kong struct scatterlist sg; 18559c46f6d4SAlex Williamson 1856ba5e4426SSridhar Samudrala if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 1857ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 1858ba5e4426SSridhar Samudrala 1859801822d1SShyam Saini addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 1860e37e2ff3SAndy Lutomirski if (!addr) 1861e37e2ff3SAndy Lutomirski return -ENOMEM; 1862e37e2ff3SAndy Lutomirski 1863e37e2ff3SAndy Lutomirski ret = eth_prepare_mac_addr_change(dev, addr); 1864f2f2c8b4SJiri Pirko if (ret) 1865e37e2ff3SAndy Lutomirski goto out; 18669c46f6d4SAlex Williamson 18677e58d5aeSAmos Kong if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 18687e58d5aeSAmos Kong sg_init_one(&sg, addr->sa_data, dev->addr_len); 18697e58d5aeSAmos Kong if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 1870d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 18717e58d5aeSAmos Kong dev_warn(&vdev->dev, 18727e58d5aeSAmos Kong "Failed to set mac address by vq command.\n"); 1873e37e2ff3SAndy Lutomirski ret = -EINVAL; 1874e37e2ff3SAndy Lutomirski goto out; 18757e58d5aeSAmos Kong } 18767e93a02fSMichael S. Tsirkin } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 18777e93a02fSMichael S. Tsirkin !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1878855e0c52SRusty Russell unsigned int i; 1879855e0c52SRusty Russell 1880855e0c52SRusty Russell /* Naturally, this has an atomicity problem. */ 1881855e0c52SRusty Russell for (i = 0; i < dev->addr_len; i++) 1882855e0c52SRusty Russell virtio_cwrite8(vdev, 1883855e0c52SRusty Russell offsetof(struct virtio_net_config, mac) + 1884855e0c52SRusty Russell i, addr->sa_data[i]); 18857e58d5aeSAmos Kong } 18867e58d5aeSAmos Kong 18877e58d5aeSAmos Kong eth_commit_mac_addr_change(dev, p); 1888e37e2ff3SAndy Lutomirski ret = 0; 18899c46f6d4SAlex Williamson 1890e37e2ff3SAndy Lutomirski out: 1891e37e2ff3SAndy Lutomirski kfree(addr); 1892e37e2ff3SAndy Lutomirski return ret; 18939c46f6d4SAlex Williamson } 18949c46f6d4SAlex Williamson 1895bc1f4470Sstephen hemminger static void virtnet_stats(struct net_device *dev, 18963fa2a1dfSstephen hemminger struct rtnl_link_stats64 *tot) 18973fa2a1dfSstephen hemminger { 18983fa2a1dfSstephen hemminger struct virtnet_info *vi = netdev_priv(dev); 18993fa2a1dfSstephen hemminger unsigned int start; 1900d7dfc5cfSToshiaki Makita int i; 19013fa2a1dfSstephen hemminger 1902d7dfc5cfSToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 1903a520794bSTony Lu u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops; 1904d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 1905d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 19063fa2a1dfSstephen hemminger 19073fa2a1dfSstephen hemminger do { 1908d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&sq->stats.syncp); 1909d7dfc5cfSToshiaki Makita tpackets = sq->stats.packets; 1910d7dfc5cfSToshiaki Makita tbytes = sq->stats.bytes; 1911a520794bSTony Lu terrors = sq->stats.tx_timeouts; 1912d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); 191383a27052SEric Dumazet 191483a27052SEric Dumazet do { 1915d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&rq->stats.syncp); 1916d46eeeafSJason Wang rpackets = rq->stats.packets; 1917d46eeeafSJason Wang rbytes = rq->stats.bytes; 1918d46eeeafSJason Wang rdrops = rq->stats.drops; 1919d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); 19203fa2a1dfSstephen hemminger 19213fa2a1dfSstephen hemminger tot->rx_packets += rpackets; 19223fa2a1dfSstephen hemminger tot->tx_packets += tpackets; 19233fa2a1dfSstephen hemminger tot->rx_bytes += rbytes; 19243fa2a1dfSstephen hemminger tot->tx_bytes += tbytes; 19252c4a2f7dSToshiaki Makita tot->rx_dropped += rdrops; 1926a520794bSTony Lu tot->tx_errors += terrors; 19273fa2a1dfSstephen hemminger } 19283fa2a1dfSstephen hemminger 19293fa2a1dfSstephen hemminger tot->tx_dropped = dev->stats.tx_dropped; 1930021ac8d3SRick Jones tot->tx_fifo_errors = dev->stats.tx_fifo_errors; 19313fa2a1dfSstephen hemminger tot->rx_length_errors = dev->stats.rx_length_errors; 19323fa2a1dfSstephen hemminger tot->rx_frame_errors = dev->stats.rx_frame_errors; 19333fa2a1dfSstephen hemminger } 19343fa2a1dfSstephen hemminger 1935586d17c5SJason Wang static void virtnet_ack_link_announce(struct virtnet_info *vi) 1936586d17c5SJason Wang { 1937586d17c5SJason Wang rtnl_lock(); 1938586d17c5SJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 1939d24bae32Sstephen hemminger VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 1940586d17c5SJason Wang dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 1941586d17c5SJason Wang rtnl_unlock(); 1942586d17c5SJason Wang } 1943586d17c5SJason Wang 194447315329SJohn Fastabend static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 1945986a4f4dSJason Wang { 1946986a4f4dSJason Wang struct scatterlist sg; 1947986a4f4dSJason Wang struct net_device *dev = vi->dev; 1948986a4f4dSJason Wang 1949986a4f4dSJason Wang if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 1950986a4f4dSJason Wang return 0; 1951986a4f4dSJason Wang 195212e57169SMichael S. Tsirkin vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 195312e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq)); 1954986a4f4dSJason Wang 1955986a4f4dSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 1956d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 1957986a4f4dSJason Wang dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 1958986a4f4dSJason Wang queue_pairs); 1959986a4f4dSJason Wang return -EINVAL; 196055257d72SSasha Levin } else { 1961986a4f4dSJason Wang vi->curr_queue_pairs = queue_pairs; 196235ed159bSJason Wang /* virtnet_open() will refill when device is going to up. */ 196335ed159bSJason Wang if (dev->flags & IFF_UP) 19649b9cd802SJason Wang schedule_delayed_work(&vi->refill, 0); 196555257d72SSasha Levin } 1966986a4f4dSJason Wang 1967986a4f4dSJason Wang return 0; 1968986a4f4dSJason Wang } 1969986a4f4dSJason Wang 197047315329SJohn Fastabend static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 197147315329SJohn Fastabend { 197247315329SJohn Fastabend int err; 197347315329SJohn Fastabend 197447315329SJohn Fastabend rtnl_lock(); 197547315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 197647315329SJohn Fastabend rtnl_unlock(); 197747315329SJohn Fastabend return err; 197847315329SJohn Fastabend } 197947315329SJohn Fastabend 1980296f96fcSRusty Russell static int virtnet_close(struct net_device *dev) 1981296f96fcSRusty Russell { 1982296f96fcSRusty Russell struct virtnet_info *vi = netdev_priv(dev); 1983986a4f4dSJason Wang int i; 1984296f96fcSRusty Russell 1985b2baed69SRusty Russell /* Make sure refill_work doesn't re-enable napi! */ 1986b2baed69SRusty Russell cancel_delayed_work_sync(&vi->refill); 1987986a4f4dSJason Wang 1988b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 1989754b8a21SJesper Dangaard Brouer xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 1990986a4f4dSJason Wang napi_disable(&vi->rq[i].napi); 199178a57b48SWillem de Bruijn virtnet_napi_tx_disable(&vi->sq[i].napi); 1992b92f1e67SWillem de Bruijn } 1993296f96fcSRusty Russell 1994296f96fcSRusty Russell return 0; 1995296f96fcSRusty Russell } 1996296f96fcSRusty Russell 19972af7698eSAlex Williamson static void virtnet_set_rx_mode(struct net_device *dev) 19982af7698eSAlex Williamson { 19992af7698eSAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 2000f565a7c2SAlex Williamson struct scatterlist sg[2]; 2001f565a7c2SAlex Williamson struct virtio_net_ctrl_mac *mac_data; 2002ccffad25SJiri Pirko struct netdev_hw_addr *ha; 200332e7bfc4SJiri Pirko int uc_count; 20044cd24eafSJiri Pirko int mc_count; 2005f565a7c2SAlex Williamson void *buf; 2006f565a7c2SAlex Williamson int i; 20072af7698eSAlex Williamson 2008788a8b6dSstephen hemminger /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 20092af7698eSAlex Williamson if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 20102af7698eSAlex Williamson return; 20112af7698eSAlex Williamson 201212e57169SMichael S. Tsirkin vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0); 201312e57169SMichael S. Tsirkin vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0); 20142af7698eSAlex Williamson 201512e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc)); 20162af7698eSAlex Williamson 20172af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 2018d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_PROMISC, sg)) 20192af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 202012e57169SMichael S. Tsirkin vi->ctrl->promisc ? "en" : "dis"); 20212af7698eSAlex Williamson 202212e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti)); 20232af7698eSAlex Williamson 20242af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 2025d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 20262af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 202712e57169SMichael S. Tsirkin vi->ctrl->allmulti ? "en" : "dis"); 2028f565a7c2SAlex Williamson 202932e7bfc4SJiri Pirko uc_count = netdev_uc_count(dev); 20304cd24eafSJiri Pirko mc_count = netdev_mc_count(dev); 2031f565a7c2SAlex Williamson /* MAC filter - use one buffer for both lists */ 20324cd24eafSJiri Pirko buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 2033f565a7c2SAlex Williamson (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 20344cd24eafSJiri Pirko mac_data = buf; 2035e68ed8f0SJoe Perches if (!buf) 2036f565a7c2SAlex Williamson return; 2037f565a7c2SAlex Williamson 203823e258e1SAlex Williamson sg_init_table(sg, 2); 203923e258e1SAlex Williamson 2040f565a7c2SAlex Williamson /* Store the unicast list and count in the front of the buffer */ 2041fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 2042ccffad25SJiri Pirko i = 0; 204332e7bfc4SJiri Pirko netdev_for_each_uc_addr(ha, dev) 2044ccffad25SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2045f565a7c2SAlex Williamson 2046f565a7c2SAlex Williamson sg_set_buf(&sg[0], mac_data, 204732e7bfc4SJiri Pirko sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 2048f565a7c2SAlex Williamson 2049f565a7c2SAlex Williamson /* multicast list and count fill the end */ 205032e7bfc4SJiri Pirko mac_data = (void *)&mac_data->macs[uc_count][0]; 2051f565a7c2SAlex Williamson 2052fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 2053567ec874SJiri Pirko i = 0; 205422bedad3SJiri Pirko netdev_for_each_mc_addr(ha, dev) 205522bedad3SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2056f565a7c2SAlex Williamson 2057f565a7c2SAlex Williamson sg_set_buf(&sg[1], mac_data, 20584cd24eafSJiri Pirko sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 2059f565a7c2SAlex Williamson 2060f565a7c2SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 2061d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 206299e872aeSThomas Huth dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 2063f565a7c2SAlex Williamson 2064f565a7c2SAlex Williamson kfree(buf); 20652af7698eSAlex Williamson } 20662af7698eSAlex Williamson 206780d5c368SPatrick McHardy static int virtnet_vlan_rx_add_vid(struct net_device *dev, 206880d5c368SPatrick McHardy __be16 proto, u16 vid) 20690bde9569SAlex Williamson { 20700bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 20710bde9569SAlex Williamson struct scatterlist sg; 20720bde9569SAlex Williamson 2073d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 207412e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 20750bde9569SAlex Williamson 20760bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2077d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 20780bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 20798e586137SJiri Pirko return 0; 20800bde9569SAlex Williamson } 20810bde9569SAlex Williamson 208280d5c368SPatrick McHardy static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 208380d5c368SPatrick McHardy __be16 proto, u16 vid) 20840bde9569SAlex Williamson { 20850bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 20860bde9569SAlex Williamson struct scatterlist sg; 20870bde9569SAlex Williamson 2088d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 208912e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 20900bde9569SAlex Williamson 20910bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2092d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 20930bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 20948e586137SJiri Pirko return 0; 20950bde9569SAlex Williamson } 20960bde9569SAlex Williamson 2097310974faSPeter Xu static void virtnet_clean_affinity(struct virtnet_info *vi) 2098986a4f4dSJason Wang { 2099986a4f4dSJason Wang int i; 21008898c21cSWanlong Gao 21018898c21cSWanlong Gao if (vi->affinity_hint_set) { 21028898c21cSWanlong Gao for (i = 0; i < vi->max_queue_pairs; i++) { 210319e226e8SCaleb Raitto virtqueue_set_affinity(vi->rq[i].vq, NULL); 210419e226e8SCaleb Raitto virtqueue_set_affinity(vi->sq[i].vq, NULL); 21058898c21cSWanlong Gao } 21068898c21cSWanlong Gao 21078898c21cSWanlong Gao vi->affinity_hint_set = false; 21088898c21cSWanlong Gao } 21098898c21cSWanlong Gao } 21108898c21cSWanlong Gao 21118898c21cSWanlong Gao static void virtnet_set_affinity(struct virtnet_info *vi) 2112986a4f4dSJason Wang { 21132ca653d6SCaleb Raitto cpumask_var_t mask; 21142ca653d6SCaleb Raitto int stragglers; 21152ca653d6SCaleb Raitto int group_size; 21162ca653d6SCaleb Raitto int i, j, cpu; 21172ca653d6SCaleb Raitto int num_cpu; 21182ca653d6SCaleb Raitto int stride; 2119986a4f4dSJason Wang 21202ca653d6SCaleb Raitto if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 2121310974faSPeter Xu virtnet_clean_affinity(vi); 2122986a4f4dSJason Wang return; 2123986a4f4dSJason Wang } 2124986a4f4dSJason Wang 21252ca653d6SCaleb Raitto num_cpu = num_online_cpus(); 21262ca653d6SCaleb Raitto stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); 21272ca653d6SCaleb Raitto stragglers = num_cpu >= vi->curr_queue_pairs ? 21282ca653d6SCaleb Raitto num_cpu % vi->curr_queue_pairs : 21292ca653d6SCaleb Raitto 0; 21309b51d9d8SYury Norov cpu = cpumask_first(cpu_online_mask); 21314d99f660SAndrei Vagin 21322ca653d6SCaleb Raitto for (i = 0; i < vi->curr_queue_pairs; i++) { 21332ca653d6SCaleb Raitto group_size = stride + (i < stragglers ? 1 : 0); 21342ca653d6SCaleb Raitto 21352ca653d6SCaleb Raitto for (j = 0; j < group_size; j++) { 21362ca653d6SCaleb Raitto cpumask_set_cpu(cpu, mask); 21372ca653d6SCaleb Raitto cpu = cpumask_next_wrap(cpu, cpu_online_mask, 21382ca653d6SCaleb Raitto nr_cpu_ids, false); 21392ca653d6SCaleb Raitto } 21402ca653d6SCaleb Raitto virtqueue_set_affinity(vi->rq[i].vq, mask); 21412ca653d6SCaleb Raitto virtqueue_set_affinity(vi->sq[i].vq, mask); 2142044ab86dSAntoine Tenart __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); 21432ca653d6SCaleb Raitto cpumask_clear(mask); 2144986a4f4dSJason Wang } 2145986a4f4dSJason Wang 2146986a4f4dSJason Wang vi->affinity_hint_set = true; 21472ca653d6SCaleb Raitto free_cpumask_var(mask); 214847be2479SWanlong Gao } 2149986a4f4dSJason Wang 21508017c279SSebastian Andrzej Siewior static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 21518de4b2f3SWanlong Gao { 21528017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 21538017c279SSebastian Andrzej Siewior node); 21548de4b2f3SWanlong Gao virtnet_set_affinity(vi); 21558017c279SSebastian Andrzej Siewior return 0; 21568de4b2f3SWanlong Gao } 21573ab098dfSJason Wang 21588017c279SSebastian Andrzej Siewior static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 21598017c279SSebastian Andrzej Siewior { 21608017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 21618017c279SSebastian Andrzej Siewior node_dead); 21628017c279SSebastian Andrzej Siewior virtnet_set_affinity(vi); 21638017c279SSebastian Andrzej Siewior return 0; 21648017c279SSebastian Andrzej Siewior } 21658017c279SSebastian Andrzej Siewior 21668017c279SSebastian Andrzej Siewior static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 21678017c279SSebastian Andrzej Siewior { 21688017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 21698017c279SSebastian Andrzej Siewior node); 21708017c279SSebastian Andrzej Siewior 2171310974faSPeter Xu virtnet_clean_affinity(vi); 21728017c279SSebastian Andrzej Siewior return 0; 21738017c279SSebastian Andrzej Siewior } 21748017c279SSebastian Andrzej Siewior 21758017c279SSebastian Andrzej Siewior static enum cpuhp_state virtionet_online; 21768017c279SSebastian Andrzej Siewior 21778017c279SSebastian Andrzej Siewior static int virtnet_cpu_notif_add(struct virtnet_info *vi) 21788017c279SSebastian Andrzej Siewior { 21798017c279SSebastian Andrzej Siewior int ret; 21808017c279SSebastian Andrzej Siewior 21818017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 21828017c279SSebastian Andrzej Siewior if (ret) 21838017c279SSebastian Andrzej Siewior return ret; 21848017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 21858017c279SSebastian Andrzej Siewior &vi->node_dead); 21868017c279SSebastian Andrzej Siewior if (!ret) 21878017c279SSebastian Andrzej Siewior return ret; 21888017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 21898017c279SSebastian Andrzej Siewior return ret; 21908017c279SSebastian Andrzej Siewior } 21918017c279SSebastian Andrzej Siewior 21928017c279SSebastian Andrzej Siewior static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 21938017c279SSebastian Andrzej Siewior { 21948017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 21958017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 21968017c279SSebastian Andrzej Siewior &vi->node_dead); 2197a9ea3fc6SHerbert Xu } 2198a9ea3fc6SHerbert Xu 21998f9f4668SRick Jones static void virtnet_get_ringparam(struct net_device *dev, 220074624944SHao Chen struct ethtool_ringparam *ring, 220174624944SHao Chen struct kernel_ethtool_ringparam *kernel_ring, 220274624944SHao Chen struct netlink_ext_ack *extack) 22038f9f4668SRick Jones { 22048f9f4668SRick Jones struct virtnet_info *vi = netdev_priv(dev); 22058f9f4668SRick Jones 2206986a4f4dSJason Wang ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq); 2207986a4f4dSJason Wang ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq); 22088f9f4668SRick Jones ring->rx_pending = ring->rx_max_pending; 22098f9f4668SRick Jones ring->tx_pending = ring->tx_max_pending; 22108f9f4668SRick Jones } 22118f9f4668SRick Jones 2212*c7114b12SAndrew Melnychenko static bool virtnet_commit_rss_command(struct virtnet_info *vi) 2213*c7114b12SAndrew Melnychenko { 2214*c7114b12SAndrew Melnychenko struct net_device *dev = vi->dev; 2215*c7114b12SAndrew Melnychenko struct scatterlist sgs[4]; 2216*c7114b12SAndrew Melnychenko unsigned int sg_buf_size; 2217*c7114b12SAndrew Melnychenko 2218*c7114b12SAndrew Melnychenko /* prepare sgs */ 2219*c7114b12SAndrew Melnychenko sg_init_table(sgs, 4); 2220*c7114b12SAndrew Melnychenko 2221*c7114b12SAndrew Melnychenko sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table); 2222*c7114b12SAndrew Melnychenko sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size); 2223*c7114b12SAndrew Melnychenko 2224*c7114b12SAndrew Melnychenko sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1); 2225*c7114b12SAndrew Melnychenko sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size); 2226*c7114b12SAndrew Melnychenko 2227*c7114b12SAndrew Melnychenko sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key) 2228*c7114b12SAndrew Melnychenko - offsetof(struct virtio_net_ctrl_rss, max_tx_vq); 2229*c7114b12SAndrew Melnychenko sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size); 2230*c7114b12SAndrew Melnychenko 2231*c7114b12SAndrew Melnychenko sg_buf_size = vi->rss_key_size; 2232*c7114b12SAndrew Melnychenko sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size); 2233*c7114b12SAndrew Melnychenko 2234*c7114b12SAndrew Melnychenko if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 2235*c7114b12SAndrew Melnychenko VIRTIO_NET_CTRL_MQ_RSS_CONFIG, sgs)) { 2236*c7114b12SAndrew Melnychenko dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n"); 2237*c7114b12SAndrew Melnychenko return false; 2238*c7114b12SAndrew Melnychenko } 2239*c7114b12SAndrew Melnychenko return true; 2240*c7114b12SAndrew Melnychenko } 2241*c7114b12SAndrew Melnychenko 2242*c7114b12SAndrew Melnychenko static void virtnet_init_default_rss(struct virtnet_info *vi) 2243*c7114b12SAndrew Melnychenko { 2244*c7114b12SAndrew Melnychenko u32 indir_val = 0; 2245*c7114b12SAndrew Melnychenko int i = 0; 2246*c7114b12SAndrew Melnychenko 2247*c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_types = vi->rss_hash_types_supported; 2248*c7114b12SAndrew Melnychenko vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size 2249*c7114b12SAndrew Melnychenko ? vi->rss_indir_table_size - 1 : 0; 2250*c7114b12SAndrew Melnychenko vi->ctrl->rss.unclassified_queue = 0; 2251*c7114b12SAndrew Melnychenko 2252*c7114b12SAndrew Melnychenko for (; i < vi->rss_indir_table_size; ++i) { 2253*c7114b12SAndrew Melnychenko indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs); 2254*c7114b12SAndrew Melnychenko vi->ctrl->rss.indirection_table[i] = indir_val; 2255*c7114b12SAndrew Melnychenko } 2256*c7114b12SAndrew Melnychenko 2257*c7114b12SAndrew Melnychenko vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs; 2258*c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_key_length = vi->rss_key_size; 2259*c7114b12SAndrew Melnychenko 2260*c7114b12SAndrew Melnychenko netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size); 2261*c7114b12SAndrew Melnychenko } 2262*c7114b12SAndrew Melnychenko 226366846048SRick Jones 226466846048SRick Jones static void virtnet_get_drvinfo(struct net_device *dev, 226566846048SRick Jones struct ethtool_drvinfo *info) 226666846048SRick Jones { 226766846048SRick Jones struct virtnet_info *vi = netdev_priv(dev); 226866846048SRick Jones struct virtio_device *vdev = vi->vdev; 226966846048SRick Jones 227066846048SRick Jones strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 227166846048SRick Jones strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 227266846048SRick Jones strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 227366846048SRick Jones 227466846048SRick Jones } 227566846048SRick Jones 2276d73bcd2cSJason Wang /* TODO: Eliminate OOO packets during switching */ 2277d73bcd2cSJason Wang static int virtnet_set_channels(struct net_device *dev, 2278d73bcd2cSJason Wang struct ethtool_channels *channels) 2279d73bcd2cSJason Wang { 2280d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 2281d73bcd2cSJason Wang u16 queue_pairs = channels->combined_count; 2282d73bcd2cSJason Wang int err; 2283d73bcd2cSJason Wang 2284d73bcd2cSJason Wang /* We don't support separate rx/tx channels. 2285d73bcd2cSJason Wang * We don't allow setting 'other' channels. 2286d73bcd2cSJason Wang */ 2287d73bcd2cSJason Wang if (channels->rx_count || channels->tx_count || channels->other_count) 2288d73bcd2cSJason Wang return -EINVAL; 2289d73bcd2cSJason Wang 2290c18e9cd6SAmos Kong if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 2291d73bcd2cSJason Wang return -EINVAL; 2292d73bcd2cSJason Wang 2293f600b690SJohn Fastabend /* For now we don't support modifying channels while XDP is loaded 2294f600b690SJohn Fastabend * also when XDP is loaded all RX queues have XDP programs so we only 2295f600b690SJohn Fastabend * need to check a single RX queue. 2296f600b690SJohn Fastabend */ 2297f600b690SJohn Fastabend if (vi->rq[0].xdp_prog) 2298f600b690SJohn Fastabend return -EINVAL; 2299f600b690SJohn Fastabend 2300a0d1d0f4SSebastian Andrzej Siewior cpus_read_lock(); 230147315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 2302de33212fSJeff Dike if (err) { 2303a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 2304de33212fSJeff Dike goto err; 2305d73bcd2cSJason Wang } 2306de33212fSJeff Dike virtnet_set_affinity(vi); 2307a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 2308d73bcd2cSJason Wang 2309de33212fSJeff Dike netif_set_real_num_tx_queues(dev, queue_pairs); 2310de33212fSJeff Dike netif_set_real_num_rx_queues(dev, queue_pairs); 2311de33212fSJeff Dike err: 2312d73bcd2cSJason Wang return err; 2313d73bcd2cSJason Wang } 2314d73bcd2cSJason Wang 2315d7dfc5cfSToshiaki Makita static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 2316d7dfc5cfSToshiaki Makita { 2317d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 2318d7dfc5cfSToshiaki Makita unsigned int i, j; 2319d7a9a01bSAlexander Duyck u8 *p = data; 2320d7dfc5cfSToshiaki Makita 2321d7dfc5cfSToshiaki Makita switch (stringset) { 2322d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 2323d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2324d7a9a01bSAlexander Duyck for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) 2325d7a9a01bSAlexander Duyck ethtool_sprintf(&p, "rx_queue_%u_%s", i, 2326d7a9a01bSAlexander Duyck virtnet_rq_stats_desc[j].desc); 2327d7dfc5cfSToshiaki Makita } 2328d7dfc5cfSToshiaki Makita 2329d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2330d7a9a01bSAlexander Duyck for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) 2331d7a9a01bSAlexander Duyck ethtool_sprintf(&p, "tx_queue_%u_%s", i, 2332d7a9a01bSAlexander Duyck virtnet_sq_stats_desc[j].desc); 2333d7dfc5cfSToshiaki Makita } 2334d7dfc5cfSToshiaki Makita break; 2335d7dfc5cfSToshiaki Makita } 2336d7dfc5cfSToshiaki Makita } 2337d7dfc5cfSToshiaki Makita 2338d7dfc5cfSToshiaki Makita static int virtnet_get_sset_count(struct net_device *dev, int sset) 2339d7dfc5cfSToshiaki Makita { 2340d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 2341d7dfc5cfSToshiaki Makita 2342d7dfc5cfSToshiaki Makita switch (sset) { 2343d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 2344d7dfc5cfSToshiaki Makita return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN + 2345d7dfc5cfSToshiaki Makita VIRTNET_SQ_STATS_LEN); 2346d7dfc5cfSToshiaki Makita default: 2347d7dfc5cfSToshiaki Makita return -EOPNOTSUPP; 2348d7dfc5cfSToshiaki Makita } 2349d7dfc5cfSToshiaki Makita } 2350d7dfc5cfSToshiaki Makita 2351d7dfc5cfSToshiaki Makita static void virtnet_get_ethtool_stats(struct net_device *dev, 2352d7dfc5cfSToshiaki Makita struct ethtool_stats *stats, u64 *data) 2353d7dfc5cfSToshiaki Makita { 2354d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 2355d7dfc5cfSToshiaki Makita unsigned int idx = 0, start, i, j; 2356d7dfc5cfSToshiaki Makita const u8 *stats_base; 2357d7dfc5cfSToshiaki Makita size_t offset; 2358d7dfc5cfSToshiaki Makita 2359d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2360d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 2361d7dfc5cfSToshiaki Makita 2362d46eeeafSJason Wang stats_base = (u8 *)&rq->stats; 2363d7dfc5cfSToshiaki Makita do { 2364d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&rq->stats.syncp); 2365d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { 2366d7dfc5cfSToshiaki Makita offset = virtnet_rq_stats_desc[j].offset; 2367d7dfc5cfSToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 2368d7dfc5cfSToshiaki Makita } 2369d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); 2370d7dfc5cfSToshiaki Makita idx += VIRTNET_RQ_STATS_LEN; 2371d7dfc5cfSToshiaki Makita } 2372d7dfc5cfSToshiaki Makita 2373d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2374d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 2375d7dfc5cfSToshiaki Makita 2376d7dfc5cfSToshiaki Makita stats_base = (u8 *)&sq->stats; 2377d7dfc5cfSToshiaki Makita do { 2378d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&sq->stats.syncp); 2379d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { 2380d7dfc5cfSToshiaki Makita offset = virtnet_sq_stats_desc[j].offset; 2381d7dfc5cfSToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 2382d7dfc5cfSToshiaki Makita } 2383d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); 2384d7dfc5cfSToshiaki Makita idx += VIRTNET_SQ_STATS_LEN; 2385d7dfc5cfSToshiaki Makita } 2386d7dfc5cfSToshiaki Makita } 2387d7dfc5cfSToshiaki Makita 2388d73bcd2cSJason Wang static void virtnet_get_channels(struct net_device *dev, 2389d73bcd2cSJason Wang struct ethtool_channels *channels) 2390d73bcd2cSJason Wang { 2391d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 2392d73bcd2cSJason Wang 2393d73bcd2cSJason Wang channels->combined_count = vi->curr_queue_pairs; 2394d73bcd2cSJason Wang channels->max_combined = vi->max_queue_pairs; 2395d73bcd2cSJason Wang channels->max_other = 0; 2396d73bcd2cSJason Wang channels->rx_count = 0; 2397d73bcd2cSJason Wang channels->tx_count = 0; 2398d73bcd2cSJason Wang channels->other_count = 0; 2399d73bcd2cSJason Wang } 2400d73bcd2cSJason Wang 2401ebb6b4b1SPhilippe Reynes static int virtnet_set_link_ksettings(struct net_device *dev, 2402ebb6b4b1SPhilippe Reynes const struct ethtool_link_ksettings *cmd) 240316032be5SNikolay Aleksandrov { 240416032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 240516032be5SNikolay Aleksandrov 24069aedc6e2SCris Forno return ethtool_virtdev_set_link_ksettings(dev, cmd, 24079aedc6e2SCris Forno &vi->speed, &vi->duplex); 240816032be5SNikolay Aleksandrov } 240916032be5SNikolay Aleksandrov 2410ebb6b4b1SPhilippe Reynes static int virtnet_get_link_ksettings(struct net_device *dev, 2411ebb6b4b1SPhilippe Reynes struct ethtool_link_ksettings *cmd) 241216032be5SNikolay Aleksandrov { 241316032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 241416032be5SNikolay Aleksandrov 2415ebb6b4b1SPhilippe Reynes cmd->base.speed = vi->speed; 2416ebb6b4b1SPhilippe Reynes cmd->base.duplex = vi->duplex; 2417ebb6b4b1SPhilippe Reynes cmd->base.port = PORT_OTHER; 241816032be5SNikolay Aleksandrov 241916032be5SNikolay Aleksandrov return 0; 242016032be5SNikolay Aleksandrov } 242116032be5SNikolay Aleksandrov 24220c465be1SJason Wang static int virtnet_set_coalesce(struct net_device *dev, 2423f3ccfda1SYufeng Mo struct ethtool_coalesce *ec, 2424f3ccfda1SYufeng Mo struct kernel_ethtool_coalesce *kernel_coal, 2425f3ccfda1SYufeng Mo struct netlink_ext_ack *extack) 24260c465be1SJason Wang { 24270c465be1SJason Wang struct virtnet_info *vi = netdev_priv(dev); 24280c465be1SJason Wang int i, napi_weight; 24290c465be1SJason Wang 2430a51e5206SJakub Kicinski if (ec->tx_max_coalesced_frames > 1 || 2431a51e5206SJakub Kicinski ec->rx_max_coalesced_frames != 1) 24320c465be1SJason Wang return -EINVAL; 24330c465be1SJason Wang 24340c465be1SJason Wang napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 24350c465be1SJason Wang if (napi_weight ^ vi->sq[0].napi.weight) { 24360c465be1SJason Wang if (dev->flags & IFF_UP) 24370c465be1SJason Wang return -EBUSY; 24380c465be1SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 24390c465be1SJason Wang vi->sq[i].napi.weight = napi_weight; 24400c465be1SJason Wang } 24410c465be1SJason Wang 24420c465be1SJason Wang return 0; 24430c465be1SJason Wang } 24440c465be1SJason Wang 24450c465be1SJason Wang static int virtnet_get_coalesce(struct net_device *dev, 2446f3ccfda1SYufeng Mo struct ethtool_coalesce *ec, 2447f3ccfda1SYufeng Mo struct kernel_ethtool_coalesce *kernel_coal, 2448f3ccfda1SYufeng Mo struct netlink_ext_ack *extack) 24490c465be1SJason Wang { 24500c465be1SJason Wang struct ethtool_coalesce ec_default = { 24510c465be1SJason Wang .cmd = ETHTOOL_GCOALESCE, 24520c465be1SJason Wang .rx_max_coalesced_frames = 1, 24530c465be1SJason Wang }; 24540c465be1SJason Wang struct virtnet_info *vi = netdev_priv(dev); 24550c465be1SJason Wang 24560c465be1SJason Wang memcpy(ec, &ec_default, sizeof(ec_default)); 24570c465be1SJason Wang 24580c465be1SJason Wang if (vi->sq[0].napi.weight) 24590c465be1SJason Wang ec->tx_max_coalesced_frames = 1; 24600c465be1SJason Wang 24610c465be1SJason Wang return 0; 24620c465be1SJason Wang } 24630c465be1SJason Wang 246416032be5SNikolay Aleksandrov static void virtnet_init_settings(struct net_device *dev) 246516032be5SNikolay Aleksandrov { 246616032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 246716032be5SNikolay Aleksandrov 246816032be5SNikolay Aleksandrov vi->speed = SPEED_UNKNOWN; 246916032be5SNikolay Aleksandrov vi->duplex = DUPLEX_UNKNOWN; 247016032be5SNikolay Aleksandrov } 247116032be5SNikolay Aleksandrov 2472faa9b39fSJason Baron static void virtnet_update_settings(struct virtnet_info *vi) 2473faa9b39fSJason Baron { 2474faa9b39fSJason Baron u32 speed; 2475faa9b39fSJason Baron u8 duplex; 2476faa9b39fSJason Baron 2477faa9b39fSJason Baron if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 2478faa9b39fSJason Baron return; 2479faa9b39fSJason Baron 248064ffa39dSMichael S. Tsirkin virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); 248164ffa39dSMichael S. Tsirkin 2482faa9b39fSJason Baron if (ethtool_validate_speed(speed)) 2483faa9b39fSJason Baron vi->speed = speed; 248464ffa39dSMichael S. Tsirkin 248564ffa39dSMichael S. Tsirkin virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); 248664ffa39dSMichael S. Tsirkin 2487faa9b39fSJason Baron if (ethtool_validate_duplex(duplex)) 2488faa9b39fSJason Baron vi->duplex = duplex; 2489faa9b39fSJason Baron } 2490faa9b39fSJason Baron 2491*c7114b12SAndrew Melnychenko static u32 virtnet_get_rxfh_key_size(struct net_device *dev) 2492*c7114b12SAndrew Melnychenko { 2493*c7114b12SAndrew Melnychenko return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size; 2494*c7114b12SAndrew Melnychenko } 2495*c7114b12SAndrew Melnychenko 2496*c7114b12SAndrew Melnychenko static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) 2497*c7114b12SAndrew Melnychenko { 2498*c7114b12SAndrew Melnychenko return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; 2499*c7114b12SAndrew Melnychenko } 2500*c7114b12SAndrew Melnychenko 2501*c7114b12SAndrew Melnychenko static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) 2502*c7114b12SAndrew Melnychenko { 2503*c7114b12SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 2504*c7114b12SAndrew Melnychenko int i; 2505*c7114b12SAndrew Melnychenko 2506*c7114b12SAndrew Melnychenko if (indir) { 2507*c7114b12SAndrew Melnychenko for (i = 0; i < vi->rss_indir_table_size; ++i) 2508*c7114b12SAndrew Melnychenko indir[i] = vi->ctrl->rss.indirection_table[i]; 2509*c7114b12SAndrew Melnychenko } 2510*c7114b12SAndrew Melnychenko 2511*c7114b12SAndrew Melnychenko if (key) 2512*c7114b12SAndrew Melnychenko memcpy(key, vi->ctrl->rss.key, vi->rss_key_size); 2513*c7114b12SAndrew Melnychenko 2514*c7114b12SAndrew Melnychenko if (hfunc) 2515*c7114b12SAndrew Melnychenko *hfunc = ETH_RSS_HASH_TOP; 2516*c7114b12SAndrew Melnychenko 2517*c7114b12SAndrew Melnychenko return 0; 2518*c7114b12SAndrew Melnychenko } 2519*c7114b12SAndrew Melnychenko 2520*c7114b12SAndrew Melnychenko static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc) 2521*c7114b12SAndrew Melnychenko { 2522*c7114b12SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 2523*c7114b12SAndrew Melnychenko int i; 2524*c7114b12SAndrew Melnychenko 2525*c7114b12SAndrew Melnychenko if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) 2526*c7114b12SAndrew Melnychenko return -EOPNOTSUPP; 2527*c7114b12SAndrew Melnychenko 2528*c7114b12SAndrew Melnychenko if (indir) { 2529*c7114b12SAndrew Melnychenko for (i = 0; i < vi->rss_indir_table_size; ++i) 2530*c7114b12SAndrew Melnychenko vi->ctrl->rss.indirection_table[i] = indir[i]; 2531*c7114b12SAndrew Melnychenko } 2532*c7114b12SAndrew Melnychenko if (key) 2533*c7114b12SAndrew Melnychenko memcpy(vi->ctrl->rss.key, key, vi->rss_key_size); 2534*c7114b12SAndrew Melnychenko 2535*c7114b12SAndrew Melnychenko virtnet_commit_rss_command(vi); 2536*c7114b12SAndrew Melnychenko 2537*c7114b12SAndrew Melnychenko return 0; 2538*c7114b12SAndrew Melnychenko } 2539*c7114b12SAndrew Melnychenko 2540*c7114b12SAndrew Melnychenko static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs) 2541*c7114b12SAndrew Melnychenko { 2542*c7114b12SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 2543*c7114b12SAndrew Melnychenko int rc = 0; 2544*c7114b12SAndrew Melnychenko 2545*c7114b12SAndrew Melnychenko switch (info->cmd) { 2546*c7114b12SAndrew Melnychenko case ETHTOOL_GRXRINGS: 2547*c7114b12SAndrew Melnychenko info->data = vi->curr_queue_pairs; 2548*c7114b12SAndrew Melnychenko break; 2549*c7114b12SAndrew Melnychenko default: 2550*c7114b12SAndrew Melnychenko rc = -EOPNOTSUPP; 2551*c7114b12SAndrew Melnychenko } 2552*c7114b12SAndrew Melnychenko 2553*c7114b12SAndrew Melnychenko return rc; 2554*c7114b12SAndrew Melnychenko } 2555*c7114b12SAndrew Melnychenko 25560fc0b732SStephen Hemminger static const struct ethtool_ops virtnet_ethtool_ops = { 2557a51e5206SJakub Kicinski .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES, 255866846048SRick Jones .get_drvinfo = virtnet_get_drvinfo, 25599f4d26d0SMark McLoughlin .get_link = ethtool_op_get_link, 25608f9f4668SRick Jones .get_ringparam = virtnet_get_ringparam, 2561d7dfc5cfSToshiaki Makita .get_strings = virtnet_get_strings, 2562d7dfc5cfSToshiaki Makita .get_sset_count = virtnet_get_sset_count, 2563d7dfc5cfSToshiaki Makita .get_ethtool_stats = virtnet_get_ethtool_stats, 2564d73bcd2cSJason Wang .set_channels = virtnet_set_channels, 2565d73bcd2cSJason Wang .get_channels = virtnet_get_channels, 2566074c3582SJacob Keller .get_ts_info = ethtool_op_get_ts_info, 2567ebb6b4b1SPhilippe Reynes .get_link_ksettings = virtnet_get_link_ksettings, 2568ebb6b4b1SPhilippe Reynes .set_link_ksettings = virtnet_set_link_ksettings, 25690c465be1SJason Wang .set_coalesce = virtnet_set_coalesce, 25700c465be1SJason Wang .get_coalesce = virtnet_get_coalesce, 2571*c7114b12SAndrew Melnychenko .get_rxfh_key_size = virtnet_get_rxfh_key_size, 2572*c7114b12SAndrew Melnychenko .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, 2573*c7114b12SAndrew Melnychenko .get_rxfh = virtnet_get_rxfh, 2574*c7114b12SAndrew Melnychenko .set_rxfh = virtnet_set_rxfh, 2575*c7114b12SAndrew Melnychenko .get_rxnfc = virtnet_get_rxnfc, 2576a9ea3fc6SHerbert Xu }; 2577a9ea3fc6SHerbert Xu 25789fe7bfceSJohn Fastabend static void virtnet_freeze_down(struct virtio_device *vdev) 25799fe7bfceSJohn Fastabend { 25809fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 25819fe7bfceSJohn Fastabend int i; 25829fe7bfceSJohn Fastabend 25839fe7bfceSJohn Fastabend /* Make sure no work handler is accessing the device */ 25849fe7bfceSJohn Fastabend flush_work(&vi->config_work); 25859fe7bfceSJohn Fastabend 258605c998b7SAke Koomsin netif_tx_lock_bh(vi->dev); 25879fe7bfceSJohn Fastabend netif_device_detach(vi->dev); 258805c998b7SAke Koomsin netif_tx_unlock_bh(vi->dev); 25899fe7bfceSJohn Fastabend cancel_delayed_work_sync(&vi->refill); 25909fe7bfceSJohn Fastabend 25919fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 2592b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 25939fe7bfceSJohn Fastabend napi_disable(&vi->rq[i].napi); 259478a57b48SWillem de Bruijn virtnet_napi_tx_disable(&vi->sq[i].napi); 2595b92f1e67SWillem de Bruijn } 25969fe7bfceSJohn Fastabend } 25979fe7bfceSJohn Fastabend } 25989fe7bfceSJohn Fastabend 25999fe7bfceSJohn Fastabend static int init_vqs(struct virtnet_info *vi); 26009fe7bfceSJohn Fastabend 26019fe7bfceSJohn Fastabend static int virtnet_restore_up(struct virtio_device *vdev) 26029fe7bfceSJohn Fastabend { 26039fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 26049fe7bfceSJohn Fastabend int err, i; 26059fe7bfceSJohn Fastabend 26069fe7bfceSJohn Fastabend err = init_vqs(vi); 26079fe7bfceSJohn Fastabend if (err) 26089fe7bfceSJohn Fastabend return err; 26099fe7bfceSJohn Fastabend 26109fe7bfceSJohn Fastabend virtio_device_ready(vdev); 26119fe7bfceSJohn Fastabend 26129fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 26139fe7bfceSJohn Fastabend for (i = 0; i < vi->curr_queue_pairs; i++) 26149fe7bfceSJohn Fastabend if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 26159fe7bfceSJohn Fastabend schedule_delayed_work(&vi->refill, 0); 26169fe7bfceSJohn Fastabend 2617b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 2618e4e8452aSWillem de Bruijn virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2619b92f1e67SWillem de Bruijn virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2620b92f1e67SWillem de Bruijn &vi->sq[i].napi); 2621b92f1e67SWillem de Bruijn } 26229fe7bfceSJohn Fastabend } 26239fe7bfceSJohn Fastabend 262405c998b7SAke Koomsin netif_tx_lock_bh(vi->dev); 26259fe7bfceSJohn Fastabend netif_device_attach(vi->dev); 262605c998b7SAke Koomsin netif_tx_unlock_bh(vi->dev); 26279fe7bfceSJohn Fastabend return err; 26289fe7bfceSJohn Fastabend } 26299fe7bfceSJohn Fastabend 26303f93522fSJason Wang static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 26313f93522fSJason Wang { 26323f93522fSJason Wang struct scatterlist sg; 263312e57169SMichael S. Tsirkin vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads); 26343f93522fSJason Wang 263512e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads)); 26363f93522fSJason Wang 26373f93522fSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 26383f93522fSJason Wang VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 26393f93522fSJason Wang dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); 26403f93522fSJason Wang return -EINVAL; 26413f93522fSJason Wang } 26423f93522fSJason Wang 26433f93522fSJason Wang return 0; 26443f93522fSJason Wang } 26453f93522fSJason Wang 26463f93522fSJason Wang static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 26473f93522fSJason Wang { 26483f93522fSJason Wang u64 offloads = 0; 26493f93522fSJason Wang 26503f93522fSJason Wang if (!vi->guest_offloads) 26513f93522fSJason Wang return 0; 26523f93522fSJason Wang 26533f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 26543f93522fSJason Wang } 26553f93522fSJason Wang 26563f93522fSJason Wang static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 26573f93522fSJason Wang { 26583f93522fSJason Wang u64 offloads = vi->guest_offloads; 26593f93522fSJason Wang 26603f93522fSJason Wang if (!vi->guest_offloads) 26613f93522fSJason Wang return 0; 26623f93522fSJason Wang 26633f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 26643f93522fSJason Wang } 26653f93522fSJason Wang 26669861ce03SJakub Kicinski static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 26679861ce03SJakub Kicinski struct netlink_ext_ack *extack) 2668f600b690SJohn Fastabend { 2669f600b690SJohn Fastabend unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr); 2670f600b690SJohn Fastabend struct virtnet_info *vi = netdev_priv(dev); 2671f600b690SJohn Fastabend struct bpf_prog *old_prog; 2672017b29c3SJason Wang u16 xdp_qp = 0, curr_qp; 2673672aafd5SJohn Fastabend int i, err; 2674f600b690SJohn Fastabend 26753f93522fSJason Wang if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 26763f93522fSJason Wang && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 267792502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 267892502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 267918ba58e1SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 268018ba58e1SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) { 2681dbcf24d1SJason Wang NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); 2682f600b690SJohn Fastabend return -EOPNOTSUPP; 2683f600b690SJohn Fastabend } 2684f600b690SJohn Fastabend 2685f600b690SJohn Fastabend if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 26864d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 2687f600b690SJohn Fastabend return -EINVAL; 2688f600b690SJohn Fastabend } 2689f600b690SJohn Fastabend 2690f600b690SJohn Fastabend if (dev->mtu > max_sz) { 26914d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP"); 2692f600b690SJohn Fastabend netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz); 2693f600b690SJohn Fastabend return -EINVAL; 2694f600b690SJohn Fastabend } 2695f600b690SJohn Fastabend 2696672aafd5SJohn Fastabend curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 2697672aafd5SJohn Fastabend if (prog) 2698672aafd5SJohn Fastabend xdp_qp = nr_cpu_ids; 2699672aafd5SJohn Fastabend 2700672aafd5SJohn Fastabend /* XDP requires extra queues for XDP_TX */ 2701672aafd5SJohn Fastabend if (curr_qp + xdp_qp > vi->max_queue_pairs) { 27029ce4e3d6SXuan Zhuo netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n", 2703672aafd5SJohn Fastabend curr_qp + xdp_qp, vi->max_queue_pairs); 270497c2c69eSXuan Zhuo xdp_qp = 0; 2705672aafd5SJohn Fastabend } 2706672aafd5SJohn Fastabend 270703aa6d34SToshiaki Makita old_prog = rtnl_dereference(vi->rq[0].xdp_prog); 270803aa6d34SToshiaki Makita if (!prog && !old_prog) 270903aa6d34SToshiaki Makita return 0; 271003aa6d34SToshiaki Makita 271185192dbfSAndrii Nakryiko if (prog) 271285192dbfSAndrii Nakryiko bpf_prog_add(prog, vi->max_queue_pairs - 1); 27132de2f7f4SJohn Fastabend 27144941d472SJason Wang /* Make sure NAPI is not using any XDP TX queues for RX. */ 2715534da5e8SToshiaki Makita if (netif_running(dev)) { 2716534da5e8SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 27174941d472SJason Wang napi_disable(&vi->rq[i].napi); 2718534da5e8SToshiaki Makita virtnet_napi_tx_disable(&vi->sq[i].napi); 2719534da5e8SToshiaki Makita } 2720534da5e8SToshiaki Makita } 27212de2f7f4SJohn Fastabend 272203aa6d34SToshiaki Makita if (!prog) { 272303aa6d34SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 272403aa6d34SToshiaki Makita rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 272503aa6d34SToshiaki Makita if (i == 0) 272603aa6d34SToshiaki Makita virtnet_restore_guest_offloads(vi); 272703aa6d34SToshiaki Makita } 272803aa6d34SToshiaki Makita synchronize_net(); 272903aa6d34SToshiaki Makita } 273003aa6d34SToshiaki Makita 27314941d472SJason Wang err = _virtnet_set_queues(vi, curr_qp + xdp_qp); 27324941d472SJason Wang if (err) 27334941d472SJason Wang goto err; 2734188313c1SToshiaki Makita netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 27354941d472SJason Wang vi->xdp_queue_pairs = xdp_qp; 2736f600b690SJohn Fastabend 273703aa6d34SToshiaki Makita if (prog) { 273897c2c69eSXuan Zhuo vi->xdp_enabled = true; 2739f600b690SJohn Fastabend for (i = 0; i < vi->max_queue_pairs; i++) { 2740f600b690SJohn Fastabend rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 274103aa6d34SToshiaki Makita if (i == 0 && !old_prog) 27423f93522fSJason Wang virtnet_clear_guest_offloads(vi); 27433f93522fSJason Wang } 274497c2c69eSXuan Zhuo } else { 274597c2c69eSXuan Zhuo vi->xdp_enabled = false; 274603aa6d34SToshiaki Makita } 274703aa6d34SToshiaki Makita 274803aa6d34SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 2749f600b690SJohn Fastabend if (old_prog) 2750f600b690SJohn Fastabend bpf_prog_put(old_prog); 2751534da5e8SToshiaki Makita if (netif_running(dev)) { 27524941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2753534da5e8SToshiaki Makita virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2754534da5e8SToshiaki Makita &vi->sq[i].napi); 2755534da5e8SToshiaki Makita } 2756f600b690SJohn Fastabend } 2757f600b690SJohn Fastabend 2758f600b690SJohn Fastabend return 0; 27592de2f7f4SJohn Fastabend 27604941d472SJason Wang err: 276103aa6d34SToshiaki Makita if (!prog) { 276203aa6d34SToshiaki Makita virtnet_clear_guest_offloads(vi); 27634941d472SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 276403aa6d34SToshiaki Makita rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); 276503aa6d34SToshiaki Makita } 276603aa6d34SToshiaki Makita 27678be4d9a4SToshiaki Makita if (netif_running(dev)) { 2768534da5e8SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 27694941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2770534da5e8SToshiaki Makita virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2771534da5e8SToshiaki Makita &vi->sq[i].napi); 2772534da5e8SToshiaki Makita } 27738be4d9a4SToshiaki Makita } 27742de2f7f4SJohn Fastabend if (prog) 27752de2f7f4SJohn Fastabend bpf_prog_sub(prog, vi->max_queue_pairs - 1); 27762de2f7f4SJohn Fastabend return err; 2777f600b690SJohn Fastabend } 2778f600b690SJohn Fastabend 2779f4e63525SJakub Kicinski static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2780f600b690SJohn Fastabend { 2781f600b690SJohn Fastabend switch (xdp->command) { 2782f600b690SJohn Fastabend case XDP_SETUP_PROG: 27839861ce03SJakub Kicinski return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 2784f600b690SJohn Fastabend default: 2785f600b690SJohn Fastabend return -EINVAL; 2786f600b690SJohn Fastabend } 2787f600b690SJohn Fastabend } 2788f600b690SJohn Fastabend 2789ba5e4426SSridhar Samudrala static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, 2790ba5e4426SSridhar Samudrala size_t len) 2791ba5e4426SSridhar Samudrala { 2792ba5e4426SSridhar Samudrala struct virtnet_info *vi = netdev_priv(dev); 2793ba5e4426SSridhar Samudrala int ret; 2794ba5e4426SSridhar Samudrala 2795ba5e4426SSridhar Samudrala if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 2796ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 2797ba5e4426SSridhar Samudrala 2798ba5e4426SSridhar Samudrala ret = snprintf(buf, len, "sby"); 2799ba5e4426SSridhar Samudrala if (ret >= len) 2800ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 2801ba5e4426SSridhar Samudrala 2802ba5e4426SSridhar Samudrala return 0; 2803ba5e4426SSridhar Samudrala } 2804ba5e4426SSridhar Samudrala 2805a02e8964SWillem de Bruijn static int virtnet_set_features(struct net_device *dev, 2806a02e8964SWillem de Bruijn netdev_features_t features) 2807a02e8964SWillem de Bruijn { 2808a02e8964SWillem de Bruijn struct virtnet_info *vi = netdev_priv(dev); 2809cf8691cbSMichael S. Tsirkin u64 offloads; 2810a02e8964SWillem de Bruijn int err; 2811a02e8964SWillem de Bruijn 2812dbcf24d1SJason Wang if ((dev->features ^ features) & NETIF_F_GRO_HW) { 281397c2c69eSXuan Zhuo if (vi->xdp_enabled) 2814a02e8964SWillem de Bruijn return -EBUSY; 2815a02e8964SWillem de Bruijn 2816dbcf24d1SJason Wang if (features & NETIF_F_GRO_HW) 2817cf8691cbSMichael S. Tsirkin offloads = vi->guest_offloads_capable; 2818a02e8964SWillem de Bruijn else 2819cf8691cbSMichael S. Tsirkin offloads = vi->guest_offloads_capable & 2820dbcf24d1SJason Wang ~GUEST_OFFLOAD_GRO_HW_MASK; 2821a02e8964SWillem de Bruijn 2822a02e8964SWillem de Bruijn err = virtnet_set_guest_offloads(vi, offloads); 2823a02e8964SWillem de Bruijn if (err) 2824a02e8964SWillem de Bruijn return err; 28253618ad2aSTonghao Zhang vi->guest_offloads = offloads; 2826cf8691cbSMichael S. Tsirkin } 2827cf8691cbSMichael S. Tsirkin 2828*c7114b12SAndrew Melnychenko if ((dev->features ^ features) & NETIF_F_RXHASH) { 2829*c7114b12SAndrew Melnychenko if (features & NETIF_F_RXHASH) 2830*c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_types = vi->rss_hash_types_supported; 2831*c7114b12SAndrew Melnychenko else 2832*c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE; 2833*c7114b12SAndrew Melnychenko 2834*c7114b12SAndrew Melnychenko if (!virtnet_commit_rss_command(vi)) 2835*c7114b12SAndrew Melnychenko return -EINVAL; 2836*c7114b12SAndrew Melnychenko } 2837*c7114b12SAndrew Melnychenko 2838a02e8964SWillem de Bruijn return 0; 2839a02e8964SWillem de Bruijn } 2840a02e8964SWillem de Bruijn 2841a520794bSTony Lu static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) 2842a520794bSTony Lu { 2843a520794bSTony Lu struct virtnet_info *priv = netdev_priv(dev); 2844a520794bSTony Lu struct send_queue *sq = &priv->sq[txqueue]; 2845a520794bSTony Lu struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); 2846a520794bSTony Lu 2847a520794bSTony Lu u64_stats_update_begin(&sq->stats.syncp); 2848a520794bSTony Lu sq->stats.tx_timeouts++; 2849a520794bSTony Lu u64_stats_update_end(&sq->stats.syncp); 2850a520794bSTony Lu 2851a520794bSTony Lu netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", 2852a520794bSTony Lu txqueue, sq->name, sq->vq->index, sq->vq->name, 28535337824fSEric Dumazet jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); 2854a520794bSTony Lu } 2855a520794bSTony Lu 285676288b4eSStephen Hemminger static const struct net_device_ops virtnet_netdev = { 285776288b4eSStephen Hemminger .ndo_open = virtnet_open, 285876288b4eSStephen Hemminger .ndo_stop = virtnet_close, 285976288b4eSStephen Hemminger .ndo_start_xmit = start_xmit, 286076288b4eSStephen Hemminger .ndo_validate_addr = eth_validate_addr, 28619c46f6d4SAlex Williamson .ndo_set_mac_address = virtnet_set_mac_address, 28622af7698eSAlex Williamson .ndo_set_rx_mode = virtnet_set_rx_mode, 28633fa2a1dfSstephen hemminger .ndo_get_stats64 = virtnet_stats, 28641824a989SAlex Williamson .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 28651824a989SAlex Williamson .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 2866f4e63525SJakub Kicinski .ndo_bpf = virtnet_xdp, 2867186b3c99SJason Wang .ndo_xdp_xmit = virtnet_xdp_xmit, 28682836b4f2SVlad Yasevich .ndo_features_check = passthru_features_check, 2869ba5e4426SSridhar Samudrala .ndo_get_phys_port_name = virtnet_get_phys_port_name, 2870a02e8964SWillem de Bruijn .ndo_set_features = virtnet_set_features, 2871a520794bSTony Lu .ndo_tx_timeout = virtnet_tx_timeout, 287276288b4eSStephen Hemminger }; 287376288b4eSStephen Hemminger 2874586d17c5SJason Wang static void virtnet_config_changed_work(struct work_struct *work) 28759f4d26d0SMark McLoughlin { 2876586d17c5SJason Wang struct virtnet_info *vi = 2877586d17c5SJason Wang container_of(work, struct virtnet_info, config_work); 28789f4d26d0SMark McLoughlin u16 v; 28799f4d26d0SMark McLoughlin 2880855e0c52SRusty Russell if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 2881855e0c52SRusty Russell struct virtio_net_config, status, &v) < 0) 2882507613bfSMichael S. Tsirkin return; 2883586d17c5SJason Wang 2884586d17c5SJason Wang if (v & VIRTIO_NET_S_ANNOUNCE) { 2885ee89bab1SAmerigo Wang netdev_notify_peers(vi->dev); 2886586d17c5SJason Wang virtnet_ack_link_announce(vi); 2887586d17c5SJason Wang } 28889f4d26d0SMark McLoughlin 28899f4d26d0SMark McLoughlin /* Ignore unknown (future) status bits */ 28909f4d26d0SMark McLoughlin v &= VIRTIO_NET_S_LINK_UP; 28919f4d26d0SMark McLoughlin 28929f4d26d0SMark McLoughlin if (vi->status == v) 2893507613bfSMichael S. Tsirkin return; 28949f4d26d0SMark McLoughlin 28959f4d26d0SMark McLoughlin vi->status = v; 28969f4d26d0SMark McLoughlin 28979f4d26d0SMark McLoughlin if (vi->status & VIRTIO_NET_S_LINK_UP) { 2898faa9b39fSJason Baron virtnet_update_settings(vi); 28999f4d26d0SMark McLoughlin netif_carrier_on(vi->dev); 2900986a4f4dSJason Wang netif_tx_wake_all_queues(vi->dev); 29019f4d26d0SMark McLoughlin } else { 29029f4d26d0SMark McLoughlin netif_carrier_off(vi->dev); 2903986a4f4dSJason Wang netif_tx_stop_all_queues(vi->dev); 29049f4d26d0SMark McLoughlin } 29059f4d26d0SMark McLoughlin } 29069f4d26d0SMark McLoughlin 29079f4d26d0SMark McLoughlin static void virtnet_config_changed(struct virtio_device *vdev) 29089f4d26d0SMark McLoughlin { 29099f4d26d0SMark McLoughlin struct virtnet_info *vi = vdev->priv; 29109f4d26d0SMark McLoughlin 29113b07e9caSTejun Heo schedule_work(&vi->config_work); 29129f4d26d0SMark McLoughlin } 29139f4d26d0SMark McLoughlin 2914986a4f4dSJason Wang static void virtnet_free_queues(struct virtnet_info *vi) 2915986a4f4dSJason Wang { 2916d4fb84eeSAndrey Vagin int i; 2917d4fb84eeSAndrey Vagin 2918ab3971b1SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 29195198d545SJakub Kicinski __netif_napi_del(&vi->rq[i].napi); 29205198d545SJakub Kicinski __netif_napi_del(&vi->sq[i].napi); 2921ab3971b1SJason Wang } 2922d4fb84eeSAndrey Vagin 29235198d545SJakub Kicinski /* We called __netif_napi_del(), 2924963abe5cSEric Dumazet * we need to respect an RCU grace period before freeing vi->rq 2925963abe5cSEric Dumazet */ 2926963abe5cSEric Dumazet synchronize_net(); 2927963abe5cSEric Dumazet 2928986a4f4dSJason Wang kfree(vi->rq); 2929986a4f4dSJason Wang kfree(vi->sq); 293012e57169SMichael S. Tsirkin kfree(vi->ctrl); 2931986a4f4dSJason Wang } 2932986a4f4dSJason Wang 293347315329SJohn Fastabend static void _free_receive_bufs(struct virtnet_info *vi) 2934986a4f4dSJason Wang { 2935f600b690SJohn Fastabend struct bpf_prog *old_prog; 2936986a4f4dSJason Wang int i; 2937986a4f4dSJason Wang 2938986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2939986a4f4dSJason Wang while (vi->rq[i].pages) 2940986a4f4dSJason Wang __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 2941f600b690SJohn Fastabend 2942f600b690SJohn Fastabend old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 2943f600b690SJohn Fastabend RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 2944f600b690SJohn Fastabend if (old_prog) 2945f600b690SJohn Fastabend bpf_prog_put(old_prog); 2946986a4f4dSJason Wang } 294747315329SJohn Fastabend } 294847315329SJohn Fastabend 294947315329SJohn Fastabend static void free_receive_bufs(struct virtnet_info *vi) 295047315329SJohn Fastabend { 295147315329SJohn Fastabend rtnl_lock(); 295247315329SJohn Fastabend _free_receive_bufs(vi); 2953f600b690SJohn Fastabend rtnl_unlock(); 2954986a4f4dSJason Wang } 2955986a4f4dSJason Wang 2956fb51879dSMichael Dalton static void free_receive_page_frags(struct virtnet_info *vi) 2957fb51879dSMichael Dalton { 2958fb51879dSMichael Dalton int i; 2959fb51879dSMichael Dalton for (i = 0; i < vi->max_queue_pairs; i++) 2960fb51879dSMichael Dalton if (vi->rq[i].alloc_frag.page) 2961fb51879dSMichael Dalton put_page(vi->rq[i].alloc_frag.page); 2962fb51879dSMichael Dalton } 2963fb51879dSMichael Dalton 2964986a4f4dSJason Wang static void free_unused_bufs(struct virtnet_info *vi) 2965986a4f4dSJason Wang { 2966986a4f4dSJason Wang void *buf; 2967986a4f4dSJason Wang int i; 2968986a4f4dSJason Wang 2969986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2970986a4f4dSJason Wang struct virtqueue *vq = vi->sq[i].vq; 297156434a01SJohn Fastabend while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { 29725050471dSToshiaki Makita if (!is_xdp_frame(buf)) 2973986a4f4dSJason Wang dev_kfree_skb(buf); 297456434a01SJohn Fastabend else 29755050471dSToshiaki Makita xdp_return_frame(ptr_to_xdp(buf)); 297656434a01SJohn Fastabend } 2977986a4f4dSJason Wang } 2978986a4f4dSJason Wang 2979986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2980986a4f4dSJason Wang struct virtqueue *vq = vi->rq[i].vq; 2981986a4f4dSJason Wang 2982986a4f4dSJason Wang while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { 2983ab7db917SMichael Dalton if (vi->mergeable_rx_bufs) { 2984680557cfSMichael S. Tsirkin put_page(virt_to_head_page(buf)); 2985ab7db917SMichael Dalton } else if (vi->big_packets) { 2986fa9fac17SAndrey Vagin give_pages(&vi->rq[i], buf); 2987ab7db917SMichael Dalton } else { 2988f6b10209SJason Wang put_page(virt_to_head_page(buf)); 2989986a4f4dSJason Wang } 2990986a4f4dSJason Wang } 2991986a4f4dSJason Wang } 2992ab7db917SMichael Dalton } 2993986a4f4dSJason Wang 2994e9d7417bSJason Wang static void virtnet_del_vqs(struct virtnet_info *vi) 2995e9d7417bSJason Wang { 2996e9d7417bSJason Wang struct virtio_device *vdev = vi->vdev; 2997e9d7417bSJason Wang 2998310974faSPeter Xu virtnet_clean_affinity(vi); 2999986a4f4dSJason Wang 3000e9d7417bSJason Wang vdev->config->del_vqs(vdev); 3001986a4f4dSJason Wang 3002986a4f4dSJason Wang virtnet_free_queues(vi); 3003986a4f4dSJason Wang } 3004986a4f4dSJason Wang 3005d85b758fSMichael S. Tsirkin /* How large should a single buffer be so a queue full of these can fit at 3006d85b758fSMichael S. Tsirkin * least one full packet? 3007d85b758fSMichael S. Tsirkin * Logic below assumes the mergeable buffer header is used. 3008d85b758fSMichael S. Tsirkin */ 3009d85b758fSMichael S. Tsirkin static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 3010d85b758fSMichael S. Tsirkin { 3011c1ddc42dSAndrew Melnychenko const unsigned int hdr_len = vi->hdr_len; 3012d85b758fSMichael S. Tsirkin unsigned int rq_size = virtqueue_get_vring_size(vq); 3013d85b758fSMichael S. Tsirkin unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 3014d85b758fSMichael S. Tsirkin unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 3015d85b758fSMichael S. Tsirkin unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 3016d85b758fSMichael S. Tsirkin 3017f0c3192cSMichael S. Tsirkin return max(max(min_buf_len, hdr_len) - hdr_len, 3018f0c3192cSMichael S. Tsirkin (unsigned int)GOOD_PACKET_LEN); 3019d85b758fSMichael S. Tsirkin } 3020d85b758fSMichael S. Tsirkin 3021986a4f4dSJason Wang static int virtnet_find_vqs(struct virtnet_info *vi) 3022986a4f4dSJason Wang { 3023986a4f4dSJason Wang vq_callback_t **callbacks; 3024986a4f4dSJason Wang struct virtqueue **vqs; 3025986a4f4dSJason Wang int ret = -ENOMEM; 3026986a4f4dSJason Wang int i, total_vqs; 3027986a4f4dSJason Wang const char **names; 3028d45b897bSMichael S. Tsirkin bool *ctx; 3029986a4f4dSJason Wang 3030986a4f4dSJason Wang /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 3031986a4f4dSJason Wang * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 3032986a4f4dSJason Wang * possible control vq. 3033986a4f4dSJason Wang */ 3034986a4f4dSJason Wang total_vqs = vi->max_queue_pairs * 2 + 3035986a4f4dSJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 3036986a4f4dSJason Wang 3037986a4f4dSJason Wang /* Allocate space for find_vqs parameters */ 30386396bb22SKees Cook vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); 3039986a4f4dSJason Wang if (!vqs) 3040986a4f4dSJason Wang goto err_vq; 30416da2ec56SKees Cook callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL); 3042986a4f4dSJason Wang if (!callbacks) 3043986a4f4dSJason Wang goto err_callback; 30446da2ec56SKees Cook names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL); 3045986a4f4dSJason Wang if (!names) 3046986a4f4dSJason Wang goto err_names; 3047192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 30486396bb22SKees Cook ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); 3049d45b897bSMichael S. Tsirkin if (!ctx) 3050d45b897bSMichael S. Tsirkin goto err_ctx; 3051d45b897bSMichael S. Tsirkin } else { 3052d45b897bSMichael S. Tsirkin ctx = NULL; 3053d45b897bSMichael S. Tsirkin } 3054986a4f4dSJason Wang 3055986a4f4dSJason Wang /* Parameters for control virtqueue, if any */ 3056986a4f4dSJason Wang if (vi->has_cvq) { 3057986a4f4dSJason Wang callbacks[total_vqs - 1] = NULL; 3058986a4f4dSJason Wang names[total_vqs - 1] = "control"; 3059986a4f4dSJason Wang } 3060986a4f4dSJason Wang 3061986a4f4dSJason Wang /* Allocate/initialize parameters for send/receive virtqueues */ 3062986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 3063986a4f4dSJason Wang callbacks[rxq2vq(i)] = skb_recv_done; 3064986a4f4dSJason Wang callbacks[txq2vq(i)] = skb_xmit_done; 3065986a4f4dSJason Wang sprintf(vi->rq[i].name, "input.%d", i); 3066986a4f4dSJason Wang sprintf(vi->sq[i].name, "output.%d", i); 3067986a4f4dSJason Wang names[rxq2vq(i)] = vi->rq[i].name; 3068986a4f4dSJason Wang names[txq2vq(i)] = vi->sq[i].name; 3069d45b897bSMichael S. Tsirkin if (ctx) 3070d45b897bSMichael S. Tsirkin ctx[rxq2vq(i)] = true; 3071986a4f4dSJason Wang } 3072986a4f4dSJason Wang 3073a2f7dc00SXianting Tian ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks, 3074d45b897bSMichael S. Tsirkin names, ctx, NULL); 3075986a4f4dSJason Wang if (ret) 3076986a4f4dSJason Wang goto err_find; 3077986a4f4dSJason Wang 3078986a4f4dSJason Wang if (vi->has_cvq) { 3079986a4f4dSJason Wang vi->cvq = vqs[total_vqs - 1]; 3080986a4f4dSJason Wang if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 3081f646968fSPatrick McHardy vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 3082986a4f4dSJason Wang } 3083986a4f4dSJason Wang 3084986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 3085986a4f4dSJason Wang vi->rq[i].vq = vqs[rxq2vq(i)]; 3086d85b758fSMichael S. Tsirkin vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 3087986a4f4dSJason Wang vi->sq[i].vq = vqs[txq2vq(i)]; 3088986a4f4dSJason Wang } 3089986a4f4dSJason Wang 30902fa3c8a8STonghao Zhang /* run here: ret == 0. */ 3091986a4f4dSJason Wang 3092986a4f4dSJason Wang 3093986a4f4dSJason Wang err_find: 3094d45b897bSMichael S. Tsirkin kfree(ctx); 3095d45b897bSMichael S. Tsirkin err_ctx: 3096986a4f4dSJason Wang kfree(names); 3097986a4f4dSJason Wang err_names: 3098986a4f4dSJason Wang kfree(callbacks); 3099986a4f4dSJason Wang err_callback: 3100986a4f4dSJason Wang kfree(vqs); 3101986a4f4dSJason Wang err_vq: 3102986a4f4dSJason Wang return ret; 3103986a4f4dSJason Wang } 3104986a4f4dSJason Wang 3105986a4f4dSJason Wang static int virtnet_alloc_queues(struct virtnet_info *vi) 3106986a4f4dSJason Wang { 3107986a4f4dSJason Wang int i; 3108986a4f4dSJason Wang 3109122b84a1SMax Gurtovoy if (vi->has_cvq) { 311012e57169SMichael S. Tsirkin vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 311112e57169SMichael S. Tsirkin if (!vi->ctrl) 311212e57169SMichael S. Tsirkin goto err_ctrl; 3113122b84a1SMax Gurtovoy } else { 3114122b84a1SMax Gurtovoy vi->ctrl = NULL; 3115122b84a1SMax Gurtovoy } 31166396bb22SKees Cook vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL); 3117986a4f4dSJason Wang if (!vi->sq) 3118986a4f4dSJason Wang goto err_sq; 31196396bb22SKees Cook vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL); 3120008d4278SAmerigo Wang if (!vi->rq) 3121986a4f4dSJason Wang goto err_rq; 3122986a4f4dSJason Wang 3123986a4f4dSJason Wang INIT_DELAYED_WORK(&vi->refill, refill_work); 3124986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 3125986a4f4dSJason Wang vi->rq[i].pages = NULL; 3126986a4f4dSJason Wang netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll, 3127986a4f4dSJason Wang napi_weight); 31281d11e732SWillem de Bruijn netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx, 3129b92f1e67SWillem de Bruijn napi_tx ? napi_weight : 0); 3130986a4f4dSJason Wang 3131986a4f4dSJason Wang sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 31325377d758SJohannes Berg ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 3133986a4f4dSJason Wang sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 3134d7dfc5cfSToshiaki Makita 3135d7dfc5cfSToshiaki Makita u64_stats_init(&vi->rq[i].stats.syncp); 3136d7dfc5cfSToshiaki Makita u64_stats_init(&vi->sq[i].stats.syncp); 3137986a4f4dSJason Wang } 3138986a4f4dSJason Wang 3139986a4f4dSJason Wang return 0; 3140986a4f4dSJason Wang 3141986a4f4dSJason Wang err_rq: 3142986a4f4dSJason Wang kfree(vi->sq); 3143986a4f4dSJason Wang err_sq: 314412e57169SMichael S. Tsirkin kfree(vi->ctrl); 314512e57169SMichael S. Tsirkin err_ctrl: 3146986a4f4dSJason Wang return -ENOMEM; 3147e9d7417bSJason Wang } 3148e9d7417bSJason Wang 31493f9c10b0SAmit Shah static int init_vqs(struct virtnet_info *vi) 31503f9c10b0SAmit Shah { 3151986a4f4dSJason Wang int ret; 31523f9c10b0SAmit Shah 3153986a4f4dSJason Wang /* Allocate send & receive queues */ 3154986a4f4dSJason Wang ret = virtnet_alloc_queues(vi); 3155986a4f4dSJason Wang if (ret) 3156986a4f4dSJason Wang goto err; 31573f9c10b0SAmit Shah 3158986a4f4dSJason Wang ret = virtnet_find_vqs(vi); 3159986a4f4dSJason Wang if (ret) 3160986a4f4dSJason Wang goto err_free; 31613f9c10b0SAmit Shah 3162a0d1d0f4SSebastian Andrzej Siewior cpus_read_lock(); 31638898c21cSWanlong Gao virtnet_set_affinity(vi); 3164a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 316547be2479SWanlong Gao 31663f9c10b0SAmit Shah return 0; 3167986a4f4dSJason Wang 3168986a4f4dSJason Wang err_free: 3169986a4f4dSJason Wang virtnet_free_queues(vi); 3170986a4f4dSJason Wang err: 3171986a4f4dSJason Wang return ret; 31723f9c10b0SAmit Shah } 31733f9c10b0SAmit Shah 3174fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 3175fbf28d78SMichael Dalton static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 3176718ad681Sstephen hemminger char *buf) 3177fbf28d78SMichael Dalton { 3178fbf28d78SMichael Dalton struct virtnet_info *vi = netdev_priv(queue->dev); 3179fbf28d78SMichael Dalton unsigned int queue_index = get_netdev_rx_queue_index(queue); 31803cc81a9aSJason Wang unsigned int headroom = virtnet_get_headroom(vi); 31813cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 31825377d758SJohannes Berg struct ewma_pkt_len *avg; 3183fbf28d78SMichael Dalton 3184fbf28d78SMichael Dalton BUG_ON(queue_index >= vi->max_queue_pairs); 3185fbf28d78SMichael Dalton avg = &vi->rq[queue_index].mrg_avg_pkt_len; 3186d85b758fSMichael S. Tsirkin return sprintf(buf, "%u\n", 31873cc81a9aSJason Wang get_mergeable_buf_len(&vi->rq[queue_index], avg, 31883cc81a9aSJason Wang SKB_DATA_ALIGN(headroom + tailroom))); 3189fbf28d78SMichael Dalton } 3190fbf28d78SMichael Dalton 3191fbf28d78SMichael Dalton static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 3192fbf28d78SMichael Dalton __ATTR_RO(mergeable_rx_buffer_size); 3193fbf28d78SMichael Dalton 3194fbf28d78SMichael Dalton static struct attribute *virtio_net_mrg_rx_attrs[] = { 3195fbf28d78SMichael Dalton &mergeable_rx_buffer_size_attribute.attr, 3196fbf28d78SMichael Dalton NULL 3197fbf28d78SMichael Dalton }; 3198fbf28d78SMichael Dalton 3199fbf28d78SMichael Dalton static const struct attribute_group virtio_net_mrg_rx_group = { 3200fbf28d78SMichael Dalton .name = "virtio_net", 3201fbf28d78SMichael Dalton .attrs = virtio_net_mrg_rx_attrs 3202fbf28d78SMichael Dalton }; 3203fbf28d78SMichael Dalton #endif 3204fbf28d78SMichael Dalton 3205892d6eb1SJason Wang static bool virtnet_fail_on_feature(struct virtio_device *vdev, 3206892d6eb1SJason Wang unsigned int fbit, 3207892d6eb1SJason Wang const char *fname, const char *dname) 3208892d6eb1SJason Wang { 3209892d6eb1SJason Wang if (!virtio_has_feature(vdev, fbit)) 3210892d6eb1SJason Wang return false; 3211892d6eb1SJason Wang 3212892d6eb1SJason Wang dev_err(&vdev->dev, "device advertises feature %s but not %s", 3213892d6eb1SJason Wang fname, dname); 3214892d6eb1SJason Wang 3215892d6eb1SJason Wang return true; 3216892d6eb1SJason Wang } 3217892d6eb1SJason Wang 3218892d6eb1SJason Wang #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 3219892d6eb1SJason Wang virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 3220892d6eb1SJason Wang 3221892d6eb1SJason Wang static bool virtnet_validate_features(struct virtio_device *vdev) 3222892d6eb1SJason Wang { 3223892d6eb1SJason Wang if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 3224892d6eb1SJason Wang (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 3225892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 3226892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 3227892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 3228892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 3229892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 3230892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 3231892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 3232*c7114b12SAndrew Melnychenko "VIRTIO_NET_F_CTRL_VQ") || 3233*c7114b12SAndrew Melnychenko VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, 3234892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ"))) { 3235892d6eb1SJason Wang return false; 3236892d6eb1SJason Wang } 3237892d6eb1SJason Wang 3238892d6eb1SJason Wang return true; 3239892d6eb1SJason Wang } 3240892d6eb1SJason Wang 3241d0c2c997SJarod Wilson #define MIN_MTU ETH_MIN_MTU 3242d0c2c997SJarod Wilson #define MAX_MTU ETH_MAX_MTU 3243d0c2c997SJarod Wilson 3244fe36cbe0SMichael S. Tsirkin static int virtnet_validate(struct virtio_device *vdev) 3245296f96fcSRusty Russell { 32466ba42248SMichael S. Tsirkin if (!vdev->config->get) { 32476ba42248SMichael S. Tsirkin dev_err(&vdev->dev, "%s failure: config access disabled\n", 32486ba42248SMichael S. Tsirkin __func__); 32496ba42248SMichael S. Tsirkin return -EINVAL; 32506ba42248SMichael S. Tsirkin } 32516ba42248SMichael S. Tsirkin 3252892d6eb1SJason Wang if (!virtnet_validate_features(vdev)) 3253892d6eb1SJason Wang return -EINVAL; 3254892d6eb1SJason Wang 3255fe36cbe0SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 3256fe36cbe0SMichael S. Tsirkin int mtu = virtio_cread16(vdev, 3257fe36cbe0SMichael S. Tsirkin offsetof(struct virtio_net_config, 3258fe36cbe0SMichael S. Tsirkin mtu)); 3259fe36cbe0SMichael S. Tsirkin if (mtu < MIN_MTU) 3260fe36cbe0SMichael S. Tsirkin __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 3261fe36cbe0SMichael S. Tsirkin } 3262fe36cbe0SMichael S. Tsirkin 3263fe36cbe0SMichael S. Tsirkin return 0; 3264fe36cbe0SMichael S. Tsirkin } 3265fe36cbe0SMichael S. Tsirkin 3266fe36cbe0SMichael S. Tsirkin static int virtnet_probe(struct virtio_device *vdev) 3267fe36cbe0SMichael S. Tsirkin { 3268d7dfc5cfSToshiaki Makita int i, err = -ENOMEM; 3269fe36cbe0SMichael S. Tsirkin struct net_device *dev; 3270fe36cbe0SMichael S. Tsirkin struct virtnet_info *vi; 3271fe36cbe0SMichael S. Tsirkin u16 max_queue_pairs; 3272fe36cbe0SMichael S. Tsirkin int mtu; 3273fe36cbe0SMichael S. Tsirkin 3274*c7114b12SAndrew Melnychenko /* Find if host supports multiqueue/rss virtio_net device */ 3275*c7114b12SAndrew Melnychenko max_queue_pairs = 1; 3276*c7114b12SAndrew Melnychenko if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 3277*c7114b12SAndrew Melnychenko max_queue_pairs = 3278*c7114b12SAndrew Melnychenko virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); 3279986a4f4dSJason Wang 3280986a4f4dSJason Wang /* We need at least 2 queue's */ 3281*c7114b12SAndrew Melnychenko if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 3282986a4f4dSJason Wang max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 3283986a4f4dSJason Wang !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 3284986a4f4dSJason Wang max_queue_pairs = 1; 3285296f96fcSRusty Russell 3286296f96fcSRusty Russell /* Allocate ourselves a network device with room for our info */ 3287986a4f4dSJason Wang dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 3288296f96fcSRusty Russell if (!dev) 3289296f96fcSRusty Russell return -ENOMEM; 3290296f96fcSRusty Russell 3291296f96fcSRusty Russell /* Set up network device as normal. */ 3292ab5bd583SXuan Zhuo dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 3293ab5bd583SXuan Zhuo IFF_TX_SKB_NO_LINEAR; 329476288b4eSStephen Hemminger dev->netdev_ops = &virtnet_netdev; 3295296f96fcSRusty Russell dev->features = NETIF_F_HIGHDMA; 32963fa2a1dfSstephen hemminger 32977ad24ea4SWilfried Klaebe dev->ethtool_ops = &virtnet_ethtool_ops; 3298296f96fcSRusty Russell SET_NETDEV_DEV(dev, &vdev->dev); 3299296f96fcSRusty Russell 3300296f96fcSRusty Russell /* Do we support "hardware" checksums? */ 330198e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 3302296f96fcSRusty Russell /* This opens up the world of extra features. */ 330348900cb6SJason Wang dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 330498e778c9SMichał Mirosław if (csum) 330548900cb6SJason Wang dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 330698e778c9SMichał Mirosław 330798e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 3308e078de03SDavid S. Miller dev->hw_features |= NETIF_F_TSO 330934a48579SRusty Russell | NETIF_F_TSO_ECN | NETIF_F_TSO6; 331034a48579SRusty Russell } 33115539ae96SRusty Russell /* Individual feature bits: what can host handle? */ 331298e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 331398e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO; 331498e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 331598e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO6; 331698e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 331798e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO_ECN; 331898e778c9SMichał Mirosław 331941f2f127SJason Wang dev->features |= NETIF_F_GSO_ROBUST; 332041f2f127SJason Wang 332198e778c9SMichał Mirosław if (gso) 3322e078de03SDavid S. Miller dev->features |= dev->hw_features & NETIF_F_ALL_TSO; 332398e778c9SMichał Mirosław /* (!csum && gso) case will be fixed by register_netdev() */ 3324296f96fcSRusty Russell } 33254f49129bSThomas Huth if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM)) 33264f49129bSThomas Huth dev->features |= NETIF_F_RXCSUM; 3327a02e8964SWillem de Bruijn if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 3328a02e8964SWillem de Bruijn virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) 3329dbcf24d1SJason Wang dev->features |= NETIF_F_GRO_HW; 3330cf8691cbSMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) 3331dbcf24d1SJason Wang dev->hw_features |= NETIF_F_GRO_HW; 3332296f96fcSRusty Russell 33334fda8302SJason Wang dev->vlan_features = dev->features; 33344fda8302SJason Wang 3335d0c2c997SJarod Wilson /* MTU range: 68 - 65535 */ 3336d0c2c997SJarod Wilson dev->min_mtu = MIN_MTU; 3337d0c2c997SJarod Wilson dev->max_mtu = MAX_MTU; 3338d0c2c997SJarod Wilson 3339296f96fcSRusty Russell /* Configuration may specify what MAC to use. Otherwise random. */ 3340f2edaa4aSJakub Kicinski if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 3341f2edaa4aSJakub Kicinski u8 addr[ETH_ALEN]; 3342f2edaa4aSJakub Kicinski 3343855e0c52SRusty Russell virtio_cread_bytes(vdev, 3344a586d4f6SRusty Russell offsetof(struct virtio_net_config, mac), 3345f2edaa4aSJakub Kicinski addr, ETH_ALEN); 3346f2edaa4aSJakub Kicinski eth_hw_addr_set(dev, addr); 3347f2edaa4aSJakub Kicinski } else { 3348f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 3349f2edaa4aSJakub Kicinski } 3350296f96fcSRusty Russell 3351296f96fcSRusty Russell /* Set up our device-specific information */ 3352296f96fcSRusty Russell vi = netdev_priv(dev); 3353296f96fcSRusty Russell vi->dev = dev; 3354296f96fcSRusty Russell vi->vdev = vdev; 3355d9d5dcc8SChristian Borntraeger vdev->priv = vi; 3356827da44cSJohn Stultz 3357586d17c5SJason Wang INIT_WORK(&vi->config_work, virtnet_config_changed_work); 3358296f96fcSRusty Russell 335997402b96SHerbert Xu /* If we can receive ANY GSO packets, we must allocate large ones. */ 33608e95a202SJoe Perches if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 33618e95a202SJoe Perches virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) || 3362e3e3c423SVlad Yasevich virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) || 3363e3e3c423SVlad Yasevich virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO)) 336497402b96SHerbert Xu vi->big_packets = true; 336597402b96SHerbert Xu 33663f2c31d9SMark McLoughlin if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) 33673f2c31d9SMark McLoughlin vi->mergeable_rx_bufs = true; 33683f2c31d9SMark McLoughlin 3369*c7114b12SAndrew Melnychenko if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) { 3370*c7114b12SAndrew Melnychenko vi->has_rss = true; 3371*c7114b12SAndrew Melnychenko vi->rss_indir_table_size = 3372*c7114b12SAndrew Melnychenko virtio_cread16(vdev, offsetof(struct virtio_net_config, 3373*c7114b12SAndrew Melnychenko rss_max_indirection_table_length)); 3374*c7114b12SAndrew Melnychenko vi->rss_key_size = 3375*c7114b12SAndrew Melnychenko virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); 3376*c7114b12SAndrew Melnychenko 3377*c7114b12SAndrew Melnychenko vi->rss_hash_types_supported = 3378*c7114b12SAndrew Melnychenko virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); 3379*c7114b12SAndrew Melnychenko vi->rss_hash_types_supported &= 3380*c7114b12SAndrew Melnychenko ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX | 3381*c7114b12SAndrew Melnychenko VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 3382*c7114b12SAndrew Melnychenko VIRTIO_NET_RSS_HASH_TYPE_UDP_EX); 3383*c7114b12SAndrew Melnychenko 3384*c7114b12SAndrew Melnychenko dev->hw_features |= NETIF_F_RXHASH; 3385*c7114b12SAndrew Melnychenko } 3386d04302b3SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 3387d04302b3SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 3388012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 3389012873d0SMichael S. Tsirkin else 3390012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr); 3391012873d0SMichael S. Tsirkin 339275993300SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 339375993300SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 3394e7428e95SMichael S. Tsirkin vi->any_header_sg = true; 3395e7428e95SMichael S. Tsirkin 3396986a4f4dSJason Wang if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 3397986a4f4dSJason Wang vi->has_cvq = true; 3398986a4f4dSJason Wang 339914de9d11SAaron Conole if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 340014de9d11SAaron Conole mtu = virtio_cread16(vdev, 340114de9d11SAaron Conole offsetof(struct virtio_net_config, 340214de9d11SAaron Conole mtu)); 340393a205eeSAaron Conole if (mtu < dev->min_mtu) { 3404fe36cbe0SMichael S. Tsirkin /* Should never trigger: MTU was previously validated 3405fe36cbe0SMichael S. Tsirkin * in virtnet_validate. 3406fe36cbe0SMichael S. Tsirkin */ 34077934b481SYuval Shaia dev_err(&vdev->dev, 34087934b481SYuval Shaia "device MTU appears to have changed it is now %d < %d", 34097934b481SYuval Shaia mtu, dev->min_mtu); 3410411ea23aSDan Carpenter err = -EINVAL; 3411d7dfc5cfSToshiaki Makita goto free; 3412fe36cbe0SMichael S. Tsirkin } 3413fe36cbe0SMichael S. Tsirkin 3414d0c2c997SJarod Wilson dev->mtu = mtu; 341593a205eeSAaron Conole dev->max_mtu = mtu; 34162e123b44SMichael S. Tsirkin 34172e123b44SMichael S. Tsirkin /* TODO: size buffers correctly in this case. */ 34182e123b44SMichael S. Tsirkin if (dev->mtu > ETH_DATA_LEN) 34192e123b44SMichael S. Tsirkin vi->big_packets = true; 342014de9d11SAaron Conole } 342114de9d11SAaron Conole 3422012873d0SMichael S. Tsirkin if (vi->any_header_sg) 3423012873d0SMichael S. Tsirkin dev->needed_headroom = vi->hdr_len; 34246ebbc1a6SZhangjie \(HZ\) 342544900010SJason Wang /* Enable multiqueue by default */ 342644900010SJason Wang if (num_online_cpus() >= max_queue_pairs) 342744900010SJason Wang vi->curr_queue_pairs = max_queue_pairs; 342844900010SJason Wang else 342944900010SJason Wang vi->curr_queue_pairs = num_online_cpus(); 3430986a4f4dSJason Wang vi->max_queue_pairs = max_queue_pairs; 3431986a4f4dSJason Wang 3432986a4f4dSJason Wang /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 34333f9c10b0SAmit Shah err = init_vqs(vi); 3434d2a7dddaSMichael S. Tsirkin if (err) 3435d7dfc5cfSToshiaki Makita goto free; 3436d2a7dddaSMichael S. Tsirkin 3437fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 3438fbf28d78SMichael Dalton if (vi->mergeable_rx_bufs) 3439fbf28d78SMichael Dalton dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 3440fbf28d78SMichael Dalton #endif 34410f13b66bSZhi Yong Wu netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 34420f13b66bSZhi Yong Wu netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 3443986a4f4dSJason Wang 344416032be5SNikolay Aleksandrov virtnet_init_settings(dev); 344516032be5SNikolay Aleksandrov 3446ba5e4426SSridhar Samudrala if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 3447ba5e4426SSridhar Samudrala vi->failover = net_failover_create(vi->dev); 34484b8e6ac4SWei Yongjun if (IS_ERR(vi->failover)) { 34494b8e6ac4SWei Yongjun err = PTR_ERR(vi->failover); 3450ba5e4426SSridhar Samudrala goto free_vqs; 3451ba5e4426SSridhar Samudrala } 34524b8e6ac4SWei Yongjun } 3453ba5e4426SSridhar Samudrala 3454*c7114b12SAndrew Melnychenko if (vi->has_rss) 3455*c7114b12SAndrew Melnychenko virtnet_init_default_rss(vi); 3456*c7114b12SAndrew Melnychenko 3457296f96fcSRusty Russell err = register_netdev(dev); 3458296f96fcSRusty Russell if (err) { 3459296f96fcSRusty Russell pr_debug("virtio_net: registering device failed\n"); 3460ba5e4426SSridhar Samudrala goto free_failover; 3461296f96fcSRusty Russell } 3462b3369c1fSRusty Russell 34634baf1e33SMichael S. Tsirkin virtio_device_ready(vdev); 34644baf1e33SMichael S. Tsirkin 34658017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 34668de4b2f3SWanlong Gao if (err) { 34678de4b2f3SWanlong Gao pr_debug("virtio_net: registering cpu notifier failed\n"); 3468f00e35e2Swangyunjian goto free_unregister_netdev; 34698de4b2f3SWanlong Gao } 34708de4b2f3SWanlong Gao 3471a220871bSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 347244900010SJason Wang 3473167c25e4SJason Wang /* Assume link up if device can't report link status, 3474167c25e4SJason Wang otherwise get link status from config. */ 3475167c25e4SJason Wang netif_carrier_off(dev); 3476bda7fab5SJay Vosburgh if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 34773b07e9caSTejun Heo schedule_work(&vi->config_work); 3478167c25e4SJason Wang } else { 3479167c25e4SJason Wang vi->status = VIRTIO_NET_S_LINK_UP; 3480faa9b39fSJason Baron virtnet_update_settings(vi); 34814783256eSPantelis Koukousoulas netif_carrier_on(dev); 3482167c25e4SJason Wang } 34839f4d26d0SMark McLoughlin 34843f93522fSJason Wang for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) 34853f93522fSJason Wang if (virtio_has_feature(vi->vdev, guest_offloads[i])) 34863f93522fSJason Wang set_bit(guest_offloads[i], &vi->guest_offloads); 3487a02e8964SWillem de Bruijn vi->guest_offloads_capable = vi->guest_offloads; 34883f93522fSJason Wang 3489986a4f4dSJason Wang pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 3490986a4f4dSJason Wang dev->name, max_queue_pairs); 3491986a4f4dSJason Wang 3492296f96fcSRusty Russell return 0; 3493296f96fcSRusty Russell 3494f00e35e2Swangyunjian free_unregister_netdev: 3495d9679d00SMichael S. Tsirkin virtio_reset_device(vdev); 349602465555SMichael S. Tsirkin 3497b3369c1fSRusty Russell unregister_netdev(dev); 3498ba5e4426SSridhar Samudrala free_failover: 3499ba5e4426SSridhar Samudrala net_failover_destroy(vi->failover); 3500d2a7dddaSMichael S. Tsirkin free_vqs: 3501986a4f4dSJason Wang cancel_delayed_work_sync(&vi->refill); 3502fb51879dSMichael Dalton free_receive_page_frags(vi); 3503e9d7417bSJason Wang virtnet_del_vqs(vi); 3504296f96fcSRusty Russell free: 3505296f96fcSRusty Russell free_netdev(dev); 3506296f96fcSRusty Russell return err; 3507296f96fcSRusty Russell } 3508296f96fcSRusty Russell 350904486ed0SAmit Shah static void remove_vq_common(struct virtnet_info *vi) 3510296f96fcSRusty Russell { 3511d9679d00SMichael S. Tsirkin virtio_reset_device(vi->vdev); 3512830a8a97SShirley Ma 3513830a8a97SShirley Ma /* Free unused buffers in both send and recv, if any. */ 35149ab86bbcSShirley Ma free_unused_bufs(vi); 3515fb6813f4SRusty Russell 3516986a4f4dSJason Wang free_receive_bufs(vi); 3517d2a7dddaSMichael S. Tsirkin 3518fb51879dSMichael Dalton free_receive_page_frags(vi); 3519fb51879dSMichael Dalton 3520986a4f4dSJason Wang virtnet_del_vqs(vi); 352104486ed0SAmit Shah } 352204486ed0SAmit Shah 35238cc085d6SBill Pemberton static void virtnet_remove(struct virtio_device *vdev) 352404486ed0SAmit Shah { 352504486ed0SAmit Shah struct virtnet_info *vi = vdev->priv; 352604486ed0SAmit Shah 35278017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 35288de4b2f3SWanlong Gao 3529102a2786SMichael S. Tsirkin /* Make sure no work handler is accessing the device. */ 3530102a2786SMichael S. Tsirkin flush_work(&vi->config_work); 3531586d17c5SJason Wang 353204486ed0SAmit Shah unregister_netdev(vi->dev); 353304486ed0SAmit Shah 3534ba5e4426SSridhar Samudrala net_failover_destroy(vi->failover); 3535ba5e4426SSridhar Samudrala 353604486ed0SAmit Shah remove_vq_common(vi); 3537fb6813f4SRusty Russell 353874b2553fSRusty Russell free_netdev(vi->dev); 3539296f96fcSRusty Russell } 3540296f96fcSRusty Russell 354167a75194SArnd Bergmann static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 35420741bcb5SAmit Shah { 35430741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 35440741bcb5SAmit Shah 35458017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 35469fe7bfceSJohn Fastabend virtnet_freeze_down(vdev); 35470741bcb5SAmit Shah remove_vq_common(vi); 35480741bcb5SAmit Shah 35490741bcb5SAmit Shah return 0; 35500741bcb5SAmit Shah } 35510741bcb5SAmit Shah 355267a75194SArnd Bergmann static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 35530741bcb5SAmit Shah { 35540741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 35559fe7bfceSJohn Fastabend int err; 35560741bcb5SAmit Shah 35579fe7bfceSJohn Fastabend err = virtnet_restore_up(vdev); 35580741bcb5SAmit Shah if (err) 35590741bcb5SAmit Shah return err; 3560986a4f4dSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 3561986a4f4dSJason Wang 35628017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 35633f2869caSXie Yongji if (err) { 35643f2869caSXie Yongji virtnet_freeze_down(vdev); 35653f2869caSXie Yongji remove_vq_common(vi); 3566ec9debbdSJason Wang return err; 35673f2869caSXie Yongji } 3568ec9debbdSJason Wang 35690741bcb5SAmit Shah return 0; 35700741bcb5SAmit Shah } 35710741bcb5SAmit Shah 3572296f96fcSRusty Russell static struct virtio_device_id id_table[] = { 3573296f96fcSRusty Russell { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 3574296f96fcSRusty Russell { 0 }, 3575296f96fcSRusty Russell }; 3576296f96fcSRusty Russell 3577f3358507SMichael S. Tsirkin #define VIRTNET_FEATURES \ 3578f3358507SMichael S. Tsirkin VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 3579f3358507SMichael S. Tsirkin VIRTIO_NET_F_MAC, \ 3580f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 3581f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 3582f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 3583f3358507SMichael S. Tsirkin VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 3584f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 3585f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 3586f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_MAC_ADDR, \ 3587faa9b39fSJason Baron VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 3588*c7114b12SAndrew Melnychenko VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ 3589*c7114b12SAndrew Melnychenko VIRTIO_NET_F_RSS 3590f3358507SMichael S. Tsirkin 3591c45a6816SRusty Russell static unsigned int features[] = { 3592f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 3593f3358507SMichael S. Tsirkin }; 3594f3358507SMichael S. Tsirkin 3595f3358507SMichael S. Tsirkin static unsigned int features_legacy[] = { 3596f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 3597f3358507SMichael S. Tsirkin VIRTIO_NET_F_GSO, 3598e7428e95SMichael S. Tsirkin VIRTIO_F_ANY_LAYOUT, 3599c45a6816SRusty Russell }; 3600c45a6816SRusty Russell 360122402529SUwe Kleine-König static struct virtio_driver virtio_net_driver = { 3602c45a6816SRusty Russell .feature_table = features, 3603c45a6816SRusty Russell .feature_table_size = ARRAY_SIZE(features), 3604f3358507SMichael S. Tsirkin .feature_table_legacy = features_legacy, 3605f3358507SMichael S. Tsirkin .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 3606296f96fcSRusty Russell .driver.name = KBUILD_MODNAME, 3607296f96fcSRusty Russell .driver.owner = THIS_MODULE, 3608296f96fcSRusty Russell .id_table = id_table, 3609fe36cbe0SMichael S. Tsirkin .validate = virtnet_validate, 3610296f96fcSRusty Russell .probe = virtnet_probe, 36118cc085d6SBill Pemberton .remove = virtnet_remove, 36129f4d26d0SMark McLoughlin .config_changed = virtnet_config_changed, 361389107000SAaron Lu #ifdef CONFIG_PM_SLEEP 36140741bcb5SAmit Shah .freeze = virtnet_freeze, 36150741bcb5SAmit Shah .restore = virtnet_restore, 36160741bcb5SAmit Shah #endif 3617296f96fcSRusty Russell }; 3618296f96fcSRusty Russell 36198017c279SSebastian Andrzej Siewior static __init int virtio_net_driver_init(void) 36208017c279SSebastian Andrzej Siewior { 36218017c279SSebastian Andrzej Siewior int ret; 36228017c279SSebastian Andrzej Siewior 362373c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 36248017c279SSebastian Andrzej Siewior virtnet_cpu_online, 36258017c279SSebastian Andrzej Siewior virtnet_cpu_down_prep); 36268017c279SSebastian Andrzej Siewior if (ret < 0) 36278017c279SSebastian Andrzej Siewior goto out; 36288017c279SSebastian Andrzej Siewior virtionet_online = ret; 362973c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 36308017c279SSebastian Andrzej Siewior NULL, virtnet_cpu_dead); 36318017c279SSebastian Andrzej Siewior if (ret) 36328017c279SSebastian Andrzej Siewior goto err_dead; 36338017c279SSebastian Andrzej Siewior 36348017c279SSebastian Andrzej Siewior ret = register_virtio_driver(&virtio_net_driver); 36358017c279SSebastian Andrzej Siewior if (ret) 36368017c279SSebastian Andrzej Siewior goto err_virtio; 36378017c279SSebastian Andrzej Siewior return 0; 36388017c279SSebastian Andrzej Siewior err_virtio: 36398017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 36408017c279SSebastian Andrzej Siewior err_dead: 36418017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 36428017c279SSebastian Andrzej Siewior out: 36438017c279SSebastian Andrzej Siewior return ret; 36448017c279SSebastian Andrzej Siewior } 36458017c279SSebastian Andrzej Siewior module_init(virtio_net_driver_init); 36468017c279SSebastian Andrzej Siewior 36478017c279SSebastian Andrzej Siewior static __exit void virtio_net_driver_exit(void) 36488017c279SSebastian Andrzej Siewior { 3649cfa0ebc9SAndrew Jones unregister_virtio_driver(&virtio_net_driver); 36508017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 36518017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 36528017c279SSebastian Andrzej Siewior } 36538017c279SSebastian Andrzej Siewior module_exit(virtio_net_driver_exit); 3654296f96fcSRusty Russell 3655296f96fcSRusty Russell MODULE_DEVICE_TABLE(virtio, id_table); 3656296f96fcSRusty Russell MODULE_DESCRIPTION("Virtio network driver"); 3657296f96fcSRusty Russell MODULE_LICENSE("GPL"); 3658