11ccea77eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 248925e37SRusty Russell /* A network driver using virtio. 3296f96fcSRusty Russell * 4296f96fcSRusty Russell * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5296f96fcSRusty Russell */ 6296f96fcSRusty Russell //#define DEBUG 7296f96fcSRusty Russell #include <linux/netdevice.h> 8296f96fcSRusty Russell #include <linux/etherdevice.h> 9a9ea3fc6SHerbert Xu #include <linux/ethtool.h> 10296f96fcSRusty Russell #include <linux/module.h> 11296f96fcSRusty Russell #include <linux/virtio.h> 12296f96fcSRusty Russell #include <linux/virtio_net.h> 13f600b690SJohn Fastabend #include <linux/bpf.h> 14a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 15296f96fcSRusty Russell #include <linux/scatterlist.h> 16e918085aSAlex Williamson #include <linux/if_vlan.h> 175a0e3ad6STejun Heo #include <linux/slab.h> 188de4b2f3SWanlong Gao #include <linux/cpu.h> 19ab7db917SMichael Dalton #include <linux/average.h> 20186b3c99SJason Wang #include <linux/filter.h> 212ca653d6SCaleb Raitto #include <linux/kernel.h> 22d85b758fSMichael S. Tsirkin #include <net/route.h> 23754b8a21SJesper Dangaard Brouer #include <net/xdp.h> 24ba5e4426SSridhar Samudrala #include <net/net_failover.h> 2549e47a5bSJakub Kicinski #include <net/netdev_rx_queue.h> 26296f96fcSRusty Russell 27d34710e3SAmerigo Wang static int napi_weight = NAPI_POLL_WEIGHT; 286c0cd7c0SDor Laor module_param(napi_weight, int, 0444); 296c0cd7c0SDor Laor 3031c03aefSWillem de Bruijn static bool csum = true, gso = true, napi_tx = true; 3134a48579SRusty Russell module_param(csum, bool, 0444); 3234a48579SRusty Russell module_param(gso, bool, 0444); 33b92f1e67SWillem de Bruijn module_param(napi_tx, bool, 0644); 3434a48579SRusty Russell 35296f96fcSRusty Russell /* FIXME: MTU in config. */ 365061de36SMichael Dalton #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 373f2c31d9SMark McLoughlin #define GOOD_COPY_LEN 128 38296f96fcSRusty Russell 39f6b10209SJason Wang #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 40f6b10209SJason Wang 412de2f7f4SJohn Fastabend /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */ 422de2f7f4SJohn Fastabend #define VIRTIO_XDP_HEADROOM 256 432de2f7f4SJohn Fastabend 442471c75eSJesper Dangaard Brouer /* Separating two types of XDP xmit */ 452471c75eSJesper Dangaard Brouer #define VIRTIO_XDP_TX BIT(0) 462471c75eSJesper Dangaard Brouer #define VIRTIO_XDP_REDIR BIT(1) 472471c75eSJesper Dangaard Brouer 485050471dSToshiaki Makita #define VIRTIO_XDP_FLAG BIT(0) 495050471dSToshiaki Makita 505377d758SJohannes Berg /* RX packet size EWMA. The average packet size is used to determine the packet 515377d758SJohannes Berg * buffer size when refilling RX rings. As the entire RX ring may be refilled 525377d758SJohannes Berg * at once, the weight is chosen so that the EWMA will be insensitive to short- 535377d758SJohannes Berg * term, transient changes in packet size. 54ab7db917SMichael Dalton */ 55eb1e011aSJohannes Berg DECLARE_EWMA(pkt_len, 0, 64) 56ab7db917SMichael Dalton 5766846048SRick Jones #define VIRTNET_DRIVER_VERSION "1.0.0" 582a41f71dSAlex Williamson 597acd4329SColin Ian King static const unsigned long guest_offloads[] = { 607acd4329SColin Ian King VIRTIO_NET_F_GUEST_TSO4, 613f93522fSJason Wang VIRTIO_NET_F_GUEST_TSO6, 623f93522fSJason Wang VIRTIO_NET_F_GUEST_ECN, 63e59ff2c4SJason Wang VIRTIO_NET_F_GUEST_UFO, 64418044e1SAndrew Melnychenko VIRTIO_NET_F_GUEST_CSUM, 65418044e1SAndrew Melnychenko VIRTIO_NET_F_GUEST_USO4, 66be50da3eSJiri Pirko VIRTIO_NET_F_GUEST_USO6, 67be50da3eSJiri Pirko VIRTIO_NET_F_GUEST_HDRLEN 687acd4329SColin Ian King }; 693f93522fSJason Wang 70dbcf24d1SJason Wang #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 711a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 721a03b8a3STonghao Zhang (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 73418044e1SAndrew Melnychenko (1ULL << VIRTIO_NET_F_GUEST_UFO) | \ 74418044e1SAndrew Melnychenko (1ULL << VIRTIO_NET_F_GUEST_USO4) | \ 75418044e1SAndrew Melnychenko (1ULL << VIRTIO_NET_F_GUEST_USO6)) 761a03b8a3STonghao Zhang 77d7dfc5cfSToshiaki Makita struct virtnet_stat_desc { 78d7dfc5cfSToshiaki Makita char desc[ETH_GSTRING_LEN]; 79d7dfc5cfSToshiaki Makita size_t offset; 803fa2a1dfSstephen hemminger }; 813fa2a1dfSstephen hemminger 82d7dfc5cfSToshiaki Makita struct virtnet_sq_stats { 83d7dfc5cfSToshiaki Makita struct u64_stats_sync syncp; 8427debe3eSEric Dumazet u64_stats_t packets; 8527debe3eSEric Dumazet u64_stats_t bytes; 8627debe3eSEric Dumazet u64_stats_t xdp_tx; 8727debe3eSEric Dumazet u64_stats_t xdp_tx_drops; 8827debe3eSEric Dumazet u64_stats_t kicks; 8927debe3eSEric Dumazet u64_stats_t tx_timeouts; 90d7dfc5cfSToshiaki Makita }; 91d7dfc5cfSToshiaki Makita 92d46eeeafSJason Wang struct virtnet_rq_stats { 93d46eeeafSJason Wang struct u64_stats_sync syncp; 9427debe3eSEric Dumazet u64_stats_t packets; 9527debe3eSEric Dumazet u64_stats_t bytes; 9627debe3eSEric Dumazet u64_stats_t drops; 9727debe3eSEric Dumazet u64_stats_t xdp_packets; 9827debe3eSEric Dumazet u64_stats_t xdp_tx; 9927debe3eSEric Dumazet u64_stats_t xdp_redirects; 10027debe3eSEric Dumazet u64_stats_t xdp_drops; 10127debe3eSEric Dumazet u64_stats_t kicks; 102d7dfc5cfSToshiaki Makita }; 103d7dfc5cfSToshiaki Makita 104d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m) 105d46eeeafSJason Wang #define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m) 106d7dfc5cfSToshiaki Makita 107d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 108d7dfc5cfSToshiaki Makita { "packets", VIRTNET_SQ_STAT(packets) }, 109d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_SQ_STAT(bytes) }, 1105b8f3c8dSToshiaki Makita { "xdp_tx", VIRTNET_SQ_STAT(xdp_tx) }, 1115b8f3c8dSToshiaki Makita { "xdp_tx_drops", VIRTNET_SQ_STAT(xdp_tx_drops) }, 112461f03dcSToshiaki Makita { "kicks", VIRTNET_SQ_STAT(kicks) }, 113a520794bSTony Lu { "tx_timeouts", VIRTNET_SQ_STAT(tx_timeouts) }, 114d7dfc5cfSToshiaki Makita }; 115d7dfc5cfSToshiaki Makita 116d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 117d7dfc5cfSToshiaki Makita { "packets", VIRTNET_RQ_STAT(packets) }, 118d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_RQ_STAT(bytes) }, 1192c4a2f7dSToshiaki Makita { "drops", VIRTNET_RQ_STAT(drops) }, 1205b8f3c8dSToshiaki Makita { "xdp_packets", VIRTNET_RQ_STAT(xdp_packets) }, 1215b8f3c8dSToshiaki Makita { "xdp_tx", VIRTNET_RQ_STAT(xdp_tx) }, 1225b8f3c8dSToshiaki Makita { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) }, 1235b8f3c8dSToshiaki Makita { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) }, 124461f03dcSToshiaki Makita { "kicks", VIRTNET_RQ_STAT(kicks) }, 125d7dfc5cfSToshiaki Makita }; 126d7dfc5cfSToshiaki Makita 127d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc) 128d7dfc5cfSToshiaki Makita #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc) 129d7dfc5cfSToshiaki Makita 130308d7982SGavin Li struct virtnet_interrupt_coalesce { 131308d7982SGavin Li u32 max_packets; 132308d7982SGavin Li u32 max_usecs; 133308d7982SGavin Li }; 134308d7982SGavin Li 135295525e2SXuan Zhuo /* The dma information of pages allocated at a time. */ 136295525e2SXuan Zhuo struct virtnet_rq_dma { 137295525e2SXuan Zhuo dma_addr_t addr; 138295525e2SXuan Zhuo u32 ref; 139295525e2SXuan Zhuo u16 len; 140295525e2SXuan Zhuo u16 need_sync; 141295525e2SXuan Zhuo }; 142295525e2SXuan Zhuo 143e9d7417bSJason Wang /* Internal representation of a send virtqueue */ 144e9d7417bSJason Wang struct send_queue { 145e9d7417bSJason Wang /* Virtqueue associated with this send _queue */ 146e9d7417bSJason Wang struct virtqueue *vq; 147e9d7417bSJason Wang 148e9d7417bSJason Wang /* TX: fragments + linear part + virtio header */ 149e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 150986a4f4dSJason Wang 151986a4f4dSJason Wang /* Name of the send queue: output.$index */ 152d0671115SParav Pandit char name[16]; 153b92f1e67SWillem de Bruijn 154d7dfc5cfSToshiaki Makita struct virtnet_sq_stats stats; 155d7dfc5cfSToshiaki Makita 156394bd877SGavin Li struct virtnet_interrupt_coalesce intr_coal; 157394bd877SGavin Li 158b92f1e67SWillem de Bruijn struct napi_struct napi; 159ebcce492SXuan Zhuo 160ebcce492SXuan Zhuo /* Record whether sq is in reset state. */ 161ebcce492SXuan Zhuo bool reset; 162e9d7417bSJason Wang }; 163e9d7417bSJason Wang 164e9d7417bSJason Wang /* Internal representation of a receive virtqueue */ 165e9d7417bSJason Wang struct receive_queue { 166e9d7417bSJason Wang /* Virtqueue associated with this receive_queue */ 167e9d7417bSJason Wang struct virtqueue *vq; 168e9d7417bSJason Wang 169296f96fcSRusty Russell struct napi_struct napi; 170296f96fcSRusty Russell 171f600b690SJohn Fastabend struct bpf_prog __rcu *xdp_prog; 172f600b690SJohn Fastabend 173d7dfc5cfSToshiaki Makita struct virtnet_rq_stats stats; 174d7dfc5cfSToshiaki Makita 175394bd877SGavin Li struct virtnet_interrupt_coalesce intr_coal; 176394bd877SGavin Li 177e9d7417bSJason Wang /* Chain pages by the private ptr. */ 178e9d7417bSJason Wang struct page *pages; 179e9d7417bSJason Wang 180ab7db917SMichael Dalton /* Average packet length for mergeable receive buffers. */ 1815377d758SJohannes Berg struct ewma_pkt_len mrg_avg_pkt_len; 182ab7db917SMichael Dalton 183fb51879dSMichael Dalton /* Page frag for packet buffer allocation. */ 184fb51879dSMichael Dalton struct page_frag alloc_frag; 185fb51879dSMichael Dalton 186e9d7417bSJason Wang /* RX: fragments + linear part + virtio header */ 187e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 188986a4f4dSJason Wang 189d85b758fSMichael S. Tsirkin /* Min single buffer size for mergeable buffers case. */ 190d85b758fSMichael S. Tsirkin unsigned int min_buf_len; 191d85b758fSMichael S. Tsirkin 192986a4f4dSJason Wang /* Name of this receive queue: input.$index */ 193d0671115SParav Pandit char name[16]; 194754b8a21SJesper Dangaard Brouer 195754b8a21SJesper Dangaard Brouer struct xdp_rxq_info xdp_rxq; 196295525e2SXuan Zhuo 197295525e2SXuan Zhuo /* Record the last dma info to free after new pages is allocated. */ 198295525e2SXuan Zhuo struct virtnet_rq_dma *last_dma; 199295525e2SXuan Zhuo 200295525e2SXuan Zhuo /* Do dma by self */ 201295525e2SXuan Zhuo bool do_dma; 202e9d7417bSJason Wang }; 203e9d7417bSJason Wang 204c7114b12SAndrew Melnychenko /* This structure can contain rss message with maximum settings for indirection table and keysize 205c7114b12SAndrew Melnychenko * Note, that default structure that describes RSS configuration virtio_net_rss_config 206c7114b12SAndrew Melnychenko * contains same info but can't handle table values. 207c7114b12SAndrew Melnychenko * In any case, structure would be passed to virtio hw through sg_buf split by parts 208c7114b12SAndrew Melnychenko * because table sizes may be differ according to the device configuration. 209c7114b12SAndrew Melnychenko */ 210c7114b12SAndrew Melnychenko #define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 211c7114b12SAndrew Melnychenko #define VIRTIO_NET_RSS_MAX_TABLE_LEN 128 212c7114b12SAndrew Melnychenko struct virtio_net_ctrl_rss { 213c7114b12SAndrew Melnychenko u32 hash_types; 214c7114b12SAndrew Melnychenko u16 indirection_table_mask; 215c7114b12SAndrew Melnychenko u16 unclassified_queue; 216c7114b12SAndrew Melnychenko u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN]; 217c7114b12SAndrew Melnychenko u16 max_tx_vq; 218c7114b12SAndrew Melnychenko u8 hash_key_length; 219c7114b12SAndrew Melnychenko u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE]; 220c7114b12SAndrew Melnychenko }; 221c7114b12SAndrew Melnychenko 22212e57169SMichael S. Tsirkin /* Control VQ buffers: protected by the rtnl lock */ 22312e57169SMichael S. Tsirkin struct control_buf { 22412e57169SMichael S. Tsirkin struct virtio_net_ctrl_hdr hdr; 22512e57169SMichael S. Tsirkin virtio_net_ctrl_ack status; 22612e57169SMichael S. Tsirkin struct virtio_net_ctrl_mq mq; 22712e57169SMichael S. Tsirkin u8 promisc; 22812e57169SMichael S. Tsirkin u8 allmulti; 229d7fad4c8SMichael S. Tsirkin __virtio16 vid; 230f4ee703aSMichael S. Tsirkin __virtio64 offloads; 231c7114b12SAndrew Melnychenko struct virtio_net_ctrl_rss rss; 232accc1bf2SBrett Creeley struct virtio_net_ctrl_coal_tx coal_tx; 233accc1bf2SBrett Creeley struct virtio_net_ctrl_coal_rx coal_rx; 234394bd877SGavin Li struct virtio_net_ctrl_coal_vq coal_vq; 23512e57169SMichael S. Tsirkin }; 23612e57169SMichael S. Tsirkin 237e9d7417bSJason Wang struct virtnet_info { 238e9d7417bSJason Wang struct virtio_device *vdev; 239e9d7417bSJason Wang struct virtqueue *cvq; 240e9d7417bSJason Wang struct net_device *dev; 241986a4f4dSJason Wang struct send_queue *sq; 242986a4f4dSJason Wang struct receive_queue *rq; 243e9d7417bSJason Wang unsigned int status; 244e9d7417bSJason Wang 245986a4f4dSJason Wang /* Max # of queue pairs supported by the device */ 246986a4f4dSJason Wang u16 max_queue_pairs; 247986a4f4dSJason Wang 248986a4f4dSJason Wang /* # of queue pairs currently used by the driver */ 249986a4f4dSJason Wang u16 curr_queue_pairs; 250986a4f4dSJason Wang 251672aafd5SJohn Fastabend /* # of XDP queue pairs currently used by the driver */ 252672aafd5SJohn Fastabend u16 xdp_queue_pairs; 253672aafd5SJohn Fastabend 25497c2c69eSXuan Zhuo /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ 25597c2c69eSXuan Zhuo bool xdp_enabled; 25697c2c69eSXuan Zhuo 25797402b96SHerbert Xu /* I like... big packets and I cannot lie! */ 25897402b96SHerbert Xu bool big_packets; 25997402b96SHerbert Xu 2604959aebbSGavin Li /* number of sg entries allocated for big packets */ 2614959aebbSGavin Li unsigned int big_packets_num_skbfrags; 2624959aebbSGavin Li 2633f2c31d9SMark McLoughlin /* Host will merge rx buffers for big packets (shake it! shake it!) */ 2643f2c31d9SMark McLoughlin bool mergeable_rx_bufs; 2653f2c31d9SMark McLoughlin 266c7114b12SAndrew Melnychenko /* Host supports rss and/or hash report */ 267c7114b12SAndrew Melnychenko bool has_rss; 26891f41f01SAndrew Melnychenko bool has_rss_hash_report; 269c7114b12SAndrew Melnychenko u8 rss_key_size; 270c7114b12SAndrew Melnychenko u16 rss_indir_table_size; 271c7114b12SAndrew Melnychenko u32 rss_hash_types_supported; 272c1170820SAndrew Melnychenko u32 rss_hash_types_saved; 273c7114b12SAndrew Melnychenko 274986a4f4dSJason Wang /* Has control virtqueue */ 275986a4f4dSJason Wang bool has_cvq; 276986a4f4dSJason Wang 277e7428e95SMichael S. Tsirkin /* Host can handle any s/g split between our header and packet data */ 278e7428e95SMichael S. Tsirkin bool any_header_sg; 279e7428e95SMichael S. Tsirkin 280012873d0SMichael S. Tsirkin /* Packet virtio header size */ 281012873d0SMichael S. Tsirkin u8 hdr_len; 282012873d0SMichael S. Tsirkin 2835a159128SJason Wang /* Work struct for delayed refilling if we run low on memory. */ 2843161e453SRusty Russell struct delayed_work refill; 2853161e453SRusty Russell 2865a159128SJason Wang /* Is delayed refill enabled? */ 2875a159128SJason Wang bool refill_enabled; 2885a159128SJason Wang 2895a159128SJason Wang /* The lock to synchronize the access to refill_enabled */ 2905a159128SJason Wang spinlock_t refill_lock; 2915a159128SJason Wang 292586d17c5SJason Wang /* Work struct for config space updates */ 293586d17c5SJason Wang struct work_struct config_work; 294586d17c5SJason Wang 295986a4f4dSJason Wang /* Does the affinity hint is set for virtqueues? */ 296986a4f4dSJason Wang bool affinity_hint_set; 29747be2479SWanlong Gao 2988017c279SSebastian Andrzej Siewior /* CPU hotplug instances for online & dead */ 2998017c279SSebastian Andrzej Siewior struct hlist_node node; 3008017c279SSebastian Andrzej Siewior struct hlist_node node_dead; 3012ac46030SMichael S. Tsirkin 30212e57169SMichael S. Tsirkin struct control_buf *ctrl; 30316032be5SNikolay Aleksandrov 30416032be5SNikolay Aleksandrov /* Ethtool settings */ 30516032be5SNikolay Aleksandrov u8 duplex; 30616032be5SNikolay Aleksandrov u32 speed; 3073f93522fSJason Wang 308699b045aSAlvaro Karsz /* Interrupt coalescing settings */ 309308d7982SGavin Li struct virtnet_interrupt_coalesce intr_coal_tx; 310308d7982SGavin Li struct virtnet_interrupt_coalesce intr_coal_rx; 311699b045aSAlvaro Karsz 3123f93522fSJason Wang unsigned long guest_offloads; 313a02e8964SWillem de Bruijn unsigned long guest_offloads_capable; 314ba5e4426SSridhar Samudrala 315ba5e4426SSridhar Samudrala /* failover when STANDBY feature enabled */ 316ba5e4426SSridhar Samudrala struct failover *failover; 317296f96fcSRusty Russell }; 318296f96fcSRusty Russell 3199ab86bbcSShirley Ma struct padded_vnet_hdr { 320c1ddc42dSAndrew Melnychenko struct virtio_net_hdr_v1_hash hdr; 3219ab86bbcSShirley Ma /* 322012873d0SMichael S. Tsirkin * hdr is in a separate sg buffer, and data sg buffer shares same page 323012873d0SMichael S. Tsirkin * with this header sg. This padding makes next sg 16 byte aligned 324012873d0SMichael S. Tsirkin * after the header. 3259ab86bbcSShirley Ma */ 326c1ddc42dSAndrew Melnychenko char padding[12]; 3279ab86bbcSShirley Ma }; 3289ab86bbcSShirley Ma 329dae64749SFeng Liu struct virtio_net_common_hdr { 330dae64749SFeng Liu union { 331dae64749SFeng Liu struct virtio_net_hdr hdr; 332dae64749SFeng Liu struct virtio_net_hdr_mrg_rxbuf mrg_hdr; 333dae64749SFeng Liu struct virtio_net_hdr_v1_hash hash_v1_hdr; 334dae64749SFeng Liu }; 335dae64749SFeng Liu }; 336dae64749SFeng Liu 3376a4763e2SXuan Zhuo static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf); 338ebcce492SXuan Zhuo static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf); 3396a4763e2SXuan Zhuo 3405050471dSToshiaki Makita static bool is_xdp_frame(void *ptr) 3415050471dSToshiaki Makita { 3425050471dSToshiaki Makita return (unsigned long)ptr & VIRTIO_XDP_FLAG; 3435050471dSToshiaki Makita } 3445050471dSToshiaki Makita 3455050471dSToshiaki Makita static void *xdp_to_ptr(struct xdp_frame *ptr) 3465050471dSToshiaki Makita { 3475050471dSToshiaki Makita return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG); 3485050471dSToshiaki Makita } 3495050471dSToshiaki Makita 3505050471dSToshiaki Makita static struct xdp_frame *ptr_to_xdp(void *ptr) 3515050471dSToshiaki Makita { 3525050471dSToshiaki Makita return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); 3535050471dSToshiaki Makita } 3545050471dSToshiaki Makita 355986a4f4dSJason Wang /* Converting between virtqueue no. and kernel tx/rx queue no. 356986a4f4dSJason Wang * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 357986a4f4dSJason Wang */ 358986a4f4dSJason Wang static int vq2txq(struct virtqueue *vq) 359986a4f4dSJason Wang { 3609d0ca6edSRusty Russell return (vq->index - 1) / 2; 361986a4f4dSJason Wang } 362986a4f4dSJason Wang 363986a4f4dSJason Wang static int txq2vq(int txq) 364986a4f4dSJason Wang { 365986a4f4dSJason Wang return txq * 2 + 1; 366986a4f4dSJason Wang } 367986a4f4dSJason Wang 368986a4f4dSJason Wang static int vq2rxq(struct virtqueue *vq) 369986a4f4dSJason Wang { 3709d0ca6edSRusty Russell return vq->index / 2; 371986a4f4dSJason Wang } 372986a4f4dSJason Wang 373986a4f4dSJason Wang static int rxq2vq(int rxq) 374986a4f4dSJason Wang { 375986a4f4dSJason Wang return rxq * 2; 376986a4f4dSJason Wang } 377986a4f4dSJason Wang 378dae64749SFeng Liu static inline struct virtio_net_common_hdr * 379dae64749SFeng Liu skb_vnet_common_hdr(struct sk_buff *skb) 380296f96fcSRusty Russell { 381dae64749SFeng Liu return (struct virtio_net_common_hdr *)skb->cb; 382296f96fcSRusty Russell } 383296f96fcSRusty Russell 3849ab86bbcSShirley Ma /* 3859ab86bbcSShirley Ma * private is used to chain pages for big packets, put the whole 3869ab86bbcSShirley Ma * most recent used list in the beginning for reuse 3879ab86bbcSShirley Ma */ 388e9d7417bSJason Wang static void give_pages(struct receive_queue *rq, struct page *page) 389fb6813f4SRusty Russell { 3909ab86bbcSShirley Ma struct page *end; 3919ab86bbcSShirley Ma 392e9d7417bSJason Wang /* Find end of list, sew whole thing into vi->rq.pages. */ 3939ab86bbcSShirley Ma for (end = page; end->private; end = (struct page *)end->private); 394e9d7417bSJason Wang end->private = (unsigned long)rq->pages; 395e9d7417bSJason Wang rq->pages = page; 396fb6813f4SRusty Russell } 397fb6813f4SRusty Russell 398e9d7417bSJason Wang static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 399fb6813f4SRusty Russell { 400e9d7417bSJason Wang struct page *p = rq->pages; 401fb6813f4SRusty Russell 4029ab86bbcSShirley Ma if (p) { 403e9d7417bSJason Wang rq->pages = (struct page *)p->private; 4049ab86bbcSShirley Ma /* clear private here, it is used to chain pages */ 4059ab86bbcSShirley Ma p->private = 0; 4069ab86bbcSShirley Ma } else 407fb6813f4SRusty Russell p = alloc_page(gfp_mask); 408fb6813f4SRusty Russell return p; 409fb6813f4SRusty Russell } 410fb6813f4SRusty Russell 4115a159128SJason Wang static void enable_delayed_refill(struct virtnet_info *vi) 4125a159128SJason Wang { 4135a159128SJason Wang spin_lock_bh(&vi->refill_lock); 4145a159128SJason Wang vi->refill_enabled = true; 4155a159128SJason Wang spin_unlock_bh(&vi->refill_lock); 4165a159128SJason Wang } 4175a159128SJason Wang 4185a159128SJason Wang static void disable_delayed_refill(struct virtnet_info *vi) 4195a159128SJason Wang { 4205a159128SJason Wang spin_lock_bh(&vi->refill_lock); 4215a159128SJason Wang vi->refill_enabled = false; 4225a159128SJason Wang spin_unlock_bh(&vi->refill_lock); 4235a159128SJason Wang } 4245a159128SJason Wang 425e4e8452aSWillem de Bruijn static void virtqueue_napi_schedule(struct napi_struct *napi, 426e4e8452aSWillem de Bruijn struct virtqueue *vq) 427e4e8452aSWillem de Bruijn { 428e4e8452aSWillem de Bruijn if (napi_schedule_prep(napi)) { 429e4e8452aSWillem de Bruijn virtqueue_disable_cb(vq); 430e4e8452aSWillem de Bruijn __napi_schedule(napi); 431e4e8452aSWillem de Bruijn } 432e4e8452aSWillem de Bruijn } 433e4e8452aSWillem de Bruijn 434e4e8452aSWillem de Bruijn static void virtqueue_napi_complete(struct napi_struct *napi, 435e4e8452aSWillem de Bruijn struct virtqueue *vq, int processed) 436e4e8452aSWillem de Bruijn { 437e4e8452aSWillem de Bruijn int opaque; 438e4e8452aSWillem de Bruijn 439e4e8452aSWillem de Bruijn opaque = virtqueue_enable_cb_prepare(vq); 440fdaa767aSToshiaki Makita if (napi_complete_done(napi, processed)) { 441fdaa767aSToshiaki Makita if (unlikely(virtqueue_poll(vq, opaque))) 442e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 443fdaa767aSToshiaki Makita } else { 444fdaa767aSToshiaki Makita virtqueue_disable_cb(vq); 445fdaa767aSToshiaki Makita } 446e4e8452aSWillem de Bruijn } 447e4e8452aSWillem de Bruijn 448e9d7417bSJason Wang static void skb_xmit_done(struct virtqueue *vq) 449296f96fcSRusty Russell { 450e9d7417bSJason Wang struct virtnet_info *vi = vq->vdev->priv; 451b92f1e67SWillem de Bruijn struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi; 452296f96fcSRusty Russell 4532cb9c6baSRusty Russell /* Suppress further interrupts. */ 454e9d7417bSJason Wang virtqueue_disable_cb(vq); 45511a3a154SRusty Russell 456b92f1e67SWillem de Bruijn if (napi->weight) 457b92f1e67SWillem de Bruijn virtqueue_napi_schedule(napi, vq); 458b92f1e67SWillem de Bruijn else 459363f1514SRusty Russell /* We were probably waiting for more output buffers. */ 460986a4f4dSJason Wang netif_wake_subqueue(vi->dev, vq2txq(vq)); 461296f96fcSRusty Russell } 462296f96fcSRusty Russell 46328b39bc7SJason Wang #define MRG_CTX_HEADER_SHIFT 22 46428b39bc7SJason Wang static void *mergeable_len_to_ctx(unsigned int truesize, 46528b39bc7SJason Wang unsigned int headroom) 46628b39bc7SJason Wang { 46728b39bc7SJason Wang return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 46828b39bc7SJason Wang } 46928b39bc7SJason Wang 47028b39bc7SJason Wang static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 47128b39bc7SJason Wang { 47228b39bc7SJason Wang return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 47328b39bc7SJason Wang } 47428b39bc7SJason Wang 47528b39bc7SJason Wang static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 47628b39bc7SJason Wang { 47728b39bc7SJason Wang return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 47828b39bc7SJason Wang } 47928b39bc7SJason Wang 48021e26a71SXuan Zhuo static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen, 48121e26a71SXuan Zhuo unsigned int headroom, 48221e26a71SXuan Zhuo unsigned int len) 48321e26a71SXuan Zhuo { 48421e26a71SXuan Zhuo struct sk_buff *skb; 48521e26a71SXuan Zhuo 48621e26a71SXuan Zhuo skb = build_skb(buf, buflen); 48721e26a71SXuan Zhuo if (unlikely(!skb)) 48821e26a71SXuan Zhuo return NULL; 48921e26a71SXuan Zhuo 49021e26a71SXuan Zhuo skb_reserve(skb, headroom); 49121e26a71SXuan Zhuo skb_put(skb, len); 49221e26a71SXuan Zhuo 49321e26a71SXuan Zhuo return skb; 49421e26a71SXuan Zhuo } 49521e26a71SXuan Zhuo 4963464645aSMike Waychison /* Called from bottom half context */ 497946fa564SMichael S. Tsirkin static struct sk_buff *page_to_skb(struct virtnet_info *vi, 498946fa564SMichael S. Tsirkin struct receive_queue *rq, 4992613af0eSMichael Dalton struct page *page, unsigned int offset, 500fa0f1ba7SXuan Zhuo unsigned int len, unsigned int truesize, 501fa0f1ba7SXuan Zhuo unsigned int headroom) 5029ab86bbcSShirley Ma { 5039ab86bbcSShirley Ma struct sk_buff *skb; 504dae64749SFeng Liu struct virtio_net_common_hdr *hdr; 5052613af0eSMichael Dalton unsigned int copy, hdr_len, hdr_padded_len; 506af39c8f7SEric Dumazet struct page *page_to_free = NULL; 507fb32856bSXuan Zhuo int tailroom, shinfo_size; 508f80bd740SXuan Zhuo char *p, *hdr_p, *buf; 5099ab86bbcSShirley Ma 5102613af0eSMichael Dalton p = page_address(page) + offset; 511fb32856bSXuan Zhuo hdr_p = p; 5129ab86bbcSShirley Ma 513012873d0SMichael S. Tsirkin hdr_len = vi->hdr_len; 514012873d0SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 515c1ddc42dSAndrew Melnychenko hdr_padded_len = hdr_len; 516012873d0SMichael S. Tsirkin else 5172613af0eSMichael Dalton hdr_padded_len = sizeof(struct padded_vnet_hdr); 5183f2c31d9SMark McLoughlin 519fa0f1ba7SXuan Zhuo buf = p - headroom; 5209ab86bbcSShirley Ma len -= hdr_len; 5212613af0eSMichael Dalton offset += hdr_padded_len; 5222613af0eSMichael Dalton p += hdr_padded_len; 523fa0f1ba7SXuan Zhuo tailroom = truesize - headroom - hdr_padded_len - len; 5243f2c31d9SMark McLoughlin 525fb32856bSXuan Zhuo shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 526fb32856bSXuan Zhuo 527f80bd740SXuan Zhuo /* copy small packet so we can reuse these pages */ 528f5d7872aSEric Dumazet if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) { 52921e26a71SXuan Zhuo skb = virtnet_build_skb(buf, truesize, p - buf, len); 530fb32856bSXuan Zhuo if (unlikely(!skb)) 531fb32856bSXuan Zhuo return NULL; 532fb32856bSXuan Zhuo 533afd92d82SJason Wang page = (struct page *)page->private; 534afd92d82SJason Wang if (page) 535afd92d82SJason Wang give_pages(rq, page); 536fb32856bSXuan Zhuo goto ok; 537fb32856bSXuan Zhuo } 538fb32856bSXuan Zhuo 539fb32856bSXuan Zhuo /* copy small packet so we can reuse these pages for small data */ 540fb32856bSXuan Zhuo skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 541fb32856bSXuan Zhuo if (unlikely(!skb)) 542fb32856bSXuan Zhuo return NULL; 543fb32856bSXuan Zhuo 5440f6925b3SEric Dumazet /* Copy all frame if it fits skb->head, otherwise 5450f6925b3SEric Dumazet * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. 5460f6925b3SEric Dumazet */ 5470f6925b3SEric Dumazet if (len <= skb_tailroom(skb)) 5483f2c31d9SMark McLoughlin copy = len; 5490f6925b3SEric Dumazet else 55018117a84SHeng Qi copy = ETH_HLEN; 55159ae1d12SJohannes Berg skb_put_data(skb, p, copy); 5523f2c31d9SMark McLoughlin 5533f2c31d9SMark McLoughlin len -= copy; 5549ab86bbcSShirley Ma offset += copy; 5553f2c31d9SMark McLoughlin 5562613af0eSMichael Dalton if (vi->mergeable_rx_bufs) { 5572613af0eSMichael Dalton if (len) 5582613af0eSMichael Dalton skb_add_rx_frag(skb, 0, page, offset, len, truesize); 5592613af0eSMichael Dalton else 560af39c8f7SEric Dumazet page_to_free = page; 561fb32856bSXuan Zhuo goto ok; 5622613af0eSMichael Dalton } 5632613af0eSMichael Dalton 564e878d78bSSasha Levin /* 565e878d78bSSasha Levin * Verify that we can indeed put this data into a skb. 566e878d78bSSasha Levin * This is here to handle cases when the device erroneously 567e878d78bSSasha Levin * tries to receive more than is possible. This is usually 568e878d78bSSasha Levin * the case of a broken device. 569e878d78bSSasha Levin */ 570e878d78bSSasha Levin if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { 571be443899SAmerigo Wang net_dbg_ratelimited("%s: too much data\n", skb->dev->name); 572e878d78bSSasha Levin dev_kfree_skb(skb); 573e878d78bSSasha Levin return NULL; 574e878d78bSSasha Levin } 5752613af0eSMichael Dalton BUG_ON(offset >= PAGE_SIZE); 5769ab86bbcSShirley Ma while (len) { 5772613af0eSMichael Dalton unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 5782613af0eSMichael Dalton skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 5792613af0eSMichael Dalton frag_size, truesize); 5802613af0eSMichael Dalton len -= frag_size; 5819ab86bbcSShirley Ma page = (struct page *)page->private; 5829ab86bbcSShirley Ma offset = 0; 5833f2c31d9SMark McLoughlin } 5843f2c31d9SMark McLoughlin 5859ab86bbcSShirley Ma if (page) 586e9d7417bSJason Wang give_pages(rq, page); 5873f2c31d9SMark McLoughlin 588fb32856bSXuan Zhuo ok: 589dae64749SFeng Liu hdr = skb_vnet_common_hdr(skb); 590fb32856bSXuan Zhuo memcpy(hdr, hdr_p, hdr_len); 591af39c8f7SEric Dumazet if (page_to_free) 592af39c8f7SEric Dumazet put_page(page_to_free); 593fb32856bSXuan Zhuo 5949ab86bbcSShirley Ma return skb; 5959ab86bbcSShirley Ma } 5969ab86bbcSShirley Ma 597295525e2SXuan Zhuo static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len) 598295525e2SXuan Zhuo { 599295525e2SXuan Zhuo struct page *page = virt_to_head_page(buf); 600295525e2SXuan Zhuo struct virtnet_rq_dma *dma; 601295525e2SXuan Zhuo void *head; 602295525e2SXuan Zhuo int offset; 603295525e2SXuan Zhuo 604295525e2SXuan Zhuo head = page_address(page); 605295525e2SXuan Zhuo 606295525e2SXuan Zhuo dma = head; 607295525e2SXuan Zhuo 608295525e2SXuan Zhuo --dma->ref; 609295525e2SXuan Zhuo 610295525e2SXuan Zhuo if (dma->need_sync && len) { 611295525e2SXuan Zhuo offset = buf - (head + sizeof(*dma)); 612295525e2SXuan Zhuo 6135720c43dSXuan Zhuo virtqueue_dma_sync_single_range_for_cpu(rq->vq, dma->addr, 6145720c43dSXuan Zhuo offset, len, 6155720c43dSXuan Zhuo DMA_FROM_DEVICE); 616295525e2SXuan Zhuo } 617295525e2SXuan Zhuo 6185720c43dSXuan Zhuo if (dma->ref) 619295525e2SXuan Zhuo return; 620295525e2SXuan Zhuo 621295525e2SXuan Zhuo virtqueue_dma_unmap_single_attrs(rq->vq, dma->addr, dma->len, 622295525e2SXuan Zhuo DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 623295525e2SXuan Zhuo put_page(page); 624295525e2SXuan Zhuo } 625295525e2SXuan Zhuo 626295525e2SXuan Zhuo static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx) 627295525e2SXuan Zhuo { 628295525e2SXuan Zhuo void *buf; 629295525e2SXuan Zhuo 630295525e2SXuan Zhuo buf = virtqueue_get_buf_ctx(rq->vq, len, ctx); 631295525e2SXuan Zhuo if (buf && rq->do_dma) 632295525e2SXuan Zhuo virtnet_rq_unmap(rq, buf, *len); 633295525e2SXuan Zhuo 634295525e2SXuan Zhuo return buf; 635295525e2SXuan Zhuo } 636295525e2SXuan Zhuo 637295525e2SXuan Zhuo static void *virtnet_rq_detach_unused_buf(struct receive_queue *rq) 638295525e2SXuan Zhuo { 639295525e2SXuan Zhuo void *buf; 640295525e2SXuan Zhuo 641295525e2SXuan Zhuo buf = virtqueue_detach_unused_buf(rq->vq); 642295525e2SXuan Zhuo if (buf && rq->do_dma) 643295525e2SXuan Zhuo virtnet_rq_unmap(rq, buf, 0); 644295525e2SXuan Zhuo 645295525e2SXuan Zhuo return buf; 646295525e2SXuan Zhuo } 647295525e2SXuan Zhuo 648295525e2SXuan Zhuo static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len) 649295525e2SXuan Zhuo { 650295525e2SXuan Zhuo struct virtnet_rq_dma *dma; 651295525e2SXuan Zhuo dma_addr_t addr; 652295525e2SXuan Zhuo u32 offset; 653295525e2SXuan Zhuo void *head; 654295525e2SXuan Zhuo 655295525e2SXuan Zhuo if (!rq->do_dma) { 656295525e2SXuan Zhuo sg_init_one(rq->sg, buf, len); 657295525e2SXuan Zhuo return; 658295525e2SXuan Zhuo } 659295525e2SXuan Zhuo 660295525e2SXuan Zhuo head = page_address(rq->alloc_frag.page); 661295525e2SXuan Zhuo 662295525e2SXuan Zhuo offset = buf - head; 663295525e2SXuan Zhuo 664295525e2SXuan Zhuo dma = head; 665295525e2SXuan Zhuo 666295525e2SXuan Zhuo addr = dma->addr - sizeof(*dma) + offset; 667295525e2SXuan Zhuo 668295525e2SXuan Zhuo sg_init_table(rq->sg, 1); 669295525e2SXuan Zhuo rq->sg[0].dma_address = addr; 670295525e2SXuan Zhuo rq->sg[0].length = len; 671295525e2SXuan Zhuo } 672295525e2SXuan Zhuo 673295525e2SXuan Zhuo static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp) 674295525e2SXuan Zhuo { 675295525e2SXuan Zhuo struct page_frag *alloc_frag = &rq->alloc_frag; 676295525e2SXuan Zhuo struct virtnet_rq_dma *dma; 677295525e2SXuan Zhuo void *buf, *head; 678295525e2SXuan Zhuo dma_addr_t addr; 679295525e2SXuan Zhuo 680295525e2SXuan Zhuo if (unlikely(!skb_page_frag_refill(size, alloc_frag, gfp))) 681295525e2SXuan Zhuo return NULL; 682295525e2SXuan Zhuo 683295525e2SXuan Zhuo head = page_address(alloc_frag->page); 684295525e2SXuan Zhuo 685295525e2SXuan Zhuo if (rq->do_dma) { 686295525e2SXuan Zhuo dma = head; 687295525e2SXuan Zhuo 688295525e2SXuan Zhuo /* new pages */ 689295525e2SXuan Zhuo if (!alloc_frag->offset) { 690295525e2SXuan Zhuo if (rq->last_dma) { 691295525e2SXuan Zhuo /* Now, the new page is allocated, the last dma 692295525e2SXuan Zhuo * will not be used. So the dma can be unmapped 693295525e2SXuan Zhuo * if the ref is 0. 694295525e2SXuan Zhuo */ 695295525e2SXuan Zhuo virtnet_rq_unmap(rq, rq->last_dma, 0); 696295525e2SXuan Zhuo rq->last_dma = NULL; 697295525e2SXuan Zhuo } 698295525e2SXuan Zhuo 699295525e2SXuan Zhuo dma->len = alloc_frag->size - sizeof(*dma); 700295525e2SXuan Zhuo 701295525e2SXuan Zhuo addr = virtqueue_dma_map_single_attrs(rq->vq, dma + 1, 702295525e2SXuan Zhuo dma->len, DMA_FROM_DEVICE, 0); 703295525e2SXuan Zhuo if (virtqueue_dma_mapping_error(rq->vq, addr)) 704295525e2SXuan Zhuo return NULL; 705295525e2SXuan Zhuo 706295525e2SXuan Zhuo dma->addr = addr; 707295525e2SXuan Zhuo dma->need_sync = virtqueue_dma_need_sync(rq->vq, addr); 708295525e2SXuan Zhuo 709295525e2SXuan Zhuo /* Add a reference to dma to prevent the entire dma from 710295525e2SXuan Zhuo * being released during error handling. This reference 711295525e2SXuan Zhuo * will be freed after the pages are no longer used. 712295525e2SXuan Zhuo */ 713295525e2SXuan Zhuo get_page(alloc_frag->page); 714295525e2SXuan Zhuo dma->ref = 1; 715295525e2SXuan Zhuo alloc_frag->offset = sizeof(*dma); 716295525e2SXuan Zhuo 717295525e2SXuan Zhuo rq->last_dma = dma; 718295525e2SXuan Zhuo } 719295525e2SXuan Zhuo 720295525e2SXuan Zhuo ++dma->ref; 721295525e2SXuan Zhuo } 722295525e2SXuan Zhuo 723295525e2SXuan Zhuo buf = head + alloc_frag->offset; 724295525e2SXuan Zhuo 725295525e2SXuan Zhuo get_page(alloc_frag->page); 726295525e2SXuan Zhuo alloc_frag->offset += size; 727295525e2SXuan Zhuo 728295525e2SXuan Zhuo return buf; 729295525e2SXuan Zhuo } 730295525e2SXuan Zhuo 731295525e2SXuan Zhuo static void virtnet_rq_set_premapped(struct virtnet_info *vi) 732295525e2SXuan Zhuo { 733295525e2SXuan Zhuo int i; 734295525e2SXuan Zhuo 735295525e2SXuan Zhuo /* disable for big mode */ 736295525e2SXuan Zhuo if (!vi->mergeable_rx_bufs && vi->big_packets) 737295525e2SXuan Zhuo return; 738295525e2SXuan Zhuo 739295525e2SXuan Zhuo for (i = 0; i < vi->max_queue_pairs; i++) { 740295525e2SXuan Zhuo if (virtqueue_set_dma_premapped(vi->rq[i].vq)) 741295525e2SXuan Zhuo continue; 742295525e2SXuan Zhuo 743295525e2SXuan Zhuo vi->rq[i].do_dma = true; 744295525e2SXuan Zhuo } 745295525e2SXuan Zhuo } 746295525e2SXuan Zhuo 74725074a44SXuan Zhuo static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi) 74825074a44SXuan Zhuo { 74925074a44SXuan Zhuo unsigned int len; 75025074a44SXuan Zhuo unsigned int packets = 0; 75125074a44SXuan Zhuo unsigned int bytes = 0; 75225074a44SXuan Zhuo void *ptr; 75325074a44SXuan Zhuo 75425074a44SXuan Zhuo while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 75525074a44SXuan Zhuo if (likely(!is_xdp_frame(ptr))) { 75625074a44SXuan Zhuo struct sk_buff *skb = ptr; 75725074a44SXuan Zhuo 75825074a44SXuan Zhuo pr_debug("Sent skb %p\n", skb); 75925074a44SXuan Zhuo 76025074a44SXuan Zhuo bytes += skb->len; 76125074a44SXuan Zhuo napi_consume_skb(skb, in_napi); 76225074a44SXuan Zhuo } else { 76325074a44SXuan Zhuo struct xdp_frame *frame = ptr_to_xdp(ptr); 76425074a44SXuan Zhuo 76525074a44SXuan Zhuo bytes += xdp_get_frame_len(frame); 76625074a44SXuan Zhuo xdp_return_frame(frame); 76725074a44SXuan Zhuo } 76825074a44SXuan Zhuo packets++; 76925074a44SXuan Zhuo } 77025074a44SXuan Zhuo 77125074a44SXuan Zhuo /* Avoid overhead when no packets have been processed 77225074a44SXuan Zhuo * happens when called speculatively from start_xmit. 77325074a44SXuan Zhuo */ 77425074a44SXuan Zhuo if (!packets) 77525074a44SXuan Zhuo return; 77625074a44SXuan Zhuo 77725074a44SXuan Zhuo u64_stats_update_begin(&sq->stats.syncp); 77827debe3eSEric Dumazet u64_stats_add(&sq->stats.bytes, bytes); 77927debe3eSEric Dumazet u64_stats_add(&sq->stats.packets, packets); 78025074a44SXuan Zhuo u64_stats_update_end(&sq->stats.syncp); 78125074a44SXuan Zhuo } 78225074a44SXuan Zhuo 78325074a44SXuan Zhuo static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 78425074a44SXuan Zhuo { 78525074a44SXuan Zhuo if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 78625074a44SXuan Zhuo return false; 78725074a44SXuan Zhuo else if (q < vi->curr_queue_pairs) 78825074a44SXuan Zhuo return true; 78925074a44SXuan Zhuo else 79025074a44SXuan Zhuo return false; 79125074a44SXuan Zhuo } 79225074a44SXuan Zhuo 793b8ef4809SXuan Zhuo static void check_sq_full_and_disable(struct virtnet_info *vi, 794b8ef4809SXuan Zhuo struct net_device *dev, 795b8ef4809SXuan Zhuo struct send_queue *sq) 796b8ef4809SXuan Zhuo { 797b8ef4809SXuan Zhuo bool use_napi = sq->napi.weight; 798b8ef4809SXuan Zhuo int qnum; 799b8ef4809SXuan Zhuo 800b8ef4809SXuan Zhuo qnum = sq - vi->sq; 801b8ef4809SXuan Zhuo 802b8ef4809SXuan Zhuo /* If running out of space, stop queue to avoid getting packets that we 803b8ef4809SXuan Zhuo * are then unable to transmit. 804b8ef4809SXuan Zhuo * An alternative would be to force queuing layer to requeue the skb by 805b8ef4809SXuan Zhuo * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 806b8ef4809SXuan Zhuo * returned in a normal path of operation: it means that driver is not 807b8ef4809SXuan Zhuo * maintaining the TX queue stop/start state properly, and causes 808b8ef4809SXuan Zhuo * the stack to do a non-trivial amount of useless work. 809b8ef4809SXuan Zhuo * Since most packets only take 1 or 2 ring slots, stopping the queue 810b8ef4809SXuan Zhuo * early means 16 slots are typically wasted. 811b8ef4809SXuan Zhuo */ 812b8ef4809SXuan Zhuo if (sq->vq->num_free < 2+MAX_SKB_FRAGS) { 813b8ef4809SXuan Zhuo netif_stop_subqueue(dev, qnum); 814b8ef4809SXuan Zhuo if (use_napi) { 815b8ef4809SXuan Zhuo if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) 816b8ef4809SXuan Zhuo virtqueue_napi_schedule(&sq->napi, sq->vq); 817b8ef4809SXuan Zhuo } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 818b8ef4809SXuan Zhuo /* More just got used, free them then recheck. */ 819b8ef4809SXuan Zhuo free_old_xmit_skbs(sq, false); 820b8ef4809SXuan Zhuo if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { 821b8ef4809SXuan Zhuo netif_start_subqueue(dev, qnum); 822b8ef4809SXuan Zhuo virtqueue_disable_cb(sq->vq); 823b8ef4809SXuan Zhuo } 824b8ef4809SXuan Zhuo } 825b8ef4809SXuan Zhuo } 826b8ef4809SXuan Zhuo } 827b8ef4809SXuan Zhuo 828735fc405SJesper Dangaard Brouer static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 829735fc405SJesper Dangaard Brouer struct send_queue *sq, 83044fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf) 83156434a01SJohn Fastabend { 83256434a01SJohn Fastabend struct virtio_net_hdr_mrg_rxbuf *hdr; 83397717e8dSHeng Qi struct skb_shared_info *shinfo; 83497717e8dSHeng Qi u8 nr_frags = 0; 83597717e8dSHeng Qi int err, i; 83656434a01SJohn Fastabend 837cac320c8SJesper Dangaard Brouer if (unlikely(xdpf->headroom < vi->hdr_len)) 838cac320c8SJesper Dangaard Brouer return -EOVERFLOW; 839cac320c8SJesper Dangaard Brouer 84097717e8dSHeng Qi if (unlikely(xdp_frame_has_frags(xdpf))) { 84197717e8dSHeng Qi shinfo = xdp_get_shared_info_from_frame(xdpf); 84297717e8dSHeng Qi nr_frags = shinfo->nr_frags; 84397717e8dSHeng Qi } 84497717e8dSHeng Qi 84597717e8dSHeng Qi /* In wrapping function virtnet_xdp_xmit(), we need to free 84697717e8dSHeng Qi * up the pending old buffers, where we need to calculate the 84797717e8dSHeng Qi * position of skb_shared_info in xdp_get_frame_len() and 84897717e8dSHeng Qi * xdp_return_frame(), which will involve to xdpf->data and 84997717e8dSHeng Qi * xdpf->headroom. Therefore, we need to update the value of 85097717e8dSHeng Qi * headroom synchronously here. 85197717e8dSHeng Qi */ 85297717e8dSHeng Qi xdpf->headroom -= vi->hdr_len; 853cac320c8SJesper Dangaard Brouer xdpf->data -= vi->hdr_len; 85456434a01SJohn Fastabend /* Zero header and leave csum up to XDP layers */ 855cac320c8SJesper Dangaard Brouer hdr = xdpf->data; 85656434a01SJohn Fastabend memset(hdr, 0, vi->hdr_len); 857cac320c8SJesper Dangaard Brouer xdpf->len += vi->hdr_len; 85856434a01SJohn Fastabend 85997717e8dSHeng Qi sg_init_table(sq->sg, nr_frags + 1); 86097717e8dSHeng Qi sg_set_buf(sq->sg, xdpf->data, xdpf->len); 86197717e8dSHeng Qi for (i = 0; i < nr_frags; i++) { 86297717e8dSHeng Qi skb_frag_t *frag = &shinfo->frags[i]; 863bb91accfSJason Wang 86497717e8dSHeng Qi sg_set_page(&sq->sg[i + 1], skb_frag_page(frag), 86597717e8dSHeng Qi skb_frag_size(frag), skb_frag_off(frag)); 86697717e8dSHeng Qi } 86797717e8dSHeng Qi 86897717e8dSHeng Qi err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1, 86997717e8dSHeng Qi xdp_to_ptr(xdpf), GFP_ATOMIC); 87011b7d897SJesper Dangaard Brouer if (unlikely(err)) 871cac320c8SJesper Dangaard Brouer return -ENOSPC; /* Caller handle free/refcnt */ 87256434a01SJohn Fastabend 873cac320c8SJesper Dangaard Brouer return 0; 87456434a01SJohn Fastabend } 87556434a01SJohn Fastabend 87697c2c69eSXuan Zhuo /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on 87797c2c69eSXuan Zhuo * the current cpu, so it does not need to be locked. 87897c2c69eSXuan Zhuo * 87997c2c69eSXuan Zhuo * Here we use marco instead of inline functions because we have to deal with 88097c2c69eSXuan Zhuo * three issues at the same time: 1. the choice of sq. 2. judge and execute the 88197c2c69eSXuan Zhuo * lock/unlock of txq 3. make sparse happy. It is difficult for two inline 88297c2c69eSXuan Zhuo * functions to perfectly solve these three problems at the same time. 88397c2c69eSXuan Zhuo */ 88497c2c69eSXuan Zhuo #define virtnet_xdp_get_sq(vi) ({ \ 8853dcc1edcSLi RongQing int cpu = smp_processor_id(); \ 88697c2c69eSXuan Zhuo struct netdev_queue *txq; \ 88797c2c69eSXuan Zhuo typeof(vi) v = (vi); \ 88897c2c69eSXuan Zhuo unsigned int qp; \ 88997c2c69eSXuan Zhuo \ 89097c2c69eSXuan Zhuo if (v->curr_queue_pairs > nr_cpu_ids) { \ 89197c2c69eSXuan Zhuo qp = v->curr_queue_pairs - v->xdp_queue_pairs; \ 8923dcc1edcSLi RongQing qp += cpu; \ 89397c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, qp); \ 89497c2c69eSXuan Zhuo __netif_tx_acquire(txq); \ 89597c2c69eSXuan Zhuo } else { \ 8963dcc1edcSLi RongQing qp = cpu % v->curr_queue_pairs; \ 89797c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, qp); \ 8983dcc1edcSLi RongQing __netif_tx_lock(txq, cpu); \ 89997c2c69eSXuan Zhuo } \ 90097c2c69eSXuan Zhuo v->sq + qp; \ 90197c2c69eSXuan Zhuo }) 9022a43565cSToshiaki Makita 90397c2c69eSXuan Zhuo #define virtnet_xdp_put_sq(vi, q) { \ 90497c2c69eSXuan Zhuo struct netdev_queue *txq; \ 90597c2c69eSXuan Zhuo typeof(vi) v = (vi); \ 90697c2c69eSXuan Zhuo \ 90797c2c69eSXuan Zhuo txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \ 90897c2c69eSXuan Zhuo if (v->curr_queue_pairs > nr_cpu_ids) \ 90997c2c69eSXuan Zhuo __netif_tx_release(txq); \ 91097c2c69eSXuan Zhuo else \ 91197c2c69eSXuan Zhuo __netif_tx_unlock(txq); \ 9122a43565cSToshiaki Makita } 9132a43565cSToshiaki Makita 914735fc405SJesper Dangaard Brouer static int virtnet_xdp_xmit(struct net_device *dev, 91542b33468SJesper Dangaard Brouer int n, struct xdp_frame **frames, u32 flags) 916186b3c99SJason Wang { 917186b3c99SJason Wang struct virtnet_info *vi = netdev_priv(dev); 9188dcc5b0aSJesper Dangaard Brouer struct receive_queue *rq = vi->rq; 9198dcc5b0aSJesper Dangaard Brouer struct bpf_prog *xdp_prog; 920735fc405SJesper Dangaard Brouer struct send_queue *sq; 921735fc405SJesper Dangaard Brouer unsigned int len; 922546f2897SToshiaki Makita int packets = 0; 923546f2897SToshiaki Makita int bytes = 0; 924fdc13979SLorenzo Bianconi int nxmit = 0; 925461f03dcSToshiaki Makita int kicks = 0; 9265050471dSToshiaki Makita void *ptr; 927fdc13979SLorenzo Bianconi int ret; 928735fc405SJesper Dangaard Brouer int i; 929735fc405SJesper Dangaard Brouer 9308dcc5b0aSJesper Dangaard Brouer /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 9318dcc5b0aSJesper Dangaard Brouer * indicate XDP resources have been successfully allocated. 9328dcc5b0aSJesper Dangaard Brouer */ 9339719c6b9SJohn Fastabend xdp_prog = rcu_access_pointer(rq->xdp_prog); 9341667c08aSToshiaki Makita if (!xdp_prog) 9351667c08aSToshiaki Makita return -ENXIO; 9361667c08aSToshiaki Makita 93797c2c69eSXuan Zhuo sq = virtnet_xdp_get_sq(vi); 9389ab86bbcSShirley Ma 9399ab86bbcSShirley Ma if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 940186b3c99SJason Wang ret = -EINVAL; 941186b3c99SJason Wang goto out; 942186b3c99SJason Wang } 943186b3c99SJason Wang 944735fc405SJesper Dangaard Brouer /* Free up any pending old buffers before queueing new ones. */ 9455050471dSToshiaki Makita while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 946546f2897SToshiaki Makita if (likely(is_xdp_frame(ptr))) { 947546f2897SToshiaki Makita struct xdp_frame *frame = ptr_to_xdp(ptr); 948546f2897SToshiaki Makita 94950bd14bcSHeng Qi bytes += xdp_get_frame_len(frame); 950546f2897SToshiaki Makita xdp_return_frame(frame); 951546f2897SToshiaki Makita } else { 952546f2897SToshiaki Makita struct sk_buff *skb = ptr; 953546f2897SToshiaki Makita 954546f2897SToshiaki Makita bytes += skb->len; 955546f2897SToshiaki Makita napi_consume_skb(skb, false); 956546f2897SToshiaki Makita } 957546f2897SToshiaki Makita packets++; 9585050471dSToshiaki Makita } 959735fc405SJesper Dangaard Brouer 960735fc405SJesper Dangaard Brouer for (i = 0; i < n; i++) { 961735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf = frames[i]; 962735fc405SJesper Dangaard Brouer 963fdc13979SLorenzo Bianconi if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 964fdc13979SLorenzo Bianconi break; 965fdc13979SLorenzo Bianconi nxmit++; 966735fc405SJesper Dangaard Brouer } 967fdc13979SLorenzo Bianconi ret = nxmit; 9685d274cb4SJesper Dangaard Brouer 969cd1c604aSXuan Zhuo if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) 970cd1c604aSXuan Zhuo check_sq_full_and_disable(vi, dev, sq); 971cd1c604aSXuan Zhuo 972461f03dcSToshiaki Makita if (flags & XDP_XMIT_FLUSH) { 973461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 974461f03dcSToshiaki Makita kicks = 1; 975461f03dcSToshiaki Makita } 9765b8f3c8dSToshiaki Makita out: 9775b8f3c8dSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 97827debe3eSEric Dumazet u64_stats_add(&sq->stats.bytes, bytes); 97927debe3eSEric Dumazet u64_stats_add(&sq->stats.packets, packets); 98027debe3eSEric Dumazet u64_stats_add(&sq->stats.xdp_tx, n); 98127debe3eSEric Dumazet u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit); 98227debe3eSEric Dumazet u64_stats_add(&sq->stats.kicks, kicks); 9835b8f3c8dSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 9845d274cb4SJesper Dangaard Brouer 98597c2c69eSXuan Zhuo virtnet_xdp_put_sq(vi, sq); 9865b8f3c8dSToshiaki Makita return ret; 987186b3c99SJason Wang } 988186b3c99SJason Wang 989bb2c1e9eSXuan Zhuo static void put_xdp_frags(struct xdp_buff *xdp) 990bb2c1e9eSXuan Zhuo { 991bb2c1e9eSXuan Zhuo struct skb_shared_info *shinfo; 992bb2c1e9eSXuan Zhuo struct page *xdp_page; 993bb2c1e9eSXuan Zhuo int i; 994bb2c1e9eSXuan Zhuo 995bb2c1e9eSXuan Zhuo if (xdp_buff_has_frags(xdp)) { 996bb2c1e9eSXuan Zhuo shinfo = xdp_get_shared_info_from_buff(xdp); 997bb2c1e9eSXuan Zhuo for (i = 0; i < shinfo->nr_frags; i++) { 998bb2c1e9eSXuan Zhuo xdp_page = skb_frag_page(&shinfo->frags[i]); 999bb2c1e9eSXuan Zhuo put_page(xdp_page); 1000bb2c1e9eSXuan Zhuo } 1001bb2c1e9eSXuan Zhuo } 1002bb2c1e9eSXuan Zhuo } 1003bb2c1e9eSXuan Zhuo 100400765f8eSXuan Zhuo static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, 100500765f8eSXuan Zhuo struct net_device *dev, 100600765f8eSXuan Zhuo unsigned int *xdp_xmit, 100700765f8eSXuan Zhuo struct virtnet_rq_stats *stats) 100800765f8eSXuan Zhuo { 100900765f8eSXuan Zhuo struct xdp_frame *xdpf; 101000765f8eSXuan Zhuo int err; 101100765f8eSXuan Zhuo u32 act; 101200765f8eSXuan Zhuo 101300765f8eSXuan Zhuo act = bpf_prog_run_xdp(xdp_prog, xdp); 101427debe3eSEric Dumazet u64_stats_inc(&stats->xdp_packets); 101500765f8eSXuan Zhuo 101600765f8eSXuan Zhuo switch (act) { 101700765f8eSXuan Zhuo case XDP_PASS: 101800765f8eSXuan Zhuo return act; 101900765f8eSXuan Zhuo 102000765f8eSXuan Zhuo case XDP_TX: 102127debe3eSEric Dumazet u64_stats_inc(&stats->xdp_tx); 102200765f8eSXuan Zhuo xdpf = xdp_convert_buff_to_frame(xdp); 102300765f8eSXuan Zhuo if (unlikely(!xdpf)) { 102400765f8eSXuan Zhuo netdev_dbg(dev, "convert buff to frame failed for xdp\n"); 102500765f8eSXuan Zhuo return XDP_DROP; 102600765f8eSXuan Zhuo } 102700765f8eSXuan Zhuo 102800765f8eSXuan Zhuo err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 102900765f8eSXuan Zhuo if (unlikely(!err)) { 103000765f8eSXuan Zhuo xdp_return_frame_rx_napi(xdpf); 103100765f8eSXuan Zhuo } else if (unlikely(err < 0)) { 103200765f8eSXuan Zhuo trace_xdp_exception(dev, xdp_prog, act); 103300765f8eSXuan Zhuo return XDP_DROP; 103400765f8eSXuan Zhuo } 103500765f8eSXuan Zhuo *xdp_xmit |= VIRTIO_XDP_TX; 103600765f8eSXuan Zhuo return act; 103700765f8eSXuan Zhuo 103800765f8eSXuan Zhuo case XDP_REDIRECT: 103927debe3eSEric Dumazet u64_stats_inc(&stats->xdp_redirects); 104000765f8eSXuan Zhuo err = xdp_do_redirect(dev, xdp, xdp_prog); 104100765f8eSXuan Zhuo if (err) 104200765f8eSXuan Zhuo return XDP_DROP; 104300765f8eSXuan Zhuo 104400765f8eSXuan Zhuo *xdp_xmit |= VIRTIO_XDP_REDIR; 104500765f8eSXuan Zhuo return act; 104600765f8eSXuan Zhuo 104700765f8eSXuan Zhuo default: 104800765f8eSXuan Zhuo bpf_warn_invalid_xdp_action(dev, xdp_prog, act); 104900765f8eSXuan Zhuo fallthrough; 105000765f8eSXuan Zhuo case XDP_ABORTED: 105100765f8eSXuan Zhuo trace_xdp_exception(dev, xdp_prog, act); 105200765f8eSXuan Zhuo fallthrough; 105300765f8eSXuan Zhuo case XDP_DROP: 105400765f8eSXuan Zhuo return XDP_DROP; 105500765f8eSXuan Zhuo } 105600765f8eSXuan Zhuo } 105700765f8eSXuan Zhuo 1058f6b10209SJason Wang static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 1059f6b10209SJason Wang { 106097c2c69eSXuan Zhuo return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0; 1061f6b10209SJason Wang } 1062f6b10209SJason Wang 10634941d472SJason Wang /* We copy the packet for XDP in the following cases: 10644941d472SJason Wang * 10654941d472SJason Wang * 1) Packet is scattered across multiple rx buffers. 10664941d472SJason Wang * 2) Headroom space is insufficient. 10674941d472SJason Wang * 10684941d472SJason Wang * This is inefficient but it's a temporary condition that 10694941d472SJason Wang * we hit right after XDP is enabled and until queue is refilled 10704941d472SJason Wang * with large buffers with sufficient headroom - so it should affect 10714941d472SJason Wang * at most queue size packets. 10724941d472SJason Wang * Afterwards, the conditions to enable 10734941d472SJason Wang * XDP should preclude the underlying device from sending packets 10744941d472SJason Wang * across multiple buffers (num_buf > 1), and we make sure buffers 10754941d472SJason Wang * have enough headroom. 107672979a6cSJohn Fastabend */ 107772979a6cSJohn Fastabend static struct page *xdp_linearize_page(struct receive_queue *rq, 1078981f14d4SHeng Qi int *num_buf, 107972979a6cSJohn Fastabend struct page *p, 108072979a6cSJohn Fastabend int offset, 10814941d472SJason Wang int page_off, 108272979a6cSJohn Fastabend unsigned int *len) 108372979a6cSJohn Fastabend { 1084853618d5SXuan Zhuo int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1085853618d5SXuan Zhuo struct page *page; 108672979a6cSJohn Fastabend 1087853618d5SXuan Zhuo if (page_off + *len + tailroom > PAGE_SIZE) 1088853618d5SXuan Zhuo return NULL; 1089853618d5SXuan Zhuo 1090853618d5SXuan Zhuo page = alloc_page(GFP_ATOMIC); 109172979a6cSJohn Fastabend if (!page) 109272979a6cSJohn Fastabend return NULL; 109372979a6cSJohn Fastabend 109472979a6cSJohn Fastabend memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 109572979a6cSJohn Fastabend page_off += *len; 109672979a6cSJohn Fastabend 109756a86f84SJason Wang while (--*num_buf) { 109872979a6cSJohn Fastabend unsigned int buflen; 109972979a6cSJohn Fastabend void *buf; 110072979a6cSJohn Fastabend int off; 110172979a6cSJohn Fastabend 1102295525e2SXuan Zhuo buf = virtnet_rq_get_buf(rq, &buflen, NULL); 1103680557cfSMichael S. Tsirkin if (unlikely(!buf)) 110472979a6cSJohn Fastabend goto err_buf; 110572979a6cSJohn Fastabend 110672979a6cSJohn Fastabend p = virt_to_head_page(buf); 110772979a6cSJohn Fastabend off = buf - page_address(p); 110872979a6cSJohn Fastabend 110956a86f84SJason Wang /* guard against a misconfigured or uncooperative backend that 111056a86f84SJason Wang * is sending packet larger than the MTU. 111156a86f84SJason Wang */ 11123cc81a9aSJason Wang if ((page_off + buflen + tailroom) > PAGE_SIZE) { 111356a86f84SJason Wang put_page(p); 111456a86f84SJason Wang goto err_buf; 111556a86f84SJason Wang } 111656a86f84SJason Wang 111772979a6cSJohn Fastabend memcpy(page_address(page) + page_off, 111872979a6cSJohn Fastabend page_address(p) + off, buflen); 111972979a6cSJohn Fastabend page_off += buflen; 112056a86f84SJason Wang put_page(p); 112172979a6cSJohn Fastabend } 112272979a6cSJohn Fastabend 11232de2f7f4SJohn Fastabend /* Headroom does not contribute to packet length */ 11242de2f7f4SJohn Fastabend *len = page_off - VIRTIO_XDP_HEADROOM; 112572979a6cSJohn Fastabend return page; 112672979a6cSJohn Fastabend err_buf: 112772979a6cSJohn Fastabend __free_pages(page, 0); 112872979a6cSJohn Fastabend return NULL; 112972979a6cSJohn Fastabend } 113072979a6cSJohn Fastabend 113119e8c85eSXuan Zhuo static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi, 113219e8c85eSXuan Zhuo unsigned int xdp_headroom, 113319e8c85eSXuan Zhuo void *buf, 113419e8c85eSXuan Zhuo unsigned int len) 113519e8c85eSXuan Zhuo { 113619e8c85eSXuan Zhuo unsigned int header_offset; 113719e8c85eSXuan Zhuo unsigned int headroom; 113819e8c85eSXuan Zhuo unsigned int buflen; 113919e8c85eSXuan Zhuo struct sk_buff *skb; 114019e8c85eSXuan Zhuo 114119e8c85eSXuan Zhuo header_offset = VIRTNET_RX_PAD + xdp_headroom; 114219e8c85eSXuan Zhuo headroom = vi->hdr_len + header_offset; 114319e8c85eSXuan Zhuo buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 114419e8c85eSXuan Zhuo SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 114519e8c85eSXuan Zhuo 114621e26a71SXuan Zhuo skb = virtnet_build_skb(buf, buflen, headroom, len); 114721e26a71SXuan Zhuo if (unlikely(!skb)) 114819e8c85eSXuan Zhuo return NULL; 114919e8c85eSXuan Zhuo 115019e8c85eSXuan Zhuo buf += header_offset; 1151dae64749SFeng Liu memcpy(skb_vnet_common_hdr(skb), buf, vi->hdr_len); 115219e8c85eSXuan Zhuo 115319e8c85eSXuan Zhuo return skb; 115419e8c85eSXuan Zhuo } 115519e8c85eSXuan Zhuo 1156c5f3e72fSXuan Zhuo static struct sk_buff *receive_small_xdp(struct net_device *dev, 11574941d472SJason Wang struct virtnet_info *vi, 11584941d472SJason Wang struct receive_queue *rq, 1159c5f3e72fSXuan Zhuo struct bpf_prog *xdp_prog, 1160c5f3e72fSXuan Zhuo void *buf, 1161c5f3e72fSXuan Zhuo unsigned int xdp_headroom, 1162186b3c99SJason Wang unsigned int len, 11637d9d60fdSToshiaki Makita unsigned int *xdp_xmit, 1164d46eeeafSJason Wang struct virtnet_rq_stats *stats) 11654941d472SJason Wang { 11664941d472SJason Wang unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 11674941d472SJason Wang unsigned int headroom = vi->hdr_len + header_offset; 11684941d472SJason Wang struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 1169c5f3e72fSXuan Zhuo struct page *page = virt_to_head_page(buf); 1170c5f3e72fSXuan Zhuo struct page *xdp_page; 1171c5f3e72fSXuan Zhuo unsigned int buflen; 11724941d472SJason Wang struct xdp_buff xdp; 1173c5f3e72fSXuan Zhuo struct sk_buff *skb; 1174c5f3e72fSXuan Zhuo unsigned int metasize = 0; 11754941d472SJason Wang u32 act; 11764941d472SJason Wang 117795dbe9e7SJesper Dangaard Brouer if (unlikely(hdr->hdr.gso_type)) 11784941d472SJason Wang goto err_xdp; 11794941d472SJason Wang 1180c5f3e72fSXuan Zhuo buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1181c5f3e72fSXuan Zhuo SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1182c5f3e72fSXuan Zhuo 11834941d472SJason Wang if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 11844941d472SJason Wang int offset = buf - page_address(page) + header_offset; 11854941d472SJason Wang unsigned int tlen = len + vi->hdr_len; 1186981f14d4SHeng Qi int num_buf = 1; 11874941d472SJason Wang 11884941d472SJason Wang xdp_headroom = virtnet_get_headroom(vi); 11894941d472SJason Wang header_offset = VIRTNET_RX_PAD + xdp_headroom; 11904941d472SJason Wang headroom = vi->hdr_len + header_offset; 11914941d472SJason Wang buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 11924941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 11934941d472SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, page, 11944941d472SJason Wang offset, header_offset, 11954941d472SJason Wang &tlen); 11964941d472SJason Wang if (!xdp_page) 11974941d472SJason Wang goto err_xdp; 11984941d472SJason Wang 11994941d472SJason Wang buf = page_address(xdp_page); 12004941d472SJason Wang put_page(page); 12014941d472SJason Wang page = xdp_page; 12024941d472SJason Wang } 12034941d472SJason Wang 120443b5169dSLorenzo Bianconi xdp_init_buff(&xdp, buflen, &rq->xdp_rxq); 1205be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, 1206be9df4afSLorenzo Bianconi xdp_headroom, len, true); 120700765f8eSXuan Zhuo 120800765f8eSXuan Zhuo act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats); 12094941d472SJason Wang 12104941d472SJason Wang switch (act) { 12114941d472SJason Wang case XDP_PASS: 12124941d472SJason Wang /* Recalculate length in case bpf program changed it */ 12136870de43SNikita V. Shirokov len = xdp.data_end - xdp.data; 1214503d539aSYuya Kusakabe metasize = xdp.data - xdp.data_meta; 12154941d472SJason Wang break; 1216c5f3e72fSXuan Zhuo 12174941d472SJason Wang case XDP_TX: 1218186b3c99SJason Wang case XDP_REDIRECT: 12194941d472SJason Wang goto xdp_xmit; 1220c5f3e72fSXuan Zhuo 12214941d472SJason Wang default: 12224941d472SJason Wang goto err_xdp; 12234941d472SJason Wang } 1224c5f3e72fSXuan Zhuo 122521e26a71SXuan Zhuo skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len); 122621e26a71SXuan Zhuo if (unlikely(!skb)) 1227c5f3e72fSXuan Zhuo goto err; 1228c5f3e72fSXuan Zhuo 1229c5f3e72fSXuan Zhuo if (metasize) 1230c5f3e72fSXuan Zhuo skb_metadata_set(skb, metasize); 1231c5f3e72fSXuan Zhuo 1232c5f3e72fSXuan Zhuo return skb; 1233c5f3e72fSXuan Zhuo 1234c5f3e72fSXuan Zhuo err_xdp: 123527debe3eSEric Dumazet u64_stats_inc(&stats->xdp_drops); 1236c5f3e72fSXuan Zhuo err: 123727debe3eSEric Dumazet u64_stats_inc(&stats->drops); 1238c5f3e72fSXuan Zhuo put_page(page); 1239c5f3e72fSXuan Zhuo xdp_xmit: 1240c5f3e72fSXuan Zhuo return NULL; 1241c5f3e72fSXuan Zhuo } 1242c5f3e72fSXuan Zhuo 1243c5f3e72fSXuan Zhuo static struct sk_buff *receive_small(struct net_device *dev, 1244c5f3e72fSXuan Zhuo struct virtnet_info *vi, 1245c5f3e72fSXuan Zhuo struct receive_queue *rq, 1246c5f3e72fSXuan Zhuo void *buf, void *ctx, 1247c5f3e72fSXuan Zhuo unsigned int len, 1248c5f3e72fSXuan Zhuo unsigned int *xdp_xmit, 1249c5f3e72fSXuan Zhuo struct virtnet_rq_stats *stats) 1250c5f3e72fSXuan Zhuo { 1251c5f3e72fSXuan Zhuo unsigned int xdp_headroom = (unsigned long)ctx; 1252c5f3e72fSXuan Zhuo struct page *page = virt_to_head_page(buf); 1253aef76506SXuan Zhuo struct sk_buff *skb; 1254c5f3e72fSXuan Zhuo 1255c5f3e72fSXuan Zhuo len -= vi->hdr_len; 125627debe3eSEric Dumazet u64_stats_add(&stats->bytes, len); 1257c5f3e72fSXuan Zhuo 1258c5f3e72fSXuan Zhuo if (unlikely(len > GOOD_PACKET_LEN)) { 1259c5f3e72fSXuan Zhuo pr_debug("%s: rx error: len %u exceeds max size %d\n", 1260c5f3e72fSXuan Zhuo dev->name, len, GOOD_PACKET_LEN); 1261*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 1262c5f3e72fSXuan Zhuo goto err; 1263c5f3e72fSXuan Zhuo } 1264c5f3e72fSXuan Zhuo 1265aef76506SXuan Zhuo if (unlikely(vi->xdp_enabled)) { 1266aef76506SXuan Zhuo struct bpf_prog *xdp_prog; 1267c5f3e72fSXuan Zhuo 1268c5f3e72fSXuan Zhuo rcu_read_lock(); 1269c5f3e72fSXuan Zhuo xdp_prog = rcu_dereference(rq->xdp_prog); 1270c5f3e72fSXuan Zhuo if (xdp_prog) { 1271aef76506SXuan Zhuo skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf, 1272aef76506SXuan Zhuo xdp_headroom, len, xdp_xmit, 1273aef76506SXuan Zhuo stats); 1274c5f3e72fSXuan Zhuo rcu_read_unlock(); 1275c5f3e72fSXuan Zhuo return skb; 12764941d472SJason Wang } 12774941d472SJason Wang rcu_read_unlock(); 1278aef76506SXuan Zhuo } 12794941d472SJason Wang 128019e8c85eSXuan Zhuo skb = receive_small_build_skb(vi, xdp_headroom, buf, len); 128119e8c85eSXuan Zhuo if (likely(skb)) 12824941d472SJason Wang return skb; 12834941d472SJason Wang 1284053c9e18SWenliang Wang err: 128527debe3eSEric Dumazet u64_stats_inc(&stats->drops); 12864941d472SJason Wang put_page(page); 12874941d472SJason Wang return NULL; 12884941d472SJason Wang } 12894941d472SJason Wang 12904941d472SJason Wang static struct sk_buff *receive_big(struct net_device *dev, 12914941d472SJason Wang struct virtnet_info *vi, 12924941d472SJason Wang struct receive_queue *rq, 12934941d472SJason Wang void *buf, 12947d9d60fdSToshiaki Makita unsigned int len, 1295d46eeeafSJason Wang struct virtnet_rq_stats *stats) 12964941d472SJason Wang { 12974941d472SJason Wang struct page *page = buf; 1298503d539aSYuya Kusakabe struct sk_buff *skb = 1299fa0f1ba7SXuan Zhuo page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0); 13004941d472SJason Wang 130127debe3eSEric Dumazet u64_stats_add(&stats->bytes, len - vi->hdr_len); 13024941d472SJason Wang if (unlikely(!skb)) 13034941d472SJason Wang goto err; 13044941d472SJason Wang 13054941d472SJason Wang return skb; 13064941d472SJason Wang 13074941d472SJason Wang err: 130827debe3eSEric Dumazet u64_stats_inc(&stats->drops); 13094941d472SJason Wang give_pages(rq, page); 13104941d472SJason Wang return NULL; 13114941d472SJason Wang } 13124941d472SJason Wang 131380f50f91SXuan Zhuo static void mergeable_buf_free(struct receive_queue *rq, int num_buf, 131480f50f91SXuan Zhuo struct net_device *dev, 131580f50f91SXuan Zhuo struct virtnet_rq_stats *stats) 131680f50f91SXuan Zhuo { 131780f50f91SXuan Zhuo struct page *page; 131880f50f91SXuan Zhuo void *buf; 131980f50f91SXuan Zhuo int len; 132080f50f91SXuan Zhuo 132180f50f91SXuan Zhuo while (num_buf-- > 1) { 1322295525e2SXuan Zhuo buf = virtnet_rq_get_buf(rq, &len, NULL); 132380f50f91SXuan Zhuo if (unlikely(!buf)) { 132480f50f91SXuan Zhuo pr_debug("%s: rx error: %d buffers missing\n", 132580f50f91SXuan Zhuo dev->name, num_buf); 1326*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 132780f50f91SXuan Zhuo break; 132880f50f91SXuan Zhuo } 132927debe3eSEric Dumazet u64_stats_add(&stats->bytes, len); 133080f50f91SXuan Zhuo page = virt_to_head_page(buf); 133180f50f91SXuan Zhuo put_page(page); 133280f50f91SXuan Zhuo } 133380f50f91SXuan Zhuo } 133480f50f91SXuan Zhuo 1335b26aa481SHeng Qi /* Why not use xdp_build_skb_from_frame() ? 1336b26aa481SHeng Qi * XDP core assumes that xdp frags are PAGE_SIZE in length, while in 1337b26aa481SHeng Qi * virtio-net there are 2 points that do not match its requirements: 1338b26aa481SHeng Qi * 1. The size of the prefilled buffer is not fixed before xdp is set. 1339b26aa481SHeng Qi * 2. xdp_build_skb_from_frame() does more checks that we don't need, 1340b26aa481SHeng Qi * like eth_type_trans() (which virtio-net does in receive_buf()). 1341b26aa481SHeng Qi */ 1342b26aa481SHeng Qi static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev, 1343b26aa481SHeng Qi struct virtnet_info *vi, 1344b26aa481SHeng Qi struct xdp_buff *xdp, 1345b26aa481SHeng Qi unsigned int xdp_frags_truesz) 1346b26aa481SHeng Qi { 1347b26aa481SHeng Qi struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 1348b26aa481SHeng Qi unsigned int headroom, data_len; 1349b26aa481SHeng Qi struct sk_buff *skb; 1350b26aa481SHeng Qi int metasize; 1351b26aa481SHeng Qi u8 nr_frags; 1352b26aa481SHeng Qi 1353b26aa481SHeng Qi if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { 1354b26aa481SHeng Qi pr_debug("Error building skb as missing reserved tailroom for xdp"); 1355b26aa481SHeng Qi return NULL; 1356b26aa481SHeng Qi } 1357b26aa481SHeng Qi 1358b26aa481SHeng Qi if (unlikely(xdp_buff_has_frags(xdp))) 1359b26aa481SHeng Qi nr_frags = sinfo->nr_frags; 1360b26aa481SHeng Qi 1361b26aa481SHeng Qi skb = build_skb(xdp->data_hard_start, xdp->frame_sz); 1362b26aa481SHeng Qi if (unlikely(!skb)) 1363b26aa481SHeng Qi return NULL; 1364b26aa481SHeng Qi 1365b26aa481SHeng Qi headroom = xdp->data - xdp->data_hard_start; 1366b26aa481SHeng Qi data_len = xdp->data_end - xdp->data; 1367b26aa481SHeng Qi skb_reserve(skb, headroom); 1368b26aa481SHeng Qi __skb_put(skb, data_len); 1369b26aa481SHeng Qi 1370b26aa481SHeng Qi metasize = xdp->data - xdp->data_meta; 1371b26aa481SHeng Qi metasize = metasize > 0 ? metasize : 0; 1372b26aa481SHeng Qi if (metasize) 1373b26aa481SHeng Qi skb_metadata_set(skb, metasize); 1374b26aa481SHeng Qi 1375b26aa481SHeng Qi if (unlikely(xdp_buff_has_frags(xdp))) 1376b26aa481SHeng Qi xdp_update_skb_shared_info(skb, nr_frags, 1377b26aa481SHeng Qi sinfo->xdp_frags_size, 1378b26aa481SHeng Qi xdp_frags_truesz, 1379b26aa481SHeng Qi xdp_buff_is_frag_pfmemalloc(xdp)); 1380b26aa481SHeng Qi 1381b26aa481SHeng Qi return skb; 1382b26aa481SHeng Qi } 1383b26aa481SHeng Qi 1384ef75cb51SHeng Qi /* TODO: build xdp in big mode */ 1385ef75cb51SHeng Qi static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 1386ef75cb51SHeng Qi struct virtnet_info *vi, 1387ef75cb51SHeng Qi struct receive_queue *rq, 1388ef75cb51SHeng Qi struct xdp_buff *xdp, 1389ef75cb51SHeng Qi void *buf, 1390ef75cb51SHeng Qi unsigned int len, 1391ef75cb51SHeng Qi unsigned int frame_sz, 1392981f14d4SHeng Qi int *num_buf, 1393ef75cb51SHeng Qi unsigned int *xdp_frags_truesize, 1394ef75cb51SHeng Qi struct virtnet_rq_stats *stats) 1395ef75cb51SHeng Qi { 1396ef75cb51SHeng Qi struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1397ef75cb51SHeng Qi unsigned int headroom, tailroom, room; 1398ef75cb51SHeng Qi unsigned int truesize, cur_frag_size; 1399ef75cb51SHeng Qi struct skb_shared_info *shinfo; 1400ef75cb51SHeng Qi unsigned int xdp_frags_truesz = 0; 1401ef75cb51SHeng Qi struct page *page; 1402ef75cb51SHeng Qi skb_frag_t *frag; 1403ef75cb51SHeng Qi int offset; 1404ef75cb51SHeng Qi void *ctx; 1405ef75cb51SHeng Qi 1406ef75cb51SHeng Qi xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 1407ef75cb51SHeng Qi xdp_prepare_buff(xdp, buf - VIRTIO_XDP_HEADROOM, 1408ef75cb51SHeng Qi VIRTIO_XDP_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); 1409ef75cb51SHeng Qi 1410981f14d4SHeng Qi if (!*num_buf) 1411981f14d4SHeng Qi return 0; 1412981f14d4SHeng Qi 1413ef75cb51SHeng Qi if (*num_buf > 1) { 1414ef75cb51SHeng Qi /* If we want to build multi-buffer xdp, we need 1415ef75cb51SHeng Qi * to specify that the flags of xdp_buff have the 1416ef75cb51SHeng Qi * XDP_FLAGS_HAS_FRAG bit. 1417ef75cb51SHeng Qi */ 1418ef75cb51SHeng Qi if (!xdp_buff_has_frags(xdp)) 1419ef75cb51SHeng Qi xdp_buff_set_frags_flag(xdp); 1420ef75cb51SHeng Qi 1421ef75cb51SHeng Qi shinfo = xdp_get_shared_info_from_buff(xdp); 1422ef75cb51SHeng Qi shinfo->nr_frags = 0; 1423ef75cb51SHeng Qi shinfo->xdp_frags_size = 0; 1424ef75cb51SHeng Qi } 1425ef75cb51SHeng Qi 1426981f14d4SHeng Qi if (*num_buf > MAX_SKB_FRAGS + 1) 1427ef75cb51SHeng Qi return -EINVAL; 1428ef75cb51SHeng Qi 1429981f14d4SHeng Qi while (--*num_buf > 0) { 1430295525e2SXuan Zhuo buf = virtnet_rq_get_buf(rq, &len, &ctx); 1431ef75cb51SHeng Qi if (unlikely(!buf)) { 1432ef75cb51SHeng Qi pr_debug("%s: rx error: %d buffers out of %d missing\n", 1433ef75cb51SHeng Qi dev->name, *num_buf, 1434ef75cb51SHeng Qi virtio16_to_cpu(vi->vdev, hdr->num_buffers)); 1435*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 14364cb00b13SXuan Zhuo goto err; 1437ef75cb51SHeng Qi } 1438ef75cb51SHeng Qi 143927debe3eSEric Dumazet u64_stats_add(&stats->bytes, len); 1440ef75cb51SHeng Qi page = virt_to_head_page(buf); 1441ef75cb51SHeng Qi offset = buf - page_address(page); 1442ef75cb51SHeng Qi 1443ef75cb51SHeng Qi truesize = mergeable_ctx_to_truesize(ctx); 1444ef75cb51SHeng Qi headroom = mergeable_ctx_to_headroom(ctx); 1445ef75cb51SHeng Qi tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1446ef75cb51SHeng Qi room = SKB_DATA_ALIGN(headroom + tailroom); 1447ef75cb51SHeng Qi 1448ef75cb51SHeng Qi cur_frag_size = truesize; 1449ef75cb51SHeng Qi xdp_frags_truesz += cur_frag_size; 1450ef75cb51SHeng Qi if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) { 1451ef75cb51SHeng Qi put_page(page); 1452ef75cb51SHeng Qi pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1453ef75cb51SHeng Qi dev->name, len, (unsigned long)(truesize - room)); 1454*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 14554cb00b13SXuan Zhuo goto err; 1456ef75cb51SHeng Qi } 1457ef75cb51SHeng Qi 1458ef75cb51SHeng Qi frag = &shinfo->frags[shinfo->nr_frags++]; 1459b51f4113SYunsheng Lin skb_frag_fill_page_desc(frag, page, offset, len); 1460ef75cb51SHeng Qi if (page_is_pfmemalloc(page)) 1461ef75cb51SHeng Qi xdp_buff_set_frag_pfmemalloc(xdp); 1462ef75cb51SHeng Qi 1463ef75cb51SHeng Qi shinfo->xdp_frags_size += len; 1464ef75cb51SHeng Qi } 1465ef75cb51SHeng Qi 1466ef75cb51SHeng Qi *xdp_frags_truesize = xdp_frags_truesz; 1467ef75cb51SHeng Qi return 0; 14684cb00b13SXuan Zhuo 14694cb00b13SXuan Zhuo err: 14704cb00b13SXuan Zhuo put_xdp_frags(xdp); 14714cb00b13SXuan Zhuo return -EINVAL; 1472ef75cb51SHeng Qi } 1473ef75cb51SHeng Qi 1474ad4858beSXuan Zhuo static void *mergeable_xdp_get_buf(struct virtnet_info *vi, 1475ad4858beSXuan Zhuo struct receive_queue *rq, 1476ad4858beSXuan Zhuo struct bpf_prog *xdp_prog, 1477ad4858beSXuan Zhuo void *ctx, 1478ad4858beSXuan Zhuo unsigned int *frame_sz, 1479ad4858beSXuan Zhuo int *num_buf, 1480ad4858beSXuan Zhuo struct page **page, 1481ad4858beSXuan Zhuo int offset, 1482ad4858beSXuan Zhuo unsigned int *len, 1483ad4858beSXuan Zhuo struct virtio_net_hdr_mrg_rxbuf *hdr) 1484ad4858beSXuan Zhuo { 1485ad4858beSXuan Zhuo unsigned int truesize = mergeable_ctx_to_truesize(ctx); 1486ad4858beSXuan Zhuo unsigned int headroom = mergeable_ctx_to_headroom(ctx); 1487ad4858beSXuan Zhuo struct page *xdp_page; 1488ad4858beSXuan Zhuo unsigned int xdp_room; 1489ad4858beSXuan Zhuo 1490ad4858beSXuan Zhuo /* Transient failure which in theory could occur if 1491ad4858beSXuan Zhuo * in-flight packets from before XDP was enabled reach 1492ad4858beSXuan Zhuo * the receive path after XDP is loaded. 1493ad4858beSXuan Zhuo */ 1494ad4858beSXuan Zhuo if (unlikely(hdr->hdr.gso_type)) 1495ad4858beSXuan Zhuo return NULL; 1496ad4858beSXuan Zhuo 1497ad4858beSXuan Zhuo /* Now XDP core assumes frag size is PAGE_SIZE, but buffers 1498ad4858beSXuan Zhuo * with headroom may add hole in truesize, which 1499ad4858beSXuan Zhuo * make their length exceed PAGE_SIZE. So we disabled the 1500ad4858beSXuan Zhuo * hole mechanism for xdp. See add_recvbuf_mergeable(). 1501ad4858beSXuan Zhuo */ 1502ad4858beSXuan Zhuo *frame_sz = truesize; 1503ad4858beSXuan Zhuo 1504dbe4fec2SXuan Zhuo if (likely(headroom >= virtnet_get_headroom(vi) && 1505dbe4fec2SXuan Zhuo (*num_buf == 1 || xdp_prog->aux->xdp_has_frags))) { 1506dbe4fec2SXuan Zhuo return page_address(*page) + offset; 1507dbe4fec2SXuan Zhuo } 1508dbe4fec2SXuan Zhuo 1509ad4858beSXuan Zhuo /* This happens when headroom is not enough because 1510ad4858beSXuan Zhuo * of the buffer was prefilled before XDP is set. 1511ad4858beSXuan Zhuo * This should only happen for the first several packets. 1512ad4858beSXuan Zhuo * In fact, vq reset can be used here to help us clean up 1513ad4858beSXuan Zhuo * the prefilled buffers, but many existing devices do not 1514ad4858beSXuan Zhuo * support it, and we don't want to bother users who are 1515ad4858beSXuan Zhuo * using xdp normally. 1516ad4858beSXuan Zhuo */ 1517dbe4fec2SXuan Zhuo if (!xdp_prog->aux->xdp_has_frags) { 1518ad4858beSXuan Zhuo /* linearize data for XDP */ 1519ad4858beSXuan Zhuo xdp_page = xdp_linearize_page(rq, num_buf, 1520ad4858beSXuan Zhuo *page, offset, 1521ad4858beSXuan Zhuo VIRTIO_XDP_HEADROOM, 1522ad4858beSXuan Zhuo len); 1523ad4858beSXuan Zhuo if (!xdp_page) 1524ad4858beSXuan Zhuo return NULL; 1525dbe4fec2SXuan Zhuo } else { 1526ad4858beSXuan Zhuo xdp_room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM + 1527ad4858beSXuan Zhuo sizeof(struct skb_shared_info)); 1528ad4858beSXuan Zhuo if (*len + xdp_room > PAGE_SIZE) 1529ad4858beSXuan Zhuo return NULL; 1530ad4858beSXuan Zhuo 1531ad4858beSXuan Zhuo xdp_page = alloc_page(GFP_ATOMIC); 1532ad4858beSXuan Zhuo if (!xdp_page) 1533ad4858beSXuan Zhuo return NULL; 1534ad4858beSXuan Zhuo 1535ad4858beSXuan Zhuo memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM, 1536ad4858beSXuan Zhuo page_address(*page) + offset, *len); 1537ad4858beSXuan Zhuo } 1538ad4858beSXuan Zhuo 1539dbe4fec2SXuan Zhuo *frame_sz = PAGE_SIZE; 1540dbe4fec2SXuan Zhuo 1541dbe4fec2SXuan Zhuo put_page(*page); 1542dbe4fec2SXuan Zhuo 1543dbe4fec2SXuan Zhuo *page = xdp_page; 1544dbe4fec2SXuan Zhuo 1545dbe4fec2SXuan Zhuo return page_address(*page) + VIRTIO_XDP_HEADROOM; 1546ad4858beSXuan Zhuo } 1547ad4858beSXuan Zhuo 1548d8f2835aSXuan Zhuo static struct sk_buff *receive_mergeable_xdp(struct net_device *dev, 1549d8f2835aSXuan Zhuo struct virtnet_info *vi, 1550d8f2835aSXuan Zhuo struct receive_queue *rq, 1551d8f2835aSXuan Zhuo struct bpf_prog *xdp_prog, 1552d8f2835aSXuan Zhuo void *buf, 1553d8f2835aSXuan Zhuo void *ctx, 1554d8f2835aSXuan Zhuo unsigned int len, 1555d8f2835aSXuan Zhuo unsigned int *xdp_xmit, 1556d8f2835aSXuan Zhuo struct virtnet_rq_stats *stats) 1557d8f2835aSXuan Zhuo { 1558d8f2835aSXuan Zhuo struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1559d8f2835aSXuan Zhuo int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 1560d8f2835aSXuan Zhuo struct page *page = virt_to_head_page(buf); 1561d8f2835aSXuan Zhuo int offset = buf - page_address(page); 1562d8f2835aSXuan Zhuo unsigned int xdp_frags_truesz = 0; 1563d8f2835aSXuan Zhuo struct sk_buff *head_skb; 1564d8f2835aSXuan Zhuo unsigned int frame_sz; 1565d8f2835aSXuan Zhuo struct xdp_buff xdp; 1566d8f2835aSXuan Zhuo void *data; 1567d8f2835aSXuan Zhuo u32 act; 1568d8f2835aSXuan Zhuo int err; 1569d8f2835aSXuan Zhuo 1570d8f2835aSXuan Zhuo data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page, 1571d8f2835aSXuan Zhuo offset, &len, hdr); 1572d8f2835aSXuan Zhuo if (unlikely(!data)) 1573d8f2835aSXuan Zhuo goto err_xdp; 1574d8f2835aSXuan Zhuo 1575d8f2835aSXuan Zhuo err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz, 1576d8f2835aSXuan Zhuo &num_buf, &xdp_frags_truesz, stats); 1577d8f2835aSXuan Zhuo if (unlikely(err)) 1578d8f2835aSXuan Zhuo goto err_xdp; 1579d8f2835aSXuan Zhuo 1580d8f2835aSXuan Zhuo act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats); 1581d8f2835aSXuan Zhuo 1582d8f2835aSXuan Zhuo switch (act) { 1583d8f2835aSXuan Zhuo case XDP_PASS: 1584d8f2835aSXuan Zhuo head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz); 1585d8f2835aSXuan Zhuo if (unlikely(!head_skb)) 1586d8f2835aSXuan Zhuo break; 1587d8f2835aSXuan Zhuo return head_skb; 1588d8f2835aSXuan Zhuo 1589d8f2835aSXuan Zhuo case XDP_TX: 1590d8f2835aSXuan Zhuo case XDP_REDIRECT: 1591d8f2835aSXuan Zhuo return NULL; 1592d8f2835aSXuan Zhuo 1593d8f2835aSXuan Zhuo default: 1594d8f2835aSXuan Zhuo break; 1595d8f2835aSXuan Zhuo } 1596d8f2835aSXuan Zhuo 1597d8f2835aSXuan Zhuo put_xdp_frags(&xdp); 1598d8f2835aSXuan Zhuo 1599d8f2835aSXuan Zhuo err_xdp: 1600d8f2835aSXuan Zhuo put_page(page); 1601d8f2835aSXuan Zhuo mergeable_buf_free(rq, num_buf, dev, stats); 1602d8f2835aSXuan Zhuo 160327debe3eSEric Dumazet u64_stats_inc(&stats->xdp_drops); 160427debe3eSEric Dumazet u64_stats_inc(&stats->drops); 1605d8f2835aSXuan Zhuo return NULL; 1606d8f2835aSXuan Zhuo } 1607d8f2835aSXuan Zhuo 16088fc3b9e9SMichael S. Tsirkin static struct sk_buff *receive_mergeable(struct net_device *dev, 1609fdd819b2SMichael S. Tsirkin struct virtnet_info *vi, 16108fc3b9e9SMichael S. Tsirkin struct receive_queue *rq, 1611680557cfSMichael S. Tsirkin void *buf, 1612680557cfSMichael S. Tsirkin void *ctx, 1613186b3c99SJason Wang unsigned int len, 16147d9d60fdSToshiaki Makita unsigned int *xdp_xmit, 1615d46eeeafSJason Wang struct virtnet_rq_stats *stats) 16169ab86bbcSShirley Ma { 1617012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1618981f14d4SHeng Qi int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 16198fc3b9e9SMichael S. Tsirkin struct page *page = virt_to_head_page(buf); 16208fc3b9e9SMichael S. Tsirkin int offset = buf - page_address(page); 1621f600b690SJohn Fastabend struct sk_buff *head_skb, *curr_skb; 16229ce6146eSJesper Dangaard Brouer unsigned int truesize = mergeable_ctx_to_truesize(ctx); 16234941d472SJason Wang unsigned int headroom = mergeable_ctx_to_headroom(ctx); 1624ef75cb51SHeng Qi unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1625ef75cb51SHeng Qi unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1626ab7db917SMichael Dalton 162756434a01SJohn Fastabend head_skb = NULL; 162827debe3eSEric Dumazet u64_stats_add(&stats->bytes, len - vi->hdr_len); 162956434a01SJohn Fastabend 1630ef75cb51SHeng Qi if (unlikely(len > truesize - room)) { 1631ad993a95SXie Yongji pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1632ef75cb51SHeng Qi dev->name, len, (unsigned long)(truesize - room)); 1633*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 1634ad993a95SXie Yongji goto err_skb; 1635ad993a95SXie Yongji } 16366213f07cSLi RongQing 163759ba3b1aSXuan Zhuo if (unlikely(vi->xdp_enabled)) { 163859ba3b1aSXuan Zhuo struct bpf_prog *xdp_prog; 16396213f07cSLi RongQing 1640f600b690SJohn Fastabend rcu_read_lock(); 1641f600b690SJohn Fastabend xdp_prog = rcu_dereference(rq->xdp_prog); 1642f600b690SJohn Fastabend if (xdp_prog) { 1643d8f2835aSXuan Zhuo head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx, 1644d8f2835aSXuan Zhuo len, xdp_xmit, stats); 1645fab89bafSHeng Qi rcu_read_unlock(); 16461830f893SJason Wang return head_skb; 164756434a01SJohn Fastabend } 1648f600b690SJohn Fastabend rcu_read_unlock(); 164959ba3b1aSXuan Zhuo } 1650f600b690SJohn Fastabend 1651fa0f1ba7SXuan Zhuo head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom); 1652f600b690SJohn Fastabend curr_skb = head_skb; 16539ab86bbcSShirley Ma 16548fc3b9e9SMichael S. Tsirkin if (unlikely(!curr_skb)) 16558fc3b9e9SMichael S. Tsirkin goto err_skb; 16569ab86bbcSShirley Ma while (--num_buf) { 16578fc3b9e9SMichael S. Tsirkin int num_skb_frags; 16588fc3b9e9SMichael S. Tsirkin 1659295525e2SXuan Zhuo buf = virtnet_rq_get_buf(rq, &len, &ctx); 166003e9f8a0SYunjian Wang if (unlikely(!buf)) { 16618fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers out of %d missing\n", 1662fdd819b2SMichael S. Tsirkin dev->name, num_buf, 1663012873d0SMichael S. Tsirkin virtio16_to_cpu(vi->vdev, 1664012873d0SMichael S. Tsirkin hdr->num_buffers)); 1665*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 16668fc3b9e9SMichael S. Tsirkin goto err_buf; 16673f2c31d9SMark McLoughlin } 16688fc3b9e9SMichael S. Tsirkin 166927debe3eSEric Dumazet u64_stats_add(&stats->bytes, len); 16708fc3b9e9SMichael S. Tsirkin page = virt_to_head_page(buf); 167128b39bc7SJason Wang 167228b39bc7SJason Wang truesize = mergeable_ctx_to_truesize(ctx); 1673ef75cb51SHeng Qi headroom = mergeable_ctx_to_headroom(ctx); 1674ef75cb51SHeng Qi tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1675ef75cb51SHeng Qi room = SKB_DATA_ALIGN(headroom + tailroom); 1676ef75cb51SHeng Qi if (unlikely(len > truesize - room)) { 167756da5fd0SDan Carpenter pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 1678ef75cb51SHeng Qi dev->name, len, (unsigned long)(truesize - room)); 1679*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 1680680557cfSMichael S. Tsirkin goto err_skb; 1681680557cfSMichael S. Tsirkin } 16828fc3b9e9SMichael S. Tsirkin 16838fc3b9e9SMichael S. Tsirkin num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 16842613af0eSMichael Dalton if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 16852613af0eSMichael Dalton struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 16868fc3b9e9SMichael S. Tsirkin 16878fc3b9e9SMichael S. Tsirkin if (unlikely(!nskb)) 16888fc3b9e9SMichael S. Tsirkin goto err_skb; 16892613af0eSMichael Dalton if (curr_skb == head_skb) 16902613af0eSMichael Dalton skb_shinfo(curr_skb)->frag_list = nskb; 16912613af0eSMichael Dalton else 16922613af0eSMichael Dalton curr_skb->next = nskb; 16932613af0eSMichael Dalton curr_skb = nskb; 16942613af0eSMichael Dalton head_skb->truesize += nskb->truesize; 16952613af0eSMichael Dalton num_skb_frags = 0; 16962613af0eSMichael Dalton } 16972613af0eSMichael Dalton if (curr_skb != head_skb) { 16982613af0eSMichael Dalton head_skb->data_len += len; 16992613af0eSMichael Dalton head_skb->len += len; 1700fb51879dSMichael Dalton head_skb->truesize += truesize; 17012613af0eSMichael Dalton } 17028fc3b9e9SMichael S. Tsirkin offset = buf - page_address(page); 1703ba275241SJason Wang if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 1704ba275241SJason Wang put_page(page); 1705ba275241SJason Wang skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 1706fb51879dSMichael Dalton len, truesize); 1707ba275241SJason Wang } else { 17082613af0eSMichael Dalton skb_add_rx_frag(curr_skb, num_skb_frags, page, 1709fb51879dSMichael Dalton offset, len, truesize); 1710ba275241SJason Wang } 17118fc3b9e9SMichael S. Tsirkin } 17128fc3b9e9SMichael S. Tsirkin 17135377d758SJohannes Berg ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 17148fc3b9e9SMichael S. Tsirkin return head_skb; 17158fc3b9e9SMichael S. Tsirkin 17168fc3b9e9SMichael S. Tsirkin err_skb: 17178fc3b9e9SMichael S. Tsirkin put_page(page); 171880f50f91SXuan Zhuo mergeable_buf_free(rq, num_buf, dev, stats); 171980f50f91SXuan Zhuo 17208fc3b9e9SMichael S. Tsirkin err_buf: 172127debe3eSEric Dumazet u64_stats_inc(&stats->drops); 17228fc3b9e9SMichael S. Tsirkin dev_kfree_skb(head_skb); 17238fc3b9e9SMichael S. Tsirkin return NULL; 17249ab86bbcSShirley Ma } 17259ab86bbcSShirley Ma 172691f41f01SAndrew Melnychenko static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash, 172791f41f01SAndrew Melnychenko struct sk_buff *skb) 172891f41f01SAndrew Melnychenko { 172991f41f01SAndrew Melnychenko enum pkt_hash_types rss_hash_type; 173091f41f01SAndrew Melnychenko 173191f41f01SAndrew Melnychenko if (!hdr_hash || !skb) 173291f41f01SAndrew Melnychenko return; 173391f41f01SAndrew Melnychenko 173495bb6330SMichael S. Tsirkin switch (__le16_to_cpu(hdr_hash->hash_report)) { 173591f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_TCPv4: 173691f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_UDPv4: 173791f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_TCPv6: 173891f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_UDPv6: 173991f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_TCPv6_EX: 174091f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_UDPv6_EX: 174191f41f01SAndrew Melnychenko rss_hash_type = PKT_HASH_TYPE_L4; 174291f41f01SAndrew Melnychenko break; 174391f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_IPv4: 174491f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_IPv6: 174591f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_IPv6_EX: 174691f41f01SAndrew Melnychenko rss_hash_type = PKT_HASH_TYPE_L3; 174791f41f01SAndrew Melnychenko break; 174891f41f01SAndrew Melnychenko case VIRTIO_NET_HASH_REPORT_NONE: 174991f41f01SAndrew Melnychenko default: 175091f41f01SAndrew Melnychenko rss_hash_type = PKT_HASH_TYPE_NONE; 175191f41f01SAndrew Melnychenko } 175295bb6330SMichael S. Tsirkin skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type); 175391f41f01SAndrew Melnychenko } 175491f41f01SAndrew Melnychenko 17557d9d60fdSToshiaki Makita static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 17562471c75eSJesper Dangaard Brouer void *buf, unsigned int len, void **ctx, 1757a0929a44SToshiaki Makita unsigned int *xdp_xmit, 1758d46eeeafSJason Wang struct virtnet_rq_stats *stats) 17599ab86bbcSShirley Ma { 1760e9d7417bSJason Wang struct net_device *dev = vi->dev; 17619ab86bbcSShirley Ma struct sk_buff *skb; 1762dae64749SFeng Liu struct virtio_net_common_hdr *hdr; 17639ab86bbcSShirley Ma 1764bcff3162SMichael S. Tsirkin if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 17659ab86bbcSShirley Ma pr_debug("%s: short packet %i\n", dev->name, len); 1766*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_length_errors); 1767eb1d929fSParav Pandit virtnet_rq_free_unused_buf(rq->vq, buf); 17687d9d60fdSToshiaki Makita return; 17699ab86bbcSShirley Ma } 17709ab86bbcSShirley Ma 1771f121159dSMichael S. Tsirkin if (vi->mergeable_rx_bufs) 17727d9d60fdSToshiaki Makita skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, 1773a0929a44SToshiaki Makita stats); 1774f121159dSMichael S. Tsirkin else if (vi->big_packets) 1775a0929a44SToshiaki Makita skb = receive_big(dev, vi, rq, buf, len, stats); 1776f121159dSMichael S. Tsirkin else 1777a0929a44SToshiaki Makita skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); 1778f121159dSMichael S. Tsirkin 17798fc3b9e9SMichael S. Tsirkin if (unlikely(!skb)) 17807d9d60fdSToshiaki Makita return; 17813f2c31d9SMark McLoughlin 1782dae64749SFeng Liu hdr = skb_vnet_common_hdr(skb); 178391f41f01SAndrew Melnychenko if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report) 1784dae64749SFeng Liu virtio_skb_set_hash(&hdr->hash_v1_hdr, skb); 17853fa2a1dfSstephen hemminger 1786e858fae2SMike Rapoport if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) 178710a8d94aSJason Wang skb->ip_summed = CHECKSUM_UNNECESSARY; 1788296f96fcSRusty Russell 1789e858fae2SMike Rapoport if (virtio_net_hdr_to_skb(skb, &hdr->hdr, 1790e858fae2SMike Rapoport virtio_is_little_endian(vi->vdev))) { 1791e858fae2SMike Rapoport net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n", 1792e858fae2SMike Rapoport dev->name, hdr->hdr.gso_type, 1793fdd819b2SMichael S. Tsirkin hdr->hdr.gso_size); 1794296f96fcSRusty Russell goto frame_err; 1795296f96fcSRusty Russell } 1796296f96fcSRusty Russell 1797133bbb18SWillem de Bruijn skb_record_rx_queue(skb, vq2rxq(rq->vq)); 1798d1dc06dcSMike Rapoport skb->protocol = eth_type_trans(skb, dev); 1799d1dc06dcSMike Rapoport pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 1800d1dc06dcSMike Rapoport ntohs(skb->protocol), skb->len, skb->pkt_type); 1801d1dc06dcSMike Rapoport 18020fbd050aSEric Dumazet napi_gro_receive(&rq->napi, skb); 18037d9d60fdSToshiaki Makita return; 1804296f96fcSRusty Russell 1805296f96fcSRusty Russell frame_err: 1806*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, rx_frame_errors); 1807296f96fcSRusty Russell dev_kfree_skb(skb); 1808296f96fcSRusty Russell } 1809296f96fcSRusty Russell 1810192f68cfSJason Wang /* Unlike mergeable buffers, all buffers are allocated to the 1811192f68cfSJason Wang * same size, except for the headroom. For this reason we do 1812192f68cfSJason Wang * not need to use mergeable_len_to_ctx here - it is enough 1813192f68cfSJason Wang * to store the headroom as the context ignoring the truesize. 1814192f68cfSJason Wang */ 1815946fa564SMichael S. Tsirkin static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 1816946fa564SMichael S. Tsirkin gfp_t gfp) 1817296f96fcSRusty Russell { 1818f6b10209SJason Wang char *buf; 18192de2f7f4SJohn Fastabend unsigned int xdp_headroom = virtnet_get_headroom(vi); 1820192f68cfSJason Wang void *ctx = (void *)(unsigned long)xdp_headroom; 1821f6b10209SJason Wang int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 18229ab86bbcSShirley Ma int err; 18233f2c31d9SMark McLoughlin 1824f6b10209SJason Wang len = SKB_DATA_ALIGN(len) + 1825f6b10209SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1826295525e2SXuan Zhuo 1827295525e2SXuan Zhuo buf = virtnet_rq_alloc(rq, len, gfp); 1828295525e2SXuan Zhuo if (unlikely(!buf)) 18299ab86bbcSShirley Ma return -ENOMEM; 1830296f96fcSRusty Russell 1831295525e2SXuan Zhuo virtnet_rq_init_one_sg(rq, buf + VIRTNET_RX_PAD + xdp_headroom, 1832f6b10209SJason Wang vi->hdr_len + GOOD_PACKET_LEN); 1833295525e2SXuan Zhuo 1834192f68cfSJason Wang err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 1835295525e2SXuan Zhuo if (err < 0) { 1836295525e2SXuan Zhuo if (rq->do_dma) 1837295525e2SXuan Zhuo virtnet_rq_unmap(rq, buf, 0); 1838f6b10209SJason Wang put_page(virt_to_head_page(buf)); 1839295525e2SXuan Zhuo } 1840295525e2SXuan Zhuo 18419ab86bbcSShirley Ma return err; 184297402b96SHerbert Xu } 184397402b96SHerbert Xu 1844012873d0SMichael S. Tsirkin static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 1845012873d0SMichael S. Tsirkin gfp_t gfp) 18469ab86bbcSShirley Ma { 18479ab86bbcSShirley Ma struct page *first, *list = NULL; 18489ab86bbcSShirley Ma char *p; 18499ab86bbcSShirley Ma int i, err, offset; 1850296f96fcSRusty Russell 18514959aebbSGavin Li sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2); 1852a5835440SRusty Russell 18534959aebbSGavin Li /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */ 18544959aebbSGavin Li for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) { 1855e9d7417bSJason Wang first = get_a_page(rq, gfp); 18569ab86bbcSShirley Ma if (!first) { 18579ab86bbcSShirley Ma if (list) 1858e9d7417bSJason Wang give_pages(rq, list); 18599ab86bbcSShirley Ma return -ENOMEM; 1860296f96fcSRusty Russell } 1861e9d7417bSJason Wang sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 18629ab86bbcSShirley Ma 18639ab86bbcSShirley Ma /* chain new page in list head to match sg */ 18649ab86bbcSShirley Ma first->private = (unsigned long)list; 18659ab86bbcSShirley Ma list = first; 18669ab86bbcSShirley Ma } 18679ab86bbcSShirley Ma 1868e9d7417bSJason Wang first = get_a_page(rq, gfp); 18699ab86bbcSShirley Ma if (!first) { 1870e9d7417bSJason Wang give_pages(rq, list); 18719ab86bbcSShirley Ma return -ENOMEM; 18729ab86bbcSShirley Ma } 18739ab86bbcSShirley Ma p = page_address(first); 18749ab86bbcSShirley Ma 1875e9d7417bSJason Wang /* rq->sg[0], rq->sg[1] share the same page */ 1876012873d0SMichael S. Tsirkin /* a separated rq->sg[0] for header - required in case !any_header_sg */ 1877012873d0SMichael S. Tsirkin sg_set_buf(&rq->sg[0], p, vi->hdr_len); 18789ab86bbcSShirley Ma 1879e9d7417bSJason Wang /* rq->sg[1] for data packet, from offset */ 18809ab86bbcSShirley Ma offset = sizeof(struct padded_vnet_hdr); 1881e9d7417bSJason Wang sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 18829ab86bbcSShirley Ma 18839ab86bbcSShirley Ma /* chain first in list head */ 18849ab86bbcSShirley Ma first->private = (unsigned long)list; 18854959aebbSGavin Li err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2, 1886aa989f5eSMichael S. Tsirkin first, gfp); 18879ab86bbcSShirley Ma if (err < 0) 1888e9d7417bSJason Wang give_pages(rq, first); 18899ab86bbcSShirley Ma 18909ab86bbcSShirley Ma return err; 18919ab86bbcSShirley Ma } 18929ab86bbcSShirley Ma 1893d85b758fSMichael S. Tsirkin static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 18943cc81a9aSJason Wang struct ewma_pkt_len *avg_pkt_len, 18953cc81a9aSJason Wang unsigned int room) 18969ab86bbcSShirley Ma { 1897c1ddc42dSAndrew Melnychenko struct virtnet_info *vi = rq->vq->vdev->priv; 1898c1ddc42dSAndrew Melnychenko const size_t hdr_len = vi->hdr_len; 1899fbf28d78SMichael Dalton unsigned int len; 1900fbf28d78SMichael Dalton 19013cc81a9aSJason Wang if (room) 19023cc81a9aSJason Wang return PAGE_SIZE - room; 19033cc81a9aSJason Wang 19045377d758SJohannes Berg len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 1905f0c3192cSMichael S. Tsirkin rq->min_buf_len, PAGE_SIZE - hdr_len); 19063cc81a9aSJason Wang 1907e377fcc8SMichael S. Tsirkin return ALIGN(len, L1_CACHE_BYTES); 1908fbf28d78SMichael Dalton } 1909fbf28d78SMichael Dalton 19102de2f7f4SJohn Fastabend static int add_recvbuf_mergeable(struct virtnet_info *vi, 19112de2f7f4SJohn Fastabend struct receive_queue *rq, gfp_t gfp) 1912fbf28d78SMichael Dalton { 1913fb51879dSMichael Dalton struct page_frag *alloc_frag = &rq->alloc_frag; 19142de2f7f4SJohn Fastabend unsigned int headroom = virtnet_get_headroom(vi); 19153cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 19163cc81a9aSJason Wang unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1917fb51879dSMichael Dalton unsigned int len, hole; 1918295525e2SXuan Zhuo void *ctx; 1919295525e2SXuan Zhuo char *buf; 1920295525e2SXuan Zhuo int err; 19219ab86bbcSShirley Ma 19223cc81a9aSJason Wang /* Extra tailroom is needed to satisfy XDP's assumption. This 19233cc81a9aSJason Wang * means rx frags coalescing won't work, but consider we've 19243cc81a9aSJason Wang * disabled GSO for XDP, it won't be a big issue. 19253cc81a9aSJason Wang */ 19263cc81a9aSJason Wang len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 1927295525e2SXuan Zhuo 1928295525e2SXuan Zhuo buf = virtnet_rq_alloc(rq, len + room, gfp); 1929295525e2SXuan Zhuo if (unlikely(!buf)) 19309ab86bbcSShirley Ma return -ENOMEM; 1931ab7db917SMichael Dalton 19322de2f7f4SJohn Fastabend buf += headroom; /* advance address leaving hole at front of pkt */ 1933fb51879dSMichael Dalton hole = alloc_frag->size - alloc_frag->offset; 19343cc81a9aSJason Wang if (hole < len + room) { 1935ab7db917SMichael Dalton /* To avoid internal fragmentation, if there is very likely not 1936ab7db917SMichael Dalton * enough space for another buffer, add the remaining space to 19371daa8790SMichael S. Tsirkin * the current buffer. 1938484beac2SHeng Qi * XDP core assumes that frame_size of xdp_buff and the length 1939484beac2SHeng Qi * of the frag are PAGE_SIZE, so we disable the hole mechanism. 1940ab7db917SMichael Dalton */ 1941484beac2SHeng Qi if (!headroom) 1942fb51879dSMichael Dalton len += hole; 1943fb51879dSMichael Dalton alloc_frag->offset += hole; 1944fb51879dSMichael Dalton } 19459ab86bbcSShirley Ma 1946295525e2SXuan Zhuo virtnet_rq_init_one_sg(rq, buf, len); 1947295525e2SXuan Zhuo 1948ef75cb51SHeng Qi ctx = mergeable_len_to_ctx(len + room, headroom); 1949680557cfSMichael S. Tsirkin err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 1950295525e2SXuan Zhuo if (err < 0) { 1951295525e2SXuan Zhuo if (rq->do_dma) 1952295525e2SXuan Zhuo virtnet_rq_unmap(rq, buf, 0); 19532613af0eSMichael Dalton put_page(virt_to_head_page(buf)); 1954295525e2SXuan Zhuo } 19559ab86bbcSShirley Ma 19569ab86bbcSShirley Ma return err; 1957296f96fcSRusty Russell } 1958296f96fcSRusty Russell 1959b2baed69SRusty Russell /* 1960b2baed69SRusty Russell * Returns false if we couldn't fill entirely (OOM). 1961b2baed69SRusty Russell * 1962b2baed69SRusty Russell * Normally run in the receive path, but can also be run from ndo_open 1963b2baed69SRusty Russell * before we're receiving packets, or from refill_work which is 1964b2baed69SRusty Russell * careful to disable receiving (using napi_disable). 1965b2baed69SRusty Russell */ 1966946fa564SMichael S. Tsirkin static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 1967946fa564SMichael S. Tsirkin gfp_t gfp) 19683f2c31d9SMark McLoughlin { 19693f2c31d9SMark McLoughlin int err; 19701788f495SMichael S. Tsirkin bool oom; 19713f2c31d9SMark McLoughlin 19720aea51c3SAmit Shah do { 19739ab86bbcSShirley Ma if (vi->mergeable_rx_bufs) 19742de2f7f4SJohn Fastabend err = add_recvbuf_mergeable(vi, rq, gfp); 19759ab86bbcSShirley Ma else if (vi->big_packets) 1976012873d0SMichael S. Tsirkin err = add_recvbuf_big(vi, rq, gfp); 19779ab86bbcSShirley Ma else 1978946fa564SMichael S. Tsirkin err = add_recvbuf_small(vi, rq, gfp); 19793f2c31d9SMark McLoughlin 19801788f495SMichael S. Tsirkin oom = err == -ENOMEM; 19819ed4cb07SRusty Russell if (err) 19823f2c31d9SMark McLoughlin break; 1983b7dfde95SLinus Torvalds } while (rq->vq->num_free); 1984461f03dcSToshiaki Makita if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { 198501c32598SMichael S. Tsirkin unsigned long flags; 198601c32598SMichael S. Tsirkin 198701c32598SMichael S. Tsirkin flags = u64_stats_update_begin_irqsave(&rq->stats.syncp); 198827debe3eSEric Dumazet u64_stats_inc(&rq->stats.kicks); 198901c32598SMichael S. Tsirkin u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); 1990461f03dcSToshiaki Makita } 1991461f03dcSToshiaki Makita 19923161e453SRusty Russell return !oom; 19933f2c31d9SMark McLoughlin } 19943f2c31d9SMark McLoughlin 199518445c4dSRusty Russell static void skb_recv_done(struct virtqueue *rvq) 1996296f96fcSRusty Russell { 1997296f96fcSRusty Russell struct virtnet_info *vi = rvq->vdev->priv; 1998986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 1999e9d7417bSJason Wang 2000e4e8452aSWillem de Bruijn virtqueue_napi_schedule(&rq->napi, rvq); 2001296f96fcSRusty Russell } 2002296f96fcSRusty Russell 2003e4e8452aSWillem de Bruijn static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi) 20043e9d08ecSBruce Rogers { 2005e4e8452aSWillem de Bruijn napi_enable(napi); 20063e9d08ecSBruce Rogers 20073e9d08ecSBruce Rogers /* If all buffers were filled by other side before we napi_enabled, we 2008e4e8452aSWillem de Bruijn * won't get another interrupt, so process any outstanding packets now. 2009e4e8452aSWillem de Bruijn * Call local_bh_enable after to trigger softIRQ processing. 2010e4e8452aSWillem de Bruijn */ 2011ec13ee80SMichael S. Tsirkin local_bh_disable(); 2012e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 2013ec13ee80SMichael S. Tsirkin local_bh_enable(); 20143e9d08ecSBruce Rogers } 20153e9d08ecSBruce Rogers 2016b92f1e67SWillem de Bruijn static void virtnet_napi_tx_enable(struct virtnet_info *vi, 2017b92f1e67SWillem de Bruijn struct virtqueue *vq, 2018b92f1e67SWillem de Bruijn struct napi_struct *napi) 2019b92f1e67SWillem de Bruijn { 2020b92f1e67SWillem de Bruijn if (!napi->weight) 2021b92f1e67SWillem de Bruijn return; 2022b92f1e67SWillem de Bruijn 2023b92f1e67SWillem de Bruijn /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 2024b92f1e67SWillem de Bruijn * enable the feature if this is likely affine with the transmit path. 2025b92f1e67SWillem de Bruijn */ 2026b92f1e67SWillem de Bruijn if (!vi->affinity_hint_set) { 2027b92f1e67SWillem de Bruijn napi->weight = 0; 2028b92f1e67SWillem de Bruijn return; 2029b92f1e67SWillem de Bruijn } 2030b92f1e67SWillem de Bruijn 2031b92f1e67SWillem de Bruijn return virtnet_napi_enable(vq, napi); 2032b92f1e67SWillem de Bruijn } 2033b92f1e67SWillem de Bruijn 203478a57b48SWillem de Bruijn static void virtnet_napi_tx_disable(struct napi_struct *napi) 203578a57b48SWillem de Bruijn { 203678a57b48SWillem de Bruijn if (napi->weight) 203778a57b48SWillem de Bruijn napi_disable(napi); 203878a57b48SWillem de Bruijn } 203978a57b48SWillem de Bruijn 20403161e453SRusty Russell static void refill_work(struct work_struct *work) 20413161e453SRusty Russell { 2042e9d7417bSJason Wang struct virtnet_info *vi = 2043e9d7417bSJason Wang container_of(work, struct virtnet_info, refill.work); 20443161e453SRusty Russell bool still_empty; 2045986a4f4dSJason Wang int i; 20463161e453SRusty Russell 204755257d72SSasha Levin for (i = 0; i < vi->curr_queue_pairs; i++) { 2048986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[i]; 2049986a4f4dSJason Wang 2050986a4f4dSJason Wang napi_disable(&rq->napi); 2051946fa564SMichael S. Tsirkin still_empty = !try_fill_recv(vi, rq, GFP_KERNEL); 2052e4e8452aSWillem de Bruijn virtnet_napi_enable(rq->vq, &rq->napi); 20533161e453SRusty Russell 20543161e453SRusty Russell /* In theory, this can happen: if we don't get any buffers in 2055986a4f4dSJason Wang * we will *never* try to fill again. 2056986a4f4dSJason Wang */ 20573161e453SRusty Russell if (still_empty) 20583b07e9caSTejun Heo schedule_delayed_work(&vi->refill, HZ/2); 20593161e453SRusty Russell } 2060986a4f4dSJason Wang } 20613161e453SRusty Russell 20622471c75eSJesper Dangaard Brouer static int virtnet_receive(struct receive_queue *rq, int budget, 20632471c75eSJesper Dangaard Brouer unsigned int *xdp_xmit) 2064296f96fcSRusty Russell { 2065e9d7417bSJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 2066d46eeeafSJason Wang struct virtnet_rq_stats stats = {}; 2067a0929a44SToshiaki Makita unsigned int len; 206827debe3eSEric Dumazet int packets = 0; 20699ab86bbcSShirley Ma void *buf; 2070a0929a44SToshiaki Makita int i; 2071296f96fcSRusty Russell 2072192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 2073680557cfSMichael S. Tsirkin void *ctx; 2074680557cfSMichael S. Tsirkin 207527debe3eSEric Dumazet while (packets < budget && 2076295525e2SXuan Zhuo (buf = virtnet_rq_get_buf(rq, &len, &ctx))) { 2077a0929a44SToshiaki Makita receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats); 207827debe3eSEric Dumazet packets++; 2079680557cfSMichael S. Tsirkin } 2080680557cfSMichael S. Tsirkin } else { 208127debe3eSEric Dumazet while (packets < budget && 2082295525e2SXuan Zhuo (buf = virtnet_rq_get_buf(rq, &len, NULL)) != NULL) { 2083a0929a44SToshiaki Makita receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats); 208427debe3eSEric Dumazet packets++; 2085296f96fcSRusty Russell } 2086680557cfSMichael S. Tsirkin } 2087296f96fcSRusty Russell 2088718be6baS? jiang if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 20895a159128SJason Wang if (!try_fill_recv(vi, rq, GFP_ATOMIC)) { 20905a159128SJason Wang spin_lock(&vi->refill_lock); 20915a159128SJason Wang if (vi->refill_enabled) 20923b07e9caSTejun Heo schedule_delayed_work(&vi->refill, 0); 20935a159128SJason Wang spin_unlock(&vi->refill_lock); 20945a159128SJason Wang } 20953161e453SRusty Russell } 2096296f96fcSRusty Russell 209727debe3eSEric Dumazet u64_stats_set(&stats.packets, packets); 2098d7dfc5cfSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 2099a0929a44SToshiaki Makita for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) { 2100a0929a44SToshiaki Makita size_t offset = virtnet_rq_stats_desc[i].offset; 210127debe3eSEric Dumazet u64_stats_t *item, *src; 2102a0929a44SToshiaki Makita 210327debe3eSEric Dumazet item = (u64_stats_t *)((u8 *)&rq->stats + offset); 210427debe3eSEric Dumazet src = (u64_stats_t *)((u8 *)&stats + offset); 210527debe3eSEric Dumazet u64_stats_add(item, u64_stats_read(src)); 2106a0929a44SToshiaki Makita } 2107d7dfc5cfSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 210861845d20SJason Wang 210927debe3eSEric Dumazet return packets; 21102ffa7598SJason Wang } 21112ffa7598SJason Wang 21127b0411efSWillem de Bruijn static void virtnet_poll_cleantx(struct receive_queue *rq) 21137b0411efSWillem de Bruijn { 21147b0411efSWillem de Bruijn struct virtnet_info *vi = rq->vq->vdev->priv; 21157b0411efSWillem de Bruijn unsigned int index = vq2rxq(rq->vq); 21167b0411efSWillem de Bruijn struct send_queue *sq = &vi->sq[index]; 21177b0411efSWillem de Bruijn struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 21187b0411efSWillem de Bruijn 2119534da5e8SToshiaki Makita if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) 21207b0411efSWillem de Bruijn return; 21217b0411efSWillem de Bruijn 21227b0411efSWillem de Bruijn if (__netif_tx_trylock(txq)) { 2123ebcce492SXuan Zhuo if (sq->reset) { 2124ebcce492SXuan Zhuo __netif_tx_unlock(txq); 2125ebcce492SXuan Zhuo return; 2126ebcce492SXuan Zhuo } 2127ebcce492SXuan Zhuo 2128a7766ef1SMichael S. Tsirkin do { 2129a7766ef1SMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 2130df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, true); 2131a7766ef1SMichael S. Tsirkin } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 21327b0411efSWillem de Bruijn 21337b0411efSWillem de Bruijn if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 21347b0411efSWillem de Bruijn netif_tx_wake_queue(txq); 213522bc63c5SMichael S. Tsirkin 213622bc63c5SMichael S. Tsirkin __netif_tx_unlock(txq); 213722bc63c5SMichael S. Tsirkin } 21387b0411efSWillem de Bruijn } 21397b0411efSWillem de Bruijn 21402ffa7598SJason Wang static int virtnet_poll(struct napi_struct *napi, int budget) 21412ffa7598SJason Wang { 21422ffa7598SJason Wang struct receive_queue *rq = 21432ffa7598SJason Wang container_of(napi, struct receive_queue, napi); 21449267c430SJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 21459267c430SJason Wang struct send_queue *sq; 21462a43565cSToshiaki Makita unsigned int received; 21472471c75eSJesper Dangaard Brouer unsigned int xdp_xmit = 0; 21482ffa7598SJason Wang 21497b0411efSWillem de Bruijn virtnet_poll_cleantx(rq); 21507b0411efSWillem de Bruijn 2151186b3c99SJason Wang received = virtnet_receive(rq, budget, &xdp_xmit); 21522ffa7598SJason Wang 2153ad7e615fSMagnus Karlsson if (xdp_xmit & VIRTIO_XDP_REDIR) 2154ad7e615fSMagnus Karlsson xdp_do_flush(); 2155ad7e615fSMagnus Karlsson 21568329d98eSRusty Russell /* Out of packets? */ 2157e4e8452aSWillem de Bruijn if (received < budget) 2158e4e8452aSWillem de Bruijn virtqueue_napi_complete(napi, rq->vq, received); 2159296f96fcSRusty Russell 21602471c75eSJesper Dangaard Brouer if (xdp_xmit & VIRTIO_XDP_TX) { 216197c2c69eSXuan Zhuo sq = virtnet_xdp_get_sq(vi); 2162461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 2163461f03dcSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 216427debe3eSEric Dumazet u64_stats_inc(&sq->stats.kicks); 2165461f03dcSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 2166461f03dcSToshiaki Makita } 216797c2c69eSXuan Zhuo virtnet_xdp_put_sq(vi, sq); 21689267c430SJason Wang } 2169186b3c99SJason Wang 2170296f96fcSRusty Russell return received; 2171296f96fcSRusty Russell } 2172296f96fcSRusty Russell 21735306623aSFeng Liu static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index) 21745306623aSFeng Liu { 21755306623aSFeng Liu virtnet_napi_tx_disable(&vi->sq[qp_index].napi); 21765306623aSFeng Liu napi_disable(&vi->rq[qp_index].napi); 21775306623aSFeng Liu xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); 21785306623aSFeng Liu } 21795306623aSFeng Liu 21805306623aSFeng Liu static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index) 21815306623aSFeng Liu { 21825306623aSFeng Liu struct net_device *dev = vi->dev; 21835306623aSFeng Liu int err; 21845306623aSFeng Liu 21855306623aSFeng Liu err = xdp_rxq_info_reg(&vi->rq[qp_index].xdp_rxq, dev, qp_index, 21865306623aSFeng Liu vi->rq[qp_index].napi.napi_id); 21875306623aSFeng Liu if (err < 0) 21885306623aSFeng Liu return err; 21895306623aSFeng Liu 21905306623aSFeng Liu err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq, 21915306623aSFeng Liu MEM_TYPE_PAGE_SHARED, NULL); 21925306623aSFeng Liu if (err < 0) 21935306623aSFeng Liu goto err_xdp_reg_mem_model; 21945306623aSFeng Liu 21955306623aSFeng Liu virtnet_napi_enable(vi->rq[qp_index].vq, &vi->rq[qp_index].napi); 21965306623aSFeng Liu virtnet_napi_tx_enable(vi, vi->sq[qp_index].vq, &vi->sq[qp_index].napi); 21975306623aSFeng Liu 21985306623aSFeng Liu return 0; 21995306623aSFeng Liu 22005306623aSFeng Liu err_xdp_reg_mem_model: 22015306623aSFeng Liu xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); 22025306623aSFeng Liu return err; 22035306623aSFeng Liu } 22045306623aSFeng Liu 2205986a4f4dSJason Wang static int virtnet_open(struct net_device *dev) 2206986a4f4dSJason Wang { 2207986a4f4dSJason Wang struct virtnet_info *vi = netdev_priv(dev); 2208754b8a21SJesper Dangaard Brouer int i, err; 2209986a4f4dSJason Wang 22105a159128SJason Wang enable_delayed_refill(vi); 22115a159128SJason Wang 2212e4166625SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2213e4166625SJason Wang if (i < vi->curr_queue_pairs) 2214986a4f4dSJason Wang /* Make sure we have some buffers: if oom use wq. */ 2215946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 2216986a4f4dSJason Wang schedule_delayed_work(&vi->refill, 0); 2217754b8a21SJesper Dangaard Brouer 22185306623aSFeng Liu err = virtnet_enable_queue_pair(vi, i); 2219754b8a21SJesper Dangaard Brouer if (err < 0) 22205306623aSFeng Liu goto err_enable_qp; 2221986a4f4dSJason Wang } 2222986a4f4dSJason Wang 2223986a4f4dSJason Wang return 0; 22245306623aSFeng Liu 22255306623aSFeng Liu err_enable_qp: 22265306623aSFeng Liu disable_delayed_refill(vi); 22275306623aSFeng Liu cancel_delayed_work_sync(&vi->refill); 22285306623aSFeng Liu 22295306623aSFeng Liu for (i--; i >= 0; i--) 22305306623aSFeng Liu virtnet_disable_queue_pair(vi, i); 22315306623aSFeng Liu return err; 2232986a4f4dSJason Wang } 2233986a4f4dSJason Wang 2234b92f1e67SWillem de Bruijn static int virtnet_poll_tx(struct napi_struct *napi, int budget) 2235b92f1e67SWillem de Bruijn { 2236b92f1e67SWillem de Bruijn struct send_queue *sq = container_of(napi, struct send_queue, napi); 2237b92f1e67SWillem de Bruijn struct virtnet_info *vi = sq->vq->vdev->priv; 2238534da5e8SToshiaki Makita unsigned int index = vq2txq(sq->vq); 2239534da5e8SToshiaki Makita struct netdev_queue *txq; 22405a2f966dSMichael S. Tsirkin int opaque; 22415a2f966dSMichael S. Tsirkin bool done; 2242b92f1e67SWillem de Bruijn 2243534da5e8SToshiaki Makita if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { 2244534da5e8SToshiaki Makita /* We don't need to enable cb for XDP */ 2245534da5e8SToshiaki Makita napi_complete_done(napi, 0); 2246534da5e8SToshiaki Makita return 0; 2247534da5e8SToshiaki Makita } 2248534da5e8SToshiaki Makita 2249534da5e8SToshiaki Makita txq = netdev_get_tx_queue(vi->dev, index); 2250b92f1e67SWillem de Bruijn __netif_tx_lock(txq, raw_smp_processor_id()); 22515a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 2252df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, true); 22535a2f966dSMichael S. Tsirkin 225422bc63c5SMichael S. Tsirkin if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 225522bc63c5SMichael S. Tsirkin netif_tx_wake_queue(txq); 225622bc63c5SMichael S. Tsirkin 22575a2f966dSMichael S. Tsirkin opaque = virtqueue_enable_cb_prepare(sq->vq); 22585a2f966dSMichael S. Tsirkin 22595a2f966dSMichael S. Tsirkin done = napi_complete_done(napi, 0); 22605a2f966dSMichael S. Tsirkin 22615a2f966dSMichael S. Tsirkin if (!done) 22625a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 22635a2f966dSMichael S. Tsirkin 2264b92f1e67SWillem de Bruijn __netif_tx_unlock(txq); 2265b92f1e67SWillem de Bruijn 22665a2f966dSMichael S. Tsirkin if (done) { 22675a2f966dSMichael S. Tsirkin if (unlikely(virtqueue_poll(sq->vq, opaque))) { 22685a2f966dSMichael S. Tsirkin if (napi_schedule_prep(napi)) { 22695a2f966dSMichael S. Tsirkin __netif_tx_lock(txq, raw_smp_processor_id()); 22705a2f966dSMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 22715a2f966dSMichael S. Tsirkin __netif_tx_unlock(txq); 22725a2f966dSMichael S. Tsirkin __napi_schedule(napi); 22735a2f966dSMichael S. Tsirkin } 22745a2f966dSMichael S. Tsirkin } 22755a2f966dSMichael S. Tsirkin } 2276b92f1e67SWillem de Bruijn 2277b92f1e67SWillem de Bruijn return 0; 2278b92f1e67SWillem de Bruijn } 2279b92f1e67SWillem de Bruijn 2280e9d7417bSJason Wang static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) 2281296f96fcSRusty Russell { 2282012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 2283296f96fcSRusty Russell const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 2284e9d7417bSJason Wang struct virtnet_info *vi = sq->vq->vdev->priv; 2285e2fcad58SJason A. Donenfeld int num_sg; 2286012873d0SMichael S. Tsirkin unsigned hdr_len = vi->hdr_len; 2287e7428e95SMichael S. Tsirkin bool can_push; 2288296f96fcSRusty Russell 2289e174961cSJohannes Berg pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 2290e7428e95SMichael S. Tsirkin 2291e7428e95SMichael S. Tsirkin can_push = vi->any_header_sg && 2292e7428e95SMichael S. Tsirkin !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 2293e7428e95SMichael S. Tsirkin !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 2294e7428e95SMichael S. Tsirkin /* Even if we can, don't push here yet as this would skew 2295e7428e95SMichael S. Tsirkin * csum_start offset below. */ 2296e7428e95SMichael S. Tsirkin if (can_push) 2297012873d0SMichael S. Tsirkin hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len); 2298e7428e95SMichael S. Tsirkin else 2299dae64749SFeng Liu hdr = &skb_vnet_common_hdr(skb)->mrg_hdr; 2300296f96fcSRusty Russell 2301e858fae2SMike Rapoport if (virtio_net_hdr_from_skb(skb, &hdr->hdr, 2302fd3a8862SWillem de Bruijn virtio_is_little_endian(vi->vdev), false, 2303fd3a8862SWillem de Bruijn 0)) 230485eb1389SXianting Tian return -EPROTO; 2305296f96fcSRusty Russell 2306e7428e95SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 2307012873d0SMichael S. Tsirkin hdr->num_buffers = 0; 23083f2c31d9SMark McLoughlin 2309547c890cSJason Wang sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 2310e7428e95SMichael S. Tsirkin if (can_push) { 2311e7428e95SMichael S. Tsirkin __skb_push(skb, hdr_len); 2312e7428e95SMichael S. Tsirkin num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 2313e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 2314e2fcad58SJason A. Donenfeld return num_sg; 2315e7428e95SMichael S. Tsirkin /* Pull header back to avoid skew in tx bytes calculations. */ 2316e7428e95SMichael S. Tsirkin __skb_pull(skb, hdr_len); 2317e7428e95SMichael S. Tsirkin } else { 2318e7428e95SMichael S. Tsirkin sg_set_buf(sq->sg, hdr, hdr_len); 2319e2fcad58SJason A. Donenfeld num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 2320e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 2321e2fcad58SJason A. Donenfeld return num_sg; 2322e2fcad58SJason A. Donenfeld num_sg++; 2323e7428e95SMichael S. Tsirkin } 23249dc7b9e4SRusty Russell return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC); 232511a3a154SRusty Russell } 232611a3a154SRusty Russell 2327424efe9cSStephen Hemminger static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 232899ffc696SRusty Russell { 232999ffc696SRusty Russell struct virtnet_info *vi = netdev_priv(dev); 2330986a4f4dSJason Wang int qnum = skb_get_queue_mapping(skb); 2331986a4f4dSJason Wang struct send_queue *sq = &vi->sq[qnum]; 23329ed4cb07SRusty Russell int err; 23334b7fd2e6SMichael S. Tsirkin struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 23346b16f9eeSFlorian Westphal bool kick = !netdev_xmit_more(); 2335b92f1e67SWillem de Bruijn bool use_napi = sq->napi.weight; 23362cb9c6baSRusty Russell 23372cb9c6baSRusty Russell /* Free up any pending old buffers before queueing new ones. */ 2338a7766ef1SMichael S. Tsirkin do { 2339a7766ef1SMichael S. Tsirkin if (use_napi) 2340a7766ef1SMichael S. Tsirkin virtqueue_disable_cb(sq->vq); 2341a7766ef1SMichael S. Tsirkin 2342df133f3fSMichael S. Tsirkin free_old_xmit_skbs(sq, false); 234399ffc696SRusty Russell 2344a7766ef1SMichael S. Tsirkin } while (use_napi && kick && 2345a7766ef1SMichael S. Tsirkin unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 2346bdb12e0dSWillem de Bruijn 2347074c3582SJacob Keller /* timestamp packet in software */ 2348074c3582SJacob Keller skb_tx_timestamp(skb); 2349074c3582SJacob Keller 235003f191baSMichael S. Tsirkin /* Try to transmit */ 2351b7dfde95SLinus Torvalds err = xmit_skb(sq, skb); 235299ffc696SRusty Russell 23539ed4cb07SRusty Russell /* This should not happen! */ 2354681daee2SJason Wang if (unlikely(err)) { 2355*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, tx_fifo_errors); 23562e57b79cSRick Jones if (net_ratelimit()) 235758eba97dSRusty Russell dev_warn(&dev->dev, 23587934b481SYuval Shaia "Unexpected TXQ (%d) queue failure: %d\n", 23597934b481SYuval Shaia qnum, err); 2360*e2e5c2a3SEric Dumazet DEV_STATS_INC(dev, tx_dropped); 236185e94525SEric W. Biederman dev_kfree_skb_any(skb); 236258eba97dSRusty Russell return NETDEV_TX_OK; 2363296f96fcSRusty Russell } 236403f191baSMichael S. Tsirkin 236548925e37SRusty Russell /* Don't wait up for transmitted skbs to be freed. */ 2366b92f1e67SWillem de Bruijn if (!use_napi) { 236748925e37SRusty Russell skb_orphan(skb); 2368895b5c9fSFlorian Westphal nf_reset_ct(skb); 2369b92f1e67SWillem de Bruijn } 237048925e37SRusty Russell 2371b8ef4809SXuan Zhuo check_sq_full_and_disable(vi, dev, sq); 237248925e37SRusty Russell 2373461f03dcSToshiaki Makita if (kick || netif_xmit_stopped(txq)) { 2374461f03dcSToshiaki Makita if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 2375461f03dcSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 237627debe3eSEric Dumazet u64_stats_inc(&sq->stats.kicks); 2377461f03dcSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 2378461f03dcSToshiaki Makita } 2379461f03dcSToshiaki Makita } 23800b725a2cSDavid S. Miller 23810b725a2cSDavid S. Miller return NETDEV_TX_OK; 2382c223a078SDavid S. Miller } 2383c223a078SDavid S. Miller 23846a4763e2SXuan Zhuo static int virtnet_rx_resize(struct virtnet_info *vi, 23856a4763e2SXuan Zhuo struct receive_queue *rq, u32 ring_num) 23866a4763e2SXuan Zhuo { 23876a4763e2SXuan Zhuo bool running = netif_running(vi->dev); 23886a4763e2SXuan Zhuo int err, qindex; 23896a4763e2SXuan Zhuo 23906a4763e2SXuan Zhuo qindex = rq - vi->rq; 23916a4763e2SXuan Zhuo 23926a4763e2SXuan Zhuo if (running) 23936a4763e2SXuan Zhuo napi_disable(&rq->napi); 23946a4763e2SXuan Zhuo 23956a4763e2SXuan Zhuo err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf); 23966a4763e2SXuan Zhuo if (err) 23976a4763e2SXuan Zhuo netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); 23986a4763e2SXuan Zhuo 23996a4763e2SXuan Zhuo if (!try_fill_recv(vi, rq, GFP_KERNEL)) 24006a4763e2SXuan Zhuo schedule_delayed_work(&vi->refill, 0); 24016a4763e2SXuan Zhuo 24026a4763e2SXuan Zhuo if (running) 24036a4763e2SXuan Zhuo virtnet_napi_enable(rq->vq, &rq->napi); 24046a4763e2SXuan Zhuo return err; 24056a4763e2SXuan Zhuo } 24066a4763e2SXuan Zhuo 2407ebcce492SXuan Zhuo static int virtnet_tx_resize(struct virtnet_info *vi, 2408ebcce492SXuan Zhuo struct send_queue *sq, u32 ring_num) 2409ebcce492SXuan Zhuo { 2410ebcce492SXuan Zhuo bool running = netif_running(vi->dev); 2411ebcce492SXuan Zhuo struct netdev_queue *txq; 2412ebcce492SXuan Zhuo int err, qindex; 2413ebcce492SXuan Zhuo 2414ebcce492SXuan Zhuo qindex = sq - vi->sq; 2415ebcce492SXuan Zhuo 2416ebcce492SXuan Zhuo if (running) 2417ebcce492SXuan Zhuo virtnet_napi_tx_disable(&sq->napi); 2418ebcce492SXuan Zhuo 2419ebcce492SXuan Zhuo txq = netdev_get_tx_queue(vi->dev, qindex); 2420ebcce492SXuan Zhuo 2421ebcce492SXuan Zhuo /* 1. wait all ximt complete 2422ebcce492SXuan Zhuo * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue() 2423ebcce492SXuan Zhuo */ 2424ebcce492SXuan Zhuo __netif_tx_lock_bh(txq); 2425ebcce492SXuan Zhuo 2426ebcce492SXuan Zhuo /* Prevent rx poll from accessing sq. */ 2427ebcce492SXuan Zhuo sq->reset = true; 2428ebcce492SXuan Zhuo 2429ebcce492SXuan Zhuo /* Prevent the upper layer from trying to send packets. */ 2430ebcce492SXuan Zhuo netif_stop_subqueue(vi->dev, qindex); 2431ebcce492SXuan Zhuo 2432ebcce492SXuan Zhuo __netif_tx_unlock_bh(txq); 2433ebcce492SXuan Zhuo 2434ebcce492SXuan Zhuo err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf); 2435ebcce492SXuan Zhuo if (err) 2436ebcce492SXuan Zhuo netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err); 2437ebcce492SXuan Zhuo 2438ebcce492SXuan Zhuo __netif_tx_lock_bh(txq); 2439ebcce492SXuan Zhuo sq->reset = false; 2440ebcce492SXuan Zhuo netif_tx_wake_queue(txq); 2441ebcce492SXuan Zhuo __netif_tx_unlock_bh(txq); 2442ebcce492SXuan Zhuo 2443ebcce492SXuan Zhuo if (running) 2444ebcce492SXuan Zhuo virtnet_napi_tx_enable(vi, sq->vq, &sq->napi); 2445ebcce492SXuan Zhuo return err; 2446ebcce492SXuan Zhuo } 2447ebcce492SXuan Zhuo 244840cbfc37SAmos Kong /* 244940cbfc37SAmos Kong * Send command via the control virtqueue and check status. Commands 245040cbfc37SAmos Kong * supported by the hypervisor, as indicated by feature bits, should 2451788a8b6dSstephen hemminger * never fail unless improperly formatted. 245240cbfc37SAmos Kong */ 245340cbfc37SAmos Kong static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 2454d24bae32Sstephen hemminger struct scatterlist *out) 245540cbfc37SAmos Kong { 2456f7bc9594SRusty Russell struct scatterlist *sgs[4], hdr, stat; 2457d24bae32Sstephen hemminger unsigned out_num = 0, tmp; 2458222722bcSYunjian Wang int ret; 245940cbfc37SAmos Kong 246040cbfc37SAmos Kong /* Caller should know better */ 2461f7bc9594SRusty Russell BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 246240cbfc37SAmos Kong 246312e57169SMichael S. Tsirkin vi->ctrl->status = ~0; 246412e57169SMichael S. Tsirkin vi->ctrl->hdr.class = class; 246512e57169SMichael S. Tsirkin vi->ctrl->hdr.cmd = cmd; 2466f7bc9594SRusty Russell /* Add header */ 246712e57169SMichael S. Tsirkin sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 2468f7bc9594SRusty Russell sgs[out_num++] = &hdr; 246940cbfc37SAmos Kong 2470f7bc9594SRusty Russell if (out) 2471f7bc9594SRusty Russell sgs[out_num++] = out; 247240cbfc37SAmos Kong 2473f7bc9594SRusty Russell /* Add return status. */ 247412e57169SMichael S. Tsirkin sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 2475d24bae32Sstephen hemminger sgs[out_num] = &stat; 247640cbfc37SAmos Kong 2477d24bae32Sstephen hemminger BUG_ON(out_num + 1 > ARRAY_SIZE(sgs)); 2478222722bcSYunjian Wang ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC); 2479222722bcSYunjian Wang if (ret < 0) { 2480222722bcSYunjian Wang dev_warn(&vi->vdev->dev, 2481222722bcSYunjian Wang "Failed to add sgs for command vq: %d\n.", ret); 2482222722bcSYunjian Wang return false; 2483222722bcSYunjian Wang } 248440cbfc37SAmos Kong 248567975901SHeinz Graalfs if (unlikely(!virtqueue_kick(vi->cvq))) 248612e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 248740cbfc37SAmos Kong 248840cbfc37SAmos Kong /* Spin for a response, the kick causes an ioport write, trapping 248940cbfc37SAmos Kong * into the hypervisor, so the request should be handled immediately. 249040cbfc37SAmos Kong */ 2491047b9b94SHeinz Graalfs while (!virtqueue_get_buf(vi->cvq, &tmp) && 2492047b9b94SHeinz Graalfs !virtqueue_is_broken(vi->cvq)) 249340cbfc37SAmos Kong cpu_relax(); 249440cbfc37SAmos Kong 249512e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 249640cbfc37SAmos Kong } 249740cbfc37SAmos Kong 24989c46f6d4SAlex Williamson static int virtnet_set_mac_address(struct net_device *dev, void *p) 24999c46f6d4SAlex Williamson { 25009c46f6d4SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 25019c46f6d4SAlex Williamson struct virtio_device *vdev = vi->vdev; 2502f2f2c8b4SJiri Pirko int ret; 2503e37e2ff3SAndy Lutomirski struct sockaddr *addr; 25047e58d5aeSAmos Kong struct scatterlist sg; 25059c46f6d4SAlex Williamson 2506ba5e4426SSridhar Samudrala if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 2507ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 2508ba5e4426SSridhar Samudrala 2509801822d1SShyam Saini addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 2510e37e2ff3SAndy Lutomirski if (!addr) 2511e37e2ff3SAndy Lutomirski return -ENOMEM; 2512e37e2ff3SAndy Lutomirski 2513e37e2ff3SAndy Lutomirski ret = eth_prepare_mac_addr_change(dev, addr); 2514f2f2c8b4SJiri Pirko if (ret) 2515e37e2ff3SAndy Lutomirski goto out; 25169c46f6d4SAlex Williamson 25177e58d5aeSAmos Kong if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 25187e58d5aeSAmos Kong sg_init_one(&sg, addr->sa_data, dev->addr_len); 25197e58d5aeSAmos Kong if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 2520d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 25217e58d5aeSAmos Kong dev_warn(&vdev->dev, 25227e58d5aeSAmos Kong "Failed to set mac address by vq command.\n"); 2523e37e2ff3SAndy Lutomirski ret = -EINVAL; 2524e37e2ff3SAndy Lutomirski goto out; 25257e58d5aeSAmos Kong } 25267e93a02fSMichael S. Tsirkin } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 25277e93a02fSMichael S. Tsirkin !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2528855e0c52SRusty Russell unsigned int i; 2529855e0c52SRusty Russell 2530855e0c52SRusty Russell /* Naturally, this has an atomicity problem. */ 2531855e0c52SRusty Russell for (i = 0; i < dev->addr_len; i++) 2532855e0c52SRusty Russell virtio_cwrite8(vdev, 2533855e0c52SRusty Russell offsetof(struct virtio_net_config, mac) + 2534855e0c52SRusty Russell i, addr->sa_data[i]); 25357e58d5aeSAmos Kong } 25367e58d5aeSAmos Kong 25377e58d5aeSAmos Kong eth_commit_mac_addr_change(dev, p); 2538e37e2ff3SAndy Lutomirski ret = 0; 25399c46f6d4SAlex Williamson 2540e37e2ff3SAndy Lutomirski out: 2541e37e2ff3SAndy Lutomirski kfree(addr); 2542e37e2ff3SAndy Lutomirski return ret; 25439c46f6d4SAlex Williamson } 25449c46f6d4SAlex Williamson 2545bc1f4470Sstephen hemminger static void virtnet_stats(struct net_device *dev, 25463fa2a1dfSstephen hemminger struct rtnl_link_stats64 *tot) 25473fa2a1dfSstephen hemminger { 25483fa2a1dfSstephen hemminger struct virtnet_info *vi = netdev_priv(dev); 25493fa2a1dfSstephen hemminger unsigned int start; 2550d7dfc5cfSToshiaki Makita int i; 25513fa2a1dfSstephen hemminger 2552d7dfc5cfSToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 2553a520794bSTony Lu u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops; 2554d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 2555d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 25563fa2a1dfSstephen hemminger 25573fa2a1dfSstephen hemminger do { 2558068c38adSThomas Gleixner start = u64_stats_fetch_begin(&sq->stats.syncp); 255927debe3eSEric Dumazet tpackets = u64_stats_read(&sq->stats.packets); 256027debe3eSEric Dumazet tbytes = u64_stats_read(&sq->stats.bytes); 256127debe3eSEric Dumazet terrors = u64_stats_read(&sq->stats.tx_timeouts); 2562068c38adSThomas Gleixner } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 256383a27052SEric Dumazet 256483a27052SEric Dumazet do { 2565068c38adSThomas Gleixner start = u64_stats_fetch_begin(&rq->stats.syncp); 256627debe3eSEric Dumazet rpackets = u64_stats_read(&rq->stats.packets); 256727debe3eSEric Dumazet rbytes = u64_stats_read(&rq->stats.bytes); 256827debe3eSEric Dumazet rdrops = u64_stats_read(&rq->stats.drops); 2569068c38adSThomas Gleixner } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 25703fa2a1dfSstephen hemminger 25713fa2a1dfSstephen hemminger tot->rx_packets += rpackets; 25723fa2a1dfSstephen hemminger tot->tx_packets += tpackets; 25733fa2a1dfSstephen hemminger tot->rx_bytes += rbytes; 25743fa2a1dfSstephen hemminger tot->tx_bytes += tbytes; 25752c4a2f7dSToshiaki Makita tot->rx_dropped += rdrops; 2576a520794bSTony Lu tot->tx_errors += terrors; 25773fa2a1dfSstephen hemminger } 25783fa2a1dfSstephen hemminger 2579*e2e5c2a3SEric Dumazet tot->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 2580*e2e5c2a3SEric Dumazet tot->tx_fifo_errors = DEV_STATS_READ(dev, tx_fifo_errors); 2581*e2e5c2a3SEric Dumazet tot->rx_length_errors = DEV_STATS_READ(dev, rx_length_errors); 2582*e2e5c2a3SEric Dumazet tot->rx_frame_errors = DEV_STATS_READ(dev, rx_frame_errors); 25833fa2a1dfSstephen hemminger } 25843fa2a1dfSstephen hemminger 2585586d17c5SJason Wang static void virtnet_ack_link_announce(struct virtnet_info *vi) 2586586d17c5SJason Wang { 2587586d17c5SJason Wang rtnl_lock(); 2588586d17c5SJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 2589d24bae32Sstephen hemminger VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 2590586d17c5SJason Wang dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 2591586d17c5SJason Wang rtnl_unlock(); 2592586d17c5SJason Wang } 2593586d17c5SJason Wang 259447315329SJohn Fastabend static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 2595986a4f4dSJason Wang { 2596986a4f4dSJason Wang struct scatterlist sg; 2597986a4f4dSJason Wang struct net_device *dev = vi->dev; 2598986a4f4dSJason Wang 2599986a4f4dSJason Wang if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 2600986a4f4dSJason Wang return 0; 2601986a4f4dSJason Wang 260212e57169SMichael S. Tsirkin vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 260312e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq)); 2604986a4f4dSJason Wang 2605986a4f4dSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 2606d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 2607986a4f4dSJason Wang dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 2608986a4f4dSJason Wang queue_pairs); 2609986a4f4dSJason Wang return -EINVAL; 261055257d72SSasha Levin } else { 2611986a4f4dSJason Wang vi->curr_queue_pairs = queue_pairs; 261235ed159bSJason Wang /* virtnet_open() will refill when device is going to up. */ 261335ed159bSJason Wang if (dev->flags & IFF_UP) 26149b9cd802SJason Wang schedule_delayed_work(&vi->refill, 0); 261555257d72SSasha Levin } 2616986a4f4dSJason Wang 2617986a4f4dSJason Wang return 0; 2618986a4f4dSJason Wang } 2619986a4f4dSJason Wang 262047315329SJohn Fastabend static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 262147315329SJohn Fastabend { 262247315329SJohn Fastabend int err; 262347315329SJohn Fastabend 262447315329SJohn Fastabend rtnl_lock(); 262547315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 262647315329SJohn Fastabend rtnl_unlock(); 262747315329SJohn Fastabend return err; 262847315329SJohn Fastabend } 262947315329SJohn Fastabend 2630296f96fcSRusty Russell static int virtnet_close(struct net_device *dev) 2631296f96fcSRusty Russell { 2632296f96fcSRusty Russell struct virtnet_info *vi = netdev_priv(dev); 2633986a4f4dSJason Wang int i; 2634296f96fcSRusty Russell 26355a159128SJason Wang /* Make sure NAPI doesn't schedule refill work */ 26365a159128SJason Wang disable_delayed_refill(vi); 2637b2baed69SRusty Russell /* Make sure refill_work doesn't re-enable napi! */ 2638b2baed69SRusty Russell cancel_delayed_work_sync(&vi->refill); 2639986a4f4dSJason Wang 26405306623aSFeng Liu for (i = 0; i < vi->max_queue_pairs; i++) 26415306623aSFeng Liu virtnet_disable_queue_pair(vi, i); 2642296f96fcSRusty Russell 2643296f96fcSRusty Russell return 0; 2644296f96fcSRusty Russell } 2645296f96fcSRusty Russell 26462af7698eSAlex Williamson static void virtnet_set_rx_mode(struct net_device *dev) 26472af7698eSAlex Williamson { 26482af7698eSAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 2649f565a7c2SAlex Williamson struct scatterlist sg[2]; 2650f565a7c2SAlex Williamson struct virtio_net_ctrl_mac *mac_data; 2651ccffad25SJiri Pirko struct netdev_hw_addr *ha; 265232e7bfc4SJiri Pirko int uc_count; 26534cd24eafSJiri Pirko int mc_count; 2654f565a7c2SAlex Williamson void *buf; 2655f565a7c2SAlex Williamson int i; 26562af7698eSAlex Williamson 2657788a8b6dSstephen hemminger /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 26582af7698eSAlex Williamson if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 26592af7698eSAlex Williamson return; 26602af7698eSAlex Williamson 266112e57169SMichael S. Tsirkin vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0); 266212e57169SMichael S. Tsirkin vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0); 26632af7698eSAlex Williamson 266412e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc)); 26652af7698eSAlex Williamson 26662af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 2667d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_PROMISC, sg)) 26682af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 266912e57169SMichael S. Tsirkin vi->ctrl->promisc ? "en" : "dis"); 26702af7698eSAlex Williamson 267112e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti)); 26722af7698eSAlex Williamson 26732af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 2674d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 26752af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 267612e57169SMichael S. Tsirkin vi->ctrl->allmulti ? "en" : "dis"); 2677f565a7c2SAlex Williamson 267832e7bfc4SJiri Pirko uc_count = netdev_uc_count(dev); 26794cd24eafSJiri Pirko mc_count = netdev_mc_count(dev); 2680f565a7c2SAlex Williamson /* MAC filter - use one buffer for both lists */ 26814cd24eafSJiri Pirko buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 2682f565a7c2SAlex Williamson (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 26834cd24eafSJiri Pirko mac_data = buf; 2684e68ed8f0SJoe Perches if (!buf) 2685f565a7c2SAlex Williamson return; 2686f565a7c2SAlex Williamson 268723e258e1SAlex Williamson sg_init_table(sg, 2); 268823e258e1SAlex Williamson 2689f565a7c2SAlex Williamson /* Store the unicast list and count in the front of the buffer */ 2690fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 2691ccffad25SJiri Pirko i = 0; 269232e7bfc4SJiri Pirko netdev_for_each_uc_addr(ha, dev) 2693ccffad25SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2694f565a7c2SAlex Williamson 2695f565a7c2SAlex Williamson sg_set_buf(&sg[0], mac_data, 269632e7bfc4SJiri Pirko sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 2697f565a7c2SAlex Williamson 2698f565a7c2SAlex Williamson /* multicast list and count fill the end */ 269932e7bfc4SJiri Pirko mac_data = (void *)&mac_data->macs[uc_count][0]; 2700f565a7c2SAlex Williamson 2701fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 2702567ec874SJiri Pirko i = 0; 270322bedad3SJiri Pirko netdev_for_each_mc_addr(ha, dev) 270422bedad3SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 2705f565a7c2SAlex Williamson 2706f565a7c2SAlex Williamson sg_set_buf(&sg[1], mac_data, 27074cd24eafSJiri Pirko sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 2708f565a7c2SAlex Williamson 2709f565a7c2SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 2710d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 271199e872aeSThomas Huth dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 2712f565a7c2SAlex Williamson 2713f565a7c2SAlex Williamson kfree(buf); 27142af7698eSAlex Williamson } 27152af7698eSAlex Williamson 271680d5c368SPatrick McHardy static int virtnet_vlan_rx_add_vid(struct net_device *dev, 271780d5c368SPatrick McHardy __be16 proto, u16 vid) 27180bde9569SAlex Williamson { 27190bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 27200bde9569SAlex Williamson struct scatterlist sg; 27210bde9569SAlex Williamson 2722d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 272312e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 27240bde9569SAlex Williamson 27250bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2726d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 27270bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 27288e586137SJiri Pirko return 0; 27290bde9569SAlex Williamson } 27300bde9569SAlex Williamson 273180d5c368SPatrick McHardy static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 273280d5c368SPatrick McHardy __be16 proto, u16 vid) 27330bde9569SAlex Williamson { 27340bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 27350bde9569SAlex Williamson struct scatterlist sg; 27360bde9569SAlex Williamson 2737d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 273812e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 27390bde9569SAlex Williamson 27400bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 2741d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 27420bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 27438e586137SJiri Pirko return 0; 27440bde9569SAlex Williamson } 27450bde9569SAlex Williamson 2746310974faSPeter Xu static void virtnet_clean_affinity(struct virtnet_info *vi) 2747986a4f4dSJason Wang { 2748986a4f4dSJason Wang int i; 27498898c21cSWanlong Gao 27508898c21cSWanlong Gao if (vi->affinity_hint_set) { 27518898c21cSWanlong Gao for (i = 0; i < vi->max_queue_pairs; i++) { 275219e226e8SCaleb Raitto virtqueue_set_affinity(vi->rq[i].vq, NULL); 275319e226e8SCaleb Raitto virtqueue_set_affinity(vi->sq[i].vq, NULL); 27548898c21cSWanlong Gao } 27558898c21cSWanlong Gao 27568898c21cSWanlong Gao vi->affinity_hint_set = false; 27578898c21cSWanlong Gao } 27588898c21cSWanlong Gao } 27598898c21cSWanlong Gao 27608898c21cSWanlong Gao static void virtnet_set_affinity(struct virtnet_info *vi) 2761986a4f4dSJason Wang { 27622ca653d6SCaleb Raitto cpumask_var_t mask; 27632ca653d6SCaleb Raitto int stragglers; 27642ca653d6SCaleb Raitto int group_size; 27652ca653d6SCaleb Raitto int i, j, cpu; 27662ca653d6SCaleb Raitto int num_cpu; 27672ca653d6SCaleb Raitto int stride; 2768986a4f4dSJason Wang 27692ca653d6SCaleb Raitto if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 2770310974faSPeter Xu virtnet_clean_affinity(vi); 2771986a4f4dSJason Wang return; 2772986a4f4dSJason Wang } 2773986a4f4dSJason Wang 27742ca653d6SCaleb Raitto num_cpu = num_online_cpus(); 27752ca653d6SCaleb Raitto stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); 27762ca653d6SCaleb Raitto stragglers = num_cpu >= vi->curr_queue_pairs ? 27772ca653d6SCaleb Raitto num_cpu % vi->curr_queue_pairs : 27782ca653d6SCaleb Raitto 0; 27799b51d9d8SYury Norov cpu = cpumask_first(cpu_online_mask); 27804d99f660SAndrei Vagin 27812ca653d6SCaleb Raitto for (i = 0; i < vi->curr_queue_pairs; i++) { 27822ca653d6SCaleb Raitto group_size = stride + (i < stragglers ? 1 : 0); 27832ca653d6SCaleb Raitto 27842ca653d6SCaleb Raitto for (j = 0; j < group_size; j++) { 27852ca653d6SCaleb Raitto cpumask_set_cpu(cpu, mask); 27862ca653d6SCaleb Raitto cpu = cpumask_next_wrap(cpu, cpu_online_mask, 27872ca653d6SCaleb Raitto nr_cpu_ids, false); 27882ca653d6SCaleb Raitto } 27892ca653d6SCaleb Raitto virtqueue_set_affinity(vi->rq[i].vq, mask); 27902ca653d6SCaleb Raitto virtqueue_set_affinity(vi->sq[i].vq, mask); 2791044ab86dSAntoine Tenart __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); 27922ca653d6SCaleb Raitto cpumask_clear(mask); 2793986a4f4dSJason Wang } 2794986a4f4dSJason Wang 2795986a4f4dSJason Wang vi->affinity_hint_set = true; 27962ca653d6SCaleb Raitto free_cpumask_var(mask); 279747be2479SWanlong Gao } 2798986a4f4dSJason Wang 27998017c279SSebastian Andrzej Siewior static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 28008de4b2f3SWanlong Gao { 28018017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 28028017c279SSebastian Andrzej Siewior node); 28038de4b2f3SWanlong Gao virtnet_set_affinity(vi); 28048017c279SSebastian Andrzej Siewior return 0; 28058de4b2f3SWanlong Gao } 28063ab098dfSJason Wang 28078017c279SSebastian Andrzej Siewior static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 28088017c279SSebastian Andrzej Siewior { 28098017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 28108017c279SSebastian Andrzej Siewior node_dead); 28118017c279SSebastian Andrzej Siewior virtnet_set_affinity(vi); 28128017c279SSebastian Andrzej Siewior return 0; 28138017c279SSebastian Andrzej Siewior } 28148017c279SSebastian Andrzej Siewior 28158017c279SSebastian Andrzej Siewior static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 28168017c279SSebastian Andrzej Siewior { 28178017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 28188017c279SSebastian Andrzej Siewior node); 28198017c279SSebastian Andrzej Siewior 2820310974faSPeter Xu virtnet_clean_affinity(vi); 28218017c279SSebastian Andrzej Siewior return 0; 28228017c279SSebastian Andrzej Siewior } 28238017c279SSebastian Andrzej Siewior 28248017c279SSebastian Andrzej Siewior static enum cpuhp_state virtionet_online; 28258017c279SSebastian Andrzej Siewior 28268017c279SSebastian Andrzej Siewior static int virtnet_cpu_notif_add(struct virtnet_info *vi) 28278017c279SSebastian Andrzej Siewior { 28288017c279SSebastian Andrzej Siewior int ret; 28298017c279SSebastian Andrzej Siewior 28308017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 28318017c279SSebastian Andrzej Siewior if (ret) 28328017c279SSebastian Andrzej Siewior return ret; 28338017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 28348017c279SSebastian Andrzej Siewior &vi->node_dead); 28358017c279SSebastian Andrzej Siewior if (!ret) 28368017c279SSebastian Andrzej Siewior return ret; 28378017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 28388017c279SSebastian Andrzej Siewior return ret; 28398017c279SSebastian Andrzej Siewior } 28408017c279SSebastian Andrzej Siewior 28418017c279SSebastian Andrzej Siewior static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 28428017c279SSebastian Andrzej Siewior { 28438017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 28448017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 28458017c279SSebastian Andrzej Siewior &vi->node_dead); 2846a9ea3fc6SHerbert Xu } 2847a9ea3fc6SHerbert Xu 28488f9f4668SRick Jones static void virtnet_get_ringparam(struct net_device *dev, 284974624944SHao Chen struct ethtool_ringparam *ring, 285074624944SHao Chen struct kernel_ethtool_ringparam *kernel_ring, 285174624944SHao Chen struct netlink_ext_ack *extack) 28528f9f4668SRick Jones { 28538f9f4668SRick Jones struct virtnet_info *vi = netdev_priv(dev); 28548f9f4668SRick Jones 28558597b5ddSXuan Zhuo ring->rx_max_pending = vi->rq[0].vq->num_max; 28568597b5ddSXuan Zhuo ring->tx_max_pending = vi->sq[0].vq->num_max; 28578597b5ddSXuan Zhuo ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 28588597b5ddSXuan Zhuo ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 28598f9f4668SRick Jones } 28608f9f4668SRick Jones 286139d591b0SHeng Qi static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi, 286239d591b0SHeng Qi u16 vqn, u32 max_usecs, u32 max_packets); 286339d591b0SHeng Qi 2864a335b33fSXuan Zhuo static int virtnet_set_ringparam(struct net_device *dev, 2865a335b33fSXuan Zhuo struct ethtool_ringparam *ring, 2866a335b33fSXuan Zhuo struct kernel_ethtool_ringparam *kernel_ring, 2867a335b33fSXuan Zhuo struct netlink_ext_ack *extack) 2868a335b33fSXuan Zhuo { 2869a335b33fSXuan Zhuo struct virtnet_info *vi = netdev_priv(dev); 2870a335b33fSXuan Zhuo u32 rx_pending, tx_pending; 2871a335b33fSXuan Zhuo struct receive_queue *rq; 2872a335b33fSXuan Zhuo struct send_queue *sq; 2873a335b33fSXuan Zhuo int i, err; 2874a335b33fSXuan Zhuo 2875a335b33fSXuan Zhuo if (ring->rx_mini_pending || ring->rx_jumbo_pending) 2876a335b33fSXuan Zhuo return -EINVAL; 2877a335b33fSXuan Zhuo 2878a335b33fSXuan Zhuo rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 2879a335b33fSXuan Zhuo tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 2880a335b33fSXuan Zhuo 2881a335b33fSXuan Zhuo if (ring->rx_pending == rx_pending && 2882a335b33fSXuan Zhuo ring->tx_pending == tx_pending) 2883a335b33fSXuan Zhuo return 0; 2884a335b33fSXuan Zhuo 2885a335b33fSXuan Zhuo if (ring->rx_pending > vi->rq[0].vq->num_max) 2886a335b33fSXuan Zhuo return -EINVAL; 2887a335b33fSXuan Zhuo 2888a335b33fSXuan Zhuo if (ring->tx_pending > vi->sq[0].vq->num_max) 2889a335b33fSXuan Zhuo return -EINVAL; 2890a335b33fSXuan Zhuo 2891a335b33fSXuan Zhuo for (i = 0; i < vi->max_queue_pairs; i++) { 2892a335b33fSXuan Zhuo rq = vi->rq + i; 2893a335b33fSXuan Zhuo sq = vi->sq + i; 2894a335b33fSXuan Zhuo 2895a335b33fSXuan Zhuo if (ring->tx_pending != tx_pending) { 2896a335b33fSXuan Zhuo err = virtnet_tx_resize(vi, sq, ring->tx_pending); 2897a335b33fSXuan Zhuo if (err) 2898a335b33fSXuan Zhuo return err; 289939d591b0SHeng Qi 290039d591b0SHeng Qi /* Upon disabling and re-enabling a transmit virtqueue, the device must 290139d591b0SHeng Qi * set the coalescing parameters of the virtqueue to those configured 290239d591b0SHeng Qi * through the VIRTIO_NET_CTRL_NOTF_COAL_TX_SET command, or, if the driver 290339d591b0SHeng Qi * did not set any TX coalescing parameters, to 0. 290439d591b0SHeng Qi */ 290539d591b0SHeng Qi err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(i), 290639d591b0SHeng Qi vi->intr_coal_tx.max_usecs, 290739d591b0SHeng Qi vi->intr_coal_tx.max_packets); 290839d591b0SHeng Qi if (err) 290939d591b0SHeng Qi return err; 291039d591b0SHeng Qi 291139d591b0SHeng Qi vi->sq[i].intr_coal.max_usecs = vi->intr_coal_tx.max_usecs; 291239d591b0SHeng Qi vi->sq[i].intr_coal.max_packets = vi->intr_coal_tx.max_packets; 2913a335b33fSXuan Zhuo } 2914a335b33fSXuan Zhuo 2915a335b33fSXuan Zhuo if (ring->rx_pending != rx_pending) { 2916a335b33fSXuan Zhuo err = virtnet_rx_resize(vi, rq, ring->rx_pending); 2917a335b33fSXuan Zhuo if (err) 2918a335b33fSXuan Zhuo return err; 291939d591b0SHeng Qi 292039d591b0SHeng Qi /* The reason is same as the transmit virtqueue reset */ 292139d591b0SHeng Qi err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(i), 292239d591b0SHeng Qi vi->intr_coal_rx.max_usecs, 292339d591b0SHeng Qi vi->intr_coal_rx.max_packets); 292439d591b0SHeng Qi if (err) 292539d591b0SHeng Qi return err; 292639d591b0SHeng Qi 292739d591b0SHeng Qi vi->rq[i].intr_coal.max_usecs = vi->intr_coal_rx.max_usecs; 292839d591b0SHeng Qi vi->rq[i].intr_coal.max_packets = vi->intr_coal_rx.max_packets; 2929a335b33fSXuan Zhuo } 2930a335b33fSXuan Zhuo } 2931a335b33fSXuan Zhuo 2932a335b33fSXuan Zhuo return 0; 2933a9ea3fc6SHerbert Xu } 2934a9ea3fc6SHerbert Xu 2935c7114b12SAndrew Melnychenko static bool virtnet_commit_rss_command(struct virtnet_info *vi) 2936c7114b12SAndrew Melnychenko { 2937c7114b12SAndrew Melnychenko struct net_device *dev = vi->dev; 2938c7114b12SAndrew Melnychenko struct scatterlist sgs[4]; 2939c7114b12SAndrew Melnychenko unsigned int sg_buf_size; 2940c7114b12SAndrew Melnychenko 2941c7114b12SAndrew Melnychenko /* prepare sgs */ 2942c7114b12SAndrew Melnychenko sg_init_table(sgs, 4); 2943c7114b12SAndrew Melnychenko 2944c7114b12SAndrew Melnychenko sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table); 2945c7114b12SAndrew Melnychenko sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size); 2946c7114b12SAndrew Melnychenko 2947c7114b12SAndrew Melnychenko sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1); 2948c7114b12SAndrew Melnychenko sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size); 2949c7114b12SAndrew Melnychenko 2950c7114b12SAndrew Melnychenko sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key) 2951c7114b12SAndrew Melnychenko - offsetof(struct virtio_net_ctrl_rss, max_tx_vq); 2952c7114b12SAndrew Melnychenko sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size); 2953c7114b12SAndrew Melnychenko 2954c7114b12SAndrew Melnychenko sg_buf_size = vi->rss_key_size; 2955c7114b12SAndrew Melnychenko sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size); 2956c7114b12SAndrew Melnychenko 2957c7114b12SAndrew Melnychenko if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 295891f41f01SAndrew Melnychenko vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG 295991f41f01SAndrew Melnychenko : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) { 2960c7114b12SAndrew Melnychenko dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n"); 2961c7114b12SAndrew Melnychenko return false; 2962c7114b12SAndrew Melnychenko } 2963c7114b12SAndrew Melnychenko return true; 2964c7114b12SAndrew Melnychenko } 2965c7114b12SAndrew Melnychenko 2966c7114b12SAndrew Melnychenko static void virtnet_init_default_rss(struct virtnet_info *vi) 2967c7114b12SAndrew Melnychenko { 2968c7114b12SAndrew Melnychenko u32 indir_val = 0; 2969c7114b12SAndrew Melnychenko int i = 0; 2970c7114b12SAndrew Melnychenko 2971c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_types = vi->rss_hash_types_supported; 2972c1170820SAndrew Melnychenko vi->rss_hash_types_saved = vi->rss_hash_types_supported; 2973c7114b12SAndrew Melnychenko vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size 2974c7114b12SAndrew Melnychenko ? vi->rss_indir_table_size - 1 : 0; 2975c7114b12SAndrew Melnychenko vi->ctrl->rss.unclassified_queue = 0; 2976c7114b12SAndrew Melnychenko 2977c7114b12SAndrew Melnychenko for (; i < vi->rss_indir_table_size; ++i) { 2978c7114b12SAndrew Melnychenko indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs); 2979c7114b12SAndrew Melnychenko vi->ctrl->rss.indirection_table[i] = indir_val; 2980c7114b12SAndrew Melnychenko } 2981c7114b12SAndrew Melnychenko 29822c507ce9SHawkins Jiawei vi->ctrl->rss.max_tx_vq = vi->has_rss ? vi->curr_queue_pairs : 0; 2983c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_key_length = vi->rss_key_size; 2984c7114b12SAndrew Melnychenko 2985c7114b12SAndrew Melnychenko netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size); 2986c7114b12SAndrew Melnychenko } 2987c7114b12SAndrew Melnychenko 2988c1170820SAndrew Melnychenko static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info) 2989c1170820SAndrew Melnychenko { 2990c1170820SAndrew Melnychenko info->data = 0; 2991c1170820SAndrew Melnychenko switch (info->flow_type) { 2992c1170820SAndrew Melnychenko case TCP_V4_FLOW: 2993c1170820SAndrew Melnychenko if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { 2994c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST | 2995c1170820SAndrew Melnychenko RXH_L4_B_0_1 | RXH_L4_B_2_3; 2996c1170820SAndrew Melnychenko } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 2997c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST; 2998c1170820SAndrew Melnychenko } 2999c1170820SAndrew Melnychenko break; 3000c1170820SAndrew Melnychenko case TCP_V6_FLOW: 3001c1170820SAndrew Melnychenko if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { 3002c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST | 3003c1170820SAndrew Melnychenko RXH_L4_B_0_1 | RXH_L4_B_2_3; 3004c1170820SAndrew Melnychenko } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 3005c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST; 3006c1170820SAndrew Melnychenko } 3007c1170820SAndrew Melnychenko break; 3008c1170820SAndrew Melnychenko case UDP_V4_FLOW: 3009c1170820SAndrew Melnychenko if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { 3010c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST | 3011c1170820SAndrew Melnychenko RXH_L4_B_0_1 | RXH_L4_B_2_3; 3012c1170820SAndrew Melnychenko } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 3013c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST; 3014c1170820SAndrew Melnychenko } 3015c1170820SAndrew Melnychenko break; 3016c1170820SAndrew Melnychenko case UDP_V6_FLOW: 3017c1170820SAndrew Melnychenko if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { 3018c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST | 3019c1170820SAndrew Melnychenko RXH_L4_B_0_1 | RXH_L4_B_2_3; 3020c1170820SAndrew Melnychenko } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 3021c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST; 3022c1170820SAndrew Melnychenko } 3023c1170820SAndrew Melnychenko break; 3024c1170820SAndrew Melnychenko case IPV4_FLOW: 3025c1170820SAndrew Melnychenko if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) 3026c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST; 3027c1170820SAndrew Melnychenko 3028c1170820SAndrew Melnychenko break; 3029c1170820SAndrew Melnychenko case IPV6_FLOW: 3030c1170820SAndrew Melnychenko if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) 3031c1170820SAndrew Melnychenko info->data = RXH_IP_SRC | RXH_IP_DST; 3032c1170820SAndrew Melnychenko 3033c1170820SAndrew Melnychenko break; 3034c1170820SAndrew Melnychenko default: 3035c1170820SAndrew Melnychenko info->data = 0; 3036c1170820SAndrew Melnychenko break; 3037c1170820SAndrew Melnychenko } 3038c1170820SAndrew Melnychenko } 3039c1170820SAndrew Melnychenko 3040c1170820SAndrew Melnychenko static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info) 3041c1170820SAndrew Melnychenko { 3042c1170820SAndrew Melnychenko u32 new_hashtypes = vi->rss_hash_types_saved; 3043c1170820SAndrew Melnychenko bool is_disable = info->data & RXH_DISCARD; 3044c1170820SAndrew Melnychenko bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3); 3045c1170820SAndrew Melnychenko 3046c1170820SAndrew Melnychenko /* supports only 'sd', 'sdfn' and 'r' */ 3047c1170820SAndrew Melnychenko if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable)) 3048c1170820SAndrew Melnychenko return false; 3049c1170820SAndrew Melnychenko 3050c1170820SAndrew Melnychenko switch (info->flow_type) { 3051c1170820SAndrew Melnychenko case TCP_V4_FLOW: 3052c1170820SAndrew Melnychenko new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4); 3053c1170820SAndrew Melnychenko if (!is_disable) 3054c1170820SAndrew Melnychenko new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 3055c1170820SAndrew Melnychenko | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0); 3056c1170820SAndrew Melnychenko break; 3057c1170820SAndrew Melnychenko case UDP_V4_FLOW: 3058c1170820SAndrew Melnychenko new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4); 3059c1170820SAndrew Melnychenko if (!is_disable) 3060c1170820SAndrew Melnychenko new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 3061c1170820SAndrew Melnychenko | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0); 3062c1170820SAndrew Melnychenko break; 3063c1170820SAndrew Melnychenko case IPV4_FLOW: 3064c1170820SAndrew Melnychenko new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4; 3065c1170820SAndrew Melnychenko if (!is_disable) 3066c1170820SAndrew Melnychenko new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4; 3067c1170820SAndrew Melnychenko break; 3068c1170820SAndrew Melnychenko case TCP_V6_FLOW: 3069c1170820SAndrew Melnychenko new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6); 3070c1170820SAndrew Melnychenko if (!is_disable) 3071c1170820SAndrew Melnychenko new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 3072c1170820SAndrew Melnychenko | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0); 3073c1170820SAndrew Melnychenko break; 3074c1170820SAndrew Melnychenko case UDP_V6_FLOW: 3075c1170820SAndrew Melnychenko new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6); 3076c1170820SAndrew Melnychenko if (!is_disable) 3077c1170820SAndrew Melnychenko new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 3078c1170820SAndrew Melnychenko | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0); 3079c1170820SAndrew Melnychenko break; 3080c1170820SAndrew Melnychenko case IPV6_FLOW: 3081c1170820SAndrew Melnychenko new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6; 3082c1170820SAndrew Melnychenko if (!is_disable) 3083c1170820SAndrew Melnychenko new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6; 3084c1170820SAndrew Melnychenko break; 3085c1170820SAndrew Melnychenko default: 3086c1170820SAndrew Melnychenko /* unsupported flow */ 3087c1170820SAndrew Melnychenko return false; 3088c1170820SAndrew Melnychenko } 3089c1170820SAndrew Melnychenko 3090c1170820SAndrew Melnychenko /* if unsupported hashtype was set */ 3091c1170820SAndrew Melnychenko if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported)) 3092c1170820SAndrew Melnychenko return false; 3093c1170820SAndrew Melnychenko 3094c1170820SAndrew Melnychenko if (new_hashtypes != vi->rss_hash_types_saved) { 3095c1170820SAndrew Melnychenko vi->rss_hash_types_saved = new_hashtypes; 3096c1170820SAndrew Melnychenko vi->ctrl->rss.hash_types = vi->rss_hash_types_saved; 3097c1170820SAndrew Melnychenko if (vi->dev->features & NETIF_F_RXHASH) 3098c1170820SAndrew Melnychenko return virtnet_commit_rss_command(vi); 3099c1170820SAndrew Melnychenko } 3100c1170820SAndrew Melnychenko 3101c1170820SAndrew Melnychenko return true; 3102c1170820SAndrew Melnychenko } 310366846048SRick Jones 310466846048SRick Jones static void virtnet_get_drvinfo(struct net_device *dev, 310566846048SRick Jones struct ethtool_drvinfo *info) 310666846048SRick Jones { 310766846048SRick Jones struct virtnet_info *vi = netdev_priv(dev); 310866846048SRick Jones struct virtio_device *vdev = vi->vdev; 310966846048SRick Jones 3110fb3ceec1SWolfram Sang strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 3111fb3ceec1SWolfram Sang strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 3112fb3ceec1SWolfram Sang strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 311366846048SRick Jones 311466846048SRick Jones } 311566846048SRick Jones 3116d73bcd2cSJason Wang /* TODO: Eliminate OOO packets during switching */ 3117d73bcd2cSJason Wang static int virtnet_set_channels(struct net_device *dev, 3118d73bcd2cSJason Wang struct ethtool_channels *channels) 3119d73bcd2cSJason Wang { 3120d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 3121d73bcd2cSJason Wang u16 queue_pairs = channels->combined_count; 3122d73bcd2cSJason Wang int err; 3123d73bcd2cSJason Wang 3124d73bcd2cSJason Wang /* We don't support separate rx/tx channels. 3125d73bcd2cSJason Wang * We don't allow setting 'other' channels. 3126d73bcd2cSJason Wang */ 3127d73bcd2cSJason Wang if (channels->rx_count || channels->tx_count || channels->other_count) 3128d73bcd2cSJason Wang return -EINVAL; 3129d73bcd2cSJason Wang 3130c18e9cd6SAmos Kong if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 3131d73bcd2cSJason Wang return -EINVAL; 3132d73bcd2cSJason Wang 3133f600b690SJohn Fastabend /* For now we don't support modifying channels while XDP is loaded 3134f600b690SJohn Fastabend * also when XDP is loaded all RX queues have XDP programs so we only 3135f600b690SJohn Fastabend * need to check a single RX queue. 3136f600b690SJohn Fastabend */ 3137f600b690SJohn Fastabend if (vi->rq[0].xdp_prog) 3138f600b690SJohn Fastabend return -EINVAL; 3139f600b690SJohn Fastabend 3140a0d1d0f4SSebastian Andrzej Siewior cpus_read_lock(); 314147315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 3142de33212fSJeff Dike if (err) { 3143a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 3144de33212fSJeff Dike goto err; 3145d73bcd2cSJason Wang } 3146de33212fSJeff Dike virtnet_set_affinity(vi); 3147a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 3148d73bcd2cSJason Wang 3149de33212fSJeff Dike netif_set_real_num_tx_queues(dev, queue_pairs); 3150de33212fSJeff Dike netif_set_real_num_rx_queues(dev, queue_pairs); 3151de33212fSJeff Dike err: 3152d73bcd2cSJason Wang return err; 3153d73bcd2cSJason Wang } 3154d73bcd2cSJason Wang 3155d7dfc5cfSToshiaki Makita static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 3156d7dfc5cfSToshiaki Makita { 3157d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 3158d7dfc5cfSToshiaki Makita unsigned int i, j; 3159d7a9a01bSAlexander Duyck u8 *p = data; 3160d7dfc5cfSToshiaki Makita 3161d7dfc5cfSToshiaki Makita switch (stringset) { 3162d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 3163d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 3164d7a9a01bSAlexander Duyck for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) 3165d7a9a01bSAlexander Duyck ethtool_sprintf(&p, "rx_queue_%u_%s", i, 3166d7a9a01bSAlexander Duyck virtnet_rq_stats_desc[j].desc); 3167d7dfc5cfSToshiaki Makita } 3168d7dfc5cfSToshiaki Makita 3169d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 3170d7a9a01bSAlexander Duyck for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) 3171d7a9a01bSAlexander Duyck ethtool_sprintf(&p, "tx_queue_%u_%s", i, 3172d7a9a01bSAlexander Duyck virtnet_sq_stats_desc[j].desc); 3173d7dfc5cfSToshiaki Makita } 3174d7dfc5cfSToshiaki Makita break; 3175d7dfc5cfSToshiaki Makita } 3176d7dfc5cfSToshiaki Makita } 3177d7dfc5cfSToshiaki Makita 3178d7dfc5cfSToshiaki Makita static int virtnet_get_sset_count(struct net_device *dev, int sset) 3179d7dfc5cfSToshiaki Makita { 3180d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 3181d7dfc5cfSToshiaki Makita 3182d7dfc5cfSToshiaki Makita switch (sset) { 3183d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 3184d7dfc5cfSToshiaki Makita return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN + 3185d7dfc5cfSToshiaki Makita VIRTNET_SQ_STATS_LEN); 3186d7dfc5cfSToshiaki Makita default: 3187d7dfc5cfSToshiaki Makita return -EOPNOTSUPP; 3188d7dfc5cfSToshiaki Makita } 3189d7dfc5cfSToshiaki Makita } 3190d7dfc5cfSToshiaki Makita 3191d7dfc5cfSToshiaki Makita static void virtnet_get_ethtool_stats(struct net_device *dev, 3192d7dfc5cfSToshiaki Makita struct ethtool_stats *stats, u64 *data) 3193d7dfc5cfSToshiaki Makita { 3194d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 3195d7dfc5cfSToshiaki Makita unsigned int idx = 0, start, i, j; 3196d7dfc5cfSToshiaki Makita const u8 *stats_base; 319727debe3eSEric Dumazet const u64_stats_t *p; 3198d7dfc5cfSToshiaki Makita size_t offset; 3199d7dfc5cfSToshiaki Makita 3200d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 3201d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 3202d7dfc5cfSToshiaki Makita 320327debe3eSEric Dumazet stats_base = (const u8 *)&rq->stats; 3204d7dfc5cfSToshiaki Makita do { 3205068c38adSThomas Gleixner start = u64_stats_fetch_begin(&rq->stats.syncp); 3206d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { 3207d7dfc5cfSToshiaki Makita offset = virtnet_rq_stats_desc[j].offset; 320827debe3eSEric Dumazet p = (const u64_stats_t *)(stats_base + offset); 320927debe3eSEric Dumazet data[idx + j] = u64_stats_read(p); 3210d7dfc5cfSToshiaki Makita } 3211068c38adSThomas Gleixner } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 3212d7dfc5cfSToshiaki Makita idx += VIRTNET_RQ_STATS_LEN; 3213d7dfc5cfSToshiaki Makita } 3214d7dfc5cfSToshiaki Makita 3215d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 3216d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 3217d7dfc5cfSToshiaki Makita 321827debe3eSEric Dumazet stats_base = (const u8 *)&sq->stats; 3219d7dfc5cfSToshiaki Makita do { 3220068c38adSThomas Gleixner start = u64_stats_fetch_begin(&sq->stats.syncp); 3221d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { 3222d7dfc5cfSToshiaki Makita offset = virtnet_sq_stats_desc[j].offset; 322327debe3eSEric Dumazet p = (const u64_stats_t *)(stats_base + offset); 322427debe3eSEric Dumazet data[idx + j] = u64_stats_read(p); 3225d7dfc5cfSToshiaki Makita } 3226068c38adSThomas Gleixner } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 3227d7dfc5cfSToshiaki Makita idx += VIRTNET_SQ_STATS_LEN; 3228d7dfc5cfSToshiaki Makita } 3229d7dfc5cfSToshiaki Makita } 3230d7dfc5cfSToshiaki Makita 3231d73bcd2cSJason Wang static void virtnet_get_channels(struct net_device *dev, 3232d73bcd2cSJason Wang struct ethtool_channels *channels) 3233d73bcd2cSJason Wang { 3234d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 3235d73bcd2cSJason Wang 3236d73bcd2cSJason Wang channels->combined_count = vi->curr_queue_pairs; 3237d73bcd2cSJason Wang channels->max_combined = vi->max_queue_pairs; 3238d73bcd2cSJason Wang channels->max_other = 0; 3239d73bcd2cSJason Wang channels->rx_count = 0; 3240d73bcd2cSJason Wang channels->tx_count = 0; 3241d73bcd2cSJason Wang channels->other_count = 0; 3242d73bcd2cSJason Wang } 3243d73bcd2cSJason Wang 3244ebb6b4b1SPhilippe Reynes static int virtnet_set_link_ksettings(struct net_device *dev, 3245ebb6b4b1SPhilippe Reynes const struct ethtool_link_ksettings *cmd) 324616032be5SNikolay Aleksandrov { 324716032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 324816032be5SNikolay Aleksandrov 32499aedc6e2SCris Forno return ethtool_virtdev_set_link_ksettings(dev, cmd, 32509aedc6e2SCris Forno &vi->speed, &vi->duplex); 325116032be5SNikolay Aleksandrov } 325216032be5SNikolay Aleksandrov 3253ebb6b4b1SPhilippe Reynes static int virtnet_get_link_ksettings(struct net_device *dev, 3254ebb6b4b1SPhilippe Reynes struct ethtool_link_ksettings *cmd) 325516032be5SNikolay Aleksandrov { 325616032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 325716032be5SNikolay Aleksandrov 3258ebb6b4b1SPhilippe Reynes cmd->base.speed = vi->speed; 3259ebb6b4b1SPhilippe Reynes cmd->base.duplex = vi->duplex; 3260ebb6b4b1SPhilippe Reynes cmd->base.port = PORT_OTHER; 326116032be5SNikolay Aleksandrov 326216032be5SNikolay Aleksandrov return 0; 326316032be5SNikolay Aleksandrov } 326416032be5SNikolay Aleksandrov 3265699b045aSAlvaro Karsz static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, 3266699b045aSAlvaro Karsz struct ethtool_coalesce *ec) 3267699b045aSAlvaro Karsz { 3268699b045aSAlvaro Karsz struct scatterlist sgs_tx, sgs_rx; 3269829cce76SHeng Qi int i; 3270699b045aSAlvaro Karsz 3271accc1bf2SBrett Creeley vi->ctrl->coal_tx.tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs); 3272accc1bf2SBrett Creeley vi->ctrl->coal_tx.tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames); 3273accc1bf2SBrett Creeley sg_init_one(&sgs_tx, &vi->ctrl->coal_tx, sizeof(vi->ctrl->coal_tx)); 3274699b045aSAlvaro Karsz 3275699b045aSAlvaro Karsz if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 3276699b045aSAlvaro Karsz VIRTIO_NET_CTRL_NOTF_COAL_TX_SET, 3277699b045aSAlvaro Karsz &sgs_tx)) 3278699b045aSAlvaro Karsz return -EINVAL; 3279699b045aSAlvaro Karsz 3280699b045aSAlvaro Karsz /* Save parameters */ 3281308d7982SGavin Li vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs; 3282308d7982SGavin Li vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames; 3283829cce76SHeng Qi for (i = 0; i < vi->max_queue_pairs; i++) { 3284829cce76SHeng Qi vi->sq[i].intr_coal.max_usecs = ec->tx_coalesce_usecs; 3285829cce76SHeng Qi vi->sq[i].intr_coal.max_packets = ec->tx_max_coalesced_frames; 3286829cce76SHeng Qi } 3287699b045aSAlvaro Karsz 3288accc1bf2SBrett Creeley vi->ctrl->coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); 3289accc1bf2SBrett Creeley vi->ctrl->coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); 3290accc1bf2SBrett Creeley sg_init_one(&sgs_rx, &vi->ctrl->coal_rx, sizeof(vi->ctrl->coal_rx)); 3291699b045aSAlvaro Karsz 3292699b045aSAlvaro Karsz if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 3293699b045aSAlvaro Karsz VIRTIO_NET_CTRL_NOTF_COAL_RX_SET, 3294699b045aSAlvaro Karsz &sgs_rx)) 3295699b045aSAlvaro Karsz return -EINVAL; 3296699b045aSAlvaro Karsz 3297699b045aSAlvaro Karsz /* Save parameters */ 3298308d7982SGavin Li vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs; 3299308d7982SGavin Li vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames; 3300829cce76SHeng Qi for (i = 0; i < vi->max_queue_pairs; i++) { 3301829cce76SHeng Qi vi->rq[i].intr_coal.max_usecs = ec->rx_coalesce_usecs; 3302829cce76SHeng Qi vi->rq[i].intr_coal.max_packets = ec->rx_max_coalesced_frames; 3303829cce76SHeng Qi } 3304699b045aSAlvaro Karsz 3305699b045aSAlvaro Karsz return 0; 3306699b045aSAlvaro Karsz } 3307699b045aSAlvaro Karsz 3308394bd877SGavin Li static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi, 3309394bd877SGavin Li u16 vqn, u32 max_usecs, u32 max_packets) 3310394bd877SGavin Li { 3311394bd877SGavin Li struct scatterlist sgs; 3312394bd877SGavin Li 3313394bd877SGavin Li vi->ctrl->coal_vq.vqn = cpu_to_le16(vqn); 3314394bd877SGavin Li vi->ctrl->coal_vq.coal.max_usecs = cpu_to_le32(max_usecs); 3315394bd877SGavin Li vi->ctrl->coal_vq.coal.max_packets = cpu_to_le32(max_packets); 3316394bd877SGavin Li sg_init_one(&sgs, &vi->ctrl->coal_vq, sizeof(vi->ctrl->coal_vq)); 3317394bd877SGavin Li 3318394bd877SGavin Li if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 3319394bd877SGavin Li VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET, 3320394bd877SGavin Li &sgs)) 3321394bd877SGavin Li return -EINVAL; 3322394bd877SGavin Li 3323394bd877SGavin Li return 0; 3324394bd877SGavin Li } 3325394bd877SGavin Li 3326394bd877SGavin Li static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi, 3327394bd877SGavin Li struct ethtool_coalesce *ec, 3328394bd877SGavin Li u16 queue) 3329394bd877SGavin Li { 3330394bd877SGavin Li int err; 3331394bd877SGavin Li 3332394bd877SGavin Li err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue), 3333394bd877SGavin Li ec->rx_coalesce_usecs, 3334394bd877SGavin Li ec->rx_max_coalesced_frames); 3335394bd877SGavin Li if (err) 3336394bd877SGavin Li return err; 33377c1453abSHeng Qi 3338394bd877SGavin Li vi->rq[queue].intr_coal.max_usecs = ec->rx_coalesce_usecs; 3339394bd877SGavin Li vi->rq[queue].intr_coal.max_packets = ec->rx_max_coalesced_frames; 3340394bd877SGavin Li 3341394bd877SGavin Li err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue), 3342394bd877SGavin Li ec->tx_coalesce_usecs, 3343394bd877SGavin Li ec->tx_max_coalesced_frames); 3344394bd877SGavin Li if (err) 3345394bd877SGavin Li return err; 33467c1453abSHeng Qi 3347394bd877SGavin Li vi->sq[queue].intr_coal.max_usecs = ec->tx_coalesce_usecs; 3348394bd877SGavin Li vi->sq[queue].intr_coal.max_packets = ec->tx_max_coalesced_frames; 3349394bd877SGavin Li 3350394bd877SGavin Li return 0; 3351394bd877SGavin Li } 3352394bd877SGavin Li 3353699b045aSAlvaro Karsz static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) 3354699b045aSAlvaro Karsz { 3355699b045aSAlvaro Karsz /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL 3356699b045aSAlvaro Karsz * feature is negotiated. 3357699b045aSAlvaro Karsz */ 3358699b045aSAlvaro Karsz if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs) 3359699b045aSAlvaro Karsz return -EOPNOTSUPP; 3360699b045aSAlvaro Karsz 3361699b045aSAlvaro Karsz if (ec->tx_max_coalesced_frames > 1 || 3362699b045aSAlvaro Karsz ec->rx_max_coalesced_frames != 1) 3363699b045aSAlvaro Karsz return -EINVAL; 3364699b045aSAlvaro Karsz 3365699b045aSAlvaro Karsz return 0; 3366699b045aSAlvaro Karsz } 3367699b045aSAlvaro Karsz 3368394bd877SGavin Li static int virtnet_should_update_vq_weight(int dev_flags, int weight, 3369394bd877SGavin Li int vq_weight, bool *should_update) 3370394bd877SGavin Li { 3371394bd877SGavin Li if (weight ^ vq_weight) { 3372394bd877SGavin Li if (dev_flags & IFF_UP) 3373394bd877SGavin Li return -EBUSY; 3374394bd877SGavin Li *should_update = true; 3375394bd877SGavin Li } 3376394bd877SGavin Li 3377394bd877SGavin Li return 0; 3378394bd877SGavin Li } 3379394bd877SGavin Li 33800c465be1SJason Wang static int virtnet_set_coalesce(struct net_device *dev, 3381f3ccfda1SYufeng Mo struct ethtool_coalesce *ec, 3382f3ccfda1SYufeng Mo struct kernel_ethtool_coalesce *kernel_coal, 3383f3ccfda1SYufeng Mo struct netlink_ext_ack *extack) 33840c465be1SJason Wang { 33850c465be1SJason Wang struct virtnet_info *vi = netdev_priv(dev); 3386394bd877SGavin Li int ret, queue_number, napi_weight; 3387699b045aSAlvaro Karsz bool update_napi = false; 33880c465be1SJason Wang 3389699b045aSAlvaro Karsz /* Can't change NAPI weight if the link is up */ 33900c465be1SJason Wang napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 3391394bd877SGavin Li for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) { 3392394bd877SGavin Li ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, 3393394bd877SGavin Li vi->sq[queue_number].napi.weight, 3394394bd877SGavin Li &update_napi); 3395394bd877SGavin Li if (ret) 3396394bd877SGavin Li return ret; 3397394bd877SGavin Li 3398394bd877SGavin Li if (update_napi) { 3399394bd877SGavin Li /* All queues that belong to [queue_number, vi->max_queue_pairs] will be 3400394bd877SGavin Li * updated for the sake of simplicity, which might not be necessary 3401394bd877SGavin Li */ 3402394bd877SGavin Li break; 3403394bd877SGavin Li } 3404699b045aSAlvaro Karsz } 3405699b045aSAlvaro Karsz 3406699b045aSAlvaro Karsz if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) 3407699b045aSAlvaro Karsz ret = virtnet_send_notf_coal_cmds(vi, ec); 3408699b045aSAlvaro Karsz else 3409699b045aSAlvaro Karsz ret = virtnet_coal_params_supported(ec); 3410699b045aSAlvaro Karsz 3411699b045aSAlvaro Karsz if (ret) 3412699b045aSAlvaro Karsz return ret; 3413699b045aSAlvaro Karsz 3414699b045aSAlvaro Karsz if (update_napi) { 3415394bd877SGavin Li for (; queue_number < vi->max_queue_pairs; queue_number++) 3416394bd877SGavin Li vi->sq[queue_number].napi.weight = napi_weight; 34170c465be1SJason Wang } 34180c465be1SJason Wang 3419699b045aSAlvaro Karsz return ret; 34200c465be1SJason Wang } 34210c465be1SJason Wang 34220c465be1SJason Wang static int virtnet_get_coalesce(struct net_device *dev, 3423f3ccfda1SYufeng Mo struct ethtool_coalesce *ec, 3424f3ccfda1SYufeng Mo struct kernel_ethtool_coalesce *kernel_coal, 3425f3ccfda1SYufeng Mo struct netlink_ext_ack *extack) 34260c465be1SJason Wang { 34270c465be1SJason Wang struct virtnet_info *vi = netdev_priv(dev); 34280c465be1SJason Wang 3429699b045aSAlvaro Karsz if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 3430308d7982SGavin Li ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs; 3431308d7982SGavin Li ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs; 3432308d7982SGavin Li ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets; 3433308d7982SGavin Li ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets; 3434699b045aSAlvaro Karsz } else { 3435699b045aSAlvaro Karsz ec->rx_max_coalesced_frames = 1; 34360c465be1SJason Wang 34370c465be1SJason Wang if (vi->sq[0].napi.weight) 34380c465be1SJason Wang ec->tx_max_coalesced_frames = 1; 3439699b045aSAlvaro Karsz } 34400c465be1SJason Wang 34410c465be1SJason Wang return 0; 34420c465be1SJason Wang } 34430c465be1SJason Wang 3444394bd877SGavin Li static int virtnet_set_per_queue_coalesce(struct net_device *dev, 3445394bd877SGavin Li u32 queue, 3446394bd877SGavin Li struct ethtool_coalesce *ec) 3447394bd877SGavin Li { 3448394bd877SGavin Li struct virtnet_info *vi = netdev_priv(dev); 3449394bd877SGavin Li int ret, napi_weight; 3450394bd877SGavin Li bool update_napi = false; 3451394bd877SGavin Li 3452394bd877SGavin Li if (queue >= vi->max_queue_pairs) 3453394bd877SGavin Li return -EINVAL; 3454394bd877SGavin Li 3455394bd877SGavin Li /* Can't change NAPI weight if the link is up */ 3456394bd877SGavin Li napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 3457394bd877SGavin Li ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, 3458394bd877SGavin Li vi->sq[queue].napi.weight, 3459394bd877SGavin Li &update_napi); 3460394bd877SGavin Li if (ret) 3461394bd877SGavin Li return ret; 3462394bd877SGavin Li 3463394bd877SGavin Li if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 3464394bd877SGavin Li ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue); 3465394bd877SGavin Li else 3466394bd877SGavin Li ret = virtnet_coal_params_supported(ec); 3467394bd877SGavin Li 3468394bd877SGavin Li if (ret) 3469394bd877SGavin Li return ret; 3470394bd877SGavin Li 3471394bd877SGavin Li if (update_napi) 3472394bd877SGavin Li vi->sq[queue].napi.weight = napi_weight; 3473394bd877SGavin Li 3474394bd877SGavin Li return 0; 3475394bd877SGavin Li } 3476394bd877SGavin Li 3477394bd877SGavin Li static int virtnet_get_per_queue_coalesce(struct net_device *dev, 3478394bd877SGavin Li u32 queue, 3479394bd877SGavin Li struct ethtool_coalesce *ec) 3480394bd877SGavin Li { 3481394bd877SGavin Li struct virtnet_info *vi = netdev_priv(dev); 3482394bd877SGavin Li 3483394bd877SGavin Li if (queue >= vi->max_queue_pairs) 3484394bd877SGavin Li return -EINVAL; 3485394bd877SGavin Li 3486394bd877SGavin Li if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { 3487394bd877SGavin Li ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs; 3488394bd877SGavin Li ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs; 3489394bd877SGavin Li ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets; 3490394bd877SGavin Li ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets; 3491394bd877SGavin Li } else { 3492394bd877SGavin Li ec->rx_max_coalesced_frames = 1; 3493394bd877SGavin Li 349484a056f7SHeng Qi if (vi->sq[queue].napi.weight) 3495394bd877SGavin Li ec->tx_max_coalesced_frames = 1; 3496394bd877SGavin Li } 3497394bd877SGavin Li 3498394bd877SGavin Li return 0; 3499394bd877SGavin Li } 3500394bd877SGavin Li 350116032be5SNikolay Aleksandrov static void virtnet_init_settings(struct net_device *dev) 350216032be5SNikolay Aleksandrov { 350316032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 350416032be5SNikolay Aleksandrov 350516032be5SNikolay Aleksandrov vi->speed = SPEED_UNKNOWN; 350616032be5SNikolay Aleksandrov vi->duplex = DUPLEX_UNKNOWN; 350716032be5SNikolay Aleksandrov } 350816032be5SNikolay Aleksandrov 3509faa9b39fSJason Baron static void virtnet_update_settings(struct virtnet_info *vi) 3510faa9b39fSJason Baron { 3511faa9b39fSJason Baron u32 speed; 3512faa9b39fSJason Baron u8 duplex; 3513faa9b39fSJason Baron 3514faa9b39fSJason Baron if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 3515faa9b39fSJason Baron return; 3516faa9b39fSJason Baron 351764ffa39dSMichael S. Tsirkin virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); 351864ffa39dSMichael S. Tsirkin 3519faa9b39fSJason Baron if (ethtool_validate_speed(speed)) 3520faa9b39fSJason Baron vi->speed = speed; 352164ffa39dSMichael S. Tsirkin 352264ffa39dSMichael S. Tsirkin virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); 352364ffa39dSMichael S. Tsirkin 3524faa9b39fSJason Baron if (ethtool_validate_duplex(duplex)) 3525faa9b39fSJason Baron vi->duplex = duplex; 3526faa9b39fSJason Baron } 3527faa9b39fSJason Baron 3528c7114b12SAndrew Melnychenko static u32 virtnet_get_rxfh_key_size(struct net_device *dev) 3529c7114b12SAndrew Melnychenko { 3530c7114b12SAndrew Melnychenko return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size; 3531c7114b12SAndrew Melnychenko } 3532c7114b12SAndrew Melnychenko 3533c7114b12SAndrew Melnychenko static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) 3534c7114b12SAndrew Melnychenko { 3535c7114b12SAndrew Melnychenko return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; 3536c7114b12SAndrew Melnychenko } 3537c7114b12SAndrew Melnychenko 3538c7114b12SAndrew Melnychenko static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) 3539c7114b12SAndrew Melnychenko { 3540c7114b12SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 3541c7114b12SAndrew Melnychenko int i; 3542c7114b12SAndrew Melnychenko 3543c7114b12SAndrew Melnychenko if (indir) { 3544c7114b12SAndrew Melnychenko for (i = 0; i < vi->rss_indir_table_size; ++i) 3545c7114b12SAndrew Melnychenko indir[i] = vi->ctrl->rss.indirection_table[i]; 3546c7114b12SAndrew Melnychenko } 3547c7114b12SAndrew Melnychenko 3548c7114b12SAndrew Melnychenko if (key) 3549c7114b12SAndrew Melnychenko memcpy(key, vi->ctrl->rss.key, vi->rss_key_size); 3550c7114b12SAndrew Melnychenko 3551c7114b12SAndrew Melnychenko if (hfunc) 3552c7114b12SAndrew Melnychenko *hfunc = ETH_RSS_HASH_TOP; 3553c7114b12SAndrew Melnychenko 3554c7114b12SAndrew Melnychenko return 0; 3555c7114b12SAndrew Melnychenko } 3556c7114b12SAndrew Melnychenko 3557c7114b12SAndrew Melnychenko static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc) 3558c7114b12SAndrew Melnychenko { 3559c7114b12SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 3560c7114b12SAndrew Melnychenko int i; 3561c7114b12SAndrew Melnychenko 3562c7114b12SAndrew Melnychenko if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) 3563c7114b12SAndrew Melnychenko return -EOPNOTSUPP; 3564c7114b12SAndrew Melnychenko 3565c7114b12SAndrew Melnychenko if (indir) { 3566c7114b12SAndrew Melnychenko for (i = 0; i < vi->rss_indir_table_size; ++i) 3567c7114b12SAndrew Melnychenko vi->ctrl->rss.indirection_table[i] = indir[i]; 3568c7114b12SAndrew Melnychenko } 3569c7114b12SAndrew Melnychenko if (key) 3570c7114b12SAndrew Melnychenko memcpy(vi->ctrl->rss.key, key, vi->rss_key_size); 3571c7114b12SAndrew Melnychenko 3572c7114b12SAndrew Melnychenko virtnet_commit_rss_command(vi); 3573c7114b12SAndrew Melnychenko 3574c7114b12SAndrew Melnychenko return 0; 3575c7114b12SAndrew Melnychenko } 3576c7114b12SAndrew Melnychenko 3577c7114b12SAndrew Melnychenko static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs) 3578c7114b12SAndrew Melnychenko { 3579c7114b12SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 3580c7114b12SAndrew Melnychenko int rc = 0; 3581c7114b12SAndrew Melnychenko 3582c7114b12SAndrew Melnychenko switch (info->cmd) { 3583c7114b12SAndrew Melnychenko case ETHTOOL_GRXRINGS: 3584c7114b12SAndrew Melnychenko info->data = vi->curr_queue_pairs; 3585c7114b12SAndrew Melnychenko break; 3586c1170820SAndrew Melnychenko case ETHTOOL_GRXFH: 3587c1170820SAndrew Melnychenko virtnet_get_hashflow(vi, info); 3588c1170820SAndrew Melnychenko break; 3589c1170820SAndrew Melnychenko default: 3590c1170820SAndrew Melnychenko rc = -EOPNOTSUPP; 3591c1170820SAndrew Melnychenko } 3592c1170820SAndrew Melnychenko 3593c1170820SAndrew Melnychenko return rc; 3594c1170820SAndrew Melnychenko } 3595c1170820SAndrew Melnychenko 3596c1170820SAndrew Melnychenko static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) 3597c1170820SAndrew Melnychenko { 3598c1170820SAndrew Melnychenko struct virtnet_info *vi = netdev_priv(dev); 3599c1170820SAndrew Melnychenko int rc = 0; 3600c1170820SAndrew Melnychenko 3601c1170820SAndrew Melnychenko switch (info->cmd) { 3602c1170820SAndrew Melnychenko case ETHTOOL_SRXFH: 3603c1170820SAndrew Melnychenko if (!virtnet_set_hashflow(vi, info)) 3604c1170820SAndrew Melnychenko rc = -EINVAL; 3605c1170820SAndrew Melnychenko 3606c1170820SAndrew Melnychenko break; 3607c7114b12SAndrew Melnychenko default: 3608c7114b12SAndrew Melnychenko rc = -EOPNOTSUPP; 3609c7114b12SAndrew Melnychenko } 3610c7114b12SAndrew Melnychenko 3611c7114b12SAndrew Melnychenko return rc; 3612c7114b12SAndrew Melnychenko } 3613c7114b12SAndrew Melnychenko 36140fc0b732SStephen Hemminger static const struct ethtool_ops virtnet_ethtool_ops = { 3615699b045aSAlvaro Karsz .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | 3616699b045aSAlvaro Karsz ETHTOOL_COALESCE_USECS, 361766846048SRick Jones .get_drvinfo = virtnet_get_drvinfo, 36189f4d26d0SMark McLoughlin .get_link = ethtool_op_get_link, 36198f9f4668SRick Jones .get_ringparam = virtnet_get_ringparam, 3620a335b33fSXuan Zhuo .set_ringparam = virtnet_set_ringparam, 3621d7dfc5cfSToshiaki Makita .get_strings = virtnet_get_strings, 3622d7dfc5cfSToshiaki Makita .get_sset_count = virtnet_get_sset_count, 3623d7dfc5cfSToshiaki Makita .get_ethtool_stats = virtnet_get_ethtool_stats, 3624d73bcd2cSJason Wang .set_channels = virtnet_set_channels, 3625d73bcd2cSJason Wang .get_channels = virtnet_get_channels, 3626074c3582SJacob Keller .get_ts_info = ethtool_op_get_ts_info, 3627ebb6b4b1SPhilippe Reynes .get_link_ksettings = virtnet_get_link_ksettings, 3628ebb6b4b1SPhilippe Reynes .set_link_ksettings = virtnet_set_link_ksettings, 36290c465be1SJason Wang .set_coalesce = virtnet_set_coalesce, 36300c465be1SJason Wang .get_coalesce = virtnet_get_coalesce, 3631394bd877SGavin Li .set_per_queue_coalesce = virtnet_set_per_queue_coalesce, 3632394bd877SGavin Li .get_per_queue_coalesce = virtnet_get_per_queue_coalesce, 3633c7114b12SAndrew Melnychenko .get_rxfh_key_size = virtnet_get_rxfh_key_size, 3634c7114b12SAndrew Melnychenko .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, 3635c7114b12SAndrew Melnychenko .get_rxfh = virtnet_get_rxfh, 3636c7114b12SAndrew Melnychenko .set_rxfh = virtnet_set_rxfh, 3637c7114b12SAndrew Melnychenko .get_rxnfc = virtnet_get_rxnfc, 3638c1170820SAndrew Melnychenko .set_rxnfc = virtnet_set_rxnfc, 3639a9ea3fc6SHerbert Xu }; 3640a9ea3fc6SHerbert Xu 36419fe7bfceSJohn Fastabend static void virtnet_freeze_down(struct virtio_device *vdev) 36429fe7bfceSJohn Fastabend { 36439fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 36449fe7bfceSJohn Fastabend 36459fe7bfceSJohn Fastabend /* Make sure no work handler is accessing the device */ 36469fe7bfceSJohn Fastabend flush_work(&vi->config_work); 36479fe7bfceSJohn Fastabend 364805c998b7SAke Koomsin netif_tx_lock_bh(vi->dev); 36499fe7bfceSJohn Fastabend netif_device_detach(vi->dev); 365005c998b7SAke Koomsin netif_tx_unlock_bh(vi->dev); 36518af52fe9SStephan Gerhold if (netif_running(vi->dev)) 36528af52fe9SStephan Gerhold virtnet_close(vi->dev); 36539fe7bfceSJohn Fastabend } 36549fe7bfceSJohn Fastabend 36559fe7bfceSJohn Fastabend static int init_vqs(struct virtnet_info *vi); 36569fe7bfceSJohn Fastabend 36579fe7bfceSJohn Fastabend static int virtnet_restore_up(struct virtio_device *vdev) 36589fe7bfceSJohn Fastabend { 36599fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 36608af52fe9SStephan Gerhold int err; 36619fe7bfceSJohn Fastabend 36629fe7bfceSJohn Fastabend err = init_vqs(vi); 36639fe7bfceSJohn Fastabend if (err) 36649fe7bfceSJohn Fastabend return err; 36659fe7bfceSJohn Fastabend 36669fe7bfceSJohn Fastabend virtio_device_ready(vdev); 36679fe7bfceSJohn Fastabend 36685a159128SJason Wang enable_delayed_refill(vi); 36695a159128SJason Wang 36709fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 36718af52fe9SStephan Gerhold err = virtnet_open(vi->dev); 36728af52fe9SStephan Gerhold if (err) 36738af52fe9SStephan Gerhold return err; 36749fe7bfceSJohn Fastabend } 36759fe7bfceSJohn Fastabend 367605c998b7SAke Koomsin netif_tx_lock_bh(vi->dev); 36779fe7bfceSJohn Fastabend netif_device_attach(vi->dev); 367805c998b7SAke Koomsin netif_tx_unlock_bh(vi->dev); 36799fe7bfceSJohn Fastabend return err; 36809fe7bfceSJohn Fastabend } 36819fe7bfceSJohn Fastabend 36823f93522fSJason Wang static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 36833f93522fSJason Wang { 36843f93522fSJason Wang struct scatterlist sg; 368512e57169SMichael S. Tsirkin vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads); 36863f93522fSJason Wang 368712e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads)); 36883f93522fSJason Wang 36893f93522fSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 36903f93522fSJason Wang VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 36913f93522fSJason Wang dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); 36923f93522fSJason Wang return -EINVAL; 36933f93522fSJason Wang } 36943f93522fSJason Wang 36953f93522fSJason Wang return 0; 36963f93522fSJason Wang } 36973f93522fSJason Wang 36983f93522fSJason Wang static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 36993f93522fSJason Wang { 37003f93522fSJason Wang u64 offloads = 0; 37013f93522fSJason Wang 37023f93522fSJason Wang if (!vi->guest_offloads) 37033f93522fSJason Wang return 0; 37043f93522fSJason Wang 37053f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 37063f93522fSJason Wang } 37073f93522fSJason Wang 37083f93522fSJason Wang static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 37093f93522fSJason Wang { 37103f93522fSJason Wang u64 offloads = vi->guest_offloads; 37113f93522fSJason Wang 37123f93522fSJason Wang if (!vi->guest_offloads) 37133f93522fSJason Wang return 0; 37143f93522fSJason Wang 37153f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 37163f93522fSJason Wang } 37173f93522fSJason Wang 37189861ce03SJakub Kicinski static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 37199861ce03SJakub Kicinski struct netlink_ext_ack *extack) 3720f600b690SJohn Fastabend { 3721e814b958SHeng Qi unsigned int room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM + 3722e814b958SHeng Qi sizeof(struct skb_shared_info)); 3723e814b958SHeng Qi unsigned int max_sz = PAGE_SIZE - room - ETH_HLEN; 3724f600b690SJohn Fastabend struct virtnet_info *vi = netdev_priv(dev); 3725f600b690SJohn Fastabend struct bpf_prog *old_prog; 3726017b29c3SJason Wang u16 xdp_qp = 0, curr_qp; 3727672aafd5SJohn Fastabend int i, err; 3728f600b690SJohn Fastabend 37293f93522fSJason Wang if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 37303f93522fSJason Wang && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 373192502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 373292502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 373318ba58e1SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 3734418044e1SAndrew Melnychenko virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) || 3735418044e1SAndrew Melnychenko virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) || 3736418044e1SAndrew Melnychenko virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) { 3737dbcf24d1SJason Wang NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); 3738f600b690SJohn Fastabend return -EOPNOTSUPP; 3739f600b690SJohn Fastabend } 3740f600b690SJohn Fastabend 3741f600b690SJohn Fastabend if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 37424d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 3743f600b690SJohn Fastabend return -EINVAL; 3744f600b690SJohn Fastabend } 3745f600b690SJohn Fastabend 37468d9bc36dSHeng Qi if (prog && !prog->aux->xdp_has_frags && dev->mtu > max_sz) { 37478d9bc36dSHeng Qi NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP without frags"); 37488d9bc36dSHeng Qi netdev_warn(dev, "single-buffer XDP requires MTU less than %u\n", max_sz); 3749f600b690SJohn Fastabend return -EINVAL; 3750f600b690SJohn Fastabend } 3751f600b690SJohn Fastabend 3752672aafd5SJohn Fastabend curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 3753672aafd5SJohn Fastabend if (prog) 3754672aafd5SJohn Fastabend xdp_qp = nr_cpu_ids; 3755672aafd5SJohn Fastabend 3756672aafd5SJohn Fastabend /* XDP requires extra queues for XDP_TX */ 3757672aafd5SJohn Fastabend if (curr_qp + xdp_qp > vi->max_queue_pairs) { 37589ce4e3d6SXuan Zhuo netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n", 3759672aafd5SJohn Fastabend curr_qp + xdp_qp, vi->max_queue_pairs); 376097c2c69eSXuan Zhuo xdp_qp = 0; 3761672aafd5SJohn Fastabend } 3762672aafd5SJohn Fastabend 376303aa6d34SToshiaki Makita old_prog = rtnl_dereference(vi->rq[0].xdp_prog); 376403aa6d34SToshiaki Makita if (!prog && !old_prog) 376503aa6d34SToshiaki Makita return 0; 376603aa6d34SToshiaki Makita 376785192dbfSAndrii Nakryiko if (prog) 376885192dbfSAndrii Nakryiko bpf_prog_add(prog, vi->max_queue_pairs - 1); 37692de2f7f4SJohn Fastabend 37704941d472SJason Wang /* Make sure NAPI is not using any XDP TX queues for RX. */ 3771534da5e8SToshiaki Makita if (netif_running(dev)) { 3772534da5e8SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 37734941d472SJason Wang napi_disable(&vi->rq[i].napi); 3774534da5e8SToshiaki Makita virtnet_napi_tx_disable(&vi->sq[i].napi); 3775534da5e8SToshiaki Makita } 3776534da5e8SToshiaki Makita } 37772de2f7f4SJohn Fastabend 377803aa6d34SToshiaki Makita if (!prog) { 377903aa6d34SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 378003aa6d34SToshiaki Makita rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 378103aa6d34SToshiaki Makita if (i == 0) 378203aa6d34SToshiaki Makita virtnet_restore_guest_offloads(vi); 378303aa6d34SToshiaki Makita } 378403aa6d34SToshiaki Makita synchronize_net(); 378503aa6d34SToshiaki Makita } 378603aa6d34SToshiaki Makita 37874941d472SJason Wang err = _virtnet_set_queues(vi, curr_qp + xdp_qp); 37884941d472SJason Wang if (err) 37894941d472SJason Wang goto err; 3790188313c1SToshiaki Makita netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 37914941d472SJason Wang vi->xdp_queue_pairs = xdp_qp; 3792f600b690SJohn Fastabend 379303aa6d34SToshiaki Makita if (prog) { 379497c2c69eSXuan Zhuo vi->xdp_enabled = true; 3795f600b690SJohn Fastabend for (i = 0; i < vi->max_queue_pairs; i++) { 3796f600b690SJohn Fastabend rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 379703aa6d34SToshiaki Makita if (i == 0 && !old_prog) 37983f93522fSJason Wang virtnet_clear_guest_offloads(vi); 37993f93522fSJason Wang } 380066c0e13aSMarek Majtyka if (!old_prog) 380130bbf891SLorenzo Bianconi xdp_features_set_redirect_target(dev, true); 380297c2c69eSXuan Zhuo } else { 380366c0e13aSMarek Majtyka xdp_features_clear_redirect_target(dev); 380497c2c69eSXuan Zhuo vi->xdp_enabled = false; 380503aa6d34SToshiaki Makita } 380603aa6d34SToshiaki Makita 380703aa6d34SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 3808f600b690SJohn Fastabend if (old_prog) 3809f600b690SJohn Fastabend bpf_prog_put(old_prog); 3810534da5e8SToshiaki Makita if (netif_running(dev)) { 38114941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 3812534da5e8SToshiaki Makita virtnet_napi_tx_enable(vi, vi->sq[i].vq, 3813534da5e8SToshiaki Makita &vi->sq[i].napi); 3814534da5e8SToshiaki Makita } 3815f600b690SJohn Fastabend } 3816f600b690SJohn Fastabend 3817f600b690SJohn Fastabend return 0; 38182de2f7f4SJohn Fastabend 38194941d472SJason Wang err: 382003aa6d34SToshiaki Makita if (!prog) { 382103aa6d34SToshiaki Makita virtnet_clear_guest_offloads(vi); 38224941d472SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 382303aa6d34SToshiaki Makita rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); 382403aa6d34SToshiaki Makita } 382503aa6d34SToshiaki Makita 38268be4d9a4SToshiaki Makita if (netif_running(dev)) { 3827534da5e8SToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 38284941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 3829534da5e8SToshiaki Makita virtnet_napi_tx_enable(vi, vi->sq[i].vq, 3830534da5e8SToshiaki Makita &vi->sq[i].napi); 3831534da5e8SToshiaki Makita } 38328be4d9a4SToshiaki Makita } 38332de2f7f4SJohn Fastabend if (prog) 38342de2f7f4SJohn Fastabend bpf_prog_sub(prog, vi->max_queue_pairs - 1); 38352de2f7f4SJohn Fastabend return err; 3836f600b690SJohn Fastabend } 3837f600b690SJohn Fastabend 3838f4e63525SJakub Kicinski static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 3839f600b690SJohn Fastabend { 3840f600b690SJohn Fastabend switch (xdp->command) { 3841f600b690SJohn Fastabend case XDP_SETUP_PROG: 38429861ce03SJakub Kicinski return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 3843f600b690SJohn Fastabend default: 3844f600b690SJohn Fastabend return -EINVAL; 3845f600b690SJohn Fastabend } 3846f600b690SJohn Fastabend } 3847f600b690SJohn Fastabend 3848ba5e4426SSridhar Samudrala static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, 3849ba5e4426SSridhar Samudrala size_t len) 3850ba5e4426SSridhar Samudrala { 3851ba5e4426SSridhar Samudrala struct virtnet_info *vi = netdev_priv(dev); 3852ba5e4426SSridhar Samudrala int ret; 3853ba5e4426SSridhar Samudrala 3854ba5e4426SSridhar Samudrala if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 3855ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 3856ba5e4426SSridhar Samudrala 3857ba5e4426SSridhar Samudrala ret = snprintf(buf, len, "sby"); 3858ba5e4426SSridhar Samudrala if (ret >= len) 3859ba5e4426SSridhar Samudrala return -EOPNOTSUPP; 3860ba5e4426SSridhar Samudrala 3861ba5e4426SSridhar Samudrala return 0; 3862ba5e4426SSridhar Samudrala } 3863ba5e4426SSridhar Samudrala 3864a02e8964SWillem de Bruijn static int virtnet_set_features(struct net_device *dev, 3865a02e8964SWillem de Bruijn netdev_features_t features) 3866a02e8964SWillem de Bruijn { 3867a02e8964SWillem de Bruijn struct virtnet_info *vi = netdev_priv(dev); 3868cf8691cbSMichael S. Tsirkin u64 offloads; 3869a02e8964SWillem de Bruijn int err; 3870a02e8964SWillem de Bruijn 3871dbcf24d1SJason Wang if ((dev->features ^ features) & NETIF_F_GRO_HW) { 387297c2c69eSXuan Zhuo if (vi->xdp_enabled) 3873a02e8964SWillem de Bruijn return -EBUSY; 3874a02e8964SWillem de Bruijn 3875dbcf24d1SJason Wang if (features & NETIF_F_GRO_HW) 3876cf8691cbSMichael S. Tsirkin offloads = vi->guest_offloads_capable; 3877a02e8964SWillem de Bruijn else 3878cf8691cbSMichael S. Tsirkin offloads = vi->guest_offloads_capable & 3879dbcf24d1SJason Wang ~GUEST_OFFLOAD_GRO_HW_MASK; 3880a02e8964SWillem de Bruijn 3881a02e8964SWillem de Bruijn err = virtnet_set_guest_offloads(vi, offloads); 3882a02e8964SWillem de Bruijn if (err) 3883a02e8964SWillem de Bruijn return err; 38843618ad2aSTonghao Zhang vi->guest_offloads = offloads; 3885cf8691cbSMichael S. Tsirkin } 3886cf8691cbSMichael S. Tsirkin 3887c7114b12SAndrew Melnychenko if ((dev->features ^ features) & NETIF_F_RXHASH) { 3888c7114b12SAndrew Melnychenko if (features & NETIF_F_RXHASH) 3889c1170820SAndrew Melnychenko vi->ctrl->rss.hash_types = vi->rss_hash_types_saved; 3890c7114b12SAndrew Melnychenko else 3891c7114b12SAndrew Melnychenko vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE; 3892c7114b12SAndrew Melnychenko 3893c7114b12SAndrew Melnychenko if (!virtnet_commit_rss_command(vi)) 3894c7114b12SAndrew Melnychenko return -EINVAL; 3895c7114b12SAndrew Melnychenko } 3896c7114b12SAndrew Melnychenko 3897a02e8964SWillem de Bruijn return 0; 3898a02e8964SWillem de Bruijn } 3899a02e8964SWillem de Bruijn 3900a520794bSTony Lu static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) 3901a520794bSTony Lu { 3902a520794bSTony Lu struct virtnet_info *priv = netdev_priv(dev); 3903a520794bSTony Lu struct send_queue *sq = &priv->sq[txqueue]; 3904a520794bSTony Lu struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); 3905a520794bSTony Lu 3906a520794bSTony Lu u64_stats_update_begin(&sq->stats.syncp); 390727debe3eSEric Dumazet u64_stats_inc(&sq->stats.tx_timeouts); 3908a520794bSTony Lu u64_stats_update_end(&sq->stats.syncp); 3909a520794bSTony Lu 3910a520794bSTony Lu netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", 3911a520794bSTony Lu txqueue, sq->name, sq->vq->index, sq->vq->name, 39125337824fSEric Dumazet jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); 3913a520794bSTony Lu } 3914a520794bSTony Lu 391576288b4eSStephen Hemminger static const struct net_device_ops virtnet_netdev = { 391676288b4eSStephen Hemminger .ndo_open = virtnet_open, 391776288b4eSStephen Hemminger .ndo_stop = virtnet_close, 391876288b4eSStephen Hemminger .ndo_start_xmit = start_xmit, 391976288b4eSStephen Hemminger .ndo_validate_addr = eth_validate_addr, 39209c46f6d4SAlex Williamson .ndo_set_mac_address = virtnet_set_mac_address, 39212af7698eSAlex Williamson .ndo_set_rx_mode = virtnet_set_rx_mode, 39223fa2a1dfSstephen hemminger .ndo_get_stats64 = virtnet_stats, 39231824a989SAlex Williamson .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 39241824a989SAlex Williamson .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 3925f4e63525SJakub Kicinski .ndo_bpf = virtnet_xdp, 3926186b3c99SJason Wang .ndo_xdp_xmit = virtnet_xdp_xmit, 39272836b4f2SVlad Yasevich .ndo_features_check = passthru_features_check, 3928ba5e4426SSridhar Samudrala .ndo_get_phys_port_name = virtnet_get_phys_port_name, 3929a02e8964SWillem de Bruijn .ndo_set_features = virtnet_set_features, 3930a520794bSTony Lu .ndo_tx_timeout = virtnet_tx_timeout, 393176288b4eSStephen Hemminger }; 393276288b4eSStephen Hemminger 3933586d17c5SJason Wang static void virtnet_config_changed_work(struct work_struct *work) 39349f4d26d0SMark McLoughlin { 3935586d17c5SJason Wang struct virtnet_info *vi = 3936586d17c5SJason Wang container_of(work, struct virtnet_info, config_work); 39379f4d26d0SMark McLoughlin u16 v; 39389f4d26d0SMark McLoughlin 3939855e0c52SRusty Russell if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 3940855e0c52SRusty Russell struct virtio_net_config, status, &v) < 0) 3941507613bfSMichael S. Tsirkin return; 3942586d17c5SJason Wang 3943586d17c5SJason Wang if (v & VIRTIO_NET_S_ANNOUNCE) { 3944ee89bab1SAmerigo Wang netdev_notify_peers(vi->dev); 3945586d17c5SJason Wang virtnet_ack_link_announce(vi); 3946586d17c5SJason Wang } 39479f4d26d0SMark McLoughlin 39489f4d26d0SMark McLoughlin /* Ignore unknown (future) status bits */ 39499f4d26d0SMark McLoughlin v &= VIRTIO_NET_S_LINK_UP; 39509f4d26d0SMark McLoughlin 39519f4d26d0SMark McLoughlin if (vi->status == v) 3952507613bfSMichael S. Tsirkin return; 39539f4d26d0SMark McLoughlin 39549f4d26d0SMark McLoughlin vi->status = v; 39559f4d26d0SMark McLoughlin 39569f4d26d0SMark McLoughlin if (vi->status & VIRTIO_NET_S_LINK_UP) { 3957faa9b39fSJason Baron virtnet_update_settings(vi); 39589f4d26d0SMark McLoughlin netif_carrier_on(vi->dev); 3959986a4f4dSJason Wang netif_tx_wake_all_queues(vi->dev); 39609f4d26d0SMark McLoughlin } else { 39619f4d26d0SMark McLoughlin netif_carrier_off(vi->dev); 3962986a4f4dSJason Wang netif_tx_stop_all_queues(vi->dev); 39639f4d26d0SMark McLoughlin } 39649f4d26d0SMark McLoughlin } 39659f4d26d0SMark McLoughlin 39669f4d26d0SMark McLoughlin static void virtnet_config_changed(struct virtio_device *vdev) 39679f4d26d0SMark McLoughlin { 39689f4d26d0SMark McLoughlin struct virtnet_info *vi = vdev->priv; 39699f4d26d0SMark McLoughlin 39703b07e9caSTejun Heo schedule_work(&vi->config_work); 39719f4d26d0SMark McLoughlin } 39729f4d26d0SMark McLoughlin 3973986a4f4dSJason Wang static void virtnet_free_queues(struct virtnet_info *vi) 3974986a4f4dSJason Wang { 3975d4fb84eeSAndrey Vagin int i; 3976d4fb84eeSAndrey Vagin 3977ab3971b1SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 39785198d545SJakub Kicinski __netif_napi_del(&vi->rq[i].napi); 39795198d545SJakub Kicinski __netif_napi_del(&vi->sq[i].napi); 3980ab3971b1SJason Wang } 3981d4fb84eeSAndrey Vagin 39825198d545SJakub Kicinski /* We called __netif_napi_del(), 3983963abe5cSEric Dumazet * we need to respect an RCU grace period before freeing vi->rq 3984963abe5cSEric Dumazet */ 3985963abe5cSEric Dumazet synchronize_net(); 3986963abe5cSEric Dumazet 3987986a4f4dSJason Wang kfree(vi->rq); 3988986a4f4dSJason Wang kfree(vi->sq); 398912e57169SMichael S. Tsirkin kfree(vi->ctrl); 3990986a4f4dSJason Wang } 3991986a4f4dSJason Wang 399247315329SJohn Fastabend static void _free_receive_bufs(struct virtnet_info *vi) 3993986a4f4dSJason Wang { 3994f600b690SJohn Fastabend struct bpf_prog *old_prog; 3995986a4f4dSJason Wang int i; 3996986a4f4dSJason Wang 3997986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 3998986a4f4dSJason Wang while (vi->rq[i].pages) 3999986a4f4dSJason Wang __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 4000f600b690SJohn Fastabend 4001f600b690SJohn Fastabend old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 4002f600b690SJohn Fastabend RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 4003f600b690SJohn Fastabend if (old_prog) 4004f600b690SJohn Fastabend bpf_prog_put(old_prog); 4005986a4f4dSJason Wang } 400647315329SJohn Fastabend } 400747315329SJohn Fastabend 400847315329SJohn Fastabend static void free_receive_bufs(struct virtnet_info *vi) 400947315329SJohn Fastabend { 401047315329SJohn Fastabend rtnl_lock(); 401147315329SJohn Fastabend _free_receive_bufs(vi); 4012f600b690SJohn Fastabend rtnl_unlock(); 4013986a4f4dSJason Wang } 4014986a4f4dSJason Wang 4015fb51879dSMichael Dalton static void free_receive_page_frags(struct virtnet_info *vi) 4016fb51879dSMichael Dalton { 4017fb51879dSMichael Dalton int i; 4018fb51879dSMichael Dalton for (i = 0; i < vi->max_queue_pairs; i++) 4019295525e2SXuan Zhuo if (vi->rq[i].alloc_frag.page) { 4020295525e2SXuan Zhuo if (vi->rq[i].do_dma && vi->rq[i].last_dma) 4021295525e2SXuan Zhuo virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0); 4022fb51879dSMichael Dalton put_page(vi->rq[i].alloc_frag.page); 4023fb51879dSMichael Dalton } 4024295525e2SXuan Zhuo } 4025fb51879dSMichael Dalton 40266e345f8cSXuan Zhuo static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf) 40276e345f8cSXuan Zhuo { 40286e345f8cSXuan Zhuo if (!is_xdp_frame(buf)) 40296e345f8cSXuan Zhuo dev_kfree_skb(buf); 40306e345f8cSXuan Zhuo else 40316e345f8cSXuan Zhuo xdp_return_frame(ptr_to_xdp(buf)); 40326e345f8cSXuan Zhuo } 40336e345f8cSXuan Zhuo 40346e345f8cSXuan Zhuo static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf) 40356e345f8cSXuan Zhuo { 40366e345f8cSXuan Zhuo struct virtnet_info *vi = vq->vdev->priv; 40376e345f8cSXuan Zhuo int i = vq2rxq(vq); 40386e345f8cSXuan Zhuo 40396e345f8cSXuan Zhuo if (vi->mergeable_rx_bufs) 40406e345f8cSXuan Zhuo put_page(virt_to_head_page(buf)); 40416e345f8cSXuan Zhuo else if (vi->big_packets) 40426e345f8cSXuan Zhuo give_pages(&vi->rq[i], buf); 40436e345f8cSXuan Zhuo else 40446e345f8cSXuan Zhuo put_page(virt_to_head_page(buf)); 40456e345f8cSXuan Zhuo } 40466e345f8cSXuan Zhuo 4047986a4f4dSJason Wang static void free_unused_bufs(struct virtnet_info *vi) 4048986a4f4dSJason Wang { 4049986a4f4dSJason Wang void *buf; 4050986a4f4dSJason Wang int i; 4051986a4f4dSJason Wang 4052986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 4053986a4f4dSJason Wang struct virtqueue *vq = vi->sq[i].vq; 40546e345f8cSXuan Zhuo while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 40556e345f8cSXuan Zhuo virtnet_sq_free_unused_buf(vq, buf); 4056f8bb5104SWenliang Wang cond_resched(); 4057986a4f4dSJason Wang } 4058986a4f4dSJason Wang 4059986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 4060295525e2SXuan Zhuo struct receive_queue *rq = &vi->rq[i]; 4061295525e2SXuan Zhuo 4062295525e2SXuan Zhuo while ((buf = virtnet_rq_detach_unused_buf(rq)) != NULL) 4063295525e2SXuan Zhuo virtnet_rq_free_unused_buf(rq->vq, buf); 4064f8bb5104SWenliang Wang cond_resched(); 4065986a4f4dSJason Wang } 4066ab7db917SMichael Dalton } 4067986a4f4dSJason Wang 4068e9d7417bSJason Wang static void virtnet_del_vqs(struct virtnet_info *vi) 4069e9d7417bSJason Wang { 4070e9d7417bSJason Wang struct virtio_device *vdev = vi->vdev; 4071e9d7417bSJason Wang 4072310974faSPeter Xu virtnet_clean_affinity(vi); 4073986a4f4dSJason Wang 4074e9d7417bSJason Wang vdev->config->del_vqs(vdev); 4075986a4f4dSJason Wang 4076986a4f4dSJason Wang virtnet_free_queues(vi); 4077986a4f4dSJason Wang } 4078986a4f4dSJason Wang 4079d85b758fSMichael S. Tsirkin /* How large should a single buffer be so a queue full of these can fit at 4080d85b758fSMichael S. Tsirkin * least one full packet? 4081d85b758fSMichael S. Tsirkin * Logic below assumes the mergeable buffer header is used. 4082d85b758fSMichael S. Tsirkin */ 4083d85b758fSMichael S. Tsirkin static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 4084d85b758fSMichael S. Tsirkin { 4085c1ddc42dSAndrew Melnychenko const unsigned int hdr_len = vi->hdr_len; 4086d85b758fSMichael S. Tsirkin unsigned int rq_size = virtqueue_get_vring_size(vq); 4087d85b758fSMichael S. Tsirkin unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 4088d85b758fSMichael S. Tsirkin unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 4089d85b758fSMichael S. Tsirkin unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 4090d85b758fSMichael S. Tsirkin 4091f0c3192cSMichael S. Tsirkin return max(max(min_buf_len, hdr_len) - hdr_len, 4092f0c3192cSMichael S. Tsirkin (unsigned int)GOOD_PACKET_LEN); 4093d85b758fSMichael S. Tsirkin } 4094d85b758fSMichael S. Tsirkin 4095986a4f4dSJason Wang static int virtnet_find_vqs(struct virtnet_info *vi) 4096986a4f4dSJason Wang { 4097986a4f4dSJason Wang vq_callback_t **callbacks; 4098986a4f4dSJason Wang struct virtqueue **vqs; 4099986a4f4dSJason Wang int ret = -ENOMEM; 4100986a4f4dSJason Wang int i, total_vqs; 4101986a4f4dSJason Wang const char **names; 4102d45b897bSMichael S. Tsirkin bool *ctx; 4103986a4f4dSJason Wang 4104986a4f4dSJason Wang /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 4105986a4f4dSJason Wang * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 4106986a4f4dSJason Wang * possible control vq. 4107986a4f4dSJason Wang */ 4108986a4f4dSJason Wang total_vqs = vi->max_queue_pairs * 2 + 4109986a4f4dSJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 4110986a4f4dSJason Wang 4111986a4f4dSJason Wang /* Allocate space for find_vqs parameters */ 41126396bb22SKees Cook vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); 4113986a4f4dSJason Wang if (!vqs) 4114986a4f4dSJason Wang goto err_vq; 41156da2ec56SKees Cook callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL); 4116986a4f4dSJason Wang if (!callbacks) 4117986a4f4dSJason Wang goto err_callback; 41186da2ec56SKees Cook names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL); 4119986a4f4dSJason Wang if (!names) 4120986a4f4dSJason Wang goto err_names; 4121192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 41226396bb22SKees Cook ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); 4123d45b897bSMichael S. Tsirkin if (!ctx) 4124d45b897bSMichael S. Tsirkin goto err_ctx; 4125d45b897bSMichael S. Tsirkin } else { 4126d45b897bSMichael S. Tsirkin ctx = NULL; 4127d45b897bSMichael S. Tsirkin } 4128986a4f4dSJason Wang 4129986a4f4dSJason Wang /* Parameters for control virtqueue, if any */ 4130986a4f4dSJason Wang if (vi->has_cvq) { 4131986a4f4dSJason Wang callbacks[total_vqs - 1] = NULL; 4132986a4f4dSJason Wang names[total_vqs - 1] = "control"; 4133986a4f4dSJason Wang } 4134986a4f4dSJason Wang 4135986a4f4dSJason Wang /* Allocate/initialize parameters for send/receive virtqueues */ 4136986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 4137986a4f4dSJason Wang callbacks[rxq2vq(i)] = skb_recv_done; 4138986a4f4dSJason Wang callbacks[txq2vq(i)] = skb_xmit_done; 4139986a4f4dSJason Wang sprintf(vi->rq[i].name, "input.%d", i); 4140986a4f4dSJason Wang sprintf(vi->sq[i].name, "output.%d", i); 4141986a4f4dSJason Wang names[rxq2vq(i)] = vi->rq[i].name; 4142986a4f4dSJason Wang names[txq2vq(i)] = vi->sq[i].name; 4143d45b897bSMichael S. Tsirkin if (ctx) 4144d45b897bSMichael S. Tsirkin ctx[rxq2vq(i)] = true; 4145986a4f4dSJason Wang } 4146986a4f4dSJason Wang 41472e9ca760SMichael S. Tsirkin ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks, 41482e9ca760SMichael S. Tsirkin names, ctx, NULL); 4149986a4f4dSJason Wang if (ret) 4150986a4f4dSJason Wang goto err_find; 4151986a4f4dSJason Wang 4152986a4f4dSJason Wang if (vi->has_cvq) { 4153986a4f4dSJason Wang vi->cvq = vqs[total_vqs - 1]; 4154986a4f4dSJason Wang if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 4155f646968fSPatrick McHardy vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4156986a4f4dSJason Wang } 4157986a4f4dSJason Wang 4158986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 4159986a4f4dSJason Wang vi->rq[i].vq = vqs[rxq2vq(i)]; 4160d85b758fSMichael S. Tsirkin vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 4161986a4f4dSJason Wang vi->sq[i].vq = vqs[txq2vq(i)]; 4162986a4f4dSJason Wang } 4163986a4f4dSJason Wang 41642fa3c8a8STonghao Zhang /* run here: ret == 0. */ 4165986a4f4dSJason Wang 4166986a4f4dSJason Wang 4167986a4f4dSJason Wang err_find: 4168d45b897bSMichael S. Tsirkin kfree(ctx); 4169d45b897bSMichael S. Tsirkin err_ctx: 4170986a4f4dSJason Wang kfree(names); 4171986a4f4dSJason Wang err_names: 4172986a4f4dSJason Wang kfree(callbacks); 4173986a4f4dSJason Wang err_callback: 4174986a4f4dSJason Wang kfree(vqs); 4175986a4f4dSJason Wang err_vq: 4176986a4f4dSJason Wang return ret; 4177986a4f4dSJason Wang } 4178986a4f4dSJason Wang 4179986a4f4dSJason Wang static int virtnet_alloc_queues(struct virtnet_info *vi) 4180986a4f4dSJason Wang { 4181986a4f4dSJason Wang int i; 4182986a4f4dSJason Wang 4183122b84a1SMax Gurtovoy if (vi->has_cvq) { 418412e57169SMichael S. Tsirkin vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 418512e57169SMichael S. Tsirkin if (!vi->ctrl) 418612e57169SMichael S. Tsirkin goto err_ctrl; 4187122b84a1SMax Gurtovoy } else { 4188122b84a1SMax Gurtovoy vi->ctrl = NULL; 4189122b84a1SMax Gurtovoy } 41906396bb22SKees Cook vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL); 4191986a4f4dSJason Wang if (!vi->sq) 4192986a4f4dSJason Wang goto err_sq; 41936396bb22SKees Cook vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL); 4194008d4278SAmerigo Wang if (!vi->rq) 4195986a4f4dSJason Wang goto err_rq; 4196986a4f4dSJason Wang 4197986a4f4dSJason Wang INIT_DELAYED_WORK(&vi->refill, refill_work); 4198986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 4199986a4f4dSJason Wang vi->rq[i].pages = NULL; 4200d484735dSJakub Kicinski netif_napi_add_weight(vi->dev, &vi->rq[i].napi, virtnet_poll, 4201986a4f4dSJason Wang napi_weight); 42028d602e1aSJakub Kicinski netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi, 42038d602e1aSJakub Kicinski virtnet_poll_tx, 4204b92f1e67SWillem de Bruijn napi_tx ? napi_weight : 0); 4205986a4f4dSJason Wang 4206986a4f4dSJason Wang sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 42075377d758SJohannes Berg ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 4208986a4f4dSJason Wang sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 4209d7dfc5cfSToshiaki Makita 4210d7dfc5cfSToshiaki Makita u64_stats_init(&vi->rq[i].stats.syncp); 4211d7dfc5cfSToshiaki Makita u64_stats_init(&vi->sq[i].stats.syncp); 4212986a4f4dSJason Wang } 4213986a4f4dSJason Wang 4214986a4f4dSJason Wang return 0; 4215986a4f4dSJason Wang 4216986a4f4dSJason Wang err_rq: 4217986a4f4dSJason Wang kfree(vi->sq); 4218986a4f4dSJason Wang err_sq: 421912e57169SMichael S. Tsirkin kfree(vi->ctrl); 422012e57169SMichael S. Tsirkin err_ctrl: 4221986a4f4dSJason Wang return -ENOMEM; 4222e9d7417bSJason Wang } 4223e9d7417bSJason Wang 42243f9c10b0SAmit Shah static int init_vqs(struct virtnet_info *vi) 42253f9c10b0SAmit Shah { 4226986a4f4dSJason Wang int ret; 42273f9c10b0SAmit Shah 4228986a4f4dSJason Wang /* Allocate send & receive queues */ 4229986a4f4dSJason Wang ret = virtnet_alloc_queues(vi); 4230986a4f4dSJason Wang if (ret) 4231986a4f4dSJason Wang goto err; 42323f9c10b0SAmit Shah 4233986a4f4dSJason Wang ret = virtnet_find_vqs(vi); 4234986a4f4dSJason Wang if (ret) 4235986a4f4dSJason Wang goto err_free; 42363f9c10b0SAmit Shah 4237295525e2SXuan Zhuo virtnet_rq_set_premapped(vi); 4238295525e2SXuan Zhuo 4239a0d1d0f4SSebastian Andrzej Siewior cpus_read_lock(); 42408898c21cSWanlong Gao virtnet_set_affinity(vi); 4241a0d1d0f4SSebastian Andrzej Siewior cpus_read_unlock(); 424247be2479SWanlong Gao 42433f9c10b0SAmit Shah return 0; 4244986a4f4dSJason Wang 4245986a4f4dSJason Wang err_free: 4246986a4f4dSJason Wang virtnet_free_queues(vi); 4247986a4f4dSJason Wang err: 4248986a4f4dSJason Wang return ret; 42493f9c10b0SAmit Shah } 42503f9c10b0SAmit Shah 4251fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 4252fbf28d78SMichael Dalton static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 4253718ad681Sstephen hemminger char *buf) 4254fbf28d78SMichael Dalton { 4255fbf28d78SMichael Dalton struct virtnet_info *vi = netdev_priv(queue->dev); 4256fbf28d78SMichael Dalton unsigned int queue_index = get_netdev_rx_queue_index(queue); 42573cc81a9aSJason Wang unsigned int headroom = virtnet_get_headroom(vi); 42583cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 42595377d758SJohannes Berg struct ewma_pkt_len *avg; 4260fbf28d78SMichael Dalton 4261fbf28d78SMichael Dalton BUG_ON(queue_index >= vi->max_queue_pairs); 4262fbf28d78SMichael Dalton avg = &vi->rq[queue_index].mrg_avg_pkt_len; 4263d85b758fSMichael S. Tsirkin return sprintf(buf, "%u\n", 42643cc81a9aSJason Wang get_mergeable_buf_len(&vi->rq[queue_index], avg, 42653cc81a9aSJason Wang SKB_DATA_ALIGN(headroom + tailroom))); 4266fbf28d78SMichael Dalton } 4267fbf28d78SMichael Dalton 4268fbf28d78SMichael Dalton static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 4269fbf28d78SMichael Dalton __ATTR_RO(mergeable_rx_buffer_size); 4270fbf28d78SMichael Dalton 4271fbf28d78SMichael Dalton static struct attribute *virtio_net_mrg_rx_attrs[] = { 4272fbf28d78SMichael Dalton &mergeable_rx_buffer_size_attribute.attr, 4273fbf28d78SMichael Dalton NULL 4274fbf28d78SMichael Dalton }; 4275fbf28d78SMichael Dalton 4276fbf28d78SMichael Dalton static const struct attribute_group virtio_net_mrg_rx_group = { 4277fbf28d78SMichael Dalton .name = "virtio_net", 4278fbf28d78SMichael Dalton .attrs = virtio_net_mrg_rx_attrs 4279fbf28d78SMichael Dalton }; 4280fbf28d78SMichael Dalton #endif 4281fbf28d78SMichael Dalton 4282892d6eb1SJason Wang static bool virtnet_fail_on_feature(struct virtio_device *vdev, 4283892d6eb1SJason Wang unsigned int fbit, 4284892d6eb1SJason Wang const char *fname, const char *dname) 4285892d6eb1SJason Wang { 4286892d6eb1SJason Wang if (!virtio_has_feature(vdev, fbit)) 4287892d6eb1SJason Wang return false; 4288892d6eb1SJason Wang 4289892d6eb1SJason Wang dev_err(&vdev->dev, "device advertises feature %s but not %s", 4290892d6eb1SJason Wang fname, dname); 4291892d6eb1SJason Wang 4292892d6eb1SJason Wang return true; 4293892d6eb1SJason Wang } 4294892d6eb1SJason Wang 4295892d6eb1SJason Wang #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 4296892d6eb1SJason Wang virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 4297892d6eb1SJason Wang 4298892d6eb1SJason Wang static bool virtnet_validate_features(struct virtio_device *vdev) 4299892d6eb1SJason Wang { 4300892d6eb1SJason Wang if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 4301892d6eb1SJason Wang (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 4302892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 4303892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 4304892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 4305892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 4306892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 4307892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 4308892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 4309c7114b12SAndrew Melnychenko "VIRTIO_NET_F_CTRL_VQ") || 4310c7114b12SAndrew Melnychenko VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, 431191f41f01SAndrew Melnychenko "VIRTIO_NET_F_CTRL_VQ") || 431291f41f01SAndrew Melnychenko VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, 4313699b045aSAlvaro Karsz "VIRTIO_NET_F_CTRL_VQ") || 4314699b045aSAlvaro Karsz VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, 43158af3bf66SGavin Li "VIRTIO_NET_F_CTRL_VQ") || 43168af3bf66SGavin Li VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_VQ_NOTF_COAL, 4317892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ"))) { 4318892d6eb1SJason Wang return false; 4319892d6eb1SJason Wang } 4320892d6eb1SJason Wang 4321892d6eb1SJason Wang return true; 4322892d6eb1SJason Wang } 4323892d6eb1SJason Wang 4324d0c2c997SJarod Wilson #define MIN_MTU ETH_MIN_MTU 4325d0c2c997SJarod Wilson #define MAX_MTU ETH_MAX_MTU 4326d0c2c997SJarod Wilson 4327fe36cbe0SMichael S. Tsirkin static int virtnet_validate(struct virtio_device *vdev) 4328296f96fcSRusty Russell { 43296ba42248SMichael S. Tsirkin if (!vdev->config->get) { 43306ba42248SMichael S. Tsirkin dev_err(&vdev->dev, "%s failure: config access disabled\n", 43316ba42248SMichael S. Tsirkin __func__); 43326ba42248SMichael S. Tsirkin return -EINVAL; 43336ba42248SMichael S. Tsirkin } 43346ba42248SMichael S. Tsirkin 4335892d6eb1SJason Wang if (!virtnet_validate_features(vdev)) 4336892d6eb1SJason Wang return -EINVAL; 4337892d6eb1SJason Wang 4338fe36cbe0SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 4339fe36cbe0SMichael S. Tsirkin int mtu = virtio_cread16(vdev, 4340fe36cbe0SMichael S. Tsirkin offsetof(struct virtio_net_config, 4341fe36cbe0SMichael S. Tsirkin mtu)); 4342fe36cbe0SMichael S. Tsirkin if (mtu < MIN_MTU) 4343fe36cbe0SMichael S. Tsirkin __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 4344fe36cbe0SMichael S. Tsirkin } 4345fe36cbe0SMichael S. Tsirkin 43467c06458cSLaurent Vivier if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY) && 43477c06458cSLaurent Vivier !virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 43487c06458cSLaurent Vivier dev_warn(&vdev->dev, "device advertises feature VIRTIO_NET_F_STANDBY but not VIRTIO_NET_F_MAC, disabling standby"); 43497c06458cSLaurent Vivier __virtio_clear_bit(vdev, VIRTIO_NET_F_STANDBY); 43507c06458cSLaurent Vivier } 43517c06458cSLaurent Vivier 4352fe36cbe0SMichael S. Tsirkin return 0; 4353fe36cbe0SMichael S. Tsirkin } 4354fe36cbe0SMichael S. Tsirkin 435546cd26f4SGavin Li static bool virtnet_check_guest_gso(const struct virtnet_info *vi) 435646cd26f4SGavin Li { 435746cd26f4SGavin Li return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 435846cd26f4SGavin Li virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 435946cd26f4SGavin Li virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 4360418044e1SAndrew Melnychenko virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 4361418044e1SAndrew Melnychenko (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) && 4362418044e1SAndrew Melnychenko virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6)); 436346cd26f4SGavin Li } 436446cd26f4SGavin Li 43654959aebbSGavin Li static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) 43664959aebbSGavin Li { 43674959aebbSGavin Li bool guest_gso = virtnet_check_guest_gso(vi); 43684959aebbSGavin Li 43694959aebbSGavin Li /* If device can receive ANY guest GSO packets, regardless of mtu, 43704959aebbSGavin Li * allocate packets of maximum size, otherwise limit it to only 43714959aebbSGavin Li * mtu size worth only. 43724959aebbSGavin Li */ 43734959aebbSGavin Li if (mtu > ETH_DATA_LEN || guest_gso) { 43744959aebbSGavin Li vi->big_packets = true; 43754959aebbSGavin Li vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE); 43764959aebbSGavin Li } 43774959aebbSGavin Li } 43784959aebbSGavin Li 4379fe36cbe0SMichael S. Tsirkin static int virtnet_probe(struct virtio_device *vdev) 4380fe36cbe0SMichael S. Tsirkin { 4381d7dfc5cfSToshiaki Makita int i, err = -ENOMEM; 4382fe36cbe0SMichael S. Tsirkin struct net_device *dev; 4383fe36cbe0SMichael S. Tsirkin struct virtnet_info *vi; 4384fe36cbe0SMichael S. Tsirkin u16 max_queue_pairs; 43854959aebbSGavin Li int mtu = 0; 4386fe36cbe0SMichael S. Tsirkin 4387c7114b12SAndrew Melnychenko /* Find if host supports multiqueue/rss virtio_net device */ 4388c7114b12SAndrew Melnychenko max_queue_pairs = 1; 4389c7114b12SAndrew Melnychenko if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 4390c7114b12SAndrew Melnychenko max_queue_pairs = 4391c7114b12SAndrew Melnychenko virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); 4392986a4f4dSJason Wang 4393986a4f4dSJason Wang /* We need at least 2 queue's */ 4394c7114b12SAndrew Melnychenko if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 4395986a4f4dSJason Wang max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 4396986a4f4dSJason Wang !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 4397986a4f4dSJason Wang max_queue_pairs = 1; 4398296f96fcSRusty Russell 4399296f96fcSRusty Russell /* Allocate ourselves a network device with room for our info */ 4400986a4f4dSJason Wang dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 4401296f96fcSRusty Russell if (!dev) 4402296f96fcSRusty Russell return -ENOMEM; 4403296f96fcSRusty Russell 4404296f96fcSRusty Russell /* Set up network device as normal. */ 4405ab5bd583SXuan Zhuo dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 4406ab5bd583SXuan Zhuo IFF_TX_SKB_NO_LINEAR; 440776288b4eSStephen Hemminger dev->netdev_ops = &virtnet_netdev; 4408296f96fcSRusty Russell dev->features = NETIF_F_HIGHDMA; 44093fa2a1dfSstephen hemminger 44107ad24ea4SWilfried Klaebe dev->ethtool_ops = &virtnet_ethtool_ops; 4411296f96fcSRusty Russell SET_NETDEV_DEV(dev, &vdev->dev); 4412296f96fcSRusty Russell 4413296f96fcSRusty Russell /* Do we support "hardware" checksums? */ 441498e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 4415296f96fcSRusty Russell /* This opens up the world of extra features. */ 441648900cb6SJason Wang dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 441798e778c9SMichał Mirosław if (csum) 441848900cb6SJason Wang dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 441998e778c9SMichał Mirosław 442098e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 4421e078de03SDavid S. Miller dev->hw_features |= NETIF_F_TSO 442234a48579SRusty Russell | NETIF_F_TSO_ECN | NETIF_F_TSO6; 442334a48579SRusty Russell } 44245539ae96SRusty Russell /* Individual feature bits: what can host handle? */ 442598e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 442698e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO; 442798e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 442898e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO6; 442998e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 443098e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO_ECN; 4431418044e1SAndrew Melnychenko if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO)) 4432418044e1SAndrew Melnychenko dev->hw_features |= NETIF_F_GSO_UDP_L4; 443398e778c9SMichał Mirosław 443441f2f127SJason Wang dev->features |= NETIF_F_GSO_ROBUST; 443541f2f127SJason Wang 443698e778c9SMichał Mirosław if (gso) 4437e078de03SDavid S. Miller dev->features |= dev->hw_features & NETIF_F_ALL_TSO; 443898e778c9SMichał Mirosław /* (!csum && gso) case will be fixed by register_netdev() */ 4439296f96fcSRusty Russell } 44404f49129bSThomas Huth if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM)) 44414f49129bSThomas Huth dev->features |= NETIF_F_RXCSUM; 4442a02e8964SWillem de Bruijn if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 4443a02e8964SWillem de Bruijn virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) 4444dbcf24d1SJason Wang dev->features |= NETIF_F_GRO_HW; 4445cf8691cbSMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) 4446dbcf24d1SJason Wang dev->hw_features |= NETIF_F_GRO_HW; 4447296f96fcSRusty Russell 44484fda8302SJason Wang dev->vlan_features = dev->features; 444966c0e13aSMarek Majtyka dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; 44504fda8302SJason Wang 4451d0c2c997SJarod Wilson /* MTU range: 68 - 65535 */ 4452d0c2c997SJarod Wilson dev->min_mtu = MIN_MTU; 4453d0c2c997SJarod Wilson dev->max_mtu = MAX_MTU; 4454d0c2c997SJarod Wilson 4455296f96fcSRusty Russell /* Configuration may specify what MAC to use. Otherwise random. */ 4456f2edaa4aSJakub Kicinski if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 4457f2edaa4aSJakub Kicinski u8 addr[ETH_ALEN]; 4458f2edaa4aSJakub Kicinski 4459855e0c52SRusty Russell virtio_cread_bytes(vdev, 4460a586d4f6SRusty Russell offsetof(struct virtio_net_config, mac), 4461f2edaa4aSJakub Kicinski addr, ETH_ALEN); 4462f2edaa4aSJakub Kicinski eth_hw_addr_set(dev, addr); 4463f2edaa4aSJakub Kicinski } else { 4464f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 44659f62d221SLaurent Vivier dev_info(&vdev->dev, "Assigned random MAC address %pM\n", 44669f62d221SLaurent Vivier dev->dev_addr); 4467f2edaa4aSJakub Kicinski } 4468296f96fcSRusty Russell 4469296f96fcSRusty Russell /* Set up our device-specific information */ 4470296f96fcSRusty Russell vi = netdev_priv(dev); 4471296f96fcSRusty Russell vi->dev = dev; 4472296f96fcSRusty Russell vi->vdev = vdev; 4473d9d5dcc8SChristian Borntraeger vdev->priv = vi; 4474827da44cSJohn Stultz 4475586d17c5SJason Wang INIT_WORK(&vi->config_work, virtnet_config_changed_work); 44765a159128SJason Wang spin_lock_init(&vi->refill_lock); 4477296f96fcSRusty Russell 447830bbf891SLorenzo Bianconi if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { 44793f2c31d9SMark McLoughlin vi->mergeable_rx_bufs = true; 448030bbf891SLorenzo Bianconi dev->xdp_features |= NETDEV_XDP_ACT_RX_SG; 448130bbf891SLorenzo Bianconi } 44823f2c31d9SMark McLoughlin 4483699b045aSAlvaro Karsz if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 4484308d7982SGavin Li vi->intr_coal_rx.max_usecs = 0; 4485308d7982SGavin Li vi->intr_coal_tx.max_usecs = 0; 4486308d7982SGavin Li vi->intr_coal_tx.max_packets = 0; 4487308d7982SGavin Li vi->intr_coal_rx.max_packets = 0; 4488699b045aSAlvaro Karsz } 4489699b045aSAlvaro Karsz 449091f41f01SAndrew Melnychenko if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) 449191f41f01SAndrew Melnychenko vi->has_rss_hash_report = true; 449291f41f01SAndrew Melnychenko 449391f41f01SAndrew Melnychenko if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 4494c7114b12SAndrew Melnychenko vi->has_rss = true; 449591f41f01SAndrew Melnychenko 449691f41f01SAndrew Melnychenko if (vi->has_rss || vi->has_rss_hash_report) { 4497c7114b12SAndrew Melnychenko vi->rss_indir_table_size = 4498c7114b12SAndrew Melnychenko virtio_cread16(vdev, offsetof(struct virtio_net_config, 4499c7114b12SAndrew Melnychenko rss_max_indirection_table_length)); 4500c7114b12SAndrew Melnychenko vi->rss_key_size = 4501c7114b12SAndrew Melnychenko virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); 4502c7114b12SAndrew Melnychenko 4503c7114b12SAndrew Melnychenko vi->rss_hash_types_supported = 4504c7114b12SAndrew Melnychenko virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); 4505c7114b12SAndrew Melnychenko vi->rss_hash_types_supported &= 4506c7114b12SAndrew Melnychenko ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX | 4507c7114b12SAndrew Melnychenko VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 4508c7114b12SAndrew Melnychenko VIRTIO_NET_RSS_HASH_TYPE_UDP_EX); 4509c7114b12SAndrew Melnychenko 4510c7114b12SAndrew Melnychenko dev->hw_features |= NETIF_F_RXHASH; 4511c7114b12SAndrew Melnychenko } 451291f41f01SAndrew Melnychenko 451391f41f01SAndrew Melnychenko if (vi->has_rss_hash_report) 451491f41f01SAndrew Melnychenko vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash); 451591f41f01SAndrew Melnychenko else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 4516d04302b3SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 4517012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 4518012873d0SMichael S. Tsirkin else 4519012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr); 4520012873d0SMichael S. Tsirkin 452175993300SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 452275993300SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 4523e7428e95SMichael S. Tsirkin vi->any_header_sg = true; 4524e7428e95SMichael S. Tsirkin 4525986a4f4dSJason Wang if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 4526986a4f4dSJason Wang vi->has_cvq = true; 4527986a4f4dSJason Wang 452814de9d11SAaron Conole if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 452914de9d11SAaron Conole mtu = virtio_cread16(vdev, 453014de9d11SAaron Conole offsetof(struct virtio_net_config, 453114de9d11SAaron Conole mtu)); 453293a205eeSAaron Conole if (mtu < dev->min_mtu) { 4533fe36cbe0SMichael S. Tsirkin /* Should never trigger: MTU was previously validated 4534fe36cbe0SMichael S. Tsirkin * in virtnet_validate. 4535fe36cbe0SMichael S. Tsirkin */ 45367934b481SYuval Shaia dev_err(&vdev->dev, 45377934b481SYuval Shaia "device MTU appears to have changed it is now %d < %d", 45387934b481SYuval Shaia mtu, dev->min_mtu); 4539411ea23aSDan Carpenter err = -EINVAL; 4540d7dfc5cfSToshiaki Makita goto free; 4541fe36cbe0SMichael S. Tsirkin } 4542fe36cbe0SMichael S. Tsirkin 4543d0c2c997SJarod Wilson dev->mtu = mtu; 454493a205eeSAaron Conole dev->max_mtu = mtu; 454514de9d11SAaron Conole } 454614de9d11SAaron Conole 45474959aebbSGavin Li virtnet_set_big_packets(vi, mtu); 45484959aebbSGavin Li 4549012873d0SMichael S. Tsirkin if (vi->any_header_sg) 4550012873d0SMichael S. Tsirkin dev->needed_headroom = vi->hdr_len; 45516ebbc1a6SZhangjie \(HZ\) 455244900010SJason Wang /* Enable multiqueue by default */ 455344900010SJason Wang if (num_online_cpus() >= max_queue_pairs) 455444900010SJason Wang vi->curr_queue_pairs = max_queue_pairs; 455544900010SJason Wang else 455644900010SJason Wang vi->curr_queue_pairs = num_online_cpus(); 4557986a4f4dSJason Wang vi->max_queue_pairs = max_queue_pairs; 4558986a4f4dSJason Wang 4559986a4f4dSJason Wang /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 45603f9c10b0SAmit Shah err = init_vqs(vi); 4561d2a7dddaSMichael S. Tsirkin if (err) 4562d7dfc5cfSToshiaki Makita goto free; 4563d2a7dddaSMichael S. Tsirkin 4564fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 4565fbf28d78SMichael Dalton if (vi->mergeable_rx_bufs) 4566fbf28d78SMichael Dalton dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 4567fbf28d78SMichael Dalton #endif 45680f13b66bSZhi Yong Wu netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 45690f13b66bSZhi Yong Wu netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 4570986a4f4dSJason Wang 45712e9ca760SMichael S. Tsirkin virtnet_init_settings(dev); 45722e9ca760SMichael S. Tsirkin 4573ba5e4426SSridhar Samudrala if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 4574ba5e4426SSridhar Samudrala vi->failover = net_failover_create(vi->dev); 45754b8e6ac4SWei Yongjun if (IS_ERR(vi->failover)) { 45764b8e6ac4SWei Yongjun err = PTR_ERR(vi->failover); 4577ba5e4426SSridhar Samudrala goto free_vqs; 4578ba5e4426SSridhar Samudrala } 45794b8e6ac4SWei Yongjun } 4580ba5e4426SSridhar Samudrala 458191f41f01SAndrew Melnychenko if (vi->has_rss || vi->has_rss_hash_report) 4582c7114b12SAndrew Melnychenko virtnet_init_default_rss(vi); 4583c7114b12SAndrew Melnychenko 458450c0ada6SJason Wang /* serialize netdev register + virtio_device_ready() with ndo_open() */ 458550c0ada6SJason Wang rtnl_lock(); 458650c0ada6SJason Wang 458750c0ada6SJason Wang err = register_netdevice(dev); 4588296f96fcSRusty Russell if (err) { 4589296f96fcSRusty Russell pr_debug("virtio_net: registering device failed\n"); 459050c0ada6SJason Wang rtnl_unlock(); 4591ba5e4426SSridhar Samudrala goto free_failover; 4592296f96fcSRusty Russell } 4593b3369c1fSRusty Russell 45944baf1e33SMichael S. Tsirkin virtio_device_ready(vdev); 45954baf1e33SMichael S. Tsirkin 459651b81317SJason Wang _virtnet_set_queues(vi, vi->curr_queue_pairs); 459751b81317SJason Wang 45989f62d221SLaurent Vivier /* a random MAC address has been assigned, notify the device. 45999f62d221SLaurent Vivier * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there 46009f62d221SLaurent Vivier * because many devices work fine without getting MAC explicitly 46019f62d221SLaurent Vivier */ 46029f62d221SLaurent Vivier if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 46039f62d221SLaurent Vivier virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 46049f62d221SLaurent Vivier struct scatterlist sg; 46059f62d221SLaurent Vivier 46069f62d221SLaurent Vivier sg_init_one(&sg, dev->dev_addr, dev->addr_len); 46079f62d221SLaurent Vivier if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 46089f62d221SLaurent Vivier VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 46099f62d221SLaurent Vivier pr_debug("virtio_net: setting MAC address failed\n"); 46109f62d221SLaurent Vivier rtnl_unlock(); 46119f62d221SLaurent Vivier err = -EINVAL; 46129f62d221SLaurent Vivier goto free_unregister_netdev; 46139f62d221SLaurent Vivier } 46149f62d221SLaurent Vivier } 46159f62d221SLaurent Vivier 461650c0ada6SJason Wang rtnl_unlock(); 461750c0ada6SJason Wang 46188017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 46198de4b2f3SWanlong Gao if (err) { 46208de4b2f3SWanlong Gao pr_debug("virtio_net: registering cpu notifier failed\n"); 4621f00e35e2Swangyunjian goto free_unregister_netdev; 46228de4b2f3SWanlong Gao } 46238de4b2f3SWanlong Gao 4624167c25e4SJason Wang /* Assume link up if device can't report link status, 4625167c25e4SJason Wang otherwise get link status from config. */ 4626167c25e4SJason Wang netif_carrier_off(dev); 4627bda7fab5SJay Vosburgh if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 46283b07e9caSTejun Heo schedule_work(&vi->config_work); 4629167c25e4SJason Wang } else { 4630167c25e4SJason Wang vi->status = VIRTIO_NET_S_LINK_UP; 4631faa9b39fSJason Baron virtnet_update_settings(vi); 46324783256eSPantelis Koukousoulas netif_carrier_on(dev); 4633167c25e4SJason Wang } 46349f4d26d0SMark McLoughlin 46353f93522fSJason Wang for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) 46363f93522fSJason Wang if (virtio_has_feature(vi->vdev, guest_offloads[i])) 46373f93522fSJason Wang set_bit(guest_offloads[i], &vi->guest_offloads); 4638a02e8964SWillem de Bruijn vi->guest_offloads_capable = vi->guest_offloads; 46393f93522fSJason Wang 4640986a4f4dSJason Wang pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 4641986a4f4dSJason Wang dev->name, max_queue_pairs); 4642986a4f4dSJason Wang 4643296f96fcSRusty Russell return 0; 4644296f96fcSRusty Russell 4645f00e35e2Swangyunjian free_unregister_netdev: 4646b3369c1fSRusty Russell unregister_netdev(dev); 4647ba5e4426SSridhar Samudrala free_failover: 4648ba5e4426SSridhar Samudrala net_failover_destroy(vi->failover); 4649d2a7dddaSMichael S. Tsirkin free_vqs: 4650b0686565SLi Zetao virtio_reset_device(vdev); 4651986a4f4dSJason Wang cancel_delayed_work_sync(&vi->refill); 4652fb51879dSMichael Dalton free_receive_page_frags(vi); 4653e9d7417bSJason Wang virtnet_del_vqs(vi); 4654296f96fcSRusty Russell free: 4655296f96fcSRusty Russell free_netdev(dev); 4656296f96fcSRusty Russell return err; 4657296f96fcSRusty Russell } 4658296f96fcSRusty Russell 465904486ed0SAmit Shah static void remove_vq_common(struct virtnet_info *vi) 4660296f96fcSRusty Russell { 4661d9679d00SMichael S. Tsirkin virtio_reset_device(vi->vdev); 4662830a8a97SShirley Ma 4663830a8a97SShirley Ma /* Free unused buffers in both send and recv, if any. */ 46649ab86bbcSShirley Ma free_unused_bufs(vi); 4665fb6813f4SRusty Russell 4666986a4f4dSJason Wang free_receive_bufs(vi); 4667d2a7dddaSMichael S. Tsirkin 4668fb51879dSMichael Dalton free_receive_page_frags(vi); 4669fb51879dSMichael Dalton 4670986a4f4dSJason Wang virtnet_del_vqs(vi); 467104486ed0SAmit Shah } 467204486ed0SAmit Shah 46738cc085d6SBill Pemberton static void virtnet_remove(struct virtio_device *vdev) 467404486ed0SAmit Shah { 467504486ed0SAmit Shah struct virtnet_info *vi = vdev->priv; 467604486ed0SAmit Shah 46778017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 46788de4b2f3SWanlong Gao 4679102a2786SMichael S. Tsirkin /* Make sure no work handler is accessing the device. */ 4680102a2786SMichael S. Tsirkin flush_work(&vi->config_work); 4681586d17c5SJason Wang 468204486ed0SAmit Shah unregister_netdev(vi->dev); 468304486ed0SAmit Shah 4684ba5e4426SSridhar Samudrala net_failover_destroy(vi->failover); 4685ba5e4426SSridhar Samudrala 468604486ed0SAmit Shah remove_vq_common(vi); 4687fb6813f4SRusty Russell 468874b2553fSRusty Russell free_netdev(vi->dev); 4689296f96fcSRusty Russell } 4690296f96fcSRusty Russell 469167a75194SArnd Bergmann static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 46920741bcb5SAmit Shah { 46930741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 46940741bcb5SAmit Shah 46958017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 46969fe7bfceSJohn Fastabend virtnet_freeze_down(vdev); 46970741bcb5SAmit Shah remove_vq_common(vi); 46980741bcb5SAmit Shah 46990741bcb5SAmit Shah return 0; 47000741bcb5SAmit Shah } 47010741bcb5SAmit Shah 470267a75194SArnd Bergmann static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 47030741bcb5SAmit Shah { 47040741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 47059fe7bfceSJohn Fastabend int err; 47060741bcb5SAmit Shah 47079fe7bfceSJohn Fastabend err = virtnet_restore_up(vdev); 47080741bcb5SAmit Shah if (err) 47090741bcb5SAmit Shah return err; 4710986a4f4dSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 4711986a4f4dSJason Wang 47128017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 47133f2869caSXie Yongji if (err) { 47143f2869caSXie Yongji virtnet_freeze_down(vdev); 47153f2869caSXie Yongji remove_vq_common(vi); 4716ec9debbdSJason Wang return err; 47173f2869caSXie Yongji } 4718ec9debbdSJason Wang 47190741bcb5SAmit Shah return 0; 47200741bcb5SAmit Shah } 47210741bcb5SAmit Shah 4722296f96fcSRusty Russell static struct virtio_device_id id_table[] = { 4723296f96fcSRusty Russell { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 4724296f96fcSRusty Russell { 0 }, 4725296f96fcSRusty Russell }; 4726296f96fcSRusty Russell 4727f3358507SMichael S. Tsirkin #define VIRTNET_FEATURES \ 4728f3358507SMichael S. Tsirkin VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 4729f3358507SMichael S. Tsirkin VIRTIO_NET_F_MAC, \ 4730f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 4731f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 4732f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 4733418044e1SAndrew Melnychenko VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \ 4734f3358507SMichael S. Tsirkin VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 4735f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 4736f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 4737f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_MAC_ADDR, \ 4738faa9b39fSJason Baron VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 4739c7114b12SAndrew Melnychenko VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ 4740be50da3eSJiri Pirko VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \ 47418af3bf66SGavin Li VIRTIO_NET_F_VQ_NOTF_COAL, \ 4742be50da3eSJiri Pirko VIRTIO_NET_F_GUEST_HDRLEN 4743f3358507SMichael S. Tsirkin 4744c45a6816SRusty Russell static unsigned int features[] = { 4745f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 4746f3358507SMichael S. Tsirkin }; 4747f3358507SMichael S. Tsirkin 4748f3358507SMichael S. Tsirkin static unsigned int features_legacy[] = { 4749f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 4750f3358507SMichael S. Tsirkin VIRTIO_NET_F_GSO, 4751e7428e95SMichael S. Tsirkin VIRTIO_F_ANY_LAYOUT, 4752c45a6816SRusty Russell }; 4753c45a6816SRusty Russell 475422402529SUwe Kleine-König static struct virtio_driver virtio_net_driver = { 4755c45a6816SRusty Russell .feature_table = features, 4756c45a6816SRusty Russell .feature_table_size = ARRAY_SIZE(features), 4757f3358507SMichael S. Tsirkin .feature_table_legacy = features_legacy, 4758f3358507SMichael S. Tsirkin .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 4759296f96fcSRusty Russell .driver.name = KBUILD_MODNAME, 4760296f96fcSRusty Russell .driver.owner = THIS_MODULE, 4761296f96fcSRusty Russell .id_table = id_table, 4762fe36cbe0SMichael S. Tsirkin .validate = virtnet_validate, 4763296f96fcSRusty Russell .probe = virtnet_probe, 47648cc085d6SBill Pemberton .remove = virtnet_remove, 47659f4d26d0SMark McLoughlin .config_changed = virtnet_config_changed, 476689107000SAaron Lu #ifdef CONFIG_PM_SLEEP 47670741bcb5SAmit Shah .freeze = virtnet_freeze, 47680741bcb5SAmit Shah .restore = virtnet_restore, 47690741bcb5SAmit Shah #endif 4770296f96fcSRusty Russell }; 4771296f96fcSRusty Russell 47728017c279SSebastian Andrzej Siewior static __init int virtio_net_driver_init(void) 47738017c279SSebastian Andrzej Siewior { 47748017c279SSebastian Andrzej Siewior int ret; 47758017c279SSebastian Andrzej Siewior 477673c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 47778017c279SSebastian Andrzej Siewior virtnet_cpu_online, 47788017c279SSebastian Andrzej Siewior virtnet_cpu_down_prep); 47798017c279SSebastian Andrzej Siewior if (ret < 0) 47808017c279SSebastian Andrzej Siewior goto out; 47818017c279SSebastian Andrzej Siewior virtionet_online = ret; 478273c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 47838017c279SSebastian Andrzej Siewior NULL, virtnet_cpu_dead); 47848017c279SSebastian Andrzej Siewior if (ret) 47858017c279SSebastian Andrzej Siewior goto err_dead; 47868017c279SSebastian Andrzej Siewior ret = register_virtio_driver(&virtio_net_driver); 47878017c279SSebastian Andrzej Siewior if (ret) 47888017c279SSebastian Andrzej Siewior goto err_virtio; 47898017c279SSebastian Andrzej Siewior return 0; 47908017c279SSebastian Andrzej Siewior err_virtio: 47918017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 47928017c279SSebastian Andrzej Siewior err_dead: 47938017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 47948017c279SSebastian Andrzej Siewior out: 47958017c279SSebastian Andrzej Siewior return ret; 47968017c279SSebastian Andrzej Siewior } 47978017c279SSebastian Andrzej Siewior module_init(virtio_net_driver_init); 47988017c279SSebastian Andrzej Siewior 47998017c279SSebastian Andrzej Siewior static __exit void virtio_net_driver_exit(void) 48008017c279SSebastian Andrzej Siewior { 4801cfa0ebc9SAndrew Jones unregister_virtio_driver(&virtio_net_driver); 48028017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 48038017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 48048017c279SSebastian Andrzej Siewior } 48058017c279SSebastian Andrzej Siewior module_exit(virtio_net_driver_exit); 4806296f96fcSRusty Russell 4807296f96fcSRusty Russell MODULE_DEVICE_TABLE(virtio, id_table); 4808296f96fcSRusty Russell MODULE_DESCRIPTION("Virtio network driver"); 4809296f96fcSRusty Russell MODULE_LICENSE("GPL"); 4810