148925e37SRusty Russell /* A network driver using virtio. 2296f96fcSRusty Russell * 3296f96fcSRusty Russell * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 4296f96fcSRusty Russell * 5296f96fcSRusty Russell * This program is free software; you can redistribute it and/or modify 6296f96fcSRusty Russell * it under the terms of the GNU General Public License as published by 7296f96fcSRusty Russell * the Free Software Foundation; either version 2 of the License, or 8296f96fcSRusty Russell * (at your option) any later version. 9296f96fcSRusty Russell * 10296f96fcSRusty Russell * This program is distributed in the hope that it will be useful, 11296f96fcSRusty Russell * but WITHOUT ANY WARRANTY; without even the implied warranty of 12296f96fcSRusty Russell * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13296f96fcSRusty Russell * GNU General Public License for more details. 14296f96fcSRusty Russell * 15296f96fcSRusty Russell * You should have received a copy of the GNU General Public License 16adf8d3ffSJeff Kirsher * along with this program; if not, see <http://www.gnu.org/licenses/>. 17296f96fcSRusty Russell */ 18296f96fcSRusty Russell //#define DEBUG 19296f96fcSRusty Russell #include <linux/netdevice.h> 20296f96fcSRusty Russell #include <linux/etherdevice.h> 21a9ea3fc6SHerbert Xu #include <linux/ethtool.h> 22296f96fcSRusty Russell #include <linux/module.h> 23296f96fcSRusty Russell #include <linux/virtio.h> 24296f96fcSRusty Russell #include <linux/virtio_net.h> 25f600b690SJohn Fastabend #include <linux/bpf.h> 26a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 27296f96fcSRusty Russell #include <linux/scatterlist.h> 28e918085aSAlex Williamson #include <linux/if_vlan.h> 295a0e3ad6STejun Heo #include <linux/slab.h> 308de4b2f3SWanlong Gao #include <linux/cpu.h> 31ab7db917SMichael Dalton #include <linux/average.h> 32186b3c99SJason Wang #include <linux/filter.h> 33d85b758fSMichael S. Tsirkin #include <net/route.h> 34754b8a21SJesper Dangaard Brouer #include <net/xdp.h> 35296f96fcSRusty Russell 36d34710e3SAmerigo Wang static int napi_weight = NAPI_POLL_WEIGHT; 376c0cd7c0SDor Laor module_param(napi_weight, int, 0444); 386c0cd7c0SDor Laor 39b92f1e67SWillem de Bruijn static bool csum = true, gso = true, napi_tx; 4034a48579SRusty Russell module_param(csum, bool, 0444); 4134a48579SRusty Russell module_param(gso, bool, 0444); 42b92f1e67SWillem de Bruijn module_param(napi_tx, bool, 0644); 4334a48579SRusty Russell 44296f96fcSRusty Russell /* FIXME: MTU in config. */ 455061de36SMichael Dalton #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 463f2c31d9SMark McLoughlin #define GOOD_COPY_LEN 128 47296f96fcSRusty Russell 48f6b10209SJason Wang #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 49f6b10209SJason Wang 502de2f7f4SJohn Fastabend /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */ 512de2f7f4SJohn Fastabend #define VIRTIO_XDP_HEADROOM 256 522de2f7f4SJohn Fastabend 535377d758SJohannes Berg /* RX packet size EWMA. The average packet size is used to determine the packet 545377d758SJohannes Berg * buffer size when refilling RX rings. As the entire RX ring may be refilled 555377d758SJohannes Berg * at once, the weight is chosen so that the EWMA will be insensitive to short- 565377d758SJohannes Berg * term, transient changes in packet size. 57ab7db917SMichael Dalton */ 58eb1e011aSJohannes Berg DECLARE_EWMA(pkt_len, 0, 64) 59ab7db917SMichael Dalton 6066846048SRick Jones #define VIRTNET_DRIVER_VERSION "1.0.0" 612a41f71dSAlex Williamson 627acd4329SColin Ian King static const unsigned long guest_offloads[] = { 637acd4329SColin Ian King VIRTIO_NET_F_GUEST_TSO4, 643f93522fSJason Wang VIRTIO_NET_F_GUEST_TSO6, 653f93522fSJason Wang VIRTIO_NET_F_GUEST_ECN, 667acd4329SColin Ian King VIRTIO_NET_F_GUEST_UFO 677acd4329SColin Ian King }; 683f93522fSJason Wang 69d7dfc5cfSToshiaki Makita struct virtnet_stat_desc { 70d7dfc5cfSToshiaki Makita char desc[ETH_GSTRING_LEN]; 71d7dfc5cfSToshiaki Makita size_t offset; 723fa2a1dfSstephen hemminger }; 733fa2a1dfSstephen hemminger 74d7dfc5cfSToshiaki Makita struct virtnet_sq_stats { 75d7dfc5cfSToshiaki Makita struct u64_stats_sync syncp; 76d7dfc5cfSToshiaki Makita u64 packets; 77d7dfc5cfSToshiaki Makita u64 bytes; 78d7dfc5cfSToshiaki Makita }; 79d7dfc5cfSToshiaki Makita 80d7dfc5cfSToshiaki Makita struct virtnet_rq_stats { 81d7dfc5cfSToshiaki Makita struct u64_stats_sync syncp; 82d7dfc5cfSToshiaki Makita u64 packets; 83d7dfc5cfSToshiaki Makita u64 bytes; 84d7dfc5cfSToshiaki Makita }; 85d7dfc5cfSToshiaki Makita 86d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m) 87d7dfc5cfSToshiaki Makita #define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m) 88d7dfc5cfSToshiaki Makita 89d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 90d7dfc5cfSToshiaki Makita { "packets", VIRTNET_SQ_STAT(packets) }, 91d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_SQ_STAT(bytes) }, 92d7dfc5cfSToshiaki Makita }; 93d7dfc5cfSToshiaki Makita 94d7dfc5cfSToshiaki Makita static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 95d7dfc5cfSToshiaki Makita { "packets", VIRTNET_RQ_STAT(packets) }, 96d7dfc5cfSToshiaki Makita { "bytes", VIRTNET_RQ_STAT(bytes) }, 97d7dfc5cfSToshiaki Makita }; 98d7dfc5cfSToshiaki Makita 99d7dfc5cfSToshiaki Makita #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc) 100d7dfc5cfSToshiaki Makita #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc) 101d7dfc5cfSToshiaki Makita 102e9d7417bSJason Wang /* Internal representation of a send virtqueue */ 103e9d7417bSJason Wang struct send_queue { 104e9d7417bSJason Wang /* Virtqueue associated with this send _queue */ 105e9d7417bSJason Wang struct virtqueue *vq; 106e9d7417bSJason Wang 107e9d7417bSJason Wang /* TX: fragments + linear part + virtio header */ 108e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 109986a4f4dSJason Wang 110986a4f4dSJason Wang /* Name of the send queue: output.$index */ 111986a4f4dSJason Wang char name[40]; 112b92f1e67SWillem de Bruijn 113d7dfc5cfSToshiaki Makita struct virtnet_sq_stats stats; 114d7dfc5cfSToshiaki Makita 115b92f1e67SWillem de Bruijn struct napi_struct napi; 116e9d7417bSJason Wang }; 117e9d7417bSJason Wang 118e9d7417bSJason Wang /* Internal representation of a receive virtqueue */ 119e9d7417bSJason Wang struct receive_queue { 120e9d7417bSJason Wang /* Virtqueue associated with this receive_queue */ 121e9d7417bSJason Wang struct virtqueue *vq; 122e9d7417bSJason Wang 123296f96fcSRusty Russell struct napi_struct napi; 124296f96fcSRusty Russell 125f600b690SJohn Fastabend struct bpf_prog __rcu *xdp_prog; 126f600b690SJohn Fastabend 127d7dfc5cfSToshiaki Makita struct virtnet_rq_stats stats; 128d7dfc5cfSToshiaki Makita 129e9d7417bSJason Wang /* Chain pages by the private ptr. */ 130e9d7417bSJason Wang struct page *pages; 131e9d7417bSJason Wang 132ab7db917SMichael Dalton /* Average packet length for mergeable receive buffers. */ 1335377d758SJohannes Berg struct ewma_pkt_len mrg_avg_pkt_len; 134ab7db917SMichael Dalton 135fb51879dSMichael Dalton /* Page frag for packet buffer allocation. */ 136fb51879dSMichael Dalton struct page_frag alloc_frag; 137fb51879dSMichael Dalton 138e9d7417bSJason Wang /* RX: fragments + linear part + virtio header */ 139e9d7417bSJason Wang struct scatterlist sg[MAX_SKB_FRAGS + 2]; 140986a4f4dSJason Wang 141d85b758fSMichael S. Tsirkin /* Min single buffer size for mergeable buffers case. */ 142d85b758fSMichael S. Tsirkin unsigned int min_buf_len; 143d85b758fSMichael S. Tsirkin 144986a4f4dSJason Wang /* Name of this receive queue: input.$index */ 145986a4f4dSJason Wang char name[40]; 146754b8a21SJesper Dangaard Brouer 147754b8a21SJesper Dangaard Brouer struct xdp_rxq_info xdp_rxq; 148e9d7417bSJason Wang }; 149e9d7417bSJason Wang 15012e57169SMichael S. Tsirkin /* Control VQ buffers: protected by the rtnl lock */ 15112e57169SMichael S. Tsirkin struct control_buf { 15212e57169SMichael S. Tsirkin struct virtio_net_ctrl_hdr hdr; 15312e57169SMichael S. Tsirkin virtio_net_ctrl_ack status; 15412e57169SMichael S. Tsirkin struct virtio_net_ctrl_mq mq; 15512e57169SMichael S. Tsirkin u8 promisc; 15612e57169SMichael S. Tsirkin u8 allmulti; 157d7fad4c8SMichael S. Tsirkin __virtio16 vid; 158f4ee703aSMichael S. Tsirkin __virtio64 offloads; 15912e57169SMichael S. Tsirkin }; 16012e57169SMichael S. Tsirkin 161e9d7417bSJason Wang struct virtnet_info { 162e9d7417bSJason Wang struct virtio_device *vdev; 163e9d7417bSJason Wang struct virtqueue *cvq; 164e9d7417bSJason Wang struct net_device *dev; 165986a4f4dSJason Wang struct send_queue *sq; 166986a4f4dSJason Wang struct receive_queue *rq; 167e9d7417bSJason Wang unsigned int status; 168e9d7417bSJason Wang 169986a4f4dSJason Wang /* Max # of queue pairs supported by the device */ 170986a4f4dSJason Wang u16 max_queue_pairs; 171986a4f4dSJason Wang 172986a4f4dSJason Wang /* # of queue pairs currently used by the driver */ 173986a4f4dSJason Wang u16 curr_queue_pairs; 174986a4f4dSJason Wang 175672aafd5SJohn Fastabend /* # of XDP queue pairs currently used by the driver */ 176672aafd5SJohn Fastabend u16 xdp_queue_pairs; 177672aafd5SJohn Fastabend 17897402b96SHerbert Xu /* I like... big packets and I cannot lie! */ 17997402b96SHerbert Xu bool big_packets; 18097402b96SHerbert Xu 1813f2c31d9SMark McLoughlin /* Host will merge rx buffers for big packets (shake it! shake it!) */ 1823f2c31d9SMark McLoughlin bool mergeable_rx_bufs; 1833f2c31d9SMark McLoughlin 184986a4f4dSJason Wang /* Has control virtqueue */ 185986a4f4dSJason Wang bool has_cvq; 186986a4f4dSJason Wang 187e7428e95SMichael S. Tsirkin /* Host can handle any s/g split between our header and packet data */ 188e7428e95SMichael S. Tsirkin bool any_header_sg; 189e7428e95SMichael S. Tsirkin 190012873d0SMichael S. Tsirkin /* Packet virtio header size */ 191012873d0SMichael S. Tsirkin u8 hdr_len; 192012873d0SMichael S. Tsirkin 1933161e453SRusty Russell /* Work struct for refilling if we run low on memory. */ 1943161e453SRusty Russell struct delayed_work refill; 1953161e453SRusty Russell 196586d17c5SJason Wang /* Work struct for config space updates */ 197586d17c5SJason Wang struct work_struct config_work; 198586d17c5SJason Wang 199986a4f4dSJason Wang /* Does the affinity hint is set for virtqueues? */ 200986a4f4dSJason Wang bool affinity_hint_set; 20147be2479SWanlong Gao 2028017c279SSebastian Andrzej Siewior /* CPU hotplug instances for online & dead */ 2038017c279SSebastian Andrzej Siewior struct hlist_node node; 2048017c279SSebastian Andrzej Siewior struct hlist_node node_dead; 2052ac46030SMichael S. Tsirkin 20612e57169SMichael S. Tsirkin struct control_buf *ctrl; 20716032be5SNikolay Aleksandrov 20816032be5SNikolay Aleksandrov /* Ethtool settings */ 20916032be5SNikolay Aleksandrov u8 duplex; 21016032be5SNikolay Aleksandrov u32 speed; 2113f93522fSJason Wang 2123f93522fSJason Wang unsigned long guest_offloads; 213296f96fcSRusty Russell }; 214296f96fcSRusty Russell 2159ab86bbcSShirley Ma struct padded_vnet_hdr { 216012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf hdr; 2179ab86bbcSShirley Ma /* 218012873d0SMichael S. Tsirkin * hdr is in a separate sg buffer, and data sg buffer shares same page 219012873d0SMichael S. Tsirkin * with this header sg. This padding makes next sg 16 byte aligned 220012873d0SMichael S. Tsirkin * after the header. 2219ab86bbcSShirley Ma */ 222012873d0SMichael S. Tsirkin char padding[4]; 2239ab86bbcSShirley Ma }; 2249ab86bbcSShirley Ma 225986a4f4dSJason Wang /* Converting between virtqueue no. and kernel tx/rx queue no. 226986a4f4dSJason Wang * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 227986a4f4dSJason Wang */ 228986a4f4dSJason Wang static int vq2txq(struct virtqueue *vq) 229986a4f4dSJason Wang { 2309d0ca6edSRusty Russell return (vq->index - 1) / 2; 231986a4f4dSJason Wang } 232986a4f4dSJason Wang 233986a4f4dSJason Wang static int txq2vq(int txq) 234986a4f4dSJason Wang { 235986a4f4dSJason Wang return txq * 2 + 1; 236986a4f4dSJason Wang } 237986a4f4dSJason Wang 238986a4f4dSJason Wang static int vq2rxq(struct virtqueue *vq) 239986a4f4dSJason Wang { 2409d0ca6edSRusty Russell return vq->index / 2; 241986a4f4dSJason Wang } 242986a4f4dSJason Wang 243986a4f4dSJason Wang static int rxq2vq(int rxq) 244986a4f4dSJason Wang { 245986a4f4dSJason Wang return rxq * 2; 246986a4f4dSJason Wang } 247986a4f4dSJason Wang 248012873d0SMichael S. Tsirkin static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb) 249296f96fcSRusty Russell { 250012873d0SMichael S. Tsirkin return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb; 251296f96fcSRusty Russell } 252296f96fcSRusty Russell 2539ab86bbcSShirley Ma /* 2549ab86bbcSShirley Ma * private is used to chain pages for big packets, put the whole 2559ab86bbcSShirley Ma * most recent used list in the beginning for reuse 2569ab86bbcSShirley Ma */ 257e9d7417bSJason Wang static void give_pages(struct receive_queue *rq, struct page *page) 258fb6813f4SRusty Russell { 2599ab86bbcSShirley Ma struct page *end; 2609ab86bbcSShirley Ma 261e9d7417bSJason Wang /* Find end of list, sew whole thing into vi->rq.pages. */ 2629ab86bbcSShirley Ma for (end = page; end->private; end = (struct page *)end->private); 263e9d7417bSJason Wang end->private = (unsigned long)rq->pages; 264e9d7417bSJason Wang rq->pages = page; 265fb6813f4SRusty Russell } 266fb6813f4SRusty Russell 267e9d7417bSJason Wang static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 268fb6813f4SRusty Russell { 269e9d7417bSJason Wang struct page *p = rq->pages; 270fb6813f4SRusty Russell 2719ab86bbcSShirley Ma if (p) { 272e9d7417bSJason Wang rq->pages = (struct page *)p->private; 2739ab86bbcSShirley Ma /* clear private here, it is used to chain pages */ 2749ab86bbcSShirley Ma p->private = 0; 2759ab86bbcSShirley Ma } else 276fb6813f4SRusty Russell p = alloc_page(gfp_mask); 277fb6813f4SRusty Russell return p; 278fb6813f4SRusty Russell } 279fb6813f4SRusty Russell 280e4e8452aSWillem de Bruijn static void virtqueue_napi_schedule(struct napi_struct *napi, 281e4e8452aSWillem de Bruijn struct virtqueue *vq) 282e4e8452aSWillem de Bruijn { 283e4e8452aSWillem de Bruijn if (napi_schedule_prep(napi)) { 284e4e8452aSWillem de Bruijn virtqueue_disable_cb(vq); 285e4e8452aSWillem de Bruijn __napi_schedule(napi); 286e4e8452aSWillem de Bruijn } 287e4e8452aSWillem de Bruijn } 288e4e8452aSWillem de Bruijn 289e4e8452aSWillem de Bruijn static void virtqueue_napi_complete(struct napi_struct *napi, 290e4e8452aSWillem de Bruijn struct virtqueue *vq, int processed) 291e4e8452aSWillem de Bruijn { 292e4e8452aSWillem de Bruijn int opaque; 293e4e8452aSWillem de Bruijn 294e4e8452aSWillem de Bruijn opaque = virtqueue_enable_cb_prepare(vq); 295fdaa767aSToshiaki Makita if (napi_complete_done(napi, processed)) { 296fdaa767aSToshiaki Makita if (unlikely(virtqueue_poll(vq, opaque))) 297e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 298fdaa767aSToshiaki Makita } else { 299fdaa767aSToshiaki Makita virtqueue_disable_cb(vq); 300fdaa767aSToshiaki Makita } 301e4e8452aSWillem de Bruijn } 302e4e8452aSWillem de Bruijn 303e9d7417bSJason Wang static void skb_xmit_done(struct virtqueue *vq) 304296f96fcSRusty Russell { 305e9d7417bSJason Wang struct virtnet_info *vi = vq->vdev->priv; 306b92f1e67SWillem de Bruijn struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi; 307296f96fcSRusty Russell 3082cb9c6baSRusty Russell /* Suppress further interrupts. */ 309e9d7417bSJason Wang virtqueue_disable_cb(vq); 31011a3a154SRusty Russell 311b92f1e67SWillem de Bruijn if (napi->weight) 312b92f1e67SWillem de Bruijn virtqueue_napi_schedule(napi, vq); 313b92f1e67SWillem de Bruijn else 314363f1514SRusty Russell /* We were probably waiting for more output buffers. */ 315986a4f4dSJason Wang netif_wake_subqueue(vi->dev, vq2txq(vq)); 316296f96fcSRusty Russell } 317296f96fcSRusty Russell 31828b39bc7SJason Wang #define MRG_CTX_HEADER_SHIFT 22 31928b39bc7SJason Wang static void *mergeable_len_to_ctx(unsigned int truesize, 32028b39bc7SJason Wang unsigned int headroom) 32128b39bc7SJason Wang { 32228b39bc7SJason Wang return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 32328b39bc7SJason Wang } 32428b39bc7SJason Wang 32528b39bc7SJason Wang static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 32628b39bc7SJason Wang { 32728b39bc7SJason Wang return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 32828b39bc7SJason Wang } 32928b39bc7SJason Wang 33028b39bc7SJason Wang static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 33128b39bc7SJason Wang { 33228b39bc7SJason Wang return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 33328b39bc7SJason Wang } 33428b39bc7SJason Wang 3353464645aSMike Waychison /* Called from bottom half context */ 336946fa564SMichael S. Tsirkin static struct sk_buff *page_to_skb(struct virtnet_info *vi, 337946fa564SMichael S. Tsirkin struct receive_queue *rq, 3382613af0eSMichael Dalton struct page *page, unsigned int offset, 3392613af0eSMichael Dalton unsigned int len, unsigned int truesize) 3409ab86bbcSShirley Ma { 3419ab86bbcSShirley Ma struct sk_buff *skb; 342012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 3432613af0eSMichael Dalton unsigned int copy, hdr_len, hdr_padded_len; 3449ab86bbcSShirley Ma char *p; 3459ab86bbcSShirley Ma 3462613af0eSMichael Dalton p = page_address(page) + offset; 3479ab86bbcSShirley Ma 3489ab86bbcSShirley Ma /* copy small packet so we can reuse these pages for small data */ 349c67f5db8SPaolo Abeni skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 3509ab86bbcSShirley Ma if (unlikely(!skb)) 3519ab86bbcSShirley Ma return NULL; 3529ab86bbcSShirley Ma 3539ab86bbcSShirley Ma hdr = skb_vnet_hdr(skb); 3549ab86bbcSShirley Ma 355012873d0SMichael S. Tsirkin hdr_len = vi->hdr_len; 356012873d0SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 357a4a76503Sstephen hemminger hdr_padded_len = sizeof(*hdr); 358012873d0SMichael S. Tsirkin else 3592613af0eSMichael Dalton hdr_padded_len = sizeof(struct padded_vnet_hdr); 3603f2c31d9SMark McLoughlin 3619ab86bbcSShirley Ma memcpy(hdr, p, hdr_len); 3623f2c31d9SMark McLoughlin 3639ab86bbcSShirley Ma len -= hdr_len; 3642613af0eSMichael Dalton offset += hdr_padded_len; 3652613af0eSMichael Dalton p += hdr_padded_len; 3663f2c31d9SMark McLoughlin 3673f2c31d9SMark McLoughlin copy = len; 3683f2c31d9SMark McLoughlin if (copy > skb_tailroom(skb)) 3693f2c31d9SMark McLoughlin copy = skb_tailroom(skb); 37059ae1d12SJohannes Berg skb_put_data(skb, p, copy); 3713f2c31d9SMark McLoughlin 3723f2c31d9SMark McLoughlin len -= copy; 3739ab86bbcSShirley Ma offset += copy; 3743f2c31d9SMark McLoughlin 3752613af0eSMichael Dalton if (vi->mergeable_rx_bufs) { 3762613af0eSMichael Dalton if (len) 3772613af0eSMichael Dalton skb_add_rx_frag(skb, 0, page, offset, len, truesize); 3782613af0eSMichael Dalton else 3792613af0eSMichael Dalton put_page(page); 3802613af0eSMichael Dalton return skb; 3812613af0eSMichael Dalton } 3822613af0eSMichael Dalton 383e878d78bSSasha Levin /* 384e878d78bSSasha Levin * Verify that we can indeed put this data into a skb. 385e878d78bSSasha Levin * This is here to handle cases when the device erroneously 386e878d78bSSasha Levin * tries to receive more than is possible. This is usually 387e878d78bSSasha Levin * the case of a broken device. 388e878d78bSSasha Levin */ 389e878d78bSSasha Levin if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { 390be443899SAmerigo Wang net_dbg_ratelimited("%s: too much data\n", skb->dev->name); 391e878d78bSSasha Levin dev_kfree_skb(skb); 392e878d78bSSasha Levin return NULL; 393e878d78bSSasha Levin } 3942613af0eSMichael Dalton BUG_ON(offset >= PAGE_SIZE); 3959ab86bbcSShirley Ma while (len) { 3962613af0eSMichael Dalton unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 3972613af0eSMichael Dalton skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 3982613af0eSMichael Dalton frag_size, truesize); 3992613af0eSMichael Dalton len -= frag_size; 4009ab86bbcSShirley Ma page = (struct page *)page->private; 4019ab86bbcSShirley Ma offset = 0; 4023f2c31d9SMark McLoughlin } 4033f2c31d9SMark McLoughlin 4049ab86bbcSShirley Ma if (page) 405e9d7417bSJason Wang give_pages(rq, page); 4063f2c31d9SMark McLoughlin 4079ab86bbcSShirley Ma return skb; 4089ab86bbcSShirley Ma } 4099ab86bbcSShirley Ma 410186b3c99SJason Wang static void virtnet_xdp_flush(struct net_device *dev) 411186b3c99SJason Wang { 412186b3c99SJason Wang struct virtnet_info *vi = netdev_priv(dev); 413186b3c99SJason Wang struct send_queue *sq; 414186b3c99SJason Wang unsigned int qp; 415186b3c99SJason Wang 416186b3c99SJason Wang qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id(); 417186b3c99SJason Wang sq = &vi->sq[qp]; 418186b3c99SJason Wang 419186b3c99SJason Wang virtqueue_kick(sq->vq); 420186b3c99SJason Wang } 421186b3c99SJason Wang 422735fc405SJesper Dangaard Brouer static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 423735fc405SJesper Dangaard Brouer struct send_queue *sq, 42444fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf) 42556434a01SJohn Fastabend { 42656434a01SJohn Fastabend struct virtio_net_hdr_mrg_rxbuf *hdr; 42756434a01SJohn Fastabend int err; 42856434a01SJohn Fastabend 429cac320c8SJesper Dangaard Brouer /* virtqueue want to use data area in-front of packet */ 430cac320c8SJesper Dangaard Brouer if (unlikely(xdpf->metasize > 0)) 431cac320c8SJesper Dangaard Brouer return -EOPNOTSUPP; 432cac320c8SJesper Dangaard Brouer 433cac320c8SJesper Dangaard Brouer if (unlikely(xdpf->headroom < vi->hdr_len)) 434cac320c8SJesper Dangaard Brouer return -EOVERFLOW; 435cac320c8SJesper Dangaard Brouer 436cac320c8SJesper Dangaard Brouer /* Make room for virtqueue hdr (also change xdpf->headroom?) */ 437cac320c8SJesper Dangaard Brouer xdpf->data -= vi->hdr_len; 43856434a01SJohn Fastabend /* Zero header and leave csum up to XDP layers */ 439cac320c8SJesper Dangaard Brouer hdr = xdpf->data; 44056434a01SJohn Fastabend memset(hdr, 0, vi->hdr_len); 441cac320c8SJesper Dangaard Brouer xdpf->len += vi->hdr_len; 44256434a01SJohn Fastabend 443cac320c8SJesper Dangaard Brouer sg_init_one(sq->sg, xdpf->data, xdpf->len); 444bb91accfSJason Wang 445cac320c8SJesper Dangaard Brouer err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdpf, GFP_ATOMIC); 44611b7d897SJesper Dangaard Brouer if (unlikely(err)) 447cac320c8SJesper Dangaard Brouer return -ENOSPC; /* Caller handle free/refcnt */ 44856434a01SJohn Fastabend 449cac320c8SJesper Dangaard Brouer return 0; 45056434a01SJohn Fastabend } 45156434a01SJohn Fastabend 452735fc405SJesper Dangaard Brouer static int __virtnet_xdp_tx_xmit(struct virtnet_info *vi, 453735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf) 454735fc405SJesper Dangaard Brouer { 455735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf_sent; 456735fc405SJesper Dangaard Brouer struct send_queue *sq; 457735fc405SJesper Dangaard Brouer unsigned int len; 458735fc405SJesper Dangaard Brouer unsigned int qp; 459735fc405SJesper Dangaard Brouer 460735fc405SJesper Dangaard Brouer qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id(); 461735fc405SJesper Dangaard Brouer sq = &vi->sq[qp]; 462735fc405SJesper Dangaard Brouer 463735fc405SJesper Dangaard Brouer /* Free up any pending old buffers before queueing new ones. */ 464735fc405SJesper Dangaard Brouer while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) 465735fc405SJesper Dangaard Brouer xdp_return_frame(xdpf_sent); 466735fc405SJesper Dangaard Brouer 467735fc405SJesper Dangaard Brouer return __virtnet_xdp_xmit_one(vi, sq, xdpf); 468735fc405SJesper Dangaard Brouer } 469735fc405SJesper Dangaard Brouer 470735fc405SJesper Dangaard Brouer static int virtnet_xdp_xmit(struct net_device *dev, 471735fc405SJesper Dangaard Brouer int n, struct xdp_frame **frames) 472186b3c99SJason Wang { 473186b3c99SJason Wang struct virtnet_info *vi = netdev_priv(dev); 4748dcc5b0aSJesper Dangaard Brouer struct receive_queue *rq = vi->rq; 475735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf_sent; 4768dcc5b0aSJesper Dangaard Brouer struct bpf_prog *xdp_prog; 477735fc405SJesper Dangaard Brouer struct send_queue *sq; 478735fc405SJesper Dangaard Brouer unsigned int len; 479735fc405SJesper Dangaard Brouer unsigned int qp; 480735fc405SJesper Dangaard Brouer int drops = 0; 481735fc405SJesper Dangaard Brouer int err; 482735fc405SJesper Dangaard Brouer int i; 483735fc405SJesper Dangaard Brouer 484735fc405SJesper Dangaard Brouer qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id(); 485735fc405SJesper Dangaard Brouer sq = &vi->sq[qp]; 486186b3c99SJason Wang 4878dcc5b0aSJesper Dangaard Brouer /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 4888dcc5b0aSJesper Dangaard Brouer * indicate XDP resources have been successfully allocated. 4898dcc5b0aSJesper Dangaard Brouer */ 4908dcc5b0aSJesper Dangaard Brouer xdp_prog = rcu_dereference(rq->xdp_prog); 4918dcc5b0aSJesper Dangaard Brouer if (!xdp_prog) 4928dcc5b0aSJesper Dangaard Brouer return -ENXIO; 4938dcc5b0aSJesper Dangaard Brouer 494735fc405SJesper Dangaard Brouer /* Free up any pending old buffers before queueing new ones. */ 495735fc405SJesper Dangaard Brouer while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) 496735fc405SJesper Dangaard Brouer xdp_return_frame(xdpf_sent); 497735fc405SJesper Dangaard Brouer 498735fc405SJesper Dangaard Brouer for (i = 0; i < n; i++) { 499735fc405SJesper Dangaard Brouer struct xdp_frame *xdpf = frames[i]; 500735fc405SJesper Dangaard Brouer 501735fc405SJesper Dangaard Brouer err = __virtnet_xdp_xmit_one(vi, sq, xdpf); 502735fc405SJesper Dangaard Brouer if (err) { 503735fc405SJesper Dangaard Brouer xdp_return_frame_rx_napi(xdpf); 504735fc405SJesper Dangaard Brouer drops++; 505735fc405SJesper Dangaard Brouer } 506735fc405SJesper Dangaard Brouer } 507735fc405SJesper Dangaard Brouer return n - drops; 508186b3c99SJason Wang } 509186b3c99SJason Wang 510f6b10209SJason Wang static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 511f6b10209SJason Wang { 512f6b10209SJason Wang return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0; 513f6b10209SJason Wang } 514f6b10209SJason Wang 5154941d472SJason Wang /* We copy the packet for XDP in the following cases: 5164941d472SJason Wang * 5174941d472SJason Wang * 1) Packet is scattered across multiple rx buffers. 5184941d472SJason Wang * 2) Headroom space is insufficient. 5194941d472SJason Wang * 5204941d472SJason Wang * This is inefficient but it's a temporary condition that 5214941d472SJason Wang * we hit right after XDP is enabled and until queue is refilled 5224941d472SJason Wang * with large buffers with sufficient headroom - so it should affect 5234941d472SJason Wang * at most queue size packets. 5244941d472SJason Wang * Afterwards, the conditions to enable 5254941d472SJason Wang * XDP should preclude the underlying device from sending packets 5264941d472SJason Wang * across multiple buffers (num_buf > 1), and we make sure buffers 5274941d472SJason Wang * have enough headroom. 52872979a6cSJohn Fastabend */ 52972979a6cSJohn Fastabend static struct page *xdp_linearize_page(struct receive_queue *rq, 53056a86f84SJason Wang u16 *num_buf, 53172979a6cSJohn Fastabend struct page *p, 53272979a6cSJohn Fastabend int offset, 5334941d472SJason Wang int page_off, 53472979a6cSJohn Fastabend unsigned int *len) 53572979a6cSJohn Fastabend { 53672979a6cSJohn Fastabend struct page *page = alloc_page(GFP_ATOMIC); 53772979a6cSJohn Fastabend 53872979a6cSJohn Fastabend if (!page) 53972979a6cSJohn Fastabend return NULL; 54072979a6cSJohn Fastabend 54172979a6cSJohn Fastabend memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 54272979a6cSJohn Fastabend page_off += *len; 54372979a6cSJohn Fastabend 54456a86f84SJason Wang while (--*num_buf) { 5453cc81a9aSJason Wang int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 54672979a6cSJohn Fastabend unsigned int buflen; 54772979a6cSJohn Fastabend void *buf; 54872979a6cSJohn Fastabend int off; 54972979a6cSJohn Fastabend 550680557cfSMichael S. Tsirkin buf = virtqueue_get_buf(rq->vq, &buflen); 551680557cfSMichael S. Tsirkin if (unlikely(!buf)) 55272979a6cSJohn Fastabend goto err_buf; 55372979a6cSJohn Fastabend 55472979a6cSJohn Fastabend p = virt_to_head_page(buf); 55572979a6cSJohn Fastabend off = buf - page_address(p); 55672979a6cSJohn Fastabend 55756a86f84SJason Wang /* guard against a misconfigured or uncooperative backend that 55856a86f84SJason Wang * is sending packet larger than the MTU. 55956a86f84SJason Wang */ 5603cc81a9aSJason Wang if ((page_off + buflen + tailroom) > PAGE_SIZE) { 56156a86f84SJason Wang put_page(p); 56256a86f84SJason Wang goto err_buf; 56356a86f84SJason Wang } 56456a86f84SJason Wang 56572979a6cSJohn Fastabend memcpy(page_address(page) + page_off, 56672979a6cSJohn Fastabend page_address(p) + off, buflen); 56772979a6cSJohn Fastabend page_off += buflen; 56856a86f84SJason Wang put_page(p); 56972979a6cSJohn Fastabend } 57072979a6cSJohn Fastabend 5712de2f7f4SJohn Fastabend /* Headroom does not contribute to packet length */ 5722de2f7f4SJohn Fastabend *len = page_off - VIRTIO_XDP_HEADROOM; 57372979a6cSJohn Fastabend return page; 57472979a6cSJohn Fastabend err_buf: 57572979a6cSJohn Fastabend __free_pages(page, 0); 57672979a6cSJohn Fastabend return NULL; 57772979a6cSJohn Fastabend } 57872979a6cSJohn Fastabend 5794941d472SJason Wang static struct sk_buff *receive_small(struct net_device *dev, 5804941d472SJason Wang struct virtnet_info *vi, 5814941d472SJason Wang struct receive_queue *rq, 5824941d472SJason Wang void *buf, void *ctx, 583186b3c99SJason Wang unsigned int len, 584186b3c99SJason Wang bool *xdp_xmit) 5854941d472SJason Wang { 5864941d472SJason Wang struct sk_buff *skb; 5874941d472SJason Wang struct bpf_prog *xdp_prog; 5884941d472SJason Wang unsigned int xdp_headroom = (unsigned long)ctx; 5894941d472SJason Wang unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 5904941d472SJason Wang unsigned int headroom = vi->hdr_len + header_offset; 5914941d472SJason Wang unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 5924941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 5934941d472SJason Wang struct page *page = virt_to_head_page(buf); 59411b7d897SJesper Dangaard Brouer unsigned int delta = 0; 5954941d472SJason Wang struct page *xdp_page; 59611b7d897SJesper Dangaard Brouer int err; 59711b7d897SJesper Dangaard Brouer 5984941d472SJason Wang len -= vi->hdr_len; 5994941d472SJason Wang 6004941d472SJason Wang rcu_read_lock(); 6014941d472SJason Wang xdp_prog = rcu_dereference(rq->xdp_prog); 6024941d472SJason Wang if (xdp_prog) { 6034941d472SJason Wang struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 60444fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf; 6054941d472SJason Wang struct xdp_buff xdp; 6064941d472SJason Wang void *orig_data; 6074941d472SJason Wang u32 act; 6084941d472SJason Wang 60995dbe9e7SJesper Dangaard Brouer if (unlikely(hdr->hdr.gso_type)) 6104941d472SJason Wang goto err_xdp; 6114941d472SJason Wang 6124941d472SJason Wang if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 6134941d472SJason Wang int offset = buf - page_address(page) + header_offset; 6144941d472SJason Wang unsigned int tlen = len + vi->hdr_len; 6154941d472SJason Wang u16 num_buf = 1; 6164941d472SJason Wang 6174941d472SJason Wang xdp_headroom = virtnet_get_headroom(vi); 6184941d472SJason Wang header_offset = VIRTNET_RX_PAD + xdp_headroom; 6194941d472SJason Wang headroom = vi->hdr_len + header_offset; 6204941d472SJason Wang buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 6214941d472SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 6224941d472SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, page, 6234941d472SJason Wang offset, header_offset, 6244941d472SJason Wang &tlen); 6254941d472SJason Wang if (!xdp_page) 6264941d472SJason Wang goto err_xdp; 6274941d472SJason Wang 6284941d472SJason Wang buf = page_address(xdp_page); 6294941d472SJason Wang put_page(page); 6304941d472SJason Wang page = xdp_page; 6314941d472SJason Wang } 6324941d472SJason Wang 6334941d472SJason Wang xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; 6344941d472SJason Wang xdp.data = xdp.data_hard_start + xdp_headroom; 635de8f3a83SDaniel Borkmann xdp_set_data_meta_invalid(&xdp); 6364941d472SJason Wang xdp.data_end = xdp.data + len; 637754b8a21SJesper Dangaard Brouer xdp.rxq = &rq->xdp_rxq; 6384941d472SJason Wang orig_data = xdp.data; 6394941d472SJason Wang act = bpf_prog_run_xdp(xdp_prog, &xdp); 6404941d472SJason Wang 6414941d472SJason Wang switch (act) { 6424941d472SJason Wang case XDP_PASS: 6434941d472SJason Wang /* Recalculate length in case bpf program changed it */ 6444941d472SJason Wang delta = orig_data - xdp.data; 6456870de43SNikita V. Shirokov len = xdp.data_end - xdp.data; 6464941d472SJason Wang break; 6474941d472SJason Wang case XDP_TX: 64844fa2dbdSJesper Dangaard Brouer xdpf = convert_to_xdp_frame(&xdp); 64944fa2dbdSJesper Dangaard Brouer if (unlikely(!xdpf)) 65044fa2dbdSJesper Dangaard Brouer goto err_xdp; 651735fc405SJesper Dangaard Brouer err = __virtnet_xdp_tx_xmit(vi, xdpf); 652cac320c8SJesper Dangaard Brouer if (unlikely(err)) { 6534941d472SJason Wang trace_xdp_exception(vi->dev, xdp_prog, act); 65411b7d897SJesper Dangaard Brouer goto err_xdp; 65511b7d897SJesper Dangaard Brouer } 656186b3c99SJason Wang *xdp_xmit = true; 657186b3c99SJason Wang rcu_read_unlock(); 658186b3c99SJason Wang goto xdp_xmit; 659186b3c99SJason Wang case XDP_REDIRECT: 660186b3c99SJason Wang err = xdp_do_redirect(dev, &xdp, xdp_prog); 66111b7d897SJesper Dangaard Brouer if (err) 66211b7d897SJesper Dangaard Brouer goto err_xdp; 663186b3c99SJason Wang *xdp_xmit = true; 6644941d472SJason Wang rcu_read_unlock(); 6654941d472SJason Wang goto xdp_xmit; 6664941d472SJason Wang default: 6674941d472SJason Wang bpf_warn_invalid_xdp_action(act); 6684941d472SJason Wang case XDP_ABORTED: 6694941d472SJason Wang trace_xdp_exception(vi->dev, xdp_prog, act); 6704941d472SJason Wang case XDP_DROP: 6714941d472SJason Wang goto err_xdp; 6724941d472SJason Wang } 6734941d472SJason Wang } 6744941d472SJason Wang rcu_read_unlock(); 6754941d472SJason Wang 6764941d472SJason Wang skb = build_skb(buf, buflen); 6774941d472SJason Wang if (!skb) { 6784941d472SJason Wang put_page(page); 6794941d472SJason Wang goto err; 6804941d472SJason Wang } 6814941d472SJason Wang skb_reserve(skb, headroom - delta); 6826870de43SNikita V. Shirokov skb_put(skb, len); 6834941d472SJason Wang if (!delta) { 6844941d472SJason Wang buf += header_offset; 6854941d472SJason Wang memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); 6864941d472SJason Wang } /* keep zeroed vnet hdr since packet was changed by bpf */ 6874941d472SJason Wang 6884941d472SJason Wang err: 6894941d472SJason Wang return skb; 6904941d472SJason Wang 6914941d472SJason Wang err_xdp: 6924941d472SJason Wang rcu_read_unlock(); 6934941d472SJason Wang dev->stats.rx_dropped++; 6944941d472SJason Wang put_page(page); 6954941d472SJason Wang xdp_xmit: 6964941d472SJason Wang return NULL; 6974941d472SJason Wang } 6984941d472SJason Wang 6994941d472SJason Wang static struct sk_buff *receive_big(struct net_device *dev, 7004941d472SJason Wang struct virtnet_info *vi, 7014941d472SJason Wang struct receive_queue *rq, 7024941d472SJason Wang void *buf, 7034941d472SJason Wang unsigned int len) 7044941d472SJason Wang { 7054941d472SJason Wang struct page *page = buf; 7064941d472SJason Wang struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE); 7074941d472SJason Wang 7084941d472SJason Wang if (unlikely(!skb)) 7094941d472SJason Wang goto err; 7104941d472SJason Wang 7114941d472SJason Wang return skb; 7124941d472SJason Wang 7134941d472SJason Wang err: 7144941d472SJason Wang dev->stats.rx_dropped++; 7154941d472SJason Wang give_pages(rq, page); 7164941d472SJason Wang return NULL; 7174941d472SJason Wang } 7184941d472SJason Wang 7198fc3b9e9SMichael S. Tsirkin static struct sk_buff *receive_mergeable(struct net_device *dev, 720fdd819b2SMichael S. Tsirkin struct virtnet_info *vi, 7218fc3b9e9SMichael S. Tsirkin struct receive_queue *rq, 722680557cfSMichael S. Tsirkin void *buf, 723680557cfSMichael S. Tsirkin void *ctx, 724186b3c99SJason Wang unsigned int len, 725186b3c99SJason Wang bool *xdp_xmit) 7269ab86bbcSShirley Ma { 727012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 728012873d0SMichael S. Tsirkin u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 7298fc3b9e9SMichael S. Tsirkin struct page *page = virt_to_head_page(buf); 7308fc3b9e9SMichael S. Tsirkin int offset = buf - page_address(page); 731f600b690SJohn Fastabend struct sk_buff *head_skb, *curr_skb; 732f600b690SJohn Fastabend struct bpf_prog *xdp_prog; 733f600b690SJohn Fastabend unsigned int truesize; 7344941d472SJason Wang unsigned int headroom = mergeable_ctx_to_headroom(ctx); 7353cc81a9aSJason Wang int err; 736ab7db917SMichael Dalton 73756434a01SJohn Fastabend head_skb = NULL; 73856434a01SJohn Fastabend 739f600b690SJohn Fastabend rcu_read_lock(); 740f600b690SJohn Fastabend xdp_prog = rcu_dereference(rq->xdp_prog); 741f600b690SJohn Fastabend if (xdp_prog) { 74244fa2dbdSJesper Dangaard Brouer struct xdp_frame *xdpf; 74372979a6cSJohn Fastabend struct page *xdp_page; 7440354e4d1SJohn Fastabend struct xdp_buff xdp; 7450354e4d1SJohn Fastabend void *data; 746f600b690SJohn Fastabend u32 act; 747f600b690SJohn Fastabend 7483d62b2a0SJason Wang /* Transient failure which in theory could occur if 7493d62b2a0SJason Wang * in-flight packets from before XDP was enabled reach 7503d62b2a0SJason Wang * the receive path after XDP is loaded. 7513d62b2a0SJason Wang */ 7523d62b2a0SJason Wang if (unlikely(hdr->hdr.gso_type)) 7533d62b2a0SJason Wang goto err_xdp; 7543d62b2a0SJason Wang 7553cc81a9aSJason Wang /* This happens when rx buffer size is underestimated 7563cc81a9aSJason Wang * or headroom is not enough because of the buffer 7573cc81a9aSJason Wang * was refilled before XDP is set. This should only 7583cc81a9aSJason Wang * happen for the first several packets, so we don't 7593cc81a9aSJason Wang * care much about its performance. 7603cc81a9aSJason Wang */ 7614941d472SJason Wang if (unlikely(num_buf > 1 || 7624941d472SJason Wang headroom < virtnet_get_headroom(vi))) { 76372979a6cSJohn Fastabend /* linearize data for XDP */ 76456a86f84SJason Wang xdp_page = xdp_linearize_page(rq, &num_buf, 7654941d472SJason Wang page, offset, 7664941d472SJason Wang VIRTIO_XDP_HEADROOM, 7674941d472SJason Wang &len); 76872979a6cSJohn Fastabend if (!xdp_page) 769f600b690SJohn Fastabend goto err_xdp; 7702de2f7f4SJohn Fastabend offset = VIRTIO_XDP_HEADROOM; 77172979a6cSJohn Fastabend } else { 77272979a6cSJohn Fastabend xdp_page = page; 773f600b690SJohn Fastabend } 774f600b690SJohn Fastabend 7752de2f7f4SJohn Fastabend /* Allow consuming headroom but reserve enough space to push 7762de2f7f4SJohn Fastabend * the descriptor on if we get an XDP_TX return code. 7772de2f7f4SJohn Fastabend */ 7780354e4d1SJohn Fastabend data = page_address(xdp_page) + offset; 7792de2f7f4SJohn Fastabend xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; 7800354e4d1SJohn Fastabend xdp.data = data + vi->hdr_len; 781de8f3a83SDaniel Borkmann xdp_set_data_meta_invalid(&xdp); 7820354e4d1SJohn Fastabend xdp.data_end = xdp.data + (len - vi->hdr_len); 783754b8a21SJesper Dangaard Brouer xdp.rxq = &rq->xdp_rxq; 784754b8a21SJesper Dangaard Brouer 7850354e4d1SJohn Fastabend act = bpf_prog_run_xdp(xdp_prog, &xdp); 7860354e4d1SJohn Fastabend 78756434a01SJohn Fastabend switch (act) { 78856434a01SJohn Fastabend case XDP_PASS: 7892de2f7f4SJohn Fastabend /* recalculate offset to account for any header 7902de2f7f4SJohn Fastabend * adjustments. Note other cases do not build an 7912de2f7f4SJohn Fastabend * skb and avoid using offset 7922de2f7f4SJohn Fastabend */ 7932de2f7f4SJohn Fastabend offset = xdp.data - 7942de2f7f4SJohn Fastabend page_address(xdp_page) - vi->hdr_len; 7952de2f7f4SJohn Fastabend 7966870de43SNikita V. Shirokov /* recalculate len if xdp.data or xdp.data_end were 7976870de43SNikita V. Shirokov * adjusted 7986870de43SNikita V. Shirokov */ 799aaa64527SNikita V. Shirokov len = xdp.data_end - xdp.data + vi->hdr_len; 8001830f893SJason Wang /* We can only create skb based on xdp_page. */ 8011830f893SJason Wang if (unlikely(xdp_page != page)) { 8021830f893SJason Wang rcu_read_unlock(); 8031830f893SJason Wang put_page(page); 8041830f893SJason Wang head_skb = page_to_skb(vi, rq, xdp_page, 8052de2f7f4SJohn Fastabend offset, len, PAGE_SIZE); 8061830f893SJason Wang return head_skb; 8071830f893SJason Wang } 80856434a01SJohn Fastabend break; 80956434a01SJohn Fastabend case XDP_TX: 81044fa2dbdSJesper Dangaard Brouer xdpf = convert_to_xdp_frame(&xdp); 81144fa2dbdSJesper Dangaard Brouer if (unlikely(!xdpf)) 81244fa2dbdSJesper Dangaard Brouer goto err_xdp; 813735fc405SJesper Dangaard Brouer err = __virtnet_xdp_tx_xmit(vi, xdpf); 814cac320c8SJesper Dangaard Brouer if (unlikely(err)) { 8150354e4d1SJohn Fastabend trace_xdp_exception(vi->dev, xdp_prog, act); 81611b7d897SJesper Dangaard Brouer if (unlikely(xdp_page != page)) 81711b7d897SJesper Dangaard Brouer put_page(xdp_page); 81811b7d897SJesper Dangaard Brouer goto err_xdp; 81911b7d897SJesper Dangaard Brouer } 820186b3c99SJason Wang *xdp_xmit = true; 82172979a6cSJohn Fastabend if (unlikely(xdp_page != page)) 8225d458a13SJason Wang put_page(page); 82356434a01SJohn Fastabend rcu_read_unlock(); 82456434a01SJohn Fastabend goto xdp_xmit; 8253cc81a9aSJason Wang case XDP_REDIRECT: 8263cc81a9aSJason Wang err = xdp_do_redirect(dev, &xdp, xdp_prog); 8273cc81a9aSJason Wang if (err) { 8283cc81a9aSJason Wang if (unlikely(xdp_page != page)) 8293cc81a9aSJason Wang put_page(xdp_page); 8303cc81a9aSJason Wang goto err_xdp; 8313cc81a9aSJason Wang } 8323cc81a9aSJason Wang *xdp_xmit = true; 8333cc81a9aSJason Wang if (unlikely(xdp_page != page)) 8346890418bSJason Wang put_page(page); 8353cc81a9aSJason Wang rcu_read_unlock(); 8363cc81a9aSJason Wang goto xdp_xmit; 83756434a01SJohn Fastabend default: 8380354e4d1SJohn Fastabend bpf_warn_invalid_xdp_action(act); 8390354e4d1SJohn Fastabend case XDP_ABORTED: 8400354e4d1SJohn Fastabend trace_xdp_exception(vi->dev, xdp_prog, act); 8410354e4d1SJohn Fastabend case XDP_DROP: 84272979a6cSJohn Fastabend if (unlikely(xdp_page != page)) 84372979a6cSJohn Fastabend __free_pages(xdp_page, 0); 844f600b690SJohn Fastabend goto err_xdp; 845f600b690SJohn Fastabend } 84656434a01SJohn Fastabend } 847f600b690SJohn Fastabend rcu_read_unlock(); 848f600b690SJohn Fastabend 84928b39bc7SJason Wang truesize = mergeable_ctx_to_truesize(ctx); 85028b39bc7SJason Wang if (unlikely(len > truesize)) { 85156da5fd0SDan Carpenter pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 852680557cfSMichael S. Tsirkin dev->name, len, (unsigned long)ctx); 853680557cfSMichael S. Tsirkin dev->stats.rx_length_errors++; 854680557cfSMichael S. Tsirkin goto err_skb; 855680557cfSMichael S. Tsirkin } 85628b39bc7SJason Wang 857f600b690SJohn Fastabend head_skb = page_to_skb(vi, rq, page, offset, len, truesize); 858f600b690SJohn Fastabend curr_skb = head_skb; 8599ab86bbcSShirley Ma 8608fc3b9e9SMichael S. Tsirkin if (unlikely(!curr_skb)) 8618fc3b9e9SMichael S. Tsirkin goto err_skb; 8629ab86bbcSShirley Ma while (--num_buf) { 8638fc3b9e9SMichael S. Tsirkin int num_skb_frags; 8648fc3b9e9SMichael S. Tsirkin 865680557cfSMichael S. Tsirkin buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 86603e9f8a0SYunjian Wang if (unlikely(!buf)) { 8678fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers out of %d missing\n", 868fdd819b2SMichael S. Tsirkin dev->name, num_buf, 869012873d0SMichael S. Tsirkin virtio16_to_cpu(vi->vdev, 870012873d0SMichael S. Tsirkin hdr->num_buffers)); 8718fc3b9e9SMichael S. Tsirkin dev->stats.rx_length_errors++; 8728fc3b9e9SMichael S. Tsirkin goto err_buf; 8733f2c31d9SMark McLoughlin } 8748fc3b9e9SMichael S. Tsirkin 8758fc3b9e9SMichael S. Tsirkin page = virt_to_head_page(buf); 87628b39bc7SJason Wang 87728b39bc7SJason Wang truesize = mergeable_ctx_to_truesize(ctx); 87828b39bc7SJason Wang if (unlikely(len > truesize)) { 87956da5fd0SDan Carpenter pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 880680557cfSMichael S. Tsirkin dev->name, len, (unsigned long)ctx); 881680557cfSMichael S. Tsirkin dev->stats.rx_length_errors++; 882680557cfSMichael S. Tsirkin goto err_skb; 883680557cfSMichael S. Tsirkin } 8848fc3b9e9SMichael S. Tsirkin 8858fc3b9e9SMichael S. Tsirkin num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 8862613af0eSMichael Dalton if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 8872613af0eSMichael Dalton struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 8888fc3b9e9SMichael S. Tsirkin 8898fc3b9e9SMichael S. Tsirkin if (unlikely(!nskb)) 8908fc3b9e9SMichael S. Tsirkin goto err_skb; 8912613af0eSMichael Dalton if (curr_skb == head_skb) 8922613af0eSMichael Dalton skb_shinfo(curr_skb)->frag_list = nskb; 8932613af0eSMichael Dalton else 8942613af0eSMichael Dalton curr_skb->next = nskb; 8952613af0eSMichael Dalton curr_skb = nskb; 8962613af0eSMichael Dalton head_skb->truesize += nskb->truesize; 8972613af0eSMichael Dalton num_skb_frags = 0; 8982613af0eSMichael Dalton } 8992613af0eSMichael Dalton if (curr_skb != head_skb) { 9002613af0eSMichael Dalton head_skb->data_len += len; 9012613af0eSMichael Dalton head_skb->len += len; 902fb51879dSMichael Dalton head_skb->truesize += truesize; 9032613af0eSMichael Dalton } 9048fc3b9e9SMichael S. Tsirkin offset = buf - page_address(page); 905ba275241SJason Wang if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 906ba275241SJason Wang put_page(page); 907ba275241SJason Wang skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 908fb51879dSMichael Dalton len, truesize); 909ba275241SJason Wang } else { 9102613af0eSMichael Dalton skb_add_rx_frag(curr_skb, num_skb_frags, page, 911fb51879dSMichael Dalton offset, len, truesize); 912ba275241SJason Wang } 9138fc3b9e9SMichael S. Tsirkin } 9148fc3b9e9SMichael S. Tsirkin 9155377d758SJohannes Berg ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 9168fc3b9e9SMichael S. Tsirkin return head_skb; 9178fc3b9e9SMichael S. Tsirkin 918f600b690SJohn Fastabend err_xdp: 919f600b690SJohn Fastabend rcu_read_unlock(); 9208fc3b9e9SMichael S. Tsirkin err_skb: 9218fc3b9e9SMichael S. Tsirkin put_page(page); 922850e088dSJason Wang while (num_buf-- > 1) { 923680557cfSMichael S. Tsirkin buf = virtqueue_get_buf(rq->vq, &len); 924680557cfSMichael S. Tsirkin if (unlikely(!buf)) { 9258fc3b9e9SMichael S. Tsirkin pr_debug("%s: rx error: %d buffers missing\n", 9268fc3b9e9SMichael S. Tsirkin dev->name, num_buf); 9278fc3b9e9SMichael S. Tsirkin dev->stats.rx_length_errors++; 9288fc3b9e9SMichael S. Tsirkin break; 9298fc3b9e9SMichael S. Tsirkin } 930680557cfSMichael S. Tsirkin page = virt_to_head_page(buf); 9318fc3b9e9SMichael S. Tsirkin put_page(page); 9323f2c31d9SMark McLoughlin } 9338fc3b9e9SMichael S. Tsirkin err_buf: 9348fc3b9e9SMichael S. Tsirkin dev->stats.rx_dropped++; 9358fc3b9e9SMichael S. Tsirkin dev_kfree_skb(head_skb); 93656434a01SJohn Fastabend xdp_xmit: 9378fc3b9e9SMichael S. Tsirkin return NULL; 9389ab86bbcSShirley Ma } 9399ab86bbcSShirley Ma 94061845d20SJason Wang static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 941186b3c99SJason Wang void *buf, unsigned int len, void **ctx, bool *xdp_xmit) 9429ab86bbcSShirley Ma { 943e9d7417bSJason Wang struct net_device *dev = vi->dev; 9449ab86bbcSShirley Ma struct sk_buff *skb; 945012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 94661845d20SJason Wang int ret; 9479ab86bbcSShirley Ma 948bcff3162SMichael S. Tsirkin if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 9499ab86bbcSShirley Ma pr_debug("%s: short packet %i\n", dev->name, len); 9509ab86bbcSShirley Ma dev->stats.rx_length_errors++; 951ab7db917SMichael Dalton if (vi->mergeable_rx_bufs) { 952680557cfSMichael S. Tsirkin put_page(virt_to_head_page(buf)); 953ab7db917SMichael Dalton } else if (vi->big_packets) { 95498bfd23cSMichael Dalton give_pages(rq, buf); 955ab7db917SMichael Dalton } else { 956f6b10209SJason Wang put_page(virt_to_head_page(buf)); 957ab7db917SMichael Dalton } 95861845d20SJason Wang return 0; 9599ab86bbcSShirley Ma } 9609ab86bbcSShirley Ma 961f121159dSMichael S. Tsirkin if (vi->mergeable_rx_bufs) 962186b3c99SJason Wang skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit); 963f121159dSMichael S. Tsirkin else if (vi->big_packets) 964946fa564SMichael S. Tsirkin skb = receive_big(dev, vi, rq, buf, len); 965f121159dSMichael S. Tsirkin else 966186b3c99SJason Wang skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit); 967f121159dSMichael S. Tsirkin 9688fc3b9e9SMichael S. Tsirkin if (unlikely(!skb)) 96961845d20SJason Wang return 0; 9703f2c31d9SMark McLoughlin 9719ab86bbcSShirley Ma hdr = skb_vnet_hdr(skb); 9723fa2a1dfSstephen hemminger 97361845d20SJason Wang ret = skb->len; 974296f96fcSRusty Russell 975e858fae2SMike Rapoport if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) 97610a8d94aSJason Wang skb->ip_summed = CHECKSUM_UNNECESSARY; 977296f96fcSRusty Russell 978e858fae2SMike Rapoport if (virtio_net_hdr_to_skb(skb, &hdr->hdr, 979e858fae2SMike Rapoport virtio_is_little_endian(vi->vdev))) { 980e858fae2SMike Rapoport net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n", 981e858fae2SMike Rapoport dev->name, hdr->hdr.gso_type, 982fdd819b2SMichael S. Tsirkin hdr->hdr.gso_size); 983296f96fcSRusty Russell goto frame_err; 984296f96fcSRusty Russell } 985296f96fcSRusty Russell 986d1dc06dcSMike Rapoport skb->protocol = eth_type_trans(skb, dev); 987d1dc06dcSMike Rapoport pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 988d1dc06dcSMike Rapoport ntohs(skb->protocol), skb->len, skb->pkt_type); 989d1dc06dcSMike Rapoport 9900fbd050aSEric Dumazet napi_gro_receive(&rq->napi, skb); 99161845d20SJason Wang return ret; 992296f96fcSRusty Russell 993296f96fcSRusty Russell frame_err: 994296f96fcSRusty Russell dev->stats.rx_frame_errors++; 995296f96fcSRusty Russell dev_kfree_skb(skb); 99661845d20SJason Wang return 0; 997296f96fcSRusty Russell } 998296f96fcSRusty Russell 999192f68cfSJason Wang /* Unlike mergeable buffers, all buffers are allocated to the 1000192f68cfSJason Wang * same size, except for the headroom. For this reason we do 1001192f68cfSJason Wang * not need to use mergeable_len_to_ctx here - it is enough 1002192f68cfSJason Wang * to store the headroom as the context ignoring the truesize. 1003192f68cfSJason Wang */ 1004946fa564SMichael S. Tsirkin static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 1005946fa564SMichael S. Tsirkin gfp_t gfp) 1006296f96fcSRusty Russell { 1007f6b10209SJason Wang struct page_frag *alloc_frag = &rq->alloc_frag; 1008f6b10209SJason Wang char *buf; 10092de2f7f4SJohn Fastabend unsigned int xdp_headroom = virtnet_get_headroom(vi); 1010192f68cfSJason Wang void *ctx = (void *)(unsigned long)xdp_headroom; 1011f6b10209SJason Wang int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 10129ab86bbcSShirley Ma int err; 10133f2c31d9SMark McLoughlin 1014f6b10209SJason Wang len = SKB_DATA_ALIGN(len) + 1015f6b10209SJason Wang SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1016f6b10209SJason Wang if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp))) 10179ab86bbcSShirley Ma return -ENOMEM; 1018296f96fcSRusty Russell 1019f6b10209SJason Wang buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1020f6b10209SJason Wang get_page(alloc_frag->page); 1021f6b10209SJason Wang alloc_frag->offset += len; 1022f6b10209SJason Wang sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom, 1023f6b10209SJason Wang vi->hdr_len + GOOD_PACKET_LEN); 1024192f68cfSJason Wang err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 10259ab86bbcSShirley Ma if (err < 0) 1026f6b10209SJason Wang put_page(virt_to_head_page(buf)); 10279ab86bbcSShirley Ma return err; 102897402b96SHerbert Xu } 102997402b96SHerbert Xu 1030012873d0SMichael S. Tsirkin static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 1031012873d0SMichael S. Tsirkin gfp_t gfp) 10329ab86bbcSShirley Ma { 10339ab86bbcSShirley Ma struct page *first, *list = NULL; 10349ab86bbcSShirley Ma char *p; 10359ab86bbcSShirley Ma int i, err, offset; 1036296f96fcSRusty Russell 1037a5835440SRusty Russell sg_init_table(rq->sg, MAX_SKB_FRAGS + 2); 1038a5835440SRusty Russell 1039e9d7417bSJason Wang /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */ 10409ab86bbcSShirley Ma for (i = MAX_SKB_FRAGS + 1; i > 1; --i) { 1041e9d7417bSJason Wang first = get_a_page(rq, gfp); 10429ab86bbcSShirley Ma if (!first) { 10439ab86bbcSShirley Ma if (list) 1044e9d7417bSJason Wang give_pages(rq, list); 10459ab86bbcSShirley Ma return -ENOMEM; 1046296f96fcSRusty Russell } 1047e9d7417bSJason Wang sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 10489ab86bbcSShirley Ma 10499ab86bbcSShirley Ma /* chain new page in list head to match sg */ 10509ab86bbcSShirley Ma first->private = (unsigned long)list; 10519ab86bbcSShirley Ma list = first; 10529ab86bbcSShirley Ma } 10539ab86bbcSShirley Ma 1054e9d7417bSJason Wang first = get_a_page(rq, gfp); 10559ab86bbcSShirley Ma if (!first) { 1056e9d7417bSJason Wang give_pages(rq, list); 10579ab86bbcSShirley Ma return -ENOMEM; 10589ab86bbcSShirley Ma } 10599ab86bbcSShirley Ma p = page_address(first); 10609ab86bbcSShirley Ma 1061e9d7417bSJason Wang /* rq->sg[0], rq->sg[1] share the same page */ 1062012873d0SMichael S. Tsirkin /* a separated rq->sg[0] for header - required in case !any_header_sg */ 1063012873d0SMichael S. Tsirkin sg_set_buf(&rq->sg[0], p, vi->hdr_len); 10649ab86bbcSShirley Ma 1065e9d7417bSJason Wang /* rq->sg[1] for data packet, from offset */ 10669ab86bbcSShirley Ma offset = sizeof(struct padded_vnet_hdr); 1067e9d7417bSJason Wang sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 10689ab86bbcSShirley Ma 10699ab86bbcSShirley Ma /* chain first in list head */ 10709ab86bbcSShirley Ma first->private = (unsigned long)list; 10719dc7b9e4SRusty Russell err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2, 1072aa989f5eSMichael S. Tsirkin first, gfp); 10739ab86bbcSShirley Ma if (err < 0) 1074e9d7417bSJason Wang give_pages(rq, first); 10759ab86bbcSShirley Ma 10769ab86bbcSShirley Ma return err; 10779ab86bbcSShirley Ma } 10789ab86bbcSShirley Ma 1079d85b758fSMichael S. Tsirkin static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 10803cc81a9aSJason Wang struct ewma_pkt_len *avg_pkt_len, 10813cc81a9aSJason Wang unsigned int room) 10829ab86bbcSShirley Ma { 1083ab7db917SMichael Dalton const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1084fbf28d78SMichael Dalton unsigned int len; 1085fbf28d78SMichael Dalton 10863cc81a9aSJason Wang if (room) 10873cc81a9aSJason Wang return PAGE_SIZE - room; 10883cc81a9aSJason Wang 10895377d758SJohannes Berg len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 1090f0c3192cSMichael S. Tsirkin rq->min_buf_len, PAGE_SIZE - hdr_len); 10913cc81a9aSJason Wang 1092e377fcc8SMichael S. Tsirkin return ALIGN(len, L1_CACHE_BYTES); 1093fbf28d78SMichael Dalton } 1094fbf28d78SMichael Dalton 10952de2f7f4SJohn Fastabend static int add_recvbuf_mergeable(struct virtnet_info *vi, 10962de2f7f4SJohn Fastabend struct receive_queue *rq, gfp_t gfp) 1097fbf28d78SMichael Dalton { 1098fb51879dSMichael Dalton struct page_frag *alloc_frag = &rq->alloc_frag; 10992de2f7f4SJohn Fastabend unsigned int headroom = virtnet_get_headroom(vi); 11003cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 11013cc81a9aSJason Wang unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 1102fb51879dSMichael Dalton char *buf; 1103680557cfSMichael S. Tsirkin void *ctx; 11049ab86bbcSShirley Ma int err; 1105fb51879dSMichael Dalton unsigned int len, hole; 11069ab86bbcSShirley Ma 11073cc81a9aSJason Wang /* Extra tailroom is needed to satisfy XDP's assumption. This 11083cc81a9aSJason Wang * means rx frags coalescing won't work, but consider we've 11093cc81a9aSJason Wang * disabled GSO for XDP, it won't be a big issue. 11103cc81a9aSJason Wang */ 11113cc81a9aSJason Wang len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 11123cc81a9aSJason Wang if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp))) 11139ab86bbcSShirley Ma return -ENOMEM; 1114ab7db917SMichael Dalton 1115fb51879dSMichael Dalton buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 11162de2f7f4SJohn Fastabend buf += headroom; /* advance address leaving hole at front of pkt */ 1117fb51879dSMichael Dalton get_page(alloc_frag->page); 11183cc81a9aSJason Wang alloc_frag->offset += len + room; 1119fb51879dSMichael Dalton hole = alloc_frag->size - alloc_frag->offset; 11203cc81a9aSJason Wang if (hole < len + room) { 1121ab7db917SMichael Dalton /* To avoid internal fragmentation, if there is very likely not 1122ab7db917SMichael Dalton * enough space for another buffer, add the remaining space to 11231daa8790SMichael S. Tsirkin * the current buffer. 1124ab7db917SMichael Dalton */ 1125fb51879dSMichael Dalton len += hole; 1126fb51879dSMichael Dalton alloc_frag->offset += hole; 1127fb51879dSMichael Dalton } 11289ab86bbcSShirley Ma 1129fb51879dSMichael Dalton sg_init_one(rq->sg, buf, len); 113029fda25aSDavid S. Miller ctx = mergeable_len_to_ctx(len, headroom); 1131680557cfSMichael S. Tsirkin err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); 11329ab86bbcSShirley Ma if (err < 0) 11332613af0eSMichael Dalton put_page(virt_to_head_page(buf)); 11349ab86bbcSShirley Ma 11359ab86bbcSShirley Ma return err; 1136296f96fcSRusty Russell } 1137296f96fcSRusty Russell 1138b2baed69SRusty Russell /* 1139b2baed69SRusty Russell * Returns false if we couldn't fill entirely (OOM). 1140b2baed69SRusty Russell * 1141b2baed69SRusty Russell * Normally run in the receive path, but can also be run from ndo_open 1142b2baed69SRusty Russell * before we're receiving packets, or from refill_work which is 1143b2baed69SRusty Russell * careful to disable receiving (using napi_disable). 1144b2baed69SRusty Russell */ 1145946fa564SMichael S. Tsirkin static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 1146946fa564SMichael S. Tsirkin gfp_t gfp) 11473f2c31d9SMark McLoughlin { 11483f2c31d9SMark McLoughlin int err; 11491788f495SMichael S. Tsirkin bool oom; 11503f2c31d9SMark McLoughlin 11510aea51c3SAmit Shah do { 11529ab86bbcSShirley Ma if (vi->mergeable_rx_bufs) 11532de2f7f4SJohn Fastabend err = add_recvbuf_mergeable(vi, rq, gfp); 11549ab86bbcSShirley Ma else if (vi->big_packets) 1155012873d0SMichael S. Tsirkin err = add_recvbuf_big(vi, rq, gfp); 11569ab86bbcSShirley Ma else 1157946fa564SMichael S. Tsirkin err = add_recvbuf_small(vi, rq, gfp); 11583f2c31d9SMark McLoughlin 11591788f495SMichael S. Tsirkin oom = err == -ENOMEM; 11609ed4cb07SRusty Russell if (err) 11613f2c31d9SMark McLoughlin break; 1162b7dfde95SLinus Torvalds } while (rq->vq->num_free); 1163681daee2SJason Wang virtqueue_kick(rq->vq); 11643161e453SRusty Russell return !oom; 11653f2c31d9SMark McLoughlin } 11663f2c31d9SMark McLoughlin 116718445c4dSRusty Russell static void skb_recv_done(struct virtqueue *rvq) 1168296f96fcSRusty Russell { 1169296f96fcSRusty Russell struct virtnet_info *vi = rvq->vdev->priv; 1170986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 1171e9d7417bSJason Wang 1172e4e8452aSWillem de Bruijn virtqueue_napi_schedule(&rq->napi, rvq); 1173296f96fcSRusty Russell } 1174296f96fcSRusty Russell 1175e4e8452aSWillem de Bruijn static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi) 11763e9d08ecSBruce Rogers { 1177e4e8452aSWillem de Bruijn napi_enable(napi); 11783e9d08ecSBruce Rogers 11793e9d08ecSBruce Rogers /* If all buffers were filled by other side before we napi_enabled, we 1180e4e8452aSWillem de Bruijn * won't get another interrupt, so process any outstanding packets now. 1181e4e8452aSWillem de Bruijn * Call local_bh_enable after to trigger softIRQ processing. 1182e4e8452aSWillem de Bruijn */ 1183ec13ee80SMichael S. Tsirkin local_bh_disable(); 1184e4e8452aSWillem de Bruijn virtqueue_napi_schedule(napi, vq); 1185ec13ee80SMichael S. Tsirkin local_bh_enable(); 11863e9d08ecSBruce Rogers } 11873e9d08ecSBruce Rogers 1188b92f1e67SWillem de Bruijn static void virtnet_napi_tx_enable(struct virtnet_info *vi, 1189b92f1e67SWillem de Bruijn struct virtqueue *vq, 1190b92f1e67SWillem de Bruijn struct napi_struct *napi) 1191b92f1e67SWillem de Bruijn { 1192b92f1e67SWillem de Bruijn if (!napi->weight) 1193b92f1e67SWillem de Bruijn return; 1194b92f1e67SWillem de Bruijn 1195b92f1e67SWillem de Bruijn /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 1196b92f1e67SWillem de Bruijn * enable the feature if this is likely affine with the transmit path. 1197b92f1e67SWillem de Bruijn */ 1198b92f1e67SWillem de Bruijn if (!vi->affinity_hint_set) { 1199b92f1e67SWillem de Bruijn napi->weight = 0; 1200b92f1e67SWillem de Bruijn return; 1201b92f1e67SWillem de Bruijn } 1202b92f1e67SWillem de Bruijn 1203b92f1e67SWillem de Bruijn return virtnet_napi_enable(vq, napi); 1204b92f1e67SWillem de Bruijn } 1205b92f1e67SWillem de Bruijn 120678a57b48SWillem de Bruijn static void virtnet_napi_tx_disable(struct napi_struct *napi) 120778a57b48SWillem de Bruijn { 120878a57b48SWillem de Bruijn if (napi->weight) 120978a57b48SWillem de Bruijn napi_disable(napi); 121078a57b48SWillem de Bruijn } 121178a57b48SWillem de Bruijn 12123161e453SRusty Russell static void refill_work(struct work_struct *work) 12133161e453SRusty Russell { 1214e9d7417bSJason Wang struct virtnet_info *vi = 1215e9d7417bSJason Wang container_of(work, struct virtnet_info, refill.work); 12163161e453SRusty Russell bool still_empty; 1217986a4f4dSJason Wang int i; 12183161e453SRusty Russell 121955257d72SSasha Levin for (i = 0; i < vi->curr_queue_pairs; i++) { 1220986a4f4dSJason Wang struct receive_queue *rq = &vi->rq[i]; 1221986a4f4dSJason Wang 1222986a4f4dSJason Wang napi_disable(&rq->napi); 1223946fa564SMichael S. Tsirkin still_empty = !try_fill_recv(vi, rq, GFP_KERNEL); 1224e4e8452aSWillem de Bruijn virtnet_napi_enable(rq->vq, &rq->napi); 12253161e453SRusty Russell 12263161e453SRusty Russell /* In theory, this can happen: if we don't get any buffers in 1227986a4f4dSJason Wang * we will *never* try to fill again. 1228986a4f4dSJason Wang */ 12293161e453SRusty Russell if (still_empty) 12303b07e9caSTejun Heo schedule_delayed_work(&vi->refill, HZ/2); 12313161e453SRusty Russell } 1232986a4f4dSJason Wang } 12333161e453SRusty Russell 1234186b3c99SJason Wang static int virtnet_receive(struct receive_queue *rq, int budget, bool *xdp_xmit) 1235296f96fcSRusty Russell { 1236e9d7417bSJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 123761845d20SJason Wang unsigned int len, received = 0, bytes = 0; 12389ab86bbcSShirley Ma void *buf; 1239296f96fcSRusty Russell 1240192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 1241680557cfSMichael S. Tsirkin void *ctx; 1242680557cfSMichael S. Tsirkin 1243680557cfSMichael S. Tsirkin while (received < budget && 1244680557cfSMichael S. Tsirkin (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) { 1245186b3c99SJason Wang bytes += receive_buf(vi, rq, buf, len, ctx, xdp_xmit); 1246680557cfSMichael S. Tsirkin received++; 1247680557cfSMichael S. Tsirkin } 1248680557cfSMichael S. Tsirkin } else { 1249296f96fcSRusty Russell while (received < budget && 1250e9d7417bSJason Wang (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 1251186b3c99SJason Wang bytes += receive_buf(vi, rq, buf, len, NULL, xdp_xmit); 1252296f96fcSRusty Russell received++; 1253296f96fcSRusty Russell } 1254680557cfSMichael S. Tsirkin } 1255296f96fcSRusty Russell 1256be121f46SJason Wang if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) { 1257946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, rq, GFP_ATOMIC)) 12583b07e9caSTejun Heo schedule_delayed_work(&vi->refill, 0); 12593161e453SRusty Russell } 1260296f96fcSRusty Russell 1261d7dfc5cfSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 1262d7dfc5cfSToshiaki Makita rq->stats.bytes += bytes; 1263d7dfc5cfSToshiaki Makita rq->stats.packets += received; 1264d7dfc5cfSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 126561845d20SJason Wang 12662ffa7598SJason Wang return received; 12672ffa7598SJason Wang } 12682ffa7598SJason Wang 1269ea7735d9SWillem de Bruijn static void free_old_xmit_skbs(struct send_queue *sq) 1270ea7735d9SWillem de Bruijn { 1271ea7735d9SWillem de Bruijn struct sk_buff *skb; 1272ea7735d9SWillem de Bruijn unsigned int len; 1273ea7735d9SWillem de Bruijn unsigned int packets = 0; 1274ea7735d9SWillem de Bruijn unsigned int bytes = 0; 1275ea7735d9SWillem de Bruijn 1276ea7735d9SWillem de Bruijn while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) { 1277ea7735d9SWillem de Bruijn pr_debug("Sent skb %p\n", skb); 1278ea7735d9SWillem de Bruijn 1279ea7735d9SWillem de Bruijn bytes += skb->len; 1280ea7735d9SWillem de Bruijn packets++; 1281ea7735d9SWillem de Bruijn 1282dadc0736SEric Dumazet dev_consume_skb_any(skb); 1283ea7735d9SWillem de Bruijn } 1284ea7735d9SWillem de Bruijn 1285ea7735d9SWillem de Bruijn /* Avoid overhead when no packets have been processed 1286ea7735d9SWillem de Bruijn * happens when called speculatively from start_xmit. 1287ea7735d9SWillem de Bruijn */ 1288ea7735d9SWillem de Bruijn if (!packets) 1289ea7735d9SWillem de Bruijn return; 1290ea7735d9SWillem de Bruijn 1291d7dfc5cfSToshiaki Makita u64_stats_update_begin(&sq->stats.syncp); 1292d7dfc5cfSToshiaki Makita sq->stats.bytes += bytes; 1293d7dfc5cfSToshiaki Makita sq->stats.packets += packets; 1294d7dfc5cfSToshiaki Makita u64_stats_update_end(&sq->stats.syncp); 1295ea7735d9SWillem de Bruijn } 1296ea7735d9SWillem de Bruijn 12977b0411efSWillem de Bruijn static void virtnet_poll_cleantx(struct receive_queue *rq) 12987b0411efSWillem de Bruijn { 12997b0411efSWillem de Bruijn struct virtnet_info *vi = rq->vq->vdev->priv; 13007b0411efSWillem de Bruijn unsigned int index = vq2rxq(rq->vq); 13017b0411efSWillem de Bruijn struct send_queue *sq = &vi->sq[index]; 13027b0411efSWillem de Bruijn struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 13037b0411efSWillem de Bruijn 13047b0411efSWillem de Bruijn if (!sq->napi.weight) 13057b0411efSWillem de Bruijn return; 13067b0411efSWillem de Bruijn 13077b0411efSWillem de Bruijn if (__netif_tx_trylock(txq)) { 13087b0411efSWillem de Bruijn free_old_xmit_skbs(sq); 13097b0411efSWillem de Bruijn __netif_tx_unlock(txq); 13107b0411efSWillem de Bruijn } 13117b0411efSWillem de Bruijn 13127b0411efSWillem de Bruijn if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 13137b0411efSWillem de Bruijn netif_tx_wake_queue(txq); 13147b0411efSWillem de Bruijn } 13157b0411efSWillem de Bruijn 13162ffa7598SJason Wang static int virtnet_poll(struct napi_struct *napi, int budget) 13172ffa7598SJason Wang { 13182ffa7598SJason Wang struct receive_queue *rq = 13192ffa7598SJason Wang container_of(napi, struct receive_queue, napi); 13209267c430SJason Wang struct virtnet_info *vi = rq->vq->vdev->priv; 13219267c430SJason Wang struct send_queue *sq; 13229267c430SJason Wang unsigned int received, qp; 1323186b3c99SJason Wang bool xdp_xmit = false; 13242ffa7598SJason Wang 13257b0411efSWillem de Bruijn virtnet_poll_cleantx(rq); 13267b0411efSWillem de Bruijn 1327186b3c99SJason Wang received = virtnet_receive(rq, budget, &xdp_xmit); 13282ffa7598SJason Wang 13298329d98eSRusty Russell /* Out of packets? */ 1330e4e8452aSWillem de Bruijn if (received < budget) 1331e4e8452aSWillem de Bruijn virtqueue_napi_complete(napi, rq->vq, received); 1332296f96fcSRusty Russell 13339267c430SJason Wang if (xdp_xmit) { 13349267c430SJason Wang qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + 13359267c430SJason Wang smp_processor_id(); 13369267c430SJason Wang sq = &vi->sq[qp]; 13379267c430SJason Wang virtqueue_kick(sq->vq); 1338186b3c99SJason Wang xdp_do_flush_map(); 13399267c430SJason Wang } 1340186b3c99SJason Wang 1341296f96fcSRusty Russell return received; 1342296f96fcSRusty Russell } 1343296f96fcSRusty Russell 1344986a4f4dSJason Wang static int virtnet_open(struct net_device *dev) 1345986a4f4dSJason Wang { 1346986a4f4dSJason Wang struct virtnet_info *vi = netdev_priv(dev); 1347754b8a21SJesper Dangaard Brouer int i, err; 1348986a4f4dSJason Wang 1349e4166625SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 1350e4166625SJason Wang if (i < vi->curr_queue_pairs) 1351986a4f4dSJason Wang /* Make sure we have some buffers: if oom use wq. */ 1352946fa564SMichael S. Tsirkin if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 1353986a4f4dSJason Wang schedule_delayed_work(&vi->refill, 0); 1354754b8a21SJesper Dangaard Brouer 1355754b8a21SJesper Dangaard Brouer err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i); 1356754b8a21SJesper Dangaard Brouer if (err < 0) 1357754b8a21SJesper Dangaard Brouer return err; 1358754b8a21SJesper Dangaard Brouer 13598d5d8852SJesper Dangaard Brouer err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq, 13608d5d8852SJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED, NULL); 13618d5d8852SJesper Dangaard Brouer if (err < 0) { 13628d5d8852SJesper Dangaard Brouer xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 13638d5d8852SJesper Dangaard Brouer return err; 13648d5d8852SJesper Dangaard Brouer } 13658d5d8852SJesper Dangaard Brouer 1366e4e8452aSWillem de Bruijn virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 1367b92f1e67SWillem de Bruijn virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi); 1368986a4f4dSJason Wang } 1369986a4f4dSJason Wang 1370986a4f4dSJason Wang return 0; 1371986a4f4dSJason Wang } 1372986a4f4dSJason Wang 1373b92f1e67SWillem de Bruijn static int virtnet_poll_tx(struct napi_struct *napi, int budget) 1374b92f1e67SWillem de Bruijn { 1375b92f1e67SWillem de Bruijn struct send_queue *sq = container_of(napi, struct send_queue, napi); 1376b92f1e67SWillem de Bruijn struct virtnet_info *vi = sq->vq->vdev->priv; 1377b92f1e67SWillem de Bruijn struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, vq2txq(sq->vq)); 1378b92f1e67SWillem de Bruijn 1379b92f1e67SWillem de Bruijn __netif_tx_lock(txq, raw_smp_processor_id()); 1380b92f1e67SWillem de Bruijn free_old_xmit_skbs(sq); 1381b92f1e67SWillem de Bruijn __netif_tx_unlock(txq); 1382b92f1e67SWillem de Bruijn 1383b92f1e67SWillem de Bruijn virtqueue_napi_complete(napi, sq->vq, 0); 1384b92f1e67SWillem de Bruijn 1385b92f1e67SWillem de Bruijn if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) 1386b92f1e67SWillem de Bruijn netif_tx_wake_queue(txq); 1387b92f1e67SWillem de Bruijn 1388b92f1e67SWillem de Bruijn return 0; 1389b92f1e67SWillem de Bruijn } 1390b92f1e67SWillem de Bruijn 1391e9d7417bSJason Wang static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) 1392296f96fcSRusty Russell { 1393012873d0SMichael S. Tsirkin struct virtio_net_hdr_mrg_rxbuf *hdr; 1394296f96fcSRusty Russell const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 1395e9d7417bSJason Wang struct virtnet_info *vi = sq->vq->vdev->priv; 1396e2fcad58SJason A. Donenfeld int num_sg; 1397012873d0SMichael S. Tsirkin unsigned hdr_len = vi->hdr_len; 1398e7428e95SMichael S. Tsirkin bool can_push; 1399296f96fcSRusty Russell 1400e174961cSJohannes Berg pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 1401e7428e95SMichael S. Tsirkin 1402e7428e95SMichael S. Tsirkin can_push = vi->any_header_sg && 1403e7428e95SMichael S. Tsirkin !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 1404e7428e95SMichael S. Tsirkin !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 1405e7428e95SMichael S. Tsirkin /* Even if we can, don't push here yet as this would skew 1406e7428e95SMichael S. Tsirkin * csum_start offset below. */ 1407e7428e95SMichael S. Tsirkin if (can_push) 1408012873d0SMichael S. Tsirkin hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len); 1409e7428e95SMichael S. Tsirkin else 1410e7428e95SMichael S. Tsirkin hdr = skb_vnet_hdr(skb); 1411296f96fcSRusty Russell 1412e858fae2SMike Rapoport if (virtio_net_hdr_from_skb(skb, &hdr->hdr, 14136391a448SJason Wang virtio_is_little_endian(vi->vdev), false)) 1414296f96fcSRusty Russell BUG(); 1415296f96fcSRusty Russell 1416e7428e95SMichael S. Tsirkin if (vi->mergeable_rx_bufs) 1417012873d0SMichael S. Tsirkin hdr->num_buffers = 0; 14183f2c31d9SMark McLoughlin 1419547c890cSJason Wang sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 1420e7428e95SMichael S. Tsirkin if (can_push) { 1421e7428e95SMichael S. Tsirkin __skb_push(skb, hdr_len); 1422e7428e95SMichael S. Tsirkin num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 1423e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 1424e2fcad58SJason A. Donenfeld return num_sg; 1425e7428e95SMichael S. Tsirkin /* Pull header back to avoid skew in tx bytes calculations. */ 1426e7428e95SMichael S. Tsirkin __skb_pull(skb, hdr_len); 1427e7428e95SMichael S. Tsirkin } else { 1428e7428e95SMichael S. Tsirkin sg_set_buf(sq->sg, hdr, hdr_len); 1429e2fcad58SJason A. Donenfeld num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 1430e2fcad58SJason A. Donenfeld if (unlikely(num_sg < 0)) 1431e2fcad58SJason A. Donenfeld return num_sg; 1432e2fcad58SJason A. Donenfeld num_sg++; 1433e7428e95SMichael S. Tsirkin } 14349dc7b9e4SRusty Russell return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC); 143511a3a154SRusty Russell } 143611a3a154SRusty Russell 1437424efe9cSStephen Hemminger static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 143899ffc696SRusty Russell { 143999ffc696SRusty Russell struct virtnet_info *vi = netdev_priv(dev); 1440986a4f4dSJason Wang int qnum = skb_get_queue_mapping(skb); 1441986a4f4dSJason Wang struct send_queue *sq = &vi->sq[qnum]; 14429ed4cb07SRusty Russell int err; 14434b7fd2e6SMichael S. Tsirkin struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 14444b7fd2e6SMichael S. Tsirkin bool kick = !skb->xmit_more; 1445b92f1e67SWillem de Bruijn bool use_napi = sq->napi.weight; 14462cb9c6baSRusty Russell 14472cb9c6baSRusty Russell /* Free up any pending old buffers before queueing new ones. */ 1448e9d7417bSJason Wang free_old_xmit_skbs(sq); 144999ffc696SRusty Russell 1450bdb12e0dSWillem de Bruijn if (use_napi && kick) 1451bdb12e0dSWillem de Bruijn virtqueue_enable_cb_delayed(sq->vq); 1452bdb12e0dSWillem de Bruijn 1453074c3582SJacob Keller /* timestamp packet in software */ 1454074c3582SJacob Keller skb_tx_timestamp(skb); 1455074c3582SJacob Keller 145603f191baSMichael S. Tsirkin /* Try to transmit */ 1457b7dfde95SLinus Torvalds err = xmit_skb(sq, skb); 145899ffc696SRusty Russell 14599ed4cb07SRusty Russell /* This should not happen! */ 1460681daee2SJason Wang if (unlikely(err)) { 146158eba97dSRusty Russell dev->stats.tx_fifo_errors++; 14622e57b79cSRick Jones if (net_ratelimit()) 146358eba97dSRusty Russell dev_warn(&dev->dev, 1464b7dfde95SLinus Torvalds "Unexpected TXQ (%d) queue failure: %d\n", qnum, err); 146558eba97dSRusty Russell dev->stats.tx_dropped++; 146685e94525SEric W. Biederman dev_kfree_skb_any(skb); 146758eba97dSRusty Russell return NETDEV_TX_OK; 1468296f96fcSRusty Russell } 146903f191baSMichael S. Tsirkin 147048925e37SRusty Russell /* Don't wait up for transmitted skbs to be freed. */ 1471b92f1e67SWillem de Bruijn if (!use_napi) { 147248925e37SRusty Russell skb_orphan(skb); 147348925e37SRusty Russell nf_reset(skb); 1474b92f1e67SWillem de Bruijn } 147548925e37SRusty Russell 147660302ff6SMichael S. Tsirkin /* If running out of space, stop queue to avoid getting packets that we 147760302ff6SMichael S. Tsirkin * are then unable to transmit. 147860302ff6SMichael S. Tsirkin * An alternative would be to force queuing layer to requeue the skb by 147960302ff6SMichael S. Tsirkin * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 148060302ff6SMichael S. Tsirkin * returned in a normal path of operation: it means that driver is not 148160302ff6SMichael S. Tsirkin * maintaining the TX queue stop/start state properly, and causes 148260302ff6SMichael S. Tsirkin * the stack to do a non-trivial amount of useless work. 148360302ff6SMichael S. Tsirkin * Since most packets only take 1 or 2 ring slots, stopping the queue 148460302ff6SMichael S. Tsirkin * early means 16 slots are typically wasted. 1485d631b94eSstephen hemminger */ 1486b7dfde95SLinus Torvalds if (sq->vq->num_free < 2+MAX_SKB_FRAGS) { 1487986a4f4dSJason Wang netif_stop_subqueue(dev, qnum); 1488b92f1e67SWillem de Bruijn if (!use_napi && 1489b92f1e67SWillem de Bruijn unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 149048925e37SRusty Russell /* More just got used, free them then recheck. */ 1491b7dfde95SLinus Torvalds free_old_xmit_skbs(sq); 1492b7dfde95SLinus Torvalds if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { 1493986a4f4dSJason Wang netif_start_subqueue(dev, qnum); 1494e9d7417bSJason Wang virtqueue_disable_cb(sq->vq); 149548925e37SRusty Russell } 149648925e37SRusty Russell } 149748925e37SRusty Russell } 149848925e37SRusty Russell 14994b7fd2e6SMichael S. Tsirkin if (kick || netif_xmit_stopped(txq)) 1500c223a078SDavid S. Miller virtqueue_kick(sq->vq); 15010b725a2cSDavid S. Miller 15020b725a2cSDavid S. Miller return NETDEV_TX_OK; 1503c223a078SDavid S. Miller } 1504c223a078SDavid S. Miller 150540cbfc37SAmos Kong /* 150640cbfc37SAmos Kong * Send command via the control virtqueue and check status. Commands 150740cbfc37SAmos Kong * supported by the hypervisor, as indicated by feature bits, should 1508788a8b6dSstephen hemminger * never fail unless improperly formatted. 150940cbfc37SAmos Kong */ 151040cbfc37SAmos Kong static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 1511d24bae32Sstephen hemminger struct scatterlist *out) 151240cbfc37SAmos Kong { 1513f7bc9594SRusty Russell struct scatterlist *sgs[4], hdr, stat; 1514d24bae32Sstephen hemminger unsigned out_num = 0, tmp; 151540cbfc37SAmos Kong 151640cbfc37SAmos Kong /* Caller should know better */ 1517f7bc9594SRusty Russell BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 151840cbfc37SAmos Kong 151912e57169SMichael S. Tsirkin vi->ctrl->status = ~0; 152012e57169SMichael S. Tsirkin vi->ctrl->hdr.class = class; 152112e57169SMichael S. Tsirkin vi->ctrl->hdr.cmd = cmd; 1522f7bc9594SRusty Russell /* Add header */ 152312e57169SMichael S. Tsirkin sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 1524f7bc9594SRusty Russell sgs[out_num++] = &hdr; 152540cbfc37SAmos Kong 1526f7bc9594SRusty Russell if (out) 1527f7bc9594SRusty Russell sgs[out_num++] = out; 152840cbfc37SAmos Kong 1529f7bc9594SRusty Russell /* Add return status. */ 153012e57169SMichael S. Tsirkin sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 1531d24bae32Sstephen hemminger sgs[out_num] = &stat; 153240cbfc37SAmos Kong 1533d24bae32Sstephen hemminger BUG_ON(out_num + 1 > ARRAY_SIZE(sgs)); 1534a7c58146SRusty Russell virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC); 153540cbfc37SAmos Kong 153667975901SHeinz Graalfs if (unlikely(!virtqueue_kick(vi->cvq))) 153712e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 153840cbfc37SAmos Kong 153940cbfc37SAmos Kong /* Spin for a response, the kick causes an ioport write, trapping 154040cbfc37SAmos Kong * into the hypervisor, so the request should be handled immediately. 154140cbfc37SAmos Kong */ 1542047b9b94SHeinz Graalfs while (!virtqueue_get_buf(vi->cvq, &tmp) && 1543047b9b94SHeinz Graalfs !virtqueue_is_broken(vi->cvq)) 154440cbfc37SAmos Kong cpu_relax(); 154540cbfc37SAmos Kong 154612e57169SMichael S. Tsirkin return vi->ctrl->status == VIRTIO_NET_OK; 154740cbfc37SAmos Kong } 154840cbfc37SAmos Kong 15499c46f6d4SAlex Williamson static int virtnet_set_mac_address(struct net_device *dev, void *p) 15509c46f6d4SAlex Williamson { 15519c46f6d4SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 15529c46f6d4SAlex Williamson struct virtio_device *vdev = vi->vdev; 1553f2f2c8b4SJiri Pirko int ret; 1554e37e2ff3SAndy Lutomirski struct sockaddr *addr; 15557e58d5aeSAmos Kong struct scatterlist sg; 15569c46f6d4SAlex Williamson 1557801822d1SShyam Saini addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 1558e37e2ff3SAndy Lutomirski if (!addr) 1559e37e2ff3SAndy Lutomirski return -ENOMEM; 1560e37e2ff3SAndy Lutomirski 1561e37e2ff3SAndy Lutomirski ret = eth_prepare_mac_addr_change(dev, addr); 1562f2f2c8b4SJiri Pirko if (ret) 1563e37e2ff3SAndy Lutomirski goto out; 15649c46f6d4SAlex Williamson 15657e58d5aeSAmos Kong if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 15667e58d5aeSAmos Kong sg_init_one(&sg, addr->sa_data, dev->addr_len); 15677e58d5aeSAmos Kong if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 1568d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 15697e58d5aeSAmos Kong dev_warn(&vdev->dev, 15707e58d5aeSAmos Kong "Failed to set mac address by vq command.\n"); 1571e37e2ff3SAndy Lutomirski ret = -EINVAL; 1572e37e2ff3SAndy Lutomirski goto out; 15737e58d5aeSAmos Kong } 15747e93a02fSMichael S. Tsirkin } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 15757e93a02fSMichael S. Tsirkin !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1576855e0c52SRusty Russell unsigned int i; 1577855e0c52SRusty Russell 1578855e0c52SRusty Russell /* Naturally, this has an atomicity problem. */ 1579855e0c52SRusty Russell for (i = 0; i < dev->addr_len; i++) 1580855e0c52SRusty Russell virtio_cwrite8(vdev, 1581855e0c52SRusty Russell offsetof(struct virtio_net_config, mac) + 1582855e0c52SRusty Russell i, addr->sa_data[i]); 15837e58d5aeSAmos Kong } 15847e58d5aeSAmos Kong 15857e58d5aeSAmos Kong eth_commit_mac_addr_change(dev, p); 1586e37e2ff3SAndy Lutomirski ret = 0; 15879c46f6d4SAlex Williamson 1588e37e2ff3SAndy Lutomirski out: 1589e37e2ff3SAndy Lutomirski kfree(addr); 1590e37e2ff3SAndy Lutomirski return ret; 15919c46f6d4SAlex Williamson } 15929c46f6d4SAlex Williamson 1593bc1f4470Sstephen hemminger static void virtnet_stats(struct net_device *dev, 15943fa2a1dfSstephen hemminger struct rtnl_link_stats64 *tot) 15953fa2a1dfSstephen hemminger { 15963fa2a1dfSstephen hemminger struct virtnet_info *vi = netdev_priv(dev); 15973fa2a1dfSstephen hemminger unsigned int start; 1598d7dfc5cfSToshiaki Makita int i; 15993fa2a1dfSstephen hemminger 1600d7dfc5cfSToshiaki Makita for (i = 0; i < vi->max_queue_pairs; i++) { 16013fa2a1dfSstephen hemminger u64 tpackets, tbytes, rpackets, rbytes; 1602d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 1603d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 16043fa2a1dfSstephen hemminger 16053fa2a1dfSstephen hemminger do { 1606d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&sq->stats.syncp); 1607d7dfc5cfSToshiaki Makita tpackets = sq->stats.packets; 1608d7dfc5cfSToshiaki Makita tbytes = sq->stats.bytes; 1609d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); 161083a27052SEric Dumazet 161183a27052SEric Dumazet do { 1612d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&rq->stats.syncp); 1613d7dfc5cfSToshiaki Makita rpackets = rq->stats.packets; 1614d7dfc5cfSToshiaki Makita rbytes = rq->stats.bytes; 1615d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); 16163fa2a1dfSstephen hemminger 16173fa2a1dfSstephen hemminger tot->rx_packets += rpackets; 16183fa2a1dfSstephen hemminger tot->tx_packets += tpackets; 16193fa2a1dfSstephen hemminger tot->rx_bytes += rbytes; 16203fa2a1dfSstephen hemminger tot->tx_bytes += tbytes; 16213fa2a1dfSstephen hemminger } 16223fa2a1dfSstephen hemminger 16233fa2a1dfSstephen hemminger tot->tx_dropped = dev->stats.tx_dropped; 1624021ac8d3SRick Jones tot->tx_fifo_errors = dev->stats.tx_fifo_errors; 16253fa2a1dfSstephen hemminger tot->rx_dropped = dev->stats.rx_dropped; 16263fa2a1dfSstephen hemminger tot->rx_length_errors = dev->stats.rx_length_errors; 16273fa2a1dfSstephen hemminger tot->rx_frame_errors = dev->stats.rx_frame_errors; 16283fa2a1dfSstephen hemminger } 16293fa2a1dfSstephen hemminger 1630da74e89dSAmit Shah #ifdef CONFIG_NET_POLL_CONTROLLER 1631da74e89dSAmit Shah static void virtnet_netpoll(struct net_device *dev) 1632da74e89dSAmit Shah { 1633da74e89dSAmit Shah struct virtnet_info *vi = netdev_priv(dev); 1634986a4f4dSJason Wang int i; 1635da74e89dSAmit Shah 1636986a4f4dSJason Wang for (i = 0; i < vi->curr_queue_pairs; i++) 1637986a4f4dSJason Wang napi_schedule(&vi->rq[i].napi); 1638da74e89dSAmit Shah } 1639da74e89dSAmit Shah #endif 1640da74e89dSAmit Shah 1641586d17c5SJason Wang static void virtnet_ack_link_announce(struct virtnet_info *vi) 1642586d17c5SJason Wang { 1643586d17c5SJason Wang rtnl_lock(); 1644586d17c5SJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 1645d24bae32Sstephen hemminger VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 1646586d17c5SJason Wang dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 1647586d17c5SJason Wang rtnl_unlock(); 1648586d17c5SJason Wang } 1649586d17c5SJason Wang 165047315329SJohn Fastabend static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 1651986a4f4dSJason Wang { 1652986a4f4dSJason Wang struct scatterlist sg; 1653986a4f4dSJason Wang struct net_device *dev = vi->dev; 1654986a4f4dSJason Wang 1655986a4f4dSJason Wang if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 1656986a4f4dSJason Wang return 0; 1657986a4f4dSJason Wang 165812e57169SMichael S. Tsirkin vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 165912e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq)); 1660986a4f4dSJason Wang 1661986a4f4dSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 1662d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 1663986a4f4dSJason Wang dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 1664986a4f4dSJason Wang queue_pairs); 1665986a4f4dSJason Wang return -EINVAL; 166655257d72SSasha Levin } else { 1667986a4f4dSJason Wang vi->curr_queue_pairs = queue_pairs; 166835ed159bSJason Wang /* virtnet_open() will refill when device is going to up. */ 166935ed159bSJason Wang if (dev->flags & IFF_UP) 16709b9cd802SJason Wang schedule_delayed_work(&vi->refill, 0); 167155257d72SSasha Levin } 1672986a4f4dSJason Wang 1673986a4f4dSJason Wang return 0; 1674986a4f4dSJason Wang } 1675986a4f4dSJason Wang 167647315329SJohn Fastabend static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 167747315329SJohn Fastabend { 167847315329SJohn Fastabend int err; 167947315329SJohn Fastabend 168047315329SJohn Fastabend rtnl_lock(); 168147315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 168247315329SJohn Fastabend rtnl_unlock(); 168347315329SJohn Fastabend return err; 168447315329SJohn Fastabend } 168547315329SJohn Fastabend 1686296f96fcSRusty Russell static int virtnet_close(struct net_device *dev) 1687296f96fcSRusty Russell { 1688296f96fcSRusty Russell struct virtnet_info *vi = netdev_priv(dev); 1689986a4f4dSJason Wang int i; 1690296f96fcSRusty Russell 1691b2baed69SRusty Russell /* Make sure refill_work doesn't re-enable napi! */ 1692b2baed69SRusty Russell cancel_delayed_work_sync(&vi->refill); 1693986a4f4dSJason Wang 1694b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 1695754b8a21SJesper Dangaard Brouer xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq); 1696986a4f4dSJason Wang napi_disable(&vi->rq[i].napi); 169778a57b48SWillem de Bruijn virtnet_napi_tx_disable(&vi->sq[i].napi); 1698b92f1e67SWillem de Bruijn } 1699296f96fcSRusty Russell 1700296f96fcSRusty Russell return 0; 1701296f96fcSRusty Russell } 1702296f96fcSRusty Russell 17032af7698eSAlex Williamson static void virtnet_set_rx_mode(struct net_device *dev) 17042af7698eSAlex Williamson { 17052af7698eSAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 1706f565a7c2SAlex Williamson struct scatterlist sg[2]; 1707f565a7c2SAlex Williamson struct virtio_net_ctrl_mac *mac_data; 1708ccffad25SJiri Pirko struct netdev_hw_addr *ha; 170932e7bfc4SJiri Pirko int uc_count; 17104cd24eafSJiri Pirko int mc_count; 1711f565a7c2SAlex Williamson void *buf; 1712f565a7c2SAlex Williamson int i; 17132af7698eSAlex Williamson 1714788a8b6dSstephen hemminger /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 17152af7698eSAlex Williamson if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 17162af7698eSAlex Williamson return; 17172af7698eSAlex Williamson 171812e57169SMichael S. Tsirkin vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0); 171912e57169SMichael S. Tsirkin vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0); 17202af7698eSAlex Williamson 172112e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc)); 17222af7698eSAlex Williamson 17232af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 1724d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_PROMISC, sg)) 17252af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 172612e57169SMichael S. Tsirkin vi->ctrl->promisc ? "en" : "dis"); 17272af7698eSAlex Williamson 172812e57169SMichael S. Tsirkin sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti)); 17292af7698eSAlex Williamson 17302af7698eSAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 1731d24bae32Sstephen hemminger VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 17322af7698eSAlex Williamson dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 173312e57169SMichael S. Tsirkin vi->ctrl->allmulti ? "en" : "dis"); 1734f565a7c2SAlex Williamson 173532e7bfc4SJiri Pirko uc_count = netdev_uc_count(dev); 17364cd24eafSJiri Pirko mc_count = netdev_mc_count(dev); 1737f565a7c2SAlex Williamson /* MAC filter - use one buffer for both lists */ 17384cd24eafSJiri Pirko buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 1739f565a7c2SAlex Williamson (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 17404cd24eafSJiri Pirko mac_data = buf; 1741e68ed8f0SJoe Perches if (!buf) 1742f565a7c2SAlex Williamson return; 1743f565a7c2SAlex Williamson 174423e258e1SAlex Williamson sg_init_table(sg, 2); 174523e258e1SAlex Williamson 1746f565a7c2SAlex Williamson /* Store the unicast list and count in the front of the buffer */ 1747fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 1748ccffad25SJiri Pirko i = 0; 174932e7bfc4SJiri Pirko netdev_for_each_uc_addr(ha, dev) 1750ccffad25SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 1751f565a7c2SAlex Williamson 1752f565a7c2SAlex Williamson sg_set_buf(&sg[0], mac_data, 175332e7bfc4SJiri Pirko sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 1754f565a7c2SAlex Williamson 1755f565a7c2SAlex Williamson /* multicast list and count fill the end */ 175632e7bfc4SJiri Pirko mac_data = (void *)&mac_data->macs[uc_count][0]; 1757f565a7c2SAlex Williamson 1758fdd819b2SMichael S. Tsirkin mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 1759567ec874SJiri Pirko i = 0; 176022bedad3SJiri Pirko netdev_for_each_mc_addr(ha, dev) 176122bedad3SJiri Pirko memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 1762f565a7c2SAlex Williamson 1763f565a7c2SAlex Williamson sg_set_buf(&sg[1], mac_data, 17644cd24eafSJiri Pirko sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 1765f565a7c2SAlex Williamson 1766f565a7c2SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 1767d24bae32Sstephen hemminger VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 176899e872aeSThomas Huth dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 1769f565a7c2SAlex Williamson 1770f565a7c2SAlex Williamson kfree(buf); 17712af7698eSAlex Williamson } 17722af7698eSAlex Williamson 177380d5c368SPatrick McHardy static int virtnet_vlan_rx_add_vid(struct net_device *dev, 177480d5c368SPatrick McHardy __be16 proto, u16 vid) 17750bde9569SAlex Williamson { 17760bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 17770bde9569SAlex Williamson struct scatterlist sg; 17780bde9569SAlex Williamson 1779d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 178012e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 17810bde9569SAlex Williamson 17820bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 1783d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 17840bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 17858e586137SJiri Pirko return 0; 17860bde9569SAlex Williamson } 17870bde9569SAlex Williamson 178880d5c368SPatrick McHardy static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 178980d5c368SPatrick McHardy __be16 proto, u16 vid) 17900bde9569SAlex Williamson { 17910bde9569SAlex Williamson struct virtnet_info *vi = netdev_priv(dev); 17920bde9569SAlex Williamson struct scatterlist sg; 17930bde9569SAlex Williamson 1794d7fad4c8SMichael S. Tsirkin vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid); 179512e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid)); 17960bde9569SAlex Williamson 17970bde9569SAlex Williamson if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 1798d24bae32Sstephen hemminger VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 17990bde9569SAlex Williamson dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 18008e586137SJiri Pirko return 0; 18010bde9569SAlex Williamson } 18020bde9569SAlex Williamson 18038898c21cSWanlong Gao static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu) 1804986a4f4dSJason Wang { 1805986a4f4dSJason Wang int i; 18068898c21cSWanlong Gao 18078898c21cSWanlong Gao if (vi->affinity_hint_set) { 18088898c21cSWanlong Gao for (i = 0; i < vi->max_queue_pairs; i++) { 18098898c21cSWanlong Gao virtqueue_set_affinity(vi->rq[i].vq, -1); 18108898c21cSWanlong Gao virtqueue_set_affinity(vi->sq[i].vq, -1); 18118898c21cSWanlong Gao } 18128898c21cSWanlong Gao 18138898c21cSWanlong Gao vi->affinity_hint_set = false; 18148898c21cSWanlong Gao } 18158898c21cSWanlong Gao } 18168898c21cSWanlong Gao 18178898c21cSWanlong Gao static void virtnet_set_affinity(struct virtnet_info *vi) 1818986a4f4dSJason Wang { 1819986a4f4dSJason Wang int i; 182047be2479SWanlong Gao int cpu; 1821986a4f4dSJason Wang 1822986a4f4dSJason Wang /* In multiqueue mode, when the number of cpu is equal to the number of 1823986a4f4dSJason Wang * queue pairs, we let the queue pairs to be private to one cpu by 1824986a4f4dSJason Wang * setting the affinity hint to eliminate the contention. 1825986a4f4dSJason Wang */ 18268898c21cSWanlong Gao if (vi->curr_queue_pairs == 1 || 18278898c21cSWanlong Gao vi->max_queue_pairs != num_online_cpus()) { 18288898c21cSWanlong Gao virtnet_clean_affinity(vi, -1); 1829986a4f4dSJason Wang return; 1830986a4f4dSJason Wang } 1831986a4f4dSJason Wang 183247be2479SWanlong Gao i = 0; 183347be2479SWanlong Gao for_each_online_cpu(cpu) { 1834986a4f4dSJason Wang virtqueue_set_affinity(vi->rq[i].vq, cpu); 1835986a4f4dSJason Wang virtqueue_set_affinity(vi->sq[i].vq, cpu); 18369bb8ca86SJason Wang netif_set_xps_queue(vi->dev, cpumask_of(cpu), i); 183747be2479SWanlong Gao i++; 1838986a4f4dSJason Wang } 1839986a4f4dSJason Wang 1840986a4f4dSJason Wang vi->affinity_hint_set = true; 184147be2479SWanlong Gao } 1842986a4f4dSJason Wang 18438017c279SSebastian Andrzej Siewior static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 18448de4b2f3SWanlong Gao { 18458017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 18468017c279SSebastian Andrzej Siewior node); 18478de4b2f3SWanlong Gao virtnet_set_affinity(vi); 18488017c279SSebastian Andrzej Siewior return 0; 18498de4b2f3SWanlong Gao } 18503ab098dfSJason Wang 18518017c279SSebastian Andrzej Siewior static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 18528017c279SSebastian Andrzej Siewior { 18538017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 18548017c279SSebastian Andrzej Siewior node_dead); 18558017c279SSebastian Andrzej Siewior virtnet_set_affinity(vi); 18568017c279SSebastian Andrzej Siewior return 0; 18578017c279SSebastian Andrzej Siewior } 18588017c279SSebastian Andrzej Siewior 18598017c279SSebastian Andrzej Siewior static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 18608017c279SSebastian Andrzej Siewior { 18618017c279SSebastian Andrzej Siewior struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 18628017c279SSebastian Andrzej Siewior node); 18638017c279SSebastian Andrzej Siewior 18648017c279SSebastian Andrzej Siewior virtnet_clean_affinity(vi, cpu); 18658017c279SSebastian Andrzej Siewior return 0; 18668017c279SSebastian Andrzej Siewior } 18678017c279SSebastian Andrzej Siewior 18688017c279SSebastian Andrzej Siewior static enum cpuhp_state virtionet_online; 18698017c279SSebastian Andrzej Siewior 18708017c279SSebastian Andrzej Siewior static int virtnet_cpu_notif_add(struct virtnet_info *vi) 18718017c279SSebastian Andrzej Siewior { 18728017c279SSebastian Andrzej Siewior int ret; 18738017c279SSebastian Andrzej Siewior 18748017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 18758017c279SSebastian Andrzej Siewior if (ret) 18768017c279SSebastian Andrzej Siewior return ret; 18778017c279SSebastian Andrzej Siewior ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 18788017c279SSebastian Andrzej Siewior &vi->node_dead); 18798017c279SSebastian Andrzej Siewior if (!ret) 18808017c279SSebastian Andrzej Siewior return ret; 18818017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 18828017c279SSebastian Andrzej Siewior return ret; 18838017c279SSebastian Andrzej Siewior } 18848017c279SSebastian Andrzej Siewior 18858017c279SSebastian Andrzej Siewior static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 18868017c279SSebastian Andrzej Siewior { 18878017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 18888017c279SSebastian Andrzej Siewior cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 18898017c279SSebastian Andrzej Siewior &vi->node_dead); 1890a9ea3fc6SHerbert Xu } 1891a9ea3fc6SHerbert Xu 18928f9f4668SRick Jones static void virtnet_get_ringparam(struct net_device *dev, 18938f9f4668SRick Jones struct ethtool_ringparam *ring) 18948f9f4668SRick Jones { 18958f9f4668SRick Jones struct virtnet_info *vi = netdev_priv(dev); 18968f9f4668SRick Jones 1897986a4f4dSJason Wang ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq); 1898986a4f4dSJason Wang ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq); 18998f9f4668SRick Jones ring->rx_pending = ring->rx_max_pending; 19008f9f4668SRick Jones ring->tx_pending = ring->tx_max_pending; 19018f9f4668SRick Jones } 19028f9f4668SRick Jones 190366846048SRick Jones 190466846048SRick Jones static void virtnet_get_drvinfo(struct net_device *dev, 190566846048SRick Jones struct ethtool_drvinfo *info) 190666846048SRick Jones { 190766846048SRick Jones struct virtnet_info *vi = netdev_priv(dev); 190866846048SRick Jones struct virtio_device *vdev = vi->vdev; 190966846048SRick Jones 191066846048SRick Jones strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 191166846048SRick Jones strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 191266846048SRick Jones strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 191366846048SRick Jones 191466846048SRick Jones } 191566846048SRick Jones 1916d73bcd2cSJason Wang /* TODO: Eliminate OOO packets during switching */ 1917d73bcd2cSJason Wang static int virtnet_set_channels(struct net_device *dev, 1918d73bcd2cSJason Wang struct ethtool_channels *channels) 1919d73bcd2cSJason Wang { 1920d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 1921d73bcd2cSJason Wang u16 queue_pairs = channels->combined_count; 1922d73bcd2cSJason Wang int err; 1923d73bcd2cSJason Wang 1924d73bcd2cSJason Wang /* We don't support separate rx/tx channels. 1925d73bcd2cSJason Wang * We don't allow setting 'other' channels. 1926d73bcd2cSJason Wang */ 1927d73bcd2cSJason Wang if (channels->rx_count || channels->tx_count || channels->other_count) 1928d73bcd2cSJason Wang return -EINVAL; 1929d73bcd2cSJason Wang 1930c18e9cd6SAmos Kong if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 1931d73bcd2cSJason Wang return -EINVAL; 1932d73bcd2cSJason Wang 1933f600b690SJohn Fastabend /* For now we don't support modifying channels while XDP is loaded 1934f600b690SJohn Fastabend * also when XDP is loaded all RX queues have XDP programs so we only 1935f600b690SJohn Fastabend * need to check a single RX queue. 1936f600b690SJohn Fastabend */ 1937f600b690SJohn Fastabend if (vi->rq[0].xdp_prog) 1938f600b690SJohn Fastabend return -EINVAL; 1939f600b690SJohn Fastabend 194047be2479SWanlong Gao get_online_cpus(); 194147315329SJohn Fastabend err = _virtnet_set_queues(vi, queue_pairs); 1942d73bcd2cSJason Wang if (!err) { 1943d73bcd2cSJason Wang netif_set_real_num_tx_queues(dev, queue_pairs); 1944d73bcd2cSJason Wang netif_set_real_num_rx_queues(dev, queue_pairs); 1945d73bcd2cSJason Wang 19468898c21cSWanlong Gao virtnet_set_affinity(vi); 1947d73bcd2cSJason Wang } 194847be2479SWanlong Gao put_online_cpus(); 1949d73bcd2cSJason Wang 1950d73bcd2cSJason Wang return err; 1951d73bcd2cSJason Wang } 1952d73bcd2cSJason Wang 1953d7dfc5cfSToshiaki Makita static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 1954d7dfc5cfSToshiaki Makita { 1955d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 1956d7dfc5cfSToshiaki Makita char *p = (char *)data; 1957d7dfc5cfSToshiaki Makita unsigned int i, j; 1958d7dfc5cfSToshiaki Makita 1959d7dfc5cfSToshiaki Makita switch (stringset) { 1960d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 1961d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 1962d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { 1963d7dfc5cfSToshiaki Makita snprintf(p, ETH_GSTRING_LEN, "rx_queue_%u_%s", 1964d7dfc5cfSToshiaki Makita i, virtnet_rq_stats_desc[j].desc); 1965d7dfc5cfSToshiaki Makita p += ETH_GSTRING_LEN; 1966d7dfc5cfSToshiaki Makita } 1967d7dfc5cfSToshiaki Makita } 1968d7dfc5cfSToshiaki Makita 1969d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 1970d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { 1971d7dfc5cfSToshiaki Makita snprintf(p, ETH_GSTRING_LEN, "tx_queue_%u_%s", 1972d7dfc5cfSToshiaki Makita i, virtnet_sq_stats_desc[j].desc); 1973d7dfc5cfSToshiaki Makita p += ETH_GSTRING_LEN; 1974d7dfc5cfSToshiaki Makita } 1975d7dfc5cfSToshiaki Makita } 1976d7dfc5cfSToshiaki Makita break; 1977d7dfc5cfSToshiaki Makita } 1978d7dfc5cfSToshiaki Makita } 1979d7dfc5cfSToshiaki Makita 1980d7dfc5cfSToshiaki Makita static int virtnet_get_sset_count(struct net_device *dev, int sset) 1981d7dfc5cfSToshiaki Makita { 1982d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 1983d7dfc5cfSToshiaki Makita 1984d7dfc5cfSToshiaki Makita switch (sset) { 1985d7dfc5cfSToshiaki Makita case ETH_SS_STATS: 1986d7dfc5cfSToshiaki Makita return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN + 1987d7dfc5cfSToshiaki Makita VIRTNET_SQ_STATS_LEN); 1988d7dfc5cfSToshiaki Makita default: 1989d7dfc5cfSToshiaki Makita return -EOPNOTSUPP; 1990d7dfc5cfSToshiaki Makita } 1991d7dfc5cfSToshiaki Makita } 1992d7dfc5cfSToshiaki Makita 1993d7dfc5cfSToshiaki Makita static void virtnet_get_ethtool_stats(struct net_device *dev, 1994d7dfc5cfSToshiaki Makita struct ethtool_stats *stats, u64 *data) 1995d7dfc5cfSToshiaki Makita { 1996d7dfc5cfSToshiaki Makita struct virtnet_info *vi = netdev_priv(dev); 1997d7dfc5cfSToshiaki Makita unsigned int idx = 0, start, i, j; 1998d7dfc5cfSToshiaki Makita const u8 *stats_base; 1999d7dfc5cfSToshiaki Makita size_t offset; 2000d7dfc5cfSToshiaki Makita 2001d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2002d7dfc5cfSToshiaki Makita struct receive_queue *rq = &vi->rq[i]; 2003d7dfc5cfSToshiaki Makita 2004d7dfc5cfSToshiaki Makita stats_base = (u8 *)&rq->stats; 2005d7dfc5cfSToshiaki Makita do { 2006d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&rq->stats.syncp); 2007d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { 2008d7dfc5cfSToshiaki Makita offset = virtnet_rq_stats_desc[j].offset; 2009d7dfc5cfSToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 2010d7dfc5cfSToshiaki Makita } 2011d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); 2012d7dfc5cfSToshiaki Makita idx += VIRTNET_RQ_STATS_LEN; 2013d7dfc5cfSToshiaki Makita } 2014d7dfc5cfSToshiaki Makita 2015d7dfc5cfSToshiaki Makita for (i = 0; i < vi->curr_queue_pairs; i++) { 2016d7dfc5cfSToshiaki Makita struct send_queue *sq = &vi->sq[i]; 2017d7dfc5cfSToshiaki Makita 2018d7dfc5cfSToshiaki Makita stats_base = (u8 *)&sq->stats; 2019d7dfc5cfSToshiaki Makita do { 2020d7dfc5cfSToshiaki Makita start = u64_stats_fetch_begin_irq(&sq->stats.syncp); 2021d7dfc5cfSToshiaki Makita for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { 2022d7dfc5cfSToshiaki Makita offset = virtnet_sq_stats_desc[j].offset; 2023d7dfc5cfSToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 2024d7dfc5cfSToshiaki Makita } 2025d7dfc5cfSToshiaki Makita } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); 2026d7dfc5cfSToshiaki Makita idx += VIRTNET_SQ_STATS_LEN; 2027d7dfc5cfSToshiaki Makita } 2028d7dfc5cfSToshiaki Makita } 2029d7dfc5cfSToshiaki Makita 2030d73bcd2cSJason Wang static void virtnet_get_channels(struct net_device *dev, 2031d73bcd2cSJason Wang struct ethtool_channels *channels) 2032d73bcd2cSJason Wang { 2033d73bcd2cSJason Wang struct virtnet_info *vi = netdev_priv(dev); 2034d73bcd2cSJason Wang 2035d73bcd2cSJason Wang channels->combined_count = vi->curr_queue_pairs; 2036d73bcd2cSJason Wang channels->max_combined = vi->max_queue_pairs; 2037d73bcd2cSJason Wang channels->max_other = 0; 2038d73bcd2cSJason Wang channels->rx_count = 0; 2039d73bcd2cSJason Wang channels->tx_count = 0; 2040d73bcd2cSJason Wang channels->other_count = 0; 2041d73bcd2cSJason Wang } 2042d73bcd2cSJason Wang 204316032be5SNikolay Aleksandrov /* Check if the user is trying to change anything besides speed/duplex */ 2044ebb6b4b1SPhilippe Reynes static bool 2045ebb6b4b1SPhilippe Reynes virtnet_validate_ethtool_cmd(const struct ethtool_link_ksettings *cmd) 204616032be5SNikolay Aleksandrov { 2047ebb6b4b1SPhilippe Reynes struct ethtool_link_ksettings diff1 = *cmd; 2048ebb6b4b1SPhilippe Reynes struct ethtool_link_ksettings diff2 = {}; 204916032be5SNikolay Aleksandrov 20500cf3ace9SNikolay Aleksandrov /* cmd is always set so we need to clear it, validate the port type 20510cf3ace9SNikolay Aleksandrov * and also without autonegotiation we can ignore advertising 20520cf3ace9SNikolay Aleksandrov */ 2053ebb6b4b1SPhilippe Reynes diff1.base.speed = 0; 2054ebb6b4b1SPhilippe Reynes diff2.base.port = PORT_OTHER; 2055ebb6b4b1SPhilippe Reynes ethtool_link_ksettings_zero_link_mode(&diff1, advertising); 2056ebb6b4b1SPhilippe Reynes diff1.base.duplex = 0; 2057ebb6b4b1SPhilippe Reynes diff1.base.cmd = 0; 2058ebb6b4b1SPhilippe Reynes diff1.base.link_mode_masks_nwords = 0; 205916032be5SNikolay Aleksandrov 2060ebb6b4b1SPhilippe Reynes return !memcmp(&diff1.base, &diff2.base, sizeof(diff1.base)) && 2061ebb6b4b1SPhilippe Reynes bitmap_empty(diff1.link_modes.supported, 2062ebb6b4b1SPhilippe Reynes __ETHTOOL_LINK_MODE_MASK_NBITS) && 2063ebb6b4b1SPhilippe Reynes bitmap_empty(diff1.link_modes.advertising, 2064ebb6b4b1SPhilippe Reynes __ETHTOOL_LINK_MODE_MASK_NBITS) && 2065ebb6b4b1SPhilippe Reynes bitmap_empty(diff1.link_modes.lp_advertising, 2066ebb6b4b1SPhilippe Reynes __ETHTOOL_LINK_MODE_MASK_NBITS); 206716032be5SNikolay Aleksandrov } 206816032be5SNikolay Aleksandrov 2069ebb6b4b1SPhilippe Reynes static int virtnet_set_link_ksettings(struct net_device *dev, 2070ebb6b4b1SPhilippe Reynes const struct ethtool_link_ksettings *cmd) 207116032be5SNikolay Aleksandrov { 207216032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 207316032be5SNikolay Aleksandrov u32 speed; 207416032be5SNikolay Aleksandrov 2075ebb6b4b1SPhilippe Reynes speed = cmd->base.speed; 207616032be5SNikolay Aleksandrov /* don't allow custom speed and duplex */ 207716032be5SNikolay Aleksandrov if (!ethtool_validate_speed(speed) || 2078ebb6b4b1SPhilippe Reynes !ethtool_validate_duplex(cmd->base.duplex) || 207916032be5SNikolay Aleksandrov !virtnet_validate_ethtool_cmd(cmd)) 208016032be5SNikolay Aleksandrov return -EINVAL; 208116032be5SNikolay Aleksandrov vi->speed = speed; 2082ebb6b4b1SPhilippe Reynes vi->duplex = cmd->base.duplex; 208316032be5SNikolay Aleksandrov 208416032be5SNikolay Aleksandrov return 0; 208516032be5SNikolay Aleksandrov } 208616032be5SNikolay Aleksandrov 2087ebb6b4b1SPhilippe Reynes static int virtnet_get_link_ksettings(struct net_device *dev, 2088ebb6b4b1SPhilippe Reynes struct ethtool_link_ksettings *cmd) 208916032be5SNikolay Aleksandrov { 209016032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 209116032be5SNikolay Aleksandrov 2092ebb6b4b1SPhilippe Reynes cmd->base.speed = vi->speed; 2093ebb6b4b1SPhilippe Reynes cmd->base.duplex = vi->duplex; 2094ebb6b4b1SPhilippe Reynes cmd->base.port = PORT_OTHER; 209516032be5SNikolay Aleksandrov 209616032be5SNikolay Aleksandrov return 0; 209716032be5SNikolay Aleksandrov } 209816032be5SNikolay Aleksandrov 209916032be5SNikolay Aleksandrov static void virtnet_init_settings(struct net_device *dev) 210016032be5SNikolay Aleksandrov { 210116032be5SNikolay Aleksandrov struct virtnet_info *vi = netdev_priv(dev); 210216032be5SNikolay Aleksandrov 210316032be5SNikolay Aleksandrov vi->speed = SPEED_UNKNOWN; 210416032be5SNikolay Aleksandrov vi->duplex = DUPLEX_UNKNOWN; 210516032be5SNikolay Aleksandrov } 210616032be5SNikolay Aleksandrov 2107faa9b39fSJason Baron static void virtnet_update_settings(struct virtnet_info *vi) 2108faa9b39fSJason Baron { 2109faa9b39fSJason Baron u32 speed; 2110faa9b39fSJason Baron u8 duplex; 2111faa9b39fSJason Baron 2112faa9b39fSJason Baron if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 2113faa9b39fSJason Baron return; 2114faa9b39fSJason Baron 2115faa9b39fSJason Baron speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config, 2116faa9b39fSJason Baron speed)); 2117faa9b39fSJason Baron if (ethtool_validate_speed(speed)) 2118faa9b39fSJason Baron vi->speed = speed; 2119faa9b39fSJason Baron duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config, 2120faa9b39fSJason Baron duplex)); 2121faa9b39fSJason Baron if (ethtool_validate_duplex(duplex)) 2122faa9b39fSJason Baron vi->duplex = duplex; 2123faa9b39fSJason Baron } 2124faa9b39fSJason Baron 21250fc0b732SStephen Hemminger static const struct ethtool_ops virtnet_ethtool_ops = { 212666846048SRick Jones .get_drvinfo = virtnet_get_drvinfo, 21279f4d26d0SMark McLoughlin .get_link = ethtool_op_get_link, 21288f9f4668SRick Jones .get_ringparam = virtnet_get_ringparam, 2129d7dfc5cfSToshiaki Makita .get_strings = virtnet_get_strings, 2130d7dfc5cfSToshiaki Makita .get_sset_count = virtnet_get_sset_count, 2131d7dfc5cfSToshiaki Makita .get_ethtool_stats = virtnet_get_ethtool_stats, 2132d73bcd2cSJason Wang .set_channels = virtnet_set_channels, 2133d73bcd2cSJason Wang .get_channels = virtnet_get_channels, 2134074c3582SJacob Keller .get_ts_info = ethtool_op_get_ts_info, 2135ebb6b4b1SPhilippe Reynes .get_link_ksettings = virtnet_get_link_ksettings, 2136ebb6b4b1SPhilippe Reynes .set_link_ksettings = virtnet_set_link_ksettings, 2137a9ea3fc6SHerbert Xu }; 2138a9ea3fc6SHerbert Xu 21399fe7bfceSJohn Fastabend static void virtnet_freeze_down(struct virtio_device *vdev) 21409fe7bfceSJohn Fastabend { 21419fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 21429fe7bfceSJohn Fastabend int i; 21439fe7bfceSJohn Fastabend 21449fe7bfceSJohn Fastabend /* Make sure no work handler is accessing the device */ 21459fe7bfceSJohn Fastabend flush_work(&vi->config_work); 21469fe7bfceSJohn Fastabend 21479fe7bfceSJohn Fastabend netif_device_detach(vi->dev); 2148713a98d9SJason Wang netif_tx_disable(vi->dev); 21499fe7bfceSJohn Fastabend cancel_delayed_work_sync(&vi->refill); 21509fe7bfceSJohn Fastabend 21519fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 2152b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 21539fe7bfceSJohn Fastabend napi_disable(&vi->rq[i].napi); 215478a57b48SWillem de Bruijn virtnet_napi_tx_disable(&vi->sq[i].napi); 2155b92f1e67SWillem de Bruijn } 21569fe7bfceSJohn Fastabend } 21579fe7bfceSJohn Fastabend } 21589fe7bfceSJohn Fastabend 21599fe7bfceSJohn Fastabend static int init_vqs(struct virtnet_info *vi); 21609fe7bfceSJohn Fastabend 21619fe7bfceSJohn Fastabend static int virtnet_restore_up(struct virtio_device *vdev) 21629fe7bfceSJohn Fastabend { 21639fe7bfceSJohn Fastabend struct virtnet_info *vi = vdev->priv; 21649fe7bfceSJohn Fastabend int err, i; 21659fe7bfceSJohn Fastabend 21669fe7bfceSJohn Fastabend err = init_vqs(vi); 21679fe7bfceSJohn Fastabend if (err) 21689fe7bfceSJohn Fastabend return err; 21699fe7bfceSJohn Fastabend 21709fe7bfceSJohn Fastabend virtio_device_ready(vdev); 21719fe7bfceSJohn Fastabend 21729fe7bfceSJohn Fastabend if (netif_running(vi->dev)) { 21739fe7bfceSJohn Fastabend for (i = 0; i < vi->curr_queue_pairs; i++) 21749fe7bfceSJohn Fastabend if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 21759fe7bfceSJohn Fastabend schedule_delayed_work(&vi->refill, 0); 21769fe7bfceSJohn Fastabend 2177b92f1e67SWillem de Bruijn for (i = 0; i < vi->max_queue_pairs; i++) { 2178e4e8452aSWillem de Bruijn virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2179b92f1e67SWillem de Bruijn virtnet_napi_tx_enable(vi, vi->sq[i].vq, 2180b92f1e67SWillem de Bruijn &vi->sq[i].napi); 2181b92f1e67SWillem de Bruijn } 21829fe7bfceSJohn Fastabend } 21839fe7bfceSJohn Fastabend 21849fe7bfceSJohn Fastabend netif_device_attach(vi->dev); 21859fe7bfceSJohn Fastabend return err; 21869fe7bfceSJohn Fastabend } 21879fe7bfceSJohn Fastabend 21883f93522fSJason Wang static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 21893f93522fSJason Wang { 21903f93522fSJason Wang struct scatterlist sg; 219112e57169SMichael S. Tsirkin vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads); 21923f93522fSJason Wang 219312e57169SMichael S. Tsirkin sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads)); 21943f93522fSJason Wang 21953f93522fSJason Wang if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 21963f93522fSJason Wang VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 21973f93522fSJason Wang dev_warn(&vi->dev->dev, "Fail to set guest offload. \n"); 21983f93522fSJason Wang return -EINVAL; 21993f93522fSJason Wang } 22003f93522fSJason Wang 22013f93522fSJason Wang return 0; 22023f93522fSJason Wang } 22033f93522fSJason Wang 22043f93522fSJason Wang static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 22053f93522fSJason Wang { 22063f93522fSJason Wang u64 offloads = 0; 22073f93522fSJason Wang 22083f93522fSJason Wang if (!vi->guest_offloads) 22093f93522fSJason Wang return 0; 22103f93522fSJason Wang 22113f93522fSJason Wang if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM)) 22123f93522fSJason Wang offloads = 1ULL << VIRTIO_NET_F_GUEST_CSUM; 22133f93522fSJason Wang 22143f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 22153f93522fSJason Wang } 22163f93522fSJason Wang 22173f93522fSJason Wang static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 22183f93522fSJason Wang { 22193f93522fSJason Wang u64 offloads = vi->guest_offloads; 22203f93522fSJason Wang 22213f93522fSJason Wang if (!vi->guest_offloads) 22223f93522fSJason Wang return 0; 22233f93522fSJason Wang if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM)) 22243f93522fSJason Wang offloads |= 1ULL << VIRTIO_NET_F_GUEST_CSUM; 22253f93522fSJason Wang 22263f93522fSJason Wang return virtnet_set_guest_offloads(vi, offloads); 22273f93522fSJason Wang } 22283f93522fSJason Wang 22299861ce03SJakub Kicinski static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 22309861ce03SJakub Kicinski struct netlink_ext_ack *extack) 2231f600b690SJohn Fastabend { 2232f600b690SJohn Fastabend unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr); 2233f600b690SJohn Fastabend struct virtnet_info *vi = netdev_priv(dev); 2234f600b690SJohn Fastabend struct bpf_prog *old_prog; 2235017b29c3SJason Wang u16 xdp_qp = 0, curr_qp; 2236672aafd5SJohn Fastabend int i, err; 2237f600b690SJohn Fastabend 22383f93522fSJason Wang if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 22393f93522fSJason Wang && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 224092502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 224192502fe8SJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 22423f93522fSJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO))) { 22434d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO, disable LRO first"); 2244f600b690SJohn Fastabend return -EOPNOTSUPP; 2245f600b690SJohn Fastabend } 2246f600b690SJohn Fastabend 2247f600b690SJohn Fastabend if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 22484d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 2249f600b690SJohn Fastabend return -EINVAL; 2250f600b690SJohn Fastabend } 2251f600b690SJohn Fastabend 2252f600b690SJohn Fastabend if (dev->mtu > max_sz) { 22534d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP"); 2254f600b690SJohn Fastabend netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz); 2255f600b690SJohn Fastabend return -EINVAL; 2256f600b690SJohn Fastabend } 2257f600b690SJohn Fastabend 2258672aafd5SJohn Fastabend curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 2259672aafd5SJohn Fastabend if (prog) 2260672aafd5SJohn Fastabend xdp_qp = nr_cpu_ids; 2261672aafd5SJohn Fastabend 2262672aafd5SJohn Fastabend /* XDP requires extra queues for XDP_TX */ 2263672aafd5SJohn Fastabend if (curr_qp + xdp_qp > vi->max_queue_pairs) { 22644d463c4dSDaniel Borkmann NL_SET_ERR_MSG_MOD(extack, "Too few free TX rings available"); 2265672aafd5SJohn Fastabend netdev_warn(dev, "request %i queues but max is %i\n", 2266672aafd5SJohn Fastabend curr_qp + xdp_qp, vi->max_queue_pairs); 2267672aafd5SJohn Fastabend return -ENOMEM; 2268672aafd5SJohn Fastabend } 2269672aafd5SJohn Fastabend 22702de2f7f4SJohn Fastabend if (prog) { 22712de2f7f4SJohn Fastabend prog = bpf_prog_add(prog, vi->max_queue_pairs - 1); 22722de2f7f4SJohn Fastabend if (IS_ERR(prog)) 22732de2f7f4SJohn Fastabend return PTR_ERR(prog); 22742de2f7f4SJohn Fastabend } 22752de2f7f4SJohn Fastabend 22764941d472SJason Wang /* Make sure NAPI is not using any XDP TX queues for RX. */ 22774e09ff53SJason Wang if (netif_running(dev)) 22784941d472SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 22794941d472SJason Wang napi_disable(&vi->rq[i].napi); 22802de2f7f4SJohn Fastabend 2281672aafd5SJohn Fastabend netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 22824941d472SJason Wang err = _virtnet_set_queues(vi, curr_qp + xdp_qp); 22834941d472SJason Wang if (err) 22844941d472SJason Wang goto err; 22854941d472SJason Wang vi->xdp_queue_pairs = xdp_qp; 2286f600b690SJohn Fastabend 2287f600b690SJohn Fastabend for (i = 0; i < vi->max_queue_pairs; i++) { 2288f600b690SJohn Fastabend old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 2289f600b690SJohn Fastabend rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 22903f93522fSJason Wang if (i == 0) { 22913f93522fSJason Wang if (!old_prog) 22923f93522fSJason Wang virtnet_clear_guest_offloads(vi); 22933f93522fSJason Wang if (!prog) 22943f93522fSJason Wang virtnet_restore_guest_offloads(vi); 22953f93522fSJason Wang } 2296f600b690SJohn Fastabend if (old_prog) 2297f600b690SJohn Fastabend bpf_prog_put(old_prog); 22984e09ff53SJason Wang if (netif_running(dev)) 22994941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 2300f600b690SJohn Fastabend } 2301f600b690SJohn Fastabend 2302f600b690SJohn Fastabend return 0; 23032de2f7f4SJohn Fastabend 23044941d472SJason Wang err: 23054941d472SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) 23064941d472SJason Wang virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi); 23072de2f7f4SJohn Fastabend if (prog) 23082de2f7f4SJohn Fastabend bpf_prog_sub(prog, vi->max_queue_pairs - 1); 23092de2f7f4SJohn Fastabend return err; 2310f600b690SJohn Fastabend } 2311f600b690SJohn Fastabend 23125b0e6629SMartin KaFai Lau static u32 virtnet_xdp_query(struct net_device *dev) 2313f600b690SJohn Fastabend { 2314f600b690SJohn Fastabend struct virtnet_info *vi = netdev_priv(dev); 23155b0e6629SMartin KaFai Lau const struct bpf_prog *xdp_prog; 2316f600b690SJohn Fastabend int i; 2317f600b690SJohn Fastabend 2318f600b690SJohn Fastabend for (i = 0; i < vi->max_queue_pairs; i++) { 23195b0e6629SMartin KaFai Lau xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog); 23205b0e6629SMartin KaFai Lau if (xdp_prog) 23215b0e6629SMartin KaFai Lau return xdp_prog->aux->id; 2322f600b690SJohn Fastabend } 23235b0e6629SMartin KaFai Lau return 0; 2324f600b690SJohn Fastabend } 2325f600b690SJohn Fastabend 2326f4e63525SJakub Kicinski static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2327f600b690SJohn Fastabend { 2328f600b690SJohn Fastabend switch (xdp->command) { 2329f600b690SJohn Fastabend case XDP_SETUP_PROG: 23309861ce03SJakub Kicinski return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 2331f600b690SJohn Fastabend case XDP_QUERY_PROG: 23325b0e6629SMartin KaFai Lau xdp->prog_id = virtnet_xdp_query(dev); 23335b0e6629SMartin KaFai Lau xdp->prog_attached = !!xdp->prog_id; 2334f600b690SJohn Fastabend return 0; 2335f600b690SJohn Fastabend default: 2336f600b690SJohn Fastabend return -EINVAL; 2337f600b690SJohn Fastabend } 2338f600b690SJohn Fastabend } 2339f600b690SJohn Fastabend 234076288b4eSStephen Hemminger static const struct net_device_ops virtnet_netdev = { 234176288b4eSStephen Hemminger .ndo_open = virtnet_open, 234276288b4eSStephen Hemminger .ndo_stop = virtnet_close, 234376288b4eSStephen Hemminger .ndo_start_xmit = start_xmit, 234476288b4eSStephen Hemminger .ndo_validate_addr = eth_validate_addr, 23459c46f6d4SAlex Williamson .ndo_set_mac_address = virtnet_set_mac_address, 23462af7698eSAlex Williamson .ndo_set_rx_mode = virtnet_set_rx_mode, 23473fa2a1dfSstephen hemminger .ndo_get_stats64 = virtnet_stats, 23481824a989SAlex Williamson .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 23491824a989SAlex Williamson .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 235076288b4eSStephen Hemminger #ifdef CONFIG_NET_POLL_CONTROLLER 235176288b4eSStephen Hemminger .ndo_poll_controller = virtnet_netpoll, 235276288b4eSStephen Hemminger #endif 2353f4e63525SJakub Kicinski .ndo_bpf = virtnet_xdp, 2354186b3c99SJason Wang .ndo_xdp_xmit = virtnet_xdp_xmit, 2355186b3c99SJason Wang .ndo_xdp_flush = virtnet_xdp_flush, 23562836b4f2SVlad Yasevich .ndo_features_check = passthru_features_check, 235776288b4eSStephen Hemminger }; 235876288b4eSStephen Hemminger 2359586d17c5SJason Wang static void virtnet_config_changed_work(struct work_struct *work) 23609f4d26d0SMark McLoughlin { 2361586d17c5SJason Wang struct virtnet_info *vi = 2362586d17c5SJason Wang container_of(work, struct virtnet_info, config_work); 23639f4d26d0SMark McLoughlin u16 v; 23649f4d26d0SMark McLoughlin 2365855e0c52SRusty Russell if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 2366855e0c52SRusty Russell struct virtio_net_config, status, &v) < 0) 2367507613bfSMichael S. Tsirkin return; 2368586d17c5SJason Wang 2369586d17c5SJason Wang if (v & VIRTIO_NET_S_ANNOUNCE) { 2370ee89bab1SAmerigo Wang netdev_notify_peers(vi->dev); 2371586d17c5SJason Wang virtnet_ack_link_announce(vi); 2372586d17c5SJason Wang } 23739f4d26d0SMark McLoughlin 23749f4d26d0SMark McLoughlin /* Ignore unknown (future) status bits */ 23759f4d26d0SMark McLoughlin v &= VIRTIO_NET_S_LINK_UP; 23769f4d26d0SMark McLoughlin 23779f4d26d0SMark McLoughlin if (vi->status == v) 2378507613bfSMichael S. Tsirkin return; 23799f4d26d0SMark McLoughlin 23809f4d26d0SMark McLoughlin vi->status = v; 23819f4d26d0SMark McLoughlin 23829f4d26d0SMark McLoughlin if (vi->status & VIRTIO_NET_S_LINK_UP) { 2383faa9b39fSJason Baron virtnet_update_settings(vi); 23849f4d26d0SMark McLoughlin netif_carrier_on(vi->dev); 2385986a4f4dSJason Wang netif_tx_wake_all_queues(vi->dev); 23869f4d26d0SMark McLoughlin } else { 23879f4d26d0SMark McLoughlin netif_carrier_off(vi->dev); 2388986a4f4dSJason Wang netif_tx_stop_all_queues(vi->dev); 23899f4d26d0SMark McLoughlin } 23909f4d26d0SMark McLoughlin } 23919f4d26d0SMark McLoughlin 23929f4d26d0SMark McLoughlin static void virtnet_config_changed(struct virtio_device *vdev) 23939f4d26d0SMark McLoughlin { 23949f4d26d0SMark McLoughlin struct virtnet_info *vi = vdev->priv; 23959f4d26d0SMark McLoughlin 23963b07e9caSTejun Heo schedule_work(&vi->config_work); 23979f4d26d0SMark McLoughlin } 23989f4d26d0SMark McLoughlin 2399986a4f4dSJason Wang static void virtnet_free_queues(struct virtnet_info *vi) 2400986a4f4dSJason Wang { 2401d4fb84eeSAndrey Vagin int i; 2402d4fb84eeSAndrey Vagin 2403ab3971b1SJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2404ab3971b1SJason Wang napi_hash_del(&vi->rq[i].napi); 2405d4fb84eeSAndrey Vagin netif_napi_del(&vi->rq[i].napi); 2406b92f1e67SWillem de Bruijn netif_napi_del(&vi->sq[i].napi); 2407ab3971b1SJason Wang } 2408d4fb84eeSAndrey Vagin 2409963abe5cSEric Dumazet /* We called napi_hash_del() before netif_napi_del(), 2410963abe5cSEric Dumazet * we need to respect an RCU grace period before freeing vi->rq 2411963abe5cSEric Dumazet */ 2412963abe5cSEric Dumazet synchronize_net(); 2413963abe5cSEric Dumazet 2414986a4f4dSJason Wang kfree(vi->rq); 2415986a4f4dSJason Wang kfree(vi->sq); 241612e57169SMichael S. Tsirkin kfree(vi->ctrl); 2417986a4f4dSJason Wang } 2418986a4f4dSJason Wang 241947315329SJohn Fastabend static void _free_receive_bufs(struct virtnet_info *vi) 2420986a4f4dSJason Wang { 2421f600b690SJohn Fastabend struct bpf_prog *old_prog; 2422986a4f4dSJason Wang int i; 2423986a4f4dSJason Wang 2424986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2425986a4f4dSJason Wang while (vi->rq[i].pages) 2426986a4f4dSJason Wang __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 2427f600b690SJohn Fastabend 2428f600b690SJohn Fastabend old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 2429f600b690SJohn Fastabend RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 2430f600b690SJohn Fastabend if (old_prog) 2431f600b690SJohn Fastabend bpf_prog_put(old_prog); 2432986a4f4dSJason Wang } 243347315329SJohn Fastabend } 243447315329SJohn Fastabend 243547315329SJohn Fastabend static void free_receive_bufs(struct virtnet_info *vi) 243647315329SJohn Fastabend { 243747315329SJohn Fastabend rtnl_lock(); 243847315329SJohn Fastabend _free_receive_bufs(vi); 2439f600b690SJohn Fastabend rtnl_unlock(); 2440986a4f4dSJason Wang } 2441986a4f4dSJason Wang 2442fb51879dSMichael Dalton static void free_receive_page_frags(struct virtnet_info *vi) 2443fb51879dSMichael Dalton { 2444fb51879dSMichael Dalton int i; 2445fb51879dSMichael Dalton for (i = 0; i < vi->max_queue_pairs; i++) 2446fb51879dSMichael Dalton if (vi->rq[i].alloc_frag.page) 2447fb51879dSMichael Dalton put_page(vi->rq[i].alloc_frag.page); 2448fb51879dSMichael Dalton } 2449fb51879dSMichael Dalton 2450b68df015SJohn Fastabend static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 245156434a01SJohn Fastabend { 245256434a01SJohn Fastabend if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 245356434a01SJohn Fastabend return false; 245456434a01SJohn Fastabend else if (q < vi->curr_queue_pairs) 245556434a01SJohn Fastabend return true; 245656434a01SJohn Fastabend else 245756434a01SJohn Fastabend return false; 245856434a01SJohn Fastabend } 245956434a01SJohn Fastabend 2460986a4f4dSJason Wang static void free_unused_bufs(struct virtnet_info *vi) 2461986a4f4dSJason Wang { 2462986a4f4dSJason Wang void *buf; 2463986a4f4dSJason Wang int i; 2464986a4f4dSJason Wang 2465986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2466986a4f4dSJason Wang struct virtqueue *vq = vi->sq[i].vq; 246756434a01SJohn Fastabend while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { 2468b68df015SJohn Fastabend if (!is_xdp_raw_buffer_queue(vi, i)) 2469986a4f4dSJason Wang dev_kfree_skb(buf); 247056434a01SJohn Fastabend else 247156434a01SJohn Fastabend put_page(virt_to_head_page(buf)); 247256434a01SJohn Fastabend } 2473986a4f4dSJason Wang } 2474986a4f4dSJason Wang 2475986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2476986a4f4dSJason Wang struct virtqueue *vq = vi->rq[i].vq; 2477986a4f4dSJason Wang 2478986a4f4dSJason Wang while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { 2479ab7db917SMichael Dalton if (vi->mergeable_rx_bufs) { 2480680557cfSMichael S. Tsirkin put_page(virt_to_head_page(buf)); 2481ab7db917SMichael Dalton } else if (vi->big_packets) { 2482fa9fac17SAndrey Vagin give_pages(&vi->rq[i], buf); 2483ab7db917SMichael Dalton } else { 2484f6b10209SJason Wang put_page(virt_to_head_page(buf)); 2485986a4f4dSJason Wang } 2486986a4f4dSJason Wang } 2487986a4f4dSJason Wang } 2488ab7db917SMichael Dalton } 2489986a4f4dSJason Wang 2490e9d7417bSJason Wang static void virtnet_del_vqs(struct virtnet_info *vi) 2491e9d7417bSJason Wang { 2492e9d7417bSJason Wang struct virtio_device *vdev = vi->vdev; 2493e9d7417bSJason Wang 24948898c21cSWanlong Gao virtnet_clean_affinity(vi, -1); 2495986a4f4dSJason Wang 2496e9d7417bSJason Wang vdev->config->del_vqs(vdev); 2497986a4f4dSJason Wang 2498986a4f4dSJason Wang virtnet_free_queues(vi); 2499986a4f4dSJason Wang } 2500986a4f4dSJason Wang 2501d85b758fSMichael S. Tsirkin /* How large should a single buffer be so a queue full of these can fit at 2502d85b758fSMichael S. Tsirkin * least one full packet? 2503d85b758fSMichael S. Tsirkin * Logic below assumes the mergeable buffer header is used. 2504d85b758fSMichael S. Tsirkin */ 2505d85b758fSMichael S. Tsirkin static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 2506d85b758fSMichael S. Tsirkin { 2507d85b758fSMichael S. Tsirkin const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 2508d85b758fSMichael S. Tsirkin unsigned int rq_size = virtqueue_get_vring_size(vq); 2509d85b758fSMichael S. Tsirkin unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 2510d85b758fSMichael S. Tsirkin unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 2511d85b758fSMichael S. Tsirkin unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 2512d85b758fSMichael S. Tsirkin 2513f0c3192cSMichael S. Tsirkin return max(max(min_buf_len, hdr_len) - hdr_len, 2514f0c3192cSMichael S. Tsirkin (unsigned int)GOOD_PACKET_LEN); 2515d85b758fSMichael S. Tsirkin } 2516d85b758fSMichael S. Tsirkin 2517986a4f4dSJason Wang static int virtnet_find_vqs(struct virtnet_info *vi) 2518986a4f4dSJason Wang { 2519986a4f4dSJason Wang vq_callback_t **callbacks; 2520986a4f4dSJason Wang struct virtqueue **vqs; 2521986a4f4dSJason Wang int ret = -ENOMEM; 2522986a4f4dSJason Wang int i, total_vqs; 2523986a4f4dSJason Wang const char **names; 2524d45b897bSMichael S. Tsirkin bool *ctx; 2525986a4f4dSJason Wang 2526986a4f4dSJason Wang /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 2527986a4f4dSJason Wang * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 2528986a4f4dSJason Wang * possible control vq. 2529986a4f4dSJason Wang */ 2530986a4f4dSJason Wang total_vqs = vi->max_queue_pairs * 2 + 2531986a4f4dSJason Wang virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 2532986a4f4dSJason Wang 2533986a4f4dSJason Wang /* Allocate space for find_vqs parameters */ 2534986a4f4dSJason Wang vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL); 2535986a4f4dSJason Wang if (!vqs) 2536986a4f4dSJason Wang goto err_vq; 2537986a4f4dSJason Wang callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL); 2538986a4f4dSJason Wang if (!callbacks) 2539986a4f4dSJason Wang goto err_callback; 2540986a4f4dSJason Wang names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL); 2541986a4f4dSJason Wang if (!names) 2542986a4f4dSJason Wang goto err_names; 2543192f68cfSJason Wang if (!vi->big_packets || vi->mergeable_rx_bufs) { 2544d45b897bSMichael S. Tsirkin ctx = kzalloc(total_vqs * sizeof(*ctx), GFP_KERNEL); 2545d45b897bSMichael S. Tsirkin if (!ctx) 2546d45b897bSMichael S. Tsirkin goto err_ctx; 2547d45b897bSMichael S. Tsirkin } else { 2548d45b897bSMichael S. Tsirkin ctx = NULL; 2549d45b897bSMichael S. Tsirkin } 2550986a4f4dSJason Wang 2551986a4f4dSJason Wang /* Parameters for control virtqueue, if any */ 2552986a4f4dSJason Wang if (vi->has_cvq) { 2553986a4f4dSJason Wang callbacks[total_vqs - 1] = NULL; 2554986a4f4dSJason Wang names[total_vqs - 1] = "control"; 2555986a4f4dSJason Wang } 2556986a4f4dSJason Wang 2557986a4f4dSJason Wang /* Allocate/initialize parameters for send/receive virtqueues */ 2558986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2559986a4f4dSJason Wang callbacks[rxq2vq(i)] = skb_recv_done; 2560986a4f4dSJason Wang callbacks[txq2vq(i)] = skb_xmit_done; 2561986a4f4dSJason Wang sprintf(vi->rq[i].name, "input.%d", i); 2562986a4f4dSJason Wang sprintf(vi->sq[i].name, "output.%d", i); 2563986a4f4dSJason Wang names[rxq2vq(i)] = vi->rq[i].name; 2564986a4f4dSJason Wang names[txq2vq(i)] = vi->sq[i].name; 2565d45b897bSMichael S. Tsirkin if (ctx) 2566d45b897bSMichael S. Tsirkin ctx[rxq2vq(i)] = true; 2567986a4f4dSJason Wang } 2568986a4f4dSJason Wang 2569986a4f4dSJason Wang ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks, 2570d45b897bSMichael S. Tsirkin names, ctx, NULL); 2571986a4f4dSJason Wang if (ret) 2572986a4f4dSJason Wang goto err_find; 2573986a4f4dSJason Wang 2574986a4f4dSJason Wang if (vi->has_cvq) { 2575986a4f4dSJason Wang vi->cvq = vqs[total_vqs - 1]; 2576986a4f4dSJason Wang if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 2577f646968fSPatrick McHardy vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 2578986a4f4dSJason Wang } 2579986a4f4dSJason Wang 2580986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2581986a4f4dSJason Wang vi->rq[i].vq = vqs[rxq2vq(i)]; 2582d85b758fSMichael S. Tsirkin vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 2583986a4f4dSJason Wang vi->sq[i].vq = vqs[txq2vq(i)]; 2584986a4f4dSJason Wang } 2585986a4f4dSJason Wang 2586986a4f4dSJason Wang kfree(names); 2587986a4f4dSJason Wang kfree(callbacks); 2588986a4f4dSJason Wang kfree(vqs); 258955281621SJason Wang kfree(ctx); 2590986a4f4dSJason Wang 2591986a4f4dSJason Wang return 0; 2592986a4f4dSJason Wang 2593986a4f4dSJason Wang err_find: 2594d45b897bSMichael S. Tsirkin kfree(ctx); 2595d45b897bSMichael S. Tsirkin err_ctx: 2596986a4f4dSJason Wang kfree(names); 2597986a4f4dSJason Wang err_names: 2598986a4f4dSJason Wang kfree(callbacks); 2599986a4f4dSJason Wang err_callback: 2600986a4f4dSJason Wang kfree(vqs); 2601986a4f4dSJason Wang err_vq: 2602986a4f4dSJason Wang return ret; 2603986a4f4dSJason Wang } 2604986a4f4dSJason Wang 2605986a4f4dSJason Wang static int virtnet_alloc_queues(struct virtnet_info *vi) 2606986a4f4dSJason Wang { 2607986a4f4dSJason Wang int i; 2608986a4f4dSJason Wang 260912e57169SMichael S. Tsirkin vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 261012e57169SMichael S. Tsirkin if (!vi->ctrl) 261112e57169SMichael S. Tsirkin goto err_ctrl; 2612986a4f4dSJason Wang vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL); 2613986a4f4dSJason Wang if (!vi->sq) 2614986a4f4dSJason Wang goto err_sq; 2615986a4f4dSJason Wang vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL); 2616008d4278SAmerigo Wang if (!vi->rq) 2617986a4f4dSJason Wang goto err_rq; 2618986a4f4dSJason Wang 2619986a4f4dSJason Wang INIT_DELAYED_WORK(&vi->refill, refill_work); 2620986a4f4dSJason Wang for (i = 0; i < vi->max_queue_pairs; i++) { 2621986a4f4dSJason Wang vi->rq[i].pages = NULL; 2622986a4f4dSJason Wang netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll, 2623986a4f4dSJason Wang napi_weight); 26241d11e732SWillem de Bruijn netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx, 2625b92f1e67SWillem de Bruijn napi_tx ? napi_weight : 0); 2626986a4f4dSJason Wang 2627986a4f4dSJason Wang sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 26285377d758SJohannes Berg ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 2629986a4f4dSJason Wang sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 2630d7dfc5cfSToshiaki Makita 2631d7dfc5cfSToshiaki Makita u64_stats_init(&vi->rq[i].stats.syncp); 2632d7dfc5cfSToshiaki Makita u64_stats_init(&vi->sq[i].stats.syncp); 2633986a4f4dSJason Wang } 2634986a4f4dSJason Wang 2635986a4f4dSJason Wang return 0; 2636986a4f4dSJason Wang 2637986a4f4dSJason Wang err_rq: 2638986a4f4dSJason Wang kfree(vi->sq); 2639986a4f4dSJason Wang err_sq: 264012e57169SMichael S. Tsirkin kfree(vi->ctrl); 264112e57169SMichael S. Tsirkin err_ctrl: 2642986a4f4dSJason Wang return -ENOMEM; 2643e9d7417bSJason Wang } 2644e9d7417bSJason Wang 26453f9c10b0SAmit Shah static int init_vqs(struct virtnet_info *vi) 26463f9c10b0SAmit Shah { 2647986a4f4dSJason Wang int ret; 26483f9c10b0SAmit Shah 2649986a4f4dSJason Wang /* Allocate send & receive queues */ 2650986a4f4dSJason Wang ret = virtnet_alloc_queues(vi); 2651986a4f4dSJason Wang if (ret) 2652986a4f4dSJason Wang goto err; 26533f9c10b0SAmit Shah 2654986a4f4dSJason Wang ret = virtnet_find_vqs(vi); 2655986a4f4dSJason Wang if (ret) 2656986a4f4dSJason Wang goto err_free; 26573f9c10b0SAmit Shah 265847be2479SWanlong Gao get_online_cpus(); 26598898c21cSWanlong Gao virtnet_set_affinity(vi); 266047be2479SWanlong Gao put_online_cpus(); 266147be2479SWanlong Gao 26623f9c10b0SAmit Shah return 0; 2663986a4f4dSJason Wang 2664986a4f4dSJason Wang err_free: 2665986a4f4dSJason Wang virtnet_free_queues(vi); 2666986a4f4dSJason Wang err: 2667986a4f4dSJason Wang return ret; 26683f9c10b0SAmit Shah } 26693f9c10b0SAmit Shah 2670fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 2671fbf28d78SMichael Dalton static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 2672718ad681Sstephen hemminger char *buf) 2673fbf28d78SMichael Dalton { 2674fbf28d78SMichael Dalton struct virtnet_info *vi = netdev_priv(queue->dev); 2675fbf28d78SMichael Dalton unsigned int queue_index = get_netdev_rx_queue_index(queue); 26763cc81a9aSJason Wang unsigned int headroom = virtnet_get_headroom(vi); 26773cc81a9aSJason Wang unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 26785377d758SJohannes Berg struct ewma_pkt_len *avg; 2679fbf28d78SMichael Dalton 2680fbf28d78SMichael Dalton BUG_ON(queue_index >= vi->max_queue_pairs); 2681fbf28d78SMichael Dalton avg = &vi->rq[queue_index].mrg_avg_pkt_len; 2682d85b758fSMichael S. Tsirkin return sprintf(buf, "%u\n", 26833cc81a9aSJason Wang get_mergeable_buf_len(&vi->rq[queue_index], avg, 26843cc81a9aSJason Wang SKB_DATA_ALIGN(headroom + tailroom))); 2685fbf28d78SMichael Dalton } 2686fbf28d78SMichael Dalton 2687fbf28d78SMichael Dalton static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 2688fbf28d78SMichael Dalton __ATTR_RO(mergeable_rx_buffer_size); 2689fbf28d78SMichael Dalton 2690fbf28d78SMichael Dalton static struct attribute *virtio_net_mrg_rx_attrs[] = { 2691fbf28d78SMichael Dalton &mergeable_rx_buffer_size_attribute.attr, 2692fbf28d78SMichael Dalton NULL 2693fbf28d78SMichael Dalton }; 2694fbf28d78SMichael Dalton 2695fbf28d78SMichael Dalton static const struct attribute_group virtio_net_mrg_rx_group = { 2696fbf28d78SMichael Dalton .name = "virtio_net", 2697fbf28d78SMichael Dalton .attrs = virtio_net_mrg_rx_attrs 2698fbf28d78SMichael Dalton }; 2699fbf28d78SMichael Dalton #endif 2700fbf28d78SMichael Dalton 2701892d6eb1SJason Wang static bool virtnet_fail_on_feature(struct virtio_device *vdev, 2702892d6eb1SJason Wang unsigned int fbit, 2703892d6eb1SJason Wang const char *fname, const char *dname) 2704892d6eb1SJason Wang { 2705892d6eb1SJason Wang if (!virtio_has_feature(vdev, fbit)) 2706892d6eb1SJason Wang return false; 2707892d6eb1SJason Wang 2708892d6eb1SJason Wang dev_err(&vdev->dev, "device advertises feature %s but not %s", 2709892d6eb1SJason Wang fname, dname); 2710892d6eb1SJason Wang 2711892d6eb1SJason Wang return true; 2712892d6eb1SJason Wang } 2713892d6eb1SJason Wang 2714892d6eb1SJason Wang #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 2715892d6eb1SJason Wang virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 2716892d6eb1SJason Wang 2717892d6eb1SJason Wang static bool virtnet_validate_features(struct virtio_device *vdev) 2718892d6eb1SJason Wang { 2719892d6eb1SJason Wang if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 2720892d6eb1SJason Wang (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 2721892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 2722892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 2723892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 2724892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 2725892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ") || 2726892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 2727892d6eb1SJason Wang VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 2728892d6eb1SJason Wang "VIRTIO_NET_F_CTRL_VQ"))) { 2729892d6eb1SJason Wang return false; 2730892d6eb1SJason Wang } 2731892d6eb1SJason Wang 2732892d6eb1SJason Wang return true; 2733892d6eb1SJason Wang } 2734892d6eb1SJason Wang 2735d0c2c997SJarod Wilson #define MIN_MTU ETH_MIN_MTU 2736d0c2c997SJarod Wilson #define MAX_MTU ETH_MAX_MTU 2737d0c2c997SJarod Wilson 2738fe36cbe0SMichael S. Tsirkin static int virtnet_validate(struct virtio_device *vdev) 2739296f96fcSRusty Russell { 27406ba42248SMichael S. Tsirkin if (!vdev->config->get) { 27416ba42248SMichael S. Tsirkin dev_err(&vdev->dev, "%s failure: config access disabled\n", 27426ba42248SMichael S. Tsirkin __func__); 27436ba42248SMichael S. Tsirkin return -EINVAL; 27446ba42248SMichael S. Tsirkin } 27456ba42248SMichael S. Tsirkin 2746892d6eb1SJason Wang if (!virtnet_validate_features(vdev)) 2747892d6eb1SJason Wang return -EINVAL; 2748892d6eb1SJason Wang 2749fe36cbe0SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 2750fe36cbe0SMichael S. Tsirkin int mtu = virtio_cread16(vdev, 2751fe36cbe0SMichael S. Tsirkin offsetof(struct virtio_net_config, 2752fe36cbe0SMichael S. Tsirkin mtu)); 2753fe36cbe0SMichael S. Tsirkin if (mtu < MIN_MTU) 2754fe36cbe0SMichael S. Tsirkin __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 2755fe36cbe0SMichael S. Tsirkin } 2756fe36cbe0SMichael S. Tsirkin 2757fe36cbe0SMichael S. Tsirkin return 0; 2758fe36cbe0SMichael S. Tsirkin } 2759fe36cbe0SMichael S. Tsirkin 2760fe36cbe0SMichael S. Tsirkin static int virtnet_probe(struct virtio_device *vdev) 2761fe36cbe0SMichael S. Tsirkin { 2762d7dfc5cfSToshiaki Makita int i, err = -ENOMEM; 2763fe36cbe0SMichael S. Tsirkin struct net_device *dev; 2764fe36cbe0SMichael S. Tsirkin struct virtnet_info *vi; 2765fe36cbe0SMichael S. Tsirkin u16 max_queue_pairs; 2766fe36cbe0SMichael S. Tsirkin int mtu; 2767fe36cbe0SMichael S. Tsirkin 2768986a4f4dSJason Wang /* Find if host supports multiqueue virtio_net device */ 2769855e0c52SRusty Russell err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ, 2770855e0c52SRusty Russell struct virtio_net_config, 2771855e0c52SRusty Russell max_virtqueue_pairs, &max_queue_pairs); 2772986a4f4dSJason Wang 2773986a4f4dSJason Wang /* We need at least 2 queue's */ 2774986a4f4dSJason Wang if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 2775986a4f4dSJason Wang max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 2776986a4f4dSJason Wang !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 2777986a4f4dSJason Wang max_queue_pairs = 1; 2778296f96fcSRusty Russell 2779296f96fcSRusty Russell /* Allocate ourselves a network device with room for our info */ 2780986a4f4dSJason Wang dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 2781296f96fcSRusty Russell if (!dev) 2782296f96fcSRusty Russell return -ENOMEM; 2783296f96fcSRusty Russell 2784296f96fcSRusty Russell /* Set up network device as normal. */ 2785f2f2c8b4SJiri Pirko dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; 278676288b4eSStephen Hemminger dev->netdev_ops = &virtnet_netdev; 2787296f96fcSRusty Russell dev->features = NETIF_F_HIGHDMA; 27883fa2a1dfSstephen hemminger 27897ad24ea4SWilfried Klaebe dev->ethtool_ops = &virtnet_ethtool_ops; 2790296f96fcSRusty Russell SET_NETDEV_DEV(dev, &vdev->dev); 2791296f96fcSRusty Russell 2792296f96fcSRusty Russell /* Do we support "hardware" checksums? */ 279398e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 2794296f96fcSRusty Russell /* This opens up the world of extra features. */ 279548900cb6SJason Wang dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 279698e778c9SMichał Mirosław if (csum) 279748900cb6SJason Wang dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 279898e778c9SMichał Mirosław 279998e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 2800e078de03SDavid S. Miller dev->hw_features |= NETIF_F_TSO 280134a48579SRusty Russell | NETIF_F_TSO_ECN | NETIF_F_TSO6; 280234a48579SRusty Russell } 28035539ae96SRusty Russell /* Individual feature bits: what can host handle? */ 280498e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 280598e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO; 280698e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 280798e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO6; 280898e778c9SMichał Mirosław if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 280998e778c9SMichał Mirosław dev->hw_features |= NETIF_F_TSO_ECN; 281098e778c9SMichał Mirosław 281141f2f127SJason Wang dev->features |= NETIF_F_GSO_ROBUST; 281241f2f127SJason Wang 281398e778c9SMichał Mirosław if (gso) 2814e078de03SDavid S. Miller dev->features |= dev->hw_features & NETIF_F_ALL_TSO; 281598e778c9SMichał Mirosław /* (!csum && gso) case will be fixed by register_netdev() */ 2816296f96fcSRusty Russell } 28174f49129bSThomas Huth if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM)) 28184f49129bSThomas Huth dev->features |= NETIF_F_RXCSUM; 2819296f96fcSRusty Russell 28204fda8302SJason Wang dev->vlan_features = dev->features; 28214fda8302SJason Wang 2822d0c2c997SJarod Wilson /* MTU range: 68 - 65535 */ 2823d0c2c997SJarod Wilson dev->min_mtu = MIN_MTU; 2824d0c2c997SJarod Wilson dev->max_mtu = MAX_MTU; 2825d0c2c997SJarod Wilson 2826296f96fcSRusty Russell /* Configuration may specify what MAC to use. Otherwise random. */ 2827855e0c52SRusty Russell if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) 2828855e0c52SRusty Russell virtio_cread_bytes(vdev, 2829a586d4f6SRusty Russell offsetof(struct virtio_net_config, mac), 2830855e0c52SRusty Russell dev->dev_addr, dev->addr_len); 2831855e0c52SRusty Russell else 2832f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 2833296f96fcSRusty Russell 2834296f96fcSRusty Russell /* Set up our device-specific information */ 2835296f96fcSRusty Russell vi = netdev_priv(dev); 2836296f96fcSRusty Russell vi->dev = dev; 2837296f96fcSRusty Russell vi->vdev = vdev; 2838d9d5dcc8SChristian Borntraeger vdev->priv = vi; 2839827da44cSJohn Stultz 2840586d17c5SJason Wang INIT_WORK(&vi->config_work, virtnet_config_changed_work); 2841296f96fcSRusty Russell 284297402b96SHerbert Xu /* If we can receive ANY GSO packets, we must allocate large ones. */ 28438e95a202SJoe Perches if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 28448e95a202SJoe Perches virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) || 2845e3e3c423SVlad Yasevich virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) || 2846e3e3c423SVlad Yasevich virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO)) 284797402b96SHerbert Xu vi->big_packets = true; 284897402b96SHerbert Xu 28493f2c31d9SMark McLoughlin if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) 28503f2c31d9SMark McLoughlin vi->mergeable_rx_bufs = true; 28513f2c31d9SMark McLoughlin 2852d04302b3SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 2853d04302b3SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 2854012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 2855012873d0SMichael S. Tsirkin else 2856012873d0SMichael S. Tsirkin vi->hdr_len = sizeof(struct virtio_net_hdr); 2857012873d0SMichael S. Tsirkin 285875993300SMichael S. Tsirkin if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 285975993300SMichael S. Tsirkin virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 2860e7428e95SMichael S. Tsirkin vi->any_header_sg = true; 2861e7428e95SMichael S. Tsirkin 2862986a4f4dSJason Wang if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 2863986a4f4dSJason Wang vi->has_cvq = true; 2864986a4f4dSJason Wang 286514de9d11SAaron Conole if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 286614de9d11SAaron Conole mtu = virtio_cread16(vdev, 286714de9d11SAaron Conole offsetof(struct virtio_net_config, 286814de9d11SAaron Conole mtu)); 286993a205eeSAaron Conole if (mtu < dev->min_mtu) { 2870fe36cbe0SMichael S. Tsirkin /* Should never trigger: MTU was previously validated 2871fe36cbe0SMichael S. Tsirkin * in virtnet_validate. 2872fe36cbe0SMichael S. Tsirkin */ 2873fe36cbe0SMichael S. Tsirkin dev_err(&vdev->dev, "device MTU appears to have changed " 2874fe36cbe0SMichael S. Tsirkin "it is now %d < %d", mtu, dev->min_mtu); 2875d7dfc5cfSToshiaki Makita goto free; 2876fe36cbe0SMichael S. Tsirkin } 2877fe36cbe0SMichael S. Tsirkin 2878d0c2c997SJarod Wilson dev->mtu = mtu; 287993a205eeSAaron Conole dev->max_mtu = mtu; 28802e123b44SMichael S. Tsirkin 28812e123b44SMichael S. Tsirkin /* TODO: size buffers correctly in this case. */ 28822e123b44SMichael S. Tsirkin if (dev->mtu > ETH_DATA_LEN) 28832e123b44SMichael S. Tsirkin vi->big_packets = true; 288414de9d11SAaron Conole } 288514de9d11SAaron Conole 2886012873d0SMichael S. Tsirkin if (vi->any_header_sg) 2887012873d0SMichael S. Tsirkin dev->needed_headroom = vi->hdr_len; 28886ebbc1a6SZhangjie \(HZ\) 288944900010SJason Wang /* Enable multiqueue by default */ 289044900010SJason Wang if (num_online_cpus() >= max_queue_pairs) 289144900010SJason Wang vi->curr_queue_pairs = max_queue_pairs; 289244900010SJason Wang else 289344900010SJason Wang vi->curr_queue_pairs = num_online_cpus(); 2894986a4f4dSJason Wang vi->max_queue_pairs = max_queue_pairs; 2895986a4f4dSJason Wang 2896986a4f4dSJason Wang /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 28973f9c10b0SAmit Shah err = init_vqs(vi); 2898d2a7dddaSMichael S. Tsirkin if (err) 2899d7dfc5cfSToshiaki Makita goto free; 2900d2a7dddaSMichael S. Tsirkin 2901fbf28d78SMichael Dalton #ifdef CONFIG_SYSFS 2902fbf28d78SMichael Dalton if (vi->mergeable_rx_bufs) 2903fbf28d78SMichael Dalton dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 2904fbf28d78SMichael Dalton #endif 29050f13b66bSZhi Yong Wu netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 29060f13b66bSZhi Yong Wu netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 2907986a4f4dSJason Wang 290816032be5SNikolay Aleksandrov virtnet_init_settings(dev); 290916032be5SNikolay Aleksandrov 2910296f96fcSRusty Russell err = register_netdev(dev); 2911296f96fcSRusty Russell if (err) { 2912296f96fcSRusty Russell pr_debug("virtio_net: registering device failed\n"); 2913d2a7dddaSMichael S. Tsirkin goto free_vqs; 2914296f96fcSRusty Russell } 2915b3369c1fSRusty Russell 29164baf1e33SMichael S. Tsirkin virtio_device_ready(vdev); 29174baf1e33SMichael S. Tsirkin 29188017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 29198de4b2f3SWanlong Gao if (err) { 29208de4b2f3SWanlong Gao pr_debug("virtio_net: registering cpu notifier failed\n"); 2921f00e35e2Swangyunjian goto free_unregister_netdev; 29228de4b2f3SWanlong Gao } 29238de4b2f3SWanlong Gao 2924a220871bSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 292544900010SJason Wang 2926167c25e4SJason Wang /* Assume link up if device can't report link status, 2927167c25e4SJason Wang otherwise get link status from config. */ 2928167c25e4SJason Wang netif_carrier_off(dev); 2929bda7fab5SJay Vosburgh if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 29303b07e9caSTejun Heo schedule_work(&vi->config_work); 2931167c25e4SJason Wang } else { 2932167c25e4SJason Wang vi->status = VIRTIO_NET_S_LINK_UP; 2933faa9b39fSJason Baron virtnet_update_settings(vi); 29344783256eSPantelis Koukousoulas netif_carrier_on(dev); 2935167c25e4SJason Wang } 29369f4d26d0SMark McLoughlin 29373f93522fSJason Wang for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) 29383f93522fSJason Wang if (virtio_has_feature(vi->vdev, guest_offloads[i])) 29393f93522fSJason Wang set_bit(guest_offloads[i], &vi->guest_offloads); 29403f93522fSJason Wang 2941986a4f4dSJason Wang pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 2942986a4f4dSJason Wang dev->name, max_queue_pairs); 2943986a4f4dSJason Wang 2944296f96fcSRusty Russell return 0; 2945296f96fcSRusty Russell 2946f00e35e2Swangyunjian free_unregister_netdev: 294702465555SMichael S. Tsirkin vi->vdev->config->reset(vdev); 294802465555SMichael S. Tsirkin 2949b3369c1fSRusty Russell unregister_netdev(dev); 2950d2a7dddaSMichael S. Tsirkin free_vqs: 2951986a4f4dSJason Wang cancel_delayed_work_sync(&vi->refill); 2952fb51879dSMichael Dalton free_receive_page_frags(vi); 2953e9d7417bSJason Wang virtnet_del_vqs(vi); 2954296f96fcSRusty Russell free: 2955296f96fcSRusty Russell free_netdev(dev); 2956296f96fcSRusty Russell return err; 2957296f96fcSRusty Russell } 2958296f96fcSRusty Russell 295904486ed0SAmit Shah static void remove_vq_common(struct virtnet_info *vi) 2960296f96fcSRusty Russell { 296104486ed0SAmit Shah vi->vdev->config->reset(vi->vdev); 2962830a8a97SShirley Ma 2963830a8a97SShirley Ma /* Free unused buffers in both send and recv, if any. */ 29649ab86bbcSShirley Ma free_unused_bufs(vi); 2965fb6813f4SRusty Russell 2966986a4f4dSJason Wang free_receive_bufs(vi); 2967d2a7dddaSMichael S. Tsirkin 2968fb51879dSMichael Dalton free_receive_page_frags(vi); 2969fb51879dSMichael Dalton 2970986a4f4dSJason Wang virtnet_del_vqs(vi); 297104486ed0SAmit Shah } 297204486ed0SAmit Shah 29738cc085d6SBill Pemberton static void virtnet_remove(struct virtio_device *vdev) 297404486ed0SAmit Shah { 297504486ed0SAmit Shah struct virtnet_info *vi = vdev->priv; 297604486ed0SAmit Shah 29778017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 29788de4b2f3SWanlong Gao 2979102a2786SMichael S. Tsirkin /* Make sure no work handler is accessing the device. */ 2980102a2786SMichael S. Tsirkin flush_work(&vi->config_work); 2981586d17c5SJason Wang 298204486ed0SAmit Shah unregister_netdev(vi->dev); 298304486ed0SAmit Shah 298404486ed0SAmit Shah remove_vq_common(vi); 2985fb6813f4SRusty Russell 298674b2553fSRusty Russell free_netdev(vi->dev); 2987296f96fcSRusty Russell } 2988296f96fcSRusty Russell 298967a75194SArnd Bergmann static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 29900741bcb5SAmit Shah { 29910741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 29920741bcb5SAmit Shah 29938017c279SSebastian Andrzej Siewior virtnet_cpu_notif_remove(vi); 29949fe7bfceSJohn Fastabend virtnet_freeze_down(vdev); 29950741bcb5SAmit Shah remove_vq_common(vi); 29960741bcb5SAmit Shah 29970741bcb5SAmit Shah return 0; 29980741bcb5SAmit Shah } 29990741bcb5SAmit Shah 300067a75194SArnd Bergmann static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 30010741bcb5SAmit Shah { 30020741bcb5SAmit Shah struct virtnet_info *vi = vdev->priv; 30039fe7bfceSJohn Fastabend int err; 30040741bcb5SAmit Shah 30059fe7bfceSJohn Fastabend err = virtnet_restore_up(vdev); 30060741bcb5SAmit Shah if (err) 30070741bcb5SAmit Shah return err; 3008986a4f4dSJason Wang virtnet_set_queues(vi, vi->curr_queue_pairs); 3009986a4f4dSJason Wang 30108017c279SSebastian Andrzej Siewior err = virtnet_cpu_notif_add(vi); 3011ec9debbdSJason Wang if (err) 3012ec9debbdSJason Wang return err; 3013ec9debbdSJason Wang 30140741bcb5SAmit Shah return 0; 30150741bcb5SAmit Shah } 30160741bcb5SAmit Shah 3017296f96fcSRusty Russell static struct virtio_device_id id_table[] = { 3018296f96fcSRusty Russell { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 3019296f96fcSRusty Russell { 0 }, 3020296f96fcSRusty Russell }; 3021296f96fcSRusty Russell 3022f3358507SMichael S. Tsirkin #define VIRTNET_FEATURES \ 3023f3358507SMichael S. Tsirkin VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 3024f3358507SMichael S. Tsirkin VIRTIO_NET_F_MAC, \ 3025f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 3026f3358507SMichael S. Tsirkin VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 3027f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 3028f3358507SMichael S. Tsirkin VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 3029f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 3030f3358507SMichael S. Tsirkin VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 3031f3358507SMichael S. Tsirkin VIRTIO_NET_F_CTRL_MAC_ADDR, \ 3032faa9b39fSJason Baron VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 30339805069dSSridhar Samudrala VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY 3034f3358507SMichael S. Tsirkin 3035c45a6816SRusty Russell static unsigned int features[] = { 3036f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 3037f3358507SMichael S. Tsirkin }; 3038f3358507SMichael S. Tsirkin 3039f3358507SMichael S. Tsirkin static unsigned int features_legacy[] = { 3040f3358507SMichael S. Tsirkin VIRTNET_FEATURES, 3041f3358507SMichael S. Tsirkin VIRTIO_NET_F_GSO, 3042e7428e95SMichael S. Tsirkin VIRTIO_F_ANY_LAYOUT, 3043c45a6816SRusty Russell }; 3044c45a6816SRusty Russell 304522402529SUwe Kleine-König static struct virtio_driver virtio_net_driver = { 3046c45a6816SRusty Russell .feature_table = features, 3047c45a6816SRusty Russell .feature_table_size = ARRAY_SIZE(features), 3048f3358507SMichael S. Tsirkin .feature_table_legacy = features_legacy, 3049f3358507SMichael S. Tsirkin .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 3050296f96fcSRusty Russell .driver.name = KBUILD_MODNAME, 3051296f96fcSRusty Russell .driver.owner = THIS_MODULE, 3052296f96fcSRusty Russell .id_table = id_table, 3053fe36cbe0SMichael S. Tsirkin .validate = virtnet_validate, 3054296f96fcSRusty Russell .probe = virtnet_probe, 30558cc085d6SBill Pemberton .remove = virtnet_remove, 30569f4d26d0SMark McLoughlin .config_changed = virtnet_config_changed, 305789107000SAaron Lu #ifdef CONFIG_PM_SLEEP 30580741bcb5SAmit Shah .freeze = virtnet_freeze, 30590741bcb5SAmit Shah .restore = virtnet_restore, 30600741bcb5SAmit Shah #endif 3061296f96fcSRusty Russell }; 3062296f96fcSRusty Russell 30638017c279SSebastian Andrzej Siewior static __init int virtio_net_driver_init(void) 30648017c279SSebastian Andrzej Siewior { 30658017c279SSebastian Andrzej Siewior int ret; 30668017c279SSebastian Andrzej Siewior 306773c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 30688017c279SSebastian Andrzej Siewior virtnet_cpu_online, 30698017c279SSebastian Andrzej Siewior virtnet_cpu_down_prep); 30708017c279SSebastian Andrzej Siewior if (ret < 0) 30718017c279SSebastian Andrzej Siewior goto out; 30728017c279SSebastian Andrzej Siewior virtionet_online = ret; 307373c1b41eSThomas Gleixner ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 30748017c279SSebastian Andrzej Siewior NULL, virtnet_cpu_dead); 30758017c279SSebastian Andrzej Siewior if (ret) 30768017c279SSebastian Andrzej Siewior goto err_dead; 30778017c279SSebastian Andrzej Siewior 30788017c279SSebastian Andrzej Siewior ret = register_virtio_driver(&virtio_net_driver); 30798017c279SSebastian Andrzej Siewior if (ret) 30808017c279SSebastian Andrzej Siewior goto err_virtio; 30818017c279SSebastian Andrzej Siewior return 0; 30828017c279SSebastian Andrzej Siewior err_virtio: 30838017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 30848017c279SSebastian Andrzej Siewior err_dead: 30858017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 30868017c279SSebastian Andrzej Siewior out: 30878017c279SSebastian Andrzej Siewior return ret; 30888017c279SSebastian Andrzej Siewior } 30898017c279SSebastian Andrzej Siewior module_init(virtio_net_driver_init); 30908017c279SSebastian Andrzej Siewior 30918017c279SSebastian Andrzej Siewior static __exit void virtio_net_driver_exit(void) 30928017c279SSebastian Andrzej Siewior { 3093cfa0ebc9SAndrew Jones unregister_virtio_driver(&virtio_net_driver); 30948017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 30958017c279SSebastian Andrzej Siewior cpuhp_remove_multi_state(virtionet_online); 30968017c279SSebastian Andrzej Siewior } 30978017c279SSebastian Andrzej Siewior module_exit(virtio_net_driver_exit); 3098296f96fcSRusty Russell 3099296f96fcSRusty Russell MODULE_DEVICE_TABLE(virtio, id_table); 3100296f96fcSRusty Russell MODULE_DESCRIPTION("Virtio network driver"); 3101296f96fcSRusty Russell MODULE_LICENSE("GPL"); 3102