xref: /openbmc/linux/drivers/net/virtio_net.c (revision e330fb14)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* A network driver using virtio.
3  *
4  * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5  */
6 //#define DEBUG
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/ethtool.h>
10 #include <linux/module.h>
11 #include <linux/virtio.h>
12 #include <linux/virtio_net.h>
13 #include <linux/bpf.h>
14 #include <linux/bpf_trace.h>
15 #include <linux/scatterlist.h>
16 #include <linux/if_vlan.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/average.h>
20 #include <linux/filter.h>
21 #include <linux/kernel.h>
22 #include <net/route.h>
23 #include <net/xdp.h>
24 #include <net/net_failover.h>
25 
26 static int napi_weight = NAPI_POLL_WEIGHT;
27 module_param(napi_weight, int, 0444);
28 
29 static bool csum = true, gso = true, napi_tx = true;
30 module_param(csum, bool, 0444);
31 module_param(gso, bool, 0444);
32 module_param(napi_tx, bool, 0644);
33 
34 /* FIXME: MTU in config. */
35 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
36 #define GOOD_COPY_LEN	128
37 
38 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
39 
40 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
41 #define VIRTIO_XDP_HEADROOM 256
42 
43 /* Separating two types of XDP xmit */
44 #define VIRTIO_XDP_TX		BIT(0)
45 #define VIRTIO_XDP_REDIR	BIT(1)
46 
47 #define VIRTIO_XDP_FLAG	BIT(0)
48 
49 /* RX packet size EWMA. The average packet size is used to determine the packet
50  * buffer size when refilling RX rings. As the entire RX ring may be refilled
51  * at once, the weight is chosen so that the EWMA will be insensitive to short-
52  * term, transient changes in packet size.
53  */
54 DECLARE_EWMA(pkt_len, 0, 64)
55 
56 #define VIRTNET_DRIVER_VERSION "1.0.0"
57 
58 static const unsigned long guest_offloads[] = {
59 	VIRTIO_NET_F_GUEST_TSO4,
60 	VIRTIO_NET_F_GUEST_TSO6,
61 	VIRTIO_NET_F_GUEST_ECN,
62 	VIRTIO_NET_F_GUEST_UFO,
63 	VIRTIO_NET_F_GUEST_CSUM
64 };
65 
66 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
67 				(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
68 				(1ULL << VIRTIO_NET_F_GUEST_ECN)  | \
69 				(1ULL << VIRTIO_NET_F_GUEST_UFO))
70 
71 struct virtnet_stat_desc {
72 	char desc[ETH_GSTRING_LEN];
73 	size_t offset;
74 };
75 
76 struct virtnet_sq_stats {
77 	struct u64_stats_sync syncp;
78 	u64 packets;
79 	u64 bytes;
80 	u64 xdp_tx;
81 	u64 xdp_tx_drops;
82 	u64 kicks;
83 	u64 tx_timeouts;
84 };
85 
86 struct virtnet_rq_stats {
87 	struct u64_stats_sync syncp;
88 	u64 packets;
89 	u64 bytes;
90 	u64 drops;
91 	u64 xdp_packets;
92 	u64 xdp_tx;
93 	u64 xdp_redirects;
94 	u64 xdp_drops;
95 	u64 kicks;
96 };
97 
98 #define VIRTNET_SQ_STAT(m)	offsetof(struct virtnet_sq_stats, m)
99 #define VIRTNET_RQ_STAT(m)	offsetof(struct virtnet_rq_stats, m)
100 
101 static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
102 	{ "packets",		VIRTNET_SQ_STAT(packets) },
103 	{ "bytes",		VIRTNET_SQ_STAT(bytes) },
104 	{ "xdp_tx",		VIRTNET_SQ_STAT(xdp_tx) },
105 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
106 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
107 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
108 };
109 
110 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
111 	{ "packets",		VIRTNET_RQ_STAT(packets) },
112 	{ "bytes",		VIRTNET_RQ_STAT(bytes) },
113 	{ "drops",		VIRTNET_RQ_STAT(drops) },
114 	{ "xdp_packets",	VIRTNET_RQ_STAT(xdp_packets) },
115 	{ "xdp_tx",		VIRTNET_RQ_STAT(xdp_tx) },
116 	{ "xdp_redirects",	VIRTNET_RQ_STAT(xdp_redirects) },
117 	{ "xdp_drops",		VIRTNET_RQ_STAT(xdp_drops) },
118 	{ "kicks",		VIRTNET_RQ_STAT(kicks) },
119 };
120 
121 #define VIRTNET_SQ_STATS_LEN	ARRAY_SIZE(virtnet_sq_stats_desc)
122 #define VIRTNET_RQ_STATS_LEN	ARRAY_SIZE(virtnet_rq_stats_desc)
123 
124 /* Internal representation of a send virtqueue */
125 struct send_queue {
126 	/* Virtqueue associated with this send _queue */
127 	struct virtqueue *vq;
128 
129 	/* TX: fragments + linear part + virtio header */
130 	struct scatterlist sg[MAX_SKB_FRAGS + 2];
131 
132 	/* Name of the send queue: output.$index */
133 	char name[40];
134 
135 	struct virtnet_sq_stats stats;
136 
137 	struct napi_struct napi;
138 };
139 
140 /* Internal representation of a receive virtqueue */
141 struct receive_queue {
142 	/* Virtqueue associated with this receive_queue */
143 	struct virtqueue *vq;
144 
145 	struct napi_struct napi;
146 
147 	struct bpf_prog __rcu *xdp_prog;
148 
149 	struct virtnet_rq_stats stats;
150 
151 	/* Chain pages by the private ptr. */
152 	struct page *pages;
153 
154 	/* Average packet length for mergeable receive buffers. */
155 	struct ewma_pkt_len mrg_avg_pkt_len;
156 
157 	/* Page frag for packet buffer allocation. */
158 	struct page_frag alloc_frag;
159 
160 	/* RX: fragments + linear part + virtio header */
161 	struct scatterlist sg[MAX_SKB_FRAGS + 2];
162 
163 	/* Min single buffer size for mergeable buffers case. */
164 	unsigned int min_buf_len;
165 
166 	/* Name of this receive queue: input.$index */
167 	char name[40];
168 
169 	struct xdp_rxq_info xdp_rxq;
170 };
171 
172 /* Control VQ buffers: protected by the rtnl lock */
173 struct control_buf {
174 	struct virtio_net_ctrl_hdr hdr;
175 	virtio_net_ctrl_ack status;
176 	struct virtio_net_ctrl_mq mq;
177 	u8 promisc;
178 	u8 allmulti;
179 	__virtio16 vid;
180 	__virtio64 offloads;
181 };
182 
183 struct virtnet_info {
184 	struct virtio_device *vdev;
185 	struct virtqueue *cvq;
186 	struct net_device *dev;
187 	struct send_queue *sq;
188 	struct receive_queue *rq;
189 	unsigned int status;
190 
191 	/* Max # of queue pairs supported by the device */
192 	u16 max_queue_pairs;
193 
194 	/* # of queue pairs currently used by the driver */
195 	u16 curr_queue_pairs;
196 
197 	/* # of XDP queue pairs currently used by the driver */
198 	u16 xdp_queue_pairs;
199 
200 	/* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
201 	bool xdp_enabled;
202 
203 	/* I like... big packets and I cannot lie! */
204 	bool big_packets;
205 
206 	/* Host will merge rx buffers for big packets (shake it! shake it!) */
207 	bool mergeable_rx_bufs;
208 
209 	/* Has control virtqueue */
210 	bool has_cvq;
211 
212 	/* Host can handle any s/g split between our header and packet data */
213 	bool any_header_sg;
214 
215 	/* Packet virtio header size */
216 	u8 hdr_len;
217 
218 	/* Work struct for refilling if we run low on memory. */
219 	struct delayed_work refill;
220 
221 	/* Work struct for config space updates */
222 	struct work_struct config_work;
223 
224 	/* Does the affinity hint is set for virtqueues? */
225 	bool affinity_hint_set;
226 
227 	/* CPU hotplug instances for online & dead */
228 	struct hlist_node node;
229 	struct hlist_node node_dead;
230 
231 	struct control_buf *ctrl;
232 
233 	/* Ethtool settings */
234 	u8 duplex;
235 	u32 speed;
236 
237 	unsigned long guest_offloads;
238 	unsigned long guest_offloads_capable;
239 
240 	/* failover when STANDBY feature enabled */
241 	struct failover *failover;
242 };
243 
244 struct padded_vnet_hdr {
245 	struct virtio_net_hdr_mrg_rxbuf hdr;
246 	/*
247 	 * hdr is in a separate sg buffer, and data sg buffer shares same page
248 	 * with this header sg. This padding makes next sg 16 byte aligned
249 	 * after the header.
250 	 */
251 	char padding[4];
252 };
253 
254 static bool is_xdp_frame(void *ptr)
255 {
256 	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
257 }
258 
259 static void *xdp_to_ptr(struct xdp_frame *ptr)
260 {
261 	return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
262 }
263 
264 static struct xdp_frame *ptr_to_xdp(void *ptr)
265 {
266 	return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
267 }
268 
269 /* Converting between virtqueue no. and kernel tx/rx queue no.
270  * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
271  */
272 static int vq2txq(struct virtqueue *vq)
273 {
274 	return (vq->index - 1) / 2;
275 }
276 
277 static int txq2vq(int txq)
278 {
279 	return txq * 2 + 1;
280 }
281 
282 static int vq2rxq(struct virtqueue *vq)
283 {
284 	return vq->index / 2;
285 }
286 
287 static int rxq2vq(int rxq)
288 {
289 	return rxq * 2;
290 }
291 
292 static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
293 {
294 	return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
295 }
296 
297 /*
298  * private is used to chain pages for big packets, put the whole
299  * most recent used list in the beginning for reuse
300  */
301 static void give_pages(struct receive_queue *rq, struct page *page)
302 {
303 	struct page *end;
304 
305 	/* Find end of list, sew whole thing into vi->rq.pages. */
306 	for (end = page; end->private; end = (struct page *)end->private);
307 	end->private = (unsigned long)rq->pages;
308 	rq->pages = page;
309 }
310 
311 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
312 {
313 	struct page *p = rq->pages;
314 
315 	if (p) {
316 		rq->pages = (struct page *)p->private;
317 		/* clear private here, it is used to chain pages */
318 		p->private = 0;
319 	} else
320 		p = alloc_page(gfp_mask);
321 	return p;
322 }
323 
324 static void virtqueue_napi_schedule(struct napi_struct *napi,
325 				    struct virtqueue *vq)
326 {
327 	if (napi_schedule_prep(napi)) {
328 		virtqueue_disable_cb(vq);
329 		__napi_schedule(napi);
330 	}
331 }
332 
333 static void virtqueue_napi_complete(struct napi_struct *napi,
334 				    struct virtqueue *vq, int processed)
335 {
336 	int opaque;
337 
338 	opaque = virtqueue_enable_cb_prepare(vq);
339 	if (napi_complete_done(napi, processed)) {
340 		if (unlikely(virtqueue_poll(vq, opaque)))
341 			virtqueue_napi_schedule(napi, vq);
342 	} else {
343 		virtqueue_disable_cb(vq);
344 	}
345 }
346 
347 static void skb_xmit_done(struct virtqueue *vq)
348 {
349 	struct virtnet_info *vi = vq->vdev->priv;
350 	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
351 
352 	/* Suppress further interrupts. */
353 	virtqueue_disable_cb(vq);
354 
355 	if (napi->weight)
356 		virtqueue_napi_schedule(napi, vq);
357 	else
358 		/* We were probably waiting for more output buffers. */
359 		netif_wake_subqueue(vi->dev, vq2txq(vq));
360 }
361 
362 #define MRG_CTX_HEADER_SHIFT 22
363 static void *mergeable_len_to_ctx(unsigned int truesize,
364 				  unsigned int headroom)
365 {
366 	return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
367 }
368 
369 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
370 {
371 	return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
372 }
373 
374 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
375 {
376 	return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
377 }
378 
379 /* Called from bottom half context */
380 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
381 				   struct receive_queue *rq,
382 				   struct page *page, unsigned int offset,
383 				   unsigned int len, unsigned int truesize,
384 				   bool hdr_valid, unsigned int metasize,
385 				   unsigned int headroom)
386 {
387 	struct sk_buff *skb;
388 	struct virtio_net_hdr_mrg_rxbuf *hdr;
389 	unsigned int copy, hdr_len, hdr_padded_len;
390 	struct page *page_to_free = NULL;
391 	int tailroom, shinfo_size;
392 	char *p, *hdr_p, *buf;
393 
394 	p = page_address(page) + offset;
395 	hdr_p = p;
396 
397 	hdr_len = vi->hdr_len;
398 	if (vi->mergeable_rx_bufs)
399 		hdr_padded_len = sizeof(*hdr);
400 	else
401 		hdr_padded_len = sizeof(struct padded_vnet_hdr);
402 
403 	/* If headroom is not 0, there is an offset between the beginning of the
404 	 * data and the allocated space, otherwise the data and the allocated
405 	 * space are aligned.
406 	 *
407 	 * Buffers with headroom use PAGE_SIZE as alloc size, see
408 	 * add_recvbuf_mergeable() + get_mergeable_buf_len()
409 	 */
410 	truesize = headroom ? PAGE_SIZE : truesize;
411 	tailroom = truesize - len - headroom;
412 	buf = p - headroom;
413 
414 	len -= hdr_len;
415 	offset += hdr_padded_len;
416 	p += hdr_padded_len;
417 
418 	shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
419 
420 	/* copy small packet so we can reuse these pages */
421 	if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
422 		skb = build_skb(buf, truesize);
423 		if (unlikely(!skb))
424 			return NULL;
425 
426 		skb_reserve(skb, p - buf);
427 		skb_put(skb, len);
428 
429 		page = (struct page *)page->private;
430 		if (page)
431 			give_pages(rq, page);
432 		goto ok;
433 	}
434 
435 	/* copy small packet so we can reuse these pages for small data */
436 	skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
437 	if (unlikely(!skb))
438 		return NULL;
439 
440 	/* Copy all frame if it fits skb->head, otherwise
441 	 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
442 	 */
443 	if (len <= skb_tailroom(skb))
444 		copy = len;
445 	else
446 		copy = ETH_HLEN + metasize;
447 	skb_put_data(skb, p, copy);
448 
449 	len -= copy;
450 	offset += copy;
451 
452 	if (vi->mergeable_rx_bufs) {
453 		if (len)
454 			skb_add_rx_frag(skb, 0, page, offset, len, truesize);
455 		else
456 			page_to_free = page;
457 		goto ok;
458 	}
459 
460 	/*
461 	 * Verify that we can indeed put this data into a skb.
462 	 * This is here to handle cases when the device erroneously
463 	 * tries to receive more than is possible. This is usually
464 	 * the case of a broken device.
465 	 */
466 	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
467 		net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
468 		dev_kfree_skb(skb);
469 		return NULL;
470 	}
471 	BUG_ON(offset >= PAGE_SIZE);
472 	while (len) {
473 		unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
474 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
475 				frag_size, truesize);
476 		len -= frag_size;
477 		page = (struct page *)page->private;
478 		offset = 0;
479 	}
480 
481 	if (page)
482 		give_pages(rq, page);
483 
484 ok:
485 	/* hdr_valid means no XDP, so we can copy the vnet header */
486 	if (hdr_valid) {
487 		hdr = skb_vnet_hdr(skb);
488 		memcpy(hdr, hdr_p, hdr_len);
489 	}
490 	if (page_to_free)
491 		put_page(page_to_free);
492 
493 	if (metasize) {
494 		__skb_pull(skb, metasize);
495 		skb_metadata_set(skb, metasize);
496 	}
497 
498 	return skb;
499 }
500 
501 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
502 				   struct send_queue *sq,
503 				   struct xdp_frame *xdpf)
504 {
505 	struct virtio_net_hdr_mrg_rxbuf *hdr;
506 	int err;
507 
508 	if (unlikely(xdpf->headroom < vi->hdr_len))
509 		return -EOVERFLOW;
510 
511 	/* Make room for virtqueue hdr (also change xdpf->headroom?) */
512 	xdpf->data -= vi->hdr_len;
513 	/* Zero header and leave csum up to XDP layers */
514 	hdr = xdpf->data;
515 	memset(hdr, 0, vi->hdr_len);
516 	xdpf->len   += vi->hdr_len;
517 
518 	sg_init_one(sq->sg, xdpf->data, xdpf->len);
519 
520 	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf),
521 				   GFP_ATOMIC);
522 	if (unlikely(err))
523 		return -ENOSPC; /* Caller handle free/refcnt */
524 
525 	return 0;
526 }
527 
528 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
529  * the current cpu, so it does not need to be locked.
530  *
531  * Here we use marco instead of inline functions because we have to deal with
532  * three issues at the same time: 1. the choice of sq. 2. judge and execute the
533  * lock/unlock of txq 3. make sparse happy. It is difficult for two inline
534  * functions to perfectly solve these three problems at the same time.
535  */
536 #define virtnet_xdp_get_sq(vi) ({                                       \
537 	int cpu = smp_processor_id();                                   \
538 	struct netdev_queue *txq;                                       \
539 	typeof(vi) v = (vi);                                            \
540 	unsigned int qp;                                                \
541 									\
542 	if (v->curr_queue_pairs > nr_cpu_ids) {                         \
543 		qp = v->curr_queue_pairs - v->xdp_queue_pairs;          \
544 		qp += cpu;                                              \
545 		txq = netdev_get_tx_queue(v->dev, qp);                  \
546 		__netif_tx_acquire(txq);                                \
547 	} else {                                                        \
548 		qp = cpu % v->curr_queue_pairs;                         \
549 		txq = netdev_get_tx_queue(v->dev, qp);                  \
550 		__netif_tx_lock(txq, cpu);                              \
551 	}                                                               \
552 	v->sq + qp;                                                     \
553 })
554 
555 #define virtnet_xdp_put_sq(vi, q) {                                     \
556 	struct netdev_queue *txq;                                       \
557 	typeof(vi) v = (vi);                                            \
558 									\
559 	txq = netdev_get_tx_queue(v->dev, (q) - v->sq);                 \
560 	if (v->curr_queue_pairs > nr_cpu_ids)                           \
561 		__netif_tx_release(txq);                                \
562 	else                                                            \
563 		__netif_tx_unlock(txq);                                 \
564 }
565 
566 static int virtnet_xdp_xmit(struct net_device *dev,
567 			    int n, struct xdp_frame **frames, u32 flags)
568 {
569 	struct virtnet_info *vi = netdev_priv(dev);
570 	struct receive_queue *rq = vi->rq;
571 	struct bpf_prog *xdp_prog;
572 	struct send_queue *sq;
573 	unsigned int len;
574 	int packets = 0;
575 	int bytes = 0;
576 	int nxmit = 0;
577 	int kicks = 0;
578 	void *ptr;
579 	int ret;
580 	int i;
581 
582 	/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
583 	 * indicate XDP resources have been successfully allocated.
584 	 */
585 	xdp_prog = rcu_access_pointer(rq->xdp_prog);
586 	if (!xdp_prog)
587 		return -ENXIO;
588 
589 	sq = virtnet_xdp_get_sq(vi);
590 
591 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
592 		ret = -EINVAL;
593 		goto out;
594 	}
595 
596 	/* Free up any pending old buffers before queueing new ones. */
597 	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
598 		if (likely(is_xdp_frame(ptr))) {
599 			struct xdp_frame *frame = ptr_to_xdp(ptr);
600 
601 			bytes += frame->len;
602 			xdp_return_frame(frame);
603 		} else {
604 			struct sk_buff *skb = ptr;
605 
606 			bytes += skb->len;
607 			napi_consume_skb(skb, false);
608 		}
609 		packets++;
610 	}
611 
612 	for (i = 0; i < n; i++) {
613 		struct xdp_frame *xdpf = frames[i];
614 
615 		if (__virtnet_xdp_xmit_one(vi, sq, xdpf))
616 			break;
617 		nxmit++;
618 	}
619 	ret = nxmit;
620 
621 	if (flags & XDP_XMIT_FLUSH) {
622 		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
623 			kicks = 1;
624 	}
625 out:
626 	u64_stats_update_begin(&sq->stats.syncp);
627 	sq->stats.bytes += bytes;
628 	sq->stats.packets += packets;
629 	sq->stats.xdp_tx += n;
630 	sq->stats.xdp_tx_drops += n - nxmit;
631 	sq->stats.kicks += kicks;
632 	u64_stats_update_end(&sq->stats.syncp);
633 
634 	virtnet_xdp_put_sq(vi, sq);
635 	return ret;
636 }
637 
638 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
639 {
640 	return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0;
641 }
642 
643 /* We copy the packet for XDP in the following cases:
644  *
645  * 1) Packet is scattered across multiple rx buffers.
646  * 2) Headroom space is insufficient.
647  *
648  * This is inefficient but it's a temporary condition that
649  * we hit right after XDP is enabled and until queue is refilled
650  * with large buffers with sufficient headroom - so it should affect
651  * at most queue size packets.
652  * Afterwards, the conditions to enable
653  * XDP should preclude the underlying device from sending packets
654  * across multiple buffers (num_buf > 1), and we make sure buffers
655  * have enough headroom.
656  */
657 static struct page *xdp_linearize_page(struct receive_queue *rq,
658 				       u16 *num_buf,
659 				       struct page *p,
660 				       int offset,
661 				       int page_off,
662 				       unsigned int *len)
663 {
664 	struct page *page = alloc_page(GFP_ATOMIC);
665 
666 	if (!page)
667 		return NULL;
668 
669 	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
670 	page_off += *len;
671 
672 	while (--*num_buf) {
673 		int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
674 		unsigned int buflen;
675 		void *buf;
676 		int off;
677 
678 		buf = virtqueue_get_buf(rq->vq, &buflen);
679 		if (unlikely(!buf))
680 			goto err_buf;
681 
682 		p = virt_to_head_page(buf);
683 		off = buf - page_address(p);
684 
685 		/* guard against a misconfigured or uncooperative backend that
686 		 * is sending packet larger than the MTU.
687 		 */
688 		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
689 			put_page(p);
690 			goto err_buf;
691 		}
692 
693 		memcpy(page_address(page) + page_off,
694 		       page_address(p) + off, buflen);
695 		page_off += buflen;
696 		put_page(p);
697 	}
698 
699 	/* Headroom does not contribute to packet length */
700 	*len = page_off - VIRTIO_XDP_HEADROOM;
701 	return page;
702 err_buf:
703 	__free_pages(page, 0);
704 	return NULL;
705 }
706 
707 static struct sk_buff *receive_small(struct net_device *dev,
708 				     struct virtnet_info *vi,
709 				     struct receive_queue *rq,
710 				     void *buf, void *ctx,
711 				     unsigned int len,
712 				     unsigned int *xdp_xmit,
713 				     struct virtnet_rq_stats *stats)
714 {
715 	struct sk_buff *skb;
716 	struct bpf_prog *xdp_prog;
717 	unsigned int xdp_headroom = (unsigned long)ctx;
718 	unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
719 	unsigned int headroom = vi->hdr_len + header_offset;
720 	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
721 			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
722 	struct page *page = virt_to_head_page(buf);
723 	unsigned int delta = 0;
724 	struct page *xdp_page;
725 	int err;
726 	unsigned int metasize = 0;
727 
728 	len -= vi->hdr_len;
729 	stats->bytes += len;
730 
731 	if (unlikely(len > GOOD_PACKET_LEN)) {
732 		pr_debug("%s: rx error: len %u exceeds max size %d\n",
733 			 dev->name, len, GOOD_PACKET_LEN);
734 		dev->stats.rx_length_errors++;
735 		goto err_len;
736 	}
737 	rcu_read_lock();
738 	xdp_prog = rcu_dereference(rq->xdp_prog);
739 	if (xdp_prog) {
740 		struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
741 		struct xdp_frame *xdpf;
742 		struct xdp_buff xdp;
743 		void *orig_data;
744 		u32 act;
745 
746 		if (unlikely(hdr->hdr.gso_type))
747 			goto err_xdp;
748 
749 		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
750 			int offset = buf - page_address(page) + header_offset;
751 			unsigned int tlen = len + vi->hdr_len;
752 			u16 num_buf = 1;
753 
754 			xdp_headroom = virtnet_get_headroom(vi);
755 			header_offset = VIRTNET_RX_PAD + xdp_headroom;
756 			headroom = vi->hdr_len + header_offset;
757 			buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
758 				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
759 			xdp_page = xdp_linearize_page(rq, &num_buf, page,
760 						      offset, header_offset,
761 						      &tlen);
762 			if (!xdp_page)
763 				goto err_xdp;
764 
765 			buf = page_address(xdp_page);
766 			put_page(page);
767 			page = xdp_page;
768 		}
769 
770 		xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
771 		xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
772 				 xdp_headroom, len, true);
773 		orig_data = xdp.data;
774 		act = bpf_prog_run_xdp(xdp_prog, &xdp);
775 		stats->xdp_packets++;
776 
777 		switch (act) {
778 		case XDP_PASS:
779 			/* Recalculate length in case bpf program changed it */
780 			delta = orig_data - xdp.data;
781 			len = xdp.data_end - xdp.data;
782 			metasize = xdp.data - xdp.data_meta;
783 			break;
784 		case XDP_TX:
785 			stats->xdp_tx++;
786 			xdpf = xdp_convert_buff_to_frame(&xdp);
787 			if (unlikely(!xdpf))
788 				goto err_xdp;
789 			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
790 			if (unlikely(!err)) {
791 				xdp_return_frame_rx_napi(xdpf);
792 			} else if (unlikely(err < 0)) {
793 				trace_xdp_exception(vi->dev, xdp_prog, act);
794 				goto err_xdp;
795 			}
796 			*xdp_xmit |= VIRTIO_XDP_TX;
797 			rcu_read_unlock();
798 			goto xdp_xmit;
799 		case XDP_REDIRECT:
800 			stats->xdp_redirects++;
801 			err = xdp_do_redirect(dev, &xdp, xdp_prog);
802 			if (err)
803 				goto err_xdp;
804 			*xdp_xmit |= VIRTIO_XDP_REDIR;
805 			rcu_read_unlock();
806 			goto xdp_xmit;
807 		default:
808 			bpf_warn_invalid_xdp_action(act);
809 			fallthrough;
810 		case XDP_ABORTED:
811 			trace_xdp_exception(vi->dev, xdp_prog, act);
812 			goto err_xdp;
813 		case XDP_DROP:
814 			goto err_xdp;
815 		}
816 	}
817 	rcu_read_unlock();
818 
819 	skb = build_skb(buf, buflen);
820 	if (!skb) {
821 		put_page(page);
822 		goto err;
823 	}
824 	skb_reserve(skb, headroom - delta);
825 	skb_put(skb, len);
826 	if (!xdp_prog) {
827 		buf += header_offset;
828 		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
829 	} /* keep zeroed vnet hdr since XDP is loaded */
830 
831 	if (metasize)
832 		skb_metadata_set(skb, metasize);
833 
834 err:
835 	return skb;
836 
837 err_xdp:
838 	rcu_read_unlock();
839 	stats->xdp_drops++;
840 err_len:
841 	stats->drops++;
842 	put_page(page);
843 xdp_xmit:
844 	return NULL;
845 }
846 
847 static struct sk_buff *receive_big(struct net_device *dev,
848 				   struct virtnet_info *vi,
849 				   struct receive_queue *rq,
850 				   void *buf,
851 				   unsigned int len,
852 				   struct virtnet_rq_stats *stats)
853 {
854 	struct page *page = buf;
855 	struct sk_buff *skb =
856 		page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0);
857 
858 	stats->bytes += len - vi->hdr_len;
859 	if (unlikely(!skb))
860 		goto err;
861 
862 	return skb;
863 
864 err:
865 	stats->drops++;
866 	give_pages(rq, page);
867 	return NULL;
868 }
869 
870 static struct sk_buff *receive_mergeable(struct net_device *dev,
871 					 struct virtnet_info *vi,
872 					 struct receive_queue *rq,
873 					 void *buf,
874 					 void *ctx,
875 					 unsigned int len,
876 					 unsigned int *xdp_xmit,
877 					 struct virtnet_rq_stats *stats)
878 {
879 	struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
880 	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
881 	struct page *page = virt_to_head_page(buf);
882 	int offset = buf - page_address(page);
883 	struct sk_buff *head_skb, *curr_skb;
884 	struct bpf_prog *xdp_prog;
885 	unsigned int truesize = mergeable_ctx_to_truesize(ctx);
886 	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
887 	unsigned int metasize = 0;
888 	unsigned int frame_sz;
889 	int err;
890 
891 	head_skb = NULL;
892 	stats->bytes += len - vi->hdr_len;
893 
894 	if (unlikely(len > truesize)) {
895 		pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
896 			 dev->name, len, (unsigned long)ctx);
897 		dev->stats.rx_length_errors++;
898 		goto err_skb;
899 	}
900 	rcu_read_lock();
901 	xdp_prog = rcu_dereference(rq->xdp_prog);
902 	if (xdp_prog) {
903 		struct xdp_frame *xdpf;
904 		struct page *xdp_page;
905 		struct xdp_buff xdp;
906 		void *data;
907 		u32 act;
908 
909 		/* Transient failure which in theory could occur if
910 		 * in-flight packets from before XDP was enabled reach
911 		 * the receive path after XDP is loaded.
912 		 */
913 		if (unlikely(hdr->hdr.gso_type))
914 			goto err_xdp;
915 
916 		/* Buffers with headroom use PAGE_SIZE as alloc size,
917 		 * see add_recvbuf_mergeable() + get_mergeable_buf_len()
918 		 */
919 		frame_sz = headroom ? PAGE_SIZE : truesize;
920 
921 		/* This happens when rx buffer size is underestimated
922 		 * or headroom is not enough because of the buffer
923 		 * was refilled before XDP is set. This should only
924 		 * happen for the first several packets, so we don't
925 		 * care much about its performance.
926 		 */
927 		if (unlikely(num_buf > 1 ||
928 			     headroom < virtnet_get_headroom(vi))) {
929 			/* linearize data for XDP */
930 			xdp_page = xdp_linearize_page(rq, &num_buf,
931 						      page, offset,
932 						      VIRTIO_XDP_HEADROOM,
933 						      &len);
934 			frame_sz = PAGE_SIZE;
935 
936 			if (!xdp_page)
937 				goto err_xdp;
938 			offset = VIRTIO_XDP_HEADROOM;
939 		} else {
940 			xdp_page = page;
941 		}
942 
943 		/* Allow consuming headroom but reserve enough space to push
944 		 * the descriptor on if we get an XDP_TX return code.
945 		 */
946 		data = page_address(xdp_page) + offset;
947 		xdp_init_buff(&xdp, frame_sz - vi->hdr_len, &rq->xdp_rxq);
948 		xdp_prepare_buff(&xdp, data - VIRTIO_XDP_HEADROOM + vi->hdr_len,
949 				 VIRTIO_XDP_HEADROOM, len - vi->hdr_len, true);
950 
951 		act = bpf_prog_run_xdp(xdp_prog, &xdp);
952 		stats->xdp_packets++;
953 
954 		switch (act) {
955 		case XDP_PASS:
956 			metasize = xdp.data - xdp.data_meta;
957 
958 			/* recalculate offset to account for any header
959 			 * adjustments and minus the metasize to copy the
960 			 * metadata in page_to_skb(). Note other cases do not
961 			 * build an skb and avoid using offset
962 			 */
963 			offset = xdp.data - page_address(xdp_page) -
964 				 vi->hdr_len - metasize;
965 
966 			/* recalculate len if xdp.data, xdp.data_end or
967 			 * xdp.data_meta were adjusted
968 			 */
969 			len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
970 			/* We can only create skb based on xdp_page. */
971 			if (unlikely(xdp_page != page)) {
972 				rcu_read_unlock();
973 				put_page(page);
974 				head_skb = page_to_skb(vi, rq, xdp_page, offset,
975 						       len, PAGE_SIZE, false,
976 						       metasize,
977 						       VIRTIO_XDP_HEADROOM);
978 				return head_skb;
979 			}
980 			break;
981 		case XDP_TX:
982 			stats->xdp_tx++;
983 			xdpf = xdp_convert_buff_to_frame(&xdp);
984 			if (unlikely(!xdpf))
985 				goto err_xdp;
986 			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
987 			if (unlikely(!err)) {
988 				xdp_return_frame_rx_napi(xdpf);
989 			} else if (unlikely(err < 0)) {
990 				trace_xdp_exception(vi->dev, xdp_prog, act);
991 				if (unlikely(xdp_page != page))
992 					put_page(xdp_page);
993 				goto err_xdp;
994 			}
995 			*xdp_xmit |= VIRTIO_XDP_TX;
996 			if (unlikely(xdp_page != page))
997 				put_page(page);
998 			rcu_read_unlock();
999 			goto xdp_xmit;
1000 		case XDP_REDIRECT:
1001 			stats->xdp_redirects++;
1002 			err = xdp_do_redirect(dev, &xdp, xdp_prog);
1003 			if (err) {
1004 				if (unlikely(xdp_page != page))
1005 					put_page(xdp_page);
1006 				goto err_xdp;
1007 			}
1008 			*xdp_xmit |= VIRTIO_XDP_REDIR;
1009 			if (unlikely(xdp_page != page))
1010 				put_page(page);
1011 			rcu_read_unlock();
1012 			goto xdp_xmit;
1013 		default:
1014 			bpf_warn_invalid_xdp_action(act);
1015 			fallthrough;
1016 		case XDP_ABORTED:
1017 			trace_xdp_exception(vi->dev, xdp_prog, act);
1018 			fallthrough;
1019 		case XDP_DROP:
1020 			if (unlikely(xdp_page != page))
1021 				__free_pages(xdp_page, 0);
1022 			goto err_xdp;
1023 		}
1024 	}
1025 	rcu_read_unlock();
1026 
1027 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,
1028 			       metasize, headroom);
1029 	curr_skb = head_skb;
1030 
1031 	if (unlikely(!curr_skb))
1032 		goto err_skb;
1033 	while (--num_buf) {
1034 		int num_skb_frags;
1035 
1036 		buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
1037 		if (unlikely(!buf)) {
1038 			pr_debug("%s: rx error: %d buffers out of %d missing\n",
1039 				 dev->name, num_buf,
1040 				 virtio16_to_cpu(vi->vdev,
1041 						 hdr->num_buffers));
1042 			dev->stats.rx_length_errors++;
1043 			goto err_buf;
1044 		}
1045 
1046 		stats->bytes += len;
1047 		page = virt_to_head_page(buf);
1048 
1049 		truesize = mergeable_ctx_to_truesize(ctx);
1050 		if (unlikely(len > truesize)) {
1051 			pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1052 				 dev->name, len, (unsigned long)ctx);
1053 			dev->stats.rx_length_errors++;
1054 			goto err_skb;
1055 		}
1056 
1057 		num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
1058 		if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
1059 			struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
1060 
1061 			if (unlikely(!nskb))
1062 				goto err_skb;
1063 			if (curr_skb == head_skb)
1064 				skb_shinfo(curr_skb)->frag_list = nskb;
1065 			else
1066 				curr_skb->next = nskb;
1067 			curr_skb = nskb;
1068 			head_skb->truesize += nskb->truesize;
1069 			num_skb_frags = 0;
1070 		}
1071 		if (curr_skb != head_skb) {
1072 			head_skb->data_len += len;
1073 			head_skb->len += len;
1074 			head_skb->truesize += truesize;
1075 		}
1076 		offset = buf - page_address(page);
1077 		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
1078 			put_page(page);
1079 			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
1080 					     len, truesize);
1081 		} else {
1082 			skb_add_rx_frag(curr_skb, num_skb_frags, page,
1083 					offset, len, truesize);
1084 		}
1085 	}
1086 
1087 	ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
1088 	return head_skb;
1089 
1090 err_xdp:
1091 	rcu_read_unlock();
1092 	stats->xdp_drops++;
1093 err_skb:
1094 	put_page(page);
1095 	while (num_buf-- > 1) {
1096 		buf = virtqueue_get_buf(rq->vq, &len);
1097 		if (unlikely(!buf)) {
1098 			pr_debug("%s: rx error: %d buffers missing\n",
1099 				 dev->name, num_buf);
1100 			dev->stats.rx_length_errors++;
1101 			break;
1102 		}
1103 		stats->bytes += len;
1104 		page = virt_to_head_page(buf);
1105 		put_page(page);
1106 	}
1107 err_buf:
1108 	stats->drops++;
1109 	dev_kfree_skb(head_skb);
1110 xdp_xmit:
1111 	return NULL;
1112 }
1113 
1114 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
1115 			void *buf, unsigned int len, void **ctx,
1116 			unsigned int *xdp_xmit,
1117 			struct virtnet_rq_stats *stats)
1118 {
1119 	struct net_device *dev = vi->dev;
1120 	struct sk_buff *skb;
1121 	struct virtio_net_hdr_mrg_rxbuf *hdr;
1122 
1123 	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
1124 		pr_debug("%s: short packet %i\n", dev->name, len);
1125 		dev->stats.rx_length_errors++;
1126 		if (vi->mergeable_rx_bufs) {
1127 			put_page(virt_to_head_page(buf));
1128 		} else if (vi->big_packets) {
1129 			give_pages(rq, buf);
1130 		} else {
1131 			put_page(virt_to_head_page(buf));
1132 		}
1133 		return;
1134 	}
1135 
1136 	if (vi->mergeable_rx_bufs)
1137 		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
1138 					stats);
1139 	else if (vi->big_packets)
1140 		skb = receive_big(dev, vi, rq, buf, len, stats);
1141 	else
1142 		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
1143 
1144 	if (unlikely(!skb))
1145 		return;
1146 
1147 	hdr = skb_vnet_hdr(skb);
1148 
1149 	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
1150 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1151 
1152 	if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
1153 				  virtio_is_little_endian(vi->vdev))) {
1154 		net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
1155 				     dev->name, hdr->hdr.gso_type,
1156 				     hdr->hdr.gso_size);
1157 		goto frame_err;
1158 	}
1159 
1160 	skb_record_rx_queue(skb, vq2rxq(rq->vq));
1161 	skb->protocol = eth_type_trans(skb, dev);
1162 	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
1163 		 ntohs(skb->protocol), skb->len, skb->pkt_type);
1164 
1165 	napi_gro_receive(&rq->napi, skb);
1166 	return;
1167 
1168 frame_err:
1169 	dev->stats.rx_frame_errors++;
1170 	dev_kfree_skb(skb);
1171 }
1172 
1173 /* Unlike mergeable buffers, all buffers are allocated to the
1174  * same size, except for the headroom. For this reason we do
1175  * not need to use  mergeable_len_to_ctx here - it is enough
1176  * to store the headroom as the context ignoring the truesize.
1177  */
1178 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
1179 			     gfp_t gfp)
1180 {
1181 	struct page_frag *alloc_frag = &rq->alloc_frag;
1182 	char *buf;
1183 	unsigned int xdp_headroom = virtnet_get_headroom(vi);
1184 	void *ctx = (void *)(unsigned long)xdp_headroom;
1185 	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
1186 	int err;
1187 
1188 	len = SKB_DATA_ALIGN(len) +
1189 	      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1190 	if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
1191 		return -ENOMEM;
1192 
1193 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1194 	get_page(alloc_frag->page);
1195 	alloc_frag->offset += len;
1196 	sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
1197 		    vi->hdr_len + GOOD_PACKET_LEN);
1198 	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1199 	if (err < 0)
1200 		put_page(virt_to_head_page(buf));
1201 	return err;
1202 }
1203 
1204 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
1205 			   gfp_t gfp)
1206 {
1207 	struct page *first, *list = NULL;
1208 	char *p;
1209 	int i, err, offset;
1210 
1211 	sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
1212 
1213 	/* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
1214 	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
1215 		first = get_a_page(rq, gfp);
1216 		if (!first) {
1217 			if (list)
1218 				give_pages(rq, list);
1219 			return -ENOMEM;
1220 		}
1221 		sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
1222 
1223 		/* chain new page in list head to match sg */
1224 		first->private = (unsigned long)list;
1225 		list = first;
1226 	}
1227 
1228 	first = get_a_page(rq, gfp);
1229 	if (!first) {
1230 		give_pages(rq, list);
1231 		return -ENOMEM;
1232 	}
1233 	p = page_address(first);
1234 
1235 	/* rq->sg[0], rq->sg[1] share the same page */
1236 	/* a separated rq->sg[0] for header - required in case !any_header_sg */
1237 	sg_set_buf(&rq->sg[0], p, vi->hdr_len);
1238 
1239 	/* rq->sg[1] for data packet, from offset */
1240 	offset = sizeof(struct padded_vnet_hdr);
1241 	sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
1242 
1243 	/* chain first in list head */
1244 	first->private = (unsigned long)list;
1245 	err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
1246 				  first, gfp);
1247 	if (err < 0)
1248 		give_pages(rq, first);
1249 
1250 	return err;
1251 }
1252 
1253 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
1254 					  struct ewma_pkt_len *avg_pkt_len,
1255 					  unsigned int room)
1256 {
1257 	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1258 	unsigned int len;
1259 
1260 	if (room)
1261 		return PAGE_SIZE - room;
1262 
1263 	len = hdr_len +	clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
1264 				rq->min_buf_len, PAGE_SIZE - hdr_len);
1265 
1266 	return ALIGN(len, L1_CACHE_BYTES);
1267 }
1268 
1269 static int add_recvbuf_mergeable(struct virtnet_info *vi,
1270 				 struct receive_queue *rq, gfp_t gfp)
1271 {
1272 	struct page_frag *alloc_frag = &rq->alloc_frag;
1273 	unsigned int headroom = virtnet_get_headroom(vi);
1274 	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1275 	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1276 	char *buf;
1277 	void *ctx;
1278 	int err;
1279 	unsigned int len, hole;
1280 
1281 	/* Extra tailroom is needed to satisfy XDP's assumption. This
1282 	 * means rx frags coalescing won't work, but consider we've
1283 	 * disabled GSO for XDP, it won't be a big issue.
1284 	 */
1285 	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1286 	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
1287 		return -ENOMEM;
1288 
1289 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1290 	buf += headroom; /* advance address leaving hole at front of pkt */
1291 	get_page(alloc_frag->page);
1292 	alloc_frag->offset += len + room;
1293 	hole = alloc_frag->size - alloc_frag->offset;
1294 	if (hole < len + room) {
1295 		/* To avoid internal fragmentation, if there is very likely not
1296 		 * enough space for another buffer, add the remaining space to
1297 		 * the current buffer.
1298 		 */
1299 		len += hole;
1300 		alloc_frag->offset += hole;
1301 	}
1302 
1303 	sg_init_one(rq->sg, buf, len);
1304 	ctx = mergeable_len_to_ctx(len, headroom);
1305 	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1306 	if (err < 0)
1307 		put_page(virt_to_head_page(buf));
1308 
1309 	return err;
1310 }
1311 
1312 /*
1313  * Returns false if we couldn't fill entirely (OOM).
1314  *
1315  * Normally run in the receive path, but can also be run from ndo_open
1316  * before we're receiving packets, or from refill_work which is
1317  * careful to disable receiving (using napi_disable).
1318  */
1319 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
1320 			  gfp_t gfp)
1321 {
1322 	int err;
1323 	bool oom;
1324 
1325 	do {
1326 		if (vi->mergeable_rx_bufs)
1327 			err = add_recvbuf_mergeable(vi, rq, gfp);
1328 		else if (vi->big_packets)
1329 			err = add_recvbuf_big(vi, rq, gfp);
1330 		else
1331 			err = add_recvbuf_small(vi, rq, gfp);
1332 
1333 		oom = err == -ENOMEM;
1334 		if (err)
1335 			break;
1336 	} while (rq->vq->num_free);
1337 	if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
1338 		unsigned long flags;
1339 
1340 		flags = u64_stats_update_begin_irqsave(&rq->stats.syncp);
1341 		rq->stats.kicks++;
1342 		u64_stats_update_end_irqrestore(&rq->stats.syncp, flags);
1343 	}
1344 
1345 	return !oom;
1346 }
1347 
1348 static void skb_recv_done(struct virtqueue *rvq)
1349 {
1350 	struct virtnet_info *vi = rvq->vdev->priv;
1351 	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
1352 
1353 	virtqueue_napi_schedule(&rq->napi, rvq);
1354 }
1355 
1356 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
1357 {
1358 	napi_enable(napi);
1359 
1360 	/* If all buffers were filled by other side before we napi_enabled, we
1361 	 * won't get another interrupt, so process any outstanding packets now.
1362 	 * Call local_bh_enable after to trigger softIRQ processing.
1363 	 */
1364 	local_bh_disable();
1365 	virtqueue_napi_schedule(napi, vq);
1366 	local_bh_enable();
1367 }
1368 
1369 static void virtnet_napi_tx_enable(struct virtnet_info *vi,
1370 				   struct virtqueue *vq,
1371 				   struct napi_struct *napi)
1372 {
1373 	if (!napi->weight)
1374 		return;
1375 
1376 	/* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1377 	 * enable the feature if this is likely affine with the transmit path.
1378 	 */
1379 	if (!vi->affinity_hint_set) {
1380 		napi->weight = 0;
1381 		return;
1382 	}
1383 
1384 	return virtnet_napi_enable(vq, napi);
1385 }
1386 
1387 static void virtnet_napi_tx_disable(struct napi_struct *napi)
1388 {
1389 	if (napi->weight)
1390 		napi_disable(napi);
1391 }
1392 
1393 static void refill_work(struct work_struct *work)
1394 {
1395 	struct virtnet_info *vi =
1396 		container_of(work, struct virtnet_info, refill.work);
1397 	bool still_empty;
1398 	int i;
1399 
1400 	for (i = 0; i < vi->curr_queue_pairs; i++) {
1401 		struct receive_queue *rq = &vi->rq[i];
1402 
1403 		napi_disable(&rq->napi);
1404 		still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
1405 		virtnet_napi_enable(rq->vq, &rq->napi);
1406 
1407 		/* In theory, this can happen: if we don't get any buffers in
1408 		 * we will *never* try to fill again.
1409 		 */
1410 		if (still_empty)
1411 			schedule_delayed_work(&vi->refill, HZ/2);
1412 	}
1413 }
1414 
1415 static int virtnet_receive(struct receive_queue *rq, int budget,
1416 			   unsigned int *xdp_xmit)
1417 {
1418 	struct virtnet_info *vi = rq->vq->vdev->priv;
1419 	struct virtnet_rq_stats stats = {};
1420 	unsigned int len;
1421 	void *buf;
1422 	int i;
1423 
1424 	if (!vi->big_packets || vi->mergeable_rx_bufs) {
1425 		void *ctx;
1426 
1427 		while (stats.packets < budget &&
1428 		       (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
1429 			receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
1430 			stats.packets++;
1431 		}
1432 	} else {
1433 		while (stats.packets < budget &&
1434 		       (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
1435 			receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
1436 			stats.packets++;
1437 		}
1438 	}
1439 
1440 	if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
1441 		if (!try_fill_recv(vi, rq, GFP_ATOMIC))
1442 			schedule_delayed_work(&vi->refill, 0);
1443 	}
1444 
1445 	u64_stats_update_begin(&rq->stats.syncp);
1446 	for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) {
1447 		size_t offset = virtnet_rq_stats_desc[i].offset;
1448 		u64 *item;
1449 
1450 		item = (u64 *)((u8 *)&rq->stats + offset);
1451 		*item += *(u64 *)((u8 *)&stats + offset);
1452 	}
1453 	u64_stats_update_end(&rq->stats.syncp);
1454 
1455 	return stats.packets;
1456 }
1457 
1458 static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
1459 {
1460 	unsigned int len;
1461 	unsigned int packets = 0;
1462 	unsigned int bytes = 0;
1463 	void *ptr;
1464 
1465 	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1466 		if (likely(!is_xdp_frame(ptr))) {
1467 			struct sk_buff *skb = ptr;
1468 
1469 			pr_debug("Sent skb %p\n", skb);
1470 
1471 			bytes += skb->len;
1472 			napi_consume_skb(skb, in_napi);
1473 		} else {
1474 			struct xdp_frame *frame = ptr_to_xdp(ptr);
1475 
1476 			bytes += frame->len;
1477 			xdp_return_frame(frame);
1478 		}
1479 		packets++;
1480 	}
1481 
1482 	/* Avoid overhead when no packets have been processed
1483 	 * happens when called speculatively from start_xmit.
1484 	 */
1485 	if (!packets)
1486 		return;
1487 
1488 	u64_stats_update_begin(&sq->stats.syncp);
1489 	sq->stats.bytes += bytes;
1490 	sq->stats.packets += packets;
1491 	u64_stats_update_end(&sq->stats.syncp);
1492 }
1493 
1494 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1495 {
1496 	if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1497 		return false;
1498 	else if (q < vi->curr_queue_pairs)
1499 		return true;
1500 	else
1501 		return false;
1502 }
1503 
1504 static void virtnet_poll_cleantx(struct receive_queue *rq)
1505 {
1506 	struct virtnet_info *vi = rq->vq->vdev->priv;
1507 	unsigned int index = vq2rxq(rq->vq);
1508 	struct send_queue *sq = &vi->sq[index];
1509 	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1510 
1511 	if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
1512 		return;
1513 
1514 	if (__netif_tx_trylock(txq)) {
1515 		do {
1516 			virtqueue_disable_cb(sq->vq);
1517 			free_old_xmit_skbs(sq, true);
1518 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
1519 
1520 		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1521 			netif_tx_wake_queue(txq);
1522 
1523 		__netif_tx_unlock(txq);
1524 	}
1525 }
1526 
1527 static int virtnet_poll(struct napi_struct *napi, int budget)
1528 {
1529 	struct receive_queue *rq =
1530 		container_of(napi, struct receive_queue, napi);
1531 	struct virtnet_info *vi = rq->vq->vdev->priv;
1532 	struct send_queue *sq;
1533 	unsigned int received;
1534 	unsigned int xdp_xmit = 0;
1535 
1536 	virtnet_poll_cleantx(rq);
1537 
1538 	received = virtnet_receive(rq, budget, &xdp_xmit);
1539 
1540 	/* Out of packets? */
1541 	if (received < budget)
1542 		virtqueue_napi_complete(napi, rq->vq, received);
1543 
1544 	if (xdp_xmit & VIRTIO_XDP_REDIR)
1545 		xdp_do_flush();
1546 
1547 	if (xdp_xmit & VIRTIO_XDP_TX) {
1548 		sq = virtnet_xdp_get_sq(vi);
1549 		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
1550 			u64_stats_update_begin(&sq->stats.syncp);
1551 			sq->stats.kicks++;
1552 			u64_stats_update_end(&sq->stats.syncp);
1553 		}
1554 		virtnet_xdp_put_sq(vi, sq);
1555 	}
1556 
1557 	return received;
1558 }
1559 
1560 static int virtnet_open(struct net_device *dev)
1561 {
1562 	struct virtnet_info *vi = netdev_priv(dev);
1563 	int i, err;
1564 
1565 	for (i = 0; i < vi->max_queue_pairs; i++) {
1566 		if (i < vi->curr_queue_pairs)
1567 			/* Make sure we have some buffers: if oom use wq. */
1568 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1569 				schedule_delayed_work(&vi->refill, 0);
1570 
1571 		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id);
1572 		if (err < 0)
1573 			return err;
1574 
1575 		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,
1576 						 MEM_TYPE_PAGE_SHARED, NULL);
1577 		if (err < 0) {
1578 			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
1579 			return err;
1580 		}
1581 
1582 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1583 		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
1584 	}
1585 
1586 	return 0;
1587 }
1588 
1589 static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1590 {
1591 	struct send_queue *sq = container_of(napi, struct send_queue, napi);
1592 	struct virtnet_info *vi = sq->vq->vdev->priv;
1593 	unsigned int index = vq2txq(sq->vq);
1594 	struct netdev_queue *txq;
1595 	int opaque;
1596 	bool done;
1597 
1598 	if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
1599 		/* We don't need to enable cb for XDP */
1600 		napi_complete_done(napi, 0);
1601 		return 0;
1602 	}
1603 
1604 	txq = netdev_get_tx_queue(vi->dev, index);
1605 	__netif_tx_lock(txq, raw_smp_processor_id());
1606 	virtqueue_disable_cb(sq->vq);
1607 	free_old_xmit_skbs(sq, true);
1608 
1609 	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1610 		netif_tx_wake_queue(txq);
1611 
1612 	opaque = virtqueue_enable_cb_prepare(sq->vq);
1613 
1614 	done = napi_complete_done(napi, 0);
1615 
1616 	if (!done)
1617 		virtqueue_disable_cb(sq->vq);
1618 
1619 	__netif_tx_unlock(txq);
1620 
1621 	if (done) {
1622 		if (unlikely(virtqueue_poll(sq->vq, opaque))) {
1623 			if (napi_schedule_prep(napi)) {
1624 				__netif_tx_lock(txq, raw_smp_processor_id());
1625 				virtqueue_disable_cb(sq->vq);
1626 				__netif_tx_unlock(txq);
1627 				__napi_schedule(napi);
1628 			}
1629 		}
1630 	}
1631 
1632 	return 0;
1633 }
1634 
1635 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
1636 {
1637 	struct virtio_net_hdr_mrg_rxbuf *hdr;
1638 	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1639 	struct virtnet_info *vi = sq->vq->vdev->priv;
1640 	int num_sg;
1641 	unsigned hdr_len = vi->hdr_len;
1642 	bool can_push;
1643 
1644 	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1645 
1646 	can_push = vi->any_header_sg &&
1647 		!((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1648 		!skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1649 	/* Even if we can, don't push here yet as this would skew
1650 	 * csum_start offset below. */
1651 	if (can_push)
1652 		hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1653 	else
1654 		hdr = skb_vnet_hdr(skb);
1655 
1656 	if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1657 				    virtio_is_little_endian(vi->vdev), false,
1658 				    0))
1659 		return -EPROTO;
1660 
1661 	if (vi->mergeable_rx_bufs)
1662 		hdr->num_buffers = 0;
1663 
1664 	sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1665 	if (can_push) {
1666 		__skb_push(skb, hdr_len);
1667 		num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1668 		if (unlikely(num_sg < 0))
1669 			return num_sg;
1670 		/* Pull header back to avoid skew in tx bytes calculations. */
1671 		__skb_pull(skb, hdr_len);
1672 	} else {
1673 		sg_set_buf(sq->sg, hdr, hdr_len);
1674 		num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
1675 		if (unlikely(num_sg < 0))
1676 			return num_sg;
1677 		num_sg++;
1678 	}
1679 	return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1680 }
1681 
1682 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1683 {
1684 	struct virtnet_info *vi = netdev_priv(dev);
1685 	int qnum = skb_get_queue_mapping(skb);
1686 	struct send_queue *sq = &vi->sq[qnum];
1687 	int err;
1688 	struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1689 	bool kick = !netdev_xmit_more();
1690 	bool use_napi = sq->napi.weight;
1691 
1692 	/* Free up any pending old buffers before queueing new ones. */
1693 	do {
1694 		if (use_napi)
1695 			virtqueue_disable_cb(sq->vq);
1696 
1697 		free_old_xmit_skbs(sq, false);
1698 
1699 	} while (use_napi && kick &&
1700 	       unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
1701 
1702 	/* timestamp packet in software */
1703 	skb_tx_timestamp(skb);
1704 
1705 	/* Try to transmit */
1706 	err = xmit_skb(sq, skb);
1707 
1708 	/* This should not happen! */
1709 	if (unlikely(err)) {
1710 		dev->stats.tx_fifo_errors++;
1711 		if (net_ratelimit())
1712 			dev_warn(&dev->dev,
1713 				 "Unexpected TXQ (%d) queue failure: %d\n",
1714 				 qnum, err);
1715 		dev->stats.tx_dropped++;
1716 		dev_kfree_skb_any(skb);
1717 		return NETDEV_TX_OK;
1718 	}
1719 
1720 	/* Don't wait up for transmitted skbs to be freed. */
1721 	if (!use_napi) {
1722 		skb_orphan(skb);
1723 		nf_reset_ct(skb);
1724 	}
1725 
1726 	/* If running out of space, stop queue to avoid getting packets that we
1727 	 * are then unable to transmit.
1728 	 * An alternative would be to force queuing layer to requeue the skb by
1729 	 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1730 	 * returned in a normal path of operation: it means that driver is not
1731 	 * maintaining the TX queue stop/start state properly, and causes
1732 	 * the stack to do a non-trivial amount of useless work.
1733 	 * Since most packets only take 1 or 2 ring slots, stopping the queue
1734 	 * early means 16 slots are typically wasted.
1735 	 */
1736 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1737 		netif_stop_subqueue(dev, qnum);
1738 		if (!use_napi &&
1739 		    unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1740 			/* More just got used, free them then recheck. */
1741 			free_old_xmit_skbs(sq, false);
1742 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1743 				netif_start_subqueue(dev, qnum);
1744 				virtqueue_disable_cb(sq->vq);
1745 			}
1746 		}
1747 	}
1748 
1749 	if (kick || netif_xmit_stopped(txq)) {
1750 		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
1751 			u64_stats_update_begin(&sq->stats.syncp);
1752 			sq->stats.kicks++;
1753 			u64_stats_update_end(&sq->stats.syncp);
1754 		}
1755 	}
1756 
1757 	return NETDEV_TX_OK;
1758 }
1759 
1760 /*
1761  * Send command via the control virtqueue and check status.  Commands
1762  * supported by the hypervisor, as indicated by feature bits, should
1763  * never fail unless improperly formatted.
1764  */
1765 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1766 				 struct scatterlist *out)
1767 {
1768 	struct scatterlist *sgs[4], hdr, stat;
1769 	unsigned out_num = 0, tmp;
1770 	int ret;
1771 
1772 	/* Caller should know better */
1773 	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1774 
1775 	vi->ctrl->status = ~0;
1776 	vi->ctrl->hdr.class = class;
1777 	vi->ctrl->hdr.cmd = cmd;
1778 	/* Add header */
1779 	sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
1780 	sgs[out_num++] = &hdr;
1781 
1782 	if (out)
1783 		sgs[out_num++] = out;
1784 
1785 	/* Add return status. */
1786 	sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
1787 	sgs[out_num] = &stat;
1788 
1789 	BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1790 	ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1791 	if (ret < 0) {
1792 		dev_warn(&vi->vdev->dev,
1793 			 "Failed to add sgs for command vq: %d\n.", ret);
1794 		return false;
1795 	}
1796 
1797 	if (unlikely(!virtqueue_kick(vi->cvq)))
1798 		return vi->ctrl->status == VIRTIO_NET_OK;
1799 
1800 	/* Spin for a response, the kick causes an ioport write, trapping
1801 	 * into the hypervisor, so the request should be handled immediately.
1802 	 */
1803 	while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1804 	       !virtqueue_is_broken(vi->cvq))
1805 		cpu_relax();
1806 
1807 	return vi->ctrl->status == VIRTIO_NET_OK;
1808 }
1809 
1810 static int virtnet_set_mac_address(struct net_device *dev, void *p)
1811 {
1812 	struct virtnet_info *vi = netdev_priv(dev);
1813 	struct virtio_device *vdev = vi->vdev;
1814 	int ret;
1815 	struct sockaddr *addr;
1816 	struct scatterlist sg;
1817 
1818 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
1819 		return -EOPNOTSUPP;
1820 
1821 	addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1822 	if (!addr)
1823 		return -ENOMEM;
1824 
1825 	ret = eth_prepare_mac_addr_change(dev, addr);
1826 	if (ret)
1827 		goto out;
1828 
1829 	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1830 		sg_init_one(&sg, addr->sa_data, dev->addr_len);
1831 		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1832 					  VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1833 			dev_warn(&vdev->dev,
1834 				 "Failed to set mac address by vq command.\n");
1835 			ret = -EINVAL;
1836 			goto out;
1837 		}
1838 	} else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1839 		   !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1840 		unsigned int i;
1841 
1842 		/* Naturally, this has an atomicity problem. */
1843 		for (i = 0; i < dev->addr_len; i++)
1844 			virtio_cwrite8(vdev,
1845 				       offsetof(struct virtio_net_config, mac) +
1846 				       i, addr->sa_data[i]);
1847 	}
1848 
1849 	eth_commit_mac_addr_change(dev, p);
1850 	ret = 0;
1851 
1852 out:
1853 	kfree(addr);
1854 	return ret;
1855 }
1856 
1857 static void virtnet_stats(struct net_device *dev,
1858 			  struct rtnl_link_stats64 *tot)
1859 {
1860 	struct virtnet_info *vi = netdev_priv(dev);
1861 	unsigned int start;
1862 	int i;
1863 
1864 	for (i = 0; i < vi->max_queue_pairs; i++) {
1865 		u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops;
1866 		struct receive_queue *rq = &vi->rq[i];
1867 		struct send_queue *sq = &vi->sq[i];
1868 
1869 		do {
1870 			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
1871 			tpackets = sq->stats.packets;
1872 			tbytes   = sq->stats.bytes;
1873 			terrors  = sq->stats.tx_timeouts;
1874 		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
1875 
1876 		do {
1877 			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
1878 			rpackets = rq->stats.packets;
1879 			rbytes   = rq->stats.bytes;
1880 			rdrops   = rq->stats.drops;
1881 		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
1882 
1883 		tot->rx_packets += rpackets;
1884 		tot->tx_packets += tpackets;
1885 		tot->rx_bytes   += rbytes;
1886 		tot->tx_bytes   += tbytes;
1887 		tot->rx_dropped += rdrops;
1888 		tot->tx_errors  += terrors;
1889 	}
1890 
1891 	tot->tx_dropped = dev->stats.tx_dropped;
1892 	tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1893 	tot->rx_length_errors = dev->stats.rx_length_errors;
1894 	tot->rx_frame_errors = dev->stats.rx_frame_errors;
1895 }
1896 
1897 static void virtnet_ack_link_announce(struct virtnet_info *vi)
1898 {
1899 	rtnl_lock();
1900 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1901 				  VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1902 		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1903 	rtnl_unlock();
1904 }
1905 
1906 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1907 {
1908 	struct scatterlist sg;
1909 	struct net_device *dev = vi->dev;
1910 
1911 	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1912 		return 0;
1913 
1914 	vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1915 	sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));
1916 
1917 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1918 				  VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
1919 		dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1920 			 queue_pairs);
1921 		return -EINVAL;
1922 	} else {
1923 		vi->curr_queue_pairs = queue_pairs;
1924 		/* virtnet_open() will refill when device is going to up. */
1925 		if (dev->flags & IFF_UP)
1926 			schedule_delayed_work(&vi->refill, 0);
1927 	}
1928 
1929 	return 0;
1930 }
1931 
1932 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1933 {
1934 	int err;
1935 
1936 	rtnl_lock();
1937 	err = _virtnet_set_queues(vi, queue_pairs);
1938 	rtnl_unlock();
1939 	return err;
1940 }
1941 
1942 static int virtnet_close(struct net_device *dev)
1943 {
1944 	struct virtnet_info *vi = netdev_priv(dev);
1945 	int i;
1946 
1947 	/* Make sure refill_work doesn't re-enable napi! */
1948 	cancel_delayed_work_sync(&vi->refill);
1949 
1950 	for (i = 0; i < vi->max_queue_pairs; i++) {
1951 		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
1952 		napi_disable(&vi->rq[i].napi);
1953 		virtnet_napi_tx_disable(&vi->sq[i].napi);
1954 	}
1955 
1956 	return 0;
1957 }
1958 
1959 static void virtnet_set_rx_mode(struct net_device *dev)
1960 {
1961 	struct virtnet_info *vi = netdev_priv(dev);
1962 	struct scatterlist sg[2];
1963 	struct virtio_net_ctrl_mac *mac_data;
1964 	struct netdev_hw_addr *ha;
1965 	int uc_count;
1966 	int mc_count;
1967 	void *buf;
1968 	int i;
1969 
1970 	/* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1971 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1972 		return;
1973 
1974 	vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
1975 	vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1976 
1977 	sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));
1978 
1979 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1980 				  VIRTIO_NET_CTRL_RX_PROMISC, sg))
1981 		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
1982 			 vi->ctrl->promisc ? "en" : "dis");
1983 
1984 	sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));
1985 
1986 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1987 				  VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1988 		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
1989 			 vi->ctrl->allmulti ? "en" : "dis");
1990 
1991 	uc_count = netdev_uc_count(dev);
1992 	mc_count = netdev_mc_count(dev);
1993 	/* MAC filter - use one buffer for both lists */
1994 	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
1995 		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
1996 	mac_data = buf;
1997 	if (!buf)
1998 		return;
1999 
2000 	sg_init_table(sg, 2);
2001 
2002 	/* Store the unicast list and count in the front of the buffer */
2003 	mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
2004 	i = 0;
2005 	netdev_for_each_uc_addr(ha, dev)
2006 		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
2007 
2008 	sg_set_buf(&sg[0], mac_data,
2009 		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
2010 
2011 	/* multicast list and count fill the end */
2012 	mac_data = (void *)&mac_data->macs[uc_count][0];
2013 
2014 	mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
2015 	i = 0;
2016 	netdev_for_each_mc_addr(ha, dev)
2017 		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
2018 
2019 	sg_set_buf(&sg[1], mac_data,
2020 		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
2021 
2022 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
2023 				  VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
2024 		dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
2025 
2026 	kfree(buf);
2027 }
2028 
2029 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
2030 				   __be16 proto, u16 vid)
2031 {
2032 	struct virtnet_info *vi = netdev_priv(dev);
2033 	struct scatterlist sg;
2034 
2035 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
2036 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
2037 
2038 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
2039 				  VIRTIO_NET_CTRL_VLAN_ADD, &sg))
2040 		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
2041 	return 0;
2042 }
2043 
2044 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
2045 				    __be16 proto, u16 vid)
2046 {
2047 	struct virtnet_info *vi = netdev_priv(dev);
2048 	struct scatterlist sg;
2049 
2050 	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
2051 	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
2052 
2053 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
2054 				  VIRTIO_NET_CTRL_VLAN_DEL, &sg))
2055 		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
2056 	return 0;
2057 }
2058 
2059 static void virtnet_clean_affinity(struct virtnet_info *vi)
2060 {
2061 	int i;
2062 
2063 	if (vi->affinity_hint_set) {
2064 		for (i = 0; i < vi->max_queue_pairs; i++) {
2065 			virtqueue_set_affinity(vi->rq[i].vq, NULL);
2066 			virtqueue_set_affinity(vi->sq[i].vq, NULL);
2067 		}
2068 
2069 		vi->affinity_hint_set = false;
2070 	}
2071 }
2072 
2073 static void virtnet_set_affinity(struct virtnet_info *vi)
2074 {
2075 	cpumask_var_t mask;
2076 	int stragglers;
2077 	int group_size;
2078 	int i, j, cpu;
2079 	int num_cpu;
2080 	int stride;
2081 
2082 	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2083 		virtnet_clean_affinity(vi);
2084 		return;
2085 	}
2086 
2087 	num_cpu = num_online_cpus();
2088 	stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);
2089 	stragglers = num_cpu >= vi->curr_queue_pairs ?
2090 			num_cpu % vi->curr_queue_pairs :
2091 			0;
2092 	cpu = cpumask_next(-1, cpu_online_mask);
2093 
2094 	for (i = 0; i < vi->curr_queue_pairs; i++) {
2095 		group_size = stride + (i < stragglers ? 1 : 0);
2096 
2097 		for (j = 0; j < group_size; j++) {
2098 			cpumask_set_cpu(cpu, mask);
2099 			cpu = cpumask_next_wrap(cpu, cpu_online_mask,
2100 						nr_cpu_ids, false);
2101 		}
2102 		virtqueue_set_affinity(vi->rq[i].vq, mask);
2103 		virtqueue_set_affinity(vi->sq[i].vq, mask);
2104 		__netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS);
2105 		cpumask_clear(mask);
2106 	}
2107 
2108 	vi->affinity_hint_set = true;
2109 	free_cpumask_var(mask);
2110 }
2111 
2112 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
2113 {
2114 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2115 						   node);
2116 	virtnet_set_affinity(vi);
2117 	return 0;
2118 }
2119 
2120 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
2121 {
2122 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2123 						   node_dead);
2124 	virtnet_set_affinity(vi);
2125 	return 0;
2126 }
2127 
2128 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
2129 {
2130 	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2131 						   node);
2132 
2133 	virtnet_clean_affinity(vi);
2134 	return 0;
2135 }
2136 
2137 static enum cpuhp_state virtionet_online;
2138 
2139 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
2140 {
2141 	int ret;
2142 
2143 	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
2144 	if (ret)
2145 		return ret;
2146 	ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
2147 					       &vi->node_dead);
2148 	if (!ret)
2149 		return ret;
2150 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
2151 	return ret;
2152 }
2153 
2154 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
2155 {
2156 	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
2157 	cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
2158 					    &vi->node_dead);
2159 }
2160 
2161 static void virtnet_get_ringparam(struct net_device *dev,
2162 				struct ethtool_ringparam *ring)
2163 {
2164 	struct virtnet_info *vi = netdev_priv(dev);
2165 
2166 	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
2167 	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
2168 	ring->rx_pending = ring->rx_max_pending;
2169 	ring->tx_pending = ring->tx_max_pending;
2170 }
2171 
2172 
2173 static void virtnet_get_drvinfo(struct net_device *dev,
2174 				struct ethtool_drvinfo *info)
2175 {
2176 	struct virtnet_info *vi = netdev_priv(dev);
2177 	struct virtio_device *vdev = vi->vdev;
2178 
2179 	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
2180 	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
2181 	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
2182 
2183 }
2184 
2185 /* TODO: Eliminate OOO packets during switching */
2186 static int virtnet_set_channels(struct net_device *dev,
2187 				struct ethtool_channels *channels)
2188 {
2189 	struct virtnet_info *vi = netdev_priv(dev);
2190 	u16 queue_pairs = channels->combined_count;
2191 	int err;
2192 
2193 	/* We don't support separate rx/tx channels.
2194 	 * We don't allow setting 'other' channels.
2195 	 */
2196 	if (channels->rx_count || channels->tx_count || channels->other_count)
2197 		return -EINVAL;
2198 
2199 	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
2200 		return -EINVAL;
2201 
2202 	/* For now we don't support modifying channels while XDP is loaded
2203 	 * also when XDP is loaded all RX queues have XDP programs so we only
2204 	 * need to check a single RX queue.
2205 	 */
2206 	if (vi->rq[0].xdp_prog)
2207 		return -EINVAL;
2208 
2209 	cpus_read_lock();
2210 	err = _virtnet_set_queues(vi, queue_pairs);
2211 	if (err) {
2212 		cpus_read_unlock();
2213 		goto err;
2214 	}
2215 	virtnet_set_affinity(vi);
2216 	cpus_read_unlock();
2217 
2218 	netif_set_real_num_tx_queues(dev, queue_pairs);
2219 	netif_set_real_num_rx_queues(dev, queue_pairs);
2220  err:
2221 	return err;
2222 }
2223 
2224 static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2225 {
2226 	struct virtnet_info *vi = netdev_priv(dev);
2227 	unsigned int i, j;
2228 	u8 *p = data;
2229 
2230 	switch (stringset) {
2231 	case ETH_SS_STATS:
2232 		for (i = 0; i < vi->curr_queue_pairs; i++) {
2233 			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++)
2234 				ethtool_sprintf(&p, "rx_queue_%u_%s", i,
2235 						virtnet_rq_stats_desc[j].desc);
2236 		}
2237 
2238 		for (i = 0; i < vi->curr_queue_pairs; i++) {
2239 			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++)
2240 				ethtool_sprintf(&p, "tx_queue_%u_%s", i,
2241 						virtnet_sq_stats_desc[j].desc);
2242 		}
2243 		break;
2244 	}
2245 }
2246 
2247 static int virtnet_get_sset_count(struct net_device *dev, int sset)
2248 {
2249 	struct virtnet_info *vi = netdev_priv(dev);
2250 
2251 	switch (sset) {
2252 	case ETH_SS_STATS:
2253 		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
2254 					       VIRTNET_SQ_STATS_LEN);
2255 	default:
2256 		return -EOPNOTSUPP;
2257 	}
2258 }
2259 
2260 static void virtnet_get_ethtool_stats(struct net_device *dev,
2261 				      struct ethtool_stats *stats, u64 *data)
2262 {
2263 	struct virtnet_info *vi = netdev_priv(dev);
2264 	unsigned int idx = 0, start, i, j;
2265 	const u8 *stats_base;
2266 	size_t offset;
2267 
2268 	for (i = 0; i < vi->curr_queue_pairs; i++) {
2269 		struct receive_queue *rq = &vi->rq[i];
2270 
2271 		stats_base = (u8 *)&rq->stats;
2272 		do {
2273 			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
2274 			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
2275 				offset = virtnet_rq_stats_desc[j].offset;
2276 				data[idx + j] = *(u64 *)(stats_base + offset);
2277 			}
2278 		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
2279 		idx += VIRTNET_RQ_STATS_LEN;
2280 	}
2281 
2282 	for (i = 0; i < vi->curr_queue_pairs; i++) {
2283 		struct send_queue *sq = &vi->sq[i];
2284 
2285 		stats_base = (u8 *)&sq->stats;
2286 		do {
2287 			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
2288 			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
2289 				offset = virtnet_sq_stats_desc[j].offset;
2290 				data[idx + j] = *(u64 *)(stats_base + offset);
2291 			}
2292 		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
2293 		idx += VIRTNET_SQ_STATS_LEN;
2294 	}
2295 }
2296 
2297 static void virtnet_get_channels(struct net_device *dev,
2298 				 struct ethtool_channels *channels)
2299 {
2300 	struct virtnet_info *vi = netdev_priv(dev);
2301 
2302 	channels->combined_count = vi->curr_queue_pairs;
2303 	channels->max_combined = vi->max_queue_pairs;
2304 	channels->max_other = 0;
2305 	channels->rx_count = 0;
2306 	channels->tx_count = 0;
2307 	channels->other_count = 0;
2308 }
2309 
2310 static int virtnet_set_link_ksettings(struct net_device *dev,
2311 				      const struct ethtool_link_ksettings *cmd)
2312 {
2313 	struct virtnet_info *vi = netdev_priv(dev);
2314 
2315 	return ethtool_virtdev_set_link_ksettings(dev, cmd,
2316 						  &vi->speed, &vi->duplex);
2317 }
2318 
2319 static int virtnet_get_link_ksettings(struct net_device *dev,
2320 				      struct ethtool_link_ksettings *cmd)
2321 {
2322 	struct virtnet_info *vi = netdev_priv(dev);
2323 
2324 	cmd->base.speed = vi->speed;
2325 	cmd->base.duplex = vi->duplex;
2326 	cmd->base.port = PORT_OTHER;
2327 
2328 	return 0;
2329 }
2330 
2331 static int virtnet_set_coalesce(struct net_device *dev,
2332 				struct ethtool_coalesce *ec,
2333 				struct kernel_ethtool_coalesce *kernel_coal,
2334 				struct netlink_ext_ack *extack)
2335 {
2336 	struct virtnet_info *vi = netdev_priv(dev);
2337 	int i, napi_weight;
2338 
2339 	if (ec->tx_max_coalesced_frames > 1 ||
2340 	    ec->rx_max_coalesced_frames != 1)
2341 		return -EINVAL;
2342 
2343 	napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
2344 	if (napi_weight ^ vi->sq[0].napi.weight) {
2345 		if (dev->flags & IFF_UP)
2346 			return -EBUSY;
2347 		for (i = 0; i < vi->max_queue_pairs; i++)
2348 			vi->sq[i].napi.weight = napi_weight;
2349 	}
2350 
2351 	return 0;
2352 }
2353 
2354 static int virtnet_get_coalesce(struct net_device *dev,
2355 				struct ethtool_coalesce *ec,
2356 				struct kernel_ethtool_coalesce *kernel_coal,
2357 				struct netlink_ext_ack *extack)
2358 {
2359 	struct ethtool_coalesce ec_default = {
2360 		.cmd = ETHTOOL_GCOALESCE,
2361 		.rx_max_coalesced_frames = 1,
2362 	};
2363 	struct virtnet_info *vi = netdev_priv(dev);
2364 
2365 	memcpy(ec, &ec_default, sizeof(ec_default));
2366 
2367 	if (vi->sq[0].napi.weight)
2368 		ec->tx_max_coalesced_frames = 1;
2369 
2370 	return 0;
2371 }
2372 
2373 static void virtnet_init_settings(struct net_device *dev)
2374 {
2375 	struct virtnet_info *vi = netdev_priv(dev);
2376 
2377 	vi->speed = SPEED_UNKNOWN;
2378 	vi->duplex = DUPLEX_UNKNOWN;
2379 }
2380 
2381 static void virtnet_update_settings(struct virtnet_info *vi)
2382 {
2383 	u32 speed;
2384 	u8 duplex;
2385 
2386 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2387 		return;
2388 
2389 	virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed);
2390 
2391 	if (ethtool_validate_speed(speed))
2392 		vi->speed = speed;
2393 
2394 	virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex);
2395 
2396 	if (ethtool_validate_duplex(duplex))
2397 		vi->duplex = duplex;
2398 }
2399 
2400 static const struct ethtool_ops virtnet_ethtool_ops = {
2401 	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
2402 	.get_drvinfo = virtnet_get_drvinfo,
2403 	.get_link = ethtool_op_get_link,
2404 	.get_ringparam = virtnet_get_ringparam,
2405 	.get_strings = virtnet_get_strings,
2406 	.get_sset_count = virtnet_get_sset_count,
2407 	.get_ethtool_stats = virtnet_get_ethtool_stats,
2408 	.set_channels = virtnet_set_channels,
2409 	.get_channels = virtnet_get_channels,
2410 	.get_ts_info = ethtool_op_get_ts_info,
2411 	.get_link_ksettings = virtnet_get_link_ksettings,
2412 	.set_link_ksettings = virtnet_set_link_ksettings,
2413 	.set_coalesce = virtnet_set_coalesce,
2414 	.get_coalesce = virtnet_get_coalesce,
2415 };
2416 
2417 static void virtnet_freeze_down(struct virtio_device *vdev)
2418 {
2419 	struct virtnet_info *vi = vdev->priv;
2420 	int i;
2421 
2422 	/* Make sure no work handler is accessing the device */
2423 	flush_work(&vi->config_work);
2424 
2425 	netif_tx_lock_bh(vi->dev);
2426 	netif_device_detach(vi->dev);
2427 	netif_tx_unlock_bh(vi->dev);
2428 	cancel_delayed_work_sync(&vi->refill);
2429 
2430 	if (netif_running(vi->dev)) {
2431 		for (i = 0; i < vi->max_queue_pairs; i++) {
2432 			napi_disable(&vi->rq[i].napi);
2433 			virtnet_napi_tx_disable(&vi->sq[i].napi);
2434 		}
2435 	}
2436 }
2437 
2438 static int init_vqs(struct virtnet_info *vi);
2439 
2440 static int virtnet_restore_up(struct virtio_device *vdev)
2441 {
2442 	struct virtnet_info *vi = vdev->priv;
2443 	int err, i;
2444 
2445 	err = init_vqs(vi);
2446 	if (err)
2447 		return err;
2448 
2449 	virtio_device_ready(vdev);
2450 
2451 	if (netif_running(vi->dev)) {
2452 		for (i = 0; i < vi->curr_queue_pairs; i++)
2453 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2454 				schedule_delayed_work(&vi->refill, 0);
2455 
2456 		for (i = 0; i < vi->max_queue_pairs; i++) {
2457 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2458 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2459 					       &vi->sq[i].napi);
2460 		}
2461 	}
2462 
2463 	netif_tx_lock_bh(vi->dev);
2464 	netif_device_attach(vi->dev);
2465 	netif_tx_unlock_bh(vi->dev);
2466 	return err;
2467 }
2468 
2469 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
2470 {
2471 	struct scatterlist sg;
2472 	vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);
2473 
2474 	sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));
2475 
2476 	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
2477 				  VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
2478 		dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");
2479 		return -EINVAL;
2480 	}
2481 
2482 	return 0;
2483 }
2484 
2485 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
2486 {
2487 	u64 offloads = 0;
2488 
2489 	if (!vi->guest_offloads)
2490 		return 0;
2491 
2492 	return virtnet_set_guest_offloads(vi, offloads);
2493 }
2494 
2495 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
2496 {
2497 	u64 offloads = vi->guest_offloads;
2498 
2499 	if (!vi->guest_offloads)
2500 		return 0;
2501 
2502 	return virtnet_set_guest_offloads(vi, offloads);
2503 }
2504 
2505 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
2506 			   struct netlink_ext_ack *extack)
2507 {
2508 	unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
2509 	struct virtnet_info *vi = netdev_priv(dev);
2510 	struct bpf_prog *old_prog;
2511 	u16 xdp_qp = 0, curr_qp;
2512 	int i, err;
2513 
2514 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
2515 	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2516 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2517 	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2518 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
2519 		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {
2520 		NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first");
2521 		return -EOPNOTSUPP;
2522 	}
2523 
2524 	if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
2525 		NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
2526 		return -EINVAL;
2527 	}
2528 
2529 	if (dev->mtu > max_sz) {
2530 		NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
2531 		netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
2532 		return -EINVAL;
2533 	}
2534 
2535 	curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
2536 	if (prog)
2537 		xdp_qp = nr_cpu_ids;
2538 
2539 	/* XDP requires extra queues for XDP_TX */
2540 	if (curr_qp + xdp_qp > vi->max_queue_pairs) {
2541 		netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n",
2542 				 curr_qp + xdp_qp, vi->max_queue_pairs);
2543 		xdp_qp = 0;
2544 	}
2545 
2546 	old_prog = rtnl_dereference(vi->rq[0].xdp_prog);
2547 	if (!prog && !old_prog)
2548 		return 0;
2549 
2550 	if (prog)
2551 		bpf_prog_add(prog, vi->max_queue_pairs - 1);
2552 
2553 	/* Make sure NAPI is not using any XDP TX queues for RX. */
2554 	if (netif_running(dev)) {
2555 		for (i = 0; i < vi->max_queue_pairs; i++) {
2556 			napi_disable(&vi->rq[i].napi);
2557 			virtnet_napi_tx_disable(&vi->sq[i].napi);
2558 		}
2559 	}
2560 
2561 	if (!prog) {
2562 		for (i = 0; i < vi->max_queue_pairs; i++) {
2563 			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2564 			if (i == 0)
2565 				virtnet_restore_guest_offloads(vi);
2566 		}
2567 		synchronize_net();
2568 	}
2569 
2570 	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
2571 	if (err)
2572 		goto err;
2573 	netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
2574 	vi->xdp_queue_pairs = xdp_qp;
2575 
2576 	if (prog) {
2577 		vi->xdp_enabled = true;
2578 		for (i = 0; i < vi->max_queue_pairs; i++) {
2579 			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2580 			if (i == 0 && !old_prog)
2581 				virtnet_clear_guest_offloads(vi);
2582 		}
2583 	} else {
2584 		vi->xdp_enabled = false;
2585 	}
2586 
2587 	for (i = 0; i < vi->max_queue_pairs; i++) {
2588 		if (old_prog)
2589 			bpf_prog_put(old_prog);
2590 		if (netif_running(dev)) {
2591 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2592 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2593 					       &vi->sq[i].napi);
2594 		}
2595 	}
2596 
2597 	return 0;
2598 
2599 err:
2600 	if (!prog) {
2601 		virtnet_clear_guest_offloads(vi);
2602 		for (i = 0; i < vi->max_queue_pairs; i++)
2603 			rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);
2604 	}
2605 
2606 	if (netif_running(dev)) {
2607 		for (i = 0; i < vi->max_queue_pairs; i++) {
2608 			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2609 			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2610 					       &vi->sq[i].napi);
2611 		}
2612 	}
2613 	if (prog)
2614 		bpf_prog_sub(prog, vi->max_queue_pairs - 1);
2615 	return err;
2616 }
2617 
2618 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2619 {
2620 	switch (xdp->command) {
2621 	case XDP_SETUP_PROG:
2622 		return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
2623 	default:
2624 		return -EINVAL;
2625 	}
2626 }
2627 
2628 static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
2629 				      size_t len)
2630 {
2631 	struct virtnet_info *vi = netdev_priv(dev);
2632 	int ret;
2633 
2634 	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
2635 		return -EOPNOTSUPP;
2636 
2637 	ret = snprintf(buf, len, "sby");
2638 	if (ret >= len)
2639 		return -EOPNOTSUPP;
2640 
2641 	return 0;
2642 }
2643 
2644 static int virtnet_set_features(struct net_device *dev,
2645 				netdev_features_t features)
2646 {
2647 	struct virtnet_info *vi = netdev_priv(dev);
2648 	u64 offloads;
2649 	int err;
2650 
2651 	if ((dev->features ^ features) & NETIF_F_GRO_HW) {
2652 		if (vi->xdp_enabled)
2653 			return -EBUSY;
2654 
2655 		if (features & NETIF_F_GRO_HW)
2656 			offloads = vi->guest_offloads_capable;
2657 		else
2658 			offloads = vi->guest_offloads_capable &
2659 				   ~GUEST_OFFLOAD_GRO_HW_MASK;
2660 
2661 		err = virtnet_set_guest_offloads(vi, offloads);
2662 		if (err)
2663 			return err;
2664 		vi->guest_offloads = offloads;
2665 	}
2666 
2667 	return 0;
2668 }
2669 
2670 static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue)
2671 {
2672 	struct virtnet_info *priv = netdev_priv(dev);
2673 	struct send_queue *sq = &priv->sq[txqueue];
2674 	struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
2675 
2676 	u64_stats_update_begin(&sq->stats.syncp);
2677 	sq->stats.tx_timeouts++;
2678 	u64_stats_update_end(&sq->stats.syncp);
2679 
2680 	netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n",
2681 		   txqueue, sq->name, sq->vq->index, sq->vq->name,
2682 		   jiffies_to_usecs(jiffies - txq->trans_start));
2683 }
2684 
2685 static const struct net_device_ops virtnet_netdev = {
2686 	.ndo_open            = virtnet_open,
2687 	.ndo_stop   	     = virtnet_close,
2688 	.ndo_start_xmit      = start_xmit,
2689 	.ndo_validate_addr   = eth_validate_addr,
2690 	.ndo_set_mac_address = virtnet_set_mac_address,
2691 	.ndo_set_rx_mode     = virtnet_set_rx_mode,
2692 	.ndo_get_stats64     = virtnet_stats,
2693 	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
2694 	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
2695 	.ndo_bpf		= virtnet_xdp,
2696 	.ndo_xdp_xmit		= virtnet_xdp_xmit,
2697 	.ndo_features_check	= passthru_features_check,
2698 	.ndo_get_phys_port_name	= virtnet_get_phys_port_name,
2699 	.ndo_set_features	= virtnet_set_features,
2700 	.ndo_tx_timeout		= virtnet_tx_timeout,
2701 };
2702 
2703 static void virtnet_config_changed_work(struct work_struct *work)
2704 {
2705 	struct virtnet_info *vi =
2706 		container_of(work, struct virtnet_info, config_work);
2707 	u16 v;
2708 
2709 	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
2710 				 struct virtio_net_config, status, &v) < 0)
2711 		return;
2712 
2713 	if (v & VIRTIO_NET_S_ANNOUNCE) {
2714 		netdev_notify_peers(vi->dev);
2715 		virtnet_ack_link_announce(vi);
2716 	}
2717 
2718 	/* Ignore unknown (future) status bits */
2719 	v &= VIRTIO_NET_S_LINK_UP;
2720 
2721 	if (vi->status == v)
2722 		return;
2723 
2724 	vi->status = v;
2725 
2726 	if (vi->status & VIRTIO_NET_S_LINK_UP) {
2727 		virtnet_update_settings(vi);
2728 		netif_carrier_on(vi->dev);
2729 		netif_tx_wake_all_queues(vi->dev);
2730 	} else {
2731 		netif_carrier_off(vi->dev);
2732 		netif_tx_stop_all_queues(vi->dev);
2733 	}
2734 }
2735 
2736 static void virtnet_config_changed(struct virtio_device *vdev)
2737 {
2738 	struct virtnet_info *vi = vdev->priv;
2739 
2740 	schedule_work(&vi->config_work);
2741 }
2742 
2743 static void virtnet_free_queues(struct virtnet_info *vi)
2744 {
2745 	int i;
2746 
2747 	for (i = 0; i < vi->max_queue_pairs; i++) {
2748 		__netif_napi_del(&vi->rq[i].napi);
2749 		__netif_napi_del(&vi->sq[i].napi);
2750 	}
2751 
2752 	/* We called __netif_napi_del(),
2753 	 * we need to respect an RCU grace period before freeing vi->rq
2754 	 */
2755 	synchronize_net();
2756 
2757 	kfree(vi->rq);
2758 	kfree(vi->sq);
2759 	kfree(vi->ctrl);
2760 }
2761 
2762 static void _free_receive_bufs(struct virtnet_info *vi)
2763 {
2764 	struct bpf_prog *old_prog;
2765 	int i;
2766 
2767 	for (i = 0; i < vi->max_queue_pairs; i++) {
2768 		while (vi->rq[i].pages)
2769 			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
2770 
2771 		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2772 		RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
2773 		if (old_prog)
2774 			bpf_prog_put(old_prog);
2775 	}
2776 }
2777 
2778 static void free_receive_bufs(struct virtnet_info *vi)
2779 {
2780 	rtnl_lock();
2781 	_free_receive_bufs(vi);
2782 	rtnl_unlock();
2783 }
2784 
2785 static void free_receive_page_frags(struct virtnet_info *vi)
2786 {
2787 	int i;
2788 	for (i = 0; i < vi->max_queue_pairs; i++)
2789 		if (vi->rq[i].alloc_frag.page)
2790 			put_page(vi->rq[i].alloc_frag.page);
2791 }
2792 
2793 static void free_unused_bufs(struct virtnet_info *vi)
2794 {
2795 	void *buf;
2796 	int i;
2797 
2798 	for (i = 0; i < vi->max_queue_pairs; i++) {
2799 		struct virtqueue *vq = vi->sq[i].vq;
2800 		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2801 			if (!is_xdp_frame(buf))
2802 				dev_kfree_skb(buf);
2803 			else
2804 				xdp_return_frame(ptr_to_xdp(buf));
2805 		}
2806 	}
2807 
2808 	for (i = 0; i < vi->max_queue_pairs; i++) {
2809 		struct virtqueue *vq = vi->rq[i].vq;
2810 
2811 		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2812 			if (vi->mergeable_rx_bufs) {
2813 				put_page(virt_to_head_page(buf));
2814 			} else if (vi->big_packets) {
2815 				give_pages(&vi->rq[i], buf);
2816 			} else {
2817 				put_page(virt_to_head_page(buf));
2818 			}
2819 		}
2820 	}
2821 }
2822 
2823 static void virtnet_del_vqs(struct virtnet_info *vi)
2824 {
2825 	struct virtio_device *vdev = vi->vdev;
2826 
2827 	virtnet_clean_affinity(vi);
2828 
2829 	vdev->config->del_vqs(vdev);
2830 
2831 	virtnet_free_queues(vi);
2832 }
2833 
2834 /* How large should a single buffer be so a queue full of these can fit at
2835  * least one full packet?
2836  * Logic below assumes the mergeable buffer header is used.
2837  */
2838 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2839 {
2840 	const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2841 	unsigned int rq_size = virtqueue_get_vring_size(vq);
2842 	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2843 	unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2844 	unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2845 
2846 	return max(max(min_buf_len, hdr_len) - hdr_len,
2847 		   (unsigned int)GOOD_PACKET_LEN);
2848 }
2849 
2850 static int virtnet_find_vqs(struct virtnet_info *vi)
2851 {
2852 	vq_callback_t **callbacks;
2853 	struct virtqueue **vqs;
2854 	int ret = -ENOMEM;
2855 	int i, total_vqs;
2856 	const char **names;
2857 	bool *ctx;
2858 
2859 	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2860 	 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2861 	 * possible control vq.
2862 	 */
2863 	total_vqs = vi->max_queue_pairs * 2 +
2864 		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2865 
2866 	/* Allocate space for find_vqs parameters */
2867 	vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
2868 	if (!vqs)
2869 		goto err_vq;
2870 	callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL);
2871 	if (!callbacks)
2872 		goto err_callback;
2873 	names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL);
2874 	if (!names)
2875 		goto err_names;
2876 	if (!vi->big_packets || vi->mergeable_rx_bufs) {
2877 		ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
2878 		if (!ctx)
2879 			goto err_ctx;
2880 	} else {
2881 		ctx = NULL;
2882 	}
2883 
2884 	/* Parameters for control virtqueue, if any */
2885 	if (vi->has_cvq) {
2886 		callbacks[total_vqs - 1] = NULL;
2887 		names[total_vqs - 1] = "control";
2888 	}
2889 
2890 	/* Allocate/initialize parameters for send/receive virtqueues */
2891 	for (i = 0; i < vi->max_queue_pairs; i++) {
2892 		callbacks[rxq2vq(i)] = skb_recv_done;
2893 		callbacks[txq2vq(i)] = skb_xmit_done;
2894 		sprintf(vi->rq[i].name, "input.%d", i);
2895 		sprintf(vi->sq[i].name, "output.%d", i);
2896 		names[rxq2vq(i)] = vi->rq[i].name;
2897 		names[txq2vq(i)] = vi->sq[i].name;
2898 		if (ctx)
2899 			ctx[rxq2vq(i)] = true;
2900 	}
2901 
2902 	ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks,
2903 				  names, ctx, NULL);
2904 	if (ret)
2905 		goto err_find;
2906 
2907 	if (vi->has_cvq) {
2908 		vi->cvq = vqs[total_vqs - 1];
2909 		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2910 			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2911 	}
2912 
2913 	for (i = 0; i < vi->max_queue_pairs; i++) {
2914 		vi->rq[i].vq = vqs[rxq2vq(i)];
2915 		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
2916 		vi->sq[i].vq = vqs[txq2vq(i)];
2917 	}
2918 
2919 	/* run here: ret == 0. */
2920 
2921 
2922 err_find:
2923 	kfree(ctx);
2924 err_ctx:
2925 	kfree(names);
2926 err_names:
2927 	kfree(callbacks);
2928 err_callback:
2929 	kfree(vqs);
2930 err_vq:
2931 	return ret;
2932 }
2933 
2934 static int virtnet_alloc_queues(struct virtnet_info *vi)
2935 {
2936 	int i;
2937 
2938 	if (vi->has_cvq) {
2939 		vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
2940 		if (!vi->ctrl)
2941 			goto err_ctrl;
2942 	} else {
2943 		vi->ctrl = NULL;
2944 	}
2945 	vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
2946 	if (!vi->sq)
2947 		goto err_sq;
2948 	vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
2949 	if (!vi->rq)
2950 		goto err_rq;
2951 
2952 	INIT_DELAYED_WORK(&vi->refill, refill_work);
2953 	for (i = 0; i < vi->max_queue_pairs; i++) {
2954 		vi->rq[i].pages = NULL;
2955 		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2956 			       napi_weight);
2957 		netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
2958 				  napi_tx ? napi_weight : 0);
2959 
2960 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
2961 		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
2962 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
2963 
2964 		u64_stats_init(&vi->rq[i].stats.syncp);
2965 		u64_stats_init(&vi->sq[i].stats.syncp);
2966 	}
2967 
2968 	return 0;
2969 
2970 err_rq:
2971 	kfree(vi->sq);
2972 err_sq:
2973 	kfree(vi->ctrl);
2974 err_ctrl:
2975 	return -ENOMEM;
2976 }
2977 
2978 static int init_vqs(struct virtnet_info *vi)
2979 {
2980 	int ret;
2981 
2982 	/* Allocate send & receive queues */
2983 	ret = virtnet_alloc_queues(vi);
2984 	if (ret)
2985 		goto err;
2986 
2987 	ret = virtnet_find_vqs(vi);
2988 	if (ret)
2989 		goto err_free;
2990 
2991 	cpus_read_lock();
2992 	virtnet_set_affinity(vi);
2993 	cpus_read_unlock();
2994 
2995 	return 0;
2996 
2997 err_free:
2998 	virtnet_free_queues(vi);
2999 err:
3000 	return ret;
3001 }
3002 
3003 #ifdef CONFIG_SYSFS
3004 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
3005 		char *buf)
3006 {
3007 	struct virtnet_info *vi = netdev_priv(queue->dev);
3008 	unsigned int queue_index = get_netdev_rx_queue_index(queue);
3009 	unsigned int headroom = virtnet_get_headroom(vi);
3010 	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
3011 	struct ewma_pkt_len *avg;
3012 
3013 	BUG_ON(queue_index >= vi->max_queue_pairs);
3014 	avg = &vi->rq[queue_index].mrg_avg_pkt_len;
3015 	return sprintf(buf, "%u\n",
3016 		       get_mergeable_buf_len(&vi->rq[queue_index], avg,
3017 				       SKB_DATA_ALIGN(headroom + tailroom)));
3018 }
3019 
3020 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
3021 	__ATTR_RO(mergeable_rx_buffer_size);
3022 
3023 static struct attribute *virtio_net_mrg_rx_attrs[] = {
3024 	&mergeable_rx_buffer_size_attribute.attr,
3025 	NULL
3026 };
3027 
3028 static const struct attribute_group virtio_net_mrg_rx_group = {
3029 	.name = "virtio_net",
3030 	.attrs = virtio_net_mrg_rx_attrs
3031 };
3032 #endif
3033 
3034 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
3035 				    unsigned int fbit,
3036 				    const char *fname, const char *dname)
3037 {
3038 	if (!virtio_has_feature(vdev, fbit))
3039 		return false;
3040 
3041 	dev_err(&vdev->dev, "device advertises feature %s but not %s",
3042 		fname, dname);
3043 
3044 	return true;
3045 }
3046 
3047 #define VIRTNET_FAIL_ON(vdev, fbit, dbit)			\
3048 	virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
3049 
3050 static bool virtnet_validate_features(struct virtio_device *vdev)
3051 {
3052 	if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
3053 	    (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
3054 			     "VIRTIO_NET_F_CTRL_VQ") ||
3055 	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
3056 			     "VIRTIO_NET_F_CTRL_VQ") ||
3057 	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
3058 			     "VIRTIO_NET_F_CTRL_VQ") ||
3059 	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
3060 	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
3061 			     "VIRTIO_NET_F_CTRL_VQ"))) {
3062 		return false;
3063 	}
3064 
3065 	return true;
3066 }
3067 
3068 #define MIN_MTU ETH_MIN_MTU
3069 #define MAX_MTU ETH_MAX_MTU
3070 
3071 static int virtnet_validate(struct virtio_device *vdev)
3072 {
3073 	if (!vdev->config->get) {
3074 		dev_err(&vdev->dev, "%s failure: config access disabled\n",
3075 			__func__);
3076 		return -EINVAL;
3077 	}
3078 
3079 	if (!virtnet_validate_features(vdev))
3080 		return -EINVAL;
3081 
3082 	if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
3083 		int mtu = virtio_cread16(vdev,
3084 					 offsetof(struct virtio_net_config,
3085 						  mtu));
3086 		if (mtu < MIN_MTU)
3087 			__virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
3088 	}
3089 
3090 	return 0;
3091 }
3092 
3093 static int virtnet_probe(struct virtio_device *vdev)
3094 {
3095 	int i, err = -ENOMEM;
3096 	struct net_device *dev;
3097 	struct virtnet_info *vi;
3098 	u16 max_queue_pairs;
3099 	int mtu;
3100 
3101 	/* Find if host supports multiqueue virtio_net device */
3102 	err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
3103 				   struct virtio_net_config,
3104 				   max_virtqueue_pairs, &max_queue_pairs);
3105 
3106 	/* We need at least 2 queue's */
3107 	if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
3108 	    max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
3109 	    !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3110 		max_queue_pairs = 1;
3111 
3112 	/* Allocate ourselves a network device with room for our info */
3113 	dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
3114 	if (!dev)
3115 		return -ENOMEM;
3116 
3117 	/* Set up network device as normal. */
3118 	dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE |
3119 			   IFF_TX_SKB_NO_LINEAR;
3120 	dev->netdev_ops = &virtnet_netdev;
3121 	dev->features = NETIF_F_HIGHDMA;
3122 
3123 	dev->ethtool_ops = &virtnet_ethtool_ops;
3124 	SET_NETDEV_DEV(dev, &vdev->dev);
3125 
3126 	/* Do we support "hardware" checksums? */
3127 	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
3128 		/* This opens up the world of extra features. */
3129 		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3130 		if (csum)
3131 			dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3132 
3133 		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
3134 			dev->hw_features |= NETIF_F_TSO
3135 				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
3136 		}
3137 		/* Individual feature bits: what can host handle? */
3138 		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
3139 			dev->hw_features |= NETIF_F_TSO;
3140 		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
3141 			dev->hw_features |= NETIF_F_TSO6;
3142 		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
3143 			dev->hw_features |= NETIF_F_TSO_ECN;
3144 
3145 		dev->features |= NETIF_F_GSO_ROBUST;
3146 
3147 		if (gso)
3148 			dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
3149 		/* (!csum && gso) case will be fixed by register_netdev() */
3150 	}
3151 	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
3152 		dev->features |= NETIF_F_RXCSUM;
3153 	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
3154 	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6))
3155 		dev->features |= NETIF_F_GRO_HW;
3156 	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))
3157 		dev->hw_features |= NETIF_F_GRO_HW;
3158 
3159 	dev->vlan_features = dev->features;
3160 
3161 	/* MTU range: 68 - 65535 */
3162 	dev->min_mtu = MIN_MTU;
3163 	dev->max_mtu = MAX_MTU;
3164 
3165 	/* Configuration may specify what MAC to use.  Otherwise random. */
3166 	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
3167 		virtio_cread_bytes(vdev,
3168 				   offsetof(struct virtio_net_config, mac),
3169 				   dev->dev_addr, dev->addr_len);
3170 	else
3171 		eth_hw_addr_random(dev);
3172 
3173 	/* Set up our device-specific information */
3174 	vi = netdev_priv(dev);
3175 	vi->dev = dev;
3176 	vi->vdev = vdev;
3177 	vdev->priv = vi;
3178 
3179 	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
3180 
3181 	/* If we can receive ANY GSO packets, we must allocate large ones. */
3182 	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
3183 	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
3184 	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
3185 	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
3186 		vi->big_packets = true;
3187 
3188 	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3189 		vi->mergeable_rx_bufs = true;
3190 
3191 	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3192 	    virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3193 		vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
3194 	else
3195 		vi->hdr_len = sizeof(struct virtio_net_hdr);
3196 
3197 	if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
3198 	    virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3199 		vi->any_header_sg = true;
3200 
3201 	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3202 		vi->has_cvq = true;
3203 
3204 	if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
3205 		mtu = virtio_cread16(vdev,
3206 				     offsetof(struct virtio_net_config,
3207 					      mtu));
3208 		if (mtu < dev->min_mtu) {
3209 			/* Should never trigger: MTU was previously validated
3210 			 * in virtnet_validate.
3211 			 */
3212 			dev_err(&vdev->dev,
3213 				"device MTU appears to have changed it is now %d < %d",
3214 				mtu, dev->min_mtu);
3215 			err = -EINVAL;
3216 			goto free;
3217 		}
3218 
3219 		dev->mtu = mtu;
3220 		dev->max_mtu = mtu;
3221 
3222 		/* TODO: size buffers correctly in this case. */
3223 		if (dev->mtu > ETH_DATA_LEN)
3224 			vi->big_packets = true;
3225 	}
3226 
3227 	if (vi->any_header_sg)
3228 		dev->needed_headroom = vi->hdr_len;
3229 
3230 	/* Enable multiqueue by default */
3231 	if (num_online_cpus() >= max_queue_pairs)
3232 		vi->curr_queue_pairs = max_queue_pairs;
3233 	else
3234 		vi->curr_queue_pairs = num_online_cpus();
3235 	vi->max_queue_pairs = max_queue_pairs;
3236 
3237 	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3238 	err = init_vqs(vi);
3239 	if (err)
3240 		goto free;
3241 
3242 #ifdef CONFIG_SYSFS
3243 	if (vi->mergeable_rx_bufs)
3244 		dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
3245 #endif
3246 	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
3247 	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
3248 
3249 	virtnet_init_settings(dev);
3250 
3251 	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3252 		vi->failover = net_failover_create(vi->dev);
3253 		if (IS_ERR(vi->failover)) {
3254 			err = PTR_ERR(vi->failover);
3255 			goto free_vqs;
3256 		}
3257 	}
3258 
3259 	err = register_netdev(dev);
3260 	if (err) {
3261 		pr_debug("virtio_net: registering device failed\n");
3262 		goto free_failover;
3263 	}
3264 
3265 	virtio_device_ready(vdev);
3266 
3267 	err = virtnet_cpu_notif_add(vi);
3268 	if (err) {
3269 		pr_debug("virtio_net: registering cpu notifier failed\n");
3270 		goto free_unregister_netdev;
3271 	}
3272 
3273 	virtnet_set_queues(vi, vi->curr_queue_pairs);
3274 
3275 	/* Assume link up if device can't report link status,
3276 	   otherwise get link status from config. */
3277 	netif_carrier_off(dev);
3278 	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
3279 		schedule_work(&vi->config_work);
3280 	} else {
3281 		vi->status = VIRTIO_NET_S_LINK_UP;
3282 		virtnet_update_settings(vi);
3283 		netif_carrier_on(dev);
3284 	}
3285 
3286 	for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
3287 		if (virtio_has_feature(vi->vdev, guest_offloads[i]))
3288 			set_bit(guest_offloads[i], &vi->guest_offloads);
3289 	vi->guest_offloads_capable = vi->guest_offloads;
3290 
3291 	pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
3292 		 dev->name, max_queue_pairs);
3293 
3294 	return 0;
3295 
3296 free_unregister_netdev:
3297 	vi->vdev->config->reset(vdev);
3298 
3299 	unregister_netdev(dev);
3300 free_failover:
3301 	net_failover_destroy(vi->failover);
3302 free_vqs:
3303 	cancel_delayed_work_sync(&vi->refill);
3304 	free_receive_page_frags(vi);
3305 	virtnet_del_vqs(vi);
3306 free:
3307 	free_netdev(dev);
3308 	return err;
3309 }
3310 
3311 static void remove_vq_common(struct virtnet_info *vi)
3312 {
3313 	vi->vdev->config->reset(vi->vdev);
3314 
3315 	/* Free unused buffers in both send and recv, if any. */
3316 	free_unused_bufs(vi);
3317 
3318 	free_receive_bufs(vi);
3319 
3320 	free_receive_page_frags(vi);
3321 
3322 	virtnet_del_vqs(vi);
3323 }
3324 
3325 static void virtnet_remove(struct virtio_device *vdev)
3326 {
3327 	struct virtnet_info *vi = vdev->priv;
3328 
3329 	virtnet_cpu_notif_remove(vi);
3330 
3331 	/* Make sure no work handler is accessing the device. */
3332 	flush_work(&vi->config_work);
3333 
3334 	unregister_netdev(vi->dev);
3335 
3336 	net_failover_destroy(vi->failover);
3337 
3338 	remove_vq_common(vi);
3339 
3340 	free_netdev(vi->dev);
3341 }
3342 
3343 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
3344 {
3345 	struct virtnet_info *vi = vdev->priv;
3346 
3347 	virtnet_cpu_notif_remove(vi);
3348 	virtnet_freeze_down(vdev);
3349 	remove_vq_common(vi);
3350 
3351 	return 0;
3352 }
3353 
3354 static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
3355 {
3356 	struct virtnet_info *vi = vdev->priv;
3357 	int err;
3358 
3359 	err = virtnet_restore_up(vdev);
3360 	if (err)
3361 		return err;
3362 	virtnet_set_queues(vi, vi->curr_queue_pairs);
3363 
3364 	err = virtnet_cpu_notif_add(vi);
3365 	if (err) {
3366 		virtnet_freeze_down(vdev);
3367 		remove_vq_common(vi);
3368 		return err;
3369 	}
3370 
3371 	return 0;
3372 }
3373 
3374 static struct virtio_device_id id_table[] = {
3375 	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
3376 	{ 0 },
3377 };
3378 
3379 #define VIRTNET_FEATURES \
3380 	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
3381 	VIRTIO_NET_F_MAC, \
3382 	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
3383 	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
3384 	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
3385 	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
3386 	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
3387 	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3388 	VIRTIO_NET_F_CTRL_MAC_ADDR, \
3389 	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3390 	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
3391 
3392 static unsigned int features[] = {
3393 	VIRTNET_FEATURES,
3394 };
3395 
3396 static unsigned int features_legacy[] = {
3397 	VIRTNET_FEATURES,
3398 	VIRTIO_NET_F_GSO,
3399 	VIRTIO_F_ANY_LAYOUT,
3400 };
3401 
3402 static struct virtio_driver virtio_net_driver = {
3403 	.feature_table = features,
3404 	.feature_table_size = ARRAY_SIZE(features),
3405 	.feature_table_legacy = features_legacy,
3406 	.feature_table_size_legacy = ARRAY_SIZE(features_legacy),
3407 	.driver.name =	KBUILD_MODNAME,
3408 	.driver.owner =	THIS_MODULE,
3409 	.id_table =	id_table,
3410 	.validate =	virtnet_validate,
3411 	.probe =	virtnet_probe,
3412 	.remove =	virtnet_remove,
3413 	.config_changed = virtnet_config_changed,
3414 #ifdef CONFIG_PM_SLEEP
3415 	.freeze =	virtnet_freeze,
3416 	.restore =	virtnet_restore,
3417 #endif
3418 };
3419 
3420 static __init int virtio_net_driver_init(void)
3421 {
3422 	int ret;
3423 
3424 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
3425 				      virtnet_cpu_online,
3426 				      virtnet_cpu_down_prep);
3427 	if (ret < 0)
3428 		goto out;
3429 	virtionet_online = ret;
3430 	ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
3431 				      NULL, virtnet_cpu_dead);
3432 	if (ret)
3433 		goto err_dead;
3434 
3435         ret = register_virtio_driver(&virtio_net_driver);
3436 	if (ret)
3437 		goto err_virtio;
3438 	return 0;
3439 err_virtio:
3440 	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3441 err_dead:
3442 	cpuhp_remove_multi_state(virtionet_online);
3443 out:
3444 	return ret;
3445 }
3446 module_init(virtio_net_driver_init);
3447 
3448 static __exit void virtio_net_driver_exit(void)
3449 {
3450 	unregister_virtio_driver(&virtio_net_driver);
3451 	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3452 	cpuhp_remove_multi_state(virtionet_online);
3453 }
3454 module_exit(virtio_net_driver_exit);
3455 
3456 MODULE_DEVICE_TABLE(virtio, id_table);
3457 MODULE_DESCRIPTION("Virtio network driver");
3458 MODULE_LICENSE("GPL");
3459