1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
3 
4 #include <linux/bpf_trace.h>
5 #include <linux/prefetch.h>
6 #include <linux/sctp.h>
7 #include <net/mpls.h>
8 #include <net/xdp.h>
9 #include "i40e_txrx_common.h"
10 #include "i40e_trace.h"
11 #include "i40e_xsk.h"
12 
13 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
14 /**
15  * i40e_fdir - Generate a Flow Director descriptor based on fdata
16  * @tx_ring: Tx ring to send buffer on
17  * @fdata: Flow director filter data
18  * @add: Indicate if we are adding a rule or deleting one
19  *
20  **/
21 static void i40e_fdir(struct i40e_ring *tx_ring,
22 		      struct i40e_fdir_filter *fdata, bool add)
23 {
24 	struct i40e_filter_program_desc *fdir_desc;
25 	struct i40e_pf *pf = tx_ring->vsi->back;
26 	u32 flex_ptype, dtype_cmd;
27 	u16 i;
28 
29 	/* grab the next descriptor */
30 	i = tx_ring->next_to_use;
31 	fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
32 
33 	i++;
34 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
35 
36 	flex_ptype = I40E_TXD_FLTR_QW0_QINDEX_MASK &
37 		     (fdata->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT);
38 
39 	flex_ptype |= I40E_TXD_FLTR_QW0_FLEXOFF_MASK &
40 		      (fdata->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT);
41 
42 	flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK &
43 		      (fdata->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
44 
45 	/* Use LAN VSI Id if not programmed by user */
46 	flex_ptype |= I40E_TXD_FLTR_QW0_DEST_VSI_MASK &
47 		      ((u32)(fdata->dest_vsi ? : pf->vsi[pf->lan_vsi]->id) <<
48 		       I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT);
49 
50 	dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
51 
52 	dtype_cmd |= add ?
53 		     I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
54 		     I40E_TXD_FLTR_QW1_PCMD_SHIFT :
55 		     I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
56 		     I40E_TXD_FLTR_QW1_PCMD_SHIFT;
57 
58 	dtype_cmd |= I40E_TXD_FLTR_QW1_DEST_MASK &
59 		     (fdata->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT);
60 
61 	dtype_cmd |= I40E_TXD_FLTR_QW1_FD_STATUS_MASK &
62 		     (fdata->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT);
63 
64 	if (fdata->cnt_index) {
65 		dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
66 		dtype_cmd |= I40E_TXD_FLTR_QW1_CNTINDEX_MASK &
67 			     ((u32)fdata->cnt_index <<
68 			      I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT);
69 	}
70 
71 	fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
72 	fdir_desc->rsvd = cpu_to_le32(0);
73 	fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
74 	fdir_desc->fd_id = cpu_to_le32(fdata->fd_id);
75 }
76 
77 #define I40E_FD_CLEAN_DELAY 10
78 /**
79  * i40e_program_fdir_filter - Program a Flow Director filter
80  * @fdir_data: Packet data that will be filter parameters
81  * @raw_packet: the pre-allocated packet buffer for FDir
82  * @pf: The PF pointer
83  * @add: True for add/update, False for remove
84  **/
85 static int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data,
86 				    u8 *raw_packet, struct i40e_pf *pf,
87 				    bool add)
88 {
89 	struct i40e_tx_buffer *tx_buf, *first;
90 	struct i40e_tx_desc *tx_desc;
91 	struct i40e_ring *tx_ring;
92 	struct i40e_vsi *vsi;
93 	struct device *dev;
94 	dma_addr_t dma;
95 	u32 td_cmd = 0;
96 	u16 i;
97 
98 	/* find existing FDIR VSI */
99 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
100 	if (!vsi)
101 		return -ENOENT;
102 
103 	tx_ring = vsi->tx_rings[0];
104 	dev = tx_ring->dev;
105 
106 	/* we need two descriptors to add/del a filter and we can wait */
107 	for (i = I40E_FD_CLEAN_DELAY; I40E_DESC_UNUSED(tx_ring) < 2; i--) {
108 		if (!i)
109 			return -EAGAIN;
110 		msleep_interruptible(1);
111 	}
112 
113 	dma = dma_map_single(dev, raw_packet,
114 			     I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
115 	if (dma_mapping_error(dev, dma))
116 		goto dma_fail;
117 
118 	/* grab the next descriptor */
119 	i = tx_ring->next_to_use;
120 	first = &tx_ring->tx_bi[i];
121 	i40e_fdir(tx_ring, fdir_data, add);
122 
123 	/* Now program a dummy descriptor */
124 	i = tx_ring->next_to_use;
125 	tx_desc = I40E_TX_DESC(tx_ring, i);
126 	tx_buf = &tx_ring->tx_bi[i];
127 
128 	tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0;
129 
130 	memset(tx_buf, 0, sizeof(struct i40e_tx_buffer));
131 
132 	/* record length, and DMA address */
133 	dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
134 	dma_unmap_addr_set(tx_buf, dma, dma);
135 
136 	tx_desc->buffer_addr = cpu_to_le64(dma);
137 	td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
138 
139 	tx_buf->tx_flags = I40E_TX_FLAGS_FD_SB;
140 	tx_buf->raw_buf = (void *)raw_packet;
141 
142 	tx_desc->cmd_type_offset_bsz =
143 		build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
144 
145 	/* Force memory writes to complete before letting h/w
146 	 * know there are new descriptors to fetch.
147 	 */
148 	wmb();
149 
150 	/* Mark the data descriptor to be watched */
151 	first->next_to_watch = tx_desc;
152 
153 	writel(tx_ring->next_to_use, tx_ring->tail);
154 	return 0;
155 
156 dma_fail:
157 	return -1;
158 }
159 
160 /**
161  * i40e_create_dummy_packet - Constructs dummy packet for HW
162  * @dummy_packet: preallocated space for dummy packet
163  * @ipv4: is layer 3 packet of version 4 or 6
164  * @l4proto: next level protocol used in data portion of l3
165  * @data: filter data
166  *
167  * Returns address of layer 4 protocol dummy packet.
168  **/
169 static char *i40e_create_dummy_packet(u8 *dummy_packet, bool ipv4, u8 l4proto,
170 				      struct i40e_fdir_filter *data)
171 {
172 	bool is_vlan = !!data->vlan_tag;
173 	struct vlan_hdr vlan = {};
174 	struct ipv6hdr ipv6 = {};
175 	struct ethhdr eth = {};
176 	struct iphdr ip = {};
177 	u8 *tmp;
178 
179 	if (ipv4) {
180 		eth.h_proto = cpu_to_be16(ETH_P_IP);
181 		ip.protocol = l4proto;
182 		ip.version = 0x4;
183 		ip.ihl = 0x5;
184 
185 		ip.daddr = data->dst_ip;
186 		ip.saddr = data->src_ip;
187 	} else {
188 		eth.h_proto = cpu_to_be16(ETH_P_IPV6);
189 		ipv6.nexthdr = l4proto;
190 		ipv6.version = 0x6;
191 
192 		memcpy(&ipv6.saddr.in6_u.u6_addr32, data->src_ip6,
193 		       sizeof(__be32) * 4);
194 		memcpy(&ipv6.daddr.in6_u.u6_addr32, data->dst_ip6,
195 		       sizeof(__be32) * 4);
196 	}
197 
198 	if (is_vlan) {
199 		vlan.h_vlan_TCI = data->vlan_tag;
200 		vlan.h_vlan_encapsulated_proto = eth.h_proto;
201 		eth.h_proto = data->vlan_etype;
202 	}
203 
204 	tmp = dummy_packet;
205 	memcpy(tmp, &eth, sizeof(eth));
206 	tmp += sizeof(eth);
207 
208 	if (is_vlan) {
209 		memcpy(tmp, &vlan, sizeof(vlan));
210 		tmp += sizeof(vlan);
211 	}
212 
213 	if (ipv4) {
214 		memcpy(tmp, &ip, sizeof(ip));
215 		tmp += sizeof(ip);
216 	} else {
217 		memcpy(tmp, &ipv6, sizeof(ipv6));
218 		tmp += sizeof(ipv6);
219 	}
220 
221 	return tmp;
222 }
223 
224 /**
225  * i40e_create_dummy_udp_packet - helper function to create UDP packet
226  * @raw_packet: preallocated space for dummy packet
227  * @ipv4: is layer 3 packet of version 4 or 6
228  * @l4proto: next level protocol used in data portion of l3
229  * @data: filter data
230  *
231  * Helper function to populate udp fields.
232  **/
233 static void i40e_create_dummy_udp_packet(u8 *raw_packet, bool ipv4, u8 l4proto,
234 					 struct i40e_fdir_filter *data)
235 {
236 	struct udphdr *udp;
237 	u8 *tmp;
238 
239 	tmp = i40e_create_dummy_packet(raw_packet, ipv4, IPPROTO_UDP, data);
240 	udp = (struct udphdr *)(tmp);
241 	udp->dest = data->dst_port;
242 	udp->source = data->src_port;
243 }
244 
245 /**
246  * i40e_create_dummy_tcp_packet - helper function to create TCP packet
247  * @raw_packet: preallocated space for dummy packet
248  * @ipv4: is layer 3 packet of version 4 or 6
249  * @l4proto: next level protocol used in data portion of l3
250  * @data: filter data
251  *
252  * Helper function to populate tcp fields.
253  **/
254 static void i40e_create_dummy_tcp_packet(u8 *raw_packet, bool ipv4, u8 l4proto,
255 					 struct i40e_fdir_filter *data)
256 {
257 	struct tcphdr *tcp;
258 	u8 *tmp;
259 	/* Dummy tcp packet */
260 	static const char tcp_packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
261 		0x50, 0x11, 0x0, 0x72, 0, 0, 0, 0};
262 
263 	tmp = i40e_create_dummy_packet(raw_packet, ipv4, IPPROTO_TCP, data);
264 
265 	tcp = (struct tcphdr *)tmp;
266 	memcpy(tcp, tcp_packet, sizeof(tcp_packet));
267 	tcp->dest = data->dst_port;
268 	tcp->source = data->src_port;
269 }
270 
271 /**
272  * i40e_create_dummy_sctp_packet - helper function to create SCTP packet
273  * @raw_packet: preallocated space for dummy packet
274  * @ipv4: is layer 3 packet of version 4 or 6
275  * @l4proto: next level protocol used in data portion of l3
276  * @data: filter data
277  *
278  * Helper function to populate sctp fields.
279  **/
280 static void i40e_create_dummy_sctp_packet(u8 *raw_packet, bool ipv4,
281 					  u8 l4proto,
282 					  struct i40e_fdir_filter *data)
283 {
284 	struct sctphdr *sctp;
285 	u8 *tmp;
286 
287 	tmp = i40e_create_dummy_packet(raw_packet, ipv4, IPPROTO_SCTP, data);
288 
289 	sctp = (struct sctphdr *)tmp;
290 	sctp->dest = data->dst_port;
291 	sctp->source = data->src_port;
292 }
293 
294 /**
295  * i40e_prepare_fdir_filter - Prepare and program fdir filter
296  * @pf: physical function to attach filter to
297  * @fd_data: filter data
298  * @add: add or delete filter
299  * @packet_addr: address of dummy packet, used in filtering
300  * @payload_offset: offset from dummy packet address to user defined data
301  * @pctype: Packet type for which filter is used
302  *
303  * Helper function to offset data of dummy packet, program it and
304  * handle errors.
305  **/
306 static int i40e_prepare_fdir_filter(struct i40e_pf *pf,
307 				    struct i40e_fdir_filter *fd_data,
308 				    bool add, char *packet_addr,
309 				    int payload_offset, u8 pctype)
310 {
311 	int ret;
312 
313 	if (fd_data->flex_filter) {
314 		u8 *payload;
315 		__be16 pattern = fd_data->flex_word;
316 		u16 off = fd_data->flex_offset;
317 
318 		payload = packet_addr + payload_offset;
319 
320 		/* If user provided vlan, offset payload by vlan header length */
321 		if (!!fd_data->vlan_tag)
322 			payload += VLAN_HLEN;
323 
324 		*((__force __be16 *)(payload + off)) = pattern;
325 	}
326 
327 	fd_data->pctype = pctype;
328 	ret = i40e_program_fdir_filter(fd_data, packet_addr, pf, add);
329 	if (ret) {
330 		dev_info(&pf->pdev->dev,
331 			 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n",
332 			 fd_data->pctype, fd_data->fd_id, ret);
333 		/* Free the packet buffer since it wasn't added to the ring */
334 		return -EOPNOTSUPP;
335 	} else if (I40E_DEBUG_FD & pf->hw.debug_mask) {
336 		if (add)
337 			dev_info(&pf->pdev->dev,
338 				 "Filter OK for PCTYPE %d loc = %d\n",
339 				 fd_data->pctype, fd_data->fd_id);
340 		else
341 			dev_info(&pf->pdev->dev,
342 				 "Filter deleted for PCTYPE %d loc = %d\n",
343 				 fd_data->pctype, fd_data->fd_id);
344 	}
345 
346 	return ret;
347 }
348 
349 /**
350  * i40e_change_filter_num - Prepare and program fdir filter
351  * @ipv4: is layer 3 packet of version 4 or 6
352  * @add: add or delete filter
353  * @ipv4_filter_num: field to update
354  * @ipv6_filter_num: field to update
355  *
356  * Update filter number field for pf.
357  **/
358 static void i40e_change_filter_num(bool ipv4, bool add, u16 *ipv4_filter_num,
359 				   u16 *ipv6_filter_num)
360 {
361 	if (add) {
362 		if (ipv4)
363 			(*ipv4_filter_num)++;
364 		else
365 			(*ipv6_filter_num)++;
366 	} else {
367 		if (ipv4)
368 			(*ipv4_filter_num)--;
369 		else
370 			(*ipv6_filter_num)--;
371 	}
372 }
373 
374 #define I40E_UDPIP_DUMMY_PACKET_LEN	42
375 #define I40E_UDPIP6_DUMMY_PACKET_LEN	62
376 /**
377  * i40e_add_del_fdir_udp - Add/Remove UDP filters
378  * @vsi: pointer to the targeted VSI
379  * @fd_data: the flow director data required for the FDir descriptor
380  * @add: true adds a filter, false removes it
381  * @ipv4: true is v4, false is v6
382  *
383  * Returns 0 if the filters were successfully added or removed
384  **/
385 static int i40e_add_del_fdir_udp(struct i40e_vsi *vsi,
386 				 struct i40e_fdir_filter *fd_data,
387 				 bool add,
388 				 bool ipv4)
389 {
390 	struct i40e_pf *pf = vsi->back;
391 	u8 *raw_packet;
392 	int ret;
393 
394 	raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
395 	if (!raw_packet)
396 		return -ENOMEM;
397 
398 	i40e_create_dummy_udp_packet(raw_packet, ipv4, IPPROTO_UDP, fd_data);
399 
400 	if (ipv4)
401 		ret = i40e_prepare_fdir_filter
402 			(pf, fd_data, add, raw_packet,
403 			 I40E_UDPIP_DUMMY_PACKET_LEN,
404 			 I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
405 	else
406 		ret = i40e_prepare_fdir_filter
407 			(pf, fd_data, add, raw_packet,
408 			 I40E_UDPIP6_DUMMY_PACKET_LEN,
409 			 I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
410 
411 	if (ret) {
412 		kfree(raw_packet);
413 		return ret;
414 	}
415 
416 	i40e_change_filter_num(ipv4, add, &pf->fd_udp4_filter_cnt,
417 			       &pf->fd_udp6_filter_cnt);
418 
419 	return 0;
420 }
421 
422 #define I40E_TCPIP_DUMMY_PACKET_LEN	54
423 #define I40E_TCPIP6_DUMMY_PACKET_LEN	74
424 /**
425  * i40e_add_del_fdir_tcp - Add/Remove TCPv4 filters
426  * @vsi: pointer to the targeted VSI
427  * @fd_data: the flow director data required for the FDir descriptor
428  * @add: true adds a filter, false removes it
429  * @ipv4: true is v4, false is v6
430  *
431  * Returns 0 if the filters were successfully added or removed
432  **/
433 static int i40e_add_del_fdir_tcp(struct i40e_vsi *vsi,
434 				 struct i40e_fdir_filter *fd_data,
435 				 bool add,
436 				 bool ipv4)
437 {
438 	struct i40e_pf *pf = vsi->back;
439 	u8 *raw_packet;
440 	int ret;
441 
442 	raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
443 	if (!raw_packet)
444 		return -ENOMEM;
445 
446 	i40e_create_dummy_tcp_packet(raw_packet, ipv4, IPPROTO_TCP, fd_data);
447 	if (ipv4)
448 		ret = i40e_prepare_fdir_filter
449 			(pf, fd_data, add, raw_packet,
450 			 I40E_TCPIP_DUMMY_PACKET_LEN,
451 			 I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
452 	else
453 		ret = i40e_prepare_fdir_filter
454 			(pf, fd_data, add, raw_packet,
455 			 I40E_TCPIP6_DUMMY_PACKET_LEN,
456 			 I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
457 
458 	if (ret) {
459 		kfree(raw_packet);
460 		return ret;
461 	}
462 
463 	i40e_change_filter_num(ipv4, add, &pf->fd_tcp4_filter_cnt,
464 			       &pf->fd_tcp6_filter_cnt);
465 
466 	if (add) {
467 		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
468 		    I40E_DEBUG_FD & pf->hw.debug_mask)
469 			dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
470 		set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
471 	}
472 	return 0;
473 }
474 
475 #define I40E_SCTPIP_DUMMY_PACKET_LEN	46
476 #define I40E_SCTPIP6_DUMMY_PACKET_LEN	66
477 /**
478  * i40e_add_del_fdir_sctp - Add/Remove SCTPv4 Flow Director filters for
479  * a specific flow spec
480  * @vsi: pointer to the targeted VSI
481  * @fd_data: the flow director data required for the FDir descriptor
482  * @add: true adds a filter, false removes it
483  * @ipv4: true is v4, false is v6
484  *
485  * Returns 0 if the filters were successfully added or removed
486  **/
487 static int i40e_add_del_fdir_sctp(struct i40e_vsi *vsi,
488 				  struct i40e_fdir_filter *fd_data,
489 				  bool add,
490 				  bool ipv4)
491 {
492 	struct i40e_pf *pf = vsi->back;
493 	u8 *raw_packet;
494 	int ret;
495 
496 	raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
497 	if (!raw_packet)
498 		return -ENOMEM;
499 
500 	i40e_create_dummy_sctp_packet(raw_packet, ipv4, IPPROTO_SCTP, fd_data);
501 
502 	if (ipv4)
503 		ret = i40e_prepare_fdir_filter
504 			(pf, fd_data, add, raw_packet,
505 			 I40E_SCTPIP_DUMMY_PACKET_LEN,
506 			 I40E_FILTER_PCTYPE_NONF_IPV4_SCTP);
507 	else
508 		ret = i40e_prepare_fdir_filter
509 			(pf, fd_data, add, raw_packet,
510 			 I40E_SCTPIP6_DUMMY_PACKET_LEN,
511 			 I40E_FILTER_PCTYPE_NONF_IPV6_SCTP);
512 
513 	if (ret) {
514 		kfree(raw_packet);
515 		return ret;
516 	}
517 
518 	i40e_change_filter_num(ipv4, add, &pf->fd_sctp4_filter_cnt,
519 			       &pf->fd_sctp6_filter_cnt);
520 
521 	return 0;
522 }
523 
524 #define I40E_IP_DUMMY_PACKET_LEN	34
525 #define I40E_IP6_DUMMY_PACKET_LEN	54
526 /**
527  * i40e_add_del_fdir_ip - Add/Remove IPv4 Flow Director filters for
528  * a specific flow spec
529  * @vsi: pointer to the targeted VSI
530  * @fd_data: the flow director data required for the FDir descriptor
531  * @add: true adds a filter, false removes it
532  * @ipv4: true is v4, false is v6
533  *
534  * Returns 0 if the filters were successfully added or removed
535  **/
536 static int i40e_add_del_fdir_ip(struct i40e_vsi *vsi,
537 				struct i40e_fdir_filter *fd_data,
538 				bool add,
539 				bool ipv4)
540 {
541 	struct i40e_pf *pf = vsi->back;
542 	int payload_offset;
543 	u8 *raw_packet;
544 	int iter_start;
545 	int iter_end;
546 	int ret;
547 	int i;
548 
549 	if (ipv4) {
550 		iter_start = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
551 		iter_end = I40E_FILTER_PCTYPE_FRAG_IPV4;
552 	} else {
553 		iter_start = I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
554 		iter_end = I40E_FILTER_PCTYPE_FRAG_IPV6;
555 	}
556 
557 	for (i = iter_start; i <= iter_end; i++) {
558 		raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
559 		if (!raw_packet)
560 			return -ENOMEM;
561 
562 		/* IPv6 no header option differs from IPv4 */
563 		(void)i40e_create_dummy_packet
564 			(raw_packet, ipv4, (ipv4) ? IPPROTO_IP : IPPROTO_NONE,
565 			 fd_data);
566 
567 		payload_offset = (ipv4) ? I40E_IP_DUMMY_PACKET_LEN :
568 			I40E_IP6_DUMMY_PACKET_LEN;
569 		ret = i40e_prepare_fdir_filter(pf, fd_data, add, raw_packet,
570 					       payload_offset, i);
571 		if (ret)
572 			goto err;
573 	}
574 
575 	i40e_change_filter_num(ipv4, add, &pf->fd_ip4_filter_cnt,
576 			       &pf->fd_ip6_filter_cnt);
577 
578 	return 0;
579 err:
580 	kfree(raw_packet);
581 	return ret;
582 }
583 
584 /**
585  * i40e_add_del_fdir - Build raw packets to add/del fdir filter
586  * @vsi: pointer to the targeted VSI
587  * @input: filter to add or delete
588  * @add: true adds a filter, false removes it
589  *
590  **/
591 int i40e_add_del_fdir(struct i40e_vsi *vsi,
592 		      struct i40e_fdir_filter *input, bool add)
593 {
594 	enum ip_ver { ipv6 = 0, ipv4 = 1 };
595 	struct i40e_pf *pf = vsi->back;
596 	int ret;
597 
598 	switch (input->flow_type & ~FLOW_EXT) {
599 	case TCP_V4_FLOW:
600 		ret = i40e_add_del_fdir_tcp(vsi, input, add, ipv4);
601 		break;
602 	case UDP_V4_FLOW:
603 		ret = i40e_add_del_fdir_udp(vsi, input, add, ipv4);
604 		break;
605 	case SCTP_V4_FLOW:
606 		ret = i40e_add_del_fdir_sctp(vsi, input, add, ipv4);
607 		break;
608 	case TCP_V6_FLOW:
609 		ret = i40e_add_del_fdir_tcp(vsi, input, add, ipv6);
610 		break;
611 	case UDP_V6_FLOW:
612 		ret = i40e_add_del_fdir_udp(vsi, input, add, ipv6);
613 		break;
614 	case SCTP_V6_FLOW:
615 		ret = i40e_add_del_fdir_sctp(vsi, input, add, ipv6);
616 		break;
617 	case IP_USER_FLOW:
618 		switch (input->ipl4_proto) {
619 		case IPPROTO_TCP:
620 			ret = i40e_add_del_fdir_tcp(vsi, input, add, ipv4);
621 			break;
622 		case IPPROTO_UDP:
623 			ret = i40e_add_del_fdir_udp(vsi, input, add, ipv4);
624 			break;
625 		case IPPROTO_SCTP:
626 			ret = i40e_add_del_fdir_sctp(vsi, input, add, ipv4);
627 			break;
628 		case IPPROTO_IP:
629 			ret = i40e_add_del_fdir_ip(vsi, input, add, ipv4);
630 			break;
631 		default:
632 			/* We cannot support masking based on protocol */
633 			dev_info(&pf->pdev->dev, "Unsupported IPv4 protocol 0x%02x\n",
634 				 input->ipl4_proto);
635 			return -EINVAL;
636 		}
637 		break;
638 	case IPV6_USER_FLOW:
639 		switch (input->ipl4_proto) {
640 		case IPPROTO_TCP:
641 			ret = i40e_add_del_fdir_tcp(vsi, input, add, ipv6);
642 			break;
643 		case IPPROTO_UDP:
644 			ret = i40e_add_del_fdir_udp(vsi, input, add, ipv6);
645 			break;
646 		case IPPROTO_SCTP:
647 			ret = i40e_add_del_fdir_sctp(vsi, input, add, ipv6);
648 			break;
649 		case IPPROTO_IP:
650 			ret = i40e_add_del_fdir_ip(vsi, input, add, ipv6);
651 			break;
652 		default:
653 			/* We cannot support masking based on protocol */
654 			dev_info(&pf->pdev->dev, "Unsupported IPv6 protocol 0x%02x\n",
655 				 input->ipl4_proto);
656 			return -EINVAL;
657 		}
658 		break;
659 	default:
660 		dev_info(&pf->pdev->dev, "Unsupported flow type 0x%02x\n",
661 			 input->flow_type);
662 		return -EINVAL;
663 	}
664 
665 	/* The buffer allocated here will be normally be freed by
666 	 * i40e_clean_fdir_tx_irq() as it reclaims resources after transmit
667 	 * completion. In the event of an error adding the buffer to the FDIR
668 	 * ring, it will immediately be freed. It may also be freed by
669 	 * i40e_clean_tx_ring() when closing the VSI.
670 	 */
671 	return ret;
672 }
673 
674 /**
675  * i40e_fd_handle_status - check the Programming Status for FD
676  * @rx_ring: the Rx ring for this descriptor
677  * @qword0_raw: qword0
678  * @qword1: qword1 after le_to_cpu
679  * @prog_id: the id originally used for programming
680  *
681  * This is used to verify if the FD programming or invalidation
682  * requested by SW to the HW is successful or not and take actions accordingly.
683  **/
684 static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u64 qword0_raw,
685 				  u64 qword1, u8 prog_id)
686 {
687 	struct i40e_pf *pf = rx_ring->vsi->back;
688 	struct pci_dev *pdev = pf->pdev;
689 	struct i40e_16b_rx_wb_qw0 *qw0;
690 	u32 fcnt_prog, fcnt_avail;
691 	u32 error;
692 
693 	qw0 = (struct i40e_16b_rx_wb_qw0 *)&qword0_raw;
694 	error = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
695 		I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
696 
697 	if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
698 		pf->fd_inv = le32_to_cpu(qw0->hi_dword.fd_id);
699 		if (qw0->hi_dword.fd_id != 0 ||
700 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
701 			dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
702 				 pf->fd_inv);
703 
704 		/* Check if the programming error is for ATR.
705 		 * If so, auto disable ATR and set a state for
706 		 * flush in progress. Next time we come here if flush is in
707 		 * progress do nothing, once flush is complete the state will
708 		 * be cleared.
709 		 */
710 		if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
711 			return;
712 
713 		pf->fd_add_err++;
714 		/* store the current atr filter count */
715 		pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf);
716 
717 		if (qw0->hi_dword.fd_id == 0 &&
718 		    test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) {
719 			/* These set_bit() calls aren't atomic with the
720 			 * test_bit() here, but worse case we potentially
721 			 * disable ATR and queue a flush right after SB
722 			 * support is re-enabled. That shouldn't cause an
723 			 * issue in practice
724 			 */
725 			set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
726 			set_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
727 		}
728 
729 		/* filter programming failed most likely due to table full */
730 		fcnt_prog = i40e_get_global_fd_count(pf);
731 		fcnt_avail = pf->fdir_pf_filter_count;
732 		/* If ATR is running fcnt_prog can quickly change,
733 		 * if we are very close to full, it makes sense to disable
734 		 * FD ATR/SB and then re-enable it when there is room.
735 		 */
736 		if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
737 			if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
738 			    !test_and_set_bit(__I40E_FD_SB_AUTO_DISABLED,
739 					      pf->state))
740 				if (I40E_DEBUG_FD & pf->hw.debug_mask)
741 					dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
742 		}
743 	} else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
744 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
745 			dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n",
746 				 qw0->hi_dword.fd_id);
747 	}
748 }
749 
750 /**
751  * i40e_unmap_and_free_tx_resource - Release a Tx buffer
752  * @ring:      the ring that owns the buffer
753  * @tx_buffer: the buffer to free
754  **/
755 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
756 					    struct i40e_tx_buffer *tx_buffer)
757 {
758 	if (tx_buffer->skb) {
759 		if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
760 			kfree(tx_buffer->raw_buf);
761 		else if (ring_is_xdp(ring))
762 			xdp_return_frame(tx_buffer->xdpf);
763 		else
764 			dev_kfree_skb_any(tx_buffer->skb);
765 		if (dma_unmap_len(tx_buffer, len))
766 			dma_unmap_single(ring->dev,
767 					 dma_unmap_addr(tx_buffer, dma),
768 					 dma_unmap_len(tx_buffer, len),
769 					 DMA_TO_DEVICE);
770 	} else if (dma_unmap_len(tx_buffer, len)) {
771 		dma_unmap_page(ring->dev,
772 			       dma_unmap_addr(tx_buffer, dma),
773 			       dma_unmap_len(tx_buffer, len),
774 			       DMA_TO_DEVICE);
775 	}
776 
777 	tx_buffer->next_to_watch = NULL;
778 	tx_buffer->skb = NULL;
779 	dma_unmap_len_set(tx_buffer, len, 0);
780 	/* tx_buffer must be completely set up in the transmit path */
781 }
782 
783 /**
784  * i40e_clean_tx_ring - Free any empty Tx buffers
785  * @tx_ring: ring to be cleaned
786  **/
787 void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
788 {
789 	unsigned long bi_size;
790 	u16 i;
791 
792 	if (ring_is_xdp(tx_ring) && tx_ring->xsk_pool) {
793 		i40e_xsk_clean_tx_ring(tx_ring);
794 	} else {
795 		/* ring already cleared, nothing to do */
796 		if (!tx_ring->tx_bi)
797 			return;
798 
799 		/* Free all the Tx ring sk_buffs */
800 		for (i = 0; i < tx_ring->count; i++)
801 			i40e_unmap_and_free_tx_resource(tx_ring,
802 							&tx_ring->tx_bi[i]);
803 	}
804 
805 	bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
806 	memset(tx_ring->tx_bi, 0, bi_size);
807 
808 	/* Zero out the descriptor ring */
809 	memset(tx_ring->desc, 0, tx_ring->size);
810 
811 	tx_ring->next_to_use = 0;
812 	tx_ring->next_to_clean = 0;
813 
814 	if (!tx_ring->netdev)
815 		return;
816 
817 	/* cleanup Tx queue statistics */
818 	netdev_tx_reset_queue(txring_txq(tx_ring));
819 }
820 
821 /**
822  * i40e_free_tx_resources - Free Tx resources per queue
823  * @tx_ring: Tx descriptor ring for a specific queue
824  *
825  * Free all transmit software resources
826  **/
827 void i40e_free_tx_resources(struct i40e_ring *tx_ring)
828 {
829 	i40e_clean_tx_ring(tx_ring);
830 	kfree(tx_ring->tx_bi);
831 	tx_ring->tx_bi = NULL;
832 
833 	if (tx_ring->desc) {
834 		dma_free_coherent(tx_ring->dev, tx_ring->size,
835 				  tx_ring->desc, tx_ring->dma);
836 		tx_ring->desc = NULL;
837 	}
838 }
839 
840 /**
841  * i40e_get_tx_pending - how many tx descriptors not processed
842  * @ring: the ring of descriptors
843  * @in_sw: use SW variables
844  *
845  * Since there is no access to the ring head register
846  * in XL710, we need to use our local copies
847  **/
848 u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw)
849 {
850 	u32 head, tail;
851 
852 	if (!in_sw) {
853 		head = i40e_get_head(ring);
854 		tail = readl(ring->tail);
855 	} else {
856 		head = ring->next_to_clean;
857 		tail = ring->next_to_use;
858 	}
859 
860 	if (head != tail)
861 		return (head < tail) ?
862 			tail - head : (tail + ring->count - head);
863 
864 	return 0;
865 }
866 
867 /**
868  * i40e_detect_recover_hung - Function to detect and recover hung_queues
869  * @vsi:  pointer to vsi struct with tx queues
870  *
871  * VSI has netdev and netdev has TX queues. This function is to check each of
872  * those TX queues if they are hung, trigger recovery by issuing SW interrupt.
873  **/
874 void i40e_detect_recover_hung(struct i40e_vsi *vsi)
875 {
876 	struct i40e_ring *tx_ring = NULL;
877 	struct net_device *netdev;
878 	unsigned int i;
879 	int packets;
880 
881 	if (!vsi)
882 		return;
883 
884 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
885 		return;
886 
887 	netdev = vsi->netdev;
888 	if (!netdev)
889 		return;
890 
891 	if (!netif_carrier_ok(netdev))
892 		return;
893 
894 	for (i = 0; i < vsi->num_queue_pairs; i++) {
895 		tx_ring = vsi->tx_rings[i];
896 		if (tx_ring && tx_ring->desc) {
897 			/* If packet counter has not changed the queue is
898 			 * likely stalled, so force an interrupt for this
899 			 * queue.
900 			 *
901 			 * prev_pkt_ctr would be negative if there was no
902 			 * pending work.
903 			 */
904 			packets = tx_ring->stats.packets & INT_MAX;
905 			if (tx_ring->tx_stats.prev_pkt_ctr == packets) {
906 				i40e_force_wb(vsi, tx_ring->q_vector);
907 				continue;
908 			}
909 
910 			/* Memory barrier between read of packet count and call
911 			 * to i40e_get_tx_pending()
912 			 */
913 			smp_rmb();
914 			tx_ring->tx_stats.prev_pkt_ctr =
915 			    i40e_get_tx_pending(tx_ring, true) ? packets : -1;
916 		}
917 	}
918 }
919 
920 /**
921  * i40e_clean_tx_irq - Reclaim resources after transmit completes
922  * @vsi: the VSI we care about
923  * @tx_ring: Tx ring to clean
924  * @napi_budget: Used to determine if we are in netpoll
925  * @tx_cleaned: Out parameter set to the number of TXes cleaned
926  *
927  * Returns true if there's any budget left (e.g. the clean is finished)
928  **/
929 static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
930 			      struct i40e_ring *tx_ring, int napi_budget,
931 			      unsigned int *tx_cleaned)
932 {
933 	int i = tx_ring->next_to_clean;
934 	struct i40e_tx_buffer *tx_buf;
935 	struct i40e_tx_desc *tx_head;
936 	struct i40e_tx_desc *tx_desc;
937 	unsigned int total_bytes = 0, total_packets = 0;
938 	unsigned int budget = vsi->work_limit;
939 
940 	tx_buf = &tx_ring->tx_bi[i];
941 	tx_desc = I40E_TX_DESC(tx_ring, i);
942 	i -= tx_ring->count;
943 
944 	tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
945 
946 	do {
947 		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
948 
949 		/* if next_to_watch is not set then there is no work pending */
950 		if (!eop_desc)
951 			break;
952 
953 		/* prevent any other reads prior to eop_desc */
954 		smp_rmb();
955 
956 		i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
957 		/* we have caught up to head, no work left to do */
958 		if (tx_head == tx_desc)
959 			break;
960 
961 		/* clear next_to_watch to prevent false hangs */
962 		tx_buf->next_to_watch = NULL;
963 
964 		/* update the statistics for this packet */
965 		total_bytes += tx_buf->bytecount;
966 		total_packets += tx_buf->gso_segs;
967 
968 		/* free the skb/XDP data */
969 		if (ring_is_xdp(tx_ring))
970 			xdp_return_frame(tx_buf->xdpf);
971 		else
972 			napi_consume_skb(tx_buf->skb, napi_budget);
973 
974 		/* unmap skb header data */
975 		dma_unmap_single(tx_ring->dev,
976 				 dma_unmap_addr(tx_buf, dma),
977 				 dma_unmap_len(tx_buf, len),
978 				 DMA_TO_DEVICE);
979 
980 		/* clear tx_buffer data */
981 		tx_buf->skb = NULL;
982 		dma_unmap_len_set(tx_buf, len, 0);
983 
984 		/* unmap remaining buffers */
985 		while (tx_desc != eop_desc) {
986 			i40e_trace(clean_tx_irq_unmap,
987 				   tx_ring, tx_desc, tx_buf);
988 
989 			tx_buf++;
990 			tx_desc++;
991 			i++;
992 			if (unlikely(!i)) {
993 				i -= tx_ring->count;
994 				tx_buf = tx_ring->tx_bi;
995 				tx_desc = I40E_TX_DESC(tx_ring, 0);
996 			}
997 
998 			/* unmap any remaining paged data */
999 			if (dma_unmap_len(tx_buf, len)) {
1000 				dma_unmap_page(tx_ring->dev,
1001 					       dma_unmap_addr(tx_buf, dma),
1002 					       dma_unmap_len(tx_buf, len),
1003 					       DMA_TO_DEVICE);
1004 				dma_unmap_len_set(tx_buf, len, 0);
1005 			}
1006 		}
1007 
1008 		/* move us one more past the eop_desc for start of next pkt */
1009 		tx_buf++;
1010 		tx_desc++;
1011 		i++;
1012 		if (unlikely(!i)) {
1013 			i -= tx_ring->count;
1014 			tx_buf = tx_ring->tx_bi;
1015 			tx_desc = I40E_TX_DESC(tx_ring, 0);
1016 		}
1017 
1018 		prefetch(tx_desc);
1019 
1020 		/* update budget accounting */
1021 		budget--;
1022 	} while (likely(budget));
1023 
1024 	i += tx_ring->count;
1025 	tx_ring->next_to_clean = i;
1026 	i40e_update_tx_stats(tx_ring, total_packets, total_bytes);
1027 	i40e_arm_wb(tx_ring, vsi, budget);
1028 
1029 	if (ring_is_xdp(tx_ring))
1030 		return !!budget;
1031 
1032 	/* notify netdev of completed buffers */
1033 	netdev_tx_completed_queue(txring_txq(tx_ring),
1034 				  total_packets, total_bytes);
1035 
1036 #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
1037 	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
1038 		     (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
1039 		/* Make sure that anybody stopping the queue after this
1040 		 * sees the new next_to_clean.
1041 		 */
1042 		smp_mb();
1043 		if (__netif_subqueue_stopped(tx_ring->netdev,
1044 					     tx_ring->queue_index) &&
1045 		   !test_bit(__I40E_VSI_DOWN, vsi->state)) {
1046 			netif_wake_subqueue(tx_ring->netdev,
1047 					    tx_ring->queue_index);
1048 			++tx_ring->tx_stats.restart_queue;
1049 		}
1050 	}
1051 
1052 	*tx_cleaned = total_packets;
1053 	return !!budget;
1054 }
1055 
1056 /**
1057  * i40e_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled
1058  * @vsi: the VSI we care about
1059  * @q_vector: the vector on which to enable writeback
1060  *
1061  **/
1062 static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
1063 				  struct i40e_q_vector *q_vector)
1064 {
1065 	u16 flags = q_vector->tx.ring[0].flags;
1066 	u32 val;
1067 
1068 	if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
1069 		return;
1070 
1071 	if (q_vector->arm_wb_state)
1072 		return;
1073 
1074 	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1075 		val = I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK |
1076 		      I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */
1077 
1078 		wr32(&vsi->back->hw,
1079 		     I40E_PFINT_DYN_CTLN(q_vector->reg_idx),
1080 		     val);
1081 	} else {
1082 		val = I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK |
1083 		      I40E_PFINT_DYN_CTL0_ITR_INDX_MASK; /* set noitr */
1084 
1085 		wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val);
1086 	}
1087 	q_vector->arm_wb_state = true;
1088 }
1089 
1090 /**
1091  * i40e_force_wb - Issue SW Interrupt so HW does a wb
1092  * @vsi: the VSI we care about
1093  * @q_vector: the vector  on which to force writeback
1094  *
1095  **/
1096 void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
1097 {
1098 	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1099 		u32 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
1100 			  I40E_PFINT_DYN_CTLN_ITR_INDX_MASK | /* set noitr */
1101 			  I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK |
1102 			  I40E_PFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK;
1103 			  /* allow 00 to be written to the index */
1104 
1105 		wr32(&vsi->back->hw,
1106 		     I40E_PFINT_DYN_CTLN(q_vector->reg_idx), val);
1107 	} else {
1108 		u32 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
1109 			  I40E_PFINT_DYN_CTL0_ITR_INDX_MASK | /* set noitr */
1110 			  I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK |
1111 			  I40E_PFINT_DYN_CTL0_SW_ITR_INDX_ENA_MASK;
1112 			/* allow 00 to be written to the index */
1113 
1114 		wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val);
1115 	}
1116 }
1117 
1118 static inline bool i40e_container_is_rx(struct i40e_q_vector *q_vector,
1119 					struct i40e_ring_container *rc)
1120 {
1121 	return &q_vector->rx == rc;
1122 }
1123 
1124 static inline unsigned int i40e_itr_divisor(struct i40e_q_vector *q_vector)
1125 {
1126 	unsigned int divisor;
1127 
1128 	switch (q_vector->vsi->back->hw.phy.link_info.link_speed) {
1129 	case I40E_LINK_SPEED_40GB:
1130 		divisor = I40E_ITR_ADAPTIVE_MIN_INC * 1024;
1131 		break;
1132 	case I40E_LINK_SPEED_25GB:
1133 	case I40E_LINK_SPEED_20GB:
1134 		divisor = I40E_ITR_ADAPTIVE_MIN_INC * 512;
1135 		break;
1136 	default:
1137 	case I40E_LINK_SPEED_10GB:
1138 		divisor = I40E_ITR_ADAPTIVE_MIN_INC * 256;
1139 		break;
1140 	case I40E_LINK_SPEED_1GB:
1141 	case I40E_LINK_SPEED_100MB:
1142 		divisor = I40E_ITR_ADAPTIVE_MIN_INC * 32;
1143 		break;
1144 	}
1145 
1146 	return divisor;
1147 }
1148 
1149 /**
1150  * i40e_update_itr - update the dynamic ITR value based on statistics
1151  * @q_vector: structure containing interrupt and ring information
1152  * @rc: structure containing ring performance data
1153  *
1154  * Stores a new ITR value based on packets and byte
1155  * counts during the last interrupt.  The advantage of per interrupt
1156  * computation is faster updates and more accurate ITR for the current
1157  * traffic pattern.  Constants in this function were computed
1158  * based on theoretical maximum wire speed and thresholds were set based
1159  * on testing data as well as attempting to minimize response time
1160  * while increasing bulk throughput.
1161  **/
1162 static void i40e_update_itr(struct i40e_q_vector *q_vector,
1163 			    struct i40e_ring_container *rc)
1164 {
1165 	unsigned int avg_wire_size, packets, bytes, itr;
1166 	unsigned long next_update = jiffies;
1167 
1168 	/* If we don't have any rings just leave ourselves set for maximum
1169 	 * possible latency so we take ourselves out of the equation.
1170 	 */
1171 	if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting))
1172 		return;
1173 
1174 	/* For Rx we want to push the delay up and default to low latency.
1175 	 * for Tx we want to pull the delay down and default to high latency.
1176 	 */
1177 	itr = i40e_container_is_rx(q_vector, rc) ?
1178 	      I40E_ITR_ADAPTIVE_MIN_USECS | I40E_ITR_ADAPTIVE_LATENCY :
1179 	      I40E_ITR_ADAPTIVE_MAX_USECS | I40E_ITR_ADAPTIVE_LATENCY;
1180 
1181 	/* If we didn't update within up to 1 - 2 jiffies we can assume
1182 	 * that either packets are coming in so slow there hasn't been
1183 	 * any work, or that there is so much work that NAPI is dealing
1184 	 * with interrupt moderation and we don't need to do anything.
1185 	 */
1186 	if (time_after(next_update, rc->next_update))
1187 		goto clear_counts;
1188 
1189 	/* If itr_countdown is set it means we programmed an ITR within
1190 	 * the last 4 interrupt cycles. This has a side effect of us
1191 	 * potentially firing an early interrupt. In order to work around
1192 	 * this we need to throw out any data received for a few
1193 	 * interrupts following the update.
1194 	 */
1195 	if (q_vector->itr_countdown) {
1196 		itr = rc->target_itr;
1197 		goto clear_counts;
1198 	}
1199 
1200 	packets = rc->total_packets;
1201 	bytes = rc->total_bytes;
1202 
1203 	if (i40e_container_is_rx(q_vector, rc)) {
1204 		/* If Rx there are 1 to 4 packets and bytes are less than
1205 		 * 9000 assume insufficient data to use bulk rate limiting
1206 		 * approach unless Tx is already in bulk rate limiting. We
1207 		 * are likely latency driven.
1208 		 */
1209 		if (packets && packets < 4 && bytes < 9000 &&
1210 		    (q_vector->tx.target_itr & I40E_ITR_ADAPTIVE_LATENCY)) {
1211 			itr = I40E_ITR_ADAPTIVE_LATENCY;
1212 			goto adjust_by_size;
1213 		}
1214 	} else if (packets < 4) {
1215 		/* If we have Tx and Rx ITR maxed and Tx ITR is running in
1216 		 * bulk mode and we are receiving 4 or fewer packets just
1217 		 * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so
1218 		 * that the Rx can relax.
1219 		 */
1220 		if (rc->target_itr == I40E_ITR_ADAPTIVE_MAX_USECS &&
1221 		    (q_vector->rx.target_itr & I40E_ITR_MASK) ==
1222 		     I40E_ITR_ADAPTIVE_MAX_USECS)
1223 			goto clear_counts;
1224 	} else if (packets > 32) {
1225 		/* If we have processed over 32 packets in a single interrupt
1226 		 * for Tx assume we need to switch over to "bulk" mode.
1227 		 */
1228 		rc->target_itr &= ~I40E_ITR_ADAPTIVE_LATENCY;
1229 	}
1230 
1231 	/* We have no packets to actually measure against. This means
1232 	 * either one of the other queues on this vector is active or
1233 	 * we are a Tx queue doing TSO with too high of an interrupt rate.
1234 	 *
1235 	 * Between 4 and 56 we can assume that our current interrupt delay
1236 	 * is only slightly too low. As such we should increase it by a small
1237 	 * fixed amount.
1238 	 */
1239 	if (packets < 56) {
1240 		itr = rc->target_itr + I40E_ITR_ADAPTIVE_MIN_INC;
1241 		if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
1242 			itr &= I40E_ITR_ADAPTIVE_LATENCY;
1243 			itr += I40E_ITR_ADAPTIVE_MAX_USECS;
1244 		}
1245 		goto clear_counts;
1246 	}
1247 
1248 	if (packets <= 256) {
1249 		itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);
1250 		itr &= I40E_ITR_MASK;
1251 
1252 		/* Between 56 and 112 is our "goldilocks" zone where we are
1253 		 * working out "just right". Just report that our current
1254 		 * ITR is good for us.
1255 		 */
1256 		if (packets <= 112)
1257 			goto clear_counts;
1258 
1259 		/* If packet count is 128 or greater we are likely looking
1260 		 * at a slight overrun of the delay we want. Try halving
1261 		 * our delay to see if that will cut the number of packets
1262 		 * in half per interrupt.
1263 		 */
1264 		itr /= 2;
1265 		itr &= I40E_ITR_MASK;
1266 		if (itr < I40E_ITR_ADAPTIVE_MIN_USECS)
1267 			itr = I40E_ITR_ADAPTIVE_MIN_USECS;
1268 
1269 		goto clear_counts;
1270 	}
1271 
1272 	/* The paths below assume we are dealing with a bulk ITR since
1273 	 * number of packets is greater than 256. We are just going to have
1274 	 * to compute a value and try to bring the count under control,
1275 	 * though for smaller packet sizes there isn't much we can do as
1276 	 * NAPI polling will likely be kicking in sooner rather than later.
1277 	 */
1278 	itr = I40E_ITR_ADAPTIVE_BULK;
1279 
1280 adjust_by_size:
1281 	/* If packet counts are 256 or greater we can assume we have a gross
1282 	 * overestimation of what the rate should be. Instead of trying to fine
1283 	 * tune it just use the formula below to try and dial in an exact value
1284 	 * give the current packet size of the frame.
1285 	 */
1286 	avg_wire_size = bytes / packets;
1287 
1288 	/* The following is a crude approximation of:
1289 	 *  wmem_default / (size + overhead) = desired_pkts_per_int
1290 	 *  rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
1291 	 *  (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
1292 	 *
1293 	 * Assuming wmem_default is 212992 and overhead is 640 bytes per
1294 	 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
1295 	 * formula down to
1296 	 *
1297 	 *  (170 * (size + 24)) / (size + 640) = ITR
1298 	 *
1299 	 * We first do some math on the packet size and then finally bitshift
1300 	 * by 8 after rounding up. We also have to account for PCIe link speed
1301 	 * difference as ITR scales based on this.
1302 	 */
1303 	if (avg_wire_size <= 60) {
1304 		/* Start at 250k ints/sec */
1305 		avg_wire_size = 4096;
1306 	} else if (avg_wire_size <= 380) {
1307 		/* 250K ints/sec to 60K ints/sec */
1308 		avg_wire_size *= 40;
1309 		avg_wire_size += 1696;
1310 	} else if (avg_wire_size <= 1084) {
1311 		/* 60K ints/sec to 36K ints/sec */
1312 		avg_wire_size *= 15;
1313 		avg_wire_size += 11452;
1314 	} else if (avg_wire_size <= 1980) {
1315 		/* 36K ints/sec to 30K ints/sec */
1316 		avg_wire_size *= 5;
1317 		avg_wire_size += 22420;
1318 	} else {
1319 		/* plateau at a limit of 30K ints/sec */
1320 		avg_wire_size = 32256;
1321 	}
1322 
1323 	/* If we are in low latency mode halve our delay which doubles the
1324 	 * rate to somewhere between 100K to 16K ints/sec
1325 	 */
1326 	if (itr & I40E_ITR_ADAPTIVE_LATENCY)
1327 		avg_wire_size /= 2;
1328 
1329 	/* Resultant value is 256 times larger than it needs to be. This
1330 	 * gives us room to adjust the value as needed to either increase
1331 	 * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
1332 	 *
1333 	 * Use addition as we have already recorded the new latency flag
1334 	 * for the ITR value.
1335 	 */
1336 	itr += DIV_ROUND_UP(avg_wire_size, i40e_itr_divisor(q_vector)) *
1337 	       I40E_ITR_ADAPTIVE_MIN_INC;
1338 
1339 	if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
1340 		itr &= I40E_ITR_ADAPTIVE_LATENCY;
1341 		itr += I40E_ITR_ADAPTIVE_MAX_USECS;
1342 	}
1343 
1344 clear_counts:
1345 	/* write back value */
1346 	rc->target_itr = itr;
1347 
1348 	/* next update should occur within next jiffy */
1349 	rc->next_update = next_update + 1;
1350 
1351 	rc->total_bytes = 0;
1352 	rc->total_packets = 0;
1353 }
1354 
1355 static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
1356 {
1357 	return &rx_ring->rx_bi[idx];
1358 }
1359 
1360 /**
1361  * i40e_reuse_rx_page - page flip buffer and store it back on the ring
1362  * @rx_ring: rx descriptor ring to store buffers on
1363  * @old_buff: donor buffer to have page reused
1364  *
1365  * Synchronizes page for reuse by the adapter
1366  **/
1367 static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
1368 			       struct i40e_rx_buffer *old_buff)
1369 {
1370 	struct i40e_rx_buffer *new_buff;
1371 	u16 nta = rx_ring->next_to_alloc;
1372 
1373 	new_buff = i40e_rx_bi(rx_ring, nta);
1374 
1375 	/* update, and store next to alloc */
1376 	nta++;
1377 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1378 
1379 	/* transfer page from old buffer to new buffer */
1380 	new_buff->dma		= old_buff->dma;
1381 	new_buff->page		= old_buff->page;
1382 	new_buff->page_offset	= old_buff->page_offset;
1383 	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
1384 
1385 	/* clear contents of buffer_info */
1386 	old_buff->page = NULL;
1387 }
1388 
1389 /**
1390  * i40e_clean_programming_status - clean the programming status descriptor
1391  * @rx_ring: the rx ring that has this descriptor
1392  * @qword0_raw: qword0
1393  * @qword1: qword1 representing status_error_len in CPU ordering
1394  *
1395  * Flow director should handle FD_FILTER_STATUS to check its filter programming
1396  * status being successful or not and take actions accordingly. FCoE should
1397  * handle its context/filter programming/invalidation status and take actions.
1398  *
1399  * Returns an i40e_rx_buffer to reuse if the cleanup occurred, otherwise NULL.
1400  **/
1401 void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw,
1402 				   u64 qword1)
1403 {
1404 	u8 id;
1405 
1406 	id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
1407 		  I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
1408 
1409 	if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
1410 		i40e_fd_handle_status(rx_ring, qword0_raw, qword1, id);
1411 }
1412 
1413 /**
1414  * i40e_setup_tx_descriptors - Allocate the Tx descriptors
1415  * @tx_ring: the tx ring to set up
1416  *
1417  * Return 0 on success, negative on error
1418  **/
1419 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
1420 {
1421 	struct device *dev = tx_ring->dev;
1422 	int bi_size;
1423 
1424 	if (!dev)
1425 		return -ENOMEM;
1426 
1427 	/* warn if we are about to overwrite the pointer */
1428 	WARN_ON(tx_ring->tx_bi);
1429 	bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
1430 	tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
1431 	if (!tx_ring->tx_bi)
1432 		goto err;
1433 
1434 	u64_stats_init(&tx_ring->syncp);
1435 
1436 	/* round up to nearest 4K */
1437 	tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
1438 	/* add u32 for head writeback, align after this takes care of
1439 	 * guaranteeing this is at least one cache line in size
1440 	 */
1441 	tx_ring->size += sizeof(u32);
1442 	tx_ring->size = ALIGN(tx_ring->size, 4096);
1443 	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
1444 					   &tx_ring->dma, GFP_KERNEL);
1445 	if (!tx_ring->desc) {
1446 		dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
1447 			 tx_ring->size);
1448 		goto err;
1449 	}
1450 
1451 	tx_ring->next_to_use = 0;
1452 	tx_ring->next_to_clean = 0;
1453 	tx_ring->tx_stats.prev_pkt_ctr = -1;
1454 	return 0;
1455 
1456 err:
1457 	kfree(tx_ring->tx_bi);
1458 	tx_ring->tx_bi = NULL;
1459 	return -ENOMEM;
1460 }
1461 
1462 static void i40e_clear_rx_bi(struct i40e_ring *rx_ring)
1463 {
1464 	memset(rx_ring->rx_bi, 0, sizeof(*rx_ring->rx_bi) * rx_ring->count);
1465 }
1466 
1467 /**
1468  * i40e_clean_rx_ring - Free Rx buffers
1469  * @rx_ring: ring to be cleaned
1470  **/
1471 void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
1472 {
1473 	u16 i;
1474 
1475 	/* ring already cleared, nothing to do */
1476 	if (!rx_ring->rx_bi)
1477 		return;
1478 
1479 	if (rx_ring->xsk_pool) {
1480 		i40e_xsk_clean_rx_ring(rx_ring);
1481 		goto skip_free;
1482 	}
1483 
1484 	/* Free all the Rx ring sk_buffs */
1485 	for (i = 0; i < rx_ring->count; i++) {
1486 		struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i);
1487 
1488 		if (!rx_bi->page)
1489 			continue;
1490 
1491 		/* Invalidate cache lines that may have been written to by
1492 		 * device so that we avoid corrupting memory.
1493 		 */
1494 		dma_sync_single_range_for_cpu(rx_ring->dev,
1495 					      rx_bi->dma,
1496 					      rx_bi->page_offset,
1497 					      rx_ring->rx_buf_len,
1498 					      DMA_FROM_DEVICE);
1499 
1500 		/* free resources associated with mapping */
1501 		dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
1502 				     i40e_rx_pg_size(rx_ring),
1503 				     DMA_FROM_DEVICE,
1504 				     I40E_RX_DMA_ATTR);
1505 
1506 		__page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias);
1507 
1508 		rx_bi->page = NULL;
1509 		rx_bi->page_offset = 0;
1510 	}
1511 
1512 skip_free:
1513 	if (rx_ring->xsk_pool)
1514 		i40e_clear_rx_bi_zc(rx_ring);
1515 	else
1516 		i40e_clear_rx_bi(rx_ring);
1517 
1518 	/* Zero out the descriptor ring */
1519 	memset(rx_ring->desc, 0, rx_ring->size);
1520 
1521 	rx_ring->next_to_alloc = 0;
1522 	rx_ring->next_to_clean = 0;
1523 	rx_ring->next_to_process = 0;
1524 	rx_ring->next_to_use = 0;
1525 }
1526 
1527 /**
1528  * i40e_free_rx_resources - Free Rx resources
1529  * @rx_ring: ring to clean the resources from
1530  *
1531  * Free all receive software resources
1532  **/
1533 void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1534 {
1535 	i40e_clean_rx_ring(rx_ring);
1536 	if (rx_ring->vsi->type == I40E_VSI_MAIN)
1537 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
1538 	rx_ring->xdp_prog = NULL;
1539 	kfree(rx_ring->rx_bi);
1540 	rx_ring->rx_bi = NULL;
1541 
1542 	if (rx_ring->desc) {
1543 		dma_free_coherent(rx_ring->dev, rx_ring->size,
1544 				  rx_ring->desc, rx_ring->dma);
1545 		rx_ring->desc = NULL;
1546 	}
1547 }
1548 
1549 /**
1550  * i40e_setup_rx_descriptors - Allocate Rx descriptors
1551  * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1552  *
1553  * Returns 0 on success, negative on failure
1554  **/
1555 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1556 {
1557 	struct device *dev = rx_ring->dev;
1558 
1559 	u64_stats_init(&rx_ring->syncp);
1560 
1561 	/* Round up to nearest 4K */
1562 	rx_ring->size = rx_ring->count * sizeof(union i40e_rx_desc);
1563 	rx_ring->size = ALIGN(rx_ring->size, 4096);
1564 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1565 					   &rx_ring->dma, GFP_KERNEL);
1566 
1567 	if (!rx_ring->desc) {
1568 		dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1569 			 rx_ring->size);
1570 		return -ENOMEM;
1571 	}
1572 
1573 	rx_ring->next_to_alloc = 0;
1574 	rx_ring->next_to_clean = 0;
1575 	rx_ring->next_to_process = 0;
1576 	rx_ring->next_to_use = 0;
1577 
1578 	rx_ring->xdp_prog = rx_ring->vsi->xdp_prog;
1579 
1580 	rx_ring->rx_bi =
1581 		kcalloc(rx_ring->count, sizeof(*rx_ring->rx_bi), GFP_KERNEL);
1582 	if (!rx_ring->rx_bi)
1583 		return -ENOMEM;
1584 
1585 	return 0;
1586 }
1587 
1588 /**
1589  * i40e_release_rx_desc - Store the new tail and head values
1590  * @rx_ring: ring to bump
1591  * @val: new head index
1592  **/
1593 void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1594 {
1595 	rx_ring->next_to_use = val;
1596 
1597 	/* update next to alloc since we have filled the ring */
1598 	rx_ring->next_to_alloc = val;
1599 
1600 	/* Force memory writes to complete before letting h/w
1601 	 * know there are new descriptors to fetch.  (Only
1602 	 * applicable for weak-ordered memory model archs,
1603 	 * such as IA-64).
1604 	 */
1605 	wmb();
1606 	writel(val, rx_ring->tail);
1607 }
1608 
1609 #if (PAGE_SIZE >= 8192)
1610 static unsigned int i40e_rx_frame_truesize(struct i40e_ring *rx_ring,
1611 					   unsigned int size)
1612 {
1613 	unsigned int truesize;
1614 
1615 	truesize = rx_ring->rx_offset ?
1616 		SKB_DATA_ALIGN(size + rx_ring->rx_offset) +
1617 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
1618 		SKB_DATA_ALIGN(size);
1619 	return truesize;
1620 }
1621 #endif
1622 
1623 /**
1624  * i40e_alloc_mapped_page - recycle or make a new page
1625  * @rx_ring: ring to use
1626  * @bi: rx_buffer struct to modify
1627  *
1628  * Returns true if the page was successfully allocated or
1629  * reused.
1630  **/
1631 static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
1632 				   struct i40e_rx_buffer *bi)
1633 {
1634 	struct page *page = bi->page;
1635 	dma_addr_t dma;
1636 
1637 	/* since we are recycling buffers we should seldom need to alloc */
1638 	if (likely(page)) {
1639 		rx_ring->rx_stats.page_reuse_count++;
1640 		return true;
1641 	}
1642 
1643 	/* alloc new page for storage */
1644 	page = dev_alloc_pages(i40e_rx_pg_order(rx_ring));
1645 	if (unlikely(!page)) {
1646 		rx_ring->rx_stats.alloc_page_failed++;
1647 		return false;
1648 	}
1649 
1650 	rx_ring->rx_stats.page_alloc_count++;
1651 
1652 	/* map page for use */
1653 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1654 				 i40e_rx_pg_size(rx_ring),
1655 				 DMA_FROM_DEVICE,
1656 				 I40E_RX_DMA_ATTR);
1657 
1658 	/* if mapping failed free memory back to system since
1659 	 * there isn't much point in holding memory we can't use
1660 	 */
1661 	if (dma_mapping_error(rx_ring->dev, dma)) {
1662 		__free_pages(page, i40e_rx_pg_order(rx_ring));
1663 		rx_ring->rx_stats.alloc_page_failed++;
1664 		return false;
1665 	}
1666 
1667 	bi->dma = dma;
1668 	bi->page = page;
1669 	bi->page_offset = rx_ring->rx_offset;
1670 	page_ref_add(page, USHRT_MAX - 1);
1671 	bi->pagecnt_bias = USHRT_MAX;
1672 
1673 	return true;
1674 }
1675 
1676 /**
1677  * i40e_alloc_rx_buffers - Replace used receive buffers
1678  * @rx_ring: ring to place buffers on
1679  * @cleaned_count: number of buffers to replace
1680  *
1681  * Returns false if all allocations were successful, true if any fail
1682  **/
1683 bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1684 {
1685 	u16 ntu = rx_ring->next_to_use;
1686 	union i40e_rx_desc *rx_desc;
1687 	struct i40e_rx_buffer *bi;
1688 
1689 	/* do nothing if no valid netdev defined */
1690 	if (!rx_ring->netdev || !cleaned_count)
1691 		return false;
1692 
1693 	rx_desc = I40E_RX_DESC(rx_ring, ntu);
1694 	bi = i40e_rx_bi(rx_ring, ntu);
1695 
1696 	do {
1697 		if (!i40e_alloc_mapped_page(rx_ring, bi))
1698 			goto no_buffers;
1699 
1700 		/* sync the buffer for use by the device */
1701 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
1702 						 bi->page_offset,
1703 						 rx_ring->rx_buf_len,
1704 						 DMA_FROM_DEVICE);
1705 
1706 		/* Refresh the desc even if buffer_addrs didn't change
1707 		 * because each write-back erases this info.
1708 		 */
1709 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1710 
1711 		rx_desc++;
1712 		bi++;
1713 		ntu++;
1714 		if (unlikely(ntu == rx_ring->count)) {
1715 			rx_desc = I40E_RX_DESC(rx_ring, 0);
1716 			bi = i40e_rx_bi(rx_ring, 0);
1717 			ntu = 0;
1718 		}
1719 
1720 		/* clear the status bits for the next_to_use descriptor */
1721 		rx_desc->wb.qword1.status_error_len = 0;
1722 
1723 		cleaned_count--;
1724 	} while (cleaned_count);
1725 
1726 	if (rx_ring->next_to_use != ntu)
1727 		i40e_release_rx_desc(rx_ring, ntu);
1728 
1729 	return false;
1730 
1731 no_buffers:
1732 	if (rx_ring->next_to_use != ntu)
1733 		i40e_release_rx_desc(rx_ring, ntu);
1734 
1735 	/* make sure to come back via polling to try again after
1736 	 * allocation failure
1737 	 */
1738 	return true;
1739 }
1740 
1741 /**
1742  * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1743  * @vsi: the VSI we care about
1744  * @skb: skb currently being received and modified
1745  * @rx_desc: the receive descriptor
1746  **/
1747 static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1748 				    struct sk_buff *skb,
1749 				    union i40e_rx_desc *rx_desc)
1750 {
1751 	struct i40e_rx_ptype_decoded decoded;
1752 	u32 rx_error, rx_status;
1753 	bool ipv4, ipv6;
1754 	u8 ptype;
1755 	u64 qword;
1756 
1757 	qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1758 	ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
1759 	rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1760 		   I40E_RXD_QW1_ERROR_SHIFT;
1761 	rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1762 		    I40E_RXD_QW1_STATUS_SHIFT;
1763 	decoded = decode_rx_desc_ptype(ptype);
1764 
1765 	skb->ip_summed = CHECKSUM_NONE;
1766 
1767 	skb_checksum_none_assert(skb);
1768 
1769 	/* Rx csum enabled and ip headers found? */
1770 	if (!(vsi->netdev->features & NETIF_F_RXCSUM))
1771 		return;
1772 
1773 	/* did the hardware decode the packet and checksum? */
1774 	if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1775 		return;
1776 
1777 	/* both known and outer_ip must be set for the below code to work */
1778 	if (!(decoded.known && decoded.outer_ip))
1779 		return;
1780 
1781 	ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
1782 	       (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
1783 	ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
1784 	       (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
1785 
1786 	if (ipv4 &&
1787 	    (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
1788 			 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
1789 		goto checksum_fail;
1790 
1791 	/* likely incorrect csum if alternate IP extension headers found */
1792 	if (ipv6 &&
1793 	    rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1794 		/* don't increment checksum err here, non-fatal err */
1795 		return;
1796 
1797 	/* there was some L4 error, count error and punt packet to the stack */
1798 	if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
1799 		goto checksum_fail;
1800 
1801 	/* handle packets that were not able to be checksummed due
1802 	 * to arrival speed, in this case the stack can compute
1803 	 * the csum.
1804 	 */
1805 	if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
1806 		return;
1807 
1808 	/* If there is an outer header present that might contain a checksum
1809 	 * we need to bump the checksum level by 1 to reflect the fact that
1810 	 * we are indicating we validated the inner checksum.
1811 	 */
1812 	if (decoded.tunnel_type >= I40E_RX_PTYPE_TUNNEL_IP_GRENAT)
1813 		skb->csum_level = 1;
1814 
1815 	/* Only report checksum unnecessary for TCP, UDP, or SCTP */
1816 	switch (decoded.inner_prot) {
1817 	case I40E_RX_PTYPE_INNER_PROT_TCP:
1818 	case I40E_RX_PTYPE_INNER_PROT_UDP:
1819 	case I40E_RX_PTYPE_INNER_PROT_SCTP:
1820 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1821 		fallthrough;
1822 	default:
1823 		break;
1824 	}
1825 
1826 	return;
1827 
1828 checksum_fail:
1829 	vsi->back->hw_csum_rx_error++;
1830 }
1831 
1832 /**
1833  * i40e_ptype_to_htype - get a hash type
1834  * @ptype: the ptype value from the descriptor
1835  *
1836  * Returns a hash type to be used by skb_set_hash
1837  **/
1838 static inline int i40e_ptype_to_htype(u8 ptype)
1839 {
1840 	struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1841 
1842 	if (!decoded.known)
1843 		return PKT_HASH_TYPE_NONE;
1844 
1845 	if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1846 	    decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1847 		return PKT_HASH_TYPE_L4;
1848 	else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1849 		 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1850 		return PKT_HASH_TYPE_L3;
1851 	else
1852 		return PKT_HASH_TYPE_L2;
1853 }
1854 
1855 /**
1856  * i40e_rx_hash - set the hash value in the skb
1857  * @ring: descriptor ring
1858  * @rx_desc: specific descriptor
1859  * @skb: skb currently being received and modified
1860  * @rx_ptype: Rx packet type
1861  **/
1862 static inline void i40e_rx_hash(struct i40e_ring *ring,
1863 				union i40e_rx_desc *rx_desc,
1864 				struct sk_buff *skb,
1865 				u8 rx_ptype)
1866 {
1867 	u32 hash;
1868 	const __le64 rss_mask =
1869 		cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1870 			    I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1871 
1872 	if (!(ring->netdev->features & NETIF_F_RXHASH))
1873 		return;
1874 
1875 	if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
1876 		hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1877 		skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
1878 	}
1879 }
1880 
1881 /**
1882  * i40e_process_skb_fields - Populate skb header fields from Rx descriptor
1883  * @rx_ring: rx descriptor ring packet is being transacted on
1884  * @rx_desc: pointer to the EOP Rx descriptor
1885  * @skb: pointer to current skb being populated
1886  *
1887  * This function checks the ring, descriptor, and packet information in
1888  * order to populate the hash, checksum, VLAN, protocol, and
1889  * other fields within the skb.
1890  **/
1891 void i40e_process_skb_fields(struct i40e_ring *rx_ring,
1892 			     union i40e_rx_desc *rx_desc, struct sk_buff *skb)
1893 {
1894 	u64 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1895 	u32 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1896 			I40E_RXD_QW1_STATUS_SHIFT;
1897 	u32 tsynvalid = rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK;
1898 	u32 tsyn = (rx_status & I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1899 		   I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT;
1900 	u8 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1901 		      I40E_RXD_QW1_PTYPE_SHIFT;
1902 
1903 	if (unlikely(tsynvalid))
1904 		i40e_ptp_rx_hwtstamp(rx_ring->vsi->back, skb, tsyn);
1905 
1906 	i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
1907 
1908 	i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
1909 
1910 	skb_record_rx_queue(skb, rx_ring->queue_index);
1911 
1912 	if (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
1913 		__le16 vlan_tag = rx_desc->wb.qword0.lo_dword.l2tag1;
1914 
1915 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1916 				       le16_to_cpu(vlan_tag));
1917 	}
1918 
1919 	/* modifies the skb - consumes the enet header */
1920 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1921 }
1922 
1923 /**
1924  * i40e_cleanup_headers - Correct empty headers
1925  * @rx_ring: rx descriptor ring packet is being transacted on
1926  * @skb: pointer to current skb being fixed
1927  * @rx_desc: pointer to the EOP Rx descriptor
1928  *
1929  * In addition if skb is not at least 60 bytes we need to pad it so that
1930  * it is large enough to qualify as a valid Ethernet frame.
1931  *
1932  * Returns true if an error was encountered and skb was freed.
1933  **/
1934 static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb,
1935 				 union i40e_rx_desc *rx_desc)
1936 
1937 {
1938 	/* ERR_MASK will only have valid bits if EOP set, and
1939 	 * what we are doing here is actually checking
1940 	 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in
1941 	 * the error field
1942 	 */
1943 	if (unlikely(i40e_test_staterr(rx_desc,
1944 				       BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1945 		dev_kfree_skb_any(skb);
1946 		return true;
1947 	}
1948 
1949 	/* if eth_skb_pad returns an error the skb was freed */
1950 	if (eth_skb_pad(skb))
1951 		return true;
1952 
1953 	return false;
1954 }
1955 
1956 /**
1957  * i40e_can_reuse_rx_page - Determine if page can be reused for another Rx
1958  * @rx_buffer: buffer containing the page
1959  * @rx_stats: rx stats structure for the rx ring
1960  *
1961  * If page is reusable, we have a green light for calling i40e_reuse_rx_page,
1962  * which will assign the current buffer to the buffer that next_to_alloc is
1963  * pointing to; otherwise, the DMA mapping needs to be destroyed and
1964  * page freed.
1965  *
1966  * rx_stats will be updated to indicate whether the page was waived
1967  * or busy if it could not be reused.
1968  */
1969 static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
1970 				   struct i40e_rx_queue_stats *rx_stats)
1971 {
1972 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1973 	struct page *page = rx_buffer->page;
1974 
1975 	/* Is any reuse possible? */
1976 	if (!dev_page_is_reusable(page)) {
1977 		rx_stats->page_waive_count++;
1978 		return false;
1979 	}
1980 
1981 #if (PAGE_SIZE < 8192)
1982 	/* if we are only owner of page we can reuse it */
1983 	if (unlikely((rx_buffer->page_count - pagecnt_bias) > 1)) {
1984 		rx_stats->page_busy_count++;
1985 		return false;
1986 	}
1987 #else
1988 #define I40E_LAST_OFFSET \
1989 	(SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048)
1990 	if (rx_buffer->page_offset > I40E_LAST_OFFSET) {
1991 		rx_stats->page_busy_count++;
1992 		return false;
1993 	}
1994 #endif
1995 
1996 	/* If we have drained the page fragment pool we need to update
1997 	 * the pagecnt_bias and page count so that we fully restock the
1998 	 * number of references the driver holds.
1999 	 */
2000 	if (unlikely(pagecnt_bias == 1)) {
2001 		page_ref_add(page, USHRT_MAX - 1);
2002 		rx_buffer->pagecnt_bias = USHRT_MAX;
2003 	}
2004 
2005 	return true;
2006 }
2007 
2008 /**
2009  * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region
2010  * @rx_buffer: Rx buffer to adjust
2011  * @truesize: Size of adjustment
2012  **/
2013 static void i40e_rx_buffer_flip(struct i40e_rx_buffer *rx_buffer,
2014 				unsigned int truesize)
2015 {
2016 #if (PAGE_SIZE < 8192)
2017 	rx_buffer->page_offset ^= truesize;
2018 #else
2019 	rx_buffer->page_offset += truesize;
2020 #endif
2021 }
2022 
2023 /**
2024  * i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use
2025  * @rx_ring: rx descriptor ring to transact packets on
2026  * @size: size of buffer to add to skb
2027  *
2028  * This function will pull an Rx buffer from the ring and synchronize it
2029  * for use by the CPU.
2030  */
2031 static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
2032 						 const unsigned int size)
2033 {
2034 	struct i40e_rx_buffer *rx_buffer;
2035 
2036 	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_process);
2037 	rx_buffer->page_count =
2038 #if (PAGE_SIZE < 8192)
2039 		page_count(rx_buffer->page);
2040 #else
2041 		0;
2042 #endif
2043 	prefetch_page_address(rx_buffer->page);
2044 
2045 	/* we are reusing so sync this buffer for CPU use */
2046 	dma_sync_single_range_for_cpu(rx_ring->dev,
2047 				      rx_buffer->dma,
2048 				      rx_buffer->page_offset,
2049 				      size,
2050 				      DMA_FROM_DEVICE);
2051 
2052 	/* We have pulled a buffer for use, so decrement pagecnt_bias */
2053 	rx_buffer->pagecnt_bias--;
2054 
2055 	return rx_buffer;
2056 }
2057 
2058 /**
2059  * i40e_put_rx_buffer - Clean up used buffer and either recycle or free
2060  * @rx_ring: rx descriptor ring to transact packets on
2061  * @rx_buffer: rx buffer to pull data from
2062  *
2063  * This function will clean up the contents of the rx_buffer.  It will
2064  * either recycle the buffer or unmap it and free the associated resources.
2065  */
2066 static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
2067 			       struct i40e_rx_buffer *rx_buffer)
2068 {
2069 	if (i40e_can_reuse_rx_page(rx_buffer, &rx_ring->rx_stats)) {
2070 		/* hand second half of page back to the ring */
2071 		i40e_reuse_rx_page(rx_ring, rx_buffer);
2072 	} else {
2073 		/* we are not reusing the buffer so unmap it */
2074 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
2075 				     i40e_rx_pg_size(rx_ring),
2076 				     DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
2077 		__page_frag_cache_drain(rx_buffer->page,
2078 					rx_buffer->pagecnt_bias);
2079 		/* clear contents of buffer_info */
2080 		rx_buffer->page = NULL;
2081 	}
2082 }
2083 
2084 /**
2085  * i40e_process_rx_buffs- Processing of buffers post XDP prog or on error
2086  * @rx_ring: Rx descriptor ring to transact packets on
2087  * @xdp_res: Result of the XDP program
2088  * @xdp: xdp_buff pointing to the data
2089  **/
2090 static void i40e_process_rx_buffs(struct i40e_ring *rx_ring, int xdp_res,
2091 				  struct xdp_buff *xdp)
2092 {
2093 	u32 nr_frags = xdp_get_shared_info_from_buff(xdp)->nr_frags;
2094 	u32 next = rx_ring->next_to_clean, i = 0;
2095 	struct i40e_rx_buffer *rx_buffer;
2096 
2097 	xdp->flags = 0;
2098 
2099 	while (1) {
2100 		rx_buffer = i40e_rx_bi(rx_ring, next);
2101 		if (++next == rx_ring->count)
2102 			next = 0;
2103 
2104 		if (!rx_buffer->page)
2105 			continue;
2106 
2107 		if (xdp_res != I40E_XDP_CONSUMED)
2108 			i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
2109 		else if (i++ <= nr_frags)
2110 			rx_buffer->pagecnt_bias++;
2111 
2112 		/* EOP buffer will be put in i40e_clean_rx_irq() */
2113 		if (next == rx_ring->next_to_process)
2114 			return;
2115 
2116 		i40e_put_rx_buffer(rx_ring, rx_buffer);
2117 	}
2118 }
2119 
2120 /**
2121  * i40e_construct_skb - Allocate skb and populate it
2122  * @rx_ring: rx descriptor ring to transact packets on
2123  * @xdp: xdp_buff pointing to the data
2124  *
2125  * This function allocates an skb.  It then populates it with the page
2126  * data from the current receive descriptor, taking care to set up the
2127  * skb correctly.
2128  */
2129 static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
2130 					  struct xdp_buff *xdp)
2131 {
2132 	unsigned int size = xdp->data_end - xdp->data;
2133 	struct i40e_rx_buffer *rx_buffer;
2134 	struct skb_shared_info *sinfo;
2135 	unsigned int headlen;
2136 	struct sk_buff *skb;
2137 	u32 nr_frags = 0;
2138 
2139 	/* prefetch first cache line of first page */
2140 	net_prefetch(xdp->data);
2141 
2142 	/* Note, we get here by enabling legacy-rx via:
2143 	 *
2144 	 *    ethtool --set-priv-flags <dev> legacy-rx on
2145 	 *
2146 	 * In this mode, we currently get 0 extra XDP headroom as
2147 	 * opposed to having legacy-rx off, where we process XDP
2148 	 * packets going to stack via i40e_build_skb(). The latter
2149 	 * provides us currently with 192 bytes of headroom.
2150 	 *
2151 	 * For i40e_construct_skb() mode it means that the
2152 	 * xdp->data_meta will always point to xdp->data, since
2153 	 * the helper cannot expand the head. Should this ever
2154 	 * change in future for legacy-rx mode on, then lets also
2155 	 * add xdp->data_meta handling here.
2156 	 */
2157 
2158 	/* allocate a skb to store the frags */
2159 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
2160 			       I40E_RX_HDR_SIZE,
2161 			       GFP_ATOMIC | __GFP_NOWARN);
2162 	if (unlikely(!skb))
2163 		return NULL;
2164 
2165 	/* Determine available headroom for copy */
2166 	headlen = size;
2167 	if (headlen > I40E_RX_HDR_SIZE)
2168 		headlen = eth_get_headlen(skb->dev, xdp->data,
2169 					  I40E_RX_HDR_SIZE);
2170 
2171 	/* align pull length to size of long to optimize memcpy performance */
2172 	memcpy(__skb_put(skb, headlen), xdp->data,
2173 	       ALIGN(headlen, sizeof(long)));
2174 
2175 	if (unlikely(xdp_buff_has_frags(xdp))) {
2176 		sinfo = xdp_get_shared_info_from_buff(xdp);
2177 		nr_frags = sinfo->nr_frags;
2178 	}
2179 	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
2180 	/* update all of the pointers */
2181 	size -= headlen;
2182 	if (size) {
2183 		if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2184 			dev_kfree_skb(skb);
2185 			return NULL;
2186 		}
2187 		skb_add_rx_frag(skb, 0, rx_buffer->page,
2188 				rx_buffer->page_offset + headlen,
2189 				size, xdp->frame_sz);
2190 		/* buffer is used by skb, update page_offset */
2191 		i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
2192 	} else {
2193 		/* buffer is unused, reset bias back to rx_buffer */
2194 		rx_buffer->pagecnt_bias++;
2195 	}
2196 
2197 	if (unlikely(xdp_buff_has_frags(xdp))) {
2198 		struct skb_shared_info *skinfo = skb_shinfo(skb);
2199 
2200 		memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0],
2201 		       sizeof(skb_frag_t) * nr_frags);
2202 
2203 		xdp_update_skb_shared_info(skb, skinfo->nr_frags + nr_frags,
2204 					   sinfo->xdp_frags_size,
2205 					   nr_frags * xdp->frame_sz,
2206 					   xdp_buff_is_frag_pfmemalloc(xdp));
2207 
2208 		/* First buffer has already been processed, so bump ntc */
2209 		if (++rx_ring->next_to_clean == rx_ring->count)
2210 			rx_ring->next_to_clean = 0;
2211 
2212 		i40e_process_rx_buffs(rx_ring, I40E_XDP_PASS, xdp);
2213 	}
2214 
2215 	return skb;
2216 }
2217 
2218 /**
2219  * i40e_build_skb - Build skb around an existing buffer
2220  * @rx_ring: Rx descriptor ring to transact packets on
2221  * @xdp: xdp_buff pointing to the data
2222  *
2223  * This function builds an skb around an existing Rx buffer, taking care
2224  * to set up the skb correctly and avoid any memcpy overhead.
2225  */
2226 static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
2227 				      struct xdp_buff *xdp)
2228 {
2229 	unsigned int metasize = xdp->data - xdp->data_meta;
2230 	struct skb_shared_info *sinfo;
2231 	struct sk_buff *skb;
2232 	u32 nr_frags;
2233 
2234 	/* Prefetch first cache line of first page. If xdp->data_meta
2235 	 * is unused, this points exactly as xdp->data, otherwise we
2236 	 * likely have a consumer accessing first few bytes of meta
2237 	 * data, and then actual data.
2238 	 */
2239 	net_prefetch(xdp->data_meta);
2240 
2241 	if (unlikely(xdp_buff_has_frags(xdp))) {
2242 		sinfo = xdp_get_shared_info_from_buff(xdp);
2243 		nr_frags = sinfo->nr_frags;
2244 	}
2245 
2246 	/* build an skb around the page buffer */
2247 	skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);
2248 	if (unlikely(!skb))
2249 		return NULL;
2250 
2251 	/* update pointers within the skb to store the data */
2252 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
2253 	__skb_put(skb, xdp->data_end - xdp->data);
2254 	if (metasize)
2255 		skb_metadata_set(skb, metasize);
2256 
2257 	if (unlikely(xdp_buff_has_frags(xdp))) {
2258 		xdp_update_skb_shared_info(skb, nr_frags,
2259 					   sinfo->xdp_frags_size,
2260 					   nr_frags * xdp->frame_sz,
2261 					   xdp_buff_is_frag_pfmemalloc(xdp));
2262 
2263 		i40e_process_rx_buffs(rx_ring, I40E_XDP_PASS, xdp);
2264 	} else {
2265 		struct i40e_rx_buffer *rx_buffer;
2266 
2267 		rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
2268 		/* buffer is used by skb, update page_offset */
2269 		i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
2270 	}
2271 
2272 	return skb;
2273 }
2274 
2275 /**
2276  * i40e_is_non_eop - process handling of non-EOP buffers
2277  * @rx_ring: Rx ring being processed
2278  * @rx_desc: Rx descriptor for current buffer
2279  *
2280  * If the buffer is an EOP buffer, this function exits returning false,
2281  * otherwise return true indicating that this is in fact a non-EOP buffer.
2282  */
2283 bool i40e_is_non_eop(struct i40e_ring *rx_ring,
2284 		     union i40e_rx_desc *rx_desc)
2285 {
2286 	/* if we are the last buffer then there is nothing else to do */
2287 #define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
2288 	if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
2289 		return false;
2290 
2291 	rx_ring->rx_stats.non_eop_descs++;
2292 
2293 	return true;
2294 }
2295 
2296 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
2297 			      struct i40e_ring *xdp_ring);
2298 
2299 int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring)
2300 {
2301 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2302 
2303 	if (unlikely(!xdpf))
2304 		return I40E_XDP_CONSUMED;
2305 
2306 	return i40e_xmit_xdp_ring(xdpf, xdp_ring);
2307 }
2308 
2309 /**
2310  * i40e_run_xdp - run an XDP program
2311  * @rx_ring: Rx ring being processed
2312  * @xdp: XDP buffer containing the frame
2313  * @xdp_prog: XDP program to run
2314  **/
2315 static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog)
2316 {
2317 	int err, result = I40E_XDP_PASS;
2318 	struct i40e_ring *xdp_ring;
2319 	u32 act;
2320 
2321 	if (!xdp_prog)
2322 		goto xdp_out;
2323 
2324 	prefetchw(xdp->data_hard_start); /* xdp_frame write */
2325 
2326 	act = bpf_prog_run_xdp(xdp_prog, xdp);
2327 	switch (act) {
2328 	case XDP_PASS:
2329 		break;
2330 	case XDP_TX:
2331 		xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];
2332 		result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);
2333 		if (result == I40E_XDP_CONSUMED)
2334 			goto out_failure;
2335 		break;
2336 	case XDP_REDIRECT:
2337 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
2338 		if (err)
2339 			goto out_failure;
2340 		result = I40E_XDP_REDIR;
2341 		break;
2342 	default:
2343 		bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
2344 		fallthrough;
2345 	case XDP_ABORTED:
2346 out_failure:
2347 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
2348 		fallthrough; /* handle aborts by dropping packet */
2349 	case XDP_DROP:
2350 		result = I40E_XDP_CONSUMED;
2351 		break;
2352 	}
2353 xdp_out:
2354 	return result;
2355 }
2356 
2357 /**
2358  * i40e_xdp_ring_update_tail - Updates the XDP Tx ring tail register
2359  * @xdp_ring: XDP Tx ring
2360  *
2361  * This function updates the XDP Tx ring tail register.
2362  **/
2363 void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring)
2364 {
2365 	/* Force memory writes to complete before letting h/w
2366 	 * know there are new descriptors to fetch.
2367 	 */
2368 	wmb();
2369 	writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail);
2370 }
2371 
2372 /**
2373  * i40e_update_rx_stats - Update Rx ring statistics
2374  * @rx_ring: rx descriptor ring
2375  * @total_rx_bytes: number of bytes received
2376  * @total_rx_packets: number of packets received
2377  *
2378  * This function updates the Rx ring statistics.
2379  **/
2380 void i40e_update_rx_stats(struct i40e_ring *rx_ring,
2381 			  unsigned int total_rx_bytes,
2382 			  unsigned int total_rx_packets)
2383 {
2384 	u64_stats_update_begin(&rx_ring->syncp);
2385 	rx_ring->stats.packets += total_rx_packets;
2386 	rx_ring->stats.bytes += total_rx_bytes;
2387 	u64_stats_update_end(&rx_ring->syncp);
2388 	rx_ring->q_vector->rx.total_packets += total_rx_packets;
2389 	rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
2390 }
2391 
2392 /**
2393  * i40e_finalize_xdp_rx - Bump XDP Tx tail and/or flush redirect map
2394  * @rx_ring: Rx ring
2395  * @xdp_res: Result of the receive batch
2396  *
2397  * This function bumps XDP Tx tail and/or flush redirect map, and
2398  * should be called when a batch of packets has been processed in the
2399  * napi loop.
2400  **/
2401 void i40e_finalize_xdp_rx(struct i40e_ring *rx_ring, unsigned int xdp_res)
2402 {
2403 	if (xdp_res & I40E_XDP_REDIR)
2404 		xdp_do_flush_map();
2405 
2406 	if (xdp_res & I40E_XDP_TX) {
2407 		struct i40e_ring *xdp_ring =
2408 			rx_ring->vsi->xdp_rings[rx_ring->queue_index];
2409 
2410 		i40e_xdp_ring_update_tail(xdp_ring);
2411 	}
2412 }
2413 
2414 /**
2415  * i40e_inc_ntp: Advance the next_to_process index
2416  * @rx_ring: Rx ring
2417  **/
2418 static void i40e_inc_ntp(struct i40e_ring *rx_ring)
2419 {
2420 	u32 ntp = rx_ring->next_to_process + 1;
2421 
2422 	ntp = (ntp < rx_ring->count) ? ntp : 0;
2423 	rx_ring->next_to_process = ntp;
2424 	prefetch(I40E_RX_DESC(rx_ring, ntp));
2425 }
2426 
2427 /**
2428  * i40e_add_xdp_frag: Add a frag to xdp_buff
2429  * @xdp: xdp_buff pointing to the data
2430  * @nr_frags: return number of buffers for the packet
2431  * @rx_buffer: rx_buffer holding data of the current frag
2432  * @size: size of data of current frag
2433  */
2434 static int i40e_add_xdp_frag(struct xdp_buff *xdp, u32 *nr_frags,
2435 			     struct i40e_rx_buffer *rx_buffer, u32 size)
2436 {
2437 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2438 
2439 	if (!xdp_buff_has_frags(xdp)) {
2440 		sinfo->nr_frags = 0;
2441 		sinfo->xdp_frags_size = 0;
2442 		xdp_buff_set_frags_flag(xdp);
2443 	} else if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
2444 		/* Overflowing packet: All frags need to be dropped */
2445 		return -ENOMEM;
2446 	}
2447 
2448 	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buffer->page,
2449 				   rx_buffer->page_offset, size);
2450 
2451 	sinfo->xdp_frags_size += size;
2452 
2453 	if (page_is_pfmemalloc(rx_buffer->page))
2454 		xdp_buff_set_frag_pfmemalloc(xdp);
2455 	*nr_frags = sinfo->nr_frags;
2456 
2457 	return 0;
2458 }
2459 
2460 /**
2461  * i40e_consume_xdp_buff - Consume all the buffers of the packet and update ntc
2462  * @rx_ring: rx descriptor ring to transact packets on
2463  * @xdp: xdp_buff pointing to the data
2464  * @rx_buffer: rx_buffer of eop desc
2465  */
2466 static void i40e_consume_xdp_buff(struct i40e_ring *rx_ring,
2467 				  struct xdp_buff *xdp,
2468 				  struct i40e_rx_buffer *rx_buffer)
2469 {
2470 	i40e_process_rx_buffs(rx_ring, I40E_XDP_CONSUMED, xdp);
2471 	i40e_put_rx_buffer(rx_ring, rx_buffer);
2472 	rx_ring->next_to_clean = rx_ring->next_to_process;
2473 	xdp->data = NULL;
2474 }
2475 
2476 /**
2477  * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
2478  * @rx_ring: rx descriptor ring to transact packets on
2479  * @budget: Total limit on number of packets to process
2480  * @rx_cleaned: Out parameter of the number of packets processed
2481  *
2482  * This function provides a "bounce buffer" approach to Rx interrupt
2483  * processing.  The advantage to this is that on systems that have
2484  * expensive overhead for IOMMU access this provides a means of avoiding
2485  * it by maintaining the mapping of the page to the system.
2486  *
2487  * Returns amount of work completed
2488  **/
2489 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
2490 			     unsigned int *rx_cleaned)
2491 {
2492 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
2493 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
2494 	u16 clean_threshold = rx_ring->count / 2;
2495 	unsigned int offset = rx_ring->rx_offset;
2496 	struct xdp_buff *xdp = &rx_ring->xdp;
2497 	unsigned int xdp_xmit = 0;
2498 	struct bpf_prog *xdp_prog;
2499 	bool failure = false;
2500 	int xdp_res = 0;
2501 
2502 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
2503 
2504 	while (likely(total_rx_packets < (unsigned int)budget)) {
2505 		u16 ntp = rx_ring->next_to_process;
2506 		struct i40e_rx_buffer *rx_buffer;
2507 		union i40e_rx_desc *rx_desc;
2508 		struct sk_buff *skb;
2509 		unsigned int size;
2510 		u32 nfrags = 0;
2511 		bool neop;
2512 		u64 qword;
2513 
2514 		/* return some buffers to hardware, one at a time is too slow */
2515 		if (cleaned_count >= clean_threshold) {
2516 			failure = failure ||
2517 				  i40e_alloc_rx_buffers(rx_ring, cleaned_count);
2518 			cleaned_count = 0;
2519 		}
2520 
2521 		rx_desc = I40E_RX_DESC(rx_ring, ntp);
2522 
2523 		/* status_error_len will always be zero for unused descriptors
2524 		 * because it's cleared in cleanup, and overlaps with hdr_addr
2525 		 * which is always zero because packet split isn't used, if the
2526 		 * hardware wrote DD then the length will be non-zero
2527 		 */
2528 		qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
2529 
2530 		/* This memory barrier is needed to keep us from reading
2531 		 * any other fields out of the rx_desc until we have
2532 		 * verified the descriptor has been written back.
2533 		 */
2534 		dma_rmb();
2535 
2536 		if (i40e_rx_is_programming_status(qword)) {
2537 			i40e_clean_programming_status(rx_ring,
2538 						      rx_desc->raw.qword[0],
2539 						      qword);
2540 			rx_buffer = i40e_rx_bi(rx_ring, ntp);
2541 			i40e_inc_ntp(rx_ring);
2542 			i40e_reuse_rx_page(rx_ring, rx_buffer);
2543 			/* Update ntc and bump cleaned count if not in the
2544 			 * middle of mb packet.
2545 			 */
2546 			if (rx_ring->next_to_clean == ntp) {
2547 				rx_ring->next_to_clean =
2548 					rx_ring->next_to_process;
2549 				cleaned_count++;
2550 			}
2551 			continue;
2552 		}
2553 
2554 		size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
2555 		       I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
2556 		if (!size)
2557 			break;
2558 
2559 		i40e_trace(clean_rx_irq, rx_ring, rx_desc, xdp);
2560 		/* retrieve a buffer from the ring */
2561 		rx_buffer = i40e_get_rx_buffer(rx_ring, size);
2562 
2563 		neop = i40e_is_non_eop(rx_ring, rx_desc);
2564 		i40e_inc_ntp(rx_ring);
2565 
2566 		if (!xdp->data) {
2567 			unsigned char *hard_start;
2568 
2569 			hard_start = page_address(rx_buffer->page) +
2570 				     rx_buffer->page_offset - offset;
2571 			xdp_prepare_buff(xdp, hard_start, offset, size, true);
2572 #if (PAGE_SIZE > 4096)
2573 			/* At larger PAGE_SIZE, frame_sz depend on len size */
2574 			xdp->frame_sz = i40e_rx_frame_truesize(rx_ring, size);
2575 #endif
2576 		} else if (i40e_add_xdp_frag(xdp, &nfrags, rx_buffer, size) &&
2577 			   !neop) {
2578 			/* Overflowing packet: Drop all frags on EOP */
2579 			i40e_consume_xdp_buff(rx_ring, xdp, rx_buffer);
2580 			break;
2581 		}
2582 
2583 		if (neop)
2584 			continue;
2585 
2586 		xdp_res = i40e_run_xdp(rx_ring, xdp, xdp_prog);
2587 
2588 		if (xdp_res) {
2589 			xdp_xmit |= xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR);
2590 
2591 			if (unlikely(xdp_buff_has_frags(xdp))) {
2592 				i40e_process_rx_buffs(rx_ring, xdp_res, xdp);
2593 				size = xdp_get_buff_len(xdp);
2594 			} else if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) {
2595 				i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
2596 			} else {
2597 				rx_buffer->pagecnt_bias++;
2598 			}
2599 			total_rx_bytes += size;
2600 		} else {
2601 			if (ring_uses_build_skb(rx_ring))
2602 				skb = i40e_build_skb(rx_ring, xdp);
2603 			else
2604 				skb = i40e_construct_skb(rx_ring, xdp);
2605 
2606 			/* drop if we failed to retrieve a buffer */
2607 			if (!skb) {
2608 				rx_ring->rx_stats.alloc_buff_failed++;
2609 				i40e_consume_xdp_buff(rx_ring, xdp, rx_buffer);
2610 				break;
2611 			}
2612 
2613 			if (i40e_cleanup_headers(rx_ring, skb, rx_desc))
2614 				goto process_next;
2615 
2616 			/* probably a little skewed due to removing CRC */
2617 			total_rx_bytes += skb->len;
2618 
2619 			/* populate checksum, VLAN, and protocol */
2620 			i40e_process_skb_fields(rx_ring, rx_desc, skb);
2621 
2622 			i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, xdp);
2623 			napi_gro_receive(&rx_ring->q_vector->napi, skb);
2624 		}
2625 
2626 		/* update budget accounting */
2627 		total_rx_packets++;
2628 process_next:
2629 		cleaned_count += nfrags + 1;
2630 		i40e_put_rx_buffer(rx_ring, rx_buffer);
2631 		rx_ring->next_to_clean = rx_ring->next_to_process;
2632 
2633 		xdp->data = NULL;
2634 	}
2635 
2636 	i40e_finalize_xdp_rx(rx_ring, xdp_xmit);
2637 
2638 	i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets);
2639 
2640 	*rx_cleaned = total_rx_packets;
2641 
2642 	/* guarantee a trip back through this routine if there was a failure */
2643 	return failure ? budget : (int)total_rx_packets;
2644 }
2645 
2646 /**
2647  * i40e_buildreg_itr - build a value for writing to I40E_PFINT_DYN_CTLN register
2648  * @itr_idx: interrupt throttling index
2649  * @interval: interrupt throttling interval value in usecs
2650  * @force_swint: force software interrupt
2651  *
2652  * The function builds a value for I40E_PFINT_DYN_CTLN register that
2653  * is used to update interrupt throttling interval for specified ITR index
2654  * and optionally enforces a software interrupt. If the @itr_idx is equal
2655  * to I40E_ITR_NONE then no interval change is applied and only @force_swint
2656  * parameter is taken into account. If the interval change and enforced
2657  * software interrupt are not requested then the built value just enables
2658  * appropriate vector interrupt.
2659  **/
2660 static u32 i40e_buildreg_itr(enum i40e_dyn_idx itr_idx, u16 interval,
2661 			     bool force_swint)
2662 {
2663 	u32 val;
2664 
2665 	/* We don't bother with setting the CLEARPBA bit as the data sheet
2666 	 * points out doing so is "meaningless since it was already
2667 	 * auto-cleared". The auto-clearing happens when the interrupt is
2668 	 * asserted.
2669 	 *
2670 	 * Hardware errata 28 for also indicates that writing to a
2671 	 * xxINT_DYN_CTLx CSR with INTENA_MSK (bit 31) set to 0 will clear
2672 	 * an event in the PBA anyway so we need to rely on the automask
2673 	 * to hold pending events for us until the interrupt is re-enabled
2674 	 *
2675 	 * We have to shift the given value as it is reported in microseconds
2676 	 * and the register value is recorded in 2 microsecond units.
2677 	 */
2678 	interval >>= 1;
2679 
2680 	/* 1. Enable vector interrupt
2681 	 * 2. Update the interval for the specified ITR index
2682 	 *    (I40E_ITR_NONE in the register is used to indicate that
2683 	 *     no interval update is requested)
2684 	 */
2685 	val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2686 	      FIELD_PREP(I40E_PFINT_DYN_CTLN_ITR_INDX_MASK, itr_idx) |
2687 	      FIELD_PREP(I40E_PFINT_DYN_CTLN_INTERVAL_MASK, interval);
2688 
2689 	/* 3. Enforce software interrupt trigger if requested
2690 	 *    (These software interrupts rate is limited by ITR2 that is
2691 	 *     set to 20K interrupts per second)
2692 	 */
2693 	if (force_swint)
2694 		val |= I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK |
2695 		       I40E_PFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK |
2696 		       FIELD_PREP(I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK,
2697 				  I40E_SW_ITR);
2698 
2699 	return val;
2700 }
2701 
2702 /* The act of updating the ITR will cause it to immediately trigger. In order
2703  * to prevent this from throwing off adaptive update statistics we defer the
2704  * update so that it can only happen so often. So after either Tx or Rx are
2705  * updated we make the adaptive scheme wait until either the ITR completely
2706  * expires via the next_update expiration or we have been through at least
2707  * 3 interrupts.
2708  */
2709 #define ITR_COUNTDOWN_START 3
2710 
2711 /**
2712  * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
2713  * @vsi: the VSI we care about
2714  * @q_vector: q_vector for which itr is being updated and interrupt enabled
2715  *
2716  **/
2717 static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
2718 					  struct i40e_q_vector *q_vector)
2719 {
2720 	enum i40e_dyn_idx itr_idx = I40E_ITR_NONE;
2721 	struct i40e_hw *hw = &vsi->back->hw;
2722 	u16 interval = 0;
2723 	u32 itr_val;
2724 
2725 	/* If we don't have MSIX, then we only need to re-enable icr0 */
2726 	if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) {
2727 		i40e_irq_dynamic_enable_icr0(vsi->back);
2728 		return;
2729 	}
2730 
2731 	/* These will do nothing if dynamic updates are not enabled */
2732 	i40e_update_itr(q_vector, &q_vector->tx);
2733 	i40e_update_itr(q_vector, &q_vector->rx);
2734 
2735 	/* This block of logic allows us to get away with only updating
2736 	 * one ITR value with each interrupt. The idea is to perform a
2737 	 * pseudo-lazy update with the following criteria.
2738 	 *
2739 	 * 1. Rx is given higher priority than Tx if both are in same state
2740 	 * 2. If we must reduce an ITR that is given highest priority.
2741 	 * 3. We then give priority to increasing ITR based on amount.
2742 	 */
2743 	if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
2744 		/* Rx ITR needs to be reduced, this is highest priority */
2745 		itr_idx = I40E_RX_ITR;
2746 		interval = q_vector->rx.target_itr;
2747 		q_vector->rx.current_itr = q_vector->rx.target_itr;
2748 		q_vector->itr_countdown = ITR_COUNTDOWN_START;
2749 	} else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
2750 		   ((q_vector->rx.target_itr - q_vector->rx.current_itr) <
2751 		    (q_vector->tx.target_itr - q_vector->tx.current_itr))) {
2752 		/* Tx ITR needs to be reduced, this is second priority
2753 		 * Tx ITR needs to be increased more than Rx, fourth priority
2754 		 */
2755 		itr_idx = I40E_TX_ITR;
2756 		interval = q_vector->tx.target_itr;
2757 		q_vector->tx.current_itr = q_vector->tx.target_itr;
2758 		q_vector->itr_countdown = ITR_COUNTDOWN_START;
2759 	} else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
2760 		/* Rx ITR needs to be increased, third priority */
2761 		itr_idx = I40E_RX_ITR;
2762 		interval = q_vector->rx.target_itr;
2763 		q_vector->rx.current_itr = q_vector->rx.target_itr;
2764 		q_vector->itr_countdown = ITR_COUNTDOWN_START;
2765 	} else {
2766 		/* No ITR update, lowest priority */
2767 		if (q_vector->itr_countdown)
2768 			q_vector->itr_countdown--;
2769 	}
2770 
2771 	/* Do not update interrupt control register if VSI is down */
2772 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
2773 		return;
2774 
2775 	/* Update ITR interval if necessary and enforce software interrupt
2776 	 * if we are exiting busy poll.
2777 	 */
2778 	if (q_vector->in_busy_poll) {
2779 		itr_val = i40e_buildreg_itr(itr_idx, interval, true);
2780 		q_vector->in_busy_poll = false;
2781 	} else {
2782 		itr_val = i40e_buildreg_itr(itr_idx, interval, false);
2783 	}
2784 	wr32(hw, I40E_PFINT_DYN_CTLN(q_vector->reg_idx), itr_val);
2785 }
2786 
2787 /**
2788  * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
2789  * @napi: napi struct with our devices info in it
2790  * @budget: amount of work driver is allowed to do this pass, in packets
2791  *
2792  * This function will clean all queues associated with a q_vector.
2793  *
2794  * Returns the amount of work done
2795  **/
2796 int i40e_napi_poll(struct napi_struct *napi, int budget)
2797 {
2798 	struct i40e_q_vector *q_vector =
2799 			       container_of(napi, struct i40e_q_vector, napi);
2800 	struct i40e_vsi *vsi = q_vector->vsi;
2801 	struct i40e_ring *ring;
2802 	bool tx_clean_complete = true;
2803 	bool rx_clean_complete = true;
2804 	unsigned int tx_cleaned = 0;
2805 	unsigned int rx_cleaned = 0;
2806 	bool clean_complete = true;
2807 	bool arm_wb = false;
2808 	int budget_per_ring;
2809 	int work_done = 0;
2810 
2811 	if (test_bit(__I40E_VSI_DOWN, vsi->state)) {
2812 		napi_complete(napi);
2813 		return 0;
2814 	}
2815 
2816 	/* Since the actual Tx work is minimal, we can give the Tx a larger
2817 	 * budget and be more aggressive about cleaning up the Tx descriptors.
2818 	 */
2819 	i40e_for_each_ring(ring, q_vector->tx) {
2820 		bool wd = ring->xsk_pool ?
2821 			  i40e_clean_xdp_tx_irq(vsi, ring) :
2822 			  i40e_clean_tx_irq(vsi, ring, budget, &tx_cleaned);
2823 
2824 		if (!wd) {
2825 			clean_complete = tx_clean_complete = false;
2826 			continue;
2827 		}
2828 		arm_wb |= ring->arm_wb;
2829 		ring->arm_wb = false;
2830 	}
2831 
2832 	/* Handle case where we are called by netpoll with a budget of 0 */
2833 	if (budget <= 0)
2834 		goto tx_only;
2835 
2836 	/* normally we have 1 Rx ring per q_vector */
2837 	if (unlikely(q_vector->num_ringpairs > 1))
2838 		/* We attempt to distribute budget to each Rx queue fairly, but
2839 		 * don't allow the budget to go below 1 because that would exit
2840 		 * polling early.
2841 		 */
2842 		budget_per_ring = max_t(int, budget / q_vector->num_ringpairs, 1);
2843 	else
2844 		/* Max of 1 Rx ring in this q_vector so give it the budget */
2845 		budget_per_ring = budget;
2846 
2847 	i40e_for_each_ring(ring, q_vector->rx) {
2848 		int cleaned = ring->xsk_pool ?
2849 			      i40e_clean_rx_irq_zc(ring, budget_per_ring) :
2850 			      i40e_clean_rx_irq(ring, budget_per_ring, &rx_cleaned);
2851 
2852 		work_done += cleaned;
2853 		/* if we clean as many as budgeted, we must not be done */
2854 		if (cleaned >= budget_per_ring)
2855 			clean_complete = rx_clean_complete = false;
2856 	}
2857 
2858 	if (!i40e_enabled_xdp_vsi(vsi))
2859 		trace_i40e_napi_poll(napi, q_vector, budget, budget_per_ring, rx_cleaned,
2860 				     tx_cleaned, rx_clean_complete, tx_clean_complete);
2861 
2862 	/* If work not completed, return budget and polling will return */
2863 	if (!clean_complete) {
2864 		int cpu_id = smp_processor_id();
2865 
2866 		/* It is possible that the interrupt affinity has changed but,
2867 		 * if the cpu is pegged at 100%, polling will never exit while
2868 		 * traffic continues and the interrupt will be stuck on this
2869 		 * cpu.  We check to make sure affinity is correct before we
2870 		 * continue to poll, otherwise we must stop polling so the
2871 		 * interrupt can move to the correct cpu.
2872 		 */
2873 		if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
2874 			/* Tell napi that we are done polling */
2875 			napi_complete_done(napi, work_done);
2876 
2877 			/* Force an interrupt */
2878 			i40e_force_wb(vsi, q_vector);
2879 
2880 			/* Return budget-1 so that polling stops */
2881 			return budget - 1;
2882 		}
2883 tx_only:
2884 		if (arm_wb) {
2885 			q_vector->tx.ring[0].tx_stats.tx_force_wb++;
2886 			i40e_enable_wb_on_itr(vsi, q_vector);
2887 		}
2888 		return budget;
2889 	}
2890 
2891 	if (q_vector->tx.ring[0].flags & I40E_TXR_FLAGS_WB_ON_ITR)
2892 		q_vector->arm_wb_state = false;
2893 
2894 	/* Exit the polling mode, but don't re-enable interrupts if stack might
2895 	 * poll us due to busy-polling
2896 	 */
2897 	if (likely(napi_complete_done(napi, work_done)))
2898 		i40e_update_enable_itr(vsi, q_vector);
2899 	else
2900 		q_vector->in_busy_poll = true;
2901 
2902 	return min(work_done, budget - 1);
2903 }
2904 
2905 /**
2906  * i40e_atr - Add a Flow Director ATR filter
2907  * @tx_ring:  ring to add programming descriptor to
2908  * @skb:      send buffer
2909  * @tx_flags: send tx flags
2910  **/
2911 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
2912 		     u32 tx_flags)
2913 {
2914 	struct i40e_filter_program_desc *fdir_desc;
2915 	struct i40e_pf *pf = tx_ring->vsi->back;
2916 	union {
2917 		unsigned char *network;
2918 		struct iphdr *ipv4;
2919 		struct ipv6hdr *ipv6;
2920 	} hdr;
2921 	struct tcphdr *th;
2922 	unsigned int hlen;
2923 	u32 flex_ptype, dtype_cmd;
2924 	int l4_proto;
2925 	u16 i;
2926 
2927 	/* make sure ATR is enabled */
2928 	if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
2929 		return;
2930 
2931 	if (test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
2932 		return;
2933 
2934 	/* if sampling is disabled do nothing */
2935 	if (!tx_ring->atr_sample_rate)
2936 		return;
2937 
2938 	/* Currently only IPv4/IPv6 with TCP is supported */
2939 	if (!(tx_flags & (I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6)))
2940 		return;
2941 
2942 	/* snag network header to get L4 type and address */
2943 	hdr.network = (tx_flags & I40E_TX_FLAGS_UDP_TUNNEL) ?
2944 		      skb_inner_network_header(skb) : skb_network_header(skb);
2945 
2946 	/* Note: tx_flags gets modified to reflect inner protocols in
2947 	 * tx_enable_csum function if encap is enabled.
2948 	 */
2949 	if (tx_flags & I40E_TX_FLAGS_IPV4) {
2950 		/* access ihl as u8 to avoid unaligned access on ia64 */
2951 		hlen = (hdr.network[0] & 0x0F) << 2;
2952 		l4_proto = hdr.ipv4->protocol;
2953 	} else {
2954 		/* find the start of the innermost ipv6 header */
2955 		unsigned int inner_hlen = hdr.network - skb->data;
2956 		unsigned int h_offset = inner_hlen;
2957 
2958 		/* this function updates h_offset to the end of the header */
2959 		l4_proto =
2960 		  ipv6_find_hdr(skb, &h_offset, IPPROTO_TCP, NULL, NULL);
2961 		/* hlen will contain our best estimate of the tcp header */
2962 		hlen = h_offset - inner_hlen;
2963 	}
2964 
2965 	if (l4_proto != IPPROTO_TCP)
2966 		return;
2967 
2968 	th = (struct tcphdr *)(hdr.network + hlen);
2969 
2970 	/* Due to lack of space, no more new filters can be programmed */
2971 	if (th->syn && test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
2972 		return;
2973 	if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) {
2974 		/* HW ATR eviction will take care of removing filters on FIN
2975 		 * and RST packets.
2976 		 */
2977 		if (th->fin || th->rst)
2978 			return;
2979 	}
2980 
2981 	tx_ring->atr_count++;
2982 
2983 	/* sample on all syn/fin/rst packets or once every atr sample rate */
2984 	if (!th->fin &&
2985 	    !th->syn &&
2986 	    !th->rst &&
2987 	    (tx_ring->atr_count < tx_ring->atr_sample_rate))
2988 		return;
2989 
2990 	tx_ring->atr_count = 0;
2991 
2992 	/* grab the next descriptor */
2993 	i = tx_ring->next_to_use;
2994 	fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
2995 
2996 	i++;
2997 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2998 
2999 	flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
3000 		      I40E_TXD_FLTR_QW0_QINDEX_MASK;
3001 	flex_ptype |= (tx_flags & I40E_TX_FLAGS_IPV4) ?
3002 		      (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
3003 		       I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
3004 		      (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
3005 		       I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
3006 
3007 	flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
3008 
3009 	dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
3010 
3011 	dtype_cmd |= (th->fin || th->rst) ?
3012 		     (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
3013 		      I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
3014 		     (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
3015 		      I40E_TXD_FLTR_QW1_PCMD_SHIFT);
3016 
3017 	dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
3018 		     I40E_TXD_FLTR_QW1_DEST_SHIFT;
3019 
3020 	dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
3021 		     I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
3022 
3023 	dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
3024 	if (!(tx_flags & I40E_TX_FLAGS_UDP_TUNNEL))
3025 		dtype_cmd |=
3026 			((u32)I40E_FD_ATR_STAT_IDX(pf->hw.pf_id) <<
3027 			I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
3028 			I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
3029 	else
3030 		dtype_cmd |=
3031 			((u32)I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id) <<
3032 			I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
3033 			I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
3034 
3035 	if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED)
3036 		dtype_cmd |= I40E_TXD_FLTR_QW1_ATR_MASK;
3037 
3038 	fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
3039 	fdir_desc->rsvd = cpu_to_le32(0);
3040 	fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
3041 	fdir_desc->fd_id = cpu_to_le32(0);
3042 }
3043 
3044 /**
3045  * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
3046  * @skb:     send buffer
3047  * @tx_ring: ring to send buffer on
3048  * @flags:   the tx flags to be set
3049  *
3050  * Checks the skb and set up correspondingly several generic transmit flags
3051  * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
3052  *
3053  * Returns error code indicate the frame should be dropped upon error and the
3054  * otherwise  returns 0 to indicate the flags has been set properly.
3055  **/
3056 static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
3057 					     struct i40e_ring *tx_ring,
3058 					     u32 *flags)
3059 {
3060 	__be16 protocol = skb->protocol;
3061 	u32  tx_flags = 0;
3062 
3063 	if (protocol == htons(ETH_P_8021Q) &&
3064 	    !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
3065 		/* When HW VLAN acceleration is turned off by the user the
3066 		 * stack sets the protocol to 8021q so that the driver
3067 		 * can take any steps required to support the SW only
3068 		 * VLAN handling.  In our case the driver doesn't need
3069 		 * to take any further steps so just set the protocol
3070 		 * to the encapsulated ethertype.
3071 		 */
3072 		skb->protocol = vlan_get_protocol(skb);
3073 		goto out;
3074 	}
3075 
3076 	/* if we have a HW VLAN tag being added, default to the HW one */
3077 	if (skb_vlan_tag_present(skb)) {
3078 		tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
3079 		tx_flags |= I40E_TX_FLAGS_HW_VLAN;
3080 	/* else if it is a SW VLAN, check the next protocol and store the tag */
3081 	} else if (protocol == htons(ETH_P_8021Q)) {
3082 		struct vlan_hdr *vhdr, _vhdr;
3083 
3084 		vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
3085 		if (!vhdr)
3086 			return -EINVAL;
3087 
3088 		protocol = vhdr->h_vlan_encapsulated_proto;
3089 		tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
3090 		tx_flags |= I40E_TX_FLAGS_SW_VLAN;
3091 	}
3092 
3093 	if (!(tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED))
3094 		goto out;
3095 
3096 	/* Insert 802.1p priority into VLAN header */
3097 	if ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
3098 	    (skb->priority != TC_PRIO_CONTROL)) {
3099 		tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
3100 		tx_flags |= (skb->priority & 0x7) <<
3101 				I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
3102 		if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
3103 			struct vlan_ethhdr *vhdr;
3104 			int rc;
3105 
3106 			rc = skb_cow_head(skb, 0);
3107 			if (rc < 0)
3108 				return rc;
3109 			vhdr = skb_vlan_eth_hdr(skb);
3110 			vhdr->h_vlan_TCI = htons(tx_flags >>
3111 						 I40E_TX_FLAGS_VLAN_SHIFT);
3112 		} else {
3113 			tx_flags |= I40E_TX_FLAGS_HW_VLAN;
3114 		}
3115 	}
3116 
3117 out:
3118 	*flags = tx_flags;
3119 	return 0;
3120 }
3121 
3122 /**
3123  * i40e_tso - set up the tso context descriptor
3124  * @first:    pointer to first Tx buffer for xmit
3125  * @hdr_len:  ptr to the size of the packet header
3126  * @cd_type_cmd_tso_mss: Quad Word 1
3127  *
3128  * Returns 0 if no TSO can happen, 1 if tso is going, or error
3129  **/
3130 static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len,
3131 		    u64 *cd_type_cmd_tso_mss)
3132 {
3133 	struct sk_buff *skb = first->skb;
3134 	u64 cd_cmd, cd_tso_len, cd_mss;
3135 	__be16 protocol;
3136 	union {
3137 		struct iphdr *v4;
3138 		struct ipv6hdr *v6;
3139 		unsigned char *hdr;
3140 	} ip;
3141 	union {
3142 		struct tcphdr *tcp;
3143 		struct udphdr *udp;
3144 		unsigned char *hdr;
3145 	} l4;
3146 	u32 paylen, l4_offset;
3147 	u16 gso_size;
3148 	int err;
3149 
3150 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3151 		return 0;
3152 
3153 	if (!skb_is_gso(skb))
3154 		return 0;
3155 
3156 	err = skb_cow_head(skb, 0);
3157 	if (err < 0)
3158 		return err;
3159 
3160 	protocol = vlan_get_protocol(skb);
3161 
3162 	if (eth_p_mpls(protocol))
3163 		ip.hdr = skb_inner_network_header(skb);
3164 	else
3165 		ip.hdr = skb_network_header(skb);
3166 	l4.hdr = skb_checksum_start(skb);
3167 
3168 	/* initialize outer IP header fields */
3169 	if (ip.v4->version == 4) {
3170 		ip.v4->tot_len = 0;
3171 		ip.v4->check = 0;
3172 
3173 		first->tx_flags |= I40E_TX_FLAGS_TSO;
3174 	} else {
3175 		ip.v6->payload_len = 0;
3176 		first->tx_flags |= I40E_TX_FLAGS_TSO;
3177 	}
3178 
3179 	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
3180 					 SKB_GSO_GRE_CSUM |
3181 					 SKB_GSO_IPXIP4 |
3182 					 SKB_GSO_IPXIP6 |
3183 					 SKB_GSO_UDP_TUNNEL |
3184 					 SKB_GSO_UDP_TUNNEL_CSUM)) {
3185 		if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
3186 		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
3187 			l4.udp->len = 0;
3188 
3189 			/* determine offset of outer transport header */
3190 			l4_offset = l4.hdr - skb->data;
3191 
3192 			/* remove payload length from outer checksum */
3193 			paylen = skb->len - l4_offset;
3194 			csum_replace_by_diff(&l4.udp->check,
3195 					     (__force __wsum)htonl(paylen));
3196 		}
3197 
3198 		/* reset pointers to inner headers */
3199 		ip.hdr = skb_inner_network_header(skb);
3200 		l4.hdr = skb_inner_transport_header(skb);
3201 
3202 		/* initialize inner IP header fields */
3203 		if (ip.v4->version == 4) {
3204 			ip.v4->tot_len = 0;
3205 			ip.v4->check = 0;
3206 		} else {
3207 			ip.v6->payload_len = 0;
3208 		}
3209 	}
3210 
3211 	/* determine offset of inner transport header */
3212 	l4_offset = l4.hdr - skb->data;
3213 
3214 	/* remove payload length from inner checksum */
3215 	paylen = skb->len - l4_offset;
3216 
3217 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
3218 		csum_replace_by_diff(&l4.udp->check, (__force __wsum)htonl(paylen));
3219 		/* compute length of segmentation header */
3220 		*hdr_len = sizeof(*l4.udp) + l4_offset;
3221 	} else {
3222 		csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
3223 		/* compute length of segmentation header */
3224 		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
3225 	}
3226 
3227 	/* pull values out of skb_shinfo */
3228 	gso_size = skb_shinfo(skb)->gso_size;
3229 
3230 	/* update GSO size and bytecount with header size */
3231 	first->gso_segs = skb_shinfo(skb)->gso_segs;
3232 	first->bytecount += (first->gso_segs - 1) * *hdr_len;
3233 
3234 	/* find the field values */
3235 	cd_cmd = I40E_TX_CTX_DESC_TSO;
3236 	cd_tso_len = skb->len - *hdr_len;
3237 	cd_mss = gso_size;
3238 	*cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
3239 				(cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
3240 				(cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
3241 	return 1;
3242 }
3243 
3244 /**
3245  * i40e_tsyn - set up the tsyn context descriptor
3246  * @tx_ring:  ptr to the ring to send
3247  * @skb:      ptr to the skb we're sending
3248  * @tx_flags: the collected send information
3249  * @cd_type_cmd_tso_mss: Quad Word 1
3250  *
3251  * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
3252  **/
3253 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
3254 		     u32 tx_flags, u64 *cd_type_cmd_tso_mss)
3255 {
3256 	struct i40e_pf *pf;
3257 
3258 	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
3259 		return 0;
3260 
3261 	/* Tx timestamps cannot be sampled when doing TSO */
3262 	if (tx_flags & I40E_TX_FLAGS_TSO)
3263 		return 0;
3264 
3265 	/* only timestamp the outbound packet if the user has requested it and
3266 	 * we are not already transmitting a packet to be timestamped
3267 	 */
3268 	pf = i40e_netdev_to_pf(tx_ring->netdev);
3269 	if (!(pf->flags & I40E_FLAG_PTP))
3270 		return 0;
3271 
3272 	if (pf->ptp_tx &&
3273 	    !test_and_set_bit_lock(__I40E_PTP_TX_IN_PROGRESS, pf->state)) {
3274 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3275 		pf->ptp_tx_start = jiffies;
3276 		pf->ptp_tx_skb = skb_get(skb);
3277 	} else {
3278 		pf->tx_hwtstamp_skipped++;
3279 		return 0;
3280 	}
3281 
3282 	*cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
3283 				I40E_TXD_CTX_QW1_CMD_SHIFT;
3284 
3285 	return 1;
3286 }
3287 
3288 /**
3289  * i40e_tx_enable_csum - Enable Tx checksum offloads
3290  * @skb: send buffer
3291  * @tx_flags: pointer to Tx flags currently set
3292  * @td_cmd: Tx descriptor command bits to set
3293  * @td_offset: Tx descriptor header offsets to set
3294  * @tx_ring: Tx descriptor ring
3295  * @cd_tunneling: ptr to context desc bits
3296  **/
3297 static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
3298 			       u32 *td_cmd, u32 *td_offset,
3299 			       struct i40e_ring *tx_ring,
3300 			       u32 *cd_tunneling)
3301 {
3302 	union {
3303 		struct iphdr *v4;
3304 		struct ipv6hdr *v6;
3305 		unsigned char *hdr;
3306 	} ip;
3307 	union {
3308 		struct tcphdr *tcp;
3309 		struct udphdr *udp;
3310 		unsigned char *hdr;
3311 	} l4;
3312 	unsigned char *exthdr;
3313 	u32 offset, cmd = 0;
3314 	__be16 frag_off;
3315 	__be16 protocol;
3316 	u8 l4_proto = 0;
3317 
3318 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3319 		return 0;
3320 
3321 	protocol = vlan_get_protocol(skb);
3322 
3323 	if (eth_p_mpls(protocol)) {
3324 		ip.hdr = skb_inner_network_header(skb);
3325 		l4.hdr = skb_checksum_start(skb);
3326 	} else {
3327 		ip.hdr = skb_network_header(skb);
3328 		l4.hdr = skb_transport_header(skb);
3329 	}
3330 
3331 	/* set the tx_flags to indicate the IP protocol type. this is
3332 	 * required so that checksum header computation below is accurate.
3333 	 */
3334 	if (ip.v4->version == 4)
3335 		*tx_flags |= I40E_TX_FLAGS_IPV4;
3336 	else
3337 		*tx_flags |= I40E_TX_FLAGS_IPV6;
3338 
3339 	/* compute outer L2 header size */
3340 	offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
3341 
3342 	if (skb->encapsulation) {
3343 		u32 tunnel = 0;
3344 		/* define outer network header type */
3345 		if (*tx_flags & I40E_TX_FLAGS_IPV4) {
3346 			tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
3347 				  I40E_TX_CTX_EXT_IP_IPV4 :
3348 				  I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
3349 
3350 			l4_proto = ip.v4->protocol;
3351 		} else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
3352 			int ret;
3353 
3354 			tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
3355 
3356 			exthdr = ip.hdr + sizeof(*ip.v6);
3357 			l4_proto = ip.v6->nexthdr;
3358 			ret = ipv6_skip_exthdr(skb, exthdr - skb->data,
3359 					       &l4_proto, &frag_off);
3360 			if (ret < 0)
3361 				return -1;
3362 		}
3363 
3364 		/* define outer transport */
3365 		switch (l4_proto) {
3366 		case IPPROTO_UDP:
3367 			tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
3368 			*tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL;
3369 			break;
3370 		case IPPROTO_GRE:
3371 			tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
3372 			*tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL;
3373 			break;
3374 		case IPPROTO_IPIP:
3375 		case IPPROTO_IPV6:
3376 			*tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL;
3377 			l4.hdr = skb_inner_network_header(skb);
3378 			break;
3379 		default:
3380 			if (*tx_flags & I40E_TX_FLAGS_TSO)
3381 				return -1;
3382 
3383 			skb_checksum_help(skb);
3384 			return 0;
3385 		}
3386 
3387 		/* compute outer L3 header size */
3388 		tunnel |= ((l4.hdr - ip.hdr) / 4) <<
3389 			  I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
3390 
3391 		/* switch IP header pointer from outer to inner header */
3392 		ip.hdr = skb_inner_network_header(skb);
3393 
3394 		/* compute tunnel header size */
3395 		tunnel |= ((ip.hdr - l4.hdr) / 2) <<
3396 			  I40E_TXD_CTX_QW0_NATLEN_SHIFT;
3397 
3398 		/* indicate if we need to offload outer UDP header */
3399 		if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
3400 		    !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
3401 		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
3402 			tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;
3403 
3404 		/* record tunnel offload values */
3405 		*cd_tunneling |= tunnel;
3406 
3407 		/* switch L4 header pointer from outer to inner */
3408 		l4.hdr = skb_inner_transport_header(skb);
3409 		l4_proto = 0;
3410 
3411 		/* reset type as we transition from outer to inner headers */
3412 		*tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
3413 		if (ip.v4->version == 4)
3414 			*tx_flags |= I40E_TX_FLAGS_IPV4;
3415 		if (ip.v6->version == 6)
3416 			*tx_flags |= I40E_TX_FLAGS_IPV6;
3417 	}
3418 
3419 	/* Enable IP checksum offloads */
3420 	if (*tx_flags & I40E_TX_FLAGS_IPV4) {
3421 		l4_proto = ip.v4->protocol;
3422 		/* the stack computes the IP header already, the only time we
3423 		 * need the hardware to recompute it is in the case of TSO.
3424 		 */
3425 		cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
3426 		       I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
3427 		       I40E_TX_DESC_CMD_IIPT_IPV4;
3428 	} else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
3429 		cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
3430 
3431 		exthdr = ip.hdr + sizeof(*ip.v6);
3432 		l4_proto = ip.v6->nexthdr;
3433 		if (l4.hdr != exthdr)
3434 			ipv6_skip_exthdr(skb, exthdr - skb->data,
3435 					 &l4_proto, &frag_off);
3436 	}
3437 
3438 	/* compute inner L3 header size */
3439 	offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
3440 
3441 	/* Enable L4 checksum offloads */
3442 	switch (l4_proto) {
3443 	case IPPROTO_TCP:
3444 		/* enable checksum offloads */
3445 		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
3446 		offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
3447 		break;
3448 	case IPPROTO_SCTP:
3449 		/* enable SCTP checksum offload */
3450 		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
3451 		offset |= (sizeof(struct sctphdr) >> 2) <<
3452 			  I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
3453 		break;
3454 	case IPPROTO_UDP:
3455 		/* enable UDP checksum offload */
3456 		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
3457 		offset |= (sizeof(struct udphdr) >> 2) <<
3458 			  I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
3459 		break;
3460 	default:
3461 		if (*tx_flags & I40E_TX_FLAGS_TSO)
3462 			return -1;
3463 		skb_checksum_help(skb);
3464 		return 0;
3465 	}
3466 
3467 	*td_cmd |= cmd;
3468 	*td_offset |= offset;
3469 
3470 	return 1;
3471 }
3472 
3473 /**
3474  * i40e_create_tx_ctx - Build the Tx context descriptor
3475  * @tx_ring:  ring to create the descriptor on
3476  * @cd_type_cmd_tso_mss: Quad Word 1
3477  * @cd_tunneling: Quad Word 0 - bits 0-31
3478  * @cd_l2tag2: Quad Word 0 - bits 32-63
3479  **/
3480 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
3481 			       const u64 cd_type_cmd_tso_mss,
3482 			       const u32 cd_tunneling, const u32 cd_l2tag2)
3483 {
3484 	struct i40e_tx_context_desc *context_desc;
3485 	int i = tx_ring->next_to_use;
3486 
3487 	if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
3488 	    !cd_tunneling && !cd_l2tag2)
3489 		return;
3490 
3491 	/* grab the next descriptor */
3492 	context_desc = I40E_TX_CTXTDESC(tx_ring, i);
3493 
3494 	i++;
3495 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3496 
3497 	/* cpu_to_le32 and assign to struct fields */
3498 	context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
3499 	context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
3500 	context_desc->rsvd = cpu_to_le16(0);
3501 	context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
3502 }
3503 
3504 /**
3505  * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
3506  * @tx_ring: the ring to be checked
3507  * @size:    the size buffer we want to assure is available
3508  *
3509  * Returns -EBUSY if a stop is needed, else 0
3510  **/
3511 int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
3512 {
3513 	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
3514 	/* Memory barrier before checking head and tail */
3515 	smp_mb();
3516 
3517 	++tx_ring->tx_stats.tx_stopped;
3518 
3519 	/* Check again in a case another CPU has just made room available. */
3520 	if (likely(I40E_DESC_UNUSED(tx_ring) < size))
3521 		return -EBUSY;
3522 
3523 	/* A reprieve! - use start_queue because it doesn't call schedule */
3524 	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
3525 	++tx_ring->tx_stats.restart_queue;
3526 	return 0;
3527 }
3528 
3529 /**
3530  * __i40e_chk_linearize - Check if there are more than 8 buffers per packet
3531  * @skb:      send buffer
3532  *
3533  * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire
3534  * and so we need to figure out the cases where we need to linearize the skb.
3535  *
3536  * For TSO we need to count the TSO header and segment payload separately.
3537  * As such we need to check cases where we have 7 fragments or more as we
3538  * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for
3539  * the segment payload in the first descriptor, and another 7 for the
3540  * fragments.
3541  **/
3542 bool __i40e_chk_linearize(struct sk_buff *skb)
3543 {
3544 	const skb_frag_t *frag, *stale;
3545 	int nr_frags, sum;
3546 
3547 	/* no need to check if number of frags is less than 7 */
3548 	nr_frags = skb_shinfo(skb)->nr_frags;
3549 	if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
3550 		return false;
3551 
3552 	/* We need to walk through the list and validate that each group
3553 	 * of 6 fragments totals at least gso_size.
3554 	 */
3555 	nr_frags -= I40E_MAX_BUFFER_TXD - 2;
3556 	frag = &skb_shinfo(skb)->frags[0];
3557 
3558 	/* Initialize size to the negative value of gso_size minus 1.  We
3559 	 * use this as the worst case scenerio in which the frag ahead
3560 	 * of us only provides one byte which is why we are limited to 6
3561 	 * descriptors for a single transmit as the header and previous
3562 	 * fragment are already consuming 2 descriptors.
3563 	 */
3564 	sum = 1 - skb_shinfo(skb)->gso_size;
3565 
3566 	/* Add size of frags 0 through 4 to create our initial sum */
3567 	sum += skb_frag_size(frag++);
3568 	sum += skb_frag_size(frag++);
3569 	sum += skb_frag_size(frag++);
3570 	sum += skb_frag_size(frag++);
3571 	sum += skb_frag_size(frag++);
3572 
3573 	/* Walk through fragments adding latest fragment, testing it, and
3574 	 * then removing stale fragments from the sum.
3575 	 */
3576 	for (stale = &skb_shinfo(skb)->frags[0];; stale++) {
3577 		int stale_size = skb_frag_size(stale);
3578 
3579 		sum += skb_frag_size(frag++);
3580 
3581 		/* The stale fragment may present us with a smaller
3582 		 * descriptor than the actual fragment size. To account
3583 		 * for that we need to remove all the data on the front and
3584 		 * figure out what the remainder would be in the last
3585 		 * descriptor associated with the fragment.
3586 		 */
3587 		if (stale_size > I40E_MAX_DATA_PER_TXD) {
3588 			int align_pad = -(skb_frag_off(stale)) &
3589 					(I40E_MAX_READ_REQ_SIZE - 1);
3590 
3591 			sum -= align_pad;
3592 			stale_size -= align_pad;
3593 
3594 			do {
3595 				sum -= I40E_MAX_DATA_PER_TXD_ALIGNED;
3596 				stale_size -= I40E_MAX_DATA_PER_TXD_ALIGNED;
3597 			} while (stale_size > I40E_MAX_DATA_PER_TXD);
3598 		}
3599 
3600 		/* if sum is negative we failed to make sufficient progress */
3601 		if (sum < 0)
3602 			return true;
3603 
3604 		if (!nr_frags--)
3605 			break;
3606 
3607 		sum -= stale_size;
3608 	}
3609 
3610 	return false;
3611 }
3612 
3613 /**
3614  * i40e_tx_map - Build the Tx descriptor
3615  * @tx_ring:  ring to send buffer on
3616  * @skb:      send buffer
3617  * @first:    first buffer info buffer to use
3618  * @tx_flags: collected send information
3619  * @hdr_len:  size of the packet header
3620  * @td_cmd:   the command field in the descriptor
3621  * @td_offset: offset for checksum or crc
3622  *
3623  * Returns 0 on success, -1 on failure to DMA
3624  **/
3625 static inline int i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
3626 			      struct i40e_tx_buffer *first, u32 tx_flags,
3627 			      const u8 hdr_len, u32 td_cmd, u32 td_offset)
3628 {
3629 	unsigned int data_len = skb->data_len;
3630 	unsigned int size = skb_headlen(skb);
3631 	skb_frag_t *frag;
3632 	struct i40e_tx_buffer *tx_bi;
3633 	struct i40e_tx_desc *tx_desc;
3634 	u16 i = tx_ring->next_to_use;
3635 	u32 td_tag = 0;
3636 	dma_addr_t dma;
3637 	u16 desc_count = 1;
3638 
3639 	if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
3640 		td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
3641 		td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
3642 			 I40E_TX_FLAGS_VLAN_SHIFT;
3643 	}
3644 
3645 	first->tx_flags = tx_flags;
3646 
3647 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3648 
3649 	tx_desc = I40E_TX_DESC(tx_ring, i);
3650 	tx_bi = first;
3651 
3652 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
3653 		unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
3654 
3655 		if (dma_mapping_error(tx_ring->dev, dma))
3656 			goto dma_error;
3657 
3658 		/* record length, and DMA address */
3659 		dma_unmap_len_set(tx_bi, len, size);
3660 		dma_unmap_addr_set(tx_bi, dma, dma);
3661 
3662 		/* align size to end of page */
3663 		max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
3664 		tx_desc->buffer_addr = cpu_to_le64(dma);
3665 
3666 		while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
3667 			tx_desc->cmd_type_offset_bsz =
3668 				build_ctob(td_cmd, td_offset,
3669 					   max_data, td_tag);
3670 
3671 			tx_desc++;
3672 			i++;
3673 			desc_count++;
3674 
3675 			if (i == tx_ring->count) {
3676 				tx_desc = I40E_TX_DESC(tx_ring, 0);
3677 				i = 0;
3678 			}
3679 
3680 			dma += max_data;
3681 			size -= max_data;
3682 
3683 			max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
3684 			tx_desc->buffer_addr = cpu_to_le64(dma);
3685 		}
3686 
3687 		if (likely(!data_len))
3688 			break;
3689 
3690 		tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
3691 							  size, td_tag);
3692 
3693 		tx_desc++;
3694 		i++;
3695 		desc_count++;
3696 
3697 		if (i == tx_ring->count) {
3698 			tx_desc = I40E_TX_DESC(tx_ring, 0);
3699 			i = 0;
3700 		}
3701 
3702 		size = skb_frag_size(frag);
3703 		data_len -= size;
3704 
3705 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3706 				       DMA_TO_DEVICE);
3707 
3708 		tx_bi = &tx_ring->tx_bi[i];
3709 	}
3710 
3711 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
3712 
3713 	i++;
3714 	if (i == tx_ring->count)
3715 		i = 0;
3716 
3717 	tx_ring->next_to_use = i;
3718 
3719 	i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
3720 
3721 	/* write last descriptor with EOP bit */
3722 	td_cmd |= I40E_TX_DESC_CMD_EOP;
3723 
3724 	/* We OR these values together to check both against 4 (WB_STRIDE)
3725 	 * below. This is safe since we don't re-use desc_count afterwards.
3726 	 */
3727 	desc_count |= ++tx_ring->packet_stride;
3728 
3729 	if (desc_count >= WB_STRIDE) {
3730 		/* write last descriptor with RS bit set */
3731 		td_cmd |= I40E_TX_DESC_CMD_RS;
3732 		tx_ring->packet_stride = 0;
3733 	}
3734 
3735 	tx_desc->cmd_type_offset_bsz =
3736 			build_ctob(td_cmd, td_offset, size, td_tag);
3737 
3738 	skb_tx_timestamp(skb);
3739 
3740 	/* Force memory writes to complete before letting h/w know there
3741 	 * are new descriptors to fetch.
3742 	 *
3743 	 * We also use this memory barrier to make certain all of the
3744 	 * status bits have been updated before next_to_watch is written.
3745 	 */
3746 	wmb();
3747 
3748 	/* set next_to_watch value indicating a packet is present */
3749 	first->next_to_watch = tx_desc;
3750 
3751 	/* notify HW of packet */
3752 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
3753 		writel(i, tx_ring->tail);
3754 	}
3755 
3756 	return 0;
3757 
3758 dma_error:
3759 	dev_info(tx_ring->dev, "TX DMA map failed\n");
3760 
3761 	/* clear dma mappings for failed tx_bi map */
3762 	for (;;) {
3763 		tx_bi = &tx_ring->tx_bi[i];
3764 		i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
3765 		if (tx_bi == first)
3766 			break;
3767 		if (i == 0)
3768 			i = tx_ring->count;
3769 		i--;
3770 	}
3771 
3772 	tx_ring->next_to_use = i;
3773 
3774 	return -1;
3775 }
3776 
3777 static u16 i40e_swdcb_skb_tx_hash(struct net_device *dev,
3778 				  const struct sk_buff *skb,
3779 				  u16 num_tx_queues)
3780 {
3781 	u32 jhash_initval_salt = 0xd631614b;
3782 	u32 hash;
3783 
3784 	if (skb->sk && skb->sk->sk_hash)
3785 		hash = skb->sk->sk_hash;
3786 	else
3787 		hash = (__force u16)skb->protocol ^ skb->hash;
3788 
3789 	hash = jhash_1word(hash, jhash_initval_salt);
3790 
3791 	return (u16)(((u64)hash * num_tx_queues) >> 32);
3792 }
3793 
3794 u16 i40e_lan_select_queue(struct net_device *netdev,
3795 			  struct sk_buff *skb,
3796 			  struct net_device __always_unused *sb_dev)
3797 {
3798 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3799 	struct i40e_vsi *vsi = np->vsi;
3800 	struct i40e_hw *hw;
3801 	u16 qoffset;
3802 	u16 qcount;
3803 	u8 tclass;
3804 	u16 hash;
3805 	u8 prio;
3806 
3807 	/* is DCB enabled at all? */
3808 	if (vsi->tc_config.numtc == 1 ||
3809 	    i40e_is_tc_mqprio_enabled(vsi->back))
3810 		return netdev_pick_tx(netdev, skb, sb_dev);
3811 
3812 	prio = skb->priority;
3813 	hw = &vsi->back->hw;
3814 	tclass = hw->local_dcbx_config.etscfg.prioritytable[prio];
3815 	/* sanity check */
3816 	if (unlikely(!(vsi->tc_config.enabled_tc & BIT(tclass))))
3817 		tclass = 0;
3818 
3819 	/* select a queue assigned for the given TC */
3820 	qcount = vsi->tc_config.tc_info[tclass].qcount;
3821 	hash = i40e_swdcb_skb_tx_hash(netdev, skb, qcount);
3822 
3823 	qoffset = vsi->tc_config.tc_info[tclass].qoffset;
3824 	return qoffset + hash;
3825 }
3826 
3827 /**
3828  * i40e_xmit_xdp_ring - transmits an XDP buffer to an XDP Tx ring
3829  * @xdpf: data to transmit
3830  * @xdp_ring: XDP Tx ring
3831  **/
3832 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
3833 			      struct i40e_ring *xdp_ring)
3834 {
3835 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
3836 	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
3837 	u16 i = 0, index = xdp_ring->next_to_use;
3838 	struct i40e_tx_buffer *tx_head = &xdp_ring->tx_bi[index];
3839 	struct i40e_tx_buffer *tx_bi = tx_head;
3840 	struct i40e_tx_desc *tx_desc = I40E_TX_DESC(xdp_ring, index);
3841 	void *data = xdpf->data;
3842 	u32 size = xdpf->len;
3843 
3844 	if (unlikely(I40E_DESC_UNUSED(xdp_ring) < 1 + nr_frags)) {
3845 		xdp_ring->tx_stats.tx_busy++;
3846 		return I40E_XDP_CONSUMED;
3847 	}
3848 
3849 	tx_head->bytecount = xdp_get_frame_len(xdpf);
3850 	tx_head->gso_segs = 1;
3851 	tx_head->xdpf = xdpf;
3852 
3853 	for (;;) {
3854 		dma_addr_t dma;
3855 
3856 		dma = dma_map_single(xdp_ring->dev, data, size, DMA_TO_DEVICE);
3857 		if (dma_mapping_error(xdp_ring->dev, dma))
3858 			goto unmap;
3859 
3860 		/* record length, and DMA address */
3861 		dma_unmap_len_set(tx_bi, len, size);
3862 		dma_unmap_addr_set(tx_bi, dma, dma);
3863 
3864 		tx_desc->buffer_addr = cpu_to_le64(dma);
3865 		tx_desc->cmd_type_offset_bsz =
3866 			build_ctob(I40E_TX_DESC_CMD_ICRC, 0, size, 0);
3867 
3868 		if (++index == xdp_ring->count)
3869 			index = 0;
3870 
3871 		if (i == nr_frags)
3872 			break;
3873 
3874 		tx_bi = &xdp_ring->tx_bi[index];
3875 		tx_desc = I40E_TX_DESC(xdp_ring, index);
3876 
3877 		data = skb_frag_address(&sinfo->frags[i]);
3878 		size = skb_frag_size(&sinfo->frags[i]);
3879 		i++;
3880 	}
3881 
3882 	tx_desc->cmd_type_offset_bsz |=
3883 		cpu_to_le64(I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT);
3884 
3885 	/* Make certain all of the status bits have been updated
3886 	 * before next_to_watch is written.
3887 	 */
3888 	smp_wmb();
3889 
3890 	xdp_ring->xdp_tx_active++;
3891 
3892 	tx_head->next_to_watch = tx_desc;
3893 	xdp_ring->next_to_use = index;
3894 
3895 	return I40E_XDP_TX;
3896 
3897 unmap:
3898 	for (;;) {
3899 		tx_bi = &xdp_ring->tx_bi[index];
3900 		if (dma_unmap_len(tx_bi, len))
3901 			dma_unmap_page(xdp_ring->dev,
3902 				       dma_unmap_addr(tx_bi, dma),
3903 				       dma_unmap_len(tx_bi, len),
3904 				       DMA_TO_DEVICE);
3905 		dma_unmap_len_set(tx_bi, len, 0);
3906 		if (tx_bi == tx_head)
3907 			break;
3908 
3909 		if (!index)
3910 			index += xdp_ring->count;
3911 		index--;
3912 	}
3913 
3914 	return I40E_XDP_CONSUMED;
3915 }
3916 
3917 /**
3918  * i40e_xmit_frame_ring - Sends buffer on Tx ring
3919  * @skb:     send buffer
3920  * @tx_ring: ring to send buffer on
3921  *
3922  * Returns NETDEV_TX_OK if sent, else an error code
3923  **/
3924 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
3925 					struct i40e_ring *tx_ring)
3926 {
3927 	u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
3928 	u32 cd_tunneling = 0, cd_l2tag2 = 0;
3929 	struct i40e_tx_buffer *first;
3930 	u32 td_offset = 0;
3931 	u32 tx_flags = 0;
3932 	u32 td_cmd = 0;
3933 	u8 hdr_len = 0;
3934 	int tso, count;
3935 	int tsyn;
3936 
3937 	/* prefetch the data, we'll need it later */
3938 	prefetch(skb->data);
3939 
3940 	i40e_trace(xmit_frame_ring, skb, tx_ring);
3941 
3942 	count = i40e_xmit_descriptor_count(skb);
3943 	if (i40e_chk_linearize(skb, count)) {
3944 		if (__skb_linearize(skb)) {
3945 			dev_kfree_skb_any(skb);
3946 			return NETDEV_TX_OK;
3947 		}
3948 		count = i40e_txd_use_count(skb->len);
3949 		tx_ring->tx_stats.tx_linearize++;
3950 	}
3951 
3952 	/* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
3953 	 *       + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
3954 	 *       + 4 desc gap to avoid the cache line where head is,
3955 	 *       + 1 desc for context descriptor,
3956 	 * otherwise try next time
3957 	 */
3958 	if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
3959 		tx_ring->tx_stats.tx_busy++;
3960 		return NETDEV_TX_BUSY;
3961 	}
3962 
3963 	/* record the location of the first descriptor for this packet */
3964 	first = &tx_ring->tx_bi[tx_ring->next_to_use];
3965 	first->skb = skb;
3966 	first->bytecount = skb->len;
3967 	first->gso_segs = 1;
3968 
3969 	/* prepare the xmit flags */
3970 	if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
3971 		goto out_drop;
3972 
3973 	tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
3974 
3975 	if (tso < 0)
3976 		goto out_drop;
3977 	else if (tso)
3978 		tx_flags |= I40E_TX_FLAGS_TSO;
3979 
3980 	/* Always offload the checksum, since it's in the data descriptor */
3981 	tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
3982 				  tx_ring, &cd_tunneling);
3983 	if (tso < 0)
3984 		goto out_drop;
3985 
3986 	tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
3987 
3988 	if (tsyn)
3989 		tx_flags |= I40E_TX_FLAGS_TSYN;
3990 
3991 	/* always enable CRC insertion offload */
3992 	td_cmd |= I40E_TX_DESC_CMD_ICRC;
3993 
3994 	i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
3995 			   cd_tunneling, cd_l2tag2);
3996 
3997 	/* Add Flow Director ATR if it's enabled.
3998 	 *
3999 	 * NOTE: this must always be directly before the data descriptor.
4000 	 */
4001 	i40e_atr(tx_ring, skb, tx_flags);
4002 
4003 	if (i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
4004 			td_cmd, td_offset))
4005 		goto cleanup_tx_tstamp;
4006 
4007 	return NETDEV_TX_OK;
4008 
4009 out_drop:
4010 	i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring);
4011 	dev_kfree_skb_any(first->skb);
4012 	first->skb = NULL;
4013 cleanup_tx_tstamp:
4014 	if (unlikely(tx_flags & I40E_TX_FLAGS_TSYN)) {
4015 		struct i40e_pf *pf = i40e_netdev_to_pf(tx_ring->netdev);
4016 
4017 		dev_kfree_skb_any(pf->ptp_tx_skb);
4018 		pf->ptp_tx_skb = NULL;
4019 		clear_bit_unlock(__I40E_PTP_TX_IN_PROGRESS, pf->state);
4020 	}
4021 
4022 	return NETDEV_TX_OK;
4023 }
4024 
4025 /**
4026  * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
4027  * @skb:    send buffer
4028  * @netdev: network interface device structure
4029  *
4030  * Returns NETDEV_TX_OK if sent, else an error code
4031  **/
4032 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
4033 {
4034 	struct i40e_netdev_priv *np = netdev_priv(netdev);
4035 	struct i40e_vsi *vsi = np->vsi;
4036 	struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
4037 
4038 	/* hardware can't handle really short frames, hardware padding works
4039 	 * beyond this point
4040 	 */
4041 	if (skb_put_padto(skb, I40E_MIN_TX_LEN))
4042 		return NETDEV_TX_OK;
4043 
4044 	return i40e_xmit_frame_ring(skb, tx_ring);
4045 }
4046 
4047 /**
4048  * i40e_xdp_xmit - Implements ndo_xdp_xmit
4049  * @dev: netdev
4050  * @n: number of frames
4051  * @frames: array of XDP buffer pointers
4052  * @flags: XDP extra info
4053  *
4054  * Returns number of frames successfully sent. Failed frames
4055  * will be free'ed by XDP core.
4056  *
4057  * For error cases, a negative errno code is returned and no-frames
4058  * are transmitted (caller must handle freeing frames).
4059  **/
4060 int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
4061 		  u32 flags)
4062 {
4063 	struct i40e_netdev_priv *np = netdev_priv(dev);
4064 	unsigned int queue_index = smp_processor_id();
4065 	struct i40e_vsi *vsi = np->vsi;
4066 	struct i40e_pf *pf = vsi->back;
4067 	struct i40e_ring *xdp_ring;
4068 	int nxmit = 0;
4069 	int i;
4070 
4071 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
4072 		return -ENETDOWN;
4073 
4074 	if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs ||
4075 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
4076 		return -ENXIO;
4077 
4078 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
4079 		return -EINVAL;
4080 
4081 	xdp_ring = vsi->xdp_rings[queue_index];
4082 
4083 	for (i = 0; i < n; i++) {
4084 		struct xdp_frame *xdpf = frames[i];
4085 		int err;
4086 
4087 		err = i40e_xmit_xdp_ring(xdpf, xdp_ring);
4088 		if (err != I40E_XDP_TX)
4089 			break;
4090 		nxmit++;
4091 	}
4092 
4093 	if (unlikely(flags & XDP_XMIT_FLUSH))
4094 		i40e_xdp_ring_update_tail(xdp_ring);
4095 
4096 	return nxmit;
4097 }
4098