1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c)  2018 Intel Corporation */
3 
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
7 #include <linux/aer.h>
8 #include <linux/tcp.h>
9 #include <linux/udp.h>
10 #include <linux/ip.h>
11 #include <linux/pm_runtime.h>
12 #include <net/pkt_sched.h>
13 #include <linux/bpf_trace.h>
14 #include <net/xdp_sock_drv.h>
15 #include <linux/pci.h>
16 
17 #include <net/ipv6.h>
18 
19 #include "igc.h"
20 #include "igc_hw.h"
21 #include "igc_tsn.h"
22 #include "igc_xdp.h"
23 
24 #define DRV_SUMMARY	"Intel(R) 2.5G Ethernet Linux Driver"
25 
26 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
27 
28 #define IGC_XDP_PASS		0
29 #define IGC_XDP_CONSUMED	BIT(0)
30 #define IGC_XDP_TX		BIT(1)
31 #define IGC_XDP_REDIRECT	BIT(2)
32 
33 static int debug = -1;
34 
35 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
36 MODULE_DESCRIPTION(DRV_SUMMARY);
37 MODULE_LICENSE("GPL v2");
38 module_param(debug, int, 0);
39 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
40 
41 char igc_driver_name[] = "igc";
42 static const char igc_driver_string[] = DRV_SUMMARY;
43 static const char igc_copyright[] =
44 	"Copyright(c) 2018 Intel Corporation.";
45 
46 static const struct igc_info *igc_info_tbl[] = {
47 	[board_base] = &igc_base_info,
48 };
49 
50 static const struct pci_device_id igc_pci_tbl[] = {
51 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
52 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
53 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
54 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
55 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
56 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
57 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
58 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
59 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
60 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
61 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
62 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
63 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
64 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
65 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
66 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
67 	/* required last entry */
68 	{0, }
69 };
70 
71 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
72 
73 enum latency_range {
74 	lowest_latency = 0,
75 	low_latency = 1,
76 	bulk_latency = 2,
77 	latency_invalid = 255
78 };
79 
80 void igc_reset(struct igc_adapter *adapter)
81 {
82 	struct net_device *dev = adapter->netdev;
83 	struct igc_hw *hw = &adapter->hw;
84 	struct igc_fc_info *fc = &hw->fc;
85 	u32 pba, hwm;
86 
87 	/* Repartition PBA for greater than 9k MTU if required */
88 	pba = IGC_PBA_34K;
89 
90 	/* flow control settings
91 	 * The high water mark must be low enough to fit one full frame
92 	 * after transmitting the pause frame.  As such we must have enough
93 	 * space to allow for us to complete our current transmit and then
94 	 * receive the frame that is in progress from the link partner.
95 	 * Set it to:
96 	 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
97 	 */
98 	hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
99 
100 	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
101 	fc->low_water = fc->high_water - 16;
102 	fc->pause_time = 0xFFFF;
103 	fc->send_xon = 1;
104 	fc->current_mode = fc->requested_mode;
105 
106 	hw->mac.ops.reset_hw(hw);
107 
108 	if (hw->mac.ops.init_hw(hw))
109 		netdev_err(dev, "Error on hardware initialization\n");
110 
111 	/* Re-establish EEE setting */
112 	igc_set_eee_i225(hw, true, true, true);
113 
114 	if (!netif_running(adapter->netdev))
115 		igc_power_down_phy_copper_base(&adapter->hw);
116 
117 	/* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
118 	wr32(IGC_VET, ETH_P_8021Q);
119 
120 	/* Re-enable PTP, where applicable. */
121 	igc_ptp_reset(adapter);
122 
123 	/* Re-enable TSN offloading, where applicable. */
124 	igc_tsn_reset(adapter);
125 
126 	igc_get_phy_info(hw);
127 }
128 
129 /**
130  * igc_power_up_link - Power up the phy link
131  * @adapter: address of board private structure
132  */
133 static void igc_power_up_link(struct igc_adapter *adapter)
134 {
135 	igc_reset_phy(&adapter->hw);
136 
137 	igc_power_up_phy_copper(&adapter->hw);
138 
139 	igc_setup_link(&adapter->hw);
140 }
141 
142 /**
143  * igc_release_hw_control - release control of the h/w to f/w
144  * @adapter: address of board private structure
145  *
146  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
147  * For ASF and Pass Through versions of f/w this means that the
148  * driver is no longer loaded.
149  */
150 static void igc_release_hw_control(struct igc_adapter *adapter)
151 {
152 	struct igc_hw *hw = &adapter->hw;
153 	u32 ctrl_ext;
154 
155 	if (!pci_device_is_present(adapter->pdev))
156 		return;
157 
158 	/* Let firmware take over control of h/w */
159 	ctrl_ext = rd32(IGC_CTRL_EXT);
160 	wr32(IGC_CTRL_EXT,
161 	     ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
162 }
163 
164 /**
165  * igc_get_hw_control - get control of the h/w from f/w
166  * @adapter: address of board private structure
167  *
168  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
169  * For ASF and Pass Through versions of f/w this means that
170  * the driver is loaded.
171  */
172 static void igc_get_hw_control(struct igc_adapter *adapter)
173 {
174 	struct igc_hw *hw = &adapter->hw;
175 	u32 ctrl_ext;
176 
177 	/* Let firmware know the driver has taken over */
178 	ctrl_ext = rd32(IGC_CTRL_EXT);
179 	wr32(IGC_CTRL_EXT,
180 	     ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
181 }
182 
183 static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf)
184 {
185 	dma_unmap_single(dev, dma_unmap_addr(buf, dma),
186 			 dma_unmap_len(buf, len), DMA_TO_DEVICE);
187 
188 	dma_unmap_len_set(buf, len, 0);
189 }
190 
191 /**
192  * igc_clean_tx_ring - Free Tx Buffers
193  * @tx_ring: ring to be cleaned
194  */
195 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
196 {
197 	u16 i = tx_ring->next_to_clean;
198 	struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
199 	u32 xsk_frames = 0;
200 
201 	while (i != tx_ring->next_to_use) {
202 		union igc_adv_tx_desc *eop_desc, *tx_desc;
203 
204 		switch (tx_buffer->type) {
205 		case IGC_TX_BUFFER_TYPE_XSK:
206 			xsk_frames++;
207 			break;
208 		case IGC_TX_BUFFER_TYPE_XDP:
209 			xdp_return_frame(tx_buffer->xdpf);
210 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
211 			break;
212 		case IGC_TX_BUFFER_TYPE_SKB:
213 			dev_kfree_skb_any(tx_buffer->skb);
214 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
215 			break;
216 		default:
217 			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
218 			break;
219 		}
220 
221 		/* check for eop_desc to determine the end of the packet */
222 		eop_desc = tx_buffer->next_to_watch;
223 		tx_desc = IGC_TX_DESC(tx_ring, i);
224 
225 		/* unmap remaining buffers */
226 		while (tx_desc != eop_desc) {
227 			tx_buffer++;
228 			tx_desc++;
229 			i++;
230 			if (unlikely(i == tx_ring->count)) {
231 				i = 0;
232 				tx_buffer = tx_ring->tx_buffer_info;
233 				tx_desc = IGC_TX_DESC(tx_ring, 0);
234 			}
235 
236 			/* unmap any remaining paged data */
237 			if (dma_unmap_len(tx_buffer, len))
238 				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
239 		}
240 
241 		tx_buffer->next_to_watch = NULL;
242 
243 		/* move us one more past the eop_desc for start of next pkt */
244 		tx_buffer++;
245 		i++;
246 		if (unlikely(i == tx_ring->count)) {
247 			i = 0;
248 			tx_buffer = tx_ring->tx_buffer_info;
249 		}
250 	}
251 
252 	if (tx_ring->xsk_pool && xsk_frames)
253 		xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
254 
255 	/* reset BQL for queue */
256 	netdev_tx_reset_queue(txring_txq(tx_ring));
257 
258 	/* reset next_to_use and next_to_clean */
259 	tx_ring->next_to_use = 0;
260 	tx_ring->next_to_clean = 0;
261 }
262 
263 /**
264  * igc_free_tx_resources - Free Tx Resources per Queue
265  * @tx_ring: Tx descriptor ring for a specific queue
266  *
267  * Free all transmit software resources
268  */
269 void igc_free_tx_resources(struct igc_ring *tx_ring)
270 {
271 	igc_clean_tx_ring(tx_ring);
272 
273 	vfree(tx_ring->tx_buffer_info);
274 	tx_ring->tx_buffer_info = NULL;
275 
276 	/* if not set, then don't free */
277 	if (!tx_ring->desc)
278 		return;
279 
280 	dma_free_coherent(tx_ring->dev, tx_ring->size,
281 			  tx_ring->desc, tx_ring->dma);
282 
283 	tx_ring->desc = NULL;
284 }
285 
286 /**
287  * igc_free_all_tx_resources - Free Tx Resources for All Queues
288  * @adapter: board private structure
289  *
290  * Free all transmit software resources
291  */
292 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
293 {
294 	int i;
295 
296 	for (i = 0; i < adapter->num_tx_queues; i++)
297 		igc_free_tx_resources(adapter->tx_ring[i]);
298 }
299 
300 /**
301  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
302  * @adapter: board private structure
303  */
304 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
305 {
306 	int i;
307 
308 	for (i = 0; i < adapter->num_tx_queues; i++)
309 		if (adapter->tx_ring[i])
310 			igc_clean_tx_ring(adapter->tx_ring[i]);
311 }
312 
313 /**
314  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
315  * @tx_ring: tx descriptor ring (for a specific queue) to setup
316  *
317  * Return 0 on success, negative on failure
318  */
319 int igc_setup_tx_resources(struct igc_ring *tx_ring)
320 {
321 	struct net_device *ndev = tx_ring->netdev;
322 	struct device *dev = tx_ring->dev;
323 	int size = 0;
324 
325 	size = sizeof(struct igc_tx_buffer) * tx_ring->count;
326 	tx_ring->tx_buffer_info = vzalloc(size);
327 	if (!tx_ring->tx_buffer_info)
328 		goto err;
329 
330 	/* round up to nearest 4K */
331 	tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
332 	tx_ring->size = ALIGN(tx_ring->size, 4096);
333 
334 	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
335 					   &tx_ring->dma, GFP_KERNEL);
336 
337 	if (!tx_ring->desc)
338 		goto err;
339 
340 	tx_ring->next_to_use = 0;
341 	tx_ring->next_to_clean = 0;
342 
343 	return 0;
344 
345 err:
346 	vfree(tx_ring->tx_buffer_info);
347 	netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
348 	return -ENOMEM;
349 }
350 
351 /**
352  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
353  * @adapter: board private structure
354  *
355  * Return 0 on success, negative on failure
356  */
357 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
358 {
359 	struct net_device *dev = adapter->netdev;
360 	int i, err = 0;
361 
362 	for (i = 0; i < adapter->num_tx_queues; i++) {
363 		err = igc_setup_tx_resources(adapter->tx_ring[i]);
364 		if (err) {
365 			netdev_err(dev, "Error on Tx queue %u setup\n", i);
366 			for (i--; i >= 0; i--)
367 				igc_free_tx_resources(adapter->tx_ring[i]);
368 			break;
369 		}
370 	}
371 
372 	return err;
373 }
374 
375 static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring)
376 {
377 	u16 i = rx_ring->next_to_clean;
378 
379 	dev_kfree_skb(rx_ring->skb);
380 	rx_ring->skb = NULL;
381 
382 	/* Free all the Rx ring sk_buffs */
383 	while (i != rx_ring->next_to_alloc) {
384 		struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
385 
386 		/* Invalidate cache lines that may have been written to by
387 		 * device so that we avoid corrupting memory.
388 		 */
389 		dma_sync_single_range_for_cpu(rx_ring->dev,
390 					      buffer_info->dma,
391 					      buffer_info->page_offset,
392 					      igc_rx_bufsz(rx_ring),
393 					      DMA_FROM_DEVICE);
394 
395 		/* free resources associated with mapping */
396 		dma_unmap_page_attrs(rx_ring->dev,
397 				     buffer_info->dma,
398 				     igc_rx_pg_size(rx_ring),
399 				     DMA_FROM_DEVICE,
400 				     IGC_RX_DMA_ATTR);
401 		__page_frag_cache_drain(buffer_info->page,
402 					buffer_info->pagecnt_bias);
403 
404 		i++;
405 		if (i == rx_ring->count)
406 			i = 0;
407 	}
408 }
409 
410 static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring)
411 {
412 	struct igc_rx_buffer *bi;
413 	u16 i;
414 
415 	for (i = 0; i < ring->count; i++) {
416 		bi = &ring->rx_buffer_info[i];
417 		if (!bi->xdp)
418 			continue;
419 
420 		xsk_buff_free(bi->xdp);
421 		bi->xdp = NULL;
422 	}
423 }
424 
425 /**
426  * igc_clean_rx_ring - Free Rx Buffers per Queue
427  * @ring: ring to free buffers from
428  */
429 static void igc_clean_rx_ring(struct igc_ring *ring)
430 {
431 	if (ring->xsk_pool)
432 		igc_clean_rx_ring_xsk_pool(ring);
433 	else
434 		igc_clean_rx_ring_page_shared(ring);
435 
436 	clear_ring_uses_large_buffer(ring);
437 
438 	ring->next_to_alloc = 0;
439 	ring->next_to_clean = 0;
440 	ring->next_to_use = 0;
441 }
442 
443 /**
444  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
445  * @adapter: board private structure
446  */
447 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
448 {
449 	int i;
450 
451 	for (i = 0; i < adapter->num_rx_queues; i++)
452 		if (adapter->rx_ring[i])
453 			igc_clean_rx_ring(adapter->rx_ring[i]);
454 }
455 
456 /**
457  * igc_free_rx_resources - Free Rx Resources
458  * @rx_ring: ring to clean the resources from
459  *
460  * Free all receive software resources
461  */
462 void igc_free_rx_resources(struct igc_ring *rx_ring)
463 {
464 	igc_clean_rx_ring(rx_ring);
465 
466 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
467 
468 	vfree(rx_ring->rx_buffer_info);
469 	rx_ring->rx_buffer_info = NULL;
470 
471 	/* if not set, then don't free */
472 	if (!rx_ring->desc)
473 		return;
474 
475 	dma_free_coherent(rx_ring->dev, rx_ring->size,
476 			  rx_ring->desc, rx_ring->dma);
477 
478 	rx_ring->desc = NULL;
479 }
480 
481 /**
482  * igc_free_all_rx_resources - Free Rx Resources for All Queues
483  * @adapter: board private structure
484  *
485  * Free all receive software resources
486  */
487 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
488 {
489 	int i;
490 
491 	for (i = 0; i < adapter->num_rx_queues; i++)
492 		igc_free_rx_resources(adapter->rx_ring[i]);
493 }
494 
495 /**
496  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
497  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
498  *
499  * Returns 0 on success, negative on failure
500  */
501 int igc_setup_rx_resources(struct igc_ring *rx_ring)
502 {
503 	struct net_device *ndev = rx_ring->netdev;
504 	struct device *dev = rx_ring->dev;
505 	u8 index = rx_ring->queue_index;
506 	int size, desc_len, res;
507 
508 	/* XDP RX-queue info */
509 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
510 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
511 	res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
512 			       rx_ring->q_vector->napi.napi_id);
513 	if (res < 0) {
514 		netdev_err(ndev, "Failed to register xdp_rxq index %u\n",
515 			   index);
516 		return res;
517 	}
518 
519 	size = sizeof(struct igc_rx_buffer) * rx_ring->count;
520 	rx_ring->rx_buffer_info = vzalloc(size);
521 	if (!rx_ring->rx_buffer_info)
522 		goto err;
523 
524 	desc_len = sizeof(union igc_adv_rx_desc);
525 
526 	/* Round up to nearest 4K */
527 	rx_ring->size = rx_ring->count * desc_len;
528 	rx_ring->size = ALIGN(rx_ring->size, 4096);
529 
530 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
531 					   &rx_ring->dma, GFP_KERNEL);
532 
533 	if (!rx_ring->desc)
534 		goto err;
535 
536 	rx_ring->next_to_alloc = 0;
537 	rx_ring->next_to_clean = 0;
538 	rx_ring->next_to_use = 0;
539 
540 	return 0;
541 
542 err:
543 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
544 	vfree(rx_ring->rx_buffer_info);
545 	rx_ring->rx_buffer_info = NULL;
546 	netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
547 	return -ENOMEM;
548 }
549 
550 /**
551  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
552  *                                (Descriptors) for all queues
553  * @adapter: board private structure
554  *
555  * Return 0 on success, negative on failure
556  */
557 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
558 {
559 	struct net_device *dev = adapter->netdev;
560 	int i, err = 0;
561 
562 	for (i = 0; i < adapter->num_rx_queues; i++) {
563 		err = igc_setup_rx_resources(adapter->rx_ring[i]);
564 		if (err) {
565 			netdev_err(dev, "Error on Rx queue %u setup\n", i);
566 			for (i--; i >= 0; i--)
567 				igc_free_rx_resources(adapter->rx_ring[i]);
568 			break;
569 		}
570 	}
571 
572 	return err;
573 }
574 
575 static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter,
576 					      struct igc_ring *ring)
577 {
578 	if (!igc_xdp_is_enabled(adapter) ||
579 	    !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags))
580 		return NULL;
581 
582 	return xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
583 }
584 
585 /**
586  * igc_configure_rx_ring - Configure a receive ring after Reset
587  * @adapter: board private structure
588  * @ring: receive ring to be configured
589  *
590  * Configure the Rx unit of the MAC after a reset.
591  */
592 static void igc_configure_rx_ring(struct igc_adapter *adapter,
593 				  struct igc_ring *ring)
594 {
595 	struct igc_hw *hw = &adapter->hw;
596 	union igc_adv_rx_desc *rx_desc;
597 	int reg_idx = ring->reg_idx;
598 	u32 srrctl = 0, rxdctl = 0;
599 	u64 rdba = ring->dma;
600 	u32 buf_size;
601 
602 	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
603 	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
604 	if (ring->xsk_pool) {
605 		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
606 						   MEM_TYPE_XSK_BUFF_POOL,
607 						   NULL));
608 		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
609 	} else {
610 		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
611 						   MEM_TYPE_PAGE_SHARED,
612 						   NULL));
613 	}
614 
615 	if (igc_xdp_is_enabled(adapter))
616 		set_ring_uses_large_buffer(ring);
617 
618 	/* disable the queue */
619 	wr32(IGC_RXDCTL(reg_idx), 0);
620 
621 	/* Set DMA base address registers */
622 	wr32(IGC_RDBAL(reg_idx),
623 	     rdba & 0x00000000ffffffffULL);
624 	wr32(IGC_RDBAH(reg_idx), rdba >> 32);
625 	wr32(IGC_RDLEN(reg_idx),
626 	     ring->count * sizeof(union igc_adv_rx_desc));
627 
628 	/* initialize head and tail */
629 	ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
630 	wr32(IGC_RDH(reg_idx), 0);
631 	writel(0, ring->tail);
632 
633 	/* reset next-to- use/clean to place SW in sync with hardware */
634 	ring->next_to_clean = 0;
635 	ring->next_to_use = 0;
636 
637 	if (ring->xsk_pool)
638 		buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool);
639 	else if (ring_uses_large_buffer(ring))
640 		buf_size = IGC_RXBUFFER_3072;
641 	else
642 		buf_size = IGC_RXBUFFER_2048;
643 
644 	srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
645 	srrctl |= buf_size >> IGC_SRRCTL_BSIZEPKT_SHIFT;
646 	srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
647 
648 	wr32(IGC_SRRCTL(reg_idx), srrctl);
649 
650 	rxdctl |= IGC_RX_PTHRESH;
651 	rxdctl |= IGC_RX_HTHRESH << 8;
652 	rxdctl |= IGC_RX_WTHRESH << 16;
653 
654 	/* initialize rx_buffer_info */
655 	memset(ring->rx_buffer_info, 0,
656 	       sizeof(struct igc_rx_buffer) * ring->count);
657 
658 	/* initialize Rx descriptor 0 */
659 	rx_desc = IGC_RX_DESC(ring, 0);
660 	rx_desc->wb.upper.length = 0;
661 
662 	/* enable receive descriptor fetching */
663 	rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
664 
665 	wr32(IGC_RXDCTL(reg_idx), rxdctl);
666 }
667 
668 /**
669  * igc_configure_rx - Configure receive Unit after Reset
670  * @adapter: board private structure
671  *
672  * Configure the Rx unit of the MAC after a reset.
673  */
674 static void igc_configure_rx(struct igc_adapter *adapter)
675 {
676 	int i;
677 
678 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
679 	 * the Base and Length of the Rx Descriptor Ring
680 	 */
681 	for (i = 0; i < adapter->num_rx_queues; i++)
682 		igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
683 }
684 
685 /**
686  * igc_configure_tx_ring - Configure transmit ring after Reset
687  * @adapter: board private structure
688  * @ring: tx ring to configure
689  *
690  * Configure a transmit ring after a reset.
691  */
692 static void igc_configure_tx_ring(struct igc_adapter *adapter,
693 				  struct igc_ring *ring)
694 {
695 	struct igc_hw *hw = &adapter->hw;
696 	int reg_idx = ring->reg_idx;
697 	u64 tdba = ring->dma;
698 	u32 txdctl = 0;
699 
700 	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
701 
702 	/* disable the queue */
703 	wr32(IGC_TXDCTL(reg_idx), 0);
704 	wrfl();
705 	mdelay(10);
706 
707 	wr32(IGC_TDLEN(reg_idx),
708 	     ring->count * sizeof(union igc_adv_tx_desc));
709 	wr32(IGC_TDBAL(reg_idx),
710 	     tdba & 0x00000000ffffffffULL);
711 	wr32(IGC_TDBAH(reg_idx), tdba >> 32);
712 
713 	ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
714 	wr32(IGC_TDH(reg_idx), 0);
715 	writel(0, ring->tail);
716 
717 	txdctl |= IGC_TX_PTHRESH;
718 	txdctl |= IGC_TX_HTHRESH << 8;
719 	txdctl |= IGC_TX_WTHRESH << 16;
720 
721 	txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
722 	wr32(IGC_TXDCTL(reg_idx), txdctl);
723 }
724 
725 /**
726  * igc_configure_tx - Configure transmit Unit after Reset
727  * @adapter: board private structure
728  *
729  * Configure the Tx unit of the MAC after a reset.
730  */
731 static void igc_configure_tx(struct igc_adapter *adapter)
732 {
733 	int i;
734 
735 	for (i = 0; i < adapter->num_tx_queues; i++)
736 		igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
737 }
738 
739 /**
740  * igc_setup_mrqc - configure the multiple receive queue control registers
741  * @adapter: Board private structure
742  */
743 static void igc_setup_mrqc(struct igc_adapter *adapter)
744 {
745 	struct igc_hw *hw = &adapter->hw;
746 	u32 j, num_rx_queues;
747 	u32 mrqc, rxcsum;
748 	u32 rss_key[10];
749 
750 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
751 	for (j = 0; j < 10; j++)
752 		wr32(IGC_RSSRK(j), rss_key[j]);
753 
754 	num_rx_queues = adapter->rss_queues;
755 
756 	if (adapter->rss_indir_tbl_init != num_rx_queues) {
757 		for (j = 0; j < IGC_RETA_SIZE; j++)
758 			adapter->rss_indir_tbl[j] =
759 			(j * num_rx_queues) / IGC_RETA_SIZE;
760 		adapter->rss_indir_tbl_init = num_rx_queues;
761 	}
762 	igc_write_rss_indir_tbl(adapter);
763 
764 	/* Disable raw packet checksumming so that RSS hash is placed in
765 	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
766 	 * offloads as they are enabled by default
767 	 */
768 	rxcsum = rd32(IGC_RXCSUM);
769 	rxcsum |= IGC_RXCSUM_PCSD;
770 
771 	/* Enable Receive Checksum Offload for SCTP */
772 	rxcsum |= IGC_RXCSUM_CRCOFL;
773 
774 	/* Don't need to set TUOFL or IPOFL, they default to 1 */
775 	wr32(IGC_RXCSUM, rxcsum);
776 
777 	/* Generate RSS hash based on packet types, TCP/UDP
778 	 * port numbers and/or IPv4/v6 src and dst addresses
779 	 */
780 	mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
781 	       IGC_MRQC_RSS_FIELD_IPV4_TCP |
782 	       IGC_MRQC_RSS_FIELD_IPV6 |
783 	       IGC_MRQC_RSS_FIELD_IPV6_TCP |
784 	       IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
785 
786 	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
787 		mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
788 	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
789 		mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
790 
791 	mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
792 
793 	wr32(IGC_MRQC, mrqc);
794 }
795 
796 /**
797  * igc_setup_rctl - configure the receive control registers
798  * @adapter: Board private structure
799  */
800 static void igc_setup_rctl(struct igc_adapter *adapter)
801 {
802 	struct igc_hw *hw = &adapter->hw;
803 	u32 rctl;
804 
805 	rctl = rd32(IGC_RCTL);
806 
807 	rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
808 	rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
809 
810 	rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
811 		(hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
812 
813 	/* enable stripping of CRC. Newer features require
814 	 * that the HW strips the CRC.
815 	 */
816 	rctl |= IGC_RCTL_SECRC;
817 
818 	/* disable store bad packets and clear size bits. */
819 	rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
820 
821 	/* enable LPE to allow for reception of jumbo frames */
822 	rctl |= IGC_RCTL_LPE;
823 
824 	/* disable queue 0 to prevent tail write w/o re-config */
825 	wr32(IGC_RXDCTL(0), 0);
826 
827 	/* This is useful for sniffing bad packets. */
828 	if (adapter->netdev->features & NETIF_F_RXALL) {
829 		/* UPE and MPE will be handled by normal PROMISC logic
830 		 * in set_rx_mode
831 		 */
832 		rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
833 			 IGC_RCTL_BAM | /* RX All Bcast Pkts */
834 			 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
835 
836 		rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
837 			  IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
838 	}
839 
840 	wr32(IGC_RCTL, rctl);
841 }
842 
843 /**
844  * igc_setup_tctl - configure the transmit control registers
845  * @adapter: Board private structure
846  */
847 static void igc_setup_tctl(struct igc_adapter *adapter)
848 {
849 	struct igc_hw *hw = &adapter->hw;
850 	u32 tctl;
851 
852 	/* disable queue 0 which icould be enabled by default */
853 	wr32(IGC_TXDCTL(0), 0);
854 
855 	/* Program the Transmit Control Register */
856 	tctl = rd32(IGC_TCTL);
857 	tctl &= ~IGC_TCTL_CT;
858 	tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
859 		(IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
860 
861 	/* Enable transmits */
862 	tctl |= IGC_TCTL_EN;
863 
864 	wr32(IGC_TCTL, tctl);
865 }
866 
867 /**
868  * igc_set_mac_filter_hw() - Set MAC address filter in hardware
869  * @adapter: Pointer to adapter where the filter should be set
870  * @index: Filter index
871  * @type: MAC address filter type (source or destination)
872  * @addr: MAC address
873  * @queue: If non-negative, queue assignment feature is enabled and frames
874  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
875  *         assignment is disabled.
876  */
877 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
878 				  enum igc_mac_filter_type type,
879 				  const u8 *addr, int queue)
880 {
881 	struct net_device *dev = adapter->netdev;
882 	struct igc_hw *hw = &adapter->hw;
883 	u32 ral, rah;
884 
885 	if (WARN_ON(index >= hw->mac.rar_entry_count))
886 		return;
887 
888 	ral = le32_to_cpup((__le32 *)(addr));
889 	rah = le16_to_cpup((__le16 *)(addr + 4));
890 
891 	if (type == IGC_MAC_FILTER_TYPE_SRC) {
892 		rah &= ~IGC_RAH_ASEL_MASK;
893 		rah |= IGC_RAH_ASEL_SRC_ADDR;
894 	}
895 
896 	if (queue >= 0) {
897 		rah &= ~IGC_RAH_QSEL_MASK;
898 		rah |= (queue << IGC_RAH_QSEL_SHIFT);
899 		rah |= IGC_RAH_QSEL_ENABLE;
900 	}
901 
902 	rah |= IGC_RAH_AV;
903 
904 	wr32(IGC_RAL(index), ral);
905 	wr32(IGC_RAH(index), rah);
906 
907 	netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
908 }
909 
910 /**
911  * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
912  * @adapter: Pointer to adapter where the filter should be cleared
913  * @index: Filter index
914  */
915 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
916 {
917 	struct net_device *dev = adapter->netdev;
918 	struct igc_hw *hw = &adapter->hw;
919 
920 	if (WARN_ON(index >= hw->mac.rar_entry_count))
921 		return;
922 
923 	wr32(IGC_RAL(index), 0);
924 	wr32(IGC_RAH(index), 0);
925 
926 	netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
927 }
928 
929 /* Set default MAC address for the PF in the first RAR entry */
930 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
931 {
932 	struct net_device *dev = adapter->netdev;
933 	u8 *addr = adapter->hw.mac.addr;
934 
935 	netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
936 
937 	igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
938 }
939 
940 /**
941  * igc_set_mac - Change the Ethernet Address of the NIC
942  * @netdev: network interface device structure
943  * @p: pointer to an address structure
944  *
945  * Returns 0 on success, negative on failure
946  */
947 static int igc_set_mac(struct net_device *netdev, void *p)
948 {
949 	struct igc_adapter *adapter = netdev_priv(netdev);
950 	struct igc_hw *hw = &adapter->hw;
951 	struct sockaddr *addr = p;
952 
953 	if (!is_valid_ether_addr(addr->sa_data))
954 		return -EADDRNOTAVAIL;
955 
956 	eth_hw_addr_set(netdev, addr->sa_data);
957 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
958 
959 	/* set the correct pool for the new PF MAC address in entry 0 */
960 	igc_set_default_mac_filter(adapter);
961 
962 	return 0;
963 }
964 
965 /**
966  *  igc_write_mc_addr_list - write multicast addresses to MTA
967  *  @netdev: network interface device structure
968  *
969  *  Writes multicast address list to the MTA hash table.
970  *  Returns: -ENOMEM on failure
971  *           0 on no addresses written
972  *           X on writing X addresses to MTA
973  **/
974 static int igc_write_mc_addr_list(struct net_device *netdev)
975 {
976 	struct igc_adapter *adapter = netdev_priv(netdev);
977 	struct igc_hw *hw = &adapter->hw;
978 	struct netdev_hw_addr *ha;
979 	u8  *mta_list;
980 	int i;
981 
982 	if (netdev_mc_empty(netdev)) {
983 		/* nothing to program, so clear mc list */
984 		igc_update_mc_addr_list(hw, NULL, 0);
985 		return 0;
986 	}
987 
988 	mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
989 	if (!mta_list)
990 		return -ENOMEM;
991 
992 	/* The shared function expects a packed array of only addresses. */
993 	i = 0;
994 	netdev_for_each_mc_addr(ha, netdev)
995 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
996 
997 	igc_update_mc_addr_list(hw, mta_list, i);
998 	kfree(mta_list);
999 
1000 	return netdev_mc_count(netdev);
1001 }
1002 
1003 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime)
1004 {
1005 	ktime_t cycle_time = adapter->cycle_time;
1006 	ktime_t base_time = adapter->base_time;
1007 	u32 launchtime;
1008 
1009 	/* FIXME: when using ETF together with taprio, we may have a
1010 	 * case where 'delta' is larger than the cycle_time, this may
1011 	 * cause problems if we don't read the current value of
1012 	 * IGC_BASET, as the value writen into the launchtime
1013 	 * descriptor field may be misinterpreted.
1014 	 */
1015 	div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime);
1016 
1017 	return cpu_to_le32(launchtime);
1018 }
1019 
1020 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1021 			    struct igc_tx_buffer *first,
1022 			    u32 vlan_macip_lens, u32 type_tucmd,
1023 			    u32 mss_l4len_idx)
1024 {
1025 	struct igc_adv_tx_context_desc *context_desc;
1026 	u16 i = tx_ring->next_to_use;
1027 
1028 	context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1029 
1030 	i++;
1031 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1032 
1033 	/* set bits to identify this as an advanced context descriptor */
1034 	type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1035 
1036 	/* For i225, context index must be unique per ring. */
1037 	if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1038 		mss_l4len_idx |= tx_ring->reg_idx << 4;
1039 
1040 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1041 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1042 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1043 
1044 	/* We assume there is always a valid Tx time available. Invalid times
1045 	 * should have been handled by the upper layers.
1046 	 */
1047 	if (tx_ring->launchtime_enable) {
1048 		struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1049 		ktime_t txtime = first->skb->tstamp;
1050 
1051 		skb_txtime_consumed(first->skb);
1052 		context_desc->launch_time = igc_tx_launchtime(adapter,
1053 							      txtime);
1054 	} else {
1055 		context_desc->launch_time = 0;
1056 	}
1057 }
1058 
1059 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first)
1060 {
1061 	struct sk_buff *skb = first->skb;
1062 	u32 vlan_macip_lens = 0;
1063 	u32 type_tucmd = 0;
1064 
1065 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
1066 csum_failed:
1067 		if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1068 		    !tx_ring->launchtime_enable)
1069 			return;
1070 		goto no_csum;
1071 	}
1072 
1073 	switch (skb->csum_offset) {
1074 	case offsetof(struct tcphdr, check):
1075 		type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1076 		fallthrough;
1077 	case offsetof(struct udphdr, check):
1078 		break;
1079 	case offsetof(struct sctphdr, checksum):
1080 		/* validate that this is actually an SCTP request */
1081 		if (skb_csum_is_sctp(skb)) {
1082 			type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1083 			break;
1084 		}
1085 		fallthrough;
1086 	default:
1087 		skb_checksum_help(skb);
1088 		goto csum_failed;
1089 	}
1090 
1091 	/* update TX checksum flag */
1092 	first->tx_flags |= IGC_TX_FLAGS_CSUM;
1093 	vlan_macip_lens = skb_checksum_start_offset(skb) -
1094 			  skb_network_offset(skb);
1095 no_csum:
1096 	vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1097 	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1098 
1099 	igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
1100 }
1101 
1102 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1103 {
1104 	struct net_device *netdev = tx_ring->netdev;
1105 
1106 	netif_stop_subqueue(netdev, tx_ring->queue_index);
1107 
1108 	/* memory barriier comment */
1109 	smp_mb();
1110 
1111 	/* We need to check again in a case another CPU has just
1112 	 * made room available.
1113 	 */
1114 	if (igc_desc_unused(tx_ring) < size)
1115 		return -EBUSY;
1116 
1117 	/* A reprieve! */
1118 	netif_wake_subqueue(netdev, tx_ring->queue_index);
1119 
1120 	u64_stats_update_begin(&tx_ring->tx_syncp2);
1121 	tx_ring->tx_stats.restart_queue2++;
1122 	u64_stats_update_end(&tx_ring->tx_syncp2);
1123 
1124 	return 0;
1125 }
1126 
1127 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1128 {
1129 	if (igc_desc_unused(tx_ring) >= size)
1130 		return 0;
1131 	return __igc_maybe_stop_tx(tx_ring, size);
1132 }
1133 
1134 #define IGC_SET_FLAG(_input, _flag, _result) \
1135 	(((_flag) <= (_result)) ?				\
1136 	 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) :	\
1137 	 ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1138 
1139 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
1140 {
1141 	/* set type for advanced descriptor with frame checksum insertion */
1142 	u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
1143 		       IGC_ADVTXD_DCMD_DEXT |
1144 		       IGC_ADVTXD_DCMD_IFCS;
1145 
1146 	/* set HW vlan bit if vlan is present */
1147 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN,
1148 				 IGC_ADVTXD_DCMD_VLE);
1149 
1150 	/* set segmentation bits for TSO */
1151 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1152 				 (IGC_ADVTXD_DCMD_TSE));
1153 
1154 	/* set timestamp bit if present */
1155 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
1156 				 (IGC_ADVTXD_MAC_TSTAMP));
1157 
1158 	/* insert frame checksum */
1159 	cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS);
1160 
1161 	return cmd_type;
1162 }
1163 
1164 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
1165 				 union igc_adv_tx_desc *tx_desc,
1166 				 u32 tx_flags, unsigned int paylen)
1167 {
1168 	u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
1169 
1170 	/* insert L4 checksum */
1171 	olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
1172 			  ((IGC_TXD_POPTS_TXSM << 8) /
1173 			  IGC_TX_FLAGS_CSUM);
1174 
1175 	/* insert IPv4 checksum */
1176 	olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
1177 			  (((IGC_TXD_POPTS_IXSM << 8)) /
1178 			  IGC_TX_FLAGS_IPV4);
1179 
1180 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1181 }
1182 
1183 static int igc_tx_map(struct igc_ring *tx_ring,
1184 		      struct igc_tx_buffer *first,
1185 		      const u8 hdr_len)
1186 {
1187 	struct sk_buff *skb = first->skb;
1188 	struct igc_tx_buffer *tx_buffer;
1189 	union igc_adv_tx_desc *tx_desc;
1190 	u32 tx_flags = first->tx_flags;
1191 	skb_frag_t *frag;
1192 	u16 i = tx_ring->next_to_use;
1193 	unsigned int data_len, size;
1194 	dma_addr_t dma;
1195 	u32 cmd_type;
1196 
1197 	cmd_type = igc_tx_cmd_type(skb, tx_flags);
1198 	tx_desc = IGC_TX_DESC(tx_ring, i);
1199 
1200 	igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
1201 
1202 	size = skb_headlen(skb);
1203 	data_len = skb->data_len;
1204 
1205 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1206 
1207 	tx_buffer = first;
1208 
1209 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1210 		if (dma_mapping_error(tx_ring->dev, dma))
1211 			goto dma_error;
1212 
1213 		/* record length, and DMA address */
1214 		dma_unmap_len_set(tx_buffer, len, size);
1215 		dma_unmap_addr_set(tx_buffer, dma, dma);
1216 
1217 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
1218 
1219 		while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
1220 			tx_desc->read.cmd_type_len =
1221 				cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
1222 
1223 			i++;
1224 			tx_desc++;
1225 			if (i == tx_ring->count) {
1226 				tx_desc = IGC_TX_DESC(tx_ring, 0);
1227 				i = 0;
1228 			}
1229 			tx_desc->read.olinfo_status = 0;
1230 
1231 			dma += IGC_MAX_DATA_PER_TXD;
1232 			size -= IGC_MAX_DATA_PER_TXD;
1233 
1234 			tx_desc->read.buffer_addr = cpu_to_le64(dma);
1235 		}
1236 
1237 		if (likely(!data_len))
1238 			break;
1239 
1240 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1241 
1242 		i++;
1243 		tx_desc++;
1244 		if (i == tx_ring->count) {
1245 			tx_desc = IGC_TX_DESC(tx_ring, 0);
1246 			i = 0;
1247 		}
1248 		tx_desc->read.olinfo_status = 0;
1249 
1250 		size = skb_frag_size(frag);
1251 		data_len -= size;
1252 
1253 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1254 				       size, DMA_TO_DEVICE);
1255 
1256 		tx_buffer = &tx_ring->tx_buffer_info[i];
1257 	}
1258 
1259 	/* write last descriptor with RS and EOP bits */
1260 	cmd_type |= size | IGC_TXD_DCMD;
1261 	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1262 
1263 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1264 
1265 	/* set the timestamp */
1266 	first->time_stamp = jiffies;
1267 
1268 	skb_tx_timestamp(skb);
1269 
1270 	/* Force memory writes to complete before letting h/w know there
1271 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
1272 	 * memory model archs, such as IA-64).
1273 	 *
1274 	 * We also need this memory barrier to make certain all of the
1275 	 * status bits have been updated before next_to_watch is written.
1276 	 */
1277 	wmb();
1278 
1279 	/* set next_to_watch value indicating a packet is present */
1280 	first->next_to_watch = tx_desc;
1281 
1282 	i++;
1283 	if (i == tx_ring->count)
1284 		i = 0;
1285 
1286 	tx_ring->next_to_use = i;
1287 
1288 	/* Make sure there is space in the ring for the next send. */
1289 	igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1290 
1291 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1292 		writel(i, tx_ring->tail);
1293 	}
1294 
1295 	return 0;
1296 dma_error:
1297 	netdev_err(tx_ring->netdev, "TX DMA map failed\n");
1298 	tx_buffer = &tx_ring->tx_buffer_info[i];
1299 
1300 	/* clear dma mappings for failed tx_buffer_info map */
1301 	while (tx_buffer != first) {
1302 		if (dma_unmap_len(tx_buffer, len))
1303 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1304 
1305 		if (i-- == 0)
1306 			i += tx_ring->count;
1307 		tx_buffer = &tx_ring->tx_buffer_info[i];
1308 	}
1309 
1310 	if (dma_unmap_len(tx_buffer, len))
1311 		igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1312 
1313 	dev_kfree_skb_any(tx_buffer->skb);
1314 	tx_buffer->skb = NULL;
1315 
1316 	tx_ring->next_to_use = i;
1317 
1318 	return -1;
1319 }
1320 
1321 static int igc_tso(struct igc_ring *tx_ring,
1322 		   struct igc_tx_buffer *first,
1323 		   u8 *hdr_len)
1324 {
1325 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1326 	struct sk_buff *skb = first->skb;
1327 	union {
1328 		struct iphdr *v4;
1329 		struct ipv6hdr *v6;
1330 		unsigned char *hdr;
1331 	} ip;
1332 	union {
1333 		struct tcphdr *tcp;
1334 		struct udphdr *udp;
1335 		unsigned char *hdr;
1336 	} l4;
1337 	u32 paylen, l4_offset;
1338 	int err;
1339 
1340 	if (skb->ip_summed != CHECKSUM_PARTIAL)
1341 		return 0;
1342 
1343 	if (!skb_is_gso(skb))
1344 		return 0;
1345 
1346 	err = skb_cow_head(skb, 0);
1347 	if (err < 0)
1348 		return err;
1349 
1350 	ip.hdr = skb_network_header(skb);
1351 	l4.hdr = skb_checksum_start(skb);
1352 
1353 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1354 	type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1355 
1356 	/* initialize outer IP header fields */
1357 	if (ip.v4->version == 4) {
1358 		unsigned char *csum_start = skb_checksum_start(skb);
1359 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1360 
1361 		/* IP header will have to cancel out any data that
1362 		 * is not a part of the outer IP header
1363 		 */
1364 		ip.v4->check = csum_fold(csum_partial(trans_start,
1365 						      csum_start - trans_start,
1366 						      0));
1367 		type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1368 
1369 		ip.v4->tot_len = 0;
1370 		first->tx_flags |= IGC_TX_FLAGS_TSO |
1371 				   IGC_TX_FLAGS_CSUM |
1372 				   IGC_TX_FLAGS_IPV4;
1373 	} else {
1374 		ip.v6->payload_len = 0;
1375 		first->tx_flags |= IGC_TX_FLAGS_TSO |
1376 				   IGC_TX_FLAGS_CSUM;
1377 	}
1378 
1379 	/* determine offset of inner transport header */
1380 	l4_offset = l4.hdr - skb->data;
1381 
1382 	/* remove payload length from inner checksum */
1383 	paylen = skb->len - l4_offset;
1384 	if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1385 		/* compute length of segmentation header */
1386 		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
1387 		csum_replace_by_diff(&l4.tcp->check,
1388 				     (__force __wsum)htonl(paylen));
1389 	} else {
1390 		/* compute length of segmentation header */
1391 		*hdr_len = sizeof(*l4.udp) + l4_offset;
1392 		csum_replace_by_diff(&l4.udp->check,
1393 				     (__force __wsum)htonl(paylen));
1394 	}
1395 
1396 	/* update gso size and bytecount with header size */
1397 	first->gso_segs = skb_shinfo(skb)->gso_segs;
1398 	first->bytecount += (first->gso_segs - 1) * *hdr_len;
1399 
1400 	/* MSS L4LEN IDX */
1401 	mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1402 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1403 
1404 	/* VLAN MACLEN IPLEN */
1405 	vlan_macip_lens = l4.hdr - ip.hdr;
1406 	vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1407 	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1408 
1409 	igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
1410 			type_tucmd, mss_l4len_idx);
1411 
1412 	return 1;
1413 }
1414 
1415 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1416 				       struct igc_ring *tx_ring)
1417 {
1418 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
1419 	__be16 protocol = vlan_get_protocol(skb);
1420 	struct igc_tx_buffer *first;
1421 	u32 tx_flags = 0;
1422 	unsigned short f;
1423 	u8 hdr_len = 0;
1424 	int tso = 0;
1425 
1426 	/* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1427 	 *	+ 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1428 	 *	+ 2 desc gap to keep tail from touching head,
1429 	 *	+ 1 desc for context descriptor,
1430 	 * otherwise try next time
1431 	 */
1432 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1433 		count += TXD_USE_COUNT(skb_frag_size(
1434 						&skb_shinfo(skb)->frags[f]));
1435 
1436 	if (igc_maybe_stop_tx(tx_ring, count + 3)) {
1437 		/* this is a hard error */
1438 		return NETDEV_TX_BUSY;
1439 	}
1440 
1441 	/* record the location of the first descriptor for this packet */
1442 	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1443 	first->type = IGC_TX_BUFFER_TYPE_SKB;
1444 	first->skb = skb;
1445 	first->bytecount = skb->len;
1446 	first->gso_segs = 1;
1447 
1448 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1449 		struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1450 
1451 		/* FIXME: add support for retrieving timestamps from
1452 		 * the other timer registers before skipping the
1453 		 * timestamping request.
1454 		 */
1455 		if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1456 		    !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS,
1457 					   &adapter->state)) {
1458 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1459 			tx_flags |= IGC_TX_FLAGS_TSTAMP;
1460 
1461 			adapter->ptp_tx_skb = skb_get(skb);
1462 			adapter->ptp_tx_start = jiffies;
1463 		} else {
1464 			adapter->tx_hwtstamp_skipped++;
1465 		}
1466 	}
1467 
1468 	if (skb_vlan_tag_present(skb)) {
1469 		tx_flags |= IGC_TX_FLAGS_VLAN;
1470 		tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT);
1471 	}
1472 
1473 	/* record initial flags and protocol */
1474 	first->tx_flags = tx_flags;
1475 	first->protocol = protocol;
1476 
1477 	tso = igc_tso(tx_ring, first, &hdr_len);
1478 	if (tso < 0)
1479 		goto out_drop;
1480 	else if (!tso)
1481 		igc_tx_csum(tx_ring, first);
1482 
1483 	igc_tx_map(tx_ring, first, hdr_len);
1484 
1485 	return NETDEV_TX_OK;
1486 
1487 out_drop:
1488 	dev_kfree_skb_any(first->skb);
1489 	first->skb = NULL;
1490 
1491 	return NETDEV_TX_OK;
1492 }
1493 
1494 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1495 						    struct sk_buff *skb)
1496 {
1497 	unsigned int r_idx = skb->queue_mapping;
1498 
1499 	if (r_idx >= adapter->num_tx_queues)
1500 		r_idx = r_idx % adapter->num_tx_queues;
1501 
1502 	return adapter->tx_ring[r_idx];
1503 }
1504 
1505 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1506 				  struct net_device *netdev)
1507 {
1508 	struct igc_adapter *adapter = netdev_priv(netdev);
1509 
1510 	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1511 	 * in order to meet this minimum size requirement.
1512 	 */
1513 	if (skb->len < 17) {
1514 		if (skb_padto(skb, 17))
1515 			return NETDEV_TX_OK;
1516 		skb->len = 17;
1517 	}
1518 
1519 	return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1520 }
1521 
1522 static void igc_rx_checksum(struct igc_ring *ring,
1523 			    union igc_adv_rx_desc *rx_desc,
1524 			    struct sk_buff *skb)
1525 {
1526 	skb_checksum_none_assert(skb);
1527 
1528 	/* Ignore Checksum bit is set */
1529 	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
1530 		return;
1531 
1532 	/* Rx checksum disabled via ethtool */
1533 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
1534 		return;
1535 
1536 	/* TCP/UDP checksum error bit is set */
1537 	if (igc_test_staterr(rx_desc,
1538 			     IGC_RXDEXT_STATERR_L4E |
1539 			     IGC_RXDEXT_STATERR_IPE)) {
1540 		/* work around errata with sctp packets where the TCPE aka
1541 		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1542 		 * packets (aka let the stack check the crc32c)
1543 		 */
1544 		if (!(skb->len == 60 &&
1545 		      test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
1546 			u64_stats_update_begin(&ring->rx_syncp);
1547 			ring->rx_stats.csum_err++;
1548 			u64_stats_update_end(&ring->rx_syncp);
1549 		}
1550 		/* let the stack verify checksum errors */
1551 		return;
1552 	}
1553 	/* It must be a TCP or UDP packet with a valid checksum */
1554 	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
1555 				      IGC_RXD_STAT_UDPCS))
1556 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1557 
1558 	netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
1559 		   le32_to_cpu(rx_desc->wb.upper.status_error));
1560 }
1561 
1562 static inline void igc_rx_hash(struct igc_ring *ring,
1563 			       union igc_adv_rx_desc *rx_desc,
1564 			       struct sk_buff *skb)
1565 {
1566 	if (ring->netdev->features & NETIF_F_RXHASH)
1567 		skb_set_hash(skb,
1568 			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1569 			     PKT_HASH_TYPE_L3);
1570 }
1571 
1572 static void igc_rx_vlan(struct igc_ring *rx_ring,
1573 			union igc_adv_rx_desc *rx_desc,
1574 			struct sk_buff *skb)
1575 {
1576 	struct net_device *dev = rx_ring->netdev;
1577 	u16 vid;
1578 
1579 	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1580 	    igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) {
1581 		if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) &&
1582 		    test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
1583 			vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
1584 		else
1585 			vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1586 
1587 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1588 	}
1589 }
1590 
1591 /**
1592  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1593  * @rx_ring: rx descriptor ring packet is being transacted on
1594  * @rx_desc: pointer to the EOP Rx descriptor
1595  * @skb: pointer to current skb being populated
1596  *
1597  * This function checks the ring, descriptor, and packet information in order
1598  * to populate the hash, checksum, VLAN, protocol, and other fields within the
1599  * skb.
1600  */
1601 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1602 				   union igc_adv_rx_desc *rx_desc,
1603 				   struct sk_buff *skb)
1604 {
1605 	igc_rx_hash(rx_ring, rx_desc, skb);
1606 
1607 	igc_rx_checksum(rx_ring, rx_desc, skb);
1608 
1609 	igc_rx_vlan(rx_ring, rx_desc, skb);
1610 
1611 	skb_record_rx_queue(skb, rx_ring->queue_index);
1612 
1613 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1614 }
1615 
1616 static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features)
1617 {
1618 	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1619 	struct igc_adapter *adapter = netdev_priv(netdev);
1620 	struct igc_hw *hw = &adapter->hw;
1621 	u32 ctrl;
1622 
1623 	ctrl = rd32(IGC_CTRL);
1624 
1625 	if (enable) {
1626 		/* enable VLAN tag insert/strip */
1627 		ctrl |= IGC_CTRL_VME;
1628 	} else {
1629 		/* disable VLAN tag insert/strip */
1630 		ctrl &= ~IGC_CTRL_VME;
1631 	}
1632 	wr32(IGC_CTRL, ctrl);
1633 }
1634 
1635 static void igc_restore_vlan(struct igc_adapter *adapter)
1636 {
1637 	igc_vlan_mode(adapter->netdev, adapter->netdev->features);
1638 }
1639 
1640 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1641 					       const unsigned int size,
1642 					       int *rx_buffer_pgcnt)
1643 {
1644 	struct igc_rx_buffer *rx_buffer;
1645 
1646 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1647 	*rx_buffer_pgcnt =
1648 #if (PAGE_SIZE < 8192)
1649 		page_count(rx_buffer->page);
1650 #else
1651 		0;
1652 #endif
1653 	prefetchw(rx_buffer->page);
1654 
1655 	/* we are reusing so sync this buffer for CPU use */
1656 	dma_sync_single_range_for_cpu(rx_ring->dev,
1657 				      rx_buffer->dma,
1658 				      rx_buffer->page_offset,
1659 				      size,
1660 				      DMA_FROM_DEVICE);
1661 
1662 	rx_buffer->pagecnt_bias--;
1663 
1664 	return rx_buffer;
1665 }
1666 
1667 static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer,
1668 			       unsigned int truesize)
1669 {
1670 #if (PAGE_SIZE < 8192)
1671 	buffer->page_offset ^= truesize;
1672 #else
1673 	buffer->page_offset += truesize;
1674 #endif
1675 }
1676 
1677 static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring,
1678 					      unsigned int size)
1679 {
1680 	unsigned int truesize;
1681 
1682 #if (PAGE_SIZE < 8192)
1683 	truesize = igc_rx_pg_size(ring) / 2;
1684 #else
1685 	truesize = ring_uses_build_skb(ring) ?
1686 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1687 		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1688 		   SKB_DATA_ALIGN(size);
1689 #endif
1690 	return truesize;
1691 }
1692 
1693 /**
1694  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1695  * @rx_ring: rx descriptor ring to transact packets on
1696  * @rx_buffer: buffer containing page to add
1697  * @skb: sk_buff to place the data into
1698  * @size: size of buffer to be added
1699  *
1700  * This function will add the data contained in rx_buffer->page to the skb.
1701  */
1702 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1703 			    struct igc_rx_buffer *rx_buffer,
1704 			    struct sk_buff *skb,
1705 			    unsigned int size)
1706 {
1707 	unsigned int truesize;
1708 
1709 #if (PAGE_SIZE < 8192)
1710 	truesize = igc_rx_pg_size(rx_ring) / 2;
1711 #else
1712 	truesize = ring_uses_build_skb(rx_ring) ?
1713 		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1714 		   SKB_DATA_ALIGN(size);
1715 #endif
1716 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1717 			rx_buffer->page_offset, size, truesize);
1718 
1719 	igc_rx_buffer_flip(rx_buffer, truesize);
1720 }
1721 
1722 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1723 				     struct igc_rx_buffer *rx_buffer,
1724 				     struct xdp_buff *xdp)
1725 {
1726 	unsigned int size = xdp->data_end - xdp->data;
1727 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1728 	unsigned int metasize = xdp->data - xdp->data_meta;
1729 	struct sk_buff *skb;
1730 
1731 	/* prefetch first cache line of first page */
1732 	net_prefetch(xdp->data_meta);
1733 
1734 	/* build an skb around the page buffer */
1735 	skb = napi_build_skb(xdp->data_hard_start, truesize);
1736 	if (unlikely(!skb))
1737 		return NULL;
1738 
1739 	/* update pointers within the skb to store the data */
1740 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
1741 	__skb_put(skb, size);
1742 	if (metasize)
1743 		skb_metadata_set(skb, metasize);
1744 
1745 	igc_rx_buffer_flip(rx_buffer, truesize);
1746 	return skb;
1747 }
1748 
1749 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1750 					 struct igc_rx_buffer *rx_buffer,
1751 					 struct xdp_buff *xdp,
1752 					 ktime_t timestamp)
1753 {
1754 	unsigned int metasize = xdp->data - xdp->data_meta;
1755 	unsigned int size = xdp->data_end - xdp->data;
1756 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1757 	void *va = xdp->data;
1758 	unsigned int headlen;
1759 	struct sk_buff *skb;
1760 
1761 	/* prefetch first cache line of first page */
1762 	net_prefetch(xdp->data_meta);
1763 
1764 	/* allocate a skb to store the frags */
1765 	skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1766 			     IGC_RX_HDR_LEN + metasize);
1767 	if (unlikely(!skb))
1768 		return NULL;
1769 
1770 	if (timestamp)
1771 		skb_hwtstamps(skb)->hwtstamp = timestamp;
1772 
1773 	/* Determine available headroom for copy */
1774 	headlen = size;
1775 	if (headlen > IGC_RX_HDR_LEN)
1776 		headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1777 
1778 	/* align pull length to size of long to optimize memcpy performance */
1779 	memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta,
1780 	       ALIGN(headlen + metasize, sizeof(long)));
1781 
1782 	if (metasize) {
1783 		skb_metadata_set(skb, metasize);
1784 		__skb_pull(skb, metasize);
1785 	}
1786 
1787 	/* update all of the pointers */
1788 	size -= headlen;
1789 	if (size) {
1790 		skb_add_rx_frag(skb, 0, rx_buffer->page,
1791 				(va + headlen) - page_address(rx_buffer->page),
1792 				size, truesize);
1793 		igc_rx_buffer_flip(rx_buffer, truesize);
1794 	} else {
1795 		rx_buffer->pagecnt_bias++;
1796 	}
1797 
1798 	return skb;
1799 }
1800 
1801 /**
1802  * igc_reuse_rx_page - page flip buffer and store it back on the ring
1803  * @rx_ring: rx descriptor ring to store buffers on
1804  * @old_buff: donor buffer to have page reused
1805  *
1806  * Synchronizes page for reuse by the adapter
1807  */
1808 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1809 			      struct igc_rx_buffer *old_buff)
1810 {
1811 	u16 nta = rx_ring->next_to_alloc;
1812 	struct igc_rx_buffer *new_buff;
1813 
1814 	new_buff = &rx_ring->rx_buffer_info[nta];
1815 
1816 	/* update, and store next to alloc */
1817 	nta++;
1818 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1819 
1820 	/* Transfer page from old buffer to new buffer.
1821 	 * Move each member individually to avoid possible store
1822 	 * forwarding stalls.
1823 	 */
1824 	new_buff->dma		= old_buff->dma;
1825 	new_buff->page		= old_buff->page;
1826 	new_buff->page_offset	= old_buff->page_offset;
1827 	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
1828 }
1829 
1830 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer,
1831 				  int rx_buffer_pgcnt)
1832 {
1833 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1834 	struct page *page = rx_buffer->page;
1835 
1836 	/* avoid re-using remote and pfmemalloc pages */
1837 	if (!dev_page_is_reusable(page))
1838 		return false;
1839 
1840 #if (PAGE_SIZE < 8192)
1841 	/* if we are only owner of page we can reuse it */
1842 	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
1843 		return false;
1844 #else
1845 #define IGC_LAST_OFFSET \
1846 	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1847 
1848 	if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1849 		return false;
1850 #endif
1851 
1852 	/* If we have drained the page fragment pool we need to update
1853 	 * the pagecnt_bias and page count so that we fully restock the
1854 	 * number of references the driver holds.
1855 	 */
1856 	if (unlikely(pagecnt_bias == 1)) {
1857 		page_ref_add(page, USHRT_MAX - 1);
1858 		rx_buffer->pagecnt_bias = USHRT_MAX;
1859 	}
1860 
1861 	return true;
1862 }
1863 
1864 /**
1865  * igc_is_non_eop - process handling of non-EOP buffers
1866  * @rx_ring: Rx ring being processed
1867  * @rx_desc: Rx descriptor for current buffer
1868  *
1869  * This function updates next to clean.  If the buffer is an EOP buffer
1870  * this function exits returning false, otherwise it will place the
1871  * sk_buff in the next buffer to be chained and return true indicating
1872  * that this is in fact a non-EOP buffer.
1873  */
1874 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1875 			   union igc_adv_rx_desc *rx_desc)
1876 {
1877 	u32 ntc = rx_ring->next_to_clean + 1;
1878 
1879 	/* fetch, update, and store next to clean */
1880 	ntc = (ntc < rx_ring->count) ? ntc : 0;
1881 	rx_ring->next_to_clean = ntc;
1882 
1883 	prefetch(IGC_RX_DESC(rx_ring, ntc));
1884 
1885 	if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
1886 		return false;
1887 
1888 	return true;
1889 }
1890 
1891 /**
1892  * igc_cleanup_headers - Correct corrupted or empty headers
1893  * @rx_ring: rx descriptor ring packet is being transacted on
1894  * @rx_desc: pointer to the EOP Rx descriptor
1895  * @skb: pointer to current skb being fixed
1896  *
1897  * Address the case where we are pulling data in on pages only
1898  * and as such no data is present in the skb header.
1899  *
1900  * In addition if skb is not at least 60 bytes we need to pad it so that
1901  * it is large enough to qualify as a valid Ethernet frame.
1902  *
1903  * Returns true if an error was encountered and skb was freed.
1904  */
1905 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
1906 				union igc_adv_rx_desc *rx_desc,
1907 				struct sk_buff *skb)
1908 {
1909 	/* XDP packets use error pointer so abort at this point */
1910 	if (IS_ERR(skb))
1911 		return true;
1912 
1913 	if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
1914 		struct net_device *netdev = rx_ring->netdev;
1915 
1916 		if (!(netdev->features & NETIF_F_RXALL)) {
1917 			dev_kfree_skb_any(skb);
1918 			return true;
1919 		}
1920 	}
1921 
1922 	/* if eth_skb_pad returns an error the skb was freed */
1923 	if (eth_skb_pad(skb))
1924 		return true;
1925 
1926 	return false;
1927 }
1928 
1929 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
1930 			      struct igc_rx_buffer *rx_buffer,
1931 			      int rx_buffer_pgcnt)
1932 {
1933 	if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
1934 		/* hand second half of page back to the ring */
1935 		igc_reuse_rx_page(rx_ring, rx_buffer);
1936 	} else {
1937 		/* We are not reusing the buffer so unmap it and free
1938 		 * any references we are holding to it
1939 		 */
1940 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1941 				     igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
1942 				     IGC_RX_DMA_ATTR);
1943 		__page_frag_cache_drain(rx_buffer->page,
1944 					rx_buffer->pagecnt_bias);
1945 	}
1946 
1947 	/* clear contents of rx_buffer */
1948 	rx_buffer->page = NULL;
1949 }
1950 
1951 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
1952 {
1953 	struct igc_adapter *adapter = rx_ring->q_vector->adapter;
1954 
1955 	if (ring_uses_build_skb(rx_ring))
1956 		return IGC_SKB_PAD;
1957 	if (igc_xdp_is_enabled(adapter))
1958 		return XDP_PACKET_HEADROOM;
1959 
1960 	return 0;
1961 }
1962 
1963 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
1964 				  struct igc_rx_buffer *bi)
1965 {
1966 	struct page *page = bi->page;
1967 	dma_addr_t dma;
1968 
1969 	/* since we are recycling buffers we should seldom need to alloc */
1970 	if (likely(page))
1971 		return true;
1972 
1973 	/* alloc new page for storage */
1974 	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
1975 	if (unlikely(!page)) {
1976 		rx_ring->rx_stats.alloc_failed++;
1977 		return false;
1978 	}
1979 
1980 	/* map page for use */
1981 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1982 				 igc_rx_pg_size(rx_ring),
1983 				 DMA_FROM_DEVICE,
1984 				 IGC_RX_DMA_ATTR);
1985 
1986 	/* if mapping failed free memory back to system since
1987 	 * there isn't much point in holding memory we can't use
1988 	 */
1989 	if (dma_mapping_error(rx_ring->dev, dma)) {
1990 		__free_page(page);
1991 
1992 		rx_ring->rx_stats.alloc_failed++;
1993 		return false;
1994 	}
1995 
1996 	bi->dma = dma;
1997 	bi->page = page;
1998 	bi->page_offset = igc_rx_offset(rx_ring);
1999 	page_ref_add(page, USHRT_MAX - 1);
2000 	bi->pagecnt_bias = USHRT_MAX;
2001 
2002 	return true;
2003 }
2004 
2005 /**
2006  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2007  * @rx_ring: rx descriptor ring
2008  * @cleaned_count: number of buffers to clean
2009  */
2010 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
2011 {
2012 	union igc_adv_rx_desc *rx_desc;
2013 	u16 i = rx_ring->next_to_use;
2014 	struct igc_rx_buffer *bi;
2015 	u16 bufsz;
2016 
2017 	/* nothing to do */
2018 	if (!cleaned_count)
2019 		return;
2020 
2021 	rx_desc = IGC_RX_DESC(rx_ring, i);
2022 	bi = &rx_ring->rx_buffer_info[i];
2023 	i -= rx_ring->count;
2024 
2025 	bufsz = igc_rx_bufsz(rx_ring);
2026 
2027 	do {
2028 		if (!igc_alloc_mapped_page(rx_ring, bi))
2029 			break;
2030 
2031 		/* sync the buffer for use by the device */
2032 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
2033 						 bi->page_offset, bufsz,
2034 						 DMA_FROM_DEVICE);
2035 
2036 		/* Refresh the desc even if buffer_addrs didn't change
2037 		 * because each write-back erases this info.
2038 		 */
2039 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
2040 
2041 		rx_desc++;
2042 		bi++;
2043 		i++;
2044 		if (unlikely(!i)) {
2045 			rx_desc = IGC_RX_DESC(rx_ring, 0);
2046 			bi = rx_ring->rx_buffer_info;
2047 			i -= rx_ring->count;
2048 		}
2049 
2050 		/* clear the length for the next_to_use descriptor */
2051 		rx_desc->wb.upper.length = 0;
2052 
2053 		cleaned_count--;
2054 	} while (cleaned_count);
2055 
2056 	i += rx_ring->count;
2057 
2058 	if (rx_ring->next_to_use != i) {
2059 		/* record the next descriptor to use */
2060 		rx_ring->next_to_use = i;
2061 
2062 		/* update next to alloc since we have filled the ring */
2063 		rx_ring->next_to_alloc = i;
2064 
2065 		/* Force memory writes to complete before letting h/w
2066 		 * know there are new descriptors to fetch.  (Only
2067 		 * applicable for weak-ordered memory model archs,
2068 		 * such as IA-64).
2069 		 */
2070 		wmb();
2071 		writel(i, rx_ring->tail);
2072 	}
2073 }
2074 
2075 static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
2076 {
2077 	union igc_adv_rx_desc *desc;
2078 	u16 i = ring->next_to_use;
2079 	struct igc_rx_buffer *bi;
2080 	dma_addr_t dma;
2081 	bool ok = true;
2082 
2083 	if (!count)
2084 		return ok;
2085 
2086 	desc = IGC_RX_DESC(ring, i);
2087 	bi = &ring->rx_buffer_info[i];
2088 	i -= ring->count;
2089 
2090 	do {
2091 		bi->xdp = xsk_buff_alloc(ring->xsk_pool);
2092 		if (!bi->xdp) {
2093 			ok = false;
2094 			break;
2095 		}
2096 
2097 		dma = xsk_buff_xdp_get_dma(bi->xdp);
2098 		desc->read.pkt_addr = cpu_to_le64(dma);
2099 
2100 		desc++;
2101 		bi++;
2102 		i++;
2103 		if (unlikely(!i)) {
2104 			desc = IGC_RX_DESC(ring, 0);
2105 			bi = ring->rx_buffer_info;
2106 			i -= ring->count;
2107 		}
2108 
2109 		/* Clear the length for the next_to_use descriptor. */
2110 		desc->wb.upper.length = 0;
2111 
2112 		count--;
2113 	} while (count);
2114 
2115 	i += ring->count;
2116 
2117 	if (ring->next_to_use != i) {
2118 		ring->next_to_use = i;
2119 
2120 		/* Force memory writes to complete before letting h/w
2121 		 * know there are new descriptors to fetch.  (Only
2122 		 * applicable for weak-ordered memory model archs,
2123 		 * such as IA-64).
2124 		 */
2125 		wmb();
2126 		writel(i, ring->tail);
2127 	}
2128 
2129 	return ok;
2130 }
2131 
2132 /* This function requires __netif_tx_lock is held by the caller. */
2133 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
2134 				      struct xdp_frame *xdpf)
2135 {
2136 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
2137 	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
2138 	u16 count, index = ring->next_to_use;
2139 	struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
2140 	struct igc_tx_buffer *buffer = head;
2141 	union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
2142 	u32 olinfo_status, len = xdpf->len, cmd_type;
2143 	void *data = xdpf->data;
2144 	u16 i;
2145 
2146 	count = TXD_USE_COUNT(len);
2147 	for (i = 0; i < nr_frags; i++)
2148 		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
2149 
2150 	if (igc_maybe_stop_tx(ring, count + 3)) {
2151 		/* this is a hard error */
2152 		return -EBUSY;
2153 	}
2154 
2155 	i = 0;
2156 	head->bytecount = xdp_get_frame_len(xdpf);
2157 	head->type = IGC_TX_BUFFER_TYPE_XDP;
2158 	head->gso_segs = 1;
2159 	head->xdpf = xdpf;
2160 
2161 	olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
2162 	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2163 
2164 	for (;;) {
2165 		dma_addr_t dma;
2166 
2167 		dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
2168 		if (dma_mapping_error(ring->dev, dma)) {
2169 			netdev_err_once(ring->netdev,
2170 					"Failed to map DMA for TX\n");
2171 			goto unmap;
2172 		}
2173 
2174 		dma_unmap_len_set(buffer, len, len);
2175 		dma_unmap_addr_set(buffer, dma, dma);
2176 
2177 		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2178 			   IGC_ADVTXD_DCMD_IFCS | len;
2179 
2180 		desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2181 		desc->read.buffer_addr = cpu_to_le64(dma);
2182 
2183 		buffer->protocol = 0;
2184 
2185 		if (++index == ring->count)
2186 			index = 0;
2187 
2188 		if (i == nr_frags)
2189 			break;
2190 
2191 		buffer = &ring->tx_buffer_info[index];
2192 		desc = IGC_TX_DESC(ring, index);
2193 		desc->read.olinfo_status = 0;
2194 
2195 		data = skb_frag_address(&sinfo->frags[i]);
2196 		len = skb_frag_size(&sinfo->frags[i]);
2197 		i++;
2198 	}
2199 	desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
2200 
2201 	netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
2202 	/* set the timestamp */
2203 	head->time_stamp = jiffies;
2204 	/* set next_to_watch value indicating a packet is present */
2205 	head->next_to_watch = desc;
2206 	ring->next_to_use = index;
2207 
2208 	return 0;
2209 
2210 unmap:
2211 	for (;;) {
2212 		buffer = &ring->tx_buffer_info[index];
2213 		if (dma_unmap_len(buffer, len))
2214 			dma_unmap_page(ring->dev,
2215 				       dma_unmap_addr(buffer, dma),
2216 				       dma_unmap_len(buffer, len),
2217 				       DMA_TO_DEVICE);
2218 		dma_unmap_len_set(buffer, len, 0);
2219 		if (buffer == head)
2220 			break;
2221 
2222 		if (!index)
2223 			index += ring->count;
2224 		index--;
2225 	}
2226 
2227 	return -ENOMEM;
2228 }
2229 
2230 static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
2231 					    int cpu)
2232 {
2233 	int index = cpu;
2234 
2235 	if (unlikely(index < 0))
2236 		index = 0;
2237 
2238 	while (index >= adapter->num_tx_queues)
2239 		index -= adapter->num_tx_queues;
2240 
2241 	return adapter->tx_ring[index];
2242 }
2243 
2244 static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp)
2245 {
2246 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2247 	int cpu = smp_processor_id();
2248 	struct netdev_queue *nq;
2249 	struct igc_ring *ring;
2250 	int res;
2251 
2252 	if (unlikely(!xdpf))
2253 		return -EFAULT;
2254 
2255 	ring = igc_xdp_get_tx_ring(adapter, cpu);
2256 	nq = txring_txq(ring);
2257 
2258 	__netif_tx_lock(nq, cpu);
2259 	res = igc_xdp_init_tx_descriptor(ring, xdpf);
2260 	__netif_tx_unlock(nq);
2261 	return res;
2262 }
2263 
2264 /* This function assumes rcu_read_lock() is held by the caller. */
2265 static int __igc_xdp_run_prog(struct igc_adapter *adapter,
2266 			      struct bpf_prog *prog,
2267 			      struct xdp_buff *xdp)
2268 {
2269 	u32 act = bpf_prog_run_xdp(prog, xdp);
2270 
2271 	switch (act) {
2272 	case XDP_PASS:
2273 		return IGC_XDP_PASS;
2274 	case XDP_TX:
2275 		if (igc_xdp_xmit_back(adapter, xdp) < 0)
2276 			goto out_failure;
2277 		return IGC_XDP_TX;
2278 	case XDP_REDIRECT:
2279 		if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0)
2280 			goto out_failure;
2281 		return IGC_XDP_REDIRECT;
2282 		break;
2283 	default:
2284 		bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
2285 		fallthrough;
2286 	case XDP_ABORTED:
2287 out_failure:
2288 		trace_xdp_exception(adapter->netdev, prog, act);
2289 		fallthrough;
2290 	case XDP_DROP:
2291 		return IGC_XDP_CONSUMED;
2292 	}
2293 }
2294 
2295 static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
2296 					struct xdp_buff *xdp)
2297 {
2298 	struct bpf_prog *prog;
2299 	int res;
2300 
2301 	prog = READ_ONCE(adapter->xdp_prog);
2302 	if (!prog) {
2303 		res = IGC_XDP_PASS;
2304 		goto out;
2305 	}
2306 
2307 	res = __igc_xdp_run_prog(adapter, prog, xdp);
2308 
2309 out:
2310 	return ERR_PTR(-res);
2311 }
2312 
2313 /* This function assumes __netif_tx_lock is held by the caller. */
2314 static void igc_flush_tx_descriptors(struct igc_ring *ring)
2315 {
2316 	/* Once tail pointer is updated, hardware can fetch the descriptors
2317 	 * any time so we issue a write membar here to ensure all memory
2318 	 * writes are complete before the tail pointer is updated.
2319 	 */
2320 	wmb();
2321 	writel(ring->next_to_use, ring->tail);
2322 }
2323 
2324 static void igc_finalize_xdp(struct igc_adapter *adapter, int status)
2325 {
2326 	int cpu = smp_processor_id();
2327 	struct netdev_queue *nq;
2328 	struct igc_ring *ring;
2329 
2330 	if (status & IGC_XDP_TX) {
2331 		ring = igc_xdp_get_tx_ring(adapter, cpu);
2332 		nq = txring_txq(ring);
2333 
2334 		__netif_tx_lock(nq, cpu);
2335 		igc_flush_tx_descriptors(ring);
2336 		__netif_tx_unlock(nq);
2337 	}
2338 
2339 	if (status & IGC_XDP_REDIRECT)
2340 		xdp_do_flush();
2341 }
2342 
2343 static void igc_update_rx_stats(struct igc_q_vector *q_vector,
2344 				unsigned int packets, unsigned int bytes)
2345 {
2346 	struct igc_ring *ring = q_vector->rx.ring;
2347 
2348 	u64_stats_update_begin(&ring->rx_syncp);
2349 	ring->rx_stats.packets += packets;
2350 	ring->rx_stats.bytes += bytes;
2351 	u64_stats_update_end(&ring->rx_syncp);
2352 
2353 	q_vector->rx.total_packets += packets;
2354 	q_vector->rx.total_bytes += bytes;
2355 }
2356 
2357 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
2358 {
2359 	unsigned int total_bytes = 0, total_packets = 0;
2360 	struct igc_adapter *adapter = q_vector->adapter;
2361 	struct igc_ring *rx_ring = q_vector->rx.ring;
2362 	struct sk_buff *skb = rx_ring->skb;
2363 	u16 cleaned_count = igc_desc_unused(rx_ring);
2364 	int xdp_status = 0, rx_buffer_pgcnt;
2365 
2366 	while (likely(total_packets < budget)) {
2367 		union igc_adv_rx_desc *rx_desc;
2368 		struct igc_rx_buffer *rx_buffer;
2369 		unsigned int size, truesize;
2370 		ktime_t timestamp = 0;
2371 		struct xdp_buff xdp;
2372 		int pkt_offset = 0;
2373 		void *pktbuf;
2374 
2375 		/* return some buffers to hardware, one at a time is too slow */
2376 		if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
2377 			igc_alloc_rx_buffers(rx_ring, cleaned_count);
2378 			cleaned_count = 0;
2379 		}
2380 
2381 		rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
2382 		size = le16_to_cpu(rx_desc->wb.upper.length);
2383 		if (!size)
2384 			break;
2385 
2386 		/* This memory barrier is needed to keep us from reading
2387 		 * any other fields out of the rx_desc until we know the
2388 		 * descriptor has been written back
2389 		 */
2390 		dma_rmb();
2391 
2392 		rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
2393 		truesize = igc_get_rx_frame_truesize(rx_ring, size);
2394 
2395 		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
2396 
2397 		if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
2398 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2399 							pktbuf);
2400 			pkt_offset = IGC_TS_HDR_LEN;
2401 			size -= IGC_TS_HDR_LEN;
2402 		}
2403 
2404 		if (!skb) {
2405 			xdp_init_buff(&xdp, truesize, &rx_ring->xdp_rxq);
2406 			xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring),
2407 					 igc_rx_offset(rx_ring) + pkt_offset,
2408 					 size, true);
2409 			xdp_buff_clear_frags_flag(&xdp);
2410 
2411 			skb = igc_xdp_run_prog(adapter, &xdp);
2412 		}
2413 
2414 		if (IS_ERR(skb)) {
2415 			unsigned int xdp_res = -PTR_ERR(skb);
2416 
2417 			switch (xdp_res) {
2418 			case IGC_XDP_CONSUMED:
2419 				rx_buffer->pagecnt_bias++;
2420 				break;
2421 			case IGC_XDP_TX:
2422 			case IGC_XDP_REDIRECT:
2423 				igc_rx_buffer_flip(rx_buffer, truesize);
2424 				xdp_status |= xdp_res;
2425 				break;
2426 			}
2427 
2428 			total_packets++;
2429 			total_bytes += size;
2430 		} else if (skb)
2431 			igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
2432 		else if (ring_uses_build_skb(rx_ring))
2433 			skb = igc_build_skb(rx_ring, rx_buffer, &xdp);
2434 		else
2435 			skb = igc_construct_skb(rx_ring, rx_buffer, &xdp,
2436 						timestamp);
2437 
2438 		/* exit if we failed to retrieve a buffer */
2439 		if (!skb) {
2440 			rx_ring->rx_stats.alloc_failed++;
2441 			rx_buffer->pagecnt_bias++;
2442 			break;
2443 		}
2444 
2445 		igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
2446 		cleaned_count++;
2447 
2448 		/* fetch next buffer in frame if non-eop */
2449 		if (igc_is_non_eop(rx_ring, rx_desc))
2450 			continue;
2451 
2452 		/* verify the packet layout is correct */
2453 		if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
2454 			skb = NULL;
2455 			continue;
2456 		}
2457 
2458 		/* probably a little skewed due to removing CRC */
2459 		total_bytes += skb->len;
2460 
2461 		/* populate checksum, VLAN, and protocol */
2462 		igc_process_skb_fields(rx_ring, rx_desc, skb);
2463 
2464 		napi_gro_receive(&q_vector->napi, skb);
2465 
2466 		/* reset skb pointer */
2467 		skb = NULL;
2468 
2469 		/* update budget accounting */
2470 		total_packets++;
2471 	}
2472 
2473 	if (xdp_status)
2474 		igc_finalize_xdp(adapter, xdp_status);
2475 
2476 	/* place incomplete frames back on ring for completion */
2477 	rx_ring->skb = skb;
2478 
2479 	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2480 
2481 	if (cleaned_count)
2482 		igc_alloc_rx_buffers(rx_ring, cleaned_count);
2483 
2484 	return total_packets;
2485 }
2486 
2487 static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
2488 					    struct xdp_buff *xdp)
2489 {
2490 	unsigned int totalsize = xdp->data_end - xdp->data_meta;
2491 	unsigned int metasize = xdp->data - xdp->data_meta;
2492 	struct sk_buff *skb;
2493 
2494 	net_prefetch(xdp->data_meta);
2495 
2496 	skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,
2497 			       GFP_ATOMIC | __GFP_NOWARN);
2498 	if (unlikely(!skb))
2499 		return NULL;
2500 
2501 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,
2502 	       ALIGN(totalsize, sizeof(long)));
2503 
2504 	if (metasize) {
2505 		skb_metadata_set(skb, metasize);
2506 		__skb_pull(skb, metasize);
2507 	}
2508 
2509 	return skb;
2510 }
2511 
2512 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,
2513 				union igc_adv_rx_desc *desc,
2514 				struct xdp_buff *xdp,
2515 				ktime_t timestamp)
2516 {
2517 	struct igc_ring *ring = q_vector->rx.ring;
2518 	struct sk_buff *skb;
2519 
2520 	skb = igc_construct_skb_zc(ring, xdp);
2521 	if (!skb) {
2522 		ring->rx_stats.alloc_failed++;
2523 		return;
2524 	}
2525 
2526 	if (timestamp)
2527 		skb_hwtstamps(skb)->hwtstamp = timestamp;
2528 
2529 	if (igc_cleanup_headers(ring, desc, skb))
2530 		return;
2531 
2532 	igc_process_skb_fields(ring, desc, skb);
2533 	napi_gro_receive(&q_vector->napi, skb);
2534 }
2535 
2536 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
2537 {
2538 	struct igc_adapter *adapter = q_vector->adapter;
2539 	struct igc_ring *ring = q_vector->rx.ring;
2540 	u16 cleaned_count = igc_desc_unused(ring);
2541 	int total_bytes = 0, total_packets = 0;
2542 	u16 ntc = ring->next_to_clean;
2543 	struct bpf_prog *prog;
2544 	bool failure = false;
2545 	int xdp_status = 0;
2546 
2547 	rcu_read_lock();
2548 
2549 	prog = READ_ONCE(adapter->xdp_prog);
2550 
2551 	while (likely(total_packets < budget)) {
2552 		union igc_adv_rx_desc *desc;
2553 		struct igc_rx_buffer *bi;
2554 		ktime_t timestamp = 0;
2555 		unsigned int size;
2556 		int res;
2557 
2558 		desc = IGC_RX_DESC(ring, ntc);
2559 		size = le16_to_cpu(desc->wb.upper.length);
2560 		if (!size)
2561 			break;
2562 
2563 		/* This memory barrier is needed to keep us from reading
2564 		 * any other fields out of the rx_desc until we know the
2565 		 * descriptor has been written back
2566 		 */
2567 		dma_rmb();
2568 
2569 		bi = &ring->rx_buffer_info[ntc];
2570 
2571 		if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
2572 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2573 							bi->xdp->data);
2574 
2575 			bi->xdp->data += IGC_TS_HDR_LEN;
2576 
2577 			/* HW timestamp has been copied into local variable. Metadata
2578 			 * length when XDP program is called should be 0.
2579 			 */
2580 			bi->xdp->data_meta += IGC_TS_HDR_LEN;
2581 			size -= IGC_TS_HDR_LEN;
2582 		}
2583 
2584 		bi->xdp->data_end = bi->xdp->data + size;
2585 		xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool);
2586 
2587 		res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
2588 		switch (res) {
2589 		case IGC_XDP_PASS:
2590 			igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);
2591 			fallthrough;
2592 		case IGC_XDP_CONSUMED:
2593 			xsk_buff_free(bi->xdp);
2594 			break;
2595 		case IGC_XDP_TX:
2596 		case IGC_XDP_REDIRECT:
2597 			xdp_status |= res;
2598 			break;
2599 		}
2600 
2601 		bi->xdp = NULL;
2602 		total_bytes += size;
2603 		total_packets++;
2604 		cleaned_count++;
2605 		ntc++;
2606 		if (ntc == ring->count)
2607 			ntc = 0;
2608 	}
2609 
2610 	ring->next_to_clean = ntc;
2611 	rcu_read_unlock();
2612 
2613 	if (cleaned_count >= IGC_RX_BUFFER_WRITE)
2614 		failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count);
2615 
2616 	if (xdp_status)
2617 		igc_finalize_xdp(adapter, xdp_status);
2618 
2619 	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2620 
2621 	if (xsk_uses_need_wakeup(ring->xsk_pool)) {
2622 		if (failure || ring->next_to_clean == ring->next_to_use)
2623 			xsk_set_rx_need_wakeup(ring->xsk_pool);
2624 		else
2625 			xsk_clear_rx_need_wakeup(ring->xsk_pool);
2626 		return total_packets;
2627 	}
2628 
2629 	return failure ? budget : total_packets;
2630 }
2631 
2632 static void igc_update_tx_stats(struct igc_q_vector *q_vector,
2633 				unsigned int packets, unsigned int bytes)
2634 {
2635 	struct igc_ring *ring = q_vector->tx.ring;
2636 
2637 	u64_stats_update_begin(&ring->tx_syncp);
2638 	ring->tx_stats.bytes += bytes;
2639 	ring->tx_stats.packets += packets;
2640 	u64_stats_update_end(&ring->tx_syncp);
2641 
2642 	q_vector->tx.total_bytes += bytes;
2643 	q_vector->tx.total_packets += packets;
2644 }
2645 
2646 static void igc_xdp_xmit_zc(struct igc_ring *ring)
2647 {
2648 	struct xsk_buff_pool *pool = ring->xsk_pool;
2649 	struct netdev_queue *nq = txring_txq(ring);
2650 	union igc_adv_tx_desc *tx_desc = NULL;
2651 	int cpu = smp_processor_id();
2652 	u16 ntu = ring->next_to_use;
2653 	struct xdp_desc xdp_desc;
2654 	u16 budget;
2655 
2656 	if (!netif_carrier_ok(ring->netdev))
2657 		return;
2658 
2659 	__netif_tx_lock(nq, cpu);
2660 
2661 	budget = igc_desc_unused(ring);
2662 
2663 	while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) {
2664 		u32 cmd_type, olinfo_status;
2665 		struct igc_tx_buffer *bi;
2666 		dma_addr_t dma;
2667 
2668 		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2669 			   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
2670 			   xdp_desc.len;
2671 		olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT;
2672 
2673 		dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
2674 		xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len);
2675 
2676 		tx_desc = IGC_TX_DESC(ring, ntu);
2677 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2678 		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2679 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
2680 
2681 		bi = &ring->tx_buffer_info[ntu];
2682 		bi->type = IGC_TX_BUFFER_TYPE_XSK;
2683 		bi->protocol = 0;
2684 		bi->bytecount = xdp_desc.len;
2685 		bi->gso_segs = 1;
2686 		bi->time_stamp = jiffies;
2687 		bi->next_to_watch = tx_desc;
2688 
2689 		netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len);
2690 
2691 		ntu++;
2692 		if (ntu == ring->count)
2693 			ntu = 0;
2694 	}
2695 
2696 	ring->next_to_use = ntu;
2697 	if (tx_desc) {
2698 		igc_flush_tx_descriptors(ring);
2699 		xsk_tx_release(pool);
2700 	}
2701 
2702 	__netif_tx_unlock(nq);
2703 }
2704 
2705 /**
2706  * igc_clean_tx_irq - Reclaim resources after transmit completes
2707  * @q_vector: pointer to q_vector containing needed info
2708  * @napi_budget: Used to determine if we are in netpoll
2709  *
2710  * returns true if ring is completely cleaned
2711  */
2712 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
2713 {
2714 	struct igc_adapter *adapter = q_vector->adapter;
2715 	unsigned int total_bytes = 0, total_packets = 0;
2716 	unsigned int budget = q_vector->tx.work_limit;
2717 	struct igc_ring *tx_ring = q_vector->tx.ring;
2718 	unsigned int i = tx_ring->next_to_clean;
2719 	struct igc_tx_buffer *tx_buffer;
2720 	union igc_adv_tx_desc *tx_desc;
2721 	u32 xsk_frames = 0;
2722 
2723 	if (test_bit(__IGC_DOWN, &adapter->state))
2724 		return true;
2725 
2726 	tx_buffer = &tx_ring->tx_buffer_info[i];
2727 	tx_desc = IGC_TX_DESC(tx_ring, i);
2728 	i -= tx_ring->count;
2729 
2730 	do {
2731 		union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
2732 
2733 		/* if next_to_watch is not set then there is no work pending */
2734 		if (!eop_desc)
2735 			break;
2736 
2737 		/* prevent any other reads prior to eop_desc */
2738 		smp_rmb();
2739 
2740 		/* if DD is not set pending work has not been completed */
2741 		if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
2742 			break;
2743 
2744 		/* clear next_to_watch to prevent false hangs */
2745 		tx_buffer->next_to_watch = NULL;
2746 
2747 		/* update the statistics for this packet */
2748 		total_bytes += tx_buffer->bytecount;
2749 		total_packets += tx_buffer->gso_segs;
2750 
2751 		switch (tx_buffer->type) {
2752 		case IGC_TX_BUFFER_TYPE_XSK:
2753 			xsk_frames++;
2754 			break;
2755 		case IGC_TX_BUFFER_TYPE_XDP:
2756 			xdp_return_frame(tx_buffer->xdpf);
2757 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2758 			break;
2759 		case IGC_TX_BUFFER_TYPE_SKB:
2760 			napi_consume_skb(tx_buffer->skb, napi_budget);
2761 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2762 			break;
2763 		default:
2764 			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
2765 			break;
2766 		}
2767 
2768 		/* clear last DMA location and unmap remaining buffers */
2769 		while (tx_desc != eop_desc) {
2770 			tx_buffer++;
2771 			tx_desc++;
2772 			i++;
2773 			if (unlikely(!i)) {
2774 				i -= tx_ring->count;
2775 				tx_buffer = tx_ring->tx_buffer_info;
2776 				tx_desc = IGC_TX_DESC(tx_ring, 0);
2777 			}
2778 
2779 			/* unmap any remaining paged data */
2780 			if (dma_unmap_len(tx_buffer, len))
2781 				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2782 		}
2783 
2784 		/* move us one more past the eop_desc for start of next pkt */
2785 		tx_buffer++;
2786 		tx_desc++;
2787 		i++;
2788 		if (unlikely(!i)) {
2789 			i -= tx_ring->count;
2790 			tx_buffer = tx_ring->tx_buffer_info;
2791 			tx_desc = IGC_TX_DESC(tx_ring, 0);
2792 		}
2793 
2794 		/* issue prefetch for next Tx descriptor */
2795 		prefetch(tx_desc);
2796 
2797 		/* update budget accounting */
2798 		budget--;
2799 	} while (likely(budget));
2800 
2801 	netdev_tx_completed_queue(txring_txq(tx_ring),
2802 				  total_packets, total_bytes);
2803 
2804 	i += tx_ring->count;
2805 	tx_ring->next_to_clean = i;
2806 
2807 	igc_update_tx_stats(q_vector, total_packets, total_bytes);
2808 
2809 	if (tx_ring->xsk_pool) {
2810 		if (xsk_frames)
2811 			xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
2812 		if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
2813 			xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
2814 		igc_xdp_xmit_zc(tx_ring);
2815 	}
2816 
2817 	if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
2818 		struct igc_hw *hw = &adapter->hw;
2819 
2820 		/* Detect a transmit hang in hardware, this serializes the
2821 		 * check with the clearing of time_stamp and movement of i
2822 		 */
2823 		clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
2824 		if (tx_buffer->next_to_watch &&
2825 		    time_after(jiffies, tx_buffer->time_stamp +
2826 		    (adapter->tx_timeout_factor * HZ)) &&
2827 		    !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
2828 			/* detected Tx unit hang */
2829 			netdev_err(tx_ring->netdev,
2830 				   "Detected Tx Unit Hang\n"
2831 				   "  Tx Queue             <%d>\n"
2832 				   "  TDH                  <%x>\n"
2833 				   "  TDT                  <%x>\n"
2834 				   "  next_to_use          <%x>\n"
2835 				   "  next_to_clean        <%x>\n"
2836 				   "buffer_info[next_to_clean]\n"
2837 				   "  time_stamp           <%lx>\n"
2838 				   "  next_to_watch        <%p>\n"
2839 				   "  jiffies              <%lx>\n"
2840 				   "  desc.status          <%x>\n",
2841 				   tx_ring->queue_index,
2842 				   rd32(IGC_TDH(tx_ring->reg_idx)),
2843 				   readl(tx_ring->tail),
2844 				   tx_ring->next_to_use,
2845 				   tx_ring->next_to_clean,
2846 				   tx_buffer->time_stamp,
2847 				   tx_buffer->next_to_watch,
2848 				   jiffies,
2849 				   tx_buffer->next_to_watch->wb.status);
2850 			netif_stop_subqueue(tx_ring->netdev,
2851 					    tx_ring->queue_index);
2852 
2853 			/* we are about to reset, no point in enabling stuff */
2854 			return true;
2855 		}
2856 	}
2857 
2858 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
2859 	if (unlikely(total_packets &&
2860 		     netif_carrier_ok(tx_ring->netdev) &&
2861 		     igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
2862 		/* Make sure that anybody stopping the queue after this
2863 		 * sees the new next_to_clean.
2864 		 */
2865 		smp_mb();
2866 		if (__netif_subqueue_stopped(tx_ring->netdev,
2867 					     tx_ring->queue_index) &&
2868 		    !(test_bit(__IGC_DOWN, &adapter->state))) {
2869 			netif_wake_subqueue(tx_ring->netdev,
2870 					    tx_ring->queue_index);
2871 
2872 			u64_stats_update_begin(&tx_ring->tx_syncp);
2873 			tx_ring->tx_stats.restart_queue++;
2874 			u64_stats_update_end(&tx_ring->tx_syncp);
2875 		}
2876 	}
2877 
2878 	return !!budget;
2879 }
2880 
2881 static int igc_find_mac_filter(struct igc_adapter *adapter,
2882 			       enum igc_mac_filter_type type, const u8 *addr)
2883 {
2884 	struct igc_hw *hw = &adapter->hw;
2885 	int max_entries = hw->mac.rar_entry_count;
2886 	u32 ral, rah;
2887 	int i;
2888 
2889 	for (i = 0; i < max_entries; i++) {
2890 		ral = rd32(IGC_RAL(i));
2891 		rah = rd32(IGC_RAH(i));
2892 
2893 		if (!(rah & IGC_RAH_AV))
2894 			continue;
2895 		if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
2896 			continue;
2897 		if ((rah & IGC_RAH_RAH_MASK) !=
2898 		    le16_to_cpup((__le16 *)(addr + 4)))
2899 			continue;
2900 		if (ral != le32_to_cpup((__le32 *)(addr)))
2901 			continue;
2902 
2903 		return i;
2904 	}
2905 
2906 	return -1;
2907 }
2908 
2909 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
2910 {
2911 	struct igc_hw *hw = &adapter->hw;
2912 	int max_entries = hw->mac.rar_entry_count;
2913 	u32 rah;
2914 	int i;
2915 
2916 	for (i = 0; i < max_entries; i++) {
2917 		rah = rd32(IGC_RAH(i));
2918 
2919 		if (!(rah & IGC_RAH_AV))
2920 			return i;
2921 	}
2922 
2923 	return -1;
2924 }
2925 
2926 /**
2927  * igc_add_mac_filter() - Add MAC address filter
2928  * @adapter: Pointer to adapter where the filter should be added
2929  * @type: MAC address filter type (source or destination)
2930  * @addr: MAC address
2931  * @queue: If non-negative, queue assignment feature is enabled and frames
2932  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
2933  *         assignment is disabled.
2934  *
2935  * Return: 0 in case of success, negative errno code otherwise.
2936  */
2937 static int igc_add_mac_filter(struct igc_adapter *adapter,
2938 			      enum igc_mac_filter_type type, const u8 *addr,
2939 			      int queue)
2940 {
2941 	struct net_device *dev = adapter->netdev;
2942 	int index;
2943 
2944 	index = igc_find_mac_filter(adapter, type, addr);
2945 	if (index >= 0)
2946 		goto update_filter;
2947 
2948 	index = igc_get_avail_mac_filter_slot(adapter);
2949 	if (index < 0)
2950 		return -ENOSPC;
2951 
2952 	netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
2953 		   index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
2954 		   addr, queue);
2955 
2956 update_filter:
2957 	igc_set_mac_filter_hw(adapter, index, type, addr, queue);
2958 	return 0;
2959 }
2960 
2961 /**
2962  * igc_del_mac_filter() - Delete MAC address filter
2963  * @adapter: Pointer to adapter where the filter should be deleted from
2964  * @type: MAC address filter type (source or destination)
2965  * @addr: MAC address
2966  */
2967 static void igc_del_mac_filter(struct igc_adapter *adapter,
2968 			       enum igc_mac_filter_type type, const u8 *addr)
2969 {
2970 	struct net_device *dev = adapter->netdev;
2971 	int index;
2972 
2973 	index = igc_find_mac_filter(adapter, type, addr);
2974 	if (index < 0)
2975 		return;
2976 
2977 	if (index == 0) {
2978 		/* If this is the default filter, we don't actually delete it.
2979 		 * We just reset to its default value i.e. disable queue
2980 		 * assignment.
2981 		 */
2982 		netdev_dbg(dev, "Disable default MAC filter queue assignment");
2983 
2984 		igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
2985 	} else {
2986 		netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
2987 			   index,
2988 			   type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
2989 			   addr);
2990 
2991 		igc_clear_mac_filter_hw(adapter, index);
2992 	}
2993 }
2994 
2995 /**
2996  * igc_add_vlan_prio_filter() - Add VLAN priority filter
2997  * @adapter: Pointer to adapter where the filter should be added
2998  * @prio: VLAN priority value
2999  * @queue: Queue number which matching frames are assigned to
3000  *
3001  * Return: 0 in case of success, negative errno code otherwise.
3002  */
3003 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
3004 				    int queue)
3005 {
3006 	struct net_device *dev = adapter->netdev;
3007 	struct igc_hw *hw = &adapter->hw;
3008 	u32 vlanpqf;
3009 
3010 	vlanpqf = rd32(IGC_VLANPQF);
3011 
3012 	if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
3013 		netdev_dbg(dev, "VLAN priority filter already in use\n");
3014 		return -EEXIST;
3015 	}
3016 
3017 	vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
3018 	vlanpqf |= IGC_VLANPQF_VALID(prio);
3019 
3020 	wr32(IGC_VLANPQF, vlanpqf);
3021 
3022 	netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
3023 		   prio, queue);
3024 	return 0;
3025 }
3026 
3027 /**
3028  * igc_del_vlan_prio_filter() - Delete VLAN priority filter
3029  * @adapter: Pointer to adapter where the filter should be deleted from
3030  * @prio: VLAN priority value
3031  */
3032 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
3033 {
3034 	struct igc_hw *hw = &adapter->hw;
3035 	u32 vlanpqf;
3036 
3037 	vlanpqf = rd32(IGC_VLANPQF);
3038 
3039 	vlanpqf &= ~IGC_VLANPQF_VALID(prio);
3040 	vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
3041 
3042 	wr32(IGC_VLANPQF, vlanpqf);
3043 
3044 	netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
3045 		   prio);
3046 }
3047 
3048 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
3049 {
3050 	struct igc_hw *hw = &adapter->hw;
3051 	int i;
3052 
3053 	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3054 		u32 etqf = rd32(IGC_ETQF(i));
3055 
3056 		if (!(etqf & IGC_ETQF_FILTER_ENABLE))
3057 			return i;
3058 	}
3059 
3060 	return -1;
3061 }
3062 
3063 /**
3064  * igc_add_etype_filter() - Add ethertype filter
3065  * @adapter: Pointer to adapter where the filter should be added
3066  * @etype: Ethertype value
3067  * @queue: If non-negative, queue assignment feature is enabled and frames
3068  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3069  *         assignment is disabled.
3070  *
3071  * Return: 0 in case of success, negative errno code otherwise.
3072  */
3073 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
3074 				int queue)
3075 {
3076 	struct igc_hw *hw = &adapter->hw;
3077 	int index;
3078 	u32 etqf;
3079 
3080 	index = igc_get_avail_etype_filter_slot(adapter);
3081 	if (index < 0)
3082 		return -ENOSPC;
3083 
3084 	etqf = rd32(IGC_ETQF(index));
3085 
3086 	etqf &= ~IGC_ETQF_ETYPE_MASK;
3087 	etqf |= etype;
3088 
3089 	if (queue >= 0) {
3090 		etqf &= ~IGC_ETQF_QUEUE_MASK;
3091 		etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
3092 		etqf |= IGC_ETQF_QUEUE_ENABLE;
3093 	}
3094 
3095 	etqf |= IGC_ETQF_FILTER_ENABLE;
3096 
3097 	wr32(IGC_ETQF(index), etqf);
3098 
3099 	netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
3100 		   etype, queue);
3101 	return 0;
3102 }
3103 
3104 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
3105 {
3106 	struct igc_hw *hw = &adapter->hw;
3107 	int i;
3108 
3109 	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3110 		u32 etqf = rd32(IGC_ETQF(i));
3111 
3112 		if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
3113 			return i;
3114 	}
3115 
3116 	return -1;
3117 }
3118 
3119 /**
3120  * igc_del_etype_filter() - Delete ethertype filter
3121  * @adapter: Pointer to adapter where the filter should be deleted from
3122  * @etype: Ethertype value
3123  */
3124 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
3125 {
3126 	struct igc_hw *hw = &adapter->hw;
3127 	int index;
3128 
3129 	index = igc_find_etype_filter(adapter, etype);
3130 	if (index < 0)
3131 		return;
3132 
3133 	wr32(IGC_ETQF(index), 0);
3134 
3135 	netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
3136 		   etype);
3137 }
3138 
3139 static int igc_flex_filter_select(struct igc_adapter *adapter,
3140 				  struct igc_flex_filter *input,
3141 				  u32 *fhft)
3142 {
3143 	struct igc_hw *hw = &adapter->hw;
3144 	u8 fhft_index;
3145 	u32 fhftsl;
3146 
3147 	if (input->index >= MAX_FLEX_FILTER) {
3148 		dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n");
3149 		return -EINVAL;
3150 	}
3151 
3152 	/* Indirect table select register */
3153 	fhftsl = rd32(IGC_FHFTSL);
3154 	fhftsl &= ~IGC_FHFTSL_FTSL_MASK;
3155 	switch (input->index) {
3156 	case 0 ... 7:
3157 		fhftsl |= 0x00;
3158 		break;
3159 	case 8 ... 15:
3160 		fhftsl |= 0x01;
3161 		break;
3162 	case 16 ... 23:
3163 		fhftsl |= 0x02;
3164 		break;
3165 	case 24 ... 31:
3166 		fhftsl |= 0x03;
3167 		break;
3168 	}
3169 	wr32(IGC_FHFTSL, fhftsl);
3170 
3171 	/* Normalize index down to host table register */
3172 	fhft_index = input->index % 8;
3173 
3174 	*fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) :
3175 		IGC_FHFT_EXT(fhft_index - 4);
3176 
3177 	return 0;
3178 }
3179 
3180 static int igc_write_flex_filter_ll(struct igc_adapter *adapter,
3181 				    struct igc_flex_filter *input)
3182 {
3183 	struct device *dev = &adapter->pdev->dev;
3184 	struct igc_hw *hw = &adapter->hw;
3185 	u8 *data = input->data;
3186 	u8 *mask = input->mask;
3187 	u32 queuing;
3188 	u32 fhft;
3189 	u32 wufc;
3190 	int ret;
3191 	int i;
3192 
3193 	/* Length has to be aligned to 8. Otherwise the filter will fail. Bail
3194 	 * out early to avoid surprises later.
3195 	 */
3196 	if (input->length % 8 != 0) {
3197 		dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n");
3198 		return -EINVAL;
3199 	}
3200 
3201 	/* Select corresponding flex filter register and get base for host table. */
3202 	ret = igc_flex_filter_select(adapter, input, &fhft);
3203 	if (ret)
3204 		return ret;
3205 
3206 	/* When adding a filter globally disable flex filter feature. That is
3207 	 * recommended within the datasheet.
3208 	 */
3209 	wufc = rd32(IGC_WUFC);
3210 	wufc &= ~IGC_WUFC_FLEX_HQ;
3211 	wr32(IGC_WUFC, wufc);
3212 
3213 	/* Configure filter */
3214 	queuing = input->length & IGC_FHFT_LENGTH_MASK;
3215 	queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK;
3216 	queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK;
3217 
3218 	if (input->immediate_irq)
3219 		queuing |= IGC_FHFT_IMM_INT;
3220 
3221 	if (input->drop)
3222 		queuing |= IGC_FHFT_DROP;
3223 
3224 	wr32(fhft + 0xFC, queuing);
3225 
3226 	/* Write data (128 byte) and mask (128 bit) */
3227 	for (i = 0; i < 16; ++i) {
3228 		const size_t data_idx = i * 8;
3229 		const size_t row_idx = i * 16;
3230 		u32 dw0 =
3231 			(data[data_idx + 0] << 0) |
3232 			(data[data_idx + 1] << 8) |
3233 			(data[data_idx + 2] << 16) |
3234 			(data[data_idx + 3] << 24);
3235 		u32 dw1 =
3236 			(data[data_idx + 4] << 0) |
3237 			(data[data_idx + 5] << 8) |
3238 			(data[data_idx + 6] << 16) |
3239 			(data[data_idx + 7] << 24);
3240 		u32 tmp;
3241 
3242 		/* Write row: dw0, dw1 and mask */
3243 		wr32(fhft + row_idx, dw0);
3244 		wr32(fhft + row_idx + 4, dw1);
3245 
3246 		/* mask is only valid for MASK(7, 0) */
3247 		tmp = rd32(fhft + row_idx + 8);
3248 		tmp &= ~GENMASK(7, 0);
3249 		tmp |= mask[i];
3250 		wr32(fhft + row_idx + 8, tmp);
3251 	}
3252 
3253 	/* Enable filter. */
3254 	wufc |= IGC_WUFC_FLEX_HQ;
3255 	if (input->index > 8) {
3256 		/* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
3257 		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3258 
3259 		wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8));
3260 
3261 		wr32(IGC_WUFC_EXT, wufc_ext);
3262 	} else {
3263 		wufc |= (IGC_WUFC_FLX0 << input->index);
3264 	}
3265 	wr32(IGC_WUFC, wufc);
3266 
3267 	dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n",
3268 		input->index);
3269 
3270 	return 0;
3271 }
3272 
3273 static void igc_flex_filter_add_field(struct igc_flex_filter *flex,
3274 				      const void *src, unsigned int offset,
3275 				      size_t len, const void *mask)
3276 {
3277 	int i;
3278 
3279 	/* data */
3280 	memcpy(&flex->data[offset], src, len);
3281 
3282 	/* mask */
3283 	for (i = 0; i < len; ++i) {
3284 		const unsigned int idx = i + offset;
3285 		const u8 *ptr = mask;
3286 
3287 		if (mask) {
3288 			if (ptr[i] & 0xff)
3289 				flex->mask[idx / 8] |= BIT(idx % 8);
3290 
3291 			continue;
3292 		}
3293 
3294 		flex->mask[idx / 8] |= BIT(idx % 8);
3295 	}
3296 }
3297 
3298 static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter)
3299 {
3300 	struct igc_hw *hw = &adapter->hw;
3301 	u32 wufc, wufc_ext;
3302 	int i;
3303 
3304 	wufc = rd32(IGC_WUFC);
3305 	wufc_ext = rd32(IGC_WUFC_EXT);
3306 
3307 	for (i = 0; i < MAX_FLEX_FILTER; i++) {
3308 		if (i < 8) {
3309 			if (!(wufc & (IGC_WUFC_FLX0 << i)))
3310 				return i;
3311 		} else {
3312 			if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8))))
3313 				return i;
3314 		}
3315 	}
3316 
3317 	return -ENOSPC;
3318 }
3319 
3320 static bool igc_flex_filter_in_use(struct igc_adapter *adapter)
3321 {
3322 	struct igc_hw *hw = &adapter->hw;
3323 	u32 wufc, wufc_ext;
3324 
3325 	wufc = rd32(IGC_WUFC);
3326 	wufc_ext = rd32(IGC_WUFC_EXT);
3327 
3328 	if (wufc & IGC_WUFC_FILTER_MASK)
3329 		return true;
3330 
3331 	if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK)
3332 		return true;
3333 
3334 	return false;
3335 }
3336 
3337 static int igc_add_flex_filter(struct igc_adapter *adapter,
3338 			       struct igc_nfc_rule *rule)
3339 {
3340 	struct igc_flex_filter flex = { };
3341 	struct igc_nfc_filter *filter = &rule->filter;
3342 	unsigned int eth_offset, user_offset;
3343 	int ret, index;
3344 	bool vlan;
3345 
3346 	index = igc_find_avail_flex_filter_slot(adapter);
3347 	if (index < 0)
3348 		return -ENOSPC;
3349 
3350 	/* Construct the flex filter:
3351 	 *  -> dest_mac [6]
3352 	 *  -> src_mac [6]
3353 	 *  -> tpid [2]
3354 	 *  -> vlan tci [2]
3355 	 *  -> ether type [2]
3356 	 *  -> user data [8]
3357 	 *  -> = 26 bytes => 32 length
3358 	 */
3359 	flex.index    = index;
3360 	flex.length   = 32;
3361 	flex.rx_queue = rule->action;
3362 
3363 	vlan = rule->filter.vlan_tci || rule->filter.vlan_etype;
3364 	eth_offset = vlan ? 16 : 12;
3365 	user_offset = vlan ? 18 : 14;
3366 
3367 	/* Add destination MAC  */
3368 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3369 		igc_flex_filter_add_field(&flex, &filter->dst_addr, 0,
3370 					  ETH_ALEN, NULL);
3371 
3372 	/* Add source MAC */
3373 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3374 		igc_flex_filter_add_field(&flex, &filter->src_addr, 6,
3375 					  ETH_ALEN, NULL);
3376 
3377 	/* Add VLAN etype */
3378 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE)
3379 		igc_flex_filter_add_field(&flex, &filter->vlan_etype, 12,
3380 					  sizeof(filter->vlan_etype),
3381 					  NULL);
3382 
3383 	/* Add VLAN TCI */
3384 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI)
3385 		igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14,
3386 					  sizeof(filter->vlan_tci), NULL);
3387 
3388 	/* Add Ether type */
3389 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3390 		__be16 etype = cpu_to_be16(filter->etype);
3391 
3392 		igc_flex_filter_add_field(&flex, &etype, eth_offset,
3393 					  sizeof(etype), NULL);
3394 	}
3395 
3396 	/* Add user data */
3397 	if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA)
3398 		igc_flex_filter_add_field(&flex, &filter->user_data,
3399 					  user_offset,
3400 					  sizeof(filter->user_data),
3401 					  filter->user_mask);
3402 
3403 	/* Add it down to the hardware and enable it. */
3404 	ret = igc_write_flex_filter_ll(adapter, &flex);
3405 	if (ret)
3406 		return ret;
3407 
3408 	filter->flex_index = index;
3409 
3410 	return 0;
3411 }
3412 
3413 static void igc_del_flex_filter(struct igc_adapter *adapter,
3414 				u16 reg_index)
3415 {
3416 	struct igc_hw *hw = &adapter->hw;
3417 	u32 wufc;
3418 
3419 	/* Just disable the filter. The filter table itself is kept
3420 	 * intact. Another flex_filter_add() should override the "old" data
3421 	 * then.
3422 	 */
3423 	if (reg_index > 8) {
3424 		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3425 
3426 		wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8));
3427 		wr32(IGC_WUFC_EXT, wufc_ext);
3428 	} else {
3429 		wufc = rd32(IGC_WUFC);
3430 
3431 		wufc &= ~(IGC_WUFC_FLX0 << reg_index);
3432 		wr32(IGC_WUFC, wufc);
3433 	}
3434 
3435 	if (igc_flex_filter_in_use(adapter))
3436 		return;
3437 
3438 	/* No filters are in use, we may disable flex filters */
3439 	wufc = rd32(IGC_WUFC);
3440 	wufc &= ~IGC_WUFC_FLEX_HQ;
3441 	wr32(IGC_WUFC, wufc);
3442 }
3443 
3444 static int igc_enable_nfc_rule(struct igc_adapter *adapter,
3445 			       struct igc_nfc_rule *rule)
3446 {
3447 	int err;
3448 
3449 	if (rule->flex) {
3450 		return igc_add_flex_filter(adapter, rule);
3451 	}
3452 
3453 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3454 		err = igc_add_etype_filter(adapter, rule->filter.etype,
3455 					   rule->action);
3456 		if (err)
3457 			return err;
3458 	}
3459 
3460 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
3461 		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3462 					 rule->filter.src_addr, rule->action);
3463 		if (err)
3464 			return err;
3465 	}
3466 
3467 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
3468 		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3469 					 rule->filter.dst_addr, rule->action);
3470 		if (err)
3471 			return err;
3472 	}
3473 
3474 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3475 		int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3476 			   VLAN_PRIO_SHIFT;
3477 
3478 		err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
3479 		if (err)
3480 			return err;
3481 	}
3482 
3483 	return 0;
3484 }
3485 
3486 static void igc_disable_nfc_rule(struct igc_adapter *adapter,
3487 				 const struct igc_nfc_rule *rule)
3488 {
3489 	if (rule->flex) {
3490 		igc_del_flex_filter(adapter, rule->filter.flex_index);
3491 		return;
3492 	}
3493 
3494 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
3495 		igc_del_etype_filter(adapter, rule->filter.etype);
3496 
3497 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3498 		int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3499 			   VLAN_PRIO_SHIFT;
3500 
3501 		igc_del_vlan_prio_filter(adapter, prio);
3502 	}
3503 
3504 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3505 		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3506 				   rule->filter.src_addr);
3507 
3508 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3509 		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3510 				   rule->filter.dst_addr);
3511 }
3512 
3513 /**
3514  * igc_get_nfc_rule() - Get NFC rule
3515  * @adapter: Pointer to adapter
3516  * @location: Rule location
3517  *
3518  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3519  *
3520  * Return: Pointer to NFC rule at @location. If not found, NULL.
3521  */
3522 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
3523 				      u32 location)
3524 {
3525 	struct igc_nfc_rule *rule;
3526 
3527 	list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
3528 		if (rule->location == location)
3529 			return rule;
3530 		if (rule->location > location)
3531 			break;
3532 	}
3533 
3534 	return NULL;
3535 }
3536 
3537 /**
3538  * igc_del_nfc_rule() - Delete NFC rule
3539  * @adapter: Pointer to adapter
3540  * @rule: Pointer to rule to be deleted
3541  *
3542  * Disable NFC rule in hardware and delete it from adapter.
3543  *
3544  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3545  */
3546 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3547 {
3548 	igc_disable_nfc_rule(adapter, rule);
3549 
3550 	list_del(&rule->list);
3551 	adapter->nfc_rule_count--;
3552 
3553 	kfree(rule);
3554 }
3555 
3556 static void igc_flush_nfc_rules(struct igc_adapter *adapter)
3557 {
3558 	struct igc_nfc_rule *rule, *tmp;
3559 
3560 	mutex_lock(&adapter->nfc_rule_lock);
3561 
3562 	list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
3563 		igc_del_nfc_rule(adapter, rule);
3564 
3565 	mutex_unlock(&adapter->nfc_rule_lock);
3566 }
3567 
3568 /**
3569  * igc_add_nfc_rule() - Add NFC rule
3570  * @adapter: Pointer to adapter
3571  * @rule: Pointer to rule to be added
3572  *
3573  * Enable NFC rule in hardware and add it to adapter.
3574  *
3575  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3576  *
3577  * Return: 0 on success, negative errno on failure.
3578  */
3579 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3580 {
3581 	struct igc_nfc_rule *pred, *cur;
3582 	int err;
3583 
3584 	err = igc_enable_nfc_rule(adapter, rule);
3585 	if (err)
3586 		return err;
3587 
3588 	pred = NULL;
3589 	list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
3590 		if (cur->location >= rule->location)
3591 			break;
3592 		pred = cur;
3593 	}
3594 
3595 	list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
3596 	adapter->nfc_rule_count++;
3597 	return 0;
3598 }
3599 
3600 static void igc_restore_nfc_rules(struct igc_adapter *adapter)
3601 {
3602 	struct igc_nfc_rule *rule;
3603 
3604 	mutex_lock(&adapter->nfc_rule_lock);
3605 
3606 	list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
3607 		igc_enable_nfc_rule(adapter, rule);
3608 
3609 	mutex_unlock(&adapter->nfc_rule_lock);
3610 }
3611 
3612 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
3613 {
3614 	struct igc_adapter *adapter = netdev_priv(netdev);
3615 
3616 	return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
3617 }
3618 
3619 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
3620 {
3621 	struct igc_adapter *adapter = netdev_priv(netdev);
3622 
3623 	igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
3624 	return 0;
3625 }
3626 
3627 /**
3628  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3629  * @netdev: network interface device structure
3630  *
3631  * The set_rx_mode entry point is called whenever the unicast or multicast
3632  * address lists or the network interface flags are updated.  This routine is
3633  * responsible for configuring the hardware for proper unicast, multicast,
3634  * promiscuous mode, and all-multi behavior.
3635  */
3636 static void igc_set_rx_mode(struct net_device *netdev)
3637 {
3638 	struct igc_adapter *adapter = netdev_priv(netdev);
3639 	struct igc_hw *hw = &adapter->hw;
3640 	u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
3641 	int count;
3642 
3643 	/* Check for Promiscuous and All Multicast modes */
3644 	if (netdev->flags & IFF_PROMISC) {
3645 		rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
3646 	} else {
3647 		if (netdev->flags & IFF_ALLMULTI) {
3648 			rctl |= IGC_RCTL_MPE;
3649 		} else {
3650 			/* Write addresses to the MTA, if the attempt fails
3651 			 * then we should just turn on promiscuous mode so
3652 			 * that we can at least receive multicast traffic
3653 			 */
3654 			count = igc_write_mc_addr_list(netdev);
3655 			if (count < 0)
3656 				rctl |= IGC_RCTL_MPE;
3657 		}
3658 	}
3659 
3660 	/* Write addresses to available RAR registers, if there is not
3661 	 * sufficient space to store all the addresses then enable
3662 	 * unicast promiscuous mode
3663 	 */
3664 	if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
3665 		rctl |= IGC_RCTL_UPE;
3666 
3667 	/* update state of unicast and multicast */
3668 	rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
3669 	wr32(IGC_RCTL, rctl);
3670 
3671 #if (PAGE_SIZE < 8192)
3672 	if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
3673 		rlpml = IGC_MAX_FRAME_BUILD_SKB;
3674 #endif
3675 	wr32(IGC_RLPML, rlpml);
3676 }
3677 
3678 /**
3679  * igc_configure - configure the hardware for RX and TX
3680  * @adapter: private board structure
3681  */
3682 static void igc_configure(struct igc_adapter *adapter)
3683 {
3684 	struct net_device *netdev = adapter->netdev;
3685 	int i = 0;
3686 
3687 	igc_get_hw_control(adapter);
3688 	igc_set_rx_mode(netdev);
3689 
3690 	igc_restore_vlan(adapter);
3691 
3692 	igc_setup_tctl(adapter);
3693 	igc_setup_mrqc(adapter);
3694 	igc_setup_rctl(adapter);
3695 
3696 	igc_set_default_mac_filter(adapter);
3697 	igc_restore_nfc_rules(adapter);
3698 
3699 	igc_configure_tx(adapter);
3700 	igc_configure_rx(adapter);
3701 
3702 	igc_rx_fifo_flush_base(&adapter->hw);
3703 
3704 	/* call igc_desc_unused which always leaves
3705 	 * at least 1 descriptor unused to make sure
3706 	 * next_to_use != next_to_clean
3707 	 */
3708 	for (i = 0; i < adapter->num_rx_queues; i++) {
3709 		struct igc_ring *ring = adapter->rx_ring[i];
3710 
3711 		if (ring->xsk_pool)
3712 			igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
3713 		else
3714 			igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
3715 	}
3716 }
3717 
3718 /**
3719  * igc_write_ivar - configure ivar for given MSI-X vector
3720  * @hw: pointer to the HW structure
3721  * @msix_vector: vector number we are allocating to a given ring
3722  * @index: row index of IVAR register to write within IVAR table
3723  * @offset: column offset of in IVAR, should be multiple of 8
3724  *
3725  * The IVAR table consists of 2 columns,
3726  * each containing an cause allocation for an Rx and Tx ring, and a
3727  * variable number of rows depending on the number of queues supported.
3728  */
3729 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
3730 			   int index, int offset)
3731 {
3732 	u32 ivar = array_rd32(IGC_IVAR0, index);
3733 
3734 	/* clear any bits that are currently set */
3735 	ivar &= ~((u32)0xFF << offset);
3736 
3737 	/* write vector and valid bit */
3738 	ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
3739 
3740 	array_wr32(IGC_IVAR0, index, ivar);
3741 }
3742 
3743 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
3744 {
3745 	struct igc_adapter *adapter = q_vector->adapter;
3746 	struct igc_hw *hw = &adapter->hw;
3747 	int rx_queue = IGC_N0_QUEUE;
3748 	int tx_queue = IGC_N0_QUEUE;
3749 
3750 	if (q_vector->rx.ring)
3751 		rx_queue = q_vector->rx.ring->reg_idx;
3752 	if (q_vector->tx.ring)
3753 		tx_queue = q_vector->tx.ring->reg_idx;
3754 
3755 	switch (hw->mac.type) {
3756 	case igc_i225:
3757 		if (rx_queue > IGC_N0_QUEUE)
3758 			igc_write_ivar(hw, msix_vector,
3759 				       rx_queue >> 1,
3760 				       (rx_queue & 0x1) << 4);
3761 		if (tx_queue > IGC_N0_QUEUE)
3762 			igc_write_ivar(hw, msix_vector,
3763 				       tx_queue >> 1,
3764 				       ((tx_queue & 0x1) << 4) + 8);
3765 		q_vector->eims_value = BIT(msix_vector);
3766 		break;
3767 	default:
3768 		WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
3769 		break;
3770 	}
3771 
3772 	/* add q_vector eims value to global eims_enable_mask */
3773 	adapter->eims_enable_mask |= q_vector->eims_value;
3774 
3775 	/* configure q_vector to set itr on first interrupt */
3776 	q_vector->set_itr = 1;
3777 }
3778 
3779 /**
3780  * igc_configure_msix - Configure MSI-X hardware
3781  * @adapter: Pointer to adapter structure
3782  *
3783  * igc_configure_msix sets up the hardware to properly
3784  * generate MSI-X interrupts.
3785  */
3786 static void igc_configure_msix(struct igc_adapter *adapter)
3787 {
3788 	struct igc_hw *hw = &adapter->hw;
3789 	int i, vector = 0;
3790 	u32 tmp;
3791 
3792 	adapter->eims_enable_mask = 0;
3793 
3794 	/* set vector for other causes, i.e. link changes */
3795 	switch (hw->mac.type) {
3796 	case igc_i225:
3797 		/* Turn on MSI-X capability first, or our settings
3798 		 * won't stick.  And it will take days to debug.
3799 		 */
3800 		wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
3801 		     IGC_GPIE_PBA | IGC_GPIE_EIAME |
3802 		     IGC_GPIE_NSICR);
3803 
3804 		/* enable msix_other interrupt */
3805 		adapter->eims_other = BIT(vector);
3806 		tmp = (vector++ | IGC_IVAR_VALID) << 8;
3807 
3808 		wr32(IGC_IVAR_MISC, tmp);
3809 		break;
3810 	default:
3811 		/* do nothing, since nothing else supports MSI-X */
3812 		break;
3813 	} /* switch (hw->mac.type) */
3814 
3815 	adapter->eims_enable_mask |= adapter->eims_other;
3816 
3817 	for (i = 0; i < adapter->num_q_vectors; i++)
3818 		igc_assign_vector(adapter->q_vector[i], vector++);
3819 
3820 	wrfl();
3821 }
3822 
3823 /**
3824  * igc_irq_enable - Enable default interrupt generation settings
3825  * @adapter: board private structure
3826  */
3827 static void igc_irq_enable(struct igc_adapter *adapter)
3828 {
3829 	struct igc_hw *hw = &adapter->hw;
3830 
3831 	if (adapter->msix_entries) {
3832 		u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
3833 		u32 regval = rd32(IGC_EIAC);
3834 
3835 		wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
3836 		regval = rd32(IGC_EIAM);
3837 		wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
3838 		wr32(IGC_EIMS, adapter->eims_enable_mask);
3839 		wr32(IGC_IMS, ims);
3840 	} else {
3841 		wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3842 		wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3843 	}
3844 }
3845 
3846 /**
3847  * igc_irq_disable - Mask off interrupt generation on the NIC
3848  * @adapter: board private structure
3849  */
3850 static void igc_irq_disable(struct igc_adapter *adapter)
3851 {
3852 	struct igc_hw *hw = &adapter->hw;
3853 
3854 	if (adapter->msix_entries) {
3855 		u32 regval = rd32(IGC_EIAM);
3856 
3857 		wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
3858 		wr32(IGC_EIMC, adapter->eims_enable_mask);
3859 		regval = rd32(IGC_EIAC);
3860 		wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
3861 	}
3862 
3863 	wr32(IGC_IAM, 0);
3864 	wr32(IGC_IMC, ~0);
3865 	wrfl();
3866 
3867 	if (adapter->msix_entries) {
3868 		int vector = 0, i;
3869 
3870 		synchronize_irq(adapter->msix_entries[vector++].vector);
3871 
3872 		for (i = 0; i < adapter->num_q_vectors; i++)
3873 			synchronize_irq(adapter->msix_entries[vector++].vector);
3874 	} else {
3875 		synchronize_irq(adapter->pdev->irq);
3876 	}
3877 }
3878 
3879 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
3880 			      const u32 max_rss_queues)
3881 {
3882 	/* Determine if we need to pair queues. */
3883 	/* If rss_queues > half of max_rss_queues, pair the queues in
3884 	 * order to conserve interrupts due to limited supply.
3885 	 */
3886 	if (adapter->rss_queues > (max_rss_queues / 2))
3887 		adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
3888 	else
3889 		adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
3890 }
3891 
3892 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
3893 {
3894 	return IGC_MAX_RX_QUEUES;
3895 }
3896 
3897 static void igc_init_queue_configuration(struct igc_adapter *adapter)
3898 {
3899 	u32 max_rss_queues;
3900 
3901 	max_rss_queues = igc_get_max_rss_queues(adapter);
3902 	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
3903 
3904 	igc_set_flag_queue_pairs(adapter, max_rss_queues);
3905 }
3906 
3907 /**
3908  * igc_reset_q_vector - Reset config for interrupt vector
3909  * @adapter: board private structure to initialize
3910  * @v_idx: Index of vector to be reset
3911  *
3912  * If NAPI is enabled it will delete any references to the
3913  * NAPI struct. This is preparation for igc_free_q_vector.
3914  */
3915 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
3916 {
3917 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
3918 
3919 	/* if we're coming from igc_set_interrupt_capability, the vectors are
3920 	 * not yet allocated
3921 	 */
3922 	if (!q_vector)
3923 		return;
3924 
3925 	if (q_vector->tx.ring)
3926 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
3927 
3928 	if (q_vector->rx.ring)
3929 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
3930 
3931 	netif_napi_del(&q_vector->napi);
3932 }
3933 
3934 /**
3935  * igc_free_q_vector - Free memory allocated for specific interrupt vector
3936  * @adapter: board private structure to initialize
3937  * @v_idx: Index of vector to be freed
3938  *
3939  * This function frees the memory allocated to the q_vector.
3940  */
3941 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
3942 {
3943 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
3944 
3945 	adapter->q_vector[v_idx] = NULL;
3946 
3947 	/* igc_get_stats64() might access the rings on this vector,
3948 	 * we must wait a grace period before freeing it.
3949 	 */
3950 	if (q_vector)
3951 		kfree_rcu(q_vector, rcu);
3952 }
3953 
3954 /**
3955  * igc_free_q_vectors - Free memory allocated for interrupt vectors
3956  * @adapter: board private structure to initialize
3957  *
3958  * This function frees the memory allocated to the q_vectors.  In addition if
3959  * NAPI is enabled it will delete any references to the NAPI struct prior
3960  * to freeing the q_vector.
3961  */
3962 static void igc_free_q_vectors(struct igc_adapter *adapter)
3963 {
3964 	int v_idx = adapter->num_q_vectors;
3965 
3966 	adapter->num_tx_queues = 0;
3967 	adapter->num_rx_queues = 0;
3968 	adapter->num_q_vectors = 0;
3969 
3970 	while (v_idx--) {
3971 		igc_reset_q_vector(adapter, v_idx);
3972 		igc_free_q_vector(adapter, v_idx);
3973 	}
3974 }
3975 
3976 /**
3977  * igc_update_itr - update the dynamic ITR value based on statistics
3978  * @q_vector: pointer to q_vector
3979  * @ring_container: ring info to update the itr for
3980  *
3981  * Stores a new ITR value based on packets and byte
3982  * counts during the last interrupt.  The advantage of per interrupt
3983  * computation is faster updates and more accurate ITR for the current
3984  * traffic pattern.  Constants in this function were computed
3985  * based on theoretical maximum wire speed and thresholds were set based
3986  * on testing data as well as attempting to minimize response time
3987  * while increasing bulk throughput.
3988  * NOTE: These calculations are only valid when operating in a single-
3989  * queue environment.
3990  */
3991 static void igc_update_itr(struct igc_q_vector *q_vector,
3992 			   struct igc_ring_container *ring_container)
3993 {
3994 	unsigned int packets = ring_container->total_packets;
3995 	unsigned int bytes = ring_container->total_bytes;
3996 	u8 itrval = ring_container->itr;
3997 
3998 	/* no packets, exit with status unchanged */
3999 	if (packets == 0)
4000 		return;
4001 
4002 	switch (itrval) {
4003 	case lowest_latency:
4004 		/* handle TSO and jumbo frames */
4005 		if (bytes / packets > 8000)
4006 			itrval = bulk_latency;
4007 		else if ((packets < 5) && (bytes > 512))
4008 			itrval = low_latency;
4009 		break;
4010 	case low_latency:  /* 50 usec aka 20000 ints/s */
4011 		if (bytes > 10000) {
4012 			/* this if handles the TSO accounting */
4013 			if (bytes / packets > 8000)
4014 				itrval = bulk_latency;
4015 			else if ((packets < 10) || ((bytes / packets) > 1200))
4016 				itrval = bulk_latency;
4017 			else if ((packets > 35))
4018 				itrval = lowest_latency;
4019 		} else if (bytes / packets > 2000) {
4020 			itrval = bulk_latency;
4021 		} else if (packets <= 2 && bytes < 512) {
4022 			itrval = lowest_latency;
4023 		}
4024 		break;
4025 	case bulk_latency: /* 250 usec aka 4000 ints/s */
4026 		if (bytes > 25000) {
4027 			if (packets > 35)
4028 				itrval = low_latency;
4029 		} else if (bytes < 1500) {
4030 			itrval = low_latency;
4031 		}
4032 		break;
4033 	}
4034 
4035 	/* clear work counters since we have the values we need */
4036 	ring_container->total_bytes = 0;
4037 	ring_container->total_packets = 0;
4038 
4039 	/* write updated itr to ring container */
4040 	ring_container->itr = itrval;
4041 }
4042 
4043 static void igc_set_itr(struct igc_q_vector *q_vector)
4044 {
4045 	struct igc_adapter *adapter = q_vector->adapter;
4046 	u32 new_itr = q_vector->itr_val;
4047 	u8 current_itr = 0;
4048 
4049 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4050 	switch (adapter->link_speed) {
4051 	case SPEED_10:
4052 	case SPEED_100:
4053 		current_itr = 0;
4054 		new_itr = IGC_4K_ITR;
4055 		goto set_itr_now;
4056 	default:
4057 		break;
4058 	}
4059 
4060 	igc_update_itr(q_vector, &q_vector->tx);
4061 	igc_update_itr(q_vector, &q_vector->rx);
4062 
4063 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4064 
4065 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4066 	if (current_itr == lowest_latency &&
4067 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4068 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4069 		current_itr = low_latency;
4070 
4071 	switch (current_itr) {
4072 	/* counts and packets in update_itr are dependent on these numbers */
4073 	case lowest_latency:
4074 		new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
4075 		break;
4076 	case low_latency:
4077 		new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
4078 		break;
4079 	case bulk_latency:
4080 		new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
4081 		break;
4082 	default:
4083 		break;
4084 	}
4085 
4086 set_itr_now:
4087 	if (new_itr != q_vector->itr_val) {
4088 		/* this attempts to bias the interrupt rate towards Bulk
4089 		 * by adding intermediate steps when interrupt rate is
4090 		 * increasing
4091 		 */
4092 		new_itr = new_itr > q_vector->itr_val ?
4093 			  max((new_itr * q_vector->itr_val) /
4094 			  (new_itr + (q_vector->itr_val >> 2)),
4095 			  new_itr) : new_itr;
4096 		/* Don't write the value here; it resets the adapter's
4097 		 * internal timer, and causes us to delay far longer than
4098 		 * we should between interrupts.  Instead, we write the ITR
4099 		 * value at the beginning of the next interrupt so the timing
4100 		 * ends up being correct.
4101 		 */
4102 		q_vector->itr_val = new_itr;
4103 		q_vector->set_itr = 1;
4104 	}
4105 }
4106 
4107 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
4108 {
4109 	int v_idx = adapter->num_q_vectors;
4110 
4111 	if (adapter->msix_entries) {
4112 		pci_disable_msix(adapter->pdev);
4113 		kfree(adapter->msix_entries);
4114 		adapter->msix_entries = NULL;
4115 	} else if (adapter->flags & IGC_FLAG_HAS_MSI) {
4116 		pci_disable_msi(adapter->pdev);
4117 	}
4118 
4119 	while (v_idx--)
4120 		igc_reset_q_vector(adapter, v_idx);
4121 }
4122 
4123 /**
4124  * igc_set_interrupt_capability - set MSI or MSI-X if supported
4125  * @adapter: Pointer to adapter structure
4126  * @msix: boolean value for MSI-X capability
4127  *
4128  * Attempt to configure interrupts using the best available
4129  * capabilities of the hardware and kernel.
4130  */
4131 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
4132 					 bool msix)
4133 {
4134 	int numvecs, i;
4135 	int err;
4136 
4137 	if (!msix)
4138 		goto msi_only;
4139 	adapter->flags |= IGC_FLAG_HAS_MSIX;
4140 
4141 	/* Number of supported queues. */
4142 	adapter->num_rx_queues = adapter->rss_queues;
4143 
4144 	adapter->num_tx_queues = adapter->rss_queues;
4145 
4146 	/* start with one vector for every Rx queue */
4147 	numvecs = adapter->num_rx_queues;
4148 
4149 	/* if Tx handler is separate add 1 for every Tx queue */
4150 	if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
4151 		numvecs += adapter->num_tx_queues;
4152 
4153 	/* store the number of vectors reserved for queues */
4154 	adapter->num_q_vectors = numvecs;
4155 
4156 	/* add 1 vector for link status interrupts */
4157 	numvecs++;
4158 
4159 	adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
4160 					GFP_KERNEL);
4161 
4162 	if (!adapter->msix_entries)
4163 		return;
4164 
4165 	/* populate entry values */
4166 	for (i = 0; i < numvecs; i++)
4167 		adapter->msix_entries[i].entry = i;
4168 
4169 	err = pci_enable_msix_range(adapter->pdev,
4170 				    adapter->msix_entries,
4171 				    numvecs,
4172 				    numvecs);
4173 	if (err > 0)
4174 		return;
4175 
4176 	kfree(adapter->msix_entries);
4177 	adapter->msix_entries = NULL;
4178 
4179 	igc_reset_interrupt_capability(adapter);
4180 
4181 msi_only:
4182 	adapter->flags &= ~IGC_FLAG_HAS_MSIX;
4183 
4184 	adapter->rss_queues = 1;
4185 	adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4186 	adapter->num_rx_queues = 1;
4187 	adapter->num_tx_queues = 1;
4188 	adapter->num_q_vectors = 1;
4189 	if (!pci_enable_msi(adapter->pdev))
4190 		adapter->flags |= IGC_FLAG_HAS_MSI;
4191 }
4192 
4193 /**
4194  * igc_update_ring_itr - update the dynamic ITR value based on packet size
4195  * @q_vector: pointer to q_vector
4196  *
4197  * Stores a new ITR value based on strictly on packet size.  This
4198  * algorithm is less sophisticated than that used in igc_update_itr,
4199  * due to the difficulty of synchronizing statistics across multiple
4200  * receive rings.  The divisors and thresholds used by this function
4201  * were determined based on theoretical maximum wire speed and testing
4202  * data, in order to minimize response time while increasing bulk
4203  * throughput.
4204  * NOTE: This function is called only when operating in a multiqueue
4205  * receive environment.
4206  */
4207 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
4208 {
4209 	struct igc_adapter *adapter = q_vector->adapter;
4210 	int new_val = q_vector->itr_val;
4211 	int avg_wire_size = 0;
4212 	unsigned int packets;
4213 
4214 	/* For non-gigabit speeds, just fix the interrupt rate at 4000
4215 	 * ints/sec - ITR timer value of 120 ticks.
4216 	 */
4217 	switch (adapter->link_speed) {
4218 	case SPEED_10:
4219 	case SPEED_100:
4220 		new_val = IGC_4K_ITR;
4221 		goto set_itr_val;
4222 	default:
4223 		break;
4224 	}
4225 
4226 	packets = q_vector->rx.total_packets;
4227 	if (packets)
4228 		avg_wire_size = q_vector->rx.total_bytes / packets;
4229 
4230 	packets = q_vector->tx.total_packets;
4231 	if (packets)
4232 		avg_wire_size = max_t(u32, avg_wire_size,
4233 				      q_vector->tx.total_bytes / packets);
4234 
4235 	/* if avg_wire_size isn't set no work was done */
4236 	if (!avg_wire_size)
4237 		goto clear_counts;
4238 
4239 	/* Add 24 bytes to size to account for CRC, preamble, and gap */
4240 	avg_wire_size += 24;
4241 
4242 	/* Don't starve jumbo frames */
4243 	avg_wire_size = min(avg_wire_size, 3000);
4244 
4245 	/* Give a little boost to mid-size frames */
4246 	if (avg_wire_size > 300 && avg_wire_size < 1200)
4247 		new_val = avg_wire_size / 3;
4248 	else
4249 		new_val = avg_wire_size / 2;
4250 
4251 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4252 	if (new_val < IGC_20K_ITR &&
4253 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4254 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4255 		new_val = IGC_20K_ITR;
4256 
4257 set_itr_val:
4258 	if (new_val != q_vector->itr_val) {
4259 		q_vector->itr_val = new_val;
4260 		q_vector->set_itr = 1;
4261 	}
4262 clear_counts:
4263 	q_vector->rx.total_bytes = 0;
4264 	q_vector->rx.total_packets = 0;
4265 	q_vector->tx.total_bytes = 0;
4266 	q_vector->tx.total_packets = 0;
4267 }
4268 
4269 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
4270 {
4271 	struct igc_adapter *adapter = q_vector->adapter;
4272 	struct igc_hw *hw = &adapter->hw;
4273 
4274 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
4275 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
4276 		if (adapter->num_q_vectors == 1)
4277 			igc_set_itr(q_vector);
4278 		else
4279 			igc_update_ring_itr(q_vector);
4280 	}
4281 
4282 	if (!test_bit(__IGC_DOWN, &adapter->state)) {
4283 		if (adapter->msix_entries)
4284 			wr32(IGC_EIMS, q_vector->eims_value);
4285 		else
4286 			igc_irq_enable(adapter);
4287 	}
4288 }
4289 
4290 static void igc_add_ring(struct igc_ring *ring,
4291 			 struct igc_ring_container *head)
4292 {
4293 	head->ring = ring;
4294 	head->count++;
4295 }
4296 
4297 /**
4298  * igc_cache_ring_register - Descriptor ring to register mapping
4299  * @adapter: board private structure to initialize
4300  *
4301  * Once we know the feature-set enabled for the device, we'll cache
4302  * the register offset the descriptor ring is assigned to.
4303  */
4304 static void igc_cache_ring_register(struct igc_adapter *adapter)
4305 {
4306 	int i = 0, j = 0;
4307 
4308 	switch (adapter->hw.mac.type) {
4309 	case igc_i225:
4310 	default:
4311 		for (; i < adapter->num_rx_queues; i++)
4312 			adapter->rx_ring[i]->reg_idx = i;
4313 		for (; j < adapter->num_tx_queues; j++)
4314 			adapter->tx_ring[j]->reg_idx = j;
4315 		break;
4316 	}
4317 }
4318 
4319 /**
4320  * igc_poll - NAPI Rx polling callback
4321  * @napi: napi polling structure
4322  * @budget: count of how many packets we should handle
4323  */
4324 static int igc_poll(struct napi_struct *napi, int budget)
4325 {
4326 	struct igc_q_vector *q_vector = container_of(napi,
4327 						     struct igc_q_vector,
4328 						     napi);
4329 	struct igc_ring *rx_ring = q_vector->rx.ring;
4330 	bool clean_complete = true;
4331 	int work_done = 0;
4332 
4333 	if (q_vector->tx.ring)
4334 		clean_complete = igc_clean_tx_irq(q_vector, budget);
4335 
4336 	if (rx_ring) {
4337 		int cleaned = rx_ring->xsk_pool ?
4338 			      igc_clean_rx_irq_zc(q_vector, budget) :
4339 			      igc_clean_rx_irq(q_vector, budget);
4340 
4341 		work_done += cleaned;
4342 		if (cleaned >= budget)
4343 			clean_complete = false;
4344 	}
4345 
4346 	/* If all work not completed, return budget and keep polling */
4347 	if (!clean_complete)
4348 		return budget;
4349 
4350 	/* Exit the polling mode, but don't re-enable interrupts if stack might
4351 	 * poll us due to busy-polling
4352 	 */
4353 	if (likely(napi_complete_done(napi, work_done)))
4354 		igc_ring_irq_enable(q_vector);
4355 
4356 	return min(work_done, budget - 1);
4357 }
4358 
4359 /**
4360  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
4361  * @adapter: board private structure to initialize
4362  * @v_count: q_vectors allocated on adapter, used for ring interleaving
4363  * @v_idx: index of vector in adapter struct
4364  * @txr_count: total number of Tx rings to allocate
4365  * @txr_idx: index of first Tx ring to allocate
4366  * @rxr_count: total number of Rx rings to allocate
4367  * @rxr_idx: index of first Rx ring to allocate
4368  *
4369  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
4370  */
4371 static int igc_alloc_q_vector(struct igc_adapter *adapter,
4372 			      unsigned int v_count, unsigned int v_idx,
4373 			      unsigned int txr_count, unsigned int txr_idx,
4374 			      unsigned int rxr_count, unsigned int rxr_idx)
4375 {
4376 	struct igc_q_vector *q_vector;
4377 	struct igc_ring *ring;
4378 	int ring_count;
4379 
4380 	/* igc only supports 1 Tx and/or 1 Rx queue per vector */
4381 	if (txr_count > 1 || rxr_count > 1)
4382 		return -ENOMEM;
4383 
4384 	ring_count = txr_count + rxr_count;
4385 
4386 	/* allocate q_vector and rings */
4387 	q_vector = adapter->q_vector[v_idx];
4388 	if (!q_vector)
4389 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
4390 				   GFP_KERNEL);
4391 	else
4392 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
4393 	if (!q_vector)
4394 		return -ENOMEM;
4395 
4396 	/* initialize NAPI */
4397 	netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
4398 
4399 	/* tie q_vector and adapter together */
4400 	adapter->q_vector[v_idx] = q_vector;
4401 	q_vector->adapter = adapter;
4402 
4403 	/* initialize work limits */
4404 	q_vector->tx.work_limit = adapter->tx_work_limit;
4405 
4406 	/* initialize ITR configuration */
4407 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
4408 	q_vector->itr_val = IGC_START_ITR;
4409 
4410 	/* initialize pointer to rings */
4411 	ring = q_vector->ring;
4412 
4413 	/* initialize ITR */
4414 	if (rxr_count) {
4415 		/* rx or rx/tx vector */
4416 		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
4417 			q_vector->itr_val = adapter->rx_itr_setting;
4418 	} else {
4419 		/* tx only vector */
4420 		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
4421 			q_vector->itr_val = adapter->tx_itr_setting;
4422 	}
4423 
4424 	if (txr_count) {
4425 		/* assign generic ring traits */
4426 		ring->dev = &adapter->pdev->dev;
4427 		ring->netdev = adapter->netdev;
4428 
4429 		/* configure backlink on ring */
4430 		ring->q_vector = q_vector;
4431 
4432 		/* update q_vector Tx values */
4433 		igc_add_ring(ring, &q_vector->tx);
4434 
4435 		/* apply Tx specific ring traits */
4436 		ring->count = adapter->tx_ring_count;
4437 		ring->queue_index = txr_idx;
4438 
4439 		/* assign ring to adapter */
4440 		adapter->tx_ring[txr_idx] = ring;
4441 
4442 		/* push pointer to next ring */
4443 		ring++;
4444 	}
4445 
4446 	if (rxr_count) {
4447 		/* assign generic ring traits */
4448 		ring->dev = &adapter->pdev->dev;
4449 		ring->netdev = adapter->netdev;
4450 
4451 		/* configure backlink on ring */
4452 		ring->q_vector = q_vector;
4453 
4454 		/* update q_vector Rx values */
4455 		igc_add_ring(ring, &q_vector->rx);
4456 
4457 		/* apply Rx specific ring traits */
4458 		ring->count = adapter->rx_ring_count;
4459 		ring->queue_index = rxr_idx;
4460 
4461 		/* assign ring to adapter */
4462 		adapter->rx_ring[rxr_idx] = ring;
4463 	}
4464 
4465 	return 0;
4466 }
4467 
4468 /**
4469  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
4470  * @adapter: board private structure to initialize
4471  *
4472  * We allocate one q_vector per queue interrupt.  If allocation fails we
4473  * return -ENOMEM.
4474  */
4475 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
4476 {
4477 	int rxr_remaining = adapter->num_rx_queues;
4478 	int txr_remaining = adapter->num_tx_queues;
4479 	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
4480 	int q_vectors = adapter->num_q_vectors;
4481 	int err;
4482 
4483 	if (q_vectors >= (rxr_remaining + txr_remaining)) {
4484 		for (; rxr_remaining; v_idx++) {
4485 			err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4486 						 0, 0, 1, rxr_idx);
4487 
4488 			if (err)
4489 				goto err_out;
4490 
4491 			/* update counts and index */
4492 			rxr_remaining--;
4493 			rxr_idx++;
4494 		}
4495 	}
4496 
4497 	for (; v_idx < q_vectors; v_idx++) {
4498 		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
4499 		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
4500 
4501 		err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4502 					 tqpv, txr_idx, rqpv, rxr_idx);
4503 
4504 		if (err)
4505 			goto err_out;
4506 
4507 		/* update counts and index */
4508 		rxr_remaining -= rqpv;
4509 		txr_remaining -= tqpv;
4510 		rxr_idx++;
4511 		txr_idx++;
4512 	}
4513 
4514 	return 0;
4515 
4516 err_out:
4517 	adapter->num_tx_queues = 0;
4518 	adapter->num_rx_queues = 0;
4519 	adapter->num_q_vectors = 0;
4520 
4521 	while (v_idx--)
4522 		igc_free_q_vector(adapter, v_idx);
4523 
4524 	return -ENOMEM;
4525 }
4526 
4527 /**
4528  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
4529  * @adapter: Pointer to adapter structure
4530  * @msix: boolean for MSI-X capability
4531  *
4532  * This function initializes the interrupts and allocates all of the queues.
4533  */
4534 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
4535 {
4536 	struct net_device *dev = adapter->netdev;
4537 	int err = 0;
4538 
4539 	igc_set_interrupt_capability(adapter, msix);
4540 
4541 	err = igc_alloc_q_vectors(adapter);
4542 	if (err) {
4543 		netdev_err(dev, "Unable to allocate memory for vectors\n");
4544 		goto err_alloc_q_vectors;
4545 	}
4546 
4547 	igc_cache_ring_register(adapter);
4548 
4549 	return 0;
4550 
4551 err_alloc_q_vectors:
4552 	igc_reset_interrupt_capability(adapter);
4553 	return err;
4554 }
4555 
4556 /**
4557  * igc_sw_init - Initialize general software structures (struct igc_adapter)
4558  * @adapter: board private structure to initialize
4559  *
4560  * igc_sw_init initializes the Adapter private data structure.
4561  * Fields are initialized based on PCI device information and
4562  * OS network device settings (MTU size).
4563  */
4564 static int igc_sw_init(struct igc_adapter *adapter)
4565 {
4566 	struct net_device *netdev = adapter->netdev;
4567 	struct pci_dev *pdev = adapter->pdev;
4568 	struct igc_hw *hw = &adapter->hw;
4569 
4570 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4571 
4572 	/* set default ring sizes */
4573 	adapter->tx_ring_count = IGC_DEFAULT_TXD;
4574 	adapter->rx_ring_count = IGC_DEFAULT_RXD;
4575 
4576 	/* set default ITR values */
4577 	adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4578 	adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4579 
4580 	/* set default work limits */
4581 	adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4582 
4583 	/* adjust max frame to be at least the size of a standard frame */
4584 	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4585 				VLAN_HLEN;
4586 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4587 
4588 	mutex_init(&adapter->nfc_rule_lock);
4589 	INIT_LIST_HEAD(&adapter->nfc_rule_list);
4590 	adapter->nfc_rule_count = 0;
4591 
4592 	spin_lock_init(&adapter->stats64_lock);
4593 	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
4594 	adapter->flags |= IGC_FLAG_HAS_MSIX;
4595 
4596 	igc_init_queue_configuration(adapter);
4597 
4598 	/* This call may decrease the number of queues */
4599 	if (igc_init_interrupt_scheme(adapter, true)) {
4600 		netdev_err(netdev, "Unable to allocate memory for queues\n");
4601 		return -ENOMEM;
4602 	}
4603 
4604 	/* Explicitly disable IRQ since the NIC can be in any state. */
4605 	igc_irq_disable(adapter);
4606 
4607 	set_bit(__IGC_DOWN, &adapter->state);
4608 
4609 	return 0;
4610 }
4611 
4612 /**
4613  * igc_up - Open the interface and prepare it to handle traffic
4614  * @adapter: board private structure
4615  */
4616 void igc_up(struct igc_adapter *adapter)
4617 {
4618 	struct igc_hw *hw = &adapter->hw;
4619 	int i = 0;
4620 
4621 	/* hardware has been reset, we need to reload some things */
4622 	igc_configure(adapter);
4623 
4624 	clear_bit(__IGC_DOWN, &adapter->state);
4625 
4626 	for (i = 0; i < adapter->num_q_vectors; i++)
4627 		napi_enable(&adapter->q_vector[i]->napi);
4628 
4629 	if (adapter->msix_entries)
4630 		igc_configure_msix(adapter);
4631 	else
4632 		igc_assign_vector(adapter->q_vector[0], 0);
4633 
4634 	/* Clear any pending interrupts. */
4635 	rd32(IGC_ICR);
4636 	igc_irq_enable(adapter);
4637 
4638 	netif_tx_start_all_queues(adapter->netdev);
4639 
4640 	/* start the watchdog. */
4641 	hw->mac.get_link_status = true;
4642 	schedule_work(&adapter->watchdog_task);
4643 }
4644 
4645 /**
4646  * igc_update_stats - Update the board statistics counters
4647  * @adapter: board private structure
4648  */
4649 void igc_update_stats(struct igc_adapter *adapter)
4650 {
4651 	struct rtnl_link_stats64 *net_stats = &adapter->stats64;
4652 	struct pci_dev *pdev = adapter->pdev;
4653 	struct igc_hw *hw = &adapter->hw;
4654 	u64 _bytes, _packets;
4655 	u64 bytes, packets;
4656 	unsigned int start;
4657 	u32 mpc;
4658 	int i;
4659 
4660 	/* Prevent stats update while adapter is being reset, or if the pci
4661 	 * connection is down.
4662 	 */
4663 	if (adapter->link_speed == 0)
4664 		return;
4665 	if (pci_channel_offline(pdev))
4666 		return;
4667 
4668 	packets = 0;
4669 	bytes = 0;
4670 
4671 	rcu_read_lock();
4672 	for (i = 0; i < adapter->num_rx_queues; i++) {
4673 		struct igc_ring *ring = adapter->rx_ring[i];
4674 		u32 rqdpc = rd32(IGC_RQDPC(i));
4675 
4676 		if (hw->mac.type >= igc_i225)
4677 			wr32(IGC_RQDPC(i), 0);
4678 
4679 		if (rqdpc) {
4680 			ring->rx_stats.drops += rqdpc;
4681 			net_stats->rx_fifo_errors += rqdpc;
4682 		}
4683 
4684 		do {
4685 			start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
4686 			_bytes = ring->rx_stats.bytes;
4687 			_packets = ring->rx_stats.packets;
4688 		} while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
4689 		bytes += _bytes;
4690 		packets += _packets;
4691 	}
4692 
4693 	net_stats->rx_bytes = bytes;
4694 	net_stats->rx_packets = packets;
4695 
4696 	packets = 0;
4697 	bytes = 0;
4698 	for (i = 0; i < adapter->num_tx_queues; i++) {
4699 		struct igc_ring *ring = adapter->tx_ring[i];
4700 
4701 		do {
4702 			start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
4703 			_bytes = ring->tx_stats.bytes;
4704 			_packets = ring->tx_stats.packets;
4705 		} while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
4706 		bytes += _bytes;
4707 		packets += _packets;
4708 	}
4709 	net_stats->tx_bytes = bytes;
4710 	net_stats->tx_packets = packets;
4711 	rcu_read_unlock();
4712 
4713 	/* read stats registers */
4714 	adapter->stats.crcerrs += rd32(IGC_CRCERRS);
4715 	adapter->stats.gprc += rd32(IGC_GPRC);
4716 	adapter->stats.gorc += rd32(IGC_GORCL);
4717 	rd32(IGC_GORCH); /* clear GORCL */
4718 	adapter->stats.bprc += rd32(IGC_BPRC);
4719 	adapter->stats.mprc += rd32(IGC_MPRC);
4720 	adapter->stats.roc += rd32(IGC_ROC);
4721 
4722 	adapter->stats.prc64 += rd32(IGC_PRC64);
4723 	adapter->stats.prc127 += rd32(IGC_PRC127);
4724 	adapter->stats.prc255 += rd32(IGC_PRC255);
4725 	adapter->stats.prc511 += rd32(IGC_PRC511);
4726 	adapter->stats.prc1023 += rd32(IGC_PRC1023);
4727 	adapter->stats.prc1522 += rd32(IGC_PRC1522);
4728 	adapter->stats.tlpic += rd32(IGC_TLPIC);
4729 	adapter->stats.rlpic += rd32(IGC_RLPIC);
4730 	adapter->stats.hgptc += rd32(IGC_HGPTC);
4731 
4732 	mpc = rd32(IGC_MPC);
4733 	adapter->stats.mpc += mpc;
4734 	net_stats->rx_fifo_errors += mpc;
4735 	adapter->stats.scc += rd32(IGC_SCC);
4736 	adapter->stats.ecol += rd32(IGC_ECOL);
4737 	adapter->stats.mcc += rd32(IGC_MCC);
4738 	adapter->stats.latecol += rd32(IGC_LATECOL);
4739 	adapter->stats.dc += rd32(IGC_DC);
4740 	adapter->stats.rlec += rd32(IGC_RLEC);
4741 	adapter->stats.xonrxc += rd32(IGC_XONRXC);
4742 	adapter->stats.xontxc += rd32(IGC_XONTXC);
4743 	adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
4744 	adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
4745 	adapter->stats.fcruc += rd32(IGC_FCRUC);
4746 	adapter->stats.gptc += rd32(IGC_GPTC);
4747 	adapter->stats.gotc += rd32(IGC_GOTCL);
4748 	rd32(IGC_GOTCH); /* clear GOTCL */
4749 	adapter->stats.rnbc += rd32(IGC_RNBC);
4750 	adapter->stats.ruc += rd32(IGC_RUC);
4751 	adapter->stats.rfc += rd32(IGC_RFC);
4752 	adapter->stats.rjc += rd32(IGC_RJC);
4753 	adapter->stats.tor += rd32(IGC_TORH);
4754 	adapter->stats.tot += rd32(IGC_TOTH);
4755 	adapter->stats.tpr += rd32(IGC_TPR);
4756 
4757 	adapter->stats.ptc64 += rd32(IGC_PTC64);
4758 	adapter->stats.ptc127 += rd32(IGC_PTC127);
4759 	adapter->stats.ptc255 += rd32(IGC_PTC255);
4760 	adapter->stats.ptc511 += rd32(IGC_PTC511);
4761 	adapter->stats.ptc1023 += rd32(IGC_PTC1023);
4762 	adapter->stats.ptc1522 += rd32(IGC_PTC1522);
4763 
4764 	adapter->stats.mptc += rd32(IGC_MPTC);
4765 	adapter->stats.bptc += rd32(IGC_BPTC);
4766 
4767 	adapter->stats.tpt += rd32(IGC_TPT);
4768 	adapter->stats.colc += rd32(IGC_COLC);
4769 	adapter->stats.colc += rd32(IGC_RERC);
4770 
4771 	adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
4772 
4773 	adapter->stats.tsctc += rd32(IGC_TSCTC);
4774 
4775 	adapter->stats.iac += rd32(IGC_IAC);
4776 
4777 	/* Fill out the OS statistics structure */
4778 	net_stats->multicast = adapter->stats.mprc;
4779 	net_stats->collisions = adapter->stats.colc;
4780 
4781 	/* Rx Errors */
4782 
4783 	/* RLEC on some newer hardware can be incorrect so build
4784 	 * our own version based on RUC and ROC
4785 	 */
4786 	net_stats->rx_errors = adapter->stats.rxerrc +
4787 		adapter->stats.crcerrs + adapter->stats.algnerrc +
4788 		adapter->stats.ruc + adapter->stats.roc +
4789 		adapter->stats.cexterr;
4790 	net_stats->rx_length_errors = adapter->stats.ruc +
4791 				      adapter->stats.roc;
4792 	net_stats->rx_crc_errors = adapter->stats.crcerrs;
4793 	net_stats->rx_frame_errors = adapter->stats.algnerrc;
4794 	net_stats->rx_missed_errors = adapter->stats.mpc;
4795 
4796 	/* Tx Errors */
4797 	net_stats->tx_errors = adapter->stats.ecol +
4798 			       adapter->stats.latecol;
4799 	net_stats->tx_aborted_errors = adapter->stats.ecol;
4800 	net_stats->tx_window_errors = adapter->stats.latecol;
4801 	net_stats->tx_carrier_errors = adapter->stats.tncrs;
4802 
4803 	/* Tx Dropped needs to be maintained elsewhere */
4804 
4805 	/* Management Stats */
4806 	adapter->stats.mgptc += rd32(IGC_MGTPTC);
4807 	adapter->stats.mgprc += rd32(IGC_MGTPRC);
4808 	adapter->stats.mgpdc += rd32(IGC_MGTPDC);
4809 }
4810 
4811 /**
4812  * igc_down - Close the interface
4813  * @adapter: board private structure
4814  */
4815 void igc_down(struct igc_adapter *adapter)
4816 {
4817 	struct net_device *netdev = adapter->netdev;
4818 	struct igc_hw *hw = &adapter->hw;
4819 	u32 tctl, rctl;
4820 	int i = 0;
4821 
4822 	set_bit(__IGC_DOWN, &adapter->state);
4823 
4824 	igc_ptp_suspend(adapter);
4825 
4826 	if (pci_device_is_present(adapter->pdev)) {
4827 		/* disable receives in the hardware */
4828 		rctl = rd32(IGC_RCTL);
4829 		wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
4830 		/* flush and sleep below */
4831 	}
4832 	/* set trans_start so we don't get spurious watchdogs during reset */
4833 	netif_trans_update(netdev);
4834 
4835 	netif_carrier_off(netdev);
4836 	netif_tx_stop_all_queues(netdev);
4837 
4838 	if (pci_device_is_present(adapter->pdev)) {
4839 		/* disable transmits in the hardware */
4840 		tctl = rd32(IGC_TCTL);
4841 		tctl &= ~IGC_TCTL_EN;
4842 		wr32(IGC_TCTL, tctl);
4843 		/* flush both disables and wait for them to finish */
4844 		wrfl();
4845 		usleep_range(10000, 20000);
4846 
4847 		igc_irq_disable(adapter);
4848 	}
4849 
4850 	adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
4851 
4852 	for (i = 0; i < adapter->num_q_vectors; i++) {
4853 		if (adapter->q_vector[i]) {
4854 			napi_synchronize(&adapter->q_vector[i]->napi);
4855 			napi_disable(&adapter->q_vector[i]->napi);
4856 		}
4857 	}
4858 
4859 	del_timer_sync(&adapter->watchdog_timer);
4860 	del_timer_sync(&adapter->phy_info_timer);
4861 
4862 	/* record the stats before reset*/
4863 	spin_lock(&adapter->stats64_lock);
4864 	igc_update_stats(adapter);
4865 	spin_unlock(&adapter->stats64_lock);
4866 
4867 	adapter->link_speed = 0;
4868 	adapter->link_duplex = 0;
4869 
4870 	if (!pci_channel_offline(adapter->pdev))
4871 		igc_reset(adapter);
4872 
4873 	/* clear VLAN promisc flag so VFTA will be updated if necessary */
4874 	adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
4875 
4876 	igc_clean_all_tx_rings(adapter);
4877 	igc_clean_all_rx_rings(adapter);
4878 }
4879 
4880 void igc_reinit_locked(struct igc_adapter *adapter)
4881 {
4882 	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
4883 		usleep_range(1000, 2000);
4884 	igc_down(adapter);
4885 	igc_up(adapter);
4886 	clear_bit(__IGC_RESETTING, &adapter->state);
4887 }
4888 
4889 static void igc_reset_task(struct work_struct *work)
4890 {
4891 	struct igc_adapter *adapter;
4892 
4893 	adapter = container_of(work, struct igc_adapter, reset_task);
4894 
4895 	rtnl_lock();
4896 	/* If we're already down or resetting, just bail */
4897 	if (test_bit(__IGC_DOWN, &adapter->state) ||
4898 	    test_bit(__IGC_RESETTING, &adapter->state)) {
4899 		rtnl_unlock();
4900 		return;
4901 	}
4902 
4903 	igc_rings_dump(adapter);
4904 	igc_regs_dump(adapter);
4905 	netdev_err(adapter->netdev, "Reset adapter\n");
4906 	igc_reinit_locked(adapter);
4907 	rtnl_unlock();
4908 }
4909 
4910 /**
4911  * igc_change_mtu - Change the Maximum Transfer Unit
4912  * @netdev: network interface device structure
4913  * @new_mtu: new value for maximum frame size
4914  *
4915  * Returns 0 on success, negative on failure
4916  */
4917 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
4918 {
4919 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4920 	struct igc_adapter *adapter = netdev_priv(netdev);
4921 
4922 	if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) {
4923 		netdev_dbg(netdev, "Jumbo frames not supported with XDP");
4924 		return -EINVAL;
4925 	}
4926 
4927 	/* adjust max frame to be at least the size of a standard frame */
4928 	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
4929 		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
4930 
4931 	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
4932 		usleep_range(1000, 2000);
4933 
4934 	/* igc_down has a dependency on max_frame_size */
4935 	adapter->max_frame_size = max_frame;
4936 
4937 	if (netif_running(netdev))
4938 		igc_down(adapter);
4939 
4940 	netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
4941 	netdev->mtu = new_mtu;
4942 
4943 	if (netif_running(netdev))
4944 		igc_up(adapter);
4945 	else
4946 		igc_reset(adapter);
4947 
4948 	clear_bit(__IGC_RESETTING, &adapter->state);
4949 
4950 	return 0;
4951 }
4952 
4953 /**
4954  * igc_get_stats64 - Get System Network Statistics
4955  * @netdev: network interface device structure
4956  * @stats: rtnl_link_stats64 pointer
4957  *
4958  * Returns the address of the device statistics structure.
4959  * The statistics are updated here and also from the timer callback.
4960  */
4961 static void igc_get_stats64(struct net_device *netdev,
4962 			    struct rtnl_link_stats64 *stats)
4963 {
4964 	struct igc_adapter *adapter = netdev_priv(netdev);
4965 
4966 	spin_lock(&adapter->stats64_lock);
4967 	if (!test_bit(__IGC_RESETTING, &adapter->state))
4968 		igc_update_stats(adapter);
4969 	memcpy(stats, &adapter->stats64, sizeof(*stats));
4970 	spin_unlock(&adapter->stats64_lock);
4971 }
4972 
4973 static netdev_features_t igc_fix_features(struct net_device *netdev,
4974 					  netdev_features_t features)
4975 {
4976 	/* Since there is no support for separate Rx/Tx vlan accel
4977 	 * enable/disable make sure Tx flag is always in same state as Rx.
4978 	 */
4979 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
4980 		features |= NETIF_F_HW_VLAN_CTAG_TX;
4981 	else
4982 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4983 
4984 	return features;
4985 }
4986 
4987 static int igc_set_features(struct net_device *netdev,
4988 			    netdev_features_t features)
4989 {
4990 	netdev_features_t changed = netdev->features ^ features;
4991 	struct igc_adapter *adapter = netdev_priv(netdev);
4992 
4993 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
4994 		igc_vlan_mode(netdev, features);
4995 
4996 	/* Add VLAN support */
4997 	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
4998 		return 0;
4999 
5000 	if (!(features & NETIF_F_NTUPLE))
5001 		igc_flush_nfc_rules(adapter);
5002 
5003 	netdev->features = features;
5004 
5005 	if (netif_running(netdev))
5006 		igc_reinit_locked(adapter);
5007 	else
5008 		igc_reset(adapter);
5009 
5010 	return 1;
5011 }
5012 
5013 static netdev_features_t
5014 igc_features_check(struct sk_buff *skb, struct net_device *dev,
5015 		   netdev_features_t features)
5016 {
5017 	unsigned int network_hdr_len, mac_hdr_len;
5018 
5019 	/* Make certain the headers can be described by a context descriptor */
5020 	mac_hdr_len = skb_network_header(skb) - skb->data;
5021 	if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
5022 		return features & ~(NETIF_F_HW_CSUM |
5023 				    NETIF_F_SCTP_CRC |
5024 				    NETIF_F_HW_VLAN_CTAG_TX |
5025 				    NETIF_F_TSO |
5026 				    NETIF_F_TSO6);
5027 
5028 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
5029 	if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
5030 		return features & ~(NETIF_F_HW_CSUM |
5031 				    NETIF_F_SCTP_CRC |
5032 				    NETIF_F_TSO |
5033 				    NETIF_F_TSO6);
5034 
5035 	/* We can only support IPv4 TSO in tunnels if we can mangle the
5036 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
5037 	 */
5038 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
5039 		features &= ~NETIF_F_TSO;
5040 
5041 	return features;
5042 }
5043 
5044 static void igc_tsync_interrupt(struct igc_adapter *adapter)
5045 {
5046 	u32 ack, tsauxc, sec, nsec, tsicr;
5047 	struct igc_hw *hw = &adapter->hw;
5048 	struct ptp_clock_event event;
5049 	struct timespec64 ts;
5050 
5051 	tsicr = rd32(IGC_TSICR);
5052 	ack = 0;
5053 
5054 	if (tsicr & IGC_TSICR_SYS_WRAP) {
5055 		event.type = PTP_CLOCK_PPS;
5056 		if (adapter->ptp_caps.pps)
5057 			ptp_clock_event(adapter->ptp_clock, &event);
5058 		ack |= IGC_TSICR_SYS_WRAP;
5059 	}
5060 
5061 	if (tsicr & IGC_TSICR_TXTS) {
5062 		/* retrieve hardware timestamp */
5063 		schedule_work(&adapter->ptp_tx_work);
5064 		ack |= IGC_TSICR_TXTS;
5065 	}
5066 
5067 	if (tsicr & IGC_TSICR_TT0) {
5068 		spin_lock(&adapter->tmreg_lock);
5069 		ts = timespec64_add(adapter->perout[0].start,
5070 				    adapter->perout[0].period);
5071 		wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5072 		wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec);
5073 		tsauxc = rd32(IGC_TSAUXC);
5074 		tsauxc |= IGC_TSAUXC_EN_TT0;
5075 		wr32(IGC_TSAUXC, tsauxc);
5076 		adapter->perout[0].start = ts;
5077 		spin_unlock(&adapter->tmreg_lock);
5078 		ack |= IGC_TSICR_TT0;
5079 	}
5080 
5081 	if (tsicr & IGC_TSICR_TT1) {
5082 		spin_lock(&adapter->tmreg_lock);
5083 		ts = timespec64_add(adapter->perout[1].start,
5084 				    adapter->perout[1].period);
5085 		wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5086 		wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec);
5087 		tsauxc = rd32(IGC_TSAUXC);
5088 		tsauxc |= IGC_TSAUXC_EN_TT1;
5089 		wr32(IGC_TSAUXC, tsauxc);
5090 		adapter->perout[1].start = ts;
5091 		spin_unlock(&adapter->tmreg_lock);
5092 		ack |= IGC_TSICR_TT1;
5093 	}
5094 
5095 	if (tsicr & IGC_TSICR_AUTT0) {
5096 		nsec = rd32(IGC_AUXSTMPL0);
5097 		sec  = rd32(IGC_AUXSTMPH0);
5098 		event.type = PTP_CLOCK_EXTTS;
5099 		event.index = 0;
5100 		event.timestamp = sec * NSEC_PER_SEC + nsec;
5101 		ptp_clock_event(adapter->ptp_clock, &event);
5102 		ack |= IGC_TSICR_AUTT0;
5103 	}
5104 
5105 	if (tsicr & IGC_TSICR_AUTT1) {
5106 		nsec = rd32(IGC_AUXSTMPL1);
5107 		sec  = rd32(IGC_AUXSTMPH1);
5108 		event.type = PTP_CLOCK_EXTTS;
5109 		event.index = 1;
5110 		event.timestamp = sec * NSEC_PER_SEC + nsec;
5111 		ptp_clock_event(adapter->ptp_clock, &event);
5112 		ack |= IGC_TSICR_AUTT1;
5113 	}
5114 
5115 	/* acknowledge the interrupts */
5116 	wr32(IGC_TSICR, ack);
5117 }
5118 
5119 /**
5120  * igc_msix_other - msix other interrupt handler
5121  * @irq: interrupt number
5122  * @data: pointer to a q_vector
5123  */
5124 static irqreturn_t igc_msix_other(int irq, void *data)
5125 {
5126 	struct igc_adapter *adapter = data;
5127 	struct igc_hw *hw = &adapter->hw;
5128 	u32 icr = rd32(IGC_ICR);
5129 
5130 	/* reading ICR causes bit 31 of EICR to be cleared */
5131 	if (icr & IGC_ICR_DRSTA)
5132 		schedule_work(&adapter->reset_task);
5133 
5134 	if (icr & IGC_ICR_DOUTSYNC) {
5135 		/* HW is reporting DMA is out of sync */
5136 		adapter->stats.doosync++;
5137 	}
5138 
5139 	if (icr & IGC_ICR_LSC) {
5140 		hw->mac.get_link_status = true;
5141 		/* guard against interrupt when we're going down */
5142 		if (!test_bit(__IGC_DOWN, &adapter->state))
5143 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5144 	}
5145 
5146 	if (icr & IGC_ICR_TS)
5147 		igc_tsync_interrupt(adapter);
5148 
5149 	wr32(IGC_EIMS, adapter->eims_other);
5150 
5151 	return IRQ_HANDLED;
5152 }
5153 
5154 static void igc_write_itr(struct igc_q_vector *q_vector)
5155 {
5156 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
5157 
5158 	if (!q_vector->set_itr)
5159 		return;
5160 
5161 	if (!itr_val)
5162 		itr_val = IGC_ITR_VAL_MASK;
5163 
5164 	itr_val |= IGC_EITR_CNT_IGNR;
5165 
5166 	writel(itr_val, q_vector->itr_register);
5167 	q_vector->set_itr = 0;
5168 }
5169 
5170 static irqreturn_t igc_msix_ring(int irq, void *data)
5171 {
5172 	struct igc_q_vector *q_vector = data;
5173 
5174 	/* Write the ITR value calculated from the previous interrupt. */
5175 	igc_write_itr(q_vector);
5176 
5177 	napi_schedule(&q_vector->napi);
5178 
5179 	return IRQ_HANDLED;
5180 }
5181 
5182 /**
5183  * igc_request_msix - Initialize MSI-X interrupts
5184  * @adapter: Pointer to adapter structure
5185  *
5186  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
5187  * kernel.
5188  */
5189 static int igc_request_msix(struct igc_adapter *adapter)
5190 {
5191 	unsigned int num_q_vectors = adapter->num_q_vectors;
5192 	int i = 0, err = 0, vector = 0, free_vector = 0;
5193 	struct net_device *netdev = adapter->netdev;
5194 
5195 	err = request_irq(adapter->msix_entries[vector].vector,
5196 			  &igc_msix_other, 0, netdev->name, adapter);
5197 	if (err)
5198 		goto err_out;
5199 
5200 	if (num_q_vectors > MAX_Q_VECTORS) {
5201 		num_q_vectors = MAX_Q_VECTORS;
5202 		dev_warn(&adapter->pdev->dev,
5203 			 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5204 			 adapter->num_q_vectors, MAX_Q_VECTORS);
5205 	}
5206 	for (i = 0; i < num_q_vectors; i++) {
5207 		struct igc_q_vector *q_vector = adapter->q_vector[i];
5208 
5209 		vector++;
5210 
5211 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
5212 
5213 		if (q_vector->rx.ring && q_vector->tx.ring)
5214 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
5215 				q_vector->rx.ring->queue_index);
5216 		else if (q_vector->tx.ring)
5217 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
5218 				q_vector->tx.ring->queue_index);
5219 		else if (q_vector->rx.ring)
5220 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
5221 				q_vector->rx.ring->queue_index);
5222 		else
5223 			sprintf(q_vector->name, "%s-unused", netdev->name);
5224 
5225 		err = request_irq(adapter->msix_entries[vector].vector,
5226 				  igc_msix_ring, 0, q_vector->name,
5227 				  q_vector);
5228 		if (err)
5229 			goto err_free;
5230 	}
5231 
5232 	igc_configure_msix(adapter);
5233 	return 0;
5234 
5235 err_free:
5236 	/* free already assigned IRQs */
5237 	free_irq(adapter->msix_entries[free_vector++].vector, adapter);
5238 
5239 	vector--;
5240 	for (i = 0; i < vector; i++) {
5241 		free_irq(adapter->msix_entries[free_vector++].vector,
5242 			 adapter->q_vector[i]);
5243 	}
5244 err_out:
5245 	return err;
5246 }
5247 
5248 /**
5249  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5250  * @adapter: Pointer to adapter structure
5251  *
5252  * This function resets the device so that it has 0 rx queues, tx queues, and
5253  * MSI-X interrupts allocated.
5254  */
5255 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
5256 {
5257 	igc_free_q_vectors(adapter);
5258 	igc_reset_interrupt_capability(adapter);
5259 }
5260 
5261 /* Need to wait a few seconds after link up to get diagnostic information from
5262  * the phy
5263  */
5264 static void igc_update_phy_info(struct timer_list *t)
5265 {
5266 	struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5267 
5268 	igc_get_phy_info(&adapter->hw);
5269 }
5270 
5271 /**
5272  * igc_has_link - check shared code for link and determine up/down
5273  * @adapter: pointer to driver private info
5274  */
5275 bool igc_has_link(struct igc_adapter *adapter)
5276 {
5277 	struct igc_hw *hw = &adapter->hw;
5278 	bool link_active = false;
5279 
5280 	/* get_link_status is set on LSC (link status) interrupt or
5281 	 * rx sequence error interrupt.  get_link_status will stay
5282 	 * false until the igc_check_for_link establishes link
5283 	 * for copper adapters ONLY
5284 	 */
5285 	if (!hw->mac.get_link_status)
5286 		return true;
5287 	hw->mac.ops.check_for_link(hw);
5288 	link_active = !hw->mac.get_link_status;
5289 
5290 	if (hw->mac.type == igc_i225) {
5291 		if (!netif_carrier_ok(adapter->netdev)) {
5292 			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5293 		} else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
5294 			adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
5295 			adapter->link_check_timeout = jiffies;
5296 		}
5297 	}
5298 
5299 	return link_active;
5300 }
5301 
5302 /**
5303  * igc_watchdog - Timer Call-back
5304  * @t: timer for the watchdog
5305  */
5306 static void igc_watchdog(struct timer_list *t)
5307 {
5308 	struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5309 	/* Do the rest outside of interrupt context */
5310 	schedule_work(&adapter->watchdog_task);
5311 }
5312 
5313 static void igc_watchdog_task(struct work_struct *work)
5314 {
5315 	struct igc_adapter *adapter = container_of(work,
5316 						   struct igc_adapter,
5317 						   watchdog_task);
5318 	struct net_device *netdev = adapter->netdev;
5319 	struct igc_hw *hw = &adapter->hw;
5320 	struct igc_phy_info *phy = &hw->phy;
5321 	u16 phy_data, retry_count = 20;
5322 	u32 link;
5323 	int i;
5324 
5325 	link = igc_has_link(adapter);
5326 
5327 	if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
5328 		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5329 			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5330 		else
5331 			link = false;
5332 	}
5333 
5334 	if (link) {
5335 		/* Cancel scheduled suspend requests. */
5336 		pm_runtime_resume(netdev->dev.parent);
5337 
5338 		if (!netif_carrier_ok(netdev)) {
5339 			u32 ctrl;
5340 
5341 			hw->mac.ops.get_speed_and_duplex(hw,
5342 							 &adapter->link_speed,
5343 							 &adapter->link_duplex);
5344 
5345 			ctrl = rd32(IGC_CTRL);
5346 			/* Link status message must follow this format */
5347 			netdev_info(netdev,
5348 				    "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5349 				    adapter->link_speed,
5350 				    adapter->link_duplex == FULL_DUPLEX ?
5351 				    "Full" : "Half",
5352 				    (ctrl & IGC_CTRL_TFCE) &&
5353 				    (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
5354 				    (ctrl & IGC_CTRL_RFCE) ?  "RX" :
5355 				    (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
5356 
5357 			/* disable EEE if enabled */
5358 			if ((adapter->flags & IGC_FLAG_EEE) &&
5359 			    adapter->link_duplex == HALF_DUPLEX) {
5360 				netdev_info(netdev,
5361 					    "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
5362 				adapter->hw.dev_spec._base.eee_enable = false;
5363 				adapter->flags &= ~IGC_FLAG_EEE;
5364 			}
5365 
5366 			/* check if SmartSpeed worked */
5367 			igc_check_downshift(hw);
5368 			if (phy->speed_downgraded)
5369 				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5370 
5371 			/* adjust timeout factor according to speed/duplex */
5372 			adapter->tx_timeout_factor = 1;
5373 			switch (adapter->link_speed) {
5374 			case SPEED_10:
5375 				adapter->tx_timeout_factor = 14;
5376 				break;
5377 			case SPEED_100:
5378 			case SPEED_1000:
5379 			case SPEED_2500:
5380 				adapter->tx_timeout_factor = 7;
5381 				break;
5382 			}
5383 
5384 			if (adapter->link_speed != SPEED_1000)
5385 				goto no_wait;
5386 
5387 			/* wait for Remote receiver status OK */
5388 retry_read_status:
5389 			if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
5390 					      &phy_data)) {
5391 				if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5392 				    retry_count) {
5393 					msleep(100);
5394 					retry_count--;
5395 					goto retry_read_status;
5396 				} else if (!retry_count) {
5397 					netdev_err(netdev, "exceed max 2 second\n");
5398 				}
5399 			} else {
5400 				netdev_err(netdev, "read 1000Base-T Status Reg\n");
5401 			}
5402 no_wait:
5403 			netif_carrier_on(netdev);
5404 
5405 			/* link state has changed, schedule phy info update */
5406 			if (!test_bit(__IGC_DOWN, &adapter->state))
5407 				mod_timer(&adapter->phy_info_timer,
5408 					  round_jiffies(jiffies + 2 * HZ));
5409 		}
5410 	} else {
5411 		if (netif_carrier_ok(netdev)) {
5412 			adapter->link_speed = 0;
5413 			adapter->link_duplex = 0;
5414 
5415 			/* Links status message must follow this format */
5416 			netdev_info(netdev, "NIC Link is Down\n");
5417 			netif_carrier_off(netdev);
5418 
5419 			/* link state has changed, schedule phy info update */
5420 			if (!test_bit(__IGC_DOWN, &adapter->state))
5421 				mod_timer(&adapter->phy_info_timer,
5422 					  round_jiffies(jiffies + 2 * HZ));
5423 
5424 			/* link is down, time to check for alternate media */
5425 			if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
5426 				if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
5427 					schedule_work(&adapter->reset_task);
5428 					/* return immediately */
5429 					return;
5430 				}
5431 			}
5432 			pm_schedule_suspend(netdev->dev.parent,
5433 					    MSEC_PER_SEC * 5);
5434 
5435 		/* also check for alternate media here */
5436 		} else if (!netif_carrier_ok(netdev) &&
5437 			   (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
5438 			if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
5439 				schedule_work(&adapter->reset_task);
5440 				/* return immediately */
5441 				return;
5442 			}
5443 		}
5444 	}
5445 
5446 	spin_lock(&adapter->stats64_lock);
5447 	igc_update_stats(adapter);
5448 	spin_unlock(&adapter->stats64_lock);
5449 
5450 	for (i = 0; i < adapter->num_tx_queues; i++) {
5451 		struct igc_ring *tx_ring = adapter->tx_ring[i];
5452 
5453 		if (!netif_carrier_ok(netdev)) {
5454 			/* We've lost link, so the controller stops DMA,
5455 			 * but we've got queued Tx work that's never going
5456 			 * to get done, so reset controller to flush Tx.
5457 			 * (Do the reset outside of interrupt context).
5458 			 */
5459 			if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
5460 				adapter->tx_timeout_count++;
5461 				schedule_work(&adapter->reset_task);
5462 				/* return immediately since reset is imminent */
5463 				return;
5464 			}
5465 		}
5466 
5467 		/* Force detection of hung controller every watchdog period */
5468 		set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5469 	}
5470 
5471 	/* Cause software interrupt to ensure Rx ring is cleaned */
5472 	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5473 		u32 eics = 0;
5474 
5475 		for (i = 0; i < adapter->num_q_vectors; i++)
5476 			eics |= adapter->q_vector[i]->eims_value;
5477 		wr32(IGC_EICS, eics);
5478 	} else {
5479 		wr32(IGC_ICS, IGC_ICS_RXDMT0);
5480 	}
5481 
5482 	igc_ptp_tx_hang(adapter);
5483 
5484 	/* Reset the timer */
5485 	if (!test_bit(__IGC_DOWN, &adapter->state)) {
5486 		if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
5487 			mod_timer(&adapter->watchdog_timer,
5488 				  round_jiffies(jiffies +  HZ));
5489 		else
5490 			mod_timer(&adapter->watchdog_timer,
5491 				  round_jiffies(jiffies + 2 * HZ));
5492 	}
5493 }
5494 
5495 /**
5496  * igc_intr_msi - Interrupt Handler
5497  * @irq: interrupt number
5498  * @data: pointer to a network interface device structure
5499  */
5500 static irqreturn_t igc_intr_msi(int irq, void *data)
5501 {
5502 	struct igc_adapter *adapter = data;
5503 	struct igc_q_vector *q_vector = adapter->q_vector[0];
5504 	struct igc_hw *hw = &adapter->hw;
5505 	/* read ICR disables interrupts using IAM */
5506 	u32 icr = rd32(IGC_ICR);
5507 
5508 	igc_write_itr(q_vector);
5509 
5510 	if (icr & IGC_ICR_DRSTA)
5511 		schedule_work(&adapter->reset_task);
5512 
5513 	if (icr & IGC_ICR_DOUTSYNC) {
5514 		/* HW is reporting DMA is out of sync */
5515 		adapter->stats.doosync++;
5516 	}
5517 
5518 	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5519 		hw->mac.get_link_status = true;
5520 		if (!test_bit(__IGC_DOWN, &adapter->state))
5521 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5522 	}
5523 
5524 	if (icr & IGC_ICR_TS)
5525 		igc_tsync_interrupt(adapter);
5526 
5527 	napi_schedule(&q_vector->napi);
5528 
5529 	return IRQ_HANDLED;
5530 }
5531 
5532 /**
5533  * igc_intr - Legacy Interrupt Handler
5534  * @irq: interrupt number
5535  * @data: pointer to a network interface device structure
5536  */
5537 static irqreturn_t igc_intr(int irq, void *data)
5538 {
5539 	struct igc_adapter *adapter = data;
5540 	struct igc_q_vector *q_vector = adapter->q_vector[0];
5541 	struct igc_hw *hw = &adapter->hw;
5542 	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5543 	 * need for the IMC write
5544 	 */
5545 	u32 icr = rd32(IGC_ICR);
5546 
5547 	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5548 	 * not set, then the adapter didn't send an interrupt
5549 	 */
5550 	if (!(icr & IGC_ICR_INT_ASSERTED))
5551 		return IRQ_NONE;
5552 
5553 	igc_write_itr(q_vector);
5554 
5555 	if (icr & IGC_ICR_DRSTA)
5556 		schedule_work(&adapter->reset_task);
5557 
5558 	if (icr & IGC_ICR_DOUTSYNC) {
5559 		/* HW is reporting DMA is out of sync */
5560 		adapter->stats.doosync++;
5561 	}
5562 
5563 	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5564 		hw->mac.get_link_status = true;
5565 		/* guard against interrupt when we're going down */
5566 		if (!test_bit(__IGC_DOWN, &adapter->state))
5567 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5568 	}
5569 
5570 	if (icr & IGC_ICR_TS)
5571 		igc_tsync_interrupt(adapter);
5572 
5573 	napi_schedule(&q_vector->napi);
5574 
5575 	return IRQ_HANDLED;
5576 }
5577 
5578 static void igc_free_irq(struct igc_adapter *adapter)
5579 {
5580 	if (adapter->msix_entries) {
5581 		int vector = 0, i;
5582 
5583 		free_irq(adapter->msix_entries[vector++].vector, adapter);
5584 
5585 		for (i = 0; i < adapter->num_q_vectors; i++)
5586 			free_irq(adapter->msix_entries[vector++].vector,
5587 				 adapter->q_vector[i]);
5588 	} else {
5589 		free_irq(adapter->pdev->irq, adapter);
5590 	}
5591 }
5592 
5593 /**
5594  * igc_request_irq - initialize interrupts
5595  * @adapter: Pointer to adapter structure
5596  *
5597  * Attempts to configure interrupts using the best available
5598  * capabilities of the hardware and kernel.
5599  */
5600 static int igc_request_irq(struct igc_adapter *adapter)
5601 {
5602 	struct net_device *netdev = adapter->netdev;
5603 	struct pci_dev *pdev = adapter->pdev;
5604 	int err = 0;
5605 
5606 	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5607 		err = igc_request_msix(adapter);
5608 		if (!err)
5609 			goto request_done;
5610 		/* fall back to MSI */
5611 		igc_free_all_tx_resources(adapter);
5612 		igc_free_all_rx_resources(adapter);
5613 
5614 		igc_clear_interrupt_scheme(adapter);
5615 		err = igc_init_interrupt_scheme(adapter, false);
5616 		if (err)
5617 			goto request_done;
5618 		igc_setup_all_tx_resources(adapter);
5619 		igc_setup_all_rx_resources(adapter);
5620 		igc_configure(adapter);
5621 	}
5622 
5623 	igc_assign_vector(adapter->q_vector[0], 0);
5624 
5625 	if (adapter->flags & IGC_FLAG_HAS_MSI) {
5626 		err = request_irq(pdev->irq, &igc_intr_msi, 0,
5627 				  netdev->name, adapter);
5628 		if (!err)
5629 			goto request_done;
5630 
5631 		/* fall back to legacy interrupts */
5632 		igc_reset_interrupt_capability(adapter);
5633 		adapter->flags &= ~IGC_FLAG_HAS_MSI;
5634 	}
5635 
5636 	err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
5637 			  netdev->name, adapter);
5638 
5639 	if (err)
5640 		netdev_err(netdev, "Error %d getting interrupt\n", err);
5641 
5642 request_done:
5643 	return err;
5644 }
5645 
5646 /**
5647  * __igc_open - Called when a network interface is made active
5648  * @netdev: network interface device structure
5649  * @resuming: boolean indicating if the device is resuming
5650  *
5651  * Returns 0 on success, negative value on failure
5652  *
5653  * The open entry point is called when a network interface is made
5654  * active by the system (IFF_UP).  At this point all resources needed
5655  * for transmit and receive operations are allocated, the interrupt
5656  * handler is registered with the OS, the watchdog timer is started,
5657  * and the stack is notified that the interface is ready.
5658  */
5659 static int __igc_open(struct net_device *netdev, bool resuming)
5660 {
5661 	struct igc_adapter *adapter = netdev_priv(netdev);
5662 	struct pci_dev *pdev = adapter->pdev;
5663 	struct igc_hw *hw = &adapter->hw;
5664 	int err = 0;
5665 	int i = 0;
5666 
5667 	/* disallow open during test */
5668 
5669 	if (test_bit(__IGC_TESTING, &adapter->state)) {
5670 		WARN_ON(resuming);
5671 		return -EBUSY;
5672 	}
5673 
5674 	if (!resuming)
5675 		pm_runtime_get_sync(&pdev->dev);
5676 
5677 	netif_carrier_off(netdev);
5678 
5679 	/* allocate transmit descriptors */
5680 	err = igc_setup_all_tx_resources(adapter);
5681 	if (err)
5682 		goto err_setup_tx;
5683 
5684 	/* allocate receive descriptors */
5685 	err = igc_setup_all_rx_resources(adapter);
5686 	if (err)
5687 		goto err_setup_rx;
5688 
5689 	igc_power_up_link(adapter);
5690 
5691 	igc_configure(adapter);
5692 
5693 	err = igc_request_irq(adapter);
5694 	if (err)
5695 		goto err_req_irq;
5696 
5697 	/* Notify the stack of the actual queue counts. */
5698 	err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
5699 	if (err)
5700 		goto err_set_queues;
5701 
5702 	err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
5703 	if (err)
5704 		goto err_set_queues;
5705 
5706 	clear_bit(__IGC_DOWN, &adapter->state);
5707 
5708 	for (i = 0; i < adapter->num_q_vectors; i++)
5709 		napi_enable(&adapter->q_vector[i]->napi);
5710 
5711 	/* Clear any pending interrupts. */
5712 	rd32(IGC_ICR);
5713 	igc_irq_enable(adapter);
5714 
5715 	if (!resuming)
5716 		pm_runtime_put(&pdev->dev);
5717 
5718 	netif_tx_start_all_queues(netdev);
5719 
5720 	/* start the watchdog. */
5721 	hw->mac.get_link_status = true;
5722 	schedule_work(&adapter->watchdog_task);
5723 
5724 	return IGC_SUCCESS;
5725 
5726 err_set_queues:
5727 	igc_free_irq(adapter);
5728 err_req_irq:
5729 	igc_release_hw_control(adapter);
5730 	igc_power_down_phy_copper_base(&adapter->hw);
5731 	igc_free_all_rx_resources(adapter);
5732 err_setup_rx:
5733 	igc_free_all_tx_resources(adapter);
5734 err_setup_tx:
5735 	igc_reset(adapter);
5736 	if (!resuming)
5737 		pm_runtime_put(&pdev->dev);
5738 
5739 	return err;
5740 }
5741 
5742 int igc_open(struct net_device *netdev)
5743 {
5744 	return __igc_open(netdev, false);
5745 }
5746 
5747 /**
5748  * __igc_close - Disables a network interface
5749  * @netdev: network interface device structure
5750  * @suspending: boolean indicating the device is suspending
5751  *
5752  * Returns 0, this is not allowed to fail
5753  *
5754  * The close entry point is called when an interface is de-activated
5755  * by the OS.  The hardware is still under the driver's control, but
5756  * needs to be disabled.  A global MAC reset is issued to stop the
5757  * hardware, and all transmit and receive resources are freed.
5758  */
5759 static int __igc_close(struct net_device *netdev, bool suspending)
5760 {
5761 	struct igc_adapter *adapter = netdev_priv(netdev);
5762 	struct pci_dev *pdev = adapter->pdev;
5763 
5764 	WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
5765 
5766 	if (!suspending)
5767 		pm_runtime_get_sync(&pdev->dev);
5768 
5769 	igc_down(adapter);
5770 
5771 	igc_release_hw_control(adapter);
5772 
5773 	igc_free_irq(adapter);
5774 
5775 	igc_free_all_tx_resources(adapter);
5776 	igc_free_all_rx_resources(adapter);
5777 
5778 	if (!suspending)
5779 		pm_runtime_put_sync(&pdev->dev);
5780 
5781 	return 0;
5782 }
5783 
5784 int igc_close(struct net_device *netdev)
5785 {
5786 	if (netif_device_present(netdev) || netdev->dismantle)
5787 		return __igc_close(netdev, false);
5788 	return 0;
5789 }
5790 
5791 /**
5792  * igc_ioctl - Access the hwtstamp interface
5793  * @netdev: network interface device structure
5794  * @ifr: interface request data
5795  * @cmd: ioctl command
5796  **/
5797 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5798 {
5799 	switch (cmd) {
5800 	case SIOCGHWTSTAMP:
5801 		return igc_ptp_get_ts_config(netdev, ifr);
5802 	case SIOCSHWTSTAMP:
5803 		return igc_ptp_set_ts_config(netdev, ifr);
5804 	default:
5805 		return -EOPNOTSUPP;
5806 	}
5807 }
5808 
5809 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
5810 				      bool enable)
5811 {
5812 	struct igc_ring *ring;
5813 
5814 	if (queue < 0 || queue >= adapter->num_tx_queues)
5815 		return -EINVAL;
5816 
5817 	ring = adapter->tx_ring[queue];
5818 	ring->launchtime_enable = enable;
5819 
5820 	return 0;
5821 }
5822 
5823 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
5824 {
5825 	struct timespec64 b;
5826 
5827 	b = ktime_to_timespec64(base_time);
5828 
5829 	return timespec64_compare(now, &b) > 0;
5830 }
5831 
5832 static bool validate_schedule(struct igc_adapter *adapter,
5833 			      const struct tc_taprio_qopt_offload *qopt)
5834 {
5835 	int queue_uses[IGC_MAX_TX_QUEUES] = { };
5836 	struct timespec64 now;
5837 	size_t n;
5838 
5839 	if (qopt->cycle_time_extension)
5840 		return false;
5841 
5842 	igc_ptp_read(adapter, &now);
5843 
5844 	/* If we program the controller's BASET registers with a time
5845 	 * in the future, it will hold all the packets until that
5846 	 * time, causing a lot of TX Hangs, so to avoid that, we
5847 	 * reject schedules that would start in the future.
5848 	 */
5849 	if (!is_base_time_past(qopt->base_time, &now))
5850 		return false;
5851 
5852 	for (n = 0; n < qopt->num_entries; n++) {
5853 		const struct tc_taprio_sched_entry *e, *prev;
5854 		int i;
5855 
5856 		prev = n ? &qopt->entries[n - 1] : NULL;
5857 		e = &qopt->entries[n];
5858 
5859 		/* i225 only supports "global" frame preemption
5860 		 * settings.
5861 		 */
5862 		if (e->command != TC_TAPRIO_CMD_SET_GATES)
5863 			return false;
5864 
5865 		for (i = 0; i < adapter->num_tx_queues; i++) {
5866 			if (e->gate_mask & BIT(i))
5867 				queue_uses[i]++;
5868 
5869 			/* There are limitations: A single queue cannot be
5870 			 * opened and closed multiple times per cycle unless the
5871 			 * gate stays open. Check for it.
5872 			 */
5873 			if (queue_uses[i] > 1 &&
5874 			    !(prev->gate_mask & BIT(i)))
5875 				return false;
5876 		}
5877 	}
5878 
5879 	return true;
5880 }
5881 
5882 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
5883 				     struct tc_etf_qopt_offload *qopt)
5884 {
5885 	struct igc_hw *hw = &adapter->hw;
5886 	int err;
5887 
5888 	if (hw->mac.type != igc_i225)
5889 		return -EOPNOTSUPP;
5890 
5891 	err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
5892 	if (err)
5893 		return err;
5894 
5895 	return igc_tsn_offload_apply(adapter);
5896 }
5897 
5898 static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
5899 {
5900 	int i;
5901 
5902 	adapter->base_time = 0;
5903 	adapter->cycle_time = NSEC_PER_SEC;
5904 
5905 	for (i = 0; i < adapter->num_tx_queues; i++) {
5906 		struct igc_ring *ring = adapter->tx_ring[i];
5907 
5908 		ring->start_time = 0;
5909 		ring->end_time = NSEC_PER_SEC;
5910 	}
5911 
5912 	return 0;
5913 }
5914 
5915 static int igc_save_qbv_schedule(struct igc_adapter *adapter,
5916 				 struct tc_taprio_qopt_offload *qopt)
5917 {
5918 	bool queue_configured[IGC_MAX_TX_QUEUES] = { };
5919 	u32 start_time = 0, end_time = 0;
5920 	size_t n;
5921 
5922 	if (!qopt->enable)
5923 		return igc_tsn_clear_schedule(adapter);
5924 
5925 	if (adapter->base_time)
5926 		return -EALREADY;
5927 
5928 	if (!validate_schedule(adapter, qopt))
5929 		return -EINVAL;
5930 
5931 	adapter->cycle_time = qopt->cycle_time;
5932 	adapter->base_time = qopt->base_time;
5933 
5934 	for (n = 0; n < qopt->num_entries; n++) {
5935 		struct tc_taprio_sched_entry *e = &qopt->entries[n];
5936 		int i;
5937 
5938 		end_time += e->interval;
5939 
5940 		for (i = 0; i < adapter->num_tx_queues; i++) {
5941 			struct igc_ring *ring = adapter->tx_ring[i];
5942 
5943 			if (!(e->gate_mask & BIT(i)))
5944 				continue;
5945 
5946 			/* Check whether a queue stays open for more than one
5947 			 * entry. If so, keep the start and advance the end
5948 			 * time.
5949 			 */
5950 			if (!queue_configured[i])
5951 				ring->start_time = start_time;
5952 			ring->end_time = end_time;
5953 
5954 			queue_configured[i] = true;
5955 		}
5956 
5957 		start_time += e->interval;
5958 	}
5959 
5960 	return 0;
5961 }
5962 
5963 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
5964 					 struct tc_taprio_qopt_offload *qopt)
5965 {
5966 	struct igc_hw *hw = &adapter->hw;
5967 	int err;
5968 
5969 	if (hw->mac.type != igc_i225)
5970 		return -EOPNOTSUPP;
5971 
5972 	err = igc_save_qbv_schedule(adapter, qopt);
5973 	if (err)
5974 		return err;
5975 
5976 	return igc_tsn_offload_apply(adapter);
5977 }
5978 
5979 static int igc_save_cbs_params(struct igc_adapter *adapter, int queue,
5980 			       bool enable, int idleslope, int sendslope,
5981 			       int hicredit, int locredit)
5982 {
5983 	bool cbs_status[IGC_MAX_SR_QUEUES] = { false };
5984 	struct net_device *netdev = adapter->netdev;
5985 	struct igc_ring *ring;
5986 	int i;
5987 
5988 	/* i225 has two sets of credit-based shaper logic.
5989 	 * Supporting it only on the top two priority queues
5990 	 */
5991 	if (queue < 0 || queue > 1)
5992 		return -EINVAL;
5993 
5994 	ring = adapter->tx_ring[queue];
5995 
5996 	for (i = 0; i < IGC_MAX_SR_QUEUES; i++)
5997 		if (adapter->tx_ring[i])
5998 			cbs_status[i] = adapter->tx_ring[i]->cbs_enable;
5999 
6000 	/* CBS should be enabled on the highest priority queue first in order
6001 	 * for the CBS algorithm to operate as intended.
6002 	 */
6003 	if (enable) {
6004 		if (queue == 1 && !cbs_status[0]) {
6005 			netdev_err(netdev,
6006 				   "Enabling CBS on queue1 before queue0\n");
6007 			return -EINVAL;
6008 		}
6009 	} else {
6010 		if (queue == 0 && cbs_status[1]) {
6011 			netdev_err(netdev,
6012 				   "Disabling CBS on queue0 before queue1\n");
6013 			return -EINVAL;
6014 		}
6015 	}
6016 
6017 	ring->cbs_enable = enable;
6018 	ring->idleslope = idleslope;
6019 	ring->sendslope = sendslope;
6020 	ring->hicredit = hicredit;
6021 	ring->locredit = locredit;
6022 
6023 	return 0;
6024 }
6025 
6026 static int igc_tsn_enable_cbs(struct igc_adapter *adapter,
6027 			      struct tc_cbs_qopt_offload *qopt)
6028 {
6029 	struct igc_hw *hw = &adapter->hw;
6030 	int err;
6031 
6032 	if (hw->mac.type != igc_i225)
6033 		return -EOPNOTSUPP;
6034 
6035 	if (qopt->queue < 0 || qopt->queue > 1)
6036 		return -EINVAL;
6037 
6038 	err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable,
6039 				  qopt->idleslope, qopt->sendslope,
6040 				  qopt->hicredit, qopt->locredit);
6041 	if (err)
6042 		return err;
6043 
6044 	return igc_tsn_offload_apply(adapter);
6045 }
6046 
6047 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
6048 			void *type_data)
6049 {
6050 	struct igc_adapter *adapter = netdev_priv(dev);
6051 
6052 	switch (type) {
6053 	case TC_SETUP_QDISC_TAPRIO:
6054 		return igc_tsn_enable_qbv_scheduling(adapter, type_data);
6055 
6056 	case TC_SETUP_QDISC_ETF:
6057 		return igc_tsn_enable_launchtime(adapter, type_data);
6058 
6059 	case TC_SETUP_QDISC_CBS:
6060 		return igc_tsn_enable_cbs(adapter, type_data);
6061 
6062 	default:
6063 		return -EOPNOTSUPP;
6064 	}
6065 }
6066 
6067 static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6068 {
6069 	struct igc_adapter *adapter = netdev_priv(dev);
6070 
6071 	switch (bpf->command) {
6072 	case XDP_SETUP_PROG:
6073 		return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack);
6074 	case XDP_SETUP_XSK_POOL:
6075 		return igc_xdp_setup_pool(adapter, bpf->xsk.pool,
6076 					  bpf->xsk.queue_id);
6077 	default:
6078 		return -EOPNOTSUPP;
6079 	}
6080 }
6081 
6082 static int igc_xdp_xmit(struct net_device *dev, int num_frames,
6083 			struct xdp_frame **frames, u32 flags)
6084 {
6085 	struct igc_adapter *adapter = netdev_priv(dev);
6086 	int cpu = smp_processor_id();
6087 	struct netdev_queue *nq;
6088 	struct igc_ring *ring;
6089 	int i, drops;
6090 
6091 	if (unlikely(test_bit(__IGC_DOWN, &adapter->state)))
6092 		return -ENETDOWN;
6093 
6094 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6095 		return -EINVAL;
6096 
6097 	ring = igc_xdp_get_tx_ring(adapter, cpu);
6098 	nq = txring_txq(ring);
6099 
6100 	__netif_tx_lock(nq, cpu);
6101 
6102 	drops = 0;
6103 	for (i = 0; i < num_frames; i++) {
6104 		int err;
6105 		struct xdp_frame *xdpf = frames[i];
6106 
6107 		err = igc_xdp_init_tx_descriptor(ring, xdpf);
6108 		if (err) {
6109 			xdp_return_frame_rx_napi(xdpf);
6110 			drops++;
6111 		}
6112 	}
6113 
6114 	if (flags & XDP_XMIT_FLUSH)
6115 		igc_flush_tx_descriptors(ring);
6116 
6117 	__netif_tx_unlock(nq);
6118 
6119 	return num_frames - drops;
6120 }
6121 
6122 static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter,
6123 					struct igc_q_vector *q_vector)
6124 {
6125 	struct igc_hw *hw = &adapter->hw;
6126 	u32 eics = 0;
6127 
6128 	eics |= q_vector->eims_value;
6129 	wr32(IGC_EICS, eics);
6130 }
6131 
6132 int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
6133 {
6134 	struct igc_adapter *adapter = netdev_priv(dev);
6135 	struct igc_q_vector *q_vector;
6136 	struct igc_ring *ring;
6137 
6138 	if (test_bit(__IGC_DOWN, &adapter->state))
6139 		return -ENETDOWN;
6140 
6141 	if (!igc_xdp_is_enabled(adapter))
6142 		return -ENXIO;
6143 
6144 	if (queue_id >= adapter->num_rx_queues)
6145 		return -EINVAL;
6146 
6147 	ring = adapter->rx_ring[queue_id];
6148 
6149 	if (!ring->xsk_pool)
6150 		return -ENXIO;
6151 
6152 	q_vector = adapter->q_vector[queue_id];
6153 	if (!napi_if_scheduled_mark_missed(&q_vector->napi))
6154 		igc_trigger_rxtxq_interrupt(adapter, q_vector);
6155 
6156 	return 0;
6157 }
6158 
6159 static const struct net_device_ops igc_netdev_ops = {
6160 	.ndo_open		= igc_open,
6161 	.ndo_stop		= igc_close,
6162 	.ndo_start_xmit		= igc_xmit_frame,
6163 	.ndo_set_rx_mode	= igc_set_rx_mode,
6164 	.ndo_set_mac_address	= igc_set_mac,
6165 	.ndo_change_mtu		= igc_change_mtu,
6166 	.ndo_get_stats64	= igc_get_stats64,
6167 	.ndo_fix_features	= igc_fix_features,
6168 	.ndo_set_features	= igc_set_features,
6169 	.ndo_features_check	= igc_features_check,
6170 	.ndo_eth_ioctl		= igc_ioctl,
6171 	.ndo_setup_tc		= igc_setup_tc,
6172 	.ndo_bpf		= igc_bpf,
6173 	.ndo_xdp_xmit		= igc_xdp_xmit,
6174 	.ndo_xsk_wakeup		= igc_xsk_wakeup,
6175 };
6176 
6177 /* PCIe configuration access */
6178 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6179 {
6180 	struct igc_adapter *adapter = hw->back;
6181 
6182 	pci_read_config_word(adapter->pdev, reg, value);
6183 }
6184 
6185 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6186 {
6187 	struct igc_adapter *adapter = hw->back;
6188 
6189 	pci_write_config_word(adapter->pdev, reg, *value);
6190 }
6191 
6192 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6193 {
6194 	struct igc_adapter *adapter = hw->back;
6195 
6196 	if (!pci_is_pcie(adapter->pdev))
6197 		return -IGC_ERR_CONFIG;
6198 
6199 	pcie_capability_read_word(adapter->pdev, reg, value);
6200 
6201 	return IGC_SUCCESS;
6202 }
6203 
6204 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6205 {
6206 	struct igc_adapter *adapter = hw->back;
6207 
6208 	if (!pci_is_pcie(adapter->pdev))
6209 		return -IGC_ERR_CONFIG;
6210 
6211 	pcie_capability_write_word(adapter->pdev, reg, *value);
6212 
6213 	return IGC_SUCCESS;
6214 }
6215 
6216 u32 igc_rd32(struct igc_hw *hw, u32 reg)
6217 {
6218 	struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
6219 	u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
6220 	u32 value = 0;
6221 
6222 	if (IGC_REMOVED(hw_addr))
6223 		return ~value;
6224 
6225 	value = readl(&hw_addr[reg]);
6226 
6227 	/* reads should not return all F's */
6228 	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
6229 		struct net_device *netdev = igc->netdev;
6230 
6231 		hw->hw_addr = NULL;
6232 		netif_device_detach(netdev);
6233 		netdev_err(netdev, "PCIe link lost, device now detached\n");
6234 		WARN(pci_device_is_present(igc->pdev),
6235 		     "igc: Failed to read reg 0x%x!\n", reg);
6236 	}
6237 
6238 	return value;
6239 }
6240 
6241 /**
6242  * igc_probe - Device Initialization Routine
6243  * @pdev: PCI device information struct
6244  * @ent: entry in igc_pci_tbl
6245  *
6246  * Returns 0 on success, negative on failure
6247  *
6248  * igc_probe initializes an adapter identified by a pci_dev structure.
6249  * The OS initialization, configuring the adapter private structure,
6250  * and a hardware reset occur.
6251  */
6252 static int igc_probe(struct pci_dev *pdev,
6253 		     const struct pci_device_id *ent)
6254 {
6255 	struct igc_adapter *adapter;
6256 	struct net_device *netdev;
6257 	struct igc_hw *hw;
6258 	const struct igc_info *ei = igc_info_tbl[ent->driver_data];
6259 	int err;
6260 
6261 	err = pci_enable_device_mem(pdev);
6262 	if (err)
6263 		return err;
6264 
6265 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
6266 	if (err) {
6267 		dev_err(&pdev->dev,
6268 			"No usable DMA configuration, aborting\n");
6269 		goto err_dma;
6270 	}
6271 
6272 	err = pci_request_mem_regions(pdev, igc_driver_name);
6273 	if (err)
6274 		goto err_pci_reg;
6275 
6276 	pci_enable_pcie_error_reporting(pdev);
6277 
6278 	err = pci_enable_ptm(pdev, NULL);
6279 	if (err < 0)
6280 		dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
6281 
6282 	pci_set_master(pdev);
6283 
6284 	err = -ENOMEM;
6285 	netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
6286 				   IGC_MAX_TX_QUEUES);
6287 
6288 	if (!netdev)
6289 		goto err_alloc_etherdev;
6290 
6291 	SET_NETDEV_DEV(netdev, &pdev->dev);
6292 
6293 	pci_set_drvdata(pdev, netdev);
6294 	adapter = netdev_priv(netdev);
6295 	adapter->netdev = netdev;
6296 	adapter->pdev = pdev;
6297 	hw = &adapter->hw;
6298 	hw->back = adapter;
6299 	adapter->port_num = hw->bus.func;
6300 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6301 
6302 	err = pci_save_state(pdev);
6303 	if (err)
6304 		goto err_ioremap;
6305 
6306 	err = -EIO;
6307 	adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
6308 				   pci_resource_len(pdev, 0));
6309 	if (!adapter->io_addr)
6310 		goto err_ioremap;
6311 
6312 	/* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
6313 	hw->hw_addr = adapter->io_addr;
6314 
6315 	netdev->netdev_ops = &igc_netdev_ops;
6316 	igc_ethtool_set_ops(netdev);
6317 	netdev->watchdog_timeo = 5 * HZ;
6318 
6319 	netdev->mem_start = pci_resource_start(pdev, 0);
6320 	netdev->mem_end = pci_resource_end(pdev, 0);
6321 
6322 	/* PCI config space info */
6323 	hw->vendor_id = pdev->vendor;
6324 	hw->device_id = pdev->device;
6325 	hw->revision_id = pdev->revision;
6326 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
6327 	hw->subsystem_device_id = pdev->subsystem_device;
6328 
6329 	/* Copy the default MAC and PHY function pointers */
6330 	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6331 	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6332 
6333 	/* Initialize skew-specific constants */
6334 	err = ei->get_invariants(hw);
6335 	if (err)
6336 		goto err_sw_init;
6337 
6338 	/* Add supported features to the features list*/
6339 	netdev->features |= NETIF_F_SG;
6340 	netdev->features |= NETIF_F_TSO;
6341 	netdev->features |= NETIF_F_TSO6;
6342 	netdev->features |= NETIF_F_TSO_ECN;
6343 	netdev->features |= NETIF_F_RXCSUM;
6344 	netdev->features |= NETIF_F_HW_CSUM;
6345 	netdev->features |= NETIF_F_SCTP_CRC;
6346 	netdev->features |= NETIF_F_HW_TC;
6347 
6348 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
6349 				  NETIF_F_GSO_GRE_CSUM | \
6350 				  NETIF_F_GSO_IPXIP4 | \
6351 				  NETIF_F_GSO_IPXIP6 | \
6352 				  NETIF_F_GSO_UDP_TUNNEL | \
6353 				  NETIF_F_GSO_UDP_TUNNEL_CSUM)
6354 
6355 	netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
6356 	netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
6357 
6358 	/* setup the private structure */
6359 	err = igc_sw_init(adapter);
6360 	if (err)
6361 		goto err_sw_init;
6362 
6363 	/* copy netdev features into list of user selectable features */
6364 	netdev->hw_features |= NETIF_F_NTUPLE;
6365 	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
6366 	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
6367 	netdev->hw_features |= netdev->features;
6368 
6369 	netdev->features |= NETIF_F_HIGHDMA;
6370 
6371 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
6372 	netdev->mpls_features |= NETIF_F_HW_CSUM;
6373 	netdev->hw_enc_features |= netdev->vlan_features;
6374 
6375 	/* MTU range: 68 - 9216 */
6376 	netdev->min_mtu = ETH_MIN_MTU;
6377 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
6378 
6379 	/* before reading the NVM, reset the controller to put the device in a
6380 	 * known good starting state
6381 	 */
6382 	hw->mac.ops.reset_hw(hw);
6383 
6384 	if (igc_get_flash_presence_i225(hw)) {
6385 		if (hw->nvm.ops.validate(hw) < 0) {
6386 			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
6387 			err = -EIO;
6388 			goto err_eeprom;
6389 		}
6390 	}
6391 
6392 	if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
6393 		/* copy the MAC address out of the NVM */
6394 		if (hw->mac.ops.read_mac_addr(hw))
6395 			dev_err(&pdev->dev, "NVM Read Error\n");
6396 	}
6397 
6398 	eth_hw_addr_set(netdev, hw->mac.addr);
6399 
6400 	if (!is_valid_ether_addr(netdev->dev_addr)) {
6401 		dev_err(&pdev->dev, "Invalid MAC Address\n");
6402 		err = -EIO;
6403 		goto err_eeprom;
6404 	}
6405 
6406 	/* configure RXPBSIZE and TXPBSIZE */
6407 	wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
6408 	wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
6409 
6410 	timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
6411 	timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
6412 
6413 	INIT_WORK(&adapter->reset_task, igc_reset_task);
6414 	INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
6415 
6416 	/* Initialize link properties that are user-changeable */
6417 	adapter->fc_autoneg = true;
6418 	hw->mac.autoneg = true;
6419 	hw->phy.autoneg_advertised = 0xaf;
6420 
6421 	hw->fc.requested_mode = igc_fc_default;
6422 	hw->fc.current_mode = igc_fc_default;
6423 
6424 	/* By default, support wake on port A */
6425 	adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
6426 
6427 	/* initialize the wol settings based on the eeprom settings */
6428 	if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
6429 		adapter->wol |= IGC_WUFC_MAG;
6430 
6431 	device_set_wakeup_enable(&adapter->pdev->dev,
6432 				 adapter->flags & IGC_FLAG_WOL_SUPPORTED);
6433 
6434 	igc_ptp_init(adapter);
6435 
6436 	igc_tsn_clear_schedule(adapter);
6437 
6438 	/* reset the hardware with the new settings */
6439 	igc_reset(adapter);
6440 
6441 	/* let the f/w know that the h/w is now under the control of the
6442 	 * driver.
6443 	 */
6444 	igc_get_hw_control(adapter);
6445 
6446 	strncpy(netdev->name, "eth%d", IFNAMSIZ);
6447 	err = register_netdev(netdev);
6448 	if (err)
6449 		goto err_register;
6450 
6451 	 /* carrier off reporting is important to ethtool even BEFORE open */
6452 	netif_carrier_off(netdev);
6453 
6454 	/* Check if Media Autosense is enabled */
6455 	adapter->ei = *ei;
6456 
6457 	/* print pcie link status and MAC address */
6458 	pcie_print_link_status(pdev);
6459 	netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
6460 
6461 	dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
6462 	/* Disable EEE for internal PHY devices */
6463 	hw->dev_spec._base.eee_enable = false;
6464 	adapter->flags &= ~IGC_FLAG_EEE;
6465 	igc_set_eee_i225(hw, false, false, false);
6466 
6467 	pm_runtime_put_noidle(&pdev->dev);
6468 
6469 	return 0;
6470 
6471 err_register:
6472 	igc_release_hw_control(adapter);
6473 err_eeprom:
6474 	if (!igc_check_reset_block(hw))
6475 		igc_reset_phy(hw);
6476 err_sw_init:
6477 	igc_clear_interrupt_scheme(adapter);
6478 	iounmap(adapter->io_addr);
6479 err_ioremap:
6480 	free_netdev(netdev);
6481 err_alloc_etherdev:
6482 	pci_disable_pcie_error_reporting(pdev);
6483 	pci_release_mem_regions(pdev);
6484 err_pci_reg:
6485 err_dma:
6486 	pci_disable_device(pdev);
6487 	return err;
6488 }
6489 
6490 /**
6491  * igc_remove - Device Removal Routine
6492  * @pdev: PCI device information struct
6493  *
6494  * igc_remove is called by the PCI subsystem to alert the driver
6495  * that it should release a PCI device.  This could be caused by a
6496  * Hot-Plug event, or because the driver is going to be removed from
6497  * memory.
6498  */
6499 static void igc_remove(struct pci_dev *pdev)
6500 {
6501 	struct net_device *netdev = pci_get_drvdata(pdev);
6502 	struct igc_adapter *adapter = netdev_priv(netdev);
6503 
6504 	pm_runtime_get_noresume(&pdev->dev);
6505 
6506 	igc_flush_nfc_rules(adapter);
6507 
6508 	igc_ptp_stop(adapter);
6509 
6510 	set_bit(__IGC_DOWN, &adapter->state);
6511 
6512 	del_timer_sync(&adapter->watchdog_timer);
6513 	del_timer_sync(&adapter->phy_info_timer);
6514 
6515 	cancel_work_sync(&adapter->reset_task);
6516 	cancel_work_sync(&adapter->watchdog_task);
6517 
6518 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
6519 	 * would have already happened in close and is redundant.
6520 	 */
6521 	igc_release_hw_control(adapter);
6522 	unregister_netdev(netdev);
6523 
6524 	igc_clear_interrupt_scheme(adapter);
6525 	pci_iounmap(pdev, adapter->io_addr);
6526 	pci_release_mem_regions(pdev);
6527 
6528 	free_netdev(netdev);
6529 
6530 	pci_disable_pcie_error_reporting(pdev);
6531 
6532 	pci_disable_device(pdev);
6533 }
6534 
6535 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
6536 			  bool runtime)
6537 {
6538 	struct net_device *netdev = pci_get_drvdata(pdev);
6539 	struct igc_adapter *adapter = netdev_priv(netdev);
6540 	u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
6541 	struct igc_hw *hw = &adapter->hw;
6542 	u32 ctrl, rctl, status;
6543 	bool wake;
6544 
6545 	rtnl_lock();
6546 	netif_device_detach(netdev);
6547 
6548 	if (netif_running(netdev))
6549 		__igc_close(netdev, true);
6550 
6551 	igc_ptp_suspend(adapter);
6552 
6553 	igc_clear_interrupt_scheme(adapter);
6554 	rtnl_unlock();
6555 
6556 	status = rd32(IGC_STATUS);
6557 	if (status & IGC_STATUS_LU)
6558 		wufc &= ~IGC_WUFC_LNKC;
6559 
6560 	if (wufc) {
6561 		igc_setup_rctl(adapter);
6562 		igc_set_rx_mode(netdev);
6563 
6564 		/* turn on all-multi mode if wake on multicast is enabled */
6565 		if (wufc & IGC_WUFC_MC) {
6566 			rctl = rd32(IGC_RCTL);
6567 			rctl |= IGC_RCTL_MPE;
6568 			wr32(IGC_RCTL, rctl);
6569 		}
6570 
6571 		ctrl = rd32(IGC_CTRL);
6572 		ctrl |= IGC_CTRL_ADVD3WUC;
6573 		wr32(IGC_CTRL, ctrl);
6574 
6575 		/* Allow time for pending master requests to run */
6576 		igc_disable_pcie_master(hw);
6577 
6578 		wr32(IGC_WUC, IGC_WUC_PME_EN);
6579 		wr32(IGC_WUFC, wufc);
6580 	} else {
6581 		wr32(IGC_WUC, 0);
6582 		wr32(IGC_WUFC, 0);
6583 	}
6584 
6585 	wake = wufc || adapter->en_mng_pt;
6586 	if (!wake)
6587 		igc_power_down_phy_copper_base(&adapter->hw);
6588 	else
6589 		igc_power_up_link(adapter);
6590 
6591 	if (enable_wake)
6592 		*enable_wake = wake;
6593 
6594 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
6595 	 * would have already happened in close and is redundant.
6596 	 */
6597 	igc_release_hw_control(adapter);
6598 
6599 	pci_disable_device(pdev);
6600 
6601 	return 0;
6602 }
6603 
6604 #ifdef CONFIG_PM
6605 static int __maybe_unused igc_runtime_suspend(struct device *dev)
6606 {
6607 	return __igc_shutdown(to_pci_dev(dev), NULL, 1);
6608 }
6609 
6610 static void igc_deliver_wake_packet(struct net_device *netdev)
6611 {
6612 	struct igc_adapter *adapter = netdev_priv(netdev);
6613 	struct igc_hw *hw = &adapter->hw;
6614 	struct sk_buff *skb;
6615 	u32 wupl;
6616 
6617 	wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
6618 
6619 	/* WUPM stores only the first 128 bytes of the wake packet.
6620 	 * Read the packet only if we have the whole thing.
6621 	 */
6622 	if (wupl == 0 || wupl > IGC_WUPM_BYTES)
6623 		return;
6624 
6625 	skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
6626 	if (!skb)
6627 		return;
6628 
6629 	skb_put(skb, wupl);
6630 
6631 	/* Ensure reads are 32-bit aligned */
6632 	wupl = roundup(wupl, 4);
6633 
6634 	memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
6635 
6636 	skb->protocol = eth_type_trans(skb, netdev);
6637 	netif_rx(skb);
6638 }
6639 
6640 static int __maybe_unused igc_resume(struct device *dev)
6641 {
6642 	struct pci_dev *pdev = to_pci_dev(dev);
6643 	struct net_device *netdev = pci_get_drvdata(pdev);
6644 	struct igc_adapter *adapter = netdev_priv(netdev);
6645 	struct igc_hw *hw = &adapter->hw;
6646 	u32 err, val;
6647 
6648 	pci_set_power_state(pdev, PCI_D0);
6649 	pci_restore_state(pdev);
6650 	pci_save_state(pdev);
6651 
6652 	if (!pci_device_is_present(pdev))
6653 		return -ENODEV;
6654 	err = pci_enable_device_mem(pdev);
6655 	if (err) {
6656 		netdev_err(netdev, "Cannot enable PCI device from suspend\n");
6657 		return err;
6658 	}
6659 	pci_set_master(pdev);
6660 
6661 	pci_enable_wake(pdev, PCI_D3hot, 0);
6662 	pci_enable_wake(pdev, PCI_D3cold, 0);
6663 
6664 	if (igc_init_interrupt_scheme(adapter, true)) {
6665 		netdev_err(netdev, "Unable to allocate memory for queues\n");
6666 		return -ENOMEM;
6667 	}
6668 
6669 	igc_reset(adapter);
6670 
6671 	/* let the f/w know that the h/w is now under the control of the
6672 	 * driver.
6673 	 */
6674 	igc_get_hw_control(adapter);
6675 
6676 	val = rd32(IGC_WUS);
6677 	if (val & WAKE_PKT_WUS)
6678 		igc_deliver_wake_packet(netdev);
6679 
6680 	wr32(IGC_WUS, ~0);
6681 
6682 	rtnl_lock();
6683 	if (!err && netif_running(netdev))
6684 		err = __igc_open(netdev, true);
6685 
6686 	if (!err)
6687 		netif_device_attach(netdev);
6688 	rtnl_unlock();
6689 
6690 	return err;
6691 }
6692 
6693 static int __maybe_unused igc_runtime_resume(struct device *dev)
6694 {
6695 	return igc_resume(dev);
6696 }
6697 
6698 static int __maybe_unused igc_suspend(struct device *dev)
6699 {
6700 	return __igc_shutdown(to_pci_dev(dev), NULL, 0);
6701 }
6702 
6703 static int __maybe_unused igc_runtime_idle(struct device *dev)
6704 {
6705 	struct net_device *netdev = dev_get_drvdata(dev);
6706 	struct igc_adapter *adapter = netdev_priv(netdev);
6707 
6708 	if (!igc_has_link(adapter))
6709 		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6710 
6711 	return -EBUSY;
6712 }
6713 #endif /* CONFIG_PM */
6714 
6715 static void igc_shutdown(struct pci_dev *pdev)
6716 {
6717 	bool wake;
6718 
6719 	__igc_shutdown(pdev, &wake, 0);
6720 
6721 	if (system_state == SYSTEM_POWER_OFF) {
6722 		pci_wake_from_d3(pdev, wake);
6723 		pci_set_power_state(pdev, PCI_D3hot);
6724 	}
6725 }
6726 
6727 /**
6728  *  igc_io_error_detected - called when PCI error is detected
6729  *  @pdev: Pointer to PCI device
6730  *  @state: The current PCI connection state
6731  *
6732  *  This function is called after a PCI bus error affecting
6733  *  this device has been detected.
6734  **/
6735 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
6736 					      pci_channel_state_t state)
6737 {
6738 	struct net_device *netdev = pci_get_drvdata(pdev);
6739 	struct igc_adapter *adapter = netdev_priv(netdev);
6740 
6741 	netif_device_detach(netdev);
6742 
6743 	if (state == pci_channel_io_perm_failure)
6744 		return PCI_ERS_RESULT_DISCONNECT;
6745 
6746 	if (netif_running(netdev))
6747 		igc_down(adapter);
6748 	pci_disable_device(pdev);
6749 
6750 	/* Request a slot reset. */
6751 	return PCI_ERS_RESULT_NEED_RESET;
6752 }
6753 
6754 /**
6755  *  igc_io_slot_reset - called after the PCI bus has been reset.
6756  *  @pdev: Pointer to PCI device
6757  *
6758  *  Restart the card from scratch, as if from a cold-boot. Implementation
6759  *  resembles the first-half of the igc_resume routine.
6760  **/
6761 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
6762 {
6763 	struct net_device *netdev = pci_get_drvdata(pdev);
6764 	struct igc_adapter *adapter = netdev_priv(netdev);
6765 	struct igc_hw *hw = &adapter->hw;
6766 	pci_ers_result_t result;
6767 
6768 	if (pci_enable_device_mem(pdev)) {
6769 		netdev_err(netdev, "Could not re-enable PCI device after reset\n");
6770 		result = PCI_ERS_RESULT_DISCONNECT;
6771 	} else {
6772 		pci_set_master(pdev);
6773 		pci_restore_state(pdev);
6774 		pci_save_state(pdev);
6775 
6776 		pci_enable_wake(pdev, PCI_D3hot, 0);
6777 		pci_enable_wake(pdev, PCI_D3cold, 0);
6778 
6779 		/* In case of PCI error, adapter loses its HW address
6780 		 * so we should re-assign it here.
6781 		 */
6782 		hw->hw_addr = adapter->io_addr;
6783 
6784 		igc_reset(adapter);
6785 		wr32(IGC_WUS, ~0);
6786 		result = PCI_ERS_RESULT_RECOVERED;
6787 	}
6788 
6789 	return result;
6790 }
6791 
6792 /**
6793  *  igc_io_resume - called when traffic can start to flow again.
6794  *  @pdev: Pointer to PCI device
6795  *
6796  *  This callback is called when the error recovery driver tells us that
6797  *  its OK to resume normal operation. Implementation resembles the
6798  *  second-half of the igc_resume routine.
6799  */
6800 static void igc_io_resume(struct pci_dev *pdev)
6801 {
6802 	struct net_device *netdev = pci_get_drvdata(pdev);
6803 	struct igc_adapter *adapter = netdev_priv(netdev);
6804 
6805 	rtnl_lock();
6806 	if (netif_running(netdev)) {
6807 		if (igc_open(netdev)) {
6808 			netdev_err(netdev, "igc_open failed after reset\n");
6809 			return;
6810 		}
6811 	}
6812 
6813 	netif_device_attach(netdev);
6814 
6815 	/* let the f/w know that the h/w is now under the control of the
6816 	 * driver.
6817 	 */
6818 	igc_get_hw_control(adapter);
6819 	rtnl_unlock();
6820 }
6821 
6822 static const struct pci_error_handlers igc_err_handler = {
6823 	.error_detected = igc_io_error_detected,
6824 	.slot_reset = igc_io_slot_reset,
6825 	.resume = igc_io_resume,
6826 };
6827 
6828 #ifdef CONFIG_PM
6829 static const struct dev_pm_ops igc_pm_ops = {
6830 	SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume)
6831 	SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume,
6832 			   igc_runtime_idle)
6833 };
6834 #endif
6835 
6836 static struct pci_driver igc_driver = {
6837 	.name     = igc_driver_name,
6838 	.id_table = igc_pci_tbl,
6839 	.probe    = igc_probe,
6840 	.remove   = igc_remove,
6841 #ifdef CONFIG_PM
6842 	.driver.pm = &igc_pm_ops,
6843 #endif
6844 	.shutdown = igc_shutdown,
6845 	.err_handler = &igc_err_handler,
6846 };
6847 
6848 /**
6849  * igc_reinit_queues - return error
6850  * @adapter: pointer to adapter structure
6851  */
6852 int igc_reinit_queues(struct igc_adapter *adapter)
6853 {
6854 	struct net_device *netdev = adapter->netdev;
6855 	int err = 0;
6856 
6857 	if (netif_running(netdev))
6858 		igc_close(netdev);
6859 
6860 	igc_reset_interrupt_capability(adapter);
6861 
6862 	if (igc_init_interrupt_scheme(adapter, true)) {
6863 		netdev_err(netdev, "Unable to allocate memory for queues\n");
6864 		return -ENOMEM;
6865 	}
6866 
6867 	if (netif_running(netdev))
6868 		err = igc_open(netdev);
6869 
6870 	return err;
6871 }
6872 
6873 /**
6874  * igc_get_hw_dev - return device
6875  * @hw: pointer to hardware structure
6876  *
6877  * used by hardware layer to print debugging information
6878  */
6879 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
6880 {
6881 	struct igc_adapter *adapter = hw->back;
6882 
6883 	return adapter->netdev;
6884 }
6885 
6886 static void igc_disable_rx_ring_hw(struct igc_ring *ring)
6887 {
6888 	struct igc_hw *hw = &ring->q_vector->adapter->hw;
6889 	u8 idx = ring->reg_idx;
6890 	u32 rxdctl;
6891 
6892 	rxdctl = rd32(IGC_RXDCTL(idx));
6893 	rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE;
6894 	rxdctl |= IGC_RXDCTL_SWFLUSH;
6895 	wr32(IGC_RXDCTL(idx), rxdctl);
6896 }
6897 
6898 void igc_disable_rx_ring(struct igc_ring *ring)
6899 {
6900 	igc_disable_rx_ring_hw(ring);
6901 	igc_clean_rx_ring(ring);
6902 }
6903 
6904 void igc_enable_rx_ring(struct igc_ring *ring)
6905 {
6906 	struct igc_adapter *adapter = ring->q_vector->adapter;
6907 
6908 	igc_configure_rx_ring(adapter, ring);
6909 
6910 	if (ring->xsk_pool)
6911 		igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
6912 	else
6913 		igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
6914 }
6915 
6916 static void igc_disable_tx_ring_hw(struct igc_ring *ring)
6917 {
6918 	struct igc_hw *hw = &ring->q_vector->adapter->hw;
6919 	u8 idx = ring->reg_idx;
6920 	u32 txdctl;
6921 
6922 	txdctl = rd32(IGC_TXDCTL(idx));
6923 	txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE;
6924 	txdctl |= IGC_TXDCTL_SWFLUSH;
6925 	wr32(IGC_TXDCTL(idx), txdctl);
6926 }
6927 
6928 void igc_disable_tx_ring(struct igc_ring *ring)
6929 {
6930 	igc_disable_tx_ring_hw(ring);
6931 	igc_clean_tx_ring(ring);
6932 }
6933 
6934 void igc_enable_tx_ring(struct igc_ring *ring)
6935 {
6936 	struct igc_adapter *adapter = ring->q_vector->adapter;
6937 
6938 	igc_configure_tx_ring(adapter, ring);
6939 }
6940 
6941 /**
6942  * igc_init_module - Driver Registration Routine
6943  *
6944  * igc_init_module is the first routine called when the driver is
6945  * loaded. All it does is register with the PCI subsystem.
6946  */
6947 static int __init igc_init_module(void)
6948 {
6949 	int ret;
6950 
6951 	pr_info("%s\n", igc_driver_string);
6952 	pr_info("%s\n", igc_copyright);
6953 
6954 	ret = pci_register_driver(&igc_driver);
6955 	return ret;
6956 }
6957 
6958 module_init(igc_init_module);
6959 
6960 /**
6961  * igc_exit_module - Driver Exit Cleanup Routine
6962  *
6963  * igc_exit_module is called just before the driver is removed
6964  * from memory.
6965  */
6966 static void __exit igc_exit_module(void)
6967 {
6968 	pci_unregister_driver(&igc_driver);
6969 }
6970 
6971 module_exit(igc_exit_module);
6972 /* igc_main.c */
6973