1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c)  2018 Intel Corporation */
3 
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
7 #include <linux/tcp.h>
8 #include <linux/udp.h>
9 #include <linux/ip.h>
10 #include <linux/pm_runtime.h>
11 #include <net/pkt_sched.h>
12 #include <linux/bpf_trace.h>
13 #include <net/xdp_sock_drv.h>
14 #include <linux/pci.h>
15 
16 #include <net/ipv6.h>
17 
18 #include "igc.h"
19 #include "igc_hw.h"
20 #include "igc_tsn.h"
21 #include "igc_xdp.h"
22 
23 #define DRV_SUMMARY	"Intel(R) 2.5G Ethernet Linux Driver"
24 
25 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
26 
27 #define IGC_XDP_PASS		0
28 #define IGC_XDP_CONSUMED	BIT(0)
29 #define IGC_XDP_TX		BIT(1)
30 #define IGC_XDP_REDIRECT	BIT(2)
31 
32 static int debug = -1;
33 
34 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
35 MODULE_DESCRIPTION(DRV_SUMMARY);
36 MODULE_LICENSE("GPL v2");
37 module_param(debug, int, 0);
38 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
39 
40 char igc_driver_name[] = "igc";
41 static const char igc_driver_string[] = DRV_SUMMARY;
42 static const char igc_copyright[] =
43 	"Copyright(c) 2018 Intel Corporation.";
44 
45 static const struct igc_info *igc_info_tbl[] = {
46 	[board_base] = &igc_base_info,
47 };
48 
49 static const struct pci_device_id igc_pci_tbl[] = {
50 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
51 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
52 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
53 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
54 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
55 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
56 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
57 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
58 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
59 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
60 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
61 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
62 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
63 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
64 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
65 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
66 	/* required last entry */
67 	{0, }
68 };
69 
70 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
71 
72 enum latency_range {
73 	lowest_latency = 0,
74 	low_latency = 1,
75 	bulk_latency = 2,
76 	latency_invalid = 255
77 };
78 
igc_reset(struct igc_adapter * adapter)79 void igc_reset(struct igc_adapter *adapter)
80 {
81 	struct net_device *dev = adapter->netdev;
82 	struct igc_hw *hw = &adapter->hw;
83 	struct igc_fc_info *fc = &hw->fc;
84 	u32 pba, hwm;
85 
86 	/* Repartition PBA for greater than 9k MTU if required */
87 	pba = IGC_PBA_34K;
88 
89 	/* flow control settings
90 	 * The high water mark must be low enough to fit one full frame
91 	 * after transmitting the pause frame.  As such we must have enough
92 	 * space to allow for us to complete our current transmit and then
93 	 * receive the frame that is in progress from the link partner.
94 	 * Set it to:
95 	 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
96 	 */
97 	hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
98 
99 	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
100 	fc->low_water = fc->high_water - 16;
101 	fc->pause_time = 0xFFFF;
102 	fc->send_xon = 1;
103 	fc->current_mode = fc->requested_mode;
104 
105 	hw->mac.ops.reset_hw(hw);
106 
107 	if (hw->mac.ops.init_hw(hw))
108 		netdev_err(dev, "Error on hardware initialization\n");
109 
110 	/* Re-establish EEE setting */
111 	igc_set_eee_i225(hw, true, true, true);
112 
113 	if (!netif_running(adapter->netdev))
114 		igc_power_down_phy_copper_base(&adapter->hw);
115 
116 	/* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
117 	wr32(IGC_VET, ETH_P_8021Q);
118 
119 	/* Re-enable PTP, where applicable. */
120 	igc_ptp_reset(adapter);
121 
122 	/* Re-enable TSN offloading, where applicable. */
123 	igc_tsn_reset(adapter);
124 
125 	igc_get_phy_info(hw);
126 }
127 
128 /**
129  * igc_power_up_link - Power up the phy link
130  * @adapter: address of board private structure
131  */
igc_power_up_link(struct igc_adapter * adapter)132 static void igc_power_up_link(struct igc_adapter *adapter)
133 {
134 	igc_reset_phy(&adapter->hw);
135 
136 	igc_power_up_phy_copper(&adapter->hw);
137 
138 	igc_setup_link(&adapter->hw);
139 }
140 
141 /**
142  * igc_release_hw_control - release control of the h/w to f/w
143  * @adapter: address of board private structure
144  *
145  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
146  * For ASF and Pass Through versions of f/w this means that the
147  * driver is no longer loaded.
148  */
igc_release_hw_control(struct igc_adapter * adapter)149 static void igc_release_hw_control(struct igc_adapter *adapter)
150 {
151 	struct igc_hw *hw = &adapter->hw;
152 	u32 ctrl_ext;
153 
154 	if (!pci_device_is_present(adapter->pdev))
155 		return;
156 
157 	/* Let firmware take over control of h/w */
158 	ctrl_ext = rd32(IGC_CTRL_EXT);
159 	wr32(IGC_CTRL_EXT,
160 	     ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
161 }
162 
163 /**
164  * igc_get_hw_control - get control of the h/w from f/w
165  * @adapter: address of board private structure
166  *
167  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
168  * For ASF and Pass Through versions of f/w this means that
169  * the driver is loaded.
170  */
igc_get_hw_control(struct igc_adapter * adapter)171 static void igc_get_hw_control(struct igc_adapter *adapter)
172 {
173 	struct igc_hw *hw = &adapter->hw;
174 	u32 ctrl_ext;
175 
176 	/* Let firmware know the driver has taken over */
177 	ctrl_ext = rd32(IGC_CTRL_EXT);
178 	wr32(IGC_CTRL_EXT,
179 	     ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
180 }
181 
igc_unmap_tx_buffer(struct device * dev,struct igc_tx_buffer * buf)182 static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf)
183 {
184 	dma_unmap_single(dev, dma_unmap_addr(buf, dma),
185 			 dma_unmap_len(buf, len), DMA_TO_DEVICE);
186 
187 	dma_unmap_len_set(buf, len, 0);
188 }
189 
190 /**
191  * igc_clean_tx_ring - Free Tx Buffers
192  * @tx_ring: ring to be cleaned
193  */
igc_clean_tx_ring(struct igc_ring * tx_ring)194 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
195 {
196 	u16 i = tx_ring->next_to_clean;
197 	struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
198 	u32 xsk_frames = 0;
199 
200 	while (i != tx_ring->next_to_use) {
201 		union igc_adv_tx_desc *eop_desc, *tx_desc;
202 
203 		switch (tx_buffer->type) {
204 		case IGC_TX_BUFFER_TYPE_XSK:
205 			xsk_frames++;
206 			break;
207 		case IGC_TX_BUFFER_TYPE_XDP:
208 			xdp_return_frame(tx_buffer->xdpf);
209 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
210 			break;
211 		case IGC_TX_BUFFER_TYPE_SKB:
212 			dev_kfree_skb_any(tx_buffer->skb);
213 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
214 			break;
215 		default:
216 			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
217 			break;
218 		}
219 
220 		/* check for eop_desc to determine the end of the packet */
221 		eop_desc = tx_buffer->next_to_watch;
222 		tx_desc = IGC_TX_DESC(tx_ring, i);
223 
224 		/* unmap remaining buffers */
225 		while (tx_desc != eop_desc) {
226 			tx_buffer++;
227 			tx_desc++;
228 			i++;
229 			if (unlikely(i == tx_ring->count)) {
230 				i = 0;
231 				tx_buffer = tx_ring->tx_buffer_info;
232 				tx_desc = IGC_TX_DESC(tx_ring, 0);
233 			}
234 
235 			/* unmap any remaining paged data */
236 			if (dma_unmap_len(tx_buffer, len))
237 				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
238 		}
239 
240 		tx_buffer->next_to_watch = NULL;
241 
242 		/* move us one more past the eop_desc for start of next pkt */
243 		tx_buffer++;
244 		i++;
245 		if (unlikely(i == tx_ring->count)) {
246 			i = 0;
247 			tx_buffer = tx_ring->tx_buffer_info;
248 		}
249 	}
250 
251 	if (tx_ring->xsk_pool && xsk_frames)
252 		xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
253 
254 	/* reset BQL for queue */
255 	netdev_tx_reset_queue(txring_txq(tx_ring));
256 
257 	/* Zero out the buffer ring */
258 	memset(tx_ring->tx_buffer_info, 0,
259 	       sizeof(*tx_ring->tx_buffer_info) * tx_ring->count);
260 
261 	/* Zero out the descriptor ring */
262 	memset(tx_ring->desc, 0, tx_ring->size);
263 
264 	/* reset next_to_use and next_to_clean */
265 	tx_ring->next_to_use = 0;
266 	tx_ring->next_to_clean = 0;
267 }
268 
269 /**
270  * igc_free_tx_resources - Free Tx Resources per Queue
271  * @tx_ring: Tx descriptor ring for a specific queue
272  *
273  * Free all transmit software resources
274  */
igc_free_tx_resources(struct igc_ring * tx_ring)275 void igc_free_tx_resources(struct igc_ring *tx_ring)
276 {
277 	igc_disable_tx_ring(tx_ring);
278 
279 	vfree(tx_ring->tx_buffer_info);
280 	tx_ring->tx_buffer_info = NULL;
281 
282 	/* if not set, then don't free */
283 	if (!tx_ring->desc)
284 		return;
285 
286 	dma_free_coherent(tx_ring->dev, tx_ring->size,
287 			  tx_ring->desc, tx_ring->dma);
288 
289 	tx_ring->desc = NULL;
290 }
291 
292 /**
293  * igc_free_all_tx_resources - Free Tx Resources for All Queues
294  * @adapter: board private structure
295  *
296  * Free all transmit software resources
297  */
igc_free_all_tx_resources(struct igc_adapter * adapter)298 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
299 {
300 	int i;
301 
302 	for (i = 0; i < adapter->num_tx_queues; i++)
303 		igc_free_tx_resources(adapter->tx_ring[i]);
304 }
305 
306 /**
307  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
308  * @adapter: board private structure
309  */
igc_clean_all_tx_rings(struct igc_adapter * adapter)310 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
311 {
312 	int i;
313 
314 	for (i = 0; i < adapter->num_tx_queues; i++)
315 		if (adapter->tx_ring[i])
316 			igc_clean_tx_ring(adapter->tx_ring[i]);
317 }
318 
igc_disable_tx_ring_hw(struct igc_ring * ring)319 static void igc_disable_tx_ring_hw(struct igc_ring *ring)
320 {
321 	struct igc_hw *hw = &ring->q_vector->adapter->hw;
322 	u8 idx = ring->reg_idx;
323 	u32 txdctl;
324 
325 	txdctl = rd32(IGC_TXDCTL(idx));
326 	txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE;
327 	txdctl |= IGC_TXDCTL_SWFLUSH;
328 	wr32(IGC_TXDCTL(idx), txdctl);
329 }
330 
331 /**
332  * igc_disable_all_tx_rings_hw - Disable all transmit queue operation
333  * @adapter: board private structure
334  */
igc_disable_all_tx_rings_hw(struct igc_adapter * adapter)335 static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter)
336 {
337 	int i;
338 
339 	for (i = 0; i < adapter->num_tx_queues; i++) {
340 		struct igc_ring *tx_ring = adapter->tx_ring[i];
341 
342 		igc_disable_tx_ring_hw(tx_ring);
343 	}
344 }
345 
346 /**
347  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
348  * @tx_ring: tx descriptor ring (for a specific queue) to setup
349  *
350  * Return 0 on success, negative on failure
351  */
igc_setup_tx_resources(struct igc_ring * tx_ring)352 int igc_setup_tx_resources(struct igc_ring *tx_ring)
353 {
354 	struct net_device *ndev = tx_ring->netdev;
355 	struct device *dev = tx_ring->dev;
356 	int size = 0;
357 
358 	size = sizeof(struct igc_tx_buffer) * tx_ring->count;
359 	tx_ring->tx_buffer_info = vzalloc(size);
360 	if (!tx_ring->tx_buffer_info)
361 		goto err;
362 
363 	/* round up to nearest 4K */
364 	tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
365 	tx_ring->size = ALIGN(tx_ring->size, 4096);
366 
367 	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
368 					   &tx_ring->dma, GFP_KERNEL);
369 
370 	if (!tx_ring->desc)
371 		goto err;
372 
373 	tx_ring->next_to_use = 0;
374 	tx_ring->next_to_clean = 0;
375 
376 	return 0;
377 
378 err:
379 	vfree(tx_ring->tx_buffer_info);
380 	netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
381 	return -ENOMEM;
382 }
383 
384 /**
385  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
386  * @adapter: board private structure
387  *
388  * Return 0 on success, negative on failure
389  */
igc_setup_all_tx_resources(struct igc_adapter * adapter)390 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
391 {
392 	struct net_device *dev = adapter->netdev;
393 	int i, err = 0;
394 
395 	for (i = 0; i < adapter->num_tx_queues; i++) {
396 		err = igc_setup_tx_resources(adapter->tx_ring[i]);
397 		if (err) {
398 			netdev_err(dev, "Error on Tx queue %u setup\n", i);
399 			for (i--; i >= 0; i--)
400 				igc_free_tx_resources(adapter->tx_ring[i]);
401 			break;
402 		}
403 	}
404 
405 	return err;
406 }
407 
igc_clean_rx_ring_page_shared(struct igc_ring * rx_ring)408 static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring)
409 {
410 	u16 i = rx_ring->next_to_clean;
411 
412 	dev_kfree_skb(rx_ring->skb);
413 	rx_ring->skb = NULL;
414 
415 	/* Free all the Rx ring sk_buffs */
416 	while (i != rx_ring->next_to_alloc) {
417 		struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
418 
419 		/* Invalidate cache lines that may have been written to by
420 		 * device so that we avoid corrupting memory.
421 		 */
422 		dma_sync_single_range_for_cpu(rx_ring->dev,
423 					      buffer_info->dma,
424 					      buffer_info->page_offset,
425 					      igc_rx_bufsz(rx_ring),
426 					      DMA_FROM_DEVICE);
427 
428 		/* free resources associated with mapping */
429 		dma_unmap_page_attrs(rx_ring->dev,
430 				     buffer_info->dma,
431 				     igc_rx_pg_size(rx_ring),
432 				     DMA_FROM_DEVICE,
433 				     IGC_RX_DMA_ATTR);
434 		__page_frag_cache_drain(buffer_info->page,
435 					buffer_info->pagecnt_bias);
436 
437 		i++;
438 		if (i == rx_ring->count)
439 			i = 0;
440 	}
441 }
442 
igc_clean_rx_ring_xsk_pool(struct igc_ring * ring)443 static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring)
444 {
445 	struct igc_rx_buffer *bi;
446 	u16 i;
447 
448 	for (i = 0; i < ring->count; i++) {
449 		bi = &ring->rx_buffer_info[i];
450 		if (!bi->xdp)
451 			continue;
452 
453 		xsk_buff_free(bi->xdp);
454 		bi->xdp = NULL;
455 	}
456 }
457 
458 /**
459  * igc_clean_rx_ring - Free Rx Buffers per Queue
460  * @ring: ring to free buffers from
461  */
igc_clean_rx_ring(struct igc_ring * ring)462 static void igc_clean_rx_ring(struct igc_ring *ring)
463 {
464 	if (ring->xsk_pool)
465 		igc_clean_rx_ring_xsk_pool(ring);
466 	else
467 		igc_clean_rx_ring_page_shared(ring);
468 
469 	clear_ring_uses_large_buffer(ring);
470 
471 	ring->next_to_alloc = 0;
472 	ring->next_to_clean = 0;
473 	ring->next_to_use = 0;
474 }
475 
476 /**
477  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
478  * @adapter: board private structure
479  */
igc_clean_all_rx_rings(struct igc_adapter * adapter)480 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
481 {
482 	int i;
483 
484 	for (i = 0; i < adapter->num_rx_queues; i++)
485 		if (adapter->rx_ring[i])
486 			igc_clean_rx_ring(adapter->rx_ring[i]);
487 }
488 
489 /**
490  * igc_free_rx_resources - Free Rx Resources
491  * @rx_ring: ring to clean the resources from
492  *
493  * Free all receive software resources
494  */
igc_free_rx_resources(struct igc_ring * rx_ring)495 void igc_free_rx_resources(struct igc_ring *rx_ring)
496 {
497 	igc_clean_rx_ring(rx_ring);
498 
499 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
500 
501 	vfree(rx_ring->rx_buffer_info);
502 	rx_ring->rx_buffer_info = NULL;
503 
504 	/* if not set, then don't free */
505 	if (!rx_ring->desc)
506 		return;
507 
508 	dma_free_coherent(rx_ring->dev, rx_ring->size,
509 			  rx_ring->desc, rx_ring->dma);
510 
511 	rx_ring->desc = NULL;
512 }
513 
514 /**
515  * igc_free_all_rx_resources - Free Rx Resources for All Queues
516  * @adapter: board private structure
517  *
518  * Free all receive software resources
519  */
igc_free_all_rx_resources(struct igc_adapter * adapter)520 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
521 {
522 	int i;
523 
524 	for (i = 0; i < adapter->num_rx_queues; i++)
525 		igc_free_rx_resources(adapter->rx_ring[i]);
526 }
527 
528 /**
529  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
530  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
531  *
532  * Returns 0 on success, negative on failure
533  */
igc_setup_rx_resources(struct igc_ring * rx_ring)534 int igc_setup_rx_resources(struct igc_ring *rx_ring)
535 {
536 	struct net_device *ndev = rx_ring->netdev;
537 	struct device *dev = rx_ring->dev;
538 	u8 index = rx_ring->queue_index;
539 	int size, desc_len, res;
540 
541 	/* XDP RX-queue info */
542 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
543 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
544 	res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
545 			       rx_ring->q_vector->napi.napi_id);
546 	if (res < 0) {
547 		netdev_err(ndev, "Failed to register xdp_rxq index %u\n",
548 			   index);
549 		return res;
550 	}
551 
552 	size = sizeof(struct igc_rx_buffer) * rx_ring->count;
553 	rx_ring->rx_buffer_info = vzalloc(size);
554 	if (!rx_ring->rx_buffer_info)
555 		goto err;
556 
557 	desc_len = sizeof(union igc_adv_rx_desc);
558 
559 	/* Round up to nearest 4K */
560 	rx_ring->size = rx_ring->count * desc_len;
561 	rx_ring->size = ALIGN(rx_ring->size, 4096);
562 
563 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
564 					   &rx_ring->dma, GFP_KERNEL);
565 
566 	if (!rx_ring->desc)
567 		goto err;
568 
569 	rx_ring->next_to_alloc = 0;
570 	rx_ring->next_to_clean = 0;
571 	rx_ring->next_to_use = 0;
572 
573 	return 0;
574 
575 err:
576 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
577 	vfree(rx_ring->rx_buffer_info);
578 	rx_ring->rx_buffer_info = NULL;
579 	netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
580 	return -ENOMEM;
581 }
582 
583 /**
584  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
585  *                                (Descriptors) for all queues
586  * @adapter: board private structure
587  *
588  * Return 0 on success, negative on failure
589  */
igc_setup_all_rx_resources(struct igc_adapter * adapter)590 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
591 {
592 	struct net_device *dev = adapter->netdev;
593 	int i, err = 0;
594 
595 	for (i = 0; i < adapter->num_rx_queues; i++) {
596 		err = igc_setup_rx_resources(adapter->rx_ring[i]);
597 		if (err) {
598 			netdev_err(dev, "Error on Rx queue %u setup\n", i);
599 			for (i--; i >= 0; i--)
600 				igc_free_rx_resources(adapter->rx_ring[i]);
601 			break;
602 		}
603 	}
604 
605 	return err;
606 }
607 
igc_get_xsk_pool(struct igc_adapter * adapter,struct igc_ring * ring)608 static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter,
609 					      struct igc_ring *ring)
610 {
611 	if (!igc_xdp_is_enabled(adapter) ||
612 	    !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags))
613 		return NULL;
614 
615 	return xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
616 }
617 
618 /**
619  * igc_configure_rx_ring - Configure a receive ring after Reset
620  * @adapter: board private structure
621  * @ring: receive ring to be configured
622  *
623  * Configure the Rx unit of the MAC after a reset.
624  */
igc_configure_rx_ring(struct igc_adapter * adapter,struct igc_ring * ring)625 static void igc_configure_rx_ring(struct igc_adapter *adapter,
626 				  struct igc_ring *ring)
627 {
628 	struct igc_hw *hw = &adapter->hw;
629 	union igc_adv_rx_desc *rx_desc;
630 	int reg_idx = ring->reg_idx;
631 	u32 srrctl = 0, rxdctl = 0;
632 	u64 rdba = ring->dma;
633 	u32 buf_size;
634 
635 	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
636 	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
637 	if (ring->xsk_pool) {
638 		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
639 						   MEM_TYPE_XSK_BUFF_POOL,
640 						   NULL));
641 		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
642 	} else {
643 		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
644 						   MEM_TYPE_PAGE_SHARED,
645 						   NULL));
646 	}
647 
648 	if (igc_xdp_is_enabled(adapter))
649 		set_ring_uses_large_buffer(ring);
650 
651 	/* disable the queue */
652 	wr32(IGC_RXDCTL(reg_idx), 0);
653 
654 	/* Set DMA base address registers */
655 	wr32(IGC_RDBAL(reg_idx),
656 	     rdba & 0x00000000ffffffffULL);
657 	wr32(IGC_RDBAH(reg_idx), rdba >> 32);
658 	wr32(IGC_RDLEN(reg_idx),
659 	     ring->count * sizeof(union igc_adv_rx_desc));
660 
661 	/* initialize head and tail */
662 	ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
663 	wr32(IGC_RDH(reg_idx), 0);
664 	writel(0, ring->tail);
665 
666 	/* reset next-to- use/clean to place SW in sync with hardware */
667 	ring->next_to_clean = 0;
668 	ring->next_to_use = 0;
669 
670 	if (ring->xsk_pool)
671 		buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool);
672 	else if (ring_uses_large_buffer(ring))
673 		buf_size = IGC_RXBUFFER_3072;
674 	else
675 		buf_size = IGC_RXBUFFER_2048;
676 
677 	srrctl = rd32(IGC_SRRCTL(reg_idx));
678 	srrctl &= ~(IGC_SRRCTL_BSIZEPKT_MASK | IGC_SRRCTL_BSIZEHDR_MASK |
679 		    IGC_SRRCTL_DESCTYPE_MASK);
680 	srrctl |= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN);
681 	srrctl |= IGC_SRRCTL_BSIZEPKT(buf_size);
682 	srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
683 
684 	wr32(IGC_SRRCTL(reg_idx), srrctl);
685 
686 	rxdctl |= IGC_RX_PTHRESH;
687 	rxdctl |= IGC_RX_HTHRESH << 8;
688 	rxdctl |= IGC_RX_WTHRESH << 16;
689 
690 	/* initialize rx_buffer_info */
691 	memset(ring->rx_buffer_info, 0,
692 	       sizeof(struct igc_rx_buffer) * ring->count);
693 
694 	/* initialize Rx descriptor 0 */
695 	rx_desc = IGC_RX_DESC(ring, 0);
696 	rx_desc->wb.upper.length = 0;
697 
698 	/* enable receive descriptor fetching */
699 	rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
700 
701 	wr32(IGC_RXDCTL(reg_idx), rxdctl);
702 }
703 
704 /**
705  * igc_configure_rx - Configure receive Unit after Reset
706  * @adapter: board private structure
707  *
708  * Configure the Rx unit of the MAC after a reset.
709  */
igc_configure_rx(struct igc_adapter * adapter)710 static void igc_configure_rx(struct igc_adapter *adapter)
711 {
712 	int i;
713 
714 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
715 	 * the Base and Length of the Rx Descriptor Ring
716 	 */
717 	for (i = 0; i < adapter->num_rx_queues; i++)
718 		igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
719 }
720 
721 /**
722  * igc_configure_tx_ring - Configure transmit ring after Reset
723  * @adapter: board private structure
724  * @ring: tx ring to configure
725  *
726  * Configure a transmit ring after a reset.
727  */
igc_configure_tx_ring(struct igc_adapter * adapter,struct igc_ring * ring)728 static void igc_configure_tx_ring(struct igc_adapter *adapter,
729 				  struct igc_ring *ring)
730 {
731 	struct igc_hw *hw = &adapter->hw;
732 	int reg_idx = ring->reg_idx;
733 	u64 tdba = ring->dma;
734 	u32 txdctl = 0;
735 
736 	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
737 
738 	/* disable the queue */
739 	wr32(IGC_TXDCTL(reg_idx), 0);
740 	wrfl();
741 
742 	wr32(IGC_TDLEN(reg_idx),
743 	     ring->count * sizeof(union igc_adv_tx_desc));
744 	wr32(IGC_TDBAL(reg_idx),
745 	     tdba & 0x00000000ffffffffULL);
746 	wr32(IGC_TDBAH(reg_idx), tdba >> 32);
747 
748 	ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
749 	wr32(IGC_TDH(reg_idx), 0);
750 	writel(0, ring->tail);
751 
752 	txdctl |= IGC_TX_PTHRESH;
753 	txdctl |= IGC_TX_HTHRESH << 8;
754 	txdctl |= IGC_TX_WTHRESH << 16;
755 
756 	txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
757 	wr32(IGC_TXDCTL(reg_idx), txdctl);
758 }
759 
760 /**
761  * igc_configure_tx - Configure transmit Unit after Reset
762  * @adapter: board private structure
763  *
764  * Configure the Tx unit of the MAC after a reset.
765  */
igc_configure_tx(struct igc_adapter * adapter)766 static void igc_configure_tx(struct igc_adapter *adapter)
767 {
768 	int i;
769 
770 	for (i = 0; i < adapter->num_tx_queues; i++)
771 		igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
772 }
773 
774 /**
775  * igc_setup_mrqc - configure the multiple receive queue control registers
776  * @adapter: Board private structure
777  */
igc_setup_mrqc(struct igc_adapter * adapter)778 static void igc_setup_mrqc(struct igc_adapter *adapter)
779 {
780 	struct igc_hw *hw = &adapter->hw;
781 	u32 j, num_rx_queues;
782 	u32 mrqc, rxcsum;
783 	u32 rss_key[10];
784 
785 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
786 	for (j = 0; j < 10; j++)
787 		wr32(IGC_RSSRK(j), rss_key[j]);
788 
789 	num_rx_queues = adapter->rss_queues;
790 
791 	if (adapter->rss_indir_tbl_init != num_rx_queues) {
792 		for (j = 0; j < IGC_RETA_SIZE; j++)
793 			adapter->rss_indir_tbl[j] =
794 			(j * num_rx_queues) / IGC_RETA_SIZE;
795 		adapter->rss_indir_tbl_init = num_rx_queues;
796 	}
797 	igc_write_rss_indir_tbl(adapter);
798 
799 	/* Disable raw packet checksumming so that RSS hash is placed in
800 	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
801 	 * offloads as they are enabled by default
802 	 */
803 	rxcsum = rd32(IGC_RXCSUM);
804 	rxcsum |= IGC_RXCSUM_PCSD;
805 
806 	/* Enable Receive Checksum Offload for SCTP */
807 	rxcsum |= IGC_RXCSUM_CRCOFL;
808 
809 	/* Don't need to set TUOFL or IPOFL, they default to 1 */
810 	wr32(IGC_RXCSUM, rxcsum);
811 
812 	/* Generate RSS hash based on packet types, TCP/UDP
813 	 * port numbers and/or IPv4/v6 src and dst addresses
814 	 */
815 	mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
816 	       IGC_MRQC_RSS_FIELD_IPV4_TCP |
817 	       IGC_MRQC_RSS_FIELD_IPV6 |
818 	       IGC_MRQC_RSS_FIELD_IPV6_TCP |
819 	       IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
820 
821 	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
822 		mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
823 	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
824 		mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
825 
826 	mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
827 
828 	wr32(IGC_MRQC, mrqc);
829 }
830 
831 /**
832  * igc_setup_rctl - configure the receive control registers
833  * @adapter: Board private structure
834  */
igc_setup_rctl(struct igc_adapter * adapter)835 static void igc_setup_rctl(struct igc_adapter *adapter)
836 {
837 	struct igc_hw *hw = &adapter->hw;
838 	u32 rctl;
839 
840 	rctl = rd32(IGC_RCTL);
841 
842 	rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
843 	rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
844 
845 	rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
846 		(hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
847 
848 	/* enable stripping of CRC. Newer features require
849 	 * that the HW strips the CRC.
850 	 */
851 	rctl |= IGC_RCTL_SECRC;
852 
853 	/* disable store bad packets and clear size bits. */
854 	rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
855 
856 	/* enable LPE to allow for reception of jumbo frames */
857 	rctl |= IGC_RCTL_LPE;
858 
859 	/* disable queue 0 to prevent tail write w/o re-config */
860 	wr32(IGC_RXDCTL(0), 0);
861 
862 	/* This is useful for sniffing bad packets. */
863 	if (adapter->netdev->features & NETIF_F_RXALL) {
864 		/* UPE and MPE will be handled by normal PROMISC logic
865 		 * in set_rx_mode
866 		 */
867 		rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
868 			 IGC_RCTL_BAM | /* RX All Bcast Pkts */
869 			 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
870 
871 		rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
872 			  IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
873 	}
874 
875 	wr32(IGC_RCTL, rctl);
876 }
877 
878 /**
879  * igc_setup_tctl - configure the transmit control registers
880  * @adapter: Board private structure
881  */
igc_setup_tctl(struct igc_adapter * adapter)882 static void igc_setup_tctl(struct igc_adapter *adapter)
883 {
884 	struct igc_hw *hw = &adapter->hw;
885 	u32 tctl;
886 
887 	/* disable queue 0 which icould be enabled by default */
888 	wr32(IGC_TXDCTL(0), 0);
889 
890 	/* Program the Transmit Control Register */
891 	tctl = rd32(IGC_TCTL);
892 	tctl &= ~IGC_TCTL_CT;
893 	tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
894 		(IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
895 
896 	/* Enable transmits */
897 	tctl |= IGC_TCTL_EN;
898 
899 	wr32(IGC_TCTL, tctl);
900 }
901 
902 /**
903  * igc_set_mac_filter_hw() - Set MAC address filter in hardware
904  * @adapter: Pointer to adapter where the filter should be set
905  * @index: Filter index
906  * @type: MAC address filter type (source or destination)
907  * @addr: MAC address
908  * @queue: If non-negative, queue assignment feature is enabled and frames
909  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
910  *         assignment is disabled.
911  */
igc_set_mac_filter_hw(struct igc_adapter * adapter,int index,enum igc_mac_filter_type type,const u8 * addr,int queue)912 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
913 				  enum igc_mac_filter_type type,
914 				  const u8 *addr, int queue)
915 {
916 	struct net_device *dev = adapter->netdev;
917 	struct igc_hw *hw = &adapter->hw;
918 	u32 ral, rah;
919 
920 	if (WARN_ON(index >= hw->mac.rar_entry_count))
921 		return;
922 
923 	ral = le32_to_cpup((__le32 *)(addr));
924 	rah = le16_to_cpup((__le16 *)(addr + 4));
925 
926 	if (type == IGC_MAC_FILTER_TYPE_SRC) {
927 		rah &= ~IGC_RAH_ASEL_MASK;
928 		rah |= IGC_RAH_ASEL_SRC_ADDR;
929 	}
930 
931 	if (queue >= 0) {
932 		rah &= ~IGC_RAH_QSEL_MASK;
933 		rah |= (queue << IGC_RAH_QSEL_SHIFT);
934 		rah |= IGC_RAH_QSEL_ENABLE;
935 	}
936 
937 	rah |= IGC_RAH_AV;
938 
939 	wr32(IGC_RAL(index), ral);
940 	wr32(IGC_RAH(index), rah);
941 
942 	netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
943 }
944 
945 /**
946  * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
947  * @adapter: Pointer to adapter where the filter should be cleared
948  * @index: Filter index
949  */
igc_clear_mac_filter_hw(struct igc_adapter * adapter,int index)950 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
951 {
952 	struct net_device *dev = adapter->netdev;
953 	struct igc_hw *hw = &adapter->hw;
954 
955 	if (WARN_ON(index >= hw->mac.rar_entry_count))
956 		return;
957 
958 	wr32(IGC_RAL(index), 0);
959 	wr32(IGC_RAH(index), 0);
960 
961 	netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
962 }
963 
964 /* Set default MAC address for the PF in the first RAR entry */
igc_set_default_mac_filter(struct igc_adapter * adapter)965 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
966 {
967 	struct net_device *dev = adapter->netdev;
968 	u8 *addr = adapter->hw.mac.addr;
969 
970 	netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
971 
972 	igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
973 }
974 
975 /**
976  * igc_set_mac - Change the Ethernet Address of the NIC
977  * @netdev: network interface device structure
978  * @p: pointer to an address structure
979  *
980  * Returns 0 on success, negative on failure
981  */
igc_set_mac(struct net_device * netdev,void * p)982 static int igc_set_mac(struct net_device *netdev, void *p)
983 {
984 	struct igc_adapter *adapter = netdev_priv(netdev);
985 	struct igc_hw *hw = &adapter->hw;
986 	struct sockaddr *addr = p;
987 
988 	if (!is_valid_ether_addr(addr->sa_data))
989 		return -EADDRNOTAVAIL;
990 
991 	eth_hw_addr_set(netdev, addr->sa_data);
992 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
993 
994 	/* set the correct pool for the new PF MAC address in entry 0 */
995 	igc_set_default_mac_filter(adapter);
996 
997 	return 0;
998 }
999 
1000 /**
1001  *  igc_write_mc_addr_list - write multicast addresses to MTA
1002  *  @netdev: network interface device structure
1003  *
1004  *  Writes multicast address list to the MTA hash table.
1005  *  Returns: -ENOMEM on failure
1006  *           0 on no addresses written
1007  *           X on writing X addresses to MTA
1008  **/
igc_write_mc_addr_list(struct net_device * netdev)1009 static int igc_write_mc_addr_list(struct net_device *netdev)
1010 {
1011 	struct igc_adapter *adapter = netdev_priv(netdev);
1012 	struct igc_hw *hw = &adapter->hw;
1013 	struct netdev_hw_addr *ha;
1014 	u8  *mta_list;
1015 	int i;
1016 
1017 	if (netdev_mc_empty(netdev)) {
1018 		/* nothing to program, so clear mc list */
1019 		igc_update_mc_addr_list(hw, NULL, 0);
1020 		return 0;
1021 	}
1022 
1023 	mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
1024 	if (!mta_list)
1025 		return -ENOMEM;
1026 
1027 	/* The shared function expects a packed array of only addresses. */
1028 	i = 0;
1029 	netdev_for_each_mc_addr(ha, netdev)
1030 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1031 
1032 	igc_update_mc_addr_list(hw, mta_list, i);
1033 	kfree(mta_list);
1034 
1035 	return netdev_mc_count(netdev);
1036 }
1037 
igc_tx_launchtime(struct igc_ring * ring,ktime_t txtime,bool * first_flag,bool * insert_empty)1038 static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime,
1039 				bool *first_flag, bool *insert_empty)
1040 {
1041 	struct igc_adapter *adapter = netdev_priv(ring->netdev);
1042 	ktime_t cycle_time = adapter->cycle_time;
1043 	ktime_t base_time = adapter->base_time;
1044 	ktime_t now = ktime_get_clocktai();
1045 	ktime_t baset_est, end_of_cycle;
1046 	s32 launchtime;
1047 	s64 n;
1048 
1049 	n = div64_s64(ktime_sub_ns(now, base_time), cycle_time);
1050 
1051 	baset_est = ktime_add_ns(base_time, cycle_time * (n));
1052 	end_of_cycle = ktime_add_ns(baset_est, cycle_time);
1053 
1054 	if (ktime_compare(txtime, end_of_cycle) >= 0) {
1055 		if (baset_est != ring->last_ff_cycle) {
1056 			*first_flag = true;
1057 			ring->last_ff_cycle = baset_est;
1058 
1059 			if (ktime_compare(end_of_cycle, ring->last_tx_cycle) > 0)
1060 				*insert_empty = true;
1061 		}
1062 	}
1063 
1064 	/* Introducing a window at end of cycle on which packets
1065 	 * potentially not honor launchtime. Window of 5us chosen
1066 	 * considering software update the tail pointer and packets
1067 	 * are dma'ed to packet buffer.
1068 	 */
1069 	if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC))
1070 		netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n",
1071 			    txtime);
1072 
1073 	ring->last_tx_cycle = end_of_cycle;
1074 
1075 	launchtime = ktime_sub_ns(txtime, baset_est);
1076 	if (launchtime > 0)
1077 		div_s64_rem(launchtime, cycle_time, &launchtime);
1078 	else
1079 		launchtime = 0;
1080 
1081 	return cpu_to_le32(launchtime);
1082 }
1083 
igc_init_empty_frame(struct igc_ring * ring,struct igc_tx_buffer * buffer,struct sk_buff * skb)1084 static int igc_init_empty_frame(struct igc_ring *ring,
1085 				struct igc_tx_buffer *buffer,
1086 				struct sk_buff *skb)
1087 {
1088 	unsigned int size;
1089 	dma_addr_t dma;
1090 
1091 	size = skb_headlen(skb);
1092 
1093 	dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE);
1094 	if (dma_mapping_error(ring->dev, dma)) {
1095 		netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
1096 		return -ENOMEM;
1097 	}
1098 
1099 	buffer->skb = skb;
1100 	buffer->protocol = 0;
1101 	buffer->bytecount = skb->len;
1102 	buffer->gso_segs = 1;
1103 	buffer->time_stamp = jiffies;
1104 	dma_unmap_len_set(buffer, len, skb->len);
1105 	dma_unmap_addr_set(buffer, dma, dma);
1106 
1107 	return 0;
1108 }
1109 
igc_init_tx_empty_descriptor(struct igc_ring * ring,struct sk_buff * skb,struct igc_tx_buffer * first)1110 static int igc_init_tx_empty_descriptor(struct igc_ring *ring,
1111 					struct sk_buff *skb,
1112 					struct igc_tx_buffer *first)
1113 {
1114 	union igc_adv_tx_desc *desc;
1115 	u32 cmd_type, olinfo_status;
1116 	int err;
1117 
1118 	if (!igc_desc_unused(ring))
1119 		return -EBUSY;
1120 
1121 	err = igc_init_empty_frame(ring, first, skb);
1122 	if (err)
1123 		return err;
1124 
1125 	cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
1126 		   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
1127 		   first->bytecount;
1128 	olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
1129 
1130 	desc = IGC_TX_DESC(ring, ring->next_to_use);
1131 	desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1132 	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1133 	desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma));
1134 
1135 	netdev_tx_sent_queue(txring_txq(ring), skb->len);
1136 
1137 	first->next_to_watch = desc;
1138 
1139 	ring->next_to_use++;
1140 	if (ring->next_to_use == ring->count)
1141 		ring->next_to_use = 0;
1142 
1143 	return 0;
1144 }
1145 
1146 #define IGC_EMPTY_FRAME_SIZE 60
1147 
igc_tx_ctxtdesc(struct igc_ring * tx_ring,__le32 launch_time,bool first_flag,u32 vlan_macip_lens,u32 type_tucmd,u32 mss_l4len_idx)1148 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1149 			    __le32 launch_time, bool first_flag,
1150 			    u32 vlan_macip_lens, u32 type_tucmd,
1151 			    u32 mss_l4len_idx)
1152 {
1153 	struct igc_adv_tx_context_desc *context_desc;
1154 	u16 i = tx_ring->next_to_use;
1155 
1156 	context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1157 
1158 	i++;
1159 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1160 
1161 	/* set bits to identify this as an advanced context descriptor */
1162 	type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1163 
1164 	/* For i225, context index must be unique per ring. */
1165 	if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1166 		mss_l4len_idx |= tx_ring->reg_idx << 4;
1167 
1168 	if (first_flag)
1169 		mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST;
1170 
1171 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1172 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1173 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1174 	context_desc->launch_time	= launch_time;
1175 }
1176 
igc_tx_csum(struct igc_ring * tx_ring,struct igc_tx_buffer * first,__le32 launch_time,bool first_flag)1177 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first,
1178 			__le32 launch_time, bool first_flag)
1179 {
1180 	struct sk_buff *skb = first->skb;
1181 	u32 vlan_macip_lens = 0;
1182 	u32 type_tucmd = 0;
1183 
1184 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
1185 csum_failed:
1186 		if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1187 		    !tx_ring->launchtime_enable)
1188 			return;
1189 		goto no_csum;
1190 	}
1191 
1192 	switch (skb->csum_offset) {
1193 	case offsetof(struct tcphdr, check):
1194 		type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1195 		fallthrough;
1196 	case offsetof(struct udphdr, check):
1197 		break;
1198 	case offsetof(struct sctphdr, checksum):
1199 		/* validate that this is actually an SCTP request */
1200 		if (skb_csum_is_sctp(skb)) {
1201 			type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1202 			break;
1203 		}
1204 		fallthrough;
1205 	default:
1206 		skb_checksum_help(skb);
1207 		goto csum_failed;
1208 	}
1209 
1210 	/* update TX checksum flag */
1211 	first->tx_flags |= IGC_TX_FLAGS_CSUM;
1212 	vlan_macip_lens = skb_checksum_start_offset(skb) -
1213 			  skb_network_offset(skb);
1214 no_csum:
1215 	vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1216 	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1217 
1218 	igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1219 			vlan_macip_lens, type_tucmd, 0);
1220 }
1221 
__igc_maybe_stop_tx(struct igc_ring * tx_ring,const u16 size)1222 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1223 {
1224 	struct net_device *netdev = tx_ring->netdev;
1225 
1226 	netif_stop_subqueue(netdev, tx_ring->queue_index);
1227 
1228 	/* memory barriier comment */
1229 	smp_mb();
1230 
1231 	/* We need to check again in a case another CPU has just
1232 	 * made room available.
1233 	 */
1234 	if (igc_desc_unused(tx_ring) < size)
1235 		return -EBUSY;
1236 
1237 	/* A reprieve! */
1238 	netif_wake_subqueue(netdev, tx_ring->queue_index);
1239 
1240 	u64_stats_update_begin(&tx_ring->tx_syncp2);
1241 	tx_ring->tx_stats.restart_queue2++;
1242 	u64_stats_update_end(&tx_ring->tx_syncp2);
1243 
1244 	return 0;
1245 }
1246 
igc_maybe_stop_tx(struct igc_ring * tx_ring,const u16 size)1247 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1248 {
1249 	if (igc_desc_unused(tx_ring) >= size)
1250 		return 0;
1251 	return __igc_maybe_stop_tx(tx_ring, size);
1252 }
1253 
1254 #define IGC_SET_FLAG(_input, _flag, _result) \
1255 	(((_flag) <= (_result)) ?				\
1256 	 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) :	\
1257 	 ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1258 
igc_tx_cmd_type(struct sk_buff * skb,u32 tx_flags)1259 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
1260 {
1261 	/* set type for advanced descriptor with frame checksum insertion */
1262 	u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
1263 		       IGC_ADVTXD_DCMD_DEXT |
1264 		       IGC_ADVTXD_DCMD_IFCS;
1265 
1266 	/* set HW vlan bit if vlan is present */
1267 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN,
1268 				 IGC_ADVTXD_DCMD_VLE);
1269 
1270 	/* set segmentation bits for TSO */
1271 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1272 				 (IGC_ADVTXD_DCMD_TSE));
1273 
1274 	/* set timestamp bit if present, will select the register set
1275 	 * based on the _TSTAMP(_X) bit.
1276 	 */
1277 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
1278 				 (IGC_ADVTXD_MAC_TSTAMP));
1279 
1280 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_1,
1281 				 (IGC_ADVTXD_TSTAMP_REG_1));
1282 
1283 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_2,
1284 				 (IGC_ADVTXD_TSTAMP_REG_2));
1285 
1286 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_3,
1287 				 (IGC_ADVTXD_TSTAMP_REG_3));
1288 
1289 	/* insert frame checksum */
1290 	cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS);
1291 
1292 	return cmd_type;
1293 }
1294 
igc_tx_olinfo_status(struct igc_ring * tx_ring,union igc_adv_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)1295 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
1296 				 union igc_adv_tx_desc *tx_desc,
1297 				 u32 tx_flags, unsigned int paylen)
1298 {
1299 	u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
1300 
1301 	/* insert L4 checksum */
1302 	olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
1303 			  ((IGC_TXD_POPTS_TXSM << 8) /
1304 			  IGC_TX_FLAGS_CSUM);
1305 
1306 	/* insert IPv4 checksum */
1307 	olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
1308 			  (((IGC_TXD_POPTS_IXSM << 8)) /
1309 			  IGC_TX_FLAGS_IPV4);
1310 
1311 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1312 }
1313 
igc_tx_map(struct igc_ring * tx_ring,struct igc_tx_buffer * first,const u8 hdr_len)1314 static int igc_tx_map(struct igc_ring *tx_ring,
1315 		      struct igc_tx_buffer *first,
1316 		      const u8 hdr_len)
1317 {
1318 	struct sk_buff *skb = first->skb;
1319 	struct igc_tx_buffer *tx_buffer;
1320 	union igc_adv_tx_desc *tx_desc;
1321 	u32 tx_flags = first->tx_flags;
1322 	skb_frag_t *frag;
1323 	u16 i = tx_ring->next_to_use;
1324 	unsigned int data_len, size;
1325 	dma_addr_t dma;
1326 	u32 cmd_type;
1327 
1328 	cmd_type = igc_tx_cmd_type(skb, tx_flags);
1329 	tx_desc = IGC_TX_DESC(tx_ring, i);
1330 
1331 	igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
1332 
1333 	size = skb_headlen(skb);
1334 	data_len = skb->data_len;
1335 
1336 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1337 
1338 	tx_buffer = first;
1339 
1340 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1341 		if (dma_mapping_error(tx_ring->dev, dma))
1342 			goto dma_error;
1343 
1344 		/* record length, and DMA address */
1345 		dma_unmap_len_set(tx_buffer, len, size);
1346 		dma_unmap_addr_set(tx_buffer, dma, dma);
1347 
1348 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
1349 
1350 		while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
1351 			tx_desc->read.cmd_type_len =
1352 				cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
1353 
1354 			i++;
1355 			tx_desc++;
1356 			if (i == tx_ring->count) {
1357 				tx_desc = IGC_TX_DESC(tx_ring, 0);
1358 				i = 0;
1359 			}
1360 			tx_desc->read.olinfo_status = 0;
1361 
1362 			dma += IGC_MAX_DATA_PER_TXD;
1363 			size -= IGC_MAX_DATA_PER_TXD;
1364 
1365 			tx_desc->read.buffer_addr = cpu_to_le64(dma);
1366 		}
1367 
1368 		if (likely(!data_len))
1369 			break;
1370 
1371 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1372 
1373 		i++;
1374 		tx_desc++;
1375 		if (i == tx_ring->count) {
1376 			tx_desc = IGC_TX_DESC(tx_ring, 0);
1377 			i = 0;
1378 		}
1379 		tx_desc->read.olinfo_status = 0;
1380 
1381 		size = skb_frag_size(frag);
1382 		data_len -= size;
1383 
1384 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1385 				       size, DMA_TO_DEVICE);
1386 
1387 		tx_buffer = &tx_ring->tx_buffer_info[i];
1388 	}
1389 
1390 	/* write last descriptor with RS and EOP bits */
1391 	cmd_type |= size | IGC_TXD_DCMD;
1392 	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1393 
1394 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1395 
1396 	/* set the timestamp */
1397 	first->time_stamp = jiffies;
1398 
1399 	skb_tx_timestamp(skb);
1400 
1401 	/* Force memory writes to complete before letting h/w know there
1402 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
1403 	 * memory model archs, such as IA-64).
1404 	 *
1405 	 * We also need this memory barrier to make certain all of the
1406 	 * status bits have been updated before next_to_watch is written.
1407 	 */
1408 	wmb();
1409 
1410 	/* set next_to_watch value indicating a packet is present */
1411 	first->next_to_watch = tx_desc;
1412 
1413 	i++;
1414 	if (i == tx_ring->count)
1415 		i = 0;
1416 
1417 	tx_ring->next_to_use = i;
1418 
1419 	/* Make sure there is space in the ring for the next send. */
1420 	igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1421 
1422 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1423 		writel(i, tx_ring->tail);
1424 	}
1425 
1426 	return 0;
1427 dma_error:
1428 	netdev_err(tx_ring->netdev, "TX DMA map failed\n");
1429 	tx_buffer = &tx_ring->tx_buffer_info[i];
1430 
1431 	/* clear dma mappings for failed tx_buffer_info map */
1432 	while (tx_buffer != first) {
1433 		if (dma_unmap_len(tx_buffer, len))
1434 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1435 
1436 		if (i-- == 0)
1437 			i += tx_ring->count;
1438 		tx_buffer = &tx_ring->tx_buffer_info[i];
1439 	}
1440 
1441 	if (dma_unmap_len(tx_buffer, len))
1442 		igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1443 
1444 	dev_kfree_skb_any(tx_buffer->skb);
1445 	tx_buffer->skb = NULL;
1446 
1447 	tx_ring->next_to_use = i;
1448 
1449 	return -1;
1450 }
1451 
igc_tso(struct igc_ring * tx_ring,struct igc_tx_buffer * first,__le32 launch_time,bool first_flag,u8 * hdr_len)1452 static int igc_tso(struct igc_ring *tx_ring,
1453 		   struct igc_tx_buffer *first,
1454 		   __le32 launch_time, bool first_flag,
1455 		   u8 *hdr_len)
1456 {
1457 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1458 	struct sk_buff *skb = first->skb;
1459 	union {
1460 		struct iphdr *v4;
1461 		struct ipv6hdr *v6;
1462 		unsigned char *hdr;
1463 	} ip;
1464 	union {
1465 		struct tcphdr *tcp;
1466 		struct udphdr *udp;
1467 		unsigned char *hdr;
1468 	} l4;
1469 	u32 paylen, l4_offset;
1470 	int err;
1471 
1472 	if (skb->ip_summed != CHECKSUM_PARTIAL)
1473 		return 0;
1474 
1475 	if (!skb_is_gso(skb))
1476 		return 0;
1477 
1478 	err = skb_cow_head(skb, 0);
1479 	if (err < 0)
1480 		return err;
1481 
1482 	ip.hdr = skb_network_header(skb);
1483 	l4.hdr = skb_checksum_start(skb);
1484 
1485 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1486 	type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1487 
1488 	/* initialize outer IP header fields */
1489 	if (ip.v4->version == 4) {
1490 		unsigned char *csum_start = skb_checksum_start(skb);
1491 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1492 
1493 		/* IP header will have to cancel out any data that
1494 		 * is not a part of the outer IP header
1495 		 */
1496 		ip.v4->check = csum_fold(csum_partial(trans_start,
1497 						      csum_start - trans_start,
1498 						      0));
1499 		type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1500 
1501 		ip.v4->tot_len = 0;
1502 		first->tx_flags |= IGC_TX_FLAGS_TSO |
1503 				   IGC_TX_FLAGS_CSUM |
1504 				   IGC_TX_FLAGS_IPV4;
1505 	} else {
1506 		ip.v6->payload_len = 0;
1507 		first->tx_flags |= IGC_TX_FLAGS_TSO |
1508 				   IGC_TX_FLAGS_CSUM;
1509 	}
1510 
1511 	/* determine offset of inner transport header */
1512 	l4_offset = l4.hdr - skb->data;
1513 
1514 	/* remove payload length from inner checksum */
1515 	paylen = skb->len - l4_offset;
1516 	if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1517 		/* compute length of segmentation header */
1518 		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
1519 		csum_replace_by_diff(&l4.tcp->check,
1520 				     (__force __wsum)htonl(paylen));
1521 	} else {
1522 		/* compute length of segmentation header */
1523 		*hdr_len = sizeof(*l4.udp) + l4_offset;
1524 		csum_replace_by_diff(&l4.udp->check,
1525 				     (__force __wsum)htonl(paylen));
1526 	}
1527 
1528 	/* update gso size and bytecount with header size */
1529 	first->gso_segs = skb_shinfo(skb)->gso_segs;
1530 	first->bytecount += (first->gso_segs - 1) * *hdr_len;
1531 
1532 	/* MSS L4LEN IDX */
1533 	mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1534 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1535 
1536 	/* VLAN MACLEN IPLEN */
1537 	vlan_macip_lens = l4.hdr - ip.hdr;
1538 	vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1539 	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1540 
1541 	igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1542 			vlan_macip_lens, type_tucmd, mss_l4len_idx);
1543 
1544 	return 1;
1545 }
1546 
igc_request_tx_tstamp(struct igc_adapter * adapter,struct sk_buff * skb,u32 * flags)1547 static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags)
1548 {
1549 	int i;
1550 
1551 	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
1552 		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
1553 
1554 		if (tstamp->skb)
1555 			continue;
1556 
1557 		tstamp->skb = skb_get(skb);
1558 		tstamp->start = jiffies;
1559 		*flags = tstamp->flags;
1560 
1561 		return true;
1562 	}
1563 
1564 	return false;
1565 }
1566 
igc_xmit_frame_ring(struct sk_buff * skb,struct igc_ring * tx_ring)1567 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1568 				       struct igc_ring *tx_ring)
1569 {
1570 	struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1571 	bool first_flag = false, insert_empty = false;
1572 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
1573 	__be16 protocol = vlan_get_protocol(skb);
1574 	struct igc_tx_buffer *first;
1575 	__le32 launch_time = 0;
1576 	u32 tx_flags = 0;
1577 	unsigned short f;
1578 	ktime_t txtime;
1579 	u8 hdr_len = 0;
1580 	int tso = 0;
1581 
1582 	/* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1583 	 *	+ 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1584 	 *	+ 2 desc gap to keep tail from touching head,
1585 	 *	+ 1 desc for context descriptor,
1586 	 * otherwise try next time
1587 	 */
1588 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1589 		count += TXD_USE_COUNT(skb_frag_size(
1590 						&skb_shinfo(skb)->frags[f]));
1591 
1592 	if (igc_maybe_stop_tx(tx_ring, count + 5)) {
1593 		/* this is a hard error */
1594 		return NETDEV_TX_BUSY;
1595 	}
1596 
1597 	if (!tx_ring->launchtime_enable)
1598 		goto done;
1599 
1600 	txtime = skb->tstamp;
1601 	skb->tstamp = ktime_set(0, 0);
1602 	launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty);
1603 
1604 	if (insert_empty) {
1605 		struct igc_tx_buffer *empty_info;
1606 		struct sk_buff *empty;
1607 		void *data;
1608 
1609 		empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1610 		empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC);
1611 		if (!empty)
1612 			goto done;
1613 
1614 		data = skb_put(empty, IGC_EMPTY_FRAME_SIZE);
1615 		memset(data, 0, IGC_EMPTY_FRAME_SIZE);
1616 
1617 		igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0);
1618 
1619 		if (igc_init_tx_empty_descriptor(tx_ring,
1620 						 empty,
1621 						 empty_info) < 0)
1622 			dev_kfree_skb_any(empty);
1623 	}
1624 
1625 done:
1626 	/* record the location of the first descriptor for this packet */
1627 	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1628 	first->type = IGC_TX_BUFFER_TYPE_SKB;
1629 	first->skb = skb;
1630 	first->bytecount = skb->len;
1631 	first->gso_segs = 1;
1632 
1633 	if (adapter->qbv_transition || tx_ring->oper_gate_closed)
1634 		goto out_drop;
1635 
1636 	if (tx_ring->max_sdu > 0 && first->bytecount > tx_ring->max_sdu) {
1637 		adapter->stats.txdrop++;
1638 		goto out_drop;
1639 	}
1640 
1641 	if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags) &&
1642 		     skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1643 		unsigned long flags;
1644 		u32 tstamp_flags;
1645 
1646 		spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
1647 		if (igc_request_tx_tstamp(adapter, skb, &tstamp_flags)) {
1648 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1649 			tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags;
1650 		} else {
1651 			adapter->tx_hwtstamp_skipped++;
1652 		}
1653 
1654 		spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
1655 	}
1656 
1657 	if (skb_vlan_tag_present(skb)) {
1658 		tx_flags |= IGC_TX_FLAGS_VLAN;
1659 		tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT);
1660 	}
1661 
1662 	/* record initial flags and protocol */
1663 	first->tx_flags = tx_flags;
1664 	first->protocol = protocol;
1665 
1666 	tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len);
1667 	if (tso < 0)
1668 		goto out_drop;
1669 	else if (!tso)
1670 		igc_tx_csum(tx_ring, first, launch_time, first_flag);
1671 
1672 	igc_tx_map(tx_ring, first, hdr_len);
1673 
1674 	return NETDEV_TX_OK;
1675 
1676 out_drop:
1677 	dev_kfree_skb_any(first->skb);
1678 	first->skb = NULL;
1679 
1680 	return NETDEV_TX_OK;
1681 }
1682 
igc_tx_queue_mapping(struct igc_adapter * adapter,struct sk_buff * skb)1683 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1684 						    struct sk_buff *skb)
1685 {
1686 	unsigned int r_idx = skb->queue_mapping;
1687 
1688 	if (r_idx >= adapter->num_tx_queues)
1689 		r_idx = r_idx % adapter->num_tx_queues;
1690 
1691 	return adapter->tx_ring[r_idx];
1692 }
1693 
igc_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1694 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1695 				  struct net_device *netdev)
1696 {
1697 	struct igc_adapter *adapter = netdev_priv(netdev);
1698 
1699 	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1700 	 * in order to meet this minimum size requirement.
1701 	 */
1702 	if (skb->len < 17) {
1703 		if (skb_padto(skb, 17))
1704 			return NETDEV_TX_OK;
1705 		skb->len = 17;
1706 	}
1707 
1708 	return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1709 }
1710 
igc_rx_checksum(struct igc_ring * ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1711 static void igc_rx_checksum(struct igc_ring *ring,
1712 			    union igc_adv_rx_desc *rx_desc,
1713 			    struct sk_buff *skb)
1714 {
1715 	skb_checksum_none_assert(skb);
1716 
1717 	/* Ignore Checksum bit is set */
1718 	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
1719 		return;
1720 
1721 	/* Rx checksum disabled via ethtool */
1722 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
1723 		return;
1724 
1725 	/* TCP/UDP checksum error bit is set */
1726 	if (igc_test_staterr(rx_desc,
1727 			     IGC_RXDEXT_STATERR_L4E |
1728 			     IGC_RXDEXT_STATERR_IPE)) {
1729 		/* work around errata with sctp packets where the TCPE aka
1730 		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1731 		 * packets (aka let the stack check the crc32c)
1732 		 */
1733 		if (!(skb->len == 60 &&
1734 		      test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
1735 			u64_stats_update_begin(&ring->rx_syncp);
1736 			ring->rx_stats.csum_err++;
1737 			u64_stats_update_end(&ring->rx_syncp);
1738 		}
1739 		/* let the stack verify checksum errors */
1740 		return;
1741 	}
1742 	/* It must be a TCP or UDP packet with a valid checksum */
1743 	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
1744 				      IGC_RXD_STAT_UDPCS))
1745 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1746 
1747 	netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
1748 		   le32_to_cpu(rx_desc->wb.upper.status_error));
1749 }
1750 
1751 /* Mapping HW RSS Type to enum pkt_hash_types */
1752 static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = {
1753 	[IGC_RSS_TYPE_NO_HASH]		= PKT_HASH_TYPE_L2,
1754 	[IGC_RSS_TYPE_HASH_TCP_IPV4]	= PKT_HASH_TYPE_L4,
1755 	[IGC_RSS_TYPE_HASH_IPV4]	= PKT_HASH_TYPE_L3,
1756 	[IGC_RSS_TYPE_HASH_TCP_IPV6]	= PKT_HASH_TYPE_L4,
1757 	[IGC_RSS_TYPE_HASH_IPV6_EX]	= PKT_HASH_TYPE_L3,
1758 	[IGC_RSS_TYPE_HASH_IPV6]	= PKT_HASH_TYPE_L3,
1759 	[IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4,
1760 	[IGC_RSS_TYPE_HASH_UDP_IPV4]	= PKT_HASH_TYPE_L4,
1761 	[IGC_RSS_TYPE_HASH_UDP_IPV6]	= PKT_HASH_TYPE_L4,
1762 	[IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4,
1763 	[10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW  */
1764 	[11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask   */
1765 	[12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisons       */
1766 	[13] = PKT_HASH_TYPE_NONE,
1767 	[14] = PKT_HASH_TYPE_NONE,
1768 	[15] = PKT_HASH_TYPE_NONE,
1769 };
1770 
igc_rx_hash(struct igc_ring * ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1771 static inline void igc_rx_hash(struct igc_ring *ring,
1772 			       union igc_adv_rx_desc *rx_desc,
1773 			       struct sk_buff *skb)
1774 {
1775 	if (ring->netdev->features & NETIF_F_RXHASH) {
1776 		u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
1777 		u32 rss_type = igc_rss_type(rx_desc);
1778 
1779 		skb_set_hash(skb, rss_hash, igc_rss_type_table[rss_type]);
1780 	}
1781 }
1782 
igc_rx_vlan(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1783 static void igc_rx_vlan(struct igc_ring *rx_ring,
1784 			union igc_adv_rx_desc *rx_desc,
1785 			struct sk_buff *skb)
1786 {
1787 	struct net_device *dev = rx_ring->netdev;
1788 	u16 vid;
1789 
1790 	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1791 	    igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) {
1792 		if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) &&
1793 		    test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
1794 			vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
1795 		else
1796 			vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1797 
1798 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1799 	}
1800 }
1801 
1802 /**
1803  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1804  * @rx_ring: rx descriptor ring packet is being transacted on
1805  * @rx_desc: pointer to the EOP Rx descriptor
1806  * @skb: pointer to current skb being populated
1807  *
1808  * This function checks the ring, descriptor, and packet information in order
1809  * to populate the hash, checksum, VLAN, protocol, and other fields within the
1810  * skb.
1811  */
igc_process_skb_fields(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1812 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1813 				   union igc_adv_rx_desc *rx_desc,
1814 				   struct sk_buff *skb)
1815 {
1816 	igc_rx_hash(rx_ring, rx_desc, skb);
1817 
1818 	igc_rx_checksum(rx_ring, rx_desc, skb);
1819 
1820 	igc_rx_vlan(rx_ring, rx_desc, skb);
1821 
1822 	skb_record_rx_queue(skb, rx_ring->queue_index);
1823 
1824 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1825 }
1826 
igc_vlan_mode(struct net_device * netdev,netdev_features_t features)1827 static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features)
1828 {
1829 	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1830 	struct igc_adapter *adapter = netdev_priv(netdev);
1831 	struct igc_hw *hw = &adapter->hw;
1832 	u32 ctrl;
1833 
1834 	ctrl = rd32(IGC_CTRL);
1835 
1836 	if (enable) {
1837 		/* enable VLAN tag insert/strip */
1838 		ctrl |= IGC_CTRL_VME;
1839 	} else {
1840 		/* disable VLAN tag insert/strip */
1841 		ctrl &= ~IGC_CTRL_VME;
1842 	}
1843 	wr32(IGC_CTRL, ctrl);
1844 }
1845 
igc_restore_vlan(struct igc_adapter * adapter)1846 static void igc_restore_vlan(struct igc_adapter *adapter)
1847 {
1848 	igc_vlan_mode(adapter->netdev, adapter->netdev->features);
1849 }
1850 
igc_get_rx_buffer(struct igc_ring * rx_ring,const unsigned int size,int * rx_buffer_pgcnt)1851 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1852 					       const unsigned int size,
1853 					       int *rx_buffer_pgcnt)
1854 {
1855 	struct igc_rx_buffer *rx_buffer;
1856 
1857 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1858 	*rx_buffer_pgcnt =
1859 #if (PAGE_SIZE < 8192)
1860 		page_count(rx_buffer->page);
1861 #else
1862 		0;
1863 #endif
1864 	prefetchw(rx_buffer->page);
1865 
1866 	/* we are reusing so sync this buffer for CPU use */
1867 	dma_sync_single_range_for_cpu(rx_ring->dev,
1868 				      rx_buffer->dma,
1869 				      rx_buffer->page_offset,
1870 				      size,
1871 				      DMA_FROM_DEVICE);
1872 
1873 	rx_buffer->pagecnt_bias--;
1874 
1875 	return rx_buffer;
1876 }
1877 
igc_rx_buffer_flip(struct igc_rx_buffer * buffer,unsigned int truesize)1878 static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer,
1879 			       unsigned int truesize)
1880 {
1881 #if (PAGE_SIZE < 8192)
1882 	buffer->page_offset ^= truesize;
1883 #else
1884 	buffer->page_offset += truesize;
1885 #endif
1886 }
1887 
igc_get_rx_frame_truesize(struct igc_ring * ring,unsigned int size)1888 static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring,
1889 					      unsigned int size)
1890 {
1891 	unsigned int truesize;
1892 
1893 #if (PAGE_SIZE < 8192)
1894 	truesize = igc_rx_pg_size(ring) / 2;
1895 #else
1896 	truesize = ring_uses_build_skb(ring) ?
1897 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1898 		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1899 		   SKB_DATA_ALIGN(size);
1900 #endif
1901 	return truesize;
1902 }
1903 
1904 /**
1905  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1906  * @rx_ring: rx descriptor ring to transact packets on
1907  * @rx_buffer: buffer containing page to add
1908  * @skb: sk_buff to place the data into
1909  * @size: size of buffer to be added
1910  *
1911  * This function will add the data contained in rx_buffer->page to the skb.
1912  */
igc_add_rx_frag(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct sk_buff * skb,unsigned int size)1913 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1914 			    struct igc_rx_buffer *rx_buffer,
1915 			    struct sk_buff *skb,
1916 			    unsigned int size)
1917 {
1918 	unsigned int truesize;
1919 
1920 #if (PAGE_SIZE < 8192)
1921 	truesize = igc_rx_pg_size(rx_ring) / 2;
1922 #else
1923 	truesize = ring_uses_build_skb(rx_ring) ?
1924 		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1925 		   SKB_DATA_ALIGN(size);
1926 #endif
1927 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1928 			rx_buffer->page_offset, size, truesize);
1929 
1930 	igc_rx_buffer_flip(rx_buffer, truesize);
1931 }
1932 
igc_build_skb(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct xdp_buff * xdp)1933 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1934 				     struct igc_rx_buffer *rx_buffer,
1935 				     struct xdp_buff *xdp)
1936 {
1937 	unsigned int size = xdp->data_end - xdp->data;
1938 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1939 	unsigned int metasize = xdp->data - xdp->data_meta;
1940 	struct sk_buff *skb;
1941 
1942 	/* prefetch first cache line of first page */
1943 	net_prefetch(xdp->data_meta);
1944 
1945 	/* build an skb around the page buffer */
1946 	skb = napi_build_skb(xdp->data_hard_start, truesize);
1947 	if (unlikely(!skb))
1948 		return NULL;
1949 
1950 	/* update pointers within the skb to store the data */
1951 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
1952 	__skb_put(skb, size);
1953 	if (metasize)
1954 		skb_metadata_set(skb, metasize);
1955 
1956 	igc_rx_buffer_flip(rx_buffer, truesize);
1957 	return skb;
1958 }
1959 
igc_construct_skb(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)1960 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1961 					 struct igc_rx_buffer *rx_buffer,
1962 					 struct xdp_buff *xdp,
1963 					 ktime_t timestamp)
1964 {
1965 	unsigned int metasize = xdp->data - xdp->data_meta;
1966 	unsigned int size = xdp->data_end - xdp->data;
1967 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1968 	void *va = xdp->data;
1969 	unsigned int headlen;
1970 	struct sk_buff *skb;
1971 
1972 	/* prefetch first cache line of first page */
1973 	net_prefetch(xdp->data_meta);
1974 
1975 	/* allocate a skb to store the frags */
1976 	skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1977 			     IGC_RX_HDR_LEN + metasize);
1978 	if (unlikely(!skb))
1979 		return NULL;
1980 
1981 	if (timestamp)
1982 		skb_hwtstamps(skb)->hwtstamp = timestamp;
1983 
1984 	/* Determine available headroom for copy */
1985 	headlen = size;
1986 	if (headlen > IGC_RX_HDR_LEN)
1987 		headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1988 
1989 	/* align pull length to size of long to optimize memcpy performance */
1990 	memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta,
1991 	       ALIGN(headlen + metasize, sizeof(long)));
1992 
1993 	if (metasize) {
1994 		skb_metadata_set(skb, metasize);
1995 		__skb_pull(skb, metasize);
1996 	}
1997 
1998 	/* update all of the pointers */
1999 	size -= headlen;
2000 	if (size) {
2001 		skb_add_rx_frag(skb, 0, rx_buffer->page,
2002 				(va + headlen) - page_address(rx_buffer->page),
2003 				size, truesize);
2004 		igc_rx_buffer_flip(rx_buffer, truesize);
2005 	} else {
2006 		rx_buffer->pagecnt_bias++;
2007 	}
2008 
2009 	return skb;
2010 }
2011 
2012 /**
2013  * igc_reuse_rx_page - page flip buffer and store it back on the ring
2014  * @rx_ring: rx descriptor ring to store buffers on
2015  * @old_buff: donor buffer to have page reused
2016  *
2017  * Synchronizes page for reuse by the adapter
2018  */
igc_reuse_rx_page(struct igc_ring * rx_ring,struct igc_rx_buffer * old_buff)2019 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
2020 			      struct igc_rx_buffer *old_buff)
2021 {
2022 	u16 nta = rx_ring->next_to_alloc;
2023 	struct igc_rx_buffer *new_buff;
2024 
2025 	new_buff = &rx_ring->rx_buffer_info[nta];
2026 
2027 	/* update, and store next to alloc */
2028 	nta++;
2029 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
2030 
2031 	/* Transfer page from old buffer to new buffer.
2032 	 * Move each member individually to avoid possible store
2033 	 * forwarding stalls.
2034 	 */
2035 	new_buff->dma		= old_buff->dma;
2036 	new_buff->page		= old_buff->page;
2037 	new_buff->page_offset	= old_buff->page_offset;
2038 	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
2039 }
2040 
igc_can_reuse_rx_page(struct igc_rx_buffer * rx_buffer,int rx_buffer_pgcnt)2041 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer,
2042 				  int rx_buffer_pgcnt)
2043 {
2044 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
2045 	struct page *page = rx_buffer->page;
2046 
2047 	/* avoid re-using remote and pfmemalloc pages */
2048 	if (!dev_page_is_reusable(page))
2049 		return false;
2050 
2051 #if (PAGE_SIZE < 8192)
2052 	/* if we are only owner of page we can reuse it */
2053 	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
2054 		return false;
2055 #else
2056 #define IGC_LAST_OFFSET \
2057 	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
2058 
2059 	if (rx_buffer->page_offset > IGC_LAST_OFFSET)
2060 		return false;
2061 #endif
2062 
2063 	/* If we have drained the page fragment pool we need to update
2064 	 * the pagecnt_bias and page count so that we fully restock the
2065 	 * number of references the driver holds.
2066 	 */
2067 	if (unlikely(pagecnt_bias == 1)) {
2068 		page_ref_add(page, USHRT_MAX - 1);
2069 		rx_buffer->pagecnt_bias = USHRT_MAX;
2070 	}
2071 
2072 	return true;
2073 }
2074 
2075 /**
2076  * igc_is_non_eop - process handling of non-EOP buffers
2077  * @rx_ring: Rx ring being processed
2078  * @rx_desc: Rx descriptor for current buffer
2079  *
2080  * This function updates next to clean.  If the buffer is an EOP buffer
2081  * this function exits returning false, otherwise it will place the
2082  * sk_buff in the next buffer to be chained and return true indicating
2083  * that this is in fact a non-EOP buffer.
2084  */
igc_is_non_eop(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc)2085 static bool igc_is_non_eop(struct igc_ring *rx_ring,
2086 			   union igc_adv_rx_desc *rx_desc)
2087 {
2088 	u32 ntc = rx_ring->next_to_clean + 1;
2089 
2090 	/* fetch, update, and store next to clean */
2091 	ntc = (ntc < rx_ring->count) ? ntc : 0;
2092 	rx_ring->next_to_clean = ntc;
2093 
2094 	prefetch(IGC_RX_DESC(rx_ring, ntc));
2095 
2096 	if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
2097 		return false;
2098 
2099 	return true;
2100 }
2101 
2102 /**
2103  * igc_cleanup_headers - Correct corrupted or empty headers
2104  * @rx_ring: rx descriptor ring packet is being transacted on
2105  * @rx_desc: pointer to the EOP Rx descriptor
2106  * @skb: pointer to current skb being fixed
2107  *
2108  * Address the case where we are pulling data in on pages only
2109  * and as such no data is present in the skb header.
2110  *
2111  * In addition if skb is not at least 60 bytes we need to pad it so that
2112  * it is large enough to qualify as a valid Ethernet frame.
2113  *
2114  * Returns true if an error was encountered and skb was freed.
2115  */
igc_cleanup_headers(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)2116 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
2117 				union igc_adv_rx_desc *rx_desc,
2118 				struct sk_buff *skb)
2119 {
2120 	/* XDP packets use error pointer so abort at this point */
2121 	if (IS_ERR(skb))
2122 		return true;
2123 
2124 	if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
2125 		struct net_device *netdev = rx_ring->netdev;
2126 
2127 		if (!(netdev->features & NETIF_F_RXALL)) {
2128 			dev_kfree_skb_any(skb);
2129 			return true;
2130 		}
2131 	}
2132 
2133 	/* if eth_skb_pad returns an error the skb was freed */
2134 	if (eth_skb_pad(skb))
2135 		return true;
2136 
2137 	return false;
2138 }
2139 
igc_put_rx_buffer(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,int rx_buffer_pgcnt)2140 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
2141 			      struct igc_rx_buffer *rx_buffer,
2142 			      int rx_buffer_pgcnt)
2143 {
2144 	if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
2145 		/* hand second half of page back to the ring */
2146 		igc_reuse_rx_page(rx_ring, rx_buffer);
2147 	} else {
2148 		/* We are not reusing the buffer so unmap it and free
2149 		 * any references we are holding to it
2150 		 */
2151 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
2152 				     igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
2153 				     IGC_RX_DMA_ATTR);
2154 		__page_frag_cache_drain(rx_buffer->page,
2155 					rx_buffer->pagecnt_bias);
2156 	}
2157 
2158 	/* clear contents of rx_buffer */
2159 	rx_buffer->page = NULL;
2160 }
2161 
igc_rx_offset(struct igc_ring * rx_ring)2162 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
2163 {
2164 	struct igc_adapter *adapter = rx_ring->q_vector->adapter;
2165 
2166 	if (ring_uses_build_skb(rx_ring))
2167 		return IGC_SKB_PAD;
2168 	if (igc_xdp_is_enabled(adapter))
2169 		return XDP_PACKET_HEADROOM;
2170 
2171 	return 0;
2172 }
2173 
igc_alloc_mapped_page(struct igc_ring * rx_ring,struct igc_rx_buffer * bi)2174 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
2175 				  struct igc_rx_buffer *bi)
2176 {
2177 	struct page *page = bi->page;
2178 	dma_addr_t dma;
2179 
2180 	/* since we are recycling buffers we should seldom need to alloc */
2181 	if (likely(page))
2182 		return true;
2183 
2184 	/* alloc new page for storage */
2185 	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
2186 	if (unlikely(!page)) {
2187 		rx_ring->rx_stats.alloc_failed++;
2188 		return false;
2189 	}
2190 
2191 	/* map page for use */
2192 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
2193 				 igc_rx_pg_size(rx_ring),
2194 				 DMA_FROM_DEVICE,
2195 				 IGC_RX_DMA_ATTR);
2196 
2197 	/* if mapping failed free memory back to system since
2198 	 * there isn't much point in holding memory we can't use
2199 	 */
2200 	if (dma_mapping_error(rx_ring->dev, dma)) {
2201 		__free_page(page);
2202 
2203 		rx_ring->rx_stats.alloc_failed++;
2204 		return false;
2205 	}
2206 
2207 	bi->dma = dma;
2208 	bi->page = page;
2209 	bi->page_offset = igc_rx_offset(rx_ring);
2210 	page_ref_add(page, USHRT_MAX - 1);
2211 	bi->pagecnt_bias = USHRT_MAX;
2212 
2213 	return true;
2214 }
2215 
2216 /**
2217  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2218  * @rx_ring: rx descriptor ring
2219  * @cleaned_count: number of buffers to clean
2220  */
igc_alloc_rx_buffers(struct igc_ring * rx_ring,u16 cleaned_count)2221 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
2222 {
2223 	union igc_adv_rx_desc *rx_desc;
2224 	u16 i = rx_ring->next_to_use;
2225 	struct igc_rx_buffer *bi;
2226 	u16 bufsz;
2227 
2228 	/* nothing to do */
2229 	if (!cleaned_count)
2230 		return;
2231 
2232 	rx_desc = IGC_RX_DESC(rx_ring, i);
2233 	bi = &rx_ring->rx_buffer_info[i];
2234 	i -= rx_ring->count;
2235 
2236 	bufsz = igc_rx_bufsz(rx_ring);
2237 
2238 	do {
2239 		if (!igc_alloc_mapped_page(rx_ring, bi))
2240 			break;
2241 
2242 		/* sync the buffer for use by the device */
2243 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
2244 						 bi->page_offset, bufsz,
2245 						 DMA_FROM_DEVICE);
2246 
2247 		/* Refresh the desc even if buffer_addrs didn't change
2248 		 * because each write-back erases this info.
2249 		 */
2250 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
2251 
2252 		rx_desc++;
2253 		bi++;
2254 		i++;
2255 		if (unlikely(!i)) {
2256 			rx_desc = IGC_RX_DESC(rx_ring, 0);
2257 			bi = rx_ring->rx_buffer_info;
2258 			i -= rx_ring->count;
2259 		}
2260 
2261 		/* clear the length for the next_to_use descriptor */
2262 		rx_desc->wb.upper.length = 0;
2263 
2264 		cleaned_count--;
2265 	} while (cleaned_count);
2266 
2267 	i += rx_ring->count;
2268 
2269 	if (rx_ring->next_to_use != i) {
2270 		/* record the next descriptor to use */
2271 		rx_ring->next_to_use = i;
2272 
2273 		/* update next to alloc since we have filled the ring */
2274 		rx_ring->next_to_alloc = i;
2275 
2276 		/* Force memory writes to complete before letting h/w
2277 		 * know there are new descriptors to fetch.  (Only
2278 		 * applicable for weak-ordered memory model archs,
2279 		 * such as IA-64).
2280 		 */
2281 		wmb();
2282 		writel(i, rx_ring->tail);
2283 	}
2284 }
2285 
igc_alloc_rx_buffers_zc(struct igc_ring * ring,u16 count)2286 static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
2287 {
2288 	union igc_adv_rx_desc *desc;
2289 	u16 i = ring->next_to_use;
2290 	struct igc_rx_buffer *bi;
2291 	dma_addr_t dma;
2292 	bool ok = true;
2293 
2294 	if (!count)
2295 		return ok;
2296 
2297 	XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff);
2298 
2299 	desc = IGC_RX_DESC(ring, i);
2300 	bi = &ring->rx_buffer_info[i];
2301 	i -= ring->count;
2302 
2303 	do {
2304 		bi->xdp = xsk_buff_alloc(ring->xsk_pool);
2305 		if (!bi->xdp) {
2306 			ok = false;
2307 			break;
2308 		}
2309 
2310 		dma = xsk_buff_xdp_get_dma(bi->xdp);
2311 		desc->read.pkt_addr = cpu_to_le64(dma);
2312 
2313 		desc++;
2314 		bi++;
2315 		i++;
2316 		if (unlikely(!i)) {
2317 			desc = IGC_RX_DESC(ring, 0);
2318 			bi = ring->rx_buffer_info;
2319 			i -= ring->count;
2320 		}
2321 
2322 		/* Clear the length for the next_to_use descriptor. */
2323 		desc->wb.upper.length = 0;
2324 
2325 		count--;
2326 	} while (count);
2327 
2328 	i += ring->count;
2329 
2330 	if (ring->next_to_use != i) {
2331 		ring->next_to_use = i;
2332 
2333 		/* Force memory writes to complete before letting h/w
2334 		 * know there are new descriptors to fetch.  (Only
2335 		 * applicable for weak-ordered memory model archs,
2336 		 * such as IA-64).
2337 		 */
2338 		wmb();
2339 		writel(i, ring->tail);
2340 	}
2341 
2342 	return ok;
2343 }
2344 
2345 /* This function requires __netif_tx_lock is held by the caller. */
igc_xdp_init_tx_descriptor(struct igc_ring * ring,struct xdp_frame * xdpf)2346 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
2347 				      struct xdp_frame *xdpf)
2348 {
2349 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
2350 	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
2351 	u16 count, index = ring->next_to_use;
2352 	struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
2353 	struct igc_tx_buffer *buffer = head;
2354 	union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
2355 	u32 olinfo_status, len = xdpf->len, cmd_type;
2356 	void *data = xdpf->data;
2357 	u16 i;
2358 
2359 	count = TXD_USE_COUNT(len);
2360 	for (i = 0; i < nr_frags; i++)
2361 		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
2362 
2363 	if (igc_maybe_stop_tx(ring, count + 3)) {
2364 		/* this is a hard error */
2365 		return -EBUSY;
2366 	}
2367 
2368 	i = 0;
2369 	head->bytecount = xdp_get_frame_len(xdpf);
2370 	head->type = IGC_TX_BUFFER_TYPE_XDP;
2371 	head->gso_segs = 1;
2372 	head->xdpf = xdpf;
2373 
2374 	olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
2375 	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2376 
2377 	for (;;) {
2378 		dma_addr_t dma;
2379 
2380 		dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
2381 		if (dma_mapping_error(ring->dev, dma)) {
2382 			netdev_err_once(ring->netdev,
2383 					"Failed to map DMA for TX\n");
2384 			goto unmap;
2385 		}
2386 
2387 		dma_unmap_len_set(buffer, len, len);
2388 		dma_unmap_addr_set(buffer, dma, dma);
2389 
2390 		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2391 			   IGC_ADVTXD_DCMD_IFCS | len;
2392 
2393 		desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2394 		desc->read.buffer_addr = cpu_to_le64(dma);
2395 
2396 		buffer->protocol = 0;
2397 
2398 		if (++index == ring->count)
2399 			index = 0;
2400 
2401 		if (i == nr_frags)
2402 			break;
2403 
2404 		buffer = &ring->tx_buffer_info[index];
2405 		desc = IGC_TX_DESC(ring, index);
2406 		desc->read.olinfo_status = 0;
2407 
2408 		data = skb_frag_address(&sinfo->frags[i]);
2409 		len = skb_frag_size(&sinfo->frags[i]);
2410 		i++;
2411 	}
2412 	desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
2413 
2414 	netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
2415 	/* set the timestamp */
2416 	head->time_stamp = jiffies;
2417 	/* set next_to_watch value indicating a packet is present */
2418 	head->next_to_watch = desc;
2419 	ring->next_to_use = index;
2420 
2421 	return 0;
2422 
2423 unmap:
2424 	for (;;) {
2425 		buffer = &ring->tx_buffer_info[index];
2426 		if (dma_unmap_len(buffer, len))
2427 			dma_unmap_page(ring->dev,
2428 				       dma_unmap_addr(buffer, dma),
2429 				       dma_unmap_len(buffer, len),
2430 				       DMA_TO_DEVICE);
2431 		dma_unmap_len_set(buffer, len, 0);
2432 		if (buffer == head)
2433 			break;
2434 
2435 		if (!index)
2436 			index += ring->count;
2437 		index--;
2438 	}
2439 
2440 	return -ENOMEM;
2441 }
2442 
igc_xdp_get_tx_ring(struct igc_adapter * adapter,int cpu)2443 static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
2444 					    int cpu)
2445 {
2446 	int index = cpu;
2447 
2448 	if (unlikely(index < 0))
2449 		index = 0;
2450 
2451 	while (index >= adapter->num_tx_queues)
2452 		index -= adapter->num_tx_queues;
2453 
2454 	return adapter->tx_ring[index];
2455 }
2456 
igc_xdp_xmit_back(struct igc_adapter * adapter,struct xdp_buff * xdp)2457 static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp)
2458 {
2459 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2460 	int cpu = smp_processor_id();
2461 	struct netdev_queue *nq;
2462 	struct igc_ring *ring;
2463 	int res;
2464 
2465 	if (unlikely(!xdpf))
2466 		return -EFAULT;
2467 
2468 	ring = igc_xdp_get_tx_ring(adapter, cpu);
2469 	nq = txring_txq(ring);
2470 
2471 	__netif_tx_lock(nq, cpu);
2472 	/* Avoid transmit queue timeout since we share it with the slow path */
2473 	txq_trans_cond_update(nq);
2474 	res = igc_xdp_init_tx_descriptor(ring, xdpf);
2475 	__netif_tx_unlock(nq);
2476 	return res;
2477 }
2478 
2479 /* This function assumes rcu_read_lock() is held by the caller. */
__igc_xdp_run_prog(struct igc_adapter * adapter,struct bpf_prog * prog,struct xdp_buff * xdp)2480 static int __igc_xdp_run_prog(struct igc_adapter *adapter,
2481 			      struct bpf_prog *prog,
2482 			      struct xdp_buff *xdp)
2483 {
2484 	u32 act = bpf_prog_run_xdp(prog, xdp);
2485 
2486 	switch (act) {
2487 	case XDP_PASS:
2488 		return IGC_XDP_PASS;
2489 	case XDP_TX:
2490 		if (igc_xdp_xmit_back(adapter, xdp) < 0)
2491 			goto out_failure;
2492 		return IGC_XDP_TX;
2493 	case XDP_REDIRECT:
2494 		if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0)
2495 			goto out_failure;
2496 		return IGC_XDP_REDIRECT;
2497 		break;
2498 	default:
2499 		bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
2500 		fallthrough;
2501 	case XDP_ABORTED:
2502 out_failure:
2503 		trace_xdp_exception(adapter->netdev, prog, act);
2504 		fallthrough;
2505 	case XDP_DROP:
2506 		return IGC_XDP_CONSUMED;
2507 	}
2508 }
2509 
igc_xdp_run_prog(struct igc_adapter * adapter,struct xdp_buff * xdp)2510 static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
2511 					struct xdp_buff *xdp)
2512 {
2513 	struct bpf_prog *prog;
2514 	int res;
2515 
2516 	prog = READ_ONCE(adapter->xdp_prog);
2517 	if (!prog) {
2518 		res = IGC_XDP_PASS;
2519 		goto out;
2520 	}
2521 
2522 	res = __igc_xdp_run_prog(adapter, prog, xdp);
2523 
2524 out:
2525 	return ERR_PTR(-res);
2526 }
2527 
2528 /* This function assumes __netif_tx_lock is held by the caller. */
igc_flush_tx_descriptors(struct igc_ring * ring)2529 static void igc_flush_tx_descriptors(struct igc_ring *ring)
2530 {
2531 	/* Once tail pointer is updated, hardware can fetch the descriptors
2532 	 * any time so we issue a write membar here to ensure all memory
2533 	 * writes are complete before the tail pointer is updated.
2534 	 */
2535 	wmb();
2536 	writel(ring->next_to_use, ring->tail);
2537 }
2538 
igc_finalize_xdp(struct igc_adapter * adapter,int status)2539 static void igc_finalize_xdp(struct igc_adapter *adapter, int status)
2540 {
2541 	int cpu = smp_processor_id();
2542 	struct netdev_queue *nq;
2543 	struct igc_ring *ring;
2544 
2545 	if (status & IGC_XDP_TX) {
2546 		ring = igc_xdp_get_tx_ring(adapter, cpu);
2547 		nq = txring_txq(ring);
2548 
2549 		__netif_tx_lock(nq, cpu);
2550 		igc_flush_tx_descriptors(ring);
2551 		__netif_tx_unlock(nq);
2552 	}
2553 
2554 	if (status & IGC_XDP_REDIRECT)
2555 		xdp_do_flush();
2556 }
2557 
igc_update_rx_stats(struct igc_q_vector * q_vector,unsigned int packets,unsigned int bytes)2558 static void igc_update_rx_stats(struct igc_q_vector *q_vector,
2559 				unsigned int packets, unsigned int bytes)
2560 {
2561 	struct igc_ring *ring = q_vector->rx.ring;
2562 
2563 	u64_stats_update_begin(&ring->rx_syncp);
2564 	ring->rx_stats.packets += packets;
2565 	ring->rx_stats.bytes += bytes;
2566 	u64_stats_update_end(&ring->rx_syncp);
2567 
2568 	q_vector->rx.total_packets += packets;
2569 	q_vector->rx.total_bytes += bytes;
2570 }
2571 
igc_clean_rx_irq(struct igc_q_vector * q_vector,const int budget)2572 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
2573 {
2574 	unsigned int total_bytes = 0, total_packets = 0;
2575 	struct igc_adapter *adapter = q_vector->adapter;
2576 	struct igc_ring *rx_ring = q_vector->rx.ring;
2577 	struct sk_buff *skb = rx_ring->skb;
2578 	u16 cleaned_count = igc_desc_unused(rx_ring);
2579 	int xdp_status = 0, rx_buffer_pgcnt;
2580 
2581 	while (likely(total_packets < budget)) {
2582 		union igc_adv_rx_desc *rx_desc;
2583 		struct igc_rx_buffer *rx_buffer;
2584 		unsigned int size, truesize;
2585 		struct igc_xdp_buff ctx;
2586 		ktime_t timestamp = 0;
2587 		int pkt_offset = 0;
2588 		void *pktbuf;
2589 
2590 		/* return some buffers to hardware, one at a time is too slow */
2591 		if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
2592 			igc_alloc_rx_buffers(rx_ring, cleaned_count);
2593 			cleaned_count = 0;
2594 		}
2595 
2596 		rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
2597 		size = le16_to_cpu(rx_desc->wb.upper.length);
2598 		if (!size)
2599 			break;
2600 
2601 		/* This memory barrier is needed to keep us from reading
2602 		 * any other fields out of the rx_desc until we know the
2603 		 * descriptor has been written back
2604 		 */
2605 		dma_rmb();
2606 
2607 		rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
2608 		truesize = igc_get_rx_frame_truesize(rx_ring, size);
2609 
2610 		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
2611 
2612 		if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
2613 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2614 							pktbuf);
2615 			ctx.rx_ts = timestamp;
2616 			pkt_offset = IGC_TS_HDR_LEN;
2617 			size -= IGC_TS_HDR_LEN;
2618 		}
2619 
2620 		if (!skb) {
2621 			xdp_init_buff(&ctx.xdp, truesize, &rx_ring->xdp_rxq);
2622 			xdp_prepare_buff(&ctx.xdp, pktbuf - igc_rx_offset(rx_ring),
2623 					 igc_rx_offset(rx_ring) + pkt_offset,
2624 					 size, true);
2625 			xdp_buff_clear_frags_flag(&ctx.xdp);
2626 			ctx.rx_desc = rx_desc;
2627 
2628 			skb = igc_xdp_run_prog(adapter, &ctx.xdp);
2629 		}
2630 
2631 		if (IS_ERR(skb)) {
2632 			unsigned int xdp_res = -PTR_ERR(skb);
2633 
2634 			switch (xdp_res) {
2635 			case IGC_XDP_CONSUMED:
2636 				rx_buffer->pagecnt_bias++;
2637 				break;
2638 			case IGC_XDP_TX:
2639 			case IGC_XDP_REDIRECT:
2640 				igc_rx_buffer_flip(rx_buffer, truesize);
2641 				xdp_status |= xdp_res;
2642 				break;
2643 			}
2644 
2645 			total_packets++;
2646 			total_bytes += size;
2647 		} else if (skb)
2648 			igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
2649 		else if (ring_uses_build_skb(rx_ring))
2650 			skb = igc_build_skb(rx_ring, rx_buffer, &ctx.xdp);
2651 		else
2652 			skb = igc_construct_skb(rx_ring, rx_buffer, &ctx.xdp,
2653 						timestamp);
2654 
2655 		/* exit if we failed to retrieve a buffer */
2656 		if (!skb) {
2657 			rx_ring->rx_stats.alloc_failed++;
2658 			rx_buffer->pagecnt_bias++;
2659 			break;
2660 		}
2661 
2662 		igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
2663 		cleaned_count++;
2664 
2665 		/* fetch next buffer in frame if non-eop */
2666 		if (igc_is_non_eop(rx_ring, rx_desc))
2667 			continue;
2668 
2669 		/* verify the packet layout is correct */
2670 		if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
2671 			skb = NULL;
2672 			continue;
2673 		}
2674 
2675 		/* probably a little skewed due to removing CRC */
2676 		total_bytes += skb->len;
2677 
2678 		/* populate checksum, VLAN, and protocol */
2679 		igc_process_skb_fields(rx_ring, rx_desc, skb);
2680 
2681 		napi_gro_receive(&q_vector->napi, skb);
2682 
2683 		/* reset skb pointer */
2684 		skb = NULL;
2685 
2686 		/* update budget accounting */
2687 		total_packets++;
2688 	}
2689 
2690 	if (xdp_status)
2691 		igc_finalize_xdp(adapter, xdp_status);
2692 
2693 	/* place incomplete frames back on ring for completion */
2694 	rx_ring->skb = skb;
2695 
2696 	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2697 
2698 	if (cleaned_count)
2699 		igc_alloc_rx_buffers(rx_ring, cleaned_count);
2700 
2701 	return total_packets;
2702 }
2703 
igc_construct_skb_zc(struct igc_ring * ring,struct xdp_buff * xdp)2704 static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
2705 					    struct xdp_buff *xdp)
2706 {
2707 	unsigned int totalsize = xdp->data_end - xdp->data_meta;
2708 	unsigned int metasize = xdp->data - xdp->data_meta;
2709 	struct sk_buff *skb;
2710 
2711 	net_prefetch(xdp->data_meta);
2712 
2713 	skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,
2714 			       GFP_ATOMIC | __GFP_NOWARN);
2715 	if (unlikely(!skb))
2716 		return NULL;
2717 
2718 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,
2719 	       ALIGN(totalsize, sizeof(long)));
2720 
2721 	if (metasize) {
2722 		skb_metadata_set(skb, metasize);
2723 		__skb_pull(skb, metasize);
2724 	}
2725 
2726 	return skb;
2727 }
2728 
igc_dispatch_skb_zc(struct igc_q_vector * q_vector,union igc_adv_rx_desc * desc,struct xdp_buff * xdp,ktime_t timestamp)2729 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,
2730 				union igc_adv_rx_desc *desc,
2731 				struct xdp_buff *xdp,
2732 				ktime_t timestamp)
2733 {
2734 	struct igc_ring *ring = q_vector->rx.ring;
2735 	struct sk_buff *skb;
2736 
2737 	skb = igc_construct_skb_zc(ring, xdp);
2738 	if (!skb) {
2739 		ring->rx_stats.alloc_failed++;
2740 		return;
2741 	}
2742 
2743 	if (timestamp)
2744 		skb_hwtstamps(skb)->hwtstamp = timestamp;
2745 
2746 	if (igc_cleanup_headers(ring, desc, skb))
2747 		return;
2748 
2749 	igc_process_skb_fields(ring, desc, skb);
2750 	napi_gro_receive(&q_vector->napi, skb);
2751 }
2752 
xsk_buff_to_igc_ctx(struct xdp_buff * xdp)2753 static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp)
2754 {
2755 	/* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The
2756 	 * igc_xdp_buff shares its layout with xdp_buff_xsk and private
2757 	 * igc_xdp_buff fields fall into xdp_buff_xsk->cb
2758 	 */
2759        return (struct igc_xdp_buff *)xdp;
2760 }
2761 
igc_clean_rx_irq_zc(struct igc_q_vector * q_vector,const int budget)2762 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
2763 {
2764 	struct igc_adapter *adapter = q_vector->adapter;
2765 	struct igc_ring *ring = q_vector->rx.ring;
2766 	u16 cleaned_count = igc_desc_unused(ring);
2767 	int total_bytes = 0, total_packets = 0;
2768 	u16 ntc = ring->next_to_clean;
2769 	struct bpf_prog *prog;
2770 	bool failure = false;
2771 	int xdp_status = 0;
2772 
2773 	rcu_read_lock();
2774 
2775 	prog = READ_ONCE(adapter->xdp_prog);
2776 
2777 	while (likely(total_packets < budget)) {
2778 		union igc_adv_rx_desc *desc;
2779 		struct igc_rx_buffer *bi;
2780 		struct igc_xdp_buff *ctx;
2781 		ktime_t timestamp = 0;
2782 		unsigned int size;
2783 		int res;
2784 
2785 		desc = IGC_RX_DESC(ring, ntc);
2786 		size = le16_to_cpu(desc->wb.upper.length);
2787 		if (!size)
2788 			break;
2789 
2790 		/* This memory barrier is needed to keep us from reading
2791 		 * any other fields out of the rx_desc until we know the
2792 		 * descriptor has been written back
2793 		 */
2794 		dma_rmb();
2795 
2796 		bi = &ring->rx_buffer_info[ntc];
2797 
2798 		ctx = xsk_buff_to_igc_ctx(bi->xdp);
2799 		ctx->rx_desc = desc;
2800 
2801 		if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
2802 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2803 							bi->xdp->data);
2804 			ctx->rx_ts = timestamp;
2805 
2806 			bi->xdp->data += IGC_TS_HDR_LEN;
2807 
2808 			/* HW timestamp has been copied into local variable. Metadata
2809 			 * length when XDP program is called should be 0.
2810 			 */
2811 			bi->xdp->data_meta += IGC_TS_HDR_LEN;
2812 			size -= IGC_TS_HDR_LEN;
2813 		}
2814 
2815 		bi->xdp->data_end = bi->xdp->data + size;
2816 		xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool);
2817 
2818 		res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
2819 		switch (res) {
2820 		case IGC_XDP_PASS:
2821 			igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);
2822 			fallthrough;
2823 		case IGC_XDP_CONSUMED:
2824 			xsk_buff_free(bi->xdp);
2825 			break;
2826 		case IGC_XDP_TX:
2827 		case IGC_XDP_REDIRECT:
2828 			xdp_status |= res;
2829 			break;
2830 		}
2831 
2832 		bi->xdp = NULL;
2833 		total_bytes += size;
2834 		total_packets++;
2835 		cleaned_count++;
2836 		ntc++;
2837 		if (ntc == ring->count)
2838 			ntc = 0;
2839 	}
2840 
2841 	ring->next_to_clean = ntc;
2842 	rcu_read_unlock();
2843 
2844 	if (cleaned_count >= IGC_RX_BUFFER_WRITE)
2845 		failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count);
2846 
2847 	if (xdp_status)
2848 		igc_finalize_xdp(adapter, xdp_status);
2849 
2850 	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2851 
2852 	if (xsk_uses_need_wakeup(ring->xsk_pool)) {
2853 		if (failure || ring->next_to_clean == ring->next_to_use)
2854 			xsk_set_rx_need_wakeup(ring->xsk_pool);
2855 		else
2856 			xsk_clear_rx_need_wakeup(ring->xsk_pool);
2857 		return total_packets;
2858 	}
2859 
2860 	return failure ? budget : total_packets;
2861 }
2862 
igc_update_tx_stats(struct igc_q_vector * q_vector,unsigned int packets,unsigned int bytes)2863 static void igc_update_tx_stats(struct igc_q_vector *q_vector,
2864 				unsigned int packets, unsigned int bytes)
2865 {
2866 	struct igc_ring *ring = q_vector->tx.ring;
2867 
2868 	u64_stats_update_begin(&ring->tx_syncp);
2869 	ring->tx_stats.bytes += bytes;
2870 	ring->tx_stats.packets += packets;
2871 	u64_stats_update_end(&ring->tx_syncp);
2872 
2873 	q_vector->tx.total_bytes += bytes;
2874 	q_vector->tx.total_packets += packets;
2875 }
2876 
igc_xdp_xmit_zc(struct igc_ring * ring)2877 static void igc_xdp_xmit_zc(struct igc_ring *ring)
2878 {
2879 	struct xsk_buff_pool *pool = ring->xsk_pool;
2880 	struct netdev_queue *nq = txring_txq(ring);
2881 	union igc_adv_tx_desc *tx_desc = NULL;
2882 	int cpu = smp_processor_id();
2883 	struct xdp_desc xdp_desc;
2884 	u16 budget, ntu;
2885 
2886 	if (!netif_carrier_ok(ring->netdev))
2887 		return;
2888 
2889 	__netif_tx_lock(nq, cpu);
2890 
2891 	/* Avoid transmit queue timeout since we share it with the slow path */
2892 	txq_trans_cond_update(nq);
2893 
2894 	ntu = ring->next_to_use;
2895 	budget = igc_desc_unused(ring);
2896 
2897 	while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) {
2898 		u32 cmd_type, olinfo_status;
2899 		struct igc_tx_buffer *bi;
2900 		dma_addr_t dma;
2901 
2902 		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2903 			   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
2904 			   xdp_desc.len;
2905 		olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT;
2906 
2907 		dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
2908 		xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len);
2909 
2910 		tx_desc = IGC_TX_DESC(ring, ntu);
2911 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2912 		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2913 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
2914 
2915 		bi = &ring->tx_buffer_info[ntu];
2916 		bi->type = IGC_TX_BUFFER_TYPE_XSK;
2917 		bi->protocol = 0;
2918 		bi->bytecount = xdp_desc.len;
2919 		bi->gso_segs = 1;
2920 		bi->time_stamp = jiffies;
2921 		bi->next_to_watch = tx_desc;
2922 
2923 		netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len);
2924 
2925 		ntu++;
2926 		if (ntu == ring->count)
2927 			ntu = 0;
2928 	}
2929 
2930 	ring->next_to_use = ntu;
2931 	if (tx_desc) {
2932 		igc_flush_tx_descriptors(ring);
2933 		xsk_tx_release(pool);
2934 	}
2935 
2936 	__netif_tx_unlock(nq);
2937 }
2938 
2939 /**
2940  * igc_clean_tx_irq - Reclaim resources after transmit completes
2941  * @q_vector: pointer to q_vector containing needed info
2942  * @napi_budget: Used to determine if we are in netpoll
2943  *
2944  * returns true if ring is completely cleaned
2945  */
igc_clean_tx_irq(struct igc_q_vector * q_vector,int napi_budget)2946 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
2947 {
2948 	struct igc_adapter *adapter = q_vector->adapter;
2949 	unsigned int total_bytes = 0, total_packets = 0;
2950 	unsigned int budget = q_vector->tx.work_limit;
2951 	struct igc_ring *tx_ring = q_vector->tx.ring;
2952 	unsigned int i = tx_ring->next_to_clean;
2953 	struct igc_tx_buffer *tx_buffer;
2954 	union igc_adv_tx_desc *tx_desc;
2955 	u32 xsk_frames = 0;
2956 
2957 	if (test_bit(__IGC_DOWN, &adapter->state))
2958 		return true;
2959 
2960 	tx_buffer = &tx_ring->tx_buffer_info[i];
2961 	tx_desc = IGC_TX_DESC(tx_ring, i);
2962 	i -= tx_ring->count;
2963 
2964 	do {
2965 		union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
2966 
2967 		/* if next_to_watch is not set then there is no work pending */
2968 		if (!eop_desc)
2969 			break;
2970 
2971 		/* prevent any other reads prior to eop_desc */
2972 		smp_rmb();
2973 
2974 		/* if DD is not set pending work has not been completed */
2975 		if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
2976 			break;
2977 
2978 		/* clear next_to_watch to prevent false hangs */
2979 		tx_buffer->next_to_watch = NULL;
2980 
2981 		/* update the statistics for this packet */
2982 		total_bytes += tx_buffer->bytecount;
2983 		total_packets += tx_buffer->gso_segs;
2984 
2985 		switch (tx_buffer->type) {
2986 		case IGC_TX_BUFFER_TYPE_XSK:
2987 			xsk_frames++;
2988 			break;
2989 		case IGC_TX_BUFFER_TYPE_XDP:
2990 			xdp_return_frame(tx_buffer->xdpf);
2991 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2992 			break;
2993 		case IGC_TX_BUFFER_TYPE_SKB:
2994 			napi_consume_skb(tx_buffer->skb, napi_budget);
2995 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2996 			break;
2997 		default:
2998 			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
2999 			break;
3000 		}
3001 
3002 		/* clear last DMA location and unmap remaining buffers */
3003 		while (tx_desc != eop_desc) {
3004 			tx_buffer++;
3005 			tx_desc++;
3006 			i++;
3007 			if (unlikely(!i)) {
3008 				i -= tx_ring->count;
3009 				tx_buffer = tx_ring->tx_buffer_info;
3010 				tx_desc = IGC_TX_DESC(tx_ring, 0);
3011 			}
3012 
3013 			/* unmap any remaining paged data */
3014 			if (dma_unmap_len(tx_buffer, len))
3015 				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
3016 		}
3017 
3018 		/* move us one more past the eop_desc for start of next pkt */
3019 		tx_buffer++;
3020 		tx_desc++;
3021 		i++;
3022 		if (unlikely(!i)) {
3023 			i -= tx_ring->count;
3024 			tx_buffer = tx_ring->tx_buffer_info;
3025 			tx_desc = IGC_TX_DESC(tx_ring, 0);
3026 		}
3027 
3028 		/* issue prefetch for next Tx descriptor */
3029 		prefetch(tx_desc);
3030 
3031 		/* update budget accounting */
3032 		budget--;
3033 	} while (likely(budget));
3034 
3035 	netdev_tx_completed_queue(txring_txq(tx_ring),
3036 				  total_packets, total_bytes);
3037 
3038 	i += tx_ring->count;
3039 	tx_ring->next_to_clean = i;
3040 
3041 	igc_update_tx_stats(q_vector, total_packets, total_bytes);
3042 
3043 	if (tx_ring->xsk_pool) {
3044 		if (xsk_frames)
3045 			xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
3046 		if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
3047 			xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
3048 		igc_xdp_xmit_zc(tx_ring);
3049 	}
3050 
3051 	if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
3052 		struct igc_hw *hw = &adapter->hw;
3053 
3054 		/* Detect a transmit hang in hardware, this serializes the
3055 		 * check with the clearing of time_stamp and movement of i
3056 		 */
3057 		clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3058 		if (tx_buffer->next_to_watch &&
3059 		    time_after(jiffies, tx_buffer->time_stamp +
3060 		    (adapter->tx_timeout_factor * HZ)) &&
3061 		    !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF) &&
3062 		    (rd32(IGC_TDH(tx_ring->reg_idx)) != readl(tx_ring->tail)) &&
3063 		    !tx_ring->oper_gate_closed) {
3064 			/* detected Tx unit hang */
3065 			netdev_err(tx_ring->netdev,
3066 				   "Detected Tx Unit Hang\n"
3067 				   "  Tx Queue             <%d>\n"
3068 				   "  TDH                  <%x>\n"
3069 				   "  TDT                  <%x>\n"
3070 				   "  next_to_use          <%x>\n"
3071 				   "  next_to_clean        <%x>\n"
3072 				   "buffer_info[next_to_clean]\n"
3073 				   "  time_stamp           <%lx>\n"
3074 				   "  next_to_watch        <%p>\n"
3075 				   "  jiffies              <%lx>\n"
3076 				   "  desc.status          <%x>\n",
3077 				   tx_ring->queue_index,
3078 				   rd32(IGC_TDH(tx_ring->reg_idx)),
3079 				   readl(tx_ring->tail),
3080 				   tx_ring->next_to_use,
3081 				   tx_ring->next_to_clean,
3082 				   tx_buffer->time_stamp,
3083 				   tx_buffer->next_to_watch,
3084 				   jiffies,
3085 				   tx_buffer->next_to_watch->wb.status);
3086 			netif_stop_subqueue(tx_ring->netdev,
3087 					    tx_ring->queue_index);
3088 
3089 			/* we are about to reset, no point in enabling stuff */
3090 			return true;
3091 		}
3092 	}
3093 
3094 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
3095 	if (unlikely(total_packets &&
3096 		     netif_carrier_ok(tx_ring->netdev) &&
3097 		     igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
3098 		/* Make sure that anybody stopping the queue after this
3099 		 * sees the new next_to_clean.
3100 		 */
3101 		smp_mb();
3102 		if (__netif_subqueue_stopped(tx_ring->netdev,
3103 					     tx_ring->queue_index) &&
3104 		    !(test_bit(__IGC_DOWN, &adapter->state))) {
3105 			netif_wake_subqueue(tx_ring->netdev,
3106 					    tx_ring->queue_index);
3107 
3108 			u64_stats_update_begin(&tx_ring->tx_syncp);
3109 			tx_ring->tx_stats.restart_queue++;
3110 			u64_stats_update_end(&tx_ring->tx_syncp);
3111 		}
3112 	}
3113 
3114 	return !!budget;
3115 }
3116 
igc_find_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr)3117 static int igc_find_mac_filter(struct igc_adapter *adapter,
3118 			       enum igc_mac_filter_type type, const u8 *addr)
3119 {
3120 	struct igc_hw *hw = &adapter->hw;
3121 	int max_entries = hw->mac.rar_entry_count;
3122 	u32 ral, rah;
3123 	int i;
3124 
3125 	for (i = 0; i < max_entries; i++) {
3126 		ral = rd32(IGC_RAL(i));
3127 		rah = rd32(IGC_RAH(i));
3128 
3129 		if (!(rah & IGC_RAH_AV))
3130 			continue;
3131 		if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
3132 			continue;
3133 		if ((rah & IGC_RAH_RAH_MASK) !=
3134 		    le16_to_cpup((__le16 *)(addr + 4)))
3135 			continue;
3136 		if (ral != le32_to_cpup((__le32 *)(addr)))
3137 			continue;
3138 
3139 		return i;
3140 	}
3141 
3142 	return -1;
3143 }
3144 
igc_get_avail_mac_filter_slot(struct igc_adapter * adapter)3145 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
3146 {
3147 	struct igc_hw *hw = &adapter->hw;
3148 	int max_entries = hw->mac.rar_entry_count;
3149 	u32 rah;
3150 	int i;
3151 
3152 	for (i = 0; i < max_entries; i++) {
3153 		rah = rd32(IGC_RAH(i));
3154 
3155 		if (!(rah & IGC_RAH_AV))
3156 			return i;
3157 	}
3158 
3159 	return -1;
3160 }
3161 
3162 /**
3163  * igc_add_mac_filter() - Add MAC address filter
3164  * @adapter: Pointer to adapter where the filter should be added
3165  * @type: MAC address filter type (source or destination)
3166  * @addr: MAC address
3167  * @queue: If non-negative, queue assignment feature is enabled and frames
3168  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3169  *         assignment is disabled.
3170  *
3171  * Return: 0 in case of success, negative errno code otherwise.
3172  */
igc_add_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr,int queue)3173 static int igc_add_mac_filter(struct igc_adapter *adapter,
3174 			      enum igc_mac_filter_type type, const u8 *addr,
3175 			      int queue)
3176 {
3177 	struct net_device *dev = adapter->netdev;
3178 	int index;
3179 
3180 	index = igc_find_mac_filter(adapter, type, addr);
3181 	if (index >= 0)
3182 		goto update_filter;
3183 
3184 	index = igc_get_avail_mac_filter_slot(adapter);
3185 	if (index < 0)
3186 		return -ENOSPC;
3187 
3188 	netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
3189 		   index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3190 		   addr, queue);
3191 
3192 update_filter:
3193 	igc_set_mac_filter_hw(adapter, index, type, addr, queue);
3194 	return 0;
3195 }
3196 
3197 /**
3198  * igc_del_mac_filter() - Delete MAC address filter
3199  * @adapter: Pointer to adapter where the filter should be deleted from
3200  * @type: MAC address filter type (source or destination)
3201  * @addr: MAC address
3202  */
igc_del_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr)3203 static void igc_del_mac_filter(struct igc_adapter *adapter,
3204 			       enum igc_mac_filter_type type, const u8 *addr)
3205 {
3206 	struct net_device *dev = adapter->netdev;
3207 	int index;
3208 
3209 	index = igc_find_mac_filter(adapter, type, addr);
3210 	if (index < 0)
3211 		return;
3212 
3213 	if (index == 0) {
3214 		/* If this is the default filter, we don't actually delete it.
3215 		 * We just reset to its default value i.e. disable queue
3216 		 * assignment.
3217 		 */
3218 		netdev_dbg(dev, "Disable default MAC filter queue assignment");
3219 
3220 		igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
3221 	} else {
3222 		netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
3223 			   index,
3224 			   type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3225 			   addr);
3226 
3227 		igc_clear_mac_filter_hw(adapter, index);
3228 	}
3229 }
3230 
3231 /**
3232  * igc_add_vlan_prio_filter() - Add VLAN priority filter
3233  * @adapter: Pointer to adapter where the filter should be added
3234  * @prio: VLAN priority value
3235  * @queue: Queue number which matching frames are assigned to
3236  *
3237  * Return: 0 in case of success, negative errno code otherwise.
3238  */
igc_add_vlan_prio_filter(struct igc_adapter * adapter,int prio,int queue)3239 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
3240 				    int queue)
3241 {
3242 	struct net_device *dev = adapter->netdev;
3243 	struct igc_hw *hw = &adapter->hw;
3244 	u32 vlanpqf;
3245 
3246 	vlanpqf = rd32(IGC_VLANPQF);
3247 
3248 	if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
3249 		netdev_dbg(dev, "VLAN priority filter already in use\n");
3250 		return -EEXIST;
3251 	}
3252 
3253 	vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
3254 	vlanpqf |= IGC_VLANPQF_VALID(prio);
3255 
3256 	wr32(IGC_VLANPQF, vlanpqf);
3257 
3258 	netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
3259 		   prio, queue);
3260 	return 0;
3261 }
3262 
3263 /**
3264  * igc_del_vlan_prio_filter() - Delete VLAN priority filter
3265  * @adapter: Pointer to adapter where the filter should be deleted from
3266  * @prio: VLAN priority value
3267  */
igc_del_vlan_prio_filter(struct igc_adapter * adapter,int prio)3268 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
3269 {
3270 	struct igc_hw *hw = &adapter->hw;
3271 	u32 vlanpqf;
3272 
3273 	vlanpqf = rd32(IGC_VLANPQF);
3274 
3275 	vlanpqf &= ~IGC_VLANPQF_VALID(prio);
3276 	vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
3277 
3278 	wr32(IGC_VLANPQF, vlanpqf);
3279 
3280 	netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
3281 		   prio);
3282 }
3283 
igc_get_avail_etype_filter_slot(struct igc_adapter * adapter)3284 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
3285 {
3286 	struct igc_hw *hw = &adapter->hw;
3287 	int i;
3288 
3289 	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3290 		u32 etqf = rd32(IGC_ETQF(i));
3291 
3292 		if (!(etqf & IGC_ETQF_FILTER_ENABLE))
3293 			return i;
3294 	}
3295 
3296 	return -1;
3297 }
3298 
3299 /**
3300  * igc_add_etype_filter() - Add ethertype filter
3301  * @adapter: Pointer to adapter where the filter should be added
3302  * @etype: Ethertype value
3303  * @queue: If non-negative, queue assignment feature is enabled and frames
3304  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3305  *         assignment is disabled.
3306  *
3307  * Return: 0 in case of success, negative errno code otherwise.
3308  */
igc_add_etype_filter(struct igc_adapter * adapter,u16 etype,int queue)3309 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
3310 				int queue)
3311 {
3312 	struct igc_hw *hw = &adapter->hw;
3313 	int index;
3314 	u32 etqf;
3315 
3316 	index = igc_get_avail_etype_filter_slot(adapter);
3317 	if (index < 0)
3318 		return -ENOSPC;
3319 
3320 	etqf = rd32(IGC_ETQF(index));
3321 
3322 	etqf &= ~IGC_ETQF_ETYPE_MASK;
3323 	etqf |= etype;
3324 
3325 	if (queue >= 0) {
3326 		etqf &= ~IGC_ETQF_QUEUE_MASK;
3327 		etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
3328 		etqf |= IGC_ETQF_QUEUE_ENABLE;
3329 	}
3330 
3331 	etqf |= IGC_ETQF_FILTER_ENABLE;
3332 
3333 	wr32(IGC_ETQF(index), etqf);
3334 
3335 	netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
3336 		   etype, queue);
3337 	return 0;
3338 }
3339 
igc_find_etype_filter(struct igc_adapter * adapter,u16 etype)3340 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
3341 {
3342 	struct igc_hw *hw = &adapter->hw;
3343 	int i;
3344 
3345 	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3346 		u32 etqf = rd32(IGC_ETQF(i));
3347 
3348 		if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
3349 			return i;
3350 	}
3351 
3352 	return -1;
3353 }
3354 
3355 /**
3356  * igc_del_etype_filter() - Delete ethertype filter
3357  * @adapter: Pointer to adapter where the filter should be deleted from
3358  * @etype: Ethertype value
3359  */
igc_del_etype_filter(struct igc_adapter * adapter,u16 etype)3360 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
3361 {
3362 	struct igc_hw *hw = &adapter->hw;
3363 	int index;
3364 
3365 	index = igc_find_etype_filter(adapter, etype);
3366 	if (index < 0)
3367 		return;
3368 
3369 	wr32(IGC_ETQF(index), 0);
3370 
3371 	netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
3372 		   etype);
3373 }
3374 
igc_flex_filter_select(struct igc_adapter * adapter,struct igc_flex_filter * input,u32 * fhft)3375 static int igc_flex_filter_select(struct igc_adapter *adapter,
3376 				  struct igc_flex_filter *input,
3377 				  u32 *fhft)
3378 {
3379 	struct igc_hw *hw = &adapter->hw;
3380 	u8 fhft_index;
3381 	u32 fhftsl;
3382 
3383 	if (input->index >= MAX_FLEX_FILTER) {
3384 		dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n");
3385 		return -EINVAL;
3386 	}
3387 
3388 	/* Indirect table select register */
3389 	fhftsl = rd32(IGC_FHFTSL);
3390 	fhftsl &= ~IGC_FHFTSL_FTSL_MASK;
3391 	switch (input->index) {
3392 	case 0 ... 7:
3393 		fhftsl |= 0x00;
3394 		break;
3395 	case 8 ... 15:
3396 		fhftsl |= 0x01;
3397 		break;
3398 	case 16 ... 23:
3399 		fhftsl |= 0x02;
3400 		break;
3401 	case 24 ... 31:
3402 		fhftsl |= 0x03;
3403 		break;
3404 	}
3405 	wr32(IGC_FHFTSL, fhftsl);
3406 
3407 	/* Normalize index down to host table register */
3408 	fhft_index = input->index % 8;
3409 
3410 	*fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) :
3411 		IGC_FHFT_EXT(fhft_index - 4);
3412 
3413 	return 0;
3414 }
3415 
igc_write_flex_filter_ll(struct igc_adapter * adapter,struct igc_flex_filter * input)3416 static int igc_write_flex_filter_ll(struct igc_adapter *adapter,
3417 				    struct igc_flex_filter *input)
3418 {
3419 	struct device *dev = &adapter->pdev->dev;
3420 	struct igc_hw *hw = &adapter->hw;
3421 	u8 *data = input->data;
3422 	u8 *mask = input->mask;
3423 	u32 queuing;
3424 	u32 fhft;
3425 	u32 wufc;
3426 	int ret;
3427 	int i;
3428 
3429 	/* Length has to be aligned to 8. Otherwise the filter will fail. Bail
3430 	 * out early to avoid surprises later.
3431 	 */
3432 	if (input->length % 8 != 0) {
3433 		dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n");
3434 		return -EINVAL;
3435 	}
3436 
3437 	/* Select corresponding flex filter register and get base for host table. */
3438 	ret = igc_flex_filter_select(adapter, input, &fhft);
3439 	if (ret)
3440 		return ret;
3441 
3442 	/* When adding a filter globally disable flex filter feature. That is
3443 	 * recommended within the datasheet.
3444 	 */
3445 	wufc = rd32(IGC_WUFC);
3446 	wufc &= ~IGC_WUFC_FLEX_HQ;
3447 	wr32(IGC_WUFC, wufc);
3448 
3449 	/* Configure filter */
3450 	queuing = input->length & IGC_FHFT_LENGTH_MASK;
3451 	queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK;
3452 	queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK;
3453 
3454 	if (input->immediate_irq)
3455 		queuing |= IGC_FHFT_IMM_INT;
3456 
3457 	if (input->drop)
3458 		queuing |= IGC_FHFT_DROP;
3459 
3460 	wr32(fhft + 0xFC, queuing);
3461 
3462 	/* Write data (128 byte) and mask (128 bit) */
3463 	for (i = 0; i < 16; ++i) {
3464 		const size_t data_idx = i * 8;
3465 		const size_t row_idx = i * 16;
3466 		u32 dw0 =
3467 			(data[data_idx + 0] << 0) |
3468 			(data[data_idx + 1] << 8) |
3469 			(data[data_idx + 2] << 16) |
3470 			(data[data_idx + 3] << 24);
3471 		u32 dw1 =
3472 			(data[data_idx + 4] << 0) |
3473 			(data[data_idx + 5] << 8) |
3474 			(data[data_idx + 6] << 16) |
3475 			(data[data_idx + 7] << 24);
3476 		u32 tmp;
3477 
3478 		/* Write row: dw0, dw1 and mask */
3479 		wr32(fhft + row_idx, dw0);
3480 		wr32(fhft + row_idx + 4, dw1);
3481 
3482 		/* mask is only valid for MASK(7, 0) */
3483 		tmp = rd32(fhft + row_idx + 8);
3484 		tmp &= ~GENMASK(7, 0);
3485 		tmp |= mask[i];
3486 		wr32(fhft + row_idx + 8, tmp);
3487 	}
3488 
3489 	/* Enable filter. */
3490 	wufc |= IGC_WUFC_FLEX_HQ;
3491 	if (input->index > 8) {
3492 		/* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
3493 		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3494 
3495 		wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8));
3496 
3497 		wr32(IGC_WUFC_EXT, wufc_ext);
3498 	} else {
3499 		wufc |= (IGC_WUFC_FLX0 << input->index);
3500 	}
3501 	wr32(IGC_WUFC, wufc);
3502 
3503 	dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n",
3504 		input->index);
3505 
3506 	return 0;
3507 }
3508 
igc_flex_filter_add_field(struct igc_flex_filter * flex,const void * src,unsigned int offset,size_t len,const void * mask)3509 static void igc_flex_filter_add_field(struct igc_flex_filter *flex,
3510 				      const void *src, unsigned int offset,
3511 				      size_t len, const void *mask)
3512 {
3513 	int i;
3514 
3515 	/* data */
3516 	memcpy(&flex->data[offset], src, len);
3517 
3518 	/* mask */
3519 	for (i = 0; i < len; ++i) {
3520 		const unsigned int idx = i + offset;
3521 		const u8 *ptr = mask;
3522 
3523 		if (mask) {
3524 			if (ptr[i] & 0xff)
3525 				flex->mask[idx / 8] |= BIT(idx % 8);
3526 
3527 			continue;
3528 		}
3529 
3530 		flex->mask[idx / 8] |= BIT(idx % 8);
3531 	}
3532 }
3533 
igc_find_avail_flex_filter_slot(struct igc_adapter * adapter)3534 static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter)
3535 {
3536 	struct igc_hw *hw = &adapter->hw;
3537 	u32 wufc, wufc_ext;
3538 	int i;
3539 
3540 	wufc = rd32(IGC_WUFC);
3541 	wufc_ext = rd32(IGC_WUFC_EXT);
3542 
3543 	for (i = 0; i < MAX_FLEX_FILTER; i++) {
3544 		if (i < 8) {
3545 			if (!(wufc & (IGC_WUFC_FLX0 << i)))
3546 				return i;
3547 		} else {
3548 			if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8))))
3549 				return i;
3550 		}
3551 	}
3552 
3553 	return -ENOSPC;
3554 }
3555 
igc_flex_filter_in_use(struct igc_adapter * adapter)3556 static bool igc_flex_filter_in_use(struct igc_adapter *adapter)
3557 {
3558 	struct igc_hw *hw = &adapter->hw;
3559 	u32 wufc, wufc_ext;
3560 
3561 	wufc = rd32(IGC_WUFC);
3562 	wufc_ext = rd32(IGC_WUFC_EXT);
3563 
3564 	if (wufc & IGC_WUFC_FILTER_MASK)
3565 		return true;
3566 
3567 	if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK)
3568 		return true;
3569 
3570 	return false;
3571 }
3572 
igc_add_flex_filter(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3573 static int igc_add_flex_filter(struct igc_adapter *adapter,
3574 			       struct igc_nfc_rule *rule)
3575 {
3576 	struct igc_flex_filter flex = { };
3577 	struct igc_nfc_filter *filter = &rule->filter;
3578 	unsigned int eth_offset, user_offset;
3579 	int ret, index;
3580 	bool vlan;
3581 
3582 	index = igc_find_avail_flex_filter_slot(adapter);
3583 	if (index < 0)
3584 		return -ENOSPC;
3585 
3586 	/* Construct the flex filter:
3587 	 *  -> dest_mac [6]
3588 	 *  -> src_mac [6]
3589 	 *  -> tpid [2]
3590 	 *  -> vlan tci [2]
3591 	 *  -> ether type [2]
3592 	 *  -> user data [8]
3593 	 *  -> = 26 bytes => 32 length
3594 	 */
3595 	flex.index    = index;
3596 	flex.length   = 32;
3597 	flex.rx_queue = rule->action;
3598 
3599 	vlan = rule->filter.vlan_tci || rule->filter.vlan_etype;
3600 	eth_offset = vlan ? 16 : 12;
3601 	user_offset = vlan ? 18 : 14;
3602 
3603 	/* Add destination MAC  */
3604 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3605 		igc_flex_filter_add_field(&flex, &filter->dst_addr, 0,
3606 					  ETH_ALEN, NULL);
3607 
3608 	/* Add source MAC */
3609 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3610 		igc_flex_filter_add_field(&flex, &filter->src_addr, 6,
3611 					  ETH_ALEN, NULL);
3612 
3613 	/* Add VLAN etype */
3614 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE)
3615 		igc_flex_filter_add_field(&flex, &filter->vlan_etype, 12,
3616 					  sizeof(filter->vlan_etype),
3617 					  NULL);
3618 
3619 	/* Add VLAN TCI */
3620 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI)
3621 		igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14,
3622 					  sizeof(filter->vlan_tci), NULL);
3623 
3624 	/* Add Ether type */
3625 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3626 		__be16 etype = cpu_to_be16(filter->etype);
3627 
3628 		igc_flex_filter_add_field(&flex, &etype, eth_offset,
3629 					  sizeof(etype), NULL);
3630 	}
3631 
3632 	/* Add user data */
3633 	if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA)
3634 		igc_flex_filter_add_field(&flex, &filter->user_data,
3635 					  user_offset,
3636 					  sizeof(filter->user_data),
3637 					  filter->user_mask);
3638 
3639 	/* Add it down to the hardware and enable it. */
3640 	ret = igc_write_flex_filter_ll(adapter, &flex);
3641 	if (ret)
3642 		return ret;
3643 
3644 	filter->flex_index = index;
3645 
3646 	return 0;
3647 }
3648 
igc_del_flex_filter(struct igc_adapter * adapter,u16 reg_index)3649 static void igc_del_flex_filter(struct igc_adapter *adapter,
3650 				u16 reg_index)
3651 {
3652 	struct igc_hw *hw = &adapter->hw;
3653 	u32 wufc;
3654 
3655 	/* Just disable the filter. The filter table itself is kept
3656 	 * intact. Another flex_filter_add() should override the "old" data
3657 	 * then.
3658 	 */
3659 	if (reg_index > 8) {
3660 		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3661 
3662 		wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8));
3663 		wr32(IGC_WUFC_EXT, wufc_ext);
3664 	} else {
3665 		wufc = rd32(IGC_WUFC);
3666 
3667 		wufc &= ~(IGC_WUFC_FLX0 << reg_index);
3668 		wr32(IGC_WUFC, wufc);
3669 	}
3670 
3671 	if (igc_flex_filter_in_use(adapter))
3672 		return;
3673 
3674 	/* No filters are in use, we may disable flex filters */
3675 	wufc = rd32(IGC_WUFC);
3676 	wufc &= ~IGC_WUFC_FLEX_HQ;
3677 	wr32(IGC_WUFC, wufc);
3678 }
3679 
igc_enable_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3680 static int igc_enable_nfc_rule(struct igc_adapter *adapter,
3681 			       struct igc_nfc_rule *rule)
3682 {
3683 	int err;
3684 
3685 	if (rule->flex) {
3686 		return igc_add_flex_filter(adapter, rule);
3687 	}
3688 
3689 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3690 		err = igc_add_etype_filter(adapter, rule->filter.etype,
3691 					   rule->action);
3692 		if (err)
3693 			return err;
3694 	}
3695 
3696 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
3697 		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3698 					 rule->filter.src_addr, rule->action);
3699 		if (err)
3700 			return err;
3701 	}
3702 
3703 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
3704 		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3705 					 rule->filter.dst_addr, rule->action);
3706 		if (err)
3707 			return err;
3708 	}
3709 
3710 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3711 		int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3712 			   VLAN_PRIO_SHIFT;
3713 
3714 		err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
3715 		if (err)
3716 			return err;
3717 	}
3718 
3719 	return 0;
3720 }
3721 
igc_disable_nfc_rule(struct igc_adapter * adapter,const struct igc_nfc_rule * rule)3722 static void igc_disable_nfc_rule(struct igc_adapter *adapter,
3723 				 const struct igc_nfc_rule *rule)
3724 {
3725 	if (rule->flex) {
3726 		igc_del_flex_filter(adapter, rule->filter.flex_index);
3727 		return;
3728 	}
3729 
3730 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
3731 		igc_del_etype_filter(adapter, rule->filter.etype);
3732 
3733 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3734 		int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3735 			   VLAN_PRIO_SHIFT;
3736 
3737 		igc_del_vlan_prio_filter(adapter, prio);
3738 	}
3739 
3740 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3741 		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3742 				   rule->filter.src_addr);
3743 
3744 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3745 		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3746 				   rule->filter.dst_addr);
3747 }
3748 
3749 /**
3750  * igc_get_nfc_rule() - Get NFC rule
3751  * @adapter: Pointer to adapter
3752  * @location: Rule location
3753  *
3754  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3755  *
3756  * Return: Pointer to NFC rule at @location. If not found, NULL.
3757  */
igc_get_nfc_rule(struct igc_adapter * adapter,u32 location)3758 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
3759 				      u32 location)
3760 {
3761 	struct igc_nfc_rule *rule;
3762 
3763 	list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
3764 		if (rule->location == location)
3765 			return rule;
3766 		if (rule->location > location)
3767 			break;
3768 	}
3769 
3770 	return NULL;
3771 }
3772 
3773 /**
3774  * igc_del_nfc_rule() - Delete NFC rule
3775  * @adapter: Pointer to adapter
3776  * @rule: Pointer to rule to be deleted
3777  *
3778  * Disable NFC rule in hardware and delete it from adapter.
3779  *
3780  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3781  */
igc_del_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3782 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3783 {
3784 	igc_disable_nfc_rule(adapter, rule);
3785 
3786 	list_del(&rule->list);
3787 	adapter->nfc_rule_count--;
3788 
3789 	kfree(rule);
3790 }
3791 
igc_flush_nfc_rules(struct igc_adapter * adapter)3792 static void igc_flush_nfc_rules(struct igc_adapter *adapter)
3793 {
3794 	struct igc_nfc_rule *rule, *tmp;
3795 
3796 	mutex_lock(&adapter->nfc_rule_lock);
3797 
3798 	list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
3799 		igc_del_nfc_rule(adapter, rule);
3800 
3801 	mutex_unlock(&adapter->nfc_rule_lock);
3802 }
3803 
3804 /**
3805  * igc_add_nfc_rule() - Add NFC rule
3806  * @adapter: Pointer to adapter
3807  * @rule: Pointer to rule to be added
3808  *
3809  * Enable NFC rule in hardware and add it to adapter.
3810  *
3811  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3812  *
3813  * Return: 0 on success, negative errno on failure.
3814  */
igc_add_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3815 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3816 {
3817 	struct igc_nfc_rule *pred, *cur;
3818 	int err;
3819 
3820 	err = igc_enable_nfc_rule(adapter, rule);
3821 	if (err)
3822 		return err;
3823 
3824 	pred = NULL;
3825 	list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
3826 		if (cur->location >= rule->location)
3827 			break;
3828 		pred = cur;
3829 	}
3830 
3831 	list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
3832 	adapter->nfc_rule_count++;
3833 	return 0;
3834 }
3835 
igc_restore_nfc_rules(struct igc_adapter * adapter)3836 static void igc_restore_nfc_rules(struct igc_adapter *adapter)
3837 {
3838 	struct igc_nfc_rule *rule;
3839 
3840 	mutex_lock(&adapter->nfc_rule_lock);
3841 
3842 	list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
3843 		igc_enable_nfc_rule(adapter, rule);
3844 
3845 	mutex_unlock(&adapter->nfc_rule_lock);
3846 }
3847 
igc_uc_sync(struct net_device * netdev,const unsigned char * addr)3848 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
3849 {
3850 	struct igc_adapter *adapter = netdev_priv(netdev);
3851 
3852 	return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
3853 }
3854 
igc_uc_unsync(struct net_device * netdev,const unsigned char * addr)3855 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
3856 {
3857 	struct igc_adapter *adapter = netdev_priv(netdev);
3858 
3859 	igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
3860 	return 0;
3861 }
3862 
3863 /**
3864  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3865  * @netdev: network interface device structure
3866  *
3867  * The set_rx_mode entry point is called whenever the unicast or multicast
3868  * address lists or the network interface flags are updated.  This routine is
3869  * responsible for configuring the hardware for proper unicast, multicast,
3870  * promiscuous mode, and all-multi behavior.
3871  */
igc_set_rx_mode(struct net_device * netdev)3872 static void igc_set_rx_mode(struct net_device *netdev)
3873 {
3874 	struct igc_adapter *adapter = netdev_priv(netdev);
3875 	struct igc_hw *hw = &adapter->hw;
3876 	u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
3877 	int count;
3878 
3879 	/* Check for Promiscuous and All Multicast modes */
3880 	if (netdev->flags & IFF_PROMISC) {
3881 		rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
3882 	} else {
3883 		if (netdev->flags & IFF_ALLMULTI) {
3884 			rctl |= IGC_RCTL_MPE;
3885 		} else {
3886 			/* Write addresses to the MTA, if the attempt fails
3887 			 * then we should just turn on promiscuous mode so
3888 			 * that we can at least receive multicast traffic
3889 			 */
3890 			count = igc_write_mc_addr_list(netdev);
3891 			if (count < 0)
3892 				rctl |= IGC_RCTL_MPE;
3893 		}
3894 	}
3895 
3896 	/* Write addresses to available RAR registers, if there is not
3897 	 * sufficient space to store all the addresses then enable
3898 	 * unicast promiscuous mode
3899 	 */
3900 	if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
3901 		rctl |= IGC_RCTL_UPE;
3902 
3903 	/* update state of unicast and multicast */
3904 	rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
3905 	wr32(IGC_RCTL, rctl);
3906 
3907 #if (PAGE_SIZE < 8192)
3908 	if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
3909 		rlpml = IGC_MAX_FRAME_BUILD_SKB;
3910 #endif
3911 	wr32(IGC_RLPML, rlpml);
3912 }
3913 
3914 /**
3915  * igc_configure - configure the hardware for RX and TX
3916  * @adapter: private board structure
3917  */
igc_configure(struct igc_adapter * adapter)3918 static void igc_configure(struct igc_adapter *adapter)
3919 {
3920 	struct net_device *netdev = adapter->netdev;
3921 	int i = 0;
3922 
3923 	igc_get_hw_control(adapter);
3924 	igc_set_rx_mode(netdev);
3925 
3926 	igc_restore_vlan(adapter);
3927 
3928 	igc_setup_tctl(adapter);
3929 	igc_setup_mrqc(adapter);
3930 	igc_setup_rctl(adapter);
3931 
3932 	igc_set_default_mac_filter(adapter);
3933 	igc_restore_nfc_rules(adapter);
3934 
3935 	igc_configure_tx(adapter);
3936 	igc_configure_rx(adapter);
3937 
3938 	igc_rx_fifo_flush_base(&adapter->hw);
3939 
3940 	/* call igc_desc_unused which always leaves
3941 	 * at least 1 descriptor unused to make sure
3942 	 * next_to_use != next_to_clean
3943 	 */
3944 	for (i = 0; i < adapter->num_rx_queues; i++) {
3945 		struct igc_ring *ring = adapter->rx_ring[i];
3946 
3947 		if (ring->xsk_pool)
3948 			igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
3949 		else
3950 			igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
3951 	}
3952 }
3953 
3954 /**
3955  * igc_write_ivar - configure ivar for given MSI-X vector
3956  * @hw: pointer to the HW structure
3957  * @msix_vector: vector number we are allocating to a given ring
3958  * @index: row index of IVAR register to write within IVAR table
3959  * @offset: column offset of in IVAR, should be multiple of 8
3960  *
3961  * The IVAR table consists of 2 columns,
3962  * each containing an cause allocation for an Rx and Tx ring, and a
3963  * variable number of rows depending on the number of queues supported.
3964  */
igc_write_ivar(struct igc_hw * hw,int msix_vector,int index,int offset)3965 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
3966 			   int index, int offset)
3967 {
3968 	u32 ivar = array_rd32(IGC_IVAR0, index);
3969 
3970 	/* clear any bits that are currently set */
3971 	ivar &= ~((u32)0xFF << offset);
3972 
3973 	/* write vector and valid bit */
3974 	ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
3975 
3976 	array_wr32(IGC_IVAR0, index, ivar);
3977 }
3978 
igc_assign_vector(struct igc_q_vector * q_vector,int msix_vector)3979 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
3980 {
3981 	struct igc_adapter *adapter = q_vector->adapter;
3982 	struct igc_hw *hw = &adapter->hw;
3983 	int rx_queue = IGC_N0_QUEUE;
3984 	int tx_queue = IGC_N0_QUEUE;
3985 
3986 	if (q_vector->rx.ring)
3987 		rx_queue = q_vector->rx.ring->reg_idx;
3988 	if (q_vector->tx.ring)
3989 		tx_queue = q_vector->tx.ring->reg_idx;
3990 
3991 	switch (hw->mac.type) {
3992 	case igc_i225:
3993 		if (rx_queue > IGC_N0_QUEUE)
3994 			igc_write_ivar(hw, msix_vector,
3995 				       rx_queue >> 1,
3996 				       (rx_queue & 0x1) << 4);
3997 		if (tx_queue > IGC_N0_QUEUE)
3998 			igc_write_ivar(hw, msix_vector,
3999 				       tx_queue >> 1,
4000 				       ((tx_queue & 0x1) << 4) + 8);
4001 		q_vector->eims_value = BIT(msix_vector);
4002 		break;
4003 	default:
4004 		WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
4005 		break;
4006 	}
4007 
4008 	/* add q_vector eims value to global eims_enable_mask */
4009 	adapter->eims_enable_mask |= q_vector->eims_value;
4010 
4011 	/* configure q_vector to set itr on first interrupt */
4012 	q_vector->set_itr = 1;
4013 }
4014 
4015 /**
4016  * igc_configure_msix - Configure MSI-X hardware
4017  * @adapter: Pointer to adapter structure
4018  *
4019  * igc_configure_msix sets up the hardware to properly
4020  * generate MSI-X interrupts.
4021  */
igc_configure_msix(struct igc_adapter * adapter)4022 static void igc_configure_msix(struct igc_adapter *adapter)
4023 {
4024 	struct igc_hw *hw = &adapter->hw;
4025 	int i, vector = 0;
4026 	u32 tmp;
4027 
4028 	adapter->eims_enable_mask = 0;
4029 
4030 	/* set vector for other causes, i.e. link changes */
4031 	switch (hw->mac.type) {
4032 	case igc_i225:
4033 		/* Turn on MSI-X capability first, or our settings
4034 		 * won't stick.  And it will take days to debug.
4035 		 */
4036 		wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
4037 		     IGC_GPIE_PBA | IGC_GPIE_EIAME |
4038 		     IGC_GPIE_NSICR);
4039 
4040 		/* enable msix_other interrupt */
4041 		adapter->eims_other = BIT(vector);
4042 		tmp = (vector++ | IGC_IVAR_VALID) << 8;
4043 
4044 		wr32(IGC_IVAR_MISC, tmp);
4045 		break;
4046 	default:
4047 		/* do nothing, since nothing else supports MSI-X */
4048 		break;
4049 	} /* switch (hw->mac.type) */
4050 
4051 	adapter->eims_enable_mask |= adapter->eims_other;
4052 
4053 	for (i = 0; i < adapter->num_q_vectors; i++)
4054 		igc_assign_vector(adapter->q_vector[i], vector++);
4055 
4056 	wrfl();
4057 }
4058 
4059 /**
4060  * igc_irq_enable - Enable default interrupt generation settings
4061  * @adapter: board private structure
4062  */
igc_irq_enable(struct igc_adapter * adapter)4063 static void igc_irq_enable(struct igc_adapter *adapter)
4064 {
4065 	struct igc_hw *hw = &adapter->hw;
4066 
4067 	if (adapter->msix_entries) {
4068 		u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
4069 		u32 regval = rd32(IGC_EIAC);
4070 
4071 		wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
4072 		regval = rd32(IGC_EIAM);
4073 		wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
4074 		wr32(IGC_EIMS, adapter->eims_enable_mask);
4075 		wr32(IGC_IMS, ims);
4076 	} else {
4077 		wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
4078 		wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
4079 	}
4080 }
4081 
4082 /**
4083  * igc_irq_disable - Mask off interrupt generation on the NIC
4084  * @adapter: board private structure
4085  */
igc_irq_disable(struct igc_adapter * adapter)4086 static void igc_irq_disable(struct igc_adapter *adapter)
4087 {
4088 	struct igc_hw *hw = &adapter->hw;
4089 
4090 	if (adapter->msix_entries) {
4091 		u32 regval = rd32(IGC_EIAM);
4092 
4093 		wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
4094 		wr32(IGC_EIMC, adapter->eims_enable_mask);
4095 		regval = rd32(IGC_EIAC);
4096 		wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
4097 	}
4098 
4099 	wr32(IGC_IAM, 0);
4100 	wr32(IGC_IMC, ~0);
4101 	wrfl();
4102 
4103 	if (adapter->msix_entries) {
4104 		int vector = 0, i;
4105 
4106 		synchronize_irq(adapter->msix_entries[vector++].vector);
4107 
4108 		for (i = 0; i < adapter->num_q_vectors; i++)
4109 			synchronize_irq(adapter->msix_entries[vector++].vector);
4110 	} else {
4111 		synchronize_irq(adapter->pdev->irq);
4112 	}
4113 }
4114 
igc_set_flag_queue_pairs(struct igc_adapter * adapter,const u32 max_rss_queues)4115 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
4116 			      const u32 max_rss_queues)
4117 {
4118 	/* Determine if we need to pair queues. */
4119 	/* If rss_queues > half of max_rss_queues, pair the queues in
4120 	 * order to conserve interrupts due to limited supply.
4121 	 */
4122 	if (adapter->rss_queues > (max_rss_queues / 2))
4123 		adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4124 	else
4125 		adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4126 }
4127 
igc_get_max_rss_queues(struct igc_adapter * adapter)4128 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4129 {
4130 	return IGC_MAX_RX_QUEUES;
4131 }
4132 
igc_init_queue_configuration(struct igc_adapter * adapter)4133 static void igc_init_queue_configuration(struct igc_adapter *adapter)
4134 {
4135 	u32 max_rss_queues;
4136 
4137 	max_rss_queues = igc_get_max_rss_queues(adapter);
4138 	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4139 
4140 	igc_set_flag_queue_pairs(adapter, max_rss_queues);
4141 }
4142 
4143 /**
4144  * igc_reset_q_vector - Reset config for interrupt vector
4145  * @adapter: board private structure to initialize
4146  * @v_idx: Index of vector to be reset
4147  *
4148  * If NAPI is enabled it will delete any references to the
4149  * NAPI struct. This is preparation for igc_free_q_vector.
4150  */
igc_reset_q_vector(struct igc_adapter * adapter,int v_idx)4151 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
4152 {
4153 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4154 
4155 	/* if we're coming from igc_set_interrupt_capability, the vectors are
4156 	 * not yet allocated
4157 	 */
4158 	if (!q_vector)
4159 		return;
4160 
4161 	if (q_vector->tx.ring)
4162 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
4163 
4164 	if (q_vector->rx.ring)
4165 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
4166 
4167 	netif_napi_del(&q_vector->napi);
4168 }
4169 
4170 /**
4171  * igc_free_q_vector - Free memory allocated for specific interrupt vector
4172  * @adapter: board private structure to initialize
4173  * @v_idx: Index of vector to be freed
4174  *
4175  * This function frees the memory allocated to the q_vector.
4176  */
igc_free_q_vector(struct igc_adapter * adapter,int v_idx)4177 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
4178 {
4179 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4180 
4181 	adapter->q_vector[v_idx] = NULL;
4182 
4183 	/* igc_get_stats64() might access the rings on this vector,
4184 	 * we must wait a grace period before freeing it.
4185 	 */
4186 	if (q_vector)
4187 		kfree_rcu(q_vector, rcu);
4188 }
4189 
4190 /**
4191  * igc_free_q_vectors - Free memory allocated for interrupt vectors
4192  * @adapter: board private structure to initialize
4193  *
4194  * This function frees the memory allocated to the q_vectors.  In addition if
4195  * NAPI is enabled it will delete any references to the NAPI struct prior
4196  * to freeing the q_vector.
4197  */
igc_free_q_vectors(struct igc_adapter * adapter)4198 static void igc_free_q_vectors(struct igc_adapter *adapter)
4199 {
4200 	int v_idx = adapter->num_q_vectors;
4201 
4202 	adapter->num_tx_queues = 0;
4203 	adapter->num_rx_queues = 0;
4204 	adapter->num_q_vectors = 0;
4205 
4206 	while (v_idx--) {
4207 		igc_reset_q_vector(adapter, v_idx);
4208 		igc_free_q_vector(adapter, v_idx);
4209 	}
4210 }
4211 
4212 /**
4213  * igc_update_itr - update the dynamic ITR value based on statistics
4214  * @q_vector: pointer to q_vector
4215  * @ring_container: ring info to update the itr for
4216  *
4217  * Stores a new ITR value based on packets and byte
4218  * counts during the last interrupt.  The advantage of per interrupt
4219  * computation is faster updates and more accurate ITR for the current
4220  * traffic pattern.  Constants in this function were computed
4221  * based on theoretical maximum wire speed and thresholds were set based
4222  * on testing data as well as attempting to minimize response time
4223  * while increasing bulk throughput.
4224  * NOTE: These calculations are only valid when operating in a single-
4225  * queue environment.
4226  */
igc_update_itr(struct igc_q_vector * q_vector,struct igc_ring_container * ring_container)4227 static void igc_update_itr(struct igc_q_vector *q_vector,
4228 			   struct igc_ring_container *ring_container)
4229 {
4230 	unsigned int packets = ring_container->total_packets;
4231 	unsigned int bytes = ring_container->total_bytes;
4232 	u8 itrval = ring_container->itr;
4233 
4234 	/* no packets, exit with status unchanged */
4235 	if (packets == 0)
4236 		return;
4237 
4238 	switch (itrval) {
4239 	case lowest_latency:
4240 		/* handle TSO and jumbo frames */
4241 		if (bytes / packets > 8000)
4242 			itrval = bulk_latency;
4243 		else if ((packets < 5) && (bytes > 512))
4244 			itrval = low_latency;
4245 		break;
4246 	case low_latency:  /* 50 usec aka 20000 ints/s */
4247 		if (bytes > 10000) {
4248 			/* this if handles the TSO accounting */
4249 			if (bytes / packets > 8000)
4250 				itrval = bulk_latency;
4251 			else if ((packets < 10) || ((bytes / packets) > 1200))
4252 				itrval = bulk_latency;
4253 			else if ((packets > 35))
4254 				itrval = lowest_latency;
4255 		} else if (bytes / packets > 2000) {
4256 			itrval = bulk_latency;
4257 		} else if (packets <= 2 && bytes < 512) {
4258 			itrval = lowest_latency;
4259 		}
4260 		break;
4261 	case bulk_latency: /* 250 usec aka 4000 ints/s */
4262 		if (bytes > 25000) {
4263 			if (packets > 35)
4264 				itrval = low_latency;
4265 		} else if (bytes < 1500) {
4266 			itrval = low_latency;
4267 		}
4268 		break;
4269 	}
4270 
4271 	/* clear work counters since we have the values we need */
4272 	ring_container->total_bytes = 0;
4273 	ring_container->total_packets = 0;
4274 
4275 	/* write updated itr to ring container */
4276 	ring_container->itr = itrval;
4277 }
4278 
igc_set_itr(struct igc_q_vector * q_vector)4279 static void igc_set_itr(struct igc_q_vector *q_vector)
4280 {
4281 	struct igc_adapter *adapter = q_vector->adapter;
4282 	u32 new_itr = q_vector->itr_val;
4283 	u8 current_itr = 0;
4284 
4285 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4286 	switch (adapter->link_speed) {
4287 	case SPEED_10:
4288 	case SPEED_100:
4289 		current_itr = 0;
4290 		new_itr = IGC_4K_ITR;
4291 		goto set_itr_now;
4292 	default:
4293 		break;
4294 	}
4295 
4296 	igc_update_itr(q_vector, &q_vector->tx);
4297 	igc_update_itr(q_vector, &q_vector->rx);
4298 
4299 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4300 
4301 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4302 	if (current_itr == lowest_latency &&
4303 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4304 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4305 		current_itr = low_latency;
4306 
4307 	switch (current_itr) {
4308 	/* counts and packets in update_itr are dependent on these numbers */
4309 	case lowest_latency:
4310 		new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
4311 		break;
4312 	case low_latency:
4313 		new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
4314 		break;
4315 	case bulk_latency:
4316 		new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
4317 		break;
4318 	default:
4319 		break;
4320 	}
4321 
4322 set_itr_now:
4323 	if (new_itr != q_vector->itr_val) {
4324 		/* this attempts to bias the interrupt rate towards Bulk
4325 		 * by adding intermediate steps when interrupt rate is
4326 		 * increasing
4327 		 */
4328 		new_itr = new_itr > q_vector->itr_val ?
4329 			  max((new_itr * q_vector->itr_val) /
4330 			  (new_itr + (q_vector->itr_val >> 2)),
4331 			  new_itr) : new_itr;
4332 		/* Don't write the value here; it resets the adapter's
4333 		 * internal timer, and causes us to delay far longer than
4334 		 * we should between interrupts.  Instead, we write the ITR
4335 		 * value at the beginning of the next interrupt so the timing
4336 		 * ends up being correct.
4337 		 */
4338 		q_vector->itr_val = new_itr;
4339 		q_vector->set_itr = 1;
4340 	}
4341 }
4342 
igc_reset_interrupt_capability(struct igc_adapter * adapter)4343 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
4344 {
4345 	int v_idx = adapter->num_q_vectors;
4346 
4347 	if (adapter->msix_entries) {
4348 		pci_disable_msix(adapter->pdev);
4349 		kfree(adapter->msix_entries);
4350 		adapter->msix_entries = NULL;
4351 	} else if (adapter->flags & IGC_FLAG_HAS_MSI) {
4352 		pci_disable_msi(adapter->pdev);
4353 	}
4354 
4355 	while (v_idx--)
4356 		igc_reset_q_vector(adapter, v_idx);
4357 }
4358 
4359 /**
4360  * igc_set_interrupt_capability - set MSI or MSI-X if supported
4361  * @adapter: Pointer to adapter structure
4362  * @msix: boolean value for MSI-X capability
4363  *
4364  * Attempt to configure interrupts using the best available
4365  * capabilities of the hardware and kernel.
4366  */
igc_set_interrupt_capability(struct igc_adapter * adapter,bool msix)4367 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
4368 					 bool msix)
4369 {
4370 	int numvecs, i;
4371 	int err;
4372 
4373 	if (!msix)
4374 		goto msi_only;
4375 	adapter->flags |= IGC_FLAG_HAS_MSIX;
4376 
4377 	/* Number of supported queues. */
4378 	adapter->num_rx_queues = adapter->rss_queues;
4379 
4380 	adapter->num_tx_queues = adapter->rss_queues;
4381 
4382 	/* start with one vector for every Rx queue */
4383 	numvecs = adapter->num_rx_queues;
4384 
4385 	/* if Tx handler is separate add 1 for every Tx queue */
4386 	if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
4387 		numvecs += adapter->num_tx_queues;
4388 
4389 	/* store the number of vectors reserved for queues */
4390 	adapter->num_q_vectors = numvecs;
4391 
4392 	/* add 1 vector for link status interrupts */
4393 	numvecs++;
4394 
4395 	adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
4396 					GFP_KERNEL);
4397 
4398 	if (!adapter->msix_entries)
4399 		return;
4400 
4401 	/* populate entry values */
4402 	for (i = 0; i < numvecs; i++)
4403 		adapter->msix_entries[i].entry = i;
4404 
4405 	err = pci_enable_msix_range(adapter->pdev,
4406 				    adapter->msix_entries,
4407 				    numvecs,
4408 				    numvecs);
4409 	if (err > 0)
4410 		return;
4411 
4412 	kfree(adapter->msix_entries);
4413 	adapter->msix_entries = NULL;
4414 
4415 	igc_reset_interrupt_capability(adapter);
4416 
4417 msi_only:
4418 	adapter->flags &= ~IGC_FLAG_HAS_MSIX;
4419 
4420 	adapter->rss_queues = 1;
4421 	adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4422 	adapter->num_rx_queues = 1;
4423 	adapter->num_tx_queues = 1;
4424 	adapter->num_q_vectors = 1;
4425 	if (!pci_enable_msi(adapter->pdev))
4426 		adapter->flags |= IGC_FLAG_HAS_MSI;
4427 }
4428 
4429 /**
4430  * igc_update_ring_itr - update the dynamic ITR value based on packet size
4431  * @q_vector: pointer to q_vector
4432  *
4433  * Stores a new ITR value based on strictly on packet size.  This
4434  * algorithm is less sophisticated than that used in igc_update_itr,
4435  * due to the difficulty of synchronizing statistics across multiple
4436  * receive rings.  The divisors and thresholds used by this function
4437  * were determined based on theoretical maximum wire speed and testing
4438  * data, in order to minimize response time while increasing bulk
4439  * throughput.
4440  * NOTE: This function is called only when operating in a multiqueue
4441  * receive environment.
4442  */
igc_update_ring_itr(struct igc_q_vector * q_vector)4443 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
4444 {
4445 	struct igc_adapter *adapter = q_vector->adapter;
4446 	int new_val = q_vector->itr_val;
4447 	int avg_wire_size = 0;
4448 	unsigned int packets;
4449 
4450 	/* For non-gigabit speeds, just fix the interrupt rate at 4000
4451 	 * ints/sec - ITR timer value of 120 ticks.
4452 	 */
4453 	switch (adapter->link_speed) {
4454 	case SPEED_10:
4455 	case SPEED_100:
4456 		new_val = IGC_4K_ITR;
4457 		goto set_itr_val;
4458 	default:
4459 		break;
4460 	}
4461 
4462 	packets = q_vector->rx.total_packets;
4463 	if (packets)
4464 		avg_wire_size = q_vector->rx.total_bytes / packets;
4465 
4466 	packets = q_vector->tx.total_packets;
4467 	if (packets)
4468 		avg_wire_size = max_t(u32, avg_wire_size,
4469 				      q_vector->tx.total_bytes / packets);
4470 
4471 	/* if avg_wire_size isn't set no work was done */
4472 	if (!avg_wire_size)
4473 		goto clear_counts;
4474 
4475 	/* Add 24 bytes to size to account for CRC, preamble, and gap */
4476 	avg_wire_size += 24;
4477 
4478 	/* Don't starve jumbo frames */
4479 	avg_wire_size = min(avg_wire_size, 3000);
4480 
4481 	/* Give a little boost to mid-size frames */
4482 	if (avg_wire_size > 300 && avg_wire_size < 1200)
4483 		new_val = avg_wire_size / 3;
4484 	else
4485 		new_val = avg_wire_size / 2;
4486 
4487 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4488 	if (new_val < IGC_20K_ITR &&
4489 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4490 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4491 		new_val = IGC_20K_ITR;
4492 
4493 set_itr_val:
4494 	if (new_val != q_vector->itr_val) {
4495 		q_vector->itr_val = new_val;
4496 		q_vector->set_itr = 1;
4497 	}
4498 clear_counts:
4499 	q_vector->rx.total_bytes = 0;
4500 	q_vector->rx.total_packets = 0;
4501 	q_vector->tx.total_bytes = 0;
4502 	q_vector->tx.total_packets = 0;
4503 }
4504 
igc_ring_irq_enable(struct igc_q_vector * q_vector)4505 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
4506 {
4507 	struct igc_adapter *adapter = q_vector->adapter;
4508 	struct igc_hw *hw = &adapter->hw;
4509 
4510 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
4511 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
4512 		if (adapter->num_q_vectors == 1)
4513 			igc_set_itr(q_vector);
4514 		else
4515 			igc_update_ring_itr(q_vector);
4516 	}
4517 
4518 	if (!test_bit(__IGC_DOWN, &adapter->state)) {
4519 		if (adapter->msix_entries)
4520 			wr32(IGC_EIMS, q_vector->eims_value);
4521 		else
4522 			igc_irq_enable(adapter);
4523 	}
4524 }
4525 
igc_add_ring(struct igc_ring * ring,struct igc_ring_container * head)4526 static void igc_add_ring(struct igc_ring *ring,
4527 			 struct igc_ring_container *head)
4528 {
4529 	head->ring = ring;
4530 	head->count++;
4531 }
4532 
4533 /**
4534  * igc_cache_ring_register - Descriptor ring to register mapping
4535  * @adapter: board private structure to initialize
4536  *
4537  * Once we know the feature-set enabled for the device, we'll cache
4538  * the register offset the descriptor ring is assigned to.
4539  */
igc_cache_ring_register(struct igc_adapter * adapter)4540 static void igc_cache_ring_register(struct igc_adapter *adapter)
4541 {
4542 	int i = 0, j = 0;
4543 
4544 	switch (adapter->hw.mac.type) {
4545 	case igc_i225:
4546 	default:
4547 		for (; i < adapter->num_rx_queues; i++)
4548 			adapter->rx_ring[i]->reg_idx = i;
4549 		for (; j < adapter->num_tx_queues; j++)
4550 			adapter->tx_ring[j]->reg_idx = j;
4551 		break;
4552 	}
4553 }
4554 
4555 /**
4556  * igc_poll - NAPI Rx polling callback
4557  * @napi: napi polling structure
4558  * @budget: count of how many packets we should handle
4559  */
igc_poll(struct napi_struct * napi,int budget)4560 static int igc_poll(struct napi_struct *napi, int budget)
4561 {
4562 	struct igc_q_vector *q_vector = container_of(napi,
4563 						     struct igc_q_vector,
4564 						     napi);
4565 	struct igc_ring *rx_ring = q_vector->rx.ring;
4566 	bool clean_complete = true;
4567 	int work_done = 0;
4568 
4569 	if (q_vector->tx.ring)
4570 		clean_complete = igc_clean_tx_irq(q_vector, budget);
4571 
4572 	if (rx_ring) {
4573 		int cleaned = rx_ring->xsk_pool ?
4574 			      igc_clean_rx_irq_zc(q_vector, budget) :
4575 			      igc_clean_rx_irq(q_vector, budget);
4576 
4577 		work_done += cleaned;
4578 		if (cleaned >= budget)
4579 			clean_complete = false;
4580 	}
4581 
4582 	/* If all work not completed, return budget and keep polling */
4583 	if (!clean_complete)
4584 		return budget;
4585 
4586 	/* Exit the polling mode, but don't re-enable interrupts if stack might
4587 	 * poll us due to busy-polling
4588 	 */
4589 	if (likely(napi_complete_done(napi, work_done)))
4590 		igc_ring_irq_enable(q_vector);
4591 
4592 	return min(work_done, budget - 1);
4593 }
4594 
4595 /**
4596  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
4597  * @adapter: board private structure to initialize
4598  * @v_count: q_vectors allocated on adapter, used for ring interleaving
4599  * @v_idx: index of vector in adapter struct
4600  * @txr_count: total number of Tx rings to allocate
4601  * @txr_idx: index of first Tx ring to allocate
4602  * @rxr_count: total number of Rx rings to allocate
4603  * @rxr_idx: index of first Rx ring to allocate
4604  *
4605  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
4606  */
igc_alloc_q_vector(struct igc_adapter * adapter,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)4607 static int igc_alloc_q_vector(struct igc_adapter *adapter,
4608 			      unsigned int v_count, unsigned int v_idx,
4609 			      unsigned int txr_count, unsigned int txr_idx,
4610 			      unsigned int rxr_count, unsigned int rxr_idx)
4611 {
4612 	struct igc_q_vector *q_vector;
4613 	struct igc_ring *ring;
4614 	int ring_count;
4615 
4616 	/* igc only supports 1 Tx and/or 1 Rx queue per vector */
4617 	if (txr_count > 1 || rxr_count > 1)
4618 		return -ENOMEM;
4619 
4620 	ring_count = txr_count + rxr_count;
4621 
4622 	/* allocate q_vector and rings */
4623 	q_vector = adapter->q_vector[v_idx];
4624 	if (!q_vector)
4625 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
4626 				   GFP_KERNEL);
4627 	else
4628 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
4629 	if (!q_vector)
4630 		return -ENOMEM;
4631 
4632 	/* initialize NAPI */
4633 	netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
4634 
4635 	/* tie q_vector and adapter together */
4636 	adapter->q_vector[v_idx] = q_vector;
4637 	q_vector->adapter = adapter;
4638 
4639 	/* initialize work limits */
4640 	q_vector->tx.work_limit = adapter->tx_work_limit;
4641 
4642 	/* initialize ITR configuration */
4643 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
4644 	q_vector->itr_val = IGC_START_ITR;
4645 
4646 	/* initialize pointer to rings */
4647 	ring = q_vector->ring;
4648 
4649 	/* initialize ITR */
4650 	if (rxr_count) {
4651 		/* rx or rx/tx vector */
4652 		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
4653 			q_vector->itr_val = adapter->rx_itr_setting;
4654 	} else {
4655 		/* tx only vector */
4656 		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
4657 			q_vector->itr_val = adapter->tx_itr_setting;
4658 	}
4659 
4660 	if (txr_count) {
4661 		/* assign generic ring traits */
4662 		ring->dev = &adapter->pdev->dev;
4663 		ring->netdev = adapter->netdev;
4664 
4665 		/* configure backlink on ring */
4666 		ring->q_vector = q_vector;
4667 
4668 		/* update q_vector Tx values */
4669 		igc_add_ring(ring, &q_vector->tx);
4670 
4671 		/* apply Tx specific ring traits */
4672 		ring->count = adapter->tx_ring_count;
4673 		ring->queue_index = txr_idx;
4674 
4675 		/* assign ring to adapter */
4676 		adapter->tx_ring[txr_idx] = ring;
4677 
4678 		/* push pointer to next ring */
4679 		ring++;
4680 	}
4681 
4682 	if (rxr_count) {
4683 		/* assign generic ring traits */
4684 		ring->dev = &adapter->pdev->dev;
4685 		ring->netdev = adapter->netdev;
4686 
4687 		/* configure backlink on ring */
4688 		ring->q_vector = q_vector;
4689 
4690 		/* update q_vector Rx values */
4691 		igc_add_ring(ring, &q_vector->rx);
4692 
4693 		/* apply Rx specific ring traits */
4694 		ring->count = adapter->rx_ring_count;
4695 		ring->queue_index = rxr_idx;
4696 
4697 		/* assign ring to adapter */
4698 		adapter->rx_ring[rxr_idx] = ring;
4699 	}
4700 
4701 	return 0;
4702 }
4703 
4704 /**
4705  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
4706  * @adapter: board private structure to initialize
4707  *
4708  * We allocate one q_vector per queue interrupt.  If allocation fails we
4709  * return -ENOMEM.
4710  */
igc_alloc_q_vectors(struct igc_adapter * adapter)4711 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
4712 {
4713 	int rxr_remaining = adapter->num_rx_queues;
4714 	int txr_remaining = adapter->num_tx_queues;
4715 	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
4716 	int q_vectors = adapter->num_q_vectors;
4717 	int err;
4718 
4719 	if (q_vectors >= (rxr_remaining + txr_remaining)) {
4720 		for (; rxr_remaining; v_idx++) {
4721 			err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4722 						 0, 0, 1, rxr_idx);
4723 
4724 			if (err)
4725 				goto err_out;
4726 
4727 			/* update counts and index */
4728 			rxr_remaining--;
4729 			rxr_idx++;
4730 		}
4731 	}
4732 
4733 	for (; v_idx < q_vectors; v_idx++) {
4734 		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
4735 		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
4736 
4737 		err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4738 					 tqpv, txr_idx, rqpv, rxr_idx);
4739 
4740 		if (err)
4741 			goto err_out;
4742 
4743 		/* update counts and index */
4744 		rxr_remaining -= rqpv;
4745 		txr_remaining -= tqpv;
4746 		rxr_idx++;
4747 		txr_idx++;
4748 	}
4749 
4750 	return 0;
4751 
4752 err_out:
4753 	adapter->num_tx_queues = 0;
4754 	adapter->num_rx_queues = 0;
4755 	adapter->num_q_vectors = 0;
4756 
4757 	while (v_idx--)
4758 		igc_free_q_vector(adapter, v_idx);
4759 
4760 	return -ENOMEM;
4761 }
4762 
4763 /**
4764  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
4765  * @adapter: Pointer to adapter structure
4766  * @msix: boolean for MSI-X capability
4767  *
4768  * This function initializes the interrupts and allocates all of the queues.
4769  */
igc_init_interrupt_scheme(struct igc_adapter * adapter,bool msix)4770 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
4771 {
4772 	struct net_device *dev = adapter->netdev;
4773 	int err = 0;
4774 
4775 	igc_set_interrupt_capability(adapter, msix);
4776 
4777 	err = igc_alloc_q_vectors(adapter);
4778 	if (err) {
4779 		netdev_err(dev, "Unable to allocate memory for vectors\n");
4780 		goto err_alloc_q_vectors;
4781 	}
4782 
4783 	igc_cache_ring_register(adapter);
4784 
4785 	return 0;
4786 
4787 err_alloc_q_vectors:
4788 	igc_reset_interrupt_capability(adapter);
4789 	return err;
4790 }
4791 
4792 /**
4793  * igc_sw_init - Initialize general software structures (struct igc_adapter)
4794  * @adapter: board private structure to initialize
4795  *
4796  * igc_sw_init initializes the Adapter private data structure.
4797  * Fields are initialized based on PCI device information and
4798  * OS network device settings (MTU size).
4799  */
igc_sw_init(struct igc_adapter * adapter)4800 static int igc_sw_init(struct igc_adapter *adapter)
4801 {
4802 	struct net_device *netdev = adapter->netdev;
4803 	struct pci_dev *pdev = adapter->pdev;
4804 	struct igc_hw *hw = &adapter->hw;
4805 
4806 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4807 
4808 	/* set default ring sizes */
4809 	adapter->tx_ring_count = IGC_DEFAULT_TXD;
4810 	adapter->rx_ring_count = IGC_DEFAULT_RXD;
4811 
4812 	/* set default ITR values */
4813 	adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4814 	adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4815 
4816 	/* set default work limits */
4817 	adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4818 
4819 	/* adjust max frame to be at least the size of a standard frame */
4820 	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4821 				VLAN_HLEN;
4822 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4823 
4824 	mutex_init(&adapter->nfc_rule_lock);
4825 	INIT_LIST_HEAD(&adapter->nfc_rule_list);
4826 	adapter->nfc_rule_count = 0;
4827 
4828 	spin_lock_init(&adapter->stats64_lock);
4829 	spin_lock_init(&adapter->qbv_tx_lock);
4830 	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
4831 	adapter->flags |= IGC_FLAG_HAS_MSIX;
4832 
4833 	igc_init_queue_configuration(adapter);
4834 
4835 	/* This call may decrease the number of queues */
4836 	if (igc_init_interrupt_scheme(adapter, true)) {
4837 		netdev_err(netdev, "Unable to allocate memory for queues\n");
4838 		return -ENOMEM;
4839 	}
4840 
4841 	/* Explicitly disable IRQ since the NIC can be in any state. */
4842 	igc_irq_disable(adapter);
4843 
4844 	set_bit(__IGC_DOWN, &adapter->state);
4845 
4846 	return 0;
4847 }
4848 
4849 /**
4850  * igc_up - Open the interface and prepare it to handle traffic
4851  * @adapter: board private structure
4852  */
igc_up(struct igc_adapter * adapter)4853 void igc_up(struct igc_adapter *adapter)
4854 {
4855 	struct igc_hw *hw = &adapter->hw;
4856 	int i = 0;
4857 
4858 	/* hardware has been reset, we need to reload some things */
4859 	igc_configure(adapter);
4860 
4861 	clear_bit(__IGC_DOWN, &adapter->state);
4862 
4863 	for (i = 0; i < adapter->num_q_vectors; i++)
4864 		napi_enable(&adapter->q_vector[i]->napi);
4865 
4866 	if (adapter->msix_entries)
4867 		igc_configure_msix(adapter);
4868 	else
4869 		igc_assign_vector(adapter->q_vector[0], 0);
4870 
4871 	/* Clear any pending interrupts. */
4872 	rd32(IGC_ICR);
4873 	igc_irq_enable(adapter);
4874 
4875 	netif_tx_start_all_queues(adapter->netdev);
4876 
4877 	/* start the watchdog. */
4878 	hw->mac.get_link_status = true;
4879 	schedule_work(&adapter->watchdog_task);
4880 }
4881 
4882 /**
4883  * igc_update_stats - Update the board statistics counters
4884  * @adapter: board private structure
4885  */
igc_update_stats(struct igc_adapter * adapter)4886 void igc_update_stats(struct igc_adapter *adapter)
4887 {
4888 	struct rtnl_link_stats64 *net_stats = &adapter->stats64;
4889 	struct pci_dev *pdev = adapter->pdev;
4890 	struct igc_hw *hw = &adapter->hw;
4891 	u64 _bytes, _packets;
4892 	u64 bytes, packets;
4893 	unsigned int start;
4894 	u32 mpc;
4895 	int i;
4896 
4897 	/* Prevent stats update while adapter is being reset, or if the pci
4898 	 * connection is down.
4899 	 */
4900 	if (adapter->link_speed == 0)
4901 		return;
4902 	if (pci_channel_offline(pdev))
4903 		return;
4904 
4905 	packets = 0;
4906 	bytes = 0;
4907 
4908 	rcu_read_lock();
4909 	for (i = 0; i < adapter->num_rx_queues; i++) {
4910 		struct igc_ring *ring = adapter->rx_ring[i];
4911 		u32 rqdpc = rd32(IGC_RQDPC(i));
4912 
4913 		if (hw->mac.type >= igc_i225)
4914 			wr32(IGC_RQDPC(i), 0);
4915 
4916 		if (rqdpc) {
4917 			ring->rx_stats.drops += rqdpc;
4918 			net_stats->rx_fifo_errors += rqdpc;
4919 		}
4920 
4921 		do {
4922 			start = u64_stats_fetch_begin(&ring->rx_syncp);
4923 			_bytes = ring->rx_stats.bytes;
4924 			_packets = ring->rx_stats.packets;
4925 		} while (u64_stats_fetch_retry(&ring->rx_syncp, start));
4926 		bytes += _bytes;
4927 		packets += _packets;
4928 	}
4929 
4930 	net_stats->rx_bytes = bytes;
4931 	net_stats->rx_packets = packets;
4932 
4933 	packets = 0;
4934 	bytes = 0;
4935 	for (i = 0; i < adapter->num_tx_queues; i++) {
4936 		struct igc_ring *ring = adapter->tx_ring[i];
4937 
4938 		do {
4939 			start = u64_stats_fetch_begin(&ring->tx_syncp);
4940 			_bytes = ring->tx_stats.bytes;
4941 			_packets = ring->tx_stats.packets;
4942 		} while (u64_stats_fetch_retry(&ring->tx_syncp, start));
4943 		bytes += _bytes;
4944 		packets += _packets;
4945 	}
4946 	net_stats->tx_bytes = bytes;
4947 	net_stats->tx_packets = packets;
4948 	rcu_read_unlock();
4949 
4950 	/* read stats registers */
4951 	adapter->stats.crcerrs += rd32(IGC_CRCERRS);
4952 	adapter->stats.gprc += rd32(IGC_GPRC);
4953 	adapter->stats.gorc += rd32(IGC_GORCL);
4954 	rd32(IGC_GORCH); /* clear GORCL */
4955 	adapter->stats.bprc += rd32(IGC_BPRC);
4956 	adapter->stats.mprc += rd32(IGC_MPRC);
4957 	adapter->stats.roc += rd32(IGC_ROC);
4958 
4959 	adapter->stats.prc64 += rd32(IGC_PRC64);
4960 	adapter->stats.prc127 += rd32(IGC_PRC127);
4961 	adapter->stats.prc255 += rd32(IGC_PRC255);
4962 	adapter->stats.prc511 += rd32(IGC_PRC511);
4963 	adapter->stats.prc1023 += rd32(IGC_PRC1023);
4964 	adapter->stats.prc1522 += rd32(IGC_PRC1522);
4965 	adapter->stats.tlpic += rd32(IGC_TLPIC);
4966 	adapter->stats.rlpic += rd32(IGC_RLPIC);
4967 	adapter->stats.hgptc += rd32(IGC_HGPTC);
4968 
4969 	mpc = rd32(IGC_MPC);
4970 	adapter->stats.mpc += mpc;
4971 	net_stats->rx_fifo_errors += mpc;
4972 	adapter->stats.scc += rd32(IGC_SCC);
4973 	adapter->stats.ecol += rd32(IGC_ECOL);
4974 	adapter->stats.mcc += rd32(IGC_MCC);
4975 	adapter->stats.latecol += rd32(IGC_LATECOL);
4976 	adapter->stats.dc += rd32(IGC_DC);
4977 	adapter->stats.rlec += rd32(IGC_RLEC);
4978 	adapter->stats.xonrxc += rd32(IGC_XONRXC);
4979 	adapter->stats.xontxc += rd32(IGC_XONTXC);
4980 	adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
4981 	adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
4982 	adapter->stats.fcruc += rd32(IGC_FCRUC);
4983 	adapter->stats.gptc += rd32(IGC_GPTC);
4984 	adapter->stats.gotc += rd32(IGC_GOTCL);
4985 	rd32(IGC_GOTCH); /* clear GOTCL */
4986 	adapter->stats.rnbc += rd32(IGC_RNBC);
4987 	adapter->stats.ruc += rd32(IGC_RUC);
4988 	adapter->stats.rfc += rd32(IGC_RFC);
4989 	adapter->stats.rjc += rd32(IGC_RJC);
4990 	adapter->stats.tor += rd32(IGC_TORH);
4991 	adapter->stats.tot += rd32(IGC_TOTH);
4992 	adapter->stats.tpr += rd32(IGC_TPR);
4993 
4994 	adapter->stats.ptc64 += rd32(IGC_PTC64);
4995 	adapter->stats.ptc127 += rd32(IGC_PTC127);
4996 	adapter->stats.ptc255 += rd32(IGC_PTC255);
4997 	adapter->stats.ptc511 += rd32(IGC_PTC511);
4998 	adapter->stats.ptc1023 += rd32(IGC_PTC1023);
4999 	adapter->stats.ptc1522 += rd32(IGC_PTC1522);
5000 
5001 	adapter->stats.mptc += rd32(IGC_MPTC);
5002 	adapter->stats.bptc += rd32(IGC_BPTC);
5003 
5004 	adapter->stats.tpt += rd32(IGC_TPT);
5005 	adapter->stats.colc += rd32(IGC_COLC);
5006 	adapter->stats.colc += rd32(IGC_RERC);
5007 
5008 	adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
5009 
5010 	adapter->stats.tsctc += rd32(IGC_TSCTC);
5011 
5012 	adapter->stats.iac += rd32(IGC_IAC);
5013 
5014 	/* Fill out the OS statistics structure */
5015 	net_stats->multicast = adapter->stats.mprc;
5016 	net_stats->collisions = adapter->stats.colc;
5017 
5018 	/* Rx Errors */
5019 
5020 	/* RLEC on some newer hardware can be incorrect so build
5021 	 * our own version based on RUC and ROC
5022 	 */
5023 	net_stats->rx_errors = adapter->stats.rxerrc +
5024 		adapter->stats.crcerrs + adapter->stats.algnerrc +
5025 		adapter->stats.ruc + adapter->stats.roc +
5026 		adapter->stats.cexterr;
5027 	net_stats->rx_length_errors = adapter->stats.ruc +
5028 				      adapter->stats.roc;
5029 	net_stats->rx_crc_errors = adapter->stats.crcerrs;
5030 	net_stats->rx_frame_errors = adapter->stats.algnerrc;
5031 	net_stats->rx_missed_errors = adapter->stats.mpc;
5032 
5033 	/* Tx Errors */
5034 	net_stats->tx_errors = adapter->stats.ecol +
5035 			       adapter->stats.latecol;
5036 	net_stats->tx_aborted_errors = adapter->stats.ecol;
5037 	net_stats->tx_window_errors = adapter->stats.latecol;
5038 	net_stats->tx_carrier_errors = adapter->stats.tncrs;
5039 
5040 	/* Tx Dropped */
5041 	net_stats->tx_dropped = adapter->stats.txdrop;
5042 
5043 	/* Management Stats */
5044 	adapter->stats.mgptc += rd32(IGC_MGTPTC);
5045 	adapter->stats.mgprc += rd32(IGC_MGTPRC);
5046 	adapter->stats.mgpdc += rd32(IGC_MGTPDC);
5047 }
5048 
5049 /**
5050  * igc_down - Close the interface
5051  * @adapter: board private structure
5052  */
igc_down(struct igc_adapter * adapter)5053 void igc_down(struct igc_adapter *adapter)
5054 {
5055 	struct net_device *netdev = adapter->netdev;
5056 	struct igc_hw *hw = &adapter->hw;
5057 	u32 tctl, rctl;
5058 	int i = 0;
5059 
5060 	set_bit(__IGC_DOWN, &adapter->state);
5061 
5062 	igc_ptp_suspend(adapter);
5063 
5064 	if (pci_device_is_present(adapter->pdev)) {
5065 		/* disable receives in the hardware */
5066 		rctl = rd32(IGC_RCTL);
5067 		wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
5068 		/* flush and sleep below */
5069 	}
5070 	/* set trans_start so we don't get spurious watchdogs during reset */
5071 	netif_trans_update(netdev);
5072 
5073 	netif_carrier_off(netdev);
5074 	netif_tx_stop_all_queues(netdev);
5075 
5076 	if (pci_device_is_present(adapter->pdev)) {
5077 		/* disable transmits in the hardware */
5078 		tctl = rd32(IGC_TCTL);
5079 		tctl &= ~IGC_TCTL_EN;
5080 		wr32(IGC_TCTL, tctl);
5081 		/* flush both disables and wait for them to finish */
5082 		wrfl();
5083 		usleep_range(10000, 20000);
5084 
5085 		igc_irq_disable(adapter);
5086 	}
5087 
5088 	adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5089 
5090 	for (i = 0; i < adapter->num_q_vectors; i++) {
5091 		if (adapter->q_vector[i]) {
5092 			napi_synchronize(&adapter->q_vector[i]->napi);
5093 			napi_disable(&adapter->q_vector[i]->napi);
5094 		}
5095 	}
5096 
5097 	del_timer_sync(&adapter->watchdog_timer);
5098 	del_timer_sync(&adapter->phy_info_timer);
5099 
5100 	/* record the stats before reset*/
5101 	spin_lock(&adapter->stats64_lock);
5102 	igc_update_stats(adapter);
5103 	spin_unlock(&adapter->stats64_lock);
5104 
5105 	adapter->link_speed = 0;
5106 	adapter->link_duplex = 0;
5107 
5108 	if (!pci_channel_offline(adapter->pdev))
5109 		igc_reset(adapter);
5110 
5111 	/* clear VLAN promisc flag so VFTA will be updated if necessary */
5112 	adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
5113 
5114 	igc_disable_all_tx_rings_hw(adapter);
5115 	igc_clean_all_tx_rings(adapter);
5116 	igc_clean_all_rx_rings(adapter);
5117 }
5118 
igc_reinit_locked(struct igc_adapter * adapter)5119 void igc_reinit_locked(struct igc_adapter *adapter)
5120 {
5121 	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5122 		usleep_range(1000, 2000);
5123 	igc_down(adapter);
5124 	igc_up(adapter);
5125 	clear_bit(__IGC_RESETTING, &adapter->state);
5126 }
5127 
igc_reset_task(struct work_struct * work)5128 static void igc_reset_task(struct work_struct *work)
5129 {
5130 	struct igc_adapter *adapter;
5131 
5132 	adapter = container_of(work, struct igc_adapter, reset_task);
5133 
5134 	rtnl_lock();
5135 	/* If we're already down or resetting, just bail */
5136 	if (test_bit(__IGC_DOWN, &adapter->state) ||
5137 	    test_bit(__IGC_RESETTING, &adapter->state)) {
5138 		rtnl_unlock();
5139 		return;
5140 	}
5141 
5142 	igc_rings_dump(adapter);
5143 	igc_regs_dump(adapter);
5144 	netdev_err(adapter->netdev, "Reset adapter\n");
5145 	igc_reinit_locked(adapter);
5146 	rtnl_unlock();
5147 }
5148 
5149 /**
5150  * igc_change_mtu - Change the Maximum Transfer Unit
5151  * @netdev: network interface device structure
5152  * @new_mtu: new value for maximum frame size
5153  *
5154  * Returns 0 on success, negative on failure
5155  */
igc_change_mtu(struct net_device * netdev,int new_mtu)5156 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
5157 {
5158 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5159 	struct igc_adapter *adapter = netdev_priv(netdev);
5160 
5161 	if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) {
5162 		netdev_dbg(netdev, "Jumbo frames not supported with XDP");
5163 		return -EINVAL;
5164 	}
5165 
5166 	/* adjust max frame to be at least the size of a standard frame */
5167 	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5168 		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5169 
5170 	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5171 		usleep_range(1000, 2000);
5172 
5173 	/* igc_down has a dependency on max_frame_size */
5174 	adapter->max_frame_size = max_frame;
5175 
5176 	if (netif_running(netdev))
5177 		igc_down(adapter);
5178 
5179 	netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5180 	netdev->mtu = new_mtu;
5181 
5182 	if (netif_running(netdev))
5183 		igc_up(adapter);
5184 	else
5185 		igc_reset(adapter);
5186 
5187 	clear_bit(__IGC_RESETTING, &adapter->state);
5188 
5189 	return 0;
5190 }
5191 
5192 /**
5193  * igc_tx_timeout - Respond to a Tx Hang
5194  * @netdev: network interface device structure
5195  * @txqueue: queue number that timed out
5196  **/
igc_tx_timeout(struct net_device * netdev,unsigned int __always_unused txqueue)5197 static void igc_tx_timeout(struct net_device *netdev,
5198 			   unsigned int __always_unused txqueue)
5199 {
5200 	struct igc_adapter *adapter = netdev_priv(netdev);
5201 	struct igc_hw *hw = &adapter->hw;
5202 
5203 	/* Do the reset outside of interrupt context */
5204 	adapter->tx_timeout_count++;
5205 	schedule_work(&adapter->reset_task);
5206 	wr32(IGC_EICS,
5207 	     (adapter->eims_enable_mask & ~adapter->eims_other));
5208 }
5209 
5210 /**
5211  * igc_get_stats64 - Get System Network Statistics
5212  * @netdev: network interface device structure
5213  * @stats: rtnl_link_stats64 pointer
5214  *
5215  * Returns the address of the device statistics structure.
5216  * The statistics are updated here and also from the timer callback.
5217  */
igc_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)5218 static void igc_get_stats64(struct net_device *netdev,
5219 			    struct rtnl_link_stats64 *stats)
5220 {
5221 	struct igc_adapter *adapter = netdev_priv(netdev);
5222 
5223 	spin_lock(&adapter->stats64_lock);
5224 	if (!test_bit(__IGC_RESETTING, &adapter->state))
5225 		igc_update_stats(adapter);
5226 	memcpy(stats, &adapter->stats64, sizeof(*stats));
5227 	spin_unlock(&adapter->stats64_lock);
5228 }
5229 
igc_fix_features(struct net_device * netdev,netdev_features_t features)5230 static netdev_features_t igc_fix_features(struct net_device *netdev,
5231 					  netdev_features_t features)
5232 {
5233 	/* Since there is no support for separate Rx/Tx vlan accel
5234 	 * enable/disable make sure Tx flag is always in same state as Rx.
5235 	 */
5236 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
5237 		features |= NETIF_F_HW_VLAN_CTAG_TX;
5238 	else
5239 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
5240 
5241 	return features;
5242 }
5243 
igc_set_features(struct net_device * netdev,netdev_features_t features)5244 static int igc_set_features(struct net_device *netdev,
5245 			    netdev_features_t features)
5246 {
5247 	netdev_features_t changed = netdev->features ^ features;
5248 	struct igc_adapter *adapter = netdev_priv(netdev);
5249 
5250 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
5251 		igc_vlan_mode(netdev, features);
5252 
5253 	/* Add VLAN support */
5254 	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
5255 		return 0;
5256 
5257 	if (!(features & NETIF_F_NTUPLE))
5258 		igc_flush_nfc_rules(adapter);
5259 
5260 	netdev->features = features;
5261 
5262 	if (netif_running(netdev))
5263 		igc_reinit_locked(adapter);
5264 	else
5265 		igc_reset(adapter);
5266 
5267 	return 1;
5268 }
5269 
5270 static netdev_features_t
igc_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)5271 igc_features_check(struct sk_buff *skb, struct net_device *dev,
5272 		   netdev_features_t features)
5273 {
5274 	unsigned int network_hdr_len, mac_hdr_len;
5275 
5276 	/* Make certain the headers can be described by a context descriptor */
5277 	mac_hdr_len = skb_network_header(skb) - skb->data;
5278 	if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
5279 		return features & ~(NETIF_F_HW_CSUM |
5280 				    NETIF_F_SCTP_CRC |
5281 				    NETIF_F_HW_VLAN_CTAG_TX |
5282 				    NETIF_F_TSO |
5283 				    NETIF_F_TSO6);
5284 
5285 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
5286 	if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
5287 		return features & ~(NETIF_F_HW_CSUM |
5288 				    NETIF_F_SCTP_CRC |
5289 				    NETIF_F_TSO |
5290 				    NETIF_F_TSO6);
5291 
5292 	/* We can only support IPv4 TSO in tunnels if we can mangle the
5293 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
5294 	 */
5295 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
5296 		features &= ~NETIF_F_TSO;
5297 
5298 	return features;
5299 }
5300 
igc_tsync_interrupt(struct igc_adapter * adapter)5301 static void igc_tsync_interrupt(struct igc_adapter *adapter)
5302 {
5303 	struct igc_hw *hw = &adapter->hw;
5304 	u32 tsauxc, sec, nsec, tsicr;
5305 	struct ptp_clock_event event;
5306 	struct timespec64 ts;
5307 
5308 	tsicr = rd32(IGC_TSICR);
5309 
5310 	if (tsicr & IGC_TSICR_SYS_WRAP) {
5311 		event.type = PTP_CLOCK_PPS;
5312 		if (adapter->ptp_caps.pps)
5313 			ptp_clock_event(adapter->ptp_clock, &event);
5314 	}
5315 
5316 	if (tsicr & IGC_TSICR_TXTS) {
5317 		/* retrieve hardware timestamp */
5318 		igc_ptp_tx_tstamp_event(adapter);
5319 	}
5320 
5321 	if (tsicr & IGC_TSICR_TT0) {
5322 		spin_lock(&adapter->tmreg_lock);
5323 		ts = timespec64_add(adapter->perout[0].start,
5324 				    adapter->perout[0].period);
5325 		wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5326 		wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec);
5327 		tsauxc = rd32(IGC_TSAUXC);
5328 		tsauxc |= IGC_TSAUXC_EN_TT0;
5329 		wr32(IGC_TSAUXC, tsauxc);
5330 		adapter->perout[0].start = ts;
5331 		spin_unlock(&adapter->tmreg_lock);
5332 	}
5333 
5334 	if (tsicr & IGC_TSICR_TT1) {
5335 		spin_lock(&adapter->tmreg_lock);
5336 		ts = timespec64_add(adapter->perout[1].start,
5337 				    adapter->perout[1].period);
5338 		wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5339 		wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec);
5340 		tsauxc = rd32(IGC_TSAUXC);
5341 		tsauxc |= IGC_TSAUXC_EN_TT1;
5342 		wr32(IGC_TSAUXC, tsauxc);
5343 		adapter->perout[1].start = ts;
5344 		spin_unlock(&adapter->tmreg_lock);
5345 	}
5346 
5347 	if (tsicr & IGC_TSICR_AUTT0) {
5348 		nsec = rd32(IGC_AUXSTMPL0);
5349 		sec  = rd32(IGC_AUXSTMPH0);
5350 		event.type = PTP_CLOCK_EXTTS;
5351 		event.index = 0;
5352 		event.timestamp = sec * NSEC_PER_SEC + nsec;
5353 		ptp_clock_event(adapter->ptp_clock, &event);
5354 	}
5355 
5356 	if (tsicr & IGC_TSICR_AUTT1) {
5357 		nsec = rd32(IGC_AUXSTMPL1);
5358 		sec  = rd32(IGC_AUXSTMPH1);
5359 		event.type = PTP_CLOCK_EXTTS;
5360 		event.index = 1;
5361 		event.timestamp = sec * NSEC_PER_SEC + nsec;
5362 		ptp_clock_event(adapter->ptp_clock, &event);
5363 	}
5364 }
5365 
5366 /**
5367  * igc_msix_other - msix other interrupt handler
5368  * @irq: interrupt number
5369  * @data: pointer to a q_vector
5370  */
igc_msix_other(int irq,void * data)5371 static irqreturn_t igc_msix_other(int irq, void *data)
5372 {
5373 	struct igc_adapter *adapter = data;
5374 	struct igc_hw *hw = &adapter->hw;
5375 	u32 icr = rd32(IGC_ICR);
5376 
5377 	/* reading ICR causes bit 31 of EICR to be cleared */
5378 	if (icr & IGC_ICR_DRSTA)
5379 		schedule_work(&adapter->reset_task);
5380 
5381 	if (icr & IGC_ICR_DOUTSYNC) {
5382 		/* HW is reporting DMA is out of sync */
5383 		adapter->stats.doosync++;
5384 	}
5385 
5386 	if (icr & IGC_ICR_LSC) {
5387 		hw->mac.get_link_status = true;
5388 		/* guard against interrupt when we're going down */
5389 		if (!test_bit(__IGC_DOWN, &adapter->state))
5390 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5391 	}
5392 
5393 	if (icr & IGC_ICR_TS)
5394 		igc_tsync_interrupt(adapter);
5395 
5396 	wr32(IGC_EIMS, adapter->eims_other);
5397 
5398 	return IRQ_HANDLED;
5399 }
5400 
igc_write_itr(struct igc_q_vector * q_vector)5401 static void igc_write_itr(struct igc_q_vector *q_vector)
5402 {
5403 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
5404 
5405 	if (!q_vector->set_itr)
5406 		return;
5407 
5408 	if (!itr_val)
5409 		itr_val = IGC_ITR_VAL_MASK;
5410 
5411 	itr_val |= IGC_EITR_CNT_IGNR;
5412 
5413 	writel(itr_val, q_vector->itr_register);
5414 	q_vector->set_itr = 0;
5415 }
5416 
igc_msix_ring(int irq,void * data)5417 static irqreturn_t igc_msix_ring(int irq, void *data)
5418 {
5419 	struct igc_q_vector *q_vector = data;
5420 
5421 	/* Write the ITR value calculated from the previous interrupt. */
5422 	igc_write_itr(q_vector);
5423 
5424 	napi_schedule(&q_vector->napi);
5425 
5426 	return IRQ_HANDLED;
5427 }
5428 
5429 /**
5430  * igc_request_msix - Initialize MSI-X interrupts
5431  * @adapter: Pointer to adapter structure
5432  *
5433  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
5434  * kernel.
5435  */
igc_request_msix(struct igc_adapter * adapter)5436 static int igc_request_msix(struct igc_adapter *adapter)
5437 {
5438 	unsigned int num_q_vectors = adapter->num_q_vectors;
5439 	int i = 0, err = 0, vector = 0, free_vector = 0;
5440 	struct net_device *netdev = adapter->netdev;
5441 
5442 	err = request_irq(adapter->msix_entries[vector].vector,
5443 			  &igc_msix_other, 0, netdev->name, adapter);
5444 	if (err)
5445 		goto err_out;
5446 
5447 	if (num_q_vectors > MAX_Q_VECTORS) {
5448 		num_q_vectors = MAX_Q_VECTORS;
5449 		dev_warn(&adapter->pdev->dev,
5450 			 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5451 			 adapter->num_q_vectors, MAX_Q_VECTORS);
5452 	}
5453 	for (i = 0; i < num_q_vectors; i++) {
5454 		struct igc_q_vector *q_vector = adapter->q_vector[i];
5455 
5456 		vector++;
5457 
5458 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
5459 
5460 		if (q_vector->rx.ring && q_vector->tx.ring)
5461 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
5462 				q_vector->rx.ring->queue_index);
5463 		else if (q_vector->tx.ring)
5464 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
5465 				q_vector->tx.ring->queue_index);
5466 		else if (q_vector->rx.ring)
5467 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
5468 				q_vector->rx.ring->queue_index);
5469 		else
5470 			sprintf(q_vector->name, "%s-unused", netdev->name);
5471 
5472 		err = request_irq(adapter->msix_entries[vector].vector,
5473 				  igc_msix_ring, 0, q_vector->name,
5474 				  q_vector);
5475 		if (err)
5476 			goto err_free;
5477 	}
5478 
5479 	igc_configure_msix(adapter);
5480 	return 0;
5481 
5482 err_free:
5483 	/* free already assigned IRQs */
5484 	free_irq(adapter->msix_entries[free_vector++].vector, adapter);
5485 
5486 	vector--;
5487 	for (i = 0; i < vector; i++) {
5488 		free_irq(adapter->msix_entries[free_vector++].vector,
5489 			 adapter->q_vector[i]);
5490 	}
5491 err_out:
5492 	return err;
5493 }
5494 
5495 /**
5496  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5497  * @adapter: Pointer to adapter structure
5498  *
5499  * This function resets the device so that it has 0 rx queues, tx queues, and
5500  * MSI-X interrupts allocated.
5501  */
igc_clear_interrupt_scheme(struct igc_adapter * adapter)5502 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
5503 {
5504 	igc_free_q_vectors(adapter);
5505 	igc_reset_interrupt_capability(adapter);
5506 }
5507 
5508 /* Need to wait a few seconds after link up to get diagnostic information from
5509  * the phy
5510  */
igc_update_phy_info(struct timer_list * t)5511 static void igc_update_phy_info(struct timer_list *t)
5512 {
5513 	struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5514 
5515 	igc_get_phy_info(&adapter->hw);
5516 }
5517 
5518 /**
5519  * igc_has_link - check shared code for link and determine up/down
5520  * @adapter: pointer to driver private info
5521  */
igc_has_link(struct igc_adapter * adapter)5522 bool igc_has_link(struct igc_adapter *adapter)
5523 {
5524 	struct igc_hw *hw = &adapter->hw;
5525 	bool link_active = false;
5526 
5527 	/* get_link_status is set on LSC (link status) interrupt or
5528 	 * rx sequence error interrupt.  get_link_status will stay
5529 	 * false until the igc_check_for_link establishes link
5530 	 * for copper adapters ONLY
5531 	 */
5532 	if (!hw->mac.get_link_status)
5533 		return true;
5534 	hw->mac.ops.check_for_link(hw);
5535 	link_active = !hw->mac.get_link_status;
5536 
5537 	if (hw->mac.type == igc_i225) {
5538 		if (!netif_carrier_ok(adapter->netdev)) {
5539 			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5540 		} else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
5541 			adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
5542 			adapter->link_check_timeout = jiffies;
5543 		}
5544 	}
5545 
5546 	return link_active;
5547 }
5548 
5549 /**
5550  * igc_watchdog - Timer Call-back
5551  * @t: timer for the watchdog
5552  */
igc_watchdog(struct timer_list * t)5553 static void igc_watchdog(struct timer_list *t)
5554 {
5555 	struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5556 	/* Do the rest outside of interrupt context */
5557 	schedule_work(&adapter->watchdog_task);
5558 }
5559 
igc_watchdog_task(struct work_struct * work)5560 static void igc_watchdog_task(struct work_struct *work)
5561 {
5562 	struct igc_adapter *adapter = container_of(work,
5563 						   struct igc_adapter,
5564 						   watchdog_task);
5565 	struct net_device *netdev = adapter->netdev;
5566 	struct igc_hw *hw = &adapter->hw;
5567 	struct igc_phy_info *phy = &hw->phy;
5568 	u16 phy_data, retry_count = 20;
5569 	u32 link;
5570 	int i;
5571 
5572 	link = igc_has_link(adapter);
5573 
5574 	if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
5575 		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5576 			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5577 		else
5578 			link = false;
5579 	}
5580 
5581 	if (link) {
5582 		/* Cancel scheduled suspend requests. */
5583 		pm_runtime_resume(netdev->dev.parent);
5584 
5585 		if (!netif_carrier_ok(netdev)) {
5586 			u32 ctrl;
5587 
5588 			hw->mac.ops.get_speed_and_duplex(hw,
5589 							 &adapter->link_speed,
5590 							 &adapter->link_duplex);
5591 
5592 			ctrl = rd32(IGC_CTRL);
5593 			/* Link status message must follow this format */
5594 			netdev_info(netdev,
5595 				    "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5596 				    adapter->link_speed,
5597 				    adapter->link_duplex == FULL_DUPLEX ?
5598 				    "Full" : "Half",
5599 				    (ctrl & IGC_CTRL_TFCE) &&
5600 				    (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
5601 				    (ctrl & IGC_CTRL_RFCE) ?  "RX" :
5602 				    (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
5603 
5604 			/* disable EEE if enabled */
5605 			if ((adapter->flags & IGC_FLAG_EEE) &&
5606 			    adapter->link_duplex == HALF_DUPLEX) {
5607 				netdev_info(netdev,
5608 					    "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
5609 				adapter->hw.dev_spec._base.eee_enable = false;
5610 				adapter->flags &= ~IGC_FLAG_EEE;
5611 			}
5612 
5613 			/* check if SmartSpeed worked */
5614 			igc_check_downshift(hw);
5615 			if (phy->speed_downgraded)
5616 				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5617 
5618 			/* adjust timeout factor according to speed/duplex */
5619 			adapter->tx_timeout_factor = 1;
5620 			switch (adapter->link_speed) {
5621 			case SPEED_10:
5622 				adapter->tx_timeout_factor = 14;
5623 				break;
5624 			case SPEED_100:
5625 			case SPEED_1000:
5626 			case SPEED_2500:
5627 				adapter->tx_timeout_factor = 1;
5628 				break;
5629 			}
5630 
5631 			/* Once the launch time has been set on the wire, there
5632 			 * is a delay before the link speed can be determined
5633 			 * based on link-up activity. Write into the register
5634 			 * as soon as we know the correct link speed.
5635 			 */
5636 			igc_tsn_adjust_txtime_offset(adapter);
5637 
5638 			if (adapter->link_speed != SPEED_1000)
5639 				goto no_wait;
5640 
5641 			/* wait for Remote receiver status OK */
5642 retry_read_status:
5643 			if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
5644 					      &phy_data)) {
5645 				if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5646 				    retry_count) {
5647 					msleep(100);
5648 					retry_count--;
5649 					goto retry_read_status;
5650 				} else if (!retry_count) {
5651 					netdev_err(netdev, "exceed max 2 second\n");
5652 				}
5653 			} else {
5654 				netdev_err(netdev, "read 1000Base-T Status Reg\n");
5655 			}
5656 no_wait:
5657 			netif_carrier_on(netdev);
5658 
5659 			/* link state has changed, schedule phy info update */
5660 			if (!test_bit(__IGC_DOWN, &adapter->state))
5661 				mod_timer(&adapter->phy_info_timer,
5662 					  round_jiffies(jiffies + 2 * HZ));
5663 		}
5664 	} else {
5665 		if (netif_carrier_ok(netdev)) {
5666 			adapter->link_speed = 0;
5667 			adapter->link_duplex = 0;
5668 
5669 			/* Links status message must follow this format */
5670 			netdev_info(netdev, "NIC Link is Down\n");
5671 			netif_carrier_off(netdev);
5672 
5673 			/* link state has changed, schedule phy info update */
5674 			if (!test_bit(__IGC_DOWN, &adapter->state))
5675 				mod_timer(&adapter->phy_info_timer,
5676 					  round_jiffies(jiffies + 2 * HZ));
5677 
5678 			pm_schedule_suspend(netdev->dev.parent,
5679 					    MSEC_PER_SEC * 5);
5680 		}
5681 	}
5682 
5683 	spin_lock(&adapter->stats64_lock);
5684 	igc_update_stats(adapter);
5685 	spin_unlock(&adapter->stats64_lock);
5686 
5687 	for (i = 0; i < adapter->num_tx_queues; i++) {
5688 		struct igc_ring *tx_ring = adapter->tx_ring[i];
5689 
5690 		if (!netif_carrier_ok(netdev)) {
5691 			/* We've lost link, so the controller stops DMA,
5692 			 * but we've got queued Tx work that's never going
5693 			 * to get done, so reset controller to flush Tx.
5694 			 * (Do the reset outside of interrupt context).
5695 			 */
5696 			if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
5697 				adapter->tx_timeout_count++;
5698 				schedule_work(&adapter->reset_task);
5699 				/* return immediately since reset is imminent */
5700 				return;
5701 			}
5702 		}
5703 
5704 		/* Force detection of hung controller every watchdog period */
5705 		set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5706 	}
5707 
5708 	/* Cause software interrupt to ensure Rx ring is cleaned */
5709 	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5710 		u32 eics = 0;
5711 
5712 		for (i = 0; i < adapter->num_q_vectors; i++)
5713 			eics |= adapter->q_vector[i]->eims_value;
5714 		wr32(IGC_EICS, eics);
5715 	} else {
5716 		wr32(IGC_ICS, IGC_ICS_RXDMT0);
5717 	}
5718 
5719 	igc_ptp_tx_hang(adapter);
5720 
5721 	/* Reset the timer */
5722 	if (!test_bit(__IGC_DOWN, &adapter->state)) {
5723 		if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
5724 			mod_timer(&adapter->watchdog_timer,
5725 				  round_jiffies(jiffies +  HZ));
5726 		else
5727 			mod_timer(&adapter->watchdog_timer,
5728 				  round_jiffies(jiffies + 2 * HZ));
5729 	}
5730 }
5731 
5732 /**
5733  * igc_intr_msi - Interrupt Handler
5734  * @irq: interrupt number
5735  * @data: pointer to a network interface device structure
5736  */
igc_intr_msi(int irq,void * data)5737 static irqreturn_t igc_intr_msi(int irq, void *data)
5738 {
5739 	struct igc_adapter *adapter = data;
5740 	struct igc_q_vector *q_vector = adapter->q_vector[0];
5741 	struct igc_hw *hw = &adapter->hw;
5742 	/* read ICR disables interrupts using IAM */
5743 	u32 icr = rd32(IGC_ICR);
5744 
5745 	igc_write_itr(q_vector);
5746 
5747 	if (icr & IGC_ICR_DRSTA)
5748 		schedule_work(&adapter->reset_task);
5749 
5750 	if (icr & IGC_ICR_DOUTSYNC) {
5751 		/* HW is reporting DMA is out of sync */
5752 		adapter->stats.doosync++;
5753 	}
5754 
5755 	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5756 		hw->mac.get_link_status = true;
5757 		if (!test_bit(__IGC_DOWN, &adapter->state))
5758 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5759 	}
5760 
5761 	if (icr & IGC_ICR_TS)
5762 		igc_tsync_interrupt(adapter);
5763 
5764 	napi_schedule(&q_vector->napi);
5765 
5766 	return IRQ_HANDLED;
5767 }
5768 
5769 /**
5770  * igc_intr - Legacy Interrupt Handler
5771  * @irq: interrupt number
5772  * @data: pointer to a network interface device structure
5773  */
igc_intr(int irq,void * data)5774 static irqreturn_t igc_intr(int irq, void *data)
5775 {
5776 	struct igc_adapter *adapter = data;
5777 	struct igc_q_vector *q_vector = adapter->q_vector[0];
5778 	struct igc_hw *hw = &adapter->hw;
5779 	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5780 	 * need for the IMC write
5781 	 */
5782 	u32 icr = rd32(IGC_ICR);
5783 
5784 	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5785 	 * not set, then the adapter didn't send an interrupt
5786 	 */
5787 	if (!(icr & IGC_ICR_INT_ASSERTED))
5788 		return IRQ_NONE;
5789 
5790 	igc_write_itr(q_vector);
5791 
5792 	if (icr & IGC_ICR_DRSTA)
5793 		schedule_work(&adapter->reset_task);
5794 
5795 	if (icr & IGC_ICR_DOUTSYNC) {
5796 		/* HW is reporting DMA is out of sync */
5797 		adapter->stats.doosync++;
5798 	}
5799 
5800 	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5801 		hw->mac.get_link_status = true;
5802 		/* guard against interrupt when we're going down */
5803 		if (!test_bit(__IGC_DOWN, &adapter->state))
5804 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5805 	}
5806 
5807 	if (icr & IGC_ICR_TS)
5808 		igc_tsync_interrupt(adapter);
5809 
5810 	napi_schedule(&q_vector->napi);
5811 
5812 	return IRQ_HANDLED;
5813 }
5814 
igc_free_irq(struct igc_adapter * adapter)5815 static void igc_free_irq(struct igc_adapter *adapter)
5816 {
5817 	if (adapter->msix_entries) {
5818 		int vector = 0, i;
5819 
5820 		free_irq(adapter->msix_entries[vector++].vector, adapter);
5821 
5822 		for (i = 0; i < adapter->num_q_vectors; i++)
5823 			free_irq(adapter->msix_entries[vector++].vector,
5824 				 adapter->q_vector[i]);
5825 	} else {
5826 		free_irq(adapter->pdev->irq, adapter);
5827 	}
5828 }
5829 
5830 /**
5831  * igc_request_irq - initialize interrupts
5832  * @adapter: Pointer to adapter structure
5833  *
5834  * Attempts to configure interrupts using the best available
5835  * capabilities of the hardware and kernel.
5836  */
igc_request_irq(struct igc_adapter * adapter)5837 static int igc_request_irq(struct igc_adapter *adapter)
5838 {
5839 	struct net_device *netdev = adapter->netdev;
5840 	struct pci_dev *pdev = adapter->pdev;
5841 	int err = 0;
5842 
5843 	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5844 		err = igc_request_msix(adapter);
5845 		if (!err)
5846 			goto request_done;
5847 		/* fall back to MSI */
5848 		igc_free_all_tx_resources(adapter);
5849 		igc_free_all_rx_resources(adapter);
5850 
5851 		igc_clear_interrupt_scheme(adapter);
5852 		err = igc_init_interrupt_scheme(adapter, false);
5853 		if (err)
5854 			goto request_done;
5855 		igc_setup_all_tx_resources(adapter);
5856 		igc_setup_all_rx_resources(adapter);
5857 		igc_configure(adapter);
5858 	}
5859 
5860 	igc_assign_vector(adapter->q_vector[0], 0);
5861 
5862 	if (adapter->flags & IGC_FLAG_HAS_MSI) {
5863 		err = request_irq(pdev->irq, &igc_intr_msi, 0,
5864 				  netdev->name, adapter);
5865 		if (!err)
5866 			goto request_done;
5867 
5868 		/* fall back to legacy interrupts */
5869 		igc_reset_interrupt_capability(adapter);
5870 		adapter->flags &= ~IGC_FLAG_HAS_MSI;
5871 	}
5872 
5873 	err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
5874 			  netdev->name, adapter);
5875 
5876 	if (err)
5877 		netdev_err(netdev, "Error %d getting interrupt\n", err);
5878 
5879 request_done:
5880 	return err;
5881 }
5882 
5883 /**
5884  * __igc_open - Called when a network interface is made active
5885  * @netdev: network interface device structure
5886  * @resuming: boolean indicating if the device is resuming
5887  *
5888  * Returns 0 on success, negative value on failure
5889  *
5890  * The open entry point is called when a network interface is made
5891  * active by the system (IFF_UP).  At this point all resources needed
5892  * for transmit and receive operations are allocated, the interrupt
5893  * handler is registered with the OS, the watchdog timer is started,
5894  * and the stack is notified that the interface is ready.
5895  */
__igc_open(struct net_device * netdev,bool resuming)5896 static int __igc_open(struct net_device *netdev, bool resuming)
5897 {
5898 	struct igc_adapter *adapter = netdev_priv(netdev);
5899 	struct pci_dev *pdev = adapter->pdev;
5900 	struct igc_hw *hw = &adapter->hw;
5901 	int err = 0;
5902 	int i = 0;
5903 
5904 	/* disallow open during test */
5905 
5906 	if (test_bit(__IGC_TESTING, &adapter->state)) {
5907 		WARN_ON(resuming);
5908 		return -EBUSY;
5909 	}
5910 
5911 	if (!resuming)
5912 		pm_runtime_get_sync(&pdev->dev);
5913 
5914 	netif_carrier_off(netdev);
5915 
5916 	/* allocate transmit descriptors */
5917 	err = igc_setup_all_tx_resources(adapter);
5918 	if (err)
5919 		goto err_setup_tx;
5920 
5921 	/* allocate receive descriptors */
5922 	err = igc_setup_all_rx_resources(adapter);
5923 	if (err)
5924 		goto err_setup_rx;
5925 
5926 	igc_power_up_link(adapter);
5927 
5928 	igc_configure(adapter);
5929 
5930 	err = igc_request_irq(adapter);
5931 	if (err)
5932 		goto err_req_irq;
5933 
5934 	/* Notify the stack of the actual queue counts. */
5935 	err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
5936 	if (err)
5937 		goto err_set_queues;
5938 
5939 	err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
5940 	if (err)
5941 		goto err_set_queues;
5942 
5943 	clear_bit(__IGC_DOWN, &adapter->state);
5944 
5945 	for (i = 0; i < adapter->num_q_vectors; i++)
5946 		napi_enable(&adapter->q_vector[i]->napi);
5947 
5948 	/* Clear any pending interrupts. */
5949 	rd32(IGC_ICR);
5950 	igc_irq_enable(adapter);
5951 
5952 	if (!resuming)
5953 		pm_runtime_put(&pdev->dev);
5954 
5955 	netif_tx_start_all_queues(netdev);
5956 
5957 	/* start the watchdog. */
5958 	hw->mac.get_link_status = true;
5959 	schedule_work(&adapter->watchdog_task);
5960 
5961 	return IGC_SUCCESS;
5962 
5963 err_set_queues:
5964 	igc_free_irq(adapter);
5965 err_req_irq:
5966 	igc_release_hw_control(adapter);
5967 	igc_power_down_phy_copper_base(&adapter->hw);
5968 	igc_free_all_rx_resources(adapter);
5969 err_setup_rx:
5970 	igc_free_all_tx_resources(adapter);
5971 err_setup_tx:
5972 	igc_reset(adapter);
5973 	if (!resuming)
5974 		pm_runtime_put(&pdev->dev);
5975 
5976 	return err;
5977 }
5978 
igc_open(struct net_device * netdev)5979 int igc_open(struct net_device *netdev)
5980 {
5981 	return __igc_open(netdev, false);
5982 }
5983 
5984 /**
5985  * __igc_close - Disables a network interface
5986  * @netdev: network interface device structure
5987  * @suspending: boolean indicating the device is suspending
5988  *
5989  * Returns 0, this is not allowed to fail
5990  *
5991  * The close entry point is called when an interface is de-activated
5992  * by the OS.  The hardware is still under the driver's control, but
5993  * needs to be disabled.  A global MAC reset is issued to stop the
5994  * hardware, and all transmit and receive resources are freed.
5995  */
__igc_close(struct net_device * netdev,bool suspending)5996 static int __igc_close(struct net_device *netdev, bool suspending)
5997 {
5998 	struct igc_adapter *adapter = netdev_priv(netdev);
5999 	struct pci_dev *pdev = adapter->pdev;
6000 
6001 	WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
6002 
6003 	if (!suspending)
6004 		pm_runtime_get_sync(&pdev->dev);
6005 
6006 	igc_down(adapter);
6007 
6008 	igc_release_hw_control(adapter);
6009 
6010 	igc_free_irq(adapter);
6011 
6012 	igc_free_all_tx_resources(adapter);
6013 	igc_free_all_rx_resources(adapter);
6014 
6015 	if (!suspending)
6016 		pm_runtime_put_sync(&pdev->dev);
6017 
6018 	return 0;
6019 }
6020 
igc_close(struct net_device * netdev)6021 int igc_close(struct net_device *netdev)
6022 {
6023 	if (netif_device_present(netdev) || netdev->dismantle)
6024 		return __igc_close(netdev, false);
6025 	return 0;
6026 }
6027 
6028 /**
6029  * igc_ioctl - Access the hwtstamp interface
6030  * @netdev: network interface device structure
6031  * @ifr: interface request data
6032  * @cmd: ioctl command
6033  **/
igc_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)6034 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6035 {
6036 	switch (cmd) {
6037 	case SIOCGHWTSTAMP:
6038 		return igc_ptp_get_ts_config(netdev, ifr);
6039 	case SIOCSHWTSTAMP:
6040 		return igc_ptp_set_ts_config(netdev, ifr);
6041 	default:
6042 		return -EOPNOTSUPP;
6043 	}
6044 }
6045 
igc_save_launchtime_params(struct igc_adapter * adapter,int queue,bool enable)6046 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
6047 				      bool enable)
6048 {
6049 	struct igc_ring *ring;
6050 
6051 	if (queue < 0 || queue >= adapter->num_tx_queues)
6052 		return -EINVAL;
6053 
6054 	ring = adapter->tx_ring[queue];
6055 	ring->launchtime_enable = enable;
6056 
6057 	return 0;
6058 }
6059 
is_base_time_past(ktime_t base_time,const struct timespec64 * now)6060 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
6061 {
6062 	struct timespec64 b;
6063 
6064 	b = ktime_to_timespec64(base_time);
6065 
6066 	return timespec64_compare(now, &b) > 0;
6067 }
6068 
validate_schedule(struct igc_adapter * adapter,const struct tc_taprio_qopt_offload * qopt)6069 static bool validate_schedule(struct igc_adapter *adapter,
6070 			      const struct tc_taprio_qopt_offload *qopt)
6071 {
6072 	int queue_uses[IGC_MAX_TX_QUEUES] = { };
6073 	struct igc_hw *hw = &adapter->hw;
6074 	struct timespec64 now;
6075 	size_t n;
6076 
6077 	if (qopt->cycle_time_extension)
6078 		return false;
6079 
6080 	igc_ptp_read(adapter, &now);
6081 
6082 	/* If we program the controller's BASET registers with a time
6083 	 * in the future, it will hold all the packets until that
6084 	 * time, causing a lot of TX Hangs, so to avoid that, we
6085 	 * reject schedules that would start in the future.
6086 	 * Note: Limitation above is no longer in i226.
6087 	 */
6088 	if (!is_base_time_past(qopt->base_time, &now) &&
6089 	    igc_is_device_id_i225(hw))
6090 		return false;
6091 
6092 	for (n = 0; n < qopt->num_entries; n++) {
6093 		const struct tc_taprio_sched_entry *e, *prev;
6094 		int i;
6095 
6096 		prev = n ? &qopt->entries[n - 1] : NULL;
6097 		e = &qopt->entries[n];
6098 
6099 		/* i225 only supports "global" frame preemption
6100 		 * settings.
6101 		 */
6102 		if (e->command != TC_TAPRIO_CMD_SET_GATES)
6103 			return false;
6104 
6105 		for (i = 0; i < adapter->num_tx_queues; i++)
6106 			if (e->gate_mask & BIT(i)) {
6107 				queue_uses[i]++;
6108 
6109 				/* There are limitations: A single queue cannot
6110 				 * be opened and closed multiple times per cycle
6111 				 * unless the gate stays open. Check for it.
6112 				 */
6113 				if (queue_uses[i] > 1 &&
6114 				    !(prev->gate_mask & BIT(i)))
6115 					return false;
6116 			}
6117 	}
6118 
6119 	return true;
6120 }
6121 
igc_tsn_enable_launchtime(struct igc_adapter * adapter,struct tc_etf_qopt_offload * qopt)6122 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
6123 				     struct tc_etf_qopt_offload *qopt)
6124 {
6125 	struct igc_hw *hw = &adapter->hw;
6126 	int err;
6127 
6128 	if (hw->mac.type != igc_i225)
6129 		return -EOPNOTSUPP;
6130 
6131 	err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
6132 	if (err)
6133 		return err;
6134 
6135 	return igc_tsn_offload_apply(adapter);
6136 }
6137 
igc_qbv_clear_schedule(struct igc_adapter * adapter)6138 static int igc_qbv_clear_schedule(struct igc_adapter *adapter)
6139 {
6140 	unsigned long flags;
6141 	int i;
6142 
6143 	adapter->base_time = 0;
6144 	adapter->cycle_time = NSEC_PER_SEC;
6145 	adapter->taprio_offload_enable = false;
6146 	adapter->qbv_config_change_errors = 0;
6147 	adapter->qbv_count = 0;
6148 
6149 	for (i = 0; i < adapter->num_tx_queues; i++) {
6150 		struct igc_ring *ring = adapter->tx_ring[i];
6151 
6152 		ring->start_time = 0;
6153 		ring->end_time = NSEC_PER_SEC;
6154 		ring->max_sdu = 0;
6155 	}
6156 
6157 	spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
6158 
6159 	adapter->qbv_transition = false;
6160 
6161 	for (i = 0; i < adapter->num_tx_queues; i++) {
6162 		struct igc_ring *ring = adapter->tx_ring[i];
6163 
6164 		ring->oper_gate_closed = false;
6165 		ring->admin_gate_closed = false;
6166 	}
6167 
6168 	spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
6169 
6170 	return 0;
6171 }
6172 
igc_tsn_clear_schedule(struct igc_adapter * adapter)6173 static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
6174 {
6175 	igc_qbv_clear_schedule(adapter);
6176 
6177 	return 0;
6178 }
6179 
igc_taprio_stats(struct net_device * dev,struct tc_taprio_qopt_stats * stats)6180 static void igc_taprio_stats(struct net_device *dev,
6181 			     struct tc_taprio_qopt_stats *stats)
6182 {
6183 	/* When Strict_End is enabled, the tx_overruns counter
6184 	 * will always be zero.
6185 	 */
6186 	stats->tx_overruns = 0;
6187 }
6188 
igc_taprio_queue_stats(struct net_device * dev,struct tc_taprio_qopt_queue_stats * queue_stats)6189 static void igc_taprio_queue_stats(struct net_device *dev,
6190 				   struct tc_taprio_qopt_queue_stats *queue_stats)
6191 {
6192 	struct tc_taprio_qopt_stats *stats = &queue_stats->stats;
6193 
6194 	/* When Strict_End is enabled, the tx_overruns counter
6195 	 * will always be zero.
6196 	 */
6197 	stats->tx_overruns = 0;
6198 }
6199 
igc_save_qbv_schedule(struct igc_adapter * adapter,struct tc_taprio_qopt_offload * qopt)6200 static int igc_save_qbv_schedule(struct igc_adapter *adapter,
6201 				 struct tc_taprio_qopt_offload *qopt)
6202 {
6203 	bool queue_configured[IGC_MAX_TX_QUEUES] = { };
6204 	struct igc_hw *hw = &adapter->hw;
6205 	u32 start_time = 0, end_time = 0;
6206 	struct timespec64 now;
6207 	unsigned long flags;
6208 	size_t n;
6209 	int i;
6210 
6211 	if (qopt->base_time < 0)
6212 		return -ERANGE;
6213 
6214 	if (igc_is_device_id_i225(hw) && adapter->taprio_offload_enable)
6215 		return -EALREADY;
6216 
6217 	if (!validate_schedule(adapter, qopt))
6218 		return -EINVAL;
6219 
6220 	igc_ptp_read(adapter, &now);
6221 
6222 	if (igc_tsn_is_taprio_activated_by_user(adapter) &&
6223 	    is_base_time_past(qopt->base_time, &now))
6224 		adapter->qbv_config_change_errors++;
6225 
6226 	adapter->cycle_time = qopt->cycle_time;
6227 	adapter->base_time = qopt->base_time;
6228 	adapter->taprio_offload_enable = true;
6229 
6230 	for (n = 0; n < qopt->num_entries; n++) {
6231 		struct tc_taprio_sched_entry *e = &qopt->entries[n];
6232 
6233 		end_time += e->interval;
6234 
6235 		/* If any of the conditions below are true, we need to manually
6236 		 * control the end time of the cycle.
6237 		 * 1. Qbv users can specify a cycle time that is not equal
6238 		 * to the total GCL intervals. Hence, recalculation is
6239 		 * necessary here to exclude the time interval that
6240 		 * exceeds the cycle time.
6241 		 * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
6242 		 * once the end of the list is reached, it will switch
6243 		 * to the END_OF_CYCLE state and leave the gates in the
6244 		 * same state until the next cycle is started.
6245 		 */
6246 		if (end_time > adapter->cycle_time ||
6247 		    n + 1 == qopt->num_entries)
6248 			end_time = adapter->cycle_time;
6249 
6250 		for (i = 0; i < adapter->num_tx_queues; i++) {
6251 			struct igc_ring *ring = adapter->tx_ring[i];
6252 
6253 			if (!(e->gate_mask & BIT(i)))
6254 				continue;
6255 
6256 			/* Check whether a queue stays open for more than one
6257 			 * entry. If so, keep the start and advance the end
6258 			 * time.
6259 			 */
6260 			if (!queue_configured[i])
6261 				ring->start_time = start_time;
6262 			ring->end_time = end_time;
6263 
6264 			if (ring->start_time >= adapter->cycle_time)
6265 				queue_configured[i] = false;
6266 			else
6267 				queue_configured[i] = true;
6268 		}
6269 
6270 		start_time += e->interval;
6271 	}
6272 
6273 	spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
6274 
6275 	/* Check whether a queue gets configured.
6276 	 * If not, set the start and end time to be end time.
6277 	 */
6278 	for (i = 0; i < adapter->num_tx_queues; i++) {
6279 		struct igc_ring *ring = adapter->tx_ring[i];
6280 
6281 		if (!is_base_time_past(qopt->base_time, &now)) {
6282 			ring->admin_gate_closed = false;
6283 		} else {
6284 			ring->oper_gate_closed = false;
6285 			ring->admin_gate_closed = false;
6286 		}
6287 
6288 		if (!queue_configured[i]) {
6289 			if (!is_base_time_past(qopt->base_time, &now))
6290 				ring->admin_gate_closed = true;
6291 			else
6292 				ring->oper_gate_closed = true;
6293 
6294 			ring->start_time = end_time;
6295 			ring->end_time = end_time;
6296 		}
6297 	}
6298 
6299 	spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
6300 
6301 	for (i = 0; i < adapter->num_tx_queues; i++) {
6302 		struct igc_ring *ring = adapter->tx_ring[i];
6303 		struct net_device *dev = adapter->netdev;
6304 
6305 		if (qopt->max_sdu[i])
6306 			ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len - ETH_TLEN;
6307 		else
6308 			ring->max_sdu = 0;
6309 	}
6310 
6311 	return 0;
6312 }
6313 
igc_tsn_enable_qbv_scheduling(struct igc_adapter * adapter,struct tc_taprio_qopt_offload * qopt)6314 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
6315 					 struct tc_taprio_qopt_offload *qopt)
6316 {
6317 	struct igc_hw *hw = &adapter->hw;
6318 	int err;
6319 
6320 	if (hw->mac.type != igc_i225)
6321 		return -EOPNOTSUPP;
6322 
6323 	switch (qopt->cmd) {
6324 	case TAPRIO_CMD_REPLACE:
6325 		err = igc_save_qbv_schedule(adapter, qopt);
6326 		break;
6327 	case TAPRIO_CMD_DESTROY:
6328 		err = igc_tsn_clear_schedule(adapter);
6329 		break;
6330 	case TAPRIO_CMD_STATS:
6331 		igc_taprio_stats(adapter->netdev, &qopt->stats);
6332 		return 0;
6333 	case TAPRIO_CMD_QUEUE_STATS:
6334 		igc_taprio_queue_stats(adapter->netdev, &qopt->queue_stats);
6335 		return 0;
6336 	default:
6337 		return -EOPNOTSUPP;
6338 	}
6339 
6340 	if (err)
6341 		return err;
6342 
6343 	return igc_tsn_offload_apply(adapter);
6344 }
6345 
igc_save_cbs_params(struct igc_adapter * adapter,int queue,bool enable,int idleslope,int sendslope,int hicredit,int locredit)6346 static int igc_save_cbs_params(struct igc_adapter *adapter, int queue,
6347 			       bool enable, int idleslope, int sendslope,
6348 			       int hicredit, int locredit)
6349 {
6350 	bool cbs_status[IGC_MAX_SR_QUEUES] = { false };
6351 	struct net_device *netdev = adapter->netdev;
6352 	struct igc_ring *ring;
6353 	int i;
6354 
6355 	/* i225 has two sets of credit-based shaper logic.
6356 	 * Supporting it only on the top two priority queues
6357 	 */
6358 	if (queue < 0 || queue > 1)
6359 		return -EINVAL;
6360 
6361 	ring = adapter->tx_ring[queue];
6362 
6363 	for (i = 0; i < IGC_MAX_SR_QUEUES; i++)
6364 		if (adapter->tx_ring[i])
6365 			cbs_status[i] = adapter->tx_ring[i]->cbs_enable;
6366 
6367 	/* CBS should be enabled on the highest priority queue first in order
6368 	 * for the CBS algorithm to operate as intended.
6369 	 */
6370 	if (enable) {
6371 		if (queue == 1 && !cbs_status[0]) {
6372 			netdev_err(netdev,
6373 				   "Enabling CBS on queue1 before queue0\n");
6374 			return -EINVAL;
6375 		}
6376 	} else {
6377 		if (queue == 0 && cbs_status[1]) {
6378 			netdev_err(netdev,
6379 				   "Disabling CBS on queue0 before queue1\n");
6380 			return -EINVAL;
6381 		}
6382 	}
6383 
6384 	ring->cbs_enable = enable;
6385 	ring->idleslope = idleslope;
6386 	ring->sendslope = sendslope;
6387 	ring->hicredit = hicredit;
6388 	ring->locredit = locredit;
6389 
6390 	return 0;
6391 }
6392 
igc_tsn_enable_cbs(struct igc_adapter * adapter,struct tc_cbs_qopt_offload * qopt)6393 static int igc_tsn_enable_cbs(struct igc_adapter *adapter,
6394 			      struct tc_cbs_qopt_offload *qopt)
6395 {
6396 	struct igc_hw *hw = &adapter->hw;
6397 	int err;
6398 
6399 	if (hw->mac.type != igc_i225)
6400 		return -EOPNOTSUPP;
6401 
6402 	if (qopt->queue < 0 || qopt->queue > 1)
6403 		return -EINVAL;
6404 
6405 	err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable,
6406 				  qopt->idleslope, qopt->sendslope,
6407 				  qopt->hicredit, qopt->locredit);
6408 	if (err)
6409 		return err;
6410 
6411 	return igc_tsn_offload_apply(adapter);
6412 }
6413 
igc_tc_query_caps(struct igc_adapter * adapter,struct tc_query_caps_base * base)6414 static int igc_tc_query_caps(struct igc_adapter *adapter,
6415 			     struct tc_query_caps_base *base)
6416 {
6417 	struct igc_hw *hw = &adapter->hw;
6418 
6419 	switch (base->type) {
6420 	case TC_SETUP_QDISC_TAPRIO: {
6421 		struct tc_taprio_caps *caps = base->caps;
6422 
6423 		caps->broken_mqprio = true;
6424 
6425 		if (hw->mac.type == igc_i225) {
6426 			caps->supports_queue_max_sdu = true;
6427 			caps->gate_mask_per_txq = true;
6428 		}
6429 
6430 		return 0;
6431 	}
6432 	default:
6433 		return -EOPNOTSUPP;
6434 	}
6435 }
6436 
igc_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)6437 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
6438 			void *type_data)
6439 {
6440 	struct igc_adapter *adapter = netdev_priv(dev);
6441 
6442 	adapter->tc_setup_type = type;
6443 
6444 	switch (type) {
6445 	case TC_QUERY_CAPS:
6446 		return igc_tc_query_caps(adapter, type_data);
6447 	case TC_SETUP_QDISC_TAPRIO:
6448 		return igc_tsn_enable_qbv_scheduling(adapter, type_data);
6449 
6450 	case TC_SETUP_QDISC_ETF:
6451 		return igc_tsn_enable_launchtime(adapter, type_data);
6452 
6453 	case TC_SETUP_QDISC_CBS:
6454 		return igc_tsn_enable_cbs(adapter, type_data);
6455 
6456 	default:
6457 		return -EOPNOTSUPP;
6458 	}
6459 }
6460 
igc_bpf(struct net_device * dev,struct netdev_bpf * bpf)6461 static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6462 {
6463 	struct igc_adapter *adapter = netdev_priv(dev);
6464 
6465 	switch (bpf->command) {
6466 	case XDP_SETUP_PROG:
6467 		return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack);
6468 	case XDP_SETUP_XSK_POOL:
6469 		return igc_xdp_setup_pool(adapter, bpf->xsk.pool,
6470 					  bpf->xsk.queue_id);
6471 	default:
6472 		return -EOPNOTSUPP;
6473 	}
6474 }
6475 
igc_xdp_xmit(struct net_device * dev,int num_frames,struct xdp_frame ** frames,u32 flags)6476 static int igc_xdp_xmit(struct net_device *dev, int num_frames,
6477 			struct xdp_frame **frames, u32 flags)
6478 {
6479 	struct igc_adapter *adapter = netdev_priv(dev);
6480 	int cpu = smp_processor_id();
6481 	struct netdev_queue *nq;
6482 	struct igc_ring *ring;
6483 	int i, nxmit;
6484 
6485 	if (unlikely(!netif_carrier_ok(dev)))
6486 		return -ENETDOWN;
6487 
6488 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6489 		return -EINVAL;
6490 
6491 	ring = igc_xdp_get_tx_ring(adapter, cpu);
6492 	nq = txring_txq(ring);
6493 
6494 	__netif_tx_lock(nq, cpu);
6495 
6496 	/* Avoid transmit queue timeout since we share it with the slow path */
6497 	txq_trans_cond_update(nq);
6498 
6499 	nxmit = 0;
6500 	for (i = 0; i < num_frames; i++) {
6501 		int err;
6502 		struct xdp_frame *xdpf = frames[i];
6503 
6504 		err = igc_xdp_init_tx_descriptor(ring, xdpf);
6505 		if (err)
6506 			break;
6507 		nxmit++;
6508 	}
6509 
6510 	if (flags & XDP_XMIT_FLUSH)
6511 		igc_flush_tx_descriptors(ring);
6512 
6513 	__netif_tx_unlock(nq);
6514 
6515 	return nxmit;
6516 }
6517 
igc_trigger_rxtxq_interrupt(struct igc_adapter * adapter,struct igc_q_vector * q_vector)6518 static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter,
6519 					struct igc_q_vector *q_vector)
6520 {
6521 	struct igc_hw *hw = &adapter->hw;
6522 	u32 eics = 0;
6523 
6524 	eics |= q_vector->eims_value;
6525 	wr32(IGC_EICS, eics);
6526 }
6527 
igc_xsk_wakeup(struct net_device * dev,u32 queue_id,u32 flags)6528 int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
6529 {
6530 	struct igc_adapter *adapter = netdev_priv(dev);
6531 	struct igc_q_vector *q_vector;
6532 	struct igc_ring *ring;
6533 
6534 	if (test_bit(__IGC_DOWN, &adapter->state))
6535 		return -ENETDOWN;
6536 
6537 	if (!igc_xdp_is_enabled(adapter))
6538 		return -ENXIO;
6539 
6540 	if (queue_id >= adapter->num_rx_queues)
6541 		return -EINVAL;
6542 
6543 	ring = adapter->rx_ring[queue_id];
6544 
6545 	if (!ring->xsk_pool)
6546 		return -ENXIO;
6547 
6548 	q_vector = adapter->q_vector[queue_id];
6549 	if (!napi_if_scheduled_mark_missed(&q_vector->napi))
6550 		igc_trigger_rxtxq_interrupt(adapter, q_vector);
6551 
6552 	return 0;
6553 }
6554 
6555 static const struct net_device_ops igc_netdev_ops = {
6556 	.ndo_open		= igc_open,
6557 	.ndo_stop		= igc_close,
6558 	.ndo_start_xmit		= igc_xmit_frame,
6559 	.ndo_set_rx_mode	= igc_set_rx_mode,
6560 	.ndo_set_mac_address	= igc_set_mac,
6561 	.ndo_change_mtu		= igc_change_mtu,
6562 	.ndo_tx_timeout		= igc_tx_timeout,
6563 	.ndo_get_stats64	= igc_get_stats64,
6564 	.ndo_fix_features	= igc_fix_features,
6565 	.ndo_set_features	= igc_set_features,
6566 	.ndo_features_check	= igc_features_check,
6567 	.ndo_eth_ioctl		= igc_ioctl,
6568 	.ndo_setup_tc		= igc_setup_tc,
6569 	.ndo_bpf		= igc_bpf,
6570 	.ndo_xdp_xmit		= igc_xdp_xmit,
6571 	.ndo_xsk_wakeup		= igc_xsk_wakeup,
6572 };
6573 
6574 /* PCIe configuration access */
igc_read_pci_cfg(struct igc_hw * hw,u32 reg,u16 * value)6575 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6576 {
6577 	struct igc_adapter *adapter = hw->back;
6578 
6579 	pci_read_config_word(adapter->pdev, reg, value);
6580 }
6581 
igc_write_pci_cfg(struct igc_hw * hw,u32 reg,u16 * value)6582 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6583 {
6584 	struct igc_adapter *adapter = hw->back;
6585 
6586 	pci_write_config_word(adapter->pdev, reg, *value);
6587 }
6588 
igc_read_pcie_cap_reg(struct igc_hw * hw,u32 reg,u16 * value)6589 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6590 {
6591 	struct igc_adapter *adapter = hw->back;
6592 
6593 	if (!pci_is_pcie(adapter->pdev))
6594 		return -IGC_ERR_CONFIG;
6595 
6596 	pcie_capability_read_word(adapter->pdev, reg, value);
6597 
6598 	return IGC_SUCCESS;
6599 }
6600 
igc_write_pcie_cap_reg(struct igc_hw * hw,u32 reg,u16 * value)6601 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6602 {
6603 	struct igc_adapter *adapter = hw->back;
6604 
6605 	if (!pci_is_pcie(adapter->pdev))
6606 		return -IGC_ERR_CONFIG;
6607 
6608 	pcie_capability_write_word(adapter->pdev, reg, *value);
6609 
6610 	return IGC_SUCCESS;
6611 }
6612 
igc_rd32(struct igc_hw * hw,u32 reg)6613 u32 igc_rd32(struct igc_hw *hw, u32 reg)
6614 {
6615 	struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
6616 	u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
6617 	u32 value = 0;
6618 
6619 	if (IGC_REMOVED(hw_addr))
6620 		return ~value;
6621 
6622 	value = readl(&hw_addr[reg]);
6623 
6624 	/* reads should not return all F's */
6625 	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
6626 		struct net_device *netdev = igc->netdev;
6627 
6628 		hw->hw_addr = NULL;
6629 		netif_device_detach(netdev);
6630 		netdev_err(netdev, "PCIe link lost, device now detached\n");
6631 		WARN(pci_device_is_present(igc->pdev),
6632 		     "igc: Failed to read reg 0x%x!\n", reg);
6633 	}
6634 
6635 	return value;
6636 }
6637 
6638 /* Mapping HW RSS Type to enum xdp_rss_hash_type */
6639 static enum xdp_rss_hash_type igc_xdp_rss_type[IGC_RSS_TYPE_MAX_TABLE] = {
6640 	[IGC_RSS_TYPE_NO_HASH]		= XDP_RSS_TYPE_L2,
6641 	[IGC_RSS_TYPE_HASH_TCP_IPV4]	= XDP_RSS_TYPE_L4_IPV4_TCP,
6642 	[IGC_RSS_TYPE_HASH_IPV4]	= XDP_RSS_TYPE_L3_IPV4,
6643 	[IGC_RSS_TYPE_HASH_TCP_IPV6]	= XDP_RSS_TYPE_L4_IPV6_TCP,
6644 	[IGC_RSS_TYPE_HASH_IPV6_EX]	= XDP_RSS_TYPE_L3_IPV6_EX,
6645 	[IGC_RSS_TYPE_HASH_IPV6]	= XDP_RSS_TYPE_L3_IPV6,
6646 	[IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX,
6647 	[IGC_RSS_TYPE_HASH_UDP_IPV4]	= XDP_RSS_TYPE_L4_IPV4_UDP,
6648 	[IGC_RSS_TYPE_HASH_UDP_IPV6]	= XDP_RSS_TYPE_L4_IPV6_UDP,
6649 	[IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX,
6650 	[10] = XDP_RSS_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW  */
6651 	[11] = XDP_RSS_TYPE_NONE, /* keep array sized for SW bit-mask   */
6652 	[12] = XDP_RSS_TYPE_NONE, /* to handle future HW revisons       */
6653 	[13] = XDP_RSS_TYPE_NONE,
6654 	[14] = XDP_RSS_TYPE_NONE,
6655 	[15] = XDP_RSS_TYPE_NONE,
6656 };
6657 
igc_xdp_rx_hash(const struct xdp_md * _ctx,u32 * hash,enum xdp_rss_hash_type * rss_type)6658 static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
6659 			   enum xdp_rss_hash_type *rss_type)
6660 {
6661 	const struct igc_xdp_buff *ctx = (void *)_ctx;
6662 
6663 	if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH))
6664 		return -ENODATA;
6665 
6666 	*hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss);
6667 	*rss_type = igc_xdp_rss_type[igc_rss_type(ctx->rx_desc)];
6668 
6669 	return 0;
6670 }
6671 
igc_xdp_rx_timestamp(const struct xdp_md * _ctx,u64 * timestamp)6672 static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
6673 {
6674 	const struct igc_xdp_buff *ctx = (void *)_ctx;
6675 
6676 	if (igc_test_staterr(ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) {
6677 		*timestamp = ctx->rx_ts;
6678 
6679 		return 0;
6680 	}
6681 
6682 	return -ENODATA;
6683 }
6684 
6685 static const struct xdp_metadata_ops igc_xdp_metadata_ops = {
6686 	.xmo_rx_hash			= igc_xdp_rx_hash,
6687 	.xmo_rx_timestamp		= igc_xdp_rx_timestamp,
6688 };
6689 
igc_qbv_scheduling_timer(struct hrtimer * timer)6690 static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer)
6691 {
6692 	struct igc_adapter *adapter = container_of(timer, struct igc_adapter,
6693 						   hrtimer);
6694 	unsigned long flags;
6695 	unsigned int i;
6696 
6697 	spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
6698 
6699 	adapter->qbv_transition = true;
6700 	for (i = 0; i < adapter->num_tx_queues; i++) {
6701 		struct igc_ring *tx_ring = adapter->tx_ring[i];
6702 
6703 		if (tx_ring->admin_gate_closed) {
6704 			tx_ring->admin_gate_closed = false;
6705 			tx_ring->oper_gate_closed = true;
6706 		} else {
6707 			tx_ring->oper_gate_closed = false;
6708 		}
6709 	}
6710 	adapter->qbv_transition = false;
6711 
6712 	spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
6713 
6714 	return HRTIMER_NORESTART;
6715 }
6716 
6717 /**
6718  * igc_probe - Device Initialization Routine
6719  * @pdev: PCI device information struct
6720  * @ent: entry in igc_pci_tbl
6721  *
6722  * Returns 0 on success, negative on failure
6723  *
6724  * igc_probe initializes an adapter identified by a pci_dev structure.
6725  * The OS initialization, configuring the adapter private structure,
6726  * and a hardware reset occur.
6727  */
igc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)6728 static int igc_probe(struct pci_dev *pdev,
6729 		     const struct pci_device_id *ent)
6730 {
6731 	struct igc_adapter *adapter;
6732 	struct net_device *netdev;
6733 	struct igc_hw *hw;
6734 	const struct igc_info *ei = igc_info_tbl[ent->driver_data];
6735 	int err;
6736 
6737 	err = pci_enable_device_mem(pdev);
6738 	if (err)
6739 		return err;
6740 
6741 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
6742 	if (err) {
6743 		dev_err(&pdev->dev,
6744 			"No usable DMA configuration, aborting\n");
6745 		goto err_dma;
6746 	}
6747 
6748 	err = pci_request_mem_regions(pdev, igc_driver_name);
6749 	if (err)
6750 		goto err_pci_reg;
6751 
6752 	err = pci_enable_ptm(pdev, NULL);
6753 	if (err < 0)
6754 		dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
6755 
6756 	pci_set_master(pdev);
6757 
6758 	err = -ENOMEM;
6759 	netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
6760 				   IGC_MAX_TX_QUEUES);
6761 
6762 	if (!netdev)
6763 		goto err_alloc_etherdev;
6764 
6765 	SET_NETDEV_DEV(netdev, &pdev->dev);
6766 
6767 	pci_set_drvdata(pdev, netdev);
6768 	adapter = netdev_priv(netdev);
6769 	adapter->netdev = netdev;
6770 	adapter->pdev = pdev;
6771 	hw = &adapter->hw;
6772 	hw->back = adapter;
6773 	adapter->port_num = hw->bus.func;
6774 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6775 
6776 	err = pci_save_state(pdev);
6777 	if (err)
6778 		goto err_ioremap;
6779 
6780 	err = -EIO;
6781 	adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
6782 				   pci_resource_len(pdev, 0));
6783 	if (!adapter->io_addr)
6784 		goto err_ioremap;
6785 
6786 	/* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
6787 	hw->hw_addr = adapter->io_addr;
6788 
6789 	netdev->netdev_ops = &igc_netdev_ops;
6790 	netdev->xdp_metadata_ops = &igc_xdp_metadata_ops;
6791 	igc_ethtool_set_ops(netdev);
6792 	netdev->watchdog_timeo = 5 * HZ;
6793 
6794 	netdev->mem_start = pci_resource_start(pdev, 0);
6795 	netdev->mem_end = pci_resource_end(pdev, 0);
6796 
6797 	/* PCI config space info */
6798 	hw->vendor_id = pdev->vendor;
6799 	hw->device_id = pdev->device;
6800 	hw->revision_id = pdev->revision;
6801 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
6802 	hw->subsystem_device_id = pdev->subsystem_device;
6803 
6804 	/* Copy the default MAC and PHY function pointers */
6805 	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6806 	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6807 
6808 	/* Initialize skew-specific constants */
6809 	err = ei->get_invariants(hw);
6810 	if (err)
6811 		goto err_sw_init;
6812 
6813 	/* Add supported features to the features list*/
6814 	netdev->features |= NETIF_F_SG;
6815 	netdev->features |= NETIF_F_TSO;
6816 	netdev->features |= NETIF_F_TSO6;
6817 	netdev->features |= NETIF_F_TSO_ECN;
6818 	netdev->features |= NETIF_F_RXHASH;
6819 	netdev->features |= NETIF_F_RXCSUM;
6820 	netdev->features |= NETIF_F_HW_CSUM;
6821 	netdev->features |= NETIF_F_SCTP_CRC;
6822 	netdev->features |= NETIF_F_HW_TC;
6823 
6824 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
6825 				  NETIF_F_GSO_GRE_CSUM | \
6826 				  NETIF_F_GSO_IPXIP4 | \
6827 				  NETIF_F_GSO_IPXIP6 | \
6828 				  NETIF_F_GSO_UDP_TUNNEL | \
6829 				  NETIF_F_GSO_UDP_TUNNEL_CSUM)
6830 
6831 	netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
6832 	netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
6833 
6834 	/* setup the private structure */
6835 	err = igc_sw_init(adapter);
6836 	if (err)
6837 		goto err_sw_init;
6838 
6839 	/* copy netdev features into list of user selectable features */
6840 	netdev->hw_features |= NETIF_F_NTUPLE;
6841 	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
6842 	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
6843 	netdev->hw_features |= netdev->features;
6844 
6845 	netdev->features |= NETIF_F_HIGHDMA;
6846 
6847 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
6848 	netdev->mpls_features |= NETIF_F_HW_CSUM;
6849 	netdev->hw_enc_features |= netdev->vlan_features;
6850 
6851 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
6852 			       NETDEV_XDP_ACT_XSK_ZEROCOPY;
6853 
6854 	/* MTU range: 68 - 9216 */
6855 	netdev->min_mtu = ETH_MIN_MTU;
6856 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
6857 
6858 	/* before reading the NVM, reset the controller to put the device in a
6859 	 * known good starting state
6860 	 */
6861 	hw->mac.ops.reset_hw(hw);
6862 
6863 	if (igc_get_flash_presence_i225(hw)) {
6864 		if (hw->nvm.ops.validate(hw) < 0) {
6865 			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
6866 			err = -EIO;
6867 			goto err_eeprom;
6868 		}
6869 	}
6870 
6871 	if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
6872 		/* copy the MAC address out of the NVM */
6873 		if (hw->mac.ops.read_mac_addr(hw))
6874 			dev_err(&pdev->dev, "NVM Read Error\n");
6875 	}
6876 
6877 	eth_hw_addr_set(netdev, hw->mac.addr);
6878 
6879 	if (!is_valid_ether_addr(netdev->dev_addr)) {
6880 		dev_err(&pdev->dev, "Invalid MAC Address\n");
6881 		err = -EIO;
6882 		goto err_eeprom;
6883 	}
6884 
6885 	/* configure RXPBSIZE and TXPBSIZE */
6886 	wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
6887 	wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
6888 
6889 	timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
6890 	timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
6891 
6892 	INIT_WORK(&adapter->reset_task, igc_reset_task);
6893 	INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
6894 
6895 	hrtimer_init(&adapter->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6896 	adapter->hrtimer.function = &igc_qbv_scheduling_timer;
6897 
6898 	/* Initialize link properties that are user-changeable */
6899 	adapter->fc_autoneg = true;
6900 	hw->mac.autoneg = true;
6901 	hw->phy.autoneg_advertised = 0xaf;
6902 
6903 	hw->fc.requested_mode = igc_fc_default;
6904 	hw->fc.current_mode = igc_fc_default;
6905 
6906 	/* By default, support wake on port A */
6907 	adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
6908 
6909 	/* initialize the wol settings based on the eeprom settings */
6910 	if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
6911 		adapter->wol |= IGC_WUFC_MAG;
6912 
6913 	device_set_wakeup_enable(&adapter->pdev->dev,
6914 				 adapter->flags & IGC_FLAG_WOL_SUPPORTED);
6915 
6916 	igc_ptp_init(adapter);
6917 
6918 	igc_tsn_clear_schedule(adapter);
6919 
6920 	/* reset the hardware with the new settings */
6921 	igc_reset(adapter);
6922 
6923 	/* let the f/w know that the h/w is now under the control of the
6924 	 * driver.
6925 	 */
6926 	igc_get_hw_control(adapter);
6927 
6928 	strncpy(netdev->name, "eth%d", IFNAMSIZ);
6929 	err = register_netdev(netdev);
6930 	if (err)
6931 		goto err_register;
6932 
6933 	 /* carrier off reporting is important to ethtool even BEFORE open */
6934 	netif_carrier_off(netdev);
6935 
6936 	/* Check if Media Autosense is enabled */
6937 	adapter->ei = *ei;
6938 
6939 	/* print pcie link status and MAC address */
6940 	pcie_print_link_status(pdev);
6941 	netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
6942 
6943 	dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
6944 	/* Disable EEE for internal PHY devices */
6945 	hw->dev_spec._base.eee_enable = false;
6946 	adapter->flags &= ~IGC_FLAG_EEE;
6947 	igc_set_eee_i225(hw, false, false, false);
6948 
6949 	pm_runtime_put_noidle(&pdev->dev);
6950 
6951 	return 0;
6952 
6953 err_register:
6954 	igc_release_hw_control(adapter);
6955 err_eeprom:
6956 	if (!igc_check_reset_block(hw))
6957 		igc_reset_phy(hw);
6958 err_sw_init:
6959 	igc_clear_interrupt_scheme(adapter);
6960 	iounmap(adapter->io_addr);
6961 err_ioremap:
6962 	free_netdev(netdev);
6963 err_alloc_etherdev:
6964 	pci_release_mem_regions(pdev);
6965 err_pci_reg:
6966 err_dma:
6967 	pci_disable_device(pdev);
6968 	return err;
6969 }
6970 
6971 /**
6972  * igc_remove - Device Removal Routine
6973  * @pdev: PCI device information struct
6974  *
6975  * igc_remove is called by the PCI subsystem to alert the driver
6976  * that it should release a PCI device.  This could be caused by a
6977  * Hot-Plug event, or because the driver is going to be removed from
6978  * memory.
6979  */
igc_remove(struct pci_dev * pdev)6980 static void igc_remove(struct pci_dev *pdev)
6981 {
6982 	struct net_device *netdev = pci_get_drvdata(pdev);
6983 	struct igc_adapter *adapter = netdev_priv(netdev);
6984 
6985 	pm_runtime_get_noresume(&pdev->dev);
6986 
6987 	igc_flush_nfc_rules(adapter);
6988 
6989 	igc_ptp_stop(adapter);
6990 
6991 	pci_disable_ptm(pdev);
6992 	pci_clear_master(pdev);
6993 
6994 	set_bit(__IGC_DOWN, &adapter->state);
6995 
6996 	del_timer_sync(&adapter->watchdog_timer);
6997 	del_timer_sync(&adapter->phy_info_timer);
6998 
6999 	cancel_work_sync(&adapter->reset_task);
7000 	cancel_work_sync(&adapter->watchdog_task);
7001 	hrtimer_cancel(&adapter->hrtimer);
7002 
7003 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
7004 	 * would have already happened in close and is redundant.
7005 	 */
7006 	igc_release_hw_control(adapter);
7007 	unregister_netdev(netdev);
7008 
7009 	igc_clear_interrupt_scheme(adapter);
7010 	pci_iounmap(pdev, adapter->io_addr);
7011 	pci_release_mem_regions(pdev);
7012 
7013 	free_netdev(netdev);
7014 
7015 	pci_disable_device(pdev);
7016 }
7017 
__igc_shutdown(struct pci_dev * pdev,bool * enable_wake,bool runtime)7018 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
7019 			  bool runtime)
7020 {
7021 	struct net_device *netdev = pci_get_drvdata(pdev);
7022 	struct igc_adapter *adapter = netdev_priv(netdev);
7023 	u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
7024 	struct igc_hw *hw = &adapter->hw;
7025 	u32 ctrl, rctl, status;
7026 	bool wake;
7027 
7028 	rtnl_lock();
7029 	netif_device_detach(netdev);
7030 
7031 	if (netif_running(netdev))
7032 		__igc_close(netdev, true);
7033 
7034 	igc_ptp_suspend(adapter);
7035 
7036 	igc_clear_interrupt_scheme(adapter);
7037 	rtnl_unlock();
7038 
7039 	status = rd32(IGC_STATUS);
7040 	if (status & IGC_STATUS_LU)
7041 		wufc &= ~IGC_WUFC_LNKC;
7042 
7043 	if (wufc) {
7044 		igc_setup_rctl(adapter);
7045 		igc_set_rx_mode(netdev);
7046 
7047 		/* turn on all-multi mode if wake on multicast is enabled */
7048 		if (wufc & IGC_WUFC_MC) {
7049 			rctl = rd32(IGC_RCTL);
7050 			rctl |= IGC_RCTL_MPE;
7051 			wr32(IGC_RCTL, rctl);
7052 		}
7053 
7054 		ctrl = rd32(IGC_CTRL);
7055 		ctrl |= IGC_CTRL_ADVD3WUC;
7056 		wr32(IGC_CTRL, ctrl);
7057 
7058 		/* Allow time for pending master requests to run */
7059 		igc_disable_pcie_master(hw);
7060 
7061 		wr32(IGC_WUC, IGC_WUC_PME_EN);
7062 		wr32(IGC_WUFC, wufc);
7063 	} else {
7064 		wr32(IGC_WUC, 0);
7065 		wr32(IGC_WUFC, 0);
7066 	}
7067 
7068 	wake = wufc || adapter->en_mng_pt;
7069 	if (!wake)
7070 		igc_power_down_phy_copper_base(&adapter->hw);
7071 	else
7072 		igc_power_up_link(adapter);
7073 
7074 	if (enable_wake)
7075 		*enable_wake = wake;
7076 
7077 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
7078 	 * would have already happened in close and is redundant.
7079 	 */
7080 	igc_release_hw_control(adapter);
7081 
7082 	pci_disable_device(pdev);
7083 
7084 	return 0;
7085 }
7086 
7087 #ifdef CONFIG_PM
igc_runtime_suspend(struct device * dev)7088 static int __maybe_unused igc_runtime_suspend(struct device *dev)
7089 {
7090 	return __igc_shutdown(to_pci_dev(dev), NULL, 1);
7091 }
7092 
igc_deliver_wake_packet(struct net_device * netdev)7093 static void igc_deliver_wake_packet(struct net_device *netdev)
7094 {
7095 	struct igc_adapter *adapter = netdev_priv(netdev);
7096 	struct igc_hw *hw = &adapter->hw;
7097 	struct sk_buff *skb;
7098 	u32 wupl;
7099 
7100 	wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
7101 
7102 	/* WUPM stores only the first 128 bytes of the wake packet.
7103 	 * Read the packet only if we have the whole thing.
7104 	 */
7105 	if (wupl == 0 || wupl > IGC_WUPM_BYTES)
7106 		return;
7107 
7108 	skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
7109 	if (!skb)
7110 		return;
7111 
7112 	skb_put(skb, wupl);
7113 
7114 	/* Ensure reads are 32-bit aligned */
7115 	wupl = roundup(wupl, 4);
7116 
7117 	memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
7118 
7119 	skb->protocol = eth_type_trans(skb, netdev);
7120 	netif_rx(skb);
7121 }
7122 
igc_resume(struct device * dev)7123 static int __maybe_unused igc_resume(struct device *dev)
7124 {
7125 	struct pci_dev *pdev = to_pci_dev(dev);
7126 	struct net_device *netdev = pci_get_drvdata(pdev);
7127 	struct igc_adapter *adapter = netdev_priv(netdev);
7128 	struct igc_hw *hw = &adapter->hw;
7129 	u32 err, val;
7130 
7131 	pci_set_power_state(pdev, PCI_D0);
7132 	pci_restore_state(pdev);
7133 	pci_save_state(pdev);
7134 
7135 	if (!pci_device_is_present(pdev))
7136 		return -ENODEV;
7137 	err = pci_enable_device_mem(pdev);
7138 	if (err) {
7139 		netdev_err(netdev, "Cannot enable PCI device from suspend\n");
7140 		return err;
7141 	}
7142 	pci_set_master(pdev);
7143 
7144 	pci_enable_wake(pdev, PCI_D3hot, 0);
7145 	pci_enable_wake(pdev, PCI_D3cold, 0);
7146 
7147 	if (igc_init_interrupt_scheme(adapter, true)) {
7148 		netdev_err(netdev, "Unable to allocate memory for queues\n");
7149 		return -ENOMEM;
7150 	}
7151 
7152 	igc_reset(adapter);
7153 
7154 	/* let the f/w know that the h/w is now under the control of the
7155 	 * driver.
7156 	 */
7157 	igc_get_hw_control(adapter);
7158 
7159 	val = rd32(IGC_WUS);
7160 	if (val & WAKE_PKT_WUS)
7161 		igc_deliver_wake_packet(netdev);
7162 
7163 	wr32(IGC_WUS, ~0);
7164 
7165 	rtnl_lock();
7166 	if (!err && netif_running(netdev))
7167 		err = __igc_open(netdev, true);
7168 
7169 	if (!err)
7170 		netif_device_attach(netdev);
7171 	rtnl_unlock();
7172 
7173 	return err;
7174 }
7175 
igc_runtime_resume(struct device * dev)7176 static int __maybe_unused igc_runtime_resume(struct device *dev)
7177 {
7178 	return igc_resume(dev);
7179 }
7180 
igc_suspend(struct device * dev)7181 static int __maybe_unused igc_suspend(struct device *dev)
7182 {
7183 	return __igc_shutdown(to_pci_dev(dev), NULL, 0);
7184 }
7185 
igc_runtime_idle(struct device * dev)7186 static int __maybe_unused igc_runtime_idle(struct device *dev)
7187 {
7188 	struct net_device *netdev = dev_get_drvdata(dev);
7189 	struct igc_adapter *adapter = netdev_priv(netdev);
7190 
7191 	if (!igc_has_link(adapter))
7192 		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
7193 
7194 	return -EBUSY;
7195 }
7196 #endif /* CONFIG_PM */
7197 
igc_shutdown(struct pci_dev * pdev)7198 static void igc_shutdown(struct pci_dev *pdev)
7199 {
7200 	bool wake;
7201 
7202 	__igc_shutdown(pdev, &wake, 0);
7203 
7204 	if (system_state == SYSTEM_POWER_OFF) {
7205 		pci_wake_from_d3(pdev, wake);
7206 		pci_set_power_state(pdev, PCI_D3hot);
7207 	}
7208 }
7209 
7210 /**
7211  *  igc_io_error_detected - called when PCI error is detected
7212  *  @pdev: Pointer to PCI device
7213  *  @state: The current PCI connection state
7214  *
7215  *  This function is called after a PCI bus error affecting
7216  *  this device has been detected.
7217  **/
igc_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)7218 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
7219 					      pci_channel_state_t state)
7220 {
7221 	struct net_device *netdev = pci_get_drvdata(pdev);
7222 	struct igc_adapter *adapter = netdev_priv(netdev);
7223 
7224 	netif_device_detach(netdev);
7225 
7226 	if (state == pci_channel_io_perm_failure)
7227 		return PCI_ERS_RESULT_DISCONNECT;
7228 
7229 	if (netif_running(netdev))
7230 		igc_down(adapter);
7231 	pci_disable_device(pdev);
7232 
7233 	/* Request a slot reset. */
7234 	return PCI_ERS_RESULT_NEED_RESET;
7235 }
7236 
7237 /**
7238  *  igc_io_slot_reset - called after the PCI bus has been reset.
7239  *  @pdev: Pointer to PCI device
7240  *
7241  *  Restart the card from scratch, as if from a cold-boot. Implementation
7242  *  resembles the first-half of the igc_resume routine.
7243  **/
igc_io_slot_reset(struct pci_dev * pdev)7244 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
7245 {
7246 	struct net_device *netdev = pci_get_drvdata(pdev);
7247 	struct igc_adapter *adapter = netdev_priv(netdev);
7248 	struct igc_hw *hw = &adapter->hw;
7249 	pci_ers_result_t result;
7250 
7251 	if (pci_enable_device_mem(pdev)) {
7252 		netdev_err(netdev, "Could not re-enable PCI device after reset\n");
7253 		result = PCI_ERS_RESULT_DISCONNECT;
7254 	} else {
7255 		pci_set_master(pdev);
7256 		pci_restore_state(pdev);
7257 		pci_save_state(pdev);
7258 
7259 		pci_enable_wake(pdev, PCI_D3hot, 0);
7260 		pci_enable_wake(pdev, PCI_D3cold, 0);
7261 
7262 		/* In case of PCI error, adapter loses its HW address
7263 		 * so we should re-assign it here.
7264 		 */
7265 		hw->hw_addr = adapter->io_addr;
7266 
7267 		igc_reset(adapter);
7268 		wr32(IGC_WUS, ~0);
7269 		result = PCI_ERS_RESULT_RECOVERED;
7270 	}
7271 
7272 	return result;
7273 }
7274 
7275 /**
7276  *  igc_io_resume - called when traffic can start to flow again.
7277  *  @pdev: Pointer to PCI device
7278  *
7279  *  This callback is called when the error recovery driver tells us that
7280  *  its OK to resume normal operation. Implementation resembles the
7281  *  second-half of the igc_resume routine.
7282  */
igc_io_resume(struct pci_dev * pdev)7283 static void igc_io_resume(struct pci_dev *pdev)
7284 {
7285 	struct net_device *netdev = pci_get_drvdata(pdev);
7286 	struct igc_adapter *adapter = netdev_priv(netdev);
7287 
7288 	rtnl_lock();
7289 	if (netif_running(netdev)) {
7290 		if (igc_open(netdev)) {
7291 			rtnl_unlock();
7292 			netdev_err(netdev, "igc_open failed after reset\n");
7293 			return;
7294 		}
7295 	}
7296 
7297 	netif_device_attach(netdev);
7298 
7299 	/* let the f/w know that the h/w is now under the control of the
7300 	 * driver.
7301 	 */
7302 	igc_get_hw_control(adapter);
7303 	rtnl_unlock();
7304 }
7305 
7306 static const struct pci_error_handlers igc_err_handler = {
7307 	.error_detected = igc_io_error_detected,
7308 	.slot_reset = igc_io_slot_reset,
7309 	.resume = igc_io_resume,
7310 };
7311 
7312 #ifdef CONFIG_PM
7313 static const struct dev_pm_ops igc_pm_ops = {
7314 	SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume)
7315 	SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume,
7316 			   igc_runtime_idle)
7317 };
7318 #endif
7319 
7320 static struct pci_driver igc_driver = {
7321 	.name     = igc_driver_name,
7322 	.id_table = igc_pci_tbl,
7323 	.probe    = igc_probe,
7324 	.remove   = igc_remove,
7325 #ifdef CONFIG_PM
7326 	.driver.pm = &igc_pm_ops,
7327 #endif
7328 	.shutdown = igc_shutdown,
7329 	.err_handler = &igc_err_handler,
7330 };
7331 
7332 /**
7333  * igc_reinit_queues - return error
7334  * @adapter: pointer to adapter structure
7335  */
igc_reinit_queues(struct igc_adapter * adapter)7336 int igc_reinit_queues(struct igc_adapter *adapter)
7337 {
7338 	struct net_device *netdev = adapter->netdev;
7339 	int err = 0;
7340 
7341 	if (netif_running(netdev))
7342 		igc_close(netdev);
7343 
7344 	igc_reset_interrupt_capability(adapter);
7345 
7346 	if (igc_init_interrupt_scheme(adapter, true)) {
7347 		netdev_err(netdev, "Unable to allocate memory for queues\n");
7348 		return -ENOMEM;
7349 	}
7350 
7351 	if (netif_running(netdev))
7352 		err = igc_open(netdev);
7353 
7354 	return err;
7355 }
7356 
7357 /**
7358  * igc_get_hw_dev - return device
7359  * @hw: pointer to hardware structure
7360  *
7361  * used by hardware layer to print debugging information
7362  */
igc_get_hw_dev(struct igc_hw * hw)7363 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
7364 {
7365 	struct igc_adapter *adapter = hw->back;
7366 
7367 	return adapter->netdev;
7368 }
7369 
igc_disable_rx_ring_hw(struct igc_ring * ring)7370 static void igc_disable_rx_ring_hw(struct igc_ring *ring)
7371 {
7372 	struct igc_hw *hw = &ring->q_vector->adapter->hw;
7373 	u8 idx = ring->reg_idx;
7374 	u32 rxdctl;
7375 
7376 	rxdctl = rd32(IGC_RXDCTL(idx));
7377 	rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE;
7378 	rxdctl |= IGC_RXDCTL_SWFLUSH;
7379 	wr32(IGC_RXDCTL(idx), rxdctl);
7380 }
7381 
igc_disable_rx_ring(struct igc_ring * ring)7382 void igc_disable_rx_ring(struct igc_ring *ring)
7383 {
7384 	igc_disable_rx_ring_hw(ring);
7385 	igc_clean_rx_ring(ring);
7386 }
7387 
igc_enable_rx_ring(struct igc_ring * ring)7388 void igc_enable_rx_ring(struct igc_ring *ring)
7389 {
7390 	struct igc_adapter *adapter = ring->q_vector->adapter;
7391 
7392 	igc_configure_rx_ring(adapter, ring);
7393 
7394 	if (ring->xsk_pool)
7395 		igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
7396 	else
7397 		igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
7398 }
7399 
igc_disable_tx_ring(struct igc_ring * ring)7400 void igc_disable_tx_ring(struct igc_ring *ring)
7401 {
7402 	igc_disable_tx_ring_hw(ring);
7403 	igc_clean_tx_ring(ring);
7404 }
7405 
igc_enable_tx_ring(struct igc_ring * ring)7406 void igc_enable_tx_ring(struct igc_ring *ring)
7407 {
7408 	struct igc_adapter *adapter = ring->q_vector->adapter;
7409 
7410 	igc_configure_tx_ring(adapter, ring);
7411 }
7412 
7413 /**
7414  * igc_init_module - Driver Registration Routine
7415  *
7416  * igc_init_module is the first routine called when the driver is
7417  * loaded. All it does is register with the PCI subsystem.
7418  */
igc_init_module(void)7419 static int __init igc_init_module(void)
7420 {
7421 	int ret;
7422 
7423 	pr_info("%s\n", igc_driver_string);
7424 	pr_info("%s\n", igc_copyright);
7425 
7426 	ret = pci_register_driver(&igc_driver);
7427 	return ret;
7428 }
7429 
7430 module_init(igc_init_module);
7431 
7432 /**
7433  * igc_exit_module - Driver Exit Cleanup Routine
7434  *
7435  * igc_exit_module is called just before the driver is removed
7436  * from memory.
7437  */
igc_exit_module(void)7438 static void __exit igc_exit_module(void)
7439 {
7440 	pci_unregister_driver(&igc_driver);
7441 }
7442 
7443 module_exit(igc_exit_module);
7444 /* igc_main.c */
7445