1d89f8841SSasha Neftin // SPDX-License-Identifier: GPL-2.0
2d89f8841SSasha Neftin /* Copyright (c) 2018 Intel Corporation */
3d89f8841SSasha Neftin
4d89f8841SSasha Neftin #include <linux/module.h>
5d89f8841SSasha Neftin #include <linux/types.h>
6c9a11c23SSasha Neftin #include <linux/if_vlan.h>
7d3ae3cfbSSasha Neftin #include <linux/tcp.h>
8d3ae3cfbSSasha Neftin #include <linux/udp.h>
9d3ae3cfbSSasha Neftin #include <linux/ip.h>
109513d2a5SSasha Neftin #include <linux/pm_runtime.h>
11ec50a9d4SVinicius Costa Gomes #include <net/pkt_sched.h>
1226575105SAndre Guedes #include <linux/bpf_trace.h>
13fc9df2a0SAndre Guedes #include <net/xdp_sock_drv.h>
141b5d73fbSVinicius Costa Gomes #include <linux/pci.h>
151b5d73fbSVinicius Costa Gomes
16d3ae3cfbSSasha Neftin #include <net/ipv6.h>
17d89f8841SSasha Neftin
18d89f8841SSasha Neftin #include "igc.h"
19d89f8841SSasha Neftin #include "igc_hw.h"
20ec50a9d4SVinicius Costa Gomes #include "igc_tsn.h"
2126575105SAndre Guedes #include "igc_xdp.h"
22d89f8841SSasha Neftin
23d89f8841SSasha Neftin #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver"
24d89f8841SSasha Neftin
258c5ad0daSSasha Neftin #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
268c5ad0daSSasha Neftin
2726575105SAndre Guedes #define IGC_XDP_PASS 0
2826575105SAndre Guedes #define IGC_XDP_CONSUMED BIT(0)
2973f1071cSAndre Guedes #define IGC_XDP_TX BIT(1)
304ff32036SAndre Guedes #define IGC_XDP_REDIRECT BIT(2)
3126575105SAndre Guedes
32c9a11c23SSasha Neftin static int debug = -1;
33c9a11c23SSasha Neftin
34d89f8841SSasha Neftin MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
35d89f8841SSasha Neftin MODULE_DESCRIPTION(DRV_SUMMARY);
36d89f8841SSasha Neftin MODULE_LICENSE("GPL v2");
37c9a11c23SSasha Neftin module_param(debug, int, 0);
38c9a11c23SSasha Neftin MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
39d89f8841SSasha Neftin
40d89f8841SSasha Neftin char igc_driver_name[] = "igc";
41d89f8841SSasha Neftin static const char igc_driver_string[] = DRV_SUMMARY;
42d89f8841SSasha Neftin static const char igc_copyright[] =
43d89f8841SSasha Neftin "Copyright(c) 2018 Intel Corporation.";
44d89f8841SSasha Neftin
45ab405612SSasha Neftin static const struct igc_info *igc_info_tbl[] = {
46ab405612SSasha Neftin [board_base] = &igc_base_info,
47ab405612SSasha Neftin };
48ab405612SSasha Neftin
49d89f8841SSasha Neftin static const struct pci_device_id igc_pci_tbl[] = {
50ab405612SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
51ab405612SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
526d37a382SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
536d37a382SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
546d37a382SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
55c2a3f8feSSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
56bfa5e98cSSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
57c2a3f8feSSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
588f20571dSSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
59c2a3f8feSSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
6043546211SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
6143546211SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
6243546211SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
6343546211SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
6443546211SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
650e7d4b93SSasha Neftin { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
66d89f8841SSasha Neftin /* required last entry */
67d89f8841SSasha Neftin {0, }
68d89f8841SSasha Neftin };
69d89f8841SSasha Neftin
70d89f8841SSasha Neftin MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
71d89f8841SSasha Neftin
723df25e4cSSasha Neftin enum latency_range {
733df25e4cSSasha Neftin lowest_latency = 0,
743df25e4cSSasha Neftin low_latency = 1,
753df25e4cSSasha Neftin bulk_latency = 2,
763df25e4cSSasha Neftin latency_invalid = 255
773df25e4cSSasha Neftin };
78c9a11c23SSasha Neftin
igc_reset(struct igc_adapter * adapter)798c5ad0daSSasha Neftin void igc_reset(struct igc_adapter *adapter)
80c9a11c23SSasha Neftin {
8125f06effSAndre Guedes struct net_device *dev = adapter->netdev;
82c0071c7aSSasha Neftin struct igc_hw *hw = &adapter->hw;
830373ad4dSSasha Neftin struct igc_fc_info *fc = &hw->fc;
840373ad4dSSasha Neftin u32 pba, hwm;
850373ad4dSSasha Neftin
860373ad4dSSasha Neftin /* Repartition PBA for greater than 9k MTU if required */
870373ad4dSSasha Neftin pba = IGC_PBA_34K;
880373ad4dSSasha Neftin
890373ad4dSSasha Neftin /* flow control settings
900373ad4dSSasha Neftin * The high water mark must be low enough to fit one full frame
910373ad4dSSasha Neftin * after transmitting the pause frame. As such we must have enough
920373ad4dSSasha Neftin * space to allow for us to complete our current transmit and then
930373ad4dSSasha Neftin * receive the frame that is in progress from the link partner.
940373ad4dSSasha Neftin * Set it to:
950373ad4dSSasha Neftin * - the full Rx FIFO size minus one full Tx plus one full Rx frame
960373ad4dSSasha Neftin */
970373ad4dSSasha Neftin hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
980373ad4dSSasha Neftin
990373ad4dSSasha Neftin fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */
1000373ad4dSSasha Neftin fc->low_water = fc->high_water - 16;
1010373ad4dSSasha Neftin fc->pause_time = 0xFFFF;
1020373ad4dSSasha Neftin fc->send_xon = 1;
1030373ad4dSSasha Neftin fc->current_mode = fc->requested_mode;
104c0071c7aSSasha Neftin
105c0071c7aSSasha Neftin hw->mac.ops.reset_hw(hw);
106c0071c7aSSasha Neftin
107c0071c7aSSasha Neftin if (hw->mac.ops.init_hw(hw))
10825f06effSAndre Guedes netdev_err(dev, "Error on hardware initialization\n");
109c0071c7aSSasha Neftin
11093ec439aSSasha Neftin /* Re-establish EEE setting */
11193ec439aSSasha Neftin igc_set_eee_i225(hw, true, true, true);
11293ec439aSSasha Neftin
113c9a11c23SSasha Neftin if (!netif_running(adapter->netdev))
114a0beb3c1SSasha Neftin igc_power_down_phy_copper_base(&adapter->hw);
1155586838fSSasha Neftin
1168d744963SMuhammad Husaini Zulkifli /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
1178d744963SMuhammad Husaini Zulkifli wr32(IGC_VET, ETH_P_8021Q);
1188d744963SMuhammad Husaini Zulkifli
1195f295805SVinicius Costa Gomes /* Re-enable PTP, where applicable. */
1205f295805SVinicius Costa Gomes igc_ptp_reset(adapter);
1215f295805SVinicius Costa Gomes
122ec50a9d4SVinicius Costa Gomes /* Re-enable TSN offloading, where applicable. */
12361572d5fSVinicius Costa Gomes igc_tsn_reset(adapter);
124ec50a9d4SVinicius Costa Gomes
1255586838fSSasha Neftin igc_get_phy_info(hw);
126c9a11c23SSasha Neftin }
127c9a11c23SSasha Neftin
128c9a11c23SSasha Neftin /**
129684ea87cSSasha Neftin * igc_power_up_link - Power up the phy link
130c9a11c23SSasha Neftin * @adapter: address of board private structure
131c9a11c23SSasha Neftin */
igc_power_up_link(struct igc_adapter * adapter)132c9a11c23SSasha Neftin static void igc_power_up_link(struct igc_adapter *adapter)
133c9a11c23SSasha Neftin {
1345586838fSSasha Neftin igc_reset_phy(&adapter->hw);
1355586838fSSasha Neftin
1365586838fSSasha Neftin igc_power_up_phy_copper(&adapter->hw);
1375586838fSSasha Neftin
1385586838fSSasha Neftin igc_setup_link(&adapter->hw);
139c9a11c23SSasha Neftin }
140c9a11c23SSasha Neftin
141c9a11c23SSasha Neftin /**
142c9a11c23SSasha Neftin * igc_release_hw_control - release control of the h/w to f/w
143c9a11c23SSasha Neftin * @adapter: address of board private structure
144c9a11c23SSasha Neftin *
145c9a11c23SSasha Neftin * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
146c9a11c23SSasha Neftin * For ASF and Pass Through versions of f/w this means that the
147c9a11c23SSasha Neftin * driver is no longer loaded.
148c9a11c23SSasha Neftin */
igc_release_hw_control(struct igc_adapter * adapter)149c9a11c23SSasha Neftin static void igc_release_hw_control(struct igc_adapter *adapter)
150c9a11c23SSasha Neftin {
151c9a11c23SSasha Neftin struct igc_hw *hw = &adapter->hw;
152c9a11c23SSasha Neftin u32 ctrl_ext;
153c9a11c23SSasha Neftin
1544b799595SAaron Ma if (!pci_device_is_present(adapter->pdev))
1554b799595SAaron Ma return;
1564b799595SAaron Ma
157c9a11c23SSasha Neftin /* Let firmware take over control of h/w */
158c9a11c23SSasha Neftin ctrl_ext = rd32(IGC_CTRL_EXT);
159c9a11c23SSasha Neftin wr32(IGC_CTRL_EXT,
160c9a11c23SSasha Neftin ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
161c9a11c23SSasha Neftin }
162c9a11c23SSasha Neftin
163c9a11c23SSasha Neftin /**
164c9a11c23SSasha Neftin * igc_get_hw_control - get control of the h/w from f/w
165c9a11c23SSasha Neftin * @adapter: address of board private structure
166c9a11c23SSasha Neftin *
167c9a11c23SSasha Neftin * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
168c9a11c23SSasha Neftin * For ASF and Pass Through versions of f/w this means that
169c9a11c23SSasha Neftin * the driver is loaded.
170c9a11c23SSasha Neftin */
igc_get_hw_control(struct igc_adapter * adapter)171c9a11c23SSasha Neftin static void igc_get_hw_control(struct igc_adapter *adapter)
172c9a11c23SSasha Neftin {
173c9a11c23SSasha Neftin struct igc_hw *hw = &adapter->hw;
174c9a11c23SSasha Neftin u32 ctrl_ext;
175c9a11c23SSasha Neftin
176c9a11c23SSasha Neftin /* Let firmware know the driver has taken over */
177c9a11c23SSasha Neftin ctrl_ext = rd32(IGC_CTRL_EXT);
178c9a11c23SSasha Neftin wr32(IGC_CTRL_EXT,
179c9a11c23SSasha Neftin ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
180c9a11c23SSasha Neftin }
181c9a11c23SSasha Neftin
igc_unmap_tx_buffer(struct device * dev,struct igc_tx_buffer * buf)18261234295SAndre Guedes static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf)
18361234295SAndre Guedes {
18461234295SAndre Guedes dma_unmap_single(dev, dma_unmap_addr(buf, dma),
18561234295SAndre Guedes dma_unmap_len(buf, len), DMA_TO_DEVICE);
18661234295SAndre Guedes
18761234295SAndre Guedes dma_unmap_len_set(buf, len, 0);
18861234295SAndre Guedes }
18961234295SAndre Guedes
190c9a11c23SSasha Neftin /**
19113b5b7fdSSasha Neftin * igc_clean_tx_ring - Free Tx Buffers
19213b5b7fdSSasha Neftin * @tx_ring: ring to be cleaned
19313b5b7fdSSasha Neftin */
igc_clean_tx_ring(struct igc_ring * tx_ring)19413b5b7fdSSasha Neftin static void igc_clean_tx_ring(struct igc_ring *tx_ring)
19513b5b7fdSSasha Neftin {
19613b5b7fdSSasha Neftin u16 i = tx_ring->next_to_clean;
19713b5b7fdSSasha Neftin struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
1989acf59a7SAndre Guedes u32 xsk_frames = 0;
19913b5b7fdSSasha Neftin
20013b5b7fdSSasha Neftin while (i != tx_ring->next_to_use) {
20113b5b7fdSSasha Neftin union igc_adv_tx_desc *eop_desc, *tx_desc;
20213b5b7fdSSasha Neftin
203859b4dfaSAndre Guedes switch (tx_buffer->type) {
2049acf59a7SAndre Guedes case IGC_TX_BUFFER_TYPE_XSK:
2059acf59a7SAndre Guedes xsk_frames++;
2069acf59a7SAndre Guedes break;
207859b4dfaSAndre Guedes case IGC_TX_BUFFER_TYPE_XDP:
20873f1071cSAndre Guedes xdp_return_frame(tx_buffer->xdpf);
2099acf59a7SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
210859b4dfaSAndre Guedes break;
211859b4dfaSAndre Guedes case IGC_TX_BUFFER_TYPE_SKB:
21213b5b7fdSSasha Neftin dev_kfree_skb_any(tx_buffer->skb);
2139acf59a7SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
214859b4dfaSAndre Guedes break;
215859b4dfaSAndre Guedes default:
216859b4dfaSAndre Guedes netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
217859b4dfaSAndre Guedes break;
218859b4dfaSAndre Guedes }
21913b5b7fdSSasha Neftin
22013b5b7fdSSasha Neftin /* check for eop_desc to determine the end of the packet */
22113b5b7fdSSasha Neftin eop_desc = tx_buffer->next_to_watch;
22213b5b7fdSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, i);
22313b5b7fdSSasha Neftin
22413b5b7fdSSasha Neftin /* unmap remaining buffers */
22513b5b7fdSSasha Neftin while (tx_desc != eop_desc) {
22613b5b7fdSSasha Neftin tx_buffer++;
22713b5b7fdSSasha Neftin tx_desc++;
22813b5b7fdSSasha Neftin i++;
22913b5b7fdSSasha Neftin if (unlikely(i == tx_ring->count)) {
23013b5b7fdSSasha Neftin i = 0;
23113b5b7fdSSasha Neftin tx_buffer = tx_ring->tx_buffer_info;
23213b5b7fdSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, 0);
23313b5b7fdSSasha Neftin }
23413b5b7fdSSasha Neftin
23513b5b7fdSSasha Neftin /* unmap any remaining paged data */
23613b5b7fdSSasha Neftin if (dma_unmap_len(tx_buffer, len))
23761234295SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
23813b5b7fdSSasha Neftin }
23913b5b7fdSSasha Neftin
24056ea7ed1SVinicius Costa Gomes tx_buffer->next_to_watch = NULL;
24156ea7ed1SVinicius Costa Gomes
24213b5b7fdSSasha Neftin /* move us one more past the eop_desc for start of next pkt */
24313b5b7fdSSasha Neftin tx_buffer++;
24413b5b7fdSSasha Neftin i++;
24513b5b7fdSSasha Neftin if (unlikely(i == tx_ring->count)) {
24613b5b7fdSSasha Neftin i = 0;
24713b5b7fdSSasha Neftin tx_buffer = tx_ring->tx_buffer_info;
24813b5b7fdSSasha Neftin }
24913b5b7fdSSasha Neftin }
25013b5b7fdSSasha Neftin
2519acf59a7SAndre Guedes if (tx_ring->xsk_pool && xsk_frames)
2529acf59a7SAndre Guedes xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
2539acf59a7SAndre Guedes
25413b5b7fdSSasha Neftin /* reset BQL for queue */
25513b5b7fdSSasha Neftin netdev_tx_reset_queue(txring_txq(tx_ring));
25613b5b7fdSSasha Neftin
257e43516f5SMuhammad Husaini Zulkifli /* Zero out the buffer ring */
258e43516f5SMuhammad Husaini Zulkifli memset(tx_ring->tx_buffer_info, 0,
259e43516f5SMuhammad Husaini Zulkifli sizeof(*tx_ring->tx_buffer_info) * tx_ring->count);
260e43516f5SMuhammad Husaini Zulkifli
261e43516f5SMuhammad Husaini Zulkifli /* Zero out the descriptor ring */
262e43516f5SMuhammad Husaini Zulkifli memset(tx_ring->desc, 0, tx_ring->size);
263e43516f5SMuhammad Husaini Zulkifli
26413b5b7fdSSasha Neftin /* reset next_to_use and next_to_clean */
26513b5b7fdSSasha Neftin tx_ring->next_to_use = 0;
26613b5b7fdSSasha Neftin tx_ring->next_to_clean = 0;
26713b5b7fdSSasha Neftin }
26813b5b7fdSSasha Neftin
26913b5b7fdSSasha Neftin /**
27014504ac5SSasha Neftin * igc_free_tx_resources - Free Tx Resources per Queue
27114504ac5SSasha Neftin * @tx_ring: Tx descriptor ring for a specific queue
27214504ac5SSasha Neftin *
27314504ac5SSasha Neftin * Free all transmit software resources
27414504ac5SSasha Neftin */
igc_free_tx_resources(struct igc_ring * tx_ring)27514504ac5SSasha Neftin void igc_free_tx_resources(struct igc_ring *tx_ring)
27614504ac5SSasha Neftin {
277e43516f5SMuhammad Husaini Zulkifli igc_disable_tx_ring(tx_ring);
27814504ac5SSasha Neftin
27914504ac5SSasha Neftin vfree(tx_ring->tx_buffer_info);
28014504ac5SSasha Neftin tx_ring->tx_buffer_info = NULL;
28114504ac5SSasha Neftin
28214504ac5SSasha Neftin /* if not set, then don't free */
28314504ac5SSasha Neftin if (!tx_ring->desc)
28414504ac5SSasha Neftin return;
28514504ac5SSasha Neftin
28614504ac5SSasha Neftin dma_free_coherent(tx_ring->dev, tx_ring->size,
28714504ac5SSasha Neftin tx_ring->desc, tx_ring->dma);
28814504ac5SSasha Neftin
28914504ac5SSasha Neftin tx_ring->desc = NULL;
29014504ac5SSasha Neftin }
29114504ac5SSasha Neftin
29214504ac5SSasha Neftin /**
29314504ac5SSasha Neftin * igc_free_all_tx_resources - Free Tx Resources for All Queues
29414504ac5SSasha Neftin * @adapter: board private structure
29514504ac5SSasha Neftin *
29614504ac5SSasha Neftin * Free all transmit software resources
29714504ac5SSasha Neftin */
igc_free_all_tx_resources(struct igc_adapter * adapter)29814504ac5SSasha Neftin static void igc_free_all_tx_resources(struct igc_adapter *adapter)
29914504ac5SSasha Neftin {
30014504ac5SSasha Neftin int i;
30114504ac5SSasha Neftin
30214504ac5SSasha Neftin for (i = 0; i < adapter->num_tx_queues; i++)
30314504ac5SSasha Neftin igc_free_tx_resources(adapter->tx_ring[i]);
30414504ac5SSasha Neftin }
30514504ac5SSasha Neftin
30614504ac5SSasha Neftin /**
3070507ef8aSSasha Neftin * igc_clean_all_tx_rings - Free Tx Buffers for all queues
3080507ef8aSSasha Neftin * @adapter: board private structure
3090507ef8aSSasha Neftin */
igc_clean_all_tx_rings(struct igc_adapter * adapter)3100507ef8aSSasha Neftin static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
3110507ef8aSSasha Neftin {
3120507ef8aSSasha Neftin int i;
3130507ef8aSSasha Neftin
3140507ef8aSSasha Neftin for (i = 0; i < adapter->num_tx_queues; i++)
3150507ef8aSSasha Neftin if (adapter->tx_ring[i])
3160507ef8aSSasha Neftin igc_clean_tx_ring(adapter->tx_ring[i]);
3170507ef8aSSasha Neftin }
3180507ef8aSSasha Neftin
igc_disable_tx_ring_hw(struct igc_ring * ring)319d4a7ce64SMuhammad Husaini Zulkifli static void igc_disable_tx_ring_hw(struct igc_ring *ring)
320d4a7ce64SMuhammad Husaini Zulkifli {
321d4a7ce64SMuhammad Husaini Zulkifli struct igc_hw *hw = &ring->q_vector->adapter->hw;
322d4a7ce64SMuhammad Husaini Zulkifli u8 idx = ring->reg_idx;
323d4a7ce64SMuhammad Husaini Zulkifli u32 txdctl;
324d4a7ce64SMuhammad Husaini Zulkifli
325d4a7ce64SMuhammad Husaini Zulkifli txdctl = rd32(IGC_TXDCTL(idx));
326d4a7ce64SMuhammad Husaini Zulkifli txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE;
327d4a7ce64SMuhammad Husaini Zulkifli txdctl |= IGC_TXDCTL_SWFLUSH;
328d4a7ce64SMuhammad Husaini Zulkifli wr32(IGC_TXDCTL(idx), txdctl);
329d4a7ce64SMuhammad Husaini Zulkifli }
330d4a7ce64SMuhammad Husaini Zulkifli
331d4a7ce64SMuhammad Husaini Zulkifli /**
332d4a7ce64SMuhammad Husaini Zulkifli * igc_disable_all_tx_rings_hw - Disable all transmit queue operation
333d4a7ce64SMuhammad Husaini Zulkifli * @adapter: board private structure
334d4a7ce64SMuhammad Husaini Zulkifli */
igc_disable_all_tx_rings_hw(struct igc_adapter * adapter)335d4a7ce64SMuhammad Husaini Zulkifli static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter)
336d4a7ce64SMuhammad Husaini Zulkifli {
337d4a7ce64SMuhammad Husaini Zulkifli int i;
338d4a7ce64SMuhammad Husaini Zulkifli
339d4a7ce64SMuhammad Husaini Zulkifli for (i = 0; i < adapter->num_tx_queues; i++) {
340d4a7ce64SMuhammad Husaini Zulkifli struct igc_ring *tx_ring = adapter->tx_ring[i];
341d4a7ce64SMuhammad Husaini Zulkifli
342d4a7ce64SMuhammad Husaini Zulkifli igc_disable_tx_ring_hw(tx_ring);
343d4a7ce64SMuhammad Husaini Zulkifli }
344d4a7ce64SMuhammad Husaini Zulkifli }
345d4a7ce64SMuhammad Husaini Zulkifli
3460507ef8aSSasha Neftin /**
34713b5b7fdSSasha Neftin * igc_setup_tx_resources - allocate Tx resources (Descriptors)
34813b5b7fdSSasha Neftin * @tx_ring: tx descriptor ring (for a specific queue) to setup
34913b5b7fdSSasha Neftin *
35013b5b7fdSSasha Neftin * Return 0 on success, negative on failure
35113b5b7fdSSasha Neftin */
igc_setup_tx_resources(struct igc_ring * tx_ring)3528c5ad0daSSasha Neftin int igc_setup_tx_resources(struct igc_ring *tx_ring)
35313b5b7fdSSasha Neftin {
35425f06effSAndre Guedes struct net_device *ndev = tx_ring->netdev;
35513b5b7fdSSasha Neftin struct device *dev = tx_ring->dev;
35613b5b7fdSSasha Neftin int size = 0;
35713b5b7fdSSasha Neftin
35813b5b7fdSSasha Neftin size = sizeof(struct igc_tx_buffer) * tx_ring->count;
35913b5b7fdSSasha Neftin tx_ring->tx_buffer_info = vzalloc(size);
36013b5b7fdSSasha Neftin if (!tx_ring->tx_buffer_info)
36113b5b7fdSSasha Neftin goto err;
36213b5b7fdSSasha Neftin
36313b5b7fdSSasha Neftin /* round up to nearest 4K */
36413b5b7fdSSasha Neftin tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
36513b5b7fdSSasha Neftin tx_ring->size = ALIGN(tx_ring->size, 4096);
36613b5b7fdSSasha Neftin
36713b5b7fdSSasha Neftin tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
36813b5b7fdSSasha Neftin &tx_ring->dma, GFP_KERNEL);
36913b5b7fdSSasha Neftin
37013b5b7fdSSasha Neftin if (!tx_ring->desc)
37113b5b7fdSSasha Neftin goto err;
37213b5b7fdSSasha Neftin
37313b5b7fdSSasha Neftin tx_ring->next_to_use = 0;
37413b5b7fdSSasha Neftin tx_ring->next_to_clean = 0;
37513b5b7fdSSasha Neftin
37613b5b7fdSSasha Neftin return 0;
37713b5b7fdSSasha Neftin
37813b5b7fdSSasha Neftin err:
37913b5b7fdSSasha Neftin vfree(tx_ring->tx_buffer_info);
38025f06effSAndre Guedes netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
38113b5b7fdSSasha Neftin return -ENOMEM;
38213b5b7fdSSasha Neftin }
38313b5b7fdSSasha Neftin
38413b5b7fdSSasha Neftin /**
38513b5b7fdSSasha Neftin * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
38613b5b7fdSSasha Neftin * @adapter: board private structure
38713b5b7fdSSasha Neftin *
38813b5b7fdSSasha Neftin * Return 0 on success, negative on failure
38913b5b7fdSSasha Neftin */
igc_setup_all_tx_resources(struct igc_adapter * adapter)39013b5b7fdSSasha Neftin static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
39113b5b7fdSSasha Neftin {
39225f06effSAndre Guedes struct net_device *dev = adapter->netdev;
39313b5b7fdSSasha Neftin int i, err = 0;
39413b5b7fdSSasha Neftin
39513b5b7fdSSasha Neftin for (i = 0; i < adapter->num_tx_queues; i++) {
39613b5b7fdSSasha Neftin err = igc_setup_tx_resources(adapter->tx_ring[i]);
39713b5b7fdSSasha Neftin if (err) {
39825f06effSAndre Guedes netdev_err(dev, "Error on Tx queue %u setup\n", i);
39913b5b7fdSSasha Neftin for (i--; i >= 0; i--)
40013b5b7fdSSasha Neftin igc_free_tx_resources(adapter->tx_ring[i]);
40113b5b7fdSSasha Neftin break;
40213b5b7fdSSasha Neftin }
40313b5b7fdSSasha Neftin }
40413b5b7fdSSasha Neftin
40513b5b7fdSSasha Neftin return err;
40613b5b7fdSSasha Neftin }
40713b5b7fdSSasha Neftin
igc_clean_rx_ring_page_shared(struct igc_ring * rx_ring)408f4851648SAndre Guedes static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring)
40913b5b7fdSSasha Neftin {
41013b5b7fdSSasha Neftin u16 i = rx_ring->next_to_clean;
41113b5b7fdSSasha Neftin
41213b5b7fdSSasha Neftin dev_kfree_skb(rx_ring->skb);
41313b5b7fdSSasha Neftin rx_ring->skb = NULL;
41413b5b7fdSSasha Neftin
41513b5b7fdSSasha Neftin /* Free all the Rx ring sk_buffs */
41613b5b7fdSSasha Neftin while (i != rx_ring->next_to_alloc) {
41713b5b7fdSSasha Neftin struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
41813b5b7fdSSasha Neftin
41913b5b7fdSSasha Neftin /* Invalidate cache lines that may have been written to by
42013b5b7fdSSasha Neftin * device so that we avoid corrupting memory.
42113b5b7fdSSasha Neftin */
42213b5b7fdSSasha Neftin dma_sync_single_range_for_cpu(rx_ring->dev,
42313b5b7fdSSasha Neftin buffer_info->dma,
42413b5b7fdSSasha Neftin buffer_info->page_offset,
42513b5b7fdSSasha Neftin igc_rx_bufsz(rx_ring),
42613b5b7fdSSasha Neftin DMA_FROM_DEVICE);
42713b5b7fdSSasha Neftin
42813b5b7fdSSasha Neftin /* free resources associated with mapping */
42913b5b7fdSSasha Neftin dma_unmap_page_attrs(rx_ring->dev,
43013b5b7fdSSasha Neftin buffer_info->dma,
43113b5b7fdSSasha Neftin igc_rx_pg_size(rx_ring),
43213b5b7fdSSasha Neftin DMA_FROM_DEVICE,
43313b5b7fdSSasha Neftin IGC_RX_DMA_ATTR);
43413b5b7fdSSasha Neftin __page_frag_cache_drain(buffer_info->page,
43513b5b7fdSSasha Neftin buffer_info->pagecnt_bias);
43613b5b7fdSSasha Neftin
43713b5b7fdSSasha Neftin i++;
43813b5b7fdSSasha Neftin if (i == rx_ring->count)
43913b5b7fdSSasha Neftin i = 0;
44013b5b7fdSSasha Neftin }
441f4851648SAndre Guedes }
44213b5b7fdSSasha Neftin
igc_clean_rx_ring_xsk_pool(struct igc_ring * ring)443fc9df2a0SAndre Guedes static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring)
444fc9df2a0SAndre Guedes {
445fc9df2a0SAndre Guedes struct igc_rx_buffer *bi;
446fc9df2a0SAndre Guedes u16 i;
447fc9df2a0SAndre Guedes
448fc9df2a0SAndre Guedes for (i = 0; i < ring->count; i++) {
449fc9df2a0SAndre Guedes bi = &ring->rx_buffer_info[i];
450fc9df2a0SAndre Guedes if (!bi->xdp)
451fc9df2a0SAndre Guedes continue;
452fc9df2a0SAndre Guedes
453fc9df2a0SAndre Guedes xsk_buff_free(bi->xdp);
454fc9df2a0SAndre Guedes bi->xdp = NULL;
455fc9df2a0SAndre Guedes }
456fc9df2a0SAndre Guedes }
457fc9df2a0SAndre Guedes
458f4851648SAndre Guedes /**
459f4851648SAndre Guedes * igc_clean_rx_ring - Free Rx Buffers per Queue
460f4851648SAndre Guedes * @ring: ring to free buffers from
461f4851648SAndre Guedes */
igc_clean_rx_ring(struct igc_ring * ring)462f4851648SAndre Guedes static void igc_clean_rx_ring(struct igc_ring *ring)
463f4851648SAndre Guedes {
464fc9df2a0SAndre Guedes if (ring->xsk_pool)
465fc9df2a0SAndre Guedes igc_clean_rx_ring_xsk_pool(ring);
466fc9df2a0SAndre Guedes else
467f4851648SAndre Guedes igc_clean_rx_ring_page_shared(ring);
46826575105SAndre Guedes
469f4851648SAndre Guedes clear_ring_uses_large_buffer(ring);
470f4851648SAndre Guedes
471f4851648SAndre Guedes ring->next_to_alloc = 0;
472f4851648SAndre Guedes ring->next_to_clean = 0;
473f4851648SAndre Guedes ring->next_to_use = 0;
47413b5b7fdSSasha Neftin }
47513b5b7fdSSasha Neftin
47613b5b7fdSSasha Neftin /**
4770507ef8aSSasha Neftin * igc_clean_all_rx_rings - Free Rx Buffers for all queues
4780507ef8aSSasha Neftin * @adapter: board private structure
4790507ef8aSSasha Neftin */
igc_clean_all_rx_rings(struct igc_adapter * adapter)4800507ef8aSSasha Neftin static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
4810507ef8aSSasha Neftin {
4820507ef8aSSasha Neftin int i;
4830507ef8aSSasha Neftin
4840507ef8aSSasha Neftin for (i = 0; i < adapter->num_rx_queues; i++)
4850507ef8aSSasha Neftin if (adapter->rx_ring[i])
4860507ef8aSSasha Neftin igc_clean_rx_ring(adapter->rx_ring[i]);
4870507ef8aSSasha Neftin }
4880507ef8aSSasha Neftin
4890507ef8aSSasha Neftin /**
49013b5b7fdSSasha Neftin * igc_free_rx_resources - Free Rx Resources
49113b5b7fdSSasha Neftin * @rx_ring: ring to clean the resources from
49213b5b7fdSSasha Neftin *
49313b5b7fdSSasha Neftin * Free all receive software resources
49413b5b7fdSSasha Neftin */
igc_free_rx_resources(struct igc_ring * rx_ring)4958c5ad0daSSasha Neftin void igc_free_rx_resources(struct igc_ring *rx_ring)
49613b5b7fdSSasha Neftin {
49713b5b7fdSSasha Neftin igc_clean_rx_ring(rx_ring);
49813b5b7fdSSasha Neftin
4994609ffb9SAndre Guedes xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
50073f1071cSAndre Guedes
50113b5b7fdSSasha Neftin vfree(rx_ring->rx_buffer_info);
50213b5b7fdSSasha Neftin rx_ring->rx_buffer_info = NULL;
50313b5b7fdSSasha Neftin
50413b5b7fdSSasha Neftin /* if not set, then don't free */
50513b5b7fdSSasha Neftin if (!rx_ring->desc)
50613b5b7fdSSasha Neftin return;
50713b5b7fdSSasha Neftin
50813b5b7fdSSasha Neftin dma_free_coherent(rx_ring->dev, rx_ring->size,
50913b5b7fdSSasha Neftin rx_ring->desc, rx_ring->dma);
51013b5b7fdSSasha Neftin
51113b5b7fdSSasha Neftin rx_ring->desc = NULL;
51213b5b7fdSSasha Neftin }
51313b5b7fdSSasha Neftin
51413b5b7fdSSasha Neftin /**
51513b5b7fdSSasha Neftin * igc_free_all_rx_resources - Free Rx Resources for All Queues
51613b5b7fdSSasha Neftin * @adapter: board private structure
51713b5b7fdSSasha Neftin *
51813b5b7fdSSasha Neftin * Free all receive software resources
51913b5b7fdSSasha Neftin */
igc_free_all_rx_resources(struct igc_adapter * adapter)52013b5b7fdSSasha Neftin static void igc_free_all_rx_resources(struct igc_adapter *adapter)
52113b5b7fdSSasha Neftin {
52213b5b7fdSSasha Neftin int i;
52313b5b7fdSSasha Neftin
52413b5b7fdSSasha Neftin for (i = 0; i < adapter->num_rx_queues; i++)
52513b5b7fdSSasha Neftin igc_free_rx_resources(adapter->rx_ring[i]);
52613b5b7fdSSasha Neftin }
52713b5b7fdSSasha Neftin
52813b5b7fdSSasha Neftin /**
52913b5b7fdSSasha Neftin * igc_setup_rx_resources - allocate Rx resources (Descriptors)
53013b5b7fdSSasha Neftin * @rx_ring: rx descriptor ring (for a specific queue) to setup
53113b5b7fdSSasha Neftin *
53213b5b7fdSSasha Neftin * Returns 0 on success, negative on failure
53313b5b7fdSSasha Neftin */
igc_setup_rx_resources(struct igc_ring * rx_ring)5348c5ad0daSSasha Neftin int igc_setup_rx_resources(struct igc_ring *rx_ring)
53513b5b7fdSSasha Neftin {
53625f06effSAndre Guedes struct net_device *ndev = rx_ring->netdev;
53713b5b7fdSSasha Neftin struct device *dev = rx_ring->dev;
5384609ffb9SAndre Guedes u8 index = rx_ring->queue_index;
53973f1071cSAndre Guedes int size, desc_len, res;
54073f1071cSAndre Guedes
541453307b5SCorinna Vinschen /* XDP RX-queue info */
542453307b5SCorinna Vinschen if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
543453307b5SCorinna Vinschen xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
5444609ffb9SAndre Guedes res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
5454609ffb9SAndre Guedes rx_ring->q_vector->napi.napi_id);
5464609ffb9SAndre Guedes if (res < 0) {
5474609ffb9SAndre Guedes netdev_err(ndev, "Failed to register xdp_rxq index %u\n",
5484609ffb9SAndre Guedes index);
54973f1071cSAndre Guedes return res;
5504609ffb9SAndre Guedes }
55113b5b7fdSSasha Neftin
55213b5b7fdSSasha Neftin size = sizeof(struct igc_rx_buffer) * rx_ring->count;
55313b5b7fdSSasha Neftin rx_ring->rx_buffer_info = vzalloc(size);
55413b5b7fdSSasha Neftin if (!rx_ring->rx_buffer_info)
55513b5b7fdSSasha Neftin goto err;
55613b5b7fdSSasha Neftin
55713b5b7fdSSasha Neftin desc_len = sizeof(union igc_adv_rx_desc);
55813b5b7fdSSasha Neftin
55913b5b7fdSSasha Neftin /* Round up to nearest 4K */
56013b5b7fdSSasha Neftin rx_ring->size = rx_ring->count * desc_len;
56113b5b7fdSSasha Neftin rx_ring->size = ALIGN(rx_ring->size, 4096);
56213b5b7fdSSasha Neftin
56313b5b7fdSSasha Neftin rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
56413b5b7fdSSasha Neftin &rx_ring->dma, GFP_KERNEL);
56513b5b7fdSSasha Neftin
56613b5b7fdSSasha Neftin if (!rx_ring->desc)
56713b5b7fdSSasha Neftin goto err;
56813b5b7fdSSasha Neftin
56913b5b7fdSSasha Neftin rx_ring->next_to_alloc = 0;
57013b5b7fdSSasha Neftin rx_ring->next_to_clean = 0;
57113b5b7fdSSasha Neftin rx_ring->next_to_use = 0;
57213b5b7fdSSasha Neftin
57313b5b7fdSSasha Neftin return 0;
57413b5b7fdSSasha Neftin
57513b5b7fdSSasha Neftin err:
5764609ffb9SAndre Guedes xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
57713b5b7fdSSasha Neftin vfree(rx_ring->rx_buffer_info);
57813b5b7fdSSasha Neftin rx_ring->rx_buffer_info = NULL;
57925f06effSAndre Guedes netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
58013b5b7fdSSasha Neftin return -ENOMEM;
58113b5b7fdSSasha Neftin }
58213b5b7fdSSasha Neftin
58313b5b7fdSSasha Neftin /**
58413b5b7fdSSasha Neftin * igc_setup_all_rx_resources - wrapper to allocate Rx resources
58513b5b7fdSSasha Neftin * (Descriptors) for all queues
58613b5b7fdSSasha Neftin * @adapter: board private structure
58713b5b7fdSSasha Neftin *
58813b5b7fdSSasha Neftin * Return 0 on success, negative on failure
58913b5b7fdSSasha Neftin */
igc_setup_all_rx_resources(struct igc_adapter * adapter)59013b5b7fdSSasha Neftin static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
59113b5b7fdSSasha Neftin {
59225f06effSAndre Guedes struct net_device *dev = adapter->netdev;
59313b5b7fdSSasha Neftin int i, err = 0;
59413b5b7fdSSasha Neftin
59513b5b7fdSSasha Neftin for (i = 0; i < adapter->num_rx_queues; i++) {
59613b5b7fdSSasha Neftin err = igc_setup_rx_resources(adapter->rx_ring[i]);
59713b5b7fdSSasha Neftin if (err) {
59825f06effSAndre Guedes netdev_err(dev, "Error on Rx queue %u setup\n", i);
59913b5b7fdSSasha Neftin for (i--; i >= 0; i--)
60013b5b7fdSSasha Neftin igc_free_rx_resources(adapter->rx_ring[i]);
60113b5b7fdSSasha Neftin break;
60213b5b7fdSSasha Neftin }
60313b5b7fdSSasha Neftin }
60413b5b7fdSSasha Neftin
60513b5b7fdSSasha Neftin return err;
60613b5b7fdSSasha Neftin }
60713b5b7fdSSasha Neftin
igc_get_xsk_pool(struct igc_adapter * adapter,struct igc_ring * ring)608fc9df2a0SAndre Guedes static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter,
609fc9df2a0SAndre Guedes struct igc_ring *ring)
610fc9df2a0SAndre Guedes {
611fc9df2a0SAndre Guedes if (!igc_xdp_is_enabled(adapter) ||
612fc9df2a0SAndre Guedes !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags))
613fc9df2a0SAndre Guedes return NULL;
614fc9df2a0SAndre Guedes
615fc9df2a0SAndre Guedes return xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
616fc9df2a0SAndre Guedes }
617fc9df2a0SAndre Guedes
61813b5b7fdSSasha Neftin /**
61913b5b7fdSSasha Neftin * igc_configure_rx_ring - Configure a receive ring after Reset
62013b5b7fdSSasha Neftin * @adapter: board private structure
62113b5b7fdSSasha Neftin * @ring: receive ring to be configured
62213b5b7fdSSasha Neftin *
62313b5b7fdSSasha Neftin * Configure the Rx unit of the MAC after a reset.
62413b5b7fdSSasha Neftin */
igc_configure_rx_ring(struct igc_adapter * adapter,struct igc_ring * ring)62513b5b7fdSSasha Neftin static void igc_configure_rx_ring(struct igc_adapter *adapter,
62613b5b7fdSSasha Neftin struct igc_ring *ring)
62713b5b7fdSSasha Neftin {
62813b5b7fdSSasha Neftin struct igc_hw *hw = &adapter->hw;
62913b5b7fdSSasha Neftin union igc_adv_rx_desc *rx_desc;
63013b5b7fdSSasha Neftin int reg_idx = ring->reg_idx;
63113b5b7fdSSasha Neftin u32 srrctl = 0, rxdctl = 0;
63213b5b7fdSSasha Neftin u64 rdba = ring->dma;
633fc9df2a0SAndre Guedes u32 buf_size;
63413b5b7fdSSasha Neftin
635fc9df2a0SAndre Guedes xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
636fc9df2a0SAndre Guedes ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
637fc9df2a0SAndre Guedes if (ring->xsk_pool) {
6384609ffb9SAndre Guedes WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
639fc9df2a0SAndre Guedes MEM_TYPE_XSK_BUFF_POOL,
640fc9df2a0SAndre Guedes NULL));
641fc9df2a0SAndre Guedes xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
642fc9df2a0SAndre Guedes } else {
643fc9df2a0SAndre Guedes WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
644fc9df2a0SAndre Guedes MEM_TYPE_PAGE_SHARED,
645fc9df2a0SAndre Guedes NULL));
646fc9df2a0SAndre Guedes }
6474609ffb9SAndre Guedes
64826575105SAndre Guedes if (igc_xdp_is_enabled(adapter))
64926575105SAndre Guedes set_ring_uses_large_buffer(ring);
65026575105SAndre Guedes
65113b5b7fdSSasha Neftin /* disable the queue */
65213b5b7fdSSasha Neftin wr32(IGC_RXDCTL(reg_idx), 0);
65313b5b7fdSSasha Neftin
65413b5b7fdSSasha Neftin /* Set DMA base address registers */
65513b5b7fdSSasha Neftin wr32(IGC_RDBAL(reg_idx),
65613b5b7fdSSasha Neftin rdba & 0x00000000ffffffffULL);
65713b5b7fdSSasha Neftin wr32(IGC_RDBAH(reg_idx), rdba >> 32);
65813b5b7fdSSasha Neftin wr32(IGC_RDLEN(reg_idx),
65913b5b7fdSSasha Neftin ring->count * sizeof(union igc_adv_rx_desc));
66013b5b7fdSSasha Neftin
66113b5b7fdSSasha Neftin /* initialize head and tail */
66213b5b7fdSSasha Neftin ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
66313b5b7fdSSasha Neftin wr32(IGC_RDH(reg_idx), 0);
66413b5b7fdSSasha Neftin writel(0, ring->tail);
66513b5b7fdSSasha Neftin
66613b5b7fdSSasha Neftin /* reset next-to- use/clean to place SW in sync with hardware */
66713b5b7fdSSasha Neftin ring->next_to_clean = 0;
66813b5b7fdSSasha Neftin ring->next_to_use = 0;
66913b5b7fdSSasha Neftin
670fc9df2a0SAndre Guedes if (ring->xsk_pool)
671fc9df2a0SAndre Guedes buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool);
672fc9df2a0SAndre Guedes else if (ring_uses_large_buffer(ring))
673fc9df2a0SAndre Guedes buf_size = IGC_RXBUFFER_3072;
67413b5b7fdSSasha Neftin else
675fc9df2a0SAndre Guedes buf_size = IGC_RXBUFFER_2048;
676fc9df2a0SAndre Guedes
6773ce29c17SSong Yoong Siang srrctl = rd32(IGC_SRRCTL(reg_idx));
6783ce29c17SSong Yoong Siang srrctl &= ~(IGC_SRRCTL_BSIZEPKT_MASK | IGC_SRRCTL_BSIZEHDR_MASK |
6793ce29c17SSong Yoong Siang IGC_SRRCTL_DESCTYPE_MASK);
6803ce29c17SSong Yoong Siang srrctl |= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN);
6813ce29c17SSong Yoong Siang srrctl |= IGC_SRRCTL_BSIZEPKT(buf_size);
68213b5b7fdSSasha Neftin srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
68313b5b7fdSSasha Neftin
68413b5b7fdSSasha Neftin wr32(IGC_SRRCTL(reg_idx), srrctl);
68513b5b7fdSSasha Neftin
68613b5b7fdSSasha Neftin rxdctl |= IGC_RX_PTHRESH;
68713b5b7fdSSasha Neftin rxdctl |= IGC_RX_HTHRESH << 8;
68813b5b7fdSSasha Neftin rxdctl |= IGC_RX_WTHRESH << 16;
68913b5b7fdSSasha Neftin
69013b5b7fdSSasha Neftin /* initialize rx_buffer_info */
69113b5b7fdSSasha Neftin memset(ring->rx_buffer_info, 0,
69213b5b7fdSSasha Neftin sizeof(struct igc_rx_buffer) * ring->count);
69313b5b7fdSSasha Neftin
69413b5b7fdSSasha Neftin /* initialize Rx descriptor 0 */
69513b5b7fdSSasha Neftin rx_desc = IGC_RX_DESC(ring, 0);
69613b5b7fdSSasha Neftin rx_desc->wb.upper.length = 0;
69713b5b7fdSSasha Neftin
69813b5b7fdSSasha Neftin /* enable receive descriptor fetching */
69913b5b7fdSSasha Neftin rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
70013b5b7fdSSasha Neftin
70113b5b7fdSSasha Neftin wr32(IGC_RXDCTL(reg_idx), rxdctl);
70213b5b7fdSSasha Neftin }
70313b5b7fdSSasha Neftin
70413b5b7fdSSasha Neftin /**
70513b5b7fdSSasha Neftin * igc_configure_rx - Configure receive Unit after Reset
70613b5b7fdSSasha Neftin * @adapter: board private structure
70713b5b7fdSSasha Neftin *
70813b5b7fdSSasha Neftin * Configure the Rx unit of the MAC after a reset.
70913b5b7fdSSasha Neftin */
igc_configure_rx(struct igc_adapter * adapter)71013b5b7fdSSasha Neftin static void igc_configure_rx(struct igc_adapter *adapter)
71113b5b7fdSSasha Neftin {
71213b5b7fdSSasha Neftin int i;
71313b5b7fdSSasha Neftin
71413b5b7fdSSasha Neftin /* Setup the HW Rx Head and Tail Descriptor Pointers and
71513b5b7fdSSasha Neftin * the Base and Length of the Rx Descriptor Ring
71613b5b7fdSSasha Neftin */
71713b5b7fdSSasha Neftin for (i = 0; i < adapter->num_rx_queues; i++)
71813b5b7fdSSasha Neftin igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
71913b5b7fdSSasha Neftin }
72013b5b7fdSSasha Neftin
72113b5b7fdSSasha Neftin /**
72213b5b7fdSSasha Neftin * igc_configure_tx_ring - Configure transmit ring after Reset
72313b5b7fdSSasha Neftin * @adapter: board private structure
72413b5b7fdSSasha Neftin * @ring: tx ring to configure
72513b5b7fdSSasha Neftin *
72613b5b7fdSSasha Neftin * Configure a transmit ring after a reset.
72713b5b7fdSSasha Neftin */
igc_configure_tx_ring(struct igc_adapter * adapter,struct igc_ring * ring)72813b5b7fdSSasha Neftin static void igc_configure_tx_ring(struct igc_adapter *adapter,
72913b5b7fdSSasha Neftin struct igc_ring *ring)
73013b5b7fdSSasha Neftin {
73113b5b7fdSSasha Neftin struct igc_hw *hw = &adapter->hw;
73213b5b7fdSSasha Neftin int reg_idx = ring->reg_idx;
73313b5b7fdSSasha Neftin u64 tdba = ring->dma;
73413b5b7fdSSasha Neftin u32 txdctl = 0;
73513b5b7fdSSasha Neftin
7369acf59a7SAndre Guedes ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
7379acf59a7SAndre Guedes
73813b5b7fdSSasha Neftin /* disable the queue */
73913b5b7fdSSasha Neftin wr32(IGC_TXDCTL(reg_idx), 0);
74013b5b7fdSSasha Neftin wrfl();
74113b5b7fdSSasha Neftin
74213b5b7fdSSasha Neftin wr32(IGC_TDLEN(reg_idx),
74313b5b7fdSSasha Neftin ring->count * sizeof(union igc_adv_tx_desc));
74413b5b7fdSSasha Neftin wr32(IGC_TDBAL(reg_idx),
74513b5b7fdSSasha Neftin tdba & 0x00000000ffffffffULL);
74613b5b7fdSSasha Neftin wr32(IGC_TDBAH(reg_idx), tdba >> 32);
74713b5b7fdSSasha Neftin
74813b5b7fdSSasha Neftin ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
74913b5b7fdSSasha Neftin wr32(IGC_TDH(reg_idx), 0);
75013b5b7fdSSasha Neftin writel(0, ring->tail);
75113b5b7fdSSasha Neftin
75213b5b7fdSSasha Neftin txdctl |= IGC_TX_PTHRESH;
75313b5b7fdSSasha Neftin txdctl |= IGC_TX_HTHRESH << 8;
75413b5b7fdSSasha Neftin txdctl |= IGC_TX_WTHRESH << 16;
75513b5b7fdSSasha Neftin
75613b5b7fdSSasha Neftin txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
75713b5b7fdSSasha Neftin wr32(IGC_TXDCTL(reg_idx), txdctl);
75813b5b7fdSSasha Neftin }
75913b5b7fdSSasha Neftin
76013b5b7fdSSasha Neftin /**
76113b5b7fdSSasha Neftin * igc_configure_tx - Configure transmit Unit after Reset
76213b5b7fdSSasha Neftin * @adapter: board private structure
76313b5b7fdSSasha Neftin *
76413b5b7fdSSasha Neftin * Configure the Tx unit of the MAC after a reset.
76513b5b7fdSSasha Neftin */
igc_configure_tx(struct igc_adapter * adapter)76613b5b7fdSSasha Neftin static void igc_configure_tx(struct igc_adapter *adapter)
76713b5b7fdSSasha Neftin {
76813b5b7fdSSasha Neftin int i;
76913b5b7fdSSasha Neftin
77013b5b7fdSSasha Neftin for (i = 0; i < adapter->num_tx_queues; i++)
77113b5b7fdSSasha Neftin igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
77213b5b7fdSSasha Neftin }
77313b5b7fdSSasha Neftin
77413b5b7fdSSasha Neftin /**
77513b5b7fdSSasha Neftin * igc_setup_mrqc - configure the multiple receive queue control registers
77613b5b7fdSSasha Neftin * @adapter: Board private structure
77713b5b7fdSSasha Neftin */
igc_setup_mrqc(struct igc_adapter * adapter)77813b5b7fdSSasha Neftin static void igc_setup_mrqc(struct igc_adapter *adapter)
77913b5b7fdSSasha Neftin {
7802121c271SSasha Neftin struct igc_hw *hw = &adapter->hw;
7812121c271SSasha Neftin u32 j, num_rx_queues;
7822121c271SSasha Neftin u32 mrqc, rxcsum;
7832121c271SSasha Neftin u32 rss_key[10];
7842121c271SSasha Neftin
7852121c271SSasha Neftin netdev_rss_key_fill(rss_key, sizeof(rss_key));
7862121c271SSasha Neftin for (j = 0; j < 10; j++)
7872121c271SSasha Neftin wr32(IGC_RSSRK(j), rss_key[j]);
7882121c271SSasha Neftin
7892121c271SSasha Neftin num_rx_queues = adapter->rss_queues;
7902121c271SSasha Neftin
7912121c271SSasha Neftin if (adapter->rss_indir_tbl_init != num_rx_queues) {
7922121c271SSasha Neftin for (j = 0; j < IGC_RETA_SIZE; j++)
7932121c271SSasha Neftin adapter->rss_indir_tbl[j] =
7942121c271SSasha Neftin (j * num_rx_queues) / IGC_RETA_SIZE;
7952121c271SSasha Neftin adapter->rss_indir_tbl_init = num_rx_queues;
7962121c271SSasha Neftin }
7972121c271SSasha Neftin igc_write_rss_indir_tbl(adapter);
7982121c271SSasha Neftin
7992121c271SSasha Neftin /* Disable raw packet checksumming so that RSS hash is placed in
8002121c271SSasha Neftin * descriptor on writeback. No need to enable TCP/UDP/IP checksum
8012121c271SSasha Neftin * offloads as they are enabled by default
8022121c271SSasha Neftin */
8032121c271SSasha Neftin rxcsum = rd32(IGC_RXCSUM);
8042121c271SSasha Neftin rxcsum |= IGC_RXCSUM_PCSD;
8052121c271SSasha Neftin
8062121c271SSasha Neftin /* Enable Receive Checksum Offload for SCTP */
8072121c271SSasha Neftin rxcsum |= IGC_RXCSUM_CRCOFL;
8082121c271SSasha Neftin
8092121c271SSasha Neftin /* Don't need to set TUOFL or IPOFL, they default to 1 */
8102121c271SSasha Neftin wr32(IGC_RXCSUM, rxcsum);
8112121c271SSasha Neftin
8122121c271SSasha Neftin /* Generate RSS hash based on packet types, TCP/UDP
8132121c271SSasha Neftin * port numbers and/or IPv4/v6 src and dst addresses
8142121c271SSasha Neftin */
8152121c271SSasha Neftin mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
8162121c271SSasha Neftin IGC_MRQC_RSS_FIELD_IPV4_TCP |
8172121c271SSasha Neftin IGC_MRQC_RSS_FIELD_IPV6 |
8182121c271SSasha Neftin IGC_MRQC_RSS_FIELD_IPV6_TCP |
8192121c271SSasha Neftin IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
8202121c271SSasha Neftin
8212121c271SSasha Neftin if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
8222121c271SSasha Neftin mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
8232121c271SSasha Neftin if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
8242121c271SSasha Neftin mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
8252121c271SSasha Neftin
8262121c271SSasha Neftin mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
8272121c271SSasha Neftin
8282121c271SSasha Neftin wr32(IGC_MRQC, mrqc);
82913b5b7fdSSasha Neftin }
83013b5b7fdSSasha Neftin
83113b5b7fdSSasha Neftin /**
83213b5b7fdSSasha Neftin * igc_setup_rctl - configure the receive control registers
83313b5b7fdSSasha Neftin * @adapter: Board private structure
83413b5b7fdSSasha Neftin */
igc_setup_rctl(struct igc_adapter * adapter)83513b5b7fdSSasha Neftin static void igc_setup_rctl(struct igc_adapter *adapter)
83613b5b7fdSSasha Neftin {
83713b5b7fdSSasha Neftin struct igc_hw *hw = &adapter->hw;
83813b5b7fdSSasha Neftin u32 rctl;
83913b5b7fdSSasha Neftin
84013b5b7fdSSasha Neftin rctl = rd32(IGC_RCTL);
84113b5b7fdSSasha Neftin
84213b5b7fdSSasha Neftin rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
84313b5b7fdSSasha Neftin rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
84413b5b7fdSSasha Neftin
84513b5b7fdSSasha Neftin rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
84613b5b7fdSSasha Neftin (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
84713b5b7fdSSasha Neftin
84813b5b7fdSSasha Neftin /* enable stripping of CRC. Newer features require
84913b5b7fdSSasha Neftin * that the HW strips the CRC.
85013b5b7fdSSasha Neftin */
85113b5b7fdSSasha Neftin rctl |= IGC_RCTL_SECRC;
85213b5b7fdSSasha Neftin
85313b5b7fdSSasha Neftin /* disable store bad packets and clear size bits. */
85413b5b7fdSSasha Neftin rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
85513b5b7fdSSasha Neftin
85613b5b7fdSSasha Neftin /* enable LPE to allow for reception of jumbo frames */
85713b5b7fdSSasha Neftin rctl |= IGC_RCTL_LPE;
85813b5b7fdSSasha Neftin
85913b5b7fdSSasha Neftin /* disable queue 0 to prevent tail write w/o re-config */
86013b5b7fdSSasha Neftin wr32(IGC_RXDCTL(0), 0);
86113b5b7fdSSasha Neftin
86213b5b7fdSSasha Neftin /* This is useful for sniffing bad packets. */
86313b5b7fdSSasha Neftin if (adapter->netdev->features & NETIF_F_RXALL) {
86413b5b7fdSSasha Neftin /* UPE and MPE will be handled by normal PROMISC logic
86513b5b7fdSSasha Neftin * in set_rx_mode
86613b5b7fdSSasha Neftin */
86713b5b7fdSSasha Neftin rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
86813b5b7fdSSasha Neftin IGC_RCTL_BAM | /* RX All Bcast Pkts */
86913b5b7fdSSasha Neftin IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
87013b5b7fdSSasha Neftin
87113b5b7fdSSasha Neftin rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
87213b5b7fdSSasha Neftin IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
87313b5b7fdSSasha Neftin }
87413b5b7fdSSasha Neftin
87513b5b7fdSSasha Neftin wr32(IGC_RCTL, rctl);
87613b5b7fdSSasha Neftin }
87713b5b7fdSSasha Neftin
87813b5b7fdSSasha Neftin /**
87913b5b7fdSSasha Neftin * igc_setup_tctl - configure the transmit control registers
88013b5b7fdSSasha Neftin * @adapter: Board private structure
88113b5b7fdSSasha Neftin */
igc_setup_tctl(struct igc_adapter * adapter)88213b5b7fdSSasha Neftin static void igc_setup_tctl(struct igc_adapter *adapter)
88313b5b7fdSSasha Neftin {
88413b5b7fdSSasha Neftin struct igc_hw *hw = &adapter->hw;
88513b5b7fdSSasha Neftin u32 tctl;
88613b5b7fdSSasha Neftin
88713b5b7fdSSasha Neftin /* disable queue 0 which icould be enabled by default */
88813b5b7fdSSasha Neftin wr32(IGC_TXDCTL(0), 0);
88913b5b7fdSSasha Neftin
89013b5b7fdSSasha Neftin /* Program the Transmit Control Register */
89113b5b7fdSSasha Neftin tctl = rd32(IGC_TCTL);
89213b5b7fdSSasha Neftin tctl &= ~IGC_TCTL_CT;
89313b5b7fdSSasha Neftin tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
89413b5b7fdSSasha Neftin (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
89513b5b7fdSSasha Neftin
89613b5b7fdSSasha Neftin /* Enable transmits */
89713b5b7fdSSasha Neftin tctl |= IGC_TCTL_EN;
89813b5b7fdSSasha Neftin
89913b5b7fdSSasha Neftin wr32(IGC_TCTL, tctl);
90013b5b7fdSSasha Neftin }
90113b5b7fdSSasha Neftin
90213b5b7fdSSasha Neftin /**
903424045beSAndre Guedes * igc_set_mac_filter_hw() - Set MAC address filter in hardware
904424045beSAndre Guedes * @adapter: Pointer to adapter where the filter should be set
905424045beSAndre Guedes * @index: Filter index
906750433d0SAndre Guedes * @type: MAC address filter type (source or destination)
907750433d0SAndre Guedes * @addr: MAC address
908424045beSAndre Guedes * @queue: If non-negative, queue assignment feature is enabled and frames
909424045beSAndre Guedes * matching the filter are enqueued onto 'queue'. Otherwise, queue
910424045beSAndre Guedes * assignment is disabled.
9113988d8bfSSasha Neftin */
igc_set_mac_filter_hw(struct igc_adapter * adapter,int index,enum igc_mac_filter_type type,const u8 * addr,int queue)912424045beSAndre Guedes static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
913750433d0SAndre Guedes enum igc_mac_filter_type type,
914424045beSAndre Guedes const u8 *addr, int queue)
9153988d8bfSSasha Neftin {
916949b922eSAndre Guedes struct net_device *dev = adapter->netdev;
9173988d8bfSSasha Neftin struct igc_hw *hw = &adapter->hw;
918424045beSAndre Guedes u32 ral, rah;
9193988d8bfSSasha Neftin
920424045beSAndre Guedes if (WARN_ON(index >= hw->mac.rar_entry_count))
921424045beSAndre Guedes return;
9223988d8bfSSasha Neftin
923424045beSAndre Guedes ral = le32_to_cpup((__le32 *)(addr));
924424045beSAndre Guedes rah = le16_to_cpup((__le16 *)(addr + 4));
9253988d8bfSSasha Neftin
926750433d0SAndre Guedes if (type == IGC_MAC_FILTER_TYPE_SRC) {
927750433d0SAndre Guedes rah &= ~IGC_RAH_ASEL_MASK;
928750433d0SAndre Guedes rah |= IGC_RAH_ASEL_SRC_ADDR;
9293988d8bfSSasha Neftin }
9303988d8bfSSasha Neftin
931424045beSAndre Guedes if (queue >= 0) {
932424045beSAndre Guedes rah &= ~IGC_RAH_QSEL_MASK;
933424045beSAndre Guedes rah |= (queue << IGC_RAH_QSEL_SHIFT);
934424045beSAndre Guedes rah |= IGC_RAH_QSEL_ENABLE;
935424045beSAndre Guedes }
936424045beSAndre Guedes
937424045beSAndre Guedes rah |= IGC_RAH_AV;
938424045beSAndre Guedes
939424045beSAndre Guedes wr32(IGC_RAL(index), ral);
940424045beSAndre Guedes wr32(IGC_RAH(index), rah);
941949b922eSAndre Guedes
942949b922eSAndre Guedes netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
943424045beSAndre Guedes }
944424045beSAndre Guedes
945424045beSAndre Guedes /**
946424045beSAndre Guedes * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
947424045beSAndre Guedes * @adapter: Pointer to adapter where the filter should be cleared
948424045beSAndre Guedes * @index: Filter index
9493988d8bfSSasha Neftin */
igc_clear_mac_filter_hw(struct igc_adapter * adapter,int index)950424045beSAndre Guedes static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
951424045beSAndre Guedes {
952949b922eSAndre Guedes struct net_device *dev = adapter->netdev;
953424045beSAndre Guedes struct igc_hw *hw = &adapter->hw;
9543988d8bfSSasha Neftin
955424045beSAndre Guedes if (WARN_ON(index >= hw->mac.rar_entry_count))
956424045beSAndre Guedes return;
95727945ebeSAndre Guedes
958424045beSAndre Guedes wr32(IGC_RAL(index), 0);
959424045beSAndre Guedes wr32(IGC_RAH(index), 0);
960949b922eSAndre Guedes
961949b922eSAndre Guedes netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
9623988d8bfSSasha Neftin }
9633988d8bfSSasha Neftin
9643988d8bfSSasha Neftin /* Set default MAC address for the PF in the first RAR entry */
igc_set_default_mac_filter(struct igc_adapter * adapter)9653988d8bfSSasha Neftin static void igc_set_default_mac_filter(struct igc_adapter *adapter)
9663988d8bfSSasha Neftin {
967949b922eSAndre Guedes struct net_device *dev = adapter->netdev;
968949b922eSAndre Guedes u8 *addr = adapter->hw.mac.addr;
9693988d8bfSSasha Neftin
970949b922eSAndre Guedes netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
9713988d8bfSSasha Neftin
972750433d0SAndre Guedes igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
9733988d8bfSSasha Neftin }
9743988d8bfSSasha Neftin
9753988d8bfSSasha Neftin /**
976c9a11c23SSasha Neftin * igc_set_mac - Change the Ethernet Address of the NIC
977c9a11c23SSasha Neftin * @netdev: network interface device structure
978c9a11c23SSasha Neftin * @p: pointer to an address structure
979c9a11c23SSasha Neftin *
980c9a11c23SSasha Neftin * Returns 0 on success, negative on failure
981c9a11c23SSasha Neftin */
igc_set_mac(struct net_device * netdev,void * p)982c9a11c23SSasha Neftin static int igc_set_mac(struct net_device *netdev, void *p)
983c9a11c23SSasha Neftin {
984c9a11c23SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
985c9a11c23SSasha Neftin struct igc_hw *hw = &adapter->hw;
986c9a11c23SSasha Neftin struct sockaddr *addr = p;
987c9a11c23SSasha Neftin
988c9a11c23SSasha Neftin if (!is_valid_ether_addr(addr->sa_data))
989c9a11c23SSasha Neftin return -EADDRNOTAVAIL;
990c9a11c23SSasha Neftin
991a05e4c0aSJakub Kicinski eth_hw_addr_set(netdev, addr->sa_data);
992c9a11c23SSasha Neftin memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
993c9a11c23SSasha Neftin
994c9a11c23SSasha Neftin /* set the correct pool for the new PF MAC address in entry 0 */
995c9a11c23SSasha Neftin igc_set_default_mac_filter(adapter);
996c9a11c23SSasha Neftin
997c9a11c23SSasha Neftin return 0;
998c9a11c23SSasha Neftin }
999c9a11c23SSasha Neftin
10007f839684SSasha Neftin /**
10017f839684SSasha Neftin * igc_write_mc_addr_list - write multicast addresses to MTA
10027f839684SSasha Neftin * @netdev: network interface device structure
10037f839684SSasha Neftin *
10047f839684SSasha Neftin * Writes multicast address list to the MTA hash table.
10057f839684SSasha Neftin * Returns: -ENOMEM on failure
10067f839684SSasha Neftin * 0 on no addresses written
10077f839684SSasha Neftin * X on writing X addresses to MTA
10087f839684SSasha Neftin **/
igc_write_mc_addr_list(struct net_device * netdev)10097f839684SSasha Neftin static int igc_write_mc_addr_list(struct net_device *netdev)
10107f839684SSasha Neftin {
10117f839684SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
10127f839684SSasha Neftin struct igc_hw *hw = &adapter->hw;
10137f839684SSasha Neftin struct netdev_hw_addr *ha;
10147f839684SSasha Neftin u8 *mta_list;
10157f839684SSasha Neftin int i;
10167f839684SSasha Neftin
10177f839684SSasha Neftin if (netdev_mc_empty(netdev)) {
10187f839684SSasha Neftin /* nothing to program, so clear mc list */
10197f839684SSasha Neftin igc_update_mc_addr_list(hw, NULL, 0);
10207f839684SSasha Neftin return 0;
10217f839684SSasha Neftin }
10227f839684SSasha Neftin
10237f839684SSasha Neftin mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
10247f839684SSasha Neftin if (!mta_list)
10257f839684SSasha Neftin return -ENOMEM;
10267f839684SSasha Neftin
10277f839684SSasha Neftin /* The shared function expects a packed array of only addresses. */
10287f839684SSasha Neftin i = 0;
10297f839684SSasha Neftin netdev_for_each_mc_addr(ha, netdev)
10307f839684SSasha Neftin memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
10317f839684SSasha Neftin
10327f839684SSasha Neftin igc_update_mc_addr_list(hw, mta_list, i);
10337f839684SSasha Neftin kfree(mta_list);
10347f839684SSasha Neftin
10357f839684SSasha Neftin return netdev_mc_count(netdev);
10367f839684SSasha Neftin }
10377f839684SSasha Neftin
igc_tx_launchtime(struct igc_ring * ring,ktime_t txtime,bool * first_flag,bool * insert_empty)1038db0b124fSVinicius Costa Gomes static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime,
1039db0b124fSVinicius Costa Gomes bool *first_flag, bool *insert_empty)
104082faa9b7SVinicius Costa Gomes {
1041db0b124fSVinicius Costa Gomes struct igc_adapter *adapter = netdev_priv(ring->netdev);
104282faa9b7SVinicius Costa Gomes ktime_t cycle_time = adapter->cycle_time;
104382faa9b7SVinicius Costa Gomes ktime_t base_time = adapter->base_time;
1044db0b124fSVinicius Costa Gomes ktime_t now = ktime_get_clocktai();
1045db0b124fSVinicius Costa Gomes ktime_t baset_est, end_of_cycle;
1046c1bca9acSFlorian Kauer s32 launchtime;
1047db0b124fSVinicius Costa Gomes s64 n;
104882faa9b7SVinicius Costa Gomes
1049db0b124fSVinicius Costa Gomes n = div64_s64(ktime_sub_ns(now, base_time), cycle_time);
1050db0b124fSVinicius Costa Gomes
1051db0b124fSVinicius Costa Gomes baset_est = ktime_add_ns(base_time, cycle_time * (n));
1052db0b124fSVinicius Costa Gomes end_of_cycle = ktime_add_ns(baset_est, cycle_time);
1053db0b124fSVinicius Costa Gomes
1054db0b124fSVinicius Costa Gomes if (ktime_compare(txtime, end_of_cycle) >= 0) {
1055db0b124fSVinicius Costa Gomes if (baset_est != ring->last_ff_cycle) {
1056db0b124fSVinicius Costa Gomes *first_flag = true;
1057db0b124fSVinicius Costa Gomes ring->last_ff_cycle = baset_est;
1058db0b124fSVinicius Costa Gomes
10590bcc6285SFlorian Kauer if (ktime_compare(end_of_cycle, ring->last_tx_cycle) > 0)
1060db0b124fSVinicius Costa Gomes *insert_empty = true;
1061db0b124fSVinicius Costa Gomes }
1062db0b124fSVinicius Costa Gomes }
1063db0b124fSVinicius Costa Gomes
1064db0b124fSVinicius Costa Gomes /* Introducing a window at end of cycle on which packets
1065db0b124fSVinicius Costa Gomes * potentially not honor launchtime. Window of 5us chosen
1066db0b124fSVinicius Costa Gomes * considering software update the tail pointer and packets
1067db0b124fSVinicius Costa Gomes * are dma'ed to packet buffer.
106882faa9b7SVinicius Costa Gomes */
1069db0b124fSVinicius Costa Gomes if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC))
1070db0b124fSVinicius Costa Gomes netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n",
1071db0b124fSVinicius Costa Gomes txtime);
1072db0b124fSVinicius Costa Gomes
1073db0b124fSVinicius Costa Gomes ring->last_tx_cycle = end_of_cycle;
1074db0b124fSVinicius Costa Gomes
1075db0b124fSVinicius Costa Gomes launchtime = ktime_sub_ns(txtime, baset_est);
1076db0b124fSVinicius Costa Gomes if (launchtime > 0)
1077db0b124fSVinicius Costa Gomes div_s64_rem(launchtime, cycle_time, &launchtime);
1078db0b124fSVinicius Costa Gomes else
1079db0b124fSVinicius Costa Gomes launchtime = 0;
108082faa9b7SVinicius Costa Gomes
108182faa9b7SVinicius Costa Gomes return cpu_to_le32(launchtime);
108282faa9b7SVinicius Costa Gomes }
108382faa9b7SVinicius Costa Gomes
igc_init_empty_frame(struct igc_ring * ring,struct igc_tx_buffer * buffer,struct sk_buff * skb)1084db0b124fSVinicius Costa Gomes static int igc_init_empty_frame(struct igc_ring *ring,
1085db0b124fSVinicius Costa Gomes struct igc_tx_buffer *buffer,
1086db0b124fSVinicius Costa Gomes struct sk_buff *skb)
1087db0b124fSVinicius Costa Gomes {
1088db0b124fSVinicius Costa Gomes unsigned int size;
1089db0b124fSVinicius Costa Gomes dma_addr_t dma;
1090db0b124fSVinicius Costa Gomes
1091db0b124fSVinicius Costa Gomes size = skb_headlen(skb);
1092db0b124fSVinicius Costa Gomes
1093db0b124fSVinicius Costa Gomes dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE);
1094db0b124fSVinicius Costa Gomes if (dma_mapping_error(ring->dev, dma)) {
1095db0b124fSVinicius Costa Gomes netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
1096db0b124fSVinicius Costa Gomes return -ENOMEM;
1097db0b124fSVinicius Costa Gomes }
1098db0b124fSVinicius Costa Gomes
1099db0b124fSVinicius Costa Gomes buffer->skb = skb;
1100db0b124fSVinicius Costa Gomes buffer->protocol = 0;
1101db0b124fSVinicius Costa Gomes buffer->bytecount = skb->len;
1102db0b124fSVinicius Costa Gomes buffer->gso_segs = 1;
1103db0b124fSVinicius Costa Gomes buffer->time_stamp = jiffies;
1104db0b124fSVinicius Costa Gomes dma_unmap_len_set(buffer, len, skb->len);
1105db0b124fSVinicius Costa Gomes dma_unmap_addr_set(buffer, dma, dma);
1106db0b124fSVinicius Costa Gomes
1107db0b124fSVinicius Costa Gomes return 0;
1108db0b124fSVinicius Costa Gomes }
1109db0b124fSVinicius Costa Gomes
igc_init_tx_empty_descriptor(struct igc_ring * ring,struct sk_buff * skb,struct igc_tx_buffer * first)1110db0b124fSVinicius Costa Gomes static int igc_init_tx_empty_descriptor(struct igc_ring *ring,
1111db0b124fSVinicius Costa Gomes struct sk_buff *skb,
1112db0b124fSVinicius Costa Gomes struct igc_tx_buffer *first)
1113db0b124fSVinicius Costa Gomes {
1114db0b124fSVinicius Costa Gomes union igc_adv_tx_desc *desc;
1115db0b124fSVinicius Costa Gomes u32 cmd_type, olinfo_status;
1116db0b124fSVinicius Costa Gomes int err;
1117db0b124fSVinicius Costa Gomes
1118db0b124fSVinicius Costa Gomes if (!igc_desc_unused(ring))
1119db0b124fSVinicius Costa Gomes return -EBUSY;
1120db0b124fSVinicius Costa Gomes
1121db0b124fSVinicius Costa Gomes err = igc_init_empty_frame(ring, first, skb);
1122db0b124fSVinicius Costa Gomes if (err)
1123db0b124fSVinicius Costa Gomes return err;
1124db0b124fSVinicius Costa Gomes
1125db0b124fSVinicius Costa Gomes cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
1126db0b124fSVinicius Costa Gomes IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
1127db0b124fSVinicius Costa Gomes first->bytecount;
1128db0b124fSVinicius Costa Gomes olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
1129db0b124fSVinicius Costa Gomes
1130db0b124fSVinicius Costa Gomes desc = IGC_TX_DESC(ring, ring->next_to_use);
1131db0b124fSVinicius Costa Gomes desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1132db0b124fSVinicius Costa Gomes desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1133db0b124fSVinicius Costa Gomes desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma));
1134db0b124fSVinicius Costa Gomes
1135db0b124fSVinicius Costa Gomes netdev_tx_sent_queue(txring_txq(ring), skb->len);
1136db0b124fSVinicius Costa Gomes
1137db0b124fSVinicius Costa Gomes first->next_to_watch = desc;
1138db0b124fSVinicius Costa Gomes
1139db0b124fSVinicius Costa Gomes ring->next_to_use++;
1140db0b124fSVinicius Costa Gomes if (ring->next_to_use == ring->count)
1141db0b124fSVinicius Costa Gomes ring->next_to_use = 0;
1142db0b124fSVinicius Costa Gomes
1143db0b124fSVinicius Costa Gomes return 0;
1144db0b124fSVinicius Costa Gomes }
1145db0b124fSVinicius Costa Gomes
1146db0b124fSVinicius Costa Gomes #define IGC_EMPTY_FRAME_SIZE 60
1147db0b124fSVinicius Costa Gomes
igc_tx_ctxtdesc(struct igc_ring * tx_ring,__le32 launch_time,bool first_flag,u32 vlan_macip_lens,u32 type_tucmd,u32 mss_l4len_idx)1148d3ae3cfbSSasha Neftin static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1149db0b124fSVinicius Costa Gomes __le32 launch_time, bool first_flag,
1150d3ae3cfbSSasha Neftin u32 vlan_macip_lens, u32 type_tucmd,
1151d3ae3cfbSSasha Neftin u32 mss_l4len_idx)
1152d3ae3cfbSSasha Neftin {
1153d3ae3cfbSSasha Neftin struct igc_adv_tx_context_desc *context_desc;
1154d3ae3cfbSSasha Neftin u16 i = tx_ring->next_to_use;
1155d3ae3cfbSSasha Neftin
1156d3ae3cfbSSasha Neftin context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1157d3ae3cfbSSasha Neftin
1158d3ae3cfbSSasha Neftin i++;
1159d3ae3cfbSSasha Neftin tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1160d3ae3cfbSSasha Neftin
1161d3ae3cfbSSasha Neftin /* set bits to identify this as an advanced context descriptor */
1162d3ae3cfbSSasha Neftin type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1163d3ae3cfbSSasha Neftin
116493d85dc5SSasha Neftin /* For i225, context index must be unique per ring. */
1165d3ae3cfbSSasha Neftin if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1166d3ae3cfbSSasha Neftin mss_l4len_idx |= tx_ring->reg_idx << 4;
1167d3ae3cfbSSasha Neftin
1168db0b124fSVinicius Costa Gomes if (first_flag)
1169db0b124fSVinicius Costa Gomes mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST;
1170db0b124fSVinicius Costa Gomes
1171d3ae3cfbSSasha Neftin context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1172d3ae3cfbSSasha Neftin context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1173d3ae3cfbSSasha Neftin context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1174db0b124fSVinicius Costa Gomes context_desc->launch_time = launch_time;
1175d3ae3cfbSSasha Neftin }
1176d3ae3cfbSSasha Neftin
igc_tx_csum(struct igc_ring * tx_ring,struct igc_tx_buffer * first,__le32 launch_time,bool first_flag)1177db0b124fSVinicius Costa Gomes static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first,
1178db0b124fSVinicius Costa Gomes __le32 launch_time, bool first_flag)
1179c9a11c23SSasha Neftin {
1180d3ae3cfbSSasha Neftin struct sk_buff *skb = first->skb;
1181d3ae3cfbSSasha Neftin u32 vlan_macip_lens = 0;
1182d3ae3cfbSSasha Neftin u32 type_tucmd = 0;
1183d3ae3cfbSSasha Neftin
1184d3ae3cfbSSasha Neftin if (skb->ip_summed != CHECKSUM_PARTIAL) {
1185d3ae3cfbSSasha Neftin csum_failed:
1186d3ae3cfbSSasha Neftin if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1187d3ae3cfbSSasha Neftin !tx_ring->launchtime_enable)
1188d3ae3cfbSSasha Neftin return;
1189d3ae3cfbSSasha Neftin goto no_csum;
1190d3ae3cfbSSasha Neftin }
1191d3ae3cfbSSasha Neftin
1192d3ae3cfbSSasha Neftin switch (skb->csum_offset) {
1193d3ae3cfbSSasha Neftin case offsetof(struct tcphdr, check):
1194d3ae3cfbSSasha Neftin type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
11955463fce6SJeff Kirsher fallthrough;
1196d3ae3cfbSSasha Neftin case offsetof(struct udphdr, check):
1197d3ae3cfbSSasha Neftin break;
1198d3ae3cfbSSasha Neftin case offsetof(struct sctphdr, checksum):
1199d3ae3cfbSSasha Neftin /* validate that this is actually an SCTP request */
1200609d29a9SXin Long if (skb_csum_is_sctp(skb)) {
1201d3ae3cfbSSasha Neftin type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1202d3ae3cfbSSasha Neftin break;
1203d3ae3cfbSSasha Neftin }
12045463fce6SJeff Kirsher fallthrough;
1205d3ae3cfbSSasha Neftin default:
1206d3ae3cfbSSasha Neftin skb_checksum_help(skb);
1207d3ae3cfbSSasha Neftin goto csum_failed;
1208d3ae3cfbSSasha Neftin }
1209d3ae3cfbSSasha Neftin
1210d3ae3cfbSSasha Neftin /* update TX checksum flag */
1211d3ae3cfbSSasha Neftin first->tx_flags |= IGC_TX_FLAGS_CSUM;
1212d3ae3cfbSSasha Neftin vlan_macip_lens = skb_checksum_start_offset(skb) -
1213d3ae3cfbSSasha Neftin skb_network_offset(skb);
1214d3ae3cfbSSasha Neftin no_csum:
1215d3ae3cfbSSasha Neftin vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1216d3ae3cfbSSasha Neftin vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1217d3ae3cfbSSasha Neftin
1218db0b124fSVinicius Costa Gomes igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1219db0b124fSVinicius Costa Gomes vlan_macip_lens, type_tucmd, 0);
12200507ef8aSSasha Neftin }
12210507ef8aSSasha Neftin
__igc_maybe_stop_tx(struct igc_ring * tx_ring,const u16 size)12220507ef8aSSasha Neftin static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
12230507ef8aSSasha Neftin {
12240507ef8aSSasha Neftin struct net_device *netdev = tx_ring->netdev;
12250507ef8aSSasha Neftin
12260507ef8aSSasha Neftin netif_stop_subqueue(netdev, tx_ring->queue_index);
12270507ef8aSSasha Neftin
12280507ef8aSSasha Neftin /* memory barriier comment */
12290507ef8aSSasha Neftin smp_mb();
12300507ef8aSSasha Neftin
12310507ef8aSSasha Neftin /* We need to check again in a case another CPU has just
12320507ef8aSSasha Neftin * made room available.
12330507ef8aSSasha Neftin */
12340507ef8aSSasha Neftin if (igc_desc_unused(tx_ring) < size)
12350507ef8aSSasha Neftin return -EBUSY;
12360507ef8aSSasha Neftin
12370507ef8aSSasha Neftin /* A reprieve! */
12380507ef8aSSasha Neftin netif_wake_subqueue(netdev, tx_ring->queue_index);
12390507ef8aSSasha Neftin
12400507ef8aSSasha Neftin u64_stats_update_begin(&tx_ring->tx_syncp2);
12410507ef8aSSasha Neftin tx_ring->tx_stats.restart_queue2++;
12420507ef8aSSasha Neftin u64_stats_update_end(&tx_ring->tx_syncp2);
12430507ef8aSSasha Neftin
12440507ef8aSSasha Neftin return 0;
12450507ef8aSSasha Neftin }
12460507ef8aSSasha Neftin
igc_maybe_stop_tx(struct igc_ring * tx_ring,const u16 size)12470507ef8aSSasha Neftin static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
12480507ef8aSSasha Neftin {
12490507ef8aSSasha Neftin if (igc_desc_unused(tx_ring) >= size)
12500507ef8aSSasha Neftin return 0;
12510507ef8aSSasha Neftin return __igc_maybe_stop_tx(tx_ring, size);
12520507ef8aSSasha Neftin }
12530507ef8aSSasha Neftin
12542c344ae2SVinicius Costa Gomes #define IGC_SET_FLAG(_input, _flag, _result) \
12552c344ae2SVinicius Costa Gomes (((_flag) <= (_result)) ? \
12562c344ae2SVinicius Costa Gomes ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \
12572c344ae2SVinicius Costa Gomes ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
12582c344ae2SVinicius Costa Gomes
igc_tx_cmd_type(struct sk_buff * skb,u32 tx_flags)12598d744963SMuhammad Husaini Zulkifli static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
12600507ef8aSSasha Neftin {
12610507ef8aSSasha Neftin /* set type for advanced descriptor with frame checksum insertion */
12620507ef8aSSasha Neftin u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
12630507ef8aSSasha Neftin IGC_ADVTXD_DCMD_DEXT |
12640507ef8aSSasha Neftin IGC_ADVTXD_DCMD_IFCS;
12650507ef8aSSasha Neftin
12668d744963SMuhammad Husaini Zulkifli /* set HW vlan bit if vlan is present */
12678d744963SMuhammad Husaini Zulkifli cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN,
12688d744963SMuhammad Husaini Zulkifli IGC_ADVTXD_DCMD_VLE);
12698d744963SMuhammad Husaini Zulkifli
1270f38b782dSSasha Neftin /* set segmentation bits for TSO */
1271f38b782dSSasha Neftin cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1272f38b782dSSasha Neftin (IGC_ADVTXD_DCMD_TSE));
1273f38b782dSSasha Neftin
12743ed247e7SVinicius Costa Gomes /* set timestamp bit if present, will select the register set
12753ed247e7SVinicius Costa Gomes * based on the _TSTAMP(_X) bit.
12763ed247e7SVinicius Costa Gomes */
12772c344ae2SVinicius Costa Gomes cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
12782c344ae2SVinicius Costa Gomes (IGC_ADVTXD_MAC_TSTAMP));
12792c344ae2SVinicius Costa Gomes
12803ed247e7SVinicius Costa Gomes cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_1,
12813ed247e7SVinicius Costa Gomes (IGC_ADVTXD_TSTAMP_REG_1));
12823ed247e7SVinicius Costa Gomes
12833ed247e7SVinicius Costa Gomes cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_2,
12843ed247e7SVinicius Costa Gomes (IGC_ADVTXD_TSTAMP_REG_2));
12853ed247e7SVinicius Costa Gomes
12863ed247e7SVinicius Costa Gomes cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_3,
12873ed247e7SVinicius Costa Gomes (IGC_ADVTXD_TSTAMP_REG_3));
12883ed247e7SVinicius Costa Gomes
12898d744963SMuhammad Husaini Zulkifli /* insert frame checksum */
12908d744963SMuhammad Husaini Zulkifli cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS);
12918d744963SMuhammad Husaini Zulkifli
12920507ef8aSSasha Neftin return cmd_type;
12930507ef8aSSasha Neftin }
12940507ef8aSSasha Neftin
igc_tx_olinfo_status(struct igc_ring * tx_ring,union igc_adv_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)12950507ef8aSSasha Neftin static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
12960507ef8aSSasha Neftin union igc_adv_tx_desc *tx_desc,
12970507ef8aSSasha Neftin u32 tx_flags, unsigned int paylen)
12980507ef8aSSasha Neftin {
12990507ef8aSSasha Neftin u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
13000507ef8aSSasha Neftin
13010507ef8aSSasha Neftin /* insert L4 checksum */
13020507ef8aSSasha Neftin olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
13030507ef8aSSasha Neftin ((IGC_TXD_POPTS_TXSM << 8) /
13040507ef8aSSasha Neftin IGC_TX_FLAGS_CSUM);
13050507ef8aSSasha Neftin
13060507ef8aSSasha Neftin /* insert IPv4 checksum */
13070507ef8aSSasha Neftin olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
13080507ef8aSSasha Neftin (((IGC_TXD_POPTS_IXSM << 8)) /
13090507ef8aSSasha Neftin IGC_TX_FLAGS_IPV4);
13100507ef8aSSasha Neftin
13110507ef8aSSasha Neftin tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
13120507ef8aSSasha Neftin }
13130507ef8aSSasha Neftin
igc_tx_map(struct igc_ring * tx_ring,struct igc_tx_buffer * first,const u8 hdr_len)13140507ef8aSSasha Neftin static int igc_tx_map(struct igc_ring *tx_ring,
13150507ef8aSSasha Neftin struct igc_tx_buffer *first,
13160507ef8aSSasha Neftin const u8 hdr_len)
13170507ef8aSSasha Neftin {
13180507ef8aSSasha Neftin struct sk_buff *skb = first->skb;
13190507ef8aSSasha Neftin struct igc_tx_buffer *tx_buffer;
13200507ef8aSSasha Neftin union igc_adv_tx_desc *tx_desc;
13210507ef8aSSasha Neftin u32 tx_flags = first->tx_flags;
1322d7840976SMatthew Wilcox (Oracle) skb_frag_t *frag;
13230507ef8aSSasha Neftin u16 i = tx_ring->next_to_use;
13240507ef8aSSasha Neftin unsigned int data_len, size;
13250507ef8aSSasha Neftin dma_addr_t dma;
13268d744963SMuhammad Husaini Zulkifli u32 cmd_type;
13270507ef8aSSasha Neftin
13288d744963SMuhammad Husaini Zulkifli cmd_type = igc_tx_cmd_type(skb, tx_flags);
13290507ef8aSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, i);
13300507ef8aSSasha Neftin
13310507ef8aSSasha Neftin igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
13320507ef8aSSasha Neftin
13330507ef8aSSasha Neftin size = skb_headlen(skb);
13340507ef8aSSasha Neftin data_len = skb->data_len;
13350507ef8aSSasha Neftin
13360507ef8aSSasha Neftin dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
13370507ef8aSSasha Neftin
13380507ef8aSSasha Neftin tx_buffer = first;
13390507ef8aSSasha Neftin
13400507ef8aSSasha Neftin for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
13410507ef8aSSasha Neftin if (dma_mapping_error(tx_ring->dev, dma))
13420507ef8aSSasha Neftin goto dma_error;
13430507ef8aSSasha Neftin
13440507ef8aSSasha Neftin /* record length, and DMA address */
13450507ef8aSSasha Neftin dma_unmap_len_set(tx_buffer, len, size);
13460507ef8aSSasha Neftin dma_unmap_addr_set(tx_buffer, dma, dma);
13470507ef8aSSasha Neftin
13480507ef8aSSasha Neftin tx_desc->read.buffer_addr = cpu_to_le64(dma);
13490507ef8aSSasha Neftin
13500507ef8aSSasha Neftin while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
13510507ef8aSSasha Neftin tx_desc->read.cmd_type_len =
13520507ef8aSSasha Neftin cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
13530507ef8aSSasha Neftin
13540507ef8aSSasha Neftin i++;
13550507ef8aSSasha Neftin tx_desc++;
13560507ef8aSSasha Neftin if (i == tx_ring->count) {
13570507ef8aSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, 0);
13580507ef8aSSasha Neftin i = 0;
13590507ef8aSSasha Neftin }
13600507ef8aSSasha Neftin tx_desc->read.olinfo_status = 0;
13610507ef8aSSasha Neftin
13620507ef8aSSasha Neftin dma += IGC_MAX_DATA_PER_TXD;
13630507ef8aSSasha Neftin size -= IGC_MAX_DATA_PER_TXD;
13640507ef8aSSasha Neftin
13650507ef8aSSasha Neftin tx_desc->read.buffer_addr = cpu_to_le64(dma);
13660507ef8aSSasha Neftin }
13670507ef8aSSasha Neftin
13680507ef8aSSasha Neftin if (likely(!data_len))
13690507ef8aSSasha Neftin break;
13700507ef8aSSasha Neftin
13710507ef8aSSasha Neftin tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
13720507ef8aSSasha Neftin
13730507ef8aSSasha Neftin i++;
13740507ef8aSSasha Neftin tx_desc++;
13750507ef8aSSasha Neftin if (i == tx_ring->count) {
13760507ef8aSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, 0);
13770507ef8aSSasha Neftin i = 0;
13780507ef8aSSasha Neftin }
13790507ef8aSSasha Neftin tx_desc->read.olinfo_status = 0;
13800507ef8aSSasha Neftin
13810507ef8aSSasha Neftin size = skb_frag_size(frag);
13820507ef8aSSasha Neftin data_len -= size;
13830507ef8aSSasha Neftin
13840507ef8aSSasha Neftin dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
13850507ef8aSSasha Neftin size, DMA_TO_DEVICE);
13860507ef8aSSasha Neftin
13870507ef8aSSasha Neftin tx_buffer = &tx_ring->tx_buffer_info[i];
13880507ef8aSSasha Neftin }
13890507ef8aSSasha Neftin
13900507ef8aSSasha Neftin /* write last descriptor with RS and EOP bits */
13910507ef8aSSasha Neftin cmd_type |= size | IGC_TXD_DCMD;
13920507ef8aSSasha Neftin tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
13930507ef8aSSasha Neftin
13940507ef8aSSasha Neftin netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
13950507ef8aSSasha Neftin
13960507ef8aSSasha Neftin /* set the timestamp */
13970507ef8aSSasha Neftin first->time_stamp = jiffies;
13980507ef8aSSasha Neftin
1399a9e51058SJacob Keller skb_tx_timestamp(skb);
1400a9e51058SJacob Keller
14010507ef8aSSasha Neftin /* Force memory writes to complete before letting h/w know there
14020507ef8aSSasha Neftin * are new descriptors to fetch. (Only applicable for weak-ordered
14030507ef8aSSasha Neftin * memory model archs, such as IA-64).
14040507ef8aSSasha Neftin *
14050507ef8aSSasha Neftin * We also need this memory barrier to make certain all of the
14060507ef8aSSasha Neftin * status bits have been updated before next_to_watch is written.
14070507ef8aSSasha Neftin */
14080507ef8aSSasha Neftin wmb();
14090507ef8aSSasha Neftin
14100507ef8aSSasha Neftin /* set next_to_watch value indicating a packet is present */
14110507ef8aSSasha Neftin first->next_to_watch = tx_desc;
14120507ef8aSSasha Neftin
14130507ef8aSSasha Neftin i++;
14140507ef8aSSasha Neftin if (i == tx_ring->count)
14150507ef8aSSasha Neftin i = 0;
14160507ef8aSSasha Neftin
14170507ef8aSSasha Neftin tx_ring->next_to_use = i;
14180507ef8aSSasha Neftin
14190507ef8aSSasha Neftin /* Make sure there is space in the ring for the next send. */
14200507ef8aSSasha Neftin igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
14210507ef8aSSasha Neftin
14226b16f9eeSFlorian Westphal if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
14230507ef8aSSasha Neftin writel(i, tx_ring->tail);
14240507ef8aSSasha Neftin }
14250507ef8aSSasha Neftin
14260507ef8aSSasha Neftin return 0;
14270507ef8aSSasha Neftin dma_error:
142825f06effSAndre Guedes netdev_err(tx_ring->netdev, "TX DMA map failed\n");
14290507ef8aSSasha Neftin tx_buffer = &tx_ring->tx_buffer_info[i];
14300507ef8aSSasha Neftin
14310507ef8aSSasha Neftin /* clear dma mappings for failed tx_buffer_info map */
14320507ef8aSSasha Neftin while (tx_buffer != first) {
14330507ef8aSSasha Neftin if (dma_unmap_len(tx_buffer, len))
143461234295SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
14350507ef8aSSasha Neftin
14360507ef8aSSasha Neftin if (i-- == 0)
14370507ef8aSSasha Neftin i += tx_ring->count;
14380507ef8aSSasha Neftin tx_buffer = &tx_ring->tx_buffer_info[i];
14390507ef8aSSasha Neftin }
14400507ef8aSSasha Neftin
14410507ef8aSSasha Neftin if (dma_unmap_len(tx_buffer, len))
144261234295SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
14430507ef8aSSasha Neftin
14440507ef8aSSasha Neftin dev_kfree_skb_any(tx_buffer->skb);
14450507ef8aSSasha Neftin tx_buffer->skb = NULL;
14460507ef8aSSasha Neftin
14470507ef8aSSasha Neftin tx_ring->next_to_use = i;
14480507ef8aSSasha Neftin
14490507ef8aSSasha Neftin return -1;
14500507ef8aSSasha Neftin }
14510507ef8aSSasha Neftin
igc_tso(struct igc_ring * tx_ring,struct igc_tx_buffer * first,__le32 launch_time,bool first_flag,u8 * hdr_len)1452f38b782dSSasha Neftin static int igc_tso(struct igc_ring *tx_ring,
1453f38b782dSSasha Neftin struct igc_tx_buffer *first,
1454db0b124fSVinicius Costa Gomes __le32 launch_time, bool first_flag,
1455f38b782dSSasha Neftin u8 *hdr_len)
1456f38b782dSSasha Neftin {
1457f38b782dSSasha Neftin u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1458f38b782dSSasha Neftin struct sk_buff *skb = first->skb;
1459f38b782dSSasha Neftin union {
1460f38b782dSSasha Neftin struct iphdr *v4;
1461f38b782dSSasha Neftin struct ipv6hdr *v6;
1462f38b782dSSasha Neftin unsigned char *hdr;
1463f38b782dSSasha Neftin } ip;
1464f38b782dSSasha Neftin union {
1465f38b782dSSasha Neftin struct tcphdr *tcp;
1466f38b782dSSasha Neftin struct udphdr *udp;
1467f38b782dSSasha Neftin unsigned char *hdr;
1468f38b782dSSasha Neftin } l4;
1469f38b782dSSasha Neftin u32 paylen, l4_offset;
1470f38b782dSSasha Neftin int err;
1471f38b782dSSasha Neftin
1472f38b782dSSasha Neftin if (skb->ip_summed != CHECKSUM_PARTIAL)
1473f38b782dSSasha Neftin return 0;
1474f38b782dSSasha Neftin
1475f38b782dSSasha Neftin if (!skb_is_gso(skb))
1476f38b782dSSasha Neftin return 0;
1477f38b782dSSasha Neftin
1478f38b782dSSasha Neftin err = skb_cow_head(skb, 0);
1479f38b782dSSasha Neftin if (err < 0)
1480f38b782dSSasha Neftin return err;
1481f38b782dSSasha Neftin
1482f38b782dSSasha Neftin ip.hdr = skb_network_header(skb);
1483f38b782dSSasha Neftin l4.hdr = skb_checksum_start(skb);
1484f38b782dSSasha Neftin
1485f38b782dSSasha Neftin /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1486f38b782dSSasha Neftin type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1487f38b782dSSasha Neftin
1488f38b782dSSasha Neftin /* initialize outer IP header fields */
1489f38b782dSSasha Neftin if (ip.v4->version == 4) {
1490f38b782dSSasha Neftin unsigned char *csum_start = skb_checksum_start(skb);
1491f38b782dSSasha Neftin unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1492f38b782dSSasha Neftin
1493f38b782dSSasha Neftin /* IP header will have to cancel out any data that
1494f38b782dSSasha Neftin * is not a part of the outer IP header
1495f38b782dSSasha Neftin */
1496f38b782dSSasha Neftin ip.v4->check = csum_fold(csum_partial(trans_start,
1497f38b782dSSasha Neftin csum_start - trans_start,
1498f38b782dSSasha Neftin 0));
1499f38b782dSSasha Neftin type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1500f38b782dSSasha Neftin
1501f38b782dSSasha Neftin ip.v4->tot_len = 0;
1502f38b782dSSasha Neftin first->tx_flags |= IGC_TX_FLAGS_TSO |
1503f38b782dSSasha Neftin IGC_TX_FLAGS_CSUM |
1504f38b782dSSasha Neftin IGC_TX_FLAGS_IPV4;
1505f38b782dSSasha Neftin } else {
1506f38b782dSSasha Neftin ip.v6->payload_len = 0;
1507f38b782dSSasha Neftin first->tx_flags |= IGC_TX_FLAGS_TSO |
1508f38b782dSSasha Neftin IGC_TX_FLAGS_CSUM;
1509f38b782dSSasha Neftin }
1510f38b782dSSasha Neftin
1511f38b782dSSasha Neftin /* determine offset of inner transport header */
1512f38b782dSSasha Neftin l4_offset = l4.hdr - skb->data;
1513f38b782dSSasha Neftin
1514f38b782dSSasha Neftin /* remove payload length from inner checksum */
1515f38b782dSSasha Neftin paylen = skb->len - l4_offset;
1516f38b782dSSasha Neftin if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1517f38b782dSSasha Neftin /* compute length of segmentation header */
1518f38b782dSSasha Neftin *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1519f38b782dSSasha Neftin csum_replace_by_diff(&l4.tcp->check,
1520f38b782dSSasha Neftin (__force __wsum)htonl(paylen));
1521f38b782dSSasha Neftin } else {
1522f38b782dSSasha Neftin /* compute length of segmentation header */
1523f38b782dSSasha Neftin *hdr_len = sizeof(*l4.udp) + l4_offset;
1524f38b782dSSasha Neftin csum_replace_by_diff(&l4.udp->check,
1525f38b782dSSasha Neftin (__force __wsum)htonl(paylen));
1526f38b782dSSasha Neftin }
1527f38b782dSSasha Neftin
1528f38b782dSSasha Neftin /* update gso size and bytecount with header size */
1529f38b782dSSasha Neftin first->gso_segs = skb_shinfo(skb)->gso_segs;
1530f38b782dSSasha Neftin first->bytecount += (first->gso_segs - 1) * *hdr_len;
1531f38b782dSSasha Neftin
1532f38b782dSSasha Neftin /* MSS L4LEN IDX */
1533f38b782dSSasha Neftin mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1534f38b782dSSasha Neftin mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1535f38b782dSSasha Neftin
1536f38b782dSSasha Neftin /* VLAN MACLEN IPLEN */
1537f38b782dSSasha Neftin vlan_macip_lens = l4.hdr - ip.hdr;
1538f38b782dSSasha Neftin vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1539f38b782dSSasha Neftin vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1540f38b782dSSasha Neftin
1541db0b124fSVinicius Costa Gomes igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1542db0b124fSVinicius Costa Gomes vlan_macip_lens, type_tucmd, mss_l4len_idx);
1543f38b782dSSasha Neftin
1544f38b782dSSasha Neftin return 1;
1545f38b782dSSasha Neftin }
1546f38b782dSSasha Neftin
igc_request_tx_tstamp(struct igc_adapter * adapter,struct sk_buff * skb,u32 * flags)15473ed247e7SVinicius Costa Gomes static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags)
15483ed247e7SVinicius Costa Gomes {
15493ed247e7SVinicius Costa Gomes int i;
15503ed247e7SVinicius Costa Gomes
15513ed247e7SVinicius Costa Gomes for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
15523ed247e7SVinicius Costa Gomes struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
15533ed247e7SVinicius Costa Gomes
15543ed247e7SVinicius Costa Gomes if (tstamp->skb)
15553ed247e7SVinicius Costa Gomes continue;
15563ed247e7SVinicius Costa Gomes
15573ed247e7SVinicius Costa Gomes tstamp->skb = skb_get(skb);
15583ed247e7SVinicius Costa Gomes tstamp->start = jiffies;
15593ed247e7SVinicius Costa Gomes *flags = tstamp->flags;
15603ed247e7SVinicius Costa Gomes
15613ed247e7SVinicius Costa Gomes return true;
15623ed247e7SVinicius Costa Gomes }
15633ed247e7SVinicius Costa Gomes
15643ed247e7SVinicius Costa Gomes return false;
15653ed247e7SVinicius Costa Gomes }
15663ed247e7SVinicius Costa Gomes
igc_xmit_frame_ring(struct sk_buff * skb,struct igc_ring * tx_ring)15670507ef8aSSasha Neftin static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
15680507ef8aSSasha Neftin struct igc_ring *tx_ring)
15690507ef8aSSasha Neftin {
157092a0dcb8STan Tee Min struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1571db0b124fSVinicius Costa Gomes bool first_flag = false, insert_empty = false;
15720507ef8aSSasha Neftin u16 count = TXD_USE_COUNT(skb_headlen(skb));
15730507ef8aSSasha Neftin __be16 protocol = vlan_get_protocol(skb);
15740507ef8aSSasha Neftin struct igc_tx_buffer *first;
1575db0b124fSVinicius Costa Gomes __le32 launch_time = 0;
15760507ef8aSSasha Neftin u32 tx_flags = 0;
15770507ef8aSSasha Neftin unsigned short f;
1578db0b124fSVinicius Costa Gomes ktime_t txtime;
15790507ef8aSSasha Neftin u8 hdr_len = 0;
1580f38b782dSSasha Neftin int tso = 0;
15810507ef8aSSasha Neftin
15820507ef8aSSasha Neftin /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
15830507ef8aSSasha Neftin * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
15840507ef8aSSasha Neftin * + 2 desc gap to keep tail from touching head,
15850507ef8aSSasha Neftin * + 1 desc for context descriptor,
15860507ef8aSSasha Neftin * otherwise try next time
15870507ef8aSSasha Neftin */
15880507ef8aSSasha Neftin for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1589d7840976SMatthew Wilcox (Oracle) count += TXD_USE_COUNT(skb_frag_size(
1590d7840976SMatthew Wilcox (Oracle) &skb_shinfo(skb)->frags[f]));
15910507ef8aSSasha Neftin
1592db0b124fSVinicius Costa Gomes if (igc_maybe_stop_tx(tx_ring, count + 5)) {
15930507ef8aSSasha Neftin /* this is a hard error */
15940507ef8aSSasha Neftin return NETDEV_TX_BUSY;
15950507ef8aSSasha Neftin }
15960507ef8aSSasha Neftin
1597db0b124fSVinicius Costa Gomes if (!tx_ring->launchtime_enable)
1598db0b124fSVinicius Costa Gomes goto done;
1599db0b124fSVinicius Costa Gomes
1600db0b124fSVinicius Costa Gomes txtime = skb->tstamp;
1601db0b124fSVinicius Costa Gomes skb->tstamp = ktime_set(0, 0);
1602db0b124fSVinicius Costa Gomes launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty);
1603db0b124fSVinicius Costa Gomes
1604db0b124fSVinicius Costa Gomes if (insert_empty) {
1605db0b124fSVinicius Costa Gomes struct igc_tx_buffer *empty_info;
1606db0b124fSVinicius Costa Gomes struct sk_buff *empty;
1607db0b124fSVinicius Costa Gomes void *data;
1608db0b124fSVinicius Costa Gomes
1609db0b124fSVinicius Costa Gomes empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1610db0b124fSVinicius Costa Gomes empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC);
1611db0b124fSVinicius Costa Gomes if (!empty)
1612db0b124fSVinicius Costa Gomes goto done;
1613db0b124fSVinicius Costa Gomes
1614db0b124fSVinicius Costa Gomes data = skb_put(empty, IGC_EMPTY_FRAME_SIZE);
1615db0b124fSVinicius Costa Gomes memset(data, 0, IGC_EMPTY_FRAME_SIZE);
1616db0b124fSVinicius Costa Gomes
1617db0b124fSVinicius Costa Gomes igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0);
1618db0b124fSVinicius Costa Gomes
1619db0b124fSVinicius Costa Gomes if (igc_init_tx_empty_descriptor(tx_ring,
1620db0b124fSVinicius Costa Gomes empty,
1621db0b124fSVinicius Costa Gomes empty_info) < 0)
1622db0b124fSVinicius Costa Gomes dev_kfree_skb_any(empty);
1623db0b124fSVinicius Costa Gomes }
1624db0b124fSVinicius Costa Gomes
1625db0b124fSVinicius Costa Gomes done:
16260507ef8aSSasha Neftin /* record the location of the first descriptor for this packet */
16270507ef8aSSasha Neftin first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1628859b4dfaSAndre Guedes first->type = IGC_TX_BUFFER_TYPE_SKB;
16290507ef8aSSasha Neftin first->skb = skb;
16300507ef8aSSasha Neftin first->bytecount = skb->len;
16310507ef8aSSasha Neftin first->gso_segs = 1;
16320507ef8aSSasha Neftin
1633175c2412SMuhammad Husaini Zulkifli if (adapter->qbv_transition || tx_ring->oper_gate_closed)
1634175c2412SMuhammad Husaini Zulkifli goto out_drop;
1635175c2412SMuhammad Husaini Zulkifli
163625102893STan Tee Min if (tx_ring->max_sdu > 0 && first->bytecount > tx_ring->max_sdu) {
163792a0dcb8STan Tee Min adapter->stats.txdrop++;
163892a0dcb8STan Tee Min goto out_drop;
163992a0dcb8STan Tee Min }
164092a0dcb8STan Tee Min
1641ce58c7ccSVinicius Costa Gomes if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags) &&
1642ce58c7ccSVinicius Costa Gomes skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
16439c50e2b1SVinicius Costa Gomes unsigned long flags;
16443ed247e7SVinicius Costa Gomes u32 tstamp_flags;
16459c50e2b1SVinicius Costa Gomes
16469c50e2b1SVinicius Costa Gomes spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
16473ed247e7SVinicius Costa Gomes if (igc_request_tx_tstamp(adapter, skb, &tstamp_flags)) {
16482c344ae2SVinicius Costa Gomes skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
16493ed247e7SVinicius Costa Gomes tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags;
16502c344ae2SVinicius Costa Gomes } else {
16512c344ae2SVinicius Costa Gomes adapter->tx_hwtstamp_skipped++;
16522c344ae2SVinicius Costa Gomes }
16539c50e2b1SVinicius Costa Gomes
16549c50e2b1SVinicius Costa Gomes spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
16552c344ae2SVinicius Costa Gomes }
16562c344ae2SVinicius Costa Gomes
16578d744963SMuhammad Husaini Zulkifli if (skb_vlan_tag_present(skb)) {
16588d744963SMuhammad Husaini Zulkifli tx_flags |= IGC_TX_FLAGS_VLAN;
16598d744963SMuhammad Husaini Zulkifli tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT);
16608d744963SMuhammad Husaini Zulkifli }
16618d744963SMuhammad Husaini Zulkifli
16620507ef8aSSasha Neftin /* record initial flags and protocol */
16630507ef8aSSasha Neftin first->tx_flags = tx_flags;
16640507ef8aSSasha Neftin first->protocol = protocol;
16650507ef8aSSasha Neftin
1666db0b124fSVinicius Costa Gomes tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len);
1667f38b782dSSasha Neftin if (tso < 0)
1668f38b782dSSasha Neftin goto out_drop;
1669f38b782dSSasha Neftin else if (!tso)
1670db0b124fSVinicius Costa Gomes igc_tx_csum(tx_ring, first, launch_time, first_flag);
16710507ef8aSSasha Neftin
16720507ef8aSSasha Neftin igc_tx_map(tx_ring, first, hdr_len);
16730507ef8aSSasha Neftin
1674c9a11c23SSasha Neftin return NETDEV_TX_OK;
1675f38b782dSSasha Neftin
1676f38b782dSSasha Neftin out_drop:
1677f38b782dSSasha Neftin dev_kfree_skb_any(first->skb);
1678f38b782dSSasha Neftin first->skb = NULL;
1679f38b782dSSasha Neftin
1680f38b782dSSasha Neftin return NETDEV_TX_OK;
1681c9a11c23SSasha Neftin }
1682c9a11c23SSasha Neftin
igc_tx_queue_mapping(struct igc_adapter * adapter,struct sk_buff * skb)16830507ef8aSSasha Neftin static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
16840507ef8aSSasha Neftin struct sk_buff *skb)
168513b5b7fdSSasha Neftin {
16860507ef8aSSasha Neftin unsigned int r_idx = skb->queue_mapping;
16870507ef8aSSasha Neftin
16880507ef8aSSasha Neftin if (r_idx >= adapter->num_tx_queues)
16890507ef8aSSasha Neftin r_idx = r_idx % adapter->num_tx_queues;
16900507ef8aSSasha Neftin
16910507ef8aSSasha Neftin return adapter->tx_ring[r_idx];
169213b5b7fdSSasha Neftin }
169313b5b7fdSSasha Neftin
igc_xmit_frame(struct sk_buff * skb,struct net_device * netdev)16940507ef8aSSasha Neftin static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
16950507ef8aSSasha Neftin struct net_device *netdev)
169613b5b7fdSSasha Neftin {
16970507ef8aSSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
169813b5b7fdSSasha Neftin
16990507ef8aSSasha Neftin /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
17000507ef8aSSasha Neftin * in order to meet this minimum size requirement.
170113b5b7fdSSasha Neftin */
17020507ef8aSSasha Neftin if (skb->len < 17) {
17030507ef8aSSasha Neftin if (skb_padto(skb, 17))
17040507ef8aSSasha Neftin return NETDEV_TX_OK;
17050507ef8aSSasha Neftin skb->len = 17;
17060507ef8aSSasha Neftin }
170713b5b7fdSSasha Neftin
17080507ef8aSSasha Neftin return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
17090507ef8aSSasha Neftin }
17100507ef8aSSasha Neftin
igc_rx_checksum(struct igc_ring * ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)17113bdd7086SSasha Neftin static void igc_rx_checksum(struct igc_ring *ring,
17123bdd7086SSasha Neftin union igc_adv_rx_desc *rx_desc,
17133bdd7086SSasha Neftin struct sk_buff *skb)
17143bdd7086SSasha Neftin {
17153bdd7086SSasha Neftin skb_checksum_none_assert(skb);
17163bdd7086SSasha Neftin
17173bdd7086SSasha Neftin /* Ignore Checksum bit is set */
17183bdd7086SSasha Neftin if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
17193bdd7086SSasha Neftin return;
17203bdd7086SSasha Neftin
17213bdd7086SSasha Neftin /* Rx checksum disabled via ethtool */
17223bdd7086SSasha Neftin if (!(ring->netdev->features & NETIF_F_RXCSUM))
17233bdd7086SSasha Neftin return;
17243bdd7086SSasha Neftin
17253bdd7086SSasha Neftin /* TCP/UDP checksum error bit is set */
17263bdd7086SSasha Neftin if (igc_test_staterr(rx_desc,
1727ef8a17a2SAndre Guedes IGC_RXDEXT_STATERR_L4E |
17283bdd7086SSasha Neftin IGC_RXDEXT_STATERR_IPE)) {
17293bdd7086SSasha Neftin /* work around errata with sctp packets where the TCPE aka
17303bdd7086SSasha Neftin * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
17313bdd7086SSasha Neftin * packets (aka let the stack check the crc32c)
17323bdd7086SSasha Neftin */
17333bdd7086SSasha Neftin if (!(skb->len == 60 &&
17343bdd7086SSasha Neftin test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
17353bdd7086SSasha Neftin u64_stats_update_begin(&ring->rx_syncp);
17363bdd7086SSasha Neftin ring->rx_stats.csum_err++;
17373bdd7086SSasha Neftin u64_stats_update_end(&ring->rx_syncp);
17383bdd7086SSasha Neftin }
17393bdd7086SSasha Neftin /* let the stack verify checksum errors */
17403bdd7086SSasha Neftin return;
17413bdd7086SSasha Neftin }
17423bdd7086SSasha Neftin /* It must be a TCP or UDP packet with a valid checksum */
17433bdd7086SSasha Neftin if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
17443bdd7086SSasha Neftin IGC_RXD_STAT_UDPCS))
17453bdd7086SSasha Neftin skb->ip_summed = CHECKSUM_UNNECESSARY;
17463bdd7086SSasha Neftin
174725f06effSAndre Guedes netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
17483bdd7086SSasha Neftin le32_to_cpu(rx_desc->wb.upper.status_error));
17493bdd7086SSasha Neftin }
17503bdd7086SSasha Neftin
175184214ab4SJesper Dangaard Brouer /* Mapping HW RSS Type to enum pkt_hash_types */
175284214ab4SJesper Dangaard Brouer static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = {
175384214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_NO_HASH] = PKT_HASH_TYPE_L2,
175484214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_TCP_IPV4] = PKT_HASH_TYPE_L4,
175584214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_IPV4] = PKT_HASH_TYPE_L3,
175684214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_TCP_IPV6] = PKT_HASH_TYPE_L4,
175784214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_IPV6_EX] = PKT_HASH_TYPE_L3,
175884214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_IPV6] = PKT_HASH_TYPE_L3,
175984214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4,
176084214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_UDP_IPV4] = PKT_HASH_TYPE_L4,
176184214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_UDP_IPV6] = PKT_HASH_TYPE_L4,
176284214ab4SJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4,
176384214ab4SJesper Dangaard Brouer [10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */
176484214ab4SJesper Dangaard Brouer [11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask */
176584214ab4SJesper Dangaard Brouer [12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisons */
176684214ab4SJesper Dangaard Brouer [13] = PKT_HASH_TYPE_NONE,
176784214ab4SJesper Dangaard Brouer [14] = PKT_HASH_TYPE_NONE,
176884214ab4SJesper Dangaard Brouer [15] = PKT_HASH_TYPE_NONE,
176984214ab4SJesper Dangaard Brouer };
177084214ab4SJesper Dangaard Brouer
igc_rx_hash(struct igc_ring * ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)17710507ef8aSSasha Neftin static inline void igc_rx_hash(struct igc_ring *ring,
17720507ef8aSSasha Neftin union igc_adv_rx_desc *rx_desc,
17730507ef8aSSasha Neftin struct sk_buff *skb)
17740507ef8aSSasha Neftin {
177584214ab4SJesper Dangaard Brouer if (ring->netdev->features & NETIF_F_RXHASH) {
177684214ab4SJesper Dangaard Brouer u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
177784214ab4SJesper Dangaard Brouer u32 rss_type = igc_rss_type(rx_desc);
177884214ab4SJesper Dangaard Brouer
177984214ab4SJesper Dangaard Brouer skb_set_hash(skb, rss_hash, igc_rss_type_table[rss_type]);
178084214ab4SJesper Dangaard Brouer }
17810507ef8aSSasha Neftin }
17820507ef8aSSasha Neftin
igc_rx_vlan(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)17838d744963SMuhammad Husaini Zulkifli static void igc_rx_vlan(struct igc_ring *rx_ring,
17848d744963SMuhammad Husaini Zulkifli union igc_adv_rx_desc *rx_desc,
17858d744963SMuhammad Husaini Zulkifli struct sk_buff *skb)
17868d744963SMuhammad Husaini Zulkifli {
17878d744963SMuhammad Husaini Zulkifli struct net_device *dev = rx_ring->netdev;
17888d744963SMuhammad Husaini Zulkifli u16 vid;
17898d744963SMuhammad Husaini Zulkifli
17908d744963SMuhammad Husaini Zulkifli if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
17918d744963SMuhammad Husaini Zulkifli igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) {
17928d744963SMuhammad Husaini Zulkifli if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) &&
17938d744963SMuhammad Husaini Zulkifli test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
17948d744963SMuhammad Husaini Zulkifli vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
17958d744963SMuhammad Husaini Zulkifli else
17968d744963SMuhammad Husaini Zulkifli vid = le16_to_cpu(rx_desc->wb.upper.vlan);
17978d744963SMuhammad Husaini Zulkifli
17988d744963SMuhammad Husaini Zulkifli __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
17998d744963SMuhammad Husaini Zulkifli }
18008d744963SMuhammad Husaini Zulkifli }
18018d744963SMuhammad Husaini Zulkifli
18020507ef8aSSasha Neftin /**
18030507ef8aSSasha Neftin * igc_process_skb_fields - Populate skb header fields from Rx descriptor
18040507ef8aSSasha Neftin * @rx_ring: rx descriptor ring packet is being transacted on
18050507ef8aSSasha Neftin * @rx_desc: pointer to the EOP Rx descriptor
18060507ef8aSSasha Neftin * @skb: pointer to current skb being populated
18070507ef8aSSasha Neftin *
18083a66abe9SAndre Guedes * This function checks the ring, descriptor, and packet information in order
18093a66abe9SAndre Guedes * to populate the hash, checksum, VLAN, protocol, and other fields within the
18103a66abe9SAndre Guedes * skb.
18110507ef8aSSasha Neftin */
igc_process_skb_fields(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)18120507ef8aSSasha Neftin static void igc_process_skb_fields(struct igc_ring *rx_ring,
18130507ef8aSSasha Neftin union igc_adv_rx_desc *rx_desc,
18140507ef8aSSasha Neftin struct sk_buff *skb)
18150507ef8aSSasha Neftin {
18160507ef8aSSasha Neftin igc_rx_hash(rx_ring, rx_desc, skb);
18170507ef8aSSasha Neftin
18183bdd7086SSasha Neftin igc_rx_checksum(rx_ring, rx_desc, skb);
18193bdd7086SSasha Neftin
18208d744963SMuhammad Husaini Zulkifli igc_rx_vlan(rx_ring, rx_desc, skb);
18218d744963SMuhammad Husaini Zulkifli
18220507ef8aSSasha Neftin skb_record_rx_queue(skb, rx_ring->queue_index);
18230507ef8aSSasha Neftin
18240507ef8aSSasha Neftin skb->protocol = eth_type_trans(skb, rx_ring->netdev);
18250507ef8aSSasha Neftin }
18260507ef8aSSasha Neftin
igc_vlan_mode(struct net_device * netdev,netdev_features_t features)18278d744963SMuhammad Husaini Zulkifli static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features)
18288d744963SMuhammad Husaini Zulkifli {
18298d744963SMuhammad Husaini Zulkifli bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
18308d744963SMuhammad Husaini Zulkifli struct igc_adapter *adapter = netdev_priv(netdev);
18318d744963SMuhammad Husaini Zulkifli struct igc_hw *hw = &adapter->hw;
18328d744963SMuhammad Husaini Zulkifli u32 ctrl;
18338d744963SMuhammad Husaini Zulkifli
18348d744963SMuhammad Husaini Zulkifli ctrl = rd32(IGC_CTRL);
18358d744963SMuhammad Husaini Zulkifli
18368d744963SMuhammad Husaini Zulkifli if (enable) {
18378d744963SMuhammad Husaini Zulkifli /* enable VLAN tag insert/strip */
18388d744963SMuhammad Husaini Zulkifli ctrl |= IGC_CTRL_VME;
18398d744963SMuhammad Husaini Zulkifli } else {
18408d744963SMuhammad Husaini Zulkifli /* disable VLAN tag insert/strip */
18418d744963SMuhammad Husaini Zulkifli ctrl &= ~IGC_CTRL_VME;
18428d744963SMuhammad Husaini Zulkifli }
18438d744963SMuhammad Husaini Zulkifli wr32(IGC_CTRL, ctrl);
18448d744963SMuhammad Husaini Zulkifli }
18458d744963SMuhammad Husaini Zulkifli
igc_restore_vlan(struct igc_adapter * adapter)18468d744963SMuhammad Husaini Zulkifli static void igc_restore_vlan(struct igc_adapter *adapter)
18478d744963SMuhammad Husaini Zulkifli {
18488d744963SMuhammad Husaini Zulkifli igc_vlan_mode(adapter->netdev, adapter->netdev->features);
18498d744963SMuhammad Husaini Zulkifli }
18508d744963SMuhammad Husaini Zulkifli
igc_get_rx_buffer(struct igc_ring * rx_ring,const unsigned int size,int * rx_buffer_pgcnt)18510507ef8aSSasha Neftin static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
18524ff32036SAndre Guedes const unsigned int size,
18534ff32036SAndre Guedes int *rx_buffer_pgcnt)
18540507ef8aSSasha Neftin {
18550507ef8aSSasha Neftin struct igc_rx_buffer *rx_buffer;
18560507ef8aSSasha Neftin
18570507ef8aSSasha Neftin rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
18584ff32036SAndre Guedes *rx_buffer_pgcnt =
18594ff32036SAndre Guedes #if (PAGE_SIZE < 8192)
18604ff32036SAndre Guedes page_count(rx_buffer->page);
18614ff32036SAndre Guedes #else
18624ff32036SAndre Guedes 0;
18634ff32036SAndre Guedes #endif
18640507ef8aSSasha Neftin prefetchw(rx_buffer->page);
18650507ef8aSSasha Neftin
18660507ef8aSSasha Neftin /* we are reusing so sync this buffer for CPU use */
18670507ef8aSSasha Neftin dma_sync_single_range_for_cpu(rx_ring->dev,
18680507ef8aSSasha Neftin rx_buffer->dma,
18690507ef8aSSasha Neftin rx_buffer->page_offset,
18700507ef8aSSasha Neftin size,
18710507ef8aSSasha Neftin DMA_FROM_DEVICE);
18720507ef8aSSasha Neftin
18730507ef8aSSasha Neftin rx_buffer->pagecnt_bias--;
18740507ef8aSSasha Neftin
18750507ef8aSSasha Neftin return rx_buffer;
18760507ef8aSSasha Neftin }
18770507ef8aSSasha Neftin
igc_rx_buffer_flip(struct igc_rx_buffer * buffer,unsigned int truesize)1878613cf199SAndre Guedes static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer,
1879613cf199SAndre Guedes unsigned int truesize)
1880613cf199SAndre Guedes {
1881613cf199SAndre Guedes #if (PAGE_SIZE < 8192)
1882613cf199SAndre Guedes buffer->page_offset ^= truesize;
1883613cf199SAndre Guedes #else
1884613cf199SAndre Guedes buffer->page_offset += truesize;
1885613cf199SAndre Guedes #endif
1886613cf199SAndre Guedes }
1887613cf199SAndre Guedes
igc_get_rx_frame_truesize(struct igc_ring * ring,unsigned int size)1888a39f5e53SAndre Guedes static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring,
1889a39f5e53SAndre Guedes unsigned int size)
1890a39f5e53SAndre Guedes {
1891a39f5e53SAndre Guedes unsigned int truesize;
1892a39f5e53SAndre Guedes
1893a39f5e53SAndre Guedes #if (PAGE_SIZE < 8192)
1894a39f5e53SAndre Guedes truesize = igc_rx_pg_size(ring) / 2;
1895a39f5e53SAndre Guedes #else
1896a39f5e53SAndre Guedes truesize = ring_uses_build_skb(ring) ?
1897a39f5e53SAndre Guedes SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1898a39f5e53SAndre Guedes SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1899a39f5e53SAndre Guedes SKB_DATA_ALIGN(size);
1900a39f5e53SAndre Guedes #endif
1901a39f5e53SAndre Guedes return truesize;
1902a39f5e53SAndre Guedes }
1903a39f5e53SAndre Guedes
19040507ef8aSSasha Neftin /**
19050507ef8aSSasha Neftin * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
19060507ef8aSSasha Neftin * @rx_ring: rx descriptor ring to transact packets on
19070507ef8aSSasha Neftin * @rx_buffer: buffer containing page to add
19080507ef8aSSasha Neftin * @skb: sk_buff to place the data into
19090507ef8aSSasha Neftin * @size: size of buffer to be added
19100507ef8aSSasha Neftin *
19110507ef8aSSasha Neftin * This function will add the data contained in rx_buffer->page to the skb.
19120507ef8aSSasha Neftin */
igc_add_rx_frag(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct sk_buff * skb,unsigned int size)19130507ef8aSSasha Neftin static void igc_add_rx_frag(struct igc_ring *rx_ring,
19140507ef8aSSasha Neftin struct igc_rx_buffer *rx_buffer,
19150507ef8aSSasha Neftin struct sk_buff *skb,
19160507ef8aSSasha Neftin unsigned int size)
19170507ef8aSSasha Neftin {
1918613cf199SAndre Guedes unsigned int truesize;
19190507ef8aSSasha Neftin
1920613cf199SAndre Guedes #if (PAGE_SIZE < 8192)
1921613cf199SAndre Guedes truesize = igc_rx_pg_size(rx_ring) / 2;
19220507ef8aSSasha Neftin #else
1923613cf199SAndre Guedes truesize = ring_uses_build_skb(rx_ring) ?
19240507ef8aSSasha Neftin SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
19250507ef8aSSasha Neftin SKB_DATA_ALIGN(size);
1926613cf199SAndre Guedes #endif
19270507ef8aSSasha Neftin skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
19280507ef8aSSasha Neftin rx_buffer->page_offset, size, truesize);
1929613cf199SAndre Guedes
1930613cf199SAndre Guedes igc_rx_buffer_flip(rx_buffer, truesize);
19310507ef8aSSasha Neftin }
19320507ef8aSSasha Neftin
igc_build_skb(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct xdp_buff * xdp)19330507ef8aSSasha Neftin static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
19340507ef8aSSasha Neftin struct igc_rx_buffer *rx_buffer,
1935f51b5e2bSJesper Dangaard Brouer struct xdp_buff *xdp)
19360507ef8aSSasha Neftin {
1937f51b5e2bSJesper Dangaard Brouer unsigned int size = xdp->data_end - xdp->data;
1938a39f5e53SAndre Guedes unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1939f51b5e2bSJesper Dangaard Brouer unsigned int metasize = xdp->data - xdp->data_meta;
19400507ef8aSSasha Neftin struct sk_buff *skb;
19410507ef8aSSasha Neftin
19420507ef8aSSasha Neftin /* prefetch first cache line of first page */
1943f51b5e2bSJesper Dangaard Brouer net_prefetch(xdp->data_meta);
19440507ef8aSSasha Neftin
19450507ef8aSSasha Neftin /* build an skb around the page buffer */
19464dd330a7SAlexander Lobakin skb = napi_build_skb(xdp->data_hard_start, truesize);
19470507ef8aSSasha Neftin if (unlikely(!skb))
19480507ef8aSSasha Neftin return NULL;
19490507ef8aSSasha Neftin
19500507ef8aSSasha Neftin /* update pointers within the skb to store the data */
1951f51b5e2bSJesper Dangaard Brouer skb_reserve(skb, xdp->data - xdp->data_hard_start);
19520507ef8aSSasha Neftin __skb_put(skb, size);
1953f51b5e2bSJesper Dangaard Brouer if (metasize)
1954f51b5e2bSJesper Dangaard Brouer skb_metadata_set(skb, metasize);
19550507ef8aSSasha Neftin
1956613cf199SAndre Guedes igc_rx_buffer_flip(rx_buffer, truesize);
19570507ef8aSSasha Neftin return skb;
19580507ef8aSSasha Neftin }
19590507ef8aSSasha Neftin
igc_construct_skb(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)19600507ef8aSSasha Neftin static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
19610507ef8aSSasha Neftin struct igc_rx_buffer *rx_buffer,
196226575105SAndre Guedes struct xdp_buff *xdp,
1963e1ed4f92SAndre Guedes ktime_t timestamp)
19640507ef8aSSasha Neftin {
1965f51b5e2bSJesper Dangaard Brouer unsigned int metasize = xdp->data - xdp->data_meta;
196626575105SAndre Guedes unsigned int size = xdp->data_end - xdp->data;
1967a39f5e53SAndre Guedes unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
196826575105SAndre Guedes void *va = xdp->data;
19690507ef8aSSasha Neftin unsigned int headlen;
19700507ef8aSSasha Neftin struct sk_buff *skb;
19710507ef8aSSasha Neftin
19720507ef8aSSasha Neftin /* prefetch first cache line of first page */
1973f51b5e2bSJesper Dangaard Brouer net_prefetch(xdp->data_meta);
19740507ef8aSSasha Neftin
19750507ef8aSSasha Neftin /* allocate a skb to store the frags */
1976f51b5e2bSJesper Dangaard Brouer skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1977f51b5e2bSJesper Dangaard Brouer IGC_RX_HDR_LEN + metasize);
19780507ef8aSSasha Neftin if (unlikely(!skb))
19790507ef8aSSasha Neftin return NULL;
19800507ef8aSSasha Neftin
1981e1ed4f92SAndre Guedes if (timestamp)
1982e1ed4f92SAndre Guedes skb_hwtstamps(skb)->hwtstamp = timestamp;
198381b05520SVinicius Costa Gomes
19840507ef8aSSasha Neftin /* Determine available headroom for copy */
19850507ef8aSSasha Neftin headlen = size;
19860507ef8aSSasha Neftin if (headlen > IGC_RX_HDR_LEN)
1987c43f1255SStanislav Fomichev headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
19880507ef8aSSasha Neftin
19890507ef8aSSasha Neftin /* align pull length to size of long to optimize memcpy performance */
1990f51b5e2bSJesper Dangaard Brouer memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta,
1991f51b5e2bSJesper Dangaard Brouer ALIGN(headlen + metasize, sizeof(long)));
1992f51b5e2bSJesper Dangaard Brouer
1993f51b5e2bSJesper Dangaard Brouer if (metasize) {
1994f51b5e2bSJesper Dangaard Brouer skb_metadata_set(skb, metasize);
1995f51b5e2bSJesper Dangaard Brouer __skb_pull(skb, metasize);
1996f51b5e2bSJesper Dangaard Brouer }
19970507ef8aSSasha Neftin
19980507ef8aSSasha Neftin /* update all of the pointers */
19990507ef8aSSasha Neftin size -= headlen;
20000507ef8aSSasha Neftin if (size) {
20010507ef8aSSasha Neftin skb_add_rx_frag(skb, 0, rx_buffer->page,
20020507ef8aSSasha Neftin (va + headlen) - page_address(rx_buffer->page),
20030507ef8aSSasha Neftin size, truesize);
2004613cf199SAndre Guedes igc_rx_buffer_flip(rx_buffer, truesize);
20050507ef8aSSasha Neftin } else {
20060507ef8aSSasha Neftin rx_buffer->pagecnt_bias++;
20070507ef8aSSasha Neftin }
20080507ef8aSSasha Neftin
20090507ef8aSSasha Neftin return skb;
20100507ef8aSSasha Neftin }
20110507ef8aSSasha Neftin
20120507ef8aSSasha Neftin /**
20130507ef8aSSasha Neftin * igc_reuse_rx_page - page flip buffer and store it back on the ring
20140507ef8aSSasha Neftin * @rx_ring: rx descriptor ring to store buffers on
20150507ef8aSSasha Neftin * @old_buff: donor buffer to have page reused
20160507ef8aSSasha Neftin *
20170507ef8aSSasha Neftin * Synchronizes page for reuse by the adapter
20180507ef8aSSasha Neftin */
igc_reuse_rx_page(struct igc_ring * rx_ring,struct igc_rx_buffer * old_buff)20190507ef8aSSasha Neftin static void igc_reuse_rx_page(struct igc_ring *rx_ring,
20200507ef8aSSasha Neftin struct igc_rx_buffer *old_buff)
20210507ef8aSSasha Neftin {
20220507ef8aSSasha Neftin u16 nta = rx_ring->next_to_alloc;
20230507ef8aSSasha Neftin struct igc_rx_buffer *new_buff;
20240507ef8aSSasha Neftin
20250507ef8aSSasha Neftin new_buff = &rx_ring->rx_buffer_info[nta];
20260507ef8aSSasha Neftin
20270507ef8aSSasha Neftin /* update, and store next to alloc */
20280507ef8aSSasha Neftin nta++;
20290507ef8aSSasha Neftin rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
20300507ef8aSSasha Neftin
20310507ef8aSSasha Neftin /* Transfer page from old buffer to new buffer.
20320507ef8aSSasha Neftin * Move each member individually to avoid possible store
20330507ef8aSSasha Neftin * forwarding stalls.
20340507ef8aSSasha Neftin */
20350507ef8aSSasha Neftin new_buff->dma = old_buff->dma;
20360507ef8aSSasha Neftin new_buff->page = old_buff->page;
20370507ef8aSSasha Neftin new_buff->page_offset = old_buff->page_offset;
20380507ef8aSSasha Neftin new_buff->pagecnt_bias = old_buff->pagecnt_bias;
20390507ef8aSSasha Neftin }
20400507ef8aSSasha Neftin
igc_can_reuse_rx_page(struct igc_rx_buffer * rx_buffer,int rx_buffer_pgcnt)20414ff32036SAndre Guedes static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer,
20424ff32036SAndre Guedes int rx_buffer_pgcnt)
20430507ef8aSSasha Neftin {
20440507ef8aSSasha Neftin unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
20450507ef8aSSasha Neftin struct page *page = rx_buffer->page;
20460507ef8aSSasha Neftin
2047a79afa78SAlexander Lobakin /* avoid re-using remote and pfmemalloc pages */
2048a79afa78SAlexander Lobakin if (!dev_page_is_reusable(page))
20490507ef8aSSasha Neftin return false;
20500507ef8aSSasha Neftin
20510507ef8aSSasha Neftin #if (PAGE_SIZE < 8192)
20520507ef8aSSasha Neftin /* if we are only owner of page we can reuse it */
20534ff32036SAndre Guedes if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
20540507ef8aSSasha Neftin return false;
20550507ef8aSSasha Neftin #else
20560507ef8aSSasha Neftin #define IGC_LAST_OFFSET \
20570507ef8aSSasha Neftin (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
20580507ef8aSSasha Neftin
20590507ef8aSSasha Neftin if (rx_buffer->page_offset > IGC_LAST_OFFSET)
20600507ef8aSSasha Neftin return false;
20610507ef8aSSasha Neftin #endif
20620507ef8aSSasha Neftin
20630507ef8aSSasha Neftin /* If we have drained the page fragment pool we need to update
20640507ef8aSSasha Neftin * the pagecnt_bias and page count so that we fully restock the
20650507ef8aSSasha Neftin * number of references the driver holds.
20660507ef8aSSasha Neftin */
20674ff32036SAndre Guedes if (unlikely(pagecnt_bias == 1)) {
20684ff32036SAndre Guedes page_ref_add(page, USHRT_MAX - 1);
20690507ef8aSSasha Neftin rx_buffer->pagecnt_bias = USHRT_MAX;
20700507ef8aSSasha Neftin }
20710507ef8aSSasha Neftin
20720507ef8aSSasha Neftin return true;
20730507ef8aSSasha Neftin }
20740507ef8aSSasha Neftin
20750507ef8aSSasha Neftin /**
20760507ef8aSSasha Neftin * igc_is_non_eop - process handling of non-EOP buffers
20770507ef8aSSasha Neftin * @rx_ring: Rx ring being processed
20780507ef8aSSasha Neftin * @rx_desc: Rx descriptor for current buffer
20790507ef8aSSasha Neftin *
20800507ef8aSSasha Neftin * This function updates next to clean. If the buffer is an EOP buffer
20810507ef8aSSasha Neftin * this function exits returning false, otherwise it will place the
20820507ef8aSSasha Neftin * sk_buff in the next buffer to be chained and return true indicating
20830507ef8aSSasha Neftin * that this is in fact a non-EOP buffer.
20840507ef8aSSasha Neftin */
igc_is_non_eop(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc)20850507ef8aSSasha Neftin static bool igc_is_non_eop(struct igc_ring *rx_ring,
20860507ef8aSSasha Neftin union igc_adv_rx_desc *rx_desc)
20870507ef8aSSasha Neftin {
20880507ef8aSSasha Neftin u32 ntc = rx_ring->next_to_clean + 1;
20890507ef8aSSasha Neftin
20900507ef8aSSasha Neftin /* fetch, update, and store next to clean */
20910507ef8aSSasha Neftin ntc = (ntc < rx_ring->count) ? ntc : 0;
20920507ef8aSSasha Neftin rx_ring->next_to_clean = ntc;
20930507ef8aSSasha Neftin
20940507ef8aSSasha Neftin prefetch(IGC_RX_DESC(rx_ring, ntc));
20950507ef8aSSasha Neftin
20960507ef8aSSasha Neftin if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
20970507ef8aSSasha Neftin return false;
20980507ef8aSSasha Neftin
20990507ef8aSSasha Neftin return true;
21000507ef8aSSasha Neftin }
21010507ef8aSSasha Neftin
21020507ef8aSSasha Neftin /**
21030507ef8aSSasha Neftin * igc_cleanup_headers - Correct corrupted or empty headers
21040507ef8aSSasha Neftin * @rx_ring: rx descriptor ring packet is being transacted on
21050507ef8aSSasha Neftin * @rx_desc: pointer to the EOP Rx descriptor
21060507ef8aSSasha Neftin * @skb: pointer to current skb being fixed
21070507ef8aSSasha Neftin *
21080507ef8aSSasha Neftin * Address the case where we are pulling data in on pages only
21090507ef8aSSasha Neftin * and as such no data is present in the skb header.
21100507ef8aSSasha Neftin *
21110507ef8aSSasha Neftin * In addition if skb is not at least 60 bytes we need to pad it so that
21120507ef8aSSasha Neftin * it is large enough to qualify as a valid Ethernet frame.
21130507ef8aSSasha Neftin *
21140507ef8aSSasha Neftin * Returns true if an error was encountered and skb was freed.
21150507ef8aSSasha Neftin */
igc_cleanup_headers(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)21160507ef8aSSasha Neftin static bool igc_cleanup_headers(struct igc_ring *rx_ring,
21170507ef8aSSasha Neftin union igc_adv_rx_desc *rx_desc,
21180507ef8aSSasha Neftin struct sk_buff *skb)
21190507ef8aSSasha Neftin {
212026575105SAndre Guedes /* XDP packets use error pointer so abort at this point */
212126575105SAndre Guedes if (IS_ERR(skb))
212226575105SAndre Guedes return true;
212326575105SAndre Guedes
2124ef8a17a2SAndre Guedes if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
21250507ef8aSSasha Neftin struct net_device *netdev = rx_ring->netdev;
21260507ef8aSSasha Neftin
21270507ef8aSSasha Neftin if (!(netdev->features & NETIF_F_RXALL)) {
21280507ef8aSSasha Neftin dev_kfree_skb_any(skb);
21290507ef8aSSasha Neftin return true;
21300507ef8aSSasha Neftin }
21310507ef8aSSasha Neftin }
21320507ef8aSSasha Neftin
21330507ef8aSSasha Neftin /* if eth_skb_pad returns an error the skb was freed */
21340507ef8aSSasha Neftin if (eth_skb_pad(skb))
21350507ef8aSSasha Neftin return true;
21360507ef8aSSasha Neftin
213713b5b7fdSSasha Neftin return false;
213813b5b7fdSSasha Neftin }
213913b5b7fdSSasha Neftin
igc_put_rx_buffer(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,int rx_buffer_pgcnt)21400507ef8aSSasha Neftin static void igc_put_rx_buffer(struct igc_ring *rx_ring,
21414ff32036SAndre Guedes struct igc_rx_buffer *rx_buffer,
21424ff32036SAndre Guedes int rx_buffer_pgcnt)
21430507ef8aSSasha Neftin {
21444ff32036SAndre Guedes if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
21450507ef8aSSasha Neftin /* hand second half of page back to the ring */
21460507ef8aSSasha Neftin igc_reuse_rx_page(rx_ring, rx_buffer);
21470507ef8aSSasha Neftin } else {
21480507ef8aSSasha Neftin /* We are not reusing the buffer so unmap it and free
21490507ef8aSSasha Neftin * any references we are holding to it
21500507ef8aSSasha Neftin */
21510507ef8aSSasha Neftin dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
21520507ef8aSSasha Neftin igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
21530507ef8aSSasha Neftin IGC_RX_DMA_ATTR);
21540507ef8aSSasha Neftin __page_frag_cache_drain(rx_buffer->page,
21550507ef8aSSasha Neftin rx_buffer->pagecnt_bias);
21560507ef8aSSasha Neftin }
215713b5b7fdSSasha Neftin
21580507ef8aSSasha Neftin /* clear contents of rx_buffer */
21590507ef8aSSasha Neftin rx_buffer->page = NULL;
216013b5b7fdSSasha Neftin }
216113b5b7fdSSasha Neftin
igc_rx_offset(struct igc_ring * rx_ring)2162aac8f68cSSasha Neftin static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
2163aac8f68cSSasha Neftin {
216426575105SAndre Guedes struct igc_adapter *adapter = rx_ring->q_vector->adapter;
216526575105SAndre Guedes
216626575105SAndre Guedes if (ring_uses_build_skb(rx_ring))
216726575105SAndre Guedes return IGC_SKB_PAD;
216826575105SAndre Guedes if (igc_xdp_is_enabled(adapter))
216926575105SAndre Guedes return XDP_PACKET_HEADROOM;
217026575105SAndre Guedes
217126575105SAndre Guedes return 0;
2172aac8f68cSSasha Neftin }
2173aac8f68cSSasha Neftin
igc_alloc_mapped_page(struct igc_ring * rx_ring,struct igc_rx_buffer * bi)2174aac8f68cSSasha Neftin static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
2175aac8f68cSSasha Neftin struct igc_rx_buffer *bi)
2176aac8f68cSSasha Neftin {
2177aac8f68cSSasha Neftin struct page *page = bi->page;
2178aac8f68cSSasha Neftin dma_addr_t dma;
2179aac8f68cSSasha Neftin
2180aac8f68cSSasha Neftin /* since we are recycling buffers we should seldom need to alloc */
2181aac8f68cSSasha Neftin if (likely(page))
2182aac8f68cSSasha Neftin return true;
2183aac8f68cSSasha Neftin
2184aac8f68cSSasha Neftin /* alloc new page for storage */
2185aac8f68cSSasha Neftin page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
2186aac8f68cSSasha Neftin if (unlikely(!page)) {
2187aac8f68cSSasha Neftin rx_ring->rx_stats.alloc_failed++;
2188aac8f68cSSasha Neftin return false;
2189aac8f68cSSasha Neftin }
2190aac8f68cSSasha Neftin
2191aac8f68cSSasha Neftin /* map page for use */
2192aac8f68cSSasha Neftin dma = dma_map_page_attrs(rx_ring->dev, page, 0,
2193aac8f68cSSasha Neftin igc_rx_pg_size(rx_ring),
2194aac8f68cSSasha Neftin DMA_FROM_DEVICE,
2195aac8f68cSSasha Neftin IGC_RX_DMA_ATTR);
2196aac8f68cSSasha Neftin
2197aac8f68cSSasha Neftin /* if mapping failed free memory back to system since
2198aac8f68cSSasha Neftin * there isn't much point in holding memory we can't use
2199aac8f68cSSasha Neftin */
2200aac8f68cSSasha Neftin if (dma_mapping_error(rx_ring->dev, dma)) {
2201aac8f68cSSasha Neftin __free_page(page);
2202aac8f68cSSasha Neftin
2203aac8f68cSSasha Neftin rx_ring->rx_stats.alloc_failed++;
2204aac8f68cSSasha Neftin return false;
2205aac8f68cSSasha Neftin }
2206aac8f68cSSasha Neftin
2207aac8f68cSSasha Neftin bi->dma = dma;
2208aac8f68cSSasha Neftin bi->page = page;
2209aac8f68cSSasha Neftin bi->page_offset = igc_rx_offset(rx_ring);
22104ff32036SAndre Guedes page_ref_add(page, USHRT_MAX - 1);
22114ff32036SAndre Guedes bi->pagecnt_bias = USHRT_MAX;
2212aac8f68cSSasha Neftin
2213aac8f68cSSasha Neftin return true;
2214aac8f68cSSasha Neftin }
2215aac8f68cSSasha Neftin
221613b5b7fdSSasha Neftin /**
221713b5b7fdSSasha Neftin * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2218085c8589SSasha Neftin * @rx_ring: rx descriptor ring
2219085c8589SSasha Neftin * @cleaned_count: number of buffers to clean
222013b5b7fdSSasha Neftin */
igc_alloc_rx_buffers(struct igc_ring * rx_ring,u16 cleaned_count)222113b5b7fdSSasha Neftin static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
222213b5b7fdSSasha Neftin {
222313b5b7fdSSasha Neftin union igc_adv_rx_desc *rx_desc;
222413b5b7fdSSasha Neftin u16 i = rx_ring->next_to_use;
222513b5b7fdSSasha Neftin struct igc_rx_buffer *bi;
222613b5b7fdSSasha Neftin u16 bufsz;
222713b5b7fdSSasha Neftin
222813b5b7fdSSasha Neftin /* nothing to do */
222913b5b7fdSSasha Neftin if (!cleaned_count)
223013b5b7fdSSasha Neftin return;
223113b5b7fdSSasha Neftin
223213b5b7fdSSasha Neftin rx_desc = IGC_RX_DESC(rx_ring, i);
223313b5b7fdSSasha Neftin bi = &rx_ring->rx_buffer_info[i];
223413b5b7fdSSasha Neftin i -= rx_ring->count;
223513b5b7fdSSasha Neftin
223613b5b7fdSSasha Neftin bufsz = igc_rx_bufsz(rx_ring);
223713b5b7fdSSasha Neftin
223813b5b7fdSSasha Neftin do {
223913b5b7fdSSasha Neftin if (!igc_alloc_mapped_page(rx_ring, bi))
224013b5b7fdSSasha Neftin break;
224113b5b7fdSSasha Neftin
224213b5b7fdSSasha Neftin /* sync the buffer for use by the device */
224313b5b7fdSSasha Neftin dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
224413b5b7fdSSasha Neftin bi->page_offset, bufsz,
224513b5b7fdSSasha Neftin DMA_FROM_DEVICE);
224613b5b7fdSSasha Neftin
224713b5b7fdSSasha Neftin /* Refresh the desc even if buffer_addrs didn't change
224813b5b7fdSSasha Neftin * because each write-back erases this info.
224913b5b7fdSSasha Neftin */
225013b5b7fdSSasha Neftin rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
225113b5b7fdSSasha Neftin
225213b5b7fdSSasha Neftin rx_desc++;
225313b5b7fdSSasha Neftin bi++;
225413b5b7fdSSasha Neftin i++;
225513b5b7fdSSasha Neftin if (unlikely(!i)) {
225613b5b7fdSSasha Neftin rx_desc = IGC_RX_DESC(rx_ring, 0);
225713b5b7fdSSasha Neftin bi = rx_ring->rx_buffer_info;
225813b5b7fdSSasha Neftin i -= rx_ring->count;
225913b5b7fdSSasha Neftin }
226013b5b7fdSSasha Neftin
226113b5b7fdSSasha Neftin /* clear the length for the next_to_use descriptor */
226213b5b7fdSSasha Neftin rx_desc->wb.upper.length = 0;
226313b5b7fdSSasha Neftin
226413b5b7fdSSasha Neftin cleaned_count--;
226513b5b7fdSSasha Neftin } while (cleaned_count);
226613b5b7fdSSasha Neftin
226713b5b7fdSSasha Neftin i += rx_ring->count;
226813b5b7fdSSasha Neftin
226913b5b7fdSSasha Neftin if (rx_ring->next_to_use != i) {
227013b5b7fdSSasha Neftin /* record the next descriptor to use */
227113b5b7fdSSasha Neftin rx_ring->next_to_use = i;
227213b5b7fdSSasha Neftin
227313b5b7fdSSasha Neftin /* update next to alloc since we have filled the ring */
227413b5b7fdSSasha Neftin rx_ring->next_to_alloc = i;
227513b5b7fdSSasha Neftin
227613b5b7fdSSasha Neftin /* Force memory writes to complete before letting h/w
227713b5b7fdSSasha Neftin * know there are new descriptors to fetch. (Only
227813b5b7fdSSasha Neftin * applicable for weak-ordered memory model archs,
227913b5b7fdSSasha Neftin * such as IA-64).
228013b5b7fdSSasha Neftin */
228113b5b7fdSSasha Neftin wmb();
228213b5b7fdSSasha Neftin writel(i, rx_ring->tail);
228313b5b7fdSSasha Neftin }
228413b5b7fdSSasha Neftin }
228513b5b7fdSSasha Neftin
igc_alloc_rx_buffers_zc(struct igc_ring * ring,u16 count)2286fc9df2a0SAndre Guedes static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
2287fc9df2a0SAndre Guedes {
2288fc9df2a0SAndre Guedes union igc_adv_rx_desc *desc;
2289fc9df2a0SAndre Guedes u16 i = ring->next_to_use;
2290fc9df2a0SAndre Guedes struct igc_rx_buffer *bi;
2291fc9df2a0SAndre Guedes dma_addr_t dma;
2292fc9df2a0SAndre Guedes bool ok = true;
2293fc9df2a0SAndre Guedes
2294fc9df2a0SAndre Guedes if (!count)
2295fc9df2a0SAndre Guedes return ok;
2296fc9df2a0SAndre Guedes
229773b7123dSJesper Dangaard Brouer XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff);
229873b7123dSJesper Dangaard Brouer
2299fc9df2a0SAndre Guedes desc = IGC_RX_DESC(ring, i);
2300fc9df2a0SAndre Guedes bi = &ring->rx_buffer_info[i];
2301fc9df2a0SAndre Guedes i -= ring->count;
2302fc9df2a0SAndre Guedes
2303fc9df2a0SAndre Guedes do {
2304fc9df2a0SAndre Guedes bi->xdp = xsk_buff_alloc(ring->xsk_pool);
2305fc9df2a0SAndre Guedes if (!bi->xdp) {
2306fc9df2a0SAndre Guedes ok = false;
2307fc9df2a0SAndre Guedes break;
2308fc9df2a0SAndre Guedes }
2309fc9df2a0SAndre Guedes
2310fc9df2a0SAndre Guedes dma = xsk_buff_xdp_get_dma(bi->xdp);
2311fc9df2a0SAndre Guedes desc->read.pkt_addr = cpu_to_le64(dma);
2312fc9df2a0SAndre Guedes
2313fc9df2a0SAndre Guedes desc++;
2314fc9df2a0SAndre Guedes bi++;
2315fc9df2a0SAndre Guedes i++;
2316fc9df2a0SAndre Guedes if (unlikely(!i)) {
2317fc9df2a0SAndre Guedes desc = IGC_RX_DESC(ring, 0);
2318fc9df2a0SAndre Guedes bi = ring->rx_buffer_info;
2319fc9df2a0SAndre Guedes i -= ring->count;
2320fc9df2a0SAndre Guedes }
2321fc9df2a0SAndre Guedes
2322fc9df2a0SAndre Guedes /* Clear the length for the next_to_use descriptor. */
2323fc9df2a0SAndre Guedes desc->wb.upper.length = 0;
2324fc9df2a0SAndre Guedes
2325fc9df2a0SAndre Guedes count--;
2326fc9df2a0SAndre Guedes } while (count);
2327fc9df2a0SAndre Guedes
2328fc9df2a0SAndre Guedes i += ring->count;
2329fc9df2a0SAndre Guedes
2330fc9df2a0SAndre Guedes if (ring->next_to_use != i) {
2331fc9df2a0SAndre Guedes ring->next_to_use = i;
2332fc9df2a0SAndre Guedes
2333fc9df2a0SAndre Guedes /* Force memory writes to complete before letting h/w
2334fc9df2a0SAndre Guedes * know there are new descriptors to fetch. (Only
2335fc9df2a0SAndre Guedes * applicable for weak-ordered memory model archs,
2336fc9df2a0SAndre Guedes * such as IA-64).
2337fc9df2a0SAndre Guedes */
2338fc9df2a0SAndre Guedes wmb();
2339fc9df2a0SAndre Guedes writel(i, ring->tail);
2340fc9df2a0SAndre Guedes }
2341fc9df2a0SAndre Guedes
2342fc9df2a0SAndre Guedes return ok;
2343fc9df2a0SAndre Guedes }
2344fc9df2a0SAndre Guedes
234573f1071cSAndre Guedes /* This function requires __netif_tx_lock is held by the caller. */
igc_xdp_init_tx_descriptor(struct igc_ring * ring,struct xdp_frame * xdpf)234673f1071cSAndre Guedes static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
234773f1071cSAndre Guedes struct xdp_frame *xdpf)
234873f1071cSAndre Guedes {
23498c78c1e5SLorenzo Bianconi struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
23508c78c1e5SLorenzo Bianconi u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
23518c78c1e5SLorenzo Bianconi u16 count, index = ring->next_to_use;
23528c78c1e5SLorenzo Bianconi struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
23538c78c1e5SLorenzo Bianconi struct igc_tx_buffer *buffer = head;
23548c78c1e5SLorenzo Bianconi union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
23558c78c1e5SLorenzo Bianconi u32 olinfo_status, len = xdpf->len, cmd_type;
23568c78c1e5SLorenzo Bianconi void *data = xdpf->data;
23578c78c1e5SLorenzo Bianconi u16 i;
235873f1071cSAndre Guedes
23598c78c1e5SLorenzo Bianconi count = TXD_USE_COUNT(len);
23608c78c1e5SLorenzo Bianconi for (i = 0; i < nr_frags; i++)
23618c78c1e5SLorenzo Bianconi count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
23628c78c1e5SLorenzo Bianconi
23638c78c1e5SLorenzo Bianconi if (igc_maybe_stop_tx(ring, count + 3)) {
23648c78c1e5SLorenzo Bianconi /* this is a hard error */
236573f1071cSAndre Guedes return -EBUSY;
23668c78c1e5SLorenzo Bianconi }
236773f1071cSAndre Guedes
23688c78c1e5SLorenzo Bianconi i = 0;
23698c78c1e5SLorenzo Bianconi head->bytecount = xdp_get_frame_len(xdpf);
23708c78c1e5SLorenzo Bianconi head->type = IGC_TX_BUFFER_TYPE_XDP;
23718c78c1e5SLorenzo Bianconi head->gso_segs = 1;
23728c78c1e5SLorenzo Bianconi head->xdpf = xdpf;
23738c78c1e5SLorenzo Bianconi
23748c78c1e5SLorenzo Bianconi olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
23758c78c1e5SLorenzo Bianconi desc->read.olinfo_status = cpu_to_le32(olinfo_status);
23768c78c1e5SLorenzo Bianconi
23778c78c1e5SLorenzo Bianconi for (;;) {
23788c78c1e5SLorenzo Bianconi dma_addr_t dma;
23798c78c1e5SLorenzo Bianconi
23808c78c1e5SLorenzo Bianconi dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
23818c78c1e5SLorenzo Bianconi if (dma_mapping_error(ring->dev, dma)) {
23828c78c1e5SLorenzo Bianconi netdev_err_once(ring->netdev,
23838c78c1e5SLorenzo Bianconi "Failed to map DMA for TX\n");
23848c78c1e5SLorenzo Bianconi goto unmap;
23858c78c1e5SLorenzo Bianconi }
23868c78c1e5SLorenzo Bianconi
23878c78c1e5SLorenzo Bianconi dma_unmap_len_set(buffer, len, len);
23888c78c1e5SLorenzo Bianconi dma_unmap_addr_set(buffer, dma, dma);
238973f1071cSAndre Guedes
239073f1071cSAndre Guedes cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
23918c78c1e5SLorenzo Bianconi IGC_ADVTXD_DCMD_IFCS | len;
239273f1071cSAndre Guedes
239373f1071cSAndre Guedes desc->read.cmd_type_len = cpu_to_le32(cmd_type);
23948c78c1e5SLorenzo Bianconi desc->read.buffer_addr = cpu_to_le64(dma);
239573f1071cSAndre Guedes
23968c78c1e5SLorenzo Bianconi buffer->protocol = 0;
239773f1071cSAndre Guedes
23988c78c1e5SLorenzo Bianconi if (++index == ring->count)
23998c78c1e5SLorenzo Bianconi index = 0;
240073f1071cSAndre Guedes
24018c78c1e5SLorenzo Bianconi if (i == nr_frags)
24028c78c1e5SLorenzo Bianconi break;
24038c78c1e5SLorenzo Bianconi
24048c78c1e5SLorenzo Bianconi buffer = &ring->tx_buffer_info[index];
24058c78c1e5SLorenzo Bianconi desc = IGC_TX_DESC(ring, index);
24068c78c1e5SLorenzo Bianconi desc->read.olinfo_status = 0;
24078c78c1e5SLorenzo Bianconi
24088c78c1e5SLorenzo Bianconi data = skb_frag_address(&sinfo->frags[i]);
24098c78c1e5SLorenzo Bianconi len = skb_frag_size(&sinfo->frags[i]);
24108c78c1e5SLorenzo Bianconi i++;
24118c78c1e5SLorenzo Bianconi }
24128c78c1e5SLorenzo Bianconi desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
24138c78c1e5SLorenzo Bianconi
24148c78c1e5SLorenzo Bianconi netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
24158c78c1e5SLorenzo Bianconi /* set the timestamp */
24168c78c1e5SLorenzo Bianconi head->time_stamp = jiffies;
24178c78c1e5SLorenzo Bianconi /* set next_to_watch value indicating a packet is present */
24188c78c1e5SLorenzo Bianconi head->next_to_watch = desc;
24198c78c1e5SLorenzo Bianconi ring->next_to_use = index;
242073f1071cSAndre Guedes
242173f1071cSAndre Guedes return 0;
24228c78c1e5SLorenzo Bianconi
24238c78c1e5SLorenzo Bianconi unmap:
24248c78c1e5SLorenzo Bianconi for (;;) {
24258c78c1e5SLorenzo Bianconi buffer = &ring->tx_buffer_info[index];
24268c78c1e5SLorenzo Bianconi if (dma_unmap_len(buffer, len))
24278c78c1e5SLorenzo Bianconi dma_unmap_page(ring->dev,
24288c78c1e5SLorenzo Bianconi dma_unmap_addr(buffer, dma),
24298c78c1e5SLorenzo Bianconi dma_unmap_len(buffer, len),
24308c78c1e5SLorenzo Bianconi DMA_TO_DEVICE);
24318c78c1e5SLorenzo Bianconi dma_unmap_len_set(buffer, len, 0);
24328c78c1e5SLorenzo Bianconi if (buffer == head)
24338c78c1e5SLorenzo Bianconi break;
24348c78c1e5SLorenzo Bianconi
24358c78c1e5SLorenzo Bianconi if (!index)
24368c78c1e5SLorenzo Bianconi index += ring->count;
24378c78c1e5SLorenzo Bianconi index--;
24388c78c1e5SLorenzo Bianconi }
24398c78c1e5SLorenzo Bianconi
24408c78c1e5SLorenzo Bianconi return -ENOMEM;
244173f1071cSAndre Guedes }
244273f1071cSAndre Guedes
igc_xdp_get_tx_ring(struct igc_adapter * adapter,int cpu)244373f1071cSAndre Guedes static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
244473f1071cSAndre Guedes int cpu)
244573f1071cSAndre Guedes {
244673f1071cSAndre Guedes int index = cpu;
244773f1071cSAndre Guedes
244873f1071cSAndre Guedes if (unlikely(index < 0))
244973f1071cSAndre Guedes index = 0;
245073f1071cSAndre Guedes
245173f1071cSAndre Guedes while (index >= adapter->num_tx_queues)
245273f1071cSAndre Guedes index -= adapter->num_tx_queues;
245373f1071cSAndre Guedes
245473f1071cSAndre Guedes return adapter->tx_ring[index];
245573f1071cSAndre Guedes }
245673f1071cSAndre Guedes
igc_xdp_xmit_back(struct igc_adapter * adapter,struct xdp_buff * xdp)245773f1071cSAndre Guedes static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp)
245873f1071cSAndre Guedes {
245973f1071cSAndre Guedes struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
246073f1071cSAndre Guedes int cpu = smp_processor_id();
246173f1071cSAndre Guedes struct netdev_queue *nq;
246273f1071cSAndre Guedes struct igc_ring *ring;
246373f1071cSAndre Guedes int res;
246473f1071cSAndre Guedes
246573f1071cSAndre Guedes if (unlikely(!xdpf))
246673f1071cSAndre Guedes return -EFAULT;
246773f1071cSAndre Guedes
246873f1071cSAndre Guedes ring = igc_xdp_get_tx_ring(adapter, cpu);
246973f1071cSAndre Guedes nq = txring_txq(ring);
247073f1071cSAndre Guedes
247173f1071cSAndre Guedes __netif_tx_lock(nq, cpu);
247295b68148SKurt Kanzenbach /* Avoid transmit queue timeout since we share it with the slow path */
247395b68148SKurt Kanzenbach txq_trans_cond_update(nq);
247473f1071cSAndre Guedes res = igc_xdp_init_tx_descriptor(ring, xdpf);
247573f1071cSAndre Guedes __netif_tx_unlock(nq);
247673f1071cSAndre Guedes return res;
247773f1071cSAndre Guedes }
247873f1071cSAndre Guedes
247973a6e372SAndre Guedes /* This function assumes rcu_read_lock() is held by the caller. */
__igc_xdp_run_prog(struct igc_adapter * adapter,struct bpf_prog * prog,struct xdp_buff * xdp)248073a6e372SAndre Guedes static int __igc_xdp_run_prog(struct igc_adapter *adapter,
248173a6e372SAndre Guedes struct bpf_prog *prog,
248273a6e372SAndre Guedes struct xdp_buff *xdp)
248373a6e372SAndre Guedes {
248473a6e372SAndre Guedes u32 act = bpf_prog_run_xdp(prog, xdp);
248573a6e372SAndre Guedes
248673a6e372SAndre Guedes switch (act) {
248773a6e372SAndre Guedes case XDP_PASS:
248873a6e372SAndre Guedes return IGC_XDP_PASS;
248973a6e372SAndre Guedes case XDP_TX:
249073f1071cSAndre Guedes if (igc_xdp_xmit_back(adapter, xdp) < 0)
249145ce0859SMagnus Karlsson goto out_failure;
249212628565SDavid S. Miller return IGC_XDP_TX;
249373a6e372SAndre Guedes case XDP_REDIRECT:
24944ff32036SAndre Guedes if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0)
249545ce0859SMagnus Karlsson goto out_failure;
249612628565SDavid S. Miller return IGC_XDP_REDIRECT;
24974ff32036SAndre Guedes break;
249873a6e372SAndre Guedes default:
2499c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
250073a6e372SAndre Guedes fallthrough;
250173a6e372SAndre Guedes case XDP_ABORTED:
250245ce0859SMagnus Karlsson out_failure:
250373a6e372SAndre Guedes trace_xdp_exception(adapter->netdev, prog, act);
250473a6e372SAndre Guedes fallthrough;
250573a6e372SAndre Guedes case XDP_DROP:
250673a6e372SAndre Guedes return IGC_XDP_CONSUMED;
250773a6e372SAndre Guedes }
250873a6e372SAndre Guedes }
250973a6e372SAndre Guedes
igc_xdp_run_prog(struct igc_adapter * adapter,struct xdp_buff * xdp)2510c9a11c23SSasha Neftin static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
2511c9a11c23SSasha Neftin struct xdp_buff *xdp)
2512c9a11c23SSasha Neftin {
2513c9a11c23SSasha Neftin struct bpf_prog *prog;
2514c9a11c23SSasha Neftin int res;
251526575105SAndre Guedes
251626575105SAndre Guedes prog = READ_ONCE(adapter->xdp_prog);
251726575105SAndre Guedes if (!prog) {
251826575105SAndre Guedes res = IGC_XDP_PASS;
251949589b23SToke Høiland-Jørgensen goto out;
252026575105SAndre Guedes }
252126575105SAndre Guedes
252273a6e372SAndre Guedes res = __igc_xdp_run_prog(adapter, prog, xdp);
252326575105SAndre Guedes
252449589b23SToke Høiland-Jørgensen out:
252526575105SAndre Guedes return ERR_PTR(-res);
252626575105SAndre Guedes }
252726575105SAndre Guedes
252873f1071cSAndre Guedes /* This function assumes __netif_tx_lock is held by the caller. */
igc_flush_tx_descriptors(struct igc_ring * ring)252973f1071cSAndre Guedes static void igc_flush_tx_descriptors(struct igc_ring *ring)
253073f1071cSAndre Guedes {
253173f1071cSAndre Guedes /* Once tail pointer is updated, hardware can fetch the descriptors
253273f1071cSAndre Guedes * any time so we issue a write membar here to ensure all memory
253373f1071cSAndre Guedes * writes are complete before the tail pointer is updated.
253473f1071cSAndre Guedes */
253573f1071cSAndre Guedes wmb();
253673f1071cSAndre Guedes writel(ring->next_to_use, ring->tail);
253773f1071cSAndre Guedes }
253873f1071cSAndre Guedes
igc_finalize_xdp(struct igc_adapter * adapter,int status)253973f1071cSAndre Guedes static void igc_finalize_xdp(struct igc_adapter *adapter, int status)
254073f1071cSAndre Guedes {
254173f1071cSAndre Guedes int cpu = smp_processor_id();
254273f1071cSAndre Guedes struct netdev_queue *nq;
254373f1071cSAndre Guedes struct igc_ring *ring;
254473f1071cSAndre Guedes
254573f1071cSAndre Guedes if (status & IGC_XDP_TX) {
254673f1071cSAndre Guedes ring = igc_xdp_get_tx_ring(adapter, cpu);
254773f1071cSAndre Guedes nq = txring_txq(ring);
254873f1071cSAndre Guedes
254973f1071cSAndre Guedes __netif_tx_lock(nq, cpu);
255073f1071cSAndre Guedes igc_flush_tx_descriptors(ring);
255173f1071cSAndre Guedes __netif_tx_unlock(nq);
255273f1071cSAndre Guedes }
25534ff32036SAndre Guedes
25544ff32036SAndre Guedes if (status & IGC_XDP_REDIRECT)
25554ff32036SAndre Guedes xdp_do_flush();
255673f1071cSAndre Guedes }
255773f1071cSAndre Guedes
igc_update_rx_stats(struct igc_q_vector * q_vector,unsigned int packets,unsigned int bytes)2558a27e6e73SAndre Guedes static void igc_update_rx_stats(struct igc_q_vector *q_vector,
2559a27e6e73SAndre Guedes unsigned int packets, unsigned int bytes)
2560a27e6e73SAndre Guedes {
2561a27e6e73SAndre Guedes struct igc_ring *ring = q_vector->rx.ring;
2562a27e6e73SAndre Guedes
2563a27e6e73SAndre Guedes u64_stats_update_begin(&ring->rx_syncp);
2564a27e6e73SAndre Guedes ring->rx_stats.packets += packets;
2565a27e6e73SAndre Guedes ring->rx_stats.bytes += bytes;
2566a27e6e73SAndre Guedes u64_stats_update_end(&ring->rx_syncp);
2567a27e6e73SAndre Guedes
2568a27e6e73SAndre Guedes q_vector->rx.total_packets += packets;
2569a27e6e73SAndre Guedes q_vector->rx.total_bytes += bytes;
2570a27e6e73SAndre Guedes }
2571a27e6e73SAndre Guedes
igc_clean_rx_irq(struct igc_q_vector * q_vector,const int budget)25720507ef8aSSasha Neftin static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
25730507ef8aSSasha Neftin {
25740507ef8aSSasha Neftin unsigned int total_bytes = 0, total_packets = 0;
257573f1071cSAndre Guedes struct igc_adapter *adapter = q_vector->adapter;
25760507ef8aSSasha Neftin struct igc_ring *rx_ring = q_vector->rx.ring;
25770507ef8aSSasha Neftin struct sk_buff *skb = rx_ring->skb;
25780507ef8aSSasha Neftin u16 cleaned_count = igc_desc_unused(rx_ring);
25794ff32036SAndre Guedes int xdp_status = 0, rx_buffer_pgcnt;
25800507ef8aSSasha Neftin
25810507ef8aSSasha Neftin while (likely(total_packets < budget)) {
25820507ef8aSSasha Neftin union igc_adv_rx_desc *rx_desc;
25830507ef8aSSasha Neftin struct igc_rx_buffer *rx_buffer;
258473f1071cSAndre Guedes unsigned int size, truesize;
258573b7123dSJesper Dangaard Brouer struct igc_xdp_buff ctx;
2586e1ed4f92SAndre Guedes ktime_t timestamp = 0;
2587e1ed4f92SAndre Guedes int pkt_offset = 0;
258826575105SAndre Guedes void *pktbuf;
25890507ef8aSSasha Neftin
25900507ef8aSSasha Neftin /* return some buffers to hardware, one at a time is too slow */
25910507ef8aSSasha Neftin if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
25920507ef8aSSasha Neftin igc_alloc_rx_buffers(rx_ring, cleaned_count);
25930507ef8aSSasha Neftin cleaned_count = 0;
25940507ef8aSSasha Neftin }
25950507ef8aSSasha Neftin
25960507ef8aSSasha Neftin rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
25970507ef8aSSasha Neftin size = le16_to_cpu(rx_desc->wb.upper.length);
25980507ef8aSSasha Neftin if (!size)
25990507ef8aSSasha Neftin break;
26000507ef8aSSasha Neftin
26010507ef8aSSasha Neftin /* This memory barrier is needed to keep us from reading
26020507ef8aSSasha Neftin * any other fields out of the rx_desc until we know the
26030507ef8aSSasha Neftin * descriptor has been written back
26040507ef8aSSasha Neftin */
26050507ef8aSSasha Neftin dma_rmb();
26060507ef8aSSasha Neftin
26074ff32036SAndre Guedes rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
260873f1071cSAndre Guedes truesize = igc_get_rx_frame_truesize(rx_ring, size);
26090507ef8aSSasha Neftin
261026575105SAndre Guedes pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
2611e1ed4f92SAndre Guedes
261226575105SAndre Guedes if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
2613e1ed4f92SAndre Guedes timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2614e1ed4f92SAndre Guedes pktbuf);
2615d6772667SJesper Dangaard Brouer ctx.rx_ts = timestamp;
2616e1ed4f92SAndre Guedes pkt_offset = IGC_TS_HDR_LEN;
2617e1ed4f92SAndre Guedes size -= IGC_TS_HDR_LEN;
2618e1ed4f92SAndre Guedes }
2619e1ed4f92SAndre Guedes
262026575105SAndre Guedes if (!skb) {
262173b7123dSJesper Dangaard Brouer xdp_init_buff(&ctx.xdp, truesize, &rx_ring->xdp_rxq);
262273b7123dSJesper Dangaard Brouer xdp_prepare_buff(&ctx.xdp, pktbuf - igc_rx_offset(rx_ring),
2623f51b5e2bSJesper Dangaard Brouer igc_rx_offset(rx_ring) + pkt_offset,
2624f51b5e2bSJesper Dangaard Brouer size, true);
262573b7123dSJesper Dangaard Brouer xdp_buff_clear_frags_flag(&ctx.xdp);
26268416814fSJesper Dangaard Brouer ctx.rx_desc = rx_desc;
262726575105SAndre Guedes
262873b7123dSJesper Dangaard Brouer skb = igc_xdp_run_prog(adapter, &ctx.xdp);
262926575105SAndre Guedes }
263026575105SAndre Guedes
263126575105SAndre Guedes if (IS_ERR(skb)) {
263273f1071cSAndre Guedes unsigned int xdp_res = -PTR_ERR(skb);
263373f1071cSAndre Guedes
263473f1071cSAndre Guedes switch (xdp_res) {
263573f1071cSAndre Guedes case IGC_XDP_CONSUMED:
263626575105SAndre Guedes rx_buffer->pagecnt_bias++;
263773f1071cSAndre Guedes break;
263873f1071cSAndre Guedes case IGC_XDP_TX:
26394ff32036SAndre Guedes case IGC_XDP_REDIRECT:
264073f1071cSAndre Guedes igc_rx_buffer_flip(rx_buffer, truesize);
264173f1071cSAndre Guedes xdp_status |= xdp_res;
264273f1071cSAndre Guedes break;
264373f1071cSAndre Guedes }
264473f1071cSAndre Guedes
264526575105SAndre Guedes total_packets++;
264626575105SAndre Guedes total_bytes += size;
264726575105SAndre Guedes } else if (skb)
26480507ef8aSSasha Neftin igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
26490507ef8aSSasha Neftin else if (ring_uses_build_skb(rx_ring))
265073b7123dSJesper Dangaard Brouer skb = igc_build_skb(rx_ring, rx_buffer, &ctx.xdp);
26510507ef8aSSasha Neftin else
265273b7123dSJesper Dangaard Brouer skb = igc_construct_skb(rx_ring, rx_buffer, &ctx.xdp,
265326575105SAndre Guedes timestamp);
26540507ef8aSSasha Neftin
26550507ef8aSSasha Neftin /* exit if we failed to retrieve a buffer */
26560507ef8aSSasha Neftin if (!skb) {
26570507ef8aSSasha Neftin rx_ring->rx_stats.alloc_failed++;
26580507ef8aSSasha Neftin rx_buffer->pagecnt_bias++;
26590507ef8aSSasha Neftin break;
26600507ef8aSSasha Neftin }
26610507ef8aSSasha Neftin
26624ff32036SAndre Guedes igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
26630507ef8aSSasha Neftin cleaned_count++;
26640507ef8aSSasha Neftin
26650507ef8aSSasha Neftin /* fetch next buffer in frame if non-eop */
26660507ef8aSSasha Neftin if (igc_is_non_eop(rx_ring, rx_desc))
26670507ef8aSSasha Neftin continue;
26680507ef8aSSasha Neftin
26690507ef8aSSasha Neftin /* verify the packet layout is correct */
26700507ef8aSSasha Neftin if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
26710507ef8aSSasha Neftin skb = NULL;
26720507ef8aSSasha Neftin continue;
26730507ef8aSSasha Neftin }
26740507ef8aSSasha Neftin
26750507ef8aSSasha Neftin /* probably a little skewed due to removing CRC */
26760507ef8aSSasha Neftin total_bytes += skb->len;
26770507ef8aSSasha Neftin
26783a66abe9SAndre Guedes /* populate checksum, VLAN, and protocol */
26790507ef8aSSasha Neftin igc_process_skb_fields(rx_ring, rx_desc, skb);
26800507ef8aSSasha Neftin
26810507ef8aSSasha Neftin napi_gro_receive(&q_vector->napi, skb);
26820507ef8aSSasha Neftin
26830507ef8aSSasha Neftin /* reset skb pointer */
26840507ef8aSSasha Neftin skb = NULL;
26850507ef8aSSasha Neftin
26860507ef8aSSasha Neftin /* update budget accounting */
26870507ef8aSSasha Neftin total_packets++;
26880507ef8aSSasha Neftin }
26890507ef8aSSasha Neftin
269073f1071cSAndre Guedes if (xdp_status)
269173f1071cSAndre Guedes igc_finalize_xdp(adapter, xdp_status);
269273f1071cSAndre Guedes
26930507ef8aSSasha Neftin /* place incomplete frames back on ring for completion */
26940507ef8aSSasha Neftin rx_ring->skb = skb;
26950507ef8aSSasha Neftin
2696a27e6e73SAndre Guedes igc_update_rx_stats(q_vector, total_packets, total_bytes);
26970507ef8aSSasha Neftin
26980507ef8aSSasha Neftin if (cleaned_count)
26990507ef8aSSasha Neftin igc_alloc_rx_buffers(rx_ring, cleaned_count);
27000507ef8aSSasha Neftin
27010507ef8aSSasha Neftin return total_packets;
27020507ef8aSSasha Neftin }
27030507ef8aSSasha Neftin
igc_construct_skb_zc(struct igc_ring * ring,struct xdp_buff * xdp)2704fc9df2a0SAndre Guedes static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
2705fc9df2a0SAndre Guedes struct xdp_buff *xdp)
2706fc9df2a0SAndre Guedes {
2707f9e61d36SAlexander Lobakin unsigned int totalsize = xdp->data_end - xdp->data_meta;
2708fc9df2a0SAndre Guedes unsigned int metasize = xdp->data - xdp->data_meta;
2709fc9df2a0SAndre Guedes struct sk_buff *skb;
2710fc9df2a0SAndre Guedes
2711f9e61d36SAlexander Lobakin net_prefetch(xdp->data_meta);
2712f9e61d36SAlexander Lobakin
2713f9e61d36SAlexander Lobakin skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,
2714fc9df2a0SAndre Guedes GFP_ATOMIC | __GFP_NOWARN);
2715fc9df2a0SAndre Guedes if (unlikely(!skb))
2716fc9df2a0SAndre Guedes return NULL;
2717fc9df2a0SAndre Guedes
2718f9e61d36SAlexander Lobakin memcpy(__skb_put(skb, totalsize), xdp->data_meta,
2719f9e61d36SAlexander Lobakin ALIGN(totalsize, sizeof(long)));
2720f9e61d36SAlexander Lobakin
27214fa8fcd3SJesper Dangaard Brouer if (metasize) {
2722fc9df2a0SAndre Guedes skb_metadata_set(skb, metasize);
27234fa8fcd3SJesper Dangaard Brouer __skb_pull(skb, metasize);
27244fa8fcd3SJesper Dangaard Brouer }
2725fc9df2a0SAndre Guedes
2726fc9df2a0SAndre Guedes return skb;
2727fc9df2a0SAndre Guedes }
2728fc9df2a0SAndre Guedes
igc_dispatch_skb_zc(struct igc_q_vector * q_vector,union igc_adv_rx_desc * desc,struct xdp_buff * xdp,ktime_t timestamp)2729fc9df2a0SAndre Guedes static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,
2730fc9df2a0SAndre Guedes union igc_adv_rx_desc *desc,
2731fc9df2a0SAndre Guedes struct xdp_buff *xdp,
2732fc9df2a0SAndre Guedes ktime_t timestamp)
2733fc9df2a0SAndre Guedes {
2734fc9df2a0SAndre Guedes struct igc_ring *ring = q_vector->rx.ring;
2735fc9df2a0SAndre Guedes struct sk_buff *skb;
2736fc9df2a0SAndre Guedes
2737fc9df2a0SAndre Guedes skb = igc_construct_skb_zc(ring, xdp);
2738fc9df2a0SAndre Guedes if (!skb) {
2739fc9df2a0SAndre Guedes ring->rx_stats.alloc_failed++;
2740fc9df2a0SAndre Guedes return;
2741fc9df2a0SAndre Guedes }
2742fc9df2a0SAndre Guedes
2743fc9df2a0SAndre Guedes if (timestamp)
2744fc9df2a0SAndre Guedes skb_hwtstamps(skb)->hwtstamp = timestamp;
2745fc9df2a0SAndre Guedes
2746fc9df2a0SAndre Guedes if (igc_cleanup_headers(ring, desc, skb))
2747fc9df2a0SAndre Guedes return;
2748fc9df2a0SAndre Guedes
2749fc9df2a0SAndre Guedes igc_process_skb_fields(ring, desc, skb);
2750fc9df2a0SAndre Guedes napi_gro_receive(&q_vector->napi, skb);
2751fc9df2a0SAndre Guedes }
2752fc9df2a0SAndre Guedes
xsk_buff_to_igc_ctx(struct xdp_buff * xdp)27538416814fSJesper Dangaard Brouer static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp)
27548416814fSJesper Dangaard Brouer {
27558416814fSJesper Dangaard Brouer /* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The
27568416814fSJesper Dangaard Brouer * igc_xdp_buff shares its layout with xdp_buff_xsk and private
27578416814fSJesper Dangaard Brouer * igc_xdp_buff fields fall into xdp_buff_xsk->cb
27588416814fSJesper Dangaard Brouer */
27598416814fSJesper Dangaard Brouer return (struct igc_xdp_buff *)xdp;
27608416814fSJesper Dangaard Brouer }
27618416814fSJesper Dangaard Brouer
igc_clean_rx_irq_zc(struct igc_q_vector * q_vector,const int budget)2762fc9df2a0SAndre Guedes static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
2763fc9df2a0SAndre Guedes {
2764fc9df2a0SAndre Guedes struct igc_adapter *adapter = q_vector->adapter;
2765fc9df2a0SAndre Guedes struct igc_ring *ring = q_vector->rx.ring;
2766fc9df2a0SAndre Guedes u16 cleaned_count = igc_desc_unused(ring);
2767fc9df2a0SAndre Guedes int total_bytes = 0, total_packets = 0;
2768fc9df2a0SAndre Guedes u16 ntc = ring->next_to_clean;
2769fc9df2a0SAndre Guedes struct bpf_prog *prog;
2770fc9df2a0SAndre Guedes bool failure = false;
2771fc9df2a0SAndre Guedes int xdp_status = 0;
2772fc9df2a0SAndre Guedes
2773fc9df2a0SAndre Guedes rcu_read_lock();
2774fc9df2a0SAndre Guedes
2775fc9df2a0SAndre Guedes prog = READ_ONCE(adapter->xdp_prog);
2776fc9df2a0SAndre Guedes
2777fc9df2a0SAndre Guedes while (likely(total_packets < budget)) {
2778fc9df2a0SAndre Guedes union igc_adv_rx_desc *desc;
2779fc9df2a0SAndre Guedes struct igc_rx_buffer *bi;
27808416814fSJesper Dangaard Brouer struct igc_xdp_buff *ctx;
2781fc9df2a0SAndre Guedes ktime_t timestamp = 0;
2782fc9df2a0SAndre Guedes unsigned int size;
2783fc9df2a0SAndre Guedes int res;
2784fc9df2a0SAndre Guedes
2785fc9df2a0SAndre Guedes desc = IGC_RX_DESC(ring, ntc);
2786fc9df2a0SAndre Guedes size = le16_to_cpu(desc->wb.upper.length);
2787fc9df2a0SAndre Guedes if (!size)
2788fc9df2a0SAndre Guedes break;
2789fc9df2a0SAndre Guedes
2790fc9df2a0SAndre Guedes /* This memory barrier is needed to keep us from reading
2791fc9df2a0SAndre Guedes * any other fields out of the rx_desc until we know the
2792fc9df2a0SAndre Guedes * descriptor has been written back
2793fc9df2a0SAndre Guedes */
2794fc9df2a0SAndre Guedes dma_rmb();
2795fc9df2a0SAndre Guedes
2796fc9df2a0SAndre Guedes bi = &ring->rx_buffer_info[ntc];
2797fc9df2a0SAndre Guedes
27988416814fSJesper Dangaard Brouer ctx = xsk_buff_to_igc_ctx(bi->xdp);
27998416814fSJesper Dangaard Brouer ctx->rx_desc = desc;
28008416814fSJesper Dangaard Brouer
2801fc9df2a0SAndre Guedes if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
2802fc9df2a0SAndre Guedes timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2803fc9df2a0SAndre Guedes bi->xdp->data);
2804d6772667SJesper Dangaard Brouer ctx->rx_ts = timestamp;
2805fc9df2a0SAndre Guedes
2806fc9df2a0SAndre Guedes bi->xdp->data += IGC_TS_HDR_LEN;
2807fc9df2a0SAndre Guedes
2808fc9df2a0SAndre Guedes /* HW timestamp has been copied into local variable. Metadata
2809fc9df2a0SAndre Guedes * length when XDP program is called should be 0.
2810fc9df2a0SAndre Guedes */
2811fc9df2a0SAndre Guedes bi->xdp->data_meta += IGC_TS_HDR_LEN;
2812fc9df2a0SAndre Guedes size -= IGC_TS_HDR_LEN;
2813fc9df2a0SAndre Guedes }
2814fc9df2a0SAndre Guedes
2815fc9df2a0SAndre Guedes bi->xdp->data_end = bi->xdp->data + size;
2816fc9df2a0SAndre Guedes xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool);
2817fc9df2a0SAndre Guedes
2818fc9df2a0SAndre Guedes res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
2819fc9df2a0SAndre Guedes switch (res) {
2820fc9df2a0SAndre Guedes case IGC_XDP_PASS:
2821fc9df2a0SAndre Guedes igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);
2822fc9df2a0SAndre Guedes fallthrough;
2823fc9df2a0SAndre Guedes case IGC_XDP_CONSUMED:
2824fc9df2a0SAndre Guedes xsk_buff_free(bi->xdp);
2825fc9df2a0SAndre Guedes break;
2826fc9df2a0SAndre Guedes case IGC_XDP_TX:
2827fc9df2a0SAndre Guedes case IGC_XDP_REDIRECT:
2828fc9df2a0SAndre Guedes xdp_status |= res;
2829fc9df2a0SAndre Guedes break;
2830fc9df2a0SAndre Guedes }
2831fc9df2a0SAndre Guedes
2832fc9df2a0SAndre Guedes bi->xdp = NULL;
2833fc9df2a0SAndre Guedes total_bytes += size;
2834fc9df2a0SAndre Guedes total_packets++;
2835fc9df2a0SAndre Guedes cleaned_count++;
2836fc9df2a0SAndre Guedes ntc++;
2837fc9df2a0SAndre Guedes if (ntc == ring->count)
2838fc9df2a0SAndre Guedes ntc = 0;
2839fc9df2a0SAndre Guedes }
2840fc9df2a0SAndre Guedes
2841fc9df2a0SAndre Guedes ring->next_to_clean = ntc;
2842fc9df2a0SAndre Guedes rcu_read_unlock();
2843fc9df2a0SAndre Guedes
2844fc9df2a0SAndre Guedes if (cleaned_count >= IGC_RX_BUFFER_WRITE)
2845fc9df2a0SAndre Guedes failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count);
2846fc9df2a0SAndre Guedes
2847fc9df2a0SAndre Guedes if (xdp_status)
2848fc9df2a0SAndre Guedes igc_finalize_xdp(adapter, xdp_status);
2849fc9df2a0SAndre Guedes
2850fc9df2a0SAndre Guedes igc_update_rx_stats(q_vector, total_packets, total_bytes);
2851fc9df2a0SAndre Guedes
2852fc9df2a0SAndre Guedes if (xsk_uses_need_wakeup(ring->xsk_pool)) {
2853fc9df2a0SAndre Guedes if (failure || ring->next_to_clean == ring->next_to_use)
2854fc9df2a0SAndre Guedes xsk_set_rx_need_wakeup(ring->xsk_pool);
2855fc9df2a0SAndre Guedes else
2856fc9df2a0SAndre Guedes xsk_clear_rx_need_wakeup(ring->xsk_pool);
2857fc9df2a0SAndre Guedes return total_packets;
2858fc9df2a0SAndre Guedes }
2859fc9df2a0SAndre Guedes
2860fc9df2a0SAndre Guedes return failure ? budget : total_packets;
2861fc9df2a0SAndre Guedes }
2862fc9df2a0SAndre Guedes
igc_update_tx_stats(struct igc_q_vector * q_vector,unsigned int packets,unsigned int bytes)2863a27e6e73SAndre Guedes static void igc_update_tx_stats(struct igc_q_vector *q_vector,
2864a27e6e73SAndre Guedes unsigned int packets, unsigned int bytes)
2865a27e6e73SAndre Guedes {
2866a27e6e73SAndre Guedes struct igc_ring *ring = q_vector->tx.ring;
2867a27e6e73SAndre Guedes
2868a27e6e73SAndre Guedes u64_stats_update_begin(&ring->tx_syncp);
2869a27e6e73SAndre Guedes ring->tx_stats.bytes += bytes;
2870a27e6e73SAndre Guedes ring->tx_stats.packets += packets;
2871a27e6e73SAndre Guedes u64_stats_update_end(&ring->tx_syncp);
2872a27e6e73SAndre Guedes
2873a27e6e73SAndre Guedes q_vector->tx.total_bytes += bytes;
2874a27e6e73SAndre Guedes q_vector->tx.total_packets += packets;
2875a27e6e73SAndre Guedes }
2876a27e6e73SAndre Guedes
igc_xdp_xmit_zc(struct igc_ring * ring)28779acf59a7SAndre Guedes static void igc_xdp_xmit_zc(struct igc_ring *ring)
28789acf59a7SAndre Guedes {
28799acf59a7SAndre Guedes struct xsk_buff_pool *pool = ring->xsk_pool;
28809acf59a7SAndre Guedes struct netdev_queue *nq = txring_txq(ring);
28819acf59a7SAndre Guedes union igc_adv_tx_desc *tx_desc = NULL;
28829acf59a7SAndre Guedes int cpu = smp_processor_id();
28839acf59a7SAndre Guedes struct xdp_desc xdp_desc;
288478adb4bcSFlorian Kauer u16 budget, ntu;
28859acf59a7SAndre Guedes
28869acf59a7SAndre Guedes if (!netif_carrier_ok(ring->netdev))
28879acf59a7SAndre Guedes return;
28889acf59a7SAndre Guedes
28899acf59a7SAndre Guedes __netif_tx_lock(nq, cpu);
28909acf59a7SAndre Guedes
289195b68148SKurt Kanzenbach /* Avoid transmit queue timeout since we share it with the slow path */
289295b68148SKurt Kanzenbach txq_trans_cond_update(nq);
289395b68148SKurt Kanzenbach
289478adb4bcSFlorian Kauer ntu = ring->next_to_use;
28959acf59a7SAndre Guedes budget = igc_desc_unused(ring);
28969acf59a7SAndre Guedes
28979acf59a7SAndre Guedes while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) {
28989acf59a7SAndre Guedes u32 cmd_type, olinfo_status;
28999acf59a7SAndre Guedes struct igc_tx_buffer *bi;
29009acf59a7SAndre Guedes dma_addr_t dma;
29019acf59a7SAndre Guedes
29029acf59a7SAndre Guedes cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
29039acf59a7SAndre Guedes IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
29049acf59a7SAndre Guedes xdp_desc.len;
29059acf59a7SAndre Guedes olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT;
29069acf59a7SAndre Guedes
29079acf59a7SAndre Guedes dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
29089acf59a7SAndre Guedes xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len);
29099acf59a7SAndre Guedes
29109acf59a7SAndre Guedes tx_desc = IGC_TX_DESC(ring, ntu);
29119acf59a7SAndre Guedes tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
29129acf59a7SAndre Guedes tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
29139acf59a7SAndre Guedes tx_desc->read.buffer_addr = cpu_to_le64(dma);
29149acf59a7SAndre Guedes
29159acf59a7SAndre Guedes bi = &ring->tx_buffer_info[ntu];
29169acf59a7SAndre Guedes bi->type = IGC_TX_BUFFER_TYPE_XSK;
29179acf59a7SAndre Guedes bi->protocol = 0;
29189acf59a7SAndre Guedes bi->bytecount = xdp_desc.len;
29199acf59a7SAndre Guedes bi->gso_segs = 1;
29209acf59a7SAndre Guedes bi->time_stamp = jiffies;
29219acf59a7SAndre Guedes bi->next_to_watch = tx_desc;
29229acf59a7SAndre Guedes
29239acf59a7SAndre Guedes netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len);
29249acf59a7SAndre Guedes
29259acf59a7SAndre Guedes ntu++;
29269acf59a7SAndre Guedes if (ntu == ring->count)
29279acf59a7SAndre Guedes ntu = 0;
29289acf59a7SAndre Guedes }
29299acf59a7SAndre Guedes
29309acf59a7SAndre Guedes ring->next_to_use = ntu;
29319acf59a7SAndre Guedes if (tx_desc) {
29329acf59a7SAndre Guedes igc_flush_tx_descriptors(ring);
29339acf59a7SAndre Guedes xsk_tx_release(pool);
29349acf59a7SAndre Guedes }
29359acf59a7SAndre Guedes
29369acf59a7SAndre Guedes __netif_tx_unlock(nq);
29379acf59a7SAndre Guedes }
29389acf59a7SAndre Guedes
29390507ef8aSSasha Neftin /**
29400507ef8aSSasha Neftin * igc_clean_tx_irq - Reclaim resources after transmit completes
29410507ef8aSSasha Neftin * @q_vector: pointer to q_vector containing needed info
29420507ef8aSSasha Neftin * @napi_budget: Used to determine if we are in netpoll
29430507ef8aSSasha Neftin *
29440507ef8aSSasha Neftin * returns true if ring is completely cleaned
29450507ef8aSSasha Neftin */
igc_clean_tx_irq(struct igc_q_vector * q_vector,int napi_budget)29460507ef8aSSasha Neftin static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
29470507ef8aSSasha Neftin {
29480507ef8aSSasha Neftin struct igc_adapter *adapter = q_vector->adapter;
29490507ef8aSSasha Neftin unsigned int total_bytes = 0, total_packets = 0;
29500507ef8aSSasha Neftin unsigned int budget = q_vector->tx.work_limit;
29510507ef8aSSasha Neftin struct igc_ring *tx_ring = q_vector->tx.ring;
29520507ef8aSSasha Neftin unsigned int i = tx_ring->next_to_clean;
29530507ef8aSSasha Neftin struct igc_tx_buffer *tx_buffer;
29540507ef8aSSasha Neftin union igc_adv_tx_desc *tx_desc;
29559acf59a7SAndre Guedes u32 xsk_frames = 0;
29560507ef8aSSasha Neftin
29570507ef8aSSasha Neftin if (test_bit(__IGC_DOWN, &adapter->state))
29580507ef8aSSasha Neftin return true;
29590507ef8aSSasha Neftin
29600507ef8aSSasha Neftin tx_buffer = &tx_ring->tx_buffer_info[i];
29610507ef8aSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, i);
29620507ef8aSSasha Neftin i -= tx_ring->count;
29630507ef8aSSasha Neftin
29640507ef8aSSasha Neftin do {
29650507ef8aSSasha Neftin union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
29660507ef8aSSasha Neftin
29670507ef8aSSasha Neftin /* if next_to_watch is not set then there is no work pending */
29680507ef8aSSasha Neftin if (!eop_desc)
29690507ef8aSSasha Neftin break;
29700507ef8aSSasha Neftin
29710507ef8aSSasha Neftin /* prevent any other reads prior to eop_desc */
29720507ef8aSSasha Neftin smp_rmb();
29730507ef8aSSasha Neftin
29740507ef8aSSasha Neftin /* if DD is not set pending work has not been completed */
29750507ef8aSSasha Neftin if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
29760507ef8aSSasha Neftin break;
29770507ef8aSSasha Neftin
29780507ef8aSSasha Neftin /* clear next_to_watch to prevent false hangs */
29790507ef8aSSasha Neftin tx_buffer->next_to_watch = NULL;
29800507ef8aSSasha Neftin
29810507ef8aSSasha Neftin /* update the statistics for this packet */
29820507ef8aSSasha Neftin total_bytes += tx_buffer->bytecount;
29830507ef8aSSasha Neftin total_packets += tx_buffer->gso_segs;
29840507ef8aSSasha Neftin
2985859b4dfaSAndre Guedes switch (tx_buffer->type) {
29869acf59a7SAndre Guedes case IGC_TX_BUFFER_TYPE_XSK:
29879acf59a7SAndre Guedes xsk_frames++;
29889acf59a7SAndre Guedes break;
2989859b4dfaSAndre Guedes case IGC_TX_BUFFER_TYPE_XDP:
299073f1071cSAndre Guedes xdp_return_frame(tx_buffer->xdpf);
29919acf59a7SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2992859b4dfaSAndre Guedes break;
2993859b4dfaSAndre Guedes case IGC_TX_BUFFER_TYPE_SKB:
29940507ef8aSSasha Neftin napi_consume_skb(tx_buffer->skb, napi_budget);
29959acf59a7SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2996859b4dfaSAndre Guedes break;
2997859b4dfaSAndre Guedes default:
2998859b4dfaSAndre Guedes netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
2999859b4dfaSAndre Guedes break;
3000859b4dfaSAndre Guedes }
30010507ef8aSSasha Neftin
30020507ef8aSSasha Neftin /* clear last DMA location and unmap remaining buffers */
30030507ef8aSSasha Neftin while (tx_desc != eop_desc) {
30040507ef8aSSasha Neftin tx_buffer++;
30050507ef8aSSasha Neftin tx_desc++;
30060507ef8aSSasha Neftin i++;
30070507ef8aSSasha Neftin if (unlikely(!i)) {
30080507ef8aSSasha Neftin i -= tx_ring->count;
30090507ef8aSSasha Neftin tx_buffer = tx_ring->tx_buffer_info;
30100507ef8aSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, 0);
30110507ef8aSSasha Neftin }
30120507ef8aSSasha Neftin
30130507ef8aSSasha Neftin /* unmap any remaining paged data */
301461234295SAndre Guedes if (dma_unmap_len(tx_buffer, len))
301561234295SAndre Guedes igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
30160507ef8aSSasha Neftin }
30170507ef8aSSasha Neftin
30180507ef8aSSasha Neftin /* move us one more past the eop_desc for start of next pkt */
30190507ef8aSSasha Neftin tx_buffer++;
30200507ef8aSSasha Neftin tx_desc++;
30210507ef8aSSasha Neftin i++;
30220507ef8aSSasha Neftin if (unlikely(!i)) {
30230507ef8aSSasha Neftin i -= tx_ring->count;
30240507ef8aSSasha Neftin tx_buffer = tx_ring->tx_buffer_info;
30250507ef8aSSasha Neftin tx_desc = IGC_TX_DESC(tx_ring, 0);
30260507ef8aSSasha Neftin }
30270507ef8aSSasha Neftin
30280507ef8aSSasha Neftin /* issue prefetch for next Tx descriptor */
30290507ef8aSSasha Neftin prefetch(tx_desc);
30300507ef8aSSasha Neftin
30310507ef8aSSasha Neftin /* update budget accounting */
30320507ef8aSSasha Neftin budget--;
30330507ef8aSSasha Neftin } while (likely(budget));
30340507ef8aSSasha Neftin
30350507ef8aSSasha Neftin netdev_tx_completed_queue(txring_txq(tx_ring),
30360507ef8aSSasha Neftin total_packets, total_bytes);
30370507ef8aSSasha Neftin
30380507ef8aSSasha Neftin i += tx_ring->count;
30390507ef8aSSasha Neftin tx_ring->next_to_clean = i;
3040a27e6e73SAndre Guedes
3041a27e6e73SAndre Guedes igc_update_tx_stats(q_vector, total_packets, total_bytes);
30420507ef8aSSasha Neftin
30439acf59a7SAndre Guedes if (tx_ring->xsk_pool) {
30449acf59a7SAndre Guedes if (xsk_frames)
30459acf59a7SAndre Guedes xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
30469acf59a7SAndre Guedes if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
30479acf59a7SAndre Guedes xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
30489acf59a7SAndre Guedes igc_xdp_xmit_zc(tx_ring);
30499acf59a7SAndre Guedes }
30509acf59a7SAndre Guedes
30510507ef8aSSasha Neftin if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
30520507ef8aSSasha Neftin struct igc_hw *hw = &adapter->hw;
30530507ef8aSSasha Neftin
30540507ef8aSSasha Neftin /* Detect a transmit hang in hardware, this serializes the
30550507ef8aSSasha Neftin * check with the clearing of time_stamp and movement of i
30560507ef8aSSasha Neftin */
30570507ef8aSSasha Neftin clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
30580507ef8aSSasha Neftin if (tx_buffer->next_to_watch &&
30590507ef8aSSasha Neftin time_after(jiffies, tx_buffer->time_stamp +
30600507ef8aSSasha Neftin (adapter->tx_timeout_factor * HZ)) &&
30619b275176SSasha Neftin !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF) &&
3062175c2412SMuhammad Husaini Zulkifli (rd32(IGC_TDH(tx_ring->reg_idx)) != readl(tx_ring->tail)) &&
3063175c2412SMuhammad Husaini Zulkifli !tx_ring->oper_gate_closed) {
30640507ef8aSSasha Neftin /* detected Tx unit hang */
306525f06effSAndre Guedes netdev_err(tx_ring->netdev,
30660507ef8aSSasha Neftin "Detected Tx Unit Hang\n"
30670507ef8aSSasha Neftin " Tx Queue <%d>\n"
30680507ef8aSSasha Neftin " TDH <%x>\n"
30690507ef8aSSasha Neftin " TDT <%x>\n"
30700507ef8aSSasha Neftin " next_to_use <%x>\n"
30710507ef8aSSasha Neftin " next_to_clean <%x>\n"
30720507ef8aSSasha Neftin "buffer_info[next_to_clean]\n"
30730507ef8aSSasha Neftin " time_stamp <%lx>\n"
30740507ef8aSSasha Neftin " next_to_watch <%p>\n"
30750507ef8aSSasha Neftin " jiffies <%lx>\n"
30760507ef8aSSasha Neftin " desc.status <%x>\n",
30770507ef8aSSasha Neftin tx_ring->queue_index,
30780507ef8aSSasha Neftin rd32(IGC_TDH(tx_ring->reg_idx)),
30790507ef8aSSasha Neftin readl(tx_ring->tail),
30800507ef8aSSasha Neftin tx_ring->next_to_use,
30810507ef8aSSasha Neftin tx_ring->next_to_clean,
30820507ef8aSSasha Neftin tx_buffer->time_stamp,
30830507ef8aSSasha Neftin tx_buffer->next_to_watch,
30840507ef8aSSasha Neftin jiffies,
30850507ef8aSSasha Neftin tx_buffer->next_to_watch->wb.status);
30860507ef8aSSasha Neftin netif_stop_subqueue(tx_ring->netdev,
30870507ef8aSSasha Neftin tx_ring->queue_index);
30880507ef8aSSasha Neftin
30890507ef8aSSasha Neftin /* we are about to reset, no point in enabling stuff */
30900507ef8aSSasha Neftin return true;
30910507ef8aSSasha Neftin }
30920507ef8aSSasha Neftin }
30930507ef8aSSasha Neftin
30940507ef8aSSasha Neftin #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
30950507ef8aSSasha Neftin if (unlikely(total_packets &&
30960507ef8aSSasha Neftin netif_carrier_ok(tx_ring->netdev) &&
30970507ef8aSSasha Neftin igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
30980507ef8aSSasha Neftin /* Make sure that anybody stopping the queue after this
30990507ef8aSSasha Neftin * sees the new next_to_clean.
31000507ef8aSSasha Neftin */
31010507ef8aSSasha Neftin smp_mb();
31020507ef8aSSasha Neftin if (__netif_subqueue_stopped(tx_ring->netdev,
31030507ef8aSSasha Neftin tx_ring->queue_index) &&
31040507ef8aSSasha Neftin !(test_bit(__IGC_DOWN, &adapter->state))) {
31050507ef8aSSasha Neftin netif_wake_subqueue(tx_ring->netdev,
31060507ef8aSSasha Neftin tx_ring->queue_index);
31070507ef8aSSasha Neftin
31080507ef8aSSasha Neftin u64_stats_update_begin(&tx_ring->tx_syncp);
31090507ef8aSSasha Neftin tx_ring->tx_stats.restart_queue++;
31100507ef8aSSasha Neftin u64_stats_update_end(&tx_ring->tx_syncp);
31110507ef8aSSasha Neftin }
31120507ef8aSSasha Neftin }
31130507ef8aSSasha Neftin
31140507ef8aSSasha Neftin return !!budget;
31150507ef8aSSasha Neftin }
31160507ef8aSSasha Neftin
igc_find_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr)3117750433d0SAndre Guedes static int igc_find_mac_filter(struct igc_adapter *adapter,
3118750433d0SAndre Guedes enum igc_mac_filter_type type, const u8 *addr)
311986a4de66SSasha Neftin {
312086a4de66SSasha Neftin struct igc_hw *hw = &adapter->hw;
3121d66358caSAndre Guedes int max_entries = hw->mac.rar_entry_count;
3122d66358caSAndre Guedes u32 ral, rah;
312386a4de66SSasha Neftin int i;
312486a4de66SSasha Neftin
3125794e5bc8SAndre Guedes for (i = 0; i < max_entries; i++) {
3126d66358caSAndre Guedes ral = rd32(IGC_RAL(i));
3127d66358caSAndre Guedes rah = rd32(IGC_RAH(i));
312886a4de66SSasha Neftin
3129d66358caSAndre Guedes if (!(rah & IGC_RAH_AV))
3130794e5bc8SAndre Guedes continue;
3131750433d0SAndre Guedes if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
3132750433d0SAndre Guedes continue;
3133d66358caSAndre Guedes if ((rah & IGC_RAH_RAH_MASK) !=
3134d66358caSAndre Guedes le16_to_cpup((__le16 *)(addr + 4)))
3135d66358caSAndre Guedes continue;
3136d66358caSAndre Guedes if (ral != le32_to_cpup((__le32 *)(addr)))
313786a4de66SSasha Neftin continue;
313886a4de66SSasha Neftin
313986a4de66SSasha Neftin return i;
314086a4de66SSasha Neftin }
314186a4de66SSasha Neftin
3142794e5bc8SAndre Guedes return -1;
314386a4de66SSasha Neftin }
314486a4de66SSasha Neftin
igc_get_avail_mac_filter_slot(struct igc_adapter * adapter)3145794e5bc8SAndre Guedes static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
314686a4de66SSasha Neftin {
314786a4de66SSasha Neftin struct igc_hw *hw = &adapter->hw;
3148d66358caSAndre Guedes int max_entries = hw->mac.rar_entry_count;
3149d66358caSAndre Guedes u32 rah;
315086a4de66SSasha Neftin int i;
315186a4de66SSasha Neftin
3152794e5bc8SAndre Guedes for (i = 0; i < max_entries; i++) {
3153d66358caSAndre Guedes rah = rd32(IGC_RAH(i));
315486a4de66SSasha Neftin
3155d66358caSAndre Guedes if (!(rah & IGC_RAH_AV))
3156794e5bc8SAndre Guedes return i;
315786a4de66SSasha Neftin }
315886a4de66SSasha Neftin
3159794e5bc8SAndre Guedes return -1;
316086a4de66SSasha Neftin }
316186a4de66SSasha Neftin
3162e9736fa4SAndre Guedes /**
3163e9736fa4SAndre Guedes * igc_add_mac_filter() - Add MAC address filter
3164e9736fa4SAndre Guedes * @adapter: Pointer to adapter where the filter should be added
3165750433d0SAndre Guedes * @type: MAC address filter type (source or destination)
3166e9736fa4SAndre Guedes * @addr: MAC address
3167e9736fa4SAndre Guedes * @queue: If non-negative, queue assignment feature is enabled and frames
3168e9736fa4SAndre Guedes * matching the filter are enqueued onto 'queue'. Otherwise, queue
3169e9736fa4SAndre Guedes * assignment is disabled.
3170e9736fa4SAndre Guedes *
3171e9736fa4SAndre Guedes * Return: 0 in case of success, negative errno code otherwise.
317286a4de66SSasha Neftin */
igc_add_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr,int queue)317336fa2152SAndre Guedes static int igc_add_mac_filter(struct igc_adapter *adapter,
3174750433d0SAndre Guedes enum igc_mac_filter_type type, const u8 *addr,
3175750433d0SAndre Guedes int queue)
317686a4de66SSasha Neftin {
3177949b922eSAndre Guedes struct net_device *dev = adapter->netdev;
3178794e5bc8SAndre Guedes int index;
317986a4de66SSasha Neftin
3180750433d0SAndre Guedes index = igc_find_mac_filter(adapter, type, addr);
3181794e5bc8SAndre Guedes if (index >= 0)
3182d66358caSAndre Guedes goto update_filter;
318386a4de66SSasha Neftin
3184794e5bc8SAndre Guedes index = igc_get_avail_mac_filter_slot(adapter);
3185794e5bc8SAndre Guedes if (index < 0)
318686a4de66SSasha Neftin return -ENOSPC;
3187794e5bc8SAndre Guedes
3188750433d0SAndre Guedes netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
3189750433d0SAndre Guedes index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3190750433d0SAndre Guedes addr, queue);
3191949b922eSAndre Guedes
3192d66358caSAndre Guedes update_filter:
3193750433d0SAndre Guedes igc_set_mac_filter_hw(adapter, index, type, addr, queue);
319486a4de66SSasha Neftin return 0;
319586a4de66SSasha Neftin }
319686a4de66SSasha Neftin
3197c6aae591SAndre Guedes /**
3198c6aae591SAndre Guedes * igc_del_mac_filter() - Delete MAC address filter
3199c6aae591SAndre Guedes * @adapter: Pointer to adapter where the filter should be deleted from
3200750433d0SAndre Guedes * @type: MAC address filter type (source or destination)
3201c6aae591SAndre Guedes * @addr: MAC address
320286a4de66SSasha Neftin */
igc_del_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr)3203acda576fSAndre Guedes static void igc_del_mac_filter(struct igc_adapter *adapter,
3204750433d0SAndre Guedes enum igc_mac_filter_type type, const u8 *addr)
320586a4de66SSasha Neftin {
3206949b922eSAndre Guedes struct net_device *dev = adapter->netdev;
32075f930713SAndre Guedes int index;
320886a4de66SSasha Neftin
3209750433d0SAndre Guedes index = igc_find_mac_filter(adapter, type, addr);
32105f930713SAndre Guedes if (index < 0)
3211acda576fSAndre Guedes return;
321286a4de66SSasha Neftin
3213d66358caSAndre Guedes if (index == 0) {
32145f930713SAndre Guedes /* If this is the default filter, we don't actually delete it.
32155f930713SAndre Guedes * We just reset to its default value i.e. disable queue
32165f930713SAndre Guedes * assignment.
321786a4de66SSasha Neftin */
3218949b922eSAndre Guedes netdev_dbg(dev, "Disable default MAC filter queue assignment");
3219949b922eSAndre Guedes
3220750433d0SAndre Guedes igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
322186a4de66SSasha Neftin } else {
3222750433d0SAndre Guedes netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
3223750433d0SAndre Guedes index,
3224750433d0SAndre Guedes type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3225750433d0SAndre Guedes addr);
3226949b922eSAndre Guedes
32275f930713SAndre Guedes igc_clear_mac_filter_hw(adapter, index);
322886a4de66SSasha Neftin }
322986a4de66SSasha Neftin }
323086a4de66SSasha Neftin
323112ddee68SAndre Guedes /**
323212ddee68SAndre Guedes * igc_add_vlan_prio_filter() - Add VLAN priority filter
323312ddee68SAndre Guedes * @adapter: Pointer to adapter where the filter should be added
323412ddee68SAndre Guedes * @prio: VLAN priority value
323512ddee68SAndre Guedes * @queue: Queue number which matching frames are assigned to
323612ddee68SAndre Guedes *
323712ddee68SAndre Guedes * Return: 0 in case of success, negative errno code otherwise.
323812ddee68SAndre Guedes */
igc_add_vlan_prio_filter(struct igc_adapter * adapter,int prio,int queue)323936fa2152SAndre Guedes static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
324036fa2152SAndre Guedes int queue)
324112ddee68SAndre Guedes {
324212ddee68SAndre Guedes struct net_device *dev = adapter->netdev;
324312ddee68SAndre Guedes struct igc_hw *hw = &adapter->hw;
324412ddee68SAndre Guedes u32 vlanpqf;
324512ddee68SAndre Guedes
324612ddee68SAndre Guedes vlanpqf = rd32(IGC_VLANPQF);
324712ddee68SAndre Guedes
324812ddee68SAndre Guedes if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
324912ddee68SAndre Guedes netdev_dbg(dev, "VLAN priority filter already in use\n");
325012ddee68SAndre Guedes return -EEXIST;
325112ddee68SAndre Guedes }
325212ddee68SAndre Guedes
325312ddee68SAndre Guedes vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
325412ddee68SAndre Guedes vlanpqf |= IGC_VLANPQF_VALID(prio);
325512ddee68SAndre Guedes
325612ddee68SAndre Guedes wr32(IGC_VLANPQF, vlanpqf);
325712ddee68SAndre Guedes
325812ddee68SAndre Guedes netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
325912ddee68SAndre Guedes prio, queue);
326012ddee68SAndre Guedes return 0;
326112ddee68SAndre Guedes }
326212ddee68SAndre Guedes
326312ddee68SAndre Guedes /**
326412ddee68SAndre Guedes * igc_del_vlan_prio_filter() - Delete VLAN priority filter
326512ddee68SAndre Guedes * @adapter: Pointer to adapter where the filter should be deleted from
326612ddee68SAndre Guedes * @prio: VLAN priority value
326712ddee68SAndre Guedes */
igc_del_vlan_prio_filter(struct igc_adapter * adapter,int prio)326836fa2152SAndre Guedes static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
326912ddee68SAndre Guedes {
327012ddee68SAndre Guedes struct igc_hw *hw = &adapter->hw;
327112ddee68SAndre Guedes u32 vlanpqf;
327212ddee68SAndre Guedes
327312ddee68SAndre Guedes vlanpqf = rd32(IGC_VLANPQF);
327412ddee68SAndre Guedes
327512ddee68SAndre Guedes vlanpqf &= ~IGC_VLANPQF_VALID(prio);
327612ddee68SAndre Guedes vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
327712ddee68SAndre Guedes
327812ddee68SAndre Guedes wr32(IGC_VLANPQF, vlanpqf);
327912ddee68SAndre Guedes
328012ddee68SAndre Guedes netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
328112ddee68SAndre Guedes prio);
328212ddee68SAndre Guedes }
328312ddee68SAndre Guedes
igc_get_avail_etype_filter_slot(struct igc_adapter * adapter)3284aa7ca726SAndre Guedes static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
3285aa7ca726SAndre Guedes {
3286aa7ca726SAndre Guedes struct igc_hw *hw = &adapter->hw;
3287aa7ca726SAndre Guedes int i;
3288aa7ca726SAndre Guedes
3289aa7ca726SAndre Guedes for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3290aa7ca726SAndre Guedes u32 etqf = rd32(IGC_ETQF(i));
3291aa7ca726SAndre Guedes
3292aa7ca726SAndre Guedes if (!(etqf & IGC_ETQF_FILTER_ENABLE))
3293aa7ca726SAndre Guedes return i;
3294aa7ca726SAndre Guedes }
3295aa7ca726SAndre Guedes
3296aa7ca726SAndre Guedes return -1;
3297aa7ca726SAndre Guedes }
3298aa7ca726SAndre Guedes
3299aa7ca726SAndre Guedes /**
3300aa7ca726SAndre Guedes * igc_add_etype_filter() - Add ethertype filter
3301aa7ca726SAndre Guedes * @adapter: Pointer to adapter where the filter should be added
3302aa7ca726SAndre Guedes * @etype: Ethertype value
3303aa7ca726SAndre Guedes * @queue: If non-negative, queue assignment feature is enabled and frames
3304aa7ca726SAndre Guedes * matching the filter are enqueued onto 'queue'. Otherwise, queue
3305aa7ca726SAndre Guedes * assignment is disabled.
3306aa7ca726SAndre Guedes *
3307aa7ca726SAndre Guedes * Return: 0 in case of success, negative errno code otherwise.
3308aa7ca726SAndre Guedes */
igc_add_etype_filter(struct igc_adapter * adapter,u16 etype,int queue)330936fa2152SAndre Guedes static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
331036fa2152SAndre Guedes int queue)
3311aa7ca726SAndre Guedes {
3312aa7ca726SAndre Guedes struct igc_hw *hw = &adapter->hw;
3313aa7ca726SAndre Guedes int index;
3314aa7ca726SAndre Guedes u32 etqf;
3315aa7ca726SAndre Guedes
3316aa7ca726SAndre Guedes index = igc_get_avail_etype_filter_slot(adapter);
3317aa7ca726SAndre Guedes if (index < 0)
3318aa7ca726SAndre Guedes return -ENOSPC;
3319aa7ca726SAndre Guedes
3320aa7ca726SAndre Guedes etqf = rd32(IGC_ETQF(index));
3321aa7ca726SAndre Guedes
3322aa7ca726SAndre Guedes etqf &= ~IGC_ETQF_ETYPE_MASK;
3323aa7ca726SAndre Guedes etqf |= etype;
3324aa7ca726SAndre Guedes
3325aa7ca726SAndre Guedes if (queue >= 0) {
3326aa7ca726SAndre Guedes etqf &= ~IGC_ETQF_QUEUE_MASK;
3327aa7ca726SAndre Guedes etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
3328aa7ca726SAndre Guedes etqf |= IGC_ETQF_QUEUE_ENABLE;
3329aa7ca726SAndre Guedes }
3330aa7ca726SAndre Guedes
3331aa7ca726SAndre Guedes etqf |= IGC_ETQF_FILTER_ENABLE;
3332aa7ca726SAndre Guedes
3333aa7ca726SAndre Guedes wr32(IGC_ETQF(index), etqf);
3334aa7ca726SAndre Guedes
3335aa7ca726SAndre Guedes netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
3336aa7ca726SAndre Guedes etype, queue);
3337aa7ca726SAndre Guedes return 0;
3338aa7ca726SAndre Guedes }
3339aa7ca726SAndre Guedes
igc_find_etype_filter(struct igc_adapter * adapter,u16 etype)3340aa7ca726SAndre Guedes static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
3341aa7ca726SAndre Guedes {
3342aa7ca726SAndre Guedes struct igc_hw *hw = &adapter->hw;
3343aa7ca726SAndre Guedes int i;
3344aa7ca726SAndre Guedes
3345aa7ca726SAndre Guedes for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3346aa7ca726SAndre Guedes u32 etqf = rd32(IGC_ETQF(i));
3347aa7ca726SAndre Guedes
3348aa7ca726SAndre Guedes if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
3349aa7ca726SAndre Guedes return i;
3350aa7ca726SAndre Guedes }
3351aa7ca726SAndre Guedes
3352aa7ca726SAndre Guedes return -1;
3353aa7ca726SAndre Guedes }
3354aa7ca726SAndre Guedes
3355aa7ca726SAndre Guedes /**
3356aa7ca726SAndre Guedes * igc_del_etype_filter() - Delete ethertype filter
3357aa7ca726SAndre Guedes * @adapter: Pointer to adapter where the filter should be deleted from
3358aa7ca726SAndre Guedes * @etype: Ethertype value
3359aa7ca726SAndre Guedes */
igc_del_etype_filter(struct igc_adapter * adapter,u16 etype)3360acda576fSAndre Guedes static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
3361aa7ca726SAndre Guedes {
3362aa7ca726SAndre Guedes struct igc_hw *hw = &adapter->hw;
3363aa7ca726SAndre Guedes int index;
3364aa7ca726SAndre Guedes
3365aa7ca726SAndre Guedes index = igc_find_etype_filter(adapter, etype);
3366aa7ca726SAndre Guedes if (index < 0)
3367acda576fSAndre Guedes return;
3368aa7ca726SAndre Guedes
3369aa7ca726SAndre Guedes wr32(IGC_ETQF(index), 0);
3370aa7ca726SAndre Guedes
3371aa7ca726SAndre Guedes netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
3372aa7ca726SAndre Guedes etype);
3373aa7ca726SAndre Guedes }
3374aa7ca726SAndre Guedes
igc_flex_filter_select(struct igc_adapter * adapter,struct igc_flex_filter * input,u32 * fhft)33756574631bSKurt Kanzenbach static int igc_flex_filter_select(struct igc_adapter *adapter,
33766574631bSKurt Kanzenbach struct igc_flex_filter *input,
33776574631bSKurt Kanzenbach u32 *fhft)
33786574631bSKurt Kanzenbach {
33796574631bSKurt Kanzenbach struct igc_hw *hw = &adapter->hw;
33806574631bSKurt Kanzenbach u8 fhft_index;
33816574631bSKurt Kanzenbach u32 fhftsl;
33826574631bSKurt Kanzenbach
33836574631bSKurt Kanzenbach if (input->index >= MAX_FLEX_FILTER) {
33846574631bSKurt Kanzenbach dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n");
33856574631bSKurt Kanzenbach return -EINVAL;
33866574631bSKurt Kanzenbach }
33876574631bSKurt Kanzenbach
33886574631bSKurt Kanzenbach /* Indirect table select register */
33896574631bSKurt Kanzenbach fhftsl = rd32(IGC_FHFTSL);
33906574631bSKurt Kanzenbach fhftsl &= ~IGC_FHFTSL_FTSL_MASK;
33916574631bSKurt Kanzenbach switch (input->index) {
33926574631bSKurt Kanzenbach case 0 ... 7:
33936574631bSKurt Kanzenbach fhftsl |= 0x00;
33946574631bSKurt Kanzenbach break;
33956574631bSKurt Kanzenbach case 8 ... 15:
33966574631bSKurt Kanzenbach fhftsl |= 0x01;
33976574631bSKurt Kanzenbach break;
33986574631bSKurt Kanzenbach case 16 ... 23:
33996574631bSKurt Kanzenbach fhftsl |= 0x02;
34006574631bSKurt Kanzenbach break;
34016574631bSKurt Kanzenbach case 24 ... 31:
34026574631bSKurt Kanzenbach fhftsl |= 0x03;
34036574631bSKurt Kanzenbach break;
34046574631bSKurt Kanzenbach }
34056574631bSKurt Kanzenbach wr32(IGC_FHFTSL, fhftsl);
34066574631bSKurt Kanzenbach
34076574631bSKurt Kanzenbach /* Normalize index down to host table register */
34086574631bSKurt Kanzenbach fhft_index = input->index % 8;
34096574631bSKurt Kanzenbach
34106574631bSKurt Kanzenbach *fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) :
34116574631bSKurt Kanzenbach IGC_FHFT_EXT(fhft_index - 4);
34126574631bSKurt Kanzenbach
34136574631bSKurt Kanzenbach return 0;
34146574631bSKurt Kanzenbach }
34156574631bSKurt Kanzenbach
igc_write_flex_filter_ll(struct igc_adapter * adapter,struct igc_flex_filter * input)34162b477d05SKurt Kanzenbach static int igc_write_flex_filter_ll(struct igc_adapter *adapter,
34176574631bSKurt Kanzenbach struct igc_flex_filter *input)
34186574631bSKurt Kanzenbach {
34196574631bSKurt Kanzenbach struct device *dev = &adapter->pdev->dev;
34206574631bSKurt Kanzenbach struct igc_hw *hw = &adapter->hw;
34216574631bSKurt Kanzenbach u8 *data = input->data;
34226574631bSKurt Kanzenbach u8 *mask = input->mask;
34236574631bSKurt Kanzenbach u32 queuing;
34246574631bSKurt Kanzenbach u32 fhft;
34256574631bSKurt Kanzenbach u32 wufc;
34266574631bSKurt Kanzenbach int ret;
34276574631bSKurt Kanzenbach int i;
34286574631bSKurt Kanzenbach
34296574631bSKurt Kanzenbach /* Length has to be aligned to 8. Otherwise the filter will fail. Bail
34306574631bSKurt Kanzenbach * out early to avoid surprises later.
34316574631bSKurt Kanzenbach */
34326574631bSKurt Kanzenbach if (input->length % 8 != 0) {
34336574631bSKurt Kanzenbach dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n");
34346574631bSKurt Kanzenbach return -EINVAL;
34356574631bSKurt Kanzenbach }
34366574631bSKurt Kanzenbach
34376574631bSKurt Kanzenbach /* Select corresponding flex filter register and get base for host table. */
34386574631bSKurt Kanzenbach ret = igc_flex_filter_select(adapter, input, &fhft);
34396574631bSKurt Kanzenbach if (ret)
34406574631bSKurt Kanzenbach return ret;
34416574631bSKurt Kanzenbach
34426574631bSKurt Kanzenbach /* When adding a filter globally disable flex filter feature. That is
34436574631bSKurt Kanzenbach * recommended within the datasheet.
34446574631bSKurt Kanzenbach */
34456574631bSKurt Kanzenbach wufc = rd32(IGC_WUFC);
34466574631bSKurt Kanzenbach wufc &= ~IGC_WUFC_FLEX_HQ;
34476574631bSKurt Kanzenbach wr32(IGC_WUFC, wufc);
34486574631bSKurt Kanzenbach
34496574631bSKurt Kanzenbach /* Configure filter */
34506574631bSKurt Kanzenbach queuing = input->length & IGC_FHFT_LENGTH_MASK;
34516574631bSKurt Kanzenbach queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK;
34526574631bSKurt Kanzenbach queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK;
34536574631bSKurt Kanzenbach
34546574631bSKurt Kanzenbach if (input->immediate_irq)
34556574631bSKurt Kanzenbach queuing |= IGC_FHFT_IMM_INT;
34566574631bSKurt Kanzenbach
34576574631bSKurt Kanzenbach if (input->drop)
34586574631bSKurt Kanzenbach queuing |= IGC_FHFT_DROP;
34596574631bSKurt Kanzenbach
34606574631bSKurt Kanzenbach wr32(fhft + 0xFC, queuing);
34616574631bSKurt Kanzenbach
34626574631bSKurt Kanzenbach /* Write data (128 byte) and mask (128 bit) */
34636574631bSKurt Kanzenbach for (i = 0; i < 16; ++i) {
34646574631bSKurt Kanzenbach const size_t data_idx = i * 8;
34656574631bSKurt Kanzenbach const size_t row_idx = i * 16;
34666574631bSKurt Kanzenbach u32 dw0 =
34676574631bSKurt Kanzenbach (data[data_idx + 0] << 0) |
34686574631bSKurt Kanzenbach (data[data_idx + 1] << 8) |
34696574631bSKurt Kanzenbach (data[data_idx + 2] << 16) |
34706574631bSKurt Kanzenbach (data[data_idx + 3] << 24);
34716574631bSKurt Kanzenbach u32 dw1 =
34726574631bSKurt Kanzenbach (data[data_idx + 4] << 0) |
34736574631bSKurt Kanzenbach (data[data_idx + 5] << 8) |
34746574631bSKurt Kanzenbach (data[data_idx + 6] << 16) |
34756574631bSKurt Kanzenbach (data[data_idx + 7] << 24);
34766574631bSKurt Kanzenbach u32 tmp;
34776574631bSKurt Kanzenbach
34786574631bSKurt Kanzenbach /* Write row: dw0, dw1 and mask */
34796574631bSKurt Kanzenbach wr32(fhft + row_idx, dw0);
34806574631bSKurt Kanzenbach wr32(fhft + row_idx + 4, dw1);
34816574631bSKurt Kanzenbach
34826574631bSKurt Kanzenbach /* mask is only valid for MASK(7, 0) */
34836574631bSKurt Kanzenbach tmp = rd32(fhft + row_idx + 8);
34846574631bSKurt Kanzenbach tmp &= ~GENMASK(7, 0);
34856574631bSKurt Kanzenbach tmp |= mask[i];
34866574631bSKurt Kanzenbach wr32(fhft + row_idx + 8, tmp);
34876574631bSKurt Kanzenbach }
34886574631bSKurt Kanzenbach
34896574631bSKurt Kanzenbach /* Enable filter. */
34906574631bSKurt Kanzenbach wufc |= IGC_WUFC_FLEX_HQ;
34916574631bSKurt Kanzenbach if (input->index > 8) {
34926574631bSKurt Kanzenbach /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
34936574631bSKurt Kanzenbach u32 wufc_ext = rd32(IGC_WUFC_EXT);
34946574631bSKurt Kanzenbach
34956574631bSKurt Kanzenbach wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8));
34966574631bSKurt Kanzenbach
34976574631bSKurt Kanzenbach wr32(IGC_WUFC_EXT, wufc_ext);
34986574631bSKurt Kanzenbach } else {
34996574631bSKurt Kanzenbach wufc |= (IGC_WUFC_FLX0 << input->index);
35006574631bSKurt Kanzenbach }
35016574631bSKurt Kanzenbach wr32(IGC_WUFC, wufc);
35026574631bSKurt Kanzenbach
35036574631bSKurt Kanzenbach dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n",
35046574631bSKurt Kanzenbach input->index);
35056574631bSKurt Kanzenbach
35066574631bSKurt Kanzenbach return 0;
35076574631bSKurt Kanzenbach }
35086574631bSKurt Kanzenbach
igc_flex_filter_add_field(struct igc_flex_filter * flex,const void * src,unsigned int offset,size_t len,const void * mask)35092b477d05SKurt Kanzenbach static void igc_flex_filter_add_field(struct igc_flex_filter *flex,
35102b477d05SKurt Kanzenbach const void *src, unsigned int offset,
35112b477d05SKurt Kanzenbach size_t len, const void *mask)
35122b477d05SKurt Kanzenbach {
35132b477d05SKurt Kanzenbach int i;
35142b477d05SKurt Kanzenbach
35152b477d05SKurt Kanzenbach /* data */
35162b477d05SKurt Kanzenbach memcpy(&flex->data[offset], src, len);
35172b477d05SKurt Kanzenbach
35182b477d05SKurt Kanzenbach /* mask */
35192b477d05SKurt Kanzenbach for (i = 0; i < len; ++i) {
35202b477d05SKurt Kanzenbach const unsigned int idx = i + offset;
35212b477d05SKurt Kanzenbach const u8 *ptr = mask;
35222b477d05SKurt Kanzenbach
35232b477d05SKurt Kanzenbach if (mask) {
35242b477d05SKurt Kanzenbach if (ptr[i] & 0xff)
35252b477d05SKurt Kanzenbach flex->mask[idx / 8] |= BIT(idx % 8);
35262b477d05SKurt Kanzenbach
35272b477d05SKurt Kanzenbach continue;
35282b477d05SKurt Kanzenbach }
35292b477d05SKurt Kanzenbach
35302b477d05SKurt Kanzenbach flex->mask[idx / 8] |= BIT(idx % 8);
35312b477d05SKurt Kanzenbach }
35322b477d05SKurt Kanzenbach }
35332b477d05SKurt Kanzenbach
igc_find_avail_flex_filter_slot(struct igc_adapter * adapter)35342b477d05SKurt Kanzenbach static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter)
35352b477d05SKurt Kanzenbach {
35362b477d05SKurt Kanzenbach struct igc_hw *hw = &adapter->hw;
35372b477d05SKurt Kanzenbach u32 wufc, wufc_ext;
35382b477d05SKurt Kanzenbach int i;
35392b477d05SKurt Kanzenbach
35402b477d05SKurt Kanzenbach wufc = rd32(IGC_WUFC);
35412b477d05SKurt Kanzenbach wufc_ext = rd32(IGC_WUFC_EXT);
35422b477d05SKurt Kanzenbach
35432b477d05SKurt Kanzenbach for (i = 0; i < MAX_FLEX_FILTER; i++) {
35442b477d05SKurt Kanzenbach if (i < 8) {
35452b477d05SKurt Kanzenbach if (!(wufc & (IGC_WUFC_FLX0 << i)))
35462b477d05SKurt Kanzenbach return i;
35472b477d05SKurt Kanzenbach } else {
35482b477d05SKurt Kanzenbach if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8))))
35492b477d05SKurt Kanzenbach return i;
35502b477d05SKurt Kanzenbach }
35512b477d05SKurt Kanzenbach }
35522b477d05SKurt Kanzenbach
35532b477d05SKurt Kanzenbach return -ENOSPC;
35542b477d05SKurt Kanzenbach }
35552b477d05SKurt Kanzenbach
igc_flex_filter_in_use(struct igc_adapter * adapter)35562b477d05SKurt Kanzenbach static bool igc_flex_filter_in_use(struct igc_adapter *adapter)
35572b477d05SKurt Kanzenbach {
35582b477d05SKurt Kanzenbach struct igc_hw *hw = &adapter->hw;
35592b477d05SKurt Kanzenbach u32 wufc, wufc_ext;
35602b477d05SKurt Kanzenbach
35612b477d05SKurt Kanzenbach wufc = rd32(IGC_WUFC);
35622b477d05SKurt Kanzenbach wufc_ext = rd32(IGC_WUFC_EXT);
35632b477d05SKurt Kanzenbach
35642b477d05SKurt Kanzenbach if (wufc & IGC_WUFC_FILTER_MASK)
35652b477d05SKurt Kanzenbach return true;
35662b477d05SKurt Kanzenbach
35672b477d05SKurt Kanzenbach if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK)
35682b477d05SKurt Kanzenbach return true;
35692b477d05SKurt Kanzenbach
35702b477d05SKurt Kanzenbach return false;
35712b477d05SKurt Kanzenbach }
35722b477d05SKurt Kanzenbach
igc_add_flex_filter(struct igc_adapter * adapter,struct igc_nfc_rule * rule)35732b477d05SKurt Kanzenbach static int igc_add_flex_filter(struct igc_adapter *adapter,
35742b477d05SKurt Kanzenbach struct igc_nfc_rule *rule)
35752b477d05SKurt Kanzenbach {
35762b477d05SKurt Kanzenbach struct igc_flex_filter flex = { };
35772b477d05SKurt Kanzenbach struct igc_nfc_filter *filter = &rule->filter;
35782b477d05SKurt Kanzenbach unsigned int eth_offset, user_offset;
35792b477d05SKurt Kanzenbach int ret, index;
35802b477d05SKurt Kanzenbach bool vlan;
35812b477d05SKurt Kanzenbach
35822b477d05SKurt Kanzenbach index = igc_find_avail_flex_filter_slot(adapter);
35832b477d05SKurt Kanzenbach if (index < 0)
35842b477d05SKurt Kanzenbach return -ENOSPC;
35852b477d05SKurt Kanzenbach
35862b477d05SKurt Kanzenbach /* Construct the flex filter:
35872b477d05SKurt Kanzenbach * -> dest_mac [6]
35882b477d05SKurt Kanzenbach * -> src_mac [6]
35892b477d05SKurt Kanzenbach * -> tpid [2]
35902b477d05SKurt Kanzenbach * -> vlan tci [2]
35912b477d05SKurt Kanzenbach * -> ether type [2]
35922b477d05SKurt Kanzenbach * -> user data [8]
35932b477d05SKurt Kanzenbach * -> = 26 bytes => 32 length
35942b477d05SKurt Kanzenbach */
35952b477d05SKurt Kanzenbach flex.index = index;
35962b477d05SKurt Kanzenbach flex.length = 32;
35972b477d05SKurt Kanzenbach flex.rx_queue = rule->action;
35982b477d05SKurt Kanzenbach
35992b477d05SKurt Kanzenbach vlan = rule->filter.vlan_tci || rule->filter.vlan_etype;
36002b477d05SKurt Kanzenbach eth_offset = vlan ? 16 : 12;
36012b477d05SKurt Kanzenbach user_offset = vlan ? 18 : 14;
36022b477d05SKurt Kanzenbach
36032b477d05SKurt Kanzenbach /* Add destination MAC */
36042b477d05SKurt Kanzenbach if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
36052b477d05SKurt Kanzenbach igc_flex_filter_add_field(&flex, &filter->dst_addr, 0,
36062b477d05SKurt Kanzenbach ETH_ALEN, NULL);
36072b477d05SKurt Kanzenbach
36082b477d05SKurt Kanzenbach /* Add source MAC */
36092b477d05SKurt Kanzenbach if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
36102b477d05SKurt Kanzenbach igc_flex_filter_add_field(&flex, &filter->src_addr, 6,
36112b477d05SKurt Kanzenbach ETH_ALEN, NULL);
36122b477d05SKurt Kanzenbach
36132b477d05SKurt Kanzenbach /* Add VLAN etype */
36142b477d05SKurt Kanzenbach if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE)
36152b477d05SKurt Kanzenbach igc_flex_filter_add_field(&flex, &filter->vlan_etype, 12,
36162b477d05SKurt Kanzenbach sizeof(filter->vlan_etype),
36172b477d05SKurt Kanzenbach NULL);
36182b477d05SKurt Kanzenbach
36192b477d05SKurt Kanzenbach /* Add VLAN TCI */
36202b477d05SKurt Kanzenbach if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI)
36212b477d05SKurt Kanzenbach igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14,
36222b477d05SKurt Kanzenbach sizeof(filter->vlan_tci), NULL);
36232b477d05SKurt Kanzenbach
36242b477d05SKurt Kanzenbach /* Add Ether type */
36252b477d05SKurt Kanzenbach if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
36262b477d05SKurt Kanzenbach __be16 etype = cpu_to_be16(filter->etype);
36272b477d05SKurt Kanzenbach
36282b477d05SKurt Kanzenbach igc_flex_filter_add_field(&flex, &etype, eth_offset,
36292b477d05SKurt Kanzenbach sizeof(etype), NULL);
36302b477d05SKurt Kanzenbach }
36312b477d05SKurt Kanzenbach
36322b477d05SKurt Kanzenbach /* Add user data */
36332b477d05SKurt Kanzenbach if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA)
36342b477d05SKurt Kanzenbach igc_flex_filter_add_field(&flex, &filter->user_data,
36352b477d05SKurt Kanzenbach user_offset,
36362b477d05SKurt Kanzenbach sizeof(filter->user_data),
36372b477d05SKurt Kanzenbach filter->user_mask);
36382b477d05SKurt Kanzenbach
36392b477d05SKurt Kanzenbach /* Add it down to the hardware and enable it. */
36402b477d05SKurt Kanzenbach ret = igc_write_flex_filter_ll(adapter, &flex);
36412b477d05SKurt Kanzenbach if (ret)
36422b477d05SKurt Kanzenbach return ret;
36432b477d05SKurt Kanzenbach
36442b477d05SKurt Kanzenbach filter->flex_index = index;
36452b477d05SKurt Kanzenbach
36462b477d05SKurt Kanzenbach return 0;
36472b477d05SKurt Kanzenbach }
36482b477d05SKurt Kanzenbach
igc_del_flex_filter(struct igc_adapter * adapter,u16 reg_index)36492b477d05SKurt Kanzenbach static void igc_del_flex_filter(struct igc_adapter *adapter,
36502b477d05SKurt Kanzenbach u16 reg_index)
36512b477d05SKurt Kanzenbach {
36522b477d05SKurt Kanzenbach struct igc_hw *hw = &adapter->hw;
36532b477d05SKurt Kanzenbach u32 wufc;
36542b477d05SKurt Kanzenbach
36552b477d05SKurt Kanzenbach /* Just disable the filter. The filter table itself is kept
36562b477d05SKurt Kanzenbach * intact. Another flex_filter_add() should override the "old" data
36572b477d05SKurt Kanzenbach * then.
36582b477d05SKurt Kanzenbach */
36592b477d05SKurt Kanzenbach if (reg_index > 8) {
36602b477d05SKurt Kanzenbach u32 wufc_ext = rd32(IGC_WUFC_EXT);
36612b477d05SKurt Kanzenbach
36622b477d05SKurt Kanzenbach wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8));
36632b477d05SKurt Kanzenbach wr32(IGC_WUFC_EXT, wufc_ext);
36642b477d05SKurt Kanzenbach } else {
36652b477d05SKurt Kanzenbach wufc = rd32(IGC_WUFC);
36662b477d05SKurt Kanzenbach
36672b477d05SKurt Kanzenbach wufc &= ~(IGC_WUFC_FLX0 << reg_index);
36682b477d05SKurt Kanzenbach wr32(IGC_WUFC, wufc);
36692b477d05SKurt Kanzenbach }
36702b477d05SKurt Kanzenbach
36712b477d05SKurt Kanzenbach if (igc_flex_filter_in_use(adapter))
36722b477d05SKurt Kanzenbach return;
36732b477d05SKurt Kanzenbach
36742b477d05SKurt Kanzenbach /* No filters are in use, we may disable flex filters */
36752b477d05SKurt Kanzenbach wufc = rd32(IGC_WUFC);
36762b477d05SKurt Kanzenbach wufc &= ~IGC_WUFC_FLEX_HQ;
36772b477d05SKurt Kanzenbach wr32(IGC_WUFC, wufc);
36782b477d05SKurt Kanzenbach }
36792b477d05SKurt Kanzenbach
igc_enable_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)368036fa2152SAndre Guedes static int igc_enable_nfc_rule(struct igc_adapter *adapter,
36812b477d05SKurt Kanzenbach struct igc_nfc_rule *rule)
368236fa2152SAndre Guedes {
368336fa2152SAndre Guedes int err;
368436fa2152SAndre Guedes
368573744262SKurt Kanzenbach if (rule->flex) {
368673744262SKurt Kanzenbach return igc_add_flex_filter(adapter, rule);
36872b477d05SKurt Kanzenbach }
36882b477d05SKurt Kanzenbach
368936fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
369036fa2152SAndre Guedes err = igc_add_etype_filter(adapter, rule->filter.etype,
369136fa2152SAndre Guedes rule->action);
369236fa2152SAndre Guedes if (err)
369336fa2152SAndre Guedes return err;
369436fa2152SAndre Guedes }
369536fa2152SAndre Guedes
369636fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
369736fa2152SAndre Guedes err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
369836fa2152SAndre Guedes rule->filter.src_addr, rule->action);
369936fa2152SAndre Guedes if (err)
370036fa2152SAndre Guedes return err;
370136fa2152SAndre Guedes }
370236fa2152SAndre Guedes
370336fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
370436fa2152SAndre Guedes err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
370536fa2152SAndre Guedes rule->filter.dst_addr, rule->action);
370636fa2152SAndre Guedes if (err)
370736fa2152SAndre Guedes return err;
370836fa2152SAndre Guedes }
370936fa2152SAndre Guedes
371036fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
371136fa2152SAndre Guedes int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
371236fa2152SAndre Guedes VLAN_PRIO_SHIFT;
371336fa2152SAndre Guedes
371436fa2152SAndre Guedes err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
371536fa2152SAndre Guedes if (err)
371636fa2152SAndre Guedes return err;
371736fa2152SAndre Guedes }
371836fa2152SAndre Guedes
371936fa2152SAndre Guedes return 0;
372036fa2152SAndre Guedes }
372136fa2152SAndre Guedes
igc_disable_nfc_rule(struct igc_adapter * adapter,const struct igc_nfc_rule * rule)3722acda576fSAndre Guedes static void igc_disable_nfc_rule(struct igc_adapter *adapter,
372336fa2152SAndre Guedes const struct igc_nfc_rule *rule)
372436fa2152SAndre Guedes {
372573744262SKurt Kanzenbach if (rule->flex) {
37262b477d05SKurt Kanzenbach igc_del_flex_filter(adapter, rule->filter.flex_index);
372773744262SKurt Kanzenbach return;
372873744262SKurt Kanzenbach }
37292b477d05SKurt Kanzenbach
373036fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
373136fa2152SAndre Guedes igc_del_etype_filter(adapter, rule->filter.etype);
373236fa2152SAndre Guedes
373336fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
373436fa2152SAndre Guedes int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
373536fa2152SAndre Guedes VLAN_PRIO_SHIFT;
373636fa2152SAndre Guedes
373736fa2152SAndre Guedes igc_del_vlan_prio_filter(adapter, prio);
373836fa2152SAndre Guedes }
373936fa2152SAndre Guedes
374036fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
374136fa2152SAndre Guedes igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
374236fa2152SAndre Guedes rule->filter.src_addr);
374336fa2152SAndre Guedes
374436fa2152SAndre Guedes if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
374536fa2152SAndre Guedes igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
374636fa2152SAndre Guedes rule->filter.dst_addr);
374736fa2152SAndre Guedes }
374836fa2152SAndre Guedes
374936fa2152SAndre Guedes /**
375036fa2152SAndre Guedes * igc_get_nfc_rule() - Get NFC rule
375136fa2152SAndre Guedes * @adapter: Pointer to adapter
375236fa2152SAndre Guedes * @location: Rule location
375336fa2152SAndre Guedes *
375436fa2152SAndre Guedes * Context: Expects adapter->nfc_rule_lock to be held by caller.
375536fa2152SAndre Guedes *
375636fa2152SAndre Guedes * Return: Pointer to NFC rule at @location. If not found, NULL.
375736fa2152SAndre Guedes */
igc_get_nfc_rule(struct igc_adapter * adapter,u32 location)375836fa2152SAndre Guedes struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
375936fa2152SAndre Guedes u32 location)
376036fa2152SAndre Guedes {
376136fa2152SAndre Guedes struct igc_nfc_rule *rule;
376236fa2152SAndre Guedes
376336fa2152SAndre Guedes list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
376436fa2152SAndre Guedes if (rule->location == location)
376536fa2152SAndre Guedes return rule;
376636fa2152SAndre Guedes if (rule->location > location)
376736fa2152SAndre Guedes break;
376836fa2152SAndre Guedes }
376936fa2152SAndre Guedes
377036fa2152SAndre Guedes return NULL;
377136fa2152SAndre Guedes }
377236fa2152SAndre Guedes
377336fa2152SAndre Guedes /**
377436fa2152SAndre Guedes * igc_del_nfc_rule() - Delete NFC rule
377536fa2152SAndre Guedes * @adapter: Pointer to adapter
377636fa2152SAndre Guedes * @rule: Pointer to rule to be deleted
377736fa2152SAndre Guedes *
377836fa2152SAndre Guedes * Disable NFC rule in hardware and delete it from adapter.
377936fa2152SAndre Guedes *
378036fa2152SAndre Guedes * Context: Expects adapter->nfc_rule_lock to be held by caller.
378136fa2152SAndre Guedes */
igc_del_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)378236fa2152SAndre Guedes void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
378336fa2152SAndre Guedes {
378436fa2152SAndre Guedes igc_disable_nfc_rule(adapter, rule);
378536fa2152SAndre Guedes
378636fa2152SAndre Guedes list_del(&rule->list);
378736fa2152SAndre Guedes adapter->nfc_rule_count--;
378836fa2152SAndre Guedes
378936fa2152SAndre Guedes kfree(rule);
379036fa2152SAndre Guedes }
379136fa2152SAndre Guedes
igc_flush_nfc_rules(struct igc_adapter * adapter)3792e256ec83SAndre Guedes static void igc_flush_nfc_rules(struct igc_adapter *adapter)
3793e256ec83SAndre Guedes {
3794e256ec83SAndre Guedes struct igc_nfc_rule *rule, *tmp;
3795e256ec83SAndre Guedes
379642fc5dc0SAndre Guedes mutex_lock(&adapter->nfc_rule_lock);
3797e256ec83SAndre Guedes
3798e256ec83SAndre Guedes list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
3799e256ec83SAndre Guedes igc_del_nfc_rule(adapter, rule);
3800e256ec83SAndre Guedes
380142fc5dc0SAndre Guedes mutex_unlock(&adapter->nfc_rule_lock);
3802e256ec83SAndre Guedes }
3803e256ec83SAndre Guedes
380436fa2152SAndre Guedes /**
380536fa2152SAndre Guedes * igc_add_nfc_rule() - Add NFC rule
380636fa2152SAndre Guedes * @adapter: Pointer to adapter
380736fa2152SAndre Guedes * @rule: Pointer to rule to be added
380836fa2152SAndre Guedes *
380936fa2152SAndre Guedes * Enable NFC rule in hardware and add it to adapter.
381036fa2152SAndre Guedes *
381136fa2152SAndre Guedes * Context: Expects adapter->nfc_rule_lock to be held by caller.
381236fa2152SAndre Guedes *
381336fa2152SAndre Guedes * Return: 0 on success, negative errno on failure.
381436fa2152SAndre Guedes */
igc_add_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)381536fa2152SAndre Guedes int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
381636fa2152SAndre Guedes {
381736fa2152SAndre Guedes struct igc_nfc_rule *pred, *cur;
381836fa2152SAndre Guedes int err;
381936fa2152SAndre Guedes
382036fa2152SAndre Guedes err = igc_enable_nfc_rule(adapter, rule);
382136fa2152SAndre Guedes if (err)
382236fa2152SAndre Guedes return err;
382336fa2152SAndre Guedes
382436fa2152SAndre Guedes pred = NULL;
382536fa2152SAndre Guedes list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
382636fa2152SAndre Guedes if (cur->location >= rule->location)
382736fa2152SAndre Guedes break;
382836fa2152SAndre Guedes pred = cur;
382936fa2152SAndre Guedes }
383036fa2152SAndre Guedes
383136fa2152SAndre Guedes list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
383236fa2152SAndre Guedes adapter->nfc_rule_count++;
383336fa2152SAndre Guedes return 0;
383436fa2152SAndre Guedes }
383536fa2152SAndre Guedes
igc_restore_nfc_rules(struct igc_adapter * adapter)383636fa2152SAndre Guedes static void igc_restore_nfc_rules(struct igc_adapter *adapter)
383736fa2152SAndre Guedes {
383836fa2152SAndre Guedes struct igc_nfc_rule *rule;
383936fa2152SAndre Guedes
384042fc5dc0SAndre Guedes mutex_lock(&adapter->nfc_rule_lock);
384136fa2152SAndre Guedes
384236fa2152SAndre Guedes list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
384336fa2152SAndre Guedes igc_enable_nfc_rule(adapter, rule);
384436fa2152SAndre Guedes
384542fc5dc0SAndre Guedes mutex_unlock(&adapter->nfc_rule_lock);
384686a4de66SSasha Neftin }
384786a4de66SSasha Neftin
igc_uc_sync(struct net_device * netdev,const unsigned char * addr)384886a4de66SSasha Neftin static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
384986a4de66SSasha Neftin {
385086a4de66SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
385186a4de66SSasha Neftin
3852750433d0SAndre Guedes return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
385386a4de66SSasha Neftin }
385486a4de66SSasha Neftin
igc_uc_unsync(struct net_device * netdev,const unsigned char * addr)385586a4de66SSasha Neftin static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
385686a4de66SSasha Neftin {
385786a4de66SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
385886a4de66SSasha Neftin
3859acda576fSAndre Guedes igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
386086a4de66SSasha Neftin return 0;
386186a4de66SSasha Neftin }
386286a4de66SSasha Neftin
386386a4de66SSasha Neftin /**
386486a4de66SSasha Neftin * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
386586a4de66SSasha Neftin * @netdev: network interface device structure
386686a4de66SSasha Neftin *
386786a4de66SSasha Neftin * The set_rx_mode entry point is called whenever the unicast or multicast
386886a4de66SSasha Neftin * address lists or the network interface flags are updated. This routine is
386986a4de66SSasha Neftin * responsible for configuring the hardware for proper unicast, multicast,
387086a4de66SSasha Neftin * promiscuous mode, and all-multi behavior.
387186a4de66SSasha Neftin */
igc_set_rx_mode(struct net_device * netdev)387286a4de66SSasha Neftin static void igc_set_rx_mode(struct net_device *netdev)
387386a4de66SSasha Neftin {
387486a4de66SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
387586a4de66SSasha Neftin struct igc_hw *hw = &adapter->hw;
387686a4de66SSasha Neftin u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
387786a4de66SSasha Neftin int count;
387886a4de66SSasha Neftin
387986a4de66SSasha Neftin /* Check for Promiscuous and All Multicast modes */
388086a4de66SSasha Neftin if (netdev->flags & IFF_PROMISC) {
388186a4de66SSasha Neftin rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
388286a4de66SSasha Neftin } else {
388386a4de66SSasha Neftin if (netdev->flags & IFF_ALLMULTI) {
388486a4de66SSasha Neftin rctl |= IGC_RCTL_MPE;
388586a4de66SSasha Neftin } else {
388686a4de66SSasha Neftin /* Write addresses to the MTA, if the attempt fails
388786a4de66SSasha Neftin * then we should just turn on promiscuous mode so
388886a4de66SSasha Neftin * that we can at least receive multicast traffic
388986a4de66SSasha Neftin */
389086a4de66SSasha Neftin count = igc_write_mc_addr_list(netdev);
389186a4de66SSasha Neftin if (count < 0)
389286a4de66SSasha Neftin rctl |= IGC_RCTL_MPE;
389386a4de66SSasha Neftin }
389486a4de66SSasha Neftin }
389586a4de66SSasha Neftin
389686a4de66SSasha Neftin /* Write addresses to available RAR registers, if there is not
389786a4de66SSasha Neftin * sufficient space to store all the addresses then enable
389886a4de66SSasha Neftin * unicast promiscuous mode
389986a4de66SSasha Neftin */
390086a4de66SSasha Neftin if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
390186a4de66SSasha Neftin rctl |= IGC_RCTL_UPE;
390286a4de66SSasha Neftin
390386a4de66SSasha Neftin /* update state of unicast and multicast */
390486a4de66SSasha Neftin rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
390586a4de66SSasha Neftin wr32(IGC_RCTL, rctl);
390686a4de66SSasha Neftin
390786a4de66SSasha Neftin #if (PAGE_SIZE < 8192)
390886a4de66SSasha Neftin if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
390986a4de66SSasha Neftin rlpml = IGC_MAX_FRAME_BUILD_SKB;
391086a4de66SSasha Neftin #endif
391186a4de66SSasha Neftin wr32(IGC_RLPML, rlpml);
391286a4de66SSasha Neftin }
391386a4de66SSasha Neftin
39141a7c0f2eSSasha Neftin /**
39151a7c0f2eSSasha Neftin * igc_configure - configure the hardware for RX and TX
39161a7c0f2eSSasha Neftin * @adapter: private board structure
39171a7c0f2eSSasha Neftin */
igc_configure(struct igc_adapter * adapter)39181a7c0f2eSSasha Neftin static void igc_configure(struct igc_adapter *adapter)
39191a7c0f2eSSasha Neftin {
39201a7c0f2eSSasha Neftin struct net_device *netdev = adapter->netdev;
39211a7c0f2eSSasha Neftin int i = 0;
39221a7c0f2eSSasha Neftin
39231a7c0f2eSSasha Neftin igc_get_hw_control(adapter);
39241a7c0f2eSSasha Neftin igc_set_rx_mode(netdev);
39251a7c0f2eSSasha Neftin
39268d744963SMuhammad Husaini Zulkifli igc_restore_vlan(adapter);
39278d744963SMuhammad Husaini Zulkifli
39281a7c0f2eSSasha Neftin igc_setup_tctl(adapter);
39291a7c0f2eSSasha Neftin igc_setup_mrqc(adapter);
39301a7c0f2eSSasha Neftin igc_setup_rctl(adapter);
39311a7c0f2eSSasha Neftin
3932ac9156b2SAndre Guedes igc_set_default_mac_filter(adapter);
393397700bc8SAndre Guedes igc_restore_nfc_rules(adapter);
3934ac9156b2SAndre Guedes
39351a7c0f2eSSasha Neftin igc_configure_tx(adapter);
39361a7c0f2eSSasha Neftin igc_configure_rx(adapter);
39371a7c0f2eSSasha Neftin
39381a7c0f2eSSasha Neftin igc_rx_fifo_flush_base(&adapter->hw);
39391a7c0f2eSSasha Neftin
39401a7c0f2eSSasha Neftin /* call igc_desc_unused which always leaves
39411a7c0f2eSSasha Neftin * at least 1 descriptor unused to make sure
39421a7c0f2eSSasha Neftin * next_to_use != next_to_clean
39431a7c0f2eSSasha Neftin */
39441a7c0f2eSSasha Neftin for (i = 0; i < adapter->num_rx_queues; i++) {
39451a7c0f2eSSasha Neftin struct igc_ring *ring = adapter->rx_ring[i];
39461a7c0f2eSSasha Neftin
3947fc9df2a0SAndre Guedes if (ring->xsk_pool)
3948fc9df2a0SAndre Guedes igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
3949fc9df2a0SAndre Guedes else
39501a7c0f2eSSasha Neftin igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
39511a7c0f2eSSasha Neftin }
39521a7c0f2eSSasha Neftin }
39531a7c0f2eSSasha Neftin
3954c9a11c23SSasha Neftin /**
3955f817fa05SSasha Neftin * igc_write_ivar - configure ivar for given MSI-X vector
3956f817fa05SSasha Neftin * @hw: pointer to the HW structure
3957f817fa05SSasha Neftin * @msix_vector: vector number we are allocating to a given ring
3958f817fa05SSasha Neftin * @index: row index of IVAR register to write within IVAR table
3959f817fa05SSasha Neftin * @offset: column offset of in IVAR, should be multiple of 8
3960f817fa05SSasha Neftin *
3961f817fa05SSasha Neftin * The IVAR table consists of 2 columns,
3962f817fa05SSasha Neftin * each containing an cause allocation for an Rx and Tx ring, and a
3963f817fa05SSasha Neftin * variable number of rows depending on the number of queues supported.
3964f817fa05SSasha Neftin */
igc_write_ivar(struct igc_hw * hw,int msix_vector,int index,int offset)3965f817fa05SSasha Neftin static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
3966f817fa05SSasha Neftin int index, int offset)
3967f817fa05SSasha Neftin {
3968f817fa05SSasha Neftin u32 ivar = array_rd32(IGC_IVAR0, index);
3969f817fa05SSasha Neftin
3970f817fa05SSasha Neftin /* clear any bits that are currently set */
3971f817fa05SSasha Neftin ivar &= ~((u32)0xFF << offset);
3972f817fa05SSasha Neftin
3973f817fa05SSasha Neftin /* write vector and valid bit */
3974f817fa05SSasha Neftin ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
3975f817fa05SSasha Neftin
3976f817fa05SSasha Neftin array_wr32(IGC_IVAR0, index, ivar);
3977f817fa05SSasha Neftin }
3978f817fa05SSasha Neftin
igc_assign_vector(struct igc_q_vector * q_vector,int msix_vector)3979f817fa05SSasha Neftin static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
3980f817fa05SSasha Neftin {
3981f817fa05SSasha Neftin struct igc_adapter *adapter = q_vector->adapter;
3982f817fa05SSasha Neftin struct igc_hw *hw = &adapter->hw;
3983f817fa05SSasha Neftin int rx_queue = IGC_N0_QUEUE;
3984f817fa05SSasha Neftin int tx_queue = IGC_N0_QUEUE;
3985f817fa05SSasha Neftin
3986f817fa05SSasha Neftin if (q_vector->rx.ring)
3987f817fa05SSasha Neftin rx_queue = q_vector->rx.ring->reg_idx;
3988f817fa05SSasha Neftin if (q_vector->tx.ring)
3989f817fa05SSasha Neftin tx_queue = q_vector->tx.ring->reg_idx;
3990f817fa05SSasha Neftin
3991f817fa05SSasha Neftin switch (hw->mac.type) {
3992f817fa05SSasha Neftin case igc_i225:
3993f817fa05SSasha Neftin if (rx_queue > IGC_N0_QUEUE)
3994f817fa05SSasha Neftin igc_write_ivar(hw, msix_vector,
3995f817fa05SSasha Neftin rx_queue >> 1,
3996f817fa05SSasha Neftin (rx_queue & 0x1) << 4);
3997f817fa05SSasha Neftin if (tx_queue > IGC_N0_QUEUE)
3998f817fa05SSasha Neftin igc_write_ivar(hw, msix_vector,
3999f817fa05SSasha Neftin tx_queue >> 1,
4000f817fa05SSasha Neftin ((tx_queue & 0x1) << 4) + 8);
4001f817fa05SSasha Neftin q_vector->eims_value = BIT(msix_vector);
4002f817fa05SSasha Neftin break;
4003f817fa05SSasha Neftin default:
4004f817fa05SSasha Neftin WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
4005f817fa05SSasha Neftin break;
4006f817fa05SSasha Neftin }
4007f817fa05SSasha Neftin
4008f817fa05SSasha Neftin /* add q_vector eims value to global eims_enable_mask */
4009f817fa05SSasha Neftin adapter->eims_enable_mask |= q_vector->eims_value;
4010f817fa05SSasha Neftin
4011f817fa05SSasha Neftin /* configure q_vector to set itr on first interrupt */
4012f817fa05SSasha Neftin q_vector->set_itr = 1;
4013f817fa05SSasha Neftin }
4014f817fa05SSasha Neftin
4015f817fa05SSasha Neftin /**
4016a146ea02SSasha Neftin * igc_configure_msix - Configure MSI-X hardware
4017a146ea02SSasha Neftin * @adapter: Pointer to adapter structure
4018a146ea02SSasha Neftin *
4019a146ea02SSasha Neftin * igc_configure_msix sets up the hardware to properly
4020a146ea02SSasha Neftin * generate MSI-X interrupts.
4021a146ea02SSasha Neftin */
igc_configure_msix(struct igc_adapter * adapter)4022a146ea02SSasha Neftin static void igc_configure_msix(struct igc_adapter *adapter)
4023a146ea02SSasha Neftin {
4024a146ea02SSasha Neftin struct igc_hw *hw = &adapter->hw;
4025a146ea02SSasha Neftin int i, vector = 0;
4026a146ea02SSasha Neftin u32 tmp;
4027a146ea02SSasha Neftin
4028a146ea02SSasha Neftin adapter->eims_enable_mask = 0;
4029a146ea02SSasha Neftin
4030a146ea02SSasha Neftin /* set vector for other causes, i.e. link changes */
4031a146ea02SSasha Neftin switch (hw->mac.type) {
4032a146ea02SSasha Neftin case igc_i225:
4033a146ea02SSasha Neftin /* Turn on MSI-X capability first, or our settings
4034a146ea02SSasha Neftin * won't stick. And it will take days to debug.
4035a146ea02SSasha Neftin */
4036a146ea02SSasha Neftin wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
4037a146ea02SSasha Neftin IGC_GPIE_PBA | IGC_GPIE_EIAME |
4038a146ea02SSasha Neftin IGC_GPIE_NSICR);
4039a146ea02SSasha Neftin
4040a146ea02SSasha Neftin /* enable msix_other interrupt */
4041a146ea02SSasha Neftin adapter->eims_other = BIT(vector);
4042a146ea02SSasha Neftin tmp = (vector++ | IGC_IVAR_VALID) << 8;
4043a146ea02SSasha Neftin
4044a146ea02SSasha Neftin wr32(IGC_IVAR_MISC, tmp);
4045a146ea02SSasha Neftin break;
4046a146ea02SSasha Neftin default:
4047a146ea02SSasha Neftin /* do nothing, since nothing else supports MSI-X */
4048a146ea02SSasha Neftin break;
4049a146ea02SSasha Neftin } /* switch (hw->mac.type) */
4050a146ea02SSasha Neftin
4051a146ea02SSasha Neftin adapter->eims_enable_mask |= adapter->eims_other;
4052a146ea02SSasha Neftin
4053a146ea02SSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++)
4054a146ea02SSasha Neftin igc_assign_vector(adapter->q_vector[i], vector++);
4055a146ea02SSasha Neftin
4056a146ea02SSasha Neftin wrfl();
4057a146ea02SSasha Neftin }
4058a146ea02SSasha Neftin
4059a146ea02SSasha Neftin /**
4060fccf939eSSasha Neftin * igc_irq_enable - Enable default interrupt generation settings
4061fccf939eSSasha Neftin * @adapter: board private structure
4062fccf939eSSasha Neftin */
igc_irq_enable(struct igc_adapter * adapter)4063fccf939eSSasha Neftin static void igc_irq_enable(struct igc_adapter *adapter)
4064fccf939eSSasha Neftin {
4065fccf939eSSasha Neftin struct igc_hw *hw = &adapter->hw;
4066fccf939eSSasha Neftin
4067fccf939eSSasha Neftin if (adapter->msix_entries) {
4068fccf939eSSasha Neftin u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
4069fccf939eSSasha Neftin u32 regval = rd32(IGC_EIAC);
4070fccf939eSSasha Neftin
4071fccf939eSSasha Neftin wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
4072fccf939eSSasha Neftin regval = rd32(IGC_EIAM);
4073fccf939eSSasha Neftin wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
4074fccf939eSSasha Neftin wr32(IGC_EIMS, adapter->eims_enable_mask);
4075fccf939eSSasha Neftin wr32(IGC_IMS, ims);
4076fccf939eSSasha Neftin } else {
4077fccf939eSSasha Neftin wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
4078fccf939eSSasha Neftin wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
4079fccf939eSSasha Neftin }
4080fccf939eSSasha Neftin }
4081fccf939eSSasha Neftin
4082fccf939eSSasha Neftin /**
408335f9a78aSSasha Neftin * igc_irq_disable - Mask off interrupt generation on the NIC
408435f9a78aSSasha Neftin * @adapter: board private structure
408535f9a78aSSasha Neftin */
igc_irq_disable(struct igc_adapter * adapter)408635f9a78aSSasha Neftin static void igc_irq_disable(struct igc_adapter *adapter)
408735f9a78aSSasha Neftin {
408835f9a78aSSasha Neftin struct igc_hw *hw = &adapter->hw;
408935f9a78aSSasha Neftin
409035f9a78aSSasha Neftin if (adapter->msix_entries) {
409135f9a78aSSasha Neftin u32 regval = rd32(IGC_EIAM);
409235f9a78aSSasha Neftin
409335f9a78aSSasha Neftin wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
409435f9a78aSSasha Neftin wr32(IGC_EIMC, adapter->eims_enable_mask);
409535f9a78aSSasha Neftin regval = rd32(IGC_EIAC);
409635f9a78aSSasha Neftin wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
409735f9a78aSSasha Neftin }
409835f9a78aSSasha Neftin
409935f9a78aSSasha Neftin wr32(IGC_IAM, 0);
410035f9a78aSSasha Neftin wr32(IGC_IMC, ~0);
410135f9a78aSSasha Neftin wrfl();
410235f9a78aSSasha Neftin
410335f9a78aSSasha Neftin if (adapter->msix_entries) {
410435f9a78aSSasha Neftin int vector = 0, i;
410535f9a78aSSasha Neftin
410635f9a78aSSasha Neftin synchronize_irq(adapter->msix_entries[vector++].vector);
410735f9a78aSSasha Neftin
410835f9a78aSSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++)
410935f9a78aSSasha Neftin synchronize_irq(adapter->msix_entries[vector++].vector);
411035f9a78aSSasha Neftin } else {
411135f9a78aSSasha Neftin synchronize_irq(adapter->pdev->irq);
411235f9a78aSSasha Neftin }
411335f9a78aSSasha Neftin }
411435f9a78aSSasha Neftin
igc_set_flag_queue_pairs(struct igc_adapter * adapter,const u32 max_rss_queues)411563c92c9dSSasha Neftin void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
411663c92c9dSSasha Neftin const u32 max_rss_queues)
411763c92c9dSSasha Neftin {
411863c92c9dSSasha Neftin /* Determine if we need to pair queues. */
411963c92c9dSSasha Neftin /* If rss_queues > half of max_rss_queues, pair the queues in
412063c92c9dSSasha Neftin * order to conserve interrupts due to limited supply.
412163c92c9dSSasha Neftin */
412263c92c9dSSasha Neftin if (adapter->rss_queues > (max_rss_queues / 2))
412363c92c9dSSasha Neftin adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
412463c92c9dSSasha Neftin else
412563c92c9dSSasha Neftin adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
412663c92c9dSSasha Neftin }
412763c92c9dSSasha Neftin
igc_get_max_rss_queues(struct igc_adapter * adapter)412863c92c9dSSasha Neftin unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
412963c92c9dSSasha Neftin {
41304d0710c2SAndre Guedes return IGC_MAX_RX_QUEUES;
413163c92c9dSSasha Neftin }
413263c92c9dSSasha Neftin
igc_init_queue_configuration(struct igc_adapter * adapter)413363c92c9dSSasha Neftin static void igc_init_queue_configuration(struct igc_adapter *adapter)
413463c92c9dSSasha Neftin {
413563c92c9dSSasha Neftin u32 max_rss_queues;
413663c92c9dSSasha Neftin
413763c92c9dSSasha Neftin max_rss_queues = igc_get_max_rss_queues(adapter);
413863c92c9dSSasha Neftin adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
413963c92c9dSSasha Neftin
414063c92c9dSSasha Neftin igc_set_flag_queue_pairs(adapter, max_rss_queues);
414163c92c9dSSasha Neftin }
414263c92c9dSSasha Neftin
414363c92c9dSSasha Neftin /**
414463c92c9dSSasha Neftin * igc_reset_q_vector - Reset config for interrupt vector
414563c92c9dSSasha Neftin * @adapter: board private structure to initialize
414663c92c9dSSasha Neftin * @v_idx: Index of vector to be reset
414763c92c9dSSasha Neftin *
414863c92c9dSSasha Neftin * If NAPI is enabled it will delete any references to the
414963c92c9dSSasha Neftin * NAPI struct. This is preparation for igc_free_q_vector.
415063c92c9dSSasha Neftin */
igc_reset_q_vector(struct igc_adapter * adapter,int v_idx)415163c92c9dSSasha Neftin static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
415263c92c9dSSasha Neftin {
415363c92c9dSSasha Neftin struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
415463c92c9dSSasha Neftin
415563c92c9dSSasha Neftin /* if we're coming from igc_set_interrupt_capability, the vectors are
415663c92c9dSSasha Neftin * not yet allocated
415763c92c9dSSasha Neftin */
415863c92c9dSSasha Neftin if (!q_vector)
415963c92c9dSSasha Neftin return;
416063c92c9dSSasha Neftin
416163c92c9dSSasha Neftin if (q_vector->tx.ring)
416263c92c9dSSasha Neftin adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
416363c92c9dSSasha Neftin
416463c92c9dSSasha Neftin if (q_vector->rx.ring)
416563c92c9dSSasha Neftin adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
416663c92c9dSSasha Neftin
416763c92c9dSSasha Neftin netif_napi_del(&q_vector->napi);
416863c92c9dSSasha Neftin }
416963c92c9dSSasha Neftin
417063c92c9dSSasha Neftin /**
417163c92c9dSSasha Neftin * igc_free_q_vector - Free memory allocated for specific interrupt vector
417263c92c9dSSasha Neftin * @adapter: board private structure to initialize
417363c92c9dSSasha Neftin * @v_idx: Index of vector to be freed
417463c92c9dSSasha Neftin *
417563c92c9dSSasha Neftin * This function frees the memory allocated to the q_vector.
417663c92c9dSSasha Neftin */
igc_free_q_vector(struct igc_adapter * adapter,int v_idx)417763c92c9dSSasha Neftin static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
417863c92c9dSSasha Neftin {
417963c92c9dSSasha Neftin struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
418063c92c9dSSasha Neftin
418163c92c9dSSasha Neftin adapter->q_vector[v_idx] = NULL;
418263c92c9dSSasha Neftin
418363c92c9dSSasha Neftin /* igc_get_stats64() might access the rings on this vector,
418463c92c9dSSasha Neftin * we must wait a grace period before freeing it.
418563c92c9dSSasha Neftin */
418663c92c9dSSasha Neftin if (q_vector)
418763c92c9dSSasha Neftin kfree_rcu(q_vector, rcu);
418863c92c9dSSasha Neftin }
418963c92c9dSSasha Neftin
419063c92c9dSSasha Neftin /**
419163c92c9dSSasha Neftin * igc_free_q_vectors - Free memory allocated for interrupt vectors
419263c92c9dSSasha Neftin * @adapter: board private structure to initialize
419363c92c9dSSasha Neftin *
419463c92c9dSSasha Neftin * This function frees the memory allocated to the q_vectors. In addition if
419563c92c9dSSasha Neftin * NAPI is enabled it will delete any references to the NAPI struct prior
419663c92c9dSSasha Neftin * to freeing the q_vector.
419763c92c9dSSasha Neftin */
igc_free_q_vectors(struct igc_adapter * adapter)419863c92c9dSSasha Neftin static void igc_free_q_vectors(struct igc_adapter *adapter)
419963c92c9dSSasha Neftin {
420063c92c9dSSasha Neftin int v_idx = adapter->num_q_vectors;
420163c92c9dSSasha Neftin
420263c92c9dSSasha Neftin adapter->num_tx_queues = 0;
420363c92c9dSSasha Neftin adapter->num_rx_queues = 0;
420463c92c9dSSasha Neftin adapter->num_q_vectors = 0;
420563c92c9dSSasha Neftin
420663c92c9dSSasha Neftin while (v_idx--) {
420763c92c9dSSasha Neftin igc_reset_q_vector(adapter, v_idx);
420863c92c9dSSasha Neftin igc_free_q_vector(adapter, v_idx);
420963c92c9dSSasha Neftin }
421063c92c9dSSasha Neftin }
421163c92c9dSSasha Neftin
421263c92c9dSSasha Neftin /**
421363c92c9dSSasha Neftin * igc_update_itr - update the dynamic ITR value based on statistics
421463c92c9dSSasha Neftin * @q_vector: pointer to q_vector
421563c92c9dSSasha Neftin * @ring_container: ring info to update the itr for
421663c92c9dSSasha Neftin *
421763c92c9dSSasha Neftin * Stores a new ITR value based on packets and byte
421863c92c9dSSasha Neftin * counts during the last interrupt. The advantage of per interrupt
421963c92c9dSSasha Neftin * computation is faster updates and more accurate ITR for the current
422063c92c9dSSasha Neftin * traffic pattern. Constants in this function were computed
422163c92c9dSSasha Neftin * based on theoretical maximum wire speed and thresholds were set based
422263c92c9dSSasha Neftin * on testing data as well as attempting to minimize response time
422363c92c9dSSasha Neftin * while increasing bulk throughput.
422463c92c9dSSasha Neftin * NOTE: These calculations are only valid when operating in a single-
422563c92c9dSSasha Neftin * queue environment.
422663c92c9dSSasha Neftin */
igc_update_itr(struct igc_q_vector * q_vector,struct igc_ring_container * ring_container)422763c92c9dSSasha Neftin static void igc_update_itr(struct igc_q_vector *q_vector,
422863c92c9dSSasha Neftin struct igc_ring_container *ring_container)
422963c92c9dSSasha Neftin {
423063c92c9dSSasha Neftin unsigned int packets = ring_container->total_packets;
423163c92c9dSSasha Neftin unsigned int bytes = ring_container->total_bytes;
423263c92c9dSSasha Neftin u8 itrval = ring_container->itr;
423363c92c9dSSasha Neftin
423463c92c9dSSasha Neftin /* no packets, exit with status unchanged */
423563c92c9dSSasha Neftin if (packets == 0)
423663c92c9dSSasha Neftin return;
423763c92c9dSSasha Neftin
423863c92c9dSSasha Neftin switch (itrval) {
423963c92c9dSSasha Neftin case lowest_latency:
424063c92c9dSSasha Neftin /* handle TSO and jumbo frames */
424163c92c9dSSasha Neftin if (bytes / packets > 8000)
424263c92c9dSSasha Neftin itrval = bulk_latency;
424363c92c9dSSasha Neftin else if ((packets < 5) && (bytes > 512))
424463c92c9dSSasha Neftin itrval = low_latency;
424563c92c9dSSasha Neftin break;
424663c92c9dSSasha Neftin case low_latency: /* 50 usec aka 20000 ints/s */
424763c92c9dSSasha Neftin if (bytes > 10000) {
424863c92c9dSSasha Neftin /* this if handles the TSO accounting */
424963c92c9dSSasha Neftin if (bytes / packets > 8000)
425063c92c9dSSasha Neftin itrval = bulk_latency;
425163c92c9dSSasha Neftin else if ((packets < 10) || ((bytes / packets) > 1200))
425263c92c9dSSasha Neftin itrval = bulk_latency;
425363c92c9dSSasha Neftin else if ((packets > 35))
425463c92c9dSSasha Neftin itrval = lowest_latency;
425563c92c9dSSasha Neftin } else if (bytes / packets > 2000) {
425663c92c9dSSasha Neftin itrval = bulk_latency;
425763c92c9dSSasha Neftin } else if (packets <= 2 && bytes < 512) {
425863c92c9dSSasha Neftin itrval = lowest_latency;
425963c92c9dSSasha Neftin }
426063c92c9dSSasha Neftin break;
426163c92c9dSSasha Neftin case bulk_latency: /* 250 usec aka 4000 ints/s */
426263c92c9dSSasha Neftin if (bytes > 25000) {
426363c92c9dSSasha Neftin if (packets > 35)
426463c92c9dSSasha Neftin itrval = low_latency;
426563c92c9dSSasha Neftin } else if (bytes < 1500) {
426663c92c9dSSasha Neftin itrval = low_latency;
426763c92c9dSSasha Neftin }
426863c92c9dSSasha Neftin break;
426963c92c9dSSasha Neftin }
427063c92c9dSSasha Neftin
427163c92c9dSSasha Neftin /* clear work counters since we have the values we need */
427263c92c9dSSasha Neftin ring_container->total_bytes = 0;
427363c92c9dSSasha Neftin ring_container->total_packets = 0;
427463c92c9dSSasha Neftin
427563c92c9dSSasha Neftin /* write updated itr to ring container */
427663c92c9dSSasha Neftin ring_container->itr = itrval;
427763c92c9dSSasha Neftin }
427863c92c9dSSasha Neftin
igc_set_itr(struct igc_q_vector * q_vector)427963c92c9dSSasha Neftin static void igc_set_itr(struct igc_q_vector *q_vector)
428063c92c9dSSasha Neftin {
428163c92c9dSSasha Neftin struct igc_adapter *adapter = q_vector->adapter;
428263c92c9dSSasha Neftin u32 new_itr = q_vector->itr_val;
428363c92c9dSSasha Neftin u8 current_itr = 0;
428463c92c9dSSasha Neftin
428563c92c9dSSasha Neftin /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
428663c92c9dSSasha Neftin switch (adapter->link_speed) {
428763c92c9dSSasha Neftin case SPEED_10:
428863c92c9dSSasha Neftin case SPEED_100:
428963c92c9dSSasha Neftin current_itr = 0;
429063c92c9dSSasha Neftin new_itr = IGC_4K_ITR;
429163c92c9dSSasha Neftin goto set_itr_now;
429263c92c9dSSasha Neftin default:
429363c92c9dSSasha Neftin break;
429463c92c9dSSasha Neftin }
429563c92c9dSSasha Neftin
429663c92c9dSSasha Neftin igc_update_itr(q_vector, &q_vector->tx);
429763c92c9dSSasha Neftin igc_update_itr(q_vector, &q_vector->rx);
429863c92c9dSSasha Neftin
429963c92c9dSSasha Neftin current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
430063c92c9dSSasha Neftin
430163c92c9dSSasha Neftin /* conservative mode (itr 3) eliminates the lowest_latency setting */
430263c92c9dSSasha Neftin if (current_itr == lowest_latency &&
430363c92c9dSSasha Neftin ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
430463c92c9dSSasha Neftin (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
430563c92c9dSSasha Neftin current_itr = low_latency;
430663c92c9dSSasha Neftin
430763c92c9dSSasha Neftin switch (current_itr) {
430863c92c9dSSasha Neftin /* counts and packets in update_itr are dependent on these numbers */
430963c92c9dSSasha Neftin case lowest_latency:
431063c92c9dSSasha Neftin new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
431163c92c9dSSasha Neftin break;
431263c92c9dSSasha Neftin case low_latency:
431363c92c9dSSasha Neftin new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
431463c92c9dSSasha Neftin break;
431563c92c9dSSasha Neftin case bulk_latency:
431663c92c9dSSasha Neftin new_itr = IGC_4K_ITR; /* 4,000 ints/sec */
431763c92c9dSSasha Neftin break;
431863c92c9dSSasha Neftin default:
431963c92c9dSSasha Neftin break;
432063c92c9dSSasha Neftin }
432163c92c9dSSasha Neftin
432263c92c9dSSasha Neftin set_itr_now:
432363c92c9dSSasha Neftin if (new_itr != q_vector->itr_val) {
432463c92c9dSSasha Neftin /* this attempts to bias the interrupt rate towards Bulk
432563c92c9dSSasha Neftin * by adding intermediate steps when interrupt rate is
432663c92c9dSSasha Neftin * increasing
432763c92c9dSSasha Neftin */
432863c92c9dSSasha Neftin new_itr = new_itr > q_vector->itr_val ?
432963c92c9dSSasha Neftin max((new_itr * q_vector->itr_val) /
433063c92c9dSSasha Neftin (new_itr + (q_vector->itr_val >> 2)),
433163c92c9dSSasha Neftin new_itr) : new_itr;
433263c92c9dSSasha Neftin /* Don't write the value here; it resets the adapter's
433363c92c9dSSasha Neftin * internal timer, and causes us to delay far longer than
433463c92c9dSSasha Neftin * we should between interrupts. Instead, we write the ITR
433563c92c9dSSasha Neftin * value at the beginning of the next interrupt so the timing
433663c92c9dSSasha Neftin * ends up being correct.
433763c92c9dSSasha Neftin */
433863c92c9dSSasha Neftin q_vector->itr_val = new_itr;
433963c92c9dSSasha Neftin q_vector->set_itr = 1;
434063c92c9dSSasha Neftin }
434163c92c9dSSasha Neftin }
434263c92c9dSSasha Neftin
igc_reset_interrupt_capability(struct igc_adapter * adapter)434363c92c9dSSasha Neftin static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
434463c92c9dSSasha Neftin {
434563c92c9dSSasha Neftin int v_idx = adapter->num_q_vectors;
434663c92c9dSSasha Neftin
434763c92c9dSSasha Neftin if (adapter->msix_entries) {
434863c92c9dSSasha Neftin pci_disable_msix(adapter->pdev);
434963c92c9dSSasha Neftin kfree(adapter->msix_entries);
435063c92c9dSSasha Neftin adapter->msix_entries = NULL;
435163c92c9dSSasha Neftin } else if (adapter->flags & IGC_FLAG_HAS_MSI) {
435263c92c9dSSasha Neftin pci_disable_msi(adapter->pdev);
435363c92c9dSSasha Neftin }
435463c92c9dSSasha Neftin
435563c92c9dSSasha Neftin while (v_idx--)
435663c92c9dSSasha Neftin igc_reset_q_vector(adapter, v_idx);
435763c92c9dSSasha Neftin }
435863c92c9dSSasha Neftin
435963c92c9dSSasha Neftin /**
436063c92c9dSSasha Neftin * igc_set_interrupt_capability - set MSI or MSI-X if supported
436163c92c9dSSasha Neftin * @adapter: Pointer to adapter structure
436263c92c9dSSasha Neftin * @msix: boolean value for MSI-X capability
436363c92c9dSSasha Neftin *
436463c92c9dSSasha Neftin * Attempt to configure interrupts using the best available
436563c92c9dSSasha Neftin * capabilities of the hardware and kernel.
436663c92c9dSSasha Neftin */
igc_set_interrupt_capability(struct igc_adapter * adapter,bool msix)436763c92c9dSSasha Neftin static void igc_set_interrupt_capability(struct igc_adapter *adapter,
436863c92c9dSSasha Neftin bool msix)
436963c92c9dSSasha Neftin {
437063c92c9dSSasha Neftin int numvecs, i;
437163c92c9dSSasha Neftin int err;
437263c92c9dSSasha Neftin
437363c92c9dSSasha Neftin if (!msix)
437463c92c9dSSasha Neftin goto msi_only;
437563c92c9dSSasha Neftin adapter->flags |= IGC_FLAG_HAS_MSIX;
437663c92c9dSSasha Neftin
437763c92c9dSSasha Neftin /* Number of supported queues. */
437863c92c9dSSasha Neftin adapter->num_rx_queues = adapter->rss_queues;
437963c92c9dSSasha Neftin
438063c92c9dSSasha Neftin adapter->num_tx_queues = adapter->rss_queues;
438163c92c9dSSasha Neftin
438263c92c9dSSasha Neftin /* start with one vector for every Rx queue */
438363c92c9dSSasha Neftin numvecs = adapter->num_rx_queues;
438463c92c9dSSasha Neftin
438563c92c9dSSasha Neftin /* if Tx handler is separate add 1 for every Tx queue */
438663c92c9dSSasha Neftin if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
438763c92c9dSSasha Neftin numvecs += adapter->num_tx_queues;
438863c92c9dSSasha Neftin
438963c92c9dSSasha Neftin /* store the number of vectors reserved for queues */
439063c92c9dSSasha Neftin adapter->num_q_vectors = numvecs;
439163c92c9dSSasha Neftin
439263c92c9dSSasha Neftin /* add 1 vector for link status interrupts */
439363c92c9dSSasha Neftin numvecs++;
439463c92c9dSSasha Neftin
439563c92c9dSSasha Neftin adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
439663c92c9dSSasha Neftin GFP_KERNEL);
439763c92c9dSSasha Neftin
439863c92c9dSSasha Neftin if (!adapter->msix_entries)
439963c92c9dSSasha Neftin return;
440063c92c9dSSasha Neftin
440163c92c9dSSasha Neftin /* populate entry values */
440263c92c9dSSasha Neftin for (i = 0; i < numvecs; i++)
440363c92c9dSSasha Neftin adapter->msix_entries[i].entry = i;
440463c92c9dSSasha Neftin
440563c92c9dSSasha Neftin err = pci_enable_msix_range(adapter->pdev,
440663c92c9dSSasha Neftin adapter->msix_entries,
440763c92c9dSSasha Neftin numvecs,
440863c92c9dSSasha Neftin numvecs);
440963c92c9dSSasha Neftin if (err > 0)
441063c92c9dSSasha Neftin return;
441163c92c9dSSasha Neftin
441263c92c9dSSasha Neftin kfree(adapter->msix_entries);
441363c92c9dSSasha Neftin adapter->msix_entries = NULL;
441463c92c9dSSasha Neftin
441563c92c9dSSasha Neftin igc_reset_interrupt_capability(adapter);
441663c92c9dSSasha Neftin
441763c92c9dSSasha Neftin msi_only:
441863c92c9dSSasha Neftin adapter->flags &= ~IGC_FLAG_HAS_MSIX;
441963c92c9dSSasha Neftin
442063c92c9dSSasha Neftin adapter->rss_queues = 1;
442163c92c9dSSasha Neftin adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
442263c92c9dSSasha Neftin adapter->num_rx_queues = 1;
442363c92c9dSSasha Neftin adapter->num_tx_queues = 1;
442463c92c9dSSasha Neftin adapter->num_q_vectors = 1;
442563c92c9dSSasha Neftin if (!pci_enable_msi(adapter->pdev))
442663c92c9dSSasha Neftin adapter->flags |= IGC_FLAG_HAS_MSI;
442763c92c9dSSasha Neftin }
442863c92c9dSSasha Neftin
442963c92c9dSSasha Neftin /**
443063c92c9dSSasha Neftin * igc_update_ring_itr - update the dynamic ITR value based on packet size
443163c92c9dSSasha Neftin * @q_vector: pointer to q_vector
443263c92c9dSSasha Neftin *
443363c92c9dSSasha Neftin * Stores a new ITR value based on strictly on packet size. This
443463c92c9dSSasha Neftin * algorithm is less sophisticated than that used in igc_update_itr,
443563c92c9dSSasha Neftin * due to the difficulty of synchronizing statistics across multiple
443663c92c9dSSasha Neftin * receive rings. The divisors and thresholds used by this function
443763c92c9dSSasha Neftin * were determined based on theoretical maximum wire speed and testing
443863c92c9dSSasha Neftin * data, in order to minimize response time while increasing bulk
443963c92c9dSSasha Neftin * throughput.
444063c92c9dSSasha Neftin * NOTE: This function is called only when operating in a multiqueue
444163c92c9dSSasha Neftin * receive environment.
444263c92c9dSSasha Neftin */
igc_update_ring_itr(struct igc_q_vector * q_vector)444363c92c9dSSasha Neftin static void igc_update_ring_itr(struct igc_q_vector *q_vector)
444463c92c9dSSasha Neftin {
444563c92c9dSSasha Neftin struct igc_adapter *adapter = q_vector->adapter;
444663c92c9dSSasha Neftin int new_val = q_vector->itr_val;
444763c92c9dSSasha Neftin int avg_wire_size = 0;
444863c92c9dSSasha Neftin unsigned int packets;
444963c92c9dSSasha Neftin
445063c92c9dSSasha Neftin /* For non-gigabit speeds, just fix the interrupt rate at 4000
445163c92c9dSSasha Neftin * ints/sec - ITR timer value of 120 ticks.
445263c92c9dSSasha Neftin */
445363c92c9dSSasha Neftin switch (adapter->link_speed) {
445463c92c9dSSasha Neftin case SPEED_10:
445563c92c9dSSasha Neftin case SPEED_100:
445663c92c9dSSasha Neftin new_val = IGC_4K_ITR;
445763c92c9dSSasha Neftin goto set_itr_val;
445863c92c9dSSasha Neftin default:
445963c92c9dSSasha Neftin break;
446063c92c9dSSasha Neftin }
446163c92c9dSSasha Neftin
446263c92c9dSSasha Neftin packets = q_vector->rx.total_packets;
446363c92c9dSSasha Neftin if (packets)
446463c92c9dSSasha Neftin avg_wire_size = q_vector->rx.total_bytes / packets;
446563c92c9dSSasha Neftin
446663c92c9dSSasha Neftin packets = q_vector->tx.total_packets;
446763c92c9dSSasha Neftin if (packets)
446863c92c9dSSasha Neftin avg_wire_size = max_t(u32, avg_wire_size,
446963c92c9dSSasha Neftin q_vector->tx.total_bytes / packets);
447063c92c9dSSasha Neftin
447163c92c9dSSasha Neftin /* if avg_wire_size isn't set no work was done */
447263c92c9dSSasha Neftin if (!avg_wire_size)
447363c92c9dSSasha Neftin goto clear_counts;
447463c92c9dSSasha Neftin
447563c92c9dSSasha Neftin /* Add 24 bytes to size to account for CRC, preamble, and gap */
447663c92c9dSSasha Neftin avg_wire_size += 24;
447763c92c9dSSasha Neftin
447863c92c9dSSasha Neftin /* Don't starve jumbo frames */
447963c92c9dSSasha Neftin avg_wire_size = min(avg_wire_size, 3000);
448063c92c9dSSasha Neftin
448163c92c9dSSasha Neftin /* Give a little boost to mid-size frames */
448263c92c9dSSasha Neftin if (avg_wire_size > 300 && avg_wire_size < 1200)
448363c92c9dSSasha Neftin new_val = avg_wire_size / 3;
448463c92c9dSSasha Neftin else
448563c92c9dSSasha Neftin new_val = avg_wire_size / 2;
448663c92c9dSSasha Neftin
448763c92c9dSSasha Neftin /* conservative mode (itr 3) eliminates the lowest_latency setting */
448863c92c9dSSasha Neftin if (new_val < IGC_20K_ITR &&
448963c92c9dSSasha Neftin ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
449063c92c9dSSasha Neftin (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
449163c92c9dSSasha Neftin new_val = IGC_20K_ITR;
449263c92c9dSSasha Neftin
449363c92c9dSSasha Neftin set_itr_val:
449463c92c9dSSasha Neftin if (new_val != q_vector->itr_val) {
449563c92c9dSSasha Neftin q_vector->itr_val = new_val;
449663c92c9dSSasha Neftin q_vector->set_itr = 1;
449763c92c9dSSasha Neftin }
449863c92c9dSSasha Neftin clear_counts:
449963c92c9dSSasha Neftin q_vector->rx.total_bytes = 0;
450063c92c9dSSasha Neftin q_vector->rx.total_packets = 0;
450163c92c9dSSasha Neftin q_vector->tx.total_bytes = 0;
450263c92c9dSSasha Neftin q_vector->tx.total_packets = 0;
450363c92c9dSSasha Neftin }
450463c92c9dSSasha Neftin
igc_ring_irq_enable(struct igc_q_vector * q_vector)450563c92c9dSSasha Neftin static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
450663c92c9dSSasha Neftin {
450763c92c9dSSasha Neftin struct igc_adapter *adapter = q_vector->adapter;
450863c92c9dSSasha Neftin struct igc_hw *hw = &adapter->hw;
450963c92c9dSSasha Neftin
451063c92c9dSSasha Neftin if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
451163c92c9dSSasha Neftin (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
451263c92c9dSSasha Neftin if (adapter->num_q_vectors == 1)
451363c92c9dSSasha Neftin igc_set_itr(q_vector);
451463c92c9dSSasha Neftin else
451563c92c9dSSasha Neftin igc_update_ring_itr(q_vector);
451663c92c9dSSasha Neftin }
451763c92c9dSSasha Neftin
451863c92c9dSSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state)) {
451963c92c9dSSasha Neftin if (adapter->msix_entries)
452063c92c9dSSasha Neftin wr32(IGC_EIMS, q_vector->eims_value);
452163c92c9dSSasha Neftin else
452263c92c9dSSasha Neftin igc_irq_enable(adapter);
452363c92c9dSSasha Neftin }
452463c92c9dSSasha Neftin }
452563c92c9dSSasha Neftin
igc_add_ring(struct igc_ring * ring,struct igc_ring_container * head)452663c92c9dSSasha Neftin static void igc_add_ring(struct igc_ring *ring,
452763c92c9dSSasha Neftin struct igc_ring_container *head)
452863c92c9dSSasha Neftin {
452963c92c9dSSasha Neftin head->ring = ring;
453063c92c9dSSasha Neftin head->count++;
453163c92c9dSSasha Neftin }
453263c92c9dSSasha Neftin
453363c92c9dSSasha Neftin /**
453463c92c9dSSasha Neftin * igc_cache_ring_register - Descriptor ring to register mapping
453563c92c9dSSasha Neftin * @adapter: board private structure to initialize
453663c92c9dSSasha Neftin *
453763c92c9dSSasha Neftin * Once we know the feature-set enabled for the device, we'll cache
453863c92c9dSSasha Neftin * the register offset the descriptor ring is assigned to.
453963c92c9dSSasha Neftin */
igc_cache_ring_register(struct igc_adapter * adapter)454063c92c9dSSasha Neftin static void igc_cache_ring_register(struct igc_adapter *adapter)
454163c92c9dSSasha Neftin {
454263c92c9dSSasha Neftin int i = 0, j = 0;
454363c92c9dSSasha Neftin
454463c92c9dSSasha Neftin switch (adapter->hw.mac.type) {
454563c92c9dSSasha Neftin case igc_i225:
454663c92c9dSSasha Neftin default:
454763c92c9dSSasha Neftin for (; i < adapter->num_rx_queues; i++)
454863c92c9dSSasha Neftin adapter->rx_ring[i]->reg_idx = i;
454963c92c9dSSasha Neftin for (; j < adapter->num_tx_queues; j++)
455063c92c9dSSasha Neftin adapter->tx_ring[j]->reg_idx = j;
455163c92c9dSSasha Neftin break;
455263c92c9dSSasha Neftin }
455363c92c9dSSasha Neftin }
455463c92c9dSSasha Neftin
455563c92c9dSSasha Neftin /**
455663c92c9dSSasha Neftin * igc_poll - NAPI Rx polling callback
455763c92c9dSSasha Neftin * @napi: napi polling structure
455863c92c9dSSasha Neftin * @budget: count of how many packets we should handle
455963c92c9dSSasha Neftin */
igc_poll(struct napi_struct * napi,int budget)456063c92c9dSSasha Neftin static int igc_poll(struct napi_struct *napi, int budget)
456163c92c9dSSasha Neftin {
456263c92c9dSSasha Neftin struct igc_q_vector *q_vector = container_of(napi,
456363c92c9dSSasha Neftin struct igc_q_vector,
456463c92c9dSSasha Neftin napi);
4565fc9df2a0SAndre Guedes struct igc_ring *rx_ring = q_vector->rx.ring;
456663c92c9dSSasha Neftin bool clean_complete = true;
456763c92c9dSSasha Neftin int work_done = 0;
456863c92c9dSSasha Neftin
456963c92c9dSSasha Neftin if (q_vector->tx.ring)
457063c92c9dSSasha Neftin clean_complete = igc_clean_tx_irq(q_vector, budget);
457163c92c9dSSasha Neftin
4572fc9df2a0SAndre Guedes if (rx_ring) {
4573fc9df2a0SAndre Guedes int cleaned = rx_ring->xsk_pool ?
4574fc9df2a0SAndre Guedes igc_clean_rx_irq_zc(q_vector, budget) :
4575fc9df2a0SAndre Guedes igc_clean_rx_irq(q_vector, budget);
457663c92c9dSSasha Neftin
457763c92c9dSSasha Neftin work_done += cleaned;
457863c92c9dSSasha Neftin if (cleaned >= budget)
457963c92c9dSSasha Neftin clean_complete = false;
458063c92c9dSSasha Neftin }
458163c92c9dSSasha Neftin
458263c92c9dSSasha Neftin /* If all work not completed, return budget and keep polling */
458363c92c9dSSasha Neftin if (!clean_complete)
458463c92c9dSSasha Neftin return budget;
458563c92c9dSSasha Neftin
458663c92c9dSSasha Neftin /* Exit the polling mode, but don't re-enable interrupts if stack might
458763c92c9dSSasha Neftin * poll us due to busy-polling
458863c92c9dSSasha Neftin */
458963c92c9dSSasha Neftin if (likely(napi_complete_done(napi, work_done)))
459063c92c9dSSasha Neftin igc_ring_irq_enable(q_vector);
459163c92c9dSSasha Neftin
459263c92c9dSSasha Neftin return min(work_done, budget - 1);
459363c92c9dSSasha Neftin }
459463c92c9dSSasha Neftin
459563c92c9dSSasha Neftin /**
459663c92c9dSSasha Neftin * igc_alloc_q_vector - Allocate memory for a single interrupt vector
459763c92c9dSSasha Neftin * @adapter: board private structure to initialize
459863c92c9dSSasha Neftin * @v_count: q_vectors allocated on adapter, used for ring interleaving
459963c92c9dSSasha Neftin * @v_idx: index of vector in adapter struct
460063c92c9dSSasha Neftin * @txr_count: total number of Tx rings to allocate
460163c92c9dSSasha Neftin * @txr_idx: index of first Tx ring to allocate
460263c92c9dSSasha Neftin * @rxr_count: total number of Rx rings to allocate
460363c92c9dSSasha Neftin * @rxr_idx: index of first Rx ring to allocate
460463c92c9dSSasha Neftin *
460563c92c9dSSasha Neftin * We allocate one q_vector. If allocation fails we return -ENOMEM.
460663c92c9dSSasha Neftin */
igc_alloc_q_vector(struct igc_adapter * adapter,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)460763c92c9dSSasha Neftin static int igc_alloc_q_vector(struct igc_adapter *adapter,
460863c92c9dSSasha Neftin unsigned int v_count, unsigned int v_idx,
460963c92c9dSSasha Neftin unsigned int txr_count, unsigned int txr_idx,
461063c92c9dSSasha Neftin unsigned int rxr_count, unsigned int rxr_idx)
461163c92c9dSSasha Neftin {
461263c92c9dSSasha Neftin struct igc_q_vector *q_vector;
461363c92c9dSSasha Neftin struct igc_ring *ring;
461463c92c9dSSasha Neftin int ring_count;
461563c92c9dSSasha Neftin
461663c92c9dSSasha Neftin /* igc only supports 1 Tx and/or 1 Rx queue per vector */
461763c92c9dSSasha Neftin if (txr_count > 1 || rxr_count > 1)
461863c92c9dSSasha Neftin return -ENOMEM;
461963c92c9dSSasha Neftin
462063c92c9dSSasha Neftin ring_count = txr_count + rxr_count;
462163c92c9dSSasha Neftin
462263c92c9dSSasha Neftin /* allocate q_vector and rings */
462363c92c9dSSasha Neftin q_vector = adapter->q_vector[v_idx];
462463c92c9dSSasha Neftin if (!q_vector)
462563c92c9dSSasha Neftin q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
462663c92c9dSSasha Neftin GFP_KERNEL);
462763c92c9dSSasha Neftin else
462863c92c9dSSasha Neftin memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
462963c92c9dSSasha Neftin if (!q_vector)
463063c92c9dSSasha Neftin return -ENOMEM;
463163c92c9dSSasha Neftin
463263c92c9dSSasha Neftin /* initialize NAPI */
4633b48b89f9SJakub Kicinski netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
463463c92c9dSSasha Neftin
463563c92c9dSSasha Neftin /* tie q_vector and adapter together */
463663c92c9dSSasha Neftin adapter->q_vector[v_idx] = q_vector;
463763c92c9dSSasha Neftin q_vector->adapter = adapter;
463863c92c9dSSasha Neftin
463963c92c9dSSasha Neftin /* initialize work limits */
464063c92c9dSSasha Neftin q_vector->tx.work_limit = adapter->tx_work_limit;
464163c92c9dSSasha Neftin
464263c92c9dSSasha Neftin /* initialize ITR configuration */
464363c92c9dSSasha Neftin q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
464463c92c9dSSasha Neftin q_vector->itr_val = IGC_START_ITR;
464563c92c9dSSasha Neftin
464663c92c9dSSasha Neftin /* initialize pointer to rings */
464763c92c9dSSasha Neftin ring = q_vector->ring;
464863c92c9dSSasha Neftin
464963c92c9dSSasha Neftin /* initialize ITR */
465063c92c9dSSasha Neftin if (rxr_count) {
465163c92c9dSSasha Neftin /* rx or rx/tx vector */
465263c92c9dSSasha Neftin if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
465363c92c9dSSasha Neftin q_vector->itr_val = adapter->rx_itr_setting;
465463c92c9dSSasha Neftin } else {
465563c92c9dSSasha Neftin /* tx only vector */
465663c92c9dSSasha Neftin if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
465763c92c9dSSasha Neftin q_vector->itr_val = adapter->tx_itr_setting;
465863c92c9dSSasha Neftin }
465963c92c9dSSasha Neftin
466063c92c9dSSasha Neftin if (txr_count) {
466163c92c9dSSasha Neftin /* assign generic ring traits */
466263c92c9dSSasha Neftin ring->dev = &adapter->pdev->dev;
466363c92c9dSSasha Neftin ring->netdev = adapter->netdev;
466463c92c9dSSasha Neftin
466563c92c9dSSasha Neftin /* configure backlink on ring */
466663c92c9dSSasha Neftin ring->q_vector = q_vector;
466763c92c9dSSasha Neftin
466863c92c9dSSasha Neftin /* update q_vector Tx values */
466963c92c9dSSasha Neftin igc_add_ring(ring, &q_vector->tx);
467063c92c9dSSasha Neftin
467163c92c9dSSasha Neftin /* apply Tx specific ring traits */
467263c92c9dSSasha Neftin ring->count = adapter->tx_ring_count;
467363c92c9dSSasha Neftin ring->queue_index = txr_idx;
467463c92c9dSSasha Neftin
467563c92c9dSSasha Neftin /* assign ring to adapter */
467663c92c9dSSasha Neftin adapter->tx_ring[txr_idx] = ring;
467763c92c9dSSasha Neftin
467863c92c9dSSasha Neftin /* push pointer to next ring */
467963c92c9dSSasha Neftin ring++;
468063c92c9dSSasha Neftin }
468163c92c9dSSasha Neftin
468263c92c9dSSasha Neftin if (rxr_count) {
468363c92c9dSSasha Neftin /* assign generic ring traits */
468463c92c9dSSasha Neftin ring->dev = &adapter->pdev->dev;
468563c92c9dSSasha Neftin ring->netdev = adapter->netdev;
468663c92c9dSSasha Neftin
468763c92c9dSSasha Neftin /* configure backlink on ring */
468863c92c9dSSasha Neftin ring->q_vector = q_vector;
468963c92c9dSSasha Neftin
469063c92c9dSSasha Neftin /* update q_vector Rx values */
469163c92c9dSSasha Neftin igc_add_ring(ring, &q_vector->rx);
469263c92c9dSSasha Neftin
469363c92c9dSSasha Neftin /* apply Rx specific ring traits */
469463c92c9dSSasha Neftin ring->count = adapter->rx_ring_count;
469563c92c9dSSasha Neftin ring->queue_index = rxr_idx;
469663c92c9dSSasha Neftin
469763c92c9dSSasha Neftin /* assign ring to adapter */
469863c92c9dSSasha Neftin adapter->rx_ring[rxr_idx] = ring;
469963c92c9dSSasha Neftin }
470063c92c9dSSasha Neftin
470163c92c9dSSasha Neftin return 0;
470263c92c9dSSasha Neftin }
470363c92c9dSSasha Neftin
470463c92c9dSSasha Neftin /**
470563c92c9dSSasha Neftin * igc_alloc_q_vectors - Allocate memory for interrupt vectors
470663c92c9dSSasha Neftin * @adapter: board private structure to initialize
470763c92c9dSSasha Neftin *
470863c92c9dSSasha Neftin * We allocate one q_vector per queue interrupt. If allocation fails we
470963c92c9dSSasha Neftin * return -ENOMEM.
471063c92c9dSSasha Neftin */
igc_alloc_q_vectors(struct igc_adapter * adapter)471163c92c9dSSasha Neftin static int igc_alloc_q_vectors(struct igc_adapter *adapter)
471263c92c9dSSasha Neftin {
471363c92c9dSSasha Neftin int rxr_remaining = adapter->num_rx_queues;
471463c92c9dSSasha Neftin int txr_remaining = adapter->num_tx_queues;
471563c92c9dSSasha Neftin int rxr_idx = 0, txr_idx = 0, v_idx = 0;
471663c92c9dSSasha Neftin int q_vectors = adapter->num_q_vectors;
471763c92c9dSSasha Neftin int err;
471863c92c9dSSasha Neftin
471963c92c9dSSasha Neftin if (q_vectors >= (rxr_remaining + txr_remaining)) {
472063c92c9dSSasha Neftin for (; rxr_remaining; v_idx++) {
472163c92c9dSSasha Neftin err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
472263c92c9dSSasha Neftin 0, 0, 1, rxr_idx);
472363c92c9dSSasha Neftin
472463c92c9dSSasha Neftin if (err)
472563c92c9dSSasha Neftin goto err_out;
472663c92c9dSSasha Neftin
472763c92c9dSSasha Neftin /* update counts and index */
472863c92c9dSSasha Neftin rxr_remaining--;
472963c92c9dSSasha Neftin rxr_idx++;
473063c92c9dSSasha Neftin }
473163c92c9dSSasha Neftin }
473263c92c9dSSasha Neftin
473363c92c9dSSasha Neftin for (; v_idx < q_vectors; v_idx++) {
473463c92c9dSSasha Neftin int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
473563c92c9dSSasha Neftin int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
473663c92c9dSSasha Neftin
473763c92c9dSSasha Neftin err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
473863c92c9dSSasha Neftin tqpv, txr_idx, rqpv, rxr_idx);
473963c92c9dSSasha Neftin
474063c92c9dSSasha Neftin if (err)
474163c92c9dSSasha Neftin goto err_out;
474263c92c9dSSasha Neftin
474363c92c9dSSasha Neftin /* update counts and index */
474463c92c9dSSasha Neftin rxr_remaining -= rqpv;
474563c92c9dSSasha Neftin txr_remaining -= tqpv;
474663c92c9dSSasha Neftin rxr_idx++;
474763c92c9dSSasha Neftin txr_idx++;
474863c92c9dSSasha Neftin }
474963c92c9dSSasha Neftin
475063c92c9dSSasha Neftin return 0;
475163c92c9dSSasha Neftin
475263c92c9dSSasha Neftin err_out:
475363c92c9dSSasha Neftin adapter->num_tx_queues = 0;
475463c92c9dSSasha Neftin adapter->num_rx_queues = 0;
475563c92c9dSSasha Neftin adapter->num_q_vectors = 0;
475663c92c9dSSasha Neftin
475763c92c9dSSasha Neftin while (v_idx--)
475863c92c9dSSasha Neftin igc_free_q_vector(adapter, v_idx);
475963c92c9dSSasha Neftin
476063c92c9dSSasha Neftin return -ENOMEM;
476163c92c9dSSasha Neftin }
476263c92c9dSSasha Neftin
476363c92c9dSSasha Neftin /**
476463c92c9dSSasha Neftin * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
476563c92c9dSSasha Neftin * @adapter: Pointer to adapter structure
476663c92c9dSSasha Neftin * @msix: boolean for MSI-X capability
476763c92c9dSSasha Neftin *
476863c92c9dSSasha Neftin * This function initializes the interrupts and allocates all of the queues.
476963c92c9dSSasha Neftin */
igc_init_interrupt_scheme(struct igc_adapter * adapter,bool msix)477063c92c9dSSasha Neftin static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
477163c92c9dSSasha Neftin {
477225f06effSAndre Guedes struct net_device *dev = adapter->netdev;
477363c92c9dSSasha Neftin int err = 0;
477463c92c9dSSasha Neftin
477563c92c9dSSasha Neftin igc_set_interrupt_capability(adapter, msix);
477663c92c9dSSasha Neftin
477763c92c9dSSasha Neftin err = igc_alloc_q_vectors(adapter);
477863c92c9dSSasha Neftin if (err) {
477925f06effSAndre Guedes netdev_err(dev, "Unable to allocate memory for vectors\n");
478063c92c9dSSasha Neftin goto err_alloc_q_vectors;
478163c92c9dSSasha Neftin }
478263c92c9dSSasha Neftin
478363c92c9dSSasha Neftin igc_cache_ring_register(adapter);
478463c92c9dSSasha Neftin
478563c92c9dSSasha Neftin return 0;
478663c92c9dSSasha Neftin
478763c92c9dSSasha Neftin err_alloc_q_vectors:
478863c92c9dSSasha Neftin igc_reset_interrupt_capability(adapter);
478963c92c9dSSasha Neftin return err;
479063c92c9dSSasha Neftin }
479163c92c9dSSasha Neftin
479263c92c9dSSasha Neftin /**
479363c92c9dSSasha Neftin * igc_sw_init - Initialize general software structures (struct igc_adapter)
479463c92c9dSSasha Neftin * @adapter: board private structure to initialize
479563c92c9dSSasha Neftin *
479663c92c9dSSasha Neftin * igc_sw_init initializes the Adapter private data structure.
479763c92c9dSSasha Neftin * Fields are initialized based on PCI device information and
479863c92c9dSSasha Neftin * OS network device settings (MTU size).
479963c92c9dSSasha Neftin */
igc_sw_init(struct igc_adapter * adapter)480063c92c9dSSasha Neftin static int igc_sw_init(struct igc_adapter *adapter)
480163c92c9dSSasha Neftin {
480263c92c9dSSasha Neftin struct net_device *netdev = adapter->netdev;
480363c92c9dSSasha Neftin struct pci_dev *pdev = adapter->pdev;
480463c92c9dSSasha Neftin struct igc_hw *hw = &adapter->hw;
480563c92c9dSSasha Neftin
480663c92c9dSSasha Neftin pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
480763c92c9dSSasha Neftin
480863c92c9dSSasha Neftin /* set default ring sizes */
480963c92c9dSSasha Neftin adapter->tx_ring_count = IGC_DEFAULT_TXD;
481063c92c9dSSasha Neftin adapter->rx_ring_count = IGC_DEFAULT_RXD;
481163c92c9dSSasha Neftin
481263c92c9dSSasha Neftin /* set default ITR values */
481363c92c9dSSasha Neftin adapter->rx_itr_setting = IGC_DEFAULT_ITR;
481463c92c9dSSasha Neftin adapter->tx_itr_setting = IGC_DEFAULT_ITR;
481563c92c9dSSasha Neftin
481663c92c9dSSasha Neftin /* set default work limits */
481763c92c9dSSasha Neftin adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
481863c92c9dSSasha Neftin
481963c92c9dSSasha Neftin /* adjust max frame to be at least the size of a standard frame */
482063c92c9dSSasha Neftin adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
482163c92c9dSSasha Neftin VLAN_HLEN;
482263c92c9dSSasha Neftin adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
482363c92c9dSSasha Neftin
482442fc5dc0SAndre Guedes mutex_init(&adapter->nfc_rule_lock);
4825d957c601SAndre Guedes INIT_LIST_HEAD(&adapter->nfc_rule_list);
4826d957c601SAndre Guedes adapter->nfc_rule_count = 0;
4827d957c601SAndre Guedes
482863c92c9dSSasha Neftin spin_lock_init(&adapter->stats64_lock);
482906b41258SMuhammad Husaini Zulkifli spin_lock_init(&adapter->qbv_tx_lock);
483063c92c9dSSasha Neftin /* Assume MSI-X interrupts, will be checked during IRQ allocation */
483163c92c9dSSasha Neftin adapter->flags |= IGC_FLAG_HAS_MSIX;
483263c92c9dSSasha Neftin
483363c92c9dSSasha Neftin igc_init_queue_configuration(adapter);
483463c92c9dSSasha Neftin
483563c92c9dSSasha Neftin /* This call may decrease the number of queues */
483663c92c9dSSasha Neftin if (igc_init_interrupt_scheme(adapter, true)) {
483725f06effSAndre Guedes netdev_err(netdev, "Unable to allocate memory for queues\n");
483863c92c9dSSasha Neftin return -ENOMEM;
483963c92c9dSSasha Neftin }
484063c92c9dSSasha Neftin
484163c92c9dSSasha Neftin /* Explicitly disable IRQ since the NIC can be in any state. */
484263c92c9dSSasha Neftin igc_irq_disable(adapter);
484363c92c9dSSasha Neftin
484463c92c9dSSasha Neftin set_bit(__IGC_DOWN, &adapter->state);
484563c92c9dSSasha Neftin
484663c92c9dSSasha Neftin return 0;
484763c92c9dSSasha Neftin }
484863c92c9dSSasha Neftin
484935f9a78aSSasha Neftin /**
4850c9a11c23SSasha Neftin * igc_up - Open the interface and prepare it to handle traffic
4851c9a11c23SSasha Neftin * @adapter: board private structure
4852c9a11c23SSasha Neftin */
igc_up(struct igc_adapter * adapter)48538c5ad0daSSasha Neftin void igc_up(struct igc_adapter *adapter)
4854c9a11c23SSasha Neftin {
48553df25e4cSSasha Neftin struct igc_hw *hw = &adapter->hw;
4856c9a11c23SSasha Neftin int i = 0;
4857c9a11c23SSasha Neftin
4858c9a11c23SSasha Neftin /* hardware has been reset, we need to reload some things */
4859c9a11c23SSasha Neftin igc_configure(adapter);
4860c9a11c23SSasha Neftin
4861c9a11c23SSasha Neftin clear_bit(__IGC_DOWN, &adapter->state);
4862c9a11c23SSasha Neftin
4863c9a11c23SSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++)
4864c9a11c23SSasha Neftin napi_enable(&adapter->q_vector[i]->napi);
48653df25e4cSSasha Neftin
48663df25e4cSSasha Neftin if (adapter->msix_entries)
48673df25e4cSSasha Neftin igc_configure_msix(adapter);
48683df25e4cSSasha Neftin else
48693df25e4cSSasha Neftin igc_assign_vector(adapter->q_vector[0], 0);
48703df25e4cSSasha Neftin
48713df25e4cSSasha Neftin /* Clear any pending interrupts. */
48723df25e4cSSasha Neftin rd32(IGC_ICR);
48733df25e4cSSasha Neftin igc_irq_enable(adapter);
487413b5b7fdSSasha Neftin
487513b5b7fdSSasha Neftin netif_tx_start_all_queues(adapter->netdev);
487613b5b7fdSSasha Neftin
487713b5b7fdSSasha Neftin /* start the watchdog. */
4878501f2309SJiapeng Zhong hw->mac.get_link_status = true;
4879208983f0SSasha Neftin schedule_work(&adapter->watchdog_task);
4880c9a11c23SSasha Neftin }
4881c9a11c23SSasha Neftin
4882c9a11c23SSasha Neftin /**
4883c9a11c23SSasha Neftin * igc_update_stats - Update the board statistics counters
4884c9a11c23SSasha Neftin * @adapter: board private structure
4885c9a11c23SSasha Neftin */
igc_update_stats(struct igc_adapter * adapter)488636b9fea6SSasha Neftin void igc_update_stats(struct igc_adapter *adapter)
4887c9a11c23SSasha Neftin {
488836b9fea6SSasha Neftin struct rtnl_link_stats64 *net_stats = &adapter->stats64;
488936b9fea6SSasha Neftin struct pci_dev *pdev = adapter->pdev;
489036b9fea6SSasha Neftin struct igc_hw *hw = &adapter->hw;
489136b9fea6SSasha Neftin u64 _bytes, _packets;
489236b9fea6SSasha Neftin u64 bytes, packets;
489336b9fea6SSasha Neftin unsigned int start;
489436b9fea6SSasha Neftin u32 mpc;
489536b9fea6SSasha Neftin int i;
489636b9fea6SSasha Neftin
489736b9fea6SSasha Neftin /* Prevent stats update while adapter is being reset, or if the pci
489836b9fea6SSasha Neftin * connection is down.
489936b9fea6SSasha Neftin */
490036b9fea6SSasha Neftin if (adapter->link_speed == 0)
490136b9fea6SSasha Neftin return;
490236b9fea6SSasha Neftin if (pci_channel_offline(pdev))
490336b9fea6SSasha Neftin return;
490436b9fea6SSasha Neftin
490536b9fea6SSasha Neftin packets = 0;
490636b9fea6SSasha Neftin bytes = 0;
490736b9fea6SSasha Neftin
490836b9fea6SSasha Neftin rcu_read_lock();
490936b9fea6SSasha Neftin for (i = 0; i < adapter->num_rx_queues; i++) {
491036b9fea6SSasha Neftin struct igc_ring *ring = adapter->rx_ring[i];
491136b9fea6SSasha Neftin u32 rqdpc = rd32(IGC_RQDPC(i));
491236b9fea6SSasha Neftin
491336b9fea6SSasha Neftin if (hw->mac.type >= igc_i225)
491436b9fea6SSasha Neftin wr32(IGC_RQDPC(i), 0);
491536b9fea6SSasha Neftin
491636b9fea6SSasha Neftin if (rqdpc) {
491736b9fea6SSasha Neftin ring->rx_stats.drops += rqdpc;
491836b9fea6SSasha Neftin net_stats->rx_fifo_errors += rqdpc;
491936b9fea6SSasha Neftin }
492036b9fea6SSasha Neftin
492136b9fea6SSasha Neftin do {
4922068c38adSThomas Gleixner start = u64_stats_fetch_begin(&ring->rx_syncp);
492336b9fea6SSasha Neftin _bytes = ring->rx_stats.bytes;
492436b9fea6SSasha Neftin _packets = ring->rx_stats.packets;
4925068c38adSThomas Gleixner } while (u64_stats_fetch_retry(&ring->rx_syncp, start));
492636b9fea6SSasha Neftin bytes += _bytes;
492736b9fea6SSasha Neftin packets += _packets;
492836b9fea6SSasha Neftin }
492936b9fea6SSasha Neftin
493036b9fea6SSasha Neftin net_stats->rx_bytes = bytes;
493136b9fea6SSasha Neftin net_stats->rx_packets = packets;
493236b9fea6SSasha Neftin
493336b9fea6SSasha Neftin packets = 0;
493436b9fea6SSasha Neftin bytes = 0;
493536b9fea6SSasha Neftin for (i = 0; i < adapter->num_tx_queues; i++) {
493636b9fea6SSasha Neftin struct igc_ring *ring = adapter->tx_ring[i];
493736b9fea6SSasha Neftin
493836b9fea6SSasha Neftin do {
4939068c38adSThomas Gleixner start = u64_stats_fetch_begin(&ring->tx_syncp);
494036b9fea6SSasha Neftin _bytes = ring->tx_stats.bytes;
494136b9fea6SSasha Neftin _packets = ring->tx_stats.packets;
4942068c38adSThomas Gleixner } while (u64_stats_fetch_retry(&ring->tx_syncp, start));
494336b9fea6SSasha Neftin bytes += _bytes;
494436b9fea6SSasha Neftin packets += _packets;
494536b9fea6SSasha Neftin }
494636b9fea6SSasha Neftin net_stats->tx_bytes = bytes;
494736b9fea6SSasha Neftin net_stats->tx_packets = packets;
494836b9fea6SSasha Neftin rcu_read_unlock();
494936b9fea6SSasha Neftin
495036b9fea6SSasha Neftin /* read stats registers */
495136b9fea6SSasha Neftin adapter->stats.crcerrs += rd32(IGC_CRCERRS);
495236b9fea6SSasha Neftin adapter->stats.gprc += rd32(IGC_GPRC);
495336b9fea6SSasha Neftin adapter->stats.gorc += rd32(IGC_GORCL);
495436b9fea6SSasha Neftin rd32(IGC_GORCH); /* clear GORCL */
495536b9fea6SSasha Neftin adapter->stats.bprc += rd32(IGC_BPRC);
495636b9fea6SSasha Neftin adapter->stats.mprc += rd32(IGC_MPRC);
495736b9fea6SSasha Neftin adapter->stats.roc += rd32(IGC_ROC);
495836b9fea6SSasha Neftin
495936b9fea6SSasha Neftin adapter->stats.prc64 += rd32(IGC_PRC64);
496036b9fea6SSasha Neftin adapter->stats.prc127 += rd32(IGC_PRC127);
496136b9fea6SSasha Neftin adapter->stats.prc255 += rd32(IGC_PRC255);
496236b9fea6SSasha Neftin adapter->stats.prc511 += rd32(IGC_PRC511);
496336b9fea6SSasha Neftin adapter->stats.prc1023 += rd32(IGC_PRC1023);
496436b9fea6SSasha Neftin adapter->stats.prc1522 += rd32(IGC_PRC1522);
496540edc734SSasha Neftin adapter->stats.tlpic += rd32(IGC_TLPIC);
496640edc734SSasha Neftin adapter->stats.rlpic += rd32(IGC_RLPIC);
4967e6529944SSasha Neftin adapter->stats.hgptc += rd32(IGC_HGPTC);
496836b9fea6SSasha Neftin
496936b9fea6SSasha Neftin mpc = rd32(IGC_MPC);
497036b9fea6SSasha Neftin adapter->stats.mpc += mpc;
497136b9fea6SSasha Neftin net_stats->rx_fifo_errors += mpc;
497236b9fea6SSasha Neftin adapter->stats.scc += rd32(IGC_SCC);
497336b9fea6SSasha Neftin adapter->stats.ecol += rd32(IGC_ECOL);
497436b9fea6SSasha Neftin adapter->stats.mcc += rd32(IGC_MCC);
497536b9fea6SSasha Neftin adapter->stats.latecol += rd32(IGC_LATECOL);
497636b9fea6SSasha Neftin adapter->stats.dc += rd32(IGC_DC);
497736b9fea6SSasha Neftin adapter->stats.rlec += rd32(IGC_RLEC);
497836b9fea6SSasha Neftin adapter->stats.xonrxc += rd32(IGC_XONRXC);
497936b9fea6SSasha Neftin adapter->stats.xontxc += rd32(IGC_XONTXC);
498036b9fea6SSasha Neftin adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
498136b9fea6SSasha Neftin adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
498236b9fea6SSasha Neftin adapter->stats.fcruc += rd32(IGC_FCRUC);
498336b9fea6SSasha Neftin adapter->stats.gptc += rd32(IGC_GPTC);
498436b9fea6SSasha Neftin adapter->stats.gotc += rd32(IGC_GOTCL);
498536b9fea6SSasha Neftin rd32(IGC_GOTCH); /* clear GOTCL */
498636b9fea6SSasha Neftin adapter->stats.rnbc += rd32(IGC_RNBC);
498736b9fea6SSasha Neftin adapter->stats.ruc += rd32(IGC_RUC);
498836b9fea6SSasha Neftin adapter->stats.rfc += rd32(IGC_RFC);
498936b9fea6SSasha Neftin adapter->stats.rjc += rd32(IGC_RJC);
499036b9fea6SSasha Neftin adapter->stats.tor += rd32(IGC_TORH);
499136b9fea6SSasha Neftin adapter->stats.tot += rd32(IGC_TOTH);
499236b9fea6SSasha Neftin adapter->stats.tpr += rd32(IGC_TPR);
499336b9fea6SSasha Neftin
499436b9fea6SSasha Neftin adapter->stats.ptc64 += rd32(IGC_PTC64);
499536b9fea6SSasha Neftin adapter->stats.ptc127 += rd32(IGC_PTC127);
499636b9fea6SSasha Neftin adapter->stats.ptc255 += rd32(IGC_PTC255);
499736b9fea6SSasha Neftin adapter->stats.ptc511 += rd32(IGC_PTC511);
499836b9fea6SSasha Neftin adapter->stats.ptc1023 += rd32(IGC_PTC1023);
499936b9fea6SSasha Neftin adapter->stats.ptc1522 += rd32(IGC_PTC1522);
500036b9fea6SSasha Neftin
500136b9fea6SSasha Neftin adapter->stats.mptc += rd32(IGC_MPTC);
500236b9fea6SSasha Neftin adapter->stats.bptc += rd32(IGC_BPTC);
500336b9fea6SSasha Neftin
500436b9fea6SSasha Neftin adapter->stats.tpt += rd32(IGC_TPT);
500536b9fea6SSasha Neftin adapter->stats.colc += rd32(IGC_COLC);
500651c657b4SSasha Neftin adapter->stats.colc += rd32(IGC_RERC);
500736b9fea6SSasha Neftin
500836b9fea6SSasha Neftin adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
500936b9fea6SSasha Neftin
501036b9fea6SSasha Neftin adapter->stats.tsctc += rd32(IGC_TSCTC);
501136b9fea6SSasha Neftin
501236b9fea6SSasha Neftin adapter->stats.iac += rd32(IGC_IAC);
501336b9fea6SSasha Neftin
501436b9fea6SSasha Neftin /* Fill out the OS statistics structure */
501536b9fea6SSasha Neftin net_stats->multicast = adapter->stats.mprc;
501636b9fea6SSasha Neftin net_stats->collisions = adapter->stats.colc;
501736b9fea6SSasha Neftin
501836b9fea6SSasha Neftin /* Rx Errors */
501936b9fea6SSasha Neftin
502036b9fea6SSasha Neftin /* RLEC on some newer hardware can be incorrect so build
502136b9fea6SSasha Neftin * our own version based on RUC and ROC
502236b9fea6SSasha Neftin */
502336b9fea6SSasha Neftin net_stats->rx_errors = adapter->stats.rxerrc +
502436b9fea6SSasha Neftin adapter->stats.crcerrs + adapter->stats.algnerrc +
502536b9fea6SSasha Neftin adapter->stats.ruc + adapter->stats.roc +
502636b9fea6SSasha Neftin adapter->stats.cexterr;
502736b9fea6SSasha Neftin net_stats->rx_length_errors = adapter->stats.ruc +
502836b9fea6SSasha Neftin adapter->stats.roc;
502936b9fea6SSasha Neftin net_stats->rx_crc_errors = adapter->stats.crcerrs;
503036b9fea6SSasha Neftin net_stats->rx_frame_errors = adapter->stats.algnerrc;
503136b9fea6SSasha Neftin net_stats->rx_missed_errors = adapter->stats.mpc;
503236b9fea6SSasha Neftin
503336b9fea6SSasha Neftin /* Tx Errors */
503436b9fea6SSasha Neftin net_stats->tx_errors = adapter->stats.ecol +
503536b9fea6SSasha Neftin adapter->stats.latecol;
503636b9fea6SSasha Neftin net_stats->tx_aborted_errors = adapter->stats.ecol;
503736b9fea6SSasha Neftin net_stats->tx_window_errors = adapter->stats.latecol;
503836b9fea6SSasha Neftin net_stats->tx_carrier_errors = adapter->stats.tncrs;
503936b9fea6SSasha Neftin
504092a0dcb8STan Tee Min /* Tx Dropped */
504192a0dcb8STan Tee Min net_stats->tx_dropped = adapter->stats.txdrop;
504236b9fea6SSasha Neftin
504336b9fea6SSasha Neftin /* Management Stats */
504436b9fea6SSasha Neftin adapter->stats.mgptc += rd32(IGC_MGTPTC);
504536b9fea6SSasha Neftin adapter->stats.mgprc += rd32(IGC_MGTPRC);
504636b9fea6SSasha Neftin adapter->stats.mgpdc += rd32(IGC_MGTPDC);
5047c9a11c23SSasha Neftin }
5048c9a11c23SSasha Neftin
5049c9a11c23SSasha Neftin /**
5050c9a11c23SSasha Neftin * igc_down - Close the interface
5051c9a11c23SSasha Neftin * @adapter: board private structure
5052c9a11c23SSasha Neftin */
igc_down(struct igc_adapter * adapter)50538c5ad0daSSasha Neftin void igc_down(struct igc_adapter *adapter)
5054c9a11c23SSasha Neftin {
5055c9a11c23SSasha Neftin struct net_device *netdev = adapter->netdev;
50560507ef8aSSasha Neftin struct igc_hw *hw = &adapter->hw;
50570507ef8aSSasha Neftin u32 tctl, rctl;
5058c9a11c23SSasha Neftin int i = 0;
5059c9a11c23SSasha Neftin
5060c9a11c23SSasha Neftin set_bit(__IGC_DOWN, &adapter->state);
5061c9a11c23SSasha Neftin
5062b03c49cdSVinicius Costa Gomes igc_ptp_suspend(adapter);
5063b03c49cdSVinicius Costa Gomes
50644b799595SAaron Ma if (pci_device_is_present(adapter->pdev)) {
50650507ef8aSSasha Neftin /* disable receives in the hardware */
50660507ef8aSSasha Neftin rctl = rd32(IGC_RCTL);
50670507ef8aSSasha Neftin wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
50680507ef8aSSasha Neftin /* flush and sleep below */
50694b799595SAaron Ma }
5070c9a11c23SSasha Neftin /* set trans_start so we don't get spurious watchdogs during reset */
5071c9a11c23SSasha Neftin netif_trans_update(netdev);
5072c9a11c23SSasha Neftin
5073c9a11c23SSasha Neftin netif_carrier_off(netdev);
5074c9a11c23SSasha Neftin netif_tx_stop_all_queues(netdev);
5075c9a11c23SSasha Neftin
50764b799595SAaron Ma if (pci_device_is_present(adapter->pdev)) {
50770507ef8aSSasha Neftin /* disable transmits in the hardware */
50780507ef8aSSasha Neftin tctl = rd32(IGC_TCTL);
50790507ef8aSSasha Neftin tctl &= ~IGC_TCTL_EN;
50800507ef8aSSasha Neftin wr32(IGC_TCTL, tctl);
50810507ef8aSSasha Neftin /* flush both disables and wait for them to finish */
50820507ef8aSSasha Neftin wrfl();
50830507ef8aSSasha Neftin usleep_range(10000, 20000);
50840507ef8aSSasha Neftin
50850507ef8aSSasha Neftin igc_irq_disable(adapter);
50864b799595SAaron Ma }
50870507ef8aSSasha Neftin
50880507ef8aSSasha Neftin adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
50890507ef8aSSasha Neftin
50900507ef8aSSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++) {
50910507ef8aSSasha Neftin if (adapter->q_vector[i]) {
50920507ef8aSSasha Neftin napi_synchronize(&adapter->q_vector[i]->napi);
5093c9a11c23SSasha Neftin napi_disable(&adapter->q_vector[i]->napi);
50940507ef8aSSasha Neftin }
50950507ef8aSSasha Neftin }
50960507ef8aSSasha Neftin
50970507ef8aSSasha Neftin del_timer_sync(&adapter->watchdog_timer);
50980507ef8aSSasha Neftin del_timer_sync(&adapter->phy_info_timer);
50990507ef8aSSasha Neftin
51000507ef8aSSasha Neftin /* record the stats before reset*/
51010507ef8aSSasha Neftin spin_lock(&adapter->stats64_lock);
51020507ef8aSSasha Neftin igc_update_stats(adapter);
51030507ef8aSSasha Neftin spin_unlock(&adapter->stats64_lock);
5104c9a11c23SSasha Neftin
5105c9a11c23SSasha Neftin adapter->link_speed = 0;
5106c9a11c23SSasha Neftin adapter->link_duplex = 0;
51070507ef8aSSasha Neftin
51080507ef8aSSasha Neftin if (!pci_channel_offline(adapter->pdev))
51090507ef8aSSasha Neftin igc_reset(adapter);
51100507ef8aSSasha Neftin
51110507ef8aSSasha Neftin /* clear VLAN promisc flag so VFTA will be updated if necessary */
51120507ef8aSSasha Neftin adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
51130507ef8aSSasha Neftin
5114d4a7ce64SMuhammad Husaini Zulkifli igc_disable_all_tx_rings_hw(adapter);
51150507ef8aSSasha Neftin igc_clean_all_tx_rings(adapter);
51160507ef8aSSasha Neftin igc_clean_all_rx_rings(adapter);
51170507ef8aSSasha Neftin }
51180507ef8aSSasha Neftin
igc_reinit_locked(struct igc_adapter * adapter)51198c5ad0daSSasha Neftin void igc_reinit_locked(struct igc_adapter *adapter)
51200507ef8aSSasha Neftin {
51210507ef8aSSasha Neftin while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
51220507ef8aSSasha Neftin usleep_range(1000, 2000);
51230507ef8aSSasha Neftin igc_down(adapter);
51240507ef8aSSasha Neftin igc_up(adapter);
51250507ef8aSSasha Neftin clear_bit(__IGC_RESETTING, &adapter->state);
51260507ef8aSSasha Neftin }
51270507ef8aSSasha Neftin
igc_reset_task(struct work_struct * work)51280507ef8aSSasha Neftin static void igc_reset_task(struct work_struct *work)
51290507ef8aSSasha Neftin {
51300507ef8aSSasha Neftin struct igc_adapter *adapter;
51310507ef8aSSasha Neftin
51320507ef8aSSasha Neftin adapter = container_of(work, struct igc_adapter, reset_task);
51330507ef8aSSasha Neftin
51346da26237SSasha Neftin rtnl_lock();
51356da26237SSasha Neftin /* If we're already down or resetting, just bail */
51366da26237SSasha Neftin if (test_bit(__IGC_DOWN, &adapter->state) ||
51376da26237SSasha Neftin test_bit(__IGC_RESETTING, &adapter->state)) {
51386da26237SSasha Neftin rtnl_unlock();
51396da26237SSasha Neftin return;
51406da26237SSasha Neftin }
51416da26237SSasha Neftin
51429c384ee3SSasha Neftin igc_rings_dump(adapter);
51439c384ee3SSasha Neftin igc_regs_dump(adapter);
51440507ef8aSSasha Neftin netdev_err(adapter->netdev, "Reset adapter\n");
51450507ef8aSSasha Neftin igc_reinit_locked(adapter);
51466da26237SSasha Neftin rtnl_unlock();
5147c9a11c23SSasha Neftin }
5148c9a11c23SSasha Neftin
5149c9a11c23SSasha Neftin /**
5150c9a11c23SSasha Neftin * igc_change_mtu - Change the Maximum Transfer Unit
5151c9a11c23SSasha Neftin * @netdev: network interface device structure
5152c9a11c23SSasha Neftin * @new_mtu: new value for maximum frame size
5153c9a11c23SSasha Neftin *
5154c9a11c23SSasha Neftin * Returns 0 on success, negative on failure
5155c9a11c23SSasha Neftin */
igc_change_mtu(struct net_device * netdev,int new_mtu)5156c9a11c23SSasha Neftin static int igc_change_mtu(struct net_device *netdev, int new_mtu)
5157c9a11c23SSasha Neftin {
5158c9a11c23SSasha Neftin int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5159c9a11c23SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
5160c9a11c23SSasha Neftin
516126575105SAndre Guedes if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) {
516226575105SAndre Guedes netdev_dbg(netdev, "Jumbo frames not supported with XDP");
516326575105SAndre Guedes return -EINVAL;
516426575105SAndre Guedes }
516526575105SAndre Guedes
5166c9a11c23SSasha Neftin /* adjust max frame to be at least the size of a standard frame */
5167c9a11c23SSasha Neftin if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5168c9a11c23SSasha Neftin max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5169c9a11c23SSasha Neftin
5170c9a11c23SSasha Neftin while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5171c9a11c23SSasha Neftin usleep_range(1000, 2000);
5172c9a11c23SSasha Neftin
5173c9a11c23SSasha Neftin /* igc_down has a dependency on max_frame_size */
5174c9a11c23SSasha Neftin adapter->max_frame_size = max_frame;
5175c9a11c23SSasha Neftin
5176c9a11c23SSasha Neftin if (netif_running(netdev))
5177c9a11c23SSasha Neftin igc_down(adapter);
5178c9a11c23SSasha Neftin
517925f06effSAndre Guedes netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5180c9a11c23SSasha Neftin netdev->mtu = new_mtu;
5181c9a11c23SSasha Neftin
5182c9a11c23SSasha Neftin if (netif_running(netdev))
5183c9a11c23SSasha Neftin igc_up(adapter);
5184c9a11c23SSasha Neftin else
5185c9a11c23SSasha Neftin igc_reset(adapter);
5186c9a11c23SSasha Neftin
5187c9a11c23SSasha Neftin clear_bit(__IGC_RESETTING, &adapter->state);
5188c9a11c23SSasha Neftin
5189c9a11c23SSasha Neftin return 0;
5190c9a11c23SSasha Neftin }
5191c9a11c23SSasha Neftin
5192c9a11c23SSasha Neftin /**
51939b275176SSasha Neftin * igc_tx_timeout - Respond to a Tx Hang
51949b275176SSasha Neftin * @netdev: network interface device structure
51959b275176SSasha Neftin * @txqueue: queue number that timed out
51969b275176SSasha Neftin **/
igc_tx_timeout(struct net_device * netdev,unsigned int __always_unused txqueue)51979b275176SSasha Neftin static void igc_tx_timeout(struct net_device *netdev,
51989b275176SSasha Neftin unsigned int __always_unused txqueue)
51999b275176SSasha Neftin {
52009b275176SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
52019b275176SSasha Neftin struct igc_hw *hw = &adapter->hw;
52029b275176SSasha Neftin
52039b275176SSasha Neftin /* Do the reset outside of interrupt context */
52049b275176SSasha Neftin adapter->tx_timeout_count++;
52059b275176SSasha Neftin schedule_work(&adapter->reset_task);
52069b275176SSasha Neftin wr32(IGC_EICS,
52079b275176SSasha Neftin (adapter->eims_enable_mask & ~adapter->eims_other));
52089b275176SSasha Neftin }
52099b275176SSasha Neftin
52109b275176SSasha Neftin /**
52116b7ed22aSVinicius Costa Gomes * igc_get_stats64 - Get System Network Statistics
5212c9a11c23SSasha Neftin * @netdev: network interface device structure
52136b7ed22aSVinicius Costa Gomes * @stats: rtnl_link_stats64 pointer
5214c9a11c23SSasha Neftin *
5215c9a11c23SSasha Neftin * Returns the address of the device statistics structure.
5216c9a11c23SSasha Neftin * The statistics are updated here and also from the timer callback.
5217c9a11c23SSasha Neftin */
igc_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)52186b7ed22aSVinicius Costa Gomes static void igc_get_stats64(struct net_device *netdev,
52196b7ed22aSVinicius Costa Gomes struct rtnl_link_stats64 *stats)
5220c9a11c23SSasha Neftin {
5221c9a11c23SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
5222c9a11c23SSasha Neftin
52236b7ed22aSVinicius Costa Gomes spin_lock(&adapter->stats64_lock);
5224c9a11c23SSasha Neftin if (!test_bit(__IGC_RESETTING, &adapter->state))
5225c9a11c23SSasha Neftin igc_update_stats(adapter);
52266b7ed22aSVinicius Costa Gomes memcpy(stats, &adapter->stats64, sizeof(*stats));
52276b7ed22aSVinicius Costa Gomes spin_unlock(&adapter->stats64_lock);
5228c9a11c23SSasha Neftin }
5229c9a11c23SSasha Neftin
igc_fix_features(struct net_device * netdev,netdev_features_t features)523065cd3a72SSasha Neftin static netdev_features_t igc_fix_features(struct net_device *netdev,
523165cd3a72SSasha Neftin netdev_features_t features)
523265cd3a72SSasha Neftin {
523365cd3a72SSasha Neftin /* Since there is no support for separate Rx/Tx vlan accel
523465cd3a72SSasha Neftin * enable/disable make sure Tx flag is always in same state as Rx.
523565cd3a72SSasha Neftin */
523665cd3a72SSasha Neftin if (features & NETIF_F_HW_VLAN_CTAG_RX)
523765cd3a72SSasha Neftin features |= NETIF_F_HW_VLAN_CTAG_TX;
523865cd3a72SSasha Neftin else
523965cd3a72SSasha Neftin features &= ~NETIF_F_HW_VLAN_CTAG_TX;
524065cd3a72SSasha Neftin
524165cd3a72SSasha Neftin return features;
524265cd3a72SSasha Neftin }
524365cd3a72SSasha Neftin
igc_set_features(struct net_device * netdev,netdev_features_t features)524465cd3a72SSasha Neftin static int igc_set_features(struct net_device *netdev,
524565cd3a72SSasha Neftin netdev_features_t features)
524665cd3a72SSasha Neftin {
524765cd3a72SSasha Neftin netdev_features_t changed = netdev->features ^ features;
524865cd3a72SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
524965cd3a72SSasha Neftin
52508d744963SMuhammad Husaini Zulkifli if (changed & NETIF_F_HW_VLAN_CTAG_RX)
52518d744963SMuhammad Husaini Zulkifli igc_vlan_mode(netdev, features);
52528d744963SMuhammad Husaini Zulkifli
525365cd3a72SSasha Neftin /* Add VLAN support */
525465cd3a72SSasha Neftin if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
525565cd3a72SSasha Neftin return 0;
525665cd3a72SSasha Neftin
5257e256ec83SAndre Guedes if (!(features & NETIF_F_NTUPLE))
5258e256ec83SAndre Guedes igc_flush_nfc_rules(adapter);
525965cd3a72SSasha Neftin
526065cd3a72SSasha Neftin netdev->features = features;
526165cd3a72SSasha Neftin
526265cd3a72SSasha Neftin if (netif_running(netdev))
526365cd3a72SSasha Neftin igc_reinit_locked(adapter);
526465cd3a72SSasha Neftin else
526565cd3a72SSasha Neftin igc_reset(adapter);
526665cd3a72SSasha Neftin
526765cd3a72SSasha Neftin return 1;
526865cd3a72SSasha Neftin }
526965cd3a72SSasha Neftin
527065cd3a72SSasha Neftin static netdev_features_t
igc_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)527165cd3a72SSasha Neftin igc_features_check(struct sk_buff *skb, struct net_device *dev,
527265cd3a72SSasha Neftin netdev_features_t features)
527365cd3a72SSasha Neftin {
527465cd3a72SSasha Neftin unsigned int network_hdr_len, mac_hdr_len;
527565cd3a72SSasha Neftin
527665cd3a72SSasha Neftin /* Make certain the headers can be described by a context descriptor */
527765cd3a72SSasha Neftin mac_hdr_len = skb_network_header(skb) - skb->data;
527865cd3a72SSasha Neftin if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
527965cd3a72SSasha Neftin return features & ~(NETIF_F_HW_CSUM |
528065cd3a72SSasha Neftin NETIF_F_SCTP_CRC |
528165cd3a72SSasha Neftin NETIF_F_HW_VLAN_CTAG_TX |
528265cd3a72SSasha Neftin NETIF_F_TSO |
528365cd3a72SSasha Neftin NETIF_F_TSO6);
528465cd3a72SSasha Neftin
528565cd3a72SSasha Neftin network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
528665cd3a72SSasha Neftin if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN))
528765cd3a72SSasha Neftin return features & ~(NETIF_F_HW_CSUM |
528865cd3a72SSasha Neftin NETIF_F_SCTP_CRC |
528965cd3a72SSasha Neftin NETIF_F_TSO |
529065cd3a72SSasha Neftin NETIF_F_TSO6);
529165cd3a72SSasha Neftin
529265cd3a72SSasha Neftin /* We can only support IPv4 TSO in tunnels if we can mangle the
529365cd3a72SSasha Neftin * inner IP ID field, so strip TSO if MANGLEID is not supported.
529465cd3a72SSasha Neftin */
529565cd3a72SSasha Neftin if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
529665cd3a72SSasha Neftin features &= ~NETIF_F_TSO;
529765cd3a72SSasha Neftin
529865cd3a72SSasha Neftin return features;
529965cd3a72SSasha Neftin }
530065cd3a72SSasha Neftin
igc_tsync_interrupt(struct igc_adapter * adapter)53012c344ae2SVinicius Costa Gomes static void igc_tsync_interrupt(struct igc_adapter *adapter)
53022c344ae2SVinicius Costa Gomes {
53032c344ae2SVinicius Costa Gomes struct igc_hw *hw = &adapter->hw;
5304*9bfb3503SVinicius Costa Gomes u32 tsauxc, sec, nsec, tsicr;
530564433e5bSEderson de Souza struct ptp_clock_event event;
530687938851SEderson de Souza struct timespec64 ts;
530787938851SEderson de Souza
530887938851SEderson de Souza tsicr = rd32(IGC_TSICR);
53092c344ae2SVinicius Costa Gomes
531064433e5bSEderson de Souza if (tsicr & IGC_TSICR_SYS_WRAP) {
531164433e5bSEderson de Souza event.type = PTP_CLOCK_PPS;
531264433e5bSEderson de Souza if (adapter->ptp_caps.pps)
531364433e5bSEderson de Souza ptp_clock_event(adapter->ptp_clock, &event);
531464433e5bSEderson de Souza }
531564433e5bSEderson de Souza
53162c344ae2SVinicius Costa Gomes if (tsicr & IGC_TSICR_TXTS) {
53172c344ae2SVinicius Costa Gomes /* retrieve hardware timestamp */
5318afa14158SVinicius Costa Gomes igc_ptp_tx_tstamp_event(adapter);
53192c344ae2SVinicius Costa Gomes }
53202c344ae2SVinicius Costa Gomes
532187938851SEderson de Souza if (tsicr & IGC_TSICR_TT0) {
532287938851SEderson de Souza spin_lock(&adapter->tmreg_lock);
532387938851SEderson de Souza ts = timespec64_add(adapter->perout[0].start,
532487938851SEderson de Souza adapter->perout[0].period);
532587938851SEderson de Souza wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
532687938851SEderson de Souza wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec);
532787938851SEderson de Souza tsauxc = rd32(IGC_TSAUXC);
532887938851SEderson de Souza tsauxc |= IGC_TSAUXC_EN_TT0;
532987938851SEderson de Souza wr32(IGC_TSAUXC, tsauxc);
533087938851SEderson de Souza adapter->perout[0].start = ts;
533187938851SEderson de Souza spin_unlock(&adapter->tmreg_lock);
533287938851SEderson de Souza }
533387938851SEderson de Souza
533487938851SEderson de Souza if (tsicr & IGC_TSICR_TT1) {
533587938851SEderson de Souza spin_lock(&adapter->tmreg_lock);
533687938851SEderson de Souza ts = timespec64_add(adapter->perout[1].start,
533787938851SEderson de Souza adapter->perout[1].period);
533887938851SEderson de Souza wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
533987938851SEderson de Souza wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec);
534087938851SEderson de Souza tsauxc = rd32(IGC_TSAUXC);
534187938851SEderson de Souza tsauxc |= IGC_TSAUXC_EN_TT1;
534287938851SEderson de Souza wr32(IGC_TSAUXC, tsauxc);
534387938851SEderson de Souza adapter->perout[1].start = ts;
534487938851SEderson de Souza spin_unlock(&adapter->tmreg_lock);
534587938851SEderson de Souza }
534687938851SEderson de Souza
534787938851SEderson de Souza if (tsicr & IGC_TSICR_AUTT0) {
534887938851SEderson de Souza nsec = rd32(IGC_AUXSTMPL0);
534987938851SEderson de Souza sec = rd32(IGC_AUXSTMPH0);
535087938851SEderson de Souza event.type = PTP_CLOCK_EXTTS;
535187938851SEderson de Souza event.index = 0;
535287938851SEderson de Souza event.timestamp = sec * NSEC_PER_SEC + nsec;
535387938851SEderson de Souza ptp_clock_event(adapter->ptp_clock, &event);
535487938851SEderson de Souza }
535587938851SEderson de Souza
535687938851SEderson de Souza if (tsicr & IGC_TSICR_AUTT1) {
535787938851SEderson de Souza nsec = rd32(IGC_AUXSTMPL1);
535887938851SEderson de Souza sec = rd32(IGC_AUXSTMPH1);
535987938851SEderson de Souza event.type = PTP_CLOCK_EXTTS;
536087938851SEderson de Souza event.index = 1;
536187938851SEderson de Souza event.timestamp = sec * NSEC_PER_SEC + nsec;
536287938851SEderson de Souza ptp_clock_event(adapter->ptp_clock, &event);
536387938851SEderson de Souza }
53642c344ae2SVinicius Costa Gomes }
53652c344ae2SVinicius Costa Gomes
536613b5b7fdSSasha Neftin /**
53673df25e4cSSasha Neftin * igc_msix_other - msix other interrupt handler
53683df25e4cSSasha Neftin * @irq: interrupt number
53693df25e4cSSasha Neftin * @data: pointer to a q_vector
53703df25e4cSSasha Neftin */
igc_msix_other(int irq,void * data)53713df25e4cSSasha Neftin static irqreturn_t igc_msix_other(int irq, void *data)
53723df25e4cSSasha Neftin {
53733df25e4cSSasha Neftin struct igc_adapter *adapter = data;
53743df25e4cSSasha Neftin struct igc_hw *hw = &adapter->hw;
53753df25e4cSSasha Neftin u32 icr = rd32(IGC_ICR);
53763df25e4cSSasha Neftin
53773df25e4cSSasha Neftin /* reading ICR causes bit 31 of EICR to be cleared */
53783df25e4cSSasha Neftin if (icr & IGC_ICR_DRSTA)
53793df25e4cSSasha Neftin schedule_work(&adapter->reset_task);
53803df25e4cSSasha Neftin
53813df25e4cSSasha Neftin if (icr & IGC_ICR_DOUTSYNC) {
53823df25e4cSSasha Neftin /* HW is reporting DMA is out of sync */
53833df25e4cSSasha Neftin adapter->stats.doosync++;
53843df25e4cSSasha Neftin }
53853df25e4cSSasha Neftin
53863df25e4cSSasha Neftin if (icr & IGC_ICR_LSC) {
5387501f2309SJiapeng Zhong hw->mac.get_link_status = true;
53883df25e4cSSasha Neftin /* guard against interrupt when we're going down */
53893df25e4cSSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state))
53903df25e4cSSasha Neftin mod_timer(&adapter->watchdog_timer, jiffies + 1);
53913df25e4cSSasha Neftin }
53923df25e4cSSasha Neftin
53932c344ae2SVinicius Costa Gomes if (icr & IGC_ICR_TS)
53942c344ae2SVinicius Costa Gomes igc_tsync_interrupt(adapter);
53952c344ae2SVinicius Costa Gomes
53963df25e4cSSasha Neftin wr32(IGC_EIMS, adapter->eims_other);
53973df25e4cSSasha Neftin
53983df25e4cSSasha Neftin return IRQ_HANDLED;
53993df25e4cSSasha Neftin }
54003df25e4cSSasha Neftin
igc_write_itr(struct igc_q_vector * q_vector)540155cd7386SSasha Neftin static void igc_write_itr(struct igc_q_vector *q_vector)
540255cd7386SSasha Neftin {
540355cd7386SSasha Neftin u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
540455cd7386SSasha Neftin
540555cd7386SSasha Neftin if (!q_vector->set_itr)
540655cd7386SSasha Neftin return;
540755cd7386SSasha Neftin
540855cd7386SSasha Neftin if (!itr_val)
540955cd7386SSasha Neftin itr_val = IGC_ITR_VAL_MASK;
541055cd7386SSasha Neftin
541155cd7386SSasha Neftin itr_val |= IGC_EITR_CNT_IGNR;
541255cd7386SSasha Neftin
541355cd7386SSasha Neftin writel(itr_val, q_vector->itr_register);
541455cd7386SSasha Neftin q_vector->set_itr = 0;
541555cd7386SSasha Neftin }
541655cd7386SSasha Neftin
igc_msix_ring(int irq,void * data)54173df25e4cSSasha Neftin static irqreturn_t igc_msix_ring(int irq, void *data)
54183df25e4cSSasha Neftin {
54193df25e4cSSasha Neftin struct igc_q_vector *q_vector = data;
54203df25e4cSSasha Neftin
54213df25e4cSSasha Neftin /* Write the ITR value calculated from the previous interrupt. */
54223df25e4cSSasha Neftin igc_write_itr(q_vector);
54233df25e4cSSasha Neftin
54243df25e4cSSasha Neftin napi_schedule(&q_vector->napi);
54253df25e4cSSasha Neftin
54263df25e4cSSasha Neftin return IRQ_HANDLED;
54273df25e4cSSasha Neftin }
54283df25e4cSSasha Neftin
54293df25e4cSSasha Neftin /**
54303df25e4cSSasha Neftin * igc_request_msix - Initialize MSI-X interrupts
54313df25e4cSSasha Neftin * @adapter: Pointer to adapter structure
54323df25e4cSSasha Neftin *
54333df25e4cSSasha Neftin * igc_request_msix allocates MSI-X vectors and requests interrupts from the
54343df25e4cSSasha Neftin * kernel.
54353df25e4cSSasha Neftin */
igc_request_msix(struct igc_adapter * adapter)54363df25e4cSSasha Neftin static int igc_request_msix(struct igc_adapter *adapter)
54373df25e4cSSasha Neftin {
5438373e2829SSasha Neftin unsigned int num_q_vectors = adapter->num_q_vectors;
54393df25e4cSSasha Neftin int i = 0, err = 0, vector = 0, free_vector = 0;
54403df25e4cSSasha Neftin struct net_device *netdev = adapter->netdev;
54413df25e4cSSasha Neftin
54423df25e4cSSasha Neftin err = request_irq(adapter->msix_entries[vector].vector,
54433df25e4cSSasha Neftin &igc_msix_other, 0, netdev->name, adapter);
54443df25e4cSSasha Neftin if (err)
54453df25e4cSSasha Neftin goto err_out;
54463df25e4cSSasha Neftin
5447373e2829SSasha Neftin if (num_q_vectors > MAX_Q_VECTORS) {
5448373e2829SSasha Neftin num_q_vectors = MAX_Q_VECTORS;
5449373e2829SSasha Neftin dev_warn(&adapter->pdev->dev,
5450373e2829SSasha Neftin "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5451373e2829SSasha Neftin adapter->num_q_vectors, MAX_Q_VECTORS);
5452373e2829SSasha Neftin }
5453373e2829SSasha Neftin for (i = 0; i < num_q_vectors; i++) {
54543df25e4cSSasha Neftin struct igc_q_vector *q_vector = adapter->q_vector[i];
54553df25e4cSSasha Neftin
54563df25e4cSSasha Neftin vector++;
54573df25e4cSSasha Neftin
54583df25e4cSSasha Neftin q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
54593df25e4cSSasha Neftin
54603df25e4cSSasha Neftin if (q_vector->rx.ring && q_vector->tx.ring)
54613df25e4cSSasha Neftin sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
54623df25e4cSSasha Neftin q_vector->rx.ring->queue_index);
54633df25e4cSSasha Neftin else if (q_vector->tx.ring)
54643df25e4cSSasha Neftin sprintf(q_vector->name, "%s-tx-%u", netdev->name,
54653df25e4cSSasha Neftin q_vector->tx.ring->queue_index);
54663df25e4cSSasha Neftin else if (q_vector->rx.ring)
54673df25e4cSSasha Neftin sprintf(q_vector->name, "%s-rx-%u", netdev->name,
54683df25e4cSSasha Neftin q_vector->rx.ring->queue_index);
54693df25e4cSSasha Neftin else
54703df25e4cSSasha Neftin sprintf(q_vector->name, "%s-unused", netdev->name);
54713df25e4cSSasha Neftin
54723df25e4cSSasha Neftin err = request_irq(adapter->msix_entries[vector].vector,
54733df25e4cSSasha Neftin igc_msix_ring, 0, q_vector->name,
54743df25e4cSSasha Neftin q_vector);
54753df25e4cSSasha Neftin if (err)
54763df25e4cSSasha Neftin goto err_free;
54773df25e4cSSasha Neftin }
54783df25e4cSSasha Neftin
54793df25e4cSSasha Neftin igc_configure_msix(adapter);
54803df25e4cSSasha Neftin return 0;
54813df25e4cSSasha Neftin
54823df25e4cSSasha Neftin err_free:
54833df25e4cSSasha Neftin /* free already assigned IRQs */
54843df25e4cSSasha Neftin free_irq(adapter->msix_entries[free_vector++].vector, adapter);
54853df25e4cSSasha Neftin
54863df25e4cSSasha Neftin vector--;
54873df25e4cSSasha Neftin for (i = 0; i < vector; i++) {
54883df25e4cSSasha Neftin free_irq(adapter->msix_entries[free_vector++].vector,
54893df25e4cSSasha Neftin adapter->q_vector[i]);
54903df25e4cSSasha Neftin }
54913df25e4cSSasha Neftin err_out:
54923df25e4cSSasha Neftin return err;
54933df25e4cSSasha Neftin }
54943df25e4cSSasha Neftin
54953df25e4cSSasha Neftin /**
5496a8c4873bSSasha Neftin * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5497a8c4873bSSasha Neftin * @adapter: Pointer to adapter structure
5498a8c4873bSSasha Neftin *
5499a8c4873bSSasha Neftin * This function resets the device so that it has 0 rx queues, tx queues, and
5500a8c4873bSSasha Neftin * MSI-X interrupts allocated.
5501a8c4873bSSasha Neftin */
igc_clear_interrupt_scheme(struct igc_adapter * adapter)5502a8c4873bSSasha Neftin static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
5503a8c4873bSSasha Neftin {
5504a8c4873bSSasha Neftin igc_free_q_vectors(adapter);
5505a8c4873bSSasha Neftin igc_reset_interrupt_capability(adapter);
5506a8c4873bSSasha Neftin }
5507a8c4873bSSasha Neftin
5508208983f0SSasha Neftin /* Need to wait a few seconds after link up to get diagnostic information from
5509208983f0SSasha Neftin * the phy
5510208983f0SSasha Neftin */
igc_update_phy_info(struct timer_list * t)5511208983f0SSasha Neftin static void igc_update_phy_info(struct timer_list *t)
5512208983f0SSasha Neftin {
5513208983f0SSasha Neftin struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5514208983f0SSasha Neftin
5515208983f0SSasha Neftin igc_get_phy_info(&adapter->hw);
5516208983f0SSasha Neftin }
5517208983f0SSasha Neftin
5518208983f0SSasha Neftin /**
5519208983f0SSasha Neftin * igc_has_link - check shared code for link and determine up/down
5520208983f0SSasha Neftin * @adapter: pointer to driver private info
5521208983f0SSasha Neftin */
igc_has_link(struct igc_adapter * adapter)55228c5ad0daSSasha Neftin bool igc_has_link(struct igc_adapter *adapter)
5523208983f0SSasha Neftin {
5524208983f0SSasha Neftin struct igc_hw *hw = &adapter->hw;
5525208983f0SSasha Neftin bool link_active = false;
5526208983f0SSasha Neftin
5527208983f0SSasha Neftin /* get_link_status is set on LSC (link status) interrupt or
5528208983f0SSasha Neftin * rx sequence error interrupt. get_link_status will stay
5529208983f0SSasha Neftin * false until the igc_check_for_link establishes link
5530208983f0SSasha Neftin * for copper adapters ONLY
5531208983f0SSasha Neftin */
5532208983f0SSasha Neftin if (!hw->mac.get_link_status)
5533208983f0SSasha Neftin return true;
5534208983f0SSasha Neftin hw->mac.ops.check_for_link(hw);
5535208983f0SSasha Neftin link_active = !hw->mac.get_link_status;
5536208983f0SSasha Neftin
55377c496de5SSasha Neftin if (hw->mac.type == igc_i225) {
5538208983f0SSasha Neftin if (!netif_carrier_ok(adapter->netdev)) {
5539208983f0SSasha Neftin adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5540208983f0SSasha Neftin } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
5541208983f0SSasha Neftin adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
5542208983f0SSasha Neftin adapter->link_check_timeout = jiffies;
5543208983f0SSasha Neftin }
5544208983f0SSasha Neftin }
5545208983f0SSasha Neftin
5546208983f0SSasha Neftin return link_active;
5547208983f0SSasha Neftin }
5548208983f0SSasha Neftin
55493df25e4cSSasha Neftin /**
55500507ef8aSSasha Neftin * igc_watchdog - Timer Call-back
555186efeccdSSasha Neftin * @t: timer for the watchdog
55520507ef8aSSasha Neftin */
igc_watchdog(struct timer_list * t)55530507ef8aSSasha Neftin static void igc_watchdog(struct timer_list *t)
55540507ef8aSSasha Neftin {
55550507ef8aSSasha Neftin struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5556208983f0SSasha Neftin /* Do the rest outside of interrupt context */
5557208983f0SSasha Neftin schedule_work(&adapter->watchdog_task);
5558208983f0SSasha Neftin }
5559208983f0SSasha Neftin
igc_watchdog_task(struct work_struct * work)5560208983f0SSasha Neftin static void igc_watchdog_task(struct work_struct *work)
5561208983f0SSasha Neftin {
5562208983f0SSasha Neftin struct igc_adapter *adapter = container_of(work,
5563208983f0SSasha Neftin struct igc_adapter,
5564208983f0SSasha Neftin watchdog_task);
5565208983f0SSasha Neftin struct net_device *netdev = adapter->netdev;
5566208983f0SSasha Neftin struct igc_hw *hw = &adapter->hw;
5567208983f0SSasha Neftin struct igc_phy_info *phy = &hw->phy;
5568208983f0SSasha Neftin u16 phy_data, retry_count = 20;
5569208983f0SSasha Neftin u32 link;
5570208983f0SSasha Neftin int i;
5571208983f0SSasha Neftin
5572208983f0SSasha Neftin link = igc_has_link(adapter);
5573208983f0SSasha Neftin
5574208983f0SSasha Neftin if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
5575208983f0SSasha Neftin if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5576208983f0SSasha Neftin adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5577208983f0SSasha Neftin else
5578208983f0SSasha Neftin link = false;
5579208983f0SSasha Neftin }
5580208983f0SSasha Neftin
5581208983f0SSasha Neftin if (link) {
55828594a7f3SSasha Neftin /* Cancel scheduled suspend requests. */
55838594a7f3SSasha Neftin pm_runtime_resume(netdev->dev.parent);
55848594a7f3SSasha Neftin
5585208983f0SSasha Neftin if (!netif_carrier_ok(netdev)) {
5586208983f0SSasha Neftin u32 ctrl;
5587208983f0SSasha Neftin
5588208983f0SSasha Neftin hw->mac.ops.get_speed_and_duplex(hw,
5589208983f0SSasha Neftin &adapter->link_speed,
5590208983f0SSasha Neftin &adapter->link_duplex);
5591208983f0SSasha Neftin
5592208983f0SSasha Neftin ctrl = rd32(IGC_CTRL);
5593208983f0SSasha Neftin /* Link status message must follow this format */
5594208983f0SSasha Neftin netdev_info(netdev,
559525f06effSAndre Guedes "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5596208983f0SSasha Neftin adapter->link_speed,
5597208983f0SSasha Neftin adapter->link_duplex == FULL_DUPLEX ?
5598208983f0SSasha Neftin "Full" : "Half",
5599208983f0SSasha Neftin (ctrl & IGC_CTRL_TFCE) &&
5600208983f0SSasha Neftin (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
5601208983f0SSasha Neftin (ctrl & IGC_CTRL_RFCE) ? "RX" :
5602208983f0SSasha Neftin (ctrl & IGC_CTRL_TFCE) ? "TX" : "None");
5603208983f0SSasha Neftin
560493ec439aSSasha Neftin /* disable EEE if enabled */
560593ec439aSSasha Neftin if ((adapter->flags & IGC_FLAG_EEE) &&
560693ec439aSSasha Neftin adapter->link_duplex == HALF_DUPLEX) {
560793ec439aSSasha Neftin netdev_info(netdev,
560893ec439aSSasha Neftin "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
560993ec439aSSasha Neftin adapter->hw.dev_spec._base.eee_enable = false;
561093ec439aSSasha Neftin adapter->flags &= ~IGC_FLAG_EEE;
561193ec439aSSasha Neftin }
561293ec439aSSasha Neftin
5613208983f0SSasha Neftin /* check if SmartSpeed worked */
5614208983f0SSasha Neftin igc_check_downshift(hw);
5615208983f0SSasha Neftin if (phy->speed_downgraded)
5616208983f0SSasha Neftin netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5617208983f0SSasha Neftin
5618208983f0SSasha Neftin /* adjust timeout factor according to speed/duplex */
5619208983f0SSasha Neftin adapter->tx_timeout_factor = 1;
5620208983f0SSasha Neftin switch (adapter->link_speed) {
5621208983f0SSasha Neftin case SPEED_10:
5622208983f0SSasha Neftin adapter->tx_timeout_factor = 14;
5623208983f0SSasha Neftin break;
5624208983f0SSasha Neftin case SPEED_100:
5625b27b8dc7SMuhammad Husaini Zulkifli case SPEED_1000:
5626b27b8dc7SMuhammad Husaini Zulkifli case SPEED_2500:
56279b275176SSasha Neftin adapter->tx_timeout_factor = 1;
5628208983f0SSasha Neftin break;
5629208983f0SSasha Neftin }
5630208983f0SSasha Neftin
5631790835fcSMuhammad Husaini Zulkifli /* Once the launch time has been set on the wire, there
5632790835fcSMuhammad Husaini Zulkifli * is a delay before the link speed can be determined
5633790835fcSMuhammad Husaini Zulkifli * based on link-up activity. Write into the register
5634790835fcSMuhammad Husaini Zulkifli * as soon as we know the correct link speed.
5635790835fcSMuhammad Husaini Zulkifli */
5636790835fcSMuhammad Husaini Zulkifli igc_tsn_adjust_txtime_offset(adapter);
5637790835fcSMuhammad Husaini Zulkifli
5638208983f0SSasha Neftin if (adapter->link_speed != SPEED_1000)
5639208983f0SSasha Neftin goto no_wait;
5640208983f0SSasha Neftin
5641208983f0SSasha Neftin /* wait for Remote receiver status OK */
5642208983f0SSasha Neftin retry_read_status:
5643208983f0SSasha Neftin if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
5644208983f0SSasha Neftin &phy_data)) {
5645208983f0SSasha Neftin if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5646208983f0SSasha Neftin retry_count) {
5647208983f0SSasha Neftin msleep(100);
5648208983f0SSasha Neftin retry_count--;
5649208983f0SSasha Neftin goto retry_read_status;
5650208983f0SSasha Neftin } else if (!retry_count) {
565125f06effSAndre Guedes netdev_err(netdev, "exceed max 2 second\n");
5652208983f0SSasha Neftin }
5653208983f0SSasha Neftin } else {
565425f06effSAndre Guedes netdev_err(netdev, "read 1000Base-T Status Reg\n");
5655208983f0SSasha Neftin }
5656208983f0SSasha Neftin no_wait:
5657208983f0SSasha Neftin netif_carrier_on(netdev);
5658208983f0SSasha Neftin
5659208983f0SSasha Neftin /* link state has changed, schedule phy info update */
5660208983f0SSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state))
5661208983f0SSasha Neftin mod_timer(&adapter->phy_info_timer,
5662208983f0SSasha Neftin round_jiffies(jiffies + 2 * HZ));
5663208983f0SSasha Neftin }
5664208983f0SSasha Neftin } else {
5665208983f0SSasha Neftin if (netif_carrier_ok(netdev)) {
5666208983f0SSasha Neftin adapter->link_speed = 0;
5667208983f0SSasha Neftin adapter->link_duplex = 0;
5668208983f0SSasha Neftin
5669208983f0SSasha Neftin /* Links status message must follow this format */
567025f06effSAndre Guedes netdev_info(netdev, "NIC Link is Down\n");
5671208983f0SSasha Neftin netif_carrier_off(netdev);
5672208983f0SSasha Neftin
5673208983f0SSasha Neftin /* link state has changed, schedule phy info update */
5674208983f0SSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state))
5675208983f0SSasha Neftin mod_timer(&adapter->phy_info_timer,
5676208983f0SSasha Neftin round_jiffies(jiffies + 2 * HZ));
5677208983f0SSasha Neftin
56788594a7f3SSasha Neftin pm_schedule_suspend(netdev->dev.parent,
56798594a7f3SSasha Neftin MSEC_PER_SEC * 5);
5680208983f0SSasha Neftin }
5681208983f0SSasha Neftin }
5682208983f0SSasha Neftin
5683208983f0SSasha Neftin spin_lock(&adapter->stats64_lock);
5684208983f0SSasha Neftin igc_update_stats(adapter);
5685208983f0SSasha Neftin spin_unlock(&adapter->stats64_lock);
5686208983f0SSasha Neftin
5687208983f0SSasha Neftin for (i = 0; i < adapter->num_tx_queues; i++) {
5688208983f0SSasha Neftin struct igc_ring *tx_ring = adapter->tx_ring[i];
5689208983f0SSasha Neftin
5690208983f0SSasha Neftin if (!netif_carrier_ok(netdev)) {
5691208983f0SSasha Neftin /* We've lost link, so the controller stops DMA,
5692208983f0SSasha Neftin * but we've got queued Tx work that's never going
5693208983f0SSasha Neftin * to get done, so reset controller to flush Tx.
5694208983f0SSasha Neftin * (Do the reset outside of interrupt context).
5695208983f0SSasha Neftin */
5696208983f0SSasha Neftin if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
5697208983f0SSasha Neftin adapter->tx_timeout_count++;
5698208983f0SSasha Neftin schedule_work(&adapter->reset_task);
5699208983f0SSasha Neftin /* return immediately since reset is imminent */
5700208983f0SSasha Neftin return;
5701208983f0SSasha Neftin }
5702208983f0SSasha Neftin }
5703208983f0SSasha Neftin
5704208983f0SSasha Neftin /* Force detection of hung controller every watchdog period */
5705208983f0SSasha Neftin set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5706208983f0SSasha Neftin }
5707208983f0SSasha Neftin
5708208983f0SSasha Neftin /* Cause software interrupt to ensure Rx ring is cleaned */
5709208983f0SSasha Neftin if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5710208983f0SSasha Neftin u32 eics = 0;
5711208983f0SSasha Neftin
5712208983f0SSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++)
5713208983f0SSasha Neftin eics |= adapter->q_vector[i]->eims_value;
5714208983f0SSasha Neftin wr32(IGC_EICS, eics);
5715208983f0SSasha Neftin } else {
5716208983f0SSasha Neftin wr32(IGC_ICS, IGC_ICS_RXDMT0);
5717208983f0SSasha Neftin }
5718208983f0SSasha Neftin
57192c344ae2SVinicius Costa Gomes igc_ptp_tx_hang(adapter);
57202c344ae2SVinicius Costa Gomes
5721208983f0SSasha Neftin /* Reset the timer */
5722208983f0SSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state)) {
5723208983f0SSasha Neftin if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
5724208983f0SSasha Neftin mod_timer(&adapter->watchdog_timer,
5725208983f0SSasha Neftin round_jiffies(jiffies + HZ));
5726208983f0SSasha Neftin else
5727208983f0SSasha Neftin mod_timer(&adapter->watchdog_timer,
5728208983f0SSasha Neftin round_jiffies(jiffies + 2 * HZ));
5729208983f0SSasha Neftin }
57300507ef8aSSasha Neftin }
57310507ef8aSSasha Neftin
57320507ef8aSSasha Neftin /**
573313b5b7fdSSasha Neftin * igc_intr_msi - Interrupt Handler
573413b5b7fdSSasha Neftin * @irq: interrupt number
573513b5b7fdSSasha Neftin * @data: pointer to a network interface device structure
573613b5b7fdSSasha Neftin */
igc_intr_msi(int irq,void * data)573713b5b7fdSSasha Neftin static irqreturn_t igc_intr_msi(int irq, void *data)
573813b5b7fdSSasha Neftin {
573913b5b7fdSSasha Neftin struct igc_adapter *adapter = data;
574013b5b7fdSSasha Neftin struct igc_q_vector *q_vector = adapter->q_vector[0];
574113b5b7fdSSasha Neftin struct igc_hw *hw = &adapter->hw;
574213b5b7fdSSasha Neftin /* read ICR disables interrupts using IAM */
574313b5b7fdSSasha Neftin u32 icr = rd32(IGC_ICR);
574413b5b7fdSSasha Neftin
574513b5b7fdSSasha Neftin igc_write_itr(q_vector);
574613b5b7fdSSasha Neftin
574713b5b7fdSSasha Neftin if (icr & IGC_ICR_DRSTA)
574813b5b7fdSSasha Neftin schedule_work(&adapter->reset_task);
574913b5b7fdSSasha Neftin
575013b5b7fdSSasha Neftin if (icr & IGC_ICR_DOUTSYNC) {
575113b5b7fdSSasha Neftin /* HW is reporting DMA is out of sync */
575213b5b7fdSSasha Neftin adapter->stats.doosync++;
575313b5b7fdSSasha Neftin }
575413b5b7fdSSasha Neftin
575513b5b7fdSSasha Neftin if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5756501f2309SJiapeng Zhong hw->mac.get_link_status = true;
575713b5b7fdSSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state))
575813b5b7fdSSasha Neftin mod_timer(&adapter->watchdog_timer, jiffies + 1);
575913b5b7fdSSasha Neftin }
576013b5b7fdSSasha Neftin
5761f85846bbSJames McLaughlin if (icr & IGC_ICR_TS)
5762f85846bbSJames McLaughlin igc_tsync_interrupt(adapter);
5763f85846bbSJames McLaughlin
576413b5b7fdSSasha Neftin napi_schedule(&q_vector->napi);
576513b5b7fdSSasha Neftin
576613b5b7fdSSasha Neftin return IRQ_HANDLED;
576713b5b7fdSSasha Neftin }
576813b5b7fdSSasha Neftin
576913b5b7fdSSasha Neftin /**
577013b5b7fdSSasha Neftin * igc_intr - Legacy Interrupt Handler
577113b5b7fdSSasha Neftin * @irq: interrupt number
577213b5b7fdSSasha Neftin * @data: pointer to a network interface device structure
577313b5b7fdSSasha Neftin */
igc_intr(int irq,void * data)577413b5b7fdSSasha Neftin static irqreturn_t igc_intr(int irq, void *data)
577513b5b7fdSSasha Neftin {
577613b5b7fdSSasha Neftin struct igc_adapter *adapter = data;
577713b5b7fdSSasha Neftin struct igc_q_vector *q_vector = adapter->q_vector[0];
577813b5b7fdSSasha Neftin struct igc_hw *hw = &adapter->hw;
577913b5b7fdSSasha Neftin /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
578013b5b7fdSSasha Neftin * need for the IMC write
578113b5b7fdSSasha Neftin */
578213b5b7fdSSasha Neftin u32 icr = rd32(IGC_ICR);
578313b5b7fdSSasha Neftin
578413b5b7fdSSasha Neftin /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
578513b5b7fdSSasha Neftin * not set, then the adapter didn't send an interrupt
578613b5b7fdSSasha Neftin */
578713b5b7fdSSasha Neftin if (!(icr & IGC_ICR_INT_ASSERTED))
578813b5b7fdSSasha Neftin return IRQ_NONE;
578913b5b7fdSSasha Neftin
579013b5b7fdSSasha Neftin igc_write_itr(q_vector);
579113b5b7fdSSasha Neftin
579213b5b7fdSSasha Neftin if (icr & IGC_ICR_DRSTA)
579313b5b7fdSSasha Neftin schedule_work(&adapter->reset_task);
579413b5b7fdSSasha Neftin
579513b5b7fdSSasha Neftin if (icr & IGC_ICR_DOUTSYNC) {
579613b5b7fdSSasha Neftin /* HW is reporting DMA is out of sync */
579713b5b7fdSSasha Neftin adapter->stats.doosync++;
579813b5b7fdSSasha Neftin }
579913b5b7fdSSasha Neftin
580013b5b7fdSSasha Neftin if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5801501f2309SJiapeng Zhong hw->mac.get_link_status = true;
580213b5b7fdSSasha Neftin /* guard against interrupt when we're going down */
580313b5b7fdSSasha Neftin if (!test_bit(__IGC_DOWN, &adapter->state))
580413b5b7fdSSasha Neftin mod_timer(&adapter->watchdog_timer, jiffies + 1);
580513b5b7fdSSasha Neftin }
580613b5b7fdSSasha Neftin
5807f85846bbSJames McLaughlin if (icr & IGC_ICR_TS)
5808f85846bbSJames McLaughlin igc_tsync_interrupt(adapter);
5809f85846bbSJames McLaughlin
581013b5b7fdSSasha Neftin napi_schedule(&q_vector->napi);
581113b5b7fdSSasha Neftin
581213b5b7fdSSasha Neftin return IRQ_HANDLED;
581313b5b7fdSSasha Neftin }
581413b5b7fdSSasha Neftin
igc_free_irq(struct igc_adapter * adapter)58153df25e4cSSasha Neftin static void igc_free_irq(struct igc_adapter *adapter)
58163df25e4cSSasha Neftin {
58173df25e4cSSasha Neftin if (adapter->msix_entries) {
58183df25e4cSSasha Neftin int vector = 0, i;
58193df25e4cSSasha Neftin
58203df25e4cSSasha Neftin free_irq(adapter->msix_entries[vector++].vector, adapter);
58213df25e4cSSasha Neftin
58223df25e4cSSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++)
58233df25e4cSSasha Neftin free_irq(adapter->msix_entries[vector++].vector,
58243df25e4cSSasha Neftin adapter->q_vector[i]);
58253df25e4cSSasha Neftin } else {
58263df25e4cSSasha Neftin free_irq(adapter->pdev->irq, adapter);
58273df25e4cSSasha Neftin }
58283df25e4cSSasha Neftin }
58293df25e4cSSasha Neftin
58303df25e4cSSasha Neftin /**
58313df25e4cSSasha Neftin * igc_request_irq - initialize interrupts
58323df25e4cSSasha Neftin * @adapter: Pointer to adapter structure
58333df25e4cSSasha Neftin *
58343df25e4cSSasha Neftin * Attempts to configure interrupts using the best available
58353df25e4cSSasha Neftin * capabilities of the hardware and kernel.
58363df25e4cSSasha Neftin */
igc_request_irq(struct igc_adapter * adapter)58373df25e4cSSasha Neftin static int igc_request_irq(struct igc_adapter *adapter)
58383df25e4cSSasha Neftin {
583913b5b7fdSSasha Neftin struct net_device *netdev = adapter->netdev;
584013b5b7fdSSasha Neftin struct pci_dev *pdev = adapter->pdev;
58413df25e4cSSasha Neftin int err = 0;
58423df25e4cSSasha Neftin
58433df25e4cSSasha Neftin if (adapter->flags & IGC_FLAG_HAS_MSIX) {
58443df25e4cSSasha Neftin err = igc_request_msix(adapter);
58453df25e4cSSasha Neftin if (!err)
58463df25e4cSSasha Neftin goto request_done;
58473df25e4cSSasha Neftin /* fall back to MSI */
584813b5b7fdSSasha Neftin igc_free_all_tx_resources(adapter);
584913b5b7fdSSasha Neftin igc_free_all_rx_resources(adapter);
58503df25e4cSSasha Neftin
58513df25e4cSSasha Neftin igc_clear_interrupt_scheme(adapter);
58523df25e4cSSasha Neftin err = igc_init_interrupt_scheme(adapter, false);
58533df25e4cSSasha Neftin if (err)
58543df25e4cSSasha Neftin goto request_done;
585513b5b7fdSSasha Neftin igc_setup_all_tx_resources(adapter);
585613b5b7fdSSasha Neftin igc_setup_all_rx_resources(adapter);
58573df25e4cSSasha Neftin igc_configure(adapter);
58583df25e4cSSasha Neftin }
58593df25e4cSSasha Neftin
586013b5b7fdSSasha Neftin igc_assign_vector(adapter->q_vector[0], 0);
586113b5b7fdSSasha Neftin
586213b5b7fdSSasha Neftin if (adapter->flags & IGC_FLAG_HAS_MSI) {
586313b5b7fdSSasha Neftin err = request_irq(pdev->irq, &igc_intr_msi, 0,
586413b5b7fdSSasha Neftin netdev->name, adapter);
586513b5b7fdSSasha Neftin if (!err)
586613b5b7fdSSasha Neftin goto request_done;
586713b5b7fdSSasha Neftin
586813b5b7fdSSasha Neftin /* fall back to legacy interrupts */
586913b5b7fdSSasha Neftin igc_reset_interrupt_capability(adapter);
587013b5b7fdSSasha Neftin adapter->flags &= ~IGC_FLAG_HAS_MSI;
587113b5b7fdSSasha Neftin }
587213b5b7fdSSasha Neftin
587313b5b7fdSSasha Neftin err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
587413b5b7fdSSasha Neftin netdev->name, adapter);
587513b5b7fdSSasha Neftin
587613b5b7fdSSasha Neftin if (err)
587725f06effSAndre Guedes netdev_err(netdev, "Error %d getting interrupt\n", err);
587813b5b7fdSSasha Neftin
58793df25e4cSSasha Neftin request_done:
58803df25e4cSSasha Neftin return err;
58813df25e4cSSasha Neftin }
58823df25e4cSSasha Neftin
58833df25e4cSSasha Neftin /**
588486efeccdSSasha Neftin * __igc_open - Called when a network interface is made active
5885c9a11c23SSasha Neftin * @netdev: network interface device structure
588686efeccdSSasha Neftin * @resuming: boolean indicating if the device is resuming
5887c9a11c23SSasha Neftin *
5888c9a11c23SSasha Neftin * Returns 0 on success, negative value on failure
5889c9a11c23SSasha Neftin *
5890c9a11c23SSasha Neftin * The open entry point is called when a network interface is made
5891c9a11c23SSasha Neftin * active by the system (IFF_UP). At this point all resources needed
5892c9a11c23SSasha Neftin * for transmit and receive operations are allocated, the interrupt
5893c9a11c23SSasha Neftin * handler is registered with the OS, the watchdog timer is started,
5894c9a11c23SSasha Neftin * and the stack is notified that the interface is ready.
5895c9a11c23SSasha Neftin */
__igc_open(struct net_device * netdev,bool resuming)5896c9a11c23SSasha Neftin static int __igc_open(struct net_device *netdev, bool resuming)
5897c9a11c23SSasha Neftin {
5898c9a11c23SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
58998594a7f3SSasha Neftin struct pci_dev *pdev = adapter->pdev;
5900c9a11c23SSasha Neftin struct igc_hw *hw = &adapter->hw;
59013df25e4cSSasha Neftin int err = 0;
5902c9a11c23SSasha Neftin int i = 0;
5903c9a11c23SSasha Neftin
5904c9a11c23SSasha Neftin /* disallow open during test */
5905c9a11c23SSasha Neftin
5906c9a11c23SSasha Neftin if (test_bit(__IGC_TESTING, &adapter->state)) {
5907c9a11c23SSasha Neftin WARN_ON(resuming);
5908c9a11c23SSasha Neftin return -EBUSY;
5909c9a11c23SSasha Neftin }
5910c9a11c23SSasha Neftin
59118594a7f3SSasha Neftin if (!resuming)
59128594a7f3SSasha Neftin pm_runtime_get_sync(&pdev->dev);
59138594a7f3SSasha Neftin
5914c9a11c23SSasha Neftin netif_carrier_off(netdev);
5915c9a11c23SSasha Neftin
591613b5b7fdSSasha Neftin /* allocate transmit descriptors */
591713b5b7fdSSasha Neftin err = igc_setup_all_tx_resources(adapter);
591813b5b7fdSSasha Neftin if (err)
591913b5b7fdSSasha Neftin goto err_setup_tx;
592013b5b7fdSSasha Neftin
592113b5b7fdSSasha Neftin /* allocate receive descriptors */
592213b5b7fdSSasha Neftin err = igc_setup_all_rx_resources(adapter);
592313b5b7fdSSasha Neftin if (err)
592413b5b7fdSSasha Neftin goto err_setup_rx;
592513b5b7fdSSasha Neftin
5926c9a11c23SSasha Neftin igc_power_up_link(adapter);
5927c9a11c23SSasha Neftin
5928c9a11c23SSasha Neftin igc_configure(adapter);
5929c9a11c23SSasha Neftin
59303df25e4cSSasha Neftin err = igc_request_irq(adapter);
59313df25e4cSSasha Neftin if (err)
59323df25e4cSSasha Neftin goto err_req_irq;
59333df25e4cSSasha Neftin
59343df25e4cSSasha Neftin /* Notify the stack of the actual queue counts. */
593514b21cecSColin Ian King err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
59363df25e4cSSasha Neftin if (err)
59373df25e4cSSasha Neftin goto err_set_queues;
59383df25e4cSSasha Neftin
59393df25e4cSSasha Neftin err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
59403df25e4cSSasha Neftin if (err)
59413df25e4cSSasha Neftin goto err_set_queues;
59423df25e4cSSasha Neftin
5943c9a11c23SSasha Neftin clear_bit(__IGC_DOWN, &adapter->state);
5944c9a11c23SSasha Neftin
5945c9a11c23SSasha Neftin for (i = 0; i < adapter->num_q_vectors; i++)
5946c9a11c23SSasha Neftin napi_enable(&adapter->q_vector[i]->napi);
5947c9a11c23SSasha Neftin
59483df25e4cSSasha Neftin /* Clear any pending interrupts. */
59493df25e4cSSasha Neftin rd32(IGC_ICR);
59503df25e4cSSasha Neftin igc_irq_enable(adapter);
59513df25e4cSSasha Neftin
59528594a7f3SSasha Neftin if (!resuming)
59538594a7f3SSasha Neftin pm_runtime_put(&pdev->dev);
59548594a7f3SSasha Neftin
595513b5b7fdSSasha Neftin netif_tx_start_all_queues(netdev);
595613b5b7fdSSasha Neftin
5957c9a11c23SSasha Neftin /* start the watchdog. */
5958501f2309SJiapeng Zhong hw->mac.get_link_status = true;
5959208983f0SSasha Neftin schedule_work(&adapter->watchdog_task);
5960c9a11c23SSasha Neftin
5961c9a11c23SSasha Neftin return IGC_SUCCESS;
59623df25e4cSSasha Neftin
59633df25e4cSSasha Neftin err_set_queues:
59643df25e4cSSasha Neftin igc_free_irq(adapter);
59653df25e4cSSasha Neftin err_req_irq:
59663df25e4cSSasha Neftin igc_release_hw_control(adapter);
5967a0beb3c1SSasha Neftin igc_power_down_phy_copper_base(&adapter->hw);
596813b5b7fdSSasha Neftin igc_free_all_rx_resources(adapter);
596913b5b7fdSSasha Neftin err_setup_rx:
597013b5b7fdSSasha Neftin igc_free_all_tx_resources(adapter);
597113b5b7fdSSasha Neftin err_setup_tx:
597213b5b7fdSSasha Neftin igc_reset(adapter);
59738594a7f3SSasha Neftin if (!resuming)
59748594a7f3SSasha Neftin pm_runtime_put(&pdev->dev);
59753df25e4cSSasha Neftin
59763df25e4cSSasha Neftin return err;
5977c9a11c23SSasha Neftin }
5978c9a11c23SSasha Neftin
igc_open(struct net_device * netdev)5979f026d8caSVitaly Lifshits int igc_open(struct net_device *netdev)
5980c9a11c23SSasha Neftin {
5981c9a11c23SSasha Neftin return __igc_open(netdev, false);
5982c9a11c23SSasha Neftin }
5983c9a11c23SSasha Neftin
5984c9a11c23SSasha Neftin /**
598586efeccdSSasha Neftin * __igc_close - Disables a network interface
5986c9a11c23SSasha Neftin * @netdev: network interface device structure
598786efeccdSSasha Neftin * @suspending: boolean indicating the device is suspending
5988c9a11c23SSasha Neftin *
5989c9a11c23SSasha Neftin * Returns 0, this is not allowed to fail
5990c9a11c23SSasha Neftin *
5991c9a11c23SSasha Neftin * The close entry point is called when an interface is de-activated
5992c9a11c23SSasha Neftin * by the OS. The hardware is still under the driver's control, but
5993c9a11c23SSasha Neftin * needs to be disabled. A global MAC reset is issued to stop the
5994c9a11c23SSasha Neftin * hardware, and all transmit and receive resources are freed.
5995c9a11c23SSasha Neftin */
__igc_close(struct net_device * netdev,bool suspending)5996c9a11c23SSasha Neftin static int __igc_close(struct net_device *netdev, bool suspending)
5997c9a11c23SSasha Neftin {
5998c9a11c23SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
59998594a7f3SSasha Neftin struct pci_dev *pdev = adapter->pdev;
6000c9a11c23SSasha Neftin
6001c9a11c23SSasha Neftin WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
6002c9a11c23SSasha Neftin
60038594a7f3SSasha Neftin if (!suspending)
60048594a7f3SSasha Neftin pm_runtime_get_sync(&pdev->dev);
60058594a7f3SSasha Neftin
6006c9a11c23SSasha Neftin igc_down(adapter);
6007c9a11c23SSasha Neftin
6008c9a11c23SSasha Neftin igc_release_hw_control(adapter);
6009c9a11c23SSasha Neftin
60103df25e4cSSasha Neftin igc_free_irq(adapter);
60113df25e4cSSasha Neftin
601213b5b7fdSSasha Neftin igc_free_all_tx_resources(adapter);
601313b5b7fdSSasha Neftin igc_free_all_rx_resources(adapter);
601413b5b7fdSSasha Neftin
60158594a7f3SSasha Neftin if (!suspending)
60168594a7f3SSasha Neftin pm_runtime_put_sync(&pdev->dev);
60178594a7f3SSasha Neftin
6018c9a11c23SSasha Neftin return 0;
6019c9a11c23SSasha Neftin }
6020c9a11c23SSasha Neftin
igc_close(struct net_device * netdev)6021f026d8caSVitaly Lifshits int igc_close(struct net_device *netdev)
6022c9a11c23SSasha Neftin {
6023c9a11c23SSasha Neftin if (netif_device_present(netdev) || netdev->dismantle)
6024c9a11c23SSasha Neftin return __igc_close(netdev, false);
6025c9a11c23SSasha Neftin return 0;
6026c9a11c23SSasha Neftin }
6027c9a11c23SSasha Neftin
60285f295805SVinicius Costa Gomes /**
60295f295805SVinicius Costa Gomes * igc_ioctl - Access the hwtstamp interface
60305f295805SVinicius Costa Gomes * @netdev: network interface device structure
6031b50f7bcaSJesse Brandeburg * @ifr: interface request data
60325f295805SVinicius Costa Gomes * @cmd: ioctl command
60335f295805SVinicius Costa Gomes **/
igc_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)60345f295805SVinicius Costa Gomes static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
60355f295805SVinicius Costa Gomes {
60365f295805SVinicius Costa Gomes switch (cmd) {
60375f295805SVinicius Costa Gomes case SIOCGHWTSTAMP:
60385f295805SVinicius Costa Gomes return igc_ptp_get_ts_config(netdev, ifr);
60395f295805SVinicius Costa Gomes case SIOCSHWTSTAMP:
60405f295805SVinicius Costa Gomes return igc_ptp_set_ts_config(netdev, ifr);
60415f295805SVinicius Costa Gomes default:
60425f295805SVinicius Costa Gomes return -EOPNOTSUPP;
60435f295805SVinicius Costa Gomes }
60445f295805SVinicius Costa Gomes }
60455f295805SVinicius Costa Gomes
igc_save_launchtime_params(struct igc_adapter * adapter,int queue,bool enable)604682faa9b7SVinicius Costa Gomes static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
604782faa9b7SVinicius Costa Gomes bool enable)
604882faa9b7SVinicius Costa Gomes {
604982faa9b7SVinicius Costa Gomes struct igc_ring *ring;
605082faa9b7SVinicius Costa Gomes
605182faa9b7SVinicius Costa Gomes if (queue < 0 || queue >= adapter->num_tx_queues)
605282faa9b7SVinicius Costa Gomes return -EINVAL;
605382faa9b7SVinicius Costa Gomes
605482faa9b7SVinicius Costa Gomes ring = adapter->tx_ring[queue];
605582faa9b7SVinicius Costa Gomes ring->launchtime_enable = enable;
605682faa9b7SVinicius Costa Gomes
605782faa9b7SVinicius Costa Gomes return 0;
605882faa9b7SVinicius Costa Gomes }
605982faa9b7SVinicius Costa Gomes
is_base_time_past(ktime_t base_time,const struct timespec64 * now)606058c4ee0eSVinicius Costa Gomes static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
606158c4ee0eSVinicius Costa Gomes {
606258c4ee0eSVinicius Costa Gomes struct timespec64 b;
606358c4ee0eSVinicius Costa Gomes
606458c4ee0eSVinicius Costa Gomes b = ktime_to_timespec64(base_time);
606558c4ee0eSVinicius Costa Gomes
606658c4ee0eSVinicius Costa Gomes return timespec64_compare(now, &b) > 0;
606758c4ee0eSVinicius Costa Gomes }
606858c4ee0eSVinicius Costa Gomes
validate_schedule(struct igc_adapter * adapter,const struct tc_taprio_qopt_offload * qopt)606958c4ee0eSVinicius Costa Gomes static bool validate_schedule(struct igc_adapter *adapter,
607058c4ee0eSVinicius Costa Gomes const struct tc_taprio_qopt_offload *qopt)
6071ec50a9d4SVinicius Costa Gomes {
6072ec50a9d4SVinicius Costa Gomes int queue_uses[IGC_MAX_TX_QUEUES] = { };
6073b8897dc5SMuhammad Husaini Zulkifli struct igc_hw *hw = &adapter->hw;
607458c4ee0eSVinicius Costa Gomes struct timespec64 now;
6075ec50a9d4SVinicius Costa Gomes size_t n;
6076ec50a9d4SVinicius Costa Gomes
6077ec50a9d4SVinicius Costa Gomes if (qopt->cycle_time_extension)
6078ec50a9d4SVinicius Costa Gomes return false;
6079ec50a9d4SVinicius Costa Gomes
608058c4ee0eSVinicius Costa Gomes igc_ptp_read(adapter, &now);
608158c4ee0eSVinicius Costa Gomes
608258c4ee0eSVinicius Costa Gomes /* If we program the controller's BASET registers with a time
608358c4ee0eSVinicius Costa Gomes * in the future, it will hold all the packets until that
608458c4ee0eSVinicius Costa Gomes * time, causing a lot of TX Hangs, so to avoid that, we
608558c4ee0eSVinicius Costa Gomes * reject schedules that would start in the future.
6086b8897dc5SMuhammad Husaini Zulkifli * Note: Limitation above is no longer in i226.
608758c4ee0eSVinicius Costa Gomes */
6088b8897dc5SMuhammad Husaini Zulkifli if (!is_base_time_past(qopt->base_time, &now) &&
6089b8897dc5SMuhammad Husaini Zulkifli igc_is_device_id_i225(hw))
609058c4ee0eSVinicius Costa Gomes return false;
609158c4ee0eSVinicius Costa Gomes
6092ec50a9d4SVinicius Costa Gomes for (n = 0; n < qopt->num_entries; n++) {
6093a5fd3946SKurt Kanzenbach const struct tc_taprio_sched_entry *e, *prev;
6094ec50a9d4SVinicius Costa Gomes int i;
6095ec50a9d4SVinicius Costa Gomes
6096a5fd3946SKurt Kanzenbach prev = n ? &qopt->entries[n - 1] : NULL;
6097ec50a9d4SVinicius Costa Gomes e = &qopt->entries[n];
6098ec50a9d4SVinicius Costa Gomes
6099ec50a9d4SVinicius Costa Gomes /* i225 only supports "global" frame preemption
6100ec50a9d4SVinicius Costa Gomes * settings.
6101ec50a9d4SVinicius Costa Gomes */
6102ec50a9d4SVinicius Costa Gomes if (e->command != TC_TAPRIO_CMD_SET_GATES)
6103ec50a9d4SVinicius Costa Gomes return false;
6104ec50a9d4SVinicius Costa Gomes
61052b4cc3d3SAKASHI Takahiro for (i = 0; i < adapter->num_tx_queues; i++)
61062b4cc3d3SAKASHI Takahiro if (e->gate_mask & BIT(i)) {
6107ec50a9d4SVinicius Costa Gomes queue_uses[i]++;
6108ec50a9d4SVinicius Costa Gomes
61092b4cc3d3SAKASHI Takahiro /* There are limitations: A single queue cannot
61102b4cc3d3SAKASHI Takahiro * be opened and closed multiple times per cycle
61112b4cc3d3SAKASHI Takahiro * unless the gate stays open. Check for it.
6112a5fd3946SKurt Kanzenbach */
6113a5fd3946SKurt Kanzenbach if (queue_uses[i] > 1 &&
6114a5fd3946SKurt Kanzenbach !(prev->gate_mask & BIT(i)))
6115ec50a9d4SVinicius Costa Gomes return false;
6116ec50a9d4SVinicius Costa Gomes }
6117ec50a9d4SVinicius Costa Gomes }
6118ec50a9d4SVinicius Costa Gomes
6119ec50a9d4SVinicius Costa Gomes return true;
6120ec50a9d4SVinicius Costa Gomes }
6121ec50a9d4SVinicius Costa Gomes
igc_tsn_enable_launchtime(struct igc_adapter * adapter,struct tc_etf_qopt_offload * qopt)612282faa9b7SVinicius Costa Gomes static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
612382faa9b7SVinicius Costa Gomes struct tc_etf_qopt_offload *qopt)
612482faa9b7SVinicius Costa Gomes {
612582faa9b7SVinicius Costa Gomes struct igc_hw *hw = &adapter->hw;
612682faa9b7SVinicius Costa Gomes int err;
612782faa9b7SVinicius Costa Gomes
612882faa9b7SVinicius Costa Gomes if (hw->mac.type != igc_i225)
612982faa9b7SVinicius Costa Gomes return -EOPNOTSUPP;
613082faa9b7SVinicius Costa Gomes
613182faa9b7SVinicius Costa Gomes err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
613282faa9b7SVinicius Costa Gomes if (err)
613382faa9b7SVinicius Costa Gomes return err;
613482faa9b7SVinicius Costa Gomes
61351d1b4c63SMuhammad Husaini Zulkifli return igc_tsn_offload_apply(adapter);
613682faa9b7SVinicius Costa Gomes }
613782faa9b7SVinicius Costa Gomes
igc_qbv_clear_schedule(struct igc_adapter * adapter)613806b41258SMuhammad Husaini Zulkifli static int igc_qbv_clear_schedule(struct igc_adapter *adapter)
6139c814a2d2SVinicius Costa Gomes {
614006b41258SMuhammad Husaini Zulkifli unsigned long flags;
6141c814a2d2SVinicius Costa Gomes int i;
6142c814a2d2SVinicius Costa Gomes
6143c814a2d2SVinicius Costa Gomes adapter->base_time = 0;
6144c814a2d2SVinicius Costa Gomes adapter->cycle_time = NSEC_PER_SEC;
614582ff5f29SFlorian Kauer adapter->taprio_offload_enable = false;
6146ae4fe469SMuhammad Husaini Zulkifli adapter->qbv_config_change_errors = 0;
6147175c2412SMuhammad Husaini Zulkifli adapter->qbv_count = 0;
6148c814a2d2SVinicius Costa Gomes
6149c814a2d2SVinicius Costa Gomes for (i = 0; i < adapter->num_tx_queues; i++) {
6150c814a2d2SVinicius Costa Gomes struct igc_ring *ring = adapter->tx_ring[i];
6151c814a2d2SVinicius Costa Gomes
6152c814a2d2SVinicius Costa Gomes ring->start_time = 0;
6153c814a2d2SVinicius Costa Gomes ring->end_time = NSEC_PER_SEC;
615492a0dcb8STan Tee Min ring->max_sdu = 0;
615506b41258SMuhammad Husaini Zulkifli }
615606b41258SMuhammad Husaini Zulkifli
615706b41258SMuhammad Husaini Zulkifli spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
615806b41258SMuhammad Husaini Zulkifli
615906b41258SMuhammad Husaini Zulkifli adapter->qbv_transition = false;
616006b41258SMuhammad Husaini Zulkifli
616106b41258SMuhammad Husaini Zulkifli for (i = 0; i < adapter->num_tx_queues; i++) {
616206b41258SMuhammad Husaini Zulkifli struct igc_ring *ring = adapter->tx_ring[i];
616306b41258SMuhammad Husaini Zulkifli
6164175c2412SMuhammad Husaini Zulkifli ring->oper_gate_closed = false;
6165175c2412SMuhammad Husaini Zulkifli ring->admin_gate_closed = false;
6166c814a2d2SVinicius Costa Gomes }
6167c814a2d2SVinicius Costa Gomes
616806b41258SMuhammad Husaini Zulkifli spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
616906b41258SMuhammad Husaini Zulkifli
617006b41258SMuhammad Husaini Zulkifli return 0;
617106b41258SMuhammad Husaini Zulkifli }
617206b41258SMuhammad Husaini Zulkifli
igc_tsn_clear_schedule(struct igc_adapter * adapter)617306b41258SMuhammad Husaini Zulkifli static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
617406b41258SMuhammad Husaini Zulkifli {
617506b41258SMuhammad Husaini Zulkifli igc_qbv_clear_schedule(adapter);
617606b41258SMuhammad Husaini Zulkifli
6177c814a2d2SVinicius Costa Gomes return 0;
6178c814a2d2SVinicius Costa Gomes }
6179c814a2d2SVinicius Costa Gomes
igc_taprio_stats(struct net_device * dev,struct tc_taprio_qopt_stats * stats)6180d3750076SMuhammad Husaini Zulkifli static void igc_taprio_stats(struct net_device *dev,
6181d3750076SMuhammad Husaini Zulkifli struct tc_taprio_qopt_stats *stats)
6182d3750076SMuhammad Husaini Zulkifli {
6183d3750076SMuhammad Husaini Zulkifli /* When Strict_End is enabled, the tx_overruns counter
6184d3750076SMuhammad Husaini Zulkifli * will always be zero.
6185d3750076SMuhammad Husaini Zulkifli */
6186d3750076SMuhammad Husaini Zulkifli stats->tx_overruns = 0;
6187d3750076SMuhammad Husaini Zulkifli }
6188d3750076SMuhammad Husaini Zulkifli
igc_taprio_queue_stats(struct net_device * dev,struct tc_taprio_qopt_queue_stats * queue_stats)6189d3750076SMuhammad Husaini Zulkifli static void igc_taprio_queue_stats(struct net_device *dev,
6190d3750076SMuhammad Husaini Zulkifli struct tc_taprio_qopt_queue_stats *queue_stats)
6191d3750076SMuhammad Husaini Zulkifli {
6192d3750076SMuhammad Husaini Zulkifli struct tc_taprio_qopt_stats *stats = &queue_stats->stats;
6193d3750076SMuhammad Husaini Zulkifli
6194d3750076SMuhammad Husaini Zulkifli /* When Strict_End is enabled, the tx_overruns counter
6195d3750076SMuhammad Husaini Zulkifli * will always be zero.
6196d3750076SMuhammad Husaini Zulkifli */
6197d3750076SMuhammad Husaini Zulkifli stats->tx_overruns = 0;
6198d3750076SMuhammad Husaini Zulkifli }
6199d3750076SMuhammad Husaini Zulkifli
igc_save_qbv_schedule(struct igc_adapter * adapter,struct tc_taprio_qopt_offload * qopt)6200ec50a9d4SVinicius Costa Gomes static int igc_save_qbv_schedule(struct igc_adapter *adapter,
6201ec50a9d4SVinicius Costa Gomes struct tc_taprio_qopt_offload *qopt)
6202ec50a9d4SVinicius Costa Gomes {
6203a5fd3946SKurt Kanzenbach bool queue_configured[IGC_MAX_TX_QUEUES] = { };
62045ac1231aSTan Tee Min struct igc_hw *hw = &adapter->hw;
6205ec50a9d4SVinicius Costa Gomes u32 start_time = 0, end_time = 0;
6206175c2412SMuhammad Husaini Zulkifli struct timespec64 now;
620706b41258SMuhammad Husaini Zulkifli unsigned long flags;
6208ec50a9d4SVinicius Costa Gomes size_t n;
620972abeeddSTan Tee Min int i;
6210ec50a9d4SVinicius Costa Gomes
6211d3750076SMuhammad Husaini Zulkifli switch (qopt->cmd) {
6212d3750076SMuhammad Husaini Zulkifli case TAPRIO_CMD_REPLACE:
6213d3750076SMuhammad Husaini Zulkifli break;
6214d3750076SMuhammad Husaini Zulkifli case TAPRIO_CMD_DESTROY:
6215c814a2d2SVinicius Costa Gomes return igc_tsn_clear_schedule(adapter);
6216d3750076SMuhammad Husaini Zulkifli case TAPRIO_CMD_STATS:
6217d3750076SMuhammad Husaini Zulkifli igc_taprio_stats(adapter->netdev, &qopt->stats);
6218d3750076SMuhammad Husaini Zulkifli return 0;
6219d3750076SMuhammad Husaini Zulkifli case TAPRIO_CMD_QUEUE_STATS:
6220d3750076SMuhammad Husaini Zulkifli igc_taprio_queue_stats(adapter->netdev, &qopt->queue_stats);
6221d3750076SMuhammad Husaini Zulkifli return 0;
6222d3750076SMuhammad Husaini Zulkifli default:
622382ff5f29SFlorian Kauer return -EOPNOTSUPP;
6224d3750076SMuhammad Husaini Zulkifli }
622582ff5f29SFlorian Kauer
62263b61764fSMuhammad Husaini Zulkifli if (qopt->base_time < 0)
62273b61764fSMuhammad Husaini Zulkifli return -ERANGE;
62283b61764fSMuhammad Husaini Zulkifli
6229e5d88c53SFlorian Kauer if (igc_is_device_id_i225(hw) && adapter->taprio_offload_enable)
6230ec50a9d4SVinicius Costa Gomes return -EALREADY;
6231ec50a9d4SVinicius Costa Gomes
623258c4ee0eSVinicius Costa Gomes if (!validate_schedule(adapter, qopt))
6233ec50a9d4SVinicius Costa Gomes return -EINVAL;
6234ec50a9d4SVinicius Costa Gomes
6235ec50a9d4SVinicius Costa Gomes adapter->cycle_time = qopt->cycle_time;
6236ec50a9d4SVinicius Costa Gomes adapter->base_time = qopt->base_time;
623782ff5f29SFlorian Kauer adapter->taprio_offload_enable = true;
6238ec50a9d4SVinicius Costa Gomes
6239175c2412SMuhammad Husaini Zulkifli igc_ptp_read(adapter, &now);
6240175c2412SMuhammad Husaini Zulkifli
6241ec50a9d4SVinicius Costa Gomes for (n = 0; n < qopt->num_entries; n++) {
6242ec50a9d4SVinicius Costa Gomes struct tc_taprio_sched_entry *e = &qopt->entries[n];
6243ec50a9d4SVinicius Costa Gomes
6244ec50a9d4SVinicius Costa Gomes end_time += e->interval;
6245ec50a9d4SVinicius Costa Gomes
62466d05251dSTan Tee Min /* If any of the conditions below are true, we need to manually
62476d05251dSTan Tee Min * control the end time of the cycle.
62486d05251dSTan Tee Min * 1. Qbv users can specify a cycle time that is not equal
62496d05251dSTan Tee Min * to the total GCL intervals. Hence, recalculation is
62506d05251dSTan Tee Min * necessary here to exclude the time interval that
62516d05251dSTan Tee Min * exceeds the cycle time.
62526d05251dSTan Tee Min * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
62536d05251dSTan Tee Min * once the end of the list is reached, it will switch
62546d05251dSTan Tee Min * to the END_OF_CYCLE state and leave the gates in the
62556d05251dSTan Tee Min * same state until the next cycle is started.
62566d05251dSTan Tee Min */
62576d05251dSTan Tee Min if (end_time > adapter->cycle_time ||
62586d05251dSTan Tee Min n + 1 == qopt->num_entries)
62596d05251dSTan Tee Min end_time = adapter->cycle_time;
62606d05251dSTan Tee Min
6261691bd4d7SToshiki Nishioka for (i = 0; i < adapter->num_tx_queues; i++) {
6262ec50a9d4SVinicius Costa Gomes struct igc_ring *ring = adapter->tx_ring[i];
6263ec50a9d4SVinicius Costa Gomes
6264ec50a9d4SVinicius Costa Gomes if (!(e->gate_mask & BIT(i)))
6265ec50a9d4SVinicius Costa Gomes continue;
6266ec50a9d4SVinicius Costa Gomes
6267a5fd3946SKurt Kanzenbach /* Check whether a queue stays open for more than one
6268a5fd3946SKurt Kanzenbach * entry. If so, keep the start and advance the end
6269a5fd3946SKurt Kanzenbach * time.
6270a5fd3946SKurt Kanzenbach */
6271a5fd3946SKurt Kanzenbach if (!queue_configured[i])
6272ec50a9d4SVinicius Costa Gomes ring->start_time = start_time;
6273ec50a9d4SVinicius Costa Gomes ring->end_time = end_time;
6274a5fd3946SKurt Kanzenbach
6275175c2412SMuhammad Husaini Zulkifli if (ring->start_time >= adapter->cycle_time)
6276175c2412SMuhammad Husaini Zulkifli queue_configured[i] = false;
6277175c2412SMuhammad Husaini Zulkifli else
6278a5fd3946SKurt Kanzenbach queue_configured[i] = true;
6279ec50a9d4SVinicius Costa Gomes }
6280ec50a9d4SVinicius Costa Gomes
6281ec50a9d4SVinicius Costa Gomes start_time += e->interval;
6282ec50a9d4SVinicius Costa Gomes }
6283ec50a9d4SVinicius Costa Gomes
628406b41258SMuhammad Husaini Zulkifli spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
628506b41258SMuhammad Husaini Zulkifli
628672abeeddSTan Tee Min /* Check whether a queue gets configured.
628772abeeddSTan Tee Min * If not, set the start and end time to be end time.
628872abeeddSTan Tee Min */
628972abeeddSTan Tee Min for (i = 0; i < adapter->num_tx_queues; i++) {
629072abeeddSTan Tee Min struct igc_ring *ring = adapter->tx_ring[i];
629172abeeddSTan Tee Min
6292175c2412SMuhammad Husaini Zulkifli if (!is_base_time_past(qopt->base_time, &now)) {
6293175c2412SMuhammad Husaini Zulkifli ring->admin_gate_closed = false;
6294175c2412SMuhammad Husaini Zulkifli } else {
6295175c2412SMuhammad Husaini Zulkifli ring->oper_gate_closed = false;
6296175c2412SMuhammad Husaini Zulkifli ring->admin_gate_closed = false;
6297175c2412SMuhammad Husaini Zulkifli }
6298175c2412SMuhammad Husaini Zulkifli
6299175c2412SMuhammad Husaini Zulkifli if (!queue_configured[i]) {
6300175c2412SMuhammad Husaini Zulkifli if (!is_base_time_past(qopt->base_time, &now))
6301175c2412SMuhammad Husaini Zulkifli ring->admin_gate_closed = true;
6302175c2412SMuhammad Husaini Zulkifli else
6303175c2412SMuhammad Husaini Zulkifli ring->oper_gate_closed = true;
6304175c2412SMuhammad Husaini Zulkifli
630572abeeddSTan Tee Min ring->start_time = end_time;
630672abeeddSTan Tee Min ring->end_time = end_time;
630772abeeddSTan Tee Min }
630872abeeddSTan Tee Min }
630972abeeddSTan Tee Min
631006b41258SMuhammad Husaini Zulkifli spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
631106b41258SMuhammad Husaini Zulkifli
631292a0dcb8STan Tee Min for (i = 0; i < adapter->num_tx_queues; i++) {
631392a0dcb8STan Tee Min struct igc_ring *ring = adapter->tx_ring[i];
631492a0dcb8STan Tee Min struct net_device *dev = adapter->netdev;
631592a0dcb8STan Tee Min
631692a0dcb8STan Tee Min if (qopt->max_sdu[i])
631725102893STan Tee Min ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len - ETH_TLEN;
631892a0dcb8STan Tee Min else
631992a0dcb8STan Tee Min ring->max_sdu = 0;
632092a0dcb8STan Tee Min }
632192a0dcb8STan Tee Min
6322ec50a9d4SVinicius Costa Gomes return 0;
6323ec50a9d4SVinicius Costa Gomes }
6324ec50a9d4SVinicius Costa Gomes
igc_tsn_enable_qbv_scheduling(struct igc_adapter * adapter,struct tc_taprio_qopt_offload * qopt)6325ec50a9d4SVinicius Costa Gomes static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
6326ec50a9d4SVinicius Costa Gomes struct tc_taprio_qopt_offload *qopt)
6327ec50a9d4SVinicius Costa Gomes {
6328ec50a9d4SVinicius Costa Gomes struct igc_hw *hw = &adapter->hw;
6329ec50a9d4SVinicius Costa Gomes int err;
6330ec50a9d4SVinicius Costa Gomes
6331ec50a9d4SVinicius Costa Gomes if (hw->mac.type != igc_i225)
6332ec50a9d4SVinicius Costa Gomes return -EOPNOTSUPP;
6333ec50a9d4SVinicius Costa Gomes
6334ec50a9d4SVinicius Costa Gomes err = igc_save_qbv_schedule(adapter, qopt);
6335ec50a9d4SVinicius Costa Gomes if (err)
6336ec50a9d4SVinicius Costa Gomes return err;
6337ec50a9d4SVinicius Costa Gomes
63381d1b4c63SMuhammad Husaini Zulkifli return igc_tsn_offload_apply(adapter);
6339ec50a9d4SVinicius Costa Gomes }
6340ec50a9d4SVinicius Costa Gomes
igc_save_cbs_params(struct igc_adapter * adapter,int queue,bool enable,int idleslope,int sendslope,int hicredit,int locredit)63411ab011b0SAravindhan Gunasekaran static int igc_save_cbs_params(struct igc_adapter *adapter, int queue,
63421ab011b0SAravindhan Gunasekaran bool enable, int idleslope, int sendslope,
63431ab011b0SAravindhan Gunasekaran int hicredit, int locredit)
63441ab011b0SAravindhan Gunasekaran {
63451ab011b0SAravindhan Gunasekaran bool cbs_status[IGC_MAX_SR_QUEUES] = { false };
63461ab011b0SAravindhan Gunasekaran struct net_device *netdev = adapter->netdev;
63471ab011b0SAravindhan Gunasekaran struct igc_ring *ring;
63481ab011b0SAravindhan Gunasekaran int i;
63491ab011b0SAravindhan Gunasekaran
63501ab011b0SAravindhan Gunasekaran /* i225 has two sets of credit-based shaper logic.
63511ab011b0SAravindhan Gunasekaran * Supporting it only on the top two priority queues
63521ab011b0SAravindhan Gunasekaran */
63531ab011b0SAravindhan Gunasekaran if (queue < 0 || queue > 1)
63541ab011b0SAravindhan Gunasekaran return -EINVAL;
63551ab011b0SAravindhan Gunasekaran
63561ab011b0SAravindhan Gunasekaran ring = adapter->tx_ring[queue];
63571ab011b0SAravindhan Gunasekaran
63581ab011b0SAravindhan Gunasekaran for (i = 0; i < IGC_MAX_SR_QUEUES; i++)
63591ab011b0SAravindhan Gunasekaran if (adapter->tx_ring[i])
63601ab011b0SAravindhan Gunasekaran cbs_status[i] = adapter->tx_ring[i]->cbs_enable;
63611ab011b0SAravindhan Gunasekaran
63621ab011b0SAravindhan Gunasekaran /* CBS should be enabled on the highest priority queue first in order
63631ab011b0SAravindhan Gunasekaran * for the CBS algorithm to operate as intended.
63641ab011b0SAravindhan Gunasekaran */
63651ab011b0SAravindhan Gunasekaran if (enable) {
63661ab011b0SAravindhan Gunasekaran if (queue == 1 && !cbs_status[0]) {
63671ab011b0SAravindhan Gunasekaran netdev_err(netdev,
63681ab011b0SAravindhan Gunasekaran "Enabling CBS on queue1 before queue0\n");
63691ab011b0SAravindhan Gunasekaran return -EINVAL;
63701ab011b0SAravindhan Gunasekaran }
63711ab011b0SAravindhan Gunasekaran } else {
63721ab011b0SAravindhan Gunasekaran if (queue == 0 && cbs_status[1]) {
63731ab011b0SAravindhan Gunasekaran netdev_err(netdev,
63741ab011b0SAravindhan Gunasekaran "Disabling CBS on queue0 before queue1\n");
63751ab011b0SAravindhan Gunasekaran return -EINVAL;
63761ab011b0SAravindhan Gunasekaran }
63771ab011b0SAravindhan Gunasekaran }
63781ab011b0SAravindhan Gunasekaran
63791ab011b0SAravindhan Gunasekaran ring->cbs_enable = enable;
63801ab011b0SAravindhan Gunasekaran ring->idleslope = idleslope;
63811ab011b0SAravindhan Gunasekaran ring->sendslope = sendslope;
63821ab011b0SAravindhan Gunasekaran ring->hicredit = hicredit;
63831ab011b0SAravindhan Gunasekaran ring->locredit = locredit;
63841ab011b0SAravindhan Gunasekaran
63851ab011b0SAravindhan Gunasekaran return 0;
63861ab011b0SAravindhan Gunasekaran }
63871ab011b0SAravindhan Gunasekaran
igc_tsn_enable_cbs(struct igc_adapter * adapter,struct tc_cbs_qopt_offload * qopt)63881ab011b0SAravindhan Gunasekaran static int igc_tsn_enable_cbs(struct igc_adapter *adapter,
63891ab011b0SAravindhan Gunasekaran struct tc_cbs_qopt_offload *qopt)
63901ab011b0SAravindhan Gunasekaran {
63911ab011b0SAravindhan Gunasekaran struct igc_hw *hw = &adapter->hw;
63921ab011b0SAravindhan Gunasekaran int err;
63931ab011b0SAravindhan Gunasekaran
63941ab011b0SAravindhan Gunasekaran if (hw->mac.type != igc_i225)
63951ab011b0SAravindhan Gunasekaran return -EOPNOTSUPP;
63961ab011b0SAravindhan Gunasekaran
63971ab011b0SAravindhan Gunasekaran if (qopt->queue < 0 || qopt->queue > 1)
63981ab011b0SAravindhan Gunasekaran return -EINVAL;
63991ab011b0SAravindhan Gunasekaran
64001ab011b0SAravindhan Gunasekaran err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable,
64011ab011b0SAravindhan Gunasekaran qopt->idleslope, qopt->sendslope,
64021ab011b0SAravindhan Gunasekaran qopt->hicredit, qopt->locredit);
64031ab011b0SAravindhan Gunasekaran if (err)
64041ab011b0SAravindhan Gunasekaran return err;
64051ab011b0SAravindhan Gunasekaran
64061d1b4c63SMuhammad Husaini Zulkifli return igc_tsn_offload_apply(adapter);
64071ab011b0SAravindhan Gunasekaran }
64081ab011b0SAravindhan Gunasekaran
igc_tc_query_caps(struct igc_adapter * adapter,struct tc_query_caps_base * base)6409522d15eaSVladimir Oltean static int igc_tc_query_caps(struct igc_adapter *adapter,
6410522d15eaSVladimir Oltean struct tc_query_caps_base *base)
6411522d15eaSVladimir Oltean {
6412522d15eaSVladimir Oltean struct igc_hw *hw = &adapter->hw;
6413522d15eaSVladimir Oltean
6414522d15eaSVladimir Oltean switch (base->type) {
6415522d15eaSVladimir Oltean case TC_SETUP_QDISC_TAPRIO: {
6416522d15eaSVladimir Oltean struct tc_taprio_caps *caps = base->caps;
6417522d15eaSVladimir Oltean
64182f530df7SVladimir Oltean caps->broken_mqprio = true;
6419522d15eaSVladimir Oltean
642092a0dcb8STan Tee Min if (hw->mac.type == igc_i225) {
642192a0dcb8STan Tee Min caps->supports_queue_max_sdu = true;
6422522d15eaSVladimir Oltean caps->gate_mask_per_txq = true;
642392a0dcb8STan Tee Min }
6424522d15eaSVladimir Oltean
6425522d15eaSVladimir Oltean return 0;
6426522d15eaSVladimir Oltean }
6427522d15eaSVladimir Oltean default:
6428522d15eaSVladimir Oltean return -EOPNOTSUPP;
6429522d15eaSVladimir Oltean }
6430522d15eaSVladimir Oltean }
6431522d15eaSVladimir Oltean
igc_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)6432ec50a9d4SVinicius Costa Gomes static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
6433ec50a9d4SVinicius Costa Gomes void *type_data)
6434ec50a9d4SVinicius Costa Gomes {
6435ec50a9d4SVinicius Costa Gomes struct igc_adapter *adapter = netdev_priv(dev);
6436ec50a9d4SVinicius Costa Gomes
6437ed89b74dSMuhammad Husaini Zulkifli adapter->tc_setup_type = type;
6438ed89b74dSMuhammad Husaini Zulkifli
6439ec50a9d4SVinicius Costa Gomes switch (type) {
6440522d15eaSVladimir Oltean case TC_QUERY_CAPS:
6441522d15eaSVladimir Oltean return igc_tc_query_caps(adapter, type_data);
6442ec50a9d4SVinicius Costa Gomes case TC_SETUP_QDISC_TAPRIO:
6443ec50a9d4SVinicius Costa Gomes return igc_tsn_enable_qbv_scheduling(adapter, type_data);
6444ec50a9d4SVinicius Costa Gomes
644582faa9b7SVinicius Costa Gomes case TC_SETUP_QDISC_ETF:
644682faa9b7SVinicius Costa Gomes return igc_tsn_enable_launchtime(adapter, type_data);
644782faa9b7SVinicius Costa Gomes
64481ab011b0SAravindhan Gunasekaran case TC_SETUP_QDISC_CBS:
64491ab011b0SAravindhan Gunasekaran return igc_tsn_enable_cbs(adapter, type_data);
64501ab011b0SAravindhan Gunasekaran
6451ec50a9d4SVinicius Costa Gomes default:
6452ec50a9d4SVinicius Costa Gomes return -EOPNOTSUPP;
6453ec50a9d4SVinicius Costa Gomes }
6454ec50a9d4SVinicius Costa Gomes }
6455ec50a9d4SVinicius Costa Gomes
igc_bpf(struct net_device * dev,struct netdev_bpf * bpf)645626575105SAndre Guedes static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
645726575105SAndre Guedes {
645826575105SAndre Guedes struct igc_adapter *adapter = netdev_priv(dev);
645926575105SAndre Guedes
646026575105SAndre Guedes switch (bpf->command) {
646126575105SAndre Guedes case XDP_SETUP_PROG:
646226575105SAndre Guedes return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack);
6463fc9df2a0SAndre Guedes case XDP_SETUP_XSK_POOL:
6464fc9df2a0SAndre Guedes return igc_xdp_setup_pool(adapter, bpf->xsk.pool,
6465fc9df2a0SAndre Guedes bpf->xsk.queue_id);
646626575105SAndre Guedes default:
646726575105SAndre Guedes return -EOPNOTSUPP;
646826575105SAndre Guedes }
646926575105SAndre Guedes }
647026575105SAndre Guedes
igc_xdp_xmit(struct net_device * dev,int num_frames,struct xdp_frame ** frames,u32 flags)64714ff32036SAndre Guedes static int igc_xdp_xmit(struct net_device *dev, int num_frames,
64724ff32036SAndre Guedes struct xdp_frame **frames, u32 flags)
64734ff32036SAndre Guedes {
64744ff32036SAndre Guedes struct igc_adapter *adapter = netdev_priv(dev);
64754ff32036SAndre Guedes int cpu = smp_processor_id();
64764ff32036SAndre Guedes struct netdev_queue *nq;
64774ff32036SAndre Guedes struct igc_ring *ring;
64788df393afSFlorian Kauer int i, nxmit;
64794ff32036SAndre Guedes
6480cb47b1f6SVinicius Costa Gomes if (unlikely(!netif_carrier_ok(dev)))
64814ff32036SAndre Guedes return -ENETDOWN;
64824ff32036SAndre Guedes
64834ff32036SAndre Guedes if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
64844ff32036SAndre Guedes return -EINVAL;
64854ff32036SAndre Guedes
64864ff32036SAndre Guedes ring = igc_xdp_get_tx_ring(adapter, cpu);
64874ff32036SAndre Guedes nq = txring_txq(ring);
64884ff32036SAndre Guedes
64894ff32036SAndre Guedes __netif_tx_lock(nq, cpu);
64904ff32036SAndre Guedes
649195b68148SKurt Kanzenbach /* Avoid transmit queue timeout since we share it with the slow path */
649295b68148SKurt Kanzenbach txq_trans_cond_update(nq);
649395b68148SKurt Kanzenbach
64948df393afSFlorian Kauer nxmit = 0;
64954ff32036SAndre Guedes for (i = 0; i < num_frames; i++) {
64964ff32036SAndre Guedes int err;
64974ff32036SAndre Guedes struct xdp_frame *xdpf = frames[i];
64984ff32036SAndre Guedes
64994ff32036SAndre Guedes err = igc_xdp_init_tx_descriptor(ring, xdpf);
65008df393afSFlorian Kauer if (err)
65018df393afSFlorian Kauer break;
65028df393afSFlorian Kauer nxmit++;
65034ff32036SAndre Guedes }
65044ff32036SAndre Guedes
65054ff32036SAndre Guedes if (flags & XDP_XMIT_FLUSH)
65064ff32036SAndre Guedes igc_flush_tx_descriptors(ring);
65074ff32036SAndre Guedes
65084ff32036SAndre Guedes __netif_tx_unlock(nq);
65094ff32036SAndre Guedes
65108df393afSFlorian Kauer return nxmit;
65114ff32036SAndre Guedes }
65124ff32036SAndre Guedes
igc_trigger_rxtxq_interrupt(struct igc_adapter * adapter,struct igc_q_vector * q_vector)6513fc9df2a0SAndre Guedes static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter,
6514fc9df2a0SAndre Guedes struct igc_q_vector *q_vector)
6515fc9df2a0SAndre Guedes {
6516fc9df2a0SAndre Guedes struct igc_hw *hw = &adapter->hw;
6517fc9df2a0SAndre Guedes u32 eics = 0;
6518fc9df2a0SAndre Guedes
6519fc9df2a0SAndre Guedes eics |= q_vector->eims_value;
6520fc9df2a0SAndre Guedes wr32(IGC_EICS, eics);
6521fc9df2a0SAndre Guedes }
6522fc9df2a0SAndre Guedes
igc_xsk_wakeup(struct net_device * dev,u32 queue_id,u32 flags)6523fc9df2a0SAndre Guedes int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
6524fc9df2a0SAndre Guedes {
6525fc9df2a0SAndre Guedes struct igc_adapter *adapter = netdev_priv(dev);
6526fc9df2a0SAndre Guedes struct igc_q_vector *q_vector;
6527fc9df2a0SAndre Guedes struct igc_ring *ring;
6528fc9df2a0SAndre Guedes
6529fc9df2a0SAndre Guedes if (test_bit(__IGC_DOWN, &adapter->state))
6530fc9df2a0SAndre Guedes return -ENETDOWN;
6531fc9df2a0SAndre Guedes
6532fc9df2a0SAndre Guedes if (!igc_xdp_is_enabled(adapter))
6533fc9df2a0SAndre Guedes return -ENXIO;
6534fc9df2a0SAndre Guedes
6535fc9df2a0SAndre Guedes if (queue_id >= adapter->num_rx_queues)
6536fc9df2a0SAndre Guedes return -EINVAL;
6537fc9df2a0SAndre Guedes
6538fc9df2a0SAndre Guedes ring = adapter->rx_ring[queue_id];
6539fc9df2a0SAndre Guedes
6540fc9df2a0SAndre Guedes if (!ring->xsk_pool)
6541fc9df2a0SAndre Guedes return -ENXIO;
6542fc9df2a0SAndre Guedes
6543fc9df2a0SAndre Guedes q_vector = adapter->q_vector[queue_id];
6544fc9df2a0SAndre Guedes if (!napi_if_scheduled_mark_missed(&q_vector->napi))
6545fc9df2a0SAndre Guedes igc_trigger_rxtxq_interrupt(adapter, q_vector);
6546fc9df2a0SAndre Guedes
6547fc9df2a0SAndre Guedes return 0;
6548fc9df2a0SAndre Guedes }
6549fc9df2a0SAndre Guedes
6550c9a11c23SSasha Neftin static const struct net_device_ops igc_netdev_ops = {
6551c9a11c23SSasha Neftin .ndo_open = igc_open,
6552c9a11c23SSasha Neftin .ndo_stop = igc_close,
6553c9a11c23SSasha Neftin .ndo_start_xmit = igc_xmit_frame,
65547f839684SSasha Neftin .ndo_set_rx_mode = igc_set_rx_mode,
6555c9a11c23SSasha Neftin .ndo_set_mac_address = igc_set_mac,
6556c9a11c23SSasha Neftin .ndo_change_mtu = igc_change_mtu,
65579b275176SSasha Neftin .ndo_tx_timeout = igc_tx_timeout,
65586b7ed22aSVinicius Costa Gomes .ndo_get_stats64 = igc_get_stats64,
655965cd3a72SSasha Neftin .ndo_fix_features = igc_fix_features,
656065cd3a72SSasha Neftin .ndo_set_features = igc_set_features,
656165cd3a72SSasha Neftin .ndo_features_check = igc_features_check,
6562a7605370SArnd Bergmann .ndo_eth_ioctl = igc_ioctl,
6563ec50a9d4SVinicius Costa Gomes .ndo_setup_tc = igc_setup_tc,
656426575105SAndre Guedes .ndo_bpf = igc_bpf,
65654ff32036SAndre Guedes .ndo_xdp_xmit = igc_xdp_xmit,
6566fc9df2a0SAndre Guedes .ndo_xsk_wakeup = igc_xsk_wakeup,
6567c9a11c23SSasha Neftin };
6568146740f9SSasha Neftin
6569146740f9SSasha Neftin /* PCIe configuration access */
igc_read_pci_cfg(struct igc_hw * hw,u32 reg,u16 * value)6570146740f9SSasha Neftin void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6571146740f9SSasha Neftin {
6572146740f9SSasha Neftin struct igc_adapter *adapter = hw->back;
6573146740f9SSasha Neftin
6574146740f9SSasha Neftin pci_read_config_word(adapter->pdev, reg, value);
6575146740f9SSasha Neftin }
6576146740f9SSasha Neftin
igc_write_pci_cfg(struct igc_hw * hw,u32 reg,u16 * value)6577146740f9SSasha Neftin void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6578146740f9SSasha Neftin {
6579146740f9SSasha Neftin struct igc_adapter *adapter = hw->back;
6580146740f9SSasha Neftin
6581146740f9SSasha Neftin pci_write_config_word(adapter->pdev, reg, *value);
6582146740f9SSasha Neftin }
6583146740f9SSasha Neftin
igc_read_pcie_cap_reg(struct igc_hw * hw,u32 reg,u16 * value)6584146740f9SSasha Neftin s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6585146740f9SSasha Neftin {
6586146740f9SSasha Neftin struct igc_adapter *adapter = hw->back;
6587146740f9SSasha Neftin
6588a16f6d3aSFrederick Lawler if (!pci_is_pcie(adapter->pdev))
6589146740f9SSasha Neftin return -IGC_ERR_CONFIG;
6590146740f9SSasha Neftin
6591a16f6d3aSFrederick Lawler pcie_capability_read_word(adapter->pdev, reg, value);
6592146740f9SSasha Neftin
6593146740f9SSasha Neftin return IGC_SUCCESS;
6594146740f9SSasha Neftin }
6595146740f9SSasha Neftin
igc_write_pcie_cap_reg(struct igc_hw * hw,u32 reg,u16 * value)6596146740f9SSasha Neftin s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6597146740f9SSasha Neftin {
6598146740f9SSasha Neftin struct igc_adapter *adapter = hw->back;
6599146740f9SSasha Neftin
6600a16f6d3aSFrederick Lawler if (!pci_is_pcie(adapter->pdev))
6601146740f9SSasha Neftin return -IGC_ERR_CONFIG;
6602146740f9SSasha Neftin
6603a16f6d3aSFrederick Lawler pcie_capability_write_word(adapter->pdev, reg, *value);
6604146740f9SSasha Neftin
6605146740f9SSasha Neftin return IGC_SUCCESS;
6606146740f9SSasha Neftin }
6607146740f9SSasha Neftin
igc_rd32(struct igc_hw * hw,u32 reg)6608146740f9SSasha Neftin u32 igc_rd32(struct igc_hw *hw, u32 reg)
6609146740f9SSasha Neftin {
6610c9a11c23SSasha Neftin struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
6611146740f9SSasha Neftin u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
6612146740f9SSasha Neftin u32 value = 0;
6613146740f9SSasha Neftin
66147c1ddceeSLennert Buytenhek if (IGC_REMOVED(hw_addr))
66157c1ddceeSLennert Buytenhek return ~value;
66167c1ddceeSLennert Buytenhek
6617146740f9SSasha Neftin value = readl(&hw_addr[reg]);
6618146740f9SSasha Neftin
6619146740f9SSasha Neftin /* reads should not return all F's */
6620c9a11c23SSasha Neftin if (!(~value) && (!reg || !(~readl(hw_addr)))) {
6621c9a11c23SSasha Neftin struct net_device *netdev = igc->netdev;
6622c9a11c23SSasha Neftin
6623146740f9SSasha Neftin hw->hw_addr = NULL;
6624c9a11c23SSasha Neftin netif_device_detach(netdev);
6625c9a11c23SSasha Neftin netdev_err(netdev, "PCIe link lost, device now detached\n");
662694bc1e52SLyude Paul WARN(pci_device_is_present(igc->pdev),
662794bc1e52SLyude Paul "igc: Failed to read reg 0x%x!\n", reg);
6628c9a11c23SSasha Neftin }
6629146740f9SSasha Neftin
6630146740f9SSasha Neftin return value;
6631146740f9SSasha Neftin }
6632146740f9SSasha Neftin
66338416814fSJesper Dangaard Brouer /* Mapping HW RSS Type to enum xdp_rss_hash_type */
66348416814fSJesper Dangaard Brouer static enum xdp_rss_hash_type igc_xdp_rss_type[IGC_RSS_TYPE_MAX_TABLE] = {
66358416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_NO_HASH] = XDP_RSS_TYPE_L2,
66368416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_TCP_IPV4] = XDP_RSS_TYPE_L4_IPV4_TCP,
66378416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_IPV4] = XDP_RSS_TYPE_L3_IPV4,
66388416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_TCP_IPV6] = XDP_RSS_TYPE_L4_IPV6_TCP,
66398416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_IPV6_EX] = XDP_RSS_TYPE_L3_IPV6_EX,
66408416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_IPV6] = XDP_RSS_TYPE_L3_IPV6,
66418416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX,
66428416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_UDP_IPV4] = XDP_RSS_TYPE_L4_IPV4_UDP,
66438416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_UDP_IPV6] = XDP_RSS_TYPE_L4_IPV6_UDP,
66448416814fSJesper Dangaard Brouer [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX,
66458416814fSJesper Dangaard Brouer [10] = XDP_RSS_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */
66468416814fSJesper Dangaard Brouer [11] = XDP_RSS_TYPE_NONE, /* keep array sized for SW bit-mask */
66478416814fSJesper Dangaard Brouer [12] = XDP_RSS_TYPE_NONE, /* to handle future HW revisons */
66488416814fSJesper Dangaard Brouer [13] = XDP_RSS_TYPE_NONE,
66498416814fSJesper Dangaard Brouer [14] = XDP_RSS_TYPE_NONE,
66508416814fSJesper Dangaard Brouer [15] = XDP_RSS_TYPE_NONE,
66518416814fSJesper Dangaard Brouer };
66528416814fSJesper Dangaard Brouer
igc_xdp_rx_hash(const struct xdp_md * _ctx,u32 * hash,enum xdp_rss_hash_type * rss_type)66538416814fSJesper Dangaard Brouer static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
66548416814fSJesper Dangaard Brouer enum xdp_rss_hash_type *rss_type)
66558416814fSJesper Dangaard Brouer {
66568416814fSJesper Dangaard Brouer const struct igc_xdp_buff *ctx = (void *)_ctx;
66578416814fSJesper Dangaard Brouer
66588416814fSJesper Dangaard Brouer if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH))
66598416814fSJesper Dangaard Brouer return -ENODATA;
66608416814fSJesper Dangaard Brouer
66618416814fSJesper Dangaard Brouer *hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss);
66628416814fSJesper Dangaard Brouer *rss_type = igc_xdp_rss_type[igc_rss_type(ctx->rx_desc)];
66638416814fSJesper Dangaard Brouer
66648416814fSJesper Dangaard Brouer return 0;
66658416814fSJesper Dangaard Brouer }
66668416814fSJesper Dangaard Brouer
igc_xdp_rx_timestamp(const struct xdp_md * _ctx,u64 * timestamp)6667d6772667SJesper Dangaard Brouer static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
6668d6772667SJesper Dangaard Brouer {
6669d6772667SJesper Dangaard Brouer const struct igc_xdp_buff *ctx = (void *)_ctx;
6670d6772667SJesper Dangaard Brouer
6671d6772667SJesper Dangaard Brouer if (igc_test_staterr(ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) {
6672d6772667SJesper Dangaard Brouer *timestamp = ctx->rx_ts;
6673d6772667SJesper Dangaard Brouer
6674d6772667SJesper Dangaard Brouer return 0;
6675d6772667SJesper Dangaard Brouer }
6676d6772667SJesper Dangaard Brouer
6677d6772667SJesper Dangaard Brouer return -ENODATA;
6678d6772667SJesper Dangaard Brouer }
6679d6772667SJesper Dangaard Brouer
66808416814fSJesper Dangaard Brouer static const struct xdp_metadata_ops igc_xdp_metadata_ops = {
66818416814fSJesper Dangaard Brouer .xmo_rx_hash = igc_xdp_rx_hash,
6682d6772667SJesper Dangaard Brouer .xmo_rx_timestamp = igc_xdp_rx_timestamp,
66838416814fSJesper Dangaard Brouer };
66848416814fSJesper Dangaard Brouer
igc_qbv_scheduling_timer(struct hrtimer * timer)6685175c2412SMuhammad Husaini Zulkifli static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer)
6686175c2412SMuhammad Husaini Zulkifli {
6687175c2412SMuhammad Husaini Zulkifli struct igc_adapter *adapter = container_of(timer, struct igc_adapter,
6688175c2412SMuhammad Husaini Zulkifli hrtimer);
668906b41258SMuhammad Husaini Zulkifli unsigned long flags;
6690175c2412SMuhammad Husaini Zulkifli unsigned int i;
6691175c2412SMuhammad Husaini Zulkifli
669206b41258SMuhammad Husaini Zulkifli spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
669306b41258SMuhammad Husaini Zulkifli
6694175c2412SMuhammad Husaini Zulkifli adapter->qbv_transition = true;
6695175c2412SMuhammad Husaini Zulkifli for (i = 0; i < adapter->num_tx_queues; i++) {
6696175c2412SMuhammad Husaini Zulkifli struct igc_ring *tx_ring = adapter->tx_ring[i];
6697175c2412SMuhammad Husaini Zulkifli
6698175c2412SMuhammad Husaini Zulkifli if (tx_ring->admin_gate_closed) {
6699175c2412SMuhammad Husaini Zulkifli tx_ring->admin_gate_closed = false;
6700175c2412SMuhammad Husaini Zulkifli tx_ring->oper_gate_closed = true;
6701175c2412SMuhammad Husaini Zulkifli } else {
6702175c2412SMuhammad Husaini Zulkifli tx_ring->oper_gate_closed = false;
6703175c2412SMuhammad Husaini Zulkifli }
6704175c2412SMuhammad Husaini Zulkifli }
6705175c2412SMuhammad Husaini Zulkifli adapter->qbv_transition = false;
670606b41258SMuhammad Husaini Zulkifli
670706b41258SMuhammad Husaini Zulkifli spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
670806b41258SMuhammad Husaini Zulkifli
6709175c2412SMuhammad Husaini Zulkifli return HRTIMER_NORESTART;
6710175c2412SMuhammad Husaini Zulkifli }
6711175c2412SMuhammad Husaini Zulkifli
6712d89f8841SSasha Neftin /**
6713d89f8841SSasha Neftin * igc_probe - Device Initialization Routine
6714d89f8841SSasha Neftin * @pdev: PCI device information struct
6715d89f8841SSasha Neftin * @ent: entry in igc_pci_tbl
6716d89f8841SSasha Neftin *
6717d89f8841SSasha Neftin * Returns 0 on success, negative on failure
6718d89f8841SSasha Neftin *
6719d89f8841SSasha Neftin * igc_probe initializes an adapter identified by a pci_dev structure.
6720d89f8841SSasha Neftin * The OS initialization, configuring the adapter private structure,
6721d89f8841SSasha Neftin * and a hardware reset occur.
6722d89f8841SSasha Neftin */
igc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)6723d89f8841SSasha Neftin static int igc_probe(struct pci_dev *pdev,
6724d89f8841SSasha Neftin const struct pci_device_id *ent)
6725d89f8841SSasha Neftin {
6726146740f9SSasha Neftin struct igc_adapter *adapter;
6727c9a11c23SSasha Neftin struct net_device *netdev;
6728c9a11c23SSasha Neftin struct igc_hw *hw;
6729ab405612SSasha Neftin const struct igc_info *ei = igc_info_tbl[ent->driver_data];
6730fea89930SChristophe JAILLET int err;
6731d89f8841SSasha Neftin
6732d89f8841SSasha Neftin err = pci_enable_device_mem(pdev);
6733d89f8841SSasha Neftin if (err)
6734d89f8841SSasha Neftin return err;
6735d89f8841SSasha Neftin
673621da01fdSSasha Neftin err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
6737d89f8841SSasha Neftin if (err) {
673821da01fdSSasha Neftin dev_err(&pdev->dev,
673921da01fdSSasha Neftin "No usable DMA configuration, aborting\n");
6740d89f8841SSasha Neftin goto err_dma;
6741d89f8841SSasha Neftin }
6742d89f8841SSasha Neftin
674321da01fdSSasha Neftin err = pci_request_mem_regions(pdev, igc_driver_name);
6744d89f8841SSasha Neftin if (err)
6745d89f8841SSasha Neftin goto err_pci_reg;
6746d89f8841SSasha Neftin
67471b5d73fbSVinicius Costa Gomes err = pci_enable_ptm(pdev, NULL);
67481b5d73fbSVinicius Costa Gomes if (err < 0)
67491b5d73fbSVinicius Costa Gomes dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
67501b5d73fbSVinicius Costa Gomes
6751d89f8841SSasha Neftin pci_set_master(pdev);
6752c9a11c23SSasha Neftin
6753c9a11c23SSasha Neftin err = -ENOMEM;
6754c9a11c23SSasha Neftin netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
6755c9a11c23SSasha Neftin IGC_MAX_TX_QUEUES);
6756c9a11c23SSasha Neftin
6757c9a11c23SSasha Neftin if (!netdev)
6758c9a11c23SSasha Neftin goto err_alloc_etherdev;
6759c9a11c23SSasha Neftin
6760c9a11c23SSasha Neftin SET_NETDEV_DEV(netdev, &pdev->dev);
6761c9a11c23SSasha Neftin
6762c9a11c23SSasha Neftin pci_set_drvdata(pdev, netdev);
6763c9a11c23SSasha Neftin adapter = netdev_priv(netdev);
6764c9a11c23SSasha Neftin adapter->netdev = netdev;
6765c9a11c23SSasha Neftin adapter->pdev = pdev;
6766c9a11c23SSasha Neftin hw = &adapter->hw;
6767c9a11c23SSasha Neftin hw->back = adapter;
6768c9a11c23SSasha Neftin adapter->port_num = hw->bus.func;
67698c5ad0daSSasha Neftin adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6770c9a11c23SSasha Neftin
6771d89f8841SSasha Neftin err = pci_save_state(pdev);
6772c9a11c23SSasha Neftin if (err)
6773c9a11c23SSasha Neftin goto err_ioremap;
6774c9a11c23SSasha Neftin
6775c9a11c23SSasha Neftin err = -EIO;
6776c9a11c23SSasha Neftin adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
6777c9a11c23SSasha Neftin pci_resource_len(pdev, 0));
6778c9a11c23SSasha Neftin if (!adapter->io_addr)
6779c9a11c23SSasha Neftin goto err_ioremap;
6780c9a11c23SSasha Neftin
6781c9a11c23SSasha Neftin /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
6782c9a11c23SSasha Neftin hw->hw_addr = adapter->io_addr;
6783c9a11c23SSasha Neftin
6784c9a11c23SSasha Neftin netdev->netdev_ops = &igc_netdev_ops;
67858416814fSJesper Dangaard Brouer netdev->xdp_metadata_ops = &igc_xdp_metadata_ops;
67867df76bd1SAndre Guedes igc_ethtool_set_ops(netdev);
6787c9a11c23SSasha Neftin netdev->watchdog_timeo = 5 * HZ;
6788c9a11c23SSasha Neftin
6789c9a11c23SSasha Neftin netdev->mem_start = pci_resource_start(pdev, 0);
6790c9a11c23SSasha Neftin netdev->mem_end = pci_resource_end(pdev, 0);
6791c9a11c23SSasha Neftin
6792c9a11c23SSasha Neftin /* PCI config space info */
6793c9a11c23SSasha Neftin hw->vendor_id = pdev->vendor;
6794c9a11c23SSasha Neftin hw->device_id = pdev->device;
6795c9a11c23SSasha Neftin hw->revision_id = pdev->revision;
6796c9a11c23SSasha Neftin hw->subsystem_vendor_id = pdev->subsystem_vendor;
6797c9a11c23SSasha Neftin hw->subsystem_device_id = pdev->subsystem_device;
6798146740f9SSasha Neftin
6799ab405612SSasha Neftin /* Copy the default MAC and PHY function pointers */
6800ab405612SSasha Neftin memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
68015586838fSSasha Neftin memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6802ab405612SSasha Neftin
6803ab405612SSasha Neftin /* Initialize skew-specific constants */
6804ab405612SSasha Neftin err = ei->get_invariants(hw);
6805ab405612SSasha Neftin if (err)
6806ab405612SSasha Neftin goto err_sw_init;
6807ab405612SSasha Neftin
6808d3ae3cfbSSasha Neftin /* Add supported features to the features list*/
6809b7b46245SSasha Neftin netdev->features |= NETIF_F_SG;
6810f38b782dSSasha Neftin netdev->features |= NETIF_F_TSO;
6811f38b782dSSasha Neftin netdev->features |= NETIF_F_TSO6;
68128e8204a4SSasha Neftin netdev->features |= NETIF_F_TSO_ECN;
681384214ab4SJesper Dangaard Brouer netdev->features |= NETIF_F_RXHASH;
68143bdd7086SSasha Neftin netdev->features |= NETIF_F_RXCSUM;
6815d3ae3cfbSSasha Neftin netdev->features |= NETIF_F_HW_CSUM;
68160ac960a8SSasha Neftin netdev->features |= NETIF_F_SCTP_CRC;
6817635071e2SSasha Neftin netdev->features |= NETIF_F_HW_TC;
6818d3ae3cfbSSasha Neftin
681934428dffSSasha Neftin #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
682034428dffSSasha Neftin NETIF_F_GSO_GRE_CSUM | \
682134428dffSSasha Neftin NETIF_F_GSO_IPXIP4 | \
682234428dffSSasha Neftin NETIF_F_GSO_IPXIP6 | \
682334428dffSSasha Neftin NETIF_F_GSO_UDP_TUNNEL | \
682434428dffSSasha Neftin NETIF_F_GSO_UDP_TUNNEL_CSUM)
682534428dffSSasha Neftin
682634428dffSSasha Neftin netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
682734428dffSSasha Neftin netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
6828146740f9SSasha Neftin
6829146740f9SSasha Neftin /* setup the private structure */
6830146740f9SSasha Neftin err = igc_sw_init(adapter);
6831146740f9SSasha Neftin if (err)
6832146740f9SSasha Neftin goto err_sw_init;
6833146740f9SSasha Neftin
683465cd3a72SSasha Neftin /* copy netdev features into list of user selectable features */
683565cd3a72SSasha Neftin netdev->hw_features |= NETIF_F_NTUPLE;
68368d744963SMuhammad Husaini Zulkifli netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
68378d744963SMuhammad Husaini Zulkifli netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
6838d3ae3cfbSSasha Neftin netdev->hw_features |= netdev->features;
683965cd3a72SSasha Neftin
68404439dc42SSasha Neftin netdev->features |= NETIF_F_HIGHDMA;
68414439dc42SSasha Neftin
684240ee363cSPaolo Abeni netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
684340ee363cSPaolo Abeni netdev->mpls_features |= NETIF_F_HW_CSUM;
684440ee363cSPaolo Abeni netdev->hw_enc_features |= netdev->vlan_features;
68458d744963SMuhammad Husaini Zulkifli
684666c0e13aSMarek Majtyka netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
684766c0e13aSMarek Majtyka NETDEV_XDP_ACT_XSK_ZEROCOPY;
684866c0e13aSMarek Majtyka
6849c9a11c23SSasha Neftin /* MTU range: 68 - 9216 */
6850c9a11c23SSasha Neftin netdev->min_mtu = ETH_MIN_MTU;
6851c9a11c23SSasha Neftin netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
6852c9a11c23SSasha Neftin
68534eb80801SSasha Neftin /* before reading the NVM, reset the controller to put the device in a
68544eb80801SSasha Neftin * known good starting state
68554eb80801SSasha Neftin */
68564eb80801SSasha Neftin hw->mac.ops.reset_hw(hw);
68574eb80801SSasha Neftin
68589b924eddSSasha Neftin if (igc_get_flash_presence_i225(hw)) {
68599b924eddSSasha Neftin if (hw->nvm.ops.validate(hw) < 0) {
686025f06effSAndre Guedes dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
68619b924eddSSasha Neftin err = -EIO;
68629b924eddSSasha Neftin goto err_eeprom;
68639b924eddSSasha Neftin }
68649b924eddSSasha Neftin }
68659b924eddSSasha Neftin
68664eb80801SSasha Neftin if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
68674eb80801SSasha Neftin /* copy the MAC address out of the NVM */
68684eb80801SSasha Neftin if (hw->mac.ops.read_mac_addr(hw))
68694eb80801SSasha Neftin dev_err(&pdev->dev, "NVM Read Error\n");
68704eb80801SSasha Neftin }
68714eb80801SSasha Neftin
6872a05e4c0aSJakub Kicinski eth_hw_addr_set(netdev, hw->mac.addr);
68734eb80801SSasha Neftin
68744eb80801SSasha Neftin if (!is_valid_ether_addr(netdev->dev_addr)) {
68754eb80801SSasha Neftin dev_err(&pdev->dev, "Invalid MAC Address\n");
68764eb80801SSasha Neftin err = -EIO;
68774eb80801SSasha Neftin goto err_eeprom;
68784eb80801SSasha Neftin }
68794eb80801SSasha Neftin
68800507ef8aSSasha Neftin /* configure RXPBSIZE and TXPBSIZE */
68810507ef8aSSasha Neftin wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
68820507ef8aSSasha Neftin wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
68830507ef8aSSasha Neftin
68840507ef8aSSasha Neftin timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
6885208983f0SSasha Neftin timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
68860507ef8aSSasha Neftin
68870507ef8aSSasha Neftin INIT_WORK(&adapter->reset_task, igc_reset_task);
6888208983f0SSasha Neftin INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
68890507ef8aSSasha Neftin
6890175c2412SMuhammad Husaini Zulkifli hrtimer_init(&adapter->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6891175c2412SMuhammad Husaini Zulkifli adapter->hrtimer.function = &igc_qbv_scheduling_timer;
6892175c2412SMuhammad Husaini Zulkifli
68934eb80801SSasha Neftin /* Initialize link properties that are user-changeable */
68944eb80801SSasha Neftin adapter->fc_autoneg = true;
68954eb80801SSasha Neftin hw->mac.autoneg = true;
68964eb80801SSasha Neftin hw->phy.autoneg_advertised = 0xaf;
68974eb80801SSasha Neftin
68984eb80801SSasha Neftin hw->fc.requested_mode = igc_fc_default;
68994eb80801SSasha Neftin hw->fc.current_mode = igc_fc_default;
69004eb80801SSasha Neftin
6901e055600dSSasha Neftin /* By default, support wake on port A */
6902e055600dSSasha Neftin adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
6903e055600dSSasha Neftin
6904e055600dSSasha Neftin /* initialize the wol settings based on the eeprom settings */
6905e055600dSSasha Neftin if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
6906e055600dSSasha Neftin adapter->wol |= IGC_WUFC_MAG;
6907e055600dSSasha Neftin
6908e055600dSSasha Neftin device_set_wakeup_enable(&adapter->pdev->dev,
6909e055600dSSasha Neftin adapter->flags & IGC_FLAG_WOL_SUPPORTED);
6910e055600dSSasha Neftin
69113cda505aSVinicius Costa Gomes igc_ptp_init(adapter);
69123cda505aSVinicius Costa Gomes
6913c814a2d2SVinicius Costa Gomes igc_tsn_clear_schedule(adapter);
6914c814a2d2SVinicius Costa Gomes
6915c9a11c23SSasha Neftin /* reset the hardware with the new settings */
6916c9a11c23SSasha Neftin igc_reset(adapter);
6917c9a11c23SSasha Neftin
6918c9a11c23SSasha Neftin /* let the f/w know that the h/w is now under the control of the
6919c9a11c23SSasha Neftin * driver.
6920c9a11c23SSasha Neftin */
6921c9a11c23SSasha Neftin igc_get_hw_control(adapter);
6922c9a11c23SSasha Neftin
6923c9a11c23SSasha Neftin strncpy(netdev->name, "eth%d", IFNAMSIZ);
6924c9a11c23SSasha Neftin err = register_netdev(netdev);
6925c9a11c23SSasha Neftin if (err)
6926c9a11c23SSasha Neftin goto err_register;
6927c9a11c23SSasha Neftin
6928c9a11c23SSasha Neftin /* carrier off reporting is important to ethtool even BEFORE open */
6929c9a11c23SSasha Neftin netif_carrier_off(netdev);
6930c9a11c23SSasha Neftin
6931ab405612SSasha Neftin /* Check if Media Autosense is enabled */
6932ab405612SSasha Neftin adapter->ei = *ei;
6933ab405612SSasha Neftin
6934c9a11c23SSasha Neftin /* print pcie link status and MAC address */
6935c9a11c23SSasha Neftin pcie_print_link_status(pdev);
6936c9a11c23SSasha Neftin netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
6937c9a11c23SSasha Neftin
6938e0751556SRafael J. Wysocki dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
693993ec439aSSasha Neftin /* Disable EEE for internal PHY devices */
694093ec439aSSasha Neftin hw->dev_spec._base.eee_enable = false;
694193ec439aSSasha Neftin adapter->flags &= ~IGC_FLAG_EEE;
694293ec439aSSasha Neftin igc_set_eee_i225(hw, false, false, false);
69438594a7f3SSasha Neftin
69448594a7f3SSasha Neftin pm_runtime_put_noidle(&pdev->dev);
69458594a7f3SSasha Neftin
6946d89f8841SSasha Neftin return 0;
6947d89f8841SSasha Neftin
6948c9a11c23SSasha Neftin err_register:
6949c9a11c23SSasha Neftin igc_release_hw_control(adapter);
69504eb80801SSasha Neftin err_eeprom:
69514eb80801SSasha Neftin if (!igc_check_reset_block(hw))
69524eb80801SSasha Neftin igc_reset_phy(hw);
6953146740f9SSasha Neftin err_sw_init:
69543df25e4cSSasha Neftin igc_clear_interrupt_scheme(adapter);
69553df25e4cSSasha Neftin iounmap(adapter->io_addr);
6956c9a11c23SSasha Neftin err_ioremap:
6957c9a11c23SSasha Neftin free_netdev(netdev);
6958c9a11c23SSasha Neftin err_alloc_etherdev:
6959faf4dd52SSasha Neftin pci_release_mem_regions(pdev);
6960d89f8841SSasha Neftin err_pci_reg:
6961d89f8841SSasha Neftin err_dma:
6962d89f8841SSasha Neftin pci_disable_device(pdev);
6963d89f8841SSasha Neftin return err;
6964d89f8841SSasha Neftin }
6965d89f8841SSasha Neftin
6966d89f8841SSasha Neftin /**
6967d89f8841SSasha Neftin * igc_remove - Device Removal Routine
6968d89f8841SSasha Neftin * @pdev: PCI device information struct
6969d89f8841SSasha Neftin *
6970d89f8841SSasha Neftin * igc_remove is called by the PCI subsystem to alert the driver
6971d89f8841SSasha Neftin * that it should release a PCI device. This could be caused by a
6972d89f8841SSasha Neftin * Hot-Plug event, or because the driver is going to be removed from
6973d89f8841SSasha Neftin * memory.
6974d89f8841SSasha Neftin */
igc_remove(struct pci_dev * pdev)6975d89f8841SSasha Neftin static void igc_remove(struct pci_dev *pdev)
6976d89f8841SSasha Neftin {
6977c9a11c23SSasha Neftin struct net_device *netdev = pci_get_drvdata(pdev);
6978c9a11c23SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
6979c9a11c23SSasha Neftin
69808594a7f3SSasha Neftin pm_runtime_get_noresume(&pdev->dev);
69818594a7f3SSasha Neftin
6982e256ec83SAndre Guedes igc_flush_nfc_rules(adapter);
6983e256ec83SAndre Guedes
69845f295805SVinicius Costa Gomes igc_ptp_stop(adapter);
69855f295805SVinicius Costa Gomes
6986c080fe26SVinicius Costa Gomes pci_disable_ptm(pdev);
6987c080fe26SVinicius Costa Gomes pci_clear_master(pdev);
6988c080fe26SVinicius Costa Gomes
6989c9a11c23SSasha Neftin set_bit(__IGC_DOWN, &adapter->state);
69900507ef8aSSasha Neftin
69910507ef8aSSasha Neftin del_timer_sync(&adapter->watchdog_timer);
6992208983f0SSasha Neftin del_timer_sync(&adapter->phy_info_timer);
69930507ef8aSSasha Neftin
69940507ef8aSSasha Neftin cancel_work_sync(&adapter->reset_task);
6995208983f0SSasha Neftin cancel_work_sync(&adapter->watchdog_task);
6996175c2412SMuhammad Husaini Zulkifli hrtimer_cancel(&adapter->hrtimer);
6997c9a11c23SSasha Neftin
6998c9a11c23SSasha Neftin /* Release control of h/w to f/w. If f/w is AMT enabled, this
6999c9a11c23SSasha Neftin * would have already happened in close and is redundant.
7000c9a11c23SSasha Neftin */
7001c9a11c23SSasha Neftin igc_release_hw_control(adapter);
7002c9a11c23SSasha Neftin unregister_netdev(netdev);
7003c9a11c23SSasha Neftin
70040507ef8aSSasha Neftin igc_clear_interrupt_scheme(adapter);
70050507ef8aSSasha Neftin pci_iounmap(pdev, adapter->io_addr);
70060507ef8aSSasha Neftin pci_release_mem_regions(pdev);
7007d89f8841SSasha Neftin
7008c9a11c23SSasha Neftin free_netdev(netdev);
70090507ef8aSSasha Neftin
7010d89f8841SSasha Neftin pci_disable_device(pdev);
7011d89f8841SSasha Neftin }
7012d89f8841SSasha Neftin
__igc_shutdown(struct pci_dev * pdev,bool * enable_wake,bool runtime)70139513d2a5SSasha Neftin static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
70149513d2a5SSasha Neftin bool runtime)
70159513d2a5SSasha Neftin {
70169513d2a5SSasha Neftin struct net_device *netdev = pci_get_drvdata(pdev);
70179513d2a5SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
70189513d2a5SSasha Neftin u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
70199513d2a5SSasha Neftin struct igc_hw *hw = &adapter->hw;
70209513d2a5SSasha Neftin u32 ctrl, rctl, status;
70219513d2a5SSasha Neftin bool wake;
70229513d2a5SSasha Neftin
70239513d2a5SSasha Neftin rtnl_lock();
70249513d2a5SSasha Neftin netif_device_detach(netdev);
70259513d2a5SSasha Neftin
70269513d2a5SSasha Neftin if (netif_running(netdev))
70279513d2a5SSasha Neftin __igc_close(netdev, true);
70289513d2a5SSasha Neftin
7029a5136f76SSasha Neftin igc_ptp_suspend(adapter);
7030a5136f76SSasha Neftin
70319513d2a5SSasha Neftin igc_clear_interrupt_scheme(adapter);
70329513d2a5SSasha Neftin rtnl_unlock();
70339513d2a5SSasha Neftin
70349513d2a5SSasha Neftin status = rd32(IGC_STATUS);
70359513d2a5SSasha Neftin if (status & IGC_STATUS_LU)
70369513d2a5SSasha Neftin wufc &= ~IGC_WUFC_LNKC;
70379513d2a5SSasha Neftin
70389513d2a5SSasha Neftin if (wufc) {
70399513d2a5SSasha Neftin igc_setup_rctl(adapter);
70409513d2a5SSasha Neftin igc_set_rx_mode(netdev);
70419513d2a5SSasha Neftin
70429513d2a5SSasha Neftin /* turn on all-multi mode if wake on multicast is enabled */
70439513d2a5SSasha Neftin if (wufc & IGC_WUFC_MC) {
70449513d2a5SSasha Neftin rctl = rd32(IGC_RCTL);
70459513d2a5SSasha Neftin rctl |= IGC_RCTL_MPE;
70469513d2a5SSasha Neftin wr32(IGC_RCTL, rctl);
70479513d2a5SSasha Neftin }
70489513d2a5SSasha Neftin
70499513d2a5SSasha Neftin ctrl = rd32(IGC_CTRL);
70509513d2a5SSasha Neftin ctrl |= IGC_CTRL_ADVD3WUC;
70519513d2a5SSasha Neftin wr32(IGC_CTRL, ctrl);
70529513d2a5SSasha Neftin
70539513d2a5SSasha Neftin /* Allow time for pending master requests to run */
70549513d2a5SSasha Neftin igc_disable_pcie_master(hw);
70559513d2a5SSasha Neftin
70569513d2a5SSasha Neftin wr32(IGC_WUC, IGC_WUC_PME_EN);
70579513d2a5SSasha Neftin wr32(IGC_WUFC, wufc);
70589513d2a5SSasha Neftin } else {
70599513d2a5SSasha Neftin wr32(IGC_WUC, 0);
70609513d2a5SSasha Neftin wr32(IGC_WUFC, 0);
70619513d2a5SSasha Neftin }
70629513d2a5SSasha Neftin
70639513d2a5SSasha Neftin wake = wufc || adapter->en_mng_pt;
70649513d2a5SSasha Neftin if (!wake)
7065a0beb3c1SSasha Neftin igc_power_down_phy_copper_base(&adapter->hw);
70669513d2a5SSasha Neftin else
70679513d2a5SSasha Neftin igc_power_up_link(adapter);
70689513d2a5SSasha Neftin
70699513d2a5SSasha Neftin if (enable_wake)
70709513d2a5SSasha Neftin *enable_wake = wake;
70719513d2a5SSasha Neftin
70729513d2a5SSasha Neftin /* Release control of h/w to f/w. If f/w is AMT enabled, this
70739513d2a5SSasha Neftin * would have already happened in close and is redundant.
70749513d2a5SSasha Neftin */
70759513d2a5SSasha Neftin igc_release_hw_control(adapter);
70769513d2a5SSasha Neftin
70779513d2a5SSasha Neftin pci_disable_device(pdev);
70789513d2a5SSasha Neftin
70799513d2a5SSasha Neftin return 0;
70809513d2a5SSasha Neftin }
70819513d2a5SSasha Neftin
70829513d2a5SSasha Neftin #ifdef CONFIG_PM
igc_runtime_suspend(struct device * dev)70839513d2a5SSasha Neftin static int __maybe_unused igc_runtime_suspend(struct device *dev)
70849513d2a5SSasha Neftin {
70859513d2a5SSasha Neftin return __igc_shutdown(to_pci_dev(dev), NULL, 1);
70869513d2a5SSasha Neftin }
70879513d2a5SSasha Neftin
igc_deliver_wake_packet(struct net_device * netdev)70889513d2a5SSasha Neftin static void igc_deliver_wake_packet(struct net_device *netdev)
70899513d2a5SSasha Neftin {
70909513d2a5SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
70919513d2a5SSasha Neftin struct igc_hw *hw = &adapter->hw;
70929513d2a5SSasha Neftin struct sk_buff *skb;
70939513d2a5SSasha Neftin u32 wupl;
70949513d2a5SSasha Neftin
70959513d2a5SSasha Neftin wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
70969513d2a5SSasha Neftin
70979513d2a5SSasha Neftin /* WUPM stores only the first 128 bytes of the wake packet.
70989513d2a5SSasha Neftin * Read the packet only if we have the whole thing.
70999513d2a5SSasha Neftin */
71009513d2a5SSasha Neftin if (wupl == 0 || wupl > IGC_WUPM_BYTES)
71019513d2a5SSasha Neftin return;
71029513d2a5SSasha Neftin
71039513d2a5SSasha Neftin skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
71049513d2a5SSasha Neftin if (!skb)
71059513d2a5SSasha Neftin return;
71069513d2a5SSasha Neftin
71079513d2a5SSasha Neftin skb_put(skb, wupl);
71089513d2a5SSasha Neftin
71099513d2a5SSasha Neftin /* Ensure reads are 32-bit aligned */
71109513d2a5SSasha Neftin wupl = roundup(wupl, 4);
71119513d2a5SSasha Neftin
71129513d2a5SSasha Neftin memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
71139513d2a5SSasha Neftin
71149513d2a5SSasha Neftin skb->protocol = eth_type_trans(skb, netdev);
71159513d2a5SSasha Neftin netif_rx(skb);
71169513d2a5SSasha Neftin }
71179513d2a5SSasha Neftin
igc_resume(struct device * dev)71189513d2a5SSasha Neftin static int __maybe_unused igc_resume(struct device *dev)
71199513d2a5SSasha Neftin {
71209513d2a5SSasha Neftin struct pci_dev *pdev = to_pci_dev(dev);
71219513d2a5SSasha Neftin struct net_device *netdev = pci_get_drvdata(pdev);
71229513d2a5SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
71239513d2a5SSasha Neftin struct igc_hw *hw = &adapter->hw;
71249513d2a5SSasha Neftin u32 err, val;
71259513d2a5SSasha Neftin
71269513d2a5SSasha Neftin pci_set_power_state(pdev, PCI_D0);
71279513d2a5SSasha Neftin pci_restore_state(pdev);
71289513d2a5SSasha Neftin pci_save_state(pdev);
71299513d2a5SSasha Neftin
71309513d2a5SSasha Neftin if (!pci_device_is_present(pdev))
71319513d2a5SSasha Neftin return -ENODEV;
71329513d2a5SSasha Neftin err = pci_enable_device_mem(pdev);
71339513d2a5SSasha Neftin if (err) {
713425f06effSAndre Guedes netdev_err(netdev, "Cannot enable PCI device from suspend\n");
71359513d2a5SSasha Neftin return err;
71369513d2a5SSasha Neftin }
71379513d2a5SSasha Neftin pci_set_master(pdev);
71389513d2a5SSasha Neftin
71399513d2a5SSasha Neftin pci_enable_wake(pdev, PCI_D3hot, 0);
71409513d2a5SSasha Neftin pci_enable_wake(pdev, PCI_D3cold, 0);
71419513d2a5SSasha Neftin
71429513d2a5SSasha Neftin if (igc_init_interrupt_scheme(adapter, true)) {
714325f06effSAndre Guedes netdev_err(netdev, "Unable to allocate memory for queues\n");
71449513d2a5SSasha Neftin return -ENOMEM;
71459513d2a5SSasha Neftin }
71469513d2a5SSasha Neftin
71479513d2a5SSasha Neftin igc_reset(adapter);
71489513d2a5SSasha Neftin
71499513d2a5SSasha Neftin /* let the f/w know that the h/w is now under the control of the
71509513d2a5SSasha Neftin * driver.
71519513d2a5SSasha Neftin */
71529513d2a5SSasha Neftin igc_get_hw_control(adapter);
71539513d2a5SSasha Neftin
71549513d2a5SSasha Neftin val = rd32(IGC_WUS);
71559513d2a5SSasha Neftin if (val & WAKE_PKT_WUS)
71569513d2a5SSasha Neftin igc_deliver_wake_packet(netdev);
71579513d2a5SSasha Neftin
71589513d2a5SSasha Neftin wr32(IGC_WUS, ~0);
71599513d2a5SSasha Neftin
71609513d2a5SSasha Neftin rtnl_lock();
71619513d2a5SSasha Neftin if (!err && netif_running(netdev))
71629513d2a5SSasha Neftin err = __igc_open(netdev, true);
71639513d2a5SSasha Neftin
71649513d2a5SSasha Neftin if (!err)
71659513d2a5SSasha Neftin netif_device_attach(netdev);
71669513d2a5SSasha Neftin rtnl_unlock();
71679513d2a5SSasha Neftin
71689513d2a5SSasha Neftin return err;
71699513d2a5SSasha Neftin }
71709513d2a5SSasha Neftin
igc_runtime_resume(struct device * dev)71719513d2a5SSasha Neftin static int __maybe_unused igc_runtime_resume(struct device *dev)
71729513d2a5SSasha Neftin {
71739513d2a5SSasha Neftin return igc_resume(dev);
71749513d2a5SSasha Neftin }
71759513d2a5SSasha Neftin
igc_suspend(struct device * dev)71769513d2a5SSasha Neftin static int __maybe_unused igc_suspend(struct device *dev)
71779513d2a5SSasha Neftin {
71789513d2a5SSasha Neftin return __igc_shutdown(to_pci_dev(dev), NULL, 0);
71799513d2a5SSasha Neftin }
71809513d2a5SSasha Neftin
igc_runtime_idle(struct device * dev)71819513d2a5SSasha Neftin static int __maybe_unused igc_runtime_idle(struct device *dev)
71829513d2a5SSasha Neftin {
71839513d2a5SSasha Neftin struct net_device *netdev = dev_get_drvdata(dev);
71849513d2a5SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
71859513d2a5SSasha Neftin
71869513d2a5SSasha Neftin if (!igc_has_link(adapter))
71879513d2a5SSasha Neftin pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
71889513d2a5SSasha Neftin
71899513d2a5SSasha Neftin return -EBUSY;
71909513d2a5SSasha Neftin }
71919513d2a5SSasha Neftin #endif /* CONFIG_PM */
71929513d2a5SSasha Neftin
igc_shutdown(struct pci_dev * pdev)71939513d2a5SSasha Neftin static void igc_shutdown(struct pci_dev *pdev)
71949513d2a5SSasha Neftin {
71959513d2a5SSasha Neftin bool wake;
71969513d2a5SSasha Neftin
71979513d2a5SSasha Neftin __igc_shutdown(pdev, &wake, 0);
71989513d2a5SSasha Neftin
71999513d2a5SSasha Neftin if (system_state == SYSTEM_POWER_OFF) {
72009513d2a5SSasha Neftin pci_wake_from_d3(pdev, wake);
72019513d2a5SSasha Neftin pci_set_power_state(pdev, PCI_D3hot);
72029513d2a5SSasha Neftin }
72039513d2a5SSasha Neftin }
72049513d2a5SSasha Neftin
7205bc23aa94SSasha Neftin /**
7206bc23aa94SSasha Neftin * igc_io_error_detected - called when PCI error is detected
7207bc23aa94SSasha Neftin * @pdev: Pointer to PCI device
7208bc23aa94SSasha Neftin * @state: The current PCI connection state
7209bc23aa94SSasha Neftin *
7210bc23aa94SSasha Neftin * This function is called after a PCI bus error affecting
7211bc23aa94SSasha Neftin * this device has been detected.
7212bc23aa94SSasha Neftin **/
igc_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)7213bc23aa94SSasha Neftin static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
7214bc23aa94SSasha Neftin pci_channel_state_t state)
7215bc23aa94SSasha Neftin {
7216bc23aa94SSasha Neftin struct net_device *netdev = pci_get_drvdata(pdev);
7217bc23aa94SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
7218bc23aa94SSasha Neftin
7219bc23aa94SSasha Neftin netif_device_detach(netdev);
7220bc23aa94SSasha Neftin
7221bc23aa94SSasha Neftin if (state == pci_channel_io_perm_failure)
7222bc23aa94SSasha Neftin return PCI_ERS_RESULT_DISCONNECT;
7223bc23aa94SSasha Neftin
7224bc23aa94SSasha Neftin if (netif_running(netdev))
7225bc23aa94SSasha Neftin igc_down(adapter);
7226bc23aa94SSasha Neftin pci_disable_device(pdev);
7227bc23aa94SSasha Neftin
7228bc23aa94SSasha Neftin /* Request a slot reset. */
7229bc23aa94SSasha Neftin return PCI_ERS_RESULT_NEED_RESET;
7230bc23aa94SSasha Neftin }
7231bc23aa94SSasha Neftin
7232bc23aa94SSasha Neftin /**
7233bc23aa94SSasha Neftin * igc_io_slot_reset - called after the PCI bus has been reset.
7234bc23aa94SSasha Neftin * @pdev: Pointer to PCI device
7235bc23aa94SSasha Neftin *
7236bc23aa94SSasha Neftin * Restart the card from scratch, as if from a cold-boot. Implementation
7237bc23aa94SSasha Neftin * resembles the first-half of the igc_resume routine.
7238bc23aa94SSasha Neftin **/
igc_io_slot_reset(struct pci_dev * pdev)7239bc23aa94SSasha Neftin static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
7240bc23aa94SSasha Neftin {
7241bc23aa94SSasha Neftin struct net_device *netdev = pci_get_drvdata(pdev);
7242bc23aa94SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
7243bc23aa94SSasha Neftin struct igc_hw *hw = &adapter->hw;
7244bc23aa94SSasha Neftin pci_ers_result_t result;
7245bc23aa94SSasha Neftin
7246bc23aa94SSasha Neftin if (pci_enable_device_mem(pdev)) {
724725f06effSAndre Guedes netdev_err(netdev, "Could not re-enable PCI device after reset\n");
7248bc23aa94SSasha Neftin result = PCI_ERS_RESULT_DISCONNECT;
7249bc23aa94SSasha Neftin } else {
7250bc23aa94SSasha Neftin pci_set_master(pdev);
7251bc23aa94SSasha Neftin pci_restore_state(pdev);
7252bc23aa94SSasha Neftin pci_save_state(pdev);
7253bc23aa94SSasha Neftin
7254bc23aa94SSasha Neftin pci_enable_wake(pdev, PCI_D3hot, 0);
7255bc23aa94SSasha Neftin pci_enable_wake(pdev, PCI_D3cold, 0);
7256bc23aa94SSasha Neftin
7257bc23aa94SSasha Neftin /* In case of PCI error, adapter loses its HW address
7258bc23aa94SSasha Neftin * so we should re-assign it here.
7259bc23aa94SSasha Neftin */
7260bc23aa94SSasha Neftin hw->hw_addr = adapter->io_addr;
7261bc23aa94SSasha Neftin
7262bc23aa94SSasha Neftin igc_reset(adapter);
7263bc23aa94SSasha Neftin wr32(IGC_WUS, ~0);
7264bc23aa94SSasha Neftin result = PCI_ERS_RESULT_RECOVERED;
7265bc23aa94SSasha Neftin }
7266bc23aa94SSasha Neftin
7267bc23aa94SSasha Neftin return result;
7268bc23aa94SSasha Neftin }
7269bc23aa94SSasha Neftin
7270bc23aa94SSasha Neftin /**
7271bc23aa94SSasha Neftin * igc_io_resume - called when traffic can start to flow again.
7272bc23aa94SSasha Neftin * @pdev: Pointer to PCI device
7273bc23aa94SSasha Neftin *
7274bc23aa94SSasha Neftin * This callback is called when the error recovery driver tells us that
7275bc23aa94SSasha Neftin * its OK to resume normal operation. Implementation resembles the
7276bc23aa94SSasha Neftin * second-half of the igc_resume routine.
7277bc23aa94SSasha Neftin */
igc_io_resume(struct pci_dev * pdev)7278bc23aa94SSasha Neftin static void igc_io_resume(struct pci_dev *pdev)
7279bc23aa94SSasha Neftin {
7280bc23aa94SSasha Neftin struct net_device *netdev = pci_get_drvdata(pdev);
7281bc23aa94SSasha Neftin struct igc_adapter *adapter = netdev_priv(netdev);
7282bc23aa94SSasha Neftin
7283bc23aa94SSasha Neftin rtnl_lock();
7284bc23aa94SSasha Neftin if (netif_running(netdev)) {
7285bc23aa94SSasha Neftin if (igc_open(netdev)) {
728625f06effSAndre Guedes netdev_err(netdev, "igc_open failed after reset\n");
7287bc23aa94SSasha Neftin return;
7288bc23aa94SSasha Neftin }
7289bc23aa94SSasha Neftin }
7290bc23aa94SSasha Neftin
7291bc23aa94SSasha Neftin netif_device_attach(netdev);
7292bc23aa94SSasha Neftin
7293bc23aa94SSasha Neftin /* let the f/w know that the h/w is now under the control of the
7294bc23aa94SSasha Neftin * driver.
7295bc23aa94SSasha Neftin */
7296bc23aa94SSasha Neftin igc_get_hw_control(adapter);
7297bc23aa94SSasha Neftin rtnl_unlock();
7298bc23aa94SSasha Neftin }
7299bc23aa94SSasha Neftin
7300bc23aa94SSasha Neftin static const struct pci_error_handlers igc_err_handler = {
7301bc23aa94SSasha Neftin .error_detected = igc_io_error_detected,
7302bc23aa94SSasha Neftin .slot_reset = igc_io_slot_reset,
7303bc23aa94SSasha Neftin .resume = igc_io_resume,
7304bc23aa94SSasha Neftin };
7305bc23aa94SSasha Neftin
73069513d2a5SSasha Neftin #ifdef CONFIG_PM
73079513d2a5SSasha Neftin static const struct dev_pm_ops igc_pm_ops = {
73089513d2a5SSasha Neftin SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume)
73099513d2a5SSasha Neftin SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume,
73109513d2a5SSasha Neftin igc_runtime_idle)
73119513d2a5SSasha Neftin };
73129513d2a5SSasha Neftin #endif
73139513d2a5SSasha Neftin
7314d89f8841SSasha Neftin static struct pci_driver igc_driver = {
7315d89f8841SSasha Neftin .name = igc_driver_name,
7316d89f8841SSasha Neftin .id_table = igc_pci_tbl,
7317d89f8841SSasha Neftin .probe = igc_probe,
7318d89f8841SSasha Neftin .remove = igc_remove,
73199513d2a5SSasha Neftin #ifdef CONFIG_PM
73209513d2a5SSasha Neftin .driver.pm = &igc_pm_ops,
73219513d2a5SSasha Neftin #endif
73229513d2a5SSasha Neftin .shutdown = igc_shutdown,
7323bc23aa94SSasha Neftin .err_handler = &igc_err_handler,
7324d89f8841SSasha Neftin };
7325d89f8841SSasha Neftin
7326146740f9SSasha Neftin /**
73278c5ad0daSSasha Neftin * igc_reinit_queues - return error
73288c5ad0daSSasha Neftin * @adapter: pointer to adapter structure
73298c5ad0daSSasha Neftin */
igc_reinit_queues(struct igc_adapter * adapter)73308c5ad0daSSasha Neftin int igc_reinit_queues(struct igc_adapter *adapter)
73318c5ad0daSSasha Neftin {
73328c5ad0daSSasha Neftin struct net_device *netdev = adapter->netdev;
73338c5ad0daSSasha Neftin int err = 0;
73348c5ad0daSSasha Neftin
73358c5ad0daSSasha Neftin if (netif_running(netdev))
73368c5ad0daSSasha Neftin igc_close(netdev);
73378c5ad0daSSasha Neftin
73388c5ad0daSSasha Neftin igc_reset_interrupt_capability(adapter);
73398c5ad0daSSasha Neftin
73408c5ad0daSSasha Neftin if (igc_init_interrupt_scheme(adapter, true)) {
734125f06effSAndre Guedes netdev_err(netdev, "Unable to allocate memory for queues\n");
73428c5ad0daSSasha Neftin return -ENOMEM;
73438c5ad0daSSasha Neftin }
73448c5ad0daSSasha Neftin
73458c5ad0daSSasha Neftin if (netif_running(netdev))
73468c5ad0daSSasha Neftin err = igc_open(netdev);
73478c5ad0daSSasha Neftin
73488c5ad0daSSasha Neftin return err;
73498c5ad0daSSasha Neftin }
73508c5ad0daSSasha Neftin
73518c5ad0daSSasha Neftin /**
7352c0071c7aSSasha Neftin * igc_get_hw_dev - return device
7353c0071c7aSSasha Neftin * @hw: pointer to hardware structure
7354c0071c7aSSasha Neftin *
7355c0071c7aSSasha Neftin * used by hardware layer to print debugging information
7356c0071c7aSSasha Neftin */
igc_get_hw_dev(struct igc_hw * hw)7357c0071c7aSSasha Neftin struct net_device *igc_get_hw_dev(struct igc_hw *hw)
7358c0071c7aSSasha Neftin {
7359c0071c7aSSasha Neftin struct igc_adapter *adapter = hw->back;
7360c0071c7aSSasha Neftin
7361c0071c7aSSasha Neftin return adapter->netdev;
7362c0071c7aSSasha Neftin }
7363c0071c7aSSasha Neftin
igc_disable_rx_ring_hw(struct igc_ring * ring)7364fc9df2a0SAndre Guedes static void igc_disable_rx_ring_hw(struct igc_ring *ring)
7365fc9df2a0SAndre Guedes {
7366fc9df2a0SAndre Guedes struct igc_hw *hw = &ring->q_vector->adapter->hw;
7367fc9df2a0SAndre Guedes u8 idx = ring->reg_idx;
7368fc9df2a0SAndre Guedes u32 rxdctl;
7369fc9df2a0SAndre Guedes
7370fc9df2a0SAndre Guedes rxdctl = rd32(IGC_RXDCTL(idx));
7371fc9df2a0SAndre Guedes rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE;
7372fc9df2a0SAndre Guedes rxdctl |= IGC_RXDCTL_SWFLUSH;
7373fc9df2a0SAndre Guedes wr32(IGC_RXDCTL(idx), rxdctl);
7374fc9df2a0SAndre Guedes }
7375fc9df2a0SAndre Guedes
igc_disable_rx_ring(struct igc_ring * ring)7376fc9df2a0SAndre Guedes void igc_disable_rx_ring(struct igc_ring *ring)
7377fc9df2a0SAndre Guedes {
7378fc9df2a0SAndre Guedes igc_disable_rx_ring_hw(ring);
7379fc9df2a0SAndre Guedes igc_clean_rx_ring(ring);
7380fc9df2a0SAndre Guedes }
7381fc9df2a0SAndre Guedes
igc_enable_rx_ring(struct igc_ring * ring)7382fc9df2a0SAndre Guedes void igc_enable_rx_ring(struct igc_ring *ring)
7383fc9df2a0SAndre Guedes {
7384fc9df2a0SAndre Guedes struct igc_adapter *adapter = ring->q_vector->adapter;
7385fc9df2a0SAndre Guedes
7386fc9df2a0SAndre Guedes igc_configure_rx_ring(adapter, ring);
7387fc9df2a0SAndre Guedes
7388fc9df2a0SAndre Guedes if (ring->xsk_pool)
7389fc9df2a0SAndre Guedes igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
7390fc9df2a0SAndre Guedes else
7391fc9df2a0SAndre Guedes igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
7392fc9df2a0SAndre Guedes }
7393fc9df2a0SAndre Guedes
igc_disable_tx_ring(struct igc_ring * ring)73949acf59a7SAndre Guedes void igc_disable_tx_ring(struct igc_ring *ring)
73959acf59a7SAndre Guedes {
73969acf59a7SAndre Guedes igc_disable_tx_ring_hw(ring);
73979acf59a7SAndre Guedes igc_clean_tx_ring(ring);
73989acf59a7SAndre Guedes }
73999acf59a7SAndre Guedes
igc_enable_tx_ring(struct igc_ring * ring)74009acf59a7SAndre Guedes void igc_enable_tx_ring(struct igc_ring *ring)
74019acf59a7SAndre Guedes {
74029acf59a7SAndre Guedes struct igc_adapter *adapter = ring->q_vector->adapter;
74039acf59a7SAndre Guedes
74049acf59a7SAndre Guedes igc_configure_tx_ring(adapter, ring);
74059acf59a7SAndre Guedes }
74069acf59a7SAndre Guedes
7407c0071c7aSSasha Neftin /**
7408d89f8841SSasha Neftin * igc_init_module - Driver Registration Routine
7409d89f8841SSasha Neftin *
7410d89f8841SSasha Neftin * igc_init_module is the first routine called when the driver is
7411d89f8841SSasha Neftin * loaded. All it does is register with the PCI subsystem.
7412d89f8841SSasha Neftin */
igc_init_module(void)7413d89f8841SSasha Neftin static int __init igc_init_module(void)
7414d89f8841SSasha Neftin {
7415d89f8841SSasha Neftin int ret;
7416d89f8841SSasha Neftin
741734a2a3b8SJeff Kirsher pr_info("%s\n", igc_driver_string);
7418d89f8841SSasha Neftin pr_info("%s\n", igc_copyright);
7419d89f8841SSasha Neftin
7420d89f8841SSasha Neftin ret = pci_register_driver(&igc_driver);
7421d89f8841SSasha Neftin return ret;
7422d89f8841SSasha Neftin }
7423d89f8841SSasha Neftin
7424d89f8841SSasha Neftin module_init(igc_init_module);
7425d89f8841SSasha Neftin
7426d89f8841SSasha Neftin /**
7427d89f8841SSasha Neftin * igc_exit_module - Driver Exit Cleanup Routine
7428d89f8841SSasha Neftin *
7429d89f8841SSasha Neftin * igc_exit_module is called just before the driver is removed
7430d89f8841SSasha Neftin * from memory.
7431d89f8841SSasha Neftin */
igc_exit_module(void)7432d89f8841SSasha Neftin static void __exit igc_exit_module(void)
7433d89f8841SSasha Neftin {
7434d89f8841SSasha Neftin pci_unregister_driver(&igc_driver);
7435d89f8841SSasha Neftin }
7436d89f8841SSasha Neftin
7437d89f8841SSasha Neftin module_exit(igc_exit_module);
7438d89f8841SSasha Neftin /* igc_main.c */
7439