xref: /openbmc/linux/drivers/net/ethernet/sfc/efx.c (revision 3990a8ff)
1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2874aeea5SJeff Kirsher /****************************************************************************
3f7a6d2c4SBen Hutchings  * Driver for Solarflare network controllers and boards
4874aeea5SJeff Kirsher  * Copyright 2005-2006 Fen Systems Ltd.
5f7a6d2c4SBen Hutchings  * Copyright 2005-2013 Solarflare Communications Inc.
6874aeea5SJeff Kirsher  */
7874aeea5SJeff Kirsher 
8874aeea5SJeff Kirsher #include <linux/module.h>
9874aeea5SJeff Kirsher #include <linux/pci.h>
10874aeea5SJeff Kirsher #include <linux/netdevice.h>
11874aeea5SJeff Kirsher #include <linux/etherdevice.h>
12874aeea5SJeff Kirsher #include <linux/delay.h>
13874aeea5SJeff Kirsher #include <linux/notifier.h>
14874aeea5SJeff Kirsher #include <linux/ip.h>
15874aeea5SJeff Kirsher #include <linux/tcp.h>
16874aeea5SJeff Kirsher #include <linux/in.h>
17874aeea5SJeff Kirsher #include <linux/ethtool.h>
18874aeea5SJeff Kirsher #include <linux/topology.h>
19874aeea5SJeff Kirsher #include <linux/gfp.h>
20626950dbSAlexandre Rames #include <linux/aer.h>
21b28405b0SAlexandre Rames #include <linux/interrupt.h>
22874aeea5SJeff Kirsher #include "net_driver.h"
23e5fbd977SJon Cooper #include <net/gre.h>
24e5fbd977SJon Cooper #include <net/udp_tunnel.h>
25874aeea5SJeff Kirsher #include "efx.h"
26874aeea5SJeff Kirsher #include "nic.h"
2771827443SEdward Cree #include "io.h"
28dd40781eSBen Hutchings #include "selftest.h"
297fa8d547SShradha Shah #include "sriov.h"
30874aeea5SJeff Kirsher 
31874aeea5SJeff Kirsher #include "mcdi.h"
32e5fbd977SJon Cooper #include "mcdi_pcol.h"
33874aeea5SJeff Kirsher #include "workarounds.h"
34874aeea5SJeff Kirsher 
35874aeea5SJeff Kirsher /**************************************************************************
36874aeea5SJeff Kirsher  *
37874aeea5SJeff Kirsher  * Type name strings
38874aeea5SJeff Kirsher  *
39874aeea5SJeff Kirsher  **************************************************************************
40874aeea5SJeff Kirsher  */
41874aeea5SJeff Kirsher 
42874aeea5SJeff Kirsher /* Loopback mode names (see LOOPBACK_MODE()) */
43874aeea5SJeff Kirsher const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
4418e83e4cSBen Hutchings const char *const efx_loopback_mode_names[] = {
45874aeea5SJeff Kirsher 	[LOOPBACK_NONE]		= "NONE",
46874aeea5SJeff Kirsher 	[LOOPBACK_DATA]		= "DATAPATH",
47874aeea5SJeff Kirsher 	[LOOPBACK_GMAC]		= "GMAC",
48874aeea5SJeff Kirsher 	[LOOPBACK_XGMII]	= "XGMII",
49874aeea5SJeff Kirsher 	[LOOPBACK_XGXS]		= "XGXS",
50874aeea5SJeff Kirsher 	[LOOPBACK_XAUI]		= "XAUI",
51874aeea5SJeff Kirsher 	[LOOPBACK_GMII]		= "GMII",
52874aeea5SJeff Kirsher 	[LOOPBACK_SGMII]	= "SGMII",
53874aeea5SJeff Kirsher 	[LOOPBACK_XGBR]		= "XGBR",
54874aeea5SJeff Kirsher 	[LOOPBACK_XFI]		= "XFI",
55874aeea5SJeff Kirsher 	[LOOPBACK_XAUI_FAR]	= "XAUI_FAR",
56874aeea5SJeff Kirsher 	[LOOPBACK_GMII_FAR]	= "GMII_FAR",
57874aeea5SJeff Kirsher 	[LOOPBACK_SGMII_FAR]	= "SGMII_FAR",
58874aeea5SJeff Kirsher 	[LOOPBACK_XFI_FAR]	= "XFI_FAR",
59874aeea5SJeff Kirsher 	[LOOPBACK_GPHY]		= "GPHY",
60874aeea5SJeff Kirsher 	[LOOPBACK_PHYXS]	= "PHYXS",
61874aeea5SJeff Kirsher 	[LOOPBACK_PCS]		= "PCS",
62874aeea5SJeff Kirsher 	[LOOPBACK_PMAPMD]	= "PMA/PMD",
63874aeea5SJeff Kirsher 	[LOOPBACK_XPORT]	= "XPORT",
64874aeea5SJeff Kirsher 	[LOOPBACK_XGMII_WS]	= "XGMII_WS",
65874aeea5SJeff Kirsher 	[LOOPBACK_XAUI_WS]	= "XAUI_WS",
66874aeea5SJeff Kirsher 	[LOOPBACK_XAUI_WS_FAR]  = "XAUI_WS_FAR",
67874aeea5SJeff Kirsher 	[LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
68874aeea5SJeff Kirsher 	[LOOPBACK_GMII_WS]	= "GMII_WS",
69874aeea5SJeff Kirsher 	[LOOPBACK_XFI_WS]	= "XFI_WS",
70874aeea5SJeff Kirsher 	[LOOPBACK_XFI_WS_FAR]	= "XFI_WS_FAR",
71874aeea5SJeff Kirsher 	[LOOPBACK_PHYXS_WS]	= "PHYXS_WS",
72874aeea5SJeff Kirsher };
73874aeea5SJeff Kirsher 
74874aeea5SJeff Kirsher const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
7518e83e4cSBen Hutchings const char *const efx_reset_type_names[] = {
76874aeea5SJeff Kirsher 	[RESET_TYPE_INVISIBLE]          = "INVISIBLE",
77874aeea5SJeff Kirsher 	[RESET_TYPE_ALL]                = "ALL",
78626950dbSAlexandre Rames 	[RESET_TYPE_RECOVER_OR_ALL]     = "RECOVER_OR_ALL",
79874aeea5SJeff Kirsher 	[RESET_TYPE_WORLD]              = "WORLD",
80626950dbSAlexandre Rames 	[RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE",
81087e9025SJon Cooper 	[RESET_TYPE_DATAPATH]           = "DATAPATH",
82e283546cSEdward Cree 	[RESET_TYPE_MC_BIST]		= "MC_BIST",
83874aeea5SJeff Kirsher 	[RESET_TYPE_DISABLE]            = "DISABLE",
84874aeea5SJeff Kirsher 	[RESET_TYPE_TX_WATCHDOG]        = "TX_WATCHDOG",
85874aeea5SJeff Kirsher 	[RESET_TYPE_INT_ERROR]          = "INT_ERROR",
863de82b91SAlexandre Rames 	[RESET_TYPE_DMA_ERROR]          = "DMA_ERROR",
87874aeea5SJeff Kirsher 	[RESET_TYPE_TX_SKIP]            = "TX_SKIP",
88874aeea5SJeff Kirsher 	[RESET_TYPE_MC_FAILURE]         = "MC_FAILURE",
89e283546cSEdward Cree 	[RESET_TYPE_MCDI_TIMEOUT]	= "MCDI_TIMEOUT (FLR)",
90874aeea5SJeff Kirsher };
91874aeea5SJeff Kirsher 
92e5fbd977SJon Cooper /* UDP tunnel type names */
93e5fbd977SJon Cooper static const char *const efx_udp_tunnel_type_names[] = {
94e5fbd977SJon Cooper 	[TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan",
95e5fbd977SJon Cooper 	[TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve",
96e5fbd977SJon Cooper };
97e5fbd977SJon Cooper 
98e5fbd977SJon Cooper void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen)
99e5fbd977SJon Cooper {
100e5fbd977SJon Cooper 	if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) &&
101e5fbd977SJon Cooper 	    efx_udp_tunnel_type_names[type] != NULL)
102e5fbd977SJon Cooper 		snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]);
103e5fbd977SJon Cooper 	else
104e5fbd977SJon Cooper 		snprintf(buf, buflen, "type %d", type);
105e5fbd977SJon Cooper }
106e5fbd977SJon Cooper 
107874aeea5SJeff Kirsher /* Reset workqueue. If any NIC has a hardware failure then a reset will be
108874aeea5SJeff Kirsher  * queued onto this work queue. This is not a per-nic work queue, because
109874aeea5SJeff Kirsher  * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
110874aeea5SJeff Kirsher  */
111874aeea5SJeff Kirsher static struct workqueue_struct *reset_workqueue;
112874aeea5SJeff Kirsher 
11374cd60a4SJon Cooper /* How often and how many times to poll for a reset while waiting for a
11474cd60a4SJon Cooper  * BIST that another function started to complete.
11574cd60a4SJon Cooper  */
11674cd60a4SJon Cooper #define BIST_WAIT_DELAY_MS	100
11774cd60a4SJon Cooper #define BIST_WAIT_DELAY_COUNT	100
11874cd60a4SJon Cooper 
119874aeea5SJeff Kirsher /**************************************************************************
120874aeea5SJeff Kirsher  *
121874aeea5SJeff Kirsher  * Configurable values
122874aeea5SJeff Kirsher  *
123874aeea5SJeff Kirsher  *************************************************************************/
124874aeea5SJeff Kirsher 
125874aeea5SJeff Kirsher /*
126874aeea5SJeff Kirsher  * Use separate channels for TX and RX events
127874aeea5SJeff Kirsher  *
128874aeea5SJeff Kirsher  * Set this to 1 to use separate channels for TX and RX. It allows us
129874aeea5SJeff Kirsher  * to control interrupt affinity separately for TX and RX.
130874aeea5SJeff Kirsher  *
131874aeea5SJeff Kirsher  * This is only used in MSI-X interrupt mode
132874aeea5SJeff Kirsher  */
133b0fbdae1SShradha Shah bool efx_separate_tx_channels;
134b0fbdae1SShradha Shah module_param(efx_separate_tx_channels, bool, 0444);
135b0fbdae1SShradha Shah MODULE_PARM_DESC(efx_separate_tx_channels,
136874aeea5SJeff Kirsher 		 "Use separate channels for TX and RX");
137874aeea5SJeff Kirsher 
138874aeea5SJeff Kirsher /* This is the weight assigned to each of the (per-channel) virtual
139874aeea5SJeff Kirsher  * NAPI devices.
140874aeea5SJeff Kirsher  */
141874aeea5SJeff Kirsher static int napi_weight = 64;
142874aeea5SJeff Kirsher 
143874aeea5SJeff Kirsher /* This is the time (in jiffies) between invocations of the hardware
144626950dbSAlexandre Rames  * monitor.
145626950dbSAlexandre Rames  * On Falcon-based NICs, this will:
146874aeea5SJeff Kirsher  * - Check the on-board hardware monitor;
147874aeea5SJeff Kirsher  * - Poll the link state and reconfigure the hardware as necessary.
148626950dbSAlexandre Rames  * On Siena-based NICs for power systems with EEH support, this will give EEH a
149626950dbSAlexandre Rames  * chance to start.
150874aeea5SJeff Kirsher  */
151874aeea5SJeff Kirsher static unsigned int efx_monitor_interval = 1 * HZ;
152874aeea5SJeff Kirsher 
153874aeea5SJeff Kirsher /* Initial interrupt moderation settings.  They can be modified after
154874aeea5SJeff Kirsher  * module load with ethtool.
155874aeea5SJeff Kirsher  *
156874aeea5SJeff Kirsher  * The default for RX should strike a balance between increasing the
157874aeea5SJeff Kirsher  * round-trip latency and reducing overhead.
158874aeea5SJeff Kirsher  */
159874aeea5SJeff Kirsher static unsigned int rx_irq_mod_usec = 60;
160874aeea5SJeff Kirsher 
161874aeea5SJeff Kirsher /* Initial interrupt moderation settings.  They can be modified after
162874aeea5SJeff Kirsher  * module load with ethtool.
163874aeea5SJeff Kirsher  *
164874aeea5SJeff Kirsher  * This default is chosen to ensure that a 10G link does not go idle
165874aeea5SJeff Kirsher  * while a TX queue is stopped after it has become full.  A queue is
166874aeea5SJeff Kirsher  * restarted when it drops below half full.  The time this takes (assuming
167874aeea5SJeff Kirsher  * worst case 3 descriptors per packet and 1024 descriptors) is
168874aeea5SJeff Kirsher  *   512 / 3 * 1.2 = 205 usec.
169874aeea5SJeff Kirsher  */
170874aeea5SJeff Kirsher static unsigned int tx_irq_mod_usec = 150;
171874aeea5SJeff Kirsher 
172874aeea5SJeff Kirsher /* This is the first interrupt mode to try out of:
173874aeea5SJeff Kirsher  * 0 => MSI-X
174874aeea5SJeff Kirsher  * 1 => MSI
175874aeea5SJeff Kirsher  * 2 => legacy
176874aeea5SJeff Kirsher  */
177874aeea5SJeff Kirsher static unsigned int interrupt_mode;
178874aeea5SJeff Kirsher 
179874aeea5SJeff Kirsher /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
180874aeea5SJeff Kirsher  * i.e. the number of CPUs among which we may distribute simultaneous
181874aeea5SJeff Kirsher  * interrupt handling.
182874aeea5SJeff Kirsher  *
183874aeea5SJeff Kirsher  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
184cdb08f8fSBen Hutchings  * The default (0) means to assign an interrupt to each core.
185874aeea5SJeff Kirsher  */
186874aeea5SJeff Kirsher static unsigned int rss_cpus;
187874aeea5SJeff Kirsher module_param(rss_cpus, uint, 0444);
188874aeea5SJeff Kirsher MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
189874aeea5SJeff Kirsher 
190b9cc977dSBen Hutchings static bool phy_flash_cfg;
191b9cc977dSBen Hutchings module_param(phy_flash_cfg, bool, 0644);
192874aeea5SJeff Kirsher MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
193874aeea5SJeff Kirsher 
194e7bed9c8SBen Hutchings static unsigned irq_adapt_low_thresh = 8000;
195874aeea5SJeff Kirsher module_param(irq_adapt_low_thresh, uint, 0644);
196874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_low_thresh,
197874aeea5SJeff Kirsher 		 "Threshold score for reducing IRQ moderation");
198874aeea5SJeff Kirsher 
199e7bed9c8SBen Hutchings static unsigned irq_adapt_high_thresh = 16000;
200874aeea5SJeff Kirsher module_param(irq_adapt_high_thresh, uint, 0644);
201874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_high_thresh,
202874aeea5SJeff Kirsher 		 "Threshold score for increasing IRQ moderation");
203874aeea5SJeff Kirsher 
204874aeea5SJeff Kirsher static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
205874aeea5SJeff Kirsher 			 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
206874aeea5SJeff Kirsher 			 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
207874aeea5SJeff Kirsher 			 NETIF_MSG_TX_ERR | NETIF_MSG_HW);
208874aeea5SJeff Kirsher module_param(debug, uint, 0);
209874aeea5SJeff Kirsher MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
210874aeea5SJeff Kirsher 
211874aeea5SJeff Kirsher /**************************************************************************
212874aeea5SJeff Kirsher  *
213874aeea5SJeff Kirsher  * Utility functions and prototypes
214874aeea5SJeff Kirsher  *
215874aeea5SJeff Kirsher  *************************************************************************/
216874aeea5SJeff Kirsher 
217261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx);
218d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx);
2197f967c01SBen Hutchings static void efx_remove_channel(struct efx_channel *channel);
220874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx);
2217f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type;
222874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx);
2237f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel);
224874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx);
225874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel);
226874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx);
227874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx);
228874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx);
229e45a4fedSCharles McLachlan static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog);
230e45a4fedSCharles McLachlan static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp);
231874aeea5SJeff Kirsher 
232874aeea5SJeff Kirsher #define EFX_ASSERT_RESET_SERIALISED(efx)		\
233874aeea5SJeff Kirsher 	do {						\
234f16aeea0SBen Hutchings 		if ((efx->state == STATE_READY) ||	\
235626950dbSAlexandre Rames 		    (efx->state == STATE_RECOVERY) ||	\
236874aeea5SJeff Kirsher 		    (efx->state == STATE_DISABLED))	\
237874aeea5SJeff Kirsher 			ASSERT_RTNL();			\
238874aeea5SJeff Kirsher 	} while (0)
239874aeea5SJeff Kirsher 
2408b7325b4SBen Hutchings static int efx_check_disabled(struct efx_nic *efx)
2418b7325b4SBen Hutchings {
242626950dbSAlexandre Rames 	if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) {
2438b7325b4SBen Hutchings 		netif_err(efx, drv, efx->net_dev,
2448b7325b4SBen Hutchings 			  "device is disabled due to earlier errors\n");
2458b7325b4SBen Hutchings 		return -EIO;
2468b7325b4SBen Hutchings 	}
2478b7325b4SBen Hutchings 	return 0;
2488b7325b4SBen Hutchings }
2498b7325b4SBen Hutchings 
250874aeea5SJeff Kirsher /**************************************************************************
251874aeea5SJeff Kirsher  *
252874aeea5SJeff Kirsher  * Event queue processing
253874aeea5SJeff Kirsher  *
254874aeea5SJeff Kirsher  *************************************************************************/
255874aeea5SJeff Kirsher 
256874aeea5SJeff Kirsher /* Process channel's event queue
257874aeea5SJeff Kirsher  *
258874aeea5SJeff Kirsher  * This function is responsible for processing the event queue of a
259874aeea5SJeff Kirsher  * single channel.  The caller must guarantee that this function will
260874aeea5SJeff Kirsher  * never be concurrently called more than once on the same channel,
261874aeea5SJeff Kirsher  * though different channels may be being processed concurrently.
262874aeea5SJeff Kirsher  */
263874aeea5SJeff Kirsher static int efx_process_channel(struct efx_channel *channel, int budget)
264874aeea5SJeff Kirsher {
265c936835cSPeter Dunning 	struct efx_tx_queue *tx_queue;
266e090bfb9SEdward Cree 	struct list_head rx_list;
267874aeea5SJeff Kirsher 	int spent;
268874aeea5SJeff Kirsher 
2699f2cb71cSBen Hutchings 	if (unlikely(!channel->enabled))
270874aeea5SJeff Kirsher 		return 0;
271874aeea5SJeff Kirsher 
272e090bfb9SEdward Cree 	/* Prepare the batch receive list */
273e090bfb9SEdward Cree 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
274e090bfb9SEdward Cree 	INIT_LIST_HEAD(&rx_list);
275e090bfb9SEdward Cree 	channel->rx_list = &rx_list;
276e090bfb9SEdward Cree 
277c936835cSPeter Dunning 	efx_for_each_channel_tx_queue(tx_queue, channel) {
278c936835cSPeter Dunning 		tx_queue->pkts_compl = 0;
279c936835cSPeter Dunning 		tx_queue->bytes_compl = 0;
280c936835cSPeter Dunning 	}
281c936835cSPeter Dunning 
282874aeea5SJeff Kirsher 	spent = efx_nic_process_eventq(channel, budget);
283d9ab7007SBen Hutchings 	if (spent && efx_channel_has_rx_queue(channel)) {
284d9ab7007SBen Hutchings 		struct efx_rx_queue *rx_queue =
285d9ab7007SBen Hutchings 			efx_channel_get_rx_queue(channel);
286874aeea5SJeff Kirsher 
287ff734ef4SBen Hutchings 		efx_rx_flush_packet(channel);
288cce28794SJon Cooper 		efx_fast_push_rx_descriptors(rx_queue, true);
289d9ab7007SBen Hutchings 	}
290874aeea5SJeff Kirsher 
291c936835cSPeter Dunning 	/* Update BQL */
292c936835cSPeter Dunning 	efx_for_each_channel_tx_queue(tx_queue, channel) {
293c936835cSPeter Dunning 		if (tx_queue->bytes_compl) {
294c936835cSPeter Dunning 			netdev_tx_completed_queue(tx_queue->core_txq,
295c936835cSPeter Dunning 				tx_queue->pkts_compl, tx_queue->bytes_compl);
296c936835cSPeter Dunning 		}
297c936835cSPeter Dunning 	}
298c936835cSPeter Dunning 
299e090bfb9SEdward Cree 	/* Receive any packets we queued up */
300e090bfb9SEdward Cree 	netif_receive_skb_list(channel->rx_list);
301e090bfb9SEdward Cree 	channel->rx_list = NULL;
302e090bfb9SEdward Cree 
303874aeea5SJeff Kirsher 	return spent;
304874aeea5SJeff Kirsher }
305874aeea5SJeff Kirsher 
306874aeea5SJeff Kirsher /* NAPI poll handler
307874aeea5SJeff Kirsher  *
308874aeea5SJeff Kirsher  * NAPI guarantees serialisation of polls of the same device, which
309874aeea5SJeff Kirsher  * provides the guarantee required by efx_process_channel().
310874aeea5SJeff Kirsher  */
311539de7c5SBert Kenward static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
312539de7c5SBert Kenward {
313539de7c5SBert Kenward 	int step = efx->irq_mod_step_us;
314539de7c5SBert Kenward 
315539de7c5SBert Kenward 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
316539de7c5SBert Kenward 		if (channel->irq_moderation_us > step) {
317539de7c5SBert Kenward 			channel->irq_moderation_us -= step;
318539de7c5SBert Kenward 			efx->type->push_irq_moderation(channel);
319539de7c5SBert Kenward 		}
320539de7c5SBert Kenward 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
321539de7c5SBert Kenward 		if (channel->irq_moderation_us <
322539de7c5SBert Kenward 		    efx->irq_rx_moderation_us) {
323539de7c5SBert Kenward 			channel->irq_moderation_us += step;
324539de7c5SBert Kenward 			efx->type->push_irq_moderation(channel);
325539de7c5SBert Kenward 		}
326539de7c5SBert Kenward 	}
327539de7c5SBert Kenward 
328539de7c5SBert Kenward 	channel->irq_count = 0;
329539de7c5SBert Kenward 	channel->irq_mod_score = 0;
330539de7c5SBert Kenward }
331539de7c5SBert Kenward 
332874aeea5SJeff Kirsher static int efx_poll(struct napi_struct *napi, int budget)
333874aeea5SJeff Kirsher {
334874aeea5SJeff Kirsher 	struct efx_channel *channel =
335874aeea5SJeff Kirsher 		container_of(napi, struct efx_channel, napi_str);
336874aeea5SJeff Kirsher 	struct efx_nic *efx = channel->efx;
337874aeea5SJeff Kirsher 	int spent;
338874aeea5SJeff Kirsher 
339874aeea5SJeff Kirsher 	netif_vdbg(efx, intr, efx->net_dev,
340874aeea5SJeff Kirsher 		   "channel %d NAPI poll executing on CPU %d\n",
341874aeea5SJeff Kirsher 		   channel->channel, raw_smp_processor_id());
342874aeea5SJeff Kirsher 
343874aeea5SJeff Kirsher 	spent = efx_process_channel(channel, budget);
344874aeea5SJeff Kirsher 
345eb9a36beSCharles McLachlan 	xdp_do_flush_map();
346eb9a36beSCharles McLachlan 
347874aeea5SJeff Kirsher 	if (spent < budget) {
3489d9a6973SBen Hutchings 		if (efx_channel_has_rx_queue(channel) &&
349874aeea5SJeff Kirsher 		    efx->irq_rx_adaptive &&
350874aeea5SJeff Kirsher 		    unlikely(++channel->irq_count == 1000)) {
351539de7c5SBert Kenward 			efx_update_irq_mod(efx, channel);
352874aeea5SJeff Kirsher 		}
353874aeea5SJeff Kirsher 
3543af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL
3553af0f342SEdward Cree 		/* Perhaps expire some ARFS filters */
3563af0f342SEdward Cree 		schedule_work(&channel->filter_work);
3573af0f342SEdward Cree #endif
358874aeea5SJeff Kirsher 
359874aeea5SJeff Kirsher 		/* There is no race here; although napi_disable() will
360874aeea5SJeff Kirsher 		 * only wait for napi_complete(), this isn't a problem
361514bedbcSBen Hutchings 		 * since efx_nic_eventq_read_ack() will have no effect if
362874aeea5SJeff Kirsher 		 * interrupts have already been disabled.
363874aeea5SJeff Kirsher 		 */
364f820c0acSBert Kenward 		if (napi_complete_done(napi, spent))
365514bedbcSBen Hutchings 			efx_nic_eventq_read_ack(channel);
366874aeea5SJeff Kirsher 	}
367874aeea5SJeff Kirsher 
368874aeea5SJeff Kirsher 	return spent;
369874aeea5SJeff Kirsher }
370874aeea5SJeff Kirsher 
371874aeea5SJeff Kirsher /* Create event queue
372874aeea5SJeff Kirsher  * Event queue memory allocations are done only once.  If the channel
373874aeea5SJeff Kirsher  * is reset, the memory buffer will be reused; this guards against
374874aeea5SJeff Kirsher  * errors during channel reset and also simplifies interrupt handling.
375874aeea5SJeff Kirsher  */
376874aeea5SJeff Kirsher static int efx_probe_eventq(struct efx_channel *channel)
377874aeea5SJeff Kirsher {
378874aeea5SJeff Kirsher 	struct efx_nic *efx = channel->efx;
379874aeea5SJeff Kirsher 	unsigned long entries;
380874aeea5SJeff Kirsher 
38186ee5302SBen Hutchings 	netif_dbg(efx, probe, efx->net_dev,
382874aeea5SJeff Kirsher 		  "chan %d create event queue\n", channel->channel);
383874aeea5SJeff Kirsher 
384874aeea5SJeff Kirsher 	/* Build an event queue with room for one event per tx and rx buffer,
385874aeea5SJeff Kirsher 	 * plus some extra for link state events and MCDI completions. */
386874aeea5SJeff Kirsher 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
387e01b16a7SEdward Cree 	EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
388874aeea5SJeff Kirsher 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
389874aeea5SJeff Kirsher 
390874aeea5SJeff Kirsher 	return efx_nic_probe_eventq(channel);
391874aeea5SJeff Kirsher }
392874aeea5SJeff Kirsher 
393874aeea5SJeff Kirsher /* Prepare channel's event queue */
394261e4d96SJon Cooper static int efx_init_eventq(struct efx_channel *channel)
395874aeea5SJeff Kirsher {
39615acb1ceSBen Hutchings 	struct efx_nic *efx = channel->efx;
397261e4d96SJon Cooper 	int rc;
398261e4d96SJon Cooper 
399261e4d96SJon Cooper 	EFX_WARN_ON_PARANOID(channel->eventq_init);
400261e4d96SJon Cooper 
40115acb1ceSBen Hutchings 	netif_dbg(efx, drv, efx->net_dev,
402874aeea5SJeff Kirsher 		  "chan %d init event queue\n", channel->channel);
403874aeea5SJeff Kirsher 
404261e4d96SJon Cooper 	rc = efx_nic_init_eventq(channel);
405261e4d96SJon Cooper 	if (rc == 0) {
40615acb1ceSBen Hutchings 		efx->type->push_irq_moderation(channel);
407874aeea5SJeff Kirsher 		channel->eventq_read_ptr = 0;
408be3fc09cSBen Hutchings 		channel->eventq_init = true;
409874aeea5SJeff Kirsher 	}
410261e4d96SJon Cooper 	return rc;
411261e4d96SJon Cooper }
412874aeea5SJeff Kirsher 
4139f2cb71cSBen Hutchings /* Enable event queue processing and NAPI */
41436763266SAlexandre Rames void efx_start_eventq(struct efx_channel *channel)
4159f2cb71cSBen Hutchings {
4169f2cb71cSBen Hutchings 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
4179f2cb71cSBen Hutchings 		  "chan %d start event queue\n", channel->channel);
4189f2cb71cSBen Hutchings 
419514bedbcSBen Hutchings 	/* Make sure the NAPI handler sees the enabled flag set */
4209f2cb71cSBen Hutchings 	channel->enabled = true;
4219f2cb71cSBen Hutchings 	smp_wmb();
4229f2cb71cSBen Hutchings 
4239f2cb71cSBen Hutchings 	napi_enable(&channel->napi_str);
4249f2cb71cSBen Hutchings 	efx_nic_eventq_read_ack(channel);
4259f2cb71cSBen Hutchings }
4269f2cb71cSBen Hutchings 
4279f2cb71cSBen Hutchings /* Disable event queue processing and NAPI */
42836763266SAlexandre Rames void efx_stop_eventq(struct efx_channel *channel)
4299f2cb71cSBen Hutchings {
4309f2cb71cSBen Hutchings 	if (!channel->enabled)
4319f2cb71cSBen Hutchings 		return;
4329f2cb71cSBen Hutchings 
4339f2cb71cSBen Hutchings 	napi_disable(&channel->napi_str);
4349f2cb71cSBen Hutchings 	channel->enabled = false;
4359f2cb71cSBen Hutchings }
4369f2cb71cSBen Hutchings 
437874aeea5SJeff Kirsher static void efx_fini_eventq(struct efx_channel *channel)
438874aeea5SJeff Kirsher {
439be3fc09cSBen Hutchings 	if (!channel->eventq_init)
440be3fc09cSBen Hutchings 		return;
441be3fc09cSBen Hutchings 
442874aeea5SJeff Kirsher 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
443874aeea5SJeff Kirsher 		  "chan %d fini event queue\n", channel->channel);
444874aeea5SJeff Kirsher 
445874aeea5SJeff Kirsher 	efx_nic_fini_eventq(channel);
446be3fc09cSBen Hutchings 	channel->eventq_init = false;
447874aeea5SJeff Kirsher }
448874aeea5SJeff Kirsher 
449874aeea5SJeff Kirsher static void efx_remove_eventq(struct efx_channel *channel)
450874aeea5SJeff Kirsher {
451874aeea5SJeff Kirsher 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
452874aeea5SJeff Kirsher 		  "chan %d remove event queue\n", channel->channel);
453874aeea5SJeff Kirsher 
454874aeea5SJeff Kirsher 	efx_nic_remove_eventq(channel);
455874aeea5SJeff Kirsher }
456874aeea5SJeff Kirsher 
457874aeea5SJeff Kirsher /**************************************************************************
458874aeea5SJeff Kirsher  *
459874aeea5SJeff Kirsher  * Channel handling
460874aeea5SJeff Kirsher  *
461874aeea5SJeff Kirsher  *************************************************************************/
462874aeea5SJeff Kirsher 
4637f967c01SBen Hutchings /* Allocate and initialise a channel structure. */
464874aeea5SJeff Kirsher static struct efx_channel *
465874aeea5SJeff Kirsher efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
466874aeea5SJeff Kirsher {
467874aeea5SJeff Kirsher 	struct efx_channel *channel;
468874aeea5SJeff Kirsher 	struct efx_rx_queue *rx_queue;
469874aeea5SJeff Kirsher 	struct efx_tx_queue *tx_queue;
470874aeea5SJeff Kirsher 	int j;
471874aeea5SJeff Kirsher 
4727f967c01SBen Hutchings 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
4737f967c01SBen Hutchings 	if (!channel)
4747f967c01SBen Hutchings 		return NULL;
4757f967c01SBen Hutchings 
4767f967c01SBen Hutchings 	channel->efx = efx;
4777f967c01SBen Hutchings 	channel->channel = i;
4787f967c01SBen Hutchings 	channel->type = &efx_default_channel_type;
4797f967c01SBen Hutchings 
4807f967c01SBen Hutchings 	for (j = 0; j < EFX_TXQ_TYPES; j++) {
4817f967c01SBen Hutchings 		tx_queue = &channel->tx_queue[j];
4827f967c01SBen Hutchings 		tx_queue->efx = efx;
4837f967c01SBen Hutchings 		tx_queue->queue = i * EFX_TXQ_TYPES + j;
4847f967c01SBen Hutchings 		tx_queue->channel = channel;
4857f967c01SBen Hutchings 	}
4867f967c01SBen Hutchings 
4873af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL
4883af0f342SEdward Cree 	INIT_WORK(&channel->filter_work, efx_filter_rfs_expire);
4893af0f342SEdward Cree #endif
4903af0f342SEdward Cree 
4917f967c01SBen Hutchings 	rx_queue = &channel->rx_queue;
4927f967c01SBen Hutchings 	rx_queue->efx = efx;
4937aa1402eSKees Cook 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
4947f967c01SBen Hutchings 
4957f967c01SBen Hutchings 	return channel;
4967f967c01SBen Hutchings }
4977f967c01SBen Hutchings 
4987f967c01SBen Hutchings /* Allocate and initialise a channel structure, copying parameters
4997f967c01SBen Hutchings  * (but not resources) from an old channel structure.
5007f967c01SBen Hutchings  */
5017f967c01SBen Hutchings static struct efx_channel *
5027f967c01SBen Hutchings efx_copy_channel(const struct efx_channel *old_channel)
5037f967c01SBen Hutchings {
5047f967c01SBen Hutchings 	struct efx_channel *channel;
5057f967c01SBen Hutchings 	struct efx_rx_queue *rx_queue;
5067f967c01SBen Hutchings 	struct efx_tx_queue *tx_queue;
5077f967c01SBen Hutchings 	int j;
5087f967c01SBen Hutchings 
509874aeea5SJeff Kirsher 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
510874aeea5SJeff Kirsher 	if (!channel)
511874aeea5SJeff Kirsher 		return NULL;
512874aeea5SJeff Kirsher 
513874aeea5SJeff Kirsher 	*channel = *old_channel;
514874aeea5SJeff Kirsher 
515874aeea5SJeff Kirsher 	channel->napi_dev = NULL;
51646d054f8SBert Kenward 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
51746d054f8SBert Kenward 	channel->napi_str.napi_id = 0;
51846d054f8SBert Kenward 	channel->napi_str.state = 0;
519874aeea5SJeff Kirsher 	memset(&channel->eventq, 0, sizeof(channel->eventq));
520874aeea5SJeff Kirsher 
521874aeea5SJeff Kirsher 	for (j = 0; j < EFX_TXQ_TYPES; j++) {
522874aeea5SJeff Kirsher 		tx_queue = &channel->tx_queue[j];
523874aeea5SJeff Kirsher 		if (tx_queue->channel)
524874aeea5SJeff Kirsher 			tx_queue->channel = channel;
525874aeea5SJeff Kirsher 		tx_queue->buffer = NULL;
526874aeea5SJeff Kirsher 		memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
527874aeea5SJeff Kirsher 	}
528874aeea5SJeff Kirsher 
529874aeea5SJeff Kirsher 	rx_queue = &channel->rx_queue;
5307f967c01SBen Hutchings 	rx_queue->buffer = NULL;
5317f967c01SBen Hutchings 	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
5327aa1402eSKees Cook 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
5333af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL
5343af0f342SEdward Cree 	INIT_WORK(&channel->filter_work, efx_filter_rfs_expire);
5353af0f342SEdward Cree #endif
536874aeea5SJeff Kirsher 
537874aeea5SJeff Kirsher 	return channel;
538874aeea5SJeff Kirsher }
539874aeea5SJeff Kirsher 
540874aeea5SJeff Kirsher static int efx_probe_channel(struct efx_channel *channel)
541874aeea5SJeff Kirsher {
542874aeea5SJeff Kirsher 	struct efx_tx_queue *tx_queue;
543874aeea5SJeff Kirsher 	struct efx_rx_queue *rx_queue;
544874aeea5SJeff Kirsher 	int rc;
545874aeea5SJeff Kirsher 
546874aeea5SJeff Kirsher 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
547874aeea5SJeff Kirsher 		  "creating channel %d\n", channel->channel);
548874aeea5SJeff Kirsher 
5497f967c01SBen Hutchings 	rc = channel->type->pre_probe(channel);
5507f967c01SBen Hutchings 	if (rc)
5517f967c01SBen Hutchings 		goto fail;
5527f967c01SBen Hutchings 
553874aeea5SJeff Kirsher 	rc = efx_probe_eventq(channel);
554874aeea5SJeff Kirsher 	if (rc)
5557f967c01SBen Hutchings 		goto fail;
556874aeea5SJeff Kirsher 
557874aeea5SJeff Kirsher 	efx_for_each_channel_tx_queue(tx_queue, channel) {
558874aeea5SJeff Kirsher 		rc = efx_probe_tx_queue(tx_queue);
559874aeea5SJeff Kirsher 		if (rc)
5607f967c01SBen Hutchings 			goto fail;
561874aeea5SJeff Kirsher 	}
562874aeea5SJeff Kirsher 
563874aeea5SJeff Kirsher 	efx_for_each_channel_rx_queue(rx_queue, channel) {
564874aeea5SJeff Kirsher 		rc = efx_probe_rx_queue(rx_queue);
565874aeea5SJeff Kirsher 		if (rc)
5667f967c01SBen Hutchings 			goto fail;
567874aeea5SJeff Kirsher 	}
568874aeea5SJeff Kirsher 
569e090bfb9SEdward Cree 	channel->rx_list = NULL;
570e090bfb9SEdward Cree 
571874aeea5SJeff Kirsher 	return 0;
572874aeea5SJeff Kirsher 
5737f967c01SBen Hutchings fail:
5747f967c01SBen Hutchings 	efx_remove_channel(channel);
575874aeea5SJeff Kirsher 	return rc;
576874aeea5SJeff Kirsher }
577874aeea5SJeff Kirsher 
5787f967c01SBen Hutchings static void
5797f967c01SBen Hutchings efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
5807f967c01SBen Hutchings {
5817f967c01SBen Hutchings 	struct efx_nic *efx = channel->efx;
5827f967c01SBen Hutchings 	const char *type;
5837f967c01SBen Hutchings 	int number;
5847f967c01SBen Hutchings 
5857f967c01SBen Hutchings 	number = channel->channel;
5863990a8ffSCharles McLachlan 
5873990a8ffSCharles McLachlan 	if (number >= efx->xdp_channel_offset &&
5883990a8ffSCharles McLachlan 	    !WARN_ON_ONCE(!efx->n_xdp_channels)) {
5893990a8ffSCharles McLachlan 		type = "-xdp";
5903990a8ffSCharles McLachlan 		number -= efx->xdp_channel_offset;
5913990a8ffSCharles McLachlan 	} else if (efx->tx_channel_offset == 0) {
5927f967c01SBen Hutchings 		type = "";
5933990a8ffSCharles McLachlan 	} else if (number < efx->tx_channel_offset) {
5947f967c01SBen Hutchings 		type = "-rx";
5957f967c01SBen Hutchings 	} else {
5967f967c01SBen Hutchings 		type = "-tx";
5977f967c01SBen Hutchings 		number -= efx->tx_channel_offset;
5987f967c01SBen Hutchings 	}
5997f967c01SBen Hutchings 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);
6007f967c01SBen Hutchings }
601874aeea5SJeff Kirsher 
602874aeea5SJeff Kirsher static void efx_set_channel_names(struct efx_nic *efx)
603874aeea5SJeff Kirsher {
604874aeea5SJeff Kirsher 	struct efx_channel *channel;
605874aeea5SJeff Kirsher 
6067f967c01SBen Hutchings 	efx_for_each_channel(channel, efx)
6077f967c01SBen Hutchings 		channel->type->get_name(channel,
608d8291187SBen Hutchings 					efx->msi_context[channel->channel].name,
609d8291187SBen Hutchings 					sizeof(efx->msi_context[0].name));
610874aeea5SJeff Kirsher }
611874aeea5SJeff Kirsher 
612874aeea5SJeff Kirsher static int efx_probe_channels(struct efx_nic *efx)
613874aeea5SJeff Kirsher {
614874aeea5SJeff Kirsher 	struct efx_channel *channel;
615874aeea5SJeff Kirsher 	int rc;
616874aeea5SJeff Kirsher 
617874aeea5SJeff Kirsher 	/* Restart special buffer allocation */
618874aeea5SJeff Kirsher 	efx->next_buffer_table = 0;
619874aeea5SJeff Kirsher 
620c92aaff1SBen Hutchings 	/* Probe channels in reverse, so that any 'extra' channels
621c92aaff1SBen Hutchings 	 * use the start of the buffer table. This allows the traffic
622c92aaff1SBen Hutchings 	 * channels to be resized without moving them or wasting the
623c92aaff1SBen Hutchings 	 * entries before them.
624c92aaff1SBen Hutchings 	 */
625c92aaff1SBen Hutchings 	efx_for_each_channel_rev(channel, efx) {
626874aeea5SJeff Kirsher 		rc = efx_probe_channel(channel);
627874aeea5SJeff Kirsher 		if (rc) {
628874aeea5SJeff Kirsher 			netif_err(efx, probe, efx->net_dev,
629874aeea5SJeff Kirsher 				  "failed to create channel %d\n",
630874aeea5SJeff Kirsher 				  channel->channel);
631874aeea5SJeff Kirsher 			goto fail;
632874aeea5SJeff Kirsher 		}
633874aeea5SJeff Kirsher 	}
634874aeea5SJeff Kirsher 	efx_set_channel_names(efx);
635874aeea5SJeff Kirsher 
636874aeea5SJeff Kirsher 	return 0;
637874aeea5SJeff Kirsher 
638874aeea5SJeff Kirsher fail:
639874aeea5SJeff Kirsher 	efx_remove_channels(efx);
640874aeea5SJeff Kirsher 	return rc;
641874aeea5SJeff Kirsher }
642874aeea5SJeff Kirsher 
643874aeea5SJeff Kirsher /* Channels are shutdown and reinitialised whilst the NIC is running
644874aeea5SJeff Kirsher  * to propagate configuration changes (mtu, checksum offload), or
645874aeea5SJeff Kirsher  * to clear hardware error conditions
646874aeea5SJeff Kirsher  */
6479f2cb71cSBen Hutchings static void efx_start_datapath(struct efx_nic *efx)
648874aeea5SJeff Kirsher {
649ebfcd0fdSAndrew Rybchenko 	netdev_features_t old_features = efx->net_dev->features;
65085740cdfSBen Hutchings 	bool old_rx_scatter = efx->rx_scatter;
651874aeea5SJeff Kirsher 	struct efx_tx_queue *tx_queue;
652874aeea5SJeff Kirsher 	struct efx_rx_queue *rx_queue;
653874aeea5SJeff Kirsher 	struct efx_channel *channel;
65485740cdfSBen Hutchings 	size_t rx_buf_len;
655874aeea5SJeff Kirsher 
656874aeea5SJeff Kirsher 	/* Calculate the rx buffer allocation parameters required to
657874aeea5SJeff Kirsher 	 * support the current MTU, including padding for header
658874aeea5SJeff Kirsher 	 * alignment and overruns.
659874aeea5SJeff Kirsher 	 */
66043a3739dSJon Cooper 	efx->rx_dma_len = (efx->rx_prefix_size +
661874aeea5SJeff Kirsher 			   EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
662874aeea5SJeff Kirsher 			   efx->type->rx_buffer_padding);
663eb9a36beSCharles McLachlan 	rx_buf_len = (sizeof(struct efx_rx_page_state) + XDP_PACKET_HEADROOM +
6642ec03014SAndrew Rybchenko 		      efx->rx_ip_align + efx->rx_dma_len);
66585740cdfSBen Hutchings 	if (rx_buf_len <= PAGE_SIZE) {
666e8c68c0aSJon Cooper 		efx->rx_scatter = efx->type->always_rx_scatter;
66785740cdfSBen Hutchings 		efx->rx_buffer_order = 0;
66885740cdfSBen Hutchings 	} else if (efx->type->can_rx_scatter) {
669950c54dfSBen Hutchings 		BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES);
67085740cdfSBen Hutchings 		BUILD_BUG_ON(sizeof(struct efx_rx_page_state) +
671950c54dfSBen Hutchings 			     2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE,
672950c54dfSBen Hutchings 				       EFX_RX_BUF_ALIGNMENT) >
673950c54dfSBen Hutchings 			     PAGE_SIZE);
67485740cdfSBen Hutchings 		efx->rx_scatter = true;
67585740cdfSBen Hutchings 		efx->rx_dma_len = EFX_RX_USR_BUF_SIZE;
67685740cdfSBen Hutchings 		efx->rx_buffer_order = 0;
67785740cdfSBen Hutchings 	} else {
67885740cdfSBen Hutchings 		efx->rx_scatter = false;
67985740cdfSBen Hutchings 		efx->rx_buffer_order = get_order(rx_buf_len);
68085740cdfSBen Hutchings 	}
68185740cdfSBen Hutchings 
6821648a23fSDaniel Pieczko 	efx_rx_config_page_split(efx);
6831648a23fSDaniel Pieczko 	if (efx->rx_buffer_order)
6841648a23fSDaniel Pieczko 		netif_dbg(efx, drv, efx->net_dev,
6851648a23fSDaniel Pieczko 			  "RX buf len=%u; page order=%u batch=%u\n",
6861648a23fSDaniel Pieczko 			  efx->rx_dma_len, efx->rx_buffer_order,
6871648a23fSDaniel Pieczko 			  efx->rx_pages_per_batch);
6881648a23fSDaniel Pieczko 	else
6891648a23fSDaniel Pieczko 		netif_dbg(efx, drv, efx->net_dev,
6901648a23fSDaniel Pieczko 			  "RX buf len=%u step=%u bpp=%u; page batch=%u\n",
6911648a23fSDaniel Pieczko 			  efx->rx_dma_len, efx->rx_page_buf_step,
6921648a23fSDaniel Pieczko 			  efx->rx_bufs_per_page, efx->rx_pages_per_batch);
6932768935aSDaniel Pieczko 
694ebfcd0fdSAndrew Rybchenko 	/* Restore previously fixed features in hw_features and remove
695ebfcd0fdSAndrew Rybchenko 	 * features which are fixed now
696ebfcd0fdSAndrew Rybchenko 	 */
697ebfcd0fdSAndrew Rybchenko 	efx->net_dev->hw_features |= efx->net_dev->features;
698ebfcd0fdSAndrew Rybchenko 	efx->net_dev->hw_features &= ~efx->fixed_features;
699ebfcd0fdSAndrew Rybchenko 	efx->net_dev->features |= efx->fixed_features;
700ebfcd0fdSAndrew Rybchenko 	if (efx->net_dev->features != old_features)
701ebfcd0fdSAndrew Rybchenko 		netdev_features_change(efx->net_dev);
702ebfcd0fdSAndrew Rybchenko 
703e8c68c0aSJon Cooper 	/* RX filters may also have scatter-enabled flags */
70485740cdfSBen Hutchings 	if (efx->rx_scatter != old_rx_scatter)
705add72477SBen Hutchings 		efx->type->filter_update_rx_scatter(efx);
706874aeea5SJeff Kirsher 
70714bf718fSBen Hutchings 	/* We must keep at least one descriptor in a TX ring empty.
70814bf718fSBen Hutchings 	 * We could avoid this when the queue size does not exactly
70914bf718fSBen Hutchings 	 * match the hardware ring size, but it's not that important.
71014bf718fSBen Hutchings 	 * Therefore we stop the queue when one more skb might fill
71114bf718fSBen Hutchings 	 * the ring completely.  We wake it when half way back to
71214bf718fSBen Hutchings 	 * empty.
71314bf718fSBen Hutchings 	 */
71414bf718fSBen Hutchings 	efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx);
71514bf718fSBen Hutchings 	efx->txq_wake_thresh = efx->txq_stop_thresh / 2;
71614bf718fSBen Hutchings 
717874aeea5SJeff Kirsher 	/* Initialise the channels */
718874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx) {
7193881d8abSAlexandre Rames 		efx_for_each_channel_tx_queue(tx_queue, channel) {
720874aeea5SJeff Kirsher 			efx_init_tx_queue(tx_queue);
7213881d8abSAlexandre Rames 			atomic_inc(&efx->active_queues);
7223881d8abSAlexandre Rames 		}
723874aeea5SJeff Kirsher 
7249f2cb71cSBen Hutchings 		efx_for_each_channel_rx_queue(rx_queue, channel) {
725874aeea5SJeff Kirsher 			efx_init_rx_queue(rx_queue);
7263881d8abSAlexandre Rames 			atomic_inc(&efx->active_queues);
727cce28794SJon Cooper 			efx_stop_eventq(channel);
728cce28794SJon Cooper 			efx_fast_push_rx_descriptors(rx_queue, false);
729cce28794SJon Cooper 			efx_start_eventq(channel);
7309f2cb71cSBen Hutchings 		}
731874aeea5SJeff Kirsher 
73285740cdfSBen Hutchings 		WARN_ON(channel->rx_pkt_n_frags);
733874aeea5SJeff Kirsher 	}
7349f2cb71cSBen Hutchings 
7352ea4dc28SAlexandre Rames 	efx_ptp_start_datapath(efx);
7362ea4dc28SAlexandre Rames 
7379f2cb71cSBen Hutchings 	if (netif_device_present(efx->net_dev))
7389f2cb71cSBen Hutchings 		netif_tx_wake_all_queues(efx->net_dev);
739874aeea5SJeff Kirsher }
740874aeea5SJeff Kirsher 
7419f2cb71cSBen Hutchings static void efx_stop_datapath(struct efx_nic *efx)
742874aeea5SJeff Kirsher {
743874aeea5SJeff Kirsher 	struct efx_channel *channel;
744874aeea5SJeff Kirsher 	struct efx_tx_queue *tx_queue;
745874aeea5SJeff Kirsher 	struct efx_rx_queue *rx_queue;
746874aeea5SJeff Kirsher 	int rc;
747874aeea5SJeff Kirsher 
748874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
749874aeea5SJeff Kirsher 	BUG_ON(efx->port_enabled);
750874aeea5SJeff Kirsher 
7512ea4dc28SAlexandre Rames 	efx_ptp_stop_datapath(efx);
7522ea4dc28SAlexandre Rames 
753d8aec745SBen Hutchings 	/* Stop RX refill */
754d8aec745SBen Hutchings 	efx_for_each_channel(channel, efx) {
755d8aec745SBen Hutchings 		efx_for_each_channel_rx_queue(rx_queue, channel)
756d8aec745SBen Hutchings 			rx_queue->refill_enabled = false;
757d8aec745SBen Hutchings 	}
758d8aec745SBen Hutchings 
759874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx) {
7609f2cb71cSBen Hutchings 		/* RX packet processing is pipelined, so wait for the
7619f2cb71cSBen Hutchings 		 * NAPI handler to complete.  At least event queue 0
7629f2cb71cSBen Hutchings 		 * might be kept active by non-data events, so don't
7639f2cb71cSBen Hutchings 		 * use napi_synchronize() but actually disable NAPI
7649f2cb71cSBen Hutchings 		 * temporarily.
7659f2cb71cSBen Hutchings 		 */
7669f2cb71cSBen Hutchings 		if (efx_channel_has_rx_queue(channel)) {
7679f2cb71cSBen Hutchings 			efx_stop_eventq(channel);
7689f2cb71cSBen Hutchings 			efx_start_eventq(channel);
7699f2cb71cSBen Hutchings 		}
770e42c3d85SBen Hutchings 	}
771874aeea5SJeff Kirsher 
772e42c3d85SBen Hutchings 	rc = efx->type->fini_dmaq(efx);
7735a6681e2SEdward Cree 	if (rc) {
774e42c3d85SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
775e42c3d85SBen Hutchings 	} else {
776e42c3d85SBen Hutchings 		netif_dbg(efx, drv, efx->net_dev,
777e42c3d85SBen Hutchings 			  "successfully flushed all queues\n");
778e42c3d85SBen Hutchings 	}
779e42c3d85SBen Hutchings 
780e42c3d85SBen Hutchings 	efx_for_each_channel(channel, efx) {
781874aeea5SJeff Kirsher 		efx_for_each_channel_rx_queue(rx_queue, channel)
782874aeea5SJeff Kirsher 			efx_fini_rx_queue(rx_queue);
783874aeea5SJeff Kirsher 		efx_for_each_possible_channel_tx_queue(tx_queue, channel)
784874aeea5SJeff Kirsher 			efx_fini_tx_queue(tx_queue);
785874aeea5SJeff Kirsher 	}
786eb9a36beSCharles McLachlan 	efx->xdp_rxq_info_failed = false;
787874aeea5SJeff Kirsher }
788874aeea5SJeff Kirsher 
789874aeea5SJeff Kirsher static void efx_remove_channel(struct efx_channel *channel)
790874aeea5SJeff Kirsher {
791874aeea5SJeff Kirsher 	struct efx_tx_queue *tx_queue;
792874aeea5SJeff Kirsher 	struct efx_rx_queue *rx_queue;
793874aeea5SJeff Kirsher 
794874aeea5SJeff Kirsher 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
795874aeea5SJeff Kirsher 		  "destroy chan %d\n", channel->channel);
796874aeea5SJeff Kirsher 
797874aeea5SJeff Kirsher 	efx_for_each_channel_rx_queue(rx_queue, channel)
798874aeea5SJeff Kirsher 		efx_remove_rx_queue(rx_queue);
799874aeea5SJeff Kirsher 	efx_for_each_possible_channel_tx_queue(tx_queue, channel)
800874aeea5SJeff Kirsher 		efx_remove_tx_queue(tx_queue);
801874aeea5SJeff Kirsher 	efx_remove_eventq(channel);
802c31e5f9fSStuart Hodgson 	channel->type->post_remove(channel);
803874aeea5SJeff Kirsher }
804874aeea5SJeff Kirsher 
805874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx)
806874aeea5SJeff Kirsher {
807874aeea5SJeff Kirsher 	struct efx_channel *channel;
808874aeea5SJeff Kirsher 
809874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx)
810874aeea5SJeff Kirsher 		efx_remove_channel(channel);
8113990a8ffSCharles McLachlan 
8123990a8ffSCharles McLachlan 	kfree(efx->xdp_tx_queues);
813874aeea5SJeff Kirsher }
814874aeea5SJeff Kirsher 
815874aeea5SJeff Kirsher int
816874aeea5SJeff Kirsher efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
817874aeea5SJeff Kirsher {
818874aeea5SJeff Kirsher 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
819874aeea5SJeff Kirsher 	u32 old_rxq_entries, old_txq_entries;
8207f967c01SBen Hutchings 	unsigned i, next_buffer_table = 0;
821261e4d96SJon Cooper 	int rc, rc2;
8228b7325b4SBen Hutchings 
8238b7325b4SBen Hutchings 	rc = efx_check_disabled(efx);
8248b7325b4SBen Hutchings 	if (rc)
8258b7325b4SBen Hutchings 		return rc;
8267f967c01SBen Hutchings 
8277f967c01SBen Hutchings 	/* Not all channels should be reallocated. We must avoid
8287f967c01SBen Hutchings 	 * reallocating their buffer table entries.
8297f967c01SBen Hutchings 	 */
8307f967c01SBen Hutchings 	efx_for_each_channel(channel, efx) {
8317f967c01SBen Hutchings 		struct efx_rx_queue *rx_queue;
8327f967c01SBen Hutchings 		struct efx_tx_queue *tx_queue;
8337f967c01SBen Hutchings 
8347f967c01SBen Hutchings 		if (channel->type->copy)
8357f967c01SBen Hutchings 			continue;
8367f967c01SBen Hutchings 		next_buffer_table = max(next_buffer_table,
8377f967c01SBen Hutchings 					channel->eventq.index +
8387f967c01SBen Hutchings 					channel->eventq.entries);
8397f967c01SBen Hutchings 		efx_for_each_channel_rx_queue(rx_queue, channel)
8407f967c01SBen Hutchings 			next_buffer_table = max(next_buffer_table,
8417f967c01SBen Hutchings 						rx_queue->rxd.index +
8427f967c01SBen Hutchings 						rx_queue->rxd.entries);
8437f967c01SBen Hutchings 		efx_for_each_channel_tx_queue(tx_queue, channel)
8447f967c01SBen Hutchings 			next_buffer_table = max(next_buffer_table,
8457f967c01SBen Hutchings 						tx_queue->txd.index +
8467f967c01SBen Hutchings 						tx_queue->txd.entries);
8477f967c01SBen Hutchings 	}
848874aeea5SJeff Kirsher 
84929c69a48SBen Hutchings 	efx_device_detach_sync(efx);
850874aeea5SJeff Kirsher 	efx_stop_all(efx);
851d8291187SBen Hutchings 	efx_soft_disable_interrupts(efx);
852874aeea5SJeff Kirsher 
8537f967c01SBen Hutchings 	/* Clone channels (where possible) */
854874aeea5SJeff Kirsher 	memset(other_channel, 0, sizeof(other_channel));
855874aeea5SJeff Kirsher 	for (i = 0; i < efx->n_channels; i++) {
8567f967c01SBen Hutchings 		channel = efx->channel[i];
8577f967c01SBen Hutchings 		if (channel->type->copy)
8587f967c01SBen Hutchings 			channel = channel->type->copy(channel);
859874aeea5SJeff Kirsher 		if (!channel) {
860874aeea5SJeff Kirsher 			rc = -ENOMEM;
861874aeea5SJeff Kirsher 			goto out;
862874aeea5SJeff Kirsher 		}
863874aeea5SJeff Kirsher 		other_channel[i] = channel;
864874aeea5SJeff Kirsher 	}
865874aeea5SJeff Kirsher 
866874aeea5SJeff Kirsher 	/* Swap entry counts and channel pointers */
867874aeea5SJeff Kirsher 	old_rxq_entries = efx->rxq_entries;
868874aeea5SJeff Kirsher 	old_txq_entries = efx->txq_entries;
869874aeea5SJeff Kirsher 	efx->rxq_entries = rxq_entries;
870874aeea5SJeff Kirsher 	efx->txq_entries = txq_entries;
871874aeea5SJeff Kirsher 	for (i = 0; i < efx->n_channels; i++) {
872874aeea5SJeff Kirsher 		channel = efx->channel[i];
873874aeea5SJeff Kirsher 		efx->channel[i] = other_channel[i];
874874aeea5SJeff Kirsher 		other_channel[i] = channel;
875874aeea5SJeff Kirsher 	}
876874aeea5SJeff Kirsher 
8777f967c01SBen Hutchings 	/* Restart buffer table allocation */
8787f967c01SBen Hutchings 	efx->next_buffer_table = next_buffer_table;
8797f967c01SBen Hutchings 
8807f967c01SBen Hutchings 	for (i = 0; i < efx->n_channels; i++) {
8817f967c01SBen Hutchings 		channel = efx->channel[i];
8827f967c01SBen Hutchings 		if (!channel->type->copy)
8837f967c01SBen Hutchings 			continue;
8847f967c01SBen Hutchings 		rc = efx_probe_channel(channel);
885874aeea5SJeff Kirsher 		if (rc)
886874aeea5SJeff Kirsher 			goto rollback;
8877f967c01SBen Hutchings 		efx_init_napi_channel(efx->channel[i]);
888874aeea5SJeff Kirsher 	}
889874aeea5SJeff Kirsher 
8907f967c01SBen Hutchings out:
8917f967c01SBen Hutchings 	/* Destroy unused channel structures */
8927f967c01SBen Hutchings 	for (i = 0; i < efx->n_channels; i++) {
8937f967c01SBen Hutchings 		channel = other_channel[i];
8947f967c01SBen Hutchings 		if (channel && channel->type->copy) {
8957f967c01SBen Hutchings 			efx_fini_napi_channel(channel);
8967f967c01SBen Hutchings 			efx_remove_channel(channel);
8977f967c01SBen Hutchings 			kfree(channel);
8987f967c01SBen Hutchings 		}
8997f967c01SBen Hutchings 	}
9007f967c01SBen Hutchings 
901261e4d96SJon Cooper 	rc2 = efx_soft_enable_interrupts(efx);
902261e4d96SJon Cooper 	if (rc2) {
903261e4d96SJon Cooper 		rc = rc ? rc : rc2;
904261e4d96SJon Cooper 		netif_err(efx, drv, efx->net_dev,
905261e4d96SJon Cooper 			  "unable to restart interrupts on channel reallocation\n");
906261e4d96SJon Cooper 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);
907261e4d96SJon Cooper 	} else {
908874aeea5SJeff Kirsher 		efx_start_all(efx);
9099c568fd8SPeter Dunning 		efx_device_attach_if_not_resetting(efx);
910261e4d96SJon Cooper 	}
911874aeea5SJeff Kirsher 	return rc;
912874aeea5SJeff Kirsher 
913874aeea5SJeff Kirsher rollback:
914874aeea5SJeff Kirsher 	/* Swap back */
915874aeea5SJeff Kirsher 	efx->rxq_entries = old_rxq_entries;
916874aeea5SJeff Kirsher 	efx->txq_entries = old_txq_entries;
917874aeea5SJeff Kirsher 	for (i = 0; i < efx->n_channels; i++) {
918874aeea5SJeff Kirsher 		channel = efx->channel[i];
919874aeea5SJeff Kirsher 		efx->channel[i] = other_channel[i];
920874aeea5SJeff Kirsher 		other_channel[i] = channel;
921874aeea5SJeff Kirsher 	}
922874aeea5SJeff Kirsher 	goto out;
923874aeea5SJeff Kirsher }
924874aeea5SJeff Kirsher 
925874aeea5SJeff Kirsher void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
926874aeea5SJeff Kirsher {
92750f444aaSRobert Stonehouse 	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));
928874aeea5SJeff Kirsher }
929874aeea5SJeff Kirsher 
930e7345ba3Skbuild test robot static bool efx_default_channel_want_txqs(struct efx_channel *channel)
9312935e3c3SEdward Cree {
9322935e3c3SEdward Cree 	return channel->channel - channel->efx->tx_channel_offset <
9332935e3c3SEdward Cree 		channel->efx->n_tx_channels;
9342935e3c3SEdward Cree }
9352935e3c3SEdward Cree 
9367f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type = {
9377f967c01SBen Hutchings 	.pre_probe		= efx_channel_dummy_op_int,
938c31e5f9fSStuart Hodgson 	.post_remove		= efx_channel_dummy_op_void,
9397f967c01SBen Hutchings 	.get_name		= efx_get_channel_name,
9407f967c01SBen Hutchings 	.copy			= efx_copy_channel,
9412935e3c3SEdward Cree 	.want_txqs		= efx_default_channel_want_txqs,
9427f967c01SBen Hutchings 	.keep_eventq		= false,
9432935e3c3SEdward Cree 	.want_pio		= true,
9447f967c01SBen Hutchings };
9457f967c01SBen Hutchings 
9467f967c01SBen Hutchings int efx_channel_dummy_op_int(struct efx_channel *channel)
9477f967c01SBen Hutchings {
9487f967c01SBen Hutchings 	return 0;
9497f967c01SBen Hutchings }
9507f967c01SBen Hutchings 
951c31e5f9fSStuart Hodgson void efx_channel_dummy_op_void(struct efx_channel *channel)
952c31e5f9fSStuart Hodgson {
953c31e5f9fSStuart Hodgson }
954c31e5f9fSStuart Hodgson 
955874aeea5SJeff Kirsher /**************************************************************************
956874aeea5SJeff Kirsher  *
957874aeea5SJeff Kirsher  * Port handling
958874aeea5SJeff Kirsher  *
959874aeea5SJeff Kirsher  **************************************************************************/
960874aeea5SJeff Kirsher 
961874aeea5SJeff Kirsher /* This ensures that the kernel is kept informed (via
962874aeea5SJeff Kirsher  * netif_carrier_on/off) of the link status, and also maintains the
963874aeea5SJeff Kirsher  * link status's stop on the port's TX queue.
964874aeea5SJeff Kirsher  */
965874aeea5SJeff Kirsher void efx_link_status_changed(struct efx_nic *efx)
966874aeea5SJeff Kirsher {
967874aeea5SJeff Kirsher 	struct efx_link_state *link_state = &efx->link_state;
968874aeea5SJeff Kirsher 
969874aeea5SJeff Kirsher 	/* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
970874aeea5SJeff Kirsher 	 * that no events are triggered between unregister_netdev() and the
971874aeea5SJeff Kirsher 	 * driver unloading. A more general condition is that NETDEV_CHANGE
972874aeea5SJeff Kirsher 	 * can only be generated between NETDEV_UP and NETDEV_DOWN */
973874aeea5SJeff Kirsher 	if (!netif_running(efx->net_dev))
974874aeea5SJeff Kirsher 		return;
975874aeea5SJeff Kirsher 
976874aeea5SJeff Kirsher 	if (link_state->up != netif_carrier_ok(efx->net_dev)) {
977874aeea5SJeff Kirsher 		efx->n_link_state_changes++;
978874aeea5SJeff Kirsher 
979874aeea5SJeff Kirsher 		if (link_state->up)
980874aeea5SJeff Kirsher 			netif_carrier_on(efx->net_dev);
981874aeea5SJeff Kirsher 		else
982874aeea5SJeff Kirsher 			netif_carrier_off(efx->net_dev);
983874aeea5SJeff Kirsher 	}
984874aeea5SJeff Kirsher 
985874aeea5SJeff Kirsher 	/* Status message for kernel log */
9862aa9ef11SBen Hutchings 	if (link_state->up)
987874aeea5SJeff Kirsher 		netif_info(efx, link, efx->net_dev,
988964e6135SBen Hutchings 			   "link up at %uMbps %s-duplex (MTU %d)\n",
989874aeea5SJeff Kirsher 			   link_state->speed, link_state->fd ? "full" : "half",
990964e6135SBen Hutchings 			   efx->net_dev->mtu);
9912aa9ef11SBen Hutchings 	else
992874aeea5SJeff Kirsher 		netif_info(efx, link, efx->net_dev, "link down\n");
993874aeea5SJeff Kirsher }
994874aeea5SJeff Kirsher 
995c2ab85d2SEdward Cree void efx_link_set_advertising(struct efx_nic *efx,
996c2ab85d2SEdward Cree 			      const unsigned long *advertising)
997874aeea5SJeff Kirsher {
998c2ab85d2SEdward Cree 	memcpy(efx->link_advertising, advertising,
999c2ab85d2SEdward Cree 	       sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
1000c2ab85d2SEdward Cree 
1001c2ab85d2SEdward Cree 	efx->link_advertising[0] |= ADVERTISED_Autoneg;
1002c2ab85d2SEdward Cree 	if (advertising[0] & ADVERTISED_Pause)
1003874aeea5SJeff Kirsher 		efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
1004874aeea5SJeff Kirsher 	else
1005874aeea5SJeff Kirsher 		efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
1006c2ab85d2SEdward Cree 	if (advertising[0] & ADVERTISED_Asym_Pause)
1007874aeea5SJeff Kirsher 		efx->wanted_fc ^= EFX_FC_TX;
1008874aeea5SJeff Kirsher }
1009c2ab85d2SEdward Cree 
1010c2ab85d2SEdward Cree /* Equivalent to efx_link_set_advertising with all-zeroes, except does not
1011c2ab85d2SEdward Cree  * force the Autoneg bit on.
1012c2ab85d2SEdward Cree  */
1013c2ab85d2SEdward Cree void efx_link_clear_advertising(struct efx_nic *efx)
1014c2ab85d2SEdward Cree {
1015c2ab85d2SEdward Cree 	bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS);
1016c2ab85d2SEdward Cree 	efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
1017874aeea5SJeff Kirsher }
1018874aeea5SJeff Kirsher 
1019874aeea5SJeff Kirsher void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
1020874aeea5SJeff Kirsher {
1021874aeea5SJeff Kirsher 	efx->wanted_fc = wanted_fc;
1022c2ab85d2SEdward Cree 	if (efx->link_advertising[0]) {
1023874aeea5SJeff Kirsher 		if (wanted_fc & EFX_FC_RX)
1024c2ab85d2SEdward Cree 			efx->link_advertising[0] |= (ADVERTISED_Pause |
1025874aeea5SJeff Kirsher 						     ADVERTISED_Asym_Pause);
1026874aeea5SJeff Kirsher 		else
1027c2ab85d2SEdward Cree 			efx->link_advertising[0] &= ~(ADVERTISED_Pause |
1028874aeea5SJeff Kirsher 						      ADVERTISED_Asym_Pause);
1029874aeea5SJeff Kirsher 		if (wanted_fc & EFX_FC_TX)
1030c2ab85d2SEdward Cree 			efx->link_advertising[0] ^= ADVERTISED_Asym_Pause;
1031874aeea5SJeff Kirsher 	}
1032874aeea5SJeff Kirsher }
1033874aeea5SJeff Kirsher 
1034874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx);
1035874aeea5SJeff Kirsher 
10360d322413SEdward Cree /* We assume that efx->type->reconfigure_mac will always try to sync RX
10370d322413SEdward Cree  * filters and therefore needs to read-lock the filter table against freeing
10380d322413SEdward Cree  */
10390d322413SEdward Cree void efx_mac_reconfigure(struct efx_nic *efx)
10400d322413SEdward Cree {
10410d322413SEdward Cree 	down_read(&efx->filter_sem);
10420d322413SEdward Cree 	efx->type->reconfigure_mac(efx);
10430d322413SEdward Cree 	up_read(&efx->filter_sem);
10440d322413SEdward Cree }
10450d322413SEdward Cree 
1046874aeea5SJeff Kirsher /* Push loopback/power/transmit disable settings to the PHY, and reconfigure
1047874aeea5SJeff Kirsher  * the MAC appropriately. All other PHY configuration changes are pushed
1048874aeea5SJeff Kirsher  * through phy_op->set_settings(), and pushed asynchronously to the MAC
1049874aeea5SJeff Kirsher  * through efx_monitor().
1050874aeea5SJeff Kirsher  *
1051874aeea5SJeff Kirsher  * Callers must hold the mac_lock
1052874aeea5SJeff Kirsher  */
1053874aeea5SJeff Kirsher int __efx_reconfigure_port(struct efx_nic *efx)
1054874aeea5SJeff Kirsher {
1055874aeea5SJeff Kirsher 	enum efx_phy_mode phy_mode;
1056874aeea5SJeff Kirsher 	int rc;
1057874aeea5SJeff Kirsher 
1058874aeea5SJeff Kirsher 	WARN_ON(!mutex_is_locked(&efx->mac_lock));
1059874aeea5SJeff Kirsher 
1060874aeea5SJeff Kirsher 	/* Disable PHY transmit in mac level loopbacks */
1061874aeea5SJeff Kirsher 	phy_mode = efx->phy_mode;
1062874aeea5SJeff Kirsher 	if (LOOPBACK_INTERNAL(efx))
1063874aeea5SJeff Kirsher 		efx->phy_mode |= PHY_MODE_TX_DISABLED;
1064874aeea5SJeff Kirsher 	else
1065874aeea5SJeff Kirsher 		efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
1066874aeea5SJeff Kirsher 
1067874aeea5SJeff Kirsher 	rc = efx->type->reconfigure_port(efx);
1068874aeea5SJeff Kirsher 
1069874aeea5SJeff Kirsher 	if (rc)
1070874aeea5SJeff Kirsher 		efx->phy_mode = phy_mode;
1071874aeea5SJeff Kirsher 
1072874aeea5SJeff Kirsher 	return rc;
1073874aeea5SJeff Kirsher }
1074874aeea5SJeff Kirsher 
1075874aeea5SJeff Kirsher /* Reinitialise the MAC to pick up new PHY settings, even if the port is
1076874aeea5SJeff Kirsher  * disabled. */
1077874aeea5SJeff Kirsher int efx_reconfigure_port(struct efx_nic *efx)
1078874aeea5SJeff Kirsher {
1079874aeea5SJeff Kirsher 	int rc;
1080874aeea5SJeff Kirsher 
1081874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
1082874aeea5SJeff Kirsher 
1083874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
1084874aeea5SJeff Kirsher 	rc = __efx_reconfigure_port(efx);
1085874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
1086874aeea5SJeff Kirsher 
1087874aeea5SJeff Kirsher 	return rc;
1088874aeea5SJeff Kirsher }
1089874aeea5SJeff Kirsher 
1090874aeea5SJeff Kirsher /* Asynchronous work item for changing MAC promiscuity and multicast
1091874aeea5SJeff Kirsher  * hash.  Avoid a drain/rx_ingress enable by reconfiguring the current
1092874aeea5SJeff Kirsher  * MAC directly. */
1093874aeea5SJeff Kirsher static void efx_mac_work(struct work_struct *data)
1094874aeea5SJeff Kirsher {
1095874aeea5SJeff Kirsher 	struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
1096874aeea5SJeff Kirsher 
1097874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
109830b81cdaSBen Hutchings 	if (efx->port_enabled)
10990d322413SEdward Cree 		efx_mac_reconfigure(efx);
1100874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
1101874aeea5SJeff Kirsher }
1102874aeea5SJeff Kirsher 
1103874aeea5SJeff Kirsher static int efx_probe_port(struct efx_nic *efx)
1104874aeea5SJeff Kirsher {
1105874aeea5SJeff Kirsher 	int rc;
1106874aeea5SJeff Kirsher 
1107874aeea5SJeff Kirsher 	netif_dbg(efx, probe, efx->net_dev, "create port\n");
1108874aeea5SJeff Kirsher 
1109874aeea5SJeff Kirsher 	if (phy_flash_cfg)
1110874aeea5SJeff Kirsher 		efx->phy_mode = PHY_MODE_SPECIAL;
1111874aeea5SJeff Kirsher 
1112874aeea5SJeff Kirsher 	/* Connect up MAC/PHY operations table */
1113874aeea5SJeff Kirsher 	rc = efx->type->probe_port(efx);
1114874aeea5SJeff Kirsher 	if (rc)
1115874aeea5SJeff Kirsher 		return rc;
1116874aeea5SJeff Kirsher 
1117e332bcb3SBen Hutchings 	/* Initialise MAC address to permanent address */
1118cd84ff4dSEdward Cree 	ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr);
1119874aeea5SJeff Kirsher 
1120874aeea5SJeff Kirsher 	return 0;
1121874aeea5SJeff Kirsher }
1122874aeea5SJeff Kirsher 
1123874aeea5SJeff Kirsher static int efx_init_port(struct efx_nic *efx)
1124874aeea5SJeff Kirsher {
1125874aeea5SJeff Kirsher 	int rc;
1126874aeea5SJeff Kirsher 
1127874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "init port\n");
1128874aeea5SJeff Kirsher 
1129874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
1130874aeea5SJeff Kirsher 
1131874aeea5SJeff Kirsher 	rc = efx->phy_op->init(efx);
1132874aeea5SJeff Kirsher 	if (rc)
1133874aeea5SJeff Kirsher 		goto fail1;
1134874aeea5SJeff Kirsher 
1135874aeea5SJeff Kirsher 	efx->port_initialized = true;
1136874aeea5SJeff Kirsher 
1137874aeea5SJeff Kirsher 	/* Reconfigure the MAC before creating dma queues (required for
1138874aeea5SJeff Kirsher 	 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
11390d322413SEdward Cree 	efx_mac_reconfigure(efx);
1140874aeea5SJeff Kirsher 
1141874aeea5SJeff Kirsher 	/* Ensure the PHY advertises the correct flow control settings */
1142874aeea5SJeff Kirsher 	rc = efx->phy_op->reconfigure(efx);
1143267d9d73SEdward Cree 	if (rc && rc != -EPERM)
1144874aeea5SJeff Kirsher 		goto fail2;
1145874aeea5SJeff Kirsher 
1146874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
1147874aeea5SJeff Kirsher 	return 0;
1148874aeea5SJeff Kirsher 
1149874aeea5SJeff Kirsher fail2:
1150874aeea5SJeff Kirsher 	efx->phy_op->fini(efx);
1151874aeea5SJeff Kirsher fail1:
1152874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
1153874aeea5SJeff Kirsher 	return rc;
1154874aeea5SJeff Kirsher }
1155874aeea5SJeff Kirsher 
1156874aeea5SJeff Kirsher static void efx_start_port(struct efx_nic *efx)
1157874aeea5SJeff Kirsher {
1158874aeea5SJeff Kirsher 	netif_dbg(efx, ifup, efx->net_dev, "start port\n");
1159874aeea5SJeff Kirsher 	BUG_ON(efx->port_enabled);
1160874aeea5SJeff Kirsher 
1161874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
1162874aeea5SJeff Kirsher 	efx->port_enabled = true;
1163874aeea5SJeff Kirsher 
1164d615c039SBen Hutchings 	/* Ensure MAC ingress/egress is enabled */
11650d322413SEdward Cree 	efx_mac_reconfigure(efx);
1166874aeea5SJeff Kirsher 
1167874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
1168874aeea5SJeff Kirsher }
1169874aeea5SJeff Kirsher 
1170d615c039SBen Hutchings /* Cancel work for MAC reconfiguration, periodic hardware monitoring
1171d615c039SBen Hutchings  * and the async self-test, wait for them to finish and prevent them
1172d615c039SBen Hutchings  * being scheduled again.  This doesn't cover online resets, which
1173d615c039SBen Hutchings  * should only be cancelled when removing the device.
1174d615c039SBen Hutchings  */
1175874aeea5SJeff Kirsher static void efx_stop_port(struct efx_nic *efx)
1176874aeea5SJeff Kirsher {
1177874aeea5SJeff Kirsher 	netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
1178874aeea5SJeff Kirsher 
1179d615c039SBen Hutchings 	EFX_ASSERT_RESET_SERIALISED(efx);
1180d615c039SBen Hutchings 
1181874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
1182874aeea5SJeff Kirsher 	efx->port_enabled = false;
1183874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
1184874aeea5SJeff Kirsher 
1185874aeea5SJeff Kirsher 	/* Serialise against efx_set_multicast_list() */
1186874aeea5SJeff Kirsher 	netif_addr_lock_bh(efx->net_dev);
1187874aeea5SJeff Kirsher 	netif_addr_unlock_bh(efx->net_dev);
1188d615c039SBen Hutchings 
1189d615c039SBen Hutchings 	cancel_delayed_work_sync(&efx->monitor_work);
1190d615c039SBen Hutchings 	efx_selftest_async_cancel(efx);
1191d615c039SBen Hutchings 	cancel_work_sync(&efx->mac_work);
1192874aeea5SJeff Kirsher }
1193874aeea5SJeff Kirsher 
1194874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx)
1195874aeea5SJeff Kirsher {
1196874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
1197874aeea5SJeff Kirsher 
1198874aeea5SJeff Kirsher 	if (!efx->port_initialized)
1199874aeea5SJeff Kirsher 		return;
1200874aeea5SJeff Kirsher 
1201874aeea5SJeff Kirsher 	efx->phy_op->fini(efx);
1202874aeea5SJeff Kirsher 	efx->port_initialized = false;
1203874aeea5SJeff Kirsher 
1204874aeea5SJeff Kirsher 	efx->link_state.up = false;
1205874aeea5SJeff Kirsher 	efx_link_status_changed(efx);
1206874aeea5SJeff Kirsher }
1207874aeea5SJeff Kirsher 
1208874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx)
1209874aeea5SJeff Kirsher {
1210874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
1211874aeea5SJeff Kirsher 
1212874aeea5SJeff Kirsher 	efx->type->remove_port(efx);
1213874aeea5SJeff Kirsher }
1214874aeea5SJeff Kirsher 
1215874aeea5SJeff Kirsher /**************************************************************************
1216874aeea5SJeff Kirsher  *
1217874aeea5SJeff Kirsher  * NIC handling
1218874aeea5SJeff Kirsher  *
1219874aeea5SJeff Kirsher  **************************************************************************/
1220874aeea5SJeff Kirsher 
12210bcf4a64SBen Hutchings static LIST_HEAD(efx_primary_list);
12220bcf4a64SBen Hutchings static LIST_HEAD(efx_unassociated_list);
12230bcf4a64SBen Hutchings 
12240bcf4a64SBen Hutchings static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)
12250bcf4a64SBen Hutchings {
12260bcf4a64SBen Hutchings 	return left->type == right->type &&
12270bcf4a64SBen Hutchings 		left->vpd_sn && right->vpd_sn &&
12280bcf4a64SBen Hutchings 		!strcmp(left->vpd_sn, right->vpd_sn);
12290bcf4a64SBen Hutchings }
12300bcf4a64SBen Hutchings 
12310bcf4a64SBen Hutchings static void efx_associate(struct efx_nic *efx)
12320bcf4a64SBen Hutchings {
12330bcf4a64SBen Hutchings 	struct efx_nic *other, *next;
12340bcf4a64SBen Hutchings 
12350bcf4a64SBen Hutchings 	if (efx->primary == efx) {
12360bcf4a64SBen Hutchings 		/* Adding primary function; look for secondaries */
12370bcf4a64SBen Hutchings 
12380bcf4a64SBen Hutchings 		netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
12390bcf4a64SBen Hutchings 		list_add_tail(&efx->node, &efx_primary_list);
12400bcf4a64SBen Hutchings 
12410bcf4a64SBen Hutchings 		list_for_each_entry_safe(other, next, &efx_unassociated_list,
12420bcf4a64SBen Hutchings 					 node) {
12430bcf4a64SBen Hutchings 			if (efx_same_controller(efx, other)) {
12440bcf4a64SBen Hutchings 				list_del(&other->node);
12450bcf4a64SBen Hutchings 				netif_dbg(other, probe, other->net_dev,
12460bcf4a64SBen Hutchings 					  "moving to secondary list of %s %s\n",
12470bcf4a64SBen Hutchings 					  pci_name(efx->pci_dev),
12480bcf4a64SBen Hutchings 					  efx->net_dev->name);
12490bcf4a64SBen Hutchings 				list_add_tail(&other->node,
12500bcf4a64SBen Hutchings 					      &efx->secondary_list);
12510bcf4a64SBen Hutchings 				other->primary = efx;
12520bcf4a64SBen Hutchings 			}
12530bcf4a64SBen Hutchings 		}
12540bcf4a64SBen Hutchings 	} else {
12550bcf4a64SBen Hutchings 		/* Adding secondary function; look for primary */
12560bcf4a64SBen Hutchings 
12570bcf4a64SBen Hutchings 		list_for_each_entry(other, &efx_primary_list, node) {
12580bcf4a64SBen Hutchings 			if (efx_same_controller(efx, other)) {
12590bcf4a64SBen Hutchings 				netif_dbg(efx, probe, efx->net_dev,
12600bcf4a64SBen Hutchings 					  "adding to secondary list of %s %s\n",
12610bcf4a64SBen Hutchings 					  pci_name(other->pci_dev),
12620bcf4a64SBen Hutchings 					  other->net_dev->name);
12630bcf4a64SBen Hutchings 				list_add_tail(&efx->node,
12640bcf4a64SBen Hutchings 					      &other->secondary_list);
12650bcf4a64SBen Hutchings 				efx->primary = other;
12660bcf4a64SBen Hutchings 				return;
12670bcf4a64SBen Hutchings 			}
12680bcf4a64SBen Hutchings 		}
12690bcf4a64SBen Hutchings 
12700bcf4a64SBen Hutchings 		netif_dbg(efx, probe, efx->net_dev,
12710bcf4a64SBen Hutchings 			  "adding to unassociated list\n");
12720bcf4a64SBen Hutchings 		list_add_tail(&efx->node, &efx_unassociated_list);
12730bcf4a64SBen Hutchings 	}
12740bcf4a64SBen Hutchings }
12750bcf4a64SBen Hutchings 
12760bcf4a64SBen Hutchings static void efx_dissociate(struct efx_nic *efx)
12770bcf4a64SBen Hutchings {
12780bcf4a64SBen Hutchings 	struct efx_nic *other, *next;
12790bcf4a64SBen Hutchings 
12800bcf4a64SBen Hutchings 	list_del(&efx->node);
12810bcf4a64SBen Hutchings 	efx->primary = NULL;
12820bcf4a64SBen Hutchings 
12830bcf4a64SBen Hutchings 	list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
12840bcf4a64SBen Hutchings 		list_del(&other->node);
12850bcf4a64SBen Hutchings 		netif_dbg(other, probe, other->net_dev,
12860bcf4a64SBen Hutchings 			  "moving to unassociated list\n");
12870bcf4a64SBen Hutchings 		list_add_tail(&other->node, &efx_unassociated_list);
12880bcf4a64SBen Hutchings 		other->primary = NULL;
12890bcf4a64SBen Hutchings 	}
12900bcf4a64SBen Hutchings }
12910bcf4a64SBen Hutchings 
1292874aeea5SJeff Kirsher /* This configures the PCI device to enable I/O and DMA. */
1293874aeea5SJeff Kirsher static int efx_init_io(struct efx_nic *efx)
1294874aeea5SJeff Kirsher {
1295874aeea5SJeff Kirsher 	struct pci_dev *pci_dev = efx->pci_dev;
1296874aeea5SJeff Kirsher 	dma_addr_t dma_mask = efx->type->max_dma_mask;
1297b105798fSBen Hutchings 	unsigned int mem_map_size = efx->type->mem_map_size(efx);
129802246a7fSShradha Shah 	int rc, bar;
1299874aeea5SJeff Kirsher 
1300874aeea5SJeff Kirsher 	netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
1301874aeea5SJeff Kirsher 
130203714bbbSEdward Cree 	bar = efx->type->mem_bar(efx);
130302246a7fSShradha Shah 
1304874aeea5SJeff Kirsher 	rc = pci_enable_device(pci_dev);
1305874aeea5SJeff Kirsher 	if (rc) {
1306874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
1307874aeea5SJeff Kirsher 			  "failed to enable PCI device\n");
1308874aeea5SJeff Kirsher 		goto fail1;
1309874aeea5SJeff Kirsher 	}
1310874aeea5SJeff Kirsher 
1311874aeea5SJeff Kirsher 	pci_set_master(pci_dev);
1312874aeea5SJeff Kirsher 
131306e9552fSChristoph Hellwig 	/* Set the PCI DMA mask.  Try all possibilities from our genuine mask
131406e9552fSChristoph Hellwig 	 * down to 32 bits, because some architectures will allow 40 bit
1315874aeea5SJeff Kirsher 	 * masks event though they reject 46 bit masks.
1316874aeea5SJeff Kirsher 	 */
1317874aeea5SJeff Kirsher 	while (dma_mask > 0x7fffffffUL) {
13189663dedaSRussell King 		rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask);
1319e9e01846SBen Hutchings 		if (rc == 0)
1320874aeea5SJeff Kirsher 			break;
1321874aeea5SJeff Kirsher 		dma_mask >>= 1;
1322874aeea5SJeff Kirsher 	}
1323874aeea5SJeff Kirsher 	if (rc) {
1324874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
1325874aeea5SJeff Kirsher 			  "could not find a suitable DMA mask\n");
1326874aeea5SJeff Kirsher 		goto fail2;
1327874aeea5SJeff Kirsher 	}
1328874aeea5SJeff Kirsher 	netif_dbg(efx, probe, efx->net_dev,
1329874aeea5SJeff Kirsher 		  "using DMA mask %llx\n", (unsigned long long) dma_mask);
1330874aeea5SJeff Kirsher 
133102246a7fSShradha Shah 	efx->membase_phys = pci_resource_start(efx->pci_dev, bar);
133202246a7fSShradha Shah 	rc = pci_request_region(pci_dev, bar, "sfc");
1333874aeea5SJeff Kirsher 	if (rc) {
1334874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
1335874aeea5SJeff Kirsher 			  "request for memory BAR failed\n");
1336874aeea5SJeff Kirsher 		rc = -EIO;
1337874aeea5SJeff Kirsher 		goto fail3;
1338874aeea5SJeff Kirsher 	}
1339b105798fSBen Hutchings 	efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size);
1340874aeea5SJeff Kirsher 	if (!efx->membase) {
1341874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
1342874aeea5SJeff Kirsher 			  "could not map memory BAR at %llx+%x\n",
1343b105798fSBen Hutchings 			  (unsigned long long)efx->membase_phys, mem_map_size);
1344874aeea5SJeff Kirsher 		rc = -ENOMEM;
1345874aeea5SJeff Kirsher 		goto fail4;
1346874aeea5SJeff Kirsher 	}
1347874aeea5SJeff Kirsher 	netif_dbg(efx, probe, efx->net_dev,
1348874aeea5SJeff Kirsher 		  "memory BAR at %llx+%x (virtual %p)\n",
1349b105798fSBen Hutchings 		  (unsigned long long)efx->membase_phys, mem_map_size,
1350b105798fSBen Hutchings 		  efx->membase);
1351874aeea5SJeff Kirsher 
1352874aeea5SJeff Kirsher 	return 0;
1353874aeea5SJeff Kirsher 
1354874aeea5SJeff Kirsher  fail4:
135502246a7fSShradha Shah 	pci_release_region(efx->pci_dev, bar);
1356874aeea5SJeff Kirsher  fail3:
1357874aeea5SJeff Kirsher 	efx->membase_phys = 0;
1358874aeea5SJeff Kirsher  fail2:
1359874aeea5SJeff Kirsher 	pci_disable_device(efx->pci_dev);
1360874aeea5SJeff Kirsher  fail1:
1361874aeea5SJeff Kirsher 	return rc;
1362874aeea5SJeff Kirsher }
1363874aeea5SJeff Kirsher 
1364874aeea5SJeff Kirsher static void efx_fini_io(struct efx_nic *efx)
1365874aeea5SJeff Kirsher {
136602246a7fSShradha Shah 	int bar;
136702246a7fSShradha Shah 
1368874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
1369874aeea5SJeff Kirsher 
1370874aeea5SJeff Kirsher 	if (efx->membase) {
1371874aeea5SJeff Kirsher 		iounmap(efx->membase);
1372874aeea5SJeff Kirsher 		efx->membase = NULL;
1373874aeea5SJeff Kirsher 	}
1374874aeea5SJeff Kirsher 
1375874aeea5SJeff Kirsher 	if (efx->membase_phys) {
137603714bbbSEdward Cree 		bar = efx->type->mem_bar(efx);
137702246a7fSShradha Shah 		pci_release_region(efx->pci_dev, bar);
1378874aeea5SJeff Kirsher 		efx->membase_phys = 0;
1379874aeea5SJeff Kirsher 	}
1380874aeea5SJeff Kirsher 
13816598dad2SDaniel Pieczko 	/* Don't disable bus-mastering if VFs are assigned */
13826598dad2SDaniel Pieczko 	if (!pci_vfs_assigned(efx->pci_dev))
1383874aeea5SJeff Kirsher 		pci_disable_device(efx->pci_dev);
1384874aeea5SJeff Kirsher }
1385874aeea5SJeff Kirsher 
138642356d9aSEdward Cree void efx_set_default_rx_indir_table(struct efx_nic *efx,
138742356d9aSEdward Cree 				    struct efx_rss_context *ctx)
1388267c0157SJon Cooper {
1389267c0157SJon Cooper 	size_t i;
1390267c0157SJon Cooper 
139142356d9aSEdward Cree 	for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
139242356d9aSEdward Cree 		ctx->rx_indir_table[i] =
1393267c0157SJon Cooper 			ethtool_rxfh_indir_default(i, efx->rss_spread);
1394267c0157SJon Cooper }
1395267c0157SJon Cooper 
1396a9a52506SBen Hutchings static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
1397874aeea5SJeff Kirsher {
1398cdb08f8fSBen Hutchings 	cpumask_var_t thread_mask;
1399a16e5b24SBen Hutchings 	unsigned int count;
1400874aeea5SJeff Kirsher 	int cpu;
1401874aeea5SJeff Kirsher 
1402cd2d5b52SBen Hutchings 	if (rss_cpus) {
1403cd2d5b52SBen Hutchings 		count = rss_cpus;
1404cd2d5b52SBen Hutchings 	} else {
1405cdb08f8fSBen Hutchings 		if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
1406a9a52506SBen Hutchings 			netif_warn(efx, probe, efx->net_dev,
1407a9a52506SBen Hutchings 				   "RSS disabled due to allocation failure\n");
1408874aeea5SJeff Kirsher 			return 1;
1409874aeea5SJeff Kirsher 		}
1410874aeea5SJeff Kirsher 
1411874aeea5SJeff Kirsher 		count = 0;
1412874aeea5SJeff Kirsher 		for_each_online_cpu(cpu) {
1413cdb08f8fSBen Hutchings 			if (!cpumask_test_cpu(cpu, thread_mask)) {
1414874aeea5SJeff Kirsher 				++count;
1415cdb08f8fSBen Hutchings 				cpumask_or(thread_mask, thread_mask,
141606931e62SBartosz Golaszewski 					   topology_sibling_cpumask(cpu));
1417874aeea5SJeff Kirsher 			}
1418874aeea5SJeff Kirsher 		}
1419874aeea5SJeff Kirsher 
1420cdb08f8fSBen Hutchings 		free_cpumask_var(thread_mask);
1421cd2d5b52SBen Hutchings 	}
1422cd2d5b52SBen Hutchings 
1423271a8b42SBert Kenward 	if (count > EFX_MAX_RX_QUEUES) {
1424271a8b42SBert Kenward 		netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
1425271a8b42SBert Kenward 			       "Reducing number of rx queues from %u to %u.\n",
1426271a8b42SBert Kenward 			       count, EFX_MAX_RX_QUEUES);
1427271a8b42SBert Kenward 		count = EFX_MAX_RX_QUEUES;
1428271a8b42SBert Kenward 	}
1429271a8b42SBert Kenward 
1430cd2d5b52SBen Hutchings 	/* If RSS is requested for the PF *and* VFs then we can't write RSS
1431cd2d5b52SBen Hutchings 	 * table entries that are inaccessible to VFs
1432cd2d5b52SBen Hutchings 	 */
14337fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV
14347fa8d547SShradha Shah 	if (efx->type->sriov_wanted) {
1435d98a4ffeSShradha Shah 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
1436cd2d5b52SBen Hutchings 		    count > efx_vf_size(efx)) {
1437cd2d5b52SBen Hutchings 			netif_warn(efx, probe, efx->net_dev,
1438cd2d5b52SBen Hutchings 				   "Reducing number of RSS channels from %u to %u for "
1439cd2d5b52SBen Hutchings 				   "VF support. Increase vf-msix-limit to use more "
1440cd2d5b52SBen Hutchings 				   "channels on the PF.\n",
1441cd2d5b52SBen Hutchings 				   count, efx_vf_size(efx));
1442cd2d5b52SBen Hutchings 			count = efx_vf_size(efx);
1443cd2d5b52SBen Hutchings 		}
14447fa8d547SShradha Shah 	}
14457fa8d547SShradha Shah #endif
1446cd2d5b52SBen Hutchings 
1447874aeea5SJeff Kirsher 	return count;
1448874aeea5SJeff Kirsher }
1449874aeea5SJeff Kirsher 
14503990a8ffSCharles McLachlan static int efx_allocate_msix_channels(struct efx_nic *efx,
14513990a8ffSCharles McLachlan 				      unsigned int max_channels,
14523990a8ffSCharles McLachlan 				      unsigned int extra_channels,
14533990a8ffSCharles McLachlan 				      unsigned int parallelism)
14543990a8ffSCharles McLachlan {
14553990a8ffSCharles McLachlan 	unsigned int n_channels = parallelism;
14563990a8ffSCharles McLachlan 	int vec_count;
14573990a8ffSCharles McLachlan 	int n_xdp_tx;
14583990a8ffSCharles McLachlan 	int n_xdp_ev;
14593990a8ffSCharles McLachlan 
14603990a8ffSCharles McLachlan 	if (efx_separate_tx_channels)
14613990a8ffSCharles McLachlan 		n_channels *= 2;
14623990a8ffSCharles McLachlan 	n_channels += extra_channels;
14633990a8ffSCharles McLachlan 
14643990a8ffSCharles McLachlan 	/* To allow XDP transmit to happen from arbitrary NAPI contexts
14653990a8ffSCharles McLachlan 	 * we allocate a TX queue per CPU. We share event queues across
14663990a8ffSCharles McLachlan 	 * multiple tx queues, assuming tx and ev queues are both
14673990a8ffSCharles McLachlan 	 * maximum size.
14683990a8ffSCharles McLachlan 	 */
14693990a8ffSCharles McLachlan 
14703990a8ffSCharles McLachlan 	n_xdp_tx = num_possible_cpus();
14713990a8ffSCharles McLachlan 	n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_TXQ_TYPES);
14723990a8ffSCharles McLachlan 
14733990a8ffSCharles McLachlan 	/* Check resources.
14743990a8ffSCharles McLachlan 	 * We need a channel per event queue, plus a VI per tx queue.
14753990a8ffSCharles McLachlan 	 * This may be more pessimistic than it needs to be.
14763990a8ffSCharles McLachlan 	 */
14773990a8ffSCharles McLachlan 	if (n_channels + n_xdp_ev > max_channels) {
14783990a8ffSCharles McLachlan 		netif_err(efx, drv, efx->net_dev,
14793990a8ffSCharles McLachlan 			  "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
14803990a8ffSCharles McLachlan 			  n_xdp_ev, n_channels, max_channels);
14813990a8ffSCharles McLachlan 		efx->n_xdp_channels = 0;
14823990a8ffSCharles McLachlan 		efx->xdp_tx_per_channel = 0;
14833990a8ffSCharles McLachlan 		efx->xdp_tx_queue_count = 0;
14843990a8ffSCharles McLachlan 	} else {
14853990a8ffSCharles McLachlan 		efx->n_xdp_channels = n_xdp_ev;
14863990a8ffSCharles McLachlan 		efx->xdp_tx_per_channel = EFX_TXQ_TYPES;
14873990a8ffSCharles McLachlan 		efx->xdp_tx_queue_count = n_xdp_tx;
14883990a8ffSCharles McLachlan 		n_channels += n_xdp_ev;
14893990a8ffSCharles McLachlan 		netif_dbg(efx, drv, efx->net_dev,
14903990a8ffSCharles McLachlan 			  "Allocating %d TX and %d event queues for XDP\n",
14913990a8ffSCharles McLachlan 			  n_xdp_tx, n_xdp_ev);
14923990a8ffSCharles McLachlan 	}
14933990a8ffSCharles McLachlan 
14943990a8ffSCharles McLachlan 	n_channels = min(n_channels, max_channels);
14953990a8ffSCharles McLachlan 
14963990a8ffSCharles McLachlan 	vec_count = pci_msix_vec_count(efx->pci_dev);
14973990a8ffSCharles McLachlan 	if (vec_count < 0)
14983990a8ffSCharles McLachlan 		return vec_count;
14993990a8ffSCharles McLachlan 	if (vec_count < n_channels) {
15003990a8ffSCharles McLachlan 		netif_err(efx, drv, efx->net_dev,
15013990a8ffSCharles McLachlan 			  "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
15023990a8ffSCharles McLachlan 			  vec_count, n_channels);
15033990a8ffSCharles McLachlan 		netif_err(efx, drv, efx->net_dev,
15043990a8ffSCharles McLachlan 			  "WARNING: Performance may be reduced.\n");
15053990a8ffSCharles McLachlan 		n_channels = vec_count;
15063990a8ffSCharles McLachlan 	}
15073990a8ffSCharles McLachlan 
15083990a8ffSCharles McLachlan 	efx->n_channels = n_channels;
15093990a8ffSCharles McLachlan 
15103990a8ffSCharles McLachlan 	/* Do not create the PTP TX queue(s) if PTP uses the MC directly. */
15113990a8ffSCharles McLachlan 	if (extra_channels && !efx_ptp_use_mac_tx_timestamps(efx))
15123990a8ffSCharles McLachlan 		n_channels--;
15133990a8ffSCharles McLachlan 
15143990a8ffSCharles McLachlan 	/* Ignore XDP tx channels when creating rx channels. */
15153990a8ffSCharles McLachlan 	n_channels -= efx->n_xdp_channels;
15163990a8ffSCharles McLachlan 
15173990a8ffSCharles McLachlan 	if (efx_separate_tx_channels) {
15183990a8ffSCharles McLachlan 		efx->n_tx_channels =
15193990a8ffSCharles McLachlan 			min(max(n_channels / 2, 1U),
15203990a8ffSCharles McLachlan 			    efx->max_tx_channels);
15213990a8ffSCharles McLachlan 		efx->tx_channel_offset =
15223990a8ffSCharles McLachlan 			n_channels - efx->n_tx_channels;
15233990a8ffSCharles McLachlan 		efx->n_rx_channels =
15243990a8ffSCharles McLachlan 			max(n_channels -
15253990a8ffSCharles McLachlan 			    efx->n_tx_channels, 1U);
15263990a8ffSCharles McLachlan 	} else {
15273990a8ffSCharles McLachlan 		efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
15283990a8ffSCharles McLachlan 		efx->tx_channel_offset = 0;
15293990a8ffSCharles McLachlan 		efx->n_rx_channels = n_channels;
15303990a8ffSCharles McLachlan 	}
15313990a8ffSCharles McLachlan 
15323990a8ffSCharles McLachlan 	if (efx->n_xdp_channels)
15333990a8ffSCharles McLachlan 		efx->xdp_channel_offset = efx->tx_channel_offset +
15343990a8ffSCharles McLachlan 					  efx->n_tx_channels;
15353990a8ffSCharles McLachlan 	else
15363990a8ffSCharles McLachlan 		efx->xdp_channel_offset = efx->n_channels;
15373990a8ffSCharles McLachlan 
15383990a8ffSCharles McLachlan 	netif_dbg(efx, drv, efx->net_dev,
15393990a8ffSCharles McLachlan 		  "Allocating %u RX channels\n",
15403990a8ffSCharles McLachlan 		  efx->n_rx_channels);
15413990a8ffSCharles McLachlan 
15423990a8ffSCharles McLachlan 	return efx->n_channels;
15433990a8ffSCharles McLachlan }
15443990a8ffSCharles McLachlan 
1545874aeea5SJeff Kirsher /* Probe the number and type of interrupts we are able to obtain, and
1546874aeea5SJeff Kirsher  * the resulting numbers of channels and RX queues.
1547874aeea5SJeff Kirsher  */
1548874aeea5SJeff Kirsher static int efx_probe_interrupts(struct efx_nic *efx)
1549874aeea5SJeff Kirsher {
15507f967c01SBen Hutchings 	unsigned int extra_channels = 0;
15517f967c01SBen Hutchings 	unsigned int i, j;
1552a16e5b24SBen Hutchings 	int rc;
1553874aeea5SJeff Kirsher 
15547f967c01SBen Hutchings 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
15557f967c01SBen Hutchings 		if (efx->extra_channel_type[i])
15567f967c01SBen Hutchings 			++extra_channels;
15577f967c01SBen Hutchings 
1558874aeea5SJeff Kirsher 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
15593990a8ffSCharles McLachlan 		unsigned int parallelism = efx_wanted_parallelism(efx);
1560874aeea5SJeff Kirsher 		struct msix_entry xentries[EFX_MAX_CHANNELS];
1561a16e5b24SBen Hutchings 		unsigned int n_channels;
1562874aeea5SJeff Kirsher 
15633990a8ffSCharles McLachlan 		rc = efx_allocate_msix_channels(efx, efx->max_channels,
15643990a8ffSCharles McLachlan 						extra_channels, parallelism);
15653990a8ffSCharles McLachlan 		if (rc >= 0) {
15663990a8ffSCharles McLachlan 			n_channels = rc;
1567874aeea5SJeff Kirsher 			for (i = 0; i < n_channels; i++)
1568874aeea5SJeff Kirsher 				xentries[i].entry = i;
15693990a8ffSCharles McLachlan 			rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
15703990a8ffSCharles McLachlan 						   n_channels);
15713990a8ffSCharles McLachlan 		}
1572184603d8SAlexander Gordeev 		if (rc < 0) {
1573184603d8SAlexander Gordeev 			/* Fall back to single channel MSI */
1574184603d8SAlexander Gordeev 			netif_err(efx, drv, efx->net_dev,
1575184603d8SAlexander Gordeev 				  "could not enable MSI-X\n");
157662980cb6SAndrew Rybchenko 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
157762980cb6SAndrew Rybchenko 				efx->interrupt_mode = EFX_INT_MODE_MSI;
157862980cb6SAndrew Rybchenko 			else
157962980cb6SAndrew Rybchenko 				return rc;
1580184603d8SAlexander Gordeev 		} else if (rc < n_channels) {
1581874aeea5SJeff Kirsher 			netif_err(efx, drv, efx->net_dev,
1582874aeea5SJeff Kirsher 				  "WARNING: Insufficient MSI-X vectors"
1583a16e5b24SBen Hutchings 				  " available (%d < %u).\n", rc, n_channels);
1584874aeea5SJeff Kirsher 			netif_err(efx, drv, efx->net_dev,
1585874aeea5SJeff Kirsher 				  "WARNING: Performance may be reduced.\n");
1586874aeea5SJeff Kirsher 			n_channels = rc;
1587874aeea5SJeff Kirsher 		}
1588874aeea5SJeff Kirsher 
1589184603d8SAlexander Gordeev 		if (rc > 0) {
15907f967c01SBen Hutchings 			for (i = 0; i < efx->n_channels; i++)
1591874aeea5SJeff Kirsher 				efx_get_channel(efx, i)->irq =
1592874aeea5SJeff Kirsher 					xentries[i].vector;
1593874aeea5SJeff Kirsher 		}
1594874aeea5SJeff Kirsher 	}
1595874aeea5SJeff Kirsher 
1596874aeea5SJeff Kirsher 	/* Try single interrupt MSI */
1597874aeea5SJeff Kirsher 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
1598874aeea5SJeff Kirsher 		efx->n_channels = 1;
1599874aeea5SJeff Kirsher 		efx->n_rx_channels = 1;
1600874aeea5SJeff Kirsher 		efx->n_tx_channels = 1;
16013990a8ffSCharles McLachlan 		efx->n_xdp_channels = 0;
16023990a8ffSCharles McLachlan 		efx->xdp_channel_offset = efx->n_channels;
1603874aeea5SJeff Kirsher 		rc = pci_enable_msi(efx->pci_dev);
1604874aeea5SJeff Kirsher 		if (rc == 0) {
1605874aeea5SJeff Kirsher 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1606874aeea5SJeff Kirsher 		} else {
1607874aeea5SJeff Kirsher 			netif_err(efx, drv, efx->net_dev,
1608874aeea5SJeff Kirsher 				  "could not enable MSI\n");
160962980cb6SAndrew Rybchenko 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
1610874aeea5SJeff Kirsher 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;
161162980cb6SAndrew Rybchenko 			else
161262980cb6SAndrew Rybchenko 				return rc;
1613874aeea5SJeff Kirsher 		}
1614874aeea5SJeff Kirsher 	}
1615874aeea5SJeff Kirsher 
1616874aeea5SJeff Kirsher 	/* Assume legacy interrupts */
1617874aeea5SJeff Kirsher 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1618b0fbdae1SShradha Shah 		efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
1619874aeea5SJeff Kirsher 		efx->n_rx_channels = 1;
1620874aeea5SJeff Kirsher 		efx->n_tx_channels = 1;
16213990a8ffSCharles McLachlan 		efx->n_xdp_channels = 0;
16223990a8ffSCharles McLachlan 		efx->xdp_channel_offset = efx->n_channels;
1623874aeea5SJeff Kirsher 		efx->legacy_irq = efx->pci_dev->irq;
1624874aeea5SJeff Kirsher 	}
1625874aeea5SJeff Kirsher 
16263990a8ffSCharles McLachlan 	/* Assign extra channels if possible, before XDP channels */
16272935e3c3SEdward Cree 	efx->n_extra_tx_channels = 0;
16283990a8ffSCharles McLachlan 	j = efx->xdp_channel_offset;
16297f967c01SBen Hutchings 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
16307f967c01SBen Hutchings 		if (!efx->extra_channel_type[i])
16317f967c01SBen Hutchings 			continue;
16327f967c01SBen Hutchings 		if (efx->interrupt_mode != EFX_INT_MODE_MSIX ||
16337f967c01SBen Hutchings 		    efx->n_channels <= extra_channels) {
16347f967c01SBen Hutchings 			efx->extra_channel_type[i]->handle_no_channel(efx);
16357f967c01SBen Hutchings 		} else {
16367f967c01SBen Hutchings 			--j;
16377f967c01SBen Hutchings 			efx_get_channel(efx, j)->type =
16387f967c01SBen Hutchings 				efx->extra_channel_type[i];
16392935e3c3SEdward Cree 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
16402935e3c3SEdward Cree 				efx->n_extra_tx_channels++;
16417f967c01SBen Hutchings 		}
16427f967c01SBen Hutchings 	}
16437f967c01SBen Hutchings 
1644cd2d5b52SBen Hutchings 	/* RSS might be usable on VFs even if it is disabled on the PF */
16457fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV
16467fa8d547SShradha Shah 	if (efx->type->sriov_wanted) {
1647327c685eSShradha Shah 		efx->rss_spread = ((efx->n_rx_channels > 1 ||
1648d98a4ffeSShradha Shah 				    !efx->type->sriov_wanted(efx)) ?
1649cd2d5b52SBen Hutchings 				   efx->n_rx_channels : efx_vf_size(efx));
16507fa8d547SShradha Shah 		return 0;
16517fa8d547SShradha Shah 	}
16527fa8d547SShradha Shah #endif
16537fa8d547SShradha Shah 	efx->rss_spread = efx->n_rx_channels;
1654cd2d5b52SBen Hutchings 
1655874aeea5SJeff Kirsher 	return 0;
1656874aeea5SJeff Kirsher }
1657874aeea5SJeff Kirsher 
1658a83762d9SBert Kenward #if defined(CONFIG_SMP)
1659a83762d9SBert Kenward static void efx_set_interrupt_affinity(struct efx_nic *efx)
1660a83762d9SBert Kenward {
1661a83762d9SBert Kenward 	struct efx_channel *channel;
1662a83762d9SBert Kenward 	unsigned int cpu;
1663a83762d9SBert Kenward 
1664a83762d9SBert Kenward 	efx_for_each_channel(channel, efx) {
1665a83762d9SBert Kenward 		cpu = cpumask_local_spread(channel->channel,
1666a83762d9SBert Kenward 					   pcibus_to_node(efx->pci_dev->bus));
1667a83762d9SBert Kenward 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
1668a83762d9SBert Kenward 	}
1669a83762d9SBert Kenward }
1670a83762d9SBert Kenward 
1671a83762d9SBert Kenward static void efx_clear_interrupt_affinity(struct efx_nic *efx)
1672a83762d9SBert Kenward {
1673a83762d9SBert Kenward 	struct efx_channel *channel;
1674a83762d9SBert Kenward 
1675a83762d9SBert Kenward 	efx_for_each_channel(channel, efx)
1676a83762d9SBert Kenward 		irq_set_affinity_hint(channel->irq, NULL);
1677a83762d9SBert Kenward }
1678a83762d9SBert Kenward #else
1679a83762d9SBert Kenward static void
1680a83762d9SBert Kenward efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
1681a83762d9SBert Kenward {
1682a83762d9SBert Kenward }
1683a83762d9SBert Kenward 
1684a83762d9SBert Kenward static void
1685a83762d9SBert Kenward efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
1686a83762d9SBert Kenward {
1687a83762d9SBert Kenward }
1688a83762d9SBert Kenward #endif /* CONFIG_SMP */
1689a83762d9SBert Kenward 
1690261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx)
1691d8291187SBen Hutchings {
1692261e4d96SJon Cooper 	struct efx_channel *channel, *end_channel;
1693261e4d96SJon Cooper 	int rc;
1694d8291187SBen Hutchings 
1695d8291187SBen Hutchings 	BUG_ON(efx->state == STATE_DISABLED);
1696d8291187SBen Hutchings 
1697d8291187SBen Hutchings 	efx->irq_soft_enabled = true;
1698d8291187SBen Hutchings 	smp_wmb();
1699d8291187SBen Hutchings 
1700d8291187SBen Hutchings 	efx_for_each_channel(channel, efx) {
1701261e4d96SJon Cooper 		if (!channel->type->keep_eventq) {
1702261e4d96SJon Cooper 			rc = efx_init_eventq(channel);
1703261e4d96SJon Cooper 			if (rc)
1704261e4d96SJon Cooper 				goto fail;
1705261e4d96SJon Cooper 		}
1706d8291187SBen Hutchings 		efx_start_eventq(channel);
1707d8291187SBen Hutchings 	}
1708d8291187SBen Hutchings 
1709d8291187SBen Hutchings 	efx_mcdi_mode_event(efx);
1710261e4d96SJon Cooper 
1711261e4d96SJon Cooper 	return 0;
1712261e4d96SJon Cooper fail:
1713261e4d96SJon Cooper 	end_channel = channel;
1714261e4d96SJon Cooper 	efx_for_each_channel(channel, efx) {
1715261e4d96SJon Cooper 		if (channel == end_channel)
1716261e4d96SJon Cooper 			break;
1717261e4d96SJon Cooper 		efx_stop_eventq(channel);
1718261e4d96SJon Cooper 		if (!channel->type->keep_eventq)
1719261e4d96SJon Cooper 			efx_fini_eventq(channel);
1720261e4d96SJon Cooper 	}
1721261e4d96SJon Cooper 
1722261e4d96SJon Cooper 	return rc;
1723d8291187SBen Hutchings }
1724d8291187SBen Hutchings 
1725d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx)
1726d8291187SBen Hutchings {
1727d8291187SBen Hutchings 	struct efx_channel *channel;
1728d8291187SBen Hutchings 
1729d8291187SBen Hutchings 	if (efx->state == STATE_DISABLED)
1730d8291187SBen Hutchings 		return;
1731d8291187SBen Hutchings 
1732d8291187SBen Hutchings 	efx_mcdi_mode_poll(efx);
1733d8291187SBen Hutchings 
1734d8291187SBen Hutchings 	efx->irq_soft_enabled = false;
1735d8291187SBen Hutchings 	smp_wmb();
1736d8291187SBen Hutchings 
1737d8291187SBen Hutchings 	if (efx->legacy_irq)
1738d8291187SBen Hutchings 		synchronize_irq(efx->legacy_irq);
1739d8291187SBen Hutchings 
1740d8291187SBen Hutchings 	efx_for_each_channel(channel, efx) {
1741d8291187SBen Hutchings 		if (channel->irq)
1742d8291187SBen Hutchings 			synchronize_irq(channel->irq);
1743d8291187SBen Hutchings 
1744d8291187SBen Hutchings 		efx_stop_eventq(channel);
1745d8291187SBen Hutchings 		if (!channel->type->keep_eventq)
1746d8291187SBen Hutchings 			efx_fini_eventq(channel);
1747d8291187SBen Hutchings 	}
1748cade715fSBen Hutchings 
1749cade715fSBen Hutchings 	/* Flush the asynchronous MCDI request queue */
1750cade715fSBen Hutchings 	efx_mcdi_flush_async(efx);
1751d8291187SBen Hutchings }
1752d8291187SBen Hutchings 
1753261e4d96SJon Cooper static int efx_enable_interrupts(struct efx_nic *efx)
17549f2cb71cSBen Hutchings {
1755261e4d96SJon Cooper 	struct efx_channel *channel, *end_channel;
1756261e4d96SJon Cooper 	int rc;
17579f2cb71cSBen Hutchings 
17588b7325b4SBen Hutchings 	BUG_ON(efx->state == STATE_DISABLED);
17598b7325b4SBen Hutchings 
1760b28405b0SAlexandre Rames 	if (efx->eeh_disabled_legacy_irq) {
1761b28405b0SAlexandre Rames 		enable_irq(efx->legacy_irq);
1762b28405b0SAlexandre Rames 		efx->eeh_disabled_legacy_irq = false;
1763b28405b0SAlexandre Rames 	}
1764d8291187SBen Hutchings 
176586094f7fSBen Hutchings 	efx->type->irq_enable_master(efx);
17669f2cb71cSBen Hutchings 
17679f2cb71cSBen Hutchings 	efx_for_each_channel(channel, efx) {
1768261e4d96SJon Cooper 		if (channel->type->keep_eventq) {
1769261e4d96SJon Cooper 			rc = efx_init_eventq(channel);
1770261e4d96SJon Cooper 			if (rc)
1771261e4d96SJon Cooper 				goto fail;
1772261e4d96SJon Cooper 		}
17739f2cb71cSBen Hutchings 	}
17749f2cb71cSBen Hutchings 
1775261e4d96SJon Cooper 	rc = efx_soft_enable_interrupts(efx);
1776261e4d96SJon Cooper 	if (rc)
1777261e4d96SJon Cooper 		goto fail;
1778261e4d96SJon Cooper 
1779261e4d96SJon Cooper 	return 0;
1780261e4d96SJon Cooper 
1781261e4d96SJon Cooper fail:
1782261e4d96SJon Cooper 	end_channel = channel;
1783261e4d96SJon Cooper 	efx_for_each_channel(channel, efx) {
1784261e4d96SJon Cooper 		if (channel == end_channel)
1785261e4d96SJon Cooper 			break;
1786261e4d96SJon Cooper 		if (channel->type->keep_eventq)
1787261e4d96SJon Cooper 			efx_fini_eventq(channel);
1788261e4d96SJon Cooper 	}
1789261e4d96SJon Cooper 
1790261e4d96SJon Cooper 	efx->type->irq_disable_non_ev(efx);
1791261e4d96SJon Cooper 
1792261e4d96SJon Cooper 	return rc;
17939f2cb71cSBen Hutchings }
17949f2cb71cSBen Hutchings 
1795d8291187SBen Hutchings static void efx_disable_interrupts(struct efx_nic *efx)
17969f2cb71cSBen Hutchings {
17979f2cb71cSBen Hutchings 	struct efx_channel *channel;
17989f2cb71cSBen Hutchings 
1799d8291187SBen Hutchings 	efx_soft_disable_interrupts(efx);
18009f2cb71cSBen Hutchings 
18019f2cb71cSBen Hutchings 	efx_for_each_channel(channel, efx) {
1802d8291187SBen Hutchings 		if (channel->type->keep_eventq)
18039f2cb71cSBen Hutchings 			efx_fini_eventq(channel);
18049f2cb71cSBen Hutchings 	}
1805d8291187SBen Hutchings 
180686094f7fSBen Hutchings 	efx->type->irq_disable_non_ev(efx);
18079f2cb71cSBen Hutchings }
18089f2cb71cSBen Hutchings 
1809874aeea5SJeff Kirsher static void efx_remove_interrupts(struct efx_nic *efx)
1810874aeea5SJeff Kirsher {
1811874aeea5SJeff Kirsher 	struct efx_channel *channel;
1812874aeea5SJeff Kirsher 
1813874aeea5SJeff Kirsher 	/* Remove MSI/MSI-X interrupts */
1814874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx)
1815874aeea5SJeff Kirsher 		channel->irq = 0;
1816874aeea5SJeff Kirsher 	pci_disable_msi(efx->pci_dev);
1817874aeea5SJeff Kirsher 	pci_disable_msix(efx->pci_dev);
1818874aeea5SJeff Kirsher 
1819874aeea5SJeff Kirsher 	/* Remove legacy interrupt */
1820874aeea5SJeff Kirsher 	efx->legacy_irq = 0;
1821874aeea5SJeff Kirsher }
1822874aeea5SJeff Kirsher 
18233990a8ffSCharles McLachlan static int efx_set_channels(struct efx_nic *efx)
1824874aeea5SJeff Kirsher {
1825874aeea5SJeff Kirsher 	struct efx_channel *channel;
1826874aeea5SJeff Kirsher 	struct efx_tx_queue *tx_queue;
18273990a8ffSCharles McLachlan 	int xdp_queue_number;
1828874aeea5SJeff Kirsher 
1829874aeea5SJeff Kirsher 	efx->tx_channel_offset =
1830b0fbdae1SShradha Shah 		efx_separate_tx_channels ?
1831b0fbdae1SShradha Shah 		efx->n_channels - efx->n_tx_channels : 0;
1832874aeea5SJeff Kirsher 
18333990a8ffSCharles McLachlan 	if (efx->xdp_tx_queue_count) {
18343990a8ffSCharles McLachlan 		EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
18353990a8ffSCharles McLachlan 
18363990a8ffSCharles McLachlan 		/* Allocate array for XDP TX queue lookup. */
18373990a8ffSCharles McLachlan 		efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
18383990a8ffSCharles McLachlan 					     sizeof(*efx->xdp_tx_queues),
18393990a8ffSCharles McLachlan 					     GFP_KERNEL);
18403990a8ffSCharles McLachlan 		if (!efx->xdp_tx_queues)
18413990a8ffSCharles McLachlan 			return -ENOMEM;
18423990a8ffSCharles McLachlan 	}
18433990a8ffSCharles McLachlan 
184479d68b37SStuart Hodgson 	/* We need to mark which channels really have RX and TX
184579d68b37SStuart Hodgson 	 * queues, and adjust the TX queue numbers if we have separate
1846874aeea5SJeff Kirsher 	 * RX-only and TX-only channels.
1847874aeea5SJeff Kirsher 	 */
18483990a8ffSCharles McLachlan 	xdp_queue_number = 0;
1849874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx) {
185079d68b37SStuart Hodgson 		if (channel->channel < efx->n_rx_channels)
185179d68b37SStuart Hodgson 			channel->rx_queue.core_index = channel->channel;
185279d68b37SStuart Hodgson 		else
185379d68b37SStuart Hodgson 			channel->rx_queue.core_index = -1;
185479d68b37SStuart Hodgson 
18553990a8ffSCharles McLachlan 		efx_for_each_channel_tx_queue(tx_queue, channel) {
1856874aeea5SJeff Kirsher 			tx_queue->queue -= (efx->tx_channel_offset *
1857874aeea5SJeff Kirsher 					    EFX_TXQ_TYPES);
18583990a8ffSCharles McLachlan 
18593990a8ffSCharles McLachlan 			if (efx_channel_is_xdp_tx(channel) &&
18603990a8ffSCharles McLachlan 			    xdp_queue_number < efx->xdp_tx_queue_count) {
18613990a8ffSCharles McLachlan 				efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
18623990a8ffSCharles McLachlan 				xdp_queue_number++;
1863874aeea5SJeff Kirsher 			}
1864874aeea5SJeff Kirsher 		}
18653990a8ffSCharles McLachlan 	}
18663990a8ffSCharles McLachlan 	return 0;
18673990a8ffSCharles McLachlan }
1868874aeea5SJeff Kirsher 
1869874aeea5SJeff Kirsher static int efx_probe_nic(struct efx_nic *efx)
1870874aeea5SJeff Kirsher {
1871874aeea5SJeff Kirsher 	int rc;
1872874aeea5SJeff Kirsher 
1873874aeea5SJeff Kirsher 	netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1874874aeea5SJeff Kirsher 
1875874aeea5SJeff Kirsher 	/* Carry out hardware-type specific initialisation */
1876874aeea5SJeff Kirsher 	rc = efx->type->probe(efx);
1877874aeea5SJeff Kirsher 	if (rc)
1878874aeea5SJeff Kirsher 		return rc;
1879874aeea5SJeff Kirsher 
1880b0fbdae1SShradha Shah 	do {
1881b0fbdae1SShradha Shah 		if (!efx->max_channels || !efx->max_tx_channels) {
1882b0fbdae1SShradha Shah 			netif_err(efx, drv, efx->net_dev,
1883b0fbdae1SShradha Shah 				  "Insufficient resources to allocate"
1884b0fbdae1SShradha Shah 				  " any channels\n");
1885b0fbdae1SShradha Shah 			rc = -ENOSPC;
1886b0fbdae1SShradha Shah 			goto fail1;
1887b0fbdae1SShradha Shah 		}
1888b0fbdae1SShradha Shah 
1889b0fbdae1SShradha Shah 		/* Determine the number of channels and queues by trying
1890b0fbdae1SShradha Shah 		 * to hook in MSI-X interrupts.
1891b0fbdae1SShradha Shah 		 */
1892874aeea5SJeff Kirsher 		rc = efx_probe_interrupts(efx);
1893874aeea5SJeff Kirsher 		if (rc)
1894c15eed22SBen Hutchings 			goto fail1;
1895874aeea5SJeff Kirsher 
18963990a8ffSCharles McLachlan 		rc = efx_set_channels(efx);
18973990a8ffSCharles McLachlan 		if (rc)
18983990a8ffSCharles McLachlan 			goto fail1;
189952ad762bSDaniel Pieczko 
1900b0fbdae1SShradha Shah 		/* dimension_resources can fail with EAGAIN */
1901c15eed22SBen Hutchings 		rc = efx->type->dimension_resources(efx);
1902b0fbdae1SShradha Shah 		if (rc != 0 && rc != -EAGAIN)
1903c15eed22SBen Hutchings 			goto fail2;
190428e47c49SBen Hutchings 
1905b0fbdae1SShradha Shah 		if (rc == -EAGAIN)
1906b0fbdae1SShradha Shah 			/* try again with new max_channels */
1907b0fbdae1SShradha Shah 			efx_remove_interrupts(efx);
1908b0fbdae1SShradha Shah 
1909b0fbdae1SShradha Shah 	} while (rc == -EAGAIN);
1910b0fbdae1SShradha Shah 
1911874aeea5SJeff Kirsher 	if (efx->n_channels > 1)
191242356d9aSEdward Cree 		netdev_rss_key_fill(efx->rss_context.rx_hash_key,
191342356d9aSEdward Cree 				    sizeof(efx->rss_context.rx_hash_key));
191442356d9aSEdward Cree 	efx_set_default_rx_indir_table(efx, &efx->rss_context);
1915874aeea5SJeff Kirsher 
1916874aeea5SJeff Kirsher 	netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1917874aeea5SJeff Kirsher 	netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1918874aeea5SJeff Kirsher 
1919874aeea5SJeff Kirsher 	/* Initialise the interrupt moderation settings */
1920539de7c5SBert Kenward 	efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
19219e393b30SBen Hutchings 	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
19229e393b30SBen Hutchings 				true);
1923874aeea5SJeff Kirsher 
1924874aeea5SJeff Kirsher 	return 0;
1925874aeea5SJeff Kirsher 
1926c15eed22SBen Hutchings fail2:
1927c15eed22SBen Hutchings 	efx_remove_interrupts(efx);
1928c15eed22SBen Hutchings fail1:
1929874aeea5SJeff Kirsher 	efx->type->remove(efx);
1930874aeea5SJeff Kirsher 	return rc;
1931874aeea5SJeff Kirsher }
1932874aeea5SJeff Kirsher 
1933874aeea5SJeff Kirsher static void efx_remove_nic(struct efx_nic *efx)
1934874aeea5SJeff Kirsher {
1935874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1936874aeea5SJeff Kirsher 
1937874aeea5SJeff Kirsher 	efx_remove_interrupts(efx);
1938874aeea5SJeff Kirsher 	efx->type->remove(efx);
1939874aeea5SJeff Kirsher }
1940874aeea5SJeff Kirsher 
1941add72477SBen Hutchings static int efx_probe_filters(struct efx_nic *efx)
1942add72477SBen Hutchings {
1943add72477SBen Hutchings 	int rc;
1944add72477SBen Hutchings 
19450d322413SEdward Cree 	init_rwsem(&efx->filter_sem);
1946d248953aSMartin Habets 	mutex_lock(&efx->mac_lock);
19470d322413SEdward Cree 	down_write(&efx->filter_sem);
1948add72477SBen Hutchings 	rc = efx->type->filter_table_probe(efx);
1949add72477SBen Hutchings 	if (rc)
19500d322413SEdward Cree 		goto out_unlock;
1951add72477SBen Hutchings 
1952add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL
1953add72477SBen Hutchings 	if (efx->type->offload_features & NETIF_F_NTUPLE) {
1954faf8dcc1SJon Cooper 		struct efx_channel *channel;
1955faf8dcc1SJon Cooper 		int i, success = 1;
1956faf8dcc1SJon Cooper 
1957faf8dcc1SJon Cooper 		efx_for_each_channel(channel, efx) {
1958faf8dcc1SJon Cooper 			channel->rps_flow_id =
1959faf8dcc1SJon Cooper 				kcalloc(efx->type->max_rx_ip_filters,
1960faf8dcc1SJon Cooper 					sizeof(*channel->rps_flow_id),
1961add72477SBen Hutchings 					GFP_KERNEL);
1962faf8dcc1SJon Cooper 			if (!channel->rps_flow_id)
1963faf8dcc1SJon Cooper 				success = 0;
1964faf8dcc1SJon Cooper 			else
1965faf8dcc1SJon Cooper 				for (i = 0;
1966faf8dcc1SJon Cooper 				     i < efx->type->max_rx_ip_filters;
1967faf8dcc1SJon Cooper 				     ++i)
1968faf8dcc1SJon Cooper 					channel->rps_flow_id[i] =
1969faf8dcc1SJon Cooper 						RPS_FLOW_ID_INVALID;
1970faf8dcc1SJon Cooper 		}
1971faf8dcc1SJon Cooper 
1972faf8dcc1SJon Cooper 		if (!success) {
1973faf8dcc1SJon Cooper 			efx_for_each_channel(channel, efx)
1974faf8dcc1SJon Cooper 				kfree(channel->rps_flow_id);
1975add72477SBen Hutchings 			efx->type->filter_table_remove(efx);
19760d322413SEdward Cree 			rc = -ENOMEM;
19770d322413SEdward Cree 			goto out_unlock;
1978add72477SBen Hutchings 		}
1979faf8dcc1SJon Cooper 
1980faf8dcc1SJon Cooper 		efx->rps_expire_index = efx->rps_expire_channel = 0;
1981add72477SBen Hutchings 	}
1982add72477SBen Hutchings #endif
19830d322413SEdward Cree out_unlock:
19840d322413SEdward Cree 	up_write(&efx->filter_sem);
1985d248953aSMartin Habets 	mutex_unlock(&efx->mac_lock);
19860d322413SEdward Cree 	return rc;
1987add72477SBen Hutchings }
1988add72477SBen Hutchings 
1989add72477SBen Hutchings static void efx_remove_filters(struct efx_nic *efx)
1990add72477SBen Hutchings {
1991add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL
1992faf8dcc1SJon Cooper 	struct efx_channel *channel;
1993faf8dcc1SJon Cooper 
1994faf8dcc1SJon Cooper 	efx_for_each_channel(channel, efx)
1995faf8dcc1SJon Cooper 		kfree(channel->rps_flow_id);
1996add72477SBen Hutchings #endif
19970d322413SEdward Cree 	down_write(&efx->filter_sem);
1998add72477SBen Hutchings 	efx->type->filter_table_remove(efx);
19990d322413SEdward Cree 	up_write(&efx->filter_sem);
2000add72477SBen Hutchings }
2001add72477SBen Hutchings 
2002add72477SBen Hutchings 
2003874aeea5SJeff Kirsher /**************************************************************************
2004874aeea5SJeff Kirsher  *
2005874aeea5SJeff Kirsher  * NIC startup/shutdown
2006874aeea5SJeff Kirsher  *
2007874aeea5SJeff Kirsher  *************************************************************************/
2008874aeea5SJeff Kirsher 
2009874aeea5SJeff Kirsher static int efx_probe_all(struct efx_nic *efx)
2010874aeea5SJeff Kirsher {
2011874aeea5SJeff Kirsher 	int rc;
2012874aeea5SJeff Kirsher 
2013874aeea5SJeff Kirsher 	rc = efx_probe_nic(efx);
2014874aeea5SJeff Kirsher 	if (rc) {
2015874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
2016874aeea5SJeff Kirsher 		goto fail1;
2017874aeea5SJeff Kirsher 	}
2018874aeea5SJeff Kirsher 
2019874aeea5SJeff Kirsher 	rc = efx_probe_port(efx);
2020874aeea5SJeff Kirsher 	if (rc) {
2021874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev, "failed to create port\n");
2022874aeea5SJeff Kirsher 		goto fail2;
2023874aeea5SJeff Kirsher 	}
2024874aeea5SJeff Kirsher 
20257e6d06f0SBen Hutchings 	BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
20267e6d06f0SBen Hutchings 	if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
20277e6d06f0SBen Hutchings 		rc = -EINVAL;
20287e6d06f0SBen Hutchings 		goto fail3;
20297e6d06f0SBen Hutchings 	}
2030874aeea5SJeff Kirsher 	efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
2031874aeea5SJeff Kirsher 
20326d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV
20336d8aaaf6SDaniel Pieczko 	rc = efx->type->vswitching_probe(efx);
20346d8aaaf6SDaniel Pieczko 	if (rc) /* not fatal; the PF will still work fine */
20356d8aaaf6SDaniel Pieczko 		netif_warn(efx, probe, efx->net_dev,
20366d8aaaf6SDaniel Pieczko 			   "failed to setup vswitching rc=%d;"
20376d8aaaf6SDaniel Pieczko 			   " VFs may not function\n", rc);
20386d8aaaf6SDaniel Pieczko #endif
20396d8aaaf6SDaniel Pieczko 
2040874aeea5SJeff Kirsher 	rc = efx_probe_filters(efx);
2041874aeea5SJeff Kirsher 	if (rc) {
2042874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
2043874aeea5SJeff Kirsher 			  "failed to create filter tables\n");
20446d8aaaf6SDaniel Pieczko 		goto fail4;
2045874aeea5SJeff Kirsher 	}
2046874aeea5SJeff Kirsher 
20477f967c01SBen Hutchings 	rc = efx_probe_channels(efx);
20487f967c01SBen Hutchings 	if (rc)
20496d8aaaf6SDaniel Pieczko 		goto fail5;
20507f967c01SBen Hutchings 
2051874aeea5SJeff Kirsher 	return 0;
2052874aeea5SJeff Kirsher 
20536d8aaaf6SDaniel Pieczko  fail5:
20547f967c01SBen Hutchings 	efx_remove_filters(efx);
20556d8aaaf6SDaniel Pieczko  fail4:
20566d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV
20576d8aaaf6SDaniel Pieczko 	efx->type->vswitching_remove(efx);
20586d8aaaf6SDaniel Pieczko #endif
2059874aeea5SJeff Kirsher  fail3:
2060874aeea5SJeff Kirsher 	efx_remove_port(efx);
2061874aeea5SJeff Kirsher  fail2:
2062874aeea5SJeff Kirsher 	efx_remove_nic(efx);
2063874aeea5SJeff Kirsher  fail1:
2064874aeea5SJeff Kirsher 	return rc;
2065874aeea5SJeff Kirsher }
2066874aeea5SJeff Kirsher 
20678b7325b4SBen Hutchings /* If the interface is supposed to be running but is not, start
20688b7325b4SBen Hutchings  * the hardware and software data path, regular activity for the port
20698b7325b4SBen Hutchings  * (MAC statistics, link polling, etc.) and schedule the port to be
20708b7325b4SBen Hutchings  * reconfigured.  Interrupts must already be enabled.  This function
20718b7325b4SBen Hutchings  * is safe to call multiple times, so long as the NIC is not disabled.
20728b7325b4SBen Hutchings  * Requires the RTNL lock.
20739f2cb71cSBen Hutchings  */
2074874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx)
2075874aeea5SJeff Kirsher {
2076874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
20778b7325b4SBen Hutchings 	BUG_ON(efx->state == STATE_DISABLED);
2078874aeea5SJeff Kirsher 
2079874aeea5SJeff Kirsher 	/* Check that it is appropriate to restart the interface. All
2080874aeea5SJeff Kirsher 	 * of these flags are safe to read under just the rtnl lock */
2081e283546cSEdward Cree 	if (efx->port_enabled || !netif_running(efx->net_dev) ||
2082e283546cSEdward Cree 	    efx->reset_pending)
2083874aeea5SJeff Kirsher 		return;
2084874aeea5SJeff Kirsher 
2085874aeea5SJeff Kirsher 	efx_start_port(efx);
20869f2cb71cSBen Hutchings 	efx_start_datapath(efx);
2087874aeea5SJeff Kirsher 
2088626950dbSAlexandre Rames 	/* Start the hardware monitor if there is one */
2089626950dbSAlexandre Rames 	if (efx->type->monitor != NULL)
2090874aeea5SJeff Kirsher 		queue_delayed_work(efx->workqueue, &efx->monitor_work,
2091874aeea5SJeff Kirsher 				   efx_monitor_interval);
2092626950dbSAlexandre Rames 
20935a6681e2SEdward Cree 	/* Link state detection is normally event-driven; we have
2094626950dbSAlexandre Rames 	 * to poll now because we could have missed a change
2095626950dbSAlexandre Rames 	 */
2096874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
2097874aeea5SJeff Kirsher 	if (efx->phy_op->poll(efx))
2098874aeea5SJeff Kirsher 		efx_link_status_changed(efx);
2099874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
2100874aeea5SJeff Kirsher 
2101874aeea5SJeff Kirsher 	efx->type->start_stats(efx);
2102f8f3b5aeSJon Cooper 	efx->type->pull_stats(efx);
2103f8f3b5aeSJon Cooper 	spin_lock_bh(&efx->stats_lock);
2104f8f3b5aeSJon Cooper 	efx->type->update_stats(efx, NULL, NULL);
2105f8f3b5aeSJon Cooper 	spin_unlock_bh(&efx->stats_lock);
2106874aeea5SJeff Kirsher }
2107874aeea5SJeff Kirsher 
21088b7325b4SBen Hutchings /* Quiesce the hardware and software data path, and regular activity
21098b7325b4SBen Hutchings  * for the port without bringing the link down.  Safe to call multiple
21108b7325b4SBen Hutchings  * times with the NIC in almost any state, but interrupts should be
21118b7325b4SBen Hutchings  * enabled.  Requires the RTNL lock.
21128b7325b4SBen Hutchings  */
2113874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx)
2114874aeea5SJeff Kirsher {
2115874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
2116874aeea5SJeff Kirsher 
2117874aeea5SJeff Kirsher 	/* port_enabled can be read safely under the rtnl lock */
2118874aeea5SJeff Kirsher 	if (!efx->port_enabled)
2119874aeea5SJeff Kirsher 		return;
2120874aeea5SJeff Kirsher 
2121f8f3b5aeSJon Cooper 	/* update stats before we go down so we can accurately count
2122f8f3b5aeSJon Cooper 	 * rx_nodesc_drops
2123f8f3b5aeSJon Cooper 	 */
2124f8f3b5aeSJon Cooper 	efx->type->pull_stats(efx);
2125f8f3b5aeSJon Cooper 	spin_lock_bh(&efx->stats_lock);
2126f8f3b5aeSJon Cooper 	efx->type->update_stats(efx, NULL, NULL);
2127f8f3b5aeSJon Cooper 	spin_unlock_bh(&efx->stats_lock);
2128874aeea5SJeff Kirsher 	efx->type->stop_stats(efx);
2129874aeea5SJeff Kirsher 	efx_stop_port(efx);
2130874aeea5SJeff Kirsher 
213129c69a48SBen Hutchings 	/* Stop the kernel transmit interface.  This is only valid if
213229c69a48SBen Hutchings 	 * the device is stopped or detached; otherwise the watchdog
213329c69a48SBen Hutchings 	 * may fire immediately.
213429c69a48SBen Hutchings 	 */
213529c69a48SBen Hutchings 	WARN_ON(netif_running(efx->net_dev) &&
213629c69a48SBen Hutchings 		netif_device_present(efx->net_dev));
21379f2cb71cSBen Hutchings 	netif_tx_disable(efx->net_dev);
21389f2cb71cSBen Hutchings 
21399f2cb71cSBen Hutchings 	efx_stop_datapath(efx);
2140874aeea5SJeff Kirsher }
2141874aeea5SJeff Kirsher 
2142874aeea5SJeff Kirsher static void efx_remove_all(struct efx_nic *efx)
2143874aeea5SJeff Kirsher {
2144e45a4fedSCharles McLachlan 	rtnl_lock();
2145e45a4fedSCharles McLachlan 	efx_xdp_setup_prog(efx, NULL);
2146e45a4fedSCharles McLachlan 	rtnl_unlock();
2147e45a4fedSCharles McLachlan 
2148874aeea5SJeff Kirsher 	efx_remove_channels(efx);
21497f967c01SBen Hutchings 	efx_remove_filters(efx);
21506d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV
21516d8aaaf6SDaniel Pieczko 	efx->type->vswitching_remove(efx);
21526d8aaaf6SDaniel Pieczko #endif
2153874aeea5SJeff Kirsher 	efx_remove_port(efx);
2154874aeea5SJeff Kirsher 	efx_remove_nic(efx);
2155874aeea5SJeff Kirsher }
2156874aeea5SJeff Kirsher 
2157874aeea5SJeff Kirsher /**************************************************************************
2158874aeea5SJeff Kirsher  *
2159874aeea5SJeff Kirsher  * Interrupt moderation
2160874aeea5SJeff Kirsher  *
2161874aeea5SJeff Kirsher  **************************************************************************/
2162539de7c5SBert Kenward unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)
2163874aeea5SJeff Kirsher {
2164b548f976SBen Hutchings 	if (usecs == 0)
2165b548f976SBen Hutchings 		return 0;
2166539de7c5SBert Kenward 	if (usecs * 1000 < efx->timer_quantum_ns)
2167874aeea5SJeff Kirsher 		return 1; /* never round down to 0 */
2168539de7c5SBert Kenward 	return usecs * 1000 / efx->timer_quantum_ns;
2169539de7c5SBert Kenward }
2170539de7c5SBert Kenward 
2171539de7c5SBert Kenward unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks)
2172539de7c5SBert Kenward {
2173539de7c5SBert Kenward 	/* We must round up when converting ticks to microseconds
2174539de7c5SBert Kenward 	 * because we round down when converting the other way.
2175539de7c5SBert Kenward 	 */
2176539de7c5SBert Kenward 	return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000);
2177874aeea5SJeff Kirsher }
2178874aeea5SJeff Kirsher 
2179874aeea5SJeff Kirsher /* Set interrupt moderation parameters */
21809e393b30SBen Hutchings int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
21819e393b30SBen Hutchings 			    unsigned int rx_usecs, bool rx_adaptive,
21829e393b30SBen Hutchings 			    bool rx_may_override_tx)
2183874aeea5SJeff Kirsher {
2184874aeea5SJeff Kirsher 	struct efx_channel *channel;
2185d95e329aSBert Kenward 	unsigned int timer_max_us;
2186d95e329aSBert Kenward 
2187874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
2188874aeea5SJeff Kirsher 
2189d95e329aSBert Kenward 	timer_max_us = efx->timer_max_ns / 1000;
2190d95e329aSBert Kenward 
2191d95e329aSBert Kenward 	if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)
21929e393b30SBen Hutchings 		return -EINVAL;
21939e393b30SBen Hutchings 
2194539de7c5SBert Kenward 	if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
21959e393b30SBen Hutchings 	    !rx_may_override_tx) {
21969e393b30SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "Channels are shared. "
21979e393b30SBen Hutchings 			  "RX and TX IRQ moderation must be equal\n");
21989e393b30SBen Hutchings 		return -EINVAL;
21999e393b30SBen Hutchings 	}
22009e393b30SBen Hutchings 
2201874aeea5SJeff Kirsher 	efx->irq_rx_adaptive = rx_adaptive;
2202539de7c5SBert Kenward 	efx->irq_rx_moderation_us = rx_usecs;
2203874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx) {
2204874aeea5SJeff Kirsher 		if (efx_channel_has_rx_queue(channel))
2205539de7c5SBert Kenward 			channel->irq_moderation_us = rx_usecs;
2206874aeea5SJeff Kirsher 		else if (efx_channel_has_tx_queues(channel))
2207539de7c5SBert Kenward 			channel->irq_moderation_us = tx_usecs;
22083990a8ffSCharles McLachlan 		else if (efx_channel_is_xdp_tx(channel))
22093990a8ffSCharles McLachlan 			channel->irq_moderation_us = tx_usecs;
2210874aeea5SJeff Kirsher 	}
22119e393b30SBen Hutchings 
22129e393b30SBen Hutchings 	return 0;
2213874aeea5SJeff Kirsher }
2214874aeea5SJeff Kirsher 
2215a0c4faf5SBen Hutchings void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
2216a0c4faf5SBen Hutchings 			    unsigned int *rx_usecs, bool *rx_adaptive)
2217a0c4faf5SBen Hutchings {
2218a0c4faf5SBen Hutchings 	*rx_adaptive = efx->irq_rx_adaptive;
2219539de7c5SBert Kenward 	*rx_usecs = efx->irq_rx_moderation_us;
2220a0c4faf5SBen Hutchings 
2221a0c4faf5SBen Hutchings 	/* If channels are shared between RX and TX, so is IRQ
2222a0c4faf5SBen Hutchings 	 * moderation.  Otherwise, IRQ moderation is the same for all
2223a0c4faf5SBen Hutchings 	 * TX channels and is not adaptive.
2224a0c4faf5SBen Hutchings 	 */
2225539de7c5SBert Kenward 	if (efx->tx_channel_offset == 0) {
2226a0c4faf5SBen Hutchings 		*tx_usecs = *rx_usecs;
2227539de7c5SBert Kenward 	} else {
2228539de7c5SBert Kenward 		struct efx_channel *tx_channel;
2229539de7c5SBert Kenward 
2230539de7c5SBert Kenward 		tx_channel = efx->channel[efx->tx_channel_offset];
2231539de7c5SBert Kenward 		*tx_usecs = tx_channel->irq_moderation_us;
2232539de7c5SBert Kenward 	}
2233a0c4faf5SBen Hutchings }
2234a0c4faf5SBen Hutchings 
2235874aeea5SJeff Kirsher /**************************************************************************
2236874aeea5SJeff Kirsher  *
2237874aeea5SJeff Kirsher  * Hardware monitor
2238874aeea5SJeff Kirsher  *
2239874aeea5SJeff Kirsher  **************************************************************************/
2240874aeea5SJeff Kirsher 
2241874aeea5SJeff Kirsher /* Run periodically off the general workqueue */
2242874aeea5SJeff Kirsher static void efx_monitor(struct work_struct *data)
2243874aeea5SJeff Kirsher {
2244874aeea5SJeff Kirsher 	struct efx_nic *efx = container_of(data, struct efx_nic,
2245874aeea5SJeff Kirsher 					   monitor_work.work);
2246874aeea5SJeff Kirsher 
2247874aeea5SJeff Kirsher 	netif_vdbg(efx, timer, efx->net_dev,
2248874aeea5SJeff Kirsher 		   "hardware monitor executing on CPU %d\n",
2249874aeea5SJeff Kirsher 		   raw_smp_processor_id());
2250874aeea5SJeff Kirsher 	BUG_ON(efx->type->monitor == NULL);
2251874aeea5SJeff Kirsher 
2252874aeea5SJeff Kirsher 	/* If the mac_lock is already held then it is likely a port
2253874aeea5SJeff Kirsher 	 * reconfiguration is already in place, which will likely do
2254874aeea5SJeff Kirsher 	 * most of the work of monitor() anyway. */
2255874aeea5SJeff Kirsher 	if (mutex_trylock(&efx->mac_lock)) {
2256874aeea5SJeff Kirsher 		if (efx->port_enabled)
2257874aeea5SJeff Kirsher 			efx->type->monitor(efx);
2258874aeea5SJeff Kirsher 		mutex_unlock(&efx->mac_lock);
2259874aeea5SJeff Kirsher 	}
2260874aeea5SJeff Kirsher 
2261874aeea5SJeff Kirsher 	queue_delayed_work(efx->workqueue, &efx->monitor_work,
2262874aeea5SJeff Kirsher 			   efx_monitor_interval);
2263874aeea5SJeff Kirsher }
2264874aeea5SJeff Kirsher 
2265874aeea5SJeff Kirsher /**************************************************************************
2266874aeea5SJeff Kirsher  *
2267874aeea5SJeff Kirsher  * ioctls
2268874aeea5SJeff Kirsher  *
2269874aeea5SJeff Kirsher  *************************************************************************/
2270874aeea5SJeff Kirsher 
2271874aeea5SJeff Kirsher /* Net device ioctl
2272874aeea5SJeff Kirsher  * Context: process, rtnl_lock() held.
2273874aeea5SJeff Kirsher  */
2274874aeea5SJeff Kirsher static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
2275874aeea5SJeff Kirsher {
2276874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
2277874aeea5SJeff Kirsher 	struct mii_ioctl_data *data = if_mii(ifr);
2278874aeea5SJeff Kirsher 
22797c236c43SStuart Hodgson 	if (cmd == SIOCSHWTSTAMP)
2280433dc9b3SBen Hutchings 		return efx_ptp_set_ts_config(efx, ifr);
2281433dc9b3SBen Hutchings 	if (cmd == SIOCGHWTSTAMP)
2282433dc9b3SBen Hutchings 		return efx_ptp_get_ts_config(efx, ifr);
22837c236c43SStuart Hodgson 
2284874aeea5SJeff Kirsher 	/* Convert phy_id from older PRTAD/DEVAD format */
2285874aeea5SJeff Kirsher 	if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
2286874aeea5SJeff Kirsher 	    (data->phy_id & 0xfc00) == 0x0400)
2287874aeea5SJeff Kirsher 		data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
2288874aeea5SJeff Kirsher 
2289874aeea5SJeff Kirsher 	return mdio_mii_ioctl(&efx->mdio, data, cmd);
2290874aeea5SJeff Kirsher }
2291874aeea5SJeff Kirsher 
2292874aeea5SJeff Kirsher /**************************************************************************
2293874aeea5SJeff Kirsher  *
2294874aeea5SJeff Kirsher  * NAPI interface
2295874aeea5SJeff Kirsher  *
2296874aeea5SJeff Kirsher  **************************************************************************/
2297874aeea5SJeff Kirsher 
22987f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel)
2299874aeea5SJeff Kirsher {
23007f967c01SBen Hutchings 	struct efx_nic *efx = channel->efx;
2301874aeea5SJeff Kirsher 
2302874aeea5SJeff Kirsher 	channel->napi_dev = efx->net_dev;
2303874aeea5SJeff Kirsher 	netif_napi_add(channel->napi_dev, &channel->napi_str,
2304874aeea5SJeff Kirsher 		       efx_poll, napi_weight);
2305874aeea5SJeff Kirsher }
23067f967c01SBen Hutchings 
23077f967c01SBen Hutchings static void efx_init_napi(struct efx_nic *efx)
23087f967c01SBen Hutchings {
23097f967c01SBen Hutchings 	struct efx_channel *channel;
23107f967c01SBen Hutchings 
23117f967c01SBen Hutchings 	efx_for_each_channel(channel, efx)
23127f967c01SBen Hutchings 		efx_init_napi_channel(channel);
2313874aeea5SJeff Kirsher }
2314874aeea5SJeff Kirsher 
2315874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel)
2316874aeea5SJeff Kirsher {
2317973334a1SEric Dumazet 	if (channel->napi_dev)
2318874aeea5SJeff Kirsher 		netif_napi_del(&channel->napi_str);
2319973334a1SEric Dumazet 
2320874aeea5SJeff Kirsher 	channel->napi_dev = NULL;
2321874aeea5SJeff Kirsher }
2322874aeea5SJeff Kirsher 
2323874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx)
2324874aeea5SJeff Kirsher {
2325874aeea5SJeff Kirsher 	struct efx_channel *channel;
2326874aeea5SJeff Kirsher 
2327874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx)
2328874aeea5SJeff Kirsher 		efx_fini_napi_channel(channel);
2329874aeea5SJeff Kirsher }
2330874aeea5SJeff Kirsher 
2331874aeea5SJeff Kirsher /**************************************************************************
2332874aeea5SJeff Kirsher  *
2333874aeea5SJeff Kirsher  * Kernel net device interface
2334874aeea5SJeff Kirsher  *
2335874aeea5SJeff Kirsher  *************************************************************************/
2336874aeea5SJeff Kirsher 
2337874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */
2338e340be92SShradha Shah int efx_net_open(struct net_device *net_dev)
2339874aeea5SJeff Kirsher {
2340874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
23418b7325b4SBen Hutchings 	int rc;
23428b7325b4SBen Hutchings 
2343874aeea5SJeff Kirsher 	netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
2344874aeea5SJeff Kirsher 		  raw_smp_processor_id());
2345874aeea5SJeff Kirsher 
23468b7325b4SBen Hutchings 	rc = efx_check_disabled(efx);
23478b7325b4SBen Hutchings 	if (rc)
23488b7325b4SBen Hutchings 		return rc;
2349874aeea5SJeff Kirsher 	if (efx->phy_mode & PHY_MODE_SPECIAL)
2350874aeea5SJeff Kirsher 		return -EBUSY;
2351874aeea5SJeff Kirsher 	if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
2352874aeea5SJeff Kirsher 		return -EIO;
2353874aeea5SJeff Kirsher 
2354874aeea5SJeff Kirsher 	/* Notify the kernel of the link state polled during driver load,
2355874aeea5SJeff Kirsher 	 * before the monitor starts running */
2356874aeea5SJeff Kirsher 	efx_link_status_changed(efx);
2357874aeea5SJeff Kirsher 
2358874aeea5SJeff Kirsher 	efx_start_all(efx);
23599c568fd8SPeter Dunning 	if (efx->state == STATE_DISABLED || efx->reset_pending)
23609c568fd8SPeter Dunning 		netif_device_detach(efx->net_dev);
2361dd40781eSBen Hutchings 	efx_selftest_async_start(efx);
2362874aeea5SJeff Kirsher 	return 0;
2363874aeea5SJeff Kirsher }
2364874aeea5SJeff Kirsher 
2365874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held.
2366874aeea5SJeff Kirsher  * Note that the kernel will ignore our return code; this method
2367874aeea5SJeff Kirsher  * should really be a void.
2368874aeea5SJeff Kirsher  */
2369e340be92SShradha Shah int efx_net_stop(struct net_device *net_dev)
2370874aeea5SJeff Kirsher {
2371874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
2372874aeea5SJeff Kirsher 
2373874aeea5SJeff Kirsher 	netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
2374874aeea5SJeff Kirsher 		  raw_smp_processor_id());
2375874aeea5SJeff Kirsher 
2376874aeea5SJeff Kirsher 	/* Stop the device and flush all the channels */
2377874aeea5SJeff Kirsher 	efx_stop_all(efx);
2378874aeea5SJeff Kirsher 
2379874aeea5SJeff Kirsher 	return 0;
2380874aeea5SJeff Kirsher }
2381874aeea5SJeff Kirsher 
2382874aeea5SJeff Kirsher /* Context: process, dev_base_lock or RTNL held, non-blocking. */
2383bc1f4470Sstephen hemminger static void efx_net_stats(struct net_device *net_dev,
23842aa9ef11SBen Hutchings 			  struct rtnl_link_stats64 *stats)
2385874aeea5SJeff Kirsher {
2386874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
2387874aeea5SJeff Kirsher 
2388874aeea5SJeff Kirsher 	spin_lock_bh(&efx->stats_lock);
2389cd0ecc9aSBen Hutchings 	efx->type->update_stats(efx, NULL, stats);
23901cb34522SBen Hutchings 	spin_unlock_bh(&efx->stats_lock);
2391874aeea5SJeff Kirsher }
2392874aeea5SJeff Kirsher 
2393874aeea5SJeff Kirsher /* Context: netif_tx_lock held, BHs disabled. */
2394874aeea5SJeff Kirsher static void efx_watchdog(struct net_device *net_dev)
2395874aeea5SJeff Kirsher {
2396874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
2397874aeea5SJeff Kirsher 
2398874aeea5SJeff Kirsher 	netif_err(efx, tx_err, efx->net_dev,
2399874aeea5SJeff Kirsher 		  "TX stuck with port_enabled=%d: resetting channels\n",
2400874aeea5SJeff Kirsher 		  efx->port_enabled);
2401874aeea5SJeff Kirsher 
2402874aeea5SJeff Kirsher 	efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
2403874aeea5SJeff Kirsher }
2404874aeea5SJeff Kirsher 
2405e45a4fedSCharles McLachlan static unsigned int efx_xdp_max_mtu(struct efx_nic *efx)
2406e45a4fedSCharles McLachlan {
2407e45a4fedSCharles McLachlan 	/* The maximum MTU that we can fit in a single page, allowing for
2408e45a4fedSCharles McLachlan 	 * framing, overhead and XDP headroom.
2409e45a4fedSCharles McLachlan 	 */
2410e45a4fedSCharles McLachlan 	int overhead = EFX_MAX_FRAME_LEN(0) + sizeof(struct efx_rx_page_state) +
2411e45a4fedSCharles McLachlan 		       efx->rx_prefix_size + efx->type->rx_buffer_padding +
2412e45a4fedSCharles McLachlan 		       efx->rx_ip_align + XDP_PACKET_HEADROOM;
2413e45a4fedSCharles McLachlan 
2414e45a4fedSCharles McLachlan 	return PAGE_SIZE - overhead;
2415e45a4fedSCharles McLachlan }
2416874aeea5SJeff Kirsher 
2417874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */
2418874aeea5SJeff Kirsher static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
2419874aeea5SJeff Kirsher {
2420874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
24218b7325b4SBen Hutchings 	int rc;
2422874aeea5SJeff Kirsher 
24238b7325b4SBen Hutchings 	rc = efx_check_disabled(efx);
24248b7325b4SBen Hutchings 	if (rc)
24258b7325b4SBen Hutchings 		return rc;
2426874aeea5SJeff Kirsher 
2427e45a4fedSCharles McLachlan 	if (rtnl_dereference(efx->xdp_prog) &&
2428e45a4fedSCharles McLachlan 	    new_mtu > efx_xdp_max_mtu(efx)) {
2429e45a4fedSCharles McLachlan 		netif_err(efx, drv, efx->net_dev,
2430e45a4fedSCharles McLachlan 			  "Requested MTU of %d too big for XDP (max: %d)\n",
2431e45a4fedSCharles McLachlan 			  new_mtu, efx_xdp_max_mtu(efx));
2432e45a4fedSCharles McLachlan 		return -EINVAL;
2433e45a4fedSCharles McLachlan 	}
2434e45a4fedSCharles McLachlan 
2435874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
2436874aeea5SJeff Kirsher 
243729c69a48SBen Hutchings 	efx_device_detach_sync(efx);
243829c69a48SBen Hutchings 	efx_stop_all(efx);
243929c69a48SBen Hutchings 
2440874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
2441874aeea5SJeff Kirsher 	net_dev->mtu = new_mtu;
24420d322413SEdward Cree 	efx_mac_reconfigure(efx);
2443874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
2444874aeea5SJeff Kirsher 
2445874aeea5SJeff Kirsher 	efx_start_all(efx);
24469c568fd8SPeter Dunning 	efx_device_attach_if_not_resetting(efx);
24476c8eef4aSBen Hutchings 	return 0;
2448874aeea5SJeff Kirsher }
2449874aeea5SJeff Kirsher 
2450874aeea5SJeff Kirsher static int efx_set_mac_address(struct net_device *net_dev, void *data)
2451874aeea5SJeff Kirsher {
2452874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
2453874aeea5SJeff Kirsher 	struct sockaddr *addr = data;
2454e0b3ae30SBen Hutchings 	u8 *new_addr = addr->sa_data;
2455cfc77c2fSShradha Shah 	u8 old_addr[6];
2456cfc77c2fSShradha Shah 	int rc;
2457874aeea5SJeff Kirsher 
2458874aeea5SJeff Kirsher 	if (!is_valid_ether_addr(new_addr)) {
2459874aeea5SJeff Kirsher 		netif_err(efx, drv, efx->net_dev,
2460874aeea5SJeff Kirsher 			  "invalid ethernet MAC address requested: %pM\n",
2461874aeea5SJeff Kirsher 			  new_addr);
2462504f9b5aSDanny Kukawka 		return -EADDRNOTAVAIL;
2463874aeea5SJeff Kirsher 	}
2464874aeea5SJeff Kirsher 
2465cfc77c2fSShradha Shah 	/* save old address */
2466cfc77c2fSShradha Shah 	ether_addr_copy(old_addr, net_dev->dev_addr);
2467cd84ff4dSEdward Cree 	ether_addr_copy(net_dev->dev_addr, new_addr);
2468910c8789SShradha Shah 	if (efx->type->set_mac_address) {
2469910c8789SShradha Shah 		rc = efx->type->set_mac_address(efx);
2470cfc77c2fSShradha Shah 		if (rc) {
2471cfc77c2fSShradha Shah 			ether_addr_copy(net_dev->dev_addr, old_addr);
2472cfc77c2fSShradha Shah 			return rc;
2473cfc77c2fSShradha Shah 		}
2474cfc77c2fSShradha Shah 	}
2475874aeea5SJeff Kirsher 
2476874aeea5SJeff Kirsher 	/* Reconfigure the MAC */
2477874aeea5SJeff Kirsher 	mutex_lock(&efx->mac_lock);
24780d322413SEdward Cree 	efx_mac_reconfigure(efx);
2479874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
2480874aeea5SJeff Kirsher 
2481874aeea5SJeff Kirsher 	return 0;
2482874aeea5SJeff Kirsher }
2483874aeea5SJeff Kirsher 
2484874aeea5SJeff Kirsher /* Context: netif_addr_lock held, BHs disabled. */
24850fca8c97SBen Hutchings static void efx_set_rx_mode(struct net_device *net_dev)
2486874aeea5SJeff Kirsher {
2487874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
2488874aeea5SJeff Kirsher 
2489874aeea5SJeff Kirsher 	if (efx->port_enabled)
2490874aeea5SJeff Kirsher 		queue_work(efx->workqueue, &efx->mac_work);
2491874aeea5SJeff Kirsher 	/* Otherwise efx_start_port() will do this */
2492874aeea5SJeff Kirsher }
2493874aeea5SJeff Kirsher 
2494c8f44affSMichał Mirosław static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
2495874aeea5SJeff Kirsher {
2496874aeea5SJeff Kirsher 	struct efx_nic *efx = netdev_priv(net_dev);
24974a53ea8aSAndrew Rybchenko 	int rc;
2498874aeea5SJeff Kirsher 
2499874aeea5SJeff Kirsher 	/* If disabling RX n-tuple filtering, clear existing filters */
25004a53ea8aSAndrew Rybchenko 	if (net_dev->features & ~data & NETIF_F_NTUPLE) {
25014a53ea8aSAndrew Rybchenko 		rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
25024a53ea8aSAndrew Rybchenko 		if (rc)
25034a53ea8aSAndrew Rybchenko 			return rc;
25044a53ea8aSAndrew Rybchenko 	}
25054a53ea8aSAndrew Rybchenko 
25066978729fSEdward Cree 	/* If Rx VLAN filter is changed, update filters via mac_reconfigure.
25076978729fSEdward Cree 	 * If rx-fcs is changed, mac_reconfigure updates that too.
25086978729fSEdward Cree 	 */
25096978729fSEdward Cree 	if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER |
25106978729fSEdward Cree 					  NETIF_F_RXFCS)) {
25114a53ea8aSAndrew Rybchenko 		/* efx_set_rx_mode() will schedule MAC work to update filters
25124a53ea8aSAndrew Rybchenko 		 * when a new features are finally set in net_dev.
25134a53ea8aSAndrew Rybchenko 		 */
25144a53ea8aSAndrew Rybchenko 		efx_set_rx_mode(net_dev);
25154a53ea8aSAndrew Rybchenko 	}
2516874aeea5SJeff Kirsher 
2517874aeea5SJeff Kirsher 	return 0;
2518874aeea5SJeff Kirsher }
2519874aeea5SJeff Kirsher 
2520b40296fcSWei Yongjun static int efx_get_phys_port_id(struct net_device *net_dev,
252108a7b29bSBert Kenward 				struct netdev_phys_item_id *ppid)
252208a7b29bSBert Kenward {
252308a7b29bSBert Kenward 	struct efx_nic *efx = netdev_priv(net_dev);
252408a7b29bSBert Kenward 
252508a7b29bSBert Kenward 	if (efx->type->get_phys_port_id)
252608a7b29bSBert Kenward 		return efx->type->get_phys_port_id(efx, ppid);
252708a7b29bSBert Kenward 	else
252808a7b29bSBert Kenward 		return -EOPNOTSUPP;
252908a7b29bSBert Kenward }
253008a7b29bSBert Kenward 
2531ac019f08SBert Kenward static int efx_get_phys_port_name(struct net_device *net_dev,
2532ac019f08SBert Kenward 				  char *name, size_t len)
2533ac019f08SBert Kenward {
2534ac019f08SBert Kenward 	struct efx_nic *efx = netdev_priv(net_dev);
2535ac019f08SBert Kenward 
2536ac019f08SBert Kenward 	if (snprintf(name, len, "p%u", efx->port_num) >= len)
2537ac019f08SBert Kenward 		return -EINVAL;
2538ac019f08SBert Kenward 	return 0;
2539ac019f08SBert Kenward }
2540ac019f08SBert Kenward 
25414a53ea8aSAndrew Rybchenko static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)
25424a53ea8aSAndrew Rybchenko {
25434a53ea8aSAndrew Rybchenko 	struct efx_nic *efx = netdev_priv(net_dev);
25444a53ea8aSAndrew Rybchenko 
25454a53ea8aSAndrew Rybchenko 	if (efx->type->vlan_rx_add_vid)
25464a53ea8aSAndrew Rybchenko 		return efx->type->vlan_rx_add_vid(efx, proto, vid);
25474a53ea8aSAndrew Rybchenko 	else
25484a53ea8aSAndrew Rybchenko 		return -EOPNOTSUPP;
25494a53ea8aSAndrew Rybchenko }
25504a53ea8aSAndrew Rybchenko 
25514a53ea8aSAndrew Rybchenko static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)
25524a53ea8aSAndrew Rybchenko {
25534a53ea8aSAndrew Rybchenko 	struct efx_nic *efx = netdev_priv(net_dev);
25544a53ea8aSAndrew Rybchenko 
25554a53ea8aSAndrew Rybchenko 	if (efx->type->vlan_rx_kill_vid)
25564a53ea8aSAndrew Rybchenko 		return efx->type->vlan_rx_kill_vid(efx, proto, vid);
25574a53ea8aSAndrew Rybchenko 	else
25584a53ea8aSAndrew Rybchenko 		return -EOPNOTSUPP;
25594a53ea8aSAndrew Rybchenko }
25604a53ea8aSAndrew Rybchenko 
2561e5fbd977SJon Cooper static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in)
2562e5fbd977SJon Cooper {
2563e5fbd977SJon Cooper 	switch (in) {
2564e5fbd977SJon Cooper 	case UDP_TUNNEL_TYPE_VXLAN:
2565e5fbd977SJon Cooper 		return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN;
2566e5fbd977SJon Cooper 	case UDP_TUNNEL_TYPE_GENEVE:
2567e5fbd977SJon Cooper 		return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE;
2568e5fbd977SJon Cooper 	default:
2569e5fbd977SJon Cooper 		return -1;
2570e5fbd977SJon Cooper 	}
2571e5fbd977SJon Cooper }
2572e5fbd977SJon Cooper 
2573e5fbd977SJon Cooper static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti)
2574e5fbd977SJon Cooper {
2575e5fbd977SJon Cooper 	struct efx_nic *efx = netdev_priv(dev);
2576e5fbd977SJon Cooper 	struct efx_udp_tunnel tnl;
2577e5fbd977SJon Cooper 	int efx_tunnel_type;
2578e5fbd977SJon Cooper 
2579e5fbd977SJon Cooper 	efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2580e5fbd977SJon Cooper 	if (efx_tunnel_type < 0)
2581e5fbd977SJon Cooper 		return;
2582e5fbd977SJon Cooper 
2583e5fbd977SJon Cooper 	tnl.type = (u16)efx_tunnel_type;
2584e5fbd977SJon Cooper 	tnl.port = ti->port;
2585e5fbd977SJon Cooper 
2586e5fbd977SJon Cooper 	if (efx->type->udp_tnl_add_port)
2587e5fbd977SJon Cooper 		(void)efx->type->udp_tnl_add_port(efx, tnl);
2588e5fbd977SJon Cooper }
2589e5fbd977SJon Cooper 
2590e5fbd977SJon Cooper static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti)
2591e5fbd977SJon Cooper {
2592e5fbd977SJon Cooper 	struct efx_nic *efx = netdev_priv(dev);
2593e5fbd977SJon Cooper 	struct efx_udp_tunnel tnl;
2594e5fbd977SJon Cooper 	int efx_tunnel_type;
2595e5fbd977SJon Cooper 
2596e5fbd977SJon Cooper 	efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2597e5fbd977SJon Cooper 	if (efx_tunnel_type < 0)
2598e5fbd977SJon Cooper 		return;
2599e5fbd977SJon Cooper 
2600e5fbd977SJon Cooper 	tnl.type = (u16)efx_tunnel_type;
2601e5fbd977SJon Cooper 	tnl.port = ti->port;
2602e5fbd977SJon Cooper 
2603c04ca616SDan Carpenter 	if (efx->type->udp_tnl_del_port)
2604e5fbd977SJon Cooper 		(void)efx->type->udp_tnl_del_port(efx, tnl);
2605e5fbd977SJon Cooper }
2606e5fbd977SJon Cooper 
26077fa8d547SShradha Shah static const struct net_device_ops efx_netdev_ops = {
2608874aeea5SJeff Kirsher 	.ndo_open		= efx_net_open,
2609874aeea5SJeff Kirsher 	.ndo_stop		= efx_net_stop,
2610874aeea5SJeff Kirsher 	.ndo_get_stats64	= efx_net_stats,
2611874aeea5SJeff Kirsher 	.ndo_tx_timeout		= efx_watchdog,
2612874aeea5SJeff Kirsher 	.ndo_start_xmit		= efx_hard_start_xmit,
2613874aeea5SJeff Kirsher 	.ndo_validate_addr	= eth_validate_addr,
2614874aeea5SJeff Kirsher 	.ndo_do_ioctl		= efx_ioctl,
2615874aeea5SJeff Kirsher 	.ndo_change_mtu		= efx_change_mtu,
2616874aeea5SJeff Kirsher 	.ndo_set_mac_address	= efx_set_mac_address,
26170fca8c97SBen Hutchings 	.ndo_set_rx_mode	= efx_set_rx_mode,
2618874aeea5SJeff Kirsher 	.ndo_set_features	= efx_set_features,
26194a53ea8aSAndrew Rybchenko 	.ndo_vlan_rx_add_vid	= efx_vlan_rx_add_vid,
26204a53ea8aSAndrew Rybchenko 	.ndo_vlan_rx_kill_vid	= efx_vlan_rx_kill_vid,
2621cd2d5b52SBen Hutchings #ifdef CONFIG_SFC_SRIOV
26227fa8d547SShradha Shah 	.ndo_set_vf_mac		= efx_sriov_set_vf_mac,
26237fa8d547SShradha Shah 	.ndo_set_vf_vlan	= efx_sriov_set_vf_vlan,
26247fa8d547SShradha Shah 	.ndo_set_vf_spoofchk	= efx_sriov_set_vf_spoofchk,
26257fa8d547SShradha Shah 	.ndo_get_vf_config	= efx_sriov_get_vf_config,
26264392dc69SEdward Cree 	.ndo_set_vf_link_state  = efx_sriov_set_vf_link_state,
2627cd2d5b52SBen Hutchings #endif
262808a7b29bSBert Kenward 	.ndo_get_phys_port_id   = efx_get_phys_port_id,
2629ac019f08SBert Kenward 	.ndo_get_phys_port_name	= efx_get_phys_port_name,
2630874aeea5SJeff Kirsher 	.ndo_setup_tc		= efx_setup_tc,
2631874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL
2632874aeea5SJeff Kirsher 	.ndo_rx_flow_steer	= efx_filter_rfs,
2633874aeea5SJeff Kirsher #endif
2634e5fbd977SJon Cooper 	.ndo_udp_tunnel_add	= efx_udp_tunnel_add,
2635e5fbd977SJon Cooper 	.ndo_udp_tunnel_del	= efx_udp_tunnel_del,
2636e45a4fedSCharles McLachlan 	.ndo_bpf		= efx_xdp
2637874aeea5SJeff Kirsher };
2638874aeea5SJeff Kirsher 
2639e45a4fedSCharles McLachlan static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog)
2640e45a4fedSCharles McLachlan {
2641e45a4fedSCharles McLachlan 	struct bpf_prog *old_prog;
2642e45a4fedSCharles McLachlan 
2643e45a4fedSCharles McLachlan 	if (efx->xdp_rxq_info_failed) {
2644e45a4fedSCharles McLachlan 		netif_err(efx, drv, efx->net_dev,
2645e45a4fedSCharles McLachlan 			  "Unable to bind XDP program due to previous failure of rxq_info\n");
2646e45a4fedSCharles McLachlan 		return -EINVAL;
2647e45a4fedSCharles McLachlan 	}
2648e45a4fedSCharles McLachlan 
2649e45a4fedSCharles McLachlan 	if (prog && efx->net_dev->mtu > efx_xdp_max_mtu(efx)) {
2650e45a4fedSCharles McLachlan 		netif_err(efx, drv, efx->net_dev,
2651e45a4fedSCharles McLachlan 			  "Unable to configure XDP with MTU of %d (max: %d)\n",
2652e45a4fedSCharles McLachlan 			  efx->net_dev->mtu, efx_xdp_max_mtu(efx));
2653e45a4fedSCharles McLachlan 		return -EINVAL;
2654e45a4fedSCharles McLachlan 	}
2655e45a4fedSCharles McLachlan 
2656e45a4fedSCharles McLachlan 	old_prog = rtnl_dereference(efx->xdp_prog);
2657e45a4fedSCharles McLachlan 	rcu_assign_pointer(efx->xdp_prog, prog);
2658e45a4fedSCharles McLachlan 	/* Release the reference that was originally passed by the caller. */
2659e45a4fedSCharles McLachlan 	if (old_prog)
2660e45a4fedSCharles McLachlan 		bpf_prog_put(old_prog);
2661e45a4fedSCharles McLachlan 
2662e45a4fedSCharles McLachlan 	return 0;
2663e45a4fedSCharles McLachlan }
2664e45a4fedSCharles McLachlan 
2665e45a4fedSCharles McLachlan /* Context: process, rtnl_lock() held. */
2666e45a4fedSCharles McLachlan static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2667e45a4fedSCharles McLachlan {
2668e45a4fedSCharles McLachlan 	struct efx_nic *efx = netdev_priv(dev);
2669e45a4fedSCharles McLachlan 	struct bpf_prog *xdp_prog;
2670e45a4fedSCharles McLachlan 
2671e45a4fedSCharles McLachlan 	switch (xdp->command) {
2672e45a4fedSCharles McLachlan 	case XDP_SETUP_PROG:
2673e45a4fedSCharles McLachlan 		return efx_xdp_setup_prog(efx, xdp->prog);
2674e45a4fedSCharles McLachlan 	case XDP_QUERY_PROG:
2675e45a4fedSCharles McLachlan 		xdp_prog = rtnl_dereference(efx->xdp_prog);
2676e45a4fedSCharles McLachlan 		xdp->prog_id = xdp_prog ? xdp_prog->aux->id : 0;
2677e45a4fedSCharles McLachlan 		return 0;
2678e45a4fedSCharles McLachlan 	default:
2679e45a4fedSCharles McLachlan 		return -EINVAL;
2680e45a4fedSCharles McLachlan 	}
2681e45a4fedSCharles McLachlan }
2682e45a4fedSCharles McLachlan 
2683874aeea5SJeff Kirsher static void efx_update_name(struct efx_nic *efx)
2684874aeea5SJeff Kirsher {
2685874aeea5SJeff Kirsher 	strcpy(efx->name, efx->net_dev->name);
2686874aeea5SJeff Kirsher 	efx_mtd_rename(efx);
2687874aeea5SJeff Kirsher 	efx_set_channel_names(efx);
2688874aeea5SJeff Kirsher }
2689874aeea5SJeff Kirsher 
2690874aeea5SJeff Kirsher static int efx_netdev_event(struct notifier_block *this,
2691874aeea5SJeff Kirsher 			    unsigned long event, void *ptr)
2692874aeea5SJeff Kirsher {
2693351638e7SJiri Pirko 	struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
2694874aeea5SJeff Kirsher 
26957fa8d547SShradha Shah 	if ((net_dev->netdev_ops == &efx_netdev_ops) &&
2696874aeea5SJeff Kirsher 	    event == NETDEV_CHANGENAME)
2697874aeea5SJeff Kirsher 		efx_update_name(netdev_priv(net_dev));
2698874aeea5SJeff Kirsher 
2699874aeea5SJeff Kirsher 	return NOTIFY_DONE;
2700874aeea5SJeff Kirsher }
2701874aeea5SJeff Kirsher 
2702874aeea5SJeff Kirsher static struct notifier_block efx_netdev_notifier = {
2703874aeea5SJeff Kirsher 	.notifier_call = efx_netdev_event,
2704874aeea5SJeff Kirsher };
2705874aeea5SJeff Kirsher 
2706874aeea5SJeff Kirsher static ssize_t
2707874aeea5SJeff Kirsher show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
2708874aeea5SJeff Kirsher {
27093e03a8baSChuhong Yuan 	struct efx_nic *efx = dev_get_drvdata(dev);
2710874aeea5SJeff Kirsher 	return sprintf(buf, "%d\n", efx->phy_type);
2711874aeea5SJeff Kirsher }
2712776fbcc9SBen Hutchings static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL);
2713874aeea5SJeff Kirsher 
2714e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING
2715e7fef9b4SEdward Cree static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr,
2716e7fef9b4SEdward Cree 			     char *buf)
2717e7fef9b4SEdward Cree {
27183e03a8baSChuhong Yuan 	struct efx_nic *efx = dev_get_drvdata(dev);
2719e7fef9b4SEdward Cree 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2720e7fef9b4SEdward Cree 
2721e7fef9b4SEdward Cree 	return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled);
2722e7fef9b4SEdward Cree }
2723e7fef9b4SEdward Cree static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr,
2724e7fef9b4SEdward Cree 			    const char *buf, size_t count)
2725e7fef9b4SEdward Cree {
27263e03a8baSChuhong Yuan 	struct efx_nic *efx = dev_get_drvdata(dev);
2727e7fef9b4SEdward Cree 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2728e7fef9b4SEdward Cree 	bool enable = count > 0 && *buf != '0';
2729e7fef9b4SEdward Cree 
2730e7fef9b4SEdward Cree 	mcdi->logging_enabled = enable;
2731e7fef9b4SEdward Cree 	return count;
2732e7fef9b4SEdward Cree }
2733e7fef9b4SEdward Cree static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log);
2734e7fef9b4SEdward Cree #endif
2735e7fef9b4SEdward Cree 
2736874aeea5SJeff Kirsher static int efx_register_netdev(struct efx_nic *efx)
2737874aeea5SJeff Kirsher {
2738874aeea5SJeff Kirsher 	struct net_device *net_dev = efx->net_dev;
2739874aeea5SJeff Kirsher 	struct efx_channel *channel;
2740874aeea5SJeff Kirsher 	int rc;
2741874aeea5SJeff Kirsher 
2742874aeea5SJeff Kirsher 	net_dev->watchdog_timeo = 5 * HZ;
2743874aeea5SJeff Kirsher 	net_dev->irq = efx->pci_dev->irq;
27447fa8d547SShradha Shah 	net_dev->netdev_ops = &efx_netdev_ops;
27457fa8d547SShradha Shah 	if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
27468127d661SBen Hutchings 		net_dev->priv_flags |= IFF_UNICAST_FLT;
27477ad24ea4SWilfried Klaebe 	net_dev->ethtool_ops = &efx_ethtool_ops;
27487e6d06f0SBen Hutchings 	net_dev->gso_max_segs = EFX_TSO_MAX_SEGS;
2749cd94e519SBert Kenward 	net_dev->min_mtu = EFX_MIN_MTU;
2750cd94e519SBert Kenward 	net_dev->max_mtu = EFX_MAX_MTU;
2751874aeea5SJeff Kirsher 
2752874aeea5SJeff Kirsher 	rtnl_lock();
2753874aeea5SJeff Kirsher 
27547153f623SBen Hutchings 	/* Enable resets to be scheduled and check whether any were
27557153f623SBen Hutchings 	 * already requested.  If so, the NIC is probably hosed so we
27567153f623SBen Hutchings 	 * abort.
27577153f623SBen Hutchings 	 */
27587153f623SBen Hutchings 	efx->state = STATE_READY;
27597153f623SBen Hutchings 	smp_mb(); /* ensure we change state before checking reset_pending */
27607153f623SBen Hutchings 	if (efx->reset_pending) {
27617153f623SBen Hutchings 		netif_err(efx, probe, efx->net_dev,
27627153f623SBen Hutchings 			  "aborting probe due to scheduled reset\n");
27637153f623SBen Hutchings 		rc = -EIO;
27647153f623SBen Hutchings 		goto fail_locked;
27657153f623SBen Hutchings 	}
27667153f623SBen Hutchings 
2767874aeea5SJeff Kirsher 	rc = dev_alloc_name(net_dev, net_dev->name);
2768874aeea5SJeff Kirsher 	if (rc < 0)
2769874aeea5SJeff Kirsher 		goto fail_locked;
2770874aeea5SJeff Kirsher 	efx_update_name(efx);
2771874aeea5SJeff Kirsher 
27728f8b3d51SBen Hutchings 	/* Always start with carrier off; PHY events will detect the link */
27738f8b3d51SBen Hutchings 	netif_carrier_off(net_dev);
27748f8b3d51SBen Hutchings 
2775874aeea5SJeff Kirsher 	rc = register_netdevice(net_dev);
2776874aeea5SJeff Kirsher 	if (rc)
2777874aeea5SJeff Kirsher 		goto fail_locked;
2778874aeea5SJeff Kirsher 
2779874aeea5SJeff Kirsher 	efx_for_each_channel(channel, efx) {
2780874aeea5SJeff Kirsher 		struct efx_tx_queue *tx_queue;
2781874aeea5SJeff Kirsher 		efx_for_each_channel_tx_queue(tx_queue, channel)
2782874aeea5SJeff Kirsher 			efx_init_tx_queue_core_txq(tx_queue);
2783874aeea5SJeff Kirsher 	}
2784874aeea5SJeff Kirsher 
27850bcf4a64SBen Hutchings 	efx_associate(efx);
27860bcf4a64SBen Hutchings 
2787874aeea5SJeff Kirsher 	rtnl_unlock();
2788874aeea5SJeff Kirsher 
2789874aeea5SJeff Kirsher 	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2790874aeea5SJeff Kirsher 	if (rc) {
2791874aeea5SJeff Kirsher 		netif_err(efx, drv, efx->net_dev,
2792874aeea5SJeff Kirsher 			  "failed to init net dev attributes\n");
2793874aeea5SJeff Kirsher 		goto fail_registered;
2794874aeea5SJeff Kirsher 	}
2795e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING
2796e7fef9b4SEdward Cree 	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2797e7fef9b4SEdward Cree 	if (rc) {
2798e7fef9b4SEdward Cree 		netif_err(efx, drv, efx->net_dev,
2799e7fef9b4SEdward Cree 			  "failed to init net dev attributes\n");
2800e7fef9b4SEdward Cree 		goto fail_attr_mcdi_logging;
2801e7fef9b4SEdward Cree 	}
2802e7fef9b4SEdward Cree #endif
2803874aeea5SJeff Kirsher 
2804874aeea5SJeff Kirsher 	return 0;
2805874aeea5SJeff Kirsher 
2806e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING
2807e7fef9b4SEdward Cree fail_attr_mcdi_logging:
2808e7fef9b4SEdward Cree 	device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2809e7fef9b4SEdward Cree #endif
28107153f623SBen Hutchings fail_registered:
28117153f623SBen Hutchings 	rtnl_lock();
28120bcf4a64SBen Hutchings 	efx_dissociate(efx);
28137153f623SBen Hutchings 	unregister_netdevice(net_dev);
2814874aeea5SJeff Kirsher fail_locked:
28157153f623SBen Hutchings 	efx->state = STATE_UNINIT;
2816874aeea5SJeff Kirsher 	rtnl_unlock();
2817874aeea5SJeff Kirsher 	netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
2818874aeea5SJeff Kirsher 	return rc;
2819874aeea5SJeff Kirsher }
2820874aeea5SJeff Kirsher 
2821874aeea5SJeff Kirsher static void efx_unregister_netdev(struct efx_nic *efx)
2822874aeea5SJeff Kirsher {
2823874aeea5SJeff Kirsher 	if (!efx->net_dev)
2824874aeea5SJeff Kirsher 		return;
2825874aeea5SJeff Kirsher 
2826874aeea5SJeff Kirsher 	BUG_ON(netdev_priv(efx->net_dev) != efx);
2827874aeea5SJeff Kirsher 
2828e7fef9b4SEdward Cree 	if (efx_dev_registered(efx)) {
2829874aeea5SJeff Kirsher 		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2830e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING
2831e7fef9b4SEdward Cree 		device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2832e7fef9b4SEdward Cree #endif
2833874aeea5SJeff Kirsher 		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2834e7fef9b4SEdward Cree 		unregister_netdev(efx->net_dev);
2835e7fef9b4SEdward Cree 	}
2836874aeea5SJeff Kirsher }
2837874aeea5SJeff Kirsher 
2838874aeea5SJeff Kirsher /**************************************************************************
2839874aeea5SJeff Kirsher  *
2840874aeea5SJeff Kirsher  * Device reset and suspend
2841874aeea5SJeff Kirsher  *
2842874aeea5SJeff Kirsher  **************************************************************************/
2843874aeea5SJeff Kirsher 
2844874aeea5SJeff Kirsher /* Tears down the entire software state and most of the hardware state
2845874aeea5SJeff Kirsher  * before reset.  */
2846874aeea5SJeff Kirsher void efx_reset_down(struct efx_nic *efx, enum reset_type method)
2847874aeea5SJeff Kirsher {
2848874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
2849874aeea5SJeff Kirsher 
2850e283546cSEdward Cree 	if (method == RESET_TYPE_MCDI_TIMEOUT)
2851e283546cSEdward Cree 		efx->type->prepare_flr(efx);
2852e283546cSEdward Cree 
2853874aeea5SJeff Kirsher 	efx_stop_all(efx);
2854d8291187SBen Hutchings 	efx_disable_interrupts(efx);
28555642ceefSBen Hutchings 
28565642ceefSBen Hutchings 	mutex_lock(&efx->mac_lock);
2857193f2003SBert Kenward 	down_write(&efx->filter_sem);
2858e0a65e3cSEdward Cree 	mutex_lock(&efx->rss_lock);
2859087e9025SJon Cooper 	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2860087e9025SJon Cooper 	    method != RESET_TYPE_DATAPATH)
2861874aeea5SJeff Kirsher 		efx->phy_op->fini(efx);
2862874aeea5SJeff Kirsher 	efx->type->fini(efx);
2863874aeea5SJeff Kirsher }
2864874aeea5SJeff Kirsher 
2865874aeea5SJeff Kirsher /* This function will always ensure that the locks acquired in
2866874aeea5SJeff Kirsher  * efx_reset_down() are released. A failure return code indicates
2867874aeea5SJeff Kirsher  * that we were unable to reinitialise the hardware, and the
2868874aeea5SJeff Kirsher  * driver should be disabled. If ok is false, then the rx and tx
2869874aeea5SJeff Kirsher  * engines are not restarted, pending a RESET_DISABLE. */
2870874aeea5SJeff Kirsher int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
2871874aeea5SJeff Kirsher {
2872874aeea5SJeff Kirsher 	int rc;
2873874aeea5SJeff Kirsher 
2874874aeea5SJeff Kirsher 	EFX_ASSERT_RESET_SERIALISED(efx);
2875874aeea5SJeff Kirsher 
2876e283546cSEdward Cree 	if (method == RESET_TYPE_MCDI_TIMEOUT)
2877e283546cSEdward Cree 		efx->type->finish_flr(efx);
2878e283546cSEdward Cree 
2879e283546cSEdward Cree 	/* Ensure that SRAM is initialised even if we're disabling the device */
2880874aeea5SJeff Kirsher 	rc = efx->type->init(efx);
2881874aeea5SJeff Kirsher 	if (rc) {
2882874aeea5SJeff Kirsher 		netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
2883874aeea5SJeff Kirsher 		goto fail;
2884874aeea5SJeff Kirsher 	}
2885874aeea5SJeff Kirsher 
2886874aeea5SJeff Kirsher 	if (!ok)
2887874aeea5SJeff Kirsher 		goto fail;
2888874aeea5SJeff Kirsher 
2889087e9025SJon Cooper 	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2890087e9025SJon Cooper 	    method != RESET_TYPE_DATAPATH) {
2891874aeea5SJeff Kirsher 		rc = efx->phy_op->init(efx);
2892874aeea5SJeff Kirsher 		if (rc)
2893874aeea5SJeff Kirsher 			goto fail;
2894267d9d73SEdward Cree 		rc = efx->phy_op->reconfigure(efx);
2895267d9d73SEdward Cree 		if (rc && rc != -EPERM)
2896874aeea5SJeff Kirsher 			netif_err(efx, drv, efx->net_dev,
2897874aeea5SJeff Kirsher 				  "could not restore PHY settings\n");
2898874aeea5SJeff Kirsher 	}
2899874aeea5SJeff Kirsher 
2900261e4d96SJon Cooper 	rc = efx_enable_interrupts(efx);
2901261e4d96SJon Cooper 	if (rc)
2902261e4d96SJon Cooper 		goto fail;
29036d8aaaf6SDaniel Pieczko 
29046d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV
29056d8aaaf6SDaniel Pieczko 	rc = efx->type->vswitching_restore(efx);
29066d8aaaf6SDaniel Pieczko 	if (rc) /* not fatal; the PF will still work fine */
29076d8aaaf6SDaniel Pieczko 		netif_warn(efx, probe, efx->net_dev,
29086d8aaaf6SDaniel Pieczko 			   "failed to restore vswitching rc=%d;"
29096d8aaaf6SDaniel Pieczko 			   " VFs may not function\n", rc);
29106d8aaaf6SDaniel Pieczko #endif
29116d8aaaf6SDaniel Pieczko 
291242356d9aSEdward Cree 	if (efx->type->rx_restore_rss_contexts)
291342356d9aSEdward Cree 		efx->type->rx_restore_rss_contexts(efx);
2914e0a65e3cSEdward Cree 	mutex_unlock(&efx->rss_lock);
2915193f2003SBert Kenward 	efx->type->filter_table_restore(efx);
2916193f2003SBert Kenward 	up_write(&efx->filter_sem);
29177fa8d547SShradha Shah 	if (efx->type->sriov_reset)
2918d98a4ffeSShradha Shah 		efx->type->sriov_reset(efx);
2919874aeea5SJeff Kirsher 
2920874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
2921874aeea5SJeff Kirsher 
2922874aeea5SJeff Kirsher 	efx_start_all(efx);
2923874aeea5SJeff Kirsher 
2924e5fbd977SJon Cooper 	if (efx->type->udp_tnl_push_ports)
2925e5fbd977SJon Cooper 		efx->type->udp_tnl_push_ports(efx);
2926e5fbd977SJon Cooper 
2927874aeea5SJeff Kirsher 	return 0;
2928874aeea5SJeff Kirsher 
2929874aeea5SJeff Kirsher fail:
2930874aeea5SJeff Kirsher 	efx->port_initialized = false;
2931874aeea5SJeff Kirsher 
2932e0a65e3cSEdward Cree 	mutex_unlock(&efx->rss_lock);
2933193f2003SBert Kenward 	up_write(&efx->filter_sem);
2934874aeea5SJeff Kirsher 	mutex_unlock(&efx->mac_lock);
2935874aeea5SJeff Kirsher 
2936874aeea5SJeff Kirsher 	return rc;
2937874aeea5SJeff Kirsher }
2938874aeea5SJeff Kirsher 
2939874aeea5SJeff Kirsher /* Reset the NIC using the specified method.  Note that the reset may
2940874aeea5SJeff Kirsher  * fail, in which case the card will be left in an unusable state.
2941874aeea5SJeff Kirsher  *
2942874aeea5SJeff Kirsher  * Caller must hold the rtnl_lock.
2943874aeea5SJeff Kirsher  */
2944874aeea5SJeff Kirsher int efx_reset(struct efx_nic *efx, enum reset_type method)
2945874aeea5SJeff Kirsher {
2946874aeea5SJeff Kirsher 	int rc, rc2;
2947874aeea5SJeff Kirsher 	bool disabled;
2948874aeea5SJeff Kirsher 
2949874aeea5SJeff Kirsher 	netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
2950874aeea5SJeff Kirsher 		   RESET_TYPE(method));
2951874aeea5SJeff Kirsher 
2952c2f3b8e3SDaniel Pieczko 	efx_device_detach_sync(efx);
2953874aeea5SJeff Kirsher 	efx_reset_down(efx, method);
2954874aeea5SJeff Kirsher 
2955874aeea5SJeff Kirsher 	rc = efx->type->reset(efx, method);
2956874aeea5SJeff Kirsher 	if (rc) {
2957874aeea5SJeff Kirsher 		netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
2958874aeea5SJeff Kirsher 		goto out;
2959874aeea5SJeff Kirsher 	}
2960874aeea5SJeff Kirsher 
2961874aeea5SJeff Kirsher 	/* Clear flags for the scopes we covered.  We assume the NIC and
2962874aeea5SJeff Kirsher 	 * driver are now quiescent so that there is no race here.
2963874aeea5SJeff Kirsher 	 */
2964e283546cSEdward Cree 	if (method < RESET_TYPE_MAX_METHOD)
2965874aeea5SJeff Kirsher 		efx->reset_pending &= -(1 << (method + 1));
2966e283546cSEdward Cree 	else /* it doesn't fit into the well-ordered scope hierarchy */
2967e283546cSEdward Cree 		__clear_bit(method, &efx->reset_pending);
2968874aeea5SJeff Kirsher 
2969874aeea5SJeff Kirsher 	/* Reinitialise bus-mastering, which may have been turned off before
2970874aeea5SJeff Kirsher 	 * the reset was scheduled. This is still appropriate, even in the
2971874aeea5SJeff Kirsher 	 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
2972874aeea5SJeff Kirsher 	 * can respond to requests. */
2973874aeea5SJeff Kirsher 	pci_set_master(efx->pci_dev);
2974874aeea5SJeff Kirsher 
2975874aeea5SJeff Kirsher out:
2976874aeea5SJeff Kirsher 	/* Leave device stopped if necessary */
2977626950dbSAlexandre Rames 	disabled = rc ||
2978626950dbSAlexandre Rames 		method == RESET_TYPE_DISABLE ||
2979626950dbSAlexandre Rames 		method == RESET_TYPE_RECOVER_OR_DISABLE;
2980874aeea5SJeff Kirsher 	rc2 = efx_reset_up(efx, method, !disabled);
2981874aeea5SJeff Kirsher 	if (rc2) {
2982874aeea5SJeff Kirsher 		disabled = true;
2983874aeea5SJeff Kirsher 		if (!rc)
2984874aeea5SJeff Kirsher 			rc = rc2;
2985874aeea5SJeff Kirsher 	}
2986874aeea5SJeff Kirsher 
2987874aeea5SJeff Kirsher 	if (disabled) {
2988874aeea5SJeff Kirsher 		dev_close(efx->net_dev);
2989874aeea5SJeff Kirsher 		netif_err(efx, drv, efx->net_dev, "has been disabled\n");
2990874aeea5SJeff Kirsher 		efx->state = STATE_DISABLED;
2991874aeea5SJeff Kirsher 	} else {
2992874aeea5SJeff Kirsher 		netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
29939c568fd8SPeter Dunning 		efx_device_attach_if_not_resetting(efx);
2994874aeea5SJeff Kirsher 	}
2995874aeea5SJeff Kirsher 	return rc;
2996874aeea5SJeff Kirsher }
2997874aeea5SJeff Kirsher 
2998626950dbSAlexandre Rames /* Try recovery mechanisms.
2999626950dbSAlexandre Rames  * For now only EEH is supported.
3000626950dbSAlexandre Rames  * Returns 0 if the recovery mechanisms are unsuccessful.
3001626950dbSAlexandre Rames  * Returns a non-zero value otherwise.
3002626950dbSAlexandre Rames  */
3003b28405b0SAlexandre Rames int efx_try_recovery(struct efx_nic *efx)
3004626950dbSAlexandre Rames {
3005626950dbSAlexandre Rames #ifdef CONFIG_EEH
3006626950dbSAlexandre Rames 	/* A PCI error can occur and not be seen by EEH because nothing
3007626950dbSAlexandre Rames 	 * happens on the PCI bus. In this case the driver may fail and
3008626950dbSAlexandre Rames 	 * schedule a 'recover or reset', leading to this recovery handler.
3009626950dbSAlexandre Rames 	 * Manually call the eeh failure check function.
3010626950dbSAlexandre Rames 	 */
301112a89dbaSBenjamin Herrenschmidt 	struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev);
3012626950dbSAlexandre Rames 	if (eeh_dev_check_failure(eehdev)) {
3013626950dbSAlexandre Rames 		/* The EEH mechanisms will handle the error and reset the
3014626950dbSAlexandre Rames 		 * device if necessary.
3015626950dbSAlexandre Rames 		 */
3016626950dbSAlexandre Rames 		return 1;
3017626950dbSAlexandre Rames 	}
3018626950dbSAlexandre Rames #endif
3019626950dbSAlexandre Rames 	return 0;
3020626950dbSAlexandre Rames }
3021626950dbSAlexandre Rames 
302274cd60a4SJon Cooper static void efx_wait_for_bist_end(struct efx_nic *efx)
302374cd60a4SJon Cooper {
302474cd60a4SJon Cooper 	int i;
302574cd60a4SJon Cooper 
302674cd60a4SJon Cooper 	for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) {
302774cd60a4SJon Cooper 		if (efx_mcdi_poll_reboot(efx))
302874cd60a4SJon Cooper 			goto out;
302974cd60a4SJon Cooper 		msleep(BIST_WAIT_DELAY_MS);
303074cd60a4SJon Cooper 	}
303174cd60a4SJon Cooper 
303274cd60a4SJon Cooper 	netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n");
303374cd60a4SJon Cooper out:
303474cd60a4SJon Cooper 	/* Either way unset the BIST flag. If we found no reboot we probably
303574cd60a4SJon Cooper 	 * won't recover, but we should try.
303674cd60a4SJon Cooper 	 */
303774cd60a4SJon Cooper 	efx->mc_bist_for_other_fn = false;
303874cd60a4SJon Cooper }
303974cd60a4SJon Cooper 
3040874aeea5SJeff Kirsher /* The worker thread exists so that code that cannot sleep can
3041874aeea5SJeff Kirsher  * schedule a reset for later.
3042874aeea5SJeff Kirsher  */
3043874aeea5SJeff Kirsher static void efx_reset_work(struct work_struct *data)
3044874aeea5SJeff Kirsher {
3045874aeea5SJeff Kirsher 	struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
3046626950dbSAlexandre Rames 	unsigned long pending;
3047626950dbSAlexandre Rames 	enum reset_type method;
3048626950dbSAlexandre Rames 
30496aa7de05SMark Rutland 	pending = READ_ONCE(efx->reset_pending);
3050626950dbSAlexandre Rames 	method = fls(pending) - 1;
3051626950dbSAlexandre Rames 
305274cd60a4SJon Cooper 	if (method == RESET_TYPE_MC_BIST)
305374cd60a4SJon Cooper 		efx_wait_for_bist_end(efx);
305474cd60a4SJon Cooper 
3055626950dbSAlexandre Rames 	if ((method == RESET_TYPE_RECOVER_OR_DISABLE ||
3056626950dbSAlexandre Rames 	     method == RESET_TYPE_RECOVER_OR_ALL) &&
3057626950dbSAlexandre Rames 	    efx_try_recovery(efx))
3058626950dbSAlexandre Rames 		return;
3059874aeea5SJeff Kirsher 
3060874aeea5SJeff Kirsher 	if (!pending)
3061874aeea5SJeff Kirsher 		return;
3062874aeea5SJeff Kirsher 
3063874aeea5SJeff Kirsher 	rtnl_lock();
30647153f623SBen Hutchings 
30657153f623SBen Hutchings 	/* We checked the state in efx_schedule_reset() but it may
30667153f623SBen Hutchings 	 * have changed by now.  Now that we have the RTNL lock,
30677153f623SBen Hutchings 	 * it cannot change again.
30687153f623SBen Hutchings 	 */
30697153f623SBen Hutchings 	if (efx->state == STATE_READY)
3070626950dbSAlexandre Rames 		(void)efx_reset(efx, method);
30717153f623SBen Hutchings 
3072874aeea5SJeff Kirsher 	rtnl_unlock();
3073874aeea5SJeff Kirsher }
3074874aeea5SJeff Kirsher 
3075874aeea5SJeff Kirsher void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
3076874aeea5SJeff Kirsher {
3077874aeea5SJeff Kirsher 	enum reset_type method;
3078874aeea5SJeff Kirsher 
3079626950dbSAlexandre Rames 	if (efx->state == STATE_RECOVERY) {
3080626950dbSAlexandre Rames 		netif_dbg(efx, drv, efx->net_dev,
3081626950dbSAlexandre Rames 			  "recovering: skip scheduling %s reset\n",
3082626950dbSAlexandre Rames 			  RESET_TYPE(type));
3083626950dbSAlexandre Rames 		return;
3084626950dbSAlexandre Rames 	}
3085626950dbSAlexandre Rames 
3086874aeea5SJeff Kirsher 	switch (type) {
3087874aeea5SJeff Kirsher 	case RESET_TYPE_INVISIBLE:
3088874aeea5SJeff Kirsher 	case RESET_TYPE_ALL:
3089626950dbSAlexandre Rames 	case RESET_TYPE_RECOVER_OR_ALL:
3090874aeea5SJeff Kirsher 	case RESET_TYPE_WORLD:
3091874aeea5SJeff Kirsher 	case RESET_TYPE_DISABLE:
3092626950dbSAlexandre Rames 	case RESET_TYPE_RECOVER_OR_DISABLE:
3093087e9025SJon Cooper 	case RESET_TYPE_DATAPATH:
309474cd60a4SJon Cooper 	case RESET_TYPE_MC_BIST:
3095e283546cSEdward Cree 	case RESET_TYPE_MCDI_TIMEOUT:
3096874aeea5SJeff Kirsher 		method = type;
3097874aeea5SJeff Kirsher 		netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
3098874aeea5SJeff Kirsher 			  RESET_TYPE(method));
3099874aeea5SJeff Kirsher 		break;
3100874aeea5SJeff Kirsher 	default:
3101874aeea5SJeff Kirsher 		method = efx->type->map_reset_reason(type);
3102874aeea5SJeff Kirsher 		netif_dbg(efx, drv, efx->net_dev,
3103874aeea5SJeff Kirsher 			  "scheduling %s reset for %s\n",
3104874aeea5SJeff Kirsher 			  RESET_TYPE(method), RESET_TYPE(type));
3105874aeea5SJeff Kirsher 		break;
3106874aeea5SJeff Kirsher 	}
3107874aeea5SJeff Kirsher 
3108874aeea5SJeff Kirsher 	set_bit(method, &efx->reset_pending);
31097153f623SBen Hutchings 	smp_mb(); /* ensure we change reset_pending before checking state */
31107153f623SBen Hutchings 
31117153f623SBen Hutchings 	/* If we're not READY then just leave the flags set as the cue
31127153f623SBen Hutchings 	 * to abort probing or reschedule the reset later.
31137153f623SBen Hutchings 	 */
31146aa7de05SMark Rutland 	if (READ_ONCE(efx->state) != STATE_READY)
31157153f623SBen Hutchings 		return;
3116874aeea5SJeff Kirsher 
3117874aeea5SJeff Kirsher 	/* efx_process_channel() will no longer read events once a
3118874aeea5SJeff Kirsher 	 * reset is scheduled. So switch back to poll'd MCDI completions. */
3119874aeea5SJeff Kirsher 	efx_mcdi_mode_poll(efx);
3120874aeea5SJeff Kirsher 
3121874aeea5SJeff Kirsher 	queue_work(reset_workqueue, &efx->reset_work);
3122874aeea5SJeff Kirsher }
3123874aeea5SJeff Kirsher 
3124874aeea5SJeff Kirsher /**************************************************************************
3125874aeea5SJeff Kirsher  *
3126874aeea5SJeff Kirsher  * List of NICs we support
3127874aeea5SJeff Kirsher  *
3128874aeea5SJeff Kirsher  **************************************************************************/
3129874aeea5SJeff Kirsher 
3130874aeea5SJeff Kirsher /* PCI device ID table */
31319baa3c34SBenoit Taine static const struct pci_device_id efx_pci_table[] = {
3132547c474fSBen Hutchings 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803),	/* SFC9020 */
3133874aeea5SJeff Kirsher 	 .driver_data = (unsigned long) &siena_a0_nic_type},
3134547c474fSBen Hutchings 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813),	/* SFL9021 */
3135874aeea5SJeff Kirsher 	 .driver_data = (unsigned long) &siena_a0_nic_type},
31368127d661SBen Hutchings 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903),  /* SFC9120 PF */
31378127d661SBen Hutchings 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
31386f7f8aa6SShradha Shah 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903),  /* SFC9120 VF */
31396f7f8aa6SShradha Shah 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
31403b06a00eSMateusz Wrzesinski 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923),  /* SFC9140 PF */
31413b06a00eSMateusz Wrzesinski 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
3142dd248f1bSBert Kenward 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923),  /* SFC9140 VF */
3143dd248f1bSBert Kenward 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
3144dd248f1bSBert Kenward 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03),  /* SFC9220 PF */
3145dd248f1bSBert Kenward 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
3146dd248f1bSBert Kenward 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03),  /* SFC9220 VF */
3147dd248f1bSBert Kenward 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
3148aae5a316SEdward Cree 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03),  /* SFC9250 PF */
3149aae5a316SEdward Cree 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
3150aae5a316SEdward Cree 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03),  /* SFC9250 VF */
3151aae5a316SEdward Cree 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
3152874aeea5SJeff Kirsher 	{0}			/* end of list */
3153874aeea5SJeff Kirsher };
3154874aeea5SJeff Kirsher 
3155874aeea5SJeff Kirsher /**************************************************************************
3156874aeea5SJeff Kirsher  *
3157874aeea5SJeff Kirsher  * Dummy PHY/MAC operations
3158874aeea5SJeff Kirsher  *
3159874aeea5SJeff Kirsher  * Can be used for some unimplemented operations
3160874aeea5SJeff Kirsher  * Needed so all function pointers are valid and do not have to be tested
3161874aeea5SJeff Kirsher  * before use
3162874aeea5SJeff Kirsher  *
3163874aeea5SJeff Kirsher  **************************************************************************/
3164874aeea5SJeff Kirsher int efx_port_dummy_op_int(struct efx_nic *efx)
3165874aeea5SJeff Kirsher {
3166874aeea5SJeff Kirsher 	return 0;
3167874aeea5SJeff Kirsher }
3168874aeea5SJeff Kirsher void efx_port_dummy_op_void(struct efx_nic *efx) {}
3169874aeea5SJeff Kirsher 
3170874aeea5SJeff Kirsher static bool efx_port_dummy_op_poll(struct efx_nic *efx)
3171874aeea5SJeff Kirsher {
3172874aeea5SJeff Kirsher 	return false;
3173874aeea5SJeff Kirsher }
3174874aeea5SJeff Kirsher 
3175874aeea5SJeff Kirsher static const struct efx_phy_operations efx_dummy_phy_operations = {
3176874aeea5SJeff Kirsher 	.init		 = efx_port_dummy_op_int,
3177874aeea5SJeff Kirsher 	.reconfigure	 = efx_port_dummy_op_int,
3178874aeea5SJeff Kirsher 	.poll		 = efx_port_dummy_op_poll,
3179874aeea5SJeff Kirsher 	.fini		 = efx_port_dummy_op_void,
3180874aeea5SJeff Kirsher };
3181874aeea5SJeff Kirsher 
3182874aeea5SJeff Kirsher /**************************************************************************
3183874aeea5SJeff Kirsher  *
3184874aeea5SJeff Kirsher  * Data housekeeping
3185874aeea5SJeff Kirsher  *
3186874aeea5SJeff Kirsher  **************************************************************************/
3187874aeea5SJeff Kirsher 
3188874aeea5SJeff Kirsher /* This zeroes out and then fills in the invariants in a struct
3189874aeea5SJeff Kirsher  * efx_nic (including all sub-structures).
3190874aeea5SJeff Kirsher  */
3191adeb15aaSBen Hutchings static int efx_init_struct(struct efx_nic *efx,
3192874aeea5SJeff Kirsher 			   struct pci_dev *pci_dev, struct net_device *net_dev)
3193874aeea5SJeff Kirsher {
31946f9f6ec2SAndrew Rybchenko 	int rc = -ENOMEM, i;
3195874aeea5SJeff Kirsher 
3196874aeea5SJeff Kirsher 	/* Initialise common structures */
31970bcf4a64SBen Hutchings 	INIT_LIST_HEAD(&efx->node);
31980bcf4a64SBen Hutchings 	INIT_LIST_HEAD(&efx->secondary_list);
3199874aeea5SJeff Kirsher 	spin_lock_init(&efx->biu_lock);
3200874aeea5SJeff Kirsher #ifdef CONFIG_SFC_MTD
3201874aeea5SJeff Kirsher 	INIT_LIST_HEAD(&efx->mtd_list);
3202874aeea5SJeff Kirsher #endif
3203874aeea5SJeff Kirsher 	INIT_WORK(&efx->reset_work, efx_reset_work);
3204874aeea5SJeff Kirsher 	INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
3205dd40781eSBen Hutchings 	INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work);
3206874aeea5SJeff Kirsher 	efx->pci_dev = pci_dev;
3207874aeea5SJeff Kirsher 	efx->msg_enable = debug;
3208f16aeea0SBen Hutchings 	efx->state = STATE_UNINIT;
3209874aeea5SJeff Kirsher 	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
3210874aeea5SJeff Kirsher 
3211874aeea5SJeff Kirsher 	efx->net_dev = net_dev;
321243a3739dSJon Cooper 	efx->rx_prefix_size = efx->type->rx_prefix_size;
32132ec03014SAndrew Rybchenko 	efx->rx_ip_align =
32142ec03014SAndrew Rybchenko 		NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0;
321543a3739dSJon Cooper 	efx->rx_packet_hash_offset =
321643a3739dSJon Cooper 		efx->type->rx_hash_offset - efx->type->rx_prefix_size;
3217bd9a265dSJon Cooper 	efx->rx_packet_ts_offset =
3218bd9a265dSJon Cooper 		efx->type->rx_ts_offset - efx->type->rx_prefix_size;
321942356d9aSEdward Cree 	INIT_LIST_HEAD(&efx->rss_context.list);
3220e0a65e3cSEdward Cree 	mutex_init(&efx->rss_lock);
3221874aeea5SJeff Kirsher 	spin_lock_init(&efx->stats_lock);
322271827443SEdward Cree 	efx->vi_stride = EFX_DEFAULT_VI_STRIDE;
3223c1be4821SEdward Cree 	efx->num_mac_stats = MC_CMD_MAC_NSTATS;
3224c1be4821SEdward Cree 	BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END);
3225874aeea5SJeff Kirsher 	mutex_init(&efx->mac_lock);
32263af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL
32273af0f342SEdward Cree 	mutex_init(&efx->rps_mutex);
3228f8d62037SEdward Cree 	spin_lock_init(&efx->rps_hash_lock);
3229f8d62037SEdward Cree 	/* Failure to allocate is not fatal, but may degrade ARFS performance */
3230f8d62037SEdward Cree 	efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE,
3231f8d62037SEdward Cree 				      sizeof(*efx->rps_hash_table), GFP_KERNEL);
32323af0f342SEdward Cree #endif
3233874aeea5SJeff Kirsher 	efx->phy_op = &efx_dummy_phy_operations;
3234874aeea5SJeff Kirsher 	efx->mdio.dev = net_dev;
3235874aeea5SJeff Kirsher 	INIT_WORK(&efx->mac_work, efx_mac_work);
32369f2cb71cSBen Hutchings 	init_waitqueue_head(&efx->flush_wq);
3237874aeea5SJeff Kirsher 
3238874aeea5SJeff Kirsher 	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
3239874aeea5SJeff Kirsher 		efx->channel[i] = efx_alloc_channel(efx, i, NULL);
3240874aeea5SJeff Kirsher 		if (!efx->channel[i])
3241874aeea5SJeff Kirsher 			goto fail;
3242d8291187SBen Hutchings 		efx->msi_context[i].efx = efx;
3243d8291187SBen Hutchings 		efx->msi_context[i].index = i;
3244874aeea5SJeff Kirsher 	}
3245874aeea5SJeff Kirsher 
3246874aeea5SJeff Kirsher 	/* Higher numbered interrupt modes are less capable! */
32476f9f6ec2SAndrew Rybchenko 	if (WARN_ON_ONCE(efx->type->max_interrupt_mode >
32486f9f6ec2SAndrew Rybchenko 			 efx->type->min_interrupt_mode)) {
32496f9f6ec2SAndrew Rybchenko 		rc = -EIO;
32506f9f6ec2SAndrew Rybchenko 		goto fail;
32516f9f6ec2SAndrew Rybchenko 	}
3252874aeea5SJeff Kirsher 	efx->interrupt_mode = max(efx->type->max_interrupt_mode,
3253874aeea5SJeff Kirsher 				  interrupt_mode);
32546f9f6ec2SAndrew Rybchenko 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,
32556f9f6ec2SAndrew Rybchenko 				  interrupt_mode);
3256874aeea5SJeff Kirsher 
3257874aeea5SJeff Kirsher 	/* Would be good to use the net_dev name, but we're too early */
3258874aeea5SJeff Kirsher 	snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
3259874aeea5SJeff Kirsher 		 pci_name(pci_dev));
3260874aeea5SJeff Kirsher 	efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
3261874aeea5SJeff Kirsher 	if (!efx->workqueue)
3262874aeea5SJeff Kirsher 		goto fail;
3263874aeea5SJeff Kirsher 
3264874aeea5SJeff Kirsher 	return 0;
3265874aeea5SJeff Kirsher 
3266874aeea5SJeff Kirsher fail:
3267874aeea5SJeff Kirsher 	efx_fini_struct(efx);
32686f9f6ec2SAndrew Rybchenko 	return rc;
3269874aeea5SJeff Kirsher }
3270874aeea5SJeff Kirsher 
3271874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx)
3272874aeea5SJeff Kirsher {
3273874aeea5SJeff Kirsher 	int i;
3274874aeea5SJeff Kirsher 
3275f8d62037SEdward Cree #ifdef CONFIG_RFS_ACCEL
3276f8d62037SEdward Cree 	kfree(efx->rps_hash_table);
3277f8d62037SEdward Cree #endif
3278f8d62037SEdward Cree 
3279874aeea5SJeff Kirsher 	for (i = 0; i < EFX_MAX_CHANNELS; i++)
3280874aeea5SJeff Kirsher 		kfree(efx->channel[i]);
3281874aeea5SJeff Kirsher 
3282ef215e64SBen Hutchings 	kfree(efx->vpd_sn);
3283ef215e64SBen Hutchings 
3284874aeea5SJeff Kirsher 	if (efx->workqueue) {
3285874aeea5SJeff Kirsher 		destroy_workqueue(efx->workqueue);
3286874aeea5SJeff Kirsher 		efx->workqueue = NULL;
3287874aeea5SJeff Kirsher 	}
3288874aeea5SJeff Kirsher }
3289874aeea5SJeff Kirsher 
3290e4d112e4SEdward Cree void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
3291e4d112e4SEdward Cree {
3292e4d112e4SEdward Cree 	u64 n_rx_nodesc_trunc = 0;
3293e4d112e4SEdward Cree 	struct efx_channel *channel;
3294e4d112e4SEdward Cree 
3295e4d112e4SEdward Cree 	efx_for_each_channel(channel, efx)
3296e4d112e4SEdward Cree 		n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
3297e4d112e4SEdward Cree 	stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
3298e4d112e4SEdward Cree 	stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
3299e4d112e4SEdward Cree }
3300e4d112e4SEdward Cree 
3301f8d62037SEdward Cree bool efx_filter_spec_equal(const struct efx_filter_spec *left,
3302f8d62037SEdward Cree 			   const struct efx_filter_spec *right)
3303f8d62037SEdward Cree {
3304f8d62037SEdward Cree 	if ((left->match_flags ^ right->match_flags) |
3305f8d62037SEdward Cree 	    ((left->flags ^ right->flags) &
3306f8d62037SEdward Cree 	     (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
3307f8d62037SEdward Cree 		return false;
3308f8d62037SEdward Cree 
3309f8d62037SEdward Cree 	return memcmp(&left->outer_vid, &right->outer_vid,
3310f8d62037SEdward Cree 		      sizeof(struct efx_filter_spec) -
3311f8d62037SEdward Cree 		      offsetof(struct efx_filter_spec, outer_vid)) == 0;
3312f8d62037SEdward Cree }
3313f8d62037SEdward Cree 
3314f8d62037SEdward Cree u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)
3315f8d62037SEdward Cree {
3316f8d62037SEdward Cree 	BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);
3317f8d62037SEdward Cree 	return jhash2((const u32 *)&spec->outer_vid,
3318f8d62037SEdward Cree 		      (sizeof(struct efx_filter_spec) -
3319f8d62037SEdward Cree 		       offsetof(struct efx_filter_spec, outer_vid)) / 4,
3320f8d62037SEdward Cree 		      0);
3321f8d62037SEdward Cree }
3322f8d62037SEdward Cree 
3323f8d62037SEdward Cree #ifdef CONFIG_RFS_ACCEL
3324f8d62037SEdward Cree bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,
3325f8d62037SEdward Cree 			bool *force)
3326f8d62037SEdward Cree {
3327f8d62037SEdward Cree 	if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {
3328f8d62037SEdward Cree 		/* ARFS is currently updating this entry, leave it */
3329f8d62037SEdward Cree 		return false;
3330f8d62037SEdward Cree 	}
3331f8d62037SEdward Cree 	if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {
3332f8d62037SEdward Cree 		/* ARFS tried and failed to update this, so it's probably out
3333f8d62037SEdward Cree 		 * of date.  Remove the filter and the ARFS rule entry.
3334f8d62037SEdward Cree 		 */
3335f8d62037SEdward Cree 		rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
3336f8d62037SEdward Cree 		*force = true;
3337f8d62037SEdward Cree 		return true;
3338f8d62037SEdward Cree 	} else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */
3339f8d62037SEdward Cree 		/* ARFS has moved on, so old filter is not needed.  Since we did
3340f8d62037SEdward Cree 		 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
3341f8d62037SEdward Cree 		 * not be removed by efx_rps_hash_del() subsequently.
3342f8d62037SEdward Cree 		 */
3343f8d62037SEdward Cree 		*force = true;
3344f8d62037SEdward Cree 		return true;
3345f8d62037SEdward Cree 	}
3346f8d62037SEdward Cree 	/* Remove it iff ARFS wants to. */
3347f8d62037SEdward Cree 	return true;
3348f8d62037SEdward Cree }
3349f8d62037SEdward Cree 
3350829eb053SColin Ian King static
3351f8d62037SEdward Cree struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,
3352f8d62037SEdward Cree 				       const struct efx_filter_spec *spec)
3353f8d62037SEdward Cree {
3354f8d62037SEdward Cree 	u32 hash = efx_filter_spec_hash(spec);
3355f8d62037SEdward Cree 
3356013ff4a6SLance Roy 	lockdep_assert_held(&efx->rps_hash_lock);
3357f8d62037SEdward Cree 	if (!efx->rps_hash_table)
3358f8d62037SEdward Cree 		return NULL;
3359f8d62037SEdward Cree 	return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];
3360f8d62037SEdward Cree }
3361f8d62037SEdward Cree 
3362f8d62037SEdward Cree struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,
3363f8d62037SEdward Cree 					const struct efx_filter_spec *spec)
3364f8d62037SEdward Cree {
3365f8d62037SEdward Cree 	struct efx_arfs_rule *rule;
3366f8d62037SEdward Cree 	struct hlist_head *head;
3367f8d62037SEdward Cree 	struct hlist_node *node;
3368f8d62037SEdward Cree 
3369f8d62037SEdward Cree 	head = efx_rps_hash_bucket(efx, spec);
3370f8d62037SEdward Cree 	if (!head)
3371f8d62037SEdward Cree 		return NULL;
3372f8d62037SEdward Cree 	hlist_for_each(node, head) {
3373f8d62037SEdward Cree 		rule = container_of(node, struct efx_arfs_rule, node);
3374f8d62037SEdward Cree 		if (efx_filter_spec_equal(spec, &rule->spec))
3375f8d62037SEdward Cree 			return rule;
3376f8d62037SEdward Cree 	}
3377f8d62037SEdward Cree 	return NULL;
3378f8d62037SEdward Cree }
3379f8d62037SEdward Cree 
3380f8d62037SEdward Cree struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,
3381f8d62037SEdward Cree 				       const struct efx_filter_spec *spec,
3382f8d62037SEdward Cree 				       bool *new)
3383f8d62037SEdward Cree {
3384f8d62037SEdward Cree 	struct efx_arfs_rule *rule;
3385f8d62037SEdward Cree 	struct hlist_head *head;
3386f8d62037SEdward Cree 	struct hlist_node *node;
3387f8d62037SEdward Cree 
3388f8d62037SEdward Cree 	head = efx_rps_hash_bucket(efx, spec);
3389f8d62037SEdward Cree 	if (!head)
3390f8d62037SEdward Cree 		return NULL;
3391f8d62037SEdward Cree 	hlist_for_each(node, head) {
3392f8d62037SEdward Cree 		rule = container_of(node, struct efx_arfs_rule, node);
3393f8d62037SEdward Cree 		if (efx_filter_spec_equal(spec, &rule->spec)) {
3394f8d62037SEdward Cree 			*new = false;
3395f8d62037SEdward Cree 			return rule;
3396f8d62037SEdward Cree 		}
3397f8d62037SEdward Cree 	}
3398f8d62037SEdward Cree 	rule = kmalloc(sizeof(*rule), GFP_ATOMIC);
3399f8d62037SEdward Cree 	*new = true;
3400f8d62037SEdward Cree 	if (rule) {
3401f8d62037SEdward Cree 		memcpy(&rule->spec, spec, sizeof(rule->spec));
3402f8d62037SEdward Cree 		hlist_add_head(&rule->node, head);
3403f8d62037SEdward Cree 	}
3404f8d62037SEdward Cree 	return rule;
3405f8d62037SEdward Cree }
3406f8d62037SEdward Cree 
3407f8d62037SEdward Cree void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)
3408f8d62037SEdward Cree {
3409f8d62037SEdward Cree 	struct efx_arfs_rule *rule;
3410f8d62037SEdward Cree 	struct hlist_head *head;
3411f8d62037SEdward Cree 	struct hlist_node *node;
3412f8d62037SEdward Cree 
3413f8d62037SEdward Cree 	head = efx_rps_hash_bucket(efx, spec);
3414f8d62037SEdward Cree 	if (WARN_ON(!head))
3415f8d62037SEdward Cree 		return;
3416f8d62037SEdward Cree 	hlist_for_each(node, head) {
3417f8d62037SEdward Cree 		rule = container_of(node, struct efx_arfs_rule, node);
3418f8d62037SEdward Cree 		if (efx_filter_spec_equal(spec, &rule->spec)) {
3419f8d62037SEdward Cree 			/* Someone already reused the entry.  We know that if
3420f8d62037SEdward Cree 			 * this check doesn't fire (i.e. filter_id == REMOVING)
3421f8d62037SEdward Cree 			 * then the REMOVING mark was put there by our caller,
3422f8d62037SEdward Cree 			 * because caller is holding a lock on filter table and
3423f8d62037SEdward Cree 			 * only holders of that lock set REMOVING.
3424f8d62037SEdward Cree 			 */
3425f8d62037SEdward Cree 			if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)
3426f8d62037SEdward Cree 				return;
3427f8d62037SEdward Cree 			hlist_del(node);
3428f8d62037SEdward Cree 			kfree(rule);
3429f8d62037SEdward Cree 			return;
3430f8d62037SEdward Cree 		}
3431f8d62037SEdward Cree 	}
3432f8d62037SEdward Cree 	/* We didn't find it. */
3433f8d62037SEdward Cree 	WARN_ON(1);
3434f8d62037SEdward Cree }
3435f8d62037SEdward Cree #endif
3436f8d62037SEdward Cree 
343742356d9aSEdward Cree /* RSS contexts.  We're using linked lists and crappy O(n) algorithms, because
343842356d9aSEdward Cree  * (a) this is an infrequent control-plane operation and (b) n is small (max 64)
343942356d9aSEdward Cree  */
3440e0a65e3cSEdward Cree struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx)
344142356d9aSEdward Cree {
3442e0a65e3cSEdward Cree 	struct list_head *head = &efx->rss_context.list;
344342356d9aSEdward Cree 	struct efx_rss_context *ctx, *new;
344442356d9aSEdward Cree 	u32 id = 1; /* Don't use zero, that refers to the master RSS context */
344542356d9aSEdward Cree 
3446e0a65e3cSEdward Cree 	WARN_ON(!mutex_is_locked(&efx->rss_lock));
3447e0a65e3cSEdward Cree 
344842356d9aSEdward Cree 	/* Search for first gap in the numbering */
344942356d9aSEdward Cree 	list_for_each_entry(ctx, head, list) {
345042356d9aSEdward Cree 		if (ctx->user_id != id)
345142356d9aSEdward Cree 			break;
345242356d9aSEdward Cree 		id++;
345342356d9aSEdward Cree 		/* Check for wrap.  If this happens, we have nearly 2^32
345442356d9aSEdward Cree 		 * allocated RSS contexts, which seems unlikely.
345542356d9aSEdward Cree 		 */
345642356d9aSEdward Cree 		if (WARN_ON_ONCE(!id))
345742356d9aSEdward Cree 			return NULL;
345842356d9aSEdward Cree 	}
345942356d9aSEdward Cree 
346042356d9aSEdward Cree 	/* Create the new entry */
346142356d9aSEdward Cree 	new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL);
346242356d9aSEdward Cree 	if (!new)
346342356d9aSEdward Cree 		return NULL;
346442356d9aSEdward Cree 	new->context_id = EFX_EF10_RSS_CONTEXT_INVALID;
346542356d9aSEdward Cree 	new->rx_hash_udp_4tuple = false;
346642356d9aSEdward Cree 
346742356d9aSEdward Cree 	/* Insert the new entry into the gap */
346842356d9aSEdward Cree 	new->user_id = id;
346942356d9aSEdward Cree 	list_add_tail(&new->list, &ctx->list);
347042356d9aSEdward Cree 	return new;
347142356d9aSEdward Cree }
347242356d9aSEdward Cree 
3473e0a65e3cSEdward Cree struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id)
347442356d9aSEdward Cree {
3475e0a65e3cSEdward Cree 	struct list_head *head = &efx->rss_context.list;
347642356d9aSEdward Cree 	struct efx_rss_context *ctx;
347742356d9aSEdward Cree 
3478e0a65e3cSEdward Cree 	WARN_ON(!mutex_is_locked(&efx->rss_lock));
3479e0a65e3cSEdward Cree 
348042356d9aSEdward Cree 	list_for_each_entry(ctx, head, list)
348142356d9aSEdward Cree 		if (ctx->user_id == id)
348242356d9aSEdward Cree 			return ctx;
348342356d9aSEdward Cree 	return NULL;
348442356d9aSEdward Cree }
348542356d9aSEdward Cree 
348642356d9aSEdward Cree void efx_free_rss_context_entry(struct efx_rss_context *ctx)
348742356d9aSEdward Cree {
348842356d9aSEdward Cree 	list_del(&ctx->list);
348942356d9aSEdward Cree 	kfree(ctx);
349042356d9aSEdward Cree }
349142356d9aSEdward Cree 
3492874aeea5SJeff Kirsher /**************************************************************************
3493874aeea5SJeff Kirsher  *
3494874aeea5SJeff Kirsher  * PCI interface
3495874aeea5SJeff Kirsher  *
3496874aeea5SJeff Kirsher  **************************************************************************/
3497874aeea5SJeff Kirsher 
3498874aeea5SJeff Kirsher /* Main body of final NIC shutdown code
3499874aeea5SJeff Kirsher  * This is called only at module unload (or hotplug removal).
3500874aeea5SJeff Kirsher  */
3501874aeea5SJeff Kirsher static void efx_pci_remove_main(struct efx_nic *efx)
3502874aeea5SJeff Kirsher {
35037153f623SBen Hutchings 	/* Flush reset_work. It can no longer be scheduled since we
35047153f623SBen Hutchings 	 * are not READY.
35057153f623SBen Hutchings 	 */
35067153f623SBen Hutchings 	BUG_ON(efx->state == STATE_READY);
35077153f623SBen Hutchings 	cancel_work_sync(&efx->reset_work);
35087153f623SBen Hutchings 
3509d8291187SBen Hutchings 	efx_disable_interrupts(efx);
3510a83762d9SBert Kenward 	efx_clear_interrupt_affinity(efx);
3511874aeea5SJeff Kirsher 	efx_nic_fini_interrupt(efx);
3512874aeea5SJeff Kirsher 	efx_fini_port(efx);
3513874aeea5SJeff Kirsher 	efx->type->fini(efx);
3514874aeea5SJeff Kirsher 	efx_fini_napi(efx);
3515874aeea5SJeff Kirsher 	efx_remove_all(efx);
3516874aeea5SJeff Kirsher }
3517874aeea5SJeff Kirsher 
3518874aeea5SJeff Kirsher /* Final NIC shutdown
35192a3fc311SDaniel Pieczko  * This is called only at module unload (or hotplug removal).  A PF can call
35202a3fc311SDaniel Pieczko  * this on its VFs to ensure they are unbound first.
3521874aeea5SJeff Kirsher  */
3522874aeea5SJeff Kirsher static void efx_pci_remove(struct pci_dev *pci_dev)
3523874aeea5SJeff Kirsher {
3524874aeea5SJeff Kirsher 	struct efx_nic *efx;
3525874aeea5SJeff Kirsher 
3526874aeea5SJeff Kirsher 	efx = pci_get_drvdata(pci_dev);
3527874aeea5SJeff Kirsher 	if (!efx)
3528874aeea5SJeff Kirsher 		return;
3529874aeea5SJeff Kirsher 
3530874aeea5SJeff Kirsher 	/* Mark the NIC as fini, then stop the interface */
3531874aeea5SJeff Kirsher 	rtnl_lock();
35320bcf4a64SBen Hutchings 	efx_dissociate(efx);
3533874aeea5SJeff Kirsher 	dev_close(efx->net_dev);
3534d8291187SBen Hutchings 	efx_disable_interrupts(efx);
3535ea6bb99eSEdward Cree 	efx->state = STATE_UNINIT;
3536874aeea5SJeff Kirsher 	rtnl_unlock();
3537874aeea5SJeff Kirsher 
35387fa8d547SShradha Shah 	if (efx->type->sriov_fini)
3539d98a4ffeSShradha Shah 		efx->type->sriov_fini(efx);
35407fa8d547SShradha Shah 
3541874aeea5SJeff Kirsher 	efx_unregister_netdev(efx);
3542874aeea5SJeff Kirsher 
3543874aeea5SJeff Kirsher 	efx_mtd_remove(efx);
3544874aeea5SJeff Kirsher 
3545874aeea5SJeff Kirsher 	efx_pci_remove_main(efx);
3546874aeea5SJeff Kirsher 
3547874aeea5SJeff Kirsher 	efx_fini_io(efx);
3548874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
3549874aeea5SJeff Kirsher 
3550874aeea5SJeff Kirsher 	efx_fini_struct(efx);
3551874aeea5SJeff Kirsher 	free_netdev(efx->net_dev);
3552626950dbSAlexandre Rames 
3553626950dbSAlexandre Rames 	pci_disable_pcie_error_reporting(pci_dev);
3554874aeea5SJeff Kirsher };
3555874aeea5SJeff Kirsher 
3556460eeaa0SBen Hutchings /* NIC VPD information
3557460eeaa0SBen Hutchings  * Called during probe to display the part number of the
3558460eeaa0SBen Hutchings  * installed NIC.  VPD is potentially very large but this should
3559460eeaa0SBen Hutchings  * always appear within the first 512 bytes.
3560460eeaa0SBen Hutchings  */
3561460eeaa0SBen Hutchings #define SFC_VPD_LEN 512
3562ef215e64SBen Hutchings static void efx_probe_vpd_strings(struct efx_nic *efx)
3563460eeaa0SBen Hutchings {
3564460eeaa0SBen Hutchings 	struct pci_dev *dev = efx->pci_dev;
3565460eeaa0SBen Hutchings 	char vpd_data[SFC_VPD_LEN];
3566460eeaa0SBen Hutchings 	ssize_t vpd_size;
3567ef215e64SBen Hutchings 	int ro_start, ro_size, i, j;
3568460eeaa0SBen Hutchings 
3569460eeaa0SBen Hutchings 	/* Get the vpd data from the device */
3570460eeaa0SBen Hutchings 	vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
3571460eeaa0SBen Hutchings 	if (vpd_size <= 0) {
3572460eeaa0SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
3573460eeaa0SBen Hutchings 		return;
3574460eeaa0SBen Hutchings 	}
3575460eeaa0SBen Hutchings 
3576460eeaa0SBen Hutchings 	/* Get the Read only section */
3577ef215e64SBen Hutchings 	ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
3578ef215e64SBen Hutchings 	if (ro_start < 0) {
3579460eeaa0SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
3580460eeaa0SBen Hutchings 		return;
3581460eeaa0SBen Hutchings 	}
3582460eeaa0SBen Hutchings 
3583ef215e64SBen Hutchings 	ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]);
3584ef215e64SBen Hutchings 	j = ro_size;
3585ef215e64SBen Hutchings 	i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
3586460eeaa0SBen Hutchings 	if (i + j > vpd_size)
3587460eeaa0SBen Hutchings 		j = vpd_size - i;
3588460eeaa0SBen Hutchings 
3589460eeaa0SBen Hutchings 	/* Get the Part number */
3590460eeaa0SBen Hutchings 	i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN");
3591460eeaa0SBen Hutchings 	if (i < 0) {
3592460eeaa0SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "Part number not found\n");
3593460eeaa0SBen Hutchings 		return;
3594460eeaa0SBen Hutchings 	}
3595460eeaa0SBen Hutchings 
3596460eeaa0SBen Hutchings 	j = pci_vpd_info_field_size(&vpd_data[i]);
3597460eeaa0SBen Hutchings 	i += PCI_VPD_INFO_FLD_HDR_SIZE;
3598460eeaa0SBen Hutchings 	if (i + j > vpd_size) {
3599460eeaa0SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
3600460eeaa0SBen Hutchings 		return;
3601460eeaa0SBen Hutchings 	}
3602460eeaa0SBen Hutchings 
3603460eeaa0SBen Hutchings 	netif_info(efx, drv, efx->net_dev,
3604460eeaa0SBen Hutchings 		   "Part Number : %.*s\n", j, &vpd_data[i]);
3605ef215e64SBen Hutchings 
3606ef215e64SBen Hutchings 	i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
3607ef215e64SBen Hutchings 	j = ro_size;
3608ef215e64SBen Hutchings 	i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN");
3609ef215e64SBen Hutchings 	if (i < 0) {
3610ef215e64SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "Serial number not found\n");
3611ef215e64SBen Hutchings 		return;
3612ef215e64SBen Hutchings 	}
3613ef215e64SBen Hutchings 
3614ef215e64SBen Hutchings 	j = pci_vpd_info_field_size(&vpd_data[i]);
3615ef215e64SBen Hutchings 	i += PCI_VPD_INFO_FLD_HDR_SIZE;
3616ef215e64SBen Hutchings 	if (i + j > vpd_size) {
3617ef215e64SBen Hutchings 		netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n");
3618ef215e64SBen Hutchings 		return;
3619ef215e64SBen Hutchings 	}
3620ef215e64SBen Hutchings 
3621ef215e64SBen Hutchings 	efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL);
3622ef215e64SBen Hutchings 	if (!efx->vpd_sn)
3623ef215e64SBen Hutchings 		return;
3624ef215e64SBen Hutchings 
3625ef215e64SBen Hutchings 	snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]);
3626460eeaa0SBen Hutchings }
3627460eeaa0SBen Hutchings 
3628460eeaa0SBen Hutchings 
3629874aeea5SJeff Kirsher /* Main body of NIC initialisation
3630874aeea5SJeff Kirsher  * This is called at module load (or hotplug insertion, theoretically).
3631874aeea5SJeff Kirsher  */
3632874aeea5SJeff Kirsher static int efx_pci_probe_main(struct efx_nic *efx)
3633874aeea5SJeff Kirsher {
3634874aeea5SJeff Kirsher 	int rc;
3635874aeea5SJeff Kirsher 
3636874aeea5SJeff Kirsher 	/* Do start-of-day initialisation */
3637874aeea5SJeff Kirsher 	rc = efx_probe_all(efx);
3638874aeea5SJeff Kirsher 	if (rc)
3639874aeea5SJeff Kirsher 		goto fail1;
3640874aeea5SJeff Kirsher 
3641874aeea5SJeff Kirsher 	efx_init_napi(efx);
3642874aeea5SJeff Kirsher 
3643193f2003SBert Kenward 	down_write(&efx->filter_sem);
3644874aeea5SJeff Kirsher 	rc = efx->type->init(efx);
3645193f2003SBert Kenward 	up_write(&efx->filter_sem);
3646874aeea5SJeff Kirsher 	if (rc) {
3647874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
3648874aeea5SJeff Kirsher 			  "failed to initialise NIC\n");
3649874aeea5SJeff Kirsher 		goto fail3;
3650874aeea5SJeff Kirsher 	}
3651874aeea5SJeff Kirsher 
3652874aeea5SJeff Kirsher 	rc = efx_init_port(efx);
3653874aeea5SJeff Kirsher 	if (rc) {
3654874aeea5SJeff Kirsher 		netif_err(efx, probe, efx->net_dev,
3655874aeea5SJeff Kirsher 			  "failed to initialise port\n");
3656874aeea5SJeff Kirsher 		goto fail4;
3657874aeea5SJeff Kirsher 	}
3658874aeea5SJeff Kirsher 
3659874aeea5SJeff Kirsher 	rc = efx_nic_init_interrupt(efx);
3660874aeea5SJeff Kirsher 	if (rc)
3661874aeea5SJeff Kirsher 		goto fail5;
3662a83762d9SBert Kenward 
3663a83762d9SBert Kenward 	efx_set_interrupt_affinity(efx);
3664261e4d96SJon Cooper 	rc = efx_enable_interrupts(efx);
3665261e4d96SJon Cooper 	if (rc)
3666261e4d96SJon Cooper 		goto fail6;
3667874aeea5SJeff Kirsher 
3668874aeea5SJeff Kirsher 	return 0;
3669874aeea5SJeff Kirsher 
3670261e4d96SJon Cooper  fail6:
3671a83762d9SBert Kenward 	efx_clear_interrupt_affinity(efx);
3672261e4d96SJon Cooper 	efx_nic_fini_interrupt(efx);
3673874aeea5SJeff Kirsher  fail5:
3674874aeea5SJeff Kirsher 	efx_fini_port(efx);
3675874aeea5SJeff Kirsher  fail4:
3676874aeea5SJeff Kirsher 	efx->type->fini(efx);
3677874aeea5SJeff Kirsher  fail3:
3678874aeea5SJeff Kirsher 	efx_fini_napi(efx);
3679874aeea5SJeff Kirsher 	efx_remove_all(efx);
3680874aeea5SJeff Kirsher  fail1:
3681874aeea5SJeff Kirsher 	return rc;
3682874aeea5SJeff Kirsher }
3683874aeea5SJeff Kirsher 
36848a531400SJon Cooper static int efx_pci_probe_post_io(struct efx_nic *efx)
36858a531400SJon Cooper {
36868a531400SJon Cooper 	struct net_device *net_dev = efx->net_dev;
36878a531400SJon Cooper 	int rc = efx_pci_probe_main(efx);
36888a531400SJon Cooper 
36898a531400SJon Cooper 	if (rc)
36908a531400SJon Cooper 		return rc;
36918a531400SJon Cooper 
36928a531400SJon Cooper 	if (efx->type->sriov_init) {
36938a531400SJon Cooper 		rc = efx->type->sriov_init(efx);
36948a531400SJon Cooper 		if (rc)
36958a531400SJon Cooper 			netif_err(efx, probe, efx->net_dev,
36968a531400SJon Cooper 				  "SR-IOV can't be enabled rc %d\n", rc);
36978a531400SJon Cooper 	}
36988a531400SJon Cooper 
36998a531400SJon Cooper 	/* Determine netdevice features */
37008a531400SJon Cooper 	net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
37016978729fSEdward Cree 			      NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL);
37028a531400SJon Cooper 	if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
37038a531400SJon Cooper 		net_dev->features |= NETIF_F_TSO6;
37048a531400SJon Cooper 	/* Check whether device supports TSO */
37058a531400SJon Cooper 	if (!efx->type->tso_versions || !efx->type->tso_versions(efx))
37068a531400SJon Cooper 		net_dev->features &= ~NETIF_F_ALL_TSO;
37078a531400SJon Cooper 	/* Mask for features that also apply to VLAN devices */
37088a531400SJon Cooper 	net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |
37098a531400SJon Cooper 				   NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
37108a531400SJon Cooper 				   NETIF_F_RXCSUM);
37118a531400SJon Cooper 
37126978729fSEdward Cree 	net_dev->hw_features |= net_dev->features & ~efx->fixed_features;
37136978729fSEdward Cree 
37146978729fSEdward Cree 	/* Disable receiving frames with bad FCS, by default. */
37156978729fSEdward Cree 	net_dev->features &= ~NETIF_F_RXALL;
37168a531400SJon Cooper 
37178a531400SJon Cooper 	/* Disable VLAN filtering by default.  It may be enforced if
37188a531400SJon Cooper 	 * the feature is fixed (i.e. VLAN filters are required to
37198a531400SJon Cooper 	 * receive VLAN tagged packets due to vPort restrictions).
37208a531400SJon Cooper 	 */
37218a531400SJon Cooper 	net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
37228a531400SJon Cooper 	net_dev->features |= efx->fixed_features;
37238a531400SJon Cooper 
37248a531400SJon Cooper 	rc = efx_register_netdev(efx);
37258a531400SJon Cooper 	if (!rc)
37268a531400SJon Cooper 		return 0;
37278a531400SJon Cooper 
37288a531400SJon Cooper 	efx_pci_remove_main(efx);
37298a531400SJon Cooper 	return rc;
37308a531400SJon Cooper }
37318a531400SJon Cooper 
3732874aeea5SJeff Kirsher /* NIC initialisation
3733874aeea5SJeff Kirsher  *
3734874aeea5SJeff Kirsher  * This is called at module load (or hotplug insertion,
373573ba7b68SBen Hutchings  * theoretically).  It sets up PCI mappings, resets the NIC,
3736874aeea5SJeff Kirsher  * sets up and registers the network devices with the kernel and hooks
3737874aeea5SJeff Kirsher  * the interrupt service routine.  It does not prepare the device for
3738874aeea5SJeff Kirsher  * transmission; this is left to the first time one of the network
3739874aeea5SJeff Kirsher  * interfaces is brought up (i.e. efx_net_open).
3740874aeea5SJeff Kirsher  */
374187d1fc11SBill Pemberton static int efx_pci_probe(struct pci_dev *pci_dev,
3742874aeea5SJeff Kirsher 			 const struct pci_device_id *entry)
3743874aeea5SJeff Kirsher {
3744874aeea5SJeff Kirsher 	struct net_device *net_dev;
3745874aeea5SJeff Kirsher 	struct efx_nic *efx;
3746fadac6aaSBen Hutchings 	int rc;
3747874aeea5SJeff Kirsher 
3748874aeea5SJeff Kirsher 	/* Allocate and initialise a struct net_device and struct efx_nic */
3749874aeea5SJeff Kirsher 	net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
3750874aeea5SJeff Kirsher 				     EFX_MAX_RX_QUEUES);
3751874aeea5SJeff Kirsher 	if (!net_dev)
3752874aeea5SJeff Kirsher 		return -ENOMEM;
3753adeb15aaSBen Hutchings 	efx = netdev_priv(net_dev);
3754adeb15aaSBen Hutchings 	efx->type = (const struct efx_nic_type *) entry->driver_data;
3755ebfcd0fdSAndrew Rybchenko 	efx->fixed_features |= NETIF_F_HIGHDMA;
3756eb7cfd8cSAndrew Rybchenko 
3757874aeea5SJeff Kirsher 	pci_set_drvdata(pci_dev, efx);
3758874aeea5SJeff Kirsher 	SET_NETDEV_DEV(net_dev, &pci_dev->dev);
3759adeb15aaSBen Hutchings 	rc = efx_init_struct(efx, pci_dev, net_dev);
3760874aeea5SJeff Kirsher 	if (rc)
3761874aeea5SJeff Kirsher 		goto fail1;
3762874aeea5SJeff Kirsher 
3763874aeea5SJeff Kirsher 	netif_info(efx, probe, efx->net_dev,
3764874aeea5SJeff Kirsher 		   "Solarflare NIC detected\n");
3765874aeea5SJeff Kirsher 
37666f7f8aa6SShradha Shah 	if (!efx->type->is_vf)
3767ef215e64SBen Hutchings 		efx_probe_vpd_strings(efx);
3768460eeaa0SBen Hutchings 
3769874aeea5SJeff Kirsher 	/* Set up basic I/O (BAR mappings etc) */
3770874aeea5SJeff Kirsher 	rc = efx_init_io(efx);
3771874aeea5SJeff Kirsher 	if (rc)
3772874aeea5SJeff Kirsher 		goto fail2;
3773874aeea5SJeff Kirsher 
37748a531400SJon Cooper 	rc = efx_pci_probe_post_io(efx);
37758a531400SJon Cooper 	if (rc) {
37768a531400SJon Cooper 		/* On failure, retry once immediately.
37778a531400SJon Cooper 		 * If we aborted probe due to a scheduled reset, dismiss it.
37788a531400SJon Cooper 		 */
37798a531400SJon Cooper 		efx->reset_pending = 0;
37808a531400SJon Cooper 		rc = efx_pci_probe_post_io(efx);
37818a531400SJon Cooper 		if (rc) {
37828a531400SJon Cooper 			/* On another failure, retry once more
37838a531400SJon Cooper 			 * after a 50-305ms delay.
37848a531400SJon Cooper 			 */
37858a531400SJon Cooper 			unsigned char r;
37868a531400SJon Cooper 
37878a531400SJon Cooper 			get_random_bytes(&r, 1);
37888a531400SJon Cooper 			msleep((unsigned int)r + 50);
37898a531400SJon Cooper 			efx->reset_pending = 0;
37908a531400SJon Cooper 			rc = efx_pci_probe_post_io(efx);
37918a531400SJon Cooper 		}
37928a531400SJon Cooper 	}
3793fadac6aaSBen Hutchings 	if (rc)
3794874aeea5SJeff Kirsher 		goto fail3;
3795874aeea5SJeff Kirsher 
3796874aeea5SJeff Kirsher 	netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
3797874aeea5SJeff Kirsher 
37987c43161cSBen Hutchings 	/* Try to create MTDs, but allow this to fail */
3799874aeea5SJeff Kirsher 	rtnl_lock();
38007c43161cSBen Hutchings 	rc = efx_mtd_probe(efx);
3801874aeea5SJeff Kirsher 	rtnl_unlock();
380209a04204SBert Kenward 	if (rc && rc != -EPERM)
38037c43161cSBen Hutchings 		netif_warn(efx, probe, efx->net_dev,
38047c43161cSBen Hutchings 			   "failed to create MTDs (%d)\n", rc);
38057c43161cSBen Hutchings 
380605cfee98SMartin Habets 	(void)pci_enable_pcie_error_reporting(pci_dev);
3807626950dbSAlexandre Rames 
3808e5fbd977SJon Cooper 	if (efx->type->udp_tnl_push_ports)
3809e5fbd977SJon Cooper 		efx->type->udp_tnl_push_ports(efx);
3810e5fbd977SJon Cooper 
3811874aeea5SJeff Kirsher 	return 0;
3812874aeea5SJeff Kirsher 
3813874aeea5SJeff Kirsher  fail3:
3814874aeea5SJeff Kirsher 	efx_fini_io(efx);
3815874aeea5SJeff Kirsher  fail2:
3816874aeea5SJeff Kirsher 	efx_fini_struct(efx);
3817874aeea5SJeff Kirsher  fail1:
3818874aeea5SJeff Kirsher 	WARN_ON(rc > 0);
3819874aeea5SJeff Kirsher 	netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
3820874aeea5SJeff Kirsher 	free_netdev(net_dev);
3821874aeea5SJeff Kirsher 	return rc;
3822874aeea5SJeff Kirsher }
3823874aeea5SJeff Kirsher 
3824834e23ddSShradha Shah /* efx_pci_sriov_configure returns the actual number of Virtual Functions
3825834e23ddSShradha Shah  * enabled on success
3826834e23ddSShradha Shah  */
3827834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV
3828834e23ddSShradha Shah static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
3829834e23ddSShradha Shah {
3830834e23ddSShradha Shah 	int rc;
3831834e23ddSShradha Shah 	struct efx_nic *efx = pci_get_drvdata(dev);
3832834e23ddSShradha Shah 
3833834e23ddSShradha Shah 	if (efx->type->sriov_configure) {
3834834e23ddSShradha Shah 		rc = efx->type->sriov_configure(efx, num_vfs);
3835834e23ddSShradha Shah 		if (rc)
3836834e23ddSShradha Shah 			return rc;
3837834e23ddSShradha Shah 		else
3838834e23ddSShradha Shah 			return num_vfs;
3839834e23ddSShradha Shah 	} else
3840834e23ddSShradha Shah 		return -EOPNOTSUPP;
3841834e23ddSShradha Shah }
3842834e23ddSShradha Shah #endif
3843834e23ddSShradha Shah 
3844874aeea5SJeff Kirsher static int efx_pm_freeze(struct device *dev)
3845874aeea5SJeff Kirsher {
38463e03a8baSChuhong Yuan 	struct efx_nic *efx = dev_get_drvdata(dev);
3847874aeea5SJeff Kirsher 
384861da026dSBen Hutchings 	rtnl_lock();
384961da026dSBen Hutchings 
38506032fb56SBen Hutchings 	if (efx->state != STATE_DISABLED) {
3851f16aeea0SBen Hutchings 		efx->state = STATE_UNINIT;
3852874aeea5SJeff Kirsher 
3853c2f3b8e3SDaniel Pieczko 		efx_device_detach_sync(efx);
3854874aeea5SJeff Kirsher 
3855874aeea5SJeff Kirsher 		efx_stop_all(efx);
3856d8291187SBen Hutchings 		efx_disable_interrupts(efx);
38576032fb56SBen Hutchings 	}
3858874aeea5SJeff Kirsher 
385961da026dSBen Hutchings 	rtnl_unlock();
386061da026dSBen Hutchings 
3861874aeea5SJeff Kirsher 	return 0;
3862874aeea5SJeff Kirsher }
3863874aeea5SJeff Kirsher 
3864874aeea5SJeff Kirsher static int efx_pm_thaw(struct device *dev)
3865874aeea5SJeff Kirsher {
3866261e4d96SJon Cooper 	int rc;
38673e03a8baSChuhong Yuan 	struct efx_nic *efx = dev_get_drvdata(dev);
3868874aeea5SJeff Kirsher 
386961da026dSBen Hutchings 	rtnl_lock();
387061da026dSBen Hutchings 
38716032fb56SBen Hutchings 	if (efx->state != STATE_DISABLED) {
3872261e4d96SJon Cooper 		rc = efx_enable_interrupts(efx);
3873261e4d96SJon Cooper 		if (rc)
3874261e4d96SJon Cooper 			goto fail;
3875874aeea5SJeff Kirsher 
3876874aeea5SJeff Kirsher 		mutex_lock(&efx->mac_lock);
3877874aeea5SJeff Kirsher 		efx->phy_op->reconfigure(efx);
3878874aeea5SJeff Kirsher 		mutex_unlock(&efx->mac_lock);
3879874aeea5SJeff Kirsher 
3880874aeea5SJeff Kirsher 		efx_start_all(efx);
3881874aeea5SJeff Kirsher 
38829c568fd8SPeter Dunning 		efx_device_attach_if_not_resetting(efx);
3883874aeea5SJeff Kirsher 
3884f16aeea0SBen Hutchings 		efx->state = STATE_READY;
3885874aeea5SJeff Kirsher 
3886874aeea5SJeff Kirsher 		efx->type->resume_wol(efx);
38876032fb56SBen Hutchings 	}
3888874aeea5SJeff Kirsher 
388961da026dSBen Hutchings 	rtnl_unlock();
389061da026dSBen Hutchings 
3891874aeea5SJeff Kirsher 	/* Reschedule any quenched resets scheduled during efx_pm_freeze() */
3892874aeea5SJeff Kirsher 	queue_work(reset_workqueue, &efx->reset_work);
3893874aeea5SJeff Kirsher 
3894874aeea5SJeff Kirsher 	return 0;
3895261e4d96SJon Cooper 
3896261e4d96SJon Cooper fail:
3897261e4d96SJon Cooper 	rtnl_unlock();
3898261e4d96SJon Cooper 
3899261e4d96SJon Cooper 	return rc;
3900874aeea5SJeff Kirsher }
3901874aeea5SJeff Kirsher 
3902874aeea5SJeff Kirsher static int efx_pm_poweroff(struct device *dev)
3903874aeea5SJeff Kirsher {
3904874aeea5SJeff Kirsher 	struct pci_dev *pci_dev = to_pci_dev(dev);
3905874aeea5SJeff Kirsher 	struct efx_nic *efx = pci_get_drvdata(pci_dev);
3906874aeea5SJeff Kirsher 
3907874aeea5SJeff Kirsher 	efx->type->fini(efx);
3908874aeea5SJeff Kirsher 
3909874aeea5SJeff Kirsher 	efx->reset_pending = 0;
3910874aeea5SJeff Kirsher 
3911874aeea5SJeff Kirsher 	pci_save_state(pci_dev);
3912874aeea5SJeff Kirsher 	return pci_set_power_state(pci_dev, PCI_D3hot);
3913874aeea5SJeff Kirsher }
3914874aeea5SJeff Kirsher 
3915874aeea5SJeff Kirsher /* Used for both resume and restore */
3916874aeea5SJeff Kirsher static int efx_pm_resume(struct device *dev)
3917874aeea5SJeff Kirsher {
3918874aeea5SJeff Kirsher 	struct pci_dev *pci_dev = to_pci_dev(dev);
3919874aeea5SJeff Kirsher 	struct efx_nic *efx = pci_get_drvdata(pci_dev);
3920874aeea5SJeff Kirsher 	int rc;
3921874aeea5SJeff Kirsher 
3922874aeea5SJeff Kirsher 	rc = pci_set_power_state(pci_dev, PCI_D0);
3923874aeea5SJeff Kirsher 	if (rc)
3924874aeea5SJeff Kirsher 		return rc;
3925874aeea5SJeff Kirsher 	pci_restore_state(pci_dev);
3926874aeea5SJeff Kirsher 	rc = pci_enable_device(pci_dev);
3927874aeea5SJeff Kirsher 	if (rc)
3928874aeea5SJeff Kirsher 		return rc;
3929874aeea5SJeff Kirsher 	pci_set_master(efx->pci_dev);
3930874aeea5SJeff Kirsher 	rc = efx->type->reset(efx, RESET_TYPE_ALL);
3931874aeea5SJeff Kirsher 	if (rc)
3932874aeea5SJeff Kirsher 		return rc;
3933193f2003SBert Kenward 	down_write(&efx->filter_sem);
3934874aeea5SJeff Kirsher 	rc = efx->type->init(efx);
3935193f2003SBert Kenward 	up_write(&efx->filter_sem);
3936874aeea5SJeff Kirsher 	if (rc)
3937874aeea5SJeff Kirsher 		return rc;
3938261e4d96SJon Cooper 	rc = efx_pm_thaw(dev);
3939261e4d96SJon Cooper 	return rc;
3940874aeea5SJeff Kirsher }
3941874aeea5SJeff Kirsher 
3942874aeea5SJeff Kirsher static int efx_pm_suspend(struct device *dev)
3943874aeea5SJeff Kirsher {
3944874aeea5SJeff Kirsher 	int rc;
3945874aeea5SJeff Kirsher 
3946874aeea5SJeff Kirsher 	efx_pm_freeze(dev);
3947874aeea5SJeff Kirsher 	rc = efx_pm_poweroff(dev);
3948874aeea5SJeff Kirsher 	if (rc)
3949874aeea5SJeff Kirsher 		efx_pm_resume(dev);
3950874aeea5SJeff Kirsher 	return rc;
3951874aeea5SJeff Kirsher }
3952874aeea5SJeff Kirsher 
395318e83e4cSBen Hutchings static const struct dev_pm_ops efx_pm_ops = {
3954874aeea5SJeff Kirsher 	.suspend	= efx_pm_suspend,
3955874aeea5SJeff Kirsher 	.resume		= efx_pm_resume,
3956874aeea5SJeff Kirsher 	.freeze		= efx_pm_freeze,
3957874aeea5SJeff Kirsher 	.thaw		= efx_pm_thaw,
3958874aeea5SJeff Kirsher 	.poweroff	= efx_pm_poweroff,
3959874aeea5SJeff Kirsher 	.restore	= efx_pm_resume,
3960874aeea5SJeff Kirsher };
3961874aeea5SJeff Kirsher 
3962626950dbSAlexandre Rames /* A PCI error affecting this device was detected.
3963626950dbSAlexandre Rames  * At this point MMIO and DMA may be disabled.
3964626950dbSAlexandre Rames  * Stop the software path and request a slot reset.
3965626950dbSAlexandre Rames  */
3966debd0034Sstephen hemminger static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev,
3967626950dbSAlexandre Rames 					      enum pci_channel_state state)
3968626950dbSAlexandre Rames {
3969626950dbSAlexandre Rames 	pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3970626950dbSAlexandre Rames 	struct efx_nic *efx = pci_get_drvdata(pdev);
3971626950dbSAlexandre Rames 
3972626950dbSAlexandre Rames 	if (state == pci_channel_io_perm_failure)
3973626950dbSAlexandre Rames 		return PCI_ERS_RESULT_DISCONNECT;
3974626950dbSAlexandre Rames 
3975626950dbSAlexandre Rames 	rtnl_lock();
3976626950dbSAlexandre Rames 
3977626950dbSAlexandre Rames 	if (efx->state != STATE_DISABLED) {
3978626950dbSAlexandre Rames 		efx->state = STATE_RECOVERY;
3979626950dbSAlexandre Rames 		efx->reset_pending = 0;
3980626950dbSAlexandre Rames 
3981626950dbSAlexandre Rames 		efx_device_detach_sync(efx);
3982626950dbSAlexandre Rames 
3983626950dbSAlexandre Rames 		efx_stop_all(efx);
3984d8291187SBen Hutchings 		efx_disable_interrupts(efx);
3985626950dbSAlexandre Rames 
3986626950dbSAlexandre Rames 		status = PCI_ERS_RESULT_NEED_RESET;
3987626950dbSAlexandre Rames 	} else {
3988626950dbSAlexandre Rames 		/* If the interface is disabled we don't want to do anything
3989626950dbSAlexandre Rames 		 * with it.
3990626950dbSAlexandre Rames 		 */
3991626950dbSAlexandre Rames 		status = PCI_ERS_RESULT_RECOVERED;
3992626950dbSAlexandre Rames 	}
3993626950dbSAlexandre Rames 
3994626950dbSAlexandre Rames 	rtnl_unlock();
3995626950dbSAlexandre Rames 
3996626950dbSAlexandre Rames 	pci_disable_device(pdev);
3997626950dbSAlexandre Rames 
3998626950dbSAlexandre Rames 	return status;
3999626950dbSAlexandre Rames }
4000626950dbSAlexandre Rames 
4001dbedd44eSJoe Perches /* Fake a successful reset, which will be performed later in efx_io_resume. */
4002debd0034Sstephen hemminger static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev)
4003626950dbSAlexandre Rames {
4004626950dbSAlexandre Rames 	struct efx_nic *efx = pci_get_drvdata(pdev);
4005626950dbSAlexandre Rames 	pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
4006626950dbSAlexandre Rames 
4007626950dbSAlexandre Rames 	if (pci_enable_device(pdev)) {
4008626950dbSAlexandre Rames 		netif_err(efx, hw, efx->net_dev,
4009626950dbSAlexandre Rames 			  "Cannot re-enable PCI device after reset.\n");
4010626950dbSAlexandre Rames 		status =  PCI_ERS_RESULT_DISCONNECT;
4011626950dbSAlexandre Rames 	}
4012626950dbSAlexandre Rames 
4013626950dbSAlexandre Rames 	return status;
4014626950dbSAlexandre Rames }
4015626950dbSAlexandre Rames 
4016626950dbSAlexandre Rames /* Perform the actual reset and resume I/O operations. */
4017626950dbSAlexandre Rames static void efx_io_resume(struct pci_dev *pdev)
4018626950dbSAlexandre Rames {
4019626950dbSAlexandre Rames 	struct efx_nic *efx = pci_get_drvdata(pdev);
4020626950dbSAlexandre Rames 	int rc;
4021626950dbSAlexandre Rames 
4022626950dbSAlexandre Rames 	rtnl_lock();
4023626950dbSAlexandre Rames 
4024626950dbSAlexandre Rames 	if (efx->state == STATE_DISABLED)
4025626950dbSAlexandre Rames 		goto out;
4026626950dbSAlexandre Rames 
4027626950dbSAlexandre Rames 	rc = efx_reset(efx, RESET_TYPE_ALL);
4028626950dbSAlexandre Rames 	if (rc) {
4029626950dbSAlexandre Rames 		netif_err(efx, hw, efx->net_dev,
4030626950dbSAlexandre Rames 			  "efx_reset failed after PCI error (%d)\n", rc);
4031626950dbSAlexandre Rames 	} else {
4032626950dbSAlexandre Rames 		efx->state = STATE_READY;
4033626950dbSAlexandre Rames 		netif_dbg(efx, hw, efx->net_dev,
4034626950dbSAlexandre Rames 			  "Done resetting and resuming IO after PCI error.\n");
4035626950dbSAlexandre Rames 	}
4036626950dbSAlexandre Rames 
4037626950dbSAlexandre Rames out:
4038626950dbSAlexandre Rames 	rtnl_unlock();
4039626950dbSAlexandre Rames }
4040626950dbSAlexandre Rames 
4041626950dbSAlexandre Rames /* For simplicity and reliability, we always require a slot reset and try to
4042626950dbSAlexandre Rames  * reset the hardware when a pci error affecting the device is detected.
4043626950dbSAlexandre Rames  * We leave both the link_reset and mmio_enabled callback unimplemented:
4044626950dbSAlexandre Rames  * with our request for slot reset the mmio_enabled callback will never be
4045626950dbSAlexandre Rames  * called, and the link_reset callback is not used by AER or EEH mechanisms.
4046626950dbSAlexandre Rames  */
4047c300366bSJulia Lawall static const struct pci_error_handlers efx_err_handlers = {
4048626950dbSAlexandre Rames 	.error_detected = efx_io_error_detected,
4049626950dbSAlexandre Rames 	.slot_reset	= efx_io_slot_reset,
4050626950dbSAlexandre Rames 	.resume		= efx_io_resume,
4051626950dbSAlexandre Rames };
4052626950dbSAlexandre Rames 
4053874aeea5SJeff Kirsher static struct pci_driver efx_pci_driver = {
4054874aeea5SJeff Kirsher 	.name		= KBUILD_MODNAME,
4055874aeea5SJeff Kirsher 	.id_table	= efx_pci_table,
4056874aeea5SJeff Kirsher 	.probe		= efx_pci_probe,
4057874aeea5SJeff Kirsher 	.remove		= efx_pci_remove,
4058874aeea5SJeff Kirsher 	.driver.pm	= &efx_pm_ops,
4059626950dbSAlexandre Rames 	.err_handler	= &efx_err_handlers,
4060834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV
4061834e23ddSShradha Shah 	.sriov_configure = efx_pci_sriov_configure,
4062834e23ddSShradha Shah #endif
4063874aeea5SJeff Kirsher };
4064874aeea5SJeff Kirsher 
4065874aeea5SJeff Kirsher /**************************************************************************
4066874aeea5SJeff Kirsher  *
4067874aeea5SJeff Kirsher  * Kernel module interface
4068874aeea5SJeff Kirsher  *
4069874aeea5SJeff Kirsher  *************************************************************************/
4070874aeea5SJeff Kirsher 
4071874aeea5SJeff Kirsher module_param(interrupt_mode, uint, 0444);
4072874aeea5SJeff Kirsher MODULE_PARM_DESC(interrupt_mode,
4073874aeea5SJeff Kirsher 		 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
4074874aeea5SJeff Kirsher 
4075874aeea5SJeff Kirsher static int __init efx_init_module(void)
4076874aeea5SJeff Kirsher {
4077874aeea5SJeff Kirsher 	int rc;
4078874aeea5SJeff Kirsher 
4079874aeea5SJeff Kirsher 	printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
4080874aeea5SJeff Kirsher 
4081874aeea5SJeff Kirsher 	rc = register_netdevice_notifier(&efx_netdev_notifier);
4082874aeea5SJeff Kirsher 	if (rc)
4083874aeea5SJeff Kirsher 		goto err_notifier;
4084874aeea5SJeff Kirsher 
40857fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV
4086cd2d5b52SBen Hutchings 	rc = efx_init_sriov();
4087cd2d5b52SBen Hutchings 	if (rc)
4088cd2d5b52SBen Hutchings 		goto err_sriov;
40897fa8d547SShradha Shah #endif
4090cd2d5b52SBen Hutchings 
4091874aeea5SJeff Kirsher 	reset_workqueue = create_singlethread_workqueue("sfc_reset");
4092874aeea5SJeff Kirsher 	if (!reset_workqueue) {
4093874aeea5SJeff Kirsher 		rc = -ENOMEM;
4094874aeea5SJeff Kirsher 		goto err_reset;
4095874aeea5SJeff Kirsher 	}
4096874aeea5SJeff Kirsher 
4097874aeea5SJeff Kirsher 	rc = pci_register_driver(&efx_pci_driver);
4098874aeea5SJeff Kirsher 	if (rc < 0)
4099874aeea5SJeff Kirsher 		goto err_pci;
4100874aeea5SJeff Kirsher 
4101874aeea5SJeff Kirsher 	return 0;
4102874aeea5SJeff Kirsher 
4103874aeea5SJeff Kirsher  err_pci:
4104874aeea5SJeff Kirsher 	destroy_workqueue(reset_workqueue);
4105874aeea5SJeff Kirsher  err_reset:
41067fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV
4107cd2d5b52SBen Hutchings 	efx_fini_sriov();
4108cd2d5b52SBen Hutchings  err_sriov:
41097fa8d547SShradha Shah #endif
4110874aeea5SJeff Kirsher 	unregister_netdevice_notifier(&efx_netdev_notifier);
4111874aeea5SJeff Kirsher  err_notifier:
4112874aeea5SJeff Kirsher 	return rc;
4113874aeea5SJeff Kirsher }
4114874aeea5SJeff Kirsher 
4115874aeea5SJeff Kirsher static void __exit efx_exit_module(void)
4116874aeea5SJeff Kirsher {
4117874aeea5SJeff Kirsher 	printk(KERN_INFO "Solarflare NET driver unloading\n");
4118874aeea5SJeff Kirsher 
4119874aeea5SJeff Kirsher 	pci_unregister_driver(&efx_pci_driver);
4120874aeea5SJeff Kirsher 	destroy_workqueue(reset_workqueue);
41217fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV
4122cd2d5b52SBen Hutchings 	efx_fini_sriov();
41237fa8d547SShradha Shah #endif
4124874aeea5SJeff Kirsher 	unregister_netdevice_notifier(&efx_netdev_notifier);
4125874aeea5SJeff Kirsher 
4126874aeea5SJeff Kirsher }
4127874aeea5SJeff Kirsher 
4128874aeea5SJeff Kirsher module_init(efx_init_module);
4129874aeea5SJeff Kirsher module_exit(efx_exit_module);
4130874aeea5SJeff Kirsher 
4131874aeea5SJeff Kirsher MODULE_AUTHOR("Solarflare Communications and "
4132874aeea5SJeff Kirsher 	      "Michael Brown <mbrown@fensystems.co.uk>");
41336a350fdbSBen Hutchings MODULE_DESCRIPTION("Solarflare network driver");
4134874aeea5SJeff Kirsher MODULE_LICENSE("GPL");
4135874aeea5SJeff Kirsher MODULE_DEVICE_TABLE(pci, efx_pci_table);
413614077e9eSEdward Cree MODULE_VERSION(EFX_DRIVER_VERSION);
4137