xref: /openbmc/linux/drivers/net/ethernet/sfc/efx.c (revision 407e7517)
1 /****************************************************************************
2  * Driver for Solarflare network controllers and boards
3  * Copyright 2005-2006 Fen Systems Ltd.
4  * Copyright 2005-2013 Solarflare Communications Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation, incorporated herein by reference.
9  */
10 
11 #include <linux/module.h>
12 #include <linux/pci.h>
13 #include <linux/netdevice.h>
14 #include <linux/etherdevice.h>
15 #include <linux/delay.h>
16 #include <linux/notifier.h>
17 #include <linux/ip.h>
18 #include <linux/tcp.h>
19 #include <linux/in.h>
20 #include <linux/ethtool.h>
21 #include <linux/topology.h>
22 #include <linux/gfp.h>
23 #include <linux/aer.h>
24 #include <linux/interrupt.h>
25 #include "net_driver.h"
26 #include <net/gre.h>
27 #include <net/udp_tunnel.h>
28 #include "efx.h"
29 #include "nic.h"
30 #include "io.h"
31 #include "selftest.h"
32 #include "sriov.h"
33 
34 #include "mcdi.h"
35 #include "mcdi_pcol.h"
36 #include "workarounds.h"
37 
38 /**************************************************************************
39  *
40  * Type name strings
41  *
42  **************************************************************************
43  */
44 
45 /* Loopback mode names (see LOOPBACK_MODE()) */
46 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
47 const char *const efx_loopback_mode_names[] = {
48 	[LOOPBACK_NONE]		= "NONE",
49 	[LOOPBACK_DATA]		= "DATAPATH",
50 	[LOOPBACK_GMAC]		= "GMAC",
51 	[LOOPBACK_XGMII]	= "XGMII",
52 	[LOOPBACK_XGXS]		= "XGXS",
53 	[LOOPBACK_XAUI]		= "XAUI",
54 	[LOOPBACK_GMII]		= "GMII",
55 	[LOOPBACK_SGMII]	= "SGMII",
56 	[LOOPBACK_XGBR]		= "XGBR",
57 	[LOOPBACK_XFI]		= "XFI",
58 	[LOOPBACK_XAUI_FAR]	= "XAUI_FAR",
59 	[LOOPBACK_GMII_FAR]	= "GMII_FAR",
60 	[LOOPBACK_SGMII_FAR]	= "SGMII_FAR",
61 	[LOOPBACK_XFI_FAR]	= "XFI_FAR",
62 	[LOOPBACK_GPHY]		= "GPHY",
63 	[LOOPBACK_PHYXS]	= "PHYXS",
64 	[LOOPBACK_PCS]		= "PCS",
65 	[LOOPBACK_PMAPMD]	= "PMA/PMD",
66 	[LOOPBACK_XPORT]	= "XPORT",
67 	[LOOPBACK_XGMII_WS]	= "XGMII_WS",
68 	[LOOPBACK_XAUI_WS]	= "XAUI_WS",
69 	[LOOPBACK_XAUI_WS_FAR]  = "XAUI_WS_FAR",
70 	[LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
71 	[LOOPBACK_GMII_WS]	= "GMII_WS",
72 	[LOOPBACK_XFI_WS]	= "XFI_WS",
73 	[LOOPBACK_XFI_WS_FAR]	= "XFI_WS_FAR",
74 	[LOOPBACK_PHYXS_WS]	= "PHYXS_WS",
75 };
76 
77 const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
78 const char *const efx_reset_type_names[] = {
79 	[RESET_TYPE_INVISIBLE]          = "INVISIBLE",
80 	[RESET_TYPE_ALL]                = "ALL",
81 	[RESET_TYPE_RECOVER_OR_ALL]     = "RECOVER_OR_ALL",
82 	[RESET_TYPE_WORLD]              = "WORLD",
83 	[RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE",
84 	[RESET_TYPE_DATAPATH]           = "DATAPATH",
85 	[RESET_TYPE_MC_BIST]		= "MC_BIST",
86 	[RESET_TYPE_DISABLE]            = "DISABLE",
87 	[RESET_TYPE_TX_WATCHDOG]        = "TX_WATCHDOG",
88 	[RESET_TYPE_INT_ERROR]          = "INT_ERROR",
89 	[RESET_TYPE_DMA_ERROR]          = "DMA_ERROR",
90 	[RESET_TYPE_TX_SKIP]            = "TX_SKIP",
91 	[RESET_TYPE_MC_FAILURE]         = "MC_FAILURE",
92 	[RESET_TYPE_MCDI_TIMEOUT]	= "MCDI_TIMEOUT (FLR)",
93 };
94 
95 /* UDP tunnel type names */
96 static const char *const efx_udp_tunnel_type_names[] = {
97 	[TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan",
98 	[TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve",
99 };
100 
101 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen)
102 {
103 	if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) &&
104 	    efx_udp_tunnel_type_names[type] != NULL)
105 		snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]);
106 	else
107 		snprintf(buf, buflen, "type %d", type);
108 }
109 
110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be
111  * queued onto this work queue. This is not a per-nic work queue, because
112  * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
113  */
114 static struct workqueue_struct *reset_workqueue;
115 
116 /* How often and how many times to poll for a reset while waiting for a
117  * BIST that another function started to complete.
118  */
119 #define BIST_WAIT_DELAY_MS	100
120 #define BIST_WAIT_DELAY_COUNT	100
121 
122 /**************************************************************************
123  *
124  * Configurable values
125  *
126  *************************************************************************/
127 
128 /*
129  * Use separate channels for TX and RX events
130  *
131  * Set this to 1 to use separate channels for TX and RX. It allows us
132  * to control interrupt affinity separately for TX and RX.
133  *
134  * This is only used in MSI-X interrupt mode
135  */
136 bool efx_separate_tx_channels;
137 module_param(efx_separate_tx_channels, bool, 0444);
138 MODULE_PARM_DESC(efx_separate_tx_channels,
139 		 "Use separate channels for TX and RX");
140 
141 /* This is the weight assigned to each of the (per-channel) virtual
142  * NAPI devices.
143  */
144 static int napi_weight = 64;
145 
146 /* This is the time (in jiffies) between invocations of the hardware
147  * monitor.
148  * On Falcon-based NICs, this will:
149  * - Check the on-board hardware monitor;
150  * - Poll the link state and reconfigure the hardware as necessary.
151  * On Siena-based NICs for power systems with EEH support, this will give EEH a
152  * chance to start.
153  */
154 static unsigned int efx_monitor_interval = 1 * HZ;
155 
156 /* Initial interrupt moderation settings.  They can be modified after
157  * module load with ethtool.
158  *
159  * The default for RX should strike a balance between increasing the
160  * round-trip latency and reducing overhead.
161  */
162 static unsigned int rx_irq_mod_usec = 60;
163 
164 /* Initial interrupt moderation settings.  They can be modified after
165  * module load with ethtool.
166  *
167  * This default is chosen to ensure that a 10G link does not go idle
168  * while a TX queue is stopped after it has become full.  A queue is
169  * restarted when it drops below half full.  The time this takes (assuming
170  * worst case 3 descriptors per packet and 1024 descriptors) is
171  *   512 / 3 * 1.2 = 205 usec.
172  */
173 static unsigned int tx_irq_mod_usec = 150;
174 
175 /* This is the first interrupt mode to try out of:
176  * 0 => MSI-X
177  * 1 => MSI
178  * 2 => legacy
179  */
180 static unsigned int interrupt_mode;
181 
182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
183  * i.e. the number of CPUs among which we may distribute simultaneous
184  * interrupt handling.
185  *
186  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
187  * The default (0) means to assign an interrupt to each core.
188  */
189 static unsigned int rss_cpus;
190 module_param(rss_cpus, uint, 0444);
191 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
192 
193 static bool phy_flash_cfg;
194 module_param(phy_flash_cfg, bool, 0644);
195 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
196 
197 static unsigned irq_adapt_low_thresh = 8000;
198 module_param(irq_adapt_low_thresh, uint, 0644);
199 MODULE_PARM_DESC(irq_adapt_low_thresh,
200 		 "Threshold score for reducing IRQ moderation");
201 
202 static unsigned irq_adapt_high_thresh = 16000;
203 module_param(irq_adapt_high_thresh, uint, 0644);
204 MODULE_PARM_DESC(irq_adapt_high_thresh,
205 		 "Threshold score for increasing IRQ moderation");
206 
207 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
208 			 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
209 			 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
210 			 NETIF_MSG_TX_ERR | NETIF_MSG_HW);
211 module_param(debug, uint, 0);
212 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
213 
214 /**************************************************************************
215  *
216  * Utility functions and prototypes
217  *
218  *************************************************************************/
219 
220 static int efx_soft_enable_interrupts(struct efx_nic *efx);
221 static void efx_soft_disable_interrupts(struct efx_nic *efx);
222 static void efx_remove_channel(struct efx_channel *channel);
223 static void efx_remove_channels(struct efx_nic *efx);
224 static const struct efx_channel_type efx_default_channel_type;
225 static void efx_remove_port(struct efx_nic *efx);
226 static void efx_init_napi_channel(struct efx_channel *channel);
227 static void efx_fini_napi(struct efx_nic *efx);
228 static void efx_fini_napi_channel(struct efx_channel *channel);
229 static void efx_fini_struct(struct efx_nic *efx);
230 static void efx_start_all(struct efx_nic *efx);
231 static void efx_stop_all(struct efx_nic *efx);
232 
233 #define EFX_ASSERT_RESET_SERIALISED(efx)		\
234 	do {						\
235 		if ((efx->state == STATE_READY) ||	\
236 		    (efx->state == STATE_RECOVERY) ||	\
237 		    (efx->state == STATE_DISABLED))	\
238 			ASSERT_RTNL();			\
239 	} while (0)
240 
241 static int efx_check_disabled(struct efx_nic *efx)
242 {
243 	if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) {
244 		netif_err(efx, drv, efx->net_dev,
245 			  "device is disabled due to earlier errors\n");
246 		return -EIO;
247 	}
248 	return 0;
249 }
250 
251 /**************************************************************************
252  *
253  * Event queue processing
254  *
255  *************************************************************************/
256 
257 /* Process channel's event queue
258  *
259  * This function is responsible for processing the event queue of a
260  * single channel.  The caller must guarantee that this function will
261  * never be concurrently called more than once on the same channel,
262  * though different channels may be being processed concurrently.
263  */
264 static int efx_process_channel(struct efx_channel *channel, int budget)
265 {
266 	struct efx_tx_queue *tx_queue;
267 	int spent;
268 
269 	if (unlikely(!channel->enabled))
270 		return 0;
271 
272 	efx_for_each_channel_tx_queue(tx_queue, channel) {
273 		tx_queue->pkts_compl = 0;
274 		tx_queue->bytes_compl = 0;
275 	}
276 
277 	spent = efx_nic_process_eventq(channel, budget);
278 	if (spent && efx_channel_has_rx_queue(channel)) {
279 		struct efx_rx_queue *rx_queue =
280 			efx_channel_get_rx_queue(channel);
281 
282 		efx_rx_flush_packet(channel);
283 		efx_fast_push_rx_descriptors(rx_queue, true);
284 	}
285 
286 	/* Update BQL */
287 	efx_for_each_channel_tx_queue(tx_queue, channel) {
288 		if (tx_queue->bytes_compl) {
289 			netdev_tx_completed_queue(tx_queue->core_txq,
290 				tx_queue->pkts_compl, tx_queue->bytes_compl);
291 		}
292 	}
293 
294 	return spent;
295 }
296 
297 /* NAPI poll handler
298  *
299  * NAPI guarantees serialisation of polls of the same device, which
300  * provides the guarantee required by efx_process_channel().
301  */
302 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
303 {
304 	int step = efx->irq_mod_step_us;
305 
306 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
307 		if (channel->irq_moderation_us > step) {
308 			channel->irq_moderation_us -= step;
309 			efx->type->push_irq_moderation(channel);
310 		}
311 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
312 		if (channel->irq_moderation_us <
313 		    efx->irq_rx_moderation_us) {
314 			channel->irq_moderation_us += step;
315 			efx->type->push_irq_moderation(channel);
316 		}
317 	}
318 
319 	channel->irq_count = 0;
320 	channel->irq_mod_score = 0;
321 }
322 
323 static int efx_poll(struct napi_struct *napi, int budget)
324 {
325 	struct efx_channel *channel =
326 		container_of(napi, struct efx_channel, napi_str);
327 	struct efx_nic *efx = channel->efx;
328 	int spent;
329 
330 	netif_vdbg(efx, intr, efx->net_dev,
331 		   "channel %d NAPI poll executing on CPU %d\n",
332 		   channel->channel, raw_smp_processor_id());
333 
334 	spent = efx_process_channel(channel, budget);
335 
336 	if (spent < budget) {
337 		if (efx_channel_has_rx_queue(channel) &&
338 		    efx->irq_rx_adaptive &&
339 		    unlikely(++channel->irq_count == 1000)) {
340 			efx_update_irq_mod(efx, channel);
341 		}
342 
343 		efx_filter_rfs_expire(channel);
344 
345 		/* There is no race here; although napi_disable() will
346 		 * only wait for napi_complete(), this isn't a problem
347 		 * since efx_nic_eventq_read_ack() will have no effect if
348 		 * interrupts have already been disabled.
349 		 */
350 		if (napi_complete_done(napi, spent))
351 			efx_nic_eventq_read_ack(channel);
352 	}
353 
354 	return spent;
355 }
356 
357 /* Create event queue
358  * Event queue memory allocations are done only once.  If the channel
359  * is reset, the memory buffer will be reused; this guards against
360  * errors during channel reset and also simplifies interrupt handling.
361  */
362 static int efx_probe_eventq(struct efx_channel *channel)
363 {
364 	struct efx_nic *efx = channel->efx;
365 	unsigned long entries;
366 
367 	netif_dbg(efx, probe, efx->net_dev,
368 		  "chan %d create event queue\n", channel->channel);
369 
370 	/* Build an event queue with room for one event per tx and rx buffer,
371 	 * plus some extra for link state events and MCDI completions. */
372 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
373 	EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
374 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
375 
376 	return efx_nic_probe_eventq(channel);
377 }
378 
379 /* Prepare channel's event queue */
380 static int efx_init_eventq(struct efx_channel *channel)
381 {
382 	struct efx_nic *efx = channel->efx;
383 	int rc;
384 
385 	EFX_WARN_ON_PARANOID(channel->eventq_init);
386 
387 	netif_dbg(efx, drv, efx->net_dev,
388 		  "chan %d init event queue\n", channel->channel);
389 
390 	rc = efx_nic_init_eventq(channel);
391 	if (rc == 0) {
392 		efx->type->push_irq_moderation(channel);
393 		channel->eventq_read_ptr = 0;
394 		channel->eventq_init = true;
395 	}
396 	return rc;
397 }
398 
399 /* Enable event queue processing and NAPI */
400 void efx_start_eventq(struct efx_channel *channel)
401 {
402 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
403 		  "chan %d start event queue\n", channel->channel);
404 
405 	/* Make sure the NAPI handler sees the enabled flag set */
406 	channel->enabled = true;
407 	smp_wmb();
408 
409 	napi_enable(&channel->napi_str);
410 	efx_nic_eventq_read_ack(channel);
411 }
412 
413 /* Disable event queue processing and NAPI */
414 void efx_stop_eventq(struct efx_channel *channel)
415 {
416 	if (!channel->enabled)
417 		return;
418 
419 	napi_disable(&channel->napi_str);
420 	channel->enabled = false;
421 }
422 
423 static void efx_fini_eventq(struct efx_channel *channel)
424 {
425 	if (!channel->eventq_init)
426 		return;
427 
428 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
429 		  "chan %d fini event queue\n", channel->channel);
430 
431 	efx_nic_fini_eventq(channel);
432 	channel->eventq_init = false;
433 }
434 
435 static void efx_remove_eventq(struct efx_channel *channel)
436 {
437 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
438 		  "chan %d remove event queue\n", channel->channel);
439 
440 	efx_nic_remove_eventq(channel);
441 }
442 
443 /**************************************************************************
444  *
445  * Channel handling
446  *
447  *************************************************************************/
448 
449 /* Allocate and initialise a channel structure. */
450 static struct efx_channel *
451 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
452 {
453 	struct efx_channel *channel;
454 	struct efx_rx_queue *rx_queue;
455 	struct efx_tx_queue *tx_queue;
456 	int j;
457 
458 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
459 	if (!channel)
460 		return NULL;
461 
462 	channel->efx = efx;
463 	channel->channel = i;
464 	channel->type = &efx_default_channel_type;
465 
466 	for (j = 0; j < EFX_TXQ_TYPES; j++) {
467 		tx_queue = &channel->tx_queue[j];
468 		tx_queue->efx = efx;
469 		tx_queue->queue = i * EFX_TXQ_TYPES + j;
470 		tx_queue->channel = channel;
471 	}
472 
473 	rx_queue = &channel->rx_queue;
474 	rx_queue->efx = efx;
475 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
476 
477 	return channel;
478 }
479 
480 /* Allocate and initialise a channel structure, copying parameters
481  * (but not resources) from an old channel structure.
482  */
483 static struct efx_channel *
484 efx_copy_channel(const struct efx_channel *old_channel)
485 {
486 	struct efx_channel *channel;
487 	struct efx_rx_queue *rx_queue;
488 	struct efx_tx_queue *tx_queue;
489 	int j;
490 
491 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
492 	if (!channel)
493 		return NULL;
494 
495 	*channel = *old_channel;
496 
497 	channel->napi_dev = NULL;
498 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
499 	channel->napi_str.napi_id = 0;
500 	channel->napi_str.state = 0;
501 	memset(&channel->eventq, 0, sizeof(channel->eventq));
502 
503 	for (j = 0; j < EFX_TXQ_TYPES; j++) {
504 		tx_queue = &channel->tx_queue[j];
505 		if (tx_queue->channel)
506 			tx_queue->channel = channel;
507 		tx_queue->buffer = NULL;
508 		memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
509 	}
510 
511 	rx_queue = &channel->rx_queue;
512 	rx_queue->buffer = NULL;
513 	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
514 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
515 
516 	return channel;
517 }
518 
519 static int efx_probe_channel(struct efx_channel *channel)
520 {
521 	struct efx_tx_queue *tx_queue;
522 	struct efx_rx_queue *rx_queue;
523 	int rc;
524 
525 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
526 		  "creating channel %d\n", channel->channel);
527 
528 	rc = channel->type->pre_probe(channel);
529 	if (rc)
530 		goto fail;
531 
532 	rc = efx_probe_eventq(channel);
533 	if (rc)
534 		goto fail;
535 
536 	efx_for_each_channel_tx_queue(tx_queue, channel) {
537 		rc = efx_probe_tx_queue(tx_queue);
538 		if (rc)
539 			goto fail;
540 	}
541 
542 	efx_for_each_channel_rx_queue(rx_queue, channel) {
543 		rc = efx_probe_rx_queue(rx_queue);
544 		if (rc)
545 			goto fail;
546 	}
547 
548 	return 0;
549 
550 fail:
551 	efx_remove_channel(channel);
552 	return rc;
553 }
554 
555 static void
556 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
557 {
558 	struct efx_nic *efx = channel->efx;
559 	const char *type;
560 	int number;
561 
562 	number = channel->channel;
563 	if (efx->tx_channel_offset == 0) {
564 		type = "";
565 	} else if (channel->channel < efx->tx_channel_offset) {
566 		type = "-rx";
567 	} else {
568 		type = "-tx";
569 		number -= efx->tx_channel_offset;
570 	}
571 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);
572 }
573 
574 static void efx_set_channel_names(struct efx_nic *efx)
575 {
576 	struct efx_channel *channel;
577 
578 	efx_for_each_channel(channel, efx)
579 		channel->type->get_name(channel,
580 					efx->msi_context[channel->channel].name,
581 					sizeof(efx->msi_context[0].name));
582 }
583 
584 static int efx_probe_channels(struct efx_nic *efx)
585 {
586 	struct efx_channel *channel;
587 	int rc;
588 
589 	/* Restart special buffer allocation */
590 	efx->next_buffer_table = 0;
591 
592 	/* Probe channels in reverse, so that any 'extra' channels
593 	 * use the start of the buffer table. This allows the traffic
594 	 * channels to be resized without moving them or wasting the
595 	 * entries before them.
596 	 */
597 	efx_for_each_channel_rev(channel, efx) {
598 		rc = efx_probe_channel(channel);
599 		if (rc) {
600 			netif_err(efx, probe, efx->net_dev,
601 				  "failed to create channel %d\n",
602 				  channel->channel);
603 			goto fail;
604 		}
605 	}
606 	efx_set_channel_names(efx);
607 
608 	return 0;
609 
610 fail:
611 	efx_remove_channels(efx);
612 	return rc;
613 }
614 
615 /* Channels are shutdown and reinitialised whilst the NIC is running
616  * to propagate configuration changes (mtu, checksum offload), or
617  * to clear hardware error conditions
618  */
619 static void efx_start_datapath(struct efx_nic *efx)
620 {
621 	netdev_features_t old_features = efx->net_dev->features;
622 	bool old_rx_scatter = efx->rx_scatter;
623 	struct efx_tx_queue *tx_queue;
624 	struct efx_rx_queue *rx_queue;
625 	struct efx_channel *channel;
626 	size_t rx_buf_len;
627 
628 	/* Calculate the rx buffer allocation parameters required to
629 	 * support the current MTU, including padding for header
630 	 * alignment and overruns.
631 	 */
632 	efx->rx_dma_len = (efx->rx_prefix_size +
633 			   EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
634 			   efx->type->rx_buffer_padding);
635 	rx_buf_len = (sizeof(struct efx_rx_page_state) +
636 		      efx->rx_ip_align + efx->rx_dma_len);
637 	if (rx_buf_len <= PAGE_SIZE) {
638 		efx->rx_scatter = efx->type->always_rx_scatter;
639 		efx->rx_buffer_order = 0;
640 	} else if (efx->type->can_rx_scatter) {
641 		BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES);
642 		BUILD_BUG_ON(sizeof(struct efx_rx_page_state) +
643 			     2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE,
644 				       EFX_RX_BUF_ALIGNMENT) >
645 			     PAGE_SIZE);
646 		efx->rx_scatter = true;
647 		efx->rx_dma_len = EFX_RX_USR_BUF_SIZE;
648 		efx->rx_buffer_order = 0;
649 	} else {
650 		efx->rx_scatter = false;
651 		efx->rx_buffer_order = get_order(rx_buf_len);
652 	}
653 
654 	efx_rx_config_page_split(efx);
655 	if (efx->rx_buffer_order)
656 		netif_dbg(efx, drv, efx->net_dev,
657 			  "RX buf len=%u; page order=%u batch=%u\n",
658 			  efx->rx_dma_len, efx->rx_buffer_order,
659 			  efx->rx_pages_per_batch);
660 	else
661 		netif_dbg(efx, drv, efx->net_dev,
662 			  "RX buf len=%u step=%u bpp=%u; page batch=%u\n",
663 			  efx->rx_dma_len, efx->rx_page_buf_step,
664 			  efx->rx_bufs_per_page, efx->rx_pages_per_batch);
665 
666 	/* Restore previously fixed features in hw_features and remove
667 	 * features which are fixed now
668 	 */
669 	efx->net_dev->hw_features |= efx->net_dev->features;
670 	efx->net_dev->hw_features &= ~efx->fixed_features;
671 	efx->net_dev->features |= efx->fixed_features;
672 	if (efx->net_dev->features != old_features)
673 		netdev_features_change(efx->net_dev);
674 
675 	/* RX filters may also have scatter-enabled flags */
676 	if (efx->rx_scatter != old_rx_scatter)
677 		efx->type->filter_update_rx_scatter(efx);
678 
679 	/* We must keep at least one descriptor in a TX ring empty.
680 	 * We could avoid this when the queue size does not exactly
681 	 * match the hardware ring size, but it's not that important.
682 	 * Therefore we stop the queue when one more skb might fill
683 	 * the ring completely.  We wake it when half way back to
684 	 * empty.
685 	 */
686 	efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx);
687 	efx->txq_wake_thresh = efx->txq_stop_thresh / 2;
688 
689 	/* Initialise the channels */
690 	efx_for_each_channel(channel, efx) {
691 		efx_for_each_channel_tx_queue(tx_queue, channel) {
692 			efx_init_tx_queue(tx_queue);
693 			atomic_inc(&efx->active_queues);
694 		}
695 
696 		efx_for_each_channel_rx_queue(rx_queue, channel) {
697 			efx_init_rx_queue(rx_queue);
698 			atomic_inc(&efx->active_queues);
699 			efx_stop_eventq(channel);
700 			efx_fast_push_rx_descriptors(rx_queue, false);
701 			efx_start_eventq(channel);
702 		}
703 
704 		WARN_ON(channel->rx_pkt_n_frags);
705 	}
706 
707 	efx_ptp_start_datapath(efx);
708 
709 	if (netif_device_present(efx->net_dev))
710 		netif_tx_wake_all_queues(efx->net_dev);
711 }
712 
713 static void efx_stop_datapath(struct efx_nic *efx)
714 {
715 	struct efx_channel *channel;
716 	struct efx_tx_queue *tx_queue;
717 	struct efx_rx_queue *rx_queue;
718 	int rc;
719 
720 	EFX_ASSERT_RESET_SERIALISED(efx);
721 	BUG_ON(efx->port_enabled);
722 
723 	efx_ptp_stop_datapath(efx);
724 
725 	/* Stop RX refill */
726 	efx_for_each_channel(channel, efx) {
727 		efx_for_each_channel_rx_queue(rx_queue, channel)
728 			rx_queue->refill_enabled = false;
729 	}
730 
731 	efx_for_each_channel(channel, efx) {
732 		/* RX packet processing is pipelined, so wait for the
733 		 * NAPI handler to complete.  At least event queue 0
734 		 * might be kept active by non-data events, so don't
735 		 * use napi_synchronize() but actually disable NAPI
736 		 * temporarily.
737 		 */
738 		if (efx_channel_has_rx_queue(channel)) {
739 			efx_stop_eventq(channel);
740 			efx_start_eventq(channel);
741 		}
742 	}
743 
744 	rc = efx->type->fini_dmaq(efx);
745 	if (rc) {
746 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
747 	} else {
748 		netif_dbg(efx, drv, efx->net_dev,
749 			  "successfully flushed all queues\n");
750 	}
751 
752 	efx_for_each_channel(channel, efx) {
753 		efx_for_each_channel_rx_queue(rx_queue, channel)
754 			efx_fini_rx_queue(rx_queue);
755 		efx_for_each_possible_channel_tx_queue(tx_queue, channel)
756 			efx_fini_tx_queue(tx_queue);
757 	}
758 }
759 
760 static void efx_remove_channel(struct efx_channel *channel)
761 {
762 	struct efx_tx_queue *tx_queue;
763 	struct efx_rx_queue *rx_queue;
764 
765 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
766 		  "destroy chan %d\n", channel->channel);
767 
768 	efx_for_each_channel_rx_queue(rx_queue, channel)
769 		efx_remove_rx_queue(rx_queue);
770 	efx_for_each_possible_channel_tx_queue(tx_queue, channel)
771 		efx_remove_tx_queue(tx_queue);
772 	efx_remove_eventq(channel);
773 	channel->type->post_remove(channel);
774 }
775 
776 static void efx_remove_channels(struct efx_nic *efx)
777 {
778 	struct efx_channel *channel;
779 
780 	efx_for_each_channel(channel, efx)
781 		efx_remove_channel(channel);
782 }
783 
784 int
785 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
786 {
787 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
788 	u32 old_rxq_entries, old_txq_entries;
789 	unsigned i, next_buffer_table = 0;
790 	int rc, rc2;
791 
792 	rc = efx_check_disabled(efx);
793 	if (rc)
794 		return rc;
795 
796 	/* Not all channels should be reallocated. We must avoid
797 	 * reallocating their buffer table entries.
798 	 */
799 	efx_for_each_channel(channel, efx) {
800 		struct efx_rx_queue *rx_queue;
801 		struct efx_tx_queue *tx_queue;
802 
803 		if (channel->type->copy)
804 			continue;
805 		next_buffer_table = max(next_buffer_table,
806 					channel->eventq.index +
807 					channel->eventq.entries);
808 		efx_for_each_channel_rx_queue(rx_queue, channel)
809 			next_buffer_table = max(next_buffer_table,
810 						rx_queue->rxd.index +
811 						rx_queue->rxd.entries);
812 		efx_for_each_channel_tx_queue(tx_queue, channel)
813 			next_buffer_table = max(next_buffer_table,
814 						tx_queue->txd.index +
815 						tx_queue->txd.entries);
816 	}
817 
818 	efx_device_detach_sync(efx);
819 	efx_stop_all(efx);
820 	efx_soft_disable_interrupts(efx);
821 
822 	/* Clone channels (where possible) */
823 	memset(other_channel, 0, sizeof(other_channel));
824 	for (i = 0; i < efx->n_channels; i++) {
825 		channel = efx->channel[i];
826 		if (channel->type->copy)
827 			channel = channel->type->copy(channel);
828 		if (!channel) {
829 			rc = -ENOMEM;
830 			goto out;
831 		}
832 		other_channel[i] = channel;
833 	}
834 
835 	/* Swap entry counts and channel pointers */
836 	old_rxq_entries = efx->rxq_entries;
837 	old_txq_entries = efx->txq_entries;
838 	efx->rxq_entries = rxq_entries;
839 	efx->txq_entries = txq_entries;
840 	for (i = 0; i < efx->n_channels; i++) {
841 		channel = efx->channel[i];
842 		efx->channel[i] = other_channel[i];
843 		other_channel[i] = channel;
844 	}
845 
846 	/* Restart buffer table allocation */
847 	efx->next_buffer_table = next_buffer_table;
848 
849 	for (i = 0; i < efx->n_channels; i++) {
850 		channel = efx->channel[i];
851 		if (!channel->type->copy)
852 			continue;
853 		rc = efx_probe_channel(channel);
854 		if (rc)
855 			goto rollback;
856 		efx_init_napi_channel(efx->channel[i]);
857 	}
858 
859 out:
860 	/* Destroy unused channel structures */
861 	for (i = 0; i < efx->n_channels; i++) {
862 		channel = other_channel[i];
863 		if (channel && channel->type->copy) {
864 			efx_fini_napi_channel(channel);
865 			efx_remove_channel(channel);
866 			kfree(channel);
867 		}
868 	}
869 
870 	rc2 = efx_soft_enable_interrupts(efx);
871 	if (rc2) {
872 		rc = rc ? rc : rc2;
873 		netif_err(efx, drv, efx->net_dev,
874 			  "unable to restart interrupts on channel reallocation\n");
875 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);
876 	} else {
877 		efx_start_all(efx);
878 		efx_device_attach_if_not_resetting(efx);
879 	}
880 	return rc;
881 
882 rollback:
883 	/* Swap back */
884 	efx->rxq_entries = old_rxq_entries;
885 	efx->txq_entries = old_txq_entries;
886 	for (i = 0; i < efx->n_channels; i++) {
887 		channel = efx->channel[i];
888 		efx->channel[i] = other_channel[i];
889 		other_channel[i] = channel;
890 	}
891 	goto out;
892 }
893 
894 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
895 {
896 	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
897 }
898 
899 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
900 {
901 	return channel->channel - channel->efx->tx_channel_offset <
902 		channel->efx->n_tx_channels;
903 }
904 
905 static const struct efx_channel_type efx_default_channel_type = {
906 	.pre_probe		= efx_channel_dummy_op_int,
907 	.post_remove		= efx_channel_dummy_op_void,
908 	.get_name		= efx_get_channel_name,
909 	.copy			= efx_copy_channel,
910 	.want_txqs		= efx_default_channel_want_txqs,
911 	.keep_eventq		= false,
912 	.want_pio		= true,
913 };
914 
915 int efx_channel_dummy_op_int(struct efx_channel *channel)
916 {
917 	return 0;
918 }
919 
920 void efx_channel_dummy_op_void(struct efx_channel *channel)
921 {
922 }
923 
924 /**************************************************************************
925  *
926  * Port handling
927  *
928  **************************************************************************/
929 
930 /* This ensures that the kernel is kept informed (via
931  * netif_carrier_on/off) of the link status, and also maintains the
932  * link status's stop on the port's TX queue.
933  */
934 void efx_link_status_changed(struct efx_nic *efx)
935 {
936 	struct efx_link_state *link_state = &efx->link_state;
937 
938 	/* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
939 	 * that no events are triggered between unregister_netdev() and the
940 	 * driver unloading. A more general condition is that NETDEV_CHANGE
941 	 * can only be generated between NETDEV_UP and NETDEV_DOWN */
942 	if (!netif_running(efx->net_dev))
943 		return;
944 
945 	if (link_state->up != netif_carrier_ok(efx->net_dev)) {
946 		efx->n_link_state_changes++;
947 
948 		if (link_state->up)
949 			netif_carrier_on(efx->net_dev);
950 		else
951 			netif_carrier_off(efx->net_dev);
952 	}
953 
954 	/* Status message for kernel log */
955 	if (link_state->up)
956 		netif_info(efx, link, efx->net_dev,
957 			   "link up at %uMbps %s-duplex (MTU %d)\n",
958 			   link_state->speed, link_state->fd ? "full" : "half",
959 			   efx->net_dev->mtu);
960 	else
961 		netif_info(efx, link, efx->net_dev, "link down\n");
962 }
963 
964 void efx_link_set_advertising(struct efx_nic *efx,
965 			      const unsigned long *advertising)
966 {
967 	memcpy(efx->link_advertising, advertising,
968 	       sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
969 
970 	efx->link_advertising[0] |= ADVERTISED_Autoneg;
971 	if (advertising[0] & ADVERTISED_Pause)
972 		efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
973 	else
974 		efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
975 	if (advertising[0] & ADVERTISED_Asym_Pause)
976 		efx->wanted_fc ^= EFX_FC_TX;
977 }
978 
979 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not
980  * force the Autoneg bit on.
981  */
982 void efx_link_clear_advertising(struct efx_nic *efx)
983 {
984 	bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS);
985 	efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
986 }
987 
988 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
989 {
990 	efx->wanted_fc = wanted_fc;
991 	if (efx->link_advertising[0]) {
992 		if (wanted_fc & EFX_FC_RX)
993 			efx->link_advertising[0] |= (ADVERTISED_Pause |
994 						     ADVERTISED_Asym_Pause);
995 		else
996 			efx->link_advertising[0] &= ~(ADVERTISED_Pause |
997 						      ADVERTISED_Asym_Pause);
998 		if (wanted_fc & EFX_FC_TX)
999 			efx->link_advertising[0] ^= ADVERTISED_Asym_Pause;
1000 	}
1001 }
1002 
1003 static void efx_fini_port(struct efx_nic *efx);
1004 
1005 /* We assume that efx->type->reconfigure_mac will always try to sync RX
1006  * filters and therefore needs to read-lock the filter table against freeing
1007  */
1008 void efx_mac_reconfigure(struct efx_nic *efx)
1009 {
1010 	down_read(&efx->filter_sem);
1011 	efx->type->reconfigure_mac(efx);
1012 	up_read(&efx->filter_sem);
1013 }
1014 
1015 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure
1016  * the MAC appropriately. All other PHY configuration changes are pushed
1017  * through phy_op->set_settings(), and pushed asynchronously to the MAC
1018  * through efx_monitor().
1019  *
1020  * Callers must hold the mac_lock
1021  */
1022 int __efx_reconfigure_port(struct efx_nic *efx)
1023 {
1024 	enum efx_phy_mode phy_mode;
1025 	int rc;
1026 
1027 	WARN_ON(!mutex_is_locked(&efx->mac_lock));
1028 
1029 	/* Disable PHY transmit in mac level loopbacks */
1030 	phy_mode = efx->phy_mode;
1031 	if (LOOPBACK_INTERNAL(efx))
1032 		efx->phy_mode |= PHY_MODE_TX_DISABLED;
1033 	else
1034 		efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
1035 
1036 	rc = efx->type->reconfigure_port(efx);
1037 
1038 	if (rc)
1039 		efx->phy_mode = phy_mode;
1040 
1041 	return rc;
1042 }
1043 
1044 /* Reinitialise the MAC to pick up new PHY settings, even if the port is
1045  * disabled. */
1046 int efx_reconfigure_port(struct efx_nic *efx)
1047 {
1048 	int rc;
1049 
1050 	EFX_ASSERT_RESET_SERIALISED(efx);
1051 
1052 	mutex_lock(&efx->mac_lock);
1053 	rc = __efx_reconfigure_port(efx);
1054 	mutex_unlock(&efx->mac_lock);
1055 
1056 	return rc;
1057 }
1058 
1059 /* Asynchronous work item for changing MAC promiscuity and multicast
1060  * hash.  Avoid a drain/rx_ingress enable by reconfiguring the current
1061  * MAC directly. */
1062 static void efx_mac_work(struct work_struct *data)
1063 {
1064 	struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
1065 
1066 	mutex_lock(&efx->mac_lock);
1067 	if (efx->port_enabled)
1068 		efx_mac_reconfigure(efx);
1069 	mutex_unlock(&efx->mac_lock);
1070 }
1071 
1072 static int efx_probe_port(struct efx_nic *efx)
1073 {
1074 	int rc;
1075 
1076 	netif_dbg(efx, probe, efx->net_dev, "create port\n");
1077 
1078 	if (phy_flash_cfg)
1079 		efx->phy_mode = PHY_MODE_SPECIAL;
1080 
1081 	/* Connect up MAC/PHY operations table */
1082 	rc = efx->type->probe_port(efx);
1083 	if (rc)
1084 		return rc;
1085 
1086 	/* Initialise MAC address to permanent address */
1087 	ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr);
1088 
1089 	return 0;
1090 }
1091 
1092 static int efx_init_port(struct efx_nic *efx)
1093 {
1094 	int rc;
1095 
1096 	netif_dbg(efx, drv, efx->net_dev, "init port\n");
1097 
1098 	mutex_lock(&efx->mac_lock);
1099 
1100 	rc = efx->phy_op->init(efx);
1101 	if (rc)
1102 		goto fail1;
1103 
1104 	efx->port_initialized = true;
1105 
1106 	/* Reconfigure the MAC before creating dma queues (required for
1107 	 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
1108 	efx_mac_reconfigure(efx);
1109 
1110 	/* Ensure the PHY advertises the correct flow control settings */
1111 	rc = efx->phy_op->reconfigure(efx);
1112 	if (rc && rc != -EPERM)
1113 		goto fail2;
1114 
1115 	mutex_unlock(&efx->mac_lock);
1116 	return 0;
1117 
1118 fail2:
1119 	efx->phy_op->fini(efx);
1120 fail1:
1121 	mutex_unlock(&efx->mac_lock);
1122 	return rc;
1123 }
1124 
1125 static void efx_start_port(struct efx_nic *efx)
1126 {
1127 	netif_dbg(efx, ifup, efx->net_dev, "start port\n");
1128 	BUG_ON(efx->port_enabled);
1129 
1130 	mutex_lock(&efx->mac_lock);
1131 	efx->port_enabled = true;
1132 
1133 	/* Ensure MAC ingress/egress is enabled */
1134 	efx_mac_reconfigure(efx);
1135 
1136 	mutex_unlock(&efx->mac_lock);
1137 }
1138 
1139 /* Cancel work for MAC reconfiguration, periodic hardware monitoring
1140  * and the async self-test, wait for them to finish and prevent them
1141  * being scheduled again.  This doesn't cover online resets, which
1142  * should only be cancelled when removing the device.
1143  */
1144 static void efx_stop_port(struct efx_nic *efx)
1145 {
1146 	netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
1147 
1148 	EFX_ASSERT_RESET_SERIALISED(efx);
1149 
1150 	mutex_lock(&efx->mac_lock);
1151 	efx->port_enabled = false;
1152 	mutex_unlock(&efx->mac_lock);
1153 
1154 	/* Serialise against efx_set_multicast_list() */
1155 	netif_addr_lock_bh(efx->net_dev);
1156 	netif_addr_unlock_bh(efx->net_dev);
1157 
1158 	cancel_delayed_work_sync(&efx->monitor_work);
1159 	efx_selftest_async_cancel(efx);
1160 	cancel_work_sync(&efx->mac_work);
1161 }
1162 
1163 static void efx_fini_port(struct efx_nic *efx)
1164 {
1165 	netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
1166 
1167 	if (!efx->port_initialized)
1168 		return;
1169 
1170 	efx->phy_op->fini(efx);
1171 	efx->port_initialized = false;
1172 
1173 	efx->link_state.up = false;
1174 	efx_link_status_changed(efx);
1175 }
1176 
1177 static void efx_remove_port(struct efx_nic *efx)
1178 {
1179 	netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
1180 
1181 	efx->type->remove_port(efx);
1182 }
1183 
1184 /**************************************************************************
1185  *
1186  * NIC handling
1187  *
1188  **************************************************************************/
1189 
1190 static LIST_HEAD(efx_primary_list);
1191 static LIST_HEAD(efx_unassociated_list);
1192 
1193 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)
1194 {
1195 	return left->type == right->type &&
1196 		left->vpd_sn && right->vpd_sn &&
1197 		!strcmp(left->vpd_sn, right->vpd_sn);
1198 }
1199 
1200 static void efx_associate(struct efx_nic *efx)
1201 {
1202 	struct efx_nic *other, *next;
1203 
1204 	if (efx->primary == efx) {
1205 		/* Adding primary function; look for secondaries */
1206 
1207 		netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
1208 		list_add_tail(&efx->node, &efx_primary_list);
1209 
1210 		list_for_each_entry_safe(other, next, &efx_unassociated_list,
1211 					 node) {
1212 			if (efx_same_controller(efx, other)) {
1213 				list_del(&other->node);
1214 				netif_dbg(other, probe, other->net_dev,
1215 					  "moving to secondary list of %s %s\n",
1216 					  pci_name(efx->pci_dev),
1217 					  efx->net_dev->name);
1218 				list_add_tail(&other->node,
1219 					      &efx->secondary_list);
1220 				other->primary = efx;
1221 			}
1222 		}
1223 	} else {
1224 		/* Adding secondary function; look for primary */
1225 
1226 		list_for_each_entry(other, &efx_primary_list, node) {
1227 			if (efx_same_controller(efx, other)) {
1228 				netif_dbg(efx, probe, efx->net_dev,
1229 					  "adding to secondary list of %s %s\n",
1230 					  pci_name(other->pci_dev),
1231 					  other->net_dev->name);
1232 				list_add_tail(&efx->node,
1233 					      &other->secondary_list);
1234 				efx->primary = other;
1235 				return;
1236 			}
1237 		}
1238 
1239 		netif_dbg(efx, probe, efx->net_dev,
1240 			  "adding to unassociated list\n");
1241 		list_add_tail(&efx->node, &efx_unassociated_list);
1242 	}
1243 }
1244 
1245 static void efx_dissociate(struct efx_nic *efx)
1246 {
1247 	struct efx_nic *other, *next;
1248 
1249 	list_del(&efx->node);
1250 	efx->primary = NULL;
1251 
1252 	list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
1253 		list_del(&other->node);
1254 		netif_dbg(other, probe, other->net_dev,
1255 			  "moving to unassociated list\n");
1256 		list_add_tail(&other->node, &efx_unassociated_list);
1257 		other->primary = NULL;
1258 	}
1259 }
1260 
1261 /* This configures the PCI device to enable I/O and DMA. */
1262 static int efx_init_io(struct efx_nic *efx)
1263 {
1264 	struct pci_dev *pci_dev = efx->pci_dev;
1265 	dma_addr_t dma_mask = efx->type->max_dma_mask;
1266 	unsigned int mem_map_size = efx->type->mem_map_size(efx);
1267 	int rc, bar;
1268 
1269 	netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
1270 
1271 	bar = efx->type->mem_bar(efx);
1272 
1273 	rc = pci_enable_device(pci_dev);
1274 	if (rc) {
1275 		netif_err(efx, probe, efx->net_dev,
1276 			  "failed to enable PCI device\n");
1277 		goto fail1;
1278 	}
1279 
1280 	pci_set_master(pci_dev);
1281 
1282 	/* Set the PCI DMA mask.  Try all possibilities from our
1283 	 * genuine mask down to 32 bits, because some architectures
1284 	 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
1285 	 * masks event though they reject 46 bit masks.
1286 	 */
1287 	while (dma_mask > 0x7fffffffUL) {
1288 		rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask);
1289 		if (rc == 0)
1290 			break;
1291 		dma_mask >>= 1;
1292 	}
1293 	if (rc) {
1294 		netif_err(efx, probe, efx->net_dev,
1295 			  "could not find a suitable DMA mask\n");
1296 		goto fail2;
1297 	}
1298 	netif_dbg(efx, probe, efx->net_dev,
1299 		  "using DMA mask %llx\n", (unsigned long long) dma_mask);
1300 
1301 	efx->membase_phys = pci_resource_start(efx->pci_dev, bar);
1302 	rc = pci_request_region(pci_dev, bar, "sfc");
1303 	if (rc) {
1304 		netif_err(efx, probe, efx->net_dev,
1305 			  "request for memory BAR failed\n");
1306 		rc = -EIO;
1307 		goto fail3;
1308 	}
1309 	efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size);
1310 	if (!efx->membase) {
1311 		netif_err(efx, probe, efx->net_dev,
1312 			  "could not map memory BAR at %llx+%x\n",
1313 			  (unsigned long long)efx->membase_phys, mem_map_size);
1314 		rc = -ENOMEM;
1315 		goto fail4;
1316 	}
1317 	netif_dbg(efx, probe, efx->net_dev,
1318 		  "memory BAR at %llx+%x (virtual %p)\n",
1319 		  (unsigned long long)efx->membase_phys, mem_map_size,
1320 		  efx->membase);
1321 
1322 	return 0;
1323 
1324  fail4:
1325 	pci_release_region(efx->pci_dev, bar);
1326  fail3:
1327 	efx->membase_phys = 0;
1328  fail2:
1329 	pci_disable_device(efx->pci_dev);
1330  fail1:
1331 	return rc;
1332 }
1333 
1334 static void efx_fini_io(struct efx_nic *efx)
1335 {
1336 	int bar;
1337 
1338 	netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
1339 
1340 	if (efx->membase) {
1341 		iounmap(efx->membase);
1342 		efx->membase = NULL;
1343 	}
1344 
1345 	if (efx->membase_phys) {
1346 		bar = efx->type->mem_bar(efx);
1347 		pci_release_region(efx->pci_dev, bar);
1348 		efx->membase_phys = 0;
1349 	}
1350 
1351 	/* Don't disable bus-mastering if VFs are assigned */
1352 	if (!pci_vfs_assigned(efx->pci_dev))
1353 		pci_disable_device(efx->pci_dev);
1354 }
1355 
1356 void efx_set_default_rx_indir_table(struct efx_nic *efx)
1357 {
1358 	size_t i;
1359 
1360 	for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)
1361 		efx->rx_indir_table[i] =
1362 			ethtool_rxfh_indir_default(i, efx->rss_spread);
1363 }
1364 
1365 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
1366 {
1367 	cpumask_var_t thread_mask;
1368 	unsigned int count;
1369 	int cpu;
1370 
1371 	if (rss_cpus) {
1372 		count = rss_cpus;
1373 	} else {
1374 		if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
1375 			netif_warn(efx, probe, efx->net_dev,
1376 				   "RSS disabled due to allocation failure\n");
1377 			return 1;
1378 		}
1379 
1380 		count = 0;
1381 		for_each_online_cpu(cpu) {
1382 			if (!cpumask_test_cpu(cpu, thread_mask)) {
1383 				++count;
1384 				cpumask_or(thread_mask, thread_mask,
1385 					   topology_sibling_cpumask(cpu));
1386 			}
1387 		}
1388 
1389 		free_cpumask_var(thread_mask);
1390 	}
1391 
1392 	if (count > EFX_MAX_RX_QUEUES) {
1393 		netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
1394 			       "Reducing number of rx queues from %u to %u.\n",
1395 			       count, EFX_MAX_RX_QUEUES);
1396 		count = EFX_MAX_RX_QUEUES;
1397 	}
1398 
1399 	/* If RSS is requested for the PF *and* VFs then we can't write RSS
1400 	 * table entries that are inaccessible to VFs
1401 	 */
1402 #ifdef CONFIG_SFC_SRIOV
1403 	if (efx->type->sriov_wanted) {
1404 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
1405 		    count > efx_vf_size(efx)) {
1406 			netif_warn(efx, probe, efx->net_dev,
1407 				   "Reducing number of RSS channels from %u to %u for "
1408 				   "VF support. Increase vf-msix-limit to use more "
1409 				   "channels on the PF.\n",
1410 				   count, efx_vf_size(efx));
1411 			count = efx_vf_size(efx);
1412 		}
1413 	}
1414 #endif
1415 
1416 	return count;
1417 }
1418 
1419 /* Probe the number and type of interrupts we are able to obtain, and
1420  * the resulting numbers of channels and RX queues.
1421  */
1422 static int efx_probe_interrupts(struct efx_nic *efx)
1423 {
1424 	unsigned int extra_channels = 0;
1425 	unsigned int i, j;
1426 	int rc;
1427 
1428 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
1429 		if (efx->extra_channel_type[i])
1430 			++extra_channels;
1431 
1432 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
1433 		struct msix_entry xentries[EFX_MAX_CHANNELS];
1434 		unsigned int n_channels;
1435 
1436 		n_channels = efx_wanted_parallelism(efx);
1437 		if (efx_separate_tx_channels)
1438 			n_channels *= 2;
1439 		n_channels += extra_channels;
1440 		n_channels = min(n_channels, efx->max_channels);
1441 
1442 		for (i = 0; i < n_channels; i++)
1443 			xentries[i].entry = i;
1444 		rc = pci_enable_msix_range(efx->pci_dev,
1445 					   xentries, 1, n_channels);
1446 		if (rc < 0) {
1447 			/* Fall back to single channel MSI */
1448 			netif_err(efx, drv, efx->net_dev,
1449 				  "could not enable MSI-X\n");
1450 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
1451 				efx->interrupt_mode = EFX_INT_MODE_MSI;
1452 			else
1453 				return rc;
1454 		} else if (rc < n_channels) {
1455 			netif_err(efx, drv, efx->net_dev,
1456 				  "WARNING: Insufficient MSI-X vectors"
1457 				  " available (%d < %u).\n", rc, n_channels);
1458 			netif_err(efx, drv, efx->net_dev,
1459 				  "WARNING: Performance may be reduced.\n");
1460 			n_channels = rc;
1461 		}
1462 
1463 		if (rc > 0) {
1464 			efx->n_channels = n_channels;
1465 			if (n_channels > extra_channels)
1466 				n_channels -= extra_channels;
1467 			if (efx_separate_tx_channels) {
1468 				efx->n_tx_channels = min(max(n_channels / 2,
1469 							     1U),
1470 							 efx->max_tx_channels);
1471 				efx->n_rx_channels = max(n_channels -
1472 							 efx->n_tx_channels,
1473 							 1U);
1474 			} else {
1475 				efx->n_tx_channels = min(n_channels,
1476 							 efx->max_tx_channels);
1477 				efx->n_rx_channels = n_channels;
1478 			}
1479 			for (i = 0; i < efx->n_channels; i++)
1480 				efx_get_channel(efx, i)->irq =
1481 					xentries[i].vector;
1482 		}
1483 	}
1484 
1485 	/* Try single interrupt MSI */
1486 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
1487 		efx->n_channels = 1;
1488 		efx->n_rx_channels = 1;
1489 		efx->n_tx_channels = 1;
1490 		rc = pci_enable_msi(efx->pci_dev);
1491 		if (rc == 0) {
1492 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1493 		} else {
1494 			netif_err(efx, drv, efx->net_dev,
1495 				  "could not enable MSI\n");
1496 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
1497 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;
1498 			else
1499 				return rc;
1500 		}
1501 	}
1502 
1503 	/* Assume legacy interrupts */
1504 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1505 		efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
1506 		efx->n_rx_channels = 1;
1507 		efx->n_tx_channels = 1;
1508 		efx->legacy_irq = efx->pci_dev->irq;
1509 	}
1510 
1511 	/* Assign extra channels if possible */
1512 	efx->n_extra_tx_channels = 0;
1513 	j = efx->n_channels;
1514 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
1515 		if (!efx->extra_channel_type[i])
1516 			continue;
1517 		if (efx->interrupt_mode != EFX_INT_MODE_MSIX ||
1518 		    efx->n_channels <= extra_channels) {
1519 			efx->extra_channel_type[i]->handle_no_channel(efx);
1520 		} else {
1521 			--j;
1522 			efx_get_channel(efx, j)->type =
1523 				efx->extra_channel_type[i];
1524 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
1525 				efx->n_extra_tx_channels++;
1526 		}
1527 	}
1528 
1529 	/* RSS might be usable on VFs even if it is disabled on the PF */
1530 #ifdef CONFIG_SFC_SRIOV
1531 	if (efx->type->sriov_wanted) {
1532 		efx->rss_spread = ((efx->n_rx_channels > 1 ||
1533 				    !efx->type->sriov_wanted(efx)) ?
1534 				   efx->n_rx_channels : efx_vf_size(efx));
1535 		return 0;
1536 	}
1537 #endif
1538 	efx->rss_spread = efx->n_rx_channels;
1539 
1540 	return 0;
1541 }
1542 
1543 static int efx_soft_enable_interrupts(struct efx_nic *efx)
1544 {
1545 	struct efx_channel *channel, *end_channel;
1546 	int rc;
1547 
1548 	BUG_ON(efx->state == STATE_DISABLED);
1549 
1550 	efx->irq_soft_enabled = true;
1551 	smp_wmb();
1552 
1553 	efx_for_each_channel(channel, efx) {
1554 		if (!channel->type->keep_eventq) {
1555 			rc = efx_init_eventq(channel);
1556 			if (rc)
1557 				goto fail;
1558 		}
1559 		efx_start_eventq(channel);
1560 	}
1561 
1562 	efx_mcdi_mode_event(efx);
1563 
1564 	return 0;
1565 fail:
1566 	end_channel = channel;
1567 	efx_for_each_channel(channel, efx) {
1568 		if (channel == end_channel)
1569 			break;
1570 		efx_stop_eventq(channel);
1571 		if (!channel->type->keep_eventq)
1572 			efx_fini_eventq(channel);
1573 	}
1574 
1575 	return rc;
1576 }
1577 
1578 static void efx_soft_disable_interrupts(struct efx_nic *efx)
1579 {
1580 	struct efx_channel *channel;
1581 
1582 	if (efx->state == STATE_DISABLED)
1583 		return;
1584 
1585 	efx_mcdi_mode_poll(efx);
1586 
1587 	efx->irq_soft_enabled = false;
1588 	smp_wmb();
1589 
1590 	if (efx->legacy_irq)
1591 		synchronize_irq(efx->legacy_irq);
1592 
1593 	efx_for_each_channel(channel, efx) {
1594 		if (channel->irq)
1595 			synchronize_irq(channel->irq);
1596 
1597 		efx_stop_eventq(channel);
1598 		if (!channel->type->keep_eventq)
1599 			efx_fini_eventq(channel);
1600 	}
1601 
1602 	/* Flush the asynchronous MCDI request queue */
1603 	efx_mcdi_flush_async(efx);
1604 }
1605 
1606 static int efx_enable_interrupts(struct efx_nic *efx)
1607 {
1608 	struct efx_channel *channel, *end_channel;
1609 	int rc;
1610 
1611 	BUG_ON(efx->state == STATE_DISABLED);
1612 
1613 	if (efx->eeh_disabled_legacy_irq) {
1614 		enable_irq(efx->legacy_irq);
1615 		efx->eeh_disabled_legacy_irq = false;
1616 	}
1617 
1618 	efx->type->irq_enable_master(efx);
1619 
1620 	efx_for_each_channel(channel, efx) {
1621 		if (channel->type->keep_eventq) {
1622 			rc = efx_init_eventq(channel);
1623 			if (rc)
1624 				goto fail;
1625 		}
1626 	}
1627 
1628 	rc = efx_soft_enable_interrupts(efx);
1629 	if (rc)
1630 		goto fail;
1631 
1632 	return 0;
1633 
1634 fail:
1635 	end_channel = channel;
1636 	efx_for_each_channel(channel, efx) {
1637 		if (channel == end_channel)
1638 			break;
1639 		if (channel->type->keep_eventq)
1640 			efx_fini_eventq(channel);
1641 	}
1642 
1643 	efx->type->irq_disable_non_ev(efx);
1644 
1645 	return rc;
1646 }
1647 
1648 static void efx_disable_interrupts(struct efx_nic *efx)
1649 {
1650 	struct efx_channel *channel;
1651 
1652 	efx_soft_disable_interrupts(efx);
1653 
1654 	efx_for_each_channel(channel, efx) {
1655 		if (channel->type->keep_eventq)
1656 			efx_fini_eventq(channel);
1657 	}
1658 
1659 	efx->type->irq_disable_non_ev(efx);
1660 }
1661 
1662 static void efx_remove_interrupts(struct efx_nic *efx)
1663 {
1664 	struct efx_channel *channel;
1665 
1666 	/* Remove MSI/MSI-X interrupts */
1667 	efx_for_each_channel(channel, efx)
1668 		channel->irq = 0;
1669 	pci_disable_msi(efx->pci_dev);
1670 	pci_disable_msix(efx->pci_dev);
1671 
1672 	/* Remove legacy interrupt */
1673 	efx->legacy_irq = 0;
1674 }
1675 
1676 static void efx_set_channels(struct efx_nic *efx)
1677 {
1678 	struct efx_channel *channel;
1679 	struct efx_tx_queue *tx_queue;
1680 
1681 	efx->tx_channel_offset =
1682 		efx_separate_tx_channels ?
1683 		efx->n_channels - efx->n_tx_channels : 0;
1684 
1685 	/* We need to mark which channels really have RX and TX
1686 	 * queues, and adjust the TX queue numbers if we have separate
1687 	 * RX-only and TX-only channels.
1688 	 */
1689 	efx_for_each_channel(channel, efx) {
1690 		if (channel->channel < efx->n_rx_channels)
1691 			channel->rx_queue.core_index = channel->channel;
1692 		else
1693 			channel->rx_queue.core_index = -1;
1694 
1695 		efx_for_each_channel_tx_queue(tx_queue, channel)
1696 			tx_queue->queue -= (efx->tx_channel_offset *
1697 					    EFX_TXQ_TYPES);
1698 	}
1699 }
1700 
1701 static int efx_probe_nic(struct efx_nic *efx)
1702 {
1703 	int rc;
1704 
1705 	netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1706 
1707 	/* Carry out hardware-type specific initialisation */
1708 	rc = efx->type->probe(efx);
1709 	if (rc)
1710 		return rc;
1711 
1712 	do {
1713 		if (!efx->max_channels || !efx->max_tx_channels) {
1714 			netif_err(efx, drv, efx->net_dev,
1715 				  "Insufficient resources to allocate"
1716 				  " any channels\n");
1717 			rc = -ENOSPC;
1718 			goto fail1;
1719 		}
1720 
1721 		/* Determine the number of channels and queues by trying
1722 		 * to hook in MSI-X interrupts.
1723 		 */
1724 		rc = efx_probe_interrupts(efx);
1725 		if (rc)
1726 			goto fail1;
1727 
1728 		efx_set_channels(efx);
1729 
1730 		/* dimension_resources can fail with EAGAIN */
1731 		rc = efx->type->dimension_resources(efx);
1732 		if (rc != 0 && rc != -EAGAIN)
1733 			goto fail2;
1734 
1735 		if (rc == -EAGAIN)
1736 			/* try again with new max_channels */
1737 			efx_remove_interrupts(efx);
1738 
1739 	} while (rc == -EAGAIN);
1740 
1741 	if (efx->n_channels > 1)
1742 		netdev_rss_key_fill(&efx->rx_hash_key,
1743 				    sizeof(efx->rx_hash_key));
1744 	efx_set_default_rx_indir_table(efx);
1745 
1746 	netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1747 	netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1748 
1749 	/* Initialise the interrupt moderation settings */
1750 	efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
1751 	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
1752 				true);
1753 
1754 	return 0;
1755 
1756 fail2:
1757 	efx_remove_interrupts(efx);
1758 fail1:
1759 	efx->type->remove(efx);
1760 	return rc;
1761 }
1762 
1763 static void efx_remove_nic(struct efx_nic *efx)
1764 {
1765 	netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1766 
1767 	efx_remove_interrupts(efx);
1768 	efx->type->remove(efx);
1769 }
1770 
1771 static int efx_probe_filters(struct efx_nic *efx)
1772 {
1773 	int rc;
1774 
1775 	spin_lock_init(&efx->filter_lock);
1776 	init_rwsem(&efx->filter_sem);
1777 	mutex_lock(&efx->mac_lock);
1778 	down_write(&efx->filter_sem);
1779 	rc = efx->type->filter_table_probe(efx);
1780 	if (rc)
1781 		goto out_unlock;
1782 
1783 #ifdef CONFIG_RFS_ACCEL
1784 	if (efx->type->offload_features & NETIF_F_NTUPLE) {
1785 		struct efx_channel *channel;
1786 		int i, success = 1;
1787 
1788 		efx_for_each_channel(channel, efx) {
1789 			channel->rps_flow_id =
1790 				kcalloc(efx->type->max_rx_ip_filters,
1791 					sizeof(*channel->rps_flow_id),
1792 					GFP_KERNEL);
1793 			if (!channel->rps_flow_id)
1794 				success = 0;
1795 			else
1796 				for (i = 0;
1797 				     i < efx->type->max_rx_ip_filters;
1798 				     ++i)
1799 					channel->rps_flow_id[i] =
1800 						RPS_FLOW_ID_INVALID;
1801 		}
1802 
1803 		if (!success) {
1804 			efx_for_each_channel(channel, efx)
1805 				kfree(channel->rps_flow_id);
1806 			efx->type->filter_table_remove(efx);
1807 			rc = -ENOMEM;
1808 			goto out_unlock;
1809 		}
1810 
1811 		efx->rps_expire_index = efx->rps_expire_channel = 0;
1812 	}
1813 #endif
1814 out_unlock:
1815 	up_write(&efx->filter_sem);
1816 	mutex_unlock(&efx->mac_lock);
1817 	return rc;
1818 }
1819 
1820 static void efx_remove_filters(struct efx_nic *efx)
1821 {
1822 #ifdef CONFIG_RFS_ACCEL
1823 	struct efx_channel *channel;
1824 
1825 	efx_for_each_channel(channel, efx)
1826 		kfree(channel->rps_flow_id);
1827 #endif
1828 	down_write(&efx->filter_sem);
1829 	efx->type->filter_table_remove(efx);
1830 	up_write(&efx->filter_sem);
1831 }
1832 
1833 static void efx_restore_filters(struct efx_nic *efx)
1834 {
1835 	down_read(&efx->filter_sem);
1836 	efx->type->filter_table_restore(efx);
1837 	up_read(&efx->filter_sem);
1838 }
1839 
1840 /**************************************************************************
1841  *
1842  * NIC startup/shutdown
1843  *
1844  *************************************************************************/
1845 
1846 static int efx_probe_all(struct efx_nic *efx)
1847 {
1848 	int rc;
1849 
1850 	rc = efx_probe_nic(efx);
1851 	if (rc) {
1852 		netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
1853 		goto fail1;
1854 	}
1855 
1856 	rc = efx_probe_port(efx);
1857 	if (rc) {
1858 		netif_err(efx, probe, efx->net_dev, "failed to create port\n");
1859 		goto fail2;
1860 	}
1861 
1862 	BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
1863 	if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
1864 		rc = -EINVAL;
1865 		goto fail3;
1866 	}
1867 	efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
1868 
1869 #ifdef CONFIG_SFC_SRIOV
1870 	rc = efx->type->vswitching_probe(efx);
1871 	if (rc) /* not fatal; the PF will still work fine */
1872 		netif_warn(efx, probe, efx->net_dev,
1873 			   "failed to setup vswitching rc=%d;"
1874 			   " VFs may not function\n", rc);
1875 #endif
1876 
1877 	rc = efx_probe_filters(efx);
1878 	if (rc) {
1879 		netif_err(efx, probe, efx->net_dev,
1880 			  "failed to create filter tables\n");
1881 		goto fail4;
1882 	}
1883 
1884 	rc = efx_probe_channels(efx);
1885 	if (rc)
1886 		goto fail5;
1887 
1888 	return 0;
1889 
1890  fail5:
1891 	efx_remove_filters(efx);
1892  fail4:
1893 #ifdef CONFIG_SFC_SRIOV
1894 	efx->type->vswitching_remove(efx);
1895 #endif
1896  fail3:
1897 	efx_remove_port(efx);
1898  fail2:
1899 	efx_remove_nic(efx);
1900  fail1:
1901 	return rc;
1902 }
1903 
1904 /* If the interface is supposed to be running but is not, start
1905  * the hardware and software data path, regular activity for the port
1906  * (MAC statistics, link polling, etc.) and schedule the port to be
1907  * reconfigured.  Interrupts must already be enabled.  This function
1908  * is safe to call multiple times, so long as the NIC is not disabled.
1909  * Requires the RTNL lock.
1910  */
1911 static void efx_start_all(struct efx_nic *efx)
1912 {
1913 	EFX_ASSERT_RESET_SERIALISED(efx);
1914 	BUG_ON(efx->state == STATE_DISABLED);
1915 
1916 	/* Check that it is appropriate to restart the interface. All
1917 	 * of these flags are safe to read under just the rtnl lock */
1918 	if (efx->port_enabled || !netif_running(efx->net_dev) ||
1919 	    efx->reset_pending)
1920 		return;
1921 
1922 	efx_start_port(efx);
1923 	efx_start_datapath(efx);
1924 
1925 	/* Start the hardware monitor if there is one */
1926 	if (efx->type->monitor != NULL)
1927 		queue_delayed_work(efx->workqueue, &efx->monitor_work,
1928 				   efx_monitor_interval);
1929 
1930 	/* Link state detection is normally event-driven; we have
1931 	 * to poll now because we could have missed a change
1932 	 */
1933 	mutex_lock(&efx->mac_lock);
1934 	if (efx->phy_op->poll(efx))
1935 		efx_link_status_changed(efx);
1936 	mutex_unlock(&efx->mac_lock);
1937 
1938 	efx->type->start_stats(efx);
1939 	efx->type->pull_stats(efx);
1940 	spin_lock_bh(&efx->stats_lock);
1941 	efx->type->update_stats(efx, NULL, NULL);
1942 	spin_unlock_bh(&efx->stats_lock);
1943 }
1944 
1945 /* Quiesce the hardware and software data path, and regular activity
1946  * for the port without bringing the link down.  Safe to call multiple
1947  * times with the NIC in almost any state, but interrupts should be
1948  * enabled.  Requires the RTNL lock.
1949  */
1950 static void efx_stop_all(struct efx_nic *efx)
1951 {
1952 	EFX_ASSERT_RESET_SERIALISED(efx);
1953 
1954 	/* port_enabled can be read safely under the rtnl lock */
1955 	if (!efx->port_enabled)
1956 		return;
1957 
1958 	/* update stats before we go down so we can accurately count
1959 	 * rx_nodesc_drops
1960 	 */
1961 	efx->type->pull_stats(efx);
1962 	spin_lock_bh(&efx->stats_lock);
1963 	efx->type->update_stats(efx, NULL, NULL);
1964 	spin_unlock_bh(&efx->stats_lock);
1965 	efx->type->stop_stats(efx);
1966 	efx_stop_port(efx);
1967 
1968 	/* Stop the kernel transmit interface.  This is only valid if
1969 	 * the device is stopped or detached; otherwise the watchdog
1970 	 * may fire immediately.
1971 	 */
1972 	WARN_ON(netif_running(efx->net_dev) &&
1973 		netif_device_present(efx->net_dev));
1974 	netif_tx_disable(efx->net_dev);
1975 
1976 	efx_stop_datapath(efx);
1977 }
1978 
1979 static void efx_remove_all(struct efx_nic *efx)
1980 {
1981 	efx_remove_channels(efx);
1982 	efx_remove_filters(efx);
1983 #ifdef CONFIG_SFC_SRIOV
1984 	efx->type->vswitching_remove(efx);
1985 #endif
1986 	efx_remove_port(efx);
1987 	efx_remove_nic(efx);
1988 }
1989 
1990 /**************************************************************************
1991  *
1992  * Interrupt moderation
1993  *
1994  **************************************************************************/
1995 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)
1996 {
1997 	if (usecs == 0)
1998 		return 0;
1999 	if (usecs * 1000 < efx->timer_quantum_ns)
2000 		return 1; /* never round down to 0 */
2001 	return usecs * 1000 / efx->timer_quantum_ns;
2002 }
2003 
2004 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks)
2005 {
2006 	/* We must round up when converting ticks to microseconds
2007 	 * because we round down when converting the other way.
2008 	 */
2009 	return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000);
2010 }
2011 
2012 /* Set interrupt moderation parameters */
2013 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
2014 			    unsigned int rx_usecs, bool rx_adaptive,
2015 			    bool rx_may_override_tx)
2016 {
2017 	struct efx_channel *channel;
2018 	unsigned int timer_max_us;
2019 
2020 	EFX_ASSERT_RESET_SERIALISED(efx);
2021 
2022 	timer_max_us = efx->timer_max_ns / 1000;
2023 
2024 	if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)
2025 		return -EINVAL;
2026 
2027 	if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
2028 	    !rx_may_override_tx) {
2029 		netif_err(efx, drv, efx->net_dev, "Channels are shared. "
2030 			  "RX and TX IRQ moderation must be equal\n");
2031 		return -EINVAL;
2032 	}
2033 
2034 	efx->irq_rx_adaptive = rx_adaptive;
2035 	efx->irq_rx_moderation_us = rx_usecs;
2036 	efx_for_each_channel(channel, efx) {
2037 		if (efx_channel_has_rx_queue(channel))
2038 			channel->irq_moderation_us = rx_usecs;
2039 		else if (efx_channel_has_tx_queues(channel))
2040 			channel->irq_moderation_us = tx_usecs;
2041 	}
2042 
2043 	return 0;
2044 }
2045 
2046 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
2047 			    unsigned int *rx_usecs, bool *rx_adaptive)
2048 {
2049 	*rx_adaptive = efx->irq_rx_adaptive;
2050 	*rx_usecs = efx->irq_rx_moderation_us;
2051 
2052 	/* If channels are shared between RX and TX, so is IRQ
2053 	 * moderation.  Otherwise, IRQ moderation is the same for all
2054 	 * TX channels and is not adaptive.
2055 	 */
2056 	if (efx->tx_channel_offset == 0) {
2057 		*tx_usecs = *rx_usecs;
2058 	} else {
2059 		struct efx_channel *tx_channel;
2060 
2061 		tx_channel = efx->channel[efx->tx_channel_offset];
2062 		*tx_usecs = tx_channel->irq_moderation_us;
2063 	}
2064 }
2065 
2066 /**************************************************************************
2067  *
2068  * Hardware monitor
2069  *
2070  **************************************************************************/
2071 
2072 /* Run periodically off the general workqueue */
2073 static void efx_monitor(struct work_struct *data)
2074 {
2075 	struct efx_nic *efx = container_of(data, struct efx_nic,
2076 					   monitor_work.work);
2077 
2078 	netif_vdbg(efx, timer, efx->net_dev,
2079 		   "hardware monitor executing on CPU %d\n",
2080 		   raw_smp_processor_id());
2081 	BUG_ON(efx->type->monitor == NULL);
2082 
2083 	/* If the mac_lock is already held then it is likely a port
2084 	 * reconfiguration is already in place, which will likely do
2085 	 * most of the work of monitor() anyway. */
2086 	if (mutex_trylock(&efx->mac_lock)) {
2087 		if (efx->port_enabled)
2088 			efx->type->monitor(efx);
2089 		mutex_unlock(&efx->mac_lock);
2090 	}
2091 
2092 	queue_delayed_work(efx->workqueue, &efx->monitor_work,
2093 			   efx_monitor_interval);
2094 }
2095 
2096 /**************************************************************************
2097  *
2098  * ioctls
2099  *
2100  *************************************************************************/
2101 
2102 /* Net device ioctl
2103  * Context: process, rtnl_lock() held.
2104  */
2105 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
2106 {
2107 	struct efx_nic *efx = netdev_priv(net_dev);
2108 	struct mii_ioctl_data *data = if_mii(ifr);
2109 
2110 	if (cmd == SIOCSHWTSTAMP)
2111 		return efx_ptp_set_ts_config(efx, ifr);
2112 	if (cmd == SIOCGHWTSTAMP)
2113 		return efx_ptp_get_ts_config(efx, ifr);
2114 
2115 	/* Convert phy_id from older PRTAD/DEVAD format */
2116 	if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
2117 	    (data->phy_id & 0xfc00) == 0x0400)
2118 		data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
2119 
2120 	return mdio_mii_ioctl(&efx->mdio, data, cmd);
2121 }
2122 
2123 /**************************************************************************
2124  *
2125  * NAPI interface
2126  *
2127  **************************************************************************/
2128 
2129 static void efx_init_napi_channel(struct efx_channel *channel)
2130 {
2131 	struct efx_nic *efx = channel->efx;
2132 
2133 	channel->napi_dev = efx->net_dev;
2134 	netif_napi_add(channel->napi_dev, &channel->napi_str,
2135 		       efx_poll, napi_weight);
2136 }
2137 
2138 static void efx_init_napi(struct efx_nic *efx)
2139 {
2140 	struct efx_channel *channel;
2141 
2142 	efx_for_each_channel(channel, efx)
2143 		efx_init_napi_channel(channel);
2144 }
2145 
2146 static void efx_fini_napi_channel(struct efx_channel *channel)
2147 {
2148 	if (channel->napi_dev)
2149 		netif_napi_del(&channel->napi_str);
2150 
2151 	channel->napi_dev = NULL;
2152 }
2153 
2154 static void efx_fini_napi(struct efx_nic *efx)
2155 {
2156 	struct efx_channel *channel;
2157 
2158 	efx_for_each_channel(channel, efx)
2159 		efx_fini_napi_channel(channel);
2160 }
2161 
2162 /**************************************************************************
2163  *
2164  * Kernel netpoll interface
2165  *
2166  *************************************************************************/
2167 
2168 #ifdef CONFIG_NET_POLL_CONTROLLER
2169 
2170 /* Although in the common case interrupts will be disabled, this is not
2171  * guaranteed. However, all our work happens inside the NAPI callback,
2172  * so no locking is required.
2173  */
2174 static void efx_netpoll(struct net_device *net_dev)
2175 {
2176 	struct efx_nic *efx = netdev_priv(net_dev);
2177 	struct efx_channel *channel;
2178 
2179 	efx_for_each_channel(channel, efx)
2180 		efx_schedule_channel(channel);
2181 }
2182 
2183 #endif
2184 
2185 /**************************************************************************
2186  *
2187  * Kernel net device interface
2188  *
2189  *************************************************************************/
2190 
2191 /* Context: process, rtnl_lock() held. */
2192 int efx_net_open(struct net_device *net_dev)
2193 {
2194 	struct efx_nic *efx = netdev_priv(net_dev);
2195 	int rc;
2196 
2197 	netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
2198 		  raw_smp_processor_id());
2199 
2200 	rc = efx_check_disabled(efx);
2201 	if (rc)
2202 		return rc;
2203 	if (efx->phy_mode & PHY_MODE_SPECIAL)
2204 		return -EBUSY;
2205 	if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
2206 		return -EIO;
2207 
2208 	/* Notify the kernel of the link state polled during driver load,
2209 	 * before the monitor starts running */
2210 	efx_link_status_changed(efx);
2211 
2212 	efx_start_all(efx);
2213 	if (efx->state == STATE_DISABLED || efx->reset_pending)
2214 		netif_device_detach(efx->net_dev);
2215 	efx_selftest_async_start(efx);
2216 	return 0;
2217 }
2218 
2219 /* Context: process, rtnl_lock() held.
2220  * Note that the kernel will ignore our return code; this method
2221  * should really be a void.
2222  */
2223 int efx_net_stop(struct net_device *net_dev)
2224 {
2225 	struct efx_nic *efx = netdev_priv(net_dev);
2226 
2227 	netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
2228 		  raw_smp_processor_id());
2229 
2230 	/* Stop the device and flush all the channels */
2231 	efx_stop_all(efx);
2232 
2233 	return 0;
2234 }
2235 
2236 /* Context: process, dev_base_lock or RTNL held, non-blocking. */
2237 static void efx_net_stats(struct net_device *net_dev,
2238 			  struct rtnl_link_stats64 *stats)
2239 {
2240 	struct efx_nic *efx = netdev_priv(net_dev);
2241 
2242 	spin_lock_bh(&efx->stats_lock);
2243 	efx->type->update_stats(efx, NULL, stats);
2244 	spin_unlock_bh(&efx->stats_lock);
2245 }
2246 
2247 /* Context: netif_tx_lock held, BHs disabled. */
2248 static void efx_watchdog(struct net_device *net_dev)
2249 {
2250 	struct efx_nic *efx = netdev_priv(net_dev);
2251 
2252 	netif_err(efx, tx_err, efx->net_dev,
2253 		  "TX stuck with port_enabled=%d: resetting channels\n",
2254 		  efx->port_enabled);
2255 
2256 	efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
2257 }
2258 
2259 
2260 /* Context: process, rtnl_lock() held. */
2261 static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
2262 {
2263 	struct efx_nic *efx = netdev_priv(net_dev);
2264 	int rc;
2265 
2266 	rc = efx_check_disabled(efx);
2267 	if (rc)
2268 		return rc;
2269 
2270 	netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
2271 
2272 	efx_device_detach_sync(efx);
2273 	efx_stop_all(efx);
2274 
2275 	mutex_lock(&efx->mac_lock);
2276 	net_dev->mtu = new_mtu;
2277 	efx_mac_reconfigure(efx);
2278 	mutex_unlock(&efx->mac_lock);
2279 
2280 	efx_start_all(efx);
2281 	efx_device_attach_if_not_resetting(efx);
2282 	return 0;
2283 }
2284 
2285 static int efx_set_mac_address(struct net_device *net_dev, void *data)
2286 {
2287 	struct efx_nic *efx = netdev_priv(net_dev);
2288 	struct sockaddr *addr = data;
2289 	u8 *new_addr = addr->sa_data;
2290 	u8 old_addr[6];
2291 	int rc;
2292 
2293 	if (!is_valid_ether_addr(new_addr)) {
2294 		netif_err(efx, drv, efx->net_dev,
2295 			  "invalid ethernet MAC address requested: %pM\n",
2296 			  new_addr);
2297 		return -EADDRNOTAVAIL;
2298 	}
2299 
2300 	/* save old address */
2301 	ether_addr_copy(old_addr, net_dev->dev_addr);
2302 	ether_addr_copy(net_dev->dev_addr, new_addr);
2303 	if (efx->type->set_mac_address) {
2304 		rc = efx->type->set_mac_address(efx);
2305 		if (rc) {
2306 			ether_addr_copy(net_dev->dev_addr, old_addr);
2307 			return rc;
2308 		}
2309 	}
2310 
2311 	/* Reconfigure the MAC */
2312 	mutex_lock(&efx->mac_lock);
2313 	efx_mac_reconfigure(efx);
2314 	mutex_unlock(&efx->mac_lock);
2315 
2316 	return 0;
2317 }
2318 
2319 /* Context: netif_addr_lock held, BHs disabled. */
2320 static void efx_set_rx_mode(struct net_device *net_dev)
2321 {
2322 	struct efx_nic *efx = netdev_priv(net_dev);
2323 
2324 	if (efx->port_enabled)
2325 		queue_work(efx->workqueue, &efx->mac_work);
2326 	/* Otherwise efx_start_port() will do this */
2327 }
2328 
2329 static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
2330 {
2331 	struct efx_nic *efx = netdev_priv(net_dev);
2332 	int rc;
2333 
2334 	/* If disabling RX n-tuple filtering, clear existing filters */
2335 	if (net_dev->features & ~data & NETIF_F_NTUPLE) {
2336 		rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
2337 		if (rc)
2338 			return rc;
2339 	}
2340 
2341 	/* If Rx VLAN filter is changed, update filters via mac_reconfigure.
2342 	 * If rx-fcs is changed, mac_reconfigure updates that too.
2343 	 */
2344 	if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER |
2345 					  NETIF_F_RXFCS)) {
2346 		/* efx_set_rx_mode() will schedule MAC work to update filters
2347 		 * when a new features are finally set in net_dev.
2348 		 */
2349 		efx_set_rx_mode(net_dev);
2350 	}
2351 
2352 	return 0;
2353 }
2354 
2355 static int efx_get_phys_port_id(struct net_device *net_dev,
2356 				struct netdev_phys_item_id *ppid)
2357 {
2358 	struct efx_nic *efx = netdev_priv(net_dev);
2359 
2360 	if (efx->type->get_phys_port_id)
2361 		return efx->type->get_phys_port_id(efx, ppid);
2362 	else
2363 		return -EOPNOTSUPP;
2364 }
2365 
2366 static int efx_get_phys_port_name(struct net_device *net_dev,
2367 				  char *name, size_t len)
2368 {
2369 	struct efx_nic *efx = netdev_priv(net_dev);
2370 
2371 	if (snprintf(name, len, "p%u", efx->port_num) >= len)
2372 		return -EINVAL;
2373 	return 0;
2374 }
2375 
2376 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)
2377 {
2378 	struct efx_nic *efx = netdev_priv(net_dev);
2379 
2380 	if (efx->type->vlan_rx_add_vid)
2381 		return efx->type->vlan_rx_add_vid(efx, proto, vid);
2382 	else
2383 		return -EOPNOTSUPP;
2384 }
2385 
2386 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)
2387 {
2388 	struct efx_nic *efx = netdev_priv(net_dev);
2389 
2390 	if (efx->type->vlan_rx_kill_vid)
2391 		return efx->type->vlan_rx_kill_vid(efx, proto, vid);
2392 	else
2393 		return -EOPNOTSUPP;
2394 }
2395 
2396 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in)
2397 {
2398 	switch (in) {
2399 	case UDP_TUNNEL_TYPE_VXLAN:
2400 		return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN;
2401 	case UDP_TUNNEL_TYPE_GENEVE:
2402 		return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE;
2403 	default:
2404 		return -1;
2405 	}
2406 }
2407 
2408 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti)
2409 {
2410 	struct efx_nic *efx = netdev_priv(dev);
2411 	struct efx_udp_tunnel tnl;
2412 	int efx_tunnel_type;
2413 
2414 	efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2415 	if (efx_tunnel_type < 0)
2416 		return;
2417 
2418 	tnl.type = (u16)efx_tunnel_type;
2419 	tnl.port = ti->port;
2420 
2421 	if (efx->type->udp_tnl_add_port)
2422 		(void)efx->type->udp_tnl_add_port(efx, tnl);
2423 }
2424 
2425 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti)
2426 {
2427 	struct efx_nic *efx = netdev_priv(dev);
2428 	struct efx_udp_tunnel tnl;
2429 	int efx_tunnel_type;
2430 
2431 	efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2432 	if (efx_tunnel_type < 0)
2433 		return;
2434 
2435 	tnl.type = (u16)efx_tunnel_type;
2436 	tnl.port = ti->port;
2437 
2438 	if (efx->type->udp_tnl_del_port)
2439 		(void)efx->type->udp_tnl_del_port(efx, tnl);
2440 }
2441 
2442 static const struct net_device_ops efx_netdev_ops = {
2443 	.ndo_open		= efx_net_open,
2444 	.ndo_stop		= efx_net_stop,
2445 	.ndo_get_stats64	= efx_net_stats,
2446 	.ndo_tx_timeout		= efx_watchdog,
2447 	.ndo_start_xmit		= efx_hard_start_xmit,
2448 	.ndo_validate_addr	= eth_validate_addr,
2449 	.ndo_do_ioctl		= efx_ioctl,
2450 	.ndo_change_mtu		= efx_change_mtu,
2451 	.ndo_set_mac_address	= efx_set_mac_address,
2452 	.ndo_set_rx_mode	= efx_set_rx_mode,
2453 	.ndo_set_features	= efx_set_features,
2454 	.ndo_vlan_rx_add_vid	= efx_vlan_rx_add_vid,
2455 	.ndo_vlan_rx_kill_vid	= efx_vlan_rx_kill_vid,
2456 #ifdef CONFIG_SFC_SRIOV
2457 	.ndo_set_vf_mac		= efx_sriov_set_vf_mac,
2458 	.ndo_set_vf_vlan	= efx_sriov_set_vf_vlan,
2459 	.ndo_set_vf_spoofchk	= efx_sriov_set_vf_spoofchk,
2460 	.ndo_get_vf_config	= efx_sriov_get_vf_config,
2461 	.ndo_set_vf_link_state  = efx_sriov_set_vf_link_state,
2462 #endif
2463 	.ndo_get_phys_port_id   = efx_get_phys_port_id,
2464 	.ndo_get_phys_port_name	= efx_get_phys_port_name,
2465 #ifdef CONFIG_NET_POLL_CONTROLLER
2466 	.ndo_poll_controller = efx_netpoll,
2467 #endif
2468 	.ndo_setup_tc		= efx_setup_tc,
2469 #ifdef CONFIG_RFS_ACCEL
2470 	.ndo_rx_flow_steer	= efx_filter_rfs,
2471 #endif
2472 	.ndo_udp_tunnel_add	= efx_udp_tunnel_add,
2473 	.ndo_udp_tunnel_del	= efx_udp_tunnel_del,
2474 };
2475 
2476 static void efx_update_name(struct efx_nic *efx)
2477 {
2478 	strcpy(efx->name, efx->net_dev->name);
2479 	efx_mtd_rename(efx);
2480 	efx_set_channel_names(efx);
2481 }
2482 
2483 static int efx_netdev_event(struct notifier_block *this,
2484 			    unsigned long event, void *ptr)
2485 {
2486 	struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
2487 
2488 	if ((net_dev->netdev_ops == &efx_netdev_ops) &&
2489 	    event == NETDEV_CHANGENAME)
2490 		efx_update_name(netdev_priv(net_dev));
2491 
2492 	return NOTIFY_DONE;
2493 }
2494 
2495 static struct notifier_block efx_netdev_notifier = {
2496 	.notifier_call = efx_netdev_event,
2497 };
2498 
2499 static ssize_t
2500 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
2501 {
2502 	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2503 	return sprintf(buf, "%d\n", efx->phy_type);
2504 }
2505 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL);
2506 
2507 #ifdef CONFIG_SFC_MCDI_LOGGING
2508 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr,
2509 			     char *buf)
2510 {
2511 	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2512 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2513 
2514 	return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled);
2515 }
2516 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr,
2517 			    const char *buf, size_t count)
2518 {
2519 	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2520 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2521 	bool enable = count > 0 && *buf != '0';
2522 
2523 	mcdi->logging_enabled = enable;
2524 	return count;
2525 }
2526 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log);
2527 #endif
2528 
2529 static int efx_register_netdev(struct efx_nic *efx)
2530 {
2531 	struct net_device *net_dev = efx->net_dev;
2532 	struct efx_channel *channel;
2533 	int rc;
2534 
2535 	net_dev->watchdog_timeo = 5 * HZ;
2536 	net_dev->irq = efx->pci_dev->irq;
2537 	net_dev->netdev_ops = &efx_netdev_ops;
2538 	if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
2539 		net_dev->priv_flags |= IFF_UNICAST_FLT;
2540 	net_dev->ethtool_ops = &efx_ethtool_ops;
2541 	net_dev->gso_max_segs = EFX_TSO_MAX_SEGS;
2542 	net_dev->min_mtu = EFX_MIN_MTU;
2543 	net_dev->max_mtu = EFX_MAX_MTU;
2544 
2545 	rtnl_lock();
2546 
2547 	/* Enable resets to be scheduled and check whether any were
2548 	 * already requested.  If so, the NIC is probably hosed so we
2549 	 * abort.
2550 	 */
2551 	efx->state = STATE_READY;
2552 	smp_mb(); /* ensure we change state before checking reset_pending */
2553 	if (efx->reset_pending) {
2554 		netif_err(efx, probe, efx->net_dev,
2555 			  "aborting probe due to scheduled reset\n");
2556 		rc = -EIO;
2557 		goto fail_locked;
2558 	}
2559 
2560 	rc = dev_alloc_name(net_dev, net_dev->name);
2561 	if (rc < 0)
2562 		goto fail_locked;
2563 	efx_update_name(efx);
2564 
2565 	/* Always start with carrier off; PHY events will detect the link */
2566 	netif_carrier_off(net_dev);
2567 
2568 	rc = register_netdevice(net_dev);
2569 	if (rc)
2570 		goto fail_locked;
2571 
2572 	efx_for_each_channel(channel, efx) {
2573 		struct efx_tx_queue *tx_queue;
2574 		efx_for_each_channel_tx_queue(tx_queue, channel)
2575 			efx_init_tx_queue_core_txq(tx_queue);
2576 	}
2577 
2578 	efx_associate(efx);
2579 
2580 	rtnl_unlock();
2581 
2582 	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2583 	if (rc) {
2584 		netif_err(efx, drv, efx->net_dev,
2585 			  "failed to init net dev attributes\n");
2586 		goto fail_registered;
2587 	}
2588 #ifdef CONFIG_SFC_MCDI_LOGGING
2589 	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2590 	if (rc) {
2591 		netif_err(efx, drv, efx->net_dev,
2592 			  "failed to init net dev attributes\n");
2593 		goto fail_attr_mcdi_logging;
2594 	}
2595 #endif
2596 
2597 	return 0;
2598 
2599 #ifdef CONFIG_SFC_MCDI_LOGGING
2600 fail_attr_mcdi_logging:
2601 	device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2602 #endif
2603 fail_registered:
2604 	rtnl_lock();
2605 	efx_dissociate(efx);
2606 	unregister_netdevice(net_dev);
2607 fail_locked:
2608 	efx->state = STATE_UNINIT;
2609 	rtnl_unlock();
2610 	netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
2611 	return rc;
2612 }
2613 
2614 static void efx_unregister_netdev(struct efx_nic *efx)
2615 {
2616 	if (!efx->net_dev)
2617 		return;
2618 
2619 	BUG_ON(netdev_priv(efx->net_dev) != efx);
2620 
2621 	if (efx_dev_registered(efx)) {
2622 		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2623 #ifdef CONFIG_SFC_MCDI_LOGGING
2624 		device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2625 #endif
2626 		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2627 		unregister_netdev(efx->net_dev);
2628 	}
2629 }
2630 
2631 /**************************************************************************
2632  *
2633  * Device reset and suspend
2634  *
2635  **************************************************************************/
2636 
2637 /* Tears down the entire software state and most of the hardware state
2638  * before reset.  */
2639 void efx_reset_down(struct efx_nic *efx, enum reset_type method)
2640 {
2641 	EFX_ASSERT_RESET_SERIALISED(efx);
2642 
2643 	if (method == RESET_TYPE_MCDI_TIMEOUT)
2644 		efx->type->prepare_flr(efx);
2645 
2646 	efx_stop_all(efx);
2647 	efx_disable_interrupts(efx);
2648 
2649 	mutex_lock(&efx->mac_lock);
2650 	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2651 	    method != RESET_TYPE_DATAPATH)
2652 		efx->phy_op->fini(efx);
2653 	efx->type->fini(efx);
2654 }
2655 
2656 /* This function will always ensure that the locks acquired in
2657  * efx_reset_down() are released. A failure return code indicates
2658  * that we were unable to reinitialise the hardware, and the
2659  * driver should be disabled. If ok is false, then the rx and tx
2660  * engines are not restarted, pending a RESET_DISABLE. */
2661 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
2662 {
2663 	int rc;
2664 
2665 	EFX_ASSERT_RESET_SERIALISED(efx);
2666 
2667 	if (method == RESET_TYPE_MCDI_TIMEOUT)
2668 		efx->type->finish_flr(efx);
2669 
2670 	/* Ensure that SRAM is initialised even if we're disabling the device */
2671 	rc = efx->type->init(efx);
2672 	if (rc) {
2673 		netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
2674 		goto fail;
2675 	}
2676 
2677 	if (!ok)
2678 		goto fail;
2679 
2680 	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2681 	    method != RESET_TYPE_DATAPATH) {
2682 		rc = efx->phy_op->init(efx);
2683 		if (rc)
2684 			goto fail;
2685 		rc = efx->phy_op->reconfigure(efx);
2686 		if (rc && rc != -EPERM)
2687 			netif_err(efx, drv, efx->net_dev,
2688 				  "could not restore PHY settings\n");
2689 	}
2690 
2691 	rc = efx_enable_interrupts(efx);
2692 	if (rc)
2693 		goto fail;
2694 
2695 #ifdef CONFIG_SFC_SRIOV
2696 	rc = efx->type->vswitching_restore(efx);
2697 	if (rc) /* not fatal; the PF will still work fine */
2698 		netif_warn(efx, probe, efx->net_dev,
2699 			   "failed to restore vswitching rc=%d;"
2700 			   " VFs may not function\n", rc);
2701 #endif
2702 
2703 	down_read(&efx->filter_sem);
2704 	efx_restore_filters(efx);
2705 	up_read(&efx->filter_sem);
2706 	if (efx->type->sriov_reset)
2707 		efx->type->sriov_reset(efx);
2708 
2709 	mutex_unlock(&efx->mac_lock);
2710 
2711 	efx_start_all(efx);
2712 
2713 	if (efx->type->udp_tnl_push_ports)
2714 		efx->type->udp_tnl_push_ports(efx);
2715 
2716 	return 0;
2717 
2718 fail:
2719 	efx->port_initialized = false;
2720 
2721 	mutex_unlock(&efx->mac_lock);
2722 
2723 	return rc;
2724 }
2725 
2726 /* Reset the NIC using the specified method.  Note that the reset may
2727  * fail, in which case the card will be left in an unusable state.
2728  *
2729  * Caller must hold the rtnl_lock.
2730  */
2731 int efx_reset(struct efx_nic *efx, enum reset_type method)
2732 {
2733 	int rc, rc2;
2734 	bool disabled;
2735 
2736 	netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
2737 		   RESET_TYPE(method));
2738 
2739 	efx_device_detach_sync(efx);
2740 	efx_reset_down(efx, method);
2741 
2742 	rc = efx->type->reset(efx, method);
2743 	if (rc) {
2744 		netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
2745 		goto out;
2746 	}
2747 
2748 	/* Clear flags for the scopes we covered.  We assume the NIC and
2749 	 * driver are now quiescent so that there is no race here.
2750 	 */
2751 	if (method < RESET_TYPE_MAX_METHOD)
2752 		efx->reset_pending &= -(1 << (method + 1));
2753 	else /* it doesn't fit into the well-ordered scope hierarchy */
2754 		__clear_bit(method, &efx->reset_pending);
2755 
2756 	/* Reinitialise bus-mastering, which may have been turned off before
2757 	 * the reset was scheduled. This is still appropriate, even in the
2758 	 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
2759 	 * can respond to requests. */
2760 	pci_set_master(efx->pci_dev);
2761 
2762 out:
2763 	/* Leave device stopped if necessary */
2764 	disabled = rc ||
2765 		method == RESET_TYPE_DISABLE ||
2766 		method == RESET_TYPE_RECOVER_OR_DISABLE;
2767 	rc2 = efx_reset_up(efx, method, !disabled);
2768 	if (rc2) {
2769 		disabled = true;
2770 		if (!rc)
2771 			rc = rc2;
2772 	}
2773 
2774 	if (disabled) {
2775 		dev_close(efx->net_dev);
2776 		netif_err(efx, drv, efx->net_dev, "has been disabled\n");
2777 		efx->state = STATE_DISABLED;
2778 	} else {
2779 		netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
2780 		efx_device_attach_if_not_resetting(efx);
2781 	}
2782 	return rc;
2783 }
2784 
2785 /* Try recovery mechanisms.
2786  * For now only EEH is supported.
2787  * Returns 0 if the recovery mechanisms are unsuccessful.
2788  * Returns a non-zero value otherwise.
2789  */
2790 int efx_try_recovery(struct efx_nic *efx)
2791 {
2792 #ifdef CONFIG_EEH
2793 	/* A PCI error can occur and not be seen by EEH because nothing
2794 	 * happens on the PCI bus. In this case the driver may fail and
2795 	 * schedule a 'recover or reset', leading to this recovery handler.
2796 	 * Manually call the eeh failure check function.
2797 	 */
2798 	struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev);
2799 	if (eeh_dev_check_failure(eehdev)) {
2800 		/* The EEH mechanisms will handle the error and reset the
2801 		 * device if necessary.
2802 		 */
2803 		return 1;
2804 	}
2805 #endif
2806 	return 0;
2807 }
2808 
2809 static void efx_wait_for_bist_end(struct efx_nic *efx)
2810 {
2811 	int i;
2812 
2813 	for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) {
2814 		if (efx_mcdi_poll_reboot(efx))
2815 			goto out;
2816 		msleep(BIST_WAIT_DELAY_MS);
2817 	}
2818 
2819 	netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n");
2820 out:
2821 	/* Either way unset the BIST flag. If we found no reboot we probably
2822 	 * won't recover, but we should try.
2823 	 */
2824 	efx->mc_bist_for_other_fn = false;
2825 }
2826 
2827 /* The worker thread exists so that code that cannot sleep can
2828  * schedule a reset for later.
2829  */
2830 static void efx_reset_work(struct work_struct *data)
2831 {
2832 	struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
2833 	unsigned long pending;
2834 	enum reset_type method;
2835 
2836 	pending = READ_ONCE(efx->reset_pending);
2837 	method = fls(pending) - 1;
2838 
2839 	if (method == RESET_TYPE_MC_BIST)
2840 		efx_wait_for_bist_end(efx);
2841 
2842 	if ((method == RESET_TYPE_RECOVER_OR_DISABLE ||
2843 	     method == RESET_TYPE_RECOVER_OR_ALL) &&
2844 	    efx_try_recovery(efx))
2845 		return;
2846 
2847 	if (!pending)
2848 		return;
2849 
2850 	rtnl_lock();
2851 
2852 	/* We checked the state in efx_schedule_reset() but it may
2853 	 * have changed by now.  Now that we have the RTNL lock,
2854 	 * it cannot change again.
2855 	 */
2856 	if (efx->state == STATE_READY)
2857 		(void)efx_reset(efx, method);
2858 
2859 	rtnl_unlock();
2860 }
2861 
2862 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
2863 {
2864 	enum reset_type method;
2865 
2866 	if (efx->state == STATE_RECOVERY) {
2867 		netif_dbg(efx, drv, efx->net_dev,
2868 			  "recovering: skip scheduling %s reset\n",
2869 			  RESET_TYPE(type));
2870 		return;
2871 	}
2872 
2873 	switch (type) {
2874 	case RESET_TYPE_INVISIBLE:
2875 	case RESET_TYPE_ALL:
2876 	case RESET_TYPE_RECOVER_OR_ALL:
2877 	case RESET_TYPE_WORLD:
2878 	case RESET_TYPE_DISABLE:
2879 	case RESET_TYPE_RECOVER_OR_DISABLE:
2880 	case RESET_TYPE_DATAPATH:
2881 	case RESET_TYPE_MC_BIST:
2882 	case RESET_TYPE_MCDI_TIMEOUT:
2883 		method = type;
2884 		netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
2885 			  RESET_TYPE(method));
2886 		break;
2887 	default:
2888 		method = efx->type->map_reset_reason(type);
2889 		netif_dbg(efx, drv, efx->net_dev,
2890 			  "scheduling %s reset for %s\n",
2891 			  RESET_TYPE(method), RESET_TYPE(type));
2892 		break;
2893 	}
2894 
2895 	set_bit(method, &efx->reset_pending);
2896 	smp_mb(); /* ensure we change reset_pending before checking state */
2897 
2898 	/* If we're not READY then just leave the flags set as the cue
2899 	 * to abort probing or reschedule the reset later.
2900 	 */
2901 	if (READ_ONCE(efx->state) != STATE_READY)
2902 		return;
2903 
2904 	/* efx_process_channel() will no longer read events once a
2905 	 * reset is scheduled. So switch back to poll'd MCDI completions. */
2906 	efx_mcdi_mode_poll(efx);
2907 
2908 	queue_work(reset_workqueue, &efx->reset_work);
2909 }
2910 
2911 /**************************************************************************
2912  *
2913  * List of NICs we support
2914  *
2915  **************************************************************************/
2916 
2917 /* PCI device ID table */
2918 static const struct pci_device_id efx_pci_table[] = {
2919 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803),	/* SFC9020 */
2920 	 .driver_data = (unsigned long) &siena_a0_nic_type},
2921 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813),	/* SFL9021 */
2922 	 .driver_data = (unsigned long) &siena_a0_nic_type},
2923 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903),  /* SFC9120 PF */
2924 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2925 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903),  /* SFC9120 VF */
2926 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2927 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923),  /* SFC9140 PF */
2928 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2929 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923),  /* SFC9140 VF */
2930 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2931 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03),  /* SFC9220 PF */
2932 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2933 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03),  /* SFC9220 VF */
2934 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2935 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03),  /* SFC9250 PF */
2936 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2937 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03),  /* SFC9250 VF */
2938 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2939 	{0}			/* end of list */
2940 };
2941 
2942 /**************************************************************************
2943  *
2944  * Dummy PHY/MAC operations
2945  *
2946  * Can be used for some unimplemented operations
2947  * Needed so all function pointers are valid and do not have to be tested
2948  * before use
2949  *
2950  **************************************************************************/
2951 int efx_port_dummy_op_int(struct efx_nic *efx)
2952 {
2953 	return 0;
2954 }
2955 void efx_port_dummy_op_void(struct efx_nic *efx) {}
2956 
2957 static bool efx_port_dummy_op_poll(struct efx_nic *efx)
2958 {
2959 	return false;
2960 }
2961 
2962 static const struct efx_phy_operations efx_dummy_phy_operations = {
2963 	.init		 = efx_port_dummy_op_int,
2964 	.reconfigure	 = efx_port_dummy_op_int,
2965 	.poll		 = efx_port_dummy_op_poll,
2966 	.fini		 = efx_port_dummy_op_void,
2967 };
2968 
2969 /**************************************************************************
2970  *
2971  * Data housekeeping
2972  *
2973  **************************************************************************/
2974 
2975 /* This zeroes out and then fills in the invariants in a struct
2976  * efx_nic (including all sub-structures).
2977  */
2978 static int efx_init_struct(struct efx_nic *efx,
2979 			   struct pci_dev *pci_dev, struct net_device *net_dev)
2980 {
2981 	int rc = -ENOMEM, i;
2982 
2983 	/* Initialise common structures */
2984 	INIT_LIST_HEAD(&efx->node);
2985 	INIT_LIST_HEAD(&efx->secondary_list);
2986 	spin_lock_init(&efx->biu_lock);
2987 #ifdef CONFIG_SFC_MTD
2988 	INIT_LIST_HEAD(&efx->mtd_list);
2989 #endif
2990 	INIT_WORK(&efx->reset_work, efx_reset_work);
2991 	INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
2992 	INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work);
2993 	efx->pci_dev = pci_dev;
2994 	efx->msg_enable = debug;
2995 	efx->state = STATE_UNINIT;
2996 	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
2997 
2998 	efx->net_dev = net_dev;
2999 	efx->rx_prefix_size = efx->type->rx_prefix_size;
3000 	efx->rx_ip_align =
3001 		NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0;
3002 	efx->rx_packet_hash_offset =
3003 		efx->type->rx_hash_offset - efx->type->rx_prefix_size;
3004 	efx->rx_packet_ts_offset =
3005 		efx->type->rx_ts_offset - efx->type->rx_prefix_size;
3006 	spin_lock_init(&efx->stats_lock);
3007 	efx->vi_stride = EFX_DEFAULT_VI_STRIDE;
3008 	efx->num_mac_stats = MC_CMD_MAC_NSTATS;
3009 	BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END);
3010 	mutex_init(&efx->mac_lock);
3011 	efx->phy_op = &efx_dummy_phy_operations;
3012 	efx->mdio.dev = net_dev;
3013 	INIT_WORK(&efx->mac_work, efx_mac_work);
3014 	init_waitqueue_head(&efx->flush_wq);
3015 
3016 	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
3017 		efx->channel[i] = efx_alloc_channel(efx, i, NULL);
3018 		if (!efx->channel[i])
3019 			goto fail;
3020 		efx->msi_context[i].efx = efx;
3021 		efx->msi_context[i].index = i;
3022 	}
3023 
3024 	/* Higher numbered interrupt modes are less capable! */
3025 	if (WARN_ON_ONCE(efx->type->max_interrupt_mode >
3026 			 efx->type->min_interrupt_mode)) {
3027 		rc = -EIO;
3028 		goto fail;
3029 	}
3030 	efx->interrupt_mode = max(efx->type->max_interrupt_mode,
3031 				  interrupt_mode);
3032 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,
3033 				  interrupt_mode);
3034 
3035 	/* Would be good to use the net_dev name, but we're too early */
3036 	snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
3037 		 pci_name(pci_dev));
3038 	efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
3039 	if (!efx->workqueue)
3040 		goto fail;
3041 
3042 	return 0;
3043 
3044 fail:
3045 	efx_fini_struct(efx);
3046 	return rc;
3047 }
3048 
3049 static void efx_fini_struct(struct efx_nic *efx)
3050 {
3051 	int i;
3052 
3053 	for (i = 0; i < EFX_MAX_CHANNELS; i++)
3054 		kfree(efx->channel[i]);
3055 
3056 	kfree(efx->vpd_sn);
3057 
3058 	if (efx->workqueue) {
3059 		destroy_workqueue(efx->workqueue);
3060 		efx->workqueue = NULL;
3061 	}
3062 }
3063 
3064 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
3065 {
3066 	u64 n_rx_nodesc_trunc = 0;
3067 	struct efx_channel *channel;
3068 
3069 	efx_for_each_channel(channel, efx)
3070 		n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
3071 	stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
3072 	stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
3073 }
3074 
3075 /**************************************************************************
3076  *
3077  * PCI interface
3078  *
3079  **************************************************************************/
3080 
3081 /* Main body of final NIC shutdown code
3082  * This is called only at module unload (or hotplug removal).
3083  */
3084 static void efx_pci_remove_main(struct efx_nic *efx)
3085 {
3086 	/* Flush reset_work. It can no longer be scheduled since we
3087 	 * are not READY.
3088 	 */
3089 	BUG_ON(efx->state == STATE_READY);
3090 	cancel_work_sync(&efx->reset_work);
3091 
3092 	efx_disable_interrupts(efx);
3093 	efx_nic_fini_interrupt(efx);
3094 	efx_fini_port(efx);
3095 	efx->type->fini(efx);
3096 	efx_fini_napi(efx);
3097 	efx_remove_all(efx);
3098 }
3099 
3100 /* Final NIC shutdown
3101  * This is called only at module unload (or hotplug removal).  A PF can call
3102  * this on its VFs to ensure they are unbound first.
3103  */
3104 static void efx_pci_remove(struct pci_dev *pci_dev)
3105 {
3106 	struct efx_nic *efx;
3107 
3108 	efx = pci_get_drvdata(pci_dev);
3109 	if (!efx)
3110 		return;
3111 
3112 	/* Mark the NIC as fini, then stop the interface */
3113 	rtnl_lock();
3114 	efx_dissociate(efx);
3115 	dev_close(efx->net_dev);
3116 	efx_disable_interrupts(efx);
3117 	efx->state = STATE_UNINIT;
3118 	rtnl_unlock();
3119 
3120 	if (efx->type->sriov_fini)
3121 		efx->type->sriov_fini(efx);
3122 
3123 	efx_unregister_netdev(efx);
3124 
3125 	efx_mtd_remove(efx);
3126 
3127 	efx_pci_remove_main(efx);
3128 
3129 	efx_fini_io(efx);
3130 	netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
3131 
3132 	efx_fini_struct(efx);
3133 	free_netdev(efx->net_dev);
3134 
3135 	pci_disable_pcie_error_reporting(pci_dev);
3136 };
3137 
3138 /* NIC VPD information
3139  * Called during probe to display the part number of the
3140  * installed NIC.  VPD is potentially very large but this should
3141  * always appear within the first 512 bytes.
3142  */
3143 #define SFC_VPD_LEN 512
3144 static void efx_probe_vpd_strings(struct efx_nic *efx)
3145 {
3146 	struct pci_dev *dev = efx->pci_dev;
3147 	char vpd_data[SFC_VPD_LEN];
3148 	ssize_t vpd_size;
3149 	int ro_start, ro_size, i, j;
3150 
3151 	/* Get the vpd data from the device */
3152 	vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
3153 	if (vpd_size <= 0) {
3154 		netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
3155 		return;
3156 	}
3157 
3158 	/* Get the Read only section */
3159 	ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
3160 	if (ro_start < 0) {
3161 		netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
3162 		return;
3163 	}
3164 
3165 	ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]);
3166 	j = ro_size;
3167 	i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
3168 	if (i + j > vpd_size)
3169 		j = vpd_size - i;
3170 
3171 	/* Get the Part number */
3172 	i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN");
3173 	if (i < 0) {
3174 		netif_err(efx, drv, efx->net_dev, "Part number not found\n");
3175 		return;
3176 	}
3177 
3178 	j = pci_vpd_info_field_size(&vpd_data[i]);
3179 	i += PCI_VPD_INFO_FLD_HDR_SIZE;
3180 	if (i + j > vpd_size) {
3181 		netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
3182 		return;
3183 	}
3184 
3185 	netif_info(efx, drv, efx->net_dev,
3186 		   "Part Number : %.*s\n", j, &vpd_data[i]);
3187 
3188 	i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
3189 	j = ro_size;
3190 	i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN");
3191 	if (i < 0) {
3192 		netif_err(efx, drv, efx->net_dev, "Serial number not found\n");
3193 		return;
3194 	}
3195 
3196 	j = pci_vpd_info_field_size(&vpd_data[i]);
3197 	i += PCI_VPD_INFO_FLD_HDR_SIZE;
3198 	if (i + j > vpd_size) {
3199 		netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n");
3200 		return;
3201 	}
3202 
3203 	efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL);
3204 	if (!efx->vpd_sn)
3205 		return;
3206 
3207 	snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]);
3208 }
3209 
3210 
3211 /* Main body of NIC initialisation
3212  * This is called at module load (or hotplug insertion, theoretically).
3213  */
3214 static int efx_pci_probe_main(struct efx_nic *efx)
3215 {
3216 	int rc;
3217 
3218 	/* Do start-of-day initialisation */
3219 	rc = efx_probe_all(efx);
3220 	if (rc)
3221 		goto fail1;
3222 
3223 	efx_init_napi(efx);
3224 
3225 	rc = efx->type->init(efx);
3226 	if (rc) {
3227 		netif_err(efx, probe, efx->net_dev,
3228 			  "failed to initialise NIC\n");
3229 		goto fail3;
3230 	}
3231 
3232 	rc = efx_init_port(efx);
3233 	if (rc) {
3234 		netif_err(efx, probe, efx->net_dev,
3235 			  "failed to initialise port\n");
3236 		goto fail4;
3237 	}
3238 
3239 	rc = efx_nic_init_interrupt(efx);
3240 	if (rc)
3241 		goto fail5;
3242 	rc = efx_enable_interrupts(efx);
3243 	if (rc)
3244 		goto fail6;
3245 
3246 	return 0;
3247 
3248  fail6:
3249 	efx_nic_fini_interrupt(efx);
3250  fail5:
3251 	efx_fini_port(efx);
3252  fail4:
3253 	efx->type->fini(efx);
3254  fail3:
3255 	efx_fini_napi(efx);
3256 	efx_remove_all(efx);
3257  fail1:
3258 	return rc;
3259 }
3260 
3261 static int efx_pci_probe_post_io(struct efx_nic *efx)
3262 {
3263 	struct net_device *net_dev = efx->net_dev;
3264 	int rc = efx_pci_probe_main(efx);
3265 
3266 	if (rc)
3267 		return rc;
3268 
3269 	if (efx->type->sriov_init) {
3270 		rc = efx->type->sriov_init(efx);
3271 		if (rc)
3272 			netif_err(efx, probe, efx->net_dev,
3273 				  "SR-IOV can't be enabled rc %d\n", rc);
3274 	}
3275 
3276 	/* Determine netdevice features */
3277 	net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
3278 			      NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL);
3279 	if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
3280 		net_dev->features |= NETIF_F_TSO6;
3281 	/* Check whether device supports TSO */
3282 	if (!efx->type->tso_versions || !efx->type->tso_versions(efx))
3283 		net_dev->features &= ~NETIF_F_ALL_TSO;
3284 	/* Mask for features that also apply to VLAN devices */
3285 	net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |
3286 				   NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
3287 				   NETIF_F_RXCSUM);
3288 
3289 	net_dev->hw_features |= net_dev->features & ~efx->fixed_features;
3290 
3291 	/* Disable receiving frames with bad FCS, by default. */
3292 	net_dev->features &= ~NETIF_F_RXALL;
3293 
3294 	/* Disable VLAN filtering by default.  It may be enforced if
3295 	 * the feature is fixed (i.e. VLAN filters are required to
3296 	 * receive VLAN tagged packets due to vPort restrictions).
3297 	 */
3298 	net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
3299 	net_dev->features |= efx->fixed_features;
3300 
3301 	rc = efx_register_netdev(efx);
3302 	if (!rc)
3303 		return 0;
3304 
3305 	efx_pci_remove_main(efx);
3306 	return rc;
3307 }
3308 
3309 /* NIC initialisation
3310  *
3311  * This is called at module load (or hotplug insertion,
3312  * theoretically).  It sets up PCI mappings, resets the NIC,
3313  * sets up and registers the network devices with the kernel and hooks
3314  * the interrupt service routine.  It does not prepare the device for
3315  * transmission; this is left to the first time one of the network
3316  * interfaces is brought up (i.e. efx_net_open).
3317  */
3318 static int efx_pci_probe(struct pci_dev *pci_dev,
3319 			 const struct pci_device_id *entry)
3320 {
3321 	struct net_device *net_dev;
3322 	struct efx_nic *efx;
3323 	int rc;
3324 
3325 	/* Allocate and initialise a struct net_device and struct efx_nic */
3326 	net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
3327 				     EFX_MAX_RX_QUEUES);
3328 	if (!net_dev)
3329 		return -ENOMEM;
3330 	efx = netdev_priv(net_dev);
3331 	efx->type = (const struct efx_nic_type *) entry->driver_data;
3332 	efx->fixed_features |= NETIF_F_HIGHDMA;
3333 
3334 	pci_set_drvdata(pci_dev, efx);
3335 	SET_NETDEV_DEV(net_dev, &pci_dev->dev);
3336 	rc = efx_init_struct(efx, pci_dev, net_dev);
3337 	if (rc)
3338 		goto fail1;
3339 
3340 	netif_info(efx, probe, efx->net_dev,
3341 		   "Solarflare NIC detected\n");
3342 
3343 	if (!efx->type->is_vf)
3344 		efx_probe_vpd_strings(efx);
3345 
3346 	/* Set up basic I/O (BAR mappings etc) */
3347 	rc = efx_init_io(efx);
3348 	if (rc)
3349 		goto fail2;
3350 
3351 	rc = efx_pci_probe_post_io(efx);
3352 	if (rc) {
3353 		/* On failure, retry once immediately.
3354 		 * If we aborted probe due to a scheduled reset, dismiss it.
3355 		 */
3356 		efx->reset_pending = 0;
3357 		rc = efx_pci_probe_post_io(efx);
3358 		if (rc) {
3359 			/* On another failure, retry once more
3360 			 * after a 50-305ms delay.
3361 			 */
3362 			unsigned char r;
3363 
3364 			get_random_bytes(&r, 1);
3365 			msleep((unsigned int)r + 50);
3366 			efx->reset_pending = 0;
3367 			rc = efx_pci_probe_post_io(efx);
3368 		}
3369 	}
3370 	if (rc)
3371 		goto fail3;
3372 
3373 	netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
3374 
3375 	/* Try to create MTDs, but allow this to fail */
3376 	rtnl_lock();
3377 	rc = efx_mtd_probe(efx);
3378 	rtnl_unlock();
3379 	if (rc && rc != -EPERM)
3380 		netif_warn(efx, probe, efx->net_dev,
3381 			   "failed to create MTDs (%d)\n", rc);
3382 
3383 	rc = pci_enable_pcie_error_reporting(pci_dev);
3384 	if (rc && rc != -EINVAL)
3385 		netif_notice(efx, probe, efx->net_dev,
3386 			     "PCIE error reporting unavailable (%d).\n",
3387 			     rc);
3388 
3389 	if (efx->type->udp_tnl_push_ports)
3390 		efx->type->udp_tnl_push_ports(efx);
3391 
3392 	return 0;
3393 
3394  fail3:
3395 	efx_fini_io(efx);
3396  fail2:
3397 	efx_fini_struct(efx);
3398  fail1:
3399 	WARN_ON(rc > 0);
3400 	netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
3401 	free_netdev(net_dev);
3402 	return rc;
3403 }
3404 
3405 /* efx_pci_sriov_configure returns the actual number of Virtual Functions
3406  * enabled on success
3407  */
3408 #ifdef CONFIG_SFC_SRIOV
3409 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
3410 {
3411 	int rc;
3412 	struct efx_nic *efx = pci_get_drvdata(dev);
3413 
3414 	if (efx->type->sriov_configure) {
3415 		rc = efx->type->sriov_configure(efx, num_vfs);
3416 		if (rc)
3417 			return rc;
3418 		else
3419 			return num_vfs;
3420 	} else
3421 		return -EOPNOTSUPP;
3422 }
3423 #endif
3424 
3425 static int efx_pm_freeze(struct device *dev)
3426 {
3427 	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
3428 
3429 	rtnl_lock();
3430 
3431 	if (efx->state != STATE_DISABLED) {
3432 		efx->state = STATE_UNINIT;
3433 
3434 		efx_device_detach_sync(efx);
3435 
3436 		efx_stop_all(efx);
3437 		efx_disable_interrupts(efx);
3438 	}
3439 
3440 	rtnl_unlock();
3441 
3442 	return 0;
3443 }
3444 
3445 static int efx_pm_thaw(struct device *dev)
3446 {
3447 	int rc;
3448 	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
3449 
3450 	rtnl_lock();
3451 
3452 	if (efx->state != STATE_DISABLED) {
3453 		rc = efx_enable_interrupts(efx);
3454 		if (rc)
3455 			goto fail;
3456 
3457 		mutex_lock(&efx->mac_lock);
3458 		efx->phy_op->reconfigure(efx);
3459 		mutex_unlock(&efx->mac_lock);
3460 
3461 		efx_start_all(efx);
3462 
3463 		efx_device_attach_if_not_resetting(efx);
3464 
3465 		efx->state = STATE_READY;
3466 
3467 		efx->type->resume_wol(efx);
3468 	}
3469 
3470 	rtnl_unlock();
3471 
3472 	/* Reschedule any quenched resets scheduled during efx_pm_freeze() */
3473 	queue_work(reset_workqueue, &efx->reset_work);
3474 
3475 	return 0;
3476 
3477 fail:
3478 	rtnl_unlock();
3479 
3480 	return rc;
3481 }
3482 
3483 static int efx_pm_poweroff(struct device *dev)
3484 {
3485 	struct pci_dev *pci_dev = to_pci_dev(dev);
3486 	struct efx_nic *efx = pci_get_drvdata(pci_dev);
3487 
3488 	efx->type->fini(efx);
3489 
3490 	efx->reset_pending = 0;
3491 
3492 	pci_save_state(pci_dev);
3493 	return pci_set_power_state(pci_dev, PCI_D3hot);
3494 }
3495 
3496 /* Used for both resume and restore */
3497 static int efx_pm_resume(struct device *dev)
3498 {
3499 	struct pci_dev *pci_dev = to_pci_dev(dev);
3500 	struct efx_nic *efx = pci_get_drvdata(pci_dev);
3501 	int rc;
3502 
3503 	rc = pci_set_power_state(pci_dev, PCI_D0);
3504 	if (rc)
3505 		return rc;
3506 	pci_restore_state(pci_dev);
3507 	rc = pci_enable_device(pci_dev);
3508 	if (rc)
3509 		return rc;
3510 	pci_set_master(efx->pci_dev);
3511 	rc = efx->type->reset(efx, RESET_TYPE_ALL);
3512 	if (rc)
3513 		return rc;
3514 	rc = efx->type->init(efx);
3515 	if (rc)
3516 		return rc;
3517 	rc = efx_pm_thaw(dev);
3518 	return rc;
3519 }
3520 
3521 static int efx_pm_suspend(struct device *dev)
3522 {
3523 	int rc;
3524 
3525 	efx_pm_freeze(dev);
3526 	rc = efx_pm_poweroff(dev);
3527 	if (rc)
3528 		efx_pm_resume(dev);
3529 	return rc;
3530 }
3531 
3532 static const struct dev_pm_ops efx_pm_ops = {
3533 	.suspend	= efx_pm_suspend,
3534 	.resume		= efx_pm_resume,
3535 	.freeze		= efx_pm_freeze,
3536 	.thaw		= efx_pm_thaw,
3537 	.poweroff	= efx_pm_poweroff,
3538 	.restore	= efx_pm_resume,
3539 };
3540 
3541 /* A PCI error affecting this device was detected.
3542  * At this point MMIO and DMA may be disabled.
3543  * Stop the software path and request a slot reset.
3544  */
3545 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev,
3546 					      enum pci_channel_state state)
3547 {
3548 	pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3549 	struct efx_nic *efx = pci_get_drvdata(pdev);
3550 
3551 	if (state == pci_channel_io_perm_failure)
3552 		return PCI_ERS_RESULT_DISCONNECT;
3553 
3554 	rtnl_lock();
3555 
3556 	if (efx->state != STATE_DISABLED) {
3557 		efx->state = STATE_RECOVERY;
3558 		efx->reset_pending = 0;
3559 
3560 		efx_device_detach_sync(efx);
3561 
3562 		efx_stop_all(efx);
3563 		efx_disable_interrupts(efx);
3564 
3565 		status = PCI_ERS_RESULT_NEED_RESET;
3566 	} else {
3567 		/* If the interface is disabled we don't want to do anything
3568 		 * with it.
3569 		 */
3570 		status = PCI_ERS_RESULT_RECOVERED;
3571 	}
3572 
3573 	rtnl_unlock();
3574 
3575 	pci_disable_device(pdev);
3576 
3577 	return status;
3578 }
3579 
3580 /* Fake a successful reset, which will be performed later in efx_io_resume. */
3581 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev)
3582 {
3583 	struct efx_nic *efx = pci_get_drvdata(pdev);
3584 	pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3585 	int rc;
3586 
3587 	if (pci_enable_device(pdev)) {
3588 		netif_err(efx, hw, efx->net_dev,
3589 			  "Cannot re-enable PCI device after reset.\n");
3590 		status =  PCI_ERS_RESULT_DISCONNECT;
3591 	}
3592 
3593 	rc = pci_cleanup_aer_uncorrect_error_status(pdev);
3594 	if (rc) {
3595 		netif_err(efx, hw, efx->net_dev,
3596 		"pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc);
3597 		/* Non-fatal error. Continue. */
3598 	}
3599 
3600 	return status;
3601 }
3602 
3603 /* Perform the actual reset and resume I/O operations. */
3604 static void efx_io_resume(struct pci_dev *pdev)
3605 {
3606 	struct efx_nic *efx = pci_get_drvdata(pdev);
3607 	int rc;
3608 
3609 	rtnl_lock();
3610 
3611 	if (efx->state == STATE_DISABLED)
3612 		goto out;
3613 
3614 	rc = efx_reset(efx, RESET_TYPE_ALL);
3615 	if (rc) {
3616 		netif_err(efx, hw, efx->net_dev,
3617 			  "efx_reset failed after PCI error (%d)\n", rc);
3618 	} else {
3619 		efx->state = STATE_READY;
3620 		netif_dbg(efx, hw, efx->net_dev,
3621 			  "Done resetting and resuming IO after PCI error.\n");
3622 	}
3623 
3624 out:
3625 	rtnl_unlock();
3626 }
3627 
3628 /* For simplicity and reliability, we always require a slot reset and try to
3629  * reset the hardware when a pci error affecting the device is detected.
3630  * We leave both the link_reset and mmio_enabled callback unimplemented:
3631  * with our request for slot reset the mmio_enabled callback will never be
3632  * called, and the link_reset callback is not used by AER or EEH mechanisms.
3633  */
3634 static const struct pci_error_handlers efx_err_handlers = {
3635 	.error_detected = efx_io_error_detected,
3636 	.slot_reset	= efx_io_slot_reset,
3637 	.resume		= efx_io_resume,
3638 };
3639 
3640 static struct pci_driver efx_pci_driver = {
3641 	.name		= KBUILD_MODNAME,
3642 	.id_table	= efx_pci_table,
3643 	.probe		= efx_pci_probe,
3644 	.remove		= efx_pci_remove,
3645 	.driver.pm	= &efx_pm_ops,
3646 	.err_handler	= &efx_err_handlers,
3647 #ifdef CONFIG_SFC_SRIOV
3648 	.sriov_configure = efx_pci_sriov_configure,
3649 #endif
3650 };
3651 
3652 /**************************************************************************
3653  *
3654  * Kernel module interface
3655  *
3656  *************************************************************************/
3657 
3658 module_param(interrupt_mode, uint, 0444);
3659 MODULE_PARM_DESC(interrupt_mode,
3660 		 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
3661 
3662 static int __init efx_init_module(void)
3663 {
3664 	int rc;
3665 
3666 	printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
3667 
3668 	rc = register_netdevice_notifier(&efx_netdev_notifier);
3669 	if (rc)
3670 		goto err_notifier;
3671 
3672 #ifdef CONFIG_SFC_SRIOV
3673 	rc = efx_init_sriov();
3674 	if (rc)
3675 		goto err_sriov;
3676 #endif
3677 
3678 	reset_workqueue = create_singlethread_workqueue("sfc_reset");
3679 	if (!reset_workqueue) {
3680 		rc = -ENOMEM;
3681 		goto err_reset;
3682 	}
3683 
3684 	rc = pci_register_driver(&efx_pci_driver);
3685 	if (rc < 0)
3686 		goto err_pci;
3687 
3688 	return 0;
3689 
3690  err_pci:
3691 	destroy_workqueue(reset_workqueue);
3692  err_reset:
3693 #ifdef CONFIG_SFC_SRIOV
3694 	efx_fini_sriov();
3695  err_sriov:
3696 #endif
3697 	unregister_netdevice_notifier(&efx_netdev_notifier);
3698  err_notifier:
3699 	return rc;
3700 }
3701 
3702 static void __exit efx_exit_module(void)
3703 {
3704 	printk(KERN_INFO "Solarflare NET driver unloading\n");
3705 
3706 	pci_unregister_driver(&efx_pci_driver);
3707 	destroy_workqueue(reset_workqueue);
3708 #ifdef CONFIG_SFC_SRIOV
3709 	efx_fini_sriov();
3710 #endif
3711 	unregister_netdevice_notifier(&efx_netdev_notifier);
3712 
3713 }
3714 
3715 module_init(efx_init_module);
3716 module_exit(efx_exit_module);
3717 
3718 MODULE_AUTHOR("Solarflare Communications and "
3719 	      "Michael Brown <mbrown@fensystems.co.uk>");
3720 MODULE_DESCRIPTION("Solarflare network driver");
3721 MODULE_LICENSE("GPL");
3722 MODULE_DEVICE_TABLE(pci, efx_pci_table);
3723 MODULE_VERSION(EFX_DRIVER_VERSION);
3724