1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include <net/gre.h> 27 #include <net/udp_tunnel.h> 28 #include "efx.h" 29 #include "nic.h" 30 #include "io.h" 31 #include "selftest.h" 32 #include "sriov.h" 33 34 #include "mcdi.h" 35 #include "mcdi_pcol.h" 36 #include "workarounds.h" 37 38 /************************************************************************** 39 * 40 * Type name strings 41 * 42 ************************************************************************** 43 */ 44 45 /* Loopback mode names (see LOOPBACK_MODE()) */ 46 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 47 const char *const efx_loopback_mode_names[] = { 48 [LOOPBACK_NONE] = "NONE", 49 [LOOPBACK_DATA] = "DATAPATH", 50 [LOOPBACK_GMAC] = "GMAC", 51 [LOOPBACK_XGMII] = "XGMII", 52 [LOOPBACK_XGXS] = "XGXS", 53 [LOOPBACK_XAUI] = "XAUI", 54 [LOOPBACK_GMII] = "GMII", 55 [LOOPBACK_SGMII] = "SGMII", 56 [LOOPBACK_XGBR] = "XGBR", 57 [LOOPBACK_XFI] = "XFI", 58 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 59 [LOOPBACK_GMII_FAR] = "GMII_FAR", 60 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 61 [LOOPBACK_XFI_FAR] = "XFI_FAR", 62 [LOOPBACK_GPHY] = "GPHY", 63 [LOOPBACK_PHYXS] = "PHYXS", 64 [LOOPBACK_PCS] = "PCS", 65 [LOOPBACK_PMAPMD] = "PMA/PMD", 66 [LOOPBACK_XPORT] = "XPORT", 67 [LOOPBACK_XGMII_WS] = "XGMII_WS", 68 [LOOPBACK_XAUI_WS] = "XAUI_WS", 69 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 70 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 71 [LOOPBACK_GMII_WS] = "GMII_WS", 72 [LOOPBACK_XFI_WS] = "XFI_WS", 73 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 74 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 75 }; 76 77 const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 78 const char *const efx_reset_type_names[] = { 79 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 80 [RESET_TYPE_ALL] = "ALL", 81 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 82 [RESET_TYPE_WORLD] = "WORLD", 83 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 84 [RESET_TYPE_DATAPATH] = "DATAPATH", 85 [RESET_TYPE_MC_BIST] = "MC_BIST", 86 [RESET_TYPE_DISABLE] = "DISABLE", 87 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 88 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 89 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 90 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 91 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 92 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 93 }; 94 95 /* UDP tunnel type names */ 96 static const char *const efx_udp_tunnel_type_names[] = { 97 [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan", 98 [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve", 99 }; 100 101 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen) 102 { 103 if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) && 104 efx_udp_tunnel_type_names[type] != NULL) 105 snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]); 106 else 107 snprintf(buf, buflen, "type %d", type); 108 } 109 110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 111 * queued onto this work queue. This is not a per-nic work queue, because 112 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 113 */ 114 static struct workqueue_struct *reset_workqueue; 115 116 /* How often and how many times to poll for a reset while waiting for a 117 * BIST that another function started to complete. 118 */ 119 #define BIST_WAIT_DELAY_MS 100 120 #define BIST_WAIT_DELAY_COUNT 100 121 122 /************************************************************************** 123 * 124 * Configurable values 125 * 126 *************************************************************************/ 127 128 /* 129 * Use separate channels for TX and RX events 130 * 131 * Set this to 1 to use separate channels for TX and RX. It allows us 132 * to control interrupt affinity separately for TX and RX. 133 * 134 * This is only used in MSI-X interrupt mode 135 */ 136 bool efx_separate_tx_channels; 137 module_param(efx_separate_tx_channels, bool, 0444); 138 MODULE_PARM_DESC(efx_separate_tx_channels, 139 "Use separate channels for TX and RX"); 140 141 /* This is the weight assigned to each of the (per-channel) virtual 142 * NAPI devices. 143 */ 144 static int napi_weight = 64; 145 146 /* This is the time (in jiffies) between invocations of the hardware 147 * monitor. 148 * On Falcon-based NICs, this will: 149 * - Check the on-board hardware monitor; 150 * - Poll the link state and reconfigure the hardware as necessary. 151 * On Siena-based NICs for power systems with EEH support, this will give EEH a 152 * chance to start. 153 */ 154 static unsigned int efx_monitor_interval = 1 * HZ; 155 156 /* Initial interrupt moderation settings. They can be modified after 157 * module load with ethtool. 158 * 159 * The default for RX should strike a balance between increasing the 160 * round-trip latency and reducing overhead. 161 */ 162 static unsigned int rx_irq_mod_usec = 60; 163 164 /* Initial interrupt moderation settings. They can be modified after 165 * module load with ethtool. 166 * 167 * This default is chosen to ensure that a 10G link does not go idle 168 * while a TX queue is stopped after it has become full. A queue is 169 * restarted when it drops below half full. The time this takes (assuming 170 * worst case 3 descriptors per packet and 1024 descriptors) is 171 * 512 / 3 * 1.2 = 205 usec. 172 */ 173 static unsigned int tx_irq_mod_usec = 150; 174 175 /* This is the first interrupt mode to try out of: 176 * 0 => MSI-X 177 * 1 => MSI 178 * 2 => legacy 179 */ 180 static unsigned int interrupt_mode; 181 182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 183 * i.e. the number of CPUs among which we may distribute simultaneous 184 * interrupt handling. 185 * 186 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 187 * The default (0) means to assign an interrupt to each core. 188 */ 189 static unsigned int rss_cpus; 190 module_param(rss_cpus, uint, 0444); 191 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 192 193 static bool phy_flash_cfg; 194 module_param(phy_flash_cfg, bool, 0644); 195 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 196 197 static unsigned irq_adapt_low_thresh = 8000; 198 module_param(irq_adapt_low_thresh, uint, 0644); 199 MODULE_PARM_DESC(irq_adapt_low_thresh, 200 "Threshold score for reducing IRQ moderation"); 201 202 static unsigned irq_adapt_high_thresh = 16000; 203 module_param(irq_adapt_high_thresh, uint, 0644); 204 MODULE_PARM_DESC(irq_adapt_high_thresh, 205 "Threshold score for increasing IRQ moderation"); 206 207 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 208 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 209 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 210 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 211 module_param(debug, uint, 0); 212 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 213 214 /************************************************************************** 215 * 216 * Utility functions and prototypes 217 * 218 *************************************************************************/ 219 220 static int efx_soft_enable_interrupts(struct efx_nic *efx); 221 static void efx_soft_disable_interrupts(struct efx_nic *efx); 222 static void efx_remove_channel(struct efx_channel *channel); 223 static void efx_remove_channels(struct efx_nic *efx); 224 static const struct efx_channel_type efx_default_channel_type; 225 static void efx_remove_port(struct efx_nic *efx); 226 static void efx_init_napi_channel(struct efx_channel *channel); 227 static void efx_fini_napi(struct efx_nic *efx); 228 static void efx_fini_napi_channel(struct efx_channel *channel); 229 static void efx_fini_struct(struct efx_nic *efx); 230 static void efx_start_all(struct efx_nic *efx); 231 static void efx_stop_all(struct efx_nic *efx); 232 233 #define EFX_ASSERT_RESET_SERIALISED(efx) \ 234 do { \ 235 if ((efx->state == STATE_READY) || \ 236 (efx->state == STATE_RECOVERY) || \ 237 (efx->state == STATE_DISABLED)) \ 238 ASSERT_RTNL(); \ 239 } while (0) 240 241 static int efx_check_disabled(struct efx_nic *efx) 242 { 243 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 244 netif_err(efx, drv, efx->net_dev, 245 "device is disabled due to earlier errors\n"); 246 return -EIO; 247 } 248 return 0; 249 } 250 251 /************************************************************************** 252 * 253 * Event queue processing 254 * 255 *************************************************************************/ 256 257 /* Process channel's event queue 258 * 259 * This function is responsible for processing the event queue of a 260 * single channel. The caller must guarantee that this function will 261 * never be concurrently called more than once on the same channel, 262 * though different channels may be being processed concurrently. 263 */ 264 static int efx_process_channel(struct efx_channel *channel, int budget) 265 { 266 struct efx_tx_queue *tx_queue; 267 int spent; 268 269 if (unlikely(!channel->enabled)) 270 return 0; 271 272 efx_for_each_channel_tx_queue(tx_queue, channel) { 273 tx_queue->pkts_compl = 0; 274 tx_queue->bytes_compl = 0; 275 } 276 277 spent = efx_nic_process_eventq(channel, budget); 278 if (spent && efx_channel_has_rx_queue(channel)) { 279 struct efx_rx_queue *rx_queue = 280 efx_channel_get_rx_queue(channel); 281 282 efx_rx_flush_packet(channel); 283 efx_fast_push_rx_descriptors(rx_queue, true); 284 } 285 286 /* Update BQL */ 287 efx_for_each_channel_tx_queue(tx_queue, channel) { 288 if (tx_queue->bytes_compl) { 289 netdev_tx_completed_queue(tx_queue->core_txq, 290 tx_queue->pkts_compl, tx_queue->bytes_compl); 291 } 292 } 293 294 return spent; 295 } 296 297 /* NAPI poll handler 298 * 299 * NAPI guarantees serialisation of polls of the same device, which 300 * provides the guarantee required by efx_process_channel(). 301 */ 302 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 303 { 304 int step = efx->irq_mod_step_us; 305 306 if (channel->irq_mod_score < irq_adapt_low_thresh) { 307 if (channel->irq_moderation_us > step) { 308 channel->irq_moderation_us -= step; 309 efx->type->push_irq_moderation(channel); 310 } 311 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 312 if (channel->irq_moderation_us < 313 efx->irq_rx_moderation_us) { 314 channel->irq_moderation_us += step; 315 efx->type->push_irq_moderation(channel); 316 } 317 } 318 319 channel->irq_count = 0; 320 channel->irq_mod_score = 0; 321 } 322 323 static int efx_poll(struct napi_struct *napi, int budget) 324 { 325 struct efx_channel *channel = 326 container_of(napi, struct efx_channel, napi_str); 327 struct efx_nic *efx = channel->efx; 328 int spent; 329 330 netif_vdbg(efx, intr, efx->net_dev, 331 "channel %d NAPI poll executing on CPU %d\n", 332 channel->channel, raw_smp_processor_id()); 333 334 spent = efx_process_channel(channel, budget); 335 336 if (spent < budget) { 337 if (efx_channel_has_rx_queue(channel) && 338 efx->irq_rx_adaptive && 339 unlikely(++channel->irq_count == 1000)) { 340 efx_update_irq_mod(efx, channel); 341 } 342 343 #ifdef CONFIG_RFS_ACCEL 344 /* Perhaps expire some ARFS filters */ 345 schedule_work(&channel->filter_work); 346 #endif 347 348 /* There is no race here; although napi_disable() will 349 * only wait for napi_complete(), this isn't a problem 350 * since efx_nic_eventq_read_ack() will have no effect if 351 * interrupts have already been disabled. 352 */ 353 if (napi_complete_done(napi, spent)) 354 efx_nic_eventq_read_ack(channel); 355 } 356 357 return spent; 358 } 359 360 /* Create event queue 361 * Event queue memory allocations are done only once. If the channel 362 * is reset, the memory buffer will be reused; this guards against 363 * errors during channel reset and also simplifies interrupt handling. 364 */ 365 static int efx_probe_eventq(struct efx_channel *channel) 366 { 367 struct efx_nic *efx = channel->efx; 368 unsigned long entries; 369 370 netif_dbg(efx, probe, efx->net_dev, 371 "chan %d create event queue\n", channel->channel); 372 373 /* Build an event queue with room for one event per tx and rx buffer, 374 * plus some extra for link state events and MCDI completions. */ 375 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 376 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 377 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 378 379 return efx_nic_probe_eventq(channel); 380 } 381 382 /* Prepare channel's event queue */ 383 static int efx_init_eventq(struct efx_channel *channel) 384 { 385 struct efx_nic *efx = channel->efx; 386 int rc; 387 388 EFX_WARN_ON_PARANOID(channel->eventq_init); 389 390 netif_dbg(efx, drv, efx->net_dev, 391 "chan %d init event queue\n", channel->channel); 392 393 rc = efx_nic_init_eventq(channel); 394 if (rc == 0) { 395 efx->type->push_irq_moderation(channel); 396 channel->eventq_read_ptr = 0; 397 channel->eventq_init = true; 398 } 399 return rc; 400 } 401 402 /* Enable event queue processing and NAPI */ 403 void efx_start_eventq(struct efx_channel *channel) 404 { 405 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 406 "chan %d start event queue\n", channel->channel); 407 408 /* Make sure the NAPI handler sees the enabled flag set */ 409 channel->enabled = true; 410 smp_wmb(); 411 412 napi_enable(&channel->napi_str); 413 efx_nic_eventq_read_ack(channel); 414 } 415 416 /* Disable event queue processing and NAPI */ 417 void efx_stop_eventq(struct efx_channel *channel) 418 { 419 if (!channel->enabled) 420 return; 421 422 napi_disable(&channel->napi_str); 423 channel->enabled = false; 424 } 425 426 static void efx_fini_eventq(struct efx_channel *channel) 427 { 428 if (!channel->eventq_init) 429 return; 430 431 netif_dbg(channel->efx, drv, channel->efx->net_dev, 432 "chan %d fini event queue\n", channel->channel); 433 434 efx_nic_fini_eventq(channel); 435 channel->eventq_init = false; 436 } 437 438 static void efx_remove_eventq(struct efx_channel *channel) 439 { 440 netif_dbg(channel->efx, drv, channel->efx->net_dev, 441 "chan %d remove event queue\n", channel->channel); 442 443 efx_nic_remove_eventq(channel); 444 } 445 446 /************************************************************************** 447 * 448 * Channel handling 449 * 450 *************************************************************************/ 451 452 /* Allocate and initialise a channel structure. */ 453 static struct efx_channel * 454 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 455 { 456 struct efx_channel *channel; 457 struct efx_rx_queue *rx_queue; 458 struct efx_tx_queue *tx_queue; 459 int j; 460 461 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 462 if (!channel) 463 return NULL; 464 465 channel->efx = efx; 466 channel->channel = i; 467 channel->type = &efx_default_channel_type; 468 469 for (j = 0; j < EFX_TXQ_TYPES; j++) { 470 tx_queue = &channel->tx_queue[j]; 471 tx_queue->efx = efx; 472 tx_queue->queue = i * EFX_TXQ_TYPES + j; 473 tx_queue->channel = channel; 474 } 475 476 #ifdef CONFIG_RFS_ACCEL 477 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 478 #endif 479 480 rx_queue = &channel->rx_queue; 481 rx_queue->efx = efx; 482 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 483 484 return channel; 485 } 486 487 /* Allocate and initialise a channel structure, copying parameters 488 * (but not resources) from an old channel structure. 489 */ 490 static struct efx_channel * 491 efx_copy_channel(const struct efx_channel *old_channel) 492 { 493 struct efx_channel *channel; 494 struct efx_rx_queue *rx_queue; 495 struct efx_tx_queue *tx_queue; 496 int j; 497 498 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 499 if (!channel) 500 return NULL; 501 502 *channel = *old_channel; 503 504 channel->napi_dev = NULL; 505 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 506 channel->napi_str.napi_id = 0; 507 channel->napi_str.state = 0; 508 memset(&channel->eventq, 0, sizeof(channel->eventq)); 509 510 for (j = 0; j < EFX_TXQ_TYPES; j++) { 511 tx_queue = &channel->tx_queue[j]; 512 if (tx_queue->channel) 513 tx_queue->channel = channel; 514 tx_queue->buffer = NULL; 515 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 516 } 517 518 rx_queue = &channel->rx_queue; 519 rx_queue->buffer = NULL; 520 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 521 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 522 #ifdef CONFIG_RFS_ACCEL 523 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 524 #endif 525 526 return channel; 527 } 528 529 static int efx_probe_channel(struct efx_channel *channel) 530 { 531 struct efx_tx_queue *tx_queue; 532 struct efx_rx_queue *rx_queue; 533 int rc; 534 535 netif_dbg(channel->efx, probe, channel->efx->net_dev, 536 "creating channel %d\n", channel->channel); 537 538 rc = channel->type->pre_probe(channel); 539 if (rc) 540 goto fail; 541 542 rc = efx_probe_eventq(channel); 543 if (rc) 544 goto fail; 545 546 efx_for_each_channel_tx_queue(tx_queue, channel) { 547 rc = efx_probe_tx_queue(tx_queue); 548 if (rc) 549 goto fail; 550 } 551 552 efx_for_each_channel_rx_queue(rx_queue, channel) { 553 rc = efx_probe_rx_queue(rx_queue); 554 if (rc) 555 goto fail; 556 } 557 558 return 0; 559 560 fail: 561 efx_remove_channel(channel); 562 return rc; 563 } 564 565 static void 566 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 567 { 568 struct efx_nic *efx = channel->efx; 569 const char *type; 570 int number; 571 572 number = channel->channel; 573 if (efx->tx_channel_offset == 0) { 574 type = ""; 575 } else if (channel->channel < efx->tx_channel_offset) { 576 type = "-rx"; 577 } else { 578 type = "-tx"; 579 number -= efx->tx_channel_offset; 580 } 581 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 582 } 583 584 static void efx_set_channel_names(struct efx_nic *efx) 585 { 586 struct efx_channel *channel; 587 588 efx_for_each_channel(channel, efx) 589 channel->type->get_name(channel, 590 efx->msi_context[channel->channel].name, 591 sizeof(efx->msi_context[0].name)); 592 } 593 594 static int efx_probe_channels(struct efx_nic *efx) 595 { 596 struct efx_channel *channel; 597 int rc; 598 599 /* Restart special buffer allocation */ 600 efx->next_buffer_table = 0; 601 602 /* Probe channels in reverse, so that any 'extra' channels 603 * use the start of the buffer table. This allows the traffic 604 * channels to be resized without moving them or wasting the 605 * entries before them. 606 */ 607 efx_for_each_channel_rev(channel, efx) { 608 rc = efx_probe_channel(channel); 609 if (rc) { 610 netif_err(efx, probe, efx->net_dev, 611 "failed to create channel %d\n", 612 channel->channel); 613 goto fail; 614 } 615 } 616 efx_set_channel_names(efx); 617 618 return 0; 619 620 fail: 621 efx_remove_channels(efx); 622 return rc; 623 } 624 625 /* Channels are shutdown and reinitialised whilst the NIC is running 626 * to propagate configuration changes (mtu, checksum offload), or 627 * to clear hardware error conditions 628 */ 629 static void efx_start_datapath(struct efx_nic *efx) 630 { 631 netdev_features_t old_features = efx->net_dev->features; 632 bool old_rx_scatter = efx->rx_scatter; 633 struct efx_tx_queue *tx_queue; 634 struct efx_rx_queue *rx_queue; 635 struct efx_channel *channel; 636 size_t rx_buf_len; 637 638 /* Calculate the rx buffer allocation parameters required to 639 * support the current MTU, including padding for header 640 * alignment and overruns. 641 */ 642 efx->rx_dma_len = (efx->rx_prefix_size + 643 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 644 efx->type->rx_buffer_padding); 645 rx_buf_len = (sizeof(struct efx_rx_page_state) + 646 efx->rx_ip_align + efx->rx_dma_len); 647 if (rx_buf_len <= PAGE_SIZE) { 648 efx->rx_scatter = efx->type->always_rx_scatter; 649 efx->rx_buffer_order = 0; 650 } else if (efx->type->can_rx_scatter) { 651 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 652 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 653 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 654 EFX_RX_BUF_ALIGNMENT) > 655 PAGE_SIZE); 656 efx->rx_scatter = true; 657 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 658 efx->rx_buffer_order = 0; 659 } else { 660 efx->rx_scatter = false; 661 efx->rx_buffer_order = get_order(rx_buf_len); 662 } 663 664 efx_rx_config_page_split(efx); 665 if (efx->rx_buffer_order) 666 netif_dbg(efx, drv, efx->net_dev, 667 "RX buf len=%u; page order=%u batch=%u\n", 668 efx->rx_dma_len, efx->rx_buffer_order, 669 efx->rx_pages_per_batch); 670 else 671 netif_dbg(efx, drv, efx->net_dev, 672 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 673 efx->rx_dma_len, efx->rx_page_buf_step, 674 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 675 676 /* Restore previously fixed features in hw_features and remove 677 * features which are fixed now 678 */ 679 efx->net_dev->hw_features |= efx->net_dev->features; 680 efx->net_dev->hw_features &= ~efx->fixed_features; 681 efx->net_dev->features |= efx->fixed_features; 682 if (efx->net_dev->features != old_features) 683 netdev_features_change(efx->net_dev); 684 685 /* RX filters may also have scatter-enabled flags */ 686 if (efx->rx_scatter != old_rx_scatter) 687 efx->type->filter_update_rx_scatter(efx); 688 689 /* We must keep at least one descriptor in a TX ring empty. 690 * We could avoid this when the queue size does not exactly 691 * match the hardware ring size, but it's not that important. 692 * Therefore we stop the queue when one more skb might fill 693 * the ring completely. We wake it when half way back to 694 * empty. 695 */ 696 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 697 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 698 699 /* Initialise the channels */ 700 efx_for_each_channel(channel, efx) { 701 efx_for_each_channel_tx_queue(tx_queue, channel) { 702 efx_init_tx_queue(tx_queue); 703 atomic_inc(&efx->active_queues); 704 } 705 706 efx_for_each_channel_rx_queue(rx_queue, channel) { 707 efx_init_rx_queue(rx_queue); 708 atomic_inc(&efx->active_queues); 709 efx_stop_eventq(channel); 710 efx_fast_push_rx_descriptors(rx_queue, false); 711 efx_start_eventq(channel); 712 } 713 714 WARN_ON(channel->rx_pkt_n_frags); 715 } 716 717 efx_ptp_start_datapath(efx); 718 719 if (netif_device_present(efx->net_dev)) 720 netif_tx_wake_all_queues(efx->net_dev); 721 } 722 723 static void efx_stop_datapath(struct efx_nic *efx) 724 { 725 struct efx_channel *channel; 726 struct efx_tx_queue *tx_queue; 727 struct efx_rx_queue *rx_queue; 728 int rc; 729 730 EFX_ASSERT_RESET_SERIALISED(efx); 731 BUG_ON(efx->port_enabled); 732 733 efx_ptp_stop_datapath(efx); 734 735 /* Stop RX refill */ 736 efx_for_each_channel(channel, efx) { 737 efx_for_each_channel_rx_queue(rx_queue, channel) 738 rx_queue->refill_enabled = false; 739 } 740 741 efx_for_each_channel(channel, efx) { 742 /* RX packet processing is pipelined, so wait for the 743 * NAPI handler to complete. At least event queue 0 744 * might be kept active by non-data events, so don't 745 * use napi_synchronize() but actually disable NAPI 746 * temporarily. 747 */ 748 if (efx_channel_has_rx_queue(channel)) { 749 efx_stop_eventq(channel); 750 efx_start_eventq(channel); 751 } 752 } 753 754 rc = efx->type->fini_dmaq(efx); 755 if (rc) { 756 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 757 } else { 758 netif_dbg(efx, drv, efx->net_dev, 759 "successfully flushed all queues\n"); 760 } 761 762 efx_for_each_channel(channel, efx) { 763 efx_for_each_channel_rx_queue(rx_queue, channel) 764 efx_fini_rx_queue(rx_queue); 765 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 766 efx_fini_tx_queue(tx_queue); 767 } 768 } 769 770 static void efx_remove_channel(struct efx_channel *channel) 771 { 772 struct efx_tx_queue *tx_queue; 773 struct efx_rx_queue *rx_queue; 774 775 netif_dbg(channel->efx, drv, channel->efx->net_dev, 776 "destroy chan %d\n", channel->channel); 777 778 efx_for_each_channel_rx_queue(rx_queue, channel) 779 efx_remove_rx_queue(rx_queue); 780 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 781 efx_remove_tx_queue(tx_queue); 782 efx_remove_eventq(channel); 783 channel->type->post_remove(channel); 784 } 785 786 static void efx_remove_channels(struct efx_nic *efx) 787 { 788 struct efx_channel *channel; 789 790 efx_for_each_channel(channel, efx) 791 efx_remove_channel(channel); 792 } 793 794 int 795 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 796 { 797 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 798 u32 old_rxq_entries, old_txq_entries; 799 unsigned i, next_buffer_table = 0; 800 int rc, rc2; 801 802 rc = efx_check_disabled(efx); 803 if (rc) 804 return rc; 805 806 /* Not all channels should be reallocated. We must avoid 807 * reallocating their buffer table entries. 808 */ 809 efx_for_each_channel(channel, efx) { 810 struct efx_rx_queue *rx_queue; 811 struct efx_tx_queue *tx_queue; 812 813 if (channel->type->copy) 814 continue; 815 next_buffer_table = max(next_buffer_table, 816 channel->eventq.index + 817 channel->eventq.entries); 818 efx_for_each_channel_rx_queue(rx_queue, channel) 819 next_buffer_table = max(next_buffer_table, 820 rx_queue->rxd.index + 821 rx_queue->rxd.entries); 822 efx_for_each_channel_tx_queue(tx_queue, channel) 823 next_buffer_table = max(next_buffer_table, 824 tx_queue->txd.index + 825 tx_queue->txd.entries); 826 } 827 828 efx_device_detach_sync(efx); 829 efx_stop_all(efx); 830 efx_soft_disable_interrupts(efx); 831 832 /* Clone channels (where possible) */ 833 memset(other_channel, 0, sizeof(other_channel)); 834 for (i = 0; i < efx->n_channels; i++) { 835 channel = efx->channel[i]; 836 if (channel->type->copy) 837 channel = channel->type->copy(channel); 838 if (!channel) { 839 rc = -ENOMEM; 840 goto out; 841 } 842 other_channel[i] = channel; 843 } 844 845 /* Swap entry counts and channel pointers */ 846 old_rxq_entries = efx->rxq_entries; 847 old_txq_entries = efx->txq_entries; 848 efx->rxq_entries = rxq_entries; 849 efx->txq_entries = txq_entries; 850 for (i = 0; i < efx->n_channels; i++) { 851 channel = efx->channel[i]; 852 efx->channel[i] = other_channel[i]; 853 other_channel[i] = channel; 854 } 855 856 /* Restart buffer table allocation */ 857 efx->next_buffer_table = next_buffer_table; 858 859 for (i = 0; i < efx->n_channels; i++) { 860 channel = efx->channel[i]; 861 if (!channel->type->copy) 862 continue; 863 rc = efx_probe_channel(channel); 864 if (rc) 865 goto rollback; 866 efx_init_napi_channel(efx->channel[i]); 867 } 868 869 out: 870 /* Destroy unused channel structures */ 871 for (i = 0; i < efx->n_channels; i++) { 872 channel = other_channel[i]; 873 if (channel && channel->type->copy) { 874 efx_fini_napi_channel(channel); 875 efx_remove_channel(channel); 876 kfree(channel); 877 } 878 } 879 880 rc2 = efx_soft_enable_interrupts(efx); 881 if (rc2) { 882 rc = rc ? rc : rc2; 883 netif_err(efx, drv, efx->net_dev, 884 "unable to restart interrupts on channel reallocation\n"); 885 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 886 } else { 887 efx_start_all(efx); 888 efx_device_attach_if_not_resetting(efx); 889 } 890 return rc; 891 892 rollback: 893 /* Swap back */ 894 efx->rxq_entries = old_rxq_entries; 895 efx->txq_entries = old_txq_entries; 896 for (i = 0; i < efx->n_channels; i++) { 897 channel = efx->channel[i]; 898 efx->channel[i] = other_channel[i]; 899 other_channel[i] = channel; 900 } 901 goto out; 902 } 903 904 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 905 { 906 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 907 } 908 909 static bool efx_default_channel_want_txqs(struct efx_channel *channel) 910 { 911 return channel->channel - channel->efx->tx_channel_offset < 912 channel->efx->n_tx_channels; 913 } 914 915 static const struct efx_channel_type efx_default_channel_type = { 916 .pre_probe = efx_channel_dummy_op_int, 917 .post_remove = efx_channel_dummy_op_void, 918 .get_name = efx_get_channel_name, 919 .copy = efx_copy_channel, 920 .want_txqs = efx_default_channel_want_txqs, 921 .keep_eventq = false, 922 .want_pio = true, 923 }; 924 925 int efx_channel_dummy_op_int(struct efx_channel *channel) 926 { 927 return 0; 928 } 929 930 void efx_channel_dummy_op_void(struct efx_channel *channel) 931 { 932 } 933 934 /************************************************************************** 935 * 936 * Port handling 937 * 938 **************************************************************************/ 939 940 /* This ensures that the kernel is kept informed (via 941 * netif_carrier_on/off) of the link status, and also maintains the 942 * link status's stop on the port's TX queue. 943 */ 944 void efx_link_status_changed(struct efx_nic *efx) 945 { 946 struct efx_link_state *link_state = &efx->link_state; 947 948 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 949 * that no events are triggered between unregister_netdev() and the 950 * driver unloading. A more general condition is that NETDEV_CHANGE 951 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 952 if (!netif_running(efx->net_dev)) 953 return; 954 955 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 956 efx->n_link_state_changes++; 957 958 if (link_state->up) 959 netif_carrier_on(efx->net_dev); 960 else 961 netif_carrier_off(efx->net_dev); 962 } 963 964 /* Status message for kernel log */ 965 if (link_state->up) 966 netif_info(efx, link, efx->net_dev, 967 "link up at %uMbps %s-duplex (MTU %d)\n", 968 link_state->speed, link_state->fd ? "full" : "half", 969 efx->net_dev->mtu); 970 else 971 netif_info(efx, link, efx->net_dev, "link down\n"); 972 } 973 974 void efx_link_set_advertising(struct efx_nic *efx, 975 const unsigned long *advertising) 976 { 977 memcpy(efx->link_advertising, advertising, 978 sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK())); 979 980 efx->link_advertising[0] |= ADVERTISED_Autoneg; 981 if (advertising[0] & ADVERTISED_Pause) 982 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 983 else 984 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 985 if (advertising[0] & ADVERTISED_Asym_Pause) 986 efx->wanted_fc ^= EFX_FC_TX; 987 } 988 989 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not 990 * force the Autoneg bit on. 991 */ 992 void efx_link_clear_advertising(struct efx_nic *efx) 993 { 994 bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS); 995 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 996 } 997 998 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 999 { 1000 efx->wanted_fc = wanted_fc; 1001 if (efx->link_advertising[0]) { 1002 if (wanted_fc & EFX_FC_RX) 1003 efx->link_advertising[0] |= (ADVERTISED_Pause | 1004 ADVERTISED_Asym_Pause); 1005 else 1006 efx->link_advertising[0] &= ~(ADVERTISED_Pause | 1007 ADVERTISED_Asym_Pause); 1008 if (wanted_fc & EFX_FC_TX) 1009 efx->link_advertising[0] ^= ADVERTISED_Asym_Pause; 1010 } 1011 } 1012 1013 static void efx_fini_port(struct efx_nic *efx); 1014 1015 /* We assume that efx->type->reconfigure_mac will always try to sync RX 1016 * filters and therefore needs to read-lock the filter table against freeing 1017 */ 1018 void efx_mac_reconfigure(struct efx_nic *efx) 1019 { 1020 down_read(&efx->filter_sem); 1021 efx->type->reconfigure_mac(efx); 1022 up_read(&efx->filter_sem); 1023 } 1024 1025 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 1026 * the MAC appropriately. All other PHY configuration changes are pushed 1027 * through phy_op->set_settings(), and pushed asynchronously to the MAC 1028 * through efx_monitor(). 1029 * 1030 * Callers must hold the mac_lock 1031 */ 1032 int __efx_reconfigure_port(struct efx_nic *efx) 1033 { 1034 enum efx_phy_mode phy_mode; 1035 int rc; 1036 1037 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1038 1039 /* Disable PHY transmit in mac level loopbacks */ 1040 phy_mode = efx->phy_mode; 1041 if (LOOPBACK_INTERNAL(efx)) 1042 efx->phy_mode |= PHY_MODE_TX_DISABLED; 1043 else 1044 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1045 1046 rc = efx->type->reconfigure_port(efx); 1047 1048 if (rc) 1049 efx->phy_mode = phy_mode; 1050 1051 return rc; 1052 } 1053 1054 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1055 * disabled. */ 1056 int efx_reconfigure_port(struct efx_nic *efx) 1057 { 1058 int rc; 1059 1060 EFX_ASSERT_RESET_SERIALISED(efx); 1061 1062 mutex_lock(&efx->mac_lock); 1063 rc = __efx_reconfigure_port(efx); 1064 mutex_unlock(&efx->mac_lock); 1065 1066 return rc; 1067 } 1068 1069 /* Asynchronous work item for changing MAC promiscuity and multicast 1070 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1071 * MAC directly. */ 1072 static void efx_mac_work(struct work_struct *data) 1073 { 1074 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1075 1076 mutex_lock(&efx->mac_lock); 1077 if (efx->port_enabled) 1078 efx_mac_reconfigure(efx); 1079 mutex_unlock(&efx->mac_lock); 1080 } 1081 1082 static int efx_probe_port(struct efx_nic *efx) 1083 { 1084 int rc; 1085 1086 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1087 1088 if (phy_flash_cfg) 1089 efx->phy_mode = PHY_MODE_SPECIAL; 1090 1091 /* Connect up MAC/PHY operations table */ 1092 rc = efx->type->probe_port(efx); 1093 if (rc) 1094 return rc; 1095 1096 /* Initialise MAC address to permanent address */ 1097 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1098 1099 return 0; 1100 } 1101 1102 static int efx_init_port(struct efx_nic *efx) 1103 { 1104 int rc; 1105 1106 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1107 1108 mutex_lock(&efx->mac_lock); 1109 1110 rc = efx->phy_op->init(efx); 1111 if (rc) 1112 goto fail1; 1113 1114 efx->port_initialized = true; 1115 1116 /* Reconfigure the MAC before creating dma queues (required for 1117 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1118 efx_mac_reconfigure(efx); 1119 1120 /* Ensure the PHY advertises the correct flow control settings */ 1121 rc = efx->phy_op->reconfigure(efx); 1122 if (rc && rc != -EPERM) 1123 goto fail2; 1124 1125 mutex_unlock(&efx->mac_lock); 1126 return 0; 1127 1128 fail2: 1129 efx->phy_op->fini(efx); 1130 fail1: 1131 mutex_unlock(&efx->mac_lock); 1132 return rc; 1133 } 1134 1135 static void efx_start_port(struct efx_nic *efx) 1136 { 1137 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1138 BUG_ON(efx->port_enabled); 1139 1140 mutex_lock(&efx->mac_lock); 1141 efx->port_enabled = true; 1142 1143 /* Ensure MAC ingress/egress is enabled */ 1144 efx_mac_reconfigure(efx); 1145 1146 mutex_unlock(&efx->mac_lock); 1147 } 1148 1149 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1150 * and the async self-test, wait for them to finish and prevent them 1151 * being scheduled again. This doesn't cover online resets, which 1152 * should only be cancelled when removing the device. 1153 */ 1154 static void efx_stop_port(struct efx_nic *efx) 1155 { 1156 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1157 1158 EFX_ASSERT_RESET_SERIALISED(efx); 1159 1160 mutex_lock(&efx->mac_lock); 1161 efx->port_enabled = false; 1162 mutex_unlock(&efx->mac_lock); 1163 1164 /* Serialise against efx_set_multicast_list() */ 1165 netif_addr_lock_bh(efx->net_dev); 1166 netif_addr_unlock_bh(efx->net_dev); 1167 1168 cancel_delayed_work_sync(&efx->monitor_work); 1169 efx_selftest_async_cancel(efx); 1170 cancel_work_sync(&efx->mac_work); 1171 } 1172 1173 static void efx_fini_port(struct efx_nic *efx) 1174 { 1175 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1176 1177 if (!efx->port_initialized) 1178 return; 1179 1180 efx->phy_op->fini(efx); 1181 efx->port_initialized = false; 1182 1183 efx->link_state.up = false; 1184 efx_link_status_changed(efx); 1185 } 1186 1187 static void efx_remove_port(struct efx_nic *efx) 1188 { 1189 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1190 1191 efx->type->remove_port(efx); 1192 } 1193 1194 /************************************************************************** 1195 * 1196 * NIC handling 1197 * 1198 **************************************************************************/ 1199 1200 static LIST_HEAD(efx_primary_list); 1201 static LIST_HEAD(efx_unassociated_list); 1202 1203 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 1204 { 1205 return left->type == right->type && 1206 left->vpd_sn && right->vpd_sn && 1207 !strcmp(left->vpd_sn, right->vpd_sn); 1208 } 1209 1210 static void efx_associate(struct efx_nic *efx) 1211 { 1212 struct efx_nic *other, *next; 1213 1214 if (efx->primary == efx) { 1215 /* Adding primary function; look for secondaries */ 1216 1217 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1218 list_add_tail(&efx->node, &efx_primary_list); 1219 1220 list_for_each_entry_safe(other, next, &efx_unassociated_list, 1221 node) { 1222 if (efx_same_controller(efx, other)) { 1223 list_del(&other->node); 1224 netif_dbg(other, probe, other->net_dev, 1225 "moving to secondary list of %s %s\n", 1226 pci_name(efx->pci_dev), 1227 efx->net_dev->name); 1228 list_add_tail(&other->node, 1229 &efx->secondary_list); 1230 other->primary = efx; 1231 } 1232 } 1233 } else { 1234 /* Adding secondary function; look for primary */ 1235 1236 list_for_each_entry(other, &efx_primary_list, node) { 1237 if (efx_same_controller(efx, other)) { 1238 netif_dbg(efx, probe, efx->net_dev, 1239 "adding to secondary list of %s %s\n", 1240 pci_name(other->pci_dev), 1241 other->net_dev->name); 1242 list_add_tail(&efx->node, 1243 &other->secondary_list); 1244 efx->primary = other; 1245 return; 1246 } 1247 } 1248 1249 netif_dbg(efx, probe, efx->net_dev, 1250 "adding to unassociated list\n"); 1251 list_add_tail(&efx->node, &efx_unassociated_list); 1252 } 1253 } 1254 1255 static void efx_dissociate(struct efx_nic *efx) 1256 { 1257 struct efx_nic *other, *next; 1258 1259 list_del(&efx->node); 1260 efx->primary = NULL; 1261 1262 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1263 list_del(&other->node); 1264 netif_dbg(other, probe, other->net_dev, 1265 "moving to unassociated list\n"); 1266 list_add_tail(&other->node, &efx_unassociated_list); 1267 other->primary = NULL; 1268 } 1269 } 1270 1271 /* This configures the PCI device to enable I/O and DMA. */ 1272 static int efx_init_io(struct efx_nic *efx) 1273 { 1274 struct pci_dev *pci_dev = efx->pci_dev; 1275 dma_addr_t dma_mask = efx->type->max_dma_mask; 1276 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1277 int rc, bar; 1278 1279 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1280 1281 bar = efx->type->mem_bar(efx); 1282 1283 rc = pci_enable_device(pci_dev); 1284 if (rc) { 1285 netif_err(efx, probe, efx->net_dev, 1286 "failed to enable PCI device\n"); 1287 goto fail1; 1288 } 1289 1290 pci_set_master(pci_dev); 1291 1292 /* Set the PCI DMA mask. Try all possibilities from our 1293 * genuine mask down to 32 bits, because some architectures 1294 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1295 * masks event though they reject 46 bit masks. 1296 */ 1297 while (dma_mask > 0x7fffffffUL) { 1298 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1299 if (rc == 0) 1300 break; 1301 dma_mask >>= 1; 1302 } 1303 if (rc) { 1304 netif_err(efx, probe, efx->net_dev, 1305 "could not find a suitable DMA mask\n"); 1306 goto fail2; 1307 } 1308 netif_dbg(efx, probe, efx->net_dev, 1309 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1310 1311 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1312 rc = pci_request_region(pci_dev, bar, "sfc"); 1313 if (rc) { 1314 netif_err(efx, probe, efx->net_dev, 1315 "request for memory BAR failed\n"); 1316 rc = -EIO; 1317 goto fail3; 1318 } 1319 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1320 if (!efx->membase) { 1321 netif_err(efx, probe, efx->net_dev, 1322 "could not map memory BAR at %llx+%x\n", 1323 (unsigned long long)efx->membase_phys, mem_map_size); 1324 rc = -ENOMEM; 1325 goto fail4; 1326 } 1327 netif_dbg(efx, probe, efx->net_dev, 1328 "memory BAR at %llx+%x (virtual %p)\n", 1329 (unsigned long long)efx->membase_phys, mem_map_size, 1330 efx->membase); 1331 1332 return 0; 1333 1334 fail4: 1335 pci_release_region(efx->pci_dev, bar); 1336 fail3: 1337 efx->membase_phys = 0; 1338 fail2: 1339 pci_disable_device(efx->pci_dev); 1340 fail1: 1341 return rc; 1342 } 1343 1344 static void efx_fini_io(struct efx_nic *efx) 1345 { 1346 int bar; 1347 1348 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1349 1350 if (efx->membase) { 1351 iounmap(efx->membase); 1352 efx->membase = NULL; 1353 } 1354 1355 if (efx->membase_phys) { 1356 bar = efx->type->mem_bar(efx); 1357 pci_release_region(efx->pci_dev, bar); 1358 efx->membase_phys = 0; 1359 } 1360 1361 /* Don't disable bus-mastering if VFs are assigned */ 1362 if (!pci_vfs_assigned(efx->pci_dev)) 1363 pci_disable_device(efx->pci_dev); 1364 } 1365 1366 void efx_set_default_rx_indir_table(struct efx_nic *efx, 1367 struct efx_rss_context *ctx) 1368 { 1369 size_t i; 1370 1371 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 1372 ctx->rx_indir_table[i] = 1373 ethtool_rxfh_indir_default(i, efx->rss_spread); 1374 } 1375 1376 static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1377 { 1378 cpumask_var_t thread_mask; 1379 unsigned int count; 1380 int cpu; 1381 1382 if (rss_cpus) { 1383 count = rss_cpus; 1384 } else { 1385 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1386 netif_warn(efx, probe, efx->net_dev, 1387 "RSS disabled due to allocation failure\n"); 1388 return 1; 1389 } 1390 1391 count = 0; 1392 for_each_online_cpu(cpu) { 1393 if (!cpumask_test_cpu(cpu, thread_mask)) { 1394 ++count; 1395 cpumask_or(thread_mask, thread_mask, 1396 topology_sibling_cpumask(cpu)); 1397 } 1398 } 1399 1400 free_cpumask_var(thread_mask); 1401 } 1402 1403 if (count > EFX_MAX_RX_QUEUES) { 1404 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1405 "Reducing number of rx queues from %u to %u.\n", 1406 count, EFX_MAX_RX_QUEUES); 1407 count = EFX_MAX_RX_QUEUES; 1408 } 1409 1410 /* If RSS is requested for the PF *and* VFs then we can't write RSS 1411 * table entries that are inaccessible to VFs 1412 */ 1413 #ifdef CONFIG_SFC_SRIOV 1414 if (efx->type->sriov_wanted) { 1415 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1416 count > efx_vf_size(efx)) { 1417 netif_warn(efx, probe, efx->net_dev, 1418 "Reducing number of RSS channels from %u to %u for " 1419 "VF support. Increase vf-msix-limit to use more " 1420 "channels on the PF.\n", 1421 count, efx_vf_size(efx)); 1422 count = efx_vf_size(efx); 1423 } 1424 } 1425 #endif 1426 1427 return count; 1428 } 1429 1430 /* Probe the number and type of interrupts we are able to obtain, and 1431 * the resulting numbers of channels and RX queues. 1432 */ 1433 static int efx_probe_interrupts(struct efx_nic *efx) 1434 { 1435 unsigned int extra_channels = 0; 1436 unsigned int i, j; 1437 int rc; 1438 1439 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 1440 if (efx->extra_channel_type[i]) 1441 ++extra_channels; 1442 1443 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1444 struct msix_entry xentries[EFX_MAX_CHANNELS]; 1445 unsigned int n_channels; 1446 1447 n_channels = efx_wanted_parallelism(efx); 1448 if (efx_separate_tx_channels) 1449 n_channels *= 2; 1450 n_channels += extra_channels; 1451 n_channels = min(n_channels, efx->max_channels); 1452 1453 for (i = 0; i < n_channels; i++) 1454 xentries[i].entry = i; 1455 rc = pci_enable_msix_range(efx->pci_dev, 1456 xentries, 1, n_channels); 1457 if (rc < 0) { 1458 /* Fall back to single channel MSI */ 1459 netif_err(efx, drv, efx->net_dev, 1460 "could not enable MSI-X\n"); 1461 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 1462 efx->interrupt_mode = EFX_INT_MODE_MSI; 1463 else 1464 return rc; 1465 } else if (rc < n_channels) { 1466 netif_err(efx, drv, efx->net_dev, 1467 "WARNING: Insufficient MSI-X vectors" 1468 " available (%d < %u).\n", rc, n_channels); 1469 netif_err(efx, drv, efx->net_dev, 1470 "WARNING: Performance may be reduced.\n"); 1471 n_channels = rc; 1472 } 1473 1474 if (rc > 0) { 1475 efx->n_channels = n_channels; 1476 if (n_channels > extra_channels) 1477 n_channels -= extra_channels; 1478 if (efx_separate_tx_channels) { 1479 efx->n_tx_channels = min(max(n_channels / 2, 1480 1U), 1481 efx->max_tx_channels); 1482 efx->n_rx_channels = max(n_channels - 1483 efx->n_tx_channels, 1484 1U); 1485 } else { 1486 efx->n_tx_channels = min(n_channels, 1487 efx->max_tx_channels); 1488 efx->n_rx_channels = n_channels; 1489 } 1490 for (i = 0; i < efx->n_channels; i++) 1491 efx_get_channel(efx, i)->irq = 1492 xentries[i].vector; 1493 } 1494 } 1495 1496 /* Try single interrupt MSI */ 1497 if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1498 efx->n_channels = 1; 1499 efx->n_rx_channels = 1; 1500 efx->n_tx_channels = 1; 1501 rc = pci_enable_msi(efx->pci_dev); 1502 if (rc == 0) { 1503 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1504 } else { 1505 netif_err(efx, drv, efx->net_dev, 1506 "could not enable MSI\n"); 1507 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 1508 efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1509 else 1510 return rc; 1511 } 1512 } 1513 1514 /* Assume legacy interrupts */ 1515 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1516 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1517 efx->n_rx_channels = 1; 1518 efx->n_tx_channels = 1; 1519 efx->legacy_irq = efx->pci_dev->irq; 1520 } 1521 1522 /* Assign extra channels if possible */ 1523 efx->n_extra_tx_channels = 0; 1524 j = efx->n_channels; 1525 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 1526 if (!efx->extra_channel_type[i]) 1527 continue; 1528 if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 1529 efx->n_channels <= extra_channels) { 1530 efx->extra_channel_type[i]->handle_no_channel(efx); 1531 } else { 1532 --j; 1533 efx_get_channel(efx, j)->type = 1534 efx->extra_channel_type[i]; 1535 if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 1536 efx->n_extra_tx_channels++; 1537 } 1538 } 1539 1540 /* RSS might be usable on VFs even if it is disabled on the PF */ 1541 #ifdef CONFIG_SFC_SRIOV 1542 if (efx->type->sriov_wanted) { 1543 efx->rss_spread = ((efx->n_rx_channels > 1 || 1544 !efx->type->sriov_wanted(efx)) ? 1545 efx->n_rx_channels : efx_vf_size(efx)); 1546 return 0; 1547 } 1548 #endif 1549 efx->rss_spread = efx->n_rx_channels; 1550 1551 return 0; 1552 } 1553 1554 #if defined(CONFIG_SMP) 1555 static void efx_set_interrupt_affinity(struct efx_nic *efx) 1556 { 1557 struct efx_channel *channel; 1558 unsigned int cpu; 1559 1560 efx_for_each_channel(channel, efx) { 1561 cpu = cpumask_local_spread(channel->channel, 1562 pcibus_to_node(efx->pci_dev->bus)); 1563 irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 1564 } 1565 } 1566 1567 static void efx_clear_interrupt_affinity(struct efx_nic *efx) 1568 { 1569 struct efx_channel *channel; 1570 1571 efx_for_each_channel(channel, efx) 1572 irq_set_affinity_hint(channel->irq, NULL); 1573 } 1574 #else 1575 static void 1576 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1577 { 1578 } 1579 1580 static void 1581 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1582 { 1583 } 1584 #endif /* CONFIG_SMP */ 1585 1586 static int efx_soft_enable_interrupts(struct efx_nic *efx) 1587 { 1588 struct efx_channel *channel, *end_channel; 1589 int rc; 1590 1591 BUG_ON(efx->state == STATE_DISABLED); 1592 1593 efx->irq_soft_enabled = true; 1594 smp_wmb(); 1595 1596 efx_for_each_channel(channel, efx) { 1597 if (!channel->type->keep_eventq) { 1598 rc = efx_init_eventq(channel); 1599 if (rc) 1600 goto fail; 1601 } 1602 efx_start_eventq(channel); 1603 } 1604 1605 efx_mcdi_mode_event(efx); 1606 1607 return 0; 1608 fail: 1609 end_channel = channel; 1610 efx_for_each_channel(channel, efx) { 1611 if (channel == end_channel) 1612 break; 1613 efx_stop_eventq(channel); 1614 if (!channel->type->keep_eventq) 1615 efx_fini_eventq(channel); 1616 } 1617 1618 return rc; 1619 } 1620 1621 static void efx_soft_disable_interrupts(struct efx_nic *efx) 1622 { 1623 struct efx_channel *channel; 1624 1625 if (efx->state == STATE_DISABLED) 1626 return; 1627 1628 efx_mcdi_mode_poll(efx); 1629 1630 efx->irq_soft_enabled = false; 1631 smp_wmb(); 1632 1633 if (efx->legacy_irq) 1634 synchronize_irq(efx->legacy_irq); 1635 1636 efx_for_each_channel(channel, efx) { 1637 if (channel->irq) 1638 synchronize_irq(channel->irq); 1639 1640 efx_stop_eventq(channel); 1641 if (!channel->type->keep_eventq) 1642 efx_fini_eventq(channel); 1643 } 1644 1645 /* Flush the asynchronous MCDI request queue */ 1646 efx_mcdi_flush_async(efx); 1647 } 1648 1649 static int efx_enable_interrupts(struct efx_nic *efx) 1650 { 1651 struct efx_channel *channel, *end_channel; 1652 int rc; 1653 1654 BUG_ON(efx->state == STATE_DISABLED); 1655 1656 if (efx->eeh_disabled_legacy_irq) { 1657 enable_irq(efx->legacy_irq); 1658 efx->eeh_disabled_legacy_irq = false; 1659 } 1660 1661 efx->type->irq_enable_master(efx); 1662 1663 efx_for_each_channel(channel, efx) { 1664 if (channel->type->keep_eventq) { 1665 rc = efx_init_eventq(channel); 1666 if (rc) 1667 goto fail; 1668 } 1669 } 1670 1671 rc = efx_soft_enable_interrupts(efx); 1672 if (rc) 1673 goto fail; 1674 1675 return 0; 1676 1677 fail: 1678 end_channel = channel; 1679 efx_for_each_channel(channel, efx) { 1680 if (channel == end_channel) 1681 break; 1682 if (channel->type->keep_eventq) 1683 efx_fini_eventq(channel); 1684 } 1685 1686 efx->type->irq_disable_non_ev(efx); 1687 1688 return rc; 1689 } 1690 1691 static void efx_disable_interrupts(struct efx_nic *efx) 1692 { 1693 struct efx_channel *channel; 1694 1695 efx_soft_disable_interrupts(efx); 1696 1697 efx_for_each_channel(channel, efx) { 1698 if (channel->type->keep_eventq) 1699 efx_fini_eventq(channel); 1700 } 1701 1702 efx->type->irq_disable_non_ev(efx); 1703 } 1704 1705 static void efx_remove_interrupts(struct efx_nic *efx) 1706 { 1707 struct efx_channel *channel; 1708 1709 /* Remove MSI/MSI-X interrupts */ 1710 efx_for_each_channel(channel, efx) 1711 channel->irq = 0; 1712 pci_disable_msi(efx->pci_dev); 1713 pci_disable_msix(efx->pci_dev); 1714 1715 /* Remove legacy interrupt */ 1716 efx->legacy_irq = 0; 1717 } 1718 1719 static void efx_set_channels(struct efx_nic *efx) 1720 { 1721 struct efx_channel *channel; 1722 struct efx_tx_queue *tx_queue; 1723 1724 efx->tx_channel_offset = 1725 efx_separate_tx_channels ? 1726 efx->n_channels - efx->n_tx_channels : 0; 1727 1728 /* We need to mark which channels really have RX and TX 1729 * queues, and adjust the TX queue numbers if we have separate 1730 * RX-only and TX-only channels. 1731 */ 1732 efx_for_each_channel(channel, efx) { 1733 if (channel->channel < efx->n_rx_channels) 1734 channel->rx_queue.core_index = channel->channel; 1735 else 1736 channel->rx_queue.core_index = -1; 1737 1738 efx_for_each_channel_tx_queue(tx_queue, channel) 1739 tx_queue->queue -= (efx->tx_channel_offset * 1740 EFX_TXQ_TYPES); 1741 } 1742 } 1743 1744 static int efx_probe_nic(struct efx_nic *efx) 1745 { 1746 int rc; 1747 1748 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1749 1750 /* Carry out hardware-type specific initialisation */ 1751 rc = efx->type->probe(efx); 1752 if (rc) 1753 return rc; 1754 1755 do { 1756 if (!efx->max_channels || !efx->max_tx_channels) { 1757 netif_err(efx, drv, efx->net_dev, 1758 "Insufficient resources to allocate" 1759 " any channels\n"); 1760 rc = -ENOSPC; 1761 goto fail1; 1762 } 1763 1764 /* Determine the number of channels and queues by trying 1765 * to hook in MSI-X interrupts. 1766 */ 1767 rc = efx_probe_interrupts(efx); 1768 if (rc) 1769 goto fail1; 1770 1771 efx_set_channels(efx); 1772 1773 /* dimension_resources can fail with EAGAIN */ 1774 rc = efx->type->dimension_resources(efx); 1775 if (rc != 0 && rc != -EAGAIN) 1776 goto fail2; 1777 1778 if (rc == -EAGAIN) 1779 /* try again with new max_channels */ 1780 efx_remove_interrupts(efx); 1781 1782 } while (rc == -EAGAIN); 1783 1784 if (efx->n_channels > 1) 1785 netdev_rss_key_fill(efx->rss_context.rx_hash_key, 1786 sizeof(efx->rss_context.rx_hash_key)); 1787 efx_set_default_rx_indir_table(efx, &efx->rss_context); 1788 1789 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1790 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1791 1792 /* Initialise the interrupt moderation settings */ 1793 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1794 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1795 true); 1796 1797 return 0; 1798 1799 fail2: 1800 efx_remove_interrupts(efx); 1801 fail1: 1802 efx->type->remove(efx); 1803 return rc; 1804 } 1805 1806 static void efx_remove_nic(struct efx_nic *efx) 1807 { 1808 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1809 1810 efx_remove_interrupts(efx); 1811 efx->type->remove(efx); 1812 } 1813 1814 static int efx_probe_filters(struct efx_nic *efx) 1815 { 1816 int rc; 1817 1818 init_rwsem(&efx->filter_sem); 1819 mutex_lock(&efx->mac_lock); 1820 down_write(&efx->filter_sem); 1821 rc = efx->type->filter_table_probe(efx); 1822 if (rc) 1823 goto out_unlock; 1824 1825 #ifdef CONFIG_RFS_ACCEL 1826 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1827 struct efx_channel *channel; 1828 int i, success = 1; 1829 1830 efx_for_each_channel(channel, efx) { 1831 channel->rps_flow_id = 1832 kcalloc(efx->type->max_rx_ip_filters, 1833 sizeof(*channel->rps_flow_id), 1834 GFP_KERNEL); 1835 if (!channel->rps_flow_id) 1836 success = 0; 1837 else 1838 for (i = 0; 1839 i < efx->type->max_rx_ip_filters; 1840 ++i) 1841 channel->rps_flow_id[i] = 1842 RPS_FLOW_ID_INVALID; 1843 } 1844 1845 if (!success) { 1846 efx_for_each_channel(channel, efx) 1847 kfree(channel->rps_flow_id); 1848 efx->type->filter_table_remove(efx); 1849 rc = -ENOMEM; 1850 goto out_unlock; 1851 } 1852 1853 efx->rps_expire_index = efx->rps_expire_channel = 0; 1854 } 1855 #endif 1856 out_unlock: 1857 up_write(&efx->filter_sem); 1858 mutex_unlock(&efx->mac_lock); 1859 return rc; 1860 } 1861 1862 static void efx_remove_filters(struct efx_nic *efx) 1863 { 1864 #ifdef CONFIG_RFS_ACCEL 1865 struct efx_channel *channel; 1866 1867 efx_for_each_channel(channel, efx) 1868 kfree(channel->rps_flow_id); 1869 #endif 1870 down_write(&efx->filter_sem); 1871 efx->type->filter_table_remove(efx); 1872 up_write(&efx->filter_sem); 1873 } 1874 1875 static void efx_restore_filters(struct efx_nic *efx) 1876 { 1877 down_read(&efx->filter_sem); 1878 efx->type->filter_table_restore(efx); 1879 up_read(&efx->filter_sem); 1880 } 1881 1882 /************************************************************************** 1883 * 1884 * NIC startup/shutdown 1885 * 1886 *************************************************************************/ 1887 1888 static int efx_probe_all(struct efx_nic *efx) 1889 { 1890 int rc; 1891 1892 rc = efx_probe_nic(efx); 1893 if (rc) { 1894 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1895 goto fail1; 1896 } 1897 1898 rc = efx_probe_port(efx); 1899 if (rc) { 1900 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1901 goto fail2; 1902 } 1903 1904 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 1905 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 1906 rc = -EINVAL; 1907 goto fail3; 1908 } 1909 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1910 1911 #ifdef CONFIG_SFC_SRIOV 1912 rc = efx->type->vswitching_probe(efx); 1913 if (rc) /* not fatal; the PF will still work fine */ 1914 netif_warn(efx, probe, efx->net_dev, 1915 "failed to setup vswitching rc=%d;" 1916 " VFs may not function\n", rc); 1917 #endif 1918 1919 rc = efx_probe_filters(efx); 1920 if (rc) { 1921 netif_err(efx, probe, efx->net_dev, 1922 "failed to create filter tables\n"); 1923 goto fail4; 1924 } 1925 1926 rc = efx_probe_channels(efx); 1927 if (rc) 1928 goto fail5; 1929 1930 return 0; 1931 1932 fail5: 1933 efx_remove_filters(efx); 1934 fail4: 1935 #ifdef CONFIG_SFC_SRIOV 1936 efx->type->vswitching_remove(efx); 1937 #endif 1938 fail3: 1939 efx_remove_port(efx); 1940 fail2: 1941 efx_remove_nic(efx); 1942 fail1: 1943 return rc; 1944 } 1945 1946 /* If the interface is supposed to be running but is not, start 1947 * the hardware and software data path, regular activity for the port 1948 * (MAC statistics, link polling, etc.) and schedule the port to be 1949 * reconfigured. Interrupts must already be enabled. This function 1950 * is safe to call multiple times, so long as the NIC is not disabled. 1951 * Requires the RTNL lock. 1952 */ 1953 static void efx_start_all(struct efx_nic *efx) 1954 { 1955 EFX_ASSERT_RESET_SERIALISED(efx); 1956 BUG_ON(efx->state == STATE_DISABLED); 1957 1958 /* Check that it is appropriate to restart the interface. All 1959 * of these flags are safe to read under just the rtnl lock */ 1960 if (efx->port_enabled || !netif_running(efx->net_dev) || 1961 efx->reset_pending) 1962 return; 1963 1964 efx_start_port(efx); 1965 efx_start_datapath(efx); 1966 1967 /* Start the hardware monitor if there is one */ 1968 if (efx->type->monitor != NULL) 1969 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1970 efx_monitor_interval); 1971 1972 /* Link state detection is normally event-driven; we have 1973 * to poll now because we could have missed a change 1974 */ 1975 mutex_lock(&efx->mac_lock); 1976 if (efx->phy_op->poll(efx)) 1977 efx_link_status_changed(efx); 1978 mutex_unlock(&efx->mac_lock); 1979 1980 efx->type->start_stats(efx); 1981 efx->type->pull_stats(efx); 1982 spin_lock_bh(&efx->stats_lock); 1983 efx->type->update_stats(efx, NULL, NULL); 1984 spin_unlock_bh(&efx->stats_lock); 1985 } 1986 1987 /* Quiesce the hardware and software data path, and regular activity 1988 * for the port without bringing the link down. Safe to call multiple 1989 * times with the NIC in almost any state, but interrupts should be 1990 * enabled. Requires the RTNL lock. 1991 */ 1992 static void efx_stop_all(struct efx_nic *efx) 1993 { 1994 EFX_ASSERT_RESET_SERIALISED(efx); 1995 1996 /* port_enabled can be read safely under the rtnl lock */ 1997 if (!efx->port_enabled) 1998 return; 1999 2000 /* update stats before we go down so we can accurately count 2001 * rx_nodesc_drops 2002 */ 2003 efx->type->pull_stats(efx); 2004 spin_lock_bh(&efx->stats_lock); 2005 efx->type->update_stats(efx, NULL, NULL); 2006 spin_unlock_bh(&efx->stats_lock); 2007 efx->type->stop_stats(efx); 2008 efx_stop_port(efx); 2009 2010 /* Stop the kernel transmit interface. This is only valid if 2011 * the device is stopped or detached; otherwise the watchdog 2012 * may fire immediately. 2013 */ 2014 WARN_ON(netif_running(efx->net_dev) && 2015 netif_device_present(efx->net_dev)); 2016 netif_tx_disable(efx->net_dev); 2017 2018 efx_stop_datapath(efx); 2019 } 2020 2021 static void efx_remove_all(struct efx_nic *efx) 2022 { 2023 efx_remove_channels(efx); 2024 efx_remove_filters(efx); 2025 #ifdef CONFIG_SFC_SRIOV 2026 efx->type->vswitching_remove(efx); 2027 #endif 2028 efx_remove_port(efx); 2029 efx_remove_nic(efx); 2030 } 2031 2032 /************************************************************************** 2033 * 2034 * Interrupt moderation 2035 * 2036 **************************************************************************/ 2037 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 2038 { 2039 if (usecs == 0) 2040 return 0; 2041 if (usecs * 1000 < efx->timer_quantum_ns) 2042 return 1; /* never round down to 0 */ 2043 return usecs * 1000 / efx->timer_quantum_ns; 2044 } 2045 2046 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 2047 { 2048 /* We must round up when converting ticks to microseconds 2049 * because we round down when converting the other way. 2050 */ 2051 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 2052 } 2053 2054 /* Set interrupt moderation parameters */ 2055 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 2056 unsigned int rx_usecs, bool rx_adaptive, 2057 bool rx_may_override_tx) 2058 { 2059 struct efx_channel *channel; 2060 unsigned int timer_max_us; 2061 2062 EFX_ASSERT_RESET_SERIALISED(efx); 2063 2064 timer_max_us = efx->timer_max_ns / 1000; 2065 2066 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 2067 return -EINVAL; 2068 2069 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 2070 !rx_may_override_tx) { 2071 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 2072 "RX and TX IRQ moderation must be equal\n"); 2073 return -EINVAL; 2074 } 2075 2076 efx->irq_rx_adaptive = rx_adaptive; 2077 efx->irq_rx_moderation_us = rx_usecs; 2078 efx_for_each_channel(channel, efx) { 2079 if (efx_channel_has_rx_queue(channel)) 2080 channel->irq_moderation_us = rx_usecs; 2081 else if (efx_channel_has_tx_queues(channel)) 2082 channel->irq_moderation_us = tx_usecs; 2083 } 2084 2085 return 0; 2086 } 2087 2088 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2089 unsigned int *rx_usecs, bool *rx_adaptive) 2090 { 2091 *rx_adaptive = efx->irq_rx_adaptive; 2092 *rx_usecs = efx->irq_rx_moderation_us; 2093 2094 /* If channels are shared between RX and TX, so is IRQ 2095 * moderation. Otherwise, IRQ moderation is the same for all 2096 * TX channels and is not adaptive. 2097 */ 2098 if (efx->tx_channel_offset == 0) { 2099 *tx_usecs = *rx_usecs; 2100 } else { 2101 struct efx_channel *tx_channel; 2102 2103 tx_channel = efx->channel[efx->tx_channel_offset]; 2104 *tx_usecs = tx_channel->irq_moderation_us; 2105 } 2106 } 2107 2108 /************************************************************************** 2109 * 2110 * Hardware monitor 2111 * 2112 **************************************************************************/ 2113 2114 /* Run periodically off the general workqueue */ 2115 static void efx_monitor(struct work_struct *data) 2116 { 2117 struct efx_nic *efx = container_of(data, struct efx_nic, 2118 monitor_work.work); 2119 2120 netif_vdbg(efx, timer, efx->net_dev, 2121 "hardware monitor executing on CPU %d\n", 2122 raw_smp_processor_id()); 2123 BUG_ON(efx->type->monitor == NULL); 2124 2125 /* If the mac_lock is already held then it is likely a port 2126 * reconfiguration is already in place, which will likely do 2127 * most of the work of monitor() anyway. */ 2128 if (mutex_trylock(&efx->mac_lock)) { 2129 if (efx->port_enabled) 2130 efx->type->monitor(efx); 2131 mutex_unlock(&efx->mac_lock); 2132 } 2133 2134 queue_delayed_work(efx->workqueue, &efx->monitor_work, 2135 efx_monitor_interval); 2136 } 2137 2138 /************************************************************************** 2139 * 2140 * ioctls 2141 * 2142 *************************************************************************/ 2143 2144 /* Net device ioctl 2145 * Context: process, rtnl_lock() held. 2146 */ 2147 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2148 { 2149 struct efx_nic *efx = netdev_priv(net_dev); 2150 struct mii_ioctl_data *data = if_mii(ifr); 2151 2152 if (cmd == SIOCSHWTSTAMP) 2153 return efx_ptp_set_ts_config(efx, ifr); 2154 if (cmd == SIOCGHWTSTAMP) 2155 return efx_ptp_get_ts_config(efx, ifr); 2156 2157 /* Convert phy_id from older PRTAD/DEVAD format */ 2158 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2159 (data->phy_id & 0xfc00) == 0x0400) 2160 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2161 2162 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2163 } 2164 2165 /************************************************************************** 2166 * 2167 * NAPI interface 2168 * 2169 **************************************************************************/ 2170 2171 static void efx_init_napi_channel(struct efx_channel *channel) 2172 { 2173 struct efx_nic *efx = channel->efx; 2174 2175 channel->napi_dev = efx->net_dev; 2176 netif_napi_add(channel->napi_dev, &channel->napi_str, 2177 efx_poll, napi_weight); 2178 } 2179 2180 static void efx_init_napi(struct efx_nic *efx) 2181 { 2182 struct efx_channel *channel; 2183 2184 efx_for_each_channel(channel, efx) 2185 efx_init_napi_channel(channel); 2186 } 2187 2188 static void efx_fini_napi_channel(struct efx_channel *channel) 2189 { 2190 if (channel->napi_dev) 2191 netif_napi_del(&channel->napi_str); 2192 2193 channel->napi_dev = NULL; 2194 } 2195 2196 static void efx_fini_napi(struct efx_nic *efx) 2197 { 2198 struct efx_channel *channel; 2199 2200 efx_for_each_channel(channel, efx) 2201 efx_fini_napi_channel(channel); 2202 } 2203 2204 /************************************************************************** 2205 * 2206 * Kernel netpoll interface 2207 * 2208 *************************************************************************/ 2209 2210 #ifdef CONFIG_NET_POLL_CONTROLLER 2211 2212 /* Although in the common case interrupts will be disabled, this is not 2213 * guaranteed. However, all our work happens inside the NAPI callback, 2214 * so no locking is required. 2215 */ 2216 static void efx_netpoll(struct net_device *net_dev) 2217 { 2218 struct efx_nic *efx = netdev_priv(net_dev); 2219 struct efx_channel *channel; 2220 2221 efx_for_each_channel(channel, efx) 2222 efx_schedule_channel(channel); 2223 } 2224 2225 #endif 2226 2227 /************************************************************************** 2228 * 2229 * Kernel net device interface 2230 * 2231 *************************************************************************/ 2232 2233 /* Context: process, rtnl_lock() held. */ 2234 int efx_net_open(struct net_device *net_dev) 2235 { 2236 struct efx_nic *efx = netdev_priv(net_dev); 2237 int rc; 2238 2239 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2240 raw_smp_processor_id()); 2241 2242 rc = efx_check_disabled(efx); 2243 if (rc) 2244 return rc; 2245 if (efx->phy_mode & PHY_MODE_SPECIAL) 2246 return -EBUSY; 2247 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2248 return -EIO; 2249 2250 /* Notify the kernel of the link state polled during driver load, 2251 * before the monitor starts running */ 2252 efx_link_status_changed(efx); 2253 2254 efx_start_all(efx); 2255 if (efx->state == STATE_DISABLED || efx->reset_pending) 2256 netif_device_detach(efx->net_dev); 2257 efx_selftest_async_start(efx); 2258 return 0; 2259 } 2260 2261 /* Context: process, rtnl_lock() held. 2262 * Note that the kernel will ignore our return code; this method 2263 * should really be a void. 2264 */ 2265 int efx_net_stop(struct net_device *net_dev) 2266 { 2267 struct efx_nic *efx = netdev_priv(net_dev); 2268 2269 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2270 raw_smp_processor_id()); 2271 2272 /* Stop the device and flush all the channels */ 2273 efx_stop_all(efx); 2274 2275 return 0; 2276 } 2277 2278 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2279 static void efx_net_stats(struct net_device *net_dev, 2280 struct rtnl_link_stats64 *stats) 2281 { 2282 struct efx_nic *efx = netdev_priv(net_dev); 2283 2284 spin_lock_bh(&efx->stats_lock); 2285 efx->type->update_stats(efx, NULL, stats); 2286 spin_unlock_bh(&efx->stats_lock); 2287 } 2288 2289 /* Context: netif_tx_lock held, BHs disabled. */ 2290 static void efx_watchdog(struct net_device *net_dev) 2291 { 2292 struct efx_nic *efx = netdev_priv(net_dev); 2293 2294 netif_err(efx, tx_err, efx->net_dev, 2295 "TX stuck with port_enabled=%d: resetting channels\n", 2296 efx->port_enabled); 2297 2298 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2299 } 2300 2301 2302 /* Context: process, rtnl_lock() held. */ 2303 static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2304 { 2305 struct efx_nic *efx = netdev_priv(net_dev); 2306 int rc; 2307 2308 rc = efx_check_disabled(efx); 2309 if (rc) 2310 return rc; 2311 2312 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2313 2314 efx_device_detach_sync(efx); 2315 efx_stop_all(efx); 2316 2317 mutex_lock(&efx->mac_lock); 2318 net_dev->mtu = new_mtu; 2319 efx_mac_reconfigure(efx); 2320 mutex_unlock(&efx->mac_lock); 2321 2322 efx_start_all(efx); 2323 efx_device_attach_if_not_resetting(efx); 2324 return 0; 2325 } 2326 2327 static int efx_set_mac_address(struct net_device *net_dev, void *data) 2328 { 2329 struct efx_nic *efx = netdev_priv(net_dev); 2330 struct sockaddr *addr = data; 2331 u8 *new_addr = addr->sa_data; 2332 u8 old_addr[6]; 2333 int rc; 2334 2335 if (!is_valid_ether_addr(new_addr)) { 2336 netif_err(efx, drv, efx->net_dev, 2337 "invalid ethernet MAC address requested: %pM\n", 2338 new_addr); 2339 return -EADDRNOTAVAIL; 2340 } 2341 2342 /* save old address */ 2343 ether_addr_copy(old_addr, net_dev->dev_addr); 2344 ether_addr_copy(net_dev->dev_addr, new_addr); 2345 if (efx->type->set_mac_address) { 2346 rc = efx->type->set_mac_address(efx); 2347 if (rc) { 2348 ether_addr_copy(net_dev->dev_addr, old_addr); 2349 return rc; 2350 } 2351 } 2352 2353 /* Reconfigure the MAC */ 2354 mutex_lock(&efx->mac_lock); 2355 efx_mac_reconfigure(efx); 2356 mutex_unlock(&efx->mac_lock); 2357 2358 return 0; 2359 } 2360 2361 /* Context: netif_addr_lock held, BHs disabled. */ 2362 static void efx_set_rx_mode(struct net_device *net_dev) 2363 { 2364 struct efx_nic *efx = netdev_priv(net_dev); 2365 2366 if (efx->port_enabled) 2367 queue_work(efx->workqueue, &efx->mac_work); 2368 /* Otherwise efx_start_port() will do this */ 2369 } 2370 2371 static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2372 { 2373 struct efx_nic *efx = netdev_priv(net_dev); 2374 int rc; 2375 2376 /* If disabling RX n-tuple filtering, clear existing filters */ 2377 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2378 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2379 if (rc) 2380 return rc; 2381 } 2382 2383 /* If Rx VLAN filter is changed, update filters via mac_reconfigure. 2384 * If rx-fcs is changed, mac_reconfigure updates that too. 2385 */ 2386 if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER | 2387 NETIF_F_RXFCS)) { 2388 /* efx_set_rx_mode() will schedule MAC work to update filters 2389 * when a new features are finally set in net_dev. 2390 */ 2391 efx_set_rx_mode(net_dev); 2392 } 2393 2394 return 0; 2395 } 2396 2397 static int efx_get_phys_port_id(struct net_device *net_dev, 2398 struct netdev_phys_item_id *ppid) 2399 { 2400 struct efx_nic *efx = netdev_priv(net_dev); 2401 2402 if (efx->type->get_phys_port_id) 2403 return efx->type->get_phys_port_id(efx, ppid); 2404 else 2405 return -EOPNOTSUPP; 2406 } 2407 2408 static int efx_get_phys_port_name(struct net_device *net_dev, 2409 char *name, size_t len) 2410 { 2411 struct efx_nic *efx = netdev_priv(net_dev); 2412 2413 if (snprintf(name, len, "p%u", efx->port_num) >= len) 2414 return -EINVAL; 2415 return 0; 2416 } 2417 2418 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2419 { 2420 struct efx_nic *efx = netdev_priv(net_dev); 2421 2422 if (efx->type->vlan_rx_add_vid) 2423 return efx->type->vlan_rx_add_vid(efx, proto, vid); 2424 else 2425 return -EOPNOTSUPP; 2426 } 2427 2428 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2429 { 2430 struct efx_nic *efx = netdev_priv(net_dev); 2431 2432 if (efx->type->vlan_rx_kill_vid) 2433 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 2434 else 2435 return -EOPNOTSUPP; 2436 } 2437 2438 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in) 2439 { 2440 switch (in) { 2441 case UDP_TUNNEL_TYPE_VXLAN: 2442 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 2443 case UDP_TUNNEL_TYPE_GENEVE: 2444 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 2445 default: 2446 return -1; 2447 } 2448 } 2449 2450 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti) 2451 { 2452 struct efx_nic *efx = netdev_priv(dev); 2453 struct efx_udp_tunnel tnl; 2454 int efx_tunnel_type; 2455 2456 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2457 if (efx_tunnel_type < 0) 2458 return; 2459 2460 tnl.type = (u16)efx_tunnel_type; 2461 tnl.port = ti->port; 2462 2463 if (efx->type->udp_tnl_add_port) 2464 (void)efx->type->udp_tnl_add_port(efx, tnl); 2465 } 2466 2467 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti) 2468 { 2469 struct efx_nic *efx = netdev_priv(dev); 2470 struct efx_udp_tunnel tnl; 2471 int efx_tunnel_type; 2472 2473 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2474 if (efx_tunnel_type < 0) 2475 return; 2476 2477 tnl.type = (u16)efx_tunnel_type; 2478 tnl.port = ti->port; 2479 2480 if (efx->type->udp_tnl_del_port) 2481 (void)efx->type->udp_tnl_del_port(efx, tnl); 2482 } 2483 2484 static const struct net_device_ops efx_netdev_ops = { 2485 .ndo_open = efx_net_open, 2486 .ndo_stop = efx_net_stop, 2487 .ndo_get_stats64 = efx_net_stats, 2488 .ndo_tx_timeout = efx_watchdog, 2489 .ndo_start_xmit = efx_hard_start_xmit, 2490 .ndo_validate_addr = eth_validate_addr, 2491 .ndo_do_ioctl = efx_ioctl, 2492 .ndo_change_mtu = efx_change_mtu, 2493 .ndo_set_mac_address = efx_set_mac_address, 2494 .ndo_set_rx_mode = efx_set_rx_mode, 2495 .ndo_set_features = efx_set_features, 2496 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 2497 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2498 #ifdef CONFIG_SFC_SRIOV 2499 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2500 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2501 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2502 .ndo_get_vf_config = efx_sriov_get_vf_config, 2503 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2504 #endif 2505 .ndo_get_phys_port_id = efx_get_phys_port_id, 2506 .ndo_get_phys_port_name = efx_get_phys_port_name, 2507 #ifdef CONFIG_NET_POLL_CONTROLLER 2508 .ndo_poll_controller = efx_netpoll, 2509 #endif 2510 .ndo_setup_tc = efx_setup_tc, 2511 #ifdef CONFIG_RFS_ACCEL 2512 .ndo_rx_flow_steer = efx_filter_rfs, 2513 #endif 2514 .ndo_udp_tunnel_add = efx_udp_tunnel_add, 2515 .ndo_udp_tunnel_del = efx_udp_tunnel_del, 2516 }; 2517 2518 static void efx_update_name(struct efx_nic *efx) 2519 { 2520 strcpy(efx->name, efx->net_dev->name); 2521 efx_mtd_rename(efx); 2522 efx_set_channel_names(efx); 2523 } 2524 2525 static int efx_netdev_event(struct notifier_block *this, 2526 unsigned long event, void *ptr) 2527 { 2528 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2529 2530 if ((net_dev->netdev_ops == &efx_netdev_ops) && 2531 event == NETDEV_CHANGENAME) 2532 efx_update_name(netdev_priv(net_dev)); 2533 2534 return NOTIFY_DONE; 2535 } 2536 2537 static struct notifier_block efx_netdev_notifier = { 2538 .notifier_call = efx_netdev_event, 2539 }; 2540 2541 static ssize_t 2542 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2543 { 2544 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2545 return sprintf(buf, "%d\n", efx->phy_type); 2546 } 2547 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2548 2549 #ifdef CONFIG_SFC_MCDI_LOGGING 2550 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2551 char *buf) 2552 { 2553 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2554 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2555 2556 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2557 } 2558 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2559 const char *buf, size_t count) 2560 { 2561 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2562 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2563 bool enable = count > 0 && *buf != '0'; 2564 2565 mcdi->logging_enabled = enable; 2566 return count; 2567 } 2568 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2569 #endif 2570 2571 static int efx_register_netdev(struct efx_nic *efx) 2572 { 2573 struct net_device *net_dev = efx->net_dev; 2574 struct efx_channel *channel; 2575 int rc; 2576 2577 net_dev->watchdog_timeo = 5 * HZ; 2578 net_dev->irq = efx->pci_dev->irq; 2579 net_dev->netdev_ops = &efx_netdev_ops; 2580 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 2581 net_dev->priv_flags |= IFF_UNICAST_FLT; 2582 net_dev->ethtool_ops = &efx_ethtool_ops; 2583 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2584 net_dev->min_mtu = EFX_MIN_MTU; 2585 net_dev->max_mtu = EFX_MAX_MTU; 2586 2587 rtnl_lock(); 2588 2589 /* Enable resets to be scheduled and check whether any were 2590 * already requested. If so, the NIC is probably hosed so we 2591 * abort. 2592 */ 2593 efx->state = STATE_READY; 2594 smp_mb(); /* ensure we change state before checking reset_pending */ 2595 if (efx->reset_pending) { 2596 netif_err(efx, probe, efx->net_dev, 2597 "aborting probe due to scheduled reset\n"); 2598 rc = -EIO; 2599 goto fail_locked; 2600 } 2601 2602 rc = dev_alloc_name(net_dev, net_dev->name); 2603 if (rc < 0) 2604 goto fail_locked; 2605 efx_update_name(efx); 2606 2607 /* Always start with carrier off; PHY events will detect the link */ 2608 netif_carrier_off(net_dev); 2609 2610 rc = register_netdevice(net_dev); 2611 if (rc) 2612 goto fail_locked; 2613 2614 efx_for_each_channel(channel, efx) { 2615 struct efx_tx_queue *tx_queue; 2616 efx_for_each_channel_tx_queue(tx_queue, channel) 2617 efx_init_tx_queue_core_txq(tx_queue); 2618 } 2619 2620 efx_associate(efx); 2621 2622 rtnl_unlock(); 2623 2624 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2625 if (rc) { 2626 netif_err(efx, drv, efx->net_dev, 2627 "failed to init net dev attributes\n"); 2628 goto fail_registered; 2629 } 2630 #ifdef CONFIG_SFC_MCDI_LOGGING 2631 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2632 if (rc) { 2633 netif_err(efx, drv, efx->net_dev, 2634 "failed to init net dev attributes\n"); 2635 goto fail_attr_mcdi_logging; 2636 } 2637 #endif 2638 2639 return 0; 2640 2641 #ifdef CONFIG_SFC_MCDI_LOGGING 2642 fail_attr_mcdi_logging: 2643 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2644 #endif 2645 fail_registered: 2646 rtnl_lock(); 2647 efx_dissociate(efx); 2648 unregister_netdevice(net_dev); 2649 fail_locked: 2650 efx->state = STATE_UNINIT; 2651 rtnl_unlock(); 2652 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2653 return rc; 2654 } 2655 2656 static void efx_unregister_netdev(struct efx_nic *efx) 2657 { 2658 if (!efx->net_dev) 2659 return; 2660 2661 BUG_ON(netdev_priv(efx->net_dev) != efx); 2662 2663 if (efx_dev_registered(efx)) { 2664 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2665 #ifdef CONFIG_SFC_MCDI_LOGGING 2666 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2667 #endif 2668 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2669 unregister_netdev(efx->net_dev); 2670 } 2671 } 2672 2673 /************************************************************************** 2674 * 2675 * Device reset and suspend 2676 * 2677 **************************************************************************/ 2678 2679 /* Tears down the entire software state and most of the hardware state 2680 * before reset. */ 2681 void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2682 { 2683 EFX_ASSERT_RESET_SERIALISED(efx); 2684 2685 if (method == RESET_TYPE_MCDI_TIMEOUT) 2686 efx->type->prepare_flr(efx); 2687 2688 efx_stop_all(efx); 2689 efx_disable_interrupts(efx); 2690 2691 mutex_lock(&efx->mac_lock); 2692 mutex_lock(&efx->rss_lock); 2693 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2694 method != RESET_TYPE_DATAPATH) 2695 efx->phy_op->fini(efx); 2696 efx->type->fini(efx); 2697 } 2698 2699 /* This function will always ensure that the locks acquired in 2700 * efx_reset_down() are released. A failure return code indicates 2701 * that we were unable to reinitialise the hardware, and the 2702 * driver should be disabled. If ok is false, then the rx and tx 2703 * engines are not restarted, pending a RESET_DISABLE. */ 2704 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2705 { 2706 int rc; 2707 2708 EFX_ASSERT_RESET_SERIALISED(efx); 2709 2710 if (method == RESET_TYPE_MCDI_TIMEOUT) 2711 efx->type->finish_flr(efx); 2712 2713 /* Ensure that SRAM is initialised even if we're disabling the device */ 2714 rc = efx->type->init(efx); 2715 if (rc) { 2716 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2717 goto fail; 2718 } 2719 2720 if (!ok) 2721 goto fail; 2722 2723 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2724 method != RESET_TYPE_DATAPATH) { 2725 rc = efx->phy_op->init(efx); 2726 if (rc) 2727 goto fail; 2728 rc = efx->phy_op->reconfigure(efx); 2729 if (rc && rc != -EPERM) 2730 netif_err(efx, drv, efx->net_dev, 2731 "could not restore PHY settings\n"); 2732 } 2733 2734 rc = efx_enable_interrupts(efx); 2735 if (rc) 2736 goto fail; 2737 2738 #ifdef CONFIG_SFC_SRIOV 2739 rc = efx->type->vswitching_restore(efx); 2740 if (rc) /* not fatal; the PF will still work fine */ 2741 netif_warn(efx, probe, efx->net_dev, 2742 "failed to restore vswitching rc=%d;" 2743 " VFs may not function\n", rc); 2744 #endif 2745 2746 if (efx->type->rx_restore_rss_contexts) 2747 efx->type->rx_restore_rss_contexts(efx); 2748 mutex_unlock(&efx->rss_lock); 2749 down_read(&efx->filter_sem); 2750 efx_restore_filters(efx); 2751 up_read(&efx->filter_sem); 2752 if (efx->type->sriov_reset) 2753 efx->type->sriov_reset(efx); 2754 2755 mutex_unlock(&efx->mac_lock); 2756 2757 efx_start_all(efx); 2758 2759 if (efx->type->udp_tnl_push_ports) 2760 efx->type->udp_tnl_push_ports(efx); 2761 2762 return 0; 2763 2764 fail: 2765 efx->port_initialized = false; 2766 2767 mutex_unlock(&efx->rss_lock); 2768 mutex_unlock(&efx->mac_lock); 2769 2770 return rc; 2771 } 2772 2773 /* Reset the NIC using the specified method. Note that the reset may 2774 * fail, in which case the card will be left in an unusable state. 2775 * 2776 * Caller must hold the rtnl_lock. 2777 */ 2778 int efx_reset(struct efx_nic *efx, enum reset_type method) 2779 { 2780 int rc, rc2; 2781 bool disabled; 2782 2783 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2784 RESET_TYPE(method)); 2785 2786 efx_device_detach_sync(efx); 2787 efx_reset_down(efx, method); 2788 2789 rc = efx->type->reset(efx, method); 2790 if (rc) { 2791 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2792 goto out; 2793 } 2794 2795 /* Clear flags for the scopes we covered. We assume the NIC and 2796 * driver are now quiescent so that there is no race here. 2797 */ 2798 if (method < RESET_TYPE_MAX_METHOD) 2799 efx->reset_pending &= -(1 << (method + 1)); 2800 else /* it doesn't fit into the well-ordered scope hierarchy */ 2801 __clear_bit(method, &efx->reset_pending); 2802 2803 /* Reinitialise bus-mastering, which may have been turned off before 2804 * the reset was scheduled. This is still appropriate, even in the 2805 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2806 * can respond to requests. */ 2807 pci_set_master(efx->pci_dev); 2808 2809 out: 2810 /* Leave device stopped if necessary */ 2811 disabled = rc || 2812 method == RESET_TYPE_DISABLE || 2813 method == RESET_TYPE_RECOVER_OR_DISABLE; 2814 rc2 = efx_reset_up(efx, method, !disabled); 2815 if (rc2) { 2816 disabled = true; 2817 if (!rc) 2818 rc = rc2; 2819 } 2820 2821 if (disabled) { 2822 dev_close(efx->net_dev); 2823 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2824 efx->state = STATE_DISABLED; 2825 } else { 2826 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2827 efx_device_attach_if_not_resetting(efx); 2828 } 2829 return rc; 2830 } 2831 2832 /* Try recovery mechanisms. 2833 * For now only EEH is supported. 2834 * Returns 0 if the recovery mechanisms are unsuccessful. 2835 * Returns a non-zero value otherwise. 2836 */ 2837 int efx_try_recovery(struct efx_nic *efx) 2838 { 2839 #ifdef CONFIG_EEH 2840 /* A PCI error can occur and not be seen by EEH because nothing 2841 * happens on the PCI bus. In this case the driver may fail and 2842 * schedule a 'recover or reset', leading to this recovery handler. 2843 * Manually call the eeh failure check function. 2844 */ 2845 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2846 if (eeh_dev_check_failure(eehdev)) { 2847 /* The EEH mechanisms will handle the error and reset the 2848 * device if necessary. 2849 */ 2850 return 1; 2851 } 2852 #endif 2853 return 0; 2854 } 2855 2856 static void efx_wait_for_bist_end(struct efx_nic *efx) 2857 { 2858 int i; 2859 2860 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 2861 if (efx_mcdi_poll_reboot(efx)) 2862 goto out; 2863 msleep(BIST_WAIT_DELAY_MS); 2864 } 2865 2866 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 2867 out: 2868 /* Either way unset the BIST flag. If we found no reboot we probably 2869 * won't recover, but we should try. 2870 */ 2871 efx->mc_bist_for_other_fn = false; 2872 } 2873 2874 /* The worker thread exists so that code that cannot sleep can 2875 * schedule a reset for later. 2876 */ 2877 static void efx_reset_work(struct work_struct *data) 2878 { 2879 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2880 unsigned long pending; 2881 enum reset_type method; 2882 2883 pending = READ_ONCE(efx->reset_pending); 2884 method = fls(pending) - 1; 2885 2886 if (method == RESET_TYPE_MC_BIST) 2887 efx_wait_for_bist_end(efx); 2888 2889 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2890 method == RESET_TYPE_RECOVER_OR_ALL) && 2891 efx_try_recovery(efx)) 2892 return; 2893 2894 if (!pending) 2895 return; 2896 2897 rtnl_lock(); 2898 2899 /* We checked the state in efx_schedule_reset() but it may 2900 * have changed by now. Now that we have the RTNL lock, 2901 * it cannot change again. 2902 */ 2903 if (efx->state == STATE_READY) 2904 (void)efx_reset(efx, method); 2905 2906 rtnl_unlock(); 2907 } 2908 2909 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2910 { 2911 enum reset_type method; 2912 2913 if (efx->state == STATE_RECOVERY) { 2914 netif_dbg(efx, drv, efx->net_dev, 2915 "recovering: skip scheduling %s reset\n", 2916 RESET_TYPE(type)); 2917 return; 2918 } 2919 2920 switch (type) { 2921 case RESET_TYPE_INVISIBLE: 2922 case RESET_TYPE_ALL: 2923 case RESET_TYPE_RECOVER_OR_ALL: 2924 case RESET_TYPE_WORLD: 2925 case RESET_TYPE_DISABLE: 2926 case RESET_TYPE_RECOVER_OR_DISABLE: 2927 case RESET_TYPE_DATAPATH: 2928 case RESET_TYPE_MC_BIST: 2929 case RESET_TYPE_MCDI_TIMEOUT: 2930 method = type; 2931 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2932 RESET_TYPE(method)); 2933 break; 2934 default: 2935 method = efx->type->map_reset_reason(type); 2936 netif_dbg(efx, drv, efx->net_dev, 2937 "scheduling %s reset for %s\n", 2938 RESET_TYPE(method), RESET_TYPE(type)); 2939 break; 2940 } 2941 2942 set_bit(method, &efx->reset_pending); 2943 smp_mb(); /* ensure we change reset_pending before checking state */ 2944 2945 /* If we're not READY then just leave the flags set as the cue 2946 * to abort probing or reschedule the reset later. 2947 */ 2948 if (READ_ONCE(efx->state) != STATE_READY) 2949 return; 2950 2951 /* efx_process_channel() will no longer read events once a 2952 * reset is scheduled. So switch back to poll'd MCDI completions. */ 2953 efx_mcdi_mode_poll(efx); 2954 2955 queue_work(reset_workqueue, &efx->reset_work); 2956 } 2957 2958 /************************************************************************** 2959 * 2960 * List of NICs we support 2961 * 2962 **************************************************************************/ 2963 2964 /* PCI device ID table */ 2965 static const struct pci_device_id efx_pci_table[] = { 2966 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2967 .driver_data = (unsigned long) &siena_a0_nic_type}, 2968 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2969 .driver_data = (unsigned long) &siena_a0_nic_type}, 2970 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 2971 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2972 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 2973 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2974 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 2975 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2976 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2977 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2978 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2979 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2980 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2981 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2982 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 2983 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2984 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 2985 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2986 {0} /* end of list */ 2987 }; 2988 2989 /************************************************************************** 2990 * 2991 * Dummy PHY/MAC operations 2992 * 2993 * Can be used for some unimplemented operations 2994 * Needed so all function pointers are valid and do not have to be tested 2995 * before use 2996 * 2997 **************************************************************************/ 2998 int efx_port_dummy_op_int(struct efx_nic *efx) 2999 { 3000 return 0; 3001 } 3002 void efx_port_dummy_op_void(struct efx_nic *efx) {} 3003 3004 static bool efx_port_dummy_op_poll(struct efx_nic *efx) 3005 { 3006 return false; 3007 } 3008 3009 static const struct efx_phy_operations efx_dummy_phy_operations = { 3010 .init = efx_port_dummy_op_int, 3011 .reconfigure = efx_port_dummy_op_int, 3012 .poll = efx_port_dummy_op_poll, 3013 .fini = efx_port_dummy_op_void, 3014 }; 3015 3016 /************************************************************************** 3017 * 3018 * Data housekeeping 3019 * 3020 **************************************************************************/ 3021 3022 /* This zeroes out and then fills in the invariants in a struct 3023 * efx_nic (including all sub-structures). 3024 */ 3025 static int efx_init_struct(struct efx_nic *efx, 3026 struct pci_dev *pci_dev, struct net_device *net_dev) 3027 { 3028 int rc = -ENOMEM, i; 3029 3030 /* Initialise common structures */ 3031 INIT_LIST_HEAD(&efx->node); 3032 INIT_LIST_HEAD(&efx->secondary_list); 3033 spin_lock_init(&efx->biu_lock); 3034 #ifdef CONFIG_SFC_MTD 3035 INIT_LIST_HEAD(&efx->mtd_list); 3036 #endif 3037 INIT_WORK(&efx->reset_work, efx_reset_work); 3038 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 3039 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 3040 efx->pci_dev = pci_dev; 3041 efx->msg_enable = debug; 3042 efx->state = STATE_UNINIT; 3043 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 3044 3045 efx->net_dev = net_dev; 3046 efx->rx_prefix_size = efx->type->rx_prefix_size; 3047 efx->rx_ip_align = 3048 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 3049 efx->rx_packet_hash_offset = 3050 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 3051 efx->rx_packet_ts_offset = 3052 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 3053 INIT_LIST_HEAD(&efx->rss_context.list); 3054 mutex_init(&efx->rss_lock); 3055 spin_lock_init(&efx->stats_lock); 3056 efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 3057 efx->num_mac_stats = MC_CMD_MAC_NSTATS; 3058 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 3059 mutex_init(&efx->mac_lock); 3060 #ifdef CONFIG_RFS_ACCEL 3061 mutex_init(&efx->rps_mutex); 3062 spin_lock_init(&efx->rps_hash_lock); 3063 /* Failure to allocate is not fatal, but may degrade ARFS performance */ 3064 efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, 3065 sizeof(*efx->rps_hash_table), GFP_KERNEL); 3066 #endif 3067 efx->phy_op = &efx_dummy_phy_operations; 3068 efx->mdio.dev = net_dev; 3069 INIT_WORK(&efx->mac_work, efx_mac_work); 3070 init_waitqueue_head(&efx->flush_wq); 3071 3072 for (i = 0; i < EFX_MAX_CHANNELS; i++) { 3073 efx->channel[i] = efx_alloc_channel(efx, i, NULL); 3074 if (!efx->channel[i]) 3075 goto fail; 3076 efx->msi_context[i].efx = efx; 3077 efx->msi_context[i].index = i; 3078 } 3079 3080 /* Higher numbered interrupt modes are less capable! */ 3081 if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 3082 efx->type->min_interrupt_mode)) { 3083 rc = -EIO; 3084 goto fail; 3085 } 3086 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 3087 interrupt_mode); 3088 efx->interrupt_mode = min(efx->type->min_interrupt_mode, 3089 interrupt_mode); 3090 3091 /* Would be good to use the net_dev name, but we're too early */ 3092 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 3093 pci_name(pci_dev)); 3094 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 3095 if (!efx->workqueue) 3096 goto fail; 3097 3098 return 0; 3099 3100 fail: 3101 efx_fini_struct(efx); 3102 return rc; 3103 } 3104 3105 static void efx_fini_struct(struct efx_nic *efx) 3106 { 3107 int i; 3108 3109 #ifdef CONFIG_RFS_ACCEL 3110 kfree(efx->rps_hash_table); 3111 #endif 3112 3113 for (i = 0; i < EFX_MAX_CHANNELS; i++) 3114 kfree(efx->channel[i]); 3115 3116 kfree(efx->vpd_sn); 3117 3118 if (efx->workqueue) { 3119 destroy_workqueue(efx->workqueue); 3120 efx->workqueue = NULL; 3121 } 3122 } 3123 3124 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 3125 { 3126 u64 n_rx_nodesc_trunc = 0; 3127 struct efx_channel *channel; 3128 3129 efx_for_each_channel(channel, efx) 3130 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 3131 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 3132 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 3133 } 3134 3135 bool efx_filter_spec_equal(const struct efx_filter_spec *left, 3136 const struct efx_filter_spec *right) 3137 { 3138 if ((left->match_flags ^ right->match_flags) | 3139 ((left->flags ^ right->flags) & 3140 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) 3141 return false; 3142 3143 return memcmp(&left->outer_vid, &right->outer_vid, 3144 sizeof(struct efx_filter_spec) - 3145 offsetof(struct efx_filter_spec, outer_vid)) == 0; 3146 } 3147 3148 u32 efx_filter_spec_hash(const struct efx_filter_spec *spec) 3149 { 3150 BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); 3151 return jhash2((const u32 *)&spec->outer_vid, 3152 (sizeof(struct efx_filter_spec) - 3153 offsetof(struct efx_filter_spec, outer_vid)) / 4, 3154 0); 3155 } 3156 3157 #ifdef CONFIG_RFS_ACCEL 3158 bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx, 3159 bool *force) 3160 { 3161 if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) { 3162 /* ARFS is currently updating this entry, leave it */ 3163 return false; 3164 } 3165 if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) { 3166 /* ARFS tried and failed to update this, so it's probably out 3167 * of date. Remove the filter and the ARFS rule entry. 3168 */ 3169 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 3170 *force = true; 3171 return true; 3172 } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */ 3173 /* ARFS has moved on, so old filter is not needed. Since we did 3174 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will 3175 * not be removed by efx_rps_hash_del() subsequently. 3176 */ 3177 *force = true; 3178 return true; 3179 } 3180 /* Remove it iff ARFS wants to. */ 3181 return true; 3182 } 3183 3184 struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx, 3185 const struct efx_filter_spec *spec) 3186 { 3187 u32 hash = efx_filter_spec_hash(spec); 3188 3189 WARN_ON(!spin_is_locked(&efx->rps_hash_lock)); 3190 if (!efx->rps_hash_table) 3191 return NULL; 3192 return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE]; 3193 } 3194 3195 struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx, 3196 const struct efx_filter_spec *spec) 3197 { 3198 struct efx_arfs_rule *rule; 3199 struct hlist_head *head; 3200 struct hlist_node *node; 3201 3202 head = efx_rps_hash_bucket(efx, spec); 3203 if (!head) 3204 return NULL; 3205 hlist_for_each(node, head) { 3206 rule = container_of(node, struct efx_arfs_rule, node); 3207 if (efx_filter_spec_equal(spec, &rule->spec)) 3208 return rule; 3209 } 3210 return NULL; 3211 } 3212 3213 struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx, 3214 const struct efx_filter_spec *spec, 3215 bool *new) 3216 { 3217 struct efx_arfs_rule *rule; 3218 struct hlist_head *head; 3219 struct hlist_node *node; 3220 3221 head = efx_rps_hash_bucket(efx, spec); 3222 if (!head) 3223 return NULL; 3224 hlist_for_each(node, head) { 3225 rule = container_of(node, struct efx_arfs_rule, node); 3226 if (efx_filter_spec_equal(spec, &rule->spec)) { 3227 *new = false; 3228 return rule; 3229 } 3230 } 3231 rule = kmalloc(sizeof(*rule), GFP_ATOMIC); 3232 *new = true; 3233 if (rule) { 3234 memcpy(&rule->spec, spec, sizeof(rule->spec)); 3235 hlist_add_head(&rule->node, head); 3236 } 3237 return rule; 3238 } 3239 3240 void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec) 3241 { 3242 struct efx_arfs_rule *rule; 3243 struct hlist_head *head; 3244 struct hlist_node *node; 3245 3246 head = efx_rps_hash_bucket(efx, spec); 3247 if (WARN_ON(!head)) 3248 return; 3249 hlist_for_each(node, head) { 3250 rule = container_of(node, struct efx_arfs_rule, node); 3251 if (efx_filter_spec_equal(spec, &rule->spec)) { 3252 /* Someone already reused the entry. We know that if 3253 * this check doesn't fire (i.e. filter_id == REMOVING) 3254 * then the REMOVING mark was put there by our caller, 3255 * because caller is holding a lock on filter table and 3256 * only holders of that lock set REMOVING. 3257 */ 3258 if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING) 3259 return; 3260 hlist_del(node); 3261 kfree(rule); 3262 return; 3263 } 3264 } 3265 /* We didn't find it. */ 3266 WARN_ON(1); 3267 } 3268 #endif 3269 3270 /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because 3271 * (a) this is an infrequent control-plane operation and (b) n is small (max 64) 3272 */ 3273 struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx) 3274 { 3275 struct list_head *head = &efx->rss_context.list; 3276 struct efx_rss_context *ctx, *new; 3277 u32 id = 1; /* Don't use zero, that refers to the master RSS context */ 3278 3279 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3280 3281 /* Search for first gap in the numbering */ 3282 list_for_each_entry(ctx, head, list) { 3283 if (ctx->user_id != id) 3284 break; 3285 id++; 3286 /* Check for wrap. If this happens, we have nearly 2^32 3287 * allocated RSS contexts, which seems unlikely. 3288 */ 3289 if (WARN_ON_ONCE(!id)) 3290 return NULL; 3291 } 3292 3293 /* Create the new entry */ 3294 new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL); 3295 if (!new) 3296 return NULL; 3297 new->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 3298 new->rx_hash_udp_4tuple = false; 3299 3300 /* Insert the new entry into the gap */ 3301 new->user_id = id; 3302 list_add_tail(&new->list, &ctx->list); 3303 return new; 3304 } 3305 3306 struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id) 3307 { 3308 struct list_head *head = &efx->rss_context.list; 3309 struct efx_rss_context *ctx; 3310 3311 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3312 3313 list_for_each_entry(ctx, head, list) 3314 if (ctx->user_id == id) 3315 return ctx; 3316 return NULL; 3317 } 3318 3319 void efx_free_rss_context_entry(struct efx_rss_context *ctx) 3320 { 3321 list_del(&ctx->list); 3322 kfree(ctx); 3323 } 3324 3325 /************************************************************************** 3326 * 3327 * PCI interface 3328 * 3329 **************************************************************************/ 3330 3331 /* Main body of final NIC shutdown code 3332 * This is called only at module unload (or hotplug removal). 3333 */ 3334 static void efx_pci_remove_main(struct efx_nic *efx) 3335 { 3336 /* Flush reset_work. It can no longer be scheduled since we 3337 * are not READY. 3338 */ 3339 BUG_ON(efx->state == STATE_READY); 3340 cancel_work_sync(&efx->reset_work); 3341 3342 efx_disable_interrupts(efx); 3343 efx_clear_interrupt_affinity(efx); 3344 efx_nic_fini_interrupt(efx); 3345 efx_fini_port(efx); 3346 efx->type->fini(efx); 3347 efx_fini_napi(efx); 3348 efx_remove_all(efx); 3349 } 3350 3351 /* Final NIC shutdown 3352 * This is called only at module unload (or hotplug removal). A PF can call 3353 * this on its VFs to ensure they are unbound first. 3354 */ 3355 static void efx_pci_remove(struct pci_dev *pci_dev) 3356 { 3357 struct efx_nic *efx; 3358 3359 efx = pci_get_drvdata(pci_dev); 3360 if (!efx) 3361 return; 3362 3363 /* Mark the NIC as fini, then stop the interface */ 3364 rtnl_lock(); 3365 efx_dissociate(efx); 3366 dev_close(efx->net_dev); 3367 efx_disable_interrupts(efx); 3368 efx->state = STATE_UNINIT; 3369 rtnl_unlock(); 3370 3371 if (efx->type->sriov_fini) 3372 efx->type->sriov_fini(efx); 3373 3374 efx_unregister_netdev(efx); 3375 3376 efx_mtd_remove(efx); 3377 3378 efx_pci_remove_main(efx); 3379 3380 efx_fini_io(efx); 3381 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3382 3383 efx_fini_struct(efx); 3384 free_netdev(efx->net_dev); 3385 3386 pci_disable_pcie_error_reporting(pci_dev); 3387 }; 3388 3389 /* NIC VPD information 3390 * Called during probe to display the part number of the 3391 * installed NIC. VPD is potentially very large but this should 3392 * always appear within the first 512 bytes. 3393 */ 3394 #define SFC_VPD_LEN 512 3395 static void efx_probe_vpd_strings(struct efx_nic *efx) 3396 { 3397 struct pci_dev *dev = efx->pci_dev; 3398 char vpd_data[SFC_VPD_LEN]; 3399 ssize_t vpd_size; 3400 int ro_start, ro_size, i, j; 3401 3402 /* Get the vpd data from the device */ 3403 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3404 if (vpd_size <= 0) { 3405 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3406 return; 3407 } 3408 3409 /* Get the Read only section */ 3410 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3411 if (ro_start < 0) { 3412 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3413 return; 3414 } 3415 3416 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3417 j = ro_size; 3418 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3419 if (i + j > vpd_size) 3420 j = vpd_size - i; 3421 3422 /* Get the Part number */ 3423 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3424 if (i < 0) { 3425 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3426 return; 3427 } 3428 3429 j = pci_vpd_info_field_size(&vpd_data[i]); 3430 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3431 if (i + j > vpd_size) { 3432 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3433 return; 3434 } 3435 3436 netif_info(efx, drv, efx->net_dev, 3437 "Part Number : %.*s\n", j, &vpd_data[i]); 3438 3439 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3440 j = ro_size; 3441 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3442 if (i < 0) { 3443 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3444 return; 3445 } 3446 3447 j = pci_vpd_info_field_size(&vpd_data[i]); 3448 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3449 if (i + j > vpd_size) { 3450 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3451 return; 3452 } 3453 3454 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3455 if (!efx->vpd_sn) 3456 return; 3457 3458 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3459 } 3460 3461 3462 /* Main body of NIC initialisation 3463 * This is called at module load (or hotplug insertion, theoretically). 3464 */ 3465 static int efx_pci_probe_main(struct efx_nic *efx) 3466 { 3467 int rc; 3468 3469 /* Do start-of-day initialisation */ 3470 rc = efx_probe_all(efx); 3471 if (rc) 3472 goto fail1; 3473 3474 efx_init_napi(efx); 3475 3476 rc = efx->type->init(efx); 3477 if (rc) { 3478 netif_err(efx, probe, efx->net_dev, 3479 "failed to initialise NIC\n"); 3480 goto fail3; 3481 } 3482 3483 rc = efx_init_port(efx); 3484 if (rc) { 3485 netif_err(efx, probe, efx->net_dev, 3486 "failed to initialise port\n"); 3487 goto fail4; 3488 } 3489 3490 rc = efx_nic_init_interrupt(efx); 3491 if (rc) 3492 goto fail5; 3493 3494 efx_set_interrupt_affinity(efx); 3495 rc = efx_enable_interrupts(efx); 3496 if (rc) 3497 goto fail6; 3498 3499 return 0; 3500 3501 fail6: 3502 efx_clear_interrupt_affinity(efx); 3503 efx_nic_fini_interrupt(efx); 3504 fail5: 3505 efx_fini_port(efx); 3506 fail4: 3507 efx->type->fini(efx); 3508 fail3: 3509 efx_fini_napi(efx); 3510 efx_remove_all(efx); 3511 fail1: 3512 return rc; 3513 } 3514 3515 static int efx_pci_probe_post_io(struct efx_nic *efx) 3516 { 3517 struct net_device *net_dev = efx->net_dev; 3518 int rc = efx_pci_probe_main(efx); 3519 3520 if (rc) 3521 return rc; 3522 3523 if (efx->type->sriov_init) { 3524 rc = efx->type->sriov_init(efx); 3525 if (rc) 3526 netif_err(efx, probe, efx->net_dev, 3527 "SR-IOV can't be enabled rc %d\n", rc); 3528 } 3529 3530 /* Determine netdevice features */ 3531 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3532 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL); 3533 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 3534 net_dev->features |= NETIF_F_TSO6; 3535 /* Check whether device supports TSO */ 3536 if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 3537 net_dev->features &= ~NETIF_F_ALL_TSO; 3538 /* Mask for features that also apply to VLAN devices */ 3539 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3540 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 3541 NETIF_F_RXCSUM); 3542 3543 net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 3544 3545 /* Disable receiving frames with bad FCS, by default. */ 3546 net_dev->features &= ~NETIF_F_RXALL; 3547 3548 /* Disable VLAN filtering by default. It may be enforced if 3549 * the feature is fixed (i.e. VLAN filters are required to 3550 * receive VLAN tagged packets due to vPort restrictions). 3551 */ 3552 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3553 net_dev->features |= efx->fixed_features; 3554 3555 rc = efx_register_netdev(efx); 3556 if (!rc) 3557 return 0; 3558 3559 efx_pci_remove_main(efx); 3560 return rc; 3561 } 3562 3563 /* NIC initialisation 3564 * 3565 * This is called at module load (or hotplug insertion, 3566 * theoretically). It sets up PCI mappings, resets the NIC, 3567 * sets up and registers the network devices with the kernel and hooks 3568 * the interrupt service routine. It does not prepare the device for 3569 * transmission; this is left to the first time one of the network 3570 * interfaces is brought up (i.e. efx_net_open). 3571 */ 3572 static int efx_pci_probe(struct pci_dev *pci_dev, 3573 const struct pci_device_id *entry) 3574 { 3575 struct net_device *net_dev; 3576 struct efx_nic *efx; 3577 int rc; 3578 3579 /* Allocate and initialise a struct net_device and struct efx_nic */ 3580 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3581 EFX_MAX_RX_QUEUES); 3582 if (!net_dev) 3583 return -ENOMEM; 3584 efx = netdev_priv(net_dev); 3585 efx->type = (const struct efx_nic_type *) entry->driver_data; 3586 efx->fixed_features |= NETIF_F_HIGHDMA; 3587 3588 pci_set_drvdata(pci_dev, efx); 3589 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3590 rc = efx_init_struct(efx, pci_dev, net_dev); 3591 if (rc) 3592 goto fail1; 3593 3594 netif_info(efx, probe, efx->net_dev, 3595 "Solarflare NIC detected\n"); 3596 3597 if (!efx->type->is_vf) 3598 efx_probe_vpd_strings(efx); 3599 3600 /* Set up basic I/O (BAR mappings etc) */ 3601 rc = efx_init_io(efx); 3602 if (rc) 3603 goto fail2; 3604 3605 rc = efx_pci_probe_post_io(efx); 3606 if (rc) { 3607 /* On failure, retry once immediately. 3608 * If we aborted probe due to a scheduled reset, dismiss it. 3609 */ 3610 efx->reset_pending = 0; 3611 rc = efx_pci_probe_post_io(efx); 3612 if (rc) { 3613 /* On another failure, retry once more 3614 * after a 50-305ms delay. 3615 */ 3616 unsigned char r; 3617 3618 get_random_bytes(&r, 1); 3619 msleep((unsigned int)r + 50); 3620 efx->reset_pending = 0; 3621 rc = efx_pci_probe_post_io(efx); 3622 } 3623 } 3624 if (rc) 3625 goto fail3; 3626 3627 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3628 3629 /* Try to create MTDs, but allow this to fail */ 3630 rtnl_lock(); 3631 rc = efx_mtd_probe(efx); 3632 rtnl_unlock(); 3633 if (rc && rc != -EPERM) 3634 netif_warn(efx, probe, efx->net_dev, 3635 "failed to create MTDs (%d)\n", rc); 3636 3637 rc = pci_enable_pcie_error_reporting(pci_dev); 3638 if (rc && rc != -EINVAL) 3639 netif_notice(efx, probe, efx->net_dev, 3640 "PCIE error reporting unavailable (%d).\n", 3641 rc); 3642 3643 if (efx->type->udp_tnl_push_ports) 3644 efx->type->udp_tnl_push_ports(efx); 3645 3646 return 0; 3647 3648 fail3: 3649 efx_fini_io(efx); 3650 fail2: 3651 efx_fini_struct(efx); 3652 fail1: 3653 WARN_ON(rc > 0); 3654 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3655 free_netdev(net_dev); 3656 return rc; 3657 } 3658 3659 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3660 * enabled on success 3661 */ 3662 #ifdef CONFIG_SFC_SRIOV 3663 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3664 { 3665 int rc; 3666 struct efx_nic *efx = pci_get_drvdata(dev); 3667 3668 if (efx->type->sriov_configure) { 3669 rc = efx->type->sriov_configure(efx, num_vfs); 3670 if (rc) 3671 return rc; 3672 else 3673 return num_vfs; 3674 } else 3675 return -EOPNOTSUPP; 3676 } 3677 #endif 3678 3679 static int efx_pm_freeze(struct device *dev) 3680 { 3681 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3682 3683 rtnl_lock(); 3684 3685 if (efx->state != STATE_DISABLED) { 3686 efx->state = STATE_UNINIT; 3687 3688 efx_device_detach_sync(efx); 3689 3690 efx_stop_all(efx); 3691 efx_disable_interrupts(efx); 3692 } 3693 3694 rtnl_unlock(); 3695 3696 return 0; 3697 } 3698 3699 static int efx_pm_thaw(struct device *dev) 3700 { 3701 int rc; 3702 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3703 3704 rtnl_lock(); 3705 3706 if (efx->state != STATE_DISABLED) { 3707 rc = efx_enable_interrupts(efx); 3708 if (rc) 3709 goto fail; 3710 3711 mutex_lock(&efx->mac_lock); 3712 efx->phy_op->reconfigure(efx); 3713 mutex_unlock(&efx->mac_lock); 3714 3715 efx_start_all(efx); 3716 3717 efx_device_attach_if_not_resetting(efx); 3718 3719 efx->state = STATE_READY; 3720 3721 efx->type->resume_wol(efx); 3722 } 3723 3724 rtnl_unlock(); 3725 3726 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3727 queue_work(reset_workqueue, &efx->reset_work); 3728 3729 return 0; 3730 3731 fail: 3732 rtnl_unlock(); 3733 3734 return rc; 3735 } 3736 3737 static int efx_pm_poweroff(struct device *dev) 3738 { 3739 struct pci_dev *pci_dev = to_pci_dev(dev); 3740 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3741 3742 efx->type->fini(efx); 3743 3744 efx->reset_pending = 0; 3745 3746 pci_save_state(pci_dev); 3747 return pci_set_power_state(pci_dev, PCI_D3hot); 3748 } 3749 3750 /* Used for both resume and restore */ 3751 static int efx_pm_resume(struct device *dev) 3752 { 3753 struct pci_dev *pci_dev = to_pci_dev(dev); 3754 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3755 int rc; 3756 3757 rc = pci_set_power_state(pci_dev, PCI_D0); 3758 if (rc) 3759 return rc; 3760 pci_restore_state(pci_dev); 3761 rc = pci_enable_device(pci_dev); 3762 if (rc) 3763 return rc; 3764 pci_set_master(efx->pci_dev); 3765 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3766 if (rc) 3767 return rc; 3768 rc = efx->type->init(efx); 3769 if (rc) 3770 return rc; 3771 rc = efx_pm_thaw(dev); 3772 return rc; 3773 } 3774 3775 static int efx_pm_suspend(struct device *dev) 3776 { 3777 int rc; 3778 3779 efx_pm_freeze(dev); 3780 rc = efx_pm_poweroff(dev); 3781 if (rc) 3782 efx_pm_resume(dev); 3783 return rc; 3784 } 3785 3786 static const struct dev_pm_ops efx_pm_ops = { 3787 .suspend = efx_pm_suspend, 3788 .resume = efx_pm_resume, 3789 .freeze = efx_pm_freeze, 3790 .thaw = efx_pm_thaw, 3791 .poweroff = efx_pm_poweroff, 3792 .restore = efx_pm_resume, 3793 }; 3794 3795 /* A PCI error affecting this device was detected. 3796 * At this point MMIO and DMA may be disabled. 3797 * Stop the software path and request a slot reset. 3798 */ 3799 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3800 enum pci_channel_state state) 3801 { 3802 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3803 struct efx_nic *efx = pci_get_drvdata(pdev); 3804 3805 if (state == pci_channel_io_perm_failure) 3806 return PCI_ERS_RESULT_DISCONNECT; 3807 3808 rtnl_lock(); 3809 3810 if (efx->state != STATE_DISABLED) { 3811 efx->state = STATE_RECOVERY; 3812 efx->reset_pending = 0; 3813 3814 efx_device_detach_sync(efx); 3815 3816 efx_stop_all(efx); 3817 efx_disable_interrupts(efx); 3818 3819 status = PCI_ERS_RESULT_NEED_RESET; 3820 } else { 3821 /* If the interface is disabled we don't want to do anything 3822 * with it. 3823 */ 3824 status = PCI_ERS_RESULT_RECOVERED; 3825 } 3826 3827 rtnl_unlock(); 3828 3829 pci_disable_device(pdev); 3830 3831 return status; 3832 } 3833 3834 /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3835 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3836 { 3837 struct efx_nic *efx = pci_get_drvdata(pdev); 3838 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3839 int rc; 3840 3841 if (pci_enable_device(pdev)) { 3842 netif_err(efx, hw, efx->net_dev, 3843 "Cannot re-enable PCI device after reset.\n"); 3844 status = PCI_ERS_RESULT_DISCONNECT; 3845 } 3846 3847 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3848 if (rc) { 3849 netif_err(efx, hw, efx->net_dev, 3850 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3851 /* Non-fatal error. Continue. */ 3852 } 3853 3854 return status; 3855 } 3856 3857 /* Perform the actual reset and resume I/O operations. */ 3858 static void efx_io_resume(struct pci_dev *pdev) 3859 { 3860 struct efx_nic *efx = pci_get_drvdata(pdev); 3861 int rc; 3862 3863 rtnl_lock(); 3864 3865 if (efx->state == STATE_DISABLED) 3866 goto out; 3867 3868 rc = efx_reset(efx, RESET_TYPE_ALL); 3869 if (rc) { 3870 netif_err(efx, hw, efx->net_dev, 3871 "efx_reset failed after PCI error (%d)\n", rc); 3872 } else { 3873 efx->state = STATE_READY; 3874 netif_dbg(efx, hw, efx->net_dev, 3875 "Done resetting and resuming IO after PCI error.\n"); 3876 } 3877 3878 out: 3879 rtnl_unlock(); 3880 } 3881 3882 /* For simplicity and reliability, we always require a slot reset and try to 3883 * reset the hardware when a pci error affecting the device is detected. 3884 * We leave both the link_reset and mmio_enabled callback unimplemented: 3885 * with our request for slot reset the mmio_enabled callback will never be 3886 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3887 */ 3888 static const struct pci_error_handlers efx_err_handlers = { 3889 .error_detected = efx_io_error_detected, 3890 .slot_reset = efx_io_slot_reset, 3891 .resume = efx_io_resume, 3892 }; 3893 3894 static struct pci_driver efx_pci_driver = { 3895 .name = KBUILD_MODNAME, 3896 .id_table = efx_pci_table, 3897 .probe = efx_pci_probe, 3898 .remove = efx_pci_remove, 3899 .driver.pm = &efx_pm_ops, 3900 .err_handler = &efx_err_handlers, 3901 #ifdef CONFIG_SFC_SRIOV 3902 .sriov_configure = efx_pci_sriov_configure, 3903 #endif 3904 }; 3905 3906 /************************************************************************** 3907 * 3908 * Kernel module interface 3909 * 3910 *************************************************************************/ 3911 3912 module_param(interrupt_mode, uint, 0444); 3913 MODULE_PARM_DESC(interrupt_mode, 3914 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3915 3916 static int __init efx_init_module(void) 3917 { 3918 int rc; 3919 3920 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3921 3922 rc = register_netdevice_notifier(&efx_netdev_notifier); 3923 if (rc) 3924 goto err_notifier; 3925 3926 #ifdef CONFIG_SFC_SRIOV 3927 rc = efx_init_sriov(); 3928 if (rc) 3929 goto err_sriov; 3930 #endif 3931 3932 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3933 if (!reset_workqueue) { 3934 rc = -ENOMEM; 3935 goto err_reset; 3936 } 3937 3938 rc = pci_register_driver(&efx_pci_driver); 3939 if (rc < 0) 3940 goto err_pci; 3941 3942 return 0; 3943 3944 err_pci: 3945 destroy_workqueue(reset_workqueue); 3946 err_reset: 3947 #ifdef CONFIG_SFC_SRIOV 3948 efx_fini_sriov(); 3949 err_sriov: 3950 #endif 3951 unregister_netdevice_notifier(&efx_netdev_notifier); 3952 err_notifier: 3953 return rc; 3954 } 3955 3956 static void __exit efx_exit_module(void) 3957 { 3958 printk(KERN_INFO "Solarflare NET driver unloading\n"); 3959 3960 pci_unregister_driver(&efx_pci_driver); 3961 destroy_workqueue(reset_workqueue); 3962 #ifdef CONFIG_SFC_SRIOV 3963 efx_fini_sriov(); 3964 #endif 3965 unregister_netdevice_notifier(&efx_netdev_notifier); 3966 3967 } 3968 3969 module_init(efx_init_module); 3970 module_exit(efx_exit_module); 3971 3972 MODULE_AUTHOR("Solarflare Communications and " 3973 "Michael Brown <mbrown@fensystems.co.uk>"); 3974 MODULE_DESCRIPTION("Solarflare network driver"); 3975 MODULE_LICENSE("GPL"); 3976 MODULE_DEVICE_TABLE(pci, efx_pci_table); 3977 MODULE_VERSION(EFX_DRIVER_VERSION); 3978