1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include <net/gre.h> 27 #include <net/udp_tunnel.h> 28 #include "efx.h" 29 #include "nic.h" 30 #include "io.h" 31 #include "selftest.h" 32 #include "sriov.h" 33 34 #include "mcdi.h" 35 #include "mcdi_pcol.h" 36 #include "workarounds.h" 37 38 /************************************************************************** 39 * 40 * Type name strings 41 * 42 ************************************************************************** 43 */ 44 45 /* Loopback mode names (see LOOPBACK_MODE()) */ 46 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 47 const char *const efx_loopback_mode_names[] = { 48 [LOOPBACK_NONE] = "NONE", 49 [LOOPBACK_DATA] = "DATAPATH", 50 [LOOPBACK_GMAC] = "GMAC", 51 [LOOPBACK_XGMII] = "XGMII", 52 [LOOPBACK_XGXS] = "XGXS", 53 [LOOPBACK_XAUI] = "XAUI", 54 [LOOPBACK_GMII] = "GMII", 55 [LOOPBACK_SGMII] = "SGMII", 56 [LOOPBACK_XGBR] = "XGBR", 57 [LOOPBACK_XFI] = "XFI", 58 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 59 [LOOPBACK_GMII_FAR] = "GMII_FAR", 60 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 61 [LOOPBACK_XFI_FAR] = "XFI_FAR", 62 [LOOPBACK_GPHY] = "GPHY", 63 [LOOPBACK_PHYXS] = "PHYXS", 64 [LOOPBACK_PCS] = "PCS", 65 [LOOPBACK_PMAPMD] = "PMA/PMD", 66 [LOOPBACK_XPORT] = "XPORT", 67 [LOOPBACK_XGMII_WS] = "XGMII_WS", 68 [LOOPBACK_XAUI_WS] = "XAUI_WS", 69 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 70 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 71 [LOOPBACK_GMII_WS] = "GMII_WS", 72 [LOOPBACK_XFI_WS] = "XFI_WS", 73 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 74 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 75 }; 76 77 const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 78 const char *const efx_reset_type_names[] = { 79 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 80 [RESET_TYPE_ALL] = "ALL", 81 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 82 [RESET_TYPE_WORLD] = "WORLD", 83 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 84 [RESET_TYPE_DATAPATH] = "DATAPATH", 85 [RESET_TYPE_MC_BIST] = "MC_BIST", 86 [RESET_TYPE_DISABLE] = "DISABLE", 87 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 88 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 89 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 90 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 91 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 92 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 93 }; 94 95 /* UDP tunnel type names */ 96 static const char *const efx_udp_tunnel_type_names[] = { 97 [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan", 98 [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve", 99 }; 100 101 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen) 102 { 103 if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) && 104 efx_udp_tunnel_type_names[type] != NULL) 105 snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]); 106 else 107 snprintf(buf, buflen, "type %d", type); 108 } 109 110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 111 * queued onto this work queue. This is not a per-nic work queue, because 112 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 113 */ 114 static struct workqueue_struct *reset_workqueue; 115 116 /* How often and how many times to poll for a reset while waiting for a 117 * BIST that another function started to complete. 118 */ 119 #define BIST_WAIT_DELAY_MS 100 120 #define BIST_WAIT_DELAY_COUNT 100 121 122 /************************************************************************** 123 * 124 * Configurable values 125 * 126 *************************************************************************/ 127 128 /* 129 * Use separate channels for TX and RX events 130 * 131 * Set this to 1 to use separate channels for TX and RX. It allows us 132 * to control interrupt affinity separately for TX and RX. 133 * 134 * This is only used in MSI-X interrupt mode 135 */ 136 bool efx_separate_tx_channels; 137 module_param(efx_separate_tx_channels, bool, 0444); 138 MODULE_PARM_DESC(efx_separate_tx_channels, 139 "Use separate channels for TX and RX"); 140 141 /* This is the weight assigned to each of the (per-channel) virtual 142 * NAPI devices. 143 */ 144 static int napi_weight = 64; 145 146 /* This is the time (in jiffies) between invocations of the hardware 147 * monitor. 148 * On Falcon-based NICs, this will: 149 * - Check the on-board hardware monitor; 150 * - Poll the link state and reconfigure the hardware as necessary. 151 * On Siena-based NICs for power systems with EEH support, this will give EEH a 152 * chance to start. 153 */ 154 static unsigned int efx_monitor_interval = 1 * HZ; 155 156 /* Initial interrupt moderation settings. They can be modified after 157 * module load with ethtool. 158 * 159 * The default for RX should strike a balance between increasing the 160 * round-trip latency and reducing overhead. 161 */ 162 static unsigned int rx_irq_mod_usec = 60; 163 164 /* Initial interrupt moderation settings. They can be modified after 165 * module load with ethtool. 166 * 167 * This default is chosen to ensure that a 10G link does not go idle 168 * while a TX queue is stopped after it has become full. A queue is 169 * restarted when it drops below half full. The time this takes (assuming 170 * worst case 3 descriptors per packet and 1024 descriptors) is 171 * 512 / 3 * 1.2 = 205 usec. 172 */ 173 static unsigned int tx_irq_mod_usec = 150; 174 175 /* This is the first interrupt mode to try out of: 176 * 0 => MSI-X 177 * 1 => MSI 178 * 2 => legacy 179 */ 180 static unsigned int interrupt_mode; 181 182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 183 * i.e. the number of CPUs among which we may distribute simultaneous 184 * interrupt handling. 185 * 186 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 187 * The default (0) means to assign an interrupt to each core. 188 */ 189 static unsigned int rss_cpus; 190 module_param(rss_cpus, uint, 0444); 191 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 192 193 static bool phy_flash_cfg; 194 module_param(phy_flash_cfg, bool, 0644); 195 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 196 197 static unsigned irq_adapt_low_thresh = 8000; 198 module_param(irq_adapt_low_thresh, uint, 0644); 199 MODULE_PARM_DESC(irq_adapt_low_thresh, 200 "Threshold score for reducing IRQ moderation"); 201 202 static unsigned irq_adapt_high_thresh = 16000; 203 module_param(irq_adapt_high_thresh, uint, 0644); 204 MODULE_PARM_DESC(irq_adapt_high_thresh, 205 "Threshold score for increasing IRQ moderation"); 206 207 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 208 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 209 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 210 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 211 module_param(debug, uint, 0); 212 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 213 214 /************************************************************************** 215 * 216 * Utility functions and prototypes 217 * 218 *************************************************************************/ 219 220 static int efx_soft_enable_interrupts(struct efx_nic *efx); 221 static void efx_soft_disable_interrupts(struct efx_nic *efx); 222 static void efx_remove_channel(struct efx_channel *channel); 223 static void efx_remove_channels(struct efx_nic *efx); 224 static const struct efx_channel_type efx_default_channel_type; 225 static void efx_remove_port(struct efx_nic *efx); 226 static void efx_init_napi_channel(struct efx_channel *channel); 227 static void efx_fini_napi(struct efx_nic *efx); 228 static void efx_fini_napi_channel(struct efx_channel *channel); 229 static void efx_fini_struct(struct efx_nic *efx); 230 static void efx_start_all(struct efx_nic *efx); 231 static void efx_stop_all(struct efx_nic *efx); 232 233 #define EFX_ASSERT_RESET_SERIALISED(efx) \ 234 do { \ 235 if ((efx->state == STATE_READY) || \ 236 (efx->state == STATE_RECOVERY) || \ 237 (efx->state == STATE_DISABLED)) \ 238 ASSERT_RTNL(); \ 239 } while (0) 240 241 static int efx_check_disabled(struct efx_nic *efx) 242 { 243 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 244 netif_err(efx, drv, efx->net_dev, 245 "device is disabled due to earlier errors\n"); 246 return -EIO; 247 } 248 return 0; 249 } 250 251 /************************************************************************** 252 * 253 * Event queue processing 254 * 255 *************************************************************************/ 256 257 /* Process channel's event queue 258 * 259 * This function is responsible for processing the event queue of a 260 * single channel. The caller must guarantee that this function will 261 * never be concurrently called more than once on the same channel, 262 * though different channels may be being processed concurrently. 263 */ 264 static int efx_process_channel(struct efx_channel *channel, int budget) 265 { 266 struct efx_tx_queue *tx_queue; 267 int spent; 268 269 if (unlikely(!channel->enabled)) 270 return 0; 271 272 efx_for_each_channel_tx_queue(tx_queue, channel) { 273 tx_queue->pkts_compl = 0; 274 tx_queue->bytes_compl = 0; 275 } 276 277 spent = efx_nic_process_eventq(channel, budget); 278 if (spent && efx_channel_has_rx_queue(channel)) { 279 struct efx_rx_queue *rx_queue = 280 efx_channel_get_rx_queue(channel); 281 282 efx_rx_flush_packet(channel); 283 efx_fast_push_rx_descriptors(rx_queue, true); 284 } 285 286 /* Update BQL */ 287 efx_for_each_channel_tx_queue(tx_queue, channel) { 288 if (tx_queue->bytes_compl) { 289 netdev_tx_completed_queue(tx_queue->core_txq, 290 tx_queue->pkts_compl, tx_queue->bytes_compl); 291 } 292 } 293 294 return spent; 295 } 296 297 /* NAPI poll handler 298 * 299 * NAPI guarantees serialisation of polls of the same device, which 300 * provides the guarantee required by efx_process_channel(). 301 */ 302 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 303 { 304 int step = efx->irq_mod_step_us; 305 306 if (channel->irq_mod_score < irq_adapt_low_thresh) { 307 if (channel->irq_moderation_us > step) { 308 channel->irq_moderation_us -= step; 309 efx->type->push_irq_moderation(channel); 310 } 311 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 312 if (channel->irq_moderation_us < 313 efx->irq_rx_moderation_us) { 314 channel->irq_moderation_us += step; 315 efx->type->push_irq_moderation(channel); 316 } 317 } 318 319 channel->irq_count = 0; 320 channel->irq_mod_score = 0; 321 } 322 323 static int efx_poll(struct napi_struct *napi, int budget) 324 { 325 struct efx_channel *channel = 326 container_of(napi, struct efx_channel, napi_str); 327 struct efx_nic *efx = channel->efx; 328 int spent; 329 330 netif_vdbg(efx, intr, efx->net_dev, 331 "channel %d NAPI poll executing on CPU %d\n", 332 channel->channel, raw_smp_processor_id()); 333 334 spent = efx_process_channel(channel, budget); 335 336 if (spent < budget) { 337 if (efx_channel_has_rx_queue(channel) && 338 efx->irq_rx_adaptive && 339 unlikely(++channel->irq_count == 1000)) { 340 efx_update_irq_mod(efx, channel); 341 } 342 343 #ifdef CONFIG_RFS_ACCEL 344 /* Perhaps expire some ARFS filters */ 345 schedule_work(&channel->filter_work); 346 #endif 347 348 /* There is no race here; although napi_disable() will 349 * only wait for napi_complete(), this isn't a problem 350 * since efx_nic_eventq_read_ack() will have no effect if 351 * interrupts have already been disabled. 352 */ 353 if (napi_complete_done(napi, spent)) 354 efx_nic_eventq_read_ack(channel); 355 } 356 357 return spent; 358 } 359 360 /* Create event queue 361 * Event queue memory allocations are done only once. If the channel 362 * is reset, the memory buffer will be reused; this guards against 363 * errors during channel reset and also simplifies interrupt handling. 364 */ 365 static int efx_probe_eventq(struct efx_channel *channel) 366 { 367 struct efx_nic *efx = channel->efx; 368 unsigned long entries; 369 370 netif_dbg(efx, probe, efx->net_dev, 371 "chan %d create event queue\n", channel->channel); 372 373 /* Build an event queue with room for one event per tx and rx buffer, 374 * plus some extra for link state events and MCDI completions. */ 375 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 376 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 377 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 378 379 return efx_nic_probe_eventq(channel); 380 } 381 382 /* Prepare channel's event queue */ 383 static int efx_init_eventq(struct efx_channel *channel) 384 { 385 struct efx_nic *efx = channel->efx; 386 int rc; 387 388 EFX_WARN_ON_PARANOID(channel->eventq_init); 389 390 netif_dbg(efx, drv, efx->net_dev, 391 "chan %d init event queue\n", channel->channel); 392 393 rc = efx_nic_init_eventq(channel); 394 if (rc == 0) { 395 efx->type->push_irq_moderation(channel); 396 channel->eventq_read_ptr = 0; 397 channel->eventq_init = true; 398 } 399 return rc; 400 } 401 402 /* Enable event queue processing and NAPI */ 403 void efx_start_eventq(struct efx_channel *channel) 404 { 405 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 406 "chan %d start event queue\n", channel->channel); 407 408 /* Make sure the NAPI handler sees the enabled flag set */ 409 channel->enabled = true; 410 smp_wmb(); 411 412 napi_enable(&channel->napi_str); 413 efx_nic_eventq_read_ack(channel); 414 } 415 416 /* Disable event queue processing and NAPI */ 417 void efx_stop_eventq(struct efx_channel *channel) 418 { 419 if (!channel->enabled) 420 return; 421 422 napi_disable(&channel->napi_str); 423 channel->enabled = false; 424 } 425 426 static void efx_fini_eventq(struct efx_channel *channel) 427 { 428 if (!channel->eventq_init) 429 return; 430 431 netif_dbg(channel->efx, drv, channel->efx->net_dev, 432 "chan %d fini event queue\n", channel->channel); 433 434 efx_nic_fini_eventq(channel); 435 channel->eventq_init = false; 436 } 437 438 static void efx_remove_eventq(struct efx_channel *channel) 439 { 440 netif_dbg(channel->efx, drv, channel->efx->net_dev, 441 "chan %d remove event queue\n", channel->channel); 442 443 efx_nic_remove_eventq(channel); 444 } 445 446 /************************************************************************** 447 * 448 * Channel handling 449 * 450 *************************************************************************/ 451 452 /* Allocate and initialise a channel structure. */ 453 static struct efx_channel * 454 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 455 { 456 struct efx_channel *channel; 457 struct efx_rx_queue *rx_queue; 458 struct efx_tx_queue *tx_queue; 459 int j; 460 461 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 462 if (!channel) 463 return NULL; 464 465 channel->efx = efx; 466 channel->channel = i; 467 channel->type = &efx_default_channel_type; 468 469 for (j = 0; j < EFX_TXQ_TYPES; j++) { 470 tx_queue = &channel->tx_queue[j]; 471 tx_queue->efx = efx; 472 tx_queue->queue = i * EFX_TXQ_TYPES + j; 473 tx_queue->channel = channel; 474 } 475 476 #ifdef CONFIG_RFS_ACCEL 477 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 478 #endif 479 480 rx_queue = &channel->rx_queue; 481 rx_queue->efx = efx; 482 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 483 484 return channel; 485 } 486 487 /* Allocate and initialise a channel structure, copying parameters 488 * (but not resources) from an old channel structure. 489 */ 490 static struct efx_channel * 491 efx_copy_channel(const struct efx_channel *old_channel) 492 { 493 struct efx_channel *channel; 494 struct efx_rx_queue *rx_queue; 495 struct efx_tx_queue *tx_queue; 496 int j; 497 498 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 499 if (!channel) 500 return NULL; 501 502 *channel = *old_channel; 503 504 channel->napi_dev = NULL; 505 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 506 channel->napi_str.napi_id = 0; 507 channel->napi_str.state = 0; 508 memset(&channel->eventq, 0, sizeof(channel->eventq)); 509 510 for (j = 0; j < EFX_TXQ_TYPES; j++) { 511 tx_queue = &channel->tx_queue[j]; 512 if (tx_queue->channel) 513 tx_queue->channel = channel; 514 tx_queue->buffer = NULL; 515 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 516 } 517 518 rx_queue = &channel->rx_queue; 519 rx_queue->buffer = NULL; 520 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 521 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 522 #ifdef CONFIG_RFS_ACCEL 523 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 524 #endif 525 526 return channel; 527 } 528 529 static int efx_probe_channel(struct efx_channel *channel) 530 { 531 struct efx_tx_queue *tx_queue; 532 struct efx_rx_queue *rx_queue; 533 int rc; 534 535 netif_dbg(channel->efx, probe, channel->efx->net_dev, 536 "creating channel %d\n", channel->channel); 537 538 rc = channel->type->pre_probe(channel); 539 if (rc) 540 goto fail; 541 542 rc = efx_probe_eventq(channel); 543 if (rc) 544 goto fail; 545 546 efx_for_each_channel_tx_queue(tx_queue, channel) { 547 rc = efx_probe_tx_queue(tx_queue); 548 if (rc) 549 goto fail; 550 } 551 552 efx_for_each_channel_rx_queue(rx_queue, channel) { 553 rc = efx_probe_rx_queue(rx_queue); 554 if (rc) 555 goto fail; 556 } 557 558 return 0; 559 560 fail: 561 efx_remove_channel(channel); 562 return rc; 563 } 564 565 static void 566 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 567 { 568 struct efx_nic *efx = channel->efx; 569 const char *type; 570 int number; 571 572 number = channel->channel; 573 if (efx->tx_channel_offset == 0) { 574 type = ""; 575 } else if (channel->channel < efx->tx_channel_offset) { 576 type = "-rx"; 577 } else { 578 type = "-tx"; 579 number -= efx->tx_channel_offset; 580 } 581 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 582 } 583 584 static void efx_set_channel_names(struct efx_nic *efx) 585 { 586 struct efx_channel *channel; 587 588 efx_for_each_channel(channel, efx) 589 channel->type->get_name(channel, 590 efx->msi_context[channel->channel].name, 591 sizeof(efx->msi_context[0].name)); 592 } 593 594 static int efx_probe_channels(struct efx_nic *efx) 595 { 596 struct efx_channel *channel; 597 int rc; 598 599 /* Restart special buffer allocation */ 600 efx->next_buffer_table = 0; 601 602 /* Probe channels in reverse, so that any 'extra' channels 603 * use the start of the buffer table. This allows the traffic 604 * channels to be resized without moving them or wasting the 605 * entries before them. 606 */ 607 efx_for_each_channel_rev(channel, efx) { 608 rc = efx_probe_channel(channel); 609 if (rc) { 610 netif_err(efx, probe, efx->net_dev, 611 "failed to create channel %d\n", 612 channel->channel); 613 goto fail; 614 } 615 } 616 efx_set_channel_names(efx); 617 618 return 0; 619 620 fail: 621 efx_remove_channels(efx); 622 return rc; 623 } 624 625 /* Channels are shutdown and reinitialised whilst the NIC is running 626 * to propagate configuration changes (mtu, checksum offload), or 627 * to clear hardware error conditions 628 */ 629 static void efx_start_datapath(struct efx_nic *efx) 630 { 631 netdev_features_t old_features = efx->net_dev->features; 632 bool old_rx_scatter = efx->rx_scatter; 633 struct efx_tx_queue *tx_queue; 634 struct efx_rx_queue *rx_queue; 635 struct efx_channel *channel; 636 size_t rx_buf_len; 637 638 /* Calculate the rx buffer allocation parameters required to 639 * support the current MTU, including padding for header 640 * alignment and overruns. 641 */ 642 efx->rx_dma_len = (efx->rx_prefix_size + 643 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 644 efx->type->rx_buffer_padding); 645 rx_buf_len = (sizeof(struct efx_rx_page_state) + 646 efx->rx_ip_align + efx->rx_dma_len); 647 if (rx_buf_len <= PAGE_SIZE) { 648 efx->rx_scatter = efx->type->always_rx_scatter; 649 efx->rx_buffer_order = 0; 650 } else if (efx->type->can_rx_scatter) { 651 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 652 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 653 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 654 EFX_RX_BUF_ALIGNMENT) > 655 PAGE_SIZE); 656 efx->rx_scatter = true; 657 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 658 efx->rx_buffer_order = 0; 659 } else { 660 efx->rx_scatter = false; 661 efx->rx_buffer_order = get_order(rx_buf_len); 662 } 663 664 efx_rx_config_page_split(efx); 665 if (efx->rx_buffer_order) 666 netif_dbg(efx, drv, efx->net_dev, 667 "RX buf len=%u; page order=%u batch=%u\n", 668 efx->rx_dma_len, efx->rx_buffer_order, 669 efx->rx_pages_per_batch); 670 else 671 netif_dbg(efx, drv, efx->net_dev, 672 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 673 efx->rx_dma_len, efx->rx_page_buf_step, 674 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 675 676 /* Restore previously fixed features in hw_features and remove 677 * features which are fixed now 678 */ 679 efx->net_dev->hw_features |= efx->net_dev->features; 680 efx->net_dev->hw_features &= ~efx->fixed_features; 681 efx->net_dev->features |= efx->fixed_features; 682 if (efx->net_dev->features != old_features) 683 netdev_features_change(efx->net_dev); 684 685 /* RX filters may also have scatter-enabled flags */ 686 if (efx->rx_scatter != old_rx_scatter) 687 efx->type->filter_update_rx_scatter(efx); 688 689 /* We must keep at least one descriptor in a TX ring empty. 690 * We could avoid this when the queue size does not exactly 691 * match the hardware ring size, but it's not that important. 692 * Therefore we stop the queue when one more skb might fill 693 * the ring completely. We wake it when half way back to 694 * empty. 695 */ 696 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 697 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 698 699 /* Initialise the channels */ 700 efx_for_each_channel(channel, efx) { 701 efx_for_each_channel_tx_queue(tx_queue, channel) { 702 efx_init_tx_queue(tx_queue); 703 atomic_inc(&efx->active_queues); 704 } 705 706 efx_for_each_channel_rx_queue(rx_queue, channel) { 707 efx_init_rx_queue(rx_queue); 708 atomic_inc(&efx->active_queues); 709 efx_stop_eventq(channel); 710 efx_fast_push_rx_descriptors(rx_queue, false); 711 efx_start_eventq(channel); 712 } 713 714 WARN_ON(channel->rx_pkt_n_frags); 715 } 716 717 efx_ptp_start_datapath(efx); 718 719 if (netif_device_present(efx->net_dev)) 720 netif_tx_wake_all_queues(efx->net_dev); 721 } 722 723 static void efx_stop_datapath(struct efx_nic *efx) 724 { 725 struct efx_channel *channel; 726 struct efx_tx_queue *tx_queue; 727 struct efx_rx_queue *rx_queue; 728 int rc; 729 730 EFX_ASSERT_RESET_SERIALISED(efx); 731 BUG_ON(efx->port_enabled); 732 733 efx_ptp_stop_datapath(efx); 734 735 /* Stop RX refill */ 736 efx_for_each_channel(channel, efx) { 737 efx_for_each_channel_rx_queue(rx_queue, channel) 738 rx_queue->refill_enabled = false; 739 } 740 741 efx_for_each_channel(channel, efx) { 742 /* RX packet processing is pipelined, so wait for the 743 * NAPI handler to complete. At least event queue 0 744 * might be kept active by non-data events, so don't 745 * use napi_synchronize() but actually disable NAPI 746 * temporarily. 747 */ 748 if (efx_channel_has_rx_queue(channel)) { 749 efx_stop_eventq(channel); 750 efx_start_eventq(channel); 751 } 752 } 753 754 rc = efx->type->fini_dmaq(efx); 755 if (rc) { 756 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 757 } else { 758 netif_dbg(efx, drv, efx->net_dev, 759 "successfully flushed all queues\n"); 760 } 761 762 efx_for_each_channel(channel, efx) { 763 efx_for_each_channel_rx_queue(rx_queue, channel) 764 efx_fini_rx_queue(rx_queue); 765 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 766 efx_fini_tx_queue(tx_queue); 767 } 768 } 769 770 static void efx_remove_channel(struct efx_channel *channel) 771 { 772 struct efx_tx_queue *tx_queue; 773 struct efx_rx_queue *rx_queue; 774 775 netif_dbg(channel->efx, drv, channel->efx->net_dev, 776 "destroy chan %d\n", channel->channel); 777 778 efx_for_each_channel_rx_queue(rx_queue, channel) 779 efx_remove_rx_queue(rx_queue); 780 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 781 efx_remove_tx_queue(tx_queue); 782 efx_remove_eventq(channel); 783 channel->type->post_remove(channel); 784 } 785 786 static void efx_remove_channels(struct efx_nic *efx) 787 { 788 struct efx_channel *channel; 789 790 efx_for_each_channel(channel, efx) 791 efx_remove_channel(channel); 792 } 793 794 int 795 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 796 { 797 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 798 u32 old_rxq_entries, old_txq_entries; 799 unsigned i, next_buffer_table = 0; 800 int rc, rc2; 801 802 rc = efx_check_disabled(efx); 803 if (rc) 804 return rc; 805 806 /* Not all channels should be reallocated. We must avoid 807 * reallocating their buffer table entries. 808 */ 809 efx_for_each_channel(channel, efx) { 810 struct efx_rx_queue *rx_queue; 811 struct efx_tx_queue *tx_queue; 812 813 if (channel->type->copy) 814 continue; 815 next_buffer_table = max(next_buffer_table, 816 channel->eventq.index + 817 channel->eventq.entries); 818 efx_for_each_channel_rx_queue(rx_queue, channel) 819 next_buffer_table = max(next_buffer_table, 820 rx_queue->rxd.index + 821 rx_queue->rxd.entries); 822 efx_for_each_channel_tx_queue(tx_queue, channel) 823 next_buffer_table = max(next_buffer_table, 824 tx_queue->txd.index + 825 tx_queue->txd.entries); 826 } 827 828 efx_device_detach_sync(efx); 829 efx_stop_all(efx); 830 efx_soft_disable_interrupts(efx); 831 832 /* Clone channels (where possible) */ 833 memset(other_channel, 0, sizeof(other_channel)); 834 for (i = 0; i < efx->n_channels; i++) { 835 channel = efx->channel[i]; 836 if (channel->type->copy) 837 channel = channel->type->copy(channel); 838 if (!channel) { 839 rc = -ENOMEM; 840 goto out; 841 } 842 other_channel[i] = channel; 843 } 844 845 /* Swap entry counts and channel pointers */ 846 old_rxq_entries = efx->rxq_entries; 847 old_txq_entries = efx->txq_entries; 848 efx->rxq_entries = rxq_entries; 849 efx->txq_entries = txq_entries; 850 for (i = 0; i < efx->n_channels; i++) { 851 channel = efx->channel[i]; 852 efx->channel[i] = other_channel[i]; 853 other_channel[i] = channel; 854 } 855 856 /* Restart buffer table allocation */ 857 efx->next_buffer_table = next_buffer_table; 858 859 for (i = 0; i < efx->n_channels; i++) { 860 channel = efx->channel[i]; 861 if (!channel->type->copy) 862 continue; 863 rc = efx_probe_channel(channel); 864 if (rc) 865 goto rollback; 866 efx_init_napi_channel(efx->channel[i]); 867 } 868 869 out: 870 /* Destroy unused channel structures */ 871 for (i = 0; i < efx->n_channels; i++) { 872 channel = other_channel[i]; 873 if (channel && channel->type->copy) { 874 efx_fini_napi_channel(channel); 875 efx_remove_channel(channel); 876 kfree(channel); 877 } 878 } 879 880 rc2 = efx_soft_enable_interrupts(efx); 881 if (rc2) { 882 rc = rc ? rc : rc2; 883 netif_err(efx, drv, efx->net_dev, 884 "unable to restart interrupts on channel reallocation\n"); 885 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 886 } else { 887 efx_start_all(efx); 888 efx_device_attach_if_not_resetting(efx); 889 } 890 return rc; 891 892 rollback: 893 /* Swap back */ 894 efx->rxq_entries = old_rxq_entries; 895 efx->txq_entries = old_txq_entries; 896 for (i = 0; i < efx->n_channels; i++) { 897 channel = efx->channel[i]; 898 efx->channel[i] = other_channel[i]; 899 other_channel[i] = channel; 900 } 901 goto out; 902 } 903 904 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 905 { 906 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 907 } 908 909 static bool efx_default_channel_want_txqs(struct efx_channel *channel) 910 { 911 return channel->channel - channel->efx->tx_channel_offset < 912 channel->efx->n_tx_channels; 913 } 914 915 static const struct efx_channel_type efx_default_channel_type = { 916 .pre_probe = efx_channel_dummy_op_int, 917 .post_remove = efx_channel_dummy_op_void, 918 .get_name = efx_get_channel_name, 919 .copy = efx_copy_channel, 920 .want_txqs = efx_default_channel_want_txqs, 921 .keep_eventq = false, 922 .want_pio = true, 923 }; 924 925 int efx_channel_dummy_op_int(struct efx_channel *channel) 926 { 927 return 0; 928 } 929 930 void efx_channel_dummy_op_void(struct efx_channel *channel) 931 { 932 } 933 934 /************************************************************************** 935 * 936 * Port handling 937 * 938 **************************************************************************/ 939 940 /* This ensures that the kernel is kept informed (via 941 * netif_carrier_on/off) of the link status, and also maintains the 942 * link status's stop on the port's TX queue. 943 */ 944 void efx_link_status_changed(struct efx_nic *efx) 945 { 946 struct efx_link_state *link_state = &efx->link_state; 947 948 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 949 * that no events are triggered between unregister_netdev() and the 950 * driver unloading. A more general condition is that NETDEV_CHANGE 951 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 952 if (!netif_running(efx->net_dev)) 953 return; 954 955 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 956 efx->n_link_state_changes++; 957 958 if (link_state->up) 959 netif_carrier_on(efx->net_dev); 960 else 961 netif_carrier_off(efx->net_dev); 962 } 963 964 /* Status message for kernel log */ 965 if (link_state->up) 966 netif_info(efx, link, efx->net_dev, 967 "link up at %uMbps %s-duplex (MTU %d)\n", 968 link_state->speed, link_state->fd ? "full" : "half", 969 efx->net_dev->mtu); 970 else 971 netif_info(efx, link, efx->net_dev, "link down\n"); 972 } 973 974 void efx_link_set_advertising(struct efx_nic *efx, 975 const unsigned long *advertising) 976 { 977 memcpy(efx->link_advertising, advertising, 978 sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK())); 979 980 efx->link_advertising[0] |= ADVERTISED_Autoneg; 981 if (advertising[0] & ADVERTISED_Pause) 982 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 983 else 984 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 985 if (advertising[0] & ADVERTISED_Asym_Pause) 986 efx->wanted_fc ^= EFX_FC_TX; 987 } 988 989 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not 990 * force the Autoneg bit on. 991 */ 992 void efx_link_clear_advertising(struct efx_nic *efx) 993 { 994 bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS); 995 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 996 } 997 998 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 999 { 1000 efx->wanted_fc = wanted_fc; 1001 if (efx->link_advertising[0]) { 1002 if (wanted_fc & EFX_FC_RX) 1003 efx->link_advertising[0] |= (ADVERTISED_Pause | 1004 ADVERTISED_Asym_Pause); 1005 else 1006 efx->link_advertising[0] &= ~(ADVERTISED_Pause | 1007 ADVERTISED_Asym_Pause); 1008 if (wanted_fc & EFX_FC_TX) 1009 efx->link_advertising[0] ^= ADVERTISED_Asym_Pause; 1010 } 1011 } 1012 1013 static void efx_fini_port(struct efx_nic *efx); 1014 1015 /* We assume that efx->type->reconfigure_mac will always try to sync RX 1016 * filters and therefore needs to read-lock the filter table against freeing 1017 */ 1018 void efx_mac_reconfigure(struct efx_nic *efx) 1019 { 1020 down_read(&efx->filter_sem); 1021 efx->type->reconfigure_mac(efx); 1022 up_read(&efx->filter_sem); 1023 } 1024 1025 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 1026 * the MAC appropriately. All other PHY configuration changes are pushed 1027 * through phy_op->set_settings(), and pushed asynchronously to the MAC 1028 * through efx_monitor(). 1029 * 1030 * Callers must hold the mac_lock 1031 */ 1032 int __efx_reconfigure_port(struct efx_nic *efx) 1033 { 1034 enum efx_phy_mode phy_mode; 1035 int rc; 1036 1037 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1038 1039 /* Disable PHY transmit in mac level loopbacks */ 1040 phy_mode = efx->phy_mode; 1041 if (LOOPBACK_INTERNAL(efx)) 1042 efx->phy_mode |= PHY_MODE_TX_DISABLED; 1043 else 1044 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1045 1046 rc = efx->type->reconfigure_port(efx); 1047 1048 if (rc) 1049 efx->phy_mode = phy_mode; 1050 1051 return rc; 1052 } 1053 1054 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1055 * disabled. */ 1056 int efx_reconfigure_port(struct efx_nic *efx) 1057 { 1058 int rc; 1059 1060 EFX_ASSERT_RESET_SERIALISED(efx); 1061 1062 mutex_lock(&efx->mac_lock); 1063 rc = __efx_reconfigure_port(efx); 1064 mutex_unlock(&efx->mac_lock); 1065 1066 return rc; 1067 } 1068 1069 /* Asynchronous work item for changing MAC promiscuity and multicast 1070 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1071 * MAC directly. */ 1072 static void efx_mac_work(struct work_struct *data) 1073 { 1074 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1075 1076 mutex_lock(&efx->mac_lock); 1077 if (efx->port_enabled) 1078 efx_mac_reconfigure(efx); 1079 mutex_unlock(&efx->mac_lock); 1080 } 1081 1082 static int efx_probe_port(struct efx_nic *efx) 1083 { 1084 int rc; 1085 1086 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1087 1088 if (phy_flash_cfg) 1089 efx->phy_mode = PHY_MODE_SPECIAL; 1090 1091 /* Connect up MAC/PHY operations table */ 1092 rc = efx->type->probe_port(efx); 1093 if (rc) 1094 return rc; 1095 1096 /* Initialise MAC address to permanent address */ 1097 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1098 1099 return 0; 1100 } 1101 1102 static int efx_init_port(struct efx_nic *efx) 1103 { 1104 int rc; 1105 1106 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1107 1108 mutex_lock(&efx->mac_lock); 1109 1110 rc = efx->phy_op->init(efx); 1111 if (rc) 1112 goto fail1; 1113 1114 efx->port_initialized = true; 1115 1116 /* Reconfigure the MAC before creating dma queues (required for 1117 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1118 efx_mac_reconfigure(efx); 1119 1120 /* Ensure the PHY advertises the correct flow control settings */ 1121 rc = efx->phy_op->reconfigure(efx); 1122 if (rc && rc != -EPERM) 1123 goto fail2; 1124 1125 mutex_unlock(&efx->mac_lock); 1126 return 0; 1127 1128 fail2: 1129 efx->phy_op->fini(efx); 1130 fail1: 1131 mutex_unlock(&efx->mac_lock); 1132 return rc; 1133 } 1134 1135 static void efx_start_port(struct efx_nic *efx) 1136 { 1137 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1138 BUG_ON(efx->port_enabled); 1139 1140 mutex_lock(&efx->mac_lock); 1141 efx->port_enabled = true; 1142 1143 /* Ensure MAC ingress/egress is enabled */ 1144 efx_mac_reconfigure(efx); 1145 1146 mutex_unlock(&efx->mac_lock); 1147 } 1148 1149 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1150 * and the async self-test, wait for them to finish and prevent them 1151 * being scheduled again. This doesn't cover online resets, which 1152 * should only be cancelled when removing the device. 1153 */ 1154 static void efx_stop_port(struct efx_nic *efx) 1155 { 1156 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1157 1158 EFX_ASSERT_RESET_SERIALISED(efx); 1159 1160 mutex_lock(&efx->mac_lock); 1161 efx->port_enabled = false; 1162 mutex_unlock(&efx->mac_lock); 1163 1164 /* Serialise against efx_set_multicast_list() */ 1165 netif_addr_lock_bh(efx->net_dev); 1166 netif_addr_unlock_bh(efx->net_dev); 1167 1168 cancel_delayed_work_sync(&efx->monitor_work); 1169 efx_selftest_async_cancel(efx); 1170 cancel_work_sync(&efx->mac_work); 1171 } 1172 1173 static void efx_fini_port(struct efx_nic *efx) 1174 { 1175 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1176 1177 if (!efx->port_initialized) 1178 return; 1179 1180 efx->phy_op->fini(efx); 1181 efx->port_initialized = false; 1182 1183 efx->link_state.up = false; 1184 efx_link_status_changed(efx); 1185 } 1186 1187 static void efx_remove_port(struct efx_nic *efx) 1188 { 1189 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1190 1191 efx->type->remove_port(efx); 1192 } 1193 1194 /************************************************************************** 1195 * 1196 * NIC handling 1197 * 1198 **************************************************************************/ 1199 1200 static LIST_HEAD(efx_primary_list); 1201 static LIST_HEAD(efx_unassociated_list); 1202 1203 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 1204 { 1205 return left->type == right->type && 1206 left->vpd_sn && right->vpd_sn && 1207 !strcmp(left->vpd_sn, right->vpd_sn); 1208 } 1209 1210 static void efx_associate(struct efx_nic *efx) 1211 { 1212 struct efx_nic *other, *next; 1213 1214 if (efx->primary == efx) { 1215 /* Adding primary function; look for secondaries */ 1216 1217 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1218 list_add_tail(&efx->node, &efx_primary_list); 1219 1220 list_for_each_entry_safe(other, next, &efx_unassociated_list, 1221 node) { 1222 if (efx_same_controller(efx, other)) { 1223 list_del(&other->node); 1224 netif_dbg(other, probe, other->net_dev, 1225 "moving to secondary list of %s %s\n", 1226 pci_name(efx->pci_dev), 1227 efx->net_dev->name); 1228 list_add_tail(&other->node, 1229 &efx->secondary_list); 1230 other->primary = efx; 1231 } 1232 } 1233 } else { 1234 /* Adding secondary function; look for primary */ 1235 1236 list_for_each_entry(other, &efx_primary_list, node) { 1237 if (efx_same_controller(efx, other)) { 1238 netif_dbg(efx, probe, efx->net_dev, 1239 "adding to secondary list of %s %s\n", 1240 pci_name(other->pci_dev), 1241 other->net_dev->name); 1242 list_add_tail(&efx->node, 1243 &other->secondary_list); 1244 efx->primary = other; 1245 return; 1246 } 1247 } 1248 1249 netif_dbg(efx, probe, efx->net_dev, 1250 "adding to unassociated list\n"); 1251 list_add_tail(&efx->node, &efx_unassociated_list); 1252 } 1253 } 1254 1255 static void efx_dissociate(struct efx_nic *efx) 1256 { 1257 struct efx_nic *other, *next; 1258 1259 list_del(&efx->node); 1260 efx->primary = NULL; 1261 1262 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1263 list_del(&other->node); 1264 netif_dbg(other, probe, other->net_dev, 1265 "moving to unassociated list\n"); 1266 list_add_tail(&other->node, &efx_unassociated_list); 1267 other->primary = NULL; 1268 } 1269 } 1270 1271 /* This configures the PCI device to enable I/O and DMA. */ 1272 static int efx_init_io(struct efx_nic *efx) 1273 { 1274 struct pci_dev *pci_dev = efx->pci_dev; 1275 dma_addr_t dma_mask = efx->type->max_dma_mask; 1276 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1277 int rc, bar; 1278 1279 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1280 1281 bar = efx->type->mem_bar(efx); 1282 1283 rc = pci_enable_device(pci_dev); 1284 if (rc) { 1285 netif_err(efx, probe, efx->net_dev, 1286 "failed to enable PCI device\n"); 1287 goto fail1; 1288 } 1289 1290 pci_set_master(pci_dev); 1291 1292 /* Set the PCI DMA mask. Try all possibilities from our genuine mask 1293 * down to 32 bits, because some architectures will allow 40 bit 1294 * masks event though they reject 46 bit masks. 1295 */ 1296 while (dma_mask > 0x7fffffffUL) { 1297 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1298 if (rc == 0) 1299 break; 1300 dma_mask >>= 1; 1301 } 1302 if (rc) { 1303 netif_err(efx, probe, efx->net_dev, 1304 "could not find a suitable DMA mask\n"); 1305 goto fail2; 1306 } 1307 netif_dbg(efx, probe, efx->net_dev, 1308 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1309 1310 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1311 rc = pci_request_region(pci_dev, bar, "sfc"); 1312 if (rc) { 1313 netif_err(efx, probe, efx->net_dev, 1314 "request for memory BAR failed\n"); 1315 rc = -EIO; 1316 goto fail3; 1317 } 1318 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1319 if (!efx->membase) { 1320 netif_err(efx, probe, efx->net_dev, 1321 "could not map memory BAR at %llx+%x\n", 1322 (unsigned long long)efx->membase_phys, mem_map_size); 1323 rc = -ENOMEM; 1324 goto fail4; 1325 } 1326 netif_dbg(efx, probe, efx->net_dev, 1327 "memory BAR at %llx+%x (virtual %p)\n", 1328 (unsigned long long)efx->membase_phys, mem_map_size, 1329 efx->membase); 1330 1331 return 0; 1332 1333 fail4: 1334 pci_release_region(efx->pci_dev, bar); 1335 fail3: 1336 efx->membase_phys = 0; 1337 fail2: 1338 pci_disable_device(efx->pci_dev); 1339 fail1: 1340 return rc; 1341 } 1342 1343 static void efx_fini_io(struct efx_nic *efx) 1344 { 1345 int bar; 1346 1347 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1348 1349 if (efx->membase) { 1350 iounmap(efx->membase); 1351 efx->membase = NULL; 1352 } 1353 1354 if (efx->membase_phys) { 1355 bar = efx->type->mem_bar(efx); 1356 pci_release_region(efx->pci_dev, bar); 1357 efx->membase_phys = 0; 1358 } 1359 1360 /* Don't disable bus-mastering if VFs are assigned */ 1361 if (!pci_vfs_assigned(efx->pci_dev)) 1362 pci_disable_device(efx->pci_dev); 1363 } 1364 1365 void efx_set_default_rx_indir_table(struct efx_nic *efx, 1366 struct efx_rss_context *ctx) 1367 { 1368 size_t i; 1369 1370 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 1371 ctx->rx_indir_table[i] = 1372 ethtool_rxfh_indir_default(i, efx->rss_spread); 1373 } 1374 1375 static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1376 { 1377 cpumask_var_t thread_mask; 1378 unsigned int count; 1379 int cpu; 1380 1381 if (rss_cpus) { 1382 count = rss_cpus; 1383 } else { 1384 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1385 netif_warn(efx, probe, efx->net_dev, 1386 "RSS disabled due to allocation failure\n"); 1387 return 1; 1388 } 1389 1390 count = 0; 1391 for_each_online_cpu(cpu) { 1392 if (!cpumask_test_cpu(cpu, thread_mask)) { 1393 ++count; 1394 cpumask_or(thread_mask, thread_mask, 1395 topology_sibling_cpumask(cpu)); 1396 } 1397 } 1398 1399 free_cpumask_var(thread_mask); 1400 } 1401 1402 if (count > EFX_MAX_RX_QUEUES) { 1403 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1404 "Reducing number of rx queues from %u to %u.\n", 1405 count, EFX_MAX_RX_QUEUES); 1406 count = EFX_MAX_RX_QUEUES; 1407 } 1408 1409 /* If RSS is requested for the PF *and* VFs then we can't write RSS 1410 * table entries that are inaccessible to VFs 1411 */ 1412 #ifdef CONFIG_SFC_SRIOV 1413 if (efx->type->sriov_wanted) { 1414 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1415 count > efx_vf_size(efx)) { 1416 netif_warn(efx, probe, efx->net_dev, 1417 "Reducing number of RSS channels from %u to %u for " 1418 "VF support. Increase vf-msix-limit to use more " 1419 "channels on the PF.\n", 1420 count, efx_vf_size(efx)); 1421 count = efx_vf_size(efx); 1422 } 1423 } 1424 #endif 1425 1426 return count; 1427 } 1428 1429 /* Probe the number and type of interrupts we are able to obtain, and 1430 * the resulting numbers of channels and RX queues. 1431 */ 1432 static int efx_probe_interrupts(struct efx_nic *efx) 1433 { 1434 unsigned int extra_channels = 0; 1435 unsigned int i, j; 1436 int rc; 1437 1438 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 1439 if (efx->extra_channel_type[i]) 1440 ++extra_channels; 1441 1442 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1443 struct msix_entry xentries[EFX_MAX_CHANNELS]; 1444 unsigned int n_channels; 1445 1446 n_channels = efx_wanted_parallelism(efx); 1447 if (efx_separate_tx_channels) 1448 n_channels *= 2; 1449 n_channels += extra_channels; 1450 n_channels = min(n_channels, efx->max_channels); 1451 1452 for (i = 0; i < n_channels; i++) 1453 xentries[i].entry = i; 1454 rc = pci_enable_msix_range(efx->pci_dev, 1455 xentries, 1, n_channels); 1456 if (rc < 0) { 1457 /* Fall back to single channel MSI */ 1458 netif_err(efx, drv, efx->net_dev, 1459 "could not enable MSI-X\n"); 1460 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 1461 efx->interrupt_mode = EFX_INT_MODE_MSI; 1462 else 1463 return rc; 1464 } else if (rc < n_channels) { 1465 netif_err(efx, drv, efx->net_dev, 1466 "WARNING: Insufficient MSI-X vectors" 1467 " available (%d < %u).\n", rc, n_channels); 1468 netif_err(efx, drv, efx->net_dev, 1469 "WARNING: Performance may be reduced.\n"); 1470 n_channels = rc; 1471 } 1472 1473 if (rc > 0) { 1474 efx->n_channels = n_channels; 1475 if (n_channels > extra_channels) 1476 n_channels -= extra_channels; 1477 if (efx_separate_tx_channels) { 1478 efx->n_tx_channels = min(max(n_channels / 2, 1479 1U), 1480 efx->max_tx_channels); 1481 efx->n_rx_channels = max(n_channels - 1482 efx->n_tx_channels, 1483 1U); 1484 } else { 1485 efx->n_tx_channels = min(n_channels, 1486 efx->max_tx_channels); 1487 efx->n_rx_channels = n_channels; 1488 } 1489 for (i = 0; i < efx->n_channels; i++) 1490 efx_get_channel(efx, i)->irq = 1491 xentries[i].vector; 1492 } 1493 } 1494 1495 /* Try single interrupt MSI */ 1496 if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1497 efx->n_channels = 1; 1498 efx->n_rx_channels = 1; 1499 efx->n_tx_channels = 1; 1500 rc = pci_enable_msi(efx->pci_dev); 1501 if (rc == 0) { 1502 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1503 } else { 1504 netif_err(efx, drv, efx->net_dev, 1505 "could not enable MSI\n"); 1506 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 1507 efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1508 else 1509 return rc; 1510 } 1511 } 1512 1513 /* Assume legacy interrupts */ 1514 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1515 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1516 efx->n_rx_channels = 1; 1517 efx->n_tx_channels = 1; 1518 efx->legacy_irq = efx->pci_dev->irq; 1519 } 1520 1521 /* Assign extra channels if possible */ 1522 efx->n_extra_tx_channels = 0; 1523 j = efx->n_channels; 1524 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 1525 if (!efx->extra_channel_type[i]) 1526 continue; 1527 if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 1528 efx->n_channels <= extra_channels) { 1529 efx->extra_channel_type[i]->handle_no_channel(efx); 1530 } else { 1531 --j; 1532 efx_get_channel(efx, j)->type = 1533 efx->extra_channel_type[i]; 1534 if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 1535 efx->n_extra_tx_channels++; 1536 } 1537 } 1538 1539 /* RSS might be usable on VFs even if it is disabled on the PF */ 1540 #ifdef CONFIG_SFC_SRIOV 1541 if (efx->type->sriov_wanted) { 1542 efx->rss_spread = ((efx->n_rx_channels > 1 || 1543 !efx->type->sriov_wanted(efx)) ? 1544 efx->n_rx_channels : efx_vf_size(efx)); 1545 return 0; 1546 } 1547 #endif 1548 efx->rss_spread = efx->n_rx_channels; 1549 1550 return 0; 1551 } 1552 1553 #if defined(CONFIG_SMP) 1554 static void efx_set_interrupt_affinity(struct efx_nic *efx) 1555 { 1556 struct efx_channel *channel; 1557 unsigned int cpu; 1558 1559 efx_for_each_channel(channel, efx) { 1560 cpu = cpumask_local_spread(channel->channel, 1561 pcibus_to_node(efx->pci_dev->bus)); 1562 irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 1563 } 1564 } 1565 1566 static void efx_clear_interrupt_affinity(struct efx_nic *efx) 1567 { 1568 struct efx_channel *channel; 1569 1570 efx_for_each_channel(channel, efx) 1571 irq_set_affinity_hint(channel->irq, NULL); 1572 } 1573 #else 1574 static void 1575 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1576 { 1577 } 1578 1579 static void 1580 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1581 { 1582 } 1583 #endif /* CONFIG_SMP */ 1584 1585 static int efx_soft_enable_interrupts(struct efx_nic *efx) 1586 { 1587 struct efx_channel *channel, *end_channel; 1588 int rc; 1589 1590 BUG_ON(efx->state == STATE_DISABLED); 1591 1592 efx->irq_soft_enabled = true; 1593 smp_wmb(); 1594 1595 efx_for_each_channel(channel, efx) { 1596 if (!channel->type->keep_eventq) { 1597 rc = efx_init_eventq(channel); 1598 if (rc) 1599 goto fail; 1600 } 1601 efx_start_eventq(channel); 1602 } 1603 1604 efx_mcdi_mode_event(efx); 1605 1606 return 0; 1607 fail: 1608 end_channel = channel; 1609 efx_for_each_channel(channel, efx) { 1610 if (channel == end_channel) 1611 break; 1612 efx_stop_eventq(channel); 1613 if (!channel->type->keep_eventq) 1614 efx_fini_eventq(channel); 1615 } 1616 1617 return rc; 1618 } 1619 1620 static void efx_soft_disable_interrupts(struct efx_nic *efx) 1621 { 1622 struct efx_channel *channel; 1623 1624 if (efx->state == STATE_DISABLED) 1625 return; 1626 1627 efx_mcdi_mode_poll(efx); 1628 1629 efx->irq_soft_enabled = false; 1630 smp_wmb(); 1631 1632 if (efx->legacy_irq) 1633 synchronize_irq(efx->legacy_irq); 1634 1635 efx_for_each_channel(channel, efx) { 1636 if (channel->irq) 1637 synchronize_irq(channel->irq); 1638 1639 efx_stop_eventq(channel); 1640 if (!channel->type->keep_eventq) 1641 efx_fini_eventq(channel); 1642 } 1643 1644 /* Flush the asynchronous MCDI request queue */ 1645 efx_mcdi_flush_async(efx); 1646 } 1647 1648 static int efx_enable_interrupts(struct efx_nic *efx) 1649 { 1650 struct efx_channel *channel, *end_channel; 1651 int rc; 1652 1653 BUG_ON(efx->state == STATE_DISABLED); 1654 1655 if (efx->eeh_disabled_legacy_irq) { 1656 enable_irq(efx->legacy_irq); 1657 efx->eeh_disabled_legacy_irq = false; 1658 } 1659 1660 efx->type->irq_enable_master(efx); 1661 1662 efx_for_each_channel(channel, efx) { 1663 if (channel->type->keep_eventq) { 1664 rc = efx_init_eventq(channel); 1665 if (rc) 1666 goto fail; 1667 } 1668 } 1669 1670 rc = efx_soft_enable_interrupts(efx); 1671 if (rc) 1672 goto fail; 1673 1674 return 0; 1675 1676 fail: 1677 end_channel = channel; 1678 efx_for_each_channel(channel, efx) { 1679 if (channel == end_channel) 1680 break; 1681 if (channel->type->keep_eventq) 1682 efx_fini_eventq(channel); 1683 } 1684 1685 efx->type->irq_disable_non_ev(efx); 1686 1687 return rc; 1688 } 1689 1690 static void efx_disable_interrupts(struct efx_nic *efx) 1691 { 1692 struct efx_channel *channel; 1693 1694 efx_soft_disable_interrupts(efx); 1695 1696 efx_for_each_channel(channel, efx) { 1697 if (channel->type->keep_eventq) 1698 efx_fini_eventq(channel); 1699 } 1700 1701 efx->type->irq_disable_non_ev(efx); 1702 } 1703 1704 static void efx_remove_interrupts(struct efx_nic *efx) 1705 { 1706 struct efx_channel *channel; 1707 1708 /* Remove MSI/MSI-X interrupts */ 1709 efx_for_each_channel(channel, efx) 1710 channel->irq = 0; 1711 pci_disable_msi(efx->pci_dev); 1712 pci_disable_msix(efx->pci_dev); 1713 1714 /* Remove legacy interrupt */ 1715 efx->legacy_irq = 0; 1716 } 1717 1718 static void efx_set_channels(struct efx_nic *efx) 1719 { 1720 struct efx_channel *channel; 1721 struct efx_tx_queue *tx_queue; 1722 1723 efx->tx_channel_offset = 1724 efx_separate_tx_channels ? 1725 efx->n_channels - efx->n_tx_channels : 0; 1726 1727 /* We need to mark which channels really have RX and TX 1728 * queues, and adjust the TX queue numbers if we have separate 1729 * RX-only and TX-only channels. 1730 */ 1731 efx_for_each_channel(channel, efx) { 1732 if (channel->channel < efx->n_rx_channels) 1733 channel->rx_queue.core_index = channel->channel; 1734 else 1735 channel->rx_queue.core_index = -1; 1736 1737 efx_for_each_channel_tx_queue(tx_queue, channel) 1738 tx_queue->queue -= (efx->tx_channel_offset * 1739 EFX_TXQ_TYPES); 1740 } 1741 } 1742 1743 static int efx_probe_nic(struct efx_nic *efx) 1744 { 1745 int rc; 1746 1747 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1748 1749 /* Carry out hardware-type specific initialisation */ 1750 rc = efx->type->probe(efx); 1751 if (rc) 1752 return rc; 1753 1754 do { 1755 if (!efx->max_channels || !efx->max_tx_channels) { 1756 netif_err(efx, drv, efx->net_dev, 1757 "Insufficient resources to allocate" 1758 " any channels\n"); 1759 rc = -ENOSPC; 1760 goto fail1; 1761 } 1762 1763 /* Determine the number of channels and queues by trying 1764 * to hook in MSI-X interrupts. 1765 */ 1766 rc = efx_probe_interrupts(efx); 1767 if (rc) 1768 goto fail1; 1769 1770 efx_set_channels(efx); 1771 1772 /* dimension_resources can fail with EAGAIN */ 1773 rc = efx->type->dimension_resources(efx); 1774 if (rc != 0 && rc != -EAGAIN) 1775 goto fail2; 1776 1777 if (rc == -EAGAIN) 1778 /* try again with new max_channels */ 1779 efx_remove_interrupts(efx); 1780 1781 } while (rc == -EAGAIN); 1782 1783 if (efx->n_channels > 1) 1784 netdev_rss_key_fill(efx->rss_context.rx_hash_key, 1785 sizeof(efx->rss_context.rx_hash_key)); 1786 efx_set_default_rx_indir_table(efx, &efx->rss_context); 1787 1788 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1789 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1790 1791 /* Initialise the interrupt moderation settings */ 1792 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1793 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1794 true); 1795 1796 return 0; 1797 1798 fail2: 1799 efx_remove_interrupts(efx); 1800 fail1: 1801 efx->type->remove(efx); 1802 return rc; 1803 } 1804 1805 static void efx_remove_nic(struct efx_nic *efx) 1806 { 1807 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1808 1809 efx_remove_interrupts(efx); 1810 efx->type->remove(efx); 1811 } 1812 1813 static int efx_probe_filters(struct efx_nic *efx) 1814 { 1815 int rc; 1816 1817 init_rwsem(&efx->filter_sem); 1818 mutex_lock(&efx->mac_lock); 1819 down_write(&efx->filter_sem); 1820 rc = efx->type->filter_table_probe(efx); 1821 if (rc) 1822 goto out_unlock; 1823 1824 #ifdef CONFIG_RFS_ACCEL 1825 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1826 struct efx_channel *channel; 1827 int i, success = 1; 1828 1829 efx_for_each_channel(channel, efx) { 1830 channel->rps_flow_id = 1831 kcalloc(efx->type->max_rx_ip_filters, 1832 sizeof(*channel->rps_flow_id), 1833 GFP_KERNEL); 1834 if (!channel->rps_flow_id) 1835 success = 0; 1836 else 1837 for (i = 0; 1838 i < efx->type->max_rx_ip_filters; 1839 ++i) 1840 channel->rps_flow_id[i] = 1841 RPS_FLOW_ID_INVALID; 1842 } 1843 1844 if (!success) { 1845 efx_for_each_channel(channel, efx) 1846 kfree(channel->rps_flow_id); 1847 efx->type->filter_table_remove(efx); 1848 rc = -ENOMEM; 1849 goto out_unlock; 1850 } 1851 1852 efx->rps_expire_index = efx->rps_expire_channel = 0; 1853 } 1854 #endif 1855 out_unlock: 1856 up_write(&efx->filter_sem); 1857 mutex_unlock(&efx->mac_lock); 1858 return rc; 1859 } 1860 1861 static void efx_remove_filters(struct efx_nic *efx) 1862 { 1863 #ifdef CONFIG_RFS_ACCEL 1864 struct efx_channel *channel; 1865 1866 efx_for_each_channel(channel, efx) 1867 kfree(channel->rps_flow_id); 1868 #endif 1869 down_write(&efx->filter_sem); 1870 efx->type->filter_table_remove(efx); 1871 up_write(&efx->filter_sem); 1872 } 1873 1874 static void efx_restore_filters(struct efx_nic *efx) 1875 { 1876 down_read(&efx->filter_sem); 1877 efx->type->filter_table_restore(efx); 1878 up_read(&efx->filter_sem); 1879 } 1880 1881 /************************************************************************** 1882 * 1883 * NIC startup/shutdown 1884 * 1885 *************************************************************************/ 1886 1887 static int efx_probe_all(struct efx_nic *efx) 1888 { 1889 int rc; 1890 1891 rc = efx_probe_nic(efx); 1892 if (rc) { 1893 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1894 goto fail1; 1895 } 1896 1897 rc = efx_probe_port(efx); 1898 if (rc) { 1899 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1900 goto fail2; 1901 } 1902 1903 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 1904 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 1905 rc = -EINVAL; 1906 goto fail3; 1907 } 1908 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1909 1910 #ifdef CONFIG_SFC_SRIOV 1911 rc = efx->type->vswitching_probe(efx); 1912 if (rc) /* not fatal; the PF will still work fine */ 1913 netif_warn(efx, probe, efx->net_dev, 1914 "failed to setup vswitching rc=%d;" 1915 " VFs may not function\n", rc); 1916 #endif 1917 1918 rc = efx_probe_filters(efx); 1919 if (rc) { 1920 netif_err(efx, probe, efx->net_dev, 1921 "failed to create filter tables\n"); 1922 goto fail4; 1923 } 1924 1925 rc = efx_probe_channels(efx); 1926 if (rc) 1927 goto fail5; 1928 1929 return 0; 1930 1931 fail5: 1932 efx_remove_filters(efx); 1933 fail4: 1934 #ifdef CONFIG_SFC_SRIOV 1935 efx->type->vswitching_remove(efx); 1936 #endif 1937 fail3: 1938 efx_remove_port(efx); 1939 fail2: 1940 efx_remove_nic(efx); 1941 fail1: 1942 return rc; 1943 } 1944 1945 /* If the interface is supposed to be running but is not, start 1946 * the hardware and software data path, regular activity for the port 1947 * (MAC statistics, link polling, etc.) and schedule the port to be 1948 * reconfigured. Interrupts must already be enabled. This function 1949 * is safe to call multiple times, so long as the NIC is not disabled. 1950 * Requires the RTNL lock. 1951 */ 1952 static void efx_start_all(struct efx_nic *efx) 1953 { 1954 EFX_ASSERT_RESET_SERIALISED(efx); 1955 BUG_ON(efx->state == STATE_DISABLED); 1956 1957 /* Check that it is appropriate to restart the interface. All 1958 * of these flags are safe to read under just the rtnl lock */ 1959 if (efx->port_enabled || !netif_running(efx->net_dev) || 1960 efx->reset_pending) 1961 return; 1962 1963 efx_start_port(efx); 1964 efx_start_datapath(efx); 1965 1966 /* Start the hardware monitor if there is one */ 1967 if (efx->type->monitor != NULL) 1968 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1969 efx_monitor_interval); 1970 1971 /* Link state detection is normally event-driven; we have 1972 * to poll now because we could have missed a change 1973 */ 1974 mutex_lock(&efx->mac_lock); 1975 if (efx->phy_op->poll(efx)) 1976 efx_link_status_changed(efx); 1977 mutex_unlock(&efx->mac_lock); 1978 1979 efx->type->start_stats(efx); 1980 efx->type->pull_stats(efx); 1981 spin_lock_bh(&efx->stats_lock); 1982 efx->type->update_stats(efx, NULL, NULL); 1983 spin_unlock_bh(&efx->stats_lock); 1984 } 1985 1986 /* Quiesce the hardware and software data path, and regular activity 1987 * for the port without bringing the link down. Safe to call multiple 1988 * times with the NIC in almost any state, but interrupts should be 1989 * enabled. Requires the RTNL lock. 1990 */ 1991 static void efx_stop_all(struct efx_nic *efx) 1992 { 1993 EFX_ASSERT_RESET_SERIALISED(efx); 1994 1995 /* port_enabled can be read safely under the rtnl lock */ 1996 if (!efx->port_enabled) 1997 return; 1998 1999 /* update stats before we go down so we can accurately count 2000 * rx_nodesc_drops 2001 */ 2002 efx->type->pull_stats(efx); 2003 spin_lock_bh(&efx->stats_lock); 2004 efx->type->update_stats(efx, NULL, NULL); 2005 spin_unlock_bh(&efx->stats_lock); 2006 efx->type->stop_stats(efx); 2007 efx_stop_port(efx); 2008 2009 /* Stop the kernel transmit interface. This is only valid if 2010 * the device is stopped or detached; otherwise the watchdog 2011 * may fire immediately. 2012 */ 2013 WARN_ON(netif_running(efx->net_dev) && 2014 netif_device_present(efx->net_dev)); 2015 netif_tx_disable(efx->net_dev); 2016 2017 efx_stop_datapath(efx); 2018 } 2019 2020 static void efx_remove_all(struct efx_nic *efx) 2021 { 2022 efx_remove_channels(efx); 2023 efx_remove_filters(efx); 2024 #ifdef CONFIG_SFC_SRIOV 2025 efx->type->vswitching_remove(efx); 2026 #endif 2027 efx_remove_port(efx); 2028 efx_remove_nic(efx); 2029 } 2030 2031 /************************************************************************** 2032 * 2033 * Interrupt moderation 2034 * 2035 **************************************************************************/ 2036 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 2037 { 2038 if (usecs == 0) 2039 return 0; 2040 if (usecs * 1000 < efx->timer_quantum_ns) 2041 return 1; /* never round down to 0 */ 2042 return usecs * 1000 / efx->timer_quantum_ns; 2043 } 2044 2045 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 2046 { 2047 /* We must round up when converting ticks to microseconds 2048 * because we round down when converting the other way. 2049 */ 2050 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 2051 } 2052 2053 /* Set interrupt moderation parameters */ 2054 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 2055 unsigned int rx_usecs, bool rx_adaptive, 2056 bool rx_may_override_tx) 2057 { 2058 struct efx_channel *channel; 2059 unsigned int timer_max_us; 2060 2061 EFX_ASSERT_RESET_SERIALISED(efx); 2062 2063 timer_max_us = efx->timer_max_ns / 1000; 2064 2065 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 2066 return -EINVAL; 2067 2068 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 2069 !rx_may_override_tx) { 2070 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 2071 "RX and TX IRQ moderation must be equal\n"); 2072 return -EINVAL; 2073 } 2074 2075 efx->irq_rx_adaptive = rx_adaptive; 2076 efx->irq_rx_moderation_us = rx_usecs; 2077 efx_for_each_channel(channel, efx) { 2078 if (efx_channel_has_rx_queue(channel)) 2079 channel->irq_moderation_us = rx_usecs; 2080 else if (efx_channel_has_tx_queues(channel)) 2081 channel->irq_moderation_us = tx_usecs; 2082 } 2083 2084 return 0; 2085 } 2086 2087 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2088 unsigned int *rx_usecs, bool *rx_adaptive) 2089 { 2090 *rx_adaptive = efx->irq_rx_adaptive; 2091 *rx_usecs = efx->irq_rx_moderation_us; 2092 2093 /* If channels are shared between RX and TX, so is IRQ 2094 * moderation. Otherwise, IRQ moderation is the same for all 2095 * TX channels and is not adaptive. 2096 */ 2097 if (efx->tx_channel_offset == 0) { 2098 *tx_usecs = *rx_usecs; 2099 } else { 2100 struct efx_channel *tx_channel; 2101 2102 tx_channel = efx->channel[efx->tx_channel_offset]; 2103 *tx_usecs = tx_channel->irq_moderation_us; 2104 } 2105 } 2106 2107 /************************************************************************** 2108 * 2109 * Hardware monitor 2110 * 2111 **************************************************************************/ 2112 2113 /* Run periodically off the general workqueue */ 2114 static void efx_monitor(struct work_struct *data) 2115 { 2116 struct efx_nic *efx = container_of(data, struct efx_nic, 2117 monitor_work.work); 2118 2119 netif_vdbg(efx, timer, efx->net_dev, 2120 "hardware monitor executing on CPU %d\n", 2121 raw_smp_processor_id()); 2122 BUG_ON(efx->type->monitor == NULL); 2123 2124 /* If the mac_lock is already held then it is likely a port 2125 * reconfiguration is already in place, which will likely do 2126 * most of the work of monitor() anyway. */ 2127 if (mutex_trylock(&efx->mac_lock)) { 2128 if (efx->port_enabled) 2129 efx->type->monitor(efx); 2130 mutex_unlock(&efx->mac_lock); 2131 } 2132 2133 queue_delayed_work(efx->workqueue, &efx->monitor_work, 2134 efx_monitor_interval); 2135 } 2136 2137 /************************************************************************** 2138 * 2139 * ioctls 2140 * 2141 *************************************************************************/ 2142 2143 /* Net device ioctl 2144 * Context: process, rtnl_lock() held. 2145 */ 2146 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2147 { 2148 struct efx_nic *efx = netdev_priv(net_dev); 2149 struct mii_ioctl_data *data = if_mii(ifr); 2150 2151 if (cmd == SIOCSHWTSTAMP) 2152 return efx_ptp_set_ts_config(efx, ifr); 2153 if (cmd == SIOCGHWTSTAMP) 2154 return efx_ptp_get_ts_config(efx, ifr); 2155 2156 /* Convert phy_id from older PRTAD/DEVAD format */ 2157 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2158 (data->phy_id & 0xfc00) == 0x0400) 2159 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2160 2161 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2162 } 2163 2164 /************************************************************************** 2165 * 2166 * NAPI interface 2167 * 2168 **************************************************************************/ 2169 2170 static void efx_init_napi_channel(struct efx_channel *channel) 2171 { 2172 struct efx_nic *efx = channel->efx; 2173 2174 channel->napi_dev = efx->net_dev; 2175 netif_napi_add(channel->napi_dev, &channel->napi_str, 2176 efx_poll, napi_weight); 2177 } 2178 2179 static void efx_init_napi(struct efx_nic *efx) 2180 { 2181 struct efx_channel *channel; 2182 2183 efx_for_each_channel(channel, efx) 2184 efx_init_napi_channel(channel); 2185 } 2186 2187 static void efx_fini_napi_channel(struct efx_channel *channel) 2188 { 2189 if (channel->napi_dev) 2190 netif_napi_del(&channel->napi_str); 2191 2192 channel->napi_dev = NULL; 2193 } 2194 2195 static void efx_fini_napi(struct efx_nic *efx) 2196 { 2197 struct efx_channel *channel; 2198 2199 efx_for_each_channel(channel, efx) 2200 efx_fini_napi_channel(channel); 2201 } 2202 2203 /************************************************************************** 2204 * 2205 * Kernel netpoll interface 2206 * 2207 *************************************************************************/ 2208 2209 #ifdef CONFIG_NET_POLL_CONTROLLER 2210 2211 /* Although in the common case interrupts will be disabled, this is not 2212 * guaranteed. However, all our work happens inside the NAPI callback, 2213 * so no locking is required. 2214 */ 2215 static void efx_netpoll(struct net_device *net_dev) 2216 { 2217 struct efx_nic *efx = netdev_priv(net_dev); 2218 struct efx_channel *channel; 2219 2220 efx_for_each_channel(channel, efx) 2221 efx_schedule_channel(channel); 2222 } 2223 2224 #endif 2225 2226 /************************************************************************** 2227 * 2228 * Kernel net device interface 2229 * 2230 *************************************************************************/ 2231 2232 /* Context: process, rtnl_lock() held. */ 2233 int efx_net_open(struct net_device *net_dev) 2234 { 2235 struct efx_nic *efx = netdev_priv(net_dev); 2236 int rc; 2237 2238 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2239 raw_smp_processor_id()); 2240 2241 rc = efx_check_disabled(efx); 2242 if (rc) 2243 return rc; 2244 if (efx->phy_mode & PHY_MODE_SPECIAL) 2245 return -EBUSY; 2246 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2247 return -EIO; 2248 2249 /* Notify the kernel of the link state polled during driver load, 2250 * before the monitor starts running */ 2251 efx_link_status_changed(efx); 2252 2253 efx_start_all(efx); 2254 if (efx->state == STATE_DISABLED || efx->reset_pending) 2255 netif_device_detach(efx->net_dev); 2256 efx_selftest_async_start(efx); 2257 return 0; 2258 } 2259 2260 /* Context: process, rtnl_lock() held. 2261 * Note that the kernel will ignore our return code; this method 2262 * should really be a void. 2263 */ 2264 int efx_net_stop(struct net_device *net_dev) 2265 { 2266 struct efx_nic *efx = netdev_priv(net_dev); 2267 2268 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2269 raw_smp_processor_id()); 2270 2271 /* Stop the device and flush all the channels */ 2272 efx_stop_all(efx); 2273 2274 return 0; 2275 } 2276 2277 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2278 static void efx_net_stats(struct net_device *net_dev, 2279 struct rtnl_link_stats64 *stats) 2280 { 2281 struct efx_nic *efx = netdev_priv(net_dev); 2282 2283 spin_lock_bh(&efx->stats_lock); 2284 efx->type->update_stats(efx, NULL, stats); 2285 spin_unlock_bh(&efx->stats_lock); 2286 } 2287 2288 /* Context: netif_tx_lock held, BHs disabled. */ 2289 static void efx_watchdog(struct net_device *net_dev) 2290 { 2291 struct efx_nic *efx = netdev_priv(net_dev); 2292 2293 netif_err(efx, tx_err, efx->net_dev, 2294 "TX stuck with port_enabled=%d: resetting channels\n", 2295 efx->port_enabled); 2296 2297 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2298 } 2299 2300 2301 /* Context: process, rtnl_lock() held. */ 2302 static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2303 { 2304 struct efx_nic *efx = netdev_priv(net_dev); 2305 int rc; 2306 2307 rc = efx_check_disabled(efx); 2308 if (rc) 2309 return rc; 2310 2311 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2312 2313 efx_device_detach_sync(efx); 2314 efx_stop_all(efx); 2315 2316 mutex_lock(&efx->mac_lock); 2317 net_dev->mtu = new_mtu; 2318 efx_mac_reconfigure(efx); 2319 mutex_unlock(&efx->mac_lock); 2320 2321 efx_start_all(efx); 2322 efx_device_attach_if_not_resetting(efx); 2323 return 0; 2324 } 2325 2326 static int efx_set_mac_address(struct net_device *net_dev, void *data) 2327 { 2328 struct efx_nic *efx = netdev_priv(net_dev); 2329 struct sockaddr *addr = data; 2330 u8 *new_addr = addr->sa_data; 2331 u8 old_addr[6]; 2332 int rc; 2333 2334 if (!is_valid_ether_addr(new_addr)) { 2335 netif_err(efx, drv, efx->net_dev, 2336 "invalid ethernet MAC address requested: %pM\n", 2337 new_addr); 2338 return -EADDRNOTAVAIL; 2339 } 2340 2341 /* save old address */ 2342 ether_addr_copy(old_addr, net_dev->dev_addr); 2343 ether_addr_copy(net_dev->dev_addr, new_addr); 2344 if (efx->type->set_mac_address) { 2345 rc = efx->type->set_mac_address(efx); 2346 if (rc) { 2347 ether_addr_copy(net_dev->dev_addr, old_addr); 2348 return rc; 2349 } 2350 } 2351 2352 /* Reconfigure the MAC */ 2353 mutex_lock(&efx->mac_lock); 2354 efx_mac_reconfigure(efx); 2355 mutex_unlock(&efx->mac_lock); 2356 2357 return 0; 2358 } 2359 2360 /* Context: netif_addr_lock held, BHs disabled. */ 2361 static void efx_set_rx_mode(struct net_device *net_dev) 2362 { 2363 struct efx_nic *efx = netdev_priv(net_dev); 2364 2365 if (efx->port_enabled) 2366 queue_work(efx->workqueue, &efx->mac_work); 2367 /* Otherwise efx_start_port() will do this */ 2368 } 2369 2370 static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2371 { 2372 struct efx_nic *efx = netdev_priv(net_dev); 2373 int rc; 2374 2375 /* If disabling RX n-tuple filtering, clear existing filters */ 2376 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2377 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2378 if (rc) 2379 return rc; 2380 } 2381 2382 /* If Rx VLAN filter is changed, update filters via mac_reconfigure. 2383 * If rx-fcs is changed, mac_reconfigure updates that too. 2384 */ 2385 if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER | 2386 NETIF_F_RXFCS)) { 2387 /* efx_set_rx_mode() will schedule MAC work to update filters 2388 * when a new features are finally set in net_dev. 2389 */ 2390 efx_set_rx_mode(net_dev); 2391 } 2392 2393 return 0; 2394 } 2395 2396 static int efx_get_phys_port_id(struct net_device *net_dev, 2397 struct netdev_phys_item_id *ppid) 2398 { 2399 struct efx_nic *efx = netdev_priv(net_dev); 2400 2401 if (efx->type->get_phys_port_id) 2402 return efx->type->get_phys_port_id(efx, ppid); 2403 else 2404 return -EOPNOTSUPP; 2405 } 2406 2407 static int efx_get_phys_port_name(struct net_device *net_dev, 2408 char *name, size_t len) 2409 { 2410 struct efx_nic *efx = netdev_priv(net_dev); 2411 2412 if (snprintf(name, len, "p%u", efx->port_num) >= len) 2413 return -EINVAL; 2414 return 0; 2415 } 2416 2417 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2418 { 2419 struct efx_nic *efx = netdev_priv(net_dev); 2420 2421 if (efx->type->vlan_rx_add_vid) 2422 return efx->type->vlan_rx_add_vid(efx, proto, vid); 2423 else 2424 return -EOPNOTSUPP; 2425 } 2426 2427 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2428 { 2429 struct efx_nic *efx = netdev_priv(net_dev); 2430 2431 if (efx->type->vlan_rx_kill_vid) 2432 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 2433 else 2434 return -EOPNOTSUPP; 2435 } 2436 2437 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in) 2438 { 2439 switch (in) { 2440 case UDP_TUNNEL_TYPE_VXLAN: 2441 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 2442 case UDP_TUNNEL_TYPE_GENEVE: 2443 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 2444 default: 2445 return -1; 2446 } 2447 } 2448 2449 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti) 2450 { 2451 struct efx_nic *efx = netdev_priv(dev); 2452 struct efx_udp_tunnel tnl; 2453 int efx_tunnel_type; 2454 2455 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2456 if (efx_tunnel_type < 0) 2457 return; 2458 2459 tnl.type = (u16)efx_tunnel_type; 2460 tnl.port = ti->port; 2461 2462 if (efx->type->udp_tnl_add_port) 2463 (void)efx->type->udp_tnl_add_port(efx, tnl); 2464 } 2465 2466 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti) 2467 { 2468 struct efx_nic *efx = netdev_priv(dev); 2469 struct efx_udp_tunnel tnl; 2470 int efx_tunnel_type; 2471 2472 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2473 if (efx_tunnel_type < 0) 2474 return; 2475 2476 tnl.type = (u16)efx_tunnel_type; 2477 tnl.port = ti->port; 2478 2479 if (efx->type->udp_tnl_del_port) 2480 (void)efx->type->udp_tnl_del_port(efx, tnl); 2481 } 2482 2483 static const struct net_device_ops efx_netdev_ops = { 2484 .ndo_open = efx_net_open, 2485 .ndo_stop = efx_net_stop, 2486 .ndo_get_stats64 = efx_net_stats, 2487 .ndo_tx_timeout = efx_watchdog, 2488 .ndo_start_xmit = efx_hard_start_xmit, 2489 .ndo_validate_addr = eth_validate_addr, 2490 .ndo_do_ioctl = efx_ioctl, 2491 .ndo_change_mtu = efx_change_mtu, 2492 .ndo_set_mac_address = efx_set_mac_address, 2493 .ndo_set_rx_mode = efx_set_rx_mode, 2494 .ndo_set_features = efx_set_features, 2495 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 2496 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2497 #ifdef CONFIG_SFC_SRIOV 2498 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2499 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2500 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2501 .ndo_get_vf_config = efx_sriov_get_vf_config, 2502 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2503 #endif 2504 .ndo_get_phys_port_id = efx_get_phys_port_id, 2505 .ndo_get_phys_port_name = efx_get_phys_port_name, 2506 #ifdef CONFIG_NET_POLL_CONTROLLER 2507 .ndo_poll_controller = efx_netpoll, 2508 #endif 2509 .ndo_setup_tc = efx_setup_tc, 2510 #ifdef CONFIG_RFS_ACCEL 2511 .ndo_rx_flow_steer = efx_filter_rfs, 2512 #endif 2513 .ndo_udp_tunnel_add = efx_udp_tunnel_add, 2514 .ndo_udp_tunnel_del = efx_udp_tunnel_del, 2515 }; 2516 2517 static void efx_update_name(struct efx_nic *efx) 2518 { 2519 strcpy(efx->name, efx->net_dev->name); 2520 efx_mtd_rename(efx); 2521 efx_set_channel_names(efx); 2522 } 2523 2524 static int efx_netdev_event(struct notifier_block *this, 2525 unsigned long event, void *ptr) 2526 { 2527 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2528 2529 if ((net_dev->netdev_ops == &efx_netdev_ops) && 2530 event == NETDEV_CHANGENAME) 2531 efx_update_name(netdev_priv(net_dev)); 2532 2533 return NOTIFY_DONE; 2534 } 2535 2536 static struct notifier_block efx_netdev_notifier = { 2537 .notifier_call = efx_netdev_event, 2538 }; 2539 2540 static ssize_t 2541 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2542 { 2543 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2544 return sprintf(buf, "%d\n", efx->phy_type); 2545 } 2546 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2547 2548 #ifdef CONFIG_SFC_MCDI_LOGGING 2549 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2550 char *buf) 2551 { 2552 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2553 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2554 2555 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2556 } 2557 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2558 const char *buf, size_t count) 2559 { 2560 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2561 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2562 bool enable = count > 0 && *buf != '0'; 2563 2564 mcdi->logging_enabled = enable; 2565 return count; 2566 } 2567 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2568 #endif 2569 2570 static int efx_register_netdev(struct efx_nic *efx) 2571 { 2572 struct net_device *net_dev = efx->net_dev; 2573 struct efx_channel *channel; 2574 int rc; 2575 2576 net_dev->watchdog_timeo = 5 * HZ; 2577 net_dev->irq = efx->pci_dev->irq; 2578 net_dev->netdev_ops = &efx_netdev_ops; 2579 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 2580 net_dev->priv_flags |= IFF_UNICAST_FLT; 2581 net_dev->ethtool_ops = &efx_ethtool_ops; 2582 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2583 net_dev->min_mtu = EFX_MIN_MTU; 2584 net_dev->max_mtu = EFX_MAX_MTU; 2585 2586 rtnl_lock(); 2587 2588 /* Enable resets to be scheduled and check whether any were 2589 * already requested. If so, the NIC is probably hosed so we 2590 * abort. 2591 */ 2592 efx->state = STATE_READY; 2593 smp_mb(); /* ensure we change state before checking reset_pending */ 2594 if (efx->reset_pending) { 2595 netif_err(efx, probe, efx->net_dev, 2596 "aborting probe due to scheduled reset\n"); 2597 rc = -EIO; 2598 goto fail_locked; 2599 } 2600 2601 rc = dev_alloc_name(net_dev, net_dev->name); 2602 if (rc < 0) 2603 goto fail_locked; 2604 efx_update_name(efx); 2605 2606 /* Always start with carrier off; PHY events will detect the link */ 2607 netif_carrier_off(net_dev); 2608 2609 rc = register_netdevice(net_dev); 2610 if (rc) 2611 goto fail_locked; 2612 2613 efx_for_each_channel(channel, efx) { 2614 struct efx_tx_queue *tx_queue; 2615 efx_for_each_channel_tx_queue(tx_queue, channel) 2616 efx_init_tx_queue_core_txq(tx_queue); 2617 } 2618 2619 efx_associate(efx); 2620 2621 rtnl_unlock(); 2622 2623 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2624 if (rc) { 2625 netif_err(efx, drv, efx->net_dev, 2626 "failed to init net dev attributes\n"); 2627 goto fail_registered; 2628 } 2629 #ifdef CONFIG_SFC_MCDI_LOGGING 2630 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2631 if (rc) { 2632 netif_err(efx, drv, efx->net_dev, 2633 "failed to init net dev attributes\n"); 2634 goto fail_attr_mcdi_logging; 2635 } 2636 #endif 2637 2638 return 0; 2639 2640 #ifdef CONFIG_SFC_MCDI_LOGGING 2641 fail_attr_mcdi_logging: 2642 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2643 #endif 2644 fail_registered: 2645 rtnl_lock(); 2646 efx_dissociate(efx); 2647 unregister_netdevice(net_dev); 2648 fail_locked: 2649 efx->state = STATE_UNINIT; 2650 rtnl_unlock(); 2651 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2652 return rc; 2653 } 2654 2655 static void efx_unregister_netdev(struct efx_nic *efx) 2656 { 2657 if (!efx->net_dev) 2658 return; 2659 2660 BUG_ON(netdev_priv(efx->net_dev) != efx); 2661 2662 if (efx_dev_registered(efx)) { 2663 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2664 #ifdef CONFIG_SFC_MCDI_LOGGING 2665 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2666 #endif 2667 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2668 unregister_netdev(efx->net_dev); 2669 } 2670 } 2671 2672 /************************************************************************** 2673 * 2674 * Device reset and suspend 2675 * 2676 **************************************************************************/ 2677 2678 /* Tears down the entire software state and most of the hardware state 2679 * before reset. */ 2680 void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2681 { 2682 EFX_ASSERT_RESET_SERIALISED(efx); 2683 2684 if (method == RESET_TYPE_MCDI_TIMEOUT) 2685 efx->type->prepare_flr(efx); 2686 2687 efx_stop_all(efx); 2688 efx_disable_interrupts(efx); 2689 2690 mutex_lock(&efx->mac_lock); 2691 mutex_lock(&efx->rss_lock); 2692 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2693 method != RESET_TYPE_DATAPATH) 2694 efx->phy_op->fini(efx); 2695 efx->type->fini(efx); 2696 } 2697 2698 /* This function will always ensure that the locks acquired in 2699 * efx_reset_down() are released. A failure return code indicates 2700 * that we were unable to reinitialise the hardware, and the 2701 * driver should be disabled. If ok is false, then the rx and tx 2702 * engines are not restarted, pending a RESET_DISABLE. */ 2703 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2704 { 2705 int rc; 2706 2707 EFX_ASSERT_RESET_SERIALISED(efx); 2708 2709 if (method == RESET_TYPE_MCDI_TIMEOUT) 2710 efx->type->finish_flr(efx); 2711 2712 /* Ensure that SRAM is initialised even if we're disabling the device */ 2713 rc = efx->type->init(efx); 2714 if (rc) { 2715 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2716 goto fail; 2717 } 2718 2719 if (!ok) 2720 goto fail; 2721 2722 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2723 method != RESET_TYPE_DATAPATH) { 2724 rc = efx->phy_op->init(efx); 2725 if (rc) 2726 goto fail; 2727 rc = efx->phy_op->reconfigure(efx); 2728 if (rc && rc != -EPERM) 2729 netif_err(efx, drv, efx->net_dev, 2730 "could not restore PHY settings\n"); 2731 } 2732 2733 rc = efx_enable_interrupts(efx); 2734 if (rc) 2735 goto fail; 2736 2737 #ifdef CONFIG_SFC_SRIOV 2738 rc = efx->type->vswitching_restore(efx); 2739 if (rc) /* not fatal; the PF will still work fine */ 2740 netif_warn(efx, probe, efx->net_dev, 2741 "failed to restore vswitching rc=%d;" 2742 " VFs may not function\n", rc); 2743 #endif 2744 2745 if (efx->type->rx_restore_rss_contexts) 2746 efx->type->rx_restore_rss_contexts(efx); 2747 mutex_unlock(&efx->rss_lock); 2748 down_read(&efx->filter_sem); 2749 efx_restore_filters(efx); 2750 up_read(&efx->filter_sem); 2751 if (efx->type->sriov_reset) 2752 efx->type->sriov_reset(efx); 2753 2754 mutex_unlock(&efx->mac_lock); 2755 2756 efx_start_all(efx); 2757 2758 if (efx->type->udp_tnl_push_ports) 2759 efx->type->udp_tnl_push_ports(efx); 2760 2761 return 0; 2762 2763 fail: 2764 efx->port_initialized = false; 2765 2766 mutex_unlock(&efx->rss_lock); 2767 mutex_unlock(&efx->mac_lock); 2768 2769 return rc; 2770 } 2771 2772 /* Reset the NIC using the specified method. Note that the reset may 2773 * fail, in which case the card will be left in an unusable state. 2774 * 2775 * Caller must hold the rtnl_lock. 2776 */ 2777 int efx_reset(struct efx_nic *efx, enum reset_type method) 2778 { 2779 int rc, rc2; 2780 bool disabled; 2781 2782 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2783 RESET_TYPE(method)); 2784 2785 efx_device_detach_sync(efx); 2786 efx_reset_down(efx, method); 2787 2788 rc = efx->type->reset(efx, method); 2789 if (rc) { 2790 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2791 goto out; 2792 } 2793 2794 /* Clear flags for the scopes we covered. We assume the NIC and 2795 * driver are now quiescent so that there is no race here. 2796 */ 2797 if (method < RESET_TYPE_MAX_METHOD) 2798 efx->reset_pending &= -(1 << (method + 1)); 2799 else /* it doesn't fit into the well-ordered scope hierarchy */ 2800 __clear_bit(method, &efx->reset_pending); 2801 2802 /* Reinitialise bus-mastering, which may have been turned off before 2803 * the reset was scheduled. This is still appropriate, even in the 2804 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2805 * can respond to requests. */ 2806 pci_set_master(efx->pci_dev); 2807 2808 out: 2809 /* Leave device stopped if necessary */ 2810 disabled = rc || 2811 method == RESET_TYPE_DISABLE || 2812 method == RESET_TYPE_RECOVER_OR_DISABLE; 2813 rc2 = efx_reset_up(efx, method, !disabled); 2814 if (rc2) { 2815 disabled = true; 2816 if (!rc) 2817 rc = rc2; 2818 } 2819 2820 if (disabled) { 2821 dev_close(efx->net_dev); 2822 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2823 efx->state = STATE_DISABLED; 2824 } else { 2825 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2826 efx_device_attach_if_not_resetting(efx); 2827 } 2828 return rc; 2829 } 2830 2831 /* Try recovery mechanisms. 2832 * For now only EEH is supported. 2833 * Returns 0 if the recovery mechanisms are unsuccessful. 2834 * Returns a non-zero value otherwise. 2835 */ 2836 int efx_try_recovery(struct efx_nic *efx) 2837 { 2838 #ifdef CONFIG_EEH 2839 /* A PCI error can occur and not be seen by EEH because nothing 2840 * happens on the PCI bus. In this case the driver may fail and 2841 * schedule a 'recover or reset', leading to this recovery handler. 2842 * Manually call the eeh failure check function. 2843 */ 2844 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2845 if (eeh_dev_check_failure(eehdev)) { 2846 /* The EEH mechanisms will handle the error and reset the 2847 * device if necessary. 2848 */ 2849 return 1; 2850 } 2851 #endif 2852 return 0; 2853 } 2854 2855 static void efx_wait_for_bist_end(struct efx_nic *efx) 2856 { 2857 int i; 2858 2859 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 2860 if (efx_mcdi_poll_reboot(efx)) 2861 goto out; 2862 msleep(BIST_WAIT_DELAY_MS); 2863 } 2864 2865 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 2866 out: 2867 /* Either way unset the BIST flag. If we found no reboot we probably 2868 * won't recover, but we should try. 2869 */ 2870 efx->mc_bist_for_other_fn = false; 2871 } 2872 2873 /* The worker thread exists so that code that cannot sleep can 2874 * schedule a reset for later. 2875 */ 2876 static void efx_reset_work(struct work_struct *data) 2877 { 2878 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2879 unsigned long pending; 2880 enum reset_type method; 2881 2882 pending = READ_ONCE(efx->reset_pending); 2883 method = fls(pending) - 1; 2884 2885 if (method == RESET_TYPE_MC_BIST) 2886 efx_wait_for_bist_end(efx); 2887 2888 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2889 method == RESET_TYPE_RECOVER_OR_ALL) && 2890 efx_try_recovery(efx)) 2891 return; 2892 2893 if (!pending) 2894 return; 2895 2896 rtnl_lock(); 2897 2898 /* We checked the state in efx_schedule_reset() but it may 2899 * have changed by now. Now that we have the RTNL lock, 2900 * it cannot change again. 2901 */ 2902 if (efx->state == STATE_READY) 2903 (void)efx_reset(efx, method); 2904 2905 rtnl_unlock(); 2906 } 2907 2908 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2909 { 2910 enum reset_type method; 2911 2912 if (efx->state == STATE_RECOVERY) { 2913 netif_dbg(efx, drv, efx->net_dev, 2914 "recovering: skip scheduling %s reset\n", 2915 RESET_TYPE(type)); 2916 return; 2917 } 2918 2919 switch (type) { 2920 case RESET_TYPE_INVISIBLE: 2921 case RESET_TYPE_ALL: 2922 case RESET_TYPE_RECOVER_OR_ALL: 2923 case RESET_TYPE_WORLD: 2924 case RESET_TYPE_DISABLE: 2925 case RESET_TYPE_RECOVER_OR_DISABLE: 2926 case RESET_TYPE_DATAPATH: 2927 case RESET_TYPE_MC_BIST: 2928 case RESET_TYPE_MCDI_TIMEOUT: 2929 method = type; 2930 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2931 RESET_TYPE(method)); 2932 break; 2933 default: 2934 method = efx->type->map_reset_reason(type); 2935 netif_dbg(efx, drv, efx->net_dev, 2936 "scheduling %s reset for %s\n", 2937 RESET_TYPE(method), RESET_TYPE(type)); 2938 break; 2939 } 2940 2941 set_bit(method, &efx->reset_pending); 2942 smp_mb(); /* ensure we change reset_pending before checking state */ 2943 2944 /* If we're not READY then just leave the flags set as the cue 2945 * to abort probing or reschedule the reset later. 2946 */ 2947 if (READ_ONCE(efx->state) != STATE_READY) 2948 return; 2949 2950 /* efx_process_channel() will no longer read events once a 2951 * reset is scheduled. So switch back to poll'd MCDI completions. */ 2952 efx_mcdi_mode_poll(efx); 2953 2954 queue_work(reset_workqueue, &efx->reset_work); 2955 } 2956 2957 /************************************************************************** 2958 * 2959 * List of NICs we support 2960 * 2961 **************************************************************************/ 2962 2963 /* PCI device ID table */ 2964 static const struct pci_device_id efx_pci_table[] = { 2965 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2966 .driver_data = (unsigned long) &siena_a0_nic_type}, 2967 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2968 .driver_data = (unsigned long) &siena_a0_nic_type}, 2969 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 2970 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2971 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 2972 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2973 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 2974 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2975 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2976 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2977 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2978 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2979 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2980 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2981 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 2982 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2983 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 2984 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2985 {0} /* end of list */ 2986 }; 2987 2988 /************************************************************************** 2989 * 2990 * Dummy PHY/MAC operations 2991 * 2992 * Can be used for some unimplemented operations 2993 * Needed so all function pointers are valid and do not have to be tested 2994 * before use 2995 * 2996 **************************************************************************/ 2997 int efx_port_dummy_op_int(struct efx_nic *efx) 2998 { 2999 return 0; 3000 } 3001 void efx_port_dummy_op_void(struct efx_nic *efx) {} 3002 3003 static bool efx_port_dummy_op_poll(struct efx_nic *efx) 3004 { 3005 return false; 3006 } 3007 3008 static const struct efx_phy_operations efx_dummy_phy_operations = { 3009 .init = efx_port_dummy_op_int, 3010 .reconfigure = efx_port_dummy_op_int, 3011 .poll = efx_port_dummy_op_poll, 3012 .fini = efx_port_dummy_op_void, 3013 }; 3014 3015 /************************************************************************** 3016 * 3017 * Data housekeeping 3018 * 3019 **************************************************************************/ 3020 3021 /* This zeroes out and then fills in the invariants in a struct 3022 * efx_nic (including all sub-structures). 3023 */ 3024 static int efx_init_struct(struct efx_nic *efx, 3025 struct pci_dev *pci_dev, struct net_device *net_dev) 3026 { 3027 int rc = -ENOMEM, i; 3028 3029 /* Initialise common structures */ 3030 INIT_LIST_HEAD(&efx->node); 3031 INIT_LIST_HEAD(&efx->secondary_list); 3032 spin_lock_init(&efx->biu_lock); 3033 #ifdef CONFIG_SFC_MTD 3034 INIT_LIST_HEAD(&efx->mtd_list); 3035 #endif 3036 INIT_WORK(&efx->reset_work, efx_reset_work); 3037 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 3038 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 3039 efx->pci_dev = pci_dev; 3040 efx->msg_enable = debug; 3041 efx->state = STATE_UNINIT; 3042 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 3043 3044 efx->net_dev = net_dev; 3045 efx->rx_prefix_size = efx->type->rx_prefix_size; 3046 efx->rx_ip_align = 3047 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 3048 efx->rx_packet_hash_offset = 3049 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 3050 efx->rx_packet_ts_offset = 3051 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 3052 INIT_LIST_HEAD(&efx->rss_context.list); 3053 mutex_init(&efx->rss_lock); 3054 spin_lock_init(&efx->stats_lock); 3055 efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 3056 efx->num_mac_stats = MC_CMD_MAC_NSTATS; 3057 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 3058 mutex_init(&efx->mac_lock); 3059 #ifdef CONFIG_RFS_ACCEL 3060 mutex_init(&efx->rps_mutex); 3061 spin_lock_init(&efx->rps_hash_lock); 3062 /* Failure to allocate is not fatal, but may degrade ARFS performance */ 3063 efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, 3064 sizeof(*efx->rps_hash_table), GFP_KERNEL); 3065 #endif 3066 efx->phy_op = &efx_dummy_phy_operations; 3067 efx->mdio.dev = net_dev; 3068 INIT_WORK(&efx->mac_work, efx_mac_work); 3069 init_waitqueue_head(&efx->flush_wq); 3070 3071 for (i = 0; i < EFX_MAX_CHANNELS; i++) { 3072 efx->channel[i] = efx_alloc_channel(efx, i, NULL); 3073 if (!efx->channel[i]) 3074 goto fail; 3075 efx->msi_context[i].efx = efx; 3076 efx->msi_context[i].index = i; 3077 } 3078 3079 /* Higher numbered interrupt modes are less capable! */ 3080 if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 3081 efx->type->min_interrupt_mode)) { 3082 rc = -EIO; 3083 goto fail; 3084 } 3085 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 3086 interrupt_mode); 3087 efx->interrupt_mode = min(efx->type->min_interrupt_mode, 3088 interrupt_mode); 3089 3090 /* Would be good to use the net_dev name, but we're too early */ 3091 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 3092 pci_name(pci_dev)); 3093 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 3094 if (!efx->workqueue) 3095 goto fail; 3096 3097 return 0; 3098 3099 fail: 3100 efx_fini_struct(efx); 3101 return rc; 3102 } 3103 3104 static void efx_fini_struct(struct efx_nic *efx) 3105 { 3106 int i; 3107 3108 #ifdef CONFIG_RFS_ACCEL 3109 kfree(efx->rps_hash_table); 3110 #endif 3111 3112 for (i = 0; i < EFX_MAX_CHANNELS; i++) 3113 kfree(efx->channel[i]); 3114 3115 kfree(efx->vpd_sn); 3116 3117 if (efx->workqueue) { 3118 destroy_workqueue(efx->workqueue); 3119 efx->workqueue = NULL; 3120 } 3121 } 3122 3123 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 3124 { 3125 u64 n_rx_nodesc_trunc = 0; 3126 struct efx_channel *channel; 3127 3128 efx_for_each_channel(channel, efx) 3129 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 3130 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 3131 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 3132 } 3133 3134 bool efx_filter_spec_equal(const struct efx_filter_spec *left, 3135 const struct efx_filter_spec *right) 3136 { 3137 if ((left->match_flags ^ right->match_flags) | 3138 ((left->flags ^ right->flags) & 3139 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) 3140 return false; 3141 3142 return memcmp(&left->outer_vid, &right->outer_vid, 3143 sizeof(struct efx_filter_spec) - 3144 offsetof(struct efx_filter_spec, outer_vid)) == 0; 3145 } 3146 3147 u32 efx_filter_spec_hash(const struct efx_filter_spec *spec) 3148 { 3149 BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); 3150 return jhash2((const u32 *)&spec->outer_vid, 3151 (sizeof(struct efx_filter_spec) - 3152 offsetof(struct efx_filter_spec, outer_vid)) / 4, 3153 0); 3154 } 3155 3156 #ifdef CONFIG_RFS_ACCEL 3157 bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx, 3158 bool *force) 3159 { 3160 if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) { 3161 /* ARFS is currently updating this entry, leave it */ 3162 return false; 3163 } 3164 if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) { 3165 /* ARFS tried and failed to update this, so it's probably out 3166 * of date. Remove the filter and the ARFS rule entry. 3167 */ 3168 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 3169 *force = true; 3170 return true; 3171 } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */ 3172 /* ARFS has moved on, so old filter is not needed. Since we did 3173 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will 3174 * not be removed by efx_rps_hash_del() subsequently. 3175 */ 3176 *force = true; 3177 return true; 3178 } 3179 /* Remove it iff ARFS wants to. */ 3180 return true; 3181 } 3182 3183 static 3184 struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx, 3185 const struct efx_filter_spec *spec) 3186 { 3187 u32 hash = efx_filter_spec_hash(spec); 3188 3189 WARN_ON(!spin_is_locked(&efx->rps_hash_lock)); 3190 if (!efx->rps_hash_table) 3191 return NULL; 3192 return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE]; 3193 } 3194 3195 struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx, 3196 const struct efx_filter_spec *spec) 3197 { 3198 struct efx_arfs_rule *rule; 3199 struct hlist_head *head; 3200 struct hlist_node *node; 3201 3202 head = efx_rps_hash_bucket(efx, spec); 3203 if (!head) 3204 return NULL; 3205 hlist_for_each(node, head) { 3206 rule = container_of(node, struct efx_arfs_rule, node); 3207 if (efx_filter_spec_equal(spec, &rule->spec)) 3208 return rule; 3209 } 3210 return NULL; 3211 } 3212 3213 struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx, 3214 const struct efx_filter_spec *spec, 3215 bool *new) 3216 { 3217 struct efx_arfs_rule *rule; 3218 struct hlist_head *head; 3219 struct hlist_node *node; 3220 3221 head = efx_rps_hash_bucket(efx, spec); 3222 if (!head) 3223 return NULL; 3224 hlist_for_each(node, head) { 3225 rule = container_of(node, struct efx_arfs_rule, node); 3226 if (efx_filter_spec_equal(spec, &rule->spec)) { 3227 *new = false; 3228 return rule; 3229 } 3230 } 3231 rule = kmalloc(sizeof(*rule), GFP_ATOMIC); 3232 *new = true; 3233 if (rule) { 3234 memcpy(&rule->spec, spec, sizeof(rule->spec)); 3235 hlist_add_head(&rule->node, head); 3236 } 3237 return rule; 3238 } 3239 3240 void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec) 3241 { 3242 struct efx_arfs_rule *rule; 3243 struct hlist_head *head; 3244 struct hlist_node *node; 3245 3246 head = efx_rps_hash_bucket(efx, spec); 3247 if (WARN_ON(!head)) 3248 return; 3249 hlist_for_each(node, head) { 3250 rule = container_of(node, struct efx_arfs_rule, node); 3251 if (efx_filter_spec_equal(spec, &rule->spec)) { 3252 /* Someone already reused the entry. We know that if 3253 * this check doesn't fire (i.e. filter_id == REMOVING) 3254 * then the REMOVING mark was put there by our caller, 3255 * because caller is holding a lock on filter table and 3256 * only holders of that lock set REMOVING. 3257 */ 3258 if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING) 3259 return; 3260 hlist_del(node); 3261 kfree(rule); 3262 return; 3263 } 3264 } 3265 /* We didn't find it. */ 3266 WARN_ON(1); 3267 } 3268 #endif 3269 3270 /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because 3271 * (a) this is an infrequent control-plane operation and (b) n is small (max 64) 3272 */ 3273 struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx) 3274 { 3275 struct list_head *head = &efx->rss_context.list; 3276 struct efx_rss_context *ctx, *new; 3277 u32 id = 1; /* Don't use zero, that refers to the master RSS context */ 3278 3279 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3280 3281 /* Search for first gap in the numbering */ 3282 list_for_each_entry(ctx, head, list) { 3283 if (ctx->user_id != id) 3284 break; 3285 id++; 3286 /* Check for wrap. If this happens, we have nearly 2^32 3287 * allocated RSS contexts, which seems unlikely. 3288 */ 3289 if (WARN_ON_ONCE(!id)) 3290 return NULL; 3291 } 3292 3293 /* Create the new entry */ 3294 new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL); 3295 if (!new) 3296 return NULL; 3297 new->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 3298 new->rx_hash_udp_4tuple = false; 3299 3300 /* Insert the new entry into the gap */ 3301 new->user_id = id; 3302 list_add_tail(&new->list, &ctx->list); 3303 return new; 3304 } 3305 3306 struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id) 3307 { 3308 struct list_head *head = &efx->rss_context.list; 3309 struct efx_rss_context *ctx; 3310 3311 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3312 3313 list_for_each_entry(ctx, head, list) 3314 if (ctx->user_id == id) 3315 return ctx; 3316 return NULL; 3317 } 3318 3319 void efx_free_rss_context_entry(struct efx_rss_context *ctx) 3320 { 3321 list_del(&ctx->list); 3322 kfree(ctx); 3323 } 3324 3325 /************************************************************************** 3326 * 3327 * PCI interface 3328 * 3329 **************************************************************************/ 3330 3331 /* Main body of final NIC shutdown code 3332 * This is called only at module unload (or hotplug removal). 3333 */ 3334 static void efx_pci_remove_main(struct efx_nic *efx) 3335 { 3336 /* Flush reset_work. It can no longer be scheduled since we 3337 * are not READY. 3338 */ 3339 BUG_ON(efx->state == STATE_READY); 3340 cancel_work_sync(&efx->reset_work); 3341 3342 efx_disable_interrupts(efx); 3343 efx_clear_interrupt_affinity(efx); 3344 efx_nic_fini_interrupt(efx); 3345 efx_fini_port(efx); 3346 efx->type->fini(efx); 3347 efx_fini_napi(efx); 3348 efx_remove_all(efx); 3349 } 3350 3351 /* Final NIC shutdown 3352 * This is called only at module unload (or hotplug removal). A PF can call 3353 * this on its VFs to ensure they are unbound first. 3354 */ 3355 static void efx_pci_remove(struct pci_dev *pci_dev) 3356 { 3357 struct efx_nic *efx; 3358 3359 efx = pci_get_drvdata(pci_dev); 3360 if (!efx) 3361 return; 3362 3363 /* Mark the NIC as fini, then stop the interface */ 3364 rtnl_lock(); 3365 efx_dissociate(efx); 3366 dev_close(efx->net_dev); 3367 efx_disable_interrupts(efx); 3368 efx->state = STATE_UNINIT; 3369 rtnl_unlock(); 3370 3371 if (efx->type->sriov_fini) 3372 efx->type->sriov_fini(efx); 3373 3374 efx_unregister_netdev(efx); 3375 3376 efx_mtd_remove(efx); 3377 3378 efx_pci_remove_main(efx); 3379 3380 efx_fini_io(efx); 3381 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3382 3383 efx_fini_struct(efx); 3384 free_netdev(efx->net_dev); 3385 3386 pci_disable_pcie_error_reporting(pci_dev); 3387 }; 3388 3389 /* NIC VPD information 3390 * Called during probe to display the part number of the 3391 * installed NIC. VPD is potentially very large but this should 3392 * always appear within the first 512 bytes. 3393 */ 3394 #define SFC_VPD_LEN 512 3395 static void efx_probe_vpd_strings(struct efx_nic *efx) 3396 { 3397 struct pci_dev *dev = efx->pci_dev; 3398 char vpd_data[SFC_VPD_LEN]; 3399 ssize_t vpd_size; 3400 int ro_start, ro_size, i, j; 3401 3402 /* Get the vpd data from the device */ 3403 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3404 if (vpd_size <= 0) { 3405 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3406 return; 3407 } 3408 3409 /* Get the Read only section */ 3410 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3411 if (ro_start < 0) { 3412 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3413 return; 3414 } 3415 3416 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3417 j = ro_size; 3418 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3419 if (i + j > vpd_size) 3420 j = vpd_size - i; 3421 3422 /* Get the Part number */ 3423 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3424 if (i < 0) { 3425 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3426 return; 3427 } 3428 3429 j = pci_vpd_info_field_size(&vpd_data[i]); 3430 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3431 if (i + j > vpd_size) { 3432 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3433 return; 3434 } 3435 3436 netif_info(efx, drv, efx->net_dev, 3437 "Part Number : %.*s\n", j, &vpd_data[i]); 3438 3439 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3440 j = ro_size; 3441 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3442 if (i < 0) { 3443 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3444 return; 3445 } 3446 3447 j = pci_vpd_info_field_size(&vpd_data[i]); 3448 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3449 if (i + j > vpd_size) { 3450 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3451 return; 3452 } 3453 3454 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3455 if (!efx->vpd_sn) 3456 return; 3457 3458 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3459 } 3460 3461 3462 /* Main body of NIC initialisation 3463 * This is called at module load (or hotplug insertion, theoretically). 3464 */ 3465 static int efx_pci_probe_main(struct efx_nic *efx) 3466 { 3467 int rc; 3468 3469 /* Do start-of-day initialisation */ 3470 rc = efx_probe_all(efx); 3471 if (rc) 3472 goto fail1; 3473 3474 efx_init_napi(efx); 3475 3476 rc = efx->type->init(efx); 3477 if (rc) { 3478 netif_err(efx, probe, efx->net_dev, 3479 "failed to initialise NIC\n"); 3480 goto fail3; 3481 } 3482 3483 rc = efx_init_port(efx); 3484 if (rc) { 3485 netif_err(efx, probe, efx->net_dev, 3486 "failed to initialise port\n"); 3487 goto fail4; 3488 } 3489 3490 rc = efx_nic_init_interrupt(efx); 3491 if (rc) 3492 goto fail5; 3493 3494 efx_set_interrupt_affinity(efx); 3495 rc = efx_enable_interrupts(efx); 3496 if (rc) 3497 goto fail6; 3498 3499 return 0; 3500 3501 fail6: 3502 efx_clear_interrupt_affinity(efx); 3503 efx_nic_fini_interrupt(efx); 3504 fail5: 3505 efx_fini_port(efx); 3506 fail4: 3507 efx->type->fini(efx); 3508 fail3: 3509 efx_fini_napi(efx); 3510 efx_remove_all(efx); 3511 fail1: 3512 return rc; 3513 } 3514 3515 static int efx_pci_probe_post_io(struct efx_nic *efx) 3516 { 3517 struct net_device *net_dev = efx->net_dev; 3518 int rc = efx_pci_probe_main(efx); 3519 3520 if (rc) 3521 return rc; 3522 3523 if (efx->type->sriov_init) { 3524 rc = efx->type->sriov_init(efx); 3525 if (rc) 3526 netif_err(efx, probe, efx->net_dev, 3527 "SR-IOV can't be enabled rc %d\n", rc); 3528 } 3529 3530 /* Determine netdevice features */ 3531 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3532 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL); 3533 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 3534 net_dev->features |= NETIF_F_TSO6; 3535 /* Check whether device supports TSO */ 3536 if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 3537 net_dev->features &= ~NETIF_F_ALL_TSO; 3538 /* Mask for features that also apply to VLAN devices */ 3539 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3540 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 3541 NETIF_F_RXCSUM); 3542 3543 net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 3544 3545 /* Disable receiving frames with bad FCS, by default. */ 3546 net_dev->features &= ~NETIF_F_RXALL; 3547 3548 /* Disable VLAN filtering by default. It may be enforced if 3549 * the feature is fixed (i.e. VLAN filters are required to 3550 * receive VLAN tagged packets due to vPort restrictions). 3551 */ 3552 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3553 net_dev->features |= efx->fixed_features; 3554 3555 rc = efx_register_netdev(efx); 3556 if (!rc) 3557 return 0; 3558 3559 efx_pci_remove_main(efx); 3560 return rc; 3561 } 3562 3563 /* NIC initialisation 3564 * 3565 * This is called at module load (or hotplug insertion, 3566 * theoretically). It sets up PCI mappings, resets the NIC, 3567 * sets up and registers the network devices with the kernel and hooks 3568 * the interrupt service routine. It does not prepare the device for 3569 * transmission; this is left to the first time one of the network 3570 * interfaces is brought up (i.e. efx_net_open). 3571 */ 3572 static int efx_pci_probe(struct pci_dev *pci_dev, 3573 const struct pci_device_id *entry) 3574 { 3575 struct net_device *net_dev; 3576 struct efx_nic *efx; 3577 int rc; 3578 3579 /* Allocate and initialise a struct net_device and struct efx_nic */ 3580 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3581 EFX_MAX_RX_QUEUES); 3582 if (!net_dev) 3583 return -ENOMEM; 3584 efx = netdev_priv(net_dev); 3585 efx->type = (const struct efx_nic_type *) entry->driver_data; 3586 efx->fixed_features |= NETIF_F_HIGHDMA; 3587 3588 pci_set_drvdata(pci_dev, efx); 3589 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3590 rc = efx_init_struct(efx, pci_dev, net_dev); 3591 if (rc) 3592 goto fail1; 3593 3594 netif_info(efx, probe, efx->net_dev, 3595 "Solarflare NIC detected\n"); 3596 3597 if (!efx->type->is_vf) 3598 efx_probe_vpd_strings(efx); 3599 3600 /* Set up basic I/O (BAR mappings etc) */ 3601 rc = efx_init_io(efx); 3602 if (rc) 3603 goto fail2; 3604 3605 rc = efx_pci_probe_post_io(efx); 3606 if (rc) { 3607 /* On failure, retry once immediately. 3608 * If we aborted probe due to a scheduled reset, dismiss it. 3609 */ 3610 efx->reset_pending = 0; 3611 rc = efx_pci_probe_post_io(efx); 3612 if (rc) { 3613 /* On another failure, retry once more 3614 * after a 50-305ms delay. 3615 */ 3616 unsigned char r; 3617 3618 get_random_bytes(&r, 1); 3619 msleep((unsigned int)r + 50); 3620 efx->reset_pending = 0; 3621 rc = efx_pci_probe_post_io(efx); 3622 } 3623 } 3624 if (rc) 3625 goto fail3; 3626 3627 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3628 3629 /* Try to create MTDs, but allow this to fail */ 3630 rtnl_lock(); 3631 rc = efx_mtd_probe(efx); 3632 rtnl_unlock(); 3633 if (rc && rc != -EPERM) 3634 netif_warn(efx, probe, efx->net_dev, 3635 "failed to create MTDs (%d)\n", rc); 3636 3637 rc = pci_enable_pcie_error_reporting(pci_dev); 3638 if (rc && rc != -EINVAL) 3639 netif_notice(efx, probe, efx->net_dev, 3640 "PCIE error reporting unavailable (%d).\n", 3641 rc); 3642 3643 if (efx->type->udp_tnl_push_ports) 3644 efx->type->udp_tnl_push_ports(efx); 3645 3646 return 0; 3647 3648 fail3: 3649 efx_fini_io(efx); 3650 fail2: 3651 efx_fini_struct(efx); 3652 fail1: 3653 WARN_ON(rc > 0); 3654 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3655 free_netdev(net_dev); 3656 return rc; 3657 } 3658 3659 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3660 * enabled on success 3661 */ 3662 #ifdef CONFIG_SFC_SRIOV 3663 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3664 { 3665 int rc; 3666 struct efx_nic *efx = pci_get_drvdata(dev); 3667 3668 if (efx->type->sriov_configure) { 3669 rc = efx->type->sriov_configure(efx, num_vfs); 3670 if (rc) 3671 return rc; 3672 else 3673 return num_vfs; 3674 } else 3675 return -EOPNOTSUPP; 3676 } 3677 #endif 3678 3679 static int efx_pm_freeze(struct device *dev) 3680 { 3681 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3682 3683 rtnl_lock(); 3684 3685 if (efx->state != STATE_DISABLED) { 3686 efx->state = STATE_UNINIT; 3687 3688 efx_device_detach_sync(efx); 3689 3690 efx_stop_all(efx); 3691 efx_disable_interrupts(efx); 3692 } 3693 3694 rtnl_unlock(); 3695 3696 return 0; 3697 } 3698 3699 static int efx_pm_thaw(struct device *dev) 3700 { 3701 int rc; 3702 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3703 3704 rtnl_lock(); 3705 3706 if (efx->state != STATE_DISABLED) { 3707 rc = efx_enable_interrupts(efx); 3708 if (rc) 3709 goto fail; 3710 3711 mutex_lock(&efx->mac_lock); 3712 efx->phy_op->reconfigure(efx); 3713 mutex_unlock(&efx->mac_lock); 3714 3715 efx_start_all(efx); 3716 3717 efx_device_attach_if_not_resetting(efx); 3718 3719 efx->state = STATE_READY; 3720 3721 efx->type->resume_wol(efx); 3722 } 3723 3724 rtnl_unlock(); 3725 3726 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3727 queue_work(reset_workqueue, &efx->reset_work); 3728 3729 return 0; 3730 3731 fail: 3732 rtnl_unlock(); 3733 3734 return rc; 3735 } 3736 3737 static int efx_pm_poweroff(struct device *dev) 3738 { 3739 struct pci_dev *pci_dev = to_pci_dev(dev); 3740 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3741 3742 efx->type->fini(efx); 3743 3744 efx->reset_pending = 0; 3745 3746 pci_save_state(pci_dev); 3747 return pci_set_power_state(pci_dev, PCI_D3hot); 3748 } 3749 3750 /* Used for both resume and restore */ 3751 static int efx_pm_resume(struct device *dev) 3752 { 3753 struct pci_dev *pci_dev = to_pci_dev(dev); 3754 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3755 int rc; 3756 3757 rc = pci_set_power_state(pci_dev, PCI_D0); 3758 if (rc) 3759 return rc; 3760 pci_restore_state(pci_dev); 3761 rc = pci_enable_device(pci_dev); 3762 if (rc) 3763 return rc; 3764 pci_set_master(efx->pci_dev); 3765 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3766 if (rc) 3767 return rc; 3768 rc = efx->type->init(efx); 3769 if (rc) 3770 return rc; 3771 rc = efx_pm_thaw(dev); 3772 return rc; 3773 } 3774 3775 static int efx_pm_suspend(struct device *dev) 3776 { 3777 int rc; 3778 3779 efx_pm_freeze(dev); 3780 rc = efx_pm_poweroff(dev); 3781 if (rc) 3782 efx_pm_resume(dev); 3783 return rc; 3784 } 3785 3786 static const struct dev_pm_ops efx_pm_ops = { 3787 .suspend = efx_pm_suspend, 3788 .resume = efx_pm_resume, 3789 .freeze = efx_pm_freeze, 3790 .thaw = efx_pm_thaw, 3791 .poweroff = efx_pm_poweroff, 3792 .restore = efx_pm_resume, 3793 }; 3794 3795 /* A PCI error affecting this device was detected. 3796 * At this point MMIO and DMA may be disabled. 3797 * Stop the software path and request a slot reset. 3798 */ 3799 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3800 enum pci_channel_state state) 3801 { 3802 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3803 struct efx_nic *efx = pci_get_drvdata(pdev); 3804 3805 if (state == pci_channel_io_perm_failure) 3806 return PCI_ERS_RESULT_DISCONNECT; 3807 3808 rtnl_lock(); 3809 3810 if (efx->state != STATE_DISABLED) { 3811 efx->state = STATE_RECOVERY; 3812 efx->reset_pending = 0; 3813 3814 efx_device_detach_sync(efx); 3815 3816 efx_stop_all(efx); 3817 efx_disable_interrupts(efx); 3818 3819 status = PCI_ERS_RESULT_NEED_RESET; 3820 } else { 3821 /* If the interface is disabled we don't want to do anything 3822 * with it. 3823 */ 3824 status = PCI_ERS_RESULT_RECOVERED; 3825 } 3826 3827 rtnl_unlock(); 3828 3829 pci_disable_device(pdev); 3830 3831 return status; 3832 } 3833 3834 /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3835 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3836 { 3837 struct efx_nic *efx = pci_get_drvdata(pdev); 3838 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3839 int rc; 3840 3841 if (pci_enable_device(pdev)) { 3842 netif_err(efx, hw, efx->net_dev, 3843 "Cannot re-enable PCI device after reset.\n"); 3844 status = PCI_ERS_RESULT_DISCONNECT; 3845 } 3846 3847 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3848 if (rc) { 3849 netif_err(efx, hw, efx->net_dev, 3850 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3851 /* Non-fatal error. Continue. */ 3852 } 3853 3854 return status; 3855 } 3856 3857 /* Perform the actual reset and resume I/O operations. */ 3858 static void efx_io_resume(struct pci_dev *pdev) 3859 { 3860 struct efx_nic *efx = pci_get_drvdata(pdev); 3861 int rc; 3862 3863 rtnl_lock(); 3864 3865 if (efx->state == STATE_DISABLED) 3866 goto out; 3867 3868 rc = efx_reset(efx, RESET_TYPE_ALL); 3869 if (rc) { 3870 netif_err(efx, hw, efx->net_dev, 3871 "efx_reset failed after PCI error (%d)\n", rc); 3872 } else { 3873 efx->state = STATE_READY; 3874 netif_dbg(efx, hw, efx->net_dev, 3875 "Done resetting and resuming IO after PCI error.\n"); 3876 } 3877 3878 out: 3879 rtnl_unlock(); 3880 } 3881 3882 /* For simplicity and reliability, we always require a slot reset and try to 3883 * reset the hardware when a pci error affecting the device is detected. 3884 * We leave both the link_reset and mmio_enabled callback unimplemented: 3885 * with our request for slot reset the mmio_enabled callback will never be 3886 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3887 */ 3888 static const struct pci_error_handlers efx_err_handlers = { 3889 .error_detected = efx_io_error_detected, 3890 .slot_reset = efx_io_slot_reset, 3891 .resume = efx_io_resume, 3892 }; 3893 3894 static struct pci_driver efx_pci_driver = { 3895 .name = KBUILD_MODNAME, 3896 .id_table = efx_pci_table, 3897 .probe = efx_pci_probe, 3898 .remove = efx_pci_remove, 3899 .driver.pm = &efx_pm_ops, 3900 .err_handler = &efx_err_handlers, 3901 #ifdef CONFIG_SFC_SRIOV 3902 .sriov_configure = efx_pci_sriov_configure, 3903 #endif 3904 }; 3905 3906 /************************************************************************** 3907 * 3908 * Kernel module interface 3909 * 3910 *************************************************************************/ 3911 3912 module_param(interrupt_mode, uint, 0444); 3913 MODULE_PARM_DESC(interrupt_mode, 3914 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3915 3916 static int __init efx_init_module(void) 3917 { 3918 int rc; 3919 3920 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3921 3922 rc = register_netdevice_notifier(&efx_netdev_notifier); 3923 if (rc) 3924 goto err_notifier; 3925 3926 #ifdef CONFIG_SFC_SRIOV 3927 rc = efx_init_sriov(); 3928 if (rc) 3929 goto err_sriov; 3930 #endif 3931 3932 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3933 if (!reset_workqueue) { 3934 rc = -ENOMEM; 3935 goto err_reset; 3936 } 3937 3938 rc = pci_register_driver(&efx_pci_driver); 3939 if (rc < 0) 3940 goto err_pci; 3941 3942 return 0; 3943 3944 err_pci: 3945 destroy_workqueue(reset_workqueue); 3946 err_reset: 3947 #ifdef CONFIG_SFC_SRIOV 3948 efx_fini_sriov(); 3949 err_sriov: 3950 #endif 3951 unregister_netdevice_notifier(&efx_netdev_notifier); 3952 err_notifier: 3953 return rc; 3954 } 3955 3956 static void __exit efx_exit_module(void) 3957 { 3958 printk(KERN_INFO "Solarflare NET driver unloading\n"); 3959 3960 pci_unregister_driver(&efx_pci_driver); 3961 destroy_workqueue(reset_workqueue); 3962 #ifdef CONFIG_SFC_SRIOV 3963 efx_fini_sriov(); 3964 #endif 3965 unregister_netdevice_notifier(&efx_netdev_notifier); 3966 3967 } 3968 3969 module_init(efx_init_module); 3970 module_exit(efx_exit_module); 3971 3972 MODULE_AUTHOR("Solarflare Communications and " 3973 "Michael Brown <mbrown@fensystems.co.uk>"); 3974 MODULE_DESCRIPTION("Solarflare network driver"); 3975 MODULE_LICENSE("GPL"); 3976 MODULE_DEVICE_TABLE(pci, efx_pci_table); 3977 MODULE_VERSION(EFX_DRIVER_VERSION); 3978