1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include <net/gre.h> 27 #include <net/udp_tunnel.h> 28 #include "efx.h" 29 #include "nic.h" 30 #include "io.h" 31 #include "selftest.h" 32 #include "sriov.h" 33 34 #include "mcdi.h" 35 #include "mcdi_pcol.h" 36 #include "workarounds.h" 37 38 /************************************************************************** 39 * 40 * Type name strings 41 * 42 ************************************************************************** 43 */ 44 45 /* Loopback mode names (see LOOPBACK_MODE()) */ 46 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 47 const char *const efx_loopback_mode_names[] = { 48 [LOOPBACK_NONE] = "NONE", 49 [LOOPBACK_DATA] = "DATAPATH", 50 [LOOPBACK_GMAC] = "GMAC", 51 [LOOPBACK_XGMII] = "XGMII", 52 [LOOPBACK_XGXS] = "XGXS", 53 [LOOPBACK_XAUI] = "XAUI", 54 [LOOPBACK_GMII] = "GMII", 55 [LOOPBACK_SGMII] = "SGMII", 56 [LOOPBACK_XGBR] = "XGBR", 57 [LOOPBACK_XFI] = "XFI", 58 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 59 [LOOPBACK_GMII_FAR] = "GMII_FAR", 60 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 61 [LOOPBACK_XFI_FAR] = "XFI_FAR", 62 [LOOPBACK_GPHY] = "GPHY", 63 [LOOPBACK_PHYXS] = "PHYXS", 64 [LOOPBACK_PCS] = "PCS", 65 [LOOPBACK_PMAPMD] = "PMA/PMD", 66 [LOOPBACK_XPORT] = "XPORT", 67 [LOOPBACK_XGMII_WS] = "XGMII_WS", 68 [LOOPBACK_XAUI_WS] = "XAUI_WS", 69 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 70 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 71 [LOOPBACK_GMII_WS] = "GMII_WS", 72 [LOOPBACK_XFI_WS] = "XFI_WS", 73 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 74 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 75 }; 76 77 const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 78 const char *const efx_reset_type_names[] = { 79 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 80 [RESET_TYPE_ALL] = "ALL", 81 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 82 [RESET_TYPE_WORLD] = "WORLD", 83 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 84 [RESET_TYPE_DATAPATH] = "DATAPATH", 85 [RESET_TYPE_MC_BIST] = "MC_BIST", 86 [RESET_TYPE_DISABLE] = "DISABLE", 87 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 88 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 89 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 90 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 91 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 92 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 93 }; 94 95 /* UDP tunnel type names */ 96 static const char *const efx_udp_tunnel_type_names[] = { 97 [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan", 98 [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve", 99 }; 100 101 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen) 102 { 103 if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) && 104 efx_udp_tunnel_type_names[type] != NULL) 105 snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]); 106 else 107 snprintf(buf, buflen, "type %d", type); 108 } 109 110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 111 * queued onto this work queue. This is not a per-nic work queue, because 112 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 113 */ 114 static struct workqueue_struct *reset_workqueue; 115 116 /* How often and how many times to poll for a reset while waiting for a 117 * BIST that another function started to complete. 118 */ 119 #define BIST_WAIT_DELAY_MS 100 120 #define BIST_WAIT_DELAY_COUNT 100 121 122 /************************************************************************** 123 * 124 * Configurable values 125 * 126 *************************************************************************/ 127 128 /* 129 * Use separate channels for TX and RX events 130 * 131 * Set this to 1 to use separate channels for TX and RX. It allows us 132 * to control interrupt affinity separately for TX and RX. 133 * 134 * This is only used in MSI-X interrupt mode 135 */ 136 bool efx_separate_tx_channels; 137 module_param(efx_separate_tx_channels, bool, 0444); 138 MODULE_PARM_DESC(efx_separate_tx_channels, 139 "Use separate channels for TX and RX"); 140 141 /* This is the weight assigned to each of the (per-channel) virtual 142 * NAPI devices. 143 */ 144 static int napi_weight = 64; 145 146 /* This is the time (in jiffies) between invocations of the hardware 147 * monitor. 148 * On Falcon-based NICs, this will: 149 * - Check the on-board hardware monitor; 150 * - Poll the link state and reconfigure the hardware as necessary. 151 * On Siena-based NICs for power systems with EEH support, this will give EEH a 152 * chance to start. 153 */ 154 static unsigned int efx_monitor_interval = 1 * HZ; 155 156 /* Initial interrupt moderation settings. They can be modified after 157 * module load with ethtool. 158 * 159 * The default for RX should strike a balance between increasing the 160 * round-trip latency and reducing overhead. 161 */ 162 static unsigned int rx_irq_mod_usec = 60; 163 164 /* Initial interrupt moderation settings. They can be modified after 165 * module load with ethtool. 166 * 167 * This default is chosen to ensure that a 10G link does not go idle 168 * while a TX queue is stopped after it has become full. A queue is 169 * restarted when it drops below half full. The time this takes (assuming 170 * worst case 3 descriptors per packet and 1024 descriptors) is 171 * 512 / 3 * 1.2 = 205 usec. 172 */ 173 static unsigned int tx_irq_mod_usec = 150; 174 175 /* This is the first interrupt mode to try out of: 176 * 0 => MSI-X 177 * 1 => MSI 178 * 2 => legacy 179 */ 180 static unsigned int interrupt_mode; 181 182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 183 * i.e. the number of CPUs among which we may distribute simultaneous 184 * interrupt handling. 185 * 186 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 187 * The default (0) means to assign an interrupt to each core. 188 */ 189 static unsigned int rss_cpus; 190 module_param(rss_cpus, uint, 0444); 191 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 192 193 static bool phy_flash_cfg; 194 module_param(phy_flash_cfg, bool, 0644); 195 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 196 197 static unsigned irq_adapt_low_thresh = 8000; 198 module_param(irq_adapt_low_thresh, uint, 0644); 199 MODULE_PARM_DESC(irq_adapt_low_thresh, 200 "Threshold score for reducing IRQ moderation"); 201 202 static unsigned irq_adapt_high_thresh = 16000; 203 module_param(irq_adapt_high_thresh, uint, 0644); 204 MODULE_PARM_DESC(irq_adapt_high_thresh, 205 "Threshold score for increasing IRQ moderation"); 206 207 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 208 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 209 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 210 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 211 module_param(debug, uint, 0); 212 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 213 214 /************************************************************************** 215 * 216 * Utility functions and prototypes 217 * 218 *************************************************************************/ 219 220 static int efx_soft_enable_interrupts(struct efx_nic *efx); 221 static void efx_soft_disable_interrupts(struct efx_nic *efx); 222 static void efx_remove_channel(struct efx_channel *channel); 223 static void efx_remove_channels(struct efx_nic *efx); 224 static const struct efx_channel_type efx_default_channel_type; 225 static void efx_remove_port(struct efx_nic *efx); 226 static void efx_init_napi_channel(struct efx_channel *channel); 227 static void efx_fini_napi(struct efx_nic *efx); 228 static void efx_fini_napi_channel(struct efx_channel *channel); 229 static void efx_fini_struct(struct efx_nic *efx); 230 static void efx_start_all(struct efx_nic *efx); 231 static void efx_stop_all(struct efx_nic *efx); 232 233 #define EFX_ASSERT_RESET_SERIALISED(efx) \ 234 do { \ 235 if ((efx->state == STATE_READY) || \ 236 (efx->state == STATE_RECOVERY) || \ 237 (efx->state == STATE_DISABLED)) \ 238 ASSERT_RTNL(); \ 239 } while (0) 240 241 static int efx_check_disabled(struct efx_nic *efx) 242 { 243 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 244 netif_err(efx, drv, efx->net_dev, 245 "device is disabled due to earlier errors\n"); 246 return -EIO; 247 } 248 return 0; 249 } 250 251 /************************************************************************** 252 * 253 * Event queue processing 254 * 255 *************************************************************************/ 256 257 /* Process channel's event queue 258 * 259 * This function is responsible for processing the event queue of a 260 * single channel. The caller must guarantee that this function will 261 * never be concurrently called more than once on the same channel, 262 * though different channels may be being processed concurrently. 263 */ 264 static int efx_process_channel(struct efx_channel *channel, int budget) 265 { 266 struct efx_tx_queue *tx_queue; 267 int spent; 268 269 if (unlikely(!channel->enabled)) 270 return 0; 271 272 efx_for_each_channel_tx_queue(tx_queue, channel) { 273 tx_queue->pkts_compl = 0; 274 tx_queue->bytes_compl = 0; 275 } 276 277 spent = efx_nic_process_eventq(channel, budget); 278 if (spent && efx_channel_has_rx_queue(channel)) { 279 struct efx_rx_queue *rx_queue = 280 efx_channel_get_rx_queue(channel); 281 282 efx_rx_flush_packet(channel); 283 efx_fast_push_rx_descriptors(rx_queue, true); 284 } 285 286 /* Update BQL */ 287 efx_for_each_channel_tx_queue(tx_queue, channel) { 288 if (tx_queue->bytes_compl) { 289 netdev_tx_completed_queue(tx_queue->core_txq, 290 tx_queue->pkts_compl, tx_queue->bytes_compl); 291 } 292 } 293 294 return spent; 295 } 296 297 /* NAPI poll handler 298 * 299 * NAPI guarantees serialisation of polls of the same device, which 300 * provides the guarantee required by efx_process_channel(). 301 */ 302 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 303 { 304 int step = efx->irq_mod_step_us; 305 306 if (channel->irq_mod_score < irq_adapt_low_thresh) { 307 if (channel->irq_moderation_us > step) { 308 channel->irq_moderation_us -= step; 309 efx->type->push_irq_moderation(channel); 310 } 311 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 312 if (channel->irq_moderation_us < 313 efx->irq_rx_moderation_us) { 314 channel->irq_moderation_us += step; 315 efx->type->push_irq_moderation(channel); 316 } 317 } 318 319 channel->irq_count = 0; 320 channel->irq_mod_score = 0; 321 } 322 323 static int efx_poll(struct napi_struct *napi, int budget) 324 { 325 struct efx_channel *channel = 326 container_of(napi, struct efx_channel, napi_str); 327 struct efx_nic *efx = channel->efx; 328 int spent; 329 330 netif_vdbg(efx, intr, efx->net_dev, 331 "channel %d NAPI poll executing on CPU %d\n", 332 channel->channel, raw_smp_processor_id()); 333 334 spent = efx_process_channel(channel, budget); 335 336 if (spent < budget) { 337 if (efx_channel_has_rx_queue(channel) && 338 efx->irq_rx_adaptive && 339 unlikely(++channel->irq_count == 1000)) { 340 efx_update_irq_mod(efx, channel); 341 } 342 343 #ifdef CONFIG_RFS_ACCEL 344 /* Perhaps expire some ARFS filters */ 345 schedule_work(&channel->filter_work); 346 #endif 347 348 /* There is no race here; although napi_disable() will 349 * only wait for napi_complete(), this isn't a problem 350 * since efx_nic_eventq_read_ack() will have no effect if 351 * interrupts have already been disabled. 352 */ 353 if (napi_complete_done(napi, spent)) 354 efx_nic_eventq_read_ack(channel); 355 } 356 357 return spent; 358 } 359 360 /* Create event queue 361 * Event queue memory allocations are done only once. If the channel 362 * is reset, the memory buffer will be reused; this guards against 363 * errors during channel reset and also simplifies interrupt handling. 364 */ 365 static int efx_probe_eventq(struct efx_channel *channel) 366 { 367 struct efx_nic *efx = channel->efx; 368 unsigned long entries; 369 370 netif_dbg(efx, probe, efx->net_dev, 371 "chan %d create event queue\n", channel->channel); 372 373 /* Build an event queue with room for one event per tx and rx buffer, 374 * plus some extra for link state events and MCDI completions. */ 375 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 376 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 377 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 378 379 return efx_nic_probe_eventq(channel); 380 } 381 382 /* Prepare channel's event queue */ 383 static int efx_init_eventq(struct efx_channel *channel) 384 { 385 struct efx_nic *efx = channel->efx; 386 int rc; 387 388 EFX_WARN_ON_PARANOID(channel->eventq_init); 389 390 netif_dbg(efx, drv, efx->net_dev, 391 "chan %d init event queue\n", channel->channel); 392 393 rc = efx_nic_init_eventq(channel); 394 if (rc == 0) { 395 efx->type->push_irq_moderation(channel); 396 channel->eventq_read_ptr = 0; 397 channel->eventq_init = true; 398 } 399 return rc; 400 } 401 402 /* Enable event queue processing and NAPI */ 403 void efx_start_eventq(struct efx_channel *channel) 404 { 405 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 406 "chan %d start event queue\n", channel->channel); 407 408 /* Make sure the NAPI handler sees the enabled flag set */ 409 channel->enabled = true; 410 smp_wmb(); 411 412 napi_enable(&channel->napi_str); 413 efx_nic_eventq_read_ack(channel); 414 } 415 416 /* Disable event queue processing and NAPI */ 417 void efx_stop_eventq(struct efx_channel *channel) 418 { 419 if (!channel->enabled) 420 return; 421 422 napi_disable(&channel->napi_str); 423 channel->enabled = false; 424 } 425 426 static void efx_fini_eventq(struct efx_channel *channel) 427 { 428 if (!channel->eventq_init) 429 return; 430 431 netif_dbg(channel->efx, drv, channel->efx->net_dev, 432 "chan %d fini event queue\n", channel->channel); 433 434 efx_nic_fini_eventq(channel); 435 channel->eventq_init = false; 436 } 437 438 static void efx_remove_eventq(struct efx_channel *channel) 439 { 440 netif_dbg(channel->efx, drv, channel->efx->net_dev, 441 "chan %d remove event queue\n", channel->channel); 442 443 efx_nic_remove_eventq(channel); 444 } 445 446 /************************************************************************** 447 * 448 * Channel handling 449 * 450 *************************************************************************/ 451 452 /* Allocate and initialise a channel structure. */ 453 static struct efx_channel * 454 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 455 { 456 struct efx_channel *channel; 457 struct efx_rx_queue *rx_queue; 458 struct efx_tx_queue *tx_queue; 459 int j; 460 461 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 462 if (!channel) 463 return NULL; 464 465 channel->efx = efx; 466 channel->channel = i; 467 channel->type = &efx_default_channel_type; 468 469 for (j = 0; j < EFX_TXQ_TYPES; j++) { 470 tx_queue = &channel->tx_queue[j]; 471 tx_queue->efx = efx; 472 tx_queue->queue = i * EFX_TXQ_TYPES + j; 473 tx_queue->channel = channel; 474 } 475 476 #ifdef CONFIG_RFS_ACCEL 477 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 478 #endif 479 480 rx_queue = &channel->rx_queue; 481 rx_queue->efx = efx; 482 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 483 484 return channel; 485 } 486 487 /* Allocate and initialise a channel structure, copying parameters 488 * (but not resources) from an old channel structure. 489 */ 490 static struct efx_channel * 491 efx_copy_channel(const struct efx_channel *old_channel) 492 { 493 struct efx_channel *channel; 494 struct efx_rx_queue *rx_queue; 495 struct efx_tx_queue *tx_queue; 496 int j; 497 498 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 499 if (!channel) 500 return NULL; 501 502 *channel = *old_channel; 503 504 channel->napi_dev = NULL; 505 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 506 channel->napi_str.napi_id = 0; 507 channel->napi_str.state = 0; 508 memset(&channel->eventq, 0, sizeof(channel->eventq)); 509 510 for (j = 0; j < EFX_TXQ_TYPES; j++) { 511 tx_queue = &channel->tx_queue[j]; 512 if (tx_queue->channel) 513 tx_queue->channel = channel; 514 tx_queue->buffer = NULL; 515 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 516 } 517 518 rx_queue = &channel->rx_queue; 519 rx_queue->buffer = NULL; 520 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 521 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 522 #ifdef CONFIG_RFS_ACCEL 523 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 524 #endif 525 526 return channel; 527 } 528 529 static int efx_probe_channel(struct efx_channel *channel) 530 { 531 struct efx_tx_queue *tx_queue; 532 struct efx_rx_queue *rx_queue; 533 int rc; 534 535 netif_dbg(channel->efx, probe, channel->efx->net_dev, 536 "creating channel %d\n", channel->channel); 537 538 rc = channel->type->pre_probe(channel); 539 if (rc) 540 goto fail; 541 542 rc = efx_probe_eventq(channel); 543 if (rc) 544 goto fail; 545 546 efx_for_each_channel_tx_queue(tx_queue, channel) { 547 rc = efx_probe_tx_queue(tx_queue); 548 if (rc) 549 goto fail; 550 } 551 552 efx_for_each_channel_rx_queue(rx_queue, channel) { 553 rc = efx_probe_rx_queue(rx_queue); 554 if (rc) 555 goto fail; 556 } 557 558 return 0; 559 560 fail: 561 efx_remove_channel(channel); 562 return rc; 563 } 564 565 static void 566 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 567 { 568 struct efx_nic *efx = channel->efx; 569 const char *type; 570 int number; 571 572 number = channel->channel; 573 if (efx->tx_channel_offset == 0) { 574 type = ""; 575 } else if (channel->channel < efx->tx_channel_offset) { 576 type = "-rx"; 577 } else { 578 type = "-tx"; 579 number -= efx->tx_channel_offset; 580 } 581 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 582 } 583 584 static void efx_set_channel_names(struct efx_nic *efx) 585 { 586 struct efx_channel *channel; 587 588 efx_for_each_channel(channel, efx) 589 channel->type->get_name(channel, 590 efx->msi_context[channel->channel].name, 591 sizeof(efx->msi_context[0].name)); 592 } 593 594 static int efx_probe_channels(struct efx_nic *efx) 595 { 596 struct efx_channel *channel; 597 int rc; 598 599 /* Restart special buffer allocation */ 600 efx->next_buffer_table = 0; 601 602 /* Probe channels in reverse, so that any 'extra' channels 603 * use the start of the buffer table. This allows the traffic 604 * channels to be resized without moving them or wasting the 605 * entries before them. 606 */ 607 efx_for_each_channel_rev(channel, efx) { 608 rc = efx_probe_channel(channel); 609 if (rc) { 610 netif_err(efx, probe, efx->net_dev, 611 "failed to create channel %d\n", 612 channel->channel); 613 goto fail; 614 } 615 } 616 efx_set_channel_names(efx); 617 618 return 0; 619 620 fail: 621 efx_remove_channels(efx); 622 return rc; 623 } 624 625 /* Channels are shutdown and reinitialised whilst the NIC is running 626 * to propagate configuration changes (mtu, checksum offload), or 627 * to clear hardware error conditions 628 */ 629 static void efx_start_datapath(struct efx_nic *efx) 630 { 631 netdev_features_t old_features = efx->net_dev->features; 632 bool old_rx_scatter = efx->rx_scatter; 633 struct efx_tx_queue *tx_queue; 634 struct efx_rx_queue *rx_queue; 635 struct efx_channel *channel; 636 size_t rx_buf_len; 637 638 /* Calculate the rx buffer allocation parameters required to 639 * support the current MTU, including padding for header 640 * alignment and overruns. 641 */ 642 efx->rx_dma_len = (efx->rx_prefix_size + 643 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 644 efx->type->rx_buffer_padding); 645 rx_buf_len = (sizeof(struct efx_rx_page_state) + 646 efx->rx_ip_align + efx->rx_dma_len); 647 if (rx_buf_len <= PAGE_SIZE) { 648 efx->rx_scatter = efx->type->always_rx_scatter; 649 efx->rx_buffer_order = 0; 650 } else if (efx->type->can_rx_scatter) { 651 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 652 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 653 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 654 EFX_RX_BUF_ALIGNMENT) > 655 PAGE_SIZE); 656 efx->rx_scatter = true; 657 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 658 efx->rx_buffer_order = 0; 659 } else { 660 efx->rx_scatter = false; 661 efx->rx_buffer_order = get_order(rx_buf_len); 662 } 663 664 efx_rx_config_page_split(efx); 665 if (efx->rx_buffer_order) 666 netif_dbg(efx, drv, efx->net_dev, 667 "RX buf len=%u; page order=%u batch=%u\n", 668 efx->rx_dma_len, efx->rx_buffer_order, 669 efx->rx_pages_per_batch); 670 else 671 netif_dbg(efx, drv, efx->net_dev, 672 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 673 efx->rx_dma_len, efx->rx_page_buf_step, 674 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 675 676 /* Restore previously fixed features in hw_features and remove 677 * features which are fixed now 678 */ 679 efx->net_dev->hw_features |= efx->net_dev->features; 680 efx->net_dev->hw_features &= ~efx->fixed_features; 681 efx->net_dev->features |= efx->fixed_features; 682 if (efx->net_dev->features != old_features) 683 netdev_features_change(efx->net_dev); 684 685 /* RX filters may also have scatter-enabled flags */ 686 if (efx->rx_scatter != old_rx_scatter) 687 efx->type->filter_update_rx_scatter(efx); 688 689 /* We must keep at least one descriptor in a TX ring empty. 690 * We could avoid this when the queue size does not exactly 691 * match the hardware ring size, but it's not that important. 692 * Therefore we stop the queue when one more skb might fill 693 * the ring completely. We wake it when half way back to 694 * empty. 695 */ 696 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 697 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 698 699 /* Initialise the channels */ 700 efx_for_each_channel(channel, efx) { 701 efx_for_each_channel_tx_queue(tx_queue, channel) { 702 efx_init_tx_queue(tx_queue); 703 atomic_inc(&efx->active_queues); 704 } 705 706 efx_for_each_channel_rx_queue(rx_queue, channel) { 707 efx_init_rx_queue(rx_queue); 708 atomic_inc(&efx->active_queues); 709 efx_stop_eventq(channel); 710 efx_fast_push_rx_descriptors(rx_queue, false); 711 efx_start_eventq(channel); 712 } 713 714 WARN_ON(channel->rx_pkt_n_frags); 715 } 716 717 efx_ptp_start_datapath(efx); 718 719 if (netif_device_present(efx->net_dev)) 720 netif_tx_wake_all_queues(efx->net_dev); 721 } 722 723 static void efx_stop_datapath(struct efx_nic *efx) 724 { 725 struct efx_channel *channel; 726 struct efx_tx_queue *tx_queue; 727 struct efx_rx_queue *rx_queue; 728 int rc; 729 730 EFX_ASSERT_RESET_SERIALISED(efx); 731 BUG_ON(efx->port_enabled); 732 733 efx_ptp_stop_datapath(efx); 734 735 /* Stop RX refill */ 736 efx_for_each_channel(channel, efx) { 737 efx_for_each_channel_rx_queue(rx_queue, channel) 738 rx_queue->refill_enabled = false; 739 } 740 741 efx_for_each_channel(channel, efx) { 742 /* RX packet processing is pipelined, so wait for the 743 * NAPI handler to complete. At least event queue 0 744 * might be kept active by non-data events, so don't 745 * use napi_synchronize() but actually disable NAPI 746 * temporarily. 747 */ 748 if (efx_channel_has_rx_queue(channel)) { 749 efx_stop_eventq(channel); 750 efx_start_eventq(channel); 751 } 752 } 753 754 rc = efx->type->fini_dmaq(efx); 755 if (rc) { 756 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 757 } else { 758 netif_dbg(efx, drv, efx->net_dev, 759 "successfully flushed all queues\n"); 760 } 761 762 efx_for_each_channel(channel, efx) { 763 efx_for_each_channel_rx_queue(rx_queue, channel) 764 efx_fini_rx_queue(rx_queue); 765 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 766 efx_fini_tx_queue(tx_queue); 767 } 768 } 769 770 static void efx_remove_channel(struct efx_channel *channel) 771 { 772 struct efx_tx_queue *tx_queue; 773 struct efx_rx_queue *rx_queue; 774 775 netif_dbg(channel->efx, drv, channel->efx->net_dev, 776 "destroy chan %d\n", channel->channel); 777 778 efx_for_each_channel_rx_queue(rx_queue, channel) 779 efx_remove_rx_queue(rx_queue); 780 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 781 efx_remove_tx_queue(tx_queue); 782 efx_remove_eventq(channel); 783 channel->type->post_remove(channel); 784 } 785 786 static void efx_remove_channels(struct efx_nic *efx) 787 { 788 struct efx_channel *channel; 789 790 efx_for_each_channel(channel, efx) 791 efx_remove_channel(channel); 792 } 793 794 int 795 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 796 { 797 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 798 u32 old_rxq_entries, old_txq_entries; 799 unsigned i, next_buffer_table = 0; 800 int rc, rc2; 801 802 rc = efx_check_disabled(efx); 803 if (rc) 804 return rc; 805 806 /* Not all channels should be reallocated. We must avoid 807 * reallocating their buffer table entries. 808 */ 809 efx_for_each_channel(channel, efx) { 810 struct efx_rx_queue *rx_queue; 811 struct efx_tx_queue *tx_queue; 812 813 if (channel->type->copy) 814 continue; 815 next_buffer_table = max(next_buffer_table, 816 channel->eventq.index + 817 channel->eventq.entries); 818 efx_for_each_channel_rx_queue(rx_queue, channel) 819 next_buffer_table = max(next_buffer_table, 820 rx_queue->rxd.index + 821 rx_queue->rxd.entries); 822 efx_for_each_channel_tx_queue(tx_queue, channel) 823 next_buffer_table = max(next_buffer_table, 824 tx_queue->txd.index + 825 tx_queue->txd.entries); 826 } 827 828 efx_device_detach_sync(efx); 829 efx_stop_all(efx); 830 efx_soft_disable_interrupts(efx); 831 832 /* Clone channels (where possible) */ 833 memset(other_channel, 0, sizeof(other_channel)); 834 for (i = 0; i < efx->n_channels; i++) { 835 channel = efx->channel[i]; 836 if (channel->type->copy) 837 channel = channel->type->copy(channel); 838 if (!channel) { 839 rc = -ENOMEM; 840 goto out; 841 } 842 other_channel[i] = channel; 843 } 844 845 /* Swap entry counts and channel pointers */ 846 old_rxq_entries = efx->rxq_entries; 847 old_txq_entries = efx->txq_entries; 848 efx->rxq_entries = rxq_entries; 849 efx->txq_entries = txq_entries; 850 for (i = 0; i < efx->n_channels; i++) { 851 channel = efx->channel[i]; 852 efx->channel[i] = other_channel[i]; 853 other_channel[i] = channel; 854 } 855 856 /* Restart buffer table allocation */ 857 efx->next_buffer_table = next_buffer_table; 858 859 for (i = 0; i < efx->n_channels; i++) { 860 channel = efx->channel[i]; 861 if (!channel->type->copy) 862 continue; 863 rc = efx_probe_channel(channel); 864 if (rc) 865 goto rollback; 866 efx_init_napi_channel(efx->channel[i]); 867 } 868 869 out: 870 /* Destroy unused channel structures */ 871 for (i = 0; i < efx->n_channels; i++) { 872 channel = other_channel[i]; 873 if (channel && channel->type->copy) { 874 efx_fini_napi_channel(channel); 875 efx_remove_channel(channel); 876 kfree(channel); 877 } 878 } 879 880 rc2 = efx_soft_enable_interrupts(efx); 881 if (rc2) { 882 rc = rc ? rc : rc2; 883 netif_err(efx, drv, efx->net_dev, 884 "unable to restart interrupts on channel reallocation\n"); 885 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 886 } else { 887 efx_start_all(efx); 888 efx_device_attach_if_not_resetting(efx); 889 } 890 return rc; 891 892 rollback: 893 /* Swap back */ 894 efx->rxq_entries = old_rxq_entries; 895 efx->txq_entries = old_txq_entries; 896 for (i = 0; i < efx->n_channels; i++) { 897 channel = efx->channel[i]; 898 efx->channel[i] = other_channel[i]; 899 other_channel[i] = channel; 900 } 901 goto out; 902 } 903 904 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 905 { 906 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 907 } 908 909 static bool efx_default_channel_want_txqs(struct efx_channel *channel) 910 { 911 return channel->channel - channel->efx->tx_channel_offset < 912 channel->efx->n_tx_channels; 913 } 914 915 static const struct efx_channel_type efx_default_channel_type = { 916 .pre_probe = efx_channel_dummy_op_int, 917 .post_remove = efx_channel_dummy_op_void, 918 .get_name = efx_get_channel_name, 919 .copy = efx_copy_channel, 920 .want_txqs = efx_default_channel_want_txqs, 921 .keep_eventq = false, 922 .want_pio = true, 923 }; 924 925 int efx_channel_dummy_op_int(struct efx_channel *channel) 926 { 927 return 0; 928 } 929 930 void efx_channel_dummy_op_void(struct efx_channel *channel) 931 { 932 } 933 934 /************************************************************************** 935 * 936 * Port handling 937 * 938 **************************************************************************/ 939 940 /* This ensures that the kernel is kept informed (via 941 * netif_carrier_on/off) of the link status, and also maintains the 942 * link status's stop on the port's TX queue. 943 */ 944 void efx_link_status_changed(struct efx_nic *efx) 945 { 946 struct efx_link_state *link_state = &efx->link_state; 947 948 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 949 * that no events are triggered between unregister_netdev() and the 950 * driver unloading. A more general condition is that NETDEV_CHANGE 951 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 952 if (!netif_running(efx->net_dev)) 953 return; 954 955 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 956 efx->n_link_state_changes++; 957 958 if (link_state->up) 959 netif_carrier_on(efx->net_dev); 960 else 961 netif_carrier_off(efx->net_dev); 962 } 963 964 /* Status message for kernel log */ 965 if (link_state->up) 966 netif_info(efx, link, efx->net_dev, 967 "link up at %uMbps %s-duplex (MTU %d)\n", 968 link_state->speed, link_state->fd ? "full" : "half", 969 efx->net_dev->mtu); 970 else 971 netif_info(efx, link, efx->net_dev, "link down\n"); 972 } 973 974 void efx_link_set_advertising(struct efx_nic *efx, 975 const unsigned long *advertising) 976 { 977 memcpy(efx->link_advertising, advertising, 978 sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK())); 979 980 efx->link_advertising[0] |= ADVERTISED_Autoneg; 981 if (advertising[0] & ADVERTISED_Pause) 982 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 983 else 984 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 985 if (advertising[0] & ADVERTISED_Asym_Pause) 986 efx->wanted_fc ^= EFX_FC_TX; 987 } 988 989 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not 990 * force the Autoneg bit on. 991 */ 992 void efx_link_clear_advertising(struct efx_nic *efx) 993 { 994 bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS); 995 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 996 } 997 998 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 999 { 1000 efx->wanted_fc = wanted_fc; 1001 if (efx->link_advertising[0]) { 1002 if (wanted_fc & EFX_FC_RX) 1003 efx->link_advertising[0] |= (ADVERTISED_Pause | 1004 ADVERTISED_Asym_Pause); 1005 else 1006 efx->link_advertising[0] &= ~(ADVERTISED_Pause | 1007 ADVERTISED_Asym_Pause); 1008 if (wanted_fc & EFX_FC_TX) 1009 efx->link_advertising[0] ^= ADVERTISED_Asym_Pause; 1010 } 1011 } 1012 1013 static void efx_fini_port(struct efx_nic *efx); 1014 1015 /* We assume that efx->type->reconfigure_mac will always try to sync RX 1016 * filters and therefore needs to read-lock the filter table against freeing 1017 */ 1018 void efx_mac_reconfigure(struct efx_nic *efx) 1019 { 1020 down_read(&efx->filter_sem); 1021 efx->type->reconfigure_mac(efx); 1022 up_read(&efx->filter_sem); 1023 } 1024 1025 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 1026 * the MAC appropriately. All other PHY configuration changes are pushed 1027 * through phy_op->set_settings(), and pushed asynchronously to the MAC 1028 * through efx_monitor(). 1029 * 1030 * Callers must hold the mac_lock 1031 */ 1032 int __efx_reconfigure_port(struct efx_nic *efx) 1033 { 1034 enum efx_phy_mode phy_mode; 1035 int rc; 1036 1037 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1038 1039 /* Disable PHY transmit in mac level loopbacks */ 1040 phy_mode = efx->phy_mode; 1041 if (LOOPBACK_INTERNAL(efx)) 1042 efx->phy_mode |= PHY_MODE_TX_DISABLED; 1043 else 1044 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1045 1046 rc = efx->type->reconfigure_port(efx); 1047 1048 if (rc) 1049 efx->phy_mode = phy_mode; 1050 1051 return rc; 1052 } 1053 1054 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1055 * disabled. */ 1056 int efx_reconfigure_port(struct efx_nic *efx) 1057 { 1058 int rc; 1059 1060 EFX_ASSERT_RESET_SERIALISED(efx); 1061 1062 mutex_lock(&efx->mac_lock); 1063 rc = __efx_reconfigure_port(efx); 1064 mutex_unlock(&efx->mac_lock); 1065 1066 return rc; 1067 } 1068 1069 /* Asynchronous work item for changing MAC promiscuity and multicast 1070 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1071 * MAC directly. */ 1072 static void efx_mac_work(struct work_struct *data) 1073 { 1074 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1075 1076 mutex_lock(&efx->mac_lock); 1077 if (efx->port_enabled) 1078 efx_mac_reconfigure(efx); 1079 mutex_unlock(&efx->mac_lock); 1080 } 1081 1082 static int efx_probe_port(struct efx_nic *efx) 1083 { 1084 int rc; 1085 1086 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1087 1088 if (phy_flash_cfg) 1089 efx->phy_mode = PHY_MODE_SPECIAL; 1090 1091 /* Connect up MAC/PHY operations table */ 1092 rc = efx->type->probe_port(efx); 1093 if (rc) 1094 return rc; 1095 1096 /* Initialise MAC address to permanent address */ 1097 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1098 1099 return 0; 1100 } 1101 1102 static int efx_init_port(struct efx_nic *efx) 1103 { 1104 int rc; 1105 1106 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1107 1108 mutex_lock(&efx->mac_lock); 1109 1110 rc = efx->phy_op->init(efx); 1111 if (rc) 1112 goto fail1; 1113 1114 efx->port_initialized = true; 1115 1116 /* Reconfigure the MAC before creating dma queues (required for 1117 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1118 efx_mac_reconfigure(efx); 1119 1120 /* Ensure the PHY advertises the correct flow control settings */ 1121 rc = efx->phy_op->reconfigure(efx); 1122 if (rc && rc != -EPERM) 1123 goto fail2; 1124 1125 mutex_unlock(&efx->mac_lock); 1126 return 0; 1127 1128 fail2: 1129 efx->phy_op->fini(efx); 1130 fail1: 1131 mutex_unlock(&efx->mac_lock); 1132 return rc; 1133 } 1134 1135 static void efx_start_port(struct efx_nic *efx) 1136 { 1137 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1138 BUG_ON(efx->port_enabled); 1139 1140 mutex_lock(&efx->mac_lock); 1141 efx->port_enabled = true; 1142 1143 /* Ensure MAC ingress/egress is enabled */ 1144 efx_mac_reconfigure(efx); 1145 1146 mutex_unlock(&efx->mac_lock); 1147 } 1148 1149 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1150 * and the async self-test, wait for them to finish and prevent them 1151 * being scheduled again. This doesn't cover online resets, which 1152 * should only be cancelled when removing the device. 1153 */ 1154 static void efx_stop_port(struct efx_nic *efx) 1155 { 1156 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1157 1158 EFX_ASSERT_RESET_SERIALISED(efx); 1159 1160 mutex_lock(&efx->mac_lock); 1161 efx->port_enabled = false; 1162 mutex_unlock(&efx->mac_lock); 1163 1164 /* Serialise against efx_set_multicast_list() */ 1165 netif_addr_lock_bh(efx->net_dev); 1166 netif_addr_unlock_bh(efx->net_dev); 1167 1168 cancel_delayed_work_sync(&efx->monitor_work); 1169 efx_selftest_async_cancel(efx); 1170 cancel_work_sync(&efx->mac_work); 1171 } 1172 1173 static void efx_fini_port(struct efx_nic *efx) 1174 { 1175 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1176 1177 if (!efx->port_initialized) 1178 return; 1179 1180 efx->phy_op->fini(efx); 1181 efx->port_initialized = false; 1182 1183 efx->link_state.up = false; 1184 efx_link_status_changed(efx); 1185 } 1186 1187 static void efx_remove_port(struct efx_nic *efx) 1188 { 1189 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1190 1191 efx->type->remove_port(efx); 1192 } 1193 1194 /************************************************************************** 1195 * 1196 * NIC handling 1197 * 1198 **************************************************************************/ 1199 1200 static LIST_HEAD(efx_primary_list); 1201 static LIST_HEAD(efx_unassociated_list); 1202 1203 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 1204 { 1205 return left->type == right->type && 1206 left->vpd_sn && right->vpd_sn && 1207 !strcmp(left->vpd_sn, right->vpd_sn); 1208 } 1209 1210 static void efx_associate(struct efx_nic *efx) 1211 { 1212 struct efx_nic *other, *next; 1213 1214 if (efx->primary == efx) { 1215 /* Adding primary function; look for secondaries */ 1216 1217 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1218 list_add_tail(&efx->node, &efx_primary_list); 1219 1220 list_for_each_entry_safe(other, next, &efx_unassociated_list, 1221 node) { 1222 if (efx_same_controller(efx, other)) { 1223 list_del(&other->node); 1224 netif_dbg(other, probe, other->net_dev, 1225 "moving to secondary list of %s %s\n", 1226 pci_name(efx->pci_dev), 1227 efx->net_dev->name); 1228 list_add_tail(&other->node, 1229 &efx->secondary_list); 1230 other->primary = efx; 1231 } 1232 } 1233 } else { 1234 /* Adding secondary function; look for primary */ 1235 1236 list_for_each_entry(other, &efx_primary_list, node) { 1237 if (efx_same_controller(efx, other)) { 1238 netif_dbg(efx, probe, efx->net_dev, 1239 "adding to secondary list of %s %s\n", 1240 pci_name(other->pci_dev), 1241 other->net_dev->name); 1242 list_add_tail(&efx->node, 1243 &other->secondary_list); 1244 efx->primary = other; 1245 return; 1246 } 1247 } 1248 1249 netif_dbg(efx, probe, efx->net_dev, 1250 "adding to unassociated list\n"); 1251 list_add_tail(&efx->node, &efx_unassociated_list); 1252 } 1253 } 1254 1255 static void efx_dissociate(struct efx_nic *efx) 1256 { 1257 struct efx_nic *other, *next; 1258 1259 list_del(&efx->node); 1260 efx->primary = NULL; 1261 1262 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1263 list_del(&other->node); 1264 netif_dbg(other, probe, other->net_dev, 1265 "moving to unassociated list\n"); 1266 list_add_tail(&other->node, &efx_unassociated_list); 1267 other->primary = NULL; 1268 } 1269 } 1270 1271 /* This configures the PCI device to enable I/O and DMA. */ 1272 static int efx_init_io(struct efx_nic *efx) 1273 { 1274 struct pci_dev *pci_dev = efx->pci_dev; 1275 dma_addr_t dma_mask = efx->type->max_dma_mask; 1276 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1277 int rc, bar; 1278 1279 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1280 1281 bar = efx->type->mem_bar(efx); 1282 1283 rc = pci_enable_device(pci_dev); 1284 if (rc) { 1285 netif_err(efx, probe, efx->net_dev, 1286 "failed to enable PCI device\n"); 1287 goto fail1; 1288 } 1289 1290 pci_set_master(pci_dev); 1291 1292 /* Set the PCI DMA mask. Try all possibilities from our genuine mask 1293 * down to 32 bits, because some architectures will allow 40 bit 1294 * masks event though they reject 46 bit masks. 1295 */ 1296 while (dma_mask > 0x7fffffffUL) { 1297 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1298 if (rc == 0) 1299 break; 1300 dma_mask >>= 1; 1301 } 1302 if (rc) { 1303 netif_err(efx, probe, efx->net_dev, 1304 "could not find a suitable DMA mask\n"); 1305 goto fail2; 1306 } 1307 netif_dbg(efx, probe, efx->net_dev, 1308 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1309 1310 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1311 rc = pci_request_region(pci_dev, bar, "sfc"); 1312 if (rc) { 1313 netif_err(efx, probe, efx->net_dev, 1314 "request for memory BAR failed\n"); 1315 rc = -EIO; 1316 goto fail3; 1317 } 1318 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1319 if (!efx->membase) { 1320 netif_err(efx, probe, efx->net_dev, 1321 "could not map memory BAR at %llx+%x\n", 1322 (unsigned long long)efx->membase_phys, mem_map_size); 1323 rc = -ENOMEM; 1324 goto fail4; 1325 } 1326 netif_dbg(efx, probe, efx->net_dev, 1327 "memory BAR at %llx+%x (virtual %p)\n", 1328 (unsigned long long)efx->membase_phys, mem_map_size, 1329 efx->membase); 1330 1331 return 0; 1332 1333 fail4: 1334 pci_release_region(efx->pci_dev, bar); 1335 fail3: 1336 efx->membase_phys = 0; 1337 fail2: 1338 pci_disable_device(efx->pci_dev); 1339 fail1: 1340 return rc; 1341 } 1342 1343 static void efx_fini_io(struct efx_nic *efx) 1344 { 1345 int bar; 1346 1347 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1348 1349 if (efx->membase) { 1350 iounmap(efx->membase); 1351 efx->membase = NULL; 1352 } 1353 1354 if (efx->membase_phys) { 1355 bar = efx->type->mem_bar(efx); 1356 pci_release_region(efx->pci_dev, bar); 1357 efx->membase_phys = 0; 1358 } 1359 1360 /* Don't disable bus-mastering if VFs are assigned */ 1361 if (!pci_vfs_assigned(efx->pci_dev)) 1362 pci_disable_device(efx->pci_dev); 1363 } 1364 1365 void efx_set_default_rx_indir_table(struct efx_nic *efx, 1366 struct efx_rss_context *ctx) 1367 { 1368 size_t i; 1369 1370 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 1371 ctx->rx_indir_table[i] = 1372 ethtool_rxfh_indir_default(i, efx->rss_spread); 1373 } 1374 1375 static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1376 { 1377 cpumask_var_t thread_mask; 1378 unsigned int count; 1379 int cpu; 1380 1381 if (rss_cpus) { 1382 count = rss_cpus; 1383 } else { 1384 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1385 netif_warn(efx, probe, efx->net_dev, 1386 "RSS disabled due to allocation failure\n"); 1387 return 1; 1388 } 1389 1390 count = 0; 1391 for_each_online_cpu(cpu) { 1392 if (!cpumask_test_cpu(cpu, thread_mask)) { 1393 ++count; 1394 cpumask_or(thread_mask, thread_mask, 1395 topology_sibling_cpumask(cpu)); 1396 } 1397 } 1398 1399 free_cpumask_var(thread_mask); 1400 } 1401 1402 if (count > EFX_MAX_RX_QUEUES) { 1403 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1404 "Reducing number of rx queues from %u to %u.\n", 1405 count, EFX_MAX_RX_QUEUES); 1406 count = EFX_MAX_RX_QUEUES; 1407 } 1408 1409 /* If RSS is requested for the PF *and* VFs then we can't write RSS 1410 * table entries that are inaccessible to VFs 1411 */ 1412 #ifdef CONFIG_SFC_SRIOV 1413 if (efx->type->sriov_wanted) { 1414 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1415 count > efx_vf_size(efx)) { 1416 netif_warn(efx, probe, efx->net_dev, 1417 "Reducing number of RSS channels from %u to %u for " 1418 "VF support. Increase vf-msix-limit to use more " 1419 "channels on the PF.\n", 1420 count, efx_vf_size(efx)); 1421 count = efx_vf_size(efx); 1422 } 1423 } 1424 #endif 1425 1426 return count; 1427 } 1428 1429 /* Probe the number and type of interrupts we are able to obtain, and 1430 * the resulting numbers of channels and RX queues. 1431 */ 1432 static int efx_probe_interrupts(struct efx_nic *efx) 1433 { 1434 unsigned int extra_channels = 0; 1435 unsigned int i, j; 1436 int rc; 1437 1438 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 1439 if (efx->extra_channel_type[i]) 1440 ++extra_channels; 1441 1442 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1443 struct msix_entry xentries[EFX_MAX_CHANNELS]; 1444 unsigned int n_channels; 1445 1446 n_channels = efx_wanted_parallelism(efx); 1447 if (efx_separate_tx_channels) 1448 n_channels *= 2; 1449 n_channels += extra_channels; 1450 n_channels = min(n_channels, efx->max_channels); 1451 1452 for (i = 0; i < n_channels; i++) 1453 xentries[i].entry = i; 1454 rc = pci_enable_msix_range(efx->pci_dev, 1455 xentries, 1, n_channels); 1456 if (rc < 0) { 1457 /* Fall back to single channel MSI */ 1458 netif_err(efx, drv, efx->net_dev, 1459 "could not enable MSI-X\n"); 1460 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 1461 efx->interrupt_mode = EFX_INT_MODE_MSI; 1462 else 1463 return rc; 1464 } else if (rc < n_channels) { 1465 netif_err(efx, drv, efx->net_dev, 1466 "WARNING: Insufficient MSI-X vectors" 1467 " available (%d < %u).\n", rc, n_channels); 1468 netif_err(efx, drv, efx->net_dev, 1469 "WARNING: Performance may be reduced.\n"); 1470 n_channels = rc; 1471 } 1472 1473 if (rc > 0) { 1474 efx->n_channels = n_channels; 1475 if (n_channels > extra_channels) 1476 n_channels -= extra_channels; 1477 if (efx_separate_tx_channels) { 1478 efx->n_tx_channels = min(max(n_channels / 2, 1479 1U), 1480 efx->max_tx_channels); 1481 efx->n_rx_channels = max(n_channels - 1482 efx->n_tx_channels, 1483 1U); 1484 } else { 1485 efx->n_tx_channels = min(n_channels, 1486 efx->max_tx_channels); 1487 efx->n_rx_channels = n_channels; 1488 } 1489 for (i = 0; i < efx->n_channels; i++) 1490 efx_get_channel(efx, i)->irq = 1491 xentries[i].vector; 1492 } 1493 } 1494 1495 /* Try single interrupt MSI */ 1496 if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1497 efx->n_channels = 1; 1498 efx->n_rx_channels = 1; 1499 efx->n_tx_channels = 1; 1500 rc = pci_enable_msi(efx->pci_dev); 1501 if (rc == 0) { 1502 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1503 } else { 1504 netif_err(efx, drv, efx->net_dev, 1505 "could not enable MSI\n"); 1506 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 1507 efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1508 else 1509 return rc; 1510 } 1511 } 1512 1513 /* Assume legacy interrupts */ 1514 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1515 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1516 efx->n_rx_channels = 1; 1517 efx->n_tx_channels = 1; 1518 efx->legacy_irq = efx->pci_dev->irq; 1519 } 1520 1521 /* Assign extra channels if possible */ 1522 efx->n_extra_tx_channels = 0; 1523 j = efx->n_channels; 1524 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 1525 if (!efx->extra_channel_type[i]) 1526 continue; 1527 if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 1528 efx->n_channels <= extra_channels) { 1529 efx->extra_channel_type[i]->handle_no_channel(efx); 1530 } else { 1531 --j; 1532 efx_get_channel(efx, j)->type = 1533 efx->extra_channel_type[i]; 1534 if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 1535 efx->n_extra_tx_channels++; 1536 } 1537 } 1538 1539 /* RSS might be usable on VFs even if it is disabled on the PF */ 1540 #ifdef CONFIG_SFC_SRIOV 1541 if (efx->type->sriov_wanted) { 1542 efx->rss_spread = ((efx->n_rx_channels > 1 || 1543 !efx->type->sriov_wanted(efx)) ? 1544 efx->n_rx_channels : efx_vf_size(efx)); 1545 return 0; 1546 } 1547 #endif 1548 efx->rss_spread = efx->n_rx_channels; 1549 1550 return 0; 1551 } 1552 1553 #if defined(CONFIG_SMP) 1554 static void efx_set_interrupt_affinity(struct efx_nic *efx) 1555 { 1556 struct efx_channel *channel; 1557 unsigned int cpu; 1558 1559 efx_for_each_channel(channel, efx) { 1560 cpu = cpumask_local_spread(channel->channel, 1561 pcibus_to_node(efx->pci_dev->bus)); 1562 irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 1563 } 1564 } 1565 1566 static void efx_clear_interrupt_affinity(struct efx_nic *efx) 1567 { 1568 struct efx_channel *channel; 1569 1570 efx_for_each_channel(channel, efx) 1571 irq_set_affinity_hint(channel->irq, NULL); 1572 } 1573 #else 1574 static void 1575 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1576 { 1577 } 1578 1579 static void 1580 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1581 { 1582 } 1583 #endif /* CONFIG_SMP */ 1584 1585 static int efx_soft_enable_interrupts(struct efx_nic *efx) 1586 { 1587 struct efx_channel *channel, *end_channel; 1588 int rc; 1589 1590 BUG_ON(efx->state == STATE_DISABLED); 1591 1592 efx->irq_soft_enabled = true; 1593 smp_wmb(); 1594 1595 efx_for_each_channel(channel, efx) { 1596 if (!channel->type->keep_eventq) { 1597 rc = efx_init_eventq(channel); 1598 if (rc) 1599 goto fail; 1600 } 1601 efx_start_eventq(channel); 1602 } 1603 1604 efx_mcdi_mode_event(efx); 1605 1606 return 0; 1607 fail: 1608 end_channel = channel; 1609 efx_for_each_channel(channel, efx) { 1610 if (channel == end_channel) 1611 break; 1612 efx_stop_eventq(channel); 1613 if (!channel->type->keep_eventq) 1614 efx_fini_eventq(channel); 1615 } 1616 1617 return rc; 1618 } 1619 1620 static void efx_soft_disable_interrupts(struct efx_nic *efx) 1621 { 1622 struct efx_channel *channel; 1623 1624 if (efx->state == STATE_DISABLED) 1625 return; 1626 1627 efx_mcdi_mode_poll(efx); 1628 1629 efx->irq_soft_enabled = false; 1630 smp_wmb(); 1631 1632 if (efx->legacy_irq) 1633 synchronize_irq(efx->legacy_irq); 1634 1635 efx_for_each_channel(channel, efx) { 1636 if (channel->irq) 1637 synchronize_irq(channel->irq); 1638 1639 efx_stop_eventq(channel); 1640 if (!channel->type->keep_eventq) 1641 efx_fini_eventq(channel); 1642 } 1643 1644 /* Flush the asynchronous MCDI request queue */ 1645 efx_mcdi_flush_async(efx); 1646 } 1647 1648 static int efx_enable_interrupts(struct efx_nic *efx) 1649 { 1650 struct efx_channel *channel, *end_channel; 1651 int rc; 1652 1653 BUG_ON(efx->state == STATE_DISABLED); 1654 1655 if (efx->eeh_disabled_legacy_irq) { 1656 enable_irq(efx->legacy_irq); 1657 efx->eeh_disabled_legacy_irq = false; 1658 } 1659 1660 efx->type->irq_enable_master(efx); 1661 1662 efx_for_each_channel(channel, efx) { 1663 if (channel->type->keep_eventq) { 1664 rc = efx_init_eventq(channel); 1665 if (rc) 1666 goto fail; 1667 } 1668 } 1669 1670 rc = efx_soft_enable_interrupts(efx); 1671 if (rc) 1672 goto fail; 1673 1674 return 0; 1675 1676 fail: 1677 end_channel = channel; 1678 efx_for_each_channel(channel, efx) { 1679 if (channel == end_channel) 1680 break; 1681 if (channel->type->keep_eventq) 1682 efx_fini_eventq(channel); 1683 } 1684 1685 efx->type->irq_disable_non_ev(efx); 1686 1687 return rc; 1688 } 1689 1690 static void efx_disable_interrupts(struct efx_nic *efx) 1691 { 1692 struct efx_channel *channel; 1693 1694 efx_soft_disable_interrupts(efx); 1695 1696 efx_for_each_channel(channel, efx) { 1697 if (channel->type->keep_eventq) 1698 efx_fini_eventq(channel); 1699 } 1700 1701 efx->type->irq_disable_non_ev(efx); 1702 } 1703 1704 static void efx_remove_interrupts(struct efx_nic *efx) 1705 { 1706 struct efx_channel *channel; 1707 1708 /* Remove MSI/MSI-X interrupts */ 1709 efx_for_each_channel(channel, efx) 1710 channel->irq = 0; 1711 pci_disable_msi(efx->pci_dev); 1712 pci_disable_msix(efx->pci_dev); 1713 1714 /* Remove legacy interrupt */ 1715 efx->legacy_irq = 0; 1716 } 1717 1718 static void efx_set_channels(struct efx_nic *efx) 1719 { 1720 struct efx_channel *channel; 1721 struct efx_tx_queue *tx_queue; 1722 1723 efx->tx_channel_offset = 1724 efx_separate_tx_channels ? 1725 efx->n_channels - efx->n_tx_channels : 0; 1726 1727 /* We need to mark which channels really have RX and TX 1728 * queues, and adjust the TX queue numbers if we have separate 1729 * RX-only and TX-only channels. 1730 */ 1731 efx_for_each_channel(channel, efx) { 1732 if (channel->channel < efx->n_rx_channels) 1733 channel->rx_queue.core_index = channel->channel; 1734 else 1735 channel->rx_queue.core_index = -1; 1736 1737 efx_for_each_channel_tx_queue(tx_queue, channel) 1738 tx_queue->queue -= (efx->tx_channel_offset * 1739 EFX_TXQ_TYPES); 1740 } 1741 } 1742 1743 static int efx_probe_nic(struct efx_nic *efx) 1744 { 1745 int rc; 1746 1747 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1748 1749 /* Carry out hardware-type specific initialisation */ 1750 rc = efx->type->probe(efx); 1751 if (rc) 1752 return rc; 1753 1754 do { 1755 if (!efx->max_channels || !efx->max_tx_channels) { 1756 netif_err(efx, drv, efx->net_dev, 1757 "Insufficient resources to allocate" 1758 " any channels\n"); 1759 rc = -ENOSPC; 1760 goto fail1; 1761 } 1762 1763 /* Determine the number of channels and queues by trying 1764 * to hook in MSI-X interrupts. 1765 */ 1766 rc = efx_probe_interrupts(efx); 1767 if (rc) 1768 goto fail1; 1769 1770 efx_set_channels(efx); 1771 1772 /* dimension_resources can fail with EAGAIN */ 1773 rc = efx->type->dimension_resources(efx); 1774 if (rc != 0 && rc != -EAGAIN) 1775 goto fail2; 1776 1777 if (rc == -EAGAIN) 1778 /* try again with new max_channels */ 1779 efx_remove_interrupts(efx); 1780 1781 } while (rc == -EAGAIN); 1782 1783 if (efx->n_channels > 1) 1784 netdev_rss_key_fill(efx->rss_context.rx_hash_key, 1785 sizeof(efx->rss_context.rx_hash_key)); 1786 efx_set_default_rx_indir_table(efx, &efx->rss_context); 1787 1788 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1789 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1790 1791 /* Initialise the interrupt moderation settings */ 1792 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1793 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1794 true); 1795 1796 return 0; 1797 1798 fail2: 1799 efx_remove_interrupts(efx); 1800 fail1: 1801 efx->type->remove(efx); 1802 return rc; 1803 } 1804 1805 static void efx_remove_nic(struct efx_nic *efx) 1806 { 1807 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1808 1809 efx_remove_interrupts(efx); 1810 efx->type->remove(efx); 1811 } 1812 1813 static int efx_probe_filters(struct efx_nic *efx) 1814 { 1815 int rc; 1816 1817 init_rwsem(&efx->filter_sem); 1818 mutex_lock(&efx->mac_lock); 1819 down_write(&efx->filter_sem); 1820 rc = efx->type->filter_table_probe(efx); 1821 if (rc) 1822 goto out_unlock; 1823 1824 #ifdef CONFIG_RFS_ACCEL 1825 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1826 struct efx_channel *channel; 1827 int i, success = 1; 1828 1829 efx_for_each_channel(channel, efx) { 1830 channel->rps_flow_id = 1831 kcalloc(efx->type->max_rx_ip_filters, 1832 sizeof(*channel->rps_flow_id), 1833 GFP_KERNEL); 1834 if (!channel->rps_flow_id) 1835 success = 0; 1836 else 1837 for (i = 0; 1838 i < efx->type->max_rx_ip_filters; 1839 ++i) 1840 channel->rps_flow_id[i] = 1841 RPS_FLOW_ID_INVALID; 1842 } 1843 1844 if (!success) { 1845 efx_for_each_channel(channel, efx) 1846 kfree(channel->rps_flow_id); 1847 efx->type->filter_table_remove(efx); 1848 rc = -ENOMEM; 1849 goto out_unlock; 1850 } 1851 1852 efx->rps_expire_index = efx->rps_expire_channel = 0; 1853 } 1854 #endif 1855 out_unlock: 1856 up_write(&efx->filter_sem); 1857 mutex_unlock(&efx->mac_lock); 1858 return rc; 1859 } 1860 1861 static void efx_remove_filters(struct efx_nic *efx) 1862 { 1863 #ifdef CONFIG_RFS_ACCEL 1864 struct efx_channel *channel; 1865 1866 efx_for_each_channel(channel, efx) 1867 kfree(channel->rps_flow_id); 1868 #endif 1869 down_write(&efx->filter_sem); 1870 efx->type->filter_table_remove(efx); 1871 up_write(&efx->filter_sem); 1872 } 1873 1874 1875 /************************************************************************** 1876 * 1877 * NIC startup/shutdown 1878 * 1879 *************************************************************************/ 1880 1881 static int efx_probe_all(struct efx_nic *efx) 1882 { 1883 int rc; 1884 1885 rc = efx_probe_nic(efx); 1886 if (rc) { 1887 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1888 goto fail1; 1889 } 1890 1891 rc = efx_probe_port(efx); 1892 if (rc) { 1893 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1894 goto fail2; 1895 } 1896 1897 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 1898 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 1899 rc = -EINVAL; 1900 goto fail3; 1901 } 1902 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1903 1904 #ifdef CONFIG_SFC_SRIOV 1905 rc = efx->type->vswitching_probe(efx); 1906 if (rc) /* not fatal; the PF will still work fine */ 1907 netif_warn(efx, probe, efx->net_dev, 1908 "failed to setup vswitching rc=%d;" 1909 " VFs may not function\n", rc); 1910 #endif 1911 1912 rc = efx_probe_filters(efx); 1913 if (rc) { 1914 netif_err(efx, probe, efx->net_dev, 1915 "failed to create filter tables\n"); 1916 goto fail4; 1917 } 1918 1919 rc = efx_probe_channels(efx); 1920 if (rc) 1921 goto fail5; 1922 1923 return 0; 1924 1925 fail5: 1926 efx_remove_filters(efx); 1927 fail4: 1928 #ifdef CONFIG_SFC_SRIOV 1929 efx->type->vswitching_remove(efx); 1930 #endif 1931 fail3: 1932 efx_remove_port(efx); 1933 fail2: 1934 efx_remove_nic(efx); 1935 fail1: 1936 return rc; 1937 } 1938 1939 /* If the interface is supposed to be running but is not, start 1940 * the hardware and software data path, regular activity for the port 1941 * (MAC statistics, link polling, etc.) and schedule the port to be 1942 * reconfigured. Interrupts must already be enabled. This function 1943 * is safe to call multiple times, so long as the NIC is not disabled. 1944 * Requires the RTNL lock. 1945 */ 1946 static void efx_start_all(struct efx_nic *efx) 1947 { 1948 EFX_ASSERT_RESET_SERIALISED(efx); 1949 BUG_ON(efx->state == STATE_DISABLED); 1950 1951 /* Check that it is appropriate to restart the interface. All 1952 * of these flags are safe to read under just the rtnl lock */ 1953 if (efx->port_enabled || !netif_running(efx->net_dev) || 1954 efx->reset_pending) 1955 return; 1956 1957 efx_start_port(efx); 1958 efx_start_datapath(efx); 1959 1960 /* Start the hardware monitor if there is one */ 1961 if (efx->type->monitor != NULL) 1962 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1963 efx_monitor_interval); 1964 1965 /* Link state detection is normally event-driven; we have 1966 * to poll now because we could have missed a change 1967 */ 1968 mutex_lock(&efx->mac_lock); 1969 if (efx->phy_op->poll(efx)) 1970 efx_link_status_changed(efx); 1971 mutex_unlock(&efx->mac_lock); 1972 1973 efx->type->start_stats(efx); 1974 efx->type->pull_stats(efx); 1975 spin_lock_bh(&efx->stats_lock); 1976 efx->type->update_stats(efx, NULL, NULL); 1977 spin_unlock_bh(&efx->stats_lock); 1978 } 1979 1980 /* Quiesce the hardware and software data path, and regular activity 1981 * for the port without bringing the link down. Safe to call multiple 1982 * times with the NIC in almost any state, but interrupts should be 1983 * enabled. Requires the RTNL lock. 1984 */ 1985 static void efx_stop_all(struct efx_nic *efx) 1986 { 1987 EFX_ASSERT_RESET_SERIALISED(efx); 1988 1989 /* port_enabled can be read safely under the rtnl lock */ 1990 if (!efx->port_enabled) 1991 return; 1992 1993 /* update stats before we go down so we can accurately count 1994 * rx_nodesc_drops 1995 */ 1996 efx->type->pull_stats(efx); 1997 spin_lock_bh(&efx->stats_lock); 1998 efx->type->update_stats(efx, NULL, NULL); 1999 spin_unlock_bh(&efx->stats_lock); 2000 efx->type->stop_stats(efx); 2001 efx_stop_port(efx); 2002 2003 /* Stop the kernel transmit interface. This is only valid if 2004 * the device is stopped or detached; otherwise the watchdog 2005 * may fire immediately. 2006 */ 2007 WARN_ON(netif_running(efx->net_dev) && 2008 netif_device_present(efx->net_dev)); 2009 netif_tx_disable(efx->net_dev); 2010 2011 efx_stop_datapath(efx); 2012 } 2013 2014 static void efx_remove_all(struct efx_nic *efx) 2015 { 2016 efx_remove_channels(efx); 2017 efx_remove_filters(efx); 2018 #ifdef CONFIG_SFC_SRIOV 2019 efx->type->vswitching_remove(efx); 2020 #endif 2021 efx_remove_port(efx); 2022 efx_remove_nic(efx); 2023 } 2024 2025 /************************************************************************** 2026 * 2027 * Interrupt moderation 2028 * 2029 **************************************************************************/ 2030 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 2031 { 2032 if (usecs == 0) 2033 return 0; 2034 if (usecs * 1000 < efx->timer_quantum_ns) 2035 return 1; /* never round down to 0 */ 2036 return usecs * 1000 / efx->timer_quantum_ns; 2037 } 2038 2039 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 2040 { 2041 /* We must round up when converting ticks to microseconds 2042 * because we round down when converting the other way. 2043 */ 2044 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 2045 } 2046 2047 /* Set interrupt moderation parameters */ 2048 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 2049 unsigned int rx_usecs, bool rx_adaptive, 2050 bool rx_may_override_tx) 2051 { 2052 struct efx_channel *channel; 2053 unsigned int timer_max_us; 2054 2055 EFX_ASSERT_RESET_SERIALISED(efx); 2056 2057 timer_max_us = efx->timer_max_ns / 1000; 2058 2059 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 2060 return -EINVAL; 2061 2062 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 2063 !rx_may_override_tx) { 2064 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 2065 "RX and TX IRQ moderation must be equal\n"); 2066 return -EINVAL; 2067 } 2068 2069 efx->irq_rx_adaptive = rx_adaptive; 2070 efx->irq_rx_moderation_us = rx_usecs; 2071 efx_for_each_channel(channel, efx) { 2072 if (efx_channel_has_rx_queue(channel)) 2073 channel->irq_moderation_us = rx_usecs; 2074 else if (efx_channel_has_tx_queues(channel)) 2075 channel->irq_moderation_us = tx_usecs; 2076 } 2077 2078 return 0; 2079 } 2080 2081 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2082 unsigned int *rx_usecs, bool *rx_adaptive) 2083 { 2084 *rx_adaptive = efx->irq_rx_adaptive; 2085 *rx_usecs = efx->irq_rx_moderation_us; 2086 2087 /* If channels are shared between RX and TX, so is IRQ 2088 * moderation. Otherwise, IRQ moderation is the same for all 2089 * TX channels and is not adaptive. 2090 */ 2091 if (efx->tx_channel_offset == 0) { 2092 *tx_usecs = *rx_usecs; 2093 } else { 2094 struct efx_channel *tx_channel; 2095 2096 tx_channel = efx->channel[efx->tx_channel_offset]; 2097 *tx_usecs = tx_channel->irq_moderation_us; 2098 } 2099 } 2100 2101 /************************************************************************** 2102 * 2103 * Hardware monitor 2104 * 2105 **************************************************************************/ 2106 2107 /* Run periodically off the general workqueue */ 2108 static void efx_monitor(struct work_struct *data) 2109 { 2110 struct efx_nic *efx = container_of(data, struct efx_nic, 2111 monitor_work.work); 2112 2113 netif_vdbg(efx, timer, efx->net_dev, 2114 "hardware monitor executing on CPU %d\n", 2115 raw_smp_processor_id()); 2116 BUG_ON(efx->type->monitor == NULL); 2117 2118 /* If the mac_lock is already held then it is likely a port 2119 * reconfiguration is already in place, which will likely do 2120 * most of the work of monitor() anyway. */ 2121 if (mutex_trylock(&efx->mac_lock)) { 2122 if (efx->port_enabled) 2123 efx->type->monitor(efx); 2124 mutex_unlock(&efx->mac_lock); 2125 } 2126 2127 queue_delayed_work(efx->workqueue, &efx->monitor_work, 2128 efx_monitor_interval); 2129 } 2130 2131 /************************************************************************** 2132 * 2133 * ioctls 2134 * 2135 *************************************************************************/ 2136 2137 /* Net device ioctl 2138 * Context: process, rtnl_lock() held. 2139 */ 2140 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2141 { 2142 struct efx_nic *efx = netdev_priv(net_dev); 2143 struct mii_ioctl_data *data = if_mii(ifr); 2144 2145 if (cmd == SIOCSHWTSTAMP) 2146 return efx_ptp_set_ts_config(efx, ifr); 2147 if (cmd == SIOCGHWTSTAMP) 2148 return efx_ptp_get_ts_config(efx, ifr); 2149 2150 /* Convert phy_id from older PRTAD/DEVAD format */ 2151 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2152 (data->phy_id & 0xfc00) == 0x0400) 2153 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2154 2155 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2156 } 2157 2158 /************************************************************************** 2159 * 2160 * NAPI interface 2161 * 2162 **************************************************************************/ 2163 2164 static void efx_init_napi_channel(struct efx_channel *channel) 2165 { 2166 struct efx_nic *efx = channel->efx; 2167 2168 channel->napi_dev = efx->net_dev; 2169 netif_napi_add(channel->napi_dev, &channel->napi_str, 2170 efx_poll, napi_weight); 2171 } 2172 2173 static void efx_init_napi(struct efx_nic *efx) 2174 { 2175 struct efx_channel *channel; 2176 2177 efx_for_each_channel(channel, efx) 2178 efx_init_napi_channel(channel); 2179 } 2180 2181 static void efx_fini_napi_channel(struct efx_channel *channel) 2182 { 2183 if (channel->napi_dev) 2184 netif_napi_del(&channel->napi_str); 2185 2186 channel->napi_dev = NULL; 2187 } 2188 2189 static void efx_fini_napi(struct efx_nic *efx) 2190 { 2191 struct efx_channel *channel; 2192 2193 efx_for_each_channel(channel, efx) 2194 efx_fini_napi_channel(channel); 2195 } 2196 2197 /************************************************************************** 2198 * 2199 * Kernel netpoll interface 2200 * 2201 *************************************************************************/ 2202 2203 #ifdef CONFIG_NET_POLL_CONTROLLER 2204 2205 /* Although in the common case interrupts will be disabled, this is not 2206 * guaranteed. However, all our work happens inside the NAPI callback, 2207 * so no locking is required. 2208 */ 2209 static void efx_netpoll(struct net_device *net_dev) 2210 { 2211 struct efx_nic *efx = netdev_priv(net_dev); 2212 struct efx_channel *channel; 2213 2214 efx_for_each_channel(channel, efx) 2215 efx_schedule_channel(channel); 2216 } 2217 2218 #endif 2219 2220 /************************************************************************** 2221 * 2222 * Kernel net device interface 2223 * 2224 *************************************************************************/ 2225 2226 /* Context: process, rtnl_lock() held. */ 2227 int efx_net_open(struct net_device *net_dev) 2228 { 2229 struct efx_nic *efx = netdev_priv(net_dev); 2230 int rc; 2231 2232 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2233 raw_smp_processor_id()); 2234 2235 rc = efx_check_disabled(efx); 2236 if (rc) 2237 return rc; 2238 if (efx->phy_mode & PHY_MODE_SPECIAL) 2239 return -EBUSY; 2240 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2241 return -EIO; 2242 2243 /* Notify the kernel of the link state polled during driver load, 2244 * before the monitor starts running */ 2245 efx_link_status_changed(efx); 2246 2247 efx_start_all(efx); 2248 if (efx->state == STATE_DISABLED || efx->reset_pending) 2249 netif_device_detach(efx->net_dev); 2250 efx_selftest_async_start(efx); 2251 return 0; 2252 } 2253 2254 /* Context: process, rtnl_lock() held. 2255 * Note that the kernel will ignore our return code; this method 2256 * should really be a void. 2257 */ 2258 int efx_net_stop(struct net_device *net_dev) 2259 { 2260 struct efx_nic *efx = netdev_priv(net_dev); 2261 2262 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2263 raw_smp_processor_id()); 2264 2265 /* Stop the device and flush all the channels */ 2266 efx_stop_all(efx); 2267 2268 return 0; 2269 } 2270 2271 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2272 static void efx_net_stats(struct net_device *net_dev, 2273 struct rtnl_link_stats64 *stats) 2274 { 2275 struct efx_nic *efx = netdev_priv(net_dev); 2276 2277 spin_lock_bh(&efx->stats_lock); 2278 efx->type->update_stats(efx, NULL, stats); 2279 spin_unlock_bh(&efx->stats_lock); 2280 } 2281 2282 /* Context: netif_tx_lock held, BHs disabled. */ 2283 static void efx_watchdog(struct net_device *net_dev) 2284 { 2285 struct efx_nic *efx = netdev_priv(net_dev); 2286 2287 netif_err(efx, tx_err, efx->net_dev, 2288 "TX stuck with port_enabled=%d: resetting channels\n", 2289 efx->port_enabled); 2290 2291 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2292 } 2293 2294 2295 /* Context: process, rtnl_lock() held. */ 2296 static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2297 { 2298 struct efx_nic *efx = netdev_priv(net_dev); 2299 int rc; 2300 2301 rc = efx_check_disabled(efx); 2302 if (rc) 2303 return rc; 2304 2305 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2306 2307 efx_device_detach_sync(efx); 2308 efx_stop_all(efx); 2309 2310 mutex_lock(&efx->mac_lock); 2311 net_dev->mtu = new_mtu; 2312 efx_mac_reconfigure(efx); 2313 mutex_unlock(&efx->mac_lock); 2314 2315 efx_start_all(efx); 2316 efx_device_attach_if_not_resetting(efx); 2317 return 0; 2318 } 2319 2320 static int efx_set_mac_address(struct net_device *net_dev, void *data) 2321 { 2322 struct efx_nic *efx = netdev_priv(net_dev); 2323 struct sockaddr *addr = data; 2324 u8 *new_addr = addr->sa_data; 2325 u8 old_addr[6]; 2326 int rc; 2327 2328 if (!is_valid_ether_addr(new_addr)) { 2329 netif_err(efx, drv, efx->net_dev, 2330 "invalid ethernet MAC address requested: %pM\n", 2331 new_addr); 2332 return -EADDRNOTAVAIL; 2333 } 2334 2335 /* save old address */ 2336 ether_addr_copy(old_addr, net_dev->dev_addr); 2337 ether_addr_copy(net_dev->dev_addr, new_addr); 2338 if (efx->type->set_mac_address) { 2339 rc = efx->type->set_mac_address(efx); 2340 if (rc) { 2341 ether_addr_copy(net_dev->dev_addr, old_addr); 2342 return rc; 2343 } 2344 } 2345 2346 /* Reconfigure the MAC */ 2347 mutex_lock(&efx->mac_lock); 2348 efx_mac_reconfigure(efx); 2349 mutex_unlock(&efx->mac_lock); 2350 2351 return 0; 2352 } 2353 2354 /* Context: netif_addr_lock held, BHs disabled. */ 2355 static void efx_set_rx_mode(struct net_device *net_dev) 2356 { 2357 struct efx_nic *efx = netdev_priv(net_dev); 2358 2359 if (efx->port_enabled) 2360 queue_work(efx->workqueue, &efx->mac_work); 2361 /* Otherwise efx_start_port() will do this */ 2362 } 2363 2364 static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2365 { 2366 struct efx_nic *efx = netdev_priv(net_dev); 2367 int rc; 2368 2369 /* If disabling RX n-tuple filtering, clear existing filters */ 2370 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2371 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2372 if (rc) 2373 return rc; 2374 } 2375 2376 /* If Rx VLAN filter is changed, update filters via mac_reconfigure. 2377 * If rx-fcs is changed, mac_reconfigure updates that too. 2378 */ 2379 if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER | 2380 NETIF_F_RXFCS)) { 2381 /* efx_set_rx_mode() will schedule MAC work to update filters 2382 * when a new features are finally set in net_dev. 2383 */ 2384 efx_set_rx_mode(net_dev); 2385 } 2386 2387 return 0; 2388 } 2389 2390 static int efx_get_phys_port_id(struct net_device *net_dev, 2391 struct netdev_phys_item_id *ppid) 2392 { 2393 struct efx_nic *efx = netdev_priv(net_dev); 2394 2395 if (efx->type->get_phys_port_id) 2396 return efx->type->get_phys_port_id(efx, ppid); 2397 else 2398 return -EOPNOTSUPP; 2399 } 2400 2401 static int efx_get_phys_port_name(struct net_device *net_dev, 2402 char *name, size_t len) 2403 { 2404 struct efx_nic *efx = netdev_priv(net_dev); 2405 2406 if (snprintf(name, len, "p%u", efx->port_num) >= len) 2407 return -EINVAL; 2408 return 0; 2409 } 2410 2411 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2412 { 2413 struct efx_nic *efx = netdev_priv(net_dev); 2414 2415 if (efx->type->vlan_rx_add_vid) 2416 return efx->type->vlan_rx_add_vid(efx, proto, vid); 2417 else 2418 return -EOPNOTSUPP; 2419 } 2420 2421 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2422 { 2423 struct efx_nic *efx = netdev_priv(net_dev); 2424 2425 if (efx->type->vlan_rx_kill_vid) 2426 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 2427 else 2428 return -EOPNOTSUPP; 2429 } 2430 2431 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in) 2432 { 2433 switch (in) { 2434 case UDP_TUNNEL_TYPE_VXLAN: 2435 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 2436 case UDP_TUNNEL_TYPE_GENEVE: 2437 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 2438 default: 2439 return -1; 2440 } 2441 } 2442 2443 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti) 2444 { 2445 struct efx_nic *efx = netdev_priv(dev); 2446 struct efx_udp_tunnel tnl; 2447 int efx_tunnel_type; 2448 2449 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2450 if (efx_tunnel_type < 0) 2451 return; 2452 2453 tnl.type = (u16)efx_tunnel_type; 2454 tnl.port = ti->port; 2455 2456 if (efx->type->udp_tnl_add_port) 2457 (void)efx->type->udp_tnl_add_port(efx, tnl); 2458 } 2459 2460 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti) 2461 { 2462 struct efx_nic *efx = netdev_priv(dev); 2463 struct efx_udp_tunnel tnl; 2464 int efx_tunnel_type; 2465 2466 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2467 if (efx_tunnel_type < 0) 2468 return; 2469 2470 tnl.type = (u16)efx_tunnel_type; 2471 tnl.port = ti->port; 2472 2473 if (efx->type->udp_tnl_del_port) 2474 (void)efx->type->udp_tnl_del_port(efx, tnl); 2475 } 2476 2477 static const struct net_device_ops efx_netdev_ops = { 2478 .ndo_open = efx_net_open, 2479 .ndo_stop = efx_net_stop, 2480 .ndo_get_stats64 = efx_net_stats, 2481 .ndo_tx_timeout = efx_watchdog, 2482 .ndo_start_xmit = efx_hard_start_xmit, 2483 .ndo_validate_addr = eth_validate_addr, 2484 .ndo_do_ioctl = efx_ioctl, 2485 .ndo_change_mtu = efx_change_mtu, 2486 .ndo_set_mac_address = efx_set_mac_address, 2487 .ndo_set_rx_mode = efx_set_rx_mode, 2488 .ndo_set_features = efx_set_features, 2489 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 2490 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2491 #ifdef CONFIG_SFC_SRIOV 2492 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2493 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2494 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2495 .ndo_get_vf_config = efx_sriov_get_vf_config, 2496 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2497 #endif 2498 .ndo_get_phys_port_id = efx_get_phys_port_id, 2499 .ndo_get_phys_port_name = efx_get_phys_port_name, 2500 #ifdef CONFIG_NET_POLL_CONTROLLER 2501 .ndo_poll_controller = efx_netpoll, 2502 #endif 2503 .ndo_setup_tc = efx_setup_tc, 2504 #ifdef CONFIG_RFS_ACCEL 2505 .ndo_rx_flow_steer = efx_filter_rfs, 2506 #endif 2507 .ndo_udp_tunnel_add = efx_udp_tunnel_add, 2508 .ndo_udp_tunnel_del = efx_udp_tunnel_del, 2509 }; 2510 2511 static void efx_update_name(struct efx_nic *efx) 2512 { 2513 strcpy(efx->name, efx->net_dev->name); 2514 efx_mtd_rename(efx); 2515 efx_set_channel_names(efx); 2516 } 2517 2518 static int efx_netdev_event(struct notifier_block *this, 2519 unsigned long event, void *ptr) 2520 { 2521 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2522 2523 if ((net_dev->netdev_ops == &efx_netdev_ops) && 2524 event == NETDEV_CHANGENAME) 2525 efx_update_name(netdev_priv(net_dev)); 2526 2527 return NOTIFY_DONE; 2528 } 2529 2530 static struct notifier_block efx_netdev_notifier = { 2531 .notifier_call = efx_netdev_event, 2532 }; 2533 2534 static ssize_t 2535 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2536 { 2537 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2538 return sprintf(buf, "%d\n", efx->phy_type); 2539 } 2540 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2541 2542 #ifdef CONFIG_SFC_MCDI_LOGGING 2543 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2544 char *buf) 2545 { 2546 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2547 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2548 2549 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2550 } 2551 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2552 const char *buf, size_t count) 2553 { 2554 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2555 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2556 bool enable = count > 0 && *buf != '0'; 2557 2558 mcdi->logging_enabled = enable; 2559 return count; 2560 } 2561 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2562 #endif 2563 2564 static int efx_register_netdev(struct efx_nic *efx) 2565 { 2566 struct net_device *net_dev = efx->net_dev; 2567 struct efx_channel *channel; 2568 int rc; 2569 2570 net_dev->watchdog_timeo = 5 * HZ; 2571 net_dev->irq = efx->pci_dev->irq; 2572 net_dev->netdev_ops = &efx_netdev_ops; 2573 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 2574 net_dev->priv_flags |= IFF_UNICAST_FLT; 2575 net_dev->ethtool_ops = &efx_ethtool_ops; 2576 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2577 net_dev->min_mtu = EFX_MIN_MTU; 2578 net_dev->max_mtu = EFX_MAX_MTU; 2579 2580 rtnl_lock(); 2581 2582 /* Enable resets to be scheduled and check whether any were 2583 * already requested. If so, the NIC is probably hosed so we 2584 * abort. 2585 */ 2586 efx->state = STATE_READY; 2587 smp_mb(); /* ensure we change state before checking reset_pending */ 2588 if (efx->reset_pending) { 2589 netif_err(efx, probe, efx->net_dev, 2590 "aborting probe due to scheduled reset\n"); 2591 rc = -EIO; 2592 goto fail_locked; 2593 } 2594 2595 rc = dev_alloc_name(net_dev, net_dev->name); 2596 if (rc < 0) 2597 goto fail_locked; 2598 efx_update_name(efx); 2599 2600 /* Always start with carrier off; PHY events will detect the link */ 2601 netif_carrier_off(net_dev); 2602 2603 rc = register_netdevice(net_dev); 2604 if (rc) 2605 goto fail_locked; 2606 2607 efx_for_each_channel(channel, efx) { 2608 struct efx_tx_queue *tx_queue; 2609 efx_for_each_channel_tx_queue(tx_queue, channel) 2610 efx_init_tx_queue_core_txq(tx_queue); 2611 } 2612 2613 efx_associate(efx); 2614 2615 rtnl_unlock(); 2616 2617 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2618 if (rc) { 2619 netif_err(efx, drv, efx->net_dev, 2620 "failed to init net dev attributes\n"); 2621 goto fail_registered; 2622 } 2623 #ifdef CONFIG_SFC_MCDI_LOGGING 2624 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2625 if (rc) { 2626 netif_err(efx, drv, efx->net_dev, 2627 "failed to init net dev attributes\n"); 2628 goto fail_attr_mcdi_logging; 2629 } 2630 #endif 2631 2632 return 0; 2633 2634 #ifdef CONFIG_SFC_MCDI_LOGGING 2635 fail_attr_mcdi_logging: 2636 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2637 #endif 2638 fail_registered: 2639 rtnl_lock(); 2640 efx_dissociate(efx); 2641 unregister_netdevice(net_dev); 2642 fail_locked: 2643 efx->state = STATE_UNINIT; 2644 rtnl_unlock(); 2645 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2646 return rc; 2647 } 2648 2649 static void efx_unregister_netdev(struct efx_nic *efx) 2650 { 2651 if (!efx->net_dev) 2652 return; 2653 2654 BUG_ON(netdev_priv(efx->net_dev) != efx); 2655 2656 if (efx_dev_registered(efx)) { 2657 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2658 #ifdef CONFIG_SFC_MCDI_LOGGING 2659 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2660 #endif 2661 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2662 unregister_netdev(efx->net_dev); 2663 } 2664 } 2665 2666 /************************************************************************** 2667 * 2668 * Device reset and suspend 2669 * 2670 **************************************************************************/ 2671 2672 /* Tears down the entire software state and most of the hardware state 2673 * before reset. */ 2674 void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2675 { 2676 EFX_ASSERT_RESET_SERIALISED(efx); 2677 2678 if (method == RESET_TYPE_MCDI_TIMEOUT) 2679 efx->type->prepare_flr(efx); 2680 2681 efx_stop_all(efx); 2682 efx_disable_interrupts(efx); 2683 2684 mutex_lock(&efx->mac_lock); 2685 down_write(&efx->filter_sem); 2686 mutex_lock(&efx->rss_lock); 2687 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2688 method != RESET_TYPE_DATAPATH) 2689 efx->phy_op->fini(efx); 2690 efx->type->fini(efx); 2691 } 2692 2693 /* This function will always ensure that the locks acquired in 2694 * efx_reset_down() are released. A failure return code indicates 2695 * that we were unable to reinitialise the hardware, and the 2696 * driver should be disabled. If ok is false, then the rx and tx 2697 * engines are not restarted, pending a RESET_DISABLE. */ 2698 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2699 { 2700 int rc; 2701 2702 EFX_ASSERT_RESET_SERIALISED(efx); 2703 2704 if (method == RESET_TYPE_MCDI_TIMEOUT) 2705 efx->type->finish_flr(efx); 2706 2707 /* Ensure that SRAM is initialised even if we're disabling the device */ 2708 rc = efx->type->init(efx); 2709 if (rc) { 2710 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2711 goto fail; 2712 } 2713 2714 if (!ok) 2715 goto fail; 2716 2717 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2718 method != RESET_TYPE_DATAPATH) { 2719 rc = efx->phy_op->init(efx); 2720 if (rc) 2721 goto fail; 2722 rc = efx->phy_op->reconfigure(efx); 2723 if (rc && rc != -EPERM) 2724 netif_err(efx, drv, efx->net_dev, 2725 "could not restore PHY settings\n"); 2726 } 2727 2728 rc = efx_enable_interrupts(efx); 2729 if (rc) 2730 goto fail; 2731 2732 #ifdef CONFIG_SFC_SRIOV 2733 rc = efx->type->vswitching_restore(efx); 2734 if (rc) /* not fatal; the PF will still work fine */ 2735 netif_warn(efx, probe, efx->net_dev, 2736 "failed to restore vswitching rc=%d;" 2737 " VFs may not function\n", rc); 2738 #endif 2739 2740 if (efx->type->rx_restore_rss_contexts) 2741 efx->type->rx_restore_rss_contexts(efx); 2742 mutex_unlock(&efx->rss_lock); 2743 efx->type->filter_table_restore(efx); 2744 up_write(&efx->filter_sem); 2745 if (efx->type->sriov_reset) 2746 efx->type->sriov_reset(efx); 2747 2748 mutex_unlock(&efx->mac_lock); 2749 2750 efx_start_all(efx); 2751 2752 if (efx->type->udp_tnl_push_ports) 2753 efx->type->udp_tnl_push_ports(efx); 2754 2755 return 0; 2756 2757 fail: 2758 efx->port_initialized = false; 2759 2760 mutex_unlock(&efx->rss_lock); 2761 up_write(&efx->filter_sem); 2762 mutex_unlock(&efx->mac_lock); 2763 2764 return rc; 2765 } 2766 2767 /* Reset the NIC using the specified method. Note that the reset may 2768 * fail, in which case the card will be left in an unusable state. 2769 * 2770 * Caller must hold the rtnl_lock. 2771 */ 2772 int efx_reset(struct efx_nic *efx, enum reset_type method) 2773 { 2774 int rc, rc2; 2775 bool disabled; 2776 2777 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2778 RESET_TYPE(method)); 2779 2780 efx_device_detach_sync(efx); 2781 efx_reset_down(efx, method); 2782 2783 rc = efx->type->reset(efx, method); 2784 if (rc) { 2785 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2786 goto out; 2787 } 2788 2789 /* Clear flags for the scopes we covered. We assume the NIC and 2790 * driver are now quiescent so that there is no race here. 2791 */ 2792 if (method < RESET_TYPE_MAX_METHOD) 2793 efx->reset_pending &= -(1 << (method + 1)); 2794 else /* it doesn't fit into the well-ordered scope hierarchy */ 2795 __clear_bit(method, &efx->reset_pending); 2796 2797 /* Reinitialise bus-mastering, which may have been turned off before 2798 * the reset was scheduled. This is still appropriate, even in the 2799 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2800 * can respond to requests. */ 2801 pci_set_master(efx->pci_dev); 2802 2803 out: 2804 /* Leave device stopped if necessary */ 2805 disabled = rc || 2806 method == RESET_TYPE_DISABLE || 2807 method == RESET_TYPE_RECOVER_OR_DISABLE; 2808 rc2 = efx_reset_up(efx, method, !disabled); 2809 if (rc2) { 2810 disabled = true; 2811 if (!rc) 2812 rc = rc2; 2813 } 2814 2815 if (disabled) { 2816 dev_close(efx->net_dev); 2817 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2818 efx->state = STATE_DISABLED; 2819 } else { 2820 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2821 efx_device_attach_if_not_resetting(efx); 2822 } 2823 return rc; 2824 } 2825 2826 /* Try recovery mechanisms. 2827 * For now only EEH is supported. 2828 * Returns 0 if the recovery mechanisms are unsuccessful. 2829 * Returns a non-zero value otherwise. 2830 */ 2831 int efx_try_recovery(struct efx_nic *efx) 2832 { 2833 #ifdef CONFIG_EEH 2834 /* A PCI error can occur and not be seen by EEH because nothing 2835 * happens on the PCI bus. In this case the driver may fail and 2836 * schedule a 'recover or reset', leading to this recovery handler. 2837 * Manually call the eeh failure check function. 2838 */ 2839 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2840 if (eeh_dev_check_failure(eehdev)) { 2841 /* The EEH mechanisms will handle the error and reset the 2842 * device if necessary. 2843 */ 2844 return 1; 2845 } 2846 #endif 2847 return 0; 2848 } 2849 2850 static void efx_wait_for_bist_end(struct efx_nic *efx) 2851 { 2852 int i; 2853 2854 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 2855 if (efx_mcdi_poll_reboot(efx)) 2856 goto out; 2857 msleep(BIST_WAIT_DELAY_MS); 2858 } 2859 2860 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 2861 out: 2862 /* Either way unset the BIST flag. If we found no reboot we probably 2863 * won't recover, but we should try. 2864 */ 2865 efx->mc_bist_for_other_fn = false; 2866 } 2867 2868 /* The worker thread exists so that code that cannot sleep can 2869 * schedule a reset for later. 2870 */ 2871 static void efx_reset_work(struct work_struct *data) 2872 { 2873 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2874 unsigned long pending; 2875 enum reset_type method; 2876 2877 pending = READ_ONCE(efx->reset_pending); 2878 method = fls(pending) - 1; 2879 2880 if (method == RESET_TYPE_MC_BIST) 2881 efx_wait_for_bist_end(efx); 2882 2883 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2884 method == RESET_TYPE_RECOVER_OR_ALL) && 2885 efx_try_recovery(efx)) 2886 return; 2887 2888 if (!pending) 2889 return; 2890 2891 rtnl_lock(); 2892 2893 /* We checked the state in efx_schedule_reset() but it may 2894 * have changed by now. Now that we have the RTNL lock, 2895 * it cannot change again. 2896 */ 2897 if (efx->state == STATE_READY) 2898 (void)efx_reset(efx, method); 2899 2900 rtnl_unlock(); 2901 } 2902 2903 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2904 { 2905 enum reset_type method; 2906 2907 if (efx->state == STATE_RECOVERY) { 2908 netif_dbg(efx, drv, efx->net_dev, 2909 "recovering: skip scheduling %s reset\n", 2910 RESET_TYPE(type)); 2911 return; 2912 } 2913 2914 switch (type) { 2915 case RESET_TYPE_INVISIBLE: 2916 case RESET_TYPE_ALL: 2917 case RESET_TYPE_RECOVER_OR_ALL: 2918 case RESET_TYPE_WORLD: 2919 case RESET_TYPE_DISABLE: 2920 case RESET_TYPE_RECOVER_OR_DISABLE: 2921 case RESET_TYPE_DATAPATH: 2922 case RESET_TYPE_MC_BIST: 2923 case RESET_TYPE_MCDI_TIMEOUT: 2924 method = type; 2925 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2926 RESET_TYPE(method)); 2927 break; 2928 default: 2929 method = efx->type->map_reset_reason(type); 2930 netif_dbg(efx, drv, efx->net_dev, 2931 "scheduling %s reset for %s\n", 2932 RESET_TYPE(method), RESET_TYPE(type)); 2933 break; 2934 } 2935 2936 set_bit(method, &efx->reset_pending); 2937 smp_mb(); /* ensure we change reset_pending before checking state */ 2938 2939 /* If we're not READY then just leave the flags set as the cue 2940 * to abort probing or reschedule the reset later. 2941 */ 2942 if (READ_ONCE(efx->state) != STATE_READY) 2943 return; 2944 2945 /* efx_process_channel() will no longer read events once a 2946 * reset is scheduled. So switch back to poll'd MCDI completions. */ 2947 efx_mcdi_mode_poll(efx); 2948 2949 queue_work(reset_workqueue, &efx->reset_work); 2950 } 2951 2952 /************************************************************************** 2953 * 2954 * List of NICs we support 2955 * 2956 **************************************************************************/ 2957 2958 /* PCI device ID table */ 2959 static const struct pci_device_id efx_pci_table[] = { 2960 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2961 .driver_data = (unsigned long) &siena_a0_nic_type}, 2962 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2963 .driver_data = (unsigned long) &siena_a0_nic_type}, 2964 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 2965 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2966 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 2967 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2968 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 2969 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2970 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2971 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2972 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2973 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2974 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2975 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2976 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 2977 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2978 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 2979 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2980 {0} /* end of list */ 2981 }; 2982 2983 /************************************************************************** 2984 * 2985 * Dummy PHY/MAC operations 2986 * 2987 * Can be used for some unimplemented operations 2988 * Needed so all function pointers are valid and do not have to be tested 2989 * before use 2990 * 2991 **************************************************************************/ 2992 int efx_port_dummy_op_int(struct efx_nic *efx) 2993 { 2994 return 0; 2995 } 2996 void efx_port_dummy_op_void(struct efx_nic *efx) {} 2997 2998 static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2999 { 3000 return false; 3001 } 3002 3003 static const struct efx_phy_operations efx_dummy_phy_operations = { 3004 .init = efx_port_dummy_op_int, 3005 .reconfigure = efx_port_dummy_op_int, 3006 .poll = efx_port_dummy_op_poll, 3007 .fini = efx_port_dummy_op_void, 3008 }; 3009 3010 /************************************************************************** 3011 * 3012 * Data housekeeping 3013 * 3014 **************************************************************************/ 3015 3016 /* This zeroes out and then fills in the invariants in a struct 3017 * efx_nic (including all sub-structures). 3018 */ 3019 static int efx_init_struct(struct efx_nic *efx, 3020 struct pci_dev *pci_dev, struct net_device *net_dev) 3021 { 3022 int rc = -ENOMEM, i; 3023 3024 /* Initialise common structures */ 3025 INIT_LIST_HEAD(&efx->node); 3026 INIT_LIST_HEAD(&efx->secondary_list); 3027 spin_lock_init(&efx->biu_lock); 3028 #ifdef CONFIG_SFC_MTD 3029 INIT_LIST_HEAD(&efx->mtd_list); 3030 #endif 3031 INIT_WORK(&efx->reset_work, efx_reset_work); 3032 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 3033 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 3034 efx->pci_dev = pci_dev; 3035 efx->msg_enable = debug; 3036 efx->state = STATE_UNINIT; 3037 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 3038 3039 efx->net_dev = net_dev; 3040 efx->rx_prefix_size = efx->type->rx_prefix_size; 3041 efx->rx_ip_align = 3042 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 3043 efx->rx_packet_hash_offset = 3044 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 3045 efx->rx_packet_ts_offset = 3046 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 3047 INIT_LIST_HEAD(&efx->rss_context.list); 3048 mutex_init(&efx->rss_lock); 3049 spin_lock_init(&efx->stats_lock); 3050 efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 3051 efx->num_mac_stats = MC_CMD_MAC_NSTATS; 3052 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 3053 mutex_init(&efx->mac_lock); 3054 #ifdef CONFIG_RFS_ACCEL 3055 mutex_init(&efx->rps_mutex); 3056 spin_lock_init(&efx->rps_hash_lock); 3057 /* Failure to allocate is not fatal, but may degrade ARFS performance */ 3058 efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, 3059 sizeof(*efx->rps_hash_table), GFP_KERNEL); 3060 #endif 3061 efx->phy_op = &efx_dummy_phy_operations; 3062 efx->mdio.dev = net_dev; 3063 INIT_WORK(&efx->mac_work, efx_mac_work); 3064 init_waitqueue_head(&efx->flush_wq); 3065 3066 for (i = 0; i < EFX_MAX_CHANNELS; i++) { 3067 efx->channel[i] = efx_alloc_channel(efx, i, NULL); 3068 if (!efx->channel[i]) 3069 goto fail; 3070 efx->msi_context[i].efx = efx; 3071 efx->msi_context[i].index = i; 3072 } 3073 3074 /* Higher numbered interrupt modes are less capable! */ 3075 if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 3076 efx->type->min_interrupt_mode)) { 3077 rc = -EIO; 3078 goto fail; 3079 } 3080 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 3081 interrupt_mode); 3082 efx->interrupt_mode = min(efx->type->min_interrupt_mode, 3083 interrupt_mode); 3084 3085 /* Would be good to use the net_dev name, but we're too early */ 3086 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 3087 pci_name(pci_dev)); 3088 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 3089 if (!efx->workqueue) 3090 goto fail; 3091 3092 return 0; 3093 3094 fail: 3095 efx_fini_struct(efx); 3096 return rc; 3097 } 3098 3099 static void efx_fini_struct(struct efx_nic *efx) 3100 { 3101 int i; 3102 3103 #ifdef CONFIG_RFS_ACCEL 3104 kfree(efx->rps_hash_table); 3105 #endif 3106 3107 for (i = 0; i < EFX_MAX_CHANNELS; i++) 3108 kfree(efx->channel[i]); 3109 3110 kfree(efx->vpd_sn); 3111 3112 if (efx->workqueue) { 3113 destroy_workqueue(efx->workqueue); 3114 efx->workqueue = NULL; 3115 } 3116 } 3117 3118 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 3119 { 3120 u64 n_rx_nodesc_trunc = 0; 3121 struct efx_channel *channel; 3122 3123 efx_for_each_channel(channel, efx) 3124 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 3125 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 3126 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 3127 } 3128 3129 bool efx_filter_spec_equal(const struct efx_filter_spec *left, 3130 const struct efx_filter_spec *right) 3131 { 3132 if ((left->match_flags ^ right->match_flags) | 3133 ((left->flags ^ right->flags) & 3134 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) 3135 return false; 3136 3137 return memcmp(&left->outer_vid, &right->outer_vid, 3138 sizeof(struct efx_filter_spec) - 3139 offsetof(struct efx_filter_spec, outer_vid)) == 0; 3140 } 3141 3142 u32 efx_filter_spec_hash(const struct efx_filter_spec *spec) 3143 { 3144 BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); 3145 return jhash2((const u32 *)&spec->outer_vid, 3146 (sizeof(struct efx_filter_spec) - 3147 offsetof(struct efx_filter_spec, outer_vid)) / 4, 3148 0); 3149 } 3150 3151 #ifdef CONFIG_RFS_ACCEL 3152 bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx, 3153 bool *force) 3154 { 3155 if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) { 3156 /* ARFS is currently updating this entry, leave it */ 3157 return false; 3158 } 3159 if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) { 3160 /* ARFS tried and failed to update this, so it's probably out 3161 * of date. Remove the filter and the ARFS rule entry. 3162 */ 3163 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 3164 *force = true; 3165 return true; 3166 } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */ 3167 /* ARFS has moved on, so old filter is not needed. Since we did 3168 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will 3169 * not be removed by efx_rps_hash_del() subsequently. 3170 */ 3171 *force = true; 3172 return true; 3173 } 3174 /* Remove it iff ARFS wants to. */ 3175 return true; 3176 } 3177 3178 static 3179 struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx, 3180 const struct efx_filter_spec *spec) 3181 { 3182 u32 hash = efx_filter_spec_hash(spec); 3183 3184 WARN_ON(!spin_is_locked(&efx->rps_hash_lock)); 3185 if (!efx->rps_hash_table) 3186 return NULL; 3187 return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE]; 3188 } 3189 3190 struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx, 3191 const struct efx_filter_spec *spec) 3192 { 3193 struct efx_arfs_rule *rule; 3194 struct hlist_head *head; 3195 struct hlist_node *node; 3196 3197 head = efx_rps_hash_bucket(efx, spec); 3198 if (!head) 3199 return NULL; 3200 hlist_for_each(node, head) { 3201 rule = container_of(node, struct efx_arfs_rule, node); 3202 if (efx_filter_spec_equal(spec, &rule->spec)) 3203 return rule; 3204 } 3205 return NULL; 3206 } 3207 3208 struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx, 3209 const struct efx_filter_spec *spec, 3210 bool *new) 3211 { 3212 struct efx_arfs_rule *rule; 3213 struct hlist_head *head; 3214 struct hlist_node *node; 3215 3216 head = efx_rps_hash_bucket(efx, spec); 3217 if (!head) 3218 return NULL; 3219 hlist_for_each(node, head) { 3220 rule = container_of(node, struct efx_arfs_rule, node); 3221 if (efx_filter_spec_equal(spec, &rule->spec)) { 3222 *new = false; 3223 return rule; 3224 } 3225 } 3226 rule = kmalloc(sizeof(*rule), GFP_ATOMIC); 3227 *new = true; 3228 if (rule) { 3229 memcpy(&rule->spec, spec, sizeof(rule->spec)); 3230 hlist_add_head(&rule->node, head); 3231 } 3232 return rule; 3233 } 3234 3235 void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec) 3236 { 3237 struct efx_arfs_rule *rule; 3238 struct hlist_head *head; 3239 struct hlist_node *node; 3240 3241 head = efx_rps_hash_bucket(efx, spec); 3242 if (WARN_ON(!head)) 3243 return; 3244 hlist_for_each(node, head) { 3245 rule = container_of(node, struct efx_arfs_rule, node); 3246 if (efx_filter_spec_equal(spec, &rule->spec)) { 3247 /* Someone already reused the entry. We know that if 3248 * this check doesn't fire (i.e. filter_id == REMOVING) 3249 * then the REMOVING mark was put there by our caller, 3250 * because caller is holding a lock on filter table and 3251 * only holders of that lock set REMOVING. 3252 */ 3253 if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING) 3254 return; 3255 hlist_del(node); 3256 kfree(rule); 3257 return; 3258 } 3259 } 3260 /* We didn't find it. */ 3261 WARN_ON(1); 3262 } 3263 #endif 3264 3265 /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because 3266 * (a) this is an infrequent control-plane operation and (b) n is small (max 64) 3267 */ 3268 struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx) 3269 { 3270 struct list_head *head = &efx->rss_context.list; 3271 struct efx_rss_context *ctx, *new; 3272 u32 id = 1; /* Don't use zero, that refers to the master RSS context */ 3273 3274 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3275 3276 /* Search for first gap in the numbering */ 3277 list_for_each_entry(ctx, head, list) { 3278 if (ctx->user_id != id) 3279 break; 3280 id++; 3281 /* Check for wrap. If this happens, we have nearly 2^32 3282 * allocated RSS contexts, which seems unlikely. 3283 */ 3284 if (WARN_ON_ONCE(!id)) 3285 return NULL; 3286 } 3287 3288 /* Create the new entry */ 3289 new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL); 3290 if (!new) 3291 return NULL; 3292 new->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 3293 new->rx_hash_udp_4tuple = false; 3294 3295 /* Insert the new entry into the gap */ 3296 new->user_id = id; 3297 list_add_tail(&new->list, &ctx->list); 3298 return new; 3299 } 3300 3301 struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id) 3302 { 3303 struct list_head *head = &efx->rss_context.list; 3304 struct efx_rss_context *ctx; 3305 3306 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3307 3308 list_for_each_entry(ctx, head, list) 3309 if (ctx->user_id == id) 3310 return ctx; 3311 return NULL; 3312 } 3313 3314 void efx_free_rss_context_entry(struct efx_rss_context *ctx) 3315 { 3316 list_del(&ctx->list); 3317 kfree(ctx); 3318 } 3319 3320 /************************************************************************** 3321 * 3322 * PCI interface 3323 * 3324 **************************************************************************/ 3325 3326 /* Main body of final NIC shutdown code 3327 * This is called only at module unload (or hotplug removal). 3328 */ 3329 static void efx_pci_remove_main(struct efx_nic *efx) 3330 { 3331 /* Flush reset_work. It can no longer be scheduled since we 3332 * are not READY. 3333 */ 3334 BUG_ON(efx->state == STATE_READY); 3335 cancel_work_sync(&efx->reset_work); 3336 3337 efx_disable_interrupts(efx); 3338 efx_clear_interrupt_affinity(efx); 3339 efx_nic_fini_interrupt(efx); 3340 efx_fini_port(efx); 3341 efx->type->fini(efx); 3342 efx_fini_napi(efx); 3343 efx_remove_all(efx); 3344 } 3345 3346 /* Final NIC shutdown 3347 * This is called only at module unload (or hotplug removal). A PF can call 3348 * this on its VFs to ensure they are unbound first. 3349 */ 3350 static void efx_pci_remove(struct pci_dev *pci_dev) 3351 { 3352 struct efx_nic *efx; 3353 3354 efx = pci_get_drvdata(pci_dev); 3355 if (!efx) 3356 return; 3357 3358 /* Mark the NIC as fini, then stop the interface */ 3359 rtnl_lock(); 3360 efx_dissociate(efx); 3361 dev_close(efx->net_dev); 3362 efx_disable_interrupts(efx); 3363 efx->state = STATE_UNINIT; 3364 rtnl_unlock(); 3365 3366 if (efx->type->sriov_fini) 3367 efx->type->sriov_fini(efx); 3368 3369 efx_unregister_netdev(efx); 3370 3371 efx_mtd_remove(efx); 3372 3373 efx_pci_remove_main(efx); 3374 3375 efx_fini_io(efx); 3376 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3377 3378 efx_fini_struct(efx); 3379 free_netdev(efx->net_dev); 3380 3381 pci_disable_pcie_error_reporting(pci_dev); 3382 }; 3383 3384 /* NIC VPD information 3385 * Called during probe to display the part number of the 3386 * installed NIC. VPD is potentially very large but this should 3387 * always appear within the first 512 bytes. 3388 */ 3389 #define SFC_VPD_LEN 512 3390 static void efx_probe_vpd_strings(struct efx_nic *efx) 3391 { 3392 struct pci_dev *dev = efx->pci_dev; 3393 char vpd_data[SFC_VPD_LEN]; 3394 ssize_t vpd_size; 3395 int ro_start, ro_size, i, j; 3396 3397 /* Get the vpd data from the device */ 3398 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3399 if (vpd_size <= 0) { 3400 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3401 return; 3402 } 3403 3404 /* Get the Read only section */ 3405 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3406 if (ro_start < 0) { 3407 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3408 return; 3409 } 3410 3411 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3412 j = ro_size; 3413 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3414 if (i + j > vpd_size) 3415 j = vpd_size - i; 3416 3417 /* Get the Part number */ 3418 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3419 if (i < 0) { 3420 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3421 return; 3422 } 3423 3424 j = pci_vpd_info_field_size(&vpd_data[i]); 3425 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3426 if (i + j > vpd_size) { 3427 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3428 return; 3429 } 3430 3431 netif_info(efx, drv, efx->net_dev, 3432 "Part Number : %.*s\n", j, &vpd_data[i]); 3433 3434 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3435 j = ro_size; 3436 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3437 if (i < 0) { 3438 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3439 return; 3440 } 3441 3442 j = pci_vpd_info_field_size(&vpd_data[i]); 3443 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3444 if (i + j > vpd_size) { 3445 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3446 return; 3447 } 3448 3449 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3450 if (!efx->vpd_sn) 3451 return; 3452 3453 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3454 } 3455 3456 3457 /* Main body of NIC initialisation 3458 * This is called at module load (or hotplug insertion, theoretically). 3459 */ 3460 static int efx_pci_probe_main(struct efx_nic *efx) 3461 { 3462 int rc; 3463 3464 /* Do start-of-day initialisation */ 3465 rc = efx_probe_all(efx); 3466 if (rc) 3467 goto fail1; 3468 3469 efx_init_napi(efx); 3470 3471 down_write(&efx->filter_sem); 3472 rc = efx->type->init(efx); 3473 up_write(&efx->filter_sem); 3474 if (rc) { 3475 netif_err(efx, probe, efx->net_dev, 3476 "failed to initialise NIC\n"); 3477 goto fail3; 3478 } 3479 3480 rc = efx_init_port(efx); 3481 if (rc) { 3482 netif_err(efx, probe, efx->net_dev, 3483 "failed to initialise port\n"); 3484 goto fail4; 3485 } 3486 3487 rc = efx_nic_init_interrupt(efx); 3488 if (rc) 3489 goto fail5; 3490 3491 efx_set_interrupt_affinity(efx); 3492 rc = efx_enable_interrupts(efx); 3493 if (rc) 3494 goto fail6; 3495 3496 return 0; 3497 3498 fail6: 3499 efx_clear_interrupt_affinity(efx); 3500 efx_nic_fini_interrupt(efx); 3501 fail5: 3502 efx_fini_port(efx); 3503 fail4: 3504 efx->type->fini(efx); 3505 fail3: 3506 efx_fini_napi(efx); 3507 efx_remove_all(efx); 3508 fail1: 3509 return rc; 3510 } 3511 3512 static int efx_pci_probe_post_io(struct efx_nic *efx) 3513 { 3514 struct net_device *net_dev = efx->net_dev; 3515 int rc = efx_pci_probe_main(efx); 3516 3517 if (rc) 3518 return rc; 3519 3520 if (efx->type->sriov_init) { 3521 rc = efx->type->sriov_init(efx); 3522 if (rc) 3523 netif_err(efx, probe, efx->net_dev, 3524 "SR-IOV can't be enabled rc %d\n", rc); 3525 } 3526 3527 /* Determine netdevice features */ 3528 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3529 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL); 3530 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 3531 net_dev->features |= NETIF_F_TSO6; 3532 /* Check whether device supports TSO */ 3533 if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 3534 net_dev->features &= ~NETIF_F_ALL_TSO; 3535 /* Mask for features that also apply to VLAN devices */ 3536 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3537 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 3538 NETIF_F_RXCSUM); 3539 3540 net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 3541 3542 /* Disable receiving frames with bad FCS, by default. */ 3543 net_dev->features &= ~NETIF_F_RXALL; 3544 3545 /* Disable VLAN filtering by default. It may be enforced if 3546 * the feature is fixed (i.e. VLAN filters are required to 3547 * receive VLAN tagged packets due to vPort restrictions). 3548 */ 3549 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3550 net_dev->features |= efx->fixed_features; 3551 3552 rc = efx_register_netdev(efx); 3553 if (!rc) 3554 return 0; 3555 3556 efx_pci_remove_main(efx); 3557 return rc; 3558 } 3559 3560 /* NIC initialisation 3561 * 3562 * This is called at module load (or hotplug insertion, 3563 * theoretically). It sets up PCI mappings, resets the NIC, 3564 * sets up and registers the network devices with the kernel and hooks 3565 * the interrupt service routine. It does not prepare the device for 3566 * transmission; this is left to the first time one of the network 3567 * interfaces is brought up (i.e. efx_net_open). 3568 */ 3569 static int efx_pci_probe(struct pci_dev *pci_dev, 3570 const struct pci_device_id *entry) 3571 { 3572 struct net_device *net_dev; 3573 struct efx_nic *efx; 3574 int rc; 3575 3576 /* Allocate and initialise a struct net_device and struct efx_nic */ 3577 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3578 EFX_MAX_RX_QUEUES); 3579 if (!net_dev) 3580 return -ENOMEM; 3581 efx = netdev_priv(net_dev); 3582 efx->type = (const struct efx_nic_type *) entry->driver_data; 3583 efx->fixed_features |= NETIF_F_HIGHDMA; 3584 3585 pci_set_drvdata(pci_dev, efx); 3586 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3587 rc = efx_init_struct(efx, pci_dev, net_dev); 3588 if (rc) 3589 goto fail1; 3590 3591 netif_info(efx, probe, efx->net_dev, 3592 "Solarflare NIC detected\n"); 3593 3594 if (!efx->type->is_vf) 3595 efx_probe_vpd_strings(efx); 3596 3597 /* Set up basic I/O (BAR mappings etc) */ 3598 rc = efx_init_io(efx); 3599 if (rc) 3600 goto fail2; 3601 3602 rc = efx_pci_probe_post_io(efx); 3603 if (rc) { 3604 /* On failure, retry once immediately. 3605 * If we aborted probe due to a scheduled reset, dismiss it. 3606 */ 3607 efx->reset_pending = 0; 3608 rc = efx_pci_probe_post_io(efx); 3609 if (rc) { 3610 /* On another failure, retry once more 3611 * after a 50-305ms delay. 3612 */ 3613 unsigned char r; 3614 3615 get_random_bytes(&r, 1); 3616 msleep((unsigned int)r + 50); 3617 efx->reset_pending = 0; 3618 rc = efx_pci_probe_post_io(efx); 3619 } 3620 } 3621 if (rc) 3622 goto fail3; 3623 3624 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3625 3626 /* Try to create MTDs, but allow this to fail */ 3627 rtnl_lock(); 3628 rc = efx_mtd_probe(efx); 3629 rtnl_unlock(); 3630 if (rc && rc != -EPERM) 3631 netif_warn(efx, probe, efx->net_dev, 3632 "failed to create MTDs (%d)\n", rc); 3633 3634 rc = pci_enable_pcie_error_reporting(pci_dev); 3635 if (rc && rc != -EINVAL) 3636 netif_notice(efx, probe, efx->net_dev, 3637 "PCIE error reporting unavailable (%d).\n", 3638 rc); 3639 3640 if (efx->type->udp_tnl_push_ports) 3641 efx->type->udp_tnl_push_ports(efx); 3642 3643 return 0; 3644 3645 fail3: 3646 efx_fini_io(efx); 3647 fail2: 3648 efx_fini_struct(efx); 3649 fail1: 3650 WARN_ON(rc > 0); 3651 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3652 free_netdev(net_dev); 3653 return rc; 3654 } 3655 3656 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3657 * enabled on success 3658 */ 3659 #ifdef CONFIG_SFC_SRIOV 3660 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3661 { 3662 int rc; 3663 struct efx_nic *efx = pci_get_drvdata(dev); 3664 3665 if (efx->type->sriov_configure) { 3666 rc = efx->type->sriov_configure(efx, num_vfs); 3667 if (rc) 3668 return rc; 3669 else 3670 return num_vfs; 3671 } else 3672 return -EOPNOTSUPP; 3673 } 3674 #endif 3675 3676 static int efx_pm_freeze(struct device *dev) 3677 { 3678 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3679 3680 rtnl_lock(); 3681 3682 if (efx->state != STATE_DISABLED) { 3683 efx->state = STATE_UNINIT; 3684 3685 efx_device_detach_sync(efx); 3686 3687 efx_stop_all(efx); 3688 efx_disable_interrupts(efx); 3689 } 3690 3691 rtnl_unlock(); 3692 3693 return 0; 3694 } 3695 3696 static int efx_pm_thaw(struct device *dev) 3697 { 3698 int rc; 3699 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3700 3701 rtnl_lock(); 3702 3703 if (efx->state != STATE_DISABLED) { 3704 rc = efx_enable_interrupts(efx); 3705 if (rc) 3706 goto fail; 3707 3708 mutex_lock(&efx->mac_lock); 3709 efx->phy_op->reconfigure(efx); 3710 mutex_unlock(&efx->mac_lock); 3711 3712 efx_start_all(efx); 3713 3714 efx_device_attach_if_not_resetting(efx); 3715 3716 efx->state = STATE_READY; 3717 3718 efx->type->resume_wol(efx); 3719 } 3720 3721 rtnl_unlock(); 3722 3723 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3724 queue_work(reset_workqueue, &efx->reset_work); 3725 3726 return 0; 3727 3728 fail: 3729 rtnl_unlock(); 3730 3731 return rc; 3732 } 3733 3734 static int efx_pm_poweroff(struct device *dev) 3735 { 3736 struct pci_dev *pci_dev = to_pci_dev(dev); 3737 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3738 3739 efx->type->fini(efx); 3740 3741 efx->reset_pending = 0; 3742 3743 pci_save_state(pci_dev); 3744 return pci_set_power_state(pci_dev, PCI_D3hot); 3745 } 3746 3747 /* Used for both resume and restore */ 3748 static int efx_pm_resume(struct device *dev) 3749 { 3750 struct pci_dev *pci_dev = to_pci_dev(dev); 3751 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3752 int rc; 3753 3754 rc = pci_set_power_state(pci_dev, PCI_D0); 3755 if (rc) 3756 return rc; 3757 pci_restore_state(pci_dev); 3758 rc = pci_enable_device(pci_dev); 3759 if (rc) 3760 return rc; 3761 pci_set_master(efx->pci_dev); 3762 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3763 if (rc) 3764 return rc; 3765 down_write(&efx->filter_sem); 3766 rc = efx->type->init(efx); 3767 up_write(&efx->filter_sem); 3768 if (rc) 3769 return rc; 3770 rc = efx_pm_thaw(dev); 3771 return rc; 3772 } 3773 3774 static int efx_pm_suspend(struct device *dev) 3775 { 3776 int rc; 3777 3778 efx_pm_freeze(dev); 3779 rc = efx_pm_poweroff(dev); 3780 if (rc) 3781 efx_pm_resume(dev); 3782 return rc; 3783 } 3784 3785 static const struct dev_pm_ops efx_pm_ops = { 3786 .suspend = efx_pm_suspend, 3787 .resume = efx_pm_resume, 3788 .freeze = efx_pm_freeze, 3789 .thaw = efx_pm_thaw, 3790 .poweroff = efx_pm_poweroff, 3791 .restore = efx_pm_resume, 3792 }; 3793 3794 /* A PCI error affecting this device was detected. 3795 * At this point MMIO and DMA may be disabled. 3796 * Stop the software path and request a slot reset. 3797 */ 3798 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3799 enum pci_channel_state state) 3800 { 3801 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3802 struct efx_nic *efx = pci_get_drvdata(pdev); 3803 3804 if (state == pci_channel_io_perm_failure) 3805 return PCI_ERS_RESULT_DISCONNECT; 3806 3807 rtnl_lock(); 3808 3809 if (efx->state != STATE_DISABLED) { 3810 efx->state = STATE_RECOVERY; 3811 efx->reset_pending = 0; 3812 3813 efx_device_detach_sync(efx); 3814 3815 efx_stop_all(efx); 3816 efx_disable_interrupts(efx); 3817 3818 status = PCI_ERS_RESULT_NEED_RESET; 3819 } else { 3820 /* If the interface is disabled we don't want to do anything 3821 * with it. 3822 */ 3823 status = PCI_ERS_RESULT_RECOVERED; 3824 } 3825 3826 rtnl_unlock(); 3827 3828 pci_disable_device(pdev); 3829 3830 return status; 3831 } 3832 3833 /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3834 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3835 { 3836 struct efx_nic *efx = pci_get_drvdata(pdev); 3837 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3838 int rc; 3839 3840 if (pci_enable_device(pdev)) { 3841 netif_err(efx, hw, efx->net_dev, 3842 "Cannot re-enable PCI device after reset.\n"); 3843 status = PCI_ERS_RESULT_DISCONNECT; 3844 } 3845 3846 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3847 if (rc) { 3848 netif_err(efx, hw, efx->net_dev, 3849 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3850 /* Non-fatal error. Continue. */ 3851 } 3852 3853 return status; 3854 } 3855 3856 /* Perform the actual reset and resume I/O operations. */ 3857 static void efx_io_resume(struct pci_dev *pdev) 3858 { 3859 struct efx_nic *efx = pci_get_drvdata(pdev); 3860 int rc; 3861 3862 rtnl_lock(); 3863 3864 if (efx->state == STATE_DISABLED) 3865 goto out; 3866 3867 rc = efx_reset(efx, RESET_TYPE_ALL); 3868 if (rc) { 3869 netif_err(efx, hw, efx->net_dev, 3870 "efx_reset failed after PCI error (%d)\n", rc); 3871 } else { 3872 efx->state = STATE_READY; 3873 netif_dbg(efx, hw, efx->net_dev, 3874 "Done resetting and resuming IO after PCI error.\n"); 3875 } 3876 3877 out: 3878 rtnl_unlock(); 3879 } 3880 3881 /* For simplicity and reliability, we always require a slot reset and try to 3882 * reset the hardware when a pci error affecting the device is detected. 3883 * We leave both the link_reset and mmio_enabled callback unimplemented: 3884 * with our request for slot reset the mmio_enabled callback will never be 3885 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3886 */ 3887 static const struct pci_error_handlers efx_err_handlers = { 3888 .error_detected = efx_io_error_detected, 3889 .slot_reset = efx_io_slot_reset, 3890 .resume = efx_io_resume, 3891 }; 3892 3893 static struct pci_driver efx_pci_driver = { 3894 .name = KBUILD_MODNAME, 3895 .id_table = efx_pci_table, 3896 .probe = efx_pci_probe, 3897 .remove = efx_pci_remove, 3898 .driver.pm = &efx_pm_ops, 3899 .err_handler = &efx_err_handlers, 3900 #ifdef CONFIG_SFC_SRIOV 3901 .sriov_configure = efx_pci_sriov_configure, 3902 #endif 3903 }; 3904 3905 /************************************************************************** 3906 * 3907 * Kernel module interface 3908 * 3909 *************************************************************************/ 3910 3911 module_param(interrupt_mode, uint, 0444); 3912 MODULE_PARM_DESC(interrupt_mode, 3913 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3914 3915 static int __init efx_init_module(void) 3916 { 3917 int rc; 3918 3919 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3920 3921 rc = register_netdevice_notifier(&efx_netdev_notifier); 3922 if (rc) 3923 goto err_notifier; 3924 3925 #ifdef CONFIG_SFC_SRIOV 3926 rc = efx_init_sriov(); 3927 if (rc) 3928 goto err_sriov; 3929 #endif 3930 3931 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3932 if (!reset_workqueue) { 3933 rc = -ENOMEM; 3934 goto err_reset; 3935 } 3936 3937 rc = pci_register_driver(&efx_pci_driver); 3938 if (rc < 0) 3939 goto err_pci; 3940 3941 return 0; 3942 3943 err_pci: 3944 destroy_workqueue(reset_workqueue); 3945 err_reset: 3946 #ifdef CONFIG_SFC_SRIOV 3947 efx_fini_sriov(); 3948 err_sriov: 3949 #endif 3950 unregister_netdevice_notifier(&efx_netdev_notifier); 3951 err_notifier: 3952 return rc; 3953 } 3954 3955 static void __exit efx_exit_module(void) 3956 { 3957 printk(KERN_INFO "Solarflare NET driver unloading\n"); 3958 3959 pci_unregister_driver(&efx_pci_driver); 3960 destroy_workqueue(reset_workqueue); 3961 #ifdef CONFIG_SFC_SRIOV 3962 efx_fini_sriov(); 3963 #endif 3964 unregister_netdevice_notifier(&efx_netdev_notifier); 3965 3966 } 3967 3968 module_init(efx_init_module); 3969 module_exit(efx_exit_module); 3970 3971 MODULE_AUTHOR("Solarflare Communications and " 3972 "Michael Brown <mbrown@fensystems.co.uk>"); 3973 MODULE_DESCRIPTION("Solarflare network driver"); 3974 MODULE_LICENSE("GPL"); 3975 MODULE_DEVICE_TABLE(pci, efx_pci_table); 3976 MODULE_VERSION(EFX_DRIVER_VERSION); 3977