1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include <net/gre.h> 27 #include <net/udp_tunnel.h> 28 #include "efx.h" 29 #include "nic.h" 30 #include "io.h" 31 #include "selftest.h" 32 #include "sriov.h" 33 34 #include "mcdi.h" 35 #include "mcdi_pcol.h" 36 #include "workarounds.h" 37 38 /************************************************************************** 39 * 40 * Type name strings 41 * 42 ************************************************************************** 43 */ 44 45 /* Loopback mode names (see LOOPBACK_MODE()) */ 46 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 47 const char *const efx_loopback_mode_names[] = { 48 [LOOPBACK_NONE] = "NONE", 49 [LOOPBACK_DATA] = "DATAPATH", 50 [LOOPBACK_GMAC] = "GMAC", 51 [LOOPBACK_XGMII] = "XGMII", 52 [LOOPBACK_XGXS] = "XGXS", 53 [LOOPBACK_XAUI] = "XAUI", 54 [LOOPBACK_GMII] = "GMII", 55 [LOOPBACK_SGMII] = "SGMII", 56 [LOOPBACK_XGBR] = "XGBR", 57 [LOOPBACK_XFI] = "XFI", 58 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 59 [LOOPBACK_GMII_FAR] = "GMII_FAR", 60 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 61 [LOOPBACK_XFI_FAR] = "XFI_FAR", 62 [LOOPBACK_GPHY] = "GPHY", 63 [LOOPBACK_PHYXS] = "PHYXS", 64 [LOOPBACK_PCS] = "PCS", 65 [LOOPBACK_PMAPMD] = "PMA/PMD", 66 [LOOPBACK_XPORT] = "XPORT", 67 [LOOPBACK_XGMII_WS] = "XGMII_WS", 68 [LOOPBACK_XAUI_WS] = "XAUI_WS", 69 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 70 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 71 [LOOPBACK_GMII_WS] = "GMII_WS", 72 [LOOPBACK_XFI_WS] = "XFI_WS", 73 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 74 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 75 }; 76 77 const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 78 const char *const efx_reset_type_names[] = { 79 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 80 [RESET_TYPE_ALL] = "ALL", 81 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 82 [RESET_TYPE_WORLD] = "WORLD", 83 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 84 [RESET_TYPE_DATAPATH] = "DATAPATH", 85 [RESET_TYPE_MC_BIST] = "MC_BIST", 86 [RESET_TYPE_DISABLE] = "DISABLE", 87 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 88 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 89 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 90 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 91 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 92 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 93 }; 94 95 /* UDP tunnel type names */ 96 static const char *const efx_udp_tunnel_type_names[] = { 97 [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan", 98 [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve", 99 }; 100 101 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen) 102 { 103 if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) && 104 efx_udp_tunnel_type_names[type] != NULL) 105 snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]); 106 else 107 snprintf(buf, buflen, "type %d", type); 108 } 109 110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 111 * queued onto this work queue. This is not a per-nic work queue, because 112 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 113 */ 114 static struct workqueue_struct *reset_workqueue; 115 116 /* How often and how many times to poll for a reset while waiting for a 117 * BIST that another function started to complete. 118 */ 119 #define BIST_WAIT_DELAY_MS 100 120 #define BIST_WAIT_DELAY_COUNT 100 121 122 /************************************************************************** 123 * 124 * Configurable values 125 * 126 *************************************************************************/ 127 128 /* 129 * Use separate channels for TX and RX events 130 * 131 * Set this to 1 to use separate channels for TX and RX. It allows us 132 * to control interrupt affinity separately for TX and RX. 133 * 134 * This is only used in MSI-X interrupt mode 135 */ 136 bool efx_separate_tx_channels; 137 module_param(efx_separate_tx_channels, bool, 0444); 138 MODULE_PARM_DESC(efx_separate_tx_channels, 139 "Use separate channels for TX and RX"); 140 141 /* This is the weight assigned to each of the (per-channel) virtual 142 * NAPI devices. 143 */ 144 static int napi_weight = 64; 145 146 /* This is the time (in jiffies) between invocations of the hardware 147 * monitor. 148 * On Falcon-based NICs, this will: 149 * - Check the on-board hardware monitor; 150 * - Poll the link state and reconfigure the hardware as necessary. 151 * On Siena-based NICs for power systems with EEH support, this will give EEH a 152 * chance to start. 153 */ 154 static unsigned int efx_monitor_interval = 1 * HZ; 155 156 /* Initial interrupt moderation settings. They can be modified after 157 * module load with ethtool. 158 * 159 * The default for RX should strike a balance between increasing the 160 * round-trip latency and reducing overhead. 161 */ 162 static unsigned int rx_irq_mod_usec = 60; 163 164 /* Initial interrupt moderation settings. They can be modified after 165 * module load with ethtool. 166 * 167 * This default is chosen to ensure that a 10G link does not go idle 168 * while a TX queue is stopped after it has become full. A queue is 169 * restarted when it drops below half full. The time this takes (assuming 170 * worst case 3 descriptors per packet and 1024 descriptors) is 171 * 512 / 3 * 1.2 = 205 usec. 172 */ 173 static unsigned int tx_irq_mod_usec = 150; 174 175 /* This is the first interrupt mode to try out of: 176 * 0 => MSI-X 177 * 1 => MSI 178 * 2 => legacy 179 */ 180 static unsigned int interrupt_mode; 181 182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 183 * i.e. the number of CPUs among which we may distribute simultaneous 184 * interrupt handling. 185 * 186 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 187 * The default (0) means to assign an interrupt to each core. 188 */ 189 static unsigned int rss_cpus; 190 module_param(rss_cpus, uint, 0444); 191 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 192 193 static bool phy_flash_cfg; 194 module_param(phy_flash_cfg, bool, 0644); 195 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 196 197 static unsigned irq_adapt_low_thresh = 8000; 198 module_param(irq_adapt_low_thresh, uint, 0644); 199 MODULE_PARM_DESC(irq_adapt_low_thresh, 200 "Threshold score for reducing IRQ moderation"); 201 202 static unsigned irq_adapt_high_thresh = 16000; 203 module_param(irq_adapt_high_thresh, uint, 0644); 204 MODULE_PARM_DESC(irq_adapt_high_thresh, 205 "Threshold score for increasing IRQ moderation"); 206 207 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 208 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 209 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 210 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 211 module_param(debug, uint, 0); 212 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 213 214 /************************************************************************** 215 * 216 * Utility functions and prototypes 217 * 218 *************************************************************************/ 219 220 static int efx_soft_enable_interrupts(struct efx_nic *efx); 221 static void efx_soft_disable_interrupts(struct efx_nic *efx); 222 static void efx_remove_channel(struct efx_channel *channel); 223 static void efx_remove_channels(struct efx_nic *efx); 224 static const struct efx_channel_type efx_default_channel_type; 225 static void efx_remove_port(struct efx_nic *efx); 226 static void efx_init_napi_channel(struct efx_channel *channel); 227 static void efx_fini_napi(struct efx_nic *efx); 228 static void efx_fini_napi_channel(struct efx_channel *channel); 229 static void efx_fini_struct(struct efx_nic *efx); 230 static void efx_start_all(struct efx_nic *efx); 231 static void efx_stop_all(struct efx_nic *efx); 232 233 #define EFX_ASSERT_RESET_SERIALISED(efx) \ 234 do { \ 235 if ((efx->state == STATE_READY) || \ 236 (efx->state == STATE_RECOVERY) || \ 237 (efx->state == STATE_DISABLED)) \ 238 ASSERT_RTNL(); \ 239 } while (0) 240 241 static int efx_check_disabled(struct efx_nic *efx) 242 { 243 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 244 netif_err(efx, drv, efx->net_dev, 245 "device is disabled due to earlier errors\n"); 246 return -EIO; 247 } 248 return 0; 249 } 250 251 /************************************************************************** 252 * 253 * Event queue processing 254 * 255 *************************************************************************/ 256 257 /* Process channel's event queue 258 * 259 * This function is responsible for processing the event queue of a 260 * single channel. The caller must guarantee that this function will 261 * never be concurrently called more than once on the same channel, 262 * though different channels may be being processed concurrently. 263 */ 264 static int efx_process_channel(struct efx_channel *channel, int budget) 265 { 266 struct efx_tx_queue *tx_queue; 267 int spent; 268 269 if (unlikely(!channel->enabled)) 270 return 0; 271 272 efx_for_each_channel_tx_queue(tx_queue, channel) { 273 tx_queue->pkts_compl = 0; 274 tx_queue->bytes_compl = 0; 275 } 276 277 spent = efx_nic_process_eventq(channel, budget); 278 if (spent && efx_channel_has_rx_queue(channel)) { 279 struct efx_rx_queue *rx_queue = 280 efx_channel_get_rx_queue(channel); 281 282 efx_rx_flush_packet(channel); 283 efx_fast_push_rx_descriptors(rx_queue, true); 284 } 285 286 /* Update BQL */ 287 efx_for_each_channel_tx_queue(tx_queue, channel) { 288 if (tx_queue->bytes_compl) { 289 netdev_tx_completed_queue(tx_queue->core_txq, 290 tx_queue->pkts_compl, tx_queue->bytes_compl); 291 } 292 } 293 294 return spent; 295 } 296 297 /* NAPI poll handler 298 * 299 * NAPI guarantees serialisation of polls of the same device, which 300 * provides the guarantee required by efx_process_channel(). 301 */ 302 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 303 { 304 int step = efx->irq_mod_step_us; 305 306 if (channel->irq_mod_score < irq_adapt_low_thresh) { 307 if (channel->irq_moderation_us > step) { 308 channel->irq_moderation_us -= step; 309 efx->type->push_irq_moderation(channel); 310 } 311 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 312 if (channel->irq_moderation_us < 313 efx->irq_rx_moderation_us) { 314 channel->irq_moderation_us += step; 315 efx->type->push_irq_moderation(channel); 316 } 317 } 318 319 channel->irq_count = 0; 320 channel->irq_mod_score = 0; 321 } 322 323 static int efx_poll(struct napi_struct *napi, int budget) 324 { 325 struct efx_channel *channel = 326 container_of(napi, struct efx_channel, napi_str); 327 struct efx_nic *efx = channel->efx; 328 int spent; 329 330 netif_vdbg(efx, intr, efx->net_dev, 331 "channel %d NAPI poll executing on CPU %d\n", 332 channel->channel, raw_smp_processor_id()); 333 334 spent = efx_process_channel(channel, budget); 335 336 if (spent < budget) { 337 if (efx_channel_has_rx_queue(channel) && 338 efx->irq_rx_adaptive && 339 unlikely(++channel->irq_count == 1000)) { 340 efx_update_irq_mod(efx, channel); 341 } 342 343 efx_filter_rfs_expire(channel); 344 345 /* There is no race here; although napi_disable() will 346 * only wait for napi_complete(), this isn't a problem 347 * since efx_nic_eventq_read_ack() will have no effect if 348 * interrupts have already been disabled. 349 */ 350 if (napi_complete_done(napi, spent)) 351 efx_nic_eventq_read_ack(channel); 352 } 353 354 return spent; 355 } 356 357 /* Create event queue 358 * Event queue memory allocations are done only once. If the channel 359 * is reset, the memory buffer will be reused; this guards against 360 * errors during channel reset and also simplifies interrupt handling. 361 */ 362 static int efx_probe_eventq(struct efx_channel *channel) 363 { 364 struct efx_nic *efx = channel->efx; 365 unsigned long entries; 366 367 netif_dbg(efx, probe, efx->net_dev, 368 "chan %d create event queue\n", channel->channel); 369 370 /* Build an event queue with room for one event per tx and rx buffer, 371 * plus some extra for link state events and MCDI completions. */ 372 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 373 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 374 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 375 376 return efx_nic_probe_eventq(channel); 377 } 378 379 /* Prepare channel's event queue */ 380 static int efx_init_eventq(struct efx_channel *channel) 381 { 382 struct efx_nic *efx = channel->efx; 383 int rc; 384 385 EFX_WARN_ON_PARANOID(channel->eventq_init); 386 387 netif_dbg(efx, drv, efx->net_dev, 388 "chan %d init event queue\n", channel->channel); 389 390 rc = efx_nic_init_eventq(channel); 391 if (rc == 0) { 392 efx->type->push_irq_moderation(channel); 393 channel->eventq_read_ptr = 0; 394 channel->eventq_init = true; 395 } 396 return rc; 397 } 398 399 /* Enable event queue processing and NAPI */ 400 void efx_start_eventq(struct efx_channel *channel) 401 { 402 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 403 "chan %d start event queue\n", channel->channel); 404 405 /* Make sure the NAPI handler sees the enabled flag set */ 406 channel->enabled = true; 407 smp_wmb(); 408 409 napi_enable(&channel->napi_str); 410 efx_nic_eventq_read_ack(channel); 411 } 412 413 /* Disable event queue processing and NAPI */ 414 void efx_stop_eventq(struct efx_channel *channel) 415 { 416 if (!channel->enabled) 417 return; 418 419 napi_disable(&channel->napi_str); 420 channel->enabled = false; 421 } 422 423 static void efx_fini_eventq(struct efx_channel *channel) 424 { 425 if (!channel->eventq_init) 426 return; 427 428 netif_dbg(channel->efx, drv, channel->efx->net_dev, 429 "chan %d fini event queue\n", channel->channel); 430 431 efx_nic_fini_eventq(channel); 432 channel->eventq_init = false; 433 } 434 435 static void efx_remove_eventq(struct efx_channel *channel) 436 { 437 netif_dbg(channel->efx, drv, channel->efx->net_dev, 438 "chan %d remove event queue\n", channel->channel); 439 440 efx_nic_remove_eventq(channel); 441 } 442 443 /************************************************************************** 444 * 445 * Channel handling 446 * 447 *************************************************************************/ 448 449 /* Allocate and initialise a channel structure. */ 450 static struct efx_channel * 451 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 452 { 453 struct efx_channel *channel; 454 struct efx_rx_queue *rx_queue; 455 struct efx_tx_queue *tx_queue; 456 int j; 457 458 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 459 if (!channel) 460 return NULL; 461 462 channel->efx = efx; 463 channel->channel = i; 464 channel->type = &efx_default_channel_type; 465 466 for (j = 0; j < EFX_TXQ_TYPES; j++) { 467 tx_queue = &channel->tx_queue[j]; 468 tx_queue->efx = efx; 469 tx_queue->queue = i * EFX_TXQ_TYPES + j; 470 tx_queue->channel = channel; 471 } 472 473 rx_queue = &channel->rx_queue; 474 rx_queue->efx = efx; 475 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 476 477 return channel; 478 } 479 480 /* Allocate and initialise a channel structure, copying parameters 481 * (but not resources) from an old channel structure. 482 */ 483 static struct efx_channel * 484 efx_copy_channel(const struct efx_channel *old_channel) 485 { 486 struct efx_channel *channel; 487 struct efx_rx_queue *rx_queue; 488 struct efx_tx_queue *tx_queue; 489 int j; 490 491 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 492 if (!channel) 493 return NULL; 494 495 *channel = *old_channel; 496 497 channel->napi_dev = NULL; 498 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 499 channel->napi_str.napi_id = 0; 500 channel->napi_str.state = 0; 501 memset(&channel->eventq, 0, sizeof(channel->eventq)); 502 503 for (j = 0; j < EFX_TXQ_TYPES; j++) { 504 tx_queue = &channel->tx_queue[j]; 505 if (tx_queue->channel) 506 tx_queue->channel = channel; 507 tx_queue->buffer = NULL; 508 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 509 } 510 511 rx_queue = &channel->rx_queue; 512 rx_queue->buffer = NULL; 513 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 514 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 515 516 return channel; 517 } 518 519 static int efx_probe_channel(struct efx_channel *channel) 520 { 521 struct efx_tx_queue *tx_queue; 522 struct efx_rx_queue *rx_queue; 523 int rc; 524 525 netif_dbg(channel->efx, probe, channel->efx->net_dev, 526 "creating channel %d\n", channel->channel); 527 528 rc = channel->type->pre_probe(channel); 529 if (rc) 530 goto fail; 531 532 rc = efx_probe_eventq(channel); 533 if (rc) 534 goto fail; 535 536 efx_for_each_channel_tx_queue(tx_queue, channel) { 537 rc = efx_probe_tx_queue(tx_queue); 538 if (rc) 539 goto fail; 540 } 541 542 efx_for_each_channel_rx_queue(rx_queue, channel) { 543 rc = efx_probe_rx_queue(rx_queue); 544 if (rc) 545 goto fail; 546 } 547 548 return 0; 549 550 fail: 551 efx_remove_channel(channel); 552 return rc; 553 } 554 555 static void 556 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 557 { 558 struct efx_nic *efx = channel->efx; 559 const char *type; 560 int number; 561 562 number = channel->channel; 563 if (efx->tx_channel_offset == 0) { 564 type = ""; 565 } else if (channel->channel < efx->tx_channel_offset) { 566 type = "-rx"; 567 } else { 568 type = "-tx"; 569 number -= efx->tx_channel_offset; 570 } 571 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 572 } 573 574 static void efx_set_channel_names(struct efx_nic *efx) 575 { 576 struct efx_channel *channel; 577 578 efx_for_each_channel(channel, efx) 579 channel->type->get_name(channel, 580 efx->msi_context[channel->channel].name, 581 sizeof(efx->msi_context[0].name)); 582 } 583 584 static int efx_probe_channels(struct efx_nic *efx) 585 { 586 struct efx_channel *channel; 587 int rc; 588 589 /* Restart special buffer allocation */ 590 efx->next_buffer_table = 0; 591 592 /* Probe channels in reverse, so that any 'extra' channels 593 * use the start of the buffer table. This allows the traffic 594 * channels to be resized without moving them or wasting the 595 * entries before them. 596 */ 597 efx_for_each_channel_rev(channel, efx) { 598 rc = efx_probe_channel(channel); 599 if (rc) { 600 netif_err(efx, probe, efx->net_dev, 601 "failed to create channel %d\n", 602 channel->channel); 603 goto fail; 604 } 605 } 606 efx_set_channel_names(efx); 607 608 return 0; 609 610 fail: 611 efx_remove_channels(efx); 612 return rc; 613 } 614 615 /* Channels are shutdown and reinitialised whilst the NIC is running 616 * to propagate configuration changes (mtu, checksum offload), or 617 * to clear hardware error conditions 618 */ 619 static void efx_start_datapath(struct efx_nic *efx) 620 { 621 netdev_features_t old_features = efx->net_dev->features; 622 bool old_rx_scatter = efx->rx_scatter; 623 struct efx_tx_queue *tx_queue; 624 struct efx_rx_queue *rx_queue; 625 struct efx_channel *channel; 626 size_t rx_buf_len; 627 628 /* Calculate the rx buffer allocation parameters required to 629 * support the current MTU, including padding for header 630 * alignment and overruns. 631 */ 632 efx->rx_dma_len = (efx->rx_prefix_size + 633 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 634 efx->type->rx_buffer_padding); 635 rx_buf_len = (sizeof(struct efx_rx_page_state) + 636 efx->rx_ip_align + efx->rx_dma_len); 637 if (rx_buf_len <= PAGE_SIZE) { 638 efx->rx_scatter = efx->type->always_rx_scatter; 639 efx->rx_buffer_order = 0; 640 } else if (efx->type->can_rx_scatter) { 641 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 642 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 643 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 644 EFX_RX_BUF_ALIGNMENT) > 645 PAGE_SIZE); 646 efx->rx_scatter = true; 647 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 648 efx->rx_buffer_order = 0; 649 } else { 650 efx->rx_scatter = false; 651 efx->rx_buffer_order = get_order(rx_buf_len); 652 } 653 654 efx_rx_config_page_split(efx); 655 if (efx->rx_buffer_order) 656 netif_dbg(efx, drv, efx->net_dev, 657 "RX buf len=%u; page order=%u batch=%u\n", 658 efx->rx_dma_len, efx->rx_buffer_order, 659 efx->rx_pages_per_batch); 660 else 661 netif_dbg(efx, drv, efx->net_dev, 662 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 663 efx->rx_dma_len, efx->rx_page_buf_step, 664 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 665 666 /* Restore previously fixed features in hw_features and remove 667 * features which are fixed now 668 */ 669 efx->net_dev->hw_features |= efx->net_dev->features; 670 efx->net_dev->hw_features &= ~efx->fixed_features; 671 efx->net_dev->features |= efx->fixed_features; 672 if (efx->net_dev->features != old_features) 673 netdev_features_change(efx->net_dev); 674 675 /* RX filters may also have scatter-enabled flags */ 676 if (efx->rx_scatter != old_rx_scatter) 677 efx->type->filter_update_rx_scatter(efx); 678 679 /* We must keep at least one descriptor in a TX ring empty. 680 * We could avoid this when the queue size does not exactly 681 * match the hardware ring size, but it's not that important. 682 * Therefore we stop the queue when one more skb might fill 683 * the ring completely. We wake it when half way back to 684 * empty. 685 */ 686 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 687 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 688 689 /* Initialise the channels */ 690 efx_for_each_channel(channel, efx) { 691 efx_for_each_channel_tx_queue(tx_queue, channel) { 692 efx_init_tx_queue(tx_queue); 693 atomic_inc(&efx->active_queues); 694 } 695 696 efx_for_each_channel_rx_queue(rx_queue, channel) { 697 efx_init_rx_queue(rx_queue); 698 atomic_inc(&efx->active_queues); 699 efx_stop_eventq(channel); 700 efx_fast_push_rx_descriptors(rx_queue, false); 701 efx_start_eventq(channel); 702 } 703 704 WARN_ON(channel->rx_pkt_n_frags); 705 } 706 707 efx_ptp_start_datapath(efx); 708 709 if (netif_device_present(efx->net_dev)) 710 netif_tx_wake_all_queues(efx->net_dev); 711 } 712 713 static void efx_stop_datapath(struct efx_nic *efx) 714 { 715 struct efx_channel *channel; 716 struct efx_tx_queue *tx_queue; 717 struct efx_rx_queue *rx_queue; 718 int rc; 719 720 EFX_ASSERT_RESET_SERIALISED(efx); 721 BUG_ON(efx->port_enabled); 722 723 efx_ptp_stop_datapath(efx); 724 725 /* Stop RX refill */ 726 efx_for_each_channel(channel, efx) { 727 efx_for_each_channel_rx_queue(rx_queue, channel) 728 rx_queue->refill_enabled = false; 729 } 730 731 efx_for_each_channel(channel, efx) { 732 /* RX packet processing is pipelined, so wait for the 733 * NAPI handler to complete. At least event queue 0 734 * might be kept active by non-data events, so don't 735 * use napi_synchronize() but actually disable NAPI 736 * temporarily. 737 */ 738 if (efx_channel_has_rx_queue(channel)) { 739 efx_stop_eventq(channel); 740 efx_start_eventq(channel); 741 } 742 } 743 744 rc = efx->type->fini_dmaq(efx); 745 if (rc) { 746 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 747 } else { 748 netif_dbg(efx, drv, efx->net_dev, 749 "successfully flushed all queues\n"); 750 } 751 752 efx_for_each_channel(channel, efx) { 753 efx_for_each_channel_rx_queue(rx_queue, channel) 754 efx_fini_rx_queue(rx_queue); 755 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 756 efx_fini_tx_queue(tx_queue); 757 } 758 } 759 760 static void efx_remove_channel(struct efx_channel *channel) 761 { 762 struct efx_tx_queue *tx_queue; 763 struct efx_rx_queue *rx_queue; 764 765 netif_dbg(channel->efx, drv, channel->efx->net_dev, 766 "destroy chan %d\n", channel->channel); 767 768 efx_for_each_channel_rx_queue(rx_queue, channel) 769 efx_remove_rx_queue(rx_queue); 770 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 771 efx_remove_tx_queue(tx_queue); 772 efx_remove_eventq(channel); 773 channel->type->post_remove(channel); 774 } 775 776 static void efx_remove_channels(struct efx_nic *efx) 777 { 778 struct efx_channel *channel; 779 780 efx_for_each_channel(channel, efx) 781 efx_remove_channel(channel); 782 } 783 784 int 785 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 786 { 787 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 788 u32 old_rxq_entries, old_txq_entries; 789 unsigned i, next_buffer_table = 0; 790 int rc, rc2; 791 792 rc = efx_check_disabled(efx); 793 if (rc) 794 return rc; 795 796 /* Not all channels should be reallocated. We must avoid 797 * reallocating their buffer table entries. 798 */ 799 efx_for_each_channel(channel, efx) { 800 struct efx_rx_queue *rx_queue; 801 struct efx_tx_queue *tx_queue; 802 803 if (channel->type->copy) 804 continue; 805 next_buffer_table = max(next_buffer_table, 806 channel->eventq.index + 807 channel->eventq.entries); 808 efx_for_each_channel_rx_queue(rx_queue, channel) 809 next_buffer_table = max(next_buffer_table, 810 rx_queue->rxd.index + 811 rx_queue->rxd.entries); 812 efx_for_each_channel_tx_queue(tx_queue, channel) 813 next_buffer_table = max(next_buffer_table, 814 tx_queue->txd.index + 815 tx_queue->txd.entries); 816 } 817 818 efx_device_detach_sync(efx); 819 efx_stop_all(efx); 820 efx_soft_disable_interrupts(efx); 821 822 /* Clone channels (where possible) */ 823 memset(other_channel, 0, sizeof(other_channel)); 824 for (i = 0; i < efx->n_channels; i++) { 825 channel = efx->channel[i]; 826 if (channel->type->copy) 827 channel = channel->type->copy(channel); 828 if (!channel) { 829 rc = -ENOMEM; 830 goto out; 831 } 832 other_channel[i] = channel; 833 } 834 835 /* Swap entry counts and channel pointers */ 836 old_rxq_entries = efx->rxq_entries; 837 old_txq_entries = efx->txq_entries; 838 efx->rxq_entries = rxq_entries; 839 efx->txq_entries = txq_entries; 840 for (i = 0; i < efx->n_channels; i++) { 841 channel = efx->channel[i]; 842 efx->channel[i] = other_channel[i]; 843 other_channel[i] = channel; 844 } 845 846 /* Restart buffer table allocation */ 847 efx->next_buffer_table = next_buffer_table; 848 849 for (i = 0; i < efx->n_channels; i++) { 850 channel = efx->channel[i]; 851 if (!channel->type->copy) 852 continue; 853 rc = efx_probe_channel(channel); 854 if (rc) 855 goto rollback; 856 efx_init_napi_channel(efx->channel[i]); 857 } 858 859 out: 860 /* Destroy unused channel structures */ 861 for (i = 0; i < efx->n_channels; i++) { 862 channel = other_channel[i]; 863 if (channel && channel->type->copy) { 864 efx_fini_napi_channel(channel); 865 efx_remove_channel(channel); 866 kfree(channel); 867 } 868 } 869 870 rc2 = efx_soft_enable_interrupts(efx); 871 if (rc2) { 872 rc = rc ? rc : rc2; 873 netif_err(efx, drv, efx->net_dev, 874 "unable to restart interrupts on channel reallocation\n"); 875 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 876 } else { 877 efx_start_all(efx); 878 efx_device_attach_if_not_resetting(efx); 879 } 880 return rc; 881 882 rollback: 883 /* Swap back */ 884 efx->rxq_entries = old_rxq_entries; 885 efx->txq_entries = old_txq_entries; 886 for (i = 0; i < efx->n_channels; i++) { 887 channel = efx->channel[i]; 888 efx->channel[i] = other_channel[i]; 889 other_channel[i] = channel; 890 } 891 goto out; 892 } 893 894 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 895 { 896 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 897 } 898 899 static const struct efx_channel_type efx_default_channel_type = { 900 .pre_probe = efx_channel_dummy_op_int, 901 .post_remove = efx_channel_dummy_op_void, 902 .get_name = efx_get_channel_name, 903 .copy = efx_copy_channel, 904 .keep_eventq = false, 905 }; 906 907 int efx_channel_dummy_op_int(struct efx_channel *channel) 908 { 909 return 0; 910 } 911 912 void efx_channel_dummy_op_void(struct efx_channel *channel) 913 { 914 } 915 916 /************************************************************************** 917 * 918 * Port handling 919 * 920 **************************************************************************/ 921 922 /* This ensures that the kernel is kept informed (via 923 * netif_carrier_on/off) of the link status, and also maintains the 924 * link status's stop on the port's TX queue. 925 */ 926 void efx_link_status_changed(struct efx_nic *efx) 927 { 928 struct efx_link_state *link_state = &efx->link_state; 929 930 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 931 * that no events are triggered between unregister_netdev() and the 932 * driver unloading. A more general condition is that NETDEV_CHANGE 933 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 934 if (!netif_running(efx->net_dev)) 935 return; 936 937 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 938 efx->n_link_state_changes++; 939 940 if (link_state->up) 941 netif_carrier_on(efx->net_dev); 942 else 943 netif_carrier_off(efx->net_dev); 944 } 945 946 /* Status message for kernel log */ 947 if (link_state->up) 948 netif_info(efx, link, efx->net_dev, 949 "link up at %uMbps %s-duplex (MTU %d)\n", 950 link_state->speed, link_state->fd ? "full" : "half", 951 efx->net_dev->mtu); 952 else 953 netif_info(efx, link, efx->net_dev, "link down\n"); 954 } 955 956 void efx_link_set_advertising(struct efx_nic *efx, u32 advertising) 957 { 958 efx->link_advertising = advertising; 959 if (advertising) { 960 if (advertising & ADVERTISED_Pause) 961 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 962 else 963 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 964 if (advertising & ADVERTISED_Asym_Pause) 965 efx->wanted_fc ^= EFX_FC_TX; 966 } 967 } 968 969 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 970 { 971 efx->wanted_fc = wanted_fc; 972 if (efx->link_advertising) { 973 if (wanted_fc & EFX_FC_RX) 974 efx->link_advertising |= (ADVERTISED_Pause | 975 ADVERTISED_Asym_Pause); 976 else 977 efx->link_advertising &= ~(ADVERTISED_Pause | 978 ADVERTISED_Asym_Pause); 979 if (wanted_fc & EFX_FC_TX) 980 efx->link_advertising ^= ADVERTISED_Asym_Pause; 981 } 982 } 983 984 static void efx_fini_port(struct efx_nic *efx); 985 986 /* We assume that efx->type->reconfigure_mac will always try to sync RX 987 * filters and therefore needs to read-lock the filter table against freeing 988 */ 989 void efx_mac_reconfigure(struct efx_nic *efx) 990 { 991 down_read(&efx->filter_sem); 992 efx->type->reconfigure_mac(efx); 993 up_read(&efx->filter_sem); 994 } 995 996 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 997 * the MAC appropriately. All other PHY configuration changes are pushed 998 * through phy_op->set_settings(), and pushed asynchronously to the MAC 999 * through efx_monitor(). 1000 * 1001 * Callers must hold the mac_lock 1002 */ 1003 int __efx_reconfigure_port(struct efx_nic *efx) 1004 { 1005 enum efx_phy_mode phy_mode; 1006 int rc; 1007 1008 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1009 1010 /* Disable PHY transmit in mac level loopbacks */ 1011 phy_mode = efx->phy_mode; 1012 if (LOOPBACK_INTERNAL(efx)) 1013 efx->phy_mode |= PHY_MODE_TX_DISABLED; 1014 else 1015 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1016 1017 rc = efx->type->reconfigure_port(efx); 1018 1019 if (rc) 1020 efx->phy_mode = phy_mode; 1021 1022 return rc; 1023 } 1024 1025 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1026 * disabled. */ 1027 int efx_reconfigure_port(struct efx_nic *efx) 1028 { 1029 int rc; 1030 1031 EFX_ASSERT_RESET_SERIALISED(efx); 1032 1033 mutex_lock(&efx->mac_lock); 1034 rc = __efx_reconfigure_port(efx); 1035 mutex_unlock(&efx->mac_lock); 1036 1037 return rc; 1038 } 1039 1040 /* Asynchronous work item for changing MAC promiscuity and multicast 1041 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1042 * MAC directly. */ 1043 static void efx_mac_work(struct work_struct *data) 1044 { 1045 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1046 1047 mutex_lock(&efx->mac_lock); 1048 if (efx->port_enabled) 1049 efx_mac_reconfigure(efx); 1050 mutex_unlock(&efx->mac_lock); 1051 } 1052 1053 static int efx_probe_port(struct efx_nic *efx) 1054 { 1055 int rc; 1056 1057 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1058 1059 if (phy_flash_cfg) 1060 efx->phy_mode = PHY_MODE_SPECIAL; 1061 1062 /* Connect up MAC/PHY operations table */ 1063 rc = efx->type->probe_port(efx); 1064 if (rc) 1065 return rc; 1066 1067 /* Initialise MAC address to permanent address */ 1068 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1069 1070 return 0; 1071 } 1072 1073 static int efx_init_port(struct efx_nic *efx) 1074 { 1075 int rc; 1076 1077 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1078 1079 mutex_lock(&efx->mac_lock); 1080 1081 rc = efx->phy_op->init(efx); 1082 if (rc) 1083 goto fail1; 1084 1085 efx->port_initialized = true; 1086 1087 /* Reconfigure the MAC before creating dma queues (required for 1088 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1089 efx_mac_reconfigure(efx); 1090 1091 /* Ensure the PHY advertises the correct flow control settings */ 1092 rc = efx->phy_op->reconfigure(efx); 1093 if (rc && rc != -EPERM) 1094 goto fail2; 1095 1096 mutex_unlock(&efx->mac_lock); 1097 return 0; 1098 1099 fail2: 1100 efx->phy_op->fini(efx); 1101 fail1: 1102 mutex_unlock(&efx->mac_lock); 1103 return rc; 1104 } 1105 1106 static void efx_start_port(struct efx_nic *efx) 1107 { 1108 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1109 BUG_ON(efx->port_enabled); 1110 1111 mutex_lock(&efx->mac_lock); 1112 efx->port_enabled = true; 1113 1114 /* Ensure MAC ingress/egress is enabled */ 1115 efx_mac_reconfigure(efx); 1116 1117 mutex_unlock(&efx->mac_lock); 1118 } 1119 1120 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1121 * and the async self-test, wait for them to finish and prevent them 1122 * being scheduled again. This doesn't cover online resets, which 1123 * should only be cancelled when removing the device. 1124 */ 1125 static void efx_stop_port(struct efx_nic *efx) 1126 { 1127 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1128 1129 EFX_ASSERT_RESET_SERIALISED(efx); 1130 1131 mutex_lock(&efx->mac_lock); 1132 efx->port_enabled = false; 1133 mutex_unlock(&efx->mac_lock); 1134 1135 /* Serialise against efx_set_multicast_list() */ 1136 netif_addr_lock_bh(efx->net_dev); 1137 netif_addr_unlock_bh(efx->net_dev); 1138 1139 cancel_delayed_work_sync(&efx->monitor_work); 1140 efx_selftest_async_cancel(efx); 1141 cancel_work_sync(&efx->mac_work); 1142 } 1143 1144 static void efx_fini_port(struct efx_nic *efx) 1145 { 1146 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1147 1148 if (!efx->port_initialized) 1149 return; 1150 1151 efx->phy_op->fini(efx); 1152 efx->port_initialized = false; 1153 1154 efx->link_state.up = false; 1155 efx_link_status_changed(efx); 1156 } 1157 1158 static void efx_remove_port(struct efx_nic *efx) 1159 { 1160 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1161 1162 efx->type->remove_port(efx); 1163 } 1164 1165 /************************************************************************** 1166 * 1167 * NIC handling 1168 * 1169 **************************************************************************/ 1170 1171 static LIST_HEAD(efx_primary_list); 1172 static LIST_HEAD(efx_unassociated_list); 1173 1174 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 1175 { 1176 return left->type == right->type && 1177 left->vpd_sn && right->vpd_sn && 1178 !strcmp(left->vpd_sn, right->vpd_sn); 1179 } 1180 1181 static void efx_associate(struct efx_nic *efx) 1182 { 1183 struct efx_nic *other, *next; 1184 1185 if (efx->primary == efx) { 1186 /* Adding primary function; look for secondaries */ 1187 1188 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1189 list_add_tail(&efx->node, &efx_primary_list); 1190 1191 list_for_each_entry_safe(other, next, &efx_unassociated_list, 1192 node) { 1193 if (efx_same_controller(efx, other)) { 1194 list_del(&other->node); 1195 netif_dbg(other, probe, other->net_dev, 1196 "moving to secondary list of %s %s\n", 1197 pci_name(efx->pci_dev), 1198 efx->net_dev->name); 1199 list_add_tail(&other->node, 1200 &efx->secondary_list); 1201 other->primary = efx; 1202 } 1203 } 1204 } else { 1205 /* Adding secondary function; look for primary */ 1206 1207 list_for_each_entry(other, &efx_primary_list, node) { 1208 if (efx_same_controller(efx, other)) { 1209 netif_dbg(efx, probe, efx->net_dev, 1210 "adding to secondary list of %s %s\n", 1211 pci_name(other->pci_dev), 1212 other->net_dev->name); 1213 list_add_tail(&efx->node, 1214 &other->secondary_list); 1215 efx->primary = other; 1216 return; 1217 } 1218 } 1219 1220 netif_dbg(efx, probe, efx->net_dev, 1221 "adding to unassociated list\n"); 1222 list_add_tail(&efx->node, &efx_unassociated_list); 1223 } 1224 } 1225 1226 static void efx_dissociate(struct efx_nic *efx) 1227 { 1228 struct efx_nic *other, *next; 1229 1230 list_del(&efx->node); 1231 efx->primary = NULL; 1232 1233 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1234 list_del(&other->node); 1235 netif_dbg(other, probe, other->net_dev, 1236 "moving to unassociated list\n"); 1237 list_add_tail(&other->node, &efx_unassociated_list); 1238 other->primary = NULL; 1239 } 1240 } 1241 1242 /* This configures the PCI device to enable I/O and DMA. */ 1243 static int efx_init_io(struct efx_nic *efx) 1244 { 1245 struct pci_dev *pci_dev = efx->pci_dev; 1246 dma_addr_t dma_mask = efx->type->max_dma_mask; 1247 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1248 int rc, bar; 1249 1250 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1251 1252 bar = efx->type->mem_bar(efx); 1253 1254 rc = pci_enable_device(pci_dev); 1255 if (rc) { 1256 netif_err(efx, probe, efx->net_dev, 1257 "failed to enable PCI device\n"); 1258 goto fail1; 1259 } 1260 1261 pci_set_master(pci_dev); 1262 1263 /* Set the PCI DMA mask. Try all possibilities from our 1264 * genuine mask down to 32 bits, because some architectures 1265 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1266 * masks event though they reject 46 bit masks. 1267 */ 1268 while (dma_mask > 0x7fffffffUL) { 1269 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1270 if (rc == 0) 1271 break; 1272 dma_mask >>= 1; 1273 } 1274 if (rc) { 1275 netif_err(efx, probe, efx->net_dev, 1276 "could not find a suitable DMA mask\n"); 1277 goto fail2; 1278 } 1279 netif_dbg(efx, probe, efx->net_dev, 1280 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1281 1282 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1283 rc = pci_request_region(pci_dev, bar, "sfc"); 1284 if (rc) { 1285 netif_err(efx, probe, efx->net_dev, 1286 "request for memory BAR failed\n"); 1287 rc = -EIO; 1288 goto fail3; 1289 } 1290 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1291 if (!efx->membase) { 1292 netif_err(efx, probe, efx->net_dev, 1293 "could not map memory BAR at %llx+%x\n", 1294 (unsigned long long)efx->membase_phys, mem_map_size); 1295 rc = -ENOMEM; 1296 goto fail4; 1297 } 1298 netif_dbg(efx, probe, efx->net_dev, 1299 "memory BAR at %llx+%x (virtual %p)\n", 1300 (unsigned long long)efx->membase_phys, mem_map_size, 1301 efx->membase); 1302 1303 return 0; 1304 1305 fail4: 1306 pci_release_region(efx->pci_dev, bar); 1307 fail3: 1308 efx->membase_phys = 0; 1309 fail2: 1310 pci_disable_device(efx->pci_dev); 1311 fail1: 1312 return rc; 1313 } 1314 1315 static void efx_fini_io(struct efx_nic *efx) 1316 { 1317 int bar; 1318 1319 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1320 1321 if (efx->membase) { 1322 iounmap(efx->membase); 1323 efx->membase = NULL; 1324 } 1325 1326 if (efx->membase_phys) { 1327 bar = efx->type->mem_bar(efx); 1328 pci_release_region(efx->pci_dev, bar); 1329 efx->membase_phys = 0; 1330 } 1331 1332 /* Don't disable bus-mastering if VFs are assigned */ 1333 if (!pci_vfs_assigned(efx->pci_dev)) 1334 pci_disable_device(efx->pci_dev); 1335 } 1336 1337 void efx_set_default_rx_indir_table(struct efx_nic *efx) 1338 { 1339 size_t i; 1340 1341 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1342 efx->rx_indir_table[i] = 1343 ethtool_rxfh_indir_default(i, efx->rss_spread); 1344 } 1345 1346 static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1347 { 1348 cpumask_var_t thread_mask; 1349 unsigned int count; 1350 int cpu; 1351 1352 if (rss_cpus) { 1353 count = rss_cpus; 1354 } else { 1355 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1356 netif_warn(efx, probe, efx->net_dev, 1357 "RSS disabled due to allocation failure\n"); 1358 return 1; 1359 } 1360 1361 count = 0; 1362 for_each_online_cpu(cpu) { 1363 if (!cpumask_test_cpu(cpu, thread_mask)) { 1364 ++count; 1365 cpumask_or(thread_mask, thread_mask, 1366 topology_sibling_cpumask(cpu)); 1367 } 1368 } 1369 1370 free_cpumask_var(thread_mask); 1371 } 1372 1373 if (count > EFX_MAX_RX_QUEUES) { 1374 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1375 "Reducing number of rx queues from %u to %u.\n", 1376 count, EFX_MAX_RX_QUEUES); 1377 count = EFX_MAX_RX_QUEUES; 1378 } 1379 1380 /* If RSS is requested for the PF *and* VFs then we can't write RSS 1381 * table entries that are inaccessible to VFs 1382 */ 1383 #ifdef CONFIG_SFC_SRIOV 1384 if (efx->type->sriov_wanted) { 1385 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1386 count > efx_vf_size(efx)) { 1387 netif_warn(efx, probe, efx->net_dev, 1388 "Reducing number of RSS channels from %u to %u for " 1389 "VF support. Increase vf-msix-limit to use more " 1390 "channels on the PF.\n", 1391 count, efx_vf_size(efx)); 1392 count = efx_vf_size(efx); 1393 } 1394 } 1395 #endif 1396 1397 return count; 1398 } 1399 1400 /* Probe the number and type of interrupts we are able to obtain, and 1401 * the resulting numbers of channels and RX queues. 1402 */ 1403 static int efx_probe_interrupts(struct efx_nic *efx) 1404 { 1405 unsigned int extra_channels = 0; 1406 unsigned int i, j; 1407 int rc; 1408 1409 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 1410 if (efx->extra_channel_type[i]) 1411 ++extra_channels; 1412 1413 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1414 struct msix_entry xentries[EFX_MAX_CHANNELS]; 1415 unsigned int n_channels; 1416 1417 n_channels = efx_wanted_parallelism(efx); 1418 if (efx_separate_tx_channels) 1419 n_channels *= 2; 1420 n_channels += extra_channels; 1421 n_channels = min(n_channels, efx->max_channels); 1422 1423 for (i = 0; i < n_channels; i++) 1424 xentries[i].entry = i; 1425 rc = pci_enable_msix_range(efx->pci_dev, 1426 xentries, 1, n_channels); 1427 if (rc < 0) { 1428 /* Fall back to single channel MSI */ 1429 netif_err(efx, drv, efx->net_dev, 1430 "could not enable MSI-X\n"); 1431 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 1432 efx->interrupt_mode = EFX_INT_MODE_MSI; 1433 else 1434 return rc; 1435 } else if (rc < n_channels) { 1436 netif_err(efx, drv, efx->net_dev, 1437 "WARNING: Insufficient MSI-X vectors" 1438 " available (%d < %u).\n", rc, n_channels); 1439 netif_err(efx, drv, efx->net_dev, 1440 "WARNING: Performance may be reduced.\n"); 1441 n_channels = rc; 1442 } 1443 1444 if (rc > 0) { 1445 efx->n_channels = n_channels; 1446 if (n_channels > extra_channels) 1447 n_channels -= extra_channels; 1448 if (efx_separate_tx_channels) { 1449 efx->n_tx_channels = min(max(n_channels / 2, 1450 1U), 1451 efx->max_tx_channels); 1452 efx->n_rx_channels = max(n_channels - 1453 efx->n_tx_channels, 1454 1U); 1455 } else { 1456 efx->n_tx_channels = min(n_channels, 1457 efx->max_tx_channels); 1458 efx->n_rx_channels = n_channels; 1459 } 1460 for (i = 0; i < efx->n_channels; i++) 1461 efx_get_channel(efx, i)->irq = 1462 xentries[i].vector; 1463 } 1464 } 1465 1466 /* Try single interrupt MSI */ 1467 if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1468 efx->n_channels = 1; 1469 efx->n_rx_channels = 1; 1470 efx->n_tx_channels = 1; 1471 rc = pci_enable_msi(efx->pci_dev); 1472 if (rc == 0) { 1473 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1474 } else { 1475 netif_err(efx, drv, efx->net_dev, 1476 "could not enable MSI\n"); 1477 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 1478 efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1479 else 1480 return rc; 1481 } 1482 } 1483 1484 /* Assume legacy interrupts */ 1485 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1486 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1487 efx->n_rx_channels = 1; 1488 efx->n_tx_channels = 1; 1489 efx->legacy_irq = efx->pci_dev->irq; 1490 } 1491 1492 /* Assign extra channels if possible */ 1493 j = efx->n_channels; 1494 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 1495 if (!efx->extra_channel_type[i]) 1496 continue; 1497 if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 1498 efx->n_channels <= extra_channels) { 1499 efx->extra_channel_type[i]->handle_no_channel(efx); 1500 } else { 1501 --j; 1502 efx_get_channel(efx, j)->type = 1503 efx->extra_channel_type[i]; 1504 } 1505 } 1506 1507 /* RSS might be usable on VFs even if it is disabled on the PF */ 1508 #ifdef CONFIG_SFC_SRIOV 1509 if (efx->type->sriov_wanted) { 1510 efx->rss_spread = ((efx->n_rx_channels > 1 || 1511 !efx->type->sriov_wanted(efx)) ? 1512 efx->n_rx_channels : efx_vf_size(efx)); 1513 return 0; 1514 } 1515 #endif 1516 efx->rss_spread = efx->n_rx_channels; 1517 1518 return 0; 1519 } 1520 1521 static int efx_soft_enable_interrupts(struct efx_nic *efx) 1522 { 1523 struct efx_channel *channel, *end_channel; 1524 int rc; 1525 1526 BUG_ON(efx->state == STATE_DISABLED); 1527 1528 efx->irq_soft_enabled = true; 1529 smp_wmb(); 1530 1531 efx_for_each_channel(channel, efx) { 1532 if (!channel->type->keep_eventq) { 1533 rc = efx_init_eventq(channel); 1534 if (rc) 1535 goto fail; 1536 } 1537 efx_start_eventq(channel); 1538 } 1539 1540 efx_mcdi_mode_event(efx); 1541 1542 return 0; 1543 fail: 1544 end_channel = channel; 1545 efx_for_each_channel(channel, efx) { 1546 if (channel == end_channel) 1547 break; 1548 efx_stop_eventq(channel); 1549 if (!channel->type->keep_eventq) 1550 efx_fini_eventq(channel); 1551 } 1552 1553 return rc; 1554 } 1555 1556 static void efx_soft_disable_interrupts(struct efx_nic *efx) 1557 { 1558 struct efx_channel *channel; 1559 1560 if (efx->state == STATE_DISABLED) 1561 return; 1562 1563 efx_mcdi_mode_poll(efx); 1564 1565 efx->irq_soft_enabled = false; 1566 smp_wmb(); 1567 1568 if (efx->legacy_irq) 1569 synchronize_irq(efx->legacy_irq); 1570 1571 efx_for_each_channel(channel, efx) { 1572 if (channel->irq) 1573 synchronize_irq(channel->irq); 1574 1575 efx_stop_eventq(channel); 1576 if (!channel->type->keep_eventq) 1577 efx_fini_eventq(channel); 1578 } 1579 1580 /* Flush the asynchronous MCDI request queue */ 1581 efx_mcdi_flush_async(efx); 1582 } 1583 1584 static int efx_enable_interrupts(struct efx_nic *efx) 1585 { 1586 struct efx_channel *channel, *end_channel; 1587 int rc; 1588 1589 BUG_ON(efx->state == STATE_DISABLED); 1590 1591 if (efx->eeh_disabled_legacy_irq) { 1592 enable_irq(efx->legacy_irq); 1593 efx->eeh_disabled_legacy_irq = false; 1594 } 1595 1596 efx->type->irq_enable_master(efx); 1597 1598 efx_for_each_channel(channel, efx) { 1599 if (channel->type->keep_eventq) { 1600 rc = efx_init_eventq(channel); 1601 if (rc) 1602 goto fail; 1603 } 1604 } 1605 1606 rc = efx_soft_enable_interrupts(efx); 1607 if (rc) 1608 goto fail; 1609 1610 return 0; 1611 1612 fail: 1613 end_channel = channel; 1614 efx_for_each_channel(channel, efx) { 1615 if (channel == end_channel) 1616 break; 1617 if (channel->type->keep_eventq) 1618 efx_fini_eventq(channel); 1619 } 1620 1621 efx->type->irq_disable_non_ev(efx); 1622 1623 return rc; 1624 } 1625 1626 static void efx_disable_interrupts(struct efx_nic *efx) 1627 { 1628 struct efx_channel *channel; 1629 1630 efx_soft_disable_interrupts(efx); 1631 1632 efx_for_each_channel(channel, efx) { 1633 if (channel->type->keep_eventq) 1634 efx_fini_eventq(channel); 1635 } 1636 1637 efx->type->irq_disable_non_ev(efx); 1638 } 1639 1640 static void efx_remove_interrupts(struct efx_nic *efx) 1641 { 1642 struct efx_channel *channel; 1643 1644 /* Remove MSI/MSI-X interrupts */ 1645 efx_for_each_channel(channel, efx) 1646 channel->irq = 0; 1647 pci_disable_msi(efx->pci_dev); 1648 pci_disable_msix(efx->pci_dev); 1649 1650 /* Remove legacy interrupt */ 1651 efx->legacy_irq = 0; 1652 } 1653 1654 static void efx_set_channels(struct efx_nic *efx) 1655 { 1656 struct efx_channel *channel; 1657 struct efx_tx_queue *tx_queue; 1658 1659 efx->tx_channel_offset = 1660 efx_separate_tx_channels ? 1661 efx->n_channels - efx->n_tx_channels : 0; 1662 1663 /* We need to mark which channels really have RX and TX 1664 * queues, and adjust the TX queue numbers if we have separate 1665 * RX-only and TX-only channels. 1666 */ 1667 efx_for_each_channel(channel, efx) { 1668 if (channel->channel < efx->n_rx_channels) 1669 channel->rx_queue.core_index = channel->channel; 1670 else 1671 channel->rx_queue.core_index = -1; 1672 1673 efx_for_each_channel_tx_queue(tx_queue, channel) 1674 tx_queue->queue -= (efx->tx_channel_offset * 1675 EFX_TXQ_TYPES); 1676 } 1677 } 1678 1679 static int efx_probe_nic(struct efx_nic *efx) 1680 { 1681 int rc; 1682 1683 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1684 1685 /* Carry out hardware-type specific initialisation */ 1686 rc = efx->type->probe(efx); 1687 if (rc) 1688 return rc; 1689 1690 do { 1691 if (!efx->max_channels || !efx->max_tx_channels) { 1692 netif_err(efx, drv, efx->net_dev, 1693 "Insufficient resources to allocate" 1694 " any channels\n"); 1695 rc = -ENOSPC; 1696 goto fail1; 1697 } 1698 1699 /* Determine the number of channels and queues by trying 1700 * to hook in MSI-X interrupts. 1701 */ 1702 rc = efx_probe_interrupts(efx); 1703 if (rc) 1704 goto fail1; 1705 1706 efx_set_channels(efx); 1707 1708 /* dimension_resources can fail with EAGAIN */ 1709 rc = efx->type->dimension_resources(efx); 1710 if (rc != 0 && rc != -EAGAIN) 1711 goto fail2; 1712 1713 if (rc == -EAGAIN) 1714 /* try again with new max_channels */ 1715 efx_remove_interrupts(efx); 1716 1717 } while (rc == -EAGAIN); 1718 1719 if (efx->n_channels > 1) 1720 netdev_rss_key_fill(&efx->rx_hash_key, 1721 sizeof(efx->rx_hash_key)); 1722 efx_set_default_rx_indir_table(efx); 1723 1724 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1725 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1726 1727 /* Initialise the interrupt moderation settings */ 1728 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1729 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1730 true); 1731 1732 return 0; 1733 1734 fail2: 1735 efx_remove_interrupts(efx); 1736 fail1: 1737 efx->type->remove(efx); 1738 return rc; 1739 } 1740 1741 static void efx_remove_nic(struct efx_nic *efx) 1742 { 1743 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1744 1745 efx_remove_interrupts(efx); 1746 efx->type->remove(efx); 1747 } 1748 1749 static int efx_probe_filters(struct efx_nic *efx) 1750 { 1751 int rc; 1752 1753 spin_lock_init(&efx->filter_lock); 1754 init_rwsem(&efx->filter_sem); 1755 mutex_lock(&efx->mac_lock); 1756 down_write(&efx->filter_sem); 1757 rc = efx->type->filter_table_probe(efx); 1758 if (rc) 1759 goto out_unlock; 1760 1761 #ifdef CONFIG_RFS_ACCEL 1762 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1763 struct efx_channel *channel; 1764 int i, success = 1; 1765 1766 efx_for_each_channel(channel, efx) { 1767 channel->rps_flow_id = 1768 kcalloc(efx->type->max_rx_ip_filters, 1769 sizeof(*channel->rps_flow_id), 1770 GFP_KERNEL); 1771 if (!channel->rps_flow_id) 1772 success = 0; 1773 else 1774 for (i = 0; 1775 i < efx->type->max_rx_ip_filters; 1776 ++i) 1777 channel->rps_flow_id[i] = 1778 RPS_FLOW_ID_INVALID; 1779 } 1780 1781 if (!success) { 1782 efx_for_each_channel(channel, efx) 1783 kfree(channel->rps_flow_id); 1784 efx->type->filter_table_remove(efx); 1785 rc = -ENOMEM; 1786 goto out_unlock; 1787 } 1788 1789 efx->rps_expire_index = efx->rps_expire_channel = 0; 1790 } 1791 #endif 1792 out_unlock: 1793 up_write(&efx->filter_sem); 1794 mutex_unlock(&efx->mac_lock); 1795 return rc; 1796 } 1797 1798 static void efx_remove_filters(struct efx_nic *efx) 1799 { 1800 #ifdef CONFIG_RFS_ACCEL 1801 struct efx_channel *channel; 1802 1803 efx_for_each_channel(channel, efx) 1804 kfree(channel->rps_flow_id); 1805 #endif 1806 down_write(&efx->filter_sem); 1807 efx->type->filter_table_remove(efx); 1808 up_write(&efx->filter_sem); 1809 } 1810 1811 static void efx_restore_filters(struct efx_nic *efx) 1812 { 1813 down_read(&efx->filter_sem); 1814 efx->type->filter_table_restore(efx); 1815 up_read(&efx->filter_sem); 1816 } 1817 1818 /************************************************************************** 1819 * 1820 * NIC startup/shutdown 1821 * 1822 *************************************************************************/ 1823 1824 static int efx_probe_all(struct efx_nic *efx) 1825 { 1826 int rc; 1827 1828 rc = efx_probe_nic(efx); 1829 if (rc) { 1830 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1831 goto fail1; 1832 } 1833 1834 rc = efx_probe_port(efx); 1835 if (rc) { 1836 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1837 goto fail2; 1838 } 1839 1840 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 1841 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 1842 rc = -EINVAL; 1843 goto fail3; 1844 } 1845 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1846 1847 #ifdef CONFIG_SFC_SRIOV 1848 rc = efx->type->vswitching_probe(efx); 1849 if (rc) /* not fatal; the PF will still work fine */ 1850 netif_warn(efx, probe, efx->net_dev, 1851 "failed to setup vswitching rc=%d;" 1852 " VFs may not function\n", rc); 1853 #endif 1854 1855 rc = efx_probe_filters(efx); 1856 if (rc) { 1857 netif_err(efx, probe, efx->net_dev, 1858 "failed to create filter tables\n"); 1859 goto fail4; 1860 } 1861 1862 rc = efx_probe_channels(efx); 1863 if (rc) 1864 goto fail5; 1865 1866 return 0; 1867 1868 fail5: 1869 efx_remove_filters(efx); 1870 fail4: 1871 #ifdef CONFIG_SFC_SRIOV 1872 efx->type->vswitching_remove(efx); 1873 #endif 1874 fail3: 1875 efx_remove_port(efx); 1876 fail2: 1877 efx_remove_nic(efx); 1878 fail1: 1879 return rc; 1880 } 1881 1882 /* If the interface is supposed to be running but is not, start 1883 * the hardware and software data path, regular activity for the port 1884 * (MAC statistics, link polling, etc.) and schedule the port to be 1885 * reconfigured. Interrupts must already be enabled. This function 1886 * is safe to call multiple times, so long as the NIC is not disabled. 1887 * Requires the RTNL lock. 1888 */ 1889 static void efx_start_all(struct efx_nic *efx) 1890 { 1891 EFX_ASSERT_RESET_SERIALISED(efx); 1892 BUG_ON(efx->state == STATE_DISABLED); 1893 1894 /* Check that it is appropriate to restart the interface. All 1895 * of these flags are safe to read under just the rtnl lock */ 1896 if (efx->port_enabled || !netif_running(efx->net_dev) || 1897 efx->reset_pending) 1898 return; 1899 1900 efx_start_port(efx); 1901 efx_start_datapath(efx); 1902 1903 /* Start the hardware monitor if there is one */ 1904 if (efx->type->monitor != NULL) 1905 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1906 efx_monitor_interval); 1907 1908 /* Link state detection is normally event-driven; we have 1909 * to poll now because we could have missed a change 1910 */ 1911 mutex_lock(&efx->mac_lock); 1912 if (efx->phy_op->poll(efx)) 1913 efx_link_status_changed(efx); 1914 mutex_unlock(&efx->mac_lock); 1915 1916 efx->type->start_stats(efx); 1917 efx->type->pull_stats(efx); 1918 spin_lock_bh(&efx->stats_lock); 1919 efx->type->update_stats(efx, NULL, NULL); 1920 spin_unlock_bh(&efx->stats_lock); 1921 } 1922 1923 /* Quiesce the hardware and software data path, and regular activity 1924 * for the port without bringing the link down. Safe to call multiple 1925 * times with the NIC in almost any state, but interrupts should be 1926 * enabled. Requires the RTNL lock. 1927 */ 1928 static void efx_stop_all(struct efx_nic *efx) 1929 { 1930 EFX_ASSERT_RESET_SERIALISED(efx); 1931 1932 /* port_enabled can be read safely under the rtnl lock */ 1933 if (!efx->port_enabled) 1934 return; 1935 1936 /* update stats before we go down so we can accurately count 1937 * rx_nodesc_drops 1938 */ 1939 efx->type->pull_stats(efx); 1940 spin_lock_bh(&efx->stats_lock); 1941 efx->type->update_stats(efx, NULL, NULL); 1942 spin_unlock_bh(&efx->stats_lock); 1943 efx->type->stop_stats(efx); 1944 efx_stop_port(efx); 1945 1946 /* Stop the kernel transmit interface. This is only valid if 1947 * the device is stopped or detached; otherwise the watchdog 1948 * may fire immediately. 1949 */ 1950 WARN_ON(netif_running(efx->net_dev) && 1951 netif_device_present(efx->net_dev)); 1952 netif_tx_disable(efx->net_dev); 1953 1954 efx_stop_datapath(efx); 1955 } 1956 1957 static void efx_remove_all(struct efx_nic *efx) 1958 { 1959 efx_remove_channels(efx); 1960 efx_remove_filters(efx); 1961 #ifdef CONFIG_SFC_SRIOV 1962 efx->type->vswitching_remove(efx); 1963 #endif 1964 efx_remove_port(efx); 1965 efx_remove_nic(efx); 1966 } 1967 1968 /************************************************************************** 1969 * 1970 * Interrupt moderation 1971 * 1972 **************************************************************************/ 1973 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 1974 { 1975 if (usecs == 0) 1976 return 0; 1977 if (usecs * 1000 < efx->timer_quantum_ns) 1978 return 1; /* never round down to 0 */ 1979 return usecs * 1000 / efx->timer_quantum_ns; 1980 } 1981 1982 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 1983 { 1984 /* We must round up when converting ticks to microseconds 1985 * because we round down when converting the other way. 1986 */ 1987 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 1988 } 1989 1990 /* Set interrupt moderation parameters */ 1991 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 1992 unsigned int rx_usecs, bool rx_adaptive, 1993 bool rx_may_override_tx) 1994 { 1995 struct efx_channel *channel; 1996 unsigned int timer_max_us; 1997 1998 EFX_ASSERT_RESET_SERIALISED(efx); 1999 2000 timer_max_us = efx->timer_max_ns / 1000; 2001 2002 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 2003 return -EINVAL; 2004 2005 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 2006 !rx_may_override_tx) { 2007 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 2008 "RX and TX IRQ moderation must be equal\n"); 2009 return -EINVAL; 2010 } 2011 2012 efx->irq_rx_adaptive = rx_adaptive; 2013 efx->irq_rx_moderation_us = rx_usecs; 2014 efx_for_each_channel(channel, efx) { 2015 if (efx_channel_has_rx_queue(channel)) 2016 channel->irq_moderation_us = rx_usecs; 2017 else if (efx_channel_has_tx_queues(channel)) 2018 channel->irq_moderation_us = tx_usecs; 2019 } 2020 2021 return 0; 2022 } 2023 2024 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2025 unsigned int *rx_usecs, bool *rx_adaptive) 2026 { 2027 *rx_adaptive = efx->irq_rx_adaptive; 2028 *rx_usecs = efx->irq_rx_moderation_us; 2029 2030 /* If channels are shared between RX and TX, so is IRQ 2031 * moderation. Otherwise, IRQ moderation is the same for all 2032 * TX channels and is not adaptive. 2033 */ 2034 if (efx->tx_channel_offset == 0) { 2035 *tx_usecs = *rx_usecs; 2036 } else { 2037 struct efx_channel *tx_channel; 2038 2039 tx_channel = efx->channel[efx->tx_channel_offset]; 2040 *tx_usecs = tx_channel->irq_moderation_us; 2041 } 2042 } 2043 2044 /************************************************************************** 2045 * 2046 * Hardware monitor 2047 * 2048 **************************************************************************/ 2049 2050 /* Run periodically off the general workqueue */ 2051 static void efx_monitor(struct work_struct *data) 2052 { 2053 struct efx_nic *efx = container_of(data, struct efx_nic, 2054 monitor_work.work); 2055 2056 netif_vdbg(efx, timer, efx->net_dev, 2057 "hardware monitor executing on CPU %d\n", 2058 raw_smp_processor_id()); 2059 BUG_ON(efx->type->monitor == NULL); 2060 2061 /* If the mac_lock is already held then it is likely a port 2062 * reconfiguration is already in place, which will likely do 2063 * most of the work of monitor() anyway. */ 2064 if (mutex_trylock(&efx->mac_lock)) { 2065 if (efx->port_enabled) 2066 efx->type->monitor(efx); 2067 mutex_unlock(&efx->mac_lock); 2068 } 2069 2070 queue_delayed_work(efx->workqueue, &efx->monitor_work, 2071 efx_monitor_interval); 2072 } 2073 2074 /************************************************************************** 2075 * 2076 * ioctls 2077 * 2078 *************************************************************************/ 2079 2080 /* Net device ioctl 2081 * Context: process, rtnl_lock() held. 2082 */ 2083 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2084 { 2085 struct efx_nic *efx = netdev_priv(net_dev); 2086 struct mii_ioctl_data *data = if_mii(ifr); 2087 2088 if (cmd == SIOCSHWTSTAMP) 2089 return efx_ptp_set_ts_config(efx, ifr); 2090 if (cmd == SIOCGHWTSTAMP) 2091 return efx_ptp_get_ts_config(efx, ifr); 2092 2093 /* Convert phy_id from older PRTAD/DEVAD format */ 2094 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2095 (data->phy_id & 0xfc00) == 0x0400) 2096 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2097 2098 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2099 } 2100 2101 /************************************************************************** 2102 * 2103 * NAPI interface 2104 * 2105 **************************************************************************/ 2106 2107 static void efx_init_napi_channel(struct efx_channel *channel) 2108 { 2109 struct efx_nic *efx = channel->efx; 2110 2111 channel->napi_dev = efx->net_dev; 2112 netif_napi_add(channel->napi_dev, &channel->napi_str, 2113 efx_poll, napi_weight); 2114 } 2115 2116 static void efx_init_napi(struct efx_nic *efx) 2117 { 2118 struct efx_channel *channel; 2119 2120 efx_for_each_channel(channel, efx) 2121 efx_init_napi_channel(channel); 2122 } 2123 2124 static void efx_fini_napi_channel(struct efx_channel *channel) 2125 { 2126 if (channel->napi_dev) 2127 netif_napi_del(&channel->napi_str); 2128 2129 channel->napi_dev = NULL; 2130 } 2131 2132 static void efx_fini_napi(struct efx_nic *efx) 2133 { 2134 struct efx_channel *channel; 2135 2136 efx_for_each_channel(channel, efx) 2137 efx_fini_napi_channel(channel); 2138 } 2139 2140 /************************************************************************** 2141 * 2142 * Kernel netpoll interface 2143 * 2144 *************************************************************************/ 2145 2146 #ifdef CONFIG_NET_POLL_CONTROLLER 2147 2148 /* Although in the common case interrupts will be disabled, this is not 2149 * guaranteed. However, all our work happens inside the NAPI callback, 2150 * so no locking is required. 2151 */ 2152 static void efx_netpoll(struct net_device *net_dev) 2153 { 2154 struct efx_nic *efx = netdev_priv(net_dev); 2155 struct efx_channel *channel; 2156 2157 efx_for_each_channel(channel, efx) 2158 efx_schedule_channel(channel); 2159 } 2160 2161 #endif 2162 2163 /************************************************************************** 2164 * 2165 * Kernel net device interface 2166 * 2167 *************************************************************************/ 2168 2169 /* Context: process, rtnl_lock() held. */ 2170 int efx_net_open(struct net_device *net_dev) 2171 { 2172 struct efx_nic *efx = netdev_priv(net_dev); 2173 int rc; 2174 2175 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2176 raw_smp_processor_id()); 2177 2178 rc = efx_check_disabled(efx); 2179 if (rc) 2180 return rc; 2181 if (efx->phy_mode & PHY_MODE_SPECIAL) 2182 return -EBUSY; 2183 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2184 return -EIO; 2185 2186 /* Notify the kernel of the link state polled during driver load, 2187 * before the monitor starts running */ 2188 efx_link_status_changed(efx); 2189 2190 efx_start_all(efx); 2191 if (efx->state == STATE_DISABLED || efx->reset_pending) 2192 netif_device_detach(efx->net_dev); 2193 efx_selftest_async_start(efx); 2194 return 0; 2195 } 2196 2197 /* Context: process, rtnl_lock() held. 2198 * Note that the kernel will ignore our return code; this method 2199 * should really be a void. 2200 */ 2201 int efx_net_stop(struct net_device *net_dev) 2202 { 2203 struct efx_nic *efx = netdev_priv(net_dev); 2204 2205 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2206 raw_smp_processor_id()); 2207 2208 /* Stop the device and flush all the channels */ 2209 efx_stop_all(efx); 2210 2211 return 0; 2212 } 2213 2214 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2215 static void efx_net_stats(struct net_device *net_dev, 2216 struct rtnl_link_stats64 *stats) 2217 { 2218 struct efx_nic *efx = netdev_priv(net_dev); 2219 2220 spin_lock_bh(&efx->stats_lock); 2221 efx->type->update_stats(efx, NULL, stats); 2222 spin_unlock_bh(&efx->stats_lock); 2223 } 2224 2225 /* Context: netif_tx_lock held, BHs disabled. */ 2226 static void efx_watchdog(struct net_device *net_dev) 2227 { 2228 struct efx_nic *efx = netdev_priv(net_dev); 2229 2230 netif_err(efx, tx_err, efx->net_dev, 2231 "TX stuck with port_enabled=%d: resetting channels\n", 2232 efx->port_enabled); 2233 2234 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2235 } 2236 2237 2238 /* Context: process, rtnl_lock() held. */ 2239 static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2240 { 2241 struct efx_nic *efx = netdev_priv(net_dev); 2242 int rc; 2243 2244 rc = efx_check_disabled(efx); 2245 if (rc) 2246 return rc; 2247 2248 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2249 2250 efx_device_detach_sync(efx); 2251 efx_stop_all(efx); 2252 2253 mutex_lock(&efx->mac_lock); 2254 net_dev->mtu = new_mtu; 2255 efx_mac_reconfigure(efx); 2256 mutex_unlock(&efx->mac_lock); 2257 2258 efx_start_all(efx); 2259 efx_device_attach_if_not_resetting(efx); 2260 return 0; 2261 } 2262 2263 static int efx_set_mac_address(struct net_device *net_dev, void *data) 2264 { 2265 struct efx_nic *efx = netdev_priv(net_dev); 2266 struct sockaddr *addr = data; 2267 u8 *new_addr = addr->sa_data; 2268 u8 old_addr[6]; 2269 int rc; 2270 2271 if (!is_valid_ether_addr(new_addr)) { 2272 netif_err(efx, drv, efx->net_dev, 2273 "invalid ethernet MAC address requested: %pM\n", 2274 new_addr); 2275 return -EADDRNOTAVAIL; 2276 } 2277 2278 /* save old address */ 2279 ether_addr_copy(old_addr, net_dev->dev_addr); 2280 ether_addr_copy(net_dev->dev_addr, new_addr); 2281 if (efx->type->set_mac_address) { 2282 rc = efx->type->set_mac_address(efx); 2283 if (rc) { 2284 ether_addr_copy(net_dev->dev_addr, old_addr); 2285 return rc; 2286 } 2287 } 2288 2289 /* Reconfigure the MAC */ 2290 mutex_lock(&efx->mac_lock); 2291 efx_mac_reconfigure(efx); 2292 mutex_unlock(&efx->mac_lock); 2293 2294 return 0; 2295 } 2296 2297 /* Context: netif_addr_lock held, BHs disabled. */ 2298 static void efx_set_rx_mode(struct net_device *net_dev) 2299 { 2300 struct efx_nic *efx = netdev_priv(net_dev); 2301 2302 if (efx->port_enabled) 2303 queue_work(efx->workqueue, &efx->mac_work); 2304 /* Otherwise efx_start_port() will do this */ 2305 } 2306 2307 static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2308 { 2309 struct efx_nic *efx = netdev_priv(net_dev); 2310 int rc; 2311 2312 /* If disabling RX n-tuple filtering, clear existing filters */ 2313 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2314 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2315 if (rc) 2316 return rc; 2317 } 2318 2319 /* If Rx VLAN filter is changed, update filters via mac_reconfigure. 2320 * If rx-fcs is changed, mac_reconfigure updates that too. 2321 */ 2322 if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER | 2323 NETIF_F_RXFCS)) { 2324 /* efx_set_rx_mode() will schedule MAC work to update filters 2325 * when a new features are finally set in net_dev. 2326 */ 2327 efx_set_rx_mode(net_dev); 2328 } 2329 2330 return 0; 2331 } 2332 2333 static int efx_get_phys_port_id(struct net_device *net_dev, 2334 struct netdev_phys_item_id *ppid) 2335 { 2336 struct efx_nic *efx = netdev_priv(net_dev); 2337 2338 if (efx->type->get_phys_port_id) 2339 return efx->type->get_phys_port_id(efx, ppid); 2340 else 2341 return -EOPNOTSUPP; 2342 } 2343 2344 static int efx_get_phys_port_name(struct net_device *net_dev, 2345 char *name, size_t len) 2346 { 2347 struct efx_nic *efx = netdev_priv(net_dev); 2348 2349 if (snprintf(name, len, "p%u", efx->port_num) >= len) 2350 return -EINVAL; 2351 return 0; 2352 } 2353 2354 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2355 { 2356 struct efx_nic *efx = netdev_priv(net_dev); 2357 2358 if (efx->type->vlan_rx_add_vid) 2359 return efx->type->vlan_rx_add_vid(efx, proto, vid); 2360 else 2361 return -EOPNOTSUPP; 2362 } 2363 2364 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2365 { 2366 struct efx_nic *efx = netdev_priv(net_dev); 2367 2368 if (efx->type->vlan_rx_kill_vid) 2369 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 2370 else 2371 return -EOPNOTSUPP; 2372 } 2373 2374 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in) 2375 { 2376 switch (in) { 2377 case UDP_TUNNEL_TYPE_VXLAN: 2378 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 2379 case UDP_TUNNEL_TYPE_GENEVE: 2380 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 2381 default: 2382 return -1; 2383 } 2384 } 2385 2386 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti) 2387 { 2388 struct efx_nic *efx = netdev_priv(dev); 2389 struct efx_udp_tunnel tnl; 2390 int efx_tunnel_type; 2391 2392 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2393 if (efx_tunnel_type < 0) 2394 return; 2395 2396 tnl.type = (u16)efx_tunnel_type; 2397 tnl.port = ti->port; 2398 2399 if (efx->type->udp_tnl_add_port) 2400 (void)efx->type->udp_tnl_add_port(efx, tnl); 2401 } 2402 2403 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti) 2404 { 2405 struct efx_nic *efx = netdev_priv(dev); 2406 struct efx_udp_tunnel tnl; 2407 int efx_tunnel_type; 2408 2409 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2410 if (efx_tunnel_type < 0) 2411 return; 2412 2413 tnl.type = (u16)efx_tunnel_type; 2414 tnl.port = ti->port; 2415 2416 if (efx->type->udp_tnl_del_port) 2417 (void)efx->type->udp_tnl_del_port(efx, tnl); 2418 } 2419 2420 static const struct net_device_ops efx_netdev_ops = { 2421 .ndo_open = efx_net_open, 2422 .ndo_stop = efx_net_stop, 2423 .ndo_get_stats64 = efx_net_stats, 2424 .ndo_tx_timeout = efx_watchdog, 2425 .ndo_start_xmit = efx_hard_start_xmit, 2426 .ndo_validate_addr = eth_validate_addr, 2427 .ndo_do_ioctl = efx_ioctl, 2428 .ndo_change_mtu = efx_change_mtu, 2429 .ndo_set_mac_address = efx_set_mac_address, 2430 .ndo_set_rx_mode = efx_set_rx_mode, 2431 .ndo_set_features = efx_set_features, 2432 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 2433 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2434 #ifdef CONFIG_SFC_SRIOV 2435 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2436 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2437 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2438 .ndo_get_vf_config = efx_sriov_get_vf_config, 2439 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2440 #endif 2441 .ndo_get_phys_port_id = efx_get_phys_port_id, 2442 .ndo_get_phys_port_name = efx_get_phys_port_name, 2443 #ifdef CONFIG_NET_POLL_CONTROLLER 2444 .ndo_poll_controller = efx_netpoll, 2445 #endif 2446 .ndo_setup_tc = efx_setup_tc, 2447 #ifdef CONFIG_RFS_ACCEL 2448 .ndo_rx_flow_steer = efx_filter_rfs, 2449 #endif 2450 .ndo_udp_tunnel_add = efx_udp_tunnel_add, 2451 .ndo_udp_tunnel_del = efx_udp_tunnel_del, 2452 }; 2453 2454 static void efx_update_name(struct efx_nic *efx) 2455 { 2456 strcpy(efx->name, efx->net_dev->name); 2457 efx_mtd_rename(efx); 2458 efx_set_channel_names(efx); 2459 } 2460 2461 static int efx_netdev_event(struct notifier_block *this, 2462 unsigned long event, void *ptr) 2463 { 2464 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2465 2466 if ((net_dev->netdev_ops == &efx_netdev_ops) && 2467 event == NETDEV_CHANGENAME) 2468 efx_update_name(netdev_priv(net_dev)); 2469 2470 return NOTIFY_DONE; 2471 } 2472 2473 static struct notifier_block efx_netdev_notifier = { 2474 .notifier_call = efx_netdev_event, 2475 }; 2476 2477 static ssize_t 2478 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2479 { 2480 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2481 return sprintf(buf, "%d\n", efx->phy_type); 2482 } 2483 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2484 2485 #ifdef CONFIG_SFC_MCDI_LOGGING 2486 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2487 char *buf) 2488 { 2489 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2490 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2491 2492 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2493 } 2494 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2495 const char *buf, size_t count) 2496 { 2497 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2498 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2499 bool enable = count > 0 && *buf != '0'; 2500 2501 mcdi->logging_enabled = enable; 2502 return count; 2503 } 2504 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2505 #endif 2506 2507 static int efx_register_netdev(struct efx_nic *efx) 2508 { 2509 struct net_device *net_dev = efx->net_dev; 2510 struct efx_channel *channel; 2511 int rc; 2512 2513 net_dev->watchdog_timeo = 5 * HZ; 2514 net_dev->irq = efx->pci_dev->irq; 2515 net_dev->netdev_ops = &efx_netdev_ops; 2516 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 2517 net_dev->priv_flags |= IFF_UNICAST_FLT; 2518 net_dev->ethtool_ops = &efx_ethtool_ops; 2519 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2520 net_dev->min_mtu = EFX_MIN_MTU; 2521 net_dev->max_mtu = EFX_MAX_MTU; 2522 2523 rtnl_lock(); 2524 2525 /* Enable resets to be scheduled and check whether any were 2526 * already requested. If so, the NIC is probably hosed so we 2527 * abort. 2528 */ 2529 efx->state = STATE_READY; 2530 smp_mb(); /* ensure we change state before checking reset_pending */ 2531 if (efx->reset_pending) { 2532 netif_err(efx, probe, efx->net_dev, 2533 "aborting probe due to scheduled reset\n"); 2534 rc = -EIO; 2535 goto fail_locked; 2536 } 2537 2538 rc = dev_alloc_name(net_dev, net_dev->name); 2539 if (rc < 0) 2540 goto fail_locked; 2541 efx_update_name(efx); 2542 2543 /* Always start with carrier off; PHY events will detect the link */ 2544 netif_carrier_off(net_dev); 2545 2546 rc = register_netdevice(net_dev); 2547 if (rc) 2548 goto fail_locked; 2549 2550 efx_for_each_channel(channel, efx) { 2551 struct efx_tx_queue *tx_queue; 2552 efx_for_each_channel_tx_queue(tx_queue, channel) 2553 efx_init_tx_queue_core_txq(tx_queue); 2554 } 2555 2556 efx_associate(efx); 2557 2558 rtnl_unlock(); 2559 2560 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2561 if (rc) { 2562 netif_err(efx, drv, efx->net_dev, 2563 "failed to init net dev attributes\n"); 2564 goto fail_registered; 2565 } 2566 #ifdef CONFIG_SFC_MCDI_LOGGING 2567 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2568 if (rc) { 2569 netif_err(efx, drv, efx->net_dev, 2570 "failed to init net dev attributes\n"); 2571 goto fail_attr_mcdi_logging; 2572 } 2573 #endif 2574 2575 return 0; 2576 2577 #ifdef CONFIG_SFC_MCDI_LOGGING 2578 fail_attr_mcdi_logging: 2579 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2580 #endif 2581 fail_registered: 2582 rtnl_lock(); 2583 efx_dissociate(efx); 2584 unregister_netdevice(net_dev); 2585 fail_locked: 2586 efx->state = STATE_UNINIT; 2587 rtnl_unlock(); 2588 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2589 return rc; 2590 } 2591 2592 static void efx_unregister_netdev(struct efx_nic *efx) 2593 { 2594 if (!efx->net_dev) 2595 return; 2596 2597 BUG_ON(netdev_priv(efx->net_dev) != efx); 2598 2599 if (efx_dev_registered(efx)) { 2600 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2601 #ifdef CONFIG_SFC_MCDI_LOGGING 2602 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2603 #endif 2604 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2605 unregister_netdev(efx->net_dev); 2606 } 2607 } 2608 2609 /************************************************************************** 2610 * 2611 * Device reset and suspend 2612 * 2613 **************************************************************************/ 2614 2615 /* Tears down the entire software state and most of the hardware state 2616 * before reset. */ 2617 void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2618 { 2619 EFX_ASSERT_RESET_SERIALISED(efx); 2620 2621 if (method == RESET_TYPE_MCDI_TIMEOUT) 2622 efx->type->prepare_flr(efx); 2623 2624 efx_stop_all(efx); 2625 efx_disable_interrupts(efx); 2626 2627 mutex_lock(&efx->mac_lock); 2628 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2629 method != RESET_TYPE_DATAPATH) 2630 efx->phy_op->fini(efx); 2631 efx->type->fini(efx); 2632 } 2633 2634 /* This function will always ensure that the locks acquired in 2635 * efx_reset_down() are released. A failure return code indicates 2636 * that we were unable to reinitialise the hardware, and the 2637 * driver should be disabled. If ok is false, then the rx and tx 2638 * engines are not restarted, pending a RESET_DISABLE. */ 2639 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2640 { 2641 int rc; 2642 2643 EFX_ASSERT_RESET_SERIALISED(efx); 2644 2645 if (method == RESET_TYPE_MCDI_TIMEOUT) 2646 efx->type->finish_flr(efx); 2647 2648 /* Ensure that SRAM is initialised even if we're disabling the device */ 2649 rc = efx->type->init(efx); 2650 if (rc) { 2651 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2652 goto fail; 2653 } 2654 2655 if (!ok) 2656 goto fail; 2657 2658 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2659 method != RESET_TYPE_DATAPATH) { 2660 rc = efx->phy_op->init(efx); 2661 if (rc) 2662 goto fail; 2663 rc = efx->phy_op->reconfigure(efx); 2664 if (rc && rc != -EPERM) 2665 netif_err(efx, drv, efx->net_dev, 2666 "could not restore PHY settings\n"); 2667 } 2668 2669 rc = efx_enable_interrupts(efx); 2670 if (rc) 2671 goto fail; 2672 2673 #ifdef CONFIG_SFC_SRIOV 2674 rc = efx->type->vswitching_restore(efx); 2675 if (rc) /* not fatal; the PF will still work fine */ 2676 netif_warn(efx, probe, efx->net_dev, 2677 "failed to restore vswitching rc=%d;" 2678 " VFs may not function\n", rc); 2679 #endif 2680 2681 down_read(&efx->filter_sem); 2682 efx_restore_filters(efx); 2683 up_read(&efx->filter_sem); 2684 if (efx->type->sriov_reset) 2685 efx->type->sriov_reset(efx); 2686 2687 mutex_unlock(&efx->mac_lock); 2688 2689 efx_start_all(efx); 2690 2691 if (efx->type->udp_tnl_push_ports) 2692 efx->type->udp_tnl_push_ports(efx); 2693 2694 return 0; 2695 2696 fail: 2697 efx->port_initialized = false; 2698 2699 mutex_unlock(&efx->mac_lock); 2700 2701 return rc; 2702 } 2703 2704 /* Reset the NIC using the specified method. Note that the reset may 2705 * fail, in which case the card will be left in an unusable state. 2706 * 2707 * Caller must hold the rtnl_lock. 2708 */ 2709 int efx_reset(struct efx_nic *efx, enum reset_type method) 2710 { 2711 int rc, rc2; 2712 bool disabled; 2713 2714 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2715 RESET_TYPE(method)); 2716 2717 efx_device_detach_sync(efx); 2718 efx_reset_down(efx, method); 2719 2720 rc = efx->type->reset(efx, method); 2721 if (rc) { 2722 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2723 goto out; 2724 } 2725 2726 /* Clear flags for the scopes we covered. We assume the NIC and 2727 * driver are now quiescent so that there is no race here. 2728 */ 2729 if (method < RESET_TYPE_MAX_METHOD) 2730 efx->reset_pending &= -(1 << (method + 1)); 2731 else /* it doesn't fit into the well-ordered scope hierarchy */ 2732 __clear_bit(method, &efx->reset_pending); 2733 2734 /* Reinitialise bus-mastering, which may have been turned off before 2735 * the reset was scheduled. This is still appropriate, even in the 2736 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2737 * can respond to requests. */ 2738 pci_set_master(efx->pci_dev); 2739 2740 out: 2741 /* Leave device stopped if necessary */ 2742 disabled = rc || 2743 method == RESET_TYPE_DISABLE || 2744 method == RESET_TYPE_RECOVER_OR_DISABLE; 2745 rc2 = efx_reset_up(efx, method, !disabled); 2746 if (rc2) { 2747 disabled = true; 2748 if (!rc) 2749 rc = rc2; 2750 } 2751 2752 if (disabled) { 2753 dev_close(efx->net_dev); 2754 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2755 efx->state = STATE_DISABLED; 2756 } else { 2757 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2758 efx_device_attach_if_not_resetting(efx); 2759 } 2760 return rc; 2761 } 2762 2763 /* Try recovery mechanisms. 2764 * For now only EEH is supported. 2765 * Returns 0 if the recovery mechanisms are unsuccessful. 2766 * Returns a non-zero value otherwise. 2767 */ 2768 int efx_try_recovery(struct efx_nic *efx) 2769 { 2770 #ifdef CONFIG_EEH 2771 /* A PCI error can occur and not be seen by EEH because nothing 2772 * happens on the PCI bus. In this case the driver may fail and 2773 * schedule a 'recover or reset', leading to this recovery handler. 2774 * Manually call the eeh failure check function. 2775 */ 2776 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2777 if (eeh_dev_check_failure(eehdev)) { 2778 /* The EEH mechanisms will handle the error and reset the 2779 * device if necessary. 2780 */ 2781 return 1; 2782 } 2783 #endif 2784 return 0; 2785 } 2786 2787 static void efx_wait_for_bist_end(struct efx_nic *efx) 2788 { 2789 int i; 2790 2791 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 2792 if (efx_mcdi_poll_reboot(efx)) 2793 goto out; 2794 msleep(BIST_WAIT_DELAY_MS); 2795 } 2796 2797 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 2798 out: 2799 /* Either way unset the BIST flag. If we found no reboot we probably 2800 * won't recover, but we should try. 2801 */ 2802 efx->mc_bist_for_other_fn = false; 2803 } 2804 2805 /* The worker thread exists so that code that cannot sleep can 2806 * schedule a reset for later. 2807 */ 2808 static void efx_reset_work(struct work_struct *data) 2809 { 2810 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2811 unsigned long pending; 2812 enum reset_type method; 2813 2814 pending = READ_ONCE(efx->reset_pending); 2815 method = fls(pending) - 1; 2816 2817 if (method == RESET_TYPE_MC_BIST) 2818 efx_wait_for_bist_end(efx); 2819 2820 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2821 method == RESET_TYPE_RECOVER_OR_ALL) && 2822 efx_try_recovery(efx)) 2823 return; 2824 2825 if (!pending) 2826 return; 2827 2828 rtnl_lock(); 2829 2830 /* We checked the state in efx_schedule_reset() but it may 2831 * have changed by now. Now that we have the RTNL lock, 2832 * it cannot change again. 2833 */ 2834 if (efx->state == STATE_READY) 2835 (void)efx_reset(efx, method); 2836 2837 rtnl_unlock(); 2838 } 2839 2840 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2841 { 2842 enum reset_type method; 2843 2844 if (efx->state == STATE_RECOVERY) { 2845 netif_dbg(efx, drv, efx->net_dev, 2846 "recovering: skip scheduling %s reset\n", 2847 RESET_TYPE(type)); 2848 return; 2849 } 2850 2851 switch (type) { 2852 case RESET_TYPE_INVISIBLE: 2853 case RESET_TYPE_ALL: 2854 case RESET_TYPE_RECOVER_OR_ALL: 2855 case RESET_TYPE_WORLD: 2856 case RESET_TYPE_DISABLE: 2857 case RESET_TYPE_RECOVER_OR_DISABLE: 2858 case RESET_TYPE_DATAPATH: 2859 case RESET_TYPE_MC_BIST: 2860 case RESET_TYPE_MCDI_TIMEOUT: 2861 method = type; 2862 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2863 RESET_TYPE(method)); 2864 break; 2865 default: 2866 method = efx->type->map_reset_reason(type); 2867 netif_dbg(efx, drv, efx->net_dev, 2868 "scheduling %s reset for %s\n", 2869 RESET_TYPE(method), RESET_TYPE(type)); 2870 break; 2871 } 2872 2873 set_bit(method, &efx->reset_pending); 2874 smp_mb(); /* ensure we change reset_pending before checking state */ 2875 2876 /* If we're not READY then just leave the flags set as the cue 2877 * to abort probing or reschedule the reset later. 2878 */ 2879 if (READ_ONCE(efx->state) != STATE_READY) 2880 return; 2881 2882 /* efx_process_channel() will no longer read events once a 2883 * reset is scheduled. So switch back to poll'd MCDI completions. */ 2884 efx_mcdi_mode_poll(efx); 2885 2886 queue_work(reset_workqueue, &efx->reset_work); 2887 } 2888 2889 /************************************************************************** 2890 * 2891 * List of NICs we support 2892 * 2893 **************************************************************************/ 2894 2895 /* PCI device ID table */ 2896 static const struct pci_device_id efx_pci_table[] = { 2897 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2898 .driver_data = (unsigned long) &siena_a0_nic_type}, 2899 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2900 .driver_data = (unsigned long) &siena_a0_nic_type}, 2901 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 2902 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2903 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 2904 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2905 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 2906 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2907 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2908 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2909 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2910 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2911 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2912 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2913 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 2914 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2915 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 2916 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2917 {0} /* end of list */ 2918 }; 2919 2920 /************************************************************************** 2921 * 2922 * Dummy PHY/MAC operations 2923 * 2924 * Can be used for some unimplemented operations 2925 * Needed so all function pointers are valid and do not have to be tested 2926 * before use 2927 * 2928 **************************************************************************/ 2929 int efx_port_dummy_op_int(struct efx_nic *efx) 2930 { 2931 return 0; 2932 } 2933 void efx_port_dummy_op_void(struct efx_nic *efx) {} 2934 2935 static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2936 { 2937 return false; 2938 } 2939 2940 static const struct efx_phy_operations efx_dummy_phy_operations = { 2941 .init = efx_port_dummy_op_int, 2942 .reconfigure = efx_port_dummy_op_int, 2943 .poll = efx_port_dummy_op_poll, 2944 .fini = efx_port_dummy_op_void, 2945 }; 2946 2947 /************************************************************************** 2948 * 2949 * Data housekeeping 2950 * 2951 **************************************************************************/ 2952 2953 /* This zeroes out and then fills in the invariants in a struct 2954 * efx_nic (including all sub-structures). 2955 */ 2956 static int efx_init_struct(struct efx_nic *efx, 2957 struct pci_dev *pci_dev, struct net_device *net_dev) 2958 { 2959 int rc = -ENOMEM, i; 2960 2961 /* Initialise common structures */ 2962 INIT_LIST_HEAD(&efx->node); 2963 INIT_LIST_HEAD(&efx->secondary_list); 2964 spin_lock_init(&efx->biu_lock); 2965 #ifdef CONFIG_SFC_MTD 2966 INIT_LIST_HEAD(&efx->mtd_list); 2967 #endif 2968 INIT_WORK(&efx->reset_work, efx_reset_work); 2969 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 2970 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 2971 efx->pci_dev = pci_dev; 2972 efx->msg_enable = debug; 2973 efx->state = STATE_UNINIT; 2974 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2975 2976 efx->net_dev = net_dev; 2977 efx->rx_prefix_size = efx->type->rx_prefix_size; 2978 efx->rx_ip_align = 2979 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 2980 efx->rx_packet_hash_offset = 2981 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 2982 efx->rx_packet_ts_offset = 2983 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 2984 spin_lock_init(&efx->stats_lock); 2985 efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 2986 efx->num_mac_stats = MC_CMD_MAC_NSTATS; 2987 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 2988 mutex_init(&efx->mac_lock); 2989 efx->phy_op = &efx_dummy_phy_operations; 2990 efx->mdio.dev = net_dev; 2991 INIT_WORK(&efx->mac_work, efx_mac_work); 2992 init_waitqueue_head(&efx->flush_wq); 2993 2994 for (i = 0; i < EFX_MAX_CHANNELS; i++) { 2995 efx->channel[i] = efx_alloc_channel(efx, i, NULL); 2996 if (!efx->channel[i]) 2997 goto fail; 2998 efx->msi_context[i].efx = efx; 2999 efx->msi_context[i].index = i; 3000 } 3001 3002 /* Higher numbered interrupt modes are less capable! */ 3003 if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 3004 efx->type->min_interrupt_mode)) { 3005 rc = -EIO; 3006 goto fail; 3007 } 3008 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 3009 interrupt_mode); 3010 efx->interrupt_mode = min(efx->type->min_interrupt_mode, 3011 interrupt_mode); 3012 3013 /* Would be good to use the net_dev name, but we're too early */ 3014 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 3015 pci_name(pci_dev)); 3016 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 3017 if (!efx->workqueue) 3018 goto fail; 3019 3020 return 0; 3021 3022 fail: 3023 efx_fini_struct(efx); 3024 return rc; 3025 } 3026 3027 static void efx_fini_struct(struct efx_nic *efx) 3028 { 3029 int i; 3030 3031 for (i = 0; i < EFX_MAX_CHANNELS; i++) 3032 kfree(efx->channel[i]); 3033 3034 kfree(efx->vpd_sn); 3035 3036 if (efx->workqueue) { 3037 destroy_workqueue(efx->workqueue); 3038 efx->workqueue = NULL; 3039 } 3040 } 3041 3042 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 3043 { 3044 u64 n_rx_nodesc_trunc = 0; 3045 struct efx_channel *channel; 3046 3047 efx_for_each_channel(channel, efx) 3048 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 3049 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 3050 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 3051 } 3052 3053 /************************************************************************** 3054 * 3055 * PCI interface 3056 * 3057 **************************************************************************/ 3058 3059 /* Main body of final NIC shutdown code 3060 * This is called only at module unload (or hotplug removal). 3061 */ 3062 static void efx_pci_remove_main(struct efx_nic *efx) 3063 { 3064 /* Flush reset_work. It can no longer be scheduled since we 3065 * are not READY. 3066 */ 3067 BUG_ON(efx->state == STATE_READY); 3068 cancel_work_sync(&efx->reset_work); 3069 3070 efx_disable_interrupts(efx); 3071 efx_nic_fini_interrupt(efx); 3072 efx_fini_port(efx); 3073 efx->type->fini(efx); 3074 efx_fini_napi(efx); 3075 efx_remove_all(efx); 3076 } 3077 3078 /* Final NIC shutdown 3079 * This is called only at module unload (or hotplug removal). A PF can call 3080 * this on its VFs to ensure they are unbound first. 3081 */ 3082 static void efx_pci_remove(struct pci_dev *pci_dev) 3083 { 3084 struct efx_nic *efx; 3085 3086 efx = pci_get_drvdata(pci_dev); 3087 if (!efx) 3088 return; 3089 3090 /* Mark the NIC as fini, then stop the interface */ 3091 rtnl_lock(); 3092 efx_dissociate(efx); 3093 dev_close(efx->net_dev); 3094 efx_disable_interrupts(efx); 3095 efx->state = STATE_UNINIT; 3096 rtnl_unlock(); 3097 3098 if (efx->type->sriov_fini) 3099 efx->type->sriov_fini(efx); 3100 3101 efx_unregister_netdev(efx); 3102 3103 efx_mtd_remove(efx); 3104 3105 efx_pci_remove_main(efx); 3106 3107 efx_fini_io(efx); 3108 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3109 3110 efx_fini_struct(efx); 3111 free_netdev(efx->net_dev); 3112 3113 pci_disable_pcie_error_reporting(pci_dev); 3114 }; 3115 3116 /* NIC VPD information 3117 * Called during probe to display the part number of the 3118 * installed NIC. VPD is potentially very large but this should 3119 * always appear within the first 512 bytes. 3120 */ 3121 #define SFC_VPD_LEN 512 3122 static void efx_probe_vpd_strings(struct efx_nic *efx) 3123 { 3124 struct pci_dev *dev = efx->pci_dev; 3125 char vpd_data[SFC_VPD_LEN]; 3126 ssize_t vpd_size; 3127 int ro_start, ro_size, i, j; 3128 3129 /* Get the vpd data from the device */ 3130 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3131 if (vpd_size <= 0) { 3132 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3133 return; 3134 } 3135 3136 /* Get the Read only section */ 3137 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3138 if (ro_start < 0) { 3139 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3140 return; 3141 } 3142 3143 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3144 j = ro_size; 3145 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3146 if (i + j > vpd_size) 3147 j = vpd_size - i; 3148 3149 /* Get the Part number */ 3150 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3151 if (i < 0) { 3152 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3153 return; 3154 } 3155 3156 j = pci_vpd_info_field_size(&vpd_data[i]); 3157 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3158 if (i + j > vpd_size) { 3159 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3160 return; 3161 } 3162 3163 netif_info(efx, drv, efx->net_dev, 3164 "Part Number : %.*s\n", j, &vpd_data[i]); 3165 3166 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3167 j = ro_size; 3168 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3169 if (i < 0) { 3170 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3171 return; 3172 } 3173 3174 j = pci_vpd_info_field_size(&vpd_data[i]); 3175 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3176 if (i + j > vpd_size) { 3177 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3178 return; 3179 } 3180 3181 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3182 if (!efx->vpd_sn) 3183 return; 3184 3185 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3186 } 3187 3188 3189 /* Main body of NIC initialisation 3190 * This is called at module load (or hotplug insertion, theoretically). 3191 */ 3192 static int efx_pci_probe_main(struct efx_nic *efx) 3193 { 3194 int rc; 3195 3196 /* Do start-of-day initialisation */ 3197 rc = efx_probe_all(efx); 3198 if (rc) 3199 goto fail1; 3200 3201 efx_init_napi(efx); 3202 3203 rc = efx->type->init(efx); 3204 if (rc) { 3205 netif_err(efx, probe, efx->net_dev, 3206 "failed to initialise NIC\n"); 3207 goto fail3; 3208 } 3209 3210 rc = efx_init_port(efx); 3211 if (rc) { 3212 netif_err(efx, probe, efx->net_dev, 3213 "failed to initialise port\n"); 3214 goto fail4; 3215 } 3216 3217 rc = efx_nic_init_interrupt(efx); 3218 if (rc) 3219 goto fail5; 3220 rc = efx_enable_interrupts(efx); 3221 if (rc) 3222 goto fail6; 3223 3224 return 0; 3225 3226 fail6: 3227 efx_nic_fini_interrupt(efx); 3228 fail5: 3229 efx_fini_port(efx); 3230 fail4: 3231 efx->type->fini(efx); 3232 fail3: 3233 efx_fini_napi(efx); 3234 efx_remove_all(efx); 3235 fail1: 3236 return rc; 3237 } 3238 3239 static int efx_pci_probe_post_io(struct efx_nic *efx) 3240 { 3241 struct net_device *net_dev = efx->net_dev; 3242 int rc = efx_pci_probe_main(efx); 3243 3244 if (rc) 3245 return rc; 3246 3247 if (efx->type->sriov_init) { 3248 rc = efx->type->sriov_init(efx); 3249 if (rc) 3250 netif_err(efx, probe, efx->net_dev, 3251 "SR-IOV can't be enabled rc %d\n", rc); 3252 } 3253 3254 /* Determine netdevice features */ 3255 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3256 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL); 3257 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 3258 net_dev->features |= NETIF_F_TSO6; 3259 /* Check whether device supports TSO */ 3260 if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 3261 net_dev->features &= ~NETIF_F_ALL_TSO; 3262 /* Mask for features that also apply to VLAN devices */ 3263 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3264 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 3265 NETIF_F_RXCSUM); 3266 3267 net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 3268 3269 /* Disable receiving frames with bad FCS, by default. */ 3270 net_dev->features &= ~NETIF_F_RXALL; 3271 3272 /* Disable VLAN filtering by default. It may be enforced if 3273 * the feature is fixed (i.e. VLAN filters are required to 3274 * receive VLAN tagged packets due to vPort restrictions). 3275 */ 3276 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3277 net_dev->features |= efx->fixed_features; 3278 3279 rc = efx_register_netdev(efx); 3280 if (!rc) 3281 return 0; 3282 3283 efx_pci_remove_main(efx); 3284 return rc; 3285 } 3286 3287 /* NIC initialisation 3288 * 3289 * This is called at module load (or hotplug insertion, 3290 * theoretically). It sets up PCI mappings, resets the NIC, 3291 * sets up and registers the network devices with the kernel and hooks 3292 * the interrupt service routine. It does not prepare the device for 3293 * transmission; this is left to the first time one of the network 3294 * interfaces is brought up (i.e. efx_net_open). 3295 */ 3296 static int efx_pci_probe(struct pci_dev *pci_dev, 3297 const struct pci_device_id *entry) 3298 { 3299 struct net_device *net_dev; 3300 struct efx_nic *efx; 3301 int rc; 3302 3303 /* Allocate and initialise a struct net_device and struct efx_nic */ 3304 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3305 EFX_MAX_RX_QUEUES); 3306 if (!net_dev) 3307 return -ENOMEM; 3308 efx = netdev_priv(net_dev); 3309 efx->type = (const struct efx_nic_type *) entry->driver_data; 3310 efx->fixed_features |= NETIF_F_HIGHDMA; 3311 3312 pci_set_drvdata(pci_dev, efx); 3313 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3314 rc = efx_init_struct(efx, pci_dev, net_dev); 3315 if (rc) 3316 goto fail1; 3317 3318 netif_info(efx, probe, efx->net_dev, 3319 "Solarflare NIC detected\n"); 3320 3321 if (!efx->type->is_vf) 3322 efx_probe_vpd_strings(efx); 3323 3324 /* Set up basic I/O (BAR mappings etc) */ 3325 rc = efx_init_io(efx); 3326 if (rc) 3327 goto fail2; 3328 3329 rc = efx_pci_probe_post_io(efx); 3330 if (rc) { 3331 /* On failure, retry once immediately. 3332 * If we aborted probe due to a scheduled reset, dismiss it. 3333 */ 3334 efx->reset_pending = 0; 3335 rc = efx_pci_probe_post_io(efx); 3336 if (rc) { 3337 /* On another failure, retry once more 3338 * after a 50-305ms delay. 3339 */ 3340 unsigned char r; 3341 3342 get_random_bytes(&r, 1); 3343 msleep((unsigned int)r + 50); 3344 efx->reset_pending = 0; 3345 rc = efx_pci_probe_post_io(efx); 3346 } 3347 } 3348 if (rc) 3349 goto fail3; 3350 3351 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3352 3353 /* Try to create MTDs, but allow this to fail */ 3354 rtnl_lock(); 3355 rc = efx_mtd_probe(efx); 3356 rtnl_unlock(); 3357 if (rc && rc != -EPERM) 3358 netif_warn(efx, probe, efx->net_dev, 3359 "failed to create MTDs (%d)\n", rc); 3360 3361 rc = pci_enable_pcie_error_reporting(pci_dev); 3362 if (rc && rc != -EINVAL) 3363 netif_notice(efx, probe, efx->net_dev, 3364 "PCIE error reporting unavailable (%d).\n", 3365 rc); 3366 3367 if (efx->type->udp_tnl_push_ports) 3368 efx->type->udp_tnl_push_ports(efx); 3369 3370 return 0; 3371 3372 fail3: 3373 efx_fini_io(efx); 3374 fail2: 3375 efx_fini_struct(efx); 3376 fail1: 3377 WARN_ON(rc > 0); 3378 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3379 free_netdev(net_dev); 3380 return rc; 3381 } 3382 3383 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3384 * enabled on success 3385 */ 3386 #ifdef CONFIG_SFC_SRIOV 3387 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3388 { 3389 int rc; 3390 struct efx_nic *efx = pci_get_drvdata(dev); 3391 3392 if (efx->type->sriov_configure) { 3393 rc = efx->type->sriov_configure(efx, num_vfs); 3394 if (rc) 3395 return rc; 3396 else 3397 return num_vfs; 3398 } else 3399 return -EOPNOTSUPP; 3400 } 3401 #endif 3402 3403 static int efx_pm_freeze(struct device *dev) 3404 { 3405 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3406 3407 rtnl_lock(); 3408 3409 if (efx->state != STATE_DISABLED) { 3410 efx->state = STATE_UNINIT; 3411 3412 efx_device_detach_sync(efx); 3413 3414 efx_stop_all(efx); 3415 efx_disable_interrupts(efx); 3416 } 3417 3418 rtnl_unlock(); 3419 3420 return 0; 3421 } 3422 3423 static int efx_pm_thaw(struct device *dev) 3424 { 3425 int rc; 3426 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3427 3428 rtnl_lock(); 3429 3430 if (efx->state != STATE_DISABLED) { 3431 rc = efx_enable_interrupts(efx); 3432 if (rc) 3433 goto fail; 3434 3435 mutex_lock(&efx->mac_lock); 3436 efx->phy_op->reconfigure(efx); 3437 mutex_unlock(&efx->mac_lock); 3438 3439 efx_start_all(efx); 3440 3441 efx_device_attach_if_not_resetting(efx); 3442 3443 efx->state = STATE_READY; 3444 3445 efx->type->resume_wol(efx); 3446 } 3447 3448 rtnl_unlock(); 3449 3450 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3451 queue_work(reset_workqueue, &efx->reset_work); 3452 3453 return 0; 3454 3455 fail: 3456 rtnl_unlock(); 3457 3458 return rc; 3459 } 3460 3461 static int efx_pm_poweroff(struct device *dev) 3462 { 3463 struct pci_dev *pci_dev = to_pci_dev(dev); 3464 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3465 3466 efx->type->fini(efx); 3467 3468 efx->reset_pending = 0; 3469 3470 pci_save_state(pci_dev); 3471 return pci_set_power_state(pci_dev, PCI_D3hot); 3472 } 3473 3474 /* Used for both resume and restore */ 3475 static int efx_pm_resume(struct device *dev) 3476 { 3477 struct pci_dev *pci_dev = to_pci_dev(dev); 3478 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3479 int rc; 3480 3481 rc = pci_set_power_state(pci_dev, PCI_D0); 3482 if (rc) 3483 return rc; 3484 pci_restore_state(pci_dev); 3485 rc = pci_enable_device(pci_dev); 3486 if (rc) 3487 return rc; 3488 pci_set_master(efx->pci_dev); 3489 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3490 if (rc) 3491 return rc; 3492 rc = efx->type->init(efx); 3493 if (rc) 3494 return rc; 3495 rc = efx_pm_thaw(dev); 3496 return rc; 3497 } 3498 3499 static int efx_pm_suspend(struct device *dev) 3500 { 3501 int rc; 3502 3503 efx_pm_freeze(dev); 3504 rc = efx_pm_poweroff(dev); 3505 if (rc) 3506 efx_pm_resume(dev); 3507 return rc; 3508 } 3509 3510 static const struct dev_pm_ops efx_pm_ops = { 3511 .suspend = efx_pm_suspend, 3512 .resume = efx_pm_resume, 3513 .freeze = efx_pm_freeze, 3514 .thaw = efx_pm_thaw, 3515 .poweroff = efx_pm_poweroff, 3516 .restore = efx_pm_resume, 3517 }; 3518 3519 /* A PCI error affecting this device was detected. 3520 * At this point MMIO and DMA may be disabled. 3521 * Stop the software path and request a slot reset. 3522 */ 3523 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3524 enum pci_channel_state state) 3525 { 3526 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3527 struct efx_nic *efx = pci_get_drvdata(pdev); 3528 3529 if (state == pci_channel_io_perm_failure) 3530 return PCI_ERS_RESULT_DISCONNECT; 3531 3532 rtnl_lock(); 3533 3534 if (efx->state != STATE_DISABLED) { 3535 efx->state = STATE_RECOVERY; 3536 efx->reset_pending = 0; 3537 3538 efx_device_detach_sync(efx); 3539 3540 efx_stop_all(efx); 3541 efx_disable_interrupts(efx); 3542 3543 status = PCI_ERS_RESULT_NEED_RESET; 3544 } else { 3545 /* If the interface is disabled we don't want to do anything 3546 * with it. 3547 */ 3548 status = PCI_ERS_RESULT_RECOVERED; 3549 } 3550 3551 rtnl_unlock(); 3552 3553 pci_disable_device(pdev); 3554 3555 return status; 3556 } 3557 3558 /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3559 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3560 { 3561 struct efx_nic *efx = pci_get_drvdata(pdev); 3562 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3563 int rc; 3564 3565 if (pci_enable_device(pdev)) { 3566 netif_err(efx, hw, efx->net_dev, 3567 "Cannot re-enable PCI device after reset.\n"); 3568 status = PCI_ERS_RESULT_DISCONNECT; 3569 } 3570 3571 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3572 if (rc) { 3573 netif_err(efx, hw, efx->net_dev, 3574 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3575 /* Non-fatal error. Continue. */ 3576 } 3577 3578 return status; 3579 } 3580 3581 /* Perform the actual reset and resume I/O operations. */ 3582 static void efx_io_resume(struct pci_dev *pdev) 3583 { 3584 struct efx_nic *efx = pci_get_drvdata(pdev); 3585 int rc; 3586 3587 rtnl_lock(); 3588 3589 if (efx->state == STATE_DISABLED) 3590 goto out; 3591 3592 rc = efx_reset(efx, RESET_TYPE_ALL); 3593 if (rc) { 3594 netif_err(efx, hw, efx->net_dev, 3595 "efx_reset failed after PCI error (%d)\n", rc); 3596 } else { 3597 efx->state = STATE_READY; 3598 netif_dbg(efx, hw, efx->net_dev, 3599 "Done resetting and resuming IO after PCI error.\n"); 3600 } 3601 3602 out: 3603 rtnl_unlock(); 3604 } 3605 3606 /* For simplicity and reliability, we always require a slot reset and try to 3607 * reset the hardware when a pci error affecting the device is detected. 3608 * We leave both the link_reset and mmio_enabled callback unimplemented: 3609 * with our request for slot reset the mmio_enabled callback will never be 3610 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3611 */ 3612 static const struct pci_error_handlers efx_err_handlers = { 3613 .error_detected = efx_io_error_detected, 3614 .slot_reset = efx_io_slot_reset, 3615 .resume = efx_io_resume, 3616 }; 3617 3618 static struct pci_driver efx_pci_driver = { 3619 .name = KBUILD_MODNAME, 3620 .id_table = efx_pci_table, 3621 .probe = efx_pci_probe, 3622 .remove = efx_pci_remove, 3623 .driver.pm = &efx_pm_ops, 3624 .err_handler = &efx_err_handlers, 3625 #ifdef CONFIG_SFC_SRIOV 3626 .sriov_configure = efx_pci_sriov_configure, 3627 #endif 3628 }; 3629 3630 /************************************************************************** 3631 * 3632 * Kernel module interface 3633 * 3634 *************************************************************************/ 3635 3636 module_param(interrupt_mode, uint, 0444); 3637 MODULE_PARM_DESC(interrupt_mode, 3638 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3639 3640 static int __init efx_init_module(void) 3641 { 3642 int rc; 3643 3644 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3645 3646 rc = register_netdevice_notifier(&efx_netdev_notifier); 3647 if (rc) 3648 goto err_notifier; 3649 3650 #ifdef CONFIG_SFC_SRIOV 3651 rc = efx_init_sriov(); 3652 if (rc) 3653 goto err_sriov; 3654 #endif 3655 3656 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3657 if (!reset_workqueue) { 3658 rc = -ENOMEM; 3659 goto err_reset; 3660 } 3661 3662 rc = pci_register_driver(&efx_pci_driver); 3663 if (rc < 0) 3664 goto err_pci; 3665 3666 return 0; 3667 3668 err_pci: 3669 destroy_workqueue(reset_workqueue); 3670 err_reset: 3671 #ifdef CONFIG_SFC_SRIOV 3672 efx_fini_sriov(); 3673 err_sriov: 3674 #endif 3675 unregister_netdevice_notifier(&efx_netdev_notifier); 3676 err_notifier: 3677 return rc; 3678 } 3679 3680 static void __exit efx_exit_module(void) 3681 { 3682 printk(KERN_INFO "Solarflare NET driver unloading\n"); 3683 3684 pci_unregister_driver(&efx_pci_driver); 3685 destroy_workqueue(reset_workqueue); 3686 #ifdef CONFIG_SFC_SRIOV 3687 efx_fini_sriov(); 3688 #endif 3689 unregister_netdevice_notifier(&efx_netdev_notifier); 3690 3691 } 3692 3693 module_init(efx_init_module); 3694 module_exit(efx_exit_module); 3695 3696 MODULE_AUTHOR("Solarflare Communications and " 3697 "Michael Brown <mbrown@fensystems.co.uk>"); 3698 MODULE_DESCRIPTION("Solarflare network driver"); 3699 MODULE_LICENSE("GPL"); 3700 MODULE_DEVICE_TABLE(pci, efx_pci_table); 3701 MODULE_VERSION(EFX_DRIVER_VERSION); 3702