1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2018 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include "net_driver.h" 12 #include <linux/module.h> 13 #include <linux/netdevice.h> 14 #include "efx_common.h" 15 #include "efx_channels.h" 16 #include "efx.h" 17 #include "mcdi.h" 18 #include "selftest.h" 19 #include "rx_common.h" 20 #include "tx_common.h" 21 #include "nic.h" 22 #include "io.h" 23 #include "mcdi_pcol.h" 24 25 static unsigned int debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 26 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 27 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 28 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 29 module_param(debug, uint, 0); 30 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 31 32 /* This is the time (in jiffies) between invocations of the hardware 33 * monitor. 34 * On Falcon-based NICs, this will: 35 * - Check the on-board hardware monitor; 36 * - Poll the link state and reconfigure the hardware as necessary. 37 * On Siena-based NICs for power systems with EEH support, this will give EEH a 38 * chance to start. 39 */ 40 static unsigned int efx_monitor_interval = 1 * HZ; 41 42 /* How often and how many times to poll for a reset while waiting for a 43 * BIST that another function started to complete. 44 */ 45 #define BIST_WAIT_DELAY_MS 100 46 #define BIST_WAIT_DELAY_COUNT 100 47 48 /* Default stats update time */ 49 #define STATS_PERIOD_MS_DEFAULT 1000 50 51 const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 52 const char *const efx_reset_type_names[] = { 53 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 54 [RESET_TYPE_ALL] = "ALL", 55 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 56 [RESET_TYPE_WORLD] = "WORLD", 57 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 58 [RESET_TYPE_DATAPATH] = "DATAPATH", 59 [RESET_TYPE_MC_BIST] = "MC_BIST", 60 [RESET_TYPE_DISABLE] = "DISABLE", 61 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 62 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 63 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 64 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 65 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 66 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 67 }; 68 69 #define RESET_TYPE(type) \ 70 STRING_TABLE_LOOKUP(type, efx_reset_type) 71 72 /* Loopback mode names (see LOOPBACK_MODE()) */ 73 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 74 const char *const efx_loopback_mode_names[] = { 75 [LOOPBACK_NONE] = "NONE", 76 [LOOPBACK_DATA] = "DATAPATH", 77 [LOOPBACK_GMAC] = "GMAC", 78 [LOOPBACK_XGMII] = "XGMII", 79 [LOOPBACK_XGXS] = "XGXS", 80 [LOOPBACK_XAUI] = "XAUI", 81 [LOOPBACK_GMII] = "GMII", 82 [LOOPBACK_SGMII] = "SGMII", 83 [LOOPBACK_XGBR] = "XGBR", 84 [LOOPBACK_XFI] = "XFI", 85 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 86 [LOOPBACK_GMII_FAR] = "GMII_FAR", 87 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 88 [LOOPBACK_XFI_FAR] = "XFI_FAR", 89 [LOOPBACK_GPHY] = "GPHY", 90 [LOOPBACK_PHYXS] = "PHYXS", 91 [LOOPBACK_PCS] = "PCS", 92 [LOOPBACK_PMAPMD] = "PMA/PMD", 93 [LOOPBACK_XPORT] = "XPORT", 94 [LOOPBACK_XGMII_WS] = "XGMII_WS", 95 [LOOPBACK_XAUI_WS] = "XAUI_WS", 96 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 97 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 98 [LOOPBACK_GMII_WS] = "GMII_WS", 99 [LOOPBACK_XFI_WS] = "XFI_WS", 100 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 101 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 102 }; 103 104 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 105 * queued onto this work queue. This is not a per-nic work queue, because 106 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 107 */ 108 static struct workqueue_struct *reset_workqueue; 109 110 int efx_create_reset_workqueue(void) 111 { 112 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 113 if (!reset_workqueue) { 114 printk(KERN_ERR "Failed to create reset workqueue\n"); 115 return -ENOMEM; 116 } 117 118 return 0; 119 } 120 121 void efx_queue_reset_work(struct efx_nic *efx) 122 { 123 queue_work(reset_workqueue, &efx->reset_work); 124 } 125 126 void efx_flush_reset_workqueue(struct efx_nic *efx) 127 { 128 cancel_work_sync(&efx->reset_work); 129 } 130 131 void efx_destroy_reset_workqueue(void) 132 { 133 if (reset_workqueue) { 134 destroy_workqueue(reset_workqueue); 135 reset_workqueue = NULL; 136 } 137 } 138 139 /* We assume that efx->type->reconfigure_mac will always try to sync RX 140 * filters and therefore needs to read-lock the filter table against freeing 141 */ 142 void efx_mac_reconfigure(struct efx_nic *efx) 143 { 144 if (efx->type->reconfigure_mac) { 145 down_read(&efx->filter_sem); 146 efx->type->reconfigure_mac(efx); 147 up_read(&efx->filter_sem); 148 } 149 } 150 151 /* Asynchronous work item for changing MAC promiscuity and multicast 152 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 153 * MAC directly. 154 */ 155 static void efx_mac_work(struct work_struct *data) 156 { 157 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 158 159 mutex_lock(&efx->mac_lock); 160 if (efx->port_enabled) 161 efx_mac_reconfigure(efx); 162 mutex_unlock(&efx->mac_lock); 163 } 164 165 /* This ensures that the kernel is kept informed (via 166 * netif_carrier_on/off) of the link status, and also maintains the 167 * link status's stop on the port's TX queue. 168 */ 169 void efx_link_status_changed(struct efx_nic *efx) 170 { 171 struct efx_link_state *link_state = &efx->link_state; 172 173 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 174 * that no events are triggered between unregister_netdev() and the 175 * driver unloading. A more general condition is that NETDEV_CHANGE 176 * can only be generated between NETDEV_UP and NETDEV_DOWN 177 */ 178 if (!netif_running(efx->net_dev)) 179 return; 180 181 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 182 efx->n_link_state_changes++; 183 184 if (link_state->up) 185 netif_carrier_on(efx->net_dev); 186 else 187 netif_carrier_off(efx->net_dev); 188 } 189 190 /* Status message for kernel log */ 191 if (link_state->up) 192 netif_info(efx, link, efx->net_dev, 193 "link up at %uMbps %s-duplex (MTU %d)\n", 194 link_state->speed, link_state->fd ? "full" : "half", 195 efx->net_dev->mtu); 196 else 197 netif_info(efx, link, efx->net_dev, "link down\n"); 198 } 199 200 unsigned int efx_xdp_max_mtu(struct efx_nic *efx) 201 { 202 /* The maximum MTU that we can fit in a single page, allowing for 203 * framing, overhead and XDP headroom. 204 */ 205 int overhead = EFX_MAX_FRAME_LEN(0) + sizeof(struct efx_rx_page_state) + 206 efx->rx_prefix_size + efx->type->rx_buffer_padding + 207 efx->rx_ip_align + XDP_PACKET_HEADROOM; 208 209 return PAGE_SIZE - overhead; 210 } 211 212 /* Context: process, rtnl_lock() held. */ 213 int efx_change_mtu(struct net_device *net_dev, int new_mtu) 214 { 215 struct efx_nic *efx = netdev_priv(net_dev); 216 int rc; 217 218 rc = efx_check_disabled(efx); 219 if (rc) 220 return rc; 221 222 if (rtnl_dereference(efx->xdp_prog) && 223 new_mtu > efx_xdp_max_mtu(efx)) { 224 netif_err(efx, drv, efx->net_dev, 225 "Requested MTU of %d too big for XDP (max: %d)\n", 226 new_mtu, efx_xdp_max_mtu(efx)); 227 return -EINVAL; 228 } 229 230 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 231 232 efx_device_detach_sync(efx); 233 efx_stop_all(efx); 234 235 mutex_lock(&efx->mac_lock); 236 net_dev->mtu = new_mtu; 237 efx_mac_reconfigure(efx); 238 mutex_unlock(&efx->mac_lock); 239 240 efx_start_all(efx); 241 efx_device_attach_if_not_resetting(efx); 242 return 0; 243 } 244 245 /************************************************************************** 246 * 247 * Hardware monitor 248 * 249 **************************************************************************/ 250 251 /* Run periodically off the general workqueue */ 252 static void efx_monitor(struct work_struct *data) 253 { 254 struct efx_nic *efx = container_of(data, struct efx_nic, 255 monitor_work.work); 256 257 netif_vdbg(efx, timer, efx->net_dev, 258 "hardware monitor executing on CPU %d\n", 259 raw_smp_processor_id()); 260 BUG_ON(efx->type->monitor == NULL); 261 262 /* If the mac_lock is already held then it is likely a port 263 * reconfiguration is already in place, which will likely do 264 * most of the work of monitor() anyway. 265 */ 266 if (mutex_trylock(&efx->mac_lock)) { 267 if (efx->port_enabled && efx->type->monitor) 268 efx->type->monitor(efx); 269 mutex_unlock(&efx->mac_lock); 270 } 271 272 efx_start_monitor(efx); 273 } 274 275 void efx_start_monitor(struct efx_nic *efx) 276 { 277 if (efx->type->monitor) 278 queue_delayed_work(efx->workqueue, &efx->monitor_work, 279 efx_monitor_interval); 280 } 281 282 /************************************************************************** 283 * 284 * Event queue processing 285 * 286 *************************************************************************/ 287 288 /* Channels are shutdown and reinitialised whilst the NIC is running 289 * to propagate configuration changes (mtu, checksum offload), or 290 * to clear hardware error conditions 291 */ 292 static void efx_start_datapath(struct efx_nic *efx) 293 { 294 netdev_features_t old_features = efx->net_dev->features; 295 bool old_rx_scatter = efx->rx_scatter; 296 size_t rx_buf_len; 297 298 /* Calculate the rx buffer allocation parameters required to 299 * support the current MTU, including padding for header 300 * alignment and overruns. 301 */ 302 efx->rx_dma_len = (efx->rx_prefix_size + 303 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 304 efx->type->rx_buffer_padding); 305 rx_buf_len = (sizeof(struct efx_rx_page_state) + XDP_PACKET_HEADROOM + 306 efx->rx_ip_align + efx->rx_dma_len); 307 if (rx_buf_len <= PAGE_SIZE) { 308 efx->rx_scatter = efx->type->always_rx_scatter; 309 efx->rx_buffer_order = 0; 310 } else if (efx->type->can_rx_scatter) { 311 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 312 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 313 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 314 EFX_RX_BUF_ALIGNMENT) > 315 PAGE_SIZE); 316 efx->rx_scatter = true; 317 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 318 efx->rx_buffer_order = 0; 319 } else { 320 efx->rx_scatter = false; 321 efx->rx_buffer_order = get_order(rx_buf_len); 322 } 323 324 efx_rx_config_page_split(efx); 325 if (efx->rx_buffer_order) 326 netif_dbg(efx, drv, efx->net_dev, 327 "RX buf len=%u; page order=%u batch=%u\n", 328 efx->rx_dma_len, efx->rx_buffer_order, 329 efx->rx_pages_per_batch); 330 else 331 netif_dbg(efx, drv, efx->net_dev, 332 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 333 efx->rx_dma_len, efx->rx_page_buf_step, 334 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 335 336 /* Restore previously fixed features in hw_features and remove 337 * features which are fixed now 338 */ 339 efx->net_dev->hw_features |= efx->net_dev->features; 340 efx->net_dev->hw_features &= ~efx->fixed_features; 341 efx->net_dev->features |= efx->fixed_features; 342 if (efx->net_dev->features != old_features) 343 netdev_features_change(efx->net_dev); 344 345 /* RX filters may also have scatter-enabled flags */ 346 if ((efx->rx_scatter != old_rx_scatter) && 347 efx->type->filter_update_rx_scatter) 348 efx->type->filter_update_rx_scatter(efx); 349 350 /* We must keep at least one descriptor in a TX ring empty. 351 * We could avoid this when the queue size does not exactly 352 * match the hardware ring size, but it's not that important. 353 * Therefore we stop the queue when one more skb might fill 354 * the ring completely. We wake it when half way back to 355 * empty. 356 */ 357 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 358 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 359 360 /* Initialise the channels */ 361 efx_start_channels(efx); 362 363 efx_ptp_start_datapath(efx); 364 365 if (netif_device_present(efx->net_dev)) 366 netif_tx_wake_all_queues(efx->net_dev); 367 } 368 369 static void efx_stop_datapath(struct efx_nic *efx) 370 { 371 EFX_ASSERT_RESET_SERIALISED(efx); 372 BUG_ON(efx->port_enabled); 373 374 efx_ptp_stop_datapath(efx); 375 376 efx_stop_channels(efx); 377 } 378 379 /************************************************************************** 380 * 381 * Port handling 382 * 383 **************************************************************************/ 384 385 static void efx_start_port(struct efx_nic *efx) 386 { 387 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 388 BUG_ON(efx->port_enabled); 389 390 mutex_lock(&efx->mac_lock); 391 efx->port_enabled = true; 392 393 /* Ensure MAC ingress/egress is enabled */ 394 efx_mac_reconfigure(efx); 395 396 mutex_unlock(&efx->mac_lock); 397 } 398 399 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 400 * and the async self-test, wait for them to finish and prevent them 401 * being scheduled again. This doesn't cover online resets, which 402 * should only be cancelled when removing the device. 403 */ 404 static void efx_stop_port(struct efx_nic *efx) 405 { 406 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 407 408 EFX_ASSERT_RESET_SERIALISED(efx); 409 410 mutex_lock(&efx->mac_lock); 411 efx->port_enabled = false; 412 mutex_unlock(&efx->mac_lock); 413 414 /* Serialise against efx_set_multicast_list() */ 415 netif_addr_lock_bh(efx->net_dev); 416 netif_addr_unlock_bh(efx->net_dev); 417 418 cancel_delayed_work_sync(&efx->monitor_work); 419 efx_selftest_async_cancel(efx); 420 cancel_work_sync(&efx->mac_work); 421 } 422 423 /* If the interface is supposed to be running but is not, start 424 * the hardware and software data path, regular activity for the port 425 * (MAC statistics, link polling, etc.) and schedule the port to be 426 * reconfigured. Interrupts must already be enabled. This function 427 * is safe to call multiple times, so long as the NIC is not disabled. 428 * Requires the RTNL lock. 429 */ 430 void efx_start_all(struct efx_nic *efx) 431 { 432 EFX_ASSERT_RESET_SERIALISED(efx); 433 BUG_ON(efx->state == STATE_DISABLED); 434 435 /* Check that it is appropriate to restart the interface. All 436 * of these flags are safe to read under just the rtnl lock 437 */ 438 if (efx->port_enabled || !netif_running(efx->net_dev) || 439 efx->reset_pending) 440 return; 441 442 efx_start_port(efx); 443 efx_start_datapath(efx); 444 445 /* Start the hardware monitor if there is one */ 446 efx_start_monitor(efx); 447 448 /* Link state detection is normally event-driven; we have 449 * to poll now because we could have missed a change 450 */ 451 mutex_lock(&efx->mac_lock); 452 if (efx->phy_op->poll(efx)) 453 efx_link_status_changed(efx); 454 mutex_unlock(&efx->mac_lock); 455 456 if (efx->type->start_stats) { 457 efx->type->start_stats(efx); 458 efx->type->pull_stats(efx); 459 spin_lock_bh(&efx->stats_lock); 460 efx->type->update_stats(efx, NULL, NULL); 461 spin_unlock_bh(&efx->stats_lock); 462 } 463 } 464 465 /* Quiesce the hardware and software data path, and regular activity 466 * for the port without bringing the link down. Safe to call multiple 467 * times with the NIC in almost any state, but interrupts should be 468 * enabled. Requires the RTNL lock. 469 */ 470 void efx_stop_all(struct efx_nic *efx) 471 { 472 EFX_ASSERT_RESET_SERIALISED(efx); 473 474 /* port_enabled can be read safely under the rtnl lock */ 475 if (!efx->port_enabled) 476 return; 477 478 if (efx->type->update_stats) { 479 /* update stats before we go down so we can accurately count 480 * rx_nodesc_drops 481 */ 482 efx->type->pull_stats(efx); 483 spin_lock_bh(&efx->stats_lock); 484 efx->type->update_stats(efx, NULL, NULL); 485 spin_unlock_bh(&efx->stats_lock); 486 efx->type->stop_stats(efx); 487 } 488 489 efx_stop_port(efx); 490 491 /* Stop the kernel transmit interface. This is only valid if 492 * the device is stopped or detached; otherwise the watchdog 493 * may fire immediately. 494 */ 495 WARN_ON(netif_running(efx->net_dev) && 496 netif_device_present(efx->net_dev)); 497 netif_tx_disable(efx->net_dev); 498 499 efx_stop_datapath(efx); 500 } 501 502 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 503 void efx_net_stats(struct net_device *net_dev, struct rtnl_link_stats64 *stats) 504 { 505 struct efx_nic *efx = netdev_priv(net_dev); 506 507 spin_lock_bh(&efx->stats_lock); 508 efx->type->update_stats(efx, NULL, stats); 509 spin_unlock_bh(&efx->stats_lock); 510 } 511 512 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 513 * the MAC appropriately. All other PHY configuration changes are pushed 514 * through phy_op->set_settings(), and pushed asynchronously to the MAC 515 * through efx_monitor(). 516 * 517 * Callers must hold the mac_lock 518 */ 519 int __efx_reconfigure_port(struct efx_nic *efx) 520 { 521 enum efx_phy_mode phy_mode; 522 int rc = 0; 523 524 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 525 526 /* Disable PHY transmit in mac level loopbacks */ 527 phy_mode = efx->phy_mode; 528 if (LOOPBACK_INTERNAL(efx)) 529 efx->phy_mode |= PHY_MODE_TX_DISABLED; 530 else 531 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 532 533 if (efx->type->reconfigure_port) 534 rc = efx->type->reconfigure_port(efx); 535 536 if (rc) 537 efx->phy_mode = phy_mode; 538 539 return rc; 540 } 541 542 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 543 * disabled. 544 */ 545 int efx_reconfigure_port(struct efx_nic *efx) 546 { 547 int rc; 548 549 EFX_ASSERT_RESET_SERIALISED(efx); 550 551 mutex_lock(&efx->mac_lock); 552 rc = __efx_reconfigure_port(efx); 553 mutex_unlock(&efx->mac_lock); 554 555 return rc; 556 } 557 558 /************************************************************************** 559 * 560 * Device reset and suspend 561 * 562 **************************************************************************/ 563 564 static void efx_wait_for_bist_end(struct efx_nic *efx) 565 { 566 int i; 567 568 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 569 if (efx_mcdi_poll_reboot(efx)) 570 goto out; 571 msleep(BIST_WAIT_DELAY_MS); 572 } 573 574 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 575 out: 576 /* Either way unset the BIST flag. If we found no reboot we probably 577 * won't recover, but we should try. 578 */ 579 efx->mc_bist_for_other_fn = false; 580 } 581 582 /* Try recovery mechanisms. 583 * For now only EEH is supported. 584 * Returns 0 if the recovery mechanisms are unsuccessful. 585 * Returns a non-zero value otherwise. 586 */ 587 int efx_try_recovery(struct efx_nic *efx) 588 { 589 #ifdef CONFIG_EEH 590 /* A PCI error can occur and not be seen by EEH because nothing 591 * happens on the PCI bus. In this case the driver may fail and 592 * schedule a 'recover or reset', leading to this recovery handler. 593 * Manually call the eeh failure check function. 594 */ 595 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 596 if (eeh_dev_check_failure(eehdev)) { 597 /* The EEH mechanisms will handle the error and reset the 598 * device if necessary. 599 */ 600 return 1; 601 } 602 #endif 603 return 0; 604 } 605 606 /* Tears down the entire software state and most of the hardware state 607 * before reset. 608 */ 609 void efx_reset_down(struct efx_nic *efx, enum reset_type method) 610 { 611 EFX_ASSERT_RESET_SERIALISED(efx); 612 613 if (method == RESET_TYPE_MCDI_TIMEOUT) 614 efx->type->prepare_flr(efx); 615 616 efx_stop_all(efx); 617 efx_disable_interrupts(efx); 618 619 mutex_lock(&efx->mac_lock); 620 down_write(&efx->filter_sem); 621 mutex_lock(&efx->rss_lock); 622 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 623 method != RESET_TYPE_DATAPATH) 624 efx->phy_op->fini(efx); 625 efx->type->fini(efx); 626 } 627 628 /* This function will always ensure that the locks acquired in 629 * efx_reset_down() are released. A failure return code indicates 630 * that we were unable to reinitialise the hardware, and the 631 * driver should be disabled. If ok is false, then the rx and tx 632 * engines are not restarted, pending a RESET_DISABLE. 633 */ 634 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 635 { 636 int rc; 637 638 EFX_ASSERT_RESET_SERIALISED(efx); 639 640 if (method == RESET_TYPE_MCDI_TIMEOUT) 641 efx->type->finish_flr(efx); 642 643 /* Ensure that SRAM is initialised even if we're disabling the device */ 644 rc = efx->type->init(efx); 645 if (rc) { 646 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 647 goto fail; 648 } 649 650 if (!ok) 651 goto fail; 652 653 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 654 method != RESET_TYPE_DATAPATH) { 655 rc = efx->phy_op->init(efx); 656 if (rc) 657 goto fail; 658 rc = efx->phy_op->reconfigure(efx); 659 if (rc && rc != -EPERM) 660 netif_err(efx, drv, efx->net_dev, 661 "could not restore PHY settings\n"); 662 } 663 664 rc = efx_enable_interrupts(efx); 665 if (rc) 666 goto fail; 667 668 #ifdef CONFIG_SFC_SRIOV 669 rc = efx->type->vswitching_restore(efx); 670 if (rc) /* not fatal; the PF will still work fine */ 671 netif_warn(efx, probe, efx->net_dev, 672 "failed to restore vswitching rc=%d;" 673 " VFs may not function\n", rc); 674 #endif 675 676 if (efx->type->rx_restore_rss_contexts) 677 efx->type->rx_restore_rss_contexts(efx); 678 mutex_unlock(&efx->rss_lock); 679 efx->type->filter_table_restore(efx); 680 up_write(&efx->filter_sem); 681 if (efx->type->sriov_reset) 682 efx->type->sriov_reset(efx); 683 684 mutex_unlock(&efx->mac_lock); 685 686 efx_start_all(efx); 687 688 if (efx->type->udp_tnl_push_ports) 689 efx->type->udp_tnl_push_ports(efx); 690 691 return 0; 692 693 fail: 694 efx->port_initialized = false; 695 696 mutex_unlock(&efx->rss_lock); 697 up_write(&efx->filter_sem); 698 mutex_unlock(&efx->mac_lock); 699 700 return rc; 701 } 702 703 /* Reset the NIC using the specified method. Note that the reset may 704 * fail, in which case the card will be left in an unusable state. 705 * 706 * Caller must hold the rtnl_lock. 707 */ 708 int efx_reset(struct efx_nic *efx, enum reset_type method) 709 { 710 bool disabled; 711 int rc, rc2; 712 713 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 714 RESET_TYPE(method)); 715 716 efx_device_detach_sync(efx); 717 efx_reset_down(efx, method); 718 719 rc = efx->type->reset(efx, method); 720 if (rc) { 721 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 722 goto out; 723 } 724 725 /* Clear flags for the scopes we covered. We assume the NIC and 726 * driver are now quiescent so that there is no race here. 727 */ 728 if (method < RESET_TYPE_MAX_METHOD) 729 efx->reset_pending &= -(1 << (method + 1)); 730 else /* it doesn't fit into the well-ordered scope hierarchy */ 731 __clear_bit(method, &efx->reset_pending); 732 733 /* Reinitialise bus-mastering, which may have been turned off before 734 * the reset was scheduled. This is still appropriate, even in the 735 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 736 * can respond to requests. 737 */ 738 pci_set_master(efx->pci_dev); 739 740 out: 741 /* Leave device stopped if necessary */ 742 disabled = rc || 743 method == RESET_TYPE_DISABLE || 744 method == RESET_TYPE_RECOVER_OR_DISABLE; 745 rc2 = efx_reset_up(efx, method, !disabled); 746 if (rc2) { 747 disabled = true; 748 if (!rc) 749 rc = rc2; 750 } 751 752 if (disabled) { 753 dev_close(efx->net_dev); 754 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 755 efx->state = STATE_DISABLED; 756 } else { 757 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 758 efx_device_attach_if_not_resetting(efx); 759 } 760 return rc; 761 } 762 763 /* The worker thread exists so that code that cannot sleep can 764 * schedule a reset for later. 765 */ 766 static void efx_reset_work(struct work_struct *data) 767 { 768 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 769 unsigned long pending; 770 enum reset_type method; 771 772 pending = READ_ONCE(efx->reset_pending); 773 method = fls(pending) - 1; 774 775 if (method == RESET_TYPE_MC_BIST) 776 efx_wait_for_bist_end(efx); 777 778 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 779 method == RESET_TYPE_RECOVER_OR_ALL) && 780 efx_try_recovery(efx)) 781 return; 782 783 if (!pending) 784 return; 785 786 rtnl_lock(); 787 788 /* We checked the state in efx_schedule_reset() but it may 789 * have changed by now. Now that we have the RTNL lock, 790 * it cannot change again. 791 */ 792 if (efx->state == STATE_READY) 793 (void)efx_reset(efx, method); 794 795 rtnl_unlock(); 796 } 797 798 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 799 { 800 enum reset_type method; 801 802 if (efx->state == STATE_RECOVERY) { 803 netif_dbg(efx, drv, efx->net_dev, 804 "recovering: skip scheduling %s reset\n", 805 RESET_TYPE(type)); 806 return; 807 } 808 809 switch (type) { 810 case RESET_TYPE_INVISIBLE: 811 case RESET_TYPE_ALL: 812 case RESET_TYPE_RECOVER_OR_ALL: 813 case RESET_TYPE_WORLD: 814 case RESET_TYPE_DISABLE: 815 case RESET_TYPE_RECOVER_OR_DISABLE: 816 case RESET_TYPE_DATAPATH: 817 case RESET_TYPE_MC_BIST: 818 case RESET_TYPE_MCDI_TIMEOUT: 819 method = type; 820 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 821 RESET_TYPE(method)); 822 break; 823 default: 824 method = efx->type->map_reset_reason(type); 825 netif_dbg(efx, drv, efx->net_dev, 826 "scheduling %s reset for %s\n", 827 RESET_TYPE(method), RESET_TYPE(type)); 828 break; 829 } 830 831 set_bit(method, &efx->reset_pending); 832 smp_mb(); /* ensure we change reset_pending before checking state */ 833 834 /* If we're not READY then just leave the flags set as the cue 835 * to abort probing or reschedule the reset later. 836 */ 837 if (READ_ONCE(efx->state) != STATE_READY) 838 return; 839 840 /* efx_process_channel() will no longer read events once a 841 * reset is scheduled. So switch back to poll'd MCDI completions. 842 */ 843 efx_mcdi_mode_poll(efx); 844 845 efx_queue_reset_work(efx); 846 } 847 848 /************************************************************************** 849 * 850 * Dummy PHY/MAC operations 851 * 852 * Can be used for some unimplemented operations 853 * Needed so all function pointers are valid and do not have to be tested 854 * before use 855 * 856 **************************************************************************/ 857 int efx_port_dummy_op_int(struct efx_nic *efx) 858 { 859 return 0; 860 } 861 void efx_port_dummy_op_void(struct efx_nic *efx) {} 862 863 static bool efx_port_dummy_op_poll(struct efx_nic *efx) 864 { 865 return false; 866 } 867 868 static const struct efx_phy_operations efx_dummy_phy_operations = { 869 .init = efx_port_dummy_op_int, 870 .reconfigure = efx_port_dummy_op_int, 871 .poll = efx_port_dummy_op_poll, 872 .fini = efx_port_dummy_op_void, 873 }; 874 875 /************************************************************************** 876 * 877 * Data housekeeping 878 * 879 **************************************************************************/ 880 881 /* This zeroes out and then fills in the invariants in a struct 882 * efx_nic (including all sub-structures). 883 */ 884 int efx_init_struct(struct efx_nic *efx, 885 struct pci_dev *pci_dev, struct net_device *net_dev) 886 { 887 int rc = -ENOMEM; 888 889 /* Initialise common structures */ 890 INIT_LIST_HEAD(&efx->node); 891 INIT_LIST_HEAD(&efx->secondary_list); 892 spin_lock_init(&efx->biu_lock); 893 #ifdef CONFIG_SFC_MTD 894 INIT_LIST_HEAD(&efx->mtd_list); 895 #endif 896 INIT_WORK(&efx->reset_work, efx_reset_work); 897 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 898 efx_selftest_async_init(efx); 899 efx->pci_dev = pci_dev; 900 efx->msg_enable = debug; 901 efx->state = STATE_UNINIT; 902 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 903 904 efx->net_dev = net_dev; 905 efx->rx_prefix_size = efx->type->rx_prefix_size; 906 efx->rx_ip_align = 907 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 908 efx->rx_packet_hash_offset = 909 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 910 efx->rx_packet_ts_offset = 911 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 912 INIT_LIST_HEAD(&efx->rss_context.list); 913 mutex_init(&efx->rss_lock); 914 spin_lock_init(&efx->stats_lock); 915 efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 916 efx->num_mac_stats = MC_CMD_MAC_NSTATS; 917 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 918 mutex_init(&efx->mac_lock); 919 #ifdef CONFIG_RFS_ACCEL 920 mutex_init(&efx->rps_mutex); 921 spin_lock_init(&efx->rps_hash_lock); 922 /* Failure to allocate is not fatal, but may degrade ARFS performance */ 923 efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, 924 sizeof(*efx->rps_hash_table), GFP_KERNEL); 925 #endif 926 efx->phy_op = &efx_dummy_phy_operations; 927 efx->mdio.dev = net_dev; 928 INIT_WORK(&efx->mac_work, efx_mac_work); 929 init_waitqueue_head(&efx->flush_wq); 930 931 rc = efx_init_channels(efx); 932 if (rc) 933 goto fail; 934 935 /* Would be good to use the net_dev name, but we're too early */ 936 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 937 pci_name(pci_dev)); 938 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 939 if (!efx->workqueue) { 940 rc = -ENOMEM; 941 goto fail; 942 } 943 944 return 0; 945 946 fail: 947 efx_fini_struct(efx); 948 return rc; 949 } 950 951 void efx_fini_struct(struct efx_nic *efx) 952 { 953 #ifdef CONFIG_RFS_ACCEL 954 kfree(efx->rps_hash_table); 955 #endif 956 957 efx_fini_channels(efx); 958 959 kfree(efx->vpd_sn); 960 961 if (efx->workqueue) { 962 destroy_workqueue(efx->workqueue); 963 efx->workqueue = NULL; 964 } 965 } 966 967 /* This configures the PCI device to enable I/O and DMA. */ 968 int efx_init_io(struct efx_nic *efx, int bar, dma_addr_t dma_mask, 969 unsigned int mem_map_size) 970 { 971 struct pci_dev *pci_dev = efx->pci_dev; 972 int rc; 973 974 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 975 976 rc = pci_enable_device(pci_dev); 977 if (rc) { 978 netif_err(efx, probe, efx->net_dev, 979 "failed to enable PCI device\n"); 980 goto fail1; 981 } 982 983 pci_set_master(pci_dev); 984 985 /* Set the PCI DMA mask. Try all possibilities from our 986 * genuine mask down to 32 bits, because some architectures 987 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 988 * masks event though they reject 46 bit masks. 989 */ 990 while (dma_mask > 0x7fffffffUL) { 991 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 992 if (rc == 0) 993 break; 994 dma_mask >>= 1; 995 } 996 if (rc) { 997 netif_err(efx, probe, efx->net_dev, 998 "could not find a suitable DMA mask\n"); 999 goto fail2; 1000 } 1001 netif_dbg(efx, probe, efx->net_dev, 1002 "using DMA mask %llx\n", (unsigned long long)dma_mask); 1003 1004 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1005 if (!efx->membase_phys) { 1006 netif_err(efx, probe, efx->net_dev, 1007 "ERROR: No BAR%d mapping from the BIOS. " 1008 "Try pci=realloc on the kernel command line\n", bar); 1009 rc = -ENODEV; 1010 goto fail3; 1011 } 1012 1013 rc = pci_request_region(pci_dev, bar, "sfc"); 1014 if (rc) { 1015 netif_err(efx, probe, efx->net_dev, 1016 "request for memory BAR failed\n"); 1017 rc = -EIO; 1018 goto fail3; 1019 } 1020 1021 efx->membase = ioremap(efx->membase_phys, mem_map_size); 1022 if (!efx->membase) { 1023 netif_err(efx, probe, efx->net_dev, 1024 "could not map memory BAR at %llx+%x\n", 1025 (unsigned long long)efx->membase_phys, mem_map_size); 1026 rc = -ENOMEM; 1027 goto fail4; 1028 } 1029 netif_dbg(efx, probe, efx->net_dev, 1030 "memory BAR at %llx+%x (virtual %p)\n", 1031 (unsigned long long)efx->membase_phys, mem_map_size, 1032 efx->membase); 1033 1034 return 0; 1035 1036 fail4: 1037 pci_release_region(efx->pci_dev, bar); 1038 fail3: 1039 efx->membase_phys = 0; 1040 fail2: 1041 pci_disable_device(efx->pci_dev); 1042 fail1: 1043 return rc; 1044 } 1045 1046 void efx_fini_io(struct efx_nic *efx, int bar) 1047 { 1048 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1049 1050 if (efx->membase) { 1051 iounmap(efx->membase); 1052 efx->membase = NULL; 1053 } 1054 1055 if (efx->membase_phys) { 1056 pci_release_region(efx->pci_dev, bar); 1057 efx->membase_phys = 0; 1058 } 1059 1060 /* Don't disable bus-mastering if VFs are assigned */ 1061 if (!pci_vfs_assigned(efx->pci_dev)) 1062 pci_disable_device(efx->pci_dev); 1063 } 1064 1065 #ifdef CONFIG_SFC_MCDI_LOGGING 1066 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 1067 char *buf) 1068 { 1069 struct efx_nic *efx = dev_get_drvdata(dev); 1070 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 1071 1072 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 1073 } 1074 1075 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 1076 const char *buf, size_t count) 1077 { 1078 struct efx_nic *efx = dev_get_drvdata(dev); 1079 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 1080 bool enable = count > 0 && *buf != '0'; 1081 1082 mcdi->logging_enabled = enable; 1083 return count; 1084 } 1085 1086 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 1087 1088 void efx_init_mcdi_logging(struct efx_nic *efx) 1089 { 1090 int rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 1091 1092 if (rc) { 1093 netif_warn(efx, drv, efx->net_dev, 1094 "failed to init net dev attributes\n"); 1095 } 1096 } 1097 1098 void efx_fini_mcdi_logging(struct efx_nic *efx) 1099 { 1100 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 1101 } 1102 #endif 1103