1874aeea5SJeff Kirsher /**************************************************************************** 2f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 3874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 4f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 5874aeea5SJeff Kirsher * 6874aeea5SJeff Kirsher * This program is free software; you can redistribute it and/or modify it 7874aeea5SJeff Kirsher * under the terms of the GNU General Public License version 2 as published 8874aeea5SJeff Kirsher * by the Free Software Foundation, incorporated herein by reference. 9874aeea5SJeff Kirsher */ 10874aeea5SJeff Kirsher 11874aeea5SJeff Kirsher #include <linux/module.h> 12874aeea5SJeff Kirsher #include <linux/pci.h> 13874aeea5SJeff Kirsher #include <linux/netdevice.h> 14874aeea5SJeff Kirsher #include <linux/etherdevice.h> 15874aeea5SJeff Kirsher #include <linux/delay.h> 16874aeea5SJeff Kirsher #include <linux/notifier.h> 17874aeea5SJeff Kirsher #include <linux/ip.h> 18874aeea5SJeff Kirsher #include <linux/tcp.h> 19874aeea5SJeff Kirsher #include <linux/in.h> 20874aeea5SJeff Kirsher #include <linux/ethtool.h> 21874aeea5SJeff Kirsher #include <linux/topology.h> 22874aeea5SJeff Kirsher #include <linux/gfp.h> 23626950dbSAlexandre Rames #include <linux/aer.h> 24b28405b0SAlexandre Rames #include <linux/interrupt.h> 25874aeea5SJeff Kirsher #include "net_driver.h" 26874aeea5SJeff Kirsher #include "efx.h" 27874aeea5SJeff Kirsher #include "nic.h" 28dd40781eSBen Hutchings #include "selftest.h" 297fa8d547SShradha Shah #include "sriov.h" 30874aeea5SJeff Kirsher 31874aeea5SJeff Kirsher #include "mcdi.h" 32874aeea5SJeff Kirsher #include "workarounds.h" 33874aeea5SJeff Kirsher 34874aeea5SJeff Kirsher /************************************************************************** 35874aeea5SJeff Kirsher * 36874aeea5SJeff Kirsher * Type name strings 37874aeea5SJeff Kirsher * 38874aeea5SJeff Kirsher ************************************************************************** 39874aeea5SJeff Kirsher */ 40874aeea5SJeff Kirsher 41874aeea5SJeff Kirsher /* Loopback mode names (see LOOPBACK_MODE()) */ 42874aeea5SJeff Kirsher const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 4318e83e4cSBen Hutchings const char *const efx_loopback_mode_names[] = { 44874aeea5SJeff Kirsher [LOOPBACK_NONE] = "NONE", 45874aeea5SJeff Kirsher [LOOPBACK_DATA] = "DATAPATH", 46874aeea5SJeff Kirsher [LOOPBACK_GMAC] = "GMAC", 47874aeea5SJeff Kirsher [LOOPBACK_XGMII] = "XGMII", 48874aeea5SJeff Kirsher [LOOPBACK_XGXS] = "XGXS", 49874aeea5SJeff Kirsher [LOOPBACK_XAUI] = "XAUI", 50874aeea5SJeff Kirsher [LOOPBACK_GMII] = "GMII", 51874aeea5SJeff Kirsher [LOOPBACK_SGMII] = "SGMII", 52874aeea5SJeff Kirsher [LOOPBACK_XGBR] = "XGBR", 53874aeea5SJeff Kirsher [LOOPBACK_XFI] = "XFI", 54874aeea5SJeff Kirsher [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 55874aeea5SJeff Kirsher [LOOPBACK_GMII_FAR] = "GMII_FAR", 56874aeea5SJeff Kirsher [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 57874aeea5SJeff Kirsher [LOOPBACK_XFI_FAR] = "XFI_FAR", 58874aeea5SJeff Kirsher [LOOPBACK_GPHY] = "GPHY", 59874aeea5SJeff Kirsher [LOOPBACK_PHYXS] = "PHYXS", 60874aeea5SJeff Kirsher [LOOPBACK_PCS] = "PCS", 61874aeea5SJeff Kirsher [LOOPBACK_PMAPMD] = "PMA/PMD", 62874aeea5SJeff Kirsher [LOOPBACK_XPORT] = "XPORT", 63874aeea5SJeff Kirsher [LOOPBACK_XGMII_WS] = "XGMII_WS", 64874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS] = "XAUI_WS", 65874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 66874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 67874aeea5SJeff Kirsher [LOOPBACK_GMII_WS] = "GMII_WS", 68874aeea5SJeff Kirsher [LOOPBACK_XFI_WS] = "XFI_WS", 69874aeea5SJeff Kirsher [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 70874aeea5SJeff Kirsher [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 71874aeea5SJeff Kirsher }; 72874aeea5SJeff Kirsher 73874aeea5SJeff Kirsher const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 7418e83e4cSBen Hutchings const char *const efx_reset_type_names[] = { 75874aeea5SJeff Kirsher [RESET_TYPE_INVISIBLE] = "INVISIBLE", 76874aeea5SJeff Kirsher [RESET_TYPE_ALL] = "ALL", 77626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 78874aeea5SJeff Kirsher [RESET_TYPE_WORLD] = "WORLD", 79626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 80087e9025SJon Cooper [RESET_TYPE_DATAPATH] = "DATAPATH", 81e283546cSEdward Cree [RESET_TYPE_MC_BIST] = "MC_BIST", 82874aeea5SJeff Kirsher [RESET_TYPE_DISABLE] = "DISABLE", 83874aeea5SJeff Kirsher [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 84874aeea5SJeff Kirsher [RESET_TYPE_INT_ERROR] = "INT_ERROR", 85874aeea5SJeff Kirsher [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY", 863de82b91SAlexandre Rames [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 87874aeea5SJeff Kirsher [RESET_TYPE_TX_SKIP] = "TX_SKIP", 88874aeea5SJeff Kirsher [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 89e283546cSEdward Cree [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 90874aeea5SJeff Kirsher }; 91874aeea5SJeff Kirsher 92874aeea5SJeff Kirsher /* Reset workqueue. If any NIC has a hardware failure then a reset will be 93874aeea5SJeff Kirsher * queued onto this work queue. This is not a per-nic work queue, because 94874aeea5SJeff Kirsher * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 95874aeea5SJeff Kirsher */ 96874aeea5SJeff Kirsher static struct workqueue_struct *reset_workqueue; 97874aeea5SJeff Kirsher 9874cd60a4SJon Cooper /* How often and how many times to poll for a reset while waiting for a 9974cd60a4SJon Cooper * BIST that another function started to complete. 10074cd60a4SJon Cooper */ 10174cd60a4SJon Cooper #define BIST_WAIT_DELAY_MS 100 10274cd60a4SJon Cooper #define BIST_WAIT_DELAY_COUNT 100 10374cd60a4SJon Cooper 104874aeea5SJeff Kirsher /************************************************************************** 105874aeea5SJeff Kirsher * 106874aeea5SJeff Kirsher * Configurable values 107874aeea5SJeff Kirsher * 108874aeea5SJeff Kirsher *************************************************************************/ 109874aeea5SJeff Kirsher 110874aeea5SJeff Kirsher /* 111874aeea5SJeff Kirsher * Use separate channels for TX and RX events 112874aeea5SJeff Kirsher * 113874aeea5SJeff Kirsher * Set this to 1 to use separate channels for TX and RX. It allows us 114874aeea5SJeff Kirsher * to control interrupt affinity separately for TX and RX. 115874aeea5SJeff Kirsher * 116874aeea5SJeff Kirsher * This is only used in MSI-X interrupt mode 117874aeea5SJeff Kirsher */ 118b0fbdae1SShradha Shah bool efx_separate_tx_channels; 119b0fbdae1SShradha Shah module_param(efx_separate_tx_channels, bool, 0444); 120b0fbdae1SShradha Shah MODULE_PARM_DESC(efx_separate_tx_channels, 121874aeea5SJeff Kirsher "Use separate channels for TX and RX"); 122874aeea5SJeff Kirsher 123874aeea5SJeff Kirsher /* This is the weight assigned to each of the (per-channel) virtual 124874aeea5SJeff Kirsher * NAPI devices. 125874aeea5SJeff Kirsher */ 126874aeea5SJeff Kirsher static int napi_weight = 64; 127874aeea5SJeff Kirsher 128874aeea5SJeff Kirsher /* This is the time (in jiffies) between invocations of the hardware 129626950dbSAlexandre Rames * monitor. 130626950dbSAlexandre Rames * On Falcon-based NICs, this will: 131874aeea5SJeff Kirsher * - Check the on-board hardware monitor; 132874aeea5SJeff Kirsher * - Poll the link state and reconfigure the hardware as necessary. 133626950dbSAlexandre Rames * On Siena-based NICs for power systems with EEH support, this will give EEH a 134626950dbSAlexandre Rames * chance to start. 135874aeea5SJeff Kirsher */ 136874aeea5SJeff Kirsher static unsigned int efx_monitor_interval = 1 * HZ; 137874aeea5SJeff Kirsher 138874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 139874aeea5SJeff Kirsher * module load with ethtool. 140874aeea5SJeff Kirsher * 141874aeea5SJeff Kirsher * The default for RX should strike a balance between increasing the 142874aeea5SJeff Kirsher * round-trip latency and reducing overhead. 143874aeea5SJeff Kirsher */ 144874aeea5SJeff Kirsher static unsigned int rx_irq_mod_usec = 60; 145874aeea5SJeff Kirsher 146874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 147874aeea5SJeff Kirsher * module load with ethtool. 148874aeea5SJeff Kirsher * 149874aeea5SJeff Kirsher * This default is chosen to ensure that a 10G link does not go idle 150874aeea5SJeff Kirsher * while a TX queue is stopped after it has become full. A queue is 151874aeea5SJeff Kirsher * restarted when it drops below half full. The time this takes (assuming 152874aeea5SJeff Kirsher * worst case 3 descriptors per packet and 1024 descriptors) is 153874aeea5SJeff Kirsher * 512 / 3 * 1.2 = 205 usec. 154874aeea5SJeff Kirsher */ 155874aeea5SJeff Kirsher static unsigned int tx_irq_mod_usec = 150; 156874aeea5SJeff Kirsher 157874aeea5SJeff Kirsher /* This is the first interrupt mode to try out of: 158874aeea5SJeff Kirsher * 0 => MSI-X 159874aeea5SJeff Kirsher * 1 => MSI 160874aeea5SJeff Kirsher * 2 => legacy 161874aeea5SJeff Kirsher */ 162874aeea5SJeff Kirsher static unsigned int interrupt_mode; 163874aeea5SJeff Kirsher 164874aeea5SJeff Kirsher /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 165874aeea5SJeff Kirsher * i.e. the number of CPUs among which we may distribute simultaneous 166874aeea5SJeff Kirsher * interrupt handling. 167874aeea5SJeff Kirsher * 168874aeea5SJeff Kirsher * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 169cdb08f8fSBen Hutchings * The default (0) means to assign an interrupt to each core. 170874aeea5SJeff Kirsher */ 171874aeea5SJeff Kirsher static unsigned int rss_cpus; 172874aeea5SJeff Kirsher module_param(rss_cpus, uint, 0444); 173874aeea5SJeff Kirsher MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 174874aeea5SJeff Kirsher 175b9cc977dSBen Hutchings static bool phy_flash_cfg; 176b9cc977dSBen Hutchings module_param(phy_flash_cfg, bool, 0644); 177874aeea5SJeff Kirsher MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 178874aeea5SJeff Kirsher 179e7bed9c8SBen Hutchings static unsigned irq_adapt_low_thresh = 8000; 180874aeea5SJeff Kirsher module_param(irq_adapt_low_thresh, uint, 0644); 181874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_low_thresh, 182874aeea5SJeff Kirsher "Threshold score for reducing IRQ moderation"); 183874aeea5SJeff Kirsher 184e7bed9c8SBen Hutchings static unsigned irq_adapt_high_thresh = 16000; 185874aeea5SJeff Kirsher module_param(irq_adapt_high_thresh, uint, 0644); 186874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_high_thresh, 187874aeea5SJeff Kirsher "Threshold score for increasing IRQ moderation"); 188874aeea5SJeff Kirsher 189874aeea5SJeff Kirsher static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 190874aeea5SJeff Kirsher NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 191874aeea5SJeff Kirsher NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 192874aeea5SJeff Kirsher NETIF_MSG_TX_ERR | NETIF_MSG_HW); 193874aeea5SJeff Kirsher module_param(debug, uint, 0); 194874aeea5SJeff Kirsher MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 195874aeea5SJeff Kirsher 196874aeea5SJeff Kirsher /************************************************************************** 197874aeea5SJeff Kirsher * 198874aeea5SJeff Kirsher * Utility functions and prototypes 199874aeea5SJeff Kirsher * 200874aeea5SJeff Kirsher *************************************************************************/ 201874aeea5SJeff Kirsher 202261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx); 203d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx); 2047f967c01SBen Hutchings static void efx_remove_channel(struct efx_channel *channel); 205874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx); 2067f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type; 207874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx); 2087f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel); 209874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx); 210874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel); 211874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx); 212874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx); 213874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx); 214874aeea5SJeff Kirsher 215874aeea5SJeff Kirsher #define EFX_ASSERT_RESET_SERIALISED(efx) \ 216874aeea5SJeff Kirsher do { \ 217f16aeea0SBen Hutchings if ((efx->state == STATE_READY) || \ 218626950dbSAlexandre Rames (efx->state == STATE_RECOVERY) || \ 219874aeea5SJeff Kirsher (efx->state == STATE_DISABLED)) \ 220874aeea5SJeff Kirsher ASSERT_RTNL(); \ 221874aeea5SJeff Kirsher } while (0) 222874aeea5SJeff Kirsher 2238b7325b4SBen Hutchings static int efx_check_disabled(struct efx_nic *efx) 2248b7325b4SBen Hutchings { 225626950dbSAlexandre Rames if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 2268b7325b4SBen Hutchings netif_err(efx, drv, efx->net_dev, 2278b7325b4SBen Hutchings "device is disabled due to earlier errors\n"); 2288b7325b4SBen Hutchings return -EIO; 2298b7325b4SBen Hutchings } 2308b7325b4SBen Hutchings return 0; 2318b7325b4SBen Hutchings } 2328b7325b4SBen Hutchings 233874aeea5SJeff Kirsher /************************************************************************** 234874aeea5SJeff Kirsher * 235874aeea5SJeff Kirsher * Event queue processing 236874aeea5SJeff Kirsher * 237874aeea5SJeff Kirsher *************************************************************************/ 238874aeea5SJeff Kirsher 239874aeea5SJeff Kirsher /* Process channel's event queue 240874aeea5SJeff Kirsher * 241874aeea5SJeff Kirsher * This function is responsible for processing the event queue of a 242874aeea5SJeff Kirsher * single channel. The caller must guarantee that this function will 243874aeea5SJeff Kirsher * never be concurrently called more than once on the same channel, 244874aeea5SJeff Kirsher * though different channels may be being processed concurrently. 245874aeea5SJeff Kirsher */ 246874aeea5SJeff Kirsher static int efx_process_channel(struct efx_channel *channel, int budget) 247874aeea5SJeff Kirsher { 248c936835cSPeter Dunning struct efx_tx_queue *tx_queue; 249874aeea5SJeff Kirsher int spent; 250874aeea5SJeff Kirsher 2519f2cb71cSBen Hutchings if (unlikely(!channel->enabled)) 252874aeea5SJeff Kirsher return 0; 253874aeea5SJeff Kirsher 254c936835cSPeter Dunning efx_for_each_channel_tx_queue(tx_queue, channel) { 255c936835cSPeter Dunning tx_queue->pkts_compl = 0; 256c936835cSPeter Dunning tx_queue->bytes_compl = 0; 257c936835cSPeter Dunning } 258c936835cSPeter Dunning 259874aeea5SJeff Kirsher spent = efx_nic_process_eventq(channel, budget); 260d9ab7007SBen Hutchings if (spent && efx_channel_has_rx_queue(channel)) { 261d9ab7007SBen Hutchings struct efx_rx_queue *rx_queue = 262d9ab7007SBen Hutchings efx_channel_get_rx_queue(channel); 263874aeea5SJeff Kirsher 264ff734ef4SBen Hutchings efx_rx_flush_packet(channel); 265cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, true); 266d9ab7007SBen Hutchings } 267874aeea5SJeff Kirsher 268c936835cSPeter Dunning /* Update BQL */ 269c936835cSPeter Dunning efx_for_each_channel_tx_queue(tx_queue, channel) { 270c936835cSPeter Dunning if (tx_queue->bytes_compl) { 271c936835cSPeter Dunning netdev_tx_completed_queue(tx_queue->core_txq, 272c936835cSPeter Dunning tx_queue->pkts_compl, tx_queue->bytes_compl); 273c936835cSPeter Dunning } 274c936835cSPeter Dunning } 275c936835cSPeter Dunning 276874aeea5SJeff Kirsher return spent; 277874aeea5SJeff Kirsher } 278874aeea5SJeff Kirsher 279874aeea5SJeff Kirsher /* NAPI poll handler 280874aeea5SJeff Kirsher * 281874aeea5SJeff Kirsher * NAPI guarantees serialisation of polls of the same device, which 282874aeea5SJeff Kirsher * provides the guarantee required by efx_process_channel(). 283874aeea5SJeff Kirsher */ 284874aeea5SJeff Kirsher static int efx_poll(struct napi_struct *napi, int budget) 285874aeea5SJeff Kirsher { 286874aeea5SJeff Kirsher struct efx_channel *channel = 287874aeea5SJeff Kirsher container_of(napi, struct efx_channel, napi_str); 288874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 289874aeea5SJeff Kirsher int spent; 290874aeea5SJeff Kirsher 29136763266SAlexandre Rames if (!efx_channel_lock_napi(channel)) 29236763266SAlexandre Rames return budget; 29336763266SAlexandre Rames 294874aeea5SJeff Kirsher netif_vdbg(efx, intr, efx->net_dev, 295874aeea5SJeff Kirsher "channel %d NAPI poll executing on CPU %d\n", 296874aeea5SJeff Kirsher channel->channel, raw_smp_processor_id()); 297874aeea5SJeff Kirsher 298874aeea5SJeff Kirsher spent = efx_process_channel(channel, budget); 299874aeea5SJeff Kirsher 300874aeea5SJeff Kirsher if (spent < budget) { 3019d9a6973SBen Hutchings if (efx_channel_has_rx_queue(channel) && 302874aeea5SJeff Kirsher efx->irq_rx_adaptive && 303874aeea5SJeff Kirsher unlikely(++channel->irq_count == 1000)) { 304874aeea5SJeff Kirsher if (unlikely(channel->irq_mod_score < 305874aeea5SJeff Kirsher irq_adapt_low_thresh)) { 306874aeea5SJeff Kirsher if (channel->irq_moderation > 1) { 307874aeea5SJeff Kirsher channel->irq_moderation -= 1; 308874aeea5SJeff Kirsher efx->type->push_irq_moderation(channel); 309874aeea5SJeff Kirsher } 310874aeea5SJeff Kirsher } else if (unlikely(channel->irq_mod_score > 311874aeea5SJeff Kirsher irq_adapt_high_thresh)) { 312874aeea5SJeff Kirsher if (channel->irq_moderation < 313874aeea5SJeff Kirsher efx->irq_rx_moderation) { 314874aeea5SJeff Kirsher channel->irq_moderation += 1; 315874aeea5SJeff Kirsher efx->type->push_irq_moderation(channel); 316874aeea5SJeff Kirsher } 317874aeea5SJeff Kirsher } 318874aeea5SJeff Kirsher channel->irq_count = 0; 319874aeea5SJeff Kirsher channel->irq_mod_score = 0; 320874aeea5SJeff Kirsher } 321874aeea5SJeff Kirsher 322874aeea5SJeff Kirsher efx_filter_rfs_expire(channel); 323874aeea5SJeff Kirsher 324874aeea5SJeff Kirsher /* There is no race here; although napi_disable() will 325874aeea5SJeff Kirsher * only wait for napi_complete(), this isn't a problem 326514bedbcSBen Hutchings * since efx_nic_eventq_read_ack() will have no effect if 327874aeea5SJeff Kirsher * interrupts have already been disabled. 328874aeea5SJeff Kirsher */ 329874aeea5SJeff Kirsher napi_complete(napi); 330514bedbcSBen Hutchings efx_nic_eventq_read_ack(channel); 331874aeea5SJeff Kirsher } 332874aeea5SJeff Kirsher 33336763266SAlexandre Rames efx_channel_unlock_napi(channel); 334874aeea5SJeff Kirsher return spent; 335874aeea5SJeff Kirsher } 336874aeea5SJeff Kirsher 337874aeea5SJeff Kirsher /* Create event queue 338874aeea5SJeff Kirsher * Event queue memory allocations are done only once. If the channel 339874aeea5SJeff Kirsher * is reset, the memory buffer will be reused; this guards against 340874aeea5SJeff Kirsher * errors during channel reset and also simplifies interrupt handling. 341874aeea5SJeff Kirsher */ 342874aeea5SJeff Kirsher static int efx_probe_eventq(struct efx_channel *channel) 343874aeea5SJeff Kirsher { 344874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 345874aeea5SJeff Kirsher unsigned long entries; 346874aeea5SJeff Kirsher 34786ee5302SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 348874aeea5SJeff Kirsher "chan %d create event queue\n", channel->channel); 349874aeea5SJeff Kirsher 350874aeea5SJeff Kirsher /* Build an event queue with room for one event per tx and rx buffer, 351874aeea5SJeff Kirsher * plus some extra for link state events and MCDI completions. */ 352874aeea5SJeff Kirsher entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 353874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 354874aeea5SJeff Kirsher channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 355874aeea5SJeff Kirsher 356874aeea5SJeff Kirsher return efx_nic_probe_eventq(channel); 357874aeea5SJeff Kirsher } 358874aeea5SJeff Kirsher 359874aeea5SJeff Kirsher /* Prepare channel's event queue */ 360261e4d96SJon Cooper static int efx_init_eventq(struct efx_channel *channel) 361874aeea5SJeff Kirsher { 36215acb1ceSBen Hutchings struct efx_nic *efx = channel->efx; 363261e4d96SJon Cooper int rc; 364261e4d96SJon Cooper 365261e4d96SJon Cooper EFX_WARN_ON_PARANOID(channel->eventq_init); 366261e4d96SJon Cooper 36715acb1ceSBen Hutchings netif_dbg(efx, drv, efx->net_dev, 368874aeea5SJeff Kirsher "chan %d init event queue\n", channel->channel); 369874aeea5SJeff Kirsher 370261e4d96SJon Cooper rc = efx_nic_init_eventq(channel); 371261e4d96SJon Cooper if (rc == 0) { 37215acb1ceSBen Hutchings efx->type->push_irq_moderation(channel); 373874aeea5SJeff Kirsher channel->eventq_read_ptr = 0; 374be3fc09cSBen Hutchings channel->eventq_init = true; 375874aeea5SJeff Kirsher } 376261e4d96SJon Cooper return rc; 377261e4d96SJon Cooper } 378874aeea5SJeff Kirsher 3799f2cb71cSBen Hutchings /* Enable event queue processing and NAPI */ 38036763266SAlexandre Rames void efx_start_eventq(struct efx_channel *channel) 3819f2cb71cSBen Hutchings { 3829f2cb71cSBen Hutchings netif_dbg(channel->efx, ifup, channel->efx->net_dev, 3839f2cb71cSBen Hutchings "chan %d start event queue\n", channel->channel); 3849f2cb71cSBen Hutchings 385514bedbcSBen Hutchings /* Make sure the NAPI handler sees the enabled flag set */ 3869f2cb71cSBen Hutchings channel->enabled = true; 3879f2cb71cSBen Hutchings smp_wmb(); 3889f2cb71cSBen Hutchings 38936763266SAlexandre Rames efx_channel_enable(channel); 3909f2cb71cSBen Hutchings napi_enable(&channel->napi_str); 3919f2cb71cSBen Hutchings efx_nic_eventq_read_ack(channel); 3929f2cb71cSBen Hutchings } 3939f2cb71cSBen Hutchings 3949f2cb71cSBen Hutchings /* Disable event queue processing and NAPI */ 39536763266SAlexandre Rames void efx_stop_eventq(struct efx_channel *channel) 3969f2cb71cSBen Hutchings { 3979f2cb71cSBen Hutchings if (!channel->enabled) 3989f2cb71cSBen Hutchings return; 3999f2cb71cSBen Hutchings 4009f2cb71cSBen Hutchings napi_disable(&channel->napi_str); 40136763266SAlexandre Rames while (!efx_channel_disable(channel)) 40236763266SAlexandre Rames usleep_range(1000, 20000); 4039f2cb71cSBen Hutchings channel->enabled = false; 4049f2cb71cSBen Hutchings } 4059f2cb71cSBen Hutchings 406874aeea5SJeff Kirsher static void efx_fini_eventq(struct efx_channel *channel) 407874aeea5SJeff Kirsher { 408be3fc09cSBen Hutchings if (!channel->eventq_init) 409be3fc09cSBen Hutchings return; 410be3fc09cSBen Hutchings 411874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 412874aeea5SJeff Kirsher "chan %d fini event queue\n", channel->channel); 413874aeea5SJeff Kirsher 414874aeea5SJeff Kirsher efx_nic_fini_eventq(channel); 415be3fc09cSBen Hutchings channel->eventq_init = false; 416874aeea5SJeff Kirsher } 417874aeea5SJeff Kirsher 418874aeea5SJeff Kirsher static void efx_remove_eventq(struct efx_channel *channel) 419874aeea5SJeff Kirsher { 420874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 421874aeea5SJeff Kirsher "chan %d remove event queue\n", channel->channel); 422874aeea5SJeff Kirsher 423874aeea5SJeff Kirsher efx_nic_remove_eventq(channel); 424874aeea5SJeff Kirsher } 425874aeea5SJeff Kirsher 426874aeea5SJeff Kirsher /************************************************************************** 427874aeea5SJeff Kirsher * 428874aeea5SJeff Kirsher * Channel handling 429874aeea5SJeff Kirsher * 430874aeea5SJeff Kirsher *************************************************************************/ 431874aeea5SJeff Kirsher 4327f967c01SBen Hutchings /* Allocate and initialise a channel structure. */ 433874aeea5SJeff Kirsher static struct efx_channel * 434874aeea5SJeff Kirsher efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 435874aeea5SJeff Kirsher { 436874aeea5SJeff Kirsher struct efx_channel *channel; 437874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 438874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 439874aeea5SJeff Kirsher int j; 440874aeea5SJeff Kirsher 4417f967c01SBen Hutchings channel = kzalloc(sizeof(*channel), GFP_KERNEL); 4427f967c01SBen Hutchings if (!channel) 4437f967c01SBen Hutchings return NULL; 4447f967c01SBen Hutchings 4457f967c01SBen Hutchings channel->efx = efx; 4467f967c01SBen Hutchings channel->channel = i; 4477f967c01SBen Hutchings channel->type = &efx_default_channel_type; 4487f967c01SBen Hutchings 4497f967c01SBen Hutchings for (j = 0; j < EFX_TXQ_TYPES; j++) { 4507f967c01SBen Hutchings tx_queue = &channel->tx_queue[j]; 4517f967c01SBen Hutchings tx_queue->efx = efx; 4527f967c01SBen Hutchings tx_queue->queue = i * EFX_TXQ_TYPES + j; 4537f967c01SBen Hutchings tx_queue->channel = channel; 4547f967c01SBen Hutchings } 4557f967c01SBen Hutchings 4567f967c01SBen Hutchings rx_queue = &channel->rx_queue; 4577f967c01SBen Hutchings rx_queue->efx = efx; 4587f967c01SBen Hutchings setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 4597f967c01SBen Hutchings (unsigned long)rx_queue); 4607f967c01SBen Hutchings 4617f967c01SBen Hutchings return channel; 4627f967c01SBen Hutchings } 4637f967c01SBen Hutchings 4647f967c01SBen Hutchings /* Allocate and initialise a channel structure, copying parameters 4657f967c01SBen Hutchings * (but not resources) from an old channel structure. 4667f967c01SBen Hutchings */ 4677f967c01SBen Hutchings static struct efx_channel * 4687f967c01SBen Hutchings efx_copy_channel(const struct efx_channel *old_channel) 4697f967c01SBen Hutchings { 4707f967c01SBen Hutchings struct efx_channel *channel; 4717f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 4727f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 4737f967c01SBen Hutchings int j; 4747f967c01SBen Hutchings 475874aeea5SJeff Kirsher channel = kmalloc(sizeof(*channel), GFP_KERNEL); 476874aeea5SJeff Kirsher if (!channel) 477874aeea5SJeff Kirsher return NULL; 478874aeea5SJeff Kirsher 479874aeea5SJeff Kirsher *channel = *old_channel; 480874aeea5SJeff Kirsher 481874aeea5SJeff Kirsher channel->napi_dev = NULL; 482874aeea5SJeff Kirsher memset(&channel->eventq, 0, sizeof(channel->eventq)); 483874aeea5SJeff Kirsher 484874aeea5SJeff Kirsher for (j = 0; j < EFX_TXQ_TYPES; j++) { 485874aeea5SJeff Kirsher tx_queue = &channel->tx_queue[j]; 486874aeea5SJeff Kirsher if (tx_queue->channel) 487874aeea5SJeff Kirsher tx_queue->channel = channel; 488874aeea5SJeff Kirsher tx_queue->buffer = NULL; 489874aeea5SJeff Kirsher memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 490874aeea5SJeff Kirsher } 491874aeea5SJeff Kirsher 492874aeea5SJeff Kirsher rx_queue = &channel->rx_queue; 4937f967c01SBen Hutchings rx_queue->buffer = NULL; 4947f967c01SBen Hutchings memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 495874aeea5SJeff Kirsher setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 496874aeea5SJeff Kirsher (unsigned long)rx_queue); 497874aeea5SJeff Kirsher 498874aeea5SJeff Kirsher return channel; 499874aeea5SJeff Kirsher } 500874aeea5SJeff Kirsher 501874aeea5SJeff Kirsher static int efx_probe_channel(struct efx_channel *channel) 502874aeea5SJeff Kirsher { 503874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 504874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 505874aeea5SJeff Kirsher int rc; 506874aeea5SJeff Kirsher 507874aeea5SJeff Kirsher netif_dbg(channel->efx, probe, channel->efx->net_dev, 508874aeea5SJeff Kirsher "creating channel %d\n", channel->channel); 509874aeea5SJeff Kirsher 5107f967c01SBen Hutchings rc = channel->type->pre_probe(channel); 5117f967c01SBen Hutchings if (rc) 5127f967c01SBen Hutchings goto fail; 5137f967c01SBen Hutchings 514874aeea5SJeff Kirsher rc = efx_probe_eventq(channel); 515874aeea5SJeff Kirsher if (rc) 5167f967c01SBen Hutchings goto fail; 517874aeea5SJeff Kirsher 518874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) { 519874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 520874aeea5SJeff Kirsher if (rc) 5217f967c01SBen Hutchings goto fail; 522874aeea5SJeff Kirsher } 523874aeea5SJeff Kirsher 524874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) { 525874aeea5SJeff Kirsher rc = efx_probe_rx_queue(rx_queue); 526874aeea5SJeff Kirsher if (rc) 5277f967c01SBen Hutchings goto fail; 528874aeea5SJeff Kirsher } 529874aeea5SJeff Kirsher 530874aeea5SJeff Kirsher return 0; 531874aeea5SJeff Kirsher 5327f967c01SBen Hutchings fail: 5337f967c01SBen Hutchings efx_remove_channel(channel); 534874aeea5SJeff Kirsher return rc; 535874aeea5SJeff Kirsher } 536874aeea5SJeff Kirsher 5377f967c01SBen Hutchings static void 5387f967c01SBen Hutchings efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 5397f967c01SBen Hutchings { 5407f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 5417f967c01SBen Hutchings const char *type; 5427f967c01SBen Hutchings int number; 5437f967c01SBen Hutchings 5447f967c01SBen Hutchings number = channel->channel; 5457f967c01SBen Hutchings if (efx->tx_channel_offset == 0) { 5467f967c01SBen Hutchings type = ""; 5477f967c01SBen Hutchings } else if (channel->channel < efx->tx_channel_offset) { 5487f967c01SBen Hutchings type = "-rx"; 5497f967c01SBen Hutchings } else { 5507f967c01SBen Hutchings type = "-tx"; 5517f967c01SBen Hutchings number -= efx->tx_channel_offset; 5527f967c01SBen Hutchings } 5537f967c01SBen Hutchings snprintf(buf, len, "%s%s-%d", efx->name, type, number); 5547f967c01SBen Hutchings } 555874aeea5SJeff Kirsher 556874aeea5SJeff Kirsher static void efx_set_channel_names(struct efx_nic *efx) 557874aeea5SJeff Kirsher { 558874aeea5SJeff Kirsher struct efx_channel *channel; 559874aeea5SJeff Kirsher 5607f967c01SBen Hutchings efx_for_each_channel(channel, efx) 5617f967c01SBen Hutchings channel->type->get_name(channel, 562d8291187SBen Hutchings efx->msi_context[channel->channel].name, 563d8291187SBen Hutchings sizeof(efx->msi_context[0].name)); 564874aeea5SJeff Kirsher } 565874aeea5SJeff Kirsher 566874aeea5SJeff Kirsher static int efx_probe_channels(struct efx_nic *efx) 567874aeea5SJeff Kirsher { 568874aeea5SJeff Kirsher struct efx_channel *channel; 569874aeea5SJeff Kirsher int rc; 570874aeea5SJeff Kirsher 571874aeea5SJeff Kirsher /* Restart special buffer allocation */ 572874aeea5SJeff Kirsher efx->next_buffer_table = 0; 573874aeea5SJeff Kirsher 574c92aaff1SBen Hutchings /* Probe channels in reverse, so that any 'extra' channels 575c92aaff1SBen Hutchings * use the start of the buffer table. This allows the traffic 576c92aaff1SBen Hutchings * channels to be resized without moving them or wasting the 577c92aaff1SBen Hutchings * entries before them. 578c92aaff1SBen Hutchings */ 579c92aaff1SBen Hutchings efx_for_each_channel_rev(channel, efx) { 580874aeea5SJeff Kirsher rc = efx_probe_channel(channel); 581874aeea5SJeff Kirsher if (rc) { 582874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 583874aeea5SJeff Kirsher "failed to create channel %d\n", 584874aeea5SJeff Kirsher channel->channel); 585874aeea5SJeff Kirsher goto fail; 586874aeea5SJeff Kirsher } 587874aeea5SJeff Kirsher } 588874aeea5SJeff Kirsher efx_set_channel_names(efx); 589874aeea5SJeff Kirsher 590874aeea5SJeff Kirsher return 0; 591874aeea5SJeff Kirsher 592874aeea5SJeff Kirsher fail: 593874aeea5SJeff Kirsher efx_remove_channels(efx); 594874aeea5SJeff Kirsher return rc; 595874aeea5SJeff Kirsher } 596874aeea5SJeff Kirsher 597874aeea5SJeff Kirsher /* Channels are shutdown and reinitialised whilst the NIC is running 598874aeea5SJeff Kirsher * to propagate configuration changes (mtu, checksum offload), or 599874aeea5SJeff Kirsher * to clear hardware error conditions 600874aeea5SJeff Kirsher */ 6019f2cb71cSBen Hutchings static void efx_start_datapath(struct efx_nic *efx) 602874aeea5SJeff Kirsher { 60385740cdfSBen Hutchings bool old_rx_scatter = efx->rx_scatter; 604874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 605874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 606874aeea5SJeff Kirsher struct efx_channel *channel; 60785740cdfSBen Hutchings size_t rx_buf_len; 608874aeea5SJeff Kirsher 609874aeea5SJeff Kirsher /* Calculate the rx buffer allocation parameters required to 610874aeea5SJeff Kirsher * support the current MTU, including padding for header 611874aeea5SJeff Kirsher * alignment and overruns. 612874aeea5SJeff Kirsher */ 61343a3739dSJon Cooper efx->rx_dma_len = (efx->rx_prefix_size + 614874aeea5SJeff Kirsher EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 615874aeea5SJeff Kirsher efx->type->rx_buffer_padding); 61685740cdfSBen Hutchings rx_buf_len = (sizeof(struct efx_rx_page_state) + 6172ec03014SAndrew Rybchenko efx->rx_ip_align + efx->rx_dma_len); 61885740cdfSBen Hutchings if (rx_buf_len <= PAGE_SIZE) { 619e8c68c0aSJon Cooper efx->rx_scatter = efx->type->always_rx_scatter; 62085740cdfSBen Hutchings efx->rx_buffer_order = 0; 62185740cdfSBen Hutchings } else if (efx->type->can_rx_scatter) { 622950c54dfSBen Hutchings BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 62385740cdfSBen Hutchings BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 624950c54dfSBen Hutchings 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 625950c54dfSBen Hutchings EFX_RX_BUF_ALIGNMENT) > 626950c54dfSBen Hutchings PAGE_SIZE); 62785740cdfSBen Hutchings efx->rx_scatter = true; 62885740cdfSBen Hutchings efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 62985740cdfSBen Hutchings efx->rx_buffer_order = 0; 63085740cdfSBen Hutchings } else { 63185740cdfSBen Hutchings efx->rx_scatter = false; 63285740cdfSBen Hutchings efx->rx_buffer_order = get_order(rx_buf_len); 63385740cdfSBen Hutchings } 63485740cdfSBen Hutchings 6351648a23fSDaniel Pieczko efx_rx_config_page_split(efx); 6361648a23fSDaniel Pieczko if (efx->rx_buffer_order) 6371648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6381648a23fSDaniel Pieczko "RX buf len=%u; page order=%u batch=%u\n", 6391648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_buffer_order, 6401648a23fSDaniel Pieczko efx->rx_pages_per_batch); 6411648a23fSDaniel Pieczko else 6421648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6431648a23fSDaniel Pieczko "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 6441648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_page_buf_step, 6451648a23fSDaniel Pieczko efx->rx_bufs_per_page, efx->rx_pages_per_batch); 6462768935aSDaniel Pieczko 647e8c68c0aSJon Cooper /* RX filters may also have scatter-enabled flags */ 64885740cdfSBen Hutchings if (efx->rx_scatter != old_rx_scatter) 649add72477SBen Hutchings efx->type->filter_update_rx_scatter(efx); 650874aeea5SJeff Kirsher 65114bf718fSBen Hutchings /* We must keep at least one descriptor in a TX ring empty. 65214bf718fSBen Hutchings * We could avoid this when the queue size does not exactly 65314bf718fSBen Hutchings * match the hardware ring size, but it's not that important. 65414bf718fSBen Hutchings * Therefore we stop the queue when one more skb might fill 65514bf718fSBen Hutchings * the ring completely. We wake it when half way back to 65614bf718fSBen Hutchings * empty. 65714bf718fSBen Hutchings */ 65814bf718fSBen Hutchings efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 65914bf718fSBen Hutchings efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 66014bf718fSBen Hutchings 661874aeea5SJeff Kirsher /* Initialise the channels */ 662874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 6633881d8abSAlexandre Rames efx_for_each_channel_tx_queue(tx_queue, channel) { 664874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 6653881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 6663881d8abSAlexandre Rames } 667874aeea5SJeff Kirsher 6689f2cb71cSBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) { 669874aeea5SJeff Kirsher efx_init_rx_queue(rx_queue); 6703881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 671cce28794SJon Cooper efx_stop_eventq(channel); 672cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, false); 673cce28794SJon Cooper efx_start_eventq(channel); 6749f2cb71cSBen Hutchings } 675874aeea5SJeff Kirsher 67685740cdfSBen Hutchings WARN_ON(channel->rx_pkt_n_frags); 677874aeea5SJeff Kirsher } 6789f2cb71cSBen Hutchings 6792ea4dc28SAlexandre Rames efx_ptp_start_datapath(efx); 6802ea4dc28SAlexandre Rames 6819f2cb71cSBen Hutchings if (netif_device_present(efx->net_dev)) 6829f2cb71cSBen Hutchings netif_tx_wake_all_queues(efx->net_dev); 683874aeea5SJeff Kirsher } 684874aeea5SJeff Kirsher 6859f2cb71cSBen Hutchings static void efx_stop_datapath(struct efx_nic *efx) 686874aeea5SJeff Kirsher { 687874aeea5SJeff Kirsher struct efx_channel *channel; 688874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 689874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 690874aeea5SJeff Kirsher int rc; 691874aeea5SJeff Kirsher 692874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 693874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 694874aeea5SJeff Kirsher 6952ea4dc28SAlexandre Rames efx_ptp_stop_datapath(efx); 6962ea4dc28SAlexandre Rames 697d8aec745SBen Hutchings /* Stop RX refill */ 698d8aec745SBen Hutchings efx_for_each_channel(channel, efx) { 699d8aec745SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 700d8aec745SBen Hutchings rx_queue->refill_enabled = false; 701d8aec745SBen Hutchings } 702d8aec745SBen Hutchings 703874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 7049f2cb71cSBen Hutchings /* RX packet processing is pipelined, so wait for the 7059f2cb71cSBen Hutchings * NAPI handler to complete. At least event queue 0 7069f2cb71cSBen Hutchings * might be kept active by non-data events, so don't 7079f2cb71cSBen Hutchings * use napi_synchronize() but actually disable NAPI 7089f2cb71cSBen Hutchings * temporarily. 7099f2cb71cSBen Hutchings */ 7109f2cb71cSBen Hutchings if (efx_channel_has_rx_queue(channel)) { 7119f2cb71cSBen Hutchings efx_stop_eventq(channel); 7129f2cb71cSBen Hutchings efx_start_eventq(channel); 7139f2cb71cSBen Hutchings } 714e42c3d85SBen Hutchings } 715874aeea5SJeff Kirsher 716e42c3d85SBen Hutchings rc = efx->type->fini_dmaq(efx); 717e42c3d85SBen Hutchings if (rc && EFX_WORKAROUND_7803(efx)) { 718e42c3d85SBen Hutchings /* Schedule a reset to recover from the flush failure. The 719e42c3d85SBen Hutchings * descriptor caches reference memory we're about to free, 720e42c3d85SBen Hutchings * but falcon_reconfigure_mac_wrapper() won't reconnect 721e42c3d85SBen Hutchings * the MACs because of the pending reset. 722e42c3d85SBen Hutchings */ 723e42c3d85SBen Hutchings netif_err(efx, drv, efx->net_dev, 724e42c3d85SBen Hutchings "Resetting to recover from flush failure\n"); 725e42c3d85SBen Hutchings efx_schedule_reset(efx, RESET_TYPE_ALL); 726e42c3d85SBen Hutchings } else if (rc) { 727e42c3d85SBen Hutchings netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 728e42c3d85SBen Hutchings } else { 729e42c3d85SBen Hutchings netif_dbg(efx, drv, efx->net_dev, 730e42c3d85SBen Hutchings "successfully flushed all queues\n"); 731e42c3d85SBen Hutchings } 732e42c3d85SBen Hutchings 733e42c3d85SBen Hutchings efx_for_each_channel(channel, efx) { 734874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 735874aeea5SJeff Kirsher efx_fini_rx_queue(rx_queue); 736874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 737874aeea5SJeff Kirsher efx_fini_tx_queue(tx_queue); 738874aeea5SJeff Kirsher } 739874aeea5SJeff Kirsher } 740874aeea5SJeff Kirsher 741874aeea5SJeff Kirsher static void efx_remove_channel(struct efx_channel *channel) 742874aeea5SJeff Kirsher { 743874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 744874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 745874aeea5SJeff Kirsher 746874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 747874aeea5SJeff Kirsher "destroy chan %d\n", channel->channel); 748874aeea5SJeff Kirsher 749874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 750874aeea5SJeff Kirsher efx_remove_rx_queue(rx_queue); 751874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 752874aeea5SJeff Kirsher efx_remove_tx_queue(tx_queue); 753874aeea5SJeff Kirsher efx_remove_eventq(channel); 754c31e5f9fSStuart Hodgson channel->type->post_remove(channel); 755874aeea5SJeff Kirsher } 756874aeea5SJeff Kirsher 757874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx) 758874aeea5SJeff Kirsher { 759874aeea5SJeff Kirsher struct efx_channel *channel; 760874aeea5SJeff Kirsher 761874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 762874aeea5SJeff Kirsher efx_remove_channel(channel); 763874aeea5SJeff Kirsher } 764874aeea5SJeff Kirsher 765874aeea5SJeff Kirsher int 766874aeea5SJeff Kirsher efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 767874aeea5SJeff Kirsher { 768874aeea5SJeff Kirsher struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 769874aeea5SJeff Kirsher u32 old_rxq_entries, old_txq_entries; 7707f967c01SBen Hutchings unsigned i, next_buffer_table = 0; 771261e4d96SJon Cooper int rc, rc2; 7728b7325b4SBen Hutchings 7738b7325b4SBen Hutchings rc = efx_check_disabled(efx); 7748b7325b4SBen Hutchings if (rc) 7758b7325b4SBen Hutchings return rc; 7767f967c01SBen Hutchings 7777f967c01SBen Hutchings /* Not all channels should be reallocated. We must avoid 7787f967c01SBen Hutchings * reallocating their buffer table entries. 7797f967c01SBen Hutchings */ 7807f967c01SBen Hutchings efx_for_each_channel(channel, efx) { 7817f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 7827f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 7837f967c01SBen Hutchings 7847f967c01SBen Hutchings if (channel->type->copy) 7857f967c01SBen Hutchings continue; 7867f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7877f967c01SBen Hutchings channel->eventq.index + 7887f967c01SBen Hutchings channel->eventq.entries); 7897f967c01SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 7907f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7917f967c01SBen Hutchings rx_queue->rxd.index + 7927f967c01SBen Hutchings rx_queue->rxd.entries); 7937f967c01SBen Hutchings efx_for_each_channel_tx_queue(tx_queue, channel) 7947f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7957f967c01SBen Hutchings tx_queue->txd.index + 7967f967c01SBen Hutchings tx_queue->txd.entries); 7977f967c01SBen Hutchings } 798874aeea5SJeff Kirsher 79929c69a48SBen Hutchings efx_device_detach_sync(efx); 800874aeea5SJeff Kirsher efx_stop_all(efx); 801d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 802874aeea5SJeff Kirsher 8037f967c01SBen Hutchings /* Clone channels (where possible) */ 804874aeea5SJeff Kirsher memset(other_channel, 0, sizeof(other_channel)); 805874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 8067f967c01SBen Hutchings channel = efx->channel[i]; 8077f967c01SBen Hutchings if (channel->type->copy) 8087f967c01SBen Hutchings channel = channel->type->copy(channel); 809874aeea5SJeff Kirsher if (!channel) { 810874aeea5SJeff Kirsher rc = -ENOMEM; 811874aeea5SJeff Kirsher goto out; 812874aeea5SJeff Kirsher } 813874aeea5SJeff Kirsher other_channel[i] = channel; 814874aeea5SJeff Kirsher } 815874aeea5SJeff Kirsher 816874aeea5SJeff Kirsher /* Swap entry counts and channel pointers */ 817874aeea5SJeff Kirsher old_rxq_entries = efx->rxq_entries; 818874aeea5SJeff Kirsher old_txq_entries = efx->txq_entries; 819874aeea5SJeff Kirsher efx->rxq_entries = rxq_entries; 820874aeea5SJeff Kirsher efx->txq_entries = txq_entries; 821874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 822874aeea5SJeff Kirsher channel = efx->channel[i]; 823874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 824874aeea5SJeff Kirsher other_channel[i] = channel; 825874aeea5SJeff Kirsher } 826874aeea5SJeff Kirsher 8277f967c01SBen Hutchings /* Restart buffer table allocation */ 8287f967c01SBen Hutchings efx->next_buffer_table = next_buffer_table; 8297f967c01SBen Hutchings 8307f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8317f967c01SBen Hutchings channel = efx->channel[i]; 8327f967c01SBen Hutchings if (!channel->type->copy) 8337f967c01SBen Hutchings continue; 8347f967c01SBen Hutchings rc = efx_probe_channel(channel); 835874aeea5SJeff Kirsher if (rc) 836874aeea5SJeff Kirsher goto rollback; 8377f967c01SBen Hutchings efx_init_napi_channel(efx->channel[i]); 838874aeea5SJeff Kirsher } 839874aeea5SJeff Kirsher 8407f967c01SBen Hutchings out: 8417f967c01SBen Hutchings /* Destroy unused channel structures */ 8427f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8437f967c01SBen Hutchings channel = other_channel[i]; 8447f967c01SBen Hutchings if (channel && channel->type->copy) { 8457f967c01SBen Hutchings efx_fini_napi_channel(channel); 8467f967c01SBen Hutchings efx_remove_channel(channel); 8477f967c01SBen Hutchings kfree(channel); 8487f967c01SBen Hutchings } 8497f967c01SBen Hutchings } 8507f967c01SBen Hutchings 851261e4d96SJon Cooper rc2 = efx_soft_enable_interrupts(efx); 852261e4d96SJon Cooper if (rc2) { 853261e4d96SJon Cooper rc = rc ? rc : rc2; 854261e4d96SJon Cooper netif_err(efx, drv, efx->net_dev, 855261e4d96SJon Cooper "unable to restart interrupts on channel reallocation\n"); 856261e4d96SJon Cooper efx_schedule_reset(efx, RESET_TYPE_DISABLE); 857261e4d96SJon Cooper } else { 858874aeea5SJeff Kirsher efx_start_all(efx); 85929c69a48SBen Hutchings netif_device_attach(efx->net_dev); 860261e4d96SJon Cooper } 861874aeea5SJeff Kirsher return rc; 862874aeea5SJeff Kirsher 863874aeea5SJeff Kirsher rollback: 864874aeea5SJeff Kirsher /* Swap back */ 865874aeea5SJeff Kirsher efx->rxq_entries = old_rxq_entries; 866874aeea5SJeff Kirsher efx->txq_entries = old_txq_entries; 867874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 868874aeea5SJeff Kirsher channel = efx->channel[i]; 869874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 870874aeea5SJeff Kirsher other_channel[i] = channel; 871874aeea5SJeff Kirsher } 872874aeea5SJeff Kirsher goto out; 873874aeea5SJeff Kirsher } 874874aeea5SJeff Kirsher 875874aeea5SJeff Kirsher void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 876874aeea5SJeff Kirsher { 877874aeea5SJeff Kirsher mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 878874aeea5SJeff Kirsher } 879874aeea5SJeff Kirsher 8807f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type = { 8817f967c01SBen Hutchings .pre_probe = efx_channel_dummy_op_int, 882c31e5f9fSStuart Hodgson .post_remove = efx_channel_dummy_op_void, 8837f967c01SBen Hutchings .get_name = efx_get_channel_name, 8847f967c01SBen Hutchings .copy = efx_copy_channel, 8857f967c01SBen Hutchings .keep_eventq = false, 8867f967c01SBen Hutchings }; 8877f967c01SBen Hutchings 8887f967c01SBen Hutchings int efx_channel_dummy_op_int(struct efx_channel *channel) 8897f967c01SBen Hutchings { 8907f967c01SBen Hutchings return 0; 8917f967c01SBen Hutchings } 8927f967c01SBen Hutchings 893c31e5f9fSStuart Hodgson void efx_channel_dummy_op_void(struct efx_channel *channel) 894c31e5f9fSStuart Hodgson { 895c31e5f9fSStuart Hodgson } 896c31e5f9fSStuart Hodgson 897874aeea5SJeff Kirsher /************************************************************************** 898874aeea5SJeff Kirsher * 899874aeea5SJeff Kirsher * Port handling 900874aeea5SJeff Kirsher * 901874aeea5SJeff Kirsher **************************************************************************/ 902874aeea5SJeff Kirsher 903874aeea5SJeff Kirsher /* This ensures that the kernel is kept informed (via 904874aeea5SJeff Kirsher * netif_carrier_on/off) of the link status, and also maintains the 905874aeea5SJeff Kirsher * link status's stop on the port's TX queue. 906874aeea5SJeff Kirsher */ 907874aeea5SJeff Kirsher void efx_link_status_changed(struct efx_nic *efx) 908874aeea5SJeff Kirsher { 909874aeea5SJeff Kirsher struct efx_link_state *link_state = &efx->link_state; 910874aeea5SJeff Kirsher 911874aeea5SJeff Kirsher /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 912874aeea5SJeff Kirsher * that no events are triggered between unregister_netdev() and the 913874aeea5SJeff Kirsher * driver unloading. A more general condition is that NETDEV_CHANGE 914874aeea5SJeff Kirsher * can only be generated between NETDEV_UP and NETDEV_DOWN */ 915874aeea5SJeff Kirsher if (!netif_running(efx->net_dev)) 916874aeea5SJeff Kirsher return; 917874aeea5SJeff Kirsher 918874aeea5SJeff Kirsher if (link_state->up != netif_carrier_ok(efx->net_dev)) { 919874aeea5SJeff Kirsher efx->n_link_state_changes++; 920874aeea5SJeff Kirsher 921874aeea5SJeff Kirsher if (link_state->up) 922874aeea5SJeff Kirsher netif_carrier_on(efx->net_dev); 923874aeea5SJeff Kirsher else 924874aeea5SJeff Kirsher netif_carrier_off(efx->net_dev); 925874aeea5SJeff Kirsher } 926874aeea5SJeff Kirsher 927874aeea5SJeff Kirsher /* Status message for kernel log */ 9282aa9ef11SBen Hutchings if (link_state->up) 929874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, 930964e6135SBen Hutchings "link up at %uMbps %s-duplex (MTU %d)\n", 931874aeea5SJeff Kirsher link_state->speed, link_state->fd ? "full" : "half", 932964e6135SBen Hutchings efx->net_dev->mtu); 9332aa9ef11SBen Hutchings else 934874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, "link down\n"); 935874aeea5SJeff Kirsher } 936874aeea5SJeff Kirsher 937874aeea5SJeff Kirsher void efx_link_set_advertising(struct efx_nic *efx, u32 advertising) 938874aeea5SJeff Kirsher { 939874aeea5SJeff Kirsher efx->link_advertising = advertising; 940874aeea5SJeff Kirsher if (advertising) { 941874aeea5SJeff Kirsher if (advertising & ADVERTISED_Pause) 942874aeea5SJeff Kirsher efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 943874aeea5SJeff Kirsher else 944874aeea5SJeff Kirsher efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 945874aeea5SJeff Kirsher if (advertising & ADVERTISED_Asym_Pause) 946874aeea5SJeff Kirsher efx->wanted_fc ^= EFX_FC_TX; 947874aeea5SJeff Kirsher } 948874aeea5SJeff Kirsher } 949874aeea5SJeff Kirsher 950874aeea5SJeff Kirsher void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 951874aeea5SJeff Kirsher { 952874aeea5SJeff Kirsher efx->wanted_fc = wanted_fc; 953874aeea5SJeff Kirsher if (efx->link_advertising) { 954874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_RX) 955874aeea5SJeff Kirsher efx->link_advertising |= (ADVERTISED_Pause | 956874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 957874aeea5SJeff Kirsher else 958874aeea5SJeff Kirsher efx->link_advertising &= ~(ADVERTISED_Pause | 959874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 960874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_TX) 961874aeea5SJeff Kirsher efx->link_advertising ^= ADVERTISED_Asym_Pause; 962874aeea5SJeff Kirsher } 963874aeea5SJeff Kirsher } 964874aeea5SJeff Kirsher 965874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx); 966874aeea5SJeff Kirsher 9670d322413SEdward Cree /* We assume that efx->type->reconfigure_mac will always try to sync RX 9680d322413SEdward Cree * filters and therefore needs to read-lock the filter table against freeing 9690d322413SEdward Cree */ 9700d322413SEdward Cree void efx_mac_reconfigure(struct efx_nic *efx) 9710d322413SEdward Cree { 9720d322413SEdward Cree down_read(&efx->filter_sem); 9730d322413SEdward Cree efx->type->reconfigure_mac(efx); 9740d322413SEdward Cree up_read(&efx->filter_sem); 9750d322413SEdward Cree } 9760d322413SEdward Cree 977874aeea5SJeff Kirsher /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 978874aeea5SJeff Kirsher * the MAC appropriately. All other PHY configuration changes are pushed 979874aeea5SJeff Kirsher * through phy_op->set_settings(), and pushed asynchronously to the MAC 980874aeea5SJeff Kirsher * through efx_monitor(). 981874aeea5SJeff Kirsher * 982874aeea5SJeff Kirsher * Callers must hold the mac_lock 983874aeea5SJeff Kirsher */ 984874aeea5SJeff Kirsher int __efx_reconfigure_port(struct efx_nic *efx) 985874aeea5SJeff Kirsher { 986874aeea5SJeff Kirsher enum efx_phy_mode phy_mode; 987874aeea5SJeff Kirsher int rc; 988874aeea5SJeff Kirsher 989874aeea5SJeff Kirsher WARN_ON(!mutex_is_locked(&efx->mac_lock)); 990874aeea5SJeff Kirsher 991874aeea5SJeff Kirsher /* Disable PHY transmit in mac level loopbacks */ 992874aeea5SJeff Kirsher phy_mode = efx->phy_mode; 993874aeea5SJeff Kirsher if (LOOPBACK_INTERNAL(efx)) 994874aeea5SJeff Kirsher efx->phy_mode |= PHY_MODE_TX_DISABLED; 995874aeea5SJeff Kirsher else 996874aeea5SJeff Kirsher efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 997874aeea5SJeff Kirsher 998874aeea5SJeff Kirsher rc = efx->type->reconfigure_port(efx); 999874aeea5SJeff Kirsher 1000874aeea5SJeff Kirsher if (rc) 1001874aeea5SJeff Kirsher efx->phy_mode = phy_mode; 1002874aeea5SJeff Kirsher 1003874aeea5SJeff Kirsher return rc; 1004874aeea5SJeff Kirsher } 1005874aeea5SJeff Kirsher 1006874aeea5SJeff Kirsher /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1007874aeea5SJeff Kirsher * disabled. */ 1008874aeea5SJeff Kirsher int efx_reconfigure_port(struct efx_nic *efx) 1009874aeea5SJeff Kirsher { 1010874aeea5SJeff Kirsher int rc; 1011874aeea5SJeff Kirsher 1012874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1013874aeea5SJeff Kirsher 1014874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1015874aeea5SJeff Kirsher rc = __efx_reconfigure_port(efx); 1016874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1017874aeea5SJeff Kirsher 1018874aeea5SJeff Kirsher return rc; 1019874aeea5SJeff Kirsher } 1020874aeea5SJeff Kirsher 1021874aeea5SJeff Kirsher /* Asynchronous work item for changing MAC promiscuity and multicast 1022874aeea5SJeff Kirsher * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1023874aeea5SJeff Kirsher * MAC directly. */ 1024874aeea5SJeff Kirsher static void efx_mac_work(struct work_struct *data) 1025874aeea5SJeff Kirsher { 1026874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1027874aeea5SJeff Kirsher 1028874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 102930b81cdaSBen Hutchings if (efx->port_enabled) 10300d322413SEdward Cree efx_mac_reconfigure(efx); 1031874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1032874aeea5SJeff Kirsher } 1033874aeea5SJeff Kirsher 1034874aeea5SJeff Kirsher static int efx_probe_port(struct efx_nic *efx) 1035874aeea5SJeff Kirsher { 1036874aeea5SJeff Kirsher int rc; 1037874aeea5SJeff Kirsher 1038874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1039874aeea5SJeff Kirsher 1040874aeea5SJeff Kirsher if (phy_flash_cfg) 1041874aeea5SJeff Kirsher efx->phy_mode = PHY_MODE_SPECIAL; 1042874aeea5SJeff Kirsher 1043874aeea5SJeff Kirsher /* Connect up MAC/PHY operations table */ 1044874aeea5SJeff Kirsher rc = efx->type->probe_port(efx); 1045874aeea5SJeff Kirsher if (rc) 1046874aeea5SJeff Kirsher return rc; 1047874aeea5SJeff Kirsher 1048e332bcb3SBen Hutchings /* Initialise MAC address to permanent address */ 1049cd84ff4dSEdward Cree ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1050874aeea5SJeff Kirsher 1051874aeea5SJeff Kirsher return 0; 1052874aeea5SJeff Kirsher } 1053874aeea5SJeff Kirsher 1054874aeea5SJeff Kirsher static int efx_init_port(struct efx_nic *efx) 1055874aeea5SJeff Kirsher { 1056874aeea5SJeff Kirsher int rc; 1057874aeea5SJeff Kirsher 1058874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1059874aeea5SJeff Kirsher 1060874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1061874aeea5SJeff Kirsher 1062874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 1063874aeea5SJeff Kirsher if (rc) 1064874aeea5SJeff Kirsher goto fail1; 1065874aeea5SJeff Kirsher 1066874aeea5SJeff Kirsher efx->port_initialized = true; 1067874aeea5SJeff Kirsher 1068874aeea5SJeff Kirsher /* Reconfigure the MAC before creating dma queues (required for 1069874aeea5SJeff Kirsher * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 10700d322413SEdward Cree efx_mac_reconfigure(efx); 1071874aeea5SJeff Kirsher 1072874aeea5SJeff Kirsher /* Ensure the PHY advertises the correct flow control settings */ 1073874aeea5SJeff Kirsher rc = efx->phy_op->reconfigure(efx); 1074267d9d73SEdward Cree if (rc && rc != -EPERM) 1075874aeea5SJeff Kirsher goto fail2; 1076874aeea5SJeff Kirsher 1077874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1078874aeea5SJeff Kirsher return 0; 1079874aeea5SJeff Kirsher 1080874aeea5SJeff Kirsher fail2: 1081874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1082874aeea5SJeff Kirsher fail1: 1083874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1084874aeea5SJeff Kirsher return rc; 1085874aeea5SJeff Kirsher } 1086874aeea5SJeff Kirsher 1087874aeea5SJeff Kirsher static void efx_start_port(struct efx_nic *efx) 1088874aeea5SJeff Kirsher { 1089874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1090874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 1091874aeea5SJeff Kirsher 1092874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1093874aeea5SJeff Kirsher efx->port_enabled = true; 1094874aeea5SJeff Kirsher 1095d615c039SBen Hutchings /* Ensure MAC ingress/egress is enabled */ 10960d322413SEdward Cree efx_mac_reconfigure(efx); 1097874aeea5SJeff Kirsher 1098874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1099874aeea5SJeff Kirsher } 1100874aeea5SJeff Kirsher 1101d615c039SBen Hutchings /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1102d615c039SBen Hutchings * and the async self-test, wait for them to finish and prevent them 1103d615c039SBen Hutchings * being scheduled again. This doesn't cover online resets, which 1104d615c039SBen Hutchings * should only be cancelled when removing the device. 1105d615c039SBen Hutchings */ 1106874aeea5SJeff Kirsher static void efx_stop_port(struct efx_nic *efx) 1107874aeea5SJeff Kirsher { 1108874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1109874aeea5SJeff Kirsher 1110d615c039SBen Hutchings EFX_ASSERT_RESET_SERIALISED(efx); 1111d615c039SBen Hutchings 1112874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1113874aeea5SJeff Kirsher efx->port_enabled = false; 1114874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1115874aeea5SJeff Kirsher 1116874aeea5SJeff Kirsher /* Serialise against efx_set_multicast_list() */ 1117874aeea5SJeff Kirsher netif_addr_lock_bh(efx->net_dev); 1118874aeea5SJeff Kirsher netif_addr_unlock_bh(efx->net_dev); 1119d615c039SBen Hutchings 1120d615c039SBen Hutchings cancel_delayed_work_sync(&efx->monitor_work); 1121d615c039SBen Hutchings efx_selftest_async_cancel(efx); 1122d615c039SBen Hutchings cancel_work_sync(&efx->mac_work); 1123874aeea5SJeff Kirsher } 1124874aeea5SJeff Kirsher 1125874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx) 1126874aeea5SJeff Kirsher { 1127874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1128874aeea5SJeff Kirsher 1129874aeea5SJeff Kirsher if (!efx->port_initialized) 1130874aeea5SJeff Kirsher return; 1131874aeea5SJeff Kirsher 1132874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1133874aeea5SJeff Kirsher efx->port_initialized = false; 1134874aeea5SJeff Kirsher 1135874aeea5SJeff Kirsher efx->link_state.up = false; 1136874aeea5SJeff Kirsher efx_link_status_changed(efx); 1137874aeea5SJeff Kirsher } 1138874aeea5SJeff Kirsher 1139874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx) 1140874aeea5SJeff Kirsher { 1141874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1142874aeea5SJeff Kirsher 1143874aeea5SJeff Kirsher efx->type->remove_port(efx); 1144874aeea5SJeff Kirsher } 1145874aeea5SJeff Kirsher 1146874aeea5SJeff Kirsher /************************************************************************** 1147874aeea5SJeff Kirsher * 1148874aeea5SJeff Kirsher * NIC handling 1149874aeea5SJeff Kirsher * 1150874aeea5SJeff Kirsher **************************************************************************/ 1151874aeea5SJeff Kirsher 11520bcf4a64SBen Hutchings static LIST_HEAD(efx_primary_list); 11530bcf4a64SBen Hutchings static LIST_HEAD(efx_unassociated_list); 11540bcf4a64SBen Hutchings 11550bcf4a64SBen Hutchings static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 11560bcf4a64SBen Hutchings { 11570bcf4a64SBen Hutchings return left->type == right->type && 11580bcf4a64SBen Hutchings left->vpd_sn && right->vpd_sn && 11590bcf4a64SBen Hutchings !strcmp(left->vpd_sn, right->vpd_sn); 11600bcf4a64SBen Hutchings } 11610bcf4a64SBen Hutchings 11620bcf4a64SBen Hutchings static void efx_associate(struct efx_nic *efx) 11630bcf4a64SBen Hutchings { 11640bcf4a64SBen Hutchings struct efx_nic *other, *next; 11650bcf4a64SBen Hutchings 11660bcf4a64SBen Hutchings if (efx->primary == efx) { 11670bcf4a64SBen Hutchings /* Adding primary function; look for secondaries */ 11680bcf4a64SBen Hutchings 11690bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 11700bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_primary_list); 11710bcf4a64SBen Hutchings 11720bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx_unassociated_list, 11730bcf4a64SBen Hutchings node) { 11740bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 11750bcf4a64SBen Hutchings list_del(&other->node); 11760bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 11770bcf4a64SBen Hutchings "moving to secondary list of %s %s\n", 11780bcf4a64SBen Hutchings pci_name(efx->pci_dev), 11790bcf4a64SBen Hutchings efx->net_dev->name); 11800bcf4a64SBen Hutchings list_add_tail(&other->node, 11810bcf4a64SBen Hutchings &efx->secondary_list); 11820bcf4a64SBen Hutchings other->primary = efx; 11830bcf4a64SBen Hutchings } 11840bcf4a64SBen Hutchings } 11850bcf4a64SBen Hutchings } else { 11860bcf4a64SBen Hutchings /* Adding secondary function; look for primary */ 11870bcf4a64SBen Hutchings 11880bcf4a64SBen Hutchings list_for_each_entry(other, &efx_primary_list, node) { 11890bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 11900bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 11910bcf4a64SBen Hutchings "adding to secondary list of %s %s\n", 11920bcf4a64SBen Hutchings pci_name(other->pci_dev), 11930bcf4a64SBen Hutchings other->net_dev->name); 11940bcf4a64SBen Hutchings list_add_tail(&efx->node, 11950bcf4a64SBen Hutchings &other->secondary_list); 11960bcf4a64SBen Hutchings efx->primary = other; 11970bcf4a64SBen Hutchings return; 11980bcf4a64SBen Hutchings } 11990bcf4a64SBen Hutchings } 12000bcf4a64SBen Hutchings 12010bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 12020bcf4a64SBen Hutchings "adding to unassociated list\n"); 12030bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_unassociated_list); 12040bcf4a64SBen Hutchings } 12050bcf4a64SBen Hutchings } 12060bcf4a64SBen Hutchings 12070bcf4a64SBen Hutchings static void efx_dissociate(struct efx_nic *efx) 12080bcf4a64SBen Hutchings { 12090bcf4a64SBen Hutchings struct efx_nic *other, *next; 12100bcf4a64SBen Hutchings 12110bcf4a64SBen Hutchings list_del(&efx->node); 12120bcf4a64SBen Hutchings efx->primary = NULL; 12130bcf4a64SBen Hutchings 12140bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 12150bcf4a64SBen Hutchings list_del(&other->node); 12160bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 12170bcf4a64SBen Hutchings "moving to unassociated list\n"); 12180bcf4a64SBen Hutchings list_add_tail(&other->node, &efx_unassociated_list); 12190bcf4a64SBen Hutchings other->primary = NULL; 12200bcf4a64SBen Hutchings } 12210bcf4a64SBen Hutchings } 12220bcf4a64SBen Hutchings 1223874aeea5SJeff Kirsher /* This configures the PCI device to enable I/O and DMA. */ 1224874aeea5SJeff Kirsher static int efx_init_io(struct efx_nic *efx) 1225874aeea5SJeff Kirsher { 1226874aeea5SJeff Kirsher struct pci_dev *pci_dev = efx->pci_dev; 1227874aeea5SJeff Kirsher dma_addr_t dma_mask = efx->type->max_dma_mask; 1228b105798fSBen Hutchings unsigned int mem_map_size = efx->type->mem_map_size(efx); 122902246a7fSShradha Shah int rc, bar; 1230874aeea5SJeff Kirsher 1231874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1232874aeea5SJeff Kirsher 123302246a7fSShradha Shah bar = efx->type->mem_bar; 123402246a7fSShradha Shah 1235874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 1236874aeea5SJeff Kirsher if (rc) { 1237874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1238874aeea5SJeff Kirsher "failed to enable PCI device\n"); 1239874aeea5SJeff Kirsher goto fail1; 1240874aeea5SJeff Kirsher } 1241874aeea5SJeff Kirsher 1242874aeea5SJeff Kirsher pci_set_master(pci_dev); 1243874aeea5SJeff Kirsher 1244874aeea5SJeff Kirsher /* Set the PCI DMA mask. Try all possibilities from our 1245874aeea5SJeff Kirsher * genuine mask down to 32 bits, because some architectures 1246874aeea5SJeff Kirsher * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1247874aeea5SJeff Kirsher * masks event though they reject 46 bit masks. 1248874aeea5SJeff Kirsher */ 1249874aeea5SJeff Kirsher while (dma_mask > 0x7fffffffUL) { 12509663dedaSRussell King rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1251e9e01846SBen Hutchings if (rc == 0) 1252874aeea5SJeff Kirsher break; 1253874aeea5SJeff Kirsher dma_mask >>= 1; 1254874aeea5SJeff Kirsher } 1255874aeea5SJeff Kirsher if (rc) { 1256874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1257874aeea5SJeff Kirsher "could not find a suitable DMA mask\n"); 1258874aeea5SJeff Kirsher goto fail2; 1259874aeea5SJeff Kirsher } 1260874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1261874aeea5SJeff Kirsher "using DMA mask %llx\n", (unsigned long long) dma_mask); 1262874aeea5SJeff Kirsher 126302246a7fSShradha Shah efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 126402246a7fSShradha Shah rc = pci_request_region(pci_dev, bar, "sfc"); 1265874aeea5SJeff Kirsher if (rc) { 1266874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1267874aeea5SJeff Kirsher "request for memory BAR failed\n"); 1268874aeea5SJeff Kirsher rc = -EIO; 1269874aeea5SJeff Kirsher goto fail3; 1270874aeea5SJeff Kirsher } 1271b105798fSBen Hutchings efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1272874aeea5SJeff Kirsher if (!efx->membase) { 1273874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1274874aeea5SJeff Kirsher "could not map memory BAR at %llx+%x\n", 1275b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size); 1276874aeea5SJeff Kirsher rc = -ENOMEM; 1277874aeea5SJeff Kirsher goto fail4; 1278874aeea5SJeff Kirsher } 1279874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1280874aeea5SJeff Kirsher "memory BAR at %llx+%x (virtual %p)\n", 1281b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size, 1282b105798fSBen Hutchings efx->membase); 1283874aeea5SJeff Kirsher 1284874aeea5SJeff Kirsher return 0; 1285874aeea5SJeff Kirsher 1286874aeea5SJeff Kirsher fail4: 128702246a7fSShradha Shah pci_release_region(efx->pci_dev, bar); 1288874aeea5SJeff Kirsher fail3: 1289874aeea5SJeff Kirsher efx->membase_phys = 0; 1290874aeea5SJeff Kirsher fail2: 1291874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1292874aeea5SJeff Kirsher fail1: 1293874aeea5SJeff Kirsher return rc; 1294874aeea5SJeff Kirsher } 1295874aeea5SJeff Kirsher 1296874aeea5SJeff Kirsher static void efx_fini_io(struct efx_nic *efx) 1297874aeea5SJeff Kirsher { 129802246a7fSShradha Shah int bar; 129902246a7fSShradha Shah 1300874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1301874aeea5SJeff Kirsher 1302874aeea5SJeff Kirsher if (efx->membase) { 1303874aeea5SJeff Kirsher iounmap(efx->membase); 1304874aeea5SJeff Kirsher efx->membase = NULL; 1305874aeea5SJeff Kirsher } 1306874aeea5SJeff Kirsher 1307874aeea5SJeff Kirsher if (efx->membase_phys) { 130802246a7fSShradha Shah bar = efx->type->mem_bar; 130902246a7fSShradha Shah pci_release_region(efx->pci_dev, bar); 1310874aeea5SJeff Kirsher efx->membase_phys = 0; 1311874aeea5SJeff Kirsher } 1312874aeea5SJeff Kirsher 13136598dad2SDaniel Pieczko /* Don't disable bus-mastering if VFs are assigned */ 13146598dad2SDaniel Pieczko if (!pci_vfs_assigned(efx->pci_dev)) 1315874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1316874aeea5SJeff Kirsher } 1317874aeea5SJeff Kirsher 1318267c0157SJon Cooper void efx_set_default_rx_indir_table(struct efx_nic *efx) 1319267c0157SJon Cooper { 1320267c0157SJon Cooper size_t i; 1321267c0157SJon Cooper 1322267c0157SJon Cooper for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1323267c0157SJon Cooper efx->rx_indir_table[i] = 1324267c0157SJon Cooper ethtool_rxfh_indir_default(i, efx->rss_spread); 1325267c0157SJon Cooper } 1326267c0157SJon Cooper 1327a9a52506SBen Hutchings static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1328874aeea5SJeff Kirsher { 1329cdb08f8fSBen Hutchings cpumask_var_t thread_mask; 1330a16e5b24SBen Hutchings unsigned int count; 1331874aeea5SJeff Kirsher int cpu; 1332874aeea5SJeff Kirsher 1333cd2d5b52SBen Hutchings if (rss_cpus) { 1334cd2d5b52SBen Hutchings count = rss_cpus; 1335cd2d5b52SBen Hutchings } else { 1336cdb08f8fSBen Hutchings if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1337a9a52506SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1338a9a52506SBen Hutchings "RSS disabled due to allocation failure\n"); 1339874aeea5SJeff Kirsher return 1; 1340874aeea5SJeff Kirsher } 1341874aeea5SJeff Kirsher 1342874aeea5SJeff Kirsher count = 0; 1343874aeea5SJeff Kirsher for_each_online_cpu(cpu) { 1344cdb08f8fSBen Hutchings if (!cpumask_test_cpu(cpu, thread_mask)) { 1345874aeea5SJeff Kirsher ++count; 1346cdb08f8fSBen Hutchings cpumask_or(thread_mask, thread_mask, 134706931e62SBartosz Golaszewski topology_sibling_cpumask(cpu)); 1348874aeea5SJeff Kirsher } 1349874aeea5SJeff Kirsher } 1350874aeea5SJeff Kirsher 1351cdb08f8fSBen Hutchings free_cpumask_var(thread_mask); 1352cd2d5b52SBen Hutchings } 1353cd2d5b52SBen Hutchings 1354cd2d5b52SBen Hutchings /* If RSS is requested for the PF *and* VFs then we can't write RSS 1355cd2d5b52SBen Hutchings * table entries that are inaccessible to VFs 1356cd2d5b52SBen Hutchings */ 13577fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 13587fa8d547SShradha Shah if (efx->type->sriov_wanted) { 1359d98a4ffeSShradha Shah if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1360cd2d5b52SBen Hutchings count > efx_vf_size(efx)) { 1361cd2d5b52SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1362cd2d5b52SBen Hutchings "Reducing number of RSS channels from %u to %u for " 1363cd2d5b52SBen Hutchings "VF support. Increase vf-msix-limit to use more " 1364cd2d5b52SBen Hutchings "channels on the PF.\n", 1365cd2d5b52SBen Hutchings count, efx_vf_size(efx)); 1366cd2d5b52SBen Hutchings count = efx_vf_size(efx); 1367cd2d5b52SBen Hutchings } 13687fa8d547SShradha Shah } 13697fa8d547SShradha Shah #endif 1370cd2d5b52SBen Hutchings 1371874aeea5SJeff Kirsher return count; 1372874aeea5SJeff Kirsher } 1373874aeea5SJeff Kirsher 1374874aeea5SJeff Kirsher /* Probe the number and type of interrupts we are able to obtain, and 1375874aeea5SJeff Kirsher * the resulting numbers of channels and RX queues. 1376874aeea5SJeff Kirsher */ 1377874aeea5SJeff Kirsher static int efx_probe_interrupts(struct efx_nic *efx) 1378874aeea5SJeff Kirsher { 13797f967c01SBen Hutchings unsigned int extra_channels = 0; 13807f967c01SBen Hutchings unsigned int i, j; 1381a16e5b24SBen Hutchings int rc; 1382874aeea5SJeff Kirsher 13837f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 13847f967c01SBen Hutchings if (efx->extra_channel_type[i]) 13857f967c01SBen Hutchings ++extra_channels; 13867f967c01SBen Hutchings 1387874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1388874aeea5SJeff Kirsher struct msix_entry xentries[EFX_MAX_CHANNELS]; 1389a16e5b24SBen Hutchings unsigned int n_channels; 1390874aeea5SJeff Kirsher 1391a9a52506SBen Hutchings n_channels = efx_wanted_parallelism(efx); 1392b0fbdae1SShradha Shah if (efx_separate_tx_channels) 1393874aeea5SJeff Kirsher n_channels *= 2; 13947f967c01SBen Hutchings n_channels += extra_channels; 1395b105798fSBen Hutchings n_channels = min(n_channels, efx->max_channels); 1396874aeea5SJeff Kirsher 1397874aeea5SJeff Kirsher for (i = 0; i < n_channels; i++) 1398874aeea5SJeff Kirsher xentries[i].entry = i; 1399184603d8SAlexander Gordeev rc = pci_enable_msix_range(efx->pci_dev, 1400184603d8SAlexander Gordeev xentries, 1, n_channels); 1401184603d8SAlexander Gordeev if (rc < 0) { 1402184603d8SAlexander Gordeev /* Fall back to single channel MSI */ 1403184603d8SAlexander Gordeev efx->interrupt_mode = EFX_INT_MODE_MSI; 1404184603d8SAlexander Gordeev netif_err(efx, drv, efx->net_dev, 1405184603d8SAlexander Gordeev "could not enable MSI-X\n"); 1406184603d8SAlexander Gordeev } else if (rc < n_channels) { 1407874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1408874aeea5SJeff Kirsher "WARNING: Insufficient MSI-X vectors" 1409a16e5b24SBen Hutchings " available (%d < %u).\n", rc, n_channels); 1410874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1411874aeea5SJeff Kirsher "WARNING: Performance may be reduced.\n"); 1412874aeea5SJeff Kirsher n_channels = rc; 1413874aeea5SJeff Kirsher } 1414874aeea5SJeff Kirsher 1415184603d8SAlexander Gordeev if (rc > 0) { 1416874aeea5SJeff Kirsher efx->n_channels = n_channels; 14177f967c01SBen Hutchings if (n_channels > extra_channels) 14187f967c01SBen Hutchings n_channels -= extra_channels; 1419b0fbdae1SShradha Shah if (efx_separate_tx_channels) { 1420b0fbdae1SShradha Shah efx->n_tx_channels = min(max(n_channels / 2, 1421b0fbdae1SShradha Shah 1U), 1422b0fbdae1SShradha Shah efx->max_tx_channels); 14237f967c01SBen Hutchings efx->n_rx_channels = max(n_channels - 14247f967c01SBen Hutchings efx->n_tx_channels, 14257f967c01SBen Hutchings 1U); 1426874aeea5SJeff Kirsher } else { 1427b0fbdae1SShradha Shah efx->n_tx_channels = min(n_channels, 1428b0fbdae1SShradha Shah efx->max_tx_channels); 14297f967c01SBen Hutchings efx->n_rx_channels = n_channels; 1430874aeea5SJeff Kirsher } 14317f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) 1432874aeea5SJeff Kirsher efx_get_channel(efx, i)->irq = 1433874aeea5SJeff Kirsher xentries[i].vector; 1434874aeea5SJeff Kirsher } 1435874aeea5SJeff Kirsher } 1436874aeea5SJeff Kirsher 1437874aeea5SJeff Kirsher /* Try single interrupt MSI */ 1438874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1439874aeea5SJeff Kirsher efx->n_channels = 1; 1440874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1441874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1442874aeea5SJeff Kirsher rc = pci_enable_msi(efx->pci_dev); 1443874aeea5SJeff Kirsher if (rc == 0) { 1444874aeea5SJeff Kirsher efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1445874aeea5SJeff Kirsher } else { 1446874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1447874aeea5SJeff Kirsher "could not enable MSI\n"); 1448874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1449874aeea5SJeff Kirsher } 1450874aeea5SJeff Kirsher } 1451874aeea5SJeff Kirsher 1452874aeea5SJeff Kirsher /* Assume legacy interrupts */ 1453874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1454b0fbdae1SShradha Shah efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1455874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1456874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1457874aeea5SJeff Kirsher efx->legacy_irq = efx->pci_dev->irq; 1458874aeea5SJeff Kirsher } 1459874aeea5SJeff Kirsher 14607f967c01SBen Hutchings /* Assign extra channels if possible */ 14617f967c01SBen Hutchings j = efx->n_channels; 14627f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 14637f967c01SBen Hutchings if (!efx->extra_channel_type[i]) 14647f967c01SBen Hutchings continue; 14657f967c01SBen Hutchings if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 14667f967c01SBen Hutchings efx->n_channels <= extra_channels) { 14677f967c01SBen Hutchings efx->extra_channel_type[i]->handle_no_channel(efx); 14687f967c01SBen Hutchings } else { 14697f967c01SBen Hutchings --j; 14707f967c01SBen Hutchings efx_get_channel(efx, j)->type = 14717f967c01SBen Hutchings efx->extra_channel_type[i]; 14727f967c01SBen Hutchings } 14737f967c01SBen Hutchings } 14747f967c01SBen Hutchings 1475cd2d5b52SBen Hutchings /* RSS might be usable on VFs even if it is disabled on the PF */ 14767fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 14777fa8d547SShradha Shah if (efx->type->sriov_wanted) { 1478327c685eSShradha Shah efx->rss_spread = ((efx->n_rx_channels > 1 || 1479d98a4ffeSShradha Shah !efx->type->sriov_wanted(efx)) ? 1480cd2d5b52SBen Hutchings efx->n_rx_channels : efx_vf_size(efx)); 14817fa8d547SShradha Shah return 0; 14827fa8d547SShradha Shah } 14837fa8d547SShradha Shah #endif 14847fa8d547SShradha Shah efx->rss_spread = efx->n_rx_channels; 1485cd2d5b52SBen Hutchings 1486874aeea5SJeff Kirsher return 0; 1487874aeea5SJeff Kirsher } 1488874aeea5SJeff Kirsher 1489261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx) 1490d8291187SBen Hutchings { 1491261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1492261e4d96SJon Cooper int rc; 1493d8291187SBen Hutchings 1494d8291187SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1495d8291187SBen Hutchings 1496d8291187SBen Hutchings efx->irq_soft_enabled = true; 1497d8291187SBen Hutchings smp_wmb(); 1498d8291187SBen Hutchings 1499d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1500261e4d96SJon Cooper if (!channel->type->keep_eventq) { 1501261e4d96SJon Cooper rc = efx_init_eventq(channel); 1502261e4d96SJon Cooper if (rc) 1503261e4d96SJon Cooper goto fail; 1504261e4d96SJon Cooper } 1505d8291187SBen Hutchings efx_start_eventq(channel); 1506d8291187SBen Hutchings } 1507d8291187SBen Hutchings 1508d8291187SBen Hutchings efx_mcdi_mode_event(efx); 1509261e4d96SJon Cooper 1510261e4d96SJon Cooper return 0; 1511261e4d96SJon Cooper fail: 1512261e4d96SJon Cooper end_channel = channel; 1513261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1514261e4d96SJon Cooper if (channel == end_channel) 1515261e4d96SJon Cooper break; 1516261e4d96SJon Cooper efx_stop_eventq(channel); 1517261e4d96SJon Cooper if (!channel->type->keep_eventq) 1518261e4d96SJon Cooper efx_fini_eventq(channel); 1519261e4d96SJon Cooper } 1520261e4d96SJon Cooper 1521261e4d96SJon Cooper return rc; 1522d8291187SBen Hutchings } 1523d8291187SBen Hutchings 1524d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx) 1525d8291187SBen Hutchings { 1526d8291187SBen Hutchings struct efx_channel *channel; 1527d8291187SBen Hutchings 1528d8291187SBen Hutchings if (efx->state == STATE_DISABLED) 1529d8291187SBen Hutchings return; 1530d8291187SBen Hutchings 1531d8291187SBen Hutchings efx_mcdi_mode_poll(efx); 1532d8291187SBen Hutchings 1533d8291187SBen Hutchings efx->irq_soft_enabled = false; 1534d8291187SBen Hutchings smp_wmb(); 1535d8291187SBen Hutchings 1536d8291187SBen Hutchings if (efx->legacy_irq) 1537d8291187SBen Hutchings synchronize_irq(efx->legacy_irq); 1538d8291187SBen Hutchings 1539d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1540d8291187SBen Hutchings if (channel->irq) 1541d8291187SBen Hutchings synchronize_irq(channel->irq); 1542d8291187SBen Hutchings 1543d8291187SBen Hutchings efx_stop_eventq(channel); 1544d8291187SBen Hutchings if (!channel->type->keep_eventq) 1545d8291187SBen Hutchings efx_fini_eventq(channel); 1546d8291187SBen Hutchings } 1547cade715fSBen Hutchings 1548cade715fSBen Hutchings /* Flush the asynchronous MCDI request queue */ 1549cade715fSBen Hutchings efx_mcdi_flush_async(efx); 1550d8291187SBen Hutchings } 1551d8291187SBen Hutchings 1552261e4d96SJon Cooper static int efx_enable_interrupts(struct efx_nic *efx) 15539f2cb71cSBen Hutchings { 1554261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1555261e4d96SJon Cooper int rc; 15569f2cb71cSBen Hutchings 15578b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 15588b7325b4SBen Hutchings 1559b28405b0SAlexandre Rames if (efx->eeh_disabled_legacy_irq) { 1560b28405b0SAlexandre Rames enable_irq(efx->legacy_irq); 1561b28405b0SAlexandre Rames efx->eeh_disabled_legacy_irq = false; 1562b28405b0SAlexandre Rames } 1563d8291187SBen Hutchings 156486094f7fSBen Hutchings efx->type->irq_enable_master(efx); 15659f2cb71cSBen Hutchings 15669f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1567261e4d96SJon Cooper if (channel->type->keep_eventq) { 1568261e4d96SJon Cooper rc = efx_init_eventq(channel); 1569261e4d96SJon Cooper if (rc) 1570261e4d96SJon Cooper goto fail; 1571261e4d96SJon Cooper } 15729f2cb71cSBen Hutchings } 15739f2cb71cSBen Hutchings 1574261e4d96SJon Cooper rc = efx_soft_enable_interrupts(efx); 1575261e4d96SJon Cooper if (rc) 1576261e4d96SJon Cooper goto fail; 1577261e4d96SJon Cooper 1578261e4d96SJon Cooper return 0; 1579261e4d96SJon Cooper 1580261e4d96SJon Cooper fail: 1581261e4d96SJon Cooper end_channel = channel; 1582261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1583261e4d96SJon Cooper if (channel == end_channel) 1584261e4d96SJon Cooper break; 1585261e4d96SJon Cooper if (channel->type->keep_eventq) 1586261e4d96SJon Cooper efx_fini_eventq(channel); 1587261e4d96SJon Cooper } 1588261e4d96SJon Cooper 1589261e4d96SJon Cooper efx->type->irq_disable_non_ev(efx); 1590261e4d96SJon Cooper 1591261e4d96SJon Cooper return rc; 15929f2cb71cSBen Hutchings } 15939f2cb71cSBen Hutchings 1594d8291187SBen Hutchings static void efx_disable_interrupts(struct efx_nic *efx) 15959f2cb71cSBen Hutchings { 15969f2cb71cSBen Hutchings struct efx_channel *channel; 15979f2cb71cSBen Hutchings 1598d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 15999f2cb71cSBen Hutchings 16009f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1601d8291187SBen Hutchings if (channel->type->keep_eventq) 16029f2cb71cSBen Hutchings efx_fini_eventq(channel); 16039f2cb71cSBen Hutchings } 1604d8291187SBen Hutchings 160586094f7fSBen Hutchings efx->type->irq_disable_non_ev(efx); 16069f2cb71cSBen Hutchings } 16079f2cb71cSBen Hutchings 1608874aeea5SJeff Kirsher static void efx_remove_interrupts(struct efx_nic *efx) 1609874aeea5SJeff Kirsher { 1610874aeea5SJeff Kirsher struct efx_channel *channel; 1611874aeea5SJeff Kirsher 1612874aeea5SJeff Kirsher /* Remove MSI/MSI-X interrupts */ 1613874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1614874aeea5SJeff Kirsher channel->irq = 0; 1615874aeea5SJeff Kirsher pci_disable_msi(efx->pci_dev); 1616874aeea5SJeff Kirsher pci_disable_msix(efx->pci_dev); 1617874aeea5SJeff Kirsher 1618874aeea5SJeff Kirsher /* Remove legacy interrupt */ 1619874aeea5SJeff Kirsher efx->legacy_irq = 0; 1620874aeea5SJeff Kirsher } 1621874aeea5SJeff Kirsher 1622874aeea5SJeff Kirsher static void efx_set_channels(struct efx_nic *efx) 1623874aeea5SJeff Kirsher { 1624874aeea5SJeff Kirsher struct efx_channel *channel; 1625874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 1626874aeea5SJeff Kirsher 1627874aeea5SJeff Kirsher efx->tx_channel_offset = 1628b0fbdae1SShradha Shah efx_separate_tx_channels ? 1629b0fbdae1SShradha Shah efx->n_channels - efx->n_tx_channels : 0; 1630874aeea5SJeff Kirsher 163179d68b37SStuart Hodgson /* We need to mark which channels really have RX and TX 163279d68b37SStuart Hodgson * queues, and adjust the TX queue numbers if we have separate 1633874aeea5SJeff Kirsher * RX-only and TX-only channels. 1634874aeea5SJeff Kirsher */ 1635874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 163679d68b37SStuart Hodgson if (channel->channel < efx->n_rx_channels) 163779d68b37SStuart Hodgson channel->rx_queue.core_index = channel->channel; 163879d68b37SStuart Hodgson else 163979d68b37SStuart Hodgson channel->rx_queue.core_index = -1; 164079d68b37SStuart Hodgson 1641874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 1642874aeea5SJeff Kirsher tx_queue->queue -= (efx->tx_channel_offset * 1643874aeea5SJeff Kirsher EFX_TXQ_TYPES); 1644874aeea5SJeff Kirsher } 1645874aeea5SJeff Kirsher } 1646874aeea5SJeff Kirsher 1647874aeea5SJeff Kirsher static int efx_probe_nic(struct efx_nic *efx) 1648874aeea5SJeff Kirsher { 1649874aeea5SJeff Kirsher int rc; 1650874aeea5SJeff Kirsher 1651874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1652874aeea5SJeff Kirsher 1653874aeea5SJeff Kirsher /* Carry out hardware-type specific initialisation */ 1654874aeea5SJeff Kirsher rc = efx->type->probe(efx); 1655874aeea5SJeff Kirsher if (rc) 1656874aeea5SJeff Kirsher return rc; 1657874aeea5SJeff Kirsher 1658b0fbdae1SShradha Shah do { 1659b0fbdae1SShradha Shah if (!efx->max_channels || !efx->max_tx_channels) { 1660b0fbdae1SShradha Shah netif_err(efx, drv, efx->net_dev, 1661b0fbdae1SShradha Shah "Insufficient resources to allocate" 1662b0fbdae1SShradha Shah " any channels\n"); 1663b0fbdae1SShradha Shah rc = -ENOSPC; 1664b0fbdae1SShradha Shah goto fail1; 1665b0fbdae1SShradha Shah } 1666b0fbdae1SShradha Shah 1667b0fbdae1SShradha Shah /* Determine the number of channels and queues by trying 1668b0fbdae1SShradha Shah * to hook in MSI-X interrupts. 1669b0fbdae1SShradha Shah */ 1670874aeea5SJeff Kirsher rc = efx_probe_interrupts(efx); 1671874aeea5SJeff Kirsher if (rc) 1672c15eed22SBen Hutchings goto fail1; 1673874aeea5SJeff Kirsher 167452ad762bSDaniel Pieczko efx_set_channels(efx); 167552ad762bSDaniel Pieczko 1676b0fbdae1SShradha Shah /* dimension_resources can fail with EAGAIN */ 1677c15eed22SBen Hutchings rc = efx->type->dimension_resources(efx); 1678b0fbdae1SShradha Shah if (rc != 0 && rc != -EAGAIN) 1679c15eed22SBen Hutchings goto fail2; 168028e47c49SBen Hutchings 1681b0fbdae1SShradha Shah if (rc == -EAGAIN) 1682b0fbdae1SShradha Shah /* try again with new max_channels */ 1683b0fbdae1SShradha Shah efx_remove_interrupts(efx); 1684b0fbdae1SShradha Shah 1685b0fbdae1SShradha Shah } while (rc == -EAGAIN); 1686b0fbdae1SShradha Shah 1687874aeea5SJeff Kirsher if (efx->n_channels > 1) 1688267c0157SJon Cooper netdev_rss_key_fill(&efx->rx_hash_key, 1689267c0157SJon Cooper sizeof(efx->rx_hash_key)); 1690267c0157SJon Cooper efx_set_default_rx_indir_table(efx); 1691874aeea5SJeff Kirsher 1692874aeea5SJeff Kirsher netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1693874aeea5SJeff Kirsher netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1694874aeea5SJeff Kirsher 1695874aeea5SJeff Kirsher /* Initialise the interrupt moderation settings */ 16969e393b30SBen Hutchings efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 16979e393b30SBen Hutchings true); 1698874aeea5SJeff Kirsher 1699874aeea5SJeff Kirsher return 0; 1700874aeea5SJeff Kirsher 1701c15eed22SBen Hutchings fail2: 1702c15eed22SBen Hutchings efx_remove_interrupts(efx); 1703c15eed22SBen Hutchings fail1: 1704874aeea5SJeff Kirsher efx->type->remove(efx); 1705874aeea5SJeff Kirsher return rc; 1706874aeea5SJeff Kirsher } 1707874aeea5SJeff Kirsher 1708874aeea5SJeff Kirsher static void efx_remove_nic(struct efx_nic *efx) 1709874aeea5SJeff Kirsher { 1710874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1711874aeea5SJeff Kirsher 1712874aeea5SJeff Kirsher efx_remove_interrupts(efx); 1713874aeea5SJeff Kirsher efx->type->remove(efx); 1714874aeea5SJeff Kirsher } 1715874aeea5SJeff Kirsher 1716add72477SBen Hutchings static int efx_probe_filters(struct efx_nic *efx) 1717add72477SBen Hutchings { 1718add72477SBen Hutchings int rc; 1719add72477SBen Hutchings 1720add72477SBen Hutchings spin_lock_init(&efx->filter_lock); 17210d322413SEdward Cree init_rwsem(&efx->filter_sem); 17220d322413SEdward Cree down_write(&efx->filter_sem); 1723add72477SBen Hutchings rc = efx->type->filter_table_probe(efx); 1724add72477SBen Hutchings if (rc) 17250d322413SEdward Cree goto out_unlock; 1726add72477SBen Hutchings 1727add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1728add72477SBen Hutchings if (efx->type->offload_features & NETIF_F_NTUPLE) { 1729add72477SBen Hutchings efx->rps_flow_id = kcalloc(efx->type->max_rx_ip_filters, 1730add72477SBen Hutchings sizeof(*efx->rps_flow_id), 1731add72477SBen Hutchings GFP_KERNEL); 1732add72477SBen Hutchings if (!efx->rps_flow_id) { 1733add72477SBen Hutchings efx->type->filter_table_remove(efx); 17340d322413SEdward Cree rc = -ENOMEM; 17350d322413SEdward Cree goto out_unlock; 1736add72477SBen Hutchings } 1737add72477SBen Hutchings } 1738add72477SBen Hutchings #endif 17390d322413SEdward Cree out_unlock: 17400d322413SEdward Cree up_write(&efx->filter_sem); 17410d322413SEdward Cree return rc; 1742add72477SBen Hutchings } 1743add72477SBen Hutchings 1744add72477SBen Hutchings static void efx_remove_filters(struct efx_nic *efx) 1745add72477SBen Hutchings { 1746add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1747add72477SBen Hutchings kfree(efx->rps_flow_id); 1748add72477SBen Hutchings #endif 17490d322413SEdward Cree down_write(&efx->filter_sem); 1750add72477SBen Hutchings efx->type->filter_table_remove(efx); 17510d322413SEdward Cree up_write(&efx->filter_sem); 1752add72477SBen Hutchings } 1753add72477SBen Hutchings 1754add72477SBen Hutchings static void efx_restore_filters(struct efx_nic *efx) 1755add72477SBen Hutchings { 17560d322413SEdward Cree down_read(&efx->filter_sem); 1757add72477SBen Hutchings efx->type->filter_table_restore(efx); 17580d322413SEdward Cree up_read(&efx->filter_sem); 1759add72477SBen Hutchings } 1760add72477SBen Hutchings 1761874aeea5SJeff Kirsher /************************************************************************** 1762874aeea5SJeff Kirsher * 1763874aeea5SJeff Kirsher * NIC startup/shutdown 1764874aeea5SJeff Kirsher * 1765874aeea5SJeff Kirsher *************************************************************************/ 1766874aeea5SJeff Kirsher 1767874aeea5SJeff Kirsher static int efx_probe_all(struct efx_nic *efx) 1768874aeea5SJeff Kirsher { 1769874aeea5SJeff Kirsher int rc; 1770874aeea5SJeff Kirsher 1771874aeea5SJeff Kirsher rc = efx_probe_nic(efx); 1772874aeea5SJeff Kirsher if (rc) { 1773874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1774874aeea5SJeff Kirsher goto fail1; 1775874aeea5SJeff Kirsher } 1776874aeea5SJeff Kirsher 1777874aeea5SJeff Kirsher rc = efx_probe_port(efx); 1778874aeea5SJeff Kirsher if (rc) { 1779874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1780874aeea5SJeff Kirsher goto fail2; 1781874aeea5SJeff Kirsher } 1782874aeea5SJeff Kirsher 17837e6d06f0SBen Hutchings BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 17847e6d06f0SBen Hutchings if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 17857e6d06f0SBen Hutchings rc = -EINVAL; 17867e6d06f0SBen Hutchings goto fail3; 17877e6d06f0SBen Hutchings } 1788874aeea5SJeff Kirsher efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1789874aeea5SJeff Kirsher 17906d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 17916d8aaaf6SDaniel Pieczko rc = efx->type->vswitching_probe(efx); 17926d8aaaf6SDaniel Pieczko if (rc) /* not fatal; the PF will still work fine */ 17936d8aaaf6SDaniel Pieczko netif_warn(efx, probe, efx->net_dev, 17946d8aaaf6SDaniel Pieczko "failed to setup vswitching rc=%d;" 17956d8aaaf6SDaniel Pieczko " VFs may not function\n", rc); 17966d8aaaf6SDaniel Pieczko #endif 17976d8aaaf6SDaniel Pieczko 1798874aeea5SJeff Kirsher rc = efx_probe_filters(efx); 1799874aeea5SJeff Kirsher if (rc) { 1800874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1801874aeea5SJeff Kirsher "failed to create filter tables\n"); 18026d8aaaf6SDaniel Pieczko goto fail4; 1803874aeea5SJeff Kirsher } 1804874aeea5SJeff Kirsher 18057f967c01SBen Hutchings rc = efx_probe_channels(efx); 18067f967c01SBen Hutchings if (rc) 18076d8aaaf6SDaniel Pieczko goto fail5; 18087f967c01SBen Hutchings 1809874aeea5SJeff Kirsher return 0; 1810874aeea5SJeff Kirsher 18116d8aaaf6SDaniel Pieczko fail5: 18127f967c01SBen Hutchings efx_remove_filters(efx); 18136d8aaaf6SDaniel Pieczko fail4: 18146d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 18156d8aaaf6SDaniel Pieczko efx->type->vswitching_remove(efx); 18166d8aaaf6SDaniel Pieczko #endif 1817874aeea5SJeff Kirsher fail3: 1818874aeea5SJeff Kirsher efx_remove_port(efx); 1819874aeea5SJeff Kirsher fail2: 1820874aeea5SJeff Kirsher efx_remove_nic(efx); 1821874aeea5SJeff Kirsher fail1: 1822874aeea5SJeff Kirsher return rc; 1823874aeea5SJeff Kirsher } 1824874aeea5SJeff Kirsher 18258b7325b4SBen Hutchings /* If the interface is supposed to be running but is not, start 18268b7325b4SBen Hutchings * the hardware and software data path, regular activity for the port 18278b7325b4SBen Hutchings * (MAC statistics, link polling, etc.) and schedule the port to be 18288b7325b4SBen Hutchings * reconfigured. Interrupts must already be enabled. This function 18298b7325b4SBen Hutchings * is safe to call multiple times, so long as the NIC is not disabled. 18308b7325b4SBen Hutchings * Requires the RTNL lock. 18319f2cb71cSBen Hutchings */ 1832874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx) 1833874aeea5SJeff Kirsher { 1834874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 18358b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1836874aeea5SJeff Kirsher 1837874aeea5SJeff Kirsher /* Check that it is appropriate to restart the interface. All 1838874aeea5SJeff Kirsher * of these flags are safe to read under just the rtnl lock */ 1839e283546cSEdward Cree if (efx->port_enabled || !netif_running(efx->net_dev) || 1840e283546cSEdward Cree efx->reset_pending) 1841874aeea5SJeff Kirsher return; 1842874aeea5SJeff Kirsher 1843874aeea5SJeff Kirsher efx_start_port(efx); 18449f2cb71cSBen Hutchings efx_start_datapath(efx); 1845874aeea5SJeff Kirsher 1846626950dbSAlexandre Rames /* Start the hardware monitor if there is one */ 1847626950dbSAlexandre Rames if (efx->type->monitor != NULL) 1848874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 1849874aeea5SJeff Kirsher efx_monitor_interval); 1850626950dbSAlexandre Rames 1851626950dbSAlexandre Rames /* If link state detection is normally event-driven, we have 1852626950dbSAlexandre Rames * to poll now because we could have missed a change 1853626950dbSAlexandre Rames */ 1854626950dbSAlexandre Rames if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { 1855874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1856874aeea5SJeff Kirsher if (efx->phy_op->poll(efx)) 1857874aeea5SJeff Kirsher efx_link_status_changed(efx); 1858874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1859874aeea5SJeff Kirsher } 1860874aeea5SJeff Kirsher 1861874aeea5SJeff Kirsher efx->type->start_stats(efx); 1862f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 1863f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 1864f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 1865f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 1866874aeea5SJeff Kirsher } 1867874aeea5SJeff Kirsher 18688b7325b4SBen Hutchings /* Quiesce the hardware and software data path, and regular activity 18698b7325b4SBen Hutchings * for the port without bringing the link down. Safe to call multiple 18708b7325b4SBen Hutchings * times with the NIC in almost any state, but interrupts should be 18718b7325b4SBen Hutchings * enabled. Requires the RTNL lock. 18728b7325b4SBen Hutchings */ 1873874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx) 1874874aeea5SJeff Kirsher { 1875874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1876874aeea5SJeff Kirsher 1877874aeea5SJeff Kirsher /* port_enabled can be read safely under the rtnl lock */ 1878874aeea5SJeff Kirsher if (!efx->port_enabled) 1879874aeea5SJeff Kirsher return; 1880874aeea5SJeff Kirsher 1881f8f3b5aeSJon Cooper /* update stats before we go down so we can accurately count 1882f8f3b5aeSJon Cooper * rx_nodesc_drops 1883f8f3b5aeSJon Cooper */ 1884f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 1885f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 1886f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 1887f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 1888874aeea5SJeff Kirsher efx->type->stop_stats(efx); 1889874aeea5SJeff Kirsher efx_stop_port(efx); 1890874aeea5SJeff Kirsher 189129c69a48SBen Hutchings /* Stop the kernel transmit interface. This is only valid if 189229c69a48SBen Hutchings * the device is stopped or detached; otherwise the watchdog 189329c69a48SBen Hutchings * may fire immediately. 189429c69a48SBen Hutchings */ 189529c69a48SBen Hutchings WARN_ON(netif_running(efx->net_dev) && 189629c69a48SBen Hutchings netif_device_present(efx->net_dev)); 18979f2cb71cSBen Hutchings netif_tx_disable(efx->net_dev); 18989f2cb71cSBen Hutchings 18999f2cb71cSBen Hutchings efx_stop_datapath(efx); 1900874aeea5SJeff Kirsher } 1901874aeea5SJeff Kirsher 1902874aeea5SJeff Kirsher static void efx_remove_all(struct efx_nic *efx) 1903874aeea5SJeff Kirsher { 1904874aeea5SJeff Kirsher efx_remove_channels(efx); 19057f967c01SBen Hutchings efx_remove_filters(efx); 19066d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 19076d8aaaf6SDaniel Pieczko efx->type->vswitching_remove(efx); 19086d8aaaf6SDaniel Pieczko #endif 1909874aeea5SJeff Kirsher efx_remove_port(efx); 1910874aeea5SJeff Kirsher efx_remove_nic(efx); 1911874aeea5SJeff Kirsher } 1912874aeea5SJeff Kirsher 1913874aeea5SJeff Kirsher /************************************************************************** 1914874aeea5SJeff Kirsher * 1915874aeea5SJeff Kirsher * Interrupt moderation 1916874aeea5SJeff Kirsher * 1917874aeea5SJeff Kirsher **************************************************************************/ 1918874aeea5SJeff Kirsher 1919cc180b69SBen Hutchings static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns) 1920874aeea5SJeff Kirsher { 1921b548f976SBen Hutchings if (usecs == 0) 1922b548f976SBen Hutchings return 0; 1923cc180b69SBen Hutchings if (usecs * 1000 < quantum_ns) 1924874aeea5SJeff Kirsher return 1; /* never round down to 0 */ 1925cc180b69SBen Hutchings return usecs * 1000 / quantum_ns; 1926874aeea5SJeff Kirsher } 1927874aeea5SJeff Kirsher 1928874aeea5SJeff Kirsher /* Set interrupt moderation parameters */ 19299e393b30SBen Hutchings int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 19309e393b30SBen Hutchings unsigned int rx_usecs, bool rx_adaptive, 19319e393b30SBen Hutchings bool rx_may_override_tx) 1932874aeea5SJeff Kirsher { 1933874aeea5SJeff Kirsher struct efx_channel *channel; 1934cc180b69SBen Hutchings unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max * 1935cc180b69SBen Hutchings efx->timer_quantum_ns, 1936cc180b69SBen Hutchings 1000); 1937cc180b69SBen Hutchings unsigned int tx_ticks; 1938cc180b69SBen Hutchings unsigned int rx_ticks; 1939874aeea5SJeff Kirsher 1940874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1941874aeea5SJeff Kirsher 1942cc180b69SBen Hutchings if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max) 19439e393b30SBen Hutchings return -EINVAL; 19449e393b30SBen Hutchings 1945cc180b69SBen Hutchings tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns); 1946cc180b69SBen Hutchings rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns); 1947cc180b69SBen Hutchings 19489e393b30SBen Hutchings if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 && 19499e393b30SBen Hutchings !rx_may_override_tx) { 19509e393b30SBen Hutchings netif_err(efx, drv, efx->net_dev, "Channels are shared. " 19519e393b30SBen Hutchings "RX and TX IRQ moderation must be equal\n"); 19529e393b30SBen Hutchings return -EINVAL; 19539e393b30SBen Hutchings } 19549e393b30SBen Hutchings 1955874aeea5SJeff Kirsher efx->irq_rx_adaptive = rx_adaptive; 1956874aeea5SJeff Kirsher efx->irq_rx_moderation = rx_ticks; 1957874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 1958874aeea5SJeff Kirsher if (efx_channel_has_rx_queue(channel)) 1959874aeea5SJeff Kirsher channel->irq_moderation = rx_ticks; 1960874aeea5SJeff Kirsher else if (efx_channel_has_tx_queues(channel)) 1961874aeea5SJeff Kirsher channel->irq_moderation = tx_ticks; 1962874aeea5SJeff Kirsher } 19639e393b30SBen Hutchings 19649e393b30SBen Hutchings return 0; 1965874aeea5SJeff Kirsher } 1966874aeea5SJeff Kirsher 1967a0c4faf5SBen Hutchings void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 1968a0c4faf5SBen Hutchings unsigned int *rx_usecs, bool *rx_adaptive) 1969a0c4faf5SBen Hutchings { 1970cc180b69SBen Hutchings /* We must round up when converting ticks to microseconds 1971cc180b69SBen Hutchings * because we round down when converting the other way. 1972cc180b69SBen Hutchings */ 1973cc180b69SBen Hutchings 1974a0c4faf5SBen Hutchings *rx_adaptive = efx->irq_rx_adaptive; 1975cc180b69SBen Hutchings *rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation * 1976cc180b69SBen Hutchings efx->timer_quantum_ns, 1977cc180b69SBen Hutchings 1000); 1978a0c4faf5SBen Hutchings 1979a0c4faf5SBen Hutchings /* If channels are shared between RX and TX, so is IRQ 1980a0c4faf5SBen Hutchings * moderation. Otherwise, IRQ moderation is the same for all 1981a0c4faf5SBen Hutchings * TX channels and is not adaptive. 1982a0c4faf5SBen Hutchings */ 1983a0c4faf5SBen Hutchings if (efx->tx_channel_offset == 0) 1984a0c4faf5SBen Hutchings *tx_usecs = *rx_usecs; 1985a0c4faf5SBen Hutchings else 1986cc180b69SBen Hutchings *tx_usecs = DIV_ROUND_UP( 1987a0c4faf5SBen Hutchings efx->channel[efx->tx_channel_offset]->irq_moderation * 1988cc180b69SBen Hutchings efx->timer_quantum_ns, 1989cc180b69SBen Hutchings 1000); 1990a0c4faf5SBen Hutchings } 1991a0c4faf5SBen Hutchings 1992874aeea5SJeff Kirsher /************************************************************************** 1993874aeea5SJeff Kirsher * 1994874aeea5SJeff Kirsher * Hardware monitor 1995874aeea5SJeff Kirsher * 1996874aeea5SJeff Kirsher **************************************************************************/ 1997874aeea5SJeff Kirsher 1998874aeea5SJeff Kirsher /* Run periodically off the general workqueue */ 1999874aeea5SJeff Kirsher static void efx_monitor(struct work_struct *data) 2000874aeea5SJeff Kirsher { 2001874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, 2002874aeea5SJeff Kirsher monitor_work.work); 2003874aeea5SJeff Kirsher 2004874aeea5SJeff Kirsher netif_vdbg(efx, timer, efx->net_dev, 2005874aeea5SJeff Kirsher "hardware monitor executing on CPU %d\n", 2006874aeea5SJeff Kirsher raw_smp_processor_id()); 2007874aeea5SJeff Kirsher BUG_ON(efx->type->monitor == NULL); 2008874aeea5SJeff Kirsher 2009874aeea5SJeff Kirsher /* If the mac_lock is already held then it is likely a port 2010874aeea5SJeff Kirsher * reconfiguration is already in place, which will likely do 2011874aeea5SJeff Kirsher * most of the work of monitor() anyway. */ 2012874aeea5SJeff Kirsher if (mutex_trylock(&efx->mac_lock)) { 2013874aeea5SJeff Kirsher if (efx->port_enabled) 2014874aeea5SJeff Kirsher efx->type->monitor(efx); 2015874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2016874aeea5SJeff Kirsher } 2017874aeea5SJeff Kirsher 2018874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 2019874aeea5SJeff Kirsher efx_monitor_interval); 2020874aeea5SJeff Kirsher } 2021874aeea5SJeff Kirsher 2022874aeea5SJeff Kirsher /************************************************************************** 2023874aeea5SJeff Kirsher * 2024874aeea5SJeff Kirsher * ioctls 2025874aeea5SJeff Kirsher * 2026874aeea5SJeff Kirsher *************************************************************************/ 2027874aeea5SJeff Kirsher 2028874aeea5SJeff Kirsher /* Net device ioctl 2029874aeea5SJeff Kirsher * Context: process, rtnl_lock() held. 2030874aeea5SJeff Kirsher */ 2031874aeea5SJeff Kirsher static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2032874aeea5SJeff Kirsher { 2033874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2034874aeea5SJeff Kirsher struct mii_ioctl_data *data = if_mii(ifr); 2035874aeea5SJeff Kirsher 20367c236c43SStuart Hodgson if (cmd == SIOCSHWTSTAMP) 2037433dc9b3SBen Hutchings return efx_ptp_set_ts_config(efx, ifr); 2038433dc9b3SBen Hutchings if (cmd == SIOCGHWTSTAMP) 2039433dc9b3SBen Hutchings return efx_ptp_get_ts_config(efx, ifr); 20407c236c43SStuart Hodgson 2041874aeea5SJeff Kirsher /* Convert phy_id from older PRTAD/DEVAD format */ 2042874aeea5SJeff Kirsher if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2043874aeea5SJeff Kirsher (data->phy_id & 0xfc00) == 0x0400) 2044874aeea5SJeff Kirsher data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2045874aeea5SJeff Kirsher 2046874aeea5SJeff Kirsher return mdio_mii_ioctl(&efx->mdio, data, cmd); 2047874aeea5SJeff Kirsher } 2048874aeea5SJeff Kirsher 2049874aeea5SJeff Kirsher /************************************************************************** 2050874aeea5SJeff Kirsher * 2051874aeea5SJeff Kirsher * NAPI interface 2052874aeea5SJeff Kirsher * 2053874aeea5SJeff Kirsher **************************************************************************/ 2054874aeea5SJeff Kirsher 20557f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel) 2056874aeea5SJeff Kirsher { 20577f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 2058874aeea5SJeff Kirsher 2059874aeea5SJeff Kirsher channel->napi_dev = efx->net_dev; 2060874aeea5SJeff Kirsher netif_napi_add(channel->napi_dev, &channel->napi_str, 2061874aeea5SJeff Kirsher efx_poll, napi_weight); 2062c0f9c7e4SBert Kenward efx_channel_busy_poll_init(channel); 2063874aeea5SJeff Kirsher } 20647f967c01SBen Hutchings 20657f967c01SBen Hutchings static void efx_init_napi(struct efx_nic *efx) 20667f967c01SBen Hutchings { 20677f967c01SBen Hutchings struct efx_channel *channel; 20687f967c01SBen Hutchings 20697f967c01SBen Hutchings efx_for_each_channel(channel, efx) 20707f967c01SBen Hutchings efx_init_napi_channel(channel); 2071874aeea5SJeff Kirsher } 2072874aeea5SJeff Kirsher 2073874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel) 2074874aeea5SJeff Kirsher { 207536763266SAlexandre Rames if (channel->napi_dev) { 2076874aeea5SJeff Kirsher netif_napi_del(&channel->napi_str); 207736763266SAlexandre Rames napi_hash_del(&channel->napi_str); 207836763266SAlexandre Rames } 2079874aeea5SJeff Kirsher channel->napi_dev = NULL; 2080874aeea5SJeff Kirsher } 2081874aeea5SJeff Kirsher 2082874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx) 2083874aeea5SJeff Kirsher { 2084874aeea5SJeff Kirsher struct efx_channel *channel; 2085874aeea5SJeff Kirsher 2086874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 2087874aeea5SJeff Kirsher efx_fini_napi_channel(channel); 2088874aeea5SJeff Kirsher } 2089874aeea5SJeff Kirsher 2090874aeea5SJeff Kirsher /************************************************************************** 2091874aeea5SJeff Kirsher * 2092874aeea5SJeff Kirsher * Kernel netpoll interface 2093874aeea5SJeff Kirsher * 2094874aeea5SJeff Kirsher *************************************************************************/ 2095874aeea5SJeff Kirsher 2096874aeea5SJeff Kirsher #ifdef CONFIG_NET_POLL_CONTROLLER 2097874aeea5SJeff Kirsher 2098874aeea5SJeff Kirsher /* Although in the common case interrupts will be disabled, this is not 2099874aeea5SJeff Kirsher * guaranteed. However, all our work happens inside the NAPI callback, 2100874aeea5SJeff Kirsher * so no locking is required. 2101874aeea5SJeff Kirsher */ 2102874aeea5SJeff Kirsher static void efx_netpoll(struct net_device *net_dev) 2103874aeea5SJeff Kirsher { 2104874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2105874aeea5SJeff Kirsher struct efx_channel *channel; 2106874aeea5SJeff Kirsher 2107874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 2108874aeea5SJeff Kirsher efx_schedule_channel(channel); 2109874aeea5SJeff Kirsher } 2110874aeea5SJeff Kirsher 2111874aeea5SJeff Kirsher #endif 2112874aeea5SJeff Kirsher 211336763266SAlexandre Rames #ifdef CONFIG_NET_RX_BUSY_POLL 211436763266SAlexandre Rames static int efx_busy_poll(struct napi_struct *napi) 211536763266SAlexandre Rames { 211636763266SAlexandre Rames struct efx_channel *channel = 211736763266SAlexandre Rames container_of(napi, struct efx_channel, napi_str); 211836763266SAlexandre Rames struct efx_nic *efx = channel->efx; 211936763266SAlexandre Rames int budget = 4; 212036763266SAlexandre Rames int old_rx_packets, rx_packets; 212136763266SAlexandre Rames 212236763266SAlexandre Rames if (!netif_running(efx->net_dev)) 212336763266SAlexandre Rames return LL_FLUSH_FAILED; 212436763266SAlexandre Rames 2125c0f9c7e4SBert Kenward if (!efx_channel_try_lock_poll(channel)) 212636763266SAlexandre Rames return LL_FLUSH_BUSY; 212736763266SAlexandre Rames 212836763266SAlexandre Rames old_rx_packets = channel->rx_queue.rx_packets; 212936763266SAlexandre Rames efx_process_channel(channel, budget); 213036763266SAlexandre Rames 213136763266SAlexandre Rames rx_packets = channel->rx_queue.rx_packets - old_rx_packets; 213236763266SAlexandre Rames 213336763266SAlexandre Rames /* There is no race condition with NAPI here. 213436763266SAlexandre Rames * NAPI will automatically be rescheduled if it yielded during busy 213536763266SAlexandre Rames * polling, because it was not able to take the lock and thus returned 213636763266SAlexandre Rames * the full budget. 213736763266SAlexandre Rames */ 213836763266SAlexandre Rames efx_channel_unlock_poll(channel); 213936763266SAlexandre Rames 214036763266SAlexandre Rames return rx_packets; 214136763266SAlexandre Rames } 214236763266SAlexandre Rames #endif 214336763266SAlexandre Rames 2144874aeea5SJeff Kirsher /************************************************************************** 2145874aeea5SJeff Kirsher * 2146874aeea5SJeff Kirsher * Kernel net device interface 2147874aeea5SJeff Kirsher * 2148874aeea5SJeff Kirsher *************************************************************************/ 2149874aeea5SJeff Kirsher 2150874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2151e340be92SShradha Shah int efx_net_open(struct net_device *net_dev) 2152874aeea5SJeff Kirsher { 2153874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 21548b7325b4SBen Hutchings int rc; 21558b7325b4SBen Hutchings 2156874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2157874aeea5SJeff Kirsher raw_smp_processor_id()); 2158874aeea5SJeff Kirsher 21598b7325b4SBen Hutchings rc = efx_check_disabled(efx); 21608b7325b4SBen Hutchings if (rc) 21618b7325b4SBen Hutchings return rc; 2162874aeea5SJeff Kirsher if (efx->phy_mode & PHY_MODE_SPECIAL) 2163874aeea5SJeff Kirsher return -EBUSY; 2164874aeea5SJeff Kirsher if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2165874aeea5SJeff Kirsher return -EIO; 2166874aeea5SJeff Kirsher 2167874aeea5SJeff Kirsher /* Notify the kernel of the link state polled during driver load, 2168874aeea5SJeff Kirsher * before the monitor starts running */ 2169874aeea5SJeff Kirsher efx_link_status_changed(efx); 2170874aeea5SJeff Kirsher 2171874aeea5SJeff Kirsher efx_start_all(efx); 2172dd40781eSBen Hutchings efx_selftest_async_start(efx); 2173874aeea5SJeff Kirsher return 0; 2174874aeea5SJeff Kirsher } 2175874aeea5SJeff Kirsher 2176874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. 2177874aeea5SJeff Kirsher * Note that the kernel will ignore our return code; this method 2178874aeea5SJeff Kirsher * should really be a void. 2179874aeea5SJeff Kirsher */ 2180e340be92SShradha Shah int efx_net_stop(struct net_device *net_dev) 2181874aeea5SJeff Kirsher { 2182874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2183874aeea5SJeff Kirsher 2184874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2185874aeea5SJeff Kirsher raw_smp_processor_id()); 2186874aeea5SJeff Kirsher 2187874aeea5SJeff Kirsher /* Stop the device and flush all the channels */ 2188874aeea5SJeff Kirsher efx_stop_all(efx); 2189874aeea5SJeff Kirsher 2190874aeea5SJeff Kirsher return 0; 2191874aeea5SJeff Kirsher } 2192874aeea5SJeff Kirsher 2193874aeea5SJeff Kirsher /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 21942aa9ef11SBen Hutchings static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev, 21952aa9ef11SBen Hutchings struct rtnl_link_stats64 *stats) 2196874aeea5SJeff Kirsher { 2197874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2198874aeea5SJeff Kirsher 2199874aeea5SJeff Kirsher spin_lock_bh(&efx->stats_lock); 2200cd0ecc9aSBen Hutchings efx->type->update_stats(efx, NULL, stats); 22011cb34522SBen Hutchings spin_unlock_bh(&efx->stats_lock); 22021cb34522SBen Hutchings 2203874aeea5SJeff Kirsher return stats; 2204874aeea5SJeff Kirsher } 2205874aeea5SJeff Kirsher 2206874aeea5SJeff Kirsher /* Context: netif_tx_lock held, BHs disabled. */ 2207874aeea5SJeff Kirsher static void efx_watchdog(struct net_device *net_dev) 2208874aeea5SJeff Kirsher { 2209874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2210874aeea5SJeff Kirsher 2211874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 2212874aeea5SJeff Kirsher "TX stuck with port_enabled=%d: resetting channels\n", 2213874aeea5SJeff Kirsher efx->port_enabled); 2214874aeea5SJeff Kirsher 2215874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2216874aeea5SJeff Kirsher } 2217874aeea5SJeff Kirsher 2218874aeea5SJeff Kirsher 2219874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2220874aeea5SJeff Kirsher static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2221874aeea5SJeff Kirsher { 2222874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 22238b7325b4SBen Hutchings int rc; 2224874aeea5SJeff Kirsher 22258b7325b4SBen Hutchings rc = efx_check_disabled(efx); 22268b7325b4SBen Hutchings if (rc) 22278b7325b4SBen Hutchings return rc; 2228874aeea5SJeff Kirsher if (new_mtu > EFX_MAX_MTU) 2229874aeea5SJeff Kirsher return -EINVAL; 2230874aeea5SJeff Kirsher 2231874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2232874aeea5SJeff Kirsher 223329c69a48SBen Hutchings efx_device_detach_sync(efx); 223429c69a48SBen Hutchings efx_stop_all(efx); 223529c69a48SBen Hutchings 2236874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2237874aeea5SJeff Kirsher net_dev->mtu = new_mtu; 22380d322413SEdward Cree efx_mac_reconfigure(efx); 2239874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2240874aeea5SJeff Kirsher 2241874aeea5SJeff Kirsher efx_start_all(efx); 224229c69a48SBen Hutchings netif_device_attach(efx->net_dev); 22436c8eef4aSBen Hutchings return 0; 2244874aeea5SJeff Kirsher } 2245874aeea5SJeff Kirsher 2246874aeea5SJeff Kirsher static int efx_set_mac_address(struct net_device *net_dev, void *data) 2247874aeea5SJeff Kirsher { 2248874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2249874aeea5SJeff Kirsher struct sockaddr *addr = data; 2250e0b3ae30SBen Hutchings u8 *new_addr = addr->sa_data; 2251cfc77c2fSShradha Shah u8 old_addr[6]; 2252cfc77c2fSShradha Shah int rc; 2253874aeea5SJeff Kirsher 2254874aeea5SJeff Kirsher if (!is_valid_ether_addr(new_addr)) { 2255874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2256874aeea5SJeff Kirsher "invalid ethernet MAC address requested: %pM\n", 2257874aeea5SJeff Kirsher new_addr); 2258504f9b5aSDanny Kukawka return -EADDRNOTAVAIL; 2259874aeea5SJeff Kirsher } 2260874aeea5SJeff Kirsher 2261cfc77c2fSShradha Shah /* save old address */ 2262cfc77c2fSShradha Shah ether_addr_copy(old_addr, net_dev->dev_addr); 2263cd84ff4dSEdward Cree ether_addr_copy(net_dev->dev_addr, new_addr); 2264910c8789SShradha Shah if (efx->type->set_mac_address) { 2265910c8789SShradha Shah rc = efx->type->set_mac_address(efx); 2266cfc77c2fSShradha Shah if (rc) { 2267cfc77c2fSShradha Shah ether_addr_copy(net_dev->dev_addr, old_addr); 2268cfc77c2fSShradha Shah return rc; 2269cfc77c2fSShradha Shah } 2270cfc77c2fSShradha Shah } 2271874aeea5SJeff Kirsher 2272874aeea5SJeff Kirsher /* Reconfigure the MAC */ 2273874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 22740d322413SEdward Cree efx_mac_reconfigure(efx); 2275874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2276874aeea5SJeff Kirsher 2277874aeea5SJeff Kirsher return 0; 2278874aeea5SJeff Kirsher } 2279874aeea5SJeff Kirsher 2280874aeea5SJeff Kirsher /* Context: netif_addr_lock held, BHs disabled. */ 22810fca8c97SBen Hutchings static void efx_set_rx_mode(struct net_device *net_dev) 2282874aeea5SJeff Kirsher { 2283874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2284874aeea5SJeff Kirsher 2285874aeea5SJeff Kirsher if (efx->port_enabled) 2286874aeea5SJeff Kirsher queue_work(efx->workqueue, &efx->mac_work); 2287874aeea5SJeff Kirsher /* Otherwise efx_start_port() will do this */ 2288874aeea5SJeff Kirsher } 2289874aeea5SJeff Kirsher 2290c8f44affSMichał Mirosław static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2291874aeea5SJeff Kirsher { 2292874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2293874aeea5SJeff Kirsher 2294874aeea5SJeff Kirsher /* If disabling RX n-tuple filtering, clear existing filters */ 2295874aeea5SJeff Kirsher if (net_dev->features & ~data & NETIF_F_NTUPLE) 2296fbd79120SBen Hutchings return efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2297874aeea5SJeff Kirsher 2298874aeea5SJeff Kirsher return 0; 2299874aeea5SJeff Kirsher } 2300874aeea5SJeff Kirsher 23017fa8d547SShradha Shah static const struct net_device_ops efx_netdev_ops = { 2302874aeea5SJeff Kirsher .ndo_open = efx_net_open, 2303874aeea5SJeff Kirsher .ndo_stop = efx_net_stop, 2304874aeea5SJeff Kirsher .ndo_get_stats64 = efx_net_stats, 2305874aeea5SJeff Kirsher .ndo_tx_timeout = efx_watchdog, 2306874aeea5SJeff Kirsher .ndo_start_xmit = efx_hard_start_xmit, 2307874aeea5SJeff Kirsher .ndo_validate_addr = eth_validate_addr, 2308874aeea5SJeff Kirsher .ndo_do_ioctl = efx_ioctl, 2309874aeea5SJeff Kirsher .ndo_change_mtu = efx_change_mtu, 2310874aeea5SJeff Kirsher .ndo_set_mac_address = efx_set_mac_address, 23110fca8c97SBen Hutchings .ndo_set_rx_mode = efx_set_rx_mode, 2312874aeea5SJeff Kirsher .ndo_set_features = efx_set_features, 2313cd2d5b52SBen Hutchings #ifdef CONFIG_SFC_SRIOV 23147fa8d547SShradha Shah .ndo_set_vf_mac = efx_sriov_set_vf_mac, 23157fa8d547SShradha Shah .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 23167fa8d547SShradha Shah .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 23177fa8d547SShradha Shah .ndo_get_vf_config = efx_sriov_get_vf_config, 23184392dc69SEdward Cree .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 23191d051e00SShradha Shah .ndo_get_phys_port_id = efx_sriov_get_phys_port_id, 2320cd2d5b52SBen Hutchings #endif 2321874aeea5SJeff Kirsher #ifdef CONFIG_NET_POLL_CONTROLLER 2322874aeea5SJeff Kirsher .ndo_poll_controller = efx_netpoll, 2323874aeea5SJeff Kirsher #endif 2324874aeea5SJeff Kirsher .ndo_setup_tc = efx_setup_tc, 232536763266SAlexandre Rames #ifdef CONFIG_NET_RX_BUSY_POLL 232636763266SAlexandre Rames .ndo_busy_poll = efx_busy_poll, 232736763266SAlexandre Rames #endif 2328874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 2329874aeea5SJeff Kirsher .ndo_rx_flow_steer = efx_filter_rfs, 2330874aeea5SJeff Kirsher #endif 2331874aeea5SJeff Kirsher }; 2332874aeea5SJeff Kirsher 2333874aeea5SJeff Kirsher static void efx_update_name(struct efx_nic *efx) 2334874aeea5SJeff Kirsher { 2335874aeea5SJeff Kirsher strcpy(efx->name, efx->net_dev->name); 2336874aeea5SJeff Kirsher efx_mtd_rename(efx); 2337874aeea5SJeff Kirsher efx_set_channel_names(efx); 2338874aeea5SJeff Kirsher } 2339874aeea5SJeff Kirsher 2340874aeea5SJeff Kirsher static int efx_netdev_event(struct notifier_block *this, 2341874aeea5SJeff Kirsher unsigned long event, void *ptr) 2342874aeea5SJeff Kirsher { 2343351638e7SJiri Pirko struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2344874aeea5SJeff Kirsher 23457fa8d547SShradha Shah if ((net_dev->netdev_ops == &efx_netdev_ops) && 2346874aeea5SJeff Kirsher event == NETDEV_CHANGENAME) 2347874aeea5SJeff Kirsher efx_update_name(netdev_priv(net_dev)); 2348874aeea5SJeff Kirsher 2349874aeea5SJeff Kirsher return NOTIFY_DONE; 2350874aeea5SJeff Kirsher } 2351874aeea5SJeff Kirsher 2352874aeea5SJeff Kirsher static struct notifier_block efx_netdev_notifier = { 2353874aeea5SJeff Kirsher .notifier_call = efx_netdev_event, 2354874aeea5SJeff Kirsher }; 2355874aeea5SJeff Kirsher 2356874aeea5SJeff Kirsher static ssize_t 2357874aeea5SJeff Kirsher show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2358874aeea5SJeff Kirsher { 2359874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2360874aeea5SJeff Kirsher return sprintf(buf, "%d\n", efx->phy_type); 2361874aeea5SJeff Kirsher } 2362776fbcc9SBen Hutchings static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2363874aeea5SJeff Kirsher 2364e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2365e7fef9b4SEdward Cree static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2366e7fef9b4SEdward Cree char *buf) 2367e7fef9b4SEdward Cree { 2368e7fef9b4SEdward Cree struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2369e7fef9b4SEdward Cree struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2370e7fef9b4SEdward Cree 2371e7fef9b4SEdward Cree return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2372e7fef9b4SEdward Cree } 2373e7fef9b4SEdward Cree static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2374e7fef9b4SEdward Cree const char *buf, size_t count) 2375e7fef9b4SEdward Cree { 2376e7fef9b4SEdward Cree struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2377e7fef9b4SEdward Cree struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2378e7fef9b4SEdward Cree bool enable = count > 0 && *buf != '0'; 2379e7fef9b4SEdward Cree 2380e7fef9b4SEdward Cree mcdi->logging_enabled = enable; 2381e7fef9b4SEdward Cree return count; 2382e7fef9b4SEdward Cree } 2383e7fef9b4SEdward Cree static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2384e7fef9b4SEdward Cree #endif 2385e7fef9b4SEdward Cree 2386874aeea5SJeff Kirsher static int efx_register_netdev(struct efx_nic *efx) 2387874aeea5SJeff Kirsher { 2388874aeea5SJeff Kirsher struct net_device *net_dev = efx->net_dev; 2389874aeea5SJeff Kirsher struct efx_channel *channel; 2390874aeea5SJeff Kirsher int rc; 2391874aeea5SJeff Kirsher 2392874aeea5SJeff Kirsher net_dev->watchdog_timeo = 5 * HZ; 2393874aeea5SJeff Kirsher net_dev->irq = efx->pci_dev->irq; 23947fa8d547SShradha Shah net_dev->netdev_ops = &efx_netdev_ops; 23957fa8d547SShradha Shah if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 23968127d661SBen Hutchings net_dev->priv_flags |= IFF_UNICAST_FLT; 23977ad24ea4SWilfried Klaebe net_dev->ethtool_ops = &efx_ethtool_ops; 23987e6d06f0SBen Hutchings net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2399874aeea5SJeff Kirsher 2400874aeea5SJeff Kirsher rtnl_lock(); 2401874aeea5SJeff Kirsher 24027153f623SBen Hutchings /* Enable resets to be scheduled and check whether any were 24037153f623SBen Hutchings * already requested. If so, the NIC is probably hosed so we 24047153f623SBen Hutchings * abort. 24057153f623SBen Hutchings */ 24067153f623SBen Hutchings efx->state = STATE_READY; 24077153f623SBen Hutchings smp_mb(); /* ensure we change state before checking reset_pending */ 24087153f623SBen Hutchings if (efx->reset_pending) { 24097153f623SBen Hutchings netif_err(efx, probe, efx->net_dev, 24107153f623SBen Hutchings "aborting probe due to scheduled reset\n"); 24117153f623SBen Hutchings rc = -EIO; 24127153f623SBen Hutchings goto fail_locked; 24137153f623SBen Hutchings } 24147153f623SBen Hutchings 2415874aeea5SJeff Kirsher rc = dev_alloc_name(net_dev, net_dev->name); 2416874aeea5SJeff Kirsher if (rc < 0) 2417874aeea5SJeff Kirsher goto fail_locked; 2418874aeea5SJeff Kirsher efx_update_name(efx); 2419874aeea5SJeff Kirsher 24208f8b3d51SBen Hutchings /* Always start with carrier off; PHY events will detect the link */ 24218f8b3d51SBen Hutchings netif_carrier_off(net_dev); 24228f8b3d51SBen Hutchings 2423874aeea5SJeff Kirsher rc = register_netdevice(net_dev); 2424874aeea5SJeff Kirsher if (rc) 2425874aeea5SJeff Kirsher goto fail_locked; 2426874aeea5SJeff Kirsher 2427874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2428874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 2429874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 2430874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 2431874aeea5SJeff Kirsher } 2432874aeea5SJeff Kirsher 24330bcf4a64SBen Hutchings efx_associate(efx); 24340bcf4a64SBen Hutchings 2435874aeea5SJeff Kirsher rtnl_unlock(); 2436874aeea5SJeff Kirsher 2437874aeea5SJeff Kirsher rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2438874aeea5SJeff Kirsher if (rc) { 2439874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2440874aeea5SJeff Kirsher "failed to init net dev attributes\n"); 2441874aeea5SJeff Kirsher goto fail_registered; 2442874aeea5SJeff Kirsher } 2443e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2444e7fef9b4SEdward Cree rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2445e7fef9b4SEdward Cree if (rc) { 2446e7fef9b4SEdward Cree netif_err(efx, drv, efx->net_dev, 2447e7fef9b4SEdward Cree "failed to init net dev attributes\n"); 2448e7fef9b4SEdward Cree goto fail_attr_mcdi_logging; 2449e7fef9b4SEdward Cree } 2450e7fef9b4SEdward Cree #endif 2451874aeea5SJeff Kirsher 2452874aeea5SJeff Kirsher return 0; 2453874aeea5SJeff Kirsher 2454e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2455e7fef9b4SEdward Cree fail_attr_mcdi_logging: 2456e7fef9b4SEdward Cree device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2457e7fef9b4SEdward Cree #endif 24587153f623SBen Hutchings fail_registered: 24597153f623SBen Hutchings rtnl_lock(); 24600bcf4a64SBen Hutchings efx_dissociate(efx); 24617153f623SBen Hutchings unregister_netdevice(net_dev); 2462874aeea5SJeff Kirsher fail_locked: 24637153f623SBen Hutchings efx->state = STATE_UNINIT; 2464874aeea5SJeff Kirsher rtnl_unlock(); 2465874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2466874aeea5SJeff Kirsher return rc; 2467874aeea5SJeff Kirsher } 2468874aeea5SJeff Kirsher 2469874aeea5SJeff Kirsher static void efx_unregister_netdev(struct efx_nic *efx) 2470874aeea5SJeff Kirsher { 2471874aeea5SJeff Kirsher if (!efx->net_dev) 2472874aeea5SJeff Kirsher return; 2473874aeea5SJeff Kirsher 2474874aeea5SJeff Kirsher BUG_ON(netdev_priv(efx->net_dev) != efx); 2475874aeea5SJeff Kirsher 2476e7fef9b4SEdward Cree if (efx_dev_registered(efx)) { 2477874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2478e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2479e7fef9b4SEdward Cree device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2480e7fef9b4SEdward Cree #endif 2481874aeea5SJeff Kirsher device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2482e7fef9b4SEdward Cree unregister_netdev(efx->net_dev); 2483e7fef9b4SEdward Cree } 2484874aeea5SJeff Kirsher } 2485874aeea5SJeff Kirsher 2486874aeea5SJeff Kirsher /************************************************************************** 2487874aeea5SJeff Kirsher * 2488874aeea5SJeff Kirsher * Device reset and suspend 2489874aeea5SJeff Kirsher * 2490874aeea5SJeff Kirsher **************************************************************************/ 2491874aeea5SJeff Kirsher 2492874aeea5SJeff Kirsher /* Tears down the entire software state and most of the hardware state 2493874aeea5SJeff Kirsher * before reset. */ 2494874aeea5SJeff Kirsher void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2495874aeea5SJeff Kirsher { 2496874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2497874aeea5SJeff Kirsher 2498e283546cSEdward Cree if (method == RESET_TYPE_MCDI_TIMEOUT) 2499e283546cSEdward Cree efx->type->prepare_flr(efx); 2500e283546cSEdward Cree 2501874aeea5SJeff Kirsher efx_stop_all(efx); 2502d8291187SBen Hutchings efx_disable_interrupts(efx); 25035642ceefSBen Hutchings 25045642ceefSBen Hutchings mutex_lock(&efx->mac_lock); 2505087e9025SJon Cooper if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2506087e9025SJon Cooper method != RESET_TYPE_DATAPATH) 2507874aeea5SJeff Kirsher efx->phy_op->fini(efx); 2508874aeea5SJeff Kirsher efx->type->fini(efx); 2509874aeea5SJeff Kirsher } 2510874aeea5SJeff Kirsher 2511874aeea5SJeff Kirsher /* This function will always ensure that the locks acquired in 2512874aeea5SJeff Kirsher * efx_reset_down() are released. A failure return code indicates 2513874aeea5SJeff Kirsher * that we were unable to reinitialise the hardware, and the 2514874aeea5SJeff Kirsher * driver should be disabled. If ok is false, then the rx and tx 2515874aeea5SJeff Kirsher * engines are not restarted, pending a RESET_DISABLE. */ 2516874aeea5SJeff Kirsher int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2517874aeea5SJeff Kirsher { 2518874aeea5SJeff Kirsher int rc; 2519874aeea5SJeff Kirsher 2520874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2521874aeea5SJeff Kirsher 2522e283546cSEdward Cree if (method == RESET_TYPE_MCDI_TIMEOUT) 2523e283546cSEdward Cree efx->type->finish_flr(efx); 2524e283546cSEdward Cree 2525e283546cSEdward Cree /* Ensure that SRAM is initialised even if we're disabling the device */ 2526874aeea5SJeff Kirsher rc = efx->type->init(efx); 2527874aeea5SJeff Kirsher if (rc) { 2528874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2529874aeea5SJeff Kirsher goto fail; 2530874aeea5SJeff Kirsher } 2531874aeea5SJeff Kirsher 2532874aeea5SJeff Kirsher if (!ok) 2533874aeea5SJeff Kirsher goto fail; 2534874aeea5SJeff Kirsher 2535087e9025SJon Cooper if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2536087e9025SJon Cooper method != RESET_TYPE_DATAPATH) { 2537874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 2538874aeea5SJeff Kirsher if (rc) 2539874aeea5SJeff Kirsher goto fail; 2540267d9d73SEdward Cree rc = efx->phy_op->reconfigure(efx); 2541267d9d73SEdward Cree if (rc && rc != -EPERM) 2542874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2543874aeea5SJeff Kirsher "could not restore PHY settings\n"); 2544874aeea5SJeff Kirsher } 2545874aeea5SJeff Kirsher 2546261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 2547261e4d96SJon Cooper if (rc) 2548261e4d96SJon Cooper goto fail; 25496d8aaaf6SDaniel Pieczko 25506d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 25516d8aaaf6SDaniel Pieczko rc = efx->type->vswitching_restore(efx); 25526d8aaaf6SDaniel Pieczko if (rc) /* not fatal; the PF will still work fine */ 25536d8aaaf6SDaniel Pieczko netif_warn(efx, probe, efx->net_dev, 25546d8aaaf6SDaniel Pieczko "failed to restore vswitching rc=%d;" 25556d8aaaf6SDaniel Pieczko " VFs may not function\n", rc); 25566d8aaaf6SDaniel Pieczko #endif 25576d8aaaf6SDaniel Pieczko 25580d322413SEdward Cree down_read(&efx->filter_sem); 2559874aeea5SJeff Kirsher efx_restore_filters(efx); 25600d322413SEdward Cree up_read(&efx->filter_sem); 25617fa8d547SShradha Shah if (efx->type->sriov_reset) 2562d98a4ffeSShradha Shah efx->type->sriov_reset(efx); 2563874aeea5SJeff Kirsher 2564874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2565874aeea5SJeff Kirsher 2566874aeea5SJeff Kirsher efx_start_all(efx); 2567874aeea5SJeff Kirsher 2568874aeea5SJeff Kirsher return 0; 2569874aeea5SJeff Kirsher 2570874aeea5SJeff Kirsher fail: 2571874aeea5SJeff Kirsher efx->port_initialized = false; 2572874aeea5SJeff Kirsher 2573874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2574874aeea5SJeff Kirsher 2575874aeea5SJeff Kirsher return rc; 2576874aeea5SJeff Kirsher } 2577874aeea5SJeff Kirsher 2578874aeea5SJeff Kirsher /* Reset the NIC using the specified method. Note that the reset may 2579874aeea5SJeff Kirsher * fail, in which case the card will be left in an unusable state. 2580874aeea5SJeff Kirsher * 2581874aeea5SJeff Kirsher * Caller must hold the rtnl_lock. 2582874aeea5SJeff Kirsher */ 2583874aeea5SJeff Kirsher int efx_reset(struct efx_nic *efx, enum reset_type method) 2584874aeea5SJeff Kirsher { 2585874aeea5SJeff Kirsher int rc, rc2; 2586874aeea5SJeff Kirsher bool disabled; 2587874aeea5SJeff Kirsher 2588874aeea5SJeff Kirsher netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2589874aeea5SJeff Kirsher RESET_TYPE(method)); 2590874aeea5SJeff Kirsher 2591c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2592874aeea5SJeff Kirsher efx_reset_down(efx, method); 2593874aeea5SJeff Kirsher 2594874aeea5SJeff Kirsher rc = efx->type->reset(efx, method); 2595874aeea5SJeff Kirsher if (rc) { 2596874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2597874aeea5SJeff Kirsher goto out; 2598874aeea5SJeff Kirsher } 2599874aeea5SJeff Kirsher 2600874aeea5SJeff Kirsher /* Clear flags for the scopes we covered. We assume the NIC and 2601874aeea5SJeff Kirsher * driver are now quiescent so that there is no race here. 2602874aeea5SJeff Kirsher */ 2603e283546cSEdward Cree if (method < RESET_TYPE_MAX_METHOD) 2604874aeea5SJeff Kirsher efx->reset_pending &= -(1 << (method + 1)); 2605e283546cSEdward Cree else /* it doesn't fit into the well-ordered scope hierarchy */ 2606e283546cSEdward Cree __clear_bit(method, &efx->reset_pending); 2607874aeea5SJeff Kirsher 2608874aeea5SJeff Kirsher /* Reinitialise bus-mastering, which may have been turned off before 2609874aeea5SJeff Kirsher * the reset was scheduled. This is still appropriate, even in the 2610874aeea5SJeff Kirsher * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2611874aeea5SJeff Kirsher * can respond to requests. */ 2612874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 2613874aeea5SJeff Kirsher 2614874aeea5SJeff Kirsher out: 2615874aeea5SJeff Kirsher /* Leave device stopped if necessary */ 2616626950dbSAlexandre Rames disabled = rc || 2617626950dbSAlexandre Rames method == RESET_TYPE_DISABLE || 2618626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_DISABLE; 2619874aeea5SJeff Kirsher rc2 = efx_reset_up(efx, method, !disabled); 2620874aeea5SJeff Kirsher if (rc2) { 2621874aeea5SJeff Kirsher disabled = true; 2622874aeea5SJeff Kirsher if (!rc) 2623874aeea5SJeff Kirsher rc = rc2; 2624874aeea5SJeff Kirsher } 2625874aeea5SJeff Kirsher 2626874aeea5SJeff Kirsher if (disabled) { 2627874aeea5SJeff Kirsher dev_close(efx->net_dev); 2628874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2629874aeea5SJeff Kirsher efx->state = STATE_DISABLED; 2630874aeea5SJeff Kirsher } else { 2631874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2632874aeea5SJeff Kirsher netif_device_attach(efx->net_dev); 2633874aeea5SJeff Kirsher } 2634874aeea5SJeff Kirsher return rc; 2635874aeea5SJeff Kirsher } 2636874aeea5SJeff Kirsher 2637626950dbSAlexandre Rames /* Try recovery mechanisms. 2638626950dbSAlexandre Rames * For now only EEH is supported. 2639626950dbSAlexandre Rames * Returns 0 if the recovery mechanisms are unsuccessful. 2640626950dbSAlexandre Rames * Returns a non-zero value otherwise. 2641626950dbSAlexandre Rames */ 2642b28405b0SAlexandre Rames int efx_try_recovery(struct efx_nic *efx) 2643626950dbSAlexandre Rames { 2644626950dbSAlexandre Rames #ifdef CONFIG_EEH 2645626950dbSAlexandre Rames /* A PCI error can occur and not be seen by EEH because nothing 2646626950dbSAlexandre Rames * happens on the PCI bus. In this case the driver may fail and 2647626950dbSAlexandre Rames * schedule a 'recover or reset', leading to this recovery handler. 2648626950dbSAlexandre Rames * Manually call the eeh failure check function. 2649626950dbSAlexandre Rames */ 265012a89dbaSBenjamin Herrenschmidt struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2651626950dbSAlexandre Rames if (eeh_dev_check_failure(eehdev)) { 2652626950dbSAlexandre Rames /* The EEH mechanisms will handle the error and reset the 2653626950dbSAlexandre Rames * device if necessary. 2654626950dbSAlexandre Rames */ 2655626950dbSAlexandre Rames return 1; 2656626950dbSAlexandre Rames } 2657626950dbSAlexandre Rames #endif 2658626950dbSAlexandre Rames return 0; 2659626950dbSAlexandre Rames } 2660626950dbSAlexandre Rames 266174cd60a4SJon Cooper static void efx_wait_for_bist_end(struct efx_nic *efx) 266274cd60a4SJon Cooper { 266374cd60a4SJon Cooper int i; 266474cd60a4SJon Cooper 266574cd60a4SJon Cooper for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 266674cd60a4SJon Cooper if (efx_mcdi_poll_reboot(efx)) 266774cd60a4SJon Cooper goto out; 266874cd60a4SJon Cooper msleep(BIST_WAIT_DELAY_MS); 266974cd60a4SJon Cooper } 267074cd60a4SJon Cooper 267174cd60a4SJon Cooper netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 267274cd60a4SJon Cooper out: 267374cd60a4SJon Cooper /* Either way unset the BIST flag. If we found no reboot we probably 267474cd60a4SJon Cooper * won't recover, but we should try. 267574cd60a4SJon Cooper */ 267674cd60a4SJon Cooper efx->mc_bist_for_other_fn = false; 267774cd60a4SJon Cooper } 267874cd60a4SJon Cooper 2679874aeea5SJeff Kirsher /* The worker thread exists so that code that cannot sleep can 2680874aeea5SJeff Kirsher * schedule a reset for later. 2681874aeea5SJeff Kirsher */ 2682874aeea5SJeff Kirsher static void efx_reset_work(struct work_struct *data) 2683874aeea5SJeff Kirsher { 2684874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2685626950dbSAlexandre Rames unsigned long pending; 2686626950dbSAlexandre Rames enum reset_type method; 2687626950dbSAlexandre Rames 2688626950dbSAlexandre Rames pending = ACCESS_ONCE(efx->reset_pending); 2689626950dbSAlexandre Rames method = fls(pending) - 1; 2690626950dbSAlexandre Rames 269174cd60a4SJon Cooper if (method == RESET_TYPE_MC_BIST) 269274cd60a4SJon Cooper efx_wait_for_bist_end(efx); 269374cd60a4SJon Cooper 2694626950dbSAlexandre Rames if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2695626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_ALL) && 2696626950dbSAlexandre Rames efx_try_recovery(efx)) 2697626950dbSAlexandre Rames return; 2698874aeea5SJeff Kirsher 2699874aeea5SJeff Kirsher if (!pending) 2700874aeea5SJeff Kirsher return; 2701874aeea5SJeff Kirsher 2702874aeea5SJeff Kirsher rtnl_lock(); 27037153f623SBen Hutchings 27047153f623SBen Hutchings /* We checked the state in efx_schedule_reset() but it may 27057153f623SBen Hutchings * have changed by now. Now that we have the RTNL lock, 27067153f623SBen Hutchings * it cannot change again. 27077153f623SBen Hutchings */ 27087153f623SBen Hutchings if (efx->state == STATE_READY) 2709626950dbSAlexandre Rames (void)efx_reset(efx, method); 27107153f623SBen Hutchings 2711874aeea5SJeff Kirsher rtnl_unlock(); 2712874aeea5SJeff Kirsher } 2713874aeea5SJeff Kirsher 2714874aeea5SJeff Kirsher void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2715874aeea5SJeff Kirsher { 2716874aeea5SJeff Kirsher enum reset_type method; 2717874aeea5SJeff Kirsher 2718626950dbSAlexandre Rames if (efx->state == STATE_RECOVERY) { 2719626950dbSAlexandre Rames netif_dbg(efx, drv, efx->net_dev, 2720626950dbSAlexandre Rames "recovering: skip scheduling %s reset\n", 2721626950dbSAlexandre Rames RESET_TYPE(type)); 2722626950dbSAlexandre Rames return; 2723626950dbSAlexandre Rames } 2724626950dbSAlexandre Rames 2725874aeea5SJeff Kirsher switch (type) { 2726874aeea5SJeff Kirsher case RESET_TYPE_INVISIBLE: 2727874aeea5SJeff Kirsher case RESET_TYPE_ALL: 2728626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_ALL: 2729874aeea5SJeff Kirsher case RESET_TYPE_WORLD: 2730874aeea5SJeff Kirsher case RESET_TYPE_DISABLE: 2731626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_DISABLE: 2732087e9025SJon Cooper case RESET_TYPE_DATAPATH: 273374cd60a4SJon Cooper case RESET_TYPE_MC_BIST: 2734e283546cSEdward Cree case RESET_TYPE_MCDI_TIMEOUT: 2735874aeea5SJeff Kirsher method = type; 2736874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2737874aeea5SJeff Kirsher RESET_TYPE(method)); 2738874aeea5SJeff Kirsher break; 2739874aeea5SJeff Kirsher default: 2740874aeea5SJeff Kirsher method = efx->type->map_reset_reason(type); 2741874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, 2742874aeea5SJeff Kirsher "scheduling %s reset for %s\n", 2743874aeea5SJeff Kirsher RESET_TYPE(method), RESET_TYPE(type)); 2744874aeea5SJeff Kirsher break; 2745874aeea5SJeff Kirsher } 2746874aeea5SJeff Kirsher 2747874aeea5SJeff Kirsher set_bit(method, &efx->reset_pending); 27487153f623SBen Hutchings smp_mb(); /* ensure we change reset_pending before checking state */ 27497153f623SBen Hutchings 27507153f623SBen Hutchings /* If we're not READY then just leave the flags set as the cue 27517153f623SBen Hutchings * to abort probing or reschedule the reset later. 27527153f623SBen Hutchings */ 27537153f623SBen Hutchings if (ACCESS_ONCE(efx->state) != STATE_READY) 27547153f623SBen Hutchings return; 2755874aeea5SJeff Kirsher 2756874aeea5SJeff Kirsher /* efx_process_channel() will no longer read events once a 2757874aeea5SJeff Kirsher * reset is scheduled. So switch back to poll'd MCDI completions. */ 2758874aeea5SJeff Kirsher efx_mcdi_mode_poll(efx); 2759874aeea5SJeff Kirsher 2760874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 2761874aeea5SJeff Kirsher } 2762874aeea5SJeff Kirsher 2763874aeea5SJeff Kirsher /************************************************************************** 2764874aeea5SJeff Kirsher * 2765874aeea5SJeff Kirsher * List of NICs we support 2766874aeea5SJeff Kirsher * 2767874aeea5SJeff Kirsher **************************************************************************/ 2768874aeea5SJeff Kirsher 2769874aeea5SJeff Kirsher /* PCI device ID table */ 27709baa3c34SBenoit Taine static const struct pci_device_id efx_pci_table[] = { 27710e59e7e7SLinus Torvalds {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 27720e59e7e7SLinus Torvalds PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0), 2773874aeea5SJeff Kirsher .driver_data = (unsigned long) &falcon_a1_nic_type}, 27740e59e7e7SLinus Torvalds {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 27750e59e7e7SLinus Torvalds PCI_DEVICE_ID_SOLARFLARE_SFC4000B), 2776874aeea5SJeff Kirsher .driver_data = (unsigned long) &falcon_b0_nic_type}, 2777547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2778874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 2779547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2780874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 27818127d661SBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 27828127d661SBen Hutchings .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 27836f7f8aa6SShradha Shah {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 27846f7f8aa6SShradha Shah .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 27853b06a00eSMateusz Wrzesinski {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 27863b06a00eSMateusz Wrzesinski .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2787dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2788dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2789dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2790dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2791dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2792dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2793874aeea5SJeff Kirsher {0} /* end of list */ 2794874aeea5SJeff Kirsher }; 2795874aeea5SJeff Kirsher 2796874aeea5SJeff Kirsher /************************************************************************** 2797874aeea5SJeff Kirsher * 2798874aeea5SJeff Kirsher * Dummy PHY/MAC operations 2799874aeea5SJeff Kirsher * 2800874aeea5SJeff Kirsher * Can be used for some unimplemented operations 2801874aeea5SJeff Kirsher * Needed so all function pointers are valid and do not have to be tested 2802874aeea5SJeff Kirsher * before use 2803874aeea5SJeff Kirsher * 2804874aeea5SJeff Kirsher **************************************************************************/ 2805874aeea5SJeff Kirsher int efx_port_dummy_op_int(struct efx_nic *efx) 2806874aeea5SJeff Kirsher { 2807874aeea5SJeff Kirsher return 0; 2808874aeea5SJeff Kirsher } 2809874aeea5SJeff Kirsher void efx_port_dummy_op_void(struct efx_nic *efx) {} 2810874aeea5SJeff Kirsher 2811874aeea5SJeff Kirsher static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2812874aeea5SJeff Kirsher { 2813874aeea5SJeff Kirsher return false; 2814874aeea5SJeff Kirsher } 2815874aeea5SJeff Kirsher 2816874aeea5SJeff Kirsher static const struct efx_phy_operations efx_dummy_phy_operations = { 2817874aeea5SJeff Kirsher .init = efx_port_dummy_op_int, 2818874aeea5SJeff Kirsher .reconfigure = efx_port_dummy_op_int, 2819874aeea5SJeff Kirsher .poll = efx_port_dummy_op_poll, 2820874aeea5SJeff Kirsher .fini = efx_port_dummy_op_void, 2821874aeea5SJeff Kirsher }; 2822874aeea5SJeff Kirsher 2823874aeea5SJeff Kirsher /************************************************************************** 2824874aeea5SJeff Kirsher * 2825874aeea5SJeff Kirsher * Data housekeeping 2826874aeea5SJeff Kirsher * 2827874aeea5SJeff Kirsher **************************************************************************/ 2828874aeea5SJeff Kirsher 2829874aeea5SJeff Kirsher /* This zeroes out and then fills in the invariants in a struct 2830874aeea5SJeff Kirsher * efx_nic (including all sub-structures). 2831874aeea5SJeff Kirsher */ 2832adeb15aaSBen Hutchings static int efx_init_struct(struct efx_nic *efx, 2833874aeea5SJeff Kirsher struct pci_dev *pci_dev, struct net_device *net_dev) 2834874aeea5SJeff Kirsher { 2835874aeea5SJeff Kirsher int i; 2836874aeea5SJeff Kirsher 2837874aeea5SJeff Kirsher /* Initialise common structures */ 28380bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->node); 28390bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->secondary_list); 2840874aeea5SJeff Kirsher spin_lock_init(&efx->biu_lock); 2841874aeea5SJeff Kirsher #ifdef CONFIG_SFC_MTD 2842874aeea5SJeff Kirsher INIT_LIST_HEAD(&efx->mtd_list); 2843874aeea5SJeff Kirsher #endif 2844874aeea5SJeff Kirsher INIT_WORK(&efx->reset_work, efx_reset_work); 2845874aeea5SJeff Kirsher INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 2846dd40781eSBen Hutchings INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 2847874aeea5SJeff Kirsher efx->pci_dev = pci_dev; 2848874aeea5SJeff Kirsher efx->msg_enable = debug; 2849f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 2850874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2851874aeea5SJeff Kirsher 2852874aeea5SJeff Kirsher efx->net_dev = net_dev; 285343a3739dSJon Cooper efx->rx_prefix_size = efx->type->rx_prefix_size; 28542ec03014SAndrew Rybchenko efx->rx_ip_align = 28552ec03014SAndrew Rybchenko NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 285643a3739dSJon Cooper efx->rx_packet_hash_offset = 285743a3739dSJon Cooper efx->type->rx_hash_offset - efx->type->rx_prefix_size; 2858bd9a265dSJon Cooper efx->rx_packet_ts_offset = 2859bd9a265dSJon Cooper efx->type->rx_ts_offset - efx->type->rx_prefix_size; 2860874aeea5SJeff Kirsher spin_lock_init(&efx->stats_lock); 2861874aeea5SJeff Kirsher mutex_init(&efx->mac_lock); 2862874aeea5SJeff Kirsher efx->phy_op = &efx_dummy_phy_operations; 2863874aeea5SJeff Kirsher efx->mdio.dev = net_dev; 2864874aeea5SJeff Kirsher INIT_WORK(&efx->mac_work, efx_mac_work); 28659f2cb71cSBen Hutchings init_waitqueue_head(&efx->flush_wq); 2866874aeea5SJeff Kirsher 2867874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) { 2868874aeea5SJeff Kirsher efx->channel[i] = efx_alloc_channel(efx, i, NULL); 2869874aeea5SJeff Kirsher if (!efx->channel[i]) 2870874aeea5SJeff Kirsher goto fail; 2871d8291187SBen Hutchings efx->msi_context[i].efx = efx; 2872d8291187SBen Hutchings efx->msi_context[i].index = i; 2873874aeea5SJeff Kirsher } 2874874aeea5SJeff Kirsher 2875874aeea5SJeff Kirsher /* Higher numbered interrupt modes are less capable! */ 2876874aeea5SJeff Kirsher efx->interrupt_mode = max(efx->type->max_interrupt_mode, 2877874aeea5SJeff Kirsher interrupt_mode); 2878874aeea5SJeff Kirsher 2879874aeea5SJeff Kirsher /* Would be good to use the net_dev name, but we're too early */ 2880874aeea5SJeff Kirsher snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 2881874aeea5SJeff Kirsher pci_name(pci_dev)); 2882874aeea5SJeff Kirsher efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 2883874aeea5SJeff Kirsher if (!efx->workqueue) 2884874aeea5SJeff Kirsher goto fail; 2885874aeea5SJeff Kirsher 2886874aeea5SJeff Kirsher return 0; 2887874aeea5SJeff Kirsher 2888874aeea5SJeff Kirsher fail: 2889874aeea5SJeff Kirsher efx_fini_struct(efx); 2890874aeea5SJeff Kirsher return -ENOMEM; 2891874aeea5SJeff Kirsher } 2892874aeea5SJeff Kirsher 2893874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx) 2894874aeea5SJeff Kirsher { 2895874aeea5SJeff Kirsher int i; 2896874aeea5SJeff Kirsher 2897874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) 2898874aeea5SJeff Kirsher kfree(efx->channel[i]); 2899874aeea5SJeff Kirsher 2900ef215e64SBen Hutchings kfree(efx->vpd_sn); 2901ef215e64SBen Hutchings 2902874aeea5SJeff Kirsher if (efx->workqueue) { 2903874aeea5SJeff Kirsher destroy_workqueue(efx->workqueue); 2904874aeea5SJeff Kirsher efx->workqueue = NULL; 2905874aeea5SJeff Kirsher } 2906874aeea5SJeff Kirsher } 2907874aeea5SJeff Kirsher 2908e4d112e4SEdward Cree void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 2909e4d112e4SEdward Cree { 2910e4d112e4SEdward Cree u64 n_rx_nodesc_trunc = 0; 2911e4d112e4SEdward Cree struct efx_channel *channel; 2912e4d112e4SEdward Cree 2913e4d112e4SEdward Cree efx_for_each_channel(channel, efx) 2914e4d112e4SEdward Cree n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 2915e4d112e4SEdward Cree stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 2916e4d112e4SEdward Cree stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 2917e4d112e4SEdward Cree } 2918e4d112e4SEdward Cree 2919874aeea5SJeff Kirsher /************************************************************************** 2920874aeea5SJeff Kirsher * 2921874aeea5SJeff Kirsher * PCI interface 2922874aeea5SJeff Kirsher * 2923874aeea5SJeff Kirsher **************************************************************************/ 2924874aeea5SJeff Kirsher 2925874aeea5SJeff Kirsher /* Main body of final NIC shutdown code 2926874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 2927874aeea5SJeff Kirsher */ 2928874aeea5SJeff Kirsher static void efx_pci_remove_main(struct efx_nic *efx) 2929874aeea5SJeff Kirsher { 29307153f623SBen Hutchings /* Flush reset_work. It can no longer be scheduled since we 29317153f623SBen Hutchings * are not READY. 29327153f623SBen Hutchings */ 29337153f623SBen Hutchings BUG_ON(efx->state == STATE_READY); 29347153f623SBen Hutchings cancel_work_sync(&efx->reset_work); 29357153f623SBen Hutchings 2936d8291187SBen Hutchings efx_disable_interrupts(efx); 2937874aeea5SJeff Kirsher efx_nic_fini_interrupt(efx); 2938874aeea5SJeff Kirsher efx_fini_port(efx); 2939874aeea5SJeff Kirsher efx->type->fini(efx); 2940874aeea5SJeff Kirsher efx_fini_napi(efx); 2941874aeea5SJeff Kirsher efx_remove_all(efx); 2942874aeea5SJeff Kirsher } 2943874aeea5SJeff Kirsher 2944874aeea5SJeff Kirsher /* Final NIC shutdown 29452a3fc311SDaniel Pieczko * This is called only at module unload (or hotplug removal). A PF can call 29462a3fc311SDaniel Pieczko * this on its VFs to ensure they are unbound first. 2947874aeea5SJeff Kirsher */ 2948874aeea5SJeff Kirsher static void efx_pci_remove(struct pci_dev *pci_dev) 2949874aeea5SJeff Kirsher { 2950874aeea5SJeff Kirsher struct efx_nic *efx; 2951874aeea5SJeff Kirsher 2952874aeea5SJeff Kirsher efx = pci_get_drvdata(pci_dev); 2953874aeea5SJeff Kirsher if (!efx) 2954874aeea5SJeff Kirsher return; 2955874aeea5SJeff Kirsher 2956874aeea5SJeff Kirsher /* Mark the NIC as fini, then stop the interface */ 2957874aeea5SJeff Kirsher rtnl_lock(); 29580bcf4a64SBen Hutchings efx_dissociate(efx); 2959874aeea5SJeff Kirsher dev_close(efx->net_dev); 2960d8291187SBen Hutchings efx_disable_interrupts(efx); 2961ea6bb99eSEdward Cree efx->state = STATE_UNINIT; 2962874aeea5SJeff Kirsher rtnl_unlock(); 2963874aeea5SJeff Kirsher 29647fa8d547SShradha Shah if (efx->type->sriov_fini) 2965d98a4ffeSShradha Shah efx->type->sriov_fini(efx); 29667fa8d547SShradha Shah 2967874aeea5SJeff Kirsher efx_unregister_netdev(efx); 2968874aeea5SJeff Kirsher 2969874aeea5SJeff Kirsher efx_mtd_remove(efx); 2970874aeea5SJeff Kirsher 2971874aeea5SJeff Kirsher efx_pci_remove_main(efx); 2972874aeea5SJeff Kirsher 2973874aeea5SJeff Kirsher efx_fini_io(efx); 2974874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 2975874aeea5SJeff Kirsher 2976874aeea5SJeff Kirsher efx_fini_struct(efx); 2977874aeea5SJeff Kirsher free_netdev(efx->net_dev); 2978626950dbSAlexandre Rames 2979626950dbSAlexandre Rames pci_disable_pcie_error_reporting(pci_dev); 2980874aeea5SJeff Kirsher }; 2981874aeea5SJeff Kirsher 2982460eeaa0SBen Hutchings /* NIC VPD information 2983460eeaa0SBen Hutchings * Called during probe to display the part number of the 2984460eeaa0SBen Hutchings * installed NIC. VPD is potentially very large but this should 2985460eeaa0SBen Hutchings * always appear within the first 512 bytes. 2986460eeaa0SBen Hutchings */ 2987460eeaa0SBen Hutchings #define SFC_VPD_LEN 512 2988ef215e64SBen Hutchings static void efx_probe_vpd_strings(struct efx_nic *efx) 2989460eeaa0SBen Hutchings { 2990460eeaa0SBen Hutchings struct pci_dev *dev = efx->pci_dev; 2991460eeaa0SBen Hutchings char vpd_data[SFC_VPD_LEN]; 2992460eeaa0SBen Hutchings ssize_t vpd_size; 2993ef215e64SBen Hutchings int ro_start, ro_size, i, j; 2994460eeaa0SBen Hutchings 2995460eeaa0SBen Hutchings /* Get the vpd data from the device */ 2996460eeaa0SBen Hutchings vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 2997460eeaa0SBen Hutchings if (vpd_size <= 0) { 2998460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 2999460eeaa0SBen Hutchings return; 3000460eeaa0SBen Hutchings } 3001460eeaa0SBen Hutchings 3002460eeaa0SBen Hutchings /* Get the Read only section */ 3003ef215e64SBen Hutchings ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3004ef215e64SBen Hutchings if (ro_start < 0) { 3005460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3006460eeaa0SBen Hutchings return; 3007460eeaa0SBen Hutchings } 3008460eeaa0SBen Hutchings 3009ef215e64SBen Hutchings ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3010ef215e64SBen Hutchings j = ro_size; 3011ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3012460eeaa0SBen Hutchings if (i + j > vpd_size) 3013460eeaa0SBen Hutchings j = vpd_size - i; 3014460eeaa0SBen Hutchings 3015460eeaa0SBen Hutchings /* Get the Part number */ 3016460eeaa0SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3017460eeaa0SBen Hutchings if (i < 0) { 3018460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3019460eeaa0SBen Hutchings return; 3020460eeaa0SBen Hutchings } 3021460eeaa0SBen Hutchings 3022460eeaa0SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 3023460eeaa0SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 3024460eeaa0SBen Hutchings if (i + j > vpd_size) { 3025460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3026460eeaa0SBen Hutchings return; 3027460eeaa0SBen Hutchings } 3028460eeaa0SBen Hutchings 3029460eeaa0SBen Hutchings netif_info(efx, drv, efx->net_dev, 3030460eeaa0SBen Hutchings "Part Number : %.*s\n", j, &vpd_data[i]); 3031ef215e64SBen Hutchings 3032ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3033ef215e64SBen Hutchings j = ro_size; 3034ef215e64SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3035ef215e64SBen Hutchings if (i < 0) { 3036ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3037ef215e64SBen Hutchings return; 3038ef215e64SBen Hutchings } 3039ef215e64SBen Hutchings 3040ef215e64SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 3041ef215e64SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 3042ef215e64SBen Hutchings if (i + j > vpd_size) { 3043ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3044ef215e64SBen Hutchings return; 3045ef215e64SBen Hutchings } 3046ef215e64SBen Hutchings 3047ef215e64SBen Hutchings efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3048ef215e64SBen Hutchings if (!efx->vpd_sn) 3049ef215e64SBen Hutchings return; 3050ef215e64SBen Hutchings 3051ef215e64SBen Hutchings snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3052460eeaa0SBen Hutchings } 3053460eeaa0SBen Hutchings 3054460eeaa0SBen Hutchings 3055874aeea5SJeff Kirsher /* Main body of NIC initialisation 3056874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, theoretically). 3057874aeea5SJeff Kirsher */ 3058874aeea5SJeff Kirsher static int efx_pci_probe_main(struct efx_nic *efx) 3059874aeea5SJeff Kirsher { 3060874aeea5SJeff Kirsher int rc; 3061874aeea5SJeff Kirsher 3062874aeea5SJeff Kirsher /* Do start-of-day initialisation */ 3063874aeea5SJeff Kirsher rc = efx_probe_all(efx); 3064874aeea5SJeff Kirsher if (rc) 3065874aeea5SJeff Kirsher goto fail1; 3066874aeea5SJeff Kirsher 3067874aeea5SJeff Kirsher efx_init_napi(efx); 3068874aeea5SJeff Kirsher 3069874aeea5SJeff Kirsher rc = efx->type->init(efx); 3070874aeea5SJeff Kirsher if (rc) { 3071874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 3072874aeea5SJeff Kirsher "failed to initialise NIC\n"); 3073874aeea5SJeff Kirsher goto fail3; 3074874aeea5SJeff Kirsher } 3075874aeea5SJeff Kirsher 3076874aeea5SJeff Kirsher rc = efx_init_port(efx); 3077874aeea5SJeff Kirsher if (rc) { 3078874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 3079874aeea5SJeff Kirsher "failed to initialise port\n"); 3080874aeea5SJeff Kirsher goto fail4; 3081874aeea5SJeff Kirsher } 3082874aeea5SJeff Kirsher 3083874aeea5SJeff Kirsher rc = efx_nic_init_interrupt(efx); 3084874aeea5SJeff Kirsher if (rc) 3085874aeea5SJeff Kirsher goto fail5; 3086261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3087261e4d96SJon Cooper if (rc) 3088261e4d96SJon Cooper goto fail6; 3089874aeea5SJeff Kirsher 3090874aeea5SJeff Kirsher return 0; 3091874aeea5SJeff Kirsher 3092261e4d96SJon Cooper fail6: 3093261e4d96SJon Cooper efx_nic_fini_interrupt(efx); 3094874aeea5SJeff Kirsher fail5: 3095874aeea5SJeff Kirsher efx_fini_port(efx); 3096874aeea5SJeff Kirsher fail4: 3097874aeea5SJeff Kirsher efx->type->fini(efx); 3098874aeea5SJeff Kirsher fail3: 3099874aeea5SJeff Kirsher efx_fini_napi(efx); 3100874aeea5SJeff Kirsher efx_remove_all(efx); 3101874aeea5SJeff Kirsher fail1: 3102874aeea5SJeff Kirsher return rc; 3103874aeea5SJeff Kirsher } 3104874aeea5SJeff Kirsher 3105874aeea5SJeff Kirsher /* NIC initialisation 3106874aeea5SJeff Kirsher * 3107874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, 310873ba7b68SBen Hutchings * theoretically). It sets up PCI mappings, resets the NIC, 3109874aeea5SJeff Kirsher * sets up and registers the network devices with the kernel and hooks 3110874aeea5SJeff Kirsher * the interrupt service routine. It does not prepare the device for 3111874aeea5SJeff Kirsher * transmission; this is left to the first time one of the network 3112874aeea5SJeff Kirsher * interfaces is brought up (i.e. efx_net_open). 3113874aeea5SJeff Kirsher */ 311487d1fc11SBill Pemberton static int efx_pci_probe(struct pci_dev *pci_dev, 3115874aeea5SJeff Kirsher const struct pci_device_id *entry) 3116874aeea5SJeff Kirsher { 3117874aeea5SJeff Kirsher struct net_device *net_dev; 3118874aeea5SJeff Kirsher struct efx_nic *efx; 3119fadac6aaSBen Hutchings int rc; 3120874aeea5SJeff Kirsher 3121874aeea5SJeff Kirsher /* Allocate and initialise a struct net_device and struct efx_nic */ 3122874aeea5SJeff Kirsher net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3123874aeea5SJeff Kirsher EFX_MAX_RX_QUEUES); 3124874aeea5SJeff Kirsher if (!net_dev) 3125874aeea5SJeff Kirsher return -ENOMEM; 3126adeb15aaSBen Hutchings efx = netdev_priv(net_dev); 3127adeb15aaSBen Hutchings efx->type = (const struct efx_nic_type *) entry->driver_data; 3128adeb15aaSBen Hutchings net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3129874aeea5SJeff Kirsher NETIF_F_HIGHDMA | NETIF_F_TSO | 3130874aeea5SJeff Kirsher NETIF_F_RXCSUM); 3131c8cd0989STom Herbert if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 3132874aeea5SJeff Kirsher net_dev->features |= NETIF_F_TSO6; 3133874aeea5SJeff Kirsher /* Mask for features that also apply to VLAN devices */ 3134a188222bSTom Herbert net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3135874aeea5SJeff Kirsher NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 3136874aeea5SJeff Kirsher NETIF_F_RXCSUM); 3137874aeea5SJeff Kirsher /* All offloads can be toggled */ 3138874aeea5SJeff Kirsher net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA; 3139874aeea5SJeff Kirsher pci_set_drvdata(pci_dev, efx); 3140874aeea5SJeff Kirsher SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3141adeb15aaSBen Hutchings rc = efx_init_struct(efx, pci_dev, net_dev); 3142874aeea5SJeff Kirsher if (rc) 3143874aeea5SJeff Kirsher goto fail1; 3144874aeea5SJeff Kirsher 3145874aeea5SJeff Kirsher netif_info(efx, probe, efx->net_dev, 3146874aeea5SJeff Kirsher "Solarflare NIC detected\n"); 3147874aeea5SJeff Kirsher 31486f7f8aa6SShradha Shah if (!efx->type->is_vf) 3149ef215e64SBen Hutchings efx_probe_vpd_strings(efx); 3150460eeaa0SBen Hutchings 3151874aeea5SJeff Kirsher /* Set up basic I/O (BAR mappings etc) */ 3152874aeea5SJeff Kirsher rc = efx_init_io(efx); 3153874aeea5SJeff Kirsher if (rc) 3154874aeea5SJeff Kirsher goto fail2; 3155874aeea5SJeff Kirsher 3156874aeea5SJeff Kirsher rc = efx_pci_probe_main(efx); 3157fadac6aaSBen Hutchings if (rc) 3158874aeea5SJeff Kirsher goto fail3; 3159874aeea5SJeff Kirsher 3160874aeea5SJeff Kirsher rc = efx_register_netdev(efx); 3161874aeea5SJeff Kirsher if (rc) 3162fadac6aaSBen Hutchings goto fail4; 3163874aeea5SJeff Kirsher 31647fa8d547SShradha Shah if (efx->type->sriov_init) { 3165d98a4ffeSShradha Shah rc = efx->type->sriov_init(efx); 3166cd2d5b52SBen Hutchings if (rc) 3167cd2d5b52SBen Hutchings netif_err(efx, probe, efx->net_dev, 3168cd2d5b52SBen Hutchings "SR-IOV can't be enabled rc %d\n", rc); 31697fa8d547SShradha Shah } 3170cd2d5b52SBen Hutchings 3171874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3172874aeea5SJeff Kirsher 31737c43161cSBen Hutchings /* Try to create MTDs, but allow this to fail */ 3174874aeea5SJeff Kirsher rtnl_lock(); 31757c43161cSBen Hutchings rc = efx_mtd_probe(efx); 3176874aeea5SJeff Kirsher rtnl_unlock(); 317709a04204SBert Kenward if (rc && rc != -EPERM) 31787c43161cSBen Hutchings netif_warn(efx, probe, efx->net_dev, 31797c43161cSBen Hutchings "failed to create MTDs (%d)\n", rc); 31807c43161cSBen Hutchings 3181626950dbSAlexandre Rames rc = pci_enable_pcie_error_reporting(pci_dev); 3182626950dbSAlexandre Rames if (rc && rc != -EINVAL) 318309a04204SBert Kenward netif_notice(efx, probe, efx->net_dev, 318409a04204SBert Kenward "PCIE error reporting unavailable (%d).\n", 318509a04204SBert Kenward rc); 3186626950dbSAlexandre Rames 3187874aeea5SJeff Kirsher return 0; 3188874aeea5SJeff Kirsher 3189874aeea5SJeff Kirsher fail4: 3190fadac6aaSBen Hutchings efx_pci_remove_main(efx); 3191874aeea5SJeff Kirsher fail3: 3192874aeea5SJeff Kirsher efx_fini_io(efx); 3193874aeea5SJeff Kirsher fail2: 3194874aeea5SJeff Kirsher efx_fini_struct(efx); 3195874aeea5SJeff Kirsher fail1: 3196874aeea5SJeff Kirsher WARN_ON(rc > 0); 3197874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3198874aeea5SJeff Kirsher free_netdev(net_dev); 3199874aeea5SJeff Kirsher return rc; 3200874aeea5SJeff Kirsher } 3201874aeea5SJeff Kirsher 3202834e23ddSShradha Shah /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3203834e23ddSShradha Shah * enabled on success 3204834e23ddSShradha Shah */ 3205834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV 3206834e23ddSShradha Shah static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3207834e23ddSShradha Shah { 3208834e23ddSShradha Shah int rc; 3209834e23ddSShradha Shah struct efx_nic *efx = pci_get_drvdata(dev); 3210834e23ddSShradha Shah 3211834e23ddSShradha Shah if (efx->type->sriov_configure) { 3212834e23ddSShradha Shah rc = efx->type->sriov_configure(efx, num_vfs); 3213834e23ddSShradha Shah if (rc) 3214834e23ddSShradha Shah return rc; 3215834e23ddSShradha Shah else 3216834e23ddSShradha Shah return num_vfs; 3217834e23ddSShradha Shah } else 3218834e23ddSShradha Shah return -EOPNOTSUPP; 3219834e23ddSShradha Shah } 3220834e23ddSShradha Shah #endif 3221834e23ddSShradha Shah 3222874aeea5SJeff Kirsher static int efx_pm_freeze(struct device *dev) 3223874aeea5SJeff Kirsher { 3224874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3225874aeea5SJeff Kirsher 322661da026dSBen Hutchings rtnl_lock(); 322761da026dSBen Hutchings 32286032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3229f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 3230874aeea5SJeff Kirsher 3231c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 3232874aeea5SJeff Kirsher 3233874aeea5SJeff Kirsher efx_stop_all(efx); 3234d8291187SBen Hutchings efx_disable_interrupts(efx); 32356032fb56SBen Hutchings } 3236874aeea5SJeff Kirsher 323761da026dSBen Hutchings rtnl_unlock(); 323861da026dSBen Hutchings 3239874aeea5SJeff Kirsher return 0; 3240874aeea5SJeff Kirsher } 3241874aeea5SJeff Kirsher 3242874aeea5SJeff Kirsher static int efx_pm_thaw(struct device *dev) 3243874aeea5SJeff Kirsher { 3244261e4d96SJon Cooper int rc; 3245874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3246874aeea5SJeff Kirsher 324761da026dSBen Hutchings rtnl_lock(); 324861da026dSBen Hutchings 32496032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3250261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3251261e4d96SJon Cooper if (rc) 3252261e4d96SJon Cooper goto fail; 3253874aeea5SJeff Kirsher 3254874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 3255874aeea5SJeff Kirsher efx->phy_op->reconfigure(efx); 3256874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 3257874aeea5SJeff Kirsher 3258874aeea5SJeff Kirsher efx_start_all(efx); 3259874aeea5SJeff Kirsher 3260874aeea5SJeff Kirsher netif_device_attach(efx->net_dev); 3261874aeea5SJeff Kirsher 3262f16aeea0SBen Hutchings efx->state = STATE_READY; 3263874aeea5SJeff Kirsher 3264874aeea5SJeff Kirsher efx->type->resume_wol(efx); 32656032fb56SBen Hutchings } 3266874aeea5SJeff Kirsher 326761da026dSBen Hutchings rtnl_unlock(); 326861da026dSBen Hutchings 3269874aeea5SJeff Kirsher /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3270874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 3271874aeea5SJeff Kirsher 3272874aeea5SJeff Kirsher return 0; 3273261e4d96SJon Cooper 3274261e4d96SJon Cooper fail: 3275261e4d96SJon Cooper rtnl_unlock(); 3276261e4d96SJon Cooper 3277261e4d96SJon Cooper return rc; 3278874aeea5SJeff Kirsher } 3279874aeea5SJeff Kirsher 3280874aeea5SJeff Kirsher static int efx_pm_poweroff(struct device *dev) 3281874aeea5SJeff Kirsher { 3282874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3283874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3284874aeea5SJeff Kirsher 3285874aeea5SJeff Kirsher efx->type->fini(efx); 3286874aeea5SJeff Kirsher 3287874aeea5SJeff Kirsher efx->reset_pending = 0; 3288874aeea5SJeff Kirsher 3289874aeea5SJeff Kirsher pci_save_state(pci_dev); 3290874aeea5SJeff Kirsher return pci_set_power_state(pci_dev, PCI_D3hot); 3291874aeea5SJeff Kirsher } 3292874aeea5SJeff Kirsher 3293874aeea5SJeff Kirsher /* Used for both resume and restore */ 3294874aeea5SJeff Kirsher static int efx_pm_resume(struct device *dev) 3295874aeea5SJeff Kirsher { 3296874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3297874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3298874aeea5SJeff Kirsher int rc; 3299874aeea5SJeff Kirsher 3300874aeea5SJeff Kirsher rc = pci_set_power_state(pci_dev, PCI_D0); 3301874aeea5SJeff Kirsher if (rc) 3302874aeea5SJeff Kirsher return rc; 3303874aeea5SJeff Kirsher pci_restore_state(pci_dev); 3304874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 3305874aeea5SJeff Kirsher if (rc) 3306874aeea5SJeff Kirsher return rc; 3307874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 3308874aeea5SJeff Kirsher rc = efx->type->reset(efx, RESET_TYPE_ALL); 3309874aeea5SJeff Kirsher if (rc) 3310874aeea5SJeff Kirsher return rc; 3311874aeea5SJeff Kirsher rc = efx->type->init(efx); 3312874aeea5SJeff Kirsher if (rc) 3313874aeea5SJeff Kirsher return rc; 3314261e4d96SJon Cooper rc = efx_pm_thaw(dev); 3315261e4d96SJon Cooper return rc; 3316874aeea5SJeff Kirsher } 3317874aeea5SJeff Kirsher 3318874aeea5SJeff Kirsher static int efx_pm_suspend(struct device *dev) 3319874aeea5SJeff Kirsher { 3320874aeea5SJeff Kirsher int rc; 3321874aeea5SJeff Kirsher 3322874aeea5SJeff Kirsher efx_pm_freeze(dev); 3323874aeea5SJeff Kirsher rc = efx_pm_poweroff(dev); 3324874aeea5SJeff Kirsher if (rc) 3325874aeea5SJeff Kirsher efx_pm_resume(dev); 3326874aeea5SJeff Kirsher return rc; 3327874aeea5SJeff Kirsher } 3328874aeea5SJeff Kirsher 332918e83e4cSBen Hutchings static const struct dev_pm_ops efx_pm_ops = { 3330874aeea5SJeff Kirsher .suspend = efx_pm_suspend, 3331874aeea5SJeff Kirsher .resume = efx_pm_resume, 3332874aeea5SJeff Kirsher .freeze = efx_pm_freeze, 3333874aeea5SJeff Kirsher .thaw = efx_pm_thaw, 3334874aeea5SJeff Kirsher .poweroff = efx_pm_poweroff, 3335874aeea5SJeff Kirsher .restore = efx_pm_resume, 3336874aeea5SJeff Kirsher }; 3337874aeea5SJeff Kirsher 3338626950dbSAlexandre Rames /* A PCI error affecting this device was detected. 3339626950dbSAlexandre Rames * At this point MMIO and DMA may be disabled. 3340626950dbSAlexandre Rames * Stop the software path and request a slot reset. 3341626950dbSAlexandre Rames */ 3342debd0034Sstephen hemminger static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3343626950dbSAlexandre Rames enum pci_channel_state state) 3344626950dbSAlexandre Rames { 3345626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3346626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3347626950dbSAlexandre Rames 3348626950dbSAlexandre Rames if (state == pci_channel_io_perm_failure) 3349626950dbSAlexandre Rames return PCI_ERS_RESULT_DISCONNECT; 3350626950dbSAlexandre Rames 3351626950dbSAlexandre Rames rtnl_lock(); 3352626950dbSAlexandre Rames 3353626950dbSAlexandre Rames if (efx->state != STATE_DISABLED) { 3354626950dbSAlexandre Rames efx->state = STATE_RECOVERY; 3355626950dbSAlexandre Rames efx->reset_pending = 0; 3356626950dbSAlexandre Rames 3357626950dbSAlexandre Rames efx_device_detach_sync(efx); 3358626950dbSAlexandre Rames 3359626950dbSAlexandre Rames efx_stop_all(efx); 3360d8291187SBen Hutchings efx_disable_interrupts(efx); 3361626950dbSAlexandre Rames 3362626950dbSAlexandre Rames status = PCI_ERS_RESULT_NEED_RESET; 3363626950dbSAlexandre Rames } else { 3364626950dbSAlexandre Rames /* If the interface is disabled we don't want to do anything 3365626950dbSAlexandre Rames * with it. 3366626950dbSAlexandre Rames */ 3367626950dbSAlexandre Rames status = PCI_ERS_RESULT_RECOVERED; 3368626950dbSAlexandre Rames } 3369626950dbSAlexandre Rames 3370626950dbSAlexandre Rames rtnl_unlock(); 3371626950dbSAlexandre Rames 3372626950dbSAlexandre Rames pci_disable_device(pdev); 3373626950dbSAlexandre Rames 3374626950dbSAlexandre Rames return status; 3375626950dbSAlexandre Rames } 3376626950dbSAlexandre Rames 3377dbedd44eSJoe Perches /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3378debd0034Sstephen hemminger static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3379626950dbSAlexandre Rames { 3380626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3381626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3382626950dbSAlexandre Rames int rc; 3383626950dbSAlexandre Rames 3384626950dbSAlexandre Rames if (pci_enable_device(pdev)) { 3385626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3386626950dbSAlexandre Rames "Cannot re-enable PCI device after reset.\n"); 3387626950dbSAlexandre Rames status = PCI_ERS_RESULT_DISCONNECT; 3388626950dbSAlexandre Rames } 3389626950dbSAlexandre Rames 3390626950dbSAlexandre Rames rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3391626950dbSAlexandre Rames if (rc) { 3392626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3393626950dbSAlexandre Rames "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3394626950dbSAlexandre Rames /* Non-fatal error. Continue. */ 3395626950dbSAlexandre Rames } 3396626950dbSAlexandre Rames 3397626950dbSAlexandre Rames return status; 3398626950dbSAlexandre Rames } 3399626950dbSAlexandre Rames 3400626950dbSAlexandre Rames /* Perform the actual reset and resume I/O operations. */ 3401626950dbSAlexandre Rames static void efx_io_resume(struct pci_dev *pdev) 3402626950dbSAlexandre Rames { 3403626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3404626950dbSAlexandre Rames int rc; 3405626950dbSAlexandre Rames 3406626950dbSAlexandre Rames rtnl_lock(); 3407626950dbSAlexandre Rames 3408626950dbSAlexandre Rames if (efx->state == STATE_DISABLED) 3409626950dbSAlexandre Rames goto out; 3410626950dbSAlexandre Rames 3411626950dbSAlexandre Rames rc = efx_reset(efx, RESET_TYPE_ALL); 3412626950dbSAlexandre Rames if (rc) { 3413626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3414626950dbSAlexandre Rames "efx_reset failed after PCI error (%d)\n", rc); 3415626950dbSAlexandre Rames } else { 3416626950dbSAlexandre Rames efx->state = STATE_READY; 3417626950dbSAlexandre Rames netif_dbg(efx, hw, efx->net_dev, 3418626950dbSAlexandre Rames "Done resetting and resuming IO after PCI error.\n"); 3419626950dbSAlexandre Rames } 3420626950dbSAlexandre Rames 3421626950dbSAlexandre Rames out: 3422626950dbSAlexandre Rames rtnl_unlock(); 3423626950dbSAlexandre Rames } 3424626950dbSAlexandre Rames 3425626950dbSAlexandre Rames /* For simplicity and reliability, we always require a slot reset and try to 3426626950dbSAlexandre Rames * reset the hardware when a pci error affecting the device is detected. 3427626950dbSAlexandre Rames * We leave both the link_reset and mmio_enabled callback unimplemented: 3428626950dbSAlexandre Rames * with our request for slot reset the mmio_enabled callback will never be 3429626950dbSAlexandre Rames * called, and the link_reset callback is not used by AER or EEH mechanisms. 3430626950dbSAlexandre Rames */ 3431c300366bSJulia Lawall static const struct pci_error_handlers efx_err_handlers = { 3432626950dbSAlexandre Rames .error_detected = efx_io_error_detected, 3433626950dbSAlexandre Rames .slot_reset = efx_io_slot_reset, 3434626950dbSAlexandre Rames .resume = efx_io_resume, 3435626950dbSAlexandre Rames }; 3436626950dbSAlexandre Rames 3437874aeea5SJeff Kirsher static struct pci_driver efx_pci_driver = { 3438874aeea5SJeff Kirsher .name = KBUILD_MODNAME, 3439874aeea5SJeff Kirsher .id_table = efx_pci_table, 3440874aeea5SJeff Kirsher .probe = efx_pci_probe, 3441874aeea5SJeff Kirsher .remove = efx_pci_remove, 3442874aeea5SJeff Kirsher .driver.pm = &efx_pm_ops, 3443626950dbSAlexandre Rames .err_handler = &efx_err_handlers, 3444834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV 3445834e23ddSShradha Shah .sriov_configure = efx_pci_sriov_configure, 3446834e23ddSShradha Shah #endif 3447874aeea5SJeff Kirsher }; 3448874aeea5SJeff Kirsher 3449874aeea5SJeff Kirsher /************************************************************************** 3450874aeea5SJeff Kirsher * 3451874aeea5SJeff Kirsher * Kernel module interface 3452874aeea5SJeff Kirsher * 3453874aeea5SJeff Kirsher *************************************************************************/ 3454874aeea5SJeff Kirsher 3455874aeea5SJeff Kirsher module_param(interrupt_mode, uint, 0444); 3456874aeea5SJeff Kirsher MODULE_PARM_DESC(interrupt_mode, 3457874aeea5SJeff Kirsher "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3458874aeea5SJeff Kirsher 3459874aeea5SJeff Kirsher static int __init efx_init_module(void) 3460874aeea5SJeff Kirsher { 3461874aeea5SJeff Kirsher int rc; 3462874aeea5SJeff Kirsher 3463874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3464874aeea5SJeff Kirsher 3465874aeea5SJeff Kirsher rc = register_netdevice_notifier(&efx_netdev_notifier); 3466874aeea5SJeff Kirsher if (rc) 3467874aeea5SJeff Kirsher goto err_notifier; 3468874aeea5SJeff Kirsher 34697fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 3470cd2d5b52SBen Hutchings rc = efx_init_sriov(); 3471cd2d5b52SBen Hutchings if (rc) 3472cd2d5b52SBen Hutchings goto err_sriov; 34737fa8d547SShradha Shah #endif 3474cd2d5b52SBen Hutchings 3475874aeea5SJeff Kirsher reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3476874aeea5SJeff Kirsher if (!reset_workqueue) { 3477874aeea5SJeff Kirsher rc = -ENOMEM; 3478874aeea5SJeff Kirsher goto err_reset; 3479874aeea5SJeff Kirsher } 3480874aeea5SJeff Kirsher 3481874aeea5SJeff Kirsher rc = pci_register_driver(&efx_pci_driver); 3482874aeea5SJeff Kirsher if (rc < 0) 3483874aeea5SJeff Kirsher goto err_pci; 3484874aeea5SJeff Kirsher 3485874aeea5SJeff Kirsher return 0; 3486874aeea5SJeff Kirsher 3487874aeea5SJeff Kirsher err_pci: 3488874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 3489874aeea5SJeff Kirsher err_reset: 34907fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 3491cd2d5b52SBen Hutchings efx_fini_sriov(); 3492cd2d5b52SBen Hutchings err_sriov: 34937fa8d547SShradha Shah #endif 3494874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3495874aeea5SJeff Kirsher err_notifier: 3496874aeea5SJeff Kirsher return rc; 3497874aeea5SJeff Kirsher } 3498874aeea5SJeff Kirsher 3499874aeea5SJeff Kirsher static void __exit efx_exit_module(void) 3500874aeea5SJeff Kirsher { 3501874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver unloading\n"); 3502874aeea5SJeff Kirsher 3503874aeea5SJeff Kirsher pci_unregister_driver(&efx_pci_driver); 3504874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 35057fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 3506cd2d5b52SBen Hutchings efx_fini_sriov(); 35077fa8d547SShradha Shah #endif 3508874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3509874aeea5SJeff Kirsher 3510874aeea5SJeff Kirsher } 3511874aeea5SJeff Kirsher 3512874aeea5SJeff Kirsher module_init(efx_init_module); 3513874aeea5SJeff Kirsher module_exit(efx_exit_module); 3514874aeea5SJeff Kirsher 3515874aeea5SJeff Kirsher MODULE_AUTHOR("Solarflare Communications and " 3516874aeea5SJeff Kirsher "Michael Brown <mbrown@fensystems.co.uk>"); 35176a350fdbSBen Hutchings MODULE_DESCRIPTION("Solarflare network driver"); 3518874aeea5SJeff Kirsher MODULE_LICENSE("GPL"); 3519874aeea5SJeff Kirsher MODULE_DEVICE_TABLE(pci, efx_pci_table); 3520