1874aeea5SJeff Kirsher /**************************************************************************** 2f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 3874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 4f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 5874aeea5SJeff Kirsher * 6874aeea5SJeff Kirsher * This program is free software; you can redistribute it and/or modify it 7874aeea5SJeff Kirsher * under the terms of the GNU General Public License version 2 as published 8874aeea5SJeff Kirsher * by the Free Software Foundation, incorporated herein by reference. 9874aeea5SJeff Kirsher */ 10874aeea5SJeff Kirsher 11874aeea5SJeff Kirsher #include <linux/module.h> 12874aeea5SJeff Kirsher #include <linux/pci.h> 13874aeea5SJeff Kirsher #include <linux/netdevice.h> 14874aeea5SJeff Kirsher #include <linux/etherdevice.h> 15874aeea5SJeff Kirsher #include <linux/delay.h> 16874aeea5SJeff Kirsher #include <linux/notifier.h> 17874aeea5SJeff Kirsher #include <linux/ip.h> 18874aeea5SJeff Kirsher #include <linux/tcp.h> 19874aeea5SJeff Kirsher #include <linux/in.h> 20874aeea5SJeff Kirsher #include <linux/ethtool.h> 21874aeea5SJeff Kirsher #include <linux/topology.h> 22874aeea5SJeff Kirsher #include <linux/gfp.h> 23626950dbSAlexandre Rames #include <linux/aer.h> 24b28405b0SAlexandre Rames #include <linux/interrupt.h> 25874aeea5SJeff Kirsher #include "net_driver.h" 26874aeea5SJeff Kirsher #include "efx.h" 27874aeea5SJeff Kirsher #include "nic.h" 28dd40781eSBen Hutchings #include "selftest.h" 29874aeea5SJeff Kirsher 30874aeea5SJeff Kirsher #include "mcdi.h" 31874aeea5SJeff Kirsher #include "workarounds.h" 32874aeea5SJeff Kirsher 33874aeea5SJeff Kirsher /************************************************************************** 34874aeea5SJeff Kirsher * 35874aeea5SJeff Kirsher * Type name strings 36874aeea5SJeff Kirsher * 37874aeea5SJeff Kirsher ************************************************************************** 38874aeea5SJeff Kirsher */ 39874aeea5SJeff Kirsher 40874aeea5SJeff Kirsher /* Loopback mode names (see LOOPBACK_MODE()) */ 41874aeea5SJeff Kirsher const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 4218e83e4cSBen Hutchings const char *const efx_loopback_mode_names[] = { 43874aeea5SJeff Kirsher [LOOPBACK_NONE] = "NONE", 44874aeea5SJeff Kirsher [LOOPBACK_DATA] = "DATAPATH", 45874aeea5SJeff Kirsher [LOOPBACK_GMAC] = "GMAC", 46874aeea5SJeff Kirsher [LOOPBACK_XGMII] = "XGMII", 47874aeea5SJeff Kirsher [LOOPBACK_XGXS] = "XGXS", 48874aeea5SJeff Kirsher [LOOPBACK_XAUI] = "XAUI", 49874aeea5SJeff Kirsher [LOOPBACK_GMII] = "GMII", 50874aeea5SJeff Kirsher [LOOPBACK_SGMII] = "SGMII", 51874aeea5SJeff Kirsher [LOOPBACK_XGBR] = "XGBR", 52874aeea5SJeff Kirsher [LOOPBACK_XFI] = "XFI", 53874aeea5SJeff Kirsher [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 54874aeea5SJeff Kirsher [LOOPBACK_GMII_FAR] = "GMII_FAR", 55874aeea5SJeff Kirsher [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 56874aeea5SJeff Kirsher [LOOPBACK_XFI_FAR] = "XFI_FAR", 57874aeea5SJeff Kirsher [LOOPBACK_GPHY] = "GPHY", 58874aeea5SJeff Kirsher [LOOPBACK_PHYXS] = "PHYXS", 59874aeea5SJeff Kirsher [LOOPBACK_PCS] = "PCS", 60874aeea5SJeff Kirsher [LOOPBACK_PMAPMD] = "PMA/PMD", 61874aeea5SJeff Kirsher [LOOPBACK_XPORT] = "XPORT", 62874aeea5SJeff Kirsher [LOOPBACK_XGMII_WS] = "XGMII_WS", 63874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS] = "XAUI_WS", 64874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 65874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 66874aeea5SJeff Kirsher [LOOPBACK_GMII_WS] = "GMII_WS", 67874aeea5SJeff Kirsher [LOOPBACK_XFI_WS] = "XFI_WS", 68874aeea5SJeff Kirsher [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 69874aeea5SJeff Kirsher [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 70874aeea5SJeff Kirsher }; 71874aeea5SJeff Kirsher 72874aeea5SJeff Kirsher const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 7318e83e4cSBen Hutchings const char *const efx_reset_type_names[] = { 74874aeea5SJeff Kirsher [RESET_TYPE_INVISIBLE] = "INVISIBLE", 75874aeea5SJeff Kirsher [RESET_TYPE_ALL] = "ALL", 76626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 77874aeea5SJeff Kirsher [RESET_TYPE_WORLD] = "WORLD", 78626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 79874aeea5SJeff Kirsher [RESET_TYPE_DISABLE] = "DISABLE", 80874aeea5SJeff Kirsher [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 81874aeea5SJeff Kirsher [RESET_TYPE_INT_ERROR] = "INT_ERROR", 82874aeea5SJeff Kirsher [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY", 833de82b91SAlexandre Rames [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 84874aeea5SJeff Kirsher [RESET_TYPE_TX_SKIP] = "TX_SKIP", 85874aeea5SJeff Kirsher [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 8674cd60a4SJon Cooper [RESET_TYPE_MC_BIST] = "MC_BIST", 87874aeea5SJeff Kirsher }; 88874aeea5SJeff Kirsher 89874aeea5SJeff Kirsher /* Reset workqueue. If any NIC has a hardware failure then a reset will be 90874aeea5SJeff Kirsher * queued onto this work queue. This is not a per-nic work queue, because 91874aeea5SJeff Kirsher * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 92874aeea5SJeff Kirsher */ 93874aeea5SJeff Kirsher static struct workqueue_struct *reset_workqueue; 94874aeea5SJeff Kirsher 9574cd60a4SJon Cooper /* How often and how many times to poll for a reset while waiting for a 9674cd60a4SJon Cooper * BIST that another function started to complete. 9774cd60a4SJon Cooper */ 9874cd60a4SJon Cooper #define BIST_WAIT_DELAY_MS 100 9974cd60a4SJon Cooper #define BIST_WAIT_DELAY_COUNT 100 10074cd60a4SJon Cooper 101874aeea5SJeff Kirsher /************************************************************************** 102874aeea5SJeff Kirsher * 103874aeea5SJeff Kirsher * Configurable values 104874aeea5SJeff Kirsher * 105874aeea5SJeff Kirsher *************************************************************************/ 106874aeea5SJeff Kirsher 107874aeea5SJeff Kirsher /* 108874aeea5SJeff Kirsher * Use separate channels for TX and RX events 109874aeea5SJeff Kirsher * 110874aeea5SJeff Kirsher * Set this to 1 to use separate channels for TX and RX. It allows us 111874aeea5SJeff Kirsher * to control interrupt affinity separately for TX and RX. 112874aeea5SJeff Kirsher * 113874aeea5SJeff Kirsher * This is only used in MSI-X interrupt mode 114874aeea5SJeff Kirsher */ 115b9cc977dSBen Hutchings static bool separate_tx_channels; 116b9cc977dSBen Hutchings module_param(separate_tx_channels, bool, 0444); 117874aeea5SJeff Kirsher MODULE_PARM_DESC(separate_tx_channels, 118874aeea5SJeff Kirsher "Use separate channels for TX and RX"); 119874aeea5SJeff Kirsher 120874aeea5SJeff Kirsher /* This is the weight assigned to each of the (per-channel) virtual 121874aeea5SJeff Kirsher * NAPI devices. 122874aeea5SJeff Kirsher */ 123874aeea5SJeff Kirsher static int napi_weight = 64; 124874aeea5SJeff Kirsher 125874aeea5SJeff Kirsher /* This is the time (in jiffies) between invocations of the hardware 126626950dbSAlexandre Rames * monitor. 127626950dbSAlexandre Rames * On Falcon-based NICs, this will: 128874aeea5SJeff Kirsher * - Check the on-board hardware monitor; 129874aeea5SJeff Kirsher * - Poll the link state and reconfigure the hardware as necessary. 130626950dbSAlexandre Rames * On Siena-based NICs for power systems with EEH support, this will give EEH a 131626950dbSAlexandre Rames * chance to start. 132874aeea5SJeff Kirsher */ 133874aeea5SJeff Kirsher static unsigned int efx_monitor_interval = 1 * HZ; 134874aeea5SJeff Kirsher 135874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 136874aeea5SJeff Kirsher * module load with ethtool. 137874aeea5SJeff Kirsher * 138874aeea5SJeff Kirsher * The default for RX should strike a balance between increasing the 139874aeea5SJeff Kirsher * round-trip latency and reducing overhead. 140874aeea5SJeff Kirsher */ 141874aeea5SJeff Kirsher static unsigned int rx_irq_mod_usec = 60; 142874aeea5SJeff Kirsher 143874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 144874aeea5SJeff Kirsher * module load with ethtool. 145874aeea5SJeff Kirsher * 146874aeea5SJeff Kirsher * This default is chosen to ensure that a 10G link does not go idle 147874aeea5SJeff Kirsher * while a TX queue is stopped after it has become full. A queue is 148874aeea5SJeff Kirsher * restarted when it drops below half full. The time this takes (assuming 149874aeea5SJeff Kirsher * worst case 3 descriptors per packet and 1024 descriptors) is 150874aeea5SJeff Kirsher * 512 / 3 * 1.2 = 205 usec. 151874aeea5SJeff Kirsher */ 152874aeea5SJeff Kirsher static unsigned int tx_irq_mod_usec = 150; 153874aeea5SJeff Kirsher 154874aeea5SJeff Kirsher /* This is the first interrupt mode to try out of: 155874aeea5SJeff Kirsher * 0 => MSI-X 156874aeea5SJeff Kirsher * 1 => MSI 157874aeea5SJeff Kirsher * 2 => legacy 158874aeea5SJeff Kirsher */ 159874aeea5SJeff Kirsher static unsigned int interrupt_mode; 160874aeea5SJeff Kirsher 161874aeea5SJeff Kirsher /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 162874aeea5SJeff Kirsher * i.e. the number of CPUs among which we may distribute simultaneous 163874aeea5SJeff Kirsher * interrupt handling. 164874aeea5SJeff Kirsher * 165874aeea5SJeff Kirsher * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 166cdb08f8fSBen Hutchings * The default (0) means to assign an interrupt to each core. 167874aeea5SJeff Kirsher */ 168874aeea5SJeff Kirsher static unsigned int rss_cpus; 169874aeea5SJeff Kirsher module_param(rss_cpus, uint, 0444); 170874aeea5SJeff Kirsher MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 171874aeea5SJeff Kirsher 172b9cc977dSBen Hutchings static bool phy_flash_cfg; 173b9cc977dSBen Hutchings module_param(phy_flash_cfg, bool, 0644); 174874aeea5SJeff Kirsher MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 175874aeea5SJeff Kirsher 176e7bed9c8SBen Hutchings static unsigned irq_adapt_low_thresh = 8000; 177874aeea5SJeff Kirsher module_param(irq_adapt_low_thresh, uint, 0644); 178874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_low_thresh, 179874aeea5SJeff Kirsher "Threshold score for reducing IRQ moderation"); 180874aeea5SJeff Kirsher 181e7bed9c8SBen Hutchings static unsigned irq_adapt_high_thresh = 16000; 182874aeea5SJeff Kirsher module_param(irq_adapt_high_thresh, uint, 0644); 183874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_high_thresh, 184874aeea5SJeff Kirsher "Threshold score for increasing IRQ moderation"); 185874aeea5SJeff Kirsher 186874aeea5SJeff Kirsher static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 187874aeea5SJeff Kirsher NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 188874aeea5SJeff Kirsher NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 189874aeea5SJeff Kirsher NETIF_MSG_TX_ERR | NETIF_MSG_HW); 190874aeea5SJeff Kirsher module_param(debug, uint, 0); 191874aeea5SJeff Kirsher MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 192874aeea5SJeff Kirsher 193874aeea5SJeff Kirsher /************************************************************************** 194874aeea5SJeff Kirsher * 195874aeea5SJeff Kirsher * Utility functions and prototypes 196874aeea5SJeff Kirsher * 197874aeea5SJeff Kirsher *************************************************************************/ 198874aeea5SJeff Kirsher 199261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx); 200d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx); 2017f967c01SBen Hutchings static void efx_remove_channel(struct efx_channel *channel); 202874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx); 2037f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type; 204874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx); 2057f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel); 206874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx); 207874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel); 208874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx); 209874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx); 210874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx); 211874aeea5SJeff Kirsher 212874aeea5SJeff Kirsher #define EFX_ASSERT_RESET_SERIALISED(efx) \ 213874aeea5SJeff Kirsher do { \ 214f16aeea0SBen Hutchings if ((efx->state == STATE_READY) || \ 215626950dbSAlexandre Rames (efx->state == STATE_RECOVERY) || \ 216874aeea5SJeff Kirsher (efx->state == STATE_DISABLED)) \ 217874aeea5SJeff Kirsher ASSERT_RTNL(); \ 218874aeea5SJeff Kirsher } while (0) 219874aeea5SJeff Kirsher 2208b7325b4SBen Hutchings static int efx_check_disabled(struct efx_nic *efx) 2218b7325b4SBen Hutchings { 222626950dbSAlexandre Rames if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 2238b7325b4SBen Hutchings netif_err(efx, drv, efx->net_dev, 2248b7325b4SBen Hutchings "device is disabled due to earlier errors\n"); 2258b7325b4SBen Hutchings return -EIO; 2268b7325b4SBen Hutchings } 2278b7325b4SBen Hutchings return 0; 2288b7325b4SBen Hutchings } 2298b7325b4SBen Hutchings 230874aeea5SJeff Kirsher /************************************************************************** 231874aeea5SJeff Kirsher * 232874aeea5SJeff Kirsher * Event queue processing 233874aeea5SJeff Kirsher * 234874aeea5SJeff Kirsher *************************************************************************/ 235874aeea5SJeff Kirsher 236874aeea5SJeff Kirsher /* Process channel's event queue 237874aeea5SJeff Kirsher * 238874aeea5SJeff Kirsher * This function is responsible for processing the event queue of a 239874aeea5SJeff Kirsher * single channel. The caller must guarantee that this function will 240874aeea5SJeff Kirsher * never be concurrently called more than once on the same channel, 241874aeea5SJeff Kirsher * though different channels may be being processed concurrently. 242874aeea5SJeff Kirsher */ 243874aeea5SJeff Kirsher static int efx_process_channel(struct efx_channel *channel, int budget) 244874aeea5SJeff Kirsher { 245874aeea5SJeff Kirsher int spent; 246874aeea5SJeff Kirsher 2479f2cb71cSBen Hutchings if (unlikely(!channel->enabled)) 248874aeea5SJeff Kirsher return 0; 249874aeea5SJeff Kirsher 250874aeea5SJeff Kirsher spent = efx_nic_process_eventq(channel, budget); 251d9ab7007SBen Hutchings if (spent && efx_channel_has_rx_queue(channel)) { 252d9ab7007SBen Hutchings struct efx_rx_queue *rx_queue = 253d9ab7007SBen Hutchings efx_channel_get_rx_queue(channel); 254874aeea5SJeff Kirsher 255ff734ef4SBen Hutchings efx_rx_flush_packet(channel); 256cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, true); 257d9ab7007SBen Hutchings } 258874aeea5SJeff Kirsher 259874aeea5SJeff Kirsher return spent; 260874aeea5SJeff Kirsher } 261874aeea5SJeff Kirsher 262874aeea5SJeff Kirsher /* NAPI poll handler 263874aeea5SJeff Kirsher * 264874aeea5SJeff Kirsher * NAPI guarantees serialisation of polls of the same device, which 265874aeea5SJeff Kirsher * provides the guarantee required by efx_process_channel(). 266874aeea5SJeff Kirsher */ 267874aeea5SJeff Kirsher static int efx_poll(struct napi_struct *napi, int budget) 268874aeea5SJeff Kirsher { 269874aeea5SJeff Kirsher struct efx_channel *channel = 270874aeea5SJeff Kirsher container_of(napi, struct efx_channel, napi_str); 271874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 272874aeea5SJeff Kirsher int spent; 273874aeea5SJeff Kirsher 274874aeea5SJeff Kirsher netif_vdbg(efx, intr, efx->net_dev, 275874aeea5SJeff Kirsher "channel %d NAPI poll executing on CPU %d\n", 276874aeea5SJeff Kirsher channel->channel, raw_smp_processor_id()); 277874aeea5SJeff Kirsher 278874aeea5SJeff Kirsher spent = efx_process_channel(channel, budget); 279874aeea5SJeff Kirsher 280874aeea5SJeff Kirsher if (spent < budget) { 2819d9a6973SBen Hutchings if (efx_channel_has_rx_queue(channel) && 282874aeea5SJeff Kirsher efx->irq_rx_adaptive && 283874aeea5SJeff Kirsher unlikely(++channel->irq_count == 1000)) { 284874aeea5SJeff Kirsher if (unlikely(channel->irq_mod_score < 285874aeea5SJeff Kirsher irq_adapt_low_thresh)) { 286874aeea5SJeff Kirsher if (channel->irq_moderation > 1) { 287874aeea5SJeff Kirsher channel->irq_moderation -= 1; 288874aeea5SJeff Kirsher efx->type->push_irq_moderation(channel); 289874aeea5SJeff Kirsher } 290874aeea5SJeff Kirsher } else if (unlikely(channel->irq_mod_score > 291874aeea5SJeff Kirsher irq_adapt_high_thresh)) { 292874aeea5SJeff Kirsher if (channel->irq_moderation < 293874aeea5SJeff Kirsher efx->irq_rx_moderation) { 294874aeea5SJeff Kirsher channel->irq_moderation += 1; 295874aeea5SJeff Kirsher efx->type->push_irq_moderation(channel); 296874aeea5SJeff Kirsher } 297874aeea5SJeff Kirsher } 298874aeea5SJeff Kirsher channel->irq_count = 0; 299874aeea5SJeff Kirsher channel->irq_mod_score = 0; 300874aeea5SJeff Kirsher } 301874aeea5SJeff Kirsher 302874aeea5SJeff Kirsher efx_filter_rfs_expire(channel); 303874aeea5SJeff Kirsher 304874aeea5SJeff Kirsher /* There is no race here; although napi_disable() will 305874aeea5SJeff Kirsher * only wait for napi_complete(), this isn't a problem 306514bedbcSBen Hutchings * since efx_nic_eventq_read_ack() will have no effect if 307874aeea5SJeff Kirsher * interrupts have already been disabled. 308874aeea5SJeff Kirsher */ 309874aeea5SJeff Kirsher napi_complete(napi); 310514bedbcSBen Hutchings efx_nic_eventq_read_ack(channel); 311874aeea5SJeff Kirsher } 312874aeea5SJeff Kirsher 313874aeea5SJeff Kirsher return spent; 314874aeea5SJeff Kirsher } 315874aeea5SJeff Kirsher 316874aeea5SJeff Kirsher /* Create event queue 317874aeea5SJeff Kirsher * Event queue memory allocations are done only once. If the channel 318874aeea5SJeff Kirsher * is reset, the memory buffer will be reused; this guards against 319874aeea5SJeff Kirsher * errors during channel reset and also simplifies interrupt handling. 320874aeea5SJeff Kirsher */ 321874aeea5SJeff Kirsher static int efx_probe_eventq(struct efx_channel *channel) 322874aeea5SJeff Kirsher { 323874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 324874aeea5SJeff Kirsher unsigned long entries; 325874aeea5SJeff Kirsher 32686ee5302SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 327874aeea5SJeff Kirsher "chan %d create event queue\n", channel->channel); 328874aeea5SJeff Kirsher 329874aeea5SJeff Kirsher /* Build an event queue with room for one event per tx and rx buffer, 330874aeea5SJeff Kirsher * plus some extra for link state events and MCDI completions. */ 331874aeea5SJeff Kirsher entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 332874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 333874aeea5SJeff Kirsher channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 334874aeea5SJeff Kirsher 335874aeea5SJeff Kirsher return efx_nic_probe_eventq(channel); 336874aeea5SJeff Kirsher } 337874aeea5SJeff Kirsher 338874aeea5SJeff Kirsher /* Prepare channel's event queue */ 339261e4d96SJon Cooper static int efx_init_eventq(struct efx_channel *channel) 340874aeea5SJeff Kirsher { 34115acb1ceSBen Hutchings struct efx_nic *efx = channel->efx; 342261e4d96SJon Cooper int rc; 343261e4d96SJon Cooper 344261e4d96SJon Cooper EFX_WARN_ON_PARANOID(channel->eventq_init); 345261e4d96SJon Cooper 34615acb1ceSBen Hutchings netif_dbg(efx, drv, efx->net_dev, 347874aeea5SJeff Kirsher "chan %d init event queue\n", channel->channel); 348874aeea5SJeff Kirsher 349261e4d96SJon Cooper rc = efx_nic_init_eventq(channel); 350261e4d96SJon Cooper if (rc == 0) { 35115acb1ceSBen Hutchings efx->type->push_irq_moderation(channel); 352874aeea5SJeff Kirsher channel->eventq_read_ptr = 0; 353be3fc09cSBen Hutchings channel->eventq_init = true; 354874aeea5SJeff Kirsher } 355261e4d96SJon Cooper return rc; 356261e4d96SJon Cooper } 357874aeea5SJeff Kirsher 3589f2cb71cSBen Hutchings /* Enable event queue processing and NAPI */ 3599f2cb71cSBen Hutchings static void efx_start_eventq(struct efx_channel *channel) 3609f2cb71cSBen Hutchings { 3619f2cb71cSBen Hutchings netif_dbg(channel->efx, ifup, channel->efx->net_dev, 3629f2cb71cSBen Hutchings "chan %d start event queue\n", channel->channel); 3639f2cb71cSBen Hutchings 364514bedbcSBen Hutchings /* Make sure the NAPI handler sees the enabled flag set */ 3659f2cb71cSBen Hutchings channel->enabled = true; 3669f2cb71cSBen Hutchings smp_wmb(); 3679f2cb71cSBen Hutchings 3689f2cb71cSBen Hutchings napi_enable(&channel->napi_str); 3699f2cb71cSBen Hutchings efx_nic_eventq_read_ack(channel); 3709f2cb71cSBen Hutchings } 3719f2cb71cSBen Hutchings 3729f2cb71cSBen Hutchings /* Disable event queue processing and NAPI */ 3739f2cb71cSBen Hutchings static void efx_stop_eventq(struct efx_channel *channel) 3749f2cb71cSBen Hutchings { 3759f2cb71cSBen Hutchings if (!channel->enabled) 3769f2cb71cSBen Hutchings return; 3779f2cb71cSBen Hutchings 3789f2cb71cSBen Hutchings napi_disable(&channel->napi_str); 3799f2cb71cSBen Hutchings channel->enabled = false; 3809f2cb71cSBen Hutchings } 3819f2cb71cSBen Hutchings 382874aeea5SJeff Kirsher static void efx_fini_eventq(struct efx_channel *channel) 383874aeea5SJeff Kirsher { 384be3fc09cSBen Hutchings if (!channel->eventq_init) 385be3fc09cSBen Hutchings return; 386be3fc09cSBen Hutchings 387874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 388874aeea5SJeff Kirsher "chan %d fini event queue\n", channel->channel); 389874aeea5SJeff Kirsher 390874aeea5SJeff Kirsher efx_nic_fini_eventq(channel); 391be3fc09cSBen Hutchings channel->eventq_init = false; 392874aeea5SJeff Kirsher } 393874aeea5SJeff Kirsher 394874aeea5SJeff Kirsher static void efx_remove_eventq(struct efx_channel *channel) 395874aeea5SJeff Kirsher { 396874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 397874aeea5SJeff Kirsher "chan %d remove event queue\n", channel->channel); 398874aeea5SJeff Kirsher 399874aeea5SJeff Kirsher efx_nic_remove_eventq(channel); 400874aeea5SJeff Kirsher } 401874aeea5SJeff Kirsher 402874aeea5SJeff Kirsher /************************************************************************** 403874aeea5SJeff Kirsher * 404874aeea5SJeff Kirsher * Channel handling 405874aeea5SJeff Kirsher * 406874aeea5SJeff Kirsher *************************************************************************/ 407874aeea5SJeff Kirsher 4087f967c01SBen Hutchings /* Allocate and initialise a channel structure. */ 409874aeea5SJeff Kirsher static struct efx_channel * 410874aeea5SJeff Kirsher efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 411874aeea5SJeff Kirsher { 412874aeea5SJeff Kirsher struct efx_channel *channel; 413874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 414874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 415874aeea5SJeff Kirsher int j; 416874aeea5SJeff Kirsher 4177f967c01SBen Hutchings channel = kzalloc(sizeof(*channel), GFP_KERNEL); 4187f967c01SBen Hutchings if (!channel) 4197f967c01SBen Hutchings return NULL; 4207f967c01SBen Hutchings 4217f967c01SBen Hutchings channel->efx = efx; 4227f967c01SBen Hutchings channel->channel = i; 4237f967c01SBen Hutchings channel->type = &efx_default_channel_type; 4247f967c01SBen Hutchings 4257f967c01SBen Hutchings for (j = 0; j < EFX_TXQ_TYPES; j++) { 4267f967c01SBen Hutchings tx_queue = &channel->tx_queue[j]; 4277f967c01SBen Hutchings tx_queue->efx = efx; 4287f967c01SBen Hutchings tx_queue->queue = i * EFX_TXQ_TYPES + j; 4297f967c01SBen Hutchings tx_queue->channel = channel; 4307f967c01SBen Hutchings } 4317f967c01SBen Hutchings 4327f967c01SBen Hutchings rx_queue = &channel->rx_queue; 4337f967c01SBen Hutchings rx_queue->efx = efx; 4347f967c01SBen Hutchings setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 4357f967c01SBen Hutchings (unsigned long)rx_queue); 4367f967c01SBen Hutchings 4377f967c01SBen Hutchings return channel; 4387f967c01SBen Hutchings } 4397f967c01SBen Hutchings 4407f967c01SBen Hutchings /* Allocate and initialise a channel structure, copying parameters 4417f967c01SBen Hutchings * (but not resources) from an old channel structure. 4427f967c01SBen Hutchings */ 4437f967c01SBen Hutchings static struct efx_channel * 4447f967c01SBen Hutchings efx_copy_channel(const struct efx_channel *old_channel) 4457f967c01SBen Hutchings { 4467f967c01SBen Hutchings struct efx_channel *channel; 4477f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 4487f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 4497f967c01SBen Hutchings int j; 4507f967c01SBen Hutchings 451874aeea5SJeff Kirsher channel = kmalloc(sizeof(*channel), GFP_KERNEL); 452874aeea5SJeff Kirsher if (!channel) 453874aeea5SJeff Kirsher return NULL; 454874aeea5SJeff Kirsher 455874aeea5SJeff Kirsher *channel = *old_channel; 456874aeea5SJeff Kirsher 457874aeea5SJeff Kirsher channel->napi_dev = NULL; 458874aeea5SJeff Kirsher memset(&channel->eventq, 0, sizeof(channel->eventq)); 459874aeea5SJeff Kirsher 460874aeea5SJeff Kirsher for (j = 0; j < EFX_TXQ_TYPES; j++) { 461874aeea5SJeff Kirsher tx_queue = &channel->tx_queue[j]; 462874aeea5SJeff Kirsher if (tx_queue->channel) 463874aeea5SJeff Kirsher tx_queue->channel = channel; 464874aeea5SJeff Kirsher tx_queue->buffer = NULL; 465874aeea5SJeff Kirsher memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 466874aeea5SJeff Kirsher } 467874aeea5SJeff Kirsher 468874aeea5SJeff Kirsher rx_queue = &channel->rx_queue; 4697f967c01SBen Hutchings rx_queue->buffer = NULL; 4707f967c01SBen Hutchings memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 471874aeea5SJeff Kirsher setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 472874aeea5SJeff Kirsher (unsigned long)rx_queue); 473874aeea5SJeff Kirsher 474874aeea5SJeff Kirsher return channel; 475874aeea5SJeff Kirsher } 476874aeea5SJeff Kirsher 477874aeea5SJeff Kirsher static int efx_probe_channel(struct efx_channel *channel) 478874aeea5SJeff Kirsher { 479874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 480874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 481874aeea5SJeff Kirsher int rc; 482874aeea5SJeff Kirsher 483874aeea5SJeff Kirsher netif_dbg(channel->efx, probe, channel->efx->net_dev, 484874aeea5SJeff Kirsher "creating channel %d\n", channel->channel); 485874aeea5SJeff Kirsher 4867f967c01SBen Hutchings rc = channel->type->pre_probe(channel); 4877f967c01SBen Hutchings if (rc) 4887f967c01SBen Hutchings goto fail; 4897f967c01SBen Hutchings 490874aeea5SJeff Kirsher rc = efx_probe_eventq(channel); 491874aeea5SJeff Kirsher if (rc) 4927f967c01SBen Hutchings goto fail; 493874aeea5SJeff Kirsher 494874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) { 495874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 496874aeea5SJeff Kirsher if (rc) 4977f967c01SBen Hutchings goto fail; 498874aeea5SJeff Kirsher } 499874aeea5SJeff Kirsher 500874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) { 501874aeea5SJeff Kirsher rc = efx_probe_rx_queue(rx_queue); 502874aeea5SJeff Kirsher if (rc) 5037f967c01SBen Hutchings goto fail; 504874aeea5SJeff Kirsher } 505874aeea5SJeff Kirsher 506874aeea5SJeff Kirsher return 0; 507874aeea5SJeff Kirsher 5087f967c01SBen Hutchings fail: 5097f967c01SBen Hutchings efx_remove_channel(channel); 510874aeea5SJeff Kirsher return rc; 511874aeea5SJeff Kirsher } 512874aeea5SJeff Kirsher 5137f967c01SBen Hutchings static void 5147f967c01SBen Hutchings efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 5157f967c01SBen Hutchings { 5167f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 5177f967c01SBen Hutchings const char *type; 5187f967c01SBen Hutchings int number; 5197f967c01SBen Hutchings 5207f967c01SBen Hutchings number = channel->channel; 5217f967c01SBen Hutchings if (efx->tx_channel_offset == 0) { 5227f967c01SBen Hutchings type = ""; 5237f967c01SBen Hutchings } else if (channel->channel < efx->tx_channel_offset) { 5247f967c01SBen Hutchings type = "-rx"; 5257f967c01SBen Hutchings } else { 5267f967c01SBen Hutchings type = "-tx"; 5277f967c01SBen Hutchings number -= efx->tx_channel_offset; 5287f967c01SBen Hutchings } 5297f967c01SBen Hutchings snprintf(buf, len, "%s%s-%d", efx->name, type, number); 5307f967c01SBen Hutchings } 531874aeea5SJeff Kirsher 532874aeea5SJeff Kirsher static void efx_set_channel_names(struct efx_nic *efx) 533874aeea5SJeff Kirsher { 534874aeea5SJeff Kirsher struct efx_channel *channel; 535874aeea5SJeff Kirsher 5367f967c01SBen Hutchings efx_for_each_channel(channel, efx) 5377f967c01SBen Hutchings channel->type->get_name(channel, 538d8291187SBen Hutchings efx->msi_context[channel->channel].name, 539d8291187SBen Hutchings sizeof(efx->msi_context[0].name)); 540874aeea5SJeff Kirsher } 541874aeea5SJeff Kirsher 542874aeea5SJeff Kirsher static int efx_probe_channels(struct efx_nic *efx) 543874aeea5SJeff Kirsher { 544874aeea5SJeff Kirsher struct efx_channel *channel; 545874aeea5SJeff Kirsher int rc; 546874aeea5SJeff Kirsher 547874aeea5SJeff Kirsher /* Restart special buffer allocation */ 548874aeea5SJeff Kirsher efx->next_buffer_table = 0; 549874aeea5SJeff Kirsher 550c92aaff1SBen Hutchings /* Probe channels in reverse, so that any 'extra' channels 551c92aaff1SBen Hutchings * use the start of the buffer table. This allows the traffic 552c92aaff1SBen Hutchings * channels to be resized without moving them or wasting the 553c92aaff1SBen Hutchings * entries before them. 554c92aaff1SBen Hutchings */ 555c92aaff1SBen Hutchings efx_for_each_channel_rev(channel, efx) { 556874aeea5SJeff Kirsher rc = efx_probe_channel(channel); 557874aeea5SJeff Kirsher if (rc) { 558874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 559874aeea5SJeff Kirsher "failed to create channel %d\n", 560874aeea5SJeff Kirsher channel->channel); 561874aeea5SJeff Kirsher goto fail; 562874aeea5SJeff Kirsher } 563874aeea5SJeff Kirsher } 564874aeea5SJeff Kirsher efx_set_channel_names(efx); 565874aeea5SJeff Kirsher 566874aeea5SJeff Kirsher return 0; 567874aeea5SJeff Kirsher 568874aeea5SJeff Kirsher fail: 569874aeea5SJeff Kirsher efx_remove_channels(efx); 570874aeea5SJeff Kirsher return rc; 571874aeea5SJeff Kirsher } 572874aeea5SJeff Kirsher 573874aeea5SJeff Kirsher /* Channels are shutdown and reinitialised whilst the NIC is running 574874aeea5SJeff Kirsher * to propagate configuration changes (mtu, checksum offload), or 575874aeea5SJeff Kirsher * to clear hardware error conditions 576874aeea5SJeff Kirsher */ 5779f2cb71cSBen Hutchings static void efx_start_datapath(struct efx_nic *efx) 578874aeea5SJeff Kirsher { 57985740cdfSBen Hutchings bool old_rx_scatter = efx->rx_scatter; 580874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 581874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 582874aeea5SJeff Kirsher struct efx_channel *channel; 58385740cdfSBen Hutchings size_t rx_buf_len; 584874aeea5SJeff Kirsher 585874aeea5SJeff Kirsher /* Calculate the rx buffer allocation parameters required to 586874aeea5SJeff Kirsher * support the current MTU, including padding for header 587874aeea5SJeff Kirsher * alignment and overruns. 588874aeea5SJeff Kirsher */ 58943a3739dSJon Cooper efx->rx_dma_len = (efx->rx_prefix_size + 590874aeea5SJeff Kirsher EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 591874aeea5SJeff Kirsher efx->type->rx_buffer_padding); 59285740cdfSBen Hutchings rx_buf_len = (sizeof(struct efx_rx_page_state) + 5932ec03014SAndrew Rybchenko efx->rx_ip_align + efx->rx_dma_len); 59485740cdfSBen Hutchings if (rx_buf_len <= PAGE_SIZE) { 595e8c68c0aSJon Cooper efx->rx_scatter = efx->type->always_rx_scatter; 59685740cdfSBen Hutchings efx->rx_buffer_order = 0; 59785740cdfSBen Hutchings } else if (efx->type->can_rx_scatter) { 598950c54dfSBen Hutchings BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 59985740cdfSBen Hutchings BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 600950c54dfSBen Hutchings 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 601950c54dfSBen Hutchings EFX_RX_BUF_ALIGNMENT) > 602950c54dfSBen Hutchings PAGE_SIZE); 60385740cdfSBen Hutchings efx->rx_scatter = true; 60485740cdfSBen Hutchings efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 60585740cdfSBen Hutchings efx->rx_buffer_order = 0; 60685740cdfSBen Hutchings } else { 60785740cdfSBen Hutchings efx->rx_scatter = false; 60885740cdfSBen Hutchings efx->rx_buffer_order = get_order(rx_buf_len); 60985740cdfSBen Hutchings } 61085740cdfSBen Hutchings 6111648a23fSDaniel Pieczko efx_rx_config_page_split(efx); 6121648a23fSDaniel Pieczko if (efx->rx_buffer_order) 6131648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6141648a23fSDaniel Pieczko "RX buf len=%u; page order=%u batch=%u\n", 6151648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_buffer_order, 6161648a23fSDaniel Pieczko efx->rx_pages_per_batch); 6171648a23fSDaniel Pieczko else 6181648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6191648a23fSDaniel Pieczko "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 6201648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_page_buf_step, 6211648a23fSDaniel Pieczko efx->rx_bufs_per_page, efx->rx_pages_per_batch); 6222768935aSDaniel Pieczko 623e8c68c0aSJon Cooper /* RX filters may also have scatter-enabled flags */ 62485740cdfSBen Hutchings if (efx->rx_scatter != old_rx_scatter) 625add72477SBen Hutchings efx->type->filter_update_rx_scatter(efx); 626874aeea5SJeff Kirsher 62714bf718fSBen Hutchings /* We must keep at least one descriptor in a TX ring empty. 62814bf718fSBen Hutchings * We could avoid this when the queue size does not exactly 62914bf718fSBen Hutchings * match the hardware ring size, but it's not that important. 63014bf718fSBen Hutchings * Therefore we stop the queue when one more skb might fill 63114bf718fSBen Hutchings * the ring completely. We wake it when half way back to 63214bf718fSBen Hutchings * empty. 63314bf718fSBen Hutchings */ 63414bf718fSBen Hutchings efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 63514bf718fSBen Hutchings efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 63614bf718fSBen Hutchings 637874aeea5SJeff Kirsher /* Initialise the channels */ 638874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 6393881d8abSAlexandre Rames efx_for_each_channel_tx_queue(tx_queue, channel) { 640874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 6413881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 6423881d8abSAlexandre Rames } 643874aeea5SJeff Kirsher 6449f2cb71cSBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) { 645874aeea5SJeff Kirsher efx_init_rx_queue(rx_queue); 6463881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 647cce28794SJon Cooper efx_stop_eventq(channel); 648cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, false); 649cce28794SJon Cooper efx_start_eventq(channel); 6509f2cb71cSBen Hutchings } 651874aeea5SJeff Kirsher 65285740cdfSBen Hutchings WARN_ON(channel->rx_pkt_n_frags); 653874aeea5SJeff Kirsher } 6549f2cb71cSBen Hutchings 6552ea4dc28SAlexandre Rames efx_ptp_start_datapath(efx); 6562ea4dc28SAlexandre Rames 6579f2cb71cSBen Hutchings if (netif_device_present(efx->net_dev)) 6589f2cb71cSBen Hutchings netif_tx_wake_all_queues(efx->net_dev); 659874aeea5SJeff Kirsher } 660874aeea5SJeff Kirsher 6619f2cb71cSBen Hutchings static void efx_stop_datapath(struct efx_nic *efx) 662874aeea5SJeff Kirsher { 663874aeea5SJeff Kirsher struct efx_channel *channel; 664874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 665874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 666874aeea5SJeff Kirsher int rc; 667874aeea5SJeff Kirsher 668874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 669874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 670874aeea5SJeff Kirsher 6712ea4dc28SAlexandre Rames efx_ptp_stop_datapath(efx); 6722ea4dc28SAlexandre Rames 673d8aec745SBen Hutchings /* Stop RX refill */ 674d8aec745SBen Hutchings efx_for_each_channel(channel, efx) { 675d8aec745SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 676d8aec745SBen Hutchings rx_queue->refill_enabled = false; 677d8aec745SBen Hutchings } 678d8aec745SBen Hutchings 679874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 6809f2cb71cSBen Hutchings /* RX packet processing is pipelined, so wait for the 6819f2cb71cSBen Hutchings * NAPI handler to complete. At least event queue 0 6829f2cb71cSBen Hutchings * might be kept active by non-data events, so don't 6839f2cb71cSBen Hutchings * use napi_synchronize() but actually disable NAPI 6849f2cb71cSBen Hutchings * temporarily. 6859f2cb71cSBen Hutchings */ 6869f2cb71cSBen Hutchings if (efx_channel_has_rx_queue(channel)) { 6879f2cb71cSBen Hutchings efx_stop_eventq(channel); 6889f2cb71cSBen Hutchings efx_start_eventq(channel); 6899f2cb71cSBen Hutchings } 690e42c3d85SBen Hutchings } 691874aeea5SJeff Kirsher 692e42c3d85SBen Hutchings rc = efx->type->fini_dmaq(efx); 693e42c3d85SBen Hutchings if (rc && EFX_WORKAROUND_7803(efx)) { 694e42c3d85SBen Hutchings /* Schedule a reset to recover from the flush failure. The 695e42c3d85SBen Hutchings * descriptor caches reference memory we're about to free, 696e42c3d85SBen Hutchings * but falcon_reconfigure_mac_wrapper() won't reconnect 697e42c3d85SBen Hutchings * the MACs because of the pending reset. 698e42c3d85SBen Hutchings */ 699e42c3d85SBen Hutchings netif_err(efx, drv, efx->net_dev, 700e42c3d85SBen Hutchings "Resetting to recover from flush failure\n"); 701e42c3d85SBen Hutchings efx_schedule_reset(efx, RESET_TYPE_ALL); 702e42c3d85SBen Hutchings } else if (rc) { 703e42c3d85SBen Hutchings netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 704e42c3d85SBen Hutchings } else { 705e42c3d85SBen Hutchings netif_dbg(efx, drv, efx->net_dev, 706e42c3d85SBen Hutchings "successfully flushed all queues\n"); 707e42c3d85SBen Hutchings } 708e42c3d85SBen Hutchings 709e42c3d85SBen Hutchings efx_for_each_channel(channel, efx) { 710874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 711874aeea5SJeff Kirsher efx_fini_rx_queue(rx_queue); 712874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 713874aeea5SJeff Kirsher efx_fini_tx_queue(tx_queue); 714874aeea5SJeff Kirsher } 715874aeea5SJeff Kirsher } 716874aeea5SJeff Kirsher 717874aeea5SJeff Kirsher static void efx_remove_channel(struct efx_channel *channel) 718874aeea5SJeff Kirsher { 719874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 720874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 721874aeea5SJeff Kirsher 722874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 723874aeea5SJeff Kirsher "destroy chan %d\n", channel->channel); 724874aeea5SJeff Kirsher 725874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 726874aeea5SJeff Kirsher efx_remove_rx_queue(rx_queue); 727874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 728874aeea5SJeff Kirsher efx_remove_tx_queue(tx_queue); 729874aeea5SJeff Kirsher efx_remove_eventq(channel); 730c31e5f9fSStuart Hodgson channel->type->post_remove(channel); 731874aeea5SJeff Kirsher } 732874aeea5SJeff Kirsher 733874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx) 734874aeea5SJeff Kirsher { 735874aeea5SJeff Kirsher struct efx_channel *channel; 736874aeea5SJeff Kirsher 737874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 738874aeea5SJeff Kirsher efx_remove_channel(channel); 739874aeea5SJeff Kirsher } 740874aeea5SJeff Kirsher 741874aeea5SJeff Kirsher int 742874aeea5SJeff Kirsher efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 743874aeea5SJeff Kirsher { 744874aeea5SJeff Kirsher struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 745874aeea5SJeff Kirsher u32 old_rxq_entries, old_txq_entries; 7467f967c01SBen Hutchings unsigned i, next_buffer_table = 0; 747261e4d96SJon Cooper int rc, rc2; 7488b7325b4SBen Hutchings 7498b7325b4SBen Hutchings rc = efx_check_disabled(efx); 7508b7325b4SBen Hutchings if (rc) 7518b7325b4SBen Hutchings return rc; 7527f967c01SBen Hutchings 7537f967c01SBen Hutchings /* Not all channels should be reallocated. We must avoid 7547f967c01SBen Hutchings * reallocating their buffer table entries. 7557f967c01SBen Hutchings */ 7567f967c01SBen Hutchings efx_for_each_channel(channel, efx) { 7577f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 7587f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 7597f967c01SBen Hutchings 7607f967c01SBen Hutchings if (channel->type->copy) 7617f967c01SBen Hutchings continue; 7627f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7637f967c01SBen Hutchings channel->eventq.index + 7647f967c01SBen Hutchings channel->eventq.entries); 7657f967c01SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 7667f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7677f967c01SBen Hutchings rx_queue->rxd.index + 7687f967c01SBen Hutchings rx_queue->rxd.entries); 7697f967c01SBen Hutchings efx_for_each_channel_tx_queue(tx_queue, channel) 7707f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7717f967c01SBen Hutchings tx_queue->txd.index + 7727f967c01SBen Hutchings tx_queue->txd.entries); 7737f967c01SBen Hutchings } 774874aeea5SJeff Kirsher 77529c69a48SBen Hutchings efx_device_detach_sync(efx); 776874aeea5SJeff Kirsher efx_stop_all(efx); 777d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 778874aeea5SJeff Kirsher 7797f967c01SBen Hutchings /* Clone channels (where possible) */ 780874aeea5SJeff Kirsher memset(other_channel, 0, sizeof(other_channel)); 781874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 7827f967c01SBen Hutchings channel = efx->channel[i]; 7837f967c01SBen Hutchings if (channel->type->copy) 7847f967c01SBen Hutchings channel = channel->type->copy(channel); 785874aeea5SJeff Kirsher if (!channel) { 786874aeea5SJeff Kirsher rc = -ENOMEM; 787874aeea5SJeff Kirsher goto out; 788874aeea5SJeff Kirsher } 789874aeea5SJeff Kirsher other_channel[i] = channel; 790874aeea5SJeff Kirsher } 791874aeea5SJeff Kirsher 792874aeea5SJeff Kirsher /* Swap entry counts and channel pointers */ 793874aeea5SJeff Kirsher old_rxq_entries = efx->rxq_entries; 794874aeea5SJeff Kirsher old_txq_entries = efx->txq_entries; 795874aeea5SJeff Kirsher efx->rxq_entries = rxq_entries; 796874aeea5SJeff Kirsher efx->txq_entries = txq_entries; 797874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 798874aeea5SJeff Kirsher channel = efx->channel[i]; 799874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 800874aeea5SJeff Kirsher other_channel[i] = channel; 801874aeea5SJeff Kirsher } 802874aeea5SJeff Kirsher 8037f967c01SBen Hutchings /* Restart buffer table allocation */ 8047f967c01SBen Hutchings efx->next_buffer_table = next_buffer_table; 8057f967c01SBen Hutchings 8067f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8077f967c01SBen Hutchings channel = efx->channel[i]; 8087f967c01SBen Hutchings if (!channel->type->copy) 8097f967c01SBen Hutchings continue; 8107f967c01SBen Hutchings rc = efx_probe_channel(channel); 811874aeea5SJeff Kirsher if (rc) 812874aeea5SJeff Kirsher goto rollback; 8137f967c01SBen Hutchings efx_init_napi_channel(efx->channel[i]); 814874aeea5SJeff Kirsher } 815874aeea5SJeff Kirsher 8167f967c01SBen Hutchings out: 8177f967c01SBen Hutchings /* Destroy unused channel structures */ 8187f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8197f967c01SBen Hutchings channel = other_channel[i]; 8207f967c01SBen Hutchings if (channel && channel->type->copy) { 8217f967c01SBen Hutchings efx_fini_napi_channel(channel); 8227f967c01SBen Hutchings efx_remove_channel(channel); 8237f967c01SBen Hutchings kfree(channel); 8247f967c01SBen Hutchings } 8257f967c01SBen Hutchings } 8267f967c01SBen Hutchings 827261e4d96SJon Cooper rc2 = efx_soft_enable_interrupts(efx); 828261e4d96SJon Cooper if (rc2) { 829261e4d96SJon Cooper rc = rc ? rc : rc2; 830261e4d96SJon Cooper netif_err(efx, drv, efx->net_dev, 831261e4d96SJon Cooper "unable to restart interrupts on channel reallocation\n"); 832261e4d96SJon Cooper efx_schedule_reset(efx, RESET_TYPE_DISABLE); 833261e4d96SJon Cooper } else { 834874aeea5SJeff Kirsher efx_start_all(efx); 83529c69a48SBen Hutchings netif_device_attach(efx->net_dev); 836261e4d96SJon Cooper } 837874aeea5SJeff Kirsher return rc; 838874aeea5SJeff Kirsher 839874aeea5SJeff Kirsher rollback: 840874aeea5SJeff Kirsher /* Swap back */ 841874aeea5SJeff Kirsher efx->rxq_entries = old_rxq_entries; 842874aeea5SJeff Kirsher efx->txq_entries = old_txq_entries; 843874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 844874aeea5SJeff Kirsher channel = efx->channel[i]; 845874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 846874aeea5SJeff Kirsher other_channel[i] = channel; 847874aeea5SJeff Kirsher } 848874aeea5SJeff Kirsher goto out; 849874aeea5SJeff Kirsher } 850874aeea5SJeff Kirsher 851874aeea5SJeff Kirsher void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 852874aeea5SJeff Kirsher { 853874aeea5SJeff Kirsher mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 854874aeea5SJeff Kirsher } 855874aeea5SJeff Kirsher 8567f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type = { 8577f967c01SBen Hutchings .pre_probe = efx_channel_dummy_op_int, 858c31e5f9fSStuart Hodgson .post_remove = efx_channel_dummy_op_void, 8597f967c01SBen Hutchings .get_name = efx_get_channel_name, 8607f967c01SBen Hutchings .copy = efx_copy_channel, 8617f967c01SBen Hutchings .keep_eventq = false, 8627f967c01SBen Hutchings }; 8637f967c01SBen Hutchings 8647f967c01SBen Hutchings int efx_channel_dummy_op_int(struct efx_channel *channel) 8657f967c01SBen Hutchings { 8667f967c01SBen Hutchings return 0; 8677f967c01SBen Hutchings } 8687f967c01SBen Hutchings 869c31e5f9fSStuart Hodgson void efx_channel_dummy_op_void(struct efx_channel *channel) 870c31e5f9fSStuart Hodgson { 871c31e5f9fSStuart Hodgson } 872c31e5f9fSStuart Hodgson 873874aeea5SJeff Kirsher /************************************************************************** 874874aeea5SJeff Kirsher * 875874aeea5SJeff Kirsher * Port handling 876874aeea5SJeff Kirsher * 877874aeea5SJeff Kirsher **************************************************************************/ 878874aeea5SJeff Kirsher 879874aeea5SJeff Kirsher /* This ensures that the kernel is kept informed (via 880874aeea5SJeff Kirsher * netif_carrier_on/off) of the link status, and also maintains the 881874aeea5SJeff Kirsher * link status's stop on the port's TX queue. 882874aeea5SJeff Kirsher */ 883874aeea5SJeff Kirsher void efx_link_status_changed(struct efx_nic *efx) 884874aeea5SJeff Kirsher { 885874aeea5SJeff Kirsher struct efx_link_state *link_state = &efx->link_state; 886874aeea5SJeff Kirsher 887874aeea5SJeff Kirsher /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 888874aeea5SJeff Kirsher * that no events are triggered between unregister_netdev() and the 889874aeea5SJeff Kirsher * driver unloading. A more general condition is that NETDEV_CHANGE 890874aeea5SJeff Kirsher * can only be generated between NETDEV_UP and NETDEV_DOWN */ 891874aeea5SJeff Kirsher if (!netif_running(efx->net_dev)) 892874aeea5SJeff Kirsher return; 893874aeea5SJeff Kirsher 894874aeea5SJeff Kirsher if (link_state->up != netif_carrier_ok(efx->net_dev)) { 895874aeea5SJeff Kirsher efx->n_link_state_changes++; 896874aeea5SJeff Kirsher 897874aeea5SJeff Kirsher if (link_state->up) 898874aeea5SJeff Kirsher netif_carrier_on(efx->net_dev); 899874aeea5SJeff Kirsher else 900874aeea5SJeff Kirsher netif_carrier_off(efx->net_dev); 901874aeea5SJeff Kirsher } 902874aeea5SJeff Kirsher 903874aeea5SJeff Kirsher /* Status message for kernel log */ 9042aa9ef11SBen Hutchings if (link_state->up) 905874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, 906964e6135SBen Hutchings "link up at %uMbps %s-duplex (MTU %d)\n", 907874aeea5SJeff Kirsher link_state->speed, link_state->fd ? "full" : "half", 908964e6135SBen Hutchings efx->net_dev->mtu); 9092aa9ef11SBen Hutchings else 910874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, "link down\n"); 911874aeea5SJeff Kirsher } 912874aeea5SJeff Kirsher 913874aeea5SJeff Kirsher void efx_link_set_advertising(struct efx_nic *efx, u32 advertising) 914874aeea5SJeff Kirsher { 915874aeea5SJeff Kirsher efx->link_advertising = advertising; 916874aeea5SJeff Kirsher if (advertising) { 917874aeea5SJeff Kirsher if (advertising & ADVERTISED_Pause) 918874aeea5SJeff Kirsher efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 919874aeea5SJeff Kirsher else 920874aeea5SJeff Kirsher efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 921874aeea5SJeff Kirsher if (advertising & ADVERTISED_Asym_Pause) 922874aeea5SJeff Kirsher efx->wanted_fc ^= EFX_FC_TX; 923874aeea5SJeff Kirsher } 924874aeea5SJeff Kirsher } 925874aeea5SJeff Kirsher 926874aeea5SJeff Kirsher void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 927874aeea5SJeff Kirsher { 928874aeea5SJeff Kirsher efx->wanted_fc = wanted_fc; 929874aeea5SJeff Kirsher if (efx->link_advertising) { 930874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_RX) 931874aeea5SJeff Kirsher efx->link_advertising |= (ADVERTISED_Pause | 932874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 933874aeea5SJeff Kirsher else 934874aeea5SJeff Kirsher efx->link_advertising &= ~(ADVERTISED_Pause | 935874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 936874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_TX) 937874aeea5SJeff Kirsher efx->link_advertising ^= ADVERTISED_Asym_Pause; 938874aeea5SJeff Kirsher } 939874aeea5SJeff Kirsher } 940874aeea5SJeff Kirsher 941874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx); 942874aeea5SJeff Kirsher 943874aeea5SJeff Kirsher /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 944874aeea5SJeff Kirsher * the MAC appropriately. All other PHY configuration changes are pushed 945874aeea5SJeff Kirsher * through phy_op->set_settings(), and pushed asynchronously to the MAC 946874aeea5SJeff Kirsher * through efx_monitor(). 947874aeea5SJeff Kirsher * 948874aeea5SJeff Kirsher * Callers must hold the mac_lock 949874aeea5SJeff Kirsher */ 950874aeea5SJeff Kirsher int __efx_reconfigure_port(struct efx_nic *efx) 951874aeea5SJeff Kirsher { 952874aeea5SJeff Kirsher enum efx_phy_mode phy_mode; 953874aeea5SJeff Kirsher int rc; 954874aeea5SJeff Kirsher 955874aeea5SJeff Kirsher WARN_ON(!mutex_is_locked(&efx->mac_lock)); 956874aeea5SJeff Kirsher 957874aeea5SJeff Kirsher /* Disable PHY transmit in mac level loopbacks */ 958874aeea5SJeff Kirsher phy_mode = efx->phy_mode; 959874aeea5SJeff Kirsher if (LOOPBACK_INTERNAL(efx)) 960874aeea5SJeff Kirsher efx->phy_mode |= PHY_MODE_TX_DISABLED; 961874aeea5SJeff Kirsher else 962874aeea5SJeff Kirsher efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 963874aeea5SJeff Kirsher 964874aeea5SJeff Kirsher rc = efx->type->reconfigure_port(efx); 965874aeea5SJeff Kirsher 966874aeea5SJeff Kirsher if (rc) 967874aeea5SJeff Kirsher efx->phy_mode = phy_mode; 968874aeea5SJeff Kirsher 969874aeea5SJeff Kirsher return rc; 970874aeea5SJeff Kirsher } 971874aeea5SJeff Kirsher 972874aeea5SJeff Kirsher /* Reinitialise the MAC to pick up new PHY settings, even if the port is 973874aeea5SJeff Kirsher * disabled. */ 974874aeea5SJeff Kirsher int efx_reconfigure_port(struct efx_nic *efx) 975874aeea5SJeff Kirsher { 976874aeea5SJeff Kirsher int rc; 977874aeea5SJeff Kirsher 978874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 979874aeea5SJeff Kirsher 980874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 981874aeea5SJeff Kirsher rc = __efx_reconfigure_port(efx); 982874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 983874aeea5SJeff Kirsher 984874aeea5SJeff Kirsher return rc; 985874aeea5SJeff Kirsher } 986874aeea5SJeff Kirsher 987874aeea5SJeff Kirsher /* Asynchronous work item for changing MAC promiscuity and multicast 988874aeea5SJeff Kirsher * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 989874aeea5SJeff Kirsher * MAC directly. */ 990874aeea5SJeff Kirsher static void efx_mac_work(struct work_struct *data) 991874aeea5SJeff Kirsher { 992874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 993874aeea5SJeff Kirsher 994874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 99530b81cdaSBen Hutchings if (efx->port_enabled) 996710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 997874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 998874aeea5SJeff Kirsher } 999874aeea5SJeff Kirsher 1000874aeea5SJeff Kirsher static int efx_probe_port(struct efx_nic *efx) 1001874aeea5SJeff Kirsher { 1002874aeea5SJeff Kirsher int rc; 1003874aeea5SJeff Kirsher 1004874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1005874aeea5SJeff Kirsher 1006874aeea5SJeff Kirsher if (phy_flash_cfg) 1007874aeea5SJeff Kirsher efx->phy_mode = PHY_MODE_SPECIAL; 1008874aeea5SJeff Kirsher 1009874aeea5SJeff Kirsher /* Connect up MAC/PHY operations table */ 1010874aeea5SJeff Kirsher rc = efx->type->probe_port(efx); 1011874aeea5SJeff Kirsher if (rc) 1012874aeea5SJeff Kirsher return rc; 1013874aeea5SJeff Kirsher 1014e332bcb3SBen Hutchings /* Initialise MAC address to permanent address */ 1015e332bcb3SBen Hutchings memcpy(efx->net_dev->dev_addr, efx->net_dev->perm_addr, ETH_ALEN); 1016874aeea5SJeff Kirsher 1017874aeea5SJeff Kirsher return 0; 1018874aeea5SJeff Kirsher } 1019874aeea5SJeff Kirsher 1020874aeea5SJeff Kirsher static int efx_init_port(struct efx_nic *efx) 1021874aeea5SJeff Kirsher { 1022874aeea5SJeff Kirsher int rc; 1023874aeea5SJeff Kirsher 1024874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1025874aeea5SJeff Kirsher 1026874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1027874aeea5SJeff Kirsher 1028874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 1029874aeea5SJeff Kirsher if (rc) 1030874aeea5SJeff Kirsher goto fail1; 1031874aeea5SJeff Kirsher 1032874aeea5SJeff Kirsher efx->port_initialized = true; 1033874aeea5SJeff Kirsher 1034874aeea5SJeff Kirsher /* Reconfigure the MAC before creating dma queues (required for 1035874aeea5SJeff Kirsher * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1036710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 1037874aeea5SJeff Kirsher 1038874aeea5SJeff Kirsher /* Ensure the PHY advertises the correct flow control settings */ 1039874aeea5SJeff Kirsher rc = efx->phy_op->reconfigure(efx); 1040874aeea5SJeff Kirsher if (rc) 1041874aeea5SJeff Kirsher goto fail2; 1042874aeea5SJeff Kirsher 1043874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1044874aeea5SJeff Kirsher return 0; 1045874aeea5SJeff Kirsher 1046874aeea5SJeff Kirsher fail2: 1047874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1048874aeea5SJeff Kirsher fail1: 1049874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1050874aeea5SJeff Kirsher return rc; 1051874aeea5SJeff Kirsher } 1052874aeea5SJeff Kirsher 1053874aeea5SJeff Kirsher static void efx_start_port(struct efx_nic *efx) 1054874aeea5SJeff Kirsher { 1055874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1056874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 1057874aeea5SJeff Kirsher 1058874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1059874aeea5SJeff Kirsher efx->port_enabled = true; 1060874aeea5SJeff Kirsher 1061d615c039SBen Hutchings /* Ensure MAC ingress/egress is enabled */ 1062710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 1063874aeea5SJeff Kirsher 1064874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1065874aeea5SJeff Kirsher } 1066874aeea5SJeff Kirsher 1067d615c039SBen Hutchings /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1068d615c039SBen Hutchings * and the async self-test, wait for them to finish and prevent them 1069d615c039SBen Hutchings * being scheduled again. This doesn't cover online resets, which 1070d615c039SBen Hutchings * should only be cancelled when removing the device. 1071d615c039SBen Hutchings */ 1072874aeea5SJeff Kirsher static void efx_stop_port(struct efx_nic *efx) 1073874aeea5SJeff Kirsher { 1074874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1075874aeea5SJeff Kirsher 1076d615c039SBen Hutchings EFX_ASSERT_RESET_SERIALISED(efx); 1077d615c039SBen Hutchings 1078874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1079874aeea5SJeff Kirsher efx->port_enabled = false; 1080874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1081874aeea5SJeff Kirsher 1082874aeea5SJeff Kirsher /* Serialise against efx_set_multicast_list() */ 1083874aeea5SJeff Kirsher netif_addr_lock_bh(efx->net_dev); 1084874aeea5SJeff Kirsher netif_addr_unlock_bh(efx->net_dev); 1085d615c039SBen Hutchings 1086d615c039SBen Hutchings cancel_delayed_work_sync(&efx->monitor_work); 1087d615c039SBen Hutchings efx_selftest_async_cancel(efx); 1088d615c039SBen Hutchings cancel_work_sync(&efx->mac_work); 1089874aeea5SJeff Kirsher } 1090874aeea5SJeff Kirsher 1091874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx) 1092874aeea5SJeff Kirsher { 1093874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1094874aeea5SJeff Kirsher 1095874aeea5SJeff Kirsher if (!efx->port_initialized) 1096874aeea5SJeff Kirsher return; 1097874aeea5SJeff Kirsher 1098874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1099874aeea5SJeff Kirsher efx->port_initialized = false; 1100874aeea5SJeff Kirsher 1101874aeea5SJeff Kirsher efx->link_state.up = false; 1102874aeea5SJeff Kirsher efx_link_status_changed(efx); 1103874aeea5SJeff Kirsher } 1104874aeea5SJeff Kirsher 1105874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx) 1106874aeea5SJeff Kirsher { 1107874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1108874aeea5SJeff Kirsher 1109874aeea5SJeff Kirsher efx->type->remove_port(efx); 1110874aeea5SJeff Kirsher } 1111874aeea5SJeff Kirsher 1112874aeea5SJeff Kirsher /************************************************************************** 1113874aeea5SJeff Kirsher * 1114874aeea5SJeff Kirsher * NIC handling 1115874aeea5SJeff Kirsher * 1116874aeea5SJeff Kirsher **************************************************************************/ 1117874aeea5SJeff Kirsher 11180bcf4a64SBen Hutchings static LIST_HEAD(efx_primary_list); 11190bcf4a64SBen Hutchings static LIST_HEAD(efx_unassociated_list); 11200bcf4a64SBen Hutchings 11210bcf4a64SBen Hutchings static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 11220bcf4a64SBen Hutchings { 11230bcf4a64SBen Hutchings return left->type == right->type && 11240bcf4a64SBen Hutchings left->vpd_sn && right->vpd_sn && 11250bcf4a64SBen Hutchings !strcmp(left->vpd_sn, right->vpd_sn); 11260bcf4a64SBen Hutchings } 11270bcf4a64SBen Hutchings 11280bcf4a64SBen Hutchings static void efx_associate(struct efx_nic *efx) 11290bcf4a64SBen Hutchings { 11300bcf4a64SBen Hutchings struct efx_nic *other, *next; 11310bcf4a64SBen Hutchings 11320bcf4a64SBen Hutchings if (efx->primary == efx) { 11330bcf4a64SBen Hutchings /* Adding primary function; look for secondaries */ 11340bcf4a64SBen Hutchings 11350bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 11360bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_primary_list); 11370bcf4a64SBen Hutchings 11380bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx_unassociated_list, 11390bcf4a64SBen Hutchings node) { 11400bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 11410bcf4a64SBen Hutchings list_del(&other->node); 11420bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 11430bcf4a64SBen Hutchings "moving to secondary list of %s %s\n", 11440bcf4a64SBen Hutchings pci_name(efx->pci_dev), 11450bcf4a64SBen Hutchings efx->net_dev->name); 11460bcf4a64SBen Hutchings list_add_tail(&other->node, 11470bcf4a64SBen Hutchings &efx->secondary_list); 11480bcf4a64SBen Hutchings other->primary = efx; 11490bcf4a64SBen Hutchings } 11500bcf4a64SBen Hutchings } 11510bcf4a64SBen Hutchings } else { 11520bcf4a64SBen Hutchings /* Adding secondary function; look for primary */ 11530bcf4a64SBen Hutchings 11540bcf4a64SBen Hutchings list_for_each_entry(other, &efx_primary_list, node) { 11550bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 11560bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 11570bcf4a64SBen Hutchings "adding to secondary list of %s %s\n", 11580bcf4a64SBen Hutchings pci_name(other->pci_dev), 11590bcf4a64SBen Hutchings other->net_dev->name); 11600bcf4a64SBen Hutchings list_add_tail(&efx->node, 11610bcf4a64SBen Hutchings &other->secondary_list); 11620bcf4a64SBen Hutchings efx->primary = other; 11630bcf4a64SBen Hutchings return; 11640bcf4a64SBen Hutchings } 11650bcf4a64SBen Hutchings } 11660bcf4a64SBen Hutchings 11670bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 11680bcf4a64SBen Hutchings "adding to unassociated list\n"); 11690bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_unassociated_list); 11700bcf4a64SBen Hutchings } 11710bcf4a64SBen Hutchings } 11720bcf4a64SBen Hutchings 11730bcf4a64SBen Hutchings static void efx_dissociate(struct efx_nic *efx) 11740bcf4a64SBen Hutchings { 11750bcf4a64SBen Hutchings struct efx_nic *other, *next; 11760bcf4a64SBen Hutchings 11770bcf4a64SBen Hutchings list_del(&efx->node); 11780bcf4a64SBen Hutchings efx->primary = NULL; 11790bcf4a64SBen Hutchings 11800bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 11810bcf4a64SBen Hutchings list_del(&other->node); 11820bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 11830bcf4a64SBen Hutchings "moving to unassociated list\n"); 11840bcf4a64SBen Hutchings list_add_tail(&other->node, &efx_unassociated_list); 11850bcf4a64SBen Hutchings other->primary = NULL; 11860bcf4a64SBen Hutchings } 11870bcf4a64SBen Hutchings } 11880bcf4a64SBen Hutchings 1189874aeea5SJeff Kirsher /* This configures the PCI device to enable I/O and DMA. */ 1190874aeea5SJeff Kirsher static int efx_init_io(struct efx_nic *efx) 1191874aeea5SJeff Kirsher { 1192874aeea5SJeff Kirsher struct pci_dev *pci_dev = efx->pci_dev; 1193874aeea5SJeff Kirsher dma_addr_t dma_mask = efx->type->max_dma_mask; 1194b105798fSBen Hutchings unsigned int mem_map_size = efx->type->mem_map_size(efx); 1195874aeea5SJeff Kirsher int rc; 1196874aeea5SJeff Kirsher 1197874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1198874aeea5SJeff Kirsher 1199874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 1200874aeea5SJeff Kirsher if (rc) { 1201874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1202874aeea5SJeff Kirsher "failed to enable PCI device\n"); 1203874aeea5SJeff Kirsher goto fail1; 1204874aeea5SJeff Kirsher } 1205874aeea5SJeff Kirsher 1206874aeea5SJeff Kirsher pci_set_master(pci_dev); 1207874aeea5SJeff Kirsher 1208874aeea5SJeff Kirsher /* Set the PCI DMA mask. Try all possibilities from our 1209874aeea5SJeff Kirsher * genuine mask down to 32 bits, because some architectures 1210874aeea5SJeff Kirsher * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1211874aeea5SJeff Kirsher * masks event though they reject 46 bit masks. 1212874aeea5SJeff Kirsher */ 1213874aeea5SJeff Kirsher while (dma_mask > 0x7fffffffUL) { 12140e33d870SBen Hutchings if (dma_supported(&pci_dev->dev, dma_mask)) { 12159663dedaSRussell King rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1216e9e01846SBen Hutchings if (rc == 0) 1217874aeea5SJeff Kirsher break; 1218e9e01846SBen Hutchings } 1219874aeea5SJeff Kirsher dma_mask >>= 1; 1220874aeea5SJeff Kirsher } 1221874aeea5SJeff Kirsher if (rc) { 1222874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1223874aeea5SJeff Kirsher "could not find a suitable DMA mask\n"); 1224874aeea5SJeff Kirsher goto fail2; 1225874aeea5SJeff Kirsher } 1226874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1227874aeea5SJeff Kirsher "using DMA mask %llx\n", (unsigned long long) dma_mask); 1228874aeea5SJeff Kirsher 1229874aeea5SJeff Kirsher efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR); 1230874aeea5SJeff Kirsher rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc"); 1231874aeea5SJeff Kirsher if (rc) { 1232874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1233874aeea5SJeff Kirsher "request for memory BAR failed\n"); 1234874aeea5SJeff Kirsher rc = -EIO; 1235874aeea5SJeff Kirsher goto fail3; 1236874aeea5SJeff Kirsher } 1237b105798fSBen Hutchings efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1238874aeea5SJeff Kirsher if (!efx->membase) { 1239874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1240874aeea5SJeff Kirsher "could not map memory BAR at %llx+%x\n", 1241b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size); 1242874aeea5SJeff Kirsher rc = -ENOMEM; 1243874aeea5SJeff Kirsher goto fail4; 1244874aeea5SJeff Kirsher } 1245874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1246874aeea5SJeff Kirsher "memory BAR at %llx+%x (virtual %p)\n", 1247b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size, 1248b105798fSBen Hutchings efx->membase); 1249874aeea5SJeff Kirsher 1250874aeea5SJeff Kirsher return 0; 1251874aeea5SJeff Kirsher 1252874aeea5SJeff Kirsher fail4: 1253874aeea5SJeff Kirsher pci_release_region(efx->pci_dev, EFX_MEM_BAR); 1254874aeea5SJeff Kirsher fail3: 1255874aeea5SJeff Kirsher efx->membase_phys = 0; 1256874aeea5SJeff Kirsher fail2: 1257874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1258874aeea5SJeff Kirsher fail1: 1259874aeea5SJeff Kirsher return rc; 1260874aeea5SJeff Kirsher } 1261874aeea5SJeff Kirsher 1262874aeea5SJeff Kirsher static void efx_fini_io(struct efx_nic *efx) 1263874aeea5SJeff Kirsher { 1264874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1265874aeea5SJeff Kirsher 1266874aeea5SJeff Kirsher if (efx->membase) { 1267874aeea5SJeff Kirsher iounmap(efx->membase); 1268874aeea5SJeff Kirsher efx->membase = NULL; 1269874aeea5SJeff Kirsher } 1270874aeea5SJeff Kirsher 1271874aeea5SJeff Kirsher if (efx->membase_phys) { 1272874aeea5SJeff Kirsher pci_release_region(efx->pci_dev, EFX_MEM_BAR); 1273874aeea5SJeff Kirsher efx->membase_phys = 0; 1274874aeea5SJeff Kirsher } 1275874aeea5SJeff Kirsher 1276874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1277874aeea5SJeff Kirsher } 1278874aeea5SJeff Kirsher 1279a9a52506SBen Hutchings static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1280874aeea5SJeff Kirsher { 1281cdb08f8fSBen Hutchings cpumask_var_t thread_mask; 1282a16e5b24SBen Hutchings unsigned int count; 1283874aeea5SJeff Kirsher int cpu; 1284874aeea5SJeff Kirsher 1285cd2d5b52SBen Hutchings if (rss_cpus) { 1286cd2d5b52SBen Hutchings count = rss_cpus; 1287cd2d5b52SBen Hutchings } else { 1288cdb08f8fSBen Hutchings if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1289a9a52506SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1290a9a52506SBen Hutchings "RSS disabled due to allocation failure\n"); 1291874aeea5SJeff Kirsher return 1; 1292874aeea5SJeff Kirsher } 1293874aeea5SJeff Kirsher 1294874aeea5SJeff Kirsher count = 0; 1295874aeea5SJeff Kirsher for_each_online_cpu(cpu) { 1296cdb08f8fSBen Hutchings if (!cpumask_test_cpu(cpu, thread_mask)) { 1297874aeea5SJeff Kirsher ++count; 1298cdb08f8fSBen Hutchings cpumask_or(thread_mask, thread_mask, 1299cdb08f8fSBen Hutchings topology_thread_cpumask(cpu)); 1300874aeea5SJeff Kirsher } 1301874aeea5SJeff Kirsher } 1302874aeea5SJeff Kirsher 1303cdb08f8fSBen Hutchings free_cpumask_var(thread_mask); 1304cd2d5b52SBen Hutchings } 1305cd2d5b52SBen Hutchings 1306cd2d5b52SBen Hutchings /* If RSS is requested for the PF *and* VFs then we can't write RSS 1307cd2d5b52SBen Hutchings * table entries that are inaccessible to VFs 1308cd2d5b52SBen Hutchings */ 1309cd2d5b52SBen Hutchings if (efx_sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1310cd2d5b52SBen Hutchings count > efx_vf_size(efx)) { 1311cd2d5b52SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1312cd2d5b52SBen Hutchings "Reducing number of RSS channels from %u to %u for " 1313cd2d5b52SBen Hutchings "VF support. Increase vf-msix-limit to use more " 1314cd2d5b52SBen Hutchings "channels on the PF.\n", 1315cd2d5b52SBen Hutchings count, efx_vf_size(efx)); 1316cd2d5b52SBen Hutchings count = efx_vf_size(efx); 1317cd2d5b52SBen Hutchings } 1318cd2d5b52SBen Hutchings 1319874aeea5SJeff Kirsher return count; 1320874aeea5SJeff Kirsher } 1321874aeea5SJeff Kirsher 1322874aeea5SJeff Kirsher /* Probe the number and type of interrupts we are able to obtain, and 1323874aeea5SJeff Kirsher * the resulting numbers of channels and RX queues. 1324874aeea5SJeff Kirsher */ 1325874aeea5SJeff Kirsher static int efx_probe_interrupts(struct efx_nic *efx) 1326874aeea5SJeff Kirsher { 13277f967c01SBen Hutchings unsigned int extra_channels = 0; 13287f967c01SBen Hutchings unsigned int i, j; 1329a16e5b24SBen Hutchings int rc; 1330874aeea5SJeff Kirsher 13317f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 13327f967c01SBen Hutchings if (efx->extra_channel_type[i]) 13337f967c01SBen Hutchings ++extra_channels; 13347f967c01SBen Hutchings 1335874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1336874aeea5SJeff Kirsher struct msix_entry xentries[EFX_MAX_CHANNELS]; 1337a16e5b24SBen Hutchings unsigned int n_channels; 1338874aeea5SJeff Kirsher 1339a9a52506SBen Hutchings n_channels = efx_wanted_parallelism(efx); 1340874aeea5SJeff Kirsher if (separate_tx_channels) 1341874aeea5SJeff Kirsher n_channels *= 2; 13427f967c01SBen Hutchings n_channels += extra_channels; 1343b105798fSBen Hutchings n_channels = min(n_channels, efx->max_channels); 1344874aeea5SJeff Kirsher 1345874aeea5SJeff Kirsher for (i = 0; i < n_channels; i++) 1346874aeea5SJeff Kirsher xentries[i].entry = i; 1347874aeea5SJeff Kirsher rc = pci_enable_msix(efx->pci_dev, xentries, n_channels); 1348874aeea5SJeff Kirsher if (rc > 0) { 1349874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1350874aeea5SJeff Kirsher "WARNING: Insufficient MSI-X vectors" 1351a16e5b24SBen Hutchings " available (%d < %u).\n", rc, n_channels); 1352874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1353874aeea5SJeff Kirsher "WARNING: Performance may be reduced.\n"); 1354874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(rc >= n_channels); 1355874aeea5SJeff Kirsher n_channels = rc; 1356874aeea5SJeff Kirsher rc = pci_enable_msix(efx->pci_dev, xentries, 1357874aeea5SJeff Kirsher n_channels); 1358874aeea5SJeff Kirsher } 1359874aeea5SJeff Kirsher 1360874aeea5SJeff Kirsher if (rc == 0) { 1361874aeea5SJeff Kirsher efx->n_channels = n_channels; 13627f967c01SBen Hutchings if (n_channels > extra_channels) 13637f967c01SBen Hutchings n_channels -= extra_channels; 1364874aeea5SJeff Kirsher if (separate_tx_channels) { 13657f967c01SBen Hutchings efx->n_tx_channels = max(n_channels / 2, 1U); 13667f967c01SBen Hutchings efx->n_rx_channels = max(n_channels - 13677f967c01SBen Hutchings efx->n_tx_channels, 13687f967c01SBen Hutchings 1U); 1369874aeea5SJeff Kirsher } else { 13707f967c01SBen Hutchings efx->n_tx_channels = n_channels; 13717f967c01SBen Hutchings efx->n_rx_channels = n_channels; 1372874aeea5SJeff Kirsher } 13737f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) 1374874aeea5SJeff Kirsher efx_get_channel(efx, i)->irq = 1375874aeea5SJeff Kirsher xentries[i].vector; 1376874aeea5SJeff Kirsher } else { 1377874aeea5SJeff Kirsher /* Fall back to single channel MSI */ 1378874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_MSI; 1379874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1380874aeea5SJeff Kirsher "could not enable MSI-X\n"); 1381874aeea5SJeff Kirsher } 1382874aeea5SJeff Kirsher } 1383874aeea5SJeff Kirsher 1384874aeea5SJeff Kirsher /* Try single interrupt MSI */ 1385874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1386874aeea5SJeff Kirsher efx->n_channels = 1; 1387874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1388874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1389874aeea5SJeff Kirsher rc = pci_enable_msi(efx->pci_dev); 1390874aeea5SJeff Kirsher if (rc == 0) { 1391874aeea5SJeff Kirsher efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1392874aeea5SJeff Kirsher } else { 1393874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1394874aeea5SJeff Kirsher "could not enable MSI\n"); 1395874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1396874aeea5SJeff Kirsher } 1397874aeea5SJeff Kirsher } 1398874aeea5SJeff Kirsher 1399874aeea5SJeff Kirsher /* Assume legacy interrupts */ 1400874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1401874aeea5SJeff Kirsher efx->n_channels = 1 + (separate_tx_channels ? 1 : 0); 1402874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1403874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1404874aeea5SJeff Kirsher efx->legacy_irq = efx->pci_dev->irq; 1405874aeea5SJeff Kirsher } 1406874aeea5SJeff Kirsher 14077f967c01SBen Hutchings /* Assign extra channels if possible */ 14087f967c01SBen Hutchings j = efx->n_channels; 14097f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 14107f967c01SBen Hutchings if (!efx->extra_channel_type[i]) 14117f967c01SBen Hutchings continue; 14127f967c01SBen Hutchings if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 14137f967c01SBen Hutchings efx->n_channels <= extra_channels) { 14147f967c01SBen Hutchings efx->extra_channel_type[i]->handle_no_channel(efx); 14157f967c01SBen Hutchings } else { 14167f967c01SBen Hutchings --j; 14177f967c01SBen Hutchings efx_get_channel(efx, j)->type = 14187f967c01SBen Hutchings efx->extra_channel_type[i]; 14197f967c01SBen Hutchings } 14207f967c01SBen Hutchings } 14217f967c01SBen Hutchings 1422cd2d5b52SBen Hutchings /* RSS might be usable on VFs even if it is disabled on the PF */ 14233132d282SBen Hutchings efx->rss_spread = ((efx->n_rx_channels > 1 || !efx_sriov_wanted(efx)) ? 1424cd2d5b52SBen Hutchings efx->n_rx_channels : efx_vf_size(efx)); 1425cd2d5b52SBen Hutchings 1426874aeea5SJeff Kirsher return 0; 1427874aeea5SJeff Kirsher } 1428874aeea5SJeff Kirsher 1429261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx) 1430d8291187SBen Hutchings { 1431261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1432261e4d96SJon Cooper int rc; 1433d8291187SBen Hutchings 1434d8291187SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1435d8291187SBen Hutchings 1436d8291187SBen Hutchings efx->irq_soft_enabled = true; 1437d8291187SBen Hutchings smp_wmb(); 1438d8291187SBen Hutchings 1439d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1440261e4d96SJon Cooper if (!channel->type->keep_eventq) { 1441261e4d96SJon Cooper rc = efx_init_eventq(channel); 1442261e4d96SJon Cooper if (rc) 1443261e4d96SJon Cooper goto fail; 1444261e4d96SJon Cooper } 1445d8291187SBen Hutchings efx_start_eventq(channel); 1446d8291187SBen Hutchings } 1447d8291187SBen Hutchings 1448d8291187SBen Hutchings efx_mcdi_mode_event(efx); 1449261e4d96SJon Cooper 1450261e4d96SJon Cooper return 0; 1451261e4d96SJon Cooper fail: 1452261e4d96SJon Cooper end_channel = channel; 1453261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1454261e4d96SJon Cooper if (channel == end_channel) 1455261e4d96SJon Cooper break; 1456261e4d96SJon Cooper efx_stop_eventq(channel); 1457261e4d96SJon Cooper if (!channel->type->keep_eventq) 1458261e4d96SJon Cooper efx_fini_eventq(channel); 1459261e4d96SJon Cooper } 1460261e4d96SJon Cooper 1461261e4d96SJon Cooper return rc; 1462d8291187SBen Hutchings } 1463d8291187SBen Hutchings 1464d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx) 1465d8291187SBen Hutchings { 1466d8291187SBen Hutchings struct efx_channel *channel; 1467d8291187SBen Hutchings 1468d8291187SBen Hutchings if (efx->state == STATE_DISABLED) 1469d8291187SBen Hutchings return; 1470d8291187SBen Hutchings 1471d8291187SBen Hutchings efx_mcdi_mode_poll(efx); 1472d8291187SBen Hutchings 1473d8291187SBen Hutchings efx->irq_soft_enabled = false; 1474d8291187SBen Hutchings smp_wmb(); 1475d8291187SBen Hutchings 1476d8291187SBen Hutchings if (efx->legacy_irq) 1477d8291187SBen Hutchings synchronize_irq(efx->legacy_irq); 1478d8291187SBen Hutchings 1479d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1480d8291187SBen Hutchings if (channel->irq) 1481d8291187SBen Hutchings synchronize_irq(channel->irq); 1482d8291187SBen Hutchings 1483d8291187SBen Hutchings efx_stop_eventq(channel); 1484d8291187SBen Hutchings if (!channel->type->keep_eventq) 1485d8291187SBen Hutchings efx_fini_eventq(channel); 1486d8291187SBen Hutchings } 1487cade715fSBen Hutchings 1488cade715fSBen Hutchings /* Flush the asynchronous MCDI request queue */ 1489cade715fSBen Hutchings efx_mcdi_flush_async(efx); 1490d8291187SBen Hutchings } 1491d8291187SBen Hutchings 1492261e4d96SJon Cooper static int efx_enable_interrupts(struct efx_nic *efx) 14939f2cb71cSBen Hutchings { 1494261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1495261e4d96SJon Cooper int rc; 14969f2cb71cSBen Hutchings 14978b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 14988b7325b4SBen Hutchings 1499b28405b0SAlexandre Rames if (efx->eeh_disabled_legacy_irq) { 1500b28405b0SAlexandre Rames enable_irq(efx->legacy_irq); 1501b28405b0SAlexandre Rames efx->eeh_disabled_legacy_irq = false; 1502b28405b0SAlexandre Rames } 1503d8291187SBen Hutchings 150486094f7fSBen Hutchings efx->type->irq_enable_master(efx); 15059f2cb71cSBen Hutchings 15069f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1507261e4d96SJon Cooper if (channel->type->keep_eventq) { 1508261e4d96SJon Cooper rc = efx_init_eventq(channel); 1509261e4d96SJon Cooper if (rc) 1510261e4d96SJon Cooper goto fail; 1511261e4d96SJon Cooper } 15129f2cb71cSBen Hutchings } 15139f2cb71cSBen Hutchings 1514261e4d96SJon Cooper rc = efx_soft_enable_interrupts(efx); 1515261e4d96SJon Cooper if (rc) 1516261e4d96SJon Cooper goto fail; 1517261e4d96SJon Cooper 1518261e4d96SJon Cooper return 0; 1519261e4d96SJon Cooper 1520261e4d96SJon Cooper fail: 1521261e4d96SJon Cooper end_channel = channel; 1522261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1523261e4d96SJon Cooper if (channel == end_channel) 1524261e4d96SJon Cooper break; 1525261e4d96SJon Cooper if (channel->type->keep_eventq) 1526261e4d96SJon Cooper efx_fini_eventq(channel); 1527261e4d96SJon Cooper } 1528261e4d96SJon Cooper 1529261e4d96SJon Cooper efx->type->irq_disable_non_ev(efx); 1530261e4d96SJon Cooper 1531261e4d96SJon Cooper return rc; 15329f2cb71cSBen Hutchings } 15339f2cb71cSBen Hutchings 1534d8291187SBen Hutchings static void efx_disable_interrupts(struct efx_nic *efx) 15359f2cb71cSBen Hutchings { 15369f2cb71cSBen Hutchings struct efx_channel *channel; 15379f2cb71cSBen Hutchings 1538d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 15399f2cb71cSBen Hutchings 15409f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1541d8291187SBen Hutchings if (channel->type->keep_eventq) 15429f2cb71cSBen Hutchings efx_fini_eventq(channel); 15439f2cb71cSBen Hutchings } 1544d8291187SBen Hutchings 154586094f7fSBen Hutchings efx->type->irq_disable_non_ev(efx); 15469f2cb71cSBen Hutchings } 15479f2cb71cSBen Hutchings 1548874aeea5SJeff Kirsher static void efx_remove_interrupts(struct efx_nic *efx) 1549874aeea5SJeff Kirsher { 1550874aeea5SJeff Kirsher struct efx_channel *channel; 1551874aeea5SJeff Kirsher 1552874aeea5SJeff Kirsher /* Remove MSI/MSI-X interrupts */ 1553874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1554874aeea5SJeff Kirsher channel->irq = 0; 1555874aeea5SJeff Kirsher pci_disable_msi(efx->pci_dev); 1556874aeea5SJeff Kirsher pci_disable_msix(efx->pci_dev); 1557874aeea5SJeff Kirsher 1558874aeea5SJeff Kirsher /* Remove legacy interrupt */ 1559874aeea5SJeff Kirsher efx->legacy_irq = 0; 1560874aeea5SJeff Kirsher } 1561874aeea5SJeff Kirsher 1562874aeea5SJeff Kirsher static void efx_set_channels(struct efx_nic *efx) 1563874aeea5SJeff Kirsher { 1564874aeea5SJeff Kirsher struct efx_channel *channel; 1565874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 1566874aeea5SJeff Kirsher 1567874aeea5SJeff Kirsher efx->tx_channel_offset = 1568874aeea5SJeff Kirsher separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0; 1569874aeea5SJeff Kirsher 157079d68b37SStuart Hodgson /* We need to mark which channels really have RX and TX 157179d68b37SStuart Hodgson * queues, and adjust the TX queue numbers if we have separate 1572874aeea5SJeff Kirsher * RX-only and TX-only channels. 1573874aeea5SJeff Kirsher */ 1574874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 157579d68b37SStuart Hodgson if (channel->channel < efx->n_rx_channels) 157679d68b37SStuart Hodgson channel->rx_queue.core_index = channel->channel; 157779d68b37SStuart Hodgson else 157879d68b37SStuart Hodgson channel->rx_queue.core_index = -1; 157979d68b37SStuart Hodgson 1580874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 1581874aeea5SJeff Kirsher tx_queue->queue -= (efx->tx_channel_offset * 1582874aeea5SJeff Kirsher EFX_TXQ_TYPES); 1583874aeea5SJeff Kirsher } 1584874aeea5SJeff Kirsher } 1585874aeea5SJeff Kirsher 1586874aeea5SJeff Kirsher static int efx_probe_nic(struct efx_nic *efx) 1587874aeea5SJeff Kirsher { 1588874aeea5SJeff Kirsher size_t i; 1589874aeea5SJeff Kirsher int rc; 1590874aeea5SJeff Kirsher 1591874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1592874aeea5SJeff Kirsher 1593874aeea5SJeff Kirsher /* Carry out hardware-type specific initialisation */ 1594874aeea5SJeff Kirsher rc = efx->type->probe(efx); 1595874aeea5SJeff Kirsher if (rc) 1596874aeea5SJeff Kirsher return rc; 1597874aeea5SJeff Kirsher 1598874aeea5SJeff Kirsher /* Determine the number of channels and queues by trying to hook 1599874aeea5SJeff Kirsher * in MSI-X interrupts. */ 1600874aeea5SJeff Kirsher rc = efx_probe_interrupts(efx); 1601874aeea5SJeff Kirsher if (rc) 1602c15eed22SBen Hutchings goto fail1; 1603874aeea5SJeff Kirsher 1604c15eed22SBen Hutchings rc = efx->type->dimension_resources(efx); 1605c15eed22SBen Hutchings if (rc) 1606c15eed22SBen Hutchings goto fail2; 160728e47c49SBen Hutchings 1608874aeea5SJeff Kirsher if (efx->n_channels > 1) 1609874aeea5SJeff Kirsher get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key)); 1610874aeea5SJeff Kirsher for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1611278bc429SBen Hutchings efx->rx_indir_table[i] = 1612cd2d5b52SBen Hutchings ethtool_rxfh_indir_default(i, efx->rss_spread); 1613874aeea5SJeff Kirsher 1614874aeea5SJeff Kirsher efx_set_channels(efx); 1615874aeea5SJeff Kirsher netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1616874aeea5SJeff Kirsher netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1617874aeea5SJeff Kirsher 1618874aeea5SJeff Kirsher /* Initialise the interrupt moderation settings */ 16199e393b30SBen Hutchings efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 16209e393b30SBen Hutchings true); 1621874aeea5SJeff Kirsher 1622874aeea5SJeff Kirsher return 0; 1623874aeea5SJeff Kirsher 1624c15eed22SBen Hutchings fail2: 1625c15eed22SBen Hutchings efx_remove_interrupts(efx); 1626c15eed22SBen Hutchings fail1: 1627874aeea5SJeff Kirsher efx->type->remove(efx); 1628874aeea5SJeff Kirsher return rc; 1629874aeea5SJeff Kirsher } 1630874aeea5SJeff Kirsher 1631874aeea5SJeff Kirsher static void efx_remove_nic(struct efx_nic *efx) 1632874aeea5SJeff Kirsher { 1633874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1634874aeea5SJeff Kirsher 1635874aeea5SJeff Kirsher efx_remove_interrupts(efx); 1636874aeea5SJeff Kirsher efx->type->remove(efx); 1637874aeea5SJeff Kirsher } 1638874aeea5SJeff Kirsher 1639add72477SBen Hutchings static int efx_probe_filters(struct efx_nic *efx) 1640add72477SBen Hutchings { 1641add72477SBen Hutchings int rc; 1642add72477SBen Hutchings 1643add72477SBen Hutchings spin_lock_init(&efx->filter_lock); 1644add72477SBen Hutchings 1645add72477SBen Hutchings rc = efx->type->filter_table_probe(efx); 1646add72477SBen Hutchings if (rc) 1647add72477SBen Hutchings return rc; 1648add72477SBen Hutchings 1649add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1650add72477SBen Hutchings if (efx->type->offload_features & NETIF_F_NTUPLE) { 1651add72477SBen Hutchings efx->rps_flow_id = kcalloc(efx->type->max_rx_ip_filters, 1652add72477SBen Hutchings sizeof(*efx->rps_flow_id), 1653add72477SBen Hutchings GFP_KERNEL); 1654add72477SBen Hutchings if (!efx->rps_flow_id) { 1655add72477SBen Hutchings efx->type->filter_table_remove(efx); 1656add72477SBen Hutchings return -ENOMEM; 1657add72477SBen Hutchings } 1658add72477SBen Hutchings } 1659add72477SBen Hutchings #endif 1660add72477SBen Hutchings 1661add72477SBen Hutchings return 0; 1662add72477SBen Hutchings } 1663add72477SBen Hutchings 1664add72477SBen Hutchings static void efx_remove_filters(struct efx_nic *efx) 1665add72477SBen Hutchings { 1666add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1667add72477SBen Hutchings kfree(efx->rps_flow_id); 1668add72477SBen Hutchings #endif 1669add72477SBen Hutchings efx->type->filter_table_remove(efx); 1670add72477SBen Hutchings } 1671add72477SBen Hutchings 1672add72477SBen Hutchings static void efx_restore_filters(struct efx_nic *efx) 1673add72477SBen Hutchings { 1674add72477SBen Hutchings efx->type->filter_table_restore(efx); 1675add72477SBen Hutchings } 1676add72477SBen Hutchings 1677874aeea5SJeff Kirsher /************************************************************************** 1678874aeea5SJeff Kirsher * 1679874aeea5SJeff Kirsher * NIC startup/shutdown 1680874aeea5SJeff Kirsher * 1681874aeea5SJeff Kirsher *************************************************************************/ 1682874aeea5SJeff Kirsher 1683874aeea5SJeff Kirsher static int efx_probe_all(struct efx_nic *efx) 1684874aeea5SJeff Kirsher { 1685874aeea5SJeff Kirsher int rc; 1686874aeea5SJeff Kirsher 1687874aeea5SJeff Kirsher rc = efx_probe_nic(efx); 1688874aeea5SJeff Kirsher if (rc) { 1689874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1690874aeea5SJeff Kirsher goto fail1; 1691874aeea5SJeff Kirsher } 1692874aeea5SJeff Kirsher 1693874aeea5SJeff Kirsher rc = efx_probe_port(efx); 1694874aeea5SJeff Kirsher if (rc) { 1695874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1696874aeea5SJeff Kirsher goto fail2; 1697874aeea5SJeff Kirsher } 1698874aeea5SJeff Kirsher 16997e6d06f0SBen Hutchings BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 17007e6d06f0SBen Hutchings if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 17017e6d06f0SBen Hutchings rc = -EINVAL; 17027e6d06f0SBen Hutchings goto fail3; 17037e6d06f0SBen Hutchings } 1704874aeea5SJeff Kirsher efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1705874aeea5SJeff Kirsher 1706874aeea5SJeff Kirsher rc = efx_probe_filters(efx); 1707874aeea5SJeff Kirsher if (rc) { 1708874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1709874aeea5SJeff Kirsher "failed to create filter tables\n"); 17107f967c01SBen Hutchings goto fail3; 1711874aeea5SJeff Kirsher } 1712874aeea5SJeff Kirsher 17137f967c01SBen Hutchings rc = efx_probe_channels(efx); 17147f967c01SBen Hutchings if (rc) 17157f967c01SBen Hutchings goto fail4; 17167f967c01SBen Hutchings 1717874aeea5SJeff Kirsher return 0; 1718874aeea5SJeff Kirsher 1719874aeea5SJeff Kirsher fail4: 17207f967c01SBen Hutchings efx_remove_filters(efx); 1721874aeea5SJeff Kirsher fail3: 1722874aeea5SJeff Kirsher efx_remove_port(efx); 1723874aeea5SJeff Kirsher fail2: 1724874aeea5SJeff Kirsher efx_remove_nic(efx); 1725874aeea5SJeff Kirsher fail1: 1726874aeea5SJeff Kirsher return rc; 1727874aeea5SJeff Kirsher } 1728874aeea5SJeff Kirsher 17298b7325b4SBen Hutchings /* If the interface is supposed to be running but is not, start 17308b7325b4SBen Hutchings * the hardware and software data path, regular activity for the port 17318b7325b4SBen Hutchings * (MAC statistics, link polling, etc.) and schedule the port to be 17328b7325b4SBen Hutchings * reconfigured. Interrupts must already be enabled. This function 17338b7325b4SBen Hutchings * is safe to call multiple times, so long as the NIC is not disabled. 17348b7325b4SBen Hutchings * Requires the RTNL lock. 17359f2cb71cSBen Hutchings */ 1736874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx) 1737874aeea5SJeff Kirsher { 1738874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 17398b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1740874aeea5SJeff Kirsher 1741874aeea5SJeff Kirsher /* Check that it is appropriate to restart the interface. All 1742874aeea5SJeff Kirsher * of these flags are safe to read under just the rtnl lock */ 17438b7325b4SBen Hutchings if (efx->port_enabled || !netif_running(efx->net_dev)) 1744874aeea5SJeff Kirsher return; 1745874aeea5SJeff Kirsher 1746874aeea5SJeff Kirsher efx_start_port(efx); 17479f2cb71cSBen Hutchings efx_start_datapath(efx); 1748874aeea5SJeff Kirsher 1749626950dbSAlexandre Rames /* Start the hardware monitor if there is one */ 1750626950dbSAlexandre Rames if (efx->type->monitor != NULL) 1751874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 1752874aeea5SJeff Kirsher efx_monitor_interval); 1753626950dbSAlexandre Rames 1754626950dbSAlexandre Rames /* If link state detection is normally event-driven, we have 1755626950dbSAlexandre Rames * to poll now because we could have missed a change 1756626950dbSAlexandre Rames */ 1757626950dbSAlexandre Rames if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { 1758874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1759874aeea5SJeff Kirsher if (efx->phy_op->poll(efx)) 1760874aeea5SJeff Kirsher efx_link_status_changed(efx); 1761874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1762874aeea5SJeff Kirsher } 1763874aeea5SJeff Kirsher 1764874aeea5SJeff Kirsher efx->type->start_stats(efx); 1765f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 1766f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 1767f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 1768f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 1769874aeea5SJeff Kirsher } 1770874aeea5SJeff Kirsher 17718b7325b4SBen Hutchings /* Quiesce the hardware and software data path, and regular activity 17728b7325b4SBen Hutchings * for the port without bringing the link down. Safe to call multiple 17738b7325b4SBen Hutchings * times with the NIC in almost any state, but interrupts should be 17748b7325b4SBen Hutchings * enabled. Requires the RTNL lock. 17758b7325b4SBen Hutchings */ 1776874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx) 1777874aeea5SJeff Kirsher { 1778874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1779874aeea5SJeff Kirsher 1780874aeea5SJeff Kirsher /* port_enabled can be read safely under the rtnl lock */ 1781874aeea5SJeff Kirsher if (!efx->port_enabled) 1782874aeea5SJeff Kirsher return; 1783874aeea5SJeff Kirsher 1784f8f3b5aeSJon Cooper /* update stats before we go down so we can accurately count 1785f8f3b5aeSJon Cooper * rx_nodesc_drops 1786f8f3b5aeSJon Cooper */ 1787f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 1788f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 1789f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 1790f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 1791874aeea5SJeff Kirsher efx->type->stop_stats(efx); 1792874aeea5SJeff Kirsher efx_stop_port(efx); 1793874aeea5SJeff Kirsher 179429c69a48SBen Hutchings /* Stop the kernel transmit interface. This is only valid if 179529c69a48SBen Hutchings * the device is stopped or detached; otherwise the watchdog 179629c69a48SBen Hutchings * may fire immediately. 179729c69a48SBen Hutchings */ 179829c69a48SBen Hutchings WARN_ON(netif_running(efx->net_dev) && 179929c69a48SBen Hutchings netif_device_present(efx->net_dev)); 18009f2cb71cSBen Hutchings netif_tx_disable(efx->net_dev); 18019f2cb71cSBen Hutchings 18029f2cb71cSBen Hutchings efx_stop_datapath(efx); 1803874aeea5SJeff Kirsher } 1804874aeea5SJeff Kirsher 1805874aeea5SJeff Kirsher static void efx_remove_all(struct efx_nic *efx) 1806874aeea5SJeff Kirsher { 1807874aeea5SJeff Kirsher efx_remove_channels(efx); 18087f967c01SBen Hutchings efx_remove_filters(efx); 1809874aeea5SJeff Kirsher efx_remove_port(efx); 1810874aeea5SJeff Kirsher efx_remove_nic(efx); 1811874aeea5SJeff Kirsher } 1812874aeea5SJeff Kirsher 1813874aeea5SJeff Kirsher /************************************************************************** 1814874aeea5SJeff Kirsher * 1815874aeea5SJeff Kirsher * Interrupt moderation 1816874aeea5SJeff Kirsher * 1817874aeea5SJeff Kirsher **************************************************************************/ 1818874aeea5SJeff Kirsher 1819cc180b69SBen Hutchings static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns) 1820874aeea5SJeff Kirsher { 1821b548f976SBen Hutchings if (usecs == 0) 1822b548f976SBen Hutchings return 0; 1823cc180b69SBen Hutchings if (usecs * 1000 < quantum_ns) 1824874aeea5SJeff Kirsher return 1; /* never round down to 0 */ 1825cc180b69SBen Hutchings return usecs * 1000 / quantum_ns; 1826874aeea5SJeff Kirsher } 1827874aeea5SJeff Kirsher 1828874aeea5SJeff Kirsher /* Set interrupt moderation parameters */ 18299e393b30SBen Hutchings int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 18309e393b30SBen Hutchings unsigned int rx_usecs, bool rx_adaptive, 18319e393b30SBen Hutchings bool rx_may_override_tx) 1832874aeea5SJeff Kirsher { 1833874aeea5SJeff Kirsher struct efx_channel *channel; 1834cc180b69SBen Hutchings unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max * 1835cc180b69SBen Hutchings efx->timer_quantum_ns, 1836cc180b69SBen Hutchings 1000); 1837cc180b69SBen Hutchings unsigned int tx_ticks; 1838cc180b69SBen Hutchings unsigned int rx_ticks; 1839874aeea5SJeff Kirsher 1840874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1841874aeea5SJeff Kirsher 1842cc180b69SBen Hutchings if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max) 18439e393b30SBen Hutchings return -EINVAL; 18449e393b30SBen Hutchings 1845cc180b69SBen Hutchings tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns); 1846cc180b69SBen Hutchings rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns); 1847cc180b69SBen Hutchings 18489e393b30SBen Hutchings if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 && 18499e393b30SBen Hutchings !rx_may_override_tx) { 18509e393b30SBen Hutchings netif_err(efx, drv, efx->net_dev, "Channels are shared. " 18519e393b30SBen Hutchings "RX and TX IRQ moderation must be equal\n"); 18529e393b30SBen Hutchings return -EINVAL; 18539e393b30SBen Hutchings } 18549e393b30SBen Hutchings 1855874aeea5SJeff Kirsher efx->irq_rx_adaptive = rx_adaptive; 1856874aeea5SJeff Kirsher efx->irq_rx_moderation = rx_ticks; 1857874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 1858874aeea5SJeff Kirsher if (efx_channel_has_rx_queue(channel)) 1859874aeea5SJeff Kirsher channel->irq_moderation = rx_ticks; 1860874aeea5SJeff Kirsher else if (efx_channel_has_tx_queues(channel)) 1861874aeea5SJeff Kirsher channel->irq_moderation = tx_ticks; 1862874aeea5SJeff Kirsher } 18639e393b30SBen Hutchings 18649e393b30SBen Hutchings return 0; 1865874aeea5SJeff Kirsher } 1866874aeea5SJeff Kirsher 1867a0c4faf5SBen Hutchings void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 1868a0c4faf5SBen Hutchings unsigned int *rx_usecs, bool *rx_adaptive) 1869a0c4faf5SBen Hutchings { 1870cc180b69SBen Hutchings /* We must round up when converting ticks to microseconds 1871cc180b69SBen Hutchings * because we round down when converting the other way. 1872cc180b69SBen Hutchings */ 1873cc180b69SBen Hutchings 1874a0c4faf5SBen Hutchings *rx_adaptive = efx->irq_rx_adaptive; 1875cc180b69SBen Hutchings *rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation * 1876cc180b69SBen Hutchings efx->timer_quantum_ns, 1877cc180b69SBen Hutchings 1000); 1878a0c4faf5SBen Hutchings 1879a0c4faf5SBen Hutchings /* If channels are shared between RX and TX, so is IRQ 1880a0c4faf5SBen Hutchings * moderation. Otherwise, IRQ moderation is the same for all 1881a0c4faf5SBen Hutchings * TX channels and is not adaptive. 1882a0c4faf5SBen Hutchings */ 1883a0c4faf5SBen Hutchings if (efx->tx_channel_offset == 0) 1884a0c4faf5SBen Hutchings *tx_usecs = *rx_usecs; 1885a0c4faf5SBen Hutchings else 1886cc180b69SBen Hutchings *tx_usecs = DIV_ROUND_UP( 1887a0c4faf5SBen Hutchings efx->channel[efx->tx_channel_offset]->irq_moderation * 1888cc180b69SBen Hutchings efx->timer_quantum_ns, 1889cc180b69SBen Hutchings 1000); 1890a0c4faf5SBen Hutchings } 1891a0c4faf5SBen Hutchings 1892874aeea5SJeff Kirsher /************************************************************************** 1893874aeea5SJeff Kirsher * 1894874aeea5SJeff Kirsher * Hardware monitor 1895874aeea5SJeff Kirsher * 1896874aeea5SJeff Kirsher **************************************************************************/ 1897874aeea5SJeff Kirsher 1898874aeea5SJeff Kirsher /* Run periodically off the general workqueue */ 1899874aeea5SJeff Kirsher static void efx_monitor(struct work_struct *data) 1900874aeea5SJeff Kirsher { 1901874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, 1902874aeea5SJeff Kirsher monitor_work.work); 1903874aeea5SJeff Kirsher 1904874aeea5SJeff Kirsher netif_vdbg(efx, timer, efx->net_dev, 1905874aeea5SJeff Kirsher "hardware monitor executing on CPU %d\n", 1906874aeea5SJeff Kirsher raw_smp_processor_id()); 1907874aeea5SJeff Kirsher BUG_ON(efx->type->monitor == NULL); 1908874aeea5SJeff Kirsher 1909874aeea5SJeff Kirsher /* If the mac_lock is already held then it is likely a port 1910874aeea5SJeff Kirsher * reconfiguration is already in place, which will likely do 1911874aeea5SJeff Kirsher * most of the work of monitor() anyway. */ 1912874aeea5SJeff Kirsher if (mutex_trylock(&efx->mac_lock)) { 1913874aeea5SJeff Kirsher if (efx->port_enabled) 1914874aeea5SJeff Kirsher efx->type->monitor(efx); 1915874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1916874aeea5SJeff Kirsher } 1917874aeea5SJeff Kirsher 1918874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 1919874aeea5SJeff Kirsher efx_monitor_interval); 1920874aeea5SJeff Kirsher } 1921874aeea5SJeff Kirsher 1922874aeea5SJeff Kirsher /************************************************************************** 1923874aeea5SJeff Kirsher * 1924874aeea5SJeff Kirsher * ioctls 1925874aeea5SJeff Kirsher * 1926874aeea5SJeff Kirsher *************************************************************************/ 1927874aeea5SJeff Kirsher 1928874aeea5SJeff Kirsher /* Net device ioctl 1929874aeea5SJeff Kirsher * Context: process, rtnl_lock() held. 1930874aeea5SJeff Kirsher */ 1931874aeea5SJeff Kirsher static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 1932874aeea5SJeff Kirsher { 1933874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 1934874aeea5SJeff Kirsher struct mii_ioctl_data *data = if_mii(ifr); 1935874aeea5SJeff Kirsher 19367c236c43SStuart Hodgson if (cmd == SIOCSHWTSTAMP) 1937433dc9b3SBen Hutchings return efx_ptp_set_ts_config(efx, ifr); 1938433dc9b3SBen Hutchings if (cmd == SIOCGHWTSTAMP) 1939433dc9b3SBen Hutchings return efx_ptp_get_ts_config(efx, ifr); 19407c236c43SStuart Hodgson 1941874aeea5SJeff Kirsher /* Convert phy_id from older PRTAD/DEVAD format */ 1942874aeea5SJeff Kirsher if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 1943874aeea5SJeff Kirsher (data->phy_id & 0xfc00) == 0x0400) 1944874aeea5SJeff Kirsher data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 1945874aeea5SJeff Kirsher 1946874aeea5SJeff Kirsher return mdio_mii_ioctl(&efx->mdio, data, cmd); 1947874aeea5SJeff Kirsher } 1948874aeea5SJeff Kirsher 1949874aeea5SJeff Kirsher /************************************************************************** 1950874aeea5SJeff Kirsher * 1951874aeea5SJeff Kirsher * NAPI interface 1952874aeea5SJeff Kirsher * 1953874aeea5SJeff Kirsher **************************************************************************/ 1954874aeea5SJeff Kirsher 19557f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel) 1956874aeea5SJeff Kirsher { 19577f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 1958874aeea5SJeff Kirsher 1959874aeea5SJeff Kirsher channel->napi_dev = efx->net_dev; 1960874aeea5SJeff Kirsher netif_napi_add(channel->napi_dev, &channel->napi_str, 1961874aeea5SJeff Kirsher efx_poll, napi_weight); 1962874aeea5SJeff Kirsher } 19637f967c01SBen Hutchings 19647f967c01SBen Hutchings static void efx_init_napi(struct efx_nic *efx) 19657f967c01SBen Hutchings { 19667f967c01SBen Hutchings struct efx_channel *channel; 19677f967c01SBen Hutchings 19687f967c01SBen Hutchings efx_for_each_channel(channel, efx) 19697f967c01SBen Hutchings efx_init_napi_channel(channel); 1970874aeea5SJeff Kirsher } 1971874aeea5SJeff Kirsher 1972874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel) 1973874aeea5SJeff Kirsher { 1974874aeea5SJeff Kirsher if (channel->napi_dev) 1975874aeea5SJeff Kirsher netif_napi_del(&channel->napi_str); 1976874aeea5SJeff Kirsher channel->napi_dev = NULL; 1977874aeea5SJeff Kirsher } 1978874aeea5SJeff Kirsher 1979874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx) 1980874aeea5SJeff Kirsher { 1981874aeea5SJeff Kirsher struct efx_channel *channel; 1982874aeea5SJeff Kirsher 1983874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1984874aeea5SJeff Kirsher efx_fini_napi_channel(channel); 1985874aeea5SJeff Kirsher } 1986874aeea5SJeff Kirsher 1987874aeea5SJeff Kirsher /************************************************************************** 1988874aeea5SJeff Kirsher * 1989874aeea5SJeff Kirsher * Kernel netpoll interface 1990874aeea5SJeff Kirsher * 1991874aeea5SJeff Kirsher *************************************************************************/ 1992874aeea5SJeff Kirsher 1993874aeea5SJeff Kirsher #ifdef CONFIG_NET_POLL_CONTROLLER 1994874aeea5SJeff Kirsher 1995874aeea5SJeff Kirsher /* Although in the common case interrupts will be disabled, this is not 1996874aeea5SJeff Kirsher * guaranteed. However, all our work happens inside the NAPI callback, 1997874aeea5SJeff Kirsher * so no locking is required. 1998874aeea5SJeff Kirsher */ 1999874aeea5SJeff Kirsher static void efx_netpoll(struct net_device *net_dev) 2000874aeea5SJeff Kirsher { 2001874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2002874aeea5SJeff Kirsher struct efx_channel *channel; 2003874aeea5SJeff Kirsher 2004874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 2005874aeea5SJeff Kirsher efx_schedule_channel(channel); 2006874aeea5SJeff Kirsher } 2007874aeea5SJeff Kirsher 2008874aeea5SJeff Kirsher #endif 2009874aeea5SJeff Kirsher 2010874aeea5SJeff Kirsher /************************************************************************** 2011874aeea5SJeff Kirsher * 2012874aeea5SJeff Kirsher * Kernel net device interface 2013874aeea5SJeff Kirsher * 2014874aeea5SJeff Kirsher *************************************************************************/ 2015874aeea5SJeff Kirsher 2016874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2017874aeea5SJeff Kirsher static int efx_net_open(struct net_device *net_dev) 2018874aeea5SJeff Kirsher { 2019874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 20208b7325b4SBen Hutchings int rc; 20218b7325b4SBen Hutchings 2022874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2023874aeea5SJeff Kirsher raw_smp_processor_id()); 2024874aeea5SJeff Kirsher 20258b7325b4SBen Hutchings rc = efx_check_disabled(efx); 20268b7325b4SBen Hutchings if (rc) 20278b7325b4SBen Hutchings return rc; 2028874aeea5SJeff Kirsher if (efx->phy_mode & PHY_MODE_SPECIAL) 2029874aeea5SJeff Kirsher return -EBUSY; 2030874aeea5SJeff Kirsher if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2031874aeea5SJeff Kirsher return -EIO; 2032874aeea5SJeff Kirsher 2033874aeea5SJeff Kirsher /* Notify the kernel of the link state polled during driver load, 2034874aeea5SJeff Kirsher * before the monitor starts running */ 2035874aeea5SJeff Kirsher efx_link_status_changed(efx); 2036874aeea5SJeff Kirsher 2037874aeea5SJeff Kirsher efx_start_all(efx); 2038dd40781eSBen Hutchings efx_selftest_async_start(efx); 2039874aeea5SJeff Kirsher return 0; 2040874aeea5SJeff Kirsher } 2041874aeea5SJeff Kirsher 2042874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. 2043874aeea5SJeff Kirsher * Note that the kernel will ignore our return code; this method 2044874aeea5SJeff Kirsher * should really be a void. 2045874aeea5SJeff Kirsher */ 2046874aeea5SJeff Kirsher static int efx_net_stop(struct net_device *net_dev) 2047874aeea5SJeff Kirsher { 2048874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2049874aeea5SJeff Kirsher 2050874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2051874aeea5SJeff Kirsher raw_smp_processor_id()); 2052874aeea5SJeff Kirsher 2053874aeea5SJeff Kirsher /* Stop the device and flush all the channels */ 2054874aeea5SJeff Kirsher efx_stop_all(efx); 2055874aeea5SJeff Kirsher 2056874aeea5SJeff Kirsher return 0; 2057874aeea5SJeff Kirsher } 2058874aeea5SJeff Kirsher 2059874aeea5SJeff Kirsher /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 20602aa9ef11SBen Hutchings static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev, 20612aa9ef11SBen Hutchings struct rtnl_link_stats64 *stats) 2062874aeea5SJeff Kirsher { 2063874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2064874aeea5SJeff Kirsher 2065874aeea5SJeff Kirsher spin_lock_bh(&efx->stats_lock); 2066cd0ecc9aSBen Hutchings efx->type->update_stats(efx, NULL, stats); 20671cb34522SBen Hutchings spin_unlock_bh(&efx->stats_lock); 20681cb34522SBen Hutchings 2069874aeea5SJeff Kirsher return stats; 2070874aeea5SJeff Kirsher } 2071874aeea5SJeff Kirsher 2072874aeea5SJeff Kirsher /* Context: netif_tx_lock held, BHs disabled. */ 2073874aeea5SJeff Kirsher static void efx_watchdog(struct net_device *net_dev) 2074874aeea5SJeff Kirsher { 2075874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2076874aeea5SJeff Kirsher 2077874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 2078874aeea5SJeff Kirsher "TX stuck with port_enabled=%d: resetting channels\n", 2079874aeea5SJeff Kirsher efx->port_enabled); 2080874aeea5SJeff Kirsher 2081874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2082874aeea5SJeff Kirsher } 2083874aeea5SJeff Kirsher 2084874aeea5SJeff Kirsher 2085874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2086874aeea5SJeff Kirsher static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2087874aeea5SJeff Kirsher { 2088874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 20898b7325b4SBen Hutchings int rc; 2090874aeea5SJeff Kirsher 20918b7325b4SBen Hutchings rc = efx_check_disabled(efx); 20928b7325b4SBen Hutchings if (rc) 20938b7325b4SBen Hutchings return rc; 2094874aeea5SJeff Kirsher if (new_mtu > EFX_MAX_MTU) 2095874aeea5SJeff Kirsher return -EINVAL; 2096874aeea5SJeff Kirsher 2097874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2098874aeea5SJeff Kirsher 209929c69a48SBen Hutchings efx_device_detach_sync(efx); 210029c69a48SBen Hutchings efx_stop_all(efx); 210129c69a48SBen Hutchings 2102874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2103874aeea5SJeff Kirsher net_dev->mtu = new_mtu; 2104710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 2105874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2106874aeea5SJeff Kirsher 2107874aeea5SJeff Kirsher efx_start_all(efx); 210829c69a48SBen Hutchings netif_device_attach(efx->net_dev); 21096c8eef4aSBen Hutchings return 0; 2110874aeea5SJeff Kirsher } 2111874aeea5SJeff Kirsher 2112874aeea5SJeff Kirsher static int efx_set_mac_address(struct net_device *net_dev, void *data) 2113874aeea5SJeff Kirsher { 2114874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2115874aeea5SJeff Kirsher struct sockaddr *addr = data; 2116e0b3ae30SBen Hutchings u8 *new_addr = addr->sa_data; 2117874aeea5SJeff Kirsher 2118874aeea5SJeff Kirsher if (!is_valid_ether_addr(new_addr)) { 2119874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2120874aeea5SJeff Kirsher "invalid ethernet MAC address requested: %pM\n", 2121874aeea5SJeff Kirsher new_addr); 2122504f9b5aSDanny Kukawka return -EADDRNOTAVAIL; 2123874aeea5SJeff Kirsher } 2124874aeea5SJeff Kirsher 2125874aeea5SJeff Kirsher memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len); 2126cd2d5b52SBen Hutchings efx_sriov_mac_address_changed(efx); 2127874aeea5SJeff Kirsher 2128874aeea5SJeff Kirsher /* Reconfigure the MAC */ 2129874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2130710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 2131874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2132874aeea5SJeff Kirsher 2133874aeea5SJeff Kirsher return 0; 2134874aeea5SJeff Kirsher } 2135874aeea5SJeff Kirsher 2136874aeea5SJeff Kirsher /* Context: netif_addr_lock held, BHs disabled. */ 21370fca8c97SBen Hutchings static void efx_set_rx_mode(struct net_device *net_dev) 2138874aeea5SJeff Kirsher { 2139874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2140874aeea5SJeff Kirsher 2141874aeea5SJeff Kirsher if (efx->port_enabled) 2142874aeea5SJeff Kirsher queue_work(efx->workqueue, &efx->mac_work); 2143874aeea5SJeff Kirsher /* Otherwise efx_start_port() will do this */ 2144874aeea5SJeff Kirsher } 2145874aeea5SJeff Kirsher 2146c8f44affSMichał Mirosław static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2147874aeea5SJeff Kirsher { 2148874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2149874aeea5SJeff Kirsher 2150874aeea5SJeff Kirsher /* If disabling RX n-tuple filtering, clear existing filters */ 2151874aeea5SJeff Kirsher if (net_dev->features & ~data & NETIF_F_NTUPLE) 2152fbd79120SBen Hutchings return efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2153874aeea5SJeff Kirsher 2154874aeea5SJeff Kirsher return 0; 2155874aeea5SJeff Kirsher } 2156874aeea5SJeff Kirsher 21578127d661SBen Hutchings static const struct net_device_ops efx_farch_netdev_ops = { 2158874aeea5SJeff Kirsher .ndo_open = efx_net_open, 2159874aeea5SJeff Kirsher .ndo_stop = efx_net_stop, 2160874aeea5SJeff Kirsher .ndo_get_stats64 = efx_net_stats, 2161874aeea5SJeff Kirsher .ndo_tx_timeout = efx_watchdog, 2162874aeea5SJeff Kirsher .ndo_start_xmit = efx_hard_start_xmit, 2163874aeea5SJeff Kirsher .ndo_validate_addr = eth_validate_addr, 2164874aeea5SJeff Kirsher .ndo_do_ioctl = efx_ioctl, 2165874aeea5SJeff Kirsher .ndo_change_mtu = efx_change_mtu, 2166874aeea5SJeff Kirsher .ndo_set_mac_address = efx_set_mac_address, 21670fca8c97SBen Hutchings .ndo_set_rx_mode = efx_set_rx_mode, 2168874aeea5SJeff Kirsher .ndo_set_features = efx_set_features, 2169cd2d5b52SBen Hutchings #ifdef CONFIG_SFC_SRIOV 2170cd2d5b52SBen Hutchings .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2171cd2d5b52SBen Hutchings .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2172cd2d5b52SBen Hutchings .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2173cd2d5b52SBen Hutchings .ndo_get_vf_config = efx_sriov_get_vf_config, 2174cd2d5b52SBen Hutchings #endif 2175874aeea5SJeff Kirsher #ifdef CONFIG_NET_POLL_CONTROLLER 2176874aeea5SJeff Kirsher .ndo_poll_controller = efx_netpoll, 2177874aeea5SJeff Kirsher #endif 2178874aeea5SJeff Kirsher .ndo_setup_tc = efx_setup_tc, 2179874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 2180874aeea5SJeff Kirsher .ndo_rx_flow_steer = efx_filter_rfs, 2181874aeea5SJeff Kirsher #endif 2182874aeea5SJeff Kirsher }; 2183874aeea5SJeff Kirsher 21848127d661SBen Hutchings static const struct net_device_ops efx_ef10_netdev_ops = { 21858127d661SBen Hutchings .ndo_open = efx_net_open, 21868127d661SBen Hutchings .ndo_stop = efx_net_stop, 21878127d661SBen Hutchings .ndo_get_stats64 = efx_net_stats, 21888127d661SBen Hutchings .ndo_tx_timeout = efx_watchdog, 21898127d661SBen Hutchings .ndo_start_xmit = efx_hard_start_xmit, 21908127d661SBen Hutchings .ndo_validate_addr = eth_validate_addr, 21918127d661SBen Hutchings .ndo_do_ioctl = efx_ioctl, 21928127d661SBen Hutchings .ndo_change_mtu = efx_change_mtu, 21938127d661SBen Hutchings .ndo_set_mac_address = efx_set_mac_address, 21948127d661SBen Hutchings .ndo_set_rx_mode = efx_set_rx_mode, 21958127d661SBen Hutchings .ndo_set_features = efx_set_features, 21968127d661SBen Hutchings #ifdef CONFIG_NET_POLL_CONTROLLER 21978127d661SBen Hutchings .ndo_poll_controller = efx_netpoll, 21988127d661SBen Hutchings #endif 21998127d661SBen Hutchings #ifdef CONFIG_RFS_ACCEL 22008127d661SBen Hutchings .ndo_rx_flow_steer = efx_filter_rfs, 22018127d661SBen Hutchings #endif 22028127d661SBen Hutchings }; 22038127d661SBen Hutchings 2204874aeea5SJeff Kirsher static void efx_update_name(struct efx_nic *efx) 2205874aeea5SJeff Kirsher { 2206874aeea5SJeff Kirsher strcpy(efx->name, efx->net_dev->name); 2207874aeea5SJeff Kirsher efx_mtd_rename(efx); 2208874aeea5SJeff Kirsher efx_set_channel_names(efx); 2209874aeea5SJeff Kirsher } 2210874aeea5SJeff Kirsher 2211874aeea5SJeff Kirsher static int efx_netdev_event(struct notifier_block *this, 2212874aeea5SJeff Kirsher unsigned long event, void *ptr) 2213874aeea5SJeff Kirsher { 2214351638e7SJiri Pirko struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2215874aeea5SJeff Kirsher 22168127d661SBen Hutchings if ((net_dev->netdev_ops == &efx_farch_netdev_ops || 22178127d661SBen Hutchings net_dev->netdev_ops == &efx_ef10_netdev_ops) && 2218874aeea5SJeff Kirsher event == NETDEV_CHANGENAME) 2219874aeea5SJeff Kirsher efx_update_name(netdev_priv(net_dev)); 2220874aeea5SJeff Kirsher 2221874aeea5SJeff Kirsher return NOTIFY_DONE; 2222874aeea5SJeff Kirsher } 2223874aeea5SJeff Kirsher 2224874aeea5SJeff Kirsher static struct notifier_block efx_netdev_notifier = { 2225874aeea5SJeff Kirsher .notifier_call = efx_netdev_event, 2226874aeea5SJeff Kirsher }; 2227874aeea5SJeff Kirsher 2228874aeea5SJeff Kirsher static ssize_t 2229874aeea5SJeff Kirsher show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2230874aeea5SJeff Kirsher { 2231874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2232874aeea5SJeff Kirsher return sprintf(buf, "%d\n", efx->phy_type); 2233874aeea5SJeff Kirsher } 2234776fbcc9SBen Hutchings static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2235874aeea5SJeff Kirsher 2236874aeea5SJeff Kirsher static int efx_register_netdev(struct efx_nic *efx) 2237874aeea5SJeff Kirsher { 2238874aeea5SJeff Kirsher struct net_device *net_dev = efx->net_dev; 2239874aeea5SJeff Kirsher struct efx_channel *channel; 2240874aeea5SJeff Kirsher int rc; 2241874aeea5SJeff Kirsher 2242874aeea5SJeff Kirsher net_dev->watchdog_timeo = 5 * HZ; 2243874aeea5SJeff Kirsher net_dev->irq = efx->pci_dev->irq; 22448127d661SBen Hutchings if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) { 22458127d661SBen Hutchings net_dev->netdev_ops = &efx_ef10_netdev_ops; 22468127d661SBen Hutchings net_dev->priv_flags |= IFF_UNICAST_FLT; 22478127d661SBen Hutchings } else { 22488127d661SBen Hutchings net_dev->netdev_ops = &efx_farch_netdev_ops; 22498127d661SBen Hutchings } 2250874aeea5SJeff Kirsher SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops); 22517e6d06f0SBen Hutchings net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2252874aeea5SJeff Kirsher 2253874aeea5SJeff Kirsher rtnl_lock(); 2254874aeea5SJeff Kirsher 22557153f623SBen Hutchings /* Enable resets to be scheduled and check whether any were 22567153f623SBen Hutchings * already requested. If so, the NIC is probably hosed so we 22577153f623SBen Hutchings * abort. 22587153f623SBen Hutchings */ 22597153f623SBen Hutchings efx->state = STATE_READY; 22607153f623SBen Hutchings smp_mb(); /* ensure we change state before checking reset_pending */ 22617153f623SBen Hutchings if (efx->reset_pending) { 22627153f623SBen Hutchings netif_err(efx, probe, efx->net_dev, 22637153f623SBen Hutchings "aborting probe due to scheduled reset\n"); 22647153f623SBen Hutchings rc = -EIO; 22657153f623SBen Hutchings goto fail_locked; 22667153f623SBen Hutchings } 22677153f623SBen Hutchings 2268874aeea5SJeff Kirsher rc = dev_alloc_name(net_dev, net_dev->name); 2269874aeea5SJeff Kirsher if (rc < 0) 2270874aeea5SJeff Kirsher goto fail_locked; 2271874aeea5SJeff Kirsher efx_update_name(efx); 2272874aeea5SJeff Kirsher 22738f8b3d51SBen Hutchings /* Always start with carrier off; PHY events will detect the link */ 22748f8b3d51SBen Hutchings netif_carrier_off(net_dev); 22758f8b3d51SBen Hutchings 2276874aeea5SJeff Kirsher rc = register_netdevice(net_dev); 2277874aeea5SJeff Kirsher if (rc) 2278874aeea5SJeff Kirsher goto fail_locked; 2279874aeea5SJeff Kirsher 2280874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2281874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 2282874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 2283874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 2284874aeea5SJeff Kirsher } 2285874aeea5SJeff Kirsher 22860bcf4a64SBen Hutchings efx_associate(efx); 22870bcf4a64SBen Hutchings 2288874aeea5SJeff Kirsher rtnl_unlock(); 2289874aeea5SJeff Kirsher 2290874aeea5SJeff Kirsher rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2291874aeea5SJeff Kirsher if (rc) { 2292874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2293874aeea5SJeff Kirsher "failed to init net dev attributes\n"); 2294874aeea5SJeff Kirsher goto fail_registered; 2295874aeea5SJeff Kirsher } 2296874aeea5SJeff Kirsher 2297874aeea5SJeff Kirsher return 0; 2298874aeea5SJeff Kirsher 22997153f623SBen Hutchings fail_registered: 23007153f623SBen Hutchings rtnl_lock(); 23010bcf4a64SBen Hutchings efx_dissociate(efx); 23027153f623SBen Hutchings unregister_netdevice(net_dev); 2303874aeea5SJeff Kirsher fail_locked: 23047153f623SBen Hutchings efx->state = STATE_UNINIT; 2305874aeea5SJeff Kirsher rtnl_unlock(); 2306874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2307874aeea5SJeff Kirsher return rc; 2308874aeea5SJeff Kirsher } 2309874aeea5SJeff Kirsher 2310874aeea5SJeff Kirsher static void efx_unregister_netdev(struct efx_nic *efx) 2311874aeea5SJeff Kirsher { 2312874aeea5SJeff Kirsher if (!efx->net_dev) 2313874aeea5SJeff Kirsher return; 2314874aeea5SJeff Kirsher 2315874aeea5SJeff Kirsher BUG_ON(netdev_priv(efx->net_dev) != efx); 2316874aeea5SJeff Kirsher 2317874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2318874aeea5SJeff Kirsher device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 23197153f623SBen Hutchings 23207153f623SBen Hutchings rtnl_lock(); 23217153f623SBen Hutchings unregister_netdevice(efx->net_dev); 23227153f623SBen Hutchings efx->state = STATE_UNINIT; 23237153f623SBen Hutchings rtnl_unlock(); 2324874aeea5SJeff Kirsher } 2325874aeea5SJeff Kirsher 2326874aeea5SJeff Kirsher /************************************************************************** 2327874aeea5SJeff Kirsher * 2328874aeea5SJeff Kirsher * Device reset and suspend 2329874aeea5SJeff Kirsher * 2330874aeea5SJeff Kirsher **************************************************************************/ 2331874aeea5SJeff Kirsher 2332874aeea5SJeff Kirsher /* Tears down the entire software state and most of the hardware state 2333874aeea5SJeff Kirsher * before reset. */ 2334874aeea5SJeff Kirsher void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2335874aeea5SJeff Kirsher { 2336874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2337874aeea5SJeff Kirsher 2338874aeea5SJeff Kirsher efx_stop_all(efx); 2339d8291187SBen Hutchings efx_disable_interrupts(efx); 23405642ceefSBen Hutchings 23415642ceefSBen Hutchings mutex_lock(&efx->mac_lock); 2342874aeea5SJeff Kirsher if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) 2343874aeea5SJeff Kirsher efx->phy_op->fini(efx); 2344874aeea5SJeff Kirsher efx->type->fini(efx); 2345874aeea5SJeff Kirsher } 2346874aeea5SJeff Kirsher 2347874aeea5SJeff Kirsher /* This function will always ensure that the locks acquired in 2348874aeea5SJeff Kirsher * efx_reset_down() are released. A failure return code indicates 2349874aeea5SJeff Kirsher * that we were unable to reinitialise the hardware, and the 2350874aeea5SJeff Kirsher * driver should be disabled. If ok is false, then the rx and tx 2351874aeea5SJeff Kirsher * engines are not restarted, pending a RESET_DISABLE. */ 2352874aeea5SJeff Kirsher int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2353874aeea5SJeff Kirsher { 2354874aeea5SJeff Kirsher int rc; 2355874aeea5SJeff Kirsher 2356874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2357874aeea5SJeff Kirsher 2358874aeea5SJeff Kirsher rc = efx->type->init(efx); 2359874aeea5SJeff Kirsher if (rc) { 2360874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2361874aeea5SJeff Kirsher goto fail; 2362874aeea5SJeff Kirsher } 2363874aeea5SJeff Kirsher 2364874aeea5SJeff Kirsher if (!ok) 2365874aeea5SJeff Kirsher goto fail; 2366874aeea5SJeff Kirsher 2367874aeea5SJeff Kirsher if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) { 2368874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 2369874aeea5SJeff Kirsher if (rc) 2370874aeea5SJeff Kirsher goto fail; 2371874aeea5SJeff Kirsher if (efx->phy_op->reconfigure(efx)) 2372874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2373874aeea5SJeff Kirsher "could not restore PHY settings\n"); 2374874aeea5SJeff Kirsher } 2375874aeea5SJeff Kirsher 2376261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 2377261e4d96SJon Cooper if (rc) 2378261e4d96SJon Cooper goto fail; 2379874aeea5SJeff Kirsher efx_restore_filters(efx); 2380cd2d5b52SBen Hutchings efx_sriov_reset(efx); 2381874aeea5SJeff Kirsher 2382874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2383874aeea5SJeff Kirsher 2384874aeea5SJeff Kirsher efx_start_all(efx); 2385874aeea5SJeff Kirsher 2386874aeea5SJeff Kirsher return 0; 2387874aeea5SJeff Kirsher 2388874aeea5SJeff Kirsher fail: 2389874aeea5SJeff Kirsher efx->port_initialized = false; 2390874aeea5SJeff Kirsher 2391874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2392874aeea5SJeff Kirsher 2393874aeea5SJeff Kirsher return rc; 2394874aeea5SJeff Kirsher } 2395874aeea5SJeff Kirsher 2396874aeea5SJeff Kirsher /* Reset the NIC using the specified method. Note that the reset may 2397874aeea5SJeff Kirsher * fail, in which case the card will be left in an unusable state. 2398874aeea5SJeff Kirsher * 2399874aeea5SJeff Kirsher * Caller must hold the rtnl_lock. 2400874aeea5SJeff Kirsher */ 2401874aeea5SJeff Kirsher int efx_reset(struct efx_nic *efx, enum reset_type method) 2402874aeea5SJeff Kirsher { 2403874aeea5SJeff Kirsher int rc, rc2; 2404874aeea5SJeff Kirsher bool disabled; 2405874aeea5SJeff Kirsher 2406874aeea5SJeff Kirsher netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2407874aeea5SJeff Kirsher RESET_TYPE(method)); 2408874aeea5SJeff Kirsher 2409c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2410874aeea5SJeff Kirsher efx_reset_down(efx, method); 2411874aeea5SJeff Kirsher 2412874aeea5SJeff Kirsher rc = efx->type->reset(efx, method); 2413874aeea5SJeff Kirsher if (rc) { 2414874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2415874aeea5SJeff Kirsher goto out; 2416874aeea5SJeff Kirsher } 2417874aeea5SJeff Kirsher 2418874aeea5SJeff Kirsher /* Clear flags for the scopes we covered. We assume the NIC and 2419874aeea5SJeff Kirsher * driver are now quiescent so that there is no race here. 2420874aeea5SJeff Kirsher */ 2421874aeea5SJeff Kirsher efx->reset_pending &= -(1 << (method + 1)); 2422874aeea5SJeff Kirsher 2423874aeea5SJeff Kirsher /* Reinitialise bus-mastering, which may have been turned off before 2424874aeea5SJeff Kirsher * the reset was scheduled. This is still appropriate, even in the 2425874aeea5SJeff Kirsher * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2426874aeea5SJeff Kirsher * can respond to requests. */ 2427874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 2428874aeea5SJeff Kirsher 2429874aeea5SJeff Kirsher out: 2430874aeea5SJeff Kirsher /* Leave device stopped if necessary */ 2431626950dbSAlexandre Rames disabled = rc || 2432626950dbSAlexandre Rames method == RESET_TYPE_DISABLE || 2433626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_DISABLE; 2434874aeea5SJeff Kirsher rc2 = efx_reset_up(efx, method, !disabled); 2435874aeea5SJeff Kirsher if (rc2) { 2436874aeea5SJeff Kirsher disabled = true; 2437874aeea5SJeff Kirsher if (!rc) 2438874aeea5SJeff Kirsher rc = rc2; 2439874aeea5SJeff Kirsher } 2440874aeea5SJeff Kirsher 2441874aeea5SJeff Kirsher if (disabled) { 2442874aeea5SJeff Kirsher dev_close(efx->net_dev); 2443874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2444874aeea5SJeff Kirsher efx->state = STATE_DISABLED; 2445874aeea5SJeff Kirsher } else { 2446874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2447874aeea5SJeff Kirsher netif_device_attach(efx->net_dev); 2448874aeea5SJeff Kirsher } 2449874aeea5SJeff Kirsher return rc; 2450874aeea5SJeff Kirsher } 2451874aeea5SJeff Kirsher 2452626950dbSAlexandre Rames /* Try recovery mechanisms. 2453626950dbSAlexandre Rames * For now only EEH is supported. 2454626950dbSAlexandre Rames * Returns 0 if the recovery mechanisms are unsuccessful. 2455626950dbSAlexandre Rames * Returns a non-zero value otherwise. 2456626950dbSAlexandre Rames */ 2457b28405b0SAlexandre Rames int efx_try_recovery(struct efx_nic *efx) 2458626950dbSAlexandre Rames { 2459626950dbSAlexandre Rames #ifdef CONFIG_EEH 2460626950dbSAlexandre Rames /* A PCI error can occur and not be seen by EEH because nothing 2461626950dbSAlexandre Rames * happens on the PCI bus. In this case the driver may fail and 2462626950dbSAlexandre Rames * schedule a 'recover or reset', leading to this recovery handler. 2463626950dbSAlexandre Rames * Manually call the eeh failure check function. 2464626950dbSAlexandre Rames */ 2465626950dbSAlexandre Rames struct eeh_dev *eehdev = 2466626950dbSAlexandre Rames of_node_to_eeh_dev(pci_device_to_OF_node(efx->pci_dev)); 2467626950dbSAlexandre Rames 2468626950dbSAlexandre Rames if (eeh_dev_check_failure(eehdev)) { 2469626950dbSAlexandre Rames /* The EEH mechanisms will handle the error and reset the 2470626950dbSAlexandre Rames * device if necessary. 2471626950dbSAlexandre Rames */ 2472626950dbSAlexandre Rames return 1; 2473626950dbSAlexandre Rames } 2474626950dbSAlexandre Rames #endif 2475626950dbSAlexandre Rames return 0; 2476626950dbSAlexandre Rames } 2477626950dbSAlexandre Rames 247874cd60a4SJon Cooper static void efx_wait_for_bist_end(struct efx_nic *efx) 247974cd60a4SJon Cooper { 248074cd60a4SJon Cooper int i; 248174cd60a4SJon Cooper 248274cd60a4SJon Cooper for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 248374cd60a4SJon Cooper if (efx_mcdi_poll_reboot(efx)) 248474cd60a4SJon Cooper goto out; 248574cd60a4SJon Cooper msleep(BIST_WAIT_DELAY_MS); 248674cd60a4SJon Cooper } 248774cd60a4SJon Cooper 248874cd60a4SJon Cooper netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 248974cd60a4SJon Cooper out: 249074cd60a4SJon Cooper /* Either way unset the BIST flag. If we found no reboot we probably 249174cd60a4SJon Cooper * won't recover, but we should try. 249274cd60a4SJon Cooper */ 249374cd60a4SJon Cooper efx->mc_bist_for_other_fn = false; 249474cd60a4SJon Cooper } 249574cd60a4SJon Cooper 2496874aeea5SJeff Kirsher /* The worker thread exists so that code that cannot sleep can 2497874aeea5SJeff Kirsher * schedule a reset for later. 2498874aeea5SJeff Kirsher */ 2499874aeea5SJeff Kirsher static void efx_reset_work(struct work_struct *data) 2500874aeea5SJeff Kirsher { 2501874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2502626950dbSAlexandre Rames unsigned long pending; 2503626950dbSAlexandre Rames enum reset_type method; 2504626950dbSAlexandre Rames 2505626950dbSAlexandre Rames pending = ACCESS_ONCE(efx->reset_pending); 2506626950dbSAlexandre Rames method = fls(pending) - 1; 2507626950dbSAlexandre Rames 250874cd60a4SJon Cooper if (method == RESET_TYPE_MC_BIST) 250974cd60a4SJon Cooper efx_wait_for_bist_end(efx); 251074cd60a4SJon Cooper 2511626950dbSAlexandre Rames if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2512626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_ALL) && 2513626950dbSAlexandre Rames efx_try_recovery(efx)) 2514626950dbSAlexandre Rames return; 2515874aeea5SJeff Kirsher 2516874aeea5SJeff Kirsher if (!pending) 2517874aeea5SJeff Kirsher return; 2518874aeea5SJeff Kirsher 2519874aeea5SJeff Kirsher rtnl_lock(); 25207153f623SBen Hutchings 25217153f623SBen Hutchings /* We checked the state in efx_schedule_reset() but it may 25227153f623SBen Hutchings * have changed by now. Now that we have the RTNL lock, 25237153f623SBen Hutchings * it cannot change again. 25247153f623SBen Hutchings */ 25257153f623SBen Hutchings if (efx->state == STATE_READY) 2526626950dbSAlexandre Rames (void)efx_reset(efx, method); 25277153f623SBen Hutchings 2528874aeea5SJeff Kirsher rtnl_unlock(); 2529874aeea5SJeff Kirsher } 2530874aeea5SJeff Kirsher 2531874aeea5SJeff Kirsher void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2532874aeea5SJeff Kirsher { 2533874aeea5SJeff Kirsher enum reset_type method; 2534874aeea5SJeff Kirsher 2535626950dbSAlexandre Rames if (efx->state == STATE_RECOVERY) { 2536626950dbSAlexandre Rames netif_dbg(efx, drv, efx->net_dev, 2537626950dbSAlexandre Rames "recovering: skip scheduling %s reset\n", 2538626950dbSAlexandre Rames RESET_TYPE(type)); 2539626950dbSAlexandre Rames return; 2540626950dbSAlexandre Rames } 2541626950dbSAlexandre Rames 2542874aeea5SJeff Kirsher switch (type) { 2543874aeea5SJeff Kirsher case RESET_TYPE_INVISIBLE: 2544874aeea5SJeff Kirsher case RESET_TYPE_ALL: 2545626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_ALL: 2546874aeea5SJeff Kirsher case RESET_TYPE_WORLD: 2547874aeea5SJeff Kirsher case RESET_TYPE_DISABLE: 2548626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_DISABLE: 254974cd60a4SJon Cooper case RESET_TYPE_MC_BIST: 2550874aeea5SJeff Kirsher method = type; 2551874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2552874aeea5SJeff Kirsher RESET_TYPE(method)); 2553874aeea5SJeff Kirsher break; 2554874aeea5SJeff Kirsher default: 2555874aeea5SJeff Kirsher method = efx->type->map_reset_reason(type); 2556874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, 2557874aeea5SJeff Kirsher "scheduling %s reset for %s\n", 2558874aeea5SJeff Kirsher RESET_TYPE(method), RESET_TYPE(type)); 2559874aeea5SJeff Kirsher break; 2560874aeea5SJeff Kirsher } 2561874aeea5SJeff Kirsher 2562874aeea5SJeff Kirsher set_bit(method, &efx->reset_pending); 25637153f623SBen Hutchings smp_mb(); /* ensure we change reset_pending before checking state */ 25647153f623SBen Hutchings 25657153f623SBen Hutchings /* If we're not READY then just leave the flags set as the cue 25667153f623SBen Hutchings * to abort probing or reschedule the reset later. 25677153f623SBen Hutchings */ 25687153f623SBen Hutchings if (ACCESS_ONCE(efx->state) != STATE_READY) 25697153f623SBen Hutchings return; 2570874aeea5SJeff Kirsher 2571874aeea5SJeff Kirsher /* efx_process_channel() will no longer read events once a 2572874aeea5SJeff Kirsher * reset is scheduled. So switch back to poll'd MCDI completions. */ 2573874aeea5SJeff Kirsher efx_mcdi_mode_poll(efx); 2574874aeea5SJeff Kirsher 2575874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 2576874aeea5SJeff Kirsher } 2577874aeea5SJeff Kirsher 2578874aeea5SJeff Kirsher /************************************************************************** 2579874aeea5SJeff Kirsher * 2580874aeea5SJeff Kirsher * List of NICs we support 2581874aeea5SJeff Kirsher * 2582874aeea5SJeff Kirsher **************************************************************************/ 2583874aeea5SJeff Kirsher 2584874aeea5SJeff Kirsher /* PCI device ID table */ 2585874aeea5SJeff Kirsher static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = { 25860e59e7e7SLinus Torvalds {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 25870e59e7e7SLinus Torvalds PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0), 2588874aeea5SJeff Kirsher .driver_data = (unsigned long) &falcon_a1_nic_type}, 25890e59e7e7SLinus Torvalds {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 25900e59e7e7SLinus Torvalds PCI_DEVICE_ID_SOLARFLARE_SFC4000B), 2591874aeea5SJeff Kirsher .driver_data = (unsigned long) &falcon_b0_nic_type}, 2592547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2593874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 2594547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2595874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 25968127d661SBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 25978127d661SBen Hutchings .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2598874aeea5SJeff Kirsher {0} /* end of list */ 2599874aeea5SJeff Kirsher }; 2600874aeea5SJeff Kirsher 2601874aeea5SJeff Kirsher /************************************************************************** 2602874aeea5SJeff Kirsher * 2603874aeea5SJeff Kirsher * Dummy PHY/MAC operations 2604874aeea5SJeff Kirsher * 2605874aeea5SJeff Kirsher * Can be used for some unimplemented operations 2606874aeea5SJeff Kirsher * Needed so all function pointers are valid and do not have to be tested 2607874aeea5SJeff Kirsher * before use 2608874aeea5SJeff Kirsher * 2609874aeea5SJeff Kirsher **************************************************************************/ 2610874aeea5SJeff Kirsher int efx_port_dummy_op_int(struct efx_nic *efx) 2611874aeea5SJeff Kirsher { 2612874aeea5SJeff Kirsher return 0; 2613874aeea5SJeff Kirsher } 2614874aeea5SJeff Kirsher void efx_port_dummy_op_void(struct efx_nic *efx) {} 2615874aeea5SJeff Kirsher 2616874aeea5SJeff Kirsher static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2617874aeea5SJeff Kirsher { 2618874aeea5SJeff Kirsher return false; 2619874aeea5SJeff Kirsher } 2620874aeea5SJeff Kirsher 2621874aeea5SJeff Kirsher static const struct efx_phy_operations efx_dummy_phy_operations = { 2622874aeea5SJeff Kirsher .init = efx_port_dummy_op_int, 2623874aeea5SJeff Kirsher .reconfigure = efx_port_dummy_op_int, 2624874aeea5SJeff Kirsher .poll = efx_port_dummy_op_poll, 2625874aeea5SJeff Kirsher .fini = efx_port_dummy_op_void, 2626874aeea5SJeff Kirsher }; 2627874aeea5SJeff Kirsher 2628874aeea5SJeff Kirsher /************************************************************************** 2629874aeea5SJeff Kirsher * 2630874aeea5SJeff Kirsher * Data housekeeping 2631874aeea5SJeff Kirsher * 2632874aeea5SJeff Kirsher **************************************************************************/ 2633874aeea5SJeff Kirsher 2634874aeea5SJeff Kirsher /* This zeroes out and then fills in the invariants in a struct 2635874aeea5SJeff Kirsher * efx_nic (including all sub-structures). 2636874aeea5SJeff Kirsher */ 2637adeb15aaSBen Hutchings static int efx_init_struct(struct efx_nic *efx, 2638874aeea5SJeff Kirsher struct pci_dev *pci_dev, struct net_device *net_dev) 2639874aeea5SJeff Kirsher { 2640874aeea5SJeff Kirsher int i; 2641874aeea5SJeff Kirsher 2642874aeea5SJeff Kirsher /* Initialise common structures */ 26430bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->node); 26440bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->secondary_list); 2645874aeea5SJeff Kirsher spin_lock_init(&efx->biu_lock); 2646874aeea5SJeff Kirsher #ifdef CONFIG_SFC_MTD 2647874aeea5SJeff Kirsher INIT_LIST_HEAD(&efx->mtd_list); 2648874aeea5SJeff Kirsher #endif 2649874aeea5SJeff Kirsher INIT_WORK(&efx->reset_work, efx_reset_work); 2650874aeea5SJeff Kirsher INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 2651dd40781eSBen Hutchings INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 2652874aeea5SJeff Kirsher efx->pci_dev = pci_dev; 2653874aeea5SJeff Kirsher efx->msg_enable = debug; 2654f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 2655874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2656874aeea5SJeff Kirsher 2657874aeea5SJeff Kirsher efx->net_dev = net_dev; 265843a3739dSJon Cooper efx->rx_prefix_size = efx->type->rx_prefix_size; 26592ec03014SAndrew Rybchenko efx->rx_ip_align = 26602ec03014SAndrew Rybchenko NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 266143a3739dSJon Cooper efx->rx_packet_hash_offset = 266243a3739dSJon Cooper efx->type->rx_hash_offset - efx->type->rx_prefix_size; 2663bd9a265dSJon Cooper efx->rx_packet_ts_offset = 2664bd9a265dSJon Cooper efx->type->rx_ts_offset - efx->type->rx_prefix_size; 2665874aeea5SJeff Kirsher spin_lock_init(&efx->stats_lock); 2666874aeea5SJeff Kirsher mutex_init(&efx->mac_lock); 2667874aeea5SJeff Kirsher efx->phy_op = &efx_dummy_phy_operations; 2668874aeea5SJeff Kirsher efx->mdio.dev = net_dev; 2669874aeea5SJeff Kirsher INIT_WORK(&efx->mac_work, efx_mac_work); 26709f2cb71cSBen Hutchings init_waitqueue_head(&efx->flush_wq); 2671874aeea5SJeff Kirsher 2672874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) { 2673874aeea5SJeff Kirsher efx->channel[i] = efx_alloc_channel(efx, i, NULL); 2674874aeea5SJeff Kirsher if (!efx->channel[i]) 2675874aeea5SJeff Kirsher goto fail; 2676d8291187SBen Hutchings efx->msi_context[i].efx = efx; 2677d8291187SBen Hutchings efx->msi_context[i].index = i; 2678874aeea5SJeff Kirsher } 2679874aeea5SJeff Kirsher 2680874aeea5SJeff Kirsher /* Higher numbered interrupt modes are less capable! */ 2681874aeea5SJeff Kirsher efx->interrupt_mode = max(efx->type->max_interrupt_mode, 2682874aeea5SJeff Kirsher interrupt_mode); 2683874aeea5SJeff Kirsher 2684874aeea5SJeff Kirsher /* Would be good to use the net_dev name, but we're too early */ 2685874aeea5SJeff Kirsher snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 2686874aeea5SJeff Kirsher pci_name(pci_dev)); 2687874aeea5SJeff Kirsher efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 2688874aeea5SJeff Kirsher if (!efx->workqueue) 2689874aeea5SJeff Kirsher goto fail; 2690874aeea5SJeff Kirsher 2691874aeea5SJeff Kirsher return 0; 2692874aeea5SJeff Kirsher 2693874aeea5SJeff Kirsher fail: 2694874aeea5SJeff Kirsher efx_fini_struct(efx); 2695874aeea5SJeff Kirsher return -ENOMEM; 2696874aeea5SJeff Kirsher } 2697874aeea5SJeff Kirsher 2698874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx) 2699874aeea5SJeff Kirsher { 2700874aeea5SJeff Kirsher int i; 2701874aeea5SJeff Kirsher 2702874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) 2703874aeea5SJeff Kirsher kfree(efx->channel[i]); 2704874aeea5SJeff Kirsher 2705ef215e64SBen Hutchings kfree(efx->vpd_sn); 2706ef215e64SBen Hutchings 2707874aeea5SJeff Kirsher if (efx->workqueue) { 2708874aeea5SJeff Kirsher destroy_workqueue(efx->workqueue); 2709874aeea5SJeff Kirsher efx->workqueue = NULL; 2710874aeea5SJeff Kirsher } 2711874aeea5SJeff Kirsher } 2712874aeea5SJeff Kirsher 2713874aeea5SJeff Kirsher /************************************************************************** 2714874aeea5SJeff Kirsher * 2715874aeea5SJeff Kirsher * PCI interface 2716874aeea5SJeff Kirsher * 2717874aeea5SJeff Kirsher **************************************************************************/ 2718874aeea5SJeff Kirsher 2719874aeea5SJeff Kirsher /* Main body of final NIC shutdown code 2720874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 2721874aeea5SJeff Kirsher */ 2722874aeea5SJeff Kirsher static void efx_pci_remove_main(struct efx_nic *efx) 2723874aeea5SJeff Kirsher { 27247153f623SBen Hutchings /* Flush reset_work. It can no longer be scheduled since we 27257153f623SBen Hutchings * are not READY. 27267153f623SBen Hutchings */ 27277153f623SBen Hutchings BUG_ON(efx->state == STATE_READY); 27287153f623SBen Hutchings cancel_work_sync(&efx->reset_work); 27297153f623SBen Hutchings 2730d8291187SBen Hutchings efx_disable_interrupts(efx); 2731874aeea5SJeff Kirsher efx_nic_fini_interrupt(efx); 2732874aeea5SJeff Kirsher efx_fini_port(efx); 2733874aeea5SJeff Kirsher efx->type->fini(efx); 2734874aeea5SJeff Kirsher efx_fini_napi(efx); 2735874aeea5SJeff Kirsher efx_remove_all(efx); 2736874aeea5SJeff Kirsher } 2737874aeea5SJeff Kirsher 2738874aeea5SJeff Kirsher /* Final NIC shutdown 2739874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 2740874aeea5SJeff Kirsher */ 2741874aeea5SJeff Kirsher static void efx_pci_remove(struct pci_dev *pci_dev) 2742874aeea5SJeff Kirsher { 2743874aeea5SJeff Kirsher struct efx_nic *efx; 2744874aeea5SJeff Kirsher 2745874aeea5SJeff Kirsher efx = pci_get_drvdata(pci_dev); 2746874aeea5SJeff Kirsher if (!efx) 2747874aeea5SJeff Kirsher return; 2748874aeea5SJeff Kirsher 2749874aeea5SJeff Kirsher /* Mark the NIC as fini, then stop the interface */ 2750874aeea5SJeff Kirsher rtnl_lock(); 27510bcf4a64SBen Hutchings efx_dissociate(efx); 2752874aeea5SJeff Kirsher dev_close(efx->net_dev); 2753d8291187SBen Hutchings efx_disable_interrupts(efx); 2754874aeea5SJeff Kirsher rtnl_unlock(); 2755874aeea5SJeff Kirsher 2756cd2d5b52SBen Hutchings efx_sriov_fini(efx); 2757874aeea5SJeff Kirsher efx_unregister_netdev(efx); 2758874aeea5SJeff Kirsher 2759874aeea5SJeff Kirsher efx_mtd_remove(efx); 2760874aeea5SJeff Kirsher 2761874aeea5SJeff Kirsher efx_pci_remove_main(efx); 2762874aeea5SJeff Kirsher 2763874aeea5SJeff Kirsher efx_fini_io(efx); 2764874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 2765874aeea5SJeff Kirsher 2766874aeea5SJeff Kirsher efx_fini_struct(efx); 2767874aeea5SJeff Kirsher free_netdev(efx->net_dev); 2768626950dbSAlexandre Rames 2769626950dbSAlexandre Rames pci_disable_pcie_error_reporting(pci_dev); 2770874aeea5SJeff Kirsher }; 2771874aeea5SJeff Kirsher 2772460eeaa0SBen Hutchings /* NIC VPD information 2773460eeaa0SBen Hutchings * Called during probe to display the part number of the 2774460eeaa0SBen Hutchings * installed NIC. VPD is potentially very large but this should 2775460eeaa0SBen Hutchings * always appear within the first 512 bytes. 2776460eeaa0SBen Hutchings */ 2777460eeaa0SBen Hutchings #define SFC_VPD_LEN 512 2778ef215e64SBen Hutchings static void efx_probe_vpd_strings(struct efx_nic *efx) 2779460eeaa0SBen Hutchings { 2780460eeaa0SBen Hutchings struct pci_dev *dev = efx->pci_dev; 2781460eeaa0SBen Hutchings char vpd_data[SFC_VPD_LEN]; 2782460eeaa0SBen Hutchings ssize_t vpd_size; 2783ef215e64SBen Hutchings int ro_start, ro_size, i, j; 2784460eeaa0SBen Hutchings 2785460eeaa0SBen Hutchings /* Get the vpd data from the device */ 2786460eeaa0SBen Hutchings vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 2787460eeaa0SBen Hutchings if (vpd_size <= 0) { 2788460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 2789460eeaa0SBen Hutchings return; 2790460eeaa0SBen Hutchings } 2791460eeaa0SBen Hutchings 2792460eeaa0SBen Hutchings /* Get the Read only section */ 2793ef215e64SBen Hutchings ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 2794ef215e64SBen Hutchings if (ro_start < 0) { 2795460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 2796460eeaa0SBen Hutchings return; 2797460eeaa0SBen Hutchings } 2798460eeaa0SBen Hutchings 2799ef215e64SBen Hutchings ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 2800ef215e64SBen Hutchings j = ro_size; 2801ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 2802460eeaa0SBen Hutchings if (i + j > vpd_size) 2803460eeaa0SBen Hutchings j = vpd_size - i; 2804460eeaa0SBen Hutchings 2805460eeaa0SBen Hutchings /* Get the Part number */ 2806460eeaa0SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 2807460eeaa0SBen Hutchings if (i < 0) { 2808460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 2809460eeaa0SBen Hutchings return; 2810460eeaa0SBen Hutchings } 2811460eeaa0SBen Hutchings 2812460eeaa0SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 2813460eeaa0SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 2814460eeaa0SBen Hutchings if (i + j > vpd_size) { 2815460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 2816460eeaa0SBen Hutchings return; 2817460eeaa0SBen Hutchings } 2818460eeaa0SBen Hutchings 2819460eeaa0SBen Hutchings netif_info(efx, drv, efx->net_dev, 2820460eeaa0SBen Hutchings "Part Number : %.*s\n", j, &vpd_data[i]); 2821ef215e64SBen Hutchings 2822ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 2823ef215e64SBen Hutchings j = ro_size; 2824ef215e64SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 2825ef215e64SBen Hutchings if (i < 0) { 2826ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 2827ef215e64SBen Hutchings return; 2828ef215e64SBen Hutchings } 2829ef215e64SBen Hutchings 2830ef215e64SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 2831ef215e64SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 2832ef215e64SBen Hutchings if (i + j > vpd_size) { 2833ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 2834ef215e64SBen Hutchings return; 2835ef215e64SBen Hutchings } 2836ef215e64SBen Hutchings 2837ef215e64SBen Hutchings efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 2838ef215e64SBen Hutchings if (!efx->vpd_sn) 2839ef215e64SBen Hutchings return; 2840ef215e64SBen Hutchings 2841ef215e64SBen Hutchings snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 2842460eeaa0SBen Hutchings } 2843460eeaa0SBen Hutchings 2844460eeaa0SBen Hutchings 2845874aeea5SJeff Kirsher /* Main body of NIC initialisation 2846874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, theoretically). 2847874aeea5SJeff Kirsher */ 2848874aeea5SJeff Kirsher static int efx_pci_probe_main(struct efx_nic *efx) 2849874aeea5SJeff Kirsher { 2850874aeea5SJeff Kirsher int rc; 2851874aeea5SJeff Kirsher 2852874aeea5SJeff Kirsher /* Do start-of-day initialisation */ 2853874aeea5SJeff Kirsher rc = efx_probe_all(efx); 2854874aeea5SJeff Kirsher if (rc) 2855874aeea5SJeff Kirsher goto fail1; 2856874aeea5SJeff Kirsher 2857874aeea5SJeff Kirsher efx_init_napi(efx); 2858874aeea5SJeff Kirsher 2859874aeea5SJeff Kirsher rc = efx->type->init(efx); 2860874aeea5SJeff Kirsher if (rc) { 2861874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 2862874aeea5SJeff Kirsher "failed to initialise NIC\n"); 2863874aeea5SJeff Kirsher goto fail3; 2864874aeea5SJeff Kirsher } 2865874aeea5SJeff Kirsher 2866874aeea5SJeff Kirsher rc = efx_init_port(efx); 2867874aeea5SJeff Kirsher if (rc) { 2868874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 2869874aeea5SJeff Kirsher "failed to initialise port\n"); 2870874aeea5SJeff Kirsher goto fail4; 2871874aeea5SJeff Kirsher } 2872874aeea5SJeff Kirsher 2873874aeea5SJeff Kirsher rc = efx_nic_init_interrupt(efx); 2874874aeea5SJeff Kirsher if (rc) 2875874aeea5SJeff Kirsher goto fail5; 2876261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 2877261e4d96SJon Cooper if (rc) 2878261e4d96SJon Cooper goto fail6; 2879874aeea5SJeff Kirsher 2880874aeea5SJeff Kirsher return 0; 2881874aeea5SJeff Kirsher 2882261e4d96SJon Cooper fail6: 2883261e4d96SJon Cooper efx_nic_fini_interrupt(efx); 2884874aeea5SJeff Kirsher fail5: 2885874aeea5SJeff Kirsher efx_fini_port(efx); 2886874aeea5SJeff Kirsher fail4: 2887874aeea5SJeff Kirsher efx->type->fini(efx); 2888874aeea5SJeff Kirsher fail3: 2889874aeea5SJeff Kirsher efx_fini_napi(efx); 2890874aeea5SJeff Kirsher efx_remove_all(efx); 2891874aeea5SJeff Kirsher fail1: 2892874aeea5SJeff Kirsher return rc; 2893874aeea5SJeff Kirsher } 2894874aeea5SJeff Kirsher 2895874aeea5SJeff Kirsher /* NIC initialisation 2896874aeea5SJeff Kirsher * 2897874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, 289873ba7b68SBen Hutchings * theoretically). It sets up PCI mappings, resets the NIC, 2899874aeea5SJeff Kirsher * sets up and registers the network devices with the kernel and hooks 2900874aeea5SJeff Kirsher * the interrupt service routine. It does not prepare the device for 2901874aeea5SJeff Kirsher * transmission; this is left to the first time one of the network 2902874aeea5SJeff Kirsher * interfaces is brought up (i.e. efx_net_open). 2903874aeea5SJeff Kirsher */ 290487d1fc11SBill Pemberton static int efx_pci_probe(struct pci_dev *pci_dev, 2905874aeea5SJeff Kirsher const struct pci_device_id *entry) 2906874aeea5SJeff Kirsher { 2907874aeea5SJeff Kirsher struct net_device *net_dev; 2908874aeea5SJeff Kirsher struct efx_nic *efx; 2909fadac6aaSBen Hutchings int rc; 2910874aeea5SJeff Kirsher 2911874aeea5SJeff Kirsher /* Allocate and initialise a struct net_device and struct efx_nic */ 2912874aeea5SJeff Kirsher net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 2913874aeea5SJeff Kirsher EFX_MAX_RX_QUEUES); 2914874aeea5SJeff Kirsher if (!net_dev) 2915874aeea5SJeff Kirsher return -ENOMEM; 2916adeb15aaSBen Hutchings efx = netdev_priv(net_dev); 2917adeb15aaSBen Hutchings efx->type = (const struct efx_nic_type *) entry->driver_data; 2918adeb15aaSBen Hutchings net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 2919874aeea5SJeff Kirsher NETIF_F_HIGHDMA | NETIF_F_TSO | 2920874aeea5SJeff Kirsher NETIF_F_RXCSUM); 2921adeb15aaSBen Hutchings if (efx->type->offload_features & NETIF_F_V6_CSUM) 2922874aeea5SJeff Kirsher net_dev->features |= NETIF_F_TSO6; 2923874aeea5SJeff Kirsher /* Mask for features that also apply to VLAN devices */ 2924874aeea5SJeff Kirsher net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG | 2925874aeea5SJeff Kirsher NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 2926874aeea5SJeff Kirsher NETIF_F_RXCSUM); 2927874aeea5SJeff Kirsher /* All offloads can be toggled */ 2928874aeea5SJeff Kirsher net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA; 2929874aeea5SJeff Kirsher pci_set_drvdata(pci_dev, efx); 2930874aeea5SJeff Kirsher SET_NETDEV_DEV(net_dev, &pci_dev->dev); 2931adeb15aaSBen Hutchings rc = efx_init_struct(efx, pci_dev, net_dev); 2932874aeea5SJeff Kirsher if (rc) 2933874aeea5SJeff Kirsher goto fail1; 2934874aeea5SJeff Kirsher 2935874aeea5SJeff Kirsher netif_info(efx, probe, efx->net_dev, 2936874aeea5SJeff Kirsher "Solarflare NIC detected\n"); 2937874aeea5SJeff Kirsher 2938ef215e64SBen Hutchings efx_probe_vpd_strings(efx); 2939460eeaa0SBen Hutchings 2940874aeea5SJeff Kirsher /* Set up basic I/O (BAR mappings etc) */ 2941874aeea5SJeff Kirsher rc = efx_init_io(efx); 2942874aeea5SJeff Kirsher if (rc) 2943874aeea5SJeff Kirsher goto fail2; 2944874aeea5SJeff Kirsher 2945874aeea5SJeff Kirsher rc = efx_pci_probe_main(efx); 2946fadac6aaSBen Hutchings if (rc) 2947874aeea5SJeff Kirsher goto fail3; 2948874aeea5SJeff Kirsher 2949874aeea5SJeff Kirsher rc = efx_register_netdev(efx); 2950874aeea5SJeff Kirsher if (rc) 2951fadac6aaSBen Hutchings goto fail4; 2952874aeea5SJeff Kirsher 2953cd2d5b52SBen Hutchings rc = efx_sriov_init(efx); 2954cd2d5b52SBen Hutchings if (rc) 2955cd2d5b52SBen Hutchings netif_err(efx, probe, efx->net_dev, 2956cd2d5b52SBen Hutchings "SR-IOV can't be enabled rc %d\n", rc); 2957cd2d5b52SBen Hutchings 2958874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 2959874aeea5SJeff Kirsher 29607c43161cSBen Hutchings /* Try to create MTDs, but allow this to fail */ 2961874aeea5SJeff Kirsher rtnl_lock(); 29627c43161cSBen Hutchings rc = efx_mtd_probe(efx); 2963874aeea5SJeff Kirsher rtnl_unlock(); 29647c43161cSBen Hutchings if (rc) 29657c43161cSBen Hutchings netif_warn(efx, probe, efx->net_dev, 29667c43161cSBen Hutchings "failed to create MTDs (%d)\n", rc); 29677c43161cSBen Hutchings 2968626950dbSAlexandre Rames rc = pci_enable_pcie_error_reporting(pci_dev); 2969626950dbSAlexandre Rames if (rc && rc != -EINVAL) 2970626950dbSAlexandre Rames netif_warn(efx, probe, efx->net_dev, 2971626950dbSAlexandre Rames "pci_enable_pcie_error_reporting failed (%d)\n", rc); 2972626950dbSAlexandre Rames 2973874aeea5SJeff Kirsher return 0; 2974874aeea5SJeff Kirsher 2975874aeea5SJeff Kirsher fail4: 2976fadac6aaSBen Hutchings efx_pci_remove_main(efx); 2977874aeea5SJeff Kirsher fail3: 2978874aeea5SJeff Kirsher efx_fini_io(efx); 2979874aeea5SJeff Kirsher fail2: 2980874aeea5SJeff Kirsher efx_fini_struct(efx); 2981874aeea5SJeff Kirsher fail1: 2982874aeea5SJeff Kirsher WARN_ON(rc > 0); 2983874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 2984874aeea5SJeff Kirsher free_netdev(net_dev); 2985874aeea5SJeff Kirsher return rc; 2986874aeea5SJeff Kirsher } 2987874aeea5SJeff Kirsher 2988874aeea5SJeff Kirsher static int efx_pm_freeze(struct device *dev) 2989874aeea5SJeff Kirsher { 2990874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2991874aeea5SJeff Kirsher 299261da026dSBen Hutchings rtnl_lock(); 299361da026dSBen Hutchings 29946032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 2995f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 2996874aeea5SJeff Kirsher 2997c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2998874aeea5SJeff Kirsher 2999874aeea5SJeff Kirsher efx_stop_all(efx); 3000d8291187SBen Hutchings efx_disable_interrupts(efx); 30016032fb56SBen Hutchings } 3002874aeea5SJeff Kirsher 300361da026dSBen Hutchings rtnl_unlock(); 300461da026dSBen Hutchings 3005874aeea5SJeff Kirsher return 0; 3006874aeea5SJeff Kirsher } 3007874aeea5SJeff Kirsher 3008874aeea5SJeff Kirsher static int efx_pm_thaw(struct device *dev) 3009874aeea5SJeff Kirsher { 3010261e4d96SJon Cooper int rc; 3011874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3012874aeea5SJeff Kirsher 301361da026dSBen Hutchings rtnl_lock(); 301461da026dSBen Hutchings 30156032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3016261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3017261e4d96SJon Cooper if (rc) 3018261e4d96SJon Cooper goto fail; 3019874aeea5SJeff Kirsher 3020874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 3021874aeea5SJeff Kirsher efx->phy_op->reconfigure(efx); 3022874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 3023874aeea5SJeff Kirsher 3024874aeea5SJeff Kirsher efx_start_all(efx); 3025874aeea5SJeff Kirsher 3026874aeea5SJeff Kirsher netif_device_attach(efx->net_dev); 3027874aeea5SJeff Kirsher 3028f16aeea0SBen Hutchings efx->state = STATE_READY; 3029874aeea5SJeff Kirsher 3030874aeea5SJeff Kirsher efx->type->resume_wol(efx); 30316032fb56SBen Hutchings } 3032874aeea5SJeff Kirsher 303361da026dSBen Hutchings rtnl_unlock(); 303461da026dSBen Hutchings 3035874aeea5SJeff Kirsher /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3036874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 3037874aeea5SJeff Kirsher 3038874aeea5SJeff Kirsher return 0; 3039261e4d96SJon Cooper 3040261e4d96SJon Cooper fail: 3041261e4d96SJon Cooper rtnl_unlock(); 3042261e4d96SJon Cooper 3043261e4d96SJon Cooper return rc; 3044874aeea5SJeff Kirsher } 3045874aeea5SJeff Kirsher 3046874aeea5SJeff Kirsher static int efx_pm_poweroff(struct device *dev) 3047874aeea5SJeff Kirsher { 3048874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3049874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3050874aeea5SJeff Kirsher 3051874aeea5SJeff Kirsher efx->type->fini(efx); 3052874aeea5SJeff Kirsher 3053874aeea5SJeff Kirsher efx->reset_pending = 0; 3054874aeea5SJeff Kirsher 3055874aeea5SJeff Kirsher pci_save_state(pci_dev); 3056874aeea5SJeff Kirsher return pci_set_power_state(pci_dev, PCI_D3hot); 3057874aeea5SJeff Kirsher } 3058874aeea5SJeff Kirsher 3059874aeea5SJeff Kirsher /* Used for both resume and restore */ 3060874aeea5SJeff Kirsher static int efx_pm_resume(struct device *dev) 3061874aeea5SJeff Kirsher { 3062874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3063874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3064874aeea5SJeff Kirsher int rc; 3065874aeea5SJeff Kirsher 3066874aeea5SJeff Kirsher rc = pci_set_power_state(pci_dev, PCI_D0); 3067874aeea5SJeff Kirsher if (rc) 3068874aeea5SJeff Kirsher return rc; 3069874aeea5SJeff Kirsher pci_restore_state(pci_dev); 3070874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 3071874aeea5SJeff Kirsher if (rc) 3072874aeea5SJeff Kirsher return rc; 3073874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 3074874aeea5SJeff Kirsher rc = efx->type->reset(efx, RESET_TYPE_ALL); 3075874aeea5SJeff Kirsher if (rc) 3076874aeea5SJeff Kirsher return rc; 3077874aeea5SJeff Kirsher rc = efx->type->init(efx); 3078874aeea5SJeff Kirsher if (rc) 3079874aeea5SJeff Kirsher return rc; 3080261e4d96SJon Cooper rc = efx_pm_thaw(dev); 3081261e4d96SJon Cooper return rc; 3082874aeea5SJeff Kirsher } 3083874aeea5SJeff Kirsher 3084874aeea5SJeff Kirsher static int efx_pm_suspend(struct device *dev) 3085874aeea5SJeff Kirsher { 3086874aeea5SJeff Kirsher int rc; 3087874aeea5SJeff Kirsher 3088874aeea5SJeff Kirsher efx_pm_freeze(dev); 3089874aeea5SJeff Kirsher rc = efx_pm_poweroff(dev); 3090874aeea5SJeff Kirsher if (rc) 3091874aeea5SJeff Kirsher efx_pm_resume(dev); 3092874aeea5SJeff Kirsher return rc; 3093874aeea5SJeff Kirsher } 3094874aeea5SJeff Kirsher 309518e83e4cSBen Hutchings static const struct dev_pm_ops efx_pm_ops = { 3096874aeea5SJeff Kirsher .suspend = efx_pm_suspend, 3097874aeea5SJeff Kirsher .resume = efx_pm_resume, 3098874aeea5SJeff Kirsher .freeze = efx_pm_freeze, 3099874aeea5SJeff Kirsher .thaw = efx_pm_thaw, 3100874aeea5SJeff Kirsher .poweroff = efx_pm_poweroff, 3101874aeea5SJeff Kirsher .restore = efx_pm_resume, 3102874aeea5SJeff Kirsher }; 3103874aeea5SJeff Kirsher 3104626950dbSAlexandre Rames /* A PCI error affecting this device was detected. 3105626950dbSAlexandre Rames * At this point MMIO and DMA may be disabled. 3106626950dbSAlexandre Rames * Stop the software path and request a slot reset. 3107626950dbSAlexandre Rames */ 3108debd0034Sstephen hemminger static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3109626950dbSAlexandre Rames enum pci_channel_state state) 3110626950dbSAlexandre Rames { 3111626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3112626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3113626950dbSAlexandre Rames 3114626950dbSAlexandre Rames if (state == pci_channel_io_perm_failure) 3115626950dbSAlexandre Rames return PCI_ERS_RESULT_DISCONNECT; 3116626950dbSAlexandre Rames 3117626950dbSAlexandre Rames rtnl_lock(); 3118626950dbSAlexandre Rames 3119626950dbSAlexandre Rames if (efx->state != STATE_DISABLED) { 3120626950dbSAlexandre Rames efx->state = STATE_RECOVERY; 3121626950dbSAlexandre Rames efx->reset_pending = 0; 3122626950dbSAlexandre Rames 3123626950dbSAlexandre Rames efx_device_detach_sync(efx); 3124626950dbSAlexandre Rames 3125626950dbSAlexandre Rames efx_stop_all(efx); 3126d8291187SBen Hutchings efx_disable_interrupts(efx); 3127626950dbSAlexandre Rames 3128626950dbSAlexandre Rames status = PCI_ERS_RESULT_NEED_RESET; 3129626950dbSAlexandre Rames } else { 3130626950dbSAlexandre Rames /* If the interface is disabled we don't want to do anything 3131626950dbSAlexandre Rames * with it. 3132626950dbSAlexandre Rames */ 3133626950dbSAlexandre Rames status = PCI_ERS_RESULT_RECOVERED; 3134626950dbSAlexandre Rames } 3135626950dbSAlexandre Rames 3136626950dbSAlexandre Rames rtnl_unlock(); 3137626950dbSAlexandre Rames 3138626950dbSAlexandre Rames pci_disable_device(pdev); 3139626950dbSAlexandre Rames 3140626950dbSAlexandre Rames return status; 3141626950dbSAlexandre Rames } 3142626950dbSAlexandre Rames 3143626950dbSAlexandre Rames /* Fake a successfull reset, which will be performed later in efx_io_resume. */ 3144debd0034Sstephen hemminger static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3145626950dbSAlexandre Rames { 3146626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3147626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3148626950dbSAlexandre Rames int rc; 3149626950dbSAlexandre Rames 3150626950dbSAlexandre Rames if (pci_enable_device(pdev)) { 3151626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3152626950dbSAlexandre Rames "Cannot re-enable PCI device after reset.\n"); 3153626950dbSAlexandre Rames status = PCI_ERS_RESULT_DISCONNECT; 3154626950dbSAlexandre Rames } 3155626950dbSAlexandre Rames 3156626950dbSAlexandre Rames rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3157626950dbSAlexandre Rames if (rc) { 3158626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3159626950dbSAlexandre Rames "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3160626950dbSAlexandre Rames /* Non-fatal error. Continue. */ 3161626950dbSAlexandre Rames } 3162626950dbSAlexandre Rames 3163626950dbSAlexandre Rames return status; 3164626950dbSAlexandre Rames } 3165626950dbSAlexandre Rames 3166626950dbSAlexandre Rames /* Perform the actual reset and resume I/O operations. */ 3167626950dbSAlexandre Rames static void efx_io_resume(struct pci_dev *pdev) 3168626950dbSAlexandre Rames { 3169626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3170626950dbSAlexandre Rames int rc; 3171626950dbSAlexandre Rames 3172626950dbSAlexandre Rames rtnl_lock(); 3173626950dbSAlexandre Rames 3174626950dbSAlexandre Rames if (efx->state == STATE_DISABLED) 3175626950dbSAlexandre Rames goto out; 3176626950dbSAlexandre Rames 3177626950dbSAlexandre Rames rc = efx_reset(efx, RESET_TYPE_ALL); 3178626950dbSAlexandre Rames if (rc) { 3179626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3180626950dbSAlexandre Rames "efx_reset failed after PCI error (%d)\n", rc); 3181626950dbSAlexandre Rames } else { 3182626950dbSAlexandre Rames efx->state = STATE_READY; 3183626950dbSAlexandre Rames netif_dbg(efx, hw, efx->net_dev, 3184626950dbSAlexandre Rames "Done resetting and resuming IO after PCI error.\n"); 3185626950dbSAlexandre Rames } 3186626950dbSAlexandre Rames 3187626950dbSAlexandre Rames out: 3188626950dbSAlexandre Rames rtnl_unlock(); 3189626950dbSAlexandre Rames } 3190626950dbSAlexandre Rames 3191626950dbSAlexandre Rames /* For simplicity and reliability, we always require a slot reset and try to 3192626950dbSAlexandre Rames * reset the hardware when a pci error affecting the device is detected. 3193626950dbSAlexandre Rames * We leave both the link_reset and mmio_enabled callback unimplemented: 3194626950dbSAlexandre Rames * with our request for slot reset the mmio_enabled callback will never be 3195626950dbSAlexandre Rames * called, and the link_reset callback is not used by AER or EEH mechanisms. 3196626950dbSAlexandre Rames */ 3197626950dbSAlexandre Rames static struct pci_error_handlers efx_err_handlers = { 3198626950dbSAlexandre Rames .error_detected = efx_io_error_detected, 3199626950dbSAlexandre Rames .slot_reset = efx_io_slot_reset, 3200626950dbSAlexandre Rames .resume = efx_io_resume, 3201626950dbSAlexandre Rames }; 3202626950dbSAlexandre Rames 3203874aeea5SJeff Kirsher static struct pci_driver efx_pci_driver = { 3204874aeea5SJeff Kirsher .name = KBUILD_MODNAME, 3205874aeea5SJeff Kirsher .id_table = efx_pci_table, 3206874aeea5SJeff Kirsher .probe = efx_pci_probe, 3207874aeea5SJeff Kirsher .remove = efx_pci_remove, 3208874aeea5SJeff Kirsher .driver.pm = &efx_pm_ops, 3209626950dbSAlexandre Rames .err_handler = &efx_err_handlers, 3210874aeea5SJeff Kirsher }; 3211874aeea5SJeff Kirsher 3212874aeea5SJeff Kirsher /************************************************************************** 3213874aeea5SJeff Kirsher * 3214874aeea5SJeff Kirsher * Kernel module interface 3215874aeea5SJeff Kirsher * 3216874aeea5SJeff Kirsher *************************************************************************/ 3217874aeea5SJeff Kirsher 3218874aeea5SJeff Kirsher module_param(interrupt_mode, uint, 0444); 3219874aeea5SJeff Kirsher MODULE_PARM_DESC(interrupt_mode, 3220874aeea5SJeff Kirsher "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3221874aeea5SJeff Kirsher 3222874aeea5SJeff Kirsher static int __init efx_init_module(void) 3223874aeea5SJeff Kirsher { 3224874aeea5SJeff Kirsher int rc; 3225874aeea5SJeff Kirsher 3226874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3227874aeea5SJeff Kirsher 3228874aeea5SJeff Kirsher rc = register_netdevice_notifier(&efx_netdev_notifier); 3229874aeea5SJeff Kirsher if (rc) 3230874aeea5SJeff Kirsher goto err_notifier; 3231874aeea5SJeff Kirsher 3232cd2d5b52SBen Hutchings rc = efx_init_sriov(); 3233cd2d5b52SBen Hutchings if (rc) 3234cd2d5b52SBen Hutchings goto err_sriov; 3235cd2d5b52SBen Hutchings 3236874aeea5SJeff Kirsher reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3237874aeea5SJeff Kirsher if (!reset_workqueue) { 3238874aeea5SJeff Kirsher rc = -ENOMEM; 3239874aeea5SJeff Kirsher goto err_reset; 3240874aeea5SJeff Kirsher } 3241874aeea5SJeff Kirsher 3242874aeea5SJeff Kirsher rc = pci_register_driver(&efx_pci_driver); 3243874aeea5SJeff Kirsher if (rc < 0) 3244874aeea5SJeff Kirsher goto err_pci; 3245874aeea5SJeff Kirsher 3246874aeea5SJeff Kirsher return 0; 3247874aeea5SJeff Kirsher 3248874aeea5SJeff Kirsher err_pci: 3249874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 3250874aeea5SJeff Kirsher err_reset: 3251cd2d5b52SBen Hutchings efx_fini_sriov(); 3252cd2d5b52SBen Hutchings err_sriov: 3253874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3254874aeea5SJeff Kirsher err_notifier: 3255874aeea5SJeff Kirsher return rc; 3256874aeea5SJeff Kirsher } 3257874aeea5SJeff Kirsher 3258874aeea5SJeff Kirsher static void __exit efx_exit_module(void) 3259874aeea5SJeff Kirsher { 3260874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver unloading\n"); 3261874aeea5SJeff Kirsher 3262874aeea5SJeff Kirsher pci_unregister_driver(&efx_pci_driver); 3263874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 3264cd2d5b52SBen Hutchings efx_fini_sriov(); 3265874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3266874aeea5SJeff Kirsher 3267874aeea5SJeff Kirsher } 3268874aeea5SJeff Kirsher 3269874aeea5SJeff Kirsher module_init(efx_init_module); 3270874aeea5SJeff Kirsher module_exit(efx_exit_module); 3271874aeea5SJeff Kirsher 3272874aeea5SJeff Kirsher MODULE_AUTHOR("Solarflare Communications and " 3273874aeea5SJeff Kirsher "Michael Brown <mbrown@fensystems.co.uk>"); 32746a350fdbSBen Hutchings MODULE_DESCRIPTION("Solarflare network driver"); 3275874aeea5SJeff Kirsher MODULE_LICENSE("GPL"); 3276874aeea5SJeff Kirsher MODULE_DEVICE_TABLE(pci, efx_pci_table); 3277