1874aeea5SJeff Kirsher /**************************************************************************** 2874aeea5SJeff Kirsher * Driver for Solarflare Solarstorm network controllers and boards 3874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 4874aeea5SJeff Kirsher * Copyright 2005-2011 Solarflare Communications Inc. 5874aeea5SJeff Kirsher * 6874aeea5SJeff Kirsher * This program is free software; you can redistribute it and/or modify it 7874aeea5SJeff Kirsher * under the terms of the GNU General Public License version 2 as published 8874aeea5SJeff Kirsher * by the Free Software Foundation, incorporated herein by reference. 9874aeea5SJeff Kirsher */ 10874aeea5SJeff Kirsher 11874aeea5SJeff Kirsher #include <linux/module.h> 12874aeea5SJeff Kirsher #include <linux/pci.h> 13874aeea5SJeff Kirsher #include <linux/netdevice.h> 14874aeea5SJeff Kirsher #include <linux/etherdevice.h> 15874aeea5SJeff Kirsher #include <linux/delay.h> 16874aeea5SJeff Kirsher #include <linux/notifier.h> 17874aeea5SJeff Kirsher #include <linux/ip.h> 18874aeea5SJeff Kirsher #include <linux/tcp.h> 19874aeea5SJeff Kirsher #include <linux/in.h> 20874aeea5SJeff Kirsher #include <linux/crc32.h> 21874aeea5SJeff Kirsher #include <linux/ethtool.h> 22874aeea5SJeff Kirsher #include <linux/topology.h> 23874aeea5SJeff Kirsher #include <linux/gfp.h> 24626950dbSAlexandre Rames #include <linux/pci.h> 25874aeea5SJeff Kirsher #include <linux/cpu_rmap.h> 26626950dbSAlexandre Rames #include <linux/aer.h> 27874aeea5SJeff Kirsher #include "net_driver.h" 28874aeea5SJeff Kirsher #include "efx.h" 29874aeea5SJeff Kirsher #include "nic.h" 30dd40781eSBen Hutchings #include "selftest.h" 31874aeea5SJeff Kirsher 32874aeea5SJeff Kirsher #include "mcdi.h" 33874aeea5SJeff Kirsher #include "workarounds.h" 34874aeea5SJeff Kirsher 35874aeea5SJeff Kirsher /************************************************************************** 36874aeea5SJeff Kirsher * 37874aeea5SJeff Kirsher * Type name strings 38874aeea5SJeff Kirsher * 39874aeea5SJeff Kirsher ************************************************************************** 40874aeea5SJeff Kirsher */ 41874aeea5SJeff Kirsher 42874aeea5SJeff Kirsher /* Loopback mode names (see LOOPBACK_MODE()) */ 43874aeea5SJeff Kirsher const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 4418e83e4cSBen Hutchings const char *const efx_loopback_mode_names[] = { 45874aeea5SJeff Kirsher [LOOPBACK_NONE] = "NONE", 46874aeea5SJeff Kirsher [LOOPBACK_DATA] = "DATAPATH", 47874aeea5SJeff Kirsher [LOOPBACK_GMAC] = "GMAC", 48874aeea5SJeff Kirsher [LOOPBACK_XGMII] = "XGMII", 49874aeea5SJeff Kirsher [LOOPBACK_XGXS] = "XGXS", 50874aeea5SJeff Kirsher [LOOPBACK_XAUI] = "XAUI", 51874aeea5SJeff Kirsher [LOOPBACK_GMII] = "GMII", 52874aeea5SJeff Kirsher [LOOPBACK_SGMII] = "SGMII", 53874aeea5SJeff Kirsher [LOOPBACK_XGBR] = "XGBR", 54874aeea5SJeff Kirsher [LOOPBACK_XFI] = "XFI", 55874aeea5SJeff Kirsher [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 56874aeea5SJeff Kirsher [LOOPBACK_GMII_FAR] = "GMII_FAR", 57874aeea5SJeff Kirsher [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 58874aeea5SJeff Kirsher [LOOPBACK_XFI_FAR] = "XFI_FAR", 59874aeea5SJeff Kirsher [LOOPBACK_GPHY] = "GPHY", 60874aeea5SJeff Kirsher [LOOPBACK_PHYXS] = "PHYXS", 61874aeea5SJeff Kirsher [LOOPBACK_PCS] = "PCS", 62874aeea5SJeff Kirsher [LOOPBACK_PMAPMD] = "PMA/PMD", 63874aeea5SJeff Kirsher [LOOPBACK_XPORT] = "XPORT", 64874aeea5SJeff Kirsher [LOOPBACK_XGMII_WS] = "XGMII_WS", 65874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS] = "XAUI_WS", 66874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 67874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 68874aeea5SJeff Kirsher [LOOPBACK_GMII_WS] = "GMII_WS", 69874aeea5SJeff Kirsher [LOOPBACK_XFI_WS] = "XFI_WS", 70874aeea5SJeff Kirsher [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 71874aeea5SJeff Kirsher [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 72874aeea5SJeff Kirsher }; 73874aeea5SJeff Kirsher 74874aeea5SJeff Kirsher const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 7518e83e4cSBen Hutchings const char *const efx_reset_type_names[] = { 76874aeea5SJeff Kirsher [RESET_TYPE_INVISIBLE] = "INVISIBLE", 77874aeea5SJeff Kirsher [RESET_TYPE_ALL] = "ALL", 78626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 79874aeea5SJeff Kirsher [RESET_TYPE_WORLD] = "WORLD", 80626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 81874aeea5SJeff Kirsher [RESET_TYPE_DISABLE] = "DISABLE", 82874aeea5SJeff Kirsher [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 83874aeea5SJeff Kirsher [RESET_TYPE_INT_ERROR] = "INT_ERROR", 84874aeea5SJeff Kirsher [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY", 85874aeea5SJeff Kirsher [RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH", 86874aeea5SJeff Kirsher [RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH", 87874aeea5SJeff Kirsher [RESET_TYPE_TX_SKIP] = "TX_SKIP", 88874aeea5SJeff Kirsher [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 89874aeea5SJeff Kirsher }; 90874aeea5SJeff Kirsher 91874aeea5SJeff Kirsher /* Reset workqueue. If any NIC has a hardware failure then a reset will be 92874aeea5SJeff Kirsher * queued onto this work queue. This is not a per-nic work queue, because 93874aeea5SJeff Kirsher * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 94874aeea5SJeff Kirsher */ 95874aeea5SJeff Kirsher static struct workqueue_struct *reset_workqueue; 96874aeea5SJeff Kirsher 97874aeea5SJeff Kirsher /************************************************************************** 98874aeea5SJeff Kirsher * 99874aeea5SJeff Kirsher * Configurable values 100874aeea5SJeff Kirsher * 101874aeea5SJeff Kirsher *************************************************************************/ 102874aeea5SJeff Kirsher 103874aeea5SJeff Kirsher /* 104874aeea5SJeff Kirsher * Use separate channels for TX and RX events 105874aeea5SJeff Kirsher * 106874aeea5SJeff Kirsher * Set this to 1 to use separate channels for TX and RX. It allows us 107874aeea5SJeff Kirsher * to control interrupt affinity separately for TX and RX. 108874aeea5SJeff Kirsher * 109874aeea5SJeff Kirsher * This is only used in MSI-X interrupt mode 110874aeea5SJeff Kirsher */ 111b9cc977dSBen Hutchings static bool separate_tx_channels; 112b9cc977dSBen Hutchings module_param(separate_tx_channels, bool, 0444); 113874aeea5SJeff Kirsher MODULE_PARM_DESC(separate_tx_channels, 114874aeea5SJeff Kirsher "Use separate channels for TX and RX"); 115874aeea5SJeff Kirsher 116874aeea5SJeff Kirsher /* This is the weight assigned to each of the (per-channel) virtual 117874aeea5SJeff Kirsher * NAPI devices. 118874aeea5SJeff Kirsher */ 119874aeea5SJeff Kirsher static int napi_weight = 64; 120874aeea5SJeff Kirsher 121874aeea5SJeff Kirsher /* This is the time (in jiffies) between invocations of the hardware 122626950dbSAlexandre Rames * monitor. 123626950dbSAlexandre Rames * On Falcon-based NICs, this will: 124874aeea5SJeff Kirsher * - Check the on-board hardware monitor; 125874aeea5SJeff Kirsher * - Poll the link state and reconfigure the hardware as necessary. 126626950dbSAlexandre Rames * On Siena-based NICs for power systems with EEH support, this will give EEH a 127626950dbSAlexandre Rames * chance to start. 128874aeea5SJeff Kirsher */ 129874aeea5SJeff Kirsher static unsigned int efx_monitor_interval = 1 * HZ; 130874aeea5SJeff Kirsher 131874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 132874aeea5SJeff Kirsher * module load with ethtool. 133874aeea5SJeff Kirsher * 134874aeea5SJeff Kirsher * The default for RX should strike a balance between increasing the 135874aeea5SJeff Kirsher * round-trip latency and reducing overhead. 136874aeea5SJeff Kirsher */ 137874aeea5SJeff Kirsher static unsigned int rx_irq_mod_usec = 60; 138874aeea5SJeff Kirsher 139874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 140874aeea5SJeff Kirsher * module load with ethtool. 141874aeea5SJeff Kirsher * 142874aeea5SJeff Kirsher * This default is chosen to ensure that a 10G link does not go idle 143874aeea5SJeff Kirsher * while a TX queue is stopped after it has become full. A queue is 144874aeea5SJeff Kirsher * restarted when it drops below half full. The time this takes (assuming 145874aeea5SJeff Kirsher * worst case 3 descriptors per packet and 1024 descriptors) is 146874aeea5SJeff Kirsher * 512 / 3 * 1.2 = 205 usec. 147874aeea5SJeff Kirsher */ 148874aeea5SJeff Kirsher static unsigned int tx_irq_mod_usec = 150; 149874aeea5SJeff Kirsher 150874aeea5SJeff Kirsher /* This is the first interrupt mode to try out of: 151874aeea5SJeff Kirsher * 0 => MSI-X 152874aeea5SJeff Kirsher * 1 => MSI 153874aeea5SJeff Kirsher * 2 => legacy 154874aeea5SJeff Kirsher */ 155874aeea5SJeff Kirsher static unsigned int interrupt_mode; 156874aeea5SJeff Kirsher 157874aeea5SJeff Kirsher /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 158874aeea5SJeff Kirsher * i.e. the number of CPUs among which we may distribute simultaneous 159874aeea5SJeff Kirsher * interrupt handling. 160874aeea5SJeff Kirsher * 161874aeea5SJeff Kirsher * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 162cdb08f8fSBen Hutchings * The default (0) means to assign an interrupt to each core. 163874aeea5SJeff Kirsher */ 164874aeea5SJeff Kirsher static unsigned int rss_cpus; 165874aeea5SJeff Kirsher module_param(rss_cpus, uint, 0444); 166874aeea5SJeff Kirsher MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 167874aeea5SJeff Kirsher 168b9cc977dSBen Hutchings static bool phy_flash_cfg; 169b9cc977dSBen Hutchings module_param(phy_flash_cfg, bool, 0644); 170874aeea5SJeff Kirsher MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 171874aeea5SJeff Kirsher 172e7bed9c8SBen Hutchings static unsigned irq_adapt_low_thresh = 8000; 173874aeea5SJeff Kirsher module_param(irq_adapt_low_thresh, uint, 0644); 174874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_low_thresh, 175874aeea5SJeff Kirsher "Threshold score for reducing IRQ moderation"); 176874aeea5SJeff Kirsher 177e7bed9c8SBen Hutchings static unsigned irq_adapt_high_thresh = 16000; 178874aeea5SJeff Kirsher module_param(irq_adapt_high_thresh, uint, 0644); 179874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_high_thresh, 180874aeea5SJeff Kirsher "Threshold score for increasing IRQ moderation"); 181874aeea5SJeff Kirsher 182874aeea5SJeff Kirsher static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 183874aeea5SJeff Kirsher NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 184874aeea5SJeff Kirsher NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 185874aeea5SJeff Kirsher NETIF_MSG_TX_ERR | NETIF_MSG_HW); 186874aeea5SJeff Kirsher module_param(debug, uint, 0); 187874aeea5SJeff Kirsher MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 188874aeea5SJeff Kirsher 189874aeea5SJeff Kirsher /************************************************************************** 190874aeea5SJeff Kirsher * 191874aeea5SJeff Kirsher * Utility functions and prototypes 192874aeea5SJeff Kirsher * 193874aeea5SJeff Kirsher *************************************************************************/ 194874aeea5SJeff Kirsher 1957f967c01SBen Hutchings static void efx_start_interrupts(struct efx_nic *efx, bool may_keep_eventq); 1967f967c01SBen Hutchings static void efx_stop_interrupts(struct efx_nic *efx, bool may_keep_eventq); 1977f967c01SBen Hutchings static void efx_remove_channel(struct efx_channel *channel); 198874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx); 1997f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type; 200874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx); 2017f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel); 202874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx); 203874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel); 204874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx); 205874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx); 206874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx); 207874aeea5SJeff Kirsher 208874aeea5SJeff Kirsher #define EFX_ASSERT_RESET_SERIALISED(efx) \ 209874aeea5SJeff Kirsher do { \ 210f16aeea0SBen Hutchings if ((efx->state == STATE_READY) || \ 211626950dbSAlexandre Rames (efx->state == STATE_RECOVERY) || \ 212874aeea5SJeff Kirsher (efx->state == STATE_DISABLED)) \ 213874aeea5SJeff Kirsher ASSERT_RTNL(); \ 214874aeea5SJeff Kirsher } while (0) 215874aeea5SJeff Kirsher 2168b7325b4SBen Hutchings static int efx_check_disabled(struct efx_nic *efx) 2178b7325b4SBen Hutchings { 218626950dbSAlexandre Rames if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 2198b7325b4SBen Hutchings netif_err(efx, drv, efx->net_dev, 2208b7325b4SBen Hutchings "device is disabled due to earlier errors\n"); 2218b7325b4SBen Hutchings return -EIO; 2228b7325b4SBen Hutchings } 2238b7325b4SBen Hutchings return 0; 2248b7325b4SBen Hutchings } 2258b7325b4SBen Hutchings 226874aeea5SJeff Kirsher /************************************************************************** 227874aeea5SJeff Kirsher * 228874aeea5SJeff Kirsher * Event queue processing 229874aeea5SJeff Kirsher * 230874aeea5SJeff Kirsher *************************************************************************/ 231874aeea5SJeff Kirsher 232874aeea5SJeff Kirsher /* Process channel's event queue 233874aeea5SJeff Kirsher * 234874aeea5SJeff Kirsher * This function is responsible for processing the event queue of a 235874aeea5SJeff Kirsher * single channel. The caller must guarantee that this function will 236874aeea5SJeff Kirsher * never be concurrently called more than once on the same channel, 237874aeea5SJeff Kirsher * though different channels may be being processed concurrently. 238874aeea5SJeff Kirsher */ 239874aeea5SJeff Kirsher static int efx_process_channel(struct efx_channel *channel, int budget) 240874aeea5SJeff Kirsher { 241874aeea5SJeff Kirsher int spent; 242874aeea5SJeff Kirsher 2439f2cb71cSBen Hutchings if (unlikely(!channel->enabled)) 244874aeea5SJeff Kirsher return 0; 245874aeea5SJeff Kirsher 246874aeea5SJeff Kirsher spent = efx_nic_process_eventq(channel, budget); 247d9ab7007SBen Hutchings if (spent && efx_channel_has_rx_queue(channel)) { 248d9ab7007SBen Hutchings struct efx_rx_queue *rx_queue = 249d9ab7007SBen Hutchings efx_channel_get_rx_queue(channel); 250874aeea5SJeff Kirsher 251ff734ef4SBen Hutchings efx_rx_flush_packet(channel); 25297d48a10SAlexandre Rames if (rx_queue->enabled) 253d9ab7007SBen Hutchings efx_fast_push_rx_descriptors(rx_queue); 254d9ab7007SBen Hutchings } 255874aeea5SJeff Kirsher 256874aeea5SJeff Kirsher return spent; 257874aeea5SJeff Kirsher } 258874aeea5SJeff Kirsher 259874aeea5SJeff Kirsher /* Mark channel as finished processing 260874aeea5SJeff Kirsher * 261874aeea5SJeff Kirsher * Note that since we will not receive further interrupts for this 262874aeea5SJeff Kirsher * channel before we finish processing and call the eventq_read_ack() 263874aeea5SJeff Kirsher * method, there is no need to use the interrupt hold-off timers. 264874aeea5SJeff Kirsher */ 265874aeea5SJeff Kirsher static inline void efx_channel_processed(struct efx_channel *channel) 266874aeea5SJeff Kirsher { 267874aeea5SJeff Kirsher /* The interrupt handler for this channel may set work_pending 268874aeea5SJeff Kirsher * as soon as we acknowledge the events we've seen. Make sure 269874aeea5SJeff Kirsher * it's cleared before then. */ 270874aeea5SJeff Kirsher channel->work_pending = false; 271874aeea5SJeff Kirsher smp_wmb(); 272874aeea5SJeff Kirsher 273874aeea5SJeff Kirsher efx_nic_eventq_read_ack(channel); 274874aeea5SJeff Kirsher } 275874aeea5SJeff Kirsher 276874aeea5SJeff Kirsher /* NAPI poll handler 277874aeea5SJeff Kirsher * 278874aeea5SJeff Kirsher * NAPI guarantees serialisation of polls of the same device, which 279874aeea5SJeff Kirsher * provides the guarantee required by efx_process_channel(). 280874aeea5SJeff Kirsher */ 281874aeea5SJeff Kirsher static int efx_poll(struct napi_struct *napi, int budget) 282874aeea5SJeff Kirsher { 283874aeea5SJeff Kirsher struct efx_channel *channel = 284874aeea5SJeff Kirsher container_of(napi, struct efx_channel, napi_str); 285874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 286874aeea5SJeff Kirsher int spent; 287874aeea5SJeff Kirsher 288874aeea5SJeff Kirsher netif_vdbg(efx, intr, efx->net_dev, 289874aeea5SJeff Kirsher "channel %d NAPI poll executing on CPU %d\n", 290874aeea5SJeff Kirsher channel->channel, raw_smp_processor_id()); 291874aeea5SJeff Kirsher 292874aeea5SJeff Kirsher spent = efx_process_channel(channel, budget); 293874aeea5SJeff Kirsher 294874aeea5SJeff Kirsher if (spent < budget) { 2959d9a6973SBen Hutchings if (efx_channel_has_rx_queue(channel) && 296874aeea5SJeff Kirsher efx->irq_rx_adaptive && 297874aeea5SJeff Kirsher unlikely(++channel->irq_count == 1000)) { 298874aeea5SJeff Kirsher if (unlikely(channel->irq_mod_score < 299874aeea5SJeff Kirsher irq_adapt_low_thresh)) { 300874aeea5SJeff Kirsher if (channel->irq_moderation > 1) { 301874aeea5SJeff Kirsher channel->irq_moderation -= 1; 302874aeea5SJeff Kirsher efx->type->push_irq_moderation(channel); 303874aeea5SJeff Kirsher } 304874aeea5SJeff Kirsher } else if (unlikely(channel->irq_mod_score > 305874aeea5SJeff Kirsher irq_adapt_high_thresh)) { 306874aeea5SJeff Kirsher if (channel->irq_moderation < 307874aeea5SJeff Kirsher efx->irq_rx_moderation) { 308874aeea5SJeff Kirsher channel->irq_moderation += 1; 309874aeea5SJeff Kirsher efx->type->push_irq_moderation(channel); 310874aeea5SJeff Kirsher } 311874aeea5SJeff Kirsher } 312874aeea5SJeff Kirsher channel->irq_count = 0; 313874aeea5SJeff Kirsher channel->irq_mod_score = 0; 314874aeea5SJeff Kirsher } 315874aeea5SJeff Kirsher 316874aeea5SJeff Kirsher efx_filter_rfs_expire(channel); 317874aeea5SJeff Kirsher 318874aeea5SJeff Kirsher /* There is no race here; although napi_disable() will 319874aeea5SJeff Kirsher * only wait for napi_complete(), this isn't a problem 320874aeea5SJeff Kirsher * since efx_channel_processed() will have no effect if 321874aeea5SJeff Kirsher * interrupts have already been disabled. 322874aeea5SJeff Kirsher */ 323874aeea5SJeff Kirsher napi_complete(napi); 324874aeea5SJeff Kirsher efx_channel_processed(channel); 325874aeea5SJeff Kirsher } 326874aeea5SJeff Kirsher 327874aeea5SJeff Kirsher return spent; 328874aeea5SJeff Kirsher } 329874aeea5SJeff Kirsher 330874aeea5SJeff Kirsher /* Process the eventq of the specified channel immediately on this CPU 331874aeea5SJeff Kirsher * 332874aeea5SJeff Kirsher * Disable hardware generated interrupts, wait for any existing 333874aeea5SJeff Kirsher * processing to finish, then directly poll (and ack ) the eventq. 334874aeea5SJeff Kirsher * Finally reenable NAPI and interrupts. 335874aeea5SJeff Kirsher * 336874aeea5SJeff Kirsher * This is for use only during a loopback self-test. It must not 337874aeea5SJeff Kirsher * deliver any packets up the stack as this can result in deadlock. 338874aeea5SJeff Kirsher */ 339874aeea5SJeff Kirsher void efx_process_channel_now(struct efx_channel *channel) 340874aeea5SJeff Kirsher { 341874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 342874aeea5SJeff Kirsher 343874aeea5SJeff Kirsher BUG_ON(channel->channel >= efx->n_channels); 344874aeea5SJeff Kirsher BUG_ON(!channel->enabled); 345874aeea5SJeff Kirsher BUG_ON(!efx->loopback_selftest); 346874aeea5SJeff Kirsher 347874aeea5SJeff Kirsher /* Disable interrupts and wait for ISRs to complete */ 348874aeea5SJeff Kirsher efx_nic_disable_interrupts(efx); 349874aeea5SJeff Kirsher if (efx->legacy_irq) { 350874aeea5SJeff Kirsher synchronize_irq(efx->legacy_irq); 351874aeea5SJeff Kirsher efx->legacy_irq_enabled = false; 352874aeea5SJeff Kirsher } 353874aeea5SJeff Kirsher if (channel->irq) 354874aeea5SJeff Kirsher synchronize_irq(channel->irq); 355874aeea5SJeff Kirsher 356874aeea5SJeff Kirsher /* Wait for any NAPI processing to complete */ 357874aeea5SJeff Kirsher napi_disable(&channel->napi_str); 358874aeea5SJeff Kirsher 359874aeea5SJeff Kirsher /* Poll the channel */ 360874aeea5SJeff Kirsher efx_process_channel(channel, channel->eventq_mask + 1); 361874aeea5SJeff Kirsher 362874aeea5SJeff Kirsher /* Ack the eventq. This may cause an interrupt to be generated 363874aeea5SJeff Kirsher * when they are reenabled */ 364874aeea5SJeff Kirsher efx_channel_processed(channel); 365874aeea5SJeff Kirsher 366874aeea5SJeff Kirsher napi_enable(&channel->napi_str); 367874aeea5SJeff Kirsher if (efx->legacy_irq) 368874aeea5SJeff Kirsher efx->legacy_irq_enabled = true; 369874aeea5SJeff Kirsher efx_nic_enable_interrupts(efx); 370874aeea5SJeff Kirsher } 371874aeea5SJeff Kirsher 372874aeea5SJeff Kirsher /* Create event queue 373874aeea5SJeff Kirsher * Event queue memory allocations are done only once. If the channel 374874aeea5SJeff Kirsher * is reset, the memory buffer will be reused; this guards against 375874aeea5SJeff Kirsher * errors during channel reset and also simplifies interrupt handling. 376874aeea5SJeff Kirsher */ 377874aeea5SJeff Kirsher static int efx_probe_eventq(struct efx_channel *channel) 378874aeea5SJeff Kirsher { 379874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 380874aeea5SJeff Kirsher unsigned long entries; 381874aeea5SJeff Kirsher 38286ee5302SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 383874aeea5SJeff Kirsher "chan %d create event queue\n", channel->channel); 384874aeea5SJeff Kirsher 385874aeea5SJeff Kirsher /* Build an event queue with room for one event per tx and rx buffer, 386874aeea5SJeff Kirsher * plus some extra for link state events and MCDI completions. */ 387874aeea5SJeff Kirsher entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 388874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 389874aeea5SJeff Kirsher channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 390874aeea5SJeff Kirsher 391874aeea5SJeff Kirsher return efx_nic_probe_eventq(channel); 392874aeea5SJeff Kirsher } 393874aeea5SJeff Kirsher 394874aeea5SJeff Kirsher /* Prepare channel's event queue */ 395874aeea5SJeff Kirsher static void efx_init_eventq(struct efx_channel *channel) 396874aeea5SJeff Kirsher { 397874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 398874aeea5SJeff Kirsher "chan %d init event queue\n", channel->channel); 399874aeea5SJeff Kirsher 400874aeea5SJeff Kirsher channel->eventq_read_ptr = 0; 401874aeea5SJeff Kirsher 402874aeea5SJeff Kirsher efx_nic_init_eventq(channel); 403874aeea5SJeff Kirsher } 404874aeea5SJeff Kirsher 4059f2cb71cSBen Hutchings /* Enable event queue processing and NAPI */ 4069f2cb71cSBen Hutchings static void efx_start_eventq(struct efx_channel *channel) 4079f2cb71cSBen Hutchings { 4089f2cb71cSBen Hutchings netif_dbg(channel->efx, ifup, channel->efx->net_dev, 4099f2cb71cSBen Hutchings "chan %d start event queue\n", channel->channel); 4109f2cb71cSBen Hutchings 4119f2cb71cSBen Hutchings /* The interrupt handler for this channel may set work_pending 4129f2cb71cSBen Hutchings * as soon as we enable it. Make sure it's cleared before 4139f2cb71cSBen Hutchings * then. Similarly, make sure it sees the enabled flag set. 4149f2cb71cSBen Hutchings */ 4159f2cb71cSBen Hutchings channel->work_pending = false; 4169f2cb71cSBen Hutchings channel->enabled = true; 4179f2cb71cSBen Hutchings smp_wmb(); 4189f2cb71cSBen Hutchings 4199f2cb71cSBen Hutchings napi_enable(&channel->napi_str); 4209f2cb71cSBen Hutchings efx_nic_eventq_read_ack(channel); 4219f2cb71cSBen Hutchings } 4229f2cb71cSBen Hutchings 4239f2cb71cSBen Hutchings /* Disable event queue processing and NAPI */ 4249f2cb71cSBen Hutchings static void efx_stop_eventq(struct efx_channel *channel) 4259f2cb71cSBen Hutchings { 4269f2cb71cSBen Hutchings if (!channel->enabled) 4279f2cb71cSBen Hutchings return; 4289f2cb71cSBen Hutchings 4299f2cb71cSBen Hutchings napi_disable(&channel->napi_str); 4309f2cb71cSBen Hutchings channel->enabled = false; 4319f2cb71cSBen Hutchings } 4329f2cb71cSBen Hutchings 433874aeea5SJeff Kirsher static void efx_fini_eventq(struct efx_channel *channel) 434874aeea5SJeff Kirsher { 435874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 436874aeea5SJeff Kirsher "chan %d fini event queue\n", channel->channel); 437874aeea5SJeff Kirsher 438874aeea5SJeff Kirsher efx_nic_fini_eventq(channel); 439874aeea5SJeff Kirsher } 440874aeea5SJeff Kirsher 441874aeea5SJeff Kirsher static void efx_remove_eventq(struct efx_channel *channel) 442874aeea5SJeff Kirsher { 443874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 444874aeea5SJeff Kirsher "chan %d remove event queue\n", channel->channel); 445874aeea5SJeff Kirsher 446874aeea5SJeff Kirsher efx_nic_remove_eventq(channel); 447874aeea5SJeff Kirsher } 448874aeea5SJeff Kirsher 449874aeea5SJeff Kirsher /************************************************************************** 450874aeea5SJeff Kirsher * 451874aeea5SJeff Kirsher * Channel handling 452874aeea5SJeff Kirsher * 453874aeea5SJeff Kirsher *************************************************************************/ 454874aeea5SJeff Kirsher 4557f967c01SBen Hutchings /* Allocate and initialise a channel structure. */ 456874aeea5SJeff Kirsher static struct efx_channel * 457874aeea5SJeff Kirsher efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 458874aeea5SJeff Kirsher { 459874aeea5SJeff Kirsher struct efx_channel *channel; 460874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 461874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 462874aeea5SJeff Kirsher int j; 463874aeea5SJeff Kirsher 4647f967c01SBen Hutchings channel = kzalloc(sizeof(*channel), GFP_KERNEL); 4657f967c01SBen Hutchings if (!channel) 4667f967c01SBen Hutchings return NULL; 4677f967c01SBen Hutchings 4687f967c01SBen Hutchings channel->efx = efx; 4697f967c01SBen Hutchings channel->channel = i; 4707f967c01SBen Hutchings channel->type = &efx_default_channel_type; 4717f967c01SBen Hutchings 4727f967c01SBen Hutchings for (j = 0; j < EFX_TXQ_TYPES; j++) { 4737f967c01SBen Hutchings tx_queue = &channel->tx_queue[j]; 4747f967c01SBen Hutchings tx_queue->efx = efx; 4757f967c01SBen Hutchings tx_queue->queue = i * EFX_TXQ_TYPES + j; 4767f967c01SBen Hutchings tx_queue->channel = channel; 4777f967c01SBen Hutchings } 4787f967c01SBen Hutchings 4797f967c01SBen Hutchings rx_queue = &channel->rx_queue; 4807f967c01SBen Hutchings rx_queue->efx = efx; 4817f967c01SBen Hutchings setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 4827f967c01SBen Hutchings (unsigned long)rx_queue); 4837f967c01SBen Hutchings 4847f967c01SBen Hutchings return channel; 4857f967c01SBen Hutchings } 4867f967c01SBen Hutchings 4877f967c01SBen Hutchings /* Allocate and initialise a channel structure, copying parameters 4887f967c01SBen Hutchings * (but not resources) from an old channel structure. 4897f967c01SBen Hutchings */ 4907f967c01SBen Hutchings static struct efx_channel * 4917f967c01SBen Hutchings efx_copy_channel(const struct efx_channel *old_channel) 4927f967c01SBen Hutchings { 4937f967c01SBen Hutchings struct efx_channel *channel; 4947f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 4957f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 4967f967c01SBen Hutchings int j; 4977f967c01SBen Hutchings 498874aeea5SJeff Kirsher channel = kmalloc(sizeof(*channel), GFP_KERNEL); 499874aeea5SJeff Kirsher if (!channel) 500874aeea5SJeff Kirsher return NULL; 501874aeea5SJeff Kirsher 502874aeea5SJeff Kirsher *channel = *old_channel; 503874aeea5SJeff Kirsher 504874aeea5SJeff Kirsher channel->napi_dev = NULL; 505874aeea5SJeff Kirsher memset(&channel->eventq, 0, sizeof(channel->eventq)); 506874aeea5SJeff Kirsher 507874aeea5SJeff Kirsher for (j = 0; j < EFX_TXQ_TYPES; j++) { 508874aeea5SJeff Kirsher tx_queue = &channel->tx_queue[j]; 509874aeea5SJeff Kirsher if (tx_queue->channel) 510874aeea5SJeff Kirsher tx_queue->channel = channel; 511874aeea5SJeff Kirsher tx_queue->buffer = NULL; 512874aeea5SJeff Kirsher memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 513874aeea5SJeff Kirsher } 514874aeea5SJeff Kirsher 515874aeea5SJeff Kirsher rx_queue = &channel->rx_queue; 5167f967c01SBen Hutchings rx_queue->buffer = NULL; 5177f967c01SBen Hutchings memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 518874aeea5SJeff Kirsher setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 519874aeea5SJeff Kirsher (unsigned long)rx_queue); 520874aeea5SJeff Kirsher 521874aeea5SJeff Kirsher return channel; 522874aeea5SJeff Kirsher } 523874aeea5SJeff Kirsher 524874aeea5SJeff Kirsher static int efx_probe_channel(struct efx_channel *channel) 525874aeea5SJeff Kirsher { 526874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 527874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 528874aeea5SJeff Kirsher int rc; 529874aeea5SJeff Kirsher 530874aeea5SJeff Kirsher netif_dbg(channel->efx, probe, channel->efx->net_dev, 531874aeea5SJeff Kirsher "creating channel %d\n", channel->channel); 532874aeea5SJeff Kirsher 5337f967c01SBen Hutchings rc = channel->type->pre_probe(channel); 5347f967c01SBen Hutchings if (rc) 5357f967c01SBen Hutchings goto fail; 5367f967c01SBen Hutchings 537874aeea5SJeff Kirsher rc = efx_probe_eventq(channel); 538874aeea5SJeff Kirsher if (rc) 5397f967c01SBen Hutchings goto fail; 540874aeea5SJeff Kirsher 541874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) { 542874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 543874aeea5SJeff Kirsher if (rc) 5447f967c01SBen Hutchings goto fail; 545874aeea5SJeff Kirsher } 546874aeea5SJeff Kirsher 547874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) { 548874aeea5SJeff Kirsher rc = efx_probe_rx_queue(rx_queue); 549874aeea5SJeff Kirsher if (rc) 5507f967c01SBen Hutchings goto fail; 551874aeea5SJeff Kirsher } 552874aeea5SJeff Kirsher 553874aeea5SJeff Kirsher channel->n_rx_frm_trunc = 0; 554874aeea5SJeff Kirsher 555874aeea5SJeff Kirsher return 0; 556874aeea5SJeff Kirsher 5577f967c01SBen Hutchings fail: 5587f967c01SBen Hutchings efx_remove_channel(channel); 559874aeea5SJeff Kirsher return rc; 560874aeea5SJeff Kirsher } 561874aeea5SJeff Kirsher 5627f967c01SBen Hutchings static void 5637f967c01SBen Hutchings efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 5647f967c01SBen Hutchings { 5657f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 5667f967c01SBen Hutchings const char *type; 5677f967c01SBen Hutchings int number; 5687f967c01SBen Hutchings 5697f967c01SBen Hutchings number = channel->channel; 5707f967c01SBen Hutchings if (efx->tx_channel_offset == 0) { 5717f967c01SBen Hutchings type = ""; 5727f967c01SBen Hutchings } else if (channel->channel < efx->tx_channel_offset) { 5737f967c01SBen Hutchings type = "-rx"; 5747f967c01SBen Hutchings } else { 5757f967c01SBen Hutchings type = "-tx"; 5767f967c01SBen Hutchings number -= efx->tx_channel_offset; 5777f967c01SBen Hutchings } 5787f967c01SBen Hutchings snprintf(buf, len, "%s%s-%d", efx->name, type, number); 5797f967c01SBen Hutchings } 580874aeea5SJeff Kirsher 581874aeea5SJeff Kirsher static void efx_set_channel_names(struct efx_nic *efx) 582874aeea5SJeff Kirsher { 583874aeea5SJeff Kirsher struct efx_channel *channel; 584874aeea5SJeff Kirsher 5857f967c01SBen Hutchings efx_for_each_channel(channel, efx) 5867f967c01SBen Hutchings channel->type->get_name(channel, 5877f967c01SBen Hutchings efx->channel_name[channel->channel], 5887f967c01SBen Hutchings sizeof(efx->channel_name[0])); 589874aeea5SJeff Kirsher } 590874aeea5SJeff Kirsher 591874aeea5SJeff Kirsher static int efx_probe_channels(struct efx_nic *efx) 592874aeea5SJeff Kirsher { 593874aeea5SJeff Kirsher struct efx_channel *channel; 594874aeea5SJeff Kirsher int rc; 595874aeea5SJeff Kirsher 596874aeea5SJeff Kirsher /* Restart special buffer allocation */ 597874aeea5SJeff Kirsher efx->next_buffer_table = 0; 598874aeea5SJeff Kirsher 599c92aaff1SBen Hutchings /* Probe channels in reverse, so that any 'extra' channels 600c92aaff1SBen Hutchings * use the start of the buffer table. This allows the traffic 601c92aaff1SBen Hutchings * channels to be resized without moving them or wasting the 602c92aaff1SBen Hutchings * entries before them. 603c92aaff1SBen Hutchings */ 604c92aaff1SBen Hutchings efx_for_each_channel_rev(channel, efx) { 605874aeea5SJeff Kirsher rc = efx_probe_channel(channel); 606874aeea5SJeff Kirsher if (rc) { 607874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 608874aeea5SJeff Kirsher "failed to create channel %d\n", 609874aeea5SJeff Kirsher channel->channel); 610874aeea5SJeff Kirsher goto fail; 611874aeea5SJeff Kirsher } 612874aeea5SJeff Kirsher } 613874aeea5SJeff Kirsher efx_set_channel_names(efx); 614874aeea5SJeff Kirsher 615874aeea5SJeff Kirsher return 0; 616874aeea5SJeff Kirsher 617874aeea5SJeff Kirsher fail: 618874aeea5SJeff Kirsher efx_remove_channels(efx); 619874aeea5SJeff Kirsher return rc; 620874aeea5SJeff Kirsher } 621874aeea5SJeff Kirsher 622874aeea5SJeff Kirsher /* Channels are shutdown and reinitialised whilst the NIC is running 623874aeea5SJeff Kirsher * to propagate configuration changes (mtu, checksum offload), or 624874aeea5SJeff Kirsher * to clear hardware error conditions 625874aeea5SJeff Kirsher */ 6269f2cb71cSBen Hutchings static void efx_start_datapath(struct efx_nic *efx) 627874aeea5SJeff Kirsher { 62885740cdfSBen Hutchings bool old_rx_scatter = efx->rx_scatter; 629874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 630874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 631874aeea5SJeff Kirsher struct efx_channel *channel; 63285740cdfSBen Hutchings size_t rx_buf_len; 633874aeea5SJeff Kirsher 634874aeea5SJeff Kirsher /* Calculate the rx buffer allocation parameters required to 635874aeea5SJeff Kirsher * support the current MTU, including padding for header 636874aeea5SJeff Kirsher * alignment and overruns. 637874aeea5SJeff Kirsher */ 638272baeebSBen Hutchings efx->rx_dma_len = (efx->type->rx_buffer_hash_size + 639874aeea5SJeff Kirsher EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 640874aeea5SJeff Kirsher efx->type->rx_buffer_padding); 64185740cdfSBen Hutchings rx_buf_len = (sizeof(struct efx_rx_page_state) + 642272baeebSBen Hutchings EFX_PAGE_IP_ALIGN + efx->rx_dma_len); 64385740cdfSBen Hutchings if (rx_buf_len <= PAGE_SIZE) { 64485740cdfSBen Hutchings efx->rx_scatter = false; 64585740cdfSBen Hutchings efx->rx_buffer_order = 0; 64685740cdfSBen Hutchings } else if (efx->type->can_rx_scatter) { 64785740cdfSBen Hutchings BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 64885740cdfSBen Hutchings EFX_PAGE_IP_ALIGN + EFX_RX_USR_BUF_SIZE > 64985740cdfSBen Hutchings PAGE_SIZE / 2); 65085740cdfSBen Hutchings efx->rx_scatter = true; 65185740cdfSBen Hutchings efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 65285740cdfSBen Hutchings efx->rx_buffer_order = 0; 65385740cdfSBen Hutchings } else { 65485740cdfSBen Hutchings efx->rx_scatter = false; 65585740cdfSBen Hutchings efx->rx_buffer_order = get_order(rx_buf_len); 65685740cdfSBen Hutchings } 65785740cdfSBen Hutchings 6581648a23fSDaniel Pieczko efx_rx_config_page_split(efx); 6591648a23fSDaniel Pieczko if (efx->rx_buffer_order) 6601648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6611648a23fSDaniel Pieczko "RX buf len=%u; page order=%u batch=%u\n", 6621648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_buffer_order, 6631648a23fSDaniel Pieczko efx->rx_pages_per_batch); 6641648a23fSDaniel Pieczko else 6651648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6661648a23fSDaniel Pieczko "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 6671648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_page_buf_step, 6681648a23fSDaniel Pieczko efx->rx_bufs_per_page, efx->rx_pages_per_batch); 6692768935aSDaniel Pieczko 67085740cdfSBen Hutchings /* RX filters also have scatter-enabled flags */ 67185740cdfSBen Hutchings if (efx->rx_scatter != old_rx_scatter) 67285740cdfSBen Hutchings efx_filter_update_rx_scatter(efx); 673874aeea5SJeff Kirsher 67414bf718fSBen Hutchings /* We must keep at least one descriptor in a TX ring empty. 67514bf718fSBen Hutchings * We could avoid this when the queue size does not exactly 67614bf718fSBen Hutchings * match the hardware ring size, but it's not that important. 67714bf718fSBen Hutchings * Therefore we stop the queue when one more skb might fill 67814bf718fSBen Hutchings * the ring completely. We wake it when half way back to 67914bf718fSBen Hutchings * empty. 68014bf718fSBen Hutchings */ 68114bf718fSBen Hutchings efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 68214bf718fSBen Hutchings efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 68314bf718fSBen Hutchings 684874aeea5SJeff Kirsher /* Initialise the channels */ 685874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 686874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 687874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 688874aeea5SJeff Kirsher 6899f2cb71cSBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) { 690874aeea5SJeff Kirsher efx_init_rx_queue(rx_queue); 6919f2cb71cSBen Hutchings efx_nic_generate_fill_event(rx_queue); 6929f2cb71cSBen Hutchings } 693874aeea5SJeff Kirsher 69485740cdfSBen Hutchings WARN_ON(channel->rx_pkt_n_frags); 695874aeea5SJeff Kirsher } 6969f2cb71cSBen Hutchings 6979f2cb71cSBen Hutchings if (netif_device_present(efx->net_dev)) 6989f2cb71cSBen Hutchings netif_tx_wake_all_queues(efx->net_dev); 699874aeea5SJeff Kirsher } 700874aeea5SJeff Kirsher 7019f2cb71cSBen Hutchings static void efx_stop_datapath(struct efx_nic *efx) 702874aeea5SJeff Kirsher { 703874aeea5SJeff Kirsher struct efx_channel *channel; 704874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 705874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 7063dca9d2dSStuart Hodgson struct pci_dev *dev = efx->pci_dev; 707874aeea5SJeff Kirsher int rc; 708874aeea5SJeff Kirsher 709874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 710874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 711874aeea5SJeff Kirsher 7123dca9d2dSStuart Hodgson /* Only perform flush if dma is enabled */ 713626950dbSAlexandre Rames if (dev->is_busmaster && efx->state != STATE_RECOVERY) { 714874aeea5SJeff Kirsher rc = efx_nic_flush_queues(efx); 7153dca9d2dSStuart Hodgson 716874aeea5SJeff Kirsher if (rc && EFX_WORKAROUND_7803(efx)) { 717874aeea5SJeff Kirsher /* Schedule a reset to recover from the flush failure. The 718874aeea5SJeff Kirsher * descriptor caches reference memory we're about to free, 719874aeea5SJeff Kirsher * but falcon_reconfigure_mac_wrapper() won't reconnect 720874aeea5SJeff Kirsher * the MACs because of the pending reset. */ 721874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 722874aeea5SJeff Kirsher "Resetting to recover from flush failure\n"); 723874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_ALL); 724874aeea5SJeff Kirsher } else if (rc) { 725874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 726874aeea5SJeff Kirsher } else { 727874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, 728874aeea5SJeff Kirsher "successfully flushed all queues\n"); 729874aeea5SJeff Kirsher } 7303dca9d2dSStuart Hodgson } 731874aeea5SJeff Kirsher 732874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 7339f2cb71cSBen Hutchings /* RX packet processing is pipelined, so wait for the 7349f2cb71cSBen Hutchings * NAPI handler to complete. At least event queue 0 7359f2cb71cSBen Hutchings * might be kept active by non-data events, so don't 7369f2cb71cSBen Hutchings * use napi_synchronize() but actually disable NAPI 7379f2cb71cSBen Hutchings * temporarily. 7389f2cb71cSBen Hutchings */ 7399f2cb71cSBen Hutchings if (efx_channel_has_rx_queue(channel)) { 7409f2cb71cSBen Hutchings efx_stop_eventq(channel); 7419f2cb71cSBen Hutchings efx_start_eventq(channel); 7429f2cb71cSBen Hutchings } 743874aeea5SJeff Kirsher 744874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 745874aeea5SJeff Kirsher efx_fini_rx_queue(rx_queue); 746874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 747874aeea5SJeff Kirsher efx_fini_tx_queue(tx_queue); 748874aeea5SJeff Kirsher } 749874aeea5SJeff Kirsher } 750874aeea5SJeff Kirsher 751874aeea5SJeff Kirsher static void efx_remove_channel(struct efx_channel *channel) 752874aeea5SJeff Kirsher { 753874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 754874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 755874aeea5SJeff Kirsher 756874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 757874aeea5SJeff Kirsher "destroy chan %d\n", channel->channel); 758874aeea5SJeff Kirsher 759874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 760874aeea5SJeff Kirsher efx_remove_rx_queue(rx_queue); 761874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 762874aeea5SJeff Kirsher efx_remove_tx_queue(tx_queue); 763874aeea5SJeff Kirsher efx_remove_eventq(channel); 764c31e5f9fSStuart Hodgson channel->type->post_remove(channel); 765874aeea5SJeff Kirsher } 766874aeea5SJeff Kirsher 767874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx) 768874aeea5SJeff Kirsher { 769874aeea5SJeff Kirsher struct efx_channel *channel; 770874aeea5SJeff Kirsher 771874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 772874aeea5SJeff Kirsher efx_remove_channel(channel); 773874aeea5SJeff Kirsher } 774874aeea5SJeff Kirsher 775874aeea5SJeff Kirsher int 776874aeea5SJeff Kirsher efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 777874aeea5SJeff Kirsher { 778874aeea5SJeff Kirsher struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 779874aeea5SJeff Kirsher u32 old_rxq_entries, old_txq_entries; 7807f967c01SBen Hutchings unsigned i, next_buffer_table = 0; 7818b7325b4SBen Hutchings int rc; 7828b7325b4SBen Hutchings 7838b7325b4SBen Hutchings rc = efx_check_disabled(efx); 7848b7325b4SBen Hutchings if (rc) 7858b7325b4SBen Hutchings return rc; 7867f967c01SBen Hutchings 7877f967c01SBen Hutchings /* Not all channels should be reallocated. We must avoid 7887f967c01SBen Hutchings * reallocating their buffer table entries. 7897f967c01SBen Hutchings */ 7907f967c01SBen Hutchings efx_for_each_channel(channel, efx) { 7917f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 7927f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 7937f967c01SBen Hutchings 7947f967c01SBen Hutchings if (channel->type->copy) 7957f967c01SBen Hutchings continue; 7967f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 7977f967c01SBen Hutchings channel->eventq.index + 7987f967c01SBen Hutchings channel->eventq.entries); 7997f967c01SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 8007f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8017f967c01SBen Hutchings rx_queue->rxd.index + 8027f967c01SBen Hutchings rx_queue->rxd.entries); 8037f967c01SBen Hutchings efx_for_each_channel_tx_queue(tx_queue, channel) 8047f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8057f967c01SBen Hutchings tx_queue->txd.index + 8067f967c01SBen Hutchings tx_queue->txd.entries); 8077f967c01SBen Hutchings } 808874aeea5SJeff Kirsher 80929c69a48SBen Hutchings efx_device_detach_sync(efx); 810874aeea5SJeff Kirsher efx_stop_all(efx); 8117f967c01SBen Hutchings efx_stop_interrupts(efx, true); 812874aeea5SJeff Kirsher 8137f967c01SBen Hutchings /* Clone channels (where possible) */ 814874aeea5SJeff Kirsher memset(other_channel, 0, sizeof(other_channel)); 815874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 8167f967c01SBen Hutchings channel = efx->channel[i]; 8177f967c01SBen Hutchings if (channel->type->copy) 8187f967c01SBen Hutchings channel = channel->type->copy(channel); 819874aeea5SJeff Kirsher if (!channel) { 820874aeea5SJeff Kirsher rc = -ENOMEM; 821874aeea5SJeff Kirsher goto out; 822874aeea5SJeff Kirsher } 823874aeea5SJeff Kirsher other_channel[i] = channel; 824874aeea5SJeff Kirsher } 825874aeea5SJeff Kirsher 826874aeea5SJeff Kirsher /* Swap entry counts and channel pointers */ 827874aeea5SJeff Kirsher old_rxq_entries = efx->rxq_entries; 828874aeea5SJeff Kirsher old_txq_entries = efx->txq_entries; 829874aeea5SJeff Kirsher efx->rxq_entries = rxq_entries; 830874aeea5SJeff Kirsher efx->txq_entries = txq_entries; 831874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 832874aeea5SJeff Kirsher channel = efx->channel[i]; 833874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 834874aeea5SJeff Kirsher other_channel[i] = channel; 835874aeea5SJeff Kirsher } 836874aeea5SJeff Kirsher 8377f967c01SBen Hutchings /* Restart buffer table allocation */ 8387f967c01SBen Hutchings efx->next_buffer_table = next_buffer_table; 8397f967c01SBen Hutchings 8407f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8417f967c01SBen Hutchings channel = efx->channel[i]; 8427f967c01SBen Hutchings if (!channel->type->copy) 8437f967c01SBen Hutchings continue; 8447f967c01SBen Hutchings rc = efx_probe_channel(channel); 845874aeea5SJeff Kirsher if (rc) 846874aeea5SJeff Kirsher goto rollback; 8477f967c01SBen Hutchings efx_init_napi_channel(efx->channel[i]); 848874aeea5SJeff Kirsher } 849874aeea5SJeff Kirsher 8507f967c01SBen Hutchings out: 8517f967c01SBen Hutchings /* Destroy unused channel structures */ 8527f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8537f967c01SBen Hutchings channel = other_channel[i]; 8547f967c01SBen Hutchings if (channel && channel->type->copy) { 8557f967c01SBen Hutchings efx_fini_napi_channel(channel); 8567f967c01SBen Hutchings efx_remove_channel(channel); 8577f967c01SBen Hutchings kfree(channel); 8587f967c01SBen Hutchings } 8597f967c01SBen Hutchings } 8607f967c01SBen Hutchings 8617f967c01SBen Hutchings efx_start_interrupts(efx, true); 862874aeea5SJeff Kirsher efx_start_all(efx); 86329c69a48SBen Hutchings netif_device_attach(efx->net_dev); 864874aeea5SJeff Kirsher return rc; 865874aeea5SJeff Kirsher 866874aeea5SJeff Kirsher rollback: 867874aeea5SJeff Kirsher /* Swap back */ 868874aeea5SJeff Kirsher efx->rxq_entries = old_rxq_entries; 869874aeea5SJeff Kirsher efx->txq_entries = old_txq_entries; 870874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 871874aeea5SJeff Kirsher channel = efx->channel[i]; 872874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 873874aeea5SJeff Kirsher other_channel[i] = channel; 874874aeea5SJeff Kirsher } 875874aeea5SJeff Kirsher goto out; 876874aeea5SJeff Kirsher } 877874aeea5SJeff Kirsher 878874aeea5SJeff Kirsher void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 879874aeea5SJeff Kirsher { 880874aeea5SJeff Kirsher mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 881874aeea5SJeff Kirsher } 882874aeea5SJeff Kirsher 8837f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type = { 8847f967c01SBen Hutchings .pre_probe = efx_channel_dummy_op_int, 885c31e5f9fSStuart Hodgson .post_remove = efx_channel_dummy_op_void, 8867f967c01SBen Hutchings .get_name = efx_get_channel_name, 8877f967c01SBen Hutchings .copy = efx_copy_channel, 8887f967c01SBen Hutchings .keep_eventq = false, 8897f967c01SBen Hutchings }; 8907f967c01SBen Hutchings 8917f967c01SBen Hutchings int efx_channel_dummy_op_int(struct efx_channel *channel) 8927f967c01SBen Hutchings { 8937f967c01SBen Hutchings return 0; 8947f967c01SBen Hutchings } 8957f967c01SBen Hutchings 896c31e5f9fSStuart Hodgson void efx_channel_dummy_op_void(struct efx_channel *channel) 897c31e5f9fSStuart Hodgson { 898c31e5f9fSStuart Hodgson } 899c31e5f9fSStuart Hodgson 900874aeea5SJeff Kirsher /************************************************************************** 901874aeea5SJeff Kirsher * 902874aeea5SJeff Kirsher * Port handling 903874aeea5SJeff Kirsher * 904874aeea5SJeff Kirsher **************************************************************************/ 905874aeea5SJeff Kirsher 906874aeea5SJeff Kirsher /* This ensures that the kernel is kept informed (via 907874aeea5SJeff Kirsher * netif_carrier_on/off) of the link status, and also maintains the 908874aeea5SJeff Kirsher * link status's stop on the port's TX queue. 909874aeea5SJeff Kirsher */ 910874aeea5SJeff Kirsher void efx_link_status_changed(struct efx_nic *efx) 911874aeea5SJeff Kirsher { 912874aeea5SJeff Kirsher struct efx_link_state *link_state = &efx->link_state; 913874aeea5SJeff Kirsher 914874aeea5SJeff Kirsher /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 915874aeea5SJeff Kirsher * that no events are triggered between unregister_netdev() and the 916874aeea5SJeff Kirsher * driver unloading. A more general condition is that NETDEV_CHANGE 917874aeea5SJeff Kirsher * can only be generated between NETDEV_UP and NETDEV_DOWN */ 918874aeea5SJeff Kirsher if (!netif_running(efx->net_dev)) 919874aeea5SJeff Kirsher return; 920874aeea5SJeff Kirsher 921874aeea5SJeff Kirsher if (link_state->up != netif_carrier_ok(efx->net_dev)) { 922874aeea5SJeff Kirsher efx->n_link_state_changes++; 923874aeea5SJeff Kirsher 924874aeea5SJeff Kirsher if (link_state->up) 925874aeea5SJeff Kirsher netif_carrier_on(efx->net_dev); 926874aeea5SJeff Kirsher else 927874aeea5SJeff Kirsher netif_carrier_off(efx->net_dev); 928874aeea5SJeff Kirsher } 929874aeea5SJeff Kirsher 930874aeea5SJeff Kirsher /* Status message for kernel log */ 9312aa9ef11SBen Hutchings if (link_state->up) 932874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, 933874aeea5SJeff Kirsher "link up at %uMbps %s-duplex (MTU %d)%s\n", 934874aeea5SJeff Kirsher link_state->speed, link_state->fd ? "full" : "half", 935874aeea5SJeff Kirsher efx->net_dev->mtu, 936874aeea5SJeff Kirsher (efx->promiscuous ? " [PROMISC]" : "")); 9372aa9ef11SBen Hutchings else 938874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, "link down\n"); 939874aeea5SJeff Kirsher } 940874aeea5SJeff Kirsher 941874aeea5SJeff Kirsher void efx_link_set_advertising(struct efx_nic *efx, u32 advertising) 942874aeea5SJeff Kirsher { 943874aeea5SJeff Kirsher efx->link_advertising = advertising; 944874aeea5SJeff Kirsher if (advertising) { 945874aeea5SJeff Kirsher if (advertising & ADVERTISED_Pause) 946874aeea5SJeff Kirsher efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 947874aeea5SJeff Kirsher else 948874aeea5SJeff Kirsher efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 949874aeea5SJeff Kirsher if (advertising & ADVERTISED_Asym_Pause) 950874aeea5SJeff Kirsher efx->wanted_fc ^= EFX_FC_TX; 951874aeea5SJeff Kirsher } 952874aeea5SJeff Kirsher } 953874aeea5SJeff Kirsher 954874aeea5SJeff Kirsher void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 955874aeea5SJeff Kirsher { 956874aeea5SJeff Kirsher efx->wanted_fc = wanted_fc; 957874aeea5SJeff Kirsher if (efx->link_advertising) { 958874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_RX) 959874aeea5SJeff Kirsher efx->link_advertising |= (ADVERTISED_Pause | 960874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 961874aeea5SJeff Kirsher else 962874aeea5SJeff Kirsher efx->link_advertising &= ~(ADVERTISED_Pause | 963874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 964874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_TX) 965874aeea5SJeff Kirsher efx->link_advertising ^= ADVERTISED_Asym_Pause; 966874aeea5SJeff Kirsher } 967874aeea5SJeff Kirsher } 968874aeea5SJeff Kirsher 969874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx); 970874aeea5SJeff Kirsher 971874aeea5SJeff Kirsher /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 972874aeea5SJeff Kirsher * the MAC appropriately. All other PHY configuration changes are pushed 973874aeea5SJeff Kirsher * through phy_op->set_settings(), and pushed asynchronously to the MAC 974874aeea5SJeff Kirsher * through efx_monitor(). 975874aeea5SJeff Kirsher * 976874aeea5SJeff Kirsher * Callers must hold the mac_lock 977874aeea5SJeff Kirsher */ 978874aeea5SJeff Kirsher int __efx_reconfigure_port(struct efx_nic *efx) 979874aeea5SJeff Kirsher { 980874aeea5SJeff Kirsher enum efx_phy_mode phy_mode; 981874aeea5SJeff Kirsher int rc; 982874aeea5SJeff Kirsher 983874aeea5SJeff Kirsher WARN_ON(!mutex_is_locked(&efx->mac_lock)); 984874aeea5SJeff Kirsher 9850fca8c97SBen Hutchings /* Serialise the promiscuous flag with efx_set_rx_mode. */ 986874aeea5SJeff Kirsher netif_addr_lock_bh(efx->net_dev); 987874aeea5SJeff Kirsher netif_addr_unlock_bh(efx->net_dev); 988874aeea5SJeff Kirsher 989874aeea5SJeff Kirsher /* Disable PHY transmit in mac level loopbacks */ 990874aeea5SJeff Kirsher phy_mode = efx->phy_mode; 991874aeea5SJeff Kirsher if (LOOPBACK_INTERNAL(efx)) 992874aeea5SJeff Kirsher efx->phy_mode |= PHY_MODE_TX_DISABLED; 993874aeea5SJeff Kirsher else 994874aeea5SJeff Kirsher efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 995874aeea5SJeff Kirsher 996874aeea5SJeff Kirsher rc = efx->type->reconfigure_port(efx); 997874aeea5SJeff Kirsher 998874aeea5SJeff Kirsher if (rc) 999874aeea5SJeff Kirsher efx->phy_mode = phy_mode; 1000874aeea5SJeff Kirsher 1001874aeea5SJeff Kirsher return rc; 1002874aeea5SJeff Kirsher } 1003874aeea5SJeff Kirsher 1004874aeea5SJeff Kirsher /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1005874aeea5SJeff Kirsher * disabled. */ 1006874aeea5SJeff Kirsher int efx_reconfigure_port(struct efx_nic *efx) 1007874aeea5SJeff Kirsher { 1008874aeea5SJeff Kirsher int rc; 1009874aeea5SJeff Kirsher 1010874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1011874aeea5SJeff Kirsher 1012874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1013874aeea5SJeff Kirsher rc = __efx_reconfigure_port(efx); 1014874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1015874aeea5SJeff Kirsher 1016874aeea5SJeff Kirsher return rc; 1017874aeea5SJeff Kirsher } 1018874aeea5SJeff Kirsher 1019874aeea5SJeff Kirsher /* Asynchronous work item for changing MAC promiscuity and multicast 1020874aeea5SJeff Kirsher * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1021874aeea5SJeff Kirsher * MAC directly. */ 1022874aeea5SJeff Kirsher static void efx_mac_work(struct work_struct *data) 1023874aeea5SJeff Kirsher { 1024874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1025874aeea5SJeff Kirsher 1026874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 102730b81cdaSBen Hutchings if (efx->port_enabled) 1028710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 1029874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1030874aeea5SJeff Kirsher } 1031874aeea5SJeff Kirsher 1032874aeea5SJeff Kirsher static int efx_probe_port(struct efx_nic *efx) 1033874aeea5SJeff Kirsher { 1034874aeea5SJeff Kirsher int rc; 1035874aeea5SJeff Kirsher 1036874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1037874aeea5SJeff Kirsher 1038874aeea5SJeff Kirsher if (phy_flash_cfg) 1039874aeea5SJeff Kirsher efx->phy_mode = PHY_MODE_SPECIAL; 1040874aeea5SJeff Kirsher 1041874aeea5SJeff Kirsher /* Connect up MAC/PHY operations table */ 1042874aeea5SJeff Kirsher rc = efx->type->probe_port(efx); 1043874aeea5SJeff Kirsher if (rc) 1044874aeea5SJeff Kirsher return rc; 1045874aeea5SJeff Kirsher 1046e332bcb3SBen Hutchings /* Initialise MAC address to permanent address */ 1047e332bcb3SBen Hutchings memcpy(efx->net_dev->dev_addr, efx->net_dev->perm_addr, ETH_ALEN); 1048874aeea5SJeff Kirsher 1049874aeea5SJeff Kirsher return 0; 1050874aeea5SJeff Kirsher } 1051874aeea5SJeff Kirsher 1052874aeea5SJeff Kirsher static int efx_init_port(struct efx_nic *efx) 1053874aeea5SJeff Kirsher { 1054874aeea5SJeff Kirsher int rc; 1055874aeea5SJeff Kirsher 1056874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1057874aeea5SJeff Kirsher 1058874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1059874aeea5SJeff Kirsher 1060874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 1061874aeea5SJeff Kirsher if (rc) 1062874aeea5SJeff Kirsher goto fail1; 1063874aeea5SJeff Kirsher 1064874aeea5SJeff Kirsher efx->port_initialized = true; 1065874aeea5SJeff Kirsher 1066874aeea5SJeff Kirsher /* Reconfigure the MAC before creating dma queues (required for 1067874aeea5SJeff Kirsher * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1068710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 1069874aeea5SJeff Kirsher 1070874aeea5SJeff Kirsher /* Ensure the PHY advertises the correct flow control settings */ 1071874aeea5SJeff Kirsher rc = efx->phy_op->reconfigure(efx); 1072874aeea5SJeff Kirsher if (rc) 1073874aeea5SJeff Kirsher goto fail2; 1074874aeea5SJeff Kirsher 1075874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1076874aeea5SJeff Kirsher return 0; 1077874aeea5SJeff Kirsher 1078874aeea5SJeff Kirsher fail2: 1079874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1080874aeea5SJeff Kirsher fail1: 1081874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1082874aeea5SJeff Kirsher return rc; 1083874aeea5SJeff Kirsher } 1084874aeea5SJeff Kirsher 1085874aeea5SJeff Kirsher static void efx_start_port(struct efx_nic *efx) 1086874aeea5SJeff Kirsher { 1087874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1088874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 1089874aeea5SJeff Kirsher 1090874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1091874aeea5SJeff Kirsher efx->port_enabled = true; 1092874aeea5SJeff Kirsher 1093874aeea5SJeff Kirsher /* efx_mac_work() might have been scheduled after efx_stop_port(), 1094874aeea5SJeff Kirsher * and then cancelled by efx_flush_all() */ 1095710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 1096874aeea5SJeff Kirsher 1097874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1098874aeea5SJeff Kirsher } 1099874aeea5SJeff Kirsher 1100874aeea5SJeff Kirsher /* Prevent efx_mac_work() and efx_monitor() from working */ 1101874aeea5SJeff Kirsher static void efx_stop_port(struct efx_nic *efx) 1102874aeea5SJeff Kirsher { 1103874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1104874aeea5SJeff Kirsher 1105874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1106874aeea5SJeff Kirsher efx->port_enabled = false; 1107874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1108874aeea5SJeff Kirsher 1109874aeea5SJeff Kirsher /* Serialise against efx_set_multicast_list() */ 1110874aeea5SJeff Kirsher netif_addr_lock_bh(efx->net_dev); 1111874aeea5SJeff Kirsher netif_addr_unlock_bh(efx->net_dev); 1112874aeea5SJeff Kirsher } 1113874aeea5SJeff Kirsher 1114874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx) 1115874aeea5SJeff Kirsher { 1116874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1117874aeea5SJeff Kirsher 1118874aeea5SJeff Kirsher if (!efx->port_initialized) 1119874aeea5SJeff Kirsher return; 1120874aeea5SJeff Kirsher 1121874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1122874aeea5SJeff Kirsher efx->port_initialized = false; 1123874aeea5SJeff Kirsher 1124874aeea5SJeff Kirsher efx->link_state.up = false; 1125874aeea5SJeff Kirsher efx_link_status_changed(efx); 1126874aeea5SJeff Kirsher } 1127874aeea5SJeff Kirsher 1128874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx) 1129874aeea5SJeff Kirsher { 1130874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1131874aeea5SJeff Kirsher 1132874aeea5SJeff Kirsher efx->type->remove_port(efx); 1133874aeea5SJeff Kirsher } 1134874aeea5SJeff Kirsher 1135874aeea5SJeff Kirsher /************************************************************************** 1136874aeea5SJeff Kirsher * 1137874aeea5SJeff Kirsher * NIC handling 1138874aeea5SJeff Kirsher * 1139874aeea5SJeff Kirsher **************************************************************************/ 1140874aeea5SJeff Kirsher 1141874aeea5SJeff Kirsher /* This configures the PCI device to enable I/O and DMA. */ 1142874aeea5SJeff Kirsher static int efx_init_io(struct efx_nic *efx) 1143874aeea5SJeff Kirsher { 1144874aeea5SJeff Kirsher struct pci_dev *pci_dev = efx->pci_dev; 1145874aeea5SJeff Kirsher dma_addr_t dma_mask = efx->type->max_dma_mask; 1146874aeea5SJeff Kirsher int rc; 1147874aeea5SJeff Kirsher 1148874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1149874aeea5SJeff Kirsher 1150874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 1151874aeea5SJeff Kirsher if (rc) { 1152874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1153874aeea5SJeff Kirsher "failed to enable PCI device\n"); 1154874aeea5SJeff Kirsher goto fail1; 1155874aeea5SJeff Kirsher } 1156874aeea5SJeff Kirsher 1157874aeea5SJeff Kirsher pci_set_master(pci_dev); 1158874aeea5SJeff Kirsher 1159874aeea5SJeff Kirsher /* Set the PCI DMA mask. Try all possibilities from our 1160874aeea5SJeff Kirsher * genuine mask down to 32 bits, because some architectures 1161874aeea5SJeff Kirsher * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1162874aeea5SJeff Kirsher * masks event though they reject 46 bit masks. 1163874aeea5SJeff Kirsher */ 1164874aeea5SJeff Kirsher while (dma_mask > 0x7fffffffUL) { 11650e33d870SBen Hutchings if (dma_supported(&pci_dev->dev, dma_mask)) { 11660e33d870SBen Hutchings rc = dma_set_mask(&pci_dev->dev, dma_mask); 1167e9e01846SBen Hutchings if (rc == 0) 1168874aeea5SJeff Kirsher break; 1169e9e01846SBen Hutchings } 1170874aeea5SJeff Kirsher dma_mask >>= 1; 1171874aeea5SJeff Kirsher } 1172874aeea5SJeff Kirsher if (rc) { 1173874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1174874aeea5SJeff Kirsher "could not find a suitable DMA mask\n"); 1175874aeea5SJeff Kirsher goto fail2; 1176874aeea5SJeff Kirsher } 1177874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1178874aeea5SJeff Kirsher "using DMA mask %llx\n", (unsigned long long) dma_mask); 11790e33d870SBen Hutchings rc = dma_set_coherent_mask(&pci_dev->dev, dma_mask); 1180874aeea5SJeff Kirsher if (rc) { 11810e33d870SBen Hutchings /* dma_set_coherent_mask() is not *allowed* to 11820e33d870SBen Hutchings * fail with a mask that dma_set_mask() accepted, 1183874aeea5SJeff Kirsher * but just in case... 1184874aeea5SJeff Kirsher */ 1185874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1186874aeea5SJeff Kirsher "failed to set consistent DMA mask\n"); 1187874aeea5SJeff Kirsher goto fail2; 1188874aeea5SJeff Kirsher } 1189874aeea5SJeff Kirsher 1190874aeea5SJeff Kirsher efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR); 1191874aeea5SJeff Kirsher rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc"); 1192874aeea5SJeff Kirsher if (rc) { 1193874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1194874aeea5SJeff Kirsher "request for memory BAR failed\n"); 1195874aeea5SJeff Kirsher rc = -EIO; 1196874aeea5SJeff Kirsher goto fail3; 1197874aeea5SJeff Kirsher } 1198874aeea5SJeff Kirsher efx->membase = ioremap_nocache(efx->membase_phys, 1199874aeea5SJeff Kirsher efx->type->mem_map_size); 1200874aeea5SJeff Kirsher if (!efx->membase) { 1201874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1202874aeea5SJeff Kirsher "could not map memory BAR at %llx+%x\n", 1203874aeea5SJeff Kirsher (unsigned long long)efx->membase_phys, 1204874aeea5SJeff Kirsher efx->type->mem_map_size); 1205874aeea5SJeff Kirsher rc = -ENOMEM; 1206874aeea5SJeff Kirsher goto fail4; 1207874aeea5SJeff Kirsher } 1208874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1209874aeea5SJeff Kirsher "memory BAR at %llx+%x (virtual %p)\n", 1210874aeea5SJeff Kirsher (unsigned long long)efx->membase_phys, 1211874aeea5SJeff Kirsher efx->type->mem_map_size, efx->membase); 1212874aeea5SJeff Kirsher 1213874aeea5SJeff Kirsher return 0; 1214874aeea5SJeff Kirsher 1215874aeea5SJeff Kirsher fail4: 1216874aeea5SJeff Kirsher pci_release_region(efx->pci_dev, EFX_MEM_BAR); 1217874aeea5SJeff Kirsher fail3: 1218874aeea5SJeff Kirsher efx->membase_phys = 0; 1219874aeea5SJeff Kirsher fail2: 1220874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1221874aeea5SJeff Kirsher fail1: 1222874aeea5SJeff Kirsher return rc; 1223874aeea5SJeff Kirsher } 1224874aeea5SJeff Kirsher 1225874aeea5SJeff Kirsher static void efx_fini_io(struct efx_nic *efx) 1226874aeea5SJeff Kirsher { 1227874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1228874aeea5SJeff Kirsher 1229874aeea5SJeff Kirsher if (efx->membase) { 1230874aeea5SJeff Kirsher iounmap(efx->membase); 1231874aeea5SJeff Kirsher efx->membase = NULL; 1232874aeea5SJeff Kirsher } 1233874aeea5SJeff Kirsher 1234874aeea5SJeff Kirsher if (efx->membase_phys) { 1235874aeea5SJeff Kirsher pci_release_region(efx->pci_dev, EFX_MEM_BAR); 1236874aeea5SJeff Kirsher efx->membase_phys = 0; 1237874aeea5SJeff Kirsher } 1238874aeea5SJeff Kirsher 1239874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1240874aeea5SJeff Kirsher } 1241874aeea5SJeff Kirsher 1242a9a52506SBen Hutchings static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1243874aeea5SJeff Kirsher { 1244cdb08f8fSBen Hutchings cpumask_var_t thread_mask; 1245a16e5b24SBen Hutchings unsigned int count; 1246874aeea5SJeff Kirsher int cpu; 1247874aeea5SJeff Kirsher 1248cd2d5b52SBen Hutchings if (rss_cpus) { 1249cd2d5b52SBen Hutchings count = rss_cpus; 1250cd2d5b52SBen Hutchings } else { 1251cdb08f8fSBen Hutchings if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1252a9a52506SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1253a9a52506SBen Hutchings "RSS disabled due to allocation failure\n"); 1254874aeea5SJeff Kirsher return 1; 1255874aeea5SJeff Kirsher } 1256874aeea5SJeff Kirsher 1257874aeea5SJeff Kirsher count = 0; 1258874aeea5SJeff Kirsher for_each_online_cpu(cpu) { 1259cdb08f8fSBen Hutchings if (!cpumask_test_cpu(cpu, thread_mask)) { 1260874aeea5SJeff Kirsher ++count; 1261cdb08f8fSBen Hutchings cpumask_or(thread_mask, thread_mask, 1262cdb08f8fSBen Hutchings topology_thread_cpumask(cpu)); 1263874aeea5SJeff Kirsher } 1264874aeea5SJeff Kirsher } 1265874aeea5SJeff Kirsher 1266cdb08f8fSBen Hutchings free_cpumask_var(thread_mask); 1267cd2d5b52SBen Hutchings } 1268cd2d5b52SBen Hutchings 1269cd2d5b52SBen Hutchings /* If RSS is requested for the PF *and* VFs then we can't write RSS 1270cd2d5b52SBen Hutchings * table entries that are inaccessible to VFs 1271cd2d5b52SBen Hutchings */ 1272cd2d5b52SBen Hutchings if (efx_sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1273cd2d5b52SBen Hutchings count > efx_vf_size(efx)) { 1274cd2d5b52SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1275cd2d5b52SBen Hutchings "Reducing number of RSS channels from %u to %u for " 1276cd2d5b52SBen Hutchings "VF support. Increase vf-msix-limit to use more " 1277cd2d5b52SBen Hutchings "channels on the PF.\n", 1278cd2d5b52SBen Hutchings count, efx_vf_size(efx)); 1279cd2d5b52SBen Hutchings count = efx_vf_size(efx); 1280cd2d5b52SBen Hutchings } 1281cd2d5b52SBen Hutchings 1282874aeea5SJeff Kirsher return count; 1283874aeea5SJeff Kirsher } 1284874aeea5SJeff Kirsher 1285874aeea5SJeff Kirsher static int 1286874aeea5SJeff Kirsher efx_init_rx_cpu_rmap(struct efx_nic *efx, struct msix_entry *xentries) 1287874aeea5SJeff Kirsher { 1288874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 1289a16e5b24SBen Hutchings unsigned int i; 1290a16e5b24SBen Hutchings int rc; 1291874aeea5SJeff Kirsher 1292874aeea5SJeff Kirsher efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels); 1293874aeea5SJeff Kirsher if (!efx->net_dev->rx_cpu_rmap) 1294874aeea5SJeff Kirsher return -ENOMEM; 1295874aeea5SJeff Kirsher for (i = 0; i < efx->n_rx_channels; i++) { 1296874aeea5SJeff Kirsher rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap, 1297874aeea5SJeff Kirsher xentries[i].vector); 1298874aeea5SJeff Kirsher if (rc) { 1299874aeea5SJeff Kirsher free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap); 1300874aeea5SJeff Kirsher efx->net_dev->rx_cpu_rmap = NULL; 1301874aeea5SJeff Kirsher return rc; 1302874aeea5SJeff Kirsher } 1303874aeea5SJeff Kirsher } 1304874aeea5SJeff Kirsher #endif 1305874aeea5SJeff Kirsher return 0; 1306874aeea5SJeff Kirsher } 1307874aeea5SJeff Kirsher 1308874aeea5SJeff Kirsher /* Probe the number and type of interrupts we are able to obtain, and 1309874aeea5SJeff Kirsher * the resulting numbers of channels and RX queues. 1310874aeea5SJeff Kirsher */ 1311874aeea5SJeff Kirsher static int efx_probe_interrupts(struct efx_nic *efx) 1312874aeea5SJeff Kirsher { 1313a16e5b24SBen Hutchings unsigned int max_channels = 1314a16e5b24SBen Hutchings min(efx->type->phys_addr_channels, EFX_MAX_CHANNELS); 13157f967c01SBen Hutchings unsigned int extra_channels = 0; 13167f967c01SBen Hutchings unsigned int i, j; 1317a16e5b24SBen Hutchings int rc; 1318874aeea5SJeff Kirsher 13197f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 13207f967c01SBen Hutchings if (efx->extra_channel_type[i]) 13217f967c01SBen Hutchings ++extra_channels; 13227f967c01SBen Hutchings 1323874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1324874aeea5SJeff Kirsher struct msix_entry xentries[EFX_MAX_CHANNELS]; 1325a16e5b24SBen Hutchings unsigned int n_channels; 1326874aeea5SJeff Kirsher 1327a9a52506SBen Hutchings n_channels = efx_wanted_parallelism(efx); 1328874aeea5SJeff Kirsher if (separate_tx_channels) 1329874aeea5SJeff Kirsher n_channels *= 2; 13307f967c01SBen Hutchings n_channels += extra_channels; 1331874aeea5SJeff Kirsher n_channels = min(n_channels, max_channels); 1332874aeea5SJeff Kirsher 1333874aeea5SJeff Kirsher for (i = 0; i < n_channels; i++) 1334874aeea5SJeff Kirsher xentries[i].entry = i; 1335874aeea5SJeff Kirsher rc = pci_enable_msix(efx->pci_dev, xentries, n_channels); 1336874aeea5SJeff Kirsher if (rc > 0) { 1337874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1338874aeea5SJeff Kirsher "WARNING: Insufficient MSI-X vectors" 1339a16e5b24SBen Hutchings " available (%d < %u).\n", rc, n_channels); 1340874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1341874aeea5SJeff Kirsher "WARNING: Performance may be reduced.\n"); 1342874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(rc >= n_channels); 1343874aeea5SJeff Kirsher n_channels = rc; 1344874aeea5SJeff Kirsher rc = pci_enable_msix(efx->pci_dev, xentries, 1345874aeea5SJeff Kirsher n_channels); 1346874aeea5SJeff Kirsher } 1347874aeea5SJeff Kirsher 1348874aeea5SJeff Kirsher if (rc == 0) { 1349874aeea5SJeff Kirsher efx->n_channels = n_channels; 13507f967c01SBen Hutchings if (n_channels > extra_channels) 13517f967c01SBen Hutchings n_channels -= extra_channels; 1352874aeea5SJeff Kirsher if (separate_tx_channels) { 13537f967c01SBen Hutchings efx->n_tx_channels = max(n_channels / 2, 1U); 13547f967c01SBen Hutchings efx->n_rx_channels = max(n_channels - 13557f967c01SBen Hutchings efx->n_tx_channels, 13567f967c01SBen Hutchings 1U); 1357874aeea5SJeff Kirsher } else { 13587f967c01SBen Hutchings efx->n_tx_channels = n_channels; 13597f967c01SBen Hutchings efx->n_rx_channels = n_channels; 1360874aeea5SJeff Kirsher } 1361874aeea5SJeff Kirsher rc = efx_init_rx_cpu_rmap(efx, xentries); 1362874aeea5SJeff Kirsher if (rc) { 1363874aeea5SJeff Kirsher pci_disable_msix(efx->pci_dev); 1364874aeea5SJeff Kirsher return rc; 1365874aeea5SJeff Kirsher } 13667f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) 1367874aeea5SJeff Kirsher efx_get_channel(efx, i)->irq = 1368874aeea5SJeff Kirsher xentries[i].vector; 1369874aeea5SJeff Kirsher } else { 1370874aeea5SJeff Kirsher /* Fall back to single channel MSI */ 1371874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_MSI; 1372874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1373874aeea5SJeff Kirsher "could not enable MSI-X\n"); 1374874aeea5SJeff Kirsher } 1375874aeea5SJeff Kirsher } 1376874aeea5SJeff Kirsher 1377874aeea5SJeff Kirsher /* Try single interrupt MSI */ 1378874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1379874aeea5SJeff Kirsher efx->n_channels = 1; 1380874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1381874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1382874aeea5SJeff Kirsher rc = pci_enable_msi(efx->pci_dev); 1383874aeea5SJeff Kirsher if (rc == 0) { 1384874aeea5SJeff Kirsher efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1385874aeea5SJeff Kirsher } else { 1386874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1387874aeea5SJeff Kirsher "could not enable MSI\n"); 1388874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1389874aeea5SJeff Kirsher } 1390874aeea5SJeff Kirsher } 1391874aeea5SJeff Kirsher 1392874aeea5SJeff Kirsher /* Assume legacy interrupts */ 1393874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1394874aeea5SJeff Kirsher efx->n_channels = 1 + (separate_tx_channels ? 1 : 0); 1395874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1396874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1397874aeea5SJeff Kirsher efx->legacy_irq = efx->pci_dev->irq; 1398874aeea5SJeff Kirsher } 1399874aeea5SJeff Kirsher 14007f967c01SBen Hutchings /* Assign extra channels if possible */ 14017f967c01SBen Hutchings j = efx->n_channels; 14027f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 14037f967c01SBen Hutchings if (!efx->extra_channel_type[i]) 14047f967c01SBen Hutchings continue; 14057f967c01SBen Hutchings if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 14067f967c01SBen Hutchings efx->n_channels <= extra_channels) { 14077f967c01SBen Hutchings efx->extra_channel_type[i]->handle_no_channel(efx); 14087f967c01SBen Hutchings } else { 14097f967c01SBen Hutchings --j; 14107f967c01SBen Hutchings efx_get_channel(efx, j)->type = 14117f967c01SBen Hutchings efx->extra_channel_type[i]; 14127f967c01SBen Hutchings } 14137f967c01SBen Hutchings } 14147f967c01SBen Hutchings 1415cd2d5b52SBen Hutchings /* RSS might be usable on VFs even if it is disabled on the PF */ 14163132d282SBen Hutchings efx->rss_spread = ((efx->n_rx_channels > 1 || !efx_sriov_wanted(efx)) ? 1417cd2d5b52SBen Hutchings efx->n_rx_channels : efx_vf_size(efx)); 1418cd2d5b52SBen Hutchings 1419874aeea5SJeff Kirsher return 0; 1420874aeea5SJeff Kirsher } 1421874aeea5SJeff Kirsher 14229f2cb71cSBen Hutchings /* Enable interrupts, then probe and start the event queues */ 14237f967c01SBen Hutchings static void efx_start_interrupts(struct efx_nic *efx, bool may_keep_eventq) 14249f2cb71cSBen Hutchings { 14259f2cb71cSBen Hutchings struct efx_channel *channel; 14269f2cb71cSBen Hutchings 14278b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 14288b7325b4SBen Hutchings 14299f2cb71cSBen Hutchings if (efx->legacy_irq) 14309f2cb71cSBen Hutchings efx->legacy_irq_enabled = true; 14319f2cb71cSBen Hutchings efx_nic_enable_interrupts(efx); 14329f2cb71cSBen Hutchings 14339f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 14347f967c01SBen Hutchings if (!channel->type->keep_eventq || !may_keep_eventq) 14359f2cb71cSBen Hutchings efx_init_eventq(channel); 14369f2cb71cSBen Hutchings efx_start_eventq(channel); 14379f2cb71cSBen Hutchings } 14389f2cb71cSBen Hutchings 14399f2cb71cSBen Hutchings efx_mcdi_mode_event(efx); 14409f2cb71cSBen Hutchings } 14419f2cb71cSBen Hutchings 14427f967c01SBen Hutchings static void efx_stop_interrupts(struct efx_nic *efx, bool may_keep_eventq) 14439f2cb71cSBen Hutchings { 14449f2cb71cSBen Hutchings struct efx_channel *channel; 14459f2cb71cSBen Hutchings 14468b7325b4SBen Hutchings if (efx->state == STATE_DISABLED) 14478b7325b4SBen Hutchings return; 14488b7325b4SBen Hutchings 14499f2cb71cSBen Hutchings efx_mcdi_mode_poll(efx); 14509f2cb71cSBen Hutchings 14519f2cb71cSBen Hutchings efx_nic_disable_interrupts(efx); 14529f2cb71cSBen Hutchings if (efx->legacy_irq) { 14539f2cb71cSBen Hutchings synchronize_irq(efx->legacy_irq); 14549f2cb71cSBen Hutchings efx->legacy_irq_enabled = false; 14559f2cb71cSBen Hutchings } 14569f2cb71cSBen Hutchings 14579f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 14589f2cb71cSBen Hutchings if (channel->irq) 14599f2cb71cSBen Hutchings synchronize_irq(channel->irq); 14609f2cb71cSBen Hutchings 14619f2cb71cSBen Hutchings efx_stop_eventq(channel); 14627f967c01SBen Hutchings if (!channel->type->keep_eventq || !may_keep_eventq) 14639f2cb71cSBen Hutchings efx_fini_eventq(channel); 14649f2cb71cSBen Hutchings } 14659f2cb71cSBen Hutchings } 14669f2cb71cSBen Hutchings 1467874aeea5SJeff Kirsher static void efx_remove_interrupts(struct efx_nic *efx) 1468874aeea5SJeff Kirsher { 1469874aeea5SJeff Kirsher struct efx_channel *channel; 1470874aeea5SJeff Kirsher 1471874aeea5SJeff Kirsher /* Remove MSI/MSI-X interrupts */ 1472874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1473874aeea5SJeff Kirsher channel->irq = 0; 1474874aeea5SJeff Kirsher pci_disable_msi(efx->pci_dev); 1475874aeea5SJeff Kirsher pci_disable_msix(efx->pci_dev); 1476874aeea5SJeff Kirsher 1477874aeea5SJeff Kirsher /* Remove legacy interrupt */ 1478874aeea5SJeff Kirsher efx->legacy_irq = 0; 1479874aeea5SJeff Kirsher } 1480874aeea5SJeff Kirsher 1481874aeea5SJeff Kirsher static void efx_set_channels(struct efx_nic *efx) 1482874aeea5SJeff Kirsher { 1483874aeea5SJeff Kirsher struct efx_channel *channel; 1484874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 1485874aeea5SJeff Kirsher 1486874aeea5SJeff Kirsher efx->tx_channel_offset = 1487874aeea5SJeff Kirsher separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0; 1488874aeea5SJeff Kirsher 148979d68b37SStuart Hodgson /* We need to mark which channels really have RX and TX 149079d68b37SStuart Hodgson * queues, and adjust the TX queue numbers if we have separate 1491874aeea5SJeff Kirsher * RX-only and TX-only channels. 1492874aeea5SJeff Kirsher */ 1493874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 149479d68b37SStuart Hodgson if (channel->channel < efx->n_rx_channels) 149579d68b37SStuart Hodgson channel->rx_queue.core_index = channel->channel; 149679d68b37SStuart Hodgson else 149779d68b37SStuart Hodgson channel->rx_queue.core_index = -1; 149879d68b37SStuart Hodgson 1499874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 1500874aeea5SJeff Kirsher tx_queue->queue -= (efx->tx_channel_offset * 1501874aeea5SJeff Kirsher EFX_TXQ_TYPES); 1502874aeea5SJeff Kirsher } 1503874aeea5SJeff Kirsher } 1504874aeea5SJeff Kirsher 1505874aeea5SJeff Kirsher static int efx_probe_nic(struct efx_nic *efx) 1506874aeea5SJeff Kirsher { 1507874aeea5SJeff Kirsher size_t i; 1508874aeea5SJeff Kirsher int rc; 1509874aeea5SJeff Kirsher 1510874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1511874aeea5SJeff Kirsher 1512874aeea5SJeff Kirsher /* Carry out hardware-type specific initialisation */ 1513874aeea5SJeff Kirsher rc = efx->type->probe(efx); 1514874aeea5SJeff Kirsher if (rc) 1515874aeea5SJeff Kirsher return rc; 1516874aeea5SJeff Kirsher 1517874aeea5SJeff Kirsher /* Determine the number of channels and queues by trying to hook 1518874aeea5SJeff Kirsher * in MSI-X interrupts. */ 1519874aeea5SJeff Kirsher rc = efx_probe_interrupts(efx); 1520874aeea5SJeff Kirsher if (rc) 1521874aeea5SJeff Kirsher goto fail; 1522874aeea5SJeff Kirsher 152328e47c49SBen Hutchings efx->type->dimension_resources(efx); 152428e47c49SBen Hutchings 1525874aeea5SJeff Kirsher if (efx->n_channels > 1) 1526874aeea5SJeff Kirsher get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key)); 1527874aeea5SJeff Kirsher for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1528278bc429SBen Hutchings efx->rx_indir_table[i] = 1529cd2d5b52SBen Hutchings ethtool_rxfh_indir_default(i, efx->rss_spread); 1530874aeea5SJeff Kirsher 1531874aeea5SJeff Kirsher efx_set_channels(efx); 1532874aeea5SJeff Kirsher netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1533874aeea5SJeff Kirsher netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1534874aeea5SJeff Kirsher 1535874aeea5SJeff Kirsher /* Initialise the interrupt moderation settings */ 15369e393b30SBen Hutchings efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 15379e393b30SBen Hutchings true); 1538874aeea5SJeff Kirsher 1539874aeea5SJeff Kirsher return 0; 1540874aeea5SJeff Kirsher 1541874aeea5SJeff Kirsher fail: 1542874aeea5SJeff Kirsher efx->type->remove(efx); 1543874aeea5SJeff Kirsher return rc; 1544874aeea5SJeff Kirsher } 1545874aeea5SJeff Kirsher 1546874aeea5SJeff Kirsher static void efx_remove_nic(struct efx_nic *efx) 1547874aeea5SJeff Kirsher { 1548874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1549874aeea5SJeff Kirsher 1550874aeea5SJeff Kirsher efx_remove_interrupts(efx); 1551874aeea5SJeff Kirsher efx->type->remove(efx); 1552874aeea5SJeff Kirsher } 1553874aeea5SJeff Kirsher 1554874aeea5SJeff Kirsher /************************************************************************** 1555874aeea5SJeff Kirsher * 1556874aeea5SJeff Kirsher * NIC startup/shutdown 1557874aeea5SJeff Kirsher * 1558874aeea5SJeff Kirsher *************************************************************************/ 1559874aeea5SJeff Kirsher 1560874aeea5SJeff Kirsher static int efx_probe_all(struct efx_nic *efx) 1561874aeea5SJeff Kirsher { 1562874aeea5SJeff Kirsher int rc; 1563874aeea5SJeff Kirsher 1564874aeea5SJeff Kirsher rc = efx_probe_nic(efx); 1565874aeea5SJeff Kirsher if (rc) { 1566874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1567874aeea5SJeff Kirsher goto fail1; 1568874aeea5SJeff Kirsher } 1569874aeea5SJeff Kirsher 1570874aeea5SJeff Kirsher rc = efx_probe_port(efx); 1571874aeea5SJeff Kirsher if (rc) { 1572874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1573874aeea5SJeff Kirsher goto fail2; 1574874aeea5SJeff Kirsher } 1575874aeea5SJeff Kirsher 15767e6d06f0SBen Hutchings BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 15777e6d06f0SBen Hutchings if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 15787e6d06f0SBen Hutchings rc = -EINVAL; 15797e6d06f0SBen Hutchings goto fail3; 15807e6d06f0SBen Hutchings } 1581874aeea5SJeff Kirsher efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1582874aeea5SJeff Kirsher 1583874aeea5SJeff Kirsher rc = efx_probe_filters(efx); 1584874aeea5SJeff Kirsher if (rc) { 1585874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1586874aeea5SJeff Kirsher "failed to create filter tables\n"); 15877f967c01SBen Hutchings goto fail3; 1588874aeea5SJeff Kirsher } 1589874aeea5SJeff Kirsher 15907f967c01SBen Hutchings rc = efx_probe_channels(efx); 15917f967c01SBen Hutchings if (rc) 15927f967c01SBen Hutchings goto fail4; 15937f967c01SBen Hutchings 1594874aeea5SJeff Kirsher return 0; 1595874aeea5SJeff Kirsher 1596874aeea5SJeff Kirsher fail4: 15977f967c01SBen Hutchings efx_remove_filters(efx); 1598874aeea5SJeff Kirsher fail3: 1599874aeea5SJeff Kirsher efx_remove_port(efx); 1600874aeea5SJeff Kirsher fail2: 1601874aeea5SJeff Kirsher efx_remove_nic(efx); 1602874aeea5SJeff Kirsher fail1: 1603874aeea5SJeff Kirsher return rc; 1604874aeea5SJeff Kirsher } 1605874aeea5SJeff Kirsher 16068b7325b4SBen Hutchings /* If the interface is supposed to be running but is not, start 16078b7325b4SBen Hutchings * the hardware and software data path, regular activity for the port 16088b7325b4SBen Hutchings * (MAC statistics, link polling, etc.) and schedule the port to be 16098b7325b4SBen Hutchings * reconfigured. Interrupts must already be enabled. This function 16108b7325b4SBen Hutchings * is safe to call multiple times, so long as the NIC is not disabled. 16118b7325b4SBen Hutchings * Requires the RTNL lock. 16129f2cb71cSBen Hutchings */ 1613874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx) 1614874aeea5SJeff Kirsher { 1615874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 16168b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1617874aeea5SJeff Kirsher 1618874aeea5SJeff Kirsher /* Check that it is appropriate to restart the interface. All 1619874aeea5SJeff Kirsher * of these flags are safe to read under just the rtnl lock */ 16208b7325b4SBen Hutchings if (efx->port_enabled || !netif_running(efx->net_dev)) 1621874aeea5SJeff Kirsher return; 1622874aeea5SJeff Kirsher 1623874aeea5SJeff Kirsher efx_start_port(efx); 16249f2cb71cSBen Hutchings efx_start_datapath(efx); 1625874aeea5SJeff Kirsher 1626626950dbSAlexandre Rames /* Start the hardware monitor if there is one */ 1627626950dbSAlexandre Rames if (efx->type->monitor != NULL) 1628874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 1629874aeea5SJeff Kirsher efx_monitor_interval); 1630626950dbSAlexandre Rames 1631626950dbSAlexandre Rames /* If link state detection is normally event-driven, we have 1632626950dbSAlexandre Rames * to poll now because we could have missed a change 1633626950dbSAlexandre Rames */ 1634626950dbSAlexandre Rames if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { 1635874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1636874aeea5SJeff Kirsher if (efx->phy_op->poll(efx)) 1637874aeea5SJeff Kirsher efx_link_status_changed(efx); 1638874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1639874aeea5SJeff Kirsher } 1640874aeea5SJeff Kirsher 1641874aeea5SJeff Kirsher efx->type->start_stats(efx); 1642874aeea5SJeff Kirsher } 1643874aeea5SJeff Kirsher 1644874aeea5SJeff Kirsher /* Flush all delayed work. Should only be called when no more delayed work 1645874aeea5SJeff Kirsher * will be scheduled. This doesn't flush pending online resets (efx_reset), 1646874aeea5SJeff Kirsher * since we're holding the rtnl_lock at this point. */ 1647874aeea5SJeff Kirsher static void efx_flush_all(struct efx_nic *efx) 1648874aeea5SJeff Kirsher { 1649dd40781eSBen Hutchings /* Make sure the hardware monitor and event self-test are stopped */ 1650874aeea5SJeff Kirsher cancel_delayed_work_sync(&efx->monitor_work); 1651dd40781eSBen Hutchings efx_selftest_async_cancel(efx); 1652874aeea5SJeff Kirsher /* Stop scheduled port reconfigurations */ 1653874aeea5SJeff Kirsher cancel_work_sync(&efx->mac_work); 1654874aeea5SJeff Kirsher } 1655874aeea5SJeff Kirsher 16568b7325b4SBen Hutchings /* Quiesce the hardware and software data path, and regular activity 16578b7325b4SBen Hutchings * for the port without bringing the link down. Safe to call multiple 16588b7325b4SBen Hutchings * times with the NIC in almost any state, but interrupts should be 16598b7325b4SBen Hutchings * enabled. Requires the RTNL lock. 16608b7325b4SBen Hutchings */ 1661874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx) 1662874aeea5SJeff Kirsher { 1663874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1664874aeea5SJeff Kirsher 1665874aeea5SJeff Kirsher /* port_enabled can be read safely under the rtnl lock */ 1666874aeea5SJeff Kirsher if (!efx->port_enabled) 1667874aeea5SJeff Kirsher return; 1668874aeea5SJeff Kirsher 1669874aeea5SJeff Kirsher efx->type->stop_stats(efx); 1670874aeea5SJeff Kirsher efx_stop_port(efx); 1671874aeea5SJeff Kirsher 1672874aeea5SJeff Kirsher /* Flush efx_mac_work(), refill_workqueue, monitor_work */ 1673874aeea5SJeff Kirsher efx_flush_all(efx); 1674874aeea5SJeff Kirsher 167529c69a48SBen Hutchings /* Stop the kernel transmit interface. This is only valid if 167629c69a48SBen Hutchings * the device is stopped or detached; otherwise the watchdog 167729c69a48SBen Hutchings * may fire immediately. 167829c69a48SBen Hutchings */ 167929c69a48SBen Hutchings WARN_ON(netif_running(efx->net_dev) && 168029c69a48SBen Hutchings netif_device_present(efx->net_dev)); 16819f2cb71cSBen Hutchings netif_tx_disable(efx->net_dev); 16829f2cb71cSBen Hutchings 16839f2cb71cSBen Hutchings efx_stop_datapath(efx); 1684874aeea5SJeff Kirsher } 1685874aeea5SJeff Kirsher 1686874aeea5SJeff Kirsher static void efx_remove_all(struct efx_nic *efx) 1687874aeea5SJeff Kirsher { 1688874aeea5SJeff Kirsher efx_remove_channels(efx); 16897f967c01SBen Hutchings efx_remove_filters(efx); 1690874aeea5SJeff Kirsher efx_remove_port(efx); 1691874aeea5SJeff Kirsher efx_remove_nic(efx); 1692874aeea5SJeff Kirsher } 1693874aeea5SJeff Kirsher 1694874aeea5SJeff Kirsher /************************************************************************** 1695874aeea5SJeff Kirsher * 1696874aeea5SJeff Kirsher * Interrupt moderation 1697874aeea5SJeff Kirsher * 1698874aeea5SJeff Kirsher **************************************************************************/ 1699874aeea5SJeff Kirsher 1700cc180b69SBen Hutchings static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns) 1701874aeea5SJeff Kirsher { 1702b548f976SBen Hutchings if (usecs == 0) 1703b548f976SBen Hutchings return 0; 1704cc180b69SBen Hutchings if (usecs * 1000 < quantum_ns) 1705874aeea5SJeff Kirsher return 1; /* never round down to 0 */ 1706cc180b69SBen Hutchings return usecs * 1000 / quantum_ns; 1707874aeea5SJeff Kirsher } 1708874aeea5SJeff Kirsher 1709874aeea5SJeff Kirsher /* Set interrupt moderation parameters */ 17109e393b30SBen Hutchings int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 17119e393b30SBen Hutchings unsigned int rx_usecs, bool rx_adaptive, 17129e393b30SBen Hutchings bool rx_may_override_tx) 1713874aeea5SJeff Kirsher { 1714874aeea5SJeff Kirsher struct efx_channel *channel; 1715cc180b69SBen Hutchings unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max * 1716cc180b69SBen Hutchings efx->timer_quantum_ns, 1717cc180b69SBen Hutchings 1000); 1718cc180b69SBen Hutchings unsigned int tx_ticks; 1719cc180b69SBen Hutchings unsigned int rx_ticks; 1720874aeea5SJeff Kirsher 1721874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1722874aeea5SJeff Kirsher 1723cc180b69SBen Hutchings if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max) 17249e393b30SBen Hutchings return -EINVAL; 17259e393b30SBen Hutchings 1726cc180b69SBen Hutchings tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns); 1727cc180b69SBen Hutchings rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns); 1728cc180b69SBen Hutchings 17299e393b30SBen Hutchings if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 && 17309e393b30SBen Hutchings !rx_may_override_tx) { 17319e393b30SBen Hutchings netif_err(efx, drv, efx->net_dev, "Channels are shared. " 17329e393b30SBen Hutchings "RX and TX IRQ moderation must be equal\n"); 17339e393b30SBen Hutchings return -EINVAL; 17349e393b30SBen Hutchings } 17359e393b30SBen Hutchings 1736874aeea5SJeff Kirsher efx->irq_rx_adaptive = rx_adaptive; 1737874aeea5SJeff Kirsher efx->irq_rx_moderation = rx_ticks; 1738874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 1739874aeea5SJeff Kirsher if (efx_channel_has_rx_queue(channel)) 1740874aeea5SJeff Kirsher channel->irq_moderation = rx_ticks; 1741874aeea5SJeff Kirsher else if (efx_channel_has_tx_queues(channel)) 1742874aeea5SJeff Kirsher channel->irq_moderation = tx_ticks; 1743874aeea5SJeff Kirsher } 17449e393b30SBen Hutchings 17459e393b30SBen Hutchings return 0; 1746874aeea5SJeff Kirsher } 1747874aeea5SJeff Kirsher 1748a0c4faf5SBen Hutchings void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 1749a0c4faf5SBen Hutchings unsigned int *rx_usecs, bool *rx_adaptive) 1750a0c4faf5SBen Hutchings { 1751cc180b69SBen Hutchings /* We must round up when converting ticks to microseconds 1752cc180b69SBen Hutchings * because we round down when converting the other way. 1753cc180b69SBen Hutchings */ 1754cc180b69SBen Hutchings 1755a0c4faf5SBen Hutchings *rx_adaptive = efx->irq_rx_adaptive; 1756cc180b69SBen Hutchings *rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation * 1757cc180b69SBen Hutchings efx->timer_quantum_ns, 1758cc180b69SBen Hutchings 1000); 1759a0c4faf5SBen Hutchings 1760a0c4faf5SBen Hutchings /* If channels are shared between RX and TX, so is IRQ 1761a0c4faf5SBen Hutchings * moderation. Otherwise, IRQ moderation is the same for all 1762a0c4faf5SBen Hutchings * TX channels and is not adaptive. 1763a0c4faf5SBen Hutchings */ 1764a0c4faf5SBen Hutchings if (efx->tx_channel_offset == 0) 1765a0c4faf5SBen Hutchings *tx_usecs = *rx_usecs; 1766a0c4faf5SBen Hutchings else 1767cc180b69SBen Hutchings *tx_usecs = DIV_ROUND_UP( 1768a0c4faf5SBen Hutchings efx->channel[efx->tx_channel_offset]->irq_moderation * 1769cc180b69SBen Hutchings efx->timer_quantum_ns, 1770cc180b69SBen Hutchings 1000); 1771a0c4faf5SBen Hutchings } 1772a0c4faf5SBen Hutchings 1773874aeea5SJeff Kirsher /************************************************************************** 1774874aeea5SJeff Kirsher * 1775874aeea5SJeff Kirsher * Hardware monitor 1776874aeea5SJeff Kirsher * 1777874aeea5SJeff Kirsher **************************************************************************/ 1778874aeea5SJeff Kirsher 1779874aeea5SJeff Kirsher /* Run periodically off the general workqueue */ 1780874aeea5SJeff Kirsher static void efx_monitor(struct work_struct *data) 1781874aeea5SJeff Kirsher { 1782874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, 1783874aeea5SJeff Kirsher monitor_work.work); 1784874aeea5SJeff Kirsher 1785874aeea5SJeff Kirsher netif_vdbg(efx, timer, efx->net_dev, 1786874aeea5SJeff Kirsher "hardware monitor executing on CPU %d\n", 1787874aeea5SJeff Kirsher raw_smp_processor_id()); 1788874aeea5SJeff Kirsher BUG_ON(efx->type->monitor == NULL); 1789874aeea5SJeff Kirsher 1790874aeea5SJeff Kirsher /* If the mac_lock is already held then it is likely a port 1791874aeea5SJeff Kirsher * reconfiguration is already in place, which will likely do 1792874aeea5SJeff Kirsher * most of the work of monitor() anyway. */ 1793874aeea5SJeff Kirsher if (mutex_trylock(&efx->mac_lock)) { 1794874aeea5SJeff Kirsher if (efx->port_enabled) 1795874aeea5SJeff Kirsher efx->type->monitor(efx); 1796874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1797874aeea5SJeff Kirsher } 1798874aeea5SJeff Kirsher 1799874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 1800874aeea5SJeff Kirsher efx_monitor_interval); 1801874aeea5SJeff Kirsher } 1802874aeea5SJeff Kirsher 1803874aeea5SJeff Kirsher /************************************************************************** 1804874aeea5SJeff Kirsher * 1805874aeea5SJeff Kirsher * ioctls 1806874aeea5SJeff Kirsher * 1807874aeea5SJeff Kirsher *************************************************************************/ 1808874aeea5SJeff Kirsher 1809874aeea5SJeff Kirsher /* Net device ioctl 1810874aeea5SJeff Kirsher * Context: process, rtnl_lock() held. 1811874aeea5SJeff Kirsher */ 1812874aeea5SJeff Kirsher static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 1813874aeea5SJeff Kirsher { 1814874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 1815874aeea5SJeff Kirsher struct mii_ioctl_data *data = if_mii(ifr); 1816874aeea5SJeff Kirsher 18177c236c43SStuart Hodgson if (cmd == SIOCSHWTSTAMP) 18187c236c43SStuart Hodgson return efx_ptp_ioctl(efx, ifr, cmd); 18197c236c43SStuart Hodgson 1820874aeea5SJeff Kirsher /* Convert phy_id from older PRTAD/DEVAD format */ 1821874aeea5SJeff Kirsher if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 1822874aeea5SJeff Kirsher (data->phy_id & 0xfc00) == 0x0400) 1823874aeea5SJeff Kirsher data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 1824874aeea5SJeff Kirsher 1825874aeea5SJeff Kirsher return mdio_mii_ioctl(&efx->mdio, data, cmd); 1826874aeea5SJeff Kirsher } 1827874aeea5SJeff Kirsher 1828874aeea5SJeff Kirsher /************************************************************************** 1829874aeea5SJeff Kirsher * 1830874aeea5SJeff Kirsher * NAPI interface 1831874aeea5SJeff Kirsher * 1832874aeea5SJeff Kirsher **************************************************************************/ 1833874aeea5SJeff Kirsher 18347f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel) 1835874aeea5SJeff Kirsher { 18367f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 1837874aeea5SJeff Kirsher 1838874aeea5SJeff Kirsher channel->napi_dev = efx->net_dev; 1839874aeea5SJeff Kirsher netif_napi_add(channel->napi_dev, &channel->napi_str, 1840874aeea5SJeff Kirsher efx_poll, napi_weight); 1841874aeea5SJeff Kirsher } 18427f967c01SBen Hutchings 18437f967c01SBen Hutchings static void efx_init_napi(struct efx_nic *efx) 18447f967c01SBen Hutchings { 18457f967c01SBen Hutchings struct efx_channel *channel; 18467f967c01SBen Hutchings 18477f967c01SBen Hutchings efx_for_each_channel(channel, efx) 18487f967c01SBen Hutchings efx_init_napi_channel(channel); 1849874aeea5SJeff Kirsher } 1850874aeea5SJeff Kirsher 1851874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel) 1852874aeea5SJeff Kirsher { 1853874aeea5SJeff Kirsher if (channel->napi_dev) 1854874aeea5SJeff Kirsher netif_napi_del(&channel->napi_str); 1855874aeea5SJeff Kirsher channel->napi_dev = NULL; 1856874aeea5SJeff Kirsher } 1857874aeea5SJeff Kirsher 1858874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx) 1859874aeea5SJeff Kirsher { 1860874aeea5SJeff Kirsher struct efx_channel *channel; 1861874aeea5SJeff Kirsher 1862874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1863874aeea5SJeff Kirsher efx_fini_napi_channel(channel); 1864874aeea5SJeff Kirsher } 1865874aeea5SJeff Kirsher 1866874aeea5SJeff Kirsher /************************************************************************** 1867874aeea5SJeff Kirsher * 1868874aeea5SJeff Kirsher * Kernel netpoll interface 1869874aeea5SJeff Kirsher * 1870874aeea5SJeff Kirsher *************************************************************************/ 1871874aeea5SJeff Kirsher 1872874aeea5SJeff Kirsher #ifdef CONFIG_NET_POLL_CONTROLLER 1873874aeea5SJeff Kirsher 1874874aeea5SJeff Kirsher /* Although in the common case interrupts will be disabled, this is not 1875874aeea5SJeff Kirsher * guaranteed. However, all our work happens inside the NAPI callback, 1876874aeea5SJeff Kirsher * so no locking is required. 1877874aeea5SJeff Kirsher */ 1878874aeea5SJeff Kirsher static void efx_netpoll(struct net_device *net_dev) 1879874aeea5SJeff Kirsher { 1880874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 1881874aeea5SJeff Kirsher struct efx_channel *channel; 1882874aeea5SJeff Kirsher 1883874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1884874aeea5SJeff Kirsher efx_schedule_channel(channel); 1885874aeea5SJeff Kirsher } 1886874aeea5SJeff Kirsher 1887874aeea5SJeff Kirsher #endif 1888874aeea5SJeff Kirsher 1889874aeea5SJeff Kirsher /************************************************************************** 1890874aeea5SJeff Kirsher * 1891874aeea5SJeff Kirsher * Kernel net device interface 1892874aeea5SJeff Kirsher * 1893874aeea5SJeff Kirsher *************************************************************************/ 1894874aeea5SJeff Kirsher 1895874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 1896874aeea5SJeff Kirsher static int efx_net_open(struct net_device *net_dev) 1897874aeea5SJeff Kirsher { 1898874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 18998b7325b4SBen Hutchings int rc; 19008b7325b4SBen Hutchings 1901874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 1902874aeea5SJeff Kirsher raw_smp_processor_id()); 1903874aeea5SJeff Kirsher 19048b7325b4SBen Hutchings rc = efx_check_disabled(efx); 19058b7325b4SBen Hutchings if (rc) 19068b7325b4SBen Hutchings return rc; 1907874aeea5SJeff Kirsher if (efx->phy_mode & PHY_MODE_SPECIAL) 1908874aeea5SJeff Kirsher return -EBUSY; 1909874aeea5SJeff Kirsher if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 1910874aeea5SJeff Kirsher return -EIO; 1911874aeea5SJeff Kirsher 1912874aeea5SJeff Kirsher /* Notify the kernel of the link state polled during driver load, 1913874aeea5SJeff Kirsher * before the monitor starts running */ 1914874aeea5SJeff Kirsher efx_link_status_changed(efx); 1915874aeea5SJeff Kirsher 1916874aeea5SJeff Kirsher efx_start_all(efx); 1917dd40781eSBen Hutchings efx_selftest_async_start(efx); 1918874aeea5SJeff Kirsher return 0; 1919874aeea5SJeff Kirsher } 1920874aeea5SJeff Kirsher 1921874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. 1922874aeea5SJeff Kirsher * Note that the kernel will ignore our return code; this method 1923874aeea5SJeff Kirsher * should really be a void. 1924874aeea5SJeff Kirsher */ 1925874aeea5SJeff Kirsher static int efx_net_stop(struct net_device *net_dev) 1926874aeea5SJeff Kirsher { 1927874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 1928874aeea5SJeff Kirsher 1929874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 1930874aeea5SJeff Kirsher raw_smp_processor_id()); 1931874aeea5SJeff Kirsher 1932874aeea5SJeff Kirsher /* Stop the device and flush all the channels */ 1933874aeea5SJeff Kirsher efx_stop_all(efx); 1934874aeea5SJeff Kirsher 1935874aeea5SJeff Kirsher return 0; 1936874aeea5SJeff Kirsher } 1937874aeea5SJeff Kirsher 1938874aeea5SJeff Kirsher /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 19392aa9ef11SBen Hutchings static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev, 19402aa9ef11SBen Hutchings struct rtnl_link_stats64 *stats) 1941874aeea5SJeff Kirsher { 1942874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 1943874aeea5SJeff Kirsher struct efx_mac_stats *mac_stats = &efx->mac_stats; 1944874aeea5SJeff Kirsher 1945874aeea5SJeff Kirsher spin_lock_bh(&efx->stats_lock); 19461cb34522SBen Hutchings 1947874aeea5SJeff Kirsher efx->type->update_stats(efx); 1948874aeea5SJeff Kirsher 1949874aeea5SJeff Kirsher stats->rx_packets = mac_stats->rx_packets; 1950874aeea5SJeff Kirsher stats->tx_packets = mac_stats->tx_packets; 1951874aeea5SJeff Kirsher stats->rx_bytes = mac_stats->rx_bytes; 1952874aeea5SJeff Kirsher stats->tx_bytes = mac_stats->tx_bytes; 1953874aeea5SJeff Kirsher stats->rx_dropped = efx->n_rx_nodesc_drop_cnt; 1954874aeea5SJeff Kirsher stats->multicast = mac_stats->rx_multicast; 1955874aeea5SJeff Kirsher stats->collisions = mac_stats->tx_collision; 1956874aeea5SJeff Kirsher stats->rx_length_errors = (mac_stats->rx_gtjumbo + 1957874aeea5SJeff Kirsher mac_stats->rx_length_error); 1958874aeea5SJeff Kirsher stats->rx_crc_errors = mac_stats->rx_bad; 1959874aeea5SJeff Kirsher stats->rx_frame_errors = mac_stats->rx_align_error; 1960874aeea5SJeff Kirsher stats->rx_fifo_errors = mac_stats->rx_overflow; 1961874aeea5SJeff Kirsher stats->rx_missed_errors = mac_stats->rx_missed; 1962874aeea5SJeff Kirsher stats->tx_window_errors = mac_stats->tx_late_collision; 1963874aeea5SJeff Kirsher 1964874aeea5SJeff Kirsher stats->rx_errors = (stats->rx_length_errors + 1965874aeea5SJeff Kirsher stats->rx_crc_errors + 1966874aeea5SJeff Kirsher stats->rx_frame_errors + 1967874aeea5SJeff Kirsher mac_stats->rx_symbol_error); 1968874aeea5SJeff Kirsher stats->tx_errors = (stats->tx_window_errors + 1969874aeea5SJeff Kirsher mac_stats->tx_bad); 1970874aeea5SJeff Kirsher 19711cb34522SBen Hutchings spin_unlock_bh(&efx->stats_lock); 19721cb34522SBen Hutchings 1973874aeea5SJeff Kirsher return stats; 1974874aeea5SJeff Kirsher } 1975874aeea5SJeff Kirsher 1976874aeea5SJeff Kirsher /* Context: netif_tx_lock held, BHs disabled. */ 1977874aeea5SJeff Kirsher static void efx_watchdog(struct net_device *net_dev) 1978874aeea5SJeff Kirsher { 1979874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 1980874aeea5SJeff Kirsher 1981874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 1982874aeea5SJeff Kirsher "TX stuck with port_enabled=%d: resetting channels\n", 1983874aeea5SJeff Kirsher efx->port_enabled); 1984874aeea5SJeff Kirsher 1985874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 1986874aeea5SJeff Kirsher } 1987874aeea5SJeff Kirsher 1988874aeea5SJeff Kirsher 1989874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 1990874aeea5SJeff Kirsher static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 1991874aeea5SJeff Kirsher { 1992874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 19938b7325b4SBen Hutchings int rc; 1994874aeea5SJeff Kirsher 19958b7325b4SBen Hutchings rc = efx_check_disabled(efx); 19968b7325b4SBen Hutchings if (rc) 19978b7325b4SBen Hutchings return rc; 1998874aeea5SJeff Kirsher if (new_mtu > EFX_MAX_MTU) 1999874aeea5SJeff Kirsher return -EINVAL; 2000874aeea5SJeff Kirsher 2001874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2002874aeea5SJeff Kirsher 200329c69a48SBen Hutchings efx_device_detach_sync(efx); 200429c69a48SBen Hutchings efx_stop_all(efx); 200529c69a48SBen Hutchings 2006874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2007874aeea5SJeff Kirsher net_dev->mtu = new_mtu; 2008710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 2009874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2010874aeea5SJeff Kirsher 2011874aeea5SJeff Kirsher efx_start_all(efx); 201229c69a48SBen Hutchings netif_device_attach(efx->net_dev); 20136c8eef4aSBen Hutchings return 0; 2014874aeea5SJeff Kirsher } 2015874aeea5SJeff Kirsher 2016874aeea5SJeff Kirsher static int efx_set_mac_address(struct net_device *net_dev, void *data) 2017874aeea5SJeff Kirsher { 2018874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2019874aeea5SJeff Kirsher struct sockaddr *addr = data; 2020874aeea5SJeff Kirsher char *new_addr = addr->sa_data; 2021874aeea5SJeff Kirsher 2022874aeea5SJeff Kirsher if (!is_valid_ether_addr(new_addr)) { 2023874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2024874aeea5SJeff Kirsher "invalid ethernet MAC address requested: %pM\n", 2025874aeea5SJeff Kirsher new_addr); 2026504f9b5aSDanny Kukawka return -EADDRNOTAVAIL; 2027874aeea5SJeff Kirsher } 2028874aeea5SJeff Kirsher 2029874aeea5SJeff Kirsher memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len); 2030cd2d5b52SBen Hutchings efx_sriov_mac_address_changed(efx); 2031874aeea5SJeff Kirsher 2032874aeea5SJeff Kirsher /* Reconfigure the MAC */ 2033874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2034710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 2035874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2036874aeea5SJeff Kirsher 2037874aeea5SJeff Kirsher return 0; 2038874aeea5SJeff Kirsher } 2039874aeea5SJeff Kirsher 2040874aeea5SJeff Kirsher /* Context: netif_addr_lock held, BHs disabled. */ 20410fca8c97SBen Hutchings static void efx_set_rx_mode(struct net_device *net_dev) 2042874aeea5SJeff Kirsher { 2043874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2044874aeea5SJeff Kirsher struct netdev_hw_addr *ha; 2045874aeea5SJeff Kirsher union efx_multicast_hash *mc_hash = &efx->multicast_hash; 2046874aeea5SJeff Kirsher u32 crc; 2047874aeea5SJeff Kirsher int bit; 2048874aeea5SJeff Kirsher 2049874aeea5SJeff Kirsher efx->promiscuous = !!(net_dev->flags & IFF_PROMISC); 2050874aeea5SJeff Kirsher 2051874aeea5SJeff Kirsher /* Build multicast hash table */ 2052874aeea5SJeff Kirsher if (efx->promiscuous || (net_dev->flags & IFF_ALLMULTI)) { 2053874aeea5SJeff Kirsher memset(mc_hash, 0xff, sizeof(*mc_hash)); 2054874aeea5SJeff Kirsher } else { 2055874aeea5SJeff Kirsher memset(mc_hash, 0x00, sizeof(*mc_hash)); 2056874aeea5SJeff Kirsher netdev_for_each_mc_addr(ha, net_dev) { 2057874aeea5SJeff Kirsher crc = ether_crc_le(ETH_ALEN, ha->addr); 2058874aeea5SJeff Kirsher bit = crc & (EFX_MCAST_HASH_ENTRIES - 1); 205932766ec8SBen Hutchings __set_bit_le(bit, mc_hash); 2060874aeea5SJeff Kirsher } 2061874aeea5SJeff Kirsher 2062874aeea5SJeff Kirsher /* Broadcast packets go through the multicast hash filter. 2063874aeea5SJeff Kirsher * ether_crc_le() of the broadcast address is 0xbe2612ff 2064874aeea5SJeff Kirsher * so we always add bit 0xff to the mask. 2065874aeea5SJeff Kirsher */ 206632766ec8SBen Hutchings __set_bit_le(0xff, mc_hash); 2067874aeea5SJeff Kirsher } 2068874aeea5SJeff Kirsher 2069874aeea5SJeff Kirsher if (efx->port_enabled) 2070874aeea5SJeff Kirsher queue_work(efx->workqueue, &efx->mac_work); 2071874aeea5SJeff Kirsher /* Otherwise efx_start_port() will do this */ 2072874aeea5SJeff Kirsher } 2073874aeea5SJeff Kirsher 2074c8f44affSMichał Mirosław static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2075874aeea5SJeff Kirsher { 2076874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2077874aeea5SJeff Kirsher 2078874aeea5SJeff Kirsher /* If disabling RX n-tuple filtering, clear existing filters */ 2079874aeea5SJeff Kirsher if (net_dev->features & ~data & NETIF_F_NTUPLE) 2080874aeea5SJeff Kirsher efx_filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2081874aeea5SJeff Kirsher 2082874aeea5SJeff Kirsher return 0; 2083874aeea5SJeff Kirsher } 2084874aeea5SJeff Kirsher 2085874aeea5SJeff Kirsher static const struct net_device_ops efx_netdev_ops = { 2086874aeea5SJeff Kirsher .ndo_open = efx_net_open, 2087874aeea5SJeff Kirsher .ndo_stop = efx_net_stop, 2088874aeea5SJeff Kirsher .ndo_get_stats64 = efx_net_stats, 2089874aeea5SJeff Kirsher .ndo_tx_timeout = efx_watchdog, 2090874aeea5SJeff Kirsher .ndo_start_xmit = efx_hard_start_xmit, 2091874aeea5SJeff Kirsher .ndo_validate_addr = eth_validate_addr, 2092874aeea5SJeff Kirsher .ndo_do_ioctl = efx_ioctl, 2093874aeea5SJeff Kirsher .ndo_change_mtu = efx_change_mtu, 2094874aeea5SJeff Kirsher .ndo_set_mac_address = efx_set_mac_address, 20950fca8c97SBen Hutchings .ndo_set_rx_mode = efx_set_rx_mode, 2096874aeea5SJeff Kirsher .ndo_set_features = efx_set_features, 2097cd2d5b52SBen Hutchings #ifdef CONFIG_SFC_SRIOV 2098cd2d5b52SBen Hutchings .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2099cd2d5b52SBen Hutchings .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2100cd2d5b52SBen Hutchings .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2101cd2d5b52SBen Hutchings .ndo_get_vf_config = efx_sriov_get_vf_config, 2102cd2d5b52SBen Hutchings #endif 2103874aeea5SJeff Kirsher #ifdef CONFIG_NET_POLL_CONTROLLER 2104874aeea5SJeff Kirsher .ndo_poll_controller = efx_netpoll, 2105874aeea5SJeff Kirsher #endif 2106874aeea5SJeff Kirsher .ndo_setup_tc = efx_setup_tc, 2107874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 2108874aeea5SJeff Kirsher .ndo_rx_flow_steer = efx_filter_rfs, 2109874aeea5SJeff Kirsher #endif 2110874aeea5SJeff Kirsher }; 2111874aeea5SJeff Kirsher 2112874aeea5SJeff Kirsher static void efx_update_name(struct efx_nic *efx) 2113874aeea5SJeff Kirsher { 2114874aeea5SJeff Kirsher strcpy(efx->name, efx->net_dev->name); 2115874aeea5SJeff Kirsher efx_mtd_rename(efx); 2116874aeea5SJeff Kirsher efx_set_channel_names(efx); 2117874aeea5SJeff Kirsher } 2118874aeea5SJeff Kirsher 2119874aeea5SJeff Kirsher static int efx_netdev_event(struct notifier_block *this, 2120874aeea5SJeff Kirsher unsigned long event, void *ptr) 2121874aeea5SJeff Kirsher { 2122874aeea5SJeff Kirsher struct net_device *net_dev = ptr; 2123874aeea5SJeff Kirsher 2124874aeea5SJeff Kirsher if (net_dev->netdev_ops == &efx_netdev_ops && 2125874aeea5SJeff Kirsher event == NETDEV_CHANGENAME) 2126874aeea5SJeff Kirsher efx_update_name(netdev_priv(net_dev)); 2127874aeea5SJeff Kirsher 2128874aeea5SJeff Kirsher return NOTIFY_DONE; 2129874aeea5SJeff Kirsher } 2130874aeea5SJeff Kirsher 2131874aeea5SJeff Kirsher static struct notifier_block efx_netdev_notifier = { 2132874aeea5SJeff Kirsher .notifier_call = efx_netdev_event, 2133874aeea5SJeff Kirsher }; 2134874aeea5SJeff Kirsher 2135874aeea5SJeff Kirsher static ssize_t 2136874aeea5SJeff Kirsher show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2137874aeea5SJeff Kirsher { 2138874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2139874aeea5SJeff Kirsher return sprintf(buf, "%d\n", efx->phy_type); 2140874aeea5SJeff Kirsher } 2141874aeea5SJeff Kirsher static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL); 2142874aeea5SJeff Kirsher 2143874aeea5SJeff Kirsher static int efx_register_netdev(struct efx_nic *efx) 2144874aeea5SJeff Kirsher { 2145874aeea5SJeff Kirsher struct net_device *net_dev = efx->net_dev; 2146874aeea5SJeff Kirsher struct efx_channel *channel; 2147874aeea5SJeff Kirsher int rc; 2148874aeea5SJeff Kirsher 2149874aeea5SJeff Kirsher net_dev->watchdog_timeo = 5 * HZ; 2150874aeea5SJeff Kirsher net_dev->irq = efx->pci_dev->irq; 2151874aeea5SJeff Kirsher net_dev->netdev_ops = &efx_netdev_ops; 2152874aeea5SJeff Kirsher SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops); 21537e6d06f0SBen Hutchings net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2154874aeea5SJeff Kirsher 2155874aeea5SJeff Kirsher rtnl_lock(); 2156874aeea5SJeff Kirsher 21577153f623SBen Hutchings /* Enable resets to be scheduled and check whether any were 21587153f623SBen Hutchings * already requested. If so, the NIC is probably hosed so we 21597153f623SBen Hutchings * abort. 21607153f623SBen Hutchings */ 21617153f623SBen Hutchings efx->state = STATE_READY; 21627153f623SBen Hutchings smp_mb(); /* ensure we change state before checking reset_pending */ 21637153f623SBen Hutchings if (efx->reset_pending) { 21647153f623SBen Hutchings netif_err(efx, probe, efx->net_dev, 21657153f623SBen Hutchings "aborting probe due to scheduled reset\n"); 21667153f623SBen Hutchings rc = -EIO; 21677153f623SBen Hutchings goto fail_locked; 21687153f623SBen Hutchings } 21697153f623SBen Hutchings 2170874aeea5SJeff Kirsher rc = dev_alloc_name(net_dev, net_dev->name); 2171874aeea5SJeff Kirsher if (rc < 0) 2172874aeea5SJeff Kirsher goto fail_locked; 2173874aeea5SJeff Kirsher efx_update_name(efx); 2174874aeea5SJeff Kirsher 21758f8b3d51SBen Hutchings /* Always start with carrier off; PHY events will detect the link */ 21768f8b3d51SBen Hutchings netif_carrier_off(net_dev); 21778f8b3d51SBen Hutchings 2178874aeea5SJeff Kirsher rc = register_netdevice(net_dev); 2179874aeea5SJeff Kirsher if (rc) 2180874aeea5SJeff Kirsher goto fail_locked; 2181874aeea5SJeff Kirsher 2182874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2183874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 2184874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 2185874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 2186874aeea5SJeff Kirsher } 2187874aeea5SJeff Kirsher 2188874aeea5SJeff Kirsher rtnl_unlock(); 2189874aeea5SJeff Kirsher 2190874aeea5SJeff Kirsher rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2191874aeea5SJeff Kirsher if (rc) { 2192874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2193874aeea5SJeff Kirsher "failed to init net dev attributes\n"); 2194874aeea5SJeff Kirsher goto fail_registered; 2195874aeea5SJeff Kirsher } 2196874aeea5SJeff Kirsher 2197874aeea5SJeff Kirsher return 0; 2198874aeea5SJeff Kirsher 21997153f623SBen Hutchings fail_registered: 22007153f623SBen Hutchings rtnl_lock(); 22017153f623SBen Hutchings unregister_netdevice(net_dev); 2202874aeea5SJeff Kirsher fail_locked: 22037153f623SBen Hutchings efx->state = STATE_UNINIT; 2204874aeea5SJeff Kirsher rtnl_unlock(); 2205874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2206874aeea5SJeff Kirsher return rc; 2207874aeea5SJeff Kirsher } 2208874aeea5SJeff Kirsher 2209874aeea5SJeff Kirsher static void efx_unregister_netdev(struct efx_nic *efx) 2210874aeea5SJeff Kirsher { 2211874aeea5SJeff Kirsher struct efx_channel *channel; 2212874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 2213874aeea5SJeff Kirsher 2214874aeea5SJeff Kirsher if (!efx->net_dev) 2215874aeea5SJeff Kirsher return; 2216874aeea5SJeff Kirsher 2217874aeea5SJeff Kirsher BUG_ON(netdev_priv(efx->net_dev) != efx); 2218874aeea5SJeff Kirsher 2219874aeea5SJeff Kirsher /* Free up any skbs still remaining. This has to happen before 2220874aeea5SJeff Kirsher * we try to unregister the netdev as running their destructors 2221874aeea5SJeff Kirsher * may be needed to get the device ref. count to 0. */ 2222874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2223874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 2224874aeea5SJeff Kirsher efx_release_tx_buffers(tx_queue); 2225874aeea5SJeff Kirsher } 2226874aeea5SJeff Kirsher 2227874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2228874aeea5SJeff Kirsher device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 22297153f623SBen Hutchings 22307153f623SBen Hutchings rtnl_lock(); 22317153f623SBen Hutchings unregister_netdevice(efx->net_dev); 22327153f623SBen Hutchings efx->state = STATE_UNINIT; 22337153f623SBen Hutchings rtnl_unlock(); 2234874aeea5SJeff Kirsher } 2235874aeea5SJeff Kirsher 2236874aeea5SJeff Kirsher /************************************************************************** 2237874aeea5SJeff Kirsher * 2238874aeea5SJeff Kirsher * Device reset and suspend 2239874aeea5SJeff Kirsher * 2240874aeea5SJeff Kirsher **************************************************************************/ 2241874aeea5SJeff Kirsher 2242874aeea5SJeff Kirsher /* Tears down the entire software state and most of the hardware state 2243874aeea5SJeff Kirsher * before reset. */ 2244874aeea5SJeff Kirsher void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2245874aeea5SJeff Kirsher { 2246874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2247874aeea5SJeff Kirsher 2248874aeea5SJeff Kirsher efx_stop_all(efx); 22497f967c01SBen Hutchings efx_stop_interrupts(efx, false); 22505642ceefSBen Hutchings 22515642ceefSBen Hutchings mutex_lock(&efx->mac_lock); 2252874aeea5SJeff Kirsher if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) 2253874aeea5SJeff Kirsher efx->phy_op->fini(efx); 2254874aeea5SJeff Kirsher efx->type->fini(efx); 2255874aeea5SJeff Kirsher } 2256874aeea5SJeff Kirsher 2257874aeea5SJeff Kirsher /* This function will always ensure that the locks acquired in 2258874aeea5SJeff Kirsher * efx_reset_down() are released. A failure return code indicates 2259874aeea5SJeff Kirsher * that we were unable to reinitialise the hardware, and the 2260874aeea5SJeff Kirsher * driver should be disabled. If ok is false, then the rx and tx 2261874aeea5SJeff Kirsher * engines are not restarted, pending a RESET_DISABLE. */ 2262874aeea5SJeff Kirsher int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2263874aeea5SJeff Kirsher { 2264874aeea5SJeff Kirsher int rc; 2265874aeea5SJeff Kirsher 2266874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2267874aeea5SJeff Kirsher 2268874aeea5SJeff Kirsher rc = efx->type->init(efx); 2269874aeea5SJeff Kirsher if (rc) { 2270874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2271874aeea5SJeff Kirsher goto fail; 2272874aeea5SJeff Kirsher } 2273874aeea5SJeff Kirsher 2274874aeea5SJeff Kirsher if (!ok) 2275874aeea5SJeff Kirsher goto fail; 2276874aeea5SJeff Kirsher 2277874aeea5SJeff Kirsher if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) { 2278874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 2279874aeea5SJeff Kirsher if (rc) 2280874aeea5SJeff Kirsher goto fail; 2281874aeea5SJeff Kirsher if (efx->phy_op->reconfigure(efx)) 2282874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2283874aeea5SJeff Kirsher "could not restore PHY settings\n"); 2284874aeea5SJeff Kirsher } 2285874aeea5SJeff Kirsher 2286710b208dSBen Hutchings efx->type->reconfigure_mac(efx); 2287874aeea5SJeff Kirsher 22887f967c01SBen Hutchings efx_start_interrupts(efx, false); 2289874aeea5SJeff Kirsher efx_restore_filters(efx); 2290cd2d5b52SBen Hutchings efx_sriov_reset(efx); 2291874aeea5SJeff Kirsher 2292874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2293874aeea5SJeff Kirsher 2294874aeea5SJeff Kirsher efx_start_all(efx); 2295874aeea5SJeff Kirsher 2296874aeea5SJeff Kirsher return 0; 2297874aeea5SJeff Kirsher 2298874aeea5SJeff Kirsher fail: 2299874aeea5SJeff Kirsher efx->port_initialized = false; 2300874aeea5SJeff Kirsher 2301874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2302874aeea5SJeff Kirsher 2303874aeea5SJeff Kirsher return rc; 2304874aeea5SJeff Kirsher } 2305874aeea5SJeff Kirsher 2306874aeea5SJeff Kirsher /* Reset the NIC using the specified method. Note that the reset may 2307874aeea5SJeff Kirsher * fail, in which case the card will be left in an unusable state. 2308874aeea5SJeff Kirsher * 2309874aeea5SJeff Kirsher * Caller must hold the rtnl_lock. 2310874aeea5SJeff Kirsher */ 2311874aeea5SJeff Kirsher int efx_reset(struct efx_nic *efx, enum reset_type method) 2312874aeea5SJeff Kirsher { 2313874aeea5SJeff Kirsher int rc, rc2; 2314874aeea5SJeff Kirsher bool disabled; 2315874aeea5SJeff Kirsher 2316874aeea5SJeff Kirsher netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2317874aeea5SJeff Kirsher RESET_TYPE(method)); 2318874aeea5SJeff Kirsher 2319c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2320874aeea5SJeff Kirsher efx_reset_down(efx, method); 2321874aeea5SJeff Kirsher 2322874aeea5SJeff Kirsher rc = efx->type->reset(efx, method); 2323874aeea5SJeff Kirsher if (rc) { 2324874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2325874aeea5SJeff Kirsher goto out; 2326874aeea5SJeff Kirsher } 2327874aeea5SJeff Kirsher 2328874aeea5SJeff Kirsher /* Clear flags for the scopes we covered. We assume the NIC and 2329874aeea5SJeff Kirsher * driver are now quiescent so that there is no race here. 2330874aeea5SJeff Kirsher */ 2331874aeea5SJeff Kirsher efx->reset_pending &= -(1 << (method + 1)); 2332874aeea5SJeff Kirsher 2333874aeea5SJeff Kirsher /* Reinitialise bus-mastering, which may have been turned off before 2334874aeea5SJeff Kirsher * the reset was scheduled. This is still appropriate, even in the 2335874aeea5SJeff Kirsher * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2336874aeea5SJeff Kirsher * can respond to requests. */ 2337874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 2338874aeea5SJeff Kirsher 2339874aeea5SJeff Kirsher out: 2340874aeea5SJeff Kirsher /* Leave device stopped if necessary */ 2341626950dbSAlexandre Rames disabled = rc || 2342626950dbSAlexandre Rames method == RESET_TYPE_DISABLE || 2343626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_DISABLE; 2344874aeea5SJeff Kirsher rc2 = efx_reset_up(efx, method, !disabled); 2345874aeea5SJeff Kirsher if (rc2) { 2346874aeea5SJeff Kirsher disabled = true; 2347874aeea5SJeff Kirsher if (!rc) 2348874aeea5SJeff Kirsher rc = rc2; 2349874aeea5SJeff Kirsher } 2350874aeea5SJeff Kirsher 2351874aeea5SJeff Kirsher if (disabled) { 2352874aeea5SJeff Kirsher dev_close(efx->net_dev); 2353874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2354874aeea5SJeff Kirsher efx->state = STATE_DISABLED; 2355874aeea5SJeff Kirsher } else { 2356874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2357874aeea5SJeff Kirsher netif_device_attach(efx->net_dev); 2358874aeea5SJeff Kirsher } 2359874aeea5SJeff Kirsher return rc; 2360874aeea5SJeff Kirsher } 2361874aeea5SJeff Kirsher 2362626950dbSAlexandre Rames /* Try recovery mechanisms. 2363626950dbSAlexandre Rames * For now only EEH is supported. 2364626950dbSAlexandre Rames * Returns 0 if the recovery mechanisms are unsuccessful. 2365626950dbSAlexandre Rames * Returns a non-zero value otherwise. 2366626950dbSAlexandre Rames */ 2367626950dbSAlexandre Rames static int efx_try_recovery(struct efx_nic *efx) 2368626950dbSAlexandre Rames { 2369626950dbSAlexandre Rames #ifdef CONFIG_EEH 2370626950dbSAlexandre Rames /* A PCI error can occur and not be seen by EEH because nothing 2371626950dbSAlexandre Rames * happens on the PCI bus. In this case the driver may fail and 2372626950dbSAlexandre Rames * schedule a 'recover or reset', leading to this recovery handler. 2373626950dbSAlexandre Rames * Manually call the eeh failure check function. 2374626950dbSAlexandre Rames */ 2375626950dbSAlexandre Rames struct eeh_dev *eehdev = 2376626950dbSAlexandre Rames of_node_to_eeh_dev(pci_device_to_OF_node(efx->pci_dev)); 2377626950dbSAlexandre Rames 2378626950dbSAlexandre Rames if (eeh_dev_check_failure(eehdev)) { 2379626950dbSAlexandre Rames /* The EEH mechanisms will handle the error and reset the 2380626950dbSAlexandre Rames * device if necessary. 2381626950dbSAlexandre Rames */ 2382626950dbSAlexandre Rames return 1; 2383626950dbSAlexandre Rames } 2384626950dbSAlexandre Rames #endif 2385626950dbSAlexandre Rames return 0; 2386626950dbSAlexandre Rames } 2387626950dbSAlexandre Rames 2388874aeea5SJeff Kirsher /* The worker thread exists so that code that cannot sleep can 2389874aeea5SJeff Kirsher * schedule a reset for later. 2390874aeea5SJeff Kirsher */ 2391874aeea5SJeff Kirsher static void efx_reset_work(struct work_struct *data) 2392874aeea5SJeff Kirsher { 2393874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2394626950dbSAlexandre Rames unsigned long pending; 2395626950dbSAlexandre Rames enum reset_type method; 2396626950dbSAlexandre Rames 2397626950dbSAlexandre Rames pending = ACCESS_ONCE(efx->reset_pending); 2398626950dbSAlexandre Rames method = fls(pending) - 1; 2399626950dbSAlexandre Rames 2400626950dbSAlexandre Rames if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2401626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_ALL) && 2402626950dbSAlexandre Rames efx_try_recovery(efx)) 2403626950dbSAlexandre Rames return; 2404874aeea5SJeff Kirsher 2405874aeea5SJeff Kirsher if (!pending) 2406874aeea5SJeff Kirsher return; 2407874aeea5SJeff Kirsher 2408874aeea5SJeff Kirsher rtnl_lock(); 24097153f623SBen Hutchings 24107153f623SBen Hutchings /* We checked the state in efx_schedule_reset() but it may 24117153f623SBen Hutchings * have changed by now. Now that we have the RTNL lock, 24127153f623SBen Hutchings * it cannot change again. 24137153f623SBen Hutchings */ 24147153f623SBen Hutchings if (efx->state == STATE_READY) 2415626950dbSAlexandre Rames (void)efx_reset(efx, method); 24167153f623SBen Hutchings 2417874aeea5SJeff Kirsher rtnl_unlock(); 2418874aeea5SJeff Kirsher } 2419874aeea5SJeff Kirsher 2420874aeea5SJeff Kirsher void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2421874aeea5SJeff Kirsher { 2422874aeea5SJeff Kirsher enum reset_type method; 2423874aeea5SJeff Kirsher 2424626950dbSAlexandre Rames if (efx->state == STATE_RECOVERY) { 2425626950dbSAlexandre Rames netif_dbg(efx, drv, efx->net_dev, 2426626950dbSAlexandre Rames "recovering: skip scheduling %s reset\n", 2427626950dbSAlexandre Rames RESET_TYPE(type)); 2428626950dbSAlexandre Rames return; 2429626950dbSAlexandre Rames } 2430626950dbSAlexandre Rames 2431874aeea5SJeff Kirsher switch (type) { 2432874aeea5SJeff Kirsher case RESET_TYPE_INVISIBLE: 2433874aeea5SJeff Kirsher case RESET_TYPE_ALL: 2434626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_ALL: 2435874aeea5SJeff Kirsher case RESET_TYPE_WORLD: 2436874aeea5SJeff Kirsher case RESET_TYPE_DISABLE: 2437626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_DISABLE: 2438874aeea5SJeff Kirsher method = type; 2439874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2440874aeea5SJeff Kirsher RESET_TYPE(method)); 2441874aeea5SJeff Kirsher break; 2442874aeea5SJeff Kirsher default: 2443874aeea5SJeff Kirsher method = efx->type->map_reset_reason(type); 2444874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, 2445874aeea5SJeff Kirsher "scheduling %s reset for %s\n", 2446874aeea5SJeff Kirsher RESET_TYPE(method), RESET_TYPE(type)); 2447874aeea5SJeff Kirsher break; 2448874aeea5SJeff Kirsher } 2449874aeea5SJeff Kirsher 2450874aeea5SJeff Kirsher set_bit(method, &efx->reset_pending); 24517153f623SBen Hutchings smp_mb(); /* ensure we change reset_pending before checking state */ 24527153f623SBen Hutchings 24537153f623SBen Hutchings /* If we're not READY then just leave the flags set as the cue 24547153f623SBen Hutchings * to abort probing or reschedule the reset later. 24557153f623SBen Hutchings */ 24567153f623SBen Hutchings if (ACCESS_ONCE(efx->state) != STATE_READY) 24577153f623SBen Hutchings return; 2458874aeea5SJeff Kirsher 2459874aeea5SJeff Kirsher /* efx_process_channel() will no longer read events once a 2460874aeea5SJeff Kirsher * reset is scheduled. So switch back to poll'd MCDI completions. */ 2461874aeea5SJeff Kirsher efx_mcdi_mode_poll(efx); 2462874aeea5SJeff Kirsher 2463874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 2464874aeea5SJeff Kirsher } 2465874aeea5SJeff Kirsher 2466874aeea5SJeff Kirsher /************************************************************************** 2467874aeea5SJeff Kirsher * 2468874aeea5SJeff Kirsher * List of NICs we support 2469874aeea5SJeff Kirsher * 2470874aeea5SJeff Kirsher **************************************************************************/ 2471874aeea5SJeff Kirsher 2472874aeea5SJeff Kirsher /* PCI device ID table */ 2473874aeea5SJeff Kirsher static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = { 24740e59e7e7SLinus Torvalds {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 24750e59e7e7SLinus Torvalds PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0), 2476874aeea5SJeff Kirsher .driver_data = (unsigned long) &falcon_a1_nic_type}, 24770e59e7e7SLinus Torvalds {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 24780e59e7e7SLinus Torvalds PCI_DEVICE_ID_SOLARFLARE_SFC4000B), 2479874aeea5SJeff Kirsher .driver_data = (unsigned long) &falcon_b0_nic_type}, 2480547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2481874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 2482547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2483874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 2484874aeea5SJeff Kirsher {0} /* end of list */ 2485874aeea5SJeff Kirsher }; 2486874aeea5SJeff Kirsher 2487874aeea5SJeff Kirsher /************************************************************************** 2488874aeea5SJeff Kirsher * 2489874aeea5SJeff Kirsher * Dummy PHY/MAC operations 2490874aeea5SJeff Kirsher * 2491874aeea5SJeff Kirsher * Can be used for some unimplemented operations 2492874aeea5SJeff Kirsher * Needed so all function pointers are valid and do not have to be tested 2493874aeea5SJeff Kirsher * before use 2494874aeea5SJeff Kirsher * 2495874aeea5SJeff Kirsher **************************************************************************/ 2496874aeea5SJeff Kirsher int efx_port_dummy_op_int(struct efx_nic *efx) 2497874aeea5SJeff Kirsher { 2498874aeea5SJeff Kirsher return 0; 2499874aeea5SJeff Kirsher } 2500874aeea5SJeff Kirsher void efx_port_dummy_op_void(struct efx_nic *efx) {} 2501874aeea5SJeff Kirsher 2502874aeea5SJeff Kirsher static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2503874aeea5SJeff Kirsher { 2504874aeea5SJeff Kirsher return false; 2505874aeea5SJeff Kirsher } 2506874aeea5SJeff Kirsher 2507874aeea5SJeff Kirsher static const struct efx_phy_operations efx_dummy_phy_operations = { 2508874aeea5SJeff Kirsher .init = efx_port_dummy_op_int, 2509874aeea5SJeff Kirsher .reconfigure = efx_port_dummy_op_int, 2510874aeea5SJeff Kirsher .poll = efx_port_dummy_op_poll, 2511874aeea5SJeff Kirsher .fini = efx_port_dummy_op_void, 2512874aeea5SJeff Kirsher }; 2513874aeea5SJeff Kirsher 2514874aeea5SJeff Kirsher /************************************************************************** 2515874aeea5SJeff Kirsher * 2516874aeea5SJeff Kirsher * Data housekeeping 2517874aeea5SJeff Kirsher * 2518874aeea5SJeff Kirsher **************************************************************************/ 2519874aeea5SJeff Kirsher 2520874aeea5SJeff Kirsher /* This zeroes out and then fills in the invariants in a struct 2521874aeea5SJeff Kirsher * efx_nic (including all sub-structures). 2522874aeea5SJeff Kirsher */ 2523adeb15aaSBen Hutchings static int efx_init_struct(struct efx_nic *efx, 2524874aeea5SJeff Kirsher struct pci_dev *pci_dev, struct net_device *net_dev) 2525874aeea5SJeff Kirsher { 2526874aeea5SJeff Kirsher int i; 2527874aeea5SJeff Kirsher 2528874aeea5SJeff Kirsher /* Initialise common structures */ 2529874aeea5SJeff Kirsher spin_lock_init(&efx->biu_lock); 2530874aeea5SJeff Kirsher #ifdef CONFIG_SFC_MTD 2531874aeea5SJeff Kirsher INIT_LIST_HEAD(&efx->mtd_list); 2532874aeea5SJeff Kirsher #endif 2533874aeea5SJeff Kirsher INIT_WORK(&efx->reset_work, efx_reset_work); 2534874aeea5SJeff Kirsher INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 2535dd40781eSBen Hutchings INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 2536874aeea5SJeff Kirsher efx->pci_dev = pci_dev; 2537874aeea5SJeff Kirsher efx->msg_enable = debug; 2538f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 2539874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2540874aeea5SJeff Kirsher 2541874aeea5SJeff Kirsher efx->net_dev = net_dev; 2542874aeea5SJeff Kirsher spin_lock_init(&efx->stats_lock); 2543874aeea5SJeff Kirsher mutex_init(&efx->mac_lock); 2544874aeea5SJeff Kirsher efx->phy_op = &efx_dummy_phy_operations; 2545874aeea5SJeff Kirsher efx->mdio.dev = net_dev; 2546874aeea5SJeff Kirsher INIT_WORK(&efx->mac_work, efx_mac_work); 25479f2cb71cSBen Hutchings init_waitqueue_head(&efx->flush_wq); 2548874aeea5SJeff Kirsher 2549874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) { 2550874aeea5SJeff Kirsher efx->channel[i] = efx_alloc_channel(efx, i, NULL); 2551874aeea5SJeff Kirsher if (!efx->channel[i]) 2552874aeea5SJeff Kirsher goto fail; 2553874aeea5SJeff Kirsher } 2554874aeea5SJeff Kirsher 2555874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS); 2556874aeea5SJeff Kirsher 2557874aeea5SJeff Kirsher /* Higher numbered interrupt modes are less capable! */ 2558874aeea5SJeff Kirsher efx->interrupt_mode = max(efx->type->max_interrupt_mode, 2559874aeea5SJeff Kirsher interrupt_mode); 2560874aeea5SJeff Kirsher 2561874aeea5SJeff Kirsher /* Would be good to use the net_dev name, but we're too early */ 2562874aeea5SJeff Kirsher snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 2563874aeea5SJeff Kirsher pci_name(pci_dev)); 2564874aeea5SJeff Kirsher efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 2565874aeea5SJeff Kirsher if (!efx->workqueue) 2566874aeea5SJeff Kirsher goto fail; 2567874aeea5SJeff Kirsher 2568874aeea5SJeff Kirsher return 0; 2569874aeea5SJeff Kirsher 2570874aeea5SJeff Kirsher fail: 2571874aeea5SJeff Kirsher efx_fini_struct(efx); 2572874aeea5SJeff Kirsher return -ENOMEM; 2573874aeea5SJeff Kirsher } 2574874aeea5SJeff Kirsher 2575874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx) 2576874aeea5SJeff Kirsher { 2577874aeea5SJeff Kirsher int i; 2578874aeea5SJeff Kirsher 2579874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) 2580874aeea5SJeff Kirsher kfree(efx->channel[i]); 2581874aeea5SJeff Kirsher 2582874aeea5SJeff Kirsher if (efx->workqueue) { 2583874aeea5SJeff Kirsher destroy_workqueue(efx->workqueue); 2584874aeea5SJeff Kirsher efx->workqueue = NULL; 2585874aeea5SJeff Kirsher } 2586874aeea5SJeff Kirsher } 2587874aeea5SJeff Kirsher 2588874aeea5SJeff Kirsher /************************************************************************** 2589874aeea5SJeff Kirsher * 2590874aeea5SJeff Kirsher * PCI interface 2591874aeea5SJeff Kirsher * 2592874aeea5SJeff Kirsher **************************************************************************/ 2593874aeea5SJeff Kirsher 2594874aeea5SJeff Kirsher /* Main body of final NIC shutdown code 2595874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 2596874aeea5SJeff Kirsher */ 2597874aeea5SJeff Kirsher static void efx_pci_remove_main(struct efx_nic *efx) 2598874aeea5SJeff Kirsher { 25997153f623SBen Hutchings /* Flush reset_work. It can no longer be scheduled since we 26007153f623SBen Hutchings * are not READY. 26017153f623SBen Hutchings */ 26027153f623SBen Hutchings BUG_ON(efx->state == STATE_READY); 26037153f623SBen Hutchings cancel_work_sync(&efx->reset_work); 26047153f623SBen Hutchings 2605874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 2606874aeea5SJeff Kirsher free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap); 2607874aeea5SJeff Kirsher efx->net_dev->rx_cpu_rmap = NULL; 2608874aeea5SJeff Kirsher #endif 26097f967c01SBen Hutchings efx_stop_interrupts(efx, false); 2610874aeea5SJeff Kirsher efx_nic_fini_interrupt(efx); 2611874aeea5SJeff Kirsher efx_fini_port(efx); 2612874aeea5SJeff Kirsher efx->type->fini(efx); 2613874aeea5SJeff Kirsher efx_fini_napi(efx); 2614874aeea5SJeff Kirsher efx_remove_all(efx); 2615874aeea5SJeff Kirsher } 2616874aeea5SJeff Kirsher 2617874aeea5SJeff Kirsher /* Final NIC shutdown 2618874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 2619874aeea5SJeff Kirsher */ 2620874aeea5SJeff Kirsher static void efx_pci_remove(struct pci_dev *pci_dev) 2621874aeea5SJeff Kirsher { 2622874aeea5SJeff Kirsher struct efx_nic *efx; 2623874aeea5SJeff Kirsher 2624874aeea5SJeff Kirsher efx = pci_get_drvdata(pci_dev); 2625874aeea5SJeff Kirsher if (!efx) 2626874aeea5SJeff Kirsher return; 2627874aeea5SJeff Kirsher 2628874aeea5SJeff Kirsher /* Mark the NIC as fini, then stop the interface */ 2629874aeea5SJeff Kirsher rtnl_lock(); 2630874aeea5SJeff Kirsher dev_close(efx->net_dev); 26315642ceefSBen Hutchings efx_stop_interrupts(efx, false); 2632874aeea5SJeff Kirsher rtnl_unlock(); 2633874aeea5SJeff Kirsher 2634cd2d5b52SBen Hutchings efx_sriov_fini(efx); 2635874aeea5SJeff Kirsher efx_unregister_netdev(efx); 2636874aeea5SJeff Kirsher 2637874aeea5SJeff Kirsher efx_mtd_remove(efx); 2638874aeea5SJeff Kirsher 2639874aeea5SJeff Kirsher efx_pci_remove_main(efx); 2640874aeea5SJeff Kirsher 2641874aeea5SJeff Kirsher efx_fini_io(efx); 2642874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 2643874aeea5SJeff Kirsher 2644874aeea5SJeff Kirsher efx_fini_struct(efx); 26453de4e301SBen Hutchings pci_set_drvdata(pci_dev, NULL); 2646874aeea5SJeff Kirsher free_netdev(efx->net_dev); 2647626950dbSAlexandre Rames 2648626950dbSAlexandre Rames pci_disable_pcie_error_reporting(pci_dev); 2649874aeea5SJeff Kirsher }; 2650874aeea5SJeff Kirsher 2651460eeaa0SBen Hutchings /* NIC VPD information 2652460eeaa0SBen Hutchings * Called during probe to display the part number of the 2653460eeaa0SBen Hutchings * installed NIC. VPD is potentially very large but this should 2654460eeaa0SBen Hutchings * always appear within the first 512 bytes. 2655460eeaa0SBen Hutchings */ 2656460eeaa0SBen Hutchings #define SFC_VPD_LEN 512 2657460eeaa0SBen Hutchings static void efx_print_product_vpd(struct efx_nic *efx) 2658460eeaa0SBen Hutchings { 2659460eeaa0SBen Hutchings struct pci_dev *dev = efx->pci_dev; 2660460eeaa0SBen Hutchings char vpd_data[SFC_VPD_LEN]; 2661460eeaa0SBen Hutchings ssize_t vpd_size; 2662460eeaa0SBen Hutchings int i, j; 2663460eeaa0SBen Hutchings 2664460eeaa0SBen Hutchings /* Get the vpd data from the device */ 2665460eeaa0SBen Hutchings vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 2666460eeaa0SBen Hutchings if (vpd_size <= 0) { 2667460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 2668460eeaa0SBen Hutchings return; 2669460eeaa0SBen Hutchings } 2670460eeaa0SBen Hutchings 2671460eeaa0SBen Hutchings /* Get the Read only section */ 2672460eeaa0SBen Hutchings i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 2673460eeaa0SBen Hutchings if (i < 0) { 2674460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 2675460eeaa0SBen Hutchings return; 2676460eeaa0SBen Hutchings } 2677460eeaa0SBen Hutchings 2678460eeaa0SBen Hutchings j = pci_vpd_lrdt_size(&vpd_data[i]); 2679460eeaa0SBen Hutchings i += PCI_VPD_LRDT_TAG_SIZE; 2680460eeaa0SBen Hutchings if (i + j > vpd_size) 2681460eeaa0SBen Hutchings j = vpd_size - i; 2682460eeaa0SBen Hutchings 2683460eeaa0SBen Hutchings /* Get the Part number */ 2684460eeaa0SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 2685460eeaa0SBen Hutchings if (i < 0) { 2686460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 2687460eeaa0SBen Hutchings return; 2688460eeaa0SBen Hutchings } 2689460eeaa0SBen Hutchings 2690460eeaa0SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 2691460eeaa0SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 2692460eeaa0SBen Hutchings if (i + j > vpd_size) { 2693460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 2694460eeaa0SBen Hutchings return; 2695460eeaa0SBen Hutchings } 2696460eeaa0SBen Hutchings 2697460eeaa0SBen Hutchings netif_info(efx, drv, efx->net_dev, 2698460eeaa0SBen Hutchings "Part Number : %.*s\n", j, &vpd_data[i]); 2699460eeaa0SBen Hutchings } 2700460eeaa0SBen Hutchings 2701460eeaa0SBen Hutchings 2702874aeea5SJeff Kirsher /* Main body of NIC initialisation 2703874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, theoretically). 2704874aeea5SJeff Kirsher */ 2705874aeea5SJeff Kirsher static int efx_pci_probe_main(struct efx_nic *efx) 2706874aeea5SJeff Kirsher { 2707874aeea5SJeff Kirsher int rc; 2708874aeea5SJeff Kirsher 2709874aeea5SJeff Kirsher /* Do start-of-day initialisation */ 2710874aeea5SJeff Kirsher rc = efx_probe_all(efx); 2711874aeea5SJeff Kirsher if (rc) 2712874aeea5SJeff Kirsher goto fail1; 2713874aeea5SJeff Kirsher 2714874aeea5SJeff Kirsher efx_init_napi(efx); 2715874aeea5SJeff Kirsher 2716874aeea5SJeff Kirsher rc = efx->type->init(efx); 2717874aeea5SJeff Kirsher if (rc) { 2718874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 2719874aeea5SJeff Kirsher "failed to initialise NIC\n"); 2720874aeea5SJeff Kirsher goto fail3; 2721874aeea5SJeff Kirsher } 2722874aeea5SJeff Kirsher 2723874aeea5SJeff Kirsher rc = efx_init_port(efx); 2724874aeea5SJeff Kirsher if (rc) { 2725874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 2726874aeea5SJeff Kirsher "failed to initialise port\n"); 2727874aeea5SJeff Kirsher goto fail4; 2728874aeea5SJeff Kirsher } 2729874aeea5SJeff Kirsher 2730874aeea5SJeff Kirsher rc = efx_nic_init_interrupt(efx); 2731874aeea5SJeff Kirsher if (rc) 2732874aeea5SJeff Kirsher goto fail5; 27337f967c01SBen Hutchings efx_start_interrupts(efx, false); 2734874aeea5SJeff Kirsher 2735874aeea5SJeff Kirsher return 0; 2736874aeea5SJeff Kirsher 2737874aeea5SJeff Kirsher fail5: 2738874aeea5SJeff Kirsher efx_fini_port(efx); 2739874aeea5SJeff Kirsher fail4: 2740874aeea5SJeff Kirsher efx->type->fini(efx); 2741874aeea5SJeff Kirsher fail3: 2742874aeea5SJeff Kirsher efx_fini_napi(efx); 2743874aeea5SJeff Kirsher efx_remove_all(efx); 2744874aeea5SJeff Kirsher fail1: 2745874aeea5SJeff Kirsher return rc; 2746874aeea5SJeff Kirsher } 2747874aeea5SJeff Kirsher 2748874aeea5SJeff Kirsher /* NIC initialisation 2749874aeea5SJeff Kirsher * 2750874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, 275173ba7b68SBen Hutchings * theoretically). It sets up PCI mappings, resets the NIC, 2752874aeea5SJeff Kirsher * sets up and registers the network devices with the kernel and hooks 2753874aeea5SJeff Kirsher * the interrupt service routine. It does not prepare the device for 2754874aeea5SJeff Kirsher * transmission; this is left to the first time one of the network 2755874aeea5SJeff Kirsher * interfaces is brought up (i.e. efx_net_open). 2756874aeea5SJeff Kirsher */ 275787d1fc11SBill Pemberton static int efx_pci_probe(struct pci_dev *pci_dev, 2758874aeea5SJeff Kirsher const struct pci_device_id *entry) 2759874aeea5SJeff Kirsher { 2760874aeea5SJeff Kirsher struct net_device *net_dev; 2761874aeea5SJeff Kirsher struct efx_nic *efx; 2762fadac6aaSBen Hutchings int rc; 2763874aeea5SJeff Kirsher 2764874aeea5SJeff Kirsher /* Allocate and initialise a struct net_device and struct efx_nic */ 2765874aeea5SJeff Kirsher net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 2766874aeea5SJeff Kirsher EFX_MAX_RX_QUEUES); 2767874aeea5SJeff Kirsher if (!net_dev) 2768874aeea5SJeff Kirsher return -ENOMEM; 2769adeb15aaSBen Hutchings efx = netdev_priv(net_dev); 2770adeb15aaSBen Hutchings efx->type = (const struct efx_nic_type *) entry->driver_data; 2771adeb15aaSBen Hutchings net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 2772874aeea5SJeff Kirsher NETIF_F_HIGHDMA | NETIF_F_TSO | 2773874aeea5SJeff Kirsher NETIF_F_RXCSUM); 2774adeb15aaSBen Hutchings if (efx->type->offload_features & NETIF_F_V6_CSUM) 2775874aeea5SJeff Kirsher net_dev->features |= NETIF_F_TSO6; 2776874aeea5SJeff Kirsher /* Mask for features that also apply to VLAN devices */ 2777874aeea5SJeff Kirsher net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG | 2778874aeea5SJeff Kirsher NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 2779874aeea5SJeff Kirsher NETIF_F_RXCSUM); 2780874aeea5SJeff Kirsher /* All offloads can be toggled */ 2781874aeea5SJeff Kirsher net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA; 2782874aeea5SJeff Kirsher pci_set_drvdata(pci_dev, efx); 2783874aeea5SJeff Kirsher SET_NETDEV_DEV(net_dev, &pci_dev->dev); 2784adeb15aaSBen Hutchings rc = efx_init_struct(efx, pci_dev, net_dev); 2785874aeea5SJeff Kirsher if (rc) 2786874aeea5SJeff Kirsher goto fail1; 2787874aeea5SJeff Kirsher 2788874aeea5SJeff Kirsher netif_info(efx, probe, efx->net_dev, 2789874aeea5SJeff Kirsher "Solarflare NIC detected\n"); 2790874aeea5SJeff Kirsher 2791460eeaa0SBen Hutchings efx_print_product_vpd(efx); 2792460eeaa0SBen Hutchings 2793874aeea5SJeff Kirsher /* Set up basic I/O (BAR mappings etc) */ 2794874aeea5SJeff Kirsher rc = efx_init_io(efx); 2795874aeea5SJeff Kirsher if (rc) 2796874aeea5SJeff Kirsher goto fail2; 2797874aeea5SJeff Kirsher 2798874aeea5SJeff Kirsher rc = efx_pci_probe_main(efx); 2799fadac6aaSBen Hutchings if (rc) 2800874aeea5SJeff Kirsher goto fail3; 2801874aeea5SJeff Kirsher 2802874aeea5SJeff Kirsher rc = efx_register_netdev(efx); 2803874aeea5SJeff Kirsher if (rc) 2804fadac6aaSBen Hutchings goto fail4; 2805874aeea5SJeff Kirsher 2806cd2d5b52SBen Hutchings rc = efx_sriov_init(efx); 2807cd2d5b52SBen Hutchings if (rc) 2808cd2d5b52SBen Hutchings netif_err(efx, probe, efx->net_dev, 2809cd2d5b52SBen Hutchings "SR-IOV can't be enabled rc %d\n", rc); 2810cd2d5b52SBen Hutchings 2811874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 2812874aeea5SJeff Kirsher 28137c43161cSBen Hutchings /* Try to create MTDs, but allow this to fail */ 2814874aeea5SJeff Kirsher rtnl_lock(); 28157c43161cSBen Hutchings rc = efx_mtd_probe(efx); 2816874aeea5SJeff Kirsher rtnl_unlock(); 28177c43161cSBen Hutchings if (rc) 28187c43161cSBen Hutchings netif_warn(efx, probe, efx->net_dev, 28197c43161cSBen Hutchings "failed to create MTDs (%d)\n", rc); 28207c43161cSBen Hutchings 2821626950dbSAlexandre Rames rc = pci_enable_pcie_error_reporting(pci_dev); 2822626950dbSAlexandre Rames if (rc && rc != -EINVAL) 2823626950dbSAlexandre Rames netif_warn(efx, probe, efx->net_dev, 2824626950dbSAlexandre Rames "pci_enable_pcie_error_reporting failed (%d)\n", rc); 2825626950dbSAlexandre Rames 2826874aeea5SJeff Kirsher return 0; 2827874aeea5SJeff Kirsher 2828874aeea5SJeff Kirsher fail4: 2829fadac6aaSBen Hutchings efx_pci_remove_main(efx); 2830874aeea5SJeff Kirsher fail3: 2831874aeea5SJeff Kirsher efx_fini_io(efx); 2832874aeea5SJeff Kirsher fail2: 2833874aeea5SJeff Kirsher efx_fini_struct(efx); 2834874aeea5SJeff Kirsher fail1: 28353de4e301SBen Hutchings pci_set_drvdata(pci_dev, NULL); 2836874aeea5SJeff Kirsher WARN_ON(rc > 0); 2837874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 2838874aeea5SJeff Kirsher free_netdev(net_dev); 2839874aeea5SJeff Kirsher return rc; 2840874aeea5SJeff Kirsher } 2841874aeea5SJeff Kirsher 2842874aeea5SJeff Kirsher static int efx_pm_freeze(struct device *dev) 2843874aeea5SJeff Kirsher { 2844874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2845874aeea5SJeff Kirsher 284661da026dSBen Hutchings rtnl_lock(); 284761da026dSBen Hutchings 28486032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 2849f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 2850874aeea5SJeff Kirsher 2851c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2852874aeea5SJeff Kirsher 2853874aeea5SJeff Kirsher efx_stop_all(efx); 28547f967c01SBen Hutchings efx_stop_interrupts(efx, false); 28556032fb56SBen Hutchings } 2856874aeea5SJeff Kirsher 285761da026dSBen Hutchings rtnl_unlock(); 285861da026dSBen Hutchings 2859874aeea5SJeff Kirsher return 0; 2860874aeea5SJeff Kirsher } 2861874aeea5SJeff Kirsher 2862874aeea5SJeff Kirsher static int efx_pm_thaw(struct device *dev) 2863874aeea5SJeff Kirsher { 2864874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2865874aeea5SJeff Kirsher 286661da026dSBen Hutchings rtnl_lock(); 286761da026dSBen Hutchings 28686032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 28697f967c01SBen Hutchings efx_start_interrupts(efx, false); 2870874aeea5SJeff Kirsher 2871874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2872874aeea5SJeff Kirsher efx->phy_op->reconfigure(efx); 2873874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2874874aeea5SJeff Kirsher 2875874aeea5SJeff Kirsher efx_start_all(efx); 2876874aeea5SJeff Kirsher 2877874aeea5SJeff Kirsher netif_device_attach(efx->net_dev); 2878874aeea5SJeff Kirsher 2879f16aeea0SBen Hutchings efx->state = STATE_READY; 2880874aeea5SJeff Kirsher 2881874aeea5SJeff Kirsher efx->type->resume_wol(efx); 28826032fb56SBen Hutchings } 2883874aeea5SJeff Kirsher 288461da026dSBen Hutchings rtnl_unlock(); 288561da026dSBen Hutchings 2886874aeea5SJeff Kirsher /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 2887874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 2888874aeea5SJeff Kirsher 2889874aeea5SJeff Kirsher return 0; 2890874aeea5SJeff Kirsher } 2891874aeea5SJeff Kirsher 2892874aeea5SJeff Kirsher static int efx_pm_poweroff(struct device *dev) 2893874aeea5SJeff Kirsher { 2894874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 2895874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 2896874aeea5SJeff Kirsher 2897874aeea5SJeff Kirsher efx->type->fini(efx); 2898874aeea5SJeff Kirsher 2899874aeea5SJeff Kirsher efx->reset_pending = 0; 2900874aeea5SJeff Kirsher 2901874aeea5SJeff Kirsher pci_save_state(pci_dev); 2902874aeea5SJeff Kirsher return pci_set_power_state(pci_dev, PCI_D3hot); 2903874aeea5SJeff Kirsher } 2904874aeea5SJeff Kirsher 2905874aeea5SJeff Kirsher /* Used for both resume and restore */ 2906874aeea5SJeff Kirsher static int efx_pm_resume(struct device *dev) 2907874aeea5SJeff Kirsher { 2908874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 2909874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 2910874aeea5SJeff Kirsher int rc; 2911874aeea5SJeff Kirsher 2912874aeea5SJeff Kirsher rc = pci_set_power_state(pci_dev, PCI_D0); 2913874aeea5SJeff Kirsher if (rc) 2914874aeea5SJeff Kirsher return rc; 2915874aeea5SJeff Kirsher pci_restore_state(pci_dev); 2916874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 2917874aeea5SJeff Kirsher if (rc) 2918874aeea5SJeff Kirsher return rc; 2919874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 2920874aeea5SJeff Kirsher rc = efx->type->reset(efx, RESET_TYPE_ALL); 2921874aeea5SJeff Kirsher if (rc) 2922874aeea5SJeff Kirsher return rc; 2923874aeea5SJeff Kirsher rc = efx->type->init(efx); 2924874aeea5SJeff Kirsher if (rc) 2925874aeea5SJeff Kirsher return rc; 2926874aeea5SJeff Kirsher efx_pm_thaw(dev); 2927874aeea5SJeff Kirsher return 0; 2928874aeea5SJeff Kirsher } 2929874aeea5SJeff Kirsher 2930874aeea5SJeff Kirsher static int efx_pm_suspend(struct device *dev) 2931874aeea5SJeff Kirsher { 2932874aeea5SJeff Kirsher int rc; 2933874aeea5SJeff Kirsher 2934874aeea5SJeff Kirsher efx_pm_freeze(dev); 2935874aeea5SJeff Kirsher rc = efx_pm_poweroff(dev); 2936874aeea5SJeff Kirsher if (rc) 2937874aeea5SJeff Kirsher efx_pm_resume(dev); 2938874aeea5SJeff Kirsher return rc; 2939874aeea5SJeff Kirsher } 2940874aeea5SJeff Kirsher 294118e83e4cSBen Hutchings static const struct dev_pm_ops efx_pm_ops = { 2942874aeea5SJeff Kirsher .suspend = efx_pm_suspend, 2943874aeea5SJeff Kirsher .resume = efx_pm_resume, 2944874aeea5SJeff Kirsher .freeze = efx_pm_freeze, 2945874aeea5SJeff Kirsher .thaw = efx_pm_thaw, 2946874aeea5SJeff Kirsher .poweroff = efx_pm_poweroff, 2947874aeea5SJeff Kirsher .restore = efx_pm_resume, 2948874aeea5SJeff Kirsher }; 2949874aeea5SJeff Kirsher 2950626950dbSAlexandre Rames /* A PCI error affecting this device was detected. 2951626950dbSAlexandre Rames * At this point MMIO and DMA may be disabled. 2952626950dbSAlexandre Rames * Stop the software path and request a slot reset. 2953626950dbSAlexandre Rames */ 2954626950dbSAlexandre Rames pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 2955626950dbSAlexandre Rames enum pci_channel_state state) 2956626950dbSAlexandre Rames { 2957626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 2958626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 2959626950dbSAlexandre Rames 2960626950dbSAlexandre Rames if (state == pci_channel_io_perm_failure) 2961626950dbSAlexandre Rames return PCI_ERS_RESULT_DISCONNECT; 2962626950dbSAlexandre Rames 2963626950dbSAlexandre Rames rtnl_lock(); 2964626950dbSAlexandre Rames 2965626950dbSAlexandre Rames if (efx->state != STATE_DISABLED) { 2966626950dbSAlexandre Rames efx->state = STATE_RECOVERY; 2967626950dbSAlexandre Rames efx->reset_pending = 0; 2968626950dbSAlexandre Rames 2969626950dbSAlexandre Rames efx_device_detach_sync(efx); 2970626950dbSAlexandre Rames 2971626950dbSAlexandre Rames efx_stop_all(efx); 2972626950dbSAlexandre Rames efx_stop_interrupts(efx, false); 2973626950dbSAlexandre Rames 2974626950dbSAlexandre Rames status = PCI_ERS_RESULT_NEED_RESET; 2975626950dbSAlexandre Rames } else { 2976626950dbSAlexandre Rames /* If the interface is disabled we don't want to do anything 2977626950dbSAlexandre Rames * with it. 2978626950dbSAlexandre Rames */ 2979626950dbSAlexandre Rames status = PCI_ERS_RESULT_RECOVERED; 2980626950dbSAlexandre Rames } 2981626950dbSAlexandre Rames 2982626950dbSAlexandre Rames rtnl_unlock(); 2983626950dbSAlexandre Rames 2984626950dbSAlexandre Rames pci_disable_device(pdev); 2985626950dbSAlexandre Rames 2986626950dbSAlexandre Rames return status; 2987626950dbSAlexandre Rames } 2988626950dbSAlexandre Rames 2989626950dbSAlexandre Rames /* Fake a successfull reset, which will be performed later in efx_io_resume. */ 2990626950dbSAlexandre Rames pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 2991626950dbSAlexandre Rames { 2992626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 2993626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 2994626950dbSAlexandre Rames int rc; 2995626950dbSAlexandre Rames 2996626950dbSAlexandre Rames if (pci_enable_device(pdev)) { 2997626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 2998626950dbSAlexandre Rames "Cannot re-enable PCI device after reset.\n"); 2999626950dbSAlexandre Rames status = PCI_ERS_RESULT_DISCONNECT; 3000626950dbSAlexandre Rames } 3001626950dbSAlexandre Rames 3002626950dbSAlexandre Rames rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3003626950dbSAlexandre Rames if (rc) { 3004626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3005626950dbSAlexandre Rames "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3006626950dbSAlexandre Rames /* Non-fatal error. Continue. */ 3007626950dbSAlexandre Rames } 3008626950dbSAlexandre Rames 3009626950dbSAlexandre Rames return status; 3010626950dbSAlexandre Rames } 3011626950dbSAlexandre Rames 3012626950dbSAlexandre Rames /* Perform the actual reset and resume I/O operations. */ 3013626950dbSAlexandre Rames static void efx_io_resume(struct pci_dev *pdev) 3014626950dbSAlexandre Rames { 3015626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3016626950dbSAlexandre Rames int rc; 3017626950dbSAlexandre Rames 3018626950dbSAlexandre Rames rtnl_lock(); 3019626950dbSAlexandre Rames 3020626950dbSAlexandre Rames if (efx->state == STATE_DISABLED) 3021626950dbSAlexandre Rames goto out; 3022626950dbSAlexandre Rames 3023626950dbSAlexandre Rames rc = efx_reset(efx, RESET_TYPE_ALL); 3024626950dbSAlexandre Rames if (rc) { 3025626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3026626950dbSAlexandre Rames "efx_reset failed after PCI error (%d)\n", rc); 3027626950dbSAlexandre Rames } else { 3028626950dbSAlexandre Rames efx->state = STATE_READY; 3029626950dbSAlexandre Rames netif_dbg(efx, hw, efx->net_dev, 3030626950dbSAlexandre Rames "Done resetting and resuming IO after PCI error.\n"); 3031626950dbSAlexandre Rames } 3032626950dbSAlexandre Rames 3033626950dbSAlexandre Rames out: 3034626950dbSAlexandre Rames rtnl_unlock(); 3035626950dbSAlexandre Rames } 3036626950dbSAlexandre Rames 3037626950dbSAlexandre Rames /* For simplicity and reliability, we always require a slot reset and try to 3038626950dbSAlexandre Rames * reset the hardware when a pci error affecting the device is detected. 3039626950dbSAlexandre Rames * We leave both the link_reset and mmio_enabled callback unimplemented: 3040626950dbSAlexandre Rames * with our request for slot reset the mmio_enabled callback will never be 3041626950dbSAlexandre Rames * called, and the link_reset callback is not used by AER or EEH mechanisms. 3042626950dbSAlexandre Rames */ 3043626950dbSAlexandre Rames static struct pci_error_handlers efx_err_handlers = { 3044626950dbSAlexandre Rames .error_detected = efx_io_error_detected, 3045626950dbSAlexandre Rames .slot_reset = efx_io_slot_reset, 3046626950dbSAlexandre Rames .resume = efx_io_resume, 3047626950dbSAlexandre Rames }; 3048626950dbSAlexandre Rames 3049874aeea5SJeff Kirsher static struct pci_driver efx_pci_driver = { 3050874aeea5SJeff Kirsher .name = KBUILD_MODNAME, 3051874aeea5SJeff Kirsher .id_table = efx_pci_table, 3052874aeea5SJeff Kirsher .probe = efx_pci_probe, 3053874aeea5SJeff Kirsher .remove = efx_pci_remove, 3054874aeea5SJeff Kirsher .driver.pm = &efx_pm_ops, 3055626950dbSAlexandre Rames .err_handler = &efx_err_handlers, 3056874aeea5SJeff Kirsher }; 3057874aeea5SJeff Kirsher 3058874aeea5SJeff Kirsher /************************************************************************** 3059874aeea5SJeff Kirsher * 3060874aeea5SJeff Kirsher * Kernel module interface 3061874aeea5SJeff Kirsher * 3062874aeea5SJeff Kirsher *************************************************************************/ 3063874aeea5SJeff Kirsher 3064874aeea5SJeff Kirsher module_param(interrupt_mode, uint, 0444); 3065874aeea5SJeff Kirsher MODULE_PARM_DESC(interrupt_mode, 3066874aeea5SJeff Kirsher "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3067874aeea5SJeff Kirsher 3068874aeea5SJeff Kirsher static int __init efx_init_module(void) 3069874aeea5SJeff Kirsher { 3070874aeea5SJeff Kirsher int rc; 3071874aeea5SJeff Kirsher 3072874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3073874aeea5SJeff Kirsher 3074874aeea5SJeff Kirsher rc = register_netdevice_notifier(&efx_netdev_notifier); 3075874aeea5SJeff Kirsher if (rc) 3076874aeea5SJeff Kirsher goto err_notifier; 3077874aeea5SJeff Kirsher 3078cd2d5b52SBen Hutchings rc = efx_init_sriov(); 3079cd2d5b52SBen Hutchings if (rc) 3080cd2d5b52SBen Hutchings goto err_sriov; 3081cd2d5b52SBen Hutchings 3082874aeea5SJeff Kirsher reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3083874aeea5SJeff Kirsher if (!reset_workqueue) { 3084874aeea5SJeff Kirsher rc = -ENOMEM; 3085874aeea5SJeff Kirsher goto err_reset; 3086874aeea5SJeff Kirsher } 3087874aeea5SJeff Kirsher 3088874aeea5SJeff Kirsher rc = pci_register_driver(&efx_pci_driver); 3089874aeea5SJeff Kirsher if (rc < 0) 3090874aeea5SJeff Kirsher goto err_pci; 3091874aeea5SJeff Kirsher 3092874aeea5SJeff Kirsher return 0; 3093874aeea5SJeff Kirsher 3094874aeea5SJeff Kirsher err_pci: 3095874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 3096874aeea5SJeff Kirsher err_reset: 3097cd2d5b52SBen Hutchings efx_fini_sriov(); 3098cd2d5b52SBen Hutchings err_sriov: 3099874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3100874aeea5SJeff Kirsher err_notifier: 3101874aeea5SJeff Kirsher return rc; 3102874aeea5SJeff Kirsher } 3103874aeea5SJeff Kirsher 3104874aeea5SJeff Kirsher static void __exit efx_exit_module(void) 3105874aeea5SJeff Kirsher { 3106874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver unloading\n"); 3107874aeea5SJeff Kirsher 3108874aeea5SJeff Kirsher pci_unregister_driver(&efx_pci_driver); 3109874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 3110cd2d5b52SBen Hutchings efx_fini_sriov(); 3111874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3112874aeea5SJeff Kirsher 3113874aeea5SJeff Kirsher } 3114874aeea5SJeff Kirsher 3115874aeea5SJeff Kirsher module_init(efx_init_module); 3116874aeea5SJeff Kirsher module_exit(efx_exit_module); 3117874aeea5SJeff Kirsher 3118874aeea5SJeff Kirsher MODULE_AUTHOR("Solarflare Communications and " 3119874aeea5SJeff Kirsher "Michael Brown <mbrown@fensystems.co.uk>"); 3120874aeea5SJeff Kirsher MODULE_DESCRIPTION("Solarflare Communications network driver"); 3121874aeea5SJeff Kirsher MODULE_LICENSE("GPL"); 3122874aeea5SJeff Kirsher MODULE_DEVICE_TABLE(pci, efx_pci_table); 3123