1874aeea5SJeff Kirsher /**************************************************************************** 2f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 3874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 4f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 5874aeea5SJeff Kirsher * 6874aeea5SJeff Kirsher * This program is free software; you can redistribute it and/or modify it 7874aeea5SJeff Kirsher * under the terms of the GNU General Public License version 2 as published 8874aeea5SJeff Kirsher * by the Free Software Foundation, incorporated herein by reference. 9874aeea5SJeff Kirsher */ 10874aeea5SJeff Kirsher 11874aeea5SJeff Kirsher #include <linux/module.h> 12874aeea5SJeff Kirsher #include <linux/pci.h> 13874aeea5SJeff Kirsher #include <linux/netdevice.h> 14874aeea5SJeff Kirsher #include <linux/etherdevice.h> 15874aeea5SJeff Kirsher #include <linux/delay.h> 16874aeea5SJeff Kirsher #include <linux/notifier.h> 17874aeea5SJeff Kirsher #include <linux/ip.h> 18874aeea5SJeff Kirsher #include <linux/tcp.h> 19874aeea5SJeff Kirsher #include <linux/in.h> 20874aeea5SJeff Kirsher #include <linux/ethtool.h> 21874aeea5SJeff Kirsher #include <linux/topology.h> 22874aeea5SJeff Kirsher #include <linux/gfp.h> 23626950dbSAlexandre Rames #include <linux/aer.h> 24b28405b0SAlexandre Rames #include <linux/interrupt.h> 25874aeea5SJeff Kirsher #include "net_driver.h" 26e5fbd977SJon Cooper #include <net/gre.h> 27e5fbd977SJon Cooper #include <net/udp_tunnel.h> 28874aeea5SJeff Kirsher #include "efx.h" 29874aeea5SJeff Kirsher #include "nic.h" 3071827443SEdward Cree #include "io.h" 31dd40781eSBen Hutchings #include "selftest.h" 327fa8d547SShradha Shah #include "sriov.h" 33874aeea5SJeff Kirsher 34874aeea5SJeff Kirsher #include "mcdi.h" 35e5fbd977SJon Cooper #include "mcdi_pcol.h" 36874aeea5SJeff Kirsher #include "workarounds.h" 37874aeea5SJeff Kirsher 38874aeea5SJeff Kirsher /************************************************************************** 39874aeea5SJeff Kirsher * 40874aeea5SJeff Kirsher * Type name strings 41874aeea5SJeff Kirsher * 42874aeea5SJeff Kirsher ************************************************************************** 43874aeea5SJeff Kirsher */ 44874aeea5SJeff Kirsher 45874aeea5SJeff Kirsher /* Loopback mode names (see LOOPBACK_MODE()) */ 46874aeea5SJeff Kirsher const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 4718e83e4cSBen Hutchings const char *const efx_loopback_mode_names[] = { 48874aeea5SJeff Kirsher [LOOPBACK_NONE] = "NONE", 49874aeea5SJeff Kirsher [LOOPBACK_DATA] = "DATAPATH", 50874aeea5SJeff Kirsher [LOOPBACK_GMAC] = "GMAC", 51874aeea5SJeff Kirsher [LOOPBACK_XGMII] = "XGMII", 52874aeea5SJeff Kirsher [LOOPBACK_XGXS] = "XGXS", 53874aeea5SJeff Kirsher [LOOPBACK_XAUI] = "XAUI", 54874aeea5SJeff Kirsher [LOOPBACK_GMII] = "GMII", 55874aeea5SJeff Kirsher [LOOPBACK_SGMII] = "SGMII", 56874aeea5SJeff Kirsher [LOOPBACK_XGBR] = "XGBR", 57874aeea5SJeff Kirsher [LOOPBACK_XFI] = "XFI", 58874aeea5SJeff Kirsher [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 59874aeea5SJeff Kirsher [LOOPBACK_GMII_FAR] = "GMII_FAR", 60874aeea5SJeff Kirsher [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 61874aeea5SJeff Kirsher [LOOPBACK_XFI_FAR] = "XFI_FAR", 62874aeea5SJeff Kirsher [LOOPBACK_GPHY] = "GPHY", 63874aeea5SJeff Kirsher [LOOPBACK_PHYXS] = "PHYXS", 64874aeea5SJeff Kirsher [LOOPBACK_PCS] = "PCS", 65874aeea5SJeff Kirsher [LOOPBACK_PMAPMD] = "PMA/PMD", 66874aeea5SJeff Kirsher [LOOPBACK_XPORT] = "XPORT", 67874aeea5SJeff Kirsher [LOOPBACK_XGMII_WS] = "XGMII_WS", 68874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS] = "XAUI_WS", 69874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 70874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 71874aeea5SJeff Kirsher [LOOPBACK_GMII_WS] = "GMII_WS", 72874aeea5SJeff Kirsher [LOOPBACK_XFI_WS] = "XFI_WS", 73874aeea5SJeff Kirsher [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 74874aeea5SJeff Kirsher [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 75874aeea5SJeff Kirsher }; 76874aeea5SJeff Kirsher 77874aeea5SJeff Kirsher const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 7818e83e4cSBen Hutchings const char *const efx_reset_type_names[] = { 79874aeea5SJeff Kirsher [RESET_TYPE_INVISIBLE] = "INVISIBLE", 80874aeea5SJeff Kirsher [RESET_TYPE_ALL] = "ALL", 81626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 82874aeea5SJeff Kirsher [RESET_TYPE_WORLD] = "WORLD", 83626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 84087e9025SJon Cooper [RESET_TYPE_DATAPATH] = "DATAPATH", 85e283546cSEdward Cree [RESET_TYPE_MC_BIST] = "MC_BIST", 86874aeea5SJeff Kirsher [RESET_TYPE_DISABLE] = "DISABLE", 87874aeea5SJeff Kirsher [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 88874aeea5SJeff Kirsher [RESET_TYPE_INT_ERROR] = "INT_ERROR", 893de82b91SAlexandre Rames [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 90874aeea5SJeff Kirsher [RESET_TYPE_TX_SKIP] = "TX_SKIP", 91874aeea5SJeff Kirsher [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 92e283546cSEdward Cree [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 93874aeea5SJeff Kirsher }; 94874aeea5SJeff Kirsher 95e5fbd977SJon Cooper /* UDP tunnel type names */ 96e5fbd977SJon Cooper static const char *const efx_udp_tunnel_type_names[] = { 97e5fbd977SJon Cooper [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan", 98e5fbd977SJon Cooper [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve", 99e5fbd977SJon Cooper }; 100e5fbd977SJon Cooper 101e5fbd977SJon Cooper void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen) 102e5fbd977SJon Cooper { 103e5fbd977SJon Cooper if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) && 104e5fbd977SJon Cooper efx_udp_tunnel_type_names[type] != NULL) 105e5fbd977SJon Cooper snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]); 106e5fbd977SJon Cooper else 107e5fbd977SJon Cooper snprintf(buf, buflen, "type %d", type); 108e5fbd977SJon Cooper } 109e5fbd977SJon Cooper 110874aeea5SJeff Kirsher /* Reset workqueue. If any NIC has a hardware failure then a reset will be 111874aeea5SJeff Kirsher * queued onto this work queue. This is not a per-nic work queue, because 112874aeea5SJeff Kirsher * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 113874aeea5SJeff Kirsher */ 114874aeea5SJeff Kirsher static struct workqueue_struct *reset_workqueue; 115874aeea5SJeff Kirsher 11674cd60a4SJon Cooper /* How often and how many times to poll for a reset while waiting for a 11774cd60a4SJon Cooper * BIST that another function started to complete. 11874cd60a4SJon Cooper */ 11974cd60a4SJon Cooper #define BIST_WAIT_DELAY_MS 100 12074cd60a4SJon Cooper #define BIST_WAIT_DELAY_COUNT 100 12174cd60a4SJon Cooper 122874aeea5SJeff Kirsher /************************************************************************** 123874aeea5SJeff Kirsher * 124874aeea5SJeff Kirsher * Configurable values 125874aeea5SJeff Kirsher * 126874aeea5SJeff Kirsher *************************************************************************/ 127874aeea5SJeff Kirsher 128874aeea5SJeff Kirsher /* 129874aeea5SJeff Kirsher * Use separate channels for TX and RX events 130874aeea5SJeff Kirsher * 131874aeea5SJeff Kirsher * Set this to 1 to use separate channels for TX and RX. It allows us 132874aeea5SJeff Kirsher * to control interrupt affinity separately for TX and RX. 133874aeea5SJeff Kirsher * 134874aeea5SJeff Kirsher * This is only used in MSI-X interrupt mode 135874aeea5SJeff Kirsher */ 136b0fbdae1SShradha Shah bool efx_separate_tx_channels; 137b0fbdae1SShradha Shah module_param(efx_separate_tx_channels, bool, 0444); 138b0fbdae1SShradha Shah MODULE_PARM_DESC(efx_separate_tx_channels, 139874aeea5SJeff Kirsher "Use separate channels for TX and RX"); 140874aeea5SJeff Kirsher 141874aeea5SJeff Kirsher /* This is the weight assigned to each of the (per-channel) virtual 142874aeea5SJeff Kirsher * NAPI devices. 143874aeea5SJeff Kirsher */ 144874aeea5SJeff Kirsher static int napi_weight = 64; 145874aeea5SJeff Kirsher 146874aeea5SJeff Kirsher /* This is the time (in jiffies) between invocations of the hardware 147626950dbSAlexandre Rames * monitor. 148626950dbSAlexandre Rames * On Falcon-based NICs, this will: 149874aeea5SJeff Kirsher * - Check the on-board hardware monitor; 150874aeea5SJeff Kirsher * - Poll the link state and reconfigure the hardware as necessary. 151626950dbSAlexandre Rames * On Siena-based NICs for power systems with EEH support, this will give EEH a 152626950dbSAlexandre Rames * chance to start. 153874aeea5SJeff Kirsher */ 154874aeea5SJeff Kirsher static unsigned int efx_monitor_interval = 1 * HZ; 155874aeea5SJeff Kirsher 156874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 157874aeea5SJeff Kirsher * module load with ethtool. 158874aeea5SJeff Kirsher * 159874aeea5SJeff Kirsher * The default for RX should strike a balance between increasing the 160874aeea5SJeff Kirsher * round-trip latency and reducing overhead. 161874aeea5SJeff Kirsher */ 162874aeea5SJeff Kirsher static unsigned int rx_irq_mod_usec = 60; 163874aeea5SJeff Kirsher 164874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 165874aeea5SJeff Kirsher * module load with ethtool. 166874aeea5SJeff Kirsher * 167874aeea5SJeff Kirsher * This default is chosen to ensure that a 10G link does not go idle 168874aeea5SJeff Kirsher * while a TX queue is stopped after it has become full. A queue is 169874aeea5SJeff Kirsher * restarted when it drops below half full. The time this takes (assuming 170874aeea5SJeff Kirsher * worst case 3 descriptors per packet and 1024 descriptors) is 171874aeea5SJeff Kirsher * 512 / 3 * 1.2 = 205 usec. 172874aeea5SJeff Kirsher */ 173874aeea5SJeff Kirsher static unsigned int tx_irq_mod_usec = 150; 174874aeea5SJeff Kirsher 175874aeea5SJeff Kirsher /* This is the first interrupt mode to try out of: 176874aeea5SJeff Kirsher * 0 => MSI-X 177874aeea5SJeff Kirsher * 1 => MSI 178874aeea5SJeff Kirsher * 2 => legacy 179874aeea5SJeff Kirsher */ 180874aeea5SJeff Kirsher static unsigned int interrupt_mode; 181874aeea5SJeff Kirsher 182874aeea5SJeff Kirsher /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 183874aeea5SJeff Kirsher * i.e. the number of CPUs among which we may distribute simultaneous 184874aeea5SJeff Kirsher * interrupt handling. 185874aeea5SJeff Kirsher * 186874aeea5SJeff Kirsher * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 187cdb08f8fSBen Hutchings * The default (0) means to assign an interrupt to each core. 188874aeea5SJeff Kirsher */ 189874aeea5SJeff Kirsher static unsigned int rss_cpus; 190874aeea5SJeff Kirsher module_param(rss_cpus, uint, 0444); 191874aeea5SJeff Kirsher MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 192874aeea5SJeff Kirsher 193b9cc977dSBen Hutchings static bool phy_flash_cfg; 194b9cc977dSBen Hutchings module_param(phy_flash_cfg, bool, 0644); 195874aeea5SJeff Kirsher MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 196874aeea5SJeff Kirsher 197e7bed9c8SBen Hutchings static unsigned irq_adapt_low_thresh = 8000; 198874aeea5SJeff Kirsher module_param(irq_adapt_low_thresh, uint, 0644); 199874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_low_thresh, 200874aeea5SJeff Kirsher "Threshold score for reducing IRQ moderation"); 201874aeea5SJeff Kirsher 202e7bed9c8SBen Hutchings static unsigned irq_adapt_high_thresh = 16000; 203874aeea5SJeff Kirsher module_param(irq_adapt_high_thresh, uint, 0644); 204874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_high_thresh, 205874aeea5SJeff Kirsher "Threshold score for increasing IRQ moderation"); 206874aeea5SJeff Kirsher 207874aeea5SJeff Kirsher static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 208874aeea5SJeff Kirsher NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 209874aeea5SJeff Kirsher NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 210874aeea5SJeff Kirsher NETIF_MSG_TX_ERR | NETIF_MSG_HW); 211874aeea5SJeff Kirsher module_param(debug, uint, 0); 212874aeea5SJeff Kirsher MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 213874aeea5SJeff Kirsher 214874aeea5SJeff Kirsher /************************************************************************** 215874aeea5SJeff Kirsher * 216874aeea5SJeff Kirsher * Utility functions and prototypes 217874aeea5SJeff Kirsher * 218874aeea5SJeff Kirsher *************************************************************************/ 219874aeea5SJeff Kirsher 220261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx); 221d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx); 2227f967c01SBen Hutchings static void efx_remove_channel(struct efx_channel *channel); 223874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx); 2247f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type; 225874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx); 2267f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel); 227874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx); 228874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel); 229874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx); 230874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx); 231874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx); 232874aeea5SJeff Kirsher 233874aeea5SJeff Kirsher #define EFX_ASSERT_RESET_SERIALISED(efx) \ 234874aeea5SJeff Kirsher do { \ 235f16aeea0SBen Hutchings if ((efx->state == STATE_READY) || \ 236626950dbSAlexandre Rames (efx->state == STATE_RECOVERY) || \ 237874aeea5SJeff Kirsher (efx->state == STATE_DISABLED)) \ 238874aeea5SJeff Kirsher ASSERT_RTNL(); \ 239874aeea5SJeff Kirsher } while (0) 240874aeea5SJeff Kirsher 2418b7325b4SBen Hutchings static int efx_check_disabled(struct efx_nic *efx) 2428b7325b4SBen Hutchings { 243626950dbSAlexandre Rames if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 2448b7325b4SBen Hutchings netif_err(efx, drv, efx->net_dev, 2458b7325b4SBen Hutchings "device is disabled due to earlier errors\n"); 2468b7325b4SBen Hutchings return -EIO; 2478b7325b4SBen Hutchings } 2488b7325b4SBen Hutchings return 0; 2498b7325b4SBen Hutchings } 2508b7325b4SBen Hutchings 251874aeea5SJeff Kirsher /************************************************************************** 252874aeea5SJeff Kirsher * 253874aeea5SJeff Kirsher * Event queue processing 254874aeea5SJeff Kirsher * 255874aeea5SJeff Kirsher *************************************************************************/ 256874aeea5SJeff Kirsher 257874aeea5SJeff Kirsher /* Process channel's event queue 258874aeea5SJeff Kirsher * 259874aeea5SJeff Kirsher * This function is responsible for processing the event queue of a 260874aeea5SJeff Kirsher * single channel. The caller must guarantee that this function will 261874aeea5SJeff Kirsher * never be concurrently called more than once on the same channel, 262874aeea5SJeff Kirsher * though different channels may be being processed concurrently. 263874aeea5SJeff Kirsher */ 264874aeea5SJeff Kirsher static int efx_process_channel(struct efx_channel *channel, int budget) 265874aeea5SJeff Kirsher { 266c936835cSPeter Dunning struct efx_tx_queue *tx_queue; 267e090bfb9SEdward Cree struct list_head rx_list; 268874aeea5SJeff Kirsher int spent; 269874aeea5SJeff Kirsher 2709f2cb71cSBen Hutchings if (unlikely(!channel->enabled)) 271874aeea5SJeff Kirsher return 0; 272874aeea5SJeff Kirsher 273e090bfb9SEdward Cree /* Prepare the batch receive list */ 274e090bfb9SEdward Cree EFX_WARN_ON_PARANOID(channel->rx_list != NULL); 275e090bfb9SEdward Cree INIT_LIST_HEAD(&rx_list); 276e090bfb9SEdward Cree channel->rx_list = &rx_list; 277e090bfb9SEdward Cree 278c936835cSPeter Dunning efx_for_each_channel_tx_queue(tx_queue, channel) { 279c936835cSPeter Dunning tx_queue->pkts_compl = 0; 280c936835cSPeter Dunning tx_queue->bytes_compl = 0; 281c936835cSPeter Dunning } 282c936835cSPeter Dunning 283874aeea5SJeff Kirsher spent = efx_nic_process_eventq(channel, budget); 284d9ab7007SBen Hutchings if (spent && efx_channel_has_rx_queue(channel)) { 285d9ab7007SBen Hutchings struct efx_rx_queue *rx_queue = 286d9ab7007SBen Hutchings efx_channel_get_rx_queue(channel); 287874aeea5SJeff Kirsher 288ff734ef4SBen Hutchings efx_rx_flush_packet(channel); 289cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, true); 290d9ab7007SBen Hutchings } 291874aeea5SJeff Kirsher 292c936835cSPeter Dunning /* Update BQL */ 293c936835cSPeter Dunning efx_for_each_channel_tx_queue(tx_queue, channel) { 294c936835cSPeter Dunning if (tx_queue->bytes_compl) { 295c936835cSPeter Dunning netdev_tx_completed_queue(tx_queue->core_txq, 296c936835cSPeter Dunning tx_queue->pkts_compl, tx_queue->bytes_compl); 297c936835cSPeter Dunning } 298c936835cSPeter Dunning } 299c936835cSPeter Dunning 300e090bfb9SEdward Cree /* Receive any packets we queued up */ 301e090bfb9SEdward Cree netif_receive_skb_list(channel->rx_list); 302e090bfb9SEdward Cree channel->rx_list = NULL; 303e090bfb9SEdward Cree 304874aeea5SJeff Kirsher return spent; 305874aeea5SJeff Kirsher } 306874aeea5SJeff Kirsher 307874aeea5SJeff Kirsher /* NAPI poll handler 308874aeea5SJeff Kirsher * 309874aeea5SJeff Kirsher * NAPI guarantees serialisation of polls of the same device, which 310874aeea5SJeff Kirsher * provides the guarantee required by efx_process_channel(). 311874aeea5SJeff Kirsher */ 312539de7c5SBert Kenward static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 313539de7c5SBert Kenward { 314539de7c5SBert Kenward int step = efx->irq_mod_step_us; 315539de7c5SBert Kenward 316539de7c5SBert Kenward if (channel->irq_mod_score < irq_adapt_low_thresh) { 317539de7c5SBert Kenward if (channel->irq_moderation_us > step) { 318539de7c5SBert Kenward channel->irq_moderation_us -= step; 319539de7c5SBert Kenward efx->type->push_irq_moderation(channel); 320539de7c5SBert Kenward } 321539de7c5SBert Kenward } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 322539de7c5SBert Kenward if (channel->irq_moderation_us < 323539de7c5SBert Kenward efx->irq_rx_moderation_us) { 324539de7c5SBert Kenward channel->irq_moderation_us += step; 325539de7c5SBert Kenward efx->type->push_irq_moderation(channel); 326539de7c5SBert Kenward } 327539de7c5SBert Kenward } 328539de7c5SBert Kenward 329539de7c5SBert Kenward channel->irq_count = 0; 330539de7c5SBert Kenward channel->irq_mod_score = 0; 331539de7c5SBert Kenward } 332539de7c5SBert Kenward 333874aeea5SJeff Kirsher static int efx_poll(struct napi_struct *napi, int budget) 334874aeea5SJeff Kirsher { 335874aeea5SJeff Kirsher struct efx_channel *channel = 336874aeea5SJeff Kirsher container_of(napi, struct efx_channel, napi_str); 337874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 338874aeea5SJeff Kirsher int spent; 339874aeea5SJeff Kirsher 340874aeea5SJeff Kirsher netif_vdbg(efx, intr, efx->net_dev, 341874aeea5SJeff Kirsher "channel %d NAPI poll executing on CPU %d\n", 342874aeea5SJeff Kirsher channel->channel, raw_smp_processor_id()); 343874aeea5SJeff Kirsher 344874aeea5SJeff Kirsher spent = efx_process_channel(channel, budget); 345874aeea5SJeff Kirsher 346874aeea5SJeff Kirsher if (spent < budget) { 3479d9a6973SBen Hutchings if (efx_channel_has_rx_queue(channel) && 348874aeea5SJeff Kirsher efx->irq_rx_adaptive && 349874aeea5SJeff Kirsher unlikely(++channel->irq_count == 1000)) { 350539de7c5SBert Kenward efx_update_irq_mod(efx, channel); 351874aeea5SJeff Kirsher } 352874aeea5SJeff Kirsher 3533af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 3543af0f342SEdward Cree /* Perhaps expire some ARFS filters */ 3553af0f342SEdward Cree schedule_work(&channel->filter_work); 3563af0f342SEdward Cree #endif 357874aeea5SJeff Kirsher 358874aeea5SJeff Kirsher /* There is no race here; although napi_disable() will 359874aeea5SJeff Kirsher * only wait for napi_complete(), this isn't a problem 360514bedbcSBen Hutchings * since efx_nic_eventq_read_ack() will have no effect if 361874aeea5SJeff Kirsher * interrupts have already been disabled. 362874aeea5SJeff Kirsher */ 363f820c0acSBert Kenward if (napi_complete_done(napi, spent)) 364514bedbcSBen Hutchings efx_nic_eventq_read_ack(channel); 365874aeea5SJeff Kirsher } 366874aeea5SJeff Kirsher 367874aeea5SJeff Kirsher return spent; 368874aeea5SJeff Kirsher } 369874aeea5SJeff Kirsher 370874aeea5SJeff Kirsher /* Create event queue 371874aeea5SJeff Kirsher * Event queue memory allocations are done only once. If the channel 372874aeea5SJeff Kirsher * is reset, the memory buffer will be reused; this guards against 373874aeea5SJeff Kirsher * errors during channel reset and also simplifies interrupt handling. 374874aeea5SJeff Kirsher */ 375874aeea5SJeff Kirsher static int efx_probe_eventq(struct efx_channel *channel) 376874aeea5SJeff Kirsher { 377874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 378874aeea5SJeff Kirsher unsigned long entries; 379874aeea5SJeff Kirsher 38086ee5302SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 381874aeea5SJeff Kirsher "chan %d create event queue\n", channel->channel); 382874aeea5SJeff Kirsher 383874aeea5SJeff Kirsher /* Build an event queue with room for one event per tx and rx buffer, 384874aeea5SJeff Kirsher * plus some extra for link state events and MCDI completions. */ 385874aeea5SJeff Kirsher entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 386e01b16a7SEdward Cree EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 387874aeea5SJeff Kirsher channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 388874aeea5SJeff Kirsher 389874aeea5SJeff Kirsher return efx_nic_probe_eventq(channel); 390874aeea5SJeff Kirsher } 391874aeea5SJeff Kirsher 392874aeea5SJeff Kirsher /* Prepare channel's event queue */ 393261e4d96SJon Cooper static int efx_init_eventq(struct efx_channel *channel) 394874aeea5SJeff Kirsher { 39515acb1ceSBen Hutchings struct efx_nic *efx = channel->efx; 396261e4d96SJon Cooper int rc; 397261e4d96SJon Cooper 398261e4d96SJon Cooper EFX_WARN_ON_PARANOID(channel->eventq_init); 399261e4d96SJon Cooper 40015acb1ceSBen Hutchings netif_dbg(efx, drv, efx->net_dev, 401874aeea5SJeff Kirsher "chan %d init event queue\n", channel->channel); 402874aeea5SJeff Kirsher 403261e4d96SJon Cooper rc = efx_nic_init_eventq(channel); 404261e4d96SJon Cooper if (rc == 0) { 40515acb1ceSBen Hutchings efx->type->push_irq_moderation(channel); 406874aeea5SJeff Kirsher channel->eventq_read_ptr = 0; 407be3fc09cSBen Hutchings channel->eventq_init = true; 408874aeea5SJeff Kirsher } 409261e4d96SJon Cooper return rc; 410261e4d96SJon Cooper } 411874aeea5SJeff Kirsher 4129f2cb71cSBen Hutchings /* Enable event queue processing and NAPI */ 41336763266SAlexandre Rames void efx_start_eventq(struct efx_channel *channel) 4149f2cb71cSBen Hutchings { 4159f2cb71cSBen Hutchings netif_dbg(channel->efx, ifup, channel->efx->net_dev, 4169f2cb71cSBen Hutchings "chan %d start event queue\n", channel->channel); 4179f2cb71cSBen Hutchings 418514bedbcSBen Hutchings /* Make sure the NAPI handler sees the enabled flag set */ 4199f2cb71cSBen Hutchings channel->enabled = true; 4209f2cb71cSBen Hutchings smp_wmb(); 4219f2cb71cSBen Hutchings 4229f2cb71cSBen Hutchings napi_enable(&channel->napi_str); 4239f2cb71cSBen Hutchings efx_nic_eventq_read_ack(channel); 4249f2cb71cSBen Hutchings } 4259f2cb71cSBen Hutchings 4269f2cb71cSBen Hutchings /* Disable event queue processing and NAPI */ 42736763266SAlexandre Rames void efx_stop_eventq(struct efx_channel *channel) 4289f2cb71cSBen Hutchings { 4299f2cb71cSBen Hutchings if (!channel->enabled) 4309f2cb71cSBen Hutchings return; 4319f2cb71cSBen Hutchings 4329f2cb71cSBen Hutchings napi_disable(&channel->napi_str); 4339f2cb71cSBen Hutchings channel->enabled = false; 4349f2cb71cSBen Hutchings } 4359f2cb71cSBen Hutchings 436874aeea5SJeff Kirsher static void efx_fini_eventq(struct efx_channel *channel) 437874aeea5SJeff Kirsher { 438be3fc09cSBen Hutchings if (!channel->eventq_init) 439be3fc09cSBen Hutchings return; 440be3fc09cSBen Hutchings 441874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 442874aeea5SJeff Kirsher "chan %d fini event queue\n", channel->channel); 443874aeea5SJeff Kirsher 444874aeea5SJeff Kirsher efx_nic_fini_eventq(channel); 445be3fc09cSBen Hutchings channel->eventq_init = false; 446874aeea5SJeff Kirsher } 447874aeea5SJeff Kirsher 448874aeea5SJeff Kirsher static void efx_remove_eventq(struct efx_channel *channel) 449874aeea5SJeff Kirsher { 450874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 451874aeea5SJeff Kirsher "chan %d remove event queue\n", channel->channel); 452874aeea5SJeff Kirsher 453874aeea5SJeff Kirsher efx_nic_remove_eventq(channel); 454874aeea5SJeff Kirsher } 455874aeea5SJeff Kirsher 456874aeea5SJeff Kirsher /************************************************************************** 457874aeea5SJeff Kirsher * 458874aeea5SJeff Kirsher * Channel handling 459874aeea5SJeff Kirsher * 460874aeea5SJeff Kirsher *************************************************************************/ 461874aeea5SJeff Kirsher 4627f967c01SBen Hutchings /* Allocate and initialise a channel structure. */ 463874aeea5SJeff Kirsher static struct efx_channel * 464874aeea5SJeff Kirsher efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 465874aeea5SJeff Kirsher { 466874aeea5SJeff Kirsher struct efx_channel *channel; 467874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 468874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 469874aeea5SJeff Kirsher int j; 470874aeea5SJeff Kirsher 4717f967c01SBen Hutchings channel = kzalloc(sizeof(*channel), GFP_KERNEL); 4727f967c01SBen Hutchings if (!channel) 4737f967c01SBen Hutchings return NULL; 4747f967c01SBen Hutchings 4757f967c01SBen Hutchings channel->efx = efx; 4767f967c01SBen Hutchings channel->channel = i; 4777f967c01SBen Hutchings channel->type = &efx_default_channel_type; 4787f967c01SBen Hutchings 4797f967c01SBen Hutchings for (j = 0; j < EFX_TXQ_TYPES; j++) { 4807f967c01SBen Hutchings tx_queue = &channel->tx_queue[j]; 4817f967c01SBen Hutchings tx_queue->efx = efx; 4827f967c01SBen Hutchings tx_queue->queue = i * EFX_TXQ_TYPES + j; 4837f967c01SBen Hutchings tx_queue->channel = channel; 4847f967c01SBen Hutchings } 4857f967c01SBen Hutchings 4863af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 4873af0f342SEdward Cree INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 4883af0f342SEdward Cree #endif 4893af0f342SEdward Cree 4907f967c01SBen Hutchings rx_queue = &channel->rx_queue; 4917f967c01SBen Hutchings rx_queue->efx = efx; 4927aa1402eSKees Cook timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 4937f967c01SBen Hutchings 4947f967c01SBen Hutchings return channel; 4957f967c01SBen Hutchings } 4967f967c01SBen Hutchings 4977f967c01SBen Hutchings /* Allocate and initialise a channel structure, copying parameters 4987f967c01SBen Hutchings * (but not resources) from an old channel structure. 4997f967c01SBen Hutchings */ 5007f967c01SBen Hutchings static struct efx_channel * 5017f967c01SBen Hutchings efx_copy_channel(const struct efx_channel *old_channel) 5027f967c01SBen Hutchings { 5037f967c01SBen Hutchings struct efx_channel *channel; 5047f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 5057f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 5067f967c01SBen Hutchings int j; 5077f967c01SBen Hutchings 508874aeea5SJeff Kirsher channel = kmalloc(sizeof(*channel), GFP_KERNEL); 509874aeea5SJeff Kirsher if (!channel) 510874aeea5SJeff Kirsher return NULL; 511874aeea5SJeff Kirsher 512874aeea5SJeff Kirsher *channel = *old_channel; 513874aeea5SJeff Kirsher 514874aeea5SJeff Kirsher channel->napi_dev = NULL; 51546d054f8SBert Kenward INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 51646d054f8SBert Kenward channel->napi_str.napi_id = 0; 51746d054f8SBert Kenward channel->napi_str.state = 0; 518874aeea5SJeff Kirsher memset(&channel->eventq, 0, sizeof(channel->eventq)); 519874aeea5SJeff Kirsher 520874aeea5SJeff Kirsher for (j = 0; j < EFX_TXQ_TYPES; j++) { 521874aeea5SJeff Kirsher tx_queue = &channel->tx_queue[j]; 522874aeea5SJeff Kirsher if (tx_queue->channel) 523874aeea5SJeff Kirsher tx_queue->channel = channel; 524874aeea5SJeff Kirsher tx_queue->buffer = NULL; 525874aeea5SJeff Kirsher memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 526874aeea5SJeff Kirsher } 527874aeea5SJeff Kirsher 528874aeea5SJeff Kirsher rx_queue = &channel->rx_queue; 5297f967c01SBen Hutchings rx_queue->buffer = NULL; 5307f967c01SBen Hutchings memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 5317aa1402eSKees Cook timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 5323af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 5333af0f342SEdward Cree INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); 5343af0f342SEdward Cree #endif 535874aeea5SJeff Kirsher 536874aeea5SJeff Kirsher return channel; 537874aeea5SJeff Kirsher } 538874aeea5SJeff Kirsher 539874aeea5SJeff Kirsher static int efx_probe_channel(struct efx_channel *channel) 540874aeea5SJeff Kirsher { 541874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 542874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 543874aeea5SJeff Kirsher int rc; 544874aeea5SJeff Kirsher 545874aeea5SJeff Kirsher netif_dbg(channel->efx, probe, channel->efx->net_dev, 546874aeea5SJeff Kirsher "creating channel %d\n", channel->channel); 547874aeea5SJeff Kirsher 5487f967c01SBen Hutchings rc = channel->type->pre_probe(channel); 5497f967c01SBen Hutchings if (rc) 5507f967c01SBen Hutchings goto fail; 5517f967c01SBen Hutchings 552874aeea5SJeff Kirsher rc = efx_probe_eventq(channel); 553874aeea5SJeff Kirsher if (rc) 5547f967c01SBen Hutchings goto fail; 555874aeea5SJeff Kirsher 556874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) { 557874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 558874aeea5SJeff Kirsher if (rc) 5597f967c01SBen Hutchings goto fail; 560874aeea5SJeff Kirsher } 561874aeea5SJeff Kirsher 562874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) { 563874aeea5SJeff Kirsher rc = efx_probe_rx_queue(rx_queue); 564874aeea5SJeff Kirsher if (rc) 5657f967c01SBen Hutchings goto fail; 566874aeea5SJeff Kirsher } 567874aeea5SJeff Kirsher 568e090bfb9SEdward Cree channel->rx_list = NULL; 569e090bfb9SEdward Cree 570874aeea5SJeff Kirsher return 0; 571874aeea5SJeff Kirsher 5727f967c01SBen Hutchings fail: 5737f967c01SBen Hutchings efx_remove_channel(channel); 574874aeea5SJeff Kirsher return rc; 575874aeea5SJeff Kirsher } 576874aeea5SJeff Kirsher 5777f967c01SBen Hutchings static void 5787f967c01SBen Hutchings efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 5797f967c01SBen Hutchings { 5807f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 5817f967c01SBen Hutchings const char *type; 5827f967c01SBen Hutchings int number; 5837f967c01SBen Hutchings 5847f967c01SBen Hutchings number = channel->channel; 5857f967c01SBen Hutchings if (efx->tx_channel_offset == 0) { 5867f967c01SBen Hutchings type = ""; 5877f967c01SBen Hutchings } else if (channel->channel < efx->tx_channel_offset) { 5887f967c01SBen Hutchings type = "-rx"; 5897f967c01SBen Hutchings } else { 5907f967c01SBen Hutchings type = "-tx"; 5917f967c01SBen Hutchings number -= efx->tx_channel_offset; 5927f967c01SBen Hutchings } 5937f967c01SBen Hutchings snprintf(buf, len, "%s%s-%d", efx->name, type, number); 5947f967c01SBen Hutchings } 595874aeea5SJeff Kirsher 596874aeea5SJeff Kirsher static void efx_set_channel_names(struct efx_nic *efx) 597874aeea5SJeff Kirsher { 598874aeea5SJeff Kirsher struct efx_channel *channel; 599874aeea5SJeff Kirsher 6007f967c01SBen Hutchings efx_for_each_channel(channel, efx) 6017f967c01SBen Hutchings channel->type->get_name(channel, 602d8291187SBen Hutchings efx->msi_context[channel->channel].name, 603d8291187SBen Hutchings sizeof(efx->msi_context[0].name)); 604874aeea5SJeff Kirsher } 605874aeea5SJeff Kirsher 606874aeea5SJeff Kirsher static int efx_probe_channels(struct efx_nic *efx) 607874aeea5SJeff Kirsher { 608874aeea5SJeff Kirsher struct efx_channel *channel; 609874aeea5SJeff Kirsher int rc; 610874aeea5SJeff Kirsher 611874aeea5SJeff Kirsher /* Restart special buffer allocation */ 612874aeea5SJeff Kirsher efx->next_buffer_table = 0; 613874aeea5SJeff Kirsher 614c92aaff1SBen Hutchings /* Probe channels in reverse, so that any 'extra' channels 615c92aaff1SBen Hutchings * use the start of the buffer table. This allows the traffic 616c92aaff1SBen Hutchings * channels to be resized without moving them or wasting the 617c92aaff1SBen Hutchings * entries before them. 618c92aaff1SBen Hutchings */ 619c92aaff1SBen Hutchings efx_for_each_channel_rev(channel, efx) { 620874aeea5SJeff Kirsher rc = efx_probe_channel(channel); 621874aeea5SJeff Kirsher if (rc) { 622874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 623874aeea5SJeff Kirsher "failed to create channel %d\n", 624874aeea5SJeff Kirsher channel->channel); 625874aeea5SJeff Kirsher goto fail; 626874aeea5SJeff Kirsher } 627874aeea5SJeff Kirsher } 628874aeea5SJeff Kirsher efx_set_channel_names(efx); 629874aeea5SJeff Kirsher 630874aeea5SJeff Kirsher return 0; 631874aeea5SJeff Kirsher 632874aeea5SJeff Kirsher fail: 633874aeea5SJeff Kirsher efx_remove_channels(efx); 634874aeea5SJeff Kirsher return rc; 635874aeea5SJeff Kirsher } 636874aeea5SJeff Kirsher 637874aeea5SJeff Kirsher /* Channels are shutdown and reinitialised whilst the NIC is running 638874aeea5SJeff Kirsher * to propagate configuration changes (mtu, checksum offload), or 639874aeea5SJeff Kirsher * to clear hardware error conditions 640874aeea5SJeff Kirsher */ 6419f2cb71cSBen Hutchings static void efx_start_datapath(struct efx_nic *efx) 642874aeea5SJeff Kirsher { 643ebfcd0fdSAndrew Rybchenko netdev_features_t old_features = efx->net_dev->features; 64485740cdfSBen Hutchings bool old_rx_scatter = efx->rx_scatter; 645874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 646874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 647874aeea5SJeff Kirsher struct efx_channel *channel; 64885740cdfSBen Hutchings size_t rx_buf_len; 649874aeea5SJeff Kirsher 650874aeea5SJeff Kirsher /* Calculate the rx buffer allocation parameters required to 651874aeea5SJeff Kirsher * support the current MTU, including padding for header 652874aeea5SJeff Kirsher * alignment and overruns. 653874aeea5SJeff Kirsher */ 65443a3739dSJon Cooper efx->rx_dma_len = (efx->rx_prefix_size + 655874aeea5SJeff Kirsher EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 656874aeea5SJeff Kirsher efx->type->rx_buffer_padding); 65785740cdfSBen Hutchings rx_buf_len = (sizeof(struct efx_rx_page_state) + 6582ec03014SAndrew Rybchenko efx->rx_ip_align + efx->rx_dma_len); 65985740cdfSBen Hutchings if (rx_buf_len <= PAGE_SIZE) { 660e8c68c0aSJon Cooper efx->rx_scatter = efx->type->always_rx_scatter; 66185740cdfSBen Hutchings efx->rx_buffer_order = 0; 66285740cdfSBen Hutchings } else if (efx->type->can_rx_scatter) { 663950c54dfSBen Hutchings BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 66485740cdfSBen Hutchings BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 665950c54dfSBen Hutchings 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 666950c54dfSBen Hutchings EFX_RX_BUF_ALIGNMENT) > 667950c54dfSBen Hutchings PAGE_SIZE); 66885740cdfSBen Hutchings efx->rx_scatter = true; 66985740cdfSBen Hutchings efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 67085740cdfSBen Hutchings efx->rx_buffer_order = 0; 67185740cdfSBen Hutchings } else { 67285740cdfSBen Hutchings efx->rx_scatter = false; 67385740cdfSBen Hutchings efx->rx_buffer_order = get_order(rx_buf_len); 67485740cdfSBen Hutchings } 67585740cdfSBen Hutchings 6761648a23fSDaniel Pieczko efx_rx_config_page_split(efx); 6771648a23fSDaniel Pieczko if (efx->rx_buffer_order) 6781648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6791648a23fSDaniel Pieczko "RX buf len=%u; page order=%u batch=%u\n", 6801648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_buffer_order, 6811648a23fSDaniel Pieczko efx->rx_pages_per_batch); 6821648a23fSDaniel Pieczko else 6831648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6841648a23fSDaniel Pieczko "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 6851648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_page_buf_step, 6861648a23fSDaniel Pieczko efx->rx_bufs_per_page, efx->rx_pages_per_batch); 6872768935aSDaniel Pieczko 688ebfcd0fdSAndrew Rybchenko /* Restore previously fixed features in hw_features and remove 689ebfcd0fdSAndrew Rybchenko * features which are fixed now 690ebfcd0fdSAndrew Rybchenko */ 691ebfcd0fdSAndrew Rybchenko efx->net_dev->hw_features |= efx->net_dev->features; 692ebfcd0fdSAndrew Rybchenko efx->net_dev->hw_features &= ~efx->fixed_features; 693ebfcd0fdSAndrew Rybchenko efx->net_dev->features |= efx->fixed_features; 694ebfcd0fdSAndrew Rybchenko if (efx->net_dev->features != old_features) 695ebfcd0fdSAndrew Rybchenko netdev_features_change(efx->net_dev); 696ebfcd0fdSAndrew Rybchenko 697e8c68c0aSJon Cooper /* RX filters may also have scatter-enabled flags */ 69885740cdfSBen Hutchings if (efx->rx_scatter != old_rx_scatter) 699add72477SBen Hutchings efx->type->filter_update_rx_scatter(efx); 700874aeea5SJeff Kirsher 70114bf718fSBen Hutchings /* We must keep at least one descriptor in a TX ring empty. 70214bf718fSBen Hutchings * We could avoid this when the queue size does not exactly 70314bf718fSBen Hutchings * match the hardware ring size, but it's not that important. 70414bf718fSBen Hutchings * Therefore we stop the queue when one more skb might fill 70514bf718fSBen Hutchings * the ring completely. We wake it when half way back to 70614bf718fSBen Hutchings * empty. 70714bf718fSBen Hutchings */ 70814bf718fSBen Hutchings efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 70914bf718fSBen Hutchings efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 71014bf718fSBen Hutchings 711874aeea5SJeff Kirsher /* Initialise the channels */ 712874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 7133881d8abSAlexandre Rames efx_for_each_channel_tx_queue(tx_queue, channel) { 714874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 7153881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 7163881d8abSAlexandre Rames } 717874aeea5SJeff Kirsher 7189f2cb71cSBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) { 719874aeea5SJeff Kirsher efx_init_rx_queue(rx_queue); 7203881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 721cce28794SJon Cooper efx_stop_eventq(channel); 722cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, false); 723cce28794SJon Cooper efx_start_eventq(channel); 7249f2cb71cSBen Hutchings } 725874aeea5SJeff Kirsher 72685740cdfSBen Hutchings WARN_ON(channel->rx_pkt_n_frags); 727874aeea5SJeff Kirsher } 7289f2cb71cSBen Hutchings 7292ea4dc28SAlexandre Rames efx_ptp_start_datapath(efx); 7302ea4dc28SAlexandre Rames 7319f2cb71cSBen Hutchings if (netif_device_present(efx->net_dev)) 7329f2cb71cSBen Hutchings netif_tx_wake_all_queues(efx->net_dev); 733874aeea5SJeff Kirsher } 734874aeea5SJeff Kirsher 7359f2cb71cSBen Hutchings static void efx_stop_datapath(struct efx_nic *efx) 736874aeea5SJeff Kirsher { 737874aeea5SJeff Kirsher struct efx_channel *channel; 738874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 739874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 740874aeea5SJeff Kirsher int rc; 741874aeea5SJeff Kirsher 742874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 743874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 744874aeea5SJeff Kirsher 7452ea4dc28SAlexandre Rames efx_ptp_stop_datapath(efx); 7462ea4dc28SAlexandre Rames 747d8aec745SBen Hutchings /* Stop RX refill */ 748d8aec745SBen Hutchings efx_for_each_channel(channel, efx) { 749d8aec745SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 750d8aec745SBen Hutchings rx_queue->refill_enabled = false; 751d8aec745SBen Hutchings } 752d8aec745SBen Hutchings 753874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 7549f2cb71cSBen Hutchings /* RX packet processing is pipelined, so wait for the 7559f2cb71cSBen Hutchings * NAPI handler to complete. At least event queue 0 7569f2cb71cSBen Hutchings * might be kept active by non-data events, so don't 7579f2cb71cSBen Hutchings * use napi_synchronize() but actually disable NAPI 7589f2cb71cSBen Hutchings * temporarily. 7599f2cb71cSBen Hutchings */ 7609f2cb71cSBen Hutchings if (efx_channel_has_rx_queue(channel)) { 7619f2cb71cSBen Hutchings efx_stop_eventq(channel); 7629f2cb71cSBen Hutchings efx_start_eventq(channel); 7639f2cb71cSBen Hutchings } 764e42c3d85SBen Hutchings } 765874aeea5SJeff Kirsher 766e42c3d85SBen Hutchings rc = efx->type->fini_dmaq(efx); 7675a6681e2SEdward Cree if (rc) { 768e42c3d85SBen Hutchings netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 769e42c3d85SBen Hutchings } else { 770e42c3d85SBen Hutchings netif_dbg(efx, drv, efx->net_dev, 771e42c3d85SBen Hutchings "successfully flushed all queues\n"); 772e42c3d85SBen Hutchings } 773e42c3d85SBen Hutchings 774e42c3d85SBen Hutchings efx_for_each_channel(channel, efx) { 775874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 776874aeea5SJeff Kirsher efx_fini_rx_queue(rx_queue); 777874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 778874aeea5SJeff Kirsher efx_fini_tx_queue(tx_queue); 779874aeea5SJeff Kirsher } 780874aeea5SJeff Kirsher } 781874aeea5SJeff Kirsher 782874aeea5SJeff Kirsher static void efx_remove_channel(struct efx_channel *channel) 783874aeea5SJeff Kirsher { 784874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 785874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 786874aeea5SJeff Kirsher 787874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 788874aeea5SJeff Kirsher "destroy chan %d\n", channel->channel); 789874aeea5SJeff Kirsher 790874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 791874aeea5SJeff Kirsher efx_remove_rx_queue(rx_queue); 792874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 793874aeea5SJeff Kirsher efx_remove_tx_queue(tx_queue); 794874aeea5SJeff Kirsher efx_remove_eventq(channel); 795c31e5f9fSStuart Hodgson channel->type->post_remove(channel); 796874aeea5SJeff Kirsher } 797874aeea5SJeff Kirsher 798874aeea5SJeff Kirsher static void efx_remove_channels(struct efx_nic *efx) 799874aeea5SJeff Kirsher { 800874aeea5SJeff Kirsher struct efx_channel *channel; 801874aeea5SJeff Kirsher 802874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 803874aeea5SJeff Kirsher efx_remove_channel(channel); 804874aeea5SJeff Kirsher } 805874aeea5SJeff Kirsher 806874aeea5SJeff Kirsher int 807874aeea5SJeff Kirsher efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 808874aeea5SJeff Kirsher { 809874aeea5SJeff Kirsher struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 810874aeea5SJeff Kirsher u32 old_rxq_entries, old_txq_entries; 8117f967c01SBen Hutchings unsigned i, next_buffer_table = 0; 812261e4d96SJon Cooper int rc, rc2; 8138b7325b4SBen Hutchings 8148b7325b4SBen Hutchings rc = efx_check_disabled(efx); 8158b7325b4SBen Hutchings if (rc) 8168b7325b4SBen Hutchings return rc; 8177f967c01SBen Hutchings 8187f967c01SBen Hutchings /* Not all channels should be reallocated. We must avoid 8197f967c01SBen Hutchings * reallocating their buffer table entries. 8207f967c01SBen Hutchings */ 8217f967c01SBen Hutchings efx_for_each_channel(channel, efx) { 8227f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 8237f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 8247f967c01SBen Hutchings 8257f967c01SBen Hutchings if (channel->type->copy) 8267f967c01SBen Hutchings continue; 8277f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8287f967c01SBen Hutchings channel->eventq.index + 8297f967c01SBen Hutchings channel->eventq.entries); 8307f967c01SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 8317f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8327f967c01SBen Hutchings rx_queue->rxd.index + 8337f967c01SBen Hutchings rx_queue->rxd.entries); 8347f967c01SBen Hutchings efx_for_each_channel_tx_queue(tx_queue, channel) 8357f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8367f967c01SBen Hutchings tx_queue->txd.index + 8377f967c01SBen Hutchings tx_queue->txd.entries); 8387f967c01SBen Hutchings } 839874aeea5SJeff Kirsher 84029c69a48SBen Hutchings efx_device_detach_sync(efx); 841874aeea5SJeff Kirsher efx_stop_all(efx); 842d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 843874aeea5SJeff Kirsher 8447f967c01SBen Hutchings /* Clone channels (where possible) */ 845874aeea5SJeff Kirsher memset(other_channel, 0, sizeof(other_channel)); 846874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 8477f967c01SBen Hutchings channel = efx->channel[i]; 8487f967c01SBen Hutchings if (channel->type->copy) 8497f967c01SBen Hutchings channel = channel->type->copy(channel); 850874aeea5SJeff Kirsher if (!channel) { 851874aeea5SJeff Kirsher rc = -ENOMEM; 852874aeea5SJeff Kirsher goto out; 853874aeea5SJeff Kirsher } 854874aeea5SJeff Kirsher other_channel[i] = channel; 855874aeea5SJeff Kirsher } 856874aeea5SJeff Kirsher 857874aeea5SJeff Kirsher /* Swap entry counts and channel pointers */ 858874aeea5SJeff Kirsher old_rxq_entries = efx->rxq_entries; 859874aeea5SJeff Kirsher old_txq_entries = efx->txq_entries; 860874aeea5SJeff Kirsher efx->rxq_entries = rxq_entries; 861874aeea5SJeff Kirsher efx->txq_entries = txq_entries; 862874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 863874aeea5SJeff Kirsher channel = efx->channel[i]; 864874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 865874aeea5SJeff Kirsher other_channel[i] = channel; 866874aeea5SJeff Kirsher } 867874aeea5SJeff Kirsher 8687f967c01SBen Hutchings /* Restart buffer table allocation */ 8697f967c01SBen Hutchings efx->next_buffer_table = next_buffer_table; 8707f967c01SBen Hutchings 8717f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8727f967c01SBen Hutchings channel = efx->channel[i]; 8737f967c01SBen Hutchings if (!channel->type->copy) 8747f967c01SBen Hutchings continue; 8757f967c01SBen Hutchings rc = efx_probe_channel(channel); 876874aeea5SJeff Kirsher if (rc) 877874aeea5SJeff Kirsher goto rollback; 8787f967c01SBen Hutchings efx_init_napi_channel(efx->channel[i]); 879874aeea5SJeff Kirsher } 880874aeea5SJeff Kirsher 8817f967c01SBen Hutchings out: 8827f967c01SBen Hutchings /* Destroy unused channel structures */ 8837f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8847f967c01SBen Hutchings channel = other_channel[i]; 8857f967c01SBen Hutchings if (channel && channel->type->copy) { 8867f967c01SBen Hutchings efx_fini_napi_channel(channel); 8877f967c01SBen Hutchings efx_remove_channel(channel); 8887f967c01SBen Hutchings kfree(channel); 8897f967c01SBen Hutchings } 8907f967c01SBen Hutchings } 8917f967c01SBen Hutchings 892261e4d96SJon Cooper rc2 = efx_soft_enable_interrupts(efx); 893261e4d96SJon Cooper if (rc2) { 894261e4d96SJon Cooper rc = rc ? rc : rc2; 895261e4d96SJon Cooper netif_err(efx, drv, efx->net_dev, 896261e4d96SJon Cooper "unable to restart interrupts on channel reallocation\n"); 897261e4d96SJon Cooper efx_schedule_reset(efx, RESET_TYPE_DISABLE); 898261e4d96SJon Cooper } else { 899874aeea5SJeff Kirsher efx_start_all(efx); 9009c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 901261e4d96SJon Cooper } 902874aeea5SJeff Kirsher return rc; 903874aeea5SJeff Kirsher 904874aeea5SJeff Kirsher rollback: 905874aeea5SJeff Kirsher /* Swap back */ 906874aeea5SJeff Kirsher efx->rxq_entries = old_rxq_entries; 907874aeea5SJeff Kirsher efx->txq_entries = old_txq_entries; 908874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 909874aeea5SJeff Kirsher channel = efx->channel[i]; 910874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 911874aeea5SJeff Kirsher other_channel[i] = channel; 912874aeea5SJeff Kirsher } 913874aeea5SJeff Kirsher goto out; 914874aeea5SJeff Kirsher } 915874aeea5SJeff Kirsher 916874aeea5SJeff Kirsher void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 917874aeea5SJeff Kirsher { 918874aeea5SJeff Kirsher mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 919874aeea5SJeff Kirsher } 920874aeea5SJeff Kirsher 921e7345ba3Skbuild test robot static bool efx_default_channel_want_txqs(struct efx_channel *channel) 9222935e3c3SEdward Cree { 9232935e3c3SEdward Cree return channel->channel - channel->efx->tx_channel_offset < 9242935e3c3SEdward Cree channel->efx->n_tx_channels; 9252935e3c3SEdward Cree } 9262935e3c3SEdward Cree 9277f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type = { 9287f967c01SBen Hutchings .pre_probe = efx_channel_dummy_op_int, 929c31e5f9fSStuart Hodgson .post_remove = efx_channel_dummy_op_void, 9307f967c01SBen Hutchings .get_name = efx_get_channel_name, 9317f967c01SBen Hutchings .copy = efx_copy_channel, 9322935e3c3SEdward Cree .want_txqs = efx_default_channel_want_txqs, 9337f967c01SBen Hutchings .keep_eventq = false, 9342935e3c3SEdward Cree .want_pio = true, 9357f967c01SBen Hutchings }; 9367f967c01SBen Hutchings 9377f967c01SBen Hutchings int efx_channel_dummy_op_int(struct efx_channel *channel) 9387f967c01SBen Hutchings { 9397f967c01SBen Hutchings return 0; 9407f967c01SBen Hutchings } 9417f967c01SBen Hutchings 942c31e5f9fSStuart Hodgson void efx_channel_dummy_op_void(struct efx_channel *channel) 943c31e5f9fSStuart Hodgson { 944c31e5f9fSStuart Hodgson } 945c31e5f9fSStuart Hodgson 946874aeea5SJeff Kirsher /************************************************************************** 947874aeea5SJeff Kirsher * 948874aeea5SJeff Kirsher * Port handling 949874aeea5SJeff Kirsher * 950874aeea5SJeff Kirsher **************************************************************************/ 951874aeea5SJeff Kirsher 952874aeea5SJeff Kirsher /* This ensures that the kernel is kept informed (via 953874aeea5SJeff Kirsher * netif_carrier_on/off) of the link status, and also maintains the 954874aeea5SJeff Kirsher * link status's stop on the port's TX queue. 955874aeea5SJeff Kirsher */ 956874aeea5SJeff Kirsher void efx_link_status_changed(struct efx_nic *efx) 957874aeea5SJeff Kirsher { 958874aeea5SJeff Kirsher struct efx_link_state *link_state = &efx->link_state; 959874aeea5SJeff Kirsher 960874aeea5SJeff Kirsher /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 961874aeea5SJeff Kirsher * that no events are triggered between unregister_netdev() and the 962874aeea5SJeff Kirsher * driver unloading. A more general condition is that NETDEV_CHANGE 963874aeea5SJeff Kirsher * can only be generated between NETDEV_UP and NETDEV_DOWN */ 964874aeea5SJeff Kirsher if (!netif_running(efx->net_dev)) 965874aeea5SJeff Kirsher return; 966874aeea5SJeff Kirsher 967874aeea5SJeff Kirsher if (link_state->up != netif_carrier_ok(efx->net_dev)) { 968874aeea5SJeff Kirsher efx->n_link_state_changes++; 969874aeea5SJeff Kirsher 970874aeea5SJeff Kirsher if (link_state->up) 971874aeea5SJeff Kirsher netif_carrier_on(efx->net_dev); 972874aeea5SJeff Kirsher else 973874aeea5SJeff Kirsher netif_carrier_off(efx->net_dev); 974874aeea5SJeff Kirsher } 975874aeea5SJeff Kirsher 976874aeea5SJeff Kirsher /* Status message for kernel log */ 9772aa9ef11SBen Hutchings if (link_state->up) 978874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, 979964e6135SBen Hutchings "link up at %uMbps %s-duplex (MTU %d)\n", 980874aeea5SJeff Kirsher link_state->speed, link_state->fd ? "full" : "half", 981964e6135SBen Hutchings efx->net_dev->mtu); 9822aa9ef11SBen Hutchings else 983874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, "link down\n"); 984874aeea5SJeff Kirsher } 985874aeea5SJeff Kirsher 986c2ab85d2SEdward Cree void efx_link_set_advertising(struct efx_nic *efx, 987c2ab85d2SEdward Cree const unsigned long *advertising) 988874aeea5SJeff Kirsher { 989c2ab85d2SEdward Cree memcpy(efx->link_advertising, advertising, 990c2ab85d2SEdward Cree sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK())); 991c2ab85d2SEdward Cree 992c2ab85d2SEdward Cree efx->link_advertising[0] |= ADVERTISED_Autoneg; 993c2ab85d2SEdward Cree if (advertising[0] & ADVERTISED_Pause) 994874aeea5SJeff Kirsher efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 995874aeea5SJeff Kirsher else 996874aeea5SJeff Kirsher efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 997c2ab85d2SEdward Cree if (advertising[0] & ADVERTISED_Asym_Pause) 998874aeea5SJeff Kirsher efx->wanted_fc ^= EFX_FC_TX; 999874aeea5SJeff Kirsher } 1000c2ab85d2SEdward Cree 1001c2ab85d2SEdward Cree /* Equivalent to efx_link_set_advertising with all-zeroes, except does not 1002c2ab85d2SEdward Cree * force the Autoneg bit on. 1003c2ab85d2SEdward Cree */ 1004c2ab85d2SEdward Cree void efx_link_clear_advertising(struct efx_nic *efx) 1005c2ab85d2SEdward Cree { 1006c2ab85d2SEdward Cree bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS); 1007c2ab85d2SEdward Cree efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 1008874aeea5SJeff Kirsher } 1009874aeea5SJeff Kirsher 1010874aeea5SJeff Kirsher void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 1011874aeea5SJeff Kirsher { 1012874aeea5SJeff Kirsher efx->wanted_fc = wanted_fc; 1013c2ab85d2SEdward Cree if (efx->link_advertising[0]) { 1014874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_RX) 1015c2ab85d2SEdward Cree efx->link_advertising[0] |= (ADVERTISED_Pause | 1016874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 1017874aeea5SJeff Kirsher else 1018c2ab85d2SEdward Cree efx->link_advertising[0] &= ~(ADVERTISED_Pause | 1019874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 1020874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_TX) 1021c2ab85d2SEdward Cree efx->link_advertising[0] ^= ADVERTISED_Asym_Pause; 1022874aeea5SJeff Kirsher } 1023874aeea5SJeff Kirsher } 1024874aeea5SJeff Kirsher 1025874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx); 1026874aeea5SJeff Kirsher 10270d322413SEdward Cree /* We assume that efx->type->reconfigure_mac will always try to sync RX 10280d322413SEdward Cree * filters and therefore needs to read-lock the filter table against freeing 10290d322413SEdward Cree */ 10300d322413SEdward Cree void efx_mac_reconfigure(struct efx_nic *efx) 10310d322413SEdward Cree { 10320d322413SEdward Cree down_read(&efx->filter_sem); 10330d322413SEdward Cree efx->type->reconfigure_mac(efx); 10340d322413SEdward Cree up_read(&efx->filter_sem); 10350d322413SEdward Cree } 10360d322413SEdward Cree 1037874aeea5SJeff Kirsher /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 1038874aeea5SJeff Kirsher * the MAC appropriately. All other PHY configuration changes are pushed 1039874aeea5SJeff Kirsher * through phy_op->set_settings(), and pushed asynchronously to the MAC 1040874aeea5SJeff Kirsher * through efx_monitor(). 1041874aeea5SJeff Kirsher * 1042874aeea5SJeff Kirsher * Callers must hold the mac_lock 1043874aeea5SJeff Kirsher */ 1044874aeea5SJeff Kirsher int __efx_reconfigure_port(struct efx_nic *efx) 1045874aeea5SJeff Kirsher { 1046874aeea5SJeff Kirsher enum efx_phy_mode phy_mode; 1047874aeea5SJeff Kirsher int rc; 1048874aeea5SJeff Kirsher 1049874aeea5SJeff Kirsher WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1050874aeea5SJeff Kirsher 1051874aeea5SJeff Kirsher /* Disable PHY transmit in mac level loopbacks */ 1052874aeea5SJeff Kirsher phy_mode = efx->phy_mode; 1053874aeea5SJeff Kirsher if (LOOPBACK_INTERNAL(efx)) 1054874aeea5SJeff Kirsher efx->phy_mode |= PHY_MODE_TX_DISABLED; 1055874aeea5SJeff Kirsher else 1056874aeea5SJeff Kirsher efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1057874aeea5SJeff Kirsher 1058874aeea5SJeff Kirsher rc = efx->type->reconfigure_port(efx); 1059874aeea5SJeff Kirsher 1060874aeea5SJeff Kirsher if (rc) 1061874aeea5SJeff Kirsher efx->phy_mode = phy_mode; 1062874aeea5SJeff Kirsher 1063874aeea5SJeff Kirsher return rc; 1064874aeea5SJeff Kirsher } 1065874aeea5SJeff Kirsher 1066874aeea5SJeff Kirsher /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1067874aeea5SJeff Kirsher * disabled. */ 1068874aeea5SJeff Kirsher int efx_reconfigure_port(struct efx_nic *efx) 1069874aeea5SJeff Kirsher { 1070874aeea5SJeff Kirsher int rc; 1071874aeea5SJeff Kirsher 1072874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1073874aeea5SJeff Kirsher 1074874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1075874aeea5SJeff Kirsher rc = __efx_reconfigure_port(efx); 1076874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1077874aeea5SJeff Kirsher 1078874aeea5SJeff Kirsher return rc; 1079874aeea5SJeff Kirsher } 1080874aeea5SJeff Kirsher 1081874aeea5SJeff Kirsher /* Asynchronous work item for changing MAC promiscuity and multicast 1082874aeea5SJeff Kirsher * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1083874aeea5SJeff Kirsher * MAC directly. */ 1084874aeea5SJeff Kirsher static void efx_mac_work(struct work_struct *data) 1085874aeea5SJeff Kirsher { 1086874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1087874aeea5SJeff Kirsher 1088874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 108930b81cdaSBen Hutchings if (efx->port_enabled) 10900d322413SEdward Cree efx_mac_reconfigure(efx); 1091874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1092874aeea5SJeff Kirsher } 1093874aeea5SJeff Kirsher 1094874aeea5SJeff Kirsher static int efx_probe_port(struct efx_nic *efx) 1095874aeea5SJeff Kirsher { 1096874aeea5SJeff Kirsher int rc; 1097874aeea5SJeff Kirsher 1098874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1099874aeea5SJeff Kirsher 1100874aeea5SJeff Kirsher if (phy_flash_cfg) 1101874aeea5SJeff Kirsher efx->phy_mode = PHY_MODE_SPECIAL; 1102874aeea5SJeff Kirsher 1103874aeea5SJeff Kirsher /* Connect up MAC/PHY operations table */ 1104874aeea5SJeff Kirsher rc = efx->type->probe_port(efx); 1105874aeea5SJeff Kirsher if (rc) 1106874aeea5SJeff Kirsher return rc; 1107874aeea5SJeff Kirsher 1108e332bcb3SBen Hutchings /* Initialise MAC address to permanent address */ 1109cd84ff4dSEdward Cree ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1110874aeea5SJeff Kirsher 1111874aeea5SJeff Kirsher return 0; 1112874aeea5SJeff Kirsher } 1113874aeea5SJeff Kirsher 1114874aeea5SJeff Kirsher static int efx_init_port(struct efx_nic *efx) 1115874aeea5SJeff Kirsher { 1116874aeea5SJeff Kirsher int rc; 1117874aeea5SJeff Kirsher 1118874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1119874aeea5SJeff Kirsher 1120874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1121874aeea5SJeff Kirsher 1122874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 1123874aeea5SJeff Kirsher if (rc) 1124874aeea5SJeff Kirsher goto fail1; 1125874aeea5SJeff Kirsher 1126874aeea5SJeff Kirsher efx->port_initialized = true; 1127874aeea5SJeff Kirsher 1128874aeea5SJeff Kirsher /* Reconfigure the MAC before creating dma queues (required for 1129874aeea5SJeff Kirsher * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 11300d322413SEdward Cree efx_mac_reconfigure(efx); 1131874aeea5SJeff Kirsher 1132874aeea5SJeff Kirsher /* Ensure the PHY advertises the correct flow control settings */ 1133874aeea5SJeff Kirsher rc = efx->phy_op->reconfigure(efx); 1134267d9d73SEdward Cree if (rc && rc != -EPERM) 1135874aeea5SJeff Kirsher goto fail2; 1136874aeea5SJeff Kirsher 1137874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1138874aeea5SJeff Kirsher return 0; 1139874aeea5SJeff Kirsher 1140874aeea5SJeff Kirsher fail2: 1141874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1142874aeea5SJeff Kirsher fail1: 1143874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1144874aeea5SJeff Kirsher return rc; 1145874aeea5SJeff Kirsher } 1146874aeea5SJeff Kirsher 1147874aeea5SJeff Kirsher static void efx_start_port(struct efx_nic *efx) 1148874aeea5SJeff Kirsher { 1149874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1150874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 1151874aeea5SJeff Kirsher 1152874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1153874aeea5SJeff Kirsher efx->port_enabled = true; 1154874aeea5SJeff Kirsher 1155d615c039SBen Hutchings /* Ensure MAC ingress/egress is enabled */ 11560d322413SEdward Cree efx_mac_reconfigure(efx); 1157874aeea5SJeff Kirsher 1158874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1159874aeea5SJeff Kirsher } 1160874aeea5SJeff Kirsher 1161d615c039SBen Hutchings /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1162d615c039SBen Hutchings * and the async self-test, wait for them to finish and prevent them 1163d615c039SBen Hutchings * being scheduled again. This doesn't cover online resets, which 1164d615c039SBen Hutchings * should only be cancelled when removing the device. 1165d615c039SBen Hutchings */ 1166874aeea5SJeff Kirsher static void efx_stop_port(struct efx_nic *efx) 1167874aeea5SJeff Kirsher { 1168874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1169874aeea5SJeff Kirsher 1170d615c039SBen Hutchings EFX_ASSERT_RESET_SERIALISED(efx); 1171d615c039SBen Hutchings 1172874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1173874aeea5SJeff Kirsher efx->port_enabled = false; 1174874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1175874aeea5SJeff Kirsher 1176874aeea5SJeff Kirsher /* Serialise against efx_set_multicast_list() */ 1177874aeea5SJeff Kirsher netif_addr_lock_bh(efx->net_dev); 1178874aeea5SJeff Kirsher netif_addr_unlock_bh(efx->net_dev); 1179d615c039SBen Hutchings 1180d615c039SBen Hutchings cancel_delayed_work_sync(&efx->monitor_work); 1181d615c039SBen Hutchings efx_selftest_async_cancel(efx); 1182d615c039SBen Hutchings cancel_work_sync(&efx->mac_work); 1183874aeea5SJeff Kirsher } 1184874aeea5SJeff Kirsher 1185874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx) 1186874aeea5SJeff Kirsher { 1187874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1188874aeea5SJeff Kirsher 1189874aeea5SJeff Kirsher if (!efx->port_initialized) 1190874aeea5SJeff Kirsher return; 1191874aeea5SJeff Kirsher 1192874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1193874aeea5SJeff Kirsher efx->port_initialized = false; 1194874aeea5SJeff Kirsher 1195874aeea5SJeff Kirsher efx->link_state.up = false; 1196874aeea5SJeff Kirsher efx_link_status_changed(efx); 1197874aeea5SJeff Kirsher } 1198874aeea5SJeff Kirsher 1199874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx) 1200874aeea5SJeff Kirsher { 1201874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1202874aeea5SJeff Kirsher 1203874aeea5SJeff Kirsher efx->type->remove_port(efx); 1204874aeea5SJeff Kirsher } 1205874aeea5SJeff Kirsher 1206874aeea5SJeff Kirsher /************************************************************************** 1207874aeea5SJeff Kirsher * 1208874aeea5SJeff Kirsher * NIC handling 1209874aeea5SJeff Kirsher * 1210874aeea5SJeff Kirsher **************************************************************************/ 1211874aeea5SJeff Kirsher 12120bcf4a64SBen Hutchings static LIST_HEAD(efx_primary_list); 12130bcf4a64SBen Hutchings static LIST_HEAD(efx_unassociated_list); 12140bcf4a64SBen Hutchings 12150bcf4a64SBen Hutchings static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 12160bcf4a64SBen Hutchings { 12170bcf4a64SBen Hutchings return left->type == right->type && 12180bcf4a64SBen Hutchings left->vpd_sn && right->vpd_sn && 12190bcf4a64SBen Hutchings !strcmp(left->vpd_sn, right->vpd_sn); 12200bcf4a64SBen Hutchings } 12210bcf4a64SBen Hutchings 12220bcf4a64SBen Hutchings static void efx_associate(struct efx_nic *efx) 12230bcf4a64SBen Hutchings { 12240bcf4a64SBen Hutchings struct efx_nic *other, *next; 12250bcf4a64SBen Hutchings 12260bcf4a64SBen Hutchings if (efx->primary == efx) { 12270bcf4a64SBen Hutchings /* Adding primary function; look for secondaries */ 12280bcf4a64SBen Hutchings 12290bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 12300bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_primary_list); 12310bcf4a64SBen Hutchings 12320bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx_unassociated_list, 12330bcf4a64SBen Hutchings node) { 12340bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 12350bcf4a64SBen Hutchings list_del(&other->node); 12360bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 12370bcf4a64SBen Hutchings "moving to secondary list of %s %s\n", 12380bcf4a64SBen Hutchings pci_name(efx->pci_dev), 12390bcf4a64SBen Hutchings efx->net_dev->name); 12400bcf4a64SBen Hutchings list_add_tail(&other->node, 12410bcf4a64SBen Hutchings &efx->secondary_list); 12420bcf4a64SBen Hutchings other->primary = efx; 12430bcf4a64SBen Hutchings } 12440bcf4a64SBen Hutchings } 12450bcf4a64SBen Hutchings } else { 12460bcf4a64SBen Hutchings /* Adding secondary function; look for primary */ 12470bcf4a64SBen Hutchings 12480bcf4a64SBen Hutchings list_for_each_entry(other, &efx_primary_list, node) { 12490bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 12500bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 12510bcf4a64SBen Hutchings "adding to secondary list of %s %s\n", 12520bcf4a64SBen Hutchings pci_name(other->pci_dev), 12530bcf4a64SBen Hutchings other->net_dev->name); 12540bcf4a64SBen Hutchings list_add_tail(&efx->node, 12550bcf4a64SBen Hutchings &other->secondary_list); 12560bcf4a64SBen Hutchings efx->primary = other; 12570bcf4a64SBen Hutchings return; 12580bcf4a64SBen Hutchings } 12590bcf4a64SBen Hutchings } 12600bcf4a64SBen Hutchings 12610bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 12620bcf4a64SBen Hutchings "adding to unassociated list\n"); 12630bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_unassociated_list); 12640bcf4a64SBen Hutchings } 12650bcf4a64SBen Hutchings } 12660bcf4a64SBen Hutchings 12670bcf4a64SBen Hutchings static void efx_dissociate(struct efx_nic *efx) 12680bcf4a64SBen Hutchings { 12690bcf4a64SBen Hutchings struct efx_nic *other, *next; 12700bcf4a64SBen Hutchings 12710bcf4a64SBen Hutchings list_del(&efx->node); 12720bcf4a64SBen Hutchings efx->primary = NULL; 12730bcf4a64SBen Hutchings 12740bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 12750bcf4a64SBen Hutchings list_del(&other->node); 12760bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 12770bcf4a64SBen Hutchings "moving to unassociated list\n"); 12780bcf4a64SBen Hutchings list_add_tail(&other->node, &efx_unassociated_list); 12790bcf4a64SBen Hutchings other->primary = NULL; 12800bcf4a64SBen Hutchings } 12810bcf4a64SBen Hutchings } 12820bcf4a64SBen Hutchings 1283874aeea5SJeff Kirsher /* This configures the PCI device to enable I/O and DMA. */ 1284874aeea5SJeff Kirsher static int efx_init_io(struct efx_nic *efx) 1285874aeea5SJeff Kirsher { 1286874aeea5SJeff Kirsher struct pci_dev *pci_dev = efx->pci_dev; 1287874aeea5SJeff Kirsher dma_addr_t dma_mask = efx->type->max_dma_mask; 1288b105798fSBen Hutchings unsigned int mem_map_size = efx->type->mem_map_size(efx); 128902246a7fSShradha Shah int rc, bar; 1290874aeea5SJeff Kirsher 1291874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1292874aeea5SJeff Kirsher 129303714bbbSEdward Cree bar = efx->type->mem_bar(efx); 129402246a7fSShradha Shah 1295874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 1296874aeea5SJeff Kirsher if (rc) { 1297874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1298874aeea5SJeff Kirsher "failed to enable PCI device\n"); 1299874aeea5SJeff Kirsher goto fail1; 1300874aeea5SJeff Kirsher } 1301874aeea5SJeff Kirsher 1302874aeea5SJeff Kirsher pci_set_master(pci_dev); 1303874aeea5SJeff Kirsher 130406e9552fSChristoph Hellwig /* Set the PCI DMA mask. Try all possibilities from our genuine mask 130506e9552fSChristoph Hellwig * down to 32 bits, because some architectures will allow 40 bit 1306874aeea5SJeff Kirsher * masks event though they reject 46 bit masks. 1307874aeea5SJeff Kirsher */ 1308874aeea5SJeff Kirsher while (dma_mask > 0x7fffffffUL) { 13099663dedaSRussell King rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1310e9e01846SBen Hutchings if (rc == 0) 1311874aeea5SJeff Kirsher break; 1312874aeea5SJeff Kirsher dma_mask >>= 1; 1313874aeea5SJeff Kirsher } 1314874aeea5SJeff Kirsher if (rc) { 1315874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1316874aeea5SJeff Kirsher "could not find a suitable DMA mask\n"); 1317874aeea5SJeff Kirsher goto fail2; 1318874aeea5SJeff Kirsher } 1319874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1320874aeea5SJeff Kirsher "using DMA mask %llx\n", (unsigned long long) dma_mask); 1321874aeea5SJeff Kirsher 132202246a7fSShradha Shah efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 132302246a7fSShradha Shah rc = pci_request_region(pci_dev, bar, "sfc"); 1324874aeea5SJeff Kirsher if (rc) { 1325874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1326874aeea5SJeff Kirsher "request for memory BAR failed\n"); 1327874aeea5SJeff Kirsher rc = -EIO; 1328874aeea5SJeff Kirsher goto fail3; 1329874aeea5SJeff Kirsher } 1330b105798fSBen Hutchings efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1331874aeea5SJeff Kirsher if (!efx->membase) { 1332874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1333874aeea5SJeff Kirsher "could not map memory BAR at %llx+%x\n", 1334b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size); 1335874aeea5SJeff Kirsher rc = -ENOMEM; 1336874aeea5SJeff Kirsher goto fail4; 1337874aeea5SJeff Kirsher } 1338874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1339874aeea5SJeff Kirsher "memory BAR at %llx+%x (virtual %p)\n", 1340b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size, 1341b105798fSBen Hutchings efx->membase); 1342874aeea5SJeff Kirsher 1343874aeea5SJeff Kirsher return 0; 1344874aeea5SJeff Kirsher 1345874aeea5SJeff Kirsher fail4: 134602246a7fSShradha Shah pci_release_region(efx->pci_dev, bar); 1347874aeea5SJeff Kirsher fail3: 1348874aeea5SJeff Kirsher efx->membase_phys = 0; 1349874aeea5SJeff Kirsher fail2: 1350874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1351874aeea5SJeff Kirsher fail1: 1352874aeea5SJeff Kirsher return rc; 1353874aeea5SJeff Kirsher } 1354874aeea5SJeff Kirsher 1355874aeea5SJeff Kirsher static void efx_fini_io(struct efx_nic *efx) 1356874aeea5SJeff Kirsher { 135702246a7fSShradha Shah int bar; 135802246a7fSShradha Shah 1359874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1360874aeea5SJeff Kirsher 1361874aeea5SJeff Kirsher if (efx->membase) { 1362874aeea5SJeff Kirsher iounmap(efx->membase); 1363874aeea5SJeff Kirsher efx->membase = NULL; 1364874aeea5SJeff Kirsher } 1365874aeea5SJeff Kirsher 1366874aeea5SJeff Kirsher if (efx->membase_phys) { 136703714bbbSEdward Cree bar = efx->type->mem_bar(efx); 136802246a7fSShradha Shah pci_release_region(efx->pci_dev, bar); 1369874aeea5SJeff Kirsher efx->membase_phys = 0; 1370874aeea5SJeff Kirsher } 1371874aeea5SJeff Kirsher 13726598dad2SDaniel Pieczko /* Don't disable bus-mastering if VFs are assigned */ 13736598dad2SDaniel Pieczko if (!pci_vfs_assigned(efx->pci_dev)) 1374874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1375874aeea5SJeff Kirsher } 1376874aeea5SJeff Kirsher 137742356d9aSEdward Cree void efx_set_default_rx_indir_table(struct efx_nic *efx, 137842356d9aSEdward Cree struct efx_rss_context *ctx) 1379267c0157SJon Cooper { 1380267c0157SJon Cooper size_t i; 1381267c0157SJon Cooper 138242356d9aSEdward Cree for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 138342356d9aSEdward Cree ctx->rx_indir_table[i] = 1384267c0157SJon Cooper ethtool_rxfh_indir_default(i, efx->rss_spread); 1385267c0157SJon Cooper } 1386267c0157SJon Cooper 1387a9a52506SBen Hutchings static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1388874aeea5SJeff Kirsher { 1389cdb08f8fSBen Hutchings cpumask_var_t thread_mask; 1390a16e5b24SBen Hutchings unsigned int count; 1391874aeea5SJeff Kirsher int cpu; 1392874aeea5SJeff Kirsher 1393cd2d5b52SBen Hutchings if (rss_cpus) { 1394cd2d5b52SBen Hutchings count = rss_cpus; 1395cd2d5b52SBen Hutchings } else { 1396cdb08f8fSBen Hutchings if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1397a9a52506SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1398a9a52506SBen Hutchings "RSS disabled due to allocation failure\n"); 1399874aeea5SJeff Kirsher return 1; 1400874aeea5SJeff Kirsher } 1401874aeea5SJeff Kirsher 1402874aeea5SJeff Kirsher count = 0; 1403874aeea5SJeff Kirsher for_each_online_cpu(cpu) { 1404cdb08f8fSBen Hutchings if (!cpumask_test_cpu(cpu, thread_mask)) { 1405874aeea5SJeff Kirsher ++count; 1406cdb08f8fSBen Hutchings cpumask_or(thread_mask, thread_mask, 140706931e62SBartosz Golaszewski topology_sibling_cpumask(cpu)); 1408874aeea5SJeff Kirsher } 1409874aeea5SJeff Kirsher } 1410874aeea5SJeff Kirsher 1411cdb08f8fSBen Hutchings free_cpumask_var(thread_mask); 1412cd2d5b52SBen Hutchings } 1413cd2d5b52SBen Hutchings 1414271a8b42SBert Kenward if (count > EFX_MAX_RX_QUEUES) { 1415271a8b42SBert Kenward netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1416271a8b42SBert Kenward "Reducing number of rx queues from %u to %u.\n", 1417271a8b42SBert Kenward count, EFX_MAX_RX_QUEUES); 1418271a8b42SBert Kenward count = EFX_MAX_RX_QUEUES; 1419271a8b42SBert Kenward } 1420271a8b42SBert Kenward 1421cd2d5b52SBen Hutchings /* If RSS is requested for the PF *and* VFs then we can't write RSS 1422cd2d5b52SBen Hutchings * table entries that are inaccessible to VFs 1423cd2d5b52SBen Hutchings */ 14247fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 14257fa8d547SShradha Shah if (efx->type->sriov_wanted) { 1426d98a4ffeSShradha Shah if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1427cd2d5b52SBen Hutchings count > efx_vf_size(efx)) { 1428cd2d5b52SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1429cd2d5b52SBen Hutchings "Reducing number of RSS channels from %u to %u for " 1430cd2d5b52SBen Hutchings "VF support. Increase vf-msix-limit to use more " 1431cd2d5b52SBen Hutchings "channels on the PF.\n", 1432cd2d5b52SBen Hutchings count, efx_vf_size(efx)); 1433cd2d5b52SBen Hutchings count = efx_vf_size(efx); 1434cd2d5b52SBen Hutchings } 14357fa8d547SShradha Shah } 14367fa8d547SShradha Shah #endif 1437cd2d5b52SBen Hutchings 1438874aeea5SJeff Kirsher return count; 1439874aeea5SJeff Kirsher } 1440874aeea5SJeff Kirsher 1441874aeea5SJeff Kirsher /* Probe the number and type of interrupts we are able to obtain, and 1442874aeea5SJeff Kirsher * the resulting numbers of channels and RX queues. 1443874aeea5SJeff Kirsher */ 1444874aeea5SJeff Kirsher static int efx_probe_interrupts(struct efx_nic *efx) 1445874aeea5SJeff Kirsher { 14467f967c01SBen Hutchings unsigned int extra_channels = 0; 14477f967c01SBen Hutchings unsigned int i, j; 1448a16e5b24SBen Hutchings int rc; 1449874aeea5SJeff Kirsher 14507f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 14517f967c01SBen Hutchings if (efx->extra_channel_type[i]) 14527f967c01SBen Hutchings ++extra_channels; 14537f967c01SBen Hutchings 1454874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1455874aeea5SJeff Kirsher struct msix_entry xentries[EFX_MAX_CHANNELS]; 1456a16e5b24SBen Hutchings unsigned int n_channels; 1457874aeea5SJeff Kirsher 1458a9a52506SBen Hutchings n_channels = efx_wanted_parallelism(efx); 1459b0fbdae1SShradha Shah if (efx_separate_tx_channels) 1460874aeea5SJeff Kirsher n_channels *= 2; 14617f967c01SBen Hutchings n_channels += extra_channels; 1462b105798fSBen Hutchings n_channels = min(n_channels, efx->max_channels); 1463874aeea5SJeff Kirsher 1464874aeea5SJeff Kirsher for (i = 0; i < n_channels; i++) 1465874aeea5SJeff Kirsher xentries[i].entry = i; 1466184603d8SAlexander Gordeev rc = pci_enable_msix_range(efx->pci_dev, 1467184603d8SAlexander Gordeev xentries, 1, n_channels); 1468184603d8SAlexander Gordeev if (rc < 0) { 1469184603d8SAlexander Gordeev /* Fall back to single channel MSI */ 1470184603d8SAlexander Gordeev netif_err(efx, drv, efx->net_dev, 1471184603d8SAlexander Gordeev "could not enable MSI-X\n"); 147262980cb6SAndrew Rybchenko if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 147362980cb6SAndrew Rybchenko efx->interrupt_mode = EFX_INT_MODE_MSI; 147462980cb6SAndrew Rybchenko else 147562980cb6SAndrew Rybchenko return rc; 1476184603d8SAlexander Gordeev } else if (rc < n_channels) { 1477874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1478874aeea5SJeff Kirsher "WARNING: Insufficient MSI-X vectors" 1479a16e5b24SBen Hutchings " available (%d < %u).\n", rc, n_channels); 1480874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1481874aeea5SJeff Kirsher "WARNING: Performance may be reduced.\n"); 1482874aeea5SJeff Kirsher n_channels = rc; 1483874aeea5SJeff Kirsher } 1484874aeea5SJeff Kirsher 1485184603d8SAlexander Gordeev if (rc > 0) { 1486874aeea5SJeff Kirsher efx->n_channels = n_channels; 14877f967c01SBen Hutchings if (n_channels > extra_channels) 14887f967c01SBen Hutchings n_channels -= extra_channels; 1489b0fbdae1SShradha Shah if (efx_separate_tx_channels) { 1490b0fbdae1SShradha Shah efx->n_tx_channels = min(max(n_channels / 2, 1491b0fbdae1SShradha Shah 1U), 1492b0fbdae1SShradha Shah efx->max_tx_channels); 14937f967c01SBen Hutchings efx->n_rx_channels = max(n_channels - 14947f967c01SBen Hutchings efx->n_tx_channels, 14957f967c01SBen Hutchings 1U); 1496874aeea5SJeff Kirsher } else { 1497b0fbdae1SShradha Shah efx->n_tx_channels = min(n_channels, 1498b0fbdae1SShradha Shah efx->max_tx_channels); 14997f967c01SBen Hutchings efx->n_rx_channels = n_channels; 1500874aeea5SJeff Kirsher } 15017f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) 1502874aeea5SJeff Kirsher efx_get_channel(efx, i)->irq = 1503874aeea5SJeff Kirsher xentries[i].vector; 1504874aeea5SJeff Kirsher } 1505874aeea5SJeff Kirsher } 1506874aeea5SJeff Kirsher 1507874aeea5SJeff Kirsher /* Try single interrupt MSI */ 1508874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1509874aeea5SJeff Kirsher efx->n_channels = 1; 1510874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1511874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1512874aeea5SJeff Kirsher rc = pci_enable_msi(efx->pci_dev); 1513874aeea5SJeff Kirsher if (rc == 0) { 1514874aeea5SJeff Kirsher efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1515874aeea5SJeff Kirsher } else { 1516874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1517874aeea5SJeff Kirsher "could not enable MSI\n"); 151862980cb6SAndrew Rybchenko if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 1519874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_LEGACY; 152062980cb6SAndrew Rybchenko else 152162980cb6SAndrew Rybchenko return rc; 1522874aeea5SJeff Kirsher } 1523874aeea5SJeff Kirsher } 1524874aeea5SJeff Kirsher 1525874aeea5SJeff Kirsher /* Assume legacy interrupts */ 1526874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1527b0fbdae1SShradha Shah efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1528874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1529874aeea5SJeff Kirsher efx->n_tx_channels = 1; 1530874aeea5SJeff Kirsher efx->legacy_irq = efx->pci_dev->irq; 1531874aeea5SJeff Kirsher } 1532874aeea5SJeff Kirsher 15337f967c01SBen Hutchings /* Assign extra channels if possible */ 15342935e3c3SEdward Cree efx->n_extra_tx_channels = 0; 15357f967c01SBen Hutchings j = efx->n_channels; 15367f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 15377f967c01SBen Hutchings if (!efx->extra_channel_type[i]) 15387f967c01SBen Hutchings continue; 15397f967c01SBen Hutchings if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 15407f967c01SBen Hutchings efx->n_channels <= extra_channels) { 15417f967c01SBen Hutchings efx->extra_channel_type[i]->handle_no_channel(efx); 15427f967c01SBen Hutchings } else { 15437f967c01SBen Hutchings --j; 15447f967c01SBen Hutchings efx_get_channel(efx, j)->type = 15457f967c01SBen Hutchings efx->extra_channel_type[i]; 15462935e3c3SEdward Cree if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 15472935e3c3SEdward Cree efx->n_extra_tx_channels++; 15487f967c01SBen Hutchings } 15497f967c01SBen Hutchings } 15507f967c01SBen Hutchings 1551cd2d5b52SBen Hutchings /* RSS might be usable on VFs even if it is disabled on the PF */ 15527fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 15537fa8d547SShradha Shah if (efx->type->sriov_wanted) { 1554327c685eSShradha Shah efx->rss_spread = ((efx->n_rx_channels > 1 || 1555d98a4ffeSShradha Shah !efx->type->sriov_wanted(efx)) ? 1556cd2d5b52SBen Hutchings efx->n_rx_channels : efx_vf_size(efx)); 15577fa8d547SShradha Shah return 0; 15587fa8d547SShradha Shah } 15597fa8d547SShradha Shah #endif 15607fa8d547SShradha Shah efx->rss_spread = efx->n_rx_channels; 1561cd2d5b52SBen Hutchings 1562874aeea5SJeff Kirsher return 0; 1563874aeea5SJeff Kirsher } 1564874aeea5SJeff Kirsher 1565a83762d9SBert Kenward #if defined(CONFIG_SMP) 1566a83762d9SBert Kenward static void efx_set_interrupt_affinity(struct efx_nic *efx) 1567a83762d9SBert Kenward { 1568a83762d9SBert Kenward struct efx_channel *channel; 1569a83762d9SBert Kenward unsigned int cpu; 1570a83762d9SBert Kenward 1571a83762d9SBert Kenward efx_for_each_channel(channel, efx) { 1572a83762d9SBert Kenward cpu = cpumask_local_spread(channel->channel, 1573a83762d9SBert Kenward pcibus_to_node(efx->pci_dev->bus)); 1574a83762d9SBert Kenward irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 1575a83762d9SBert Kenward } 1576a83762d9SBert Kenward } 1577a83762d9SBert Kenward 1578a83762d9SBert Kenward static void efx_clear_interrupt_affinity(struct efx_nic *efx) 1579a83762d9SBert Kenward { 1580a83762d9SBert Kenward struct efx_channel *channel; 1581a83762d9SBert Kenward 1582a83762d9SBert Kenward efx_for_each_channel(channel, efx) 1583a83762d9SBert Kenward irq_set_affinity_hint(channel->irq, NULL); 1584a83762d9SBert Kenward } 1585a83762d9SBert Kenward #else 1586a83762d9SBert Kenward static void 1587a83762d9SBert Kenward efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1588a83762d9SBert Kenward { 1589a83762d9SBert Kenward } 1590a83762d9SBert Kenward 1591a83762d9SBert Kenward static void 1592a83762d9SBert Kenward efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1593a83762d9SBert Kenward { 1594a83762d9SBert Kenward } 1595a83762d9SBert Kenward #endif /* CONFIG_SMP */ 1596a83762d9SBert Kenward 1597261e4d96SJon Cooper static int efx_soft_enable_interrupts(struct efx_nic *efx) 1598d8291187SBen Hutchings { 1599261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1600261e4d96SJon Cooper int rc; 1601d8291187SBen Hutchings 1602d8291187SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1603d8291187SBen Hutchings 1604d8291187SBen Hutchings efx->irq_soft_enabled = true; 1605d8291187SBen Hutchings smp_wmb(); 1606d8291187SBen Hutchings 1607d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1608261e4d96SJon Cooper if (!channel->type->keep_eventq) { 1609261e4d96SJon Cooper rc = efx_init_eventq(channel); 1610261e4d96SJon Cooper if (rc) 1611261e4d96SJon Cooper goto fail; 1612261e4d96SJon Cooper } 1613d8291187SBen Hutchings efx_start_eventq(channel); 1614d8291187SBen Hutchings } 1615d8291187SBen Hutchings 1616d8291187SBen Hutchings efx_mcdi_mode_event(efx); 1617261e4d96SJon Cooper 1618261e4d96SJon Cooper return 0; 1619261e4d96SJon Cooper fail: 1620261e4d96SJon Cooper end_channel = channel; 1621261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1622261e4d96SJon Cooper if (channel == end_channel) 1623261e4d96SJon Cooper break; 1624261e4d96SJon Cooper efx_stop_eventq(channel); 1625261e4d96SJon Cooper if (!channel->type->keep_eventq) 1626261e4d96SJon Cooper efx_fini_eventq(channel); 1627261e4d96SJon Cooper } 1628261e4d96SJon Cooper 1629261e4d96SJon Cooper return rc; 1630d8291187SBen Hutchings } 1631d8291187SBen Hutchings 1632d8291187SBen Hutchings static void efx_soft_disable_interrupts(struct efx_nic *efx) 1633d8291187SBen Hutchings { 1634d8291187SBen Hutchings struct efx_channel *channel; 1635d8291187SBen Hutchings 1636d8291187SBen Hutchings if (efx->state == STATE_DISABLED) 1637d8291187SBen Hutchings return; 1638d8291187SBen Hutchings 1639d8291187SBen Hutchings efx_mcdi_mode_poll(efx); 1640d8291187SBen Hutchings 1641d8291187SBen Hutchings efx->irq_soft_enabled = false; 1642d8291187SBen Hutchings smp_wmb(); 1643d8291187SBen Hutchings 1644d8291187SBen Hutchings if (efx->legacy_irq) 1645d8291187SBen Hutchings synchronize_irq(efx->legacy_irq); 1646d8291187SBen Hutchings 1647d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1648d8291187SBen Hutchings if (channel->irq) 1649d8291187SBen Hutchings synchronize_irq(channel->irq); 1650d8291187SBen Hutchings 1651d8291187SBen Hutchings efx_stop_eventq(channel); 1652d8291187SBen Hutchings if (!channel->type->keep_eventq) 1653d8291187SBen Hutchings efx_fini_eventq(channel); 1654d8291187SBen Hutchings } 1655cade715fSBen Hutchings 1656cade715fSBen Hutchings /* Flush the asynchronous MCDI request queue */ 1657cade715fSBen Hutchings efx_mcdi_flush_async(efx); 1658d8291187SBen Hutchings } 1659d8291187SBen Hutchings 1660261e4d96SJon Cooper static int efx_enable_interrupts(struct efx_nic *efx) 16619f2cb71cSBen Hutchings { 1662261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1663261e4d96SJon Cooper int rc; 16649f2cb71cSBen Hutchings 16658b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 16668b7325b4SBen Hutchings 1667b28405b0SAlexandre Rames if (efx->eeh_disabled_legacy_irq) { 1668b28405b0SAlexandre Rames enable_irq(efx->legacy_irq); 1669b28405b0SAlexandre Rames efx->eeh_disabled_legacy_irq = false; 1670b28405b0SAlexandre Rames } 1671d8291187SBen Hutchings 167286094f7fSBen Hutchings efx->type->irq_enable_master(efx); 16739f2cb71cSBen Hutchings 16749f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1675261e4d96SJon Cooper if (channel->type->keep_eventq) { 1676261e4d96SJon Cooper rc = efx_init_eventq(channel); 1677261e4d96SJon Cooper if (rc) 1678261e4d96SJon Cooper goto fail; 1679261e4d96SJon Cooper } 16809f2cb71cSBen Hutchings } 16819f2cb71cSBen Hutchings 1682261e4d96SJon Cooper rc = efx_soft_enable_interrupts(efx); 1683261e4d96SJon Cooper if (rc) 1684261e4d96SJon Cooper goto fail; 1685261e4d96SJon Cooper 1686261e4d96SJon Cooper return 0; 1687261e4d96SJon Cooper 1688261e4d96SJon Cooper fail: 1689261e4d96SJon Cooper end_channel = channel; 1690261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1691261e4d96SJon Cooper if (channel == end_channel) 1692261e4d96SJon Cooper break; 1693261e4d96SJon Cooper if (channel->type->keep_eventq) 1694261e4d96SJon Cooper efx_fini_eventq(channel); 1695261e4d96SJon Cooper } 1696261e4d96SJon Cooper 1697261e4d96SJon Cooper efx->type->irq_disable_non_ev(efx); 1698261e4d96SJon Cooper 1699261e4d96SJon Cooper return rc; 17009f2cb71cSBen Hutchings } 17019f2cb71cSBen Hutchings 1702d8291187SBen Hutchings static void efx_disable_interrupts(struct efx_nic *efx) 17039f2cb71cSBen Hutchings { 17049f2cb71cSBen Hutchings struct efx_channel *channel; 17059f2cb71cSBen Hutchings 1706d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 17079f2cb71cSBen Hutchings 17089f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1709d8291187SBen Hutchings if (channel->type->keep_eventq) 17109f2cb71cSBen Hutchings efx_fini_eventq(channel); 17119f2cb71cSBen Hutchings } 1712d8291187SBen Hutchings 171386094f7fSBen Hutchings efx->type->irq_disable_non_ev(efx); 17149f2cb71cSBen Hutchings } 17159f2cb71cSBen Hutchings 1716874aeea5SJeff Kirsher static void efx_remove_interrupts(struct efx_nic *efx) 1717874aeea5SJeff Kirsher { 1718874aeea5SJeff Kirsher struct efx_channel *channel; 1719874aeea5SJeff Kirsher 1720874aeea5SJeff Kirsher /* Remove MSI/MSI-X interrupts */ 1721874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1722874aeea5SJeff Kirsher channel->irq = 0; 1723874aeea5SJeff Kirsher pci_disable_msi(efx->pci_dev); 1724874aeea5SJeff Kirsher pci_disable_msix(efx->pci_dev); 1725874aeea5SJeff Kirsher 1726874aeea5SJeff Kirsher /* Remove legacy interrupt */ 1727874aeea5SJeff Kirsher efx->legacy_irq = 0; 1728874aeea5SJeff Kirsher } 1729874aeea5SJeff Kirsher 1730874aeea5SJeff Kirsher static void efx_set_channels(struct efx_nic *efx) 1731874aeea5SJeff Kirsher { 1732874aeea5SJeff Kirsher struct efx_channel *channel; 1733874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 1734874aeea5SJeff Kirsher 1735874aeea5SJeff Kirsher efx->tx_channel_offset = 1736b0fbdae1SShradha Shah efx_separate_tx_channels ? 1737b0fbdae1SShradha Shah efx->n_channels - efx->n_tx_channels : 0; 1738874aeea5SJeff Kirsher 173979d68b37SStuart Hodgson /* We need to mark which channels really have RX and TX 174079d68b37SStuart Hodgson * queues, and adjust the TX queue numbers if we have separate 1741874aeea5SJeff Kirsher * RX-only and TX-only channels. 1742874aeea5SJeff Kirsher */ 1743874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 174479d68b37SStuart Hodgson if (channel->channel < efx->n_rx_channels) 174579d68b37SStuart Hodgson channel->rx_queue.core_index = channel->channel; 174679d68b37SStuart Hodgson else 174779d68b37SStuart Hodgson channel->rx_queue.core_index = -1; 174879d68b37SStuart Hodgson 1749874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 1750874aeea5SJeff Kirsher tx_queue->queue -= (efx->tx_channel_offset * 1751874aeea5SJeff Kirsher EFX_TXQ_TYPES); 1752874aeea5SJeff Kirsher } 1753874aeea5SJeff Kirsher } 1754874aeea5SJeff Kirsher 1755874aeea5SJeff Kirsher static int efx_probe_nic(struct efx_nic *efx) 1756874aeea5SJeff Kirsher { 1757874aeea5SJeff Kirsher int rc; 1758874aeea5SJeff Kirsher 1759874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1760874aeea5SJeff Kirsher 1761874aeea5SJeff Kirsher /* Carry out hardware-type specific initialisation */ 1762874aeea5SJeff Kirsher rc = efx->type->probe(efx); 1763874aeea5SJeff Kirsher if (rc) 1764874aeea5SJeff Kirsher return rc; 1765874aeea5SJeff Kirsher 1766b0fbdae1SShradha Shah do { 1767b0fbdae1SShradha Shah if (!efx->max_channels || !efx->max_tx_channels) { 1768b0fbdae1SShradha Shah netif_err(efx, drv, efx->net_dev, 1769b0fbdae1SShradha Shah "Insufficient resources to allocate" 1770b0fbdae1SShradha Shah " any channels\n"); 1771b0fbdae1SShradha Shah rc = -ENOSPC; 1772b0fbdae1SShradha Shah goto fail1; 1773b0fbdae1SShradha Shah } 1774b0fbdae1SShradha Shah 1775b0fbdae1SShradha Shah /* Determine the number of channels and queues by trying 1776b0fbdae1SShradha Shah * to hook in MSI-X interrupts. 1777b0fbdae1SShradha Shah */ 1778874aeea5SJeff Kirsher rc = efx_probe_interrupts(efx); 1779874aeea5SJeff Kirsher if (rc) 1780c15eed22SBen Hutchings goto fail1; 1781874aeea5SJeff Kirsher 178252ad762bSDaniel Pieczko efx_set_channels(efx); 178352ad762bSDaniel Pieczko 1784b0fbdae1SShradha Shah /* dimension_resources can fail with EAGAIN */ 1785c15eed22SBen Hutchings rc = efx->type->dimension_resources(efx); 1786b0fbdae1SShradha Shah if (rc != 0 && rc != -EAGAIN) 1787c15eed22SBen Hutchings goto fail2; 178828e47c49SBen Hutchings 1789b0fbdae1SShradha Shah if (rc == -EAGAIN) 1790b0fbdae1SShradha Shah /* try again with new max_channels */ 1791b0fbdae1SShradha Shah efx_remove_interrupts(efx); 1792b0fbdae1SShradha Shah 1793b0fbdae1SShradha Shah } while (rc == -EAGAIN); 1794b0fbdae1SShradha Shah 1795874aeea5SJeff Kirsher if (efx->n_channels > 1) 179642356d9aSEdward Cree netdev_rss_key_fill(efx->rss_context.rx_hash_key, 179742356d9aSEdward Cree sizeof(efx->rss_context.rx_hash_key)); 179842356d9aSEdward Cree efx_set_default_rx_indir_table(efx, &efx->rss_context); 1799874aeea5SJeff Kirsher 1800874aeea5SJeff Kirsher netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1801874aeea5SJeff Kirsher netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1802874aeea5SJeff Kirsher 1803874aeea5SJeff Kirsher /* Initialise the interrupt moderation settings */ 1804539de7c5SBert Kenward efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 18059e393b30SBen Hutchings efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 18069e393b30SBen Hutchings true); 1807874aeea5SJeff Kirsher 1808874aeea5SJeff Kirsher return 0; 1809874aeea5SJeff Kirsher 1810c15eed22SBen Hutchings fail2: 1811c15eed22SBen Hutchings efx_remove_interrupts(efx); 1812c15eed22SBen Hutchings fail1: 1813874aeea5SJeff Kirsher efx->type->remove(efx); 1814874aeea5SJeff Kirsher return rc; 1815874aeea5SJeff Kirsher } 1816874aeea5SJeff Kirsher 1817874aeea5SJeff Kirsher static void efx_remove_nic(struct efx_nic *efx) 1818874aeea5SJeff Kirsher { 1819874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1820874aeea5SJeff Kirsher 1821874aeea5SJeff Kirsher efx_remove_interrupts(efx); 1822874aeea5SJeff Kirsher efx->type->remove(efx); 1823874aeea5SJeff Kirsher } 1824874aeea5SJeff Kirsher 1825add72477SBen Hutchings static int efx_probe_filters(struct efx_nic *efx) 1826add72477SBen Hutchings { 1827add72477SBen Hutchings int rc; 1828add72477SBen Hutchings 18290d322413SEdward Cree init_rwsem(&efx->filter_sem); 1830d248953aSMartin Habets mutex_lock(&efx->mac_lock); 18310d322413SEdward Cree down_write(&efx->filter_sem); 1832add72477SBen Hutchings rc = efx->type->filter_table_probe(efx); 1833add72477SBen Hutchings if (rc) 18340d322413SEdward Cree goto out_unlock; 1835add72477SBen Hutchings 1836add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1837add72477SBen Hutchings if (efx->type->offload_features & NETIF_F_NTUPLE) { 1838faf8dcc1SJon Cooper struct efx_channel *channel; 1839faf8dcc1SJon Cooper int i, success = 1; 1840faf8dcc1SJon Cooper 1841faf8dcc1SJon Cooper efx_for_each_channel(channel, efx) { 1842faf8dcc1SJon Cooper channel->rps_flow_id = 1843faf8dcc1SJon Cooper kcalloc(efx->type->max_rx_ip_filters, 1844faf8dcc1SJon Cooper sizeof(*channel->rps_flow_id), 1845add72477SBen Hutchings GFP_KERNEL); 1846faf8dcc1SJon Cooper if (!channel->rps_flow_id) 1847faf8dcc1SJon Cooper success = 0; 1848faf8dcc1SJon Cooper else 1849faf8dcc1SJon Cooper for (i = 0; 1850faf8dcc1SJon Cooper i < efx->type->max_rx_ip_filters; 1851faf8dcc1SJon Cooper ++i) 1852faf8dcc1SJon Cooper channel->rps_flow_id[i] = 1853faf8dcc1SJon Cooper RPS_FLOW_ID_INVALID; 1854faf8dcc1SJon Cooper } 1855faf8dcc1SJon Cooper 1856faf8dcc1SJon Cooper if (!success) { 1857faf8dcc1SJon Cooper efx_for_each_channel(channel, efx) 1858faf8dcc1SJon Cooper kfree(channel->rps_flow_id); 1859add72477SBen Hutchings efx->type->filter_table_remove(efx); 18600d322413SEdward Cree rc = -ENOMEM; 18610d322413SEdward Cree goto out_unlock; 1862add72477SBen Hutchings } 1863faf8dcc1SJon Cooper 1864faf8dcc1SJon Cooper efx->rps_expire_index = efx->rps_expire_channel = 0; 1865add72477SBen Hutchings } 1866add72477SBen Hutchings #endif 18670d322413SEdward Cree out_unlock: 18680d322413SEdward Cree up_write(&efx->filter_sem); 1869d248953aSMartin Habets mutex_unlock(&efx->mac_lock); 18700d322413SEdward Cree return rc; 1871add72477SBen Hutchings } 1872add72477SBen Hutchings 1873add72477SBen Hutchings static void efx_remove_filters(struct efx_nic *efx) 1874add72477SBen Hutchings { 1875add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1876faf8dcc1SJon Cooper struct efx_channel *channel; 1877faf8dcc1SJon Cooper 1878faf8dcc1SJon Cooper efx_for_each_channel(channel, efx) 1879faf8dcc1SJon Cooper kfree(channel->rps_flow_id); 1880add72477SBen Hutchings #endif 18810d322413SEdward Cree down_write(&efx->filter_sem); 1882add72477SBen Hutchings efx->type->filter_table_remove(efx); 18830d322413SEdward Cree up_write(&efx->filter_sem); 1884add72477SBen Hutchings } 1885add72477SBen Hutchings 1886add72477SBen Hutchings 1887874aeea5SJeff Kirsher /************************************************************************** 1888874aeea5SJeff Kirsher * 1889874aeea5SJeff Kirsher * NIC startup/shutdown 1890874aeea5SJeff Kirsher * 1891874aeea5SJeff Kirsher *************************************************************************/ 1892874aeea5SJeff Kirsher 1893874aeea5SJeff Kirsher static int efx_probe_all(struct efx_nic *efx) 1894874aeea5SJeff Kirsher { 1895874aeea5SJeff Kirsher int rc; 1896874aeea5SJeff Kirsher 1897874aeea5SJeff Kirsher rc = efx_probe_nic(efx); 1898874aeea5SJeff Kirsher if (rc) { 1899874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1900874aeea5SJeff Kirsher goto fail1; 1901874aeea5SJeff Kirsher } 1902874aeea5SJeff Kirsher 1903874aeea5SJeff Kirsher rc = efx_probe_port(efx); 1904874aeea5SJeff Kirsher if (rc) { 1905874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1906874aeea5SJeff Kirsher goto fail2; 1907874aeea5SJeff Kirsher } 1908874aeea5SJeff Kirsher 19097e6d06f0SBen Hutchings BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 19107e6d06f0SBen Hutchings if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 19117e6d06f0SBen Hutchings rc = -EINVAL; 19127e6d06f0SBen Hutchings goto fail3; 19137e6d06f0SBen Hutchings } 1914874aeea5SJeff Kirsher efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1915874aeea5SJeff Kirsher 19166d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 19176d8aaaf6SDaniel Pieczko rc = efx->type->vswitching_probe(efx); 19186d8aaaf6SDaniel Pieczko if (rc) /* not fatal; the PF will still work fine */ 19196d8aaaf6SDaniel Pieczko netif_warn(efx, probe, efx->net_dev, 19206d8aaaf6SDaniel Pieczko "failed to setup vswitching rc=%d;" 19216d8aaaf6SDaniel Pieczko " VFs may not function\n", rc); 19226d8aaaf6SDaniel Pieczko #endif 19236d8aaaf6SDaniel Pieczko 1924874aeea5SJeff Kirsher rc = efx_probe_filters(efx); 1925874aeea5SJeff Kirsher if (rc) { 1926874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1927874aeea5SJeff Kirsher "failed to create filter tables\n"); 19286d8aaaf6SDaniel Pieczko goto fail4; 1929874aeea5SJeff Kirsher } 1930874aeea5SJeff Kirsher 19317f967c01SBen Hutchings rc = efx_probe_channels(efx); 19327f967c01SBen Hutchings if (rc) 19336d8aaaf6SDaniel Pieczko goto fail5; 19347f967c01SBen Hutchings 1935874aeea5SJeff Kirsher return 0; 1936874aeea5SJeff Kirsher 19376d8aaaf6SDaniel Pieczko fail5: 19387f967c01SBen Hutchings efx_remove_filters(efx); 19396d8aaaf6SDaniel Pieczko fail4: 19406d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 19416d8aaaf6SDaniel Pieczko efx->type->vswitching_remove(efx); 19426d8aaaf6SDaniel Pieczko #endif 1943874aeea5SJeff Kirsher fail3: 1944874aeea5SJeff Kirsher efx_remove_port(efx); 1945874aeea5SJeff Kirsher fail2: 1946874aeea5SJeff Kirsher efx_remove_nic(efx); 1947874aeea5SJeff Kirsher fail1: 1948874aeea5SJeff Kirsher return rc; 1949874aeea5SJeff Kirsher } 1950874aeea5SJeff Kirsher 19518b7325b4SBen Hutchings /* If the interface is supposed to be running but is not, start 19528b7325b4SBen Hutchings * the hardware and software data path, regular activity for the port 19538b7325b4SBen Hutchings * (MAC statistics, link polling, etc.) and schedule the port to be 19548b7325b4SBen Hutchings * reconfigured. Interrupts must already be enabled. This function 19558b7325b4SBen Hutchings * is safe to call multiple times, so long as the NIC is not disabled. 19568b7325b4SBen Hutchings * Requires the RTNL lock. 19579f2cb71cSBen Hutchings */ 1958874aeea5SJeff Kirsher static void efx_start_all(struct efx_nic *efx) 1959874aeea5SJeff Kirsher { 1960874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 19618b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1962874aeea5SJeff Kirsher 1963874aeea5SJeff Kirsher /* Check that it is appropriate to restart the interface. All 1964874aeea5SJeff Kirsher * of these flags are safe to read under just the rtnl lock */ 1965e283546cSEdward Cree if (efx->port_enabled || !netif_running(efx->net_dev) || 1966e283546cSEdward Cree efx->reset_pending) 1967874aeea5SJeff Kirsher return; 1968874aeea5SJeff Kirsher 1969874aeea5SJeff Kirsher efx_start_port(efx); 19709f2cb71cSBen Hutchings efx_start_datapath(efx); 1971874aeea5SJeff Kirsher 1972626950dbSAlexandre Rames /* Start the hardware monitor if there is one */ 1973626950dbSAlexandre Rames if (efx->type->monitor != NULL) 1974874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 1975874aeea5SJeff Kirsher efx_monitor_interval); 1976626950dbSAlexandre Rames 19775a6681e2SEdward Cree /* Link state detection is normally event-driven; we have 1978626950dbSAlexandre Rames * to poll now because we could have missed a change 1979626950dbSAlexandre Rames */ 1980874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1981874aeea5SJeff Kirsher if (efx->phy_op->poll(efx)) 1982874aeea5SJeff Kirsher efx_link_status_changed(efx); 1983874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1984874aeea5SJeff Kirsher 1985874aeea5SJeff Kirsher efx->type->start_stats(efx); 1986f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 1987f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 1988f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 1989f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 1990874aeea5SJeff Kirsher } 1991874aeea5SJeff Kirsher 19928b7325b4SBen Hutchings /* Quiesce the hardware and software data path, and regular activity 19938b7325b4SBen Hutchings * for the port without bringing the link down. Safe to call multiple 19948b7325b4SBen Hutchings * times with the NIC in almost any state, but interrupts should be 19958b7325b4SBen Hutchings * enabled. Requires the RTNL lock. 19968b7325b4SBen Hutchings */ 1997874aeea5SJeff Kirsher static void efx_stop_all(struct efx_nic *efx) 1998874aeea5SJeff Kirsher { 1999874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2000874aeea5SJeff Kirsher 2001874aeea5SJeff Kirsher /* port_enabled can be read safely under the rtnl lock */ 2002874aeea5SJeff Kirsher if (!efx->port_enabled) 2003874aeea5SJeff Kirsher return; 2004874aeea5SJeff Kirsher 2005f8f3b5aeSJon Cooper /* update stats before we go down so we can accurately count 2006f8f3b5aeSJon Cooper * rx_nodesc_drops 2007f8f3b5aeSJon Cooper */ 2008f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 2009f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 2010f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 2011f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 2012874aeea5SJeff Kirsher efx->type->stop_stats(efx); 2013874aeea5SJeff Kirsher efx_stop_port(efx); 2014874aeea5SJeff Kirsher 201529c69a48SBen Hutchings /* Stop the kernel transmit interface. This is only valid if 201629c69a48SBen Hutchings * the device is stopped or detached; otherwise the watchdog 201729c69a48SBen Hutchings * may fire immediately. 201829c69a48SBen Hutchings */ 201929c69a48SBen Hutchings WARN_ON(netif_running(efx->net_dev) && 202029c69a48SBen Hutchings netif_device_present(efx->net_dev)); 20219f2cb71cSBen Hutchings netif_tx_disable(efx->net_dev); 20229f2cb71cSBen Hutchings 20239f2cb71cSBen Hutchings efx_stop_datapath(efx); 2024874aeea5SJeff Kirsher } 2025874aeea5SJeff Kirsher 2026874aeea5SJeff Kirsher static void efx_remove_all(struct efx_nic *efx) 2027874aeea5SJeff Kirsher { 2028874aeea5SJeff Kirsher efx_remove_channels(efx); 20297f967c01SBen Hutchings efx_remove_filters(efx); 20306d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 20316d8aaaf6SDaniel Pieczko efx->type->vswitching_remove(efx); 20326d8aaaf6SDaniel Pieczko #endif 2033874aeea5SJeff Kirsher efx_remove_port(efx); 2034874aeea5SJeff Kirsher efx_remove_nic(efx); 2035874aeea5SJeff Kirsher } 2036874aeea5SJeff Kirsher 2037874aeea5SJeff Kirsher /************************************************************************** 2038874aeea5SJeff Kirsher * 2039874aeea5SJeff Kirsher * Interrupt moderation 2040874aeea5SJeff Kirsher * 2041874aeea5SJeff Kirsher **************************************************************************/ 2042539de7c5SBert Kenward unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 2043874aeea5SJeff Kirsher { 2044b548f976SBen Hutchings if (usecs == 0) 2045b548f976SBen Hutchings return 0; 2046539de7c5SBert Kenward if (usecs * 1000 < efx->timer_quantum_ns) 2047874aeea5SJeff Kirsher return 1; /* never round down to 0 */ 2048539de7c5SBert Kenward return usecs * 1000 / efx->timer_quantum_ns; 2049539de7c5SBert Kenward } 2050539de7c5SBert Kenward 2051539de7c5SBert Kenward unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 2052539de7c5SBert Kenward { 2053539de7c5SBert Kenward /* We must round up when converting ticks to microseconds 2054539de7c5SBert Kenward * because we round down when converting the other way. 2055539de7c5SBert Kenward */ 2056539de7c5SBert Kenward return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 2057874aeea5SJeff Kirsher } 2058874aeea5SJeff Kirsher 2059874aeea5SJeff Kirsher /* Set interrupt moderation parameters */ 20609e393b30SBen Hutchings int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 20619e393b30SBen Hutchings unsigned int rx_usecs, bool rx_adaptive, 20629e393b30SBen Hutchings bool rx_may_override_tx) 2063874aeea5SJeff Kirsher { 2064874aeea5SJeff Kirsher struct efx_channel *channel; 2065d95e329aSBert Kenward unsigned int timer_max_us; 2066d95e329aSBert Kenward 2067874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2068874aeea5SJeff Kirsher 2069d95e329aSBert Kenward timer_max_us = efx->timer_max_ns / 1000; 2070d95e329aSBert Kenward 2071d95e329aSBert Kenward if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 20729e393b30SBen Hutchings return -EINVAL; 20739e393b30SBen Hutchings 2074539de7c5SBert Kenward if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 20759e393b30SBen Hutchings !rx_may_override_tx) { 20769e393b30SBen Hutchings netif_err(efx, drv, efx->net_dev, "Channels are shared. " 20779e393b30SBen Hutchings "RX and TX IRQ moderation must be equal\n"); 20789e393b30SBen Hutchings return -EINVAL; 20799e393b30SBen Hutchings } 20809e393b30SBen Hutchings 2081874aeea5SJeff Kirsher efx->irq_rx_adaptive = rx_adaptive; 2082539de7c5SBert Kenward efx->irq_rx_moderation_us = rx_usecs; 2083874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2084874aeea5SJeff Kirsher if (efx_channel_has_rx_queue(channel)) 2085539de7c5SBert Kenward channel->irq_moderation_us = rx_usecs; 2086874aeea5SJeff Kirsher else if (efx_channel_has_tx_queues(channel)) 2087539de7c5SBert Kenward channel->irq_moderation_us = tx_usecs; 2088874aeea5SJeff Kirsher } 20899e393b30SBen Hutchings 20909e393b30SBen Hutchings return 0; 2091874aeea5SJeff Kirsher } 2092874aeea5SJeff Kirsher 2093a0c4faf5SBen Hutchings void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2094a0c4faf5SBen Hutchings unsigned int *rx_usecs, bool *rx_adaptive) 2095a0c4faf5SBen Hutchings { 2096a0c4faf5SBen Hutchings *rx_adaptive = efx->irq_rx_adaptive; 2097539de7c5SBert Kenward *rx_usecs = efx->irq_rx_moderation_us; 2098a0c4faf5SBen Hutchings 2099a0c4faf5SBen Hutchings /* If channels are shared between RX and TX, so is IRQ 2100a0c4faf5SBen Hutchings * moderation. Otherwise, IRQ moderation is the same for all 2101a0c4faf5SBen Hutchings * TX channels and is not adaptive. 2102a0c4faf5SBen Hutchings */ 2103539de7c5SBert Kenward if (efx->tx_channel_offset == 0) { 2104a0c4faf5SBen Hutchings *tx_usecs = *rx_usecs; 2105539de7c5SBert Kenward } else { 2106539de7c5SBert Kenward struct efx_channel *tx_channel; 2107539de7c5SBert Kenward 2108539de7c5SBert Kenward tx_channel = efx->channel[efx->tx_channel_offset]; 2109539de7c5SBert Kenward *tx_usecs = tx_channel->irq_moderation_us; 2110539de7c5SBert Kenward } 2111a0c4faf5SBen Hutchings } 2112a0c4faf5SBen Hutchings 2113874aeea5SJeff Kirsher /************************************************************************** 2114874aeea5SJeff Kirsher * 2115874aeea5SJeff Kirsher * Hardware monitor 2116874aeea5SJeff Kirsher * 2117874aeea5SJeff Kirsher **************************************************************************/ 2118874aeea5SJeff Kirsher 2119874aeea5SJeff Kirsher /* Run periodically off the general workqueue */ 2120874aeea5SJeff Kirsher static void efx_monitor(struct work_struct *data) 2121874aeea5SJeff Kirsher { 2122874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, 2123874aeea5SJeff Kirsher monitor_work.work); 2124874aeea5SJeff Kirsher 2125874aeea5SJeff Kirsher netif_vdbg(efx, timer, efx->net_dev, 2126874aeea5SJeff Kirsher "hardware monitor executing on CPU %d\n", 2127874aeea5SJeff Kirsher raw_smp_processor_id()); 2128874aeea5SJeff Kirsher BUG_ON(efx->type->monitor == NULL); 2129874aeea5SJeff Kirsher 2130874aeea5SJeff Kirsher /* If the mac_lock is already held then it is likely a port 2131874aeea5SJeff Kirsher * reconfiguration is already in place, which will likely do 2132874aeea5SJeff Kirsher * most of the work of monitor() anyway. */ 2133874aeea5SJeff Kirsher if (mutex_trylock(&efx->mac_lock)) { 2134874aeea5SJeff Kirsher if (efx->port_enabled) 2135874aeea5SJeff Kirsher efx->type->monitor(efx); 2136874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2137874aeea5SJeff Kirsher } 2138874aeea5SJeff Kirsher 2139874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 2140874aeea5SJeff Kirsher efx_monitor_interval); 2141874aeea5SJeff Kirsher } 2142874aeea5SJeff Kirsher 2143874aeea5SJeff Kirsher /************************************************************************** 2144874aeea5SJeff Kirsher * 2145874aeea5SJeff Kirsher * ioctls 2146874aeea5SJeff Kirsher * 2147874aeea5SJeff Kirsher *************************************************************************/ 2148874aeea5SJeff Kirsher 2149874aeea5SJeff Kirsher /* Net device ioctl 2150874aeea5SJeff Kirsher * Context: process, rtnl_lock() held. 2151874aeea5SJeff Kirsher */ 2152874aeea5SJeff Kirsher static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2153874aeea5SJeff Kirsher { 2154874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2155874aeea5SJeff Kirsher struct mii_ioctl_data *data = if_mii(ifr); 2156874aeea5SJeff Kirsher 21577c236c43SStuart Hodgson if (cmd == SIOCSHWTSTAMP) 2158433dc9b3SBen Hutchings return efx_ptp_set_ts_config(efx, ifr); 2159433dc9b3SBen Hutchings if (cmd == SIOCGHWTSTAMP) 2160433dc9b3SBen Hutchings return efx_ptp_get_ts_config(efx, ifr); 21617c236c43SStuart Hodgson 2162874aeea5SJeff Kirsher /* Convert phy_id from older PRTAD/DEVAD format */ 2163874aeea5SJeff Kirsher if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2164874aeea5SJeff Kirsher (data->phy_id & 0xfc00) == 0x0400) 2165874aeea5SJeff Kirsher data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2166874aeea5SJeff Kirsher 2167874aeea5SJeff Kirsher return mdio_mii_ioctl(&efx->mdio, data, cmd); 2168874aeea5SJeff Kirsher } 2169874aeea5SJeff Kirsher 2170874aeea5SJeff Kirsher /************************************************************************** 2171874aeea5SJeff Kirsher * 2172874aeea5SJeff Kirsher * NAPI interface 2173874aeea5SJeff Kirsher * 2174874aeea5SJeff Kirsher **************************************************************************/ 2175874aeea5SJeff Kirsher 21767f967c01SBen Hutchings static void efx_init_napi_channel(struct efx_channel *channel) 2177874aeea5SJeff Kirsher { 21787f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 2179874aeea5SJeff Kirsher 2180874aeea5SJeff Kirsher channel->napi_dev = efx->net_dev; 2181874aeea5SJeff Kirsher netif_napi_add(channel->napi_dev, &channel->napi_str, 2182874aeea5SJeff Kirsher efx_poll, napi_weight); 2183874aeea5SJeff Kirsher } 21847f967c01SBen Hutchings 21857f967c01SBen Hutchings static void efx_init_napi(struct efx_nic *efx) 21867f967c01SBen Hutchings { 21877f967c01SBen Hutchings struct efx_channel *channel; 21887f967c01SBen Hutchings 21897f967c01SBen Hutchings efx_for_each_channel(channel, efx) 21907f967c01SBen Hutchings efx_init_napi_channel(channel); 2191874aeea5SJeff Kirsher } 2192874aeea5SJeff Kirsher 2193874aeea5SJeff Kirsher static void efx_fini_napi_channel(struct efx_channel *channel) 2194874aeea5SJeff Kirsher { 2195973334a1SEric Dumazet if (channel->napi_dev) 2196874aeea5SJeff Kirsher netif_napi_del(&channel->napi_str); 2197973334a1SEric Dumazet 2198874aeea5SJeff Kirsher channel->napi_dev = NULL; 2199874aeea5SJeff Kirsher } 2200874aeea5SJeff Kirsher 2201874aeea5SJeff Kirsher static void efx_fini_napi(struct efx_nic *efx) 2202874aeea5SJeff Kirsher { 2203874aeea5SJeff Kirsher struct efx_channel *channel; 2204874aeea5SJeff Kirsher 2205874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 2206874aeea5SJeff Kirsher efx_fini_napi_channel(channel); 2207874aeea5SJeff Kirsher } 2208874aeea5SJeff Kirsher 2209874aeea5SJeff Kirsher /************************************************************************** 2210874aeea5SJeff Kirsher * 2211874aeea5SJeff Kirsher * Kernel net device interface 2212874aeea5SJeff Kirsher * 2213874aeea5SJeff Kirsher *************************************************************************/ 2214874aeea5SJeff Kirsher 2215874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2216e340be92SShradha Shah int efx_net_open(struct net_device *net_dev) 2217874aeea5SJeff Kirsher { 2218874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 22198b7325b4SBen Hutchings int rc; 22208b7325b4SBen Hutchings 2221874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2222874aeea5SJeff Kirsher raw_smp_processor_id()); 2223874aeea5SJeff Kirsher 22248b7325b4SBen Hutchings rc = efx_check_disabled(efx); 22258b7325b4SBen Hutchings if (rc) 22268b7325b4SBen Hutchings return rc; 2227874aeea5SJeff Kirsher if (efx->phy_mode & PHY_MODE_SPECIAL) 2228874aeea5SJeff Kirsher return -EBUSY; 2229874aeea5SJeff Kirsher if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2230874aeea5SJeff Kirsher return -EIO; 2231874aeea5SJeff Kirsher 2232874aeea5SJeff Kirsher /* Notify the kernel of the link state polled during driver load, 2233874aeea5SJeff Kirsher * before the monitor starts running */ 2234874aeea5SJeff Kirsher efx_link_status_changed(efx); 2235874aeea5SJeff Kirsher 2236874aeea5SJeff Kirsher efx_start_all(efx); 22379c568fd8SPeter Dunning if (efx->state == STATE_DISABLED || efx->reset_pending) 22389c568fd8SPeter Dunning netif_device_detach(efx->net_dev); 2239dd40781eSBen Hutchings efx_selftest_async_start(efx); 2240874aeea5SJeff Kirsher return 0; 2241874aeea5SJeff Kirsher } 2242874aeea5SJeff Kirsher 2243874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. 2244874aeea5SJeff Kirsher * Note that the kernel will ignore our return code; this method 2245874aeea5SJeff Kirsher * should really be a void. 2246874aeea5SJeff Kirsher */ 2247e340be92SShradha Shah int efx_net_stop(struct net_device *net_dev) 2248874aeea5SJeff Kirsher { 2249874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2250874aeea5SJeff Kirsher 2251874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2252874aeea5SJeff Kirsher raw_smp_processor_id()); 2253874aeea5SJeff Kirsher 2254874aeea5SJeff Kirsher /* Stop the device and flush all the channels */ 2255874aeea5SJeff Kirsher efx_stop_all(efx); 2256874aeea5SJeff Kirsher 2257874aeea5SJeff Kirsher return 0; 2258874aeea5SJeff Kirsher } 2259874aeea5SJeff Kirsher 2260874aeea5SJeff Kirsher /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2261bc1f4470Sstephen hemminger static void efx_net_stats(struct net_device *net_dev, 22622aa9ef11SBen Hutchings struct rtnl_link_stats64 *stats) 2263874aeea5SJeff Kirsher { 2264874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2265874aeea5SJeff Kirsher 2266874aeea5SJeff Kirsher spin_lock_bh(&efx->stats_lock); 2267cd0ecc9aSBen Hutchings efx->type->update_stats(efx, NULL, stats); 22681cb34522SBen Hutchings spin_unlock_bh(&efx->stats_lock); 2269874aeea5SJeff Kirsher } 2270874aeea5SJeff Kirsher 2271874aeea5SJeff Kirsher /* Context: netif_tx_lock held, BHs disabled. */ 2272874aeea5SJeff Kirsher static void efx_watchdog(struct net_device *net_dev) 2273874aeea5SJeff Kirsher { 2274874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2275874aeea5SJeff Kirsher 2276874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 2277874aeea5SJeff Kirsher "TX stuck with port_enabled=%d: resetting channels\n", 2278874aeea5SJeff Kirsher efx->port_enabled); 2279874aeea5SJeff Kirsher 2280874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2281874aeea5SJeff Kirsher } 2282874aeea5SJeff Kirsher 2283874aeea5SJeff Kirsher 2284874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2285874aeea5SJeff Kirsher static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2286874aeea5SJeff Kirsher { 2287874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 22888b7325b4SBen Hutchings int rc; 2289874aeea5SJeff Kirsher 22908b7325b4SBen Hutchings rc = efx_check_disabled(efx); 22918b7325b4SBen Hutchings if (rc) 22928b7325b4SBen Hutchings return rc; 2293874aeea5SJeff Kirsher 2294874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2295874aeea5SJeff Kirsher 229629c69a48SBen Hutchings efx_device_detach_sync(efx); 229729c69a48SBen Hutchings efx_stop_all(efx); 229829c69a48SBen Hutchings 2299874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2300874aeea5SJeff Kirsher net_dev->mtu = new_mtu; 23010d322413SEdward Cree efx_mac_reconfigure(efx); 2302874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2303874aeea5SJeff Kirsher 2304874aeea5SJeff Kirsher efx_start_all(efx); 23059c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 23066c8eef4aSBen Hutchings return 0; 2307874aeea5SJeff Kirsher } 2308874aeea5SJeff Kirsher 2309874aeea5SJeff Kirsher static int efx_set_mac_address(struct net_device *net_dev, void *data) 2310874aeea5SJeff Kirsher { 2311874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2312874aeea5SJeff Kirsher struct sockaddr *addr = data; 2313e0b3ae30SBen Hutchings u8 *new_addr = addr->sa_data; 2314cfc77c2fSShradha Shah u8 old_addr[6]; 2315cfc77c2fSShradha Shah int rc; 2316874aeea5SJeff Kirsher 2317874aeea5SJeff Kirsher if (!is_valid_ether_addr(new_addr)) { 2318874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2319874aeea5SJeff Kirsher "invalid ethernet MAC address requested: %pM\n", 2320874aeea5SJeff Kirsher new_addr); 2321504f9b5aSDanny Kukawka return -EADDRNOTAVAIL; 2322874aeea5SJeff Kirsher } 2323874aeea5SJeff Kirsher 2324cfc77c2fSShradha Shah /* save old address */ 2325cfc77c2fSShradha Shah ether_addr_copy(old_addr, net_dev->dev_addr); 2326cd84ff4dSEdward Cree ether_addr_copy(net_dev->dev_addr, new_addr); 2327910c8789SShradha Shah if (efx->type->set_mac_address) { 2328910c8789SShradha Shah rc = efx->type->set_mac_address(efx); 2329cfc77c2fSShradha Shah if (rc) { 2330cfc77c2fSShradha Shah ether_addr_copy(net_dev->dev_addr, old_addr); 2331cfc77c2fSShradha Shah return rc; 2332cfc77c2fSShradha Shah } 2333cfc77c2fSShradha Shah } 2334874aeea5SJeff Kirsher 2335874aeea5SJeff Kirsher /* Reconfigure the MAC */ 2336874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 23370d322413SEdward Cree efx_mac_reconfigure(efx); 2338874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2339874aeea5SJeff Kirsher 2340874aeea5SJeff Kirsher return 0; 2341874aeea5SJeff Kirsher } 2342874aeea5SJeff Kirsher 2343874aeea5SJeff Kirsher /* Context: netif_addr_lock held, BHs disabled. */ 23440fca8c97SBen Hutchings static void efx_set_rx_mode(struct net_device *net_dev) 2345874aeea5SJeff Kirsher { 2346874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2347874aeea5SJeff Kirsher 2348874aeea5SJeff Kirsher if (efx->port_enabled) 2349874aeea5SJeff Kirsher queue_work(efx->workqueue, &efx->mac_work); 2350874aeea5SJeff Kirsher /* Otherwise efx_start_port() will do this */ 2351874aeea5SJeff Kirsher } 2352874aeea5SJeff Kirsher 2353c8f44affSMichał Mirosław static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2354874aeea5SJeff Kirsher { 2355874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 23564a53ea8aSAndrew Rybchenko int rc; 2357874aeea5SJeff Kirsher 2358874aeea5SJeff Kirsher /* If disabling RX n-tuple filtering, clear existing filters */ 23594a53ea8aSAndrew Rybchenko if (net_dev->features & ~data & NETIF_F_NTUPLE) { 23604a53ea8aSAndrew Rybchenko rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 23614a53ea8aSAndrew Rybchenko if (rc) 23624a53ea8aSAndrew Rybchenko return rc; 23634a53ea8aSAndrew Rybchenko } 23644a53ea8aSAndrew Rybchenko 23656978729fSEdward Cree /* If Rx VLAN filter is changed, update filters via mac_reconfigure. 23666978729fSEdward Cree * If rx-fcs is changed, mac_reconfigure updates that too. 23676978729fSEdward Cree */ 23686978729fSEdward Cree if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER | 23696978729fSEdward Cree NETIF_F_RXFCS)) { 23704a53ea8aSAndrew Rybchenko /* efx_set_rx_mode() will schedule MAC work to update filters 23714a53ea8aSAndrew Rybchenko * when a new features are finally set in net_dev. 23724a53ea8aSAndrew Rybchenko */ 23734a53ea8aSAndrew Rybchenko efx_set_rx_mode(net_dev); 23744a53ea8aSAndrew Rybchenko } 2375874aeea5SJeff Kirsher 2376874aeea5SJeff Kirsher return 0; 2377874aeea5SJeff Kirsher } 2378874aeea5SJeff Kirsher 2379b40296fcSWei Yongjun static int efx_get_phys_port_id(struct net_device *net_dev, 238008a7b29bSBert Kenward struct netdev_phys_item_id *ppid) 238108a7b29bSBert Kenward { 238208a7b29bSBert Kenward struct efx_nic *efx = netdev_priv(net_dev); 238308a7b29bSBert Kenward 238408a7b29bSBert Kenward if (efx->type->get_phys_port_id) 238508a7b29bSBert Kenward return efx->type->get_phys_port_id(efx, ppid); 238608a7b29bSBert Kenward else 238708a7b29bSBert Kenward return -EOPNOTSUPP; 238808a7b29bSBert Kenward } 238908a7b29bSBert Kenward 2390ac019f08SBert Kenward static int efx_get_phys_port_name(struct net_device *net_dev, 2391ac019f08SBert Kenward char *name, size_t len) 2392ac019f08SBert Kenward { 2393ac019f08SBert Kenward struct efx_nic *efx = netdev_priv(net_dev); 2394ac019f08SBert Kenward 2395ac019f08SBert Kenward if (snprintf(name, len, "p%u", efx->port_num) >= len) 2396ac019f08SBert Kenward return -EINVAL; 2397ac019f08SBert Kenward return 0; 2398ac019f08SBert Kenward } 2399ac019f08SBert Kenward 24004a53ea8aSAndrew Rybchenko static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 24014a53ea8aSAndrew Rybchenko { 24024a53ea8aSAndrew Rybchenko struct efx_nic *efx = netdev_priv(net_dev); 24034a53ea8aSAndrew Rybchenko 24044a53ea8aSAndrew Rybchenko if (efx->type->vlan_rx_add_vid) 24054a53ea8aSAndrew Rybchenko return efx->type->vlan_rx_add_vid(efx, proto, vid); 24064a53ea8aSAndrew Rybchenko else 24074a53ea8aSAndrew Rybchenko return -EOPNOTSUPP; 24084a53ea8aSAndrew Rybchenko } 24094a53ea8aSAndrew Rybchenko 24104a53ea8aSAndrew Rybchenko static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 24114a53ea8aSAndrew Rybchenko { 24124a53ea8aSAndrew Rybchenko struct efx_nic *efx = netdev_priv(net_dev); 24134a53ea8aSAndrew Rybchenko 24144a53ea8aSAndrew Rybchenko if (efx->type->vlan_rx_kill_vid) 24154a53ea8aSAndrew Rybchenko return efx->type->vlan_rx_kill_vid(efx, proto, vid); 24164a53ea8aSAndrew Rybchenko else 24174a53ea8aSAndrew Rybchenko return -EOPNOTSUPP; 24184a53ea8aSAndrew Rybchenko } 24194a53ea8aSAndrew Rybchenko 2420e5fbd977SJon Cooper static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in) 2421e5fbd977SJon Cooper { 2422e5fbd977SJon Cooper switch (in) { 2423e5fbd977SJon Cooper case UDP_TUNNEL_TYPE_VXLAN: 2424e5fbd977SJon Cooper return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 2425e5fbd977SJon Cooper case UDP_TUNNEL_TYPE_GENEVE: 2426e5fbd977SJon Cooper return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 2427e5fbd977SJon Cooper default: 2428e5fbd977SJon Cooper return -1; 2429e5fbd977SJon Cooper } 2430e5fbd977SJon Cooper } 2431e5fbd977SJon Cooper 2432e5fbd977SJon Cooper static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti) 2433e5fbd977SJon Cooper { 2434e5fbd977SJon Cooper struct efx_nic *efx = netdev_priv(dev); 2435e5fbd977SJon Cooper struct efx_udp_tunnel tnl; 2436e5fbd977SJon Cooper int efx_tunnel_type; 2437e5fbd977SJon Cooper 2438e5fbd977SJon Cooper efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2439e5fbd977SJon Cooper if (efx_tunnel_type < 0) 2440e5fbd977SJon Cooper return; 2441e5fbd977SJon Cooper 2442e5fbd977SJon Cooper tnl.type = (u16)efx_tunnel_type; 2443e5fbd977SJon Cooper tnl.port = ti->port; 2444e5fbd977SJon Cooper 2445e5fbd977SJon Cooper if (efx->type->udp_tnl_add_port) 2446e5fbd977SJon Cooper (void)efx->type->udp_tnl_add_port(efx, tnl); 2447e5fbd977SJon Cooper } 2448e5fbd977SJon Cooper 2449e5fbd977SJon Cooper static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti) 2450e5fbd977SJon Cooper { 2451e5fbd977SJon Cooper struct efx_nic *efx = netdev_priv(dev); 2452e5fbd977SJon Cooper struct efx_udp_tunnel tnl; 2453e5fbd977SJon Cooper int efx_tunnel_type; 2454e5fbd977SJon Cooper 2455e5fbd977SJon Cooper efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2456e5fbd977SJon Cooper if (efx_tunnel_type < 0) 2457e5fbd977SJon Cooper return; 2458e5fbd977SJon Cooper 2459e5fbd977SJon Cooper tnl.type = (u16)efx_tunnel_type; 2460e5fbd977SJon Cooper tnl.port = ti->port; 2461e5fbd977SJon Cooper 2462c04ca616SDan Carpenter if (efx->type->udp_tnl_del_port) 2463e5fbd977SJon Cooper (void)efx->type->udp_tnl_del_port(efx, tnl); 2464e5fbd977SJon Cooper } 2465e5fbd977SJon Cooper 24667fa8d547SShradha Shah static const struct net_device_ops efx_netdev_ops = { 2467874aeea5SJeff Kirsher .ndo_open = efx_net_open, 2468874aeea5SJeff Kirsher .ndo_stop = efx_net_stop, 2469874aeea5SJeff Kirsher .ndo_get_stats64 = efx_net_stats, 2470874aeea5SJeff Kirsher .ndo_tx_timeout = efx_watchdog, 2471874aeea5SJeff Kirsher .ndo_start_xmit = efx_hard_start_xmit, 2472874aeea5SJeff Kirsher .ndo_validate_addr = eth_validate_addr, 2473874aeea5SJeff Kirsher .ndo_do_ioctl = efx_ioctl, 2474874aeea5SJeff Kirsher .ndo_change_mtu = efx_change_mtu, 2475874aeea5SJeff Kirsher .ndo_set_mac_address = efx_set_mac_address, 24760fca8c97SBen Hutchings .ndo_set_rx_mode = efx_set_rx_mode, 2477874aeea5SJeff Kirsher .ndo_set_features = efx_set_features, 24784a53ea8aSAndrew Rybchenko .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 24794a53ea8aSAndrew Rybchenko .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2480cd2d5b52SBen Hutchings #ifdef CONFIG_SFC_SRIOV 24817fa8d547SShradha Shah .ndo_set_vf_mac = efx_sriov_set_vf_mac, 24827fa8d547SShradha Shah .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 24837fa8d547SShradha Shah .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 24847fa8d547SShradha Shah .ndo_get_vf_config = efx_sriov_get_vf_config, 24854392dc69SEdward Cree .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2486cd2d5b52SBen Hutchings #endif 248708a7b29bSBert Kenward .ndo_get_phys_port_id = efx_get_phys_port_id, 2488ac019f08SBert Kenward .ndo_get_phys_port_name = efx_get_phys_port_name, 2489874aeea5SJeff Kirsher .ndo_setup_tc = efx_setup_tc, 2490874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 2491874aeea5SJeff Kirsher .ndo_rx_flow_steer = efx_filter_rfs, 2492874aeea5SJeff Kirsher #endif 2493e5fbd977SJon Cooper .ndo_udp_tunnel_add = efx_udp_tunnel_add, 2494e5fbd977SJon Cooper .ndo_udp_tunnel_del = efx_udp_tunnel_del, 2495874aeea5SJeff Kirsher }; 2496874aeea5SJeff Kirsher 2497874aeea5SJeff Kirsher static void efx_update_name(struct efx_nic *efx) 2498874aeea5SJeff Kirsher { 2499874aeea5SJeff Kirsher strcpy(efx->name, efx->net_dev->name); 2500874aeea5SJeff Kirsher efx_mtd_rename(efx); 2501874aeea5SJeff Kirsher efx_set_channel_names(efx); 2502874aeea5SJeff Kirsher } 2503874aeea5SJeff Kirsher 2504874aeea5SJeff Kirsher static int efx_netdev_event(struct notifier_block *this, 2505874aeea5SJeff Kirsher unsigned long event, void *ptr) 2506874aeea5SJeff Kirsher { 2507351638e7SJiri Pirko struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2508874aeea5SJeff Kirsher 25097fa8d547SShradha Shah if ((net_dev->netdev_ops == &efx_netdev_ops) && 2510874aeea5SJeff Kirsher event == NETDEV_CHANGENAME) 2511874aeea5SJeff Kirsher efx_update_name(netdev_priv(net_dev)); 2512874aeea5SJeff Kirsher 2513874aeea5SJeff Kirsher return NOTIFY_DONE; 2514874aeea5SJeff Kirsher } 2515874aeea5SJeff Kirsher 2516874aeea5SJeff Kirsher static struct notifier_block efx_netdev_notifier = { 2517874aeea5SJeff Kirsher .notifier_call = efx_netdev_event, 2518874aeea5SJeff Kirsher }; 2519874aeea5SJeff Kirsher 2520874aeea5SJeff Kirsher static ssize_t 2521874aeea5SJeff Kirsher show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2522874aeea5SJeff Kirsher { 2523874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2524874aeea5SJeff Kirsher return sprintf(buf, "%d\n", efx->phy_type); 2525874aeea5SJeff Kirsher } 2526776fbcc9SBen Hutchings static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2527874aeea5SJeff Kirsher 2528e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2529e7fef9b4SEdward Cree static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2530e7fef9b4SEdward Cree char *buf) 2531e7fef9b4SEdward Cree { 2532e7fef9b4SEdward Cree struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2533e7fef9b4SEdward Cree struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2534e7fef9b4SEdward Cree 2535e7fef9b4SEdward Cree return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2536e7fef9b4SEdward Cree } 2537e7fef9b4SEdward Cree static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2538e7fef9b4SEdward Cree const char *buf, size_t count) 2539e7fef9b4SEdward Cree { 2540e7fef9b4SEdward Cree struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2541e7fef9b4SEdward Cree struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2542e7fef9b4SEdward Cree bool enable = count > 0 && *buf != '0'; 2543e7fef9b4SEdward Cree 2544e7fef9b4SEdward Cree mcdi->logging_enabled = enable; 2545e7fef9b4SEdward Cree return count; 2546e7fef9b4SEdward Cree } 2547e7fef9b4SEdward Cree static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2548e7fef9b4SEdward Cree #endif 2549e7fef9b4SEdward Cree 2550874aeea5SJeff Kirsher static int efx_register_netdev(struct efx_nic *efx) 2551874aeea5SJeff Kirsher { 2552874aeea5SJeff Kirsher struct net_device *net_dev = efx->net_dev; 2553874aeea5SJeff Kirsher struct efx_channel *channel; 2554874aeea5SJeff Kirsher int rc; 2555874aeea5SJeff Kirsher 2556874aeea5SJeff Kirsher net_dev->watchdog_timeo = 5 * HZ; 2557874aeea5SJeff Kirsher net_dev->irq = efx->pci_dev->irq; 25587fa8d547SShradha Shah net_dev->netdev_ops = &efx_netdev_ops; 25597fa8d547SShradha Shah if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 25608127d661SBen Hutchings net_dev->priv_flags |= IFF_UNICAST_FLT; 25617ad24ea4SWilfried Klaebe net_dev->ethtool_ops = &efx_ethtool_ops; 25627e6d06f0SBen Hutchings net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2563cd94e519SBert Kenward net_dev->min_mtu = EFX_MIN_MTU; 2564cd94e519SBert Kenward net_dev->max_mtu = EFX_MAX_MTU; 2565874aeea5SJeff Kirsher 2566874aeea5SJeff Kirsher rtnl_lock(); 2567874aeea5SJeff Kirsher 25687153f623SBen Hutchings /* Enable resets to be scheduled and check whether any were 25697153f623SBen Hutchings * already requested. If so, the NIC is probably hosed so we 25707153f623SBen Hutchings * abort. 25717153f623SBen Hutchings */ 25727153f623SBen Hutchings efx->state = STATE_READY; 25737153f623SBen Hutchings smp_mb(); /* ensure we change state before checking reset_pending */ 25747153f623SBen Hutchings if (efx->reset_pending) { 25757153f623SBen Hutchings netif_err(efx, probe, efx->net_dev, 25767153f623SBen Hutchings "aborting probe due to scheduled reset\n"); 25777153f623SBen Hutchings rc = -EIO; 25787153f623SBen Hutchings goto fail_locked; 25797153f623SBen Hutchings } 25807153f623SBen Hutchings 2581874aeea5SJeff Kirsher rc = dev_alloc_name(net_dev, net_dev->name); 2582874aeea5SJeff Kirsher if (rc < 0) 2583874aeea5SJeff Kirsher goto fail_locked; 2584874aeea5SJeff Kirsher efx_update_name(efx); 2585874aeea5SJeff Kirsher 25868f8b3d51SBen Hutchings /* Always start with carrier off; PHY events will detect the link */ 25878f8b3d51SBen Hutchings netif_carrier_off(net_dev); 25888f8b3d51SBen Hutchings 2589874aeea5SJeff Kirsher rc = register_netdevice(net_dev); 2590874aeea5SJeff Kirsher if (rc) 2591874aeea5SJeff Kirsher goto fail_locked; 2592874aeea5SJeff Kirsher 2593874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2594874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 2595874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 2596874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 2597874aeea5SJeff Kirsher } 2598874aeea5SJeff Kirsher 25990bcf4a64SBen Hutchings efx_associate(efx); 26000bcf4a64SBen Hutchings 2601874aeea5SJeff Kirsher rtnl_unlock(); 2602874aeea5SJeff Kirsher 2603874aeea5SJeff Kirsher rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2604874aeea5SJeff Kirsher if (rc) { 2605874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2606874aeea5SJeff Kirsher "failed to init net dev attributes\n"); 2607874aeea5SJeff Kirsher goto fail_registered; 2608874aeea5SJeff Kirsher } 2609e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2610e7fef9b4SEdward Cree rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2611e7fef9b4SEdward Cree if (rc) { 2612e7fef9b4SEdward Cree netif_err(efx, drv, efx->net_dev, 2613e7fef9b4SEdward Cree "failed to init net dev attributes\n"); 2614e7fef9b4SEdward Cree goto fail_attr_mcdi_logging; 2615e7fef9b4SEdward Cree } 2616e7fef9b4SEdward Cree #endif 2617874aeea5SJeff Kirsher 2618874aeea5SJeff Kirsher return 0; 2619874aeea5SJeff Kirsher 2620e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2621e7fef9b4SEdward Cree fail_attr_mcdi_logging: 2622e7fef9b4SEdward Cree device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2623e7fef9b4SEdward Cree #endif 26247153f623SBen Hutchings fail_registered: 26257153f623SBen Hutchings rtnl_lock(); 26260bcf4a64SBen Hutchings efx_dissociate(efx); 26277153f623SBen Hutchings unregister_netdevice(net_dev); 2628874aeea5SJeff Kirsher fail_locked: 26297153f623SBen Hutchings efx->state = STATE_UNINIT; 2630874aeea5SJeff Kirsher rtnl_unlock(); 2631874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2632874aeea5SJeff Kirsher return rc; 2633874aeea5SJeff Kirsher } 2634874aeea5SJeff Kirsher 2635874aeea5SJeff Kirsher static void efx_unregister_netdev(struct efx_nic *efx) 2636874aeea5SJeff Kirsher { 2637874aeea5SJeff Kirsher if (!efx->net_dev) 2638874aeea5SJeff Kirsher return; 2639874aeea5SJeff Kirsher 2640874aeea5SJeff Kirsher BUG_ON(netdev_priv(efx->net_dev) != efx); 2641874aeea5SJeff Kirsher 2642e7fef9b4SEdward Cree if (efx_dev_registered(efx)) { 2643874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2644e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2645e7fef9b4SEdward Cree device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2646e7fef9b4SEdward Cree #endif 2647874aeea5SJeff Kirsher device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2648e7fef9b4SEdward Cree unregister_netdev(efx->net_dev); 2649e7fef9b4SEdward Cree } 2650874aeea5SJeff Kirsher } 2651874aeea5SJeff Kirsher 2652874aeea5SJeff Kirsher /************************************************************************** 2653874aeea5SJeff Kirsher * 2654874aeea5SJeff Kirsher * Device reset and suspend 2655874aeea5SJeff Kirsher * 2656874aeea5SJeff Kirsher **************************************************************************/ 2657874aeea5SJeff Kirsher 2658874aeea5SJeff Kirsher /* Tears down the entire software state and most of the hardware state 2659874aeea5SJeff Kirsher * before reset. */ 2660874aeea5SJeff Kirsher void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2661874aeea5SJeff Kirsher { 2662874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2663874aeea5SJeff Kirsher 2664e283546cSEdward Cree if (method == RESET_TYPE_MCDI_TIMEOUT) 2665e283546cSEdward Cree efx->type->prepare_flr(efx); 2666e283546cSEdward Cree 2667874aeea5SJeff Kirsher efx_stop_all(efx); 2668d8291187SBen Hutchings efx_disable_interrupts(efx); 26695642ceefSBen Hutchings 26705642ceefSBen Hutchings mutex_lock(&efx->mac_lock); 2671193f2003SBert Kenward down_write(&efx->filter_sem); 2672e0a65e3cSEdward Cree mutex_lock(&efx->rss_lock); 2673087e9025SJon Cooper if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2674087e9025SJon Cooper method != RESET_TYPE_DATAPATH) 2675874aeea5SJeff Kirsher efx->phy_op->fini(efx); 2676874aeea5SJeff Kirsher efx->type->fini(efx); 2677874aeea5SJeff Kirsher } 2678874aeea5SJeff Kirsher 2679874aeea5SJeff Kirsher /* This function will always ensure that the locks acquired in 2680874aeea5SJeff Kirsher * efx_reset_down() are released. A failure return code indicates 2681874aeea5SJeff Kirsher * that we were unable to reinitialise the hardware, and the 2682874aeea5SJeff Kirsher * driver should be disabled. If ok is false, then the rx and tx 2683874aeea5SJeff Kirsher * engines are not restarted, pending a RESET_DISABLE. */ 2684874aeea5SJeff Kirsher int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2685874aeea5SJeff Kirsher { 2686874aeea5SJeff Kirsher int rc; 2687874aeea5SJeff Kirsher 2688874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2689874aeea5SJeff Kirsher 2690e283546cSEdward Cree if (method == RESET_TYPE_MCDI_TIMEOUT) 2691e283546cSEdward Cree efx->type->finish_flr(efx); 2692e283546cSEdward Cree 2693e283546cSEdward Cree /* Ensure that SRAM is initialised even if we're disabling the device */ 2694874aeea5SJeff Kirsher rc = efx->type->init(efx); 2695874aeea5SJeff Kirsher if (rc) { 2696874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2697874aeea5SJeff Kirsher goto fail; 2698874aeea5SJeff Kirsher } 2699874aeea5SJeff Kirsher 2700874aeea5SJeff Kirsher if (!ok) 2701874aeea5SJeff Kirsher goto fail; 2702874aeea5SJeff Kirsher 2703087e9025SJon Cooper if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2704087e9025SJon Cooper method != RESET_TYPE_DATAPATH) { 2705874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 2706874aeea5SJeff Kirsher if (rc) 2707874aeea5SJeff Kirsher goto fail; 2708267d9d73SEdward Cree rc = efx->phy_op->reconfigure(efx); 2709267d9d73SEdward Cree if (rc && rc != -EPERM) 2710874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2711874aeea5SJeff Kirsher "could not restore PHY settings\n"); 2712874aeea5SJeff Kirsher } 2713874aeea5SJeff Kirsher 2714261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 2715261e4d96SJon Cooper if (rc) 2716261e4d96SJon Cooper goto fail; 27176d8aaaf6SDaniel Pieczko 27186d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 27196d8aaaf6SDaniel Pieczko rc = efx->type->vswitching_restore(efx); 27206d8aaaf6SDaniel Pieczko if (rc) /* not fatal; the PF will still work fine */ 27216d8aaaf6SDaniel Pieczko netif_warn(efx, probe, efx->net_dev, 27226d8aaaf6SDaniel Pieczko "failed to restore vswitching rc=%d;" 27236d8aaaf6SDaniel Pieczko " VFs may not function\n", rc); 27246d8aaaf6SDaniel Pieczko #endif 27256d8aaaf6SDaniel Pieczko 272642356d9aSEdward Cree if (efx->type->rx_restore_rss_contexts) 272742356d9aSEdward Cree efx->type->rx_restore_rss_contexts(efx); 2728e0a65e3cSEdward Cree mutex_unlock(&efx->rss_lock); 2729193f2003SBert Kenward efx->type->filter_table_restore(efx); 2730193f2003SBert Kenward up_write(&efx->filter_sem); 27317fa8d547SShradha Shah if (efx->type->sriov_reset) 2732d98a4ffeSShradha Shah efx->type->sriov_reset(efx); 2733874aeea5SJeff Kirsher 2734874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2735874aeea5SJeff Kirsher 2736874aeea5SJeff Kirsher efx_start_all(efx); 2737874aeea5SJeff Kirsher 2738e5fbd977SJon Cooper if (efx->type->udp_tnl_push_ports) 2739e5fbd977SJon Cooper efx->type->udp_tnl_push_ports(efx); 2740e5fbd977SJon Cooper 2741874aeea5SJeff Kirsher return 0; 2742874aeea5SJeff Kirsher 2743874aeea5SJeff Kirsher fail: 2744874aeea5SJeff Kirsher efx->port_initialized = false; 2745874aeea5SJeff Kirsher 2746e0a65e3cSEdward Cree mutex_unlock(&efx->rss_lock); 2747193f2003SBert Kenward up_write(&efx->filter_sem); 2748874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2749874aeea5SJeff Kirsher 2750874aeea5SJeff Kirsher return rc; 2751874aeea5SJeff Kirsher } 2752874aeea5SJeff Kirsher 2753874aeea5SJeff Kirsher /* Reset the NIC using the specified method. Note that the reset may 2754874aeea5SJeff Kirsher * fail, in which case the card will be left in an unusable state. 2755874aeea5SJeff Kirsher * 2756874aeea5SJeff Kirsher * Caller must hold the rtnl_lock. 2757874aeea5SJeff Kirsher */ 2758874aeea5SJeff Kirsher int efx_reset(struct efx_nic *efx, enum reset_type method) 2759874aeea5SJeff Kirsher { 2760874aeea5SJeff Kirsher int rc, rc2; 2761874aeea5SJeff Kirsher bool disabled; 2762874aeea5SJeff Kirsher 2763874aeea5SJeff Kirsher netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2764874aeea5SJeff Kirsher RESET_TYPE(method)); 2765874aeea5SJeff Kirsher 2766c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2767874aeea5SJeff Kirsher efx_reset_down(efx, method); 2768874aeea5SJeff Kirsher 2769874aeea5SJeff Kirsher rc = efx->type->reset(efx, method); 2770874aeea5SJeff Kirsher if (rc) { 2771874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2772874aeea5SJeff Kirsher goto out; 2773874aeea5SJeff Kirsher } 2774874aeea5SJeff Kirsher 2775874aeea5SJeff Kirsher /* Clear flags for the scopes we covered. We assume the NIC and 2776874aeea5SJeff Kirsher * driver are now quiescent so that there is no race here. 2777874aeea5SJeff Kirsher */ 2778e283546cSEdward Cree if (method < RESET_TYPE_MAX_METHOD) 2779874aeea5SJeff Kirsher efx->reset_pending &= -(1 << (method + 1)); 2780e283546cSEdward Cree else /* it doesn't fit into the well-ordered scope hierarchy */ 2781e283546cSEdward Cree __clear_bit(method, &efx->reset_pending); 2782874aeea5SJeff Kirsher 2783874aeea5SJeff Kirsher /* Reinitialise bus-mastering, which may have been turned off before 2784874aeea5SJeff Kirsher * the reset was scheduled. This is still appropriate, even in the 2785874aeea5SJeff Kirsher * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2786874aeea5SJeff Kirsher * can respond to requests. */ 2787874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 2788874aeea5SJeff Kirsher 2789874aeea5SJeff Kirsher out: 2790874aeea5SJeff Kirsher /* Leave device stopped if necessary */ 2791626950dbSAlexandre Rames disabled = rc || 2792626950dbSAlexandre Rames method == RESET_TYPE_DISABLE || 2793626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_DISABLE; 2794874aeea5SJeff Kirsher rc2 = efx_reset_up(efx, method, !disabled); 2795874aeea5SJeff Kirsher if (rc2) { 2796874aeea5SJeff Kirsher disabled = true; 2797874aeea5SJeff Kirsher if (!rc) 2798874aeea5SJeff Kirsher rc = rc2; 2799874aeea5SJeff Kirsher } 2800874aeea5SJeff Kirsher 2801874aeea5SJeff Kirsher if (disabled) { 2802874aeea5SJeff Kirsher dev_close(efx->net_dev); 2803874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2804874aeea5SJeff Kirsher efx->state = STATE_DISABLED; 2805874aeea5SJeff Kirsher } else { 2806874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 28079c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 2808874aeea5SJeff Kirsher } 2809874aeea5SJeff Kirsher return rc; 2810874aeea5SJeff Kirsher } 2811874aeea5SJeff Kirsher 2812626950dbSAlexandre Rames /* Try recovery mechanisms. 2813626950dbSAlexandre Rames * For now only EEH is supported. 2814626950dbSAlexandre Rames * Returns 0 if the recovery mechanisms are unsuccessful. 2815626950dbSAlexandre Rames * Returns a non-zero value otherwise. 2816626950dbSAlexandre Rames */ 2817b28405b0SAlexandre Rames int efx_try_recovery(struct efx_nic *efx) 2818626950dbSAlexandre Rames { 2819626950dbSAlexandre Rames #ifdef CONFIG_EEH 2820626950dbSAlexandre Rames /* A PCI error can occur and not be seen by EEH because nothing 2821626950dbSAlexandre Rames * happens on the PCI bus. In this case the driver may fail and 2822626950dbSAlexandre Rames * schedule a 'recover or reset', leading to this recovery handler. 2823626950dbSAlexandre Rames * Manually call the eeh failure check function. 2824626950dbSAlexandre Rames */ 282512a89dbaSBenjamin Herrenschmidt struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2826626950dbSAlexandre Rames if (eeh_dev_check_failure(eehdev)) { 2827626950dbSAlexandre Rames /* The EEH mechanisms will handle the error and reset the 2828626950dbSAlexandre Rames * device if necessary. 2829626950dbSAlexandre Rames */ 2830626950dbSAlexandre Rames return 1; 2831626950dbSAlexandre Rames } 2832626950dbSAlexandre Rames #endif 2833626950dbSAlexandre Rames return 0; 2834626950dbSAlexandre Rames } 2835626950dbSAlexandre Rames 283674cd60a4SJon Cooper static void efx_wait_for_bist_end(struct efx_nic *efx) 283774cd60a4SJon Cooper { 283874cd60a4SJon Cooper int i; 283974cd60a4SJon Cooper 284074cd60a4SJon Cooper for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 284174cd60a4SJon Cooper if (efx_mcdi_poll_reboot(efx)) 284274cd60a4SJon Cooper goto out; 284374cd60a4SJon Cooper msleep(BIST_WAIT_DELAY_MS); 284474cd60a4SJon Cooper } 284574cd60a4SJon Cooper 284674cd60a4SJon Cooper netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 284774cd60a4SJon Cooper out: 284874cd60a4SJon Cooper /* Either way unset the BIST flag. If we found no reboot we probably 284974cd60a4SJon Cooper * won't recover, but we should try. 285074cd60a4SJon Cooper */ 285174cd60a4SJon Cooper efx->mc_bist_for_other_fn = false; 285274cd60a4SJon Cooper } 285374cd60a4SJon Cooper 2854874aeea5SJeff Kirsher /* The worker thread exists so that code that cannot sleep can 2855874aeea5SJeff Kirsher * schedule a reset for later. 2856874aeea5SJeff Kirsher */ 2857874aeea5SJeff Kirsher static void efx_reset_work(struct work_struct *data) 2858874aeea5SJeff Kirsher { 2859874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2860626950dbSAlexandre Rames unsigned long pending; 2861626950dbSAlexandre Rames enum reset_type method; 2862626950dbSAlexandre Rames 28636aa7de05SMark Rutland pending = READ_ONCE(efx->reset_pending); 2864626950dbSAlexandre Rames method = fls(pending) - 1; 2865626950dbSAlexandre Rames 286674cd60a4SJon Cooper if (method == RESET_TYPE_MC_BIST) 286774cd60a4SJon Cooper efx_wait_for_bist_end(efx); 286874cd60a4SJon Cooper 2869626950dbSAlexandre Rames if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2870626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_ALL) && 2871626950dbSAlexandre Rames efx_try_recovery(efx)) 2872626950dbSAlexandre Rames return; 2873874aeea5SJeff Kirsher 2874874aeea5SJeff Kirsher if (!pending) 2875874aeea5SJeff Kirsher return; 2876874aeea5SJeff Kirsher 2877874aeea5SJeff Kirsher rtnl_lock(); 28787153f623SBen Hutchings 28797153f623SBen Hutchings /* We checked the state in efx_schedule_reset() but it may 28807153f623SBen Hutchings * have changed by now. Now that we have the RTNL lock, 28817153f623SBen Hutchings * it cannot change again. 28827153f623SBen Hutchings */ 28837153f623SBen Hutchings if (efx->state == STATE_READY) 2884626950dbSAlexandre Rames (void)efx_reset(efx, method); 28857153f623SBen Hutchings 2886874aeea5SJeff Kirsher rtnl_unlock(); 2887874aeea5SJeff Kirsher } 2888874aeea5SJeff Kirsher 2889874aeea5SJeff Kirsher void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2890874aeea5SJeff Kirsher { 2891874aeea5SJeff Kirsher enum reset_type method; 2892874aeea5SJeff Kirsher 2893626950dbSAlexandre Rames if (efx->state == STATE_RECOVERY) { 2894626950dbSAlexandre Rames netif_dbg(efx, drv, efx->net_dev, 2895626950dbSAlexandre Rames "recovering: skip scheduling %s reset\n", 2896626950dbSAlexandre Rames RESET_TYPE(type)); 2897626950dbSAlexandre Rames return; 2898626950dbSAlexandre Rames } 2899626950dbSAlexandre Rames 2900874aeea5SJeff Kirsher switch (type) { 2901874aeea5SJeff Kirsher case RESET_TYPE_INVISIBLE: 2902874aeea5SJeff Kirsher case RESET_TYPE_ALL: 2903626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_ALL: 2904874aeea5SJeff Kirsher case RESET_TYPE_WORLD: 2905874aeea5SJeff Kirsher case RESET_TYPE_DISABLE: 2906626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_DISABLE: 2907087e9025SJon Cooper case RESET_TYPE_DATAPATH: 290874cd60a4SJon Cooper case RESET_TYPE_MC_BIST: 2909e283546cSEdward Cree case RESET_TYPE_MCDI_TIMEOUT: 2910874aeea5SJeff Kirsher method = type; 2911874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2912874aeea5SJeff Kirsher RESET_TYPE(method)); 2913874aeea5SJeff Kirsher break; 2914874aeea5SJeff Kirsher default: 2915874aeea5SJeff Kirsher method = efx->type->map_reset_reason(type); 2916874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, 2917874aeea5SJeff Kirsher "scheduling %s reset for %s\n", 2918874aeea5SJeff Kirsher RESET_TYPE(method), RESET_TYPE(type)); 2919874aeea5SJeff Kirsher break; 2920874aeea5SJeff Kirsher } 2921874aeea5SJeff Kirsher 2922874aeea5SJeff Kirsher set_bit(method, &efx->reset_pending); 29237153f623SBen Hutchings smp_mb(); /* ensure we change reset_pending before checking state */ 29247153f623SBen Hutchings 29257153f623SBen Hutchings /* If we're not READY then just leave the flags set as the cue 29267153f623SBen Hutchings * to abort probing or reschedule the reset later. 29277153f623SBen Hutchings */ 29286aa7de05SMark Rutland if (READ_ONCE(efx->state) != STATE_READY) 29297153f623SBen Hutchings return; 2930874aeea5SJeff Kirsher 2931874aeea5SJeff Kirsher /* efx_process_channel() will no longer read events once a 2932874aeea5SJeff Kirsher * reset is scheduled. So switch back to poll'd MCDI completions. */ 2933874aeea5SJeff Kirsher efx_mcdi_mode_poll(efx); 2934874aeea5SJeff Kirsher 2935874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 2936874aeea5SJeff Kirsher } 2937874aeea5SJeff Kirsher 2938874aeea5SJeff Kirsher /************************************************************************** 2939874aeea5SJeff Kirsher * 2940874aeea5SJeff Kirsher * List of NICs we support 2941874aeea5SJeff Kirsher * 2942874aeea5SJeff Kirsher **************************************************************************/ 2943874aeea5SJeff Kirsher 2944874aeea5SJeff Kirsher /* PCI device ID table */ 29459baa3c34SBenoit Taine static const struct pci_device_id efx_pci_table[] = { 2946547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2947874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 2948547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2949874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 29508127d661SBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 29518127d661SBen Hutchings .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 29526f7f8aa6SShradha Shah {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 29536f7f8aa6SShradha Shah .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 29543b06a00eSMateusz Wrzesinski {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 29553b06a00eSMateusz Wrzesinski .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2956dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2957dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2958dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2959dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2960dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2961dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2962aae5a316SEdward Cree {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 2963aae5a316SEdward Cree .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2964aae5a316SEdward Cree {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 2965aae5a316SEdward Cree .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2966874aeea5SJeff Kirsher {0} /* end of list */ 2967874aeea5SJeff Kirsher }; 2968874aeea5SJeff Kirsher 2969874aeea5SJeff Kirsher /************************************************************************** 2970874aeea5SJeff Kirsher * 2971874aeea5SJeff Kirsher * Dummy PHY/MAC operations 2972874aeea5SJeff Kirsher * 2973874aeea5SJeff Kirsher * Can be used for some unimplemented operations 2974874aeea5SJeff Kirsher * Needed so all function pointers are valid and do not have to be tested 2975874aeea5SJeff Kirsher * before use 2976874aeea5SJeff Kirsher * 2977874aeea5SJeff Kirsher **************************************************************************/ 2978874aeea5SJeff Kirsher int efx_port_dummy_op_int(struct efx_nic *efx) 2979874aeea5SJeff Kirsher { 2980874aeea5SJeff Kirsher return 0; 2981874aeea5SJeff Kirsher } 2982874aeea5SJeff Kirsher void efx_port_dummy_op_void(struct efx_nic *efx) {} 2983874aeea5SJeff Kirsher 2984874aeea5SJeff Kirsher static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2985874aeea5SJeff Kirsher { 2986874aeea5SJeff Kirsher return false; 2987874aeea5SJeff Kirsher } 2988874aeea5SJeff Kirsher 2989874aeea5SJeff Kirsher static const struct efx_phy_operations efx_dummy_phy_operations = { 2990874aeea5SJeff Kirsher .init = efx_port_dummy_op_int, 2991874aeea5SJeff Kirsher .reconfigure = efx_port_dummy_op_int, 2992874aeea5SJeff Kirsher .poll = efx_port_dummy_op_poll, 2993874aeea5SJeff Kirsher .fini = efx_port_dummy_op_void, 2994874aeea5SJeff Kirsher }; 2995874aeea5SJeff Kirsher 2996874aeea5SJeff Kirsher /************************************************************************** 2997874aeea5SJeff Kirsher * 2998874aeea5SJeff Kirsher * Data housekeeping 2999874aeea5SJeff Kirsher * 3000874aeea5SJeff Kirsher **************************************************************************/ 3001874aeea5SJeff Kirsher 3002874aeea5SJeff Kirsher /* This zeroes out and then fills in the invariants in a struct 3003874aeea5SJeff Kirsher * efx_nic (including all sub-structures). 3004874aeea5SJeff Kirsher */ 3005adeb15aaSBen Hutchings static int efx_init_struct(struct efx_nic *efx, 3006874aeea5SJeff Kirsher struct pci_dev *pci_dev, struct net_device *net_dev) 3007874aeea5SJeff Kirsher { 30086f9f6ec2SAndrew Rybchenko int rc = -ENOMEM, i; 3009874aeea5SJeff Kirsher 3010874aeea5SJeff Kirsher /* Initialise common structures */ 30110bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->node); 30120bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->secondary_list); 3013874aeea5SJeff Kirsher spin_lock_init(&efx->biu_lock); 3014874aeea5SJeff Kirsher #ifdef CONFIG_SFC_MTD 3015874aeea5SJeff Kirsher INIT_LIST_HEAD(&efx->mtd_list); 3016874aeea5SJeff Kirsher #endif 3017874aeea5SJeff Kirsher INIT_WORK(&efx->reset_work, efx_reset_work); 3018874aeea5SJeff Kirsher INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 3019dd40781eSBen Hutchings INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 3020874aeea5SJeff Kirsher efx->pci_dev = pci_dev; 3021874aeea5SJeff Kirsher efx->msg_enable = debug; 3022f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 3023874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 3024874aeea5SJeff Kirsher 3025874aeea5SJeff Kirsher efx->net_dev = net_dev; 302643a3739dSJon Cooper efx->rx_prefix_size = efx->type->rx_prefix_size; 30272ec03014SAndrew Rybchenko efx->rx_ip_align = 30282ec03014SAndrew Rybchenko NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 302943a3739dSJon Cooper efx->rx_packet_hash_offset = 303043a3739dSJon Cooper efx->type->rx_hash_offset - efx->type->rx_prefix_size; 3031bd9a265dSJon Cooper efx->rx_packet_ts_offset = 3032bd9a265dSJon Cooper efx->type->rx_ts_offset - efx->type->rx_prefix_size; 303342356d9aSEdward Cree INIT_LIST_HEAD(&efx->rss_context.list); 3034e0a65e3cSEdward Cree mutex_init(&efx->rss_lock); 3035874aeea5SJeff Kirsher spin_lock_init(&efx->stats_lock); 303671827443SEdward Cree efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 3037c1be4821SEdward Cree efx->num_mac_stats = MC_CMD_MAC_NSTATS; 3038c1be4821SEdward Cree BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 3039874aeea5SJeff Kirsher mutex_init(&efx->mac_lock); 30403af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 30413af0f342SEdward Cree mutex_init(&efx->rps_mutex); 3042f8d62037SEdward Cree spin_lock_init(&efx->rps_hash_lock); 3043f8d62037SEdward Cree /* Failure to allocate is not fatal, but may degrade ARFS performance */ 3044f8d62037SEdward Cree efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, 3045f8d62037SEdward Cree sizeof(*efx->rps_hash_table), GFP_KERNEL); 30463af0f342SEdward Cree #endif 3047874aeea5SJeff Kirsher efx->phy_op = &efx_dummy_phy_operations; 3048874aeea5SJeff Kirsher efx->mdio.dev = net_dev; 3049874aeea5SJeff Kirsher INIT_WORK(&efx->mac_work, efx_mac_work); 30509f2cb71cSBen Hutchings init_waitqueue_head(&efx->flush_wq); 3051874aeea5SJeff Kirsher 3052874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) { 3053874aeea5SJeff Kirsher efx->channel[i] = efx_alloc_channel(efx, i, NULL); 3054874aeea5SJeff Kirsher if (!efx->channel[i]) 3055874aeea5SJeff Kirsher goto fail; 3056d8291187SBen Hutchings efx->msi_context[i].efx = efx; 3057d8291187SBen Hutchings efx->msi_context[i].index = i; 3058874aeea5SJeff Kirsher } 3059874aeea5SJeff Kirsher 3060874aeea5SJeff Kirsher /* Higher numbered interrupt modes are less capable! */ 30616f9f6ec2SAndrew Rybchenko if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 30626f9f6ec2SAndrew Rybchenko efx->type->min_interrupt_mode)) { 30636f9f6ec2SAndrew Rybchenko rc = -EIO; 30646f9f6ec2SAndrew Rybchenko goto fail; 30656f9f6ec2SAndrew Rybchenko } 3066874aeea5SJeff Kirsher efx->interrupt_mode = max(efx->type->max_interrupt_mode, 3067874aeea5SJeff Kirsher interrupt_mode); 30686f9f6ec2SAndrew Rybchenko efx->interrupt_mode = min(efx->type->min_interrupt_mode, 30696f9f6ec2SAndrew Rybchenko interrupt_mode); 3070874aeea5SJeff Kirsher 3071874aeea5SJeff Kirsher /* Would be good to use the net_dev name, but we're too early */ 3072874aeea5SJeff Kirsher snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 3073874aeea5SJeff Kirsher pci_name(pci_dev)); 3074874aeea5SJeff Kirsher efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 3075874aeea5SJeff Kirsher if (!efx->workqueue) 3076874aeea5SJeff Kirsher goto fail; 3077874aeea5SJeff Kirsher 3078874aeea5SJeff Kirsher return 0; 3079874aeea5SJeff Kirsher 3080874aeea5SJeff Kirsher fail: 3081874aeea5SJeff Kirsher efx_fini_struct(efx); 30826f9f6ec2SAndrew Rybchenko return rc; 3083874aeea5SJeff Kirsher } 3084874aeea5SJeff Kirsher 3085874aeea5SJeff Kirsher static void efx_fini_struct(struct efx_nic *efx) 3086874aeea5SJeff Kirsher { 3087874aeea5SJeff Kirsher int i; 3088874aeea5SJeff Kirsher 3089f8d62037SEdward Cree #ifdef CONFIG_RFS_ACCEL 3090f8d62037SEdward Cree kfree(efx->rps_hash_table); 3091f8d62037SEdward Cree #endif 3092f8d62037SEdward Cree 3093874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) 3094874aeea5SJeff Kirsher kfree(efx->channel[i]); 3095874aeea5SJeff Kirsher 3096ef215e64SBen Hutchings kfree(efx->vpd_sn); 3097ef215e64SBen Hutchings 3098874aeea5SJeff Kirsher if (efx->workqueue) { 3099874aeea5SJeff Kirsher destroy_workqueue(efx->workqueue); 3100874aeea5SJeff Kirsher efx->workqueue = NULL; 3101874aeea5SJeff Kirsher } 3102874aeea5SJeff Kirsher } 3103874aeea5SJeff Kirsher 3104e4d112e4SEdward Cree void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 3105e4d112e4SEdward Cree { 3106e4d112e4SEdward Cree u64 n_rx_nodesc_trunc = 0; 3107e4d112e4SEdward Cree struct efx_channel *channel; 3108e4d112e4SEdward Cree 3109e4d112e4SEdward Cree efx_for_each_channel(channel, efx) 3110e4d112e4SEdward Cree n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 3111e4d112e4SEdward Cree stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 3112e4d112e4SEdward Cree stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 3113e4d112e4SEdward Cree } 3114e4d112e4SEdward Cree 3115f8d62037SEdward Cree bool efx_filter_spec_equal(const struct efx_filter_spec *left, 3116f8d62037SEdward Cree const struct efx_filter_spec *right) 3117f8d62037SEdward Cree { 3118f8d62037SEdward Cree if ((left->match_flags ^ right->match_flags) | 3119f8d62037SEdward Cree ((left->flags ^ right->flags) & 3120f8d62037SEdward Cree (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) 3121f8d62037SEdward Cree return false; 3122f8d62037SEdward Cree 3123f8d62037SEdward Cree return memcmp(&left->outer_vid, &right->outer_vid, 3124f8d62037SEdward Cree sizeof(struct efx_filter_spec) - 3125f8d62037SEdward Cree offsetof(struct efx_filter_spec, outer_vid)) == 0; 3126f8d62037SEdward Cree } 3127f8d62037SEdward Cree 3128f8d62037SEdward Cree u32 efx_filter_spec_hash(const struct efx_filter_spec *spec) 3129f8d62037SEdward Cree { 3130f8d62037SEdward Cree BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); 3131f8d62037SEdward Cree return jhash2((const u32 *)&spec->outer_vid, 3132f8d62037SEdward Cree (sizeof(struct efx_filter_spec) - 3133f8d62037SEdward Cree offsetof(struct efx_filter_spec, outer_vid)) / 4, 3134f8d62037SEdward Cree 0); 3135f8d62037SEdward Cree } 3136f8d62037SEdward Cree 3137f8d62037SEdward Cree #ifdef CONFIG_RFS_ACCEL 3138f8d62037SEdward Cree bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx, 3139f8d62037SEdward Cree bool *force) 3140f8d62037SEdward Cree { 3141f8d62037SEdward Cree if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) { 3142f8d62037SEdward Cree /* ARFS is currently updating this entry, leave it */ 3143f8d62037SEdward Cree return false; 3144f8d62037SEdward Cree } 3145f8d62037SEdward Cree if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) { 3146f8d62037SEdward Cree /* ARFS tried and failed to update this, so it's probably out 3147f8d62037SEdward Cree * of date. Remove the filter and the ARFS rule entry. 3148f8d62037SEdward Cree */ 3149f8d62037SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 3150f8d62037SEdward Cree *force = true; 3151f8d62037SEdward Cree return true; 3152f8d62037SEdward Cree } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */ 3153f8d62037SEdward Cree /* ARFS has moved on, so old filter is not needed. Since we did 3154f8d62037SEdward Cree * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will 3155f8d62037SEdward Cree * not be removed by efx_rps_hash_del() subsequently. 3156f8d62037SEdward Cree */ 3157f8d62037SEdward Cree *force = true; 3158f8d62037SEdward Cree return true; 3159f8d62037SEdward Cree } 3160f8d62037SEdward Cree /* Remove it iff ARFS wants to. */ 3161f8d62037SEdward Cree return true; 3162f8d62037SEdward Cree } 3163f8d62037SEdward Cree 3164829eb053SColin Ian King static 3165f8d62037SEdward Cree struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx, 3166f8d62037SEdward Cree const struct efx_filter_spec *spec) 3167f8d62037SEdward Cree { 3168f8d62037SEdward Cree u32 hash = efx_filter_spec_hash(spec); 3169f8d62037SEdward Cree 3170013ff4a6SLance Roy lockdep_assert_held(&efx->rps_hash_lock); 3171f8d62037SEdward Cree if (!efx->rps_hash_table) 3172f8d62037SEdward Cree return NULL; 3173f8d62037SEdward Cree return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE]; 3174f8d62037SEdward Cree } 3175f8d62037SEdward Cree 3176f8d62037SEdward Cree struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx, 3177f8d62037SEdward Cree const struct efx_filter_spec *spec) 3178f8d62037SEdward Cree { 3179f8d62037SEdward Cree struct efx_arfs_rule *rule; 3180f8d62037SEdward Cree struct hlist_head *head; 3181f8d62037SEdward Cree struct hlist_node *node; 3182f8d62037SEdward Cree 3183f8d62037SEdward Cree head = efx_rps_hash_bucket(efx, spec); 3184f8d62037SEdward Cree if (!head) 3185f8d62037SEdward Cree return NULL; 3186f8d62037SEdward Cree hlist_for_each(node, head) { 3187f8d62037SEdward Cree rule = container_of(node, struct efx_arfs_rule, node); 3188f8d62037SEdward Cree if (efx_filter_spec_equal(spec, &rule->spec)) 3189f8d62037SEdward Cree return rule; 3190f8d62037SEdward Cree } 3191f8d62037SEdward Cree return NULL; 3192f8d62037SEdward Cree } 3193f8d62037SEdward Cree 3194f8d62037SEdward Cree struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx, 3195f8d62037SEdward Cree const struct efx_filter_spec *spec, 3196f8d62037SEdward Cree bool *new) 3197f8d62037SEdward Cree { 3198f8d62037SEdward Cree struct efx_arfs_rule *rule; 3199f8d62037SEdward Cree struct hlist_head *head; 3200f8d62037SEdward Cree struct hlist_node *node; 3201f8d62037SEdward Cree 3202f8d62037SEdward Cree head = efx_rps_hash_bucket(efx, spec); 3203f8d62037SEdward Cree if (!head) 3204f8d62037SEdward Cree return NULL; 3205f8d62037SEdward Cree hlist_for_each(node, head) { 3206f8d62037SEdward Cree rule = container_of(node, struct efx_arfs_rule, node); 3207f8d62037SEdward Cree if (efx_filter_spec_equal(spec, &rule->spec)) { 3208f8d62037SEdward Cree *new = false; 3209f8d62037SEdward Cree return rule; 3210f8d62037SEdward Cree } 3211f8d62037SEdward Cree } 3212f8d62037SEdward Cree rule = kmalloc(sizeof(*rule), GFP_ATOMIC); 3213f8d62037SEdward Cree *new = true; 3214f8d62037SEdward Cree if (rule) { 3215f8d62037SEdward Cree memcpy(&rule->spec, spec, sizeof(rule->spec)); 3216f8d62037SEdward Cree hlist_add_head(&rule->node, head); 3217f8d62037SEdward Cree } 3218f8d62037SEdward Cree return rule; 3219f8d62037SEdward Cree } 3220f8d62037SEdward Cree 3221f8d62037SEdward Cree void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec) 3222f8d62037SEdward Cree { 3223f8d62037SEdward Cree struct efx_arfs_rule *rule; 3224f8d62037SEdward Cree struct hlist_head *head; 3225f8d62037SEdward Cree struct hlist_node *node; 3226f8d62037SEdward Cree 3227f8d62037SEdward Cree head = efx_rps_hash_bucket(efx, spec); 3228f8d62037SEdward Cree if (WARN_ON(!head)) 3229f8d62037SEdward Cree return; 3230f8d62037SEdward Cree hlist_for_each(node, head) { 3231f8d62037SEdward Cree rule = container_of(node, struct efx_arfs_rule, node); 3232f8d62037SEdward Cree if (efx_filter_spec_equal(spec, &rule->spec)) { 3233f8d62037SEdward Cree /* Someone already reused the entry. We know that if 3234f8d62037SEdward Cree * this check doesn't fire (i.e. filter_id == REMOVING) 3235f8d62037SEdward Cree * then the REMOVING mark was put there by our caller, 3236f8d62037SEdward Cree * because caller is holding a lock on filter table and 3237f8d62037SEdward Cree * only holders of that lock set REMOVING. 3238f8d62037SEdward Cree */ 3239f8d62037SEdward Cree if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING) 3240f8d62037SEdward Cree return; 3241f8d62037SEdward Cree hlist_del(node); 3242f8d62037SEdward Cree kfree(rule); 3243f8d62037SEdward Cree return; 3244f8d62037SEdward Cree } 3245f8d62037SEdward Cree } 3246f8d62037SEdward Cree /* We didn't find it. */ 3247f8d62037SEdward Cree WARN_ON(1); 3248f8d62037SEdward Cree } 3249f8d62037SEdward Cree #endif 3250f8d62037SEdward Cree 325142356d9aSEdward Cree /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because 325242356d9aSEdward Cree * (a) this is an infrequent control-plane operation and (b) n is small (max 64) 325342356d9aSEdward Cree */ 3254e0a65e3cSEdward Cree struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx) 325542356d9aSEdward Cree { 3256e0a65e3cSEdward Cree struct list_head *head = &efx->rss_context.list; 325742356d9aSEdward Cree struct efx_rss_context *ctx, *new; 325842356d9aSEdward Cree u32 id = 1; /* Don't use zero, that refers to the master RSS context */ 325942356d9aSEdward Cree 3260e0a65e3cSEdward Cree WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3261e0a65e3cSEdward Cree 326242356d9aSEdward Cree /* Search for first gap in the numbering */ 326342356d9aSEdward Cree list_for_each_entry(ctx, head, list) { 326442356d9aSEdward Cree if (ctx->user_id != id) 326542356d9aSEdward Cree break; 326642356d9aSEdward Cree id++; 326742356d9aSEdward Cree /* Check for wrap. If this happens, we have nearly 2^32 326842356d9aSEdward Cree * allocated RSS contexts, which seems unlikely. 326942356d9aSEdward Cree */ 327042356d9aSEdward Cree if (WARN_ON_ONCE(!id)) 327142356d9aSEdward Cree return NULL; 327242356d9aSEdward Cree } 327342356d9aSEdward Cree 327442356d9aSEdward Cree /* Create the new entry */ 327542356d9aSEdward Cree new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL); 327642356d9aSEdward Cree if (!new) 327742356d9aSEdward Cree return NULL; 327842356d9aSEdward Cree new->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 327942356d9aSEdward Cree new->rx_hash_udp_4tuple = false; 328042356d9aSEdward Cree 328142356d9aSEdward Cree /* Insert the new entry into the gap */ 328242356d9aSEdward Cree new->user_id = id; 328342356d9aSEdward Cree list_add_tail(&new->list, &ctx->list); 328442356d9aSEdward Cree return new; 328542356d9aSEdward Cree } 328642356d9aSEdward Cree 3287e0a65e3cSEdward Cree struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id) 328842356d9aSEdward Cree { 3289e0a65e3cSEdward Cree struct list_head *head = &efx->rss_context.list; 329042356d9aSEdward Cree struct efx_rss_context *ctx; 329142356d9aSEdward Cree 3292e0a65e3cSEdward Cree WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3293e0a65e3cSEdward Cree 329442356d9aSEdward Cree list_for_each_entry(ctx, head, list) 329542356d9aSEdward Cree if (ctx->user_id == id) 329642356d9aSEdward Cree return ctx; 329742356d9aSEdward Cree return NULL; 329842356d9aSEdward Cree } 329942356d9aSEdward Cree 330042356d9aSEdward Cree void efx_free_rss_context_entry(struct efx_rss_context *ctx) 330142356d9aSEdward Cree { 330242356d9aSEdward Cree list_del(&ctx->list); 330342356d9aSEdward Cree kfree(ctx); 330442356d9aSEdward Cree } 330542356d9aSEdward Cree 3306874aeea5SJeff Kirsher /************************************************************************** 3307874aeea5SJeff Kirsher * 3308874aeea5SJeff Kirsher * PCI interface 3309874aeea5SJeff Kirsher * 3310874aeea5SJeff Kirsher **************************************************************************/ 3311874aeea5SJeff Kirsher 3312874aeea5SJeff Kirsher /* Main body of final NIC shutdown code 3313874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 3314874aeea5SJeff Kirsher */ 3315874aeea5SJeff Kirsher static void efx_pci_remove_main(struct efx_nic *efx) 3316874aeea5SJeff Kirsher { 33177153f623SBen Hutchings /* Flush reset_work. It can no longer be scheduled since we 33187153f623SBen Hutchings * are not READY. 33197153f623SBen Hutchings */ 33207153f623SBen Hutchings BUG_ON(efx->state == STATE_READY); 33217153f623SBen Hutchings cancel_work_sync(&efx->reset_work); 33227153f623SBen Hutchings 3323d8291187SBen Hutchings efx_disable_interrupts(efx); 3324a83762d9SBert Kenward efx_clear_interrupt_affinity(efx); 3325874aeea5SJeff Kirsher efx_nic_fini_interrupt(efx); 3326874aeea5SJeff Kirsher efx_fini_port(efx); 3327874aeea5SJeff Kirsher efx->type->fini(efx); 3328874aeea5SJeff Kirsher efx_fini_napi(efx); 3329874aeea5SJeff Kirsher efx_remove_all(efx); 3330874aeea5SJeff Kirsher } 3331874aeea5SJeff Kirsher 3332874aeea5SJeff Kirsher /* Final NIC shutdown 33332a3fc311SDaniel Pieczko * This is called only at module unload (or hotplug removal). A PF can call 33342a3fc311SDaniel Pieczko * this on its VFs to ensure they are unbound first. 3335874aeea5SJeff Kirsher */ 3336874aeea5SJeff Kirsher static void efx_pci_remove(struct pci_dev *pci_dev) 3337874aeea5SJeff Kirsher { 3338874aeea5SJeff Kirsher struct efx_nic *efx; 3339874aeea5SJeff Kirsher 3340874aeea5SJeff Kirsher efx = pci_get_drvdata(pci_dev); 3341874aeea5SJeff Kirsher if (!efx) 3342874aeea5SJeff Kirsher return; 3343874aeea5SJeff Kirsher 3344874aeea5SJeff Kirsher /* Mark the NIC as fini, then stop the interface */ 3345874aeea5SJeff Kirsher rtnl_lock(); 33460bcf4a64SBen Hutchings efx_dissociate(efx); 3347874aeea5SJeff Kirsher dev_close(efx->net_dev); 3348d8291187SBen Hutchings efx_disable_interrupts(efx); 3349ea6bb99eSEdward Cree efx->state = STATE_UNINIT; 3350874aeea5SJeff Kirsher rtnl_unlock(); 3351874aeea5SJeff Kirsher 33527fa8d547SShradha Shah if (efx->type->sriov_fini) 3353d98a4ffeSShradha Shah efx->type->sriov_fini(efx); 33547fa8d547SShradha Shah 3355874aeea5SJeff Kirsher efx_unregister_netdev(efx); 3356874aeea5SJeff Kirsher 3357874aeea5SJeff Kirsher efx_mtd_remove(efx); 3358874aeea5SJeff Kirsher 3359874aeea5SJeff Kirsher efx_pci_remove_main(efx); 3360874aeea5SJeff Kirsher 3361874aeea5SJeff Kirsher efx_fini_io(efx); 3362874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3363874aeea5SJeff Kirsher 3364874aeea5SJeff Kirsher efx_fini_struct(efx); 3365874aeea5SJeff Kirsher free_netdev(efx->net_dev); 3366626950dbSAlexandre Rames 3367626950dbSAlexandre Rames pci_disable_pcie_error_reporting(pci_dev); 3368874aeea5SJeff Kirsher }; 3369874aeea5SJeff Kirsher 3370460eeaa0SBen Hutchings /* NIC VPD information 3371460eeaa0SBen Hutchings * Called during probe to display the part number of the 3372460eeaa0SBen Hutchings * installed NIC. VPD is potentially very large but this should 3373460eeaa0SBen Hutchings * always appear within the first 512 bytes. 3374460eeaa0SBen Hutchings */ 3375460eeaa0SBen Hutchings #define SFC_VPD_LEN 512 3376ef215e64SBen Hutchings static void efx_probe_vpd_strings(struct efx_nic *efx) 3377460eeaa0SBen Hutchings { 3378460eeaa0SBen Hutchings struct pci_dev *dev = efx->pci_dev; 3379460eeaa0SBen Hutchings char vpd_data[SFC_VPD_LEN]; 3380460eeaa0SBen Hutchings ssize_t vpd_size; 3381ef215e64SBen Hutchings int ro_start, ro_size, i, j; 3382460eeaa0SBen Hutchings 3383460eeaa0SBen Hutchings /* Get the vpd data from the device */ 3384460eeaa0SBen Hutchings vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3385460eeaa0SBen Hutchings if (vpd_size <= 0) { 3386460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3387460eeaa0SBen Hutchings return; 3388460eeaa0SBen Hutchings } 3389460eeaa0SBen Hutchings 3390460eeaa0SBen Hutchings /* Get the Read only section */ 3391ef215e64SBen Hutchings ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3392ef215e64SBen Hutchings if (ro_start < 0) { 3393460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3394460eeaa0SBen Hutchings return; 3395460eeaa0SBen Hutchings } 3396460eeaa0SBen Hutchings 3397ef215e64SBen Hutchings ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3398ef215e64SBen Hutchings j = ro_size; 3399ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3400460eeaa0SBen Hutchings if (i + j > vpd_size) 3401460eeaa0SBen Hutchings j = vpd_size - i; 3402460eeaa0SBen Hutchings 3403460eeaa0SBen Hutchings /* Get the Part number */ 3404460eeaa0SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3405460eeaa0SBen Hutchings if (i < 0) { 3406460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3407460eeaa0SBen Hutchings return; 3408460eeaa0SBen Hutchings } 3409460eeaa0SBen Hutchings 3410460eeaa0SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 3411460eeaa0SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 3412460eeaa0SBen Hutchings if (i + j > vpd_size) { 3413460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3414460eeaa0SBen Hutchings return; 3415460eeaa0SBen Hutchings } 3416460eeaa0SBen Hutchings 3417460eeaa0SBen Hutchings netif_info(efx, drv, efx->net_dev, 3418460eeaa0SBen Hutchings "Part Number : %.*s\n", j, &vpd_data[i]); 3419ef215e64SBen Hutchings 3420ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3421ef215e64SBen Hutchings j = ro_size; 3422ef215e64SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3423ef215e64SBen Hutchings if (i < 0) { 3424ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3425ef215e64SBen Hutchings return; 3426ef215e64SBen Hutchings } 3427ef215e64SBen Hutchings 3428ef215e64SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 3429ef215e64SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 3430ef215e64SBen Hutchings if (i + j > vpd_size) { 3431ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3432ef215e64SBen Hutchings return; 3433ef215e64SBen Hutchings } 3434ef215e64SBen Hutchings 3435ef215e64SBen Hutchings efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3436ef215e64SBen Hutchings if (!efx->vpd_sn) 3437ef215e64SBen Hutchings return; 3438ef215e64SBen Hutchings 3439ef215e64SBen Hutchings snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3440460eeaa0SBen Hutchings } 3441460eeaa0SBen Hutchings 3442460eeaa0SBen Hutchings 3443874aeea5SJeff Kirsher /* Main body of NIC initialisation 3444874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, theoretically). 3445874aeea5SJeff Kirsher */ 3446874aeea5SJeff Kirsher static int efx_pci_probe_main(struct efx_nic *efx) 3447874aeea5SJeff Kirsher { 3448874aeea5SJeff Kirsher int rc; 3449874aeea5SJeff Kirsher 3450874aeea5SJeff Kirsher /* Do start-of-day initialisation */ 3451874aeea5SJeff Kirsher rc = efx_probe_all(efx); 3452874aeea5SJeff Kirsher if (rc) 3453874aeea5SJeff Kirsher goto fail1; 3454874aeea5SJeff Kirsher 3455874aeea5SJeff Kirsher efx_init_napi(efx); 3456874aeea5SJeff Kirsher 3457193f2003SBert Kenward down_write(&efx->filter_sem); 3458874aeea5SJeff Kirsher rc = efx->type->init(efx); 3459193f2003SBert Kenward up_write(&efx->filter_sem); 3460874aeea5SJeff Kirsher if (rc) { 3461874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 3462874aeea5SJeff Kirsher "failed to initialise NIC\n"); 3463874aeea5SJeff Kirsher goto fail3; 3464874aeea5SJeff Kirsher } 3465874aeea5SJeff Kirsher 3466874aeea5SJeff Kirsher rc = efx_init_port(efx); 3467874aeea5SJeff Kirsher if (rc) { 3468874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 3469874aeea5SJeff Kirsher "failed to initialise port\n"); 3470874aeea5SJeff Kirsher goto fail4; 3471874aeea5SJeff Kirsher } 3472874aeea5SJeff Kirsher 3473874aeea5SJeff Kirsher rc = efx_nic_init_interrupt(efx); 3474874aeea5SJeff Kirsher if (rc) 3475874aeea5SJeff Kirsher goto fail5; 3476a83762d9SBert Kenward 3477a83762d9SBert Kenward efx_set_interrupt_affinity(efx); 3478261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3479261e4d96SJon Cooper if (rc) 3480261e4d96SJon Cooper goto fail6; 3481874aeea5SJeff Kirsher 3482874aeea5SJeff Kirsher return 0; 3483874aeea5SJeff Kirsher 3484261e4d96SJon Cooper fail6: 3485a83762d9SBert Kenward efx_clear_interrupt_affinity(efx); 3486261e4d96SJon Cooper efx_nic_fini_interrupt(efx); 3487874aeea5SJeff Kirsher fail5: 3488874aeea5SJeff Kirsher efx_fini_port(efx); 3489874aeea5SJeff Kirsher fail4: 3490874aeea5SJeff Kirsher efx->type->fini(efx); 3491874aeea5SJeff Kirsher fail3: 3492874aeea5SJeff Kirsher efx_fini_napi(efx); 3493874aeea5SJeff Kirsher efx_remove_all(efx); 3494874aeea5SJeff Kirsher fail1: 3495874aeea5SJeff Kirsher return rc; 3496874aeea5SJeff Kirsher } 3497874aeea5SJeff Kirsher 34988a531400SJon Cooper static int efx_pci_probe_post_io(struct efx_nic *efx) 34998a531400SJon Cooper { 35008a531400SJon Cooper struct net_device *net_dev = efx->net_dev; 35018a531400SJon Cooper int rc = efx_pci_probe_main(efx); 35028a531400SJon Cooper 35038a531400SJon Cooper if (rc) 35048a531400SJon Cooper return rc; 35058a531400SJon Cooper 35068a531400SJon Cooper if (efx->type->sriov_init) { 35078a531400SJon Cooper rc = efx->type->sriov_init(efx); 35088a531400SJon Cooper if (rc) 35098a531400SJon Cooper netif_err(efx, probe, efx->net_dev, 35108a531400SJon Cooper "SR-IOV can't be enabled rc %d\n", rc); 35118a531400SJon Cooper } 35128a531400SJon Cooper 35138a531400SJon Cooper /* Determine netdevice features */ 35148a531400SJon Cooper net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 35156978729fSEdward Cree NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL); 35168a531400SJon Cooper if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 35178a531400SJon Cooper net_dev->features |= NETIF_F_TSO6; 35188a531400SJon Cooper /* Check whether device supports TSO */ 35198a531400SJon Cooper if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 35208a531400SJon Cooper net_dev->features &= ~NETIF_F_ALL_TSO; 35218a531400SJon Cooper /* Mask for features that also apply to VLAN devices */ 35228a531400SJon Cooper net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 35238a531400SJon Cooper NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 35248a531400SJon Cooper NETIF_F_RXCSUM); 35258a531400SJon Cooper 35266978729fSEdward Cree net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 35276978729fSEdward Cree 35286978729fSEdward Cree /* Disable receiving frames with bad FCS, by default. */ 35296978729fSEdward Cree net_dev->features &= ~NETIF_F_RXALL; 35308a531400SJon Cooper 35318a531400SJon Cooper /* Disable VLAN filtering by default. It may be enforced if 35328a531400SJon Cooper * the feature is fixed (i.e. VLAN filters are required to 35338a531400SJon Cooper * receive VLAN tagged packets due to vPort restrictions). 35348a531400SJon Cooper */ 35358a531400SJon Cooper net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 35368a531400SJon Cooper net_dev->features |= efx->fixed_features; 35378a531400SJon Cooper 35388a531400SJon Cooper rc = efx_register_netdev(efx); 35398a531400SJon Cooper if (!rc) 35408a531400SJon Cooper return 0; 35418a531400SJon Cooper 35428a531400SJon Cooper efx_pci_remove_main(efx); 35438a531400SJon Cooper return rc; 35448a531400SJon Cooper } 35458a531400SJon Cooper 3546874aeea5SJeff Kirsher /* NIC initialisation 3547874aeea5SJeff Kirsher * 3548874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, 354973ba7b68SBen Hutchings * theoretically). It sets up PCI mappings, resets the NIC, 3550874aeea5SJeff Kirsher * sets up and registers the network devices with the kernel and hooks 3551874aeea5SJeff Kirsher * the interrupt service routine. It does not prepare the device for 3552874aeea5SJeff Kirsher * transmission; this is left to the first time one of the network 3553874aeea5SJeff Kirsher * interfaces is brought up (i.e. efx_net_open). 3554874aeea5SJeff Kirsher */ 355587d1fc11SBill Pemberton static int efx_pci_probe(struct pci_dev *pci_dev, 3556874aeea5SJeff Kirsher const struct pci_device_id *entry) 3557874aeea5SJeff Kirsher { 3558874aeea5SJeff Kirsher struct net_device *net_dev; 3559874aeea5SJeff Kirsher struct efx_nic *efx; 3560fadac6aaSBen Hutchings int rc; 3561874aeea5SJeff Kirsher 3562874aeea5SJeff Kirsher /* Allocate and initialise a struct net_device and struct efx_nic */ 3563874aeea5SJeff Kirsher net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3564874aeea5SJeff Kirsher EFX_MAX_RX_QUEUES); 3565874aeea5SJeff Kirsher if (!net_dev) 3566874aeea5SJeff Kirsher return -ENOMEM; 3567adeb15aaSBen Hutchings efx = netdev_priv(net_dev); 3568adeb15aaSBen Hutchings efx->type = (const struct efx_nic_type *) entry->driver_data; 3569ebfcd0fdSAndrew Rybchenko efx->fixed_features |= NETIF_F_HIGHDMA; 3570eb7cfd8cSAndrew Rybchenko 3571874aeea5SJeff Kirsher pci_set_drvdata(pci_dev, efx); 3572874aeea5SJeff Kirsher SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3573adeb15aaSBen Hutchings rc = efx_init_struct(efx, pci_dev, net_dev); 3574874aeea5SJeff Kirsher if (rc) 3575874aeea5SJeff Kirsher goto fail1; 3576874aeea5SJeff Kirsher 3577874aeea5SJeff Kirsher netif_info(efx, probe, efx->net_dev, 3578874aeea5SJeff Kirsher "Solarflare NIC detected\n"); 3579874aeea5SJeff Kirsher 35806f7f8aa6SShradha Shah if (!efx->type->is_vf) 3581ef215e64SBen Hutchings efx_probe_vpd_strings(efx); 3582460eeaa0SBen Hutchings 3583874aeea5SJeff Kirsher /* Set up basic I/O (BAR mappings etc) */ 3584874aeea5SJeff Kirsher rc = efx_init_io(efx); 3585874aeea5SJeff Kirsher if (rc) 3586874aeea5SJeff Kirsher goto fail2; 3587874aeea5SJeff Kirsher 35888a531400SJon Cooper rc = efx_pci_probe_post_io(efx); 35898a531400SJon Cooper if (rc) { 35908a531400SJon Cooper /* On failure, retry once immediately. 35918a531400SJon Cooper * If we aborted probe due to a scheduled reset, dismiss it. 35928a531400SJon Cooper */ 35938a531400SJon Cooper efx->reset_pending = 0; 35948a531400SJon Cooper rc = efx_pci_probe_post_io(efx); 35958a531400SJon Cooper if (rc) { 35968a531400SJon Cooper /* On another failure, retry once more 35978a531400SJon Cooper * after a 50-305ms delay. 35988a531400SJon Cooper */ 35998a531400SJon Cooper unsigned char r; 36008a531400SJon Cooper 36018a531400SJon Cooper get_random_bytes(&r, 1); 36028a531400SJon Cooper msleep((unsigned int)r + 50); 36038a531400SJon Cooper efx->reset_pending = 0; 36048a531400SJon Cooper rc = efx_pci_probe_post_io(efx); 36058a531400SJon Cooper } 36068a531400SJon Cooper } 3607fadac6aaSBen Hutchings if (rc) 3608874aeea5SJeff Kirsher goto fail3; 3609874aeea5SJeff Kirsher 3610874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3611874aeea5SJeff Kirsher 36127c43161cSBen Hutchings /* Try to create MTDs, but allow this to fail */ 3613874aeea5SJeff Kirsher rtnl_lock(); 36147c43161cSBen Hutchings rc = efx_mtd_probe(efx); 3615874aeea5SJeff Kirsher rtnl_unlock(); 361609a04204SBert Kenward if (rc && rc != -EPERM) 36177c43161cSBen Hutchings netif_warn(efx, probe, efx->net_dev, 36187c43161cSBen Hutchings "failed to create MTDs (%d)\n", rc); 36197c43161cSBen Hutchings 3620626950dbSAlexandre Rames rc = pci_enable_pcie_error_reporting(pci_dev); 3621626950dbSAlexandre Rames if (rc && rc != -EINVAL) 362209a04204SBert Kenward netif_notice(efx, probe, efx->net_dev, 362309a04204SBert Kenward "PCIE error reporting unavailable (%d).\n", 362409a04204SBert Kenward rc); 3625626950dbSAlexandre Rames 3626e5fbd977SJon Cooper if (efx->type->udp_tnl_push_ports) 3627e5fbd977SJon Cooper efx->type->udp_tnl_push_ports(efx); 3628e5fbd977SJon Cooper 3629874aeea5SJeff Kirsher return 0; 3630874aeea5SJeff Kirsher 3631874aeea5SJeff Kirsher fail3: 3632874aeea5SJeff Kirsher efx_fini_io(efx); 3633874aeea5SJeff Kirsher fail2: 3634874aeea5SJeff Kirsher efx_fini_struct(efx); 3635874aeea5SJeff Kirsher fail1: 3636874aeea5SJeff Kirsher WARN_ON(rc > 0); 3637874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3638874aeea5SJeff Kirsher free_netdev(net_dev); 3639874aeea5SJeff Kirsher return rc; 3640874aeea5SJeff Kirsher } 3641874aeea5SJeff Kirsher 3642834e23ddSShradha Shah /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3643834e23ddSShradha Shah * enabled on success 3644834e23ddSShradha Shah */ 3645834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV 3646834e23ddSShradha Shah static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3647834e23ddSShradha Shah { 3648834e23ddSShradha Shah int rc; 3649834e23ddSShradha Shah struct efx_nic *efx = pci_get_drvdata(dev); 3650834e23ddSShradha Shah 3651834e23ddSShradha Shah if (efx->type->sriov_configure) { 3652834e23ddSShradha Shah rc = efx->type->sriov_configure(efx, num_vfs); 3653834e23ddSShradha Shah if (rc) 3654834e23ddSShradha Shah return rc; 3655834e23ddSShradha Shah else 3656834e23ddSShradha Shah return num_vfs; 3657834e23ddSShradha Shah } else 3658834e23ddSShradha Shah return -EOPNOTSUPP; 3659834e23ddSShradha Shah } 3660834e23ddSShradha Shah #endif 3661834e23ddSShradha Shah 3662874aeea5SJeff Kirsher static int efx_pm_freeze(struct device *dev) 3663874aeea5SJeff Kirsher { 3664874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3665874aeea5SJeff Kirsher 366661da026dSBen Hutchings rtnl_lock(); 366761da026dSBen Hutchings 36686032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3669f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 3670874aeea5SJeff Kirsher 3671c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 3672874aeea5SJeff Kirsher 3673874aeea5SJeff Kirsher efx_stop_all(efx); 3674d8291187SBen Hutchings efx_disable_interrupts(efx); 36756032fb56SBen Hutchings } 3676874aeea5SJeff Kirsher 367761da026dSBen Hutchings rtnl_unlock(); 367861da026dSBen Hutchings 3679874aeea5SJeff Kirsher return 0; 3680874aeea5SJeff Kirsher } 3681874aeea5SJeff Kirsher 3682874aeea5SJeff Kirsher static int efx_pm_thaw(struct device *dev) 3683874aeea5SJeff Kirsher { 3684261e4d96SJon Cooper int rc; 3685874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3686874aeea5SJeff Kirsher 368761da026dSBen Hutchings rtnl_lock(); 368861da026dSBen Hutchings 36896032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3690261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3691261e4d96SJon Cooper if (rc) 3692261e4d96SJon Cooper goto fail; 3693874aeea5SJeff Kirsher 3694874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 3695874aeea5SJeff Kirsher efx->phy_op->reconfigure(efx); 3696874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 3697874aeea5SJeff Kirsher 3698874aeea5SJeff Kirsher efx_start_all(efx); 3699874aeea5SJeff Kirsher 37009c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 3701874aeea5SJeff Kirsher 3702f16aeea0SBen Hutchings efx->state = STATE_READY; 3703874aeea5SJeff Kirsher 3704874aeea5SJeff Kirsher efx->type->resume_wol(efx); 37056032fb56SBen Hutchings } 3706874aeea5SJeff Kirsher 370761da026dSBen Hutchings rtnl_unlock(); 370861da026dSBen Hutchings 3709874aeea5SJeff Kirsher /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3710874aeea5SJeff Kirsher queue_work(reset_workqueue, &efx->reset_work); 3711874aeea5SJeff Kirsher 3712874aeea5SJeff Kirsher return 0; 3713261e4d96SJon Cooper 3714261e4d96SJon Cooper fail: 3715261e4d96SJon Cooper rtnl_unlock(); 3716261e4d96SJon Cooper 3717261e4d96SJon Cooper return rc; 3718874aeea5SJeff Kirsher } 3719874aeea5SJeff Kirsher 3720874aeea5SJeff Kirsher static int efx_pm_poweroff(struct device *dev) 3721874aeea5SJeff Kirsher { 3722874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3723874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3724874aeea5SJeff Kirsher 3725874aeea5SJeff Kirsher efx->type->fini(efx); 3726874aeea5SJeff Kirsher 3727874aeea5SJeff Kirsher efx->reset_pending = 0; 3728874aeea5SJeff Kirsher 3729874aeea5SJeff Kirsher pci_save_state(pci_dev); 3730874aeea5SJeff Kirsher return pci_set_power_state(pci_dev, PCI_D3hot); 3731874aeea5SJeff Kirsher } 3732874aeea5SJeff Kirsher 3733874aeea5SJeff Kirsher /* Used for both resume and restore */ 3734874aeea5SJeff Kirsher static int efx_pm_resume(struct device *dev) 3735874aeea5SJeff Kirsher { 3736874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3737874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3738874aeea5SJeff Kirsher int rc; 3739874aeea5SJeff Kirsher 3740874aeea5SJeff Kirsher rc = pci_set_power_state(pci_dev, PCI_D0); 3741874aeea5SJeff Kirsher if (rc) 3742874aeea5SJeff Kirsher return rc; 3743874aeea5SJeff Kirsher pci_restore_state(pci_dev); 3744874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 3745874aeea5SJeff Kirsher if (rc) 3746874aeea5SJeff Kirsher return rc; 3747874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 3748874aeea5SJeff Kirsher rc = efx->type->reset(efx, RESET_TYPE_ALL); 3749874aeea5SJeff Kirsher if (rc) 3750874aeea5SJeff Kirsher return rc; 3751193f2003SBert Kenward down_write(&efx->filter_sem); 3752874aeea5SJeff Kirsher rc = efx->type->init(efx); 3753193f2003SBert Kenward up_write(&efx->filter_sem); 3754874aeea5SJeff Kirsher if (rc) 3755874aeea5SJeff Kirsher return rc; 3756261e4d96SJon Cooper rc = efx_pm_thaw(dev); 3757261e4d96SJon Cooper return rc; 3758874aeea5SJeff Kirsher } 3759874aeea5SJeff Kirsher 3760874aeea5SJeff Kirsher static int efx_pm_suspend(struct device *dev) 3761874aeea5SJeff Kirsher { 3762874aeea5SJeff Kirsher int rc; 3763874aeea5SJeff Kirsher 3764874aeea5SJeff Kirsher efx_pm_freeze(dev); 3765874aeea5SJeff Kirsher rc = efx_pm_poweroff(dev); 3766874aeea5SJeff Kirsher if (rc) 3767874aeea5SJeff Kirsher efx_pm_resume(dev); 3768874aeea5SJeff Kirsher return rc; 3769874aeea5SJeff Kirsher } 3770874aeea5SJeff Kirsher 377118e83e4cSBen Hutchings static const struct dev_pm_ops efx_pm_ops = { 3772874aeea5SJeff Kirsher .suspend = efx_pm_suspend, 3773874aeea5SJeff Kirsher .resume = efx_pm_resume, 3774874aeea5SJeff Kirsher .freeze = efx_pm_freeze, 3775874aeea5SJeff Kirsher .thaw = efx_pm_thaw, 3776874aeea5SJeff Kirsher .poweroff = efx_pm_poweroff, 3777874aeea5SJeff Kirsher .restore = efx_pm_resume, 3778874aeea5SJeff Kirsher }; 3779874aeea5SJeff Kirsher 3780626950dbSAlexandre Rames /* A PCI error affecting this device was detected. 3781626950dbSAlexandre Rames * At this point MMIO and DMA may be disabled. 3782626950dbSAlexandre Rames * Stop the software path and request a slot reset. 3783626950dbSAlexandre Rames */ 3784debd0034Sstephen hemminger static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3785626950dbSAlexandre Rames enum pci_channel_state state) 3786626950dbSAlexandre Rames { 3787626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3788626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3789626950dbSAlexandre Rames 3790626950dbSAlexandre Rames if (state == pci_channel_io_perm_failure) 3791626950dbSAlexandre Rames return PCI_ERS_RESULT_DISCONNECT; 3792626950dbSAlexandre Rames 3793626950dbSAlexandre Rames rtnl_lock(); 3794626950dbSAlexandre Rames 3795626950dbSAlexandre Rames if (efx->state != STATE_DISABLED) { 3796626950dbSAlexandre Rames efx->state = STATE_RECOVERY; 3797626950dbSAlexandre Rames efx->reset_pending = 0; 3798626950dbSAlexandre Rames 3799626950dbSAlexandre Rames efx_device_detach_sync(efx); 3800626950dbSAlexandre Rames 3801626950dbSAlexandre Rames efx_stop_all(efx); 3802d8291187SBen Hutchings efx_disable_interrupts(efx); 3803626950dbSAlexandre Rames 3804626950dbSAlexandre Rames status = PCI_ERS_RESULT_NEED_RESET; 3805626950dbSAlexandre Rames } else { 3806626950dbSAlexandre Rames /* If the interface is disabled we don't want to do anything 3807626950dbSAlexandre Rames * with it. 3808626950dbSAlexandre Rames */ 3809626950dbSAlexandre Rames status = PCI_ERS_RESULT_RECOVERED; 3810626950dbSAlexandre Rames } 3811626950dbSAlexandre Rames 3812626950dbSAlexandre Rames rtnl_unlock(); 3813626950dbSAlexandre Rames 3814626950dbSAlexandre Rames pci_disable_device(pdev); 3815626950dbSAlexandre Rames 3816626950dbSAlexandre Rames return status; 3817626950dbSAlexandre Rames } 3818626950dbSAlexandre Rames 3819dbedd44eSJoe Perches /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3820debd0034Sstephen hemminger static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3821626950dbSAlexandre Rames { 3822626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3823626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3824626950dbSAlexandre Rames 3825626950dbSAlexandre Rames if (pci_enable_device(pdev)) { 3826626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3827626950dbSAlexandre Rames "Cannot re-enable PCI device after reset.\n"); 3828626950dbSAlexandre Rames status = PCI_ERS_RESULT_DISCONNECT; 3829626950dbSAlexandre Rames } 3830626950dbSAlexandre Rames 3831626950dbSAlexandre Rames return status; 3832626950dbSAlexandre Rames } 3833626950dbSAlexandre Rames 3834626950dbSAlexandre Rames /* Perform the actual reset and resume I/O operations. */ 3835626950dbSAlexandre Rames static void efx_io_resume(struct pci_dev *pdev) 3836626950dbSAlexandre Rames { 3837626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3838626950dbSAlexandre Rames int rc; 3839626950dbSAlexandre Rames 3840626950dbSAlexandre Rames rtnl_lock(); 3841626950dbSAlexandre Rames 3842626950dbSAlexandre Rames if (efx->state == STATE_DISABLED) 3843626950dbSAlexandre Rames goto out; 3844626950dbSAlexandre Rames 3845626950dbSAlexandre Rames rc = efx_reset(efx, RESET_TYPE_ALL); 3846626950dbSAlexandre Rames if (rc) { 3847626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3848626950dbSAlexandre Rames "efx_reset failed after PCI error (%d)\n", rc); 3849626950dbSAlexandre Rames } else { 3850626950dbSAlexandre Rames efx->state = STATE_READY; 3851626950dbSAlexandre Rames netif_dbg(efx, hw, efx->net_dev, 3852626950dbSAlexandre Rames "Done resetting and resuming IO after PCI error.\n"); 3853626950dbSAlexandre Rames } 3854626950dbSAlexandre Rames 3855626950dbSAlexandre Rames out: 3856626950dbSAlexandre Rames rtnl_unlock(); 3857626950dbSAlexandre Rames } 3858626950dbSAlexandre Rames 3859626950dbSAlexandre Rames /* For simplicity and reliability, we always require a slot reset and try to 3860626950dbSAlexandre Rames * reset the hardware when a pci error affecting the device is detected. 3861626950dbSAlexandre Rames * We leave both the link_reset and mmio_enabled callback unimplemented: 3862626950dbSAlexandre Rames * with our request for slot reset the mmio_enabled callback will never be 3863626950dbSAlexandre Rames * called, and the link_reset callback is not used by AER or EEH mechanisms. 3864626950dbSAlexandre Rames */ 3865c300366bSJulia Lawall static const struct pci_error_handlers efx_err_handlers = { 3866626950dbSAlexandre Rames .error_detected = efx_io_error_detected, 3867626950dbSAlexandre Rames .slot_reset = efx_io_slot_reset, 3868626950dbSAlexandre Rames .resume = efx_io_resume, 3869626950dbSAlexandre Rames }; 3870626950dbSAlexandre Rames 3871874aeea5SJeff Kirsher static struct pci_driver efx_pci_driver = { 3872874aeea5SJeff Kirsher .name = KBUILD_MODNAME, 3873874aeea5SJeff Kirsher .id_table = efx_pci_table, 3874874aeea5SJeff Kirsher .probe = efx_pci_probe, 3875874aeea5SJeff Kirsher .remove = efx_pci_remove, 3876874aeea5SJeff Kirsher .driver.pm = &efx_pm_ops, 3877626950dbSAlexandre Rames .err_handler = &efx_err_handlers, 3878834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV 3879834e23ddSShradha Shah .sriov_configure = efx_pci_sriov_configure, 3880834e23ddSShradha Shah #endif 3881874aeea5SJeff Kirsher }; 3882874aeea5SJeff Kirsher 3883874aeea5SJeff Kirsher /************************************************************************** 3884874aeea5SJeff Kirsher * 3885874aeea5SJeff Kirsher * Kernel module interface 3886874aeea5SJeff Kirsher * 3887874aeea5SJeff Kirsher *************************************************************************/ 3888874aeea5SJeff Kirsher 3889874aeea5SJeff Kirsher module_param(interrupt_mode, uint, 0444); 3890874aeea5SJeff Kirsher MODULE_PARM_DESC(interrupt_mode, 3891874aeea5SJeff Kirsher "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3892874aeea5SJeff Kirsher 3893874aeea5SJeff Kirsher static int __init efx_init_module(void) 3894874aeea5SJeff Kirsher { 3895874aeea5SJeff Kirsher int rc; 3896874aeea5SJeff Kirsher 3897874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3898874aeea5SJeff Kirsher 3899874aeea5SJeff Kirsher rc = register_netdevice_notifier(&efx_netdev_notifier); 3900874aeea5SJeff Kirsher if (rc) 3901874aeea5SJeff Kirsher goto err_notifier; 3902874aeea5SJeff Kirsher 39037fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 3904cd2d5b52SBen Hutchings rc = efx_init_sriov(); 3905cd2d5b52SBen Hutchings if (rc) 3906cd2d5b52SBen Hutchings goto err_sriov; 39077fa8d547SShradha Shah #endif 3908cd2d5b52SBen Hutchings 3909874aeea5SJeff Kirsher reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3910874aeea5SJeff Kirsher if (!reset_workqueue) { 3911874aeea5SJeff Kirsher rc = -ENOMEM; 3912874aeea5SJeff Kirsher goto err_reset; 3913874aeea5SJeff Kirsher } 3914874aeea5SJeff Kirsher 3915874aeea5SJeff Kirsher rc = pci_register_driver(&efx_pci_driver); 3916874aeea5SJeff Kirsher if (rc < 0) 3917874aeea5SJeff Kirsher goto err_pci; 3918874aeea5SJeff Kirsher 3919874aeea5SJeff Kirsher return 0; 3920874aeea5SJeff Kirsher 3921874aeea5SJeff Kirsher err_pci: 3922874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 3923874aeea5SJeff Kirsher err_reset: 39247fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 3925cd2d5b52SBen Hutchings efx_fini_sriov(); 3926cd2d5b52SBen Hutchings err_sriov: 39277fa8d547SShradha Shah #endif 3928874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3929874aeea5SJeff Kirsher err_notifier: 3930874aeea5SJeff Kirsher return rc; 3931874aeea5SJeff Kirsher } 3932874aeea5SJeff Kirsher 3933874aeea5SJeff Kirsher static void __exit efx_exit_module(void) 3934874aeea5SJeff Kirsher { 3935874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver unloading\n"); 3936874aeea5SJeff Kirsher 3937874aeea5SJeff Kirsher pci_unregister_driver(&efx_pci_driver); 3938874aeea5SJeff Kirsher destroy_workqueue(reset_workqueue); 39397fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 3940cd2d5b52SBen Hutchings efx_fini_sriov(); 39417fa8d547SShradha Shah #endif 3942874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 3943874aeea5SJeff Kirsher 3944874aeea5SJeff Kirsher } 3945874aeea5SJeff Kirsher 3946874aeea5SJeff Kirsher module_init(efx_init_module); 3947874aeea5SJeff Kirsher module_exit(efx_exit_module); 3948874aeea5SJeff Kirsher 3949874aeea5SJeff Kirsher MODULE_AUTHOR("Solarflare Communications and " 3950874aeea5SJeff Kirsher "Michael Brown <mbrown@fensystems.co.uk>"); 39516a350fdbSBen Hutchings MODULE_DESCRIPTION("Solarflare network driver"); 3952874aeea5SJeff Kirsher MODULE_LICENSE("GPL"); 3953874aeea5SJeff Kirsher MODULE_DEVICE_TABLE(pci, efx_pci_table); 395414077e9eSEdward Cree MODULE_VERSION(EFX_DRIVER_VERSION); 3955