1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2874aeea5SJeff Kirsher /**************************************************************************** 3f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 4874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 5f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 6874aeea5SJeff Kirsher */ 7874aeea5SJeff Kirsher 8874aeea5SJeff Kirsher #include <linux/module.h> 9874aeea5SJeff Kirsher #include <linux/pci.h> 10874aeea5SJeff Kirsher #include <linux/netdevice.h> 11874aeea5SJeff Kirsher #include <linux/etherdevice.h> 12874aeea5SJeff Kirsher #include <linux/delay.h> 13874aeea5SJeff Kirsher #include <linux/notifier.h> 14874aeea5SJeff Kirsher #include <linux/ip.h> 15874aeea5SJeff Kirsher #include <linux/tcp.h> 16874aeea5SJeff Kirsher #include <linux/in.h> 17874aeea5SJeff Kirsher #include <linux/ethtool.h> 18874aeea5SJeff Kirsher #include <linux/topology.h> 19874aeea5SJeff Kirsher #include <linux/gfp.h> 20626950dbSAlexandre Rames #include <linux/aer.h> 21b28405b0SAlexandre Rames #include <linux/interrupt.h> 22874aeea5SJeff Kirsher #include "net_driver.h" 23e5fbd977SJon Cooper #include <net/gre.h> 24e5fbd977SJon Cooper #include <net/udp_tunnel.h> 25874aeea5SJeff Kirsher #include "efx.h" 26e1253f39SAlex Maftei (amaftei) #include "efx_common.h" 27e1253f39SAlex Maftei (amaftei) #include "efx_channels.h" 28e1253f39SAlex Maftei (amaftei) #include "rx_common.h" 29e1253f39SAlex Maftei (amaftei) #include "tx_common.h" 30874aeea5SJeff Kirsher #include "nic.h" 3171827443SEdward Cree #include "io.h" 32dd40781eSBen Hutchings #include "selftest.h" 337fa8d547SShradha Shah #include "sriov.h" 34874aeea5SJeff Kirsher 35874aeea5SJeff Kirsher #include "mcdi.h" 36e5fbd977SJon Cooper #include "mcdi_pcol.h" 37874aeea5SJeff Kirsher #include "workarounds.h" 38874aeea5SJeff Kirsher 39874aeea5SJeff Kirsher /************************************************************************** 40874aeea5SJeff Kirsher * 41874aeea5SJeff Kirsher * Type name strings 42874aeea5SJeff Kirsher * 43874aeea5SJeff Kirsher ************************************************************************** 44874aeea5SJeff Kirsher */ 45874aeea5SJeff Kirsher 46874aeea5SJeff Kirsher /* Loopback mode names (see LOOPBACK_MODE()) */ 47874aeea5SJeff Kirsher const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 4818e83e4cSBen Hutchings const char *const efx_loopback_mode_names[] = { 49874aeea5SJeff Kirsher [LOOPBACK_NONE] = "NONE", 50874aeea5SJeff Kirsher [LOOPBACK_DATA] = "DATAPATH", 51874aeea5SJeff Kirsher [LOOPBACK_GMAC] = "GMAC", 52874aeea5SJeff Kirsher [LOOPBACK_XGMII] = "XGMII", 53874aeea5SJeff Kirsher [LOOPBACK_XGXS] = "XGXS", 54874aeea5SJeff Kirsher [LOOPBACK_XAUI] = "XAUI", 55874aeea5SJeff Kirsher [LOOPBACK_GMII] = "GMII", 56874aeea5SJeff Kirsher [LOOPBACK_SGMII] = "SGMII", 57874aeea5SJeff Kirsher [LOOPBACK_XGBR] = "XGBR", 58874aeea5SJeff Kirsher [LOOPBACK_XFI] = "XFI", 59874aeea5SJeff Kirsher [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 60874aeea5SJeff Kirsher [LOOPBACK_GMII_FAR] = "GMII_FAR", 61874aeea5SJeff Kirsher [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 62874aeea5SJeff Kirsher [LOOPBACK_XFI_FAR] = "XFI_FAR", 63874aeea5SJeff Kirsher [LOOPBACK_GPHY] = "GPHY", 64874aeea5SJeff Kirsher [LOOPBACK_PHYXS] = "PHYXS", 65874aeea5SJeff Kirsher [LOOPBACK_PCS] = "PCS", 66874aeea5SJeff Kirsher [LOOPBACK_PMAPMD] = "PMA/PMD", 67874aeea5SJeff Kirsher [LOOPBACK_XPORT] = "XPORT", 68874aeea5SJeff Kirsher [LOOPBACK_XGMII_WS] = "XGMII_WS", 69874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS] = "XAUI_WS", 70874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 71874aeea5SJeff Kirsher [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 72874aeea5SJeff Kirsher [LOOPBACK_GMII_WS] = "GMII_WS", 73874aeea5SJeff Kirsher [LOOPBACK_XFI_WS] = "XFI_WS", 74874aeea5SJeff Kirsher [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 75874aeea5SJeff Kirsher [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 76874aeea5SJeff Kirsher }; 77874aeea5SJeff Kirsher 78874aeea5SJeff Kirsher const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 7918e83e4cSBen Hutchings const char *const efx_reset_type_names[] = { 80874aeea5SJeff Kirsher [RESET_TYPE_INVISIBLE] = "INVISIBLE", 81874aeea5SJeff Kirsher [RESET_TYPE_ALL] = "ALL", 82626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 83874aeea5SJeff Kirsher [RESET_TYPE_WORLD] = "WORLD", 84626950dbSAlexandre Rames [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 85087e9025SJon Cooper [RESET_TYPE_DATAPATH] = "DATAPATH", 86e283546cSEdward Cree [RESET_TYPE_MC_BIST] = "MC_BIST", 87874aeea5SJeff Kirsher [RESET_TYPE_DISABLE] = "DISABLE", 88874aeea5SJeff Kirsher [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 89874aeea5SJeff Kirsher [RESET_TYPE_INT_ERROR] = "INT_ERROR", 903de82b91SAlexandre Rames [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 91874aeea5SJeff Kirsher [RESET_TYPE_TX_SKIP] = "TX_SKIP", 92874aeea5SJeff Kirsher [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 93e283546cSEdward Cree [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 94874aeea5SJeff Kirsher }; 95874aeea5SJeff Kirsher 96e5fbd977SJon Cooper /* UDP tunnel type names */ 97e5fbd977SJon Cooper static const char *const efx_udp_tunnel_type_names[] = { 98e5fbd977SJon Cooper [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan", 99e5fbd977SJon Cooper [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve", 100e5fbd977SJon Cooper }; 101e5fbd977SJon Cooper 102e5fbd977SJon Cooper void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen) 103e5fbd977SJon Cooper { 104e5fbd977SJon Cooper if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) && 105e5fbd977SJon Cooper efx_udp_tunnel_type_names[type] != NULL) 106e5fbd977SJon Cooper snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]); 107e5fbd977SJon Cooper else 108e5fbd977SJon Cooper snprintf(buf, buflen, "type %d", type); 109e5fbd977SJon Cooper } 110e5fbd977SJon Cooper 11174cd60a4SJon Cooper /* How often and how many times to poll for a reset while waiting for a 11274cd60a4SJon Cooper * BIST that another function started to complete. 11374cd60a4SJon Cooper */ 11474cd60a4SJon Cooper #define BIST_WAIT_DELAY_MS 100 11574cd60a4SJon Cooper #define BIST_WAIT_DELAY_COUNT 100 11674cd60a4SJon Cooper 117874aeea5SJeff Kirsher /************************************************************************** 118874aeea5SJeff Kirsher * 119874aeea5SJeff Kirsher * Configurable values 120874aeea5SJeff Kirsher * 121874aeea5SJeff Kirsher *************************************************************************/ 122874aeea5SJeff Kirsher 123874aeea5SJeff Kirsher /* 124874aeea5SJeff Kirsher * Use separate channels for TX and RX events 125874aeea5SJeff Kirsher * 126874aeea5SJeff Kirsher * Set this to 1 to use separate channels for TX and RX. It allows us 127874aeea5SJeff Kirsher * to control interrupt affinity separately for TX and RX. 128874aeea5SJeff Kirsher * 129874aeea5SJeff Kirsher * This is only used in MSI-X interrupt mode 130874aeea5SJeff Kirsher */ 131b0fbdae1SShradha Shah bool efx_separate_tx_channels; 132b0fbdae1SShradha Shah module_param(efx_separate_tx_channels, bool, 0444); 133b0fbdae1SShradha Shah MODULE_PARM_DESC(efx_separate_tx_channels, 134874aeea5SJeff Kirsher "Use separate channels for TX and RX"); 135874aeea5SJeff Kirsher 136874aeea5SJeff Kirsher /* This is the weight assigned to each of the (per-channel) virtual 137874aeea5SJeff Kirsher * NAPI devices. 138874aeea5SJeff Kirsher */ 139874aeea5SJeff Kirsher static int napi_weight = 64; 140874aeea5SJeff Kirsher 141874aeea5SJeff Kirsher /* This is the time (in jiffies) between invocations of the hardware 142626950dbSAlexandre Rames * monitor. 143626950dbSAlexandre Rames * On Falcon-based NICs, this will: 144874aeea5SJeff Kirsher * - Check the on-board hardware monitor; 145874aeea5SJeff Kirsher * - Poll the link state and reconfigure the hardware as necessary. 146626950dbSAlexandre Rames * On Siena-based NICs for power systems with EEH support, this will give EEH a 147626950dbSAlexandre Rames * chance to start. 148874aeea5SJeff Kirsher */ 149874aeea5SJeff Kirsher static unsigned int efx_monitor_interval = 1 * HZ; 150874aeea5SJeff Kirsher 151874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 152874aeea5SJeff Kirsher * module load with ethtool. 153874aeea5SJeff Kirsher * 154874aeea5SJeff Kirsher * The default for RX should strike a balance between increasing the 155874aeea5SJeff Kirsher * round-trip latency and reducing overhead. 156874aeea5SJeff Kirsher */ 157874aeea5SJeff Kirsher static unsigned int rx_irq_mod_usec = 60; 158874aeea5SJeff Kirsher 159874aeea5SJeff Kirsher /* Initial interrupt moderation settings. They can be modified after 160874aeea5SJeff Kirsher * module load with ethtool. 161874aeea5SJeff Kirsher * 162874aeea5SJeff Kirsher * This default is chosen to ensure that a 10G link does not go idle 163874aeea5SJeff Kirsher * while a TX queue is stopped after it has become full. A queue is 164874aeea5SJeff Kirsher * restarted when it drops below half full. The time this takes (assuming 165874aeea5SJeff Kirsher * worst case 3 descriptors per packet and 1024 descriptors) is 166874aeea5SJeff Kirsher * 512 / 3 * 1.2 = 205 usec. 167874aeea5SJeff Kirsher */ 168874aeea5SJeff Kirsher static unsigned int tx_irq_mod_usec = 150; 169874aeea5SJeff Kirsher 170874aeea5SJeff Kirsher /* This is the first interrupt mode to try out of: 171874aeea5SJeff Kirsher * 0 => MSI-X 172874aeea5SJeff Kirsher * 1 => MSI 173874aeea5SJeff Kirsher * 2 => legacy 174874aeea5SJeff Kirsher */ 175874aeea5SJeff Kirsher static unsigned int interrupt_mode; 176874aeea5SJeff Kirsher 177874aeea5SJeff Kirsher /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 178874aeea5SJeff Kirsher * i.e. the number of CPUs among which we may distribute simultaneous 179874aeea5SJeff Kirsher * interrupt handling. 180874aeea5SJeff Kirsher * 181874aeea5SJeff Kirsher * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 182cdb08f8fSBen Hutchings * The default (0) means to assign an interrupt to each core. 183874aeea5SJeff Kirsher */ 184874aeea5SJeff Kirsher static unsigned int rss_cpus; 185874aeea5SJeff Kirsher module_param(rss_cpus, uint, 0444); 186874aeea5SJeff Kirsher MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 187874aeea5SJeff Kirsher 188b9cc977dSBen Hutchings static bool phy_flash_cfg; 189b9cc977dSBen Hutchings module_param(phy_flash_cfg, bool, 0644); 190874aeea5SJeff Kirsher MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 191874aeea5SJeff Kirsher 192e7bed9c8SBen Hutchings static unsigned irq_adapt_low_thresh = 8000; 193874aeea5SJeff Kirsher module_param(irq_adapt_low_thresh, uint, 0644); 194874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_low_thresh, 195874aeea5SJeff Kirsher "Threshold score for reducing IRQ moderation"); 196874aeea5SJeff Kirsher 197e7bed9c8SBen Hutchings static unsigned irq_adapt_high_thresh = 16000; 198874aeea5SJeff Kirsher module_param(irq_adapt_high_thresh, uint, 0644); 199874aeea5SJeff Kirsher MODULE_PARM_DESC(irq_adapt_high_thresh, 200874aeea5SJeff Kirsher "Threshold score for increasing IRQ moderation"); 201874aeea5SJeff Kirsher 202874aeea5SJeff Kirsher static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 203874aeea5SJeff Kirsher NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 204874aeea5SJeff Kirsher NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 205874aeea5SJeff Kirsher NETIF_MSG_TX_ERR | NETIF_MSG_HW); 206874aeea5SJeff Kirsher module_param(debug, uint, 0); 207874aeea5SJeff Kirsher MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 208874aeea5SJeff Kirsher 209874aeea5SJeff Kirsher /************************************************************************** 210874aeea5SJeff Kirsher * 211874aeea5SJeff Kirsher * Utility functions and prototypes 212874aeea5SJeff Kirsher * 213874aeea5SJeff Kirsher *************************************************************************/ 214874aeea5SJeff Kirsher 2157f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type; 216874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx); 217e45a4fedSCharles McLachlan static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog); 218e45a4fedSCharles McLachlan static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp); 219dfe44c1fSCharles McLachlan static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs, 220dfe44c1fSCharles McLachlan u32 flags); 221874aeea5SJeff Kirsher 222874aeea5SJeff Kirsher #define EFX_ASSERT_RESET_SERIALISED(efx) \ 223874aeea5SJeff Kirsher do { \ 224f16aeea0SBen Hutchings if ((efx->state == STATE_READY) || \ 225626950dbSAlexandre Rames (efx->state == STATE_RECOVERY) || \ 226874aeea5SJeff Kirsher (efx->state == STATE_DISABLED)) \ 227874aeea5SJeff Kirsher ASSERT_RTNL(); \ 228874aeea5SJeff Kirsher } while (0) 229874aeea5SJeff Kirsher 230874aeea5SJeff Kirsher /************************************************************************** 231874aeea5SJeff Kirsher * 232874aeea5SJeff Kirsher * Event queue processing 233874aeea5SJeff Kirsher * 234874aeea5SJeff Kirsher *************************************************************************/ 235874aeea5SJeff Kirsher 236874aeea5SJeff Kirsher /* Process channel's event queue 237874aeea5SJeff Kirsher * 238874aeea5SJeff Kirsher * This function is responsible for processing the event queue of a 239874aeea5SJeff Kirsher * single channel. The caller must guarantee that this function will 240874aeea5SJeff Kirsher * never be concurrently called more than once on the same channel, 241874aeea5SJeff Kirsher * though different channels may be being processed concurrently. 242874aeea5SJeff Kirsher */ 243874aeea5SJeff Kirsher static int efx_process_channel(struct efx_channel *channel, int budget) 244874aeea5SJeff Kirsher { 245c936835cSPeter Dunning struct efx_tx_queue *tx_queue; 246e090bfb9SEdward Cree struct list_head rx_list; 247874aeea5SJeff Kirsher int spent; 248874aeea5SJeff Kirsher 2499f2cb71cSBen Hutchings if (unlikely(!channel->enabled)) 250874aeea5SJeff Kirsher return 0; 251874aeea5SJeff Kirsher 252e090bfb9SEdward Cree /* Prepare the batch receive list */ 253e090bfb9SEdward Cree EFX_WARN_ON_PARANOID(channel->rx_list != NULL); 254e090bfb9SEdward Cree INIT_LIST_HEAD(&rx_list); 255e090bfb9SEdward Cree channel->rx_list = &rx_list; 256e090bfb9SEdward Cree 257c936835cSPeter Dunning efx_for_each_channel_tx_queue(tx_queue, channel) { 258c936835cSPeter Dunning tx_queue->pkts_compl = 0; 259c936835cSPeter Dunning tx_queue->bytes_compl = 0; 260c936835cSPeter Dunning } 261c936835cSPeter Dunning 262874aeea5SJeff Kirsher spent = efx_nic_process_eventq(channel, budget); 263d9ab7007SBen Hutchings if (spent && efx_channel_has_rx_queue(channel)) { 264d9ab7007SBen Hutchings struct efx_rx_queue *rx_queue = 265d9ab7007SBen Hutchings efx_channel_get_rx_queue(channel); 266874aeea5SJeff Kirsher 267ff734ef4SBen Hutchings efx_rx_flush_packet(channel); 268cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, true); 269d9ab7007SBen Hutchings } 270874aeea5SJeff Kirsher 271c936835cSPeter Dunning /* Update BQL */ 272c936835cSPeter Dunning efx_for_each_channel_tx_queue(tx_queue, channel) { 273c936835cSPeter Dunning if (tx_queue->bytes_compl) { 274c936835cSPeter Dunning netdev_tx_completed_queue(tx_queue->core_txq, 275c936835cSPeter Dunning tx_queue->pkts_compl, tx_queue->bytes_compl); 276c936835cSPeter Dunning } 277c936835cSPeter Dunning } 278c936835cSPeter Dunning 279e090bfb9SEdward Cree /* Receive any packets we queued up */ 280e090bfb9SEdward Cree netif_receive_skb_list(channel->rx_list); 281e090bfb9SEdward Cree channel->rx_list = NULL; 282e090bfb9SEdward Cree 283874aeea5SJeff Kirsher return spent; 284874aeea5SJeff Kirsher } 285874aeea5SJeff Kirsher 286874aeea5SJeff Kirsher /* NAPI poll handler 287874aeea5SJeff Kirsher * 288874aeea5SJeff Kirsher * NAPI guarantees serialisation of polls of the same device, which 289874aeea5SJeff Kirsher * provides the guarantee required by efx_process_channel(). 290874aeea5SJeff Kirsher */ 291539de7c5SBert Kenward static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 292539de7c5SBert Kenward { 293539de7c5SBert Kenward int step = efx->irq_mod_step_us; 294539de7c5SBert Kenward 295539de7c5SBert Kenward if (channel->irq_mod_score < irq_adapt_low_thresh) { 296539de7c5SBert Kenward if (channel->irq_moderation_us > step) { 297539de7c5SBert Kenward channel->irq_moderation_us -= step; 298539de7c5SBert Kenward efx->type->push_irq_moderation(channel); 299539de7c5SBert Kenward } 300539de7c5SBert Kenward } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 301539de7c5SBert Kenward if (channel->irq_moderation_us < 302539de7c5SBert Kenward efx->irq_rx_moderation_us) { 303539de7c5SBert Kenward channel->irq_moderation_us += step; 304539de7c5SBert Kenward efx->type->push_irq_moderation(channel); 305539de7c5SBert Kenward } 306539de7c5SBert Kenward } 307539de7c5SBert Kenward 308539de7c5SBert Kenward channel->irq_count = 0; 309539de7c5SBert Kenward channel->irq_mod_score = 0; 310539de7c5SBert Kenward } 311539de7c5SBert Kenward 312874aeea5SJeff Kirsher static int efx_poll(struct napi_struct *napi, int budget) 313874aeea5SJeff Kirsher { 314874aeea5SJeff Kirsher struct efx_channel *channel = 315874aeea5SJeff Kirsher container_of(napi, struct efx_channel, napi_str); 316874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 317874aeea5SJeff Kirsher int spent; 318874aeea5SJeff Kirsher 319874aeea5SJeff Kirsher netif_vdbg(efx, intr, efx->net_dev, 320874aeea5SJeff Kirsher "channel %d NAPI poll executing on CPU %d\n", 321874aeea5SJeff Kirsher channel->channel, raw_smp_processor_id()); 322874aeea5SJeff Kirsher 323874aeea5SJeff Kirsher spent = efx_process_channel(channel, budget); 324874aeea5SJeff Kirsher 325eb9a36beSCharles McLachlan xdp_do_flush_map(); 326eb9a36beSCharles McLachlan 327874aeea5SJeff Kirsher if (spent < budget) { 3289d9a6973SBen Hutchings if (efx_channel_has_rx_queue(channel) && 329874aeea5SJeff Kirsher efx->irq_rx_adaptive && 330874aeea5SJeff Kirsher unlikely(++channel->irq_count == 1000)) { 331539de7c5SBert Kenward efx_update_irq_mod(efx, channel); 332874aeea5SJeff Kirsher } 333874aeea5SJeff Kirsher 3343af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 3353af0f342SEdward Cree /* Perhaps expire some ARFS filters */ 3366fbc05e5SEdward Cree mod_delayed_work(system_wq, &channel->filter_work, 0); 3373af0f342SEdward Cree #endif 338874aeea5SJeff Kirsher 339874aeea5SJeff Kirsher /* There is no race here; although napi_disable() will 340874aeea5SJeff Kirsher * only wait for napi_complete(), this isn't a problem 341514bedbcSBen Hutchings * since efx_nic_eventq_read_ack() will have no effect if 342874aeea5SJeff Kirsher * interrupts have already been disabled. 343874aeea5SJeff Kirsher */ 344f820c0acSBert Kenward if (napi_complete_done(napi, spent)) 345514bedbcSBen Hutchings efx_nic_eventq_read_ack(channel); 346874aeea5SJeff Kirsher } 347874aeea5SJeff Kirsher 348874aeea5SJeff Kirsher return spent; 349874aeea5SJeff Kirsher } 350874aeea5SJeff Kirsher 351874aeea5SJeff Kirsher /* Create event queue 352874aeea5SJeff Kirsher * Event queue memory allocations are done only once. If the channel 353874aeea5SJeff Kirsher * is reset, the memory buffer will be reused; this guards against 354874aeea5SJeff Kirsher * errors during channel reset and also simplifies interrupt handling. 355874aeea5SJeff Kirsher */ 356e1253f39SAlex Maftei (amaftei) int efx_probe_eventq(struct efx_channel *channel) 357874aeea5SJeff Kirsher { 358874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 359874aeea5SJeff Kirsher unsigned long entries; 360874aeea5SJeff Kirsher 36186ee5302SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 362874aeea5SJeff Kirsher "chan %d create event queue\n", channel->channel); 363874aeea5SJeff Kirsher 364874aeea5SJeff Kirsher /* Build an event queue with room for one event per tx and rx buffer, 365874aeea5SJeff Kirsher * plus some extra for link state events and MCDI completions. */ 366874aeea5SJeff Kirsher entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 367e01b16a7SEdward Cree EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 368874aeea5SJeff Kirsher channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 369874aeea5SJeff Kirsher 370874aeea5SJeff Kirsher return efx_nic_probe_eventq(channel); 371874aeea5SJeff Kirsher } 372874aeea5SJeff Kirsher 373874aeea5SJeff Kirsher /* Prepare channel's event queue */ 374e1253f39SAlex Maftei (amaftei) int efx_init_eventq(struct efx_channel *channel) 375874aeea5SJeff Kirsher { 37615acb1ceSBen Hutchings struct efx_nic *efx = channel->efx; 377261e4d96SJon Cooper int rc; 378261e4d96SJon Cooper 379261e4d96SJon Cooper EFX_WARN_ON_PARANOID(channel->eventq_init); 380261e4d96SJon Cooper 38115acb1ceSBen Hutchings netif_dbg(efx, drv, efx->net_dev, 382874aeea5SJeff Kirsher "chan %d init event queue\n", channel->channel); 383874aeea5SJeff Kirsher 384261e4d96SJon Cooper rc = efx_nic_init_eventq(channel); 385261e4d96SJon Cooper if (rc == 0) { 38615acb1ceSBen Hutchings efx->type->push_irq_moderation(channel); 387874aeea5SJeff Kirsher channel->eventq_read_ptr = 0; 388be3fc09cSBen Hutchings channel->eventq_init = true; 389874aeea5SJeff Kirsher } 390261e4d96SJon Cooper return rc; 391261e4d96SJon Cooper } 392874aeea5SJeff Kirsher 3939f2cb71cSBen Hutchings /* Enable event queue processing and NAPI */ 39436763266SAlexandre Rames void efx_start_eventq(struct efx_channel *channel) 3959f2cb71cSBen Hutchings { 3969f2cb71cSBen Hutchings netif_dbg(channel->efx, ifup, channel->efx->net_dev, 3979f2cb71cSBen Hutchings "chan %d start event queue\n", channel->channel); 3989f2cb71cSBen Hutchings 399514bedbcSBen Hutchings /* Make sure the NAPI handler sees the enabled flag set */ 4009f2cb71cSBen Hutchings channel->enabled = true; 4019f2cb71cSBen Hutchings smp_wmb(); 4029f2cb71cSBen Hutchings 4039f2cb71cSBen Hutchings napi_enable(&channel->napi_str); 4049f2cb71cSBen Hutchings efx_nic_eventq_read_ack(channel); 4059f2cb71cSBen Hutchings } 4069f2cb71cSBen Hutchings 4079f2cb71cSBen Hutchings /* Disable event queue processing and NAPI */ 40836763266SAlexandre Rames void efx_stop_eventq(struct efx_channel *channel) 4099f2cb71cSBen Hutchings { 4109f2cb71cSBen Hutchings if (!channel->enabled) 4119f2cb71cSBen Hutchings return; 4129f2cb71cSBen Hutchings 4139f2cb71cSBen Hutchings napi_disable(&channel->napi_str); 4149f2cb71cSBen Hutchings channel->enabled = false; 4159f2cb71cSBen Hutchings } 4169f2cb71cSBen Hutchings 417e1253f39SAlex Maftei (amaftei) void efx_fini_eventq(struct efx_channel *channel) 418874aeea5SJeff Kirsher { 419be3fc09cSBen Hutchings if (!channel->eventq_init) 420be3fc09cSBen Hutchings return; 421be3fc09cSBen Hutchings 422874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 423874aeea5SJeff Kirsher "chan %d fini event queue\n", channel->channel); 424874aeea5SJeff Kirsher 425874aeea5SJeff Kirsher efx_nic_fini_eventq(channel); 426be3fc09cSBen Hutchings channel->eventq_init = false; 427874aeea5SJeff Kirsher } 428874aeea5SJeff Kirsher 429e1253f39SAlex Maftei (amaftei) void efx_remove_eventq(struct efx_channel *channel) 430874aeea5SJeff Kirsher { 431874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 432874aeea5SJeff Kirsher "chan %d remove event queue\n", channel->channel); 433874aeea5SJeff Kirsher 434874aeea5SJeff Kirsher efx_nic_remove_eventq(channel); 435874aeea5SJeff Kirsher } 436874aeea5SJeff Kirsher 437874aeea5SJeff Kirsher /************************************************************************** 438874aeea5SJeff Kirsher * 439874aeea5SJeff Kirsher * Channel handling 440874aeea5SJeff Kirsher * 441874aeea5SJeff Kirsher *************************************************************************/ 442874aeea5SJeff Kirsher 4437f967c01SBen Hutchings /* Allocate and initialise a channel structure. */ 444e1253f39SAlex Maftei (amaftei) struct efx_channel * 445874aeea5SJeff Kirsher efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 446874aeea5SJeff Kirsher { 447874aeea5SJeff Kirsher struct efx_channel *channel; 448874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 449874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 450874aeea5SJeff Kirsher int j; 451874aeea5SJeff Kirsher 4527f967c01SBen Hutchings channel = kzalloc(sizeof(*channel), GFP_KERNEL); 4537f967c01SBen Hutchings if (!channel) 4547f967c01SBen Hutchings return NULL; 4557f967c01SBen Hutchings 4567f967c01SBen Hutchings channel->efx = efx; 4577f967c01SBen Hutchings channel->channel = i; 4587f967c01SBen Hutchings channel->type = &efx_default_channel_type; 4597f967c01SBen Hutchings 4607f967c01SBen Hutchings for (j = 0; j < EFX_TXQ_TYPES; j++) { 4617f967c01SBen Hutchings tx_queue = &channel->tx_queue[j]; 4627f967c01SBen Hutchings tx_queue->efx = efx; 4637f967c01SBen Hutchings tx_queue->queue = i * EFX_TXQ_TYPES + j; 4647f967c01SBen Hutchings tx_queue->channel = channel; 4657f967c01SBen Hutchings } 4667f967c01SBen Hutchings 4673af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 4686fbc05e5SEdward Cree INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 4693af0f342SEdward Cree #endif 4703af0f342SEdward Cree 4717f967c01SBen Hutchings rx_queue = &channel->rx_queue; 4727f967c01SBen Hutchings rx_queue->efx = efx; 4737aa1402eSKees Cook timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 4747f967c01SBen Hutchings 4757f967c01SBen Hutchings return channel; 4767f967c01SBen Hutchings } 4777f967c01SBen Hutchings 4787f967c01SBen Hutchings /* Allocate and initialise a channel structure, copying parameters 4797f967c01SBen Hutchings * (but not resources) from an old channel structure. 4807f967c01SBen Hutchings */ 481e1253f39SAlex Maftei (amaftei) struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel) 4827f967c01SBen Hutchings { 4837f967c01SBen Hutchings struct efx_channel *channel; 4847f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 4857f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 4867f967c01SBen Hutchings int j; 4877f967c01SBen Hutchings 488874aeea5SJeff Kirsher channel = kmalloc(sizeof(*channel), GFP_KERNEL); 489874aeea5SJeff Kirsher if (!channel) 490874aeea5SJeff Kirsher return NULL; 491874aeea5SJeff Kirsher 492874aeea5SJeff Kirsher *channel = *old_channel; 493874aeea5SJeff Kirsher 494874aeea5SJeff Kirsher channel->napi_dev = NULL; 49546d054f8SBert Kenward INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 49646d054f8SBert Kenward channel->napi_str.napi_id = 0; 49746d054f8SBert Kenward channel->napi_str.state = 0; 498874aeea5SJeff Kirsher memset(&channel->eventq, 0, sizeof(channel->eventq)); 499874aeea5SJeff Kirsher 500874aeea5SJeff Kirsher for (j = 0; j < EFX_TXQ_TYPES; j++) { 501874aeea5SJeff Kirsher tx_queue = &channel->tx_queue[j]; 502874aeea5SJeff Kirsher if (tx_queue->channel) 503874aeea5SJeff Kirsher tx_queue->channel = channel; 504874aeea5SJeff Kirsher tx_queue->buffer = NULL; 505874aeea5SJeff Kirsher memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 506874aeea5SJeff Kirsher } 507874aeea5SJeff Kirsher 508874aeea5SJeff Kirsher rx_queue = &channel->rx_queue; 5097f967c01SBen Hutchings rx_queue->buffer = NULL; 5107f967c01SBen Hutchings memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 5117aa1402eSKees Cook timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 5123af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 5136fbc05e5SEdward Cree INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 5143af0f342SEdward Cree #endif 515874aeea5SJeff Kirsher 516874aeea5SJeff Kirsher return channel; 517874aeea5SJeff Kirsher } 518874aeea5SJeff Kirsher 519874aeea5SJeff Kirsher static int efx_probe_channel(struct efx_channel *channel) 520874aeea5SJeff Kirsher { 521874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 522874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 523874aeea5SJeff Kirsher int rc; 524874aeea5SJeff Kirsher 525874aeea5SJeff Kirsher netif_dbg(channel->efx, probe, channel->efx->net_dev, 526874aeea5SJeff Kirsher "creating channel %d\n", channel->channel); 527874aeea5SJeff Kirsher 5287f967c01SBen Hutchings rc = channel->type->pre_probe(channel); 5297f967c01SBen Hutchings if (rc) 5307f967c01SBen Hutchings goto fail; 5317f967c01SBen Hutchings 532874aeea5SJeff Kirsher rc = efx_probe_eventq(channel); 533874aeea5SJeff Kirsher if (rc) 5347f967c01SBen Hutchings goto fail; 535874aeea5SJeff Kirsher 536874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) { 537874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 538874aeea5SJeff Kirsher if (rc) 5397f967c01SBen Hutchings goto fail; 540874aeea5SJeff Kirsher } 541874aeea5SJeff Kirsher 542874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) { 543874aeea5SJeff Kirsher rc = efx_probe_rx_queue(rx_queue); 544874aeea5SJeff Kirsher if (rc) 5457f967c01SBen Hutchings goto fail; 546874aeea5SJeff Kirsher } 547874aeea5SJeff Kirsher 548e090bfb9SEdward Cree channel->rx_list = NULL; 549e090bfb9SEdward Cree 550874aeea5SJeff Kirsher return 0; 551874aeea5SJeff Kirsher 5527f967c01SBen Hutchings fail: 5537f967c01SBen Hutchings efx_remove_channel(channel); 554874aeea5SJeff Kirsher return rc; 555874aeea5SJeff Kirsher } 556874aeea5SJeff Kirsher 557e1253f39SAlex Maftei (amaftei) void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 5587f967c01SBen Hutchings { 5597f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 5607f967c01SBen Hutchings const char *type; 5617f967c01SBen Hutchings int number; 5627f967c01SBen Hutchings 5637f967c01SBen Hutchings number = channel->channel; 5643990a8ffSCharles McLachlan 5653990a8ffSCharles McLachlan if (number >= efx->xdp_channel_offset && 5663990a8ffSCharles McLachlan !WARN_ON_ONCE(!efx->n_xdp_channels)) { 5673990a8ffSCharles McLachlan type = "-xdp"; 5683990a8ffSCharles McLachlan number -= efx->xdp_channel_offset; 5693990a8ffSCharles McLachlan } else if (efx->tx_channel_offset == 0) { 5707f967c01SBen Hutchings type = ""; 5713990a8ffSCharles McLachlan } else if (number < efx->tx_channel_offset) { 5727f967c01SBen Hutchings type = "-rx"; 5737f967c01SBen Hutchings } else { 5747f967c01SBen Hutchings type = "-tx"; 5757f967c01SBen Hutchings number -= efx->tx_channel_offset; 5767f967c01SBen Hutchings } 5777f967c01SBen Hutchings snprintf(buf, len, "%s%s-%d", efx->name, type, number); 5787f967c01SBen Hutchings } 579874aeea5SJeff Kirsher 580e1253f39SAlex Maftei (amaftei) void efx_set_channel_names(struct efx_nic *efx) 581874aeea5SJeff Kirsher { 582874aeea5SJeff Kirsher struct efx_channel *channel; 583874aeea5SJeff Kirsher 5847f967c01SBen Hutchings efx_for_each_channel(channel, efx) 5857f967c01SBen Hutchings channel->type->get_name(channel, 586d8291187SBen Hutchings efx->msi_context[channel->channel].name, 587d8291187SBen Hutchings sizeof(efx->msi_context[0].name)); 588874aeea5SJeff Kirsher } 589874aeea5SJeff Kirsher 590e1253f39SAlex Maftei (amaftei) int efx_probe_channels(struct efx_nic *efx) 591874aeea5SJeff Kirsher { 592874aeea5SJeff Kirsher struct efx_channel *channel; 593874aeea5SJeff Kirsher int rc; 594874aeea5SJeff Kirsher 595874aeea5SJeff Kirsher /* Restart special buffer allocation */ 596874aeea5SJeff Kirsher efx->next_buffer_table = 0; 597874aeea5SJeff Kirsher 598c92aaff1SBen Hutchings /* Probe channels in reverse, so that any 'extra' channels 599c92aaff1SBen Hutchings * use the start of the buffer table. This allows the traffic 600c92aaff1SBen Hutchings * channels to be resized without moving them or wasting the 601c92aaff1SBen Hutchings * entries before them. 602c92aaff1SBen Hutchings */ 603c92aaff1SBen Hutchings efx_for_each_channel_rev(channel, efx) { 604874aeea5SJeff Kirsher rc = efx_probe_channel(channel); 605874aeea5SJeff Kirsher if (rc) { 606874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 607874aeea5SJeff Kirsher "failed to create channel %d\n", 608874aeea5SJeff Kirsher channel->channel); 609874aeea5SJeff Kirsher goto fail; 610874aeea5SJeff Kirsher } 611874aeea5SJeff Kirsher } 612874aeea5SJeff Kirsher efx_set_channel_names(efx); 613874aeea5SJeff Kirsher 614874aeea5SJeff Kirsher return 0; 615874aeea5SJeff Kirsher 616874aeea5SJeff Kirsher fail: 617874aeea5SJeff Kirsher efx_remove_channels(efx); 618874aeea5SJeff Kirsher return rc; 619874aeea5SJeff Kirsher } 620874aeea5SJeff Kirsher 621874aeea5SJeff Kirsher /* Channels are shutdown and reinitialised whilst the NIC is running 622874aeea5SJeff Kirsher * to propagate configuration changes (mtu, checksum offload), or 623874aeea5SJeff Kirsher * to clear hardware error conditions 624874aeea5SJeff Kirsher */ 6259f2cb71cSBen Hutchings static void efx_start_datapath(struct efx_nic *efx) 626874aeea5SJeff Kirsher { 627ebfcd0fdSAndrew Rybchenko netdev_features_t old_features = efx->net_dev->features; 62885740cdfSBen Hutchings bool old_rx_scatter = efx->rx_scatter; 629874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 630874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 631874aeea5SJeff Kirsher struct efx_channel *channel; 63285740cdfSBen Hutchings size_t rx_buf_len; 633874aeea5SJeff Kirsher 634874aeea5SJeff Kirsher /* Calculate the rx buffer allocation parameters required to 635874aeea5SJeff Kirsher * support the current MTU, including padding for header 636874aeea5SJeff Kirsher * alignment and overruns. 637874aeea5SJeff Kirsher */ 63843a3739dSJon Cooper efx->rx_dma_len = (efx->rx_prefix_size + 639874aeea5SJeff Kirsher EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 640874aeea5SJeff Kirsher efx->type->rx_buffer_padding); 641eb9a36beSCharles McLachlan rx_buf_len = (sizeof(struct efx_rx_page_state) + XDP_PACKET_HEADROOM + 6422ec03014SAndrew Rybchenko efx->rx_ip_align + efx->rx_dma_len); 64385740cdfSBen Hutchings if (rx_buf_len <= PAGE_SIZE) { 644e8c68c0aSJon Cooper efx->rx_scatter = efx->type->always_rx_scatter; 64585740cdfSBen Hutchings efx->rx_buffer_order = 0; 64685740cdfSBen Hutchings } else if (efx->type->can_rx_scatter) { 647950c54dfSBen Hutchings BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 64885740cdfSBen Hutchings BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 649950c54dfSBen Hutchings 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 650950c54dfSBen Hutchings EFX_RX_BUF_ALIGNMENT) > 651950c54dfSBen Hutchings PAGE_SIZE); 65285740cdfSBen Hutchings efx->rx_scatter = true; 65385740cdfSBen Hutchings efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 65485740cdfSBen Hutchings efx->rx_buffer_order = 0; 65585740cdfSBen Hutchings } else { 65685740cdfSBen Hutchings efx->rx_scatter = false; 65785740cdfSBen Hutchings efx->rx_buffer_order = get_order(rx_buf_len); 65885740cdfSBen Hutchings } 65985740cdfSBen Hutchings 6601648a23fSDaniel Pieczko efx_rx_config_page_split(efx); 6611648a23fSDaniel Pieczko if (efx->rx_buffer_order) 6621648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6631648a23fSDaniel Pieczko "RX buf len=%u; page order=%u batch=%u\n", 6641648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_buffer_order, 6651648a23fSDaniel Pieczko efx->rx_pages_per_batch); 6661648a23fSDaniel Pieczko else 6671648a23fSDaniel Pieczko netif_dbg(efx, drv, efx->net_dev, 6681648a23fSDaniel Pieczko "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 6691648a23fSDaniel Pieczko efx->rx_dma_len, efx->rx_page_buf_step, 6701648a23fSDaniel Pieczko efx->rx_bufs_per_page, efx->rx_pages_per_batch); 6712768935aSDaniel Pieczko 672ebfcd0fdSAndrew Rybchenko /* Restore previously fixed features in hw_features and remove 673ebfcd0fdSAndrew Rybchenko * features which are fixed now 674ebfcd0fdSAndrew Rybchenko */ 675ebfcd0fdSAndrew Rybchenko efx->net_dev->hw_features |= efx->net_dev->features; 676ebfcd0fdSAndrew Rybchenko efx->net_dev->hw_features &= ~efx->fixed_features; 677ebfcd0fdSAndrew Rybchenko efx->net_dev->features |= efx->fixed_features; 678ebfcd0fdSAndrew Rybchenko if (efx->net_dev->features != old_features) 679ebfcd0fdSAndrew Rybchenko netdev_features_change(efx->net_dev); 680ebfcd0fdSAndrew Rybchenko 681e8c68c0aSJon Cooper /* RX filters may also have scatter-enabled flags */ 68285740cdfSBen Hutchings if (efx->rx_scatter != old_rx_scatter) 683add72477SBen Hutchings efx->type->filter_update_rx_scatter(efx); 684874aeea5SJeff Kirsher 68514bf718fSBen Hutchings /* We must keep at least one descriptor in a TX ring empty. 68614bf718fSBen Hutchings * We could avoid this when the queue size does not exactly 68714bf718fSBen Hutchings * match the hardware ring size, but it's not that important. 68814bf718fSBen Hutchings * Therefore we stop the queue when one more skb might fill 68914bf718fSBen Hutchings * the ring completely. We wake it when half way back to 69014bf718fSBen Hutchings * empty. 69114bf718fSBen Hutchings */ 69214bf718fSBen Hutchings efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 69314bf718fSBen Hutchings efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 69414bf718fSBen Hutchings 695874aeea5SJeff Kirsher /* Initialise the channels */ 696874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 6973881d8abSAlexandre Rames efx_for_each_channel_tx_queue(tx_queue, channel) { 698874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 6993881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 7003881d8abSAlexandre Rames } 701874aeea5SJeff Kirsher 7029f2cb71cSBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) { 703874aeea5SJeff Kirsher efx_init_rx_queue(rx_queue); 7043881d8abSAlexandre Rames atomic_inc(&efx->active_queues); 705cce28794SJon Cooper efx_stop_eventq(channel); 706cce28794SJon Cooper efx_fast_push_rx_descriptors(rx_queue, false); 707cce28794SJon Cooper efx_start_eventq(channel); 7089f2cb71cSBen Hutchings } 709874aeea5SJeff Kirsher 71085740cdfSBen Hutchings WARN_ON(channel->rx_pkt_n_frags); 711874aeea5SJeff Kirsher } 7129f2cb71cSBen Hutchings 7132ea4dc28SAlexandre Rames efx_ptp_start_datapath(efx); 7142ea4dc28SAlexandre Rames 7159f2cb71cSBen Hutchings if (netif_device_present(efx->net_dev)) 7169f2cb71cSBen Hutchings netif_tx_wake_all_queues(efx->net_dev); 717874aeea5SJeff Kirsher } 718874aeea5SJeff Kirsher 7199f2cb71cSBen Hutchings static void efx_stop_datapath(struct efx_nic *efx) 720874aeea5SJeff Kirsher { 721874aeea5SJeff Kirsher struct efx_channel *channel; 722874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 723874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 724874aeea5SJeff Kirsher int rc; 725874aeea5SJeff Kirsher 726874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 727874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 728874aeea5SJeff Kirsher 7292ea4dc28SAlexandre Rames efx_ptp_stop_datapath(efx); 7302ea4dc28SAlexandre Rames 731d8aec745SBen Hutchings /* Stop RX refill */ 732d8aec745SBen Hutchings efx_for_each_channel(channel, efx) { 733d8aec745SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 734d8aec745SBen Hutchings rx_queue->refill_enabled = false; 735d8aec745SBen Hutchings } 736d8aec745SBen Hutchings 737874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 7389f2cb71cSBen Hutchings /* RX packet processing is pipelined, so wait for the 7399f2cb71cSBen Hutchings * NAPI handler to complete. At least event queue 0 7409f2cb71cSBen Hutchings * might be kept active by non-data events, so don't 7419f2cb71cSBen Hutchings * use napi_synchronize() but actually disable NAPI 7429f2cb71cSBen Hutchings * temporarily. 7439f2cb71cSBen Hutchings */ 7449f2cb71cSBen Hutchings if (efx_channel_has_rx_queue(channel)) { 7459f2cb71cSBen Hutchings efx_stop_eventq(channel); 7469f2cb71cSBen Hutchings efx_start_eventq(channel); 7479f2cb71cSBen Hutchings } 748e42c3d85SBen Hutchings } 749874aeea5SJeff Kirsher 750e42c3d85SBen Hutchings rc = efx->type->fini_dmaq(efx); 7515a6681e2SEdward Cree if (rc) { 752e42c3d85SBen Hutchings netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 753e42c3d85SBen Hutchings } else { 754e42c3d85SBen Hutchings netif_dbg(efx, drv, efx->net_dev, 755e42c3d85SBen Hutchings "successfully flushed all queues\n"); 756e42c3d85SBen Hutchings } 757e42c3d85SBen Hutchings 758e42c3d85SBen Hutchings efx_for_each_channel(channel, efx) { 759874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 760874aeea5SJeff Kirsher efx_fini_rx_queue(rx_queue); 761874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 762874aeea5SJeff Kirsher efx_fini_tx_queue(tx_queue); 763874aeea5SJeff Kirsher } 764eb9a36beSCharles McLachlan efx->xdp_rxq_info_failed = false; 765874aeea5SJeff Kirsher } 766874aeea5SJeff Kirsher 767e1253f39SAlex Maftei (amaftei) void efx_remove_channel(struct efx_channel *channel) 768874aeea5SJeff Kirsher { 769874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 770874aeea5SJeff Kirsher struct efx_rx_queue *rx_queue; 771874aeea5SJeff Kirsher 772874aeea5SJeff Kirsher netif_dbg(channel->efx, drv, channel->efx->net_dev, 773874aeea5SJeff Kirsher "destroy chan %d\n", channel->channel); 774874aeea5SJeff Kirsher 775874aeea5SJeff Kirsher efx_for_each_channel_rx_queue(rx_queue, channel) 776874aeea5SJeff Kirsher efx_remove_rx_queue(rx_queue); 777874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, channel) 778874aeea5SJeff Kirsher efx_remove_tx_queue(tx_queue); 779874aeea5SJeff Kirsher efx_remove_eventq(channel); 780c31e5f9fSStuart Hodgson channel->type->post_remove(channel); 781874aeea5SJeff Kirsher } 782874aeea5SJeff Kirsher 783e1253f39SAlex Maftei (amaftei) void efx_remove_channels(struct efx_nic *efx) 784874aeea5SJeff Kirsher { 785874aeea5SJeff Kirsher struct efx_channel *channel; 786874aeea5SJeff Kirsher 787874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 788874aeea5SJeff Kirsher efx_remove_channel(channel); 7893990a8ffSCharles McLachlan 7903990a8ffSCharles McLachlan kfree(efx->xdp_tx_queues); 791874aeea5SJeff Kirsher } 792874aeea5SJeff Kirsher 793e1253f39SAlex Maftei (amaftei) int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 794874aeea5SJeff Kirsher { 795874aeea5SJeff Kirsher struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 796874aeea5SJeff Kirsher u32 old_rxq_entries, old_txq_entries; 7977f967c01SBen Hutchings unsigned i, next_buffer_table = 0; 798261e4d96SJon Cooper int rc, rc2; 7998b7325b4SBen Hutchings 8008b7325b4SBen Hutchings rc = efx_check_disabled(efx); 8018b7325b4SBen Hutchings if (rc) 8028b7325b4SBen Hutchings return rc; 8037f967c01SBen Hutchings 8047f967c01SBen Hutchings /* Not all channels should be reallocated. We must avoid 8057f967c01SBen Hutchings * reallocating their buffer table entries. 8067f967c01SBen Hutchings */ 8077f967c01SBen Hutchings efx_for_each_channel(channel, efx) { 8087f967c01SBen Hutchings struct efx_rx_queue *rx_queue; 8097f967c01SBen Hutchings struct efx_tx_queue *tx_queue; 8107f967c01SBen Hutchings 8117f967c01SBen Hutchings if (channel->type->copy) 8127f967c01SBen Hutchings continue; 8137f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8147f967c01SBen Hutchings channel->eventq.index + 8157f967c01SBen Hutchings channel->eventq.entries); 8167f967c01SBen Hutchings efx_for_each_channel_rx_queue(rx_queue, channel) 8177f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8187f967c01SBen Hutchings rx_queue->rxd.index + 8197f967c01SBen Hutchings rx_queue->rxd.entries); 8207f967c01SBen Hutchings efx_for_each_channel_tx_queue(tx_queue, channel) 8217f967c01SBen Hutchings next_buffer_table = max(next_buffer_table, 8227f967c01SBen Hutchings tx_queue->txd.index + 8237f967c01SBen Hutchings tx_queue->txd.entries); 8247f967c01SBen Hutchings } 825874aeea5SJeff Kirsher 82629c69a48SBen Hutchings efx_device_detach_sync(efx); 827874aeea5SJeff Kirsher efx_stop_all(efx); 828d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 829874aeea5SJeff Kirsher 8307f967c01SBen Hutchings /* Clone channels (where possible) */ 831874aeea5SJeff Kirsher memset(other_channel, 0, sizeof(other_channel)); 832874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 8337f967c01SBen Hutchings channel = efx->channel[i]; 8347f967c01SBen Hutchings if (channel->type->copy) 8357f967c01SBen Hutchings channel = channel->type->copy(channel); 836874aeea5SJeff Kirsher if (!channel) { 837874aeea5SJeff Kirsher rc = -ENOMEM; 838874aeea5SJeff Kirsher goto out; 839874aeea5SJeff Kirsher } 840874aeea5SJeff Kirsher other_channel[i] = channel; 841874aeea5SJeff Kirsher } 842874aeea5SJeff Kirsher 843874aeea5SJeff Kirsher /* Swap entry counts and channel pointers */ 844874aeea5SJeff Kirsher old_rxq_entries = efx->rxq_entries; 845874aeea5SJeff Kirsher old_txq_entries = efx->txq_entries; 846874aeea5SJeff Kirsher efx->rxq_entries = rxq_entries; 847874aeea5SJeff Kirsher efx->txq_entries = txq_entries; 848874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 849874aeea5SJeff Kirsher channel = efx->channel[i]; 850874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 851874aeea5SJeff Kirsher other_channel[i] = channel; 852874aeea5SJeff Kirsher } 853874aeea5SJeff Kirsher 8547f967c01SBen Hutchings /* Restart buffer table allocation */ 8557f967c01SBen Hutchings efx->next_buffer_table = next_buffer_table; 8567f967c01SBen Hutchings 8577f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8587f967c01SBen Hutchings channel = efx->channel[i]; 8597f967c01SBen Hutchings if (!channel->type->copy) 8607f967c01SBen Hutchings continue; 8617f967c01SBen Hutchings rc = efx_probe_channel(channel); 862874aeea5SJeff Kirsher if (rc) 863874aeea5SJeff Kirsher goto rollback; 8647f967c01SBen Hutchings efx_init_napi_channel(efx->channel[i]); 865874aeea5SJeff Kirsher } 866874aeea5SJeff Kirsher 8677f967c01SBen Hutchings out: 8687f967c01SBen Hutchings /* Destroy unused channel structures */ 8697f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) { 8707f967c01SBen Hutchings channel = other_channel[i]; 8717f967c01SBen Hutchings if (channel && channel->type->copy) { 8727f967c01SBen Hutchings efx_fini_napi_channel(channel); 8737f967c01SBen Hutchings efx_remove_channel(channel); 8747f967c01SBen Hutchings kfree(channel); 8757f967c01SBen Hutchings } 8767f967c01SBen Hutchings } 8777f967c01SBen Hutchings 878261e4d96SJon Cooper rc2 = efx_soft_enable_interrupts(efx); 879261e4d96SJon Cooper if (rc2) { 880261e4d96SJon Cooper rc = rc ? rc : rc2; 881261e4d96SJon Cooper netif_err(efx, drv, efx->net_dev, 882261e4d96SJon Cooper "unable to restart interrupts on channel reallocation\n"); 883261e4d96SJon Cooper efx_schedule_reset(efx, RESET_TYPE_DISABLE); 884261e4d96SJon Cooper } else { 885874aeea5SJeff Kirsher efx_start_all(efx); 8869c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 887261e4d96SJon Cooper } 888874aeea5SJeff Kirsher return rc; 889874aeea5SJeff Kirsher 890874aeea5SJeff Kirsher rollback: 891874aeea5SJeff Kirsher /* Swap back */ 892874aeea5SJeff Kirsher efx->rxq_entries = old_rxq_entries; 893874aeea5SJeff Kirsher efx->txq_entries = old_txq_entries; 894874aeea5SJeff Kirsher for (i = 0; i < efx->n_channels; i++) { 895874aeea5SJeff Kirsher channel = efx->channel[i]; 896874aeea5SJeff Kirsher efx->channel[i] = other_channel[i]; 897874aeea5SJeff Kirsher other_channel[i] = channel; 898874aeea5SJeff Kirsher } 899874aeea5SJeff Kirsher goto out; 900874aeea5SJeff Kirsher } 901874aeea5SJeff Kirsher 902874aeea5SJeff Kirsher void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 903874aeea5SJeff Kirsher { 90450f444aaSRobert Stonehouse mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10)); 905874aeea5SJeff Kirsher } 906874aeea5SJeff Kirsher 907e1253f39SAlex Maftei (amaftei) bool efx_default_channel_want_txqs(struct efx_channel *channel) 9082935e3c3SEdward Cree { 9092935e3c3SEdward Cree return channel->channel - channel->efx->tx_channel_offset < 9102935e3c3SEdward Cree channel->efx->n_tx_channels; 9112935e3c3SEdward Cree } 9122935e3c3SEdward Cree 9137f967c01SBen Hutchings static const struct efx_channel_type efx_default_channel_type = { 9147f967c01SBen Hutchings .pre_probe = efx_channel_dummy_op_int, 915c31e5f9fSStuart Hodgson .post_remove = efx_channel_dummy_op_void, 9167f967c01SBen Hutchings .get_name = efx_get_channel_name, 9177f967c01SBen Hutchings .copy = efx_copy_channel, 9182935e3c3SEdward Cree .want_txqs = efx_default_channel_want_txqs, 9197f967c01SBen Hutchings .keep_eventq = false, 9202935e3c3SEdward Cree .want_pio = true, 9217f967c01SBen Hutchings }; 9227f967c01SBen Hutchings 9237f967c01SBen Hutchings int efx_channel_dummy_op_int(struct efx_channel *channel) 9247f967c01SBen Hutchings { 9257f967c01SBen Hutchings return 0; 9267f967c01SBen Hutchings } 9277f967c01SBen Hutchings 928c31e5f9fSStuart Hodgson void efx_channel_dummy_op_void(struct efx_channel *channel) 929c31e5f9fSStuart Hodgson { 930c31e5f9fSStuart Hodgson } 931c31e5f9fSStuart Hodgson 932874aeea5SJeff Kirsher /************************************************************************** 933874aeea5SJeff Kirsher * 934874aeea5SJeff Kirsher * Port handling 935874aeea5SJeff Kirsher * 936874aeea5SJeff Kirsher **************************************************************************/ 937874aeea5SJeff Kirsher 938874aeea5SJeff Kirsher /* This ensures that the kernel is kept informed (via 939874aeea5SJeff Kirsher * netif_carrier_on/off) of the link status, and also maintains the 940874aeea5SJeff Kirsher * link status's stop on the port's TX queue. 941874aeea5SJeff Kirsher */ 942874aeea5SJeff Kirsher void efx_link_status_changed(struct efx_nic *efx) 943874aeea5SJeff Kirsher { 944874aeea5SJeff Kirsher struct efx_link_state *link_state = &efx->link_state; 945874aeea5SJeff Kirsher 946874aeea5SJeff Kirsher /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 947874aeea5SJeff Kirsher * that no events are triggered between unregister_netdev() and the 948874aeea5SJeff Kirsher * driver unloading. A more general condition is that NETDEV_CHANGE 949874aeea5SJeff Kirsher * can only be generated between NETDEV_UP and NETDEV_DOWN */ 950874aeea5SJeff Kirsher if (!netif_running(efx->net_dev)) 951874aeea5SJeff Kirsher return; 952874aeea5SJeff Kirsher 953874aeea5SJeff Kirsher if (link_state->up != netif_carrier_ok(efx->net_dev)) { 954874aeea5SJeff Kirsher efx->n_link_state_changes++; 955874aeea5SJeff Kirsher 956874aeea5SJeff Kirsher if (link_state->up) 957874aeea5SJeff Kirsher netif_carrier_on(efx->net_dev); 958874aeea5SJeff Kirsher else 959874aeea5SJeff Kirsher netif_carrier_off(efx->net_dev); 960874aeea5SJeff Kirsher } 961874aeea5SJeff Kirsher 962874aeea5SJeff Kirsher /* Status message for kernel log */ 9632aa9ef11SBen Hutchings if (link_state->up) 964874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, 965964e6135SBen Hutchings "link up at %uMbps %s-duplex (MTU %d)\n", 966874aeea5SJeff Kirsher link_state->speed, link_state->fd ? "full" : "half", 967964e6135SBen Hutchings efx->net_dev->mtu); 9682aa9ef11SBen Hutchings else 969874aeea5SJeff Kirsher netif_info(efx, link, efx->net_dev, "link down\n"); 970874aeea5SJeff Kirsher } 971874aeea5SJeff Kirsher 972c2ab85d2SEdward Cree void efx_link_set_advertising(struct efx_nic *efx, 973c2ab85d2SEdward Cree const unsigned long *advertising) 974874aeea5SJeff Kirsher { 975c2ab85d2SEdward Cree memcpy(efx->link_advertising, advertising, 976c2ab85d2SEdward Cree sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK())); 977c2ab85d2SEdward Cree 978c2ab85d2SEdward Cree efx->link_advertising[0] |= ADVERTISED_Autoneg; 979c2ab85d2SEdward Cree if (advertising[0] & ADVERTISED_Pause) 980874aeea5SJeff Kirsher efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 981874aeea5SJeff Kirsher else 982874aeea5SJeff Kirsher efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 983c2ab85d2SEdward Cree if (advertising[0] & ADVERTISED_Asym_Pause) 984874aeea5SJeff Kirsher efx->wanted_fc ^= EFX_FC_TX; 985874aeea5SJeff Kirsher } 986c2ab85d2SEdward Cree 987c2ab85d2SEdward Cree /* Equivalent to efx_link_set_advertising with all-zeroes, except does not 988c2ab85d2SEdward Cree * force the Autoneg bit on. 989c2ab85d2SEdward Cree */ 990c2ab85d2SEdward Cree void efx_link_clear_advertising(struct efx_nic *efx) 991c2ab85d2SEdward Cree { 992c2ab85d2SEdward Cree bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS); 993c2ab85d2SEdward Cree efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 994874aeea5SJeff Kirsher } 995874aeea5SJeff Kirsher 996874aeea5SJeff Kirsher void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 997874aeea5SJeff Kirsher { 998874aeea5SJeff Kirsher efx->wanted_fc = wanted_fc; 999c2ab85d2SEdward Cree if (efx->link_advertising[0]) { 1000874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_RX) 1001c2ab85d2SEdward Cree efx->link_advertising[0] |= (ADVERTISED_Pause | 1002874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 1003874aeea5SJeff Kirsher else 1004c2ab85d2SEdward Cree efx->link_advertising[0] &= ~(ADVERTISED_Pause | 1005874aeea5SJeff Kirsher ADVERTISED_Asym_Pause); 1006874aeea5SJeff Kirsher if (wanted_fc & EFX_FC_TX) 1007c2ab85d2SEdward Cree efx->link_advertising[0] ^= ADVERTISED_Asym_Pause; 1008874aeea5SJeff Kirsher } 1009874aeea5SJeff Kirsher } 1010874aeea5SJeff Kirsher 1011874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx); 1012874aeea5SJeff Kirsher 10130d322413SEdward Cree /* We assume that efx->type->reconfigure_mac will always try to sync RX 10140d322413SEdward Cree * filters and therefore needs to read-lock the filter table against freeing 10150d322413SEdward Cree */ 10160d322413SEdward Cree void efx_mac_reconfigure(struct efx_nic *efx) 10170d322413SEdward Cree { 10180d322413SEdward Cree down_read(&efx->filter_sem); 10190d322413SEdward Cree efx->type->reconfigure_mac(efx); 10200d322413SEdward Cree up_read(&efx->filter_sem); 10210d322413SEdward Cree } 10220d322413SEdward Cree 1023874aeea5SJeff Kirsher /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 1024874aeea5SJeff Kirsher * the MAC appropriately. All other PHY configuration changes are pushed 1025874aeea5SJeff Kirsher * through phy_op->set_settings(), and pushed asynchronously to the MAC 1026874aeea5SJeff Kirsher * through efx_monitor(). 1027874aeea5SJeff Kirsher * 1028874aeea5SJeff Kirsher * Callers must hold the mac_lock 1029874aeea5SJeff Kirsher */ 1030874aeea5SJeff Kirsher int __efx_reconfigure_port(struct efx_nic *efx) 1031874aeea5SJeff Kirsher { 1032874aeea5SJeff Kirsher enum efx_phy_mode phy_mode; 1033874aeea5SJeff Kirsher int rc; 1034874aeea5SJeff Kirsher 1035874aeea5SJeff Kirsher WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1036874aeea5SJeff Kirsher 1037874aeea5SJeff Kirsher /* Disable PHY transmit in mac level loopbacks */ 1038874aeea5SJeff Kirsher phy_mode = efx->phy_mode; 1039874aeea5SJeff Kirsher if (LOOPBACK_INTERNAL(efx)) 1040874aeea5SJeff Kirsher efx->phy_mode |= PHY_MODE_TX_DISABLED; 1041874aeea5SJeff Kirsher else 1042874aeea5SJeff Kirsher efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1043874aeea5SJeff Kirsher 1044874aeea5SJeff Kirsher rc = efx->type->reconfigure_port(efx); 1045874aeea5SJeff Kirsher 1046874aeea5SJeff Kirsher if (rc) 1047874aeea5SJeff Kirsher efx->phy_mode = phy_mode; 1048874aeea5SJeff Kirsher 1049874aeea5SJeff Kirsher return rc; 1050874aeea5SJeff Kirsher } 1051874aeea5SJeff Kirsher 1052874aeea5SJeff Kirsher /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1053874aeea5SJeff Kirsher * disabled. */ 1054874aeea5SJeff Kirsher int efx_reconfigure_port(struct efx_nic *efx) 1055874aeea5SJeff Kirsher { 1056874aeea5SJeff Kirsher int rc; 1057874aeea5SJeff Kirsher 1058874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 1059874aeea5SJeff Kirsher 1060874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1061874aeea5SJeff Kirsher rc = __efx_reconfigure_port(efx); 1062874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1063874aeea5SJeff Kirsher 1064874aeea5SJeff Kirsher return rc; 1065874aeea5SJeff Kirsher } 1066874aeea5SJeff Kirsher 1067874aeea5SJeff Kirsher /* Asynchronous work item for changing MAC promiscuity and multicast 1068874aeea5SJeff Kirsher * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1069874aeea5SJeff Kirsher * MAC directly. */ 1070874aeea5SJeff Kirsher static void efx_mac_work(struct work_struct *data) 1071874aeea5SJeff Kirsher { 1072874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1073874aeea5SJeff Kirsher 1074874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 107530b81cdaSBen Hutchings if (efx->port_enabled) 10760d322413SEdward Cree efx_mac_reconfigure(efx); 1077874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1078874aeea5SJeff Kirsher } 1079874aeea5SJeff Kirsher 1080874aeea5SJeff Kirsher static int efx_probe_port(struct efx_nic *efx) 1081874aeea5SJeff Kirsher { 1082874aeea5SJeff Kirsher int rc; 1083874aeea5SJeff Kirsher 1084874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1085874aeea5SJeff Kirsher 1086874aeea5SJeff Kirsher if (phy_flash_cfg) 1087874aeea5SJeff Kirsher efx->phy_mode = PHY_MODE_SPECIAL; 1088874aeea5SJeff Kirsher 1089874aeea5SJeff Kirsher /* Connect up MAC/PHY operations table */ 1090874aeea5SJeff Kirsher rc = efx->type->probe_port(efx); 1091874aeea5SJeff Kirsher if (rc) 1092874aeea5SJeff Kirsher return rc; 1093874aeea5SJeff Kirsher 1094e332bcb3SBen Hutchings /* Initialise MAC address to permanent address */ 1095cd84ff4dSEdward Cree ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1096874aeea5SJeff Kirsher 1097874aeea5SJeff Kirsher return 0; 1098874aeea5SJeff Kirsher } 1099874aeea5SJeff Kirsher 1100874aeea5SJeff Kirsher static int efx_init_port(struct efx_nic *efx) 1101874aeea5SJeff Kirsher { 1102874aeea5SJeff Kirsher int rc; 1103874aeea5SJeff Kirsher 1104874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1105874aeea5SJeff Kirsher 1106874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1107874aeea5SJeff Kirsher 1108874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 1109874aeea5SJeff Kirsher if (rc) 1110874aeea5SJeff Kirsher goto fail1; 1111874aeea5SJeff Kirsher 1112874aeea5SJeff Kirsher efx->port_initialized = true; 1113874aeea5SJeff Kirsher 1114874aeea5SJeff Kirsher /* Reconfigure the MAC before creating dma queues (required for 1115874aeea5SJeff Kirsher * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 11160d322413SEdward Cree efx_mac_reconfigure(efx); 1117874aeea5SJeff Kirsher 1118874aeea5SJeff Kirsher /* Ensure the PHY advertises the correct flow control settings */ 1119874aeea5SJeff Kirsher rc = efx->phy_op->reconfigure(efx); 1120267d9d73SEdward Cree if (rc && rc != -EPERM) 1121874aeea5SJeff Kirsher goto fail2; 1122874aeea5SJeff Kirsher 1123874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1124874aeea5SJeff Kirsher return 0; 1125874aeea5SJeff Kirsher 1126874aeea5SJeff Kirsher fail2: 1127874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1128874aeea5SJeff Kirsher fail1: 1129874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1130874aeea5SJeff Kirsher return rc; 1131874aeea5SJeff Kirsher } 1132874aeea5SJeff Kirsher 1133874aeea5SJeff Kirsher static void efx_start_port(struct efx_nic *efx) 1134874aeea5SJeff Kirsher { 1135874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1136874aeea5SJeff Kirsher BUG_ON(efx->port_enabled); 1137874aeea5SJeff Kirsher 1138874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1139874aeea5SJeff Kirsher efx->port_enabled = true; 1140874aeea5SJeff Kirsher 1141d615c039SBen Hutchings /* Ensure MAC ingress/egress is enabled */ 11420d322413SEdward Cree efx_mac_reconfigure(efx); 1143874aeea5SJeff Kirsher 1144874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1145874aeea5SJeff Kirsher } 1146874aeea5SJeff Kirsher 1147d615c039SBen Hutchings /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1148d615c039SBen Hutchings * and the async self-test, wait for them to finish and prevent them 1149d615c039SBen Hutchings * being scheduled again. This doesn't cover online resets, which 1150d615c039SBen Hutchings * should only be cancelled when removing the device. 1151d615c039SBen Hutchings */ 1152874aeea5SJeff Kirsher static void efx_stop_port(struct efx_nic *efx) 1153874aeea5SJeff Kirsher { 1154874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1155874aeea5SJeff Kirsher 1156d615c039SBen Hutchings EFX_ASSERT_RESET_SERIALISED(efx); 1157d615c039SBen Hutchings 1158874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 1159874aeea5SJeff Kirsher efx->port_enabled = false; 1160874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 1161874aeea5SJeff Kirsher 1162874aeea5SJeff Kirsher /* Serialise against efx_set_multicast_list() */ 1163874aeea5SJeff Kirsher netif_addr_lock_bh(efx->net_dev); 1164874aeea5SJeff Kirsher netif_addr_unlock_bh(efx->net_dev); 1165d615c039SBen Hutchings 1166d615c039SBen Hutchings cancel_delayed_work_sync(&efx->monitor_work); 1167d615c039SBen Hutchings efx_selftest_async_cancel(efx); 1168d615c039SBen Hutchings cancel_work_sync(&efx->mac_work); 1169874aeea5SJeff Kirsher } 1170874aeea5SJeff Kirsher 1171874aeea5SJeff Kirsher static void efx_fini_port(struct efx_nic *efx) 1172874aeea5SJeff Kirsher { 1173874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1174874aeea5SJeff Kirsher 1175874aeea5SJeff Kirsher if (!efx->port_initialized) 1176874aeea5SJeff Kirsher return; 1177874aeea5SJeff Kirsher 1178874aeea5SJeff Kirsher efx->phy_op->fini(efx); 1179874aeea5SJeff Kirsher efx->port_initialized = false; 1180874aeea5SJeff Kirsher 1181874aeea5SJeff Kirsher efx->link_state.up = false; 1182874aeea5SJeff Kirsher efx_link_status_changed(efx); 1183874aeea5SJeff Kirsher } 1184874aeea5SJeff Kirsher 1185874aeea5SJeff Kirsher static void efx_remove_port(struct efx_nic *efx) 1186874aeea5SJeff Kirsher { 1187874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1188874aeea5SJeff Kirsher 1189874aeea5SJeff Kirsher efx->type->remove_port(efx); 1190874aeea5SJeff Kirsher } 1191874aeea5SJeff Kirsher 1192874aeea5SJeff Kirsher /************************************************************************** 1193874aeea5SJeff Kirsher * 1194874aeea5SJeff Kirsher * NIC handling 1195874aeea5SJeff Kirsher * 1196874aeea5SJeff Kirsher **************************************************************************/ 1197874aeea5SJeff Kirsher 11980bcf4a64SBen Hutchings static LIST_HEAD(efx_primary_list); 11990bcf4a64SBen Hutchings static LIST_HEAD(efx_unassociated_list); 12000bcf4a64SBen Hutchings 12010bcf4a64SBen Hutchings static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 12020bcf4a64SBen Hutchings { 12030bcf4a64SBen Hutchings return left->type == right->type && 12040bcf4a64SBen Hutchings left->vpd_sn && right->vpd_sn && 12050bcf4a64SBen Hutchings !strcmp(left->vpd_sn, right->vpd_sn); 12060bcf4a64SBen Hutchings } 12070bcf4a64SBen Hutchings 12080bcf4a64SBen Hutchings static void efx_associate(struct efx_nic *efx) 12090bcf4a64SBen Hutchings { 12100bcf4a64SBen Hutchings struct efx_nic *other, *next; 12110bcf4a64SBen Hutchings 12120bcf4a64SBen Hutchings if (efx->primary == efx) { 12130bcf4a64SBen Hutchings /* Adding primary function; look for secondaries */ 12140bcf4a64SBen Hutchings 12150bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 12160bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_primary_list); 12170bcf4a64SBen Hutchings 12180bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx_unassociated_list, 12190bcf4a64SBen Hutchings node) { 12200bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 12210bcf4a64SBen Hutchings list_del(&other->node); 12220bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 12230bcf4a64SBen Hutchings "moving to secondary list of %s %s\n", 12240bcf4a64SBen Hutchings pci_name(efx->pci_dev), 12250bcf4a64SBen Hutchings efx->net_dev->name); 12260bcf4a64SBen Hutchings list_add_tail(&other->node, 12270bcf4a64SBen Hutchings &efx->secondary_list); 12280bcf4a64SBen Hutchings other->primary = efx; 12290bcf4a64SBen Hutchings } 12300bcf4a64SBen Hutchings } 12310bcf4a64SBen Hutchings } else { 12320bcf4a64SBen Hutchings /* Adding secondary function; look for primary */ 12330bcf4a64SBen Hutchings 12340bcf4a64SBen Hutchings list_for_each_entry(other, &efx_primary_list, node) { 12350bcf4a64SBen Hutchings if (efx_same_controller(efx, other)) { 12360bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 12370bcf4a64SBen Hutchings "adding to secondary list of %s %s\n", 12380bcf4a64SBen Hutchings pci_name(other->pci_dev), 12390bcf4a64SBen Hutchings other->net_dev->name); 12400bcf4a64SBen Hutchings list_add_tail(&efx->node, 12410bcf4a64SBen Hutchings &other->secondary_list); 12420bcf4a64SBen Hutchings efx->primary = other; 12430bcf4a64SBen Hutchings return; 12440bcf4a64SBen Hutchings } 12450bcf4a64SBen Hutchings } 12460bcf4a64SBen Hutchings 12470bcf4a64SBen Hutchings netif_dbg(efx, probe, efx->net_dev, 12480bcf4a64SBen Hutchings "adding to unassociated list\n"); 12490bcf4a64SBen Hutchings list_add_tail(&efx->node, &efx_unassociated_list); 12500bcf4a64SBen Hutchings } 12510bcf4a64SBen Hutchings } 12520bcf4a64SBen Hutchings 12530bcf4a64SBen Hutchings static void efx_dissociate(struct efx_nic *efx) 12540bcf4a64SBen Hutchings { 12550bcf4a64SBen Hutchings struct efx_nic *other, *next; 12560bcf4a64SBen Hutchings 12570bcf4a64SBen Hutchings list_del(&efx->node); 12580bcf4a64SBen Hutchings efx->primary = NULL; 12590bcf4a64SBen Hutchings 12600bcf4a64SBen Hutchings list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 12610bcf4a64SBen Hutchings list_del(&other->node); 12620bcf4a64SBen Hutchings netif_dbg(other, probe, other->net_dev, 12630bcf4a64SBen Hutchings "moving to unassociated list\n"); 12640bcf4a64SBen Hutchings list_add_tail(&other->node, &efx_unassociated_list); 12650bcf4a64SBen Hutchings other->primary = NULL; 12660bcf4a64SBen Hutchings } 12670bcf4a64SBen Hutchings } 12680bcf4a64SBen Hutchings 1269874aeea5SJeff Kirsher /* This configures the PCI device to enable I/O and DMA. */ 1270b1940451SAlex Maftei (amaftei) int efx_init_io(struct efx_nic *efx, int bar, dma_addr_t dma_mask, 1271b1940451SAlex Maftei (amaftei) unsigned int mem_map_size) 1272874aeea5SJeff Kirsher { 1273874aeea5SJeff Kirsher struct pci_dev *pci_dev = efx->pci_dev; 1274b1940451SAlex Maftei (amaftei) int rc; 1275874aeea5SJeff Kirsher 1276874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1277874aeea5SJeff Kirsher 1278874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 1279874aeea5SJeff Kirsher if (rc) { 1280874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1281874aeea5SJeff Kirsher "failed to enable PCI device\n"); 1282874aeea5SJeff Kirsher goto fail1; 1283874aeea5SJeff Kirsher } 1284874aeea5SJeff Kirsher 1285874aeea5SJeff Kirsher pci_set_master(pci_dev); 1286874aeea5SJeff Kirsher 128706e9552fSChristoph Hellwig /* Set the PCI DMA mask. Try all possibilities from our genuine mask 128806e9552fSChristoph Hellwig * down to 32 bits, because some architectures will allow 40 bit 1289874aeea5SJeff Kirsher * masks event though they reject 46 bit masks. 1290874aeea5SJeff Kirsher */ 1291874aeea5SJeff Kirsher while (dma_mask > 0x7fffffffUL) { 12929663dedaSRussell King rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1293e9e01846SBen Hutchings if (rc == 0) 1294874aeea5SJeff Kirsher break; 1295874aeea5SJeff Kirsher dma_mask >>= 1; 1296874aeea5SJeff Kirsher } 1297874aeea5SJeff Kirsher if (rc) { 1298874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1299874aeea5SJeff Kirsher "could not find a suitable DMA mask\n"); 1300874aeea5SJeff Kirsher goto fail2; 1301874aeea5SJeff Kirsher } 1302874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1303874aeea5SJeff Kirsher "using DMA mask %llx\n", (unsigned long long) dma_mask); 1304874aeea5SJeff Kirsher 130502246a7fSShradha Shah efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 130602246a7fSShradha Shah rc = pci_request_region(pci_dev, bar, "sfc"); 1307874aeea5SJeff Kirsher if (rc) { 1308874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1309874aeea5SJeff Kirsher "request for memory BAR failed\n"); 1310874aeea5SJeff Kirsher rc = -EIO; 1311874aeea5SJeff Kirsher goto fail3; 1312874aeea5SJeff Kirsher } 1313b105798fSBen Hutchings efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1314874aeea5SJeff Kirsher if (!efx->membase) { 1315874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 1316874aeea5SJeff Kirsher "could not map memory BAR at %llx+%x\n", 1317b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size); 1318874aeea5SJeff Kirsher rc = -ENOMEM; 1319874aeea5SJeff Kirsher goto fail4; 1320874aeea5SJeff Kirsher } 1321874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 1322874aeea5SJeff Kirsher "memory BAR at %llx+%x (virtual %p)\n", 1323b105798fSBen Hutchings (unsigned long long)efx->membase_phys, mem_map_size, 1324b105798fSBen Hutchings efx->membase); 1325874aeea5SJeff Kirsher 1326874aeea5SJeff Kirsher return 0; 1327874aeea5SJeff Kirsher 1328874aeea5SJeff Kirsher fail4: 132902246a7fSShradha Shah pci_release_region(efx->pci_dev, bar); 1330874aeea5SJeff Kirsher fail3: 1331874aeea5SJeff Kirsher efx->membase_phys = 0; 1332874aeea5SJeff Kirsher fail2: 1333874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1334874aeea5SJeff Kirsher fail1: 1335874aeea5SJeff Kirsher return rc; 1336874aeea5SJeff Kirsher } 1337874aeea5SJeff Kirsher 1338b1940451SAlex Maftei (amaftei) void efx_fini_io(struct efx_nic *efx, int bar) 1339874aeea5SJeff Kirsher { 1340874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1341874aeea5SJeff Kirsher 1342874aeea5SJeff Kirsher if (efx->membase) { 1343874aeea5SJeff Kirsher iounmap(efx->membase); 1344874aeea5SJeff Kirsher efx->membase = NULL; 1345874aeea5SJeff Kirsher } 1346874aeea5SJeff Kirsher 1347874aeea5SJeff Kirsher if (efx->membase_phys) { 134802246a7fSShradha Shah pci_release_region(efx->pci_dev, bar); 1349874aeea5SJeff Kirsher efx->membase_phys = 0; 1350874aeea5SJeff Kirsher } 1351874aeea5SJeff Kirsher 13526598dad2SDaniel Pieczko /* Don't disable bus-mastering if VFs are assigned */ 13536598dad2SDaniel Pieczko if (!pci_vfs_assigned(efx->pci_dev)) 1354874aeea5SJeff Kirsher pci_disable_device(efx->pci_dev); 1355874aeea5SJeff Kirsher } 1356874aeea5SJeff Kirsher 135742356d9aSEdward Cree void efx_set_default_rx_indir_table(struct efx_nic *efx, 135842356d9aSEdward Cree struct efx_rss_context *ctx) 1359267c0157SJon Cooper { 1360267c0157SJon Cooper size_t i; 1361267c0157SJon Cooper 136242356d9aSEdward Cree for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 136342356d9aSEdward Cree ctx->rx_indir_table[i] = 1364267c0157SJon Cooper ethtool_rxfh_indir_default(i, efx->rss_spread); 1365267c0157SJon Cooper } 1366267c0157SJon Cooper 1367a9a52506SBen Hutchings static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1368874aeea5SJeff Kirsher { 1369cdb08f8fSBen Hutchings cpumask_var_t thread_mask; 1370a16e5b24SBen Hutchings unsigned int count; 1371874aeea5SJeff Kirsher int cpu; 1372874aeea5SJeff Kirsher 1373cd2d5b52SBen Hutchings if (rss_cpus) { 1374cd2d5b52SBen Hutchings count = rss_cpus; 1375cd2d5b52SBen Hutchings } else { 1376cdb08f8fSBen Hutchings if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1377a9a52506SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1378a9a52506SBen Hutchings "RSS disabled due to allocation failure\n"); 1379874aeea5SJeff Kirsher return 1; 1380874aeea5SJeff Kirsher } 1381874aeea5SJeff Kirsher 1382874aeea5SJeff Kirsher count = 0; 1383874aeea5SJeff Kirsher for_each_online_cpu(cpu) { 1384cdb08f8fSBen Hutchings if (!cpumask_test_cpu(cpu, thread_mask)) { 1385874aeea5SJeff Kirsher ++count; 1386cdb08f8fSBen Hutchings cpumask_or(thread_mask, thread_mask, 138706931e62SBartosz Golaszewski topology_sibling_cpumask(cpu)); 1388874aeea5SJeff Kirsher } 1389874aeea5SJeff Kirsher } 1390874aeea5SJeff Kirsher 1391cdb08f8fSBen Hutchings free_cpumask_var(thread_mask); 1392cd2d5b52SBen Hutchings } 1393cd2d5b52SBen Hutchings 1394271a8b42SBert Kenward if (count > EFX_MAX_RX_QUEUES) { 1395271a8b42SBert Kenward netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1396271a8b42SBert Kenward "Reducing number of rx queues from %u to %u.\n", 1397271a8b42SBert Kenward count, EFX_MAX_RX_QUEUES); 1398271a8b42SBert Kenward count = EFX_MAX_RX_QUEUES; 1399271a8b42SBert Kenward } 1400271a8b42SBert Kenward 1401cd2d5b52SBen Hutchings /* If RSS is requested for the PF *and* VFs then we can't write RSS 1402cd2d5b52SBen Hutchings * table entries that are inaccessible to VFs 1403cd2d5b52SBen Hutchings */ 14047fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 14057fa8d547SShradha Shah if (efx->type->sriov_wanted) { 1406d98a4ffeSShradha Shah if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1407cd2d5b52SBen Hutchings count > efx_vf_size(efx)) { 1408cd2d5b52SBen Hutchings netif_warn(efx, probe, efx->net_dev, 1409cd2d5b52SBen Hutchings "Reducing number of RSS channels from %u to %u for " 1410cd2d5b52SBen Hutchings "VF support. Increase vf-msix-limit to use more " 1411cd2d5b52SBen Hutchings "channels on the PF.\n", 1412cd2d5b52SBen Hutchings count, efx_vf_size(efx)); 1413cd2d5b52SBen Hutchings count = efx_vf_size(efx); 1414cd2d5b52SBen Hutchings } 14157fa8d547SShradha Shah } 14167fa8d547SShradha Shah #endif 1417cd2d5b52SBen Hutchings 1418874aeea5SJeff Kirsher return count; 1419874aeea5SJeff Kirsher } 1420874aeea5SJeff Kirsher 14213990a8ffSCharles McLachlan static int efx_allocate_msix_channels(struct efx_nic *efx, 14223990a8ffSCharles McLachlan unsigned int max_channels, 14233990a8ffSCharles McLachlan unsigned int extra_channels, 14243990a8ffSCharles McLachlan unsigned int parallelism) 14253990a8ffSCharles McLachlan { 14263990a8ffSCharles McLachlan unsigned int n_channels = parallelism; 14273990a8ffSCharles McLachlan int vec_count; 14283990a8ffSCharles McLachlan int n_xdp_tx; 14293990a8ffSCharles McLachlan int n_xdp_ev; 14303990a8ffSCharles McLachlan 14313990a8ffSCharles McLachlan if (efx_separate_tx_channels) 14323990a8ffSCharles McLachlan n_channels *= 2; 14333990a8ffSCharles McLachlan n_channels += extra_channels; 14343990a8ffSCharles McLachlan 14353990a8ffSCharles McLachlan /* To allow XDP transmit to happen from arbitrary NAPI contexts 14363990a8ffSCharles McLachlan * we allocate a TX queue per CPU. We share event queues across 14373990a8ffSCharles McLachlan * multiple tx queues, assuming tx and ev queues are both 14383990a8ffSCharles McLachlan * maximum size. 14393990a8ffSCharles McLachlan */ 14403990a8ffSCharles McLachlan 14413990a8ffSCharles McLachlan n_xdp_tx = num_possible_cpus(); 14423990a8ffSCharles McLachlan n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_TXQ_TYPES); 14433990a8ffSCharles McLachlan 14448700aff0SEdward Cree vec_count = pci_msix_vec_count(efx->pci_dev); 14458700aff0SEdward Cree if (vec_count < 0) 14468700aff0SEdward Cree return vec_count; 14478700aff0SEdward Cree 14488700aff0SEdward Cree max_channels = min_t(unsigned int, vec_count, max_channels); 14498700aff0SEdward Cree 14503990a8ffSCharles McLachlan /* Check resources. 14513990a8ffSCharles McLachlan * We need a channel per event queue, plus a VI per tx queue. 14523990a8ffSCharles McLachlan * This may be more pessimistic than it needs to be. 14533990a8ffSCharles McLachlan */ 14543990a8ffSCharles McLachlan if (n_channels + n_xdp_ev > max_channels) { 14553990a8ffSCharles McLachlan netif_err(efx, drv, efx->net_dev, 14563990a8ffSCharles McLachlan "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", 14573990a8ffSCharles McLachlan n_xdp_ev, n_channels, max_channels); 14583990a8ffSCharles McLachlan efx->n_xdp_channels = 0; 14593990a8ffSCharles McLachlan efx->xdp_tx_per_channel = 0; 14603990a8ffSCharles McLachlan efx->xdp_tx_queue_count = 0; 14613990a8ffSCharles McLachlan } else { 14623990a8ffSCharles McLachlan efx->n_xdp_channels = n_xdp_ev; 14633990a8ffSCharles McLachlan efx->xdp_tx_per_channel = EFX_TXQ_TYPES; 14643990a8ffSCharles McLachlan efx->xdp_tx_queue_count = n_xdp_tx; 14653990a8ffSCharles McLachlan n_channels += n_xdp_ev; 14663990a8ffSCharles McLachlan netif_dbg(efx, drv, efx->net_dev, 14673990a8ffSCharles McLachlan "Allocating %d TX and %d event queues for XDP\n", 14683990a8ffSCharles McLachlan n_xdp_tx, n_xdp_ev); 14693990a8ffSCharles McLachlan } 14703990a8ffSCharles McLachlan 14713990a8ffSCharles McLachlan if (vec_count < n_channels) { 14723990a8ffSCharles McLachlan netif_err(efx, drv, efx->net_dev, 14733990a8ffSCharles McLachlan "WARNING: Insufficient MSI-X vectors available (%d < %u).\n", 14743990a8ffSCharles McLachlan vec_count, n_channels); 14753990a8ffSCharles McLachlan netif_err(efx, drv, efx->net_dev, 14763990a8ffSCharles McLachlan "WARNING: Performance may be reduced.\n"); 14773990a8ffSCharles McLachlan n_channels = vec_count; 14783990a8ffSCharles McLachlan } 14793990a8ffSCharles McLachlan 14808700aff0SEdward Cree n_channels = min(n_channels, max_channels); 14813990a8ffSCharles McLachlan 14828700aff0SEdward Cree efx->n_channels = n_channels; 14833990a8ffSCharles McLachlan 14843990a8ffSCharles McLachlan /* Ignore XDP tx channels when creating rx channels. */ 14853990a8ffSCharles McLachlan n_channels -= efx->n_xdp_channels; 14863990a8ffSCharles McLachlan 14873990a8ffSCharles McLachlan if (efx_separate_tx_channels) { 14883990a8ffSCharles McLachlan efx->n_tx_channels = 14893990a8ffSCharles McLachlan min(max(n_channels / 2, 1U), 14903990a8ffSCharles McLachlan efx->max_tx_channels); 14913990a8ffSCharles McLachlan efx->tx_channel_offset = 14923990a8ffSCharles McLachlan n_channels - efx->n_tx_channels; 14933990a8ffSCharles McLachlan efx->n_rx_channels = 14943990a8ffSCharles McLachlan max(n_channels - 14953990a8ffSCharles McLachlan efx->n_tx_channels, 1U); 14963990a8ffSCharles McLachlan } else { 14973990a8ffSCharles McLachlan efx->n_tx_channels = min(n_channels, efx->max_tx_channels); 14983990a8ffSCharles McLachlan efx->tx_channel_offset = 0; 14993990a8ffSCharles McLachlan efx->n_rx_channels = n_channels; 15003990a8ffSCharles McLachlan } 15013990a8ffSCharles McLachlan 15028700aff0SEdward Cree efx->n_rx_channels = min(efx->n_rx_channels, parallelism); 15038700aff0SEdward Cree efx->n_tx_channels = min(efx->n_tx_channels, parallelism); 15048700aff0SEdward Cree 15058700aff0SEdward Cree efx->xdp_channel_offset = n_channels; 15063990a8ffSCharles McLachlan 15073990a8ffSCharles McLachlan netif_dbg(efx, drv, efx->net_dev, 15083990a8ffSCharles McLachlan "Allocating %u RX channels\n", 15093990a8ffSCharles McLachlan efx->n_rx_channels); 15103990a8ffSCharles McLachlan 15113990a8ffSCharles McLachlan return efx->n_channels; 15123990a8ffSCharles McLachlan } 15133990a8ffSCharles McLachlan 1514874aeea5SJeff Kirsher /* Probe the number and type of interrupts we are able to obtain, and 1515874aeea5SJeff Kirsher * the resulting numbers of channels and RX queues. 1516874aeea5SJeff Kirsher */ 1517e1253f39SAlex Maftei (amaftei) int efx_probe_interrupts(struct efx_nic *efx) 1518874aeea5SJeff Kirsher { 15197f967c01SBen Hutchings unsigned int extra_channels = 0; 15208700aff0SEdward Cree unsigned int rss_spread; 15217f967c01SBen Hutchings unsigned int i, j; 1522a16e5b24SBen Hutchings int rc; 1523874aeea5SJeff Kirsher 15247f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 15257f967c01SBen Hutchings if (efx->extra_channel_type[i]) 15267f967c01SBen Hutchings ++extra_channels; 15277f967c01SBen Hutchings 1528874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 15293990a8ffSCharles McLachlan unsigned int parallelism = efx_wanted_parallelism(efx); 1530874aeea5SJeff Kirsher struct msix_entry xentries[EFX_MAX_CHANNELS]; 1531a16e5b24SBen Hutchings unsigned int n_channels; 1532874aeea5SJeff Kirsher 15333990a8ffSCharles McLachlan rc = efx_allocate_msix_channels(efx, efx->max_channels, 15343990a8ffSCharles McLachlan extra_channels, parallelism); 15353990a8ffSCharles McLachlan if (rc >= 0) { 15363990a8ffSCharles McLachlan n_channels = rc; 1537874aeea5SJeff Kirsher for (i = 0; i < n_channels; i++) 1538874aeea5SJeff Kirsher xentries[i].entry = i; 15393990a8ffSCharles McLachlan rc = pci_enable_msix_range(efx->pci_dev, xentries, 1, 15403990a8ffSCharles McLachlan n_channels); 15413990a8ffSCharles McLachlan } 1542184603d8SAlexander Gordeev if (rc < 0) { 1543184603d8SAlexander Gordeev /* Fall back to single channel MSI */ 1544184603d8SAlexander Gordeev netif_err(efx, drv, efx->net_dev, 1545184603d8SAlexander Gordeev "could not enable MSI-X\n"); 154662980cb6SAndrew Rybchenko if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 154762980cb6SAndrew Rybchenko efx->interrupt_mode = EFX_INT_MODE_MSI; 154862980cb6SAndrew Rybchenko else 154962980cb6SAndrew Rybchenko return rc; 1550184603d8SAlexander Gordeev } else if (rc < n_channels) { 1551874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1552874aeea5SJeff Kirsher "WARNING: Insufficient MSI-X vectors" 1553a16e5b24SBen Hutchings " available (%d < %u).\n", rc, n_channels); 1554874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1555874aeea5SJeff Kirsher "WARNING: Performance may be reduced.\n"); 1556874aeea5SJeff Kirsher n_channels = rc; 1557874aeea5SJeff Kirsher } 1558874aeea5SJeff Kirsher 1559184603d8SAlexander Gordeev if (rc > 0) { 15607f967c01SBen Hutchings for (i = 0; i < efx->n_channels; i++) 1561874aeea5SJeff Kirsher efx_get_channel(efx, i)->irq = 1562874aeea5SJeff Kirsher xentries[i].vector; 1563874aeea5SJeff Kirsher } 1564874aeea5SJeff Kirsher } 1565874aeea5SJeff Kirsher 1566874aeea5SJeff Kirsher /* Try single interrupt MSI */ 1567874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1568874aeea5SJeff Kirsher efx->n_channels = 1; 1569874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1570874aeea5SJeff Kirsher efx->n_tx_channels = 1; 15713990a8ffSCharles McLachlan efx->n_xdp_channels = 0; 15723990a8ffSCharles McLachlan efx->xdp_channel_offset = efx->n_channels; 1573874aeea5SJeff Kirsher rc = pci_enable_msi(efx->pci_dev); 1574874aeea5SJeff Kirsher if (rc == 0) { 1575874aeea5SJeff Kirsher efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1576874aeea5SJeff Kirsher } else { 1577874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 1578874aeea5SJeff Kirsher "could not enable MSI\n"); 157962980cb6SAndrew Rybchenko if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 1580874aeea5SJeff Kirsher efx->interrupt_mode = EFX_INT_MODE_LEGACY; 158162980cb6SAndrew Rybchenko else 158262980cb6SAndrew Rybchenko return rc; 1583874aeea5SJeff Kirsher } 1584874aeea5SJeff Kirsher } 1585874aeea5SJeff Kirsher 1586874aeea5SJeff Kirsher /* Assume legacy interrupts */ 1587874aeea5SJeff Kirsher if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1588b0fbdae1SShradha Shah efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1589874aeea5SJeff Kirsher efx->n_rx_channels = 1; 1590874aeea5SJeff Kirsher efx->n_tx_channels = 1; 15913990a8ffSCharles McLachlan efx->n_xdp_channels = 0; 15923990a8ffSCharles McLachlan efx->xdp_channel_offset = efx->n_channels; 1593874aeea5SJeff Kirsher efx->legacy_irq = efx->pci_dev->irq; 1594874aeea5SJeff Kirsher } 1595874aeea5SJeff Kirsher 15963990a8ffSCharles McLachlan /* Assign extra channels if possible, before XDP channels */ 15972935e3c3SEdward Cree efx->n_extra_tx_channels = 0; 15983990a8ffSCharles McLachlan j = efx->xdp_channel_offset; 15997f967c01SBen Hutchings for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 16007f967c01SBen Hutchings if (!efx->extra_channel_type[i]) 16017f967c01SBen Hutchings continue; 16028700aff0SEdward Cree if (j <= efx->tx_channel_offset + efx->n_tx_channels) { 16037f967c01SBen Hutchings efx->extra_channel_type[i]->handle_no_channel(efx); 16047f967c01SBen Hutchings } else { 16057f967c01SBen Hutchings --j; 16067f967c01SBen Hutchings efx_get_channel(efx, j)->type = 16077f967c01SBen Hutchings efx->extra_channel_type[i]; 16082935e3c3SEdward Cree if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 16092935e3c3SEdward Cree efx->n_extra_tx_channels++; 16107f967c01SBen Hutchings } 16117f967c01SBen Hutchings } 16127f967c01SBen Hutchings 16138700aff0SEdward Cree rss_spread = efx->n_rx_channels; 1614cd2d5b52SBen Hutchings /* RSS might be usable on VFs even if it is disabled on the PF */ 16157fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 16167fa8d547SShradha Shah if (efx->type->sriov_wanted) { 16178700aff0SEdward Cree efx->rss_spread = ((rss_spread > 1 || 1618d98a4ffeSShradha Shah !efx->type->sriov_wanted(efx)) ? 16198700aff0SEdward Cree rss_spread : efx_vf_size(efx)); 16207fa8d547SShradha Shah return 0; 16217fa8d547SShradha Shah } 16227fa8d547SShradha Shah #endif 16238700aff0SEdward Cree efx->rss_spread = rss_spread; 1624cd2d5b52SBen Hutchings 1625874aeea5SJeff Kirsher return 0; 1626874aeea5SJeff Kirsher } 1627874aeea5SJeff Kirsher 1628a83762d9SBert Kenward #if defined(CONFIG_SMP) 1629e1253f39SAlex Maftei (amaftei) void efx_set_interrupt_affinity(struct efx_nic *efx) 1630a83762d9SBert Kenward { 1631a83762d9SBert Kenward struct efx_channel *channel; 1632a83762d9SBert Kenward unsigned int cpu; 1633a83762d9SBert Kenward 1634a83762d9SBert Kenward efx_for_each_channel(channel, efx) { 1635a83762d9SBert Kenward cpu = cpumask_local_spread(channel->channel, 1636a83762d9SBert Kenward pcibus_to_node(efx->pci_dev->bus)); 1637a83762d9SBert Kenward irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 1638a83762d9SBert Kenward } 1639a83762d9SBert Kenward } 1640a83762d9SBert Kenward 1641e1253f39SAlex Maftei (amaftei) void efx_clear_interrupt_affinity(struct efx_nic *efx) 1642a83762d9SBert Kenward { 1643a83762d9SBert Kenward struct efx_channel *channel; 1644a83762d9SBert Kenward 1645a83762d9SBert Kenward efx_for_each_channel(channel, efx) 1646a83762d9SBert Kenward irq_set_affinity_hint(channel->irq, NULL); 1647a83762d9SBert Kenward } 1648a83762d9SBert Kenward #else 1649e1253f39SAlex Maftei (amaftei) void efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1650a83762d9SBert Kenward { 1651a83762d9SBert Kenward } 1652a83762d9SBert Kenward 1653e1253f39SAlex Maftei (amaftei) void efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 1654a83762d9SBert Kenward { 1655a83762d9SBert Kenward } 1656a83762d9SBert Kenward #endif /* CONFIG_SMP */ 1657a83762d9SBert Kenward 1658e1253f39SAlex Maftei (amaftei) int efx_soft_enable_interrupts(struct efx_nic *efx) 1659d8291187SBen Hutchings { 1660261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1661261e4d96SJon Cooper int rc; 1662d8291187SBen Hutchings 1663d8291187SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 1664d8291187SBen Hutchings 1665d8291187SBen Hutchings efx->irq_soft_enabled = true; 1666d8291187SBen Hutchings smp_wmb(); 1667d8291187SBen Hutchings 1668d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1669261e4d96SJon Cooper if (!channel->type->keep_eventq) { 1670261e4d96SJon Cooper rc = efx_init_eventq(channel); 1671261e4d96SJon Cooper if (rc) 1672261e4d96SJon Cooper goto fail; 1673261e4d96SJon Cooper } 1674d8291187SBen Hutchings efx_start_eventq(channel); 1675d8291187SBen Hutchings } 1676d8291187SBen Hutchings 1677d8291187SBen Hutchings efx_mcdi_mode_event(efx); 1678261e4d96SJon Cooper 1679261e4d96SJon Cooper return 0; 1680261e4d96SJon Cooper fail: 1681261e4d96SJon Cooper end_channel = channel; 1682261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1683261e4d96SJon Cooper if (channel == end_channel) 1684261e4d96SJon Cooper break; 1685261e4d96SJon Cooper efx_stop_eventq(channel); 1686261e4d96SJon Cooper if (!channel->type->keep_eventq) 1687261e4d96SJon Cooper efx_fini_eventq(channel); 1688261e4d96SJon Cooper } 1689261e4d96SJon Cooper 1690261e4d96SJon Cooper return rc; 1691d8291187SBen Hutchings } 1692d8291187SBen Hutchings 1693e1253f39SAlex Maftei (amaftei) void efx_soft_disable_interrupts(struct efx_nic *efx) 1694d8291187SBen Hutchings { 1695d8291187SBen Hutchings struct efx_channel *channel; 1696d8291187SBen Hutchings 1697d8291187SBen Hutchings if (efx->state == STATE_DISABLED) 1698d8291187SBen Hutchings return; 1699d8291187SBen Hutchings 1700d8291187SBen Hutchings efx_mcdi_mode_poll(efx); 1701d8291187SBen Hutchings 1702d8291187SBen Hutchings efx->irq_soft_enabled = false; 1703d8291187SBen Hutchings smp_wmb(); 1704d8291187SBen Hutchings 1705d8291187SBen Hutchings if (efx->legacy_irq) 1706d8291187SBen Hutchings synchronize_irq(efx->legacy_irq); 1707d8291187SBen Hutchings 1708d8291187SBen Hutchings efx_for_each_channel(channel, efx) { 1709d8291187SBen Hutchings if (channel->irq) 1710d8291187SBen Hutchings synchronize_irq(channel->irq); 1711d8291187SBen Hutchings 1712d8291187SBen Hutchings efx_stop_eventq(channel); 1713d8291187SBen Hutchings if (!channel->type->keep_eventq) 1714d8291187SBen Hutchings efx_fini_eventq(channel); 1715d8291187SBen Hutchings } 1716cade715fSBen Hutchings 1717cade715fSBen Hutchings /* Flush the asynchronous MCDI request queue */ 1718cade715fSBen Hutchings efx_mcdi_flush_async(efx); 1719d8291187SBen Hutchings } 1720d8291187SBen Hutchings 1721e1253f39SAlex Maftei (amaftei) int efx_enable_interrupts(struct efx_nic *efx) 17229f2cb71cSBen Hutchings { 1723261e4d96SJon Cooper struct efx_channel *channel, *end_channel; 1724261e4d96SJon Cooper int rc; 17259f2cb71cSBen Hutchings 17268b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 17278b7325b4SBen Hutchings 1728b28405b0SAlexandre Rames if (efx->eeh_disabled_legacy_irq) { 1729b28405b0SAlexandre Rames enable_irq(efx->legacy_irq); 1730b28405b0SAlexandre Rames efx->eeh_disabled_legacy_irq = false; 1731b28405b0SAlexandre Rames } 1732d8291187SBen Hutchings 173386094f7fSBen Hutchings efx->type->irq_enable_master(efx); 17349f2cb71cSBen Hutchings 17359f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1736261e4d96SJon Cooper if (channel->type->keep_eventq) { 1737261e4d96SJon Cooper rc = efx_init_eventq(channel); 1738261e4d96SJon Cooper if (rc) 1739261e4d96SJon Cooper goto fail; 1740261e4d96SJon Cooper } 17419f2cb71cSBen Hutchings } 17429f2cb71cSBen Hutchings 1743261e4d96SJon Cooper rc = efx_soft_enable_interrupts(efx); 1744261e4d96SJon Cooper if (rc) 1745261e4d96SJon Cooper goto fail; 1746261e4d96SJon Cooper 1747261e4d96SJon Cooper return 0; 1748261e4d96SJon Cooper 1749261e4d96SJon Cooper fail: 1750261e4d96SJon Cooper end_channel = channel; 1751261e4d96SJon Cooper efx_for_each_channel(channel, efx) { 1752261e4d96SJon Cooper if (channel == end_channel) 1753261e4d96SJon Cooper break; 1754261e4d96SJon Cooper if (channel->type->keep_eventq) 1755261e4d96SJon Cooper efx_fini_eventq(channel); 1756261e4d96SJon Cooper } 1757261e4d96SJon Cooper 1758261e4d96SJon Cooper efx->type->irq_disable_non_ev(efx); 1759261e4d96SJon Cooper 1760261e4d96SJon Cooper return rc; 17619f2cb71cSBen Hutchings } 17629f2cb71cSBen Hutchings 1763e1253f39SAlex Maftei (amaftei) void efx_disable_interrupts(struct efx_nic *efx) 17649f2cb71cSBen Hutchings { 17659f2cb71cSBen Hutchings struct efx_channel *channel; 17669f2cb71cSBen Hutchings 1767d8291187SBen Hutchings efx_soft_disable_interrupts(efx); 17689f2cb71cSBen Hutchings 17699f2cb71cSBen Hutchings efx_for_each_channel(channel, efx) { 1770d8291187SBen Hutchings if (channel->type->keep_eventq) 17719f2cb71cSBen Hutchings efx_fini_eventq(channel); 17729f2cb71cSBen Hutchings } 1773d8291187SBen Hutchings 177486094f7fSBen Hutchings efx->type->irq_disable_non_ev(efx); 17759f2cb71cSBen Hutchings } 17769f2cb71cSBen Hutchings 1777e1253f39SAlex Maftei (amaftei) void efx_remove_interrupts(struct efx_nic *efx) 1778874aeea5SJeff Kirsher { 1779874aeea5SJeff Kirsher struct efx_channel *channel; 1780874aeea5SJeff Kirsher 1781874aeea5SJeff Kirsher /* Remove MSI/MSI-X interrupts */ 1782874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 1783874aeea5SJeff Kirsher channel->irq = 0; 1784874aeea5SJeff Kirsher pci_disable_msi(efx->pci_dev); 1785874aeea5SJeff Kirsher pci_disable_msix(efx->pci_dev); 1786874aeea5SJeff Kirsher 1787874aeea5SJeff Kirsher /* Remove legacy interrupt */ 1788874aeea5SJeff Kirsher efx->legacy_irq = 0; 1789874aeea5SJeff Kirsher } 1790874aeea5SJeff Kirsher 1791e1253f39SAlex Maftei (amaftei) int efx_set_channels(struct efx_nic *efx) 1792874aeea5SJeff Kirsher { 1793874aeea5SJeff Kirsher struct efx_channel *channel; 1794874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 17953990a8ffSCharles McLachlan int xdp_queue_number; 1796874aeea5SJeff Kirsher 1797874aeea5SJeff Kirsher efx->tx_channel_offset = 1798b0fbdae1SShradha Shah efx_separate_tx_channels ? 1799b0fbdae1SShradha Shah efx->n_channels - efx->n_tx_channels : 0; 1800874aeea5SJeff Kirsher 18013990a8ffSCharles McLachlan if (efx->xdp_tx_queue_count) { 18023990a8ffSCharles McLachlan EFX_WARN_ON_PARANOID(efx->xdp_tx_queues); 18033990a8ffSCharles McLachlan 18043990a8ffSCharles McLachlan /* Allocate array for XDP TX queue lookup. */ 18053990a8ffSCharles McLachlan efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count, 18063990a8ffSCharles McLachlan sizeof(*efx->xdp_tx_queues), 18073990a8ffSCharles McLachlan GFP_KERNEL); 18083990a8ffSCharles McLachlan if (!efx->xdp_tx_queues) 18093990a8ffSCharles McLachlan return -ENOMEM; 18103990a8ffSCharles McLachlan } 18113990a8ffSCharles McLachlan 181279d68b37SStuart Hodgson /* We need to mark which channels really have RX and TX 181379d68b37SStuart Hodgson * queues, and adjust the TX queue numbers if we have separate 1814874aeea5SJeff Kirsher * RX-only and TX-only channels. 1815874aeea5SJeff Kirsher */ 18163990a8ffSCharles McLachlan xdp_queue_number = 0; 1817874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 181879d68b37SStuart Hodgson if (channel->channel < efx->n_rx_channels) 181979d68b37SStuart Hodgson channel->rx_queue.core_index = channel->channel; 182079d68b37SStuart Hodgson else 182179d68b37SStuart Hodgson channel->rx_queue.core_index = -1; 182279d68b37SStuart Hodgson 18233990a8ffSCharles McLachlan efx_for_each_channel_tx_queue(tx_queue, channel) { 1824874aeea5SJeff Kirsher tx_queue->queue -= (efx->tx_channel_offset * 1825874aeea5SJeff Kirsher EFX_TXQ_TYPES); 18263990a8ffSCharles McLachlan 18273990a8ffSCharles McLachlan if (efx_channel_is_xdp_tx(channel) && 18283990a8ffSCharles McLachlan xdp_queue_number < efx->xdp_tx_queue_count) { 18293990a8ffSCharles McLachlan efx->xdp_tx_queues[xdp_queue_number] = tx_queue; 18303990a8ffSCharles McLachlan xdp_queue_number++; 1831874aeea5SJeff Kirsher } 1832874aeea5SJeff Kirsher } 18333990a8ffSCharles McLachlan } 18343990a8ffSCharles McLachlan return 0; 18353990a8ffSCharles McLachlan } 1836874aeea5SJeff Kirsher 1837874aeea5SJeff Kirsher static int efx_probe_nic(struct efx_nic *efx) 1838874aeea5SJeff Kirsher { 1839874aeea5SJeff Kirsher int rc; 1840874aeea5SJeff Kirsher 1841874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1842874aeea5SJeff Kirsher 1843874aeea5SJeff Kirsher /* Carry out hardware-type specific initialisation */ 1844874aeea5SJeff Kirsher rc = efx->type->probe(efx); 1845874aeea5SJeff Kirsher if (rc) 1846874aeea5SJeff Kirsher return rc; 1847874aeea5SJeff Kirsher 1848b0fbdae1SShradha Shah do { 1849b0fbdae1SShradha Shah if (!efx->max_channels || !efx->max_tx_channels) { 1850b0fbdae1SShradha Shah netif_err(efx, drv, efx->net_dev, 1851b0fbdae1SShradha Shah "Insufficient resources to allocate" 1852b0fbdae1SShradha Shah " any channels\n"); 1853b0fbdae1SShradha Shah rc = -ENOSPC; 1854b0fbdae1SShradha Shah goto fail1; 1855b0fbdae1SShradha Shah } 1856b0fbdae1SShradha Shah 1857b0fbdae1SShradha Shah /* Determine the number of channels and queues by trying 1858b0fbdae1SShradha Shah * to hook in MSI-X interrupts. 1859b0fbdae1SShradha Shah */ 1860874aeea5SJeff Kirsher rc = efx_probe_interrupts(efx); 1861874aeea5SJeff Kirsher if (rc) 1862c15eed22SBen Hutchings goto fail1; 1863874aeea5SJeff Kirsher 18643990a8ffSCharles McLachlan rc = efx_set_channels(efx); 18653990a8ffSCharles McLachlan if (rc) 18663990a8ffSCharles McLachlan goto fail1; 186752ad762bSDaniel Pieczko 1868b0fbdae1SShradha Shah /* dimension_resources can fail with EAGAIN */ 1869c15eed22SBen Hutchings rc = efx->type->dimension_resources(efx); 1870b0fbdae1SShradha Shah if (rc != 0 && rc != -EAGAIN) 1871c15eed22SBen Hutchings goto fail2; 187228e47c49SBen Hutchings 1873b0fbdae1SShradha Shah if (rc == -EAGAIN) 1874b0fbdae1SShradha Shah /* try again with new max_channels */ 1875b0fbdae1SShradha Shah efx_remove_interrupts(efx); 1876b0fbdae1SShradha Shah 1877b0fbdae1SShradha Shah } while (rc == -EAGAIN); 1878b0fbdae1SShradha Shah 1879874aeea5SJeff Kirsher if (efx->n_channels > 1) 188042356d9aSEdward Cree netdev_rss_key_fill(efx->rss_context.rx_hash_key, 188142356d9aSEdward Cree sizeof(efx->rss_context.rx_hash_key)); 188242356d9aSEdward Cree efx_set_default_rx_indir_table(efx, &efx->rss_context); 1883874aeea5SJeff Kirsher 1884874aeea5SJeff Kirsher netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1885874aeea5SJeff Kirsher netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1886874aeea5SJeff Kirsher 1887874aeea5SJeff Kirsher /* Initialise the interrupt moderation settings */ 1888539de7c5SBert Kenward efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 18899e393b30SBen Hutchings efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 18909e393b30SBen Hutchings true); 1891874aeea5SJeff Kirsher 1892874aeea5SJeff Kirsher return 0; 1893874aeea5SJeff Kirsher 1894c15eed22SBen Hutchings fail2: 1895c15eed22SBen Hutchings efx_remove_interrupts(efx); 1896c15eed22SBen Hutchings fail1: 1897874aeea5SJeff Kirsher efx->type->remove(efx); 1898874aeea5SJeff Kirsher return rc; 1899874aeea5SJeff Kirsher } 1900874aeea5SJeff Kirsher 1901874aeea5SJeff Kirsher static void efx_remove_nic(struct efx_nic *efx) 1902874aeea5SJeff Kirsher { 1903874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1904874aeea5SJeff Kirsher 1905874aeea5SJeff Kirsher efx_remove_interrupts(efx); 1906874aeea5SJeff Kirsher efx->type->remove(efx); 1907874aeea5SJeff Kirsher } 1908874aeea5SJeff Kirsher 1909add72477SBen Hutchings static int efx_probe_filters(struct efx_nic *efx) 1910add72477SBen Hutchings { 1911add72477SBen Hutchings int rc; 1912add72477SBen Hutchings 19130d322413SEdward Cree init_rwsem(&efx->filter_sem); 1914d248953aSMartin Habets mutex_lock(&efx->mac_lock); 19150d322413SEdward Cree down_write(&efx->filter_sem); 1916add72477SBen Hutchings rc = efx->type->filter_table_probe(efx); 1917add72477SBen Hutchings if (rc) 19180d322413SEdward Cree goto out_unlock; 1919add72477SBen Hutchings 1920add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1921add72477SBen Hutchings if (efx->type->offload_features & NETIF_F_NTUPLE) { 1922faf8dcc1SJon Cooper struct efx_channel *channel; 1923faf8dcc1SJon Cooper int i, success = 1; 1924faf8dcc1SJon Cooper 1925faf8dcc1SJon Cooper efx_for_each_channel(channel, efx) { 1926faf8dcc1SJon Cooper channel->rps_flow_id = 1927faf8dcc1SJon Cooper kcalloc(efx->type->max_rx_ip_filters, 1928faf8dcc1SJon Cooper sizeof(*channel->rps_flow_id), 1929add72477SBen Hutchings GFP_KERNEL); 1930faf8dcc1SJon Cooper if (!channel->rps_flow_id) 1931faf8dcc1SJon Cooper success = 0; 1932faf8dcc1SJon Cooper else 1933faf8dcc1SJon Cooper for (i = 0; 1934faf8dcc1SJon Cooper i < efx->type->max_rx_ip_filters; 1935faf8dcc1SJon Cooper ++i) 1936faf8dcc1SJon Cooper channel->rps_flow_id[i] = 1937faf8dcc1SJon Cooper RPS_FLOW_ID_INVALID; 19388490e75cSEdward Cree channel->rfs_expire_index = 0; 19398490e75cSEdward Cree channel->rfs_filter_count = 0; 1940faf8dcc1SJon Cooper } 1941faf8dcc1SJon Cooper 1942faf8dcc1SJon Cooper if (!success) { 1943faf8dcc1SJon Cooper efx_for_each_channel(channel, efx) 1944faf8dcc1SJon Cooper kfree(channel->rps_flow_id); 1945add72477SBen Hutchings efx->type->filter_table_remove(efx); 19460d322413SEdward Cree rc = -ENOMEM; 19470d322413SEdward Cree goto out_unlock; 1948add72477SBen Hutchings } 1949add72477SBen Hutchings } 1950add72477SBen Hutchings #endif 19510d322413SEdward Cree out_unlock: 19520d322413SEdward Cree up_write(&efx->filter_sem); 1953d248953aSMartin Habets mutex_unlock(&efx->mac_lock); 19540d322413SEdward Cree return rc; 1955add72477SBen Hutchings } 1956add72477SBen Hutchings 1957add72477SBen Hutchings static void efx_remove_filters(struct efx_nic *efx) 1958add72477SBen Hutchings { 1959add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 1960faf8dcc1SJon Cooper struct efx_channel *channel; 1961faf8dcc1SJon Cooper 19628490e75cSEdward Cree efx_for_each_channel(channel, efx) { 19636fbc05e5SEdward Cree cancel_delayed_work_sync(&channel->filter_work); 1964faf8dcc1SJon Cooper kfree(channel->rps_flow_id); 19658490e75cSEdward Cree } 1966add72477SBen Hutchings #endif 19670d322413SEdward Cree down_write(&efx->filter_sem); 1968add72477SBen Hutchings efx->type->filter_table_remove(efx); 19690d322413SEdward Cree up_write(&efx->filter_sem); 1970add72477SBen Hutchings } 1971add72477SBen Hutchings 1972add72477SBen Hutchings 1973874aeea5SJeff Kirsher /************************************************************************** 1974874aeea5SJeff Kirsher * 1975874aeea5SJeff Kirsher * NIC startup/shutdown 1976874aeea5SJeff Kirsher * 1977874aeea5SJeff Kirsher *************************************************************************/ 1978874aeea5SJeff Kirsher 1979874aeea5SJeff Kirsher static int efx_probe_all(struct efx_nic *efx) 1980874aeea5SJeff Kirsher { 1981874aeea5SJeff Kirsher int rc; 1982874aeea5SJeff Kirsher 1983874aeea5SJeff Kirsher rc = efx_probe_nic(efx); 1984874aeea5SJeff Kirsher if (rc) { 1985874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1986874aeea5SJeff Kirsher goto fail1; 1987874aeea5SJeff Kirsher } 1988874aeea5SJeff Kirsher 1989874aeea5SJeff Kirsher rc = efx_probe_port(efx); 1990874aeea5SJeff Kirsher if (rc) { 1991874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1992874aeea5SJeff Kirsher goto fail2; 1993874aeea5SJeff Kirsher } 1994874aeea5SJeff Kirsher 19957e6d06f0SBen Hutchings BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 19967e6d06f0SBen Hutchings if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 19977e6d06f0SBen Hutchings rc = -EINVAL; 19987e6d06f0SBen Hutchings goto fail3; 19997e6d06f0SBen Hutchings } 2000874aeea5SJeff Kirsher efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 2001874aeea5SJeff Kirsher 20026d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 20036d8aaaf6SDaniel Pieczko rc = efx->type->vswitching_probe(efx); 20046d8aaaf6SDaniel Pieczko if (rc) /* not fatal; the PF will still work fine */ 20056d8aaaf6SDaniel Pieczko netif_warn(efx, probe, efx->net_dev, 20066d8aaaf6SDaniel Pieczko "failed to setup vswitching rc=%d;" 20076d8aaaf6SDaniel Pieczko " VFs may not function\n", rc); 20086d8aaaf6SDaniel Pieczko #endif 20096d8aaaf6SDaniel Pieczko 2010874aeea5SJeff Kirsher rc = efx_probe_filters(efx); 2011874aeea5SJeff Kirsher if (rc) { 2012874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 2013874aeea5SJeff Kirsher "failed to create filter tables\n"); 20146d8aaaf6SDaniel Pieczko goto fail4; 2015874aeea5SJeff Kirsher } 2016874aeea5SJeff Kirsher 20177f967c01SBen Hutchings rc = efx_probe_channels(efx); 20187f967c01SBen Hutchings if (rc) 20196d8aaaf6SDaniel Pieczko goto fail5; 20207f967c01SBen Hutchings 2021874aeea5SJeff Kirsher return 0; 2022874aeea5SJeff Kirsher 20236d8aaaf6SDaniel Pieczko fail5: 20247f967c01SBen Hutchings efx_remove_filters(efx); 20256d8aaaf6SDaniel Pieczko fail4: 20266d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 20276d8aaaf6SDaniel Pieczko efx->type->vswitching_remove(efx); 20286d8aaaf6SDaniel Pieczko #endif 2029874aeea5SJeff Kirsher fail3: 2030874aeea5SJeff Kirsher efx_remove_port(efx); 2031874aeea5SJeff Kirsher fail2: 2032874aeea5SJeff Kirsher efx_remove_nic(efx); 2033874aeea5SJeff Kirsher fail1: 2034874aeea5SJeff Kirsher return rc; 2035874aeea5SJeff Kirsher } 2036874aeea5SJeff Kirsher 20378b7325b4SBen Hutchings /* If the interface is supposed to be running but is not, start 20388b7325b4SBen Hutchings * the hardware and software data path, regular activity for the port 20398b7325b4SBen Hutchings * (MAC statistics, link polling, etc.) and schedule the port to be 20408b7325b4SBen Hutchings * reconfigured. Interrupts must already be enabled. This function 20418b7325b4SBen Hutchings * is safe to call multiple times, so long as the NIC is not disabled. 20428b7325b4SBen Hutchings * Requires the RTNL lock. 20439f2cb71cSBen Hutchings */ 2044e1253f39SAlex Maftei (amaftei) void efx_start_all(struct efx_nic *efx) 2045874aeea5SJeff Kirsher { 2046874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 20478b7325b4SBen Hutchings BUG_ON(efx->state == STATE_DISABLED); 2048874aeea5SJeff Kirsher 2049874aeea5SJeff Kirsher /* Check that it is appropriate to restart the interface. All 2050874aeea5SJeff Kirsher * of these flags are safe to read under just the rtnl lock */ 2051e283546cSEdward Cree if (efx->port_enabled || !netif_running(efx->net_dev) || 2052e283546cSEdward Cree efx->reset_pending) 2053874aeea5SJeff Kirsher return; 2054874aeea5SJeff Kirsher 2055874aeea5SJeff Kirsher efx_start_port(efx); 20569f2cb71cSBen Hutchings efx_start_datapath(efx); 2057874aeea5SJeff Kirsher 2058626950dbSAlexandre Rames /* Start the hardware monitor if there is one */ 2059626950dbSAlexandre Rames if (efx->type->monitor != NULL) 2060874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 2061874aeea5SJeff Kirsher efx_monitor_interval); 2062626950dbSAlexandre Rames 20635a6681e2SEdward Cree /* Link state detection is normally event-driven; we have 2064626950dbSAlexandre Rames * to poll now because we could have missed a change 2065626950dbSAlexandre Rames */ 2066874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2067874aeea5SJeff Kirsher if (efx->phy_op->poll(efx)) 2068874aeea5SJeff Kirsher efx_link_status_changed(efx); 2069874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2070874aeea5SJeff Kirsher 2071874aeea5SJeff Kirsher efx->type->start_stats(efx); 2072f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 2073f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 2074f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 2075f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 2076874aeea5SJeff Kirsher } 2077874aeea5SJeff Kirsher 20788b7325b4SBen Hutchings /* Quiesce the hardware and software data path, and regular activity 20798b7325b4SBen Hutchings * for the port without bringing the link down. Safe to call multiple 20808b7325b4SBen Hutchings * times with the NIC in almost any state, but interrupts should be 20818b7325b4SBen Hutchings * enabled. Requires the RTNL lock. 20828b7325b4SBen Hutchings */ 2083e1253f39SAlex Maftei (amaftei) void efx_stop_all(struct efx_nic *efx) 2084874aeea5SJeff Kirsher { 2085874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2086874aeea5SJeff Kirsher 2087874aeea5SJeff Kirsher /* port_enabled can be read safely under the rtnl lock */ 2088874aeea5SJeff Kirsher if (!efx->port_enabled) 2089874aeea5SJeff Kirsher return; 2090874aeea5SJeff Kirsher 2091f8f3b5aeSJon Cooper /* update stats before we go down so we can accurately count 2092f8f3b5aeSJon Cooper * rx_nodesc_drops 2093f8f3b5aeSJon Cooper */ 2094f8f3b5aeSJon Cooper efx->type->pull_stats(efx); 2095f8f3b5aeSJon Cooper spin_lock_bh(&efx->stats_lock); 2096f8f3b5aeSJon Cooper efx->type->update_stats(efx, NULL, NULL); 2097f8f3b5aeSJon Cooper spin_unlock_bh(&efx->stats_lock); 2098874aeea5SJeff Kirsher efx->type->stop_stats(efx); 2099874aeea5SJeff Kirsher efx_stop_port(efx); 2100874aeea5SJeff Kirsher 210129c69a48SBen Hutchings /* Stop the kernel transmit interface. This is only valid if 210229c69a48SBen Hutchings * the device is stopped or detached; otherwise the watchdog 210329c69a48SBen Hutchings * may fire immediately. 210429c69a48SBen Hutchings */ 210529c69a48SBen Hutchings WARN_ON(netif_running(efx->net_dev) && 210629c69a48SBen Hutchings netif_device_present(efx->net_dev)); 21079f2cb71cSBen Hutchings netif_tx_disable(efx->net_dev); 21089f2cb71cSBen Hutchings 21099f2cb71cSBen Hutchings efx_stop_datapath(efx); 2110874aeea5SJeff Kirsher } 2111874aeea5SJeff Kirsher 2112874aeea5SJeff Kirsher static void efx_remove_all(struct efx_nic *efx) 2113874aeea5SJeff Kirsher { 2114e45a4fedSCharles McLachlan rtnl_lock(); 2115e45a4fedSCharles McLachlan efx_xdp_setup_prog(efx, NULL); 2116e45a4fedSCharles McLachlan rtnl_unlock(); 2117e45a4fedSCharles McLachlan 2118874aeea5SJeff Kirsher efx_remove_channels(efx); 21197f967c01SBen Hutchings efx_remove_filters(efx); 21206d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 21216d8aaaf6SDaniel Pieczko efx->type->vswitching_remove(efx); 21226d8aaaf6SDaniel Pieczko #endif 2123874aeea5SJeff Kirsher efx_remove_port(efx); 2124874aeea5SJeff Kirsher efx_remove_nic(efx); 2125874aeea5SJeff Kirsher } 2126874aeea5SJeff Kirsher 2127874aeea5SJeff Kirsher /************************************************************************** 2128874aeea5SJeff Kirsher * 2129874aeea5SJeff Kirsher * Interrupt moderation 2130874aeea5SJeff Kirsher * 2131874aeea5SJeff Kirsher **************************************************************************/ 2132539de7c5SBert Kenward unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 2133874aeea5SJeff Kirsher { 2134b548f976SBen Hutchings if (usecs == 0) 2135b548f976SBen Hutchings return 0; 2136539de7c5SBert Kenward if (usecs * 1000 < efx->timer_quantum_ns) 2137874aeea5SJeff Kirsher return 1; /* never round down to 0 */ 2138539de7c5SBert Kenward return usecs * 1000 / efx->timer_quantum_ns; 2139539de7c5SBert Kenward } 2140539de7c5SBert Kenward 2141539de7c5SBert Kenward unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 2142539de7c5SBert Kenward { 2143539de7c5SBert Kenward /* We must round up when converting ticks to microseconds 2144539de7c5SBert Kenward * because we round down when converting the other way. 2145539de7c5SBert Kenward */ 2146539de7c5SBert Kenward return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 2147874aeea5SJeff Kirsher } 2148874aeea5SJeff Kirsher 2149874aeea5SJeff Kirsher /* Set interrupt moderation parameters */ 21509e393b30SBen Hutchings int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 21519e393b30SBen Hutchings unsigned int rx_usecs, bool rx_adaptive, 21529e393b30SBen Hutchings bool rx_may_override_tx) 2153874aeea5SJeff Kirsher { 2154874aeea5SJeff Kirsher struct efx_channel *channel; 2155d95e329aSBert Kenward unsigned int timer_max_us; 2156d95e329aSBert Kenward 2157874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2158874aeea5SJeff Kirsher 2159d95e329aSBert Kenward timer_max_us = efx->timer_max_ns / 1000; 2160d95e329aSBert Kenward 2161d95e329aSBert Kenward if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 21629e393b30SBen Hutchings return -EINVAL; 21639e393b30SBen Hutchings 2164539de7c5SBert Kenward if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 21659e393b30SBen Hutchings !rx_may_override_tx) { 21669e393b30SBen Hutchings netif_err(efx, drv, efx->net_dev, "Channels are shared. " 21679e393b30SBen Hutchings "RX and TX IRQ moderation must be equal\n"); 21689e393b30SBen Hutchings return -EINVAL; 21699e393b30SBen Hutchings } 21709e393b30SBen Hutchings 2171874aeea5SJeff Kirsher efx->irq_rx_adaptive = rx_adaptive; 2172539de7c5SBert Kenward efx->irq_rx_moderation_us = rx_usecs; 2173874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2174874aeea5SJeff Kirsher if (efx_channel_has_rx_queue(channel)) 2175539de7c5SBert Kenward channel->irq_moderation_us = rx_usecs; 2176874aeea5SJeff Kirsher else if (efx_channel_has_tx_queues(channel)) 2177539de7c5SBert Kenward channel->irq_moderation_us = tx_usecs; 21783990a8ffSCharles McLachlan else if (efx_channel_is_xdp_tx(channel)) 21793990a8ffSCharles McLachlan channel->irq_moderation_us = tx_usecs; 2180874aeea5SJeff Kirsher } 21819e393b30SBen Hutchings 21829e393b30SBen Hutchings return 0; 2183874aeea5SJeff Kirsher } 2184874aeea5SJeff Kirsher 2185a0c4faf5SBen Hutchings void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2186a0c4faf5SBen Hutchings unsigned int *rx_usecs, bool *rx_adaptive) 2187a0c4faf5SBen Hutchings { 2188a0c4faf5SBen Hutchings *rx_adaptive = efx->irq_rx_adaptive; 2189539de7c5SBert Kenward *rx_usecs = efx->irq_rx_moderation_us; 2190a0c4faf5SBen Hutchings 2191a0c4faf5SBen Hutchings /* If channels are shared between RX and TX, so is IRQ 2192a0c4faf5SBen Hutchings * moderation. Otherwise, IRQ moderation is the same for all 2193a0c4faf5SBen Hutchings * TX channels and is not adaptive. 2194a0c4faf5SBen Hutchings */ 2195539de7c5SBert Kenward if (efx->tx_channel_offset == 0) { 2196a0c4faf5SBen Hutchings *tx_usecs = *rx_usecs; 2197539de7c5SBert Kenward } else { 2198539de7c5SBert Kenward struct efx_channel *tx_channel; 2199539de7c5SBert Kenward 2200539de7c5SBert Kenward tx_channel = efx->channel[efx->tx_channel_offset]; 2201539de7c5SBert Kenward *tx_usecs = tx_channel->irq_moderation_us; 2202539de7c5SBert Kenward } 2203a0c4faf5SBen Hutchings } 2204a0c4faf5SBen Hutchings 2205874aeea5SJeff Kirsher /************************************************************************** 2206874aeea5SJeff Kirsher * 2207874aeea5SJeff Kirsher * Hardware monitor 2208874aeea5SJeff Kirsher * 2209874aeea5SJeff Kirsher **************************************************************************/ 2210874aeea5SJeff Kirsher 2211874aeea5SJeff Kirsher /* Run periodically off the general workqueue */ 2212874aeea5SJeff Kirsher static void efx_monitor(struct work_struct *data) 2213874aeea5SJeff Kirsher { 2214874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, 2215874aeea5SJeff Kirsher monitor_work.work); 2216874aeea5SJeff Kirsher 2217874aeea5SJeff Kirsher netif_vdbg(efx, timer, efx->net_dev, 2218874aeea5SJeff Kirsher "hardware monitor executing on CPU %d\n", 2219874aeea5SJeff Kirsher raw_smp_processor_id()); 2220874aeea5SJeff Kirsher BUG_ON(efx->type->monitor == NULL); 2221874aeea5SJeff Kirsher 2222874aeea5SJeff Kirsher /* If the mac_lock is already held then it is likely a port 2223874aeea5SJeff Kirsher * reconfiguration is already in place, which will likely do 2224874aeea5SJeff Kirsher * most of the work of monitor() anyway. */ 2225874aeea5SJeff Kirsher if (mutex_trylock(&efx->mac_lock)) { 2226874aeea5SJeff Kirsher if (efx->port_enabled) 2227874aeea5SJeff Kirsher efx->type->monitor(efx); 2228874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2229874aeea5SJeff Kirsher } 2230874aeea5SJeff Kirsher 2231874aeea5SJeff Kirsher queue_delayed_work(efx->workqueue, &efx->monitor_work, 2232874aeea5SJeff Kirsher efx_monitor_interval); 2233874aeea5SJeff Kirsher } 2234874aeea5SJeff Kirsher 2235874aeea5SJeff Kirsher /************************************************************************** 2236874aeea5SJeff Kirsher * 2237874aeea5SJeff Kirsher * ioctls 2238874aeea5SJeff Kirsher * 2239874aeea5SJeff Kirsher *************************************************************************/ 2240874aeea5SJeff Kirsher 2241874aeea5SJeff Kirsher /* Net device ioctl 2242874aeea5SJeff Kirsher * Context: process, rtnl_lock() held. 2243874aeea5SJeff Kirsher */ 2244874aeea5SJeff Kirsher static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2245874aeea5SJeff Kirsher { 2246874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2247874aeea5SJeff Kirsher struct mii_ioctl_data *data = if_mii(ifr); 2248874aeea5SJeff Kirsher 22497c236c43SStuart Hodgson if (cmd == SIOCSHWTSTAMP) 2250433dc9b3SBen Hutchings return efx_ptp_set_ts_config(efx, ifr); 2251433dc9b3SBen Hutchings if (cmd == SIOCGHWTSTAMP) 2252433dc9b3SBen Hutchings return efx_ptp_get_ts_config(efx, ifr); 22537c236c43SStuart Hodgson 2254874aeea5SJeff Kirsher /* Convert phy_id from older PRTAD/DEVAD format */ 2255874aeea5SJeff Kirsher if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2256874aeea5SJeff Kirsher (data->phy_id & 0xfc00) == 0x0400) 2257874aeea5SJeff Kirsher data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2258874aeea5SJeff Kirsher 2259874aeea5SJeff Kirsher return mdio_mii_ioctl(&efx->mdio, data, cmd); 2260874aeea5SJeff Kirsher } 2261874aeea5SJeff Kirsher 2262874aeea5SJeff Kirsher /************************************************************************** 2263874aeea5SJeff Kirsher * 2264874aeea5SJeff Kirsher * NAPI interface 2265874aeea5SJeff Kirsher * 2266874aeea5SJeff Kirsher **************************************************************************/ 2267874aeea5SJeff Kirsher 2268e1253f39SAlex Maftei (amaftei) void efx_init_napi_channel(struct efx_channel *channel) 2269874aeea5SJeff Kirsher { 22707f967c01SBen Hutchings struct efx_nic *efx = channel->efx; 2271874aeea5SJeff Kirsher 2272874aeea5SJeff Kirsher channel->napi_dev = efx->net_dev; 2273874aeea5SJeff Kirsher netif_napi_add(channel->napi_dev, &channel->napi_str, 2274874aeea5SJeff Kirsher efx_poll, napi_weight); 2275874aeea5SJeff Kirsher } 22767f967c01SBen Hutchings 2277e1253f39SAlex Maftei (amaftei) void efx_init_napi(struct efx_nic *efx) 22787f967c01SBen Hutchings { 22797f967c01SBen Hutchings struct efx_channel *channel; 22807f967c01SBen Hutchings 22817f967c01SBen Hutchings efx_for_each_channel(channel, efx) 22827f967c01SBen Hutchings efx_init_napi_channel(channel); 2283874aeea5SJeff Kirsher } 2284874aeea5SJeff Kirsher 2285e1253f39SAlex Maftei (amaftei) void efx_fini_napi_channel(struct efx_channel *channel) 2286874aeea5SJeff Kirsher { 2287973334a1SEric Dumazet if (channel->napi_dev) 2288874aeea5SJeff Kirsher netif_napi_del(&channel->napi_str); 2289973334a1SEric Dumazet 2290874aeea5SJeff Kirsher channel->napi_dev = NULL; 2291874aeea5SJeff Kirsher } 2292874aeea5SJeff Kirsher 2293e1253f39SAlex Maftei (amaftei) void efx_fini_napi(struct efx_nic *efx) 2294874aeea5SJeff Kirsher { 2295874aeea5SJeff Kirsher struct efx_channel *channel; 2296874aeea5SJeff Kirsher 2297874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) 2298874aeea5SJeff Kirsher efx_fini_napi_channel(channel); 2299874aeea5SJeff Kirsher } 2300874aeea5SJeff Kirsher 2301874aeea5SJeff Kirsher /************************************************************************** 2302874aeea5SJeff Kirsher * 2303874aeea5SJeff Kirsher * Kernel net device interface 2304874aeea5SJeff Kirsher * 2305874aeea5SJeff Kirsher *************************************************************************/ 2306874aeea5SJeff Kirsher 2307874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2308e340be92SShradha Shah int efx_net_open(struct net_device *net_dev) 2309874aeea5SJeff Kirsher { 2310874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 23118b7325b4SBen Hutchings int rc; 23128b7325b4SBen Hutchings 2313874aeea5SJeff Kirsher netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2314874aeea5SJeff Kirsher raw_smp_processor_id()); 2315874aeea5SJeff Kirsher 23168b7325b4SBen Hutchings rc = efx_check_disabled(efx); 23178b7325b4SBen Hutchings if (rc) 23188b7325b4SBen Hutchings return rc; 2319874aeea5SJeff Kirsher if (efx->phy_mode & PHY_MODE_SPECIAL) 2320874aeea5SJeff Kirsher return -EBUSY; 2321874aeea5SJeff Kirsher if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2322874aeea5SJeff Kirsher return -EIO; 2323874aeea5SJeff Kirsher 2324874aeea5SJeff Kirsher /* Notify the kernel of the link state polled during driver load, 2325874aeea5SJeff Kirsher * before the monitor starts running */ 2326874aeea5SJeff Kirsher efx_link_status_changed(efx); 2327874aeea5SJeff Kirsher 2328874aeea5SJeff Kirsher efx_start_all(efx); 23299c568fd8SPeter Dunning if (efx->state == STATE_DISABLED || efx->reset_pending) 23309c568fd8SPeter Dunning netif_device_detach(efx->net_dev); 2331dd40781eSBen Hutchings efx_selftest_async_start(efx); 2332874aeea5SJeff Kirsher return 0; 2333874aeea5SJeff Kirsher } 2334874aeea5SJeff Kirsher 2335874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. 2336874aeea5SJeff Kirsher * Note that the kernel will ignore our return code; this method 2337874aeea5SJeff Kirsher * should really be a void. 2338874aeea5SJeff Kirsher */ 2339e340be92SShradha Shah int efx_net_stop(struct net_device *net_dev) 2340874aeea5SJeff Kirsher { 2341874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2342874aeea5SJeff Kirsher 2343874aeea5SJeff Kirsher netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2344874aeea5SJeff Kirsher raw_smp_processor_id()); 2345874aeea5SJeff Kirsher 2346874aeea5SJeff Kirsher /* Stop the device and flush all the channels */ 2347874aeea5SJeff Kirsher efx_stop_all(efx); 2348874aeea5SJeff Kirsher 2349874aeea5SJeff Kirsher return 0; 2350874aeea5SJeff Kirsher } 2351874aeea5SJeff Kirsher 2352874aeea5SJeff Kirsher /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2353bc1f4470Sstephen hemminger static void efx_net_stats(struct net_device *net_dev, 23542aa9ef11SBen Hutchings struct rtnl_link_stats64 *stats) 2355874aeea5SJeff Kirsher { 2356874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2357874aeea5SJeff Kirsher 2358874aeea5SJeff Kirsher spin_lock_bh(&efx->stats_lock); 2359cd0ecc9aSBen Hutchings efx->type->update_stats(efx, NULL, stats); 23601cb34522SBen Hutchings spin_unlock_bh(&efx->stats_lock); 2361874aeea5SJeff Kirsher } 2362874aeea5SJeff Kirsher 2363874aeea5SJeff Kirsher /* Context: netif_tx_lock held, BHs disabled. */ 23640290bd29SMichael S. Tsirkin static void efx_watchdog(struct net_device *net_dev, unsigned int txqueue) 2365874aeea5SJeff Kirsher { 2366874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2367874aeea5SJeff Kirsher 2368874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 2369874aeea5SJeff Kirsher "TX stuck with port_enabled=%d: resetting channels\n", 2370874aeea5SJeff Kirsher efx->port_enabled); 2371874aeea5SJeff Kirsher 2372874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2373874aeea5SJeff Kirsher } 2374874aeea5SJeff Kirsher 2375e45a4fedSCharles McLachlan static unsigned int efx_xdp_max_mtu(struct efx_nic *efx) 2376e45a4fedSCharles McLachlan { 2377e45a4fedSCharles McLachlan /* The maximum MTU that we can fit in a single page, allowing for 2378e45a4fedSCharles McLachlan * framing, overhead and XDP headroom. 2379e45a4fedSCharles McLachlan */ 2380e45a4fedSCharles McLachlan int overhead = EFX_MAX_FRAME_LEN(0) + sizeof(struct efx_rx_page_state) + 2381e45a4fedSCharles McLachlan efx->rx_prefix_size + efx->type->rx_buffer_padding + 2382e45a4fedSCharles McLachlan efx->rx_ip_align + XDP_PACKET_HEADROOM; 2383e45a4fedSCharles McLachlan 2384e45a4fedSCharles McLachlan return PAGE_SIZE - overhead; 2385e45a4fedSCharles McLachlan } 2386874aeea5SJeff Kirsher 2387874aeea5SJeff Kirsher /* Context: process, rtnl_lock() held. */ 2388874aeea5SJeff Kirsher static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2389874aeea5SJeff Kirsher { 2390874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 23918b7325b4SBen Hutchings int rc; 2392874aeea5SJeff Kirsher 23938b7325b4SBen Hutchings rc = efx_check_disabled(efx); 23948b7325b4SBen Hutchings if (rc) 23958b7325b4SBen Hutchings return rc; 2396874aeea5SJeff Kirsher 2397e45a4fedSCharles McLachlan if (rtnl_dereference(efx->xdp_prog) && 2398e45a4fedSCharles McLachlan new_mtu > efx_xdp_max_mtu(efx)) { 2399e45a4fedSCharles McLachlan netif_err(efx, drv, efx->net_dev, 2400e45a4fedSCharles McLachlan "Requested MTU of %d too big for XDP (max: %d)\n", 2401e45a4fedSCharles McLachlan new_mtu, efx_xdp_max_mtu(efx)); 2402e45a4fedSCharles McLachlan return -EINVAL; 2403e45a4fedSCharles McLachlan } 2404e45a4fedSCharles McLachlan 2405874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2406874aeea5SJeff Kirsher 240729c69a48SBen Hutchings efx_device_detach_sync(efx); 240829c69a48SBen Hutchings efx_stop_all(efx); 240929c69a48SBen Hutchings 2410874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 2411874aeea5SJeff Kirsher net_dev->mtu = new_mtu; 24120d322413SEdward Cree efx_mac_reconfigure(efx); 2413874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2414874aeea5SJeff Kirsher 2415874aeea5SJeff Kirsher efx_start_all(efx); 24169c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 24176c8eef4aSBen Hutchings return 0; 2418874aeea5SJeff Kirsher } 2419874aeea5SJeff Kirsher 2420874aeea5SJeff Kirsher static int efx_set_mac_address(struct net_device *net_dev, void *data) 2421874aeea5SJeff Kirsher { 2422874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2423874aeea5SJeff Kirsher struct sockaddr *addr = data; 2424e0b3ae30SBen Hutchings u8 *new_addr = addr->sa_data; 2425cfc77c2fSShradha Shah u8 old_addr[6]; 2426cfc77c2fSShradha Shah int rc; 2427874aeea5SJeff Kirsher 2428874aeea5SJeff Kirsher if (!is_valid_ether_addr(new_addr)) { 2429874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2430874aeea5SJeff Kirsher "invalid ethernet MAC address requested: %pM\n", 2431874aeea5SJeff Kirsher new_addr); 2432504f9b5aSDanny Kukawka return -EADDRNOTAVAIL; 2433874aeea5SJeff Kirsher } 2434874aeea5SJeff Kirsher 2435cfc77c2fSShradha Shah /* save old address */ 2436cfc77c2fSShradha Shah ether_addr_copy(old_addr, net_dev->dev_addr); 2437cd84ff4dSEdward Cree ether_addr_copy(net_dev->dev_addr, new_addr); 2438910c8789SShradha Shah if (efx->type->set_mac_address) { 2439910c8789SShradha Shah rc = efx->type->set_mac_address(efx); 2440cfc77c2fSShradha Shah if (rc) { 2441cfc77c2fSShradha Shah ether_addr_copy(net_dev->dev_addr, old_addr); 2442cfc77c2fSShradha Shah return rc; 2443cfc77c2fSShradha Shah } 2444cfc77c2fSShradha Shah } 2445874aeea5SJeff Kirsher 2446874aeea5SJeff Kirsher /* Reconfigure the MAC */ 2447874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 24480d322413SEdward Cree efx_mac_reconfigure(efx); 2449874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2450874aeea5SJeff Kirsher 2451874aeea5SJeff Kirsher return 0; 2452874aeea5SJeff Kirsher } 2453874aeea5SJeff Kirsher 2454874aeea5SJeff Kirsher /* Context: netif_addr_lock held, BHs disabled. */ 24550fca8c97SBen Hutchings static void efx_set_rx_mode(struct net_device *net_dev) 2456874aeea5SJeff Kirsher { 2457874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 2458874aeea5SJeff Kirsher 2459874aeea5SJeff Kirsher if (efx->port_enabled) 2460874aeea5SJeff Kirsher queue_work(efx->workqueue, &efx->mac_work); 2461874aeea5SJeff Kirsher /* Otherwise efx_start_port() will do this */ 2462874aeea5SJeff Kirsher } 2463874aeea5SJeff Kirsher 2464c8f44affSMichał Mirosław static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2465874aeea5SJeff Kirsher { 2466874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 24674a53ea8aSAndrew Rybchenko int rc; 2468874aeea5SJeff Kirsher 2469874aeea5SJeff Kirsher /* If disabling RX n-tuple filtering, clear existing filters */ 24704a53ea8aSAndrew Rybchenko if (net_dev->features & ~data & NETIF_F_NTUPLE) { 24714a53ea8aSAndrew Rybchenko rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 24724a53ea8aSAndrew Rybchenko if (rc) 24734a53ea8aSAndrew Rybchenko return rc; 24744a53ea8aSAndrew Rybchenko } 24754a53ea8aSAndrew Rybchenko 24766978729fSEdward Cree /* If Rx VLAN filter is changed, update filters via mac_reconfigure. 24776978729fSEdward Cree * If rx-fcs is changed, mac_reconfigure updates that too. 24786978729fSEdward Cree */ 24796978729fSEdward Cree if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER | 24806978729fSEdward Cree NETIF_F_RXFCS)) { 24814a53ea8aSAndrew Rybchenko /* efx_set_rx_mode() will schedule MAC work to update filters 24824a53ea8aSAndrew Rybchenko * when a new features are finally set in net_dev. 24834a53ea8aSAndrew Rybchenko */ 24844a53ea8aSAndrew Rybchenko efx_set_rx_mode(net_dev); 24854a53ea8aSAndrew Rybchenko } 2486874aeea5SJeff Kirsher 2487874aeea5SJeff Kirsher return 0; 2488874aeea5SJeff Kirsher } 2489874aeea5SJeff Kirsher 2490b40296fcSWei Yongjun static int efx_get_phys_port_id(struct net_device *net_dev, 249108a7b29bSBert Kenward struct netdev_phys_item_id *ppid) 249208a7b29bSBert Kenward { 249308a7b29bSBert Kenward struct efx_nic *efx = netdev_priv(net_dev); 249408a7b29bSBert Kenward 249508a7b29bSBert Kenward if (efx->type->get_phys_port_id) 249608a7b29bSBert Kenward return efx->type->get_phys_port_id(efx, ppid); 249708a7b29bSBert Kenward else 249808a7b29bSBert Kenward return -EOPNOTSUPP; 249908a7b29bSBert Kenward } 250008a7b29bSBert Kenward 2501ac019f08SBert Kenward static int efx_get_phys_port_name(struct net_device *net_dev, 2502ac019f08SBert Kenward char *name, size_t len) 2503ac019f08SBert Kenward { 2504ac019f08SBert Kenward struct efx_nic *efx = netdev_priv(net_dev); 2505ac019f08SBert Kenward 2506ac019f08SBert Kenward if (snprintf(name, len, "p%u", efx->port_num) >= len) 2507ac019f08SBert Kenward return -EINVAL; 2508ac019f08SBert Kenward return 0; 2509ac019f08SBert Kenward } 2510ac019f08SBert Kenward 25114a53ea8aSAndrew Rybchenko static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 25124a53ea8aSAndrew Rybchenko { 25134a53ea8aSAndrew Rybchenko struct efx_nic *efx = netdev_priv(net_dev); 25144a53ea8aSAndrew Rybchenko 25154a53ea8aSAndrew Rybchenko if (efx->type->vlan_rx_add_vid) 25164a53ea8aSAndrew Rybchenko return efx->type->vlan_rx_add_vid(efx, proto, vid); 25174a53ea8aSAndrew Rybchenko else 25184a53ea8aSAndrew Rybchenko return -EOPNOTSUPP; 25194a53ea8aSAndrew Rybchenko } 25204a53ea8aSAndrew Rybchenko 25214a53ea8aSAndrew Rybchenko static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 25224a53ea8aSAndrew Rybchenko { 25234a53ea8aSAndrew Rybchenko struct efx_nic *efx = netdev_priv(net_dev); 25244a53ea8aSAndrew Rybchenko 25254a53ea8aSAndrew Rybchenko if (efx->type->vlan_rx_kill_vid) 25264a53ea8aSAndrew Rybchenko return efx->type->vlan_rx_kill_vid(efx, proto, vid); 25274a53ea8aSAndrew Rybchenko else 25284a53ea8aSAndrew Rybchenko return -EOPNOTSUPP; 25294a53ea8aSAndrew Rybchenko } 25304a53ea8aSAndrew Rybchenko 2531e5fbd977SJon Cooper static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in) 2532e5fbd977SJon Cooper { 2533e5fbd977SJon Cooper switch (in) { 2534e5fbd977SJon Cooper case UDP_TUNNEL_TYPE_VXLAN: 2535e5fbd977SJon Cooper return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 2536e5fbd977SJon Cooper case UDP_TUNNEL_TYPE_GENEVE: 2537e5fbd977SJon Cooper return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 2538e5fbd977SJon Cooper default: 2539e5fbd977SJon Cooper return -1; 2540e5fbd977SJon Cooper } 2541e5fbd977SJon Cooper } 2542e5fbd977SJon Cooper 2543e5fbd977SJon Cooper static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti) 2544e5fbd977SJon Cooper { 2545e5fbd977SJon Cooper struct efx_nic *efx = netdev_priv(dev); 2546e5fbd977SJon Cooper struct efx_udp_tunnel tnl; 2547e5fbd977SJon Cooper int efx_tunnel_type; 2548e5fbd977SJon Cooper 2549e5fbd977SJon Cooper efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2550e5fbd977SJon Cooper if (efx_tunnel_type < 0) 2551e5fbd977SJon Cooper return; 2552e5fbd977SJon Cooper 2553e5fbd977SJon Cooper tnl.type = (u16)efx_tunnel_type; 2554e5fbd977SJon Cooper tnl.port = ti->port; 2555e5fbd977SJon Cooper 2556e5fbd977SJon Cooper if (efx->type->udp_tnl_add_port) 2557e5fbd977SJon Cooper (void)efx->type->udp_tnl_add_port(efx, tnl); 2558e5fbd977SJon Cooper } 2559e5fbd977SJon Cooper 2560e5fbd977SJon Cooper static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti) 2561e5fbd977SJon Cooper { 2562e5fbd977SJon Cooper struct efx_nic *efx = netdev_priv(dev); 2563e5fbd977SJon Cooper struct efx_udp_tunnel tnl; 2564e5fbd977SJon Cooper int efx_tunnel_type; 2565e5fbd977SJon Cooper 2566e5fbd977SJon Cooper efx_tunnel_type = efx_udp_tunnel_type_map(ti->type); 2567e5fbd977SJon Cooper if (efx_tunnel_type < 0) 2568e5fbd977SJon Cooper return; 2569e5fbd977SJon Cooper 2570e5fbd977SJon Cooper tnl.type = (u16)efx_tunnel_type; 2571e5fbd977SJon Cooper tnl.port = ti->port; 2572e5fbd977SJon Cooper 2573c04ca616SDan Carpenter if (efx->type->udp_tnl_del_port) 2574e5fbd977SJon Cooper (void)efx->type->udp_tnl_del_port(efx, tnl); 2575e5fbd977SJon Cooper } 2576e5fbd977SJon Cooper 25777fa8d547SShradha Shah static const struct net_device_ops efx_netdev_ops = { 2578874aeea5SJeff Kirsher .ndo_open = efx_net_open, 2579874aeea5SJeff Kirsher .ndo_stop = efx_net_stop, 2580874aeea5SJeff Kirsher .ndo_get_stats64 = efx_net_stats, 2581874aeea5SJeff Kirsher .ndo_tx_timeout = efx_watchdog, 2582874aeea5SJeff Kirsher .ndo_start_xmit = efx_hard_start_xmit, 2583874aeea5SJeff Kirsher .ndo_validate_addr = eth_validate_addr, 2584874aeea5SJeff Kirsher .ndo_do_ioctl = efx_ioctl, 2585874aeea5SJeff Kirsher .ndo_change_mtu = efx_change_mtu, 2586874aeea5SJeff Kirsher .ndo_set_mac_address = efx_set_mac_address, 25870fca8c97SBen Hutchings .ndo_set_rx_mode = efx_set_rx_mode, 2588874aeea5SJeff Kirsher .ndo_set_features = efx_set_features, 25894a53ea8aSAndrew Rybchenko .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 25904a53ea8aSAndrew Rybchenko .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2591cd2d5b52SBen Hutchings #ifdef CONFIG_SFC_SRIOV 25927fa8d547SShradha Shah .ndo_set_vf_mac = efx_sriov_set_vf_mac, 25937fa8d547SShradha Shah .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 25947fa8d547SShradha Shah .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 25957fa8d547SShradha Shah .ndo_get_vf_config = efx_sriov_get_vf_config, 25964392dc69SEdward Cree .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2597cd2d5b52SBen Hutchings #endif 259808a7b29bSBert Kenward .ndo_get_phys_port_id = efx_get_phys_port_id, 2599ac019f08SBert Kenward .ndo_get_phys_port_name = efx_get_phys_port_name, 2600874aeea5SJeff Kirsher .ndo_setup_tc = efx_setup_tc, 2601874aeea5SJeff Kirsher #ifdef CONFIG_RFS_ACCEL 2602874aeea5SJeff Kirsher .ndo_rx_flow_steer = efx_filter_rfs, 2603874aeea5SJeff Kirsher #endif 2604e5fbd977SJon Cooper .ndo_udp_tunnel_add = efx_udp_tunnel_add, 2605e5fbd977SJon Cooper .ndo_udp_tunnel_del = efx_udp_tunnel_del, 2606dfe44c1fSCharles McLachlan .ndo_xdp_xmit = efx_xdp_xmit, 2607e45a4fedSCharles McLachlan .ndo_bpf = efx_xdp 2608874aeea5SJeff Kirsher }; 2609874aeea5SJeff Kirsher 2610e45a4fedSCharles McLachlan static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog) 2611e45a4fedSCharles McLachlan { 2612e45a4fedSCharles McLachlan struct bpf_prog *old_prog; 2613e45a4fedSCharles McLachlan 2614e45a4fedSCharles McLachlan if (efx->xdp_rxq_info_failed) { 2615e45a4fedSCharles McLachlan netif_err(efx, drv, efx->net_dev, 2616e45a4fedSCharles McLachlan "Unable to bind XDP program due to previous failure of rxq_info\n"); 2617e45a4fedSCharles McLachlan return -EINVAL; 2618e45a4fedSCharles McLachlan } 2619e45a4fedSCharles McLachlan 2620e45a4fedSCharles McLachlan if (prog && efx->net_dev->mtu > efx_xdp_max_mtu(efx)) { 2621e45a4fedSCharles McLachlan netif_err(efx, drv, efx->net_dev, 2622e45a4fedSCharles McLachlan "Unable to configure XDP with MTU of %d (max: %d)\n", 2623e45a4fedSCharles McLachlan efx->net_dev->mtu, efx_xdp_max_mtu(efx)); 2624e45a4fedSCharles McLachlan return -EINVAL; 2625e45a4fedSCharles McLachlan } 2626e45a4fedSCharles McLachlan 2627e45a4fedSCharles McLachlan old_prog = rtnl_dereference(efx->xdp_prog); 2628e45a4fedSCharles McLachlan rcu_assign_pointer(efx->xdp_prog, prog); 2629e45a4fedSCharles McLachlan /* Release the reference that was originally passed by the caller. */ 2630e45a4fedSCharles McLachlan if (old_prog) 2631e45a4fedSCharles McLachlan bpf_prog_put(old_prog); 2632e45a4fedSCharles McLachlan 2633e45a4fedSCharles McLachlan return 0; 2634e45a4fedSCharles McLachlan } 2635e45a4fedSCharles McLachlan 2636e45a4fedSCharles McLachlan /* Context: process, rtnl_lock() held. */ 2637e45a4fedSCharles McLachlan static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2638e45a4fedSCharles McLachlan { 2639e45a4fedSCharles McLachlan struct efx_nic *efx = netdev_priv(dev); 2640e45a4fedSCharles McLachlan struct bpf_prog *xdp_prog; 2641e45a4fedSCharles McLachlan 2642e45a4fedSCharles McLachlan switch (xdp->command) { 2643e45a4fedSCharles McLachlan case XDP_SETUP_PROG: 2644e45a4fedSCharles McLachlan return efx_xdp_setup_prog(efx, xdp->prog); 2645e45a4fedSCharles McLachlan case XDP_QUERY_PROG: 2646e45a4fedSCharles McLachlan xdp_prog = rtnl_dereference(efx->xdp_prog); 2647e45a4fedSCharles McLachlan xdp->prog_id = xdp_prog ? xdp_prog->aux->id : 0; 2648e45a4fedSCharles McLachlan return 0; 2649e45a4fedSCharles McLachlan default: 2650e45a4fedSCharles McLachlan return -EINVAL; 2651e45a4fedSCharles McLachlan } 2652e45a4fedSCharles McLachlan } 2653e45a4fedSCharles McLachlan 2654dfe44c1fSCharles McLachlan static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs, 2655dfe44c1fSCharles McLachlan u32 flags) 2656dfe44c1fSCharles McLachlan { 2657dfe44c1fSCharles McLachlan struct efx_nic *efx = netdev_priv(dev); 2658dfe44c1fSCharles McLachlan 2659dfe44c1fSCharles McLachlan if (!netif_running(dev)) 2660dfe44c1fSCharles McLachlan return -EINVAL; 2661dfe44c1fSCharles McLachlan 2662dfe44c1fSCharles McLachlan return efx_xdp_tx_buffers(efx, n, xdpfs, flags & XDP_XMIT_FLUSH); 2663dfe44c1fSCharles McLachlan } 2664dfe44c1fSCharles McLachlan 2665874aeea5SJeff Kirsher static void efx_update_name(struct efx_nic *efx) 2666874aeea5SJeff Kirsher { 2667874aeea5SJeff Kirsher strcpy(efx->name, efx->net_dev->name); 2668874aeea5SJeff Kirsher efx_mtd_rename(efx); 2669874aeea5SJeff Kirsher efx_set_channel_names(efx); 2670874aeea5SJeff Kirsher } 2671874aeea5SJeff Kirsher 2672874aeea5SJeff Kirsher static int efx_netdev_event(struct notifier_block *this, 2673874aeea5SJeff Kirsher unsigned long event, void *ptr) 2674874aeea5SJeff Kirsher { 2675351638e7SJiri Pirko struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2676874aeea5SJeff Kirsher 26777fa8d547SShradha Shah if ((net_dev->netdev_ops == &efx_netdev_ops) && 2678874aeea5SJeff Kirsher event == NETDEV_CHANGENAME) 2679874aeea5SJeff Kirsher efx_update_name(netdev_priv(net_dev)); 2680874aeea5SJeff Kirsher 2681874aeea5SJeff Kirsher return NOTIFY_DONE; 2682874aeea5SJeff Kirsher } 2683874aeea5SJeff Kirsher 2684874aeea5SJeff Kirsher static struct notifier_block efx_netdev_notifier = { 2685874aeea5SJeff Kirsher .notifier_call = efx_netdev_event, 2686874aeea5SJeff Kirsher }; 2687874aeea5SJeff Kirsher 2688874aeea5SJeff Kirsher static ssize_t 2689874aeea5SJeff Kirsher show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2690874aeea5SJeff Kirsher { 26913e03a8baSChuhong Yuan struct efx_nic *efx = dev_get_drvdata(dev); 2692874aeea5SJeff Kirsher return sprintf(buf, "%d\n", efx->phy_type); 2693874aeea5SJeff Kirsher } 2694776fbcc9SBen Hutchings static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2695874aeea5SJeff Kirsher 2696e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2697e7fef9b4SEdward Cree static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2698e7fef9b4SEdward Cree char *buf) 2699e7fef9b4SEdward Cree { 27003e03a8baSChuhong Yuan struct efx_nic *efx = dev_get_drvdata(dev); 2701e7fef9b4SEdward Cree struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2702e7fef9b4SEdward Cree 2703e7fef9b4SEdward Cree return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2704e7fef9b4SEdward Cree } 2705e7fef9b4SEdward Cree static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2706e7fef9b4SEdward Cree const char *buf, size_t count) 2707e7fef9b4SEdward Cree { 27083e03a8baSChuhong Yuan struct efx_nic *efx = dev_get_drvdata(dev); 2709e7fef9b4SEdward Cree struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2710e7fef9b4SEdward Cree bool enable = count > 0 && *buf != '0'; 2711e7fef9b4SEdward Cree 2712e7fef9b4SEdward Cree mcdi->logging_enabled = enable; 2713e7fef9b4SEdward Cree return count; 2714e7fef9b4SEdward Cree } 2715e7fef9b4SEdward Cree static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2716e7fef9b4SEdward Cree #endif 2717e7fef9b4SEdward Cree 2718874aeea5SJeff Kirsher static int efx_register_netdev(struct efx_nic *efx) 2719874aeea5SJeff Kirsher { 2720874aeea5SJeff Kirsher struct net_device *net_dev = efx->net_dev; 2721874aeea5SJeff Kirsher struct efx_channel *channel; 2722874aeea5SJeff Kirsher int rc; 2723874aeea5SJeff Kirsher 2724874aeea5SJeff Kirsher net_dev->watchdog_timeo = 5 * HZ; 2725874aeea5SJeff Kirsher net_dev->irq = efx->pci_dev->irq; 27267fa8d547SShradha Shah net_dev->netdev_ops = &efx_netdev_ops; 27277fa8d547SShradha Shah if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 27288127d661SBen Hutchings net_dev->priv_flags |= IFF_UNICAST_FLT; 27297ad24ea4SWilfried Klaebe net_dev->ethtool_ops = &efx_ethtool_ops; 27307e6d06f0SBen Hutchings net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2731cd94e519SBert Kenward net_dev->min_mtu = EFX_MIN_MTU; 2732cd94e519SBert Kenward net_dev->max_mtu = EFX_MAX_MTU; 2733874aeea5SJeff Kirsher 2734874aeea5SJeff Kirsher rtnl_lock(); 2735874aeea5SJeff Kirsher 27367153f623SBen Hutchings /* Enable resets to be scheduled and check whether any were 27377153f623SBen Hutchings * already requested. If so, the NIC is probably hosed so we 27387153f623SBen Hutchings * abort. 27397153f623SBen Hutchings */ 27407153f623SBen Hutchings efx->state = STATE_READY; 27417153f623SBen Hutchings smp_mb(); /* ensure we change state before checking reset_pending */ 27427153f623SBen Hutchings if (efx->reset_pending) { 27437153f623SBen Hutchings netif_err(efx, probe, efx->net_dev, 27447153f623SBen Hutchings "aborting probe due to scheduled reset\n"); 27457153f623SBen Hutchings rc = -EIO; 27467153f623SBen Hutchings goto fail_locked; 27477153f623SBen Hutchings } 27487153f623SBen Hutchings 2749874aeea5SJeff Kirsher rc = dev_alloc_name(net_dev, net_dev->name); 2750874aeea5SJeff Kirsher if (rc < 0) 2751874aeea5SJeff Kirsher goto fail_locked; 2752874aeea5SJeff Kirsher efx_update_name(efx); 2753874aeea5SJeff Kirsher 27548f8b3d51SBen Hutchings /* Always start with carrier off; PHY events will detect the link */ 27558f8b3d51SBen Hutchings netif_carrier_off(net_dev); 27568f8b3d51SBen Hutchings 2757874aeea5SJeff Kirsher rc = register_netdevice(net_dev); 2758874aeea5SJeff Kirsher if (rc) 2759874aeea5SJeff Kirsher goto fail_locked; 2760874aeea5SJeff Kirsher 2761874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 2762874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 2763874aeea5SJeff Kirsher efx_for_each_channel_tx_queue(tx_queue, channel) 2764874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 2765874aeea5SJeff Kirsher } 2766874aeea5SJeff Kirsher 27670bcf4a64SBen Hutchings efx_associate(efx); 27680bcf4a64SBen Hutchings 2769874aeea5SJeff Kirsher rtnl_unlock(); 2770874aeea5SJeff Kirsher 2771874aeea5SJeff Kirsher rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2772874aeea5SJeff Kirsher if (rc) { 2773874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2774874aeea5SJeff Kirsher "failed to init net dev attributes\n"); 2775874aeea5SJeff Kirsher goto fail_registered; 2776874aeea5SJeff Kirsher } 2777e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2778e7fef9b4SEdward Cree rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2779e7fef9b4SEdward Cree if (rc) { 2780e7fef9b4SEdward Cree netif_err(efx, drv, efx->net_dev, 2781e7fef9b4SEdward Cree "failed to init net dev attributes\n"); 2782e7fef9b4SEdward Cree goto fail_attr_mcdi_logging; 2783e7fef9b4SEdward Cree } 2784e7fef9b4SEdward Cree #endif 2785874aeea5SJeff Kirsher 2786874aeea5SJeff Kirsher return 0; 2787874aeea5SJeff Kirsher 2788e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2789e7fef9b4SEdward Cree fail_attr_mcdi_logging: 2790e7fef9b4SEdward Cree device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2791e7fef9b4SEdward Cree #endif 27927153f623SBen Hutchings fail_registered: 27937153f623SBen Hutchings rtnl_lock(); 27940bcf4a64SBen Hutchings efx_dissociate(efx); 27957153f623SBen Hutchings unregister_netdevice(net_dev); 2796874aeea5SJeff Kirsher fail_locked: 27977153f623SBen Hutchings efx->state = STATE_UNINIT; 2798874aeea5SJeff Kirsher rtnl_unlock(); 2799874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2800874aeea5SJeff Kirsher return rc; 2801874aeea5SJeff Kirsher } 2802874aeea5SJeff Kirsher 2803874aeea5SJeff Kirsher static void efx_unregister_netdev(struct efx_nic *efx) 2804874aeea5SJeff Kirsher { 2805874aeea5SJeff Kirsher if (!efx->net_dev) 2806874aeea5SJeff Kirsher return; 2807874aeea5SJeff Kirsher 2808874aeea5SJeff Kirsher BUG_ON(netdev_priv(efx->net_dev) != efx); 2809874aeea5SJeff Kirsher 2810e7fef9b4SEdward Cree if (efx_dev_registered(efx)) { 2811874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2812e7fef9b4SEdward Cree #ifdef CONFIG_SFC_MCDI_LOGGING 2813e7fef9b4SEdward Cree device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2814e7fef9b4SEdward Cree #endif 2815874aeea5SJeff Kirsher device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2816e7fef9b4SEdward Cree unregister_netdev(efx->net_dev); 2817e7fef9b4SEdward Cree } 2818874aeea5SJeff Kirsher } 2819874aeea5SJeff Kirsher 2820874aeea5SJeff Kirsher /************************************************************************** 2821874aeea5SJeff Kirsher * 2822874aeea5SJeff Kirsher * Device reset and suspend 2823874aeea5SJeff Kirsher * 2824874aeea5SJeff Kirsher **************************************************************************/ 2825874aeea5SJeff Kirsher 2826874aeea5SJeff Kirsher /* Tears down the entire software state and most of the hardware state 2827874aeea5SJeff Kirsher * before reset. */ 2828874aeea5SJeff Kirsher void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2829874aeea5SJeff Kirsher { 2830874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2831874aeea5SJeff Kirsher 2832e283546cSEdward Cree if (method == RESET_TYPE_MCDI_TIMEOUT) 2833e283546cSEdward Cree efx->type->prepare_flr(efx); 2834e283546cSEdward Cree 2835874aeea5SJeff Kirsher efx_stop_all(efx); 2836d8291187SBen Hutchings efx_disable_interrupts(efx); 28375642ceefSBen Hutchings 28385642ceefSBen Hutchings mutex_lock(&efx->mac_lock); 2839193f2003SBert Kenward down_write(&efx->filter_sem); 2840e0a65e3cSEdward Cree mutex_lock(&efx->rss_lock); 2841087e9025SJon Cooper if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2842087e9025SJon Cooper method != RESET_TYPE_DATAPATH) 2843874aeea5SJeff Kirsher efx->phy_op->fini(efx); 2844874aeea5SJeff Kirsher efx->type->fini(efx); 2845874aeea5SJeff Kirsher } 2846874aeea5SJeff Kirsher 2847874aeea5SJeff Kirsher /* This function will always ensure that the locks acquired in 2848874aeea5SJeff Kirsher * efx_reset_down() are released. A failure return code indicates 2849874aeea5SJeff Kirsher * that we were unable to reinitialise the hardware, and the 2850874aeea5SJeff Kirsher * driver should be disabled. If ok is false, then the rx and tx 2851874aeea5SJeff Kirsher * engines are not restarted, pending a RESET_DISABLE. */ 2852874aeea5SJeff Kirsher int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2853874aeea5SJeff Kirsher { 2854874aeea5SJeff Kirsher int rc; 2855874aeea5SJeff Kirsher 2856874aeea5SJeff Kirsher EFX_ASSERT_RESET_SERIALISED(efx); 2857874aeea5SJeff Kirsher 2858e283546cSEdward Cree if (method == RESET_TYPE_MCDI_TIMEOUT) 2859e283546cSEdward Cree efx->type->finish_flr(efx); 2860e283546cSEdward Cree 2861e283546cSEdward Cree /* Ensure that SRAM is initialised even if we're disabling the device */ 2862874aeea5SJeff Kirsher rc = efx->type->init(efx); 2863874aeea5SJeff Kirsher if (rc) { 2864874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2865874aeea5SJeff Kirsher goto fail; 2866874aeea5SJeff Kirsher } 2867874aeea5SJeff Kirsher 2868874aeea5SJeff Kirsher if (!ok) 2869874aeea5SJeff Kirsher goto fail; 2870874aeea5SJeff Kirsher 2871087e9025SJon Cooper if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2872087e9025SJon Cooper method != RESET_TYPE_DATAPATH) { 2873874aeea5SJeff Kirsher rc = efx->phy_op->init(efx); 2874874aeea5SJeff Kirsher if (rc) 2875874aeea5SJeff Kirsher goto fail; 2876267d9d73SEdward Cree rc = efx->phy_op->reconfigure(efx); 2877267d9d73SEdward Cree if (rc && rc != -EPERM) 2878874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, 2879874aeea5SJeff Kirsher "could not restore PHY settings\n"); 2880874aeea5SJeff Kirsher } 2881874aeea5SJeff Kirsher 2882261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 2883261e4d96SJon Cooper if (rc) 2884261e4d96SJon Cooper goto fail; 28856d8aaaf6SDaniel Pieczko 28866d8aaaf6SDaniel Pieczko #ifdef CONFIG_SFC_SRIOV 28876d8aaaf6SDaniel Pieczko rc = efx->type->vswitching_restore(efx); 28886d8aaaf6SDaniel Pieczko if (rc) /* not fatal; the PF will still work fine */ 28896d8aaaf6SDaniel Pieczko netif_warn(efx, probe, efx->net_dev, 28906d8aaaf6SDaniel Pieczko "failed to restore vswitching rc=%d;" 28916d8aaaf6SDaniel Pieczko " VFs may not function\n", rc); 28926d8aaaf6SDaniel Pieczko #endif 28936d8aaaf6SDaniel Pieczko 289442356d9aSEdward Cree if (efx->type->rx_restore_rss_contexts) 289542356d9aSEdward Cree efx->type->rx_restore_rss_contexts(efx); 2896e0a65e3cSEdward Cree mutex_unlock(&efx->rss_lock); 2897193f2003SBert Kenward efx->type->filter_table_restore(efx); 2898193f2003SBert Kenward up_write(&efx->filter_sem); 28997fa8d547SShradha Shah if (efx->type->sriov_reset) 2900d98a4ffeSShradha Shah efx->type->sriov_reset(efx); 2901874aeea5SJeff Kirsher 2902874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2903874aeea5SJeff Kirsher 2904874aeea5SJeff Kirsher efx_start_all(efx); 2905874aeea5SJeff Kirsher 2906e5fbd977SJon Cooper if (efx->type->udp_tnl_push_ports) 2907e5fbd977SJon Cooper efx->type->udp_tnl_push_ports(efx); 2908e5fbd977SJon Cooper 2909874aeea5SJeff Kirsher return 0; 2910874aeea5SJeff Kirsher 2911874aeea5SJeff Kirsher fail: 2912874aeea5SJeff Kirsher efx->port_initialized = false; 2913874aeea5SJeff Kirsher 2914e0a65e3cSEdward Cree mutex_unlock(&efx->rss_lock); 2915193f2003SBert Kenward up_write(&efx->filter_sem); 2916874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 2917874aeea5SJeff Kirsher 2918874aeea5SJeff Kirsher return rc; 2919874aeea5SJeff Kirsher } 2920874aeea5SJeff Kirsher 2921874aeea5SJeff Kirsher /* Reset the NIC using the specified method. Note that the reset may 2922874aeea5SJeff Kirsher * fail, in which case the card will be left in an unusable state. 2923874aeea5SJeff Kirsher * 2924874aeea5SJeff Kirsher * Caller must hold the rtnl_lock. 2925874aeea5SJeff Kirsher */ 2926874aeea5SJeff Kirsher int efx_reset(struct efx_nic *efx, enum reset_type method) 2927874aeea5SJeff Kirsher { 2928874aeea5SJeff Kirsher int rc, rc2; 2929874aeea5SJeff Kirsher bool disabled; 2930874aeea5SJeff Kirsher 2931874aeea5SJeff Kirsher netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2932874aeea5SJeff Kirsher RESET_TYPE(method)); 2933874aeea5SJeff Kirsher 2934c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 2935874aeea5SJeff Kirsher efx_reset_down(efx, method); 2936874aeea5SJeff Kirsher 2937874aeea5SJeff Kirsher rc = efx->type->reset(efx, method); 2938874aeea5SJeff Kirsher if (rc) { 2939874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2940874aeea5SJeff Kirsher goto out; 2941874aeea5SJeff Kirsher } 2942874aeea5SJeff Kirsher 2943874aeea5SJeff Kirsher /* Clear flags for the scopes we covered. We assume the NIC and 2944874aeea5SJeff Kirsher * driver are now quiescent so that there is no race here. 2945874aeea5SJeff Kirsher */ 2946e283546cSEdward Cree if (method < RESET_TYPE_MAX_METHOD) 2947874aeea5SJeff Kirsher efx->reset_pending &= -(1 << (method + 1)); 2948e283546cSEdward Cree else /* it doesn't fit into the well-ordered scope hierarchy */ 2949e283546cSEdward Cree __clear_bit(method, &efx->reset_pending); 2950874aeea5SJeff Kirsher 2951874aeea5SJeff Kirsher /* Reinitialise bus-mastering, which may have been turned off before 2952874aeea5SJeff Kirsher * the reset was scheduled. This is still appropriate, even in the 2953874aeea5SJeff Kirsher * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2954874aeea5SJeff Kirsher * can respond to requests. */ 2955874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 2956874aeea5SJeff Kirsher 2957874aeea5SJeff Kirsher out: 2958874aeea5SJeff Kirsher /* Leave device stopped if necessary */ 2959626950dbSAlexandre Rames disabled = rc || 2960626950dbSAlexandre Rames method == RESET_TYPE_DISABLE || 2961626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_DISABLE; 2962874aeea5SJeff Kirsher rc2 = efx_reset_up(efx, method, !disabled); 2963874aeea5SJeff Kirsher if (rc2) { 2964874aeea5SJeff Kirsher disabled = true; 2965874aeea5SJeff Kirsher if (!rc) 2966874aeea5SJeff Kirsher rc = rc2; 2967874aeea5SJeff Kirsher } 2968874aeea5SJeff Kirsher 2969874aeea5SJeff Kirsher if (disabled) { 2970874aeea5SJeff Kirsher dev_close(efx->net_dev); 2971874aeea5SJeff Kirsher netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2972874aeea5SJeff Kirsher efx->state = STATE_DISABLED; 2973874aeea5SJeff Kirsher } else { 2974874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 29759c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 2976874aeea5SJeff Kirsher } 2977874aeea5SJeff Kirsher return rc; 2978874aeea5SJeff Kirsher } 2979874aeea5SJeff Kirsher 2980626950dbSAlexandre Rames /* Try recovery mechanisms. 2981626950dbSAlexandre Rames * For now only EEH is supported. 2982626950dbSAlexandre Rames * Returns 0 if the recovery mechanisms are unsuccessful. 2983626950dbSAlexandre Rames * Returns a non-zero value otherwise. 2984626950dbSAlexandre Rames */ 2985b28405b0SAlexandre Rames int efx_try_recovery(struct efx_nic *efx) 2986626950dbSAlexandre Rames { 2987626950dbSAlexandre Rames #ifdef CONFIG_EEH 2988626950dbSAlexandre Rames /* A PCI error can occur and not be seen by EEH because nothing 2989626950dbSAlexandre Rames * happens on the PCI bus. In this case the driver may fail and 2990626950dbSAlexandre Rames * schedule a 'recover or reset', leading to this recovery handler. 2991626950dbSAlexandre Rames * Manually call the eeh failure check function. 2992626950dbSAlexandre Rames */ 299312a89dbaSBenjamin Herrenschmidt struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2994626950dbSAlexandre Rames if (eeh_dev_check_failure(eehdev)) { 2995626950dbSAlexandre Rames /* The EEH mechanisms will handle the error and reset the 2996626950dbSAlexandre Rames * device if necessary. 2997626950dbSAlexandre Rames */ 2998626950dbSAlexandre Rames return 1; 2999626950dbSAlexandre Rames } 3000626950dbSAlexandre Rames #endif 3001626950dbSAlexandre Rames return 0; 3002626950dbSAlexandre Rames } 3003626950dbSAlexandre Rames 300474cd60a4SJon Cooper static void efx_wait_for_bist_end(struct efx_nic *efx) 300574cd60a4SJon Cooper { 300674cd60a4SJon Cooper int i; 300774cd60a4SJon Cooper 300874cd60a4SJon Cooper for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 300974cd60a4SJon Cooper if (efx_mcdi_poll_reboot(efx)) 301074cd60a4SJon Cooper goto out; 301174cd60a4SJon Cooper msleep(BIST_WAIT_DELAY_MS); 301274cd60a4SJon Cooper } 301374cd60a4SJon Cooper 301474cd60a4SJon Cooper netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 301574cd60a4SJon Cooper out: 301674cd60a4SJon Cooper /* Either way unset the BIST flag. If we found no reboot we probably 301774cd60a4SJon Cooper * won't recover, but we should try. 301874cd60a4SJon Cooper */ 301974cd60a4SJon Cooper efx->mc_bist_for_other_fn = false; 302074cd60a4SJon Cooper } 302174cd60a4SJon Cooper 3022874aeea5SJeff Kirsher /* The worker thread exists so that code that cannot sleep can 3023874aeea5SJeff Kirsher * schedule a reset for later. 3024874aeea5SJeff Kirsher */ 3025874aeea5SJeff Kirsher static void efx_reset_work(struct work_struct *data) 3026874aeea5SJeff Kirsher { 3027874aeea5SJeff Kirsher struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 3028626950dbSAlexandre Rames unsigned long pending; 3029626950dbSAlexandre Rames enum reset_type method; 3030626950dbSAlexandre Rames 30316aa7de05SMark Rutland pending = READ_ONCE(efx->reset_pending); 3032626950dbSAlexandre Rames method = fls(pending) - 1; 3033626950dbSAlexandre Rames 303474cd60a4SJon Cooper if (method == RESET_TYPE_MC_BIST) 303574cd60a4SJon Cooper efx_wait_for_bist_end(efx); 303674cd60a4SJon Cooper 3037626950dbSAlexandre Rames if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 3038626950dbSAlexandre Rames method == RESET_TYPE_RECOVER_OR_ALL) && 3039626950dbSAlexandre Rames efx_try_recovery(efx)) 3040626950dbSAlexandre Rames return; 3041874aeea5SJeff Kirsher 3042874aeea5SJeff Kirsher if (!pending) 3043874aeea5SJeff Kirsher return; 3044874aeea5SJeff Kirsher 3045874aeea5SJeff Kirsher rtnl_lock(); 30467153f623SBen Hutchings 30477153f623SBen Hutchings /* We checked the state in efx_schedule_reset() but it may 30487153f623SBen Hutchings * have changed by now. Now that we have the RTNL lock, 30497153f623SBen Hutchings * it cannot change again. 30507153f623SBen Hutchings */ 30517153f623SBen Hutchings if (efx->state == STATE_READY) 3052626950dbSAlexandre Rames (void)efx_reset(efx, method); 30537153f623SBen Hutchings 3054874aeea5SJeff Kirsher rtnl_unlock(); 3055874aeea5SJeff Kirsher } 3056874aeea5SJeff Kirsher 3057874aeea5SJeff Kirsher void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 3058874aeea5SJeff Kirsher { 3059874aeea5SJeff Kirsher enum reset_type method; 3060874aeea5SJeff Kirsher 3061626950dbSAlexandre Rames if (efx->state == STATE_RECOVERY) { 3062626950dbSAlexandre Rames netif_dbg(efx, drv, efx->net_dev, 3063626950dbSAlexandre Rames "recovering: skip scheduling %s reset\n", 3064626950dbSAlexandre Rames RESET_TYPE(type)); 3065626950dbSAlexandre Rames return; 3066626950dbSAlexandre Rames } 3067626950dbSAlexandre Rames 3068874aeea5SJeff Kirsher switch (type) { 3069874aeea5SJeff Kirsher case RESET_TYPE_INVISIBLE: 3070874aeea5SJeff Kirsher case RESET_TYPE_ALL: 3071626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_ALL: 3072874aeea5SJeff Kirsher case RESET_TYPE_WORLD: 3073874aeea5SJeff Kirsher case RESET_TYPE_DISABLE: 3074626950dbSAlexandre Rames case RESET_TYPE_RECOVER_OR_DISABLE: 3075087e9025SJon Cooper case RESET_TYPE_DATAPATH: 307674cd60a4SJon Cooper case RESET_TYPE_MC_BIST: 3077e283546cSEdward Cree case RESET_TYPE_MCDI_TIMEOUT: 3078874aeea5SJeff Kirsher method = type; 3079874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 3080874aeea5SJeff Kirsher RESET_TYPE(method)); 3081874aeea5SJeff Kirsher break; 3082874aeea5SJeff Kirsher default: 3083874aeea5SJeff Kirsher method = efx->type->map_reset_reason(type); 3084874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, 3085874aeea5SJeff Kirsher "scheduling %s reset for %s\n", 3086874aeea5SJeff Kirsher RESET_TYPE(method), RESET_TYPE(type)); 3087874aeea5SJeff Kirsher break; 3088874aeea5SJeff Kirsher } 3089874aeea5SJeff Kirsher 3090874aeea5SJeff Kirsher set_bit(method, &efx->reset_pending); 30917153f623SBen Hutchings smp_mb(); /* ensure we change reset_pending before checking state */ 30927153f623SBen Hutchings 30937153f623SBen Hutchings /* If we're not READY then just leave the flags set as the cue 30947153f623SBen Hutchings * to abort probing or reschedule the reset later. 30957153f623SBen Hutchings */ 30966aa7de05SMark Rutland if (READ_ONCE(efx->state) != STATE_READY) 30977153f623SBen Hutchings return; 3098874aeea5SJeff Kirsher 3099874aeea5SJeff Kirsher /* efx_process_channel() will no longer read events once a 3100874aeea5SJeff Kirsher * reset is scheduled. So switch back to poll'd MCDI completions. */ 3101874aeea5SJeff Kirsher efx_mcdi_mode_poll(efx); 3102874aeea5SJeff Kirsher 310382c64484SAlex Maftei (amaftei) efx_queue_reset_work(efx); 3104874aeea5SJeff Kirsher } 3105874aeea5SJeff Kirsher 3106874aeea5SJeff Kirsher /************************************************************************** 3107874aeea5SJeff Kirsher * 3108874aeea5SJeff Kirsher * List of NICs we support 3109874aeea5SJeff Kirsher * 3110874aeea5SJeff Kirsher **************************************************************************/ 3111874aeea5SJeff Kirsher 3112874aeea5SJeff Kirsher /* PCI device ID table */ 31139baa3c34SBenoit Taine static const struct pci_device_id efx_pci_table[] = { 3114547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 3115874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 3116547c474fSBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 3117874aeea5SJeff Kirsher .driver_data = (unsigned long) &siena_a0_nic_type}, 31188127d661SBen Hutchings {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 31198127d661SBen Hutchings .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 31206f7f8aa6SShradha Shah {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 31216f7f8aa6SShradha Shah .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 31223b06a00eSMateusz Wrzesinski {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 31233b06a00eSMateusz Wrzesinski .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 3124dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 3125dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 3126dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 3127dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 3128dd248f1bSBert Kenward {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 3129dd248f1bSBert Kenward .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 3130aae5a316SEdward Cree {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 3131aae5a316SEdward Cree .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 3132aae5a316SEdward Cree {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 3133aae5a316SEdward Cree .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 3134874aeea5SJeff Kirsher {0} /* end of list */ 3135874aeea5SJeff Kirsher }; 3136874aeea5SJeff Kirsher 3137874aeea5SJeff Kirsher /************************************************************************** 3138874aeea5SJeff Kirsher * 3139874aeea5SJeff Kirsher * Dummy PHY/MAC operations 3140874aeea5SJeff Kirsher * 3141874aeea5SJeff Kirsher * Can be used for some unimplemented operations 3142874aeea5SJeff Kirsher * Needed so all function pointers are valid and do not have to be tested 3143874aeea5SJeff Kirsher * before use 3144874aeea5SJeff Kirsher * 3145874aeea5SJeff Kirsher **************************************************************************/ 3146874aeea5SJeff Kirsher int efx_port_dummy_op_int(struct efx_nic *efx) 3147874aeea5SJeff Kirsher { 3148874aeea5SJeff Kirsher return 0; 3149874aeea5SJeff Kirsher } 3150874aeea5SJeff Kirsher void efx_port_dummy_op_void(struct efx_nic *efx) {} 3151874aeea5SJeff Kirsher 3152874aeea5SJeff Kirsher static bool efx_port_dummy_op_poll(struct efx_nic *efx) 3153874aeea5SJeff Kirsher { 3154874aeea5SJeff Kirsher return false; 3155874aeea5SJeff Kirsher } 3156874aeea5SJeff Kirsher 3157874aeea5SJeff Kirsher static const struct efx_phy_operations efx_dummy_phy_operations = { 3158874aeea5SJeff Kirsher .init = efx_port_dummy_op_int, 3159874aeea5SJeff Kirsher .reconfigure = efx_port_dummy_op_int, 3160874aeea5SJeff Kirsher .poll = efx_port_dummy_op_poll, 3161874aeea5SJeff Kirsher .fini = efx_port_dummy_op_void, 3162874aeea5SJeff Kirsher }; 3163874aeea5SJeff Kirsher 3164874aeea5SJeff Kirsher /************************************************************************** 3165874aeea5SJeff Kirsher * 3166874aeea5SJeff Kirsher * Data housekeeping 3167874aeea5SJeff Kirsher * 3168874aeea5SJeff Kirsher **************************************************************************/ 3169874aeea5SJeff Kirsher 3170874aeea5SJeff Kirsher /* This zeroes out and then fills in the invariants in a struct 3171874aeea5SJeff Kirsher * efx_nic (including all sub-structures). 3172874aeea5SJeff Kirsher */ 3173e1253f39SAlex Maftei (amaftei) int efx_init_struct(struct efx_nic *efx, struct pci_dev *pci_dev, 3174e1253f39SAlex Maftei (amaftei) struct net_device *net_dev) 3175874aeea5SJeff Kirsher { 31766f9f6ec2SAndrew Rybchenko int rc = -ENOMEM, i; 3177874aeea5SJeff Kirsher 3178874aeea5SJeff Kirsher /* Initialise common structures */ 31790bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->node); 31800bcf4a64SBen Hutchings INIT_LIST_HEAD(&efx->secondary_list); 3181874aeea5SJeff Kirsher spin_lock_init(&efx->biu_lock); 3182874aeea5SJeff Kirsher #ifdef CONFIG_SFC_MTD 3183874aeea5SJeff Kirsher INIT_LIST_HEAD(&efx->mtd_list); 3184874aeea5SJeff Kirsher #endif 3185874aeea5SJeff Kirsher INIT_WORK(&efx->reset_work, efx_reset_work); 3186874aeea5SJeff Kirsher INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 3187dd40781eSBen Hutchings INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 3188874aeea5SJeff Kirsher efx->pci_dev = pci_dev; 3189874aeea5SJeff Kirsher efx->msg_enable = debug; 3190f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 3191874aeea5SJeff Kirsher strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 3192874aeea5SJeff Kirsher 3193874aeea5SJeff Kirsher efx->net_dev = net_dev; 319443a3739dSJon Cooper efx->rx_prefix_size = efx->type->rx_prefix_size; 31952ec03014SAndrew Rybchenko efx->rx_ip_align = 31962ec03014SAndrew Rybchenko NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 319743a3739dSJon Cooper efx->rx_packet_hash_offset = 319843a3739dSJon Cooper efx->type->rx_hash_offset - efx->type->rx_prefix_size; 3199bd9a265dSJon Cooper efx->rx_packet_ts_offset = 3200bd9a265dSJon Cooper efx->type->rx_ts_offset - efx->type->rx_prefix_size; 320142356d9aSEdward Cree INIT_LIST_HEAD(&efx->rss_context.list); 3202e0a65e3cSEdward Cree mutex_init(&efx->rss_lock); 3203874aeea5SJeff Kirsher spin_lock_init(&efx->stats_lock); 320471827443SEdward Cree efx->vi_stride = EFX_DEFAULT_VI_STRIDE; 3205c1be4821SEdward Cree efx->num_mac_stats = MC_CMD_MAC_NSTATS; 3206c1be4821SEdward Cree BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 3207874aeea5SJeff Kirsher mutex_init(&efx->mac_lock); 32083af0f342SEdward Cree #ifdef CONFIG_RFS_ACCEL 32093af0f342SEdward Cree mutex_init(&efx->rps_mutex); 3210f8d62037SEdward Cree spin_lock_init(&efx->rps_hash_lock); 3211f8d62037SEdward Cree /* Failure to allocate is not fatal, but may degrade ARFS performance */ 3212f8d62037SEdward Cree efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, 3213f8d62037SEdward Cree sizeof(*efx->rps_hash_table), GFP_KERNEL); 32143af0f342SEdward Cree #endif 3215874aeea5SJeff Kirsher efx->phy_op = &efx_dummy_phy_operations; 3216874aeea5SJeff Kirsher efx->mdio.dev = net_dev; 3217874aeea5SJeff Kirsher INIT_WORK(&efx->mac_work, efx_mac_work); 32189f2cb71cSBen Hutchings init_waitqueue_head(&efx->flush_wq); 3219874aeea5SJeff Kirsher 3220874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) { 3221874aeea5SJeff Kirsher efx->channel[i] = efx_alloc_channel(efx, i, NULL); 3222874aeea5SJeff Kirsher if (!efx->channel[i]) 3223874aeea5SJeff Kirsher goto fail; 3224d8291187SBen Hutchings efx->msi_context[i].efx = efx; 3225d8291187SBen Hutchings efx->msi_context[i].index = i; 3226874aeea5SJeff Kirsher } 3227874aeea5SJeff Kirsher 3228874aeea5SJeff Kirsher /* Higher numbered interrupt modes are less capable! */ 32296f9f6ec2SAndrew Rybchenko if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 32306f9f6ec2SAndrew Rybchenko efx->type->min_interrupt_mode)) { 32316f9f6ec2SAndrew Rybchenko rc = -EIO; 32326f9f6ec2SAndrew Rybchenko goto fail; 32336f9f6ec2SAndrew Rybchenko } 3234874aeea5SJeff Kirsher efx->interrupt_mode = max(efx->type->max_interrupt_mode, 3235874aeea5SJeff Kirsher interrupt_mode); 32366f9f6ec2SAndrew Rybchenko efx->interrupt_mode = min(efx->type->min_interrupt_mode, 32376f9f6ec2SAndrew Rybchenko interrupt_mode); 3238874aeea5SJeff Kirsher 3239874aeea5SJeff Kirsher /* Would be good to use the net_dev name, but we're too early */ 3240874aeea5SJeff Kirsher snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 3241874aeea5SJeff Kirsher pci_name(pci_dev)); 3242874aeea5SJeff Kirsher efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 3243874aeea5SJeff Kirsher if (!efx->workqueue) 3244874aeea5SJeff Kirsher goto fail; 3245874aeea5SJeff Kirsher 3246874aeea5SJeff Kirsher return 0; 3247874aeea5SJeff Kirsher 3248874aeea5SJeff Kirsher fail: 3249874aeea5SJeff Kirsher efx_fini_struct(efx); 32506f9f6ec2SAndrew Rybchenko return rc; 3251874aeea5SJeff Kirsher } 3252874aeea5SJeff Kirsher 3253e1253f39SAlex Maftei (amaftei) void efx_fini_struct(struct efx_nic *efx) 3254874aeea5SJeff Kirsher { 3255874aeea5SJeff Kirsher int i; 3256874aeea5SJeff Kirsher 3257f8d62037SEdward Cree #ifdef CONFIG_RFS_ACCEL 3258f8d62037SEdward Cree kfree(efx->rps_hash_table); 3259f8d62037SEdward Cree #endif 3260f8d62037SEdward Cree 3261874aeea5SJeff Kirsher for (i = 0; i < EFX_MAX_CHANNELS; i++) 3262874aeea5SJeff Kirsher kfree(efx->channel[i]); 3263874aeea5SJeff Kirsher 3264ef215e64SBen Hutchings kfree(efx->vpd_sn); 3265ef215e64SBen Hutchings 3266874aeea5SJeff Kirsher if (efx->workqueue) { 3267874aeea5SJeff Kirsher destroy_workqueue(efx->workqueue); 3268874aeea5SJeff Kirsher efx->workqueue = NULL; 3269874aeea5SJeff Kirsher } 3270874aeea5SJeff Kirsher } 3271874aeea5SJeff Kirsher 3272e4d112e4SEdward Cree void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 3273e4d112e4SEdward Cree { 3274e4d112e4SEdward Cree u64 n_rx_nodesc_trunc = 0; 3275e4d112e4SEdward Cree struct efx_channel *channel; 3276e4d112e4SEdward Cree 3277e4d112e4SEdward Cree efx_for_each_channel(channel, efx) 3278e4d112e4SEdward Cree n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 3279e4d112e4SEdward Cree stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 3280e4d112e4SEdward Cree stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 3281e4d112e4SEdward Cree } 3282e4d112e4SEdward Cree 3283f8d62037SEdward Cree bool efx_filter_spec_equal(const struct efx_filter_spec *left, 3284f8d62037SEdward Cree const struct efx_filter_spec *right) 3285f8d62037SEdward Cree { 3286f8d62037SEdward Cree if ((left->match_flags ^ right->match_flags) | 3287f8d62037SEdward Cree ((left->flags ^ right->flags) & 3288f8d62037SEdward Cree (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) 3289f8d62037SEdward Cree return false; 3290f8d62037SEdward Cree 3291f8d62037SEdward Cree return memcmp(&left->outer_vid, &right->outer_vid, 3292f8d62037SEdward Cree sizeof(struct efx_filter_spec) - 3293f8d62037SEdward Cree offsetof(struct efx_filter_spec, outer_vid)) == 0; 3294f8d62037SEdward Cree } 3295f8d62037SEdward Cree 3296f8d62037SEdward Cree u32 efx_filter_spec_hash(const struct efx_filter_spec *spec) 3297f8d62037SEdward Cree { 3298f8d62037SEdward Cree BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); 3299f8d62037SEdward Cree return jhash2((const u32 *)&spec->outer_vid, 3300f8d62037SEdward Cree (sizeof(struct efx_filter_spec) - 3301f8d62037SEdward Cree offsetof(struct efx_filter_spec, outer_vid)) / 4, 3302f8d62037SEdward Cree 0); 3303f8d62037SEdward Cree } 3304f8d62037SEdward Cree 3305f8d62037SEdward Cree #ifdef CONFIG_RFS_ACCEL 3306f8d62037SEdward Cree bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx, 3307f8d62037SEdward Cree bool *force) 3308f8d62037SEdward Cree { 3309f8d62037SEdward Cree if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) { 3310f8d62037SEdward Cree /* ARFS is currently updating this entry, leave it */ 3311f8d62037SEdward Cree return false; 3312f8d62037SEdward Cree } 3313f8d62037SEdward Cree if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) { 3314f8d62037SEdward Cree /* ARFS tried and failed to update this, so it's probably out 3315f8d62037SEdward Cree * of date. Remove the filter and the ARFS rule entry. 3316f8d62037SEdward Cree */ 3317f8d62037SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 3318f8d62037SEdward Cree *force = true; 3319f8d62037SEdward Cree return true; 3320f8d62037SEdward Cree } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */ 3321f8d62037SEdward Cree /* ARFS has moved on, so old filter is not needed. Since we did 3322f8d62037SEdward Cree * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will 3323f8d62037SEdward Cree * not be removed by efx_rps_hash_del() subsequently. 3324f8d62037SEdward Cree */ 3325f8d62037SEdward Cree *force = true; 3326f8d62037SEdward Cree return true; 3327f8d62037SEdward Cree } 3328f8d62037SEdward Cree /* Remove it iff ARFS wants to. */ 3329f8d62037SEdward Cree return true; 3330f8d62037SEdward Cree } 3331f8d62037SEdward Cree 3332829eb053SColin Ian King static 3333f8d62037SEdward Cree struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx, 3334f8d62037SEdward Cree const struct efx_filter_spec *spec) 3335f8d62037SEdward Cree { 3336f8d62037SEdward Cree u32 hash = efx_filter_spec_hash(spec); 3337f8d62037SEdward Cree 3338013ff4a6SLance Roy lockdep_assert_held(&efx->rps_hash_lock); 3339f8d62037SEdward Cree if (!efx->rps_hash_table) 3340f8d62037SEdward Cree return NULL; 3341f8d62037SEdward Cree return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE]; 3342f8d62037SEdward Cree } 3343f8d62037SEdward Cree 3344f8d62037SEdward Cree struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx, 3345f8d62037SEdward Cree const struct efx_filter_spec *spec) 3346f8d62037SEdward Cree { 3347f8d62037SEdward Cree struct efx_arfs_rule *rule; 3348f8d62037SEdward Cree struct hlist_head *head; 3349f8d62037SEdward Cree struct hlist_node *node; 3350f8d62037SEdward Cree 3351f8d62037SEdward Cree head = efx_rps_hash_bucket(efx, spec); 3352f8d62037SEdward Cree if (!head) 3353f8d62037SEdward Cree return NULL; 3354f8d62037SEdward Cree hlist_for_each(node, head) { 3355f8d62037SEdward Cree rule = container_of(node, struct efx_arfs_rule, node); 3356f8d62037SEdward Cree if (efx_filter_spec_equal(spec, &rule->spec)) 3357f8d62037SEdward Cree return rule; 3358f8d62037SEdward Cree } 3359f8d62037SEdward Cree return NULL; 3360f8d62037SEdward Cree } 3361f8d62037SEdward Cree 3362f8d62037SEdward Cree struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx, 3363f8d62037SEdward Cree const struct efx_filter_spec *spec, 3364f8d62037SEdward Cree bool *new) 3365f8d62037SEdward Cree { 3366f8d62037SEdward Cree struct efx_arfs_rule *rule; 3367f8d62037SEdward Cree struct hlist_head *head; 3368f8d62037SEdward Cree struct hlist_node *node; 3369f8d62037SEdward Cree 3370f8d62037SEdward Cree head = efx_rps_hash_bucket(efx, spec); 3371f8d62037SEdward Cree if (!head) 3372f8d62037SEdward Cree return NULL; 3373f8d62037SEdward Cree hlist_for_each(node, head) { 3374f8d62037SEdward Cree rule = container_of(node, struct efx_arfs_rule, node); 3375f8d62037SEdward Cree if (efx_filter_spec_equal(spec, &rule->spec)) { 3376f8d62037SEdward Cree *new = false; 3377f8d62037SEdward Cree return rule; 3378f8d62037SEdward Cree } 3379f8d62037SEdward Cree } 3380f8d62037SEdward Cree rule = kmalloc(sizeof(*rule), GFP_ATOMIC); 3381f8d62037SEdward Cree *new = true; 3382f8d62037SEdward Cree if (rule) { 3383f8d62037SEdward Cree memcpy(&rule->spec, spec, sizeof(rule->spec)); 3384f8d62037SEdward Cree hlist_add_head(&rule->node, head); 3385f8d62037SEdward Cree } 3386f8d62037SEdward Cree return rule; 3387f8d62037SEdward Cree } 3388f8d62037SEdward Cree 3389f8d62037SEdward Cree void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec) 3390f8d62037SEdward Cree { 3391f8d62037SEdward Cree struct efx_arfs_rule *rule; 3392f8d62037SEdward Cree struct hlist_head *head; 3393f8d62037SEdward Cree struct hlist_node *node; 3394f8d62037SEdward Cree 3395f8d62037SEdward Cree head = efx_rps_hash_bucket(efx, spec); 3396f8d62037SEdward Cree if (WARN_ON(!head)) 3397f8d62037SEdward Cree return; 3398f8d62037SEdward Cree hlist_for_each(node, head) { 3399f8d62037SEdward Cree rule = container_of(node, struct efx_arfs_rule, node); 3400f8d62037SEdward Cree if (efx_filter_spec_equal(spec, &rule->spec)) { 3401f8d62037SEdward Cree /* Someone already reused the entry. We know that if 3402f8d62037SEdward Cree * this check doesn't fire (i.e. filter_id == REMOVING) 3403f8d62037SEdward Cree * then the REMOVING mark was put there by our caller, 3404f8d62037SEdward Cree * because caller is holding a lock on filter table and 3405f8d62037SEdward Cree * only holders of that lock set REMOVING. 3406f8d62037SEdward Cree */ 3407f8d62037SEdward Cree if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING) 3408f8d62037SEdward Cree return; 3409f8d62037SEdward Cree hlist_del(node); 3410f8d62037SEdward Cree kfree(rule); 3411f8d62037SEdward Cree return; 3412f8d62037SEdward Cree } 3413f8d62037SEdward Cree } 3414f8d62037SEdward Cree /* We didn't find it. */ 3415f8d62037SEdward Cree WARN_ON(1); 3416f8d62037SEdward Cree } 3417f8d62037SEdward Cree #endif 3418f8d62037SEdward Cree 341942356d9aSEdward Cree /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because 342042356d9aSEdward Cree * (a) this is an infrequent control-plane operation and (b) n is small (max 64) 342142356d9aSEdward Cree */ 3422e0a65e3cSEdward Cree struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx) 342342356d9aSEdward Cree { 3424e0a65e3cSEdward Cree struct list_head *head = &efx->rss_context.list; 342542356d9aSEdward Cree struct efx_rss_context *ctx, *new; 342642356d9aSEdward Cree u32 id = 1; /* Don't use zero, that refers to the master RSS context */ 342742356d9aSEdward Cree 3428e0a65e3cSEdward Cree WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3429e0a65e3cSEdward Cree 343042356d9aSEdward Cree /* Search for first gap in the numbering */ 343142356d9aSEdward Cree list_for_each_entry(ctx, head, list) { 343242356d9aSEdward Cree if (ctx->user_id != id) 343342356d9aSEdward Cree break; 343442356d9aSEdward Cree id++; 343542356d9aSEdward Cree /* Check for wrap. If this happens, we have nearly 2^32 343642356d9aSEdward Cree * allocated RSS contexts, which seems unlikely. 343742356d9aSEdward Cree */ 343842356d9aSEdward Cree if (WARN_ON_ONCE(!id)) 343942356d9aSEdward Cree return NULL; 344042356d9aSEdward Cree } 344142356d9aSEdward Cree 344242356d9aSEdward Cree /* Create the new entry */ 344342356d9aSEdward Cree new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL); 344442356d9aSEdward Cree if (!new) 344542356d9aSEdward Cree return NULL; 344642356d9aSEdward Cree new->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 344742356d9aSEdward Cree new->rx_hash_udp_4tuple = false; 344842356d9aSEdward Cree 344942356d9aSEdward Cree /* Insert the new entry into the gap */ 345042356d9aSEdward Cree new->user_id = id; 345142356d9aSEdward Cree list_add_tail(&new->list, &ctx->list); 345242356d9aSEdward Cree return new; 345342356d9aSEdward Cree } 345442356d9aSEdward Cree 3455e0a65e3cSEdward Cree struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id) 345642356d9aSEdward Cree { 3457e0a65e3cSEdward Cree struct list_head *head = &efx->rss_context.list; 345842356d9aSEdward Cree struct efx_rss_context *ctx; 345942356d9aSEdward Cree 3460e0a65e3cSEdward Cree WARN_ON(!mutex_is_locked(&efx->rss_lock)); 3461e0a65e3cSEdward Cree 346242356d9aSEdward Cree list_for_each_entry(ctx, head, list) 346342356d9aSEdward Cree if (ctx->user_id == id) 346442356d9aSEdward Cree return ctx; 346542356d9aSEdward Cree return NULL; 346642356d9aSEdward Cree } 346742356d9aSEdward Cree 346842356d9aSEdward Cree void efx_free_rss_context_entry(struct efx_rss_context *ctx) 346942356d9aSEdward Cree { 347042356d9aSEdward Cree list_del(&ctx->list); 347142356d9aSEdward Cree kfree(ctx); 347242356d9aSEdward Cree } 347342356d9aSEdward Cree 3474874aeea5SJeff Kirsher /************************************************************************** 3475874aeea5SJeff Kirsher * 3476874aeea5SJeff Kirsher * PCI interface 3477874aeea5SJeff Kirsher * 3478874aeea5SJeff Kirsher **************************************************************************/ 3479874aeea5SJeff Kirsher 3480874aeea5SJeff Kirsher /* Main body of final NIC shutdown code 3481874aeea5SJeff Kirsher * This is called only at module unload (or hotplug removal). 3482874aeea5SJeff Kirsher */ 3483874aeea5SJeff Kirsher static void efx_pci_remove_main(struct efx_nic *efx) 3484874aeea5SJeff Kirsher { 34857153f623SBen Hutchings /* Flush reset_work. It can no longer be scheduled since we 34867153f623SBen Hutchings * are not READY. 34877153f623SBen Hutchings */ 34887153f623SBen Hutchings BUG_ON(efx->state == STATE_READY); 348982c64484SAlex Maftei (amaftei) efx_flush_reset_workqueue(efx); 34907153f623SBen Hutchings 3491d8291187SBen Hutchings efx_disable_interrupts(efx); 3492a83762d9SBert Kenward efx_clear_interrupt_affinity(efx); 3493874aeea5SJeff Kirsher efx_nic_fini_interrupt(efx); 3494874aeea5SJeff Kirsher efx_fini_port(efx); 3495874aeea5SJeff Kirsher efx->type->fini(efx); 3496874aeea5SJeff Kirsher efx_fini_napi(efx); 3497874aeea5SJeff Kirsher efx_remove_all(efx); 3498874aeea5SJeff Kirsher } 3499874aeea5SJeff Kirsher 3500874aeea5SJeff Kirsher /* Final NIC shutdown 35012a3fc311SDaniel Pieczko * This is called only at module unload (or hotplug removal). A PF can call 35022a3fc311SDaniel Pieczko * this on its VFs to ensure they are unbound first. 3503874aeea5SJeff Kirsher */ 3504874aeea5SJeff Kirsher static void efx_pci_remove(struct pci_dev *pci_dev) 3505874aeea5SJeff Kirsher { 3506874aeea5SJeff Kirsher struct efx_nic *efx; 3507874aeea5SJeff Kirsher 3508874aeea5SJeff Kirsher efx = pci_get_drvdata(pci_dev); 3509874aeea5SJeff Kirsher if (!efx) 3510874aeea5SJeff Kirsher return; 3511874aeea5SJeff Kirsher 3512874aeea5SJeff Kirsher /* Mark the NIC as fini, then stop the interface */ 3513874aeea5SJeff Kirsher rtnl_lock(); 35140bcf4a64SBen Hutchings efx_dissociate(efx); 3515874aeea5SJeff Kirsher dev_close(efx->net_dev); 3516d8291187SBen Hutchings efx_disable_interrupts(efx); 3517ea6bb99eSEdward Cree efx->state = STATE_UNINIT; 3518874aeea5SJeff Kirsher rtnl_unlock(); 3519874aeea5SJeff Kirsher 35207fa8d547SShradha Shah if (efx->type->sriov_fini) 3521d98a4ffeSShradha Shah efx->type->sriov_fini(efx); 35227fa8d547SShradha Shah 3523874aeea5SJeff Kirsher efx_unregister_netdev(efx); 3524874aeea5SJeff Kirsher 3525874aeea5SJeff Kirsher efx_mtd_remove(efx); 3526874aeea5SJeff Kirsher 3527874aeea5SJeff Kirsher efx_pci_remove_main(efx); 3528874aeea5SJeff Kirsher 3529b1940451SAlex Maftei (amaftei) efx_fini_io(efx, efx->type->mem_bar(efx)); 3530874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3531874aeea5SJeff Kirsher 3532874aeea5SJeff Kirsher efx_fini_struct(efx); 3533874aeea5SJeff Kirsher free_netdev(efx->net_dev); 3534626950dbSAlexandre Rames 3535626950dbSAlexandre Rames pci_disable_pcie_error_reporting(pci_dev); 3536874aeea5SJeff Kirsher }; 3537874aeea5SJeff Kirsher 3538460eeaa0SBen Hutchings /* NIC VPD information 3539460eeaa0SBen Hutchings * Called during probe to display the part number of the 3540460eeaa0SBen Hutchings * installed NIC. VPD is potentially very large but this should 3541460eeaa0SBen Hutchings * always appear within the first 512 bytes. 3542460eeaa0SBen Hutchings */ 3543460eeaa0SBen Hutchings #define SFC_VPD_LEN 512 3544ef215e64SBen Hutchings static void efx_probe_vpd_strings(struct efx_nic *efx) 3545460eeaa0SBen Hutchings { 3546460eeaa0SBen Hutchings struct pci_dev *dev = efx->pci_dev; 3547460eeaa0SBen Hutchings char vpd_data[SFC_VPD_LEN]; 3548460eeaa0SBen Hutchings ssize_t vpd_size; 3549ef215e64SBen Hutchings int ro_start, ro_size, i, j; 3550460eeaa0SBen Hutchings 3551460eeaa0SBen Hutchings /* Get the vpd data from the device */ 3552460eeaa0SBen Hutchings vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3553460eeaa0SBen Hutchings if (vpd_size <= 0) { 3554460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3555460eeaa0SBen Hutchings return; 3556460eeaa0SBen Hutchings } 3557460eeaa0SBen Hutchings 3558460eeaa0SBen Hutchings /* Get the Read only section */ 3559ef215e64SBen Hutchings ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3560ef215e64SBen Hutchings if (ro_start < 0) { 3561460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3562460eeaa0SBen Hutchings return; 3563460eeaa0SBen Hutchings } 3564460eeaa0SBen Hutchings 3565ef215e64SBen Hutchings ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3566ef215e64SBen Hutchings j = ro_size; 3567ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3568460eeaa0SBen Hutchings if (i + j > vpd_size) 3569460eeaa0SBen Hutchings j = vpd_size - i; 3570460eeaa0SBen Hutchings 3571460eeaa0SBen Hutchings /* Get the Part number */ 3572460eeaa0SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3573460eeaa0SBen Hutchings if (i < 0) { 3574460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3575460eeaa0SBen Hutchings return; 3576460eeaa0SBen Hutchings } 3577460eeaa0SBen Hutchings 3578460eeaa0SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 3579460eeaa0SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 3580460eeaa0SBen Hutchings if (i + j > vpd_size) { 3581460eeaa0SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3582460eeaa0SBen Hutchings return; 3583460eeaa0SBen Hutchings } 3584460eeaa0SBen Hutchings 3585460eeaa0SBen Hutchings netif_info(efx, drv, efx->net_dev, 3586460eeaa0SBen Hutchings "Part Number : %.*s\n", j, &vpd_data[i]); 3587ef215e64SBen Hutchings 3588ef215e64SBen Hutchings i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3589ef215e64SBen Hutchings j = ro_size; 3590ef215e64SBen Hutchings i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3591ef215e64SBen Hutchings if (i < 0) { 3592ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3593ef215e64SBen Hutchings return; 3594ef215e64SBen Hutchings } 3595ef215e64SBen Hutchings 3596ef215e64SBen Hutchings j = pci_vpd_info_field_size(&vpd_data[i]); 3597ef215e64SBen Hutchings i += PCI_VPD_INFO_FLD_HDR_SIZE; 3598ef215e64SBen Hutchings if (i + j > vpd_size) { 3599ef215e64SBen Hutchings netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3600ef215e64SBen Hutchings return; 3601ef215e64SBen Hutchings } 3602ef215e64SBen Hutchings 3603ef215e64SBen Hutchings efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3604ef215e64SBen Hutchings if (!efx->vpd_sn) 3605ef215e64SBen Hutchings return; 3606ef215e64SBen Hutchings 3607ef215e64SBen Hutchings snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3608460eeaa0SBen Hutchings } 3609460eeaa0SBen Hutchings 3610460eeaa0SBen Hutchings 3611874aeea5SJeff Kirsher /* Main body of NIC initialisation 3612874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, theoretically). 3613874aeea5SJeff Kirsher */ 3614874aeea5SJeff Kirsher static int efx_pci_probe_main(struct efx_nic *efx) 3615874aeea5SJeff Kirsher { 3616874aeea5SJeff Kirsher int rc; 3617874aeea5SJeff Kirsher 3618874aeea5SJeff Kirsher /* Do start-of-day initialisation */ 3619874aeea5SJeff Kirsher rc = efx_probe_all(efx); 3620874aeea5SJeff Kirsher if (rc) 3621874aeea5SJeff Kirsher goto fail1; 3622874aeea5SJeff Kirsher 3623874aeea5SJeff Kirsher efx_init_napi(efx); 3624874aeea5SJeff Kirsher 3625193f2003SBert Kenward down_write(&efx->filter_sem); 3626874aeea5SJeff Kirsher rc = efx->type->init(efx); 3627193f2003SBert Kenward up_write(&efx->filter_sem); 3628874aeea5SJeff Kirsher if (rc) { 3629874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 3630874aeea5SJeff Kirsher "failed to initialise NIC\n"); 3631874aeea5SJeff Kirsher goto fail3; 3632874aeea5SJeff Kirsher } 3633874aeea5SJeff Kirsher 3634874aeea5SJeff Kirsher rc = efx_init_port(efx); 3635874aeea5SJeff Kirsher if (rc) { 3636874aeea5SJeff Kirsher netif_err(efx, probe, efx->net_dev, 3637874aeea5SJeff Kirsher "failed to initialise port\n"); 3638874aeea5SJeff Kirsher goto fail4; 3639874aeea5SJeff Kirsher } 3640874aeea5SJeff Kirsher 3641874aeea5SJeff Kirsher rc = efx_nic_init_interrupt(efx); 3642874aeea5SJeff Kirsher if (rc) 3643874aeea5SJeff Kirsher goto fail5; 3644a83762d9SBert Kenward 3645a83762d9SBert Kenward efx_set_interrupt_affinity(efx); 3646261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3647261e4d96SJon Cooper if (rc) 3648261e4d96SJon Cooper goto fail6; 3649874aeea5SJeff Kirsher 3650874aeea5SJeff Kirsher return 0; 3651874aeea5SJeff Kirsher 3652261e4d96SJon Cooper fail6: 3653a83762d9SBert Kenward efx_clear_interrupt_affinity(efx); 3654261e4d96SJon Cooper efx_nic_fini_interrupt(efx); 3655874aeea5SJeff Kirsher fail5: 3656874aeea5SJeff Kirsher efx_fini_port(efx); 3657874aeea5SJeff Kirsher fail4: 3658874aeea5SJeff Kirsher efx->type->fini(efx); 3659874aeea5SJeff Kirsher fail3: 3660874aeea5SJeff Kirsher efx_fini_napi(efx); 3661874aeea5SJeff Kirsher efx_remove_all(efx); 3662874aeea5SJeff Kirsher fail1: 3663874aeea5SJeff Kirsher return rc; 3664874aeea5SJeff Kirsher } 3665874aeea5SJeff Kirsher 36668a531400SJon Cooper static int efx_pci_probe_post_io(struct efx_nic *efx) 36678a531400SJon Cooper { 36688a531400SJon Cooper struct net_device *net_dev = efx->net_dev; 36698a531400SJon Cooper int rc = efx_pci_probe_main(efx); 36708a531400SJon Cooper 36718a531400SJon Cooper if (rc) 36728a531400SJon Cooper return rc; 36738a531400SJon Cooper 36748a531400SJon Cooper if (efx->type->sriov_init) { 36758a531400SJon Cooper rc = efx->type->sriov_init(efx); 36768a531400SJon Cooper if (rc) 36778a531400SJon Cooper netif_err(efx, probe, efx->net_dev, 36788a531400SJon Cooper "SR-IOV can't be enabled rc %d\n", rc); 36798a531400SJon Cooper } 36808a531400SJon Cooper 36818a531400SJon Cooper /* Determine netdevice features */ 36828a531400SJon Cooper net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 36836978729fSEdward Cree NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL); 36848a531400SJon Cooper if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 36858a531400SJon Cooper net_dev->features |= NETIF_F_TSO6; 36868a531400SJon Cooper /* Check whether device supports TSO */ 36878a531400SJon Cooper if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 36888a531400SJon Cooper net_dev->features &= ~NETIF_F_ALL_TSO; 36898a531400SJon Cooper /* Mask for features that also apply to VLAN devices */ 36908a531400SJon Cooper net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 36918a531400SJon Cooper NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 36928a531400SJon Cooper NETIF_F_RXCSUM); 36938a531400SJon Cooper 36946978729fSEdward Cree net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 36956978729fSEdward Cree 36966978729fSEdward Cree /* Disable receiving frames with bad FCS, by default. */ 36976978729fSEdward Cree net_dev->features &= ~NETIF_F_RXALL; 36988a531400SJon Cooper 36998a531400SJon Cooper /* Disable VLAN filtering by default. It may be enforced if 37008a531400SJon Cooper * the feature is fixed (i.e. VLAN filters are required to 37018a531400SJon Cooper * receive VLAN tagged packets due to vPort restrictions). 37028a531400SJon Cooper */ 37038a531400SJon Cooper net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 37048a531400SJon Cooper net_dev->features |= efx->fixed_features; 37058a531400SJon Cooper 37068a531400SJon Cooper rc = efx_register_netdev(efx); 37078a531400SJon Cooper if (!rc) 37088a531400SJon Cooper return 0; 37098a531400SJon Cooper 37108a531400SJon Cooper efx_pci_remove_main(efx); 37118a531400SJon Cooper return rc; 37128a531400SJon Cooper } 37138a531400SJon Cooper 3714874aeea5SJeff Kirsher /* NIC initialisation 3715874aeea5SJeff Kirsher * 3716874aeea5SJeff Kirsher * This is called at module load (or hotplug insertion, 371773ba7b68SBen Hutchings * theoretically). It sets up PCI mappings, resets the NIC, 3718874aeea5SJeff Kirsher * sets up and registers the network devices with the kernel and hooks 3719874aeea5SJeff Kirsher * the interrupt service routine. It does not prepare the device for 3720874aeea5SJeff Kirsher * transmission; this is left to the first time one of the network 3721874aeea5SJeff Kirsher * interfaces is brought up (i.e. efx_net_open). 3722874aeea5SJeff Kirsher */ 372387d1fc11SBill Pemberton static int efx_pci_probe(struct pci_dev *pci_dev, 3724874aeea5SJeff Kirsher const struct pci_device_id *entry) 3725874aeea5SJeff Kirsher { 3726874aeea5SJeff Kirsher struct net_device *net_dev; 3727874aeea5SJeff Kirsher struct efx_nic *efx; 3728fadac6aaSBen Hutchings int rc; 3729874aeea5SJeff Kirsher 3730874aeea5SJeff Kirsher /* Allocate and initialise a struct net_device and struct efx_nic */ 3731874aeea5SJeff Kirsher net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3732874aeea5SJeff Kirsher EFX_MAX_RX_QUEUES); 3733874aeea5SJeff Kirsher if (!net_dev) 3734874aeea5SJeff Kirsher return -ENOMEM; 3735adeb15aaSBen Hutchings efx = netdev_priv(net_dev); 3736adeb15aaSBen Hutchings efx->type = (const struct efx_nic_type *) entry->driver_data; 3737ebfcd0fdSAndrew Rybchenko efx->fixed_features |= NETIF_F_HIGHDMA; 3738eb7cfd8cSAndrew Rybchenko 3739874aeea5SJeff Kirsher pci_set_drvdata(pci_dev, efx); 3740874aeea5SJeff Kirsher SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3741adeb15aaSBen Hutchings rc = efx_init_struct(efx, pci_dev, net_dev); 3742874aeea5SJeff Kirsher if (rc) 3743874aeea5SJeff Kirsher goto fail1; 3744874aeea5SJeff Kirsher 3745874aeea5SJeff Kirsher netif_info(efx, probe, efx->net_dev, 3746874aeea5SJeff Kirsher "Solarflare NIC detected\n"); 3747874aeea5SJeff Kirsher 37486f7f8aa6SShradha Shah if (!efx->type->is_vf) 3749ef215e64SBen Hutchings efx_probe_vpd_strings(efx); 3750460eeaa0SBen Hutchings 3751874aeea5SJeff Kirsher /* Set up basic I/O (BAR mappings etc) */ 3752b1940451SAlex Maftei (amaftei) rc = efx_init_io(efx, efx->type->mem_bar(efx), efx->type->max_dma_mask, 3753b1940451SAlex Maftei (amaftei) efx->type->mem_map_size(efx)); 3754874aeea5SJeff Kirsher if (rc) 3755874aeea5SJeff Kirsher goto fail2; 3756874aeea5SJeff Kirsher 37578a531400SJon Cooper rc = efx_pci_probe_post_io(efx); 37588a531400SJon Cooper if (rc) { 37598a531400SJon Cooper /* On failure, retry once immediately. 37608a531400SJon Cooper * If we aborted probe due to a scheduled reset, dismiss it. 37618a531400SJon Cooper */ 37628a531400SJon Cooper efx->reset_pending = 0; 37638a531400SJon Cooper rc = efx_pci_probe_post_io(efx); 37648a531400SJon Cooper if (rc) { 37658a531400SJon Cooper /* On another failure, retry once more 37668a531400SJon Cooper * after a 50-305ms delay. 37678a531400SJon Cooper */ 37688a531400SJon Cooper unsigned char r; 37698a531400SJon Cooper 37708a531400SJon Cooper get_random_bytes(&r, 1); 37718a531400SJon Cooper msleep((unsigned int)r + 50); 37728a531400SJon Cooper efx->reset_pending = 0; 37738a531400SJon Cooper rc = efx_pci_probe_post_io(efx); 37748a531400SJon Cooper } 37758a531400SJon Cooper } 3776fadac6aaSBen Hutchings if (rc) 3777874aeea5SJeff Kirsher goto fail3; 3778874aeea5SJeff Kirsher 3779874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3780874aeea5SJeff Kirsher 37817c43161cSBen Hutchings /* Try to create MTDs, but allow this to fail */ 3782874aeea5SJeff Kirsher rtnl_lock(); 37837c43161cSBen Hutchings rc = efx_mtd_probe(efx); 3784874aeea5SJeff Kirsher rtnl_unlock(); 378509a04204SBert Kenward if (rc && rc != -EPERM) 37867c43161cSBen Hutchings netif_warn(efx, probe, efx->net_dev, 37877c43161cSBen Hutchings "failed to create MTDs (%d)\n", rc); 37887c43161cSBen Hutchings 378905cfee98SMartin Habets (void)pci_enable_pcie_error_reporting(pci_dev); 3790626950dbSAlexandre Rames 3791e5fbd977SJon Cooper if (efx->type->udp_tnl_push_ports) 3792e5fbd977SJon Cooper efx->type->udp_tnl_push_ports(efx); 3793e5fbd977SJon Cooper 3794874aeea5SJeff Kirsher return 0; 3795874aeea5SJeff Kirsher 3796874aeea5SJeff Kirsher fail3: 3797b1940451SAlex Maftei (amaftei) efx_fini_io(efx, efx->type->mem_bar(efx)); 3798874aeea5SJeff Kirsher fail2: 3799874aeea5SJeff Kirsher efx_fini_struct(efx); 3800874aeea5SJeff Kirsher fail1: 3801874aeea5SJeff Kirsher WARN_ON(rc > 0); 3802874aeea5SJeff Kirsher netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3803874aeea5SJeff Kirsher free_netdev(net_dev); 3804874aeea5SJeff Kirsher return rc; 3805874aeea5SJeff Kirsher } 3806874aeea5SJeff Kirsher 3807834e23ddSShradha Shah /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3808834e23ddSShradha Shah * enabled on success 3809834e23ddSShradha Shah */ 3810834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV 3811834e23ddSShradha Shah static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3812834e23ddSShradha Shah { 3813834e23ddSShradha Shah int rc; 3814834e23ddSShradha Shah struct efx_nic *efx = pci_get_drvdata(dev); 3815834e23ddSShradha Shah 3816834e23ddSShradha Shah if (efx->type->sriov_configure) { 3817834e23ddSShradha Shah rc = efx->type->sriov_configure(efx, num_vfs); 3818834e23ddSShradha Shah if (rc) 3819834e23ddSShradha Shah return rc; 3820834e23ddSShradha Shah else 3821834e23ddSShradha Shah return num_vfs; 3822834e23ddSShradha Shah } else 3823834e23ddSShradha Shah return -EOPNOTSUPP; 3824834e23ddSShradha Shah } 3825834e23ddSShradha Shah #endif 3826834e23ddSShradha Shah 3827874aeea5SJeff Kirsher static int efx_pm_freeze(struct device *dev) 3828874aeea5SJeff Kirsher { 38293e03a8baSChuhong Yuan struct efx_nic *efx = dev_get_drvdata(dev); 3830874aeea5SJeff Kirsher 383161da026dSBen Hutchings rtnl_lock(); 383261da026dSBen Hutchings 38336032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3834f16aeea0SBen Hutchings efx->state = STATE_UNINIT; 3835874aeea5SJeff Kirsher 3836c2f3b8e3SDaniel Pieczko efx_device_detach_sync(efx); 3837874aeea5SJeff Kirsher 3838874aeea5SJeff Kirsher efx_stop_all(efx); 3839d8291187SBen Hutchings efx_disable_interrupts(efx); 38406032fb56SBen Hutchings } 3841874aeea5SJeff Kirsher 384261da026dSBen Hutchings rtnl_unlock(); 384361da026dSBen Hutchings 3844874aeea5SJeff Kirsher return 0; 3845874aeea5SJeff Kirsher } 3846874aeea5SJeff Kirsher 3847874aeea5SJeff Kirsher static int efx_pm_thaw(struct device *dev) 3848874aeea5SJeff Kirsher { 3849261e4d96SJon Cooper int rc; 38503e03a8baSChuhong Yuan struct efx_nic *efx = dev_get_drvdata(dev); 3851874aeea5SJeff Kirsher 385261da026dSBen Hutchings rtnl_lock(); 385361da026dSBen Hutchings 38546032fb56SBen Hutchings if (efx->state != STATE_DISABLED) { 3855261e4d96SJon Cooper rc = efx_enable_interrupts(efx); 3856261e4d96SJon Cooper if (rc) 3857261e4d96SJon Cooper goto fail; 3858874aeea5SJeff Kirsher 3859874aeea5SJeff Kirsher mutex_lock(&efx->mac_lock); 3860874aeea5SJeff Kirsher efx->phy_op->reconfigure(efx); 3861874aeea5SJeff Kirsher mutex_unlock(&efx->mac_lock); 3862874aeea5SJeff Kirsher 3863874aeea5SJeff Kirsher efx_start_all(efx); 3864874aeea5SJeff Kirsher 38659c568fd8SPeter Dunning efx_device_attach_if_not_resetting(efx); 3866874aeea5SJeff Kirsher 3867f16aeea0SBen Hutchings efx->state = STATE_READY; 3868874aeea5SJeff Kirsher 3869874aeea5SJeff Kirsher efx->type->resume_wol(efx); 38706032fb56SBen Hutchings } 3871874aeea5SJeff Kirsher 387261da026dSBen Hutchings rtnl_unlock(); 387361da026dSBen Hutchings 3874874aeea5SJeff Kirsher /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 387582c64484SAlex Maftei (amaftei) efx_queue_reset_work(efx); 3876874aeea5SJeff Kirsher 3877874aeea5SJeff Kirsher return 0; 3878261e4d96SJon Cooper 3879261e4d96SJon Cooper fail: 3880261e4d96SJon Cooper rtnl_unlock(); 3881261e4d96SJon Cooper 3882261e4d96SJon Cooper return rc; 3883874aeea5SJeff Kirsher } 3884874aeea5SJeff Kirsher 3885874aeea5SJeff Kirsher static int efx_pm_poweroff(struct device *dev) 3886874aeea5SJeff Kirsher { 3887874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3888874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3889874aeea5SJeff Kirsher 3890874aeea5SJeff Kirsher efx->type->fini(efx); 3891874aeea5SJeff Kirsher 3892874aeea5SJeff Kirsher efx->reset_pending = 0; 3893874aeea5SJeff Kirsher 3894874aeea5SJeff Kirsher pci_save_state(pci_dev); 3895874aeea5SJeff Kirsher return pci_set_power_state(pci_dev, PCI_D3hot); 3896874aeea5SJeff Kirsher } 3897874aeea5SJeff Kirsher 3898874aeea5SJeff Kirsher /* Used for both resume and restore */ 3899874aeea5SJeff Kirsher static int efx_pm_resume(struct device *dev) 3900874aeea5SJeff Kirsher { 3901874aeea5SJeff Kirsher struct pci_dev *pci_dev = to_pci_dev(dev); 3902874aeea5SJeff Kirsher struct efx_nic *efx = pci_get_drvdata(pci_dev); 3903874aeea5SJeff Kirsher int rc; 3904874aeea5SJeff Kirsher 3905874aeea5SJeff Kirsher rc = pci_set_power_state(pci_dev, PCI_D0); 3906874aeea5SJeff Kirsher if (rc) 3907874aeea5SJeff Kirsher return rc; 3908874aeea5SJeff Kirsher pci_restore_state(pci_dev); 3909874aeea5SJeff Kirsher rc = pci_enable_device(pci_dev); 3910874aeea5SJeff Kirsher if (rc) 3911874aeea5SJeff Kirsher return rc; 3912874aeea5SJeff Kirsher pci_set_master(efx->pci_dev); 3913874aeea5SJeff Kirsher rc = efx->type->reset(efx, RESET_TYPE_ALL); 3914874aeea5SJeff Kirsher if (rc) 3915874aeea5SJeff Kirsher return rc; 3916193f2003SBert Kenward down_write(&efx->filter_sem); 3917874aeea5SJeff Kirsher rc = efx->type->init(efx); 3918193f2003SBert Kenward up_write(&efx->filter_sem); 3919874aeea5SJeff Kirsher if (rc) 3920874aeea5SJeff Kirsher return rc; 3921261e4d96SJon Cooper rc = efx_pm_thaw(dev); 3922261e4d96SJon Cooper return rc; 3923874aeea5SJeff Kirsher } 3924874aeea5SJeff Kirsher 3925874aeea5SJeff Kirsher static int efx_pm_suspend(struct device *dev) 3926874aeea5SJeff Kirsher { 3927874aeea5SJeff Kirsher int rc; 3928874aeea5SJeff Kirsher 3929874aeea5SJeff Kirsher efx_pm_freeze(dev); 3930874aeea5SJeff Kirsher rc = efx_pm_poweroff(dev); 3931874aeea5SJeff Kirsher if (rc) 3932874aeea5SJeff Kirsher efx_pm_resume(dev); 3933874aeea5SJeff Kirsher return rc; 3934874aeea5SJeff Kirsher } 3935874aeea5SJeff Kirsher 393618e83e4cSBen Hutchings static const struct dev_pm_ops efx_pm_ops = { 3937874aeea5SJeff Kirsher .suspend = efx_pm_suspend, 3938874aeea5SJeff Kirsher .resume = efx_pm_resume, 3939874aeea5SJeff Kirsher .freeze = efx_pm_freeze, 3940874aeea5SJeff Kirsher .thaw = efx_pm_thaw, 3941874aeea5SJeff Kirsher .poweroff = efx_pm_poweroff, 3942874aeea5SJeff Kirsher .restore = efx_pm_resume, 3943874aeea5SJeff Kirsher }; 3944874aeea5SJeff Kirsher 3945626950dbSAlexandre Rames /* A PCI error affecting this device was detected. 3946626950dbSAlexandre Rames * At this point MMIO and DMA may be disabled. 3947626950dbSAlexandre Rames * Stop the software path and request a slot reset. 3948626950dbSAlexandre Rames */ 3949debd0034Sstephen hemminger static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3950626950dbSAlexandre Rames enum pci_channel_state state) 3951626950dbSAlexandre Rames { 3952626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3953626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3954626950dbSAlexandre Rames 3955626950dbSAlexandre Rames if (state == pci_channel_io_perm_failure) 3956626950dbSAlexandre Rames return PCI_ERS_RESULT_DISCONNECT; 3957626950dbSAlexandre Rames 3958626950dbSAlexandre Rames rtnl_lock(); 3959626950dbSAlexandre Rames 3960626950dbSAlexandre Rames if (efx->state != STATE_DISABLED) { 3961626950dbSAlexandre Rames efx->state = STATE_RECOVERY; 3962626950dbSAlexandre Rames efx->reset_pending = 0; 3963626950dbSAlexandre Rames 3964626950dbSAlexandre Rames efx_device_detach_sync(efx); 3965626950dbSAlexandre Rames 3966626950dbSAlexandre Rames efx_stop_all(efx); 3967d8291187SBen Hutchings efx_disable_interrupts(efx); 3968626950dbSAlexandre Rames 3969626950dbSAlexandre Rames status = PCI_ERS_RESULT_NEED_RESET; 3970626950dbSAlexandre Rames } else { 3971626950dbSAlexandre Rames /* If the interface is disabled we don't want to do anything 3972626950dbSAlexandre Rames * with it. 3973626950dbSAlexandre Rames */ 3974626950dbSAlexandre Rames status = PCI_ERS_RESULT_RECOVERED; 3975626950dbSAlexandre Rames } 3976626950dbSAlexandre Rames 3977626950dbSAlexandre Rames rtnl_unlock(); 3978626950dbSAlexandre Rames 3979626950dbSAlexandre Rames pci_disable_device(pdev); 3980626950dbSAlexandre Rames 3981626950dbSAlexandre Rames return status; 3982626950dbSAlexandre Rames } 3983626950dbSAlexandre Rames 3984dbedd44eSJoe Perches /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3985debd0034Sstephen hemminger static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3986626950dbSAlexandre Rames { 3987626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 3988626950dbSAlexandre Rames pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3989626950dbSAlexandre Rames 3990626950dbSAlexandre Rames if (pci_enable_device(pdev)) { 3991626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 3992626950dbSAlexandre Rames "Cannot re-enable PCI device after reset.\n"); 3993626950dbSAlexandre Rames status = PCI_ERS_RESULT_DISCONNECT; 3994626950dbSAlexandre Rames } 3995626950dbSAlexandre Rames 3996626950dbSAlexandre Rames return status; 3997626950dbSAlexandre Rames } 3998626950dbSAlexandre Rames 3999626950dbSAlexandre Rames /* Perform the actual reset and resume I/O operations. */ 4000626950dbSAlexandre Rames static void efx_io_resume(struct pci_dev *pdev) 4001626950dbSAlexandre Rames { 4002626950dbSAlexandre Rames struct efx_nic *efx = pci_get_drvdata(pdev); 4003626950dbSAlexandre Rames int rc; 4004626950dbSAlexandre Rames 4005626950dbSAlexandre Rames rtnl_lock(); 4006626950dbSAlexandre Rames 4007626950dbSAlexandre Rames if (efx->state == STATE_DISABLED) 4008626950dbSAlexandre Rames goto out; 4009626950dbSAlexandre Rames 4010626950dbSAlexandre Rames rc = efx_reset(efx, RESET_TYPE_ALL); 4011626950dbSAlexandre Rames if (rc) { 4012626950dbSAlexandre Rames netif_err(efx, hw, efx->net_dev, 4013626950dbSAlexandre Rames "efx_reset failed after PCI error (%d)\n", rc); 4014626950dbSAlexandre Rames } else { 4015626950dbSAlexandre Rames efx->state = STATE_READY; 4016626950dbSAlexandre Rames netif_dbg(efx, hw, efx->net_dev, 4017626950dbSAlexandre Rames "Done resetting and resuming IO after PCI error.\n"); 4018626950dbSAlexandre Rames } 4019626950dbSAlexandre Rames 4020626950dbSAlexandre Rames out: 4021626950dbSAlexandre Rames rtnl_unlock(); 4022626950dbSAlexandre Rames } 4023626950dbSAlexandre Rames 4024626950dbSAlexandre Rames /* For simplicity and reliability, we always require a slot reset and try to 4025626950dbSAlexandre Rames * reset the hardware when a pci error affecting the device is detected. 4026626950dbSAlexandre Rames * We leave both the link_reset and mmio_enabled callback unimplemented: 4027626950dbSAlexandre Rames * with our request for slot reset the mmio_enabled callback will never be 4028626950dbSAlexandre Rames * called, and the link_reset callback is not used by AER or EEH mechanisms. 4029626950dbSAlexandre Rames */ 4030c300366bSJulia Lawall static const struct pci_error_handlers efx_err_handlers = { 4031626950dbSAlexandre Rames .error_detected = efx_io_error_detected, 4032626950dbSAlexandre Rames .slot_reset = efx_io_slot_reset, 4033626950dbSAlexandre Rames .resume = efx_io_resume, 4034626950dbSAlexandre Rames }; 4035626950dbSAlexandre Rames 4036874aeea5SJeff Kirsher static struct pci_driver efx_pci_driver = { 4037874aeea5SJeff Kirsher .name = KBUILD_MODNAME, 4038874aeea5SJeff Kirsher .id_table = efx_pci_table, 4039874aeea5SJeff Kirsher .probe = efx_pci_probe, 4040874aeea5SJeff Kirsher .remove = efx_pci_remove, 4041874aeea5SJeff Kirsher .driver.pm = &efx_pm_ops, 4042626950dbSAlexandre Rames .err_handler = &efx_err_handlers, 4043834e23ddSShradha Shah #ifdef CONFIG_SFC_SRIOV 4044834e23ddSShradha Shah .sriov_configure = efx_pci_sriov_configure, 4045834e23ddSShradha Shah #endif 4046874aeea5SJeff Kirsher }; 4047874aeea5SJeff Kirsher 4048874aeea5SJeff Kirsher /************************************************************************** 4049874aeea5SJeff Kirsher * 4050874aeea5SJeff Kirsher * Kernel module interface 4051874aeea5SJeff Kirsher * 4052874aeea5SJeff Kirsher *************************************************************************/ 4053874aeea5SJeff Kirsher 4054874aeea5SJeff Kirsher module_param(interrupt_mode, uint, 0444); 4055874aeea5SJeff Kirsher MODULE_PARM_DESC(interrupt_mode, 4056874aeea5SJeff Kirsher "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 4057874aeea5SJeff Kirsher 4058874aeea5SJeff Kirsher static int __init efx_init_module(void) 4059874aeea5SJeff Kirsher { 4060874aeea5SJeff Kirsher int rc; 4061874aeea5SJeff Kirsher 4062874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 4063874aeea5SJeff Kirsher 4064874aeea5SJeff Kirsher rc = register_netdevice_notifier(&efx_netdev_notifier); 4065874aeea5SJeff Kirsher if (rc) 4066874aeea5SJeff Kirsher goto err_notifier; 4067874aeea5SJeff Kirsher 40687fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 4069cd2d5b52SBen Hutchings rc = efx_init_sriov(); 4070cd2d5b52SBen Hutchings if (rc) 4071cd2d5b52SBen Hutchings goto err_sriov; 40727fa8d547SShradha Shah #endif 4073cd2d5b52SBen Hutchings 407482c64484SAlex Maftei (amaftei) rc = efx_create_reset_workqueue(); 407582c64484SAlex Maftei (amaftei) if (rc) 4076874aeea5SJeff Kirsher goto err_reset; 4077874aeea5SJeff Kirsher 4078874aeea5SJeff Kirsher rc = pci_register_driver(&efx_pci_driver); 4079874aeea5SJeff Kirsher if (rc < 0) 4080874aeea5SJeff Kirsher goto err_pci; 4081874aeea5SJeff Kirsher 4082874aeea5SJeff Kirsher return 0; 4083874aeea5SJeff Kirsher 4084874aeea5SJeff Kirsher err_pci: 408582c64484SAlex Maftei (amaftei) efx_destroy_reset_workqueue(); 4086874aeea5SJeff Kirsher err_reset: 40877fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 4088cd2d5b52SBen Hutchings efx_fini_sriov(); 4089cd2d5b52SBen Hutchings err_sriov: 40907fa8d547SShradha Shah #endif 4091874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 4092874aeea5SJeff Kirsher err_notifier: 4093874aeea5SJeff Kirsher return rc; 4094874aeea5SJeff Kirsher } 4095874aeea5SJeff Kirsher 4096874aeea5SJeff Kirsher static void __exit efx_exit_module(void) 4097874aeea5SJeff Kirsher { 4098874aeea5SJeff Kirsher printk(KERN_INFO "Solarflare NET driver unloading\n"); 4099874aeea5SJeff Kirsher 4100874aeea5SJeff Kirsher pci_unregister_driver(&efx_pci_driver); 410182c64484SAlex Maftei (amaftei) efx_destroy_reset_workqueue(); 41027fa8d547SShradha Shah #ifdef CONFIG_SFC_SRIOV 4103cd2d5b52SBen Hutchings efx_fini_sriov(); 41047fa8d547SShradha Shah #endif 4105874aeea5SJeff Kirsher unregister_netdevice_notifier(&efx_netdev_notifier); 4106874aeea5SJeff Kirsher 4107874aeea5SJeff Kirsher } 4108874aeea5SJeff Kirsher 4109874aeea5SJeff Kirsher module_init(efx_init_module); 4110874aeea5SJeff Kirsher module_exit(efx_exit_module); 4111874aeea5SJeff Kirsher 4112874aeea5SJeff Kirsher MODULE_AUTHOR("Solarflare Communications and " 4113874aeea5SJeff Kirsher "Michael Brown <mbrown@fensystems.co.uk>"); 41146a350fdbSBen Hutchings MODULE_DESCRIPTION("Solarflare network driver"); 4115874aeea5SJeff Kirsher MODULE_LICENSE("GPL"); 4116874aeea5SJeff Kirsher MODULE_DEVICE_TABLE(pci, efx_pci_table); 411714077e9eSEdward Cree MODULE_VERSION(EFX_DRIVER_VERSION); 4118