1768fd266SAlex Maftei (amaftei) // SPDX-License-Identifier: GPL-2.0-only 2768fd266SAlex Maftei (amaftei) /**************************************************************************** 3768fd266SAlex Maftei (amaftei) * Driver for Solarflare network controllers and boards 4768fd266SAlex Maftei (amaftei) * Copyright 2018 Solarflare Communications Inc. 5768fd266SAlex Maftei (amaftei) * 6768fd266SAlex Maftei (amaftei) * This program is free software; you can redistribute it and/or modify it 7768fd266SAlex Maftei (amaftei) * under the terms of the GNU General Public License version 2 as published 8768fd266SAlex Maftei (amaftei) * by the Free Software Foundation, incorporated herein by reference. 9768fd266SAlex Maftei (amaftei) */ 10768fd266SAlex Maftei (amaftei) 11768fd266SAlex Maftei (amaftei) #include "net_driver.h" 12768fd266SAlex Maftei (amaftei) #include <linux/module.h> 13768fd266SAlex Maftei (amaftei) #include "efx_channels.h" 14768fd266SAlex Maftei (amaftei) #include "efx.h" 15768fd266SAlex Maftei (amaftei) #include "efx_common.h" 16768fd266SAlex Maftei (amaftei) #include "tx_common.h" 17768fd266SAlex Maftei (amaftei) #include "rx_common.h" 18768fd266SAlex Maftei (amaftei) #include "nic.h" 19768fd266SAlex Maftei (amaftei) #include "sriov.h" 20768fd266SAlex Maftei (amaftei) 2183975485SAlex Maftei (amaftei) /* This is the first interrupt mode to try out of: 2283975485SAlex Maftei (amaftei) * 0 => MSI-X 2383975485SAlex Maftei (amaftei) * 1 => MSI 2483975485SAlex Maftei (amaftei) * 2 => legacy 2583975485SAlex Maftei (amaftei) */ 2683975485SAlex Maftei (amaftei) static unsigned int interrupt_mode; 2783975485SAlex Maftei (amaftei) module_param(interrupt_mode, uint, 0444); 2883975485SAlex Maftei (amaftei) MODULE_PARM_DESC(interrupt_mode, 2983975485SAlex Maftei (amaftei) "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3083975485SAlex Maftei (amaftei) 3137c45a4eSAlex Maftei (amaftei) /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 3237c45a4eSAlex Maftei (amaftei) * i.e. the number of CPUs among which we may distribute simultaneous 3337c45a4eSAlex Maftei (amaftei) * interrupt handling. 3437c45a4eSAlex Maftei (amaftei) * 3537c45a4eSAlex Maftei (amaftei) * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 3637c45a4eSAlex Maftei (amaftei) * The default (0) means to assign an interrupt to each core. 3737c45a4eSAlex Maftei (amaftei) */ 3837c45a4eSAlex Maftei (amaftei) static unsigned int rss_cpus; 3937c45a4eSAlex Maftei (amaftei) module_param(rss_cpus, uint, 0444); 4037c45a4eSAlex Maftei (amaftei) MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 4137c45a4eSAlex Maftei (amaftei) 42768fd266SAlex Maftei (amaftei) static unsigned int irq_adapt_low_thresh = 8000; 43768fd266SAlex Maftei (amaftei) module_param(irq_adapt_low_thresh, uint, 0644); 44768fd266SAlex Maftei (amaftei) MODULE_PARM_DESC(irq_adapt_low_thresh, 45768fd266SAlex Maftei (amaftei) "Threshold score for reducing IRQ moderation"); 46768fd266SAlex Maftei (amaftei) 47768fd266SAlex Maftei (amaftei) static unsigned int irq_adapt_high_thresh = 16000; 48768fd266SAlex Maftei (amaftei) module_param(irq_adapt_high_thresh, uint, 0644); 49768fd266SAlex Maftei (amaftei) MODULE_PARM_DESC(irq_adapt_high_thresh, 50768fd266SAlex Maftei (amaftei) "Threshold score for increasing IRQ moderation"); 51768fd266SAlex Maftei (amaftei) 52768fd266SAlex Maftei (amaftei) /* This is the weight assigned to each of the (per-channel) virtual 53768fd266SAlex Maftei (amaftei) * NAPI devices. 54768fd266SAlex Maftei (amaftei) */ 55768fd266SAlex Maftei (amaftei) static int napi_weight = 64; 56768fd266SAlex Maftei (amaftei) 5783975485SAlex Maftei (amaftei) /*************** 5883975485SAlex Maftei (amaftei) * Housekeeping 5983975485SAlex Maftei (amaftei) ***************/ 6083975485SAlex Maftei (amaftei) 6183975485SAlex Maftei (amaftei) int efx_channel_dummy_op_int(struct efx_channel *channel) 6283975485SAlex Maftei (amaftei) { 6383975485SAlex Maftei (amaftei) return 0; 6483975485SAlex Maftei (amaftei) } 6583975485SAlex Maftei (amaftei) 6683975485SAlex Maftei (amaftei) void efx_channel_dummy_op_void(struct efx_channel *channel) 6783975485SAlex Maftei (amaftei) { 6883975485SAlex Maftei (amaftei) } 6983975485SAlex Maftei (amaftei) 7083975485SAlex Maftei (amaftei) static const struct efx_channel_type efx_default_channel_type = { 7183975485SAlex Maftei (amaftei) .pre_probe = efx_channel_dummy_op_int, 7283975485SAlex Maftei (amaftei) .post_remove = efx_channel_dummy_op_void, 7383975485SAlex Maftei (amaftei) .get_name = efx_get_channel_name, 7483975485SAlex Maftei (amaftei) .copy = efx_copy_channel, 7583975485SAlex Maftei (amaftei) .want_txqs = efx_default_channel_want_txqs, 7683975485SAlex Maftei (amaftei) .keep_eventq = false, 7783975485SAlex Maftei (amaftei) .want_pio = true, 7883975485SAlex Maftei (amaftei) }; 7983975485SAlex Maftei (amaftei) 8037c45a4eSAlex Maftei (amaftei) /************* 8137c45a4eSAlex Maftei (amaftei) * INTERRUPTS 8237c45a4eSAlex Maftei (amaftei) *************/ 8337c45a4eSAlex Maftei (amaftei) 8437c45a4eSAlex Maftei (amaftei) static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 8537c45a4eSAlex Maftei (amaftei) { 8637c45a4eSAlex Maftei (amaftei) cpumask_var_t thread_mask; 8737c45a4eSAlex Maftei (amaftei) unsigned int count; 8837c45a4eSAlex Maftei (amaftei) int cpu; 8937c45a4eSAlex Maftei (amaftei) 9037c45a4eSAlex Maftei (amaftei) if (rss_cpus) { 9137c45a4eSAlex Maftei (amaftei) count = rss_cpus; 9237c45a4eSAlex Maftei (amaftei) } else { 9337c45a4eSAlex Maftei (amaftei) if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 9437c45a4eSAlex Maftei (amaftei) netif_warn(efx, probe, efx->net_dev, 9537c45a4eSAlex Maftei (amaftei) "RSS disabled due to allocation failure\n"); 9637c45a4eSAlex Maftei (amaftei) return 1; 9737c45a4eSAlex Maftei (amaftei) } 9837c45a4eSAlex Maftei (amaftei) 9937c45a4eSAlex Maftei (amaftei) count = 0; 10037c45a4eSAlex Maftei (amaftei) for_each_online_cpu(cpu) { 10137c45a4eSAlex Maftei (amaftei) if (!cpumask_test_cpu(cpu, thread_mask)) { 10237c45a4eSAlex Maftei (amaftei) ++count; 10337c45a4eSAlex Maftei (amaftei) cpumask_or(thread_mask, thread_mask, 10437c45a4eSAlex Maftei (amaftei) topology_sibling_cpumask(cpu)); 10537c45a4eSAlex Maftei (amaftei) } 10637c45a4eSAlex Maftei (amaftei) } 10737c45a4eSAlex Maftei (amaftei) 10837c45a4eSAlex Maftei (amaftei) free_cpumask_var(thread_mask); 10937c45a4eSAlex Maftei (amaftei) } 11037c45a4eSAlex Maftei (amaftei) 11137c45a4eSAlex Maftei (amaftei) if (count > EFX_MAX_RX_QUEUES) { 11237c45a4eSAlex Maftei (amaftei) netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 11337c45a4eSAlex Maftei (amaftei) "Reducing number of rx queues from %u to %u.\n", 11437c45a4eSAlex Maftei (amaftei) count, EFX_MAX_RX_QUEUES); 11537c45a4eSAlex Maftei (amaftei) count = EFX_MAX_RX_QUEUES; 11637c45a4eSAlex Maftei (amaftei) } 11737c45a4eSAlex Maftei (amaftei) 11837c45a4eSAlex Maftei (amaftei) /* If RSS is requested for the PF *and* VFs then we can't write RSS 11937c45a4eSAlex Maftei (amaftei) * table entries that are inaccessible to VFs 12037c45a4eSAlex Maftei (amaftei) */ 12137c45a4eSAlex Maftei (amaftei) #ifdef CONFIG_SFC_SRIOV 12237c45a4eSAlex Maftei (amaftei) if (efx->type->sriov_wanted) { 12337c45a4eSAlex Maftei (amaftei) if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 12437c45a4eSAlex Maftei (amaftei) count > efx_vf_size(efx)) { 12537c45a4eSAlex Maftei (amaftei) netif_warn(efx, probe, efx->net_dev, 12637c45a4eSAlex Maftei (amaftei) "Reducing number of RSS channels from %u to %u for " 12737c45a4eSAlex Maftei (amaftei) "VF support. Increase vf-msix-limit to use more " 12837c45a4eSAlex Maftei (amaftei) "channels on the PF.\n", 12937c45a4eSAlex Maftei (amaftei) count, efx_vf_size(efx)); 13037c45a4eSAlex Maftei (amaftei) count = efx_vf_size(efx); 13137c45a4eSAlex Maftei (amaftei) } 13237c45a4eSAlex Maftei (amaftei) } 13337c45a4eSAlex Maftei (amaftei) #endif 13437c45a4eSAlex Maftei (amaftei) 13537c45a4eSAlex Maftei (amaftei) return count; 13637c45a4eSAlex Maftei (amaftei) } 13737c45a4eSAlex Maftei (amaftei) 13837c45a4eSAlex Maftei (amaftei) static int efx_allocate_msix_channels(struct efx_nic *efx, 13937c45a4eSAlex Maftei (amaftei) unsigned int max_channels, 14037c45a4eSAlex Maftei (amaftei) unsigned int extra_channels, 14137c45a4eSAlex Maftei (amaftei) unsigned int parallelism) 14237c45a4eSAlex Maftei (amaftei) { 14337c45a4eSAlex Maftei (amaftei) unsigned int n_channels = parallelism; 14437c45a4eSAlex Maftei (amaftei) int vec_count; 14537c45a4eSAlex Maftei (amaftei) int n_xdp_tx; 14637c45a4eSAlex Maftei (amaftei) int n_xdp_ev; 14737c45a4eSAlex Maftei (amaftei) 14837c45a4eSAlex Maftei (amaftei) if (efx_separate_tx_channels) 14937c45a4eSAlex Maftei (amaftei) n_channels *= 2; 15037c45a4eSAlex Maftei (amaftei) n_channels += extra_channels; 15137c45a4eSAlex Maftei (amaftei) 15237c45a4eSAlex Maftei (amaftei) /* To allow XDP transmit to happen from arbitrary NAPI contexts 15337c45a4eSAlex Maftei (amaftei) * we allocate a TX queue per CPU. We share event queues across 15437c45a4eSAlex Maftei (amaftei) * multiple tx queues, assuming tx and ev queues are both 15537c45a4eSAlex Maftei (amaftei) * maximum size. 15637c45a4eSAlex Maftei (amaftei) */ 15737c45a4eSAlex Maftei (amaftei) 15837c45a4eSAlex Maftei (amaftei) n_xdp_tx = num_possible_cpus(); 15937c45a4eSAlex Maftei (amaftei) n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_TXQ_TYPES); 16037c45a4eSAlex Maftei (amaftei) 16137c45a4eSAlex Maftei (amaftei) vec_count = pci_msix_vec_count(efx->pci_dev); 16237c45a4eSAlex Maftei (amaftei) if (vec_count < 0) 16337c45a4eSAlex Maftei (amaftei) return vec_count; 16437c45a4eSAlex Maftei (amaftei) 16537c45a4eSAlex Maftei (amaftei) max_channels = min_t(unsigned int, vec_count, max_channels); 16637c45a4eSAlex Maftei (amaftei) 16737c45a4eSAlex Maftei (amaftei) /* Check resources. 16837c45a4eSAlex Maftei (amaftei) * We need a channel per event queue, plus a VI per tx queue. 16937c45a4eSAlex Maftei (amaftei) * This may be more pessimistic than it needs to be. 17037c45a4eSAlex Maftei (amaftei) */ 17137c45a4eSAlex Maftei (amaftei) if (n_channels + n_xdp_ev > max_channels) { 17237c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 17337c45a4eSAlex Maftei (amaftei) "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", 17437c45a4eSAlex Maftei (amaftei) n_xdp_ev, n_channels, max_channels); 17537c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = 0; 17637c45a4eSAlex Maftei (amaftei) efx->xdp_tx_per_channel = 0; 17737c45a4eSAlex Maftei (amaftei) efx->xdp_tx_queue_count = 0; 17837c45a4eSAlex Maftei (amaftei) } else { 17937c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = n_xdp_ev; 18037c45a4eSAlex Maftei (amaftei) efx->xdp_tx_per_channel = EFX_TXQ_TYPES; 18137c45a4eSAlex Maftei (amaftei) efx->xdp_tx_queue_count = n_xdp_tx; 18237c45a4eSAlex Maftei (amaftei) n_channels += n_xdp_ev; 18337c45a4eSAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 18437c45a4eSAlex Maftei (amaftei) "Allocating %d TX and %d event queues for XDP\n", 18537c45a4eSAlex Maftei (amaftei) n_xdp_tx, n_xdp_ev); 18637c45a4eSAlex Maftei (amaftei) } 18737c45a4eSAlex Maftei (amaftei) 18837c45a4eSAlex Maftei (amaftei) if (vec_count < n_channels) { 18937c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 19037c45a4eSAlex Maftei (amaftei) "WARNING: Insufficient MSI-X vectors available (%d < %u).\n", 19137c45a4eSAlex Maftei (amaftei) vec_count, n_channels); 19237c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 19337c45a4eSAlex Maftei (amaftei) "WARNING: Performance may be reduced.\n"); 19437c45a4eSAlex Maftei (amaftei) n_channels = vec_count; 19537c45a4eSAlex Maftei (amaftei) } 19637c45a4eSAlex Maftei (amaftei) 19737c45a4eSAlex Maftei (amaftei) n_channels = min(n_channels, max_channels); 19837c45a4eSAlex Maftei (amaftei) 19937c45a4eSAlex Maftei (amaftei) efx->n_channels = n_channels; 20037c45a4eSAlex Maftei (amaftei) 20137c45a4eSAlex Maftei (amaftei) /* Ignore XDP tx channels when creating rx channels. */ 20237c45a4eSAlex Maftei (amaftei) n_channels -= efx->n_xdp_channels; 20337c45a4eSAlex Maftei (amaftei) 20437c45a4eSAlex Maftei (amaftei) if (efx_separate_tx_channels) { 20537c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = 20637c45a4eSAlex Maftei (amaftei) min(max(n_channels / 2, 1U), 20737c45a4eSAlex Maftei (amaftei) efx->max_tx_channels); 20837c45a4eSAlex Maftei (amaftei) efx->tx_channel_offset = 20937c45a4eSAlex Maftei (amaftei) n_channels - efx->n_tx_channels; 21037c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = 21137c45a4eSAlex Maftei (amaftei) max(n_channels - 21237c45a4eSAlex Maftei (amaftei) efx->n_tx_channels, 1U); 21337c45a4eSAlex Maftei (amaftei) } else { 21437c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = min(n_channels, efx->max_tx_channels); 21537c45a4eSAlex Maftei (amaftei) efx->tx_channel_offset = 0; 21637c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = n_channels; 21737c45a4eSAlex Maftei (amaftei) } 21837c45a4eSAlex Maftei (amaftei) 21937c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = min(efx->n_rx_channels, parallelism); 22037c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = min(efx->n_tx_channels, parallelism); 22137c45a4eSAlex Maftei (amaftei) 22237c45a4eSAlex Maftei (amaftei) efx->xdp_channel_offset = n_channels; 22337c45a4eSAlex Maftei (amaftei) 22437c45a4eSAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 22537c45a4eSAlex Maftei (amaftei) "Allocating %u RX channels\n", 22637c45a4eSAlex Maftei (amaftei) efx->n_rx_channels); 22737c45a4eSAlex Maftei (amaftei) 22837c45a4eSAlex Maftei (amaftei) return efx->n_channels; 22937c45a4eSAlex Maftei (amaftei) } 23037c45a4eSAlex Maftei (amaftei) 23137c45a4eSAlex Maftei (amaftei) /* Probe the number and type of interrupts we are able to obtain, and 23237c45a4eSAlex Maftei (amaftei) * the resulting numbers of channels and RX queues. 23337c45a4eSAlex Maftei (amaftei) */ 23437c45a4eSAlex Maftei (amaftei) int efx_probe_interrupts(struct efx_nic *efx) 23537c45a4eSAlex Maftei (amaftei) { 23637c45a4eSAlex Maftei (amaftei) unsigned int extra_channels = 0; 23737c45a4eSAlex Maftei (amaftei) unsigned int rss_spread; 23837c45a4eSAlex Maftei (amaftei) unsigned int i, j; 23937c45a4eSAlex Maftei (amaftei) int rc; 24037c45a4eSAlex Maftei (amaftei) 24137c45a4eSAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 24237c45a4eSAlex Maftei (amaftei) if (efx->extra_channel_type[i]) 24337c45a4eSAlex Maftei (amaftei) ++extra_channels; 24437c45a4eSAlex Maftei (amaftei) 24537c45a4eSAlex Maftei (amaftei) if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 24637c45a4eSAlex Maftei (amaftei) unsigned int parallelism = efx_wanted_parallelism(efx); 24737c45a4eSAlex Maftei (amaftei) struct msix_entry xentries[EFX_MAX_CHANNELS]; 24837c45a4eSAlex Maftei (amaftei) unsigned int n_channels; 24937c45a4eSAlex Maftei (amaftei) 25037c45a4eSAlex Maftei (amaftei) rc = efx_allocate_msix_channels(efx, efx->max_channels, 25137c45a4eSAlex Maftei (amaftei) extra_channels, parallelism); 25237c45a4eSAlex Maftei (amaftei) if (rc >= 0) { 25337c45a4eSAlex Maftei (amaftei) n_channels = rc; 25437c45a4eSAlex Maftei (amaftei) for (i = 0; i < n_channels; i++) 25537c45a4eSAlex Maftei (amaftei) xentries[i].entry = i; 25637c45a4eSAlex Maftei (amaftei) rc = pci_enable_msix_range(efx->pci_dev, xentries, 1, 25737c45a4eSAlex Maftei (amaftei) n_channels); 25837c45a4eSAlex Maftei (amaftei) } 25937c45a4eSAlex Maftei (amaftei) if (rc < 0) { 26037c45a4eSAlex Maftei (amaftei) /* Fall back to single channel MSI */ 26137c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 26237c45a4eSAlex Maftei (amaftei) "could not enable MSI-X\n"); 26337c45a4eSAlex Maftei (amaftei) if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 26437c45a4eSAlex Maftei (amaftei) efx->interrupt_mode = EFX_INT_MODE_MSI; 26537c45a4eSAlex Maftei (amaftei) else 26637c45a4eSAlex Maftei (amaftei) return rc; 26737c45a4eSAlex Maftei (amaftei) } else if (rc < n_channels) { 26837c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 26937c45a4eSAlex Maftei (amaftei) "WARNING: Insufficient MSI-X vectors" 27037c45a4eSAlex Maftei (amaftei) " available (%d < %u).\n", rc, n_channels); 27137c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 27237c45a4eSAlex Maftei (amaftei) "WARNING: Performance may be reduced.\n"); 27337c45a4eSAlex Maftei (amaftei) n_channels = rc; 27437c45a4eSAlex Maftei (amaftei) } 27537c45a4eSAlex Maftei (amaftei) 27637c45a4eSAlex Maftei (amaftei) if (rc > 0) { 27737c45a4eSAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) 27837c45a4eSAlex Maftei (amaftei) efx_get_channel(efx, i)->irq = 27937c45a4eSAlex Maftei (amaftei) xentries[i].vector; 28037c45a4eSAlex Maftei (amaftei) } 28137c45a4eSAlex Maftei (amaftei) } 28237c45a4eSAlex Maftei (amaftei) 28337c45a4eSAlex Maftei (amaftei) /* Try single interrupt MSI */ 28437c45a4eSAlex Maftei (amaftei) if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 28537c45a4eSAlex Maftei (amaftei) efx->n_channels = 1; 28637c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = 1; 28737c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = 1; 28837c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = 0; 28937c45a4eSAlex Maftei (amaftei) efx->xdp_channel_offset = efx->n_channels; 29037c45a4eSAlex Maftei (amaftei) rc = pci_enable_msi(efx->pci_dev); 29137c45a4eSAlex Maftei (amaftei) if (rc == 0) { 29237c45a4eSAlex Maftei (amaftei) efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 29337c45a4eSAlex Maftei (amaftei) } else { 29437c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 29537c45a4eSAlex Maftei (amaftei) "could not enable MSI\n"); 29637c45a4eSAlex Maftei (amaftei) if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 29737c45a4eSAlex Maftei (amaftei) efx->interrupt_mode = EFX_INT_MODE_LEGACY; 29837c45a4eSAlex Maftei (amaftei) else 29937c45a4eSAlex Maftei (amaftei) return rc; 30037c45a4eSAlex Maftei (amaftei) } 30137c45a4eSAlex Maftei (amaftei) } 30237c45a4eSAlex Maftei (amaftei) 30337c45a4eSAlex Maftei (amaftei) /* Assume legacy interrupts */ 30437c45a4eSAlex Maftei (amaftei) if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 30537c45a4eSAlex Maftei (amaftei) efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 30637c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = 1; 30737c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = 1; 30837c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = 0; 30937c45a4eSAlex Maftei (amaftei) efx->xdp_channel_offset = efx->n_channels; 31037c45a4eSAlex Maftei (amaftei) efx->legacy_irq = efx->pci_dev->irq; 31137c45a4eSAlex Maftei (amaftei) } 31237c45a4eSAlex Maftei (amaftei) 31337c45a4eSAlex Maftei (amaftei) /* Assign extra channels if possible, before XDP channels */ 31437c45a4eSAlex Maftei (amaftei) efx->n_extra_tx_channels = 0; 31537c45a4eSAlex Maftei (amaftei) j = efx->xdp_channel_offset; 31637c45a4eSAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 31737c45a4eSAlex Maftei (amaftei) if (!efx->extra_channel_type[i]) 31837c45a4eSAlex Maftei (amaftei) continue; 31937c45a4eSAlex Maftei (amaftei) if (j <= efx->tx_channel_offset + efx->n_tx_channels) { 32037c45a4eSAlex Maftei (amaftei) efx->extra_channel_type[i]->handle_no_channel(efx); 32137c45a4eSAlex Maftei (amaftei) } else { 32237c45a4eSAlex Maftei (amaftei) --j; 32337c45a4eSAlex Maftei (amaftei) efx_get_channel(efx, j)->type = 32437c45a4eSAlex Maftei (amaftei) efx->extra_channel_type[i]; 32537c45a4eSAlex Maftei (amaftei) if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 32637c45a4eSAlex Maftei (amaftei) efx->n_extra_tx_channels++; 32737c45a4eSAlex Maftei (amaftei) } 32837c45a4eSAlex Maftei (amaftei) } 32937c45a4eSAlex Maftei (amaftei) 33037c45a4eSAlex Maftei (amaftei) rss_spread = efx->n_rx_channels; 33137c45a4eSAlex Maftei (amaftei) /* RSS might be usable on VFs even if it is disabled on the PF */ 33237c45a4eSAlex Maftei (amaftei) #ifdef CONFIG_SFC_SRIOV 33337c45a4eSAlex Maftei (amaftei) if (efx->type->sriov_wanted) { 33437c45a4eSAlex Maftei (amaftei) efx->rss_spread = ((rss_spread > 1 || 33537c45a4eSAlex Maftei (amaftei) !efx->type->sriov_wanted(efx)) ? 33637c45a4eSAlex Maftei (amaftei) rss_spread : efx_vf_size(efx)); 33737c45a4eSAlex Maftei (amaftei) return 0; 33837c45a4eSAlex Maftei (amaftei) } 33937c45a4eSAlex Maftei (amaftei) #endif 34037c45a4eSAlex Maftei (amaftei) efx->rss_spread = rss_spread; 34137c45a4eSAlex Maftei (amaftei) 34237c45a4eSAlex Maftei (amaftei) return 0; 34337c45a4eSAlex Maftei (amaftei) } 34437c45a4eSAlex Maftei (amaftei) 34537c45a4eSAlex Maftei (amaftei) #if defined(CONFIG_SMP) 34637c45a4eSAlex Maftei (amaftei) void efx_set_interrupt_affinity(struct efx_nic *efx) 34737c45a4eSAlex Maftei (amaftei) { 34837c45a4eSAlex Maftei (amaftei) struct efx_channel *channel; 34937c45a4eSAlex Maftei (amaftei) unsigned int cpu; 35037c45a4eSAlex Maftei (amaftei) 35137c45a4eSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 35237c45a4eSAlex Maftei (amaftei) cpu = cpumask_local_spread(channel->channel, 35337c45a4eSAlex Maftei (amaftei) pcibus_to_node(efx->pci_dev->bus)); 35437c45a4eSAlex Maftei (amaftei) irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 35537c45a4eSAlex Maftei (amaftei) } 35637c45a4eSAlex Maftei (amaftei) } 35737c45a4eSAlex Maftei (amaftei) 35837c45a4eSAlex Maftei (amaftei) void efx_clear_interrupt_affinity(struct efx_nic *efx) 35937c45a4eSAlex Maftei (amaftei) { 36037c45a4eSAlex Maftei (amaftei) struct efx_channel *channel; 36137c45a4eSAlex Maftei (amaftei) 36237c45a4eSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 36337c45a4eSAlex Maftei (amaftei) irq_set_affinity_hint(channel->irq, NULL); 36437c45a4eSAlex Maftei (amaftei) } 36537c45a4eSAlex Maftei (amaftei) #else 36637c45a4eSAlex Maftei (amaftei) void 36737c45a4eSAlex Maftei (amaftei) efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 36837c45a4eSAlex Maftei (amaftei) { 36937c45a4eSAlex Maftei (amaftei) } 37037c45a4eSAlex Maftei (amaftei) 37137c45a4eSAlex Maftei (amaftei) void 37237c45a4eSAlex Maftei (amaftei) efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 37337c45a4eSAlex Maftei (amaftei) { 37437c45a4eSAlex Maftei (amaftei) } 37537c45a4eSAlex Maftei (amaftei) #endif /* CONFIG_SMP */ 37637c45a4eSAlex Maftei (amaftei) 37737c45a4eSAlex Maftei (amaftei) void efx_remove_interrupts(struct efx_nic *efx) 37837c45a4eSAlex Maftei (amaftei) { 37937c45a4eSAlex Maftei (amaftei) struct efx_channel *channel; 38037c45a4eSAlex Maftei (amaftei) 38137c45a4eSAlex Maftei (amaftei) /* Remove MSI/MSI-X interrupts */ 38237c45a4eSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 38337c45a4eSAlex Maftei (amaftei) channel->irq = 0; 38437c45a4eSAlex Maftei (amaftei) pci_disable_msi(efx->pci_dev); 38537c45a4eSAlex Maftei (amaftei) pci_disable_msix(efx->pci_dev); 38637c45a4eSAlex Maftei (amaftei) 38737c45a4eSAlex Maftei (amaftei) /* Remove legacy interrupt */ 38837c45a4eSAlex Maftei (amaftei) efx->legacy_irq = 0; 38937c45a4eSAlex Maftei (amaftei) } 39037c45a4eSAlex Maftei (amaftei) 39183975485SAlex Maftei (amaftei) /************************************************************************** 39283975485SAlex Maftei (amaftei) * 39383975485SAlex Maftei (amaftei) * Channel handling 39483975485SAlex Maftei (amaftei) * 39583975485SAlex Maftei (amaftei) *************************************************************************/ 39683975485SAlex Maftei (amaftei) 39783975485SAlex Maftei (amaftei) /* Allocate and initialise a channel structure. */ 39883975485SAlex Maftei (amaftei) struct efx_channel * 39983975485SAlex Maftei (amaftei) efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 40083975485SAlex Maftei (amaftei) { 40183975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 40283975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 40383975485SAlex Maftei (amaftei) struct efx_channel *channel; 40483975485SAlex Maftei (amaftei) int j; 40583975485SAlex Maftei (amaftei) 40683975485SAlex Maftei (amaftei) channel = kzalloc(sizeof(*channel), GFP_KERNEL); 40783975485SAlex Maftei (amaftei) if (!channel) 40883975485SAlex Maftei (amaftei) return NULL; 40983975485SAlex Maftei (amaftei) 41083975485SAlex Maftei (amaftei) channel->efx = efx; 41183975485SAlex Maftei (amaftei) channel->channel = i; 41283975485SAlex Maftei (amaftei) channel->type = &efx_default_channel_type; 41383975485SAlex Maftei (amaftei) 41483975485SAlex Maftei (amaftei) for (j = 0; j < EFX_TXQ_TYPES; j++) { 41583975485SAlex Maftei (amaftei) tx_queue = &channel->tx_queue[j]; 41683975485SAlex Maftei (amaftei) tx_queue->efx = efx; 41783975485SAlex Maftei (amaftei) tx_queue->queue = i * EFX_TXQ_TYPES + j; 41883975485SAlex Maftei (amaftei) tx_queue->channel = channel; 41983975485SAlex Maftei (amaftei) } 42083975485SAlex Maftei (amaftei) 42183975485SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 42283975485SAlex Maftei (amaftei) INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 42383975485SAlex Maftei (amaftei) #endif 42483975485SAlex Maftei (amaftei) 42583975485SAlex Maftei (amaftei) rx_queue = &channel->rx_queue; 42683975485SAlex Maftei (amaftei) rx_queue->efx = efx; 42783975485SAlex Maftei (amaftei) timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 42883975485SAlex Maftei (amaftei) 42983975485SAlex Maftei (amaftei) return channel; 43083975485SAlex Maftei (amaftei) } 43183975485SAlex Maftei (amaftei) 43283975485SAlex Maftei (amaftei) int efx_init_channels(struct efx_nic *efx) 43383975485SAlex Maftei (amaftei) { 43483975485SAlex Maftei (amaftei) unsigned int i; 43583975485SAlex Maftei (amaftei) 43683975485SAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_CHANNELS; i++) { 43783975485SAlex Maftei (amaftei) efx->channel[i] = efx_alloc_channel(efx, i, NULL); 43883975485SAlex Maftei (amaftei) if (!efx->channel[i]) 43983975485SAlex Maftei (amaftei) return -ENOMEM; 44083975485SAlex Maftei (amaftei) efx->msi_context[i].efx = efx; 44183975485SAlex Maftei (amaftei) efx->msi_context[i].index = i; 44283975485SAlex Maftei (amaftei) } 44383975485SAlex Maftei (amaftei) 44483975485SAlex Maftei (amaftei) /* Higher numbered interrupt modes are less capable! */ 44583975485SAlex Maftei (amaftei) if (WARN_ON_ONCE(efx->type->max_interrupt_mode > 44683975485SAlex Maftei (amaftei) efx->type->min_interrupt_mode)) { 44783975485SAlex Maftei (amaftei) return -EIO; 44883975485SAlex Maftei (amaftei) } 44983975485SAlex Maftei (amaftei) efx->interrupt_mode = max(efx->type->max_interrupt_mode, 45083975485SAlex Maftei (amaftei) interrupt_mode); 45183975485SAlex Maftei (amaftei) efx->interrupt_mode = min(efx->type->min_interrupt_mode, 45283975485SAlex Maftei (amaftei) interrupt_mode); 45383975485SAlex Maftei (amaftei) 45483975485SAlex Maftei (amaftei) return 0; 45583975485SAlex Maftei (amaftei) } 45683975485SAlex Maftei (amaftei) 45783975485SAlex Maftei (amaftei) void efx_fini_channels(struct efx_nic *efx) 45883975485SAlex Maftei (amaftei) { 45983975485SAlex Maftei (amaftei) unsigned int i; 46083975485SAlex Maftei (amaftei) 46183975485SAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_CHANNELS; i++) 46283975485SAlex Maftei (amaftei) if (efx->channel[i]) { 46383975485SAlex Maftei (amaftei) kfree(efx->channel[i]); 46483975485SAlex Maftei (amaftei) efx->channel[i] = NULL; 46583975485SAlex Maftei (amaftei) } 46683975485SAlex Maftei (amaftei) } 46783975485SAlex Maftei (amaftei) 46883975485SAlex Maftei (amaftei) /* Allocate and initialise a channel structure, copying parameters 46983975485SAlex Maftei (amaftei) * (but not resources) from an old channel structure. 47083975485SAlex Maftei (amaftei) */ 47183975485SAlex Maftei (amaftei) struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel) 47283975485SAlex Maftei (amaftei) { 47383975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 47483975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 47583975485SAlex Maftei (amaftei) struct efx_channel *channel; 47683975485SAlex Maftei (amaftei) int j; 47783975485SAlex Maftei (amaftei) 47883975485SAlex Maftei (amaftei) channel = kmalloc(sizeof(*channel), GFP_KERNEL); 47983975485SAlex Maftei (amaftei) if (!channel) 48083975485SAlex Maftei (amaftei) return NULL; 48183975485SAlex Maftei (amaftei) 48283975485SAlex Maftei (amaftei) *channel = *old_channel; 48383975485SAlex Maftei (amaftei) 48483975485SAlex Maftei (amaftei) channel->napi_dev = NULL; 48583975485SAlex Maftei (amaftei) INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 48683975485SAlex Maftei (amaftei) channel->napi_str.napi_id = 0; 48783975485SAlex Maftei (amaftei) channel->napi_str.state = 0; 48883975485SAlex Maftei (amaftei) memset(&channel->eventq, 0, sizeof(channel->eventq)); 48983975485SAlex Maftei (amaftei) 49083975485SAlex Maftei (amaftei) for (j = 0; j < EFX_TXQ_TYPES; j++) { 49183975485SAlex Maftei (amaftei) tx_queue = &channel->tx_queue[j]; 49283975485SAlex Maftei (amaftei) if (tx_queue->channel) 49383975485SAlex Maftei (amaftei) tx_queue->channel = channel; 49483975485SAlex Maftei (amaftei) tx_queue->buffer = NULL; 49583975485SAlex Maftei (amaftei) memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 49683975485SAlex Maftei (amaftei) } 49783975485SAlex Maftei (amaftei) 49883975485SAlex Maftei (amaftei) rx_queue = &channel->rx_queue; 49983975485SAlex Maftei (amaftei) rx_queue->buffer = NULL; 50083975485SAlex Maftei (amaftei) memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 50183975485SAlex Maftei (amaftei) timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 50283975485SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 50383975485SAlex Maftei (amaftei) INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 50483975485SAlex Maftei (amaftei) #endif 50583975485SAlex Maftei (amaftei) 50683975485SAlex Maftei (amaftei) return channel; 50783975485SAlex Maftei (amaftei) } 50883975485SAlex Maftei (amaftei) 50983975485SAlex Maftei (amaftei) static int efx_probe_channel(struct efx_channel *channel) 51083975485SAlex Maftei (amaftei) { 51183975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 51283975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 51383975485SAlex Maftei (amaftei) int rc; 51483975485SAlex Maftei (amaftei) 51583975485SAlex Maftei (amaftei) netif_dbg(channel->efx, probe, channel->efx->net_dev, 51683975485SAlex Maftei (amaftei) "creating channel %d\n", channel->channel); 51783975485SAlex Maftei (amaftei) 51883975485SAlex Maftei (amaftei) rc = channel->type->pre_probe(channel); 51983975485SAlex Maftei (amaftei) if (rc) 52083975485SAlex Maftei (amaftei) goto fail; 52183975485SAlex Maftei (amaftei) 52283975485SAlex Maftei (amaftei) rc = efx_probe_eventq(channel); 52383975485SAlex Maftei (amaftei) if (rc) 52483975485SAlex Maftei (amaftei) goto fail; 52583975485SAlex Maftei (amaftei) 52683975485SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 52783975485SAlex Maftei (amaftei) rc = efx_probe_tx_queue(tx_queue); 52883975485SAlex Maftei (amaftei) if (rc) 52983975485SAlex Maftei (amaftei) goto fail; 53083975485SAlex Maftei (amaftei) } 53183975485SAlex Maftei (amaftei) 53283975485SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) { 53383975485SAlex Maftei (amaftei) rc = efx_probe_rx_queue(rx_queue); 53483975485SAlex Maftei (amaftei) if (rc) 53583975485SAlex Maftei (amaftei) goto fail; 53683975485SAlex Maftei (amaftei) } 53783975485SAlex Maftei (amaftei) 53883975485SAlex Maftei (amaftei) channel->rx_list = NULL; 53983975485SAlex Maftei (amaftei) 54083975485SAlex Maftei (amaftei) return 0; 54183975485SAlex Maftei (amaftei) 54283975485SAlex Maftei (amaftei) fail: 54383975485SAlex Maftei (amaftei) efx_remove_channel(channel); 54483975485SAlex Maftei (amaftei) return rc; 54583975485SAlex Maftei (amaftei) } 54683975485SAlex Maftei (amaftei) 54783975485SAlex Maftei (amaftei) void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 54883975485SAlex Maftei (amaftei) { 54983975485SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 55083975485SAlex Maftei (amaftei) const char *type; 55183975485SAlex Maftei (amaftei) int number; 55283975485SAlex Maftei (amaftei) 55383975485SAlex Maftei (amaftei) number = channel->channel; 55483975485SAlex Maftei (amaftei) 55583975485SAlex Maftei (amaftei) if (number >= efx->xdp_channel_offset && 55683975485SAlex Maftei (amaftei) !WARN_ON_ONCE(!efx->n_xdp_channels)) { 55783975485SAlex Maftei (amaftei) type = "-xdp"; 55883975485SAlex Maftei (amaftei) number -= efx->xdp_channel_offset; 55983975485SAlex Maftei (amaftei) } else if (efx->tx_channel_offset == 0) { 56083975485SAlex Maftei (amaftei) type = ""; 56183975485SAlex Maftei (amaftei) } else if (number < efx->tx_channel_offset) { 56283975485SAlex Maftei (amaftei) type = "-rx"; 56383975485SAlex Maftei (amaftei) } else { 56483975485SAlex Maftei (amaftei) type = "-tx"; 56583975485SAlex Maftei (amaftei) number -= efx->tx_channel_offset; 56683975485SAlex Maftei (amaftei) } 56783975485SAlex Maftei (amaftei) snprintf(buf, len, "%s%s-%d", efx->name, type, number); 56883975485SAlex Maftei (amaftei) } 56983975485SAlex Maftei (amaftei) 57083975485SAlex Maftei (amaftei) void efx_set_channel_names(struct efx_nic *efx) 57183975485SAlex Maftei (amaftei) { 57283975485SAlex Maftei (amaftei) struct efx_channel *channel; 57383975485SAlex Maftei (amaftei) 57483975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 57583975485SAlex Maftei (amaftei) channel->type->get_name(channel, 57683975485SAlex Maftei (amaftei) efx->msi_context[channel->channel].name, 57783975485SAlex Maftei (amaftei) sizeof(efx->msi_context[0].name)); 57883975485SAlex Maftei (amaftei) } 57983975485SAlex Maftei (amaftei) 58083975485SAlex Maftei (amaftei) int efx_probe_channels(struct efx_nic *efx) 58183975485SAlex Maftei (amaftei) { 58283975485SAlex Maftei (amaftei) struct efx_channel *channel; 58383975485SAlex Maftei (amaftei) int rc; 58483975485SAlex Maftei (amaftei) 58583975485SAlex Maftei (amaftei) /* Restart special buffer allocation */ 58683975485SAlex Maftei (amaftei) efx->next_buffer_table = 0; 58783975485SAlex Maftei (amaftei) 58883975485SAlex Maftei (amaftei) /* Probe channels in reverse, so that any 'extra' channels 58983975485SAlex Maftei (amaftei) * use the start of the buffer table. This allows the traffic 59083975485SAlex Maftei (amaftei) * channels to be resized without moving them or wasting the 59183975485SAlex Maftei (amaftei) * entries before them. 59283975485SAlex Maftei (amaftei) */ 59383975485SAlex Maftei (amaftei) efx_for_each_channel_rev(channel, efx) { 59483975485SAlex Maftei (amaftei) rc = efx_probe_channel(channel); 59583975485SAlex Maftei (amaftei) if (rc) { 59683975485SAlex Maftei (amaftei) netif_err(efx, probe, efx->net_dev, 59783975485SAlex Maftei (amaftei) "failed to create channel %d\n", 59883975485SAlex Maftei (amaftei) channel->channel); 59983975485SAlex Maftei (amaftei) goto fail; 60083975485SAlex Maftei (amaftei) } 60183975485SAlex Maftei (amaftei) } 60283975485SAlex Maftei (amaftei) efx_set_channel_names(efx); 60383975485SAlex Maftei (amaftei) 60483975485SAlex Maftei (amaftei) return 0; 60583975485SAlex Maftei (amaftei) 60683975485SAlex Maftei (amaftei) fail: 60783975485SAlex Maftei (amaftei) efx_remove_channels(efx); 60883975485SAlex Maftei (amaftei) return rc; 60983975485SAlex Maftei (amaftei) } 61083975485SAlex Maftei (amaftei) 61183975485SAlex Maftei (amaftei) void efx_remove_channel(struct efx_channel *channel) 61283975485SAlex Maftei (amaftei) { 61383975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 61483975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 61583975485SAlex Maftei (amaftei) 61683975485SAlex Maftei (amaftei) netif_dbg(channel->efx, drv, channel->efx->net_dev, 61783975485SAlex Maftei (amaftei) "destroy chan %d\n", channel->channel); 61883975485SAlex Maftei (amaftei) 61983975485SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 62083975485SAlex Maftei (amaftei) efx_remove_rx_queue(rx_queue); 62183975485SAlex Maftei (amaftei) efx_for_each_possible_channel_tx_queue(tx_queue, channel) 62283975485SAlex Maftei (amaftei) efx_remove_tx_queue(tx_queue); 62383975485SAlex Maftei (amaftei) efx_remove_eventq(channel); 62483975485SAlex Maftei (amaftei) channel->type->post_remove(channel); 62583975485SAlex Maftei (amaftei) } 62683975485SAlex Maftei (amaftei) 62783975485SAlex Maftei (amaftei) void efx_remove_channels(struct efx_nic *efx) 62883975485SAlex Maftei (amaftei) { 62983975485SAlex Maftei (amaftei) struct efx_channel *channel; 63083975485SAlex Maftei (amaftei) 63183975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 63283975485SAlex Maftei (amaftei) efx_remove_channel(channel); 63383975485SAlex Maftei (amaftei) 63483975485SAlex Maftei (amaftei) kfree(efx->xdp_tx_queues); 63583975485SAlex Maftei (amaftei) } 63683975485SAlex Maftei (amaftei) 63783975485SAlex Maftei (amaftei) int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 63883975485SAlex Maftei (amaftei) { 63983975485SAlex Maftei (amaftei) struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 64083975485SAlex Maftei (amaftei) unsigned int i, next_buffer_table = 0; 64183975485SAlex Maftei (amaftei) u32 old_rxq_entries, old_txq_entries; 64283975485SAlex Maftei (amaftei) int rc, rc2; 64383975485SAlex Maftei (amaftei) 64483975485SAlex Maftei (amaftei) rc = efx_check_disabled(efx); 64583975485SAlex Maftei (amaftei) if (rc) 64683975485SAlex Maftei (amaftei) return rc; 64783975485SAlex Maftei (amaftei) 64883975485SAlex Maftei (amaftei) /* Not all channels should be reallocated. We must avoid 64983975485SAlex Maftei (amaftei) * reallocating their buffer table entries. 65083975485SAlex Maftei (amaftei) */ 65183975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 65283975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 65383975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 65483975485SAlex Maftei (amaftei) 65583975485SAlex Maftei (amaftei) if (channel->type->copy) 65683975485SAlex Maftei (amaftei) continue; 65783975485SAlex Maftei (amaftei) next_buffer_table = max(next_buffer_table, 65883975485SAlex Maftei (amaftei) channel->eventq.index + 65983975485SAlex Maftei (amaftei) channel->eventq.entries); 66083975485SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 66183975485SAlex Maftei (amaftei) next_buffer_table = max(next_buffer_table, 66283975485SAlex Maftei (amaftei) rx_queue->rxd.index + 66383975485SAlex Maftei (amaftei) rx_queue->rxd.entries); 66483975485SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) 66583975485SAlex Maftei (amaftei) next_buffer_table = max(next_buffer_table, 66683975485SAlex Maftei (amaftei) tx_queue->txd.index + 66783975485SAlex Maftei (amaftei) tx_queue->txd.entries); 66883975485SAlex Maftei (amaftei) } 66983975485SAlex Maftei (amaftei) 67083975485SAlex Maftei (amaftei) efx_device_detach_sync(efx); 67183975485SAlex Maftei (amaftei) efx_stop_all(efx); 67283975485SAlex Maftei (amaftei) efx_soft_disable_interrupts(efx); 67383975485SAlex Maftei (amaftei) 67483975485SAlex Maftei (amaftei) /* Clone channels (where possible) */ 67583975485SAlex Maftei (amaftei) memset(other_channel, 0, sizeof(other_channel)); 67683975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 67783975485SAlex Maftei (amaftei) channel = efx->channel[i]; 67883975485SAlex Maftei (amaftei) if (channel->type->copy) 67983975485SAlex Maftei (amaftei) channel = channel->type->copy(channel); 68083975485SAlex Maftei (amaftei) if (!channel) { 68183975485SAlex Maftei (amaftei) rc = -ENOMEM; 68283975485SAlex Maftei (amaftei) goto out; 68383975485SAlex Maftei (amaftei) } 68483975485SAlex Maftei (amaftei) other_channel[i] = channel; 68583975485SAlex Maftei (amaftei) } 68683975485SAlex Maftei (amaftei) 68783975485SAlex Maftei (amaftei) /* Swap entry counts and channel pointers */ 68883975485SAlex Maftei (amaftei) old_rxq_entries = efx->rxq_entries; 68983975485SAlex Maftei (amaftei) old_txq_entries = efx->txq_entries; 69083975485SAlex Maftei (amaftei) efx->rxq_entries = rxq_entries; 69183975485SAlex Maftei (amaftei) efx->txq_entries = txq_entries; 69283975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 69383975485SAlex Maftei (amaftei) channel = efx->channel[i]; 69483975485SAlex Maftei (amaftei) efx->channel[i] = other_channel[i]; 69583975485SAlex Maftei (amaftei) other_channel[i] = channel; 69683975485SAlex Maftei (amaftei) } 69783975485SAlex Maftei (amaftei) 69883975485SAlex Maftei (amaftei) /* Restart buffer table allocation */ 69983975485SAlex Maftei (amaftei) efx->next_buffer_table = next_buffer_table; 70083975485SAlex Maftei (amaftei) 70183975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 70283975485SAlex Maftei (amaftei) channel = efx->channel[i]; 70383975485SAlex Maftei (amaftei) if (!channel->type->copy) 70483975485SAlex Maftei (amaftei) continue; 70583975485SAlex Maftei (amaftei) rc = efx_probe_channel(channel); 70683975485SAlex Maftei (amaftei) if (rc) 70783975485SAlex Maftei (amaftei) goto rollback; 70883975485SAlex Maftei (amaftei) efx_init_napi_channel(efx->channel[i]); 70983975485SAlex Maftei (amaftei) } 71083975485SAlex Maftei (amaftei) 71183975485SAlex Maftei (amaftei) out: 71283975485SAlex Maftei (amaftei) /* Destroy unused channel structures */ 71383975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 71483975485SAlex Maftei (amaftei) channel = other_channel[i]; 71583975485SAlex Maftei (amaftei) if (channel && channel->type->copy) { 71683975485SAlex Maftei (amaftei) efx_fini_napi_channel(channel); 71783975485SAlex Maftei (amaftei) efx_remove_channel(channel); 71883975485SAlex Maftei (amaftei) kfree(channel); 71983975485SAlex Maftei (amaftei) } 72083975485SAlex Maftei (amaftei) } 72183975485SAlex Maftei (amaftei) 72283975485SAlex Maftei (amaftei) rc2 = efx_soft_enable_interrupts(efx); 72383975485SAlex Maftei (amaftei) if (rc2) { 72483975485SAlex Maftei (amaftei) rc = rc ? rc : rc2; 72583975485SAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 72683975485SAlex Maftei (amaftei) "unable to restart interrupts on channel reallocation\n"); 72783975485SAlex Maftei (amaftei) efx_schedule_reset(efx, RESET_TYPE_DISABLE); 72883975485SAlex Maftei (amaftei) } else { 72983975485SAlex Maftei (amaftei) efx_start_all(efx); 73083975485SAlex Maftei (amaftei) efx_device_attach_if_not_resetting(efx); 73183975485SAlex Maftei (amaftei) } 73283975485SAlex Maftei (amaftei) return rc; 73383975485SAlex Maftei (amaftei) 73483975485SAlex Maftei (amaftei) rollback: 73583975485SAlex Maftei (amaftei) /* Swap back */ 73683975485SAlex Maftei (amaftei) efx->rxq_entries = old_rxq_entries; 73783975485SAlex Maftei (amaftei) efx->txq_entries = old_txq_entries; 73883975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 73983975485SAlex Maftei (amaftei) channel = efx->channel[i]; 74083975485SAlex Maftei (amaftei) efx->channel[i] = other_channel[i]; 74183975485SAlex Maftei (amaftei) other_channel[i] = channel; 74283975485SAlex Maftei (amaftei) } 74383975485SAlex Maftei (amaftei) goto out; 74483975485SAlex Maftei (amaftei) } 74583975485SAlex Maftei (amaftei) 74683975485SAlex Maftei (amaftei) int efx_set_channels(struct efx_nic *efx) 74783975485SAlex Maftei (amaftei) { 74883975485SAlex Maftei (amaftei) struct efx_channel *channel; 74983975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 75083975485SAlex Maftei (amaftei) int xdp_queue_number; 75183975485SAlex Maftei (amaftei) 75283975485SAlex Maftei (amaftei) efx->tx_channel_offset = 75383975485SAlex Maftei (amaftei) efx_separate_tx_channels ? 75483975485SAlex Maftei (amaftei) efx->n_channels - efx->n_tx_channels : 0; 75583975485SAlex Maftei (amaftei) 75683975485SAlex Maftei (amaftei) if (efx->xdp_tx_queue_count) { 75783975485SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(efx->xdp_tx_queues); 75883975485SAlex Maftei (amaftei) 75983975485SAlex Maftei (amaftei) /* Allocate array for XDP TX queue lookup. */ 76083975485SAlex Maftei (amaftei) efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count, 76183975485SAlex Maftei (amaftei) sizeof(*efx->xdp_tx_queues), 76283975485SAlex Maftei (amaftei) GFP_KERNEL); 76383975485SAlex Maftei (amaftei) if (!efx->xdp_tx_queues) 76483975485SAlex Maftei (amaftei) return -ENOMEM; 76583975485SAlex Maftei (amaftei) } 76683975485SAlex Maftei (amaftei) 76783975485SAlex Maftei (amaftei) /* We need to mark which channels really have RX and TX 76883975485SAlex Maftei (amaftei) * queues, and adjust the TX queue numbers if we have separate 76983975485SAlex Maftei (amaftei) * RX-only and TX-only channels. 77083975485SAlex Maftei (amaftei) */ 77183975485SAlex Maftei (amaftei) xdp_queue_number = 0; 77283975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 77383975485SAlex Maftei (amaftei) if (channel->channel < efx->n_rx_channels) 77483975485SAlex Maftei (amaftei) channel->rx_queue.core_index = channel->channel; 77583975485SAlex Maftei (amaftei) else 77683975485SAlex Maftei (amaftei) channel->rx_queue.core_index = -1; 77783975485SAlex Maftei (amaftei) 77883975485SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 77983975485SAlex Maftei (amaftei) tx_queue->queue -= (efx->tx_channel_offset * 78083975485SAlex Maftei (amaftei) EFX_TXQ_TYPES); 78183975485SAlex Maftei (amaftei) 78283975485SAlex Maftei (amaftei) if (efx_channel_is_xdp_tx(channel) && 78383975485SAlex Maftei (amaftei) xdp_queue_number < efx->xdp_tx_queue_count) { 78483975485SAlex Maftei (amaftei) efx->xdp_tx_queues[xdp_queue_number] = tx_queue; 78583975485SAlex Maftei (amaftei) xdp_queue_number++; 78683975485SAlex Maftei (amaftei) } 78783975485SAlex Maftei (amaftei) } 78883975485SAlex Maftei (amaftei) } 78983975485SAlex Maftei (amaftei) return 0; 79083975485SAlex Maftei (amaftei) } 79183975485SAlex Maftei (amaftei) 79283975485SAlex Maftei (amaftei) bool efx_default_channel_want_txqs(struct efx_channel *channel) 79383975485SAlex Maftei (amaftei) { 79483975485SAlex Maftei (amaftei) return channel->channel - channel->efx->tx_channel_offset < 79583975485SAlex Maftei (amaftei) channel->efx->n_tx_channels; 79683975485SAlex Maftei (amaftei) } 79783975485SAlex Maftei (amaftei) 798e20ba5b1SAlex Maftei (amaftei) /************* 799e20ba5b1SAlex Maftei (amaftei) * START/STOP 800e20ba5b1SAlex Maftei (amaftei) *************/ 801e20ba5b1SAlex Maftei (amaftei) 802e20ba5b1SAlex Maftei (amaftei) int efx_soft_enable_interrupts(struct efx_nic *efx) 803e20ba5b1SAlex Maftei (amaftei) { 804e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel, *end_channel; 805e20ba5b1SAlex Maftei (amaftei) int rc; 806e20ba5b1SAlex Maftei (amaftei) 807e20ba5b1SAlex Maftei (amaftei) BUG_ON(efx->state == STATE_DISABLED); 808e20ba5b1SAlex Maftei (amaftei) 809e20ba5b1SAlex Maftei (amaftei) efx->irq_soft_enabled = true; 810e20ba5b1SAlex Maftei (amaftei) smp_wmb(); 811e20ba5b1SAlex Maftei (amaftei) 812e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 813e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) { 814e20ba5b1SAlex Maftei (amaftei) rc = efx_init_eventq(channel); 815e20ba5b1SAlex Maftei (amaftei) if (rc) 816e20ba5b1SAlex Maftei (amaftei) goto fail; 817e20ba5b1SAlex Maftei (amaftei) } 818e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 819e20ba5b1SAlex Maftei (amaftei) } 820e20ba5b1SAlex Maftei (amaftei) 821e20ba5b1SAlex Maftei (amaftei) efx_mcdi_mode_event(efx); 822e20ba5b1SAlex Maftei (amaftei) 823e20ba5b1SAlex Maftei (amaftei) return 0; 824e20ba5b1SAlex Maftei (amaftei) fail: 825e20ba5b1SAlex Maftei (amaftei) end_channel = channel; 826e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 827e20ba5b1SAlex Maftei (amaftei) if (channel == end_channel) 828e20ba5b1SAlex Maftei (amaftei) break; 829e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 830e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) 831e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 832e20ba5b1SAlex Maftei (amaftei) } 833e20ba5b1SAlex Maftei (amaftei) 834e20ba5b1SAlex Maftei (amaftei) return rc; 835e20ba5b1SAlex Maftei (amaftei) } 836e20ba5b1SAlex Maftei (amaftei) 837e20ba5b1SAlex Maftei (amaftei) void efx_soft_disable_interrupts(struct efx_nic *efx) 838e20ba5b1SAlex Maftei (amaftei) { 839e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 840e20ba5b1SAlex Maftei (amaftei) 841e20ba5b1SAlex Maftei (amaftei) if (efx->state == STATE_DISABLED) 842e20ba5b1SAlex Maftei (amaftei) return; 843e20ba5b1SAlex Maftei (amaftei) 844e20ba5b1SAlex Maftei (amaftei) efx_mcdi_mode_poll(efx); 845e20ba5b1SAlex Maftei (amaftei) 846e20ba5b1SAlex Maftei (amaftei) efx->irq_soft_enabled = false; 847e20ba5b1SAlex Maftei (amaftei) smp_wmb(); 848e20ba5b1SAlex Maftei (amaftei) 849e20ba5b1SAlex Maftei (amaftei) if (efx->legacy_irq) 850e20ba5b1SAlex Maftei (amaftei) synchronize_irq(efx->legacy_irq); 851e20ba5b1SAlex Maftei (amaftei) 852e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 853e20ba5b1SAlex Maftei (amaftei) if (channel->irq) 854e20ba5b1SAlex Maftei (amaftei) synchronize_irq(channel->irq); 855e20ba5b1SAlex Maftei (amaftei) 856e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 857e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) 858e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 859e20ba5b1SAlex Maftei (amaftei) } 860e20ba5b1SAlex Maftei (amaftei) 861e20ba5b1SAlex Maftei (amaftei) /* Flush the asynchronous MCDI request queue */ 862e20ba5b1SAlex Maftei (amaftei) efx_mcdi_flush_async(efx); 863e20ba5b1SAlex Maftei (amaftei) } 864e20ba5b1SAlex Maftei (amaftei) 865e20ba5b1SAlex Maftei (amaftei) int efx_enable_interrupts(struct efx_nic *efx) 866e20ba5b1SAlex Maftei (amaftei) { 867e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel, *end_channel; 868e20ba5b1SAlex Maftei (amaftei) int rc; 869e20ba5b1SAlex Maftei (amaftei) 870e20ba5b1SAlex Maftei (amaftei) /* TODO: Is this really a bug? */ 871e20ba5b1SAlex Maftei (amaftei) BUG_ON(efx->state == STATE_DISABLED); 872e20ba5b1SAlex Maftei (amaftei) 873e20ba5b1SAlex Maftei (amaftei) if (efx->eeh_disabled_legacy_irq) { 874e20ba5b1SAlex Maftei (amaftei) enable_irq(efx->legacy_irq); 875e20ba5b1SAlex Maftei (amaftei) efx->eeh_disabled_legacy_irq = false; 876e20ba5b1SAlex Maftei (amaftei) } 877e20ba5b1SAlex Maftei (amaftei) 878e20ba5b1SAlex Maftei (amaftei) efx->type->irq_enable_master(efx); 879e20ba5b1SAlex Maftei (amaftei) 880e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 881e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) { 882e20ba5b1SAlex Maftei (amaftei) rc = efx_init_eventq(channel); 883e20ba5b1SAlex Maftei (amaftei) if (rc) 884e20ba5b1SAlex Maftei (amaftei) goto fail; 885e20ba5b1SAlex Maftei (amaftei) } 886e20ba5b1SAlex Maftei (amaftei) } 887e20ba5b1SAlex Maftei (amaftei) 888e20ba5b1SAlex Maftei (amaftei) rc = efx_soft_enable_interrupts(efx); 889e20ba5b1SAlex Maftei (amaftei) if (rc) 890e20ba5b1SAlex Maftei (amaftei) goto fail; 891e20ba5b1SAlex Maftei (amaftei) 892e20ba5b1SAlex Maftei (amaftei) return 0; 893e20ba5b1SAlex Maftei (amaftei) 894e20ba5b1SAlex Maftei (amaftei) fail: 895e20ba5b1SAlex Maftei (amaftei) end_channel = channel; 896e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 897e20ba5b1SAlex Maftei (amaftei) if (channel == end_channel) 898e20ba5b1SAlex Maftei (amaftei) break; 899e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) 900e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 901e20ba5b1SAlex Maftei (amaftei) } 902e20ba5b1SAlex Maftei (amaftei) 903e20ba5b1SAlex Maftei (amaftei) efx->type->irq_disable_non_ev(efx); 904e20ba5b1SAlex Maftei (amaftei) 905e20ba5b1SAlex Maftei (amaftei) return rc; 906e20ba5b1SAlex Maftei (amaftei) } 907e20ba5b1SAlex Maftei (amaftei) 908e20ba5b1SAlex Maftei (amaftei) void efx_disable_interrupts(struct efx_nic *efx) 909e20ba5b1SAlex Maftei (amaftei) { 910e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 911e20ba5b1SAlex Maftei (amaftei) 912e20ba5b1SAlex Maftei (amaftei) efx_soft_disable_interrupts(efx); 913e20ba5b1SAlex Maftei (amaftei) 914e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 915e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) 916e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 917e20ba5b1SAlex Maftei (amaftei) } 918e20ba5b1SAlex Maftei (amaftei) 919e20ba5b1SAlex Maftei (amaftei) efx->type->irq_disable_non_ev(efx); 920e20ba5b1SAlex Maftei (amaftei) } 921e20ba5b1SAlex Maftei (amaftei) 922e20ba5b1SAlex Maftei (amaftei) void efx_start_channels(struct efx_nic *efx) 923e20ba5b1SAlex Maftei (amaftei) { 924e20ba5b1SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 925e20ba5b1SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 926e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 927e20ba5b1SAlex Maftei (amaftei) 928e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 929e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 930e20ba5b1SAlex Maftei (amaftei) efx_init_tx_queue(tx_queue); 931e20ba5b1SAlex Maftei (amaftei) atomic_inc(&efx->active_queues); 932e20ba5b1SAlex Maftei (amaftei) } 933e20ba5b1SAlex Maftei (amaftei) 934e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) { 935e20ba5b1SAlex Maftei (amaftei) efx_init_rx_queue(rx_queue); 936e20ba5b1SAlex Maftei (amaftei) atomic_inc(&efx->active_queues); 937e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 938e20ba5b1SAlex Maftei (amaftei) efx_fast_push_rx_descriptors(rx_queue, false); 939e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 940e20ba5b1SAlex Maftei (amaftei) } 941e20ba5b1SAlex Maftei (amaftei) 942e20ba5b1SAlex Maftei (amaftei) WARN_ON(channel->rx_pkt_n_frags); 943e20ba5b1SAlex Maftei (amaftei) } 944e20ba5b1SAlex Maftei (amaftei) } 945e20ba5b1SAlex Maftei (amaftei) 946e20ba5b1SAlex Maftei (amaftei) void efx_stop_channels(struct efx_nic *efx) 947e20ba5b1SAlex Maftei (amaftei) { 948e20ba5b1SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 949e20ba5b1SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 950e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 951e20ba5b1SAlex Maftei (amaftei) int rc; 952e20ba5b1SAlex Maftei (amaftei) 953e20ba5b1SAlex Maftei (amaftei) /* Stop RX refill */ 954e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 955e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 956e20ba5b1SAlex Maftei (amaftei) rx_queue->refill_enabled = false; 957e20ba5b1SAlex Maftei (amaftei) } 958e20ba5b1SAlex Maftei (amaftei) 959e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 960e20ba5b1SAlex Maftei (amaftei) /* RX packet processing is pipelined, so wait for the 961e20ba5b1SAlex Maftei (amaftei) * NAPI handler to complete. At least event queue 0 962e20ba5b1SAlex Maftei (amaftei) * might be kept active by non-data events, so don't 963e20ba5b1SAlex Maftei (amaftei) * use napi_synchronize() but actually disable NAPI 964e20ba5b1SAlex Maftei (amaftei) * temporarily. 965e20ba5b1SAlex Maftei (amaftei) */ 966e20ba5b1SAlex Maftei (amaftei) if (efx_channel_has_rx_queue(channel)) { 967e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 968e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 969e20ba5b1SAlex Maftei (amaftei) } 970e20ba5b1SAlex Maftei (amaftei) } 971e20ba5b1SAlex Maftei (amaftei) 972e20ba5b1SAlex Maftei (amaftei) rc = efx->type->fini_dmaq(efx); 973e20ba5b1SAlex Maftei (amaftei) if (rc) { 974e20ba5b1SAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 975e20ba5b1SAlex Maftei (amaftei) } else { 976e20ba5b1SAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 977e20ba5b1SAlex Maftei (amaftei) "successfully flushed all queues\n"); 978e20ba5b1SAlex Maftei (amaftei) } 979e20ba5b1SAlex Maftei (amaftei) 980e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 981e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 982e20ba5b1SAlex Maftei (amaftei) efx_fini_rx_queue(rx_queue); 983e20ba5b1SAlex Maftei (amaftei) efx_for_each_possible_channel_tx_queue(tx_queue, channel) 984e20ba5b1SAlex Maftei (amaftei) efx_fini_tx_queue(tx_queue); 985e20ba5b1SAlex Maftei (amaftei) } 986e20ba5b1SAlex Maftei (amaftei) } 987e20ba5b1SAlex Maftei (amaftei) 988768fd266SAlex Maftei (amaftei) /************************************************************************** 989768fd266SAlex Maftei (amaftei) * 990768fd266SAlex Maftei (amaftei) * NAPI interface 991768fd266SAlex Maftei (amaftei) * 992768fd266SAlex Maftei (amaftei) *************************************************************************/ 993768fd266SAlex Maftei (amaftei) 994768fd266SAlex Maftei (amaftei) /* Process channel's event queue 995768fd266SAlex Maftei (amaftei) * 996768fd266SAlex Maftei (amaftei) * This function is responsible for processing the event queue of a 997768fd266SAlex Maftei (amaftei) * single channel. The caller must guarantee that this function will 998768fd266SAlex Maftei (amaftei) * never be concurrently called more than once on the same channel, 999768fd266SAlex Maftei (amaftei) * though different channels may be being processed concurrently. 1000768fd266SAlex Maftei (amaftei) */ 1001768fd266SAlex Maftei (amaftei) static int efx_process_channel(struct efx_channel *channel, int budget) 1002768fd266SAlex Maftei (amaftei) { 1003768fd266SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 1004768fd266SAlex Maftei (amaftei) struct list_head rx_list; 1005768fd266SAlex Maftei (amaftei) int spent; 1006768fd266SAlex Maftei (amaftei) 1007768fd266SAlex Maftei (amaftei) if (unlikely(!channel->enabled)) 1008768fd266SAlex Maftei (amaftei) return 0; 1009768fd266SAlex Maftei (amaftei) 1010768fd266SAlex Maftei (amaftei) /* Prepare the batch receive list */ 1011768fd266SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(channel->rx_list != NULL); 1012768fd266SAlex Maftei (amaftei) INIT_LIST_HEAD(&rx_list); 1013768fd266SAlex Maftei (amaftei) channel->rx_list = &rx_list; 1014768fd266SAlex Maftei (amaftei) 1015768fd266SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 1016768fd266SAlex Maftei (amaftei) tx_queue->pkts_compl = 0; 1017768fd266SAlex Maftei (amaftei) tx_queue->bytes_compl = 0; 1018768fd266SAlex Maftei (amaftei) } 1019768fd266SAlex Maftei (amaftei) 1020768fd266SAlex Maftei (amaftei) spent = efx_nic_process_eventq(channel, budget); 1021768fd266SAlex Maftei (amaftei) if (spent && efx_channel_has_rx_queue(channel)) { 1022768fd266SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = 1023768fd266SAlex Maftei (amaftei) efx_channel_get_rx_queue(channel); 1024768fd266SAlex Maftei (amaftei) 1025768fd266SAlex Maftei (amaftei) efx_rx_flush_packet(channel); 1026768fd266SAlex Maftei (amaftei) efx_fast_push_rx_descriptors(rx_queue, true); 1027768fd266SAlex Maftei (amaftei) } 1028768fd266SAlex Maftei (amaftei) 1029768fd266SAlex Maftei (amaftei) /* Update BQL */ 1030768fd266SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 1031768fd266SAlex Maftei (amaftei) if (tx_queue->bytes_compl) { 1032768fd266SAlex Maftei (amaftei) netdev_tx_completed_queue(tx_queue->core_txq, 1033768fd266SAlex Maftei (amaftei) tx_queue->pkts_compl, 1034768fd266SAlex Maftei (amaftei) tx_queue->bytes_compl); 1035768fd266SAlex Maftei (amaftei) } 1036768fd266SAlex Maftei (amaftei) } 1037768fd266SAlex Maftei (amaftei) 1038768fd266SAlex Maftei (amaftei) /* Receive any packets we queued up */ 1039768fd266SAlex Maftei (amaftei) netif_receive_skb_list(channel->rx_list); 1040768fd266SAlex Maftei (amaftei) channel->rx_list = NULL; 1041768fd266SAlex Maftei (amaftei) 1042768fd266SAlex Maftei (amaftei) return spent; 1043768fd266SAlex Maftei (amaftei) } 1044768fd266SAlex Maftei (amaftei) 1045768fd266SAlex Maftei (amaftei) static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 1046768fd266SAlex Maftei (amaftei) { 1047768fd266SAlex Maftei (amaftei) int step = efx->irq_mod_step_us; 1048768fd266SAlex Maftei (amaftei) 1049768fd266SAlex Maftei (amaftei) if (channel->irq_mod_score < irq_adapt_low_thresh) { 1050768fd266SAlex Maftei (amaftei) if (channel->irq_moderation_us > step) { 1051768fd266SAlex Maftei (amaftei) channel->irq_moderation_us -= step; 1052768fd266SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 1053768fd266SAlex Maftei (amaftei) } 1054768fd266SAlex Maftei (amaftei) } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 1055768fd266SAlex Maftei (amaftei) if (channel->irq_moderation_us < 1056768fd266SAlex Maftei (amaftei) efx->irq_rx_moderation_us) { 1057768fd266SAlex Maftei (amaftei) channel->irq_moderation_us += step; 1058768fd266SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 1059768fd266SAlex Maftei (amaftei) } 1060768fd266SAlex Maftei (amaftei) } 1061768fd266SAlex Maftei (amaftei) 1062768fd266SAlex Maftei (amaftei) channel->irq_count = 0; 1063768fd266SAlex Maftei (amaftei) channel->irq_mod_score = 0; 1064768fd266SAlex Maftei (amaftei) } 1065768fd266SAlex Maftei (amaftei) 1066768fd266SAlex Maftei (amaftei) /* NAPI poll handler 1067768fd266SAlex Maftei (amaftei) * 1068768fd266SAlex Maftei (amaftei) * NAPI guarantees serialisation of polls of the same device, which 1069768fd266SAlex Maftei (amaftei) * provides the guarantee required by efx_process_channel(). 1070768fd266SAlex Maftei (amaftei) */ 1071768fd266SAlex Maftei (amaftei) static int efx_poll(struct napi_struct *napi, int budget) 1072768fd266SAlex Maftei (amaftei) { 1073768fd266SAlex Maftei (amaftei) struct efx_channel *channel = 1074768fd266SAlex Maftei (amaftei) container_of(napi, struct efx_channel, napi_str); 1075768fd266SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 1076768fd266SAlex Maftei (amaftei) int spent; 1077768fd266SAlex Maftei (amaftei) 1078768fd266SAlex Maftei (amaftei) netif_vdbg(efx, intr, efx->net_dev, 1079768fd266SAlex Maftei (amaftei) "channel %d NAPI poll executing on CPU %d\n", 1080768fd266SAlex Maftei (amaftei) channel->channel, raw_smp_processor_id()); 1081768fd266SAlex Maftei (amaftei) 1082768fd266SAlex Maftei (amaftei) spent = efx_process_channel(channel, budget); 1083768fd266SAlex Maftei (amaftei) 1084768fd266SAlex Maftei (amaftei) xdp_do_flush_map(); 1085768fd266SAlex Maftei (amaftei) 1086768fd266SAlex Maftei (amaftei) if (spent < budget) { 1087768fd266SAlex Maftei (amaftei) if (efx_channel_has_rx_queue(channel) && 1088768fd266SAlex Maftei (amaftei) efx->irq_rx_adaptive && 1089768fd266SAlex Maftei (amaftei) unlikely(++channel->irq_count == 1000)) { 1090768fd266SAlex Maftei (amaftei) efx_update_irq_mod(efx, channel); 1091768fd266SAlex Maftei (amaftei) } 1092768fd266SAlex Maftei (amaftei) 1093768fd266SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 1094768fd266SAlex Maftei (amaftei) /* Perhaps expire some ARFS filters */ 1095768fd266SAlex Maftei (amaftei) mod_delayed_work(system_wq, &channel->filter_work, 0); 1096768fd266SAlex Maftei (amaftei) #endif 1097768fd266SAlex Maftei (amaftei) 1098768fd266SAlex Maftei (amaftei) /* There is no race here; although napi_disable() will 1099768fd266SAlex Maftei (amaftei) * only wait for napi_complete(), this isn't a problem 1100768fd266SAlex Maftei (amaftei) * since efx_nic_eventq_read_ack() will have no effect if 1101768fd266SAlex Maftei (amaftei) * interrupts have already been disabled. 1102768fd266SAlex Maftei (amaftei) */ 1103768fd266SAlex Maftei (amaftei) if (napi_complete_done(napi, spent)) 1104768fd266SAlex Maftei (amaftei) efx_nic_eventq_read_ack(channel); 1105768fd266SAlex Maftei (amaftei) } 1106768fd266SAlex Maftei (amaftei) 1107768fd266SAlex Maftei (amaftei) return spent; 1108768fd266SAlex Maftei (amaftei) } 1109768fd266SAlex Maftei (amaftei) 1110768fd266SAlex Maftei (amaftei) void efx_init_napi_channel(struct efx_channel *channel) 1111768fd266SAlex Maftei (amaftei) { 1112768fd266SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 1113768fd266SAlex Maftei (amaftei) 1114768fd266SAlex Maftei (amaftei) channel->napi_dev = efx->net_dev; 1115768fd266SAlex Maftei (amaftei) netif_napi_add(channel->napi_dev, &channel->napi_str, 1116768fd266SAlex Maftei (amaftei) efx_poll, napi_weight); 1117768fd266SAlex Maftei (amaftei) } 1118768fd266SAlex Maftei (amaftei) 1119768fd266SAlex Maftei (amaftei) void efx_init_napi(struct efx_nic *efx) 1120768fd266SAlex Maftei (amaftei) { 1121768fd266SAlex Maftei (amaftei) struct efx_channel *channel; 1122768fd266SAlex Maftei (amaftei) 1123768fd266SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 1124768fd266SAlex Maftei (amaftei) efx_init_napi_channel(channel); 1125768fd266SAlex Maftei (amaftei) } 1126768fd266SAlex Maftei (amaftei) 1127768fd266SAlex Maftei (amaftei) void efx_fini_napi_channel(struct efx_channel *channel) 1128768fd266SAlex Maftei (amaftei) { 1129768fd266SAlex Maftei (amaftei) if (channel->napi_dev) 1130768fd266SAlex Maftei (amaftei) netif_napi_del(&channel->napi_str); 1131768fd266SAlex Maftei (amaftei) 1132768fd266SAlex Maftei (amaftei) channel->napi_dev = NULL; 1133768fd266SAlex Maftei (amaftei) } 1134768fd266SAlex Maftei (amaftei) 1135768fd266SAlex Maftei (amaftei) void efx_fini_napi(struct efx_nic *efx) 1136768fd266SAlex Maftei (amaftei) { 1137768fd266SAlex Maftei (amaftei) struct efx_channel *channel; 1138768fd266SAlex Maftei (amaftei) 1139768fd266SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 1140768fd266SAlex Maftei (amaftei) efx_fini_napi_channel(channel); 1141768fd266SAlex Maftei (amaftei) } 1142