1768fd266SAlex Maftei (amaftei) // SPDX-License-Identifier: GPL-2.0-only 2768fd266SAlex Maftei (amaftei) /**************************************************************************** 3768fd266SAlex Maftei (amaftei) * Driver for Solarflare network controllers and boards 4768fd266SAlex Maftei (amaftei) * Copyright 2018 Solarflare Communications Inc. 5768fd266SAlex Maftei (amaftei) * 6768fd266SAlex Maftei (amaftei) * This program is free software; you can redistribute it and/or modify it 7768fd266SAlex Maftei (amaftei) * under the terms of the GNU General Public License version 2 as published 8768fd266SAlex Maftei (amaftei) * by the Free Software Foundation, incorporated herein by reference. 9768fd266SAlex Maftei (amaftei) */ 10768fd266SAlex Maftei (amaftei) 11768fd266SAlex Maftei (amaftei) #include "net_driver.h" 12768fd266SAlex Maftei (amaftei) #include <linux/module.h> 13b6459415SJakub Kicinski #include <linux/filter.h> 14768fd266SAlex Maftei (amaftei) #include "efx_channels.h" 15768fd266SAlex Maftei (amaftei) #include "efx.h" 16768fd266SAlex Maftei (amaftei) #include "efx_common.h" 17768fd266SAlex Maftei (amaftei) #include "tx_common.h" 18768fd266SAlex Maftei (amaftei) #include "rx_common.h" 19768fd266SAlex Maftei (amaftei) #include "nic.h" 20768fd266SAlex Maftei (amaftei) #include "sriov.h" 21e26ca4b5SIvan Babrou #include "workarounds.h" 22768fd266SAlex Maftei (amaftei) 2383975485SAlex Maftei (amaftei) /* This is the first interrupt mode to try out of: 2483975485SAlex Maftei (amaftei) * 0 => MSI-X 2583975485SAlex Maftei (amaftei) * 1 => MSI 2683975485SAlex Maftei (amaftei) * 2 => legacy 2783975485SAlex Maftei (amaftei) */ 28e4ff3232SEdward Cree unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX; 2983975485SAlex Maftei (amaftei) 3037c45a4eSAlex Maftei (amaftei) /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 3137c45a4eSAlex Maftei (amaftei) * i.e. the number of CPUs among which we may distribute simultaneous 3237c45a4eSAlex Maftei (amaftei) * interrupt handling. 3337c45a4eSAlex Maftei (amaftei) * 3437c45a4eSAlex Maftei (amaftei) * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 3537c45a4eSAlex Maftei (amaftei) * The default (0) means to assign an interrupt to each core. 3637c45a4eSAlex Maftei (amaftei) */ 3767e6398eSEdward Cree unsigned int rss_cpus; 3837c45a4eSAlex Maftei (amaftei) 39768fd266SAlex Maftei (amaftei) static unsigned int irq_adapt_low_thresh = 8000; 40768fd266SAlex Maftei (amaftei) module_param(irq_adapt_low_thresh, uint, 0644); 41768fd266SAlex Maftei (amaftei) MODULE_PARM_DESC(irq_adapt_low_thresh, 42768fd266SAlex Maftei (amaftei) "Threshold score for reducing IRQ moderation"); 43768fd266SAlex Maftei (amaftei) 44768fd266SAlex Maftei (amaftei) static unsigned int irq_adapt_high_thresh = 16000; 45768fd266SAlex Maftei (amaftei) module_param(irq_adapt_high_thresh, uint, 0644); 46768fd266SAlex Maftei (amaftei) MODULE_PARM_DESC(irq_adapt_high_thresh, 47768fd266SAlex Maftei (amaftei) "Threshold score for increasing IRQ moderation"); 48768fd266SAlex Maftei (amaftei) 4954fccfddSMartin Habets static const struct efx_channel_type efx_default_channel_type; 5083975485SAlex Maftei (amaftei) 5137c45a4eSAlex Maftei (amaftei) /************* 5237c45a4eSAlex Maftei (amaftei) * INTERRUPTS 5337c45a4eSAlex Maftei (amaftei) *************/ 5437c45a4eSAlex Maftei (amaftei) 55c265b569SÍñigo Huguet static unsigned int count_online_cores(struct efx_nic *efx, bool local_node) 5637c45a4eSAlex Maftei (amaftei) { 57c265b569SÍñigo Huguet cpumask_var_t filter_mask; 5837c45a4eSAlex Maftei (amaftei) unsigned int count; 5937c45a4eSAlex Maftei (amaftei) int cpu; 6037c45a4eSAlex Maftei (amaftei) 61c265b569SÍñigo Huguet if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) { 6237c45a4eSAlex Maftei (amaftei) netif_warn(efx, probe, efx->net_dev, 6337c45a4eSAlex Maftei (amaftei) "RSS disabled due to allocation failure\n"); 6437c45a4eSAlex Maftei (amaftei) return 1; 6537c45a4eSAlex Maftei (amaftei) } 6637c45a4eSAlex Maftei (amaftei) 67c265b569SÍñigo Huguet cpumask_copy(filter_mask, cpu_online_mask); 68c9ad266bSMartin Habets if (local_node) 69c9ad266bSMartin Habets cpumask_and(filter_mask, filter_mask, 70c9ad266bSMartin Habets cpumask_of_pcibus(efx->pci_dev->bus)); 7137c45a4eSAlex Maftei (amaftei) 72c265b569SÍñigo Huguet count = 0; 73c265b569SÍñigo Huguet for_each_cpu(cpu, filter_mask) { 74c265b569SÍñigo Huguet ++count; 75c265b569SÍñigo Huguet cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu)); 76c265b569SÍñigo Huguet } 77c265b569SÍñigo Huguet 78c265b569SÍñigo Huguet free_cpumask_var(filter_mask); 79c265b569SÍñigo Huguet 80c265b569SÍñigo Huguet return count; 81c265b569SÍñigo Huguet } 82c265b569SÍñigo Huguet 83c265b569SÍñigo Huguet static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 84c265b569SÍñigo Huguet { 85c265b569SÍñigo Huguet unsigned int count; 86c265b569SÍñigo Huguet 87c265b569SÍñigo Huguet if (rss_cpus) { 88c265b569SÍñigo Huguet count = rss_cpus; 89c265b569SÍñigo Huguet } else { 90c265b569SÍñigo Huguet count = count_online_cores(efx, true); 91c265b569SÍñigo Huguet 92c265b569SÍñigo Huguet /* If no online CPUs in local node, fallback to any online CPUs */ 93c265b569SÍñigo Huguet if (count == 0) 94c265b569SÍñigo Huguet count = count_online_cores(efx, false); 9537c45a4eSAlex Maftei (amaftei) } 9637c45a4eSAlex Maftei (amaftei) 9737c45a4eSAlex Maftei (amaftei) if (count > EFX_MAX_RX_QUEUES) { 9837c45a4eSAlex Maftei (amaftei) netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 9937c45a4eSAlex Maftei (amaftei) "Reducing number of rx queues from %u to %u.\n", 10037c45a4eSAlex Maftei (amaftei) count, EFX_MAX_RX_QUEUES); 10137c45a4eSAlex Maftei (amaftei) count = EFX_MAX_RX_QUEUES; 10237c45a4eSAlex Maftei (amaftei) } 10337c45a4eSAlex Maftei (amaftei) 10437c45a4eSAlex Maftei (amaftei) /* If RSS is requested for the PF *and* VFs then we can't write RSS 10537c45a4eSAlex Maftei (amaftei) * table entries that are inaccessible to VFs 10637c45a4eSAlex Maftei (amaftei) */ 10737c45a4eSAlex Maftei (amaftei) #ifdef CONFIG_SFC_SRIOV 10837c45a4eSAlex Maftei (amaftei) if (efx->type->sriov_wanted) { 10937c45a4eSAlex Maftei (amaftei) if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 11037c45a4eSAlex Maftei (amaftei) count > efx_vf_size(efx)) { 11137c45a4eSAlex Maftei (amaftei) netif_warn(efx, probe, efx->net_dev, 11237c45a4eSAlex Maftei (amaftei) "Reducing number of RSS channels from %u to %u for " 11337c45a4eSAlex Maftei (amaftei) "VF support. Increase vf-msix-limit to use more " 11437c45a4eSAlex Maftei (amaftei) "channels on the PF.\n", 11537c45a4eSAlex Maftei (amaftei) count, efx_vf_size(efx)); 11637c45a4eSAlex Maftei (amaftei) count = efx_vf_size(efx); 11737c45a4eSAlex Maftei (amaftei) } 11837c45a4eSAlex Maftei (amaftei) } 11937c45a4eSAlex Maftei (amaftei) #endif 12037c45a4eSAlex Maftei (amaftei) 12137c45a4eSAlex Maftei (amaftei) return count; 12237c45a4eSAlex Maftei (amaftei) } 12337c45a4eSAlex Maftei (amaftei) 12437c45a4eSAlex Maftei (amaftei) static int efx_allocate_msix_channels(struct efx_nic *efx, 12537c45a4eSAlex Maftei (amaftei) unsigned int max_channels, 12637c45a4eSAlex Maftei (amaftei) unsigned int extra_channels, 12737c45a4eSAlex Maftei (amaftei) unsigned int parallelism) 12837c45a4eSAlex Maftei (amaftei) { 12937c45a4eSAlex Maftei (amaftei) unsigned int n_channels = parallelism; 13037c45a4eSAlex Maftei (amaftei) int vec_count; 131e26ca4b5SIvan Babrou int tx_per_ev; 13237c45a4eSAlex Maftei (amaftei) int n_xdp_tx; 13337c45a4eSAlex Maftei (amaftei) int n_xdp_ev; 13437c45a4eSAlex Maftei (amaftei) 13537c45a4eSAlex Maftei (amaftei) if (efx_separate_tx_channels) 13637c45a4eSAlex Maftei (amaftei) n_channels *= 2; 13737c45a4eSAlex Maftei (amaftei) n_channels += extra_channels; 13837c45a4eSAlex Maftei (amaftei) 13937c45a4eSAlex Maftei (amaftei) /* To allow XDP transmit to happen from arbitrary NAPI contexts 14037c45a4eSAlex Maftei (amaftei) * we allocate a TX queue per CPU. We share event queues across 14137c45a4eSAlex Maftei (amaftei) * multiple tx queues, assuming tx and ev queues are both 14237c45a4eSAlex Maftei (amaftei) * maximum size. 14337c45a4eSAlex Maftei (amaftei) */ 144e26ca4b5SIvan Babrou tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx); 145f28100cbSÍñigo Huguet tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL); 14637c45a4eSAlex Maftei (amaftei) n_xdp_tx = num_possible_cpus(); 147e26ca4b5SIvan Babrou n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev); 14837c45a4eSAlex Maftei (amaftei) 14937c45a4eSAlex Maftei (amaftei) vec_count = pci_msix_vec_count(efx->pci_dev); 15037c45a4eSAlex Maftei (amaftei) if (vec_count < 0) 15137c45a4eSAlex Maftei (amaftei) return vec_count; 15237c45a4eSAlex Maftei (amaftei) 15337c45a4eSAlex Maftei (amaftei) max_channels = min_t(unsigned int, vec_count, max_channels); 15437c45a4eSAlex Maftei (amaftei) 15537c45a4eSAlex Maftei (amaftei) /* Check resources. 15637c45a4eSAlex Maftei (amaftei) * We need a channel per event queue, plus a VI per tx queue. 15737c45a4eSAlex Maftei (amaftei) * This may be more pessimistic than it needs to be. 15837c45a4eSAlex Maftei (amaftei) */ 15941544618SÍñigo Huguet if (n_channels >= max_channels) { 1606215b608SÍñigo Huguet efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; 1616215b608SÍñigo Huguet netif_warn(efx, drv, efx->net_dev, 16237c45a4eSAlex Maftei (amaftei) "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", 16337c45a4eSAlex Maftei (amaftei) n_xdp_ev, n_channels, max_channels); 1646215b608SÍñigo Huguet netif_warn(efx, drv, efx->net_dev, 1656215b608SÍñigo Huguet "XDP_TX and XDP_REDIRECT might decrease device's performance\n"); 166de5f32e2SEdward Cree } else if (n_channels + n_xdp_tx > efx->max_vis) { 1676215b608SÍñigo Huguet efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; 1686215b608SÍñigo Huguet netif_warn(efx, drv, efx->net_dev, 169de5f32e2SEdward Cree "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n", 170de5f32e2SEdward Cree n_xdp_tx, n_channels, efx->max_vis); 1716215b608SÍñigo Huguet netif_warn(efx, drv, efx->net_dev, 1726215b608SÍñigo Huguet "XDP_TX and XDP_REDIRECT might decrease device's performance\n"); 17341544618SÍñigo Huguet } else if (n_channels + n_xdp_ev > max_channels) { 17441544618SÍñigo Huguet efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED; 17541544618SÍñigo Huguet netif_warn(efx, drv, efx->net_dev, 17641544618SÍñigo Huguet "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", 17741544618SÍñigo Huguet n_xdp_ev, n_channels, max_channels); 17841544618SÍñigo Huguet 17941544618SÍñigo Huguet n_xdp_ev = max_channels - n_channels; 18041544618SÍñigo Huguet netif_warn(efx, drv, efx->net_dev, 18141544618SÍñigo Huguet "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n", 18241544618SÍñigo Huguet DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev)); 18337c45a4eSAlex Maftei (amaftei) } else { 18441544618SÍñigo Huguet efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED; 18541544618SÍñigo Huguet } 18641544618SÍñigo Huguet 1876215b608SÍñigo Huguet if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) { 18837c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = n_xdp_ev; 189f28100cbSÍñigo Huguet efx->xdp_tx_per_channel = tx_per_ev; 19037c45a4eSAlex Maftei (amaftei) efx->xdp_tx_queue_count = n_xdp_tx; 19137c45a4eSAlex Maftei (amaftei) n_channels += n_xdp_ev; 19237c45a4eSAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 19337c45a4eSAlex Maftei (amaftei) "Allocating %d TX and %d event queues for XDP\n", 19441544618SÍñigo Huguet n_xdp_ev * tx_per_ev, n_xdp_ev); 19541544618SÍñigo Huguet } else { 19641544618SÍñigo Huguet efx->n_xdp_channels = 0; 19741544618SÍñigo Huguet efx->xdp_tx_per_channel = 0; 1986215b608SÍñigo Huguet efx->xdp_tx_queue_count = n_xdp_tx; 19937c45a4eSAlex Maftei (amaftei) } 20037c45a4eSAlex Maftei (amaftei) 20137c45a4eSAlex Maftei (amaftei) if (vec_count < n_channels) { 20237c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 20337c45a4eSAlex Maftei (amaftei) "WARNING: Insufficient MSI-X vectors available (%d < %u).\n", 20437c45a4eSAlex Maftei (amaftei) vec_count, n_channels); 20537c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 20637c45a4eSAlex Maftei (amaftei) "WARNING: Performance may be reduced.\n"); 20737c45a4eSAlex Maftei (amaftei) n_channels = vec_count; 20837c45a4eSAlex Maftei (amaftei) } 20937c45a4eSAlex Maftei (amaftei) 21037c45a4eSAlex Maftei (amaftei) n_channels = min(n_channels, max_channels); 21137c45a4eSAlex Maftei (amaftei) 21237c45a4eSAlex Maftei (amaftei) efx->n_channels = n_channels; 21337c45a4eSAlex Maftei (amaftei) 21437c45a4eSAlex Maftei (amaftei) /* Ignore XDP tx channels when creating rx channels. */ 21537c45a4eSAlex Maftei (amaftei) n_channels -= efx->n_xdp_channels; 21637c45a4eSAlex Maftei (amaftei) 21737c45a4eSAlex Maftei (amaftei) if (efx_separate_tx_channels) { 21837c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = 21937c45a4eSAlex Maftei (amaftei) min(max(n_channels / 2, 1U), 22037c45a4eSAlex Maftei (amaftei) efx->max_tx_channels); 22137c45a4eSAlex Maftei (amaftei) efx->tx_channel_offset = 22237c45a4eSAlex Maftei (amaftei) n_channels - efx->n_tx_channels; 22337c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = 22437c45a4eSAlex Maftei (amaftei) max(n_channels - 22537c45a4eSAlex Maftei (amaftei) efx->n_tx_channels, 1U); 22637c45a4eSAlex Maftei (amaftei) } else { 22737c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = min(n_channels, efx->max_tx_channels); 22837c45a4eSAlex Maftei (amaftei) efx->tx_channel_offset = 0; 22937c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = n_channels; 23037c45a4eSAlex Maftei (amaftei) } 23137c45a4eSAlex Maftei (amaftei) 23237c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = min(efx->n_rx_channels, parallelism); 23337c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = min(efx->n_tx_channels, parallelism); 23437c45a4eSAlex Maftei (amaftei) 23537c45a4eSAlex Maftei (amaftei) efx->xdp_channel_offset = n_channels; 23637c45a4eSAlex Maftei (amaftei) 23737c45a4eSAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 23837c45a4eSAlex Maftei (amaftei) "Allocating %u RX channels\n", 23937c45a4eSAlex Maftei (amaftei) efx->n_rx_channels); 24037c45a4eSAlex Maftei (amaftei) 24137c45a4eSAlex Maftei (amaftei) return efx->n_channels; 24237c45a4eSAlex Maftei (amaftei) } 24337c45a4eSAlex Maftei (amaftei) 24437c45a4eSAlex Maftei (amaftei) /* Probe the number and type of interrupts we are able to obtain, and 24537c45a4eSAlex Maftei (amaftei) * the resulting numbers of channels and RX queues. 24637c45a4eSAlex Maftei (amaftei) */ 24737c45a4eSAlex Maftei (amaftei) int efx_probe_interrupts(struct efx_nic *efx) 24837c45a4eSAlex Maftei (amaftei) { 24937c45a4eSAlex Maftei (amaftei) unsigned int extra_channels = 0; 25037c45a4eSAlex Maftei (amaftei) unsigned int rss_spread; 25137c45a4eSAlex Maftei (amaftei) unsigned int i, j; 25237c45a4eSAlex Maftei (amaftei) int rc; 25337c45a4eSAlex Maftei (amaftei) 25437c45a4eSAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 25537c45a4eSAlex Maftei (amaftei) if (efx->extra_channel_type[i]) 25637c45a4eSAlex Maftei (amaftei) ++extra_channels; 25737c45a4eSAlex Maftei (amaftei) 25837c45a4eSAlex Maftei (amaftei) if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 25937c45a4eSAlex Maftei (amaftei) unsigned int parallelism = efx_wanted_parallelism(efx); 26037c45a4eSAlex Maftei (amaftei) struct msix_entry xentries[EFX_MAX_CHANNELS]; 26137c45a4eSAlex Maftei (amaftei) unsigned int n_channels; 26237c45a4eSAlex Maftei (amaftei) 26337c45a4eSAlex Maftei (amaftei) rc = efx_allocate_msix_channels(efx, efx->max_channels, 26437c45a4eSAlex Maftei (amaftei) extra_channels, parallelism); 26537c45a4eSAlex Maftei (amaftei) if (rc >= 0) { 26637c45a4eSAlex Maftei (amaftei) n_channels = rc; 26737c45a4eSAlex Maftei (amaftei) for (i = 0; i < n_channels; i++) 26837c45a4eSAlex Maftei (amaftei) xentries[i].entry = i; 26937c45a4eSAlex Maftei (amaftei) rc = pci_enable_msix_range(efx->pci_dev, xentries, 1, 27037c45a4eSAlex Maftei (amaftei) n_channels); 27137c45a4eSAlex Maftei (amaftei) } 27237c45a4eSAlex Maftei (amaftei) if (rc < 0) { 27337c45a4eSAlex Maftei (amaftei) /* Fall back to single channel MSI */ 27437c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 27537c45a4eSAlex Maftei (amaftei) "could not enable MSI-X\n"); 27637c45a4eSAlex Maftei (amaftei) if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 27737c45a4eSAlex Maftei (amaftei) efx->interrupt_mode = EFX_INT_MODE_MSI; 27837c45a4eSAlex Maftei (amaftei) else 27937c45a4eSAlex Maftei (amaftei) return rc; 28037c45a4eSAlex Maftei (amaftei) } else if (rc < n_channels) { 28137c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 28237c45a4eSAlex Maftei (amaftei) "WARNING: Insufficient MSI-X vectors" 28337c45a4eSAlex Maftei (amaftei) " available (%d < %u).\n", rc, n_channels); 28437c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 28537c45a4eSAlex Maftei (amaftei) "WARNING: Performance may be reduced.\n"); 28637c45a4eSAlex Maftei (amaftei) n_channels = rc; 28737c45a4eSAlex Maftei (amaftei) } 28837c45a4eSAlex Maftei (amaftei) 28937c45a4eSAlex Maftei (amaftei) if (rc > 0) { 29037c45a4eSAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) 29137c45a4eSAlex Maftei (amaftei) efx_get_channel(efx, i)->irq = 29237c45a4eSAlex Maftei (amaftei) xentries[i].vector; 29337c45a4eSAlex Maftei (amaftei) } 29437c45a4eSAlex Maftei (amaftei) } 29537c45a4eSAlex Maftei (amaftei) 29637c45a4eSAlex Maftei (amaftei) /* Try single interrupt MSI */ 29737c45a4eSAlex Maftei (amaftei) if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 29837c45a4eSAlex Maftei (amaftei) efx->n_channels = 1; 29937c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = 1; 30037c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = 1; 301c308dfd1SÍñigo Huguet efx->tx_channel_offset = 0; 30237c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = 0; 30337c45a4eSAlex Maftei (amaftei) efx->xdp_channel_offset = efx->n_channels; 304*e84a1e1eSÍñigo Huguet efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; 30537c45a4eSAlex Maftei (amaftei) rc = pci_enable_msi(efx->pci_dev); 30637c45a4eSAlex Maftei (amaftei) if (rc == 0) { 30737c45a4eSAlex Maftei (amaftei) efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 30837c45a4eSAlex Maftei (amaftei) } else { 30937c45a4eSAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 31037c45a4eSAlex Maftei (amaftei) "could not enable MSI\n"); 31137c45a4eSAlex Maftei (amaftei) if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 31237c45a4eSAlex Maftei (amaftei) efx->interrupt_mode = EFX_INT_MODE_LEGACY; 31337c45a4eSAlex Maftei (amaftei) else 31437c45a4eSAlex Maftei (amaftei) return rc; 31537c45a4eSAlex Maftei (amaftei) } 31637c45a4eSAlex Maftei (amaftei) } 31737c45a4eSAlex Maftei (amaftei) 31837c45a4eSAlex Maftei (amaftei) /* Assume legacy interrupts */ 31937c45a4eSAlex Maftei (amaftei) if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 32037c45a4eSAlex Maftei (amaftei) efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 32137c45a4eSAlex Maftei (amaftei) efx->n_rx_channels = 1; 32237c45a4eSAlex Maftei (amaftei) efx->n_tx_channels = 1; 323f232af42SÍñigo Huguet efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0; 32437c45a4eSAlex Maftei (amaftei) efx->n_xdp_channels = 0; 32537c45a4eSAlex Maftei (amaftei) efx->xdp_channel_offset = efx->n_channels; 326*e84a1e1eSÍñigo Huguet efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; 32737c45a4eSAlex Maftei (amaftei) efx->legacy_irq = efx->pci_dev->irq; 32837c45a4eSAlex Maftei (amaftei) } 32937c45a4eSAlex Maftei (amaftei) 33037c45a4eSAlex Maftei (amaftei) /* Assign extra channels if possible, before XDP channels */ 33137c45a4eSAlex Maftei (amaftei) efx->n_extra_tx_channels = 0; 33237c45a4eSAlex Maftei (amaftei) j = efx->xdp_channel_offset; 33337c45a4eSAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 33437c45a4eSAlex Maftei (amaftei) if (!efx->extra_channel_type[i]) 33537c45a4eSAlex Maftei (amaftei) continue; 33637c45a4eSAlex Maftei (amaftei) if (j <= efx->tx_channel_offset + efx->n_tx_channels) { 33737c45a4eSAlex Maftei (amaftei) efx->extra_channel_type[i]->handle_no_channel(efx); 33837c45a4eSAlex Maftei (amaftei) } else { 33937c45a4eSAlex Maftei (amaftei) --j; 34037c45a4eSAlex Maftei (amaftei) efx_get_channel(efx, j)->type = 34137c45a4eSAlex Maftei (amaftei) efx->extra_channel_type[i]; 34237c45a4eSAlex Maftei (amaftei) if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 34337c45a4eSAlex Maftei (amaftei) efx->n_extra_tx_channels++; 34437c45a4eSAlex Maftei (amaftei) } 34537c45a4eSAlex Maftei (amaftei) } 34637c45a4eSAlex Maftei (amaftei) 34737c45a4eSAlex Maftei (amaftei) rss_spread = efx->n_rx_channels; 34837c45a4eSAlex Maftei (amaftei) /* RSS might be usable on VFs even if it is disabled on the PF */ 34937c45a4eSAlex Maftei (amaftei) #ifdef CONFIG_SFC_SRIOV 35037c45a4eSAlex Maftei (amaftei) if (efx->type->sriov_wanted) { 35137c45a4eSAlex Maftei (amaftei) efx->rss_spread = ((rss_spread > 1 || 35237c45a4eSAlex Maftei (amaftei) !efx->type->sriov_wanted(efx)) ? 35337c45a4eSAlex Maftei (amaftei) rss_spread : efx_vf_size(efx)); 35437c45a4eSAlex Maftei (amaftei) return 0; 35537c45a4eSAlex Maftei (amaftei) } 35637c45a4eSAlex Maftei (amaftei) #endif 35737c45a4eSAlex Maftei (amaftei) efx->rss_spread = rss_spread; 35837c45a4eSAlex Maftei (amaftei) 35937c45a4eSAlex Maftei (amaftei) return 0; 36037c45a4eSAlex Maftei (amaftei) } 36137c45a4eSAlex Maftei (amaftei) 36237c45a4eSAlex Maftei (amaftei) #if defined(CONFIG_SMP) 36337c45a4eSAlex Maftei (amaftei) void efx_set_interrupt_affinity(struct efx_nic *efx) 36437c45a4eSAlex Maftei (amaftei) { 365c9ad266bSMartin Habets const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus); 36637c45a4eSAlex Maftei (amaftei) struct efx_channel *channel; 36737c45a4eSAlex Maftei (amaftei) unsigned int cpu; 36837c45a4eSAlex Maftei (amaftei) 36909a99ab1SÍñigo Huguet /* If no online CPUs in local node, fallback to any online CPU */ 37009a99ab1SÍñigo Huguet if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids) 37109a99ab1SÍñigo Huguet numa_mask = cpu_online_mask; 37209a99ab1SÍñigo Huguet 37309a99ab1SÍñigo Huguet cpu = -1; 37437c45a4eSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 37509a99ab1SÍñigo Huguet cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask); 37609a99ab1SÍñigo Huguet if (cpu >= nr_cpu_ids) 37709a99ab1SÍñigo Huguet cpu = cpumask_first_and(cpu_online_mask, numa_mask); 37837c45a4eSAlex Maftei (amaftei) irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 37937c45a4eSAlex Maftei (amaftei) } 38037c45a4eSAlex Maftei (amaftei) } 38137c45a4eSAlex Maftei (amaftei) 38237c45a4eSAlex Maftei (amaftei) void efx_clear_interrupt_affinity(struct efx_nic *efx) 38337c45a4eSAlex Maftei (amaftei) { 38437c45a4eSAlex Maftei (amaftei) struct efx_channel *channel; 38537c45a4eSAlex Maftei (amaftei) 38637c45a4eSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 38737c45a4eSAlex Maftei (amaftei) irq_set_affinity_hint(channel->irq, NULL); 38837c45a4eSAlex Maftei (amaftei) } 38937c45a4eSAlex Maftei (amaftei) #else 39037c45a4eSAlex Maftei (amaftei) void 39137c45a4eSAlex Maftei (amaftei) efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 39237c45a4eSAlex Maftei (amaftei) { 39337c45a4eSAlex Maftei (amaftei) } 39437c45a4eSAlex Maftei (amaftei) 39537c45a4eSAlex Maftei (amaftei) void 39637c45a4eSAlex Maftei (amaftei) efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) 39737c45a4eSAlex Maftei (amaftei) { 39837c45a4eSAlex Maftei (amaftei) } 39937c45a4eSAlex Maftei (amaftei) #endif /* CONFIG_SMP */ 40037c45a4eSAlex Maftei (amaftei) 40137c45a4eSAlex Maftei (amaftei) void efx_remove_interrupts(struct efx_nic *efx) 40237c45a4eSAlex Maftei (amaftei) { 40337c45a4eSAlex Maftei (amaftei) struct efx_channel *channel; 40437c45a4eSAlex Maftei (amaftei) 40537c45a4eSAlex Maftei (amaftei) /* Remove MSI/MSI-X interrupts */ 40637c45a4eSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 40737c45a4eSAlex Maftei (amaftei) channel->irq = 0; 40837c45a4eSAlex Maftei (amaftei) pci_disable_msi(efx->pci_dev); 40937c45a4eSAlex Maftei (amaftei) pci_disable_msix(efx->pci_dev); 41037c45a4eSAlex Maftei (amaftei) 41137c45a4eSAlex Maftei (amaftei) /* Remove legacy interrupt */ 41237c45a4eSAlex Maftei (amaftei) efx->legacy_irq = 0; 41337c45a4eSAlex Maftei (amaftei) } 41437c45a4eSAlex Maftei (amaftei) 4155f999256SAlex Maftei (amaftei) /*************** 4165f999256SAlex Maftei (amaftei) * EVENT QUEUES 4175f999256SAlex Maftei (amaftei) ***************/ 4185f999256SAlex Maftei (amaftei) 4195f999256SAlex Maftei (amaftei) /* Create event queue 4205f999256SAlex Maftei (amaftei) * Event queue memory allocations are done only once. If the channel 4215f999256SAlex Maftei (amaftei) * is reset, the memory buffer will be reused; this guards against 4225f999256SAlex Maftei (amaftei) * errors during channel reset and also simplifies interrupt handling. 4235f999256SAlex Maftei (amaftei) */ 4245f999256SAlex Maftei (amaftei) int efx_probe_eventq(struct efx_channel *channel) 4255f999256SAlex Maftei (amaftei) { 4265f999256SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 4275f999256SAlex Maftei (amaftei) unsigned long entries; 4285f999256SAlex Maftei (amaftei) 4295f999256SAlex Maftei (amaftei) netif_dbg(efx, probe, efx->net_dev, 4305f999256SAlex Maftei (amaftei) "chan %d create event queue\n", channel->channel); 4315f999256SAlex Maftei (amaftei) 4325f999256SAlex Maftei (amaftei) /* Build an event queue with room for one event per tx and rx buffer, 4335f999256SAlex Maftei (amaftei) * plus some extra for link state events and MCDI completions. 4345f999256SAlex Maftei (amaftei) */ 4355f999256SAlex Maftei (amaftei) entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 4365f999256SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 4375f999256SAlex Maftei (amaftei) channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 4385f999256SAlex Maftei (amaftei) 4395f999256SAlex Maftei (amaftei) return efx_nic_probe_eventq(channel); 4405f999256SAlex Maftei (amaftei) } 4415f999256SAlex Maftei (amaftei) 4425f999256SAlex Maftei (amaftei) /* Prepare channel's event queue */ 4435f999256SAlex Maftei (amaftei) int efx_init_eventq(struct efx_channel *channel) 4445f999256SAlex Maftei (amaftei) { 4455f999256SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 4465f999256SAlex Maftei (amaftei) int rc; 4475f999256SAlex Maftei (amaftei) 4485f999256SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(channel->eventq_init); 4495f999256SAlex Maftei (amaftei) 4505f999256SAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 4515f999256SAlex Maftei (amaftei) "chan %d init event queue\n", channel->channel); 4525f999256SAlex Maftei (amaftei) 4535f999256SAlex Maftei (amaftei) rc = efx_nic_init_eventq(channel); 4545f999256SAlex Maftei (amaftei) if (rc == 0) { 4555f999256SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 4565f999256SAlex Maftei (amaftei) channel->eventq_read_ptr = 0; 4575f999256SAlex Maftei (amaftei) channel->eventq_init = true; 4585f999256SAlex Maftei (amaftei) } 4595f999256SAlex Maftei (amaftei) return rc; 4605f999256SAlex Maftei (amaftei) } 4615f999256SAlex Maftei (amaftei) 4625f999256SAlex Maftei (amaftei) /* Enable event queue processing and NAPI */ 4635f999256SAlex Maftei (amaftei) void efx_start_eventq(struct efx_channel *channel) 4645f999256SAlex Maftei (amaftei) { 4655f999256SAlex Maftei (amaftei) netif_dbg(channel->efx, ifup, channel->efx->net_dev, 4665f999256SAlex Maftei (amaftei) "chan %d start event queue\n", channel->channel); 4675f999256SAlex Maftei (amaftei) 4685f999256SAlex Maftei (amaftei) /* Make sure the NAPI handler sees the enabled flag set */ 4695f999256SAlex Maftei (amaftei) channel->enabled = true; 4705f999256SAlex Maftei (amaftei) smp_wmb(); 4715f999256SAlex Maftei (amaftei) 4725f999256SAlex Maftei (amaftei) napi_enable(&channel->napi_str); 4735f999256SAlex Maftei (amaftei) efx_nic_eventq_read_ack(channel); 4745f999256SAlex Maftei (amaftei) } 4755f999256SAlex Maftei (amaftei) 4765f999256SAlex Maftei (amaftei) /* Disable event queue processing and NAPI */ 4775f999256SAlex Maftei (amaftei) void efx_stop_eventq(struct efx_channel *channel) 4785f999256SAlex Maftei (amaftei) { 4795f999256SAlex Maftei (amaftei) if (!channel->enabled) 4805f999256SAlex Maftei (amaftei) return; 4815f999256SAlex Maftei (amaftei) 4825f999256SAlex Maftei (amaftei) napi_disable(&channel->napi_str); 4835f999256SAlex Maftei (amaftei) channel->enabled = false; 4845f999256SAlex Maftei (amaftei) } 4855f999256SAlex Maftei (amaftei) 4865f999256SAlex Maftei (amaftei) void efx_fini_eventq(struct efx_channel *channel) 4875f999256SAlex Maftei (amaftei) { 4885f999256SAlex Maftei (amaftei) if (!channel->eventq_init) 4895f999256SAlex Maftei (amaftei) return; 4905f999256SAlex Maftei (amaftei) 4915f999256SAlex Maftei (amaftei) netif_dbg(channel->efx, drv, channel->efx->net_dev, 4925f999256SAlex Maftei (amaftei) "chan %d fini event queue\n", channel->channel); 4935f999256SAlex Maftei (amaftei) 4945f999256SAlex Maftei (amaftei) efx_nic_fini_eventq(channel); 4955f999256SAlex Maftei (amaftei) channel->eventq_init = false; 4965f999256SAlex Maftei (amaftei) } 4975f999256SAlex Maftei (amaftei) 4985f999256SAlex Maftei (amaftei) void efx_remove_eventq(struct efx_channel *channel) 4995f999256SAlex Maftei (amaftei) { 5005f999256SAlex Maftei (amaftei) netif_dbg(channel->efx, drv, channel->efx->net_dev, 5015f999256SAlex Maftei (amaftei) "chan %d remove event queue\n", channel->channel); 5025f999256SAlex Maftei (amaftei) 5035f999256SAlex Maftei (amaftei) efx_nic_remove_eventq(channel); 5045f999256SAlex Maftei (amaftei) } 5055f999256SAlex Maftei (amaftei) 50683975485SAlex Maftei (amaftei) /************************************************************************** 50783975485SAlex Maftei (amaftei) * 50883975485SAlex Maftei (amaftei) * Channel handling 50983975485SAlex Maftei (amaftei) * 51083975485SAlex Maftei (amaftei) *************************************************************************/ 51183975485SAlex Maftei (amaftei) 512025c5a0bSEdward Cree #ifdef CONFIG_RFS_ACCEL 513025c5a0bSEdward Cree static void efx_filter_rfs_expire(struct work_struct *data) 514025c5a0bSEdward Cree { 515025c5a0bSEdward Cree struct delayed_work *dwork = to_delayed_work(data); 516025c5a0bSEdward Cree struct efx_channel *channel; 517025c5a0bSEdward Cree unsigned int time, quota; 518025c5a0bSEdward Cree 519025c5a0bSEdward Cree channel = container_of(dwork, struct efx_channel, filter_work); 520025c5a0bSEdward Cree time = jiffies - channel->rfs_last_expiry; 521025c5a0bSEdward Cree quota = channel->rfs_filter_count * time / (30 * HZ); 522025c5a0bSEdward Cree if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota))) 523025c5a0bSEdward Cree channel->rfs_last_expiry += time; 524025c5a0bSEdward Cree /* Ensure we do more work eventually even if NAPI poll is not happening */ 525025c5a0bSEdward Cree schedule_delayed_work(dwork, 30 * HZ); 526025c5a0bSEdward Cree } 527025c5a0bSEdward Cree #endif 528025c5a0bSEdward Cree 52983975485SAlex Maftei (amaftei) /* Allocate and initialise a channel structure. */ 5304da24fa6SEdward Cree static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i) 53183975485SAlex Maftei (amaftei) { 53283975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 53383975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 53483975485SAlex Maftei (amaftei) struct efx_channel *channel; 53583975485SAlex Maftei (amaftei) int j; 53683975485SAlex Maftei (amaftei) 53783975485SAlex Maftei (amaftei) channel = kzalloc(sizeof(*channel), GFP_KERNEL); 53883975485SAlex Maftei (amaftei) if (!channel) 53983975485SAlex Maftei (amaftei) return NULL; 54083975485SAlex Maftei (amaftei) 54183975485SAlex Maftei (amaftei) channel->efx = efx; 54283975485SAlex Maftei (amaftei) channel->channel = i; 54383975485SAlex Maftei (amaftei) channel->type = &efx_default_channel_type; 54483975485SAlex Maftei (amaftei) 54512804793SEdward Cree for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) { 54683975485SAlex Maftei (amaftei) tx_queue = &channel->tx_queue[j]; 54783975485SAlex Maftei (amaftei) tx_queue->efx = efx; 548a81dcd85SEdward Cree tx_queue->queue = -1; 549a81dcd85SEdward Cree tx_queue->label = j; 55083975485SAlex Maftei (amaftei) tx_queue->channel = channel; 55183975485SAlex Maftei (amaftei) } 55283975485SAlex Maftei (amaftei) 55383975485SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 55483975485SAlex Maftei (amaftei) INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 55583975485SAlex Maftei (amaftei) #endif 55683975485SAlex Maftei (amaftei) 55783975485SAlex Maftei (amaftei) rx_queue = &channel->rx_queue; 55883975485SAlex Maftei (amaftei) rx_queue->efx = efx; 55983975485SAlex Maftei (amaftei) timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 56083975485SAlex Maftei (amaftei) 56183975485SAlex Maftei (amaftei) return channel; 56283975485SAlex Maftei (amaftei) } 56383975485SAlex Maftei (amaftei) 56483975485SAlex Maftei (amaftei) int efx_init_channels(struct efx_nic *efx) 56583975485SAlex Maftei (amaftei) { 56683975485SAlex Maftei (amaftei) unsigned int i; 56783975485SAlex Maftei (amaftei) 56883975485SAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_CHANNELS; i++) { 5694da24fa6SEdward Cree efx->channel[i] = efx_alloc_channel(efx, i); 57083975485SAlex Maftei (amaftei) if (!efx->channel[i]) 57183975485SAlex Maftei (amaftei) return -ENOMEM; 57283975485SAlex Maftei (amaftei) efx->msi_context[i].efx = efx; 57383975485SAlex Maftei (amaftei) efx->msi_context[i].index = i; 57483975485SAlex Maftei (amaftei) } 57583975485SAlex Maftei (amaftei) 57683975485SAlex Maftei (amaftei) /* Higher numbered interrupt modes are less capable! */ 57783975485SAlex Maftei (amaftei) efx->interrupt_mode = min(efx->type->min_interrupt_mode, 578e4ff3232SEdward Cree efx_interrupt_mode); 57983975485SAlex Maftei (amaftei) 580937aa3aeSEdward Cree efx->max_channels = EFX_MAX_CHANNELS; 581937aa3aeSEdward Cree efx->max_tx_channels = EFX_MAX_CHANNELS; 582937aa3aeSEdward Cree 58383975485SAlex Maftei (amaftei) return 0; 58483975485SAlex Maftei (amaftei) } 58583975485SAlex Maftei (amaftei) 58683975485SAlex Maftei (amaftei) void efx_fini_channels(struct efx_nic *efx) 58783975485SAlex Maftei (amaftei) { 58883975485SAlex Maftei (amaftei) unsigned int i; 58983975485SAlex Maftei (amaftei) 59083975485SAlex Maftei (amaftei) for (i = 0; i < EFX_MAX_CHANNELS; i++) 59183975485SAlex Maftei (amaftei) if (efx->channel[i]) { 59283975485SAlex Maftei (amaftei) kfree(efx->channel[i]); 59383975485SAlex Maftei (amaftei) efx->channel[i] = NULL; 59483975485SAlex Maftei (amaftei) } 59583975485SAlex Maftei (amaftei) } 59683975485SAlex Maftei (amaftei) 59783975485SAlex Maftei (amaftei) /* Allocate and initialise a channel structure, copying parameters 59883975485SAlex Maftei (amaftei) * (but not resources) from an old channel structure. 59983975485SAlex Maftei (amaftei) */ 60083975485SAlex Maftei (amaftei) struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel) 60183975485SAlex Maftei (amaftei) { 60283975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 60383975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 60483975485SAlex Maftei (amaftei) struct efx_channel *channel; 60583975485SAlex Maftei (amaftei) int j; 60683975485SAlex Maftei (amaftei) 60783975485SAlex Maftei (amaftei) channel = kmalloc(sizeof(*channel), GFP_KERNEL); 60883975485SAlex Maftei (amaftei) if (!channel) 60983975485SAlex Maftei (amaftei) return NULL; 61083975485SAlex Maftei (amaftei) 61183975485SAlex Maftei (amaftei) *channel = *old_channel; 61283975485SAlex Maftei (amaftei) 61383975485SAlex Maftei (amaftei) channel->napi_dev = NULL; 61483975485SAlex Maftei (amaftei) INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 61583975485SAlex Maftei (amaftei) channel->napi_str.napi_id = 0; 61683975485SAlex Maftei (amaftei) channel->napi_str.state = 0; 61783975485SAlex Maftei (amaftei) memset(&channel->eventq, 0, sizeof(channel->eventq)); 61883975485SAlex Maftei (amaftei) 61912804793SEdward Cree for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) { 62083975485SAlex Maftei (amaftei) tx_queue = &channel->tx_queue[j]; 62183975485SAlex Maftei (amaftei) if (tx_queue->channel) 62283975485SAlex Maftei (amaftei) tx_queue->channel = channel; 62383975485SAlex Maftei (amaftei) tx_queue->buffer = NULL; 6244b1bd9dbSEdward Cree tx_queue->cb_page = NULL; 62583975485SAlex Maftei (amaftei) memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 62683975485SAlex Maftei (amaftei) } 62783975485SAlex Maftei (amaftei) 62883975485SAlex Maftei (amaftei) rx_queue = &channel->rx_queue; 62983975485SAlex Maftei (amaftei) rx_queue->buffer = NULL; 63083975485SAlex Maftei (amaftei) memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 63183975485SAlex Maftei (amaftei) timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 63283975485SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 63383975485SAlex Maftei (amaftei) INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 63483975485SAlex Maftei (amaftei) #endif 63583975485SAlex Maftei (amaftei) 63683975485SAlex Maftei (amaftei) return channel; 63783975485SAlex Maftei (amaftei) } 63883975485SAlex Maftei (amaftei) 63983975485SAlex Maftei (amaftei) static int efx_probe_channel(struct efx_channel *channel) 64083975485SAlex Maftei (amaftei) { 64183975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 64283975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 64383975485SAlex Maftei (amaftei) int rc; 64483975485SAlex Maftei (amaftei) 64583975485SAlex Maftei (amaftei) netif_dbg(channel->efx, probe, channel->efx->net_dev, 64683975485SAlex Maftei (amaftei) "creating channel %d\n", channel->channel); 64783975485SAlex Maftei (amaftei) 64883975485SAlex Maftei (amaftei) rc = channel->type->pre_probe(channel); 64983975485SAlex Maftei (amaftei) if (rc) 65083975485SAlex Maftei (amaftei) goto fail; 65183975485SAlex Maftei (amaftei) 65283975485SAlex Maftei (amaftei) rc = efx_probe_eventq(channel); 65383975485SAlex Maftei (amaftei) if (rc) 65483975485SAlex Maftei (amaftei) goto fail; 65583975485SAlex Maftei (amaftei) 65683975485SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 65783975485SAlex Maftei (amaftei) rc = efx_probe_tx_queue(tx_queue); 65883975485SAlex Maftei (amaftei) if (rc) 65983975485SAlex Maftei (amaftei) goto fail; 66083975485SAlex Maftei (amaftei) } 66183975485SAlex Maftei (amaftei) 66283975485SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) { 66383975485SAlex Maftei (amaftei) rc = efx_probe_rx_queue(rx_queue); 66483975485SAlex Maftei (amaftei) if (rc) 66583975485SAlex Maftei (amaftei) goto fail; 66683975485SAlex Maftei (amaftei) } 66783975485SAlex Maftei (amaftei) 66883975485SAlex Maftei (amaftei) channel->rx_list = NULL; 66983975485SAlex Maftei (amaftei) 67083975485SAlex Maftei (amaftei) return 0; 67183975485SAlex Maftei (amaftei) 67283975485SAlex Maftei (amaftei) fail: 67383975485SAlex Maftei (amaftei) efx_remove_channel(channel); 67483975485SAlex Maftei (amaftei) return rc; 67583975485SAlex Maftei (amaftei) } 67683975485SAlex Maftei (amaftei) 67754fccfddSMartin Habets static void efx_get_channel_name(struct efx_channel *channel, char *buf, 67854fccfddSMartin Habets size_t len) 67983975485SAlex Maftei (amaftei) { 68083975485SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 68183975485SAlex Maftei (amaftei) const char *type; 68283975485SAlex Maftei (amaftei) int number; 68383975485SAlex Maftei (amaftei) 68483975485SAlex Maftei (amaftei) number = channel->channel; 68583975485SAlex Maftei (amaftei) 68683975485SAlex Maftei (amaftei) if (number >= efx->xdp_channel_offset && 68783975485SAlex Maftei (amaftei) !WARN_ON_ONCE(!efx->n_xdp_channels)) { 68883975485SAlex Maftei (amaftei) type = "-xdp"; 68983975485SAlex Maftei (amaftei) number -= efx->xdp_channel_offset; 69083975485SAlex Maftei (amaftei) } else if (efx->tx_channel_offset == 0) { 69183975485SAlex Maftei (amaftei) type = ""; 69283975485SAlex Maftei (amaftei) } else if (number < efx->tx_channel_offset) { 69383975485SAlex Maftei (amaftei) type = "-rx"; 69483975485SAlex Maftei (amaftei) } else { 69583975485SAlex Maftei (amaftei) type = "-tx"; 69683975485SAlex Maftei (amaftei) number -= efx->tx_channel_offset; 69783975485SAlex Maftei (amaftei) } 69883975485SAlex Maftei (amaftei) snprintf(buf, len, "%s%s-%d", efx->name, type, number); 69983975485SAlex Maftei (amaftei) } 70083975485SAlex Maftei (amaftei) 70183975485SAlex Maftei (amaftei) void efx_set_channel_names(struct efx_nic *efx) 70283975485SAlex Maftei (amaftei) { 70383975485SAlex Maftei (amaftei) struct efx_channel *channel; 70483975485SAlex Maftei (amaftei) 70583975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 70683975485SAlex Maftei (amaftei) channel->type->get_name(channel, 70783975485SAlex Maftei (amaftei) efx->msi_context[channel->channel].name, 70883975485SAlex Maftei (amaftei) sizeof(efx->msi_context[0].name)); 70983975485SAlex Maftei (amaftei) } 71083975485SAlex Maftei (amaftei) 71183975485SAlex Maftei (amaftei) int efx_probe_channels(struct efx_nic *efx) 71283975485SAlex Maftei (amaftei) { 71383975485SAlex Maftei (amaftei) struct efx_channel *channel; 71483975485SAlex Maftei (amaftei) int rc; 71583975485SAlex Maftei (amaftei) 71683975485SAlex Maftei (amaftei) /* Restart special buffer allocation */ 71783975485SAlex Maftei (amaftei) efx->next_buffer_table = 0; 71883975485SAlex Maftei (amaftei) 71983975485SAlex Maftei (amaftei) /* Probe channels in reverse, so that any 'extra' channels 72083975485SAlex Maftei (amaftei) * use the start of the buffer table. This allows the traffic 72183975485SAlex Maftei (amaftei) * channels to be resized without moving them or wasting the 72283975485SAlex Maftei (amaftei) * entries before them. 72383975485SAlex Maftei (amaftei) */ 72483975485SAlex Maftei (amaftei) efx_for_each_channel_rev(channel, efx) { 72583975485SAlex Maftei (amaftei) rc = efx_probe_channel(channel); 72683975485SAlex Maftei (amaftei) if (rc) { 72783975485SAlex Maftei (amaftei) netif_err(efx, probe, efx->net_dev, 72883975485SAlex Maftei (amaftei) "failed to create channel %d\n", 72983975485SAlex Maftei (amaftei) channel->channel); 73083975485SAlex Maftei (amaftei) goto fail; 73183975485SAlex Maftei (amaftei) } 73283975485SAlex Maftei (amaftei) } 73383975485SAlex Maftei (amaftei) efx_set_channel_names(efx); 73483975485SAlex Maftei (amaftei) 73583975485SAlex Maftei (amaftei) return 0; 73683975485SAlex Maftei (amaftei) 73783975485SAlex Maftei (amaftei) fail: 73883975485SAlex Maftei (amaftei) efx_remove_channels(efx); 73983975485SAlex Maftei (amaftei) return rc; 74083975485SAlex Maftei (amaftei) } 74183975485SAlex Maftei (amaftei) 74283975485SAlex Maftei (amaftei) void efx_remove_channel(struct efx_channel *channel) 74383975485SAlex Maftei (amaftei) { 74483975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 74583975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 74683975485SAlex Maftei (amaftei) 74783975485SAlex Maftei (amaftei) netif_dbg(channel->efx, drv, channel->efx->net_dev, 74883975485SAlex Maftei (amaftei) "destroy chan %d\n", channel->channel); 74983975485SAlex Maftei (amaftei) 75083975485SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 75183975485SAlex Maftei (amaftei) efx_remove_rx_queue(rx_queue); 752f9cac93eSEdward Cree efx_for_each_channel_tx_queue(tx_queue, channel) 75383975485SAlex Maftei (amaftei) efx_remove_tx_queue(tx_queue); 75483975485SAlex Maftei (amaftei) efx_remove_eventq(channel); 75583975485SAlex Maftei (amaftei) channel->type->post_remove(channel); 75683975485SAlex Maftei (amaftei) } 75783975485SAlex Maftei (amaftei) 75883975485SAlex Maftei (amaftei) void efx_remove_channels(struct efx_nic *efx) 75983975485SAlex Maftei (amaftei) { 76083975485SAlex Maftei (amaftei) struct efx_channel *channel; 76183975485SAlex Maftei (amaftei) 76283975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 76383975485SAlex Maftei (amaftei) efx_remove_channel(channel); 76483975485SAlex Maftei (amaftei) 76583975485SAlex Maftei (amaftei) kfree(efx->xdp_tx_queues); 76683975485SAlex Maftei (amaftei) } 76783975485SAlex Maftei (amaftei) 768059a47f1STaehee Yoo static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number, 769059a47f1STaehee Yoo struct efx_tx_queue *tx_queue) 770059a47f1STaehee Yoo { 771059a47f1STaehee Yoo if (xdp_queue_number >= efx->xdp_tx_queue_count) 772059a47f1STaehee Yoo return -EINVAL; 773059a47f1STaehee Yoo 774059a47f1STaehee Yoo netif_dbg(efx, drv, efx->net_dev, 775059a47f1STaehee Yoo "Channel %u TXQ %u is XDP %u, HW %u\n", 776059a47f1STaehee Yoo tx_queue->channel->channel, tx_queue->label, 777059a47f1STaehee Yoo xdp_queue_number, tx_queue->queue); 778059a47f1STaehee Yoo efx->xdp_tx_queues[xdp_queue_number] = tx_queue; 779059a47f1STaehee Yoo return 0; 780059a47f1STaehee Yoo } 781059a47f1STaehee Yoo 782059a47f1STaehee Yoo static void efx_set_xdp_channels(struct efx_nic *efx) 783059a47f1STaehee Yoo { 784059a47f1STaehee Yoo struct efx_tx_queue *tx_queue; 785059a47f1STaehee Yoo struct efx_channel *channel; 786059a47f1STaehee Yoo unsigned int next_queue = 0; 787059a47f1STaehee Yoo int xdp_queue_number = 0; 788059a47f1STaehee Yoo int rc; 789059a47f1STaehee Yoo 790059a47f1STaehee Yoo /* We need to mark which channels really have RX and TX 791059a47f1STaehee Yoo * queues, and adjust the TX queue numbers if we have separate 792059a47f1STaehee Yoo * RX-only and TX-only channels. 793059a47f1STaehee Yoo */ 794059a47f1STaehee Yoo efx_for_each_channel(channel, efx) { 795059a47f1STaehee Yoo if (channel->channel < efx->tx_channel_offset) 796059a47f1STaehee Yoo continue; 797059a47f1STaehee Yoo 798059a47f1STaehee Yoo if (efx_channel_is_xdp_tx(channel)) { 799059a47f1STaehee Yoo efx_for_each_channel_tx_queue(tx_queue, channel) { 800059a47f1STaehee Yoo tx_queue->queue = next_queue++; 801059a47f1STaehee Yoo rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, 802059a47f1STaehee Yoo tx_queue); 803059a47f1STaehee Yoo if (rc == 0) 804059a47f1STaehee Yoo xdp_queue_number++; 805059a47f1STaehee Yoo } 806059a47f1STaehee Yoo } else { 807059a47f1STaehee Yoo efx_for_each_channel_tx_queue(tx_queue, channel) { 808059a47f1STaehee Yoo tx_queue->queue = next_queue++; 809059a47f1STaehee Yoo netif_dbg(efx, drv, efx->net_dev, 810059a47f1STaehee Yoo "Channel %u TXQ %u is HW %u\n", 811059a47f1STaehee Yoo channel->channel, tx_queue->label, 812059a47f1STaehee Yoo tx_queue->queue); 813059a47f1STaehee Yoo } 814059a47f1STaehee Yoo 815059a47f1STaehee Yoo /* If XDP is borrowing queues from net stack, it must 816059a47f1STaehee Yoo * use the queue with no csum offload, which is the 817059a47f1STaehee Yoo * first one of the channel 818059a47f1STaehee Yoo * (note: tx_queue_by_type is not initialized yet) 819059a47f1STaehee Yoo */ 820059a47f1STaehee Yoo if (efx->xdp_txq_queues_mode == 821059a47f1STaehee Yoo EFX_XDP_TX_QUEUES_BORROWED) { 822059a47f1STaehee Yoo tx_queue = &channel->tx_queue[0]; 823059a47f1STaehee Yoo rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, 824059a47f1STaehee Yoo tx_queue); 825059a47f1STaehee Yoo if (rc == 0) 826059a47f1STaehee Yoo xdp_queue_number++; 827059a47f1STaehee Yoo } 828059a47f1STaehee Yoo } 829059a47f1STaehee Yoo } 830059a47f1STaehee Yoo WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED && 831059a47f1STaehee Yoo xdp_queue_number != efx->xdp_tx_queue_count); 832059a47f1STaehee Yoo WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED && 833059a47f1STaehee Yoo xdp_queue_number > efx->xdp_tx_queue_count); 834059a47f1STaehee Yoo 835059a47f1STaehee Yoo /* If we have more CPUs than assigned XDP TX queues, assign the already 836059a47f1STaehee Yoo * existing queues to the exceeding CPUs 837059a47f1STaehee Yoo */ 838059a47f1STaehee Yoo next_queue = 0; 839059a47f1STaehee Yoo while (xdp_queue_number < efx->xdp_tx_queue_count) { 840059a47f1STaehee Yoo tx_queue = efx->xdp_tx_queues[next_queue++]; 841059a47f1STaehee Yoo rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); 842059a47f1STaehee Yoo if (rc == 0) 843059a47f1STaehee Yoo xdp_queue_number++; 844059a47f1STaehee Yoo } 845059a47f1STaehee Yoo } 846059a47f1STaehee Yoo 84783975485SAlex Maftei (amaftei) int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 84883975485SAlex Maftei (amaftei) { 84949e6123cSTaehee Yoo struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel, 85049e6123cSTaehee Yoo *ptp_channel = efx_ptp_channel(efx); 85149e6123cSTaehee Yoo struct efx_ptp_data *ptp_data = efx->ptp_data; 85283975485SAlex Maftei (amaftei) unsigned int i, next_buffer_table = 0; 85383975485SAlex Maftei (amaftei) u32 old_rxq_entries, old_txq_entries; 85483975485SAlex Maftei (amaftei) int rc, rc2; 85583975485SAlex Maftei (amaftei) 85683975485SAlex Maftei (amaftei) rc = efx_check_disabled(efx); 85783975485SAlex Maftei (amaftei) if (rc) 85883975485SAlex Maftei (amaftei) return rc; 85983975485SAlex Maftei (amaftei) 86083975485SAlex Maftei (amaftei) /* Not all channels should be reallocated. We must avoid 86183975485SAlex Maftei (amaftei) * reallocating their buffer table entries. 86283975485SAlex Maftei (amaftei) */ 86383975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 86483975485SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 86583975485SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 86683975485SAlex Maftei (amaftei) 86783975485SAlex Maftei (amaftei) if (channel->type->copy) 86883975485SAlex Maftei (amaftei) continue; 86983975485SAlex Maftei (amaftei) next_buffer_table = max(next_buffer_table, 87083975485SAlex Maftei (amaftei) channel->eventq.index + 87183975485SAlex Maftei (amaftei) channel->eventq.entries); 87283975485SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 87383975485SAlex Maftei (amaftei) next_buffer_table = max(next_buffer_table, 87483975485SAlex Maftei (amaftei) rx_queue->rxd.index + 87583975485SAlex Maftei (amaftei) rx_queue->rxd.entries); 87683975485SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) 87783975485SAlex Maftei (amaftei) next_buffer_table = max(next_buffer_table, 87883975485SAlex Maftei (amaftei) tx_queue->txd.index + 87983975485SAlex Maftei (amaftei) tx_queue->txd.entries); 88083975485SAlex Maftei (amaftei) } 88183975485SAlex Maftei (amaftei) 88283975485SAlex Maftei (amaftei) efx_device_detach_sync(efx); 88383975485SAlex Maftei (amaftei) efx_stop_all(efx); 88483975485SAlex Maftei (amaftei) efx_soft_disable_interrupts(efx); 88583975485SAlex Maftei (amaftei) 88683975485SAlex Maftei (amaftei) /* Clone channels (where possible) */ 88783975485SAlex Maftei (amaftei) memset(other_channel, 0, sizeof(other_channel)); 88883975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 88983975485SAlex Maftei (amaftei) channel = efx->channel[i]; 89083975485SAlex Maftei (amaftei) if (channel->type->copy) 89183975485SAlex Maftei (amaftei) channel = channel->type->copy(channel); 89283975485SAlex Maftei (amaftei) if (!channel) { 89383975485SAlex Maftei (amaftei) rc = -ENOMEM; 89483975485SAlex Maftei (amaftei) goto out; 89583975485SAlex Maftei (amaftei) } 89683975485SAlex Maftei (amaftei) other_channel[i] = channel; 89783975485SAlex Maftei (amaftei) } 89883975485SAlex Maftei (amaftei) 89983975485SAlex Maftei (amaftei) /* Swap entry counts and channel pointers */ 90083975485SAlex Maftei (amaftei) old_rxq_entries = efx->rxq_entries; 90183975485SAlex Maftei (amaftei) old_txq_entries = efx->txq_entries; 90283975485SAlex Maftei (amaftei) efx->rxq_entries = rxq_entries; 90383975485SAlex Maftei (amaftei) efx->txq_entries = txq_entries; 9040cf765fbSJiapeng Chong for (i = 0; i < efx->n_channels; i++) 9050cf765fbSJiapeng Chong swap(efx->channel[i], other_channel[i]); 90683975485SAlex Maftei (amaftei) 90783975485SAlex Maftei (amaftei) /* Restart buffer table allocation */ 90883975485SAlex Maftei (amaftei) efx->next_buffer_table = next_buffer_table; 90983975485SAlex Maftei (amaftei) 91083975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 91183975485SAlex Maftei (amaftei) channel = efx->channel[i]; 91283975485SAlex Maftei (amaftei) if (!channel->type->copy) 91383975485SAlex Maftei (amaftei) continue; 91483975485SAlex Maftei (amaftei) rc = efx_probe_channel(channel); 91583975485SAlex Maftei (amaftei) if (rc) 91683975485SAlex Maftei (amaftei) goto rollback; 91783975485SAlex Maftei (amaftei) efx_init_napi_channel(efx->channel[i]); 91883975485SAlex Maftei (amaftei) } 91983975485SAlex Maftei (amaftei) 920059a47f1STaehee Yoo efx_set_xdp_channels(efx); 92183975485SAlex Maftei (amaftei) out: 92249e6123cSTaehee Yoo efx->ptp_data = NULL; 92383975485SAlex Maftei (amaftei) /* Destroy unused channel structures */ 92483975485SAlex Maftei (amaftei) for (i = 0; i < efx->n_channels; i++) { 92583975485SAlex Maftei (amaftei) channel = other_channel[i]; 92683975485SAlex Maftei (amaftei) if (channel && channel->type->copy) { 92783975485SAlex Maftei (amaftei) efx_fini_napi_channel(channel); 92883975485SAlex Maftei (amaftei) efx_remove_channel(channel); 92983975485SAlex Maftei (amaftei) kfree(channel); 93083975485SAlex Maftei (amaftei) } 93183975485SAlex Maftei (amaftei) } 93283975485SAlex Maftei (amaftei) 93349e6123cSTaehee Yoo efx->ptp_data = ptp_data; 93483975485SAlex Maftei (amaftei) rc2 = efx_soft_enable_interrupts(efx); 93583975485SAlex Maftei (amaftei) if (rc2) { 93683975485SAlex Maftei (amaftei) rc = rc ? rc : rc2; 93783975485SAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, 93883975485SAlex Maftei (amaftei) "unable to restart interrupts on channel reallocation\n"); 93983975485SAlex Maftei (amaftei) efx_schedule_reset(efx, RESET_TYPE_DISABLE); 94083975485SAlex Maftei (amaftei) } else { 94183975485SAlex Maftei (amaftei) efx_start_all(efx); 94283975485SAlex Maftei (amaftei) efx_device_attach_if_not_resetting(efx); 94383975485SAlex Maftei (amaftei) } 94483975485SAlex Maftei (amaftei) return rc; 94583975485SAlex Maftei (amaftei) 94683975485SAlex Maftei (amaftei) rollback: 94783975485SAlex Maftei (amaftei) /* Swap back */ 94883975485SAlex Maftei (amaftei) efx->rxq_entries = old_rxq_entries; 94983975485SAlex Maftei (amaftei) efx->txq_entries = old_txq_entries; 9500cf765fbSJiapeng Chong for (i = 0; i < efx->n_channels; i++) 9510cf765fbSJiapeng Chong swap(efx->channel[i], other_channel[i]); 95249e6123cSTaehee Yoo efx_ptp_update_channel(efx, ptp_channel); 95383975485SAlex Maftei (amaftei) goto out; 95483975485SAlex Maftei (amaftei) } 95583975485SAlex Maftei (amaftei) 95683975485SAlex Maftei (amaftei) int efx_set_channels(struct efx_nic *efx) 95783975485SAlex Maftei (amaftei) { 958a81dcd85SEdward Cree struct efx_channel *channel; 95969a70496SEdward Cree int rc; 96083975485SAlex Maftei (amaftei) 96183975485SAlex Maftei (amaftei) if (efx->xdp_tx_queue_count) { 96283975485SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(efx->xdp_tx_queues); 96383975485SAlex Maftei (amaftei) 96483975485SAlex Maftei (amaftei) /* Allocate array for XDP TX queue lookup. */ 96583975485SAlex Maftei (amaftei) efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count, 96683975485SAlex Maftei (amaftei) sizeof(*efx->xdp_tx_queues), 96783975485SAlex Maftei (amaftei) GFP_KERNEL); 96883975485SAlex Maftei (amaftei) if (!efx->xdp_tx_queues) 96983975485SAlex Maftei (amaftei) return -ENOMEM; 97083975485SAlex Maftei (amaftei) } 97183975485SAlex Maftei (amaftei) 97283975485SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 97383975485SAlex Maftei (amaftei) if (channel->channel < efx->n_rx_channels) 97483975485SAlex Maftei (amaftei) channel->rx_queue.core_index = channel->channel; 97583975485SAlex Maftei (amaftei) else 97683975485SAlex Maftei (amaftei) channel->rx_queue.core_index = -1; 977a81dcd85SEdward Cree } 9786215b608SÍñigo Huguet 979059a47f1STaehee Yoo efx_set_xdp_channels(efx); 98069a70496SEdward Cree 98169a70496SEdward Cree rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 98269a70496SEdward Cree if (rc) 98369a70496SEdward Cree return rc; 98469a70496SEdward Cree return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 98583975485SAlex Maftei (amaftei) } 98683975485SAlex Maftei (amaftei) 98754fccfddSMartin Habets static bool efx_default_channel_want_txqs(struct efx_channel *channel) 98883975485SAlex Maftei (amaftei) { 98983975485SAlex Maftei (amaftei) return channel->channel - channel->efx->tx_channel_offset < 99083975485SAlex Maftei (amaftei) channel->efx->n_tx_channels; 99183975485SAlex Maftei (amaftei) } 99283975485SAlex Maftei (amaftei) 993e20ba5b1SAlex Maftei (amaftei) /************* 994e20ba5b1SAlex Maftei (amaftei) * START/STOP 995e20ba5b1SAlex Maftei (amaftei) *************/ 996e20ba5b1SAlex Maftei (amaftei) 997e20ba5b1SAlex Maftei (amaftei) int efx_soft_enable_interrupts(struct efx_nic *efx) 998e20ba5b1SAlex Maftei (amaftei) { 999e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel, *end_channel; 1000e20ba5b1SAlex Maftei (amaftei) int rc; 1001e20ba5b1SAlex Maftei (amaftei) 1002e20ba5b1SAlex Maftei (amaftei) BUG_ON(efx->state == STATE_DISABLED); 1003e20ba5b1SAlex Maftei (amaftei) 1004e20ba5b1SAlex Maftei (amaftei) efx->irq_soft_enabled = true; 1005e20ba5b1SAlex Maftei (amaftei) smp_wmb(); 1006e20ba5b1SAlex Maftei (amaftei) 1007e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1008e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) { 1009e20ba5b1SAlex Maftei (amaftei) rc = efx_init_eventq(channel); 1010e20ba5b1SAlex Maftei (amaftei) if (rc) 1011e20ba5b1SAlex Maftei (amaftei) goto fail; 1012e20ba5b1SAlex Maftei (amaftei) } 1013e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 1014e20ba5b1SAlex Maftei (amaftei) } 1015e20ba5b1SAlex Maftei (amaftei) 1016e20ba5b1SAlex Maftei (amaftei) efx_mcdi_mode_event(efx); 1017e20ba5b1SAlex Maftei (amaftei) 1018e20ba5b1SAlex Maftei (amaftei) return 0; 1019e20ba5b1SAlex Maftei (amaftei) fail: 1020e20ba5b1SAlex Maftei (amaftei) end_channel = channel; 1021e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1022e20ba5b1SAlex Maftei (amaftei) if (channel == end_channel) 1023e20ba5b1SAlex Maftei (amaftei) break; 1024e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 1025e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) 1026e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 1027e20ba5b1SAlex Maftei (amaftei) } 1028e20ba5b1SAlex Maftei (amaftei) 1029e20ba5b1SAlex Maftei (amaftei) return rc; 1030e20ba5b1SAlex Maftei (amaftei) } 1031e20ba5b1SAlex Maftei (amaftei) 1032e20ba5b1SAlex Maftei (amaftei) void efx_soft_disable_interrupts(struct efx_nic *efx) 1033e20ba5b1SAlex Maftei (amaftei) { 1034e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 1035e20ba5b1SAlex Maftei (amaftei) 1036e20ba5b1SAlex Maftei (amaftei) if (efx->state == STATE_DISABLED) 1037e20ba5b1SAlex Maftei (amaftei) return; 1038e20ba5b1SAlex Maftei (amaftei) 1039e20ba5b1SAlex Maftei (amaftei) efx_mcdi_mode_poll(efx); 1040e20ba5b1SAlex Maftei (amaftei) 1041e20ba5b1SAlex Maftei (amaftei) efx->irq_soft_enabled = false; 1042e20ba5b1SAlex Maftei (amaftei) smp_wmb(); 1043e20ba5b1SAlex Maftei (amaftei) 1044e20ba5b1SAlex Maftei (amaftei) if (efx->legacy_irq) 1045e20ba5b1SAlex Maftei (amaftei) synchronize_irq(efx->legacy_irq); 1046e20ba5b1SAlex Maftei (amaftei) 1047e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1048e20ba5b1SAlex Maftei (amaftei) if (channel->irq) 1049e20ba5b1SAlex Maftei (amaftei) synchronize_irq(channel->irq); 1050e20ba5b1SAlex Maftei (amaftei) 1051e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 1052e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) 1053e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 1054e20ba5b1SAlex Maftei (amaftei) } 1055e20ba5b1SAlex Maftei (amaftei) 1056e20ba5b1SAlex Maftei (amaftei) /* Flush the asynchronous MCDI request queue */ 1057e20ba5b1SAlex Maftei (amaftei) efx_mcdi_flush_async(efx); 1058e20ba5b1SAlex Maftei (amaftei) } 1059e20ba5b1SAlex Maftei (amaftei) 1060e20ba5b1SAlex Maftei (amaftei) int efx_enable_interrupts(struct efx_nic *efx) 1061e20ba5b1SAlex Maftei (amaftei) { 1062e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel, *end_channel; 1063e20ba5b1SAlex Maftei (amaftei) int rc; 1064e20ba5b1SAlex Maftei (amaftei) 1065e20ba5b1SAlex Maftei (amaftei) /* TODO: Is this really a bug? */ 1066e20ba5b1SAlex Maftei (amaftei) BUG_ON(efx->state == STATE_DISABLED); 1067e20ba5b1SAlex Maftei (amaftei) 1068e20ba5b1SAlex Maftei (amaftei) if (efx->eeh_disabled_legacy_irq) { 1069e20ba5b1SAlex Maftei (amaftei) enable_irq(efx->legacy_irq); 1070e20ba5b1SAlex Maftei (amaftei) efx->eeh_disabled_legacy_irq = false; 1071e20ba5b1SAlex Maftei (amaftei) } 1072e20ba5b1SAlex Maftei (amaftei) 1073e20ba5b1SAlex Maftei (amaftei) efx->type->irq_enable_master(efx); 1074e20ba5b1SAlex Maftei (amaftei) 1075e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1076e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) { 1077e20ba5b1SAlex Maftei (amaftei) rc = efx_init_eventq(channel); 1078e20ba5b1SAlex Maftei (amaftei) if (rc) 1079e20ba5b1SAlex Maftei (amaftei) goto fail; 1080e20ba5b1SAlex Maftei (amaftei) } 1081e20ba5b1SAlex Maftei (amaftei) } 1082e20ba5b1SAlex Maftei (amaftei) 1083e20ba5b1SAlex Maftei (amaftei) rc = efx_soft_enable_interrupts(efx); 1084e20ba5b1SAlex Maftei (amaftei) if (rc) 1085e20ba5b1SAlex Maftei (amaftei) goto fail; 1086e20ba5b1SAlex Maftei (amaftei) 1087e20ba5b1SAlex Maftei (amaftei) return 0; 1088e20ba5b1SAlex Maftei (amaftei) 1089e20ba5b1SAlex Maftei (amaftei) fail: 1090e20ba5b1SAlex Maftei (amaftei) end_channel = channel; 1091e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1092e20ba5b1SAlex Maftei (amaftei) if (channel == end_channel) 1093e20ba5b1SAlex Maftei (amaftei) break; 1094e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) 1095e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 1096e20ba5b1SAlex Maftei (amaftei) } 1097e20ba5b1SAlex Maftei (amaftei) 1098e20ba5b1SAlex Maftei (amaftei) efx->type->irq_disable_non_ev(efx); 1099e20ba5b1SAlex Maftei (amaftei) 1100e20ba5b1SAlex Maftei (amaftei) return rc; 1101e20ba5b1SAlex Maftei (amaftei) } 1102e20ba5b1SAlex Maftei (amaftei) 1103e20ba5b1SAlex Maftei (amaftei) void efx_disable_interrupts(struct efx_nic *efx) 1104e20ba5b1SAlex Maftei (amaftei) { 1105e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 1106e20ba5b1SAlex Maftei (amaftei) 1107e20ba5b1SAlex Maftei (amaftei) efx_soft_disable_interrupts(efx); 1108e20ba5b1SAlex Maftei (amaftei) 1109e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1110e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) 1111e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 1112e20ba5b1SAlex Maftei (amaftei) } 1113e20ba5b1SAlex Maftei (amaftei) 1114e20ba5b1SAlex Maftei (amaftei) efx->type->irq_disable_non_ev(efx); 1115e20ba5b1SAlex Maftei (amaftei) } 1116e20ba5b1SAlex Maftei (amaftei) 1117e20ba5b1SAlex Maftei (amaftei) void efx_start_channels(struct efx_nic *efx) 1118e20ba5b1SAlex Maftei (amaftei) { 1119e20ba5b1SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 1120e20ba5b1SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 1121e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 1122e20ba5b1SAlex Maftei (amaftei) 1123fb5833d8STaehee Yoo efx_for_each_channel_rev(channel, efx) { 112485697f97SEdward Cree if (channel->type->start) 112585697f97SEdward Cree channel->type->start(channel); 1126e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 1127e20ba5b1SAlex Maftei (amaftei) efx_init_tx_queue(tx_queue); 1128e20ba5b1SAlex Maftei (amaftei) atomic_inc(&efx->active_queues); 1129e20ba5b1SAlex Maftei (amaftei) } 1130e20ba5b1SAlex Maftei (amaftei) 1131e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) { 1132e20ba5b1SAlex Maftei (amaftei) efx_init_rx_queue(rx_queue); 1133e20ba5b1SAlex Maftei (amaftei) atomic_inc(&efx->active_queues); 1134e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 1135e20ba5b1SAlex Maftei (amaftei) efx_fast_push_rx_descriptors(rx_queue, false); 1136e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 1137e20ba5b1SAlex Maftei (amaftei) } 1138e20ba5b1SAlex Maftei (amaftei) 1139e20ba5b1SAlex Maftei (amaftei) WARN_ON(channel->rx_pkt_n_frags); 1140e20ba5b1SAlex Maftei (amaftei) } 1141e20ba5b1SAlex Maftei (amaftei) } 1142e20ba5b1SAlex Maftei (amaftei) 1143e20ba5b1SAlex Maftei (amaftei) void efx_stop_channels(struct efx_nic *efx) 1144e20ba5b1SAlex Maftei (amaftei) { 1145e20ba5b1SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 1146e20ba5b1SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 1147e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 1148b5775b47SAlex Maftei (amaftei) int rc = 0; 1149e20ba5b1SAlex Maftei (amaftei) 115085697f97SEdward Cree /* Stop special channels and RX refill. 115185697f97SEdward Cree * The channel's stop has to be called first, since it might wait 115285697f97SEdward Cree * for a sentinel RX to indicate the channel has fully drained. 115385697f97SEdward Cree */ 1154e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 115585697f97SEdward Cree if (channel->type->stop) 115685697f97SEdward Cree channel->type->stop(channel); 1157e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 1158e20ba5b1SAlex Maftei (amaftei) rx_queue->refill_enabled = false; 1159e20ba5b1SAlex Maftei (amaftei) } 1160e20ba5b1SAlex Maftei (amaftei) 1161e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1162e20ba5b1SAlex Maftei (amaftei) /* RX packet processing is pipelined, so wait for the 1163e20ba5b1SAlex Maftei (amaftei) * NAPI handler to complete. At least event queue 0 1164e20ba5b1SAlex Maftei (amaftei) * might be kept active by non-data events, so don't 1165e20ba5b1SAlex Maftei (amaftei) * use napi_synchronize() but actually disable NAPI 1166e20ba5b1SAlex Maftei (amaftei) * temporarily. 1167e20ba5b1SAlex Maftei (amaftei) */ 1168e20ba5b1SAlex Maftei (amaftei) if (efx_channel_has_rx_queue(channel)) { 1169e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 1170e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 1171e20ba5b1SAlex Maftei (amaftei) } 1172e20ba5b1SAlex Maftei (amaftei) } 1173e20ba5b1SAlex Maftei (amaftei) 1174b5775b47SAlex Maftei (amaftei) if (efx->type->fini_dmaq) 1175e20ba5b1SAlex Maftei (amaftei) rc = efx->type->fini_dmaq(efx); 1176b5775b47SAlex Maftei (amaftei) 1177e20ba5b1SAlex Maftei (amaftei) if (rc) { 1178e20ba5b1SAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 1179e20ba5b1SAlex Maftei (amaftei) } else { 1180e20ba5b1SAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 1181e20ba5b1SAlex Maftei (amaftei) "successfully flushed all queues\n"); 1182e20ba5b1SAlex Maftei (amaftei) } 1183e20ba5b1SAlex Maftei (amaftei) 1184e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 1185e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 1186e20ba5b1SAlex Maftei (amaftei) efx_fini_rx_queue(rx_queue); 1187f9cac93eSEdward Cree efx_for_each_channel_tx_queue(tx_queue, channel) 1188e20ba5b1SAlex Maftei (amaftei) efx_fini_tx_queue(tx_queue); 1189e20ba5b1SAlex Maftei (amaftei) } 1190e20ba5b1SAlex Maftei (amaftei) } 1191e20ba5b1SAlex Maftei (amaftei) 1192768fd266SAlex Maftei (amaftei) /************************************************************************** 1193768fd266SAlex Maftei (amaftei) * 1194768fd266SAlex Maftei (amaftei) * NAPI interface 1195768fd266SAlex Maftei (amaftei) * 1196768fd266SAlex Maftei (amaftei) *************************************************************************/ 1197768fd266SAlex Maftei (amaftei) 1198768fd266SAlex Maftei (amaftei) /* Process channel's event queue 1199768fd266SAlex Maftei (amaftei) * 1200768fd266SAlex Maftei (amaftei) * This function is responsible for processing the event queue of a 1201768fd266SAlex Maftei (amaftei) * single channel. The caller must guarantee that this function will 1202768fd266SAlex Maftei (amaftei) * never be concurrently called more than once on the same channel, 1203768fd266SAlex Maftei (amaftei) * though different channels may be being processed concurrently. 1204768fd266SAlex Maftei (amaftei) */ 1205768fd266SAlex Maftei (amaftei) static int efx_process_channel(struct efx_channel *channel, int budget) 1206768fd266SAlex Maftei (amaftei) { 1207768fd266SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 1208768fd266SAlex Maftei (amaftei) struct list_head rx_list; 1209768fd266SAlex Maftei (amaftei) int spent; 1210768fd266SAlex Maftei (amaftei) 1211768fd266SAlex Maftei (amaftei) if (unlikely(!channel->enabled)) 1212768fd266SAlex Maftei (amaftei) return 0; 1213768fd266SAlex Maftei (amaftei) 1214768fd266SAlex Maftei (amaftei) /* Prepare the batch receive list */ 1215768fd266SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(channel->rx_list != NULL); 1216768fd266SAlex Maftei (amaftei) INIT_LIST_HEAD(&rx_list); 1217768fd266SAlex Maftei (amaftei) channel->rx_list = &rx_list; 1218768fd266SAlex Maftei (amaftei) 1219768fd266SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 1220768fd266SAlex Maftei (amaftei) tx_queue->pkts_compl = 0; 1221768fd266SAlex Maftei (amaftei) tx_queue->bytes_compl = 0; 1222768fd266SAlex Maftei (amaftei) } 1223768fd266SAlex Maftei (amaftei) 1224768fd266SAlex Maftei (amaftei) spent = efx_nic_process_eventq(channel, budget); 1225768fd266SAlex Maftei (amaftei) if (spent && efx_channel_has_rx_queue(channel)) { 1226768fd266SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = 1227768fd266SAlex Maftei (amaftei) efx_channel_get_rx_queue(channel); 1228768fd266SAlex Maftei (amaftei) 1229768fd266SAlex Maftei (amaftei) efx_rx_flush_packet(channel); 1230768fd266SAlex Maftei (amaftei) efx_fast_push_rx_descriptors(rx_queue, true); 1231768fd266SAlex Maftei (amaftei) } 1232768fd266SAlex Maftei (amaftei) 1233768fd266SAlex Maftei (amaftei) /* Update BQL */ 1234768fd266SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 1235768fd266SAlex Maftei (amaftei) if (tx_queue->bytes_compl) { 1236768fd266SAlex Maftei (amaftei) netdev_tx_completed_queue(tx_queue->core_txq, 1237768fd266SAlex Maftei (amaftei) tx_queue->pkts_compl, 1238768fd266SAlex Maftei (amaftei) tx_queue->bytes_compl); 1239768fd266SAlex Maftei (amaftei) } 1240768fd266SAlex Maftei (amaftei) } 1241768fd266SAlex Maftei (amaftei) 1242768fd266SAlex Maftei (amaftei) /* Receive any packets we queued up */ 1243768fd266SAlex Maftei (amaftei) netif_receive_skb_list(channel->rx_list); 1244768fd266SAlex Maftei (amaftei) channel->rx_list = NULL; 1245768fd266SAlex Maftei (amaftei) 1246768fd266SAlex Maftei (amaftei) return spent; 1247768fd266SAlex Maftei (amaftei) } 1248768fd266SAlex Maftei (amaftei) 1249768fd266SAlex Maftei (amaftei) static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 1250768fd266SAlex Maftei (amaftei) { 1251768fd266SAlex Maftei (amaftei) int step = efx->irq_mod_step_us; 1252768fd266SAlex Maftei (amaftei) 1253768fd266SAlex Maftei (amaftei) if (channel->irq_mod_score < irq_adapt_low_thresh) { 1254768fd266SAlex Maftei (amaftei) if (channel->irq_moderation_us > step) { 1255768fd266SAlex Maftei (amaftei) channel->irq_moderation_us -= step; 1256768fd266SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 1257768fd266SAlex Maftei (amaftei) } 1258768fd266SAlex Maftei (amaftei) } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 1259768fd266SAlex Maftei (amaftei) if (channel->irq_moderation_us < 1260768fd266SAlex Maftei (amaftei) efx->irq_rx_moderation_us) { 1261768fd266SAlex Maftei (amaftei) channel->irq_moderation_us += step; 1262768fd266SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 1263768fd266SAlex Maftei (amaftei) } 1264768fd266SAlex Maftei (amaftei) } 1265768fd266SAlex Maftei (amaftei) 1266768fd266SAlex Maftei (amaftei) channel->irq_count = 0; 1267768fd266SAlex Maftei (amaftei) channel->irq_mod_score = 0; 1268768fd266SAlex Maftei (amaftei) } 1269768fd266SAlex Maftei (amaftei) 1270768fd266SAlex Maftei (amaftei) /* NAPI poll handler 1271768fd266SAlex Maftei (amaftei) * 1272768fd266SAlex Maftei (amaftei) * NAPI guarantees serialisation of polls of the same device, which 1273768fd266SAlex Maftei (amaftei) * provides the guarantee required by efx_process_channel(). 1274768fd266SAlex Maftei (amaftei) */ 1275768fd266SAlex Maftei (amaftei) static int efx_poll(struct napi_struct *napi, int budget) 1276768fd266SAlex Maftei (amaftei) { 1277768fd266SAlex Maftei (amaftei) struct efx_channel *channel = 1278768fd266SAlex Maftei (amaftei) container_of(napi, struct efx_channel, napi_str); 1279768fd266SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 1280b7683155SEdward Cree #ifdef CONFIG_RFS_ACCEL 1281b7683155SEdward Cree unsigned int time; 1282b7683155SEdward Cree #endif 1283768fd266SAlex Maftei (amaftei) int spent; 1284768fd266SAlex Maftei (amaftei) 1285768fd266SAlex Maftei (amaftei) netif_vdbg(efx, intr, efx->net_dev, 1286768fd266SAlex Maftei (amaftei) "channel %d NAPI poll executing on CPU %d\n", 1287768fd266SAlex Maftei (amaftei) channel->channel, raw_smp_processor_id()); 1288768fd266SAlex Maftei (amaftei) 1289768fd266SAlex Maftei (amaftei) spent = efx_process_channel(channel, budget); 1290768fd266SAlex Maftei (amaftei) 1291768fd266SAlex Maftei (amaftei) xdp_do_flush_map(); 1292768fd266SAlex Maftei (amaftei) 1293768fd266SAlex Maftei (amaftei) if (spent < budget) { 1294768fd266SAlex Maftei (amaftei) if (efx_channel_has_rx_queue(channel) && 1295768fd266SAlex Maftei (amaftei) efx->irq_rx_adaptive && 1296768fd266SAlex Maftei (amaftei) unlikely(++channel->irq_count == 1000)) { 1297768fd266SAlex Maftei (amaftei) efx_update_irq_mod(efx, channel); 1298768fd266SAlex Maftei (amaftei) } 1299768fd266SAlex Maftei (amaftei) 1300768fd266SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 1301768fd266SAlex Maftei (amaftei) /* Perhaps expire some ARFS filters */ 1302b7683155SEdward Cree time = jiffies - channel->rfs_last_expiry; 1303b7683155SEdward Cree /* Would our quota be >= 20? */ 1304b7683155SEdward Cree if (channel->rfs_filter_count * time >= 600 * HZ) 1305768fd266SAlex Maftei (amaftei) mod_delayed_work(system_wq, &channel->filter_work, 0); 1306768fd266SAlex Maftei (amaftei) #endif 1307768fd266SAlex Maftei (amaftei) 1308768fd266SAlex Maftei (amaftei) /* There is no race here; although napi_disable() will 1309768fd266SAlex Maftei (amaftei) * only wait for napi_complete(), this isn't a problem 1310768fd266SAlex Maftei (amaftei) * since efx_nic_eventq_read_ack() will have no effect if 1311768fd266SAlex Maftei (amaftei) * interrupts have already been disabled. 1312768fd266SAlex Maftei (amaftei) */ 1313768fd266SAlex Maftei (amaftei) if (napi_complete_done(napi, spent)) 1314768fd266SAlex Maftei (amaftei) efx_nic_eventq_read_ack(channel); 1315768fd266SAlex Maftei (amaftei) } 1316768fd266SAlex Maftei (amaftei) 1317768fd266SAlex Maftei (amaftei) return spent; 1318768fd266SAlex Maftei (amaftei) } 1319768fd266SAlex Maftei (amaftei) 1320768fd266SAlex Maftei (amaftei) void efx_init_napi_channel(struct efx_channel *channel) 1321768fd266SAlex Maftei (amaftei) { 1322768fd266SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 1323768fd266SAlex Maftei (amaftei) 1324768fd266SAlex Maftei (amaftei) channel->napi_dev = efx->net_dev; 1325b48b89f9SJakub Kicinski netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll); 1326768fd266SAlex Maftei (amaftei) } 1327768fd266SAlex Maftei (amaftei) 1328768fd266SAlex Maftei (amaftei) void efx_init_napi(struct efx_nic *efx) 1329768fd266SAlex Maftei (amaftei) { 1330768fd266SAlex Maftei (amaftei) struct efx_channel *channel; 1331768fd266SAlex Maftei (amaftei) 1332768fd266SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 1333768fd266SAlex Maftei (amaftei) efx_init_napi_channel(channel); 1334768fd266SAlex Maftei (amaftei) } 1335768fd266SAlex Maftei (amaftei) 1336768fd266SAlex Maftei (amaftei) void efx_fini_napi_channel(struct efx_channel *channel) 1337768fd266SAlex Maftei (amaftei) { 1338768fd266SAlex Maftei (amaftei) if (channel->napi_dev) 1339768fd266SAlex Maftei (amaftei) netif_napi_del(&channel->napi_str); 1340768fd266SAlex Maftei (amaftei) 1341768fd266SAlex Maftei (amaftei) channel->napi_dev = NULL; 1342768fd266SAlex Maftei (amaftei) } 1343768fd266SAlex Maftei (amaftei) 1344768fd266SAlex Maftei (amaftei) void efx_fini_napi(struct efx_nic *efx) 1345768fd266SAlex Maftei (amaftei) { 1346768fd266SAlex Maftei (amaftei) struct efx_channel *channel; 1347768fd266SAlex Maftei (amaftei) 1348768fd266SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 1349768fd266SAlex Maftei (amaftei) efx_fini_napi_channel(channel); 1350768fd266SAlex Maftei (amaftei) } 135154fccfddSMartin Habets 135254fccfddSMartin Habets /*************** 135354fccfddSMartin Habets * Housekeeping 135454fccfddSMartin Habets ***************/ 135554fccfddSMartin Habets 135654fccfddSMartin Habets static int efx_channel_dummy_op_int(struct efx_channel *channel) 135754fccfddSMartin Habets { 135854fccfddSMartin Habets return 0; 135954fccfddSMartin Habets } 136054fccfddSMartin Habets 136154fccfddSMartin Habets void efx_channel_dummy_op_void(struct efx_channel *channel) 136254fccfddSMartin Habets { 136354fccfddSMartin Habets } 136454fccfddSMartin Habets 136554fccfddSMartin Habets static const struct efx_channel_type efx_default_channel_type = { 136654fccfddSMartin Habets .pre_probe = efx_channel_dummy_op_int, 136754fccfddSMartin Habets .post_remove = efx_channel_dummy_op_void, 136854fccfddSMartin Habets .get_name = efx_get_channel_name, 136954fccfddSMartin Habets .copy = efx_copy_channel, 137054fccfddSMartin Habets .want_txqs = efx_default_channel_want_txqs, 137154fccfddSMartin Habets .keep_eventq = false, 137254fccfddSMartin Habets .want_pio = true, 137354fccfddSMartin Habets }; 1374