1768fd266SAlex Maftei (amaftei) // SPDX-License-Identifier: GPL-2.0-only 2768fd266SAlex Maftei (amaftei) /**************************************************************************** 3768fd266SAlex Maftei (amaftei) * Driver for Solarflare network controllers and boards 4768fd266SAlex Maftei (amaftei) * Copyright 2018 Solarflare Communications Inc. 5768fd266SAlex Maftei (amaftei) * 6768fd266SAlex Maftei (amaftei) * This program is free software; you can redistribute it and/or modify it 7768fd266SAlex Maftei (amaftei) * under the terms of the GNU General Public License version 2 as published 8768fd266SAlex Maftei (amaftei) * by the Free Software Foundation, incorporated herein by reference. 9768fd266SAlex Maftei (amaftei) */ 10768fd266SAlex Maftei (amaftei) 11768fd266SAlex Maftei (amaftei) #include "net_driver.h" 12768fd266SAlex Maftei (amaftei) #include <linux/module.h> 13768fd266SAlex Maftei (amaftei) #include "efx_channels.h" 14768fd266SAlex Maftei (amaftei) #include "efx.h" 15768fd266SAlex Maftei (amaftei) #include "efx_common.h" 16768fd266SAlex Maftei (amaftei) #include "tx_common.h" 17768fd266SAlex Maftei (amaftei) #include "rx_common.h" 18768fd266SAlex Maftei (amaftei) #include "nic.h" 19768fd266SAlex Maftei (amaftei) #include "sriov.h" 20768fd266SAlex Maftei (amaftei) 21768fd266SAlex Maftei (amaftei) static unsigned int irq_adapt_low_thresh = 8000; 22768fd266SAlex Maftei (amaftei) module_param(irq_adapt_low_thresh, uint, 0644); 23768fd266SAlex Maftei (amaftei) MODULE_PARM_DESC(irq_adapt_low_thresh, 24768fd266SAlex Maftei (amaftei) "Threshold score for reducing IRQ moderation"); 25768fd266SAlex Maftei (amaftei) 26768fd266SAlex Maftei (amaftei) static unsigned int irq_adapt_high_thresh = 16000; 27768fd266SAlex Maftei (amaftei) module_param(irq_adapt_high_thresh, uint, 0644); 28768fd266SAlex Maftei (amaftei) MODULE_PARM_DESC(irq_adapt_high_thresh, 29768fd266SAlex Maftei (amaftei) "Threshold score for increasing IRQ moderation"); 30768fd266SAlex Maftei (amaftei) 31768fd266SAlex Maftei (amaftei) /* This is the weight assigned to each of the (per-channel) virtual 32768fd266SAlex Maftei (amaftei) * NAPI devices. 33768fd266SAlex Maftei (amaftei) */ 34768fd266SAlex Maftei (amaftei) static int napi_weight = 64; 35768fd266SAlex Maftei (amaftei) 36e20ba5b1SAlex Maftei (amaftei) /************* 37e20ba5b1SAlex Maftei (amaftei) * START/STOP 38e20ba5b1SAlex Maftei (amaftei) *************/ 39e20ba5b1SAlex Maftei (amaftei) 40e20ba5b1SAlex Maftei (amaftei) int efx_soft_enable_interrupts(struct efx_nic *efx) 41e20ba5b1SAlex Maftei (amaftei) { 42e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel, *end_channel; 43e20ba5b1SAlex Maftei (amaftei) int rc; 44e20ba5b1SAlex Maftei (amaftei) 45e20ba5b1SAlex Maftei (amaftei) BUG_ON(efx->state == STATE_DISABLED); 46e20ba5b1SAlex Maftei (amaftei) 47e20ba5b1SAlex Maftei (amaftei) efx->irq_soft_enabled = true; 48e20ba5b1SAlex Maftei (amaftei) smp_wmb(); 49e20ba5b1SAlex Maftei (amaftei) 50e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 51e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) { 52e20ba5b1SAlex Maftei (amaftei) rc = efx_init_eventq(channel); 53e20ba5b1SAlex Maftei (amaftei) if (rc) 54e20ba5b1SAlex Maftei (amaftei) goto fail; 55e20ba5b1SAlex Maftei (amaftei) } 56e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 57e20ba5b1SAlex Maftei (amaftei) } 58e20ba5b1SAlex Maftei (amaftei) 59e20ba5b1SAlex Maftei (amaftei) efx_mcdi_mode_event(efx); 60e20ba5b1SAlex Maftei (amaftei) 61e20ba5b1SAlex Maftei (amaftei) return 0; 62e20ba5b1SAlex Maftei (amaftei) fail: 63e20ba5b1SAlex Maftei (amaftei) end_channel = channel; 64e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 65e20ba5b1SAlex Maftei (amaftei) if (channel == end_channel) 66e20ba5b1SAlex Maftei (amaftei) break; 67e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 68e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) 69e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 70e20ba5b1SAlex Maftei (amaftei) } 71e20ba5b1SAlex Maftei (amaftei) 72e20ba5b1SAlex Maftei (amaftei) return rc; 73e20ba5b1SAlex Maftei (amaftei) } 74e20ba5b1SAlex Maftei (amaftei) 75e20ba5b1SAlex Maftei (amaftei) void efx_soft_disable_interrupts(struct efx_nic *efx) 76e20ba5b1SAlex Maftei (amaftei) { 77e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 78e20ba5b1SAlex Maftei (amaftei) 79e20ba5b1SAlex Maftei (amaftei) if (efx->state == STATE_DISABLED) 80e20ba5b1SAlex Maftei (amaftei) return; 81e20ba5b1SAlex Maftei (amaftei) 82e20ba5b1SAlex Maftei (amaftei) efx_mcdi_mode_poll(efx); 83e20ba5b1SAlex Maftei (amaftei) 84e20ba5b1SAlex Maftei (amaftei) efx->irq_soft_enabled = false; 85e20ba5b1SAlex Maftei (amaftei) smp_wmb(); 86e20ba5b1SAlex Maftei (amaftei) 87e20ba5b1SAlex Maftei (amaftei) if (efx->legacy_irq) 88e20ba5b1SAlex Maftei (amaftei) synchronize_irq(efx->legacy_irq); 89e20ba5b1SAlex Maftei (amaftei) 90e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 91e20ba5b1SAlex Maftei (amaftei) if (channel->irq) 92e20ba5b1SAlex Maftei (amaftei) synchronize_irq(channel->irq); 93e20ba5b1SAlex Maftei (amaftei) 94e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 95e20ba5b1SAlex Maftei (amaftei) if (!channel->type->keep_eventq) 96e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 97e20ba5b1SAlex Maftei (amaftei) } 98e20ba5b1SAlex Maftei (amaftei) 99e20ba5b1SAlex Maftei (amaftei) /* Flush the asynchronous MCDI request queue */ 100e20ba5b1SAlex Maftei (amaftei) efx_mcdi_flush_async(efx); 101e20ba5b1SAlex Maftei (amaftei) } 102e20ba5b1SAlex Maftei (amaftei) 103e20ba5b1SAlex Maftei (amaftei) int efx_enable_interrupts(struct efx_nic *efx) 104e20ba5b1SAlex Maftei (amaftei) { 105e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel, *end_channel; 106e20ba5b1SAlex Maftei (amaftei) int rc; 107e20ba5b1SAlex Maftei (amaftei) 108e20ba5b1SAlex Maftei (amaftei) /* TODO: Is this really a bug? */ 109e20ba5b1SAlex Maftei (amaftei) BUG_ON(efx->state == STATE_DISABLED); 110e20ba5b1SAlex Maftei (amaftei) 111e20ba5b1SAlex Maftei (amaftei) if (efx->eeh_disabled_legacy_irq) { 112e20ba5b1SAlex Maftei (amaftei) enable_irq(efx->legacy_irq); 113e20ba5b1SAlex Maftei (amaftei) efx->eeh_disabled_legacy_irq = false; 114e20ba5b1SAlex Maftei (amaftei) } 115e20ba5b1SAlex Maftei (amaftei) 116e20ba5b1SAlex Maftei (amaftei) efx->type->irq_enable_master(efx); 117e20ba5b1SAlex Maftei (amaftei) 118e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 119e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) { 120e20ba5b1SAlex Maftei (amaftei) rc = efx_init_eventq(channel); 121e20ba5b1SAlex Maftei (amaftei) if (rc) 122e20ba5b1SAlex Maftei (amaftei) goto fail; 123e20ba5b1SAlex Maftei (amaftei) } 124e20ba5b1SAlex Maftei (amaftei) } 125e20ba5b1SAlex Maftei (amaftei) 126e20ba5b1SAlex Maftei (amaftei) rc = efx_soft_enable_interrupts(efx); 127e20ba5b1SAlex Maftei (amaftei) if (rc) 128e20ba5b1SAlex Maftei (amaftei) goto fail; 129e20ba5b1SAlex Maftei (amaftei) 130e20ba5b1SAlex Maftei (amaftei) return 0; 131e20ba5b1SAlex Maftei (amaftei) 132e20ba5b1SAlex Maftei (amaftei) fail: 133e20ba5b1SAlex Maftei (amaftei) end_channel = channel; 134e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 135e20ba5b1SAlex Maftei (amaftei) if (channel == end_channel) 136e20ba5b1SAlex Maftei (amaftei) break; 137e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) 138e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 139e20ba5b1SAlex Maftei (amaftei) } 140e20ba5b1SAlex Maftei (amaftei) 141e20ba5b1SAlex Maftei (amaftei) efx->type->irq_disable_non_ev(efx); 142e20ba5b1SAlex Maftei (amaftei) 143e20ba5b1SAlex Maftei (amaftei) return rc; 144e20ba5b1SAlex Maftei (amaftei) } 145e20ba5b1SAlex Maftei (amaftei) 146e20ba5b1SAlex Maftei (amaftei) void efx_disable_interrupts(struct efx_nic *efx) 147e20ba5b1SAlex Maftei (amaftei) { 148e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 149e20ba5b1SAlex Maftei (amaftei) 150e20ba5b1SAlex Maftei (amaftei) efx_soft_disable_interrupts(efx); 151e20ba5b1SAlex Maftei (amaftei) 152e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 153e20ba5b1SAlex Maftei (amaftei) if (channel->type->keep_eventq) 154e20ba5b1SAlex Maftei (amaftei) efx_fini_eventq(channel); 155e20ba5b1SAlex Maftei (amaftei) } 156e20ba5b1SAlex Maftei (amaftei) 157e20ba5b1SAlex Maftei (amaftei) efx->type->irq_disable_non_ev(efx); 158e20ba5b1SAlex Maftei (amaftei) } 159e20ba5b1SAlex Maftei (amaftei) 160e20ba5b1SAlex Maftei (amaftei) void efx_start_channels(struct efx_nic *efx) 161e20ba5b1SAlex Maftei (amaftei) { 162e20ba5b1SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 163e20ba5b1SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 164e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 165e20ba5b1SAlex Maftei (amaftei) 166e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 167e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 168e20ba5b1SAlex Maftei (amaftei) efx_init_tx_queue(tx_queue); 169e20ba5b1SAlex Maftei (amaftei) atomic_inc(&efx->active_queues); 170e20ba5b1SAlex Maftei (amaftei) } 171e20ba5b1SAlex Maftei (amaftei) 172e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) { 173e20ba5b1SAlex Maftei (amaftei) efx_init_rx_queue(rx_queue); 174e20ba5b1SAlex Maftei (amaftei) atomic_inc(&efx->active_queues); 175e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 176e20ba5b1SAlex Maftei (amaftei) efx_fast_push_rx_descriptors(rx_queue, false); 177e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 178e20ba5b1SAlex Maftei (amaftei) } 179e20ba5b1SAlex Maftei (amaftei) 180e20ba5b1SAlex Maftei (amaftei) WARN_ON(channel->rx_pkt_n_frags); 181e20ba5b1SAlex Maftei (amaftei) } 182e20ba5b1SAlex Maftei (amaftei) } 183e20ba5b1SAlex Maftei (amaftei) 184e20ba5b1SAlex Maftei (amaftei) void efx_stop_channels(struct efx_nic *efx) 185e20ba5b1SAlex Maftei (amaftei) { 186e20ba5b1SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 187e20ba5b1SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue; 188e20ba5b1SAlex Maftei (amaftei) struct efx_channel *channel; 189e20ba5b1SAlex Maftei (amaftei) int rc; 190e20ba5b1SAlex Maftei (amaftei) 191e20ba5b1SAlex Maftei (amaftei) /* Stop RX refill */ 192e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 193e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 194e20ba5b1SAlex Maftei (amaftei) rx_queue->refill_enabled = false; 195e20ba5b1SAlex Maftei (amaftei) } 196e20ba5b1SAlex Maftei (amaftei) 197e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 198e20ba5b1SAlex Maftei (amaftei) /* RX packet processing is pipelined, so wait for the 199e20ba5b1SAlex Maftei (amaftei) * NAPI handler to complete. At least event queue 0 200e20ba5b1SAlex Maftei (amaftei) * might be kept active by non-data events, so don't 201e20ba5b1SAlex Maftei (amaftei) * use napi_synchronize() but actually disable NAPI 202e20ba5b1SAlex Maftei (amaftei) * temporarily. 203e20ba5b1SAlex Maftei (amaftei) */ 204e20ba5b1SAlex Maftei (amaftei) if (efx_channel_has_rx_queue(channel)) { 205e20ba5b1SAlex Maftei (amaftei) efx_stop_eventq(channel); 206e20ba5b1SAlex Maftei (amaftei) efx_start_eventq(channel); 207e20ba5b1SAlex Maftei (amaftei) } 208e20ba5b1SAlex Maftei (amaftei) } 209e20ba5b1SAlex Maftei (amaftei) 210e20ba5b1SAlex Maftei (amaftei) rc = efx->type->fini_dmaq(efx); 211e20ba5b1SAlex Maftei (amaftei) if (rc) { 212e20ba5b1SAlex Maftei (amaftei) netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 213e20ba5b1SAlex Maftei (amaftei) } else { 214e20ba5b1SAlex Maftei (amaftei) netif_dbg(efx, drv, efx->net_dev, 215e20ba5b1SAlex Maftei (amaftei) "successfully flushed all queues\n"); 216e20ba5b1SAlex Maftei (amaftei) } 217e20ba5b1SAlex Maftei (amaftei) 218e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) { 219e20ba5b1SAlex Maftei (amaftei) efx_for_each_channel_rx_queue(rx_queue, channel) 220e20ba5b1SAlex Maftei (amaftei) efx_fini_rx_queue(rx_queue); 221e20ba5b1SAlex Maftei (amaftei) efx_for_each_possible_channel_tx_queue(tx_queue, channel) 222e20ba5b1SAlex Maftei (amaftei) efx_fini_tx_queue(tx_queue); 223e20ba5b1SAlex Maftei (amaftei) } 224e20ba5b1SAlex Maftei (amaftei) } 225e20ba5b1SAlex Maftei (amaftei) 226768fd266SAlex Maftei (amaftei) /************************************************************************** 227768fd266SAlex Maftei (amaftei) * 228768fd266SAlex Maftei (amaftei) * NAPI interface 229768fd266SAlex Maftei (amaftei) * 230768fd266SAlex Maftei (amaftei) *************************************************************************/ 231768fd266SAlex Maftei (amaftei) 232768fd266SAlex Maftei (amaftei) /* Process channel's event queue 233768fd266SAlex Maftei (amaftei) * 234768fd266SAlex Maftei (amaftei) * This function is responsible for processing the event queue of a 235768fd266SAlex Maftei (amaftei) * single channel. The caller must guarantee that this function will 236768fd266SAlex Maftei (amaftei) * never be concurrently called more than once on the same channel, 237768fd266SAlex Maftei (amaftei) * though different channels may be being processed concurrently. 238768fd266SAlex Maftei (amaftei) */ 239768fd266SAlex Maftei (amaftei) static int efx_process_channel(struct efx_channel *channel, int budget) 240768fd266SAlex Maftei (amaftei) { 241768fd266SAlex Maftei (amaftei) struct efx_tx_queue *tx_queue; 242768fd266SAlex Maftei (amaftei) struct list_head rx_list; 243768fd266SAlex Maftei (amaftei) int spent; 244768fd266SAlex Maftei (amaftei) 245768fd266SAlex Maftei (amaftei) if (unlikely(!channel->enabled)) 246768fd266SAlex Maftei (amaftei) return 0; 247768fd266SAlex Maftei (amaftei) 248768fd266SAlex Maftei (amaftei) /* Prepare the batch receive list */ 249768fd266SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(channel->rx_list != NULL); 250768fd266SAlex Maftei (amaftei) INIT_LIST_HEAD(&rx_list); 251768fd266SAlex Maftei (amaftei) channel->rx_list = &rx_list; 252768fd266SAlex Maftei (amaftei) 253768fd266SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 254768fd266SAlex Maftei (amaftei) tx_queue->pkts_compl = 0; 255768fd266SAlex Maftei (amaftei) tx_queue->bytes_compl = 0; 256768fd266SAlex Maftei (amaftei) } 257768fd266SAlex Maftei (amaftei) 258768fd266SAlex Maftei (amaftei) spent = efx_nic_process_eventq(channel, budget); 259768fd266SAlex Maftei (amaftei) if (spent && efx_channel_has_rx_queue(channel)) { 260768fd266SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = 261768fd266SAlex Maftei (amaftei) efx_channel_get_rx_queue(channel); 262768fd266SAlex Maftei (amaftei) 263768fd266SAlex Maftei (amaftei) efx_rx_flush_packet(channel); 264768fd266SAlex Maftei (amaftei) efx_fast_push_rx_descriptors(rx_queue, true); 265768fd266SAlex Maftei (amaftei) } 266768fd266SAlex Maftei (amaftei) 267768fd266SAlex Maftei (amaftei) /* Update BQL */ 268768fd266SAlex Maftei (amaftei) efx_for_each_channel_tx_queue(tx_queue, channel) { 269768fd266SAlex Maftei (amaftei) if (tx_queue->bytes_compl) { 270768fd266SAlex Maftei (amaftei) netdev_tx_completed_queue(tx_queue->core_txq, 271768fd266SAlex Maftei (amaftei) tx_queue->pkts_compl, 272768fd266SAlex Maftei (amaftei) tx_queue->bytes_compl); 273768fd266SAlex Maftei (amaftei) } 274768fd266SAlex Maftei (amaftei) } 275768fd266SAlex Maftei (amaftei) 276768fd266SAlex Maftei (amaftei) /* Receive any packets we queued up */ 277768fd266SAlex Maftei (amaftei) netif_receive_skb_list(channel->rx_list); 278768fd266SAlex Maftei (amaftei) channel->rx_list = NULL; 279768fd266SAlex Maftei (amaftei) 280768fd266SAlex Maftei (amaftei) return spent; 281768fd266SAlex Maftei (amaftei) } 282768fd266SAlex Maftei (amaftei) 283768fd266SAlex Maftei (amaftei) static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 284768fd266SAlex Maftei (amaftei) { 285768fd266SAlex Maftei (amaftei) int step = efx->irq_mod_step_us; 286768fd266SAlex Maftei (amaftei) 287768fd266SAlex Maftei (amaftei) if (channel->irq_mod_score < irq_adapt_low_thresh) { 288768fd266SAlex Maftei (amaftei) if (channel->irq_moderation_us > step) { 289768fd266SAlex Maftei (amaftei) channel->irq_moderation_us -= step; 290768fd266SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 291768fd266SAlex Maftei (amaftei) } 292768fd266SAlex Maftei (amaftei) } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 293768fd266SAlex Maftei (amaftei) if (channel->irq_moderation_us < 294768fd266SAlex Maftei (amaftei) efx->irq_rx_moderation_us) { 295768fd266SAlex Maftei (amaftei) channel->irq_moderation_us += step; 296768fd266SAlex Maftei (amaftei) efx->type->push_irq_moderation(channel); 297768fd266SAlex Maftei (amaftei) } 298768fd266SAlex Maftei (amaftei) } 299768fd266SAlex Maftei (amaftei) 300768fd266SAlex Maftei (amaftei) channel->irq_count = 0; 301768fd266SAlex Maftei (amaftei) channel->irq_mod_score = 0; 302768fd266SAlex Maftei (amaftei) } 303768fd266SAlex Maftei (amaftei) 304768fd266SAlex Maftei (amaftei) /* NAPI poll handler 305768fd266SAlex Maftei (amaftei) * 306768fd266SAlex Maftei (amaftei) * NAPI guarantees serialisation of polls of the same device, which 307768fd266SAlex Maftei (amaftei) * provides the guarantee required by efx_process_channel(). 308768fd266SAlex Maftei (amaftei) */ 309768fd266SAlex Maftei (amaftei) static int efx_poll(struct napi_struct *napi, int budget) 310768fd266SAlex Maftei (amaftei) { 311768fd266SAlex Maftei (amaftei) struct efx_channel *channel = 312768fd266SAlex Maftei (amaftei) container_of(napi, struct efx_channel, napi_str); 313768fd266SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 314768fd266SAlex Maftei (amaftei) int spent; 315768fd266SAlex Maftei (amaftei) 316768fd266SAlex Maftei (amaftei) netif_vdbg(efx, intr, efx->net_dev, 317768fd266SAlex Maftei (amaftei) "channel %d NAPI poll executing on CPU %d\n", 318768fd266SAlex Maftei (amaftei) channel->channel, raw_smp_processor_id()); 319768fd266SAlex Maftei (amaftei) 320768fd266SAlex Maftei (amaftei) spent = efx_process_channel(channel, budget); 321768fd266SAlex Maftei (amaftei) 322768fd266SAlex Maftei (amaftei) xdp_do_flush_map(); 323768fd266SAlex Maftei (amaftei) 324768fd266SAlex Maftei (amaftei) if (spent < budget) { 325768fd266SAlex Maftei (amaftei) if (efx_channel_has_rx_queue(channel) && 326768fd266SAlex Maftei (amaftei) efx->irq_rx_adaptive && 327768fd266SAlex Maftei (amaftei) unlikely(++channel->irq_count == 1000)) { 328768fd266SAlex Maftei (amaftei) efx_update_irq_mod(efx, channel); 329768fd266SAlex Maftei (amaftei) } 330768fd266SAlex Maftei (amaftei) 331768fd266SAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL 332768fd266SAlex Maftei (amaftei) /* Perhaps expire some ARFS filters */ 333768fd266SAlex Maftei (amaftei) mod_delayed_work(system_wq, &channel->filter_work, 0); 334768fd266SAlex Maftei (amaftei) #endif 335768fd266SAlex Maftei (amaftei) 336768fd266SAlex Maftei (amaftei) /* There is no race here; although napi_disable() will 337768fd266SAlex Maftei (amaftei) * only wait for napi_complete(), this isn't a problem 338768fd266SAlex Maftei (amaftei) * since efx_nic_eventq_read_ack() will have no effect if 339768fd266SAlex Maftei (amaftei) * interrupts have already been disabled. 340768fd266SAlex Maftei (amaftei) */ 341768fd266SAlex Maftei (amaftei) if (napi_complete_done(napi, spent)) 342768fd266SAlex Maftei (amaftei) efx_nic_eventq_read_ack(channel); 343768fd266SAlex Maftei (amaftei) } 344768fd266SAlex Maftei (amaftei) 345768fd266SAlex Maftei (amaftei) return spent; 346768fd266SAlex Maftei (amaftei) } 347768fd266SAlex Maftei (amaftei) 348768fd266SAlex Maftei (amaftei) void efx_init_napi_channel(struct efx_channel *channel) 349768fd266SAlex Maftei (amaftei) { 350768fd266SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx; 351768fd266SAlex Maftei (amaftei) 352768fd266SAlex Maftei (amaftei) channel->napi_dev = efx->net_dev; 353768fd266SAlex Maftei (amaftei) netif_napi_add(channel->napi_dev, &channel->napi_str, 354768fd266SAlex Maftei (amaftei) efx_poll, napi_weight); 355768fd266SAlex Maftei (amaftei) } 356768fd266SAlex Maftei (amaftei) 357768fd266SAlex Maftei (amaftei) void efx_init_napi(struct efx_nic *efx) 358768fd266SAlex Maftei (amaftei) { 359768fd266SAlex Maftei (amaftei) struct efx_channel *channel; 360768fd266SAlex Maftei (amaftei) 361768fd266SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 362768fd266SAlex Maftei (amaftei) efx_init_napi_channel(channel); 363768fd266SAlex Maftei (amaftei) } 364768fd266SAlex Maftei (amaftei) 365768fd266SAlex Maftei (amaftei) void efx_fini_napi_channel(struct efx_channel *channel) 366768fd266SAlex Maftei (amaftei) { 367768fd266SAlex Maftei (amaftei) if (channel->napi_dev) 368768fd266SAlex Maftei (amaftei) netif_napi_del(&channel->napi_str); 369768fd266SAlex Maftei (amaftei) 370768fd266SAlex Maftei (amaftei) channel->napi_dev = NULL; 371768fd266SAlex Maftei (amaftei) } 372768fd266SAlex Maftei (amaftei) 373768fd266SAlex Maftei (amaftei) void efx_fini_napi(struct efx_nic *efx) 374768fd266SAlex Maftei (amaftei) { 375768fd266SAlex Maftei (amaftei) struct efx_channel *channel; 376768fd266SAlex Maftei (amaftei) 377768fd266SAlex Maftei (amaftei) efx_for_each_channel(channel, efx) 378768fd266SAlex Maftei (amaftei) efx_fini_napi_channel(channel); 379768fd266SAlex Maftei (amaftei) } 380