1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3  * Driver for Solarflare network controllers and boards
4  * Copyright 2018 Solarflare Communications Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation, incorporated herein by reference.
9  */
10 
11 #include "net_driver.h"
12 #include <linux/module.h>
13 #include <linux/filter.h>
14 #include "efx_channels.h"
15 #include "efx.h"
16 #include "efx_common.h"
17 #include "tx_common.h"
18 #include "rx_common.h"
19 #include "nic.h"
20 #include "sriov.h"
21 #include "workarounds.h"
22 
23 /* This is the first interrupt mode to try out of:
24  * 0 => MSI-X
25  * 1 => MSI
26  * 2 => legacy
27  */
28 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
29 
30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
31  * i.e. the number of CPUs among which we may distribute simultaneous
32  * interrupt handling.
33  *
34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
35  * The default (0) means to assign an interrupt to each core.
36  */
37 unsigned int rss_cpus;
38 
39 static unsigned int irq_adapt_low_thresh = 8000;
40 module_param(irq_adapt_low_thresh, uint, 0644);
41 MODULE_PARM_DESC(irq_adapt_low_thresh,
42 		 "Threshold score for reducing IRQ moderation");
43 
44 static unsigned int irq_adapt_high_thresh = 16000;
45 module_param(irq_adapt_high_thresh, uint, 0644);
46 MODULE_PARM_DESC(irq_adapt_high_thresh,
47 		 "Threshold score for increasing IRQ moderation");
48 
49 static const struct efx_channel_type efx_default_channel_type;
50 
51 /*************
52  * INTERRUPTS
53  *************/
54 
55 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
56 {
57 	cpumask_var_t filter_mask;
58 	unsigned int count;
59 	int cpu;
60 
61 	if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
62 		netif_warn(efx, probe, efx->net_dev,
63 			   "RSS disabled due to allocation failure\n");
64 		return 1;
65 	}
66 
67 	cpumask_copy(filter_mask, cpu_online_mask);
68 	if (local_node)
69 		cpumask_and(filter_mask, filter_mask,
70 			    cpumask_of_pcibus(efx->pci_dev->bus));
71 
72 	count = 0;
73 	for_each_cpu(cpu, filter_mask) {
74 		++count;
75 		cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
76 	}
77 
78 	free_cpumask_var(filter_mask);
79 
80 	return count;
81 }
82 
83 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
84 {
85 	unsigned int count;
86 
87 	if (rss_cpus) {
88 		count = rss_cpus;
89 	} else {
90 		count = count_online_cores(efx, true);
91 
92 		/* If no online CPUs in local node, fallback to any online CPUs */
93 		if (count == 0)
94 			count = count_online_cores(efx, false);
95 	}
96 
97 	if (count > EFX_MAX_RX_QUEUES) {
98 		netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
99 			       "Reducing number of rx queues from %u to %u.\n",
100 			       count, EFX_MAX_RX_QUEUES);
101 		count = EFX_MAX_RX_QUEUES;
102 	}
103 
104 	/* If RSS is requested for the PF *and* VFs then we can't write RSS
105 	 * table entries that are inaccessible to VFs
106 	 */
107 #ifdef CONFIG_SFC_SRIOV
108 	if (efx->type->sriov_wanted) {
109 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
110 		    count > efx_vf_size(efx)) {
111 			netif_warn(efx, probe, efx->net_dev,
112 				   "Reducing number of RSS channels from %u to %u for "
113 				   "VF support. Increase vf-msix-limit to use more "
114 				   "channels on the PF.\n",
115 				   count, efx_vf_size(efx));
116 			count = efx_vf_size(efx);
117 		}
118 	}
119 #endif
120 
121 	return count;
122 }
123 
124 static int efx_allocate_msix_channels(struct efx_nic *efx,
125 				      unsigned int max_channels,
126 				      unsigned int extra_channels,
127 				      unsigned int parallelism)
128 {
129 	unsigned int n_channels = parallelism;
130 	int vec_count;
131 	int tx_per_ev;
132 	int n_xdp_tx;
133 	int n_xdp_ev;
134 
135 	if (efx_separate_tx_channels)
136 		n_channels *= 2;
137 	n_channels += extra_channels;
138 
139 	/* To allow XDP transmit to happen from arbitrary NAPI contexts
140 	 * we allocate a TX queue per CPU. We share event queues across
141 	 * multiple tx queues, assuming tx and ev queues are both
142 	 * maximum size.
143 	 */
144 	tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
145 	tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
146 	n_xdp_tx = num_possible_cpus();
147 	n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
148 
149 	vec_count = pci_msix_vec_count(efx->pci_dev);
150 	if (vec_count < 0)
151 		return vec_count;
152 
153 	max_channels = min_t(unsigned int, vec_count, max_channels);
154 
155 	/* Check resources.
156 	 * We need a channel per event queue, plus a VI per tx queue.
157 	 * This may be more pessimistic than it needs to be.
158 	 */
159 	if (n_channels >= max_channels) {
160 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
161 		netif_warn(efx, drv, efx->net_dev,
162 			   "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
163 			   n_xdp_ev, n_channels, max_channels);
164 		netif_warn(efx, drv, efx->net_dev,
165 			   "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
166 	} else if (n_channels + n_xdp_tx > efx->max_vis) {
167 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
168 		netif_warn(efx, drv, efx->net_dev,
169 			   "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
170 			   n_xdp_tx, n_channels, efx->max_vis);
171 		netif_warn(efx, drv, efx->net_dev,
172 			   "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
173 	} else if (n_channels + n_xdp_ev > max_channels) {
174 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
175 		netif_warn(efx, drv, efx->net_dev,
176 			   "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
177 			   n_xdp_ev, n_channels, max_channels);
178 
179 		n_xdp_ev = max_channels - n_channels;
180 		netif_warn(efx, drv, efx->net_dev,
181 			   "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
182 			   DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
183 	} else {
184 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
185 	}
186 
187 	if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
188 		efx->n_xdp_channels = n_xdp_ev;
189 		efx->xdp_tx_per_channel = tx_per_ev;
190 		efx->xdp_tx_queue_count = n_xdp_tx;
191 		n_channels += n_xdp_ev;
192 		netif_dbg(efx, drv, efx->net_dev,
193 			  "Allocating %d TX and %d event queues for XDP\n",
194 			  n_xdp_ev * tx_per_ev, n_xdp_ev);
195 	} else {
196 		efx->n_xdp_channels = 0;
197 		efx->xdp_tx_per_channel = 0;
198 		efx->xdp_tx_queue_count = n_xdp_tx;
199 	}
200 
201 	if (vec_count < n_channels) {
202 		netif_err(efx, drv, efx->net_dev,
203 			  "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
204 			  vec_count, n_channels);
205 		netif_err(efx, drv, efx->net_dev,
206 			  "WARNING: Performance may be reduced.\n");
207 		n_channels = vec_count;
208 	}
209 
210 	n_channels = min(n_channels, max_channels);
211 
212 	efx->n_channels = n_channels;
213 
214 	/* Ignore XDP tx channels when creating rx channels. */
215 	n_channels -= efx->n_xdp_channels;
216 
217 	if (efx_separate_tx_channels) {
218 		efx->n_tx_channels =
219 			min(max(n_channels / 2, 1U),
220 			    efx->max_tx_channels);
221 		efx->tx_channel_offset =
222 			n_channels - efx->n_tx_channels;
223 		efx->n_rx_channels =
224 			max(n_channels -
225 			    efx->n_tx_channels, 1U);
226 	} else {
227 		efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
228 		efx->tx_channel_offset = 0;
229 		efx->n_rx_channels = n_channels;
230 	}
231 
232 	efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
233 	efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
234 
235 	efx->xdp_channel_offset = n_channels;
236 
237 	netif_dbg(efx, drv, efx->net_dev,
238 		  "Allocating %u RX channels\n",
239 		  efx->n_rx_channels);
240 
241 	return efx->n_channels;
242 }
243 
244 /* Probe the number and type of interrupts we are able to obtain, and
245  * the resulting numbers of channels and RX queues.
246  */
247 int efx_probe_interrupts(struct efx_nic *efx)
248 {
249 	unsigned int extra_channels = 0;
250 	unsigned int rss_spread;
251 	unsigned int i, j;
252 	int rc;
253 
254 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
255 		if (efx->extra_channel_type[i])
256 			++extra_channels;
257 
258 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
259 		unsigned int parallelism = efx_wanted_parallelism(efx);
260 		struct msix_entry xentries[EFX_MAX_CHANNELS];
261 		unsigned int n_channels;
262 
263 		rc = efx_allocate_msix_channels(efx, efx->max_channels,
264 						extra_channels, parallelism);
265 		if (rc >= 0) {
266 			n_channels = rc;
267 			for (i = 0; i < n_channels; i++)
268 				xentries[i].entry = i;
269 			rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
270 						   n_channels);
271 		}
272 		if (rc < 0) {
273 			/* Fall back to single channel MSI */
274 			netif_err(efx, drv, efx->net_dev,
275 				  "could not enable MSI-X\n");
276 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
277 				efx->interrupt_mode = EFX_INT_MODE_MSI;
278 			else
279 				return rc;
280 		} else if (rc < n_channels) {
281 			netif_err(efx, drv, efx->net_dev,
282 				  "WARNING: Insufficient MSI-X vectors"
283 				  " available (%d < %u).\n", rc, n_channels);
284 			netif_err(efx, drv, efx->net_dev,
285 				  "WARNING: Performance may be reduced.\n");
286 			n_channels = rc;
287 		}
288 
289 		if (rc > 0) {
290 			for (i = 0; i < efx->n_channels; i++)
291 				efx_get_channel(efx, i)->irq =
292 					xentries[i].vector;
293 		}
294 	}
295 
296 	/* Try single interrupt MSI */
297 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
298 		efx->n_channels = 1;
299 		efx->n_rx_channels = 1;
300 		efx->n_tx_channels = 1;
301 		efx->tx_channel_offset = 0;
302 		efx->n_xdp_channels = 0;
303 		efx->xdp_channel_offset = efx->n_channels;
304 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
305 		rc = pci_enable_msi(efx->pci_dev);
306 		if (rc == 0) {
307 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
308 		} else {
309 			netif_err(efx, drv, efx->net_dev,
310 				  "could not enable MSI\n");
311 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
312 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;
313 			else
314 				return rc;
315 		}
316 	}
317 
318 	/* Assume legacy interrupts */
319 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
320 		efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
321 		efx->n_rx_channels = 1;
322 		efx->n_tx_channels = 1;
323 		efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0;
324 		efx->n_xdp_channels = 0;
325 		efx->xdp_channel_offset = efx->n_channels;
326 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
327 		efx->legacy_irq = efx->pci_dev->irq;
328 	}
329 
330 	/* Assign extra channels if possible, before XDP channels */
331 	efx->n_extra_tx_channels = 0;
332 	j = efx->xdp_channel_offset;
333 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
334 		if (!efx->extra_channel_type[i])
335 			continue;
336 		if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
337 			efx->extra_channel_type[i]->handle_no_channel(efx);
338 		} else {
339 			--j;
340 			efx_get_channel(efx, j)->type =
341 				efx->extra_channel_type[i];
342 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
343 				efx->n_extra_tx_channels++;
344 		}
345 	}
346 
347 	rss_spread = efx->n_rx_channels;
348 	/* RSS might be usable on VFs even if it is disabled on the PF */
349 #ifdef CONFIG_SFC_SRIOV
350 	if (efx->type->sriov_wanted) {
351 		efx->rss_spread = ((rss_spread > 1 ||
352 				    !efx->type->sriov_wanted(efx)) ?
353 				   rss_spread : efx_vf_size(efx));
354 		return 0;
355 	}
356 #endif
357 	efx->rss_spread = rss_spread;
358 
359 	return 0;
360 }
361 
362 #if defined(CONFIG_SMP)
363 void efx_set_interrupt_affinity(struct efx_nic *efx)
364 {
365 	const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus);
366 	struct efx_channel *channel;
367 	unsigned int cpu;
368 
369 	/* If no online CPUs in local node, fallback to any online CPU */
370 	if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
371 		numa_mask = cpu_online_mask;
372 
373 	cpu = -1;
374 	efx_for_each_channel(channel, efx) {
375 		cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
376 		if (cpu >= nr_cpu_ids)
377 			cpu = cpumask_first_and(cpu_online_mask, numa_mask);
378 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
379 	}
380 }
381 
382 void efx_clear_interrupt_affinity(struct efx_nic *efx)
383 {
384 	struct efx_channel *channel;
385 
386 	efx_for_each_channel(channel, efx)
387 		irq_set_affinity_hint(channel->irq, NULL);
388 }
389 #else
390 void
391 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
392 {
393 }
394 
395 void
396 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
397 {
398 }
399 #endif /* CONFIG_SMP */
400 
401 void efx_remove_interrupts(struct efx_nic *efx)
402 {
403 	struct efx_channel *channel;
404 
405 	/* Remove MSI/MSI-X interrupts */
406 	efx_for_each_channel(channel, efx)
407 		channel->irq = 0;
408 	pci_disable_msi(efx->pci_dev);
409 	pci_disable_msix(efx->pci_dev);
410 
411 	/* Remove legacy interrupt */
412 	efx->legacy_irq = 0;
413 }
414 
415 /***************
416  * EVENT QUEUES
417  ***************/
418 
419 /* Create event queue
420  * Event queue memory allocations are done only once.  If the channel
421  * is reset, the memory buffer will be reused; this guards against
422  * errors during channel reset and also simplifies interrupt handling.
423  */
424 int efx_probe_eventq(struct efx_channel *channel)
425 {
426 	struct efx_nic *efx = channel->efx;
427 	unsigned long entries;
428 
429 	netif_dbg(efx, probe, efx->net_dev,
430 		  "chan %d create event queue\n", channel->channel);
431 
432 	/* Build an event queue with room for one event per tx and rx buffer,
433 	 * plus some extra for link state events and MCDI completions.
434 	 */
435 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
436 	EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
437 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
438 
439 	return efx_nic_probe_eventq(channel);
440 }
441 
442 /* Prepare channel's event queue */
443 int efx_init_eventq(struct efx_channel *channel)
444 {
445 	struct efx_nic *efx = channel->efx;
446 	int rc;
447 
448 	EFX_WARN_ON_PARANOID(channel->eventq_init);
449 
450 	netif_dbg(efx, drv, efx->net_dev,
451 		  "chan %d init event queue\n", channel->channel);
452 
453 	rc = efx_nic_init_eventq(channel);
454 	if (rc == 0) {
455 		efx->type->push_irq_moderation(channel);
456 		channel->eventq_read_ptr = 0;
457 		channel->eventq_init = true;
458 	}
459 	return rc;
460 }
461 
462 /* Enable event queue processing and NAPI */
463 void efx_start_eventq(struct efx_channel *channel)
464 {
465 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
466 		  "chan %d start event queue\n", channel->channel);
467 
468 	/* Make sure the NAPI handler sees the enabled flag set */
469 	channel->enabled = true;
470 	smp_wmb();
471 
472 	napi_enable(&channel->napi_str);
473 	efx_nic_eventq_read_ack(channel);
474 }
475 
476 /* Disable event queue processing and NAPI */
477 void efx_stop_eventq(struct efx_channel *channel)
478 {
479 	if (!channel->enabled)
480 		return;
481 
482 	napi_disable(&channel->napi_str);
483 	channel->enabled = false;
484 }
485 
486 void efx_fini_eventq(struct efx_channel *channel)
487 {
488 	if (!channel->eventq_init)
489 		return;
490 
491 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
492 		  "chan %d fini event queue\n", channel->channel);
493 
494 	efx_nic_fini_eventq(channel);
495 	channel->eventq_init = false;
496 }
497 
498 void efx_remove_eventq(struct efx_channel *channel)
499 {
500 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
501 		  "chan %d remove event queue\n", channel->channel);
502 
503 	efx_nic_remove_eventq(channel);
504 }
505 
506 /**************************************************************************
507  *
508  * Channel handling
509  *
510  *************************************************************************/
511 
512 #ifdef CONFIG_RFS_ACCEL
513 static void efx_filter_rfs_expire(struct work_struct *data)
514 {
515 	struct delayed_work *dwork = to_delayed_work(data);
516 	struct efx_channel *channel;
517 	unsigned int time, quota;
518 
519 	channel = container_of(dwork, struct efx_channel, filter_work);
520 	time = jiffies - channel->rfs_last_expiry;
521 	quota = channel->rfs_filter_count * time / (30 * HZ);
522 	if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
523 		channel->rfs_last_expiry += time;
524 	/* Ensure we do more work eventually even if NAPI poll is not happening */
525 	schedule_delayed_work(dwork, 30 * HZ);
526 }
527 #endif
528 
529 /* Allocate and initialise a channel structure. */
530 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
531 {
532 	struct efx_rx_queue *rx_queue;
533 	struct efx_tx_queue *tx_queue;
534 	struct efx_channel *channel;
535 	int j;
536 
537 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
538 	if (!channel)
539 		return NULL;
540 
541 	channel->efx = efx;
542 	channel->channel = i;
543 	channel->type = &efx_default_channel_type;
544 
545 	for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
546 		tx_queue = &channel->tx_queue[j];
547 		tx_queue->efx = efx;
548 		tx_queue->queue = -1;
549 		tx_queue->label = j;
550 		tx_queue->channel = channel;
551 	}
552 
553 #ifdef CONFIG_RFS_ACCEL
554 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
555 #endif
556 
557 	rx_queue = &channel->rx_queue;
558 	rx_queue->efx = efx;
559 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
560 
561 	return channel;
562 }
563 
564 int efx_init_channels(struct efx_nic *efx)
565 {
566 	unsigned int i;
567 
568 	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
569 		efx->channel[i] = efx_alloc_channel(efx, i);
570 		if (!efx->channel[i])
571 			return -ENOMEM;
572 		efx->msi_context[i].efx = efx;
573 		efx->msi_context[i].index = i;
574 	}
575 
576 	/* Higher numbered interrupt modes are less capable! */
577 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,
578 				  efx_interrupt_mode);
579 
580 	efx->max_channels = EFX_MAX_CHANNELS;
581 	efx->max_tx_channels = EFX_MAX_CHANNELS;
582 
583 	return 0;
584 }
585 
586 void efx_fini_channels(struct efx_nic *efx)
587 {
588 	unsigned int i;
589 
590 	for (i = 0; i < EFX_MAX_CHANNELS; i++)
591 		if (efx->channel[i]) {
592 			kfree(efx->channel[i]);
593 			efx->channel[i] = NULL;
594 		}
595 }
596 
597 /* Allocate and initialise a channel structure, copying parameters
598  * (but not resources) from an old channel structure.
599  */
600 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
601 {
602 	struct efx_rx_queue *rx_queue;
603 	struct efx_tx_queue *tx_queue;
604 	struct efx_channel *channel;
605 	int j;
606 
607 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
608 	if (!channel)
609 		return NULL;
610 
611 	*channel = *old_channel;
612 
613 	channel->napi_dev = NULL;
614 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
615 	channel->napi_str.napi_id = 0;
616 	channel->napi_str.state = 0;
617 	memset(&channel->eventq, 0, sizeof(channel->eventq));
618 
619 	for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
620 		tx_queue = &channel->tx_queue[j];
621 		if (tx_queue->channel)
622 			tx_queue->channel = channel;
623 		tx_queue->buffer = NULL;
624 		tx_queue->cb_page = NULL;
625 		memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
626 	}
627 
628 	rx_queue = &channel->rx_queue;
629 	rx_queue->buffer = NULL;
630 	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
631 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
632 #ifdef CONFIG_RFS_ACCEL
633 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
634 #endif
635 
636 	return channel;
637 }
638 
639 static int efx_probe_channel(struct efx_channel *channel)
640 {
641 	struct efx_tx_queue *tx_queue;
642 	struct efx_rx_queue *rx_queue;
643 	int rc;
644 
645 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
646 		  "creating channel %d\n", channel->channel);
647 
648 	rc = channel->type->pre_probe(channel);
649 	if (rc)
650 		goto fail;
651 
652 	rc = efx_probe_eventq(channel);
653 	if (rc)
654 		goto fail;
655 
656 	efx_for_each_channel_tx_queue(tx_queue, channel) {
657 		rc = efx_probe_tx_queue(tx_queue);
658 		if (rc)
659 			goto fail;
660 	}
661 
662 	efx_for_each_channel_rx_queue(rx_queue, channel) {
663 		rc = efx_probe_rx_queue(rx_queue);
664 		if (rc)
665 			goto fail;
666 	}
667 
668 	channel->rx_list = NULL;
669 
670 	return 0;
671 
672 fail:
673 	efx_remove_channel(channel);
674 	return rc;
675 }
676 
677 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
678 				 size_t len)
679 {
680 	struct efx_nic *efx = channel->efx;
681 	const char *type;
682 	int number;
683 
684 	number = channel->channel;
685 
686 	if (number >= efx->xdp_channel_offset &&
687 	    !WARN_ON_ONCE(!efx->n_xdp_channels)) {
688 		type = "-xdp";
689 		number -= efx->xdp_channel_offset;
690 	} else if (efx->tx_channel_offset == 0) {
691 		type = "";
692 	} else if (number < efx->tx_channel_offset) {
693 		type = "-rx";
694 	} else {
695 		type = "-tx";
696 		number -= efx->tx_channel_offset;
697 	}
698 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);
699 }
700 
701 void efx_set_channel_names(struct efx_nic *efx)
702 {
703 	struct efx_channel *channel;
704 
705 	efx_for_each_channel(channel, efx)
706 		channel->type->get_name(channel,
707 					efx->msi_context[channel->channel].name,
708 					sizeof(efx->msi_context[0].name));
709 }
710 
711 int efx_probe_channels(struct efx_nic *efx)
712 {
713 	struct efx_channel *channel;
714 	int rc;
715 
716 	/* Restart special buffer allocation */
717 	efx->next_buffer_table = 0;
718 
719 	/* Probe channels in reverse, so that any 'extra' channels
720 	 * use the start of the buffer table. This allows the traffic
721 	 * channels to be resized without moving them or wasting the
722 	 * entries before them.
723 	 */
724 	efx_for_each_channel_rev(channel, efx) {
725 		rc = efx_probe_channel(channel);
726 		if (rc) {
727 			netif_err(efx, probe, efx->net_dev,
728 				  "failed to create channel %d\n",
729 				  channel->channel);
730 			goto fail;
731 		}
732 	}
733 	efx_set_channel_names(efx);
734 
735 	return 0;
736 
737 fail:
738 	efx_remove_channels(efx);
739 	return rc;
740 }
741 
742 void efx_remove_channel(struct efx_channel *channel)
743 {
744 	struct efx_tx_queue *tx_queue;
745 	struct efx_rx_queue *rx_queue;
746 
747 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
748 		  "destroy chan %d\n", channel->channel);
749 
750 	efx_for_each_channel_rx_queue(rx_queue, channel)
751 		efx_remove_rx_queue(rx_queue);
752 	efx_for_each_channel_tx_queue(tx_queue, channel)
753 		efx_remove_tx_queue(tx_queue);
754 	efx_remove_eventq(channel);
755 	channel->type->post_remove(channel);
756 }
757 
758 void efx_remove_channels(struct efx_nic *efx)
759 {
760 	struct efx_channel *channel;
761 
762 	efx_for_each_channel(channel, efx)
763 		efx_remove_channel(channel);
764 
765 	kfree(efx->xdp_tx_queues);
766 }
767 
768 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
769 				struct efx_tx_queue *tx_queue)
770 {
771 	if (xdp_queue_number >= efx->xdp_tx_queue_count)
772 		return -EINVAL;
773 
774 	netif_dbg(efx, drv, efx->net_dev,
775 		  "Channel %u TXQ %u is XDP %u, HW %u\n",
776 		  tx_queue->channel->channel, tx_queue->label,
777 		  xdp_queue_number, tx_queue->queue);
778 	efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
779 	return 0;
780 }
781 
782 static void efx_set_xdp_channels(struct efx_nic *efx)
783 {
784 	struct efx_tx_queue *tx_queue;
785 	struct efx_channel *channel;
786 	unsigned int next_queue = 0;
787 	int xdp_queue_number = 0;
788 	int rc;
789 
790 	/* We need to mark which channels really have RX and TX
791 	 * queues, and adjust the TX queue numbers if we have separate
792 	 * RX-only and TX-only channels.
793 	 */
794 	efx_for_each_channel(channel, efx) {
795 		if (channel->channel < efx->tx_channel_offset)
796 			continue;
797 
798 		if (efx_channel_is_xdp_tx(channel)) {
799 			efx_for_each_channel_tx_queue(tx_queue, channel) {
800 				tx_queue->queue = next_queue++;
801 				rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
802 							  tx_queue);
803 				if (rc == 0)
804 					xdp_queue_number++;
805 			}
806 		} else {
807 			efx_for_each_channel_tx_queue(tx_queue, channel) {
808 				tx_queue->queue = next_queue++;
809 				netif_dbg(efx, drv, efx->net_dev,
810 					  "Channel %u TXQ %u is HW %u\n",
811 					  channel->channel, tx_queue->label,
812 					  tx_queue->queue);
813 			}
814 
815 			/* If XDP is borrowing queues from net stack, it must
816 			 * use the queue with no csum offload, which is the
817 			 * first one of the channel
818 			 * (note: tx_queue_by_type is not initialized yet)
819 			 */
820 			if (efx->xdp_txq_queues_mode ==
821 			    EFX_XDP_TX_QUEUES_BORROWED) {
822 				tx_queue = &channel->tx_queue[0];
823 				rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
824 							  tx_queue);
825 				if (rc == 0)
826 					xdp_queue_number++;
827 			}
828 		}
829 	}
830 	WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
831 		xdp_queue_number != efx->xdp_tx_queue_count);
832 	WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
833 		xdp_queue_number > efx->xdp_tx_queue_count);
834 
835 	/* If we have more CPUs than assigned XDP TX queues, assign the already
836 	 * existing queues to the exceeding CPUs
837 	 */
838 	next_queue = 0;
839 	while (xdp_queue_number < efx->xdp_tx_queue_count) {
840 		tx_queue = efx->xdp_tx_queues[next_queue++];
841 		rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
842 		if (rc == 0)
843 			xdp_queue_number++;
844 	}
845 }
846 
847 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
848 {
849 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
850 			   *ptp_channel = efx_ptp_channel(efx);
851 	struct efx_ptp_data *ptp_data = efx->ptp_data;
852 	unsigned int i, next_buffer_table = 0;
853 	u32 old_rxq_entries, old_txq_entries;
854 	int rc, rc2;
855 
856 	rc = efx_check_disabled(efx);
857 	if (rc)
858 		return rc;
859 
860 	/* Not all channels should be reallocated. We must avoid
861 	 * reallocating their buffer table entries.
862 	 */
863 	efx_for_each_channel(channel, efx) {
864 		struct efx_rx_queue *rx_queue;
865 		struct efx_tx_queue *tx_queue;
866 
867 		if (channel->type->copy)
868 			continue;
869 		next_buffer_table = max(next_buffer_table,
870 					channel->eventq.index +
871 					channel->eventq.entries);
872 		efx_for_each_channel_rx_queue(rx_queue, channel)
873 			next_buffer_table = max(next_buffer_table,
874 						rx_queue->rxd.index +
875 						rx_queue->rxd.entries);
876 		efx_for_each_channel_tx_queue(tx_queue, channel)
877 			next_buffer_table = max(next_buffer_table,
878 						tx_queue->txd.index +
879 						tx_queue->txd.entries);
880 	}
881 
882 	efx_device_detach_sync(efx);
883 	efx_stop_all(efx);
884 	efx_soft_disable_interrupts(efx);
885 
886 	/* Clone channels (where possible) */
887 	memset(other_channel, 0, sizeof(other_channel));
888 	for (i = 0; i < efx->n_channels; i++) {
889 		channel = efx->channel[i];
890 		if (channel->type->copy)
891 			channel = channel->type->copy(channel);
892 		if (!channel) {
893 			rc = -ENOMEM;
894 			goto out;
895 		}
896 		other_channel[i] = channel;
897 	}
898 
899 	/* Swap entry counts and channel pointers */
900 	old_rxq_entries = efx->rxq_entries;
901 	old_txq_entries = efx->txq_entries;
902 	efx->rxq_entries = rxq_entries;
903 	efx->txq_entries = txq_entries;
904 	for (i = 0; i < efx->n_channels; i++)
905 		swap(efx->channel[i], other_channel[i]);
906 
907 	/* Restart buffer table allocation */
908 	efx->next_buffer_table = next_buffer_table;
909 
910 	for (i = 0; i < efx->n_channels; i++) {
911 		channel = efx->channel[i];
912 		if (!channel->type->copy)
913 			continue;
914 		rc = efx_probe_channel(channel);
915 		if (rc)
916 			goto rollback;
917 		efx_init_napi_channel(efx->channel[i]);
918 	}
919 
920 	efx_set_xdp_channels(efx);
921 out:
922 	efx->ptp_data = NULL;
923 	/* Destroy unused channel structures */
924 	for (i = 0; i < efx->n_channels; i++) {
925 		channel = other_channel[i];
926 		if (channel && channel->type->copy) {
927 			efx_fini_napi_channel(channel);
928 			efx_remove_channel(channel);
929 			kfree(channel);
930 		}
931 	}
932 
933 	efx->ptp_data = ptp_data;
934 	rc2 = efx_soft_enable_interrupts(efx);
935 	if (rc2) {
936 		rc = rc ? rc : rc2;
937 		netif_err(efx, drv, efx->net_dev,
938 			  "unable to restart interrupts on channel reallocation\n");
939 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);
940 	} else {
941 		efx_start_all(efx);
942 		efx_device_attach_if_not_resetting(efx);
943 	}
944 	return rc;
945 
946 rollback:
947 	/* Swap back */
948 	efx->rxq_entries = old_rxq_entries;
949 	efx->txq_entries = old_txq_entries;
950 	for (i = 0; i < efx->n_channels; i++)
951 		swap(efx->channel[i], other_channel[i]);
952 	efx_ptp_update_channel(efx, ptp_channel);
953 	goto out;
954 }
955 
956 int efx_set_channels(struct efx_nic *efx)
957 {
958 	struct efx_channel *channel;
959 	int rc;
960 
961 	if (efx->xdp_tx_queue_count) {
962 		EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
963 
964 		/* Allocate array for XDP TX queue lookup. */
965 		efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
966 					     sizeof(*efx->xdp_tx_queues),
967 					     GFP_KERNEL);
968 		if (!efx->xdp_tx_queues)
969 			return -ENOMEM;
970 	}
971 
972 	efx_for_each_channel(channel, efx) {
973 		if (channel->channel < efx->n_rx_channels)
974 			channel->rx_queue.core_index = channel->channel;
975 		else
976 			channel->rx_queue.core_index = -1;
977 	}
978 
979 	efx_set_xdp_channels(efx);
980 
981 	rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
982 	if (rc)
983 		return rc;
984 	return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
985 }
986 
987 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
988 {
989 	return channel->channel - channel->efx->tx_channel_offset <
990 		channel->efx->n_tx_channels;
991 }
992 
993 /*************
994  * START/STOP
995  *************/
996 
997 int efx_soft_enable_interrupts(struct efx_nic *efx)
998 {
999 	struct efx_channel *channel, *end_channel;
1000 	int rc;
1001 
1002 	BUG_ON(efx->state == STATE_DISABLED);
1003 
1004 	efx->irq_soft_enabled = true;
1005 	smp_wmb();
1006 
1007 	efx_for_each_channel(channel, efx) {
1008 		if (!channel->type->keep_eventq) {
1009 			rc = efx_init_eventq(channel);
1010 			if (rc)
1011 				goto fail;
1012 		}
1013 		efx_start_eventq(channel);
1014 	}
1015 
1016 	efx_mcdi_mode_event(efx);
1017 
1018 	return 0;
1019 fail:
1020 	end_channel = channel;
1021 	efx_for_each_channel(channel, efx) {
1022 		if (channel == end_channel)
1023 			break;
1024 		efx_stop_eventq(channel);
1025 		if (!channel->type->keep_eventq)
1026 			efx_fini_eventq(channel);
1027 	}
1028 
1029 	return rc;
1030 }
1031 
1032 void efx_soft_disable_interrupts(struct efx_nic *efx)
1033 {
1034 	struct efx_channel *channel;
1035 
1036 	if (efx->state == STATE_DISABLED)
1037 		return;
1038 
1039 	efx_mcdi_mode_poll(efx);
1040 
1041 	efx->irq_soft_enabled = false;
1042 	smp_wmb();
1043 
1044 	if (efx->legacy_irq)
1045 		synchronize_irq(efx->legacy_irq);
1046 
1047 	efx_for_each_channel(channel, efx) {
1048 		if (channel->irq)
1049 			synchronize_irq(channel->irq);
1050 
1051 		efx_stop_eventq(channel);
1052 		if (!channel->type->keep_eventq)
1053 			efx_fini_eventq(channel);
1054 	}
1055 
1056 	/* Flush the asynchronous MCDI request queue */
1057 	efx_mcdi_flush_async(efx);
1058 }
1059 
1060 int efx_enable_interrupts(struct efx_nic *efx)
1061 {
1062 	struct efx_channel *channel, *end_channel;
1063 	int rc;
1064 
1065 	/* TODO: Is this really a bug? */
1066 	BUG_ON(efx->state == STATE_DISABLED);
1067 
1068 	if (efx->eeh_disabled_legacy_irq) {
1069 		enable_irq(efx->legacy_irq);
1070 		efx->eeh_disabled_legacy_irq = false;
1071 	}
1072 
1073 	efx->type->irq_enable_master(efx);
1074 
1075 	efx_for_each_channel(channel, efx) {
1076 		if (channel->type->keep_eventq) {
1077 			rc = efx_init_eventq(channel);
1078 			if (rc)
1079 				goto fail;
1080 		}
1081 	}
1082 
1083 	rc = efx_soft_enable_interrupts(efx);
1084 	if (rc)
1085 		goto fail;
1086 
1087 	return 0;
1088 
1089 fail:
1090 	end_channel = channel;
1091 	efx_for_each_channel(channel, efx) {
1092 		if (channel == end_channel)
1093 			break;
1094 		if (channel->type->keep_eventq)
1095 			efx_fini_eventq(channel);
1096 	}
1097 
1098 	efx->type->irq_disable_non_ev(efx);
1099 
1100 	return rc;
1101 }
1102 
1103 void efx_disable_interrupts(struct efx_nic *efx)
1104 {
1105 	struct efx_channel *channel;
1106 
1107 	efx_soft_disable_interrupts(efx);
1108 
1109 	efx_for_each_channel(channel, efx) {
1110 		if (channel->type->keep_eventq)
1111 			efx_fini_eventq(channel);
1112 	}
1113 
1114 	efx->type->irq_disable_non_ev(efx);
1115 }
1116 
1117 void efx_start_channels(struct efx_nic *efx)
1118 {
1119 	struct efx_tx_queue *tx_queue;
1120 	struct efx_rx_queue *rx_queue;
1121 	struct efx_channel *channel;
1122 
1123 	efx_for_each_channel_rev(channel, efx) {
1124 		if (channel->type->start)
1125 			channel->type->start(channel);
1126 		efx_for_each_channel_tx_queue(tx_queue, channel) {
1127 			efx_init_tx_queue(tx_queue);
1128 			atomic_inc(&efx->active_queues);
1129 		}
1130 
1131 		efx_for_each_channel_rx_queue(rx_queue, channel) {
1132 			efx_init_rx_queue(rx_queue);
1133 			atomic_inc(&efx->active_queues);
1134 			efx_stop_eventq(channel);
1135 			efx_fast_push_rx_descriptors(rx_queue, false);
1136 			efx_start_eventq(channel);
1137 		}
1138 
1139 		WARN_ON(channel->rx_pkt_n_frags);
1140 	}
1141 }
1142 
1143 void efx_stop_channels(struct efx_nic *efx)
1144 {
1145 	struct efx_tx_queue *tx_queue;
1146 	struct efx_rx_queue *rx_queue;
1147 	struct efx_channel *channel;
1148 	int rc = 0;
1149 
1150 	/* Stop special channels and RX refill.
1151 	 * The channel's stop has to be called first, since it might wait
1152 	 * for a sentinel RX to indicate the channel has fully drained.
1153 	 */
1154 	efx_for_each_channel(channel, efx) {
1155 		if (channel->type->stop)
1156 			channel->type->stop(channel);
1157 		efx_for_each_channel_rx_queue(rx_queue, channel)
1158 			rx_queue->refill_enabled = false;
1159 	}
1160 
1161 	efx_for_each_channel(channel, efx) {
1162 		/* RX packet processing is pipelined, so wait for the
1163 		 * NAPI handler to complete.  At least event queue 0
1164 		 * might be kept active by non-data events, so don't
1165 		 * use napi_synchronize() but actually disable NAPI
1166 		 * temporarily.
1167 		 */
1168 		if (efx_channel_has_rx_queue(channel)) {
1169 			efx_stop_eventq(channel);
1170 			efx_start_eventq(channel);
1171 		}
1172 	}
1173 
1174 	if (efx->type->fini_dmaq)
1175 		rc = efx->type->fini_dmaq(efx);
1176 
1177 	if (rc) {
1178 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1179 	} else {
1180 		netif_dbg(efx, drv, efx->net_dev,
1181 			  "successfully flushed all queues\n");
1182 	}
1183 
1184 	efx_for_each_channel(channel, efx) {
1185 		efx_for_each_channel_rx_queue(rx_queue, channel)
1186 			efx_fini_rx_queue(rx_queue);
1187 		efx_for_each_channel_tx_queue(tx_queue, channel)
1188 			efx_fini_tx_queue(tx_queue);
1189 	}
1190 }
1191 
1192 /**************************************************************************
1193  *
1194  * NAPI interface
1195  *
1196  *************************************************************************/
1197 
1198 /* Process channel's event queue
1199  *
1200  * This function is responsible for processing the event queue of a
1201  * single channel.  The caller must guarantee that this function will
1202  * never be concurrently called more than once on the same channel,
1203  * though different channels may be being processed concurrently.
1204  */
1205 static int efx_process_channel(struct efx_channel *channel, int budget)
1206 {
1207 	struct efx_tx_queue *tx_queue;
1208 	struct list_head rx_list;
1209 	int spent;
1210 
1211 	if (unlikely(!channel->enabled))
1212 		return 0;
1213 
1214 	/* Prepare the batch receive list */
1215 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1216 	INIT_LIST_HEAD(&rx_list);
1217 	channel->rx_list = &rx_list;
1218 
1219 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1220 		tx_queue->pkts_compl = 0;
1221 		tx_queue->bytes_compl = 0;
1222 	}
1223 
1224 	spent = efx_nic_process_eventq(channel, budget);
1225 	if (spent && efx_channel_has_rx_queue(channel)) {
1226 		struct efx_rx_queue *rx_queue =
1227 			efx_channel_get_rx_queue(channel);
1228 
1229 		efx_rx_flush_packet(channel);
1230 		efx_fast_push_rx_descriptors(rx_queue, true);
1231 	}
1232 
1233 	/* Update BQL */
1234 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1235 		if (tx_queue->bytes_compl) {
1236 			netdev_tx_completed_queue(tx_queue->core_txq,
1237 						  tx_queue->pkts_compl,
1238 						  tx_queue->bytes_compl);
1239 		}
1240 	}
1241 
1242 	/* Receive any packets we queued up */
1243 	netif_receive_skb_list(channel->rx_list);
1244 	channel->rx_list = NULL;
1245 
1246 	return spent;
1247 }
1248 
1249 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1250 {
1251 	int step = efx->irq_mod_step_us;
1252 
1253 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
1254 		if (channel->irq_moderation_us > step) {
1255 			channel->irq_moderation_us -= step;
1256 			efx->type->push_irq_moderation(channel);
1257 		}
1258 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1259 		if (channel->irq_moderation_us <
1260 		    efx->irq_rx_moderation_us) {
1261 			channel->irq_moderation_us += step;
1262 			efx->type->push_irq_moderation(channel);
1263 		}
1264 	}
1265 
1266 	channel->irq_count = 0;
1267 	channel->irq_mod_score = 0;
1268 }
1269 
1270 /* NAPI poll handler
1271  *
1272  * NAPI guarantees serialisation of polls of the same device, which
1273  * provides the guarantee required by efx_process_channel().
1274  */
1275 static int efx_poll(struct napi_struct *napi, int budget)
1276 {
1277 	struct efx_channel *channel =
1278 		container_of(napi, struct efx_channel, napi_str);
1279 	struct efx_nic *efx = channel->efx;
1280 #ifdef CONFIG_RFS_ACCEL
1281 	unsigned int time;
1282 #endif
1283 	int spent;
1284 
1285 	netif_vdbg(efx, intr, efx->net_dev,
1286 		   "channel %d NAPI poll executing on CPU %d\n",
1287 		   channel->channel, raw_smp_processor_id());
1288 
1289 	spent = efx_process_channel(channel, budget);
1290 
1291 	xdp_do_flush_map();
1292 
1293 	if (spent < budget) {
1294 		if (efx_channel_has_rx_queue(channel) &&
1295 		    efx->irq_rx_adaptive &&
1296 		    unlikely(++channel->irq_count == 1000)) {
1297 			efx_update_irq_mod(efx, channel);
1298 		}
1299 
1300 #ifdef CONFIG_RFS_ACCEL
1301 		/* Perhaps expire some ARFS filters */
1302 		time = jiffies - channel->rfs_last_expiry;
1303 		/* Would our quota be >= 20? */
1304 		if (channel->rfs_filter_count * time >= 600 * HZ)
1305 			mod_delayed_work(system_wq, &channel->filter_work, 0);
1306 #endif
1307 
1308 		/* There is no race here; although napi_disable() will
1309 		 * only wait for napi_complete(), this isn't a problem
1310 		 * since efx_nic_eventq_read_ack() will have no effect if
1311 		 * interrupts have already been disabled.
1312 		 */
1313 		if (napi_complete_done(napi, spent))
1314 			efx_nic_eventq_read_ack(channel);
1315 	}
1316 
1317 	return spent;
1318 }
1319 
1320 void efx_init_napi_channel(struct efx_channel *channel)
1321 {
1322 	struct efx_nic *efx = channel->efx;
1323 
1324 	channel->napi_dev = efx->net_dev;
1325 	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
1326 }
1327 
1328 void efx_init_napi(struct efx_nic *efx)
1329 {
1330 	struct efx_channel *channel;
1331 
1332 	efx_for_each_channel(channel, efx)
1333 		efx_init_napi_channel(channel);
1334 }
1335 
1336 void efx_fini_napi_channel(struct efx_channel *channel)
1337 {
1338 	if (channel->napi_dev)
1339 		netif_napi_del(&channel->napi_str);
1340 
1341 	channel->napi_dev = NULL;
1342 }
1343 
1344 void efx_fini_napi(struct efx_nic *efx)
1345 {
1346 	struct efx_channel *channel;
1347 
1348 	efx_for_each_channel(channel, efx)
1349 		efx_fini_napi_channel(channel);
1350 }
1351 
1352 /***************
1353  * Housekeeping
1354  ***************/
1355 
1356 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1357 {
1358 	return 0;
1359 }
1360 
1361 void efx_channel_dummy_op_void(struct efx_channel *channel)
1362 {
1363 }
1364 
1365 static const struct efx_channel_type efx_default_channel_type = {
1366 	.pre_probe		= efx_channel_dummy_op_int,
1367 	.post_remove		= efx_channel_dummy_op_void,
1368 	.get_name		= efx_get_channel_name,
1369 	.copy			= efx_copy_channel,
1370 	.want_txqs		= efx_default_channel_want_txqs,
1371 	.keep_eventq		= false,
1372 	.want_pio		= true,
1373 };
1374