1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3  * Driver for Solarflare network controllers and boards
4  * Copyright 2018 Solarflare Communications Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation, incorporated herein by reference.
9  */
10 
11 #include "net_driver.h"
12 #include <linux/module.h>
13 #include <linux/filter.h>
14 #include "efx_channels.h"
15 #include "efx.h"
16 #include "efx_common.h"
17 #include "tx_common.h"
18 #include "rx_common.h"
19 #include "nic.h"
20 #include "sriov.h"
21 #include "workarounds.h"
22 
23 /* This is the first interrupt mode to try out of:
24  * 0 => MSI-X
25  * 1 => MSI
26  * 2 => legacy
27  */
28 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
29 
30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
31  * i.e. the number of CPUs among which we may distribute simultaneous
32  * interrupt handling.
33  *
34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
35  * The default (0) means to assign an interrupt to each core.
36  */
37 unsigned int rss_cpus;
38 
39 static unsigned int irq_adapt_low_thresh = 8000;
40 module_param(irq_adapt_low_thresh, uint, 0644);
41 MODULE_PARM_DESC(irq_adapt_low_thresh,
42 		 "Threshold score for reducing IRQ moderation");
43 
44 static unsigned int irq_adapt_high_thresh = 16000;
45 module_param(irq_adapt_high_thresh, uint, 0644);
46 MODULE_PARM_DESC(irq_adapt_high_thresh,
47 		 "Threshold score for increasing IRQ moderation");
48 
49 static const struct efx_channel_type efx_default_channel_type;
50 
51 /*************
52  * INTERRUPTS
53  *************/
54 
55 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
56 {
57 	cpumask_var_t filter_mask;
58 	unsigned int count;
59 	int cpu;
60 
61 	if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
62 		netif_warn(efx, probe, efx->net_dev,
63 			   "RSS disabled due to allocation failure\n");
64 		return 1;
65 	}
66 
67 	cpumask_copy(filter_mask, cpu_online_mask);
68 	if (local_node)
69 		cpumask_and(filter_mask, filter_mask,
70 			    cpumask_of_pcibus(efx->pci_dev->bus));
71 
72 	count = 0;
73 	for_each_cpu(cpu, filter_mask) {
74 		++count;
75 		cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
76 	}
77 
78 	free_cpumask_var(filter_mask);
79 
80 	return count;
81 }
82 
83 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
84 {
85 	unsigned int count;
86 
87 	if (rss_cpus) {
88 		count = rss_cpus;
89 	} else {
90 		count = count_online_cores(efx, true);
91 
92 		/* If no online CPUs in local node, fallback to any online CPUs */
93 		if (count == 0)
94 			count = count_online_cores(efx, false);
95 	}
96 
97 	if (count > EFX_MAX_RX_QUEUES) {
98 		netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
99 			       "Reducing number of rx queues from %u to %u.\n",
100 			       count, EFX_MAX_RX_QUEUES);
101 		count = EFX_MAX_RX_QUEUES;
102 	}
103 
104 	/* If RSS is requested for the PF *and* VFs then we can't write RSS
105 	 * table entries that are inaccessible to VFs
106 	 */
107 #ifdef CONFIG_SFC_SRIOV
108 	if (efx->type->sriov_wanted) {
109 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
110 		    count > efx_vf_size(efx)) {
111 			netif_warn(efx, probe, efx->net_dev,
112 				   "Reducing number of RSS channels from %u to %u for "
113 				   "VF support. Increase vf-msix-limit to use more "
114 				   "channels on the PF.\n",
115 				   count, efx_vf_size(efx));
116 			count = efx_vf_size(efx);
117 		}
118 	}
119 #endif
120 
121 	return count;
122 }
123 
124 static int efx_allocate_msix_channels(struct efx_nic *efx,
125 				      unsigned int max_channels,
126 				      unsigned int extra_channels,
127 				      unsigned int parallelism)
128 {
129 	unsigned int n_channels = parallelism;
130 	int vec_count;
131 	int tx_per_ev;
132 	int n_xdp_tx;
133 	int n_xdp_ev;
134 
135 	if (efx_separate_tx_channels)
136 		n_channels *= 2;
137 	n_channels += extra_channels;
138 
139 	/* To allow XDP transmit to happen from arbitrary NAPI contexts
140 	 * we allocate a TX queue per CPU. We share event queues across
141 	 * multiple tx queues, assuming tx and ev queues are both
142 	 * maximum size.
143 	 */
144 	tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
145 	tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
146 	n_xdp_tx = num_possible_cpus();
147 	n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
148 
149 	vec_count = pci_msix_vec_count(efx->pci_dev);
150 	if (vec_count < 0)
151 		return vec_count;
152 
153 	max_channels = min_t(unsigned int, vec_count, max_channels);
154 
155 	/* Check resources.
156 	 * We need a channel per event queue, plus a VI per tx queue.
157 	 * This may be more pessimistic than it needs to be.
158 	 */
159 	if (n_channels >= max_channels) {
160 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
161 		netif_warn(efx, drv, efx->net_dev,
162 			   "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
163 			   n_xdp_ev, n_channels, max_channels);
164 		netif_warn(efx, drv, efx->net_dev,
165 			   "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
166 	} else if (n_channels + n_xdp_tx > efx->max_vis) {
167 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
168 		netif_warn(efx, drv, efx->net_dev,
169 			   "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
170 			   n_xdp_tx, n_channels, efx->max_vis);
171 		netif_warn(efx, drv, efx->net_dev,
172 			   "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
173 	} else if (n_channels + n_xdp_ev > max_channels) {
174 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
175 		netif_warn(efx, drv, efx->net_dev,
176 			   "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
177 			   n_xdp_ev, n_channels, max_channels);
178 
179 		n_xdp_ev = max_channels - n_channels;
180 		netif_warn(efx, drv, efx->net_dev,
181 			   "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
182 			   DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
183 	} else {
184 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
185 	}
186 
187 	if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
188 		efx->n_xdp_channels = n_xdp_ev;
189 		efx->xdp_tx_per_channel = tx_per_ev;
190 		efx->xdp_tx_queue_count = n_xdp_tx;
191 		n_channels += n_xdp_ev;
192 		netif_dbg(efx, drv, efx->net_dev,
193 			  "Allocating %d TX and %d event queues for XDP\n",
194 			  n_xdp_ev * tx_per_ev, n_xdp_ev);
195 	} else {
196 		efx->n_xdp_channels = 0;
197 		efx->xdp_tx_per_channel = 0;
198 		efx->xdp_tx_queue_count = n_xdp_tx;
199 	}
200 
201 	if (vec_count < n_channels) {
202 		netif_err(efx, drv, efx->net_dev,
203 			  "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
204 			  vec_count, n_channels);
205 		netif_err(efx, drv, efx->net_dev,
206 			  "WARNING: Performance may be reduced.\n");
207 		n_channels = vec_count;
208 	}
209 
210 	n_channels = min(n_channels, max_channels);
211 
212 	efx->n_channels = n_channels;
213 
214 	/* Ignore XDP tx channels when creating rx channels. */
215 	n_channels -= efx->n_xdp_channels;
216 
217 	if (efx_separate_tx_channels) {
218 		efx->n_tx_channels =
219 			min(max(n_channels / 2, 1U),
220 			    efx->max_tx_channels);
221 		efx->tx_channel_offset =
222 			n_channels - efx->n_tx_channels;
223 		efx->n_rx_channels =
224 			max(n_channels -
225 			    efx->n_tx_channels, 1U);
226 	} else {
227 		efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
228 		efx->tx_channel_offset = 0;
229 		efx->n_rx_channels = n_channels;
230 	}
231 
232 	efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
233 	efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
234 
235 	efx->xdp_channel_offset = n_channels;
236 
237 	netif_dbg(efx, drv, efx->net_dev,
238 		  "Allocating %u RX channels\n",
239 		  efx->n_rx_channels);
240 
241 	return efx->n_channels;
242 }
243 
244 /* Probe the number and type of interrupts we are able to obtain, and
245  * the resulting numbers of channels and RX queues.
246  */
247 int efx_probe_interrupts(struct efx_nic *efx)
248 {
249 	unsigned int extra_channels = 0;
250 	unsigned int rss_spread;
251 	unsigned int i, j;
252 	int rc;
253 
254 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
255 		if (efx->extra_channel_type[i])
256 			++extra_channels;
257 
258 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
259 		unsigned int parallelism = efx_wanted_parallelism(efx);
260 		struct msix_entry xentries[EFX_MAX_CHANNELS];
261 		unsigned int n_channels;
262 
263 		rc = efx_allocate_msix_channels(efx, efx->max_channels,
264 						extra_channels, parallelism);
265 		if (rc >= 0) {
266 			n_channels = rc;
267 			for (i = 0; i < n_channels; i++)
268 				xentries[i].entry = i;
269 			rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
270 						   n_channels);
271 		}
272 		if (rc < 0) {
273 			/* Fall back to single channel MSI */
274 			netif_err(efx, drv, efx->net_dev,
275 				  "could not enable MSI-X\n");
276 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
277 				efx->interrupt_mode = EFX_INT_MODE_MSI;
278 			else
279 				return rc;
280 		} else if (rc < n_channels) {
281 			netif_err(efx, drv, efx->net_dev,
282 				  "WARNING: Insufficient MSI-X vectors"
283 				  " available (%d < %u).\n", rc, n_channels);
284 			netif_err(efx, drv, efx->net_dev,
285 				  "WARNING: Performance may be reduced.\n");
286 			n_channels = rc;
287 		}
288 
289 		if (rc > 0) {
290 			for (i = 0; i < efx->n_channels; i++)
291 				efx_get_channel(efx, i)->irq =
292 					xentries[i].vector;
293 		}
294 	}
295 
296 	/* Try single interrupt MSI */
297 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
298 		efx->n_channels = 1;
299 		efx->n_rx_channels = 1;
300 		efx->n_tx_channels = 1;
301 		efx->tx_channel_offset = 0;
302 		efx->n_xdp_channels = 0;
303 		efx->xdp_channel_offset = efx->n_channels;
304 		rc = pci_enable_msi(efx->pci_dev);
305 		if (rc == 0) {
306 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
307 		} else {
308 			netif_err(efx, drv, efx->net_dev,
309 				  "could not enable MSI\n");
310 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
311 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;
312 			else
313 				return rc;
314 		}
315 	}
316 
317 	/* Assume legacy interrupts */
318 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
319 		efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
320 		efx->n_rx_channels = 1;
321 		efx->n_tx_channels = 1;
322 		efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0;
323 		efx->n_xdp_channels = 0;
324 		efx->xdp_channel_offset = efx->n_channels;
325 		efx->legacy_irq = efx->pci_dev->irq;
326 	}
327 
328 	/* Assign extra channels if possible, before XDP channels */
329 	efx->n_extra_tx_channels = 0;
330 	j = efx->xdp_channel_offset;
331 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
332 		if (!efx->extra_channel_type[i])
333 			continue;
334 		if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
335 			efx->extra_channel_type[i]->handle_no_channel(efx);
336 		} else {
337 			--j;
338 			efx_get_channel(efx, j)->type =
339 				efx->extra_channel_type[i];
340 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
341 				efx->n_extra_tx_channels++;
342 		}
343 	}
344 
345 	rss_spread = efx->n_rx_channels;
346 	/* RSS might be usable on VFs even if it is disabled on the PF */
347 #ifdef CONFIG_SFC_SRIOV
348 	if (efx->type->sriov_wanted) {
349 		efx->rss_spread = ((rss_spread > 1 ||
350 				    !efx->type->sriov_wanted(efx)) ?
351 				   rss_spread : efx_vf_size(efx));
352 		return 0;
353 	}
354 #endif
355 	efx->rss_spread = rss_spread;
356 
357 	return 0;
358 }
359 
360 #if defined(CONFIG_SMP)
361 void efx_set_interrupt_affinity(struct efx_nic *efx)
362 {
363 	const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus);
364 	struct efx_channel *channel;
365 	unsigned int cpu;
366 
367 	/* If no online CPUs in local node, fallback to any online CPU */
368 	if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
369 		numa_mask = cpu_online_mask;
370 
371 	cpu = -1;
372 	efx_for_each_channel(channel, efx) {
373 		cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
374 		if (cpu >= nr_cpu_ids)
375 			cpu = cpumask_first_and(cpu_online_mask, numa_mask);
376 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
377 	}
378 }
379 
380 void efx_clear_interrupt_affinity(struct efx_nic *efx)
381 {
382 	struct efx_channel *channel;
383 
384 	efx_for_each_channel(channel, efx)
385 		irq_set_affinity_hint(channel->irq, NULL);
386 }
387 #else
388 void
389 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
390 {
391 }
392 
393 void
394 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
395 {
396 }
397 #endif /* CONFIG_SMP */
398 
399 void efx_remove_interrupts(struct efx_nic *efx)
400 {
401 	struct efx_channel *channel;
402 
403 	/* Remove MSI/MSI-X interrupts */
404 	efx_for_each_channel(channel, efx)
405 		channel->irq = 0;
406 	pci_disable_msi(efx->pci_dev);
407 	pci_disable_msix(efx->pci_dev);
408 
409 	/* Remove legacy interrupt */
410 	efx->legacy_irq = 0;
411 }
412 
413 /***************
414  * EVENT QUEUES
415  ***************/
416 
417 /* Create event queue
418  * Event queue memory allocations are done only once.  If the channel
419  * is reset, the memory buffer will be reused; this guards against
420  * errors during channel reset and also simplifies interrupt handling.
421  */
422 int efx_probe_eventq(struct efx_channel *channel)
423 {
424 	struct efx_nic *efx = channel->efx;
425 	unsigned long entries;
426 
427 	netif_dbg(efx, probe, efx->net_dev,
428 		  "chan %d create event queue\n", channel->channel);
429 
430 	/* Build an event queue with room for one event per tx and rx buffer,
431 	 * plus some extra for link state events and MCDI completions.
432 	 */
433 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
434 	EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
435 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
436 
437 	return efx_nic_probe_eventq(channel);
438 }
439 
440 /* Prepare channel's event queue */
441 int efx_init_eventq(struct efx_channel *channel)
442 {
443 	struct efx_nic *efx = channel->efx;
444 	int rc;
445 
446 	EFX_WARN_ON_PARANOID(channel->eventq_init);
447 
448 	netif_dbg(efx, drv, efx->net_dev,
449 		  "chan %d init event queue\n", channel->channel);
450 
451 	rc = efx_nic_init_eventq(channel);
452 	if (rc == 0) {
453 		efx->type->push_irq_moderation(channel);
454 		channel->eventq_read_ptr = 0;
455 		channel->eventq_init = true;
456 	}
457 	return rc;
458 }
459 
460 /* Enable event queue processing and NAPI */
461 void efx_start_eventq(struct efx_channel *channel)
462 {
463 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
464 		  "chan %d start event queue\n", channel->channel);
465 
466 	/* Make sure the NAPI handler sees the enabled flag set */
467 	channel->enabled = true;
468 	smp_wmb();
469 
470 	napi_enable(&channel->napi_str);
471 	efx_nic_eventq_read_ack(channel);
472 }
473 
474 /* Disable event queue processing and NAPI */
475 void efx_stop_eventq(struct efx_channel *channel)
476 {
477 	if (!channel->enabled)
478 		return;
479 
480 	napi_disable(&channel->napi_str);
481 	channel->enabled = false;
482 }
483 
484 void efx_fini_eventq(struct efx_channel *channel)
485 {
486 	if (!channel->eventq_init)
487 		return;
488 
489 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
490 		  "chan %d fini event queue\n", channel->channel);
491 
492 	efx_nic_fini_eventq(channel);
493 	channel->eventq_init = false;
494 }
495 
496 void efx_remove_eventq(struct efx_channel *channel)
497 {
498 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
499 		  "chan %d remove event queue\n", channel->channel);
500 
501 	efx_nic_remove_eventq(channel);
502 }
503 
504 /**************************************************************************
505  *
506  * Channel handling
507  *
508  *************************************************************************/
509 
510 #ifdef CONFIG_RFS_ACCEL
511 static void efx_filter_rfs_expire(struct work_struct *data)
512 {
513 	struct delayed_work *dwork = to_delayed_work(data);
514 	struct efx_channel *channel;
515 	unsigned int time, quota;
516 
517 	channel = container_of(dwork, struct efx_channel, filter_work);
518 	time = jiffies - channel->rfs_last_expiry;
519 	quota = channel->rfs_filter_count * time / (30 * HZ);
520 	if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
521 		channel->rfs_last_expiry += time;
522 	/* Ensure we do more work eventually even if NAPI poll is not happening */
523 	schedule_delayed_work(dwork, 30 * HZ);
524 }
525 #endif
526 
527 /* Allocate and initialise a channel structure. */
528 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
529 {
530 	struct efx_rx_queue *rx_queue;
531 	struct efx_tx_queue *tx_queue;
532 	struct efx_channel *channel;
533 	int j;
534 
535 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
536 	if (!channel)
537 		return NULL;
538 
539 	channel->efx = efx;
540 	channel->channel = i;
541 	channel->type = &efx_default_channel_type;
542 
543 	for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
544 		tx_queue = &channel->tx_queue[j];
545 		tx_queue->efx = efx;
546 		tx_queue->queue = -1;
547 		tx_queue->label = j;
548 		tx_queue->channel = channel;
549 	}
550 
551 #ifdef CONFIG_RFS_ACCEL
552 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
553 #endif
554 
555 	rx_queue = &channel->rx_queue;
556 	rx_queue->efx = efx;
557 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
558 
559 	return channel;
560 }
561 
562 int efx_init_channels(struct efx_nic *efx)
563 {
564 	unsigned int i;
565 
566 	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
567 		efx->channel[i] = efx_alloc_channel(efx, i);
568 		if (!efx->channel[i])
569 			return -ENOMEM;
570 		efx->msi_context[i].efx = efx;
571 		efx->msi_context[i].index = i;
572 	}
573 
574 	/* Higher numbered interrupt modes are less capable! */
575 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,
576 				  efx_interrupt_mode);
577 
578 	efx->max_channels = EFX_MAX_CHANNELS;
579 	efx->max_tx_channels = EFX_MAX_CHANNELS;
580 
581 	return 0;
582 }
583 
584 void efx_fini_channels(struct efx_nic *efx)
585 {
586 	unsigned int i;
587 
588 	for (i = 0; i < EFX_MAX_CHANNELS; i++)
589 		if (efx->channel[i]) {
590 			kfree(efx->channel[i]);
591 			efx->channel[i] = NULL;
592 		}
593 }
594 
595 /* Allocate and initialise a channel structure, copying parameters
596  * (but not resources) from an old channel structure.
597  */
598 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
599 {
600 	struct efx_rx_queue *rx_queue;
601 	struct efx_tx_queue *tx_queue;
602 	struct efx_channel *channel;
603 	int j;
604 
605 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
606 	if (!channel)
607 		return NULL;
608 
609 	*channel = *old_channel;
610 
611 	channel->napi_dev = NULL;
612 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
613 	channel->napi_str.napi_id = 0;
614 	channel->napi_str.state = 0;
615 	memset(&channel->eventq, 0, sizeof(channel->eventq));
616 
617 	for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
618 		tx_queue = &channel->tx_queue[j];
619 		if (tx_queue->channel)
620 			tx_queue->channel = channel;
621 		tx_queue->buffer = NULL;
622 		tx_queue->cb_page = NULL;
623 		memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
624 	}
625 
626 	rx_queue = &channel->rx_queue;
627 	rx_queue->buffer = NULL;
628 	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
629 	timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
630 #ifdef CONFIG_RFS_ACCEL
631 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
632 #endif
633 
634 	return channel;
635 }
636 
637 static int efx_probe_channel(struct efx_channel *channel)
638 {
639 	struct efx_tx_queue *tx_queue;
640 	struct efx_rx_queue *rx_queue;
641 	int rc;
642 
643 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
644 		  "creating channel %d\n", channel->channel);
645 
646 	rc = channel->type->pre_probe(channel);
647 	if (rc)
648 		goto fail;
649 
650 	rc = efx_probe_eventq(channel);
651 	if (rc)
652 		goto fail;
653 
654 	efx_for_each_channel_tx_queue(tx_queue, channel) {
655 		rc = efx_probe_tx_queue(tx_queue);
656 		if (rc)
657 			goto fail;
658 	}
659 
660 	efx_for_each_channel_rx_queue(rx_queue, channel) {
661 		rc = efx_probe_rx_queue(rx_queue);
662 		if (rc)
663 			goto fail;
664 	}
665 
666 	channel->rx_list = NULL;
667 
668 	return 0;
669 
670 fail:
671 	efx_remove_channel(channel);
672 	return rc;
673 }
674 
675 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
676 				 size_t len)
677 {
678 	struct efx_nic *efx = channel->efx;
679 	const char *type;
680 	int number;
681 
682 	number = channel->channel;
683 
684 	if (number >= efx->xdp_channel_offset &&
685 	    !WARN_ON_ONCE(!efx->n_xdp_channels)) {
686 		type = "-xdp";
687 		number -= efx->xdp_channel_offset;
688 	} else if (efx->tx_channel_offset == 0) {
689 		type = "";
690 	} else if (number < efx->tx_channel_offset) {
691 		type = "-rx";
692 	} else {
693 		type = "-tx";
694 		number -= efx->tx_channel_offset;
695 	}
696 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);
697 }
698 
699 void efx_set_channel_names(struct efx_nic *efx)
700 {
701 	struct efx_channel *channel;
702 
703 	efx_for_each_channel(channel, efx)
704 		channel->type->get_name(channel,
705 					efx->msi_context[channel->channel].name,
706 					sizeof(efx->msi_context[0].name));
707 }
708 
709 int efx_probe_channels(struct efx_nic *efx)
710 {
711 	struct efx_channel *channel;
712 	int rc;
713 
714 	/* Restart special buffer allocation */
715 	efx->next_buffer_table = 0;
716 
717 	/* Probe channels in reverse, so that any 'extra' channels
718 	 * use the start of the buffer table. This allows the traffic
719 	 * channels to be resized without moving them or wasting the
720 	 * entries before them.
721 	 */
722 	efx_for_each_channel_rev(channel, efx) {
723 		rc = efx_probe_channel(channel);
724 		if (rc) {
725 			netif_err(efx, probe, efx->net_dev,
726 				  "failed to create channel %d\n",
727 				  channel->channel);
728 			goto fail;
729 		}
730 	}
731 	efx_set_channel_names(efx);
732 
733 	return 0;
734 
735 fail:
736 	efx_remove_channels(efx);
737 	return rc;
738 }
739 
740 void efx_remove_channel(struct efx_channel *channel)
741 {
742 	struct efx_tx_queue *tx_queue;
743 	struct efx_rx_queue *rx_queue;
744 
745 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
746 		  "destroy chan %d\n", channel->channel);
747 
748 	efx_for_each_channel_rx_queue(rx_queue, channel)
749 		efx_remove_rx_queue(rx_queue);
750 	efx_for_each_channel_tx_queue(tx_queue, channel)
751 		efx_remove_tx_queue(tx_queue);
752 	efx_remove_eventq(channel);
753 	channel->type->post_remove(channel);
754 }
755 
756 void efx_remove_channels(struct efx_nic *efx)
757 {
758 	struct efx_channel *channel;
759 
760 	efx_for_each_channel(channel, efx)
761 		efx_remove_channel(channel);
762 
763 	kfree(efx->xdp_tx_queues);
764 }
765 
766 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
767 				struct efx_tx_queue *tx_queue)
768 {
769 	if (xdp_queue_number >= efx->xdp_tx_queue_count)
770 		return -EINVAL;
771 
772 	netif_dbg(efx, drv, efx->net_dev,
773 		  "Channel %u TXQ %u is XDP %u, HW %u\n",
774 		  tx_queue->channel->channel, tx_queue->label,
775 		  xdp_queue_number, tx_queue->queue);
776 	efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
777 	return 0;
778 }
779 
780 static void efx_set_xdp_channels(struct efx_nic *efx)
781 {
782 	struct efx_tx_queue *tx_queue;
783 	struct efx_channel *channel;
784 	unsigned int next_queue = 0;
785 	int xdp_queue_number = 0;
786 	int rc;
787 
788 	/* We need to mark which channels really have RX and TX
789 	 * queues, and adjust the TX queue numbers if we have separate
790 	 * RX-only and TX-only channels.
791 	 */
792 	efx_for_each_channel(channel, efx) {
793 		if (channel->channel < efx->tx_channel_offset)
794 			continue;
795 
796 		if (efx_channel_is_xdp_tx(channel)) {
797 			efx_for_each_channel_tx_queue(tx_queue, channel) {
798 				tx_queue->queue = next_queue++;
799 				rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
800 							  tx_queue);
801 				if (rc == 0)
802 					xdp_queue_number++;
803 			}
804 		} else {
805 			efx_for_each_channel_tx_queue(tx_queue, channel) {
806 				tx_queue->queue = next_queue++;
807 				netif_dbg(efx, drv, efx->net_dev,
808 					  "Channel %u TXQ %u is HW %u\n",
809 					  channel->channel, tx_queue->label,
810 					  tx_queue->queue);
811 			}
812 
813 			/* If XDP is borrowing queues from net stack, it must
814 			 * use the queue with no csum offload, which is the
815 			 * first one of the channel
816 			 * (note: tx_queue_by_type is not initialized yet)
817 			 */
818 			if (efx->xdp_txq_queues_mode ==
819 			    EFX_XDP_TX_QUEUES_BORROWED) {
820 				tx_queue = &channel->tx_queue[0];
821 				rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
822 							  tx_queue);
823 				if (rc == 0)
824 					xdp_queue_number++;
825 			}
826 		}
827 	}
828 	WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
829 		xdp_queue_number != efx->xdp_tx_queue_count);
830 	WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
831 		xdp_queue_number > efx->xdp_tx_queue_count);
832 
833 	/* If we have more CPUs than assigned XDP TX queues, assign the already
834 	 * existing queues to the exceeding CPUs
835 	 */
836 	next_queue = 0;
837 	while (xdp_queue_number < efx->xdp_tx_queue_count) {
838 		tx_queue = efx->xdp_tx_queues[next_queue++];
839 		rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
840 		if (rc == 0)
841 			xdp_queue_number++;
842 	}
843 }
844 
845 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
846 {
847 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
848 			   *ptp_channel = efx_ptp_channel(efx);
849 	struct efx_ptp_data *ptp_data = efx->ptp_data;
850 	unsigned int i, next_buffer_table = 0;
851 	u32 old_rxq_entries, old_txq_entries;
852 	int rc, rc2;
853 
854 	rc = efx_check_disabled(efx);
855 	if (rc)
856 		return rc;
857 
858 	/* Not all channels should be reallocated. We must avoid
859 	 * reallocating their buffer table entries.
860 	 */
861 	efx_for_each_channel(channel, efx) {
862 		struct efx_rx_queue *rx_queue;
863 		struct efx_tx_queue *tx_queue;
864 
865 		if (channel->type->copy)
866 			continue;
867 		next_buffer_table = max(next_buffer_table,
868 					channel->eventq.index +
869 					channel->eventq.entries);
870 		efx_for_each_channel_rx_queue(rx_queue, channel)
871 			next_buffer_table = max(next_buffer_table,
872 						rx_queue->rxd.index +
873 						rx_queue->rxd.entries);
874 		efx_for_each_channel_tx_queue(tx_queue, channel)
875 			next_buffer_table = max(next_buffer_table,
876 						tx_queue->txd.index +
877 						tx_queue->txd.entries);
878 	}
879 
880 	efx_device_detach_sync(efx);
881 	efx_stop_all(efx);
882 	efx_soft_disable_interrupts(efx);
883 
884 	/* Clone channels (where possible) */
885 	memset(other_channel, 0, sizeof(other_channel));
886 	for (i = 0; i < efx->n_channels; i++) {
887 		channel = efx->channel[i];
888 		if (channel->type->copy)
889 			channel = channel->type->copy(channel);
890 		if (!channel) {
891 			rc = -ENOMEM;
892 			goto out;
893 		}
894 		other_channel[i] = channel;
895 	}
896 
897 	/* Swap entry counts and channel pointers */
898 	old_rxq_entries = efx->rxq_entries;
899 	old_txq_entries = efx->txq_entries;
900 	efx->rxq_entries = rxq_entries;
901 	efx->txq_entries = txq_entries;
902 	for (i = 0; i < efx->n_channels; i++)
903 		swap(efx->channel[i], other_channel[i]);
904 
905 	/* Restart buffer table allocation */
906 	efx->next_buffer_table = next_buffer_table;
907 
908 	for (i = 0; i < efx->n_channels; i++) {
909 		channel = efx->channel[i];
910 		if (!channel->type->copy)
911 			continue;
912 		rc = efx_probe_channel(channel);
913 		if (rc)
914 			goto rollback;
915 		efx_init_napi_channel(efx->channel[i]);
916 	}
917 
918 	efx_set_xdp_channels(efx);
919 out:
920 	efx->ptp_data = NULL;
921 	/* Destroy unused channel structures */
922 	for (i = 0; i < efx->n_channels; i++) {
923 		channel = other_channel[i];
924 		if (channel && channel->type->copy) {
925 			efx_fini_napi_channel(channel);
926 			efx_remove_channel(channel);
927 			kfree(channel);
928 		}
929 	}
930 
931 	efx->ptp_data = ptp_data;
932 	rc2 = efx_soft_enable_interrupts(efx);
933 	if (rc2) {
934 		rc = rc ? rc : rc2;
935 		netif_err(efx, drv, efx->net_dev,
936 			  "unable to restart interrupts on channel reallocation\n");
937 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);
938 	} else {
939 		efx_start_all(efx);
940 		efx_device_attach_if_not_resetting(efx);
941 	}
942 	return rc;
943 
944 rollback:
945 	/* Swap back */
946 	efx->rxq_entries = old_rxq_entries;
947 	efx->txq_entries = old_txq_entries;
948 	for (i = 0; i < efx->n_channels; i++)
949 		swap(efx->channel[i], other_channel[i]);
950 	efx_ptp_update_channel(efx, ptp_channel);
951 	goto out;
952 }
953 
954 int efx_set_channels(struct efx_nic *efx)
955 {
956 	struct efx_channel *channel;
957 	int rc;
958 
959 	if (efx->xdp_tx_queue_count) {
960 		EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
961 
962 		/* Allocate array for XDP TX queue lookup. */
963 		efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
964 					     sizeof(*efx->xdp_tx_queues),
965 					     GFP_KERNEL);
966 		if (!efx->xdp_tx_queues)
967 			return -ENOMEM;
968 	}
969 
970 	efx_for_each_channel(channel, efx) {
971 		if (channel->channel < efx->n_rx_channels)
972 			channel->rx_queue.core_index = channel->channel;
973 		else
974 			channel->rx_queue.core_index = -1;
975 	}
976 
977 	efx_set_xdp_channels(efx);
978 
979 	rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
980 	if (rc)
981 		return rc;
982 	return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
983 }
984 
985 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
986 {
987 	return channel->channel - channel->efx->tx_channel_offset <
988 		channel->efx->n_tx_channels;
989 }
990 
991 /*************
992  * START/STOP
993  *************/
994 
995 int efx_soft_enable_interrupts(struct efx_nic *efx)
996 {
997 	struct efx_channel *channel, *end_channel;
998 	int rc;
999 
1000 	BUG_ON(efx->state == STATE_DISABLED);
1001 
1002 	efx->irq_soft_enabled = true;
1003 	smp_wmb();
1004 
1005 	efx_for_each_channel(channel, efx) {
1006 		if (!channel->type->keep_eventq) {
1007 			rc = efx_init_eventq(channel);
1008 			if (rc)
1009 				goto fail;
1010 		}
1011 		efx_start_eventq(channel);
1012 	}
1013 
1014 	efx_mcdi_mode_event(efx);
1015 
1016 	return 0;
1017 fail:
1018 	end_channel = channel;
1019 	efx_for_each_channel(channel, efx) {
1020 		if (channel == end_channel)
1021 			break;
1022 		efx_stop_eventq(channel);
1023 		if (!channel->type->keep_eventq)
1024 			efx_fini_eventq(channel);
1025 	}
1026 
1027 	return rc;
1028 }
1029 
1030 void efx_soft_disable_interrupts(struct efx_nic *efx)
1031 {
1032 	struct efx_channel *channel;
1033 
1034 	if (efx->state == STATE_DISABLED)
1035 		return;
1036 
1037 	efx_mcdi_mode_poll(efx);
1038 
1039 	efx->irq_soft_enabled = false;
1040 	smp_wmb();
1041 
1042 	if (efx->legacy_irq)
1043 		synchronize_irq(efx->legacy_irq);
1044 
1045 	efx_for_each_channel(channel, efx) {
1046 		if (channel->irq)
1047 			synchronize_irq(channel->irq);
1048 
1049 		efx_stop_eventq(channel);
1050 		if (!channel->type->keep_eventq)
1051 			efx_fini_eventq(channel);
1052 	}
1053 
1054 	/* Flush the asynchronous MCDI request queue */
1055 	efx_mcdi_flush_async(efx);
1056 }
1057 
1058 int efx_enable_interrupts(struct efx_nic *efx)
1059 {
1060 	struct efx_channel *channel, *end_channel;
1061 	int rc;
1062 
1063 	/* TODO: Is this really a bug? */
1064 	BUG_ON(efx->state == STATE_DISABLED);
1065 
1066 	if (efx->eeh_disabled_legacy_irq) {
1067 		enable_irq(efx->legacy_irq);
1068 		efx->eeh_disabled_legacy_irq = false;
1069 	}
1070 
1071 	efx->type->irq_enable_master(efx);
1072 
1073 	efx_for_each_channel(channel, efx) {
1074 		if (channel->type->keep_eventq) {
1075 			rc = efx_init_eventq(channel);
1076 			if (rc)
1077 				goto fail;
1078 		}
1079 	}
1080 
1081 	rc = efx_soft_enable_interrupts(efx);
1082 	if (rc)
1083 		goto fail;
1084 
1085 	return 0;
1086 
1087 fail:
1088 	end_channel = channel;
1089 	efx_for_each_channel(channel, efx) {
1090 		if (channel == end_channel)
1091 			break;
1092 		if (channel->type->keep_eventq)
1093 			efx_fini_eventq(channel);
1094 	}
1095 
1096 	efx->type->irq_disable_non_ev(efx);
1097 
1098 	return rc;
1099 }
1100 
1101 void efx_disable_interrupts(struct efx_nic *efx)
1102 {
1103 	struct efx_channel *channel;
1104 
1105 	efx_soft_disable_interrupts(efx);
1106 
1107 	efx_for_each_channel(channel, efx) {
1108 		if (channel->type->keep_eventq)
1109 			efx_fini_eventq(channel);
1110 	}
1111 
1112 	efx->type->irq_disable_non_ev(efx);
1113 }
1114 
1115 void efx_start_channels(struct efx_nic *efx)
1116 {
1117 	struct efx_tx_queue *tx_queue;
1118 	struct efx_rx_queue *rx_queue;
1119 	struct efx_channel *channel;
1120 
1121 	efx_for_each_channel_rev(channel, efx) {
1122 		if (channel->type->start)
1123 			channel->type->start(channel);
1124 		efx_for_each_channel_tx_queue(tx_queue, channel) {
1125 			efx_init_tx_queue(tx_queue);
1126 			atomic_inc(&efx->active_queues);
1127 		}
1128 
1129 		efx_for_each_channel_rx_queue(rx_queue, channel) {
1130 			efx_init_rx_queue(rx_queue);
1131 			atomic_inc(&efx->active_queues);
1132 			efx_stop_eventq(channel);
1133 			efx_fast_push_rx_descriptors(rx_queue, false);
1134 			efx_start_eventq(channel);
1135 		}
1136 
1137 		WARN_ON(channel->rx_pkt_n_frags);
1138 	}
1139 }
1140 
1141 void efx_stop_channels(struct efx_nic *efx)
1142 {
1143 	struct efx_tx_queue *tx_queue;
1144 	struct efx_rx_queue *rx_queue;
1145 	struct efx_channel *channel;
1146 	int rc = 0;
1147 
1148 	/* Stop special channels and RX refill.
1149 	 * The channel's stop has to be called first, since it might wait
1150 	 * for a sentinel RX to indicate the channel has fully drained.
1151 	 */
1152 	efx_for_each_channel(channel, efx) {
1153 		if (channel->type->stop)
1154 			channel->type->stop(channel);
1155 		efx_for_each_channel_rx_queue(rx_queue, channel)
1156 			rx_queue->refill_enabled = false;
1157 	}
1158 
1159 	efx_for_each_channel(channel, efx) {
1160 		/* RX packet processing is pipelined, so wait for the
1161 		 * NAPI handler to complete.  At least event queue 0
1162 		 * might be kept active by non-data events, so don't
1163 		 * use napi_synchronize() but actually disable NAPI
1164 		 * temporarily.
1165 		 */
1166 		if (efx_channel_has_rx_queue(channel)) {
1167 			efx_stop_eventq(channel);
1168 			efx_start_eventq(channel);
1169 		}
1170 	}
1171 
1172 	if (efx->type->fini_dmaq)
1173 		rc = efx->type->fini_dmaq(efx);
1174 
1175 	if (rc) {
1176 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1177 	} else {
1178 		netif_dbg(efx, drv, efx->net_dev,
1179 			  "successfully flushed all queues\n");
1180 	}
1181 
1182 	efx_for_each_channel(channel, efx) {
1183 		efx_for_each_channel_rx_queue(rx_queue, channel)
1184 			efx_fini_rx_queue(rx_queue);
1185 		efx_for_each_channel_tx_queue(tx_queue, channel)
1186 			efx_fini_tx_queue(tx_queue);
1187 	}
1188 }
1189 
1190 /**************************************************************************
1191  *
1192  * NAPI interface
1193  *
1194  *************************************************************************/
1195 
1196 /* Process channel's event queue
1197  *
1198  * This function is responsible for processing the event queue of a
1199  * single channel.  The caller must guarantee that this function will
1200  * never be concurrently called more than once on the same channel,
1201  * though different channels may be being processed concurrently.
1202  */
1203 static int efx_process_channel(struct efx_channel *channel, int budget)
1204 {
1205 	struct efx_tx_queue *tx_queue;
1206 	struct list_head rx_list;
1207 	int spent;
1208 
1209 	if (unlikely(!channel->enabled))
1210 		return 0;
1211 
1212 	/* Prepare the batch receive list */
1213 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1214 	INIT_LIST_HEAD(&rx_list);
1215 	channel->rx_list = &rx_list;
1216 
1217 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1218 		tx_queue->pkts_compl = 0;
1219 		tx_queue->bytes_compl = 0;
1220 	}
1221 
1222 	spent = efx_nic_process_eventq(channel, budget);
1223 	if (spent && efx_channel_has_rx_queue(channel)) {
1224 		struct efx_rx_queue *rx_queue =
1225 			efx_channel_get_rx_queue(channel);
1226 
1227 		efx_rx_flush_packet(channel);
1228 		efx_fast_push_rx_descriptors(rx_queue, true);
1229 	}
1230 
1231 	/* Update BQL */
1232 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1233 		if (tx_queue->bytes_compl) {
1234 			netdev_tx_completed_queue(tx_queue->core_txq,
1235 						  tx_queue->pkts_compl,
1236 						  tx_queue->bytes_compl);
1237 		}
1238 	}
1239 
1240 	/* Receive any packets we queued up */
1241 	netif_receive_skb_list(channel->rx_list);
1242 	channel->rx_list = NULL;
1243 
1244 	return spent;
1245 }
1246 
1247 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1248 {
1249 	int step = efx->irq_mod_step_us;
1250 
1251 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
1252 		if (channel->irq_moderation_us > step) {
1253 			channel->irq_moderation_us -= step;
1254 			efx->type->push_irq_moderation(channel);
1255 		}
1256 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1257 		if (channel->irq_moderation_us <
1258 		    efx->irq_rx_moderation_us) {
1259 			channel->irq_moderation_us += step;
1260 			efx->type->push_irq_moderation(channel);
1261 		}
1262 	}
1263 
1264 	channel->irq_count = 0;
1265 	channel->irq_mod_score = 0;
1266 }
1267 
1268 /* NAPI poll handler
1269  *
1270  * NAPI guarantees serialisation of polls of the same device, which
1271  * provides the guarantee required by efx_process_channel().
1272  */
1273 static int efx_poll(struct napi_struct *napi, int budget)
1274 {
1275 	struct efx_channel *channel =
1276 		container_of(napi, struct efx_channel, napi_str);
1277 	struct efx_nic *efx = channel->efx;
1278 #ifdef CONFIG_RFS_ACCEL
1279 	unsigned int time;
1280 #endif
1281 	int spent;
1282 
1283 	netif_vdbg(efx, intr, efx->net_dev,
1284 		   "channel %d NAPI poll executing on CPU %d\n",
1285 		   channel->channel, raw_smp_processor_id());
1286 
1287 	spent = efx_process_channel(channel, budget);
1288 
1289 	xdp_do_flush_map();
1290 
1291 	if (spent < budget) {
1292 		if (efx_channel_has_rx_queue(channel) &&
1293 		    efx->irq_rx_adaptive &&
1294 		    unlikely(++channel->irq_count == 1000)) {
1295 			efx_update_irq_mod(efx, channel);
1296 		}
1297 
1298 #ifdef CONFIG_RFS_ACCEL
1299 		/* Perhaps expire some ARFS filters */
1300 		time = jiffies - channel->rfs_last_expiry;
1301 		/* Would our quota be >= 20? */
1302 		if (channel->rfs_filter_count * time >= 600 * HZ)
1303 			mod_delayed_work(system_wq, &channel->filter_work, 0);
1304 #endif
1305 
1306 		/* There is no race here; although napi_disable() will
1307 		 * only wait for napi_complete(), this isn't a problem
1308 		 * since efx_nic_eventq_read_ack() will have no effect if
1309 		 * interrupts have already been disabled.
1310 		 */
1311 		if (napi_complete_done(napi, spent))
1312 			efx_nic_eventq_read_ack(channel);
1313 	}
1314 
1315 	return spent;
1316 }
1317 
1318 void efx_init_napi_channel(struct efx_channel *channel)
1319 {
1320 	struct efx_nic *efx = channel->efx;
1321 
1322 	channel->napi_dev = efx->net_dev;
1323 	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
1324 }
1325 
1326 void efx_init_napi(struct efx_nic *efx)
1327 {
1328 	struct efx_channel *channel;
1329 
1330 	efx_for_each_channel(channel, efx)
1331 		efx_init_napi_channel(channel);
1332 }
1333 
1334 void efx_fini_napi_channel(struct efx_channel *channel)
1335 {
1336 	if (channel->napi_dev)
1337 		netif_napi_del(&channel->napi_str);
1338 
1339 	channel->napi_dev = NULL;
1340 }
1341 
1342 void efx_fini_napi(struct efx_nic *efx)
1343 {
1344 	struct efx_channel *channel;
1345 
1346 	efx_for_each_channel(channel, efx)
1347 		efx_fini_napi_channel(channel);
1348 }
1349 
1350 /***************
1351  * Housekeeping
1352  ***************/
1353 
1354 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1355 {
1356 	return 0;
1357 }
1358 
1359 void efx_channel_dummy_op_void(struct efx_channel *channel)
1360 {
1361 }
1362 
1363 static const struct efx_channel_type efx_default_channel_type = {
1364 	.pre_probe		= efx_channel_dummy_op_int,
1365 	.post_remove		= efx_channel_dummy_op_void,
1366 	.get_name		= efx_get_channel_name,
1367 	.copy			= efx_copy_channel,
1368 	.want_txqs		= efx_default_channel_want_txqs,
1369 	.keep_eventq		= false,
1370 	.want_pio		= true,
1371 };
1372