1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3  * Driver for Solarflare network controllers and boards
4  * Copyright 2018 Solarflare Communications Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation, incorporated herein by reference.
9  */
10 
11 #include "net_driver.h"
12 #include <linux/module.h>
13 #include <linux/filter.h>
14 #include "efx_channels.h"
15 #include "efx.h"
16 #include "efx_common.h"
17 #include "tx_common.h"
18 #include "rx_common.h"
19 #include "nic.h"
20 #include "sriov.h"
21 #include "workarounds.h"
22 
23 /* This is the first interrupt mode to try out of:
24  * 0 => MSI-X
25  * 1 => MSI
26  * 2 => legacy
27  */
28 unsigned int efx_siena_interrupt_mode = EFX_INT_MODE_MSIX;
29 
30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
31  * i.e. the number of CPUs among which we may distribute simultaneous
32  * interrupt handling.
33  *
34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
35  * The default (0) means to assign an interrupt to each core.
36  */
37 unsigned int efx_siena_rss_cpus;
38 
39 static unsigned int irq_adapt_low_thresh = 8000;
40 module_param(irq_adapt_low_thresh, uint, 0644);
41 MODULE_PARM_DESC(irq_adapt_low_thresh,
42 		 "Threshold score for reducing IRQ moderation");
43 
44 static unsigned int irq_adapt_high_thresh = 16000;
45 module_param(irq_adapt_high_thresh, uint, 0644);
46 MODULE_PARM_DESC(irq_adapt_high_thresh,
47 		 "Threshold score for increasing IRQ moderation");
48 
49 static const struct efx_channel_type efx_default_channel_type;
50 
51 /*************
52  * INTERRUPTS
53  *************/
54 
55 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
56 {
57 	cpumask_var_t filter_mask;
58 	unsigned int count;
59 	int cpu;
60 
61 	if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
62 		netif_warn(efx, probe, efx->net_dev,
63 			   "RSS disabled due to allocation failure\n");
64 		return 1;
65 	}
66 
67 	cpumask_copy(filter_mask, cpu_online_mask);
68 	if (local_node)
69 		cpumask_and(filter_mask, filter_mask,
70 			    cpumask_of_pcibus(efx->pci_dev->bus));
71 
72 	count = 0;
73 	for_each_cpu(cpu, filter_mask) {
74 		++count;
75 		cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
76 	}
77 
78 	free_cpumask_var(filter_mask);
79 
80 	return count;
81 }
82 
83 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
84 {
85 	unsigned int count;
86 
87 	if (efx_siena_rss_cpus) {
88 		count = efx_siena_rss_cpus;
89 	} else {
90 		count = count_online_cores(efx, true);
91 
92 		/* If no online CPUs in local node, fallback to any online CPUs */
93 		if (count == 0)
94 			count = count_online_cores(efx, false);
95 	}
96 
97 	if (count > EFX_MAX_RX_QUEUES) {
98 		netif_cond_dbg(efx, probe, efx->net_dev, !efx_siena_rss_cpus,
99 			       warn,
100 			       "Reducing number of rx queues from %u to %u.\n",
101 			       count, EFX_MAX_RX_QUEUES);
102 		count = EFX_MAX_RX_QUEUES;
103 	}
104 
105 	/* If RSS is requested for the PF *and* VFs then we can't write RSS
106 	 * table entries that are inaccessible to VFs
107 	 */
108 #ifdef CONFIG_SFC_SIENA_SRIOV
109 	if (efx->type->sriov_wanted) {
110 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
111 		    count > efx_vf_size(efx)) {
112 			netif_warn(efx, probe, efx->net_dev,
113 				   "Reducing number of RSS channels from %u to %u for "
114 				   "VF support. Increase vf-msix-limit to use more "
115 				   "channels on the PF.\n",
116 				   count, efx_vf_size(efx));
117 			count = efx_vf_size(efx);
118 		}
119 	}
120 #endif
121 
122 	return count;
123 }
124 
125 static int efx_allocate_msix_channels(struct efx_nic *efx,
126 				      unsigned int max_channels,
127 				      unsigned int extra_channels,
128 				      unsigned int parallelism)
129 {
130 	unsigned int n_channels = parallelism;
131 	int vec_count;
132 	int tx_per_ev;
133 	int n_xdp_tx;
134 	int n_xdp_ev;
135 
136 	if (efx_siena_separate_tx_channels)
137 		n_channels *= 2;
138 	n_channels += extra_channels;
139 
140 	/* To allow XDP transmit to happen from arbitrary NAPI contexts
141 	 * we allocate a TX queue per CPU. We share event queues across
142 	 * multiple tx queues, assuming tx and ev queues are both
143 	 * maximum size.
144 	 */
145 	tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
146 	tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
147 	n_xdp_tx = num_possible_cpus();
148 	n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
149 
150 	vec_count = pci_msix_vec_count(efx->pci_dev);
151 	if (vec_count < 0)
152 		return vec_count;
153 
154 	max_channels = min_t(unsigned int, vec_count, max_channels);
155 
156 	/* Check resources.
157 	 * We need a channel per event queue, plus a VI per tx queue.
158 	 * This may be more pessimistic than it needs to be.
159 	 */
160 	if (n_channels >= max_channels) {
161 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
162 		netif_warn(efx, drv, efx->net_dev,
163 			   "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
164 			   n_xdp_ev, n_channels, max_channels);
165 		netif_warn(efx, drv, efx->net_dev,
166 			   "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
167 	} else if (n_channels + n_xdp_tx > efx->max_vis) {
168 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
169 		netif_warn(efx, drv, efx->net_dev,
170 			   "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
171 			   n_xdp_tx, n_channels, efx->max_vis);
172 		netif_warn(efx, drv, efx->net_dev,
173 			   "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
174 	} else if (n_channels + n_xdp_ev > max_channels) {
175 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
176 		netif_warn(efx, drv, efx->net_dev,
177 			   "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
178 			   n_xdp_ev, n_channels, max_channels);
179 
180 		n_xdp_ev = max_channels - n_channels;
181 		netif_warn(efx, drv, efx->net_dev,
182 			   "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
183 			   DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
184 	} else {
185 		efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
186 	}
187 
188 	if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
189 		efx->n_xdp_channels = n_xdp_ev;
190 		efx->xdp_tx_per_channel = tx_per_ev;
191 		efx->xdp_tx_queue_count = n_xdp_tx;
192 		n_channels += n_xdp_ev;
193 		netif_dbg(efx, drv, efx->net_dev,
194 			  "Allocating %d TX and %d event queues for XDP\n",
195 			  n_xdp_ev * tx_per_ev, n_xdp_ev);
196 	} else {
197 		efx->n_xdp_channels = 0;
198 		efx->xdp_tx_per_channel = 0;
199 		efx->xdp_tx_queue_count = n_xdp_tx;
200 	}
201 
202 	if (vec_count < n_channels) {
203 		netif_err(efx, drv, efx->net_dev,
204 			  "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
205 			  vec_count, n_channels);
206 		netif_err(efx, drv, efx->net_dev,
207 			  "WARNING: Performance may be reduced.\n");
208 		n_channels = vec_count;
209 	}
210 
211 	n_channels = min(n_channels, max_channels);
212 
213 	efx->n_channels = n_channels;
214 
215 	/* Ignore XDP tx channels when creating rx channels. */
216 	n_channels -= efx->n_xdp_channels;
217 
218 	if (efx_siena_separate_tx_channels) {
219 		efx->n_tx_channels =
220 			min(max(n_channels / 2, 1U),
221 			    efx->max_tx_channels);
222 		efx->tx_channel_offset =
223 			n_channels - efx->n_tx_channels;
224 		efx->n_rx_channels =
225 			max(n_channels -
226 			    efx->n_tx_channels, 1U);
227 	} else {
228 		efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
229 		efx->tx_channel_offset = 0;
230 		efx->n_rx_channels = n_channels;
231 	}
232 
233 	efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
234 	efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
235 
236 	efx->xdp_channel_offset = n_channels;
237 
238 	netif_dbg(efx, drv, efx->net_dev,
239 		  "Allocating %u RX channels\n",
240 		  efx->n_rx_channels);
241 
242 	return efx->n_channels;
243 }
244 
245 /* Probe the number and type of interrupts we are able to obtain, and
246  * the resulting numbers of channels and RX queues.
247  */
248 int efx_siena_probe_interrupts(struct efx_nic *efx)
249 {
250 	unsigned int extra_channels = 0;
251 	unsigned int rss_spread;
252 	unsigned int i, j;
253 	int rc;
254 
255 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
256 		if (efx->extra_channel_type[i])
257 			++extra_channels;
258 
259 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
260 		unsigned int parallelism = efx_wanted_parallelism(efx);
261 		struct msix_entry xentries[EFX_MAX_CHANNELS];
262 		unsigned int n_channels;
263 
264 		rc = efx_allocate_msix_channels(efx, efx->max_channels,
265 						extra_channels, parallelism);
266 		if (rc >= 0) {
267 			n_channels = rc;
268 			for (i = 0; i < n_channels; i++)
269 				xentries[i].entry = i;
270 			rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
271 						   n_channels);
272 		}
273 		if (rc < 0) {
274 			/* Fall back to single channel MSI */
275 			netif_err(efx, drv, efx->net_dev,
276 				  "could not enable MSI-X\n");
277 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
278 				efx->interrupt_mode = EFX_INT_MODE_MSI;
279 			else
280 				return rc;
281 		} else if (rc < n_channels) {
282 			netif_err(efx, drv, efx->net_dev,
283 				  "WARNING: Insufficient MSI-X vectors"
284 				  " available (%d < %u).\n", rc, n_channels);
285 			netif_err(efx, drv, efx->net_dev,
286 				  "WARNING: Performance may be reduced.\n");
287 			n_channels = rc;
288 		}
289 
290 		if (rc > 0) {
291 			for (i = 0; i < efx->n_channels; i++)
292 				efx_get_channel(efx, i)->irq =
293 					xentries[i].vector;
294 		}
295 	}
296 
297 	/* Try single interrupt MSI */
298 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
299 		efx->n_channels = 1;
300 		efx->n_rx_channels = 1;
301 		efx->n_tx_channels = 1;
302 		efx->tx_channel_offset = 0;
303 		efx->n_xdp_channels = 0;
304 		efx->xdp_channel_offset = efx->n_channels;
305 		rc = pci_enable_msi(efx->pci_dev);
306 		if (rc == 0) {
307 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
308 		} else {
309 			netif_err(efx, drv, efx->net_dev,
310 				  "could not enable MSI\n");
311 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
312 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;
313 			else
314 				return rc;
315 		}
316 	}
317 
318 	/* Assume legacy interrupts */
319 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
320 		efx->n_channels = 1 + (efx_siena_separate_tx_channels ? 1 : 0);
321 		efx->n_rx_channels = 1;
322 		efx->n_tx_channels = 1;
323 		efx->tx_channel_offset = 1;
324 		efx->n_xdp_channels = 0;
325 		efx->xdp_channel_offset = efx->n_channels;
326 		efx->legacy_irq = efx->pci_dev->irq;
327 	}
328 
329 	/* Assign extra channels if possible, before XDP channels */
330 	efx->n_extra_tx_channels = 0;
331 	j = efx->xdp_channel_offset;
332 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
333 		if (!efx->extra_channel_type[i])
334 			continue;
335 		if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
336 			efx->extra_channel_type[i]->handle_no_channel(efx);
337 		} else {
338 			--j;
339 			efx_get_channel(efx, j)->type =
340 				efx->extra_channel_type[i];
341 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
342 				efx->n_extra_tx_channels++;
343 		}
344 	}
345 
346 	rss_spread = efx->n_rx_channels;
347 	/* RSS might be usable on VFs even if it is disabled on the PF */
348 #ifdef CONFIG_SFC_SIENA_SRIOV
349 	if (efx->type->sriov_wanted) {
350 		efx->rss_spread = ((rss_spread > 1 ||
351 				    !efx->type->sriov_wanted(efx)) ?
352 				   rss_spread : efx_vf_size(efx));
353 		return 0;
354 	}
355 #endif
356 	efx->rss_spread = rss_spread;
357 
358 	return 0;
359 }
360 
361 #if defined(CONFIG_SMP)
362 void efx_siena_set_interrupt_affinity(struct efx_nic *efx)
363 {
364 	const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus);
365 	struct efx_channel *channel;
366 	unsigned int cpu;
367 
368 	/* If no online CPUs in local node, fallback to any online CPU */
369 	if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
370 		numa_mask = cpu_online_mask;
371 
372 	cpu = -1;
373 	efx_for_each_channel(channel, efx) {
374 		cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
375 		if (cpu >= nr_cpu_ids)
376 			cpu = cpumask_first_and(cpu_online_mask, numa_mask);
377 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
378 	}
379 }
380 
381 void efx_siena_clear_interrupt_affinity(struct efx_nic *efx)
382 {
383 	struct efx_channel *channel;
384 
385 	efx_for_each_channel(channel, efx)
386 		irq_set_affinity_hint(channel->irq, NULL);
387 }
388 #else
389 void
390 efx_siena_set_interrupt_affinity(struct efx_nic *efx __always_unused)
391 {
392 }
393 
394 void
395 efx_siena_clear_interrupt_affinity(struct efx_nic *efx __always_unused)
396 {
397 }
398 #endif /* CONFIG_SMP */
399 
400 void efx_siena_remove_interrupts(struct efx_nic *efx)
401 {
402 	struct efx_channel *channel;
403 
404 	/* Remove MSI/MSI-X interrupts */
405 	efx_for_each_channel(channel, efx)
406 		channel->irq = 0;
407 	pci_disable_msi(efx->pci_dev);
408 	pci_disable_msix(efx->pci_dev);
409 
410 	/* Remove legacy interrupt */
411 	efx->legacy_irq = 0;
412 }
413 
414 /***************
415  * EVENT QUEUES
416  ***************/
417 
418 /* Create event queue
419  * Event queue memory allocations are done only once.  If the channel
420  * is reset, the memory buffer will be reused; this guards against
421  * errors during channel reset and also simplifies interrupt handling.
422  */
423 static int efx_probe_eventq(struct efx_channel *channel)
424 {
425 	struct efx_nic *efx = channel->efx;
426 	unsigned long entries;
427 
428 	netif_dbg(efx, probe, efx->net_dev,
429 		  "chan %d create event queue\n", channel->channel);
430 
431 	/* Build an event queue with room for one event per tx and rx buffer,
432 	 * plus some extra for link state events and MCDI completions.
433 	 */
434 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
435 	EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
436 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
437 
438 	return efx_nic_probe_eventq(channel);
439 }
440 
441 /* Prepare channel's event queue */
442 static int efx_init_eventq(struct efx_channel *channel)
443 {
444 	struct efx_nic *efx = channel->efx;
445 	int rc;
446 
447 	EFX_WARN_ON_PARANOID(channel->eventq_init);
448 
449 	netif_dbg(efx, drv, efx->net_dev,
450 		  "chan %d init event queue\n", channel->channel);
451 
452 	rc = efx_nic_init_eventq(channel);
453 	if (rc == 0) {
454 		efx->type->push_irq_moderation(channel);
455 		channel->eventq_read_ptr = 0;
456 		channel->eventq_init = true;
457 	}
458 	return rc;
459 }
460 
461 /* Enable event queue processing and NAPI */
462 void efx_siena_start_eventq(struct efx_channel *channel)
463 {
464 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
465 		  "chan %d start event queue\n", channel->channel);
466 
467 	/* Make sure the NAPI handler sees the enabled flag set */
468 	channel->enabled = true;
469 	smp_wmb();
470 
471 	napi_enable(&channel->napi_str);
472 	efx_nic_eventq_read_ack(channel);
473 }
474 
475 /* Disable event queue processing and NAPI */
476 void efx_siena_stop_eventq(struct efx_channel *channel)
477 {
478 	if (!channel->enabled)
479 		return;
480 
481 	napi_disable(&channel->napi_str);
482 	channel->enabled = false;
483 }
484 
485 static void efx_fini_eventq(struct efx_channel *channel)
486 {
487 	if (!channel->eventq_init)
488 		return;
489 
490 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
491 		  "chan %d fini event queue\n", channel->channel);
492 
493 	efx_nic_fini_eventq(channel);
494 	channel->eventq_init = false;
495 }
496 
497 static void efx_remove_eventq(struct efx_channel *channel)
498 {
499 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
500 		  "chan %d remove event queue\n", channel->channel);
501 
502 	efx_nic_remove_eventq(channel);
503 }
504 
505 /**************************************************************************
506  *
507  * Channel handling
508  *
509  *************************************************************************/
510 
511 #ifdef CONFIG_RFS_ACCEL
512 static void efx_filter_rfs_expire(struct work_struct *data)
513 {
514 	struct delayed_work *dwork = to_delayed_work(data);
515 	struct efx_channel *channel;
516 	unsigned int time, quota;
517 
518 	channel = container_of(dwork, struct efx_channel, filter_work);
519 	time = jiffies - channel->rfs_last_expiry;
520 	quota = channel->rfs_filter_count * time / (30 * HZ);
521 	if (quota >= 20 && __efx_siena_filter_rfs_expire(channel,
522 					min(channel->rfs_filter_count, quota)))
523 		channel->rfs_last_expiry += time;
524 	/* Ensure we do more work eventually even if NAPI poll is not happening */
525 	schedule_delayed_work(dwork, 30 * HZ);
526 }
527 #endif
528 
529 /* Allocate and initialise a channel structure. */
530 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
531 {
532 	struct efx_rx_queue *rx_queue;
533 	struct efx_tx_queue *tx_queue;
534 	struct efx_channel *channel;
535 	int j;
536 
537 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
538 	if (!channel)
539 		return NULL;
540 
541 	channel->efx = efx;
542 	channel->channel = i;
543 	channel->type = &efx_default_channel_type;
544 
545 	for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
546 		tx_queue = &channel->tx_queue[j];
547 		tx_queue->efx = efx;
548 		tx_queue->queue = -1;
549 		tx_queue->label = j;
550 		tx_queue->channel = channel;
551 	}
552 
553 #ifdef CONFIG_RFS_ACCEL
554 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
555 #endif
556 
557 	rx_queue = &channel->rx_queue;
558 	rx_queue->efx = efx;
559 	timer_setup(&rx_queue->slow_fill, efx_siena_rx_slow_fill, 0);
560 
561 	return channel;
562 }
563 
564 int efx_siena_init_channels(struct efx_nic *efx)
565 {
566 	unsigned int i;
567 
568 	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
569 		efx->channel[i] = efx_alloc_channel(efx, i);
570 		if (!efx->channel[i])
571 			return -ENOMEM;
572 		efx->msi_context[i].efx = efx;
573 		efx->msi_context[i].index = i;
574 	}
575 
576 	/* Higher numbered interrupt modes are less capable! */
577 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,
578 				  efx_siena_interrupt_mode);
579 
580 	efx->max_channels = EFX_MAX_CHANNELS;
581 	efx->max_tx_channels = EFX_MAX_CHANNELS;
582 
583 	return 0;
584 }
585 
586 void efx_siena_fini_channels(struct efx_nic *efx)
587 {
588 	unsigned int i;
589 
590 	for (i = 0; i < EFX_MAX_CHANNELS; i++)
591 		if (efx->channel[i]) {
592 			kfree(efx->channel[i]);
593 			efx->channel[i] = NULL;
594 		}
595 }
596 
597 /* Allocate and initialise a channel structure, copying parameters
598  * (but not resources) from an old channel structure.
599  */
600 static
601 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
602 {
603 	struct efx_rx_queue *rx_queue;
604 	struct efx_tx_queue *tx_queue;
605 	struct efx_channel *channel;
606 	int j;
607 
608 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
609 	if (!channel)
610 		return NULL;
611 
612 	*channel = *old_channel;
613 
614 	channel->napi_dev = NULL;
615 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
616 	channel->napi_str.napi_id = 0;
617 	channel->napi_str.state = 0;
618 	memset(&channel->eventq, 0, sizeof(channel->eventq));
619 
620 	for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
621 		tx_queue = &channel->tx_queue[j];
622 		if (tx_queue->channel)
623 			tx_queue->channel = channel;
624 		tx_queue->buffer = NULL;
625 		tx_queue->cb_page = NULL;
626 		memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
627 	}
628 
629 	rx_queue = &channel->rx_queue;
630 	rx_queue->buffer = NULL;
631 	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
632 	timer_setup(&rx_queue->slow_fill, efx_siena_rx_slow_fill, 0);
633 #ifdef CONFIG_RFS_ACCEL
634 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
635 #endif
636 
637 	return channel;
638 }
639 
640 static int efx_probe_channel(struct efx_channel *channel)
641 {
642 	struct efx_tx_queue *tx_queue;
643 	struct efx_rx_queue *rx_queue;
644 	int rc;
645 
646 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
647 		  "creating channel %d\n", channel->channel);
648 
649 	rc = channel->type->pre_probe(channel);
650 	if (rc)
651 		goto fail;
652 
653 	rc = efx_probe_eventq(channel);
654 	if (rc)
655 		goto fail;
656 
657 	efx_for_each_channel_tx_queue(tx_queue, channel) {
658 		rc = efx_siena_probe_tx_queue(tx_queue);
659 		if (rc)
660 			goto fail;
661 	}
662 
663 	efx_for_each_channel_rx_queue(rx_queue, channel) {
664 		rc = efx_siena_probe_rx_queue(rx_queue);
665 		if (rc)
666 			goto fail;
667 	}
668 
669 	channel->rx_list = NULL;
670 
671 	return 0;
672 
673 fail:
674 	efx_siena_remove_channel(channel);
675 	return rc;
676 }
677 
678 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
679 				 size_t len)
680 {
681 	struct efx_nic *efx = channel->efx;
682 	const char *type;
683 	int number;
684 
685 	number = channel->channel;
686 
687 	if (number >= efx->xdp_channel_offset &&
688 	    !WARN_ON_ONCE(!efx->n_xdp_channels)) {
689 		type = "-xdp";
690 		number -= efx->xdp_channel_offset;
691 	} else if (efx->tx_channel_offset == 0) {
692 		type = "";
693 	} else if (number < efx->tx_channel_offset) {
694 		type = "-rx";
695 	} else {
696 		type = "-tx";
697 		number -= efx->tx_channel_offset;
698 	}
699 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);
700 }
701 
702 void efx_siena_set_channel_names(struct efx_nic *efx)
703 {
704 	struct efx_channel *channel;
705 
706 	efx_for_each_channel(channel, efx)
707 		channel->type->get_name(channel,
708 					efx->msi_context[channel->channel].name,
709 					sizeof(efx->msi_context[0].name));
710 }
711 
712 int efx_siena_probe_channels(struct efx_nic *efx)
713 {
714 	struct efx_channel *channel;
715 	int rc;
716 
717 	/* Restart special buffer allocation */
718 	efx->next_buffer_table = 0;
719 
720 	/* Probe channels in reverse, so that any 'extra' channels
721 	 * use the start of the buffer table. This allows the traffic
722 	 * channels to be resized without moving them or wasting the
723 	 * entries before them.
724 	 */
725 	efx_for_each_channel_rev(channel, efx) {
726 		rc = efx_probe_channel(channel);
727 		if (rc) {
728 			netif_err(efx, probe, efx->net_dev,
729 				  "failed to create channel %d\n",
730 				  channel->channel);
731 			goto fail;
732 		}
733 	}
734 	efx_siena_set_channel_names(efx);
735 
736 	return 0;
737 
738 fail:
739 	efx_siena_remove_channels(efx);
740 	return rc;
741 }
742 
743 void efx_siena_remove_channel(struct efx_channel *channel)
744 {
745 	struct efx_tx_queue *tx_queue;
746 	struct efx_rx_queue *rx_queue;
747 
748 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
749 		  "destroy chan %d\n", channel->channel);
750 
751 	efx_for_each_channel_rx_queue(rx_queue, channel)
752 		efx_siena_remove_rx_queue(rx_queue);
753 	efx_for_each_channel_tx_queue(tx_queue, channel)
754 		efx_siena_remove_tx_queue(tx_queue);
755 	efx_remove_eventq(channel);
756 	channel->type->post_remove(channel);
757 }
758 
759 void efx_siena_remove_channels(struct efx_nic *efx)
760 {
761 	struct efx_channel *channel;
762 
763 	efx_for_each_channel(channel, efx)
764 		efx_siena_remove_channel(channel);
765 
766 	kfree(efx->xdp_tx_queues);
767 }
768 
769 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
770 				struct efx_tx_queue *tx_queue)
771 {
772 	if (xdp_queue_number >= efx->xdp_tx_queue_count)
773 		return -EINVAL;
774 
775 	netif_dbg(efx, drv, efx->net_dev,
776 		  "Channel %u TXQ %u is XDP %u, HW %u\n",
777 		  tx_queue->channel->channel, tx_queue->label,
778 		  xdp_queue_number, tx_queue->queue);
779 	efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
780 	return 0;
781 }
782 
783 static void efx_set_xdp_channels(struct efx_nic *efx)
784 {
785 	struct efx_tx_queue *tx_queue;
786 	struct efx_channel *channel;
787 	unsigned int next_queue = 0;
788 	int xdp_queue_number = 0;
789 	int rc;
790 
791 	/* We need to mark which channels really have RX and TX
792 	 * queues, and adjust the TX queue numbers if we have separate
793 	 * RX-only and TX-only channels.
794 	 */
795 	efx_for_each_channel(channel, efx) {
796 		if (channel->channel < efx->tx_channel_offset)
797 			continue;
798 
799 		if (efx_channel_is_xdp_tx(channel)) {
800 			efx_for_each_channel_tx_queue(tx_queue, channel) {
801 				tx_queue->queue = next_queue++;
802 				rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
803 							  tx_queue);
804 				if (rc == 0)
805 					xdp_queue_number++;
806 			}
807 		} else {
808 			efx_for_each_channel_tx_queue(tx_queue, channel) {
809 				tx_queue->queue = next_queue++;
810 				netif_dbg(efx, drv, efx->net_dev,
811 					  "Channel %u TXQ %u is HW %u\n",
812 					  channel->channel, tx_queue->label,
813 					  tx_queue->queue);
814 			}
815 
816 			/* If XDP is borrowing queues from net stack, it must
817 			 * use the queue with no csum offload, which is the
818 			 * first one of the channel
819 			 * (note: tx_queue_by_type is not initialized yet)
820 			 */
821 			if (efx->xdp_txq_queues_mode ==
822 			    EFX_XDP_TX_QUEUES_BORROWED) {
823 				tx_queue = &channel->tx_queue[0];
824 				rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
825 							  tx_queue);
826 				if (rc == 0)
827 					xdp_queue_number++;
828 			}
829 		}
830 	}
831 	WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
832 		xdp_queue_number != efx->xdp_tx_queue_count);
833 	WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
834 		xdp_queue_number > efx->xdp_tx_queue_count);
835 
836 	/* If we have more CPUs than assigned XDP TX queues, assign the already
837 	 * existing queues to the exceeding CPUs
838 	 */
839 	next_queue = 0;
840 	while (xdp_queue_number < efx->xdp_tx_queue_count) {
841 		tx_queue = efx->xdp_tx_queues[next_queue++];
842 		rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
843 		if (rc == 0)
844 			xdp_queue_number++;
845 	}
846 }
847 
848 static int efx_soft_enable_interrupts(struct efx_nic *efx);
849 static void efx_soft_disable_interrupts(struct efx_nic *efx);
850 static void efx_init_napi_channel(struct efx_channel *channel);
851 static void efx_fini_napi_channel(struct efx_channel *channel);
852 
853 int efx_siena_realloc_channels(struct efx_nic *efx, u32 rxq_entries,
854 			       u32 txq_entries)
855 {
856 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
857 	unsigned int i, next_buffer_table = 0;
858 	u32 old_rxq_entries, old_txq_entries;
859 	int rc, rc2;
860 
861 	rc = efx_check_disabled(efx);
862 	if (rc)
863 		return rc;
864 
865 	/* Not all channels should be reallocated. We must avoid
866 	 * reallocating their buffer table entries.
867 	 */
868 	efx_for_each_channel(channel, efx) {
869 		struct efx_rx_queue *rx_queue;
870 		struct efx_tx_queue *tx_queue;
871 
872 		if (channel->type->copy)
873 			continue;
874 		next_buffer_table = max(next_buffer_table,
875 					channel->eventq.index +
876 					channel->eventq.entries);
877 		efx_for_each_channel_rx_queue(rx_queue, channel)
878 			next_buffer_table = max(next_buffer_table,
879 						rx_queue->rxd.index +
880 						rx_queue->rxd.entries);
881 		efx_for_each_channel_tx_queue(tx_queue, channel)
882 			next_buffer_table = max(next_buffer_table,
883 						tx_queue->txd.index +
884 						tx_queue->txd.entries);
885 	}
886 
887 	efx_device_detach_sync(efx);
888 	efx_siena_stop_all(efx);
889 	efx_soft_disable_interrupts(efx);
890 
891 	/* Clone channels (where possible) */
892 	memset(other_channel, 0, sizeof(other_channel));
893 	for (i = 0; i < efx->n_channels; i++) {
894 		channel = efx->channel[i];
895 		if (channel->type->copy)
896 			channel = channel->type->copy(channel);
897 		if (!channel) {
898 			rc = -ENOMEM;
899 			goto out;
900 		}
901 		other_channel[i] = channel;
902 	}
903 
904 	/* Swap entry counts and channel pointers */
905 	old_rxq_entries = efx->rxq_entries;
906 	old_txq_entries = efx->txq_entries;
907 	efx->rxq_entries = rxq_entries;
908 	efx->txq_entries = txq_entries;
909 	for (i = 0; i < efx->n_channels; i++)
910 		swap(efx->channel[i], other_channel[i]);
911 
912 	/* Restart buffer table allocation */
913 	efx->next_buffer_table = next_buffer_table;
914 
915 	for (i = 0; i < efx->n_channels; i++) {
916 		channel = efx->channel[i];
917 		if (!channel->type->copy)
918 			continue;
919 		rc = efx_probe_channel(channel);
920 		if (rc)
921 			goto rollback;
922 		efx_init_napi_channel(efx->channel[i]);
923 	}
924 
925 	efx_set_xdp_channels(efx);
926 out:
927 	/* Destroy unused channel structures */
928 	for (i = 0; i < efx->n_channels; i++) {
929 		channel = other_channel[i];
930 		if (channel && channel->type->copy) {
931 			efx_fini_napi_channel(channel);
932 			efx_siena_remove_channel(channel);
933 			kfree(channel);
934 		}
935 	}
936 
937 	rc2 = efx_soft_enable_interrupts(efx);
938 	if (rc2) {
939 		rc = rc ? rc : rc2;
940 		netif_err(efx, drv, efx->net_dev,
941 			  "unable to restart interrupts on channel reallocation\n");
942 		efx_siena_schedule_reset(efx, RESET_TYPE_DISABLE);
943 	} else {
944 		efx_siena_start_all(efx);
945 		efx_device_attach_if_not_resetting(efx);
946 	}
947 	return rc;
948 
949 rollback:
950 	/* Swap back */
951 	efx->rxq_entries = old_rxq_entries;
952 	efx->txq_entries = old_txq_entries;
953 	for (i = 0; i < efx->n_channels; i++)
954 		swap(efx->channel[i], other_channel[i]);
955 	goto out;
956 }
957 
958 int efx_siena_set_channels(struct efx_nic *efx)
959 {
960 	struct efx_channel *channel;
961 	int rc;
962 
963 	if (efx->xdp_tx_queue_count) {
964 		EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
965 
966 		/* Allocate array for XDP TX queue lookup. */
967 		efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
968 					     sizeof(*efx->xdp_tx_queues),
969 					     GFP_KERNEL);
970 		if (!efx->xdp_tx_queues)
971 			return -ENOMEM;
972 	}
973 
974 	efx_for_each_channel(channel, efx) {
975 		if (channel->channel < efx->n_rx_channels)
976 			channel->rx_queue.core_index = channel->channel;
977 		else
978 			channel->rx_queue.core_index = -1;
979 	}
980 
981 	efx_set_xdp_channels(efx);
982 
983 	rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
984 	if (rc)
985 		return rc;
986 	return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
987 }
988 
989 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
990 {
991 	return channel->channel - channel->efx->tx_channel_offset <
992 		channel->efx->n_tx_channels;
993 }
994 
995 /*************
996  * START/STOP
997  *************/
998 
999 static int efx_soft_enable_interrupts(struct efx_nic *efx)
1000 {
1001 	struct efx_channel *channel, *end_channel;
1002 	int rc;
1003 
1004 	BUG_ON(efx->state == STATE_DISABLED);
1005 
1006 	efx->irq_soft_enabled = true;
1007 	smp_wmb();
1008 
1009 	efx_for_each_channel(channel, efx) {
1010 		if (!channel->type->keep_eventq) {
1011 			rc = efx_init_eventq(channel);
1012 			if (rc)
1013 				goto fail;
1014 		}
1015 		efx_siena_start_eventq(channel);
1016 	}
1017 
1018 	efx_siena_mcdi_mode_event(efx);
1019 
1020 	return 0;
1021 fail:
1022 	end_channel = channel;
1023 	efx_for_each_channel(channel, efx) {
1024 		if (channel == end_channel)
1025 			break;
1026 		efx_siena_stop_eventq(channel);
1027 		if (!channel->type->keep_eventq)
1028 			efx_fini_eventq(channel);
1029 	}
1030 
1031 	return rc;
1032 }
1033 
1034 static void efx_soft_disable_interrupts(struct efx_nic *efx)
1035 {
1036 	struct efx_channel *channel;
1037 
1038 	if (efx->state == STATE_DISABLED)
1039 		return;
1040 
1041 	efx_siena_mcdi_mode_poll(efx);
1042 
1043 	efx->irq_soft_enabled = false;
1044 	smp_wmb();
1045 
1046 	if (efx->legacy_irq)
1047 		synchronize_irq(efx->legacy_irq);
1048 
1049 	efx_for_each_channel(channel, efx) {
1050 		if (channel->irq)
1051 			synchronize_irq(channel->irq);
1052 
1053 		efx_siena_stop_eventq(channel);
1054 		if (!channel->type->keep_eventq)
1055 			efx_fini_eventq(channel);
1056 	}
1057 
1058 	/* Flush the asynchronous MCDI request queue */
1059 	efx_siena_mcdi_flush_async(efx);
1060 }
1061 
1062 int efx_siena_enable_interrupts(struct efx_nic *efx)
1063 {
1064 	struct efx_channel *channel, *end_channel;
1065 	int rc;
1066 
1067 	/* TODO: Is this really a bug? */
1068 	BUG_ON(efx->state == STATE_DISABLED);
1069 
1070 	if (efx->eeh_disabled_legacy_irq) {
1071 		enable_irq(efx->legacy_irq);
1072 		efx->eeh_disabled_legacy_irq = false;
1073 	}
1074 
1075 	efx->type->irq_enable_master(efx);
1076 
1077 	efx_for_each_channel(channel, efx) {
1078 		if (channel->type->keep_eventq) {
1079 			rc = efx_init_eventq(channel);
1080 			if (rc)
1081 				goto fail;
1082 		}
1083 	}
1084 
1085 	rc = efx_soft_enable_interrupts(efx);
1086 	if (rc)
1087 		goto fail;
1088 
1089 	return 0;
1090 
1091 fail:
1092 	end_channel = channel;
1093 	efx_for_each_channel(channel, efx) {
1094 		if (channel == end_channel)
1095 			break;
1096 		if (channel->type->keep_eventq)
1097 			efx_fini_eventq(channel);
1098 	}
1099 
1100 	efx->type->irq_disable_non_ev(efx);
1101 
1102 	return rc;
1103 }
1104 
1105 void efx_siena_disable_interrupts(struct efx_nic *efx)
1106 {
1107 	struct efx_channel *channel;
1108 
1109 	efx_soft_disable_interrupts(efx);
1110 
1111 	efx_for_each_channel(channel, efx) {
1112 		if (channel->type->keep_eventq)
1113 			efx_fini_eventq(channel);
1114 	}
1115 
1116 	efx->type->irq_disable_non_ev(efx);
1117 }
1118 
1119 void efx_siena_start_channels(struct efx_nic *efx)
1120 {
1121 	struct efx_tx_queue *tx_queue;
1122 	struct efx_rx_queue *rx_queue;
1123 	struct efx_channel *channel;
1124 
1125 	efx_for_each_channel_rev(channel, efx) {
1126 		efx_for_each_channel_tx_queue(tx_queue, channel) {
1127 			efx_siena_init_tx_queue(tx_queue);
1128 			atomic_inc(&efx->active_queues);
1129 		}
1130 
1131 		efx_for_each_channel_rx_queue(rx_queue, channel) {
1132 			efx_siena_init_rx_queue(rx_queue);
1133 			atomic_inc(&efx->active_queues);
1134 			efx_siena_stop_eventq(channel);
1135 			efx_siena_fast_push_rx_descriptors(rx_queue, false);
1136 			efx_siena_start_eventq(channel);
1137 		}
1138 
1139 		WARN_ON(channel->rx_pkt_n_frags);
1140 	}
1141 }
1142 
1143 void efx_siena_stop_channels(struct efx_nic *efx)
1144 {
1145 	struct efx_tx_queue *tx_queue;
1146 	struct efx_rx_queue *rx_queue;
1147 	struct efx_channel *channel;
1148 	int rc = 0;
1149 
1150 	/* Stop RX refill */
1151 	efx_for_each_channel(channel, efx) {
1152 		efx_for_each_channel_rx_queue(rx_queue, channel)
1153 			rx_queue->refill_enabled = false;
1154 	}
1155 
1156 	efx_for_each_channel(channel, efx) {
1157 		/* RX packet processing is pipelined, so wait for the
1158 		 * NAPI handler to complete.  At least event queue 0
1159 		 * might be kept active by non-data events, so don't
1160 		 * use napi_synchronize() but actually disable NAPI
1161 		 * temporarily.
1162 		 */
1163 		if (efx_channel_has_rx_queue(channel)) {
1164 			efx_siena_stop_eventq(channel);
1165 			efx_siena_start_eventq(channel);
1166 		}
1167 	}
1168 
1169 	if (efx->type->fini_dmaq)
1170 		rc = efx->type->fini_dmaq(efx);
1171 
1172 	if (rc) {
1173 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1174 	} else {
1175 		netif_dbg(efx, drv, efx->net_dev,
1176 			  "successfully flushed all queues\n");
1177 	}
1178 
1179 	efx_for_each_channel(channel, efx) {
1180 		efx_for_each_channel_rx_queue(rx_queue, channel)
1181 			efx_siena_fini_rx_queue(rx_queue);
1182 		efx_for_each_channel_tx_queue(tx_queue, channel)
1183 			efx_siena_fini_tx_queue(tx_queue);
1184 	}
1185 }
1186 
1187 /**************************************************************************
1188  *
1189  * NAPI interface
1190  *
1191  *************************************************************************/
1192 
1193 /* Process channel's event queue
1194  *
1195  * This function is responsible for processing the event queue of a
1196  * single channel.  The caller must guarantee that this function will
1197  * never be concurrently called more than once on the same channel,
1198  * though different channels may be being processed concurrently.
1199  */
1200 static int efx_process_channel(struct efx_channel *channel, int budget)
1201 {
1202 	struct efx_tx_queue *tx_queue;
1203 	struct list_head rx_list;
1204 	int spent;
1205 
1206 	if (unlikely(!channel->enabled))
1207 		return 0;
1208 
1209 	/* Prepare the batch receive list */
1210 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1211 	INIT_LIST_HEAD(&rx_list);
1212 	channel->rx_list = &rx_list;
1213 
1214 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1215 		tx_queue->pkts_compl = 0;
1216 		tx_queue->bytes_compl = 0;
1217 	}
1218 
1219 	spent = efx_nic_process_eventq(channel, budget);
1220 	if (spent && efx_channel_has_rx_queue(channel)) {
1221 		struct efx_rx_queue *rx_queue =
1222 			efx_channel_get_rx_queue(channel);
1223 
1224 		efx_rx_flush_packet(channel);
1225 		efx_siena_fast_push_rx_descriptors(rx_queue, true);
1226 	}
1227 
1228 	/* Update BQL */
1229 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1230 		if (tx_queue->bytes_compl) {
1231 			netdev_tx_completed_queue(tx_queue->core_txq,
1232 						  tx_queue->pkts_compl,
1233 						  tx_queue->bytes_compl);
1234 		}
1235 	}
1236 
1237 	/* Receive any packets we queued up */
1238 	netif_receive_skb_list(channel->rx_list);
1239 	channel->rx_list = NULL;
1240 
1241 	return spent;
1242 }
1243 
1244 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1245 {
1246 	int step = efx->irq_mod_step_us;
1247 
1248 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
1249 		if (channel->irq_moderation_us > step) {
1250 			channel->irq_moderation_us -= step;
1251 			efx->type->push_irq_moderation(channel);
1252 		}
1253 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1254 		if (channel->irq_moderation_us <
1255 		    efx->irq_rx_moderation_us) {
1256 			channel->irq_moderation_us += step;
1257 			efx->type->push_irq_moderation(channel);
1258 		}
1259 	}
1260 
1261 	channel->irq_count = 0;
1262 	channel->irq_mod_score = 0;
1263 }
1264 
1265 /* NAPI poll handler
1266  *
1267  * NAPI guarantees serialisation of polls of the same device, which
1268  * provides the guarantee required by efx_process_channel().
1269  */
1270 static int efx_poll(struct napi_struct *napi, int budget)
1271 {
1272 	struct efx_channel *channel =
1273 		container_of(napi, struct efx_channel, napi_str);
1274 	struct efx_nic *efx = channel->efx;
1275 #ifdef CONFIG_RFS_ACCEL
1276 	unsigned int time;
1277 #endif
1278 	int spent;
1279 
1280 	netif_vdbg(efx, intr, efx->net_dev,
1281 		   "channel %d NAPI poll executing on CPU %d\n",
1282 		   channel->channel, raw_smp_processor_id());
1283 
1284 	spent = efx_process_channel(channel, budget);
1285 
1286 	xdp_do_flush_map();
1287 
1288 	if (spent < budget) {
1289 		if (efx_channel_has_rx_queue(channel) &&
1290 		    efx->irq_rx_adaptive &&
1291 		    unlikely(++channel->irq_count == 1000)) {
1292 			efx_update_irq_mod(efx, channel);
1293 		}
1294 
1295 #ifdef CONFIG_RFS_ACCEL
1296 		/* Perhaps expire some ARFS filters */
1297 		time = jiffies - channel->rfs_last_expiry;
1298 		/* Would our quota be >= 20? */
1299 		if (channel->rfs_filter_count * time >= 600 * HZ)
1300 			mod_delayed_work(system_wq, &channel->filter_work, 0);
1301 #endif
1302 
1303 		/* There is no race here; although napi_disable() will
1304 		 * only wait for napi_complete(), this isn't a problem
1305 		 * since efx_nic_eventq_read_ack() will have no effect if
1306 		 * interrupts have already been disabled.
1307 		 */
1308 		if (napi_complete_done(napi, spent))
1309 			efx_nic_eventq_read_ack(channel);
1310 	}
1311 
1312 	return spent;
1313 }
1314 
1315 static void efx_init_napi_channel(struct efx_channel *channel)
1316 {
1317 	struct efx_nic *efx = channel->efx;
1318 
1319 	channel->napi_dev = efx->net_dev;
1320 	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, 64);
1321 }
1322 
1323 void efx_siena_init_napi(struct efx_nic *efx)
1324 {
1325 	struct efx_channel *channel;
1326 
1327 	efx_for_each_channel(channel, efx)
1328 		efx_init_napi_channel(channel);
1329 }
1330 
1331 static void efx_fini_napi_channel(struct efx_channel *channel)
1332 {
1333 	if (channel->napi_dev)
1334 		netif_napi_del(&channel->napi_str);
1335 
1336 	channel->napi_dev = NULL;
1337 }
1338 
1339 void efx_siena_fini_napi(struct efx_nic *efx)
1340 {
1341 	struct efx_channel *channel;
1342 
1343 	efx_for_each_channel(channel, efx)
1344 		efx_fini_napi_channel(channel);
1345 }
1346 
1347 /***************
1348  * Housekeeping
1349  ***************/
1350 
1351 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1352 {
1353 	return 0;
1354 }
1355 
1356 void efx_siena_channel_dummy_op_void(struct efx_channel *channel)
1357 {
1358 }
1359 
1360 static const struct efx_channel_type efx_default_channel_type = {
1361 	.pre_probe		= efx_channel_dummy_op_int,
1362 	.post_remove		= efx_siena_channel_dummy_op_void,
1363 	.get_name		= efx_get_channel_name,
1364 	.copy			= efx_copy_channel,
1365 	.want_txqs		= efx_default_channel_want_txqs,
1366 	.keep_eventq		= false,
1367 	.want_pio		= true,
1368 };
1369