190 static void ef4_remove_channel(struct ef4_channel *channel);
194 static void ef4_init_napi_channel(struct ef4_channel *channel);
196 static void ef4_fini_napi_channel(struct ef4_channel *channel);
232 static int ef4_process_channel(struct ef4_channel *channel, int budget)  in ef4_process_channel()  argument
237 	if (unlikely(!channel->enabled))  in ef4_process_channel()
240 	ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_process_channel()
245 	spent = ef4_nic_process_eventq(channel, budget);  in ef4_process_channel()
246 	if (spent && ef4_channel_has_rx_queue(channel)) {  in ef4_process_channel()
248 			ef4_channel_get_rx_queue(channel);  in ef4_process_channel()
250 		ef4_rx_flush_packet(channel);  in ef4_process_channel()
255 	ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_process_channel()
270 static void ef4_update_irq_mod(struct ef4_nic *efx, struct ef4_channel *channel)  in ef4_update_irq_mod()  argument
274 	if (channel->irq_mod_score < irq_adapt_low_thresh) {  in ef4_update_irq_mod()
275 		if (channel->irq_moderation_us > step) {  in ef4_update_irq_mod()
276 			channel->irq_moderation_us -= step;  in ef4_update_irq_mod()
277 			efx->type->push_irq_moderation(channel);  in ef4_update_irq_mod()
279 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {  in ef4_update_irq_mod()
280 		if (channel->irq_moderation_us <  in ef4_update_irq_mod()
282 			channel->irq_moderation_us += step;  in ef4_update_irq_mod()
283 			efx->type->push_irq_moderation(channel);  in ef4_update_irq_mod()
287 	channel->irq_count = 0;  in ef4_update_irq_mod()
288 	channel->irq_mod_score = 0;  in ef4_update_irq_mod()
293 	struct ef4_channel *channel =  in ef4_poll()  local
295 	struct ef4_nic *efx = channel->efx;  in ef4_poll()
300 		   channel->channel, raw_smp_processor_id());  in ef4_poll()
302 	spent = ef4_process_channel(channel, budget);  in ef4_poll()
305 		if (ef4_channel_has_rx_queue(channel) &&  in ef4_poll()
307 		    unlikely(++channel->irq_count == 1000)) {  in ef4_poll()
308 			ef4_update_irq_mod(efx, channel);  in ef4_poll()
311 		ef4_filter_rfs_expire(channel);  in ef4_poll()
319 		ef4_nic_eventq_read_ack(channel);  in ef4_poll()
330 static int ef4_probe_eventq(struct ef4_channel *channel)  in ef4_probe_eventq()  argument
332 	struct ef4_nic *efx = channel->efx;  in ef4_probe_eventq()
336 		  "chan %d create event queue\n", channel->channel);  in ef4_probe_eventq()
342 	channel->eventq_mask = max(entries, EF4_MIN_EVQ_SIZE) - 1;  in ef4_probe_eventq()
344 	return ef4_nic_probe_eventq(channel);  in ef4_probe_eventq()
348 static int ef4_init_eventq(struct ef4_channel *channel)  in ef4_init_eventq()  argument
350 	struct ef4_nic *efx = channel->efx;  in ef4_init_eventq()
353 	EF4_WARN_ON_PARANOID(channel->eventq_init);  in ef4_init_eventq()
356 		  "chan %d init event queue\n", channel->channel);  in ef4_init_eventq()
358 	rc = ef4_nic_init_eventq(channel);  in ef4_init_eventq()
360 		efx->type->push_irq_moderation(channel);  in ef4_init_eventq()
361 		channel->eventq_read_ptr = 0;  in ef4_init_eventq()
362 		channel->eventq_init = true;  in ef4_init_eventq()
368 void ef4_start_eventq(struct ef4_channel *channel)  in ef4_start_eventq()  argument
370 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,  in ef4_start_eventq()
371 		  "chan %d start event queue\n", channel->channel);  in ef4_start_eventq()
374 	channel->enabled = true;  in ef4_start_eventq()
377 	napi_enable(&channel->napi_str);  in ef4_start_eventq()
378 	ef4_nic_eventq_read_ack(channel);  in ef4_start_eventq()
382 void ef4_stop_eventq(struct ef4_channel *channel)  in ef4_stop_eventq()  argument
384 	if (!channel->enabled)  in ef4_stop_eventq()
387 	napi_disable(&channel->napi_str);  in ef4_stop_eventq()
388 	channel->enabled = false;  in ef4_stop_eventq()
391 static void ef4_fini_eventq(struct ef4_channel *channel)  in ef4_fini_eventq()  argument
393 	if (!channel->eventq_init)  in ef4_fini_eventq()
396 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in ef4_fini_eventq()
397 		  "chan %d fini event queue\n", channel->channel);  in ef4_fini_eventq()
399 	ef4_nic_fini_eventq(channel);  in ef4_fini_eventq()
400 	channel->eventq_init = false;  in ef4_fini_eventq()
403 static void ef4_remove_eventq(struct ef4_channel *channel)  in ef4_remove_eventq()  argument
405 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in ef4_remove_eventq()
406 		  "chan %d remove event queue\n", channel->channel);  in ef4_remove_eventq()
408 	ef4_nic_remove_eventq(channel);  in ef4_remove_eventq()
421 	struct ef4_channel *channel;  in ef4_alloc_channel()  local
426 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);  in ef4_alloc_channel()
427 	if (!channel)  in ef4_alloc_channel()
430 	channel->efx = efx;  in ef4_alloc_channel()
431 	channel->channel = i;  in ef4_alloc_channel()
432 	channel->type = &ef4_default_channel_type;  in ef4_alloc_channel()
435 		tx_queue = &channel->tx_queue[j];  in ef4_alloc_channel()
438 		tx_queue->channel = channel;  in ef4_alloc_channel()
441 	rx_queue = &channel->rx_queue;  in ef4_alloc_channel()
445 	return channel;  in ef4_alloc_channel()
454 	struct ef4_channel *channel;  in ef4_copy_channel()  local
459 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);  in ef4_copy_channel()
460 	if (!channel)  in ef4_copy_channel()
463 	*channel = *old_channel;  in ef4_copy_channel()
465 	channel->napi_dev = NULL;  in ef4_copy_channel()
466 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);  in ef4_copy_channel()
467 	channel->napi_str.napi_id = 0;  in ef4_copy_channel()
468 	channel->napi_str.state = 0;  in ef4_copy_channel()
469 	memset(&channel->eventq, 0, sizeof(channel->eventq));  in ef4_copy_channel()
472 		tx_queue = &channel->tx_queue[j];  in ef4_copy_channel()
473 		if (tx_queue->channel)  in ef4_copy_channel()
474 			tx_queue->channel = channel;  in ef4_copy_channel()
479 	rx_queue = &channel->rx_queue;  in ef4_copy_channel()
484 	return channel;  in ef4_copy_channel()
487 static int ef4_probe_channel(struct ef4_channel *channel)  in ef4_probe_channel()  argument
493 	netif_dbg(channel->efx, probe, channel->efx->net_dev,  in ef4_probe_channel()
494 		  "creating channel %d\n", channel->channel);  in ef4_probe_channel()
496 	rc = channel->type->pre_probe(channel);  in ef4_probe_channel()
500 	rc = ef4_probe_eventq(channel);  in ef4_probe_channel()
504 	ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_probe_channel()
510 	ef4_for_each_channel_rx_queue(rx_queue, channel) {  in ef4_probe_channel()
519 	ef4_remove_channel(channel);  in ef4_probe_channel()
524 ef4_get_channel_name(struct ef4_channel *channel, char *buf, size_t len)  in ef4_get_channel_name()  argument
526 	struct ef4_nic *efx = channel->efx;  in ef4_get_channel_name()
530 	number = channel->channel;  in ef4_get_channel_name()
533 	} else if (channel->channel < efx->tx_channel_offset) {  in ef4_get_channel_name()
544 	struct ef4_channel *channel;  in ef4_set_channel_names()  local
546 	ef4_for_each_channel(channel, efx)  in ef4_set_channel_names()
547 		channel->type->get_name(channel,  in ef4_set_channel_names()
548 					efx->msi_context[channel->channel].name,  in ef4_set_channel_names()
554 	struct ef4_channel *channel;  in ef4_probe_channels()  local
565 	ef4_for_each_channel_rev(channel, efx) {  in ef4_probe_channels()
566 		rc = ef4_probe_channel(channel);  in ef4_probe_channels()
570 				  channel->channel);  in ef4_probe_channels()
593 	struct ef4_channel *channel;  in ef4_start_datapath()  local
658 	ef4_for_each_channel(channel, efx) {  in ef4_start_datapath()
659 		ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_start_datapath()
664 		ef4_for_each_channel_rx_queue(rx_queue, channel) {  in ef4_start_datapath()
667 			ef4_stop_eventq(channel);  in ef4_start_datapath()
669 			ef4_start_eventq(channel);  in ef4_start_datapath()
672 		WARN_ON(channel->rx_pkt_n_frags);  in ef4_start_datapath()
681 	struct ef4_channel *channel;  in ef4_stop_datapath()  local
690 	ef4_for_each_channel(channel, efx) {  in ef4_stop_datapath()
691 		ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_stop_datapath()
695 	ef4_for_each_channel(channel, efx) {  in ef4_stop_datapath()
702 		if (ef4_channel_has_rx_queue(channel)) {  in ef4_stop_datapath()
703 			ef4_stop_eventq(channel);  in ef4_stop_datapath()
704 			ef4_start_eventq(channel);  in ef4_stop_datapath()
725 	ef4_for_each_channel(channel, efx) {  in ef4_stop_datapath()
726 		ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_stop_datapath()
728 		ef4_for_each_possible_channel_tx_queue(tx_queue, channel)  in ef4_stop_datapath()
733 static void ef4_remove_channel(struct ef4_channel *channel)  in ef4_remove_channel()  argument
738 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in ef4_remove_channel()
739 		  "destroy chan %d\n", channel->channel);  in ef4_remove_channel()
741 	ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_remove_channel()
743 	ef4_for_each_possible_channel_tx_queue(tx_queue, channel)  in ef4_remove_channel()
745 	ef4_remove_eventq(channel);  in ef4_remove_channel()
746 	channel->type->post_remove(channel);  in ef4_remove_channel()
751 	struct ef4_channel *channel;  in ef4_remove_channels()  local
753 	ef4_for_each_channel(channel, efx)  in ef4_remove_channels()
754 		ef4_remove_channel(channel);  in ef4_remove_channels()
760 	struct ef4_channel *other_channel[EF4_MAX_CHANNELS], *channel;  in ef4_realloc_channels()  local
772 	ef4_for_each_channel(channel, efx) {  in ef4_realloc_channels()
776 		if (channel->type->copy)  in ef4_realloc_channels()
779 					channel->eventq.index +  in ef4_realloc_channels()
780 					channel->eventq.entries);  in ef4_realloc_channels()
781 		ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_realloc_channels()
785 		ef4_for_each_channel_tx_queue(tx_queue, channel)  in ef4_realloc_channels()
798 		channel = efx->channel[i];  in ef4_realloc_channels()
799 		if (channel->type->copy)  in ef4_realloc_channels()
800 			channel = channel->type->copy(channel);  in ef4_realloc_channels()
801 		if (!channel) {  in ef4_realloc_channels()
805 		other_channel[i] = channel;  in ef4_realloc_channels()
814 		swap(efx->channel[i], other_channel[i]);  in ef4_realloc_channels()
821 		channel = efx->channel[i];  in ef4_realloc_channels()
822 		if (!channel->type->copy)  in ef4_realloc_channels()
824 		rc = ef4_probe_channel(channel);  in ef4_realloc_channels()
827 		ef4_init_napi_channel(efx->channel[i]);  in ef4_realloc_channels()
833 		channel = other_channel[i];  in ef4_realloc_channels()
834 		if (channel && channel->type->copy) {  in ef4_realloc_channels()
835 			ef4_fini_napi_channel(channel);  in ef4_realloc_channels()
836 			ef4_remove_channel(channel);  in ef4_realloc_channels()
837 			kfree(channel);  in ef4_realloc_channels()
858 		swap(efx->channel[i], other_channel[i]);  in ef4_realloc_channels()
876 int ef4_channel_dummy_op_int(struct ef4_channel *channel)  in ef4_channel_dummy_op_int()  argument
881 void ef4_channel_dummy_op_void(struct ef4_channel *channel)  in ef4_channel_dummy_op_void()  argument
1459 	struct ef4_channel *channel, *end_channel;  in ef4_soft_enable_interrupts()  local
1467 	ef4_for_each_channel(channel, efx) {  in ef4_soft_enable_interrupts()
1468 		if (!channel->type->keep_eventq) {  in ef4_soft_enable_interrupts()
1469 			rc = ef4_init_eventq(channel);  in ef4_soft_enable_interrupts()
1473 		ef4_start_eventq(channel);  in ef4_soft_enable_interrupts()
1478 	end_channel = channel;  in ef4_soft_enable_interrupts()
1479 	ef4_for_each_channel(channel, efx) {  in ef4_soft_enable_interrupts()
1480 		if (channel == end_channel)  in ef4_soft_enable_interrupts()
1482 		ef4_stop_eventq(channel);  in ef4_soft_enable_interrupts()
1483 		if (!channel->type->keep_eventq)  in ef4_soft_enable_interrupts()
1484 			ef4_fini_eventq(channel);  in ef4_soft_enable_interrupts()
1492 	struct ef4_channel *channel;  in ef4_soft_disable_interrupts()  local
1503 	ef4_for_each_channel(channel, efx) {  in ef4_soft_disable_interrupts()
1504 		if (channel->irq)  in ef4_soft_disable_interrupts()
1505 			synchronize_irq(channel->irq);  in ef4_soft_disable_interrupts()
1507 		ef4_stop_eventq(channel);  in ef4_soft_disable_interrupts()
1508 		if (!channel->type->keep_eventq)  in ef4_soft_disable_interrupts()
1509 			ef4_fini_eventq(channel);  in ef4_soft_disable_interrupts()
1515 	struct ef4_channel *channel, *end_channel;  in ef4_enable_interrupts()  local
1527 	ef4_for_each_channel(channel, efx) {  in ef4_enable_interrupts()
1528 		if (channel->type->keep_eventq) {  in ef4_enable_interrupts()
1529 			rc = ef4_init_eventq(channel);  in ef4_enable_interrupts()
1542 	end_channel = channel;  in ef4_enable_interrupts()
1543 	ef4_for_each_channel(channel, efx) {  in ef4_enable_interrupts()
1544 		if (channel == end_channel)  in ef4_enable_interrupts()
1546 		if (channel->type->keep_eventq)  in ef4_enable_interrupts()
1547 			ef4_fini_eventq(channel);  in ef4_enable_interrupts()
1557 	struct ef4_channel *channel;  in ef4_disable_interrupts()  local
1561 	ef4_for_each_channel(channel, efx) {  in ef4_disable_interrupts()
1562 		if (channel->type->keep_eventq)  in ef4_disable_interrupts()
1563 			ef4_fini_eventq(channel);  in ef4_disable_interrupts()
1571 	struct ef4_channel *channel;  in ef4_remove_interrupts()  local
1574 	ef4_for_each_channel(channel, efx)  in ef4_remove_interrupts()
1575 		channel->irq = 0;  in ef4_remove_interrupts()
1585 	struct ef4_channel *channel;  in ef4_set_channels()  local
1596 	ef4_for_each_channel(channel, efx) {  in ef4_set_channels()
1597 		if (channel->channel < efx->n_rx_channels)  in ef4_set_channels()
1598 			channel->rx_queue.core_index = channel->channel;  in ef4_set_channels()
1600 			channel->rx_queue.core_index = -1;  in ef4_set_channels()
1602 		ef4_for_each_channel_tx_queue(tx_queue, channel)  in ef4_set_channels()
1692 		struct ef4_channel *channel;  in ef4_probe_filters()  local
1695 		ef4_for_each_channel(channel, efx) {  in ef4_probe_filters()
1696 			channel->rps_flow_id =  in ef4_probe_filters()
1698 					sizeof(*channel->rps_flow_id),  in ef4_probe_filters()
1700 			if (!channel->rps_flow_id)  in ef4_probe_filters()
1706 					channel->rps_flow_id[i] =  in ef4_probe_filters()
1711 			ef4_for_each_channel(channel, efx)  in ef4_probe_filters()
1712 				kfree(channel->rps_flow_id);  in ef4_probe_filters()
1730 	struct ef4_channel *channel;  in ef4_remove_filters()  local
1732 	ef4_for_each_channel(channel, efx)  in ef4_remove_filters()
1733 		kfree(channel->rps_flow_id);  in ef4_remove_filters()
1902 	struct ef4_channel *channel;  in ef4_init_irq_moderation()  local
1921 	ef4_for_each_channel(channel, efx) {  in ef4_init_irq_moderation()
1922 		if (ef4_channel_has_rx_queue(channel))  in ef4_init_irq_moderation()
1923 			channel->irq_moderation_us = rx_usecs;  in ef4_init_irq_moderation()
1924 		else if (ef4_channel_has_tx_queues(channel))  in ef4_init_irq_moderation()
1925 			channel->irq_moderation_us = tx_usecs;  in ef4_init_irq_moderation()
1946 		tx_channel = efx->channel[efx->tx_channel_offset];  in ef4_get_irq_moderation()
2009 static void ef4_init_napi_channel(struct ef4_channel *channel)  in ef4_init_napi_channel()  argument
2011 	struct ef4_nic *efx = channel->efx;  in ef4_init_napi_channel()
2013 	channel->napi_dev = efx->net_dev;  in ef4_init_napi_channel()
2014 	netif_napi_add(channel->napi_dev, &channel->napi_str, ef4_poll);  in ef4_init_napi_channel()
2019 	struct ef4_channel *channel;  in ef4_init_napi()  local
2021 	ef4_for_each_channel(channel, efx)  in ef4_init_napi()
2022 		ef4_init_napi_channel(channel);  in ef4_init_napi()
2025 static void ef4_fini_napi_channel(struct ef4_channel *channel)  in ef4_fini_napi_channel()  argument
2027 	if (channel->napi_dev)  in ef4_fini_napi_channel()
2028 		netif_napi_del(&channel->napi_str);  in ef4_fini_napi_channel()
2030 	channel->napi_dev = NULL;  in ef4_fini_napi_channel()
2035 	struct ef4_channel *channel;  in ef4_fini_napi()  local
2037 	ef4_for_each_channel(channel, efx)  in ef4_fini_napi()
2038 		ef4_fini_napi_channel(channel);  in ef4_fini_napi()
2256 	struct ef4_channel *channel;  in ef4_register_netdev()  local
2294 	ef4_for_each_channel(channel, efx) {  in ef4_register_netdev()
2296 		ef4_for_each_channel_tx_queue(tx_queue, channel)  in ef4_register_netdev()
2660 		efx->channel[i] = ef4_alloc_channel(efx, i, NULL);  in ef4_init_struct()
2661 		if (!efx->channel[i])  in ef4_init_struct()
2690 		kfree(efx->channel[i]);  in ef4_fini_struct()
2703 	struct ef4_channel *channel;  in ef4_update_sw_stats()  local
2705 	ef4_for_each_channel(channel, efx)  in ef4_update_sw_stats()
2706 		n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;  in ef4_update_sw_stats()