1 // SPDX-License-Identifier: GPL-2.0-only
17 /* Counter-management hashtables */
35 	WARN_ON(!list_empty(&cnt->users));  in efx_tc_counter_free()
42 	flush_work(&cnt->work);  in efx_tc_counter_free()
43 	EFX_WARN_ON_PARANOID(spin_is_locked(&cnt->lock));  in efx_tc_counter_free()
51 	WARN_ON(refcount_read(&ctr->ref));  in efx_tc_counter_id_free()
59 	rc = rhashtable_init(&efx->tc->counter_id_ht, &efx_tc_counter_id_ht_params);  in efx_tc_init_counters()
62 	rc = rhashtable_init(&efx->tc->counter_ht, &efx_tc_counter_ht_params);  in efx_tc_init_counters()
67 	rhashtable_destroy(&efx->tc->counter_id_ht);  in efx_tc_init_counters()
77 	rhashtable_destroy(&efx->tc->counter_ht);  in efx_tc_destroy_counters()
78 	rhashtable_destroy(&efx->tc->counter_id_ht);  in efx_tc_destroy_counters()
83 	rhashtable_free_and_destroy(&efx->tc->counter_id_ht, efx_tc_counter_id_free, NULL);  in efx_tc_fini_counters()
84 	rhashtable_free_and_destroy(&efx->tc->counter_ht, efx_tc_counter_free, NULL);  in efx_tc_fini_counters()
95 	spin_lock_bh(&cnt->lock);  in efx_tc_counter_work()
96 	touched = READ_ONCE(cnt->touched);  in efx_tc_counter_work()
98 	list_for_each_entry(act, &cnt->users, count_user) {  in efx_tc_counter_work()
99 		encap = act->encap_md;  in efx_tc_counter_work()
102 		if (!encap->neigh) /* can't happen */  in efx_tc_counter_work()
104 		if (time_after_eq(encap->neigh->used, touched))  in efx_tc_counter_work()
106 		encap->neigh->used = touched;  in efx_tc_counter_work()
110 		if (encap->neigh->dst_ip)  in efx_tc_counter_work()
111 			n = neigh_lookup(&arp_tbl, &encap->neigh->dst_ip,  in efx_tc_counter_work()
112 					 encap->neigh->egdev);  in efx_tc_counter_work()
115 			n = neigh_lookup(ipv6_stub->nd_tbl,  in efx_tc_counter_work()
116 					 &encap->neigh->dst_ip6,  in efx_tc_counter_work()
117 					 encap->neigh->egdev);  in efx_tc_counter_work()
127 	spin_unlock_bh(&cnt->lock);  in efx_tc_counter_work()
140 		return ERR_PTR(-ENOMEM);  in efx_tc_flower_allocate_counter()
142 	spin_lock_init(&cnt->lock);  in efx_tc_flower_allocate_counter()
143 	INIT_WORK(&cnt->work, efx_tc_counter_work);  in efx_tc_flower_allocate_counter()
144 	cnt->touched = jiffies;  in efx_tc_flower_allocate_counter()
145 	cnt->type = type;  in efx_tc_flower_allocate_counter()
150 	INIT_LIST_HEAD(&cnt->users);  in efx_tc_flower_allocate_counter()
151 	rc = rhashtable_insert_fast(&efx->tc->counter_ht, &cnt->linkage,  in efx_tc_flower_allocate_counter()
164 		netif_warn(efx, hw, efx->net_dev,  in efx_tc_flower_allocate_counter()
166 			   cnt->fw_id, rc2);  in efx_tc_flower_allocate_counter()
169 	return ERR_PTR(rc > 0 ? -EIO : rc);  in efx_tc_flower_allocate_counter()
177 	rhashtable_remove_fast(&efx->tc->counter_ht, &cnt->linkage,  in efx_tc_flower_release_counter()
181 		netif_warn(efx, hw, efx->net_dev,  in efx_tc_flower_release_counter()
183 			   cnt->fw_id, rc);  in efx_tc_flower_release_counter()
184 	WARN_ON(!list_empty(&cnt->users));  in efx_tc_flower_release_counter()
186 	 * after we deleted the counter.  The RCU just ensures that we won't  in efx_tc_flower_release_counter()
188 	 * Ensuring we don't update the wrong counter if the ID gets re-used  in efx_tc_flower_release_counter()
192 	flush_work(&cnt->work);  in efx_tc_flower_release_counter()
193 	EFX_WARN_ON_PARANOID(spin_is_locked(&cnt->lock));  in efx_tc_flower_release_counter()
205 	return rhashtable_lookup_fast(&efx->tc->counter_ht, &key,  in efx_tc_flower_find_counter_by_fw_id()
214 	if (!refcount_dec_and_test(&ctr->ref))  in efx_tc_flower_put_counter_index()
216 	rhashtable_remove_fast(&efx->tc->counter_id_ht, &ctr->linkage,  in efx_tc_flower_put_counter_index()
218 	efx_tc_flower_release_counter(efx, ctr->cnt);  in efx_tc_flower_put_counter_index()
231 		return ERR_PTR(-ENOMEM);  in efx_tc_flower_get_counter_index()
232 	ctr->cookie = cookie;  in efx_tc_flower_get_counter_index()
233 	old = rhashtable_lookup_get_insert_fast(&efx->tc->counter_id_ht,  in efx_tc_flower_get_counter_index()
234 						&ctr->linkage,  in efx_tc_flower_get_counter_index()
241 		if (!refcount_inc_not_zero(&old->ref))  in efx_tc_flower_get_counter_index()
242 			return ERR_PTR(-EAGAIN);  in efx_tc_flower_get_counter_index()
248 			rhashtable_remove_fast(&efx->tc->counter_id_ht,  in efx_tc_flower_get_counter_index()
249 					       &ctr->linkage,  in efx_tc_flower_get_counter_index()
254 		ctr->cnt = cnt;  in efx_tc_flower_get_counter_index()
255 		refcount_set(&ctr->ref, 1);  in efx_tc_flower_get_counter_index()
266 	return rhashtable_lookup_fast(&efx->tc->counter_id_ht, &key,  in efx_tc_flower_find_counter_index()
274 	netif_warn(efx, drv, efx->net_dev,  in efx_tc_handle_no_channel()
275 		   "MAE counters require MSI-X and 1 additional interrupt vector.\n");  in efx_tc_handle_no_channel()
280 	struct efx_rx_queue *rx_queue = &channel->rx_queue;  in efx_tc_probe_channel()
282 	channel->irq_moderation_us = 0;  in efx_tc_probe_channel()
283 	rx_queue->core_index = 0;  in efx_tc_probe_channel()
285 	INIT_WORK(&rx_queue->grant_work, efx_mae_counters_grant_credits);  in efx_tc_probe_channel()
293 	struct efx_nic *efx = channel->efx;  in efx_tc_start_channel()
301 	struct efx_nic *efx = channel->efx;  in efx_tc_stop_channel()
306 		netif_warn(efx, drv, efx->net_dev,  in efx_tc_stop_channel()
309 	rx_queue->grant_credits = false;  in efx_tc_stop_channel()
310 	flush_work(&rx_queue->grant_work);  in efx_tc_stop_channel()
320 	snprintf(buf, len, "%s-mae", channel->efx->name);  in efx_tc_get_channel_name()
334 		 * with updates for the counter still in-flight; however this  in efx_tc_counter_update()
338 			netif_dbg(efx, drv, efx->net_dev,  in efx_tc_counter_update()
344 	spin_lock_bh(&cnt->lock);  in efx_tc_counter_update()
345 	if ((s32)mark - (s32)cnt->gen < 0) {  in efx_tc_counter_update()
353 		 * even a long-lived counter won't start getting ignored if  in efx_tc_counter_update()
357 		cnt->gen = mark;  in efx_tc_counter_update()
359 		cnt->packets += packets;  in efx_tc_counter_update()
360 		cnt->bytes += bytes;  in efx_tc_counter_update()
361 		cnt->touched = jiffies;  in efx_tc_counter_update()
363 	spin_unlock_bh(&cnt->lock);  in efx_tc_counter_update()
364 	schedule_work(&cnt->work);  in efx_tc_counter_update()
415 /* Read a little-endian 48-bit field with 16-bit alignment */
446 			netif_err(efx, drv, efx->net_dev,  in efx_tc_rx_version_2()
452 	/* mae_counter_format.h implies that this offset is fixed, since it  in efx_tc_rx_version_2()
453 	 * carries on with SOP-based LBNs for the fields in this header  in efx_tc_rx_version_2()
457 			netif_err(efx, drv, efx->net_dev,  in efx_tc_rx_version_2()
470 		/* 24-bit field with 32-bit alignment */  in efx_tc_rx_version_2()
475 		/* 48-bit field with 16-bit alignment */  in efx_tc_rx_version_2()
480 		/* 48-bit field with 16-bit alignment */  in efx_tc_rx_version_2()
487 			/* CT counters are 1-bit saturating counters to update  in efx_tc_rx_version_2()
493 				netdev_warn_once(efx->net_dev,  in efx_tc_rx_version_2()
514 						     channel->rx_pkt_index);  in efx_tc_rx()
516 	struct efx_nic *efx = rx_queue->efx;  in efx_tc_rx()
532 			netif_err(efx, drv, efx->net_dev,  in efx_tc_rx()
543 		efx->tc->seen_gen[type] = mark;  in efx_tc_rx()
544 		if (efx->tc->flush_counters &&  in efx_tc_rx()
545 		    (s32)(efx->tc->flush_gen[type] - mark) <= 0)  in efx_tc_rx()
546 			wake_up(&efx->tc->flush_wq);  in efx_tc_rx()
550 	channel->rx_pkt_n_frags = 0;  in efx_tc_rx()