1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
3 
4 #include <linux/hash.h>
5 #include <linux/hashtable.h>
6 #include <linux/jhash.h>
7 #include <linux/math64.h>
8 #include <linux/vmalloc.h>
9 #include <net/pkt_cls.h>
10 
11 #include "cmsg.h"
12 #include "conntrack.h"
13 #include "main.h"
14 #include "../nfp_app.h"
15 
16 struct nfp_mask_id_table {
17 	struct hlist_node link;
18 	u32 hash_key;
19 	u32 ref_cnt;
20 	u8 mask_id;
21 };
22 
23 struct nfp_fl_flow_table_cmp_arg {
24 	struct net_device *netdev;
25 	unsigned long cookie;
26 };
27 
28 struct nfp_fl_stats_ctx_to_flow {
29 	struct rhash_head ht_node;
30 	u32 stats_cxt;
31 	struct nfp_fl_payload *flow;
32 };
33 
34 static const struct rhashtable_params stats_ctx_table_params = {
35 	.key_offset	= offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt),
36 	.head_offset	= offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node),
37 	.key_len	= sizeof(u32),
38 };
39 
40 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id)
41 {
42 	struct nfp_flower_priv *priv = app->priv;
43 	struct circ_buf *ring;
44 
45 	ring = &priv->stats_ids.free_list;
46 	/* Check if buffer is full, stats_ring_size must be power of 2 */
47 	if (!CIRC_SPACE(ring->head, ring->tail, priv->stats_ring_size))
48 		return -ENOBUFS;
49 
50 	/* Each increment of head represents size of NFP_FL_STATS_ELEM_RS */
51 	memcpy(&ring->buf[ring->head * NFP_FL_STATS_ELEM_RS],
52 	       &stats_context_id, NFP_FL_STATS_ELEM_RS);
53 	ring->head = (ring->head + 1) & (priv->stats_ring_size - 1);
54 
55 	return 0;
56 }
57 
58 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id)
59 {
60 	struct nfp_flower_priv *priv = app->priv;
61 	u32 freed_stats_id, temp_stats_id;
62 	struct circ_buf *ring;
63 
64 	ring = &priv->stats_ids.free_list;
65 	freed_stats_id = priv->stats_ring_size;
66 	/* Check for unallocated entries first. */
67 	if (priv->stats_ids.init_unalloc > 0) {
68 		*stats_context_id =
69 			FIELD_PREP(NFP_FL_STAT_ID_STAT,
70 				   priv->stats_ids.init_unalloc - 1) |
71 			FIELD_PREP(NFP_FL_STAT_ID_MU_NUM,
72 				   priv->active_mem_unit);
73 
74 		if (++priv->active_mem_unit == priv->total_mem_units) {
75 			priv->stats_ids.init_unalloc--;
76 			priv->active_mem_unit = 0;
77 		}
78 
79 		return 0;
80 	}
81 
82 	/* Check if buffer is empty. */
83 	if (ring->head == ring->tail) {
84 		*stats_context_id = freed_stats_id;
85 		return -ENOENT;
86 	}
87 
88 	/* Each increment of tail represents size of NFP_FL_STATS_ELEM_RS */
89 	memcpy(&temp_stats_id, &ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS],
90 	       NFP_FL_STATS_ELEM_RS);
91 	*stats_context_id = temp_stats_id;
92 	memcpy(&ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], &freed_stats_id,
93 	       NFP_FL_STATS_ELEM_RS);
94 	/* stats_ring_size must be power of 2 */
95 	ring->tail = (ring->tail + 1) & (priv->stats_ring_size - 1);
96 
97 	return 0;
98 }
99 
100 /* Must be called with either RTNL or rcu_read_lock */
101 struct nfp_fl_payload *
102 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
103 			   struct net_device *netdev)
104 {
105 	struct nfp_fl_flow_table_cmp_arg flower_cmp_arg;
106 	struct nfp_flower_priv *priv = app->priv;
107 
108 	flower_cmp_arg.netdev = netdev;
109 	flower_cmp_arg.cookie = tc_flower_cookie;
110 
111 	return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg,
112 				      nfp_flower_table_params);
113 }
114 
115 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb)
116 {
117 	unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb);
118 	struct nfp_flower_priv *priv = app->priv;
119 	struct nfp_fl_stats_frame *stats;
120 	unsigned char *msg;
121 	u32 ctx_id;
122 	int i;
123 
124 	msg = nfp_flower_cmsg_get_data(skb);
125 
126 	spin_lock(&priv->stats_lock);
127 	for (i = 0; i < msg_len / sizeof(*stats); i++) {
128 		stats = (struct nfp_fl_stats_frame *)msg + i;
129 		ctx_id = be32_to_cpu(stats->stats_con_id);
130 		priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count);
131 		priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count);
132 		priv->stats[ctx_id].used = jiffies;
133 	}
134 	spin_unlock(&priv->stats_lock);
135 }
136 
137 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id)
138 {
139 	struct nfp_flower_priv *priv = app->priv;
140 	struct circ_buf *ring;
141 
142 	ring = &priv->mask_ids.mask_id_free_list;
143 	/* Checking if buffer is full,
144 	 * NFP_FLOWER_MASK_ENTRY_RS must be power of 2
145 	 */
146 	if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0)
147 		return -ENOBUFS;
148 
149 	/* Each increment of head represents size of
150 	 * NFP_FLOWER_MASK_ELEMENT_RS
151 	 */
152 	memcpy(&ring->buf[ring->head * NFP_FLOWER_MASK_ELEMENT_RS], &mask_id,
153 	       NFP_FLOWER_MASK_ELEMENT_RS);
154 	ring->head = (ring->head + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1);
155 
156 	priv->mask_ids.last_used[mask_id] = ktime_get();
157 
158 	return 0;
159 }
160 
161 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id)
162 {
163 	struct nfp_flower_priv *priv = app->priv;
164 	ktime_t reuse_timeout;
165 	struct circ_buf *ring;
166 	u8 temp_id, freed_id;
167 
168 	ring = &priv->mask_ids.mask_id_free_list;
169 	freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1;
170 	/* Checking for unallocated entries first. */
171 	if (priv->mask_ids.init_unallocated > 0) {
172 		*mask_id = priv->mask_ids.init_unallocated;
173 		priv->mask_ids.init_unallocated--;
174 		return 0;
175 	}
176 
177 	/* Checking if buffer is empty. */
178 	if (ring->head == ring->tail)
179 		goto err_not_found;
180 
181 	/* Each increment of tail represents size of
182 	 * NFP_FLOWER_MASK_ELEMENT_RS
183 	 */
184 	memcpy(&temp_id, &ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS],
185 	       NFP_FLOWER_MASK_ELEMENT_RS);
186 	*mask_id = temp_id;
187 
188 	reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id],
189 				     NFP_FL_MASK_REUSE_TIME_NS);
190 
191 	if (ktime_before(ktime_get(), reuse_timeout))
192 		goto err_not_found;
193 
194 	memcpy(&ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], &freed_id,
195 	       NFP_FLOWER_MASK_ELEMENT_RS);
196 	/* NFP_FLOWER_MASK_ENTRY_RS must be power of 2 */
197 	ring->tail = (ring->tail + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1);
198 
199 	return 0;
200 
201 err_not_found:
202 	*mask_id = freed_id;
203 	return -ENOENT;
204 }
205 
206 static int
207 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
208 {
209 	struct nfp_flower_priv *priv = app->priv;
210 	struct nfp_mask_id_table *mask_entry;
211 	unsigned long hash_key;
212 	u8 mask_id;
213 
214 	if (nfp_mask_alloc(app, &mask_id))
215 		return -ENOENT;
216 
217 	mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL);
218 	if (!mask_entry) {
219 		nfp_release_mask_id(app, mask_id);
220 		return -ENOMEM;
221 	}
222 
223 	INIT_HLIST_NODE(&mask_entry->link);
224 	mask_entry->mask_id = mask_id;
225 	hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
226 	mask_entry->hash_key = hash_key;
227 	mask_entry->ref_cnt = 1;
228 	hash_add(priv->mask_table, &mask_entry->link, hash_key);
229 
230 	return mask_id;
231 }
232 
233 static struct nfp_mask_id_table *
234 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
235 {
236 	struct nfp_flower_priv *priv = app->priv;
237 	struct nfp_mask_id_table *mask_entry;
238 	unsigned long hash_key;
239 
240 	hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
241 
242 	hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key)
243 		if (mask_entry->hash_key == hash_key)
244 			return mask_entry;
245 
246 	return NULL;
247 }
248 
249 static int
250 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
251 {
252 	struct nfp_mask_id_table *mask_entry;
253 
254 	mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
255 	if (!mask_entry)
256 		return -ENOENT;
257 
258 	mask_entry->ref_cnt++;
259 
260 	/* Casting u8 to int for later use. */
261 	return mask_entry->mask_id;
262 }
263 
264 static bool
265 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len,
266 		   u8 *meta_flags, u8 *mask_id)
267 {
268 	int id;
269 
270 	id = nfp_find_in_mask_table(app, mask_data, mask_len);
271 	if (id < 0) {
272 		id = nfp_add_mask_table(app, mask_data, mask_len);
273 		if (id < 0)
274 			return false;
275 		*meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
276 	}
277 	*mask_id = id;
278 
279 	return true;
280 }
281 
282 static bool
283 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len,
284 		      u8 *meta_flags, u8 *mask_id)
285 {
286 	struct nfp_mask_id_table *mask_entry;
287 
288 	mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
289 	if (!mask_entry)
290 		return false;
291 
292 	*mask_id = mask_entry->mask_id;
293 	mask_entry->ref_cnt--;
294 	if (!mask_entry->ref_cnt) {
295 		hash_del(&mask_entry->link);
296 		nfp_release_mask_id(app, *mask_id);
297 		kfree(mask_entry);
298 		if (meta_flags)
299 			*meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
300 	}
301 
302 	return true;
303 }
304 
305 int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie,
306 			      struct nfp_fl_payload *nfp_flow,
307 			      struct net_device *netdev,
308 			      struct netlink_ext_ack *extack)
309 {
310 	struct nfp_fl_stats_ctx_to_flow *ctx_entry;
311 	struct nfp_flower_priv *priv = app->priv;
312 	struct nfp_fl_payload *check_entry;
313 	u8 new_mask_id;
314 	u32 stats_cxt;
315 	int err;
316 
317 	err = nfp_get_stats_entry(app, &stats_cxt);
318 	if (err) {
319 		NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate new stats context");
320 		return err;
321 	}
322 
323 	nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt);
324 	nfp_flow->meta.host_cookie = cpu_to_be64(cookie);
325 	nfp_flow->ingress_dev = netdev;
326 
327 	ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL);
328 	if (!ctx_entry) {
329 		err = -ENOMEM;
330 		goto err_release_stats;
331 	}
332 
333 	ctx_entry->stats_cxt = stats_cxt;
334 	ctx_entry->flow = nfp_flow;
335 
336 	if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node,
337 				   stats_ctx_table_params)) {
338 		err = -ENOMEM;
339 		goto err_free_ctx_entry;
340 	}
341 
342 	/* Do not allocate a mask-id for pre_tun_rules. These flows are used to
343 	 * configure the pre_tun table and are never actually send to the
344 	 * firmware as an add-flow message. This causes the mask-id allocation
345 	 * on the firmware to get out of sync if allocated here.
346 	 */
347 	new_mask_id = 0;
348 	if (!nfp_flow->pre_tun_rule.dev &&
349 	    !nfp_check_mask_add(app, nfp_flow->mask_data,
350 				nfp_flow->meta.mask_len,
351 				&nfp_flow->meta.flags, &new_mask_id)) {
352 		NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id");
353 		err = -ENOENT;
354 		goto err_remove_rhash;
355 	}
356 
357 	nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
358 	priv->flower_version++;
359 
360 	/* Update flow payload with mask ids. */
361 	nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
362 	priv->stats[stats_cxt].pkts = 0;
363 	priv->stats[stats_cxt].bytes = 0;
364 	priv->stats[stats_cxt].used = jiffies;
365 
366 	check_entry = nfp_flower_search_fl_table(app, cookie, netdev);
367 	if (check_entry) {
368 		NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry");
369 		err = -EEXIST;
370 		goto err_remove_mask;
371 	}
372 
373 	return 0;
374 
375 err_remove_mask:
376 	if (!nfp_flow->pre_tun_rule.dev)
377 		nfp_check_mask_remove(app, nfp_flow->mask_data,
378 				      nfp_flow->meta.mask_len,
379 				      NULL, &new_mask_id);
380 err_remove_rhash:
381 	WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table,
382 					    &ctx_entry->ht_node,
383 					    stats_ctx_table_params));
384 err_free_ctx_entry:
385 	kfree(ctx_entry);
386 err_release_stats:
387 	nfp_release_stats_entry(app, stats_cxt);
388 
389 	return err;
390 }
391 
392 void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv,
393 				struct nfp_fl_payload *nfp_flow)
394 {
395 	nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK;
396 	nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
397 	priv->flower_version++;
398 }
399 
400 int nfp_modify_flow_metadata(struct nfp_app *app,
401 			     struct nfp_fl_payload *nfp_flow)
402 {
403 	struct nfp_fl_stats_ctx_to_flow *ctx_entry;
404 	struct nfp_flower_priv *priv = app->priv;
405 	u8 new_mask_id = 0;
406 	u32 temp_ctx_id;
407 
408 	__nfp_modify_flow_metadata(priv, nfp_flow);
409 
410 	if (!nfp_flow->pre_tun_rule.dev)
411 		nfp_check_mask_remove(app, nfp_flow->mask_data,
412 				      nfp_flow->meta.mask_len, &nfp_flow->meta.flags,
413 				      &new_mask_id);
414 
415 	/* Update flow payload with mask ids. */
416 	nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
417 
418 	/* Release the stats ctx id and ctx to flow table entry. */
419 	temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
420 
421 	ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id,
422 					   stats_ctx_table_params);
423 	if (!ctx_entry)
424 		return -ENOENT;
425 
426 	WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table,
427 					    &ctx_entry->ht_node,
428 					    stats_ctx_table_params));
429 	kfree(ctx_entry);
430 
431 	return nfp_release_stats_entry(app, temp_ctx_id);
432 }
433 
434 struct nfp_fl_payload *
435 nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id)
436 {
437 	struct nfp_fl_stats_ctx_to_flow *ctx_entry;
438 	struct nfp_flower_priv *priv = app->priv;
439 
440 	ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id,
441 					   stats_ctx_table_params);
442 	if (!ctx_entry)
443 		return NULL;
444 
445 	return ctx_entry->flow;
446 }
447 
448 static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg,
449 			    const void *obj)
450 {
451 	const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key;
452 	const struct nfp_fl_payload *flow_entry = obj;
453 
454 	if (flow_entry->ingress_dev == cmp_arg->netdev)
455 		return flow_entry->tc_flower_cookie != cmp_arg->cookie;
456 
457 	return 1;
458 }
459 
460 static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed)
461 {
462 	const struct nfp_fl_payload *flower_entry = data;
463 
464 	return jhash2((u32 *)&flower_entry->tc_flower_cookie,
465 		      sizeof(flower_entry->tc_flower_cookie) / sizeof(u32),
466 		      seed);
467 }
468 
469 static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed)
470 {
471 	const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data;
472 
473 	return jhash2((u32 *)&cmp_arg->cookie,
474 		      sizeof(cmp_arg->cookie) / sizeof(u32), seed);
475 }
476 
477 const struct rhashtable_params nfp_flower_table_params = {
478 	.head_offset		= offsetof(struct nfp_fl_payload, fl_node),
479 	.hashfn			= nfp_fl_key_hashfn,
480 	.obj_cmpfn		= nfp_fl_obj_cmpfn,
481 	.obj_hashfn		= nfp_fl_obj_hashfn,
482 	.automatic_shrinking	= true,
483 };
484 
485 const struct rhashtable_params merge_table_params = {
486 	.key_offset	= offsetof(struct nfp_merge_info, parent_ctx),
487 	.head_offset	= offsetof(struct nfp_merge_info, ht_node),
488 	.key_len	= sizeof(u64),
489 };
490 
491 const struct rhashtable_params nfp_zone_table_params = {
492 	.head_offset		= offsetof(struct nfp_fl_ct_zone_entry, hash_node),
493 	.key_len		= sizeof(u16),
494 	.key_offset		= offsetof(struct nfp_fl_ct_zone_entry, zone),
495 	.automatic_shrinking	= false,
496 };
497 
498 const struct rhashtable_params nfp_ct_map_params = {
499 	.head_offset		= offsetof(struct nfp_fl_ct_map_entry, hash_node),
500 	.key_len		= sizeof(unsigned long),
501 	.key_offset		= offsetof(struct nfp_fl_ct_map_entry, cookie),
502 	.automatic_shrinking	= true,
503 };
504 
505 const struct rhashtable_params neigh_table_params = {
506 	.key_offset	= offsetof(struct nfp_neigh_entry, neigh_cookie),
507 	.head_offset	= offsetof(struct nfp_neigh_entry, ht_node),
508 	.key_len	= sizeof(unsigned long),
509 };
510 
511 int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count,
512 			     unsigned int host_num_mems)
513 {
514 	struct nfp_flower_priv *priv = app->priv;
515 	int err, stats_size;
516 
517 	hash_init(priv->mask_table);
518 
519 	err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params);
520 	if (err)
521 		return err;
522 
523 	err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params);
524 	if (err)
525 		goto err_free_flow_table;
526 
527 	err = rhashtable_init(&priv->merge_table, &merge_table_params);
528 	if (err)
529 		goto err_free_stats_ctx_table;
530 
531 	err = rhashtable_init(&priv->ct_zone_table, &nfp_zone_table_params);
532 	if (err)
533 		goto err_free_merge_table;
534 
535 	err = rhashtable_init(&priv->ct_map_table, &nfp_ct_map_params);
536 	if (err)
537 		goto err_free_ct_zone_table;
538 
539 	err = rhashtable_init(&priv->neigh_table, &neigh_table_params);
540 	if (err)
541 		goto err_free_ct_map_table;
542 
543 	INIT_LIST_HEAD(&priv->predt_list);
544 
545 	get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed));
546 
547 	/* Init ring buffer and unallocated mask_ids. */
548 	priv->mask_ids.mask_id_free_list.buf =
549 		kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
550 			      NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL);
551 	if (!priv->mask_ids.mask_id_free_list.buf)
552 		goto err_free_neigh_table;
553 
554 	priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1;
555 
556 	/* Init timestamps for mask id*/
557 	priv->mask_ids.last_used =
558 		kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
559 			      sizeof(*priv->mask_ids.last_used), GFP_KERNEL);
560 	if (!priv->mask_ids.last_used)
561 		goto err_free_mask_id;
562 
563 	/* Init ring buffer and unallocated stats_ids. */
564 	priv->stats_ids.free_list.buf =
565 		vmalloc(array_size(NFP_FL_STATS_ELEM_RS,
566 				   priv->stats_ring_size));
567 	if (!priv->stats_ids.free_list.buf)
568 		goto err_free_last_used;
569 
570 	priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems);
571 
572 	stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) |
573 		     FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1);
574 	priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats),
575 				     GFP_KERNEL);
576 	if (!priv->stats)
577 		goto err_free_ring_buf;
578 
579 	spin_lock_init(&priv->stats_lock);
580 	spin_lock_init(&priv->predt_lock);
581 
582 	return 0;
583 
584 err_free_ring_buf:
585 	vfree(priv->stats_ids.free_list.buf);
586 err_free_last_used:
587 	kfree(priv->mask_ids.last_used);
588 err_free_mask_id:
589 	kfree(priv->mask_ids.mask_id_free_list.buf);
590 err_free_neigh_table:
591 	rhashtable_destroy(&priv->neigh_table);
592 err_free_ct_map_table:
593 	rhashtable_destroy(&priv->ct_map_table);
594 err_free_ct_zone_table:
595 	rhashtable_destroy(&priv->ct_zone_table);
596 err_free_merge_table:
597 	rhashtable_destroy(&priv->merge_table);
598 err_free_stats_ctx_table:
599 	rhashtable_destroy(&priv->stats_ctx_table);
600 err_free_flow_table:
601 	rhashtable_destroy(&priv->flow_table);
602 	return -ENOMEM;
603 }
604 
605 static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt)
606 {
607 	if (!zt)
608 		return;
609 
610 	if (!list_empty(&zt->pre_ct_list)) {
611 		struct rhashtable *m_table = &zt->priv->ct_map_table;
612 		struct nfp_fl_ct_flow_entry *entry, *tmp;
613 		struct nfp_fl_ct_map_entry *map;
614 
615 		WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n");
616 		list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list,
617 					 list_node) {
618 			map = rhashtable_lookup_fast(m_table,
619 						     &entry->cookie,
620 						     nfp_ct_map_params);
621 			WARN_ON_ONCE(rhashtable_remove_fast(m_table,
622 							    &map->hash_node,
623 							    nfp_ct_map_params));
624 			nfp_fl_ct_clean_flow_entry(entry);
625 			kfree(map);
626 		}
627 	}
628 
629 	if (!list_empty(&zt->post_ct_list)) {
630 		struct rhashtable *m_table = &zt->priv->ct_map_table;
631 		struct nfp_fl_ct_flow_entry *entry, *tmp;
632 		struct nfp_fl_ct_map_entry *map;
633 
634 		WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n");
635 		list_for_each_entry_safe(entry, tmp, &zt->post_ct_list,
636 					 list_node) {
637 			map = rhashtable_lookup_fast(m_table,
638 						     &entry->cookie,
639 						     nfp_ct_map_params);
640 			WARN_ON_ONCE(rhashtable_remove_fast(m_table,
641 							    &map->hash_node,
642 							    nfp_ct_map_params));
643 			nfp_fl_ct_clean_flow_entry(entry);
644 			kfree(map);
645 		}
646 	}
647 
648 	if (zt->nft) {
649 		nf_flow_table_offload_del_cb(zt->nft,
650 					     nfp_fl_ct_handle_nft_flow,
651 					     zt);
652 		zt->nft = NULL;
653 	}
654 
655 	if (!list_empty(&zt->nft_flows_list)) {
656 		struct rhashtable *m_table = &zt->priv->ct_map_table;
657 		struct nfp_fl_ct_flow_entry *entry, *tmp;
658 		struct nfp_fl_ct_map_entry *map;
659 
660 		WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n");
661 		list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list,
662 					 list_node) {
663 			map = rhashtable_lookup_fast(m_table,
664 						     &entry->cookie,
665 						     nfp_ct_map_params);
666 			WARN_ON_ONCE(rhashtable_remove_fast(m_table,
667 							    &map->hash_node,
668 							    nfp_ct_map_params));
669 			nfp_fl_ct_clean_flow_entry(entry);
670 			kfree(map);
671 		}
672 	}
673 
674 	rhashtable_free_and_destroy(&zt->tc_merge_tb,
675 				    nfp_check_rhashtable_empty, NULL);
676 	rhashtable_free_and_destroy(&zt->nft_merge_tb,
677 				    nfp_check_rhashtable_empty, NULL);
678 
679 	kfree(zt);
680 }
681 
682 static void nfp_free_zone_table_entry(void *ptr, void *arg)
683 {
684 	struct nfp_fl_ct_zone_entry *zt = ptr;
685 
686 	nfp_zone_table_entry_destroy(zt);
687 }
688 
689 static void nfp_free_map_table_entry(void *ptr, void *arg)
690 {
691 	struct nfp_fl_ct_map_entry *map = ptr;
692 
693 	if (!map)
694 		return;
695 
696 	kfree(map);
697 }
698 
699 void nfp_flower_metadata_cleanup(struct nfp_app *app)
700 {
701 	struct nfp_flower_priv *priv = app->priv;
702 
703 	if (!priv)
704 		return;
705 
706 	rhashtable_free_and_destroy(&priv->flow_table,
707 				    nfp_check_rhashtable_empty, NULL);
708 	rhashtable_free_and_destroy(&priv->stats_ctx_table,
709 				    nfp_check_rhashtable_empty, NULL);
710 	rhashtable_free_and_destroy(&priv->merge_table,
711 				    nfp_check_rhashtable_empty, NULL);
712 	rhashtable_free_and_destroy(&priv->ct_zone_table,
713 				    nfp_free_zone_table_entry, NULL);
714 	nfp_zone_table_entry_destroy(priv->ct_zone_wc);
715 
716 	rhashtable_free_and_destroy(&priv->ct_map_table,
717 				    nfp_free_map_table_entry, NULL);
718 	rhashtable_free_and_destroy(&priv->neigh_table,
719 				    nfp_check_rhashtable_empty, NULL);
720 	kvfree(priv->stats);
721 	kfree(priv->mask_ids.mask_id_free_list.buf);
722 	kfree(priv->mask_ids.last_used);
723 	vfree(priv->stats_ids.free_list.buf);
724 }
725