1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2021 Corigine, Inc. */
3 
4 #include <net/tc_act/tc_csum.h>
5 #include <net/tc_act/tc_ct.h>
6 
7 #include "conntrack.h"
8 #include "../nfp_port.h"
9 
10 const struct rhashtable_params nfp_tc_ct_merge_params = {
11 	.head_offset		= offsetof(struct nfp_fl_ct_tc_merge,
12 					   hash_node),
13 	.key_len		= sizeof(unsigned long) * 2,
14 	.key_offset		= offsetof(struct nfp_fl_ct_tc_merge, cookie),
15 	.automatic_shrinking	= true,
16 };
17 
18 const struct rhashtable_params nfp_nft_ct_merge_params = {
19 	.head_offset		= offsetof(struct nfp_fl_nft_tc_merge,
20 					   hash_node),
21 	.key_len		= sizeof(unsigned long) * 3,
22 	.key_offset		= offsetof(struct nfp_fl_nft_tc_merge, cookie),
23 	.automatic_shrinking	= true,
24 };
25 
26 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
27 					      enum flow_action_id act_id);
28 
29 /**
30  * get_hashentry() - Wrapper around hashtable lookup.
31  * @ht:		hashtable where entry could be found
32  * @key:	key to lookup
33  * @params:	hashtable params
34  * @size:	size of entry to allocate if not in table
35  *
36  * Returns an entry from a hashtable. If entry does not exist
37  * yet allocate the memory for it and return the new entry.
38  */
39 static void *get_hashentry(struct rhashtable *ht, void *key,
40 			   const struct rhashtable_params params, size_t size)
41 {
42 	void *result;
43 
44 	result = rhashtable_lookup_fast(ht, key, params);
45 
46 	if (result)
47 		return result;
48 
49 	result = kzalloc(size, GFP_KERNEL);
50 	if (!result)
51 		return ERR_PTR(-ENOMEM);
52 
53 	return result;
54 }
55 
56 bool is_pre_ct_flow(struct flow_cls_offload *flow)
57 {
58 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
59 	struct flow_dissector *dissector = rule->match.dissector;
60 	struct flow_action_entry *act;
61 	struct flow_match_ct ct;
62 	int i;
63 
64 	if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) {
65 		flow_rule_match_ct(rule, &ct);
66 		if (ct.key->ct_state)
67 			return false;
68 	}
69 
70 	if (flow->common.chain_index)
71 		return false;
72 
73 	flow_action_for_each(i, act, &flow->rule->action) {
74 		if (act->id == FLOW_ACTION_CT) {
75 			/* The pre_ct rule only have the ct or ct nat action, cannot
76 			 * contains other ct action e.g ct commit and so on.
77 			 */
78 			if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT))
79 				return true;
80 			else
81 				return false;
82 		}
83 	}
84 
85 	return false;
86 }
87 
88 bool is_post_ct_flow(struct flow_cls_offload *flow)
89 {
90 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
91 	struct flow_dissector *dissector = rule->match.dissector;
92 	struct flow_action_entry *act;
93 	bool exist_ct_clear = false;
94 	struct flow_match_ct ct;
95 	int i;
96 
97 	if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) {
98 		flow_rule_match_ct(rule, &ct);
99 		if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
100 			return true;
101 	} else {
102 		/* post ct entry cannot contains any ct action except ct_clear. */
103 		flow_action_for_each(i, act, &flow->rule->action) {
104 			if (act->id == FLOW_ACTION_CT) {
105 				/* ignore ct clear action. */
106 				if (act->ct.action == TCA_CT_ACT_CLEAR) {
107 					exist_ct_clear = true;
108 					continue;
109 				}
110 
111 				return false;
112 			}
113 		}
114 		/* when do nat with ct, the post ct entry ignore the ct status,
115 		 * will match the nat field(sip/dip) instead. In this situation,
116 		 * the flow chain index is not zero and contains ct clear action.
117 		 */
118 		if (flow->common.chain_index && exist_ct_clear)
119 			return true;
120 	}
121 
122 	return false;
123 }
124 
125 /**
126  * get_mangled_key() - Mangle the key if mangle act exists
127  * @rule:	rule that carries the actions
128  * @buf:	pointer to key to be mangled
129  * @offset:	used to adjust mangled offset in L2/L3/L4 header
130  * @key_sz:	key size
131  * @htype:	mangling type
132  *
133  * Returns buf where the mangled key stores.
134  */
135 static void *get_mangled_key(struct flow_rule *rule, void *buf,
136 			     u32 offset, size_t key_sz,
137 			     enum flow_action_mangle_base htype)
138 {
139 	struct flow_action_entry *act;
140 	u32 *val = (u32 *)buf;
141 	u32 off, msk, key;
142 	int i;
143 
144 	flow_action_for_each(i, act, &rule->action) {
145 		if (act->id == FLOW_ACTION_MANGLE &&
146 		    act->mangle.htype == htype) {
147 			off = act->mangle.offset - offset;
148 			msk = act->mangle.mask;
149 			key = act->mangle.val;
150 
151 			/* Mangling is supposed to be u32 aligned */
152 			if (off % 4 || off >= key_sz)
153 				continue;
154 
155 			val[off >> 2] &= msk;
156 			val[off >> 2] |= key;
157 		}
158 	}
159 
160 	return buf;
161 }
162 
163 /* Only tos and ttl are involved in flow_match_ip structure, which
164  * doesn't conform to the layout of ip/ipv6 header definition. So
165  * they need particular process here: fill them into the ip/ipv6
166  * header, so that mangling actions can work directly.
167  */
168 #define NFP_IPV4_TOS_MASK	GENMASK(23, 16)
169 #define NFP_IPV4_TTL_MASK	GENMASK(31, 24)
170 #define NFP_IPV6_TCLASS_MASK	GENMASK(27, 20)
171 #define NFP_IPV6_HLIMIT_MASK	GENMASK(7, 0)
172 static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf,
173 				 bool is_v6)
174 {
175 	struct flow_match_ip match;
176 	/* IPv4's ttl field is in third dword. */
177 	__be32 ip_hdr[3];
178 	u32 tmp, hdr_len;
179 
180 	flow_rule_match_ip(rule, &match);
181 
182 	if (is_v6) {
183 		tmp = FIELD_PREP(NFP_IPV6_TCLASS_MASK, match.key->tos);
184 		ip_hdr[0] = cpu_to_be32(tmp);
185 		tmp = FIELD_PREP(NFP_IPV6_HLIMIT_MASK, match.key->ttl);
186 		ip_hdr[1] = cpu_to_be32(tmp);
187 		hdr_len = 2 * sizeof(__be32);
188 	} else {
189 		tmp = FIELD_PREP(NFP_IPV4_TOS_MASK, match.key->tos);
190 		ip_hdr[0] = cpu_to_be32(tmp);
191 		tmp = FIELD_PREP(NFP_IPV4_TTL_MASK, match.key->ttl);
192 		ip_hdr[2] = cpu_to_be32(tmp);
193 		hdr_len = 3 * sizeof(__be32);
194 	}
195 
196 	get_mangled_key(rule, ip_hdr, 0, hdr_len,
197 			is_v6 ? FLOW_ACT_MANGLE_HDR_TYPE_IP6 :
198 				FLOW_ACT_MANGLE_HDR_TYPE_IP4);
199 
200 	match.key = buf;
201 
202 	if (is_v6) {
203 		tmp = be32_to_cpu(ip_hdr[0]);
204 		match.key->tos = FIELD_GET(NFP_IPV6_TCLASS_MASK, tmp);
205 		tmp = be32_to_cpu(ip_hdr[1]);
206 		match.key->ttl = FIELD_GET(NFP_IPV6_HLIMIT_MASK, tmp);
207 	} else {
208 		tmp = be32_to_cpu(ip_hdr[0]);
209 		match.key->tos = FIELD_GET(NFP_IPV4_TOS_MASK, tmp);
210 		tmp = be32_to_cpu(ip_hdr[2]);
211 		match.key->ttl = FIELD_GET(NFP_IPV4_TTL_MASK, tmp);
212 	}
213 
214 	return buf;
215 }
216 
217 /* Note entry1 and entry2 are not swappable. only skip ip and
218  * tport merge check for pre_ct and post_ct when pre_ct do nat.
219  */
220 static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1,
221 					   struct nfp_fl_ct_flow_entry *entry2)
222 {
223 	/* only pre_ct have NFP_FL_ACTION_DO_NAT flag. */
224 	if ((entry1->flags & NFP_FL_ACTION_DO_NAT) &&
225 	    entry2->type == CT_TYPE_POST_CT)
226 		return false;
227 
228 	return true;
229 }
230 
231 /* Note entry1 and entry2 are not swappable, entry1 should be
232  * the former flow whose mangle action need be taken into account
233  * if existed, and entry2 should be the latter flow whose action
234  * we don't care.
235  */
236 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
237 			      struct nfp_fl_ct_flow_entry *entry2)
238 {
239 	unsigned long long ovlp_keys;
240 	bool out, is_v6 = false;
241 	u8 ip_proto = 0;
242 	ovlp_keys = entry1->rule->match.dissector->used_keys &
243 			entry2->rule->match.dissector->used_keys;
244 	/* Temporary buffer for mangling keys, 64 is enough to cover max
245 	 * struct size of key in various fields that may be mangled.
246 	 * Supported fields to mangle:
247 	 * mac_src/mac_dst(struct flow_match_eth_addrs, 12B)
248 	 * nw_tos/nw_ttl(struct flow_match_ip, 2B)
249 	 * nw_src/nw_dst(struct flow_match_ipv4/6_addrs, 32B)
250 	 * tp_src/tp_dst(struct flow_match_ports, 4B)
251 	 */
252 	char buf[64];
253 
254 	if (entry1->netdev && entry2->netdev &&
255 	    entry1->netdev != entry2->netdev)
256 		return -EINVAL;
257 
258 	/* Check the overlapped fields one by one, the unmasked part
259 	 * should not conflict with each other.
260 	 */
261 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) {
262 		struct flow_match_control match1, match2;
263 
264 		flow_rule_match_control(entry1->rule, &match1);
265 		flow_rule_match_control(entry2->rule, &match2);
266 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
267 		if (out)
268 			goto check_failed;
269 	}
270 
271 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_BASIC)) {
272 		struct flow_match_basic match1, match2;
273 
274 		flow_rule_match_basic(entry1->rule, &match1);
275 		flow_rule_match_basic(entry2->rule, &match2);
276 
277 		/* n_proto field is a must in ct-related flows,
278 		 * it should be either ipv4 or ipv6.
279 		 */
280 		is_v6 = match1.key->n_proto == htons(ETH_P_IPV6);
281 		/* ip_proto field is a must when port field is cared */
282 		ip_proto = match1.key->ip_proto;
283 
284 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
285 		if (out)
286 			goto check_failed;
287 	}
288 
289 	/* if pre ct entry do nat, the nat ip exists in nft entry,
290 	 * will be do merge check when do nft and post ct merge,
291 	 * so skip this ip merge check here.
292 	 */
293 	if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) &&
294 	    nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
295 		struct flow_match_ipv4_addrs match1, match2;
296 
297 		flow_rule_match_ipv4_addrs(entry1->rule, &match1);
298 		flow_rule_match_ipv4_addrs(entry2->rule, &match2);
299 
300 		memcpy(buf, match1.key, sizeof(*match1.key));
301 		match1.key = get_mangled_key(entry1->rule, buf,
302 					     offsetof(struct iphdr, saddr),
303 					     sizeof(*match1.key),
304 					     FLOW_ACT_MANGLE_HDR_TYPE_IP4);
305 
306 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
307 		if (out)
308 			goto check_failed;
309 	}
310 
311 	/* if pre ct entry do nat, the nat ip exists in nft entry,
312 	 * will be do merge check when do nft and post ct merge,
313 	 * so skip this ip merge check here.
314 	 */
315 	if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) &&
316 	    nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
317 		struct flow_match_ipv6_addrs match1, match2;
318 
319 		flow_rule_match_ipv6_addrs(entry1->rule, &match1);
320 		flow_rule_match_ipv6_addrs(entry2->rule, &match2);
321 
322 		memcpy(buf, match1.key, sizeof(*match1.key));
323 		match1.key = get_mangled_key(entry1->rule, buf,
324 					     offsetof(struct ipv6hdr, saddr),
325 					     sizeof(*match1.key),
326 					     FLOW_ACT_MANGLE_HDR_TYPE_IP6);
327 
328 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
329 		if (out)
330 			goto check_failed;
331 	}
332 
333 	/* if pre ct entry do nat, the nat tport exists in nft entry,
334 	 * will be do merge check when do nft and post ct merge,
335 	 * so skip this tport merge check here.
336 	 */
337 	if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_PORTS)) &&
338 	    nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
339 		enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC;
340 		struct flow_match_ports match1, match2;
341 
342 		flow_rule_match_ports(entry1->rule, &match1);
343 		flow_rule_match_ports(entry2->rule, &match2);
344 
345 		if (ip_proto == IPPROTO_UDP)
346 			htype = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
347 		else if (ip_proto == IPPROTO_TCP)
348 			htype = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
349 
350 		memcpy(buf, match1.key, sizeof(*match1.key));
351 		match1.key = get_mangled_key(entry1->rule, buf, 0,
352 					     sizeof(*match1.key), htype);
353 
354 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
355 		if (out)
356 			goto check_failed;
357 	}
358 
359 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
360 		struct flow_match_eth_addrs match1, match2;
361 
362 		flow_rule_match_eth_addrs(entry1->rule, &match1);
363 		flow_rule_match_eth_addrs(entry2->rule, &match2);
364 
365 		memcpy(buf, match1.key, sizeof(*match1.key));
366 		match1.key = get_mangled_key(entry1->rule, buf, 0,
367 					     sizeof(*match1.key),
368 					     FLOW_ACT_MANGLE_HDR_TYPE_ETH);
369 
370 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
371 		if (out)
372 			goto check_failed;
373 	}
374 
375 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN)) {
376 		struct flow_match_vlan match1, match2;
377 
378 		flow_rule_match_vlan(entry1->rule, &match1);
379 		flow_rule_match_vlan(entry2->rule, &match2);
380 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
381 		if (out)
382 			goto check_failed;
383 	}
384 
385 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_MPLS)) {
386 		struct flow_match_mpls match1, match2;
387 
388 		flow_rule_match_mpls(entry1->rule, &match1);
389 		flow_rule_match_mpls(entry2->rule, &match2);
390 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
391 		if (out)
392 			goto check_failed;
393 	}
394 
395 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_TCP)) {
396 		struct flow_match_tcp match1, match2;
397 
398 		flow_rule_match_tcp(entry1->rule, &match1);
399 		flow_rule_match_tcp(entry2->rule, &match2);
400 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
401 		if (out)
402 			goto check_failed;
403 	}
404 
405 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IP)) {
406 		struct flow_match_ip match1, match2;
407 
408 		flow_rule_match_ip(entry1->rule, &match1);
409 		flow_rule_match_ip(entry2->rule, &match2);
410 
411 		match1.key = get_mangled_tos_ttl(entry1->rule, buf, is_v6);
412 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
413 		if (out)
414 			goto check_failed;
415 	}
416 
417 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
418 		struct flow_match_enc_keyid match1, match2;
419 
420 		flow_rule_match_enc_keyid(entry1->rule, &match1);
421 		flow_rule_match_enc_keyid(entry2->rule, &match2);
422 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
423 		if (out)
424 			goto check_failed;
425 	}
426 
427 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
428 		struct flow_match_ipv4_addrs match1, match2;
429 
430 		flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
431 		flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
432 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
433 		if (out)
434 			goto check_failed;
435 	}
436 
437 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
438 		struct flow_match_ipv6_addrs match1, match2;
439 
440 		flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
441 		flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
442 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
443 		if (out)
444 			goto check_failed;
445 	}
446 
447 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
448 		struct flow_match_control match1, match2;
449 
450 		flow_rule_match_enc_control(entry1->rule, &match1);
451 		flow_rule_match_enc_control(entry2->rule, &match2);
452 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
453 		if (out)
454 			goto check_failed;
455 	}
456 
457 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP)) {
458 		struct flow_match_ip match1, match2;
459 
460 		flow_rule_match_enc_ip(entry1->rule, &match1);
461 		flow_rule_match_enc_ip(entry2->rule, &match2);
462 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
463 		if (out)
464 			goto check_failed;
465 	}
466 
467 	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
468 		struct flow_match_enc_opts match1, match2;
469 
470 		flow_rule_match_enc_opts(entry1->rule, &match1);
471 		flow_rule_match_enc_opts(entry2->rule, &match2);
472 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
473 		if (out)
474 			goto check_failed;
475 	}
476 
477 	return 0;
478 
479 check_failed:
480 	return -EINVAL;
481 }
482 
483 static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in,
484 				   struct flow_rule *rule)
485 {
486 	struct flow_match_vlan match;
487 
488 	if (unlikely(flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)))
489 		return -EOPNOTSUPP;
490 
491 	/* post_ct does not match VLAN KEY, can be merged. */
492 	if (likely(!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)))
493 		return 0;
494 
495 	switch (a_in->id) {
496 	/* pre_ct has pop vlan, post_ct cannot match VLAN KEY, cannot be merged. */
497 	case FLOW_ACTION_VLAN_POP:
498 		return -EOPNOTSUPP;
499 
500 	case FLOW_ACTION_VLAN_PUSH:
501 	case FLOW_ACTION_VLAN_MANGLE:
502 		flow_rule_match_vlan(rule, &match);
503 		/* different vlan id, cannot be merged. */
504 		if ((match.key->vlan_id & match.mask->vlan_id) ^
505 		    (a_in->vlan.vid & match.mask->vlan_id))
506 			return -EOPNOTSUPP;
507 
508 		/* different tpid, cannot be merged. */
509 		if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^
510 		    (a_in->vlan.proto & match.mask->vlan_tpid))
511 			return -EOPNOTSUPP;
512 
513 		/* different priority, cannot be merged. */
514 		if ((match.key->vlan_priority & match.mask->vlan_priority) ^
515 		    (a_in->vlan.prio & match.mask->vlan_priority))
516 			return -EOPNOTSUPP;
517 
518 		break;
519 	default:
520 		return -EOPNOTSUPP;
521 	}
522 
523 	return 0;
524 }
525 
526 /* Extra check for multiple ct-zones merge
527  * currently surpport nft entries merge check in different zones
528  */
529 static int nfp_ct_merge_extra_check(struct nfp_fl_ct_flow_entry *nft_entry,
530 				    struct nfp_fl_ct_tc_merge *tc_m_entry)
531 {
532 	struct nfp_fl_nft_tc_merge *prev_nft_m_entry;
533 	struct nfp_fl_ct_flow_entry *pre_ct_entry;
534 
535 	pre_ct_entry = tc_m_entry->pre_ct_parent;
536 	prev_nft_m_entry = pre_ct_entry->prev_m_entries[pre_ct_entry->num_prev_m_entries - 1];
537 
538 	return nfp_ct_merge_check(prev_nft_m_entry->nft_parent, nft_entry);
539 }
540 
541 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
542 				  struct nfp_fl_ct_flow_entry *post_ct_entry,
543 				  struct nfp_fl_ct_flow_entry *nft_entry)
544 {
545 	struct flow_action_entry *act;
546 	int i, err;
547 
548 	/* Check for pre_ct->action conflicts */
549 	flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
550 		switch (act->id) {
551 		case FLOW_ACTION_VLAN_PUSH:
552 		case FLOW_ACTION_VLAN_POP:
553 		case FLOW_ACTION_VLAN_MANGLE:
554 			err = nfp_ct_check_vlan_merge(act, post_ct_entry->rule);
555 			if (err)
556 				return err;
557 			break;
558 		case FLOW_ACTION_MPLS_PUSH:
559 		case FLOW_ACTION_MPLS_POP:
560 		case FLOW_ACTION_MPLS_MANGLE:
561 			return -EOPNOTSUPP;
562 		default:
563 			break;
564 		}
565 	}
566 
567 	/* Check for nft->action conflicts */
568 	flow_action_for_each(i, act, &nft_entry->rule->action) {
569 		switch (act->id) {
570 		case FLOW_ACTION_VLAN_PUSH:
571 		case FLOW_ACTION_VLAN_POP:
572 		case FLOW_ACTION_VLAN_MANGLE:
573 		case FLOW_ACTION_MPLS_PUSH:
574 		case FLOW_ACTION_MPLS_POP:
575 		case FLOW_ACTION_MPLS_MANGLE:
576 			return -EOPNOTSUPP;
577 		default:
578 			break;
579 		}
580 	}
581 	return 0;
582 }
583 
584 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
585 			     struct nfp_fl_ct_flow_entry *nft_entry)
586 {
587 	struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
588 	struct flow_action_entry *ct_met;
589 	struct flow_match_ct ct;
590 	int i;
591 
592 	ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
593 	if (ct_met && (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT))) {
594 		u32 *act_lbl;
595 
596 		act_lbl = ct_met->ct_metadata.labels;
597 		flow_rule_match_ct(post_ct_entry->rule, &ct);
598 		for (i = 0; i < 4; i++) {
599 			if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
600 			    (act_lbl[i] & ct.mask->ct_labels[i]))
601 				return -EINVAL;
602 		}
603 
604 		if ((ct.key->ct_mark & ct.mask->ct_mark) ^
605 		    (ct_met->ct_metadata.mark & ct.mask->ct_mark))
606 			return -EINVAL;
607 
608 		return 0;
609 	} else {
610 		/* post_ct with ct clear action will not match the
611 		 * ct status when nft is nat entry.
612 		 */
613 		if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE)
614 			return 0;
615 	}
616 
617 	return -EINVAL;
618 }
619 
620 static int
621 nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map)
622 {
623 	int key_size;
624 
625 	/* This field must always be present */
626 	key_size = sizeof(struct nfp_flower_meta_tci);
627 	map[FLOW_PAY_META_TCI] = 0;
628 
629 	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_EXT_META) {
630 		map[FLOW_PAY_EXT_META] = key_size;
631 		key_size += sizeof(struct nfp_flower_ext_meta);
632 	}
633 	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_PORT) {
634 		map[FLOW_PAY_INPORT] = key_size;
635 		key_size += sizeof(struct nfp_flower_in_port);
636 	}
637 	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_MAC) {
638 		map[FLOW_PAY_MAC_MPLS] = key_size;
639 		key_size += sizeof(struct nfp_flower_mac_mpls);
640 	}
641 	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_TP) {
642 		map[FLOW_PAY_L4] = key_size;
643 		key_size += sizeof(struct nfp_flower_tp_ports);
644 	}
645 	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV4) {
646 		map[FLOW_PAY_IPV4] = key_size;
647 		key_size += sizeof(struct nfp_flower_ipv4);
648 	}
649 	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV6) {
650 		map[FLOW_PAY_IPV6] = key_size;
651 		key_size += sizeof(struct nfp_flower_ipv6);
652 	}
653 
654 	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_QINQ) {
655 		map[FLOW_PAY_QINQ] = key_size;
656 		key_size += sizeof(struct nfp_flower_vlan);
657 	}
658 
659 	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
660 		map[FLOW_PAY_GRE] = key_size;
661 		if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
662 			key_size += sizeof(struct nfp_flower_ipv6_gre_tun);
663 		else
664 			key_size += sizeof(struct nfp_flower_ipv4_gre_tun);
665 	}
666 
667 	if ((in_key_ls.key_layer & NFP_FLOWER_LAYER_VXLAN) ||
668 	    (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE)) {
669 		map[FLOW_PAY_UDP_TUN] = key_size;
670 		if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
671 			key_size += sizeof(struct nfp_flower_ipv6_udp_tun);
672 		else
673 			key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
674 	}
675 
676 	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
677 		map[FLOW_PAY_GENEVE_OPT] = key_size;
678 		key_size += sizeof(struct nfp_flower_geneve_options);
679 	}
680 
681 	return key_size;
682 }
683 
684 /* get the csum flag according the ip proto and mangle action. */
685 static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u32 *csum)
686 {
687 	if (a_in->id != FLOW_ACTION_MANGLE)
688 		return;
689 
690 	switch (a_in->mangle.htype) {
691 	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
692 		*csum |= TCA_CSUM_UPDATE_FLAG_IPV4HDR;
693 		if (ip_proto == IPPROTO_TCP)
694 			*csum |= TCA_CSUM_UPDATE_FLAG_TCP;
695 		else if (ip_proto == IPPROTO_UDP)
696 			*csum |= TCA_CSUM_UPDATE_FLAG_UDP;
697 		break;
698 	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
699 		*csum |= TCA_CSUM_UPDATE_FLAG_TCP;
700 		break;
701 	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
702 		*csum |= TCA_CSUM_UPDATE_FLAG_UDP;
703 		break;
704 	default:
705 		break;
706 	}
707 }
708 
709 static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
710 					struct nfp_flower_priv *priv,
711 					struct net_device *netdev,
712 					struct nfp_fl_payload *flow_pay,
713 					int num_rules)
714 {
715 	enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
716 	struct flow_action_entry *a_in;
717 	int i, j, id, num_actions = 0;
718 	struct flow_rule *a_rule;
719 	int err = 0, offset = 0;
720 
721 	for (i = 0; i < num_rules; i++)
722 		num_actions += rules[i]->action.num_entries;
723 
724 	/* Add one action to make sure there is enough room to add an checksum action
725 	 * when do nat.
726 	 */
727 	a_rule = flow_rule_alloc(num_actions + (num_rules / 2));
728 	if (!a_rule)
729 		return -ENOMEM;
730 
731 	/* post_ct entry have one action at least. */
732 	if (rules[num_rules - 1]->action.num_entries != 0)
733 		tmp_stats = rules[num_rules - 1]->action.entries[0].hw_stats;
734 
735 	/* Actions need a BASIC dissector. */
736 	a_rule->match = rules[0]->match;
737 
738 	/* Copy actions */
739 	for (j = 0; j < num_rules; j++) {
740 		u32 csum_updated = 0;
741 		u8 ip_proto = 0;
742 
743 		if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) {
744 			struct flow_match_basic match;
745 
746 			/* ip_proto is the only field that is needed in later compile_action,
747 			 * needed to set the correct checksum flags. It doesn't really matter
748 			 * which input rule's ip_proto field we take as the earlier merge checks
749 			 * would have made sure that they don't conflict. We do not know which
750 			 * of the subflows would have the ip_proto filled in, so we need to iterate
751 			 * through the subflows and assign the proper subflow to a_rule
752 			 */
753 			flow_rule_match_basic(rules[j], &match);
754 			if (match.mask->ip_proto) {
755 				a_rule->match = rules[j]->match;
756 				ip_proto = match.key->ip_proto;
757 			}
758 		}
759 
760 		for (i = 0; i < rules[j]->action.num_entries; i++) {
761 			a_in = &rules[j]->action.entries[i];
762 			id = a_in->id;
763 
764 			/* Ignore CT related actions as these would already have
765 			 * been taken care of by previous checks, and we do not send
766 			 * any CT actions to the firmware.
767 			 */
768 			switch (id) {
769 			case FLOW_ACTION_CT:
770 			case FLOW_ACTION_GOTO:
771 			case FLOW_ACTION_CT_METADATA:
772 				continue;
773 			default:
774 				/* nft entry is generated by tc ct, which mangle action do not care
775 				 * the stats, inherit the post entry stats to meet the
776 				 * flow_action_hw_stats_check.
777 				 * nft entry flow rules are at odd array index.
778 				 */
779 				if (j & 0x01) {
780 					if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE)
781 						a_in->hw_stats = tmp_stats;
782 					nfp_fl_get_csum_flag(a_in, ip_proto, &csum_updated);
783 				}
784 				memcpy(&a_rule->action.entries[offset++],
785 				       a_in, sizeof(struct flow_action_entry));
786 				break;
787 			}
788 		}
789 		/* nft entry have mangle action, but do not have checksum action when do NAT,
790 		 * hardware will automatically fix IPv4 and TCP/UDP checksum. so add an csum action
791 		 * to meet csum action check.
792 		 */
793 		if (csum_updated) {
794 			struct flow_action_entry *csum_action;
795 
796 			csum_action = &a_rule->action.entries[offset++];
797 			csum_action->id = FLOW_ACTION_CSUM;
798 			csum_action->csum_flags = csum_updated;
799 			csum_action->hw_stats = tmp_stats;
800 		}
801 	}
802 
803 	/* Some actions would have been ignored, so update the num_entries field */
804 	a_rule->action.num_entries = offset;
805 	err = nfp_flower_compile_action(priv->app, a_rule, netdev, flow_pay, NULL);
806 	kfree(a_rule);
807 
808 	return err;
809 }
810 
811 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
812 {
813 	enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
814 	struct nfp_fl_ct_zone_entry *zt = m_entry->zt;
815 	struct flow_rule *rules[NFP_MAX_ENTRY_RULES];
816 	struct nfp_fl_ct_flow_entry *pre_ct_entry;
817 	struct nfp_fl_key_ls key_layer, tmp_layer;
818 	struct nfp_flower_priv *priv = zt->priv;
819 	u16 key_map[_FLOW_PAY_LAYERS_MAX];
820 	struct nfp_fl_payload *flow_pay;
821 	u8 *key, *msk, *kdata, *mdata;
822 	struct nfp_port *port = NULL;
823 	int num_rules, err, i, j = 0;
824 	struct net_device *netdev;
825 	bool qinq_sup;
826 	u32 port_id;
827 	u16 offset;
828 
829 	netdev = m_entry->netdev;
830 	qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ);
831 
832 	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
833 	num_rules = pre_ct_entry->num_prev_m_entries * 2 + _CT_TYPE_MAX;
834 
835 	for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) {
836 		rules[j++] = pre_ct_entry->prev_m_entries[i]->tc_m_parent->pre_ct_parent->rule;
837 		rules[j++] = pre_ct_entry->prev_m_entries[i]->nft_parent->rule;
838 	}
839 
840 	rules[j++] = m_entry->tc_m_parent->pre_ct_parent->rule;
841 	rules[j++] = m_entry->nft_parent->rule;
842 	rules[j++] = m_entry->tc_m_parent->post_ct_parent->rule;
843 
844 	memset(&key_layer, 0, sizeof(struct nfp_fl_key_ls));
845 	memset(&key_map, 0, sizeof(key_map));
846 
847 	/* Calculate the resultant key layer and size for offload */
848 	for (i = 0; i < num_rules; i++) {
849 		err = nfp_flower_calculate_key_layers(priv->app,
850 						      m_entry->netdev,
851 						      &tmp_layer, rules[i],
852 						      &tun_type, NULL);
853 		if (err)
854 			return err;
855 
856 		key_layer.key_layer |= tmp_layer.key_layer;
857 		key_layer.key_layer_two |= tmp_layer.key_layer_two;
858 	}
859 	key_layer.key_size = nfp_fl_calc_key_layers_sz(key_layer, key_map);
860 
861 	flow_pay = nfp_flower_allocate_new(&key_layer);
862 	if (!flow_pay)
863 		return -ENOMEM;
864 
865 	memset(flow_pay->unmasked_data, 0, key_layer.key_size);
866 	memset(flow_pay->mask_data, 0, key_layer.key_size);
867 
868 	kdata = flow_pay->unmasked_data;
869 	mdata = flow_pay->mask_data;
870 
871 	offset = key_map[FLOW_PAY_META_TCI];
872 	key = kdata + offset;
873 	msk = mdata + offset;
874 	nfp_flower_compile_meta((struct nfp_flower_meta_tci *)key,
875 				(struct nfp_flower_meta_tci *)msk,
876 				key_layer.key_layer);
877 
878 	if (NFP_FLOWER_LAYER_EXT_META & key_layer.key_layer) {
879 		offset =  key_map[FLOW_PAY_EXT_META];
880 		key = kdata + offset;
881 		msk = mdata + offset;
882 		nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)key,
883 					    key_layer.key_layer_two);
884 		nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)msk,
885 					    key_layer.key_layer_two);
886 	}
887 
888 	/* Using in_port from the -trk rule. The tc merge checks should already
889 	 * be checking that the ingress netdevs are the same
890 	 */
891 	port_id = nfp_flower_get_port_id_from_netdev(priv->app, netdev);
892 	offset = key_map[FLOW_PAY_INPORT];
893 	key = kdata + offset;
894 	msk = mdata + offset;
895 	err = nfp_flower_compile_port((struct nfp_flower_in_port *)key,
896 				      port_id, false, tun_type, NULL);
897 	if (err)
898 		goto ct_offload_err;
899 	err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk,
900 				      port_id, true, tun_type, NULL);
901 	if (err)
902 		goto ct_offload_err;
903 
904 	/* This following part works on the assumption that previous checks has
905 	 * already filtered out flows that has different values for the different
906 	 * layers. Here we iterate through all three rules and merge their respective
907 	 * masked value(cared bits), basic method is:
908 	 * final_key = (r1_key & r1_mask) | (r2_key & r2_mask) | (r3_key & r3_mask)
909 	 * final_mask = r1_mask | r2_mask | r3_mask
910 	 * If none of the rules contains a match that is also fine, that simply means
911 	 * that the layer is not present.
912 	 */
913 	if (!qinq_sup) {
914 		for (i = 0; i < num_rules; i++) {
915 			offset = key_map[FLOW_PAY_META_TCI];
916 			key = kdata + offset;
917 			msk = mdata + offset;
918 			nfp_flower_compile_tci((struct nfp_flower_meta_tci *)key,
919 					       (struct nfp_flower_meta_tci *)msk,
920 					       rules[i]);
921 		}
922 	}
923 
924 	if (NFP_FLOWER_LAYER_MAC & key_layer.key_layer) {
925 		offset = key_map[FLOW_PAY_MAC_MPLS];
926 		key = kdata + offset;
927 		msk = mdata + offset;
928 		for (i = 0; i < num_rules; i++) {
929 			nfp_flower_compile_mac((struct nfp_flower_mac_mpls *)key,
930 					       (struct nfp_flower_mac_mpls *)msk,
931 					       rules[i]);
932 			err = nfp_flower_compile_mpls((struct nfp_flower_mac_mpls *)key,
933 						      (struct nfp_flower_mac_mpls *)msk,
934 						      rules[i], NULL);
935 			if (err)
936 				goto ct_offload_err;
937 		}
938 	}
939 
940 	if (NFP_FLOWER_LAYER_IPV4 & key_layer.key_layer) {
941 		offset = key_map[FLOW_PAY_IPV4];
942 		key = kdata + offset;
943 		msk = mdata + offset;
944 		for (i = 0; i < num_rules; i++) {
945 			nfp_flower_compile_ipv4((struct nfp_flower_ipv4 *)key,
946 						(struct nfp_flower_ipv4 *)msk,
947 						rules[i]);
948 		}
949 	}
950 
951 	if (NFP_FLOWER_LAYER_IPV6 & key_layer.key_layer) {
952 		offset = key_map[FLOW_PAY_IPV6];
953 		key = kdata + offset;
954 		msk = mdata + offset;
955 		for (i = 0; i < num_rules; i++) {
956 			nfp_flower_compile_ipv6((struct nfp_flower_ipv6 *)key,
957 						(struct nfp_flower_ipv6 *)msk,
958 						rules[i]);
959 		}
960 	}
961 
962 	if (NFP_FLOWER_LAYER_TP & key_layer.key_layer) {
963 		offset = key_map[FLOW_PAY_L4];
964 		key = kdata + offset;
965 		msk = mdata + offset;
966 		for (i = 0; i < num_rules; i++) {
967 			nfp_flower_compile_tport((struct nfp_flower_tp_ports *)key,
968 						 (struct nfp_flower_tp_ports *)msk,
969 						 rules[i]);
970 		}
971 	}
972 
973 	if (NFP_FLOWER_LAYER2_QINQ & key_layer.key_layer_two) {
974 		offset = key_map[FLOW_PAY_QINQ];
975 		key = kdata + offset;
976 		msk = mdata + offset;
977 		for (i = 0; i < num_rules; i++) {
978 			nfp_flower_compile_vlan((struct nfp_flower_vlan *)key,
979 						(struct nfp_flower_vlan *)msk,
980 						rules[i]);
981 		}
982 	}
983 
984 	if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
985 		offset = key_map[FLOW_PAY_GRE];
986 		key = kdata + offset;
987 		msk = mdata + offset;
988 		if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
989 			struct nfp_flower_ipv6_gre_tun *gre_match;
990 			struct nfp_ipv6_addr_entry *entry;
991 			struct in6_addr *dst;
992 
993 			for (i = 0; i < num_rules; i++) {
994 				nfp_flower_compile_ipv6_gre_tun((void *)key,
995 								(void *)msk, rules[i]);
996 			}
997 			gre_match = (struct nfp_flower_ipv6_gre_tun *)key;
998 			dst = &gre_match->ipv6.dst;
999 
1000 			entry = nfp_tunnel_add_ipv6_off(priv->app, dst);
1001 			if (!entry) {
1002 				err = -ENOMEM;
1003 				goto ct_offload_err;
1004 			}
1005 
1006 			flow_pay->nfp_tun_ipv6 = entry;
1007 		} else {
1008 			__be32 dst;
1009 
1010 			for (i = 0; i < num_rules; i++) {
1011 				nfp_flower_compile_ipv4_gre_tun((void *)key,
1012 								(void *)msk, rules[i]);
1013 			}
1014 			dst = ((struct nfp_flower_ipv4_gre_tun *)key)->ipv4.dst;
1015 
1016 			/* Store the tunnel destination in the rule data.
1017 			 * This must be present and be an exact match.
1018 			 */
1019 			flow_pay->nfp_tun_ipv4_addr = dst;
1020 			nfp_tunnel_add_ipv4_off(priv->app, dst);
1021 		}
1022 	}
1023 
1024 	if (key_layer.key_layer & NFP_FLOWER_LAYER_VXLAN ||
1025 	    key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE) {
1026 		offset = key_map[FLOW_PAY_UDP_TUN];
1027 		key = kdata + offset;
1028 		msk = mdata + offset;
1029 		if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
1030 			struct nfp_flower_ipv6_udp_tun *udp_match;
1031 			struct nfp_ipv6_addr_entry *entry;
1032 			struct in6_addr *dst;
1033 
1034 			for (i = 0; i < num_rules; i++) {
1035 				nfp_flower_compile_ipv6_udp_tun((void *)key,
1036 								(void *)msk, rules[i]);
1037 			}
1038 			udp_match = (struct nfp_flower_ipv6_udp_tun *)key;
1039 			dst = &udp_match->ipv6.dst;
1040 
1041 			entry = nfp_tunnel_add_ipv6_off(priv->app, dst);
1042 			if (!entry) {
1043 				err = -ENOMEM;
1044 				goto ct_offload_err;
1045 			}
1046 
1047 			flow_pay->nfp_tun_ipv6 = entry;
1048 		} else {
1049 			__be32 dst;
1050 
1051 			for (i = 0; i < num_rules; i++) {
1052 				nfp_flower_compile_ipv4_udp_tun((void *)key,
1053 								(void *)msk, rules[i]);
1054 			}
1055 			dst = ((struct nfp_flower_ipv4_udp_tun *)key)->ipv4.dst;
1056 
1057 			/* Store the tunnel destination in the rule data.
1058 			 * This must be present and be an exact match.
1059 			 */
1060 			flow_pay->nfp_tun_ipv4_addr = dst;
1061 			nfp_tunnel_add_ipv4_off(priv->app, dst);
1062 		}
1063 
1064 		if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
1065 			offset = key_map[FLOW_PAY_GENEVE_OPT];
1066 			key = kdata + offset;
1067 			msk = mdata + offset;
1068 			for (i = 0; i < num_rules; i++)
1069 				nfp_flower_compile_geneve_opt(key, msk, rules[i]);
1070 		}
1071 	}
1072 
1073 	/* Merge actions into flow_pay */
1074 	err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay, num_rules);
1075 	if (err)
1076 		goto ct_offload_err;
1077 
1078 	/* Use the pointer address as the cookie, but set the last bit to 1.
1079 	 * This is to avoid the 'is_merge_flow' check from detecting this as
1080 	 * an already merged flow. This works since address alignment means
1081 	 * that the last bit for pointer addresses will be 0.
1082 	 */
1083 	flow_pay->tc_flower_cookie = ((unsigned long)flow_pay) | 0x1;
1084 	err = nfp_compile_flow_metadata(priv->app, flow_pay->tc_flower_cookie,
1085 					flow_pay, netdev, NULL);
1086 	if (err)
1087 		goto ct_offload_err;
1088 
1089 	if (nfp_netdev_is_nfp_repr(netdev))
1090 		port = nfp_port_from_netdev(netdev);
1091 
1092 	err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node,
1093 				     nfp_flower_table_params);
1094 	if (err)
1095 		goto ct_release_offload_meta_err;
1096 
1097 	err = nfp_flower_xmit_flow(priv->app, flow_pay,
1098 				   NFP_FLOWER_CMSG_TYPE_FLOW_ADD);
1099 	if (err)
1100 		goto ct_remove_rhash_err;
1101 
1102 	m_entry->tc_flower_cookie = flow_pay->tc_flower_cookie;
1103 	m_entry->flow_pay = flow_pay;
1104 
1105 	if (port)
1106 		port->tc_offload_cnt++;
1107 
1108 	return err;
1109 
1110 ct_remove_rhash_err:
1111 	WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1112 					    &flow_pay->fl_node,
1113 					    nfp_flower_table_params));
1114 ct_release_offload_meta_err:
1115 	nfp_modify_flow_metadata(priv->app, flow_pay);
1116 ct_offload_err:
1117 	if (flow_pay->nfp_tun_ipv4_addr)
1118 		nfp_tunnel_del_ipv4_off(priv->app, flow_pay->nfp_tun_ipv4_addr);
1119 	if (flow_pay->nfp_tun_ipv6)
1120 		nfp_tunnel_put_ipv6_off(priv->app, flow_pay->nfp_tun_ipv6);
1121 	kfree(flow_pay->action_data);
1122 	kfree(flow_pay->mask_data);
1123 	kfree(flow_pay->unmasked_data);
1124 	kfree(flow_pay);
1125 	return err;
1126 }
1127 
1128 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
1129 				 struct net_device *netdev)
1130 {
1131 	struct nfp_flower_priv *priv = app->priv;
1132 	struct nfp_fl_payload *flow_pay;
1133 	struct nfp_port *port = NULL;
1134 	int err = 0;
1135 
1136 	if (nfp_netdev_is_nfp_repr(netdev))
1137 		port = nfp_port_from_netdev(netdev);
1138 
1139 	flow_pay = nfp_flower_search_fl_table(app, cookie, netdev);
1140 	if (!flow_pay)
1141 		return -ENOENT;
1142 
1143 	err = nfp_modify_flow_metadata(app, flow_pay);
1144 	if (err)
1145 		goto err_free_merge_flow;
1146 
1147 	if (flow_pay->nfp_tun_ipv4_addr)
1148 		nfp_tunnel_del_ipv4_off(app, flow_pay->nfp_tun_ipv4_addr);
1149 
1150 	if (flow_pay->nfp_tun_ipv6)
1151 		nfp_tunnel_put_ipv6_off(app, flow_pay->nfp_tun_ipv6);
1152 
1153 	if (!flow_pay->in_hw) {
1154 		err = 0;
1155 		goto err_free_merge_flow;
1156 	}
1157 
1158 	err = nfp_flower_xmit_flow(app, flow_pay,
1159 				   NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
1160 
1161 err_free_merge_flow:
1162 	nfp_flower_del_linked_merge_flows(app, flow_pay);
1163 	if (port)
1164 		port->tc_offload_cnt--;
1165 	kfree(flow_pay->action_data);
1166 	kfree(flow_pay->mask_data);
1167 	kfree(flow_pay->unmasked_data);
1168 	WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1169 					    &flow_pay->fl_node,
1170 					    nfp_flower_table_params));
1171 	kfree_rcu(flow_pay, rcu);
1172 	return err;
1173 }
1174 
1175 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
1176 			       struct nfp_fl_ct_flow_entry *nft_entry,
1177 			       struct nfp_fl_ct_tc_merge *tc_m_entry)
1178 {
1179 	struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1180 	struct nfp_fl_nft_tc_merge *nft_m_entry;
1181 	unsigned long new_cookie[3];
1182 	int err;
1183 
1184 	pre_ct_entry = tc_m_entry->pre_ct_parent;
1185 	post_ct_entry = tc_m_entry->post_ct_parent;
1186 
1187 	err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
1188 	if (err)
1189 		return err;
1190 
1191 	/* Check that the two tc flows are also compatible with
1192 	 * the nft entry. No need to check the pre_ct and post_ct
1193 	 * entries as that was already done during pre_merge.
1194 	 * The nft entry does not have a chain populated, so
1195 	 * skip this check.
1196 	 */
1197 	err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
1198 	if (err)
1199 		return err;
1200 	err = nfp_ct_merge_check(nft_entry, post_ct_entry);
1201 	if (err)
1202 		return err;
1203 	err = nfp_ct_check_meta(post_ct_entry, nft_entry);
1204 	if (err)
1205 		return err;
1206 
1207 	if (pre_ct_entry->num_prev_m_entries > 0) {
1208 		err = nfp_ct_merge_extra_check(nft_entry, tc_m_entry);
1209 		if (err)
1210 			return err;
1211 	}
1212 
1213 	/* Combine tc_merge and nft cookies for this cookie. */
1214 	new_cookie[0] = tc_m_entry->cookie[0];
1215 	new_cookie[1] = tc_m_entry->cookie[1];
1216 	new_cookie[2] = nft_entry->cookie;
1217 	nft_m_entry = get_hashentry(&zt->nft_merge_tb,
1218 				    &new_cookie,
1219 				    nfp_nft_ct_merge_params,
1220 				    sizeof(*nft_m_entry));
1221 
1222 	if (IS_ERR(nft_m_entry))
1223 		return PTR_ERR(nft_m_entry);
1224 
1225 	/* nft_m_entry already present, not merging again */
1226 	if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
1227 		return 0;
1228 
1229 	memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
1230 	nft_m_entry->zt = zt;
1231 	nft_m_entry->tc_m_parent = tc_m_entry;
1232 	nft_m_entry->nft_parent = nft_entry;
1233 	nft_m_entry->tc_flower_cookie = 0;
1234 	/* Copy the netdev from the pre_ct entry. When the tc_m_entry was created
1235 	 * it only combined them if the netdevs were the same, so can use any of them.
1236 	 */
1237 	nft_m_entry->netdev = pre_ct_entry->netdev;
1238 
1239 	/* Add this entry to the tc_m_list and nft_flow lists */
1240 	list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
1241 	list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
1242 
1243 	err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
1244 				     nfp_nft_ct_merge_params);
1245 	if (err)
1246 		goto err_nft_ct_merge_insert;
1247 
1248 	zt->nft_merge_count++;
1249 
1250 	if (post_ct_entry->goto_chain_index > 0)
1251 		return nfp_fl_create_new_pre_ct(nft_m_entry);
1252 
1253 	/* Generate offload structure and send to nfp */
1254 	err = nfp_fl_ct_add_offload(nft_m_entry);
1255 	if (err)
1256 		goto err_nft_ct_offload;
1257 
1258 	return err;
1259 
1260 err_nft_ct_offload:
1261 	nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
1262 			      nft_m_entry->netdev);
1263 err_nft_ct_merge_insert:
1264 	list_del(&nft_m_entry->tc_merge_list);
1265 	list_del(&nft_m_entry->nft_flow_list);
1266 	kfree(nft_m_entry);
1267 	return err;
1268 }
1269 
1270 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
1271 			      struct nfp_fl_ct_flow_entry *ct_entry1,
1272 			      struct nfp_fl_ct_flow_entry *ct_entry2)
1273 {
1274 	struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1275 	struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
1276 	struct nfp_fl_ct_tc_merge *m_entry;
1277 	unsigned long new_cookie[2];
1278 	int err;
1279 
1280 	if (ct_entry1->type == CT_TYPE_PRE_CT) {
1281 		pre_ct_entry = ct_entry1;
1282 		post_ct_entry = ct_entry2;
1283 	} else {
1284 		post_ct_entry = ct_entry1;
1285 		pre_ct_entry = ct_entry2;
1286 	}
1287 
1288 	/* Checks that the chain_index of the filter matches the
1289 	 * chain_index of the GOTO action.
1290 	 */
1291 	if (post_ct_entry->chain_index != pre_ct_entry->goto_chain_index)
1292 		return -EINVAL;
1293 
1294 	err = nfp_ct_merge_check(pre_ct_entry, post_ct_entry);
1295 	if (err)
1296 		return err;
1297 
1298 	new_cookie[0] = pre_ct_entry->cookie;
1299 	new_cookie[1] = post_ct_entry->cookie;
1300 	m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
1301 				nfp_tc_ct_merge_params, sizeof(*m_entry));
1302 	if (IS_ERR(m_entry))
1303 		return PTR_ERR(m_entry);
1304 
1305 	/* m_entry already present, not merging again */
1306 	if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
1307 		return 0;
1308 
1309 	memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
1310 	m_entry->zt = zt;
1311 	m_entry->post_ct_parent = post_ct_entry;
1312 	m_entry->pre_ct_parent = pre_ct_entry;
1313 
1314 	/* Add this entry to the pre_ct and post_ct lists */
1315 	list_add(&m_entry->post_ct_list, &post_ct_entry->children);
1316 	list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
1317 	INIT_LIST_HEAD(&m_entry->children);
1318 
1319 	err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
1320 				     nfp_tc_ct_merge_params);
1321 	if (err)
1322 		goto err_ct_tc_merge_insert;
1323 	zt->tc_merge_count++;
1324 
1325 	/* Merge with existing nft flows */
1326 	list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
1327 				 list_node) {
1328 		nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
1329 	}
1330 
1331 	return 0;
1332 
1333 err_ct_tc_merge_insert:
1334 	list_del(&m_entry->post_ct_list);
1335 	list_del(&m_entry->pre_ct_list);
1336 	kfree(m_entry);
1337 	return err;
1338 }
1339 
1340 static struct
1341 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
1342 					 u16 zone, bool wildcarded)
1343 {
1344 	struct nfp_fl_ct_zone_entry *zt;
1345 	int err;
1346 
1347 	if (wildcarded && priv->ct_zone_wc)
1348 		return priv->ct_zone_wc;
1349 
1350 	if (!wildcarded) {
1351 		zt = get_hashentry(&priv->ct_zone_table, &zone,
1352 				   nfp_zone_table_params, sizeof(*zt));
1353 
1354 		/* If priv is set this is an existing entry, just return it */
1355 		if (IS_ERR(zt) || zt->priv)
1356 			return zt;
1357 	} else {
1358 		zt = kzalloc(sizeof(*zt), GFP_KERNEL);
1359 		if (!zt)
1360 			return ERR_PTR(-ENOMEM);
1361 	}
1362 
1363 	zt->zone = zone;
1364 	zt->priv = priv;
1365 	zt->nft = NULL;
1366 
1367 	/* init the various hash tables and lists */
1368 	INIT_LIST_HEAD(&zt->pre_ct_list);
1369 	INIT_LIST_HEAD(&zt->post_ct_list);
1370 	INIT_LIST_HEAD(&zt->nft_flows_list);
1371 
1372 	err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
1373 	if (err)
1374 		goto err_tc_merge_tb_init;
1375 
1376 	err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
1377 	if (err)
1378 		goto err_nft_merge_tb_init;
1379 
1380 	if (wildcarded) {
1381 		priv->ct_zone_wc = zt;
1382 	} else {
1383 		err = rhashtable_insert_fast(&priv->ct_zone_table,
1384 					     &zt->hash_node,
1385 					     nfp_zone_table_params);
1386 		if (err)
1387 			goto err_zone_insert;
1388 	}
1389 
1390 	return zt;
1391 
1392 err_zone_insert:
1393 	rhashtable_destroy(&zt->nft_merge_tb);
1394 err_nft_merge_tb_init:
1395 	rhashtable_destroy(&zt->tc_merge_tb);
1396 err_tc_merge_tb_init:
1397 	kfree(zt);
1398 	return ERR_PTR(err);
1399 }
1400 
1401 static struct net_device *get_netdev_from_rule(struct flow_rule *rule)
1402 {
1403 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
1404 		struct flow_match_meta match;
1405 
1406 		flow_rule_match_meta(rule, &match);
1407 		if (match.key->ingress_ifindex & match.mask->ingress_ifindex)
1408 			return __dev_get_by_index(&init_net,
1409 						  match.key->ingress_ifindex);
1410 	}
1411 
1412 	return NULL;
1413 }
1414 
1415 static void nfp_nft_ct_translate_mangle_action(struct flow_action_entry *mangle_action)
1416 {
1417 	if (mangle_action->id != FLOW_ACTION_MANGLE)
1418 		return;
1419 
1420 	switch (mangle_action->mangle.htype) {
1421 	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1422 	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1423 		mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val);
1424 		mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask);
1425 		return;
1426 
1427 	/* Both struct tcphdr and struct udphdr start with
1428 	 *	__be16 source;
1429 	 *	__be16 dest;
1430 	 * so we can use the same code for both.
1431 	 */
1432 	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
1433 	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
1434 		if (mangle_action->mangle.offset == offsetof(struct tcphdr, source)) {
1435 			mangle_action->mangle.val =
1436 				(__force u32)cpu_to_be32(mangle_action->mangle.val << 16);
1437 			/* The mask of mangle action is inverse mask,
1438 			 * so clear the dest tp port with 0xFFFF to
1439 			 * instead of rotate-left operation.
1440 			 */
1441 			mangle_action->mangle.mask =
1442 				(__force u32)cpu_to_be32(mangle_action->mangle.mask << 16 | 0xFFFF);
1443 		}
1444 		if (mangle_action->mangle.offset == offsetof(struct tcphdr, dest)) {
1445 			mangle_action->mangle.offset = 0;
1446 			mangle_action->mangle.val =
1447 				(__force u32)cpu_to_be32(mangle_action->mangle.val);
1448 			mangle_action->mangle.mask =
1449 				(__force u32)cpu_to_be32(mangle_action->mangle.mask);
1450 		}
1451 		return;
1452 
1453 	default:
1454 		return;
1455 	}
1456 }
1457 
1458 static int nfp_nft_ct_set_flow_flag(struct flow_action_entry *act,
1459 				    struct nfp_fl_ct_flow_entry *entry)
1460 {
1461 	switch (act->id) {
1462 	case FLOW_ACTION_CT:
1463 		if (act->ct.action == TCA_CT_ACT_NAT)
1464 			entry->flags |= NFP_FL_ACTION_DO_NAT;
1465 		break;
1466 
1467 	case FLOW_ACTION_MANGLE:
1468 		entry->flags |= NFP_FL_ACTION_DO_MANGLE;
1469 		break;
1470 
1471 	default:
1472 		break;
1473 	}
1474 
1475 	return 0;
1476 }
1477 
1478 static struct
1479 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
1480 					 struct net_device *netdev,
1481 					 struct flow_cls_offload *flow,
1482 					 bool is_nft, struct netlink_ext_ack *extack)
1483 {
1484 	struct nf_flow_match *nft_match = NULL;
1485 	struct nfp_fl_ct_flow_entry *entry;
1486 	struct nfp_fl_ct_map_entry *map;
1487 	struct flow_action_entry *act;
1488 	int err, i;
1489 
1490 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1491 	if (!entry)
1492 		return ERR_PTR(-ENOMEM);
1493 
1494 	entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
1495 	if (!entry->rule) {
1496 		err = -ENOMEM;
1497 		goto err_pre_ct_rule;
1498 	}
1499 
1500 	/* nft flows gets destroyed after callback return, so need
1501 	 * to do a full copy instead of just a reference.
1502 	 */
1503 	if (is_nft) {
1504 		nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
1505 		if (!nft_match) {
1506 			err = -ENOMEM;
1507 			goto err_pre_ct_act;
1508 		}
1509 		memcpy(&nft_match->dissector, flow->rule->match.dissector,
1510 		       sizeof(nft_match->dissector));
1511 		memcpy(&nft_match->mask, flow->rule->match.mask,
1512 		       sizeof(nft_match->mask));
1513 		memcpy(&nft_match->key, flow->rule->match.key,
1514 		       sizeof(nft_match->key));
1515 		entry->rule->match.dissector = &nft_match->dissector;
1516 		entry->rule->match.mask = &nft_match->mask;
1517 		entry->rule->match.key = &nft_match->key;
1518 
1519 		if (!netdev)
1520 			netdev = get_netdev_from_rule(entry->rule);
1521 	} else {
1522 		entry->rule->match.dissector = flow->rule->match.dissector;
1523 		entry->rule->match.mask = flow->rule->match.mask;
1524 		entry->rule->match.key = flow->rule->match.key;
1525 	}
1526 
1527 	entry->zt = zt;
1528 	entry->netdev = netdev;
1529 	entry->cookie = flow->cookie > 0 ? flow->cookie : (unsigned long)entry;
1530 	entry->chain_index = flow->common.chain_index;
1531 	entry->tun_offset = NFP_FL_CT_NO_TUN;
1532 
1533 	/* Copy over action data. Unfortunately we do not get a handle to the
1534 	 * original tcf_action data, and the flow objects gets destroyed, so we
1535 	 * cannot just save a pointer to this either, so need to copy over the
1536 	 * data unfortunately.
1537 	 */
1538 	entry->rule->action.num_entries = flow->rule->action.num_entries;
1539 	flow_action_for_each(i, act, &flow->rule->action) {
1540 		struct flow_action_entry *new_act;
1541 
1542 		new_act = &entry->rule->action.entries[i];
1543 		memcpy(new_act, act, sizeof(struct flow_action_entry));
1544 		/* nft entry mangle field is host byte order, need translate to
1545 		 * network byte order.
1546 		 */
1547 		if (is_nft)
1548 			nfp_nft_ct_translate_mangle_action(new_act);
1549 
1550 		nfp_nft_ct_set_flow_flag(new_act, entry);
1551 		/* Entunnel is a special case, need to allocate and copy
1552 		 * tunnel info.
1553 		 */
1554 		if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
1555 			struct ip_tunnel_info *tun = act->tunnel;
1556 			size_t tun_size = sizeof(*tun) + tun->options_len;
1557 
1558 			new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
1559 			if (!new_act->tunnel) {
1560 				err = -ENOMEM;
1561 				goto err_pre_ct_tun_cp;
1562 			}
1563 			entry->tun_offset = i;
1564 		}
1565 	}
1566 
1567 	INIT_LIST_HEAD(&entry->children);
1568 
1569 	if (flow->cookie == 0)
1570 		return entry;
1571 
1572 	/* Now add a ct map entry to flower-priv */
1573 	map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
1574 			    nfp_ct_map_params, sizeof(*map));
1575 	if (IS_ERR(map)) {
1576 		NL_SET_ERR_MSG_MOD(extack,
1577 				   "offload error: ct map entry creation failed");
1578 		err = -ENOMEM;
1579 		goto err_ct_flow_insert;
1580 	}
1581 	map->cookie = flow->cookie;
1582 	map->ct_entry = entry;
1583 	err = rhashtable_insert_fast(&zt->priv->ct_map_table,
1584 				     &map->hash_node,
1585 				     nfp_ct_map_params);
1586 	if (err) {
1587 		NL_SET_ERR_MSG_MOD(extack,
1588 				   "offload error: ct map entry table add failed");
1589 		goto err_map_insert;
1590 	}
1591 
1592 	return entry;
1593 
1594 err_map_insert:
1595 	kfree(map);
1596 err_ct_flow_insert:
1597 	if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1598 		kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
1599 err_pre_ct_tun_cp:
1600 	kfree(nft_match);
1601 err_pre_ct_act:
1602 	kfree(entry->rule);
1603 err_pre_ct_rule:
1604 	kfree(entry);
1605 	return ERR_PTR(err);
1606 }
1607 
1608 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
1609 {
1610 	struct nfp_fl_ct_zone_entry *zt;
1611 	int err;
1612 
1613 	zt = m_entry->zt;
1614 
1615 	/* Flow is in HW, need to delete */
1616 	if (m_entry->tc_flower_cookie) {
1617 		err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
1618 					    m_entry->netdev);
1619 		if (err)
1620 			return;
1621 	}
1622 
1623 	WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
1624 					    &m_entry->hash_node,
1625 					    nfp_nft_ct_merge_params));
1626 	zt->nft_merge_count--;
1627 	list_del(&m_entry->tc_merge_list);
1628 	list_del(&m_entry->nft_flow_list);
1629 
1630 	if (m_entry->next_pre_ct_entry) {
1631 		struct nfp_fl_ct_map_entry pre_ct_map_ent;
1632 
1633 		pre_ct_map_ent.ct_entry = m_entry->next_pre_ct_entry;
1634 		pre_ct_map_ent.cookie = 0;
1635 		nfp_fl_ct_del_flow(&pre_ct_map_ent);
1636 	}
1637 
1638 	kfree(m_entry);
1639 }
1640 
1641 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
1642 {
1643 	struct nfp_fl_nft_tc_merge *m_entry, *tmp;
1644 
1645 	/* These post entries are parts of two lists, one is a list of nft_entries
1646 	 * and the other is of from a list of tc_merge structures. Iterate
1647 	 * through the relevant list and cleanup the entries.
1648 	 */
1649 
1650 	if (is_nft_flow) {
1651 		/* Need to iterate through list of nft_flow entries */
1652 		struct nfp_fl_ct_flow_entry *ct_entry = entry;
1653 
1654 		list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1655 					 nft_flow_list) {
1656 			cleanup_nft_merge_entry(m_entry);
1657 		}
1658 	} else {
1659 		/* Need to iterate through list of tc_merged_flow entries */
1660 		struct nfp_fl_ct_tc_merge *ct_entry = entry;
1661 
1662 		list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1663 					 tc_merge_list) {
1664 			cleanup_nft_merge_entry(m_entry);
1665 		}
1666 	}
1667 }
1668 
1669 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
1670 {
1671 	struct nfp_fl_ct_zone_entry *zt;
1672 	int err;
1673 
1674 	zt = m_ent->zt;
1675 	err = rhashtable_remove_fast(&zt->tc_merge_tb,
1676 				     &m_ent->hash_node,
1677 				     nfp_tc_ct_merge_params);
1678 	if (err)
1679 		pr_warn("WARNING: could not remove merge_entry from hashtable\n");
1680 	zt->tc_merge_count--;
1681 	list_del(&m_ent->post_ct_list);
1682 	list_del(&m_ent->pre_ct_list);
1683 
1684 	if (!list_empty(&m_ent->children))
1685 		nfp_free_nft_merge_children(m_ent, false);
1686 	kfree(m_ent);
1687 }
1688 
1689 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
1690 {
1691 	struct nfp_fl_ct_tc_merge *m_ent, *tmp;
1692 
1693 	switch (entry->type) {
1694 	case CT_TYPE_PRE_CT:
1695 		list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
1696 			nfp_del_tc_merge_entry(m_ent);
1697 		}
1698 		break;
1699 	case CT_TYPE_POST_CT:
1700 		list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
1701 			nfp_del_tc_merge_entry(m_ent);
1702 		}
1703 		break;
1704 	default:
1705 		break;
1706 	}
1707 }
1708 
1709 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
1710 {
1711 	list_del(&entry->list_node);
1712 
1713 	if (!list_empty(&entry->children)) {
1714 		if (entry->type == CT_TYPE_NFT)
1715 			nfp_free_nft_merge_children(entry, true);
1716 		else
1717 			nfp_free_tc_merge_children(entry);
1718 	}
1719 
1720 	if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1721 		kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
1722 
1723 	if (entry->type == CT_TYPE_NFT) {
1724 		struct nf_flow_match *nft_match;
1725 
1726 		nft_match = container_of(entry->rule->match.dissector,
1727 					 struct nf_flow_match, dissector);
1728 		kfree(nft_match);
1729 	}
1730 
1731 	kfree(entry->rule);
1732 	kfree(entry);
1733 }
1734 
1735 static struct flow_action_entry *get_flow_act_ct(struct flow_rule *rule)
1736 {
1737 	struct flow_action_entry *act;
1738 	int i;
1739 
1740 	/* More than one ct action may be present in a flow rule,
1741 	 * Return the first one that is not a CT clear action
1742 	 */
1743 	flow_action_for_each(i, act, &rule->action) {
1744 		if (act->id == FLOW_ACTION_CT && act->ct.action != TCA_CT_ACT_CLEAR)
1745 			return act;
1746 	}
1747 
1748 	return NULL;
1749 }
1750 
1751 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
1752 					      enum flow_action_id act_id)
1753 {
1754 	struct flow_action_entry *act = NULL;
1755 	int i;
1756 
1757 	flow_action_for_each(i, act, &rule->action) {
1758 		if (act->id == act_id)
1759 			return act;
1760 	}
1761 	return NULL;
1762 }
1763 
1764 static void
1765 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
1766 			struct nfp_fl_ct_zone_entry *zt_src,
1767 			struct nfp_fl_ct_zone_entry *zt_dst)
1768 {
1769 	struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
1770 	struct list_head *ct_list;
1771 
1772 	if (ct_entry1->type == CT_TYPE_PRE_CT)
1773 		ct_list = &zt_src->post_ct_list;
1774 	else if (ct_entry1->type == CT_TYPE_POST_CT)
1775 		ct_list = &zt_src->pre_ct_list;
1776 	else
1777 		return;
1778 
1779 	list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
1780 				 list_node) {
1781 		nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
1782 	}
1783 }
1784 
1785 static void
1786 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
1787 			 struct nfp_fl_ct_zone_entry *zt)
1788 {
1789 	struct nfp_fl_ct_tc_merge *tc_merge_entry;
1790 	struct rhashtable_iter iter;
1791 
1792 	rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
1793 	rhashtable_walk_start(&iter);
1794 	while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
1795 		if (IS_ERR(tc_merge_entry))
1796 			continue;
1797 		rhashtable_walk_stop(&iter);
1798 		nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
1799 		rhashtable_walk_start(&iter);
1800 	}
1801 	rhashtable_walk_stop(&iter);
1802 	rhashtable_walk_exit(&iter);
1803 }
1804 
1805 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
1806 			    struct net_device *netdev,
1807 			    struct flow_cls_offload *flow,
1808 			    struct netlink_ext_ack *extack,
1809 			    struct nfp_fl_nft_tc_merge *m_entry)
1810 {
1811 	struct flow_action_entry *ct_act, *ct_goto;
1812 	struct nfp_fl_ct_flow_entry *ct_entry;
1813 	struct nfp_fl_ct_zone_entry *zt;
1814 	int err;
1815 
1816 	ct_act = get_flow_act_ct(flow->rule);
1817 	if (!ct_act) {
1818 		NL_SET_ERR_MSG_MOD(extack,
1819 				   "unsupported offload: Conntrack action empty in conntrack offload");
1820 		return -EOPNOTSUPP;
1821 	}
1822 
1823 	ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
1824 	if (!ct_goto) {
1825 		NL_SET_ERR_MSG_MOD(extack,
1826 				   "unsupported offload: Conntrack requires ACTION_GOTO");
1827 		return -EOPNOTSUPP;
1828 	}
1829 
1830 	zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
1831 	if (IS_ERR(zt)) {
1832 		NL_SET_ERR_MSG_MOD(extack,
1833 				   "offload error: Could not create zone table entry");
1834 		return PTR_ERR(zt);
1835 	}
1836 
1837 	if (!zt->nft) {
1838 		zt->nft = ct_act->ct.flow_table;
1839 		err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
1840 		if (err) {
1841 			NL_SET_ERR_MSG_MOD(extack,
1842 					   "offload error: Could not register nft_callback");
1843 			return err;
1844 		}
1845 	}
1846 
1847 	/* Add entry to pre_ct_list */
1848 	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1849 	if (IS_ERR(ct_entry))
1850 		return PTR_ERR(ct_entry);
1851 	ct_entry->type = CT_TYPE_PRE_CT;
1852 	ct_entry->chain_index = flow->common.chain_index;
1853 	ct_entry->goto_chain_index = ct_goto->chain_index;
1854 
1855 	if (m_entry) {
1856 		struct nfp_fl_ct_flow_entry *pre_ct_entry;
1857 		int i;
1858 
1859 		pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
1860 		for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++)
1861 			ct_entry->prev_m_entries[i] = pre_ct_entry->prev_m_entries[i];
1862 		ct_entry->prev_m_entries[i++] = m_entry;
1863 		ct_entry->num_prev_m_entries = i;
1864 
1865 		m_entry->next_pre_ct_entry = ct_entry;
1866 	}
1867 
1868 	list_add(&ct_entry->list_node, &zt->pre_ct_list);
1869 	zt->pre_ct_count++;
1870 
1871 	nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1872 
1873 	/* Need to check and merge with tables in the wc_zone as well */
1874 	if (priv->ct_zone_wc)
1875 		nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);
1876 
1877 	return 0;
1878 }
1879 
1880 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
1881 			     struct net_device *netdev,
1882 			     struct flow_cls_offload *flow,
1883 			     struct netlink_ext_ack *extack)
1884 {
1885 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
1886 	struct nfp_fl_ct_flow_entry *ct_entry;
1887 	struct flow_action_entry *ct_goto;
1888 	struct nfp_fl_ct_zone_entry *zt;
1889 	struct flow_action_entry *act;
1890 	bool wildcarded = false;
1891 	struct flow_match_ct ct;
1892 	int i;
1893 
1894 	flow_action_for_each(i, act, &rule->action) {
1895 		switch (act->id) {
1896 		case FLOW_ACTION_REDIRECT:
1897 		case FLOW_ACTION_REDIRECT_INGRESS:
1898 		case FLOW_ACTION_MIRRED:
1899 		case FLOW_ACTION_MIRRED_INGRESS:
1900 			if (act->dev->rtnl_link_ops &&
1901 			    !strcmp(act->dev->rtnl_link_ops->kind, "openvswitch")) {
1902 				NL_SET_ERR_MSG_MOD(extack,
1903 						   "unsupported offload: out port is openvswitch internal port");
1904 				return -EOPNOTSUPP;
1905 			}
1906 			break;
1907 		default:
1908 			break;
1909 		}
1910 	}
1911 
1912 	flow_rule_match_ct(rule, &ct);
1913 	if (!ct.mask->ct_zone) {
1914 		wildcarded = true;
1915 	} else if (ct.mask->ct_zone != U16_MAX) {
1916 		NL_SET_ERR_MSG_MOD(extack,
1917 				   "unsupported offload: partially wildcarded ct_zone is not supported");
1918 		return -EOPNOTSUPP;
1919 	}
1920 
1921 	zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
1922 	if (IS_ERR(zt)) {
1923 		NL_SET_ERR_MSG_MOD(extack,
1924 				   "offload error: Could not create zone table entry");
1925 		return PTR_ERR(zt);
1926 	}
1927 
1928 	/* Add entry to post_ct_list */
1929 	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1930 	if (IS_ERR(ct_entry))
1931 		return PTR_ERR(ct_entry);
1932 
1933 	ct_entry->type = CT_TYPE_POST_CT;
1934 	ct_entry->chain_index = flow->common.chain_index;
1935 	ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
1936 	ct_entry->goto_chain_index = ct_goto ? ct_goto->chain_index : 0;
1937 	list_add(&ct_entry->list_node, &zt->post_ct_list);
1938 	zt->post_ct_count++;
1939 
1940 	if (wildcarded) {
1941 		/* Iterate through all zone tables if not empty, look for merges with
1942 		 * pre_ct entries and merge them.
1943 		 */
1944 		struct rhashtable_iter iter;
1945 		struct nfp_fl_ct_zone_entry *zone_table;
1946 
1947 		rhashtable_walk_enter(&priv->ct_zone_table, &iter);
1948 		rhashtable_walk_start(&iter);
1949 		while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
1950 			if (IS_ERR(zone_table))
1951 				continue;
1952 			rhashtable_walk_stop(&iter);
1953 			nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
1954 			rhashtable_walk_start(&iter);
1955 		}
1956 		rhashtable_walk_stop(&iter);
1957 		rhashtable_walk_exit(&iter);
1958 	} else {
1959 		nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1960 	}
1961 
1962 	return 0;
1963 }
1964 
1965 int nfp_fl_create_new_pre_ct(struct nfp_fl_nft_tc_merge *m_entry)
1966 {
1967 	struct nfp_fl_ct_flow_entry *pre_ct_entry, *post_ct_entry;
1968 	struct flow_cls_offload new_pre_ct_flow;
1969 	int err;
1970 
1971 	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
1972 	if (pre_ct_entry->num_prev_m_entries >= NFP_MAX_RECIRC_CT_ZONES - 1)
1973 		return -1;
1974 
1975 	post_ct_entry = m_entry->tc_m_parent->post_ct_parent;
1976 	memset(&new_pre_ct_flow, 0, sizeof(struct flow_cls_offload));
1977 	new_pre_ct_flow.rule = post_ct_entry->rule;
1978 	new_pre_ct_flow.common.chain_index = post_ct_entry->chain_index;
1979 
1980 	err = nfp_fl_ct_handle_pre_ct(pre_ct_entry->zt->priv,
1981 				      pre_ct_entry->netdev,
1982 				      &new_pre_ct_flow, NULL,
1983 				      m_entry);
1984 	return err;
1985 }
1986 
1987 static void
1988 nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge,
1989 		    enum ct_entry_type type, u64 *m_pkts,
1990 		    u64 *m_bytes, u64 *m_used)
1991 {
1992 	struct nfp_flower_priv *priv = nft_merge->zt->priv;
1993 	struct nfp_fl_payload *nfp_flow;
1994 	u32 ctx_id;
1995 
1996 	nfp_flow = nft_merge->flow_pay;
1997 	if (!nfp_flow)
1998 		return;
1999 
2000 	ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
2001 	*m_pkts += priv->stats[ctx_id].pkts;
2002 	*m_bytes += priv->stats[ctx_id].bytes;
2003 	*m_used = max_t(u64, *m_used, priv->stats[ctx_id].used);
2004 
2005 	/* If request is for a sub_flow which is part of a tunnel merged
2006 	 * flow then update stats from tunnel merged flows first.
2007 	 */
2008 	if (!list_empty(&nfp_flow->linked_flows))
2009 		nfp_flower_update_merge_stats(priv->app, nfp_flow);
2010 
2011 	if (type != CT_TYPE_NFT) {
2012 		/* Update nft cached stats */
2013 		flow_stats_update(&nft_merge->nft_parent->stats,
2014 				  priv->stats[ctx_id].bytes,
2015 				  priv->stats[ctx_id].pkts,
2016 				  0, priv->stats[ctx_id].used,
2017 				  FLOW_ACTION_HW_STATS_DELAYED);
2018 	} else {
2019 		/* Update pre_ct cached stats */
2020 		flow_stats_update(&nft_merge->tc_m_parent->pre_ct_parent->stats,
2021 				  priv->stats[ctx_id].bytes,
2022 				  priv->stats[ctx_id].pkts,
2023 				  0, priv->stats[ctx_id].used,
2024 				  FLOW_ACTION_HW_STATS_DELAYED);
2025 		/* Update post_ct cached stats */
2026 		flow_stats_update(&nft_merge->tc_m_parent->post_ct_parent->stats,
2027 				  priv->stats[ctx_id].bytes,
2028 				  priv->stats[ctx_id].pkts,
2029 				  0, priv->stats[ctx_id].used,
2030 				  FLOW_ACTION_HW_STATS_DELAYED);
2031 	}
2032 
2033 	/* Update previous pre_ct/post_ct/nft flow stats */
2034 	if (nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries > 0) {
2035 		struct nfp_fl_nft_tc_merge *tmp_nft_merge;
2036 		int i;
2037 
2038 		for (i = 0; i < nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries; i++) {
2039 			tmp_nft_merge = nft_merge->tc_m_parent->pre_ct_parent->prev_m_entries[i];
2040 			flow_stats_update(&tmp_nft_merge->tc_m_parent->pre_ct_parent->stats,
2041 					  priv->stats[ctx_id].bytes,
2042 					  priv->stats[ctx_id].pkts,
2043 					  0, priv->stats[ctx_id].used,
2044 					  FLOW_ACTION_HW_STATS_DELAYED);
2045 			flow_stats_update(&tmp_nft_merge->tc_m_parent->post_ct_parent->stats,
2046 					  priv->stats[ctx_id].bytes,
2047 					  priv->stats[ctx_id].pkts,
2048 					  0, priv->stats[ctx_id].used,
2049 					  FLOW_ACTION_HW_STATS_DELAYED);
2050 			flow_stats_update(&tmp_nft_merge->nft_parent->stats,
2051 					  priv->stats[ctx_id].bytes,
2052 					  priv->stats[ctx_id].pkts,
2053 					  0, priv->stats[ctx_id].used,
2054 					  FLOW_ACTION_HW_STATS_DELAYED);
2055 		}
2056 	}
2057 
2058 	/* Reset stats from the nfp */
2059 	priv->stats[ctx_id].pkts = 0;
2060 	priv->stats[ctx_id].bytes = 0;
2061 }
2062 
2063 int nfp_fl_ct_stats(struct flow_cls_offload *flow,
2064 		    struct nfp_fl_ct_map_entry *ct_map_ent)
2065 {
2066 	struct nfp_fl_ct_flow_entry *ct_entry = ct_map_ent->ct_entry;
2067 	struct nfp_fl_nft_tc_merge *nft_merge, *nft_m_tmp;
2068 	struct nfp_fl_ct_tc_merge *tc_merge, *tc_m_tmp;
2069 
2070 	u64 pkts = 0, bytes = 0, used = 0;
2071 	u64 m_pkts, m_bytes, m_used;
2072 
2073 	spin_lock_bh(&ct_entry->zt->priv->stats_lock);
2074 
2075 	if (ct_entry->type == CT_TYPE_PRE_CT) {
2076 		/* Iterate tc_merge entries associated with this flow */
2077 		list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
2078 					 pre_ct_list) {
2079 			m_pkts = 0;
2080 			m_bytes = 0;
2081 			m_used = 0;
2082 			/* Iterate nft_merge entries associated with this tc_merge flow */
2083 			list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
2084 						 tc_merge_list) {
2085 				nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_PRE_CT,
2086 						    &m_pkts, &m_bytes, &m_used);
2087 			}
2088 			pkts += m_pkts;
2089 			bytes += m_bytes;
2090 			used = max_t(u64, used, m_used);
2091 			/* Update post_ct partner */
2092 			flow_stats_update(&tc_merge->post_ct_parent->stats,
2093 					  m_bytes, m_pkts, 0, m_used,
2094 					  FLOW_ACTION_HW_STATS_DELAYED);
2095 		}
2096 	} else if (ct_entry->type == CT_TYPE_POST_CT) {
2097 		/* Iterate tc_merge entries associated with this flow */
2098 		list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
2099 					 post_ct_list) {
2100 			m_pkts = 0;
2101 			m_bytes = 0;
2102 			m_used = 0;
2103 			/* Iterate nft_merge entries associated with this tc_merge flow */
2104 			list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
2105 						 tc_merge_list) {
2106 				nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_POST_CT,
2107 						    &m_pkts, &m_bytes, &m_used);
2108 			}
2109 			pkts += m_pkts;
2110 			bytes += m_bytes;
2111 			used = max_t(u64, used, m_used);
2112 			/* Update pre_ct partner */
2113 			flow_stats_update(&tc_merge->pre_ct_parent->stats,
2114 					  m_bytes, m_pkts, 0, m_used,
2115 					  FLOW_ACTION_HW_STATS_DELAYED);
2116 		}
2117 	} else  {
2118 		/* Iterate nft_merge entries associated with this nft flow */
2119 		list_for_each_entry_safe(nft_merge, nft_m_tmp, &ct_entry->children,
2120 					 nft_flow_list) {
2121 			nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_NFT,
2122 					    &pkts, &bytes, &used);
2123 		}
2124 	}
2125 
2126 	/* Add stats from this request to stats potentially cached by
2127 	 * previous requests.
2128 	 */
2129 	flow_stats_update(&ct_entry->stats, bytes, pkts, 0, used,
2130 			  FLOW_ACTION_HW_STATS_DELAYED);
2131 	/* Finally update the flow stats from the original stats request */
2132 	flow_stats_update(&flow->stats, ct_entry->stats.bytes,
2133 			  ct_entry->stats.pkts, 0,
2134 			  ct_entry->stats.lastused,
2135 			  FLOW_ACTION_HW_STATS_DELAYED);
2136 	/* Stats has been synced to original flow, can now clear
2137 	 * the cache.
2138 	 */
2139 	ct_entry->stats.pkts = 0;
2140 	ct_entry->stats.bytes = 0;
2141 	spin_unlock_bh(&ct_entry->zt->priv->stats_lock);
2142 
2143 	return 0;
2144 }
2145 
2146 static bool
2147 nfp_fl_ct_offload_nft_supported(struct flow_cls_offload *flow)
2148 {
2149 	struct flow_rule *flow_rule = flow->rule;
2150 	struct flow_action *flow_action =
2151 		&flow_rule->action;
2152 	struct flow_action_entry *act;
2153 	int i;
2154 
2155 	flow_action_for_each(i, act, flow_action) {
2156 		if (act->id == FLOW_ACTION_CT_METADATA) {
2157 			enum ip_conntrack_info ctinfo =
2158 				act->ct_metadata.cookie & NFCT_INFOMASK;
2159 
2160 			return ctinfo != IP_CT_NEW;
2161 		}
2162 	}
2163 
2164 	return false;
2165 }
2166 
2167 static int
2168 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
2169 {
2170 	struct nfp_fl_ct_map_entry *ct_map_ent;
2171 	struct nfp_fl_ct_flow_entry *ct_entry;
2172 	struct netlink_ext_ack *extack = NULL;
2173 
2174 	extack = flow->common.extack;
2175 	switch (flow->command) {
2176 	case FLOW_CLS_REPLACE:
2177 		if (!nfp_fl_ct_offload_nft_supported(flow))
2178 			return -EOPNOTSUPP;
2179 
2180 		/* Netfilter can request offload multiple times for the same
2181 		 * flow - protect against adding duplicates.
2182 		 */
2183 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
2184 						    nfp_ct_map_params);
2185 		if (!ct_map_ent) {
2186 			ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
2187 			if (IS_ERR(ct_entry))
2188 				return PTR_ERR(ct_entry);
2189 			ct_entry->type = CT_TYPE_NFT;
2190 			list_add(&ct_entry->list_node, &zt->nft_flows_list);
2191 			zt->nft_flows_count++;
2192 			nfp_ct_merge_nft_with_tc(ct_entry, zt);
2193 		}
2194 		return 0;
2195 	case FLOW_CLS_DESTROY:
2196 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
2197 						    nfp_ct_map_params);
2198 		return nfp_fl_ct_del_flow(ct_map_ent);
2199 	case FLOW_CLS_STATS:
2200 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
2201 						    nfp_ct_map_params);
2202 		if (ct_map_ent)
2203 			return nfp_fl_ct_stats(flow, ct_map_ent);
2204 		break;
2205 	default:
2206 		break;
2207 	}
2208 	return -EINVAL;
2209 }
2210 
2211 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
2212 {
2213 	struct flow_cls_offload *flow = type_data;
2214 	struct nfp_fl_ct_zone_entry *zt = cb_priv;
2215 	int err = -EOPNOTSUPP;
2216 
2217 	switch (type) {
2218 	case TC_SETUP_CLSFLOWER:
2219 		while (!mutex_trylock(&zt->priv->nfp_fl_lock)) {
2220 			if (!zt->nft) /* avoid deadlock */
2221 				return err;
2222 			msleep(20);
2223 		}
2224 		err = nfp_fl_ct_offload_nft_flow(zt, flow);
2225 		mutex_unlock(&zt->priv->nfp_fl_lock);
2226 		break;
2227 	default:
2228 		return -EOPNOTSUPP;
2229 	}
2230 	return err;
2231 }
2232 
2233 static void
2234 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
2235 {
2236 	struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
2237 	struct nfp_fl_ct_map_entry *ct_map_ent;
2238 
2239 	list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
2240 				 list_node) {
2241 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
2242 						    &nft_entry->cookie,
2243 						    nfp_ct_map_params);
2244 		nfp_fl_ct_del_flow(ct_map_ent);
2245 	}
2246 }
2247 
2248 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
2249 {
2250 	struct nfp_fl_ct_flow_entry *ct_entry;
2251 	struct nfp_fl_ct_zone_entry *zt;
2252 	struct rhashtable *m_table;
2253 	struct nf_flowtable *nft;
2254 
2255 	if (!ct_map_ent)
2256 		return -ENOENT;
2257 
2258 	zt = ct_map_ent->ct_entry->zt;
2259 	ct_entry = ct_map_ent->ct_entry;
2260 	m_table = &zt->priv->ct_map_table;
2261 
2262 	switch (ct_entry->type) {
2263 	case CT_TYPE_PRE_CT:
2264 		zt->pre_ct_count--;
2265 		if (ct_map_ent->cookie > 0)
2266 			rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
2267 					       nfp_ct_map_params);
2268 		nfp_fl_ct_clean_flow_entry(ct_entry);
2269 		if (ct_map_ent->cookie > 0)
2270 			kfree(ct_map_ent);
2271 
2272 		if (!zt->pre_ct_count && zt->nft) {
2273 			nft = zt->nft;
2274 			zt->nft = NULL; /* avoid deadlock */
2275 			nf_flow_table_offload_del_cb(nft,
2276 						     nfp_fl_ct_handle_nft_flow,
2277 						     zt);
2278 			nfp_fl_ct_clean_nft_entries(zt);
2279 		}
2280 		break;
2281 	case CT_TYPE_POST_CT:
2282 		zt->post_ct_count--;
2283 		rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
2284 				       nfp_ct_map_params);
2285 		nfp_fl_ct_clean_flow_entry(ct_entry);
2286 		kfree(ct_map_ent);
2287 		break;
2288 	case CT_TYPE_NFT:
2289 		zt->nft_flows_count--;
2290 		rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
2291 				       nfp_ct_map_params);
2292 		nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);
2293 		kfree(ct_map_ent);
2294 		break;
2295 	default:
2296 		break;
2297 	}
2298 
2299 	return 0;
2300 }
2301