1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2021 Corigine, Inc. */
3 
4 #include "conntrack.h"
5 
6 const struct rhashtable_params nfp_tc_ct_merge_params = {
7 	.head_offset		= offsetof(struct nfp_fl_ct_tc_merge,
8 					   hash_node),
9 	.key_len		= sizeof(unsigned long) * 2,
10 	.key_offset		= offsetof(struct nfp_fl_ct_tc_merge, cookie),
11 	.automatic_shrinking	= true,
12 };
13 
14 const struct rhashtable_params nfp_nft_ct_merge_params = {
15 	.head_offset		= offsetof(struct nfp_fl_nft_tc_merge,
16 					   hash_node),
17 	.key_len		= sizeof(unsigned long) * 3,
18 	.key_offset		= offsetof(struct nfp_fl_nft_tc_merge, cookie),
19 	.automatic_shrinking	= true,
20 };
21 
22 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
23 					      enum flow_action_id act_id);
24 
25 /**
26  * get_hashentry() - Wrapper around hashtable lookup.
27  * @ht:		hashtable where entry could be found
28  * @key:	key to lookup
29  * @params:	hashtable params
30  * @size:	size of entry to allocate if not in table
31  *
32  * Returns an entry from a hashtable. If entry does not exist
33  * yet allocate the memory for it and return the new entry.
34  */
35 static void *get_hashentry(struct rhashtable *ht, void *key,
36 			   const struct rhashtable_params params, size_t size)
37 {
38 	void *result;
39 
40 	result = rhashtable_lookup_fast(ht, key, params);
41 
42 	if (result)
43 		return result;
44 
45 	result = kzalloc(size, GFP_KERNEL);
46 	if (!result)
47 		return ERR_PTR(-ENOMEM);
48 
49 	return result;
50 }
51 
52 bool is_pre_ct_flow(struct flow_cls_offload *flow)
53 {
54 	struct flow_action_entry *act;
55 	int i;
56 
57 	flow_action_for_each(i, act, &flow->rule->action) {
58 		if (act->id == FLOW_ACTION_CT && !act->ct.action)
59 			return true;
60 	}
61 	return false;
62 }
63 
64 bool is_post_ct_flow(struct flow_cls_offload *flow)
65 {
66 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
67 	struct flow_dissector *dissector = rule->match.dissector;
68 	struct flow_match_ct ct;
69 
70 	if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
71 		flow_rule_match_ct(rule, &ct);
72 		if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
73 			return true;
74 	}
75 	return false;
76 }
77 
78 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
79 			      struct nfp_fl_ct_flow_entry *entry2)
80 {
81 	unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys &
82 				 entry2->rule->match.dissector->used_keys;
83 	bool out;
84 
85 	/* check the overlapped fields one by one, the unmasked part
86 	 * should not conflict with each other.
87 	 */
88 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) {
89 		struct flow_match_control match1, match2;
90 
91 		flow_rule_match_control(entry1->rule, &match1);
92 		flow_rule_match_control(entry2->rule, &match2);
93 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
94 		if (out)
95 			goto check_failed;
96 	}
97 
98 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) {
99 		struct flow_match_basic match1, match2;
100 
101 		flow_rule_match_basic(entry1->rule, &match1);
102 		flow_rule_match_basic(entry2->rule, &match2);
103 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
104 		if (out)
105 			goto check_failed;
106 	}
107 
108 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
109 		struct flow_match_ipv4_addrs match1, match2;
110 
111 		flow_rule_match_ipv4_addrs(entry1->rule, &match1);
112 		flow_rule_match_ipv4_addrs(entry2->rule, &match2);
113 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
114 		if (out)
115 			goto check_failed;
116 	}
117 
118 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
119 		struct flow_match_ipv6_addrs match1, match2;
120 
121 		flow_rule_match_ipv6_addrs(entry1->rule, &match1);
122 		flow_rule_match_ipv6_addrs(entry2->rule, &match2);
123 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
124 		if (out)
125 			goto check_failed;
126 	}
127 
128 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) {
129 		struct flow_match_ports match1, match2;
130 
131 		flow_rule_match_ports(entry1->rule, &match1);
132 		flow_rule_match_ports(entry2->rule, &match2);
133 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
134 		if (out)
135 			goto check_failed;
136 	}
137 
138 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
139 		struct flow_match_eth_addrs match1, match2;
140 
141 		flow_rule_match_eth_addrs(entry1->rule, &match1);
142 		flow_rule_match_eth_addrs(entry2->rule, &match2);
143 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
144 		if (out)
145 			goto check_failed;
146 	}
147 
148 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) {
149 		struct flow_match_vlan match1, match2;
150 
151 		flow_rule_match_vlan(entry1->rule, &match1);
152 		flow_rule_match_vlan(entry2->rule, &match2);
153 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
154 		if (out)
155 			goto check_failed;
156 	}
157 
158 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) {
159 		struct flow_match_mpls match1, match2;
160 
161 		flow_rule_match_mpls(entry1->rule, &match1);
162 		flow_rule_match_mpls(entry2->rule, &match2);
163 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
164 		if (out)
165 			goto check_failed;
166 	}
167 
168 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) {
169 		struct flow_match_tcp match1, match2;
170 
171 		flow_rule_match_tcp(entry1->rule, &match1);
172 		flow_rule_match_tcp(entry2->rule, &match2);
173 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
174 		if (out)
175 			goto check_failed;
176 	}
177 
178 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) {
179 		struct flow_match_ip match1, match2;
180 
181 		flow_rule_match_ip(entry1->rule, &match1);
182 		flow_rule_match_ip(entry2->rule, &match2);
183 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
184 		if (out)
185 			goto check_failed;
186 	}
187 
188 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
189 		struct flow_match_enc_keyid match1, match2;
190 
191 		flow_rule_match_enc_keyid(entry1->rule, &match1);
192 		flow_rule_match_enc_keyid(entry2->rule, &match2);
193 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
194 		if (out)
195 			goto check_failed;
196 	}
197 
198 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
199 		struct flow_match_ipv4_addrs match1, match2;
200 
201 		flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
202 		flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
203 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
204 		if (out)
205 			goto check_failed;
206 	}
207 
208 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
209 		struct flow_match_ipv6_addrs match1, match2;
210 
211 		flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
212 		flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
213 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
214 		if (out)
215 			goto check_failed;
216 	}
217 
218 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
219 		struct flow_match_control match1, match2;
220 
221 		flow_rule_match_enc_control(entry1->rule, &match1);
222 		flow_rule_match_enc_control(entry2->rule, &match2);
223 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
224 		if (out)
225 			goto check_failed;
226 	}
227 
228 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) {
229 		struct flow_match_ip match1, match2;
230 
231 		flow_rule_match_enc_ip(entry1->rule, &match1);
232 		flow_rule_match_enc_ip(entry2->rule, &match2);
233 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
234 		if (out)
235 			goto check_failed;
236 	}
237 
238 	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
239 		struct flow_match_enc_opts match1, match2;
240 
241 		flow_rule_match_enc_opts(entry1->rule, &match1);
242 		flow_rule_match_enc_opts(entry2->rule, &match2);
243 		COMPARE_UNMASKED_FIELDS(match1, match2, &out);
244 		if (out)
245 			goto check_failed;
246 	}
247 
248 	return 0;
249 
250 check_failed:
251 	return -EINVAL;
252 }
253 
254 static int nfp_ct_check_mangle_merge(struct flow_action_entry *a_in,
255 				     struct flow_rule *rule)
256 {
257 	enum flow_action_mangle_base htype = a_in->mangle.htype;
258 	u32 offset = a_in->mangle.offset;
259 
260 	switch (htype) {
261 	case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
262 		if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS))
263 			return -EOPNOTSUPP;
264 		break;
265 	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
266 		if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
267 			struct flow_match_ip match;
268 
269 			flow_rule_match_ip(rule, &match);
270 			if (offset == offsetof(struct iphdr, ttl) &&
271 			    match.mask->ttl)
272 				return -EOPNOTSUPP;
273 			if (offset == round_down(offsetof(struct iphdr, tos), 4) &&
274 			    match.mask->tos)
275 				return -EOPNOTSUPP;
276 		}
277 		if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
278 			struct flow_match_ipv4_addrs match;
279 
280 			flow_rule_match_ipv4_addrs(rule, &match);
281 			if (offset == offsetof(struct iphdr, saddr) &&
282 			    match.mask->src)
283 				return -EOPNOTSUPP;
284 			if (offset == offsetof(struct iphdr, daddr) &&
285 			    match.mask->dst)
286 				return -EOPNOTSUPP;
287 		}
288 		break;
289 	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
290 		if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
291 			struct flow_match_ip match;
292 
293 			flow_rule_match_ip(rule, &match);
294 			if (offset == round_down(offsetof(struct ipv6hdr, hop_limit), 4) &&
295 			    match.mask->ttl)
296 				return -EOPNOTSUPP;
297 			/* for ipv6, tos and flow_lbl are in the same word */
298 			if (offset == round_down(offsetof(struct ipv6hdr, flow_lbl), 4) &&
299 			    match.mask->tos)
300 				return -EOPNOTSUPP;
301 		}
302 		if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
303 			struct flow_match_ipv6_addrs match;
304 
305 			flow_rule_match_ipv6_addrs(rule, &match);
306 			if (offset >= offsetof(struct ipv6hdr, saddr) &&
307 			    offset < offsetof(struct ipv6hdr, daddr) &&
308 			    memchr_inv(&match.mask->src, 0, sizeof(match.mask->src)))
309 				return -EOPNOTSUPP;
310 			if (offset >= offsetof(struct ipv6hdr, daddr) &&
311 			    offset < sizeof(struct ipv6hdr) &&
312 			    memchr_inv(&match.mask->dst, 0, sizeof(match.mask->dst)))
313 				return -EOPNOTSUPP;
314 		}
315 		break;
316 	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
317 	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
318 		/* currently only can modify ports */
319 		if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS))
320 			return -EOPNOTSUPP;
321 		break;
322 	default:
323 		break;
324 	}
325 	return 0;
326 }
327 
328 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
329 				  struct nfp_fl_ct_flow_entry *post_ct_entry,
330 				  struct nfp_fl_ct_flow_entry *nft_entry)
331 {
332 	struct flow_action_entry *act;
333 	int err, i;
334 
335 	/* Check for pre_ct->action conflicts */
336 	flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
337 		switch (act->id) {
338 		case FLOW_ACTION_MANGLE:
339 			err = nfp_ct_check_mangle_merge(act, nft_entry->rule);
340 			if (err)
341 				return err;
342 			err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
343 			if (err)
344 				return err;
345 			break;
346 		case FLOW_ACTION_VLAN_PUSH:
347 		case FLOW_ACTION_VLAN_POP:
348 		case FLOW_ACTION_VLAN_MANGLE:
349 		case FLOW_ACTION_MPLS_PUSH:
350 		case FLOW_ACTION_MPLS_POP:
351 		case FLOW_ACTION_MPLS_MANGLE:
352 			return -EOPNOTSUPP;
353 		default:
354 			break;
355 		}
356 	}
357 
358 	/* Check for nft->action conflicts */
359 	flow_action_for_each(i, act, &nft_entry->rule->action) {
360 		switch (act->id) {
361 		case FLOW_ACTION_MANGLE:
362 			err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
363 			if (err)
364 				return err;
365 			break;
366 		case FLOW_ACTION_VLAN_PUSH:
367 		case FLOW_ACTION_VLAN_POP:
368 		case FLOW_ACTION_VLAN_MANGLE:
369 		case FLOW_ACTION_MPLS_PUSH:
370 		case FLOW_ACTION_MPLS_POP:
371 		case FLOW_ACTION_MPLS_MANGLE:
372 			return -EOPNOTSUPP;
373 		default:
374 			break;
375 		}
376 	}
377 	return 0;
378 }
379 
380 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
381 			     struct nfp_fl_ct_flow_entry *nft_entry)
382 {
383 	struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
384 	struct flow_action_entry *ct_met;
385 	struct flow_match_ct ct;
386 	int i;
387 
388 	ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
389 	if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) {
390 		u32 *act_lbl;
391 
392 		act_lbl = ct_met->ct_metadata.labels;
393 		flow_rule_match_ct(post_ct_entry->rule, &ct);
394 		for (i = 0; i < 4; i++) {
395 			if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
396 			    (act_lbl[i] & ct.mask->ct_labels[i]))
397 				return -EINVAL;
398 		}
399 
400 		if ((ct.key->ct_mark & ct.mask->ct_mark) ^
401 		    (ct_met->ct_metadata.mark & ct.mask->ct_mark))
402 			return -EINVAL;
403 
404 		return 0;
405 	}
406 
407 	return -EINVAL;
408 }
409 
410 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
411 {
412 	return 0;
413 }
414 
415 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
416 				 struct net_device *netdev)
417 {
418 	return 0;
419 }
420 
421 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
422 			       struct nfp_fl_ct_flow_entry *nft_entry,
423 			       struct nfp_fl_ct_tc_merge *tc_m_entry)
424 {
425 	struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
426 	struct nfp_fl_nft_tc_merge *nft_m_entry;
427 	unsigned long new_cookie[3];
428 	int err;
429 
430 	pre_ct_entry = tc_m_entry->pre_ct_parent;
431 	post_ct_entry = tc_m_entry->post_ct_parent;
432 
433 	err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
434 	if (err)
435 		return err;
436 
437 	/* Check that the two tc flows are also compatible with
438 	 * the nft entry. No need to check the pre_ct and post_ct
439 	 * entries as that was already done during pre_merge.
440 	 * The nft entry does not have a netdev or chain populated, so
441 	 * skip this check.
442 	 */
443 	err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
444 	if (err)
445 		return err;
446 	err = nfp_ct_merge_check(post_ct_entry, nft_entry);
447 	if (err)
448 		return err;
449 	err = nfp_ct_check_meta(post_ct_entry, nft_entry);
450 	if (err)
451 		return err;
452 
453 	/* Combine tc_merge and nft cookies for this cookie. */
454 	new_cookie[0] = tc_m_entry->cookie[0];
455 	new_cookie[1] = tc_m_entry->cookie[1];
456 	new_cookie[2] = nft_entry->cookie;
457 	nft_m_entry = get_hashentry(&zt->nft_merge_tb,
458 				    &new_cookie,
459 				    nfp_nft_ct_merge_params,
460 				    sizeof(*nft_m_entry));
461 
462 	if (IS_ERR(nft_m_entry))
463 		return PTR_ERR(nft_m_entry);
464 
465 	/* nft_m_entry already present, not merging again */
466 	if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
467 		return 0;
468 
469 	memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
470 	nft_m_entry->zt = zt;
471 	nft_m_entry->tc_m_parent = tc_m_entry;
472 	nft_m_entry->nft_parent = nft_entry;
473 	nft_m_entry->tc_flower_cookie = 0;
474 	/* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created
475 	 * it only combined them if the netdevs were the same, so can use any of them.
476 	 */
477 	nft_m_entry->netdev = pre_ct_entry->netdev;
478 
479 	/* Add this entry to the tc_m_list and nft_flow lists */
480 	list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
481 	list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
482 
483 	/* Generate offload structure and send to nfp */
484 	err = nfp_fl_ct_add_offload(nft_m_entry);
485 	if (err)
486 		goto err_nft_ct_offload;
487 
488 	err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
489 				     nfp_nft_ct_merge_params);
490 	if (err)
491 		goto err_nft_ct_merge_insert;
492 
493 	zt->nft_merge_count++;
494 
495 	return err;
496 
497 err_nft_ct_merge_insert:
498 	nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
499 			      nft_m_entry->netdev);
500 err_nft_ct_offload:
501 	list_del(&nft_m_entry->tc_merge_list);
502 	list_del(&nft_m_entry->nft_flow_list);
503 	kfree(nft_m_entry);
504 	return err;
505 }
506 
507 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
508 			      struct nfp_fl_ct_flow_entry *ct_entry1,
509 			      struct nfp_fl_ct_flow_entry *ct_entry2)
510 {
511 	struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
512 	struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
513 	struct nfp_fl_ct_tc_merge *m_entry;
514 	unsigned long new_cookie[2];
515 	int err;
516 
517 	if (ct_entry1->type == CT_TYPE_PRE_CT) {
518 		pre_ct_entry = ct_entry1;
519 		post_ct_entry = ct_entry2;
520 	} else {
521 		post_ct_entry = ct_entry1;
522 		pre_ct_entry = ct_entry2;
523 	}
524 
525 	if (post_ct_entry->netdev != pre_ct_entry->netdev)
526 		return -EINVAL;
527 	/* Checks that the chain_index of the filter matches the
528 	 * chain_index of the GOTO action.
529 	 */
530 	if (post_ct_entry->chain_index != pre_ct_entry->chain_index)
531 		return -EINVAL;
532 
533 	err = nfp_ct_merge_check(post_ct_entry, pre_ct_entry);
534 	if (err)
535 		return err;
536 
537 	new_cookie[0] = pre_ct_entry->cookie;
538 	new_cookie[1] = post_ct_entry->cookie;
539 	m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
540 				nfp_tc_ct_merge_params, sizeof(*m_entry));
541 	if (IS_ERR(m_entry))
542 		return PTR_ERR(m_entry);
543 
544 	/* m_entry already present, not merging again */
545 	if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
546 		return 0;
547 
548 	memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
549 	m_entry->zt = zt;
550 	m_entry->post_ct_parent = post_ct_entry;
551 	m_entry->pre_ct_parent = pre_ct_entry;
552 
553 	/* Add this entry to the pre_ct and post_ct lists */
554 	list_add(&m_entry->post_ct_list, &post_ct_entry->children);
555 	list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
556 	INIT_LIST_HEAD(&m_entry->children);
557 
558 	err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
559 				     nfp_tc_ct_merge_params);
560 	if (err)
561 		goto err_ct_tc_merge_insert;
562 	zt->tc_merge_count++;
563 
564 	/* Merge with existing nft flows */
565 	list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
566 				 list_node) {
567 		nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
568 	}
569 
570 	return 0;
571 
572 err_ct_tc_merge_insert:
573 	list_del(&m_entry->post_ct_list);
574 	list_del(&m_entry->pre_ct_list);
575 	kfree(m_entry);
576 	return err;
577 }
578 
579 static struct
580 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
581 					 u16 zone, bool wildcarded)
582 {
583 	struct nfp_fl_ct_zone_entry *zt;
584 	int err;
585 
586 	if (wildcarded && priv->ct_zone_wc)
587 		return priv->ct_zone_wc;
588 
589 	if (!wildcarded) {
590 		zt = get_hashentry(&priv->ct_zone_table, &zone,
591 				   nfp_zone_table_params, sizeof(*zt));
592 
593 		/* If priv is set this is an existing entry, just return it */
594 		if (IS_ERR(zt) || zt->priv)
595 			return zt;
596 	} else {
597 		zt = kzalloc(sizeof(*zt), GFP_KERNEL);
598 		if (!zt)
599 			return ERR_PTR(-ENOMEM);
600 	}
601 
602 	zt->zone = zone;
603 	zt->priv = priv;
604 	zt->nft = NULL;
605 
606 	/* init the various hash tables and lists*/
607 	INIT_LIST_HEAD(&zt->pre_ct_list);
608 	INIT_LIST_HEAD(&zt->post_ct_list);
609 	INIT_LIST_HEAD(&zt->nft_flows_list);
610 
611 	err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
612 	if (err)
613 		goto err_tc_merge_tb_init;
614 
615 	err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
616 	if (err)
617 		goto err_nft_merge_tb_init;
618 
619 	if (wildcarded) {
620 		priv->ct_zone_wc = zt;
621 	} else {
622 		err = rhashtable_insert_fast(&priv->ct_zone_table,
623 					     &zt->hash_node,
624 					     nfp_zone_table_params);
625 		if (err)
626 			goto err_zone_insert;
627 	}
628 
629 	return zt;
630 
631 err_zone_insert:
632 	rhashtable_destroy(&zt->nft_merge_tb);
633 err_nft_merge_tb_init:
634 	rhashtable_destroy(&zt->tc_merge_tb);
635 err_tc_merge_tb_init:
636 	kfree(zt);
637 	return ERR_PTR(err);
638 }
639 
640 static struct
641 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
642 					 struct net_device *netdev,
643 					 struct flow_cls_offload *flow,
644 					 bool is_nft, struct netlink_ext_ack *extack)
645 {
646 	struct nf_flow_match *nft_match = NULL;
647 	struct nfp_fl_ct_flow_entry *entry;
648 	struct nfp_fl_ct_map_entry *map;
649 	struct flow_action_entry *act;
650 	int err, i;
651 
652 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
653 	if (!entry)
654 		return ERR_PTR(-ENOMEM);
655 
656 	entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
657 	if (!entry->rule) {
658 		err = -ENOMEM;
659 		goto err_pre_ct_rule;
660 	}
661 
662 	/* nft flows gets destroyed after callback return, so need
663 	 * to do a full copy instead of just a reference.
664 	 */
665 	if (is_nft) {
666 		nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
667 		if (!nft_match) {
668 			err = -ENOMEM;
669 			goto err_pre_ct_act;
670 		}
671 		memcpy(&nft_match->dissector, flow->rule->match.dissector,
672 		       sizeof(nft_match->dissector));
673 		memcpy(&nft_match->mask, flow->rule->match.mask,
674 		       sizeof(nft_match->mask));
675 		memcpy(&nft_match->key, flow->rule->match.key,
676 		       sizeof(nft_match->key));
677 		entry->rule->match.dissector = &nft_match->dissector;
678 		entry->rule->match.mask = &nft_match->mask;
679 		entry->rule->match.key = &nft_match->key;
680 	} else {
681 		entry->rule->match.dissector = flow->rule->match.dissector;
682 		entry->rule->match.mask = flow->rule->match.mask;
683 		entry->rule->match.key = flow->rule->match.key;
684 	}
685 
686 	entry->zt = zt;
687 	entry->netdev = netdev;
688 	entry->cookie = flow->cookie;
689 	entry->chain_index = flow->common.chain_index;
690 	entry->tun_offset = NFP_FL_CT_NO_TUN;
691 
692 	/* Copy over action data. Unfortunately we do not get a handle to the
693 	 * original tcf_action data, and the flow objects gets destroyed, so we
694 	 * cannot just save a pointer to this either, so need to copy over the
695 	 * data unfortunately.
696 	 */
697 	entry->rule->action.num_entries = flow->rule->action.num_entries;
698 	flow_action_for_each(i, act, &flow->rule->action) {
699 		struct flow_action_entry *new_act;
700 
701 		new_act = &entry->rule->action.entries[i];
702 		memcpy(new_act, act, sizeof(struct flow_action_entry));
703 		/* Entunnel is a special case, need to allocate and copy
704 		 * tunnel info.
705 		 */
706 		if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
707 			struct ip_tunnel_info *tun = act->tunnel;
708 			size_t tun_size = sizeof(*tun) + tun->options_len;
709 
710 			new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
711 			if (!new_act->tunnel) {
712 				err = -ENOMEM;
713 				goto err_pre_ct_tun_cp;
714 			}
715 			entry->tun_offset = i;
716 		}
717 	}
718 
719 	INIT_LIST_HEAD(&entry->children);
720 
721 	/* Now add a ct map entry to flower-priv */
722 	map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
723 			    nfp_ct_map_params, sizeof(*map));
724 	if (IS_ERR(map)) {
725 		NL_SET_ERR_MSG_MOD(extack,
726 				   "offload error: ct map entry creation failed");
727 		err = -ENOMEM;
728 		goto err_ct_flow_insert;
729 	}
730 	map->cookie = flow->cookie;
731 	map->ct_entry = entry;
732 	err = rhashtable_insert_fast(&zt->priv->ct_map_table,
733 				     &map->hash_node,
734 				     nfp_ct_map_params);
735 	if (err) {
736 		NL_SET_ERR_MSG_MOD(extack,
737 				   "offload error: ct map entry table add failed");
738 		goto err_map_insert;
739 	}
740 
741 	return entry;
742 
743 err_map_insert:
744 	kfree(map);
745 err_ct_flow_insert:
746 	if (entry->tun_offset != NFP_FL_CT_NO_TUN)
747 		kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
748 err_pre_ct_tun_cp:
749 	kfree(nft_match);
750 err_pre_ct_act:
751 	kfree(entry->rule);
752 err_pre_ct_rule:
753 	kfree(entry);
754 	return ERR_PTR(err);
755 }
756 
757 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
758 {
759 	struct nfp_fl_ct_zone_entry *zt;
760 	int err;
761 
762 	zt = m_entry->zt;
763 
764 	/* Flow is in HW, need to delete */
765 	if (m_entry->tc_flower_cookie) {
766 		err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
767 					    m_entry->netdev);
768 		if (err)
769 			return;
770 	}
771 
772 	WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
773 					    &m_entry->hash_node,
774 					    nfp_nft_ct_merge_params));
775 	zt->nft_merge_count--;
776 	list_del(&m_entry->tc_merge_list);
777 	list_del(&m_entry->nft_flow_list);
778 
779 	kfree(m_entry);
780 }
781 
782 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
783 {
784 	struct nfp_fl_nft_tc_merge *m_entry, *tmp;
785 
786 	/* These post entries are parts of two lists, one is a list of nft_entries
787 	 * and the other is of from a list of tc_merge structures. Iterate
788 	 * through the relevant list and cleanup the entries.
789 	 */
790 
791 	if (is_nft_flow) {
792 		/* Need to iterate through list of nft_flow entries*/
793 		struct nfp_fl_ct_flow_entry *ct_entry = entry;
794 
795 		list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
796 					 nft_flow_list) {
797 			cleanup_nft_merge_entry(m_entry);
798 		}
799 	} else {
800 		/* Need to iterate through list of tc_merged_flow entries*/
801 		struct nfp_fl_ct_tc_merge *ct_entry = entry;
802 
803 		list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
804 					 tc_merge_list) {
805 			cleanup_nft_merge_entry(m_entry);
806 		}
807 	}
808 }
809 
810 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
811 {
812 	struct nfp_fl_ct_zone_entry *zt;
813 	int err;
814 
815 	zt = m_ent->zt;
816 	err = rhashtable_remove_fast(&zt->tc_merge_tb,
817 				     &m_ent->hash_node,
818 				     nfp_tc_ct_merge_params);
819 	if (err)
820 		pr_warn("WARNING: could not remove merge_entry from hashtable\n");
821 	zt->tc_merge_count--;
822 	list_del(&m_ent->post_ct_list);
823 	list_del(&m_ent->pre_ct_list);
824 
825 	if (!list_empty(&m_ent->children))
826 		nfp_free_nft_merge_children(m_ent, false);
827 	kfree(m_ent);
828 }
829 
830 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
831 {
832 	struct nfp_fl_ct_tc_merge *m_ent, *tmp;
833 
834 	switch (entry->type) {
835 	case CT_TYPE_PRE_CT:
836 		list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
837 			nfp_del_tc_merge_entry(m_ent);
838 		}
839 		break;
840 	case CT_TYPE_POST_CT:
841 		list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
842 			nfp_del_tc_merge_entry(m_ent);
843 		}
844 		break;
845 	default:
846 		break;
847 	}
848 }
849 
850 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
851 {
852 	list_del(&entry->list_node);
853 
854 	if (!list_empty(&entry->children)) {
855 		if (entry->type == CT_TYPE_NFT)
856 			nfp_free_nft_merge_children(entry, true);
857 		else
858 			nfp_free_tc_merge_children(entry);
859 	}
860 
861 	if (entry->tun_offset != NFP_FL_CT_NO_TUN)
862 		kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
863 
864 	if (entry->type == CT_TYPE_NFT) {
865 		struct nf_flow_match *nft_match;
866 
867 		nft_match = container_of(entry->rule->match.dissector,
868 					 struct nf_flow_match, dissector);
869 		kfree(nft_match);
870 	}
871 
872 	kfree(entry->rule);
873 	kfree(entry);
874 }
875 
876 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
877 					      enum flow_action_id act_id)
878 {
879 	struct flow_action_entry *act = NULL;
880 	int i;
881 
882 	flow_action_for_each(i, act, &rule->action) {
883 		if (act->id == act_id)
884 			return act;
885 	}
886 	return NULL;
887 }
888 
889 static void
890 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
891 			struct nfp_fl_ct_zone_entry *zt_src,
892 			struct nfp_fl_ct_zone_entry *zt_dst)
893 {
894 	struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
895 	struct list_head *ct_list;
896 
897 	if (ct_entry1->type == CT_TYPE_PRE_CT)
898 		ct_list = &zt_src->post_ct_list;
899 	else if (ct_entry1->type == CT_TYPE_POST_CT)
900 		ct_list = &zt_src->pre_ct_list;
901 	else
902 		return;
903 
904 	list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
905 				 list_node) {
906 		nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
907 	}
908 }
909 
910 static void
911 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
912 			 struct nfp_fl_ct_zone_entry *zt)
913 {
914 	struct nfp_fl_ct_tc_merge *tc_merge_entry;
915 	struct rhashtable_iter iter;
916 
917 	rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
918 	rhashtable_walk_start(&iter);
919 	while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
920 		if (IS_ERR(tc_merge_entry))
921 			continue;
922 		rhashtable_walk_stop(&iter);
923 		nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
924 		rhashtable_walk_start(&iter);
925 	}
926 	rhashtable_walk_stop(&iter);
927 	rhashtable_walk_exit(&iter);
928 }
929 
930 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
931 			    struct net_device *netdev,
932 			    struct flow_cls_offload *flow,
933 			    struct netlink_ext_ack *extack)
934 {
935 	struct flow_action_entry *ct_act, *ct_goto;
936 	struct nfp_fl_ct_flow_entry *ct_entry;
937 	struct nfp_fl_ct_zone_entry *zt;
938 	int err;
939 
940 	ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT);
941 	if (!ct_act) {
942 		NL_SET_ERR_MSG_MOD(extack,
943 				   "unsupported offload: Conntrack action empty in conntrack offload");
944 		return -EOPNOTSUPP;
945 	}
946 
947 	ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
948 	if (!ct_goto) {
949 		NL_SET_ERR_MSG_MOD(extack,
950 				   "unsupported offload: Conntrack requires ACTION_GOTO");
951 		return -EOPNOTSUPP;
952 	}
953 
954 	zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
955 	if (IS_ERR(zt)) {
956 		NL_SET_ERR_MSG_MOD(extack,
957 				   "offload error: Could not create zone table entry");
958 		return PTR_ERR(zt);
959 	}
960 
961 	if (!zt->nft) {
962 		zt->nft = ct_act->ct.flow_table;
963 		err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
964 		if (err) {
965 			NL_SET_ERR_MSG_MOD(extack,
966 					   "offload error: Could not register nft_callback");
967 			return err;
968 		}
969 	}
970 
971 	/* Add entry to pre_ct_list */
972 	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
973 	if (IS_ERR(ct_entry))
974 		return PTR_ERR(ct_entry);
975 	ct_entry->type = CT_TYPE_PRE_CT;
976 	ct_entry->chain_index = ct_goto->chain_index;
977 	list_add(&ct_entry->list_node, &zt->pre_ct_list);
978 	zt->pre_ct_count++;
979 
980 	nfp_ct_merge_tc_entries(ct_entry, zt, zt);
981 
982 	/* Need to check and merge with tables in the wc_zone as well */
983 	if (priv->ct_zone_wc)
984 		nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);
985 
986 	return 0;
987 }
988 
989 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
990 			     struct net_device *netdev,
991 			     struct flow_cls_offload *flow,
992 			     struct netlink_ext_ack *extack)
993 {
994 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
995 	struct nfp_fl_ct_flow_entry *ct_entry;
996 	struct nfp_fl_ct_zone_entry *zt;
997 	bool wildcarded = false;
998 	struct flow_match_ct ct;
999 
1000 	flow_rule_match_ct(rule, &ct);
1001 	if (!ct.mask->ct_zone) {
1002 		wildcarded = true;
1003 	} else if (ct.mask->ct_zone != U16_MAX) {
1004 		NL_SET_ERR_MSG_MOD(extack,
1005 				   "unsupported offload: partially wildcarded ct_zone is not supported");
1006 		return -EOPNOTSUPP;
1007 	}
1008 
1009 	zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
1010 	if (IS_ERR(zt)) {
1011 		NL_SET_ERR_MSG_MOD(extack,
1012 				   "offload error: Could not create zone table entry");
1013 		return PTR_ERR(zt);
1014 	}
1015 
1016 	/* Add entry to post_ct_list */
1017 	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1018 	if (IS_ERR(ct_entry))
1019 		return PTR_ERR(ct_entry);
1020 
1021 	ct_entry->type = CT_TYPE_POST_CT;
1022 	ct_entry->chain_index = flow->common.chain_index;
1023 	list_add(&ct_entry->list_node, &zt->post_ct_list);
1024 	zt->post_ct_count++;
1025 
1026 	if (wildcarded) {
1027 		/* Iterate through all zone tables if not empty, look for merges with
1028 		 * pre_ct entries and merge them.
1029 		 */
1030 		struct rhashtable_iter iter;
1031 		struct nfp_fl_ct_zone_entry *zone_table;
1032 
1033 		rhashtable_walk_enter(&priv->ct_zone_table, &iter);
1034 		rhashtable_walk_start(&iter);
1035 		while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
1036 			if (IS_ERR(zone_table))
1037 				continue;
1038 			rhashtable_walk_stop(&iter);
1039 			nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
1040 			rhashtable_walk_start(&iter);
1041 		}
1042 		rhashtable_walk_stop(&iter);
1043 		rhashtable_walk_exit(&iter);
1044 	} else {
1045 		nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1046 	}
1047 
1048 	return 0;
1049 }
1050 
1051 static int
1052 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
1053 {
1054 	struct nfp_fl_ct_map_entry *ct_map_ent;
1055 	struct nfp_fl_ct_flow_entry *ct_entry;
1056 	struct netlink_ext_ack *extack = NULL;
1057 
1058 	ASSERT_RTNL();
1059 
1060 	extack = flow->common.extack;
1061 	switch (flow->command) {
1062 	case FLOW_CLS_REPLACE:
1063 		/* Netfilter can request offload multiple times for the same
1064 		 * flow - protect against adding duplicates.
1065 		 */
1066 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1067 						    nfp_ct_map_params);
1068 		if (!ct_map_ent) {
1069 			ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
1070 			if (IS_ERR(ct_entry))
1071 				return PTR_ERR(ct_entry);
1072 			ct_entry->type = CT_TYPE_NFT;
1073 			list_add(&ct_entry->list_node, &zt->nft_flows_list);
1074 			zt->nft_flows_count++;
1075 			nfp_ct_merge_nft_with_tc(ct_entry, zt);
1076 		}
1077 		return 0;
1078 	case FLOW_CLS_DESTROY:
1079 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1080 						    nfp_ct_map_params);
1081 		return nfp_fl_ct_del_flow(ct_map_ent);
1082 	case FLOW_CLS_STATS:
1083 		return 0;
1084 	default:
1085 		break;
1086 	}
1087 	return -EINVAL;
1088 }
1089 
1090 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
1091 {
1092 	struct flow_cls_offload *flow = type_data;
1093 	struct nfp_fl_ct_zone_entry *zt = cb_priv;
1094 	int err = -EOPNOTSUPP;
1095 
1096 	switch (type) {
1097 	case TC_SETUP_CLSFLOWER:
1098 		rtnl_lock();
1099 		err = nfp_fl_ct_offload_nft_flow(zt, flow);
1100 		rtnl_unlock();
1101 		break;
1102 	default:
1103 		return -EOPNOTSUPP;
1104 	}
1105 	return err;
1106 }
1107 
1108 static void
1109 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
1110 {
1111 	struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
1112 	struct nfp_fl_ct_map_entry *ct_map_ent;
1113 
1114 	list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
1115 				 list_node) {
1116 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
1117 						    &nft_entry->cookie,
1118 						    nfp_ct_map_params);
1119 		nfp_fl_ct_del_flow(ct_map_ent);
1120 	}
1121 }
1122 
1123 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
1124 {
1125 	struct nfp_fl_ct_flow_entry *ct_entry;
1126 	struct nfp_fl_ct_zone_entry *zt;
1127 	struct rhashtable *m_table;
1128 
1129 	if (!ct_map_ent)
1130 		return -ENOENT;
1131 
1132 	zt = ct_map_ent->ct_entry->zt;
1133 	ct_entry = ct_map_ent->ct_entry;
1134 	m_table = &zt->priv->ct_map_table;
1135 
1136 	switch (ct_entry->type) {
1137 	case CT_TYPE_PRE_CT:
1138 		zt->pre_ct_count--;
1139 		rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1140 				       nfp_ct_map_params);
1141 		nfp_fl_ct_clean_flow_entry(ct_entry);
1142 		kfree(ct_map_ent);
1143 
1144 		if (!zt->pre_ct_count) {
1145 			zt->nft = NULL;
1146 			nfp_fl_ct_clean_nft_entries(zt);
1147 		}
1148 		break;
1149 	case CT_TYPE_POST_CT:
1150 		zt->post_ct_count--;
1151 		rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1152 				       nfp_ct_map_params);
1153 		nfp_fl_ct_clean_flow_entry(ct_entry);
1154 		kfree(ct_map_ent);
1155 		break;
1156 	case CT_TYPE_NFT:
1157 		zt->nft_flows_count--;
1158 		rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1159 				       nfp_ct_map_params);
1160 		nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);
1161 		kfree(ct_map_ent);
1162 		break;
1163 	default:
1164 		break;
1165 	}
1166 
1167 	return 0;
1168 }
1169