1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3 
4 #include <linux/netdevice.h>
5 #include "en/fs_tt_redirect.h"
6 #include "fs_core.h"
7 
8 enum fs_udp_type {
9 	FS_IPV4_UDP,
10 	FS_IPV6_UDP,
11 	FS_UDP_NUM_TYPES,
12 };
13 
14 struct mlx5e_fs_udp {
15 	struct mlx5e_flow_table tables[FS_UDP_NUM_TYPES];
16 	struct mlx5_flow_handle *default_rules[FS_UDP_NUM_TYPES];
17 	int ref_cnt;
18 };
19 
20 struct mlx5e_fs_any {
21 	struct mlx5e_flow_table table;
22 	struct mlx5_flow_handle *default_rule;
23 	int ref_cnt;
24 };
25 
26 static char *fs_udp_type2str(enum fs_udp_type i)
27 {
28 	switch (i) {
29 	case FS_IPV4_UDP:
30 		return "UDP v4";
31 	default: /* FS_IPV6_UDP */
32 		return "UDP v6";
33 	}
34 }
35 
36 static enum mlx5_traffic_types fs_udp2tt(enum fs_udp_type i)
37 {
38 	switch (i) {
39 	case FS_IPV4_UDP:
40 		return MLX5_TT_IPV4_UDP;
41 	default: /* FS_IPV6_UDP */
42 		return MLX5_TT_IPV6_UDP;
43 	}
44 }
45 
46 static enum fs_udp_type tt2fs_udp(enum mlx5_traffic_types i)
47 {
48 	switch (i) {
49 	case MLX5_TT_IPV4_UDP:
50 		return FS_IPV4_UDP;
51 	case MLX5_TT_IPV6_UDP:
52 		return FS_IPV6_UDP;
53 	default:
54 		return FS_UDP_NUM_TYPES;
55 	}
56 }
57 
58 void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule)
59 {
60 	mlx5_del_flow_rules(rule);
61 }
62 
63 static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type type,
64 				  u16 udp_dport)
65 {
66 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
67 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
68 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
69 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
70 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
71 		 type == FS_IPV4_UDP ? 4 : 6);
72 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
73 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, udp_dport);
74 }
75 
76 struct mlx5_flow_handle *
77 mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
78 				  enum mlx5_traffic_types ttc_type,
79 				  u32 tir_num, u16 d_port)
80 {
81 	enum fs_udp_type type = tt2fs_udp(ttc_type);
82 	struct mlx5_flow_destination dest = {};
83 	struct mlx5_flow_table *ft = NULL;
84 	MLX5_DECLARE_FLOW_ACT(flow_act);
85 	struct mlx5_flow_handle *rule;
86 	struct mlx5_flow_spec *spec;
87 	struct mlx5e_fs_udp *fs_udp;
88 	int err;
89 
90 	if (type == FS_UDP_NUM_TYPES)
91 		return ERR_PTR(-EINVAL);
92 
93 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
94 	if (!spec)
95 		return ERR_PTR(-ENOMEM);
96 
97 	fs_udp = priv->fs.udp;
98 	ft = fs_udp->tables[type].t;
99 
100 	fs_udp_set_dport_flow(spec, type, d_port);
101 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
102 	dest.tir_num = tir_num;
103 
104 	rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
105 	kvfree(spec);
106 
107 	if (IS_ERR(rule)) {
108 		err = PTR_ERR(rule);
109 		netdev_err(priv->netdev, "%s: add %s rule failed, err %d\n",
110 			   __func__, fs_udp_type2str(type), err);
111 	}
112 	return rule;
113 }
114 
115 static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type type)
116 {
117 	struct mlx5e_flow_table *fs_udp_t;
118 	struct mlx5_flow_destination dest;
119 	MLX5_DECLARE_FLOW_ACT(flow_act);
120 	struct mlx5_flow_handle *rule;
121 	struct mlx5e_fs_udp *fs_udp;
122 	int err;
123 
124 	fs_udp = priv->fs.udp;
125 	fs_udp_t = &fs_udp->tables[type];
126 
127 	dest = mlx5_ttc_get_default_dest(priv->fs.ttc, fs_udp2tt(type));
128 	rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1);
129 	if (IS_ERR(rule)) {
130 		err = PTR_ERR(rule);
131 		netdev_err(priv->netdev,
132 			   "%s: add default rule failed, fs type=%d, err %d\n",
133 			   __func__, type, err);
134 		return err;
135 	}
136 
137 	fs_udp->default_rules[type] = rule;
138 	return 0;
139 }
140 
141 #define MLX5E_FS_UDP_NUM_GROUPS	(2)
142 #define MLX5E_FS_UDP_GROUP1_SIZE	(BIT(16))
143 #define MLX5E_FS_UDP_GROUP2_SIZE	(BIT(0))
144 #define MLX5E_FS_UDP_TABLE_SIZE		(MLX5E_FS_UDP_GROUP1_SIZE +\
145 					 MLX5E_FS_UDP_GROUP2_SIZE)
146 static int fs_udp_create_groups(struct mlx5e_flow_table *ft, enum fs_udp_type type)
147 {
148 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
149 	void *outer_headers_c;
150 	int ix = 0;
151 	u32 *in;
152 	int err;
153 	u8 *mc;
154 
155 	ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
156 	in = kvzalloc(inlen, GFP_KERNEL);
157 	if  (!in || !ft->g) {
158 		kfree(ft->g);
159 		kvfree(in);
160 		return -ENOMEM;
161 	}
162 
163 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
164 	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
165 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
166 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
167 
168 	switch (type) {
169 	case FS_IPV4_UDP:
170 	case FS_IPV6_UDP:
171 		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
172 		break;
173 	default:
174 		err = -EINVAL;
175 		goto out;
176 	}
177 	/* Match on udp protocol, Ipv4/6 and dport */
178 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
179 	MLX5_SET_CFG(in, start_flow_index, ix);
180 	ix += MLX5E_FS_UDP_GROUP1_SIZE;
181 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
182 	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
183 	if (IS_ERR(ft->g[ft->num_groups]))
184 		goto err;
185 	ft->num_groups++;
186 
187 	/* Default Flow Group */
188 	memset(in, 0, inlen);
189 	MLX5_SET_CFG(in, start_flow_index, ix);
190 	ix += MLX5E_FS_UDP_GROUP2_SIZE;
191 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
192 	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
193 	if (IS_ERR(ft->g[ft->num_groups]))
194 		goto err;
195 	ft->num_groups++;
196 
197 	kvfree(in);
198 	return 0;
199 
200 err:
201 	err = PTR_ERR(ft->g[ft->num_groups]);
202 	ft->g[ft->num_groups] = NULL;
203 out:
204 	kvfree(in);
205 
206 	return err;
207 }
208 
209 static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type)
210 {
211 	struct mlx5e_flow_table *ft = &priv->fs.udp->tables[type];
212 	struct mlx5_flow_table_attr ft_attr = {};
213 	int err;
214 
215 	ft->num_groups = 0;
216 
217 	ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
218 	ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL;
219 	ft_attr.prio = MLX5E_NIC_PRIO;
220 
221 	ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
222 	if (IS_ERR(ft->t)) {
223 		err = PTR_ERR(ft->t);
224 		ft->t = NULL;
225 		return err;
226 	}
227 
228 	netdev_dbg(priv->netdev, "Created fs %s table id %u level %u\n",
229 		   fs_udp_type2str(type), ft->t->id, ft->t->level);
230 
231 	err = fs_udp_create_groups(ft, type);
232 	if (err)
233 		goto err;
234 
235 	err = fs_udp_add_default_rule(priv, type);
236 	if (err)
237 		goto err;
238 
239 	return 0;
240 
241 err:
242 	mlx5e_destroy_flow_table(ft);
243 	return err;
244 }
245 
246 static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i)
247 {
248 	if (IS_ERR_OR_NULL(fs_udp->tables[i].t))
249 		return;
250 
251 	mlx5_del_flow_rules(fs_udp->default_rules[i]);
252 	mlx5e_destroy_flow_table(&fs_udp->tables[i]);
253 	fs_udp->tables[i].t = NULL;
254 }
255 
256 static int fs_udp_disable(struct mlx5e_priv *priv)
257 {
258 	int err, i;
259 
260 	for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
261 		/* Modify ttc rules destination to point back to the indir TIRs */
262 		err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_udp2tt(i));
263 		if (err) {
264 			netdev_err(priv->netdev,
265 				   "%s: modify ttc[%d] default destination failed, err(%d)\n",
266 				   __func__, fs_udp2tt(i), err);
267 			return err;
268 		}
269 	}
270 
271 	return 0;
272 }
273 
274 static int fs_udp_enable(struct mlx5e_priv *priv)
275 {
276 	struct mlx5_flow_destination dest = {};
277 	int err, i;
278 
279 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
280 	for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
281 		dest.ft = priv->fs.udp->tables[i].t;
282 
283 		/* Modify ttc rules destination to point on the accel_fs FTs */
284 		err = mlx5_ttc_fwd_dest(priv->fs.ttc, fs_udp2tt(i), &dest);
285 		if (err) {
286 			netdev_err(priv->netdev,
287 				   "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
288 				   __func__, fs_udp2tt(i), err);
289 			return err;
290 		}
291 	}
292 	return 0;
293 }
294 
295 void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv)
296 {
297 	struct mlx5e_fs_udp *fs_udp = priv->fs.udp;
298 	int i;
299 
300 	if (!fs_udp)
301 		return;
302 
303 	if (--fs_udp->ref_cnt)
304 		return;
305 
306 	fs_udp_disable(priv);
307 
308 	for (i = 0; i < FS_UDP_NUM_TYPES; i++)
309 		fs_udp_destroy_table(fs_udp, i);
310 
311 	kfree(fs_udp);
312 	priv->fs.udp = NULL;
313 }
314 
315 int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv)
316 {
317 	int i, err;
318 
319 	if (priv->fs.udp) {
320 		priv->fs.udp->ref_cnt++;
321 		return 0;
322 	}
323 
324 	priv->fs.udp = kzalloc(sizeof(*priv->fs.udp), GFP_KERNEL);
325 	if (!priv->fs.udp)
326 		return -ENOMEM;
327 
328 	for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
329 		err = fs_udp_create_table(priv, i);
330 		if (err)
331 			goto err_destroy_tables;
332 	}
333 
334 	err = fs_udp_enable(priv);
335 	if (err)
336 		goto err_destroy_tables;
337 
338 	priv->fs.udp->ref_cnt = 1;
339 
340 	return 0;
341 
342 err_destroy_tables:
343 	while (--i >= 0)
344 		fs_udp_destroy_table(priv->fs.udp, i);
345 
346 	kfree(priv->fs.udp);
347 	priv->fs.udp = NULL;
348 	return err;
349 }
350 
351 static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_type)
352 {
353 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
354 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
355 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ether_type);
356 }
357 
358 struct mlx5_flow_handle *
359 mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv,
360 				  u32 tir_num, u16 ether_type)
361 {
362 	struct mlx5_flow_destination dest = {};
363 	struct mlx5_flow_table *ft = NULL;
364 	MLX5_DECLARE_FLOW_ACT(flow_act);
365 	struct mlx5_flow_handle *rule;
366 	struct mlx5_flow_spec *spec;
367 	struct mlx5e_fs_any *fs_any;
368 	int err;
369 
370 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
371 	if (!spec)
372 		return ERR_PTR(-ENOMEM);
373 
374 	fs_any = priv->fs.any;
375 	ft = fs_any->table.t;
376 
377 	fs_any_set_ethertype_flow(spec, ether_type);
378 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
379 	dest.tir_num = tir_num;
380 
381 	rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
382 	kvfree(spec);
383 
384 	if (IS_ERR(rule)) {
385 		err = PTR_ERR(rule);
386 		netdev_err(priv->netdev, "%s: add ANY rule failed, err %d\n",
387 			   __func__, err);
388 	}
389 	return rule;
390 }
391 
392 static int fs_any_add_default_rule(struct mlx5e_priv *priv)
393 {
394 	struct mlx5e_flow_table *fs_any_t;
395 	struct mlx5_flow_destination dest;
396 	MLX5_DECLARE_FLOW_ACT(flow_act);
397 	struct mlx5_flow_handle *rule;
398 	struct mlx5e_fs_any *fs_any;
399 	int err;
400 
401 	fs_any = priv->fs.any;
402 	fs_any_t = &fs_any->table;
403 
404 	dest = mlx5_ttc_get_default_dest(priv->fs.ttc, MLX5_TT_ANY);
405 	rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1);
406 	if (IS_ERR(rule)) {
407 		err = PTR_ERR(rule);
408 		netdev_err(priv->netdev,
409 			   "%s: add default rule failed, fs type=ANY, err %d\n",
410 			   __func__, err);
411 		return err;
412 	}
413 
414 	fs_any->default_rule = rule;
415 	return 0;
416 }
417 
418 #define MLX5E_FS_ANY_NUM_GROUPS	(2)
419 #define MLX5E_FS_ANY_GROUP1_SIZE	(BIT(16))
420 #define MLX5E_FS_ANY_GROUP2_SIZE	(BIT(0))
421 #define MLX5E_FS_ANY_TABLE_SIZE		(MLX5E_FS_ANY_GROUP1_SIZE +\
422 					 MLX5E_FS_ANY_GROUP2_SIZE)
423 
424 static int fs_any_create_groups(struct mlx5e_flow_table *ft)
425 {
426 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
427 	void *outer_headers_c;
428 	int ix = 0;
429 	u32 *in;
430 	int err;
431 	u8 *mc;
432 
433 	ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
434 	in = kvzalloc(inlen, GFP_KERNEL);
435 	if  (!in || !ft->g) {
436 		kfree(ft->g);
437 		kvfree(in);
438 		return -ENOMEM;
439 	}
440 
441 	/* Match on ethertype */
442 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
443 	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
444 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
445 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
446 	MLX5_SET_CFG(in, start_flow_index, ix);
447 	ix += MLX5E_FS_ANY_GROUP1_SIZE;
448 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
449 	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
450 	if (IS_ERR(ft->g[ft->num_groups]))
451 		goto err;
452 	ft->num_groups++;
453 
454 	/* Default Flow Group */
455 	memset(in, 0, inlen);
456 	MLX5_SET_CFG(in, start_flow_index, ix);
457 	ix += MLX5E_FS_ANY_GROUP2_SIZE;
458 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
459 	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
460 	if (IS_ERR(ft->g[ft->num_groups]))
461 		goto err;
462 	ft->num_groups++;
463 
464 	kvfree(in);
465 	return 0;
466 
467 err:
468 	err = PTR_ERR(ft->g[ft->num_groups]);
469 	ft->g[ft->num_groups] = NULL;
470 	kvfree(in);
471 
472 	return err;
473 }
474 
475 static int fs_any_create_table(struct mlx5e_priv *priv)
476 {
477 	struct mlx5e_flow_table *ft = &priv->fs.any->table;
478 	struct mlx5_flow_table_attr ft_attr = {};
479 	int err;
480 
481 	ft->num_groups = 0;
482 
483 	ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
484 	ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL;
485 	ft_attr.prio = MLX5E_NIC_PRIO;
486 
487 	ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
488 	if (IS_ERR(ft->t)) {
489 		err = PTR_ERR(ft->t);
490 		ft->t = NULL;
491 		return err;
492 	}
493 
494 	netdev_dbg(priv->netdev, "Created fs ANY table id %u level %u\n",
495 		   ft->t->id, ft->t->level);
496 
497 	err = fs_any_create_groups(ft);
498 	if (err)
499 		goto err;
500 
501 	err = fs_any_add_default_rule(priv);
502 	if (err)
503 		goto err;
504 
505 	return 0;
506 
507 err:
508 	mlx5e_destroy_flow_table(ft);
509 	return err;
510 }
511 
512 static int fs_any_disable(struct mlx5e_priv *priv)
513 {
514 	int err;
515 
516 	/* Modify ttc rules destination to point back to the indir TIRs */
517 	err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, MLX5_TT_ANY);
518 	if (err) {
519 		netdev_err(priv->netdev,
520 			   "%s: modify ttc[%d] default destination failed, err(%d)\n",
521 			   __func__, MLX5_TT_ANY, err);
522 		return err;
523 	}
524 	return 0;
525 }
526 
527 static int fs_any_enable(struct mlx5e_priv *priv)
528 {
529 	struct mlx5_flow_destination dest = {};
530 	int err;
531 
532 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
533 	dest.ft = priv->fs.any->table.t;
534 
535 	/* Modify ttc rules destination to point on the accel_fs FTs */
536 	err = mlx5_ttc_fwd_dest(priv->fs.ttc, MLX5_TT_ANY, &dest);
537 	if (err) {
538 		netdev_err(priv->netdev,
539 			   "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
540 			   __func__, MLX5_TT_ANY, err);
541 		return err;
542 	}
543 	return 0;
544 }
545 
546 static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any)
547 {
548 	if (IS_ERR_OR_NULL(fs_any->table.t))
549 		return;
550 
551 	mlx5_del_flow_rules(fs_any->default_rule);
552 	mlx5e_destroy_flow_table(&fs_any->table);
553 	fs_any->table.t = NULL;
554 }
555 
556 void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv)
557 {
558 	struct mlx5e_fs_any *fs_any = priv->fs.any;
559 
560 	if (!fs_any)
561 		return;
562 
563 	if (--fs_any->ref_cnt)
564 		return;
565 
566 	fs_any_disable(priv);
567 
568 	fs_any_destroy_table(fs_any);
569 
570 	kfree(fs_any);
571 	priv->fs.any = NULL;
572 }
573 
574 int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv)
575 {
576 	int err;
577 
578 	if (priv->fs.any) {
579 		priv->fs.any->ref_cnt++;
580 		return 0;
581 	}
582 
583 	priv->fs.any = kzalloc(sizeof(*priv->fs.any), GFP_KERNEL);
584 	if (!priv->fs.any)
585 		return -ENOMEM;
586 
587 	err = fs_any_create_table(priv);
588 	if (err)
589 		return err;
590 
591 	err = fs_any_enable(priv);
592 	if (err)
593 		goto err_destroy_table;
594 
595 	priv->fs.any->ref_cnt = 1;
596 
597 	return 0;
598 
599 err_destroy_table:
600 	fs_any_destroy_table(priv->fs.any);
601 
602 	kfree(priv->fs.any);
603 	priv->fs.any = NULL;
604 	return err;
605 }
606