1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3 4 #include <linux/netdevice.h> 5 #include "en/fs_tt_redirect.h" 6 #include "fs_core.h" 7 8 enum fs_udp_type { 9 FS_IPV4_UDP, 10 FS_IPV6_UDP, 11 FS_UDP_NUM_TYPES, 12 }; 13 14 struct mlx5e_fs_udp { 15 struct mlx5e_flow_table tables[FS_UDP_NUM_TYPES]; 16 struct mlx5_flow_handle *default_rules[FS_UDP_NUM_TYPES]; 17 int ref_cnt; 18 }; 19 20 struct mlx5e_fs_any { 21 struct mlx5e_flow_table table; 22 struct mlx5_flow_handle *default_rule; 23 int ref_cnt; 24 }; 25 26 static char *fs_udp_type2str(enum fs_udp_type i) 27 { 28 switch (i) { 29 case FS_IPV4_UDP: 30 return "UDP v4"; 31 default: /* FS_IPV6_UDP */ 32 return "UDP v6"; 33 } 34 } 35 36 static enum mlx5_traffic_types fs_udp2tt(enum fs_udp_type i) 37 { 38 switch (i) { 39 case FS_IPV4_UDP: 40 return MLX5_TT_IPV4_UDP; 41 default: /* FS_IPV6_UDP */ 42 return MLX5_TT_IPV6_UDP; 43 } 44 } 45 46 static enum fs_udp_type tt2fs_udp(enum mlx5_traffic_types i) 47 { 48 switch (i) { 49 case MLX5_TT_IPV4_UDP: 50 return FS_IPV4_UDP; 51 case MLX5_TT_IPV6_UDP: 52 return FS_IPV6_UDP; 53 default: 54 return FS_UDP_NUM_TYPES; 55 } 56 } 57 58 void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule) 59 { 60 mlx5_del_flow_rules(rule); 61 } 62 63 static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type type, 64 u16 udp_dport) 65 { 66 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 67 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 68 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); 69 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 70 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 71 type == FS_IPV4_UDP ? 4 : 6); 72 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); 73 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, udp_dport); 74 } 75 76 struct mlx5_flow_handle * 77 mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv, 78 enum mlx5_traffic_types ttc_type, 79 u32 tir_num, u16 d_port) 80 { 81 enum fs_udp_type type = tt2fs_udp(ttc_type); 82 struct mlx5_flow_destination dest = {}; 83 struct mlx5_flow_table *ft = NULL; 84 MLX5_DECLARE_FLOW_ACT(flow_act); 85 struct mlx5_flow_handle *rule; 86 struct mlx5_flow_spec *spec; 87 struct mlx5e_fs_udp *fs_udp; 88 int err; 89 90 if (type == FS_UDP_NUM_TYPES) 91 return ERR_PTR(-EINVAL); 92 93 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 94 if (!spec) 95 return ERR_PTR(-ENOMEM); 96 97 fs_udp = priv->fs->udp; 98 ft = fs_udp->tables[type].t; 99 100 fs_udp_set_dport_flow(spec, type, d_port); 101 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 102 dest.tir_num = tir_num; 103 104 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 105 kvfree(spec); 106 107 if (IS_ERR(rule)) { 108 err = PTR_ERR(rule); 109 netdev_err(priv->netdev, "%s: add %s rule failed, err %d\n", 110 __func__, fs_udp_type2str(type), err); 111 } 112 return rule; 113 } 114 115 static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type type) 116 { 117 struct mlx5e_flow_table *fs_udp_t; 118 struct mlx5_flow_destination dest; 119 MLX5_DECLARE_FLOW_ACT(flow_act); 120 struct mlx5_flow_handle *rule; 121 struct mlx5e_fs_udp *fs_udp; 122 int err; 123 124 fs_udp = priv->fs->udp; 125 fs_udp_t = &fs_udp->tables[type]; 126 127 dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_udp2tt(type)); 128 rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1); 129 if (IS_ERR(rule)) { 130 err = PTR_ERR(rule); 131 netdev_err(priv->netdev, 132 "%s: add default rule failed, fs type=%d, err %d\n", 133 __func__, type, err); 134 return err; 135 } 136 137 fs_udp->default_rules[type] = rule; 138 return 0; 139 } 140 141 #define MLX5E_FS_UDP_NUM_GROUPS (2) 142 #define MLX5E_FS_UDP_GROUP1_SIZE (BIT(16)) 143 #define MLX5E_FS_UDP_GROUP2_SIZE (BIT(0)) 144 #define MLX5E_FS_UDP_TABLE_SIZE (MLX5E_FS_UDP_GROUP1_SIZE +\ 145 MLX5E_FS_UDP_GROUP2_SIZE) 146 static int fs_udp_create_groups(struct mlx5e_flow_table *ft, enum fs_udp_type type) 147 { 148 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 149 void *outer_headers_c; 150 int ix = 0; 151 u32 *in; 152 int err; 153 u8 *mc; 154 155 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 156 in = kvzalloc(inlen, GFP_KERNEL); 157 if (!in || !ft->g) { 158 kfree(ft->g); 159 kvfree(in); 160 return -ENOMEM; 161 } 162 163 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 164 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 165 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 166 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); 167 168 switch (type) { 169 case FS_IPV4_UDP: 170 case FS_IPV6_UDP: 171 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 172 break; 173 default: 174 err = -EINVAL; 175 goto out; 176 } 177 /* Match on udp protocol, Ipv4/6 and dport */ 178 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 179 MLX5_SET_CFG(in, start_flow_index, ix); 180 ix += MLX5E_FS_UDP_GROUP1_SIZE; 181 MLX5_SET_CFG(in, end_flow_index, ix - 1); 182 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 183 if (IS_ERR(ft->g[ft->num_groups])) 184 goto err; 185 ft->num_groups++; 186 187 /* Default Flow Group */ 188 memset(in, 0, inlen); 189 MLX5_SET_CFG(in, start_flow_index, ix); 190 ix += MLX5E_FS_UDP_GROUP2_SIZE; 191 MLX5_SET_CFG(in, end_flow_index, ix - 1); 192 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 193 if (IS_ERR(ft->g[ft->num_groups])) 194 goto err; 195 ft->num_groups++; 196 197 kvfree(in); 198 return 0; 199 200 err: 201 err = PTR_ERR(ft->g[ft->num_groups]); 202 ft->g[ft->num_groups] = NULL; 203 out: 204 kvfree(in); 205 206 return err; 207 } 208 209 static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type) 210 { 211 struct mlx5e_flow_table *ft = &priv->fs->udp->tables[type]; 212 struct mlx5_flow_table_attr ft_attr = {}; 213 int err; 214 215 ft->num_groups = 0; 216 217 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE; 218 ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL; 219 ft_attr.prio = MLX5E_NIC_PRIO; 220 221 ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); 222 if (IS_ERR(ft->t)) { 223 err = PTR_ERR(ft->t); 224 ft->t = NULL; 225 return err; 226 } 227 228 netdev_dbg(priv->netdev, "Created fs %s table id %u level %u\n", 229 fs_udp_type2str(type), ft->t->id, ft->t->level); 230 231 err = fs_udp_create_groups(ft, type); 232 if (err) 233 goto err; 234 235 err = fs_udp_add_default_rule(priv, type); 236 if (err) 237 goto err; 238 239 return 0; 240 241 err: 242 mlx5e_destroy_flow_table(ft); 243 return err; 244 } 245 246 static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i) 247 { 248 if (IS_ERR_OR_NULL(fs_udp->tables[i].t)) 249 return; 250 251 mlx5_del_flow_rules(fs_udp->default_rules[i]); 252 mlx5e_destroy_flow_table(&fs_udp->tables[i]); 253 fs_udp->tables[i].t = NULL; 254 } 255 256 static int fs_udp_disable(struct mlx5e_priv *priv) 257 { 258 int err, i; 259 260 for (i = 0; i < FS_UDP_NUM_TYPES; i++) { 261 /* Modify ttc rules destination to point back to the indir TIRs */ 262 err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_udp2tt(i)); 263 if (err) { 264 netdev_err(priv->netdev, 265 "%s: modify ttc[%d] default destination failed, err(%d)\n", 266 __func__, fs_udp2tt(i), err); 267 return err; 268 } 269 } 270 271 return 0; 272 } 273 274 static int fs_udp_enable(struct mlx5e_priv *priv) 275 { 276 struct mlx5_flow_destination dest = {}; 277 int err, i; 278 279 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 280 for (i = 0; i < FS_UDP_NUM_TYPES; i++) { 281 dest.ft = priv->fs->udp->tables[i].t; 282 283 /* Modify ttc rules destination to point on the accel_fs FTs */ 284 err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_udp2tt(i), &dest); 285 if (err) { 286 netdev_err(priv->netdev, 287 "%s: modify ttc[%d] destination to accel failed, err(%d)\n", 288 __func__, fs_udp2tt(i), err); 289 return err; 290 } 291 } 292 return 0; 293 } 294 295 void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv) 296 { 297 struct mlx5e_fs_udp *fs_udp = priv->fs->udp; 298 int i; 299 300 if (!fs_udp) 301 return; 302 303 if (--fs_udp->ref_cnt) 304 return; 305 306 fs_udp_disable(priv); 307 308 for (i = 0; i < FS_UDP_NUM_TYPES; i++) 309 fs_udp_destroy_table(fs_udp, i); 310 311 kfree(fs_udp); 312 priv->fs->udp = NULL; 313 } 314 315 int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv) 316 { 317 int i, err; 318 319 if (priv->fs->udp) { 320 priv->fs->udp->ref_cnt++; 321 return 0; 322 } 323 324 priv->fs->udp = kzalloc(sizeof(*priv->fs->udp), GFP_KERNEL); 325 if (!priv->fs->udp) 326 return -ENOMEM; 327 328 for (i = 0; i < FS_UDP_NUM_TYPES; i++) { 329 err = fs_udp_create_table(priv, i); 330 if (err) 331 goto err_destroy_tables; 332 } 333 334 err = fs_udp_enable(priv); 335 if (err) 336 goto err_destroy_tables; 337 338 priv->fs->udp->ref_cnt = 1; 339 340 return 0; 341 342 err_destroy_tables: 343 while (--i >= 0) 344 fs_udp_destroy_table(priv->fs->udp, i); 345 346 kfree(priv->fs->udp); 347 priv->fs->udp = NULL; 348 return err; 349 } 350 351 static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_type) 352 { 353 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 354 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); 355 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ether_type); 356 } 357 358 struct mlx5_flow_handle * 359 mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv, 360 u32 tir_num, u16 ether_type) 361 { 362 struct mlx5_flow_destination dest = {}; 363 struct mlx5_flow_table *ft = NULL; 364 MLX5_DECLARE_FLOW_ACT(flow_act); 365 struct mlx5_flow_handle *rule; 366 struct mlx5_flow_spec *spec; 367 struct mlx5e_fs_any *fs_any; 368 int err; 369 370 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 371 if (!spec) 372 return ERR_PTR(-ENOMEM); 373 374 fs_any = priv->fs->any; 375 ft = fs_any->table.t; 376 377 fs_any_set_ethertype_flow(spec, ether_type); 378 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 379 dest.tir_num = tir_num; 380 381 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 382 kvfree(spec); 383 384 if (IS_ERR(rule)) { 385 err = PTR_ERR(rule); 386 netdev_err(priv->netdev, "%s: add ANY rule failed, err %d\n", 387 __func__, err); 388 } 389 return rule; 390 } 391 392 static int fs_any_add_default_rule(struct mlx5e_priv *priv) 393 { 394 struct mlx5e_flow_table *fs_any_t; 395 struct mlx5_flow_destination dest; 396 MLX5_DECLARE_FLOW_ACT(flow_act); 397 struct mlx5_flow_handle *rule; 398 struct mlx5e_fs_any *fs_any; 399 int err; 400 401 fs_any = priv->fs->any; 402 fs_any_t = &fs_any->table; 403 404 dest = mlx5_ttc_get_default_dest(priv->fs->ttc, MLX5_TT_ANY); 405 rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1); 406 if (IS_ERR(rule)) { 407 err = PTR_ERR(rule); 408 netdev_err(priv->netdev, 409 "%s: add default rule failed, fs type=ANY, err %d\n", 410 __func__, err); 411 return err; 412 } 413 414 fs_any->default_rule = rule; 415 return 0; 416 } 417 418 #define MLX5E_FS_ANY_NUM_GROUPS (2) 419 #define MLX5E_FS_ANY_GROUP1_SIZE (BIT(16)) 420 #define MLX5E_FS_ANY_GROUP2_SIZE (BIT(0)) 421 #define MLX5E_FS_ANY_TABLE_SIZE (MLX5E_FS_ANY_GROUP1_SIZE +\ 422 MLX5E_FS_ANY_GROUP2_SIZE) 423 424 static int fs_any_create_groups(struct mlx5e_flow_table *ft) 425 { 426 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 427 void *outer_headers_c; 428 int ix = 0; 429 u32 *in; 430 int err; 431 u8 *mc; 432 433 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 434 in = kvzalloc(inlen, GFP_KERNEL); 435 if (!in || !ft->g) { 436 kfree(ft->g); 437 kvfree(in); 438 return -ENOMEM; 439 } 440 441 /* Match on ethertype */ 442 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 443 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 444 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); 445 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 446 MLX5_SET_CFG(in, start_flow_index, ix); 447 ix += MLX5E_FS_ANY_GROUP1_SIZE; 448 MLX5_SET_CFG(in, end_flow_index, ix - 1); 449 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 450 if (IS_ERR(ft->g[ft->num_groups])) 451 goto err; 452 ft->num_groups++; 453 454 /* Default Flow Group */ 455 memset(in, 0, inlen); 456 MLX5_SET_CFG(in, start_flow_index, ix); 457 ix += MLX5E_FS_ANY_GROUP2_SIZE; 458 MLX5_SET_CFG(in, end_flow_index, ix - 1); 459 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 460 if (IS_ERR(ft->g[ft->num_groups])) 461 goto err; 462 ft->num_groups++; 463 464 kvfree(in); 465 return 0; 466 467 err: 468 err = PTR_ERR(ft->g[ft->num_groups]); 469 ft->g[ft->num_groups] = NULL; 470 kvfree(in); 471 472 return err; 473 } 474 475 static int fs_any_create_table(struct mlx5e_priv *priv) 476 { 477 struct mlx5e_flow_table *ft = &priv->fs->any->table; 478 struct mlx5_flow_table_attr ft_attr = {}; 479 int err; 480 481 ft->num_groups = 0; 482 483 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE; 484 ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL; 485 ft_attr.prio = MLX5E_NIC_PRIO; 486 487 ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); 488 if (IS_ERR(ft->t)) { 489 err = PTR_ERR(ft->t); 490 ft->t = NULL; 491 return err; 492 } 493 494 netdev_dbg(priv->netdev, "Created fs ANY table id %u level %u\n", 495 ft->t->id, ft->t->level); 496 497 err = fs_any_create_groups(ft); 498 if (err) 499 goto err; 500 501 err = fs_any_add_default_rule(priv); 502 if (err) 503 goto err; 504 505 return 0; 506 507 err: 508 mlx5e_destroy_flow_table(ft); 509 return err; 510 } 511 512 static int fs_any_disable(struct mlx5e_priv *priv) 513 { 514 int err; 515 516 /* Modify ttc rules destination to point back to the indir TIRs */ 517 err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, MLX5_TT_ANY); 518 if (err) { 519 netdev_err(priv->netdev, 520 "%s: modify ttc[%d] default destination failed, err(%d)\n", 521 __func__, MLX5_TT_ANY, err); 522 return err; 523 } 524 return 0; 525 } 526 527 static int fs_any_enable(struct mlx5e_priv *priv) 528 { 529 struct mlx5_flow_destination dest = {}; 530 int err; 531 532 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 533 dest.ft = priv->fs->any->table.t; 534 535 /* Modify ttc rules destination to point on the accel_fs FTs */ 536 err = mlx5_ttc_fwd_dest(priv->fs->ttc, MLX5_TT_ANY, &dest); 537 if (err) { 538 netdev_err(priv->netdev, 539 "%s: modify ttc[%d] destination to accel failed, err(%d)\n", 540 __func__, MLX5_TT_ANY, err); 541 return err; 542 } 543 return 0; 544 } 545 546 static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any) 547 { 548 if (IS_ERR_OR_NULL(fs_any->table.t)) 549 return; 550 551 mlx5_del_flow_rules(fs_any->default_rule); 552 mlx5e_destroy_flow_table(&fs_any->table); 553 fs_any->table.t = NULL; 554 } 555 556 void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv) 557 { 558 struct mlx5e_fs_any *fs_any = priv->fs->any; 559 560 if (!fs_any) 561 return; 562 563 if (--fs_any->ref_cnt) 564 return; 565 566 fs_any_disable(priv); 567 568 fs_any_destroy_table(fs_any); 569 570 kfree(fs_any); 571 priv->fs->any = NULL; 572 } 573 574 int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv) 575 { 576 int err; 577 578 if (priv->fs->any) { 579 priv->fs->any->ref_cnt++; 580 return 0; 581 } 582 583 priv->fs->any = kzalloc(sizeof(*priv->fs->any), GFP_KERNEL); 584 if (!priv->fs->any) 585 return -ENOMEM; 586 587 err = fs_any_create_table(priv); 588 if (err) 589 return err; 590 591 err = fs_any_enable(priv); 592 if (err) 593 goto err_destroy_table; 594 595 priv->fs->any->ref_cnt = 1; 596 597 return 0; 598 599 err_destroy_table: 600 fs_any_destroy_table(priv->fs->any); 601 602 kfree(priv->fs->any); 603 priv->fs->any = NULL; 604 return err; 605 } 606