1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2020 Mellanox Technologies. 3 4 #include <linux/mlx5/driver.h> 5 #include <linux/mlx5/mlx5_ifc.h> 6 #include <linux/mlx5/fs.h> 7 8 #include "lib/fs_chains.h" 9 #include "en/mapping.h" 10 #include "mlx5_core.h" 11 #include "fs_core.h" 12 #include "eswitch.h" 13 #include "en.h" 14 #include "en_tc.h" 15 16 #define chains_lock(chains) ((chains)->lock) 17 #define chains_ht(chains) ((chains)->chains_ht) 18 #define chains_mapping(chains) ((chains)->chains_mapping) 19 #define prios_ht(chains) ((chains)->prios_ht) 20 #define ft_pool_left(chains) ((chains)->ft_left) 21 #define tc_default_ft(chains) ((chains)->tc_default_ft) 22 #define tc_end_ft(chains) ((chains)->tc_end_ft) 23 #define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \ 24 FDB_TC_OFFLOAD : MLX5E_TC_PRIO) 25 26 /* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS), 27 * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated 28 * for each flow table pool. We can allocate up to 16M of each pool, 29 * and we keep track of how much we used via get_next_avail_sz_from_pool. 30 * Firmware doesn't report any of this for now. 31 * ESW_POOL is expected to be sorted from large to small and match firmware 32 * pools. 33 */ 34 #define FT_SIZE (16 * 1024 * 1024) 35 static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024, 36 1 * 1024 * 1024, 37 64 * 1024, 38 128 }; 39 #define FT_TBL_SZ (64 * 1024) 40 41 struct mlx5_fs_chains { 42 struct mlx5_core_dev *dev; 43 44 struct rhashtable chains_ht; 45 struct rhashtable prios_ht; 46 /* Protects above chains_ht and prios_ht */ 47 struct mutex lock; 48 49 struct mlx5_flow_table *tc_default_ft; 50 struct mlx5_flow_table *tc_end_ft; 51 struct mapping_ctx *chains_mapping; 52 53 enum mlx5_flow_namespace_type ns; 54 u32 group_num; 55 u32 flags; 56 57 int ft_left[ARRAY_SIZE(FT_POOLS)]; 58 }; 59 60 struct fs_chain { 61 struct rhash_head node; 62 63 u32 chain; 64 65 int ref; 66 int id; 67 68 struct mlx5_fs_chains *chains; 69 struct list_head prios_list; 70 struct mlx5_flow_handle *restore_rule; 71 struct mlx5_modify_hdr *miss_modify_hdr; 72 }; 73 74 struct prio_key { 75 u32 chain; 76 u32 prio; 77 u32 level; 78 }; 79 80 struct prio { 81 struct rhash_head node; 82 struct list_head list; 83 84 struct prio_key key; 85 86 int ref; 87 88 struct fs_chain *chain; 89 struct mlx5_flow_table *ft; 90 struct mlx5_flow_table *next_ft; 91 struct mlx5_flow_group *miss_group; 92 struct mlx5_flow_handle *miss_rule; 93 }; 94 95 static const struct rhashtable_params chain_params = { 96 .head_offset = offsetof(struct fs_chain, node), 97 .key_offset = offsetof(struct fs_chain, chain), 98 .key_len = sizeof_field(struct fs_chain, chain), 99 .automatic_shrinking = true, 100 }; 101 102 static const struct rhashtable_params prio_params = { 103 .head_offset = offsetof(struct prio, node), 104 .key_offset = offsetof(struct prio, key), 105 .key_len = sizeof_field(struct prio, key), 106 .automatic_shrinking = true, 107 }; 108 109 bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains) 110 { 111 return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED; 112 } 113 114 static bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains) 115 { 116 return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 117 } 118 119 bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains) 120 { 121 return mlx5_chains_prios_supported(chains) && 122 mlx5_chains_ignore_flow_level_supported(chains); 123 } 124 125 u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains) 126 { 127 if (!mlx5_chains_prios_supported(chains)) 128 return 1; 129 130 if (mlx5_chains_ignore_flow_level_supported(chains)) 131 return UINT_MAX - 1; 132 133 /* We should get here only for eswitch case */ 134 return FDB_TC_MAX_CHAIN; 135 } 136 137 u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains) 138 { 139 return mlx5_chains_get_chain_range(chains) + 1; 140 } 141 142 u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains) 143 { 144 if (mlx5_chains_ignore_flow_level_supported(chains)) 145 return UINT_MAX; 146 147 /* We should get here only for eswitch case */ 148 return FDB_TC_MAX_PRIO; 149 } 150 151 static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains) 152 { 153 if (mlx5_chains_ignore_flow_level_supported(chains)) 154 return UINT_MAX; 155 156 /* Same value for FDB and NIC RX tables */ 157 return FDB_TC_LEVELS_PER_PRIO; 158 } 159 160 void 161 mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains, 162 struct mlx5_flow_table *ft) 163 { 164 tc_end_ft(chains) = ft; 165 } 166 167 #define POOL_NEXT_SIZE 0 168 static int 169 mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains *chains, 170 int desired_size) 171 { 172 int i, found_i = -1; 173 174 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { 175 if (ft_pool_left(chains)[i] && FT_POOLS[i] > desired_size) { 176 found_i = i; 177 if (desired_size != POOL_NEXT_SIZE) 178 break; 179 } 180 } 181 182 if (found_i != -1) { 183 --ft_pool_left(chains)[found_i]; 184 return FT_POOLS[found_i]; 185 } 186 187 return 0; 188 } 189 190 static void 191 mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains *chains, int sz) 192 { 193 int i; 194 195 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { 196 if (sz == FT_POOLS[i]) { 197 ++ft_pool_left(chains)[i]; 198 return; 199 } 200 } 201 202 WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz); 203 } 204 205 static void 206 mlx5_chains_init_sz_pool(struct mlx5_fs_chains *chains, u32 ft_max) 207 { 208 int i; 209 210 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) 211 ft_pool_left(chains)[i] = 212 FT_POOLS[i] <= ft_max ? FT_SIZE / FT_POOLS[i] : 0; 213 } 214 215 static struct mlx5_flow_table * 216 mlx5_chains_create_table(struct mlx5_fs_chains *chains, 217 u32 chain, u32 prio, u32 level) 218 { 219 struct mlx5_flow_table_attr ft_attr = {}; 220 struct mlx5_flow_namespace *ns; 221 struct mlx5_flow_table *ft; 222 int sz; 223 224 if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED) 225 ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 226 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 227 228 sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? 229 mlx5_chains_get_avail_sz_from_pool(chains, FT_TBL_SZ) : 230 mlx5_chains_get_avail_sz_from_pool(chains, POOL_NEXT_SIZE); 231 if (!sz) 232 return ERR_PTR(-ENOSPC); 233 ft_attr.max_fte = sz; 234 235 /* We use tc_default_ft(chains) as the table's next_ft till 236 * ignore_flow_level is allowed on FT creation and not just for FTEs. 237 * Instead caller should add an explicit miss rule if needed. 238 */ 239 ft_attr.next_ft = tc_default_ft(chains); 240 241 /* The root table(chain 0, prio 1, level 0) is required to be 242 * connected to the previous fs_core managed prio. 243 * We always create it, as a managed table, in order to align with 244 * fs_core logic. 245 */ 246 if (!mlx5_chains_ignore_flow_level_supported(chains) || 247 (chain == 0 && prio == 1 && level == 0)) { 248 ft_attr.level = level; 249 ft_attr.prio = prio - 1; 250 ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ? 251 mlx5_get_fdb_sub_ns(chains->dev, chain) : 252 mlx5_get_flow_namespace(chains->dev, chains->ns); 253 } else { 254 ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED; 255 ft_attr.prio = ns_to_chains_fs_prio(chains->ns); 256 /* Firmware doesn't allow us to create another level 0 table, 257 * so we create all unmanaged tables as level 1. 258 * 259 * To connect them, we use explicit miss rules with 260 * ignore_flow_level. Caller is responsible to create 261 * these rules (if needed). 262 */ 263 ft_attr.level = 1; 264 ns = mlx5_get_flow_namespace(chains->dev, chains->ns); 265 } 266 267 ft_attr.autogroup.num_reserved_entries = 2; 268 ft_attr.autogroup.max_num_groups = chains->group_num; 269 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 270 if (IS_ERR(ft)) { 271 mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n", 272 (int)PTR_ERR(ft), chain, prio, level, sz); 273 mlx5_chains_put_sz_to_pool(chains, sz); 274 return ft; 275 } 276 277 return ft; 278 } 279 280 static void 281 mlx5_chains_destroy_table(struct mlx5_fs_chains *chains, 282 struct mlx5_flow_table *ft) 283 { 284 mlx5_chains_put_sz_to_pool(chains, ft->max_fte); 285 mlx5_destroy_flow_table(ft); 286 } 287 288 static int 289 create_chain_restore(struct fs_chain *chain) 290 { 291 struct mlx5_eswitch *esw = chain->chains->dev->priv.eswitch; 292 char modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)]; 293 struct mlx5_fs_chains *chains = chain->chains; 294 enum mlx5e_tc_attr_to_reg chain_to_reg; 295 struct mlx5_modify_hdr *mod_hdr; 296 u32 index; 297 int err; 298 299 if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) || 300 !mlx5_chains_prios_supported(chains)) 301 return 0; 302 303 err = mapping_add(chains_mapping(chains), &chain->chain, &index); 304 if (err) 305 return err; 306 if (index == MLX5_FS_DEFAULT_FLOW_TAG) { 307 /* we got the special default flow tag id, so we won't know 308 * if we actually marked the packet with the restore rule 309 * we create. 310 * 311 * This case isn't possible with MLX5_FS_DEFAULT_FLOW_TAG = 0. 312 */ 313 err = mapping_add(chains_mapping(chains), 314 &chain->chain, &index); 315 mapping_remove(chains_mapping(chains), 316 MLX5_FS_DEFAULT_FLOW_TAG); 317 if (err) 318 return err; 319 } 320 321 chain->id = index; 322 323 if (chains->ns == MLX5_FLOW_NAMESPACE_FDB) { 324 chain_to_reg = CHAIN_TO_REG; 325 chain->restore_rule = esw_add_restore_rule(esw, chain->id); 326 if (IS_ERR(chain->restore_rule)) { 327 err = PTR_ERR(chain->restore_rule); 328 goto err_rule; 329 } 330 } else if (chains->ns == MLX5_FLOW_NAMESPACE_KERNEL) { 331 /* For NIC RX we don't need a restore rule 332 * since we write the metadata to reg_b 333 * that is passed to SW directly. 334 */ 335 chain_to_reg = NIC_CHAIN_TO_REG; 336 } else { 337 err = -EINVAL; 338 goto err_rule; 339 } 340 341 MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET); 342 MLX5_SET(set_action_in, modact, field, 343 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield); 344 MLX5_SET(set_action_in, modact, offset, 345 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset * 8); 346 MLX5_SET(set_action_in, modact, length, 347 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen * 8); 348 MLX5_SET(set_action_in, modact, data, chain->id); 349 mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns, 350 1, modact); 351 if (IS_ERR(mod_hdr)) { 352 err = PTR_ERR(mod_hdr); 353 goto err_mod_hdr; 354 } 355 chain->miss_modify_hdr = mod_hdr; 356 357 return 0; 358 359 err_mod_hdr: 360 if (!IS_ERR_OR_NULL(chain->restore_rule)) 361 mlx5_del_flow_rules(chain->restore_rule); 362 err_rule: 363 /* Datapath can't find this mapping, so we can safely remove it */ 364 mapping_remove(chains_mapping(chains), chain->id); 365 return err; 366 } 367 368 static void destroy_chain_restore(struct fs_chain *chain) 369 { 370 struct mlx5_fs_chains *chains = chain->chains; 371 372 if (!chain->miss_modify_hdr) 373 return; 374 375 if (chain->restore_rule) 376 mlx5_del_flow_rules(chain->restore_rule); 377 378 mlx5_modify_header_dealloc(chains->dev, chain->miss_modify_hdr); 379 mapping_remove(chains_mapping(chains), chain->id); 380 } 381 382 static struct fs_chain * 383 mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain) 384 { 385 struct fs_chain *chain_s = NULL; 386 int err; 387 388 chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL); 389 if (!chain_s) 390 return ERR_PTR(-ENOMEM); 391 392 chain_s->chains = chains; 393 chain_s->chain = chain; 394 INIT_LIST_HEAD(&chain_s->prios_list); 395 396 err = create_chain_restore(chain_s); 397 if (err) 398 goto err_restore; 399 400 err = rhashtable_insert_fast(&chains_ht(chains), &chain_s->node, 401 chain_params); 402 if (err) 403 goto err_insert; 404 405 return chain_s; 406 407 err_insert: 408 destroy_chain_restore(chain_s); 409 err_restore: 410 kvfree(chain_s); 411 return ERR_PTR(err); 412 } 413 414 static void 415 mlx5_chains_destroy_chain(struct fs_chain *chain) 416 { 417 struct mlx5_fs_chains *chains = chain->chains; 418 419 rhashtable_remove_fast(&chains_ht(chains), &chain->node, 420 chain_params); 421 422 destroy_chain_restore(chain); 423 kvfree(chain); 424 } 425 426 static struct fs_chain * 427 mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain) 428 { 429 struct fs_chain *chain_s; 430 431 chain_s = rhashtable_lookup_fast(&chains_ht(chains), &chain, 432 chain_params); 433 if (!chain_s) { 434 chain_s = mlx5_chains_create_chain(chains, chain); 435 if (IS_ERR(chain_s)) 436 return chain_s; 437 } 438 439 chain_s->ref++; 440 441 return chain_s; 442 } 443 444 static struct mlx5_flow_handle * 445 mlx5_chains_add_miss_rule(struct fs_chain *chain, 446 struct mlx5_flow_table *ft, 447 struct mlx5_flow_table *next_ft) 448 { 449 struct mlx5_fs_chains *chains = chain->chains; 450 struct mlx5_flow_destination dest = {}; 451 struct mlx5_flow_act act = {}; 452 453 act.flags = FLOW_ACT_NO_APPEND; 454 if (mlx5_chains_ignore_flow_level_supported(chain->chains)) 455 act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 456 457 act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 458 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 459 dest.ft = next_ft; 460 461 if (next_ft == tc_end_ft(chains) && 462 chain->chain != mlx5_chains_get_nf_ft_chain(chains) && 463 mlx5_chains_prios_supported(chains)) { 464 act.modify_hdr = chain->miss_modify_hdr; 465 act.action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 466 } 467 468 return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1); 469 } 470 471 static int 472 mlx5_chains_update_prio_prevs(struct prio *prio, 473 struct mlx5_flow_table *next_ft) 474 { 475 struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {}; 476 struct fs_chain *chain = prio->chain; 477 struct prio *pos; 478 int n = 0, err; 479 480 if (prio->key.level) 481 return 0; 482 483 /* Iterate in reverse order until reaching the level 0 rule of 484 * the previous priority, adding all the miss rules first, so we can 485 * revert them if any of them fails. 486 */ 487 pos = prio; 488 list_for_each_entry_continue_reverse(pos, 489 &chain->prios_list, 490 list) { 491 miss_rules[n] = mlx5_chains_add_miss_rule(chain, 492 pos->ft, 493 next_ft); 494 if (IS_ERR(miss_rules[n])) { 495 err = PTR_ERR(miss_rules[n]); 496 goto err_prev_rule; 497 } 498 499 n++; 500 if (!pos->key.level) 501 break; 502 } 503 504 /* Success, delete old miss rules, and update the pointers. */ 505 n = 0; 506 pos = prio; 507 list_for_each_entry_continue_reverse(pos, 508 &chain->prios_list, 509 list) { 510 mlx5_del_flow_rules(pos->miss_rule); 511 512 pos->miss_rule = miss_rules[n]; 513 pos->next_ft = next_ft; 514 515 n++; 516 if (!pos->key.level) 517 break; 518 } 519 520 return 0; 521 522 err_prev_rule: 523 while (--n >= 0) 524 mlx5_del_flow_rules(miss_rules[n]); 525 526 return err; 527 } 528 529 static void 530 mlx5_chains_put_chain(struct fs_chain *chain) 531 { 532 if (--chain->ref == 0) 533 mlx5_chains_destroy_chain(chain); 534 } 535 536 static struct prio * 537 mlx5_chains_create_prio(struct mlx5_fs_chains *chains, 538 u32 chain, u32 prio, u32 level) 539 { 540 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 541 struct mlx5_flow_handle *miss_rule; 542 struct mlx5_flow_group *miss_group; 543 struct mlx5_flow_table *next_ft; 544 struct mlx5_flow_table *ft; 545 struct fs_chain *chain_s; 546 struct list_head *pos; 547 struct prio *prio_s; 548 u32 *flow_group_in; 549 int err; 550 551 chain_s = mlx5_chains_get_chain(chains, chain); 552 if (IS_ERR(chain_s)) 553 return ERR_CAST(chain_s); 554 555 prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL); 556 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 557 if (!prio_s || !flow_group_in) { 558 err = -ENOMEM; 559 goto err_alloc; 560 } 561 562 /* Chain's prio list is sorted by prio and level. 563 * And all levels of some prio point to the next prio's level 0. 564 * Example list (prio, level): 565 * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0) 566 * In hardware, we will we have the following pointers: 567 * (3,0) -> (5,0) -> (7,0) -> Slow path 568 * (3,1) -> (5,0) 569 * (5,1) -> (7,0) 570 * (6,1) -> (7,0) 571 */ 572 573 /* Default miss for each chain: */ 574 next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? 575 tc_default_ft(chains) : 576 tc_end_ft(chains); 577 list_for_each(pos, &chain_s->prios_list) { 578 struct prio *p = list_entry(pos, struct prio, list); 579 580 /* exit on first pos that is larger */ 581 if (prio < p->key.prio || (prio == p->key.prio && 582 level < p->key.level)) { 583 /* Get next level 0 table */ 584 next_ft = p->key.level == 0 ? p->ft : p->next_ft; 585 break; 586 } 587 } 588 589 ft = mlx5_chains_create_table(chains, chain, prio, level); 590 if (IS_ERR(ft)) { 591 err = PTR_ERR(ft); 592 goto err_create; 593 } 594 595 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 596 ft->max_fte - 2); 597 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 598 ft->max_fte - 1); 599 miss_group = mlx5_create_flow_group(ft, flow_group_in); 600 if (IS_ERR(miss_group)) { 601 err = PTR_ERR(miss_group); 602 goto err_group; 603 } 604 605 /* Add miss rule to next_ft */ 606 miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft); 607 if (IS_ERR(miss_rule)) { 608 err = PTR_ERR(miss_rule); 609 goto err_miss_rule; 610 } 611 612 prio_s->miss_group = miss_group; 613 prio_s->miss_rule = miss_rule; 614 prio_s->next_ft = next_ft; 615 prio_s->chain = chain_s; 616 prio_s->key.chain = chain; 617 prio_s->key.prio = prio; 618 prio_s->key.level = level; 619 prio_s->ft = ft; 620 621 err = rhashtable_insert_fast(&prios_ht(chains), &prio_s->node, 622 prio_params); 623 if (err) 624 goto err_insert; 625 626 list_add(&prio_s->list, pos->prev); 627 628 /* Table is ready, connect it */ 629 err = mlx5_chains_update_prio_prevs(prio_s, ft); 630 if (err) 631 goto err_update; 632 633 kvfree(flow_group_in); 634 return prio_s; 635 636 err_update: 637 list_del(&prio_s->list); 638 rhashtable_remove_fast(&prios_ht(chains), &prio_s->node, 639 prio_params); 640 err_insert: 641 mlx5_del_flow_rules(miss_rule); 642 err_miss_rule: 643 mlx5_destroy_flow_group(miss_group); 644 err_group: 645 mlx5_chains_destroy_table(chains, ft); 646 err_create: 647 err_alloc: 648 kvfree(prio_s); 649 kvfree(flow_group_in); 650 mlx5_chains_put_chain(chain_s); 651 return ERR_PTR(err); 652 } 653 654 static void 655 mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains, 656 struct prio *prio) 657 { 658 struct fs_chain *chain = prio->chain; 659 660 WARN_ON(mlx5_chains_update_prio_prevs(prio, 661 prio->next_ft)); 662 663 list_del(&prio->list); 664 rhashtable_remove_fast(&prios_ht(chains), &prio->node, 665 prio_params); 666 mlx5_del_flow_rules(prio->miss_rule); 667 mlx5_destroy_flow_group(prio->miss_group); 668 mlx5_chains_destroy_table(chains, prio->ft); 669 mlx5_chains_put_chain(chain); 670 kvfree(prio); 671 } 672 673 struct mlx5_flow_table * 674 mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, 675 u32 level) 676 { 677 struct mlx5_flow_table *prev_fts; 678 struct prio *prio_s; 679 struct prio_key key; 680 int l = 0; 681 682 if ((chain > mlx5_chains_get_chain_range(chains) && 683 chain != mlx5_chains_get_nf_ft_chain(chains)) || 684 prio > mlx5_chains_get_prio_range(chains) || 685 level > mlx5_chains_get_level_range(chains)) 686 return ERR_PTR(-EOPNOTSUPP); 687 688 /* create earlier levels for correct fs_core lookup when 689 * connecting tables. 690 */ 691 for (l = 0; l < level; l++) { 692 prev_fts = mlx5_chains_get_table(chains, chain, prio, l); 693 if (IS_ERR(prev_fts)) { 694 prio_s = ERR_CAST(prev_fts); 695 goto err_get_prevs; 696 } 697 } 698 699 key.chain = chain; 700 key.prio = prio; 701 key.level = level; 702 703 mutex_lock(&chains_lock(chains)); 704 prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key, 705 prio_params); 706 if (!prio_s) { 707 prio_s = mlx5_chains_create_prio(chains, chain, 708 prio, level); 709 if (IS_ERR(prio_s)) 710 goto err_create_prio; 711 } 712 713 ++prio_s->ref; 714 mutex_unlock(&chains_lock(chains)); 715 716 return prio_s->ft; 717 718 err_create_prio: 719 mutex_unlock(&chains_lock(chains)); 720 err_get_prevs: 721 while (--l >= 0) 722 mlx5_chains_put_table(chains, chain, prio, l); 723 return ERR_CAST(prio_s); 724 } 725 726 void 727 mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, 728 u32 level) 729 { 730 struct prio *prio_s; 731 struct prio_key key; 732 733 key.chain = chain; 734 key.prio = prio; 735 key.level = level; 736 737 mutex_lock(&chains_lock(chains)); 738 prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key, 739 prio_params); 740 if (!prio_s) 741 goto err_get_prio; 742 743 if (--prio_s->ref == 0) 744 mlx5_chains_destroy_prio(chains, prio_s); 745 mutex_unlock(&chains_lock(chains)); 746 747 while (level-- > 0) 748 mlx5_chains_put_table(chains, chain, prio, level); 749 750 return; 751 752 err_get_prio: 753 mutex_unlock(&chains_lock(chains)); 754 WARN_ONCE(1, 755 "Couldn't find table: (chain: %d prio: %d level: %d)", 756 chain, prio, level); 757 } 758 759 struct mlx5_flow_table * 760 mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains) 761 { 762 return tc_end_ft(chains); 763 } 764 765 struct mlx5_flow_table * 766 mlx5_chains_create_global_table(struct mlx5_fs_chains *chains) 767 { 768 u32 chain, prio, level; 769 int err; 770 771 if (!mlx5_chains_ignore_flow_level_supported(chains)) { 772 err = -EOPNOTSUPP; 773 774 mlx5_core_warn(chains->dev, 775 "Couldn't create global flow table, ignore_flow_level not supported."); 776 goto err_ignore; 777 } 778 779 chain = mlx5_chains_get_chain_range(chains), 780 prio = mlx5_chains_get_prio_range(chains); 781 level = mlx5_chains_get_level_range(chains); 782 783 return mlx5_chains_create_table(chains, chain, prio, level); 784 785 err_ignore: 786 return ERR_PTR(err); 787 } 788 789 void 790 mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains, 791 struct mlx5_flow_table *ft) 792 { 793 mlx5_chains_destroy_table(chains, ft); 794 } 795 796 static struct mlx5_fs_chains * 797 mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr) 798 { 799 struct mlx5_fs_chains *chains_priv; 800 struct mapping_ctx *mapping; 801 u32 max_flow_counter; 802 int err; 803 804 chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL); 805 if (!chains_priv) 806 return ERR_PTR(-ENOMEM); 807 808 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) | 809 MLX5_CAP_GEN(dev, max_flow_counter_15_0); 810 811 mlx5_core_dbg(dev, 812 "Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n", 813 max_flow_counter, attr->max_grp_num, attr->max_ft_sz); 814 815 chains_priv->dev = dev; 816 chains_priv->flags = attr->flags; 817 chains_priv->ns = attr->ns; 818 chains_priv->group_num = attr->max_grp_num; 819 tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft; 820 821 mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n", 822 mlx5_chains_get_chain_range(chains_priv), 823 mlx5_chains_get_prio_range(chains_priv)); 824 825 mlx5_chains_init_sz_pool(chains_priv, attr->max_ft_sz); 826 827 err = rhashtable_init(&chains_ht(chains_priv), &chain_params); 828 if (err) 829 goto init_chains_ht_err; 830 831 err = rhashtable_init(&prios_ht(chains_priv), &prio_params); 832 if (err) 833 goto init_prios_ht_err; 834 835 mapping = mapping_create(sizeof(u32), attr->max_restore_tag, 836 true); 837 if (IS_ERR(mapping)) { 838 err = PTR_ERR(mapping); 839 goto mapping_err; 840 } 841 chains_mapping(chains_priv) = mapping; 842 843 mutex_init(&chains_lock(chains_priv)); 844 845 return chains_priv; 846 847 mapping_err: 848 rhashtable_destroy(&prios_ht(chains_priv)); 849 init_prios_ht_err: 850 rhashtable_destroy(&chains_ht(chains_priv)); 851 init_chains_ht_err: 852 kfree(chains_priv); 853 return ERR_PTR(err); 854 } 855 856 static void 857 mlx5_chains_cleanup(struct mlx5_fs_chains *chains) 858 { 859 mutex_destroy(&chains_lock(chains)); 860 mapping_destroy(chains_mapping(chains)); 861 rhashtable_destroy(&prios_ht(chains)); 862 rhashtable_destroy(&chains_ht(chains)); 863 864 kfree(chains); 865 } 866 867 struct mlx5_fs_chains * 868 mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr) 869 { 870 struct mlx5_fs_chains *chains; 871 872 chains = mlx5_chains_init(dev, attr); 873 874 return chains; 875 } 876 877 void 878 mlx5_chains_destroy(struct mlx5_fs_chains *chains) 879 { 880 mlx5_chains_cleanup(chains); 881 } 882 883 int 884 mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain, 885 u32 *chain_mapping) 886 { 887 return mapping_add(chains_mapping(chains), &chain, chain_mapping); 888 } 889 890 int 891 mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping) 892 { 893 return mapping_remove(chains_mapping(chains), chain_mapping); 894 } 895 896 int mlx5_get_chain_for_tag(struct mlx5_fs_chains *chains, u32 tag, 897 u32 *chain) 898 { 899 int err; 900 901 err = mapping_find(chains_mapping(chains), tag, chain); 902 if (err) { 903 mlx5_core_warn(chains->dev, "Can't find chain for tag: %d\n", tag); 904 return -ENOENT; 905 } 906 907 return 0; 908 } 909