1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "dr_types.h" 5 6 static bool dr_mask_is_smac_set(struct mlx5dr_match_spec *spec) 7 { 8 return (spec->smac_47_16 || spec->smac_15_0); 9 } 10 11 static bool dr_mask_is_dmac_set(struct mlx5dr_match_spec *spec) 12 { 13 return (spec->dmac_47_16 || spec->dmac_15_0); 14 } 15 16 static bool dr_mask_is_src_addr_set(struct mlx5dr_match_spec *spec) 17 { 18 return (spec->src_ip_127_96 || spec->src_ip_95_64 || 19 spec->src_ip_63_32 || spec->src_ip_31_0); 20 } 21 22 static bool dr_mask_is_dst_addr_set(struct mlx5dr_match_spec *spec) 23 { 24 return (spec->dst_ip_127_96 || spec->dst_ip_95_64 || 25 spec->dst_ip_63_32 || spec->dst_ip_31_0); 26 } 27 28 static bool dr_mask_is_l3_base_set(struct mlx5dr_match_spec *spec) 29 { 30 return (spec->ip_protocol || spec->frag || spec->tcp_flags || 31 spec->ip_ecn || spec->ip_dscp); 32 } 33 34 static bool dr_mask_is_tcp_udp_base_set(struct mlx5dr_match_spec *spec) 35 { 36 return (spec->tcp_sport || spec->tcp_dport || 37 spec->udp_sport || spec->udp_dport); 38 } 39 40 static bool dr_mask_is_ipv4_set(struct mlx5dr_match_spec *spec) 41 { 42 return (spec->dst_ip_31_0 || spec->src_ip_31_0); 43 } 44 45 static bool dr_mask_is_ipv4_5_tuple_set(struct mlx5dr_match_spec *spec) 46 { 47 return (dr_mask_is_l3_base_set(spec) || 48 dr_mask_is_tcp_udp_base_set(spec) || 49 dr_mask_is_ipv4_set(spec)); 50 } 51 52 static bool dr_mask_is_eth_l2_tnl_set(struct mlx5dr_match_misc *misc) 53 { 54 return misc->vxlan_vni; 55 } 56 57 static bool dr_mask_is_ttl_set(struct mlx5dr_match_spec *spec) 58 { 59 return spec->ttl_hoplimit; 60 } 61 62 #define DR_MASK_IS_L2_DST(_spec, _misc, _inner_outer) (_spec.first_vid || \ 63 (_spec).first_cfi || (_spec).first_prio || (_spec).cvlan_tag || \ 64 (_spec).svlan_tag || (_spec).dmac_47_16 || (_spec).dmac_15_0 || \ 65 (_spec).ethertype || (_spec).ip_version || \ 66 (_misc)._inner_outer##_second_vid || \ 67 (_misc)._inner_outer##_second_cfi || \ 68 (_misc)._inner_outer##_second_prio || \ 69 (_misc)._inner_outer##_second_cvlan_tag || \ 70 (_misc)._inner_outer##_second_svlan_tag) 71 72 #define DR_MASK_IS_ETH_L4_SET(_spec, _misc, _inner_outer) ( \ 73 dr_mask_is_l3_base_set(&(_spec)) || \ 74 dr_mask_is_tcp_udp_base_set(&(_spec)) || \ 75 dr_mask_is_ttl_set(&(_spec)) || \ 76 (_misc)._inner_outer##_ipv6_flow_label) 77 78 #define DR_MASK_IS_ETH_L4_MISC_SET(_misc3, _inner_outer) ( \ 79 (_misc3)._inner_outer##_tcp_seq_num || \ 80 (_misc3)._inner_outer##_tcp_ack_num) 81 82 #define DR_MASK_IS_FIRST_MPLS_SET(_misc2, _inner_outer) ( \ 83 (_misc2)._inner_outer##_first_mpls_label || \ 84 (_misc2)._inner_outer##_first_mpls_exp || \ 85 (_misc2)._inner_outer##_first_mpls_s_bos || \ 86 (_misc2)._inner_outer##_first_mpls_ttl) 87 88 static bool dr_mask_is_tnl_gre_set(struct mlx5dr_match_misc *misc) 89 { 90 return (misc->gre_key_h || misc->gre_key_l || 91 misc->gre_protocol || misc->gre_c_present || 92 misc->gre_k_present || misc->gre_s_present); 93 } 94 95 #define DR_MASK_IS_OUTER_MPLS_OVER_GRE_UDP_SET(_misc2, gre_udp) ( \ 96 (_misc2).outer_first_mpls_over_##gre_udp##_label || \ 97 (_misc2).outer_first_mpls_over_##gre_udp##_exp || \ 98 (_misc2).outer_first_mpls_over_##gre_udp##_s_bos || \ 99 (_misc2).outer_first_mpls_over_##gre_udp##_ttl) 100 101 #define DR_MASK_IS_TNL_MPLS_SET(_misc2) ( \ 102 DR_MASK_IS_OUTER_MPLS_OVER_GRE_UDP_SET((_misc2), gre) || \ 103 DR_MASK_IS_OUTER_MPLS_OVER_GRE_UDP_SET((_misc2), udp)) 104 105 static bool 106 dr_mask_is_vxlan_gpe_set(struct mlx5dr_match_misc3 *misc3) 107 { 108 return (misc3->outer_vxlan_gpe_vni || 109 misc3->outer_vxlan_gpe_next_protocol || 110 misc3->outer_vxlan_gpe_flags); 111 } 112 113 static bool 114 dr_matcher_supp_vxlan_gpe(struct mlx5dr_cmd_caps *caps) 115 { 116 return (caps->sw_format_ver == MLX5_STEERING_FORMAT_CONNECTX_6DX) || 117 (caps->flex_protocols & MLX5_FLEX_PARSER_VXLAN_GPE_ENABLED); 118 } 119 120 static bool 121 dr_mask_is_tnl_vxlan_gpe(struct mlx5dr_match_param *mask, 122 struct mlx5dr_domain *dmn) 123 { 124 return dr_mask_is_vxlan_gpe_set(&mask->misc3) && 125 dr_matcher_supp_vxlan_gpe(&dmn->info.caps); 126 } 127 128 static bool dr_mask_is_tnl_geneve_set(struct mlx5dr_match_misc *misc) 129 { 130 return misc->geneve_vni || 131 misc->geneve_oam || 132 misc->geneve_protocol_type || 133 misc->geneve_opt_len; 134 } 135 136 static bool 137 dr_matcher_supp_tnl_geneve(struct mlx5dr_cmd_caps *caps) 138 { 139 return (caps->sw_format_ver == MLX5_STEERING_FORMAT_CONNECTX_6DX) || 140 (caps->flex_protocols & MLX5_FLEX_PARSER_GENEVE_ENABLED); 141 } 142 143 static bool 144 dr_mask_is_tnl_geneve(struct mlx5dr_match_param *mask, 145 struct mlx5dr_domain *dmn) 146 { 147 return dr_mask_is_tnl_geneve_set(&mask->misc) && 148 dr_matcher_supp_tnl_geneve(&dmn->info.caps); 149 } 150 151 static int dr_matcher_supp_icmp_v4(struct mlx5dr_cmd_caps *caps) 152 { 153 return (caps->sw_format_ver == MLX5_STEERING_FORMAT_CONNECTX_6DX) || 154 (caps->flex_protocols & MLX5_FLEX_PARSER_ICMP_V4_ENABLED); 155 } 156 157 static int dr_matcher_supp_icmp_v6(struct mlx5dr_cmd_caps *caps) 158 { 159 return (caps->sw_format_ver == MLX5_STEERING_FORMAT_CONNECTX_6DX) || 160 (caps->flex_protocols & MLX5_FLEX_PARSER_ICMP_V6_ENABLED); 161 } 162 163 static bool dr_mask_is_icmpv6_set(struct mlx5dr_match_misc3 *misc3) 164 { 165 return (misc3->icmpv6_type || misc3->icmpv6_code || 166 misc3->icmpv6_header_data); 167 } 168 169 static bool dr_mask_is_icmp(struct mlx5dr_match_param *mask, 170 struct mlx5dr_domain *dmn) 171 { 172 if (DR_MASK_IS_ICMPV4_SET(&mask->misc3)) 173 return dr_matcher_supp_icmp_v4(&dmn->info.caps); 174 else if (dr_mask_is_icmpv6_set(&mask->misc3)) 175 return dr_matcher_supp_icmp_v6(&dmn->info.caps); 176 177 return false; 178 } 179 180 static bool dr_mask_is_wqe_metadata_set(struct mlx5dr_match_misc2 *misc2) 181 { 182 return misc2->metadata_reg_a; 183 } 184 185 static bool dr_mask_is_reg_c_0_3_set(struct mlx5dr_match_misc2 *misc2) 186 { 187 return (misc2->metadata_reg_c_0 || misc2->metadata_reg_c_1 || 188 misc2->metadata_reg_c_2 || misc2->metadata_reg_c_3); 189 } 190 191 static bool dr_mask_is_reg_c_4_7_set(struct mlx5dr_match_misc2 *misc2) 192 { 193 return (misc2->metadata_reg_c_4 || misc2->metadata_reg_c_5 || 194 misc2->metadata_reg_c_6 || misc2->metadata_reg_c_7); 195 } 196 197 static bool dr_mask_is_gvmi_or_qpn_set(struct mlx5dr_match_misc *misc) 198 { 199 return (misc->source_sqn || misc->source_port); 200 } 201 202 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher, 203 struct mlx5dr_matcher_rx_tx *nic_matcher, 204 enum mlx5dr_ipv outer_ipv, 205 enum mlx5dr_ipv inner_ipv) 206 { 207 nic_matcher->ste_builder = 208 nic_matcher->ste_builder_arr[outer_ipv][inner_ipv]; 209 nic_matcher->num_of_builders = 210 nic_matcher->num_of_builders_arr[outer_ipv][inner_ipv]; 211 212 if (!nic_matcher->num_of_builders) { 213 mlx5dr_dbg(matcher->tbl->dmn, 214 "Rule not supported on this matcher due to IP related fields\n"); 215 return -EINVAL; 216 } 217 218 return 0; 219 } 220 221 static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, 222 struct mlx5dr_matcher_rx_tx *nic_matcher, 223 enum mlx5dr_ipv outer_ipv, 224 enum mlx5dr_ipv inner_ipv) 225 { 226 struct mlx5dr_domain_rx_tx *nic_dmn = nic_matcher->nic_tbl->nic_dmn; 227 struct mlx5dr_domain *dmn = matcher->tbl->dmn; 228 struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; 229 struct mlx5dr_match_param mask = {}; 230 struct mlx5dr_ste_build *sb; 231 bool inner, rx; 232 int idx = 0; 233 int ret, i; 234 235 sb = nic_matcher->ste_builder_arr[outer_ipv][inner_ipv]; 236 rx = nic_dmn->ste_type == MLX5DR_STE_TYPE_RX; 237 238 /* Create a temporary mask to track and clear used mask fields */ 239 if (matcher->match_criteria & DR_MATCHER_CRITERIA_OUTER) 240 mask.outer = matcher->mask.outer; 241 242 if (matcher->match_criteria & DR_MATCHER_CRITERIA_MISC) 243 mask.misc = matcher->mask.misc; 244 245 if (matcher->match_criteria & DR_MATCHER_CRITERIA_INNER) 246 mask.inner = matcher->mask.inner; 247 248 if (matcher->match_criteria & DR_MATCHER_CRITERIA_MISC2) 249 mask.misc2 = matcher->mask.misc2; 250 251 if (matcher->match_criteria & DR_MATCHER_CRITERIA_MISC3) 252 mask.misc3 = matcher->mask.misc3; 253 254 ret = mlx5dr_ste_build_pre_check(dmn, matcher->match_criteria, 255 &matcher->mask, NULL); 256 if (ret) 257 return ret; 258 259 /* Outer */ 260 if (matcher->match_criteria & (DR_MATCHER_CRITERIA_OUTER | 261 DR_MATCHER_CRITERIA_MISC | 262 DR_MATCHER_CRITERIA_MISC2 | 263 DR_MATCHER_CRITERIA_MISC3)) { 264 inner = false; 265 266 if (dr_mask_is_wqe_metadata_set(&mask.misc2)) 267 mlx5dr_ste_build_general_purpose(ste_ctx, &sb[idx++], 268 &mask, inner, rx); 269 270 if (dr_mask_is_reg_c_0_3_set(&mask.misc2)) 271 mlx5dr_ste_build_register_0(ste_ctx, &sb[idx++], 272 &mask, inner, rx); 273 274 if (dr_mask_is_reg_c_4_7_set(&mask.misc2)) 275 mlx5dr_ste_build_register_1(ste_ctx, &sb[idx++], 276 &mask, inner, rx); 277 278 if (dr_mask_is_gvmi_or_qpn_set(&mask.misc) && 279 (dmn->type == MLX5DR_DOMAIN_TYPE_FDB || 280 dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX)) { 281 mlx5dr_ste_build_src_gvmi_qpn(ste_ctx, &sb[idx++], 282 &mask, dmn, inner, rx); 283 } 284 285 if (dr_mask_is_smac_set(&mask.outer) && 286 dr_mask_is_dmac_set(&mask.outer)) { 287 mlx5dr_ste_build_eth_l2_src_dst(ste_ctx, &sb[idx++], 288 &mask, inner, rx); 289 } 290 291 if (dr_mask_is_smac_set(&mask.outer)) 292 mlx5dr_ste_build_eth_l2_src(ste_ctx, &sb[idx++], 293 &mask, inner, rx); 294 295 if (DR_MASK_IS_L2_DST(mask.outer, mask.misc, outer)) 296 mlx5dr_ste_build_eth_l2_dst(ste_ctx, &sb[idx++], 297 &mask, inner, rx); 298 299 if (outer_ipv == DR_RULE_IPV6) { 300 if (dr_mask_is_dst_addr_set(&mask.outer)) 301 mlx5dr_ste_build_eth_l3_ipv6_dst(ste_ctx, &sb[idx++], 302 &mask, inner, rx); 303 304 if (dr_mask_is_src_addr_set(&mask.outer)) 305 mlx5dr_ste_build_eth_l3_ipv6_src(ste_ctx, &sb[idx++], 306 &mask, inner, rx); 307 308 if (DR_MASK_IS_ETH_L4_SET(mask.outer, mask.misc, outer)) 309 mlx5dr_ste_build_eth_ipv6_l3_l4(ste_ctx, &sb[idx++], 310 &mask, inner, rx); 311 } else { 312 if (dr_mask_is_ipv4_5_tuple_set(&mask.outer)) 313 mlx5dr_ste_build_eth_l3_ipv4_5_tuple(ste_ctx, &sb[idx++], 314 &mask, inner, rx); 315 316 if (dr_mask_is_ttl_set(&mask.outer)) 317 mlx5dr_ste_build_eth_l3_ipv4_misc(ste_ctx, &sb[idx++], 318 &mask, inner, rx); 319 } 320 321 if (dr_mask_is_tnl_vxlan_gpe(&mask, dmn)) 322 mlx5dr_ste_build_tnl_vxlan_gpe(ste_ctx, &sb[idx++], 323 &mask, inner, rx); 324 else if (dr_mask_is_tnl_geneve(&mask, dmn)) 325 mlx5dr_ste_build_tnl_geneve(ste_ctx, &sb[idx++], 326 &mask, inner, rx); 327 328 if (DR_MASK_IS_ETH_L4_MISC_SET(mask.misc3, outer)) 329 mlx5dr_ste_build_eth_l4_misc(ste_ctx, &sb[idx++], 330 &mask, inner, rx); 331 332 if (DR_MASK_IS_FIRST_MPLS_SET(mask.misc2, outer)) 333 mlx5dr_ste_build_mpls(ste_ctx, &sb[idx++], 334 &mask, inner, rx); 335 336 if (DR_MASK_IS_TNL_MPLS_SET(mask.misc2)) 337 mlx5dr_ste_build_tnl_mpls(ste_ctx, &sb[idx++], 338 &mask, inner, rx); 339 340 if (dr_mask_is_icmp(&mask, dmn)) { 341 ret = mlx5dr_ste_build_icmp(ste_ctx, &sb[idx++], 342 &mask, &dmn->info.caps, 343 inner, rx); 344 if (ret) 345 return ret; 346 } 347 if (dr_mask_is_tnl_gre_set(&mask.misc)) 348 mlx5dr_ste_build_tnl_gre(ste_ctx, &sb[idx++], 349 &mask, inner, rx); 350 } 351 352 /* Inner */ 353 if (matcher->match_criteria & (DR_MATCHER_CRITERIA_INNER | 354 DR_MATCHER_CRITERIA_MISC | 355 DR_MATCHER_CRITERIA_MISC2 | 356 DR_MATCHER_CRITERIA_MISC3)) { 357 inner = true; 358 359 if (dr_mask_is_eth_l2_tnl_set(&mask.misc)) 360 mlx5dr_ste_build_eth_l2_tnl(ste_ctx, &sb[idx++], 361 &mask, inner, rx); 362 363 if (dr_mask_is_smac_set(&mask.inner) && 364 dr_mask_is_dmac_set(&mask.inner)) { 365 mlx5dr_ste_build_eth_l2_src_dst(ste_ctx, &sb[idx++], 366 &mask, inner, rx); 367 } 368 369 if (dr_mask_is_smac_set(&mask.inner)) 370 mlx5dr_ste_build_eth_l2_src(ste_ctx, &sb[idx++], 371 &mask, inner, rx); 372 373 if (DR_MASK_IS_L2_DST(mask.inner, mask.misc, inner)) 374 mlx5dr_ste_build_eth_l2_dst(ste_ctx, &sb[idx++], 375 &mask, inner, rx); 376 377 if (inner_ipv == DR_RULE_IPV6) { 378 if (dr_mask_is_dst_addr_set(&mask.inner)) 379 mlx5dr_ste_build_eth_l3_ipv6_dst(ste_ctx, &sb[idx++], 380 &mask, inner, rx); 381 382 if (dr_mask_is_src_addr_set(&mask.inner)) 383 mlx5dr_ste_build_eth_l3_ipv6_src(ste_ctx, &sb[idx++], 384 &mask, inner, rx); 385 386 if (DR_MASK_IS_ETH_L4_SET(mask.inner, mask.misc, inner)) 387 mlx5dr_ste_build_eth_ipv6_l3_l4(ste_ctx, &sb[idx++], 388 &mask, inner, rx); 389 } else { 390 if (dr_mask_is_ipv4_5_tuple_set(&mask.inner)) 391 mlx5dr_ste_build_eth_l3_ipv4_5_tuple(ste_ctx, &sb[idx++], 392 &mask, inner, rx); 393 394 if (dr_mask_is_ttl_set(&mask.inner)) 395 mlx5dr_ste_build_eth_l3_ipv4_misc(ste_ctx, &sb[idx++], 396 &mask, inner, rx); 397 } 398 399 if (DR_MASK_IS_ETH_L4_MISC_SET(mask.misc3, inner)) 400 mlx5dr_ste_build_eth_l4_misc(ste_ctx, &sb[idx++], 401 &mask, inner, rx); 402 403 if (DR_MASK_IS_FIRST_MPLS_SET(mask.misc2, inner)) 404 mlx5dr_ste_build_mpls(ste_ctx, &sb[idx++], 405 &mask, inner, rx); 406 407 if (DR_MASK_IS_TNL_MPLS_SET(mask.misc2)) 408 mlx5dr_ste_build_tnl_mpls(ste_ctx, &sb[idx++], 409 &mask, inner, rx); 410 } 411 /* Empty matcher, takes all */ 412 if (matcher->match_criteria == DR_MATCHER_CRITERIA_EMPTY) 413 mlx5dr_ste_build_empty_always_hit(&sb[idx++], rx); 414 415 if (idx == 0) { 416 mlx5dr_err(dmn, "Cannot generate any valid rules from mask\n"); 417 return -EINVAL; 418 } 419 420 /* Check that all mask fields were consumed */ 421 for (i = 0; i < sizeof(struct mlx5dr_match_param); i++) { 422 if (((u8 *)&mask)[i] != 0) { 423 mlx5dr_dbg(dmn, "Mask contains unsupported parameters\n"); 424 return -EOPNOTSUPP; 425 } 426 } 427 428 nic_matcher->ste_builder = sb; 429 nic_matcher->num_of_builders_arr[outer_ipv][inner_ipv] = idx; 430 431 return 0; 432 } 433 434 static int dr_matcher_connect(struct mlx5dr_domain *dmn, 435 struct mlx5dr_matcher_rx_tx *curr_nic_matcher, 436 struct mlx5dr_matcher_rx_tx *next_nic_matcher, 437 struct mlx5dr_matcher_rx_tx *prev_nic_matcher) 438 { 439 struct mlx5dr_table_rx_tx *nic_tbl = curr_nic_matcher->nic_tbl; 440 struct mlx5dr_domain_rx_tx *nic_dmn = nic_tbl->nic_dmn; 441 struct mlx5dr_htbl_connect_info info; 442 struct mlx5dr_ste_htbl *prev_htbl; 443 int ret; 444 445 /* Connect end anchor hash table to next_htbl or to the default address */ 446 if (next_nic_matcher) { 447 info.type = CONNECT_HIT; 448 info.hit_next_htbl = next_nic_matcher->s_htbl; 449 } else { 450 info.type = CONNECT_MISS; 451 info.miss_icm_addr = nic_tbl->default_icm_addr; 452 } 453 ret = mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, 454 curr_nic_matcher->e_anchor, 455 &info, info.type == CONNECT_HIT); 456 if (ret) 457 return ret; 458 459 /* Connect start hash table to end anchor */ 460 info.type = CONNECT_MISS; 461 info.miss_icm_addr = curr_nic_matcher->e_anchor->chunk->icm_addr; 462 ret = mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, 463 curr_nic_matcher->s_htbl, 464 &info, false); 465 if (ret) 466 return ret; 467 468 /* Connect previous hash table to matcher start hash table */ 469 if (prev_nic_matcher) 470 prev_htbl = prev_nic_matcher->e_anchor; 471 else 472 prev_htbl = nic_tbl->s_anchor; 473 474 info.type = CONNECT_HIT; 475 info.hit_next_htbl = curr_nic_matcher->s_htbl; 476 ret = mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, prev_htbl, 477 &info, true); 478 if (ret) 479 return ret; 480 481 /* Update the pointing ste and next hash table */ 482 curr_nic_matcher->s_htbl->pointing_ste = prev_htbl->ste_arr; 483 prev_htbl->ste_arr[0].next_htbl = curr_nic_matcher->s_htbl; 484 485 if (next_nic_matcher) { 486 next_nic_matcher->s_htbl->pointing_ste = curr_nic_matcher->e_anchor->ste_arr; 487 curr_nic_matcher->e_anchor->ste_arr[0].next_htbl = next_nic_matcher->s_htbl; 488 } 489 490 return 0; 491 } 492 493 static int dr_matcher_add_to_tbl(struct mlx5dr_matcher *matcher) 494 { 495 struct mlx5dr_matcher *next_matcher, *prev_matcher, *tmp_matcher; 496 struct mlx5dr_table *tbl = matcher->tbl; 497 struct mlx5dr_domain *dmn = tbl->dmn; 498 bool first = true; 499 int ret; 500 501 next_matcher = NULL; 502 list_for_each_entry(tmp_matcher, &tbl->matcher_list, matcher_list) { 503 if (tmp_matcher->prio >= matcher->prio) { 504 next_matcher = tmp_matcher; 505 break; 506 } 507 first = false; 508 } 509 510 prev_matcher = NULL; 511 if (next_matcher && !first) 512 prev_matcher = list_prev_entry(next_matcher, matcher_list); 513 else if (!first) 514 prev_matcher = list_last_entry(&tbl->matcher_list, 515 struct mlx5dr_matcher, 516 matcher_list); 517 518 if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB || 519 dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX) { 520 ret = dr_matcher_connect(dmn, &matcher->rx, 521 next_matcher ? &next_matcher->rx : NULL, 522 prev_matcher ? &prev_matcher->rx : NULL); 523 if (ret) 524 return ret; 525 } 526 527 if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB || 528 dmn->type == MLX5DR_DOMAIN_TYPE_NIC_TX) { 529 ret = dr_matcher_connect(dmn, &matcher->tx, 530 next_matcher ? &next_matcher->tx : NULL, 531 prev_matcher ? &prev_matcher->tx : NULL); 532 if (ret) 533 return ret; 534 } 535 536 if (prev_matcher) 537 list_add(&matcher->matcher_list, &prev_matcher->matcher_list); 538 else if (next_matcher) 539 list_add_tail(&matcher->matcher_list, 540 &next_matcher->matcher_list); 541 else 542 list_add(&matcher->matcher_list, &tbl->matcher_list); 543 544 return 0; 545 } 546 547 static void dr_matcher_uninit_nic(struct mlx5dr_matcher_rx_tx *nic_matcher) 548 { 549 mlx5dr_htbl_put(nic_matcher->s_htbl); 550 mlx5dr_htbl_put(nic_matcher->e_anchor); 551 } 552 553 static void dr_matcher_uninit_fdb(struct mlx5dr_matcher *matcher) 554 { 555 dr_matcher_uninit_nic(&matcher->rx); 556 dr_matcher_uninit_nic(&matcher->tx); 557 } 558 559 static void dr_matcher_uninit(struct mlx5dr_matcher *matcher) 560 { 561 struct mlx5dr_domain *dmn = matcher->tbl->dmn; 562 563 switch (dmn->type) { 564 case MLX5DR_DOMAIN_TYPE_NIC_RX: 565 dr_matcher_uninit_nic(&matcher->rx); 566 break; 567 case MLX5DR_DOMAIN_TYPE_NIC_TX: 568 dr_matcher_uninit_nic(&matcher->tx); 569 break; 570 case MLX5DR_DOMAIN_TYPE_FDB: 571 dr_matcher_uninit_fdb(matcher); 572 break; 573 default: 574 WARN_ON(true); 575 break; 576 } 577 } 578 579 static int dr_matcher_set_all_ste_builders(struct mlx5dr_matcher *matcher, 580 struct mlx5dr_matcher_rx_tx *nic_matcher) 581 { 582 struct mlx5dr_domain *dmn = matcher->tbl->dmn; 583 584 dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV4, DR_RULE_IPV4); 585 dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV4, DR_RULE_IPV6); 586 dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV6, DR_RULE_IPV4); 587 dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV6, DR_RULE_IPV6); 588 589 if (!nic_matcher->ste_builder) { 590 mlx5dr_err(dmn, "Cannot generate IPv4 or IPv6 rules with given mask\n"); 591 return -EINVAL; 592 } 593 594 return 0; 595 } 596 597 static int dr_matcher_init_nic(struct mlx5dr_matcher *matcher, 598 struct mlx5dr_matcher_rx_tx *nic_matcher) 599 { 600 struct mlx5dr_domain *dmn = matcher->tbl->dmn; 601 int ret; 602 603 ret = dr_matcher_set_all_ste_builders(matcher, nic_matcher); 604 if (ret) 605 return ret; 606 607 nic_matcher->e_anchor = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool, 608 DR_CHUNK_SIZE_1, 609 MLX5DR_STE_LU_TYPE_DONT_CARE, 610 0); 611 if (!nic_matcher->e_anchor) 612 return -ENOMEM; 613 614 nic_matcher->s_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool, 615 DR_CHUNK_SIZE_1, 616 nic_matcher->ste_builder[0].lu_type, 617 nic_matcher->ste_builder[0].byte_mask); 618 if (!nic_matcher->s_htbl) { 619 ret = -ENOMEM; 620 goto free_e_htbl; 621 } 622 623 /* make sure the tables exist while empty */ 624 mlx5dr_htbl_get(nic_matcher->s_htbl); 625 mlx5dr_htbl_get(nic_matcher->e_anchor); 626 627 return 0; 628 629 free_e_htbl: 630 mlx5dr_ste_htbl_free(nic_matcher->e_anchor); 631 return ret; 632 } 633 634 static int dr_matcher_init_fdb(struct mlx5dr_matcher *matcher) 635 { 636 int ret; 637 638 ret = dr_matcher_init_nic(matcher, &matcher->rx); 639 if (ret) 640 return ret; 641 642 ret = dr_matcher_init_nic(matcher, &matcher->tx); 643 if (ret) 644 goto uninit_nic_rx; 645 646 return 0; 647 648 uninit_nic_rx: 649 dr_matcher_uninit_nic(&matcher->rx); 650 return ret; 651 } 652 653 static int dr_matcher_init(struct mlx5dr_matcher *matcher, 654 struct mlx5dr_match_parameters *mask) 655 { 656 struct mlx5dr_table *tbl = matcher->tbl; 657 struct mlx5dr_domain *dmn = tbl->dmn; 658 int ret; 659 660 if (matcher->match_criteria >= DR_MATCHER_CRITERIA_MAX) { 661 mlx5dr_err(dmn, "Invalid match criteria attribute\n"); 662 return -EINVAL; 663 } 664 665 if (mask) { 666 if (mask->match_sz > DR_SZ_MATCH_PARAM) { 667 mlx5dr_err(dmn, "Invalid match size attribute\n"); 668 return -EINVAL; 669 } 670 mlx5dr_ste_copy_param(matcher->match_criteria, 671 &matcher->mask, mask); 672 } 673 674 switch (dmn->type) { 675 case MLX5DR_DOMAIN_TYPE_NIC_RX: 676 matcher->rx.nic_tbl = &tbl->rx; 677 ret = dr_matcher_init_nic(matcher, &matcher->rx); 678 break; 679 case MLX5DR_DOMAIN_TYPE_NIC_TX: 680 matcher->tx.nic_tbl = &tbl->tx; 681 ret = dr_matcher_init_nic(matcher, &matcher->tx); 682 break; 683 case MLX5DR_DOMAIN_TYPE_FDB: 684 matcher->rx.nic_tbl = &tbl->rx; 685 matcher->tx.nic_tbl = &tbl->tx; 686 ret = dr_matcher_init_fdb(matcher); 687 break; 688 default: 689 WARN_ON(true); 690 return -EINVAL; 691 } 692 693 return ret; 694 } 695 696 struct mlx5dr_matcher * 697 mlx5dr_matcher_create(struct mlx5dr_table *tbl, 698 u32 priority, 699 u8 match_criteria_enable, 700 struct mlx5dr_match_parameters *mask) 701 { 702 struct mlx5dr_matcher *matcher; 703 int ret; 704 705 refcount_inc(&tbl->refcount); 706 707 matcher = kzalloc(sizeof(*matcher), GFP_KERNEL); 708 if (!matcher) 709 goto dec_ref; 710 711 matcher->tbl = tbl; 712 matcher->prio = priority; 713 matcher->match_criteria = match_criteria_enable; 714 refcount_set(&matcher->refcount, 1); 715 INIT_LIST_HEAD(&matcher->matcher_list); 716 717 mlx5dr_domain_lock(tbl->dmn); 718 719 ret = dr_matcher_init(matcher, mask); 720 if (ret) 721 goto free_matcher; 722 723 ret = dr_matcher_add_to_tbl(matcher); 724 if (ret) 725 goto matcher_uninit; 726 727 mlx5dr_domain_unlock(tbl->dmn); 728 729 return matcher; 730 731 matcher_uninit: 732 dr_matcher_uninit(matcher); 733 free_matcher: 734 mlx5dr_domain_unlock(tbl->dmn); 735 kfree(matcher); 736 dec_ref: 737 refcount_dec(&tbl->refcount); 738 return NULL; 739 } 740 741 static int dr_matcher_disconnect(struct mlx5dr_domain *dmn, 742 struct mlx5dr_table_rx_tx *nic_tbl, 743 struct mlx5dr_matcher_rx_tx *next_nic_matcher, 744 struct mlx5dr_matcher_rx_tx *prev_nic_matcher) 745 { 746 struct mlx5dr_domain_rx_tx *nic_dmn = nic_tbl->nic_dmn; 747 struct mlx5dr_htbl_connect_info info; 748 struct mlx5dr_ste_htbl *prev_anchor; 749 750 if (prev_nic_matcher) 751 prev_anchor = prev_nic_matcher->e_anchor; 752 else 753 prev_anchor = nic_tbl->s_anchor; 754 755 /* Connect previous anchor hash table to next matcher or to the default address */ 756 if (next_nic_matcher) { 757 info.type = CONNECT_HIT; 758 info.hit_next_htbl = next_nic_matcher->s_htbl; 759 next_nic_matcher->s_htbl->pointing_ste = prev_anchor->ste_arr; 760 prev_anchor->ste_arr[0].next_htbl = next_nic_matcher->s_htbl; 761 } else { 762 info.type = CONNECT_MISS; 763 info.miss_icm_addr = nic_tbl->default_icm_addr; 764 prev_anchor->ste_arr[0].next_htbl = NULL; 765 } 766 767 return mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, prev_anchor, 768 &info, true); 769 } 770 771 static int dr_matcher_remove_from_tbl(struct mlx5dr_matcher *matcher) 772 { 773 struct mlx5dr_matcher *prev_matcher, *next_matcher; 774 struct mlx5dr_table *tbl = matcher->tbl; 775 struct mlx5dr_domain *dmn = tbl->dmn; 776 int ret = 0; 777 778 if (list_is_last(&matcher->matcher_list, &tbl->matcher_list)) 779 next_matcher = NULL; 780 else 781 next_matcher = list_next_entry(matcher, matcher_list); 782 783 if (matcher->matcher_list.prev == &tbl->matcher_list) 784 prev_matcher = NULL; 785 else 786 prev_matcher = list_prev_entry(matcher, matcher_list); 787 788 if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB || 789 dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX) { 790 ret = dr_matcher_disconnect(dmn, &tbl->rx, 791 next_matcher ? &next_matcher->rx : NULL, 792 prev_matcher ? &prev_matcher->rx : NULL); 793 if (ret) 794 return ret; 795 } 796 797 if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB || 798 dmn->type == MLX5DR_DOMAIN_TYPE_NIC_TX) { 799 ret = dr_matcher_disconnect(dmn, &tbl->tx, 800 next_matcher ? &next_matcher->tx : NULL, 801 prev_matcher ? &prev_matcher->tx : NULL); 802 if (ret) 803 return ret; 804 } 805 806 list_del(&matcher->matcher_list); 807 808 return 0; 809 } 810 811 int mlx5dr_matcher_destroy(struct mlx5dr_matcher *matcher) 812 { 813 struct mlx5dr_table *tbl = matcher->tbl; 814 815 if (refcount_read(&matcher->refcount) > 1) 816 return -EBUSY; 817 818 mlx5dr_domain_lock(tbl->dmn); 819 820 dr_matcher_remove_from_tbl(matcher); 821 dr_matcher_uninit(matcher); 822 refcount_dec(&matcher->tbl->refcount); 823 824 mlx5dr_domain_unlock(tbl->dmn); 825 kfree(matcher); 826 827 return 0; 828 } 829