1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2019, Mellanox Technologies */ 3 4 #ifndef _DR_TYPES_ 5 #define _DR_TYPES_ 6 7 #include <linux/mlx5/vport.h> 8 #include <linux/refcount.h> 9 #include "fs_core.h" 10 #include "wq.h" 11 #include "lib/mlx5.h" 12 #include "mlx5_ifc_dr.h" 13 #include "mlx5dr.h" 14 #include "dr_dbg.h" 15 16 #define DR_RULE_MAX_STES 18 17 #define DR_ACTION_MAX_STES 5 18 #define DR_STE_SVLAN 0x1 19 #define DR_STE_CVLAN 0x2 20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4) 21 #define DR_NUM_OF_FLEX_PARSERS 8 22 #define DR_STE_MAX_FLEX_0_ID 3 23 #define DR_STE_MAX_FLEX_1_ID 7 24 25 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg) 26 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg) 27 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg) 28 29 static inline bool dr_is_flex_parser_0_id(u8 parser_id) 30 { 31 return parser_id <= DR_STE_MAX_FLEX_0_ID; 32 } 33 34 static inline bool dr_is_flex_parser_1_id(u8 parser_id) 35 { 36 return parser_id > DR_STE_MAX_FLEX_0_ID; 37 } 38 39 enum mlx5dr_icm_chunk_size { 40 DR_CHUNK_SIZE_1, 41 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */ 42 DR_CHUNK_SIZE_2, 43 DR_CHUNK_SIZE_4, 44 DR_CHUNK_SIZE_8, 45 DR_CHUNK_SIZE_16, 46 DR_CHUNK_SIZE_32, 47 DR_CHUNK_SIZE_64, 48 DR_CHUNK_SIZE_128, 49 DR_CHUNK_SIZE_256, 50 DR_CHUNK_SIZE_512, 51 DR_CHUNK_SIZE_1K, 52 DR_CHUNK_SIZE_2K, 53 DR_CHUNK_SIZE_4K, 54 DR_CHUNK_SIZE_8K, 55 DR_CHUNK_SIZE_16K, 56 DR_CHUNK_SIZE_32K, 57 DR_CHUNK_SIZE_64K, 58 DR_CHUNK_SIZE_128K, 59 DR_CHUNK_SIZE_256K, 60 DR_CHUNK_SIZE_512K, 61 DR_CHUNK_SIZE_1024K, 62 DR_CHUNK_SIZE_2048K, 63 DR_CHUNK_SIZE_MAX, 64 }; 65 66 enum mlx5dr_icm_type { 67 DR_ICM_TYPE_STE, 68 DR_ICM_TYPE_MODIFY_ACTION, 69 }; 70 71 static inline enum mlx5dr_icm_chunk_size 72 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk) 73 { 74 chunk += 2; 75 if (chunk < DR_CHUNK_SIZE_MAX) 76 return chunk; 77 78 return DR_CHUNK_SIZE_MAX; 79 } 80 81 enum { 82 DR_STE_SIZE = 64, 83 DR_STE_SIZE_CTRL = 32, 84 DR_STE_SIZE_TAG = 16, 85 DR_STE_SIZE_MASK = 16, 86 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK, 87 }; 88 89 enum mlx5dr_ste_ctx_action_cap { 90 DR_STE_CTX_ACTION_CAP_NONE = 0, 91 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0, 92 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1, 93 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2, 94 }; 95 96 enum { 97 DR_MODIFY_ACTION_SIZE = 8, 98 }; 99 100 enum mlx5dr_matcher_criteria { 101 DR_MATCHER_CRITERIA_EMPTY = 0, 102 DR_MATCHER_CRITERIA_OUTER = 1 << 0, 103 DR_MATCHER_CRITERIA_MISC = 1 << 1, 104 DR_MATCHER_CRITERIA_INNER = 1 << 2, 105 DR_MATCHER_CRITERIA_MISC2 = 1 << 3, 106 DR_MATCHER_CRITERIA_MISC3 = 1 << 4, 107 DR_MATCHER_CRITERIA_MISC4 = 1 << 5, 108 DR_MATCHER_CRITERIA_MISC5 = 1 << 6, 109 DR_MATCHER_CRITERIA_MAX = 1 << 7, 110 }; 111 112 enum mlx5dr_action_type { 113 DR_ACTION_TYP_TNL_L2_TO_L2, 114 DR_ACTION_TYP_L2_TO_TNL_L2, 115 DR_ACTION_TYP_TNL_L3_TO_L2, 116 DR_ACTION_TYP_L2_TO_TNL_L3, 117 DR_ACTION_TYP_DROP, 118 DR_ACTION_TYP_QP, 119 DR_ACTION_TYP_FT, 120 DR_ACTION_TYP_CTR, 121 DR_ACTION_TYP_TAG, 122 DR_ACTION_TYP_MODIFY_HDR, 123 DR_ACTION_TYP_VPORT, 124 DR_ACTION_TYP_POP_VLAN, 125 DR_ACTION_TYP_PUSH_VLAN, 126 DR_ACTION_TYP_INSERT_HDR, 127 DR_ACTION_TYP_REMOVE_HDR, 128 DR_ACTION_TYP_SAMPLER, 129 DR_ACTION_TYP_MAX, 130 }; 131 132 enum mlx5dr_ipv { 133 DR_RULE_IPV4, 134 DR_RULE_IPV6, 135 DR_RULE_IPV_MAX, 136 }; 137 138 struct mlx5dr_icm_pool; 139 struct mlx5dr_icm_chunk; 140 struct mlx5dr_icm_buddy_mem; 141 struct mlx5dr_ste_htbl; 142 struct mlx5dr_match_param; 143 struct mlx5dr_cmd_caps; 144 struct mlx5dr_rule_rx_tx; 145 struct mlx5dr_matcher_rx_tx; 146 struct mlx5dr_ste_ctx; 147 148 struct mlx5dr_ste { 149 u8 *hw_ste; 150 /* refcount: indicates the num of rules that using this ste */ 151 u32 refcount; 152 153 /* attached to the miss_list head at each htbl entry */ 154 struct list_head miss_list_node; 155 156 /* this ste is member of htbl */ 157 struct mlx5dr_ste_htbl *htbl; 158 159 struct mlx5dr_ste_htbl *next_htbl; 160 161 /* The rule this STE belongs to */ 162 struct mlx5dr_rule_rx_tx *rule_rx_tx; 163 164 /* this ste is part of a rule, located in ste's chain */ 165 u8 ste_chain_location; 166 }; 167 168 struct mlx5dr_ste_htbl_ctrl { 169 /* total number of valid entries belonging to this hash table. This 170 * includes the non collision and collision entries 171 */ 172 unsigned int num_of_valid_entries; 173 174 /* total number of collisions entries attached to this table */ 175 unsigned int num_of_collisions; 176 }; 177 178 struct mlx5dr_ste_htbl { 179 u16 lu_type; 180 u16 byte_mask; 181 u32 refcount; 182 struct mlx5dr_icm_chunk *chunk; 183 struct mlx5dr_ste *ste_arr; 184 u8 *hw_ste_arr; 185 186 struct list_head *miss_list; 187 188 enum mlx5dr_icm_chunk_size chunk_size; 189 struct mlx5dr_ste *pointing_ste; 190 191 struct mlx5dr_ste_htbl_ctrl ctrl; 192 }; 193 194 struct mlx5dr_ste_send_info { 195 struct mlx5dr_ste *ste; 196 struct list_head send_list; 197 u16 size; 198 u16 offset; 199 u8 data_cont[DR_STE_SIZE]; 200 u8 *data; 201 }; 202 203 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size, 204 u16 offset, u8 *data, 205 struct mlx5dr_ste_send_info *ste_info, 206 struct list_head *send_list, 207 bool copy_data); 208 209 struct mlx5dr_ste_build { 210 u8 inner:1; 211 u8 rx:1; 212 u8 vhca_id_valid:1; 213 struct mlx5dr_domain *dmn; 214 struct mlx5dr_cmd_caps *caps; 215 u16 lu_type; 216 u16 byte_mask; 217 u8 bit_mask[DR_STE_SIZE_MASK]; 218 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec, 219 struct mlx5dr_ste_build *sb, 220 u8 *tag); 221 }; 222 223 struct mlx5dr_ste_htbl * 224 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool, 225 enum mlx5dr_icm_chunk_size chunk_size, 226 u16 lu_type, u16 byte_mask); 227 228 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl); 229 230 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl) 231 { 232 htbl->refcount--; 233 if (!htbl->refcount) 234 mlx5dr_ste_htbl_free(htbl); 235 } 236 237 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl) 238 { 239 htbl->refcount++; 240 } 241 242 /* STE utils */ 243 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl); 244 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, 245 u8 *hw_ste, u64 miss_addr); 246 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx, 247 u8 *hw_ste, u64 icm_addr, u32 ht_size); 248 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx, 249 u8 *hw_ste, 250 struct mlx5dr_ste_htbl *next_htbl); 251 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask); 252 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher, 253 u8 ste_location); 254 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste); 255 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste); 256 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste); 257 258 #define MLX5DR_MAX_VLANS 2 259 260 struct mlx5dr_ste_actions_attr { 261 u32 modify_index; 262 u16 modify_actions; 263 u32 decap_index; 264 u16 decap_actions; 265 u8 decap_with_vlan:1; 266 u64 final_icm_addr; 267 u32 flow_tag; 268 u32 ctr_id; 269 u16 gvmi; 270 u16 hit_gvmi; 271 struct { 272 u32 id; 273 u32 size; 274 u8 param_0; 275 u8 param_1; 276 } reformat; 277 struct { 278 int count; 279 u32 headers[MLX5DR_MAX_VLANS]; 280 } vlans; 281 }; 282 283 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, 284 struct mlx5dr_domain *dmn, 285 u8 *action_type_set, 286 u8 *last_ste, 287 struct mlx5dr_ste_actions_attr *attr, 288 u32 *added_stes); 289 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx, 290 struct mlx5dr_domain *dmn, 291 u8 *action_type_set, 292 u8 *last_ste, 293 struct mlx5dr_ste_actions_attr *attr, 294 u32 *added_stes); 295 296 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx, 297 __be64 *hw_action, 298 u8 hw_field, 299 u8 shifter, 300 u8 length, 301 u32 data); 302 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx, 303 __be64 *hw_action, 304 u8 hw_field, 305 u8 shifter, 306 u8 length, 307 u32 data); 308 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx, 309 __be64 *hw_action, 310 u8 dst_hw_field, 311 u8 dst_shifter, 312 u8 dst_len, 313 u8 src_hw_field, 314 u8 src_shifter); 315 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx, 316 void *data, 317 u32 data_sz, 318 u8 *hw_action, 319 u32 hw_action_sz, 320 u16 *used_hw_action_num); 321 322 const struct mlx5dr_ste_action_modify_field * 323 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field); 324 325 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version); 326 void mlx5dr_ste_free(struct mlx5dr_ste *ste, 327 struct mlx5dr_matcher *matcher, 328 struct mlx5dr_matcher_rx_tx *nic_matcher); 329 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste, 330 struct mlx5dr_matcher *matcher, 331 struct mlx5dr_matcher_rx_tx *nic_matcher) 332 { 333 ste->refcount--; 334 if (!ste->refcount) 335 mlx5dr_ste_free(ste, matcher, nic_matcher); 336 } 337 338 /* initial as 0, increased only when ste appears in a new rule */ 339 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste) 340 { 341 ste->refcount++; 342 } 343 344 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste) 345 { 346 return !ste->refcount; 347 } 348 349 bool mlx5dr_ste_equal_tag(void *src, void *dst); 350 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher, 351 struct mlx5dr_matcher_rx_tx *nic_matcher, 352 struct mlx5dr_ste *ste, 353 u8 *cur_hw_ste, 354 enum mlx5dr_icm_chunk_size log_table_size); 355 356 /* STE build functions */ 357 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn, 358 u8 match_criteria, 359 struct mlx5dr_match_param *mask, 360 struct mlx5dr_match_param *value); 361 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, 362 struct mlx5dr_matcher_rx_tx *nic_matcher, 363 struct mlx5dr_match_param *value, 364 u8 *ste_arr); 365 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx, 366 struct mlx5dr_ste_build *builder, 367 struct mlx5dr_match_param *mask, 368 bool inner, bool rx); 369 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx, 370 struct mlx5dr_ste_build *sb, 371 struct mlx5dr_match_param *mask, 372 bool inner, bool rx); 373 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx, 374 struct mlx5dr_ste_build *sb, 375 struct mlx5dr_match_param *mask, 376 bool inner, bool rx); 377 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx, 378 struct mlx5dr_ste_build *sb, 379 struct mlx5dr_match_param *mask, 380 bool inner, bool rx); 381 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx, 382 struct mlx5dr_ste_build *sb, 383 struct mlx5dr_match_param *mask, 384 bool inner, bool rx); 385 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx, 386 struct mlx5dr_ste_build *sb, 387 struct mlx5dr_match_param *mask, 388 bool inner, bool rx); 389 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx, 390 struct mlx5dr_ste_build *sb, 391 struct mlx5dr_match_param *mask, 392 bool inner, bool rx); 393 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx, 394 struct mlx5dr_ste_build *sb, 395 struct mlx5dr_match_param *mask, 396 bool inner, bool rx); 397 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx, 398 struct mlx5dr_ste_build *sb, 399 struct mlx5dr_match_param *mask, 400 bool inner, bool rx); 401 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx, 402 struct mlx5dr_ste_build *sb, 403 struct mlx5dr_match_param *mask, 404 bool inner, bool rx); 405 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx, 406 struct mlx5dr_ste_build *sb, 407 struct mlx5dr_match_param *mask, 408 bool inner, bool rx); 409 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx, 410 struct mlx5dr_ste_build *sb, 411 struct mlx5dr_match_param *mask, 412 bool inner, bool rx); 413 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx, 414 struct mlx5dr_ste_build *sb, 415 struct mlx5dr_match_param *mask, 416 bool inner, bool rx); 417 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx, 418 struct mlx5dr_ste_build *sb, 419 struct mlx5dr_match_param *mask, 420 struct mlx5dr_cmd_caps *caps, 421 bool inner, bool rx); 422 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx, 423 struct mlx5dr_ste_build *sb, 424 struct mlx5dr_match_param *mask, 425 struct mlx5dr_cmd_caps *caps, 426 bool inner, bool rx); 427 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx, 428 struct mlx5dr_ste_build *sb, 429 struct mlx5dr_match_param *mask, 430 struct mlx5dr_cmd_caps *caps, 431 bool inner, bool rx); 432 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx, 433 struct mlx5dr_ste_build *sb, 434 struct mlx5dr_match_param *mask, 435 bool inner, bool rx); 436 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx, 437 struct mlx5dr_ste_build *sb, 438 struct mlx5dr_match_param *mask, 439 bool inner, bool rx); 440 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx, 441 struct mlx5dr_ste_build *sb, 442 struct mlx5dr_match_param *mask, 443 struct mlx5dr_cmd_caps *caps, 444 bool inner, bool rx); 445 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx, 446 struct mlx5dr_ste_build *sb, 447 struct mlx5dr_match_param *mask, 448 struct mlx5dr_cmd_caps *caps, 449 bool inner, bool rx); 450 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx, 451 struct mlx5dr_ste_build *sb, 452 struct mlx5dr_match_param *mask, 453 bool inner, bool rx); 454 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 455 struct mlx5dr_ste_build *sb, 456 struct mlx5dr_match_param *mask, 457 struct mlx5dr_cmd_caps *caps, 458 bool inner, bool rx); 459 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 460 struct mlx5dr_ste_build *sb, 461 struct mlx5dr_match_param *mask, 462 struct mlx5dr_cmd_caps *caps, 463 bool inner, bool rx); 464 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx, 465 struct mlx5dr_ste_build *sb, 466 struct mlx5dr_match_param *mask, 467 bool inner, bool rx); 468 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx, 469 struct mlx5dr_ste_build *sb, 470 struct mlx5dr_match_param *mask, 471 bool inner, bool rx); 472 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx, 473 struct mlx5dr_ste_build *sb, 474 struct mlx5dr_match_param *mask, 475 bool inner, bool rx); 476 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx, 477 struct mlx5dr_ste_build *sb, 478 struct mlx5dr_match_param *mask, 479 bool inner, bool rx); 480 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx, 481 struct mlx5dr_ste_build *sb, 482 struct mlx5dr_match_param *mask, 483 struct mlx5dr_domain *dmn, 484 bool inner, bool rx); 485 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 486 struct mlx5dr_ste_build *sb, 487 struct mlx5dr_match_param *mask, 488 bool inner, bool rx); 489 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 490 struct mlx5dr_ste_build *sb, 491 struct mlx5dr_match_param *mask, 492 bool inner, bool rx); 493 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx); 494 495 /* Actions utils */ 496 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, 497 struct mlx5dr_matcher_rx_tx *nic_matcher, 498 struct mlx5dr_action *actions[], 499 u32 num_actions, 500 u8 *ste_arr, 501 u32 *new_hw_ste_arr_sz); 502 503 struct mlx5dr_match_spec { 504 u32 smac_47_16; /* Source MAC address of incoming packet */ 505 /* Incoming packet Ethertype - this is the Ethertype 506 * following the last VLAN tag of the packet 507 */ 508 u32 smac_15_0:16; /* Source MAC address of incoming packet */ 509 u32 ethertype:16; 510 511 u32 dmac_47_16; /* Destination MAC address of incoming packet */ 512 513 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */ 514 /* Priority of first VLAN tag in the incoming packet. 515 * Valid only when cvlan_tag==1 or svlan_tag==1 516 */ 517 u32 first_prio:3; 518 /* CFI bit of first VLAN tag in the incoming packet. 519 * Valid only when cvlan_tag==1 or svlan_tag==1 520 */ 521 u32 first_cfi:1; 522 /* VLAN ID of first VLAN tag in the incoming packet. 523 * Valid only when cvlan_tag==1 or svlan_tag==1 524 */ 525 u32 first_vid:12; 526 527 u32 ip_protocol:8; /* IP protocol */ 528 /* Differentiated Services Code Point derived from 529 * Traffic Class/TOS field of IPv6/v4 530 */ 531 u32 ip_dscp:6; 532 /* Explicit Congestion Notification derived from 533 * Traffic Class/TOS field of IPv6/v4 534 */ 535 u32 ip_ecn:2; 536 /* The first vlan in the packet is c-vlan (0x8100). 537 * cvlan_tag and svlan_tag cannot be set together 538 */ 539 u32 cvlan_tag:1; 540 /* The first vlan in the packet is s-vlan (0x8a88). 541 * cvlan_tag and svlan_tag cannot be set together 542 */ 543 u32 svlan_tag:1; 544 u32 frag:1; /* Packet is an IP fragment */ 545 u32 ip_version:4; /* IP version */ 546 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK; 547 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS 548 */ 549 u32 tcp_flags:9; 550 551 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */ 552 u32 tcp_sport:16; 553 /* TCP destination port. 554 * tcp and udp sport/dport are mutually exclusive 555 */ 556 u32 tcp_dport:16; 557 558 u32 reserved_auto1:24; 559 u32 ttl_hoplimit:8; 560 561 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */ 562 u32 udp_sport:16; 563 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */ 564 u32 udp_dport:16; 565 566 /* IPv6 source address of incoming packets 567 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 568 * This field should be qualified by an appropriate ethertype 569 */ 570 u32 src_ip_127_96; 571 /* IPv6 source address of incoming packets 572 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 573 * This field should be qualified by an appropriate ethertype 574 */ 575 u32 src_ip_95_64; 576 /* IPv6 source address of incoming packets 577 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 578 * This field should be qualified by an appropriate ethertype 579 */ 580 u32 src_ip_63_32; 581 /* IPv6 source address of incoming packets 582 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 583 * This field should be qualified by an appropriate ethertype 584 */ 585 u32 src_ip_31_0; 586 /* IPv6 destination address of incoming packets 587 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 588 * This field should be qualified by an appropriate ethertype 589 */ 590 u32 dst_ip_127_96; 591 /* IPv6 destination address of incoming packets 592 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 593 * This field should be qualified by an appropriate ethertype 594 */ 595 u32 dst_ip_95_64; 596 /* IPv6 destination address of incoming packets 597 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 598 * This field should be qualified by an appropriate ethertype 599 */ 600 u32 dst_ip_63_32; 601 /* IPv6 destination address of incoming packets 602 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 603 * This field should be qualified by an appropriate ethertype 604 */ 605 u32 dst_ip_31_0; 606 }; 607 608 struct mlx5dr_match_misc { 609 /* used with GRE, checksum exist when gre_c_present == 1 */ 610 u32 gre_c_present:1; 611 u32 reserved_auto1:1; 612 /* used with GRE, key exist when gre_k_present == 1 */ 613 u32 gre_k_present:1; 614 /* used with GRE, sequence number exist when gre_s_present == 1 */ 615 u32 gre_s_present:1; 616 u32 source_vhca_port:4; 617 u32 source_sqn:24; /* Source SQN */ 618 619 u32 source_eswitch_owner_vhca_id:16; 620 /* Source port.;0xffff determines wire port */ 621 u32 source_port:16; 622 623 /* Priority of second VLAN tag in the outer header of the incoming packet. 624 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 625 */ 626 u32 outer_second_prio:3; 627 /* CFI bit of first VLAN tag in the outer header of the incoming packet. 628 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 629 */ 630 u32 outer_second_cfi:1; 631 /* VLAN ID of first VLAN tag the outer header of the incoming packet. 632 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 633 */ 634 u32 outer_second_vid:12; 635 /* Priority of second VLAN tag in the inner header of the incoming packet. 636 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 637 */ 638 u32 inner_second_prio:3; 639 /* CFI bit of first VLAN tag in the inner header of the incoming packet. 640 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 641 */ 642 u32 inner_second_cfi:1; 643 /* VLAN ID of first VLAN tag the inner header of the incoming packet. 644 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 645 */ 646 u32 inner_second_vid:12; 647 648 u32 outer_second_cvlan_tag:1; 649 u32 inner_second_cvlan_tag:1; 650 /* The second vlan in the outer header of the packet is c-vlan (0x8100). 651 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 652 */ 653 u32 outer_second_svlan_tag:1; 654 /* The second vlan in the inner header of the packet is c-vlan (0x8100). 655 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 656 */ 657 u32 inner_second_svlan_tag:1; 658 /* The second vlan in the outer header of the packet is s-vlan (0x8a88). 659 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 660 */ 661 u32 reserved_auto2:12; 662 /* The second vlan in the inner header of the packet is s-vlan (0x8a88). 663 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 664 */ 665 u32 gre_protocol:16; /* GRE Protocol (outer) */ 666 667 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */ 668 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */ 669 670 u32 vxlan_vni:24; /* VXLAN VNI (outer) */ 671 u32 reserved_auto3:8; 672 673 u32 geneve_vni:24; /* GENEVE VNI field (outer) */ 674 u32 reserved_auto4:6; 675 u32 geneve_tlv_option_0_exist:1; 676 u32 geneve_oam:1; /* GENEVE OAM field (outer) */ 677 678 u32 reserved_auto5:12; 679 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */ 680 681 u32 reserved_auto6:12; 682 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */ 683 684 u32 reserved_auto7:10; 685 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */ 686 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */ 687 688 u32 reserved_auto8:8; 689 u32 bth_dst_qp:24; /* Destination QP in BTH header */ 690 691 u32 reserved_auto9; 692 u32 outer_esp_spi; 693 u32 reserved_auto10[3]; 694 }; 695 696 struct mlx5dr_match_misc2 { 697 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */ 698 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */ 699 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */ 700 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */ 701 702 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */ 703 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */ 704 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */ 705 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */ 706 707 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */ 708 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */ 709 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */ 710 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */ 711 712 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */ 713 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */ 714 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */ 715 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */ 716 717 u32 metadata_reg_c_7; /* metadata_reg_c_7 */ 718 u32 metadata_reg_c_6; /* metadata_reg_c_6 */ 719 u32 metadata_reg_c_5; /* metadata_reg_c_5 */ 720 u32 metadata_reg_c_4; /* metadata_reg_c_4 */ 721 u32 metadata_reg_c_3; /* metadata_reg_c_3 */ 722 u32 metadata_reg_c_2; /* metadata_reg_c_2 */ 723 u32 metadata_reg_c_1; /* metadata_reg_c_1 */ 724 u32 metadata_reg_c_0; /* metadata_reg_c_0 */ 725 u32 metadata_reg_a; /* metadata_reg_a */ 726 u32 reserved_auto1[3]; 727 }; 728 729 struct mlx5dr_match_misc3 { 730 u32 inner_tcp_seq_num; 731 u32 outer_tcp_seq_num; 732 u32 inner_tcp_ack_num; 733 u32 outer_tcp_ack_num; 734 735 u32 reserved_auto1:8; 736 u32 outer_vxlan_gpe_vni:24; 737 738 u32 outer_vxlan_gpe_next_protocol:8; 739 u32 outer_vxlan_gpe_flags:8; 740 u32 reserved_auto2:16; 741 742 u32 icmpv4_header_data; 743 u32 icmpv6_header_data; 744 745 u8 icmpv4_type; 746 u8 icmpv4_code; 747 u8 icmpv6_type; 748 u8 icmpv6_code; 749 750 u32 geneve_tlv_option_0_data; 751 752 u32 gtpu_teid; 753 754 u8 gtpu_msg_type; 755 u8 gtpu_msg_flags; 756 u32 reserved_auto3:16; 757 758 u32 gtpu_dw_2; 759 u32 gtpu_first_ext_dw_0; 760 u32 gtpu_dw_0; 761 u32 reserved_auto4; 762 }; 763 764 struct mlx5dr_match_misc4 { 765 u32 prog_sample_field_value_0; 766 u32 prog_sample_field_id_0; 767 u32 prog_sample_field_value_1; 768 u32 prog_sample_field_id_1; 769 u32 prog_sample_field_value_2; 770 u32 prog_sample_field_id_2; 771 u32 prog_sample_field_value_3; 772 u32 prog_sample_field_id_3; 773 u32 reserved_auto1[8]; 774 }; 775 776 struct mlx5dr_match_misc5 { 777 u32 macsec_tag_0; 778 u32 macsec_tag_1; 779 u32 macsec_tag_2; 780 u32 macsec_tag_3; 781 u32 tunnel_header_0; 782 u32 tunnel_header_1; 783 u32 tunnel_header_2; 784 u32 tunnel_header_3; 785 }; 786 787 struct mlx5dr_match_param { 788 struct mlx5dr_match_spec outer; 789 struct mlx5dr_match_misc misc; 790 struct mlx5dr_match_spec inner; 791 struct mlx5dr_match_misc2 misc2; 792 struct mlx5dr_match_misc3 misc3; 793 struct mlx5dr_match_misc4 misc4; 794 struct mlx5dr_match_misc5 misc5; 795 }; 796 797 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \ 798 (_misc3)->icmpv4_code || \ 799 (_misc3)->icmpv4_header_data) 800 801 struct mlx5dr_esw_caps { 802 u64 drop_icm_address_rx; 803 u64 drop_icm_address_tx; 804 u64 uplink_icm_address_rx; 805 u64 uplink_icm_address_tx; 806 u8 sw_owner:1; 807 u8 sw_owner_v2:1; 808 }; 809 810 struct mlx5dr_cmd_vport_cap { 811 u16 vport_gvmi; 812 u16 vhca_gvmi; 813 u16 num; 814 u64 icm_address_rx; 815 u64 icm_address_tx; 816 }; 817 818 struct mlx5dr_roce_cap { 819 u8 roce_en:1; 820 u8 fl_rc_qp_when_roce_disabled:1; 821 u8 fl_rc_qp_when_roce_enabled:1; 822 }; 823 824 struct mlx5dr_vports { 825 struct mlx5dr_cmd_vport_cap esw_manager_caps; 826 struct mlx5dr_cmd_vport_cap uplink_caps; 827 struct xarray vports_caps_xa; 828 }; 829 830 struct mlx5dr_cmd_caps { 831 u16 gvmi; 832 u64 nic_rx_drop_address; 833 u64 nic_tx_drop_address; 834 u64 nic_tx_allow_address; 835 u64 esw_rx_drop_address; 836 u64 esw_tx_drop_address; 837 u32 log_icm_size; 838 u64 hdr_modify_icm_addr; 839 u32 flex_protocols; 840 u8 flex_parser_id_icmp_dw0; 841 u8 flex_parser_id_icmp_dw1; 842 u8 flex_parser_id_icmpv6_dw0; 843 u8 flex_parser_id_icmpv6_dw1; 844 u8 flex_parser_id_geneve_tlv_option_0; 845 u8 flex_parser_id_mpls_over_gre; 846 u8 flex_parser_id_mpls_over_udp; 847 u8 flex_parser_id_gtpu_dw_0; 848 u8 flex_parser_id_gtpu_teid; 849 u8 flex_parser_id_gtpu_dw_2; 850 u8 flex_parser_id_gtpu_first_ext_dw_0; 851 u8 flex_parser_ok_bits_supp; 852 u8 max_ft_level; 853 u16 roce_min_src_udp; 854 u8 sw_format_ver; 855 bool eswitch_manager; 856 bool rx_sw_owner; 857 bool tx_sw_owner; 858 bool fdb_sw_owner; 859 u8 rx_sw_owner_v2:1; 860 u8 tx_sw_owner_v2:1; 861 u8 fdb_sw_owner_v2:1; 862 struct mlx5dr_esw_caps esw_caps; 863 struct mlx5dr_vports vports; 864 bool prio_tag_required; 865 struct mlx5dr_roce_cap roce_caps; 866 u8 is_ecpf:1; 867 u8 isolate_vl_tc:1; 868 }; 869 870 enum mlx5dr_domain_nic_type { 871 DR_DOMAIN_NIC_TYPE_RX, 872 DR_DOMAIN_NIC_TYPE_TX, 873 }; 874 875 struct mlx5dr_domain_rx_tx { 876 u64 drop_icm_addr; 877 u64 default_icm_addr; 878 enum mlx5dr_domain_nic_type type; 879 struct mutex mutex; /* protect rx/tx domain */ 880 }; 881 882 struct mlx5dr_domain_info { 883 bool supp_sw_steering; 884 u32 max_inline_size; 885 u32 max_send_wr; 886 u32 max_log_sw_icm_sz; 887 u32 max_log_action_icm_sz; 888 struct mlx5dr_domain_rx_tx rx; 889 struct mlx5dr_domain_rx_tx tx; 890 struct mlx5dr_cmd_caps caps; 891 }; 892 893 struct mlx5dr_domain { 894 struct mlx5dr_domain *peer_dmn; 895 struct mlx5_core_dev *mdev; 896 u32 pdn; 897 struct mlx5_uars_page *uar; 898 enum mlx5dr_domain_type type; 899 refcount_t refcount; 900 struct mlx5dr_icm_pool *ste_icm_pool; 901 struct mlx5dr_icm_pool *action_icm_pool; 902 struct mlx5dr_send_ring *send_ring; 903 struct mlx5dr_domain_info info; 904 struct xarray csum_fts_xa; 905 struct mlx5dr_ste_ctx *ste_ctx; 906 struct list_head dbg_tbl_list; 907 struct mlx5dr_dbg_dump_info dump_info; 908 }; 909 910 struct mlx5dr_table_rx_tx { 911 struct mlx5dr_ste_htbl *s_anchor; 912 struct mlx5dr_domain_rx_tx *nic_dmn; 913 u64 default_icm_addr; 914 struct list_head nic_matcher_list; 915 }; 916 917 struct mlx5dr_table { 918 struct mlx5dr_domain *dmn; 919 struct mlx5dr_table_rx_tx rx; 920 struct mlx5dr_table_rx_tx tx; 921 u32 level; 922 u32 table_type; 923 u32 table_id; 924 u32 flags; 925 struct list_head matcher_list; 926 struct mlx5dr_action *miss_action; 927 refcount_t refcount; 928 struct list_head dbg_node; 929 }; 930 931 struct mlx5dr_matcher_rx_tx { 932 struct mlx5dr_ste_htbl *s_htbl; 933 struct mlx5dr_ste_htbl *e_anchor; 934 struct mlx5dr_ste_build *ste_builder; 935 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX] 936 [DR_RULE_IPV_MAX] 937 [DR_RULE_MAX_STES]; 938 u8 num_of_builders; 939 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX]; 940 u64 default_icm_addr; 941 struct mlx5dr_table_rx_tx *nic_tbl; 942 u32 prio; 943 struct list_head list_node; 944 u32 rules; 945 }; 946 947 struct mlx5dr_matcher { 948 struct mlx5dr_table *tbl; 949 struct mlx5dr_matcher_rx_tx rx; 950 struct mlx5dr_matcher_rx_tx tx; 951 struct list_head list_node; /* Used for both matchers and dbg managing */ 952 u32 prio; 953 struct mlx5dr_match_param mask; 954 u8 match_criteria; 955 refcount_t refcount; 956 struct list_head dbg_rule_list; 957 }; 958 959 struct mlx5dr_ste_action_modify_field { 960 u16 hw_field; 961 u8 start; 962 u8 end; 963 u8 l3_type; 964 u8 l4_type; 965 }; 966 967 struct mlx5dr_action_rewrite { 968 struct mlx5dr_domain *dmn; 969 struct mlx5dr_icm_chunk *chunk; 970 u8 *data; 971 u16 num_of_actions; 972 u32 index; 973 u8 allow_rx:1; 974 u8 allow_tx:1; 975 u8 modify_ttl:1; 976 }; 977 978 struct mlx5dr_action_reformat { 979 struct mlx5dr_domain *dmn; 980 u32 id; 981 u32 size; 982 u8 param_0; 983 u8 param_1; 984 }; 985 986 struct mlx5dr_action_sampler { 987 struct mlx5dr_domain *dmn; 988 u64 rx_icm_addr; 989 u64 tx_icm_addr; 990 u32 sampler_id; 991 }; 992 993 struct mlx5dr_action_dest_tbl { 994 u8 is_fw_tbl:1; 995 union { 996 struct mlx5dr_table *tbl; 997 struct { 998 struct mlx5dr_domain *dmn; 999 u32 id; 1000 u32 group_id; 1001 enum fs_flow_table_type type; 1002 u64 rx_icm_addr; 1003 u64 tx_icm_addr; 1004 struct mlx5dr_action **ref_actions; 1005 u32 num_of_ref_actions; 1006 } fw_tbl; 1007 }; 1008 }; 1009 1010 struct mlx5dr_action_ctr { 1011 u32 ctr_id; 1012 u32 offset; 1013 }; 1014 1015 struct mlx5dr_action_vport { 1016 struct mlx5dr_domain *dmn; 1017 struct mlx5dr_cmd_vport_cap *caps; 1018 }; 1019 1020 struct mlx5dr_action_push_vlan { 1021 u32 vlan_hdr; /* tpid_pcp_dei_vid */ 1022 }; 1023 1024 struct mlx5dr_action_flow_tag { 1025 u32 flow_tag; 1026 }; 1027 1028 struct mlx5dr_rule_action_member { 1029 struct mlx5dr_action *action; 1030 struct list_head list; 1031 }; 1032 1033 struct mlx5dr_action { 1034 enum mlx5dr_action_type action_type; 1035 refcount_t refcount; 1036 1037 union { 1038 void *data; 1039 struct mlx5dr_action_rewrite *rewrite; 1040 struct mlx5dr_action_reformat *reformat; 1041 struct mlx5dr_action_sampler *sampler; 1042 struct mlx5dr_action_dest_tbl *dest_tbl; 1043 struct mlx5dr_action_ctr *ctr; 1044 struct mlx5dr_action_vport *vport; 1045 struct mlx5dr_action_push_vlan *push_vlan; 1046 struct mlx5dr_action_flow_tag *flow_tag; 1047 }; 1048 }; 1049 1050 enum mlx5dr_connect_type { 1051 CONNECT_HIT = 1, 1052 CONNECT_MISS = 2, 1053 }; 1054 1055 struct mlx5dr_htbl_connect_info { 1056 enum mlx5dr_connect_type type; 1057 union { 1058 struct mlx5dr_ste_htbl *hit_next_htbl; 1059 u64 miss_icm_addr; 1060 }; 1061 }; 1062 1063 struct mlx5dr_rule_rx_tx { 1064 struct mlx5dr_matcher_rx_tx *nic_matcher; 1065 struct mlx5dr_ste *last_rule_ste; 1066 }; 1067 1068 struct mlx5dr_rule { 1069 struct mlx5dr_matcher *matcher; 1070 struct mlx5dr_rule_rx_tx rx; 1071 struct mlx5dr_rule_rx_tx tx; 1072 struct list_head rule_actions_list; 1073 struct list_head dbg_node; 1074 u32 flow_source; 1075 }; 1076 1077 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule, 1078 struct mlx5dr_ste *ste, 1079 bool force); 1080 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr, 1081 struct mlx5dr_ste *curr_ste, 1082 int *num_of_stes); 1083 1084 struct mlx5dr_icm_chunk { 1085 struct mlx5dr_icm_buddy_mem *buddy_mem; 1086 struct list_head chunk_list; 1087 u32 rkey; 1088 u32 num_of_entries; 1089 u32 byte_size; 1090 u64 icm_addr; 1091 u64 mr_addr; 1092 1093 /* indicates the index of this chunk in the whole memory, 1094 * used for deleting the chunk from the buddy 1095 */ 1096 unsigned int seg; 1097 1098 /* Memory optimisation */ 1099 struct mlx5dr_ste *ste_arr; 1100 u8 *hw_ste_arr; 1101 struct list_head *miss_list; 1102 }; 1103 1104 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn) 1105 { 1106 mutex_lock(&nic_dmn->mutex); 1107 } 1108 1109 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn) 1110 { 1111 mutex_unlock(&nic_dmn->mutex); 1112 } 1113 1114 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn) 1115 { 1116 mlx5dr_domain_nic_lock(&dmn->info.rx); 1117 mlx5dr_domain_nic_lock(&dmn->info.tx); 1118 } 1119 1120 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn) 1121 { 1122 mlx5dr_domain_nic_unlock(&dmn->info.tx); 1123 mlx5dr_domain_nic_unlock(&dmn->info.rx); 1124 } 1125 1126 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn, 1127 struct mlx5dr_matcher_rx_tx *nic_matcher); 1128 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn, 1129 struct mlx5dr_matcher_rx_tx *nic_matcher); 1130 1131 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher, 1132 struct mlx5dr_matcher_rx_tx *nic_matcher, 1133 enum mlx5dr_ipv outer_ipv, 1134 enum mlx5dr_ipv inner_ipv); 1135 1136 static inline int 1137 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type) 1138 { 1139 if (icm_type == DR_ICM_TYPE_STE) 1140 return DR_STE_SIZE; 1141 1142 return DR_MODIFY_ACTION_SIZE; 1143 } 1144 1145 static inline u32 1146 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size) 1147 { 1148 return 1 << chunk_size; 1149 } 1150 1151 static inline int 1152 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size, 1153 enum mlx5dr_icm_type icm_type) 1154 { 1155 int num_of_entries; 1156 int entry_size; 1157 1158 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type); 1159 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size); 1160 1161 return entry_size * num_of_entries; 1162 } 1163 1164 static inline int 1165 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl) 1166 { 1167 int num_of_entries = 1168 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk_size); 1169 1170 /* Threshold is 50%, one is added to table of size 1 */ 1171 return (num_of_entries + 1) / 2; 1172 } 1173 1174 static inline bool 1175 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl) 1176 { 1177 if (htbl->chunk_size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask) 1178 return false; 1179 1180 return true; 1181 } 1182 1183 struct mlx5dr_cmd_vport_cap * 1184 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport); 1185 1186 struct mlx5dr_cmd_query_flow_table_details { 1187 u8 status; 1188 u8 level; 1189 u64 sw_owner_icm_root_1; 1190 u64 sw_owner_icm_root_0; 1191 }; 1192 1193 struct mlx5dr_cmd_create_flow_table_attr { 1194 u32 table_type; 1195 u64 icm_addr_rx; 1196 u64 icm_addr_tx; 1197 u8 level; 1198 bool sw_owner; 1199 bool term_tbl; 1200 bool decap_en; 1201 bool reformat_en; 1202 }; 1203 1204 /* internal API functions */ 1205 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, 1206 struct mlx5dr_cmd_caps *caps); 1207 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev, 1208 bool other_vport, u16 vport_number, 1209 u64 *icm_address_rx, 1210 u64 *icm_address_tx); 1211 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev, 1212 bool other_vport, u16 vport_number, u16 *gvmi); 1213 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev, 1214 struct mlx5dr_esw_caps *caps); 1215 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev, 1216 u32 sampler_id, 1217 u64 *rx_icm_addr, 1218 u64 *tx_icm_addr); 1219 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev); 1220 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev, 1221 u32 table_type, 1222 u32 table_id, 1223 u32 group_id, 1224 u32 modify_header_id, 1225 u16 vport_id); 1226 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev, 1227 u32 table_type, 1228 u32 table_id); 1229 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev, 1230 u32 table_type, 1231 u8 num_of_actions, 1232 u64 *actions, 1233 u32 *modify_header_id); 1234 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev, 1235 u32 modify_header_id); 1236 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev, 1237 u32 table_type, 1238 u32 table_id, 1239 u32 *group_id); 1240 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev, 1241 u32 table_type, 1242 u32 table_id, 1243 u32 group_id); 1244 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev, 1245 struct mlx5dr_cmd_create_flow_table_attr *attr, 1246 u64 *fdb_rx_icm_addr, 1247 u32 *table_id); 1248 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev, 1249 u32 table_id, 1250 u32 table_type); 1251 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev, 1252 enum fs_flow_table_type type, 1253 u32 table_id, 1254 struct mlx5dr_cmd_query_flow_table_details *output); 1255 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, 1256 enum mlx5_reformat_ctx_type rt, 1257 u8 reformat_param_0, 1258 u8 reformat_param_1, 1259 size_t reformat_size, 1260 void *reformat_data, 1261 u32 *reformat_id); 1262 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev, 1263 u32 reformat_id); 1264 1265 struct mlx5dr_cmd_gid_attr { 1266 u8 gid[16]; 1267 u8 mac[6]; 1268 u32 roce_ver; 1269 }; 1270 1271 struct mlx5dr_cmd_qp_create_attr { 1272 u32 page_id; 1273 u32 pdn; 1274 u32 cqn; 1275 u32 pm_state; 1276 u32 service_type; 1277 u32 buff_umem_id; 1278 u32 db_umem_id; 1279 u32 sq_wqe_cnt; 1280 u32 rq_wqe_cnt; 1281 u32 rq_wqe_shift; 1282 u8 isolate_vl_tc:1; 1283 }; 1284 1285 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num, 1286 u16 index, struct mlx5dr_cmd_gid_attr *attr); 1287 1288 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 1289 enum mlx5dr_icm_type icm_type); 1290 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool); 1291 1292 struct mlx5dr_icm_chunk * 1293 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 1294 enum mlx5dr_icm_chunk_size chunk_size); 1295 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk); 1296 1297 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx, 1298 u8 *hw_ste_p, u32 ste_size); 1299 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn, 1300 struct mlx5dr_domain_rx_tx *nic_dmn, 1301 struct mlx5dr_ste_htbl *htbl, 1302 struct mlx5dr_htbl_connect_info *connect_info, 1303 bool update_hw_ste); 1304 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx, 1305 u16 gvmi, 1306 enum mlx5dr_domain_nic_type nic_type, 1307 struct mlx5dr_ste_htbl *htbl, 1308 u8 *formatted_ste, 1309 struct mlx5dr_htbl_connect_info *connect_info); 1310 void mlx5dr_ste_copy_param(u8 match_criteria, 1311 struct mlx5dr_match_param *set_param, 1312 struct mlx5dr_match_parameters *mask, 1313 bool clear); 1314 1315 struct mlx5dr_qp { 1316 struct mlx5_core_dev *mdev; 1317 struct mlx5_wq_qp wq; 1318 struct mlx5_uars_page *uar; 1319 struct mlx5_wq_ctrl wq_ctrl; 1320 u32 qpn; 1321 struct { 1322 unsigned int pc; 1323 unsigned int cc; 1324 unsigned int size; 1325 unsigned int *wqe_head; 1326 unsigned int wqe_cnt; 1327 } sq; 1328 struct { 1329 unsigned int pc; 1330 unsigned int cc; 1331 unsigned int size; 1332 unsigned int wqe_cnt; 1333 } rq; 1334 int max_inline_data; 1335 }; 1336 1337 struct mlx5dr_cq { 1338 struct mlx5_core_dev *mdev; 1339 struct mlx5_cqwq wq; 1340 struct mlx5_wq_ctrl wq_ctrl; 1341 struct mlx5_core_cq mcq; 1342 struct mlx5dr_qp *qp; 1343 }; 1344 1345 struct mlx5dr_mr { 1346 struct mlx5_core_dev *mdev; 1347 u32 mkey; 1348 dma_addr_t dma_addr; 1349 void *addr; 1350 size_t size; 1351 }; 1352 1353 #define MAX_SEND_CQE 64 1354 #define MIN_READ_SYNC 64 1355 1356 struct mlx5dr_send_ring { 1357 struct mlx5dr_cq *cq; 1358 struct mlx5dr_qp *qp; 1359 struct mlx5dr_mr *mr; 1360 /* How much wqes are waiting for completion */ 1361 u32 pending_wqe; 1362 /* Signal request per this trash hold value */ 1363 u16 signal_th; 1364 /* Each post_send_size less than max_post_send_size */ 1365 u32 max_post_send_size; 1366 /* manage the send queue */ 1367 u32 tx_head; 1368 void *buf; 1369 u32 buf_size; 1370 u8 sync_buff[MIN_READ_SYNC]; 1371 struct mlx5dr_mr *sync_mr; 1372 spinlock_t lock; /* Protect the data path of the send ring */ 1373 bool err_state; /* send_ring is not usable in err state */ 1374 }; 1375 1376 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn); 1377 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn, 1378 struct mlx5dr_send_ring *send_ring); 1379 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn); 1380 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn, 1381 struct mlx5dr_ste *ste, 1382 u8 *data, 1383 u16 size, 1384 u16 offset); 1385 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, 1386 struct mlx5dr_ste_htbl *htbl, 1387 u8 *formatted_ste, u8 *mask); 1388 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn, 1389 struct mlx5dr_ste_htbl *htbl, 1390 u8 *ste_init_data, 1391 bool update_hw_ste); 1392 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn, 1393 struct mlx5dr_action *action); 1394 1395 struct mlx5dr_cmd_ft_info { 1396 u32 id; 1397 u16 vport; 1398 enum fs_flow_table_type type; 1399 }; 1400 1401 struct mlx5dr_cmd_flow_destination_hw_info { 1402 enum mlx5_flow_destination_type type; 1403 union { 1404 u32 tir_num; 1405 u32 ft_num; 1406 u32 ft_id; 1407 u32 counter_id; 1408 u32 sampler_id; 1409 struct { 1410 u16 num; 1411 u16 vhca_id; 1412 u32 reformat_id; 1413 u8 flags; 1414 } vport; 1415 }; 1416 }; 1417 1418 struct mlx5dr_cmd_fte_info { 1419 u32 dests_size; 1420 u32 index; 1421 struct mlx5_flow_context flow_context; 1422 u32 *val; 1423 struct mlx5_flow_act action; 1424 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr; 1425 bool ignore_flow_level; 1426 }; 1427 1428 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev, 1429 int opmod, int modify_mask, 1430 struct mlx5dr_cmd_ft_info *ft, 1431 u32 group_id, 1432 struct mlx5dr_cmd_fte_info *fte); 1433 1434 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps); 1435 1436 struct mlx5dr_fw_recalc_cs_ft { 1437 u64 rx_icm_addr; 1438 u32 table_id; 1439 u32 group_id; 1440 u32 modify_hdr_id; 1441 }; 1442 1443 struct mlx5dr_fw_recalc_cs_ft * 1444 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num); 1445 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn, 1446 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft); 1447 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn, 1448 u16 vport_num, 1449 u64 *rx_icm_addr); 1450 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn, 1451 struct mlx5dr_cmd_flow_destination_hw_info *dest, 1452 int num_dest, 1453 bool reformat_req, 1454 u32 *tbl_id, 1455 u32 *group_id, 1456 bool ignore_flow_level); 1457 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id, 1458 u32 group_id); 1459 #endif /* _DR_TYPES_H_ */ 1460