1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2019, Mellanox Technologies */ 3 4 #ifndef _DR_TYPES_ 5 #define _DR_TYPES_ 6 7 #include <linux/mlx5/vport.h> 8 #include <linux/refcount.h> 9 #include "fs_core.h" 10 #include "wq.h" 11 #include "lib/mlx5.h" 12 #include "mlx5_ifc_dr.h" 13 #include "mlx5dr.h" 14 #include "dr_dbg.h" 15 16 #define DR_RULE_MAX_STES 18 17 #define DR_ACTION_MAX_STES 5 18 #define DR_STE_SVLAN 0x1 19 #define DR_STE_CVLAN 0x2 20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4) 21 #define DR_NUM_OF_FLEX_PARSERS 8 22 #define DR_STE_MAX_FLEX_0_ID 3 23 #define DR_STE_MAX_FLEX_1_ID 7 24 #define DR_ACTION_CACHE_LINE_SIZE 64 25 26 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg) 27 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg) 28 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg) 29 30 struct mlx5dr_ptrn_mgr; 31 struct mlx5dr_arg_mgr; 32 struct mlx5dr_arg_obj; 33 34 static inline bool dr_is_flex_parser_0_id(u8 parser_id) 35 { 36 return parser_id <= DR_STE_MAX_FLEX_0_ID; 37 } 38 39 static inline bool dr_is_flex_parser_1_id(u8 parser_id) 40 { 41 return parser_id > DR_STE_MAX_FLEX_0_ID; 42 } 43 44 enum mlx5dr_icm_chunk_size { 45 DR_CHUNK_SIZE_1, 46 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */ 47 DR_CHUNK_SIZE_2, 48 DR_CHUNK_SIZE_4, 49 DR_CHUNK_SIZE_8, 50 DR_CHUNK_SIZE_16, 51 DR_CHUNK_SIZE_32, 52 DR_CHUNK_SIZE_64, 53 DR_CHUNK_SIZE_128, 54 DR_CHUNK_SIZE_256, 55 DR_CHUNK_SIZE_512, 56 DR_CHUNK_SIZE_1K, 57 DR_CHUNK_SIZE_2K, 58 DR_CHUNK_SIZE_4K, 59 DR_CHUNK_SIZE_8K, 60 DR_CHUNK_SIZE_16K, 61 DR_CHUNK_SIZE_32K, 62 DR_CHUNK_SIZE_64K, 63 DR_CHUNK_SIZE_128K, 64 DR_CHUNK_SIZE_256K, 65 DR_CHUNK_SIZE_512K, 66 DR_CHUNK_SIZE_1024K, 67 DR_CHUNK_SIZE_2048K, 68 DR_CHUNK_SIZE_MAX, 69 }; 70 71 enum mlx5dr_icm_type { 72 DR_ICM_TYPE_STE, 73 DR_ICM_TYPE_MODIFY_ACTION, 74 DR_ICM_TYPE_MODIFY_HDR_PTRN, 75 }; 76 77 static inline enum mlx5dr_icm_chunk_size 78 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk) 79 { 80 chunk += 2; 81 if (chunk < DR_CHUNK_SIZE_MAX) 82 return chunk; 83 84 return DR_CHUNK_SIZE_MAX; 85 } 86 87 enum { 88 DR_STE_SIZE = 64, 89 DR_STE_SIZE_CTRL = 32, 90 DR_STE_SIZE_MATCH_TAG = 32, 91 DR_STE_SIZE_TAG = 16, 92 DR_STE_SIZE_MASK = 16, 93 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK, 94 }; 95 96 enum mlx5dr_ste_ctx_action_cap { 97 DR_STE_CTX_ACTION_CAP_NONE = 0, 98 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0, 99 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1, 100 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2, 101 DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3, 102 }; 103 104 enum { 105 DR_MODIFY_ACTION_SIZE = 8, 106 }; 107 108 enum mlx5dr_matcher_criteria { 109 DR_MATCHER_CRITERIA_EMPTY = 0, 110 DR_MATCHER_CRITERIA_OUTER = 1 << 0, 111 DR_MATCHER_CRITERIA_MISC = 1 << 1, 112 DR_MATCHER_CRITERIA_INNER = 1 << 2, 113 DR_MATCHER_CRITERIA_MISC2 = 1 << 3, 114 DR_MATCHER_CRITERIA_MISC3 = 1 << 4, 115 DR_MATCHER_CRITERIA_MISC4 = 1 << 5, 116 DR_MATCHER_CRITERIA_MISC5 = 1 << 6, 117 DR_MATCHER_CRITERIA_MAX = 1 << 7, 118 }; 119 120 enum mlx5dr_action_type { 121 DR_ACTION_TYP_TNL_L2_TO_L2, 122 DR_ACTION_TYP_L2_TO_TNL_L2, 123 DR_ACTION_TYP_TNL_L3_TO_L2, 124 DR_ACTION_TYP_L2_TO_TNL_L3, 125 DR_ACTION_TYP_DROP, 126 DR_ACTION_TYP_QP, 127 DR_ACTION_TYP_FT, 128 DR_ACTION_TYP_CTR, 129 DR_ACTION_TYP_TAG, 130 DR_ACTION_TYP_MODIFY_HDR, 131 DR_ACTION_TYP_VPORT, 132 DR_ACTION_TYP_POP_VLAN, 133 DR_ACTION_TYP_PUSH_VLAN, 134 DR_ACTION_TYP_INSERT_HDR, 135 DR_ACTION_TYP_REMOVE_HDR, 136 DR_ACTION_TYP_SAMPLER, 137 DR_ACTION_TYP_ASO_FLOW_METER, 138 DR_ACTION_TYP_RANGE, 139 DR_ACTION_TYP_MAX, 140 }; 141 142 enum mlx5dr_ipv { 143 DR_RULE_IPV4, 144 DR_RULE_IPV6, 145 DR_RULE_IPV_MAX, 146 }; 147 148 struct mlx5dr_icm_pool; 149 struct mlx5dr_icm_chunk; 150 struct mlx5dr_icm_buddy_mem; 151 struct mlx5dr_ste_htbl; 152 struct mlx5dr_match_param; 153 struct mlx5dr_cmd_caps; 154 struct mlx5dr_rule_rx_tx; 155 struct mlx5dr_matcher_rx_tx; 156 struct mlx5dr_ste_ctx; 157 struct mlx5dr_send_info_pool; 158 struct mlx5dr_icm_hot_chunk; 159 160 struct mlx5dr_ste { 161 /* refcount: indicates the num of rules that using this ste */ 162 u32 refcount; 163 164 /* this ste is part of a rule, located in ste's chain */ 165 u8 ste_chain_location; 166 167 /* attached to the miss_list head at each htbl entry */ 168 struct list_head miss_list_node; 169 170 /* this ste is member of htbl */ 171 struct mlx5dr_ste_htbl *htbl; 172 173 struct mlx5dr_ste_htbl *next_htbl; 174 175 /* The rule this STE belongs to */ 176 struct mlx5dr_rule_rx_tx *rule_rx_tx; 177 }; 178 179 struct mlx5dr_ste_htbl_ctrl { 180 /* total number of valid entries belonging to this hash table. This 181 * includes the non collision and collision entries 182 */ 183 unsigned int num_of_valid_entries; 184 185 /* total number of collisions entries attached to this table */ 186 unsigned int num_of_collisions; 187 }; 188 189 struct mlx5dr_ste_htbl { 190 u16 lu_type; 191 u16 byte_mask; 192 u32 refcount; 193 struct mlx5dr_icm_chunk *chunk; 194 struct mlx5dr_ste *pointing_ste; 195 struct mlx5dr_ste_htbl_ctrl ctrl; 196 }; 197 198 struct mlx5dr_ste_send_info { 199 struct mlx5dr_ste *ste; 200 struct list_head send_list; 201 u16 size; 202 u16 offset; 203 u8 data_cont[DR_STE_SIZE]; 204 u8 *data; 205 }; 206 207 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size, 208 u16 offset, u8 *data, 209 struct mlx5dr_ste_send_info *ste_info, 210 struct list_head *send_list, 211 bool copy_data); 212 213 struct mlx5dr_ste_build { 214 u8 inner:1; 215 u8 rx:1; 216 u8 vhca_id_valid:1; 217 struct mlx5dr_domain *dmn; 218 struct mlx5dr_cmd_caps *caps; 219 u16 lu_type; 220 u16 byte_mask; 221 u8 bit_mask[DR_STE_SIZE_MASK]; 222 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec, 223 struct mlx5dr_ste_build *sb, 224 u8 *tag); 225 }; 226 227 struct mlx5dr_ste_htbl * 228 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool, 229 enum mlx5dr_icm_chunk_size chunk_size, 230 u16 lu_type, u16 byte_mask); 231 232 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl); 233 234 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl) 235 { 236 htbl->refcount--; 237 if (!htbl->refcount) 238 mlx5dr_ste_htbl_free(htbl); 239 } 240 241 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl) 242 { 243 htbl->refcount++; 244 } 245 246 /* STE utils */ 247 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl); 248 bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste_p); 249 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, 250 u8 *hw_ste, u64 miss_addr); 251 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx, 252 u8 *hw_ste, u64 icm_addr, u32 ht_size); 253 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx, 254 u8 *hw_ste, 255 struct mlx5dr_ste_htbl *next_htbl); 256 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask); 257 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher, 258 u8 ste_location); 259 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste); 260 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste); 261 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste); 262 263 #define MLX5DR_MAX_VLANS 2 264 #define MLX5DR_INVALID_PATTERN_INDEX 0xffffffff 265 266 struct mlx5dr_ste_actions_attr { 267 u32 modify_index; 268 u32 modify_pat_idx; 269 u16 modify_actions; 270 u32 decap_index; 271 u32 decap_pat_idx; 272 u16 decap_actions; 273 u8 decap_with_vlan:1; 274 u64 final_icm_addr; 275 u32 flow_tag; 276 u32 ctr_id; 277 u16 gvmi; 278 u16 hit_gvmi; 279 struct { 280 u32 id; 281 u32 size; 282 u8 param_0; 283 u8 param_1; 284 } reformat; 285 struct { 286 int count; 287 u32 headers[MLX5DR_MAX_VLANS]; 288 } vlans; 289 290 struct { 291 u32 obj_id; 292 u32 offset; 293 u8 dest_reg_id; 294 u8 init_color; 295 } aso_flow_meter; 296 297 struct { 298 u64 miss_icm_addr; 299 u32 definer_id; 300 u32 min; 301 u32 max; 302 } range; 303 }; 304 305 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, 306 struct mlx5dr_domain *dmn, 307 u8 *action_type_set, 308 u8 *last_ste, 309 struct mlx5dr_ste_actions_attr *attr, 310 u32 *added_stes); 311 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx, 312 struct mlx5dr_domain *dmn, 313 u8 *action_type_set, 314 u8 *last_ste, 315 struct mlx5dr_ste_actions_attr *attr, 316 u32 *added_stes); 317 318 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx, 319 __be64 *hw_action, 320 u8 hw_field, 321 u8 shifter, 322 u8 length, 323 u32 data); 324 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx, 325 __be64 *hw_action, 326 u8 hw_field, 327 u8 shifter, 328 u8 length, 329 u32 data); 330 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx, 331 __be64 *hw_action, 332 u8 dst_hw_field, 333 u8 dst_shifter, 334 u8 dst_len, 335 u8 src_hw_field, 336 u8 src_shifter); 337 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx, 338 void *data, 339 u32 data_sz, 340 u8 *hw_action, 341 u32 hw_action_sz, 342 u16 *used_hw_action_num); 343 int mlx5dr_ste_alloc_modify_hdr(struct mlx5dr_action *action); 344 void mlx5dr_ste_free_modify_hdr(struct mlx5dr_action *action); 345 346 const struct mlx5dr_ste_action_modify_field * 347 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field); 348 349 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version); 350 void mlx5dr_ste_free(struct mlx5dr_ste *ste, 351 struct mlx5dr_matcher *matcher, 352 struct mlx5dr_matcher_rx_tx *nic_matcher); 353 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste, 354 struct mlx5dr_matcher *matcher, 355 struct mlx5dr_matcher_rx_tx *nic_matcher) 356 { 357 ste->refcount--; 358 if (!ste->refcount) 359 mlx5dr_ste_free(ste, matcher, nic_matcher); 360 } 361 362 /* initial as 0, increased only when ste appears in a new rule */ 363 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste) 364 { 365 ste->refcount++; 366 } 367 368 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste) 369 { 370 return !ste->refcount; 371 } 372 373 bool mlx5dr_ste_equal_tag(void *src, void *dst); 374 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher, 375 struct mlx5dr_matcher_rx_tx *nic_matcher, 376 struct mlx5dr_ste *ste, 377 u8 *cur_hw_ste, 378 enum mlx5dr_icm_chunk_size log_table_size); 379 380 /* STE build functions */ 381 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn, 382 u8 match_criteria, 383 struct mlx5dr_match_param *mask, 384 struct mlx5dr_match_param *value); 385 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, 386 struct mlx5dr_matcher_rx_tx *nic_matcher, 387 struct mlx5dr_match_param *value, 388 u8 *ste_arr); 389 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx, 390 struct mlx5dr_ste_build *builder, 391 struct mlx5dr_match_param *mask, 392 bool inner, bool rx); 393 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx, 394 struct mlx5dr_ste_build *sb, 395 struct mlx5dr_match_param *mask, 396 bool inner, bool rx); 397 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx, 398 struct mlx5dr_ste_build *sb, 399 struct mlx5dr_match_param *mask, 400 bool inner, bool rx); 401 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx, 402 struct mlx5dr_ste_build *sb, 403 struct mlx5dr_match_param *mask, 404 bool inner, bool rx); 405 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx, 406 struct mlx5dr_ste_build *sb, 407 struct mlx5dr_match_param *mask, 408 bool inner, bool rx); 409 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx, 410 struct mlx5dr_ste_build *sb, 411 struct mlx5dr_match_param *mask, 412 bool inner, bool rx); 413 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx, 414 struct mlx5dr_ste_build *sb, 415 struct mlx5dr_match_param *mask, 416 bool inner, bool rx); 417 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx, 418 struct mlx5dr_ste_build *sb, 419 struct mlx5dr_match_param *mask, 420 bool inner, bool rx); 421 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx, 422 struct mlx5dr_ste_build *sb, 423 struct mlx5dr_match_param *mask, 424 bool inner, bool rx); 425 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx, 426 struct mlx5dr_ste_build *sb, 427 struct mlx5dr_match_param *mask, 428 bool inner, bool rx); 429 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx, 430 struct mlx5dr_ste_build *sb, 431 struct mlx5dr_match_param *mask, 432 bool inner, bool rx); 433 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx, 434 struct mlx5dr_ste_build *sb, 435 struct mlx5dr_match_param *mask, 436 bool inner, bool rx); 437 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx, 438 struct mlx5dr_ste_build *sb, 439 struct mlx5dr_match_param *mask, 440 bool inner, bool rx); 441 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx, 442 struct mlx5dr_ste_build *sb, 443 struct mlx5dr_match_param *mask, 444 struct mlx5dr_cmd_caps *caps, 445 bool inner, bool rx); 446 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx, 447 struct mlx5dr_ste_build *sb, 448 struct mlx5dr_match_param *mask, 449 struct mlx5dr_cmd_caps *caps, 450 bool inner, bool rx); 451 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx, 452 struct mlx5dr_ste_build *sb, 453 struct mlx5dr_match_param *mask, 454 struct mlx5dr_cmd_caps *caps, 455 bool inner, bool rx); 456 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx, 457 struct mlx5dr_ste_build *sb, 458 struct mlx5dr_match_param *mask, 459 bool inner, bool rx); 460 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx, 461 struct mlx5dr_ste_build *sb, 462 struct mlx5dr_match_param *mask, 463 bool inner, bool rx); 464 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx, 465 struct mlx5dr_ste_build *sb, 466 struct mlx5dr_match_param *mask, 467 struct mlx5dr_cmd_caps *caps, 468 bool inner, bool rx); 469 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx, 470 struct mlx5dr_ste_build *sb, 471 struct mlx5dr_match_param *mask, 472 struct mlx5dr_cmd_caps *caps, 473 bool inner, bool rx); 474 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx, 475 struct mlx5dr_ste_build *sb, 476 struct mlx5dr_match_param *mask, 477 bool inner, bool rx); 478 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 479 struct mlx5dr_ste_build *sb, 480 struct mlx5dr_match_param *mask, 481 struct mlx5dr_cmd_caps *caps, 482 bool inner, bool rx); 483 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 484 struct mlx5dr_ste_build *sb, 485 struct mlx5dr_match_param *mask, 486 struct mlx5dr_cmd_caps *caps, 487 bool inner, bool rx); 488 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx, 489 struct mlx5dr_ste_build *sb, 490 struct mlx5dr_match_param *mask, 491 bool inner, bool rx); 492 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx, 493 struct mlx5dr_ste_build *sb, 494 struct mlx5dr_match_param *mask, 495 bool inner, bool rx); 496 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx, 497 struct mlx5dr_ste_build *sb, 498 struct mlx5dr_match_param *mask, 499 bool inner, bool rx); 500 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx, 501 struct mlx5dr_ste_build *sb, 502 struct mlx5dr_match_param *mask, 503 bool inner, bool rx); 504 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx, 505 struct mlx5dr_ste_build *sb, 506 struct mlx5dr_match_param *mask, 507 struct mlx5dr_domain *dmn, 508 bool inner, bool rx); 509 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 510 struct mlx5dr_ste_build *sb, 511 struct mlx5dr_match_param *mask, 512 bool inner, bool rx); 513 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 514 struct mlx5dr_ste_build *sb, 515 struct mlx5dr_match_param *mask, 516 bool inner, bool rx); 517 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx); 518 519 /* Actions utils */ 520 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, 521 struct mlx5dr_matcher_rx_tx *nic_matcher, 522 struct mlx5dr_action *actions[], 523 u32 num_actions, 524 u8 *ste_arr, 525 u32 *new_hw_ste_arr_sz); 526 527 struct mlx5dr_match_spec { 528 u32 smac_47_16; /* Source MAC address of incoming packet */ 529 /* Incoming packet Ethertype - this is the Ethertype 530 * following the last VLAN tag of the packet 531 */ 532 u32 smac_15_0:16; /* Source MAC address of incoming packet */ 533 u32 ethertype:16; 534 535 u32 dmac_47_16; /* Destination MAC address of incoming packet */ 536 537 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */ 538 /* Priority of first VLAN tag in the incoming packet. 539 * Valid only when cvlan_tag==1 or svlan_tag==1 540 */ 541 u32 first_prio:3; 542 /* CFI bit of first VLAN tag in the incoming packet. 543 * Valid only when cvlan_tag==1 or svlan_tag==1 544 */ 545 u32 first_cfi:1; 546 /* VLAN ID of first VLAN tag in the incoming packet. 547 * Valid only when cvlan_tag==1 or svlan_tag==1 548 */ 549 u32 first_vid:12; 550 551 u32 ip_protocol:8; /* IP protocol */ 552 /* Differentiated Services Code Point derived from 553 * Traffic Class/TOS field of IPv6/v4 554 */ 555 u32 ip_dscp:6; 556 /* Explicit Congestion Notification derived from 557 * Traffic Class/TOS field of IPv6/v4 558 */ 559 u32 ip_ecn:2; 560 /* The first vlan in the packet is c-vlan (0x8100). 561 * cvlan_tag and svlan_tag cannot be set together 562 */ 563 u32 cvlan_tag:1; 564 /* The first vlan in the packet is s-vlan (0x8a88). 565 * cvlan_tag and svlan_tag cannot be set together 566 */ 567 u32 svlan_tag:1; 568 u32 frag:1; /* Packet is an IP fragment */ 569 u32 ip_version:4; /* IP version */ 570 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK; 571 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS 572 */ 573 u32 tcp_flags:9; 574 575 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */ 576 u32 tcp_sport:16; 577 /* TCP destination port. 578 * tcp and udp sport/dport are mutually exclusive 579 */ 580 u32 tcp_dport:16; 581 582 u32 reserved_auto1:16; 583 u32 ipv4_ihl:4; 584 u32 reserved_auto2:4; 585 u32 ttl_hoplimit:8; 586 587 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */ 588 u32 udp_sport:16; 589 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */ 590 u32 udp_dport:16; 591 592 /* IPv6 source address of incoming packets 593 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 594 * This field should be qualified by an appropriate ethertype 595 */ 596 u32 src_ip_127_96; 597 /* IPv6 source address of incoming packets 598 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 599 * This field should be qualified by an appropriate ethertype 600 */ 601 u32 src_ip_95_64; 602 /* IPv6 source address of incoming packets 603 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 604 * This field should be qualified by an appropriate ethertype 605 */ 606 u32 src_ip_63_32; 607 /* IPv6 source address of incoming packets 608 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 609 * This field should be qualified by an appropriate ethertype 610 */ 611 u32 src_ip_31_0; 612 /* IPv6 destination address of incoming packets 613 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 614 * This field should be qualified by an appropriate ethertype 615 */ 616 u32 dst_ip_127_96; 617 /* IPv6 destination address of incoming packets 618 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 619 * This field should be qualified by an appropriate ethertype 620 */ 621 u32 dst_ip_95_64; 622 /* IPv6 destination address of incoming packets 623 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 624 * This field should be qualified by an appropriate ethertype 625 */ 626 u32 dst_ip_63_32; 627 /* IPv6 destination address of incoming packets 628 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 629 * This field should be qualified by an appropriate ethertype 630 */ 631 u32 dst_ip_31_0; 632 }; 633 634 struct mlx5dr_match_misc { 635 /* used with GRE, checksum exist when gre_c_present == 1 */ 636 u32 gre_c_present:1; 637 u32 reserved_auto1:1; 638 /* used with GRE, key exist when gre_k_present == 1 */ 639 u32 gre_k_present:1; 640 /* used with GRE, sequence number exist when gre_s_present == 1 */ 641 u32 gre_s_present:1; 642 u32 source_vhca_port:4; 643 u32 source_sqn:24; /* Source SQN */ 644 645 u32 source_eswitch_owner_vhca_id:16; 646 /* Source port.;0xffff determines wire port */ 647 u32 source_port:16; 648 649 /* Priority of second VLAN tag in the outer header of the incoming packet. 650 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 651 */ 652 u32 outer_second_prio:3; 653 /* CFI bit of first VLAN tag in the outer header of the incoming packet. 654 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 655 */ 656 u32 outer_second_cfi:1; 657 /* VLAN ID of first VLAN tag the outer header of the incoming packet. 658 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 659 */ 660 u32 outer_second_vid:12; 661 /* Priority of second VLAN tag in the inner header of the incoming packet. 662 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 663 */ 664 u32 inner_second_prio:3; 665 /* CFI bit of first VLAN tag in the inner header of the incoming packet. 666 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 667 */ 668 u32 inner_second_cfi:1; 669 /* VLAN ID of first VLAN tag the inner header of the incoming packet. 670 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 671 */ 672 u32 inner_second_vid:12; 673 674 u32 outer_second_cvlan_tag:1; 675 u32 inner_second_cvlan_tag:1; 676 /* The second vlan in the outer header of the packet is c-vlan (0x8100). 677 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 678 */ 679 u32 outer_second_svlan_tag:1; 680 /* The second vlan in the inner header of the packet is c-vlan (0x8100). 681 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 682 */ 683 u32 inner_second_svlan_tag:1; 684 /* The second vlan in the outer header of the packet is s-vlan (0x8a88). 685 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 686 */ 687 u32 reserved_auto2:12; 688 /* The second vlan in the inner header of the packet is s-vlan (0x8a88). 689 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 690 */ 691 u32 gre_protocol:16; /* GRE Protocol (outer) */ 692 693 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */ 694 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */ 695 696 u32 vxlan_vni:24; /* VXLAN VNI (outer) */ 697 u32 reserved_auto3:8; 698 699 u32 geneve_vni:24; /* GENEVE VNI field (outer) */ 700 u32 reserved_auto4:6; 701 u32 geneve_tlv_option_0_exist:1; 702 u32 geneve_oam:1; /* GENEVE OAM field (outer) */ 703 704 u32 reserved_auto5:12; 705 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */ 706 707 u32 reserved_auto6:12; 708 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */ 709 710 u32 reserved_auto7:10; 711 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */ 712 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */ 713 714 u32 reserved_auto8:8; 715 u32 bth_dst_qp:24; /* Destination QP in BTH header */ 716 717 u32 reserved_auto9; 718 u32 outer_esp_spi; 719 u32 reserved_auto10[3]; 720 }; 721 722 struct mlx5dr_match_misc2 { 723 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */ 724 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */ 725 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */ 726 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */ 727 728 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */ 729 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */ 730 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */ 731 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */ 732 733 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */ 734 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */ 735 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */ 736 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */ 737 738 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */ 739 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */ 740 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */ 741 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */ 742 743 u32 metadata_reg_c_7; /* metadata_reg_c_7 */ 744 u32 metadata_reg_c_6; /* metadata_reg_c_6 */ 745 u32 metadata_reg_c_5; /* metadata_reg_c_5 */ 746 u32 metadata_reg_c_4; /* metadata_reg_c_4 */ 747 u32 metadata_reg_c_3; /* metadata_reg_c_3 */ 748 u32 metadata_reg_c_2; /* metadata_reg_c_2 */ 749 u32 metadata_reg_c_1; /* metadata_reg_c_1 */ 750 u32 metadata_reg_c_0; /* metadata_reg_c_0 */ 751 u32 metadata_reg_a; /* metadata_reg_a */ 752 u32 reserved_auto1[3]; 753 }; 754 755 struct mlx5dr_match_misc3 { 756 u32 inner_tcp_seq_num; 757 u32 outer_tcp_seq_num; 758 u32 inner_tcp_ack_num; 759 u32 outer_tcp_ack_num; 760 761 u32 reserved_auto1:8; 762 u32 outer_vxlan_gpe_vni:24; 763 764 u32 outer_vxlan_gpe_next_protocol:8; 765 u32 outer_vxlan_gpe_flags:8; 766 u32 reserved_auto2:16; 767 768 u32 icmpv4_header_data; 769 u32 icmpv6_header_data; 770 771 u8 icmpv4_type; 772 u8 icmpv4_code; 773 u8 icmpv6_type; 774 u8 icmpv6_code; 775 776 u32 geneve_tlv_option_0_data; 777 778 u32 gtpu_teid; 779 780 u8 gtpu_msg_type; 781 u8 gtpu_msg_flags; 782 u32 reserved_auto3:16; 783 784 u32 gtpu_dw_2; 785 u32 gtpu_first_ext_dw_0; 786 u32 gtpu_dw_0; 787 u32 reserved_auto4; 788 }; 789 790 struct mlx5dr_match_misc4 { 791 u32 prog_sample_field_value_0; 792 u32 prog_sample_field_id_0; 793 u32 prog_sample_field_value_1; 794 u32 prog_sample_field_id_1; 795 u32 prog_sample_field_value_2; 796 u32 prog_sample_field_id_2; 797 u32 prog_sample_field_value_3; 798 u32 prog_sample_field_id_3; 799 u32 reserved_auto1[8]; 800 }; 801 802 struct mlx5dr_match_misc5 { 803 u32 macsec_tag_0; 804 u32 macsec_tag_1; 805 u32 macsec_tag_2; 806 u32 macsec_tag_3; 807 u32 tunnel_header_0; 808 u32 tunnel_header_1; 809 u32 tunnel_header_2; 810 u32 tunnel_header_3; 811 }; 812 813 struct mlx5dr_match_param { 814 struct mlx5dr_match_spec outer; 815 struct mlx5dr_match_misc misc; 816 struct mlx5dr_match_spec inner; 817 struct mlx5dr_match_misc2 misc2; 818 struct mlx5dr_match_misc3 misc3; 819 struct mlx5dr_match_misc4 misc4; 820 struct mlx5dr_match_misc5 misc5; 821 }; 822 823 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \ 824 (_misc3)->icmpv4_code || \ 825 (_misc3)->icmpv4_header_data) 826 827 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \ 828 (_spec)->src_ip_95_64 || \ 829 (_spec)->src_ip_63_32 || \ 830 (_spec)->src_ip_31_0) 831 832 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \ 833 (_spec)->dst_ip_95_64 || \ 834 (_spec)->dst_ip_63_32 || \ 835 (_spec)->dst_ip_31_0) 836 837 struct mlx5dr_esw_caps { 838 u64 drop_icm_address_rx; 839 u64 drop_icm_address_tx; 840 u64 uplink_icm_address_rx; 841 u64 uplink_icm_address_tx; 842 u8 sw_owner:1; 843 u8 sw_owner_v2:1; 844 }; 845 846 struct mlx5dr_cmd_vport_cap { 847 u16 vport_gvmi; 848 u16 vhca_gvmi; 849 u16 num; 850 u64 icm_address_rx; 851 u64 icm_address_tx; 852 }; 853 854 struct mlx5dr_roce_cap { 855 u8 roce_en:1; 856 u8 fl_rc_qp_when_roce_disabled:1; 857 u8 fl_rc_qp_when_roce_enabled:1; 858 }; 859 860 struct mlx5dr_vports { 861 struct mlx5dr_cmd_vport_cap esw_manager_caps; 862 struct mlx5dr_cmd_vport_cap uplink_caps; 863 struct xarray vports_caps_xa; 864 }; 865 866 struct mlx5dr_cmd_caps { 867 u16 gvmi; 868 u64 nic_rx_drop_address; 869 u64 nic_tx_drop_address; 870 u64 nic_tx_allow_address; 871 u64 esw_rx_drop_address; 872 u64 esw_tx_drop_address; 873 u32 log_icm_size; 874 u64 hdr_modify_icm_addr; 875 u32 log_modify_pattern_icm_size; 876 u64 hdr_modify_pattern_icm_addr; 877 u32 flex_protocols; 878 u8 flex_parser_id_icmp_dw0; 879 u8 flex_parser_id_icmp_dw1; 880 u8 flex_parser_id_icmpv6_dw0; 881 u8 flex_parser_id_icmpv6_dw1; 882 u8 flex_parser_id_geneve_tlv_option_0; 883 u8 flex_parser_id_mpls_over_gre; 884 u8 flex_parser_id_mpls_over_udp; 885 u8 flex_parser_id_gtpu_dw_0; 886 u8 flex_parser_id_gtpu_teid; 887 u8 flex_parser_id_gtpu_dw_2; 888 u8 flex_parser_id_gtpu_first_ext_dw_0; 889 u8 flex_parser_ok_bits_supp; 890 u8 max_ft_level; 891 u16 roce_min_src_udp; 892 u8 sw_format_ver; 893 bool eswitch_manager; 894 bool rx_sw_owner; 895 bool tx_sw_owner; 896 bool fdb_sw_owner; 897 u8 rx_sw_owner_v2:1; 898 u8 tx_sw_owner_v2:1; 899 u8 fdb_sw_owner_v2:1; 900 struct mlx5dr_esw_caps esw_caps; 901 struct mlx5dr_vports vports; 902 bool prio_tag_required; 903 struct mlx5dr_roce_cap roce_caps; 904 u16 log_header_modify_argument_granularity; 905 u16 log_header_modify_argument_max_alloc; 906 bool support_modify_argument; 907 u8 is_ecpf:1; 908 u8 isolate_vl_tc:1; 909 }; 910 911 enum mlx5dr_domain_nic_type { 912 DR_DOMAIN_NIC_TYPE_RX, 913 DR_DOMAIN_NIC_TYPE_TX, 914 }; 915 916 struct mlx5dr_domain_rx_tx { 917 u64 drop_icm_addr; 918 u64 default_icm_addr; 919 enum mlx5dr_domain_nic_type type; 920 struct mutex mutex; /* protect rx/tx domain */ 921 }; 922 923 struct mlx5dr_domain_info { 924 bool supp_sw_steering; 925 u32 max_inline_size; 926 u32 max_send_wr; 927 u32 max_log_sw_icm_sz; 928 u32 max_log_action_icm_sz; 929 u32 max_log_modify_hdr_pattern_icm_sz; 930 struct mlx5dr_domain_rx_tx rx; 931 struct mlx5dr_domain_rx_tx tx; 932 struct mlx5dr_cmd_caps caps; 933 }; 934 935 struct mlx5dr_domain { 936 struct mlx5dr_domain *peer_dmn; 937 struct mlx5_core_dev *mdev; 938 u32 pdn; 939 struct mlx5_uars_page *uar; 940 enum mlx5dr_domain_type type; 941 refcount_t refcount; 942 struct mlx5dr_icm_pool *ste_icm_pool; 943 struct mlx5dr_icm_pool *action_icm_pool; 944 struct mlx5dr_send_info_pool *send_info_pool_rx; 945 struct mlx5dr_send_info_pool *send_info_pool_tx; 946 struct kmem_cache *chunks_kmem_cache; 947 struct kmem_cache *htbls_kmem_cache; 948 struct mlx5dr_ptrn_mgr *ptrn_mgr; 949 struct mlx5dr_arg_mgr *arg_mgr; 950 struct mlx5dr_send_ring *send_ring; 951 struct mlx5dr_domain_info info; 952 struct xarray csum_fts_xa; 953 struct mlx5dr_ste_ctx *ste_ctx; 954 struct list_head dbg_tbl_list; 955 struct mlx5dr_dbg_dump_info dump_info; 956 struct xarray definers_xa; 957 }; 958 959 struct mlx5dr_table_rx_tx { 960 struct mlx5dr_ste_htbl *s_anchor; 961 struct mlx5dr_domain_rx_tx *nic_dmn; 962 u64 default_icm_addr; 963 struct list_head nic_matcher_list; 964 }; 965 966 struct mlx5dr_table { 967 struct mlx5dr_domain *dmn; 968 struct mlx5dr_table_rx_tx rx; 969 struct mlx5dr_table_rx_tx tx; 970 u32 level; 971 u32 table_type; 972 u32 table_id; 973 u32 flags; 974 struct list_head matcher_list; 975 struct mlx5dr_action *miss_action; 976 refcount_t refcount; 977 struct list_head dbg_node; 978 }; 979 980 struct mlx5dr_matcher_rx_tx { 981 struct mlx5dr_ste_htbl *s_htbl; 982 struct mlx5dr_ste_htbl *e_anchor; 983 struct mlx5dr_ste_build *ste_builder; 984 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX] 985 [DR_RULE_IPV_MAX] 986 [DR_RULE_MAX_STES]; 987 u8 num_of_builders; 988 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX]; 989 u64 default_icm_addr; 990 struct mlx5dr_table_rx_tx *nic_tbl; 991 u32 prio; 992 struct list_head list_node; 993 u32 rules; 994 }; 995 996 struct mlx5dr_matcher { 997 struct mlx5dr_table *tbl; 998 struct mlx5dr_matcher_rx_tx rx; 999 struct mlx5dr_matcher_rx_tx tx; 1000 struct list_head list_node; /* Used for both matchers and dbg managing */ 1001 u32 prio; 1002 struct mlx5dr_match_param mask; 1003 u8 match_criteria; 1004 refcount_t refcount; 1005 struct list_head dbg_rule_list; 1006 }; 1007 1008 struct mlx5dr_ste_action_modify_field { 1009 u16 hw_field; 1010 u8 start; 1011 u8 end; 1012 u8 l3_type; 1013 u8 l4_type; 1014 }; 1015 1016 struct mlx5dr_ptrn_obj { 1017 struct mlx5dr_icm_chunk *chunk; 1018 u8 *data; 1019 u16 num_of_actions; 1020 u32 index; 1021 refcount_t refcount; 1022 struct list_head list; 1023 }; 1024 1025 struct mlx5dr_arg_obj { 1026 u32 obj_id; 1027 u32 obj_offset; 1028 struct list_head list_node; 1029 u32 log_chunk_size; 1030 }; 1031 1032 struct mlx5dr_action_rewrite { 1033 struct mlx5dr_domain *dmn; 1034 struct mlx5dr_icm_chunk *chunk; 1035 u8 *data; 1036 u16 num_of_actions; 1037 u32 index; 1038 u8 allow_rx:1; 1039 u8 allow_tx:1; 1040 u8 modify_ttl:1; 1041 struct mlx5dr_ptrn_obj *ptrn; 1042 struct mlx5dr_arg_obj *arg; 1043 }; 1044 1045 struct mlx5dr_action_reformat { 1046 struct mlx5dr_domain *dmn; 1047 u32 id; 1048 u32 size; 1049 u8 param_0; 1050 u8 param_1; 1051 }; 1052 1053 struct mlx5dr_action_sampler { 1054 struct mlx5dr_domain *dmn; 1055 u64 rx_icm_addr; 1056 u64 tx_icm_addr; 1057 u32 sampler_id; 1058 }; 1059 1060 struct mlx5dr_action_dest_tbl { 1061 u8 is_fw_tbl:1; 1062 union { 1063 struct mlx5dr_table *tbl; 1064 struct { 1065 struct mlx5dr_domain *dmn; 1066 u32 id; 1067 u32 group_id; 1068 enum fs_flow_table_type type; 1069 u64 rx_icm_addr; 1070 u64 tx_icm_addr; 1071 struct mlx5dr_action **ref_actions; 1072 u32 num_of_ref_actions; 1073 } fw_tbl; 1074 }; 1075 }; 1076 1077 struct mlx5dr_action_range { 1078 struct mlx5dr_domain *dmn; 1079 struct mlx5dr_action *hit_tbl_action; 1080 struct mlx5dr_action *miss_tbl_action; 1081 u32 definer_id; 1082 u32 min; 1083 u32 max; 1084 }; 1085 1086 struct mlx5dr_action_ctr { 1087 u32 ctr_id; 1088 u32 offset; 1089 }; 1090 1091 struct mlx5dr_action_vport { 1092 struct mlx5dr_domain *dmn; 1093 struct mlx5dr_cmd_vport_cap *caps; 1094 }; 1095 1096 struct mlx5dr_action_push_vlan { 1097 u32 vlan_hdr; /* tpid_pcp_dei_vid */ 1098 }; 1099 1100 struct mlx5dr_action_flow_tag { 1101 u32 flow_tag; 1102 }; 1103 1104 struct mlx5dr_rule_action_member { 1105 struct mlx5dr_action *action; 1106 struct list_head list; 1107 }; 1108 1109 struct mlx5dr_action_aso_flow_meter { 1110 struct mlx5dr_domain *dmn; 1111 u32 obj_id; 1112 u32 offset; 1113 u8 dest_reg_id; 1114 u8 init_color; 1115 }; 1116 1117 struct mlx5dr_action { 1118 enum mlx5dr_action_type action_type; 1119 refcount_t refcount; 1120 1121 union { 1122 void *data; 1123 struct mlx5dr_action_rewrite *rewrite; 1124 struct mlx5dr_action_reformat *reformat; 1125 struct mlx5dr_action_sampler *sampler; 1126 struct mlx5dr_action_dest_tbl *dest_tbl; 1127 struct mlx5dr_action_ctr *ctr; 1128 struct mlx5dr_action_vport *vport; 1129 struct mlx5dr_action_push_vlan *push_vlan; 1130 struct mlx5dr_action_flow_tag *flow_tag; 1131 struct mlx5dr_action_aso_flow_meter *aso; 1132 struct mlx5dr_action_range *range; 1133 }; 1134 }; 1135 1136 enum mlx5dr_connect_type { 1137 CONNECT_HIT = 1, 1138 CONNECT_MISS = 2, 1139 }; 1140 1141 struct mlx5dr_htbl_connect_info { 1142 enum mlx5dr_connect_type type; 1143 union { 1144 struct mlx5dr_ste_htbl *hit_next_htbl; 1145 u64 miss_icm_addr; 1146 }; 1147 }; 1148 1149 struct mlx5dr_rule_rx_tx { 1150 struct mlx5dr_matcher_rx_tx *nic_matcher; 1151 struct mlx5dr_ste *last_rule_ste; 1152 }; 1153 1154 struct mlx5dr_rule { 1155 struct mlx5dr_matcher *matcher; 1156 struct mlx5dr_rule_rx_tx rx; 1157 struct mlx5dr_rule_rx_tx tx; 1158 struct list_head rule_actions_list; 1159 struct list_head dbg_node; 1160 u32 flow_source; 1161 }; 1162 1163 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule, 1164 struct mlx5dr_ste *ste, 1165 bool force); 1166 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr, 1167 struct mlx5dr_ste *curr_ste, 1168 int *num_of_stes); 1169 1170 struct mlx5dr_icm_chunk { 1171 struct mlx5dr_icm_buddy_mem *buddy_mem; 1172 1173 /* indicates the index of this chunk in the whole memory, 1174 * used for deleting the chunk from the buddy 1175 */ 1176 unsigned int seg; 1177 enum mlx5dr_icm_chunk_size size; 1178 1179 /* Memory optimisation */ 1180 struct mlx5dr_ste *ste_arr; 1181 u8 *hw_ste_arr; 1182 struct list_head *miss_list; 1183 }; 1184 1185 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn) 1186 { 1187 mutex_lock(&nic_dmn->mutex); 1188 } 1189 1190 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn) 1191 { 1192 mutex_unlock(&nic_dmn->mutex); 1193 } 1194 1195 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn) 1196 { 1197 mlx5dr_domain_nic_lock(&dmn->info.rx); 1198 mlx5dr_domain_nic_lock(&dmn->info.tx); 1199 } 1200 1201 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn) 1202 { 1203 mlx5dr_domain_nic_unlock(&dmn->info.tx); 1204 mlx5dr_domain_nic_unlock(&dmn->info.rx); 1205 } 1206 1207 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn, 1208 struct mlx5dr_matcher_rx_tx *nic_matcher); 1209 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn, 1210 struct mlx5dr_matcher_rx_tx *nic_matcher); 1211 1212 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher, 1213 struct mlx5dr_matcher_rx_tx *nic_matcher, 1214 enum mlx5dr_ipv outer_ipv, 1215 enum mlx5dr_ipv inner_ipv); 1216 1217 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk); 1218 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk); 1219 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk); 1220 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk); 1221 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk); 1222 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste); 1223 1224 struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool); 1225 void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl); 1226 1227 static inline int 1228 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type) 1229 { 1230 if (icm_type == DR_ICM_TYPE_STE) 1231 return DR_STE_SIZE; 1232 1233 return DR_MODIFY_ACTION_SIZE; 1234 } 1235 1236 static inline u32 1237 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size) 1238 { 1239 return 1 << chunk_size; 1240 } 1241 1242 static inline int 1243 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size, 1244 enum mlx5dr_icm_type icm_type) 1245 { 1246 int num_of_entries; 1247 int entry_size; 1248 1249 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type); 1250 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size); 1251 1252 return entry_size * num_of_entries; 1253 } 1254 1255 static inline int 1256 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl) 1257 { 1258 int num_of_entries = 1259 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size); 1260 1261 /* Threshold is 50%, one is added to table of size 1 */ 1262 return (num_of_entries + 1) / 2; 1263 } 1264 1265 static inline bool 1266 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl) 1267 { 1268 if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask) 1269 return false; 1270 1271 return true; 1272 } 1273 1274 struct mlx5dr_cmd_vport_cap * 1275 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport); 1276 1277 struct mlx5dr_cmd_query_flow_table_details { 1278 u8 status; 1279 u8 level; 1280 u64 sw_owner_icm_root_1; 1281 u64 sw_owner_icm_root_0; 1282 }; 1283 1284 struct mlx5dr_cmd_create_flow_table_attr { 1285 u32 table_type; 1286 u16 uid; 1287 u64 icm_addr_rx; 1288 u64 icm_addr_tx; 1289 u8 level; 1290 bool sw_owner; 1291 bool term_tbl; 1292 bool decap_en; 1293 bool reformat_en; 1294 }; 1295 1296 /* internal API functions */ 1297 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, 1298 struct mlx5dr_cmd_caps *caps); 1299 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev, 1300 bool other_vport, u16 vport_number, 1301 u64 *icm_address_rx, 1302 u64 *icm_address_tx); 1303 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev, 1304 bool other_vport, u16 vport_number, u16 *gvmi); 1305 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev, 1306 struct mlx5dr_esw_caps *caps); 1307 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev, 1308 u32 sampler_id, 1309 u64 *rx_icm_addr, 1310 u64 *tx_icm_addr); 1311 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev); 1312 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev, 1313 u32 table_type, 1314 u32 table_id, 1315 u32 group_id, 1316 u32 modify_header_id, 1317 u16 vport_id); 1318 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev, 1319 u32 table_type, 1320 u32 table_id); 1321 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev, 1322 u32 table_type, 1323 u8 num_of_actions, 1324 u64 *actions, 1325 u32 *modify_header_id); 1326 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev, 1327 u32 modify_header_id); 1328 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev, 1329 u32 table_type, 1330 u32 table_id, 1331 u32 *group_id); 1332 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev, 1333 u32 table_type, 1334 u32 table_id, 1335 u32 group_id); 1336 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev, 1337 struct mlx5dr_cmd_create_flow_table_attr *attr, 1338 u64 *fdb_rx_icm_addr, 1339 u32 *table_id); 1340 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev, 1341 u32 table_id, 1342 u32 table_type); 1343 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev, 1344 enum fs_flow_table_type type, 1345 u32 table_id, 1346 struct mlx5dr_cmd_query_flow_table_details *output); 1347 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, 1348 enum mlx5_reformat_ctx_type rt, 1349 u8 reformat_param_0, 1350 u8 reformat_param_1, 1351 size_t reformat_size, 1352 void *reformat_data, 1353 u32 *reformat_id); 1354 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev, 1355 u32 reformat_id); 1356 int mlx5dr_cmd_create_definer(struct mlx5_core_dev *mdev, 1357 u16 format_id, 1358 u8 *dw_selectors, 1359 u8 *byte_selectors, 1360 u8 *match_mask, 1361 u32 *definer_id); 1362 void mlx5dr_cmd_destroy_definer(struct mlx5_core_dev *mdev, 1363 u32 definer_id); 1364 1365 struct mlx5dr_cmd_gid_attr { 1366 u8 gid[16]; 1367 u8 mac[6]; 1368 u32 roce_ver; 1369 }; 1370 1371 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num, 1372 u16 index, struct mlx5dr_cmd_gid_attr *attr); 1373 1374 int mlx5dr_cmd_create_modify_header_arg(struct mlx5_core_dev *dev, 1375 u16 log_obj_range, u32 pd, 1376 u32 *obj_id); 1377 void mlx5dr_cmd_destroy_modify_header_arg(struct mlx5_core_dev *dev, 1378 u32 obj_id); 1379 1380 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 1381 enum mlx5dr_icm_type icm_type); 1382 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool); 1383 1384 struct mlx5dr_icm_chunk * 1385 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 1386 enum mlx5dr_icm_chunk_size chunk_size); 1387 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk); 1388 1389 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx, 1390 u8 *hw_ste_p, u32 ste_size); 1391 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn, 1392 struct mlx5dr_domain_rx_tx *nic_dmn, 1393 struct mlx5dr_ste_htbl *htbl, 1394 struct mlx5dr_htbl_connect_info *connect_info, 1395 bool update_hw_ste); 1396 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx, 1397 u16 gvmi, 1398 enum mlx5dr_domain_nic_type nic_type, 1399 struct mlx5dr_ste_htbl *htbl, 1400 u8 *formatted_ste, 1401 struct mlx5dr_htbl_connect_info *connect_info); 1402 void mlx5dr_ste_copy_param(u8 match_criteria, 1403 struct mlx5dr_match_param *set_param, 1404 struct mlx5dr_match_parameters *mask, 1405 bool clear); 1406 1407 struct mlx5dr_qp { 1408 struct mlx5_core_dev *mdev; 1409 struct mlx5_wq_qp wq; 1410 struct mlx5_uars_page *uar; 1411 struct mlx5_wq_ctrl wq_ctrl; 1412 u32 qpn; 1413 struct { 1414 unsigned int head; 1415 unsigned int pc; 1416 unsigned int cc; 1417 unsigned int size; 1418 unsigned int *wqe_head; 1419 unsigned int wqe_cnt; 1420 } sq; 1421 struct { 1422 unsigned int pc; 1423 unsigned int cc; 1424 unsigned int size; 1425 unsigned int wqe_cnt; 1426 } rq; 1427 int max_inline_data; 1428 }; 1429 1430 struct mlx5dr_cq { 1431 struct mlx5_core_dev *mdev; 1432 struct mlx5_cqwq wq; 1433 struct mlx5_wq_ctrl wq_ctrl; 1434 struct mlx5_core_cq mcq; 1435 struct mlx5dr_qp *qp; 1436 }; 1437 1438 struct mlx5dr_mr { 1439 struct mlx5_core_dev *mdev; 1440 u32 mkey; 1441 dma_addr_t dma_addr; 1442 void *addr; 1443 size_t size; 1444 }; 1445 1446 struct mlx5dr_send_ring { 1447 struct mlx5dr_cq *cq; 1448 struct mlx5dr_qp *qp; 1449 struct mlx5dr_mr *mr; 1450 /* How much wqes are waiting for completion */ 1451 u32 pending_wqe; 1452 /* Signal request per this trash hold value */ 1453 u16 signal_th; 1454 /* Each post_send_size less than max_post_send_size */ 1455 u32 max_post_send_size; 1456 /* manage the send queue */ 1457 u32 tx_head; 1458 void *buf; 1459 u32 buf_size; 1460 u8 *sync_buff; 1461 struct mlx5dr_mr *sync_mr; 1462 spinlock_t lock; /* Protect the data path of the send ring */ 1463 bool err_state; /* send_ring is not usable in err state */ 1464 }; 1465 1466 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn); 1467 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn, 1468 struct mlx5dr_send_ring *send_ring); 1469 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn); 1470 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn, 1471 struct mlx5dr_ste *ste, 1472 u8 *data, 1473 u16 size, 1474 u16 offset); 1475 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, 1476 struct mlx5dr_ste_htbl *htbl, 1477 u8 *formatted_ste, u8 *mask); 1478 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn, 1479 struct mlx5dr_ste_htbl *htbl, 1480 u8 *ste_init_data, 1481 bool update_hw_ste); 1482 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn, 1483 struct mlx5dr_action *action); 1484 int mlx5dr_send_postsend_pattern(struct mlx5dr_domain *dmn, 1485 struct mlx5dr_icm_chunk *chunk, 1486 u16 num_of_actions, 1487 u8 *data); 1488 int mlx5dr_send_postsend_args(struct mlx5dr_domain *dmn, u64 arg_id, 1489 u16 num_of_actions, u8 *actions_data); 1490 1491 int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn); 1492 void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn); 1493 struct mlx5dr_ste_send_info *mlx5dr_send_info_alloc(struct mlx5dr_domain *dmn, 1494 enum mlx5dr_domain_nic_type nic_type); 1495 void mlx5dr_send_info_free(struct mlx5dr_ste_send_info *ste_send_info); 1496 1497 struct mlx5dr_cmd_ft_info { 1498 u32 id; 1499 u16 vport; 1500 enum fs_flow_table_type type; 1501 }; 1502 1503 struct mlx5dr_cmd_flow_destination_hw_info { 1504 enum mlx5_flow_destination_type type; 1505 union { 1506 u32 tir_num; 1507 u32 ft_num; 1508 u32 ft_id; 1509 u32 counter_id; 1510 u32 sampler_id; 1511 struct { 1512 u16 num; 1513 u16 vhca_id; 1514 u32 reformat_id; 1515 u8 flags; 1516 } vport; 1517 }; 1518 }; 1519 1520 struct mlx5dr_cmd_fte_info { 1521 u32 dests_size; 1522 u32 index; 1523 struct mlx5_flow_context flow_context; 1524 u32 *val; 1525 struct mlx5_flow_act action; 1526 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr; 1527 bool ignore_flow_level; 1528 }; 1529 1530 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev, 1531 int opmod, int modify_mask, 1532 struct mlx5dr_cmd_ft_info *ft, 1533 u32 group_id, 1534 struct mlx5dr_cmd_fte_info *fte); 1535 1536 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps); 1537 1538 struct mlx5dr_fw_recalc_cs_ft { 1539 u64 rx_icm_addr; 1540 u32 table_id; 1541 u32 group_id; 1542 u32 modify_hdr_id; 1543 }; 1544 1545 struct mlx5dr_fw_recalc_cs_ft * 1546 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num); 1547 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn, 1548 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft); 1549 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn, 1550 u16 vport_num, 1551 u64 *rx_icm_addr); 1552 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn, 1553 struct mlx5dr_cmd_flow_destination_hw_info *dest, 1554 int num_dest, 1555 bool reformat_req, 1556 u32 *tbl_id, 1557 u32 *group_id, 1558 bool ignore_flow_level, 1559 u32 flow_source); 1560 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id, 1561 u32 group_id); 1562 1563 static inline bool mlx5dr_is_fw_table(struct mlx5_flow_table *ft) 1564 { 1565 return !ft->fs_dr_table.dr_table; 1566 } 1567 1568 static inline bool mlx5dr_supp_match_ranges(struct mlx5_core_dev *dev) 1569 { 1570 return (MLX5_CAP_GEN(dev, steering_format_version) >= 1571 MLX5_STEERING_FORMAT_CONNECTX_6DX) && 1572 (MLX5_CAP_GEN_64(dev, match_definer_format_supported) & 1573 (1ULL << MLX5_IFC_DEFINER_FORMAT_ID_SELECT)); 1574 } 1575 1576 bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn); 1577 struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn); 1578 void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr); 1579 struct mlx5dr_ptrn_obj *mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr, 1580 u16 num_of_actions, u8 *data); 1581 void mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr, 1582 struct mlx5dr_ptrn_obj *pattern); 1583 struct mlx5dr_arg_mgr *mlx5dr_arg_mgr_create(struct mlx5dr_domain *dmn); 1584 void mlx5dr_arg_mgr_destroy(struct mlx5dr_arg_mgr *mgr); 1585 struct mlx5dr_arg_obj *mlx5dr_arg_get_obj(struct mlx5dr_arg_mgr *mgr, 1586 u16 num_of_actions, 1587 u8 *data); 1588 void mlx5dr_arg_put_obj(struct mlx5dr_arg_mgr *mgr, 1589 struct mlx5dr_arg_obj *arg_obj); 1590 u32 mlx5dr_arg_get_obj_id(struct mlx5dr_arg_obj *arg_obj); 1591 1592 #endif /* _DR_TYPES_H_ */ 1593