1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2019, Mellanox Technologies */ 3 4 #ifndef _DR_TYPES_ 5 #define _DR_TYPES_ 6 7 #include <linux/mlx5/vport.h> 8 #include <linux/refcount.h> 9 #include "fs_core.h" 10 #include "wq.h" 11 #include "lib/mlx5.h" 12 #include "mlx5_ifc_dr.h" 13 #include "mlx5dr.h" 14 #include "dr_dbg.h" 15 16 #define DR_RULE_MAX_STES 18 17 #define DR_ACTION_MAX_STES 5 18 #define DR_STE_SVLAN 0x1 19 #define DR_STE_CVLAN 0x2 20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4) 21 #define DR_NUM_OF_FLEX_PARSERS 8 22 #define DR_STE_MAX_FLEX_0_ID 3 23 #define DR_STE_MAX_FLEX_1_ID 7 24 #define DR_ACTION_CACHE_LINE_SIZE 64 25 26 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg) 27 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg) 28 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg) 29 30 struct mlx5dr_ptrn_mgr; 31 struct mlx5dr_arg_mgr; 32 struct mlx5dr_arg_obj; 33 34 static inline bool dr_is_flex_parser_0_id(u8 parser_id) 35 { 36 return parser_id <= DR_STE_MAX_FLEX_0_ID; 37 } 38 39 static inline bool dr_is_flex_parser_1_id(u8 parser_id) 40 { 41 return parser_id > DR_STE_MAX_FLEX_0_ID; 42 } 43 44 enum mlx5dr_icm_chunk_size { 45 DR_CHUNK_SIZE_1, 46 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */ 47 DR_CHUNK_SIZE_2, 48 DR_CHUNK_SIZE_4, 49 DR_CHUNK_SIZE_8, 50 DR_CHUNK_SIZE_16, 51 DR_CHUNK_SIZE_32, 52 DR_CHUNK_SIZE_64, 53 DR_CHUNK_SIZE_128, 54 DR_CHUNK_SIZE_256, 55 DR_CHUNK_SIZE_512, 56 DR_CHUNK_SIZE_1K, 57 DR_CHUNK_SIZE_2K, 58 DR_CHUNK_SIZE_4K, 59 DR_CHUNK_SIZE_8K, 60 DR_CHUNK_SIZE_16K, 61 DR_CHUNK_SIZE_32K, 62 DR_CHUNK_SIZE_64K, 63 DR_CHUNK_SIZE_128K, 64 DR_CHUNK_SIZE_256K, 65 DR_CHUNK_SIZE_512K, 66 DR_CHUNK_SIZE_1024K, 67 DR_CHUNK_SIZE_2048K, 68 DR_CHUNK_SIZE_MAX, 69 }; 70 71 enum mlx5dr_icm_type { 72 DR_ICM_TYPE_STE, 73 DR_ICM_TYPE_MODIFY_ACTION, 74 DR_ICM_TYPE_MODIFY_HDR_PTRN, 75 DR_ICM_TYPE_MAX, 76 }; 77 78 static inline enum mlx5dr_icm_chunk_size 79 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk) 80 { 81 chunk += 2; 82 if (chunk < DR_CHUNK_SIZE_MAX) 83 return chunk; 84 85 return DR_CHUNK_SIZE_MAX; 86 } 87 88 enum { 89 DR_STE_SIZE = 64, 90 DR_STE_SIZE_CTRL = 32, 91 DR_STE_SIZE_MATCH_TAG = 32, 92 DR_STE_SIZE_TAG = 16, 93 DR_STE_SIZE_MASK = 16, 94 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK, 95 }; 96 97 enum mlx5dr_ste_ctx_action_cap { 98 DR_STE_CTX_ACTION_CAP_NONE = 0, 99 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0, 100 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1, 101 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2, 102 DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3, 103 }; 104 105 enum { 106 DR_MODIFY_ACTION_SIZE = 8, 107 }; 108 109 enum mlx5dr_matcher_criteria { 110 DR_MATCHER_CRITERIA_EMPTY = 0, 111 DR_MATCHER_CRITERIA_OUTER = 1 << 0, 112 DR_MATCHER_CRITERIA_MISC = 1 << 1, 113 DR_MATCHER_CRITERIA_INNER = 1 << 2, 114 DR_MATCHER_CRITERIA_MISC2 = 1 << 3, 115 DR_MATCHER_CRITERIA_MISC3 = 1 << 4, 116 DR_MATCHER_CRITERIA_MISC4 = 1 << 5, 117 DR_MATCHER_CRITERIA_MISC5 = 1 << 6, 118 DR_MATCHER_CRITERIA_MAX = 1 << 7, 119 }; 120 121 enum mlx5dr_action_type { 122 DR_ACTION_TYP_TNL_L2_TO_L2, 123 DR_ACTION_TYP_L2_TO_TNL_L2, 124 DR_ACTION_TYP_TNL_L3_TO_L2, 125 DR_ACTION_TYP_L2_TO_TNL_L3, 126 DR_ACTION_TYP_DROP, 127 DR_ACTION_TYP_QP, 128 DR_ACTION_TYP_FT, 129 DR_ACTION_TYP_CTR, 130 DR_ACTION_TYP_TAG, 131 DR_ACTION_TYP_MODIFY_HDR, 132 DR_ACTION_TYP_VPORT, 133 DR_ACTION_TYP_POP_VLAN, 134 DR_ACTION_TYP_PUSH_VLAN, 135 DR_ACTION_TYP_INSERT_HDR, 136 DR_ACTION_TYP_REMOVE_HDR, 137 DR_ACTION_TYP_SAMPLER, 138 DR_ACTION_TYP_ASO_FLOW_METER, 139 DR_ACTION_TYP_RANGE, 140 DR_ACTION_TYP_MAX, 141 }; 142 143 enum mlx5dr_ipv { 144 DR_RULE_IPV4, 145 DR_RULE_IPV6, 146 DR_RULE_IPV_MAX, 147 }; 148 149 struct mlx5dr_icm_pool; 150 struct mlx5dr_icm_chunk; 151 struct mlx5dr_icm_buddy_mem; 152 struct mlx5dr_ste_htbl; 153 struct mlx5dr_match_param; 154 struct mlx5dr_cmd_caps; 155 struct mlx5dr_rule_rx_tx; 156 struct mlx5dr_matcher_rx_tx; 157 struct mlx5dr_ste_ctx; 158 struct mlx5dr_send_info_pool; 159 struct mlx5dr_icm_hot_chunk; 160 161 struct mlx5dr_ste { 162 /* refcount: indicates the num of rules that using this ste */ 163 u32 refcount; 164 165 /* this ste is part of a rule, located in ste's chain */ 166 u8 ste_chain_location; 167 168 /* attached to the miss_list head at each htbl entry */ 169 struct list_head miss_list_node; 170 171 /* this ste is member of htbl */ 172 struct mlx5dr_ste_htbl *htbl; 173 174 struct mlx5dr_ste_htbl *next_htbl; 175 176 /* The rule this STE belongs to */ 177 struct mlx5dr_rule_rx_tx *rule_rx_tx; 178 }; 179 180 struct mlx5dr_ste_htbl_ctrl { 181 /* total number of valid entries belonging to this hash table. This 182 * includes the non collision and collision entries 183 */ 184 unsigned int num_of_valid_entries; 185 186 /* total number of collisions entries attached to this table */ 187 unsigned int num_of_collisions; 188 }; 189 190 struct mlx5dr_ste_htbl { 191 u16 lu_type; 192 u16 byte_mask; 193 u32 refcount; 194 struct mlx5dr_icm_chunk *chunk; 195 struct mlx5dr_ste *pointing_ste; 196 struct mlx5dr_ste_htbl_ctrl ctrl; 197 }; 198 199 struct mlx5dr_ste_send_info { 200 struct mlx5dr_ste *ste; 201 struct list_head send_list; 202 u16 size; 203 u16 offset; 204 u8 data_cont[DR_STE_SIZE]; 205 u8 *data; 206 }; 207 208 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size, 209 u16 offset, u8 *data, 210 struct mlx5dr_ste_send_info *ste_info, 211 struct list_head *send_list, 212 bool copy_data); 213 214 struct mlx5dr_ste_build { 215 u8 inner:1; 216 u8 rx:1; 217 u8 vhca_id_valid:1; 218 struct mlx5dr_domain *dmn; 219 struct mlx5dr_cmd_caps *caps; 220 u16 lu_type; 221 u16 byte_mask; 222 u8 bit_mask[DR_STE_SIZE_MASK]; 223 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec, 224 struct mlx5dr_ste_build *sb, 225 u8 *tag); 226 }; 227 228 struct mlx5dr_ste_htbl * 229 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool, 230 enum mlx5dr_icm_chunk_size chunk_size, 231 u16 lu_type, u16 byte_mask); 232 233 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl); 234 235 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl) 236 { 237 htbl->refcount--; 238 if (!htbl->refcount) 239 mlx5dr_ste_htbl_free(htbl); 240 } 241 242 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl) 243 { 244 htbl->refcount++; 245 } 246 247 /* STE utils */ 248 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl); 249 bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste_p); 250 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, 251 u8 *hw_ste, u64 miss_addr); 252 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx, 253 u8 *hw_ste, u64 icm_addr, u32 ht_size); 254 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx, 255 u8 *hw_ste, 256 struct mlx5dr_ste_htbl *next_htbl); 257 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask); 258 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher, 259 u8 ste_location); 260 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste); 261 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste); 262 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste); 263 264 #define MLX5DR_MAX_VLANS 2 265 #define MLX5DR_INVALID_PATTERN_INDEX 0xffffffff 266 267 struct mlx5dr_ste_actions_attr { 268 u32 modify_index; 269 u32 modify_pat_idx; 270 u16 modify_actions; 271 u8 *single_modify_action; 272 u32 decap_index; 273 u32 decap_pat_idx; 274 u16 decap_actions; 275 u8 decap_with_vlan:1; 276 u64 final_icm_addr; 277 u32 flow_tag; 278 u32 ctr_id; 279 u16 gvmi; 280 u16 hit_gvmi; 281 struct { 282 u32 id; 283 u32 size; 284 u8 param_0; 285 u8 param_1; 286 } reformat; 287 struct { 288 int count; 289 u32 headers[MLX5DR_MAX_VLANS]; 290 } vlans; 291 292 struct { 293 u32 obj_id; 294 u32 offset; 295 u8 dest_reg_id; 296 u8 init_color; 297 } aso_flow_meter; 298 299 struct { 300 u64 miss_icm_addr; 301 u32 definer_id; 302 u32 min; 303 u32 max; 304 } range; 305 }; 306 307 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, 308 struct mlx5dr_domain *dmn, 309 u8 *action_type_set, 310 u8 *last_ste, 311 struct mlx5dr_ste_actions_attr *attr, 312 u32 *added_stes); 313 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx, 314 struct mlx5dr_domain *dmn, 315 u8 *action_type_set, 316 u8 *last_ste, 317 struct mlx5dr_ste_actions_attr *attr, 318 u32 *added_stes); 319 320 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx, 321 __be64 *hw_action, 322 u8 hw_field, 323 u8 shifter, 324 u8 length, 325 u32 data); 326 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx, 327 __be64 *hw_action, 328 u8 hw_field, 329 u8 shifter, 330 u8 length, 331 u32 data); 332 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx, 333 __be64 *hw_action, 334 u8 dst_hw_field, 335 u8 dst_shifter, 336 u8 dst_len, 337 u8 src_hw_field, 338 u8 src_shifter); 339 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx, 340 void *data, 341 u32 data_sz, 342 u8 *hw_action, 343 u32 hw_action_sz, 344 u16 *used_hw_action_num); 345 int mlx5dr_ste_alloc_modify_hdr(struct mlx5dr_action *action); 346 void mlx5dr_ste_free_modify_hdr(struct mlx5dr_action *action); 347 348 const struct mlx5dr_ste_action_modify_field * 349 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field); 350 351 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version); 352 void mlx5dr_ste_free(struct mlx5dr_ste *ste, 353 struct mlx5dr_matcher *matcher, 354 struct mlx5dr_matcher_rx_tx *nic_matcher); 355 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste, 356 struct mlx5dr_matcher *matcher, 357 struct mlx5dr_matcher_rx_tx *nic_matcher) 358 { 359 ste->refcount--; 360 if (!ste->refcount) 361 mlx5dr_ste_free(ste, matcher, nic_matcher); 362 } 363 364 /* initial as 0, increased only when ste appears in a new rule */ 365 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste) 366 { 367 ste->refcount++; 368 } 369 370 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste) 371 { 372 return !ste->refcount; 373 } 374 375 bool mlx5dr_ste_equal_tag(void *src, void *dst); 376 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher, 377 struct mlx5dr_matcher_rx_tx *nic_matcher, 378 struct mlx5dr_ste *ste, 379 u8 *cur_hw_ste, 380 enum mlx5dr_icm_chunk_size log_table_size); 381 382 /* STE build functions */ 383 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn, 384 u8 match_criteria, 385 struct mlx5dr_match_param *mask, 386 struct mlx5dr_match_param *value); 387 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, 388 struct mlx5dr_matcher_rx_tx *nic_matcher, 389 struct mlx5dr_match_param *value, 390 u8 *ste_arr); 391 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx, 392 struct mlx5dr_ste_build *builder, 393 struct mlx5dr_match_param *mask, 394 bool inner, bool rx); 395 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx, 396 struct mlx5dr_ste_build *sb, 397 struct mlx5dr_match_param *mask, 398 bool inner, bool rx); 399 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx, 400 struct mlx5dr_ste_build *sb, 401 struct mlx5dr_match_param *mask, 402 bool inner, bool rx); 403 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx, 404 struct mlx5dr_ste_build *sb, 405 struct mlx5dr_match_param *mask, 406 bool inner, bool rx); 407 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx, 408 struct mlx5dr_ste_build *sb, 409 struct mlx5dr_match_param *mask, 410 bool inner, bool rx); 411 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx, 412 struct mlx5dr_ste_build *sb, 413 struct mlx5dr_match_param *mask, 414 bool inner, bool rx); 415 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx, 416 struct mlx5dr_ste_build *sb, 417 struct mlx5dr_match_param *mask, 418 bool inner, bool rx); 419 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx, 420 struct mlx5dr_ste_build *sb, 421 struct mlx5dr_match_param *mask, 422 bool inner, bool rx); 423 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx, 424 struct mlx5dr_ste_build *sb, 425 struct mlx5dr_match_param *mask, 426 bool inner, bool rx); 427 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx, 428 struct mlx5dr_ste_build *sb, 429 struct mlx5dr_match_param *mask, 430 bool inner, bool rx); 431 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx, 432 struct mlx5dr_ste_build *sb, 433 struct mlx5dr_match_param *mask, 434 bool inner, bool rx); 435 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx, 436 struct mlx5dr_ste_build *sb, 437 struct mlx5dr_match_param *mask, 438 bool inner, bool rx); 439 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx, 440 struct mlx5dr_ste_build *sb, 441 struct mlx5dr_match_param *mask, 442 bool inner, bool rx); 443 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx, 444 struct mlx5dr_ste_build *sb, 445 struct mlx5dr_match_param *mask, 446 struct mlx5dr_cmd_caps *caps, 447 bool inner, bool rx); 448 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx, 449 struct mlx5dr_ste_build *sb, 450 struct mlx5dr_match_param *mask, 451 struct mlx5dr_cmd_caps *caps, 452 bool inner, bool rx); 453 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx, 454 struct mlx5dr_ste_build *sb, 455 struct mlx5dr_match_param *mask, 456 struct mlx5dr_cmd_caps *caps, 457 bool inner, bool rx); 458 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx, 459 struct mlx5dr_ste_build *sb, 460 struct mlx5dr_match_param *mask, 461 bool inner, bool rx); 462 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx, 463 struct mlx5dr_ste_build *sb, 464 struct mlx5dr_match_param *mask, 465 bool inner, bool rx); 466 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx, 467 struct mlx5dr_ste_build *sb, 468 struct mlx5dr_match_param *mask, 469 struct mlx5dr_cmd_caps *caps, 470 bool inner, bool rx); 471 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx, 472 struct mlx5dr_ste_build *sb, 473 struct mlx5dr_match_param *mask, 474 struct mlx5dr_cmd_caps *caps, 475 bool inner, bool rx); 476 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx, 477 struct mlx5dr_ste_build *sb, 478 struct mlx5dr_match_param *mask, 479 bool inner, bool rx); 480 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 481 struct mlx5dr_ste_build *sb, 482 struct mlx5dr_match_param *mask, 483 struct mlx5dr_cmd_caps *caps, 484 bool inner, bool rx); 485 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 486 struct mlx5dr_ste_build *sb, 487 struct mlx5dr_match_param *mask, 488 struct mlx5dr_cmd_caps *caps, 489 bool inner, bool rx); 490 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx, 491 struct mlx5dr_ste_build *sb, 492 struct mlx5dr_match_param *mask, 493 bool inner, bool rx); 494 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx, 495 struct mlx5dr_ste_build *sb, 496 struct mlx5dr_match_param *mask, 497 bool inner, bool rx); 498 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx, 499 struct mlx5dr_ste_build *sb, 500 struct mlx5dr_match_param *mask, 501 bool inner, bool rx); 502 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx, 503 struct mlx5dr_ste_build *sb, 504 struct mlx5dr_match_param *mask, 505 bool inner, bool rx); 506 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx, 507 struct mlx5dr_ste_build *sb, 508 struct mlx5dr_match_param *mask, 509 struct mlx5dr_domain *dmn, 510 bool inner, bool rx); 511 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 512 struct mlx5dr_ste_build *sb, 513 struct mlx5dr_match_param *mask, 514 bool inner, bool rx); 515 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 516 struct mlx5dr_ste_build *sb, 517 struct mlx5dr_match_param *mask, 518 bool inner, bool rx); 519 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx); 520 521 /* Actions utils */ 522 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, 523 struct mlx5dr_matcher_rx_tx *nic_matcher, 524 struct mlx5dr_action *actions[], 525 u32 num_actions, 526 u8 *ste_arr, 527 u32 *new_hw_ste_arr_sz); 528 529 struct mlx5dr_match_spec { 530 u32 smac_47_16; /* Source MAC address of incoming packet */ 531 /* Incoming packet Ethertype - this is the Ethertype 532 * following the last VLAN tag of the packet 533 */ 534 u32 smac_15_0:16; /* Source MAC address of incoming packet */ 535 u32 ethertype:16; 536 537 u32 dmac_47_16; /* Destination MAC address of incoming packet */ 538 539 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */ 540 /* Priority of first VLAN tag in the incoming packet. 541 * Valid only when cvlan_tag==1 or svlan_tag==1 542 */ 543 u32 first_prio:3; 544 /* CFI bit of first VLAN tag in the incoming packet. 545 * Valid only when cvlan_tag==1 or svlan_tag==1 546 */ 547 u32 first_cfi:1; 548 /* VLAN ID of first VLAN tag in the incoming packet. 549 * Valid only when cvlan_tag==1 or svlan_tag==1 550 */ 551 u32 first_vid:12; 552 553 u32 ip_protocol:8; /* IP protocol */ 554 /* Differentiated Services Code Point derived from 555 * Traffic Class/TOS field of IPv6/v4 556 */ 557 u32 ip_dscp:6; 558 /* Explicit Congestion Notification derived from 559 * Traffic Class/TOS field of IPv6/v4 560 */ 561 u32 ip_ecn:2; 562 /* The first vlan in the packet is c-vlan (0x8100). 563 * cvlan_tag and svlan_tag cannot be set together 564 */ 565 u32 cvlan_tag:1; 566 /* The first vlan in the packet is s-vlan (0x8a88). 567 * cvlan_tag and svlan_tag cannot be set together 568 */ 569 u32 svlan_tag:1; 570 u32 frag:1; /* Packet is an IP fragment */ 571 u32 ip_version:4; /* IP version */ 572 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK; 573 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS 574 */ 575 u32 tcp_flags:9; 576 577 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */ 578 u32 tcp_sport:16; 579 /* TCP destination port. 580 * tcp and udp sport/dport are mutually exclusive 581 */ 582 u32 tcp_dport:16; 583 584 u32 reserved_auto1:16; 585 u32 ipv4_ihl:4; 586 u32 reserved_auto2:4; 587 u32 ttl_hoplimit:8; 588 589 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */ 590 u32 udp_sport:16; 591 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */ 592 u32 udp_dport:16; 593 594 /* IPv6 source address of incoming packets 595 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 596 * This field should be qualified by an appropriate ethertype 597 */ 598 u32 src_ip_127_96; 599 /* IPv6 source address of incoming packets 600 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 601 * This field should be qualified by an appropriate ethertype 602 */ 603 u32 src_ip_95_64; 604 /* IPv6 source address of incoming packets 605 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 606 * This field should be qualified by an appropriate ethertype 607 */ 608 u32 src_ip_63_32; 609 /* IPv6 source address of incoming packets 610 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 611 * This field should be qualified by an appropriate ethertype 612 */ 613 u32 src_ip_31_0; 614 /* IPv6 destination address of incoming packets 615 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 616 * This field should be qualified by an appropriate ethertype 617 */ 618 u32 dst_ip_127_96; 619 /* IPv6 destination address of incoming packets 620 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 621 * This field should be qualified by an appropriate ethertype 622 */ 623 u32 dst_ip_95_64; 624 /* IPv6 destination address of incoming packets 625 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 626 * This field should be qualified by an appropriate ethertype 627 */ 628 u32 dst_ip_63_32; 629 /* IPv6 destination address of incoming packets 630 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 631 * This field should be qualified by an appropriate ethertype 632 */ 633 u32 dst_ip_31_0; 634 }; 635 636 struct mlx5dr_match_misc { 637 /* used with GRE, checksum exist when gre_c_present == 1 */ 638 u32 gre_c_present:1; 639 u32 reserved_auto1:1; 640 /* used with GRE, key exist when gre_k_present == 1 */ 641 u32 gre_k_present:1; 642 /* used with GRE, sequence number exist when gre_s_present == 1 */ 643 u32 gre_s_present:1; 644 u32 source_vhca_port:4; 645 u32 source_sqn:24; /* Source SQN */ 646 647 u32 source_eswitch_owner_vhca_id:16; 648 /* Source port.;0xffff determines wire port */ 649 u32 source_port:16; 650 651 /* Priority of second VLAN tag in the outer header of the incoming packet. 652 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 653 */ 654 u32 outer_second_prio:3; 655 /* CFI bit of first VLAN tag in the outer header of the incoming packet. 656 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 657 */ 658 u32 outer_second_cfi:1; 659 /* VLAN ID of first VLAN tag the outer header of the incoming packet. 660 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 661 */ 662 u32 outer_second_vid:12; 663 /* Priority of second VLAN tag in the inner header of the incoming packet. 664 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 665 */ 666 u32 inner_second_prio:3; 667 /* CFI bit of first VLAN tag in the inner header of the incoming packet. 668 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 669 */ 670 u32 inner_second_cfi:1; 671 /* VLAN ID of first VLAN tag the inner header of the incoming packet. 672 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 673 */ 674 u32 inner_second_vid:12; 675 676 u32 outer_second_cvlan_tag:1; 677 u32 inner_second_cvlan_tag:1; 678 /* The second vlan in the outer header of the packet is c-vlan (0x8100). 679 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 680 */ 681 u32 outer_second_svlan_tag:1; 682 /* The second vlan in the inner header of the packet is c-vlan (0x8100). 683 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 684 */ 685 u32 inner_second_svlan_tag:1; 686 /* The second vlan in the outer header of the packet is s-vlan (0x8a88). 687 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 688 */ 689 u32 reserved_auto2:12; 690 /* The second vlan in the inner header of the packet is s-vlan (0x8a88). 691 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 692 */ 693 u32 gre_protocol:16; /* GRE Protocol (outer) */ 694 695 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */ 696 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */ 697 698 u32 vxlan_vni:24; /* VXLAN VNI (outer) */ 699 u32 reserved_auto3:8; 700 701 u32 geneve_vni:24; /* GENEVE VNI field (outer) */ 702 u32 reserved_auto4:6; 703 u32 geneve_tlv_option_0_exist:1; 704 u32 geneve_oam:1; /* GENEVE OAM field (outer) */ 705 706 u32 reserved_auto5:12; 707 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */ 708 709 u32 reserved_auto6:12; 710 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */ 711 712 u32 reserved_auto7:10; 713 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */ 714 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */ 715 716 u32 reserved_auto8:8; 717 u32 bth_dst_qp:24; /* Destination QP in BTH header */ 718 719 u32 reserved_auto9; 720 u32 outer_esp_spi; 721 u32 reserved_auto10[3]; 722 }; 723 724 struct mlx5dr_match_misc2 { 725 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */ 726 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */ 727 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */ 728 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */ 729 730 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */ 731 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */ 732 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */ 733 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */ 734 735 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */ 736 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */ 737 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */ 738 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */ 739 740 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */ 741 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */ 742 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */ 743 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */ 744 745 u32 metadata_reg_c_7; /* metadata_reg_c_7 */ 746 u32 metadata_reg_c_6; /* metadata_reg_c_6 */ 747 u32 metadata_reg_c_5; /* metadata_reg_c_5 */ 748 u32 metadata_reg_c_4; /* metadata_reg_c_4 */ 749 u32 metadata_reg_c_3; /* metadata_reg_c_3 */ 750 u32 metadata_reg_c_2; /* metadata_reg_c_2 */ 751 u32 metadata_reg_c_1; /* metadata_reg_c_1 */ 752 u32 metadata_reg_c_0; /* metadata_reg_c_0 */ 753 u32 metadata_reg_a; /* metadata_reg_a */ 754 u32 reserved_auto1[3]; 755 }; 756 757 struct mlx5dr_match_misc3 { 758 u32 inner_tcp_seq_num; 759 u32 outer_tcp_seq_num; 760 u32 inner_tcp_ack_num; 761 u32 outer_tcp_ack_num; 762 763 u32 reserved_auto1:8; 764 u32 outer_vxlan_gpe_vni:24; 765 766 u32 outer_vxlan_gpe_next_protocol:8; 767 u32 outer_vxlan_gpe_flags:8; 768 u32 reserved_auto2:16; 769 770 u32 icmpv4_header_data; 771 u32 icmpv6_header_data; 772 773 u8 icmpv4_type; 774 u8 icmpv4_code; 775 u8 icmpv6_type; 776 u8 icmpv6_code; 777 778 u32 geneve_tlv_option_0_data; 779 780 u32 gtpu_teid; 781 782 u8 gtpu_msg_type; 783 u8 gtpu_msg_flags; 784 u32 reserved_auto3:16; 785 786 u32 gtpu_dw_2; 787 u32 gtpu_first_ext_dw_0; 788 u32 gtpu_dw_0; 789 u32 reserved_auto4; 790 }; 791 792 struct mlx5dr_match_misc4 { 793 u32 prog_sample_field_value_0; 794 u32 prog_sample_field_id_0; 795 u32 prog_sample_field_value_1; 796 u32 prog_sample_field_id_1; 797 u32 prog_sample_field_value_2; 798 u32 prog_sample_field_id_2; 799 u32 prog_sample_field_value_3; 800 u32 prog_sample_field_id_3; 801 u32 reserved_auto1[8]; 802 }; 803 804 struct mlx5dr_match_misc5 { 805 u32 macsec_tag_0; 806 u32 macsec_tag_1; 807 u32 macsec_tag_2; 808 u32 macsec_tag_3; 809 u32 tunnel_header_0; 810 u32 tunnel_header_1; 811 u32 tunnel_header_2; 812 u32 tunnel_header_3; 813 }; 814 815 struct mlx5dr_match_param { 816 struct mlx5dr_match_spec outer; 817 struct mlx5dr_match_misc misc; 818 struct mlx5dr_match_spec inner; 819 struct mlx5dr_match_misc2 misc2; 820 struct mlx5dr_match_misc3 misc3; 821 struct mlx5dr_match_misc4 misc4; 822 struct mlx5dr_match_misc5 misc5; 823 }; 824 825 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \ 826 (_misc3)->icmpv4_code || \ 827 (_misc3)->icmpv4_header_data) 828 829 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \ 830 (_spec)->src_ip_95_64 || \ 831 (_spec)->src_ip_63_32 || \ 832 (_spec)->src_ip_31_0) 833 834 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \ 835 (_spec)->dst_ip_95_64 || \ 836 (_spec)->dst_ip_63_32 || \ 837 (_spec)->dst_ip_31_0) 838 839 struct mlx5dr_esw_caps { 840 u64 drop_icm_address_rx; 841 u64 drop_icm_address_tx; 842 u64 uplink_icm_address_rx; 843 u64 uplink_icm_address_tx; 844 u8 sw_owner:1; 845 u8 sw_owner_v2:1; 846 }; 847 848 struct mlx5dr_cmd_vport_cap { 849 u16 vport_gvmi; 850 u16 vhca_gvmi; 851 u16 num; 852 u64 icm_address_rx; 853 u64 icm_address_tx; 854 }; 855 856 struct mlx5dr_roce_cap { 857 u8 roce_en:1; 858 u8 fl_rc_qp_when_roce_disabled:1; 859 u8 fl_rc_qp_when_roce_enabled:1; 860 }; 861 862 struct mlx5dr_vports { 863 struct mlx5dr_cmd_vport_cap esw_manager_caps; 864 struct mlx5dr_cmd_vport_cap uplink_caps; 865 struct xarray vports_caps_xa; 866 }; 867 868 struct mlx5dr_cmd_caps { 869 u16 gvmi; 870 u64 nic_rx_drop_address; 871 u64 nic_tx_drop_address; 872 u64 nic_tx_allow_address; 873 u64 esw_rx_drop_address; 874 u64 esw_tx_drop_address; 875 u32 log_icm_size; 876 u64 hdr_modify_icm_addr; 877 u32 log_modify_pattern_icm_size; 878 u64 hdr_modify_pattern_icm_addr; 879 u32 flex_protocols; 880 u8 flex_parser_id_icmp_dw0; 881 u8 flex_parser_id_icmp_dw1; 882 u8 flex_parser_id_icmpv6_dw0; 883 u8 flex_parser_id_icmpv6_dw1; 884 u8 flex_parser_id_geneve_tlv_option_0; 885 u8 flex_parser_id_mpls_over_gre; 886 u8 flex_parser_id_mpls_over_udp; 887 u8 flex_parser_id_gtpu_dw_0; 888 u8 flex_parser_id_gtpu_teid; 889 u8 flex_parser_id_gtpu_dw_2; 890 u8 flex_parser_id_gtpu_first_ext_dw_0; 891 u8 flex_parser_ok_bits_supp; 892 u8 max_ft_level; 893 u16 roce_min_src_udp; 894 u8 sw_format_ver; 895 bool eswitch_manager; 896 bool rx_sw_owner; 897 bool tx_sw_owner; 898 bool fdb_sw_owner; 899 u8 rx_sw_owner_v2:1; 900 u8 tx_sw_owner_v2:1; 901 u8 fdb_sw_owner_v2:1; 902 struct mlx5dr_esw_caps esw_caps; 903 struct mlx5dr_vports vports; 904 bool prio_tag_required; 905 struct mlx5dr_roce_cap roce_caps; 906 u16 log_header_modify_argument_granularity; 907 u16 log_header_modify_argument_max_alloc; 908 bool support_modify_argument; 909 u8 is_ecpf:1; 910 u8 isolate_vl_tc:1; 911 }; 912 913 enum mlx5dr_domain_nic_type { 914 DR_DOMAIN_NIC_TYPE_RX, 915 DR_DOMAIN_NIC_TYPE_TX, 916 }; 917 918 struct mlx5dr_domain_rx_tx { 919 u64 drop_icm_addr; 920 u64 default_icm_addr; 921 enum mlx5dr_domain_nic_type type; 922 struct mutex mutex; /* protect rx/tx domain */ 923 }; 924 925 struct mlx5dr_domain_info { 926 bool supp_sw_steering; 927 u32 max_inline_size; 928 u32 max_send_wr; 929 u32 max_log_sw_icm_sz; 930 u32 max_log_action_icm_sz; 931 u32 max_log_modify_hdr_pattern_icm_sz; 932 struct mlx5dr_domain_rx_tx rx; 933 struct mlx5dr_domain_rx_tx tx; 934 struct mlx5dr_cmd_caps caps; 935 }; 936 937 struct mlx5dr_domain { 938 struct mlx5dr_domain *peer_dmn[MLX5_MAX_PORTS]; 939 struct mlx5_core_dev *mdev; 940 u32 pdn; 941 struct mlx5_uars_page *uar; 942 enum mlx5dr_domain_type type; 943 refcount_t refcount; 944 struct mlx5dr_icm_pool *ste_icm_pool; 945 struct mlx5dr_icm_pool *action_icm_pool; 946 struct mlx5dr_send_info_pool *send_info_pool_rx; 947 struct mlx5dr_send_info_pool *send_info_pool_tx; 948 struct kmem_cache *chunks_kmem_cache; 949 struct kmem_cache *htbls_kmem_cache; 950 struct mlx5dr_ptrn_mgr *ptrn_mgr; 951 struct mlx5dr_arg_mgr *arg_mgr; 952 struct mlx5dr_send_ring *send_ring; 953 struct mlx5dr_domain_info info; 954 struct xarray csum_fts_xa; 955 struct mlx5dr_ste_ctx *ste_ctx; 956 struct list_head dbg_tbl_list; 957 struct mlx5dr_dbg_dump_info dump_info; 958 struct xarray definers_xa; 959 /* memory management statistics */ 960 u32 num_buddies[DR_ICM_TYPE_MAX]; 961 }; 962 963 struct mlx5dr_table_rx_tx { 964 struct mlx5dr_ste_htbl *s_anchor; 965 struct mlx5dr_domain_rx_tx *nic_dmn; 966 u64 default_icm_addr; 967 struct list_head nic_matcher_list; 968 }; 969 970 struct mlx5dr_table { 971 struct mlx5dr_domain *dmn; 972 struct mlx5dr_table_rx_tx rx; 973 struct mlx5dr_table_rx_tx tx; 974 u32 level; 975 u32 table_type; 976 u32 table_id; 977 u32 flags; 978 struct list_head matcher_list; 979 struct mlx5dr_action *miss_action; 980 refcount_t refcount; 981 struct list_head dbg_node; 982 }; 983 984 struct mlx5dr_matcher_rx_tx { 985 struct mlx5dr_ste_htbl *s_htbl; 986 struct mlx5dr_ste_htbl *e_anchor; 987 struct mlx5dr_ste_build *ste_builder; 988 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX] 989 [DR_RULE_IPV_MAX] 990 [DR_RULE_MAX_STES]; 991 u8 num_of_builders; 992 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX]; 993 u64 default_icm_addr; 994 struct mlx5dr_table_rx_tx *nic_tbl; 995 u32 prio; 996 struct list_head list_node; 997 u32 rules; 998 }; 999 1000 struct mlx5dr_matcher { 1001 struct mlx5dr_table *tbl; 1002 struct mlx5dr_matcher_rx_tx rx; 1003 struct mlx5dr_matcher_rx_tx tx; 1004 struct list_head list_node; /* Used for both matchers and dbg managing */ 1005 u32 prio; 1006 struct mlx5dr_match_param mask; 1007 u8 match_criteria; 1008 refcount_t refcount; 1009 struct list_head dbg_rule_list; 1010 }; 1011 1012 struct mlx5dr_ste_action_modify_field { 1013 u16 hw_field; 1014 u8 start; 1015 u8 end; 1016 u8 l3_type; 1017 u8 l4_type; 1018 }; 1019 1020 struct mlx5dr_ptrn_obj { 1021 struct mlx5dr_icm_chunk *chunk; 1022 u8 *data; 1023 u16 num_of_actions; 1024 u32 index; 1025 refcount_t refcount; 1026 struct list_head list; 1027 }; 1028 1029 struct mlx5dr_arg_obj { 1030 u32 obj_id; 1031 u32 obj_offset; 1032 struct list_head list_node; 1033 u32 log_chunk_size; 1034 }; 1035 1036 struct mlx5dr_action_rewrite { 1037 struct mlx5dr_domain *dmn; 1038 struct mlx5dr_icm_chunk *chunk; 1039 u8 *data; 1040 u16 num_of_actions; 1041 u32 index; 1042 u8 single_action_opt:1; 1043 u8 allow_rx:1; 1044 u8 allow_tx:1; 1045 u8 modify_ttl:1; 1046 struct mlx5dr_ptrn_obj *ptrn; 1047 struct mlx5dr_arg_obj *arg; 1048 }; 1049 1050 struct mlx5dr_action_reformat { 1051 struct mlx5dr_domain *dmn; 1052 u32 id; 1053 u32 size; 1054 u8 param_0; 1055 u8 param_1; 1056 }; 1057 1058 struct mlx5dr_action_sampler { 1059 struct mlx5dr_domain *dmn; 1060 u64 rx_icm_addr; 1061 u64 tx_icm_addr; 1062 u32 sampler_id; 1063 }; 1064 1065 struct mlx5dr_action_dest_tbl { 1066 u8 is_fw_tbl:1; 1067 union { 1068 struct mlx5dr_table *tbl; 1069 struct { 1070 struct mlx5dr_domain *dmn; 1071 u32 id; 1072 u32 group_id; 1073 enum fs_flow_table_type type; 1074 u64 rx_icm_addr; 1075 u64 tx_icm_addr; 1076 struct mlx5dr_action **ref_actions; 1077 u32 num_of_ref_actions; 1078 } fw_tbl; 1079 }; 1080 }; 1081 1082 struct mlx5dr_action_range { 1083 struct mlx5dr_domain *dmn; 1084 struct mlx5dr_action *hit_tbl_action; 1085 struct mlx5dr_action *miss_tbl_action; 1086 u32 definer_id; 1087 u32 min; 1088 u32 max; 1089 }; 1090 1091 struct mlx5dr_action_ctr { 1092 u32 ctr_id; 1093 u32 offset; 1094 }; 1095 1096 struct mlx5dr_action_vport { 1097 struct mlx5dr_domain *dmn; 1098 struct mlx5dr_cmd_vport_cap *caps; 1099 }; 1100 1101 struct mlx5dr_action_push_vlan { 1102 u32 vlan_hdr; /* tpid_pcp_dei_vid */ 1103 }; 1104 1105 struct mlx5dr_action_flow_tag { 1106 u32 flow_tag; 1107 }; 1108 1109 struct mlx5dr_rule_action_member { 1110 struct mlx5dr_action *action; 1111 struct list_head list; 1112 }; 1113 1114 struct mlx5dr_action_aso_flow_meter { 1115 struct mlx5dr_domain *dmn; 1116 u32 obj_id; 1117 u32 offset; 1118 u8 dest_reg_id; 1119 u8 init_color; 1120 }; 1121 1122 struct mlx5dr_action { 1123 enum mlx5dr_action_type action_type; 1124 refcount_t refcount; 1125 1126 union { 1127 void *data; 1128 struct mlx5dr_action_rewrite *rewrite; 1129 struct mlx5dr_action_reformat *reformat; 1130 struct mlx5dr_action_sampler *sampler; 1131 struct mlx5dr_action_dest_tbl *dest_tbl; 1132 struct mlx5dr_action_ctr *ctr; 1133 struct mlx5dr_action_vport *vport; 1134 struct mlx5dr_action_push_vlan *push_vlan; 1135 struct mlx5dr_action_flow_tag *flow_tag; 1136 struct mlx5dr_action_aso_flow_meter *aso; 1137 struct mlx5dr_action_range *range; 1138 }; 1139 }; 1140 1141 enum mlx5dr_connect_type { 1142 CONNECT_HIT = 1, 1143 CONNECT_MISS = 2, 1144 }; 1145 1146 struct mlx5dr_htbl_connect_info { 1147 enum mlx5dr_connect_type type; 1148 union { 1149 struct mlx5dr_ste_htbl *hit_next_htbl; 1150 u64 miss_icm_addr; 1151 }; 1152 }; 1153 1154 struct mlx5dr_rule_rx_tx { 1155 struct mlx5dr_matcher_rx_tx *nic_matcher; 1156 struct mlx5dr_ste *last_rule_ste; 1157 }; 1158 1159 struct mlx5dr_rule { 1160 struct mlx5dr_matcher *matcher; 1161 struct mlx5dr_rule_rx_tx rx; 1162 struct mlx5dr_rule_rx_tx tx; 1163 struct list_head rule_actions_list; 1164 struct list_head dbg_node; 1165 u32 flow_source; 1166 }; 1167 1168 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule, 1169 struct mlx5dr_ste *ste, 1170 bool force); 1171 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr, 1172 struct mlx5dr_ste *curr_ste, 1173 int *num_of_stes); 1174 1175 struct mlx5dr_icm_chunk { 1176 struct mlx5dr_icm_buddy_mem *buddy_mem; 1177 1178 /* indicates the index of this chunk in the whole memory, 1179 * used for deleting the chunk from the buddy 1180 */ 1181 unsigned int seg; 1182 enum mlx5dr_icm_chunk_size size; 1183 1184 /* Memory optimisation */ 1185 struct mlx5dr_ste *ste_arr; 1186 u8 *hw_ste_arr; 1187 struct list_head *miss_list; 1188 }; 1189 1190 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn) 1191 { 1192 mutex_lock(&nic_dmn->mutex); 1193 } 1194 1195 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn) 1196 { 1197 mutex_unlock(&nic_dmn->mutex); 1198 } 1199 1200 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn) 1201 { 1202 mlx5dr_domain_nic_lock(&dmn->info.rx); 1203 mlx5dr_domain_nic_lock(&dmn->info.tx); 1204 } 1205 1206 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn) 1207 { 1208 mlx5dr_domain_nic_unlock(&dmn->info.tx); 1209 mlx5dr_domain_nic_unlock(&dmn->info.rx); 1210 } 1211 1212 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn, 1213 struct mlx5dr_matcher_rx_tx *nic_matcher); 1214 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn, 1215 struct mlx5dr_matcher_rx_tx *nic_matcher); 1216 1217 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher, 1218 struct mlx5dr_matcher_rx_tx *nic_matcher, 1219 enum mlx5dr_ipv outer_ipv, 1220 enum mlx5dr_ipv inner_ipv); 1221 1222 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk); 1223 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk); 1224 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk); 1225 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk); 1226 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk); 1227 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste); 1228 1229 struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool); 1230 void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl); 1231 1232 static inline int 1233 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type) 1234 { 1235 if (icm_type == DR_ICM_TYPE_STE) 1236 return DR_STE_SIZE; 1237 1238 return DR_MODIFY_ACTION_SIZE; 1239 } 1240 1241 static inline u32 1242 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size) 1243 { 1244 return 1 << chunk_size; 1245 } 1246 1247 static inline int 1248 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size, 1249 enum mlx5dr_icm_type icm_type) 1250 { 1251 int num_of_entries; 1252 int entry_size; 1253 1254 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type); 1255 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size); 1256 1257 return entry_size * num_of_entries; 1258 } 1259 1260 static inline int 1261 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl) 1262 { 1263 int num_of_entries = 1264 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size); 1265 1266 /* Threshold is 50%, one is added to table of size 1 */ 1267 return (num_of_entries + 1) / 2; 1268 } 1269 1270 static inline bool 1271 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl) 1272 { 1273 if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask) 1274 return false; 1275 1276 return true; 1277 } 1278 1279 struct mlx5dr_cmd_vport_cap * 1280 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport); 1281 1282 struct mlx5dr_cmd_query_flow_table_details { 1283 u8 status; 1284 u8 level; 1285 u64 sw_owner_icm_root_1; 1286 u64 sw_owner_icm_root_0; 1287 }; 1288 1289 struct mlx5dr_cmd_create_flow_table_attr { 1290 u32 table_type; 1291 u16 uid; 1292 u64 icm_addr_rx; 1293 u64 icm_addr_tx; 1294 u8 level; 1295 bool sw_owner; 1296 bool term_tbl; 1297 bool decap_en; 1298 bool reformat_en; 1299 }; 1300 1301 /* internal API functions */ 1302 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, 1303 struct mlx5dr_cmd_caps *caps); 1304 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev, 1305 bool other_vport, u16 vport_number, 1306 u64 *icm_address_rx, 1307 u64 *icm_address_tx); 1308 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev, 1309 bool other_vport, u16 vport_number, u16 *gvmi); 1310 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev, 1311 struct mlx5dr_esw_caps *caps); 1312 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev, 1313 u32 sampler_id, 1314 u64 *rx_icm_addr, 1315 u64 *tx_icm_addr); 1316 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev); 1317 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev, 1318 u32 table_type, 1319 u32 table_id, 1320 u32 group_id, 1321 u32 modify_header_id, 1322 u16 vport_id); 1323 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev, 1324 u32 table_type, 1325 u32 table_id); 1326 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev, 1327 u32 table_type, 1328 u8 num_of_actions, 1329 u64 *actions, 1330 u32 *modify_header_id); 1331 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev, 1332 u32 modify_header_id); 1333 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev, 1334 u32 table_type, 1335 u32 table_id, 1336 u32 *group_id); 1337 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev, 1338 u32 table_type, 1339 u32 table_id, 1340 u32 group_id); 1341 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev, 1342 struct mlx5dr_cmd_create_flow_table_attr *attr, 1343 u64 *fdb_rx_icm_addr, 1344 u32 *table_id); 1345 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev, 1346 u32 table_id, 1347 u32 table_type); 1348 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev, 1349 enum fs_flow_table_type type, 1350 u32 table_id, 1351 struct mlx5dr_cmd_query_flow_table_details *output); 1352 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, 1353 enum mlx5_reformat_ctx_type rt, 1354 u8 reformat_param_0, 1355 u8 reformat_param_1, 1356 size_t reformat_size, 1357 void *reformat_data, 1358 u32 *reformat_id); 1359 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev, 1360 u32 reformat_id); 1361 int mlx5dr_cmd_create_definer(struct mlx5_core_dev *mdev, 1362 u16 format_id, 1363 u8 *dw_selectors, 1364 u8 *byte_selectors, 1365 u8 *match_mask, 1366 u32 *definer_id); 1367 void mlx5dr_cmd_destroy_definer(struct mlx5_core_dev *mdev, 1368 u32 definer_id); 1369 1370 struct mlx5dr_cmd_gid_attr { 1371 u8 gid[16]; 1372 u8 mac[6]; 1373 u32 roce_ver; 1374 }; 1375 1376 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num, 1377 u16 index, struct mlx5dr_cmd_gid_attr *attr); 1378 1379 int mlx5dr_cmd_create_modify_header_arg(struct mlx5_core_dev *dev, 1380 u16 log_obj_range, u32 pd, 1381 u32 *obj_id); 1382 void mlx5dr_cmd_destroy_modify_header_arg(struct mlx5_core_dev *dev, 1383 u32 obj_id); 1384 1385 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 1386 enum mlx5dr_icm_type icm_type); 1387 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool); 1388 1389 struct mlx5dr_icm_chunk * 1390 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 1391 enum mlx5dr_icm_chunk_size chunk_size); 1392 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk); 1393 1394 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx, 1395 u8 *hw_ste_p, u32 ste_size); 1396 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn, 1397 struct mlx5dr_domain_rx_tx *nic_dmn, 1398 struct mlx5dr_ste_htbl *htbl, 1399 struct mlx5dr_htbl_connect_info *connect_info, 1400 bool update_hw_ste); 1401 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx, 1402 u16 gvmi, 1403 enum mlx5dr_domain_nic_type nic_type, 1404 struct mlx5dr_ste_htbl *htbl, 1405 u8 *formatted_ste, 1406 struct mlx5dr_htbl_connect_info *connect_info); 1407 void mlx5dr_ste_copy_param(u8 match_criteria, 1408 struct mlx5dr_match_param *set_param, 1409 struct mlx5dr_match_parameters *mask, 1410 bool clear); 1411 1412 struct mlx5dr_qp { 1413 struct mlx5_core_dev *mdev; 1414 struct mlx5_wq_qp wq; 1415 struct mlx5_uars_page *uar; 1416 struct mlx5_wq_ctrl wq_ctrl; 1417 u32 qpn; 1418 struct { 1419 unsigned int head; 1420 unsigned int pc; 1421 unsigned int cc; 1422 unsigned int size; 1423 unsigned int *wqe_head; 1424 unsigned int wqe_cnt; 1425 } sq; 1426 struct { 1427 unsigned int pc; 1428 unsigned int cc; 1429 unsigned int size; 1430 unsigned int wqe_cnt; 1431 } rq; 1432 int max_inline_data; 1433 }; 1434 1435 struct mlx5dr_cq { 1436 struct mlx5_core_dev *mdev; 1437 struct mlx5_cqwq wq; 1438 struct mlx5_wq_ctrl wq_ctrl; 1439 struct mlx5_core_cq mcq; 1440 struct mlx5dr_qp *qp; 1441 }; 1442 1443 struct mlx5dr_mr { 1444 struct mlx5_core_dev *mdev; 1445 u32 mkey; 1446 dma_addr_t dma_addr; 1447 void *addr; 1448 size_t size; 1449 }; 1450 1451 struct mlx5dr_send_ring { 1452 struct mlx5dr_cq *cq; 1453 struct mlx5dr_qp *qp; 1454 struct mlx5dr_mr *mr; 1455 /* How much wqes are waiting for completion */ 1456 u32 pending_wqe; 1457 /* Signal request per this trash hold value */ 1458 u16 signal_th; 1459 /* Each post_send_size less than max_post_send_size */ 1460 u32 max_post_send_size; 1461 /* manage the send queue */ 1462 u32 tx_head; 1463 void *buf; 1464 u32 buf_size; 1465 u8 *sync_buff; 1466 struct mlx5dr_mr *sync_mr; 1467 spinlock_t lock; /* Protect the data path of the send ring */ 1468 bool err_state; /* send_ring is not usable in err state */ 1469 }; 1470 1471 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn); 1472 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn, 1473 struct mlx5dr_send_ring *send_ring); 1474 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn); 1475 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn, 1476 struct mlx5dr_ste *ste, 1477 u8 *data, 1478 u16 size, 1479 u16 offset); 1480 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, 1481 struct mlx5dr_ste_htbl *htbl, 1482 u8 *formatted_ste, u8 *mask); 1483 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn, 1484 struct mlx5dr_ste_htbl *htbl, 1485 u8 *ste_init_data, 1486 bool update_hw_ste); 1487 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn, 1488 struct mlx5dr_action *action); 1489 int mlx5dr_send_postsend_pattern(struct mlx5dr_domain *dmn, 1490 struct mlx5dr_icm_chunk *chunk, 1491 u16 num_of_actions, 1492 u8 *data); 1493 int mlx5dr_send_postsend_args(struct mlx5dr_domain *dmn, u64 arg_id, 1494 u16 num_of_actions, u8 *actions_data); 1495 1496 int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn); 1497 void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn); 1498 struct mlx5dr_ste_send_info *mlx5dr_send_info_alloc(struct mlx5dr_domain *dmn, 1499 enum mlx5dr_domain_nic_type nic_type); 1500 void mlx5dr_send_info_free(struct mlx5dr_ste_send_info *ste_send_info); 1501 1502 struct mlx5dr_cmd_ft_info { 1503 u32 id; 1504 u16 vport; 1505 enum fs_flow_table_type type; 1506 }; 1507 1508 struct mlx5dr_cmd_flow_destination_hw_info { 1509 enum mlx5_flow_destination_type type; 1510 union { 1511 u32 tir_num; 1512 u32 ft_num; 1513 u32 ft_id; 1514 u32 counter_id; 1515 u32 sampler_id; 1516 struct { 1517 u16 num; 1518 u16 vhca_id; 1519 u32 reformat_id; 1520 u8 flags; 1521 } vport; 1522 }; 1523 }; 1524 1525 struct mlx5dr_cmd_fte_info { 1526 u32 dests_size; 1527 u32 index; 1528 struct mlx5_flow_context flow_context; 1529 u32 *val; 1530 struct mlx5_flow_act action; 1531 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr; 1532 bool ignore_flow_level; 1533 }; 1534 1535 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev, 1536 int opmod, int modify_mask, 1537 struct mlx5dr_cmd_ft_info *ft, 1538 u32 group_id, 1539 struct mlx5dr_cmd_fte_info *fte); 1540 1541 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps); 1542 1543 struct mlx5dr_fw_recalc_cs_ft { 1544 u64 rx_icm_addr; 1545 u32 table_id; 1546 u32 group_id; 1547 u32 modify_hdr_id; 1548 }; 1549 1550 struct mlx5dr_fw_recalc_cs_ft * 1551 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num); 1552 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn, 1553 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft); 1554 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn, 1555 u16 vport_num, 1556 u64 *rx_icm_addr); 1557 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn, 1558 struct mlx5dr_cmd_flow_destination_hw_info *dest, 1559 int num_dest, 1560 bool reformat_req, 1561 u32 *tbl_id, 1562 u32 *group_id, 1563 bool ignore_flow_level, 1564 u32 flow_source); 1565 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id, 1566 u32 group_id); 1567 1568 static inline bool mlx5dr_is_fw_table(struct mlx5_flow_table *ft) 1569 { 1570 return !ft->fs_dr_table.dr_table; 1571 } 1572 1573 static inline bool mlx5dr_supp_match_ranges(struct mlx5_core_dev *dev) 1574 { 1575 return (MLX5_CAP_GEN(dev, steering_format_version) >= 1576 MLX5_STEERING_FORMAT_CONNECTX_6DX) && 1577 (MLX5_CAP_GEN_64(dev, match_definer_format_supported) & 1578 (1ULL << MLX5_IFC_DEFINER_FORMAT_ID_SELECT)); 1579 } 1580 1581 bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn); 1582 struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn); 1583 void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr); 1584 struct mlx5dr_ptrn_obj *mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr, 1585 u16 num_of_actions, u8 *data); 1586 void mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr, 1587 struct mlx5dr_ptrn_obj *pattern); 1588 struct mlx5dr_arg_mgr *mlx5dr_arg_mgr_create(struct mlx5dr_domain *dmn); 1589 void mlx5dr_arg_mgr_destroy(struct mlx5dr_arg_mgr *mgr); 1590 struct mlx5dr_arg_obj *mlx5dr_arg_get_obj(struct mlx5dr_arg_mgr *mgr, 1591 u16 num_of_actions, 1592 u8 *data); 1593 void mlx5dr_arg_put_obj(struct mlx5dr_arg_mgr *mgr, 1594 struct mlx5dr_arg_obj *arg_obj); 1595 u32 mlx5dr_arg_get_obj_id(struct mlx5dr_arg_obj *arg_obj); 1596 1597 #endif /* _DR_TYPES_H_ */ 1598