1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2019, Mellanox Technologies */ 3 4 #ifndef _DR_TYPES_ 5 #define _DR_TYPES_ 6 7 #include <linux/mlx5/vport.h> 8 #include <linux/refcount.h> 9 #include "fs_core.h" 10 #include "wq.h" 11 #include "lib/mlx5.h" 12 #include "mlx5_ifc_dr.h" 13 #include "mlx5dr.h" 14 #include "dr_dbg.h" 15 16 #define DR_RULE_MAX_STES 18 17 #define DR_ACTION_MAX_STES 5 18 #define DR_STE_SVLAN 0x1 19 #define DR_STE_CVLAN 0x2 20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4) 21 #define DR_NUM_OF_FLEX_PARSERS 8 22 #define DR_STE_MAX_FLEX_0_ID 3 23 #define DR_STE_MAX_FLEX_1_ID 7 24 25 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg) 26 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg) 27 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg) 28 29 static inline bool dr_is_flex_parser_0_id(u8 parser_id) 30 { 31 return parser_id <= DR_STE_MAX_FLEX_0_ID; 32 } 33 34 static inline bool dr_is_flex_parser_1_id(u8 parser_id) 35 { 36 return parser_id > DR_STE_MAX_FLEX_0_ID; 37 } 38 39 enum mlx5dr_icm_chunk_size { 40 DR_CHUNK_SIZE_1, 41 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */ 42 DR_CHUNK_SIZE_2, 43 DR_CHUNK_SIZE_4, 44 DR_CHUNK_SIZE_8, 45 DR_CHUNK_SIZE_16, 46 DR_CHUNK_SIZE_32, 47 DR_CHUNK_SIZE_64, 48 DR_CHUNK_SIZE_128, 49 DR_CHUNK_SIZE_256, 50 DR_CHUNK_SIZE_512, 51 DR_CHUNK_SIZE_1K, 52 DR_CHUNK_SIZE_2K, 53 DR_CHUNK_SIZE_4K, 54 DR_CHUNK_SIZE_8K, 55 DR_CHUNK_SIZE_16K, 56 DR_CHUNK_SIZE_32K, 57 DR_CHUNK_SIZE_64K, 58 DR_CHUNK_SIZE_128K, 59 DR_CHUNK_SIZE_256K, 60 DR_CHUNK_SIZE_512K, 61 DR_CHUNK_SIZE_1024K, 62 DR_CHUNK_SIZE_2048K, 63 DR_CHUNK_SIZE_MAX, 64 }; 65 66 enum mlx5dr_icm_type { 67 DR_ICM_TYPE_STE, 68 DR_ICM_TYPE_MODIFY_ACTION, 69 }; 70 71 static inline enum mlx5dr_icm_chunk_size 72 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk) 73 { 74 chunk += 2; 75 if (chunk < DR_CHUNK_SIZE_MAX) 76 return chunk; 77 78 return DR_CHUNK_SIZE_MAX; 79 } 80 81 enum { 82 DR_STE_SIZE = 64, 83 DR_STE_SIZE_CTRL = 32, 84 DR_STE_SIZE_TAG = 16, 85 DR_STE_SIZE_MASK = 16, 86 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK, 87 }; 88 89 enum mlx5dr_ste_ctx_action_cap { 90 DR_STE_CTX_ACTION_CAP_NONE = 0, 91 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0, 92 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1, 93 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2, 94 DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3, 95 }; 96 97 enum { 98 DR_MODIFY_ACTION_SIZE = 8, 99 }; 100 101 enum mlx5dr_matcher_criteria { 102 DR_MATCHER_CRITERIA_EMPTY = 0, 103 DR_MATCHER_CRITERIA_OUTER = 1 << 0, 104 DR_MATCHER_CRITERIA_MISC = 1 << 1, 105 DR_MATCHER_CRITERIA_INNER = 1 << 2, 106 DR_MATCHER_CRITERIA_MISC2 = 1 << 3, 107 DR_MATCHER_CRITERIA_MISC3 = 1 << 4, 108 DR_MATCHER_CRITERIA_MISC4 = 1 << 5, 109 DR_MATCHER_CRITERIA_MISC5 = 1 << 6, 110 DR_MATCHER_CRITERIA_MAX = 1 << 7, 111 }; 112 113 enum mlx5dr_action_type { 114 DR_ACTION_TYP_TNL_L2_TO_L2, 115 DR_ACTION_TYP_L2_TO_TNL_L2, 116 DR_ACTION_TYP_TNL_L3_TO_L2, 117 DR_ACTION_TYP_L2_TO_TNL_L3, 118 DR_ACTION_TYP_DROP, 119 DR_ACTION_TYP_QP, 120 DR_ACTION_TYP_FT, 121 DR_ACTION_TYP_CTR, 122 DR_ACTION_TYP_TAG, 123 DR_ACTION_TYP_MODIFY_HDR, 124 DR_ACTION_TYP_VPORT, 125 DR_ACTION_TYP_POP_VLAN, 126 DR_ACTION_TYP_PUSH_VLAN, 127 DR_ACTION_TYP_INSERT_HDR, 128 DR_ACTION_TYP_REMOVE_HDR, 129 DR_ACTION_TYP_SAMPLER, 130 DR_ACTION_TYP_ASO_FLOW_METER, 131 DR_ACTION_TYP_MAX, 132 }; 133 134 enum mlx5dr_ipv { 135 DR_RULE_IPV4, 136 DR_RULE_IPV6, 137 DR_RULE_IPV_MAX, 138 }; 139 140 struct mlx5dr_icm_pool; 141 struct mlx5dr_icm_chunk; 142 struct mlx5dr_icm_buddy_mem; 143 struct mlx5dr_ste_htbl; 144 struct mlx5dr_match_param; 145 struct mlx5dr_cmd_caps; 146 struct mlx5dr_rule_rx_tx; 147 struct mlx5dr_matcher_rx_tx; 148 struct mlx5dr_ste_ctx; 149 struct mlx5dr_send_info_pool; 150 struct mlx5dr_icm_hot_chunk; 151 152 struct mlx5dr_ste { 153 /* refcount: indicates the num of rules that using this ste */ 154 u32 refcount; 155 156 /* this ste is part of a rule, located in ste's chain */ 157 u8 ste_chain_location; 158 159 /* attached to the miss_list head at each htbl entry */ 160 struct list_head miss_list_node; 161 162 /* this ste is member of htbl */ 163 struct mlx5dr_ste_htbl *htbl; 164 165 struct mlx5dr_ste_htbl *next_htbl; 166 167 /* The rule this STE belongs to */ 168 struct mlx5dr_rule_rx_tx *rule_rx_tx; 169 }; 170 171 struct mlx5dr_ste_htbl_ctrl { 172 /* total number of valid entries belonging to this hash table. This 173 * includes the non collision and collision entries 174 */ 175 unsigned int num_of_valid_entries; 176 177 /* total number of collisions entries attached to this table */ 178 unsigned int num_of_collisions; 179 }; 180 181 struct mlx5dr_ste_htbl { 182 u16 lu_type; 183 u16 byte_mask; 184 u32 refcount; 185 struct mlx5dr_icm_chunk *chunk; 186 struct mlx5dr_ste *pointing_ste; 187 struct mlx5dr_ste_htbl_ctrl ctrl; 188 }; 189 190 struct mlx5dr_ste_send_info { 191 struct mlx5dr_ste *ste; 192 struct list_head send_list; 193 u16 size; 194 u16 offset; 195 u8 data_cont[DR_STE_SIZE]; 196 u8 *data; 197 }; 198 199 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size, 200 u16 offset, u8 *data, 201 struct mlx5dr_ste_send_info *ste_info, 202 struct list_head *send_list, 203 bool copy_data); 204 205 struct mlx5dr_ste_build { 206 u8 inner:1; 207 u8 rx:1; 208 u8 vhca_id_valid:1; 209 struct mlx5dr_domain *dmn; 210 struct mlx5dr_cmd_caps *caps; 211 u16 lu_type; 212 u16 byte_mask; 213 u8 bit_mask[DR_STE_SIZE_MASK]; 214 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec, 215 struct mlx5dr_ste_build *sb, 216 u8 *tag); 217 }; 218 219 struct mlx5dr_ste_htbl * 220 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool, 221 enum mlx5dr_icm_chunk_size chunk_size, 222 u16 lu_type, u16 byte_mask); 223 224 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl); 225 226 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl) 227 { 228 htbl->refcount--; 229 if (!htbl->refcount) 230 mlx5dr_ste_htbl_free(htbl); 231 } 232 233 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl) 234 { 235 htbl->refcount++; 236 } 237 238 /* STE utils */ 239 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl); 240 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, 241 u8 *hw_ste, u64 miss_addr); 242 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx, 243 u8 *hw_ste, u64 icm_addr, u32 ht_size); 244 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx, 245 u8 *hw_ste, 246 struct mlx5dr_ste_htbl *next_htbl); 247 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask); 248 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher, 249 u8 ste_location); 250 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste); 251 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste); 252 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste); 253 254 #define MLX5DR_MAX_VLANS 2 255 256 struct mlx5dr_ste_actions_attr { 257 u32 modify_index; 258 u16 modify_actions; 259 u32 decap_index; 260 u16 decap_actions; 261 u8 decap_with_vlan:1; 262 u64 final_icm_addr; 263 u32 flow_tag; 264 u32 ctr_id; 265 u16 gvmi; 266 u16 hit_gvmi; 267 struct { 268 u32 id; 269 u32 size; 270 u8 param_0; 271 u8 param_1; 272 } reformat; 273 struct { 274 int count; 275 u32 headers[MLX5DR_MAX_VLANS]; 276 } vlans; 277 278 struct { 279 u32 obj_id; 280 u32 offset; 281 u8 dest_reg_id; 282 u8 init_color; 283 } aso_flow_meter; 284 }; 285 286 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, 287 struct mlx5dr_domain *dmn, 288 u8 *action_type_set, 289 u8 *last_ste, 290 struct mlx5dr_ste_actions_attr *attr, 291 u32 *added_stes); 292 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx, 293 struct mlx5dr_domain *dmn, 294 u8 *action_type_set, 295 u8 *last_ste, 296 struct mlx5dr_ste_actions_attr *attr, 297 u32 *added_stes); 298 299 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx, 300 __be64 *hw_action, 301 u8 hw_field, 302 u8 shifter, 303 u8 length, 304 u32 data); 305 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx, 306 __be64 *hw_action, 307 u8 hw_field, 308 u8 shifter, 309 u8 length, 310 u32 data); 311 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx, 312 __be64 *hw_action, 313 u8 dst_hw_field, 314 u8 dst_shifter, 315 u8 dst_len, 316 u8 src_hw_field, 317 u8 src_shifter); 318 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx, 319 void *data, 320 u32 data_sz, 321 u8 *hw_action, 322 u32 hw_action_sz, 323 u16 *used_hw_action_num); 324 325 const struct mlx5dr_ste_action_modify_field * 326 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field); 327 328 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version); 329 void mlx5dr_ste_free(struct mlx5dr_ste *ste, 330 struct mlx5dr_matcher *matcher, 331 struct mlx5dr_matcher_rx_tx *nic_matcher); 332 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste, 333 struct mlx5dr_matcher *matcher, 334 struct mlx5dr_matcher_rx_tx *nic_matcher) 335 { 336 ste->refcount--; 337 if (!ste->refcount) 338 mlx5dr_ste_free(ste, matcher, nic_matcher); 339 } 340 341 /* initial as 0, increased only when ste appears in a new rule */ 342 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste) 343 { 344 ste->refcount++; 345 } 346 347 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste) 348 { 349 return !ste->refcount; 350 } 351 352 bool mlx5dr_ste_equal_tag(void *src, void *dst); 353 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher, 354 struct mlx5dr_matcher_rx_tx *nic_matcher, 355 struct mlx5dr_ste *ste, 356 u8 *cur_hw_ste, 357 enum mlx5dr_icm_chunk_size log_table_size); 358 359 /* STE build functions */ 360 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn, 361 u8 match_criteria, 362 struct mlx5dr_match_param *mask, 363 struct mlx5dr_match_param *value); 364 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, 365 struct mlx5dr_matcher_rx_tx *nic_matcher, 366 struct mlx5dr_match_param *value, 367 u8 *ste_arr); 368 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx, 369 struct mlx5dr_ste_build *builder, 370 struct mlx5dr_match_param *mask, 371 bool inner, bool rx); 372 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx, 373 struct mlx5dr_ste_build *sb, 374 struct mlx5dr_match_param *mask, 375 bool inner, bool rx); 376 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx, 377 struct mlx5dr_ste_build *sb, 378 struct mlx5dr_match_param *mask, 379 bool inner, bool rx); 380 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx, 381 struct mlx5dr_ste_build *sb, 382 struct mlx5dr_match_param *mask, 383 bool inner, bool rx); 384 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx, 385 struct mlx5dr_ste_build *sb, 386 struct mlx5dr_match_param *mask, 387 bool inner, bool rx); 388 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx, 389 struct mlx5dr_ste_build *sb, 390 struct mlx5dr_match_param *mask, 391 bool inner, bool rx); 392 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx, 393 struct mlx5dr_ste_build *sb, 394 struct mlx5dr_match_param *mask, 395 bool inner, bool rx); 396 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx, 397 struct mlx5dr_ste_build *sb, 398 struct mlx5dr_match_param *mask, 399 bool inner, bool rx); 400 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx, 401 struct mlx5dr_ste_build *sb, 402 struct mlx5dr_match_param *mask, 403 bool inner, bool rx); 404 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx, 405 struct mlx5dr_ste_build *sb, 406 struct mlx5dr_match_param *mask, 407 bool inner, bool rx); 408 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx, 409 struct mlx5dr_ste_build *sb, 410 struct mlx5dr_match_param *mask, 411 bool inner, bool rx); 412 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx, 413 struct mlx5dr_ste_build *sb, 414 struct mlx5dr_match_param *mask, 415 bool inner, bool rx); 416 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx, 417 struct mlx5dr_ste_build *sb, 418 struct mlx5dr_match_param *mask, 419 bool inner, bool rx); 420 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx, 421 struct mlx5dr_ste_build *sb, 422 struct mlx5dr_match_param *mask, 423 struct mlx5dr_cmd_caps *caps, 424 bool inner, bool rx); 425 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx, 426 struct mlx5dr_ste_build *sb, 427 struct mlx5dr_match_param *mask, 428 struct mlx5dr_cmd_caps *caps, 429 bool inner, bool rx); 430 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx, 431 struct mlx5dr_ste_build *sb, 432 struct mlx5dr_match_param *mask, 433 struct mlx5dr_cmd_caps *caps, 434 bool inner, bool rx); 435 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx, 436 struct mlx5dr_ste_build *sb, 437 struct mlx5dr_match_param *mask, 438 bool inner, bool rx); 439 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx, 440 struct mlx5dr_ste_build *sb, 441 struct mlx5dr_match_param *mask, 442 bool inner, bool rx); 443 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx, 444 struct mlx5dr_ste_build *sb, 445 struct mlx5dr_match_param *mask, 446 struct mlx5dr_cmd_caps *caps, 447 bool inner, bool rx); 448 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx, 449 struct mlx5dr_ste_build *sb, 450 struct mlx5dr_match_param *mask, 451 struct mlx5dr_cmd_caps *caps, 452 bool inner, bool rx); 453 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx, 454 struct mlx5dr_ste_build *sb, 455 struct mlx5dr_match_param *mask, 456 bool inner, bool rx); 457 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 458 struct mlx5dr_ste_build *sb, 459 struct mlx5dr_match_param *mask, 460 struct mlx5dr_cmd_caps *caps, 461 bool inner, bool rx); 462 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 463 struct mlx5dr_ste_build *sb, 464 struct mlx5dr_match_param *mask, 465 struct mlx5dr_cmd_caps *caps, 466 bool inner, bool rx); 467 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx, 468 struct mlx5dr_ste_build *sb, 469 struct mlx5dr_match_param *mask, 470 bool inner, bool rx); 471 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx, 472 struct mlx5dr_ste_build *sb, 473 struct mlx5dr_match_param *mask, 474 bool inner, bool rx); 475 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx, 476 struct mlx5dr_ste_build *sb, 477 struct mlx5dr_match_param *mask, 478 bool inner, bool rx); 479 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx, 480 struct mlx5dr_ste_build *sb, 481 struct mlx5dr_match_param *mask, 482 bool inner, bool rx); 483 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx, 484 struct mlx5dr_ste_build *sb, 485 struct mlx5dr_match_param *mask, 486 struct mlx5dr_domain *dmn, 487 bool inner, bool rx); 488 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 489 struct mlx5dr_ste_build *sb, 490 struct mlx5dr_match_param *mask, 491 bool inner, bool rx); 492 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 493 struct mlx5dr_ste_build *sb, 494 struct mlx5dr_match_param *mask, 495 bool inner, bool rx); 496 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx); 497 498 /* Actions utils */ 499 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, 500 struct mlx5dr_matcher_rx_tx *nic_matcher, 501 struct mlx5dr_action *actions[], 502 u32 num_actions, 503 u8 *ste_arr, 504 u32 *new_hw_ste_arr_sz); 505 506 struct mlx5dr_match_spec { 507 u32 smac_47_16; /* Source MAC address of incoming packet */ 508 /* Incoming packet Ethertype - this is the Ethertype 509 * following the last VLAN tag of the packet 510 */ 511 u32 smac_15_0:16; /* Source MAC address of incoming packet */ 512 u32 ethertype:16; 513 514 u32 dmac_47_16; /* Destination MAC address of incoming packet */ 515 516 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */ 517 /* Priority of first VLAN tag in the incoming packet. 518 * Valid only when cvlan_tag==1 or svlan_tag==1 519 */ 520 u32 first_prio:3; 521 /* CFI bit of first VLAN tag in the incoming packet. 522 * Valid only when cvlan_tag==1 or svlan_tag==1 523 */ 524 u32 first_cfi:1; 525 /* VLAN ID of first VLAN tag in the incoming packet. 526 * Valid only when cvlan_tag==1 or svlan_tag==1 527 */ 528 u32 first_vid:12; 529 530 u32 ip_protocol:8; /* IP protocol */ 531 /* Differentiated Services Code Point derived from 532 * Traffic Class/TOS field of IPv6/v4 533 */ 534 u32 ip_dscp:6; 535 /* Explicit Congestion Notification derived from 536 * Traffic Class/TOS field of IPv6/v4 537 */ 538 u32 ip_ecn:2; 539 /* The first vlan in the packet is c-vlan (0x8100). 540 * cvlan_tag and svlan_tag cannot be set together 541 */ 542 u32 cvlan_tag:1; 543 /* The first vlan in the packet is s-vlan (0x8a88). 544 * cvlan_tag and svlan_tag cannot be set together 545 */ 546 u32 svlan_tag:1; 547 u32 frag:1; /* Packet is an IP fragment */ 548 u32 ip_version:4; /* IP version */ 549 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK; 550 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS 551 */ 552 u32 tcp_flags:9; 553 554 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */ 555 u32 tcp_sport:16; 556 /* TCP destination port. 557 * tcp and udp sport/dport are mutually exclusive 558 */ 559 u32 tcp_dport:16; 560 561 u32 reserved_auto1:16; 562 u32 ipv4_ihl:4; 563 u32 reserved_auto2:4; 564 u32 ttl_hoplimit:8; 565 566 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */ 567 u32 udp_sport:16; 568 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */ 569 u32 udp_dport:16; 570 571 /* IPv6 source address of incoming packets 572 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 573 * This field should be qualified by an appropriate ethertype 574 */ 575 u32 src_ip_127_96; 576 /* IPv6 source address of incoming packets 577 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 578 * This field should be qualified by an appropriate ethertype 579 */ 580 u32 src_ip_95_64; 581 /* IPv6 source address of incoming packets 582 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 583 * This field should be qualified by an appropriate ethertype 584 */ 585 u32 src_ip_63_32; 586 /* IPv6 source address of incoming packets 587 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 588 * This field should be qualified by an appropriate ethertype 589 */ 590 u32 src_ip_31_0; 591 /* IPv6 destination address of incoming packets 592 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 593 * This field should be qualified by an appropriate ethertype 594 */ 595 u32 dst_ip_127_96; 596 /* IPv6 destination address of incoming packets 597 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 598 * This field should be qualified by an appropriate ethertype 599 */ 600 u32 dst_ip_95_64; 601 /* IPv6 destination address of incoming packets 602 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 603 * This field should be qualified by an appropriate ethertype 604 */ 605 u32 dst_ip_63_32; 606 /* IPv6 destination address of incoming packets 607 * For IPv4 address use bits 31:0 (rest of the bits are reserved) 608 * This field should be qualified by an appropriate ethertype 609 */ 610 u32 dst_ip_31_0; 611 }; 612 613 struct mlx5dr_match_misc { 614 /* used with GRE, checksum exist when gre_c_present == 1 */ 615 u32 gre_c_present:1; 616 u32 reserved_auto1:1; 617 /* used with GRE, key exist when gre_k_present == 1 */ 618 u32 gre_k_present:1; 619 /* used with GRE, sequence number exist when gre_s_present == 1 */ 620 u32 gre_s_present:1; 621 u32 source_vhca_port:4; 622 u32 source_sqn:24; /* Source SQN */ 623 624 u32 source_eswitch_owner_vhca_id:16; 625 /* Source port.;0xffff determines wire port */ 626 u32 source_port:16; 627 628 /* Priority of second VLAN tag in the outer header of the incoming packet. 629 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 630 */ 631 u32 outer_second_prio:3; 632 /* CFI bit of first VLAN tag in the outer header of the incoming packet. 633 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 634 */ 635 u32 outer_second_cfi:1; 636 /* VLAN ID of first VLAN tag the outer header of the incoming packet. 637 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1 638 */ 639 u32 outer_second_vid:12; 640 /* Priority of second VLAN tag in the inner header of the incoming packet. 641 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 642 */ 643 u32 inner_second_prio:3; 644 /* CFI bit of first VLAN tag in the inner header of the incoming packet. 645 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 646 */ 647 u32 inner_second_cfi:1; 648 /* VLAN ID of first VLAN tag the inner header of the incoming packet. 649 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1 650 */ 651 u32 inner_second_vid:12; 652 653 u32 outer_second_cvlan_tag:1; 654 u32 inner_second_cvlan_tag:1; 655 /* The second vlan in the outer header of the packet is c-vlan (0x8100). 656 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 657 */ 658 u32 outer_second_svlan_tag:1; 659 /* The second vlan in the inner header of the packet is c-vlan (0x8100). 660 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 661 */ 662 u32 inner_second_svlan_tag:1; 663 /* The second vlan in the outer header of the packet is s-vlan (0x8a88). 664 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together 665 */ 666 u32 reserved_auto2:12; 667 /* The second vlan in the inner header of the packet is s-vlan (0x8a88). 668 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together 669 */ 670 u32 gre_protocol:16; /* GRE Protocol (outer) */ 671 672 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */ 673 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */ 674 675 u32 vxlan_vni:24; /* VXLAN VNI (outer) */ 676 u32 reserved_auto3:8; 677 678 u32 geneve_vni:24; /* GENEVE VNI field (outer) */ 679 u32 reserved_auto4:6; 680 u32 geneve_tlv_option_0_exist:1; 681 u32 geneve_oam:1; /* GENEVE OAM field (outer) */ 682 683 u32 reserved_auto5:12; 684 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */ 685 686 u32 reserved_auto6:12; 687 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */ 688 689 u32 reserved_auto7:10; 690 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */ 691 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */ 692 693 u32 reserved_auto8:8; 694 u32 bth_dst_qp:24; /* Destination QP in BTH header */ 695 696 u32 reserved_auto9; 697 u32 outer_esp_spi; 698 u32 reserved_auto10[3]; 699 }; 700 701 struct mlx5dr_match_misc2 { 702 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */ 703 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */ 704 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */ 705 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */ 706 707 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */ 708 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */ 709 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */ 710 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */ 711 712 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */ 713 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */ 714 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */ 715 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */ 716 717 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */ 718 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */ 719 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */ 720 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */ 721 722 u32 metadata_reg_c_7; /* metadata_reg_c_7 */ 723 u32 metadata_reg_c_6; /* metadata_reg_c_6 */ 724 u32 metadata_reg_c_5; /* metadata_reg_c_5 */ 725 u32 metadata_reg_c_4; /* metadata_reg_c_4 */ 726 u32 metadata_reg_c_3; /* metadata_reg_c_3 */ 727 u32 metadata_reg_c_2; /* metadata_reg_c_2 */ 728 u32 metadata_reg_c_1; /* metadata_reg_c_1 */ 729 u32 metadata_reg_c_0; /* metadata_reg_c_0 */ 730 u32 metadata_reg_a; /* metadata_reg_a */ 731 u32 reserved_auto1[3]; 732 }; 733 734 struct mlx5dr_match_misc3 { 735 u32 inner_tcp_seq_num; 736 u32 outer_tcp_seq_num; 737 u32 inner_tcp_ack_num; 738 u32 outer_tcp_ack_num; 739 740 u32 reserved_auto1:8; 741 u32 outer_vxlan_gpe_vni:24; 742 743 u32 outer_vxlan_gpe_next_protocol:8; 744 u32 outer_vxlan_gpe_flags:8; 745 u32 reserved_auto2:16; 746 747 u32 icmpv4_header_data; 748 u32 icmpv6_header_data; 749 750 u8 icmpv4_type; 751 u8 icmpv4_code; 752 u8 icmpv6_type; 753 u8 icmpv6_code; 754 755 u32 geneve_tlv_option_0_data; 756 757 u32 gtpu_teid; 758 759 u8 gtpu_msg_type; 760 u8 gtpu_msg_flags; 761 u32 reserved_auto3:16; 762 763 u32 gtpu_dw_2; 764 u32 gtpu_first_ext_dw_0; 765 u32 gtpu_dw_0; 766 u32 reserved_auto4; 767 }; 768 769 struct mlx5dr_match_misc4 { 770 u32 prog_sample_field_value_0; 771 u32 prog_sample_field_id_0; 772 u32 prog_sample_field_value_1; 773 u32 prog_sample_field_id_1; 774 u32 prog_sample_field_value_2; 775 u32 prog_sample_field_id_2; 776 u32 prog_sample_field_value_3; 777 u32 prog_sample_field_id_3; 778 u32 reserved_auto1[8]; 779 }; 780 781 struct mlx5dr_match_misc5 { 782 u32 macsec_tag_0; 783 u32 macsec_tag_1; 784 u32 macsec_tag_2; 785 u32 macsec_tag_3; 786 u32 tunnel_header_0; 787 u32 tunnel_header_1; 788 u32 tunnel_header_2; 789 u32 tunnel_header_3; 790 }; 791 792 struct mlx5dr_match_param { 793 struct mlx5dr_match_spec outer; 794 struct mlx5dr_match_misc misc; 795 struct mlx5dr_match_spec inner; 796 struct mlx5dr_match_misc2 misc2; 797 struct mlx5dr_match_misc3 misc3; 798 struct mlx5dr_match_misc4 misc4; 799 struct mlx5dr_match_misc5 misc5; 800 }; 801 802 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \ 803 (_misc3)->icmpv4_code || \ 804 (_misc3)->icmpv4_header_data) 805 806 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \ 807 (_spec)->src_ip_95_64 || \ 808 (_spec)->src_ip_63_32 || \ 809 (_spec)->src_ip_31_0) 810 811 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \ 812 (_spec)->dst_ip_95_64 || \ 813 (_spec)->dst_ip_63_32 || \ 814 (_spec)->dst_ip_31_0) 815 816 struct mlx5dr_esw_caps { 817 u64 drop_icm_address_rx; 818 u64 drop_icm_address_tx; 819 u64 uplink_icm_address_rx; 820 u64 uplink_icm_address_tx; 821 u8 sw_owner:1; 822 u8 sw_owner_v2:1; 823 }; 824 825 struct mlx5dr_cmd_vport_cap { 826 u16 vport_gvmi; 827 u16 vhca_gvmi; 828 u16 num; 829 u64 icm_address_rx; 830 u64 icm_address_tx; 831 }; 832 833 struct mlx5dr_roce_cap { 834 u8 roce_en:1; 835 u8 fl_rc_qp_when_roce_disabled:1; 836 u8 fl_rc_qp_when_roce_enabled:1; 837 }; 838 839 struct mlx5dr_vports { 840 struct mlx5dr_cmd_vport_cap esw_manager_caps; 841 struct mlx5dr_cmd_vport_cap uplink_caps; 842 struct xarray vports_caps_xa; 843 }; 844 845 struct mlx5dr_cmd_caps { 846 u16 gvmi; 847 u64 nic_rx_drop_address; 848 u64 nic_tx_drop_address; 849 u64 nic_tx_allow_address; 850 u64 esw_rx_drop_address; 851 u64 esw_tx_drop_address; 852 u32 log_icm_size; 853 u64 hdr_modify_icm_addr; 854 u32 flex_protocols; 855 u8 flex_parser_id_icmp_dw0; 856 u8 flex_parser_id_icmp_dw1; 857 u8 flex_parser_id_icmpv6_dw0; 858 u8 flex_parser_id_icmpv6_dw1; 859 u8 flex_parser_id_geneve_tlv_option_0; 860 u8 flex_parser_id_mpls_over_gre; 861 u8 flex_parser_id_mpls_over_udp; 862 u8 flex_parser_id_gtpu_dw_0; 863 u8 flex_parser_id_gtpu_teid; 864 u8 flex_parser_id_gtpu_dw_2; 865 u8 flex_parser_id_gtpu_first_ext_dw_0; 866 u8 flex_parser_ok_bits_supp; 867 u8 max_ft_level; 868 u16 roce_min_src_udp; 869 u8 sw_format_ver; 870 bool eswitch_manager; 871 bool rx_sw_owner; 872 bool tx_sw_owner; 873 bool fdb_sw_owner; 874 u8 rx_sw_owner_v2:1; 875 u8 tx_sw_owner_v2:1; 876 u8 fdb_sw_owner_v2:1; 877 struct mlx5dr_esw_caps esw_caps; 878 struct mlx5dr_vports vports; 879 bool prio_tag_required; 880 struct mlx5dr_roce_cap roce_caps; 881 u8 is_ecpf:1; 882 u8 isolate_vl_tc:1; 883 }; 884 885 enum mlx5dr_domain_nic_type { 886 DR_DOMAIN_NIC_TYPE_RX, 887 DR_DOMAIN_NIC_TYPE_TX, 888 }; 889 890 struct mlx5dr_domain_rx_tx { 891 u64 drop_icm_addr; 892 u64 default_icm_addr; 893 enum mlx5dr_domain_nic_type type; 894 struct mutex mutex; /* protect rx/tx domain */ 895 }; 896 897 struct mlx5dr_domain_info { 898 bool supp_sw_steering; 899 u32 max_inline_size; 900 u32 max_send_wr; 901 u32 max_log_sw_icm_sz; 902 u32 max_log_action_icm_sz; 903 struct mlx5dr_domain_rx_tx rx; 904 struct mlx5dr_domain_rx_tx tx; 905 struct mlx5dr_cmd_caps caps; 906 }; 907 908 struct mlx5dr_domain { 909 struct mlx5dr_domain *peer_dmn; 910 struct mlx5_core_dev *mdev; 911 u32 pdn; 912 struct mlx5_uars_page *uar; 913 enum mlx5dr_domain_type type; 914 refcount_t refcount; 915 struct mlx5dr_icm_pool *ste_icm_pool; 916 struct mlx5dr_icm_pool *action_icm_pool; 917 struct mlx5dr_send_info_pool *send_info_pool_rx; 918 struct mlx5dr_send_info_pool *send_info_pool_tx; 919 struct kmem_cache *chunks_kmem_cache; 920 struct kmem_cache *htbls_kmem_cache; 921 struct mlx5dr_send_ring *send_ring; 922 struct mlx5dr_domain_info info; 923 struct xarray csum_fts_xa; 924 struct mlx5dr_ste_ctx *ste_ctx; 925 struct list_head dbg_tbl_list; 926 struct mlx5dr_dbg_dump_info dump_info; 927 }; 928 929 struct mlx5dr_table_rx_tx { 930 struct mlx5dr_ste_htbl *s_anchor; 931 struct mlx5dr_domain_rx_tx *nic_dmn; 932 u64 default_icm_addr; 933 struct list_head nic_matcher_list; 934 }; 935 936 struct mlx5dr_table { 937 struct mlx5dr_domain *dmn; 938 struct mlx5dr_table_rx_tx rx; 939 struct mlx5dr_table_rx_tx tx; 940 u32 level; 941 u32 table_type; 942 u32 table_id; 943 u32 flags; 944 struct list_head matcher_list; 945 struct mlx5dr_action *miss_action; 946 refcount_t refcount; 947 struct list_head dbg_node; 948 }; 949 950 struct mlx5dr_matcher_rx_tx { 951 struct mlx5dr_ste_htbl *s_htbl; 952 struct mlx5dr_ste_htbl *e_anchor; 953 struct mlx5dr_ste_build *ste_builder; 954 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX] 955 [DR_RULE_IPV_MAX] 956 [DR_RULE_MAX_STES]; 957 u8 num_of_builders; 958 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX]; 959 u64 default_icm_addr; 960 struct mlx5dr_table_rx_tx *nic_tbl; 961 u32 prio; 962 struct list_head list_node; 963 u32 rules; 964 }; 965 966 struct mlx5dr_matcher { 967 struct mlx5dr_table *tbl; 968 struct mlx5dr_matcher_rx_tx rx; 969 struct mlx5dr_matcher_rx_tx tx; 970 struct list_head list_node; /* Used for both matchers and dbg managing */ 971 u32 prio; 972 struct mlx5dr_match_param mask; 973 u8 match_criteria; 974 refcount_t refcount; 975 struct list_head dbg_rule_list; 976 }; 977 978 struct mlx5dr_ste_action_modify_field { 979 u16 hw_field; 980 u8 start; 981 u8 end; 982 u8 l3_type; 983 u8 l4_type; 984 }; 985 986 struct mlx5dr_action_rewrite { 987 struct mlx5dr_domain *dmn; 988 struct mlx5dr_icm_chunk *chunk; 989 u8 *data; 990 u16 num_of_actions; 991 u32 index; 992 u8 allow_rx:1; 993 u8 allow_tx:1; 994 u8 modify_ttl:1; 995 }; 996 997 struct mlx5dr_action_reformat { 998 struct mlx5dr_domain *dmn; 999 u32 id; 1000 u32 size; 1001 u8 param_0; 1002 u8 param_1; 1003 }; 1004 1005 struct mlx5dr_action_sampler { 1006 struct mlx5dr_domain *dmn; 1007 u64 rx_icm_addr; 1008 u64 tx_icm_addr; 1009 u32 sampler_id; 1010 }; 1011 1012 struct mlx5dr_action_dest_tbl { 1013 u8 is_fw_tbl:1; 1014 union { 1015 struct mlx5dr_table *tbl; 1016 struct { 1017 struct mlx5dr_domain *dmn; 1018 u32 id; 1019 u32 group_id; 1020 enum fs_flow_table_type type; 1021 u64 rx_icm_addr; 1022 u64 tx_icm_addr; 1023 struct mlx5dr_action **ref_actions; 1024 u32 num_of_ref_actions; 1025 } fw_tbl; 1026 }; 1027 }; 1028 1029 struct mlx5dr_action_ctr { 1030 u32 ctr_id; 1031 u32 offset; 1032 }; 1033 1034 struct mlx5dr_action_vport { 1035 struct mlx5dr_domain *dmn; 1036 struct mlx5dr_cmd_vport_cap *caps; 1037 }; 1038 1039 struct mlx5dr_action_push_vlan { 1040 u32 vlan_hdr; /* tpid_pcp_dei_vid */ 1041 }; 1042 1043 struct mlx5dr_action_flow_tag { 1044 u32 flow_tag; 1045 }; 1046 1047 struct mlx5dr_rule_action_member { 1048 struct mlx5dr_action *action; 1049 struct list_head list; 1050 }; 1051 1052 struct mlx5dr_action_aso_flow_meter { 1053 struct mlx5dr_domain *dmn; 1054 u32 obj_id; 1055 u32 offset; 1056 u8 dest_reg_id; 1057 u8 init_color; 1058 }; 1059 1060 struct mlx5dr_action { 1061 enum mlx5dr_action_type action_type; 1062 refcount_t refcount; 1063 1064 union { 1065 void *data; 1066 struct mlx5dr_action_rewrite *rewrite; 1067 struct mlx5dr_action_reformat *reformat; 1068 struct mlx5dr_action_sampler *sampler; 1069 struct mlx5dr_action_dest_tbl *dest_tbl; 1070 struct mlx5dr_action_ctr *ctr; 1071 struct mlx5dr_action_vport *vport; 1072 struct mlx5dr_action_push_vlan *push_vlan; 1073 struct mlx5dr_action_flow_tag *flow_tag; 1074 struct mlx5dr_action_aso_flow_meter *aso; 1075 }; 1076 }; 1077 1078 enum mlx5dr_connect_type { 1079 CONNECT_HIT = 1, 1080 CONNECT_MISS = 2, 1081 }; 1082 1083 struct mlx5dr_htbl_connect_info { 1084 enum mlx5dr_connect_type type; 1085 union { 1086 struct mlx5dr_ste_htbl *hit_next_htbl; 1087 u64 miss_icm_addr; 1088 }; 1089 }; 1090 1091 struct mlx5dr_rule_rx_tx { 1092 struct mlx5dr_matcher_rx_tx *nic_matcher; 1093 struct mlx5dr_ste *last_rule_ste; 1094 }; 1095 1096 struct mlx5dr_rule { 1097 struct mlx5dr_matcher *matcher; 1098 struct mlx5dr_rule_rx_tx rx; 1099 struct mlx5dr_rule_rx_tx tx; 1100 struct list_head rule_actions_list; 1101 struct list_head dbg_node; 1102 u32 flow_source; 1103 }; 1104 1105 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule, 1106 struct mlx5dr_ste *ste, 1107 bool force); 1108 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr, 1109 struct mlx5dr_ste *curr_ste, 1110 int *num_of_stes); 1111 1112 struct mlx5dr_icm_chunk { 1113 struct mlx5dr_icm_buddy_mem *buddy_mem; 1114 struct list_head chunk_list; 1115 1116 /* indicates the index of this chunk in the whole memory, 1117 * used for deleting the chunk from the buddy 1118 */ 1119 unsigned int seg; 1120 enum mlx5dr_icm_chunk_size size; 1121 1122 /* Memory optimisation */ 1123 struct mlx5dr_ste *ste_arr; 1124 u8 *hw_ste_arr; 1125 struct list_head *miss_list; 1126 }; 1127 1128 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn) 1129 { 1130 mutex_lock(&nic_dmn->mutex); 1131 } 1132 1133 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn) 1134 { 1135 mutex_unlock(&nic_dmn->mutex); 1136 } 1137 1138 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn) 1139 { 1140 mlx5dr_domain_nic_lock(&dmn->info.rx); 1141 mlx5dr_domain_nic_lock(&dmn->info.tx); 1142 } 1143 1144 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn) 1145 { 1146 mlx5dr_domain_nic_unlock(&dmn->info.tx); 1147 mlx5dr_domain_nic_unlock(&dmn->info.rx); 1148 } 1149 1150 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn, 1151 struct mlx5dr_matcher_rx_tx *nic_matcher); 1152 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn, 1153 struct mlx5dr_matcher_rx_tx *nic_matcher); 1154 1155 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher, 1156 struct mlx5dr_matcher_rx_tx *nic_matcher, 1157 enum mlx5dr_ipv outer_ipv, 1158 enum mlx5dr_ipv inner_ipv); 1159 1160 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk); 1161 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk); 1162 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk); 1163 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk); 1164 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk); 1165 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste); 1166 1167 struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool); 1168 void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl); 1169 1170 static inline int 1171 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type) 1172 { 1173 if (icm_type == DR_ICM_TYPE_STE) 1174 return DR_STE_SIZE; 1175 1176 return DR_MODIFY_ACTION_SIZE; 1177 } 1178 1179 static inline u32 1180 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size) 1181 { 1182 return 1 << chunk_size; 1183 } 1184 1185 static inline int 1186 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size, 1187 enum mlx5dr_icm_type icm_type) 1188 { 1189 int num_of_entries; 1190 int entry_size; 1191 1192 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type); 1193 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size); 1194 1195 return entry_size * num_of_entries; 1196 } 1197 1198 static inline int 1199 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl) 1200 { 1201 int num_of_entries = 1202 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size); 1203 1204 /* Threshold is 50%, one is added to table of size 1 */ 1205 return (num_of_entries + 1) / 2; 1206 } 1207 1208 static inline bool 1209 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl) 1210 { 1211 if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask) 1212 return false; 1213 1214 return true; 1215 } 1216 1217 struct mlx5dr_cmd_vport_cap * 1218 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport); 1219 1220 struct mlx5dr_cmd_query_flow_table_details { 1221 u8 status; 1222 u8 level; 1223 u64 sw_owner_icm_root_1; 1224 u64 sw_owner_icm_root_0; 1225 }; 1226 1227 struct mlx5dr_cmd_create_flow_table_attr { 1228 u32 table_type; 1229 u16 uid; 1230 u64 icm_addr_rx; 1231 u64 icm_addr_tx; 1232 u8 level; 1233 bool sw_owner; 1234 bool term_tbl; 1235 bool decap_en; 1236 bool reformat_en; 1237 }; 1238 1239 /* internal API functions */ 1240 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, 1241 struct mlx5dr_cmd_caps *caps); 1242 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev, 1243 bool other_vport, u16 vport_number, 1244 u64 *icm_address_rx, 1245 u64 *icm_address_tx); 1246 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev, 1247 bool other_vport, u16 vport_number, u16 *gvmi); 1248 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev, 1249 struct mlx5dr_esw_caps *caps); 1250 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev, 1251 u32 sampler_id, 1252 u64 *rx_icm_addr, 1253 u64 *tx_icm_addr); 1254 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev); 1255 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev, 1256 u32 table_type, 1257 u32 table_id, 1258 u32 group_id, 1259 u32 modify_header_id, 1260 u16 vport_id); 1261 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev, 1262 u32 table_type, 1263 u32 table_id); 1264 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev, 1265 u32 table_type, 1266 u8 num_of_actions, 1267 u64 *actions, 1268 u32 *modify_header_id); 1269 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev, 1270 u32 modify_header_id); 1271 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev, 1272 u32 table_type, 1273 u32 table_id, 1274 u32 *group_id); 1275 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev, 1276 u32 table_type, 1277 u32 table_id, 1278 u32 group_id); 1279 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev, 1280 struct mlx5dr_cmd_create_flow_table_attr *attr, 1281 u64 *fdb_rx_icm_addr, 1282 u32 *table_id); 1283 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev, 1284 u32 table_id, 1285 u32 table_type); 1286 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev, 1287 enum fs_flow_table_type type, 1288 u32 table_id, 1289 struct mlx5dr_cmd_query_flow_table_details *output); 1290 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, 1291 enum mlx5_reformat_ctx_type rt, 1292 u8 reformat_param_0, 1293 u8 reformat_param_1, 1294 size_t reformat_size, 1295 void *reformat_data, 1296 u32 *reformat_id); 1297 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev, 1298 u32 reformat_id); 1299 1300 struct mlx5dr_cmd_gid_attr { 1301 u8 gid[16]; 1302 u8 mac[6]; 1303 u32 roce_ver; 1304 }; 1305 1306 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num, 1307 u16 index, struct mlx5dr_cmd_gid_attr *attr); 1308 1309 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 1310 enum mlx5dr_icm_type icm_type); 1311 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool); 1312 1313 struct mlx5dr_icm_chunk * 1314 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 1315 enum mlx5dr_icm_chunk_size chunk_size); 1316 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk); 1317 1318 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx, 1319 u8 *hw_ste_p, u32 ste_size); 1320 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn, 1321 struct mlx5dr_domain_rx_tx *nic_dmn, 1322 struct mlx5dr_ste_htbl *htbl, 1323 struct mlx5dr_htbl_connect_info *connect_info, 1324 bool update_hw_ste); 1325 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx, 1326 u16 gvmi, 1327 enum mlx5dr_domain_nic_type nic_type, 1328 struct mlx5dr_ste_htbl *htbl, 1329 u8 *formatted_ste, 1330 struct mlx5dr_htbl_connect_info *connect_info); 1331 void mlx5dr_ste_copy_param(u8 match_criteria, 1332 struct mlx5dr_match_param *set_param, 1333 struct mlx5dr_match_parameters *mask, 1334 bool clear); 1335 1336 struct mlx5dr_qp { 1337 struct mlx5_core_dev *mdev; 1338 struct mlx5_wq_qp wq; 1339 struct mlx5_uars_page *uar; 1340 struct mlx5_wq_ctrl wq_ctrl; 1341 u32 qpn; 1342 struct { 1343 unsigned int pc; 1344 unsigned int cc; 1345 unsigned int size; 1346 unsigned int *wqe_head; 1347 unsigned int wqe_cnt; 1348 } sq; 1349 struct { 1350 unsigned int pc; 1351 unsigned int cc; 1352 unsigned int size; 1353 unsigned int wqe_cnt; 1354 } rq; 1355 int max_inline_data; 1356 }; 1357 1358 struct mlx5dr_cq { 1359 struct mlx5_core_dev *mdev; 1360 struct mlx5_cqwq wq; 1361 struct mlx5_wq_ctrl wq_ctrl; 1362 struct mlx5_core_cq mcq; 1363 struct mlx5dr_qp *qp; 1364 }; 1365 1366 struct mlx5dr_mr { 1367 struct mlx5_core_dev *mdev; 1368 u32 mkey; 1369 dma_addr_t dma_addr; 1370 void *addr; 1371 size_t size; 1372 }; 1373 1374 #define MAX_SEND_CQE 64 1375 #define MIN_READ_SYNC 64 1376 1377 struct mlx5dr_send_ring { 1378 struct mlx5dr_cq *cq; 1379 struct mlx5dr_qp *qp; 1380 struct mlx5dr_mr *mr; 1381 /* How much wqes are waiting for completion */ 1382 u32 pending_wqe; 1383 /* Signal request per this trash hold value */ 1384 u16 signal_th; 1385 /* Each post_send_size less than max_post_send_size */ 1386 u32 max_post_send_size; 1387 /* manage the send queue */ 1388 u32 tx_head; 1389 void *buf; 1390 u32 buf_size; 1391 u8 sync_buff[MIN_READ_SYNC]; 1392 struct mlx5dr_mr *sync_mr; 1393 spinlock_t lock; /* Protect the data path of the send ring */ 1394 bool err_state; /* send_ring is not usable in err state */ 1395 }; 1396 1397 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn); 1398 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn, 1399 struct mlx5dr_send_ring *send_ring); 1400 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn); 1401 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn, 1402 struct mlx5dr_ste *ste, 1403 u8 *data, 1404 u16 size, 1405 u16 offset); 1406 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, 1407 struct mlx5dr_ste_htbl *htbl, 1408 u8 *formatted_ste, u8 *mask); 1409 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn, 1410 struct mlx5dr_ste_htbl *htbl, 1411 u8 *ste_init_data, 1412 bool update_hw_ste); 1413 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn, 1414 struct mlx5dr_action *action); 1415 1416 int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn); 1417 void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn); 1418 struct mlx5dr_ste_send_info *mlx5dr_send_info_alloc(struct mlx5dr_domain *dmn, 1419 enum mlx5dr_domain_nic_type nic_type); 1420 void mlx5dr_send_info_free(struct mlx5dr_ste_send_info *ste_send_info); 1421 1422 struct mlx5dr_cmd_ft_info { 1423 u32 id; 1424 u16 vport; 1425 enum fs_flow_table_type type; 1426 }; 1427 1428 struct mlx5dr_cmd_flow_destination_hw_info { 1429 enum mlx5_flow_destination_type type; 1430 union { 1431 u32 tir_num; 1432 u32 ft_num; 1433 u32 ft_id; 1434 u32 counter_id; 1435 u32 sampler_id; 1436 struct { 1437 u16 num; 1438 u16 vhca_id; 1439 u32 reformat_id; 1440 u8 flags; 1441 } vport; 1442 }; 1443 }; 1444 1445 struct mlx5dr_cmd_fte_info { 1446 u32 dests_size; 1447 u32 index; 1448 struct mlx5_flow_context flow_context; 1449 u32 *val; 1450 struct mlx5_flow_act action; 1451 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr; 1452 bool ignore_flow_level; 1453 }; 1454 1455 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev, 1456 int opmod, int modify_mask, 1457 struct mlx5dr_cmd_ft_info *ft, 1458 u32 group_id, 1459 struct mlx5dr_cmd_fte_info *fte); 1460 1461 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps); 1462 1463 struct mlx5dr_fw_recalc_cs_ft { 1464 u64 rx_icm_addr; 1465 u32 table_id; 1466 u32 group_id; 1467 u32 modify_hdr_id; 1468 }; 1469 1470 struct mlx5dr_fw_recalc_cs_ft * 1471 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num); 1472 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn, 1473 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft); 1474 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn, 1475 u16 vport_num, 1476 u64 *rx_icm_addr); 1477 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn, 1478 struct mlx5dr_cmd_flow_destination_hw_info *dest, 1479 int num_dest, 1480 bool reformat_req, 1481 u32 *tbl_id, 1482 u32 *group_id, 1483 bool ignore_flow_level, 1484 u32 flow_source); 1485 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id, 1486 u32 group_id); 1487 #endif /* _DR_TYPES_H_ */ 1488