126d688e3SAlex Vesker // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 226d688e3SAlex Vesker /* Copyright (c) 2019 Mellanox Technologies. */ 326d688e3SAlex Vesker 426d688e3SAlex Vesker #include <linux/types.h> 540416d8eSHamdan Igbaria #include <linux/crc32.h> 64bdba39bSYevgeny Kliteynik #include "dr_ste.h" 726d688e3SAlex Vesker 826d688e3SAlex Vesker struct dr_hw_ste_format { 926d688e3SAlex Vesker u8 ctrl[DR_STE_SIZE_CTRL]; 1026d688e3SAlex Vesker u8 tag[DR_STE_SIZE_TAG]; 1126d688e3SAlex Vesker u8 mask[DR_STE_SIZE_MASK]; 1226d688e3SAlex Vesker }; 1326d688e3SAlex Vesker 1440416d8eSHamdan Igbaria static u32 dr_ste_crc32_calc(const void *input_data, size_t length) 1540416d8eSHamdan Igbaria { 1640416d8eSHamdan Igbaria u32 crc = crc32(0, input_data, length); 1740416d8eSHamdan Igbaria 189ff2e92cSSaeed Mahameed return (__force u32)htonl(crc); 1940416d8eSHamdan Igbaria } 2040416d8eSHamdan Igbaria 21a283ea1bSYevgeny Kliteynik bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps) 22a283ea1bSYevgeny Kliteynik { 23a283ea1bSYevgeny Kliteynik return caps->sw_format_ver > MLX5_STEERING_FORMAT_CONNECTX_5; 24a283ea1bSYevgeny Kliteynik } 25a283ea1bSYevgeny Kliteynik 2626d688e3SAlex Vesker u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl) 2726d688e3SAlex Vesker { 2826d688e3SAlex Vesker struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; 2926d688e3SAlex Vesker u8 masked[DR_STE_SIZE_TAG] = {}; 3026d688e3SAlex Vesker u32 crc32, index; 3126d688e3SAlex Vesker u16 bit; 3226d688e3SAlex Vesker int i; 3326d688e3SAlex Vesker 3426d688e3SAlex Vesker /* Don't calculate CRC if the result is predicted */ 3526d688e3SAlex Vesker if (htbl->chunk->num_of_entries == 1 || htbl->byte_mask == 0) 3626d688e3SAlex Vesker return 0; 3726d688e3SAlex Vesker 3826d688e3SAlex Vesker /* Mask tag using byte mask, bit per byte */ 3926d688e3SAlex Vesker bit = 1 << (DR_STE_SIZE_TAG - 1); 4026d688e3SAlex Vesker for (i = 0; i < DR_STE_SIZE_TAG; i++) { 4126d688e3SAlex Vesker if (htbl->byte_mask & bit) 4226d688e3SAlex Vesker masked[i] = hw_ste->tag[i]; 4326d688e3SAlex Vesker 4426d688e3SAlex Vesker bit = bit >> 1; 4526d688e3SAlex Vesker } 4626d688e3SAlex Vesker 4740416d8eSHamdan Igbaria crc32 = dr_ste_crc32_calc(masked, DR_STE_SIZE_TAG); 4826d688e3SAlex Vesker index = crc32 & (htbl->chunk->num_of_entries - 1); 4926d688e3SAlex Vesker 5026d688e3SAlex Vesker return index; 5126d688e3SAlex Vesker } 5226d688e3SAlex Vesker 535212f9c6SYevgeny Kliteynik u16 mlx5dr_ste_conv_bit_to_byte_mask(u8 *bit_mask) 5426d688e3SAlex Vesker { 5526d688e3SAlex Vesker u16 byte_mask = 0; 5626d688e3SAlex Vesker int i; 5726d688e3SAlex Vesker 5826d688e3SAlex Vesker for (i = 0; i < DR_STE_SIZE_MASK; i++) { 5926d688e3SAlex Vesker byte_mask = byte_mask << 1; 6026d688e3SAlex Vesker if (bit_mask[i] == 0xff) 6126d688e3SAlex Vesker byte_mask |= 1; 6226d688e3SAlex Vesker } 6326d688e3SAlex Vesker return byte_mask; 6426d688e3SAlex Vesker } 6526d688e3SAlex Vesker 6664c78942SYevgeny Kliteynik static u8 *dr_ste_get_tag(u8 *hw_ste_p) 67e6b69bf3SYevgeny Kliteynik { 68e6b69bf3SYevgeny Kliteynik struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; 69e6b69bf3SYevgeny Kliteynik 70e6b69bf3SYevgeny Kliteynik return hw_ste->tag; 71e6b69bf3SYevgeny Kliteynik } 72e6b69bf3SYevgeny Kliteynik 7326d688e3SAlex Vesker void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask) 7426d688e3SAlex Vesker { 7526d688e3SAlex Vesker struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; 7626d688e3SAlex Vesker 7726d688e3SAlex Vesker memcpy(hw_ste->mask, bit_mask, DR_STE_SIZE_MASK); 7826d688e3SAlex Vesker } 7926d688e3SAlex Vesker 8026d688e3SAlex Vesker static void dr_ste_set_always_hit(struct dr_hw_ste_format *hw_ste) 8126d688e3SAlex Vesker { 8226d688e3SAlex Vesker memset(&hw_ste->tag, 0, sizeof(hw_ste->tag)); 8326d688e3SAlex Vesker memset(&hw_ste->mask, 0, sizeof(hw_ste->mask)); 8426d688e3SAlex Vesker } 8526d688e3SAlex Vesker 8626d688e3SAlex Vesker static void dr_ste_set_always_miss(struct dr_hw_ste_format *hw_ste) 8726d688e3SAlex Vesker { 8826d688e3SAlex Vesker hw_ste->tag[0] = 0xdc; 8926d688e3SAlex Vesker hw_ste->mask[0] = 0; 9026d688e3SAlex Vesker } 9126d688e3SAlex Vesker 926b93b400SYevgeny Kliteynik void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, 936b93b400SYevgeny Kliteynik u8 *hw_ste_p, u64 miss_addr) 9464c78942SYevgeny Kliteynik { 956b93b400SYevgeny Kliteynik ste_ctx->set_miss_addr(hw_ste_p, miss_addr); 9664c78942SYevgeny Kliteynik } 9764c78942SYevgeny Kliteynik 986b93b400SYevgeny Kliteynik static void dr_ste_always_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, 996b93b400SYevgeny Kliteynik struct mlx5dr_ste *ste, u64 miss_addr) 10064c78942SYevgeny Kliteynik { 10164c78942SYevgeny Kliteynik u8 *hw_ste_p = ste->hw_ste; 10264c78942SYevgeny Kliteynik 1036b93b400SYevgeny Kliteynik ste_ctx->set_next_lu_type(hw_ste_p, MLX5DR_STE_LU_TYPE_DONT_CARE); 1046b93b400SYevgeny Kliteynik ste_ctx->set_miss_addr(hw_ste_p, miss_addr); 10564c78942SYevgeny Kliteynik dr_ste_set_always_miss((struct dr_hw_ste_format *)ste->hw_ste); 10664c78942SYevgeny Kliteynik } 10764c78942SYevgeny Kliteynik 1086b93b400SYevgeny Kliteynik void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx, 1096b93b400SYevgeny Kliteynik u8 *hw_ste, u64 icm_addr, u32 ht_size) 11026d688e3SAlex Vesker { 1116b93b400SYevgeny Kliteynik ste_ctx->set_hit_addr(hw_ste, icm_addr, ht_size); 11226d688e3SAlex Vesker } 11326d688e3SAlex Vesker 11426d688e3SAlex Vesker u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste) 11526d688e3SAlex Vesker { 11626d688e3SAlex Vesker u32 index = ste - ste->htbl->ste_arr; 11726d688e3SAlex Vesker 11826d688e3SAlex Vesker return ste->htbl->chunk->icm_addr + DR_STE_SIZE * index; 11926d688e3SAlex Vesker } 12026d688e3SAlex Vesker 12126d688e3SAlex Vesker u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste) 12226d688e3SAlex Vesker { 12326d688e3SAlex Vesker u32 index = ste - ste->htbl->ste_arr; 12426d688e3SAlex Vesker 12526d688e3SAlex Vesker return ste->htbl->chunk->mr_addr + DR_STE_SIZE * index; 12626d688e3SAlex Vesker } 12726d688e3SAlex Vesker 12826d688e3SAlex Vesker struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste) 12926d688e3SAlex Vesker { 13026d688e3SAlex Vesker u32 index = ste - ste->htbl->ste_arr; 13126d688e3SAlex Vesker 13226d688e3SAlex Vesker return &ste->htbl->miss_list[index]; 13326d688e3SAlex Vesker } 13426d688e3SAlex Vesker 1356b93b400SYevgeny Kliteynik static void dr_ste_always_hit_htbl(struct mlx5dr_ste_ctx *ste_ctx, 1366b93b400SYevgeny Kliteynik struct mlx5dr_ste *ste, 13726d688e3SAlex Vesker struct mlx5dr_ste_htbl *next_htbl) 13826d688e3SAlex Vesker { 13926d688e3SAlex Vesker struct mlx5dr_icm_chunk *chunk = next_htbl->chunk; 14026d688e3SAlex Vesker u8 *hw_ste = ste->hw_ste; 14126d688e3SAlex Vesker 1426b93b400SYevgeny Kliteynik ste_ctx->set_byte_mask(hw_ste, next_htbl->byte_mask); 1436b93b400SYevgeny Kliteynik ste_ctx->set_next_lu_type(hw_ste, next_htbl->lu_type); 1446b93b400SYevgeny Kliteynik ste_ctx->set_hit_addr(hw_ste, chunk->icm_addr, chunk->num_of_entries); 14526d688e3SAlex Vesker 14626d688e3SAlex Vesker dr_ste_set_always_hit((struct dr_hw_ste_format *)ste->hw_ste); 14726d688e3SAlex Vesker } 14826d688e3SAlex Vesker 14926d688e3SAlex Vesker bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher, 15026d688e3SAlex Vesker u8 ste_location) 15126d688e3SAlex Vesker { 15226d688e3SAlex Vesker return ste_location == nic_matcher->num_of_builders; 15326d688e3SAlex Vesker } 15426d688e3SAlex Vesker 15526d688e3SAlex Vesker /* Replace relevant fields, except of: 15626d688e3SAlex Vesker * htbl - keep the origin htbl 15726d688e3SAlex Vesker * miss_list + list - already took the src from the list. 15826d688e3SAlex Vesker * icm_addr/mr_addr - depends on the hosting table. 15926d688e3SAlex Vesker * 16026d688e3SAlex Vesker * Before: 16126d688e3SAlex Vesker * | a | -> | b | -> | c | -> 16226d688e3SAlex Vesker * 16326d688e3SAlex Vesker * After: 16426d688e3SAlex Vesker * | a | -> | c | -> 16526d688e3SAlex Vesker * While the data that was in b copied to a. 16626d688e3SAlex Vesker */ 16726d688e3SAlex Vesker static void dr_ste_replace(struct mlx5dr_ste *dst, struct mlx5dr_ste *src) 16826d688e3SAlex Vesker { 16926d688e3SAlex Vesker memcpy(dst->hw_ste, src->hw_ste, DR_STE_SIZE_REDUCED); 17026d688e3SAlex Vesker dst->next_htbl = src->next_htbl; 17126d688e3SAlex Vesker if (dst->next_htbl) 17226d688e3SAlex Vesker dst->next_htbl->pointing_ste = dst; 17326d688e3SAlex Vesker 1744ce380caSYevgeny Kliteynik dst->refcount = src->refcount; 17526d688e3SAlex Vesker 17626d688e3SAlex Vesker INIT_LIST_HEAD(&dst->rule_list); 17726d688e3SAlex Vesker list_splice_tail_init(&src->rule_list, &dst->rule_list); 17826d688e3SAlex Vesker } 17926d688e3SAlex Vesker 18026d688e3SAlex Vesker /* Free ste which is the head and the only one in miss_list */ 18126d688e3SAlex Vesker static void 1826b93b400SYevgeny Kliteynik dr_ste_remove_head_ste(struct mlx5dr_ste_ctx *ste_ctx, 1836b93b400SYevgeny Kliteynik struct mlx5dr_ste *ste, 18426d688e3SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 18526d688e3SAlex Vesker struct mlx5dr_ste_send_info *ste_info_head, 18626d688e3SAlex Vesker struct list_head *send_ste_list, 18726d688e3SAlex Vesker struct mlx5dr_ste_htbl *stats_tbl) 18826d688e3SAlex Vesker { 18926d688e3SAlex Vesker u8 tmp_data_ste[DR_STE_SIZE] = {}; 19026d688e3SAlex Vesker struct mlx5dr_ste tmp_ste = {}; 19126d688e3SAlex Vesker u64 miss_addr; 19226d688e3SAlex Vesker 19326d688e3SAlex Vesker tmp_ste.hw_ste = tmp_data_ste; 19426d688e3SAlex Vesker 19526d688e3SAlex Vesker /* Use temp ste because dr_ste_always_miss_addr 19626d688e3SAlex Vesker * touches bit_mask area which doesn't exist at ste->hw_ste. 19726d688e3SAlex Vesker */ 19826d688e3SAlex Vesker memcpy(tmp_ste.hw_ste, ste->hw_ste, DR_STE_SIZE_REDUCED); 19926d688e3SAlex Vesker miss_addr = nic_matcher->e_anchor->chunk->icm_addr; 2006b93b400SYevgeny Kliteynik dr_ste_always_miss_addr(ste_ctx, &tmp_ste, miss_addr); 20126d688e3SAlex Vesker memcpy(ste->hw_ste, tmp_ste.hw_ste, DR_STE_SIZE_REDUCED); 20226d688e3SAlex Vesker 20326d688e3SAlex Vesker list_del_init(&ste->miss_list_node); 20426d688e3SAlex Vesker 20526d688e3SAlex Vesker /* Write full STE size in order to have "always_miss" */ 20626d688e3SAlex Vesker mlx5dr_send_fill_and_append_ste_send_info(ste, DR_STE_SIZE, 20726d688e3SAlex Vesker 0, tmp_data_ste, 20826d688e3SAlex Vesker ste_info_head, 20926d688e3SAlex Vesker send_ste_list, 21026d688e3SAlex Vesker true /* Copy data */); 21126d688e3SAlex Vesker 21226d688e3SAlex Vesker stats_tbl->ctrl.num_of_valid_entries--; 21326d688e3SAlex Vesker } 21426d688e3SAlex Vesker 21526d688e3SAlex Vesker /* Free ste which is the head but NOT the only one in miss_list: 21626d688e3SAlex Vesker * |_ste_| --> |_next_ste_| -->|__| -->|__| -->/0 21726d688e3SAlex Vesker */ 21826d688e3SAlex Vesker static void 2198fdac12aSYevgeny Kliteynik dr_ste_replace_head_ste(struct mlx5dr_matcher_rx_tx *nic_matcher, 2208fdac12aSYevgeny Kliteynik struct mlx5dr_ste *ste, 2218fdac12aSYevgeny Kliteynik struct mlx5dr_ste *next_ste, 22226d688e3SAlex Vesker struct mlx5dr_ste_send_info *ste_info_head, 22326d688e3SAlex Vesker struct list_head *send_ste_list, 22426d688e3SAlex Vesker struct mlx5dr_ste_htbl *stats_tbl) 22526d688e3SAlex Vesker 22626d688e3SAlex Vesker { 22726d688e3SAlex Vesker struct mlx5dr_ste_htbl *next_miss_htbl; 2288fdac12aSYevgeny Kliteynik u8 hw_ste[DR_STE_SIZE] = {}; 2298fdac12aSYevgeny Kliteynik int sb_idx; 23026d688e3SAlex Vesker 23126d688e3SAlex Vesker next_miss_htbl = next_ste->htbl; 23226d688e3SAlex Vesker 23326d688e3SAlex Vesker /* Remove from the miss_list the next_ste before copy */ 23426d688e3SAlex Vesker list_del_init(&next_ste->miss_list_node); 23526d688e3SAlex Vesker 23626d688e3SAlex Vesker /* All rule-members that use next_ste should know about that */ 23726d688e3SAlex Vesker mlx5dr_rule_update_rule_member(next_ste, ste); 23826d688e3SAlex Vesker 23926d688e3SAlex Vesker /* Move data from next into ste */ 24026d688e3SAlex Vesker dr_ste_replace(ste, next_ste); 24126d688e3SAlex Vesker 2428fdac12aSYevgeny Kliteynik /* Copy all 64 hw_ste bytes */ 2438fdac12aSYevgeny Kliteynik memcpy(hw_ste, ste->hw_ste, DR_STE_SIZE_REDUCED); 2448fdac12aSYevgeny Kliteynik sb_idx = ste->ste_chain_location - 1; 2458fdac12aSYevgeny Kliteynik mlx5dr_ste_set_bit_mask(hw_ste, 2468fdac12aSYevgeny Kliteynik nic_matcher->ste_builder[sb_idx].bit_mask); 2478fdac12aSYevgeny Kliteynik 24826d688e3SAlex Vesker /* Del the htbl that contains the next_ste. 24926d688e3SAlex Vesker * The origin htbl stay with the same number of entries. 25026d688e3SAlex Vesker */ 25126d688e3SAlex Vesker mlx5dr_htbl_put(next_miss_htbl); 25226d688e3SAlex Vesker 2538fdac12aSYevgeny Kliteynik mlx5dr_send_fill_and_append_ste_send_info(ste, DR_STE_SIZE, 2548fdac12aSYevgeny Kliteynik 0, hw_ste, 25526d688e3SAlex Vesker ste_info_head, 25626d688e3SAlex Vesker send_ste_list, 25726d688e3SAlex Vesker true /* Copy data */); 25826d688e3SAlex Vesker 25926d688e3SAlex Vesker stats_tbl->ctrl.num_of_collisions--; 26026d688e3SAlex Vesker stats_tbl->ctrl.num_of_valid_entries--; 26126d688e3SAlex Vesker } 26226d688e3SAlex Vesker 26326d688e3SAlex Vesker /* Free ste that is located in the middle of the miss list: 26426d688e3SAlex Vesker * |__| -->|_prev_ste_|->|_ste_|-->|_next_ste_| 26526d688e3SAlex Vesker */ 2666b93b400SYevgeny Kliteynik static void dr_ste_remove_middle_ste(struct mlx5dr_ste_ctx *ste_ctx, 2676b93b400SYevgeny Kliteynik struct mlx5dr_ste *ste, 26826d688e3SAlex Vesker struct mlx5dr_ste_send_info *ste_info, 26926d688e3SAlex Vesker struct list_head *send_ste_list, 27026d688e3SAlex Vesker struct mlx5dr_ste_htbl *stats_tbl) 27126d688e3SAlex Vesker { 27226d688e3SAlex Vesker struct mlx5dr_ste *prev_ste; 27326d688e3SAlex Vesker u64 miss_addr; 27426d688e3SAlex Vesker 27548cbde4bSAlex Vesker prev_ste = list_prev_entry(ste, miss_list_node); 27648cbde4bSAlex Vesker if (WARN_ON(!prev_ste)) 27726d688e3SAlex Vesker return; 27826d688e3SAlex Vesker 2796b93b400SYevgeny Kliteynik miss_addr = ste_ctx->get_miss_addr(ste->hw_ste); 2806b93b400SYevgeny Kliteynik ste_ctx->set_miss_addr(prev_ste->hw_ste, miss_addr); 28126d688e3SAlex Vesker 282f06d4969SYevgeny Kliteynik mlx5dr_send_fill_and_append_ste_send_info(prev_ste, DR_STE_SIZE_CTRL, 0, 28326d688e3SAlex Vesker prev_ste->hw_ste, ste_info, 28426d688e3SAlex Vesker send_ste_list, true /* Copy data*/); 28526d688e3SAlex Vesker 28626d688e3SAlex Vesker list_del_init(&ste->miss_list_node); 28726d688e3SAlex Vesker 28826d688e3SAlex Vesker stats_tbl->ctrl.num_of_valid_entries--; 28926d688e3SAlex Vesker stats_tbl->ctrl.num_of_collisions--; 29026d688e3SAlex Vesker } 29126d688e3SAlex Vesker 29226d688e3SAlex Vesker void mlx5dr_ste_free(struct mlx5dr_ste *ste, 29326d688e3SAlex Vesker struct mlx5dr_matcher *matcher, 29426d688e3SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher) 29526d688e3SAlex Vesker { 29626d688e3SAlex Vesker struct mlx5dr_ste_send_info *cur_ste_info, *tmp_ste_info; 29726d688e3SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 2986b93b400SYevgeny Kliteynik struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; 29926d688e3SAlex Vesker struct mlx5dr_ste_send_info ste_info_head; 30026d688e3SAlex Vesker struct mlx5dr_ste *next_ste, *first_ste; 30126d688e3SAlex Vesker bool put_on_origin_table = true; 30226d688e3SAlex Vesker struct mlx5dr_ste_htbl *stats_tbl; 30326d688e3SAlex Vesker LIST_HEAD(send_ste_list); 30426d688e3SAlex Vesker 30548cbde4bSAlex Vesker first_ste = list_first_entry(mlx5dr_ste_get_miss_list(ste), 30626d688e3SAlex Vesker struct mlx5dr_ste, miss_list_node); 30726d688e3SAlex Vesker stats_tbl = first_ste->htbl; 30826d688e3SAlex Vesker 30926d688e3SAlex Vesker /* Two options: 31026d688e3SAlex Vesker * 1. ste is head: 31126d688e3SAlex Vesker * a. head ste is the only ste in the miss list 31226d688e3SAlex Vesker * b. head ste is not the only ste in the miss-list 31326d688e3SAlex Vesker * 2. ste is not head 31426d688e3SAlex Vesker */ 31526d688e3SAlex Vesker if (first_ste == ste) { /* Ste is the head */ 31626d688e3SAlex Vesker struct mlx5dr_ste *last_ste; 31726d688e3SAlex Vesker 31826d688e3SAlex Vesker last_ste = list_last_entry(mlx5dr_ste_get_miss_list(ste), 31926d688e3SAlex Vesker struct mlx5dr_ste, miss_list_node); 32026d688e3SAlex Vesker if (last_ste == first_ste) 32126d688e3SAlex Vesker next_ste = NULL; 32226d688e3SAlex Vesker else 32348cbde4bSAlex Vesker next_ste = list_next_entry(ste, miss_list_node); 32426d688e3SAlex Vesker 32526d688e3SAlex Vesker if (!next_ste) { 32626d688e3SAlex Vesker /* One and only entry in the list */ 3276b93b400SYevgeny Kliteynik dr_ste_remove_head_ste(ste_ctx, ste, 3286b93b400SYevgeny Kliteynik nic_matcher, 32926d688e3SAlex Vesker &ste_info_head, 33026d688e3SAlex Vesker &send_ste_list, 33126d688e3SAlex Vesker stats_tbl); 33226d688e3SAlex Vesker } else { 33326d688e3SAlex Vesker /* First but not only entry in the list */ 3348fdac12aSYevgeny Kliteynik dr_ste_replace_head_ste(nic_matcher, ste, 3358fdac12aSYevgeny Kliteynik next_ste, &ste_info_head, 33626d688e3SAlex Vesker &send_ste_list, stats_tbl); 33726d688e3SAlex Vesker put_on_origin_table = false; 33826d688e3SAlex Vesker } 33926d688e3SAlex Vesker } else { /* Ste in the middle of the list */ 3406b93b400SYevgeny Kliteynik dr_ste_remove_middle_ste(ste_ctx, ste, 3416b93b400SYevgeny Kliteynik &ste_info_head, &send_ste_list, 3426b93b400SYevgeny Kliteynik stats_tbl); 34326d688e3SAlex Vesker } 34426d688e3SAlex Vesker 34526d688e3SAlex Vesker /* Update HW */ 34626d688e3SAlex Vesker list_for_each_entry_safe(cur_ste_info, tmp_ste_info, 34726d688e3SAlex Vesker &send_ste_list, send_list) { 34826d688e3SAlex Vesker list_del(&cur_ste_info->send_list); 34926d688e3SAlex Vesker mlx5dr_send_postsend_ste(dmn, cur_ste_info->ste, 35026d688e3SAlex Vesker cur_ste_info->data, cur_ste_info->size, 35126d688e3SAlex Vesker cur_ste_info->offset); 35226d688e3SAlex Vesker } 35326d688e3SAlex Vesker 35426d688e3SAlex Vesker if (put_on_origin_table) 35526d688e3SAlex Vesker mlx5dr_htbl_put(ste->htbl); 35626d688e3SAlex Vesker } 35726d688e3SAlex Vesker 35826d688e3SAlex Vesker bool mlx5dr_ste_equal_tag(void *src, void *dst) 35926d688e3SAlex Vesker { 36026d688e3SAlex Vesker struct dr_hw_ste_format *s_hw_ste = (struct dr_hw_ste_format *)src; 36126d688e3SAlex Vesker struct dr_hw_ste_format *d_hw_ste = (struct dr_hw_ste_format *)dst; 36226d688e3SAlex Vesker 36326d688e3SAlex Vesker return !memcmp(s_hw_ste->tag, d_hw_ste->tag, DR_STE_SIZE_TAG); 36426d688e3SAlex Vesker } 36526d688e3SAlex Vesker 3666b93b400SYevgeny Kliteynik void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx, 3676b93b400SYevgeny Kliteynik u8 *hw_ste, 36826d688e3SAlex Vesker struct mlx5dr_ste_htbl *next_htbl) 36926d688e3SAlex Vesker { 37026d688e3SAlex Vesker struct mlx5dr_icm_chunk *chunk = next_htbl->chunk; 37126d688e3SAlex Vesker 3726b93b400SYevgeny Kliteynik ste_ctx->set_hit_addr(hw_ste, chunk->icm_addr, chunk->num_of_entries); 37326d688e3SAlex Vesker } 37426d688e3SAlex Vesker 3754fe45e1dSYevgeny Kliteynik void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx, 3764fe45e1dSYevgeny Kliteynik u8 *hw_ste_p, u32 ste_size) 3774fe45e1dSYevgeny Kliteynik { 3784fe45e1dSYevgeny Kliteynik if (ste_ctx->prepare_for_postsend) 3794fe45e1dSYevgeny Kliteynik ste_ctx->prepare_for_postsend(hw_ste_p, ste_size); 3804fe45e1dSYevgeny Kliteynik } 3814fe45e1dSYevgeny Kliteynik 38226d688e3SAlex Vesker /* Init one ste as a pattern for ste data array */ 3836b93b400SYevgeny Kliteynik void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx, 3846b93b400SYevgeny Kliteynik u16 gvmi, 38526d688e3SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn, 38626d688e3SAlex Vesker struct mlx5dr_ste_htbl *htbl, 38726d688e3SAlex Vesker u8 *formatted_ste, 38826d688e3SAlex Vesker struct mlx5dr_htbl_connect_info *connect_info) 38926d688e3SAlex Vesker { 39026d688e3SAlex Vesker struct mlx5dr_ste ste = {}; 39126d688e3SAlex Vesker 3926b93b400SYevgeny Kliteynik ste_ctx->ste_init(formatted_ste, htbl->lu_type, nic_dmn->ste_type, gvmi); 39326d688e3SAlex Vesker ste.hw_ste = formatted_ste; 39426d688e3SAlex Vesker 39526d688e3SAlex Vesker if (connect_info->type == CONNECT_HIT) 3966b93b400SYevgeny Kliteynik dr_ste_always_hit_htbl(ste_ctx, &ste, connect_info->hit_next_htbl); 39726d688e3SAlex Vesker else 3986b93b400SYevgeny Kliteynik dr_ste_always_miss_addr(ste_ctx, &ste, connect_info->miss_icm_addr); 39926d688e3SAlex Vesker } 40026d688e3SAlex Vesker 40126d688e3SAlex Vesker int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn, 40226d688e3SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn, 40326d688e3SAlex Vesker struct mlx5dr_ste_htbl *htbl, 40426d688e3SAlex Vesker struct mlx5dr_htbl_connect_info *connect_info, 40526d688e3SAlex Vesker bool update_hw_ste) 40626d688e3SAlex Vesker { 40726d688e3SAlex Vesker u8 formatted_ste[DR_STE_SIZE] = {}; 40826d688e3SAlex Vesker 4096b93b400SYevgeny Kliteynik mlx5dr_ste_set_formatted_ste(dmn->ste_ctx, 4106b93b400SYevgeny Kliteynik dmn->info.caps.gvmi, 41126d688e3SAlex Vesker nic_dmn, 41226d688e3SAlex Vesker htbl, 41326d688e3SAlex Vesker formatted_ste, 41426d688e3SAlex Vesker connect_info); 41526d688e3SAlex Vesker 41626d688e3SAlex Vesker return mlx5dr_send_postsend_formatted_htbl(dmn, htbl, formatted_ste, update_hw_ste); 41726d688e3SAlex Vesker } 41826d688e3SAlex Vesker 41926d688e3SAlex Vesker int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher, 42026d688e3SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 42126d688e3SAlex Vesker struct mlx5dr_ste *ste, 42226d688e3SAlex Vesker u8 *cur_hw_ste, 42326d688e3SAlex Vesker enum mlx5dr_icm_chunk_size log_table_size) 42426d688e3SAlex Vesker { 42526d688e3SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn = nic_matcher->nic_tbl->nic_dmn; 42626d688e3SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 4276b93b400SYevgeny Kliteynik struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; 42826d688e3SAlex Vesker struct mlx5dr_htbl_connect_info info; 42926d688e3SAlex Vesker struct mlx5dr_ste_htbl *next_htbl; 43026d688e3SAlex Vesker 43126d688e3SAlex Vesker if (!mlx5dr_ste_is_last_in_rule(nic_matcher, ste->ste_chain_location)) { 432dd2d3c8dSYevgeny Kliteynik u16 next_lu_type; 43326d688e3SAlex Vesker u16 byte_mask; 43426d688e3SAlex Vesker 4356b93b400SYevgeny Kliteynik next_lu_type = ste_ctx->get_next_lu_type(cur_hw_ste); 4366b93b400SYevgeny Kliteynik byte_mask = ste_ctx->get_byte_mask(cur_hw_ste); 43726d688e3SAlex Vesker 43826d688e3SAlex Vesker next_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool, 43926d688e3SAlex Vesker log_table_size, 44026d688e3SAlex Vesker next_lu_type, 44126d688e3SAlex Vesker byte_mask); 44226d688e3SAlex Vesker if (!next_htbl) { 44326d688e3SAlex Vesker mlx5dr_dbg(dmn, "Failed allocating table\n"); 44426d688e3SAlex Vesker return -ENOMEM; 44526d688e3SAlex Vesker } 44626d688e3SAlex Vesker 44726d688e3SAlex Vesker /* Write new table to HW */ 44826d688e3SAlex Vesker info.type = CONNECT_MISS; 44926d688e3SAlex Vesker info.miss_icm_addr = nic_matcher->e_anchor->chunk->icm_addr; 45026d688e3SAlex Vesker if (mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, next_htbl, 45126d688e3SAlex Vesker &info, false)) { 45226d688e3SAlex Vesker mlx5dr_info(dmn, "Failed writing table to HW\n"); 45326d688e3SAlex Vesker goto free_table; 45426d688e3SAlex Vesker } 45526d688e3SAlex Vesker 4566b93b400SYevgeny Kliteynik mlx5dr_ste_set_hit_addr_by_next_htbl(ste_ctx, 4576b93b400SYevgeny Kliteynik cur_hw_ste, next_htbl); 45826d688e3SAlex Vesker ste->next_htbl = next_htbl; 45926d688e3SAlex Vesker next_htbl->pointing_ste = ste; 46026d688e3SAlex Vesker } 46126d688e3SAlex Vesker 46226d688e3SAlex Vesker return 0; 46326d688e3SAlex Vesker 46426d688e3SAlex Vesker free_table: 46526d688e3SAlex Vesker mlx5dr_ste_htbl_free(next_htbl); 46626d688e3SAlex Vesker return -ENOENT; 46726d688e3SAlex Vesker } 46826d688e3SAlex Vesker 46926d688e3SAlex Vesker static void dr_ste_set_ctrl(struct mlx5dr_ste_htbl *htbl) 47026d688e3SAlex Vesker { 47126d688e3SAlex Vesker struct mlx5dr_ste_htbl_ctrl *ctrl = &htbl->ctrl; 47226d688e3SAlex Vesker int num_of_entries; 47326d688e3SAlex Vesker 47426d688e3SAlex Vesker htbl->ctrl.may_grow = true; 47526d688e3SAlex Vesker 47683e79489SAlex Vesker if (htbl->chunk_size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask) 47726d688e3SAlex Vesker htbl->ctrl.may_grow = false; 47826d688e3SAlex Vesker 47926d688e3SAlex Vesker /* Threshold is 50%, one is added to table of size 1 */ 48026d688e3SAlex Vesker num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk_size); 48126d688e3SAlex Vesker ctrl->increase_threshold = (num_of_entries + 1) / 2; 48226d688e3SAlex Vesker } 48326d688e3SAlex Vesker 48426d688e3SAlex Vesker struct mlx5dr_ste_htbl *mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool, 48526d688e3SAlex Vesker enum mlx5dr_icm_chunk_size chunk_size, 486dd2d3c8dSYevgeny Kliteynik u16 lu_type, u16 byte_mask) 48726d688e3SAlex Vesker { 48826d688e3SAlex Vesker struct mlx5dr_icm_chunk *chunk; 48926d688e3SAlex Vesker struct mlx5dr_ste_htbl *htbl; 49026d688e3SAlex Vesker int i; 49126d688e3SAlex Vesker 49226d688e3SAlex Vesker htbl = kzalloc(sizeof(*htbl), GFP_KERNEL); 49326d688e3SAlex Vesker if (!htbl) 49426d688e3SAlex Vesker return NULL; 49526d688e3SAlex Vesker 49626d688e3SAlex Vesker chunk = mlx5dr_icm_alloc_chunk(pool, chunk_size); 49726d688e3SAlex Vesker if (!chunk) 49826d688e3SAlex Vesker goto out_free_htbl; 49926d688e3SAlex Vesker 50026d688e3SAlex Vesker htbl->chunk = chunk; 50126d688e3SAlex Vesker htbl->lu_type = lu_type; 50226d688e3SAlex Vesker htbl->byte_mask = byte_mask; 50326d688e3SAlex Vesker htbl->ste_arr = chunk->ste_arr; 50426d688e3SAlex Vesker htbl->hw_ste_arr = chunk->hw_ste_arr; 50526d688e3SAlex Vesker htbl->miss_list = chunk->miss_list; 5064ce380caSYevgeny Kliteynik htbl->refcount = 0; 50726d688e3SAlex Vesker 50826d688e3SAlex Vesker for (i = 0; i < chunk->num_of_entries; i++) { 50926d688e3SAlex Vesker struct mlx5dr_ste *ste = &htbl->ste_arr[i]; 51026d688e3SAlex Vesker 51126d688e3SAlex Vesker ste->hw_ste = htbl->hw_ste_arr + i * DR_STE_SIZE_REDUCED; 51226d688e3SAlex Vesker ste->htbl = htbl; 5134ce380caSYevgeny Kliteynik ste->refcount = 0; 51426d688e3SAlex Vesker INIT_LIST_HEAD(&ste->miss_list_node); 51526d688e3SAlex Vesker INIT_LIST_HEAD(&htbl->miss_list[i]); 51626d688e3SAlex Vesker INIT_LIST_HEAD(&ste->rule_list); 51726d688e3SAlex Vesker } 51826d688e3SAlex Vesker 51926d688e3SAlex Vesker htbl->chunk_size = chunk_size; 52026d688e3SAlex Vesker dr_ste_set_ctrl(htbl); 52126d688e3SAlex Vesker return htbl; 52226d688e3SAlex Vesker 52326d688e3SAlex Vesker out_free_htbl: 52426d688e3SAlex Vesker kfree(htbl); 52526d688e3SAlex Vesker return NULL; 52626d688e3SAlex Vesker } 52726d688e3SAlex Vesker 52826d688e3SAlex Vesker int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl) 52926d688e3SAlex Vesker { 5304ce380caSYevgeny Kliteynik if (htbl->refcount) 53126d688e3SAlex Vesker return -EBUSY; 53226d688e3SAlex Vesker 53326d688e3SAlex Vesker mlx5dr_icm_free_chunk(htbl->chunk); 53426d688e3SAlex Vesker kfree(htbl); 53526d688e3SAlex Vesker return 0; 53626d688e3SAlex Vesker } 53726d688e3SAlex Vesker 5386b93b400SYevgeny Kliteynik void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx, 5396b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn, 54064c78942SYevgeny Kliteynik u8 *action_type_set, 541ad17dc8cSYevgeny Kliteynik u8 *hw_ste_arr, 54264c78942SYevgeny Kliteynik struct mlx5dr_ste_actions_attr *attr, 54364c78942SYevgeny Kliteynik u32 *added_stes) 54464c78942SYevgeny Kliteynik { 545ad17dc8cSYevgeny Kliteynik ste_ctx->set_actions_tx(dmn, action_type_set, hw_ste_arr, 546ad17dc8cSYevgeny Kliteynik attr, added_stes); 54764c78942SYevgeny Kliteynik } 54864c78942SYevgeny Kliteynik 5496b93b400SYevgeny Kliteynik void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, 5506b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn, 55164c78942SYevgeny Kliteynik u8 *action_type_set, 552ad17dc8cSYevgeny Kliteynik u8 *hw_ste_arr, 55364c78942SYevgeny Kliteynik struct mlx5dr_ste_actions_attr *attr, 55464c78942SYevgeny Kliteynik u32 *added_stes) 55564c78942SYevgeny Kliteynik { 556ad17dc8cSYevgeny Kliteynik ste_ctx->set_actions_rx(dmn, action_type_set, hw_ste_arr, 557ad17dc8cSYevgeny Kliteynik attr, added_stes); 55864c78942SYevgeny Kliteynik } 55964c78942SYevgeny Kliteynik 5604781df92SYevgeny Kliteynik const struct mlx5dr_ste_action_modify_field * 5614781df92SYevgeny Kliteynik mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field) 5624781df92SYevgeny Kliteynik { 5634781df92SYevgeny Kliteynik const struct mlx5dr_ste_action_modify_field *hw_field; 5644781df92SYevgeny Kliteynik 5654781df92SYevgeny Kliteynik if (sw_field >= ste_ctx->modify_field_arr_sz) 5664781df92SYevgeny Kliteynik return NULL; 5674781df92SYevgeny Kliteynik 5684781df92SYevgeny Kliteynik hw_field = &ste_ctx->modify_field_arr[sw_field]; 5694781df92SYevgeny Kliteynik if (!hw_field->end && !hw_field->start) 5704781df92SYevgeny Kliteynik return NULL; 5714781df92SYevgeny Kliteynik 5724781df92SYevgeny Kliteynik return hw_field; 5734781df92SYevgeny Kliteynik } 5744781df92SYevgeny Kliteynik 5754781df92SYevgeny Kliteynik void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx, 5764781df92SYevgeny Kliteynik __be64 *hw_action, 5774781df92SYevgeny Kliteynik u8 hw_field, 5784781df92SYevgeny Kliteynik u8 shifter, 5794781df92SYevgeny Kliteynik u8 length, 5804781df92SYevgeny Kliteynik u32 data) 5814781df92SYevgeny Kliteynik { 5824781df92SYevgeny Kliteynik ste_ctx->set_action_set((u8 *)hw_action, 5834781df92SYevgeny Kliteynik hw_field, shifter, length, data); 5844781df92SYevgeny Kliteynik } 5854781df92SYevgeny Kliteynik 5864781df92SYevgeny Kliteynik void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx, 5874781df92SYevgeny Kliteynik __be64 *hw_action, 5884781df92SYevgeny Kliteynik u8 hw_field, 5894781df92SYevgeny Kliteynik u8 shifter, 5904781df92SYevgeny Kliteynik u8 length, 5914781df92SYevgeny Kliteynik u32 data) 5924781df92SYevgeny Kliteynik { 5934781df92SYevgeny Kliteynik ste_ctx->set_action_add((u8 *)hw_action, 5944781df92SYevgeny Kliteynik hw_field, shifter, length, data); 5954781df92SYevgeny Kliteynik } 5964781df92SYevgeny Kliteynik 5974781df92SYevgeny Kliteynik void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx, 5984781df92SYevgeny Kliteynik __be64 *hw_action, 5994781df92SYevgeny Kliteynik u8 dst_hw_field, 6004781df92SYevgeny Kliteynik u8 dst_shifter, 6014781df92SYevgeny Kliteynik u8 dst_len, 6024781df92SYevgeny Kliteynik u8 src_hw_field, 6034781df92SYevgeny Kliteynik u8 src_shifter) 6044781df92SYevgeny Kliteynik { 6054781df92SYevgeny Kliteynik ste_ctx->set_action_copy((u8 *)hw_action, 6064781df92SYevgeny Kliteynik dst_hw_field, dst_shifter, dst_len, 6074781df92SYevgeny Kliteynik src_hw_field, src_shifter); 6084781df92SYevgeny Kliteynik } 6094781df92SYevgeny Kliteynik 6104781df92SYevgeny Kliteynik int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx, 6114781df92SYevgeny Kliteynik void *data, u32 data_sz, 6124781df92SYevgeny Kliteynik u8 *hw_action, u32 hw_action_sz, 6134781df92SYevgeny Kliteynik u16 *used_hw_action_num) 6144781df92SYevgeny Kliteynik { 6154781df92SYevgeny Kliteynik /* Only Ethernet frame is supported, with VLAN (18) or without (14) */ 6164781df92SYevgeny Kliteynik if (data_sz != HDR_LEN_L2 && data_sz != HDR_LEN_L2_W_VLAN) 6174781df92SYevgeny Kliteynik return -EINVAL; 6184781df92SYevgeny Kliteynik 6194781df92SYevgeny Kliteynik return ste_ctx->set_action_decap_l3_list(data, data_sz, 6204781df92SYevgeny Kliteynik hw_action, hw_action_sz, 6214781df92SYevgeny Kliteynik used_hw_action_num); 6224781df92SYevgeny Kliteynik } 6234781df92SYevgeny Kliteynik 62426d688e3SAlex Vesker int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn, 62526d688e3SAlex Vesker u8 match_criteria, 62626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 62726d688e3SAlex Vesker struct mlx5dr_match_param *value) 62826d688e3SAlex Vesker { 62926d688e3SAlex Vesker if (!value && (match_criteria & DR_MATCHER_CRITERIA_MISC)) { 63026d688e3SAlex Vesker if (mask->misc.source_port && mask->misc.source_port != 0xffff) { 63138a5c59dSYevgeny Kliteynik mlx5dr_err(dmn, 63238a5c59dSYevgeny Kliteynik "Partial mask source_port is not supported\n"); 63338a5c59dSYevgeny Kliteynik return -EINVAL; 63438a5c59dSYevgeny Kliteynik } 63538a5c59dSYevgeny Kliteynik if (mask->misc.source_eswitch_owner_vhca_id && 63638a5c59dSYevgeny Kliteynik mask->misc.source_eswitch_owner_vhca_id != 0xffff) { 63738a5c59dSYevgeny Kliteynik mlx5dr_err(dmn, 63838a5c59dSYevgeny Kliteynik "Partial mask source_eswitch_owner_vhca_id is not supported\n"); 63926d688e3SAlex Vesker return -EINVAL; 64026d688e3SAlex Vesker } 64126d688e3SAlex Vesker } 64226d688e3SAlex Vesker 64326d688e3SAlex Vesker return 0; 64426d688e3SAlex Vesker } 64526d688e3SAlex Vesker 64626d688e3SAlex Vesker int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, 64726d688e3SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 64826d688e3SAlex Vesker struct mlx5dr_match_param *value, 64926d688e3SAlex Vesker u8 *ste_arr) 65026d688e3SAlex Vesker { 65126d688e3SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn = nic_matcher->nic_tbl->nic_dmn; 65226d688e3SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 6536b93b400SYevgeny Kliteynik struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; 65426d688e3SAlex Vesker struct mlx5dr_ste_build *sb; 65526d688e3SAlex Vesker int ret, i; 65626d688e3SAlex Vesker 65726d688e3SAlex Vesker ret = mlx5dr_ste_build_pre_check(dmn, matcher->match_criteria, 65826d688e3SAlex Vesker &matcher->mask, value); 65926d688e3SAlex Vesker if (ret) 66026d688e3SAlex Vesker return ret; 66126d688e3SAlex Vesker 66226d688e3SAlex Vesker sb = nic_matcher->ste_builder; 66326d688e3SAlex Vesker for (i = 0; i < nic_matcher->num_of_builders; i++) { 6646b93b400SYevgeny Kliteynik ste_ctx->ste_init(ste_arr, 66526d688e3SAlex Vesker sb->lu_type, 66626d688e3SAlex Vesker nic_dmn->ste_type, 66726d688e3SAlex Vesker dmn->info.caps.gvmi); 66826d688e3SAlex Vesker 66926d688e3SAlex Vesker mlx5dr_ste_set_bit_mask(ste_arr, sb->bit_mask); 67026d688e3SAlex Vesker 67164c78942SYevgeny Kliteynik ret = sb->ste_build_tag_func(value, sb, dr_ste_get_tag(ste_arr)); 67226d688e3SAlex Vesker if (ret) 67326d688e3SAlex Vesker return ret; 67426d688e3SAlex Vesker 67526d688e3SAlex Vesker /* Connect the STEs */ 67626d688e3SAlex Vesker if (i < (nic_matcher->num_of_builders - 1)) { 67726d688e3SAlex Vesker /* Need the next builder for these fields, 67826d688e3SAlex Vesker * not relevant for the last ste in the chain. 67926d688e3SAlex Vesker */ 68026d688e3SAlex Vesker sb++; 6816b93b400SYevgeny Kliteynik ste_ctx->set_next_lu_type(ste_arr, sb->lu_type); 6826b93b400SYevgeny Kliteynik ste_ctx->set_byte_mask(ste_arr, sb->byte_mask); 68326d688e3SAlex Vesker } 68426d688e3SAlex Vesker ste_arr += DR_STE_SIZE; 68526d688e3SAlex Vesker } 68626d688e3SAlex Vesker return 0; 68726d688e3SAlex Vesker } 68826d688e3SAlex Vesker 68926d688e3SAlex Vesker static void dr_ste_copy_mask_misc(char *mask, struct mlx5dr_match_misc *spec) 69026d688e3SAlex Vesker { 69126d688e3SAlex Vesker spec->gre_c_present = MLX5_GET(fte_match_set_misc, mask, gre_c_present); 69226d688e3SAlex Vesker spec->gre_k_present = MLX5_GET(fte_match_set_misc, mask, gre_k_present); 69326d688e3SAlex Vesker spec->gre_s_present = MLX5_GET(fte_match_set_misc, mask, gre_s_present); 69426d688e3SAlex Vesker spec->source_vhca_port = MLX5_GET(fte_match_set_misc, mask, source_vhca_port); 69526d688e3SAlex Vesker spec->source_sqn = MLX5_GET(fte_match_set_misc, mask, source_sqn); 69626d688e3SAlex Vesker 69726d688e3SAlex Vesker spec->source_port = MLX5_GET(fte_match_set_misc, mask, source_port); 698640bdb1fSAlaa Hleihel spec->source_eswitch_owner_vhca_id = MLX5_GET(fte_match_set_misc, mask, 699640bdb1fSAlaa Hleihel source_eswitch_owner_vhca_id); 70026d688e3SAlex Vesker 70126d688e3SAlex Vesker spec->outer_second_prio = MLX5_GET(fte_match_set_misc, mask, outer_second_prio); 70226d688e3SAlex Vesker spec->outer_second_cfi = MLX5_GET(fte_match_set_misc, mask, outer_second_cfi); 70326d688e3SAlex Vesker spec->outer_second_vid = MLX5_GET(fte_match_set_misc, mask, outer_second_vid); 70426d688e3SAlex Vesker spec->inner_second_prio = MLX5_GET(fte_match_set_misc, mask, inner_second_prio); 70526d688e3SAlex Vesker spec->inner_second_cfi = MLX5_GET(fte_match_set_misc, mask, inner_second_cfi); 70626d688e3SAlex Vesker spec->inner_second_vid = MLX5_GET(fte_match_set_misc, mask, inner_second_vid); 70726d688e3SAlex Vesker 70826d688e3SAlex Vesker spec->outer_second_cvlan_tag = 70926d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, outer_second_cvlan_tag); 71026d688e3SAlex Vesker spec->inner_second_cvlan_tag = 71126d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, inner_second_cvlan_tag); 71226d688e3SAlex Vesker spec->outer_second_svlan_tag = 71326d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, outer_second_svlan_tag); 71426d688e3SAlex Vesker spec->inner_second_svlan_tag = 71526d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, inner_second_svlan_tag); 71626d688e3SAlex Vesker 71726d688e3SAlex Vesker spec->gre_protocol = MLX5_GET(fte_match_set_misc, mask, gre_protocol); 71826d688e3SAlex Vesker 71926d688e3SAlex Vesker spec->gre_key_h = MLX5_GET(fte_match_set_misc, mask, gre_key.nvgre.hi); 72026d688e3SAlex Vesker spec->gre_key_l = MLX5_GET(fte_match_set_misc, mask, gre_key.nvgre.lo); 72126d688e3SAlex Vesker 72226d688e3SAlex Vesker spec->vxlan_vni = MLX5_GET(fte_match_set_misc, mask, vxlan_vni); 72326d688e3SAlex Vesker 72426d688e3SAlex Vesker spec->geneve_vni = MLX5_GET(fte_match_set_misc, mask, geneve_vni); 72526d688e3SAlex Vesker spec->geneve_oam = MLX5_GET(fte_match_set_misc, mask, geneve_oam); 72626d688e3SAlex Vesker 72726d688e3SAlex Vesker spec->outer_ipv6_flow_label = 72826d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, outer_ipv6_flow_label); 72926d688e3SAlex Vesker 73026d688e3SAlex Vesker spec->inner_ipv6_flow_label = 73126d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, inner_ipv6_flow_label); 73226d688e3SAlex Vesker 73326d688e3SAlex Vesker spec->geneve_opt_len = MLX5_GET(fte_match_set_misc, mask, geneve_opt_len); 73426d688e3SAlex Vesker spec->geneve_protocol_type = 73526d688e3SAlex Vesker MLX5_GET(fte_match_set_misc, mask, geneve_protocol_type); 73626d688e3SAlex Vesker 73726d688e3SAlex Vesker spec->bth_dst_qp = MLX5_GET(fte_match_set_misc, mask, bth_dst_qp); 73826d688e3SAlex Vesker } 73926d688e3SAlex Vesker 74026d688e3SAlex Vesker static void dr_ste_copy_mask_spec(char *mask, struct mlx5dr_match_spec *spec) 74126d688e3SAlex Vesker { 742c2ba2c22SSaeed Mahameed __be32 raw_ip[4]; 74326d688e3SAlex Vesker 74426d688e3SAlex Vesker spec->smac_47_16 = MLX5_GET(fte_match_set_lyr_2_4, mask, smac_47_16); 74526d688e3SAlex Vesker 74626d688e3SAlex Vesker spec->smac_15_0 = MLX5_GET(fte_match_set_lyr_2_4, mask, smac_15_0); 74726d688e3SAlex Vesker spec->ethertype = MLX5_GET(fte_match_set_lyr_2_4, mask, ethertype); 74826d688e3SAlex Vesker 74926d688e3SAlex Vesker spec->dmac_47_16 = MLX5_GET(fte_match_set_lyr_2_4, mask, dmac_47_16); 75026d688e3SAlex Vesker 75126d688e3SAlex Vesker spec->dmac_15_0 = MLX5_GET(fte_match_set_lyr_2_4, mask, dmac_15_0); 75226d688e3SAlex Vesker spec->first_prio = MLX5_GET(fte_match_set_lyr_2_4, mask, first_prio); 75326d688e3SAlex Vesker spec->first_cfi = MLX5_GET(fte_match_set_lyr_2_4, mask, first_cfi); 75426d688e3SAlex Vesker spec->first_vid = MLX5_GET(fte_match_set_lyr_2_4, mask, first_vid); 75526d688e3SAlex Vesker 75626d688e3SAlex Vesker spec->ip_protocol = MLX5_GET(fte_match_set_lyr_2_4, mask, ip_protocol); 75726d688e3SAlex Vesker spec->ip_dscp = MLX5_GET(fte_match_set_lyr_2_4, mask, ip_dscp); 75826d688e3SAlex Vesker spec->ip_ecn = MLX5_GET(fte_match_set_lyr_2_4, mask, ip_ecn); 75926d688e3SAlex Vesker spec->cvlan_tag = MLX5_GET(fte_match_set_lyr_2_4, mask, cvlan_tag); 76026d688e3SAlex Vesker spec->svlan_tag = MLX5_GET(fte_match_set_lyr_2_4, mask, svlan_tag); 76126d688e3SAlex Vesker spec->frag = MLX5_GET(fte_match_set_lyr_2_4, mask, frag); 76226d688e3SAlex Vesker spec->ip_version = MLX5_GET(fte_match_set_lyr_2_4, mask, ip_version); 76326d688e3SAlex Vesker spec->tcp_flags = MLX5_GET(fte_match_set_lyr_2_4, mask, tcp_flags); 76426d688e3SAlex Vesker spec->tcp_sport = MLX5_GET(fte_match_set_lyr_2_4, mask, tcp_sport); 76526d688e3SAlex Vesker spec->tcp_dport = MLX5_GET(fte_match_set_lyr_2_4, mask, tcp_dport); 76626d688e3SAlex Vesker 76726d688e3SAlex Vesker spec->ttl_hoplimit = MLX5_GET(fte_match_set_lyr_2_4, mask, ttl_hoplimit); 76826d688e3SAlex Vesker 76926d688e3SAlex Vesker spec->udp_sport = MLX5_GET(fte_match_set_lyr_2_4, mask, udp_sport); 77026d688e3SAlex Vesker spec->udp_dport = MLX5_GET(fte_match_set_lyr_2_4, mask, udp_dport); 77126d688e3SAlex Vesker 77226d688e3SAlex Vesker memcpy(raw_ip, MLX5_ADDR_OF(fte_match_set_lyr_2_4, mask, 77326d688e3SAlex Vesker src_ipv4_src_ipv6.ipv6_layout.ipv6), 77426d688e3SAlex Vesker sizeof(raw_ip)); 77526d688e3SAlex Vesker 77626d688e3SAlex Vesker spec->src_ip_127_96 = be32_to_cpu(raw_ip[0]); 77726d688e3SAlex Vesker spec->src_ip_95_64 = be32_to_cpu(raw_ip[1]); 77826d688e3SAlex Vesker spec->src_ip_63_32 = be32_to_cpu(raw_ip[2]); 77926d688e3SAlex Vesker spec->src_ip_31_0 = be32_to_cpu(raw_ip[3]); 78026d688e3SAlex Vesker 78126d688e3SAlex Vesker memcpy(raw_ip, MLX5_ADDR_OF(fte_match_set_lyr_2_4, mask, 78226d688e3SAlex Vesker dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 78326d688e3SAlex Vesker sizeof(raw_ip)); 78426d688e3SAlex Vesker 78526d688e3SAlex Vesker spec->dst_ip_127_96 = be32_to_cpu(raw_ip[0]); 78626d688e3SAlex Vesker spec->dst_ip_95_64 = be32_to_cpu(raw_ip[1]); 78726d688e3SAlex Vesker spec->dst_ip_63_32 = be32_to_cpu(raw_ip[2]); 78826d688e3SAlex Vesker spec->dst_ip_31_0 = be32_to_cpu(raw_ip[3]); 78926d688e3SAlex Vesker } 79026d688e3SAlex Vesker 79126d688e3SAlex Vesker static void dr_ste_copy_mask_misc2(char *mask, struct mlx5dr_match_misc2 *spec) 79226d688e3SAlex Vesker { 79326d688e3SAlex Vesker spec->outer_first_mpls_label = 79426d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls.mpls_label); 79526d688e3SAlex Vesker spec->outer_first_mpls_exp = 79626d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls.mpls_exp); 79726d688e3SAlex Vesker spec->outer_first_mpls_s_bos = 79826d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls.mpls_s_bos); 79926d688e3SAlex Vesker spec->outer_first_mpls_ttl = 80026d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls.mpls_ttl); 80126d688e3SAlex Vesker spec->inner_first_mpls_label = 80226d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, inner_first_mpls.mpls_label); 80326d688e3SAlex Vesker spec->inner_first_mpls_exp = 80426d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, inner_first_mpls.mpls_exp); 80526d688e3SAlex Vesker spec->inner_first_mpls_s_bos = 80626d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, inner_first_mpls.mpls_s_bos); 80726d688e3SAlex Vesker spec->inner_first_mpls_ttl = 80826d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, inner_first_mpls.mpls_ttl); 80926d688e3SAlex Vesker spec->outer_first_mpls_over_gre_label = 81026d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_gre.mpls_label); 81126d688e3SAlex Vesker spec->outer_first_mpls_over_gre_exp = 81226d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_gre.mpls_exp); 81326d688e3SAlex Vesker spec->outer_first_mpls_over_gre_s_bos = 81426d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_gre.mpls_s_bos); 81526d688e3SAlex Vesker spec->outer_first_mpls_over_gre_ttl = 81626d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_gre.mpls_ttl); 81726d688e3SAlex Vesker spec->outer_first_mpls_over_udp_label = 81826d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_udp.mpls_label); 81926d688e3SAlex Vesker spec->outer_first_mpls_over_udp_exp = 82026d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_udp.mpls_exp); 82126d688e3SAlex Vesker spec->outer_first_mpls_over_udp_s_bos = 82226d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_udp.mpls_s_bos); 82326d688e3SAlex Vesker spec->outer_first_mpls_over_udp_ttl = 82426d688e3SAlex Vesker MLX5_GET(fte_match_set_misc2, mask, outer_first_mpls_over_udp.mpls_ttl); 82526d688e3SAlex Vesker spec->metadata_reg_c_7 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_7); 82626d688e3SAlex Vesker spec->metadata_reg_c_6 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_6); 82726d688e3SAlex Vesker spec->metadata_reg_c_5 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_5); 82826d688e3SAlex Vesker spec->metadata_reg_c_4 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_4); 82926d688e3SAlex Vesker spec->metadata_reg_c_3 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_3); 83026d688e3SAlex Vesker spec->metadata_reg_c_2 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_2); 83126d688e3SAlex Vesker spec->metadata_reg_c_1 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_1); 83226d688e3SAlex Vesker spec->metadata_reg_c_0 = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_c_0); 83326d688e3SAlex Vesker spec->metadata_reg_a = MLX5_GET(fte_match_set_misc2, mask, metadata_reg_a); 83426d688e3SAlex Vesker } 83526d688e3SAlex Vesker 83626d688e3SAlex Vesker static void dr_ste_copy_mask_misc3(char *mask, struct mlx5dr_match_misc3 *spec) 83726d688e3SAlex Vesker { 83826d688e3SAlex Vesker spec->inner_tcp_seq_num = MLX5_GET(fte_match_set_misc3, mask, inner_tcp_seq_num); 83926d688e3SAlex Vesker spec->outer_tcp_seq_num = MLX5_GET(fte_match_set_misc3, mask, outer_tcp_seq_num); 84026d688e3SAlex Vesker spec->inner_tcp_ack_num = MLX5_GET(fte_match_set_misc3, mask, inner_tcp_ack_num); 84126d688e3SAlex Vesker spec->outer_tcp_ack_num = MLX5_GET(fte_match_set_misc3, mask, outer_tcp_ack_num); 84226d688e3SAlex Vesker spec->outer_vxlan_gpe_vni = 84326d688e3SAlex Vesker MLX5_GET(fte_match_set_misc3, mask, outer_vxlan_gpe_vni); 84426d688e3SAlex Vesker spec->outer_vxlan_gpe_next_protocol = 84526d688e3SAlex Vesker MLX5_GET(fte_match_set_misc3, mask, outer_vxlan_gpe_next_protocol); 84626d688e3SAlex Vesker spec->outer_vxlan_gpe_flags = 84726d688e3SAlex Vesker MLX5_GET(fte_match_set_misc3, mask, outer_vxlan_gpe_flags); 84826d688e3SAlex Vesker spec->icmpv4_header_data = MLX5_GET(fte_match_set_misc3, mask, icmp_header_data); 84926d688e3SAlex Vesker spec->icmpv6_header_data = 85026d688e3SAlex Vesker MLX5_GET(fte_match_set_misc3, mask, icmpv6_header_data); 85126d688e3SAlex Vesker spec->icmpv4_type = MLX5_GET(fte_match_set_misc3, mask, icmp_type); 85226d688e3SAlex Vesker spec->icmpv4_code = MLX5_GET(fte_match_set_misc3, mask, icmp_code); 85326d688e3SAlex Vesker spec->icmpv6_type = MLX5_GET(fte_match_set_misc3, mask, icmpv6_type); 85426d688e3SAlex Vesker spec->icmpv6_code = MLX5_GET(fte_match_set_misc3, mask, icmpv6_code); 85526d688e3SAlex Vesker } 85626d688e3SAlex Vesker 857*160e9cb3SYevgeny Kliteynik static void dr_ste_copy_mask_misc4(char *mask, struct mlx5dr_match_misc4 *spec) 858*160e9cb3SYevgeny Kliteynik { 859*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_id_0 = 860*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_id_0); 861*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_value_0 = 862*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_value_0); 863*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_id_1 = 864*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_id_1); 865*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_value_1 = 866*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_value_1); 867*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_id_2 = 868*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_id_2); 869*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_value_2 = 870*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_value_2); 871*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_id_3 = 872*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_id_3); 873*160e9cb3SYevgeny Kliteynik spec->prog_sample_field_value_3 = 874*160e9cb3SYevgeny Kliteynik MLX5_GET(fte_match_set_misc4, mask, prog_sample_field_value_3); 875*160e9cb3SYevgeny Kliteynik } 876*160e9cb3SYevgeny Kliteynik 87726d688e3SAlex Vesker void mlx5dr_ste_copy_param(u8 match_criteria, 87826d688e3SAlex Vesker struct mlx5dr_match_param *set_param, 87926d688e3SAlex Vesker struct mlx5dr_match_parameters *mask) 88026d688e3SAlex Vesker { 88126d688e3SAlex Vesker u8 tail_param[MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)] = {}; 88226d688e3SAlex Vesker u8 *data = (u8 *)mask->match_buf; 88326d688e3SAlex Vesker size_t param_location; 88426d688e3SAlex Vesker void *buff; 88526d688e3SAlex Vesker 88626d688e3SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_OUTER) { 88726d688e3SAlex Vesker if (mask->match_sz < sizeof(struct mlx5dr_match_spec)) { 88826d688e3SAlex Vesker memcpy(tail_param, data, mask->match_sz); 88926d688e3SAlex Vesker buff = tail_param; 89026d688e3SAlex Vesker } else { 89126d688e3SAlex Vesker buff = mask->match_buf; 89226d688e3SAlex Vesker } 89326d688e3SAlex Vesker dr_ste_copy_mask_spec(buff, &set_param->outer); 89426d688e3SAlex Vesker } 89526d688e3SAlex Vesker param_location = sizeof(struct mlx5dr_match_spec); 89626d688e3SAlex Vesker 89726d688e3SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_MISC) { 89826d688e3SAlex Vesker if (mask->match_sz < param_location + 89926d688e3SAlex Vesker sizeof(struct mlx5dr_match_misc)) { 90026d688e3SAlex Vesker memcpy(tail_param, data + param_location, 90126d688e3SAlex Vesker mask->match_sz - param_location); 90226d688e3SAlex Vesker buff = tail_param; 90326d688e3SAlex Vesker } else { 90426d688e3SAlex Vesker buff = data + param_location; 90526d688e3SAlex Vesker } 90626d688e3SAlex Vesker dr_ste_copy_mask_misc(buff, &set_param->misc); 90726d688e3SAlex Vesker } 90826d688e3SAlex Vesker param_location += sizeof(struct mlx5dr_match_misc); 90926d688e3SAlex Vesker 91026d688e3SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_INNER) { 91126d688e3SAlex Vesker if (mask->match_sz < param_location + 91226d688e3SAlex Vesker sizeof(struct mlx5dr_match_spec)) { 91326d688e3SAlex Vesker memcpy(tail_param, data + param_location, 91426d688e3SAlex Vesker mask->match_sz - param_location); 91526d688e3SAlex Vesker buff = tail_param; 91626d688e3SAlex Vesker } else { 91726d688e3SAlex Vesker buff = data + param_location; 91826d688e3SAlex Vesker } 91926d688e3SAlex Vesker dr_ste_copy_mask_spec(buff, &set_param->inner); 92026d688e3SAlex Vesker } 92126d688e3SAlex Vesker param_location += sizeof(struct mlx5dr_match_spec); 92226d688e3SAlex Vesker 92326d688e3SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_MISC2) { 92426d688e3SAlex Vesker if (mask->match_sz < param_location + 92526d688e3SAlex Vesker sizeof(struct mlx5dr_match_misc2)) { 92626d688e3SAlex Vesker memcpy(tail_param, data + param_location, 92726d688e3SAlex Vesker mask->match_sz - param_location); 92826d688e3SAlex Vesker buff = tail_param; 92926d688e3SAlex Vesker } else { 93026d688e3SAlex Vesker buff = data + param_location; 93126d688e3SAlex Vesker } 93226d688e3SAlex Vesker dr_ste_copy_mask_misc2(buff, &set_param->misc2); 93326d688e3SAlex Vesker } 93426d688e3SAlex Vesker 93526d688e3SAlex Vesker param_location += sizeof(struct mlx5dr_match_misc2); 93626d688e3SAlex Vesker 93726d688e3SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_MISC3) { 93826d688e3SAlex Vesker if (mask->match_sz < param_location + 93926d688e3SAlex Vesker sizeof(struct mlx5dr_match_misc3)) { 94026d688e3SAlex Vesker memcpy(tail_param, data + param_location, 94126d688e3SAlex Vesker mask->match_sz - param_location); 94226d688e3SAlex Vesker buff = tail_param; 94326d688e3SAlex Vesker } else { 94426d688e3SAlex Vesker buff = data + param_location; 94526d688e3SAlex Vesker } 94626d688e3SAlex Vesker dr_ste_copy_mask_misc3(buff, &set_param->misc3); 94726d688e3SAlex Vesker } 948*160e9cb3SYevgeny Kliteynik 949*160e9cb3SYevgeny Kliteynik param_location += sizeof(struct mlx5dr_match_misc3); 950*160e9cb3SYevgeny Kliteynik 951*160e9cb3SYevgeny Kliteynik if (match_criteria & DR_MATCHER_CRITERIA_MISC4) { 952*160e9cb3SYevgeny Kliteynik if (mask->match_sz < param_location + 953*160e9cb3SYevgeny Kliteynik sizeof(struct mlx5dr_match_misc4)) { 954*160e9cb3SYevgeny Kliteynik memcpy(tail_param, data + param_location, 955*160e9cb3SYevgeny Kliteynik mask->match_sz - param_location); 956*160e9cb3SYevgeny Kliteynik buff = tail_param; 957*160e9cb3SYevgeny Kliteynik } else { 958*160e9cb3SYevgeny Kliteynik buff = data + param_location; 959*160e9cb3SYevgeny Kliteynik } 960*160e9cb3SYevgeny Kliteynik dr_ste_copy_mask_misc4(buff, &set_param->misc4); 961*160e9cb3SYevgeny Kliteynik } 96226d688e3SAlex Vesker } 96326d688e3SAlex Vesker 9645212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx, 9655212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 96626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 96726d688e3SAlex Vesker bool inner, bool rx) 96826d688e3SAlex Vesker { 96926d688e3SAlex Vesker sb->rx = rx; 97026d688e3SAlex Vesker sb->inner = inner; 9715212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l2_src_dst_init(sb, mask); 97226d688e3SAlex Vesker } 97326d688e3SAlex Vesker 9745212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx, 9755212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 97626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 97726d688e3SAlex Vesker bool inner, bool rx) 97826d688e3SAlex Vesker { 97926d688e3SAlex Vesker sb->rx = rx; 98026d688e3SAlex Vesker sb->inner = inner; 9815212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l3_ipv6_dst_init(sb, mask); 98226d688e3SAlex Vesker } 98326d688e3SAlex Vesker 9845212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx, 9855212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 98626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 98726d688e3SAlex Vesker bool inner, bool rx) 98826d688e3SAlex Vesker { 98926d688e3SAlex Vesker sb->rx = rx; 99026d688e3SAlex Vesker sb->inner = inner; 9915212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l3_ipv6_src_init(sb, mask); 99226d688e3SAlex Vesker } 99326d688e3SAlex Vesker 9945212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx, 9955212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 99626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 99726d688e3SAlex Vesker bool inner, bool rx) 99826d688e3SAlex Vesker { 99926d688e3SAlex Vesker sb->rx = rx; 100026d688e3SAlex Vesker sb->inner = inner; 10015212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l3_ipv4_5_tuple_init(sb, mask); 100226d688e3SAlex Vesker } 100326d688e3SAlex Vesker 10045212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx, 10055212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 100626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 100726d688e3SAlex Vesker bool inner, bool rx) 100826d688e3SAlex Vesker { 100926d688e3SAlex Vesker sb->rx = rx; 101026d688e3SAlex Vesker sb->inner = inner; 10115212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l2_src_init(sb, mask); 101226d688e3SAlex Vesker } 101326d688e3SAlex Vesker 10145212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx, 10155212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 101626d688e3SAlex Vesker struct mlx5dr_match_param *mask, 101726d688e3SAlex Vesker bool inner, bool rx) 101826d688e3SAlex Vesker { 101926d688e3SAlex Vesker sb->rx = rx; 102026d688e3SAlex Vesker sb->inner = inner; 10215212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l2_dst_init(sb, mask); 102226d688e3SAlex Vesker } 102326d688e3SAlex Vesker 10245212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx, 10255212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 10265212f9c6SYevgeny Kliteynik struct mlx5dr_match_param *mask, bool inner, bool rx) 10275212f9c6SYevgeny Kliteynik { 10285212f9c6SYevgeny Kliteynik sb->rx = rx; 10295212f9c6SYevgeny Kliteynik sb->inner = inner; 10305212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l2_tnl_init(sb, mask); 10315212f9c6SYevgeny Kliteynik } 10325212f9c6SYevgeny Kliteynik 10335212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx, 10345212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 103526d688e3SAlex Vesker struct mlx5dr_match_param *mask, 103626d688e3SAlex Vesker bool inner, bool rx) 103726d688e3SAlex Vesker { 103826d688e3SAlex Vesker sb->rx = rx; 103926d688e3SAlex Vesker sb->inner = inner; 10405212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l3_ipv4_misc_init(sb, mask); 104126d688e3SAlex Vesker } 104226d688e3SAlex Vesker 10435212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx, 10445212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 104526d688e3SAlex Vesker struct mlx5dr_match_param *mask, 104626d688e3SAlex Vesker bool inner, bool rx) 104726d688e3SAlex Vesker { 104826d688e3SAlex Vesker sb->rx = rx; 104926d688e3SAlex Vesker sb->inner = inner; 10505212f9c6SYevgeny Kliteynik ste_ctx->build_eth_ipv6_l3_l4_init(sb, mask); 105126d688e3SAlex Vesker } 105226d688e3SAlex Vesker 105326d688e3SAlex Vesker static int dr_ste_build_empty_always_hit_tag(struct mlx5dr_match_param *value, 105426d688e3SAlex Vesker struct mlx5dr_ste_build *sb, 1055e6b69bf3SYevgeny Kliteynik u8 *tag) 105626d688e3SAlex Vesker { 105726d688e3SAlex Vesker return 0; 105826d688e3SAlex Vesker } 105926d688e3SAlex Vesker 106026d688e3SAlex Vesker void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx) 106126d688e3SAlex Vesker { 106226d688e3SAlex Vesker sb->rx = rx; 106326d688e3SAlex Vesker sb->lu_type = MLX5DR_STE_LU_TYPE_DONT_CARE; 106426d688e3SAlex Vesker sb->byte_mask = 0; 106526d688e3SAlex Vesker sb->ste_build_tag_func = &dr_ste_build_empty_always_hit_tag; 106626d688e3SAlex Vesker } 106726d688e3SAlex Vesker 10685212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx, 10695212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 107026d688e3SAlex Vesker struct mlx5dr_match_param *mask, 107126d688e3SAlex Vesker bool inner, bool rx) 107226d688e3SAlex Vesker { 107326d688e3SAlex Vesker sb->rx = rx; 107426d688e3SAlex Vesker sb->inner = inner; 10755212f9c6SYevgeny Kliteynik ste_ctx->build_mpls_init(sb, mask); 107626d688e3SAlex Vesker } 107726d688e3SAlex Vesker 10785212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx, 10795212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 10805212f9c6SYevgeny Kliteynik struct mlx5dr_match_param *mask, 10815212f9c6SYevgeny Kliteynik bool inner, bool rx) 10825212f9c6SYevgeny Kliteynik { 10835212f9c6SYevgeny Kliteynik sb->rx = rx; 10845212f9c6SYevgeny Kliteynik sb->inner = inner; 10855212f9c6SYevgeny Kliteynik ste_ctx->build_tnl_gre_init(sb, mask); 10865212f9c6SYevgeny Kliteynik } 10875212f9c6SYevgeny Kliteynik 10885212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx, 10895212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 109026d688e3SAlex Vesker struct mlx5dr_match_param *mask, 109126d688e3SAlex Vesker bool inner, bool rx) 109226d688e3SAlex Vesker { 109326d688e3SAlex Vesker sb->rx = rx; 109426d688e3SAlex Vesker sb->inner = inner; 10955212f9c6SYevgeny Kliteynik ste_ctx->build_tnl_mpls_init(sb, mask); 109626d688e3SAlex Vesker } 109726d688e3SAlex Vesker 10985212f9c6SYevgeny Kliteynik int mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx, 10995212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 11005212f9c6SYevgeny Kliteynik struct mlx5dr_match_param *mask, 11015212f9c6SYevgeny Kliteynik struct mlx5dr_cmd_caps *caps, 11025212f9c6SYevgeny Kliteynik bool inner, bool rx) 11035212f9c6SYevgeny Kliteynik { 11045212f9c6SYevgeny Kliteynik sb->rx = rx; 11055212f9c6SYevgeny Kliteynik sb->inner = inner; 11065212f9c6SYevgeny Kliteynik sb->caps = caps; 11075212f9c6SYevgeny Kliteynik return ste_ctx->build_icmp_init(sb, mask); 11085212f9c6SYevgeny Kliteynik } 11095212f9c6SYevgeny Kliteynik 11105212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx, 11115212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 111226d688e3SAlex Vesker struct mlx5dr_match_param *mask, 111326d688e3SAlex Vesker bool inner, bool rx) 111426d688e3SAlex Vesker { 111526d688e3SAlex Vesker sb->rx = rx; 111626d688e3SAlex Vesker sb->inner = inner; 11175212f9c6SYevgeny Kliteynik ste_ctx->build_general_purpose_init(sb, mask); 111826d688e3SAlex Vesker } 111926d688e3SAlex Vesker 11205212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx, 11215212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 112226d688e3SAlex Vesker struct mlx5dr_match_param *mask, 112326d688e3SAlex Vesker bool inner, bool rx) 112426d688e3SAlex Vesker { 112526d688e3SAlex Vesker sb->rx = rx; 112626d688e3SAlex Vesker sb->inner = inner; 11275212f9c6SYevgeny Kliteynik ste_ctx->build_eth_l4_misc_init(sb, mask); 112826d688e3SAlex Vesker } 112926d688e3SAlex Vesker 11305212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx, 11315212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 113226d688e3SAlex Vesker struct mlx5dr_match_param *mask, 113326d688e3SAlex Vesker bool inner, bool rx) 113426d688e3SAlex Vesker { 113526d688e3SAlex Vesker sb->rx = rx; 113626d688e3SAlex Vesker sb->inner = inner; 11375212f9c6SYevgeny Kliteynik ste_ctx->build_tnl_vxlan_gpe_init(sb, mask); 113826d688e3SAlex Vesker } 113926d688e3SAlex Vesker 11405212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx, 11415212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 1142b6d12238SYevgeny Kliteynik struct mlx5dr_match_param *mask, 1143b6d12238SYevgeny Kliteynik bool inner, bool rx) 1144b6d12238SYevgeny Kliteynik { 1145b6d12238SYevgeny Kliteynik sb->rx = rx; 1146b6d12238SYevgeny Kliteynik sb->inner = inner; 11475212f9c6SYevgeny Kliteynik ste_ctx->build_tnl_geneve_init(sb, mask); 114826d688e3SAlex Vesker } 114926d688e3SAlex Vesker 11505212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx, 11515212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 115226d688e3SAlex Vesker struct mlx5dr_match_param *mask, 115326d688e3SAlex Vesker bool inner, bool rx) 115426d688e3SAlex Vesker { 115526d688e3SAlex Vesker sb->rx = rx; 115626d688e3SAlex Vesker sb->inner = inner; 11575212f9c6SYevgeny Kliteynik ste_ctx->build_register_0_init(sb, mask); 115826d688e3SAlex Vesker } 115926d688e3SAlex Vesker 11605212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx, 11615212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 116226d688e3SAlex Vesker struct mlx5dr_match_param *mask, 116326d688e3SAlex Vesker bool inner, bool rx) 116426d688e3SAlex Vesker { 116526d688e3SAlex Vesker sb->rx = rx; 116626d688e3SAlex Vesker sb->inner = inner; 11675212f9c6SYevgeny Kliteynik ste_ctx->build_register_1_init(sb, mask); 116826d688e3SAlex Vesker } 116926d688e3SAlex Vesker 11705212f9c6SYevgeny Kliteynik void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx, 11715212f9c6SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 117226d688e3SAlex Vesker struct mlx5dr_match_param *mask, 1173640bdb1fSAlaa Hleihel struct mlx5dr_domain *dmn, 117426d688e3SAlex Vesker bool inner, bool rx) 117526d688e3SAlex Vesker { 1176640bdb1fSAlaa Hleihel /* Set vhca_id_valid before we reset source_eswitch_owner_vhca_id */ 1177640bdb1fSAlaa Hleihel sb->vhca_id_valid = mask->misc.source_eswitch_owner_vhca_id; 1178640bdb1fSAlaa Hleihel 117926d688e3SAlex Vesker sb->rx = rx; 1180640bdb1fSAlaa Hleihel sb->dmn = dmn; 118126d688e3SAlex Vesker sb->inner = inner; 11825212f9c6SYevgeny Kliteynik ste_ctx->build_src_gvmi_qpn_init(sb, mask); 11835212f9c6SYevgeny Kliteynik } 11845212f9c6SYevgeny Kliteynik 1185*160e9cb3SYevgeny Kliteynik void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx, 1186*160e9cb3SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 1187*160e9cb3SYevgeny Kliteynik struct mlx5dr_match_param *mask, 1188*160e9cb3SYevgeny Kliteynik bool inner, bool rx) 1189*160e9cb3SYevgeny Kliteynik { 1190*160e9cb3SYevgeny Kliteynik sb->rx = rx; 1191*160e9cb3SYevgeny Kliteynik sb->inner = inner; 1192*160e9cb3SYevgeny Kliteynik ste_ctx->build_flex_parser_0_init(sb, mask); 1193*160e9cb3SYevgeny Kliteynik } 1194*160e9cb3SYevgeny Kliteynik 1195*160e9cb3SYevgeny Kliteynik void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx, 1196*160e9cb3SYevgeny Kliteynik struct mlx5dr_ste_build *sb, 1197*160e9cb3SYevgeny Kliteynik struct mlx5dr_match_param *mask, 1198*160e9cb3SYevgeny Kliteynik bool inner, bool rx) 1199*160e9cb3SYevgeny Kliteynik { 1200*160e9cb3SYevgeny Kliteynik sb->rx = rx; 1201*160e9cb3SYevgeny Kliteynik sb->inner = inner; 1202*160e9cb3SYevgeny Kliteynik ste_ctx->build_flex_parser_1_init(sb, mask); 1203*160e9cb3SYevgeny Kliteynik } 1204*160e9cb3SYevgeny Kliteynik 12055212f9c6SYevgeny Kliteynik static struct mlx5dr_ste_ctx *mlx5dr_ste_ctx_arr[] = { 12065212f9c6SYevgeny Kliteynik [MLX5_STEERING_FORMAT_CONNECTX_5] = &ste_ctx_v0, 120710b69418SYevgeny Kliteynik [MLX5_STEERING_FORMAT_CONNECTX_6DX] = &ste_ctx_v1, 12085212f9c6SYevgeny Kliteynik }; 12095212f9c6SYevgeny Kliteynik 12105212f9c6SYevgeny Kliteynik struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version) 12115212f9c6SYevgeny Kliteynik { 12125212f9c6SYevgeny Kliteynik if (version > MLX5_STEERING_FORMAT_CONNECTX_6DX) 12135212f9c6SYevgeny Kliteynik return NULL; 12145212f9c6SYevgeny Kliteynik 12155212f9c6SYevgeny Kliteynik return mlx5dr_ste_ctx_arr[version]; 121626d688e3SAlex Vesker } 1217