141d07074SAlex Vesker // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 241d07074SAlex Vesker /* Copyright (c) 2019 Mellanox Technologies. */ 341d07074SAlex Vesker 441d07074SAlex Vesker #include "dr_types.h" 541d07074SAlex Vesker 641d07074SAlex Vesker #define DR_RULE_MAX_STE_CHAIN (DR_RULE_MAX_STES + DR_ACTION_MAX_STES) 741d07074SAlex Vesker 841d07074SAlex Vesker struct mlx5dr_rule_action_member { 941d07074SAlex Vesker struct mlx5dr_action *action; 1041d07074SAlex Vesker struct list_head list; 1141d07074SAlex Vesker }; 1241d07074SAlex Vesker 136b93b400SYevgeny Kliteynik static int dr_rule_append_to_miss_list(struct mlx5dr_ste_ctx *ste_ctx, 146b93b400SYevgeny Kliteynik struct mlx5dr_ste *new_last_ste, 1541d07074SAlex Vesker struct list_head *miss_list, 1641d07074SAlex Vesker struct list_head *send_list) 1741d07074SAlex Vesker { 1841d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info_last; 1941d07074SAlex Vesker struct mlx5dr_ste *last_ste; 2041d07074SAlex Vesker 2141d07074SAlex Vesker /* The new entry will be inserted after the last */ 2248cbde4bSAlex Vesker last_ste = list_last_entry(miss_list, struct mlx5dr_ste, miss_list_node); 2341d07074SAlex Vesker WARN_ON(!last_ste); 2441d07074SAlex Vesker 2541d07074SAlex Vesker ste_info_last = kzalloc(sizeof(*ste_info_last), GFP_KERNEL); 2641d07074SAlex Vesker if (!ste_info_last) 2741d07074SAlex Vesker return -ENOMEM; 2841d07074SAlex Vesker 296b93b400SYevgeny Kliteynik mlx5dr_ste_set_miss_addr(ste_ctx, last_ste->hw_ste, 3041d07074SAlex Vesker mlx5dr_ste_get_icm_addr(new_last_ste)); 3141d07074SAlex Vesker list_add_tail(&new_last_ste->miss_list_node, miss_list); 3241d07074SAlex Vesker 33f06d4969SYevgeny Kliteynik mlx5dr_send_fill_and_append_ste_send_info(last_ste, DR_STE_SIZE_CTRL, 3441d07074SAlex Vesker 0, last_ste->hw_ste, 3541d07074SAlex Vesker ste_info_last, send_list, true); 3641d07074SAlex Vesker 3741d07074SAlex Vesker return 0; 3841d07074SAlex Vesker } 3941d07074SAlex Vesker 4041d07074SAlex Vesker static struct mlx5dr_ste * 4141d07074SAlex Vesker dr_rule_create_collision_htbl(struct mlx5dr_matcher *matcher, 4241d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 4341d07074SAlex Vesker u8 *hw_ste) 4441d07074SAlex Vesker { 4541d07074SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 466b93b400SYevgeny Kliteynik struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; 4741d07074SAlex Vesker struct mlx5dr_ste_htbl *new_htbl; 4841d07074SAlex Vesker struct mlx5dr_ste *ste; 4941d07074SAlex Vesker 5041d07074SAlex Vesker /* Create new table for miss entry */ 5141d07074SAlex Vesker new_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool, 5241d07074SAlex Vesker DR_CHUNK_SIZE_1, 5341d07074SAlex Vesker MLX5DR_STE_LU_TYPE_DONT_CARE, 5441d07074SAlex Vesker 0); 5541d07074SAlex Vesker if (!new_htbl) { 5641d07074SAlex Vesker mlx5dr_dbg(dmn, "Failed allocating collision table\n"); 5741d07074SAlex Vesker return NULL; 5841d07074SAlex Vesker } 5941d07074SAlex Vesker 6041d07074SAlex Vesker /* One and only entry, never grows */ 6141d07074SAlex Vesker ste = new_htbl->ste_arr; 626b93b400SYevgeny Kliteynik mlx5dr_ste_set_miss_addr(ste_ctx, hw_ste, 636b93b400SYevgeny Kliteynik nic_matcher->e_anchor->chunk->icm_addr); 6441d07074SAlex Vesker mlx5dr_htbl_get(new_htbl); 6541d07074SAlex Vesker 6641d07074SAlex Vesker return ste; 6741d07074SAlex Vesker } 6841d07074SAlex Vesker 6941d07074SAlex Vesker static struct mlx5dr_ste * 7041d07074SAlex Vesker dr_rule_create_collision_entry(struct mlx5dr_matcher *matcher, 7141d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 7241d07074SAlex Vesker u8 *hw_ste, 7341d07074SAlex Vesker struct mlx5dr_ste *orig_ste) 7441d07074SAlex Vesker { 7541d07074SAlex Vesker struct mlx5dr_ste *ste; 7641d07074SAlex Vesker 7741d07074SAlex Vesker ste = dr_rule_create_collision_htbl(matcher, nic_matcher, hw_ste); 7841d07074SAlex Vesker if (!ste) { 7941d07074SAlex Vesker mlx5dr_dbg(matcher->tbl->dmn, "Failed creating collision entry\n"); 8041d07074SAlex Vesker return NULL; 8141d07074SAlex Vesker } 8241d07074SAlex Vesker 8341d07074SAlex Vesker ste->ste_chain_location = orig_ste->ste_chain_location; 8441d07074SAlex Vesker 8541d07074SAlex Vesker /* In collision entry, all members share the same miss_list_head */ 8641d07074SAlex Vesker ste->htbl->miss_list = mlx5dr_ste_get_miss_list(orig_ste); 8741d07074SAlex Vesker 8841d07074SAlex Vesker /* Next table */ 8941d07074SAlex Vesker if (mlx5dr_ste_create_next_htbl(matcher, nic_matcher, ste, hw_ste, 9041d07074SAlex Vesker DR_CHUNK_SIZE_1)) { 9141d07074SAlex Vesker mlx5dr_dbg(matcher->tbl->dmn, "Failed allocating table\n"); 9241d07074SAlex Vesker goto free_tbl; 9341d07074SAlex Vesker } 9441d07074SAlex Vesker 9541d07074SAlex Vesker return ste; 9641d07074SAlex Vesker 9741d07074SAlex Vesker free_tbl: 9841d07074SAlex Vesker mlx5dr_ste_free(ste, matcher, nic_matcher); 9941d07074SAlex Vesker return NULL; 10041d07074SAlex Vesker } 10141d07074SAlex Vesker 10241d07074SAlex Vesker static int 10341d07074SAlex Vesker dr_rule_handle_one_ste_in_update_list(struct mlx5dr_ste_send_info *ste_info, 10441d07074SAlex Vesker struct mlx5dr_domain *dmn) 10541d07074SAlex Vesker { 10641d07074SAlex Vesker int ret; 10741d07074SAlex Vesker 10841d07074SAlex Vesker list_del(&ste_info->send_list); 109f06d4969SYevgeny Kliteynik 110f06d4969SYevgeny Kliteynik /* Copy data to ste, only reduced size or control, the last 16B (mask) 11141d07074SAlex Vesker * is already written to the hw. 11241d07074SAlex Vesker */ 113f06d4969SYevgeny Kliteynik if (ste_info->size == DR_STE_SIZE_CTRL) 114f06d4969SYevgeny Kliteynik memcpy(ste_info->ste->hw_ste, ste_info->data, DR_STE_SIZE_CTRL); 115f06d4969SYevgeny Kliteynik else 11641d07074SAlex Vesker memcpy(ste_info->ste->hw_ste, ste_info->data, DR_STE_SIZE_REDUCED); 11741d07074SAlex Vesker 1184fe45e1dSYevgeny Kliteynik ret = mlx5dr_send_postsend_ste(dmn, ste_info->ste, ste_info->data, 1194fe45e1dSYevgeny Kliteynik ste_info->size, ste_info->offset); 1204fe45e1dSYevgeny Kliteynik if (ret) 1214fe45e1dSYevgeny Kliteynik goto out; 1224fe45e1dSYevgeny Kliteynik 12341d07074SAlex Vesker out: 12441d07074SAlex Vesker kfree(ste_info); 12541d07074SAlex Vesker return ret; 12641d07074SAlex Vesker } 12741d07074SAlex Vesker 12841d07074SAlex Vesker static int dr_rule_send_update_list(struct list_head *send_ste_list, 12941d07074SAlex Vesker struct mlx5dr_domain *dmn, 13041d07074SAlex Vesker bool is_reverse) 13141d07074SAlex Vesker { 13241d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info, *tmp_ste_info; 13341d07074SAlex Vesker int ret; 13441d07074SAlex Vesker 13541d07074SAlex Vesker if (is_reverse) { 13641d07074SAlex Vesker list_for_each_entry_safe_reverse(ste_info, tmp_ste_info, 13741d07074SAlex Vesker send_ste_list, send_list) { 13841d07074SAlex Vesker ret = dr_rule_handle_one_ste_in_update_list(ste_info, 13941d07074SAlex Vesker dmn); 14041d07074SAlex Vesker if (ret) 14141d07074SAlex Vesker return ret; 14241d07074SAlex Vesker } 14341d07074SAlex Vesker } else { 14441d07074SAlex Vesker list_for_each_entry_safe(ste_info, tmp_ste_info, 14541d07074SAlex Vesker send_ste_list, send_list) { 14641d07074SAlex Vesker ret = dr_rule_handle_one_ste_in_update_list(ste_info, 14741d07074SAlex Vesker dmn); 14841d07074SAlex Vesker if (ret) 14941d07074SAlex Vesker return ret; 15041d07074SAlex Vesker } 15141d07074SAlex Vesker } 15241d07074SAlex Vesker 15341d07074SAlex Vesker return 0; 15441d07074SAlex Vesker } 15541d07074SAlex Vesker 15641d07074SAlex Vesker static struct mlx5dr_ste * 15741d07074SAlex Vesker dr_rule_find_ste_in_miss_list(struct list_head *miss_list, u8 *hw_ste) 15841d07074SAlex Vesker { 15941d07074SAlex Vesker struct mlx5dr_ste *ste; 16041d07074SAlex Vesker 16141d07074SAlex Vesker if (list_empty(miss_list)) 16241d07074SAlex Vesker return NULL; 16341d07074SAlex Vesker 16441d07074SAlex Vesker /* Check if hw_ste is present in the list */ 16541d07074SAlex Vesker list_for_each_entry(ste, miss_list, miss_list_node) { 16641d07074SAlex Vesker if (mlx5dr_ste_equal_tag(ste->hw_ste, hw_ste)) 16741d07074SAlex Vesker return ste; 16841d07074SAlex Vesker } 16941d07074SAlex Vesker 17041d07074SAlex Vesker return NULL; 17141d07074SAlex Vesker } 17241d07074SAlex Vesker 17341d07074SAlex Vesker static struct mlx5dr_ste * 17441d07074SAlex Vesker dr_rule_rehash_handle_collision(struct mlx5dr_matcher *matcher, 17541d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 17641d07074SAlex Vesker struct list_head *update_list, 17741d07074SAlex Vesker struct mlx5dr_ste *col_ste, 17841d07074SAlex Vesker u8 *hw_ste) 17941d07074SAlex Vesker { 1806b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn = matcher->tbl->dmn; 18141d07074SAlex Vesker struct mlx5dr_ste *new_ste; 18241d07074SAlex Vesker int ret; 18341d07074SAlex Vesker 18441d07074SAlex Vesker new_ste = dr_rule_create_collision_htbl(matcher, nic_matcher, hw_ste); 18541d07074SAlex Vesker if (!new_ste) 18641d07074SAlex Vesker return NULL; 18741d07074SAlex Vesker 18841d07074SAlex Vesker /* In collision entry, all members share the same miss_list_head */ 18941d07074SAlex Vesker new_ste->htbl->miss_list = mlx5dr_ste_get_miss_list(col_ste); 19041d07074SAlex Vesker 19141d07074SAlex Vesker /* Update the previous from the list */ 1926b93b400SYevgeny Kliteynik ret = dr_rule_append_to_miss_list(dmn->ste_ctx, new_ste, 19341d07074SAlex Vesker mlx5dr_ste_get_miss_list(col_ste), 19441d07074SAlex Vesker update_list); 19541d07074SAlex Vesker if (ret) { 1966b93b400SYevgeny Kliteynik mlx5dr_dbg(dmn, "Failed update dup entry\n"); 19741d07074SAlex Vesker goto err_exit; 19841d07074SAlex Vesker } 19941d07074SAlex Vesker 20041d07074SAlex Vesker return new_ste; 20141d07074SAlex Vesker 20241d07074SAlex Vesker err_exit: 20341d07074SAlex Vesker mlx5dr_ste_free(new_ste, matcher, nic_matcher); 20441d07074SAlex Vesker return NULL; 20541d07074SAlex Vesker } 20641d07074SAlex Vesker 20741d07074SAlex Vesker static void dr_rule_rehash_copy_ste_ctrl(struct mlx5dr_matcher *matcher, 20841d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 20941d07074SAlex Vesker struct mlx5dr_ste *cur_ste, 21041d07074SAlex Vesker struct mlx5dr_ste *new_ste) 21141d07074SAlex Vesker { 21241d07074SAlex Vesker new_ste->next_htbl = cur_ste->next_htbl; 21341d07074SAlex Vesker new_ste->ste_chain_location = cur_ste->ste_chain_location; 21441d07074SAlex Vesker 21541d07074SAlex Vesker if (!mlx5dr_ste_is_last_in_rule(nic_matcher, new_ste->ste_chain_location)) 21641d07074SAlex Vesker new_ste->next_htbl->pointing_ste = new_ste; 21741d07074SAlex Vesker 21841d07074SAlex Vesker /* We need to copy the refcount since this ste 21941d07074SAlex Vesker * may have been traversed several times 22041d07074SAlex Vesker */ 2214ce380caSYevgeny Kliteynik new_ste->refcount = cur_ste->refcount; 22241d07074SAlex Vesker 22341d07074SAlex Vesker /* Link old STEs rule_mem list to the new ste */ 22441d07074SAlex Vesker mlx5dr_rule_update_rule_member(cur_ste, new_ste); 22541d07074SAlex Vesker INIT_LIST_HEAD(&new_ste->rule_list); 22641d07074SAlex Vesker list_splice_tail_init(&cur_ste->rule_list, &new_ste->rule_list); 22741d07074SAlex Vesker } 22841d07074SAlex Vesker 22941d07074SAlex Vesker static struct mlx5dr_ste * 23041d07074SAlex Vesker dr_rule_rehash_copy_ste(struct mlx5dr_matcher *matcher, 23141d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 23241d07074SAlex Vesker struct mlx5dr_ste *cur_ste, 23341d07074SAlex Vesker struct mlx5dr_ste_htbl *new_htbl, 23441d07074SAlex Vesker struct list_head *update_list) 23541d07074SAlex Vesker { 2366b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn = matcher->tbl->dmn; 23741d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info; 23841d07074SAlex Vesker bool use_update_list = false; 23941d07074SAlex Vesker u8 hw_ste[DR_STE_SIZE] = {}; 24041d07074SAlex Vesker struct mlx5dr_ste *new_ste; 24141d07074SAlex Vesker int new_idx; 24241d07074SAlex Vesker u8 sb_idx; 24341d07074SAlex Vesker 24441d07074SAlex Vesker /* Copy STE mask from the matcher */ 24541d07074SAlex Vesker sb_idx = cur_ste->ste_chain_location - 1; 24641d07074SAlex Vesker mlx5dr_ste_set_bit_mask(hw_ste, nic_matcher->ste_builder[sb_idx].bit_mask); 24741d07074SAlex Vesker 24841d07074SAlex Vesker /* Copy STE control and tag */ 24941d07074SAlex Vesker memcpy(hw_ste, cur_ste->hw_ste, DR_STE_SIZE_REDUCED); 2506b93b400SYevgeny Kliteynik mlx5dr_ste_set_miss_addr(dmn->ste_ctx, hw_ste, 2516b93b400SYevgeny Kliteynik nic_matcher->e_anchor->chunk->icm_addr); 25241d07074SAlex Vesker 25341d07074SAlex Vesker new_idx = mlx5dr_ste_calc_hash_index(hw_ste, new_htbl); 25441d07074SAlex Vesker new_ste = &new_htbl->ste_arr[new_idx]; 25541d07074SAlex Vesker 25697ffd895SYevgeny Kliteynik if (mlx5dr_ste_is_not_used(new_ste)) { 25741d07074SAlex Vesker mlx5dr_htbl_get(new_htbl); 25841d07074SAlex Vesker list_add_tail(&new_ste->miss_list_node, 25941d07074SAlex Vesker mlx5dr_ste_get_miss_list(new_ste)); 26041d07074SAlex Vesker } else { 26141d07074SAlex Vesker new_ste = dr_rule_rehash_handle_collision(matcher, 26241d07074SAlex Vesker nic_matcher, 26341d07074SAlex Vesker update_list, 26441d07074SAlex Vesker new_ste, 26541d07074SAlex Vesker hw_ste); 26641d07074SAlex Vesker if (!new_ste) { 2676b93b400SYevgeny Kliteynik mlx5dr_dbg(dmn, "Failed adding collision entry, index: %d\n", 26841d07074SAlex Vesker new_idx); 26941d07074SAlex Vesker return NULL; 27041d07074SAlex Vesker } 27141d07074SAlex Vesker new_htbl->ctrl.num_of_collisions++; 27241d07074SAlex Vesker use_update_list = true; 27341d07074SAlex Vesker } 27441d07074SAlex Vesker 27541d07074SAlex Vesker memcpy(new_ste->hw_ste, hw_ste, DR_STE_SIZE_REDUCED); 27641d07074SAlex Vesker 27741d07074SAlex Vesker new_htbl->ctrl.num_of_valid_entries++; 27841d07074SAlex Vesker 27941d07074SAlex Vesker if (use_update_list) { 28041d07074SAlex Vesker ste_info = kzalloc(sizeof(*ste_info), GFP_KERNEL); 28141d07074SAlex Vesker if (!ste_info) 28241d07074SAlex Vesker goto err_exit; 28341d07074SAlex Vesker 28441d07074SAlex Vesker mlx5dr_send_fill_and_append_ste_send_info(new_ste, DR_STE_SIZE, 0, 28541d07074SAlex Vesker hw_ste, ste_info, 28641d07074SAlex Vesker update_list, true); 28741d07074SAlex Vesker } 28841d07074SAlex Vesker 28941d07074SAlex Vesker dr_rule_rehash_copy_ste_ctrl(matcher, nic_matcher, cur_ste, new_ste); 29041d07074SAlex Vesker 29141d07074SAlex Vesker return new_ste; 29241d07074SAlex Vesker 29341d07074SAlex Vesker err_exit: 29441d07074SAlex Vesker mlx5dr_ste_free(new_ste, matcher, nic_matcher); 29541d07074SAlex Vesker return NULL; 29641d07074SAlex Vesker } 29741d07074SAlex Vesker 29841d07074SAlex Vesker static int dr_rule_rehash_copy_miss_list(struct mlx5dr_matcher *matcher, 29941d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 30041d07074SAlex Vesker struct list_head *cur_miss_list, 30141d07074SAlex Vesker struct mlx5dr_ste_htbl *new_htbl, 30241d07074SAlex Vesker struct list_head *update_list) 30341d07074SAlex Vesker { 30441d07074SAlex Vesker struct mlx5dr_ste *tmp_ste, *cur_ste, *new_ste; 30541d07074SAlex Vesker 30641d07074SAlex Vesker if (list_empty(cur_miss_list)) 30741d07074SAlex Vesker return 0; 30841d07074SAlex Vesker 30941d07074SAlex Vesker list_for_each_entry_safe(cur_ste, tmp_ste, cur_miss_list, miss_list_node) { 31041d07074SAlex Vesker new_ste = dr_rule_rehash_copy_ste(matcher, 31141d07074SAlex Vesker nic_matcher, 31241d07074SAlex Vesker cur_ste, 31341d07074SAlex Vesker new_htbl, 31441d07074SAlex Vesker update_list); 31541d07074SAlex Vesker if (!new_ste) 31641d07074SAlex Vesker goto err_insert; 31741d07074SAlex Vesker 31841d07074SAlex Vesker list_del(&cur_ste->miss_list_node); 31941d07074SAlex Vesker mlx5dr_htbl_put(cur_ste->htbl); 32041d07074SAlex Vesker } 32141d07074SAlex Vesker return 0; 32241d07074SAlex Vesker 32341d07074SAlex Vesker err_insert: 32441d07074SAlex Vesker mlx5dr_err(matcher->tbl->dmn, "Fatal error during resize\n"); 32541d07074SAlex Vesker WARN_ON(true); 32641d07074SAlex Vesker return -EINVAL; 32741d07074SAlex Vesker } 32841d07074SAlex Vesker 32941d07074SAlex Vesker static int dr_rule_rehash_copy_htbl(struct mlx5dr_matcher *matcher, 33041d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 33141d07074SAlex Vesker struct mlx5dr_ste_htbl *cur_htbl, 33241d07074SAlex Vesker struct mlx5dr_ste_htbl *new_htbl, 33341d07074SAlex Vesker struct list_head *update_list) 33441d07074SAlex Vesker { 33541d07074SAlex Vesker struct mlx5dr_ste *cur_ste; 33641d07074SAlex Vesker int cur_entries; 33741d07074SAlex Vesker int err = 0; 33841d07074SAlex Vesker int i; 33941d07074SAlex Vesker 34041d07074SAlex Vesker cur_entries = mlx5dr_icm_pool_chunk_size_to_entries(cur_htbl->chunk_size); 34141d07074SAlex Vesker 34241d07074SAlex Vesker if (cur_entries < 1) { 34341d07074SAlex Vesker mlx5dr_dbg(matcher->tbl->dmn, "Invalid number of entries\n"); 34441d07074SAlex Vesker return -EINVAL; 34541d07074SAlex Vesker } 34641d07074SAlex Vesker 34741d07074SAlex Vesker for (i = 0; i < cur_entries; i++) { 34841d07074SAlex Vesker cur_ste = &cur_htbl->ste_arr[i]; 34997ffd895SYevgeny Kliteynik if (mlx5dr_ste_is_not_used(cur_ste)) /* Empty, nothing to copy */ 35041d07074SAlex Vesker continue; 35141d07074SAlex Vesker 35241d07074SAlex Vesker err = dr_rule_rehash_copy_miss_list(matcher, 35341d07074SAlex Vesker nic_matcher, 35441d07074SAlex Vesker mlx5dr_ste_get_miss_list(cur_ste), 35541d07074SAlex Vesker new_htbl, 35641d07074SAlex Vesker update_list); 35741d07074SAlex Vesker if (err) 35841d07074SAlex Vesker goto clean_copy; 35941d07074SAlex Vesker } 36041d07074SAlex Vesker 36141d07074SAlex Vesker clean_copy: 36241d07074SAlex Vesker return err; 36341d07074SAlex Vesker } 36441d07074SAlex Vesker 36541d07074SAlex Vesker static struct mlx5dr_ste_htbl * 36641d07074SAlex Vesker dr_rule_rehash_htbl(struct mlx5dr_rule *rule, 36741d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule, 36841d07074SAlex Vesker struct mlx5dr_ste_htbl *cur_htbl, 36941d07074SAlex Vesker u8 ste_location, 37041d07074SAlex Vesker struct list_head *update_list, 37141d07074SAlex Vesker enum mlx5dr_icm_chunk_size new_size) 37241d07074SAlex Vesker { 37341d07074SAlex Vesker struct mlx5dr_ste_send_info *del_ste_info, *tmp_ste_info; 37441d07074SAlex Vesker struct mlx5dr_matcher *matcher = rule->matcher; 37541d07074SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 37641d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher; 37741d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info; 37841d07074SAlex Vesker struct mlx5dr_htbl_connect_info info; 37941d07074SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn; 38041d07074SAlex Vesker u8 formatted_ste[DR_STE_SIZE] = {}; 38141d07074SAlex Vesker LIST_HEAD(rehash_table_send_list); 38241d07074SAlex Vesker struct mlx5dr_ste *ste_to_update; 38341d07074SAlex Vesker struct mlx5dr_ste_htbl *new_htbl; 38441d07074SAlex Vesker int err; 38541d07074SAlex Vesker 38641d07074SAlex Vesker nic_matcher = nic_rule->nic_matcher; 38741d07074SAlex Vesker nic_dmn = nic_matcher->nic_tbl->nic_dmn; 38841d07074SAlex Vesker 38941d07074SAlex Vesker ste_info = kzalloc(sizeof(*ste_info), GFP_KERNEL); 39041d07074SAlex Vesker if (!ste_info) 39141d07074SAlex Vesker return NULL; 39241d07074SAlex Vesker 39341d07074SAlex Vesker new_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool, 39441d07074SAlex Vesker new_size, 39541d07074SAlex Vesker cur_htbl->lu_type, 39641d07074SAlex Vesker cur_htbl->byte_mask); 39741d07074SAlex Vesker if (!new_htbl) { 39841d07074SAlex Vesker mlx5dr_err(dmn, "Failed to allocate new hash table\n"); 39941d07074SAlex Vesker goto free_ste_info; 40041d07074SAlex Vesker } 40141d07074SAlex Vesker 40241d07074SAlex Vesker /* Write new table to HW */ 40341d07074SAlex Vesker info.type = CONNECT_MISS; 40441d07074SAlex Vesker info.miss_icm_addr = nic_matcher->e_anchor->chunk->icm_addr; 4056b93b400SYevgeny Kliteynik mlx5dr_ste_set_formatted_ste(dmn->ste_ctx, 4066b93b400SYevgeny Kliteynik dmn->info.caps.gvmi, 40741d07074SAlex Vesker nic_dmn, 40841d07074SAlex Vesker new_htbl, 40941d07074SAlex Vesker formatted_ste, 41041d07074SAlex Vesker &info); 41141d07074SAlex Vesker 41241d07074SAlex Vesker new_htbl->pointing_ste = cur_htbl->pointing_ste; 41341d07074SAlex Vesker new_htbl->pointing_ste->next_htbl = new_htbl; 41441d07074SAlex Vesker err = dr_rule_rehash_copy_htbl(matcher, 41541d07074SAlex Vesker nic_matcher, 41641d07074SAlex Vesker cur_htbl, 41741d07074SAlex Vesker new_htbl, 41841d07074SAlex Vesker &rehash_table_send_list); 41941d07074SAlex Vesker if (err) 42041d07074SAlex Vesker goto free_new_htbl; 42141d07074SAlex Vesker 42241d07074SAlex Vesker if (mlx5dr_send_postsend_htbl(dmn, new_htbl, formatted_ste, 42341d07074SAlex Vesker nic_matcher->ste_builder[ste_location - 1].bit_mask)) { 42441d07074SAlex Vesker mlx5dr_err(dmn, "Failed writing table to HW\n"); 42541d07074SAlex Vesker goto free_new_htbl; 42641d07074SAlex Vesker } 42741d07074SAlex Vesker 42841d07074SAlex Vesker /* Writing to the hw is done in regular order of rehash_table_send_list, 42941d07074SAlex Vesker * in order to have the origin data written before the miss address of 43041d07074SAlex Vesker * collision entries, if exists. 43141d07074SAlex Vesker */ 43241d07074SAlex Vesker if (dr_rule_send_update_list(&rehash_table_send_list, dmn, false)) { 43341d07074SAlex Vesker mlx5dr_err(dmn, "Failed updating table to HW\n"); 43441d07074SAlex Vesker goto free_ste_list; 43541d07074SAlex Vesker } 43641d07074SAlex Vesker 43741d07074SAlex Vesker /* Connect previous hash table to current */ 43841d07074SAlex Vesker if (ste_location == 1) { 43941d07074SAlex Vesker /* The previous table is an anchor, anchors size is always one STE */ 44041d07074SAlex Vesker struct mlx5dr_ste_htbl *prev_htbl = cur_htbl->pointing_ste->htbl; 44141d07074SAlex Vesker 44241d07074SAlex Vesker /* On matcher s_anchor we keep an extra refcount */ 44341d07074SAlex Vesker mlx5dr_htbl_get(new_htbl); 44441d07074SAlex Vesker mlx5dr_htbl_put(cur_htbl); 44541d07074SAlex Vesker 44641d07074SAlex Vesker nic_matcher->s_htbl = new_htbl; 44741d07074SAlex Vesker 44841d07074SAlex Vesker /* It is safe to operate dr_ste_set_hit_addr on the hw_ste here 44941d07074SAlex Vesker * (48B len) which works only on first 32B 45041d07074SAlex Vesker */ 4516b93b400SYevgeny Kliteynik mlx5dr_ste_set_hit_addr(dmn->ste_ctx, 4526b93b400SYevgeny Kliteynik prev_htbl->ste_arr[0].hw_ste, 45341d07074SAlex Vesker new_htbl->chunk->icm_addr, 45441d07074SAlex Vesker new_htbl->chunk->num_of_entries); 45541d07074SAlex Vesker 45641d07074SAlex Vesker ste_to_update = &prev_htbl->ste_arr[0]; 45741d07074SAlex Vesker } else { 4586b93b400SYevgeny Kliteynik mlx5dr_ste_set_hit_addr_by_next_htbl(dmn->ste_ctx, 4596b93b400SYevgeny Kliteynik cur_htbl->pointing_ste->hw_ste, 46041d07074SAlex Vesker new_htbl); 46141d07074SAlex Vesker ste_to_update = cur_htbl->pointing_ste; 46241d07074SAlex Vesker } 46341d07074SAlex Vesker 464f06d4969SYevgeny Kliteynik mlx5dr_send_fill_and_append_ste_send_info(ste_to_update, DR_STE_SIZE_CTRL, 46541d07074SAlex Vesker 0, ste_to_update->hw_ste, ste_info, 46641d07074SAlex Vesker update_list, false); 46741d07074SAlex Vesker 46841d07074SAlex Vesker return new_htbl; 46941d07074SAlex Vesker 47041d07074SAlex Vesker free_ste_list: 47141d07074SAlex Vesker /* Clean all ste_info's from the new table */ 47241d07074SAlex Vesker list_for_each_entry_safe(del_ste_info, tmp_ste_info, 47341d07074SAlex Vesker &rehash_table_send_list, send_list) { 47441d07074SAlex Vesker list_del(&del_ste_info->send_list); 47541d07074SAlex Vesker kfree(del_ste_info); 47641d07074SAlex Vesker } 47741d07074SAlex Vesker 47841d07074SAlex Vesker free_new_htbl: 47941d07074SAlex Vesker mlx5dr_ste_htbl_free(new_htbl); 48041d07074SAlex Vesker free_ste_info: 48141d07074SAlex Vesker kfree(ste_info); 48241d07074SAlex Vesker mlx5dr_info(dmn, "Failed creating rehash table\n"); 48341d07074SAlex Vesker return NULL; 48441d07074SAlex Vesker } 48541d07074SAlex Vesker 48641d07074SAlex Vesker static struct mlx5dr_ste_htbl *dr_rule_rehash(struct mlx5dr_rule *rule, 48741d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule, 48841d07074SAlex Vesker struct mlx5dr_ste_htbl *cur_htbl, 48941d07074SAlex Vesker u8 ste_location, 49041d07074SAlex Vesker struct list_head *update_list) 49141d07074SAlex Vesker { 49241d07074SAlex Vesker struct mlx5dr_domain *dmn = rule->matcher->tbl->dmn; 49341d07074SAlex Vesker enum mlx5dr_icm_chunk_size new_size; 49441d07074SAlex Vesker 49541d07074SAlex Vesker new_size = mlx5dr_icm_next_higher_chunk(cur_htbl->chunk_size); 49641d07074SAlex Vesker new_size = min_t(u32, new_size, dmn->info.max_log_sw_icm_sz); 49741d07074SAlex Vesker 49841d07074SAlex Vesker if (new_size == cur_htbl->chunk_size) 49941d07074SAlex Vesker return NULL; /* Skip rehash, we already at the max size */ 50041d07074SAlex Vesker 50141d07074SAlex Vesker return dr_rule_rehash_htbl(rule, nic_rule, cur_htbl, ste_location, 50241d07074SAlex Vesker update_list, new_size); 50341d07074SAlex Vesker } 50441d07074SAlex Vesker 50541d07074SAlex Vesker static struct mlx5dr_ste * 50641d07074SAlex Vesker dr_rule_handle_collision(struct mlx5dr_matcher *matcher, 50741d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 50841d07074SAlex Vesker struct mlx5dr_ste *ste, 50941d07074SAlex Vesker u8 *hw_ste, 51041d07074SAlex Vesker struct list_head *miss_list, 51141d07074SAlex Vesker struct list_head *send_list) 51241d07074SAlex Vesker { 5136b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn = matcher->tbl->dmn; 5146b93b400SYevgeny Kliteynik struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; 51541d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info; 51641d07074SAlex Vesker struct mlx5dr_ste *new_ste; 51741d07074SAlex Vesker 51841d07074SAlex Vesker ste_info = kzalloc(sizeof(*ste_info), GFP_KERNEL); 51941d07074SAlex Vesker if (!ste_info) 52041d07074SAlex Vesker return NULL; 52141d07074SAlex Vesker 52241d07074SAlex Vesker new_ste = dr_rule_create_collision_entry(matcher, nic_matcher, hw_ste, ste); 52341d07074SAlex Vesker if (!new_ste) 52441d07074SAlex Vesker goto free_send_info; 52541d07074SAlex Vesker 5266b93b400SYevgeny Kliteynik if (dr_rule_append_to_miss_list(ste_ctx, new_ste, 5276b93b400SYevgeny Kliteynik miss_list, send_list)) { 5286b93b400SYevgeny Kliteynik mlx5dr_dbg(dmn, "Failed to update prev miss_list\n"); 52941d07074SAlex Vesker goto err_exit; 53041d07074SAlex Vesker } 53141d07074SAlex Vesker 53241d07074SAlex Vesker mlx5dr_send_fill_and_append_ste_send_info(new_ste, DR_STE_SIZE, 0, hw_ste, 53341d07074SAlex Vesker ste_info, send_list, false); 53441d07074SAlex Vesker 53541d07074SAlex Vesker ste->htbl->ctrl.num_of_collisions++; 53641d07074SAlex Vesker ste->htbl->ctrl.num_of_valid_entries++; 53741d07074SAlex Vesker 53841d07074SAlex Vesker return new_ste; 53941d07074SAlex Vesker 54041d07074SAlex Vesker err_exit: 54141d07074SAlex Vesker mlx5dr_ste_free(new_ste, matcher, nic_matcher); 54241d07074SAlex Vesker free_send_info: 54341d07074SAlex Vesker kfree(ste_info); 54441d07074SAlex Vesker return NULL; 54541d07074SAlex Vesker } 54641d07074SAlex Vesker 54741d07074SAlex Vesker static void dr_rule_remove_action_members(struct mlx5dr_rule *rule) 54841d07074SAlex Vesker { 54941d07074SAlex Vesker struct mlx5dr_rule_action_member *action_mem; 55041d07074SAlex Vesker struct mlx5dr_rule_action_member *tmp; 55141d07074SAlex Vesker 55241d07074SAlex Vesker list_for_each_entry_safe(action_mem, tmp, &rule->rule_actions_list, list) { 55341d07074SAlex Vesker list_del(&action_mem->list); 55441d07074SAlex Vesker refcount_dec(&action_mem->action->refcount); 55541d07074SAlex Vesker kvfree(action_mem); 55641d07074SAlex Vesker } 55741d07074SAlex Vesker } 55841d07074SAlex Vesker 55941d07074SAlex Vesker static int dr_rule_add_action_members(struct mlx5dr_rule *rule, 56041d07074SAlex Vesker size_t num_actions, 56141d07074SAlex Vesker struct mlx5dr_action *actions[]) 56241d07074SAlex Vesker { 56341d07074SAlex Vesker struct mlx5dr_rule_action_member *action_mem; 56441d07074SAlex Vesker int i; 56541d07074SAlex Vesker 56641d07074SAlex Vesker for (i = 0; i < num_actions; i++) { 56741d07074SAlex Vesker action_mem = kvzalloc(sizeof(*action_mem), GFP_KERNEL); 56841d07074SAlex Vesker if (!action_mem) 56941d07074SAlex Vesker goto free_action_members; 57041d07074SAlex Vesker 57141d07074SAlex Vesker action_mem->action = actions[i]; 57241d07074SAlex Vesker INIT_LIST_HEAD(&action_mem->list); 57341d07074SAlex Vesker list_add_tail(&action_mem->list, &rule->rule_actions_list); 57441d07074SAlex Vesker refcount_inc(&action_mem->action->refcount); 57541d07074SAlex Vesker } 57641d07074SAlex Vesker 57741d07074SAlex Vesker return 0; 57841d07074SAlex Vesker 57941d07074SAlex Vesker free_action_members: 58041d07074SAlex Vesker dr_rule_remove_action_members(rule); 58141d07074SAlex Vesker return -ENOMEM; 58241d07074SAlex Vesker } 58341d07074SAlex Vesker 58441d07074SAlex Vesker /* While the pointer of ste is no longer valid, like while moving ste to be 58541d07074SAlex Vesker * the first in the miss_list, and to be in the origin table, 58641d07074SAlex Vesker * all rule-members that are attached to this ste should update their ste member 58741d07074SAlex Vesker * to the new pointer 58841d07074SAlex Vesker */ 58941d07074SAlex Vesker void mlx5dr_rule_update_rule_member(struct mlx5dr_ste *ste, 59041d07074SAlex Vesker struct mlx5dr_ste *new_ste) 59141d07074SAlex Vesker { 59241d07074SAlex Vesker struct mlx5dr_rule_member *rule_mem; 59341d07074SAlex Vesker 59441d07074SAlex Vesker list_for_each_entry(rule_mem, &ste->rule_list, use_ste_list) 59541d07074SAlex Vesker rule_mem->ste = new_ste; 59641d07074SAlex Vesker } 59741d07074SAlex Vesker 59841d07074SAlex Vesker static void dr_rule_clean_rule_members(struct mlx5dr_rule *rule, 59941d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule) 60041d07074SAlex Vesker { 60141d07074SAlex Vesker struct mlx5dr_rule_member *rule_mem; 60241d07074SAlex Vesker struct mlx5dr_rule_member *tmp_mem; 60341d07074SAlex Vesker 60441d07074SAlex Vesker if (list_empty(&nic_rule->rule_members_list)) 60541d07074SAlex Vesker return; 60641d07074SAlex Vesker list_for_each_entry_safe(rule_mem, tmp_mem, &nic_rule->rule_members_list, list) { 60741d07074SAlex Vesker list_del(&rule_mem->list); 60841d07074SAlex Vesker list_del(&rule_mem->use_ste_list); 60941d07074SAlex Vesker mlx5dr_ste_put(rule_mem->ste, rule->matcher, nic_rule->nic_matcher); 61041d07074SAlex Vesker kvfree(rule_mem); 61141d07074SAlex Vesker } 61241d07074SAlex Vesker } 61341d07074SAlex Vesker 61421586a0fSAlex Vesker static u16 dr_get_bits_per_mask(u16 byte_mask) 61521586a0fSAlex Vesker { 61621586a0fSAlex Vesker u16 bits = 0; 61721586a0fSAlex Vesker 61821586a0fSAlex Vesker while (byte_mask) { 61921586a0fSAlex Vesker byte_mask = byte_mask & (byte_mask - 1); 62021586a0fSAlex Vesker bits++; 62121586a0fSAlex Vesker } 62221586a0fSAlex Vesker 62321586a0fSAlex Vesker return bits; 62421586a0fSAlex Vesker } 62521586a0fSAlex Vesker 62641d07074SAlex Vesker static bool dr_rule_need_enlarge_hash(struct mlx5dr_ste_htbl *htbl, 62741d07074SAlex Vesker struct mlx5dr_domain *dmn, 62841d07074SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn) 62941d07074SAlex Vesker { 63041d07074SAlex Vesker struct mlx5dr_ste_htbl_ctrl *ctrl = &htbl->ctrl; 63141d07074SAlex Vesker 63241d07074SAlex Vesker if (dmn->info.max_log_sw_icm_sz <= htbl->chunk_size) 63341d07074SAlex Vesker return false; 63441d07074SAlex Vesker 63541d07074SAlex Vesker if (!ctrl->may_grow) 63641d07074SAlex Vesker return false; 63741d07074SAlex Vesker 63821586a0fSAlex Vesker if (dr_get_bits_per_mask(htbl->byte_mask) * BITS_PER_BYTE <= htbl->chunk_size) 63921586a0fSAlex Vesker return false; 64021586a0fSAlex Vesker 64141d07074SAlex Vesker if (ctrl->num_of_collisions >= ctrl->increase_threshold && 64241d07074SAlex Vesker (ctrl->num_of_valid_entries - ctrl->num_of_collisions) >= ctrl->increase_threshold) 64341d07074SAlex Vesker return true; 64441d07074SAlex Vesker 64541d07074SAlex Vesker return false; 64641d07074SAlex Vesker } 64741d07074SAlex Vesker 64841d07074SAlex Vesker static int dr_rule_add_member(struct mlx5dr_rule_rx_tx *nic_rule, 64941d07074SAlex Vesker struct mlx5dr_ste *ste) 65041d07074SAlex Vesker { 65141d07074SAlex Vesker struct mlx5dr_rule_member *rule_mem; 65241d07074SAlex Vesker 65341d07074SAlex Vesker rule_mem = kvzalloc(sizeof(*rule_mem), GFP_KERNEL); 65441d07074SAlex Vesker if (!rule_mem) 65541d07074SAlex Vesker return -ENOMEM; 65641d07074SAlex Vesker 657df55c558SErez Shitrit INIT_LIST_HEAD(&rule_mem->list); 658df55c558SErez Shitrit INIT_LIST_HEAD(&rule_mem->use_ste_list); 659df55c558SErez Shitrit 66041d07074SAlex Vesker rule_mem->ste = ste; 66141d07074SAlex Vesker list_add_tail(&rule_mem->list, &nic_rule->rule_members_list); 66241d07074SAlex Vesker 66341d07074SAlex Vesker list_add_tail(&rule_mem->use_ste_list, &ste->rule_list); 66441d07074SAlex Vesker 66541d07074SAlex Vesker return 0; 66641d07074SAlex Vesker } 66741d07074SAlex Vesker 66841d07074SAlex Vesker static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule, 66941d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule, 67041d07074SAlex Vesker struct list_head *send_ste_list, 67141d07074SAlex Vesker struct mlx5dr_ste *last_ste, 67241d07074SAlex Vesker u8 *hw_ste_arr, 67341d07074SAlex Vesker u32 new_hw_ste_arr_sz) 67441d07074SAlex Vesker { 67541d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher = nic_rule->nic_matcher; 67641d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info_arr[DR_ACTION_MAX_STES]; 67741d07074SAlex Vesker u8 num_of_builders = nic_matcher->num_of_builders; 67841d07074SAlex Vesker struct mlx5dr_matcher *matcher = rule->matcher; 6796b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn = matcher->tbl->dmn; 68041d07074SAlex Vesker u8 *curr_hw_ste, *prev_hw_ste; 68141d07074SAlex Vesker struct mlx5dr_ste *action_ste; 68241d07074SAlex Vesker int i, k, ret; 68341d07074SAlex Vesker 68441d07074SAlex Vesker /* Two cases: 68541d07074SAlex Vesker * 1. num_of_builders is equal to new_hw_ste_arr_sz, the action in the ste 68641d07074SAlex Vesker * 2. num_of_builders is less then new_hw_ste_arr_sz, new ste was added 68741d07074SAlex Vesker * to support the action. 68841d07074SAlex Vesker */ 68941d07074SAlex Vesker if (num_of_builders == new_hw_ste_arr_sz) 69041d07074SAlex Vesker return 0; 69141d07074SAlex Vesker 69241d07074SAlex Vesker for (i = num_of_builders, k = 0; i < new_hw_ste_arr_sz; i++, k++) { 69341d07074SAlex Vesker curr_hw_ste = hw_ste_arr + i * DR_STE_SIZE; 69441d07074SAlex Vesker prev_hw_ste = (i == 0) ? curr_hw_ste : hw_ste_arr + ((i - 1) * DR_STE_SIZE); 69541d07074SAlex Vesker action_ste = dr_rule_create_collision_htbl(matcher, 69641d07074SAlex Vesker nic_matcher, 69741d07074SAlex Vesker curr_hw_ste); 69841d07074SAlex Vesker if (!action_ste) 69941d07074SAlex Vesker return -ENOMEM; 70041d07074SAlex Vesker 70141d07074SAlex Vesker mlx5dr_ste_get(action_ste); 70241d07074SAlex Vesker 70341d07074SAlex Vesker /* While free ste we go over the miss list, so add this ste to the list */ 70441d07074SAlex Vesker list_add_tail(&action_ste->miss_list_node, 70541d07074SAlex Vesker mlx5dr_ste_get_miss_list(action_ste)); 70641d07074SAlex Vesker 70741d07074SAlex Vesker ste_info_arr[k] = kzalloc(sizeof(*ste_info_arr[k]), 70841d07074SAlex Vesker GFP_KERNEL); 70941d07074SAlex Vesker if (!ste_info_arr[k]) 71041d07074SAlex Vesker goto err_exit; 71141d07074SAlex Vesker 71241d07074SAlex Vesker /* Point current ste to the new action */ 7136b93b400SYevgeny Kliteynik mlx5dr_ste_set_hit_addr_by_next_htbl(dmn->ste_ctx, 7146b93b400SYevgeny Kliteynik prev_hw_ste, 7156b93b400SYevgeny Kliteynik action_ste->htbl); 71641d07074SAlex Vesker ret = dr_rule_add_member(nic_rule, action_ste); 71741d07074SAlex Vesker if (ret) { 7186b93b400SYevgeny Kliteynik mlx5dr_dbg(dmn, "Failed adding rule member\n"); 71941d07074SAlex Vesker goto free_ste_info; 72041d07074SAlex Vesker } 72141d07074SAlex Vesker mlx5dr_send_fill_and_append_ste_send_info(action_ste, DR_STE_SIZE, 0, 72241d07074SAlex Vesker curr_hw_ste, 72341d07074SAlex Vesker ste_info_arr[k], 72441d07074SAlex Vesker send_ste_list, false); 72541d07074SAlex Vesker } 72641d07074SAlex Vesker 72741d07074SAlex Vesker return 0; 72841d07074SAlex Vesker 72941d07074SAlex Vesker free_ste_info: 73041d07074SAlex Vesker kfree(ste_info_arr[k]); 73141d07074SAlex Vesker err_exit: 73241d07074SAlex Vesker mlx5dr_ste_put(action_ste, matcher, nic_matcher); 73341d07074SAlex Vesker return -ENOMEM; 73441d07074SAlex Vesker } 73541d07074SAlex Vesker 73641d07074SAlex Vesker static int dr_rule_handle_empty_entry(struct mlx5dr_matcher *matcher, 73741d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher, 73841d07074SAlex Vesker struct mlx5dr_ste_htbl *cur_htbl, 73941d07074SAlex Vesker struct mlx5dr_ste *ste, 74041d07074SAlex Vesker u8 ste_location, 74141d07074SAlex Vesker u8 *hw_ste, 74241d07074SAlex Vesker struct list_head *miss_list, 74341d07074SAlex Vesker struct list_head *send_list) 74441d07074SAlex Vesker { 7456b93b400SYevgeny Kliteynik struct mlx5dr_domain *dmn = matcher->tbl->dmn; 74641d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info; 74741d07074SAlex Vesker 74841d07074SAlex Vesker /* Take ref on table, only on first time this ste is used */ 74941d07074SAlex Vesker mlx5dr_htbl_get(cur_htbl); 75041d07074SAlex Vesker 75141d07074SAlex Vesker /* new entry -> new branch */ 75241d07074SAlex Vesker list_add_tail(&ste->miss_list_node, miss_list); 75341d07074SAlex Vesker 7546b93b400SYevgeny Kliteynik mlx5dr_ste_set_miss_addr(dmn->ste_ctx, hw_ste, 7556b93b400SYevgeny Kliteynik nic_matcher->e_anchor->chunk->icm_addr); 75641d07074SAlex Vesker 75741d07074SAlex Vesker ste->ste_chain_location = ste_location; 75841d07074SAlex Vesker 75941d07074SAlex Vesker ste_info = kzalloc(sizeof(*ste_info), GFP_KERNEL); 76041d07074SAlex Vesker if (!ste_info) 76141d07074SAlex Vesker goto clean_ste_setting; 76241d07074SAlex Vesker 76341d07074SAlex Vesker if (mlx5dr_ste_create_next_htbl(matcher, 76441d07074SAlex Vesker nic_matcher, 76541d07074SAlex Vesker ste, 76641d07074SAlex Vesker hw_ste, 76741d07074SAlex Vesker DR_CHUNK_SIZE_1)) { 7686b93b400SYevgeny Kliteynik mlx5dr_dbg(dmn, "Failed allocating table\n"); 76941d07074SAlex Vesker goto clean_ste_info; 77041d07074SAlex Vesker } 77141d07074SAlex Vesker 77241d07074SAlex Vesker cur_htbl->ctrl.num_of_valid_entries++; 77341d07074SAlex Vesker 77441d07074SAlex Vesker mlx5dr_send_fill_and_append_ste_send_info(ste, DR_STE_SIZE, 0, hw_ste, 77541d07074SAlex Vesker ste_info, send_list, false); 77641d07074SAlex Vesker 77741d07074SAlex Vesker return 0; 77841d07074SAlex Vesker 77941d07074SAlex Vesker clean_ste_info: 78041d07074SAlex Vesker kfree(ste_info); 78141d07074SAlex Vesker clean_ste_setting: 78241d07074SAlex Vesker list_del_init(&ste->miss_list_node); 78341d07074SAlex Vesker mlx5dr_htbl_put(cur_htbl); 78441d07074SAlex Vesker 78541d07074SAlex Vesker return -ENOMEM; 78641d07074SAlex Vesker } 78741d07074SAlex Vesker 78841d07074SAlex Vesker static struct mlx5dr_ste * 78941d07074SAlex Vesker dr_rule_handle_ste_branch(struct mlx5dr_rule *rule, 79041d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule, 79141d07074SAlex Vesker struct list_head *send_ste_list, 79241d07074SAlex Vesker struct mlx5dr_ste_htbl *cur_htbl, 79341d07074SAlex Vesker u8 *hw_ste, 79441d07074SAlex Vesker u8 ste_location, 79541d07074SAlex Vesker struct mlx5dr_ste_htbl **put_htbl) 79641d07074SAlex Vesker { 79741d07074SAlex Vesker struct mlx5dr_matcher *matcher = rule->matcher; 79841d07074SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 79941d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher; 80041d07074SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn; 80141d07074SAlex Vesker struct mlx5dr_ste_htbl *new_htbl; 80241d07074SAlex Vesker struct mlx5dr_ste *matched_ste; 80341d07074SAlex Vesker struct list_head *miss_list; 80441d07074SAlex Vesker bool skip_rehash = false; 80541d07074SAlex Vesker struct mlx5dr_ste *ste; 80641d07074SAlex Vesker int index; 80741d07074SAlex Vesker 80841d07074SAlex Vesker nic_matcher = nic_rule->nic_matcher; 80941d07074SAlex Vesker nic_dmn = nic_matcher->nic_tbl->nic_dmn; 81041d07074SAlex Vesker 81141d07074SAlex Vesker again: 81241d07074SAlex Vesker index = mlx5dr_ste_calc_hash_index(hw_ste, cur_htbl); 81341d07074SAlex Vesker miss_list = &cur_htbl->chunk->miss_list[index]; 81441d07074SAlex Vesker ste = &cur_htbl->ste_arr[index]; 81541d07074SAlex Vesker 81697ffd895SYevgeny Kliteynik if (mlx5dr_ste_is_not_used(ste)) { 81741d07074SAlex Vesker if (dr_rule_handle_empty_entry(matcher, nic_matcher, cur_htbl, 81841d07074SAlex Vesker ste, ste_location, 81941d07074SAlex Vesker hw_ste, miss_list, 82041d07074SAlex Vesker send_ste_list)) 82141d07074SAlex Vesker return NULL; 82241d07074SAlex Vesker } else { 82341d07074SAlex Vesker /* Hash table index in use, check if this ste is in the miss list */ 82441d07074SAlex Vesker matched_ste = dr_rule_find_ste_in_miss_list(miss_list, hw_ste); 82541d07074SAlex Vesker if (matched_ste) { 82641d07074SAlex Vesker /* If it is last STE in the chain, and has the same tag 82741d07074SAlex Vesker * it means that all the previous stes are the same, 82841d07074SAlex Vesker * if so, this rule is duplicated. 82941d07074SAlex Vesker */ 83000414126SAlex Vesker if (!mlx5dr_ste_is_last_in_rule(nic_matcher, ste_location)) 83141d07074SAlex Vesker return matched_ste; 83200414126SAlex Vesker 83300414126SAlex Vesker mlx5dr_dbg(dmn, "Duplicate rule inserted\n"); 83441d07074SAlex Vesker } 83541d07074SAlex Vesker 83641d07074SAlex Vesker if (!skip_rehash && dr_rule_need_enlarge_hash(cur_htbl, dmn, nic_dmn)) { 83741d07074SAlex Vesker /* Hash table index in use, try to resize of the hash */ 83841d07074SAlex Vesker skip_rehash = true; 83941d07074SAlex Vesker 84041d07074SAlex Vesker /* Hold the table till we update. 84141d07074SAlex Vesker * Release in dr_rule_create_rule() 84241d07074SAlex Vesker */ 84341d07074SAlex Vesker *put_htbl = cur_htbl; 84441d07074SAlex Vesker mlx5dr_htbl_get(cur_htbl); 84541d07074SAlex Vesker 84641d07074SAlex Vesker new_htbl = dr_rule_rehash(rule, nic_rule, cur_htbl, 84741d07074SAlex Vesker ste_location, send_ste_list); 84841d07074SAlex Vesker if (!new_htbl) { 849b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed creating rehash table, htbl-log_size: %d\n", 85041d07074SAlex Vesker cur_htbl->chunk_size); 851*6cc64770SWentao_Liang mlx5dr_htbl_put(cur_htbl); 85241d07074SAlex Vesker } else { 85341d07074SAlex Vesker cur_htbl = new_htbl; 85441d07074SAlex Vesker } 85541d07074SAlex Vesker goto again; 85641d07074SAlex Vesker } else { 85741d07074SAlex Vesker /* Hash table index in use, add another collision (miss) */ 85841d07074SAlex Vesker ste = dr_rule_handle_collision(matcher, 85941d07074SAlex Vesker nic_matcher, 86041d07074SAlex Vesker ste, 86141d07074SAlex Vesker hw_ste, 86241d07074SAlex Vesker miss_list, 86341d07074SAlex Vesker send_ste_list); 86441d07074SAlex Vesker if (!ste) { 86541d07074SAlex Vesker mlx5dr_dbg(dmn, "failed adding collision entry, index: %d\n", 86641d07074SAlex Vesker index); 86741d07074SAlex Vesker return NULL; 86841d07074SAlex Vesker } 86941d07074SAlex Vesker } 87041d07074SAlex Vesker } 87141d07074SAlex Vesker return ste; 87241d07074SAlex Vesker } 87341d07074SAlex Vesker 87441d07074SAlex Vesker static bool dr_rule_cmp_value_to_mask(u8 *mask, u8 *value, 87541d07074SAlex Vesker u32 s_idx, u32 e_idx) 87641d07074SAlex Vesker { 87741d07074SAlex Vesker u32 i; 87841d07074SAlex Vesker 87941d07074SAlex Vesker for (i = s_idx; i < e_idx; i++) { 88041d07074SAlex Vesker if (value[i] & ~mask[i]) { 88141d07074SAlex Vesker pr_info("Rule parameters contains a value not specified by mask\n"); 88241d07074SAlex Vesker return false; 88341d07074SAlex Vesker } 88441d07074SAlex Vesker } 88541d07074SAlex Vesker return true; 88641d07074SAlex Vesker } 88741d07074SAlex Vesker 88841d07074SAlex Vesker static bool dr_rule_verify(struct mlx5dr_matcher *matcher, 88941d07074SAlex Vesker struct mlx5dr_match_parameters *value, 89041d07074SAlex Vesker struct mlx5dr_match_param *param) 89141d07074SAlex Vesker { 89241d07074SAlex Vesker u8 match_criteria = matcher->match_criteria; 89341d07074SAlex Vesker size_t value_size = value->match_sz; 89441d07074SAlex Vesker u8 *mask_p = (u8 *)&matcher->mask; 89541d07074SAlex Vesker u8 *param_p = (u8 *)param; 89641d07074SAlex Vesker u32 s_idx, e_idx; 89741d07074SAlex Vesker 89841d07074SAlex Vesker if (!value_size || 899699d531fSMuhammad Sammar (value_size > DR_SZ_MATCH_PARAM || (value_size % sizeof(u32)))) { 900b7d0db55SErez Shitrit mlx5dr_err(matcher->tbl->dmn, "Rule parameters length is incorrect\n"); 90141d07074SAlex Vesker return false; 90241d07074SAlex Vesker } 90341d07074SAlex Vesker 90441d07074SAlex Vesker mlx5dr_ste_copy_param(matcher->match_criteria, param, value); 90541d07074SAlex Vesker 90641d07074SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_OUTER) { 90741d07074SAlex Vesker s_idx = offsetof(struct mlx5dr_match_param, outer); 90841d07074SAlex Vesker e_idx = min(s_idx + sizeof(param->outer), value_size); 90941d07074SAlex Vesker 91041d07074SAlex Vesker if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) { 911b7d0db55SErez Shitrit mlx5dr_err(matcher->tbl->dmn, "Rule outer parameters contains a value not specified by mask\n"); 91241d07074SAlex Vesker return false; 91341d07074SAlex Vesker } 91441d07074SAlex Vesker } 91541d07074SAlex Vesker 91641d07074SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_MISC) { 91741d07074SAlex Vesker s_idx = offsetof(struct mlx5dr_match_param, misc); 91841d07074SAlex Vesker e_idx = min(s_idx + sizeof(param->misc), value_size); 91941d07074SAlex Vesker 92041d07074SAlex Vesker if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) { 921b7d0db55SErez Shitrit mlx5dr_err(matcher->tbl->dmn, "Rule misc parameters contains a value not specified by mask\n"); 92241d07074SAlex Vesker return false; 92341d07074SAlex Vesker } 92441d07074SAlex Vesker } 92541d07074SAlex Vesker 92641d07074SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_INNER) { 92741d07074SAlex Vesker s_idx = offsetof(struct mlx5dr_match_param, inner); 92841d07074SAlex Vesker e_idx = min(s_idx + sizeof(param->inner), value_size); 92941d07074SAlex Vesker 93041d07074SAlex Vesker if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) { 931b7d0db55SErez Shitrit mlx5dr_err(matcher->tbl->dmn, "Rule inner parameters contains a value not specified by mask\n"); 93241d07074SAlex Vesker return false; 93341d07074SAlex Vesker } 93441d07074SAlex Vesker } 93541d07074SAlex Vesker 93641d07074SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_MISC2) { 93741d07074SAlex Vesker s_idx = offsetof(struct mlx5dr_match_param, misc2); 93841d07074SAlex Vesker e_idx = min(s_idx + sizeof(param->misc2), value_size); 93941d07074SAlex Vesker 94041d07074SAlex Vesker if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) { 941b7d0db55SErez Shitrit mlx5dr_err(matcher->tbl->dmn, "Rule misc2 parameters contains a value not specified by mask\n"); 94241d07074SAlex Vesker return false; 94341d07074SAlex Vesker } 94441d07074SAlex Vesker } 94541d07074SAlex Vesker 94641d07074SAlex Vesker if (match_criteria & DR_MATCHER_CRITERIA_MISC3) { 94741d07074SAlex Vesker s_idx = offsetof(struct mlx5dr_match_param, misc3); 94841d07074SAlex Vesker e_idx = min(s_idx + sizeof(param->misc3), value_size); 94941d07074SAlex Vesker 95041d07074SAlex Vesker if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) { 951b7d0db55SErez Shitrit mlx5dr_err(matcher->tbl->dmn, "Rule misc3 parameters contains a value not specified by mask\n"); 95241d07074SAlex Vesker return false; 95341d07074SAlex Vesker } 95441d07074SAlex Vesker } 955160e9cb3SYevgeny Kliteynik 956160e9cb3SYevgeny Kliteynik if (match_criteria & DR_MATCHER_CRITERIA_MISC4) { 957160e9cb3SYevgeny Kliteynik s_idx = offsetof(struct mlx5dr_match_param, misc4); 958160e9cb3SYevgeny Kliteynik e_idx = min(s_idx + sizeof(param->misc4), value_size); 959160e9cb3SYevgeny Kliteynik 960160e9cb3SYevgeny Kliteynik if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) { 961160e9cb3SYevgeny Kliteynik mlx5dr_err(matcher->tbl->dmn, 962160e9cb3SYevgeny Kliteynik "Rule misc4 parameters contains a value not specified by mask\n"); 963160e9cb3SYevgeny Kliteynik return false; 964160e9cb3SYevgeny Kliteynik } 965160e9cb3SYevgeny Kliteynik } 96641d07074SAlex Vesker return true; 96741d07074SAlex Vesker } 96841d07074SAlex Vesker 96941d07074SAlex Vesker static int dr_rule_destroy_rule_nic(struct mlx5dr_rule *rule, 97041d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule) 97141d07074SAlex Vesker { 972ed03a418SAlex Vesker mlx5dr_domain_nic_lock(nic_rule->nic_matcher->nic_tbl->nic_dmn); 97341d07074SAlex Vesker dr_rule_clean_rule_members(rule, nic_rule); 974ed03a418SAlex Vesker mlx5dr_domain_nic_unlock(nic_rule->nic_matcher->nic_tbl->nic_dmn); 975ed03a418SAlex Vesker 97641d07074SAlex Vesker return 0; 97741d07074SAlex Vesker } 97841d07074SAlex Vesker 97941d07074SAlex Vesker static int dr_rule_destroy_rule_fdb(struct mlx5dr_rule *rule) 98041d07074SAlex Vesker { 98141d07074SAlex Vesker dr_rule_destroy_rule_nic(rule, &rule->rx); 98241d07074SAlex Vesker dr_rule_destroy_rule_nic(rule, &rule->tx); 98341d07074SAlex Vesker return 0; 98441d07074SAlex Vesker } 98541d07074SAlex Vesker 98641d07074SAlex Vesker static int dr_rule_destroy_rule(struct mlx5dr_rule *rule) 98741d07074SAlex Vesker { 98841d07074SAlex Vesker struct mlx5dr_domain *dmn = rule->matcher->tbl->dmn; 98941d07074SAlex Vesker 99041d07074SAlex Vesker switch (dmn->type) { 99141d07074SAlex Vesker case MLX5DR_DOMAIN_TYPE_NIC_RX: 99241d07074SAlex Vesker dr_rule_destroy_rule_nic(rule, &rule->rx); 99341d07074SAlex Vesker break; 99441d07074SAlex Vesker case MLX5DR_DOMAIN_TYPE_NIC_TX: 99541d07074SAlex Vesker dr_rule_destroy_rule_nic(rule, &rule->tx); 99641d07074SAlex Vesker break; 99741d07074SAlex Vesker case MLX5DR_DOMAIN_TYPE_FDB: 99841d07074SAlex Vesker dr_rule_destroy_rule_fdb(rule); 99941d07074SAlex Vesker break; 100041d07074SAlex Vesker default: 100141d07074SAlex Vesker return -EINVAL; 100241d07074SAlex Vesker } 100341d07074SAlex Vesker 100441d07074SAlex Vesker dr_rule_remove_action_members(rule); 100541d07074SAlex Vesker kfree(rule); 100641d07074SAlex Vesker return 0; 100741d07074SAlex Vesker } 100841d07074SAlex Vesker 1009667f2646SAlex Vesker static enum mlx5dr_ipv dr_rule_get_ipv(struct mlx5dr_match_spec *spec) 101041d07074SAlex Vesker { 1011667f2646SAlex Vesker if (spec->ip_version == 6 || spec->ethertype == ETH_P_IPV6) 1012667f2646SAlex Vesker return DR_RULE_IPV6; 1013667f2646SAlex Vesker 1014667f2646SAlex Vesker return DR_RULE_IPV4; 101541d07074SAlex Vesker } 101641d07074SAlex Vesker 101741d07074SAlex Vesker static bool dr_rule_skip(enum mlx5dr_domain_type domain, 101841d07074SAlex Vesker enum mlx5dr_ste_entry_type ste_type, 101941d07074SAlex Vesker struct mlx5dr_match_param *mask, 102001723919SHamdan Igbaria struct mlx5dr_match_param *value, 102101723919SHamdan Igbaria u32 flow_source) 102241d07074SAlex Vesker { 102301723919SHamdan Igbaria bool rx = ste_type == MLX5DR_STE_TYPE_RX; 102401723919SHamdan Igbaria 102541d07074SAlex Vesker if (domain != MLX5DR_DOMAIN_TYPE_FDB) 102641d07074SAlex Vesker return false; 102741d07074SAlex Vesker 102841d07074SAlex Vesker if (mask->misc.source_port) { 102901723919SHamdan Igbaria if (rx && value->misc.source_port != WIRE_PORT) 103041d07074SAlex Vesker return true; 103141d07074SAlex Vesker 103201723919SHamdan Igbaria if (!rx && value->misc.source_port == WIRE_PORT) 103341d07074SAlex Vesker return true; 103441d07074SAlex Vesker } 103541d07074SAlex Vesker 103601723919SHamdan Igbaria if (rx && flow_source == MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT) 103741d07074SAlex Vesker return true; 103841d07074SAlex Vesker 103901723919SHamdan Igbaria if (!rx && flow_source == MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK) 104041d07074SAlex Vesker return true; 104101723919SHamdan Igbaria 104241d07074SAlex Vesker return false; 104341d07074SAlex Vesker } 104441d07074SAlex Vesker 104541d07074SAlex Vesker static int 104641d07074SAlex Vesker dr_rule_create_rule_nic(struct mlx5dr_rule *rule, 104741d07074SAlex Vesker struct mlx5dr_rule_rx_tx *nic_rule, 104841d07074SAlex Vesker struct mlx5dr_match_param *param, 104941d07074SAlex Vesker size_t num_actions, 105041d07074SAlex Vesker struct mlx5dr_action *actions[]) 105141d07074SAlex Vesker { 105241d07074SAlex Vesker struct mlx5dr_ste_send_info *ste_info, *tmp_ste_info; 105341d07074SAlex Vesker struct mlx5dr_matcher *matcher = rule->matcher; 105441d07074SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 105541d07074SAlex Vesker struct mlx5dr_matcher_rx_tx *nic_matcher; 105641d07074SAlex Vesker struct mlx5dr_domain_rx_tx *nic_dmn; 105741d07074SAlex Vesker struct mlx5dr_ste_htbl *htbl = NULL; 105841d07074SAlex Vesker struct mlx5dr_ste_htbl *cur_htbl; 105941d07074SAlex Vesker struct mlx5dr_ste *ste = NULL; 106041d07074SAlex Vesker LIST_HEAD(send_ste_list); 106141d07074SAlex Vesker u8 *hw_ste_arr = NULL; 106241d07074SAlex Vesker u32 new_hw_ste_arr_sz; 106341d07074SAlex Vesker int ret, i; 106441d07074SAlex Vesker 106541d07074SAlex Vesker nic_matcher = nic_rule->nic_matcher; 106641d07074SAlex Vesker nic_dmn = nic_matcher->nic_tbl->nic_dmn; 106741d07074SAlex Vesker 106841d07074SAlex Vesker INIT_LIST_HEAD(&nic_rule->rule_members_list); 106941d07074SAlex Vesker 107001723919SHamdan Igbaria if (dr_rule_skip(dmn->type, nic_dmn->ste_type, &matcher->mask, param, 107101723919SHamdan Igbaria rule->flow_source)) 107241d07074SAlex Vesker return 0; 107341d07074SAlex Vesker 1074ed03a418SAlex Vesker hw_ste_arr = kzalloc(DR_RULE_MAX_STE_CHAIN * DR_STE_SIZE, GFP_KERNEL); 1075ed03a418SAlex Vesker if (!hw_ste_arr) 1076ed03a418SAlex Vesker return -ENOMEM; 1077ed03a418SAlex Vesker 1078ed03a418SAlex Vesker mlx5dr_domain_nic_lock(nic_dmn); 1079ed03a418SAlex Vesker 108041d07074SAlex Vesker ret = mlx5dr_matcher_select_builders(matcher, 108141d07074SAlex Vesker nic_matcher, 1082667f2646SAlex Vesker dr_rule_get_ipv(¶m->outer), 1083667f2646SAlex Vesker dr_rule_get_ipv(¶m->inner)); 108441d07074SAlex Vesker if (ret) 1085ed03a418SAlex Vesker goto free_hw_ste; 108641d07074SAlex Vesker 108741d07074SAlex Vesker /* Set the tag values inside the ste array */ 108841d07074SAlex Vesker ret = mlx5dr_ste_build_ste_arr(matcher, nic_matcher, param, hw_ste_arr); 108941d07074SAlex Vesker if (ret) 109041d07074SAlex Vesker goto free_hw_ste; 109141d07074SAlex Vesker 109241d07074SAlex Vesker /* Set the actions values/addresses inside the ste array */ 109341d07074SAlex Vesker ret = mlx5dr_actions_build_ste_arr(matcher, nic_matcher, actions, 109441d07074SAlex Vesker num_actions, hw_ste_arr, 109541d07074SAlex Vesker &new_hw_ste_arr_sz); 109641d07074SAlex Vesker if (ret) 109741d07074SAlex Vesker goto free_hw_ste; 109841d07074SAlex Vesker 109941d07074SAlex Vesker cur_htbl = nic_matcher->s_htbl; 110041d07074SAlex Vesker 110141d07074SAlex Vesker /* Go over the array of STEs, and build dr_ste accordingly. 110241d07074SAlex Vesker * The loop is over only the builders which are equal or less to the 110341d07074SAlex Vesker * number of stes, in case we have actions that lives in other stes. 110441d07074SAlex Vesker */ 110541d07074SAlex Vesker for (i = 0; i < nic_matcher->num_of_builders; i++) { 110641d07074SAlex Vesker /* Calculate CRC and keep new ste entry */ 110741d07074SAlex Vesker u8 *cur_hw_ste_ent = hw_ste_arr + (i * DR_STE_SIZE); 110841d07074SAlex Vesker 110941d07074SAlex Vesker ste = dr_rule_handle_ste_branch(rule, 111041d07074SAlex Vesker nic_rule, 111141d07074SAlex Vesker &send_ste_list, 111241d07074SAlex Vesker cur_htbl, 111341d07074SAlex Vesker cur_hw_ste_ent, 111441d07074SAlex Vesker i + 1, 111541d07074SAlex Vesker &htbl); 111641d07074SAlex Vesker if (!ste) { 111741d07074SAlex Vesker mlx5dr_err(dmn, "Failed creating next branch\n"); 111841d07074SAlex Vesker ret = -ENOENT; 111941d07074SAlex Vesker goto free_rule; 112041d07074SAlex Vesker } 112141d07074SAlex Vesker 112241d07074SAlex Vesker cur_htbl = ste->next_htbl; 112341d07074SAlex Vesker 112441d07074SAlex Vesker /* Keep all STEs in the rule struct */ 112541d07074SAlex Vesker ret = dr_rule_add_member(nic_rule, ste); 112641d07074SAlex Vesker if (ret) { 112741d07074SAlex Vesker mlx5dr_dbg(dmn, "Failed adding rule member index %d\n", i); 112841d07074SAlex Vesker goto free_ste; 112941d07074SAlex Vesker } 113041d07074SAlex Vesker 113141d07074SAlex Vesker mlx5dr_ste_get(ste); 113241d07074SAlex Vesker } 113341d07074SAlex Vesker 113441d07074SAlex Vesker /* Connect actions */ 113541d07074SAlex Vesker ret = dr_rule_handle_action_stes(rule, nic_rule, &send_ste_list, 113641d07074SAlex Vesker ste, hw_ste_arr, new_hw_ste_arr_sz); 113741d07074SAlex Vesker if (ret) { 113841d07074SAlex Vesker mlx5dr_dbg(dmn, "Failed apply actions\n"); 113941d07074SAlex Vesker goto free_rule; 114041d07074SAlex Vesker } 114141d07074SAlex Vesker ret = dr_rule_send_update_list(&send_ste_list, dmn, true); 114241d07074SAlex Vesker if (ret) { 114341d07074SAlex Vesker mlx5dr_err(dmn, "Failed sending ste!\n"); 114441d07074SAlex Vesker goto free_rule; 114541d07074SAlex Vesker } 114641d07074SAlex Vesker 114741d07074SAlex Vesker if (htbl) 114841d07074SAlex Vesker mlx5dr_htbl_put(htbl); 114941d07074SAlex Vesker 1150ed03a418SAlex Vesker mlx5dr_domain_nic_unlock(nic_dmn); 1151ed03a418SAlex Vesker 1152260986fcSAlex Vesker kfree(hw_ste_arr); 1153260986fcSAlex Vesker 115441d07074SAlex Vesker return 0; 115541d07074SAlex Vesker 115641d07074SAlex Vesker free_ste: 115741d07074SAlex Vesker mlx5dr_ste_put(ste, matcher, nic_matcher); 115841d07074SAlex Vesker free_rule: 115941d07074SAlex Vesker dr_rule_clean_rule_members(rule, nic_rule); 116041d07074SAlex Vesker /* Clean all ste_info's */ 116141d07074SAlex Vesker list_for_each_entry_safe(ste_info, tmp_ste_info, &send_ste_list, send_list) { 116241d07074SAlex Vesker list_del(&ste_info->send_list); 116341d07074SAlex Vesker kfree(ste_info); 116441d07074SAlex Vesker } 116541d07074SAlex Vesker free_hw_ste: 1166ed03a418SAlex Vesker mlx5dr_domain_nic_unlock(nic_dmn); 116741d07074SAlex Vesker kfree(hw_ste_arr); 116841d07074SAlex Vesker return ret; 116941d07074SAlex Vesker } 117041d07074SAlex Vesker 117141d07074SAlex Vesker static int 117241d07074SAlex Vesker dr_rule_create_rule_fdb(struct mlx5dr_rule *rule, 117341d07074SAlex Vesker struct mlx5dr_match_param *param, 117441d07074SAlex Vesker size_t num_actions, 117541d07074SAlex Vesker struct mlx5dr_action *actions[]) 117641d07074SAlex Vesker { 117741d07074SAlex Vesker struct mlx5dr_match_param copy_param = {}; 117841d07074SAlex Vesker int ret; 117941d07074SAlex Vesker 118041d07074SAlex Vesker /* Copy match_param since they will be consumed during the first 118141d07074SAlex Vesker * nic_rule insertion. 118241d07074SAlex Vesker */ 118341d07074SAlex Vesker memcpy(©_param, param, sizeof(struct mlx5dr_match_param)); 118441d07074SAlex Vesker 118541d07074SAlex Vesker ret = dr_rule_create_rule_nic(rule, &rule->rx, param, 118641d07074SAlex Vesker num_actions, actions); 118741d07074SAlex Vesker if (ret) 118841d07074SAlex Vesker return ret; 118941d07074SAlex Vesker 119041d07074SAlex Vesker ret = dr_rule_create_rule_nic(rule, &rule->tx, ©_param, 119141d07074SAlex Vesker num_actions, actions); 119241d07074SAlex Vesker if (ret) 119341d07074SAlex Vesker goto destroy_rule_nic_rx; 119441d07074SAlex Vesker 119541d07074SAlex Vesker return 0; 119641d07074SAlex Vesker 119741d07074SAlex Vesker destroy_rule_nic_rx: 119841d07074SAlex Vesker dr_rule_destroy_rule_nic(rule, &rule->rx); 119941d07074SAlex Vesker return ret; 120041d07074SAlex Vesker } 120141d07074SAlex Vesker 120241d07074SAlex Vesker static struct mlx5dr_rule * 120341d07074SAlex Vesker dr_rule_create_rule(struct mlx5dr_matcher *matcher, 120441d07074SAlex Vesker struct mlx5dr_match_parameters *value, 120541d07074SAlex Vesker size_t num_actions, 120601723919SHamdan Igbaria struct mlx5dr_action *actions[], 120701723919SHamdan Igbaria u32 flow_source) 120841d07074SAlex Vesker { 120941d07074SAlex Vesker struct mlx5dr_domain *dmn = matcher->tbl->dmn; 121041d07074SAlex Vesker struct mlx5dr_match_param param = {}; 121141d07074SAlex Vesker struct mlx5dr_rule *rule; 121241d07074SAlex Vesker int ret; 121341d07074SAlex Vesker 121441d07074SAlex Vesker if (!dr_rule_verify(matcher, value, ¶m)) 121541d07074SAlex Vesker return NULL; 121641d07074SAlex Vesker 121741d07074SAlex Vesker rule = kzalloc(sizeof(*rule), GFP_KERNEL); 121841d07074SAlex Vesker if (!rule) 121941d07074SAlex Vesker return NULL; 122041d07074SAlex Vesker 122141d07074SAlex Vesker rule->matcher = matcher; 122201723919SHamdan Igbaria rule->flow_source = flow_source; 122341d07074SAlex Vesker INIT_LIST_HEAD(&rule->rule_actions_list); 122441d07074SAlex Vesker 122541d07074SAlex Vesker ret = dr_rule_add_action_members(rule, num_actions, actions); 122641d07074SAlex Vesker if (ret) 122741d07074SAlex Vesker goto free_rule; 122841d07074SAlex Vesker 122941d07074SAlex Vesker switch (dmn->type) { 123041d07074SAlex Vesker case MLX5DR_DOMAIN_TYPE_NIC_RX: 123141d07074SAlex Vesker rule->rx.nic_matcher = &matcher->rx; 123241d07074SAlex Vesker ret = dr_rule_create_rule_nic(rule, &rule->rx, ¶m, 123341d07074SAlex Vesker num_actions, actions); 123441d07074SAlex Vesker break; 123541d07074SAlex Vesker case MLX5DR_DOMAIN_TYPE_NIC_TX: 123641d07074SAlex Vesker rule->tx.nic_matcher = &matcher->tx; 123741d07074SAlex Vesker ret = dr_rule_create_rule_nic(rule, &rule->tx, ¶m, 123841d07074SAlex Vesker num_actions, actions); 123941d07074SAlex Vesker break; 124041d07074SAlex Vesker case MLX5DR_DOMAIN_TYPE_FDB: 124141d07074SAlex Vesker rule->rx.nic_matcher = &matcher->rx; 124241d07074SAlex Vesker rule->tx.nic_matcher = &matcher->tx; 124341d07074SAlex Vesker ret = dr_rule_create_rule_fdb(rule, ¶m, 124441d07074SAlex Vesker num_actions, actions); 124541d07074SAlex Vesker break; 124641d07074SAlex Vesker default: 124741d07074SAlex Vesker ret = -EINVAL; 124841d07074SAlex Vesker break; 124941d07074SAlex Vesker } 125041d07074SAlex Vesker 125141d07074SAlex Vesker if (ret) 125241d07074SAlex Vesker goto remove_action_members; 125341d07074SAlex Vesker 125441d07074SAlex Vesker return rule; 125541d07074SAlex Vesker 125641d07074SAlex Vesker remove_action_members: 125741d07074SAlex Vesker dr_rule_remove_action_members(rule); 125841d07074SAlex Vesker free_rule: 125941d07074SAlex Vesker kfree(rule); 1260b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed creating rule\n"); 126141d07074SAlex Vesker return NULL; 126241d07074SAlex Vesker } 126341d07074SAlex Vesker 126441d07074SAlex Vesker struct mlx5dr_rule *mlx5dr_rule_create(struct mlx5dr_matcher *matcher, 126541d07074SAlex Vesker struct mlx5dr_match_parameters *value, 126641d07074SAlex Vesker size_t num_actions, 126701723919SHamdan Igbaria struct mlx5dr_action *actions[], 126801723919SHamdan Igbaria u32 flow_source) 126941d07074SAlex Vesker { 127041d07074SAlex Vesker struct mlx5dr_rule *rule; 127141d07074SAlex Vesker 127241d07074SAlex Vesker refcount_inc(&matcher->refcount); 127341d07074SAlex Vesker 127401723919SHamdan Igbaria rule = dr_rule_create_rule(matcher, value, num_actions, actions, flow_source); 127541d07074SAlex Vesker if (!rule) 127641d07074SAlex Vesker refcount_dec(&matcher->refcount); 127741d07074SAlex Vesker 127841d07074SAlex Vesker return rule; 127941d07074SAlex Vesker } 128041d07074SAlex Vesker 128141d07074SAlex Vesker int mlx5dr_rule_destroy(struct mlx5dr_rule *rule) 128241d07074SAlex Vesker { 128341d07074SAlex Vesker struct mlx5dr_matcher *matcher = rule->matcher; 128441d07074SAlex Vesker int ret; 128541d07074SAlex Vesker 128641d07074SAlex Vesker ret = dr_rule_destroy_rule(rule); 128741d07074SAlex Vesker if (!ret) 128841d07074SAlex Vesker refcount_dec(&matcher->refcount); 1289ed03a418SAlex Vesker 129041d07074SAlex Vesker return ret; 129141d07074SAlex Vesker } 1292