1a6696735SMaxim Mikityanskiy // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2a6696735SMaxim Mikityanskiy /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3a6696735SMaxim Mikityanskiy 4a6696735SMaxim Mikityanskiy #include "tir.h" 5a6696735SMaxim Mikityanskiy #include "params.h" 6a6696735SMaxim Mikityanskiy #include <linux/mlx5/transobj.h> 7a6696735SMaxim Mikityanskiy 8a6696735SMaxim Mikityanskiy #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024) 9a6696735SMaxim Mikityanskiy 10a6696735SMaxim Mikityanskiy /* max() doesn't work inside square brackets. */ 11a6696735SMaxim Mikityanskiy #define MLX5E_TIR_CMD_IN_SZ_DW ( \ 12a6696735SMaxim Mikityanskiy MLX5_ST_SZ_DW(create_tir_in) > MLX5_ST_SZ_DW(modify_tir_in) ? \ 13a6696735SMaxim Mikityanskiy MLX5_ST_SZ_DW(create_tir_in) : MLX5_ST_SZ_DW(modify_tir_in) \ 14a6696735SMaxim Mikityanskiy ) 15a6696735SMaxim Mikityanskiy 16a6696735SMaxim Mikityanskiy struct mlx5e_tir_builder { 17a6696735SMaxim Mikityanskiy u32 in[MLX5E_TIR_CMD_IN_SZ_DW]; 18a6696735SMaxim Mikityanskiy bool modify; 19a6696735SMaxim Mikityanskiy }; 20a6696735SMaxim Mikityanskiy 21a6696735SMaxim Mikityanskiy struct mlx5e_tir_builder *mlx5e_tir_builder_alloc(bool modify) 22a6696735SMaxim Mikityanskiy { 23a6696735SMaxim Mikityanskiy struct mlx5e_tir_builder *builder; 24a6696735SMaxim Mikityanskiy 25a6696735SMaxim Mikityanskiy builder = kvzalloc(sizeof(*builder), GFP_KERNEL); 26a6696735SMaxim Mikityanskiy builder->modify = modify; 27a6696735SMaxim Mikityanskiy 28a6696735SMaxim Mikityanskiy return builder; 29a6696735SMaxim Mikityanskiy } 30a6696735SMaxim Mikityanskiy 31a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_free(struct mlx5e_tir_builder *builder) 32a6696735SMaxim Mikityanskiy { 33a6696735SMaxim Mikityanskiy kvfree(builder); 34a6696735SMaxim Mikityanskiy } 35a6696735SMaxim Mikityanskiy 36a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_clear(struct mlx5e_tir_builder *builder) 37a6696735SMaxim Mikityanskiy { 38a6696735SMaxim Mikityanskiy memset(builder->in, 0, sizeof(builder->in)); 39a6696735SMaxim Mikityanskiy } 40a6696735SMaxim Mikityanskiy 41a6696735SMaxim Mikityanskiy static void *mlx5e_tir_builder_get_tirc(struct mlx5e_tir_builder *builder) 42a6696735SMaxim Mikityanskiy { 43a6696735SMaxim Mikityanskiy if (builder->modify) 44a6696735SMaxim Mikityanskiy return MLX5_ADDR_OF(modify_tir_in, builder->in, ctx); 45a6696735SMaxim Mikityanskiy return MLX5_ADDR_OF(create_tir_in, builder->in, ctx); 46a6696735SMaxim Mikityanskiy } 47a6696735SMaxim Mikityanskiy 48a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder *builder, u32 tdn, u32 rqn) 49a6696735SMaxim Mikityanskiy { 50a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder); 51a6696735SMaxim Mikityanskiy 52a6696735SMaxim Mikityanskiy WARN_ON(builder->modify); 53a6696735SMaxim Mikityanskiy 54a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, transport_domain, tdn); 55a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); 56a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_NONE); 57a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, inline_rqn, rqn); 58a6696735SMaxim Mikityanskiy } 59a6696735SMaxim Mikityanskiy 60a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn, 61a6696735SMaxim Mikityanskiy u32 rqtn, bool inner_ft_support) 62a6696735SMaxim Mikityanskiy { 63a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder); 64a6696735SMaxim Mikityanskiy 65a6696735SMaxim Mikityanskiy WARN_ON(builder->modify); 66a6696735SMaxim Mikityanskiy 67a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, transport_domain, tdn); 68a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 69a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, indirect_table, rqtn); 70a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, tunneled_offload_en, inner_ft_support); 71a6696735SMaxim Mikityanskiy } 72a6696735SMaxim Mikityanskiy 73eaee12f0SKhalid Manaa void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder, 74eaee12f0SKhalid Manaa const struct mlx5e_packet_merge_param *pkt_merge_param) 75a6696735SMaxim Mikityanskiy { 76a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder); 77a6696735SMaxim Mikityanskiy const unsigned int rough_max_l2_l3_hdr_sz = 256; 78a6696735SMaxim Mikityanskiy 79a6696735SMaxim Mikityanskiy if (builder->modify) 80eaee12f0SKhalid Manaa MLX5_SET(modify_tir_in, builder->in, bitmask.packet_merge, 1); 81a6696735SMaxim Mikityanskiy 82*e5ca8fb0SBen Ben-Ishay switch (pkt_merge_param->type) { 83*e5ca8fb0SBen Ben-Ishay case MLX5E_PACKET_MERGE_LRO: 8450f477feSBen Ben-Ishay MLX5_SET(tirc, tirc, packet_merge_mask, 8550f477feSBen Ben-Ishay MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO | 8650f477feSBen Ben-Ishay MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO); 87a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, lro_max_ip_payload_size, 88a6696735SMaxim Mikityanskiy (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - rough_max_l2_l3_hdr_sz) >> 8); 89eaee12f0SKhalid Manaa MLX5_SET(tirc, tirc, lro_timeout_period_usecs, pkt_merge_param->timeout); 90*e5ca8fb0SBen Ben-Ishay break; 91*e5ca8fb0SBen Ben-Ishay case MLX5E_PACKET_MERGE_SHAMPO: 92*e5ca8fb0SBen Ben-Ishay MLX5_SET(tirc, tirc, packet_merge_mask, MLX5_TIRC_PACKET_MERGE_MASK_SHAMPO); 93*e5ca8fb0SBen Ben-Ishay break; 94*e5ca8fb0SBen Ben-Ishay default: 95*e5ca8fb0SBen Ben-Ishay break; 96*e5ca8fb0SBen Ben-Ishay } 97a6696735SMaxim Mikityanskiy } 98a6696735SMaxim Mikityanskiy 99a6696735SMaxim Mikityanskiy static int mlx5e_hfunc_to_hw(u8 hfunc) 100a6696735SMaxim Mikityanskiy { 101a6696735SMaxim Mikityanskiy switch (hfunc) { 102a6696735SMaxim Mikityanskiy case ETH_RSS_HASH_TOP: 103a6696735SMaxim Mikityanskiy return MLX5_RX_HASH_FN_TOEPLITZ; 104a6696735SMaxim Mikityanskiy case ETH_RSS_HASH_XOR: 105a6696735SMaxim Mikityanskiy return MLX5_RX_HASH_FN_INVERTED_XOR8; 106a6696735SMaxim Mikityanskiy default: 107a6696735SMaxim Mikityanskiy return MLX5_RX_HASH_FN_NONE; 108a6696735SMaxim Mikityanskiy } 109a6696735SMaxim Mikityanskiy } 110a6696735SMaxim Mikityanskiy 111a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_rss(struct mlx5e_tir_builder *builder, 112a6696735SMaxim Mikityanskiy const struct mlx5e_rss_params_hash *rss_hash, 113a6696735SMaxim Mikityanskiy const struct mlx5e_rss_params_traffic_type *rss_tt, 114a6696735SMaxim Mikityanskiy bool inner) 115a6696735SMaxim Mikityanskiy { 116a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder); 117a6696735SMaxim Mikityanskiy void *hfso; 118a6696735SMaxim Mikityanskiy 119a6696735SMaxim Mikityanskiy if (builder->modify) 120a6696735SMaxim Mikityanskiy MLX5_SET(modify_tir_in, builder->in, bitmask.hash, 1); 121a6696735SMaxim Mikityanskiy 122a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_hfunc_to_hw(rss_hash->hfunc)); 123a6696735SMaxim Mikityanskiy if (rss_hash->hfunc == ETH_RSS_HASH_TOP) { 124a6696735SMaxim Mikityanskiy const size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key); 125a6696735SMaxim Mikityanskiy void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 126a6696735SMaxim Mikityanskiy 127a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 128a6696735SMaxim Mikityanskiy memcpy(rss_key, rss_hash->toeplitz_hash_key, len); 129a6696735SMaxim Mikityanskiy } 130a6696735SMaxim Mikityanskiy 131a6696735SMaxim Mikityanskiy if (inner) 132a6696735SMaxim Mikityanskiy hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner); 133a6696735SMaxim Mikityanskiy else 134a6696735SMaxim Mikityanskiy hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 135a6696735SMaxim Mikityanskiy MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, rss_tt->l3_prot_type); 136a6696735SMaxim Mikityanskiy MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, rss_tt->l4_prot_type); 137a6696735SMaxim Mikityanskiy MLX5_SET(rx_hash_field_select, hfso, selected_fields, rss_tt->rx_hash_fields); 138a6696735SMaxim Mikityanskiy } 139a6696735SMaxim Mikityanskiy 140a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_direct(struct mlx5e_tir_builder *builder) 141a6696735SMaxim Mikityanskiy { 142a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder); 143a6696735SMaxim Mikityanskiy 144a6696735SMaxim Mikityanskiy WARN_ON(builder->modify); 145a6696735SMaxim Mikityanskiy 146a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); 147a6696735SMaxim Mikityanskiy } 148a6696735SMaxim Mikityanskiy 14909f83569SMaxim Mikityanskiy void mlx5e_tir_builder_build_tls(struct mlx5e_tir_builder *builder) 15009f83569SMaxim Mikityanskiy { 15109f83569SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder); 15209f83569SMaxim Mikityanskiy 15309f83569SMaxim Mikityanskiy WARN_ON(builder->modify); 15409f83569SMaxim Mikityanskiy 15509f83569SMaxim Mikityanskiy MLX5_SET(tirc, tirc, tls_en, 1); 15609f83569SMaxim Mikityanskiy MLX5_SET(tirc, tirc, self_lb_block, 15709f83569SMaxim Mikityanskiy MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST | 15809f83569SMaxim Mikityanskiy MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST); 15909f83569SMaxim Mikityanskiy } 16009f83569SMaxim Mikityanskiy 161a6696735SMaxim Mikityanskiy int mlx5e_tir_init(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder, 162a6696735SMaxim Mikityanskiy struct mlx5_core_dev *mdev, bool reg) 163a6696735SMaxim Mikityanskiy { 164a6696735SMaxim Mikityanskiy int err; 165a6696735SMaxim Mikityanskiy 166a6696735SMaxim Mikityanskiy tir->mdev = mdev; 167a6696735SMaxim Mikityanskiy 168a6696735SMaxim Mikityanskiy err = mlx5_core_create_tir(tir->mdev, builder->in, &tir->tirn); 169a6696735SMaxim Mikityanskiy if (err) 170a6696735SMaxim Mikityanskiy return err; 171a6696735SMaxim Mikityanskiy 172a6696735SMaxim Mikityanskiy if (reg) { 173a6696735SMaxim Mikityanskiy struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs; 174a6696735SMaxim Mikityanskiy 175a6696735SMaxim Mikityanskiy mutex_lock(&res->td.list_lock); 176a6696735SMaxim Mikityanskiy list_add(&tir->list, &res->td.tirs_list); 177a6696735SMaxim Mikityanskiy mutex_unlock(&res->td.list_lock); 178a6696735SMaxim Mikityanskiy } else { 179a6696735SMaxim Mikityanskiy INIT_LIST_HEAD(&tir->list); 180a6696735SMaxim Mikityanskiy } 181a6696735SMaxim Mikityanskiy 182a6696735SMaxim Mikityanskiy return 0; 183a6696735SMaxim Mikityanskiy } 184a6696735SMaxim Mikityanskiy 185a6696735SMaxim Mikityanskiy void mlx5e_tir_destroy(struct mlx5e_tir *tir) 186a6696735SMaxim Mikityanskiy { 187a6696735SMaxim Mikityanskiy struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs; 188a6696735SMaxim Mikityanskiy 189a6696735SMaxim Mikityanskiy /* Skip mutex if list_del is no-op (the TIR wasn't registered in the 190a6696735SMaxim Mikityanskiy * list). list_empty will never return true for an item of tirs_list, 191a6696735SMaxim Mikityanskiy * and READ_ONCE/WRITE_ONCE in list_empty/list_del guarantee consistency 192a6696735SMaxim Mikityanskiy * of the list->next value. 193a6696735SMaxim Mikityanskiy */ 194a6696735SMaxim Mikityanskiy if (!list_empty(&tir->list)) { 195a6696735SMaxim Mikityanskiy mutex_lock(&res->td.list_lock); 196a6696735SMaxim Mikityanskiy list_del(&tir->list); 197a6696735SMaxim Mikityanskiy mutex_unlock(&res->td.list_lock); 198a6696735SMaxim Mikityanskiy } 199a6696735SMaxim Mikityanskiy 200a6696735SMaxim Mikityanskiy mlx5_core_destroy_tir(tir->mdev, tir->tirn); 201a6696735SMaxim Mikityanskiy } 202a6696735SMaxim Mikityanskiy 203a6696735SMaxim Mikityanskiy int mlx5e_tir_modify(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder) 204a6696735SMaxim Mikityanskiy { 205a6696735SMaxim Mikityanskiy return mlx5_core_modify_tir(tir->mdev, tir->tirn, builder->in); 206a6696735SMaxim Mikityanskiy } 207