1a6696735SMaxim Mikityanskiy // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2a6696735SMaxim Mikityanskiy /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3a6696735SMaxim Mikityanskiy
4a6696735SMaxim Mikityanskiy #include "tir.h"
5a6696735SMaxim Mikityanskiy #include "params.h"
6a6696735SMaxim Mikityanskiy #include <linux/mlx5/transobj.h>
7a6696735SMaxim Mikityanskiy
8a6696735SMaxim Mikityanskiy #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
9a6696735SMaxim Mikityanskiy
10a6696735SMaxim Mikityanskiy /* max() doesn't work inside square brackets. */
11a6696735SMaxim Mikityanskiy #define MLX5E_TIR_CMD_IN_SZ_DW ( \
12a6696735SMaxim Mikityanskiy MLX5_ST_SZ_DW(create_tir_in) > MLX5_ST_SZ_DW(modify_tir_in) ? \
13a6696735SMaxim Mikityanskiy MLX5_ST_SZ_DW(create_tir_in) : MLX5_ST_SZ_DW(modify_tir_in) \
14a6696735SMaxim Mikityanskiy )
15a6696735SMaxim Mikityanskiy
16a6696735SMaxim Mikityanskiy struct mlx5e_tir_builder {
17a6696735SMaxim Mikityanskiy u32 in[MLX5E_TIR_CMD_IN_SZ_DW];
18a6696735SMaxim Mikityanskiy bool modify;
19a6696735SMaxim Mikityanskiy };
20a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_alloc(bool modify)21a6696735SMaxim Mikityanskiy struct mlx5e_tir_builder *mlx5e_tir_builder_alloc(bool modify)
22a6696735SMaxim Mikityanskiy {
23a6696735SMaxim Mikityanskiy struct mlx5e_tir_builder *builder;
24a6696735SMaxim Mikityanskiy
25a6696735SMaxim Mikityanskiy builder = kvzalloc(sizeof(*builder), GFP_KERNEL);
26a6696735SMaxim Mikityanskiy builder->modify = modify;
27a6696735SMaxim Mikityanskiy
28a6696735SMaxim Mikityanskiy return builder;
29a6696735SMaxim Mikityanskiy }
30a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_free(struct mlx5e_tir_builder * builder)31a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_free(struct mlx5e_tir_builder *builder)
32a6696735SMaxim Mikityanskiy {
33a6696735SMaxim Mikityanskiy kvfree(builder);
34a6696735SMaxim Mikityanskiy }
35a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_clear(struct mlx5e_tir_builder * builder)36a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_clear(struct mlx5e_tir_builder *builder)
37a6696735SMaxim Mikityanskiy {
38a6696735SMaxim Mikityanskiy memset(builder->in, 0, sizeof(builder->in));
39a6696735SMaxim Mikityanskiy }
40a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_get_tirc(struct mlx5e_tir_builder * builder)41a6696735SMaxim Mikityanskiy static void *mlx5e_tir_builder_get_tirc(struct mlx5e_tir_builder *builder)
42a6696735SMaxim Mikityanskiy {
43a6696735SMaxim Mikityanskiy if (builder->modify)
44a6696735SMaxim Mikityanskiy return MLX5_ADDR_OF(modify_tir_in, builder->in, ctx);
45a6696735SMaxim Mikityanskiy return MLX5_ADDR_OF(create_tir_in, builder->in, ctx);
46a6696735SMaxim Mikityanskiy }
47a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder * builder,u32 tdn,u32 rqn)48a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder *builder, u32 tdn, u32 rqn)
49a6696735SMaxim Mikityanskiy {
50a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder);
51a6696735SMaxim Mikityanskiy
52a6696735SMaxim Mikityanskiy WARN_ON(builder->modify);
53a6696735SMaxim Mikityanskiy
54a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, transport_domain, tdn);
55a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
56a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_NONE);
57a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, inline_rqn, rqn);
58a6696735SMaxim Mikityanskiy }
59a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder * builder,u32 tdn,u32 rqtn,bool inner_ft_support)60a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn,
61a6696735SMaxim Mikityanskiy u32 rqtn, bool inner_ft_support)
62a6696735SMaxim Mikityanskiy {
63a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder);
64a6696735SMaxim Mikityanskiy
65a6696735SMaxim Mikityanskiy WARN_ON(builder->modify);
66a6696735SMaxim Mikityanskiy
67a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, transport_domain, tdn);
68a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
69a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, indirect_table, rqtn);
70a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, tunneled_offload_en, inner_ft_support);
71a6696735SMaxim Mikityanskiy }
72a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder * builder,const struct mlx5e_packet_merge_param * pkt_merge_param)73eaee12f0SKhalid Manaa void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder,
74eaee12f0SKhalid Manaa const struct mlx5e_packet_merge_param *pkt_merge_param)
75a6696735SMaxim Mikityanskiy {
76a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder);
77a6696735SMaxim Mikityanskiy const unsigned int rough_max_l2_l3_hdr_sz = 256;
78a6696735SMaxim Mikityanskiy
79a6696735SMaxim Mikityanskiy if (builder->modify)
80eaee12f0SKhalid Manaa MLX5_SET(modify_tir_in, builder->in, bitmask.packet_merge, 1);
81a6696735SMaxim Mikityanskiy
82*e5ca8fb0SBen Ben-Ishay switch (pkt_merge_param->type) {
83*e5ca8fb0SBen Ben-Ishay case MLX5E_PACKET_MERGE_LRO:
8450f477feSBen Ben-Ishay MLX5_SET(tirc, tirc, packet_merge_mask,
8550f477feSBen Ben-Ishay MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO |
8650f477feSBen Ben-Ishay MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO);
87a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
88a6696735SMaxim Mikityanskiy (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - rough_max_l2_l3_hdr_sz) >> 8);
89eaee12f0SKhalid Manaa MLX5_SET(tirc, tirc, lro_timeout_period_usecs, pkt_merge_param->timeout);
90*e5ca8fb0SBen Ben-Ishay break;
91*e5ca8fb0SBen Ben-Ishay default:
92*e5ca8fb0SBen Ben-Ishay break;
93*e5ca8fb0SBen Ben-Ishay }
94a6696735SMaxim Mikityanskiy }
95a6696735SMaxim Mikityanskiy
mlx5e_hfunc_to_hw(u8 hfunc)96a6696735SMaxim Mikityanskiy static int mlx5e_hfunc_to_hw(u8 hfunc)
97a6696735SMaxim Mikityanskiy {
98a6696735SMaxim Mikityanskiy switch (hfunc) {
99a6696735SMaxim Mikityanskiy case ETH_RSS_HASH_TOP:
100a6696735SMaxim Mikityanskiy return MLX5_RX_HASH_FN_TOEPLITZ;
101a6696735SMaxim Mikityanskiy case ETH_RSS_HASH_XOR:
102a6696735SMaxim Mikityanskiy return MLX5_RX_HASH_FN_INVERTED_XOR8;
103a6696735SMaxim Mikityanskiy default:
104a6696735SMaxim Mikityanskiy return MLX5_RX_HASH_FN_NONE;
105a6696735SMaxim Mikityanskiy }
106a6696735SMaxim Mikityanskiy }
107a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_build_rss(struct mlx5e_tir_builder * builder,const struct mlx5e_rss_params_hash * rss_hash,const struct mlx5e_rss_params_traffic_type * rss_tt,bool inner)108a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_rss(struct mlx5e_tir_builder *builder,
109a6696735SMaxim Mikityanskiy const struct mlx5e_rss_params_hash *rss_hash,
110a6696735SMaxim Mikityanskiy const struct mlx5e_rss_params_traffic_type *rss_tt,
111a6696735SMaxim Mikityanskiy bool inner)
112a6696735SMaxim Mikityanskiy {
113a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder);
114a6696735SMaxim Mikityanskiy void *hfso;
115a6696735SMaxim Mikityanskiy
116a6696735SMaxim Mikityanskiy if (builder->modify)
117a6696735SMaxim Mikityanskiy MLX5_SET(modify_tir_in, builder->in, bitmask.hash, 1);
118a6696735SMaxim Mikityanskiy
119a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_hfunc_to_hw(rss_hash->hfunc));
120a6696735SMaxim Mikityanskiy if (rss_hash->hfunc == ETH_RSS_HASH_TOP) {
121a6696735SMaxim Mikityanskiy const size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
122a6696735SMaxim Mikityanskiy void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
123a6696735SMaxim Mikityanskiy
124a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
125a6696735SMaxim Mikityanskiy memcpy(rss_key, rss_hash->toeplitz_hash_key, len);
126a6696735SMaxim Mikityanskiy }
127a6696735SMaxim Mikityanskiy
128a6696735SMaxim Mikityanskiy if (inner)
129a6696735SMaxim Mikityanskiy hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
130a6696735SMaxim Mikityanskiy else
131a6696735SMaxim Mikityanskiy hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
132a6696735SMaxim Mikityanskiy MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, rss_tt->l3_prot_type);
133a6696735SMaxim Mikityanskiy MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, rss_tt->l4_prot_type);
134a6696735SMaxim Mikityanskiy MLX5_SET(rx_hash_field_select, hfso, selected_fields, rss_tt->rx_hash_fields);
135a6696735SMaxim Mikityanskiy }
136a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_build_direct(struct mlx5e_tir_builder * builder)137a6696735SMaxim Mikityanskiy void mlx5e_tir_builder_build_direct(struct mlx5e_tir_builder *builder)
138a6696735SMaxim Mikityanskiy {
139a6696735SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder);
140a6696735SMaxim Mikityanskiy
141a6696735SMaxim Mikityanskiy WARN_ON(builder->modify);
142a6696735SMaxim Mikityanskiy
143a6696735SMaxim Mikityanskiy MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
144a6696735SMaxim Mikityanskiy }
145a6696735SMaxim Mikityanskiy
mlx5e_tir_builder_build_tls(struct mlx5e_tir_builder * builder)14609f83569SMaxim Mikityanskiy void mlx5e_tir_builder_build_tls(struct mlx5e_tir_builder *builder)
14709f83569SMaxim Mikityanskiy {
14809f83569SMaxim Mikityanskiy void *tirc = mlx5e_tir_builder_get_tirc(builder);
14909f83569SMaxim Mikityanskiy
15009f83569SMaxim Mikityanskiy WARN_ON(builder->modify);
15109f83569SMaxim Mikityanskiy
15209f83569SMaxim Mikityanskiy MLX5_SET(tirc, tirc, tls_en, 1);
15309f83569SMaxim Mikityanskiy MLX5_SET(tirc, tirc, self_lb_block,
15409f83569SMaxim Mikityanskiy MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST |
15509f83569SMaxim Mikityanskiy MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST);
15609f83569SMaxim Mikityanskiy }
15709f83569SMaxim Mikityanskiy
mlx5e_tir_init(struct mlx5e_tir * tir,struct mlx5e_tir_builder * builder,struct mlx5_core_dev * mdev,bool reg)158a6696735SMaxim Mikityanskiy int mlx5e_tir_init(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder,
159a6696735SMaxim Mikityanskiy struct mlx5_core_dev *mdev, bool reg)
160a6696735SMaxim Mikityanskiy {
161a6696735SMaxim Mikityanskiy int err;
162a6696735SMaxim Mikityanskiy
163a6696735SMaxim Mikityanskiy tir->mdev = mdev;
164a6696735SMaxim Mikityanskiy
165a6696735SMaxim Mikityanskiy err = mlx5_core_create_tir(tir->mdev, builder->in, &tir->tirn);
166a6696735SMaxim Mikityanskiy if (err)
167a6696735SMaxim Mikityanskiy return err;
168a6696735SMaxim Mikityanskiy
169a6696735SMaxim Mikityanskiy if (reg) {
170a6696735SMaxim Mikityanskiy struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs;
171a6696735SMaxim Mikityanskiy
172a6696735SMaxim Mikityanskiy mutex_lock(&res->td.list_lock);
173a6696735SMaxim Mikityanskiy list_add(&tir->list, &res->td.tirs_list);
174a6696735SMaxim Mikityanskiy mutex_unlock(&res->td.list_lock);
175a6696735SMaxim Mikityanskiy } else {
176a6696735SMaxim Mikityanskiy INIT_LIST_HEAD(&tir->list);
177a6696735SMaxim Mikityanskiy }
178a6696735SMaxim Mikityanskiy
179a6696735SMaxim Mikityanskiy return 0;
180a6696735SMaxim Mikityanskiy }
181a6696735SMaxim Mikityanskiy
mlx5e_tir_destroy(struct mlx5e_tir * tir)182a6696735SMaxim Mikityanskiy void mlx5e_tir_destroy(struct mlx5e_tir *tir)
183a6696735SMaxim Mikityanskiy {
184a6696735SMaxim Mikityanskiy struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs;
185a6696735SMaxim Mikityanskiy
186a6696735SMaxim Mikityanskiy /* Skip mutex if list_del is no-op (the TIR wasn't registered in the
187a6696735SMaxim Mikityanskiy * list). list_empty will never return true for an item of tirs_list,
188a6696735SMaxim Mikityanskiy * and READ_ONCE/WRITE_ONCE in list_empty/list_del guarantee consistency
189a6696735SMaxim Mikityanskiy * of the list->next value.
190a6696735SMaxim Mikityanskiy */
191a6696735SMaxim Mikityanskiy if (!list_empty(&tir->list)) {
192a6696735SMaxim Mikityanskiy mutex_lock(&res->td.list_lock);
193a6696735SMaxim Mikityanskiy list_del(&tir->list);
194a6696735SMaxim Mikityanskiy mutex_unlock(&res->td.list_lock);
195a6696735SMaxim Mikityanskiy }
196a6696735SMaxim Mikityanskiy
197a6696735SMaxim Mikityanskiy mlx5_core_destroy_tir(tir->mdev, tir->tirn);
198a6696735SMaxim Mikityanskiy }
199a6696735SMaxim Mikityanskiy
mlx5e_tir_modify(struct mlx5e_tir * tir,struct mlx5e_tir_builder * builder)200a6696735SMaxim Mikityanskiy int mlx5e_tir_modify(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder)
201a6696735SMaxim Mikityanskiy {
202a6696735SMaxim Mikityanskiy return mlx5_core_modify_tir(tir->mdev, tir->tirn, builder->in);
203a6696735SMaxim Mikityanskiy }
204