1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 4 #include <linux/netdevice.h> 5 #include "en_accel/fs_tcp.h" 6 #include "fs_core.h" 7 8 enum accel_fs_tcp_type { 9 ACCEL_FS_IPV4_TCP, 10 ACCEL_FS_IPV6_TCP, 11 ACCEL_FS_TCP_NUM_TYPES, 12 }; 13 14 struct mlx5e_accel_fs_tcp { 15 struct mlx5e_flow_table tables[ACCEL_FS_TCP_NUM_TYPES]; 16 struct mlx5_flow_handle *default_rules[ACCEL_FS_TCP_NUM_TYPES]; 17 }; 18 19 static enum mlx5e_traffic_types fs_accel2tt(enum accel_fs_tcp_type i) 20 { 21 switch (i) { 22 case ACCEL_FS_IPV4_TCP: 23 return MLX5E_TT_IPV4_TCP; 24 default: /* ACCEL_FS_IPV6_TCP */ 25 return MLX5E_TT_IPV6_TCP; 26 } 27 } 28 29 static void accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct sock *sk) 30 { 31 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 32 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 33 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 34 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 35 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 36 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 37 &inet_sk(sk)->inet_daddr, 4); 38 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 39 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 40 &inet_sk(sk)->inet_rcv_saddr, 4); 41 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 42 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 43 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 44 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 45 } 46 47 static void accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct sock *sk) 48 { 49 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 50 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 51 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 52 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 53 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 54 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 55 &sk->sk_v6_daddr, 16); 56 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 57 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 58 &inet6_sk(sk)->saddr, 16); 59 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 60 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 61 0xff, 16); 62 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 63 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 64 0xff, 16); 65 } 66 67 void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule) 68 { 69 mlx5_del_flow_rules(rule); 70 } 71 72 struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, 73 struct sock *sk, u32 tirn, 74 uint32_t flow_tag) 75 { 76 struct mlx5_flow_destination dest = {}; 77 struct mlx5e_flow_table *ft = NULL; 78 struct mlx5e_accel_fs_tcp *fs_tcp; 79 MLX5_DECLARE_FLOW_ACT(flow_act); 80 struct mlx5_flow_handle *flow; 81 struct mlx5_flow_spec *spec; 82 83 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 84 if (!spec) 85 return ERR_PTR(-ENOMEM); 86 87 fs_tcp = priv->fs.accel_tcp; 88 89 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 90 91 switch (sk->sk_family) { 92 case AF_INET: 93 accel_fs_tcp_set_ipv4_flow(spec, sk); 94 ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; 95 mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__, 96 &inet_sk(sk)->inet_rcv_saddr, 97 inet_sk(sk)->inet_sport, 98 &inet_sk(sk)->inet_daddr, 99 inet_sk(sk)->inet_dport); 100 break; 101 #if IS_ENABLED(CONFIG_IPV6) 102 case AF_INET6: 103 if (!sk->sk_ipv6only && 104 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { 105 accel_fs_tcp_set_ipv4_flow(spec, sk); 106 ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; 107 } else { 108 accel_fs_tcp_set_ipv6_flow(spec, sk); 109 ft = &fs_tcp->tables[ACCEL_FS_IPV6_TCP]; 110 } 111 break; 112 #endif 113 default: 114 break; 115 } 116 117 if (!ft) { 118 flow = ERR_PTR(-EINVAL); 119 goto out; 120 } 121 122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 123 outer_headers.tcp_dport); 124 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 125 outer_headers.tcp_sport); 126 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 127 ntohs(inet_sk(sk)->inet_sport)); 128 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 129 ntohs(inet_sk(sk)->inet_dport)); 130 131 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 132 dest.tir_num = tirn; 133 if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG) { 134 spec->flow_context.flow_tag = flow_tag; 135 spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG; 136 } 137 138 flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); 139 140 if (IS_ERR(flow)) 141 netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n", 142 PTR_ERR(flow)); 143 144 out: 145 kvfree(spec); 146 return flow; 147 } 148 149 static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, 150 enum accel_fs_tcp_type type) 151 { 152 struct mlx5e_flow_table *accel_fs_t; 153 struct mlx5_flow_destination dest; 154 struct mlx5e_accel_fs_tcp *fs_tcp; 155 MLX5_DECLARE_FLOW_ACT(flow_act); 156 struct mlx5_flow_handle *rule; 157 int err = 0; 158 159 fs_tcp = priv->fs.accel_tcp; 160 accel_fs_t = &fs_tcp->tables[type]; 161 162 dest = mlx5e_ttc_get_default_dest(priv, fs_accel2tt(type)); 163 rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1); 164 if (IS_ERR(rule)) { 165 err = PTR_ERR(rule); 166 netdev_err(priv->netdev, 167 "%s: add default rule failed, accel_fs type=%d, err %d\n", 168 __func__, type, err); 169 return err; 170 } 171 172 fs_tcp->default_rules[type] = rule; 173 return 0; 174 } 175 176 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2) 177 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1) 178 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0)) 179 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\ 180 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE) 181 static int accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, 182 enum accel_fs_tcp_type type) 183 { 184 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 185 void *outer_headers_c; 186 int ix = 0; 187 u32 *in; 188 int err; 189 u8 *mc; 190 191 ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 192 in = kvzalloc(inlen, GFP_KERNEL); 193 if (!in || !ft->g) { 194 kfree(ft->g); 195 kvfree(in); 196 return -ENOMEM; 197 } 198 199 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 200 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 201 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 202 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); 203 204 switch (type) { 205 case ACCEL_FS_IPV4_TCP: 206 case ACCEL_FS_IPV6_TCP: 207 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 208 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 209 break; 210 default: 211 err = -EINVAL; 212 goto out; 213 } 214 215 switch (type) { 216 case ACCEL_FS_IPV4_TCP: 217 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 218 src_ipv4_src_ipv6.ipv4_layout.ipv4); 219 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 220 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 221 break; 222 case ACCEL_FS_IPV6_TCP: 223 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 224 src_ipv4_src_ipv6.ipv6_layout.ipv6), 225 0xff, 16); 226 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 227 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 228 0xff, 16); 229 break; 230 default: 231 err = -EINVAL; 232 goto out; 233 } 234 235 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 236 MLX5_SET_CFG(in, start_flow_index, ix); 237 ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE; 238 MLX5_SET_CFG(in, end_flow_index, ix - 1); 239 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 240 if (IS_ERR(ft->g[ft->num_groups])) 241 goto err; 242 ft->num_groups++; 243 244 /* Default Flow Group */ 245 memset(in, 0, inlen); 246 MLX5_SET_CFG(in, start_flow_index, ix); 247 ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE; 248 MLX5_SET_CFG(in, end_flow_index, ix - 1); 249 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 250 if (IS_ERR(ft->g[ft->num_groups])) 251 goto err; 252 ft->num_groups++; 253 254 kvfree(in); 255 return 0; 256 257 err: 258 err = PTR_ERR(ft->g[ft->num_groups]); 259 ft->g[ft->num_groups] = NULL; 260 out: 261 kvfree(in); 262 263 return err; 264 } 265 266 static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type) 267 { 268 struct mlx5e_flow_table *ft = &priv->fs.accel_tcp->tables[type]; 269 struct mlx5_flow_table_attr ft_attr = {}; 270 int err; 271 272 ft->num_groups = 0; 273 274 ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE; 275 ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL; 276 ft_attr.prio = MLX5E_NIC_PRIO; 277 278 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); 279 if (IS_ERR(ft->t)) { 280 err = PTR_ERR(ft->t); 281 ft->t = NULL; 282 return err; 283 } 284 285 netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n", 286 ft->t->id, ft->t->level); 287 288 err = accel_fs_tcp_create_groups(ft, type); 289 if (err) 290 goto err; 291 292 err = accel_fs_tcp_add_default_rule(priv, type); 293 if (err) 294 goto err; 295 296 return 0; 297 err: 298 mlx5e_destroy_flow_table(ft); 299 return err; 300 } 301 302 static int accel_fs_tcp_disable(struct mlx5e_priv *priv) 303 { 304 int err, i; 305 306 for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { 307 /* Modify ttc rules destination to point back to the indir TIRs */ 308 err = mlx5e_ttc_fwd_default_dest(priv, fs_accel2tt(i)); 309 if (err) { 310 netdev_err(priv->netdev, 311 "%s: modify ttc[%d] default destination failed, err(%d)\n", 312 __func__, fs_accel2tt(i), err); 313 return err; 314 } 315 } 316 317 return 0; 318 } 319 320 static int accel_fs_tcp_enable(struct mlx5e_priv *priv) 321 { 322 struct mlx5_flow_destination dest = {}; 323 int err, i; 324 325 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 326 for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { 327 dest.ft = priv->fs.accel_tcp->tables[i].t; 328 329 /* Modify ttc rules destination to point on the accel_fs FTs */ 330 err = mlx5e_ttc_fwd_dest(priv, fs_accel2tt(i), &dest); 331 if (err) { 332 netdev_err(priv->netdev, 333 "%s: modify ttc[%d] destination to accel failed, err(%d)\n", 334 __func__, fs_accel2tt(i), err); 335 return err; 336 } 337 } 338 return 0; 339 } 340 341 static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) 342 { 343 struct mlx5e_accel_fs_tcp *fs_tcp; 344 345 fs_tcp = priv->fs.accel_tcp; 346 if (IS_ERR_OR_NULL(fs_tcp->tables[i].t)) 347 return; 348 349 mlx5_del_flow_rules(fs_tcp->default_rules[i]); 350 mlx5e_destroy_flow_table(&fs_tcp->tables[i]); 351 fs_tcp->tables[i].t = NULL; 352 } 353 354 void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) 355 { 356 int i; 357 358 if (!priv->fs.accel_tcp) 359 return; 360 361 accel_fs_tcp_disable(priv); 362 363 for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) 364 accel_fs_tcp_destroy_table(priv, i); 365 366 kfree(priv->fs.accel_tcp); 367 priv->fs.accel_tcp = NULL; 368 } 369 370 int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) 371 { 372 int i, err; 373 374 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 375 return -EOPNOTSUPP; 376 377 priv->fs.accel_tcp = kzalloc(sizeof(*priv->fs.accel_tcp), GFP_KERNEL); 378 if (!priv->fs.accel_tcp) 379 return -ENOMEM; 380 381 for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { 382 err = accel_fs_tcp_create_table(priv, i); 383 if (err) 384 goto err_destroy_tables; 385 } 386 387 err = accel_fs_tcp_enable(priv); 388 if (err) 389 goto err_destroy_tables; 390 391 return 0; 392 393 err_destroy_tables: 394 while (--i >= 0) 395 accel_fs_tcp_destroy_table(priv, i); 396 397 kfree(priv->fs.accel_tcp); 398 priv->fs.accel_tcp = NULL; 399 return err; 400 } 401