1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifdef CONFIG_RFS_ACCEL 34 35 #include <linux/hash.h> 36 #include <linux/mlx5/fs.h> 37 #include <linux/ip.h> 38 #include <linux/ipv6.h> 39 #include "en.h" 40 41 struct arfs_tuple { 42 __be16 etype; 43 u8 ip_proto; 44 union { 45 __be32 src_ipv4; 46 struct in6_addr src_ipv6; 47 }; 48 union { 49 __be32 dst_ipv4; 50 struct in6_addr dst_ipv6; 51 }; 52 __be16 src_port; 53 __be16 dst_port; 54 }; 55 56 struct arfs_rule { 57 struct mlx5e_priv *priv; 58 struct work_struct arfs_work; 59 struct mlx5_flow_rule *rule; 60 struct hlist_node hlist; 61 int rxq; 62 /* Flow ID passed to ndo_rx_flow_steer */ 63 int flow_id; 64 /* Filter ID returned by ndo_rx_flow_steer */ 65 int filter_id; 66 struct arfs_tuple tuple; 67 }; 68 69 #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \ 70 for (i = 0; i < ARFS_NUM_TYPES; i++) \ 71 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j) 72 73 #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \ 74 for (j = 0; j < ARFS_HASH_SIZE; j++) \ 75 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist) 76 77 static enum mlx5e_traffic_types arfs_get_tt(enum arfs_type type) 78 { 79 switch (type) { 80 case ARFS_IPV4_TCP: 81 return MLX5E_TT_IPV4_TCP; 82 case ARFS_IPV4_UDP: 83 return MLX5E_TT_IPV4_UDP; 84 case ARFS_IPV6_TCP: 85 return MLX5E_TT_IPV6_TCP; 86 case ARFS_IPV6_UDP: 87 return MLX5E_TT_IPV6_UDP; 88 default: 89 return -EINVAL; 90 } 91 } 92 93 static int arfs_disable(struct mlx5e_priv *priv) 94 { 95 struct mlx5_flow_destination dest; 96 u32 *tirn = priv->indir_tirn; 97 int err = 0; 98 int tt; 99 int i; 100 101 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 102 for (i = 0; i < ARFS_NUM_TYPES; i++) { 103 dest.tir_num = tirn[i]; 104 tt = arfs_get_tt(i); 105 /* Modify ttc rules destination to bypass the aRFS tables*/ 106 err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], 107 &dest); 108 if (err) { 109 netdev_err(priv->netdev, 110 "%s: modify ttc destination failed\n", 111 __func__); 112 return err; 113 } 114 } 115 return 0; 116 } 117 118 static void arfs_del_rules(struct mlx5e_priv *priv); 119 120 int mlx5e_arfs_disable(struct mlx5e_priv *priv) 121 { 122 arfs_del_rules(priv); 123 124 return arfs_disable(priv); 125 } 126 127 int mlx5e_arfs_enable(struct mlx5e_priv *priv) 128 { 129 struct mlx5_flow_destination dest; 130 int err = 0; 131 int tt; 132 int i; 133 134 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 135 for (i = 0; i < ARFS_NUM_TYPES; i++) { 136 dest.ft = priv->fs.arfs.arfs_tables[i].ft.t; 137 tt = arfs_get_tt(i); 138 /* Modify ttc rules destination to point on the aRFS FTs */ 139 err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], 140 &dest); 141 if (err) { 142 netdev_err(priv->netdev, 143 "%s: modify ttc destination failed err=%d\n", 144 __func__, err); 145 arfs_disable(priv); 146 return err; 147 } 148 } 149 return 0; 150 } 151 152 static void arfs_destroy_table(struct arfs_table *arfs_t) 153 { 154 mlx5_del_flow_rule(arfs_t->default_rule); 155 mlx5e_destroy_flow_table(&arfs_t->ft); 156 } 157 158 void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) 159 { 160 int i; 161 162 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) 163 return; 164 165 arfs_del_rules(priv); 166 destroy_workqueue(priv->fs.arfs.wq); 167 for (i = 0; i < ARFS_NUM_TYPES; i++) { 168 if (!IS_ERR_OR_NULL(priv->fs.arfs.arfs_tables[i].ft.t)) 169 arfs_destroy_table(&priv->fs.arfs.arfs_tables[i]); 170 } 171 } 172 173 static int arfs_add_default_rule(struct mlx5e_priv *priv, 174 enum arfs_type type) 175 { 176 struct arfs_table *arfs_t = &priv->fs.arfs.arfs_tables[type]; 177 struct mlx5_flow_destination dest; 178 u8 match_criteria_enable = 0; 179 u32 *tirn = priv->indir_tirn; 180 u32 *match_criteria; 181 u32 *match_value; 182 int err = 0; 183 184 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 185 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 186 if (!match_value || !match_criteria) { 187 netdev_err(priv->netdev, "%s: alloc failed\n", __func__); 188 err = -ENOMEM; 189 goto out; 190 } 191 192 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 193 switch (type) { 194 case ARFS_IPV4_TCP: 195 dest.tir_num = tirn[MLX5E_TT_IPV4_TCP]; 196 break; 197 case ARFS_IPV4_UDP: 198 dest.tir_num = tirn[MLX5E_TT_IPV4_UDP]; 199 break; 200 case ARFS_IPV6_TCP: 201 dest.tir_num = tirn[MLX5E_TT_IPV6_TCP]; 202 break; 203 case ARFS_IPV6_UDP: 204 dest.tir_num = tirn[MLX5E_TT_IPV6_UDP]; 205 break; 206 default: 207 err = -EINVAL; 208 goto out; 209 } 210 211 arfs_t->default_rule = mlx5_add_flow_rule(arfs_t->ft.t, match_criteria_enable, 212 match_criteria, match_value, 213 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, 214 MLX5_FS_DEFAULT_FLOW_TAG, 215 &dest); 216 if (IS_ERR(arfs_t->default_rule)) { 217 err = PTR_ERR(arfs_t->default_rule); 218 arfs_t->default_rule = NULL; 219 netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n", 220 __func__, type); 221 } 222 out: 223 kvfree(match_criteria); 224 kvfree(match_value); 225 return err; 226 } 227 228 #define MLX5E_ARFS_NUM_GROUPS 2 229 #define MLX5E_ARFS_GROUP1_SIZE BIT(12) 230 #define MLX5E_ARFS_GROUP2_SIZE BIT(0) 231 #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\ 232 MLX5E_ARFS_GROUP2_SIZE) 233 static int arfs_create_groups(struct mlx5e_flow_table *ft, 234 enum arfs_type type) 235 { 236 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 237 void *outer_headers_c; 238 int ix = 0; 239 u32 *in; 240 int err; 241 u8 *mc; 242 243 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, 244 sizeof(*ft->g), GFP_KERNEL); 245 in = mlx5_vzalloc(inlen); 246 if (!in || !ft->g) { 247 kvfree(ft->g); 248 kvfree(in); 249 return -ENOMEM; 250 } 251 252 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 253 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, 254 outer_headers); 255 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); 256 switch (type) { 257 case ARFS_IPV4_TCP: 258 case ARFS_IPV6_TCP: 259 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 260 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 261 break; 262 case ARFS_IPV4_UDP: 263 case ARFS_IPV6_UDP: 264 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 265 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport); 266 break; 267 default: 268 err = -EINVAL; 269 goto out; 270 } 271 272 switch (type) { 273 case ARFS_IPV4_TCP: 274 case ARFS_IPV4_UDP: 275 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 276 src_ipv4_src_ipv6.ipv4_layout.ipv4); 277 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 278 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 279 break; 280 case ARFS_IPV6_TCP: 281 case ARFS_IPV6_UDP: 282 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 283 src_ipv4_src_ipv6.ipv6_layout.ipv6), 284 0xff, 16); 285 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 286 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 287 0xff, 16); 288 break; 289 default: 290 err = -EINVAL; 291 goto out; 292 } 293 294 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 295 MLX5_SET_CFG(in, start_flow_index, ix); 296 ix += MLX5E_ARFS_GROUP1_SIZE; 297 MLX5_SET_CFG(in, end_flow_index, ix - 1); 298 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 299 if (IS_ERR(ft->g[ft->num_groups])) 300 goto err; 301 ft->num_groups++; 302 303 memset(in, 0, inlen); 304 MLX5_SET_CFG(in, start_flow_index, ix); 305 ix += MLX5E_ARFS_GROUP2_SIZE; 306 MLX5_SET_CFG(in, end_flow_index, ix - 1); 307 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 308 if (IS_ERR(ft->g[ft->num_groups])) 309 goto err; 310 ft->num_groups++; 311 312 kvfree(in); 313 return 0; 314 315 err: 316 err = PTR_ERR(ft->g[ft->num_groups]); 317 ft->g[ft->num_groups] = NULL; 318 out: 319 kvfree(in); 320 321 return err; 322 } 323 324 static int arfs_create_table(struct mlx5e_priv *priv, 325 enum arfs_type type) 326 { 327 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; 328 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; 329 int err; 330 331 ft->t = mlx5_create_flow_table(priv->fs.ns, MLX5E_NIC_PRIO, 332 MLX5E_ARFS_TABLE_SIZE, MLX5E_ARFS_FT_LEVEL); 333 if (IS_ERR(ft->t)) { 334 err = PTR_ERR(ft->t); 335 ft->t = NULL; 336 return err; 337 } 338 339 err = arfs_create_groups(ft, type); 340 if (err) 341 goto err; 342 343 err = arfs_add_default_rule(priv, type); 344 if (err) 345 goto err; 346 347 return 0; 348 err: 349 mlx5e_destroy_flow_table(ft); 350 return err; 351 } 352 353 int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) 354 { 355 int err = 0; 356 int i; 357 358 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) 359 return 0; 360 361 spin_lock_init(&priv->fs.arfs.arfs_lock); 362 INIT_LIST_HEAD(&priv->fs.arfs.rules); 363 priv->fs.arfs.wq = create_singlethread_workqueue("mlx5e_arfs"); 364 if (!priv->fs.arfs.wq) 365 return -ENOMEM; 366 367 for (i = 0; i < ARFS_NUM_TYPES; i++) { 368 err = arfs_create_table(priv, i); 369 if (err) 370 goto err; 371 } 372 return 0; 373 err: 374 mlx5e_arfs_destroy_tables(priv); 375 return err; 376 } 377 378 #define MLX5E_ARFS_EXPIRY_QUOTA 60 379 380 static void arfs_may_expire_flow(struct mlx5e_priv *priv) 381 { 382 struct arfs_rule *arfs_rule; 383 struct hlist_node *htmp; 384 int quota = 0; 385 int i; 386 int j; 387 388 HLIST_HEAD(del_list); 389 spin_lock_bh(&priv->fs.arfs.arfs_lock); 390 mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs.arfs_tables, i, j) { 391 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA) 392 break; 393 if (!work_pending(&arfs_rule->arfs_work) && 394 rps_may_expire_flow(priv->netdev, 395 arfs_rule->rxq, arfs_rule->flow_id, 396 arfs_rule->filter_id)) { 397 hlist_del_init(&arfs_rule->hlist); 398 hlist_add_head(&arfs_rule->hlist, &del_list); 399 } 400 } 401 spin_unlock_bh(&priv->fs.arfs.arfs_lock); 402 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { 403 if (arfs_rule->rule) 404 mlx5_del_flow_rule(arfs_rule->rule); 405 hlist_del(&arfs_rule->hlist); 406 kfree(arfs_rule); 407 } 408 } 409 410 static void arfs_del_rules(struct mlx5e_priv *priv) 411 { 412 struct hlist_node *htmp; 413 struct arfs_rule *rule; 414 int i; 415 int j; 416 417 HLIST_HEAD(del_list); 418 spin_lock_bh(&priv->fs.arfs.arfs_lock); 419 mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs.arfs_tables, i, j) { 420 hlist_del_init(&rule->hlist); 421 hlist_add_head(&rule->hlist, &del_list); 422 } 423 spin_unlock_bh(&priv->fs.arfs.arfs_lock); 424 425 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { 426 cancel_work_sync(&rule->arfs_work); 427 if (rule->rule) 428 mlx5_del_flow_rule(rule->rule); 429 hlist_del(&rule->hlist); 430 kfree(rule); 431 } 432 } 433 434 static struct hlist_head * 435 arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port, 436 __be16 dst_port) 437 { 438 unsigned long l; 439 int bucket_idx; 440 441 l = (__force unsigned long)src_port | 442 ((__force unsigned long)dst_port << 2); 443 444 bucket_idx = hash_long(l, ARFS_HASH_SHIFT); 445 446 return &arfs_t->rules_hash[bucket_idx]; 447 } 448 449 static u8 arfs_get_ip_proto(const struct sk_buff *skb) 450 { 451 return (skb->protocol == htons(ETH_P_IP)) ? 452 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 453 } 454 455 static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, 456 u8 ip_proto, __be16 etype) 457 { 458 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP) 459 return &arfs->arfs_tables[ARFS_IPV4_TCP]; 460 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP) 461 return &arfs->arfs_tables[ARFS_IPV4_UDP]; 462 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP) 463 return &arfs->arfs_tables[ARFS_IPV6_TCP]; 464 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP) 465 return &arfs->arfs_tables[ARFS_IPV6_UDP]; 466 467 return NULL; 468 } 469 470 static struct mlx5_flow_rule *arfs_add_rule(struct mlx5e_priv *priv, 471 struct arfs_rule *arfs_rule) 472 { 473 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; 474 struct arfs_tuple *tuple = &arfs_rule->tuple; 475 struct mlx5_flow_rule *rule = NULL; 476 struct mlx5_flow_destination dest; 477 struct arfs_table *arfs_table; 478 u8 match_criteria_enable = 0; 479 struct mlx5_flow_table *ft; 480 u32 *match_criteria; 481 u32 *match_value; 482 int err = 0; 483 484 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 485 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 486 if (!match_value || !match_criteria) { 487 netdev_err(priv->netdev, "%s: alloc failed\n", __func__); 488 err = -ENOMEM; 489 goto out; 490 } 491 match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 492 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 493 outer_headers.ethertype); 494 MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, 495 ntohs(tuple->etype)); 496 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype); 497 if (!arfs_table) { 498 err = -EINVAL; 499 goto out; 500 } 501 502 ft = arfs_table->ft.t; 503 if (tuple->ip_proto == IPPROTO_TCP) { 504 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 505 outer_headers.tcp_dport); 506 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 507 outer_headers.tcp_sport); 508 MLX5_SET(fte_match_param, match_value, outer_headers.tcp_dport, 509 ntohs(tuple->dst_port)); 510 MLX5_SET(fte_match_param, match_value, outer_headers.tcp_sport, 511 ntohs(tuple->src_port)); 512 } else { 513 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 514 outer_headers.udp_dport); 515 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 516 outer_headers.udp_sport); 517 MLX5_SET(fte_match_param, match_value, outer_headers.udp_dport, 518 ntohs(tuple->dst_port)); 519 MLX5_SET(fte_match_param, match_value, outer_headers.udp_sport, 520 ntohs(tuple->src_port)); 521 } 522 if (tuple->etype == htons(ETH_P_IP)) { 523 memcpy(MLX5_ADDR_OF(fte_match_param, match_value, 524 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 525 &tuple->src_ipv4, 526 4); 527 memcpy(MLX5_ADDR_OF(fte_match_param, match_value, 528 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 529 &tuple->dst_ipv4, 530 4); 531 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 532 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 533 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 534 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 535 } else { 536 memcpy(MLX5_ADDR_OF(fte_match_param, match_value, 537 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 538 &tuple->src_ipv6, 539 16); 540 memcpy(MLX5_ADDR_OF(fte_match_param, match_value, 541 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 542 &tuple->dst_ipv6, 543 16); 544 memset(MLX5_ADDR_OF(fte_match_param, match_criteria, 545 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 546 0xff, 547 16); 548 memset(MLX5_ADDR_OF(fte_match_param, match_criteria, 549 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 550 0xff, 551 16); 552 } 553 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 554 dest.tir_num = priv->direct_tir[arfs_rule->rxq].tirn; 555 rule = mlx5_add_flow_rule(ft, match_criteria_enable, match_criteria, 556 match_value, MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, 557 MLX5_FS_DEFAULT_FLOW_TAG, 558 &dest); 559 if (IS_ERR(rule)) { 560 err = PTR_ERR(rule); 561 netdev_err(priv->netdev, "%s: add rule(filter id=%d, rq idx=%d) failed, err=%d\n", 562 __func__, arfs_rule->filter_id, arfs_rule->rxq, err); 563 } 564 565 out: 566 kvfree(match_criteria); 567 kvfree(match_value); 568 return err ? ERR_PTR(err) : rule; 569 } 570 571 static void arfs_modify_rule_rq(struct mlx5e_priv *priv, 572 struct mlx5_flow_rule *rule, u16 rxq) 573 { 574 struct mlx5_flow_destination dst; 575 int err = 0; 576 577 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 578 dst.tir_num = priv->direct_tir[rxq].tirn; 579 err = mlx5_modify_rule_destination(rule, &dst); 580 if (err) 581 netdev_warn(priv->netdev, 582 "Failed to modfiy aRFS rule destination to rq=%d\n", rxq); 583 } 584 585 static void arfs_handle_work(struct work_struct *work) 586 { 587 struct arfs_rule *arfs_rule = container_of(work, 588 struct arfs_rule, 589 arfs_work); 590 struct mlx5e_priv *priv = arfs_rule->priv; 591 struct mlx5_flow_rule *rule; 592 593 mutex_lock(&priv->state_lock); 594 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 595 spin_lock_bh(&priv->fs.arfs.arfs_lock); 596 hlist_del(&arfs_rule->hlist); 597 spin_unlock_bh(&priv->fs.arfs.arfs_lock); 598 599 mutex_unlock(&priv->state_lock); 600 kfree(arfs_rule); 601 goto out; 602 } 603 mutex_unlock(&priv->state_lock); 604 605 if (!arfs_rule->rule) { 606 rule = arfs_add_rule(priv, arfs_rule); 607 if (IS_ERR(rule)) 608 goto out; 609 arfs_rule->rule = rule; 610 } else { 611 arfs_modify_rule_rq(priv, arfs_rule->rule, 612 arfs_rule->rxq); 613 } 614 out: 615 arfs_may_expire_flow(priv); 616 } 617 618 /* return L4 destination port from ip4/6 packets */ 619 static __be16 arfs_get_dst_port(const struct sk_buff *skb) 620 { 621 char *transport_header; 622 623 transport_header = skb_transport_header(skb); 624 if (arfs_get_ip_proto(skb) == IPPROTO_TCP) 625 return ((struct tcphdr *)transport_header)->dest; 626 return ((struct udphdr *)transport_header)->dest; 627 } 628 629 /* return L4 source port from ip4/6 packets */ 630 static __be16 arfs_get_src_port(const struct sk_buff *skb) 631 { 632 char *transport_header; 633 634 transport_header = skb_transport_header(skb); 635 if (arfs_get_ip_proto(skb) == IPPROTO_TCP) 636 return ((struct tcphdr *)transport_header)->source; 637 return ((struct udphdr *)transport_header)->source; 638 } 639 640 static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, 641 struct arfs_table *arfs_t, 642 const struct sk_buff *skb, 643 u16 rxq, u32 flow_id) 644 { 645 struct arfs_rule *rule; 646 struct arfs_tuple *tuple; 647 648 rule = kzalloc(sizeof(*rule), GFP_ATOMIC); 649 if (!rule) 650 return NULL; 651 652 rule->priv = priv; 653 rule->rxq = rxq; 654 INIT_WORK(&rule->arfs_work, arfs_handle_work); 655 656 tuple = &rule->tuple; 657 tuple->etype = skb->protocol; 658 if (tuple->etype == htons(ETH_P_IP)) { 659 tuple->src_ipv4 = ip_hdr(skb)->saddr; 660 tuple->dst_ipv4 = ip_hdr(skb)->daddr; 661 } else { 662 memcpy(&tuple->src_ipv6, &ipv6_hdr(skb)->saddr, 663 sizeof(struct in6_addr)); 664 memcpy(&tuple->dst_ipv6, &ipv6_hdr(skb)->daddr, 665 sizeof(struct in6_addr)); 666 } 667 tuple->ip_proto = arfs_get_ip_proto(skb); 668 tuple->src_port = arfs_get_src_port(skb); 669 tuple->dst_port = arfs_get_dst_port(skb); 670 671 rule->flow_id = flow_id; 672 rule->filter_id = priv->fs.arfs.last_filter_id++ % RPS_NO_FILTER; 673 674 hlist_add_head(&rule->hlist, 675 arfs_hash_bucket(arfs_t, tuple->src_port, 676 tuple->dst_port)); 677 return rule; 678 } 679 680 static bool arfs_cmp_ips(struct arfs_tuple *tuple, 681 const struct sk_buff *skb) 682 { 683 if (tuple->etype == htons(ETH_P_IP) && 684 tuple->src_ipv4 == ip_hdr(skb)->saddr && 685 tuple->dst_ipv4 == ip_hdr(skb)->daddr) 686 return true; 687 if (tuple->etype == htons(ETH_P_IPV6) && 688 (!memcmp(&tuple->src_ipv6, &ipv6_hdr(skb)->saddr, 689 sizeof(struct in6_addr))) && 690 (!memcmp(&tuple->dst_ipv6, &ipv6_hdr(skb)->daddr, 691 sizeof(struct in6_addr)))) 692 return true; 693 return false; 694 } 695 696 static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t, 697 const struct sk_buff *skb) 698 { 699 struct arfs_rule *arfs_rule; 700 struct hlist_head *head; 701 __be16 src_port = arfs_get_src_port(skb); 702 __be16 dst_port = arfs_get_dst_port(skb); 703 704 head = arfs_hash_bucket(arfs_t, src_port, dst_port); 705 hlist_for_each_entry(arfs_rule, head, hlist) { 706 if (arfs_rule->tuple.src_port == src_port && 707 arfs_rule->tuple.dst_port == dst_port && 708 arfs_cmp_ips(&arfs_rule->tuple, skb)) { 709 return arfs_rule; 710 } 711 } 712 713 return NULL; 714 } 715 716 int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 717 u16 rxq_index, u32 flow_id) 718 { 719 struct mlx5e_priv *priv = netdev_priv(dev); 720 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; 721 struct arfs_table *arfs_t; 722 struct arfs_rule *arfs_rule; 723 724 if (skb->protocol != htons(ETH_P_IP) && 725 skb->protocol != htons(ETH_P_IPV6)) 726 return -EPROTONOSUPPORT; 727 728 arfs_t = arfs_get_table(arfs, arfs_get_ip_proto(skb), skb->protocol); 729 if (!arfs_t) 730 return -EPROTONOSUPPORT; 731 732 spin_lock_bh(&arfs->arfs_lock); 733 arfs_rule = arfs_find_rule(arfs_t, skb); 734 if (arfs_rule) { 735 if (arfs_rule->rxq == rxq_index) { 736 spin_unlock_bh(&arfs->arfs_lock); 737 return arfs_rule->filter_id; 738 } 739 arfs_rule->rxq = rxq_index; 740 } else { 741 arfs_rule = arfs_alloc_rule(priv, arfs_t, skb, 742 rxq_index, flow_id); 743 if (!arfs_rule) { 744 spin_unlock_bh(&arfs->arfs_lock); 745 return -ENOMEM; 746 } 747 } 748 queue_work(priv->fs.arfs.wq, &arfs_rule->arfs_work); 749 spin_unlock_bh(&arfs->arfs_lock); 750 return arfs_rule->filter_id; 751 } 752 #endif 753