1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/hash.h> 34 #include <linux/mlx5/fs.h> 35 #include <linux/ip.h> 36 #include <linux/ipv6.h> 37 #include "en.h" 38 39 #define ARFS_HASH_SHIFT BITS_PER_BYTE 40 #define ARFS_HASH_SIZE BIT(BITS_PER_BYTE) 41 42 struct arfs_table { 43 struct mlx5e_flow_table ft; 44 struct mlx5_flow_handle *default_rule; 45 struct hlist_head rules_hash[ARFS_HASH_SIZE]; 46 }; 47 48 enum arfs_type { 49 ARFS_IPV4_TCP, 50 ARFS_IPV6_TCP, 51 ARFS_IPV4_UDP, 52 ARFS_IPV6_UDP, 53 ARFS_NUM_TYPES, 54 }; 55 56 struct mlx5e_arfs_tables { 57 struct arfs_table arfs_tables[ARFS_NUM_TYPES]; 58 /* Protect aRFS rules list */ 59 spinlock_t arfs_lock; 60 struct list_head rules; 61 int last_filter_id; 62 struct workqueue_struct *wq; 63 }; 64 65 struct arfs_tuple { 66 __be16 etype; 67 u8 ip_proto; 68 union { 69 __be32 src_ipv4; 70 struct in6_addr src_ipv6; 71 }; 72 union { 73 __be32 dst_ipv4; 74 struct in6_addr dst_ipv6; 75 }; 76 __be16 src_port; 77 __be16 dst_port; 78 }; 79 80 struct arfs_rule { 81 struct mlx5e_priv *priv; 82 struct work_struct arfs_work; 83 struct mlx5_flow_handle *rule; 84 struct hlist_node hlist; 85 int rxq; 86 /* Flow ID passed to ndo_rx_flow_steer */ 87 int flow_id; 88 /* Filter ID returned by ndo_rx_flow_steer */ 89 int filter_id; 90 struct arfs_tuple tuple; 91 }; 92 93 #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \ 94 for (i = 0; i < ARFS_NUM_TYPES; i++) \ 95 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j) 96 97 #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \ 98 for (j = 0; j < ARFS_HASH_SIZE; j++) \ 99 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist) 100 101 static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type) 102 { 103 switch (type) { 104 case ARFS_IPV4_TCP: 105 return MLX5_TT_IPV4_TCP; 106 case ARFS_IPV4_UDP: 107 return MLX5_TT_IPV4_UDP; 108 case ARFS_IPV6_TCP: 109 return MLX5_TT_IPV6_TCP; 110 case ARFS_IPV6_UDP: 111 return MLX5_TT_IPV6_UDP; 112 default: 113 return -EINVAL; 114 } 115 } 116 117 static int arfs_disable(struct mlx5e_priv *priv) 118 { 119 int err, i; 120 121 for (i = 0; i < ARFS_NUM_TYPES; i++) { 122 /* Modify ttc rules destination back to their default */ 123 err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, arfs_get_tt(i)); 124 if (err) { 125 netdev_err(priv->netdev, 126 "%s: modify ttc[%d] default destination failed, err(%d)\n", 127 __func__, arfs_get_tt(i), err); 128 return err; 129 } 130 } 131 return 0; 132 } 133 134 static void arfs_del_rules(struct mlx5e_priv *priv); 135 136 int mlx5e_arfs_disable(struct mlx5e_priv *priv) 137 { 138 arfs_del_rules(priv); 139 140 return arfs_disable(priv); 141 } 142 143 int mlx5e_arfs_enable(struct mlx5e_priv *priv) 144 { 145 struct mlx5_flow_destination dest = {}; 146 int err, i; 147 148 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 149 for (i = 0; i < ARFS_NUM_TYPES; i++) { 150 dest.ft = priv->fs.arfs->arfs_tables[i].ft.t; 151 /* Modify ttc rules destination to point on the aRFS FTs */ 152 err = mlx5_ttc_fwd_dest(priv->fs.ttc, arfs_get_tt(i), &dest); 153 if (err) { 154 netdev_err(priv->netdev, 155 "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", 156 __func__, arfs_get_tt(i), err); 157 arfs_disable(priv); 158 return err; 159 } 160 } 161 return 0; 162 } 163 164 static void arfs_destroy_table(struct arfs_table *arfs_t) 165 { 166 mlx5_del_flow_rules(arfs_t->default_rule); 167 mlx5e_destroy_flow_table(&arfs_t->ft); 168 } 169 170 static void _mlx5e_cleanup_tables(struct mlx5e_priv *priv) 171 { 172 int i; 173 174 arfs_del_rules(priv); 175 destroy_workqueue(priv->fs.arfs->wq); 176 for (i = 0; i < ARFS_NUM_TYPES; i++) { 177 if (!IS_ERR_OR_NULL(priv->fs.arfs->arfs_tables[i].ft.t)) 178 arfs_destroy_table(&priv->fs.arfs->arfs_tables[i]); 179 } 180 } 181 182 void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) 183 { 184 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) 185 return; 186 187 _mlx5e_cleanup_tables(priv); 188 kvfree(priv->fs.arfs); 189 } 190 191 static int arfs_add_default_rule(struct mlx5e_priv *priv, 192 enum arfs_type type) 193 { 194 struct arfs_table *arfs_t = &priv->fs.arfs->arfs_tables[type]; 195 struct mlx5_flow_destination dest = {}; 196 MLX5_DECLARE_FLOW_ACT(flow_act); 197 enum mlx5_traffic_types tt; 198 int err = 0; 199 200 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 201 tt = arfs_get_tt(type); 202 if (tt == -EINVAL) { 203 netdev_err(priv->netdev, "%s: bad arfs_type: %d\n", 204 __func__, type); 205 return -EINVAL; 206 } 207 208 /* FIXME: Must use mlx5_ttc_get_default_dest(), 209 * but can't since TTC default is not setup yet ! 210 */ 211 dest.tir_num = mlx5e_rx_res_get_tirn_rss(priv->rx_res, tt); 212 arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL, 213 &flow_act, 214 &dest, 1); 215 if (IS_ERR(arfs_t->default_rule)) { 216 err = PTR_ERR(arfs_t->default_rule); 217 arfs_t->default_rule = NULL; 218 netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n", 219 __func__, type); 220 } 221 222 return err; 223 } 224 225 #define MLX5E_ARFS_NUM_GROUPS 2 226 #define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1) 227 #define MLX5E_ARFS_GROUP2_SIZE BIT(0) 228 #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\ 229 MLX5E_ARFS_GROUP2_SIZE) 230 static int arfs_create_groups(struct mlx5e_flow_table *ft, 231 enum arfs_type type) 232 { 233 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 234 void *outer_headers_c; 235 int ix = 0; 236 u32 *in; 237 int err; 238 u8 *mc; 239 240 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, 241 sizeof(*ft->g), GFP_KERNEL); 242 in = kvzalloc(inlen, GFP_KERNEL); 243 if (!in || !ft->g) { 244 kfree(ft->g); 245 kvfree(in); 246 return -ENOMEM; 247 } 248 249 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 250 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, 251 outer_headers); 252 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); 253 switch (type) { 254 case ARFS_IPV4_TCP: 255 case ARFS_IPV6_TCP: 256 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 257 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 258 break; 259 case ARFS_IPV4_UDP: 260 case ARFS_IPV6_UDP: 261 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 262 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport); 263 break; 264 default: 265 err = -EINVAL; 266 goto out; 267 } 268 269 switch (type) { 270 case ARFS_IPV4_TCP: 271 case ARFS_IPV4_UDP: 272 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 273 src_ipv4_src_ipv6.ipv4_layout.ipv4); 274 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 275 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 276 break; 277 case ARFS_IPV6_TCP: 278 case ARFS_IPV6_UDP: 279 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 280 src_ipv4_src_ipv6.ipv6_layout.ipv6), 281 0xff, 16); 282 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 283 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 284 0xff, 16); 285 break; 286 default: 287 err = -EINVAL; 288 goto out; 289 } 290 291 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 292 MLX5_SET_CFG(in, start_flow_index, ix); 293 ix += MLX5E_ARFS_GROUP1_SIZE; 294 MLX5_SET_CFG(in, end_flow_index, ix - 1); 295 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 296 if (IS_ERR(ft->g[ft->num_groups])) 297 goto err; 298 ft->num_groups++; 299 300 memset(in, 0, inlen); 301 MLX5_SET_CFG(in, start_flow_index, ix); 302 ix += MLX5E_ARFS_GROUP2_SIZE; 303 MLX5_SET_CFG(in, end_flow_index, ix - 1); 304 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 305 if (IS_ERR(ft->g[ft->num_groups])) 306 goto err; 307 ft->num_groups++; 308 309 kvfree(in); 310 return 0; 311 312 err: 313 err = PTR_ERR(ft->g[ft->num_groups]); 314 ft->g[ft->num_groups] = NULL; 315 out: 316 kvfree(in); 317 318 return err; 319 } 320 321 static int arfs_create_table(struct mlx5e_priv *priv, 322 enum arfs_type type) 323 { 324 struct mlx5e_arfs_tables *arfs = priv->fs.arfs; 325 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; 326 struct mlx5_flow_table_attr ft_attr = {}; 327 int err; 328 329 ft->num_groups = 0; 330 331 ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE; 332 ft_attr.level = MLX5E_ARFS_FT_LEVEL; 333 ft_attr.prio = MLX5E_NIC_PRIO; 334 335 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); 336 if (IS_ERR(ft->t)) { 337 err = PTR_ERR(ft->t); 338 ft->t = NULL; 339 return err; 340 } 341 342 err = arfs_create_groups(ft, type); 343 if (err) 344 goto err; 345 346 err = arfs_add_default_rule(priv, type); 347 if (err) 348 goto err; 349 350 return 0; 351 err: 352 mlx5e_destroy_flow_table(ft); 353 return err; 354 } 355 356 int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) 357 { 358 int err = -ENOMEM; 359 int i; 360 361 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) 362 return 0; 363 364 priv->fs.arfs = kvzalloc(sizeof(*priv->fs.arfs), GFP_KERNEL); 365 if (!priv->fs.arfs) 366 return -ENOMEM; 367 368 spin_lock_init(&priv->fs.arfs->arfs_lock); 369 INIT_LIST_HEAD(&priv->fs.arfs->rules); 370 priv->fs.arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); 371 if (!priv->fs.arfs->wq) 372 goto err; 373 374 for (i = 0; i < ARFS_NUM_TYPES; i++) { 375 err = arfs_create_table(priv, i); 376 if (err) 377 goto err_des; 378 } 379 return 0; 380 381 err_des: 382 _mlx5e_cleanup_tables(priv); 383 err: 384 kvfree(priv->fs.arfs); 385 return err; 386 } 387 388 #define MLX5E_ARFS_EXPIRY_QUOTA 60 389 390 static void arfs_may_expire_flow(struct mlx5e_priv *priv) 391 { 392 struct arfs_rule *arfs_rule; 393 struct hlist_node *htmp; 394 HLIST_HEAD(del_list); 395 int quota = 0; 396 int i; 397 int j; 398 399 spin_lock_bh(&priv->fs.arfs->arfs_lock); 400 mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs->arfs_tables, i, j) { 401 if (!work_pending(&arfs_rule->arfs_work) && 402 rps_may_expire_flow(priv->netdev, 403 arfs_rule->rxq, arfs_rule->flow_id, 404 arfs_rule->filter_id)) { 405 hlist_del_init(&arfs_rule->hlist); 406 hlist_add_head(&arfs_rule->hlist, &del_list); 407 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA) 408 break; 409 } 410 } 411 spin_unlock_bh(&priv->fs.arfs->arfs_lock); 412 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { 413 if (arfs_rule->rule) 414 mlx5_del_flow_rules(arfs_rule->rule); 415 hlist_del(&arfs_rule->hlist); 416 kfree(arfs_rule); 417 } 418 } 419 420 static void arfs_del_rules(struct mlx5e_priv *priv) 421 { 422 struct hlist_node *htmp; 423 struct arfs_rule *rule; 424 HLIST_HEAD(del_list); 425 int i; 426 int j; 427 428 spin_lock_bh(&priv->fs.arfs->arfs_lock); 429 mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs->arfs_tables, i, j) { 430 hlist_del_init(&rule->hlist); 431 hlist_add_head(&rule->hlist, &del_list); 432 } 433 spin_unlock_bh(&priv->fs.arfs->arfs_lock); 434 435 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { 436 cancel_work_sync(&rule->arfs_work); 437 if (rule->rule) 438 mlx5_del_flow_rules(rule->rule); 439 hlist_del(&rule->hlist); 440 kfree(rule); 441 } 442 } 443 444 static struct hlist_head * 445 arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port, 446 __be16 dst_port) 447 { 448 unsigned long l; 449 int bucket_idx; 450 451 l = (__force unsigned long)src_port | 452 ((__force unsigned long)dst_port << 2); 453 454 bucket_idx = hash_long(l, ARFS_HASH_SHIFT); 455 456 return &arfs_t->rules_hash[bucket_idx]; 457 } 458 459 static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, 460 u8 ip_proto, __be16 etype) 461 { 462 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP) 463 return &arfs->arfs_tables[ARFS_IPV4_TCP]; 464 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP) 465 return &arfs->arfs_tables[ARFS_IPV4_UDP]; 466 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP) 467 return &arfs->arfs_tables[ARFS_IPV6_TCP]; 468 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP) 469 return &arfs->arfs_tables[ARFS_IPV6_UDP]; 470 471 return NULL; 472 } 473 474 static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv, 475 struct arfs_rule *arfs_rule) 476 { 477 struct mlx5e_arfs_tables *arfs = priv->fs.arfs; 478 struct arfs_tuple *tuple = &arfs_rule->tuple; 479 struct mlx5_flow_handle *rule = NULL; 480 struct mlx5_flow_destination dest = {}; 481 MLX5_DECLARE_FLOW_ACT(flow_act); 482 struct arfs_table *arfs_table; 483 struct mlx5_flow_spec *spec; 484 struct mlx5_flow_table *ft; 485 int err = 0; 486 487 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 488 if (!spec) { 489 err = -ENOMEM; 490 goto out; 491 } 492 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 493 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 494 outer_headers.ethertype); 495 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, 496 ntohs(tuple->etype)); 497 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype); 498 if (!arfs_table) { 499 err = -EINVAL; 500 goto out; 501 } 502 503 ft = arfs_table->ft.t; 504 if (tuple->ip_proto == IPPROTO_TCP) { 505 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 506 outer_headers.tcp_dport); 507 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 508 outer_headers.tcp_sport); 509 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 510 ntohs(tuple->dst_port)); 511 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 512 ntohs(tuple->src_port)); 513 } else { 514 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 515 outer_headers.udp_dport); 516 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 517 outer_headers.udp_sport); 518 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, 519 ntohs(tuple->dst_port)); 520 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport, 521 ntohs(tuple->src_port)); 522 } 523 if (tuple->etype == htons(ETH_P_IP)) { 524 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 525 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 526 &tuple->src_ipv4, 527 4); 528 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 529 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 530 &tuple->dst_ipv4, 531 4); 532 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 533 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 534 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 535 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 536 } else { 537 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 538 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 539 &tuple->src_ipv6, 540 16); 541 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 542 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 543 &tuple->dst_ipv6, 544 16); 545 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 546 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 547 0xff, 548 16); 549 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 550 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 551 0xff, 552 16); 553 } 554 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 555 dest.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, arfs_rule->rxq); 556 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 557 if (IS_ERR(rule)) { 558 err = PTR_ERR(rule); 559 priv->channel_stats[arfs_rule->rxq]->rq.arfs_err++; 560 mlx5e_dbg(HW, priv, 561 "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n", 562 __func__, arfs_rule->filter_id, arfs_rule->rxq, 563 tuple->ip_proto, err); 564 } 565 566 out: 567 kvfree(spec); 568 return err ? ERR_PTR(err) : rule; 569 } 570 571 static void arfs_modify_rule_rq(struct mlx5e_priv *priv, 572 struct mlx5_flow_handle *rule, u16 rxq) 573 { 574 struct mlx5_flow_destination dst = {}; 575 int err = 0; 576 577 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 578 dst.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, rxq); 579 err = mlx5_modify_rule_destination(rule, &dst, NULL); 580 if (err) 581 netdev_warn(priv->netdev, 582 "Failed to modify aRFS rule destination to rq=%d\n", rxq); 583 } 584 585 static void arfs_handle_work(struct work_struct *work) 586 { 587 struct arfs_rule *arfs_rule = container_of(work, 588 struct arfs_rule, 589 arfs_work); 590 struct mlx5e_priv *priv = arfs_rule->priv; 591 struct mlx5_flow_handle *rule; 592 593 mutex_lock(&priv->state_lock); 594 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 595 spin_lock_bh(&priv->fs.arfs->arfs_lock); 596 hlist_del(&arfs_rule->hlist); 597 spin_unlock_bh(&priv->fs.arfs->arfs_lock); 598 599 mutex_unlock(&priv->state_lock); 600 kfree(arfs_rule); 601 goto out; 602 } 603 mutex_unlock(&priv->state_lock); 604 605 if (!arfs_rule->rule) { 606 rule = arfs_add_rule(priv, arfs_rule); 607 if (IS_ERR(rule)) 608 goto out; 609 arfs_rule->rule = rule; 610 } else { 611 arfs_modify_rule_rq(priv, arfs_rule->rule, 612 arfs_rule->rxq); 613 } 614 out: 615 arfs_may_expire_flow(priv); 616 } 617 618 static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, 619 struct arfs_table *arfs_t, 620 const struct flow_keys *fk, 621 u16 rxq, u32 flow_id) 622 { 623 struct arfs_rule *rule; 624 struct arfs_tuple *tuple; 625 626 rule = kzalloc(sizeof(*rule), GFP_ATOMIC); 627 if (!rule) 628 return NULL; 629 630 rule->priv = priv; 631 rule->rxq = rxq; 632 INIT_WORK(&rule->arfs_work, arfs_handle_work); 633 634 tuple = &rule->tuple; 635 tuple->etype = fk->basic.n_proto; 636 tuple->ip_proto = fk->basic.ip_proto; 637 if (tuple->etype == htons(ETH_P_IP)) { 638 tuple->src_ipv4 = fk->addrs.v4addrs.src; 639 tuple->dst_ipv4 = fk->addrs.v4addrs.dst; 640 } else { 641 memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src, 642 sizeof(struct in6_addr)); 643 memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, 644 sizeof(struct in6_addr)); 645 } 646 tuple->src_port = fk->ports.src; 647 tuple->dst_port = fk->ports.dst; 648 649 rule->flow_id = flow_id; 650 rule->filter_id = priv->fs.arfs->last_filter_id++ % RPS_NO_FILTER; 651 652 hlist_add_head(&rule->hlist, 653 arfs_hash_bucket(arfs_t, tuple->src_port, 654 tuple->dst_port)); 655 return rule; 656 } 657 658 static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk) 659 { 660 if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst) 661 return false; 662 if (tuple->etype != fk->basic.n_proto) 663 return false; 664 if (tuple->etype == htons(ETH_P_IP)) 665 return tuple->src_ipv4 == fk->addrs.v4addrs.src && 666 tuple->dst_ipv4 == fk->addrs.v4addrs.dst; 667 if (tuple->etype == htons(ETH_P_IPV6)) 668 return !memcmp(&tuple->src_ipv6, &fk->addrs.v6addrs.src, 669 sizeof(struct in6_addr)) && 670 !memcmp(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, 671 sizeof(struct in6_addr)); 672 return false; 673 } 674 675 static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t, 676 const struct flow_keys *fk) 677 { 678 struct arfs_rule *arfs_rule; 679 struct hlist_head *head; 680 681 head = arfs_hash_bucket(arfs_t, fk->ports.src, fk->ports.dst); 682 hlist_for_each_entry(arfs_rule, head, hlist) { 683 if (arfs_cmp(&arfs_rule->tuple, fk)) 684 return arfs_rule; 685 } 686 687 return NULL; 688 } 689 690 int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 691 u16 rxq_index, u32 flow_id) 692 { 693 struct mlx5e_priv *priv = netdev_priv(dev); 694 struct mlx5e_arfs_tables *arfs = priv->fs.arfs; 695 struct arfs_table *arfs_t; 696 struct arfs_rule *arfs_rule; 697 struct flow_keys fk; 698 699 if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) 700 return -EPROTONOSUPPORT; 701 702 if (fk.basic.n_proto != htons(ETH_P_IP) && 703 fk.basic.n_proto != htons(ETH_P_IPV6)) 704 return -EPROTONOSUPPORT; 705 706 if (skb->encapsulation) 707 return -EPROTONOSUPPORT; 708 709 arfs_t = arfs_get_table(arfs, fk.basic.ip_proto, fk.basic.n_proto); 710 if (!arfs_t) 711 return -EPROTONOSUPPORT; 712 713 spin_lock_bh(&arfs->arfs_lock); 714 arfs_rule = arfs_find_rule(arfs_t, &fk); 715 if (arfs_rule) { 716 if (arfs_rule->rxq == rxq_index) { 717 spin_unlock_bh(&arfs->arfs_lock); 718 return arfs_rule->filter_id; 719 } 720 arfs_rule->rxq = rxq_index; 721 } else { 722 arfs_rule = arfs_alloc_rule(priv, arfs_t, &fk, rxq_index, flow_id); 723 if (!arfs_rule) { 724 spin_unlock_bh(&arfs->arfs_lock); 725 return -ENOMEM; 726 } 727 } 728 queue_work(priv->fs.arfs->wq, &arfs_rule->arfs_work); 729 spin_unlock_bh(&arfs->arfs_lock); 730 return arfs_rule->filter_id; 731 } 732 733