1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/hash.h> 34 #include <linux/mlx5/fs.h> 35 #include <linux/ip.h> 36 #include <linux/ipv6.h> 37 #include "en.h" 38 39 struct arfs_tuple { 40 __be16 etype; 41 u8 ip_proto; 42 union { 43 __be32 src_ipv4; 44 struct in6_addr src_ipv6; 45 }; 46 union { 47 __be32 dst_ipv4; 48 struct in6_addr dst_ipv6; 49 }; 50 __be16 src_port; 51 __be16 dst_port; 52 }; 53 54 struct arfs_rule { 55 struct mlx5e_priv *priv; 56 struct work_struct arfs_work; 57 struct mlx5_flow_handle *rule; 58 struct hlist_node hlist; 59 int rxq; 60 /* Flow ID passed to ndo_rx_flow_steer */ 61 int flow_id; 62 /* Filter ID returned by ndo_rx_flow_steer */ 63 int filter_id; 64 struct arfs_tuple tuple; 65 }; 66 67 #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \ 68 for (i = 0; i < ARFS_NUM_TYPES; i++) \ 69 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j) 70 71 #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \ 72 for (j = 0; j < ARFS_HASH_SIZE; j++) \ 73 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist) 74 75 static enum mlx5e_traffic_types arfs_get_tt(enum arfs_type type) 76 { 77 switch (type) { 78 case ARFS_IPV4_TCP: 79 return MLX5E_TT_IPV4_TCP; 80 case ARFS_IPV4_UDP: 81 return MLX5E_TT_IPV4_UDP; 82 case ARFS_IPV6_TCP: 83 return MLX5E_TT_IPV6_TCP; 84 case ARFS_IPV6_UDP: 85 return MLX5E_TT_IPV6_UDP; 86 default: 87 return -EINVAL; 88 } 89 } 90 91 static int arfs_disable(struct mlx5e_priv *priv) 92 { 93 int err, i; 94 95 for (i = 0; i < ARFS_NUM_TYPES; i++) { 96 /* Modify ttc rules destination back to their default */ 97 err = mlx5e_ttc_fwd_default_dest(priv, arfs_get_tt(i)); 98 if (err) { 99 netdev_err(priv->netdev, 100 "%s: modify ttc[%d] default destination failed, err(%d)\n", 101 __func__, arfs_get_tt(i), err); 102 return err; 103 } 104 } 105 return 0; 106 } 107 108 static void arfs_del_rules(struct mlx5e_priv *priv); 109 110 int mlx5e_arfs_disable(struct mlx5e_priv *priv) 111 { 112 arfs_del_rules(priv); 113 114 return arfs_disable(priv); 115 } 116 117 int mlx5e_arfs_enable(struct mlx5e_priv *priv) 118 { 119 struct mlx5_flow_destination dest = {}; 120 int err, i; 121 122 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 123 for (i = 0; i < ARFS_NUM_TYPES; i++) { 124 dest.ft = priv->fs.arfs.arfs_tables[i].ft.t; 125 /* Modify ttc rules destination to point on the aRFS FTs */ 126 err = mlx5e_ttc_fwd_dest(priv, arfs_get_tt(i), &dest); 127 if (err) { 128 netdev_err(priv->netdev, 129 "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", 130 __func__, arfs_get_tt(i), err); 131 arfs_disable(priv); 132 return err; 133 } 134 } 135 return 0; 136 } 137 138 static void arfs_destroy_table(struct arfs_table *arfs_t) 139 { 140 mlx5_del_flow_rules(arfs_t->default_rule); 141 mlx5e_destroy_flow_table(&arfs_t->ft); 142 } 143 144 void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) 145 { 146 int i; 147 148 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) 149 return; 150 151 arfs_del_rules(priv); 152 destroy_workqueue(priv->fs.arfs.wq); 153 for (i = 0; i < ARFS_NUM_TYPES; i++) { 154 if (!IS_ERR_OR_NULL(priv->fs.arfs.arfs_tables[i].ft.t)) 155 arfs_destroy_table(&priv->fs.arfs.arfs_tables[i]); 156 } 157 } 158 159 static int arfs_add_default_rule(struct mlx5e_priv *priv, 160 enum arfs_type type) 161 { 162 struct arfs_table *arfs_t = &priv->fs.arfs.arfs_tables[type]; 163 struct mlx5e_tir *tir = priv->indir_tir; 164 struct mlx5_flow_destination dest = {}; 165 MLX5_DECLARE_FLOW_ACT(flow_act); 166 enum mlx5e_traffic_types tt; 167 int err = 0; 168 169 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 170 tt = arfs_get_tt(type); 171 if (tt == -EINVAL) { 172 netdev_err(priv->netdev, "%s: bad arfs_type: %d\n", 173 __func__, type); 174 return -EINVAL; 175 } 176 177 /* FIXME: Must use mlx5e_ttc_get_default_dest(), 178 * but can't since TTC default is not setup yet ! 179 */ 180 dest.tir_num = tir[tt].tirn; 181 arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL, 182 &flow_act, 183 &dest, 1); 184 if (IS_ERR(arfs_t->default_rule)) { 185 err = PTR_ERR(arfs_t->default_rule); 186 arfs_t->default_rule = NULL; 187 netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n", 188 __func__, type); 189 } 190 191 return err; 192 } 193 194 #define MLX5E_ARFS_NUM_GROUPS 2 195 #define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1) 196 #define MLX5E_ARFS_GROUP2_SIZE BIT(0) 197 #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\ 198 MLX5E_ARFS_GROUP2_SIZE) 199 static int arfs_create_groups(struct mlx5e_flow_table *ft, 200 enum arfs_type type) 201 { 202 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 203 void *outer_headers_c; 204 int ix = 0; 205 u32 *in; 206 int err; 207 u8 *mc; 208 209 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, 210 sizeof(*ft->g), GFP_KERNEL); 211 in = kvzalloc(inlen, GFP_KERNEL); 212 if (!in || !ft->g) { 213 kfree(ft->g); 214 kvfree(in); 215 return -ENOMEM; 216 } 217 218 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 219 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, 220 outer_headers); 221 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); 222 switch (type) { 223 case ARFS_IPV4_TCP: 224 case ARFS_IPV6_TCP: 225 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 226 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 227 break; 228 case ARFS_IPV4_UDP: 229 case ARFS_IPV6_UDP: 230 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 231 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport); 232 break; 233 default: 234 err = -EINVAL; 235 goto out; 236 } 237 238 switch (type) { 239 case ARFS_IPV4_TCP: 240 case ARFS_IPV4_UDP: 241 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 242 src_ipv4_src_ipv6.ipv4_layout.ipv4); 243 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 244 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 245 break; 246 case ARFS_IPV6_TCP: 247 case ARFS_IPV6_UDP: 248 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 249 src_ipv4_src_ipv6.ipv6_layout.ipv6), 250 0xff, 16); 251 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 252 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 253 0xff, 16); 254 break; 255 default: 256 err = -EINVAL; 257 goto out; 258 } 259 260 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 261 MLX5_SET_CFG(in, start_flow_index, ix); 262 ix += MLX5E_ARFS_GROUP1_SIZE; 263 MLX5_SET_CFG(in, end_flow_index, ix - 1); 264 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 265 if (IS_ERR(ft->g[ft->num_groups])) 266 goto err; 267 ft->num_groups++; 268 269 memset(in, 0, inlen); 270 MLX5_SET_CFG(in, start_flow_index, ix); 271 ix += MLX5E_ARFS_GROUP2_SIZE; 272 MLX5_SET_CFG(in, end_flow_index, ix - 1); 273 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 274 if (IS_ERR(ft->g[ft->num_groups])) 275 goto err; 276 ft->num_groups++; 277 278 kvfree(in); 279 return 0; 280 281 err: 282 err = PTR_ERR(ft->g[ft->num_groups]); 283 ft->g[ft->num_groups] = NULL; 284 out: 285 kvfree(in); 286 287 return err; 288 } 289 290 static int arfs_create_table(struct mlx5e_priv *priv, 291 enum arfs_type type) 292 { 293 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; 294 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; 295 struct mlx5_flow_table_attr ft_attr = {}; 296 int err; 297 298 ft->num_groups = 0; 299 300 ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE; 301 ft_attr.level = MLX5E_ARFS_FT_LEVEL; 302 ft_attr.prio = MLX5E_NIC_PRIO; 303 304 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); 305 if (IS_ERR(ft->t)) { 306 err = PTR_ERR(ft->t); 307 ft->t = NULL; 308 return err; 309 } 310 311 err = arfs_create_groups(ft, type); 312 if (err) 313 goto err; 314 315 err = arfs_add_default_rule(priv, type); 316 if (err) 317 goto err; 318 319 return 0; 320 err: 321 mlx5e_destroy_flow_table(ft); 322 return err; 323 } 324 325 int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) 326 { 327 int err = 0; 328 int i; 329 330 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) 331 return 0; 332 333 spin_lock_init(&priv->fs.arfs.arfs_lock); 334 INIT_LIST_HEAD(&priv->fs.arfs.rules); 335 priv->fs.arfs.wq = create_singlethread_workqueue("mlx5e_arfs"); 336 if (!priv->fs.arfs.wq) 337 return -ENOMEM; 338 339 for (i = 0; i < ARFS_NUM_TYPES; i++) { 340 err = arfs_create_table(priv, i); 341 if (err) 342 goto err; 343 } 344 return 0; 345 err: 346 mlx5e_arfs_destroy_tables(priv); 347 return err; 348 } 349 350 #define MLX5E_ARFS_EXPIRY_QUOTA 60 351 352 static void arfs_may_expire_flow(struct mlx5e_priv *priv) 353 { 354 struct arfs_rule *arfs_rule; 355 struct hlist_node *htmp; 356 int quota = 0; 357 int i; 358 int j; 359 360 HLIST_HEAD(del_list); 361 spin_lock_bh(&priv->fs.arfs.arfs_lock); 362 mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs.arfs_tables, i, j) { 363 if (!work_pending(&arfs_rule->arfs_work) && 364 rps_may_expire_flow(priv->netdev, 365 arfs_rule->rxq, arfs_rule->flow_id, 366 arfs_rule->filter_id)) { 367 hlist_del_init(&arfs_rule->hlist); 368 hlist_add_head(&arfs_rule->hlist, &del_list); 369 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA) 370 break; 371 } 372 } 373 spin_unlock_bh(&priv->fs.arfs.arfs_lock); 374 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { 375 if (arfs_rule->rule) 376 mlx5_del_flow_rules(arfs_rule->rule); 377 hlist_del(&arfs_rule->hlist); 378 kfree(arfs_rule); 379 } 380 } 381 382 static void arfs_del_rules(struct mlx5e_priv *priv) 383 { 384 struct hlist_node *htmp; 385 struct arfs_rule *rule; 386 int i; 387 int j; 388 389 HLIST_HEAD(del_list); 390 spin_lock_bh(&priv->fs.arfs.arfs_lock); 391 mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs.arfs_tables, i, j) { 392 hlist_del_init(&rule->hlist); 393 hlist_add_head(&rule->hlist, &del_list); 394 } 395 spin_unlock_bh(&priv->fs.arfs.arfs_lock); 396 397 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { 398 cancel_work_sync(&rule->arfs_work); 399 if (rule->rule) 400 mlx5_del_flow_rules(rule->rule); 401 hlist_del(&rule->hlist); 402 kfree(rule); 403 } 404 } 405 406 static struct hlist_head * 407 arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port, 408 __be16 dst_port) 409 { 410 unsigned long l; 411 int bucket_idx; 412 413 l = (__force unsigned long)src_port | 414 ((__force unsigned long)dst_port << 2); 415 416 bucket_idx = hash_long(l, ARFS_HASH_SHIFT); 417 418 return &arfs_t->rules_hash[bucket_idx]; 419 } 420 421 static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, 422 u8 ip_proto, __be16 etype) 423 { 424 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP) 425 return &arfs->arfs_tables[ARFS_IPV4_TCP]; 426 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP) 427 return &arfs->arfs_tables[ARFS_IPV4_UDP]; 428 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP) 429 return &arfs->arfs_tables[ARFS_IPV6_TCP]; 430 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP) 431 return &arfs->arfs_tables[ARFS_IPV6_UDP]; 432 433 return NULL; 434 } 435 436 static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv, 437 struct arfs_rule *arfs_rule) 438 { 439 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; 440 struct arfs_tuple *tuple = &arfs_rule->tuple; 441 struct mlx5_flow_handle *rule = NULL; 442 struct mlx5_flow_destination dest = {}; 443 MLX5_DECLARE_FLOW_ACT(flow_act); 444 struct arfs_table *arfs_table; 445 struct mlx5_flow_spec *spec; 446 struct mlx5_flow_table *ft; 447 int err = 0; 448 449 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 450 if (!spec) { 451 err = -ENOMEM; 452 goto out; 453 } 454 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 455 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 456 outer_headers.ethertype); 457 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, 458 ntohs(tuple->etype)); 459 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype); 460 if (!arfs_table) { 461 err = -EINVAL; 462 goto out; 463 } 464 465 ft = arfs_table->ft.t; 466 if (tuple->ip_proto == IPPROTO_TCP) { 467 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 468 outer_headers.tcp_dport); 469 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 470 outer_headers.tcp_sport); 471 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 472 ntohs(tuple->dst_port)); 473 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 474 ntohs(tuple->src_port)); 475 } else { 476 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 477 outer_headers.udp_dport); 478 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 479 outer_headers.udp_sport); 480 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, 481 ntohs(tuple->dst_port)); 482 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport, 483 ntohs(tuple->src_port)); 484 } 485 if (tuple->etype == htons(ETH_P_IP)) { 486 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 487 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 488 &tuple->src_ipv4, 489 4); 490 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 491 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 492 &tuple->dst_ipv4, 493 4); 494 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 495 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 496 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 497 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 498 } else { 499 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 500 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 501 &tuple->src_ipv6, 502 16); 503 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 504 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 505 &tuple->dst_ipv6, 506 16); 507 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 508 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 509 0xff, 510 16); 511 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 512 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 513 0xff, 514 16); 515 } 516 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 517 dest.tir_num = priv->direct_tir[arfs_rule->rxq].tirn; 518 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 519 if (IS_ERR(rule)) { 520 err = PTR_ERR(rule); 521 priv->channel_stats[arfs_rule->rxq].rq.arfs_err++; 522 mlx5e_dbg(HW, priv, 523 "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n", 524 __func__, arfs_rule->filter_id, arfs_rule->rxq, 525 tuple->ip_proto, err); 526 } 527 528 out: 529 kvfree(spec); 530 return err ? ERR_PTR(err) : rule; 531 } 532 533 static void arfs_modify_rule_rq(struct mlx5e_priv *priv, 534 struct mlx5_flow_handle *rule, u16 rxq) 535 { 536 struct mlx5_flow_destination dst = {}; 537 int err = 0; 538 539 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 540 dst.tir_num = priv->direct_tir[rxq].tirn; 541 err = mlx5_modify_rule_destination(rule, &dst, NULL); 542 if (err) 543 netdev_warn(priv->netdev, 544 "Failed to modify aRFS rule destination to rq=%d\n", rxq); 545 } 546 547 static void arfs_handle_work(struct work_struct *work) 548 { 549 struct arfs_rule *arfs_rule = container_of(work, 550 struct arfs_rule, 551 arfs_work); 552 struct mlx5e_priv *priv = arfs_rule->priv; 553 struct mlx5_flow_handle *rule; 554 555 mutex_lock(&priv->state_lock); 556 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 557 spin_lock_bh(&priv->fs.arfs.arfs_lock); 558 hlist_del(&arfs_rule->hlist); 559 spin_unlock_bh(&priv->fs.arfs.arfs_lock); 560 561 mutex_unlock(&priv->state_lock); 562 kfree(arfs_rule); 563 goto out; 564 } 565 mutex_unlock(&priv->state_lock); 566 567 if (!arfs_rule->rule) { 568 rule = arfs_add_rule(priv, arfs_rule); 569 if (IS_ERR(rule)) 570 goto out; 571 arfs_rule->rule = rule; 572 } else { 573 arfs_modify_rule_rq(priv, arfs_rule->rule, 574 arfs_rule->rxq); 575 } 576 out: 577 arfs_may_expire_flow(priv); 578 } 579 580 static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, 581 struct arfs_table *arfs_t, 582 const struct flow_keys *fk, 583 u16 rxq, u32 flow_id) 584 { 585 struct arfs_rule *rule; 586 struct arfs_tuple *tuple; 587 588 rule = kzalloc(sizeof(*rule), GFP_ATOMIC); 589 if (!rule) 590 return NULL; 591 592 rule->priv = priv; 593 rule->rxq = rxq; 594 INIT_WORK(&rule->arfs_work, arfs_handle_work); 595 596 tuple = &rule->tuple; 597 tuple->etype = fk->basic.n_proto; 598 tuple->ip_proto = fk->basic.ip_proto; 599 if (tuple->etype == htons(ETH_P_IP)) { 600 tuple->src_ipv4 = fk->addrs.v4addrs.src; 601 tuple->dst_ipv4 = fk->addrs.v4addrs.dst; 602 } else { 603 memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src, 604 sizeof(struct in6_addr)); 605 memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, 606 sizeof(struct in6_addr)); 607 } 608 tuple->src_port = fk->ports.src; 609 tuple->dst_port = fk->ports.dst; 610 611 rule->flow_id = flow_id; 612 rule->filter_id = priv->fs.arfs.last_filter_id++ % RPS_NO_FILTER; 613 614 hlist_add_head(&rule->hlist, 615 arfs_hash_bucket(arfs_t, tuple->src_port, 616 tuple->dst_port)); 617 return rule; 618 } 619 620 static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk) 621 { 622 if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst) 623 return false; 624 if (tuple->etype != fk->basic.n_proto) 625 return false; 626 if (tuple->etype == htons(ETH_P_IP)) 627 return tuple->src_ipv4 == fk->addrs.v4addrs.src && 628 tuple->dst_ipv4 == fk->addrs.v4addrs.dst; 629 if (tuple->etype == htons(ETH_P_IPV6)) 630 return !memcmp(&tuple->src_ipv6, &fk->addrs.v6addrs.src, 631 sizeof(struct in6_addr)) && 632 !memcmp(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, 633 sizeof(struct in6_addr)); 634 return false; 635 } 636 637 static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t, 638 const struct flow_keys *fk) 639 { 640 struct arfs_rule *arfs_rule; 641 struct hlist_head *head; 642 643 head = arfs_hash_bucket(arfs_t, fk->ports.src, fk->ports.dst); 644 hlist_for_each_entry(arfs_rule, head, hlist) { 645 if (arfs_cmp(&arfs_rule->tuple, fk)) 646 return arfs_rule; 647 } 648 649 return NULL; 650 } 651 652 int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 653 u16 rxq_index, u32 flow_id) 654 { 655 struct mlx5e_priv *priv = netdev_priv(dev); 656 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; 657 struct arfs_table *arfs_t; 658 struct arfs_rule *arfs_rule; 659 struct flow_keys fk; 660 661 if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) 662 return -EPROTONOSUPPORT; 663 664 if (fk.basic.n_proto != htons(ETH_P_IP) && 665 fk.basic.n_proto != htons(ETH_P_IPV6)) 666 return -EPROTONOSUPPORT; 667 668 if (skb->encapsulation) 669 return -EPROTONOSUPPORT; 670 671 arfs_t = arfs_get_table(arfs, fk.basic.ip_proto, fk.basic.n_proto); 672 if (!arfs_t) 673 return -EPROTONOSUPPORT; 674 675 spin_lock_bh(&arfs->arfs_lock); 676 arfs_rule = arfs_find_rule(arfs_t, &fk); 677 if (arfs_rule) { 678 if (arfs_rule->rxq == rxq_index) { 679 spin_unlock_bh(&arfs->arfs_lock); 680 return arfs_rule->filter_id; 681 } 682 arfs_rule->rxq = rxq_index; 683 } else { 684 arfs_rule = arfs_alloc_rule(priv, arfs_t, &fk, rxq_index, flow_id); 685 if (!arfs_rule) { 686 spin_unlock_bh(&arfs->arfs_lock); 687 return -ENOMEM; 688 } 689 } 690 queue_work(priv->fs.arfs.wq, &arfs_rule->arfs_work); 691 spin_unlock_bh(&arfs->arfs_lock); 692 return arfs_rule->filter_id; 693 } 694 695