1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/hash.h> 34 #include <linux/mlx5/fs.h> 35 #include <linux/ip.h> 36 #include <linux/ipv6.h> 37 #include "en.h" 38 39 #define ARFS_HASH_SHIFT BITS_PER_BYTE 40 #define ARFS_HASH_SIZE BIT(BITS_PER_BYTE) 41 42 struct arfs_table { 43 struct mlx5e_flow_table ft; 44 struct mlx5_flow_handle *default_rule; 45 struct hlist_head rules_hash[ARFS_HASH_SIZE]; 46 }; 47 48 enum arfs_type { 49 ARFS_IPV4_TCP, 50 ARFS_IPV6_TCP, 51 ARFS_IPV4_UDP, 52 ARFS_IPV6_UDP, 53 ARFS_NUM_TYPES, 54 }; 55 56 struct mlx5e_arfs_tables { 57 struct arfs_table arfs_tables[ARFS_NUM_TYPES]; 58 /* Protect aRFS rules list */ 59 spinlock_t arfs_lock; 60 struct list_head rules; 61 int last_filter_id; 62 struct workqueue_struct *wq; 63 }; 64 65 struct arfs_tuple { 66 __be16 etype; 67 u8 ip_proto; 68 union { 69 __be32 src_ipv4; 70 struct in6_addr src_ipv6; 71 }; 72 union { 73 __be32 dst_ipv4; 74 struct in6_addr dst_ipv6; 75 }; 76 __be16 src_port; 77 __be16 dst_port; 78 }; 79 80 struct arfs_rule { 81 struct mlx5e_priv *priv; 82 struct work_struct arfs_work; 83 struct mlx5_flow_handle *rule; 84 struct hlist_node hlist; 85 int rxq; 86 /* Flow ID passed to ndo_rx_flow_steer */ 87 int flow_id; 88 /* Filter ID returned by ndo_rx_flow_steer */ 89 int filter_id; 90 struct arfs_tuple tuple; 91 }; 92 93 #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \ 94 for (i = 0; i < ARFS_NUM_TYPES; i++) \ 95 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j) 96 97 #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \ 98 for (j = 0; j < ARFS_HASH_SIZE; j++) \ 99 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist) 100 101 static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type) 102 { 103 switch (type) { 104 case ARFS_IPV4_TCP: 105 return MLX5_TT_IPV4_TCP; 106 case ARFS_IPV4_UDP: 107 return MLX5_TT_IPV4_UDP; 108 case ARFS_IPV6_TCP: 109 return MLX5_TT_IPV6_TCP; 110 case ARFS_IPV6_UDP: 111 return MLX5_TT_IPV6_UDP; 112 default: 113 return -EINVAL; 114 } 115 } 116 117 static int arfs_disable(struct mlx5e_flow_steering *fs) 118 { 119 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); 120 int err, i; 121 122 for (i = 0; i < ARFS_NUM_TYPES; i++) { 123 /* Modify ttc rules destination back to their default */ 124 err = mlx5_ttc_fwd_default_dest(ttc, arfs_get_tt(i)); 125 if (err) { 126 fs_err(fs, 127 "%s: modify ttc[%d] default destination failed, err(%d)\n", 128 __func__, arfs_get_tt(i), err); 129 return err; 130 } 131 } 132 return 0; 133 } 134 135 static void arfs_del_rules(struct mlx5e_flow_steering *fs); 136 137 int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs) 138 { 139 arfs_del_rules(fs); 140 141 return arfs_disable(fs); 142 } 143 144 int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs) 145 { 146 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); 147 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); 148 struct mlx5_flow_destination dest = {}; 149 int err, i; 150 151 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 152 for (i = 0; i < ARFS_NUM_TYPES; i++) { 153 dest.ft = arfs->arfs_tables[i].ft.t; 154 /* Modify ttc rules destination to point on the aRFS FTs */ 155 err = mlx5_ttc_fwd_dest(ttc, arfs_get_tt(i), &dest); 156 if (err) { 157 fs_err(fs, "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", 158 __func__, arfs_get_tt(i), err); 159 arfs_disable(fs); 160 return err; 161 } 162 } 163 return 0; 164 } 165 166 static void arfs_destroy_table(struct arfs_table *arfs_t) 167 { 168 mlx5_del_flow_rules(arfs_t->default_rule); 169 mlx5e_destroy_flow_table(&arfs_t->ft); 170 } 171 172 static void _mlx5e_cleanup_tables(struct mlx5e_flow_steering *fs) 173 { 174 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); 175 int i; 176 177 arfs_del_rules(fs); 178 destroy_workqueue(arfs->wq); 179 for (i = 0; i < ARFS_NUM_TYPES; i++) { 180 if (!IS_ERR_OR_NULL(arfs->arfs_tables[i].ft.t)) 181 arfs_destroy_table(&arfs->arfs_tables[i]); 182 } 183 } 184 185 void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple) 186 { 187 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); 188 189 if (!ntuple) 190 return; 191 192 _mlx5e_cleanup_tables(fs); 193 mlx5e_fs_set_arfs(fs, NULL); 194 kvfree(arfs); 195 } 196 197 static int arfs_add_default_rule(struct mlx5e_flow_steering *fs, 198 struct mlx5e_rx_res *rx_res, 199 enum arfs_type type) 200 { 201 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); 202 struct arfs_table *arfs_t = &arfs->arfs_tables[type]; 203 struct mlx5_flow_destination dest = {}; 204 MLX5_DECLARE_FLOW_ACT(flow_act); 205 enum mlx5_traffic_types tt; 206 int err = 0; 207 208 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 209 tt = arfs_get_tt(type); 210 if (tt == -EINVAL) { 211 fs_err(fs, "%s: bad arfs_type: %d\n", __func__, type); 212 return -EINVAL; 213 } 214 215 /* FIXME: Must use mlx5_ttc_get_default_dest(), 216 * but can't since TTC default is not setup yet ! 217 */ 218 dest.tir_num = mlx5e_rx_res_get_tirn_rss(rx_res, tt); 219 arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL, 220 &flow_act, 221 &dest, 1); 222 if (IS_ERR(arfs_t->default_rule)) { 223 err = PTR_ERR(arfs_t->default_rule); 224 arfs_t->default_rule = NULL; 225 fs_err(fs, "%s: add rule failed, arfs type=%d\n", __func__, type); 226 } 227 228 return err; 229 } 230 231 #define MLX5E_ARFS_NUM_GROUPS 2 232 #define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1) 233 #define MLX5E_ARFS_GROUP2_SIZE BIT(0) 234 #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\ 235 MLX5E_ARFS_GROUP2_SIZE) 236 static int arfs_create_groups(struct mlx5e_flow_table *ft, 237 enum arfs_type type) 238 { 239 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 240 void *outer_headers_c; 241 int ix = 0; 242 u32 *in; 243 int err; 244 u8 *mc; 245 246 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, 247 sizeof(*ft->g), GFP_KERNEL); 248 in = kvzalloc(inlen, GFP_KERNEL); 249 if (!in || !ft->g) { 250 kfree(ft->g); 251 kvfree(in); 252 return -ENOMEM; 253 } 254 255 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 256 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, 257 outer_headers); 258 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); 259 switch (type) { 260 case ARFS_IPV4_TCP: 261 case ARFS_IPV6_TCP: 262 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 263 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 264 break; 265 case ARFS_IPV4_UDP: 266 case ARFS_IPV6_UDP: 267 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 268 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport); 269 break; 270 default: 271 err = -EINVAL; 272 goto out; 273 } 274 275 switch (type) { 276 case ARFS_IPV4_TCP: 277 case ARFS_IPV4_UDP: 278 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 279 src_ipv4_src_ipv6.ipv4_layout.ipv4); 280 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 281 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 282 break; 283 case ARFS_IPV6_TCP: 284 case ARFS_IPV6_UDP: 285 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 286 src_ipv4_src_ipv6.ipv6_layout.ipv6), 287 0xff, 16); 288 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 289 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 290 0xff, 16); 291 break; 292 default: 293 err = -EINVAL; 294 goto out; 295 } 296 297 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 298 MLX5_SET_CFG(in, start_flow_index, ix); 299 ix += MLX5E_ARFS_GROUP1_SIZE; 300 MLX5_SET_CFG(in, end_flow_index, ix - 1); 301 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 302 if (IS_ERR(ft->g[ft->num_groups])) 303 goto err; 304 ft->num_groups++; 305 306 memset(in, 0, inlen); 307 MLX5_SET_CFG(in, start_flow_index, ix); 308 ix += MLX5E_ARFS_GROUP2_SIZE; 309 MLX5_SET_CFG(in, end_flow_index, ix - 1); 310 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 311 if (IS_ERR(ft->g[ft->num_groups])) 312 goto err; 313 ft->num_groups++; 314 315 kvfree(in); 316 return 0; 317 318 err: 319 err = PTR_ERR(ft->g[ft->num_groups]); 320 ft->g[ft->num_groups] = NULL; 321 out: 322 kvfree(in); 323 324 return err; 325 } 326 327 static int arfs_create_table(struct mlx5e_flow_steering *fs, 328 struct mlx5e_rx_res *rx_res, 329 enum arfs_type type) 330 { 331 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false); 332 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); 333 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; 334 struct mlx5_flow_table_attr ft_attr = {}; 335 int err; 336 337 ft->num_groups = 0; 338 339 ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE; 340 ft_attr.level = MLX5E_ARFS_FT_LEVEL; 341 ft_attr.prio = MLX5E_NIC_PRIO; 342 343 ft->t = mlx5_create_flow_table(ns, &ft_attr); 344 if (IS_ERR(ft->t)) { 345 err = PTR_ERR(ft->t); 346 ft->t = NULL; 347 return err; 348 } 349 350 err = arfs_create_groups(ft, type); 351 if (err) 352 goto err; 353 354 err = arfs_add_default_rule(fs, rx_res, type); 355 if (err) 356 goto err; 357 358 return 0; 359 err: 360 mlx5e_destroy_flow_table(ft); 361 return err; 362 } 363 364 int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs, 365 struct mlx5e_rx_res *rx_res, bool ntuple) 366 { 367 struct mlx5e_arfs_tables *arfs; 368 int err = -ENOMEM; 369 int i; 370 371 if (!ntuple) 372 return 0; 373 374 arfs = kvzalloc(sizeof(*arfs), GFP_KERNEL); 375 if (!arfs) 376 return -ENOMEM; 377 378 spin_lock_init(&arfs->arfs_lock); 379 INIT_LIST_HEAD(&arfs->rules); 380 arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); 381 if (!arfs->wq) 382 goto err; 383 384 mlx5e_fs_set_arfs(fs, arfs); 385 386 for (i = 0; i < ARFS_NUM_TYPES; i++) { 387 err = arfs_create_table(fs, rx_res, i); 388 if (err) 389 goto err_des; 390 } 391 return 0; 392 393 err_des: 394 _mlx5e_cleanup_tables(fs); 395 err: 396 mlx5e_fs_set_arfs(fs, NULL); 397 kvfree(arfs); 398 return err; 399 } 400 401 #define MLX5E_ARFS_EXPIRY_QUOTA 60 402 403 static void arfs_may_expire_flow(struct mlx5e_priv *priv) 404 { 405 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs); 406 struct arfs_rule *arfs_rule; 407 struct hlist_node *htmp; 408 HLIST_HEAD(del_list); 409 int quota = 0; 410 int i; 411 int j; 412 413 spin_lock_bh(&arfs->arfs_lock); 414 mlx5e_for_each_arfs_rule(arfs_rule, htmp, arfs->arfs_tables, i, j) { 415 if (!work_pending(&arfs_rule->arfs_work) && 416 rps_may_expire_flow(priv->netdev, 417 arfs_rule->rxq, arfs_rule->flow_id, 418 arfs_rule->filter_id)) { 419 hlist_del_init(&arfs_rule->hlist); 420 hlist_add_head(&arfs_rule->hlist, &del_list); 421 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA) 422 break; 423 } 424 } 425 spin_unlock_bh(&arfs->arfs_lock); 426 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { 427 if (arfs_rule->rule) 428 mlx5_del_flow_rules(arfs_rule->rule); 429 hlist_del(&arfs_rule->hlist); 430 kfree(arfs_rule); 431 } 432 } 433 434 static void arfs_del_rules(struct mlx5e_flow_steering *fs) 435 { 436 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); 437 struct hlist_node *htmp; 438 struct arfs_rule *rule; 439 HLIST_HEAD(del_list); 440 int i; 441 int j; 442 443 spin_lock_bh(&arfs->arfs_lock); 444 mlx5e_for_each_arfs_rule(rule, htmp, arfs->arfs_tables, i, j) { 445 hlist_del_init(&rule->hlist); 446 hlist_add_head(&rule->hlist, &del_list); 447 } 448 spin_unlock_bh(&arfs->arfs_lock); 449 450 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { 451 cancel_work_sync(&rule->arfs_work); 452 if (rule->rule) 453 mlx5_del_flow_rules(rule->rule); 454 hlist_del(&rule->hlist); 455 kfree(rule); 456 } 457 } 458 459 static struct hlist_head * 460 arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port, 461 __be16 dst_port) 462 { 463 unsigned long l; 464 int bucket_idx; 465 466 l = (__force unsigned long)src_port | 467 ((__force unsigned long)dst_port << 2); 468 469 bucket_idx = hash_long(l, ARFS_HASH_SHIFT); 470 471 return &arfs_t->rules_hash[bucket_idx]; 472 } 473 474 static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, 475 u8 ip_proto, __be16 etype) 476 { 477 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP) 478 return &arfs->arfs_tables[ARFS_IPV4_TCP]; 479 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP) 480 return &arfs->arfs_tables[ARFS_IPV4_UDP]; 481 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP) 482 return &arfs->arfs_tables[ARFS_IPV6_TCP]; 483 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP) 484 return &arfs->arfs_tables[ARFS_IPV6_UDP]; 485 486 return NULL; 487 } 488 489 static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv, 490 struct arfs_rule *arfs_rule) 491 { 492 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs); 493 struct arfs_tuple *tuple = &arfs_rule->tuple; 494 struct mlx5_flow_handle *rule = NULL; 495 struct mlx5_flow_destination dest = {}; 496 MLX5_DECLARE_FLOW_ACT(flow_act); 497 struct arfs_table *arfs_table; 498 struct mlx5_flow_spec *spec; 499 struct mlx5_flow_table *ft; 500 int err = 0; 501 502 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 503 if (!spec) { 504 err = -ENOMEM; 505 goto out; 506 } 507 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 508 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 509 outer_headers.ethertype); 510 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, 511 ntohs(tuple->etype)); 512 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype); 513 if (!arfs_table) { 514 err = -EINVAL; 515 goto out; 516 } 517 518 ft = arfs_table->ft.t; 519 if (tuple->ip_proto == IPPROTO_TCP) { 520 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 521 outer_headers.tcp_dport); 522 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 523 outer_headers.tcp_sport); 524 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 525 ntohs(tuple->dst_port)); 526 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 527 ntohs(tuple->src_port)); 528 } else { 529 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 530 outer_headers.udp_dport); 531 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 532 outer_headers.udp_sport); 533 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, 534 ntohs(tuple->dst_port)); 535 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport, 536 ntohs(tuple->src_port)); 537 } 538 if (tuple->etype == htons(ETH_P_IP)) { 539 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 540 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 541 &tuple->src_ipv4, 542 4); 543 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 544 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 545 &tuple->dst_ipv4, 546 4); 547 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 548 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 549 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 550 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 551 } else { 552 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 553 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 554 &tuple->src_ipv6, 555 16); 556 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 557 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 558 &tuple->dst_ipv6, 559 16); 560 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 561 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 562 0xff, 563 16); 564 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 565 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 566 0xff, 567 16); 568 } 569 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 570 dest.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, arfs_rule->rxq); 571 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 572 if (IS_ERR(rule)) { 573 err = PTR_ERR(rule); 574 priv->channel_stats[arfs_rule->rxq]->rq.arfs_err++; 575 mlx5e_dbg(HW, priv, 576 "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n", 577 __func__, arfs_rule->filter_id, arfs_rule->rxq, 578 tuple->ip_proto, err); 579 } 580 581 out: 582 kvfree(spec); 583 return err ? ERR_PTR(err) : rule; 584 } 585 586 static void arfs_modify_rule_rq(struct mlx5e_priv *priv, 587 struct mlx5_flow_handle *rule, u16 rxq) 588 { 589 struct mlx5_flow_destination dst = {}; 590 int err = 0; 591 592 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 593 dst.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, rxq); 594 err = mlx5_modify_rule_destination(rule, &dst, NULL); 595 if (err) 596 netdev_warn(priv->netdev, 597 "Failed to modify aRFS rule destination to rq=%d\n", rxq); 598 } 599 600 static void arfs_handle_work(struct work_struct *work) 601 { 602 struct arfs_rule *arfs_rule = container_of(work, 603 struct arfs_rule, 604 arfs_work); 605 struct mlx5e_priv *priv = arfs_rule->priv; 606 struct mlx5e_arfs_tables *arfs; 607 struct mlx5_flow_handle *rule; 608 609 arfs = mlx5e_fs_get_arfs(priv->fs); 610 mutex_lock(&priv->state_lock); 611 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 612 spin_lock_bh(&arfs->arfs_lock); 613 hlist_del(&arfs_rule->hlist); 614 spin_unlock_bh(&arfs->arfs_lock); 615 616 mutex_unlock(&priv->state_lock); 617 kfree(arfs_rule); 618 goto out; 619 } 620 mutex_unlock(&priv->state_lock); 621 622 if (!arfs_rule->rule) { 623 rule = arfs_add_rule(priv, arfs_rule); 624 if (IS_ERR(rule)) 625 goto out; 626 arfs_rule->rule = rule; 627 } else { 628 arfs_modify_rule_rq(priv, arfs_rule->rule, 629 arfs_rule->rxq); 630 } 631 out: 632 arfs_may_expire_flow(priv); 633 } 634 635 static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, 636 struct arfs_table *arfs_t, 637 const struct flow_keys *fk, 638 u16 rxq, u32 flow_id) 639 { 640 struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs); 641 struct arfs_rule *rule; 642 struct arfs_tuple *tuple; 643 644 rule = kzalloc(sizeof(*rule), GFP_ATOMIC); 645 if (!rule) 646 return NULL; 647 648 rule->priv = priv; 649 rule->rxq = rxq; 650 INIT_WORK(&rule->arfs_work, arfs_handle_work); 651 652 tuple = &rule->tuple; 653 tuple->etype = fk->basic.n_proto; 654 tuple->ip_proto = fk->basic.ip_proto; 655 if (tuple->etype == htons(ETH_P_IP)) { 656 tuple->src_ipv4 = fk->addrs.v4addrs.src; 657 tuple->dst_ipv4 = fk->addrs.v4addrs.dst; 658 } else { 659 memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src, 660 sizeof(struct in6_addr)); 661 memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, 662 sizeof(struct in6_addr)); 663 } 664 tuple->src_port = fk->ports.src; 665 tuple->dst_port = fk->ports.dst; 666 667 rule->flow_id = flow_id; 668 rule->filter_id = arfs->last_filter_id++ % RPS_NO_FILTER; 669 670 hlist_add_head(&rule->hlist, 671 arfs_hash_bucket(arfs_t, tuple->src_port, 672 tuple->dst_port)); 673 return rule; 674 } 675 676 static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk) 677 { 678 if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst) 679 return false; 680 if (tuple->etype != fk->basic.n_proto) 681 return false; 682 if (tuple->etype == htons(ETH_P_IP)) 683 return tuple->src_ipv4 == fk->addrs.v4addrs.src && 684 tuple->dst_ipv4 == fk->addrs.v4addrs.dst; 685 if (tuple->etype == htons(ETH_P_IPV6)) 686 return !memcmp(&tuple->src_ipv6, &fk->addrs.v6addrs.src, 687 sizeof(struct in6_addr)) && 688 !memcmp(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, 689 sizeof(struct in6_addr)); 690 return false; 691 } 692 693 static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t, 694 const struct flow_keys *fk) 695 { 696 struct arfs_rule *arfs_rule; 697 struct hlist_head *head; 698 699 head = arfs_hash_bucket(arfs_t, fk->ports.src, fk->ports.dst); 700 hlist_for_each_entry(arfs_rule, head, hlist) { 701 if (arfs_cmp(&arfs_rule->tuple, fk)) 702 return arfs_rule; 703 } 704 705 return NULL; 706 } 707 708 int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 709 u16 rxq_index, u32 flow_id) 710 { 711 struct mlx5e_priv *priv = netdev_priv(dev); 712 struct mlx5e_arfs_tables *arfs; 713 struct arfs_rule *arfs_rule; 714 struct arfs_table *arfs_t; 715 struct flow_keys fk; 716 717 arfs = mlx5e_fs_get_arfs(priv->fs); 718 if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) 719 return -EPROTONOSUPPORT; 720 721 if (fk.basic.n_proto != htons(ETH_P_IP) && 722 fk.basic.n_proto != htons(ETH_P_IPV6)) 723 return -EPROTONOSUPPORT; 724 725 if (skb->encapsulation) 726 return -EPROTONOSUPPORT; 727 728 arfs_t = arfs_get_table(arfs, fk.basic.ip_proto, fk.basic.n_proto); 729 if (!arfs_t) 730 return -EPROTONOSUPPORT; 731 732 spin_lock_bh(&arfs->arfs_lock); 733 arfs_rule = arfs_find_rule(arfs_t, &fk); 734 if (arfs_rule) { 735 if (arfs_rule->rxq == rxq_index) { 736 spin_unlock_bh(&arfs->arfs_lock); 737 return arfs_rule->filter_id; 738 } 739 arfs_rule->rxq = rxq_index; 740 } else { 741 arfs_rule = arfs_alloc_rule(priv, arfs_t, &fk, rxq_index, flow_id); 742 if (!arfs_rule) { 743 spin_unlock_bh(&arfs->arfs_lock); 744 return -ENOMEM; 745 } 746 } 747 queue_work(arfs->wq, &arfs_rule->arfs_work); 748 spin_unlock_bh(&arfs->arfs_lock); 749 return arfs_rule->filter_id; 750 } 751 752