1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/mlx5/driver.h> 35 #include <linux/mlx5/mlx5_ifc.h> 36 #include <linux/mlx5/vport.h> 37 #include <linux/mlx5/fs.h> 38 #include "mlx5_core.h" 39 #include "eswitch.h" 40 41 enum { 42 FDB_FAST_PATH = 0, 43 FDB_SLOW_PATH 44 }; 45 46 struct mlx5_flow_handle * 47 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 48 struct mlx5_flow_spec *spec, 49 struct mlx5_esw_flow_attr *attr) 50 { 51 struct mlx5_flow_destination dest[2] = {}; 52 struct mlx5_flow_act flow_act = {0}; 53 struct mlx5_fc *counter = NULL; 54 struct mlx5_flow_handle *rule; 55 void *misc; 56 int i = 0; 57 58 if (esw->mode != SRIOV_OFFLOADS) 59 return ERR_PTR(-EOPNOTSUPP); 60 61 /* per flow vlan pop/push is emulated, don't set that into the firmware */ 62 flow_act.action = attr->action & ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 63 64 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 65 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 66 dest[i].vport_num = attr->out_rep->vport; 67 i++; 68 } 69 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 70 counter = mlx5_fc_create(esw->dev, true); 71 if (IS_ERR(counter)) { 72 rule = ERR_CAST(counter); 73 goto err_counter_alloc; 74 } 75 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 76 dest[i].counter = counter; 77 i++; 78 } 79 80 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 81 MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport); 82 83 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 84 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 85 86 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | 87 MLX5_MATCH_MISC_PARAMETERS; 88 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 89 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 90 91 if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 92 flow_act.modify_id = attr->mod_hdr_id; 93 94 if (attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP) 95 flow_act.encap_id = attr->encap_id; 96 97 rule = mlx5_add_flow_rules((struct mlx5_flow_table *)esw->fdb_table.fdb, 98 spec, &flow_act, dest, i); 99 if (IS_ERR(rule)) 100 goto err_add_rule; 101 else 102 esw->offloads.num_flows++; 103 104 return rule; 105 106 err_add_rule: 107 mlx5_fc_destroy(esw->dev, counter); 108 err_counter_alloc: 109 return rule; 110 } 111 112 void 113 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 114 struct mlx5_flow_handle *rule, 115 struct mlx5_esw_flow_attr *attr) 116 { 117 struct mlx5_fc *counter = NULL; 118 119 counter = mlx5_flow_rule_counter(rule); 120 mlx5_del_flow_rules(rule); 121 mlx5_fc_destroy(esw->dev, counter); 122 esw->offloads.num_flows--; 123 } 124 125 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) 126 { 127 struct mlx5_eswitch_rep *rep; 128 int vf_vport, err = 0; 129 130 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); 131 for (vf_vport = 1; vf_vport < esw->enabled_vports; vf_vport++) { 132 rep = &esw->offloads.vport_reps[vf_vport]; 133 if (!rep->valid) 134 continue; 135 136 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); 137 if (err) 138 goto out; 139 } 140 141 out: 142 return err; 143 } 144 145 static struct mlx5_eswitch_rep * 146 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) 147 { 148 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; 149 150 in_rep = attr->in_rep; 151 out_rep = attr->out_rep; 152 153 if (push) 154 vport = in_rep; 155 else if (pop) 156 vport = out_rep; 157 else 158 vport = in_rep; 159 160 return vport; 161 } 162 163 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, 164 bool push, bool pop, bool fwd) 165 { 166 struct mlx5_eswitch_rep *in_rep, *out_rep; 167 168 if ((push || pop) && !fwd) 169 goto out_notsupp; 170 171 in_rep = attr->in_rep; 172 out_rep = attr->out_rep; 173 174 if (push && in_rep->vport == FDB_UPLINK_VPORT) 175 goto out_notsupp; 176 177 if (pop && out_rep->vport == FDB_UPLINK_VPORT) 178 goto out_notsupp; 179 180 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ 181 if (!push && !pop && fwd) 182 if (in_rep->vlan && out_rep->vport == FDB_UPLINK_VPORT) 183 goto out_notsupp; 184 185 /* protects against (1) setting rules with different vlans to push and 186 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) 187 */ 188 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan)) 189 goto out_notsupp; 190 191 return 0; 192 193 out_notsupp: 194 return -EOPNOTSUPP; 195 } 196 197 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, 198 struct mlx5_esw_flow_attr *attr) 199 { 200 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 201 struct mlx5_eswitch_rep *vport = NULL; 202 bool push, pop, fwd; 203 int err = 0; 204 205 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 206 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 207 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 208 209 err = esw_add_vlan_action_check(attr, push, pop, fwd); 210 if (err) 211 return err; 212 213 attr->vlan_handled = false; 214 215 vport = esw_vlan_action_get_vport(attr, push, pop); 216 217 if (!push && !pop && fwd) { 218 /* tracks VF --> wire rules without vlan push action */ 219 if (attr->out_rep->vport == FDB_UPLINK_VPORT) { 220 vport->vlan_refcount++; 221 attr->vlan_handled = true; 222 } 223 224 return 0; 225 } 226 227 if (!push && !pop) 228 return 0; 229 230 if (!(offloads->vlan_push_pop_refcount)) { 231 /* it's the 1st vlan rule, apply global vlan pop policy */ 232 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); 233 if (err) 234 goto out; 235 } 236 offloads->vlan_push_pop_refcount++; 237 238 if (push) { 239 if (vport->vlan_refcount) 240 goto skip_set_push; 241 242 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, attr->vlan, 0, 243 SET_VLAN_INSERT | SET_VLAN_STRIP); 244 if (err) 245 goto out; 246 vport->vlan = attr->vlan; 247 skip_set_push: 248 vport->vlan_refcount++; 249 } 250 out: 251 if (!err) 252 attr->vlan_handled = true; 253 return err; 254 } 255 256 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, 257 struct mlx5_esw_flow_attr *attr) 258 { 259 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 260 struct mlx5_eswitch_rep *vport = NULL; 261 bool push, pop, fwd; 262 int err = 0; 263 264 if (!attr->vlan_handled) 265 return 0; 266 267 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 268 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 269 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 270 271 vport = esw_vlan_action_get_vport(attr, push, pop); 272 273 if (!push && !pop && fwd) { 274 /* tracks VF --> wire rules without vlan push action */ 275 if (attr->out_rep->vport == FDB_UPLINK_VPORT) 276 vport->vlan_refcount--; 277 278 return 0; 279 } 280 281 if (push) { 282 vport->vlan_refcount--; 283 if (vport->vlan_refcount) 284 goto skip_unset_push; 285 286 vport->vlan = 0; 287 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, 288 0, 0, SET_VLAN_STRIP); 289 if (err) 290 goto out; 291 } 292 293 skip_unset_push: 294 offloads->vlan_push_pop_refcount--; 295 if (offloads->vlan_push_pop_refcount) 296 return 0; 297 298 /* no more vlan rules, stop global vlan pop policy */ 299 err = esw_set_global_vlan_pop(esw, 0); 300 301 out: 302 return err; 303 } 304 305 static struct mlx5_flow_handle * 306 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn) 307 { 308 struct mlx5_flow_act flow_act = {0}; 309 struct mlx5_flow_destination dest; 310 struct mlx5_flow_handle *flow_rule; 311 struct mlx5_flow_spec *spec; 312 void *misc; 313 314 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 315 if (!spec) { 316 flow_rule = ERR_PTR(-ENOMEM); 317 goto out; 318 } 319 320 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 321 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 322 MLX5_SET(fte_match_set_misc, misc, source_port, 0x0); /* source vport is 0 */ 323 324 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 325 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 326 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 327 328 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 329 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 330 dest.vport_num = vport; 331 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 332 333 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.fdb, spec, 334 &flow_act, &dest, 1); 335 if (IS_ERR(flow_rule)) 336 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule)); 337 out: 338 kvfree(spec); 339 return flow_rule; 340 } 341 342 void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch *esw, 343 struct mlx5_eswitch_rep *rep) 344 { 345 struct mlx5_esw_sq *esw_sq, *tmp; 346 347 if (esw->mode != SRIOV_OFFLOADS) 348 return; 349 350 list_for_each_entry_safe(esw_sq, tmp, &rep->vport_sqs_list, list) { 351 mlx5_del_flow_rules(esw_sq->send_to_vport_rule); 352 list_del(&esw_sq->list); 353 kfree(esw_sq); 354 } 355 } 356 357 int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw, 358 struct mlx5_eswitch_rep *rep, 359 u16 *sqns_array, int sqns_num) 360 { 361 struct mlx5_flow_handle *flow_rule; 362 struct mlx5_esw_sq *esw_sq; 363 int err; 364 int i; 365 366 if (esw->mode != SRIOV_OFFLOADS) 367 return 0; 368 369 for (i = 0; i < sqns_num; i++) { 370 esw_sq = kzalloc(sizeof(*esw_sq), GFP_KERNEL); 371 if (!esw_sq) { 372 err = -ENOMEM; 373 goto out_err; 374 } 375 376 /* Add re-inject rule to the PF/representor sqs */ 377 flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw, 378 rep->vport, 379 sqns_array[i]); 380 if (IS_ERR(flow_rule)) { 381 err = PTR_ERR(flow_rule); 382 kfree(esw_sq); 383 goto out_err; 384 } 385 esw_sq->send_to_vport_rule = flow_rule; 386 list_add(&esw_sq->list, &rep->vport_sqs_list); 387 } 388 return 0; 389 390 out_err: 391 mlx5_eswitch_sqs2vport_stop(esw, rep); 392 return err; 393 } 394 395 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 396 { 397 struct mlx5_flow_act flow_act = {0}; 398 struct mlx5_flow_destination dest; 399 struct mlx5_flow_handle *flow_rule = NULL; 400 struct mlx5_flow_spec *spec; 401 int err = 0; 402 403 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 404 if (!spec) { 405 err = -ENOMEM; 406 goto out; 407 } 408 409 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 410 dest.vport_num = 0; 411 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 412 413 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.fdb, spec, 414 &flow_act, &dest, 1); 415 if (IS_ERR(flow_rule)) { 416 err = PTR_ERR(flow_rule); 417 esw_warn(esw->dev, "FDB: Failed to add miss flow rule err %d\n", err); 418 goto out; 419 } 420 421 esw->fdb_table.offloads.miss_rule = flow_rule; 422 out: 423 kvfree(spec); 424 return err; 425 } 426 427 #define ESW_OFFLOADS_NUM_GROUPS 4 428 429 static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch *esw) 430 { 431 struct mlx5_core_dev *dev = esw->dev; 432 struct mlx5_flow_namespace *root_ns; 433 struct mlx5_flow_table *fdb = NULL; 434 int esw_size, err = 0; 435 u32 flags = 0; 436 437 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 438 if (!root_ns) { 439 esw_warn(dev, "Failed to get FDB flow namespace\n"); 440 err = -EOPNOTSUPP; 441 goto out; 442 } 443 444 esw_debug(dev, "Create offloads FDB table, min (max esw size(2^%d), max counters(%d)*groups(%d))\n", 445 MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size), 446 MLX5_CAP_GEN(dev, max_flow_counter), ESW_OFFLOADS_NUM_GROUPS); 447 448 esw_size = min_t(int, MLX5_CAP_GEN(dev, max_flow_counter) * ESW_OFFLOADS_NUM_GROUPS, 449 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size)); 450 451 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 452 flags |= MLX5_FLOW_TABLE_TUNNEL_EN; 453 454 fdb = mlx5_create_auto_grouped_flow_table(root_ns, FDB_FAST_PATH, 455 esw_size, 456 ESW_OFFLOADS_NUM_GROUPS, 0, 457 flags); 458 if (IS_ERR(fdb)) { 459 err = PTR_ERR(fdb); 460 esw_warn(dev, "Failed to create Fast path FDB Table err %d\n", err); 461 goto out; 462 } 463 esw->fdb_table.fdb = fdb; 464 465 out: 466 return err; 467 } 468 469 static void esw_destroy_offloads_fast_fdb_table(struct mlx5_eswitch *esw) 470 { 471 mlx5_destroy_flow_table(esw->fdb_table.fdb); 472 } 473 474 #define MAX_PF_SQ 256 475 476 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) 477 { 478 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 479 struct mlx5_flow_table_attr ft_attr = {}; 480 struct mlx5_core_dev *dev = esw->dev; 481 struct mlx5_flow_namespace *root_ns; 482 struct mlx5_flow_table *fdb = NULL; 483 int table_size, ix, err = 0; 484 struct mlx5_flow_group *g; 485 void *match_criteria; 486 u32 *flow_group_in; 487 488 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 489 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 490 if (!flow_group_in) 491 return -ENOMEM; 492 493 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 494 if (!root_ns) { 495 esw_warn(dev, "Failed to get FDB flow namespace\n"); 496 err = -EOPNOTSUPP; 497 goto ns_err; 498 } 499 500 err = esw_create_offloads_fast_fdb_table(esw); 501 if (err) 502 goto fast_fdb_err; 503 504 table_size = nvports + MAX_PF_SQ + 1; 505 506 ft_attr.max_fte = table_size; 507 ft_attr.prio = FDB_SLOW_PATH; 508 509 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 510 if (IS_ERR(fdb)) { 511 err = PTR_ERR(fdb); 512 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 513 goto slow_fdb_err; 514 } 515 esw->fdb_table.offloads.fdb = fdb; 516 517 /* create send-to-vport group */ 518 memset(flow_group_in, 0, inlen); 519 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 520 MLX5_MATCH_MISC_PARAMETERS); 521 522 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 523 524 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 525 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 526 527 ix = nvports + MAX_PF_SQ; 528 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 529 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); 530 531 g = mlx5_create_flow_group(fdb, flow_group_in); 532 if (IS_ERR(g)) { 533 err = PTR_ERR(g); 534 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err); 535 goto send_vport_err; 536 } 537 esw->fdb_table.offloads.send_to_vport_grp = g; 538 539 /* create miss group */ 540 memset(flow_group_in, 0, inlen); 541 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 0); 542 543 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 544 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix + 1); 545 546 g = mlx5_create_flow_group(fdb, flow_group_in); 547 if (IS_ERR(g)) { 548 err = PTR_ERR(g); 549 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err); 550 goto miss_err; 551 } 552 esw->fdb_table.offloads.miss_grp = g; 553 554 err = esw_add_fdb_miss_rule(esw); 555 if (err) 556 goto miss_rule_err; 557 558 return 0; 559 560 miss_rule_err: 561 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 562 miss_err: 563 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 564 send_vport_err: 565 mlx5_destroy_flow_table(esw->fdb_table.offloads.fdb); 566 slow_fdb_err: 567 mlx5_destroy_flow_table(esw->fdb_table.fdb); 568 fast_fdb_err: 569 ns_err: 570 kvfree(flow_group_in); 571 return err; 572 } 573 574 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 575 { 576 if (!esw->fdb_table.fdb) 577 return; 578 579 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 580 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule); 581 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 582 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 583 584 mlx5_destroy_flow_table(esw->fdb_table.offloads.fdb); 585 esw_destroy_offloads_fast_fdb_table(esw); 586 } 587 588 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 589 { 590 struct mlx5_flow_table_attr ft_attr = {}; 591 struct mlx5_core_dev *dev = esw->dev; 592 struct mlx5_flow_table *ft_offloads; 593 struct mlx5_flow_namespace *ns; 594 int err = 0; 595 596 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 597 if (!ns) { 598 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 599 return -EOPNOTSUPP; 600 } 601 602 ft_attr.max_fte = dev->priv.sriov.num_vfs + 2; 603 604 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 605 if (IS_ERR(ft_offloads)) { 606 err = PTR_ERR(ft_offloads); 607 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 608 return err; 609 } 610 611 esw->offloads.ft_offloads = ft_offloads; 612 return 0; 613 } 614 615 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 616 { 617 struct mlx5_esw_offload *offloads = &esw->offloads; 618 619 mlx5_destroy_flow_table(offloads->ft_offloads); 620 } 621 622 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 623 { 624 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 625 struct mlx5_flow_group *g; 626 struct mlx5_priv *priv = &esw->dev->priv; 627 u32 *flow_group_in; 628 void *match_criteria, *misc; 629 int err = 0; 630 int nvports = priv->sriov.num_vfs + 2; 631 632 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 633 if (!flow_group_in) 634 return -ENOMEM; 635 636 /* create vport rx group */ 637 memset(flow_group_in, 0, inlen); 638 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 639 MLX5_MATCH_MISC_PARAMETERS); 640 641 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 642 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, misc_parameters); 643 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 644 645 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 646 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 647 648 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 649 650 if (IS_ERR(g)) { 651 err = PTR_ERR(g); 652 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 653 goto out; 654 } 655 656 esw->offloads.vport_rx_group = g; 657 out: 658 kfree(flow_group_in); 659 return err; 660 } 661 662 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 663 { 664 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 665 } 666 667 struct mlx5_flow_handle * 668 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, int vport, u32 tirn) 669 { 670 struct mlx5_flow_act flow_act = {0}; 671 struct mlx5_flow_destination dest; 672 struct mlx5_flow_handle *flow_rule; 673 struct mlx5_flow_spec *spec; 674 void *misc; 675 676 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 677 if (!spec) { 678 flow_rule = ERR_PTR(-ENOMEM); 679 goto out; 680 } 681 682 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 683 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 684 685 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 686 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 687 688 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 689 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 690 dest.tir_num = tirn; 691 692 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 693 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 694 &flow_act, &dest, 1); 695 if (IS_ERR(flow_rule)) { 696 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 697 goto out; 698 } 699 700 out: 701 kvfree(spec); 702 return flow_rule; 703 } 704 705 static int esw_offloads_start(struct mlx5_eswitch *esw) 706 { 707 int err, err1, num_vfs = esw->dev->priv.sriov.num_vfs; 708 709 if (esw->mode != SRIOV_LEGACY) { 710 esw_warn(esw->dev, "Can't set offloads mode, SRIOV legacy not enabled\n"); 711 return -EINVAL; 712 } 713 714 mlx5_eswitch_disable_sriov(esw); 715 err = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_OFFLOADS); 716 if (err) { 717 esw_warn(esw->dev, "Failed setting eswitch to offloads, err %d\n", err); 718 err1 = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_LEGACY); 719 if (err1) 720 esw_warn(esw->dev, "Failed setting eswitch back to legacy, err %d\n", err1); 721 } 722 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 723 if (mlx5_eswitch_inline_mode_get(esw, 724 num_vfs, 725 &esw->offloads.inline_mode)) { 726 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 727 esw_warn(esw->dev, "Inline mode is different between vports\n"); 728 } 729 } 730 return err; 731 } 732 733 int esw_offloads_init(struct mlx5_eswitch *esw, int nvports) 734 { 735 struct mlx5_eswitch_rep *rep; 736 int vport; 737 int err; 738 739 /* disable PF RoCE so missed packets don't go through RoCE steering */ 740 mlx5_dev_list_lock(); 741 mlx5_remove_dev_by_protocol(esw->dev, MLX5_INTERFACE_PROTOCOL_IB); 742 mlx5_dev_list_unlock(); 743 744 err = esw_create_offloads_fdb_tables(esw, nvports); 745 if (err) 746 goto create_fdb_err; 747 748 err = esw_create_offloads_table(esw); 749 if (err) 750 goto create_ft_err; 751 752 err = esw_create_vport_rx_group(esw); 753 if (err) 754 goto create_fg_err; 755 756 for (vport = 0; vport < nvports; vport++) { 757 rep = &esw->offloads.vport_reps[vport]; 758 if (!rep->valid) 759 continue; 760 761 err = rep->load(esw, rep); 762 if (err) 763 goto err_reps; 764 } 765 766 return 0; 767 768 err_reps: 769 for (vport--; vport >= 0; vport--) { 770 rep = &esw->offloads.vport_reps[vport]; 771 if (!rep->valid) 772 continue; 773 rep->unload(esw, rep); 774 } 775 esw_destroy_vport_rx_group(esw); 776 777 create_fg_err: 778 esw_destroy_offloads_table(esw); 779 780 create_ft_err: 781 esw_destroy_offloads_fdb_tables(esw); 782 783 create_fdb_err: 784 /* enable back PF RoCE */ 785 mlx5_dev_list_lock(); 786 mlx5_add_dev_by_protocol(esw->dev, MLX5_INTERFACE_PROTOCOL_IB); 787 mlx5_dev_list_unlock(); 788 789 return err; 790 } 791 792 static int esw_offloads_stop(struct mlx5_eswitch *esw) 793 { 794 int err, err1, num_vfs = esw->dev->priv.sriov.num_vfs; 795 796 mlx5_eswitch_disable_sriov(esw); 797 err = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_LEGACY); 798 if (err) { 799 esw_warn(esw->dev, "Failed setting eswitch to legacy, err %d\n", err); 800 err1 = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_OFFLOADS); 801 if (err1) 802 esw_warn(esw->dev, "Failed setting eswitch back to offloads, err %d\n", err); 803 } 804 805 /* enable back PF RoCE */ 806 mlx5_dev_list_lock(); 807 mlx5_add_dev_by_protocol(esw->dev, MLX5_INTERFACE_PROTOCOL_IB); 808 mlx5_dev_list_unlock(); 809 810 return err; 811 } 812 813 void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports) 814 { 815 struct mlx5_eswitch_rep *rep; 816 int vport; 817 818 for (vport = 0; vport < nvports; vport++) { 819 rep = &esw->offloads.vport_reps[vport]; 820 if (!rep->valid) 821 continue; 822 rep->unload(esw, rep); 823 } 824 825 esw_destroy_vport_rx_group(esw); 826 esw_destroy_offloads_table(esw); 827 esw_destroy_offloads_fdb_tables(esw); 828 } 829 830 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 831 { 832 switch (mode) { 833 case DEVLINK_ESWITCH_MODE_LEGACY: 834 *mlx5_mode = SRIOV_LEGACY; 835 break; 836 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 837 *mlx5_mode = SRIOV_OFFLOADS; 838 break; 839 default: 840 return -EINVAL; 841 } 842 843 return 0; 844 } 845 846 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 847 { 848 switch (mlx5_mode) { 849 case SRIOV_LEGACY: 850 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 851 break; 852 case SRIOV_OFFLOADS: 853 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 854 break; 855 default: 856 return -EINVAL; 857 } 858 859 return 0; 860 } 861 862 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 863 { 864 switch (mode) { 865 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 866 *mlx5_mode = MLX5_INLINE_MODE_NONE; 867 break; 868 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 869 *mlx5_mode = MLX5_INLINE_MODE_L2; 870 break; 871 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 872 *mlx5_mode = MLX5_INLINE_MODE_IP; 873 break; 874 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 875 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 876 break; 877 default: 878 return -EINVAL; 879 } 880 881 return 0; 882 } 883 884 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 885 { 886 switch (mlx5_mode) { 887 case MLX5_INLINE_MODE_NONE: 888 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 889 break; 890 case MLX5_INLINE_MODE_L2: 891 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 892 break; 893 case MLX5_INLINE_MODE_IP: 894 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 895 break; 896 case MLX5_INLINE_MODE_TCP_UDP: 897 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 898 break; 899 default: 900 return -EINVAL; 901 } 902 903 return 0; 904 } 905 906 static int mlx5_devlink_eswitch_check(struct devlink *devlink) 907 { 908 struct mlx5_core_dev *dev = devlink_priv(devlink); 909 910 if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH) 911 return -EOPNOTSUPP; 912 913 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 914 return -EOPNOTSUPP; 915 916 if (dev->priv.eswitch->mode == SRIOV_NONE) 917 return -EOPNOTSUPP; 918 919 return 0; 920 } 921 922 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode) 923 { 924 struct mlx5_core_dev *dev = devlink_priv(devlink); 925 u16 cur_mlx5_mode, mlx5_mode = 0; 926 int err; 927 928 err = mlx5_devlink_eswitch_check(devlink); 929 if (err) 930 return err; 931 932 cur_mlx5_mode = dev->priv.eswitch->mode; 933 934 if (esw_mode_from_devlink(mode, &mlx5_mode)) 935 return -EINVAL; 936 937 if (cur_mlx5_mode == mlx5_mode) 938 return 0; 939 940 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) 941 return esw_offloads_start(dev->priv.eswitch); 942 else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) 943 return esw_offloads_stop(dev->priv.eswitch); 944 else 945 return -EINVAL; 946 } 947 948 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 949 { 950 struct mlx5_core_dev *dev = devlink_priv(devlink); 951 int err; 952 953 err = mlx5_devlink_eswitch_check(devlink); 954 if (err) 955 return err; 956 957 return esw_mode_to_devlink(dev->priv.eswitch->mode, mode); 958 } 959 960 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode) 961 { 962 struct mlx5_core_dev *dev = devlink_priv(devlink); 963 struct mlx5_eswitch *esw = dev->priv.eswitch; 964 int err, vport; 965 u8 mlx5_mode; 966 967 err = mlx5_devlink_eswitch_check(devlink); 968 if (err) 969 return err; 970 971 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 972 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 973 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) 974 return 0; 975 /* fall through */ 976 case MLX5_CAP_INLINE_MODE_L2: 977 esw_warn(dev, "Inline mode can't be set\n"); 978 return -EOPNOTSUPP; 979 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 980 break; 981 } 982 983 if (esw->offloads.num_flows > 0) { 984 esw_warn(dev, "Can't set inline mode when flows are configured\n"); 985 return -EOPNOTSUPP; 986 } 987 988 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 989 if (err) 990 goto out; 991 992 for (vport = 1; vport < esw->enabled_vports; vport++) { 993 err = mlx5_modify_nic_vport_min_inline(dev, vport, mlx5_mode); 994 if (err) { 995 esw_warn(dev, "Failed to set min inline on vport %d\n", 996 vport); 997 goto revert_inline_mode; 998 } 999 } 1000 1001 esw->offloads.inline_mode = mlx5_mode; 1002 return 0; 1003 1004 revert_inline_mode: 1005 while (--vport > 0) 1006 mlx5_modify_nic_vport_min_inline(dev, 1007 vport, 1008 esw->offloads.inline_mode); 1009 out: 1010 return err; 1011 } 1012 1013 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 1014 { 1015 struct mlx5_core_dev *dev = devlink_priv(devlink); 1016 struct mlx5_eswitch *esw = dev->priv.eswitch; 1017 int err; 1018 1019 err = mlx5_devlink_eswitch_check(devlink); 1020 if (err) 1021 return err; 1022 1023 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 1024 } 1025 1026 int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, int nvfs, u8 *mode) 1027 { 1028 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 1029 struct mlx5_core_dev *dev = esw->dev; 1030 int vport; 1031 1032 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 1033 return -EOPNOTSUPP; 1034 1035 if (esw->mode == SRIOV_NONE) 1036 return -EOPNOTSUPP; 1037 1038 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 1039 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 1040 mlx5_mode = MLX5_INLINE_MODE_NONE; 1041 goto out; 1042 case MLX5_CAP_INLINE_MODE_L2: 1043 mlx5_mode = MLX5_INLINE_MODE_L2; 1044 goto out; 1045 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 1046 goto query_vports; 1047 } 1048 1049 query_vports: 1050 for (vport = 1; vport <= nvfs; vport++) { 1051 mlx5_query_nic_vport_min_inline(dev, vport, &mlx5_mode); 1052 if (vport > 1 && prev_mlx5_mode != mlx5_mode) 1053 return -EINVAL; 1054 prev_mlx5_mode = mlx5_mode; 1055 } 1056 1057 out: 1058 *mode = mlx5_mode; 1059 return 0; 1060 } 1061 1062 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, u8 encap) 1063 { 1064 struct mlx5_core_dev *dev = devlink_priv(devlink); 1065 struct mlx5_eswitch *esw = dev->priv.eswitch; 1066 int err; 1067 1068 err = mlx5_devlink_eswitch_check(devlink); 1069 if (err) 1070 return err; 1071 1072 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 1073 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, encap) || 1074 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) 1075 return -EOPNOTSUPP; 1076 1077 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) 1078 return -EOPNOTSUPP; 1079 1080 if (esw->mode == SRIOV_LEGACY) { 1081 esw->offloads.encap = encap; 1082 return 0; 1083 } 1084 1085 if (esw->offloads.encap == encap) 1086 return 0; 1087 1088 if (esw->offloads.num_flows > 0) { 1089 esw_warn(dev, "Can't set encapsulation when flows are configured\n"); 1090 return -EOPNOTSUPP; 1091 } 1092 1093 esw_destroy_offloads_fast_fdb_table(esw); 1094 1095 esw->offloads.encap = encap; 1096 err = esw_create_offloads_fast_fdb_table(esw); 1097 if (err) { 1098 esw_warn(esw->dev, "Failed re-creating fast FDB table, err %d\n", err); 1099 esw->offloads.encap = !encap; 1100 (void)esw_create_offloads_fast_fdb_table(esw); 1101 } 1102 return err; 1103 } 1104 1105 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, u8 *encap) 1106 { 1107 struct mlx5_core_dev *dev = devlink_priv(devlink); 1108 struct mlx5_eswitch *esw = dev->priv.eswitch; 1109 int err; 1110 1111 err = mlx5_devlink_eswitch_check(devlink); 1112 if (err) 1113 return err; 1114 1115 *encap = esw->offloads.encap; 1116 return 0; 1117 } 1118 1119 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch *esw, 1120 int vport_index, 1121 struct mlx5_eswitch_rep *__rep) 1122 { 1123 struct mlx5_esw_offload *offloads = &esw->offloads; 1124 struct mlx5_eswitch_rep *rep; 1125 1126 rep = &offloads->vport_reps[vport_index]; 1127 1128 memset(rep, 0, sizeof(*rep)); 1129 1130 rep->load = __rep->load; 1131 rep->unload = __rep->unload; 1132 rep->vport = __rep->vport; 1133 rep->netdev = __rep->netdev; 1134 ether_addr_copy(rep->hw_id, __rep->hw_id); 1135 1136 INIT_LIST_HEAD(&rep->vport_sqs_list); 1137 rep->valid = true; 1138 } 1139 1140 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw, 1141 int vport_index) 1142 { 1143 struct mlx5_esw_offload *offloads = &esw->offloads; 1144 struct mlx5_eswitch_rep *rep; 1145 1146 rep = &offloads->vport_reps[vport_index]; 1147 1148 if (esw->mode == SRIOV_OFFLOADS && esw->vports[vport_index].enabled) 1149 rep->unload(esw, rep); 1150 1151 rep->valid = false; 1152 } 1153 1154 struct net_device *mlx5_eswitch_get_uplink_netdev(struct mlx5_eswitch *esw) 1155 { 1156 #define UPLINK_REP_INDEX 0 1157 struct mlx5_esw_offload *offloads = &esw->offloads; 1158 struct mlx5_eswitch_rep *rep; 1159 1160 rep = &offloads->vport_reps[UPLINK_REP_INDEX]; 1161 return rep->netdev; 1162 } 1163